1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * xfrm_state.c 4 * 5 * Changes: 6 * Mitsuru KANDA @USAGI 7 * Kazunori MIYAZAWA @USAGI 8 * Kunihiro Ishiguro <kunihiro@ipinfusion.com> 9 * IPv6 support 10 * YOSHIFUJI Hideaki @USAGI 11 * Split up af-specific functions 12 * Derek Atkins <derek@ihtfp.com> 13 * Add UDP Encapsulation 14 * 15 */ 16 17 #include <linux/compat.h> 18 #include <linux/workqueue.h> 19 #include <net/xfrm.h> 20 #include <linux/pfkeyv2.h> 21 #include <linux/ipsec.h> 22 #include <linux/module.h> 23 #include <linux/cache.h> 24 #include <linux/audit.h> 25 #include <linux/uaccess.h> 26 #include <linux/ktime.h> 27 #include <linux/slab.h> 28 #include <linux/interrupt.h> 29 #include <linux/kernel.h> 30 31 #include <crypto/aead.h> 32 33 #include "xfrm_hash.h" 34 35 #define xfrm_state_deref_prot(table, net) \ 36 rcu_dereference_protected((table), lockdep_is_held(&(net)->xfrm.xfrm_state_lock)) 37 #define xfrm_state_deref_check(table, net) \ 38 rcu_dereference_check((table), lockdep_is_held(&(net)->xfrm.xfrm_state_lock)) 39 40 static void xfrm_state_gc_task(struct work_struct *work); 41 42 /* Each xfrm_state may be linked to two tables: 43 44 1. Hash table by (spi,daddr,ah/esp) to find SA by SPI. (input,ctl) 45 2. Hash table by (daddr,family,reqid) to find what SAs exist for given 46 destination/tunnel endpoint. (output) 47 */ 48 49 static unsigned int xfrm_state_hashmax __read_mostly = 1 * 1024 * 1024; 50 static struct kmem_cache *xfrm_state_cache __ro_after_init; 51 52 static DECLARE_WORK(xfrm_state_gc_work, xfrm_state_gc_task); 53 static HLIST_HEAD(xfrm_state_gc_list); 54 static HLIST_HEAD(xfrm_state_dev_gc_list); 55 56 static inline bool xfrm_state_hold_rcu(struct xfrm_state *x) 57 { 58 return refcount_inc_not_zero(&x->refcnt); 59 } 60 61 static inline unsigned int xfrm_dst_hash(struct net *net, 62 const xfrm_address_t *daddr, 63 const xfrm_address_t *saddr, 64 u32 reqid, 65 unsigned short family) 66 { 67 lockdep_assert_held(&net->xfrm.xfrm_state_lock); 68 69 return __xfrm_dst_hash(daddr, saddr, reqid, family, net->xfrm.state_hmask); 70 } 71 72 static inline unsigned int xfrm_src_hash(struct net *net, 73 const xfrm_address_t *daddr, 74 const xfrm_address_t *saddr, 75 unsigned short family) 76 { 77 lockdep_assert_held(&net->xfrm.xfrm_state_lock); 78 79 return __xfrm_src_hash(daddr, saddr, family, net->xfrm.state_hmask); 80 } 81 82 static inline unsigned int 83 xfrm_spi_hash(struct net *net, const xfrm_address_t *daddr, 84 __be32 spi, u8 proto, unsigned short family) 85 { 86 lockdep_assert_held(&net->xfrm.xfrm_state_lock); 87 88 return __xfrm_spi_hash(daddr, spi, proto, family, net->xfrm.state_hmask); 89 } 90 91 static unsigned int xfrm_seq_hash(struct net *net, u32 seq) 92 { 93 lockdep_assert_held(&net->xfrm.xfrm_state_lock); 94 95 return __xfrm_seq_hash(seq, net->xfrm.state_hmask); 96 } 97 98 #define XFRM_STATE_INSERT(by, _n, _h, _type) \ 99 { \ 100 struct xfrm_state *_x = NULL; \ 101 \ 102 if (_type != XFRM_DEV_OFFLOAD_PACKET) { \ 103 hlist_for_each_entry_rcu(_x, _h, by) { \ 104 if (_x->xso.type == XFRM_DEV_OFFLOAD_PACKET) \ 105 continue; \ 106 break; \ 107 } \ 108 } \ 109 \ 110 if (!_x || _x->xso.type == XFRM_DEV_OFFLOAD_PACKET) \ 111 /* SAD is empty or consist from HW SAs only */ \ 112 hlist_add_head_rcu(_n, _h); \ 113 else \ 114 hlist_add_before_rcu(_n, &_x->by); \ 115 } 116 117 static void xfrm_hash_transfer(struct hlist_head *list, 118 struct hlist_head *ndsttable, 119 struct hlist_head *nsrctable, 120 struct hlist_head *nspitable, 121 struct hlist_head *nseqtable, 122 unsigned int nhashmask) 123 { 124 struct hlist_node *tmp; 125 struct xfrm_state *x; 126 127 hlist_for_each_entry_safe(x, tmp, list, bydst) { 128 unsigned int h; 129 130 h = __xfrm_dst_hash(&x->id.daddr, &x->props.saddr, 131 x->props.reqid, x->props.family, 132 nhashmask); 133 XFRM_STATE_INSERT(bydst, &x->bydst, ndsttable + h, x->xso.type); 134 135 h = __xfrm_src_hash(&x->id.daddr, &x->props.saddr, 136 x->props.family, 137 nhashmask); 138 XFRM_STATE_INSERT(bysrc, &x->bysrc, nsrctable + h, x->xso.type); 139 140 if (x->id.spi) { 141 h = __xfrm_spi_hash(&x->id.daddr, x->id.spi, 142 x->id.proto, x->props.family, 143 nhashmask); 144 XFRM_STATE_INSERT(byspi, &x->byspi, nspitable + h, 145 x->xso.type); 146 } 147 148 if (x->km.seq) { 149 h = __xfrm_seq_hash(x->km.seq, nhashmask); 150 XFRM_STATE_INSERT(byseq, &x->byseq, nseqtable + h, 151 x->xso.type); 152 } 153 } 154 } 155 156 static unsigned long xfrm_hash_new_size(unsigned int state_hmask) 157 { 158 return ((state_hmask + 1) << 1) * sizeof(struct hlist_head); 159 } 160 161 static void xfrm_hash_resize(struct work_struct *work) 162 { 163 struct net *net = container_of(work, struct net, xfrm.state_hash_work); 164 struct hlist_head *ndst, *nsrc, *nspi, *nseq, *odst, *osrc, *ospi, *oseq; 165 unsigned long nsize, osize; 166 unsigned int nhashmask, ohashmask; 167 int i; 168 169 nsize = xfrm_hash_new_size(net->xfrm.state_hmask); 170 ndst = xfrm_hash_alloc(nsize); 171 if (!ndst) 172 return; 173 nsrc = xfrm_hash_alloc(nsize); 174 if (!nsrc) { 175 xfrm_hash_free(ndst, nsize); 176 return; 177 } 178 nspi = xfrm_hash_alloc(nsize); 179 if (!nspi) { 180 xfrm_hash_free(ndst, nsize); 181 xfrm_hash_free(nsrc, nsize); 182 return; 183 } 184 nseq = xfrm_hash_alloc(nsize); 185 if (!nseq) { 186 xfrm_hash_free(ndst, nsize); 187 xfrm_hash_free(nsrc, nsize); 188 xfrm_hash_free(nspi, nsize); 189 return; 190 } 191 192 spin_lock_bh(&net->xfrm.xfrm_state_lock); 193 write_seqcount_begin(&net->xfrm.xfrm_state_hash_generation); 194 195 nhashmask = (nsize / sizeof(struct hlist_head)) - 1U; 196 odst = xfrm_state_deref_prot(net->xfrm.state_bydst, net); 197 for (i = net->xfrm.state_hmask; i >= 0; i--) 198 xfrm_hash_transfer(odst + i, ndst, nsrc, nspi, nseq, nhashmask); 199 200 osrc = xfrm_state_deref_prot(net->xfrm.state_bysrc, net); 201 ospi = xfrm_state_deref_prot(net->xfrm.state_byspi, net); 202 oseq = xfrm_state_deref_prot(net->xfrm.state_byseq, net); 203 ohashmask = net->xfrm.state_hmask; 204 205 rcu_assign_pointer(net->xfrm.state_bydst, ndst); 206 rcu_assign_pointer(net->xfrm.state_bysrc, nsrc); 207 rcu_assign_pointer(net->xfrm.state_byspi, nspi); 208 rcu_assign_pointer(net->xfrm.state_byseq, nseq); 209 net->xfrm.state_hmask = nhashmask; 210 211 write_seqcount_end(&net->xfrm.xfrm_state_hash_generation); 212 spin_unlock_bh(&net->xfrm.xfrm_state_lock); 213 214 osize = (ohashmask + 1) * sizeof(struct hlist_head); 215 216 synchronize_rcu(); 217 218 xfrm_hash_free(odst, osize); 219 xfrm_hash_free(osrc, osize); 220 xfrm_hash_free(ospi, osize); 221 xfrm_hash_free(oseq, osize); 222 } 223 224 static DEFINE_SPINLOCK(xfrm_state_afinfo_lock); 225 static struct xfrm_state_afinfo __rcu *xfrm_state_afinfo[NPROTO]; 226 227 static DEFINE_SPINLOCK(xfrm_state_gc_lock); 228 static DEFINE_SPINLOCK(xfrm_state_dev_gc_lock); 229 static DEFINE_MUTEX(xfrm_state_gc_mutex); 230 231 int __xfrm_state_delete(struct xfrm_state *x); 232 233 int km_query(struct xfrm_state *x, struct xfrm_tmpl *t, struct xfrm_policy *pol); 234 static bool km_is_alive(const struct km_event *c); 235 void km_state_expired(struct xfrm_state *x, int hard, u32 portid); 236 237 int xfrm_register_type(const struct xfrm_type *type, unsigned short family) 238 { 239 struct xfrm_state_afinfo *afinfo = xfrm_state_get_afinfo(family); 240 int err = 0; 241 242 if (!afinfo) 243 return -EAFNOSUPPORT; 244 245 #define X(afi, T, name) do { \ 246 WARN_ON((afi)->type_ ## name); \ 247 (afi)->type_ ## name = (T); \ 248 } while (0) 249 250 switch (type->proto) { 251 case IPPROTO_COMP: 252 X(afinfo, type, comp); 253 break; 254 case IPPROTO_AH: 255 X(afinfo, type, ah); 256 break; 257 case IPPROTO_ESP: 258 X(afinfo, type, esp); 259 break; 260 case IPPROTO_IPIP: 261 X(afinfo, type, ipip); 262 break; 263 case IPPROTO_DSTOPTS: 264 X(afinfo, type, dstopts); 265 break; 266 case IPPROTO_ROUTING: 267 X(afinfo, type, routing); 268 break; 269 case IPPROTO_IPV6: 270 X(afinfo, type, ipip6); 271 break; 272 default: 273 WARN_ON(1); 274 err = -EPROTONOSUPPORT; 275 break; 276 } 277 #undef X 278 rcu_read_unlock(); 279 return err; 280 } 281 EXPORT_SYMBOL(xfrm_register_type); 282 283 void xfrm_unregister_type(const struct xfrm_type *type, unsigned short family) 284 { 285 struct xfrm_state_afinfo *afinfo = xfrm_state_get_afinfo(family); 286 287 if (unlikely(afinfo == NULL)) 288 return; 289 290 #define X(afi, T, name) do { \ 291 WARN_ON((afi)->type_ ## name != (T)); \ 292 (afi)->type_ ## name = NULL; \ 293 } while (0) 294 295 switch (type->proto) { 296 case IPPROTO_COMP: 297 X(afinfo, type, comp); 298 break; 299 case IPPROTO_AH: 300 X(afinfo, type, ah); 301 break; 302 case IPPROTO_ESP: 303 X(afinfo, type, esp); 304 break; 305 case IPPROTO_IPIP: 306 X(afinfo, type, ipip); 307 break; 308 case IPPROTO_DSTOPTS: 309 X(afinfo, type, dstopts); 310 break; 311 case IPPROTO_ROUTING: 312 X(afinfo, type, routing); 313 break; 314 case IPPROTO_IPV6: 315 X(afinfo, type, ipip6); 316 break; 317 default: 318 WARN_ON(1); 319 break; 320 } 321 #undef X 322 rcu_read_unlock(); 323 } 324 EXPORT_SYMBOL(xfrm_unregister_type); 325 326 static const struct xfrm_type *xfrm_get_type(u8 proto, unsigned short family) 327 { 328 const struct xfrm_type *type = NULL; 329 struct xfrm_state_afinfo *afinfo; 330 int modload_attempted = 0; 331 332 retry: 333 afinfo = xfrm_state_get_afinfo(family); 334 if (unlikely(afinfo == NULL)) 335 return NULL; 336 337 switch (proto) { 338 case IPPROTO_COMP: 339 type = afinfo->type_comp; 340 break; 341 case IPPROTO_AH: 342 type = afinfo->type_ah; 343 break; 344 case IPPROTO_ESP: 345 type = afinfo->type_esp; 346 break; 347 case IPPROTO_IPIP: 348 type = afinfo->type_ipip; 349 break; 350 case IPPROTO_DSTOPTS: 351 type = afinfo->type_dstopts; 352 break; 353 case IPPROTO_ROUTING: 354 type = afinfo->type_routing; 355 break; 356 case IPPROTO_IPV6: 357 type = afinfo->type_ipip6; 358 break; 359 default: 360 break; 361 } 362 363 if (unlikely(type && !try_module_get(type->owner))) 364 type = NULL; 365 366 rcu_read_unlock(); 367 368 if (!type && !modload_attempted) { 369 request_module("xfrm-type-%d-%d", family, proto); 370 modload_attempted = 1; 371 goto retry; 372 } 373 374 return type; 375 } 376 377 static void xfrm_put_type(const struct xfrm_type *type) 378 { 379 module_put(type->owner); 380 } 381 382 int xfrm_register_type_offload(const struct xfrm_type_offload *type, 383 unsigned short family) 384 { 385 struct xfrm_state_afinfo *afinfo = xfrm_state_get_afinfo(family); 386 int err = 0; 387 388 if (unlikely(afinfo == NULL)) 389 return -EAFNOSUPPORT; 390 391 switch (type->proto) { 392 case IPPROTO_ESP: 393 WARN_ON(afinfo->type_offload_esp); 394 afinfo->type_offload_esp = type; 395 break; 396 default: 397 WARN_ON(1); 398 err = -EPROTONOSUPPORT; 399 break; 400 } 401 402 rcu_read_unlock(); 403 return err; 404 } 405 EXPORT_SYMBOL(xfrm_register_type_offload); 406 407 void xfrm_unregister_type_offload(const struct xfrm_type_offload *type, 408 unsigned short family) 409 { 410 struct xfrm_state_afinfo *afinfo = xfrm_state_get_afinfo(family); 411 412 if (unlikely(afinfo == NULL)) 413 return; 414 415 switch (type->proto) { 416 case IPPROTO_ESP: 417 WARN_ON(afinfo->type_offload_esp != type); 418 afinfo->type_offload_esp = NULL; 419 break; 420 default: 421 WARN_ON(1); 422 break; 423 } 424 rcu_read_unlock(); 425 } 426 EXPORT_SYMBOL(xfrm_unregister_type_offload); 427 428 void xfrm_set_type_offload(struct xfrm_state *x, bool try_load) 429 { 430 const struct xfrm_type_offload *type = NULL; 431 struct xfrm_state_afinfo *afinfo; 432 433 retry: 434 afinfo = xfrm_state_get_afinfo(x->props.family); 435 if (unlikely(afinfo == NULL)) 436 goto out; 437 438 switch (x->id.proto) { 439 case IPPROTO_ESP: 440 type = afinfo->type_offload_esp; 441 break; 442 default: 443 break; 444 } 445 446 if ((type && !try_module_get(type->owner))) 447 type = NULL; 448 449 rcu_read_unlock(); 450 451 if (!type && try_load) { 452 request_module("xfrm-offload-%d-%d", x->props.family, 453 x->id.proto); 454 try_load = false; 455 goto retry; 456 } 457 458 out: 459 x->type_offload = type; 460 } 461 EXPORT_SYMBOL(xfrm_set_type_offload); 462 463 static const struct xfrm_mode xfrm4_mode_map[XFRM_MODE_MAX] = { 464 [XFRM_MODE_BEET] = { 465 .encap = XFRM_MODE_BEET, 466 .flags = XFRM_MODE_FLAG_TUNNEL, 467 .family = AF_INET, 468 }, 469 [XFRM_MODE_TRANSPORT] = { 470 .encap = XFRM_MODE_TRANSPORT, 471 .family = AF_INET, 472 }, 473 [XFRM_MODE_TUNNEL] = { 474 .encap = XFRM_MODE_TUNNEL, 475 .flags = XFRM_MODE_FLAG_TUNNEL, 476 .family = AF_INET, 477 }, 478 [XFRM_MODE_IPTFS] = { 479 .encap = XFRM_MODE_IPTFS, 480 .flags = XFRM_MODE_FLAG_TUNNEL, 481 .family = AF_INET, 482 }, 483 }; 484 485 static const struct xfrm_mode xfrm6_mode_map[XFRM_MODE_MAX] = { 486 [XFRM_MODE_BEET] = { 487 .encap = XFRM_MODE_BEET, 488 .flags = XFRM_MODE_FLAG_TUNNEL, 489 .family = AF_INET6, 490 }, 491 [XFRM_MODE_ROUTEOPTIMIZATION] = { 492 .encap = XFRM_MODE_ROUTEOPTIMIZATION, 493 .family = AF_INET6, 494 }, 495 [XFRM_MODE_TRANSPORT] = { 496 .encap = XFRM_MODE_TRANSPORT, 497 .family = AF_INET6, 498 }, 499 [XFRM_MODE_TUNNEL] = { 500 .encap = XFRM_MODE_TUNNEL, 501 .flags = XFRM_MODE_FLAG_TUNNEL, 502 .family = AF_INET6, 503 }, 504 [XFRM_MODE_IPTFS] = { 505 .encap = XFRM_MODE_IPTFS, 506 .flags = XFRM_MODE_FLAG_TUNNEL, 507 .family = AF_INET6, 508 }, 509 }; 510 511 static const struct xfrm_mode *xfrm_get_mode(unsigned int encap, int family) 512 { 513 const struct xfrm_mode *mode; 514 515 if (unlikely(encap >= XFRM_MODE_MAX)) 516 return NULL; 517 518 switch (family) { 519 case AF_INET: 520 mode = &xfrm4_mode_map[encap]; 521 if (mode->family == family) 522 return mode; 523 break; 524 case AF_INET6: 525 mode = &xfrm6_mode_map[encap]; 526 if (mode->family == family) 527 return mode; 528 break; 529 default: 530 break; 531 } 532 533 return NULL; 534 } 535 536 static const struct xfrm_mode_cbs __rcu *xfrm_mode_cbs_map[XFRM_MODE_MAX]; 537 static DEFINE_SPINLOCK(xfrm_mode_cbs_map_lock); 538 539 int xfrm_register_mode_cbs(u8 mode, const struct xfrm_mode_cbs *mode_cbs) 540 { 541 if (mode >= XFRM_MODE_MAX) 542 return -EINVAL; 543 544 spin_lock_bh(&xfrm_mode_cbs_map_lock); 545 rcu_assign_pointer(xfrm_mode_cbs_map[mode], mode_cbs); 546 spin_unlock_bh(&xfrm_mode_cbs_map_lock); 547 548 return 0; 549 } 550 EXPORT_SYMBOL(xfrm_register_mode_cbs); 551 552 void xfrm_unregister_mode_cbs(u8 mode) 553 { 554 if (mode >= XFRM_MODE_MAX) 555 return; 556 557 spin_lock_bh(&xfrm_mode_cbs_map_lock); 558 RCU_INIT_POINTER(xfrm_mode_cbs_map[mode], NULL); 559 spin_unlock_bh(&xfrm_mode_cbs_map_lock); 560 synchronize_rcu(); 561 } 562 EXPORT_SYMBOL(xfrm_unregister_mode_cbs); 563 564 static const struct xfrm_mode_cbs *xfrm_get_mode_cbs(u8 mode) 565 { 566 const struct xfrm_mode_cbs *cbs; 567 bool try_load = true; 568 569 if (mode >= XFRM_MODE_MAX) 570 return NULL; 571 572 retry: 573 rcu_read_lock(); 574 575 cbs = rcu_dereference(xfrm_mode_cbs_map[mode]); 576 if (cbs && !try_module_get(cbs->owner)) 577 cbs = NULL; 578 579 rcu_read_unlock(); 580 581 if (mode == XFRM_MODE_IPTFS && !cbs && try_load) { 582 request_module("xfrm-iptfs"); 583 try_load = false; 584 goto retry; 585 } 586 587 return cbs; 588 } 589 590 void xfrm_state_free(struct xfrm_state *x) 591 { 592 kmem_cache_free(xfrm_state_cache, x); 593 } 594 EXPORT_SYMBOL(xfrm_state_free); 595 596 static void xfrm_state_delete_tunnel(struct xfrm_state *x); 597 static void xfrm_state_gc_destroy(struct xfrm_state *x) 598 { 599 if (x->mode_cbs && x->mode_cbs->destroy_state) 600 x->mode_cbs->destroy_state(x); 601 hrtimer_cancel(&x->mtimer); 602 timer_delete_sync(&x->rtimer); 603 kfree_sensitive(x->aead); 604 kfree_sensitive(x->aalg); 605 kfree_sensitive(x->ealg); 606 kfree(x->calg); 607 kfree(x->encap); 608 kfree(x->coaddr); 609 kfree(x->replay_esn); 610 kfree(x->preplay_esn); 611 xfrm_unset_type_offload(x); 612 xfrm_state_delete_tunnel(x); 613 if (x->type) { 614 x->type->destructor(x); 615 xfrm_put_type(x->type); 616 } 617 if (x->xfrag.page) 618 put_page(x->xfrag.page); 619 xfrm_dev_state_free(x); 620 security_xfrm_state_free(x); 621 xfrm_state_free(x); 622 } 623 624 static void xfrm_state_gc_task(struct work_struct *work) 625 { 626 struct xfrm_state *x; 627 struct hlist_node *tmp; 628 struct hlist_head gc_list; 629 630 spin_lock_bh(&xfrm_state_gc_lock); 631 hlist_move_list(&xfrm_state_gc_list, &gc_list); 632 spin_unlock_bh(&xfrm_state_gc_lock); 633 634 synchronize_rcu(); 635 636 mutex_lock(&xfrm_state_gc_mutex); 637 hlist_for_each_entry_safe(x, tmp, &gc_list, gclist) 638 xfrm_state_gc_destroy(x); 639 mutex_unlock(&xfrm_state_gc_mutex); 640 } 641 642 static enum hrtimer_restart xfrm_timer_handler(struct hrtimer *me) 643 { 644 struct xfrm_state *x = container_of(me, struct xfrm_state, mtimer); 645 enum hrtimer_restart ret = HRTIMER_NORESTART; 646 time64_t now = ktime_get_real_seconds(); 647 time64_t next = TIME64_MAX; 648 int warn = 0; 649 int err = 0; 650 651 spin_lock(&x->lock); 652 xfrm_dev_state_update_stats(x); 653 654 if (x->km.state == XFRM_STATE_DEAD) 655 goto out; 656 if (x->km.state == XFRM_STATE_EXPIRED) 657 goto expired; 658 if (x->lft.hard_add_expires_seconds) { 659 time64_t tmo = x->lft.hard_add_expires_seconds + 660 x->curlft.add_time - now; 661 if (tmo <= 0) { 662 if (x->xflags & XFRM_SOFT_EXPIRE) { 663 /* enter hard expire without soft expire first?! 664 * setting a new date could trigger this. 665 * workaround: fix x->curflt.add_time by below: 666 */ 667 x->curlft.add_time = now - x->saved_tmo - 1; 668 tmo = x->lft.hard_add_expires_seconds - x->saved_tmo; 669 } else 670 goto expired; 671 } 672 if (tmo < next) 673 next = tmo; 674 } 675 if (x->lft.hard_use_expires_seconds) { 676 time64_t tmo = x->lft.hard_use_expires_seconds + 677 (READ_ONCE(x->curlft.use_time) ? : now) - now; 678 if (tmo <= 0) 679 goto expired; 680 if (tmo < next) 681 next = tmo; 682 } 683 if (x->km.dying) 684 goto resched; 685 if (x->lft.soft_add_expires_seconds) { 686 time64_t tmo = x->lft.soft_add_expires_seconds + 687 x->curlft.add_time - now; 688 if (tmo <= 0) { 689 warn = 1; 690 x->xflags &= ~XFRM_SOFT_EXPIRE; 691 } else if (tmo < next) { 692 next = tmo; 693 x->xflags |= XFRM_SOFT_EXPIRE; 694 x->saved_tmo = tmo; 695 } 696 } 697 if (x->lft.soft_use_expires_seconds) { 698 time64_t tmo = x->lft.soft_use_expires_seconds + 699 (READ_ONCE(x->curlft.use_time) ? : now) - now; 700 if (tmo <= 0) 701 warn = 1; 702 else if (tmo < next) 703 next = tmo; 704 } 705 706 x->km.dying = warn; 707 if (warn) 708 km_state_expired(x, 0, 0); 709 resched: 710 if (next != TIME64_MAX) { 711 hrtimer_forward_now(&x->mtimer, ktime_set(next, 0)); 712 ret = HRTIMER_RESTART; 713 } 714 715 goto out; 716 717 expired: 718 if (x->km.state == XFRM_STATE_ACQ && x->id.spi == 0) 719 x->km.state = XFRM_STATE_EXPIRED; 720 721 err = __xfrm_state_delete(x); 722 if (!err) 723 km_state_expired(x, 1, 0); 724 725 xfrm_audit_state_delete(x, err ? 0 : 1, true); 726 727 out: 728 spin_unlock(&x->lock); 729 return ret; 730 } 731 732 static void xfrm_replay_timer_handler(struct timer_list *t); 733 734 struct xfrm_state *xfrm_state_alloc(struct net *net) 735 { 736 struct xfrm_state *x; 737 738 x = kmem_cache_zalloc(xfrm_state_cache, GFP_ATOMIC); 739 740 if (x) { 741 write_pnet(&x->xs_net, net); 742 refcount_set(&x->refcnt, 1); 743 atomic_set(&x->tunnel_users, 0); 744 INIT_LIST_HEAD(&x->km.all); 745 INIT_HLIST_NODE(&x->state_cache); 746 INIT_HLIST_NODE(&x->bydst); 747 INIT_HLIST_NODE(&x->bysrc); 748 INIT_HLIST_NODE(&x->byspi); 749 INIT_HLIST_NODE(&x->byseq); 750 hrtimer_setup(&x->mtimer, xfrm_timer_handler, CLOCK_BOOTTIME, 751 HRTIMER_MODE_ABS_SOFT); 752 timer_setup(&x->rtimer, xfrm_replay_timer_handler, 0); 753 x->curlft.add_time = ktime_get_real_seconds(); 754 x->lft.soft_byte_limit = XFRM_INF; 755 x->lft.soft_packet_limit = XFRM_INF; 756 x->lft.hard_byte_limit = XFRM_INF; 757 x->lft.hard_packet_limit = XFRM_INF; 758 x->replay_maxage = 0; 759 x->replay_maxdiff = 0; 760 x->pcpu_num = UINT_MAX; 761 spin_lock_init(&x->lock); 762 x->mode_data = NULL; 763 } 764 return x; 765 } 766 EXPORT_SYMBOL(xfrm_state_alloc); 767 768 #ifdef CONFIG_XFRM_OFFLOAD 769 void xfrm_dev_state_delete(struct xfrm_state *x) 770 { 771 struct xfrm_dev_offload *xso = &x->xso; 772 struct net_device *dev = READ_ONCE(xso->dev); 773 774 if (dev) { 775 dev->xfrmdev_ops->xdo_dev_state_delete(dev, x); 776 spin_lock_bh(&xfrm_state_dev_gc_lock); 777 hlist_add_head(&x->dev_gclist, &xfrm_state_dev_gc_list); 778 spin_unlock_bh(&xfrm_state_dev_gc_lock); 779 } 780 } 781 EXPORT_SYMBOL_GPL(xfrm_dev_state_delete); 782 783 void xfrm_dev_state_free(struct xfrm_state *x) 784 { 785 struct xfrm_dev_offload *xso = &x->xso; 786 struct net_device *dev = READ_ONCE(xso->dev); 787 788 if (dev && dev->xfrmdev_ops) { 789 spin_lock_bh(&xfrm_state_dev_gc_lock); 790 if (!hlist_unhashed(&x->dev_gclist)) 791 hlist_del(&x->dev_gclist); 792 spin_unlock_bh(&xfrm_state_dev_gc_lock); 793 794 if (dev->xfrmdev_ops->xdo_dev_state_free) 795 dev->xfrmdev_ops->xdo_dev_state_free(dev, x); 796 WRITE_ONCE(xso->dev, NULL); 797 xso->type = XFRM_DEV_OFFLOAD_UNSPECIFIED; 798 netdev_put(dev, &xso->dev_tracker); 799 } 800 } 801 #endif 802 803 void __xfrm_state_destroy(struct xfrm_state *x) 804 { 805 WARN_ON(x->km.state != XFRM_STATE_DEAD); 806 807 spin_lock_bh(&xfrm_state_gc_lock); 808 hlist_add_head(&x->gclist, &xfrm_state_gc_list); 809 spin_unlock_bh(&xfrm_state_gc_lock); 810 schedule_work(&xfrm_state_gc_work); 811 } 812 EXPORT_SYMBOL(__xfrm_state_destroy); 813 814 int __xfrm_state_delete(struct xfrm_state *x) 815 { 816 struct net *net = xs_net(x); 817 int err = -ESRCH; 818 819 if (x->km.state != XFRM_STATE_DEAD) { 820 x->km.state = XFRM_STATE_DEAD; 821 822 spin_lock(&net->xfrm.xfrm_state_lock); 823 list_del(&x->km.all); 824 hlist_del_init_rcu(&x->bydst); 825 hlist_del_init_rcu(&x->bysrc); 826 if (!hlist_unhashed(&x->byseq)) 827 hlist_del_init_rcu(&x->byseq); 828 if (!hlist_unhashed(&x->state_cache)) 829 hlist_del_init_rcu(&x->state_cache); 830 if (!hlist_unhashed(&x->state_cache_input)) 831 hlist_del_init_rcu(&x->state_cache_input); 832 833 if (!hlist_unhashed(&x->byspi)) 834 hlist_del_init_rcu(&x->byspi); 835 net->xfrm.state_num--; 836 xfrm_nat_keepalive_state_updated(x); 837 spin_unlock(&net->xfrm.xfrm_state_lock); 838 839 xfrm_dev_state_delete(x); 840 841 xfrm_state_delete_tunnel(x); 842 843 /* All xfrm_state objects are created by xfrm_state_alloc. 844 * The xfrm_state_alloc call gives a reference, and that 845 * is what we are dropping here. 846 */ 847 xfrm_state_put(x); 848 err = 0; 849 } 850 851 return err; 852 } 853 EXPORT_SYMBOL(__xfrm_state_delete); 854 855 int xfrm_state_delete(struct xfrm_state *x) 856 { 857 int err; 858 859 spin_lock_bh(&x->lock); 860 err = __xfrm_state_delete(x); 861 spin_unlock_bh(&x->lock); 862 863 return err; 864 } 865 EXPORT_SYMBOL(xfrm_state_delete); 866 867 #ifdef CONFIG_SECURITY_NETWORK_XFRM 868 static inline int 869 xfrm_state_flush_secctx_check(struct net *net, u8 proto, bool task_valid) 870 { 871 int i, err = 0; 872 873 for (i = 0; i <= net->xfrm.state_hmask; i++) { 874 struct xfrm_state *x; 875 876 hlist_for_each_entry(x, xfrm_state_deref_prot(net->xfrm.state_bydst, net) + i, bydst) { 877 if (xfrm_id_proto_match(x->id.proto, proto) && 878 (err = security_xfrm_state_delete(x)) != 0) { 879 xfrm_audit_state_delete(x, 0, task_valid); 880 return err; 881 } 882 } 883 } 884 885 return err; 886 } 887 888 static inline int 889 xfrm_dev_state_flush_secctx_check(struct net *net, struct net_device *dev, bool task_valid) 890 { 891 int i, err = 0; 892 893 for (i = 0; i <= net->xfrm.state_hmask; i++) { 894 struct xfrm_state *x; 895 struct xfrm_dev_offload *xso; 896 897 hlist_for_each_entry(x, xfrm_state_deref_prot(net->xfrm.state_bydst, net) + i, bydst) { 898 xso = &x->xso; 899 900 if (xso->dev == dev && 901 (err = security_xfrm_state_delete(x)) != 0) { 902 xfrm_audit_state_delete(x, 0, task_valid); 903 return err; 904 } 905 } 906 } 907 908 return err; 909 } 910 #else 911 static inline int 912 xfrm_state_flush_secctx_check(struct net *net, u8 proto, bool task_valid) 913 { 914 return 0; 915 } 916 917 static inline int 918 xfrm_dev_state_flush_secctx_check(struct net *net, struct net_device *dev, bool task_valid) 919 { 920 return 0; 921 } 922 #endif 923 924 int xfrm_state_flush(struct net *net, u8 proto, bool task_valid) 925 { 926 int i, err = 0, cnt = 0; 927 928 spin_lock_bh(&net->xfrm.xfrm_state_lock); 929 err = xfrm_state_flush_secctx_check(net, proto, task_valid); 930 if (err) 931 goto out; 932 933 err = -ESRCH; 934 for (i = 0; i <= net->xfrm.state_hmask; i++) { 935 struct xfrm_state *x; 936 restart: 937 hlist_for_each_entry(x, xfrm_state_deref_prot(net->xfrm.state_bydst, net) + i, bydst) { 938 if (!xfrm_state_kern(x) && 939 xfrm_id_proto_match(x->id.proto, proto)) { 940 xfrm_state_hold(x); 941 spin_unlock_bh(&net->xfrm.xfrm_state_lock); 942 943 err = xfrm_state_delete(x); 944 xfrm_audit_state_delete(x, err ? 0 : 1, 945 task_valid); 946 xfrm_state_put(x); 947 if (!err) 948 cnt++; 949 950 spin_lock_bh(&net->xfrm.xfrm_state_lock); 951 goto restart; 952 } 953 } 954 } 955 out: 956 spin_unlock_bh(&net->xfrm.xfrm_state_lock); 957 if (cnt) 958 err = 0; 959 960 return err; 961 } 962 EXPORT_SYMBOL(xfrm_state_flush); 963 964 int xfrm_dev_state_flush(struct net *net, struct net_device *dev, bool task_valid) 965 { 966 struct xfrm_state *x; 967 struct hlist_node *tmp; 968 struct xfrm_dev_offload *xso; 969 int i, err = 0, cnt = 0; 970 971 spin_lock_bh(&net->xfrm.xfrm_state_lock); 972 err = xfrm_dev_state_flush_secctx_check(net, dev, task_valid); 973 if (err) 974 goto out; 975 976 err = -ESRCH; 977 for (i = 0; i <= net->xfrm.state_hmask; i++) { 978 restart: 979 hlist_for_each_entry(x, xfrm_state_deref_prot(net->xfrm.state_bydst, net) + i, bydst) { 980 xso = &x->xso; 981 982 if (!xfrm_state_kern(x) && xso->dev == dev) { 983 xfrm_state_hold(x); 984 spin_unlock_bh(&net->xfrm.xfrm_state_lock); 985 986 err = xfrm_state_delete(x); 987 xfrm_dev_state_free(x); 988 989 xfrm_audit_state_delete(x, err ? 0 : 1, 990 task_valid); 991 xfrm_state_put(x); 992 if (!err) 993 cnt++; 994 995 spin_lock_bh(&net->xfrm.xfrm_state_lock); 996 goto restart; 997 } 998 } 999 } 1000 if (cnt) 1001 err = 0; 1002 1003 out: 1004 spin_unlock_bh(&net->xfrm.xfrm_state_lock); 1005 1006 mutex_lock(&xfrm_state_gc_mutex); 1007 spin_lock_bh(&xfrm_state_dev_gc_lock); 1008 restart_gc: 1009 hlist_for_each_entry_safe(x, tmp, &xfrm_state_dev_gc_list, dev_gclist) { 1010 xso = &x->xso; 1011 1012 if (xso->dev == dev) { 1013 spin_unlock_bh(&xfrm_state_dev_gc_lock); 1014 xfrm_dev_state_free(x); 1015 spin_lock_bh(&xfrm_state_dev_gc_lock); 1016 goto restart_gc; 1017 } 1018 1019 } 1020 spin_unlock_bh(&xfrm_state_dev_gc_lock); 1021 mutex_unlock(&xfrm_state_gc_mutex); 1022 1023 xfrm_flush_gc(); 1024 1025 return err; 1026 } 1027 EXPORT_SYMBOL(xfrm_dev_state_flush); 1028 1029 void xfrm_sad_getinfo(struct net *net, struct xfrmk_sadinfo *si) 1030 { 1031 spin_lock_bh(&net->xfrm.xfrm_state_lock); 1032 si->sadcnt = net->xfrm.state_num; 1033 si->sadhcnt = net->xfrm.state_hmask + 1; 1034 si->sadhmcnt = xfrm_state_hashmax; 1035 spin_unlock_bh(&net->xfrm.xfrm_state_lock); 1036 } 1037 EXPORT_SYMBOL(xfrm_sad_getinfo); 1038 1039 static void 1040 __xfrm4_init_tempsel(struct xfrm_selector *sel, const struct flowi *fl) 1041 { 1042 const struct flowi4 *fl4 = &fl->u.ip4; 1043 1044 sel->daddr.a4 = fl4->daddr; 1045 sel->saddr.a4 = fl4->saddr; 1046 sel->dport = xfrm_flowi_dport(fl, &fl4->uli); 1047 sel->dport_mask = htons(0xffff); 1048 sel->sport = xfrm_flowi_sport(fl, &fl4->uli); 1049 sel->sport_mask = htons(0xffff); 1050 sel->family = AF_INET; 1051 sel->prefixlen_d = 32; 1052 sel->prefixlen_s = 32; 1053 sel->proto = fl4->flowi4_proto; 1054 sel->ifindex = fl4->flowi4_oif; 1055 } 1056 1057 static void 1058 __xfrm6_init_tempsel(struct xfrm_selector *sel, const struct flowi *fl) 1059 { 1060 const struct flowi6 *fl6 = &fl->u.ip6; 1061 1062 /* Initialize temporary selector matching only to current session. */ 1063 *(struct in6_addr *)&sel->daddr = fl6->daddr; 1064 *(struct in6_addr *)&sel->saddr = fl6->saddr; 1065 sel->dport = xfrm_flowi_dport(fl, &fl6->uli); 1066 sel->dport_mask = htons(0xffff); 1067 sel->sport = xfrm_flowi_sport(fl, &fl6->uli); 1068 sel->sport_mask = htons(0xffff); 1069 sel->family = AF_INET6; 1070 sel->prefixlen_d = 128; 1071 sel->prefixlen_s = 128; 1072 sel->proto = fl6->flowi6_proto; 1073 sel->ifindex = fl6->flowi6_oif; 1074 } 1075 1076 static void 1077 xfrm_init_tempstate(struct xfrm_state *x, const struct flowi *fl, 1078 const struct xfrm_tmpl *tmpl, 1079 const xfrm_address_t *daddr, const xfrm_address_t *saddr, 1080 unsigned short family) 1081 { 1082 switch (family) { 1083 case AF_INET: 1084 __xfrm4_init_tempsel(&x->sel, fl); 1085 break; 1086 case AF_INET6: 1087 __xfrm6_init_tempsel(&x->sel, fl); 1088 break; 1089 } 1090 1091 x->id = tmpl->id; 1092 1093 switch (tmpl->encap_family) { 1094 case AF_INET: 1095 if (x->id.daddr.a4 == 0) 1096 x->id.daddr.a4 = daddr->a4; 1097 x->props.saddr = tmpl->saddr; 1098 if (x->props.saddr.a4 == 0) 1099 x->props.saddr.a4 = saddr->a4; 1100 break; 1101 case AF_INET6: 1102 if (ipv6_addr_any((struct in6_addr *)&x->id.daddr)) 1103 memcpy(&x->id.daddr, daddr, sizeof(x->sel.daddr)); 1104 memcpy(&x->props.saddr, &tmpl->saddr, sizeof(x->props.saddr)); 1105 if (ipv6_addr_any((struct in6_addr *)&x->props.saddr)) 1106 memcpy(&x->props.saddr, saddr, sizeof(x->props.saddr)); 1107 break; 1108 } 1109 1110 x->props.mode = tmpl->mode; 1111 x->props.reqid = tmpl->reqid; 1112 x->props.family = tmpl->encap_family; 1113 } 1114 1115 struct xfrm_hash_state_ptrs { 1116 const struct hlist_head *bydst; 1117 const struct hlist_head *bysrc; 1118 const struct hlist_head *byspi; 1119 unsigned int hmask; 1120 }; 1121 1122 static void xfrm_hash_ptrs_get(const struct net *net, struct xfrm_hash_state_ptrs *ptrs) 1123 { 1124 unsigned int sequence; 1125 1126 do { 1127 sequence = read_seqcount_begin(&net->xfrm.xfrm_state_hash_generation); 1128 1129 ptrs->bydst = xfrm_state_deref_check(net->xfrm.state_bydst, net); 1130 ptrs->bysrc = xfrm_state_deref_check(net->xfrm.state_bysrc, net); 1131 ptrs->byspi = xfrm_state_deref_check(net->xfrm.state_byspi, net); 1132 ptrs->hmask = net->xfrm.state_hmask; 1133 } while (read_seqcount_retry(&net->xfrm.xfrm_state_hash_generation, sequence)); 1134 } 1135 1136 static struct xfrm_state *__xfrm_state_lookup_all(const struct xfrm_hash_state_ptrs *state_ptrs, 1137 u32 mark, 1138 const xfrm_address_t *daddr, 1139 __be32 spi, u8 proto, 1140 unsigned short family, 1141 struct xfrm_dev_offload *xdo) 1142 { 1143 unsigned int h = __xfrm_spi_hash(daddr, spi, proto, family, state_ptrs->hmask); 1144 struct xfrm_state *x; 1145 1146 hlist_for_each_entry_rcu(x, state_ptrs->byspi + h, byspi) { 1147 #ifdef CONFIG_XFRM_OFFLOAD 1148 if (xdo->type == XFRM_DEV_OFFLOAD_PACKET) { 1149 if (x->xso.type != XFRM_DEV_OFFLOAD_PACKET) 1150 /* HW states are in the head of list, there is 1151 * no need to iterate further. 1152 */ 1153 break; 1154 1155 /* Packet offload: both policy and SA should 1156 * have same device. 1157 */ 1158 if (xdo->dev != x->xso.dev) 1159 continue; 1160 } else if (x->xso.type == XFRM_DEV_OFFLOAD_PACKET) 1161 /* Skip HW policy for SW lookups */ 1162 continue; 1163 #endif 1164 if (x->props.family != family || 1165 x->id.spi != spi || 1166 x->id.proto != proto || 1167 !xfrm_addr_equal(&x->id.daddr, daddr, family)) 1168 continue; 1169 1170 if ((mark & x->mark.m) != x->mark.v) 1171 continue; 1172 if (!xfrm_state_hold_rcu(x)) 1173 continue; 1174 return x; 1175 } 1176 1177 return NULL; 1178 } 1179 1180 static struct xfrm_state *__xfrm_state_lookup(const struct xfrm_hash_state_ptrs *state_ptrs, 1181 u32 mark, 1182 const xfrm_address_t *daddr, 1183 __be32 spi, u8 proto, 1184 unsigned short family) 1185 { 1186 unsigned int h = __xfrm_spi_hash(daddr, spi, proto, family, state_ptrs->hmask); 1187 struct xfrm_state *x; 1188 1189 hlist_for_each_entry_rcu(x, state_ptrs->byspi + h, byspi) { 1190 if (x->props.family != family || 1191 x->id.spi != spi || 1192 x->id.proto != proto || 1193 !xfrm_addr_equal(&x->id.daddr, daddr, family)) 1194 continue; 1195 1196 if ((mark & x->mark.m) != x->mark.v) 1197 continue; 1198 if (!xfrm_state_hold_rcu(x)) 1199 continue; 1200 return x; 1201 } 1202 1203 return NULL; 1204 } 1205 1206 struct xfrm_state *xfrm_input_state_lookup(struct net *net, u32 mark, 1207 const xfrm_address_t *daddr, 1208 __be32 spi, u8 proto, 1209 unsigned short family) 1210 { 1211 struct xfrm_hash_state_ptrs state_ptrs; 1212 struct hlist_head *state_cache_input; 1213 struct xfrm_state *x = NULL; 1214 1215 /* BH is always disabled on the input path. */ 1216 lockdep_assert_in_softirq(); 1217 1218 state_cache_input = raw_cpu_ptr(net->xfrm.state_cache_input); 1219 1220 hlist_for_each_entry_rcu(x, state_cache_input, state_cache_input) { 1221 if (x->props.family != family || 1222 x->id.spi != spi || 1223 x->id.proto != proto || 1224 !xfrm_addr_equal(&x->id.daddr, daddr, family)) 1225 continue; 1226 1227 if ((mark & x->mark.m) != x->mark.v) 1228 continue; 1229 if (!xfrm_state_hold_rcu(x)) 1230 continue; 1231 goto out; 1232 } 1233 1234 xfrm_hash_ptrs_get(net, &state_ptrs); 1235 1236 x = __xfrm_state_lookup(&state_ptrs, mark, daddr, spi, proto, family); 1237 if (x) { 1238 spin_lock(&net->xfrm.xfrm_state_lock); 1239 if (x->km.state != XFRM_STATE_VALID) { 1240 /* 1241 * The state is about to be destroyed. 1242 * 1243 * Don't add it to the cache but still 1244 * return it to the caller. 1245 */ 1246 } else if (hlist_unhashed(&x->state_cache_input)) { 1247 hlist_add_head_rcu(&x->state_cache_input, state_cache_input); 1248 } else { 1249 hlist_del_rcu(&x->state_cache_input); 1250 hlist_add_head_rcu(&x->state_cache_input, state_cache_input); 1251 } 1252 spin_unlock(&net->xfrm.xfrm_state_lock); 1253 } 1254 1255 out: 1256 return x; 1257 } 1258 EXPORT_SYMBOL(xfrm_input_state_lookup); 1259 1260 static struct xfrm_state *__xfrm_state_lookup_byaddr(const struct xfrm_hash_state_ptrs *state_ptrs, 1261 u32 mark, 1262 const xfrm_address_t *daddr, 1263 const xfrm_address_t *saddr, 1264 u8 proto, unsigned short family) 1265 { 1266 unsigned int h = __xfrm_src_hash(daddr, saddr, family, state_ptrs->hmask); 1267 struct xfrm_state *x; 1268 1269 hlist_for_each_entry_rcu(x, state_ptrs->bysrc + h, bysrc) { 1270 if (x->props.family != family || 1271 x->id.proto != proto || 1272 !xfrm_addr_equal(&x->id.daddr, daddr, family) || 1273 !xfrm_addr_equal(&x->props.saddr, saddr, family)) 1274 continue; 1275 1276 if ((mark & x->mark.m) != x->mark.v) 1277 continue; 1278 if (!xfrm_state_hold_rcu(x)) 1279 continue; 1280 return x; 1281 } 1282 1283 return NULL; 1284 } 1285 1286 static inline struct xfrm_state * 1287 __xfrm_state_locate(struct xfrm_state *x, int use_spi, int family) 1288 { 1289 struct xfrm_hash_state_ptrs state_ptrs; 1290 struct net *net = xs_net(x); 1291 u32 mark = x->mark.v & x->mark.m; 1292 1293 xfrm_hash_ptrs_get(net, &state_ptrs); 1294 1295 if (use_spi) 1296 return __xfrm_state_lookup(&state_ptrs, mark, &x->id.daddr, 1297 x->id.spi, x->id.proto, family); 1298 else 1299 return __xfrm_state_lookup_byaddr(&state_ptrs, mark, 1300 &x->id.daddr, 1301 &x->props.saddr, 1302 x->id.proto, family); 1303 } 1304 1305 static void xfrm_hash_grow_check(struct net *net, int have_hash_collision) 1306 { 1307 if (have_hash_collision && 1308 (net->xfrm.state_hmask + 1) < xfrm_state_hashmax && 1309 net->xfrm.state_num > net->xfrm.state_hmask) 1310 schedule_work(&net->xfrm.state_hash_work); 1311 } 1312 1313 static void xfrm_state_look_at(struct xfrm_policy *pol, struct xfrm_state *x, 1314 const struct flowi *fl, unsigned short family, 1315 struct xfrm_state **best, int *acq_in_progress, 1316 int *error, unsigned int pcpu_id) 1317 { 1318 /* Resolution logic: 1319 * 1. There is a valid state with matching selector. Done. 1320 * 2. Valid state with inappropriate selector. Skip. 1321 * 1322 * Entering area of "sysdeps". 1323 * 1324 * 3. If state is not valid, selector is temporary, it selects 1325 * only session which triggered previous resolution. Key 1326 * manager will do something to install a state with proper 1327 * selector. 1328 */ 1329 if (x->km.state == XFRM_STATE_VALID) { 1330 if ((x->sel.family && 1331 (x->sel.family != family || 1332 !xfrm_selector_match(&x->sel, fl, family))) || 1333 !security_xfrm_state_pol_flow_match(x, pol, 1334 &fl->u.__fl_common)) 1335 return; 1336 1337 if (x->pcpu_num != UINT_MAX && x->pcpu_num != pcpu_id) 1338 return; 1339 1340 if (!*best || 1341 ((*best)->pcpu_num == UINT_MAX && x->pcpu_num == pcpu_id) || 1342 (*best)->km.dying > x->km.dying || 1343 ((*best)->km.dying == x->km.dying && 1344 (*best)->curlft.add_time < x->curlft.add_time)) 1345 *best = x; 1346 } else if (x->km.state == XFRM_STATE_ACQ) { 1347 if (!*best || x->pcpu_num == pcpu_id) 1348 *acq_in_progress = 1; 1349 } else if (x->km.state == XFRM_STATE_ERROR || 1350 x->km.state == XFRM_STATE_EXPIRED) { 1351 if ((!x->sel.family || 1352 (x->sel.family == family && 1353 xfrm_selector_match(&x->sel, fl, family))) && 1354 security_xfrm_state_pol_flow_match(x, pol, 1355 &fl->u.__fl_common)) 1356 *error = -ESRCH; 1357 } 1358 } 1359 1360 struct xfrm_state * 1361 xfrm_state_find(const xfrm_address_t *daddr, const xfrm_address_t *saddr, 1362 const struct flowi *fl, struct xfrm_tmpl *tmpl, 1363 struct xfrm_policy *pol, int *err, 1364 unsigned short family, u32 if_id) 1365 { 1366 static xfrm_address_t saddr_wildcard = { }; 1367 struct xfrm_hash_state_ptrs state_ptrs; 1368 struct net *net = xp_net(pol); 1369 unsigned int h, h_wildcard; 1370 struct xfrm_state *x, *x0, *to_put; 1371 int acquire_in_progress = 0; 1372 int error = 0; 1373 struct xfrm_state *best = NULL; 1374 u32 mark = pol->mark.v & pol->mark.m; 1375 unsigned short encap_family = tmpl->encap_family; 1376 unsigned int sequence; 1377 struct km_event c; 1378 unsigned int pcpu_id; 1379 bool cached = false; 1380 1381 /* We need the cpu id just as a lookup key, 1382 * we don't require it to be stable. 1383 */ 1384 pcpu_id = raw_smp_processor_id(); 1385 1386 to_put = NULL; 1387 1388 sequence = read_seqcount_begin(&net->xfrm.xfrm_state_hash_generation); 1389 1390 rcu_read_lock(); 1391 xfrm_hash_ptrs_get(net, &state_ptrs); 1392 1393 hlist_for_each_entry_rcu(x, &pol->state_cache_list, state_cache) { 1394 if (x->props.family == encap_family && 1395 x->props.reqid == tmpl->reqid && 1396 (mark & x->mark.m) == x->mark.v && 1397 x->if_id == if_id && 1398 !(x->props.flags & XFRM_STATE_WILDRECV) && 1399 xfrm_state_addr_check(x, daddr, saddr, encap_family) && 1400 tmpl->mode == x->props.mode && 1401 tmpl->id.proto == x->id.proto && 1402 (tmpl->id.spi == x->id.spi || !tmpl->id.spi)) 1403 xfrm_state_look_at(pol, x, fl, encap_family, 1404 &best, &acquire_in_progress, &error, pcpu_id); 1405 } 1406 1407 if (best) 1408 goto cached; 1409 1410 hlist_for_each_entry_rcu(x, &pol->state_cache_list, state_cache) { 1411 if (x->props.family == encap_family && 1412 x->props.reqid == tmpl->reqid && 1413 (mark & x->mark.m) == x->mark.v && 1414 x->if_id == if_id && 1415 !(x->props.flags & XFRM_STATE_WILDRECV) && 1416 xfrm_addr_equal(&x->id.daddr, daddr, encap_family) && 1417 tmpl->mode == x->props.mode && 1418 tmpl->id.proto == x->id.proto && 1419 (tmpl->id.spi == x->id.spi || !tmpl->id.spi)) 1420 xfrm_state_look_at(pol, x, fl, family, 1421 &best, &acquire_in_progress, &error, pcpu_id); 1422 } 1423 1424 cached: 1425 cached = true; 1426 if (best) 1427 goto found; 1428 else if (error) 1429 best = NULL; 1430 else if (acquire_in_progress) /* XXX: acquire_in_progress should not happen */ 1431 WARN_ON(1); 1432 1433 h = __xfrm_dst_hash(daddr, saddr, tmpl->reqid, encap_family, state_ptrs.hmask); 1434 hlist_for_each_entry_rcu(x, state_ptrs.bydst + h, bydst) { 1435 #ifdef CONFIG_XFRM_OFFLOAD 1436 if (pol->xdo.type == XFRM_DEV_OFFLOAD_PACKET) { 1437 if (x->xso.type != XFRM_DEV_OFFLOAD_PACKET) 1438 /* HW states are in the head of list, there is 1439 * no need to iterate further. 1440 */ 1441 break; 1442 1443 /* Packet offload: both policy and SA should 1444 * have same device. 1445 */ 1446 if (pol->xdo.dev != x->xso.dev) 1447 continue; 1448 } else if (x->xso.type == XFRM_DEV_OFFLOAD_PACKET) 1449 /* Skip HW policy for SW lookups */ 1450 continue; 1451 #endif 1452 if (x->props.family == encap_family && 1453 x->props.reqid == tmpl->reqid && 1454 (mark & x->mark.m) == x->mark.v && 1455 x->if_id == if_id && 1456 !(x->props.flags & XFRM_STATE_WILDRECV) && 1457 xfrm_state_addr_check(x, daddr, saddr, encap_family) && 1458 tmpl->mode == x->props.mode && 1459 tmpl->id.proto == x->id.proto && 1460 (tmpl->id.spi == x->id.spi || !tmpl->id.spi)) 1461 xfrm_state_look_at(pol, x, fl, family, 1462 &best, &acquire_in_progress, &error, pcpu_id); 1463 } 1464 if (best || acquire_in_progress) 1465 goto found; 1466 1467 h_wildcard = __xfrm_dst_hash(daddr, &saddr_wildcard, tmpl->reqid, 1468 encap_family, state_ptrs.hmask); 1469 hlist_for_each_entry_rcu(x, state_ptrs.bydst + h_wildcard, bydst) { 1470 #ifdef CONFIG_XFRM_OFFLOAD 1471 if (pol->xdo.type == XFRM_DEV_OFFLOAD_PACKET) { 1472 if (x->xso.type != XFRM_DEV_OFFLOAD_PACKET) 1473 /* HW states are in the head of list, there is 1474 * no need to iterate further. 1475 */ 1476 break; 1477 1478 /* Packet offload: both policy and SA should 1479 * have same device. 1480 */ 1481 if (pol->xdo.dev != x->xso.dev) 1482 continue; 1483 } else if (x->xso.type == XFRM_DEV_OFFLOAD_PACKET) 1484 /* Skip HW policy for SW lookups */ 1485 continue; 1486 #endif 1487 if (x->props.family == encap_family && 1488 x->props.reqid == tmpl->reqid && 1489 (mark & x->mark.m) == x->mark.v && 1490 x->if_id == if_id && 1491 !(x->props.flags & XFRM_STATE_WILDRECV) && 1492 xfrm_addr_equal(&x->id.daddr, daddr, encap_family) && 1493 tmpl->mode == x->props.mode && 1494 tmpl->id.proto == x->id.proto && 1495 (tmpl->id.spi == x->id.spi || !tmpl->id.spi)) 1496 xfrm_state_look_at(pol, x, fl, family, 1497 &best, &acquire_in_progress, &error, pcpu_id); 1498 } 1499 1500 found: 1501 if (!(pol->flags & XFRM_POLICY_CPU_ACQUIRE) || 1502 (best && (best->pcpu_num == pcpu_id))) 1503 x = best; 1504 1505 if (!x && !error && !acquire_in_progress) { 1506 if (tmpl->id.spi && 1507 (x0 = __xfrm_state_lookup_all(&state_ptrs, mark, daddr, 1508 tmpl->id.spi, tmpl->id.proto, 1509 encap_family, 1510 &pol->xdo)) != NULL) { 1511 to_put = x0; 1512 error = -EEXIST; 1513 goto out; 1514 } 1515 1516 c.net = net; 1517 /* If the KMs have no listeners (yet...), avoid allocating an SA 1518 * for each and every packet - garbage collection might not 1519 * handle the flood. 1520 */ 1521 if (!km_is_alive(&c)) { 1522 error = -ESRCH; 1523 goto out; 1524 } 1525 1526 x = xfrm_state_alloc(net); 1527 if (x == NULL) { 1528 error = -ENOMEM; 1529 goto out; 1530 } 1531 /* Initialize temporary state matching only 1532 * to current session. */ 1533 xfrm_init_tempstate(x, fl, tmpl, daddr, saddr, family); 1534 memcpy(&x->mark, &pol->mark, sizeof(x->mark)); 1535 x->if_id = if_id; 1536 if ((pol->flags & XFRM_POLICY_CPU_ACQUIRE) && best) 1537 x->pcpu_num = pcpu_id; 1538 1539 error = security_xfrm_state_alloc_acquire(x, pol->security, fl->flowi_secid); 1540 if (error) { 1541 x->km.state = XFRM_STATE_DEAD; 1542 to_put = x; 1543 x = NULL; 1544 goto out; 1545 } 1546 #ifdef CONFIG_XFRM_OFFLOAD 1547 if (pol->xdo.type == XFRM_DEV_OFFLOAD_PACKET) { 1548 struct xfrm_dev_offload *xdo = &pol->xdo; 1549 struct xfrm_dev_offload *xso = &x->xso; 1550 struct net_device *dev = xdo->dev; 1551 1552 xso->type = XFRM_DEV_OFFLOAD_PACKET; 1553 xso->dir = xdo->dir; 1554 xso->dev = dev; 1555 xso->ifindex = dev->ifindex; 1556 xso->flags = XFRM_DEV_OFFLOAD_FLAG_ACQ; 1557 netdev_hold(dev, &xso->dev_tracker, GFP_ATOMIC); 1558 error = dev->xfrmdev_ops->xdo_dev_state_add(dev, x, 1559 NULL); 1560 if (error) { 1561 xso->dir = 0; 1562 netdev_put(dev, &xso->dev_tracker); 1563 xso->dev = NULL; 1564 xso->type = XFRM_DEV_OFFLOAD_UNSPECIFIED; 1565 x->km.state = XFRM_STATE_DEAD; 1566 to_put = x; 1567 x = NULL; 1568 goto out; 1569 } 1570 } 1571 #endif 1572 if (km_query(x, tmpl, pol) == 0) { 1573 spin_lock_bh(&net->xfrm.xfrm_state_lock); 1574 x->km.state = XFRM_STATE_ACQ; 1575 x->dir = XFRM_SA_DIR_OUT; 1576 list_add(&x->km.all, &net->xfrm.state_all); 1577 h = xfrm_dst_hash(net, daddr, saddr, tmpl->reqid, encap_family); 1578 XFRM_STATE_INSERT(bydst, &x->bydst, 1579 xfrm_state_deref_prot(net->xfrm.state_bydst, net) + h, 1580 x->xso.type); 1581 h = xfrm_src_hash(net, daddr, saddr, encap_family); 1582 XFRM_STATE_INSERT(bysrc, &x->bysrc, 1583 xfrm_state_deref_prot(net->xfrm.state_bysrc, net) + h, 1584 x->xso.type); 1585 INIT_HLIST_NODE(&x->state_cache); 1586 if (x->id.spi) { 1587 h = xfrm_spi_hash(net, &x->id.daddr, x->id.spi, x->id.proto, encap_family); 1588 XFRM_STATE_INSERT(byspi, &x->byspi, 1589 xfrm_state_deref_prot(net->xfrm.state_byspi, net) + h, 1590 x->xso.type); 1591 } 1592 if (x->km.seq) { 1593 h = xfrm_seq_hash(net, x->km.seq); 1594 XFRM_STATE_INSERT(byseq, &x->byseq, 1595 xfrm_state_deref_prot(net->xfrm.state_byseq, net) + h, 1596 x->xso.type); 1597 } 1598 x->lft.hard_add_expires_seconds = net->xfrm.sysctl_acq_expires; 1599 hrtimer_start(&x->mtimer, 1600 ktime_set(net->xfrm.sysctl_acq_expires, 0), 1601 HRTIMER_MODE_REL_SOFT); 1602 net->xfrm.state_num++; 1603 xfrm_hash_grow_check(net, x->bydst.next != NULL); 1604 spin_unlock_bh(&net->xfrm.xfrm_state_lock); 1605 } else { 1606 #ifdef CONFIG_XFRM_OFFLOAD 1607 struct xfrm_dev_offload *xso = &x->xso; 1608 1609 if (xso->type == XFRM_DEV_OFFLOAD_PACKET) { 1610 xfrm_dev_state_delete(x); 1611 xfrm_dev_state_free(x); 1612 } 1613 #endif 1614 x->km.state = XFRM_STATE_DEAD; 1615 to_put = x; 1616 x = NULL; 1617 error = -ESRCH; 1618 } 1619 1620 /* Use the already installed 'fallback' while the CPU-specific 1621 * SA acquire is handled*/ 1622 if (best) 1623 x = best; 1624 } 1625 out: 1626 if (x) { 1627 if (!xfrm_state_hold_rcu(x)) { 1628 *err = -EAGAIN; 1629 x = NULL; 1630 } 1631 } else { 1632 *err = acquire_in_progress ? -EAGAIN : error; 1633 } 1634 1635 if (x && x->km.state == XFRM_STATE_VALID && !cached && 1636 (!(pol->flags & XFRM_POLICY_CPU_ACQUIRE) || x->pcpu_num == pcpu_id)) { 1637 spin_lock_bh(&net->xfrm.xfrm_state_lock); 1638 if (hlist_unhashed(&x->state_cache)) 1639 hlist_add_head_rcu(&x->state_cache, &pol->state_cache_list); 1640 spin_unlock_bh(&net->xfrm.xfrm_state_lock); 1641 } 1642 1643 rcu_read_unlock(); 1644 if (to_put) 1645 xfrm_state_put(to_put); 1646 1647 if (read_seqcount_retry(&net->xfrm.xfrm_state_hash_generation, sequence)) { 1648 *err = -EAGAIN; 1649 if (x) { 1650 xfrm_state_put(x); 1651 x = NULL; 1652 } 1653 } 1654 1655 return x; 1656 } 1657 1658 struct xfrm_state * 1659 xfrm_stateonly_find(struct net *net, u32 mark, u32 if_id, 1660 xfrm_address_t *daddr, xfrm_address_t *saddr, 1661 unsigned short family, u8 mode, u8 proto, u32 reqid) 1662 { 1663 unsigned int h; 1664 struct xfrm_state *rx = NULL, *x = NULL; 1665 1666 spin_lock_bh(&net->xfrm.xfrm_state_lock); 1667 h = xfrm_dst_hash(net, daddr, saddr, reqid, family); 1668 hlist_for_each_entry(x, xfrm_state_deref_prot(net->xfrm.state_bydst, net) + h, bydst) { 1669 if (x->props.family == family && 1670 x->props.reqid == reqid && 1671 (mark & x->mark.m) == x->mark.v && 1672 x->if_id == if_id && 1673 !(x->props.flags & XFRM_STATE_WILDRECV) && 1674 xfrm_state_addr_check(x, daddr, saddr, family) && 1675 mode == x->props.mode && 1676 proto == x->id.proto && 1677 x->km.state == XFRM_STATE_VALID) { 1678 rx = x; 1679 break; 1680 } 1681 } 1682 1683 if (rx) 1684 xfrm_state_hold(rx); 1685 spin_unlock_bh(&net->xfrm.xfrm_state_lock); 1686 1687 1688 return rx; 1689 } 1690 EXPORT_SYMBOL(xfrm_stateonly_find); 1691 1692 struct xfrm_state *xfrm_state_lookup_byspi(struct net *net, __be32 spi, 1693 unsigned short family) 1694 { 1695 struct xfrm_state *x; 1696 struct xfrm_state_walk *w; 1697 1698 spin_lock_bh(&net->xfrm.xfrm_state_lock); 1699 list_for_each_entry(w, &net->xfrm.state_all, all) { 1700 x = container_of(w, struct xfrm_state, km); 1701 if (x->props.family != family || 1702 x->id.spi != spi) 1703 continue; 1704 1705 xfrm_state_hold(x); 1706 spin_unlock_bh(&net->xfrm.xfrm_state_lock); 1707 return x; 1708 } 1709 spin_unlock_bh(&net->xfrm.xfrm_state_lock); 1710 return NULL; 1711 } 1712 EXPORT_SYMBOL(xfrm_state_lookup_byspi); 1713 1714 static struct xfrm_state *xfrm_state_lookup_spi_proto(struct net *net, __be32 spi, u8 proto) 1715 { 1716 struct xfrm_state *x; 1717 unsigned int i; 1718 1719 for (i = 0; i <= net->xfrm.state_hmask; i++) { 1720 hlist_for_each_entry(x, xfrm_state_deref_prot(net->xfrm.state_byspi, net) + i, byspi) { 1721 if (x->id.spi == spi && x->id.proto == proto) 1722 return x; 1723 } 1724 } 1725 return NULL; 1726 } 1727 1728 static void __xfrm_state_insert(struct xfrm_state *x) 1729 { 1730 struct net *net = xs_net(x); 1731 unsigned int h; 1732 1733 list_add(&x->km.all, &net->xfrm.state_all); 1734 1735 /* Sanitize mark before store */ 1736 x->mark.v &= x->mark.m; 1737 1738 h = xfrm_dst_hash(net, &x->id.daddr, &x->props.saddr, 1739 x->props.reqid, x->props.family); 1740 XFRM_STATE_INSERT(bydst, &x->bydst, 1741 xfrm_state_deref_prot(net->xfrm.state_bydst, net) + h, 1742 x->xso.type); 1743 1744 h = xfrm_src_hash(net, &x->id.daddr, &x->props.saddr, x->props.family); 1745 XFRM_STATE_INSERT(bysrc, &x->bysrc, 1746 xfrm_state_deref_prot(net->xfrm.state_bysrc, net) + h, 1747 x->xso.type); 1748 1749 if (x->id.spi) { 1750 h = xfrm_spi_hash(net, &x->id.daddr, x->id.spi, x->id.proto, 1751 x->props.family); 1752 1753 XFRM_STATE_INSERT(byspi, &x->byspi, 1754 xfrm_state_deref_prot(net->xfrm.state_byspi, net) + h, 1755 x->xso.type); 1756 } 1757 1758 if (x->km.seq) { 1759 h = xfrm_seq_hash(net, x->km.seq); 1760 1761 XFRM_STATE_INSERT(byseq, &x->byseq, 1762 xfrm_state_deref_prot(net->xfrm.state_byseq, net) + h, 1763 x->xso.type); 1764 } 1765 1766 hrtimer_start(&x->mtimer, ktime_set(1, 0), HRTIMER_MODE_REL_SOFT); 1767 if (x->replay_maxage) 1768 mod_timer(&x->rtimer, jiffies + x->replay_maxage); 1769 1770 net->xfrm.state_num++; 1771 1772 xfrm_hash_grow_check(net, x->bydst.next != NULL); 1773 xfrm_nat_keepalive_state_updated(x); 1774 } 1775 1776 /* net->xfrm.xfrm_state_lock is held */ 1777 static void __xfrm_state_bump_genids(struct xfrm_state *xnew) 1778 { 1779 struct net *net = xs_net(xnew); 1780 unsigned short family = xnew->props.family; 1781 u32 reqid = xnew->props.reqid; 1782 struct xfrm_state *x; 1783 unsigned int h; 1784 u32 mark = xnew->mark.v & xnew->mark.m; 1785 u32 if_id = xnew->if_id; 1786 u32 cpu_id = xnew->pcpu_num; 1787 1788 h = xfrm_dst_hash(net, &xnew->id.daddr, &xnew->props.saddr, reqid, family); 1789 hlist_for_each_entry(x, xfrm_state_deref_prot(net->xfrm.state_bydst, net) + h, bydst) { 1790 if (x->props.family == family && 1791 x->props.reqid == reqid && 1792 x->if_id == if_id && 1793 x->pcpu_num == cpu_id && 1794 (mark & x->mark.m) == x->mark.v && 1795 xfrm_addr_equal(&x->id.daddr, &xnew->id.daddr, family) && 1796 xfrm_addr_equal(&x->props.saddr, &xnew->props.saddr, family)) 1797 x->genid++; 1798 } 1799 } 1800 1801 void xfrm_state_insert(struct xfrm_state *x) 1802 { 1803 struct net *net = xs_net(x); 1804 1805 spin_lock_bh(&net->xfrm.xfrm_state_lock); 1806 __xfrm_state_bump_genids(x); 1807 __xfrm_state_insert(x); 1808 spin_unlock_bh(&net->xfrm.xfrm_state_lock); 1809 } 1810 EXPORT_SYMBOL(xfrm_state_insert); 1811 1812 /* net->xfrm.xfrm_state_lock is held */ 1813 static struct xfrm_state *__find_acq_core(struct net *net, 1814 const struct xfrm_mark *m, 1815 unsigned short family, u8 mode, 1816 u32 reqid, u32 if_id, u32 pcpu_num, u8 proto, 1817 const xfrm_address_t *daddr, 1818 const xfrm_address_t *saddr, 1819 int create) 1820 { 1821 unsigned int h = xfrm_dst_hash(net, daddr, saddr, reqid, family); 1822 struct xfrm_state *x; 1823 u32 mark = m->v & m->m; 1824 1825 hlist_for_each_entry(x, xfrm_state_deref_prot(net->xfrm.state_bydst, net) + h, bydst) { 1826 if (x->props.reqid != reqid || 1827 x->props.mode != mode || 1828 x->props.family != family || 1829 x->km.state != XFRM_STATE_ACQ || 1830 x->id.spi != 0 || 1831 x->id.proto != proto || 1832 (mark & x->mark.m) != x->mark.v || 1833 x->pcpu_num != pcpu_num || 1834 !xfrm_addr_equal(&x->id.daddr, daddr, family) || 1835 !xfrm_addr_equal(&x->props.saddr, saddr, family)) 1836 continue; 1837 1838 xfrm_state_hold(x); 1839 return x; 1840 } 1841 1842 if (!create) 1843 return NULL; 1844 1845 x = xfrm_state_alloc(net); 1846 if (likely(x)) { 1847 switch (family) { 1848 case AF_INET: 1849 x->sel.daddr.a4 = daddr->a4; 1850 x->sel.saddr.a4 = saddr->a4; 1851 x->sel.prefixlen_d = 32; 1852 x->sel.prefixlen_s = 32; 1853 x->props.saddr.a4 = saddr->a4; 1854 x->id.daddr.a4 = daddr->a4; 1855 break; 1856 1857 case AF_INET6: 1858 x->sel.daddr.in6 = daddr->in6; 1859 x->sel.saddr.in6 = saddr->in6; 1860 x->sel.prefixlen_d = 128; 1861 x->sel.prefixlen_s = 128; 1862 x->props.saddr.in6 = saddr->in6; 1863 x->id.daddr.in6 = daddr->in6; 1864 break; 1865 } 1866 1867 x->pcpu_num = pcpu_num; 1868 x->km.state = XFRM_STATE_ACQ; 1869 x->id.proto = proto; 1870 x->props.family = family; 1871 x->props.mode = mode; 1872 x->props.reqid = reqid; 1873 x->if_id = if_id; 1874 x->mark.v = m->v; 1875 x->mark.m = m->m; 1876 x->lft.hard_add_expires_seconds = net->xfrm.sysctl_acq_expires; 1877 xfrm_state_hold(x); 1878 hrtimer_start(&x->mtimer, 1879 ktime_set(net->xfrm.sysctl_acq_expires, 0), 1880 HRTIMER_MODE_REL_SOFT); 1881 list_add(&x->km.all, &net->xfrm.state_all); 1882 XFRM_STATE_INSERT(bydst, &x->bydst, 1883 xfrm_state_deref_prot(net->xfrm.state_bydst, net) + h, 1884 x->xso.type); 1885 h = xfrm_src_hash(net, daddr, saddr, family); 1886 XFRM_STATE_INSERT(bysrc, &x->bysrc, 1887 xfrm_state_deref_prot(net->xfrm.state_bysrc, net) + h, 1888 x->xso.type); 1889 1890 net->xfrm.state_num++; 1891 1892 xfrm_hash_grow_check(net, x->bydst.next != NULL); 1893 } 1894 1895 return x; 1896 } 1897 1898 static struct xfrm_state *__xfrm_find_acq_byseq(struct net *net, u32 mark, u32 seq, u32 pcpu_num); 1899 1900 int xfrm_state_add(struct xfrm_state *x) 1901 { 1902 struct net *net = xs_net(x); 1903 struct xfrm_state *x1, *to_put; 1904 int family; 1905 int err; 1906 u32 mark = x->mark.v & x->mark.m; 1907 int use_spi = xfrm_id_proto_match(x->id.proto, IPSEC_PROTO_ANY); 1908 1909 family = x->props.family; 1910 1911 to_put = NULL; 1912 1913 spin_lock_bh(&net->xfrm.xfrm_state_lock); 1914 1915 x1 = __xfrm_state_locate(x, use_spi, family); 1916 if (x1) { 1917 to_put = x1; 1918 x1 = NULL; 1919 err = -EEXIST; 1920 goto out; 1921 } 1922 1923 if (use_spi && x->km.seq) { 1924 x1 = __xfrm_find_acq_byseq(net, mark, x->km.seq, x->pcpu_num); 1925 if (x1 && ((x1->id.proto != x->id.proto) || 1926 !xfrm_addr_equal(&x1->id.daddr, &x->id.daddr, family))) { 1927 to_put = x1; 1928 x1 = NULL; 1929 } 1930 } 1931 1932 if (use_spi && !x1) 1933 x1 = __find_acq_core(net, &x->mark, family, x->props.mode, 1934 x->props.reqid, x->if_id, x->pcpu_num, x->id.proto, 1935 &x->id.daddr, &x->props.saddr, 0); 1936 1937 __xfrm_state_bump_genids(x); 1938 __xfrm_state_insert(x); 1939 err = 0; 1940 1941 out: 1942 spin_unlock_bh(&net->xfrm.xfrm_state_lock); 1943 1944 if (x1) { 1945 xfrm_state_delete(x1); 1946 xfrm_state_put(x1); 1947 } 1948 1949 if (to_put) 1950 xfrm_state_put(to_put); 1951 1952 return err; 1953 } 1954 EXPORT_SYMBOL(xfrm_state_add); 1955 1956 #ifdef CONFIG_XFRM_MIGRATE 1957 static inline int clone_security(struct xfrm_state *x, struct xfrm_sec_ctx *security) 1958 { 1959 struct xfrm_user_sec_ctx *uctx; 1960 int size = sizeof(*uctx) + security->ctx_len; 1961 int err; 1962 1963 uctx = kmalloc(size, GFP_KERNEL); 1964 if (!uctx) 1965 return -ENOMEM; 1966 1967 uctx->exttype = XFRMA_SEC_CTX; 1968 uctx->len = size; 1969 uctx->ctx_doi = security->ctx_doi; 1970 uctx->ctx_alg = security->ctx_alg; 1971 uctx->ctx_len = security->ctx_len; 1972 memcpy(uctx + 1, security->ctx_str, security->ctx_len); 1973 err = security_xfrm_state_alloc(x, uctx); 1974 kfree(uctx); 1975 if (err) 1976 return err; 1977 1978 return 0; 1979 } 1980 1981 static struct xfrm_state *xfrm_state_clone_and_setup(struct xfrm_state *orig, 1982 const struct xfrm_migrate *m) 1983 { 1984 struct net *net = xs_net(orig); 1985 struct xfrm_state *x = xfrm_state_alloc(net); 1986 if (!x) 1987 goto out; 1988 1989 memcpy(&x->id, &orig->id, sizeof(x->id)); 1990 if (m->msg_type == XFRM_MSG_MIGRATE_STATE) { 1991 if (m->flags & XFRM_MIGRATE_STATE_UPDATE_H2H_SEL) { 1992 u8 prefixlen = (m->new_family == AF_INET6) ? 128 : 32; 1993 1994 x->sel = orig->sel; 1995 x->sel.family = m->new_family; 1996 x->sel.prefixlen_d = prefixlen; 1997 x->sel.prefixlen_s = prefixlen; 1998 x->sel.daddr = m->new_daddr; 1999 x->sel.saddr = m->new_saddr; 2000 } else { 2001 x->sel = *m->new_sel; 2002 } 2003 } else { 2004 x->sel = orig->sel; 2005 } 2006 memcpy(&x->lft, &orig->lft, sizeof(x->lft)); 2007 x->props.mode = orig->props.mode; 2008 x->props.replay_window = orig->props.replay_window; 2009 2010 if (orig->aalg) { 2011 x->aalg = xfrm_algo_auth_clone(orig->aalg); 2012 if (!x->aalg) 2013 goto error; 2014 } 2015 x->props.aalgo = orig->props.aalgo; 2016 2017 if (orig->aead) { 2018 x->aead = xfrm_algo_aead_clone(orig->aead); 2019 x->geniv = orig->geniv; 2020 if (!x->aead) 2021 goto error; 2022 } 2023 if (orig->ealg) { 2024 x->ealg = xfrm_algo_clone(orig->ealg); 2025 if (!x->ealg) 2026 goto error; 2027 } 2028 x->props.ealgo = orig->props.ealgo; 2029 2030 if (orig->calg) { 2031 x->calg = xfrm_algo_clone(orig->calg); 2032 if (!x->calg) 2033 goto error; 2034 } 2035 x->props.calgo = orig->props.calgo; 2036 2037 if (m->encap) { 2038 x->encap = kmemdup(m->encap, sizeof(*x->encap), GFP_KERNEL); 2039 if (!x->encap) 2040 goto error; 2041 x->mapping_maxage = m->mapping_maxage; 2042 x->nat_keepalive_interval = m->nat_keepalive_interval; 2043 } 2044 2045 if (orig->security) 2046 if (clone_security(x, orig->security)) 2047 goto error; 2048 2049 if (orig->coaddr) { 2050 x->coaddr = kmemdup(orig->coaddr, sizeof(*x->coaddr), 2051 GFP_KERNEL); 2052 if (!x->coaddr) 2053 goto error; 2054 } 2055 2056 if (xfrm_replay_clone(x, orig)) 2057 goto error; 2058 2059 x->mark = m->new_mark ? *m->new_mark : m->old_mark; 2060 2061 x->props.smark = m->smark; 2062 2063 x->props.flags = orig->props.flags; 2064 x->props.extra_flags = orig->props.extra_flags; 2065 2066 x->pcpu_num = orig->pcpu_num; 2067 x->if_id = orig->if_id; 2068 x->tfcpad = orig->tfcpad; 2069 x->replay_maxdiff = orig->replay_maxdiff; 2070 x->replay_maxage = orig->replay_maxage; 2071 x->km.state = orig->km.state; 2072 x->km.seq = orig->km.seq; 2073 x->lastused = orig->lastused; 2074 x->new_mapping = 0; 2075 x->new_mapping_sport = 0; 2076 x->dir = orig->dir; 2077 2078 x->mode_cbs = orig->mode_cbs; 2079 if (x->mode_cbs && x->mode_cbs->clone_state) { 2080 if (x->mode_cbs->clone_state(x, orig)) { 2081 if (!x->mode_data) 2082 x->mode_cbs = NULL; 2083 goto error; 2084 } 2085 } 2086 2087 x->props.reqid = m->new_reqid; 2088 x->props.family = m->new_family; 2089 memcpy(&x->id.daddr, &m->new_daddr, sizeof(x->id.daddr)); 2090 memcpy(&x->props.saddr, &m->new_saddr, sizeof(x->props.saddr)); 2091 2092 return x; 2093 2094 error: 2095 x->km.state = XFRM_STATE_DEAD; 2096 xfrm_state_put(x); 2097 out: 2098 return NULL; 2099 } 2100 2101 struct xfrm_state *xfrm_migrate_state_find(struct xfrm_migrate *m, struct net *net, 2102 u32 if_id) 2103 { 2104 unsigned int h; 2105 struct xfrm_state *x = NULL; 2106 2107 spin_lock_bh(&net->xfrm.xfrm_state_lock); 2108 2109 if (m->old_reqid) { 2110 h = xfrm_dst_hash(net, &m->old_daddr, &m->old_saddr, 2111 m->old_reqid, m->old_family); 2112 hlist_for_each_entry(x, xfrm_state_deref_prot(net->xfrm.state_bydst, net) + h, bydst) { 2113 if (x->props.mode != m->mode || 2114 x->id.proto != m->proto) 2115 continue; 2116 if (m->old_reqid && x->props.reqid != m->old_reqid) 2117 continue; 2118 if (if_id != 0 && x->if_id != if_id) 2119 continue; 2120 if (!xfrm_addr_equal(&x->id.daddr, &m->old_daddr, 2121 m->old_family) || 2122 !xfrm_addr_equal(&x->props.saddr, &m->old_saddr, 2123 m->old_family)) 2124 continue; 2125 xfrm_state_hold(x); 2126 break; 2127 } 2128 } else { 2129 h = xfrm_src_hash(net, &m->old_daddr, &m->old_saddr, 2130 m->old_family); 2131 hlist_for_each_entry(x, xfrm_state_deref_prot(net->xfrm.state_bysrc, net) + h, bysrc) { 2132 if (x->props.mode != m->mode || 2133 x->id.proto != m->proto) 2134 continue; 2135 if (if_id != 0 && x->if_id != if_id) 2136 continue; 2137 if (!xfrm_addr_equal(&x->id.daddr, &m->old_daddr, 2138 m->old_family) || 2139 !xfrm_addr_equal(&x->props.saddr, &m->old_saddr, 2140 m->old_family)) 2141 continue; 2142 xfrm_state_hold(x); 2143 break; 2144 } 2145 } 2146 2147 spin_unlock_bh(&net->xfrm.xfrm_state_lock); 2148 2149 return x; 2150 } 2151 EXPORT_SYMBOL(xfrm_migrate_state_find); 2152 2153 struct xfrm_state *xfrm_state_migrate_create(struct xfrm_state *x, 2154 const struct xfrm_migrate *m, 2155 struct net *net, 2156 struct netlink_ext_ack *extack) 2157 { 2158 struct xfrm_state *xc; 2159 2160 xc = xfrm_state_clone_and_setup(x, m); 2161 if (!xc) { 2162 NL_SET_ERR_MSG(extack, "Failed to clone and setup state"); 2163 return NULL; 2164 } 2165 2166 if (xfrm_init_state(xc, extack) < 0) { 2167 NL_SET_ERR_MSG_WEAK(extack, "Failed to initialize migrated state"); 2168 goto error; 2169 } 2170 2171 /* configure the hardware if offload is requested */ 2172 if (m->xuo && xfrm_dev_state_add(net, xc, m->xuo, extack)) 2173 goto error; 2174 2175 return xc; 2176 error: 2177 xc->km.state = XFRM_STATE_DEAD; 2178 xfrm_state_put(xc); 2179 return NULL; 2180 } 2181 EXPORT_SYMBOL(xfrm_state_migrate_create); 2182 2183 int xfrm_state_migrate_install(const struct xfrm_state *x, 2184 struct xfrm_state *xc, 2185 const struct xfrm_migrate *m, 2186 struct netlink_ext_ack *extack) 2187 { 2188 if (m->new_family == m->old_family && 2189 xfrm_addr_equal(&x->id.daddr, &m->new_daddr, m->new_family)) { 2190 /* 2191 * Care is needed when the destination address of the state is 2192 * to be updated as it is a part of triplet. 2193 */ 2194 xfrm_state_insert(xc); 2195 } else { 2196 if (xfrm_state_add(xc) < 0) { 2197 NL_SET_ERR_MSG(extack, "Failed to add migrated state"); 2198 if (m->xuo) 2199 xfrm_dev_state_delete(xc); 2200 xc->km.state = XFRM_STATE_DEAD; 2201 xfrm_state_put(xc); 2202 return -EEXIST; 2203 } 2204 } 2205 2206 return 0; 2207 } 2208 EXPORT_SYMBOL(xfrm_state_migrate_install); 2209 2210 struct xfrm_state *xfrm_state_migrate(struct xfrm_state *x, 2211 struct xfrm_migrate *m, 2212 struct net *net, 2213 struct netlink_ext_ack *extack) 2214 { 2215 struct xfrm_state *xc; 2216 2217 xc = xfrm_state_migrate_create(x, m, net, extack); 2218 if (!xc) 2219 return NULL; 2220 2221 xfrm_migrate_sync(xc, x); 2222 2223 if (xfrm_state_migrate_install(x, xc, m, extack) < 0) 2224 return NULL; 2225 2226 return xc; 2227 } 2228 EXPORT_SYMBOL(xfrm_state_migrate); 2229 #endif 2230 2231 int xfrm_state_update(struct xfrm_state *x) 2232 { 2233 struct xfrm_state *x1, *to_put; 2234 int err; 2235 int use_spi = xfrm_id_proto_match(x->id.proto, IPSEC_PROTO_ANY); 2236 struct net *net = xs_net(x); 2237 2238 to_put = NULL; 2239 2240 spin_lock_bh(&net->xfrm.xfrm_state_lock); 2241 x1 = __xfrm_state_locate(x, use_spi, x->props.family); 2242 2243 err = -ESRCH; 2244 if (!x1) 2245 goto out; 2246 2247 if (xfrm_state_kern(x1)) { 2248 to_put = x1; 2249 err = -EEXIST; 2250 goto out; 2251 } 2252 2253 if (x1->km.state == XFRM_STATE_ACQ) { 2254 if (x->dir && x1->dir != x->dir) { 2255 to_put = x1; 2256 goto out; 2257 } 2258 2259 __xfrm_state_insert(x); 2260 x = NULL; 2261 } else { 2262 if (x1->dir != x->dir) { 2263 to_put = x1; 2264 goto out; 2265 } 2266 } 2267 err = 0; 2268 2269 out: 2270 spin_unlock_bh(&net->xfrm.xfrm_state_lock); 2271 2272 if (to_put) 2273 xfrm_state_put(to_put); 2274 2275 if (err) 2276 return err; 2277 2278 if (!x) { 2279 xfrm_state_delete(x1); 2280 xfrm_state_put(x1); 2281 return 0; 2282 } 2283 2284 err = -EINVAL; 2285 spin_lock_bh(&x1->lock); 2286 if (likely(x1->km.state == XFRM_STATE_VALID)) { 2287 if (x->encap && x1->encap && 2288 x->encap->encap_type == x1->encap->encap_type) 2289 memcpy(x1->encap, x->encap, sizeof(*x1->encap)); 2290 else if (x->encap || x1->encap) 2291 goto fail; 2292 2293 if (x->coaddr && x1->coaddr) { 2294 memcpy(x1->coaddr, x->coaddr, sizeof(*x1->coaddr)); 2295 } 2296 if (!use_spi && memcmp(&x1->sel, &x->sel, sizeof(x1->sel))) 2297 memcpy(&x1->sel, &x->sel, sizeof(x1->sel)); 2298 memcpy(&x1->lft, &x->lft, sizeof(x1->lft)); 2299 x1->km.dying = 0; 2300 2301 hrtimer_start(&x1->mtimer, ktime_set(1, 0), 2302 HRTIMER_MODE_REL_SOFT); 2303 if (READ_ONCE(x1->curlft.use_time)) 2304 xfrm_state_check_expire(x1); 2305 2306 if (x->props.smark.m || x->props.smark.v || x->if_id) { 2307 spin_lock_bh(&net->xfrm.xfrm_state_lock); 2308 2309 if (x->props.smark.m || x->props.smark.v) 2310 x1->props.smark = x->props.smark; 2311 2312 if (x->if_id) 2313 x1->if_id = x->if_id; 2314 2315 __xfrm_state_bump_genids(x1); 2316 spin_unlock_bh(&net->xfrm.xfrm_state_lock); 2317 } 2318 2319 err = 0; 2320 x->km.state = XFRM_STATE_DEAD; 2321 xfrm_dev_state_delete(x); 2322 __xfrm_state_put(x); 2323 } 2324 2325 fail: 2326 spin_unlock_bh(&x1->lock); 2327 2328 xfrm_state_put(x1); 2329 2330 return err; 2331 } 2332 EXPORT_SYMBOL(xfrm_state_update); 2333 2334 int xfrm_state_check_expire(struct xfrm_state *x) 2335 { 2336 /* All counters which are needed to decide if state is expired 2337 * are handled by SW for non-packet offload modes. Simply skip 2338 * the following update and save extra boilerplate in drivers. 2339 */ 2340 if (x->xso.type == XFRM_DEV_OFFLOAD_PACKET) 2341 xfrm_dev_state_update_stats(x); 2342 2343 if (!READ_ONCE(x->curlft.use_time)) 2344 WRITE_ONCE(x->curlft.use_time, ktime_get_real_seconds()); 2345 2346 if (x->curlft.bytes >= x->lft.hard_byte_limit || 2347 x->curlft.packets >= x->lft.hard_packet_limit) { 2348 x->km.state = XFRM_STATE_EXPIRED; 2349 hrtimer_start(&x->mtimer, 0, HRTIMER_MODE_REL_SOFT); 2350 return -EINVAL; 2351 } 2352 2353 if (!x->km.dying && 2354 (x->curlft.bytes >= x->lft.soft_byte_limit || 2355 x->curlft.packets >= x->lft.soft_packet_limit)) { 2356 x->km.dying = 1; 2357 km_state_expired(x, 0, 0); 2358 } 2359 return 0; 2360 } 2361 EXPORT_SYMBOL(xfrm_state_check_expire); 2362 2363 void xfrm_state_update_stats(struct net *net) 2364 { 2365 struct xfrm_state *x; 2366 int i; 2367 2368 spin_lock_bh(&net->xfrm.xfrm_state_lock); 2369 for (i = 0; i <= net->xfrm.state_hmask; i++) { 2370 hlist_for_each_entry(x, xfrm_state_deref_prot(net->xfrm.state_bydst, net) + i, bydst) 2371 xfrm_dev_state_update_stats(x); 2372 } 2373 spin_unlock_bh(&net->xfrm.xfrm_state_lock); 2374 } 2375 2376 struct xfrm_state * 2377 xfrm_state_lookup(struct net *net, u32 mark, const xfrm_address_t *daddr, __be32 spi, 2378 u8 proto, unsigned short family) 2379 { 2380 struct xfrm_hash_state_ptrs state_ptrs; 2381 struct xfrm_state *x; 2382 2383 rcu_read_lock(); 2384 xfrm_hash_ptrs_get(net, &state_ptrs); 2385 2386 x = __xfrm_state_lookup(&state_ptrs, mark, daddr, spi, proto, family); 2387 rcu_read_unlock(); 2388 return x; 2389 } 2390 EXPORT_SYMBOL(xfrm_state_lookup); 2391 2392 struct xfrm_state * 2393 xfrm_state_lookup_byaddr(struct net *net, u32 mark, 2394 const xfrm_address_t *daddr, const xfrm_address_t *saddr, 2395 u8 proto, unsigned short family) 2396 { 2397 struct xfrm_hash_state_ptrs state_ptrs; 2398 struct xfrm_state *x; 2399 2400 rcu_read_lock(); 2401 2402 xfrm_hash_ptrs_get(net, &state_ptrs); 2403 2404 x = __xfrm_state_lookup_byaddr(&state_ptrs, mark, daddr, saddr, proto, family); 2405 rcu_read_unlock(); 2406 return x; 2407 } 2408 EXPORT_SYMBOL(xfrm_state_lookup_byaddr); 2409 2410 struct xfrm_state * 2411 xfrm_find_acq(struct net *net, const struct xfrm_mark *mark, u8 mode, u32 reqid, 2412 u32 if_id, u32 pcpu_num, u8 proto, const xfrm_address_t *daddr, 2413 const xfrm_address_t *saddr, int create, unsigned short family) 2414 { 2415 struct xfrm_state *x; 2416 2417 spin_lock_bh(&net->xfrm.xfrm_state_lock); 2418 x = __find_acq_core(net, mark, family, mode, reqid, if_id, pcpu_num, 2419 proto, daddr, saddr, create); 2420 spin_unlock_bh(&net->xfrm.xfrm_state_lock); 2421 2422 return x; 2423 } 2424 EXPORT_SYMBOL(xfrm_find_acq); 2425 2426 #ifdef CONFIG_XFRM_SUB_POLICY 2427 #if IS_ENABLED(CONFIG_IPV6) 2428 /* distribution counting sort function for xfrm_state and xfrm_tmpl */ 2429 static void 2430 __xfrm6_sort(void **dst, void **src, int n, 2431 int (*cmp)(const void *p), int maxclass) 2432 { 2433 int count[XFRM_MAX_DEPTH] = { }; 2434 int class[XFRM_MAX_DEPTH]; 2435 int i; 2436 2437 for (i = 0; i < n; i++) { 2438 int c = cmp(src[i]); 2439 2440 class[i] = c; 2441 count[c]++; 2442 } 2443 2444 for (i = 2; i < maxclass; i++) 2445 count[i] += count[i - 1]; 2446 2447 for (i = 0; i < n; i++) { 2448 dst[count[class[i] - 1]++] = src[i]; 2449 src[i] = NULL; 2450 } 2451 } 2452 2453 /* Rule for xfrm_state: 2454 * 2455 * rule 1: select IPsec transport except AH 2456 * rule 2: select MIPv6 RO or inbound trigger 2457 * rule 3: select IPsec transport AH 2458 * rule 4: select IPsec tunnel 2459 * rule 5: others 2460 */ 2461 static int __xfrm6_state_sort_cmp(const void *p) 2462 { 2463 const struct xfrm_state *v = p; 2464 2465 switch (v->props.mode) { 2466 case XFRM_MODE_TRANSPORT: 2467 if (v->id.proto != IPPROTO_AH) 2468 return 1; 2469 else 2470 return 3; 2471 #if IS_ENABLED(CONFIG_IPV6_MIP6) 2472 case XFRM_MODE_ROUTEOPTIMIZATION: 2473 case XFRM_MODE_IN_TRIGGER: 2474 return 2; 2475 #endif 2476 case XFRM_MODE_TUNNEL: 2477 case XFRM_MODE_BEET: 2478 case XFRM_MODE_IPTFS: 2479 return 4; 2480 } 2481 return 5; 2482 } 2483 2484 /* Rule for xfrm_tmpl: 2485 * 2486 * rule 1: select IPsec transport 2487 * rule 2: select MIPv6 RO or inbound trigger 2488 * rule 3: select IPsec tunnel 2489 * rule 4: others 2490 */ 2491 static int __xfrm6_tmpl_sort_cmp(const void *p) 2492 { 2493 const struct xfrm_tmpl *v = p; 2494 2495 switch (v->mode) { 2496 case XFRM_MODE_TRANSPORT: 2497 return 1; 2498 #if IS_ENABLED(CONFIG_IPV6_MIP6) 2499 case XFRM_MODE_ROUTEOPTIMIZATION: 2500 case XFRM_MODE_IN_TRIGGER: 2501 return 2; 2502 #endif 2503 case XFRM_MODE_TUNNEL: 2504 case XFRM_MODE_BEET: 2505 case XFRM_MODE_IPTFS: 2506 return 3; 2507 } 2508 return 4; 2509 } 2510 #else 2511 static inline int __xfrm6_state_sort_cmp(const void *p) { return 5; } 2512 static inline int __xfrm6_tmpl_sort_cmp(const void *p) { return 4; } 2513 2514 static inline void 2515 __xfrm6_sort(void **dst, void **src, int n, 2516 int (*cmp)(const void *p), int maxclass) 2517 { 2518 int i; 2519 2520 for (i = 0; i < n; i++) 2521 dst[i] = src[i]; 2522 } 2523 #endif /* CONFIG_IPV6 */ 2524 2525 void 2526 xfrm_tmpl_sort(struct xfrm_tmpl **dst, struct xfrm_tmpl **src, int n, 2527 unsigned short family) 2528 { 2529 int i; 2530 2531 if (family == AF_INET6) 2532 __xfrm6_sort((void **)dst, (void **)src, n, 2533 __xfrm6_tmpl_sort_cmp, 5); 2534 else 2535 for (i = 0; i < n; i++) 2536 dst[i] = src[i]; 2537 } 2538 2539 void 2540 xfrm_state_sort(struct xfrm_state **dst, struct xfrm_state **src, int n, 2541 unsigned short family) 2542 { 2543 int i; 2544 2545 if (family == AF_INET6) 2546 __xfrm6_sort((void **)dst, (void **)src, n, 2547 __xfrm6_state_sort_cmp, 6); 2548 else 2549 for (i = 0; i < n; i++) 2550 dst[i] = src[i]; 2551 } 2552 #endif 2553 2554 /* Silly enough, but I'm lazy to build resolution list */ 2555 2556 static struct xfrm_state *__xfrm_find_acq_byseq(struct net *net, u32 mark, u32 seq, u32 pcpu_num) 2557 { 2558 unsigned int h = xfrm_seq_hash(net, seq); 2559 struct xfrm_state *x; 2560 2561 hlist_for_each_entry(x, xfrm_state_deref_prot(net->xfrm.state_byseq, net) + h, byseq) { 2562 if (x->km.seq == seq && 2563 (mark & x->mark.m) == x->mark.v && 2564 x->pcpu_num == pcpu_num && 2565 x->km.state == XFRM_STATE_ACQ) { 2566 xfrm_state_hold(x); 2567 return x; 2568 } 2569 } 2570 2571 return NULL; 2572 } 2573 2574 struct xfrm_state *xfrm_find_acq_byseq(struct net *net, u32 mark, u32 seq, u32 pcpu_num) 2575 { 2576 struct xfrm_state *x; 2577 2578 spin_lock_bh(&net->xfrm.xfrm_state_lock); 2579 x = __xfrm_find_acq_byseq(net, mark, seq, pcpu_num); 2580 spin_unlock_bh(&net->xfrm.xfrm_state_lock); 2581 return x; 2582 } 2583 EXPORT_SYMBOL(xfrm_find_acq_byseq); 2584 2585 u32 xfrm_get_acqseq(void) 2586 { 2587 u32 res; 2588 static atomic_t acqseq; 2589 2590 do { 2591 res = atomic_inc_return(&acqseq); 2592 } while (!res); 2593 2594 return res; 2595 } 2596 EXPORT_SYMBOL(xfrm_get_acqseq); 2597 2598 int verify_spi_info(u8 proto, u32 min, u32 max, struct netlink_ext_ack *extack) 2599 { 2600 switch (proto) { 2601 case IPPROTO_AH: 2602 case IPPROTO_ESP: 2603 break; 2604 2605 case IPPROTO_COMP: 2606 /* IPCOMP spi is 16-bits. */ 2607 if (max >= 0x10000) { 2608 NL_SET_ERR_MSG(extack, "IPCOMP SPI must be <= 65535"); 2609 return -EINVAL; 2610 } 2611 break; 2612 2613 default: 2614 NL_SET_ERR_MSG(extack, "Invalid protocol, must be one of AH, ESP, IPCOMP"); 2615 return -EINVAL; 2616 } 2617 2618 if (min > max) { 2619 NL_SET_ERR_MSG(extack, "Invalid SPI range: min > max"); 2620 return -EINVAL; 2621 } 2622 2623 return 0; 2624 } 2625 EXPORT_SYMBOL(verify_spi_info); 2626 2627 int xfrm_alloc_spi(struct xfrm_state *x, u32 low, u32 high, 2628 struct netlink_ext_ack *extack) 2629 { 2630 struct net *net = xs_net(x); 2631 unsigned int h; 2632 struct xfrm_state *x0; 2633 int err = -ENOENT; 2634 u32 range = high - low + 1; 2635 __be32 newspi = 0; 2636 2637 spin_lock_bh(&x->lock); 2638 if (x->km.state == XFRM_STATE_DEAD) { 2639 NL_SET_ERR_MSG(extack, "Target ACQUIRE is in DEAD state"); 2640 goto unlock; 2641 } 2642 2643 err = 0; 2644 if (x->id.spi) 2645 goto unlock; 2646 2647 err = -ENOENT; 2648 2649 for (h = 0; h < range; h++) { 2650 u32 spi = (low == high) ? low : get_random_u32_inclusive(low, high); 2651 if (spi == 0) 2652 goto next; 2653 newspi = htonl(spi); 2654 2655 spin_lock_bh(&net->xfrm.xfrm_state_lock); 2656 x0 = xfrm_state_lookup_spi_proto(net, newspi, x->id.proto); 2657 if (!x0) { 2658 x->id.spi = newspi; 2659 h = xfrm_spi_hash(net, &x->id.daddr, newspi, x->id.proto, x->props.family); 2660 XFRM_STATE_INSERT(byspi, &x->byspi, 2661 xfrm_state_deref_prot(net->xfrm.state_byspi, net) + h, 2662 x->xso.type); 2663 spin_unlock_bh(&net->xfrm.xfrm_state_lock); 2664 err = 0; 2665 goto unlock; 2666 } 2667 spin_unlock_bh(&net->xfrm.xfrm_state_lock); 2668 2669 next: 2670 if (signal_pending(current)) { 2671 err = -ERESTARTSYS; 2672 goto unlock; 2673 } 2674 2675 if (low == high) 2676 break; 2677 } 2678 2679 if (err) 2680 NL_SET_ERR_MSG(extack, "No SPI available in the requested range"); 2681 2682 unlock: 2683 spin_unlock_bh(&x->lock); 2684 2685 return err; 2686 } 2687 EXPORT_SYMBOL(xfrm_alloc_spi); 2688 2689 static bool __xfrm_state_filter_match(struct xfrm_state *x, 2690 struct xfrm_address_filter *filter) 2691 { 2692 if (filter) { 2693 if ((filter->family == AF_INET || 2694 filter->family == AF_INET6) && 2695 x->props.family != filter->family) 2696 return false; 2697 2698 return addr_match(&x->props.saddr, &filter->saddr, 2699 filter->splen) && 2700 addr_match(&x->id.daddr, &filter->daddr, 2701 filter->dplen); 2702 } 2703 return true; 2704 } 2705 2706 int xfrm_state_walk(struct net *net, struct xfrm_state_walk *walk, 2707 int (*func)(struct xfrm_state *, int, void*), 2708 void *data) 2709 { 2710 struct xfrm_state *state; 2711 struct xfrm_state_walk *x; 2712 int err = 0; 2713 2714 if (walk->seq != 0 && list_empty(&walk->all)) 2715 return 0; 2716 2717 spin_lock_bh(&net->xfrm.xfrm_state_lock); 2718 if (list_empty(&walk->all)) 2719 x = list_first_entry(&net->xfrm.state_all, struct xfrm_state_walk, all); 2720 else 2721 x = list_first_entry(&walk->all, struct xfrm_state_walk, all); 2722 list_for_each_entry_from(x, &net->xfrm.state_all, all) { 2723 if (x->state == XFRM_STATE_DEAD) 2724 continue; 2725 state = container_of(x, struct xfrm_state, km); 2726 if (!xfrm_id_proto_match(state->id.proto, walk->proto)) 2727 continue; 2728 if (!__xfrm_state_filter_match(state, walk->filter)) 2729 continue; 2730 err = func(state, walk->seq, data); 2731 if (err) { 2732 list_move_tail(&walk->all, &x->all); 2733 goto out; 2734 } 2735 walk->seq++; 2736 } 2737 if (walk->seq == 0) { 2738 err = -ENOENT; 2739 goto out; 2740 } 2741 list_del_init(&walk->all); 2742 out: 2743 spin_unlock_bh(&net->xfrm.xfrm_state_lock); 2744 return err; 2745 } 2746 EXPORT_SYMBOL(xfrm_state_walk); 2747 2748 void xfrm_state_walk_init(struct xfrm_state_walk *walk, u8 proto, 2749 struct xfrm_address_filter *filter) 2750 { 2751 INIT_LIST_HEAD(&walk->all); 2752 walk->proto = proto; 2753 walk->state = XFRM_STATE_DEAD; 2754 walk->seq = 0; 2755 walk->filter = filter; 2756 } 2757 EXPORT_SYMBOL(xfrm_state_walk_init); 2758 2759 void xfrm_state_walk_done(struct xfrm_state_walk *walk, struct net *net) 2760 { 2761 kfree(walk->filter); 2762 2763 if (list_empty(&walk->all)) 2764 return; 2765 2766 spin_lock_bh(&net->xfrm.xfrm_state_lock); 2767 list_del(&walk->all); 2768 spin_unlock_bh(&net->xfrm.xfrm_state_lock); 2769 } 2770 EXPORT_SYMBOL(xfrm_state_walk_done); 2771 2772 static void xfrm_replay_timer_handler(struct timer_list *t) 2773 { 2774 struct xfrm_state *x = timer_container_of(x, t, rtimer); 2775 2776 spin_lock(&x->lock); 2777 2778 if (x->km.state == XFRM_STATE_VALID) { 2779 if (xfrm_aevent_is_on(xs_net(x))) 2780 xfrm_replay_notify(x, XFRM_REPLAY_TIMEOUT); 2781 else 2782 x->xflags |= XFRM_TIME_DEFER; 2783 } 2784 2785 spin_unlock(&x->lock); 2786 } 2787 2788 static LIST_HEAD(xfrm_km_list); 2789 2790 void km_policy_notify(struct xfrm_policy *xp, int dir, const struct km_event *c) 2791 { 2792 struct xfrm_mgr *km; 2793 2794 rcu_read_lock(); 2795 list_for_each_entry_rcu(km, &xfrm_km_list, list) 2796 if (km->notify_policy) 2797 km->notify_policy(xp, dir, c); 2798 rcu_read_unlock(); 2799 } 2800 2801 void km_state_notify(struct xfrm_state *x, const struct km_event *c) 2802 { 2803 struct xfrm_mgr *km; 2804 rcu_read_lock(); 2805 list_for_each_entry_rcu(km, &xfrm_km_list, list) 2806 if (km->notify) 2807 km->notify(x, c); 2808 rcu_read_unlock(); 2809 } 2810 2811 EXPORT_SYMBOL(km_policy_notify); 2812 EXPORT_SYMBOL(km_state_notify); 2813 2814 void km_state_expired(struct xfrm_state *x, int hard, u32 portid) 2815 { 2816 struct km_event c; 2817 2818 c.data.hard = hard; 2819 c.portid = portid; 2820 c.event = XFRM_MSG_EXPIRE; 2821 km_state_notify(x, &c); 2822 } 2823 2824 EXPORT_SYMBOL(km_state_expired); 2825 /* 2826 * We send to all registered managers regardless of failure 2827 * We are happy with one success 2828 */ 2829 int km_query(struct xfrm_state *x, struct xfrm_tmpl *t, struct xfrm_policy *pol) 2830 { 2831 int err = -EINVAL, acqret; 2832 struct xfrm_mgr *km; 2833 2834 rcu_read_lock(); 2835 list_for_each_entry_rcu(km, &xfrm_km_list, list) { 2836 acqret = km->acquire(x, t, pol); 2837 if (!acqret) 2838 err = acqret; 2839 } 2840 rcu_read_unlock(); 2841 return err; 2842 } 2843 EXPORT_SYMBOL(km_query); 2844 2845 static int __km_new_mapping(struct xfrm_state *x, xfrm_address_t *ipaddr, __be16 sport) 2846 { 2847 int err = -EINVAL; 2848 struct xfrm_mgr *km; 2849 2850 rcu_read_lock(); 2851 list_for_each_entry_rcu(km, &xfrm_km_list, list) { 2852 if (km->new_mapping) 2853 err = km->new_mapping(x, ipaddr, sport); 2854 if (!err) 2855 break; 2856 } 2857 rcu_read_unlock(); 2858 return err; 2859 } 2860 2861 int km_new_mapping(struct xfrm_state *x, xfrm_address_t *ipaddr, __be16 sport) 2862 { 2863 int ret = 0; 2864 2865 if (x->mapping_maxage) { 2866 if ((jiffies / HZ - x->new_mapping) > x->mapping_maxage || 2867 x->new_mapping_sport != sport) { 2868 x->new_mapping_sport = sport; 2869 x->new_mapping = jiffies / HZ; 2870 ret = __km_new_mapping(x, ipaddr, sport); 2871 } 2872 } else { 2873 ret = __km_new_mapping(x, ipaddr, sport); 2874 } 2875 2876 return ret; 2877 } 2878 EXPORT_SYMBOL(km_new_mapping); 2879 2880 void km_policy_expired(struct xfrm_policy *pol, int dir, int hard, u32 portid) 2881 { 2882 struct km_event c; 2883 2884 c.data.hard = hard; 2885 c.portid = portid; 2886 c.event = XFRM_MSG_POLEXPIRE; 2887 km_policy_notify(pol, dir, &c); 2888 } 2889 EXPORT_SYMBOL(km_policy_expired); 2890 2891 #ifdef CONFIG_XFRM_MIGRATE 2892 int km_migrate(const struct xfrm_selector *sel, u8 dir, u8 type, 2893 const struct xfrm_migrate *m, int num_migrate, 2894 const struct xfrm_kmaddress *k, struct net *net, 2895 const struct xfrm_encap_tmpl *encap) 2896 { 2897 int err = -EINVAL; 2898 int ret; 2899 struct xfrm_mgr *km; 2900 2901 rcu_read_lock(); 2902 list_for_each_entry_rcu(km, &xfrm_km_list, list) { 2903 if (km->migrate) { 2904 ret = km->migrate(sel, dir, type, m, num_migrate, k, 2905 net, encap); 2906 if (!ret) 2907 err = ret; 2908 } 2909 } 2910 rcu_read_unlock(); 2911 return err; 2912 } 2913 EXPORT_SYMBOL(km_migrate); 2914 #endif 2915 2916 int km_report(struct net *net, u8 proto, struct xfrm_selector *sel, xfrm_address_t *addr) 2917 { 2918 int err = -EINVAL; 2919 int ret; 2920 struct xfrm_mgr *km; 2921 2922 rcu_read_lock(); 2923 list_for_each_entry_rcu(km, &xfrm_km_list, list) { 2924 if (km->report) { 2925 ret = km->report(net, proto, sel, addr); 2926 if (!ret) 2927 err = ret; 2928 } 2929 } 2930 rcu_read_unlock(); 2931 return err; 2932 } 2933 EXPORT_SYMBOL(km_report); 2934 2935 static bool km_is_alive(const struct km_event *c) 2936 { 2937 struct xfrm_mgr *km; 2938 bool is_alive = false; 2939 2940 rcu_read_lock(); 2941 list_for_each_entry_rcu(km, &xfrm_km_list, list) { 2942 if (km->is_alive && km->is_alive(c)) { 2943 is_alive = true; 2944 break; 2945 } 2946 } 2947 rcu_read_unlock(); 2948 2949 return is_alive; 2950 } 2951 2952 #if IS_ENABLED(CONFIG_XFRM_USER_COMPAT) 2953 static DEFINE_SPINLOCK(xfrm_translator_lock); 2954 static struct xfrm_translator __rcu *xfrm_translator; 2955 2956 struct xfrm_translator *xfrm_get_translator(void) 2957 { 2958 struct xfrm_translator *xtr; 2959 2960 rcu_read_lock(); 2961 xtr = rcu_dereference(xfrm_translator); 2962 if (unlikely(!xtr)) 2963 goto out; 2964 if (!try_module_get(xtr->owner)) 2965 xtr = NULL; 2966 out: 2967 rcu_read_unlock(); 2968 return xtr; 2969 } 2970 EXPORT_SYMBOL_GPL(xfrm_get_translator); 2971 2972 void xfrm_put_translator(struct xfrm_translator *xtr) 2973 { 2974 module_put(xtr->owner); 2975 } 2976 EXPORT_SYMBOL_GPL(xfrm_put_translator); 2977 2978 int xfrm_register_translator(struct xfrm_translator *xtr) 2979 { 2980 int err = 0; 2981 2982 spin_lock_bh(&xfrm_translator_lock); 2983 if (unlikely(xfrm_translator != NULL)) 2984 err = -EEXIST; 2985 else 2986 rcu_assign_pointer(xfrm_translator, xtr); 2987 spin_unlock_bh(&xfrm_translator_lock); 2988 2989 return err; 2990 } 2991 EXPORT_SYMBOL_GPL(xfrm_register_translator); 2992 2993 int xfrm_unregister_translator(struct xfrm_translator *xtr) 2994 { 2995 int err = 0; 2996 2997 spin_lock_bh(&xfrm_translator_lock); 2998 if (likely(xfrm_translator != NULL)) { 2999 if (rcu_access_pointer(xfrm_translator) != xtr) 3000 err = -EINVAL; 3001 else 3002 RCU_INIT_POINTER(xfrm_translator, NULL); 3003 } 3004 spin_unlock_bh(&xfrm_translator_lock); 3005 synchronize_rcu(); 3006 3007 return err; 3008 } 3009 EXPORT_SYMBOL_GPL(xfrm_unregister_translator); 3010 #endif 3011 3012 int xfrm_user_policy(struct sock *sk, int optname, sockptr_t optval, int optlen) 3013 { 3014 int err; 3015 u8 *data; 3016 struct xfrm_mgr *km; 3017 struct xfrm_policy *pol = NULL; 3018 3019 if (sockptr_is_null(optval) && !optlen) { 3020 xfrm_sk_policy_insert(sk, XFRM_POLICY_IN, NULL); 3021 xfrm_sk_policy_insert(sk, XFRM_POLICY_OUT, NULL); 3022 sk_dst_reset(sk); 3023 return 0; 3024 } 3025 3026 if (optlen <= 0 || optlen > PAGE_SIZE) 3027 return -EMSGSIZE; 3028 3029 data = memdup_sockptr(optval, optlen); 3030 if (IS_ERR(data)) 3031 return PTR_ERR(data); 3032 3033 if (IS_ENABLED(CONFIG_COMPAT_FOR_U64_ALIGNMENT) && in_compat_syscall()) { 3034 struct xfrm_translator *xtr = xfrm_get_translator(); 3035 3036 if (!xtr) { 3037 kfree(data); 3038 return -EOPNOTSUPP; 3039 } 3040 3041 err = xtr->xlate_user_policy_sockptr(&data, optlen); 3042 xfrm_put_translator(xtr); 3043 if (err) { 3044 kfree(data); 3045 return err; 3046 } 3047 } 3048 3049 err = -EINVAL; 3050 rcu_read_lock(); 3051 list_for_each_entry_rcu(km, &xfrm_km_list, list) { 3052 pol = km->compile_policy(sk, optname, data, 3053 optlen, &err); 3054 if (err >= 0) 3055 break; 3056 } 3057 rcu_read_unlock(); 3058 3059 if (err >= 0) { 3060 xfrm_sk_policy_insert(sk, err, pol); 3061 xfrm_pol_put(pol); 3062 sk_dst_reset(sk); 3063 err = 0; 3064 } 3065 3066 kfree(data); 3067 return err; 3068 } 3069 EXPORT_SYMBOL(xfrm_user_policy); 3070 3071 static DEFINE_SPINLOCK(xfrm_km_lock); 3072 3073 void xfrm_register_km(struct xfrm_mgr *km) 3074 { 3075 spin_lock_bh(&xfrm_km_lock); 3076 list_add_tail_rcu(&km->list, &xfrm_km_list); 3077 spin_unlock_bh(&xfrm_km_lock); 3078 } 3079 EXPORT_SYMBOL(xfrm_register_km); 3080 3081 void xfrm_unregister_km(struct xfrm_mgr *km) 3082 { 3083 spin_lock_bh(&xfrm_km_lock); 3084 list_del_rcu(&km->list); 3085 spin_unlock_bh(&xfrm_km_lock); 3086 synchronize_rcu(); 3087 } 3088 EXPORT_SYMBOL(xfrm_unregister_km); 3089 3090 int xfrm_state_register_afinfo(struct xfrm_state_afinfo *afinfo) 3091 { 3092 int err = 0; 3093 3094 if (WARN_ON(afinfo->family >= NPROTO)) 3095 return -EAFNOSUPPORT; 3096 3097 spin_lock_bh(&xfrm_state_afinfo_lock); 3098 if (unlikely(xfrm_state_afinfo[afinfo->family] != NULL)) 3099 err = -EEXIST; 3100 else 3101 rcu_assign_pointer(xfrm_state_afinfo[afinfo->family], afinfo); 3102 spin_unlock_bh(&xfrm_state_afinfo_lock); 3103 return err; 3104 } 3105 EXPORT_SYMBOL(xfrm_state_register_afinfo); 3106 3107 int xfrm_state_unregister_afinfo(struct xfrm_state_afinfo *afinfo) 3108 { 3109 int err = 0, family = afinfo->family; 3110 3111 if (WARN_ON(family >= NPROTO)) 3112 return -EAFNOSUPPORT; 3113 3114 spin_lock_bh(&xfrm_state_afinfo_lock); 3115 if (likely(xfrm_state_afinfo[afinfo->family] != NULL)) { 3116 if (rcu_access_pointer(xfrm_state_afinfo[family]) != afinfo) 3117 err = -EINVAL; 3118 else 3119 RCU_INIT_POINTER(xfrm_state_afinfo[afinfo->family], NULL); 3120 } 3121 spin_unlock_bh(&xfrm_state_afinfo_lock); 3122 synchronize_rcu(); 3123 return err; 3124 } 3125 EXPORT_SYMBOL(xfrm_state_unregister_afinfo); 3126 3127 struct xfrm_state_afinfo *xfrm_state_afinfo_get_rcu(unsigned int family) 3128 { 3129 if (unlikely(family >= NPROTO)) 3130 return NULL; 3131 3132 return rcu_dereference(xfrm_state_afinfo[family]); 3133 } 3134 EXPORT_SYMBOL_GPL(xfrm_state_afinfo_get_rcu); 3135 3136 struct xfrm_state_afinfo *xfrm_state_get_afinfo(unsigned int family) 3137 { 3138 struct xfrm_state_afinfo *afinfo; 3139 if (unlikely(family >= NPROTO)) 3140 return NULL; 3141 rcu_read_lock(); 3142 afinfo = rcu_dereference(xfrm_state_afinfo[family]); 3143 if (unlikely(!afinfo)) 3144 rcu_read_unlock(); 3145 return afinfo; 3146 } 3147 3148 void xfrm_flush_gc(void) 3149 { 3150 flush_work(&xfrm_state_gc_work); 3151 } 3152 EXPORT_SYMBOL(xfrm_flush_gc); 3153 3154 static void xfrm_state_delete_tunnel(struct xfrm_state *x) 3155 { 3156 if (x->tunnel) { 3157 struct xfrm_state *t = x->tunnel; 3158 3159 if (atomic_dec_return(&t->tunnel_users) == 1) 3160 xfrm_state_delete(t); 3161 xfrm_state_put(t); 3162 x->tunnel = NULL; 3163 } 3164 } 3165 3166 u32 xfrm_state_mtu(struct xfrm_state *x, int mtu) 3167 { 3168 const struct xfrm_type *type = READ_ONCE(x->type); 3169 struct crypto_aead *aead; 3170 u32 blksize, net_adj = 0; 3171 u32 overhead, payload_mtu; 3172 3173 if (x->km.state != XFRM_STATE_VALID || 3174 !type || type->proto != IPPROTO_ESP) { 3175 if (mtu <= x->props.header_len) 3176 return 1; 3177 return mtu - x->props.header_len; 3178 } 3179 3180 aead = x->data; 3181 blksize = ALIGN(crypto_aead_blocksize(aead), 4); 3182 3183 switch (x->props.mode) { 3184 case XFRM_MODE_TRANSPORT: 3185 case XFRM_MODE_BEET: 3186 if (x->props.family == AF_INET) 3187 net_adj = sizeof(struct iphdr); 3188 else if (x->props.family == AF_INET6) 3189 net_adj = sizeof(struct ipv6hdr); 3190 break; 3191 case XFRM_MODE_TUNNEL: 3192 break; 3193 default: 3194 if (x->mode_cbs && x->mode_cbs->get_inner_mtu) 3195 return x->mode_cbs->get_inner_mtu(x, mtu); 3196 3197 WARN_ON_ONCE(1); 3198 break; 3199 } 3200 3201 overhead = x->props.header_len + crypto_aead_authsize(aead) + net_adj; 3202 if (mtu <= overhead) 3203 return 1; 3204 3205 payload_mtu = mtu - overhead; 3206 payload_mtu &= ~(blksize - 1); 3207 if (payload_mtu <= 2) 3208 return 1; 3209 3210 return payload_mtu + net_adj - 2; 3211 3212 } 3213 EXPORT_SYMBOL_GPL(xfrm_state_mtu); 3214 3215 int __xfrm_init_state(struct xfrm_state *x, struct netlink_ext_ack *extack) 3216 { 3217 const struct xfrm_mode *inner_mode; 3218 const struct xfrm_mode *outer_mode; 3219 int family = x->props.family; 3220 int err; 3221 3222 if (family == AF_INET && 3223 (!x->dir || x->dir == XFRM_SA_DIR_OUT) && 3224 READ_ONCE(xs_net(x)->ipv4.sysctl_ip_no_pmtu_disc)) 3225 x->props.flags |= XFRM_STATE_NOPMTUDISC; 3226 3227 err = -EPROTONOSUPPORT; 3228 3229 if (x->sel.family != AF_UNSPEC) { 3230 inner_mode = xfrm_get_mode(x->props.mode, x->sel.family); 3231 if (inner_mode == NULL) { 3232 NL_SET_ERR_MSG(extack, "Requested mode not found"); 3233 goto error; 3234 } 3235 3236 if (!(inner_mode->flags & XFRM_MODE_FLAG_TUNNEL) && 3237 family != x->sel.family) { 3238 NL_SET_ERR_MSG(extack, "Only tunnel modes can accommodate a change of family"); 3239 goto error; 3240 } 3241 3242 x->inner_mode = *inner_mode; 3243 } else { 3244 const struct xfrm_mode *inner_mode_iaf; 3245 int iafamily = AF_INET; 3246 3247 inner_mode = xfrm_get_mode(x->props.mode, x->props.family); 3248 if (inner_mode == NULL) { 3249 NL_SET_ERR_MSG(extack, "Requested mode not found"); 3250 goto error; 3251 } 3252 3253 x->inner_mode = *inner_mode; 3254 3255 if (x->props.family == AF_INET) 3256 iafamily = AF_INET6; 3257 3258 inner_mode_iaf = xfrm_get_mode(x->props.mode, iafamily); 3259 if (inner_mode_iaf) { 3260 if (inner_mode_iaf->flags & XFRM_MODE_FLAG_TUNNEL) 3261 x->inner_mode_iaf = *inner_mode_iaf; 3262 } 3263 } 3264 3265 x->type = xfrm_get_type(x->id.proto, family); 3266 if (x->type == NULL) { 3267 NL_SET_ERR_MSG(extack, "Requested type not found"); 3268 goto error; 3269 } 3270 3271 err = x->type->init_state(x, extack); 3272 if (err) 3273 goto error; 3274 3275 outer_mode = xfrm_get_mode(x->props.mode, family); 3276 if (!outer_mode) { 3277 NL_SET_ERR_MSG(extack, "Requested mode not found"); 3278 err = -EPROTONOSUPPORT; 3279 goto error; 3280 } 3281 3282 x->outer_mode = *outer_mode; 3283 if (x->nat_keepalive_interval) { 3284 if (x->dir != XFRM_SA_DIR_OUT) { 3285 NL_SET_ERR_MSG(extack, "NAT keepalive is only supported for outbound SAs"); 3286 err = -EINVAL; 3287 goto error; 3288 } 3289 3290 if (!x->encap || x->encap->encap_type != UDP_ENCAP_ESPINUDP) { 3291 NL_SET_ERR_MSG(extack, 3292 "NAT keepalive is only supported for UDP encapsulation"); 3293 err = -EINVAL; 3294 goto error; 3295 } 3296 } 3297 3298 x->mode_cbs = xfrm_get_mode_cbs(x->props.mode); 3299 if (x->mode_cbs) { 3300 if (x->mode_cbs->init_state) 3301 err = x->mode_cbs->init_state(x); 3302 module_put(x->mode_cbs->owner); 3303 if (err && !x->mode_data) 3304 x->mode_cbs = NULL; 3305 } 3306 error: 3307 return err; 3308 } 3309 3310 EXPORT_SYMBOL(__xfrm_init_state); 3311 3312 int xfrm_init_state(struct xfrm_state *x, struct netlink_ext_ack *extack) 3313 { 3314 int err; 3315 3316 err = __xfrm_init_state(x, extack); 3317 if (err) 3318 return err; 3319 3320 err = xfrm_init_replay(x, NULL); 3321 if (err) 3322 return err; 3323 3324 x->km.state = XFRM_STATE_VALID; 3325 return 0; 3326 } 3327 3328 EXPORT_SYMBOL(xfrm_init_state); 3329 3330 int __net_init xfrm_state_init(struct net *net) 3331 { 3332 struct hlist_head *ndst, *nsrc, *nspi, *nseq; 3333 unsigned int sz; 3334 3335 if (net_eq(net, &init_net)) 3336 xfrm_state_cache = KMEM_CACHE(xfrm_state, 3337 SLAB_HWCACHE_ALIGN | SLAB_PANIC); 3338 3339 INIT_LIST_HEAD(&net->xfrm.state_all); 3340 3341 sz = sizeof(struct hlist_head) * 8; 3342 3343 ndst = xfrm_hash_alloc(sz); 3344 if (!ndst) 3345 goto out_bydst; 3346 rcu_assign_pointer(net->xfrm.state_bydst, ndst); 3347 3348 nsrc = xfrm_hash_alloc(sz); 3349 if (!nsrc) 3350 goto out_bysrc; 3351 rcu_assign_pointer(net->xfrm.state_bysrc, nsrc); 3352 3353 nspi = xfrm_hash_alloc(sz); 3354 if (!nspi) 3355 goto out_byspi; 3356 rcu_assign_pointer(net->xfrm.state_byspi, nspi); 3357 3358 nseq = xfrm_hash_alloc(sz); 3359 if (!nseq) 3360 goto out_byseq; 3361 rcu_assign_pointer(net->xfrm.state_byseq, nseq); 3362 3363 net->xfrm.state_cache_input = alloc_percpu(struct hlist_head); 3364 if (!net->xfrm.state_cache_input) 3365 goto out_state_cache_input; 3366 3367 net->xfrm.state_hmask = ((sz / sizeof(struct hlist_head)) - 1); 3368 3369 net->xfrm.state_num = 0; 3370 INIT_WORK(&net->xfrm.state_hash_work, xfrm_hash_resize); 3371 spin_lock_init(&net->xfrm.xfrm_state_lock); 3372 seqcount_spinlock_init(&net->xfrm.xfrm_state_hash_generation, 3373 &net->xfrm.xfrm_state_lock); 3374 return 0; 3375 3376 out_state_cache_input: 3377 xfrm_hash_free(nseq, sz); 3378 out_byseq: 3379 xfrm_hash_free(nspi, sz); 3380 out_byspi: 3381 xfrm_hash_free(nsrc, sz); 3382 out_bysrc: 3383 xfrm_hash_free(ndst, sz); 3384 out_bydst: 3385 return -ENOMEM; 3386 } 3387 3388 #define xfrm_state_deref_netexit(table) \ 3389 rcu_dereference_protected((table), true /* netns is going away */) 3390 void xfrm_state_fini(struct net *net) 3391 { 3392 unsigned int sz; 3393 int i; 3394 3395 flush_work(&net->xfrm.state_hash_work); 3396 xfrm_state_flush(net, 0, false); 3397 flush_work(&xfrm_state_gc_work); 3398 3399 WARN_ON(!list_empty(&net->xfrm.state_all)); 3400 3401 for (i = 0; i <= net->xfrm.state_hmask; i++) { 3402 WARN_ON(!hlist_empty(xfrm_state_deref_netexit(net->xfrm.state_byseq) + i)); 3403 WARN_ON(!hlist_empty(xfrm_state_deref_netexit(net->xfrm.state_byspi) + i)); 3404 WARN_ON(!hlist_empty(xfrm_state_deref_netexit(net->xfrm.state_bysrc) + i)); 3405 WARN_ON(!hlist_empty(xfrm_state_deref_netexit(net->xfrm.state_bydst) + i)); 3406 } 3407 3408 sz = (net->xfrm.state_hmask + 1) * sizeof(struct hlist_head); 3409 xfrm_hash_free(xfrm_state_deref_netexit(net->xfrm.state_byseq), sz); 3410 xfrm_hash_free(xfrm_state_deref_netexit(net->xfrm.state_byspi), sz); 3411 xfrm_hash_free(xfrm_state_deref_netexit(net->xfrm.state_bysrc), sz); 3412 xfrm_hash_free(xfrm_state_deref_netexit(net->xfrm.state_bydst), sz); 3413 free_percpu(net->xfrm.state_cache_input); 3414 } 3415 3416 #ifdef CONFIG_AUDITSYSCALL 3417 static void xfrm_audit_helper_sainfo(struct xfrm_state *x, 3418 struct audit_buffer *audit_buf) 3419 { 3420 struct xfrm_sec_ctx *ctx = x->security; 3421 u32 spi = ntohl(x->id.spi); 3422 3423 if (ctx) 3424 audit_log_format(audit_buf, " sec_alg=%u sec_doi=%u sec_obj=%s", 3425 ctx->ctx_alg, ctx->ctx_doi, ctx->ctx_str); 3426 3427 switch (x->props.family) { 3428 case AF_INET: 3429 audit_log_format(audit_buf, " src=%pI4 dst=%pI4", 3430 &x->props.saddr.a4, &x->id.daddr.a4); 3431 break; 3432 case AF_INET6: 3433 audit_log_format(audit_buf, " src=%pI6 dst=%pI6", 3434 x->props.saddr.a6, x->id.daddr.a6); 3435 break; 3436 } 3437 3438 audit_log_format(audit_buf, " spi=%u(0x%x)", spi, spi); 3439 } 3440 3441 static void xfrm_audit_helper_pktinfo(struct sk_buff *skb, u16 family, 3442 struct audit_buffer *audit_buf) 3443 { 3444 const struct iphdr *iph4; 3445 const struct ipv6hdr *iph6; 3446 3447 switch (family) { 3448 case AF_INET: 3449 iph4 = ip_hdr(skb); 3450 audit_log_format(audit_buf, " src=%pI4 dst=%pI4", 3451 &iph4->saddr, &iph4->daddr); 3452 break; 3453 case AF_INET6: 3454 iph6 = ipv6_hdr(skb); 3455 audit_log_format(audit_buf, 3456 " src=%pI6 dst=%pI6 flowlbl=0x%x%02x%02x", 3457 &iph6->saddr, &iph6->daddr, 3458 iph6->flow_lbl[0] & 0x0f, 3459 iph6->flow_lbl[1], 3460 iph6->flow_lbl[2]); 3461 break; 3462 } 3463 } 3464 3465 void xfrm_audit_state_add(struct xfrm_state *x, int result, bool task_valid) 3466 { 3467 struct audit_buffer *audit_buf; 3468 3469 audit_buf = xfrm_audit_start("SAD-add"); 3470 if (audit_buf == NULL) 3471 return; 3472 xfrm_audit_helper_usrinfo(task_valid, audit_buf); 3473 xfrm_audit_helper_sainfo(x, audit_buf); 3474 audit_log_format(audit_buf, " res=%u", result); 3475 audit_log_end(audit_buf); 3476 } 3477 EXPORT_SYMBOL_GPL(xfrm_audit_state_add); 3478 3479 void xfrm_audit_state_delete(struct xfrm_state *x, int result, bool task_valid) 3480 { 3481 struct audit_buffer *audit_buf; 3482 3483 audit_buf = xfrm_audit_start("SAD-delete"); 3484 if (audit_buf == NULL) 3485 return; 3486 xfrm_audit_helper_usrinfo(task_valid, audit_buf); 3487 xfrm_audit_helper_sainfo(x, audit_buf); 3488 audit_log_format(audit_buf, " res=%u", result); 3489 audit_log_end(audit_buf); 3490 } 3491 EXPORT_SYMBOL_GPL(xfrm_audit_state_delete); 3492 3493 void xfrm_audit_state_replay_overflow(struct xfrm_state *x, 3494 struct sk_buff *skb) 3495 { 3496 struct audit_buffer *audit_buf; 3497 u32 spi; 3498 3499 audit_buf = xfrm_audit_start("SA-replay-overflow"); 3500 if (audit_buf == NULL) 3501 return; 3502 xfrm_audit_helper_pktinfo(skb, x->props.family, audit_buf); 3503 /* don't record the sequence number because it's inherent in this kind 3504 * of audit message */ 3505 spi = ntohl(x->id.spi); 3506 audit_log_format(audit_buf, " spi=%u(0x%x)", spi, spi); 3507 audit_log_end(audit_buf); 3508 } 3509 EXPORT_SYMBOL_GPL(xfrm_audit_state_replay_overflow); 3510 3511 void xfrm_audit_state_replay(struct xfrm_state *x, 3512 struct sk_buff *skb, __be32 net_seq) 3513 { 3514 struct audit_buffer *audit_buf; 3515 u32 spi; 3516 3517 audit_buf = xfrm_audit_start("SA-replayed-pkt"); 3518 if (audit_buf == NULL) 3519 return; 3520 xfrm_audit_helper_pktinfo(skb, x->props.family, audit_buf); 3521 spi = ntohl(x->id.spi); 3522 audit_log_format(audit_buf, " spi=%u(0x%x) seqno=%u", 3523 spi, spi, ntohl(net_seq)); 3524 audit_log_end(audit_buf); 3525 } 3526 EXPORT_SYMBOL_GPL(xfrm_audit_state_replay); 3527 3528 void xfrm_audit_state_notfound_simple(struct sk_buff *skb, u16 family) 3529 { 3530 struct audit_buffer *audit_buf; 3531 3532 audit_buf = xfrm_audit_start("SA-notfound"); 3533 if (audit_buf == NULL) 3534 return; 3535 xfrm_audit_helper_pktinfo(skb, family, audit_buf); 3536 audit_log_end(audit_buf); 3537 } 3538 EXPORT_SYMBOL_GPL(xfrm_audit_state_notfound_simple); 3539 3540 void xfrm_audit_state_notfound(struct sk_buff *skb, u16 family, 3541 __be32 net_spi, __be32 net_seq) 3542 { 3543 struct audit_buffer *audit_buf; 3544 u32 spi; 3545 3546 audit_buf = xfrm_audit_start("SA-notfound"); 3547 if (audit_buf == NULL) 3548 return; 3549 xfrm_audit_helper_pktinfo(skb, family, audit_buf); 3550 spi = ntohl(net_spi); 3551 audit_log_format(audit_buf, " spi=%u(0x%x) seqno=%u", 3552 spi, spi, ntohl(net_seq)); 3553 audit_log_end(audit_buf); 3554 } 3555 EXPORT_SYMBOL_GPL(xfrm_audit_state_notfound); 3556 3557 void xfrm_audit_state_icvfail(struct xfrm_state *x, 3558 struct sk_buff *skb, u8 proto) 3559 { 3560 struct audit_buffer *audit_buf; 3561 __be32 net_spi; 3562 __be32 net_seq; 3563 3564 audit_buf = xfrm_audit_start("SA-icv-failure"); 3565 if (audit_buf == NULL) 3566 return; 3567 xfrm_audit_helper_pktinfo(skb, x->props.family, audit_buf); 3568 if (xfrm_parse_spi(skb, proto, &net_spi, &net_seq) == 0) { 3569 u32 spi = ntohl(net_spi); 3570 audit_log_format(audit_buf, " spi=%u(0x%x) seqno=%u", 3571 spi, spi, ntohl(net_seq)); 3572 } 3573 audit_log_end(audit_buf); 3574 } 3575 EXPORT_SYMBOL_GPL(xfrm_audit_state_icvfail); 3576 #endif /* CONFIG_AUDITSYSCALL */ 3577