1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * xfrm_state.c 4 * 5 * Changes: 6 * Mitsuru KANDA @USAGI 7 * Kazunori MIYAZAWA @USAGI 8 * Kunihiro Ishiguro <kunihiro@ipinfusion.com> 9 * IPv6 support 10 * YOSHIFUJI Hideaki @USAGI 11 * Split up af-specific functions 12 * Derek Atkins <derek@ihtfp.com> 13 * Add UDP Encapsulation 14 * 15 */ 16 17 #include <linux/compat.h> 18 #include <linux/workqueue.h> 19 #include <net/xfrm.h> 20 #include <linux/pfkeyv2.h> 21 #include <linux/ipsec.h> 22 #include <linux/module.h> 23 #include <linux/cache.h> 24 #include <linux/audit.h> 25 #include <linux/uaccess.h> 26 #include <linux/ktime.h> 27 #include <linux/slab.h> 28 #include <linux/interrupt.h> 29 #include <linux/kernel.h> 30 31 #include <crypto/aead.h> 32 33 #include "xfrm_hash.h" 34 35 #define xfrm_state_deref_prot(table, net) \ 36 rcu_dereference_protected((table), lockdep_is_held(&(net)->xfrm.xfrm_state_lock)) 37 38 static void xfrm_state_gc_task(struct work_struct *work); 39 40 /* Each xfrm_state may be linked to two tables: 41 42 1. Hash table by (spi,daddr,ah/esp) to find SA by SPI. (input,ctl) 43 2. Hash table by (daddr,family,reqid) to find what SAs exist for given 44 destination/tunnel endpoint. (output) 45 */ 46 47 static unsigned int xfrm_state_hashmax __read_mostly = 1 * 1024 * 1024; 48 static struct kmem_cache *xfrm_state_cache __ro_after_init; 49 50 static DECLARE_WORK(xfrm_state_gc_work, xfrm_state_gc_task); 51 static HLIST_HEAD(xfrm_state_gc_list); 52 static HLIST_HEAD(xfrm_state_dev_gc_list); 53 54 static inline bool xfrm_state_hold_rcu(struct xfrm_state __rcu *x) 55 { 56 return refcount_inc_not_zero(&x->refcnt); 57 } 58 59 static inline unsigned int xfrm_dst_hash(struct net *net, 60 const xfrm_address_t *daddr, 61 const xfrm_address_t *saddr, 62 u32 reqid, 63 unsigned short family) 64 { 65 return __xfrm_dst_hash(daddr, saddr, reqid, family, net->xfrm.state_hmask); 66 } 67 68 static inline unsigned int xfrm_src_hash(struct net *net, 69 const xfrm_address_t *daddr, 70 const xfrm_address_t *saddr, 71 unsigned short family) 72 { 73 return __xfrm_src_hash(daddr, saddr, family, net->xfrm.state_hmask); 74 } 75 76 static inline unsigned int 77 xfrm_spi_hash(struct net *net, const xfrm_address_t *daddr, 78 __be32 spi, u8 proto, unsigned short family) 79 { 80 return __xfrm_spi_hash(daddr, spi, proto, family, net->xfrm.state_hmask); 81 } 82 83 static unsigned int xfrm_seq_hash(struct net *net, u32 seq) 84 { 85 return __xfrm_seq_hash(seq, net->xfrm.state_hmask); 86 } 87 88 #define XFRM_STATE_INSERT(by, _n, _h, _type) \ 89 { \ 90 struct xfrm_state *_x = NULL; \ 91 \ 92 if (_type != XFRM_DEV_OFFLOAD_PACKET) { \ 93 hlist_for_each_entry_rcu(_x, _h, by) { \ 94 if (_x->xso.type == XFRM_DEV_OFFLOAD_PACKET) \ 95 continue; \ 96 break; \ 97 } \ 98 } \ 99 \ 100 if (!_x || _x->xso.type == XFRM_DEV_OFFLOAD_PACKET) \ 101 /* SAD is empty or consist from HW SAs only */ \ 102 hlist_add_head_rcu(_n, _h); \ 103 else \ 104 hlist_add_before_rcu(_n, &_x->by); \ 105 } 106 107 static void xfrm_hash_transfer(struct hlist_head *list, 108 struct hlist_head *ndsttable, 109 struct hlist_head *nsrctable, 110 struct hlist_head *nspitable, 111 struct hlist_head *nseqtable, 112 unsigned int nhashmask) 113 { 114 struct hlist_node *tmp; 115 struct xfrm_state *x; 116 117 hlist_for_each_entry_safe(x, tmp, list, bydst) { 118 unsigned int h; 119 120 h = __xfrm_dst_hash(&x->id.daddr, &x->props.saddr, 121 x->props.reqid, x->props.family, 122 nhashmask); 123 XFRM_STATE_INSERT(bydst, &x->bydst, ndsttable + h, x->xso.type); 124 125 h = __xfrm_src_hash(&x->id.daddr, &x->props.saddr, 126 x->props.family, 127 nhashmask); 128 XFRM_STATE_INSERT(bysrc, &x->bysrc, nsrctable + h, x->xso.type); 129 130 if (x->id.spi) { 131 h = __xfrm_spi_hash(&x->id.daddr, x->id.spi, 132 x->id.proto, x->props.family, 133 nhashmask); 134 XFRM_STATE_INSERT(byspi, &x->byspi, nspitable + h, 135 x->xso.type); 136 } 137 138 if (x->km.seq) { 139 h = __xfrm_seq_hash(x->km.seq, nhashmask); 140 XFRM_STATE_INSERT(byseq, &x->byseq, nseqtable + h, 141 x->xso.type); 142 } 143 } 144 } 145 146 static unsigned long xfrm_hash_new_size(unsigned int state_hmask) 147 { 148 return ((state_hmask + 1) << 1) * sizeof(struct hlist_head); 149 } 150 151 static void xfrm_hash_resize(struct work_struct *work) 152 { 153 struct net *net = container_of(work, struct net, xfrm.state_hash_work); 154 struct hlist_head *ndst, *nsrc, *nspi, *nseq, *odst, *osrc, *ospi, *oseq; 155 unsigned long nsize, osize; 156 unsigned int nhashmask, ohashmask; 157 int i; 158 159 nsize = xfrm_hash_new_size(net->xfrm.state_hmask); 160 ndst = xfrm_hash_alloc(nsize); 161 if (!ndst) 162 return; 163 nsrc = xfrm_hash_alloc(nsize); 164 if (!nsrc) { 165 xfrm_hash_free(ndst, nsize); 166 return; 167 } 168 nspi = xfrm_hash_alloc(nsize); 169 if (!nspi) { 170 xfrm_hash_free(ndst, nsize); 171 xfrm_hash_free(nsrc, nsize); 172 return; 173 } 174 nseq = xfrm_hash_alloc(nsize); 175 if (!nseq) { 176 xfrm_hash_free(ndst, nsize); 177 xfrm_hash_free(nsrc, nsize); 178 xfrm_hash_free(nspi, nsize); 179 return; 180 } 181 182 spin_lock_bh(&net->xfrm.xfrm_state_lock); 183 write_seqcount_begin(&net->xfrm.xfrm_state_hash_generation); 184 185 nhashmask = (nsize / sizeof(struct hlist_head)) - 1U; 186 odst = xfrm_state_deref_prot(net->xfrm.state_bydst, net); 187 for (i = net->xfrm.state_hmask; i >= 0; i--) 188 xfrm_hash_transfer(odst + i, ndst, nsrc, nspi, nseq, nhashmask); 189 190 osrc = xfrm_state_deref_prot(net->xfrm.state_bysrc, net); 191 ospi = xfrm_state_deref_prot(net->xfrm.state_byspi, net); 192 oseq = xfrm_state_deref_prot(net->xfrm.state_byseq, net); 193 ohashmask = net->xfrm.state_hmask; 194 195 rcu_assign_pointer(net->xfrm.state_bydst, ndst); 196 rcu_assign_pointer(net->xfrm.state_bysrc, nsrc); 197 rcu_assign_pointer(net->xfrm.state_byspi, nspi); 198 rcu_assign_pointer(net->xfrm.state_byseq, nseq); 199 net->xfrm.state_hmask = nhashmask; 200 201 write_seqcount_end(&net->xfrm.xfrm_state_hash_generation); 202 spin_unlock_bh(&net->xfrm.xfrm_state_lock); 203 204 osize = (ohashmask + 1) * sizeof(struct hlist_head); 205 206 synchronize_rcu(); 207 208 xfrm_hash_free(odst, osize); 209 xfrm_hash_free(osrc, osize); 210 xfrm_hash_free(ospi, osize); 211 xfrm_hash_free(oseq, osize); 212 } 213 214 static DEFINE_SPINLOCK(xfrm_state_afinfo_lock); 215 static struct xfrm_state_afinfo __rcu *xfrm_state_afinfo[NPROTO]; 216 217 static DEFINE_SPINLOCK(xfrm_state_gc_lock); 218 static DEFINE_SPINLOCK(xfrm_state_dev_gc_lock); 219 220 int __xfrm_state_delete(struct xfrm_state *x); 221 222 int km_query(struct xfrm_state *x, struct xfrm_tmpl *t, struct xfrm_policy *pol); 223 static bool km_is_alive(const struct km_event *c); 224 void km_state_expired(struct xfrm_state *x, int hard, u32 portid); 225 226 int xfrm_register_type(const struct xfrm_type *type, unsigned short family) 227 { 228 struct xfrm_state_afinfo *afinfo = xfrm_state_get_afinfo(family); 229 int err = 0; 230 231 if (!afinfo) 232 return -EAFNOSUPPORT; 233 234 #define X(afi, T, name) do { \ 235 WARN_ON((afi)->type_ ## name); \ 236 (afi)->type_ ## name = (T); \ 237 } while (0) 238 239 switch (type->proto) { 240 case IPPROTO_COMP: 241 X(afinfo, type, comp); 242 break; 243 case IPPROTO_AH: 244 X(afinfo, type, ah); 245 break; 246 case IPPROTO_ESP: 247 X(afinfo, type, esp); 248 break; 249 case IPPROTO_IPIP: 250 X(afinfo, type, ipip); 251 break; 252 case IPPROTO_DSTOPTS: 253 X(afinfo, type, dstopts); 254 break; 255 case IPPROTO_ROUTING: 256 X(afinfo, type, routing); 257 break; 258 case IPPROTO_IPV6: 259 X(afinfo, type, ipip6); 260 break; 261 default: 262 WARN_ON(1); 263 err = -EPROTONOSUPPORT; 264 break; 265 } 266 #undef X 267 rcu_read_unlock(); 268 return err; 269 } 270 EXPORT_SYMBOL(xfrm_register_type); 271 272 void xfrm_unregister_type(const struct xfrm_type *type, unsigned short family) 273 { 274 struct xfrm_state_afinfo *afinfo = xfrm_state_get_afinfo(family); 275 276 if (unlikely(afinfo == NULL)) 277 return; 278 279 #define X(afi, T, name) do { \ 280 WARN_ON((afi)->type_ ## name != (T)); \ 281 (afi)->type_ ## name = NULL; \ 282 } while (0) 283 284 switch (type->proto) { 285 case IPPROTO_COMP: 286 X(afinfo, type, comp); 287 break; 288 case IPPROTO_AH: 289 X(afinfo, type, ah); 290 break; 291 case IPPROTO_ESP: 292 X(afinfo, type, esp); 293 break; 294 case IPPROTO_IPIP: 295 X(afinfo, type, ipip); 296 break; 297 case IPPROTO_DSTOPTS: 298 X(afinfo, type, dstopts); 299 break; 300 case IPPROTO_ROUTING: 301 X(afinfo, type, routing); 302 break; 303 case IPPROTO_IPV6: 304 X(afinfo, type, ipip6); 305 break; 306 default: 307 WARN_ON(1); 308 break; 309 } 310 #undef X 311 rcu_read_unlock(); 312 } 313 EXPORT_SYMBOL(xfrm_unregister_type); 314 315 static const struct xfrm_type *xfrm_get_type(u8 proto, unsigned short family) 316 { 317 const struct xfrm_type *type = NULL; 318 struct xfrm_state_afinfo *afinfo; 319 int modload_attempted = 0; 320 321 retry: 322 afinfo = xfrm_state_get_afinfo(family); 323 if (unlikely(afinfo == NULL)) 324 return NULL; 325 326 switch (proto) { 327 case IPPROTO_COMP: 328 type = afinfo->type_comp; 329 break; 330 case IPPROTO_AH: 331 type = afinfo->type_ah; 332 break; 333 case IPPROTO_ESP: 334 type = afinfo->type_esp; 335 break; 336 case IPPROTO_IPIP: 337 type = afinfo->type_ipip; 338 break; 339 case IPPROTO_DSTOPTS: 340 type = afinfo->type_dstopts; 341 break; 342 case IPPROTO_ROUTING: 343 type = afinfo->type_routing; 344 break; 345 case IPPROTO_IPV6: 346 type = afinfo->type_ipip6; 347 break; 348 default: 349 break; 350 } 351 352 if (unlikely(type && !try_module_get(type->owner))) 353 type = NULL; 354 355 rcu_read_unlock(); 356 357 if (!type && !modload_attempted) { 358 request_module("xfrm-type-%d-%d", family, proto); 359 modload_attempted = 1; 360 goto retry; 361 } 362 363 return type; 364 } 365 366 static void xfrm_put_type(const struct xfrm_type *type) 367 { 368 module_put(type->owner); 369 } 370 371 int xfrm_register_type_offload(const struct xfrm_type_offload *type, 372 unsigned short family) 373 { 374 struct xfrm_state_afinfo *afinfo = xfrm_state_get_afinfo(family); 375 int err = 0; 376 377 if (unlikely(afinfo == NULL)) 378 return -EAFNOSUPPORT; 379 380 switch (type->proto) { 381 case IPPROTO_ESP: 382 WARN_ON(afinfo->type_offload_esp); 383 afinfo->type_offload_esp = type; 384 break; 385 default: 386 WARN_ON(1); 387 err = -EPROTONOSUPPORT; 388 break; 389 } 390 391 rcu_read_unlock(); 392 return err; 393 } 394 EXPORT_SYMBOL(xfrm_register_type_offload); 395 396 void xfrm_unregister_type_offload(const struct xfrm_type_offload *type, 397 unsigned short family) 398 { 399 struct xfrm_state_afinfo *afinfo = xfrm_state_get_afinfo(family); 400 401 if (unlikely(afinfo == NULL)) 402 return; 403 404 switch (type->proto) { 405 case IPPROTO_ESP: 406 WARN_ON(afinfo->type_offload_esp != type); 407 afinfo->type_offload_esp = NULL; 408 break; 409 default: 410 WARN_ON(1); 411 break; 412 } 413 rcu_read_unlock(); 414 } 415 EXPORT_SYMBOL(xfrm_unregister_type_offload); 416 417 static const struct xfrm_type_offload * 418 xfrm_get_type_offload(u8 proto, unsigned short family, bool try_load) 419 { 420 const struct xfrm_type_offload *type = NULL; 421 struct xfrm_state_afinfo *afinfo; 422 423 retry: 424 afinfo = xfrm_state_get_afinfo(family); 425 if (unlikely(afinfo == NULL)) 426 return NULL; 427 428 switch (proto) { 429 case IPPROTO_ESP: 430 type = afinfo->type_offload_esp; 431 break; 432 default: 433 break; 434 } 435 436 if ((type && !try_module_get(type->owner))) 437 type = NULL; 438 439 rcu_read_unlock(); 440 441 if (!type && try_load) { 442 request_module("xfrm-offload-%d-%d", family, proto); 443 try_load = false; 444 goto retry; 445 } 446 447 return type; 448 } 449 450 static void xfrm_put_type_offload(const struct xfrm_type_offload *type) 451 { 452 module_put(type->owner); 453 } 454 455 static const struct xfrm_mode xfrm4_mode_map[XFRM_MODE_MAX] = { 456 [XFRM_MODE_BEET] = { 457 .encap = XFRM_MODE_BEET, 458 .flags = XFRM_MODE_FLAG_TUNNEL, 459 .family = AF_INET, 460 }, 461 [XFRM_MODE_TRANSPORT] = { 462 .encap = XFRM_MODE_TRANSPORT, 463 .family = AF_INET, 464 }, 465 [XFRM_MODE_TUNNEL] = { 466 .encap = XFRM_MODE_TUNNEL, 467 .flags = XFRM_MODE_FLAG_TUNNEL, 468 .family = AF_INET, 469 }, 470 }; 471 472 static const struct xfrm_mode xfrm6_mode_map[XFRM_MODE_MAX] = { 473 [XFRM_MODE_BEET] = { 474 .encap = XFRM_MODE_BEET, 475 .flags = XFRM_MODE_FLAG_TUNNEL, 476 .family = AF_INET6, 477 }, 478 [XFRM_MODE_ROUTEOPTIMIZATION] = { 479 .encap = XFRM_MODE_ROUTEOPTIMIZATION, 480 .family = AF_INET6, 481 }, 482 [XFRM_MODE_TRANSPORT] = { 483 .encap = XFRM_MODE_TRANSPORT, 484 .family = AF_INET6, 485 }, 486 [XFRM_MODE_TUNNEL] = { 487 .encap = XFRM_MODE_TUNNEL, 488 .flags = XFRM_MODE_FLAG_TUNNEL, 489 .family = AF_INET6, 490 }, 491 }; 492 493 static const struct xfrm_mode *xfrm_get_mode(unsigned int encap, int family) 494 { 495 const struct xfrm_mode *mode; 496 497 if (unlikely(encap >= XFRM_MODE_MAX)) 498 return NULL; 499 500 switch (family) { 501 case AF_INET: 502 mode = &xfrm4_mode_map[encap]; 503 if (mode->family == family) 504 return mode; 505 break; 506 case AF_INET6: 507 mode = &xfrm6_mode_map[encap]; 508 if (mode->family == family) 509 return mode; 510 break; 511 default: 512 break; 513 } 514 515 return NULL; 516 } 517 518 void xfrm_state_free(struct xfrm_state *x) 519 { 520 kmem_cache_free(xfrm_state_cache, x); 521 } 522 EXPORT_SYMBOL(xfrm_state_free); 523 524 static void ___xfrm_state_destroy(struct xfrm_state *x) 525 { 526 hrtimer_cancel(&x->mtimer); 527 del_timer_sync(&x->rtimer); 528 kfree(x->aead); 529 kfree(x->aalg); 530 kfree(x->ealg); 531 kfree(x->calg); 532 kfree(x->encap); 533 kfree(x->coaddr); 534 kfree(x->replay_esn); 535 kfree(x->preplay_esn); 536 if (x->type_offload) 537 xfrm_put_type_offload(x->type_offload); 538 if (x->type) { 539 x->type->destructor(x); 540 xfrm_put_type(x->type); 541 } 542 if (x->xfrag.page) 543 put_page(x->xfrag.page); 544 xfrm_dev_state_free(x); 545 security_xfrm_state_free(x); 546 xfrm_state_free(x); 547 } 548 549 static void xfrm_state_gc_task(struct work_struct *work) 550 { 551 struct xfrm_state *x; 552 struct hlist_node *tmp; 553 struct hlist_head gc_list; 554 555 spin_lock_bh(&xfrm_state_gc_lock); 556 hlist_move_list(&xfrm_state_gc_list, &gc_list); 557 spin_unlock_bh(&xfrm_state_gc_lock); 558 559 synchronize_rcu(); 560 561 hlist_for_each_entry_safe(x, tmp, &gc_list, gclist) 562 ___xfrm_state_destroy(x); 563 } 564 565 static enum hrtimer_restart xfrm_timer_handler(struct hrtimer *me) 566 { 567 struct xfrm_state *x = container_of(me, struct xfrm_state, mtimer); 568 enum hrtimer_restart ret = HRTIMER_NORESTART; 569 time64_t now = ktime_get_real_seconds(); 570 time64_t next = TIME64_MAX; 571 int warn = 0; 572 int err = 0; 573 574 spin_lock(&x->lock); 575 xfrm_dev_state_update_stats(x); 576 577 if (x->km.state == XFRM_STATE_DEAD) 578 goto out; 579 if (x->km.state == XFRM_STATE_EXPIRED) 580 goto expired; 581 if (x->lft.hard_add_expires_seconds) { 582 time64_t tmo = x->lft.hard_add_expires_seconds + 583 x->curlft.add_time - now; 584 if (tmo <= 0) { 585 if (x->xflags & XFRM_SOFT_EXPIRE) { 586 /* enter hard expire without soft expire first?! 587 * setting a new date could trigger this. 588 * workaround: fix x->curflt.add_time by below: 589 */ 590 x->curlft.add_time = now - x->saved_tmo - 1; 591 tmo = x->lft.hard_add_expires_seconds - x->saved_tmo; 592 } else 593 goto expired; 594 } 595 if (tmo < next) 596 next = tmo; 597 } 598 if (x->lft.hard_use_expires_seconds) { 599 time64_t tmo = x->lft.hard_use_expires_seconds + 600 (READ_ONCE(x->curlft.use_time) ? : now) - now; 601 if (tmo <= 0) 602 goto expired; 603 if (tmo < next) 604 next = tmo; 605 } 606 if (x->km.dying) 607 goto resched; 608 if (x->lft.soft_add_expires_seconds) { 609 time64_t tmo = x->lft.soft_add_expires_seconds + 610 x->curlft.add_time - now; 611 if (tmo <= 0) { 612 warn = 1; 613 x->xflags &= ~XFRM_SOFT_EXPIRE; 614 } else if (tmo < next) { 615 next = tmo; 616 x->xflags |= XFRM_SOFT_EXPIRE; 617 x->saved_tmo = tmo; 618 } 619 } 620 if (x->lft.soft_use_expires_seconds) { 621 time64_t tmo = x->lft.soft_use_expires_seconds + 622 (READ_ONCE(x->curlft.use_time) ? : now) - now; 623 if (tmo <= 0) 624 warn = 1; 625 else if (tmo < next) 626 next = tmo; 627 } 628 629 x->km.dying = warn; 630 if (warn) 631 km_state_expired(x, 0, 0); 632 resched: 633 if (next != TIME64_MAX) { 634 hrtimer_forward_now(&x->mtimer, ktime_set(next, 0)); 635 ret = HRTIMER_RESTART; 636 } 637 638 goto out; 639 640 expired: 641 if (x->km.state == XFRM_STATE_ACQ && x->id.spi == 0) 642 x->km.state = XFRM_STATE_EXPIRED; 643 644 err = __xfrm_state_delete(x); 645 if (!err) 646 km_state_expired(x, 1, 0); 647 648 xfrm_audit_state_delete(x, err ? 0 : 1, true); 649 650 out: 651 spin_unlock(&x->lock); 652 return ret; 653 } 654 655 static void xfrm_replay_timer_handler(struct timer_list *t); 656 657 struct xfrm_state *xfrm_state_alloc(struct net *net) 658 { 659 struct xfrm_state *x; 660 661 x = kmem_cache_zalloc(xfrm_state_cache, GFP_ATOMIC); 662 663 if (x) { 664 write_pnet(&x->xs_net, net); 665 refcount_set(&x->refcnt, 1); 666 atomic_set(&x->tunnel_users, 0); 667 INIT_LIST_HEAD(&x->km.all); 668 INIT_HLIST_NODE(&x->bydst); 669 INIT_HLIST_NODE(&x->bysrc); 670 INIT_HLIST_NODE(&x->byspi); 671 INIT_HLIST_NODE(&x->byseq); 672 hrtimer_init(&x->mtimer, CLOCK_BOOTTIME, HRTIMER_MODE_ABS_SOFT); 673 x->mtimer.function = xfrm_timer_handler; 674 timer_setup(&x->rtimer, xfrm_replay_timer_handler, 0); 675 x->curlft.add_time = ktime_get_real_seconds(); 676 x->lft.soft_byte_limit = XFRM_INF; 677 x->lft.soft_packet_limit = XFRM_INF; 678 x->lft.hard_byte_limit = XFRM_INF; 679 x->lft.hard_packet_limit = XFRM_INF; 680 x->replay_maxage = 0; 681 x->replay_maxdiff = 0; 682 spin_lock_init(&x->lock); 683 } 684 return x; 685 } 686 EXPORT_SYMBOL(xfrm_state_alloc); 687 688 #ifdef CONFIG_XFRM_OFFLOAD 689 void xfrm_dev_state_delete(struct xfrm_state *x) 690 { 691 struct xfrm_dev_offload *xso = &x->xso; 692 struct net_device *dev = READ_ONCE(xso->dev); 693 694 if (dev) { 695 dev->xfrmdev_ops->xdo_dev_state_delete(x); 696 spin_lock_bh(&xfrm_state_dev_gc_lock); 697 hlist_add_head(&x->dev_gclist, &xfrm_state_dev_gc_list); 698 spin_unlock_bh(&xfrm_state_dev_gc_lock); 699 } 700 } 701 EXPORT_SYMBOL_GPL(xfrm_dev_state_delete); 702 703 void xfrm_dev_state_free(struct xfrm_state *x) 704 { 705 struct xfrm_dev_offload *xso = &x->xso; 706 struct net_device *dev = READ_ONCE(xso->dev); 707 708 if (dev && dev->xfrmdev_ops) { 709 spin_lock_bh(&xfrm_state_dev_gc_lock); 710 if (!hlist_unhashed(&x->dev_gclist)) 711 hlist_del(&x->dev_gclist); 712 spin_unlock_bh(&xfrm_state_dev_gc_lock); 713 714 if (dev->xfrmdev_ops->xdo_dev_state_free) 715 dev->xfrmdev_ops->xdo_dev_state_free(x); 716 WRITE_ONCE(xso->dev, NULL); 717 xso->type = XFRM_DEV_OFFLOAD_UNSPECIFIED; 718 netdev_put(dev, &xso->dev_tracker); 719 } 720 } 721 #endif 722 723 void __xfrm_state_destroy(struct xfrm_state *x, bool sync) 724 { 725 WARN_ON(x->km.state != XFRM_STATE_DEAD); 726 727 if (sync) { 728 synchronize_rcu(); 729 ___xfrm_state_destroy(x); 730 } else { 731 spin_lock_bh(&xfrm_state_gc_lock); 732 hlist_add_head(&x->gclist, &xfrm_state_gc_list); 733 spin_unlock_bh(&xfrm_state_gc_lock); 734 schedule_work(&xfrm_state_gc_work); 735 } 736 } 737 EXPORT_SYMBOL(__xfrm_state_destroy); 738 739 int __xfrm_state_delete(struct xfrm_state *x) 740 { 741 struct net *net = xs_net(x); 742 int err = -ESRCH; 743 744 if (x->km.state != XFRM_STATE_DEAD) { 745 x->km.state = XFRM_STATE_DEAD; 746 spin_lock(&net->xfrm.xfrm_state_lock); 747 list_del(&x->km.all); 748 hlist_del_rcu(&x->bydst); 749 hlist_del_rcu(&x->bysrc); 750 if (x->km.seq) 751 hlist_del_rcu(&x->byseq); 752 if (x->id.spi) 753 hlist_del_rcu(&x->byspi); 754 net->xfrm.state_num--; 755 xfrm_nat_keepalive_state_updated(x); 756 spin_unlock(&net->xfrm.xfrm_state_lock); 757 758 if (x->encap_sk) 759 sock_put(rcu_dereference_raw(x->encap_sk)); 760 761 xfrm_dev_state_delete(x); 762 763 /* All xfrm_state objects are created by xfrm_state_alloc. 764 * The xfrm_state_alloc call gives a reference, and that 765 * is what we are dropping here. 766 */ 767 xfrm_state_put(x); 768 err = 0; 769 } 770 771 return err; 772 } 773 EXPORT_SYMBOL(__xfrm_state_delete); 774 775 int xfrm_state_delete(struct xfrm_state *x) 776 { 777 int err; 778 779 spin_lock_bh(&x->lock); 780 err = __xfrm_state_delete(x); 781 spin_unlock_bh(&x->lock); 782 783 return err; 784 } 785 EXPORT_SYMBOL(xfrm_state_delete); 786 787 #ifdef CONFIG_SECURITY_NETWORK_XFRM 788 static inline int 789 xfrm_state_flush_secctx_check(struct net *net, u8 proto, bool task_valid) 790 { 791 int i, err = 0; 792 793 for (i = 0; i <= net->xfrm.state_hmask; i++) { 794 struct xfrm_state *x; 795 796 hlist_for_each_entry(x, net->xfrm.state_bydst+i, bydst) { 797 if (xfrm_id_proto_match(x->id.proto, proto) && 798 (err = security_xfrm_state_delete(x)) != 0) { 799 xfrm_audit_state_delete(x, 0, task_valid); 800 return err; 801 } 802 } 803 } 804 805 return err; 806 } 807 808 static inline int 809 xfrm_dev_state_flush_secctx_check(struct net *net, struct net_device *dev, bool task_valid) 810 { 811 int i, err = 0; 812 813 for (i = 0; i <= net->xfrm.state_hmask; i++) { 814 struct xfrm_state *x; 815 struct xfrm_dev_offload *xso; 816 817 hlist_for_each_entry(x, net->xfrm.state_bydst+i, bydst) { 818 xso = &x->xso; 819 820 if (xso->dev == dev && 821 (err = security_xfrm_state_delete(x)) != 0) { 822 xfrm_audit_state_delete(x, 0, task_valid); 823 return err; 824 } 825 } 826 } 827 828 return err; 829 } 830 #else 831 static inline int 832 xfrm_state_flush_secctx_check(struct net *net, u8 proto, bool task_valid) 833 { 834 return 0; 835 } 836 837 static inline int 838 xfrm_dev_state_flush_secctx_check(struct net *net, struct net_device *dev, bool task_valid) 839 { 840 return 0; 841 } 842 #endif 843 844 int xfrm_state_flush(struct net *net, u8 proto, bool task_valid, bool sync) 845 { 846 int i, err = 0, cnt = 0; 847 848 spin_lock_bh(&net->xfrm.xfrm_state_lock); 849 err = xfrm_state_flush_secctx_check(net, proto, task_valid); 850 if (err) 851 goto out; 852 853 err = -ESRCH; 854 for (i = 0; i <= net->xfrm.state_hmask; i++) { 855 struct xfrm_state *x; 856 restart: 857 hlist_for_each_entry(x, net->xfrm.state_bydst+i, bydst) { 858 if (!xfrm_state_kern(x) && 859 xfrm_id_proto_match(x->id.proto, proto)) { 860 xfrm_state_hold(x); 861 spin_unlock_bh(&net->xfrm.xfrm_state_lock); 862 863 err = xfrm_state_delete(x); 864 xfrm_audit_state_delete(x, err ? 0 : 1, 865 task_valid); 866 if (sync) 867 xfrm_state_put_sync(x); 868 else 869 xfrm_state_put(x); 870 if (!err) 871 cnt++; 872 873 spin_lock_bh(&net->xfrm.xfrm_state_lock); 874 goto restart; 875 } 876 } 877 } 878 out: 879 spin_unlock_bh(&net->xfrm.xfrm_state_lock); 880 if (cnt) 881 err = 0; 882 883 return err; 884 } 885 EXPORT_SYMBOL(xfrm_state_flush); 886 887 int xfrm_dev_state_flush(struct net *net, struct net_device *dev, bool task_valid) 888 { 889 struct xfrm_state *x; 890 struct hlist_node *tmp; 891 struct xfrm_dev_offload *xso; 892 int i, err = 0, cnt = 0; 893 894 spin_lock_bh(&net->xfrm.xfrm_state_lock); 895 err = xfrm_dev_state_flush_secctx_check(net, dev, task_valid); 896 if (err) 897 goto out; 898 899 err = -ESRCH; 900 for (i = 0; i <= net->xfrm.state_hmask; i++) { 901 restart: 902 hlist_for_each_entry(x, net->xfrm.state_bydst+i, bydst) { 903 xso = &x->xso; 904 905 if (!xfrm_state_kern(x) && xso->dev == dev) { 906 xfrm_state_hold(x); 907 spin_unlock_bh(&net->xfrm.xfrm_state_lock); 908 909 err = xfrm_state_delete(x); 910 xfrm_dev_state_free(x); 911 912 xfrm_audit_state_delete(x, err ? 0 : 1, 913 task_valid); 914 xfrm_state_put(x); 915 if (!err) 916 cnt++; 917 918 spin_lock_bh(&net->xfrm.xfrm_state_lock); 919 goto restart; 920 } 921 } 922 } 923 if (cnt) 924 err = 0; 925 926 out: 927 spin_unlock_bh(&net->xfrm.xfrm_state_lock); 928 929 spin_lock_bh(&xfrm_state_dev_gc_lock); 930 restart_gc: 931 hlist_for_each_entry_safe(x, tmp, &xfrm_state_dev_gc_list, dev_gclist) { 932 xso = &x->xso; 933 934 if (xso->dev == dev) { 935 spin_unlock_bh(&xfrm_state_dev_gc_lock); 936 xfrm_dev_state_free(x); 937 spin_lock_bh(&xfrm_state_dev_gc_lock); 938 goto restart_gc; 939 } 940 941 } 942 spin_unlock_bh(&xfrm_state_dev_gc_lock); 943 944 xfrm_flush_gc(); 945 946 return err; 947 } 948 EXPORT_SYMBOL(xfrm_dev_state_flush); 949 950 void xfrm_sad_getinfo(struct net *net, struct xfrmk_sadinfo *si) 951 { 952 spin_lock_bh(&net->xfrm.xfrm_state_lock); 953 si->sadcnt = net->xfrm.state_num; 954 si->sadhcnt = net->xfrm.state_hmask + 1; 955 si->sadhmcnt = xfrm_state_hashmax; 956 spin_unlock_bh(&net->xfrm.xfrm_state_lock); 957 } 958 EXPORT_SYMBOL(xfrm_sad_getinfo); 959 960 static void 961 __xfrm4_init_tempsel(struct xfrm_selector *sel, const struct flowi *fl) 962 { 963 const struct flowi4 *fl4 = &fl->u.ip4; 964 965 sel->daddr.a4 = fl4->daddr; 966 sel->saddr.a4 = fl4->saddr; 967 sel->dport = xfrm_flowi_dport(fl, &fl4->uli); 968 sel->dport_mask = htons(0xffff); 969 sel->sport = xfrm_flowi_sport(fl, &fl4->uli); 970 sel->sport_mask = htons(0xffff); 971 sel->family = AF_INET; 972 sel->prefixlen_d = 32; 973 sel->prefixlen_s = 32; 974 sel->proto = fl4->flowi4_proto; 975 sel->ifindex = fl4->flowi4_oif; 976 } 977 978 static void 979 __xfrm6_init_tempsel(struct xfrm_selector *sel, const struct flowi *fl) 980 { 981 const struct flowi6 *fl6 = &fl->u.ip6; 982 983 /* Initialize temporary selector matching only to current session. */ 984 *(struct in6_addr *)&sel->daddr = fl6->daddr; 985 *(struct in6_addr *)&sel->saddr = fl6->saddr; 986 sel->dport = xfrm_flowi_dport(fl, &fl6->uli); 987 sel->dport_mask = htons(0xffff); 988 sel->sport = xfrm_flowi_sport(fl, &fl6->uli); 989 sel->sport_mask = htons(0xffff); 990 sel->family = AF_INET6; 991 sel->prefixlen_d = 128; 992 sel->prefixlen_s = 128; 993 sel->proto = fl6->flowi6_proto; 994 sel->ifindex = fl6->flowi6_oif; 995 } 996 997 static void 998 xfrm_init_tempstate(struct xfrm_state *x, const struct flowi *fl, 999 const struct xfrm_tmpl *tmpl, 1000 const xfrm_address_t *daddr, const xfrm_address_t *saddr, 1001 unsigned short family) 1002 { 1003 switch (family) { 1004 case AF_INET: 1005 __xfrm4_init_tempsel(&x->sel, fl); 1006 break; 1007 case AF_INET6: 1008 __xfrm6_init_tempsel(&x->sel, fl); 1009 break; 1010 } 1011 1012 x->id = tmpl->id; 1013 1014 switch (tmpl->encap_family) { 1015 case AF_INET: 1016 if (x->id.daddr.a4 == 0) 1017 x->id.daddr.a4 = daddr->a4; 1018 x->props.saddr = tmpl->saddr; 1019 if (x->props.saddr.a4 == 0) 1020 x->props.saddr.a4 = saddr->a4; 1021 break; 1022 case AF_INET6: 1023 if (ipv6_addr_any((struct in6_addr *)&x->id.daddr)) 1024 memcpy(&x->id.daddr, daddr, sizeof(x->sel.daddr)); 1025 memcpy(&x->props.saddr, &tmpl->saddr, sizeof(x->props.saddr)); 1026 if (ipv6_addr_any((struct in6_addr *)&x->props.saddr)) 1027 memcpy(&x->props.saddr, saddr, sizeof(x->props.saddr)); 1028 break; 1029 } 1030 1031 x->props.mode = tmpl->mode; 1032 x->props.reqid = tmpl->reqid; 1033 x->props.family = tmpl->encap_family; 1034 } 1035 1036 static struct xfrm_state *__xfrm_state_lookup_all(struct net *net, u32 mark, 1037 const xfrm_address_t *daddr, 1038 __be32 spi, u8 proto, 1039 unsigned short family, 1040 struct xfrm_dev_offload *xdo) 1041 { 1042 unsigned int h = xfrm_spi_hash(net, daddr, spi, proto, family); 1043 struct xfrm_state *x; 1044 1045 hlist_for_each_entry_rcu(x, net->xfrm.state_byspi + h, byspi) { 1046 #ifdef CONFIG_XFRM_OFFLOAD 1047 if (xdo->type == XFRM_DEV_OFFLOAD_PACKET) { 1048 if (x->xso.type != XFRM_DEV_OFFLOAD_PACKET) 1049 /* HW states are in the head of list, there is 1050 * no need to iterate further. 1051 */ 1052 break; 1053 1054 /* Packet offload: both policy and SA should 1055 * have same device. 1056 */ 1057 if (xdo->dev != x->xso.dev) 1058 continue; 1059 } else if (x->xso.type == XFRM_DEV_OFFLOAD_PACKET) 1060 /* Skip HW policy for SW lookups */ 1061 continue; 1062 #endif 1063 if (x->props.family != family || 1064 x->id.spi != spi || 1065 x->id.proto != proto || 1066 !xfrm_addr_equal(&x->id.daddr, daddr, family)) 1067 continue; 1068 1069 if ((mark & x->mark.m) != x->mark.v) 1070 continue; 1071 if (!xfrm_state_hold_rcu(x)) 1072 continue; 1073 return x; 1074 } 1075 1076 return NULL; 1077 } 1078 1079 static struct xfrm_state *__xfrm_state_lookup(struct net *net, u32 mark, 1080 const xfrm_address_t *daddr, 1081 __be32 spi, u8 proto, 1082 unsigned short family) 1083 { 1084 unsigned int h = xfrm_spi_hash(net, daddr, spi, proto, family); 1085 struct xfrm_state *x; 1086 1087 hlist_for_each_entry_rcu(x, net->xfrm.state_byspi + h, byspi) { 1088 if (x->props.family != family || 1089 x->id.spi != spi || 1090 x->id.proto != proto || 1091 !xfrm_addr_equal(&x->id.daddr, daddr, family)) 1092 continue; 1093 1094 if ((mark & x->mark.m) != x->mark.v) 1095 continue; 1096 if (!xfrm_state_hold_rcu(x)) 1097 continue; 1098 return x; 1099 } 1100 1101 return NULL; 1102 } 1103 1104 static struct xfrm_state *__xfrm_state_lookup_byaddr(struct net *net, u32 mark, 1105 const xfrm_address_t *daddr, 1106 const xfrm_address_t *saddr, 1107 u8 proto, unsigned short family) 1108 { 1109 unsigned int h = xfrm_src_hash(net, daddr, saddr, family); 1110 struct xfrm_state *x; 1111 1112 hlist_for_each_entry_rcu(x, net->xfrm.state_bysrc + h, bysrc) { 1113 if (x->props.family != family || 1114 x->id.proto != proto || 1115 !xfrm_addr_equal(&x->id.daddr, daddr, family) || 1116 !xfrm_addr_equal(&x->props.saddr, saddr, family)) 1117 continue; 1118 1119 if ((mark & x->mark.m) != x->mark.v) 1120 continue; 1121 if (!xfrm_state_hold_rcu(x)) 1122 continue; 1123 return x; 1124 } 1125 1126 return NULL; 1127 } 1128 1129 static inline struct xfrm_state * 1130 __xfrm_state_locate(struct xfrm_state *x, int use_spi, int family) 1131 { 1132 struct net *net = xs_net(x); 1133 u32 mark = x->mark.v & x->mark.m; 1134 1135 if (use_spi) 1136 return __xfrm_state_lookup(net, mark, &x->id.daddr, 1137 x->id.spi, x->id.proto, family); 1138 else 1139 return __xfrm_state_lookup_byaddr(net, mark, 1140 &x->id.daddr, 1141 &x->props.saddr, 1142 x->id.proto, family); 1143 } 1144 1145 static void xfrm_hash_grow_check(struct net *net, int have_hash_collision) 1146 { 1147 if (have_hash_collision && 1148 (net->xfrm.state_hmask + 1) < xfrm_state_hashmax && 1149 net->xfrm.state_num > net->xfrm.state_hmask) 1150 schedule_work(&net->xfrm.state_hash_work); 1151 } 1152 1153 static void xfrm_state_look_at(struct xfrm_policy *pol, struct xfrm_state *x, 1154 const struct flowi *fl, unsigned short family, 1155 struct xfrm_state **best, int *acq_in_progress, 1156 int *error) 1157 { 1158 /* Resolution logic: 1159 * 1. There is a valid state with matching selector. Done. 1160 * 2. Valid state with inappropriate selector. Skip. 1161 * 1162 * Entering area of "sysdeps". 1163 * 1164 * 3. If state is not valid, selector is temporary, it selects 1165 * only session which triggered previous resolution. Key 1166 * manager will do something to install a state with proper 1167 * selector. 1168 */ 1169 if (x->km.state == XFRM_STATE_VALID) { 1170 if ((x->sel.family && 1171 (x->sel.family != family || 1172 !xfrm_selector_match(&x->sel, fl, family))) || 1173 !security_xfrm_state_pol_flow_match(x, pol, 1174 &fl->u.__fl_common)) 1175 return; 1176 1177 if (!*best || 1178 (*best)->km.dying > x->km.dying || 1179 ((*best)->km.dying == x->km.dying && 1180 (*best)->curlft.add_time < x->curlft.add_time)) 1181 *best = x; 1182 } else if (x->km.state == XFRM_STATE_ACQ) { 1183 *acq_in_progress = 1; 1184 } else if (x->km.state == XFRM_STATE_ERROR || 1185 x->km.state == XFRM_STATE_EXPIRED) { 1186 if ((!x->sel.family || 1187 (x->sel.family == family && 1188 xfrm_selector_match(&x->sel, fl, family))) && 1189 security_xfrm_state_pol_flow_match(x, pol, 1190 &fl->u.__fl_common)) 1191 *error = -ESRCH; 1192 } 1193 } 1194 1195 struct xfrm_state * 1196 xfrm_state_find(const xfrm_address_t *daddr, const xfrm_address_t *saddr, 1197 const struct flowi *fl, struct xfrm_tmpl *tmpl, 1198 struct xfrm_policy *pol, int *err, 1199 unsigned short family, u32 if_id) 1200 { 1201 static xfrm_address_t saddr_wildcard = { }; 1202 struct net *net = xp_net(pol); 1203 unsigned int h, h_wildcard; 1204 struct xfrm_state *x, *x0, *to_put; 1205 int acquire_in_progress = 0; 1206 int error = 0; 1207 struct xfrm_state *best = NULL; 1208 u32 mark = pol->mark.v & pol->mark.m; 1209 unsigned short encap_family = tmpl->encap_family; 1210 unsigned int sequence; 1211 struct km_event c; 1212 1213 to_put = NULL; 1214 1215 sequence = read_seqcount_begin(&net->xfrm.xfrm_state_hash_generation); 1216 1217 rcu_read_lock(); 1218 h = xfrm_dst_hash(net, daddr, saddr, tmpl->reqid, encap_family); 1219 hlist_for_each_entry_rcu(x, net->xfrm.state_bydst + h, bydst) { 1220 #ifdef CONFIG_XFRM_OFFLOAD 1221 if (pol->xdo.type == XFRM_DEV_OFFLOAD_PACKET) { 1222 if (x->xso.type != XFRM_DEV_OFFLOAD_PACKET) 1223 /* HW states are in the head of list, there is 1224 * no need to iterate further. 1225 */ 1226 break; 1227 1228 /* Packet offload: both policy and SA should 1229 * have same device. 1230 */ 1231 if (pol->xdo.dev != x->xso.dev) 1232 continue; 1233 } else if (x->xso.type == XFRM_DEV_OFFLOAD_PACKET) 1234 /* Skip HW policy for SW lookups */ 1235 continue; 1236 #endif 1237 if (x->props.family == encap_family && 1238 x->props.reqid == tmpl->reqid && 1239 (mark & x->mark.m) == x->mark.v && 1240 x->if_id == if_id && 1241 !(x->props.flags & XFRM_STATE_WILDRECV) && 1242 xfrm_state_addr_check(x, daddr, saddr, encap_family) && 1243 tmpl->mode == x->props.mode && 1244 tmpl->id.proto == x->id.proto && 1245 (tmpl->id.spi == x->id.spi || !tmpl->id.spi)) 1246 xfrm_state_look_at(pol, x, fl, family, 1247 &best, &acquire_in_progress, &error); 1248 } 1249 if (best || acquire_in_progress) 1250 goto found; 1251 1252 h_wildcard = xfrm_dst_hash(net, daddr, &saddr_wildcard, tmpl->reqid, encap_family); 1253 hlist_for_each_entry_rcu(x, net->xfrm.state_bydst + h_wildcard, bydst) { 1254 #ifdef CONFIG_XFRM_OFFLOAD 1255 if (pol->xdo.type == XFRM_DEV_OFFLOAD_PACKET) { 1256 if (x->xso.type != XFRM_DEV_OFFLOAD_PACKET) 1257 /* HW states are in the head of list, there is 1258 * no need to iterate further. 1259 */ 1260 break; 1261 1262 /* Packet offload: both policy and SA should 1263 * have same device. 1264 */ 1265 if (pol->xdo.dev != x->xso.dev) 1266 continue; 1267 } else if (x->xso.type == XFRM_DEV_OFFLOAD_PACKET) 1268 /* Skip HW policy for SW lookups */ 1269 continue; 1270 #endif 1271 if (x->props.family == encap_family && 1272 x->props.reqid == tmpl->reqid && 1273 (mark & x->mark.m) == x->mark.v && 1274 x->if_id == if_id && 1275 !(x->props.flags & XFRM_STATE_WILDRECV) && 1276 xfrm_addr_equal(&x->id.daddr, daddr, encap_family) && 1277 tmpl->mode == x->props.mode && 1278 tmpl->id.proto == x->id.proto && 1279 (tmpl->id.spi == x->id.spi || !tmpl->id.spi)) 1280 xfrm_state_look_at(pol, x, fl, family, 1281 &best, &acquire_in_progress, &error); 1282 } 1283 1284 found: 1285 x = best; 1286 if (!x && !error && !acquire_in_progress) { 1287 if (tmpl->id.spi && 1288 (x0 = __xfrm_state_lookup_all(net, mark, daddr, 1289 tmpl->id.spi, tmpl->id.proto, 1290 encap_family, 1291 &pol->xdo)) != NULL) { 1292 to_put = x0; 1293 error = -EEXIST; 1294 goto out; 1295 } 1296 1297 c.net = net; 1298 /* If the KMs have no listeners (yet...), avoid allocating an SA 1299 * for each and every packet - garbage collection might not 1300 * handle the flood. 1301 */ 1302 if (!km_is_alive(&c)) { 1303 error = -ESRCH; 1304 goto out; 1305 } 1306 1307 x = xfrm_state_alloc(net); 1308 if (x == NULL) { 1309 error = -ENOMEM; 1310 goto out; 1311 } 1312 /* Initialize temporary state matching only 1313 * to current session. */ 1314 xfrm_init_tempstate(x, fl, tmpl, daddr, saddr, family); 1315 memcpy(&x->mark, &pol->mark, sizeof(x->mark)); 1316 x->if_id = if_id; 1317 1318 error = security_xfrm_state_alloc_acquire(x, pol->security, fl->flowi_secid); 1319 if (error) { 1320 x->km.state = XFRM_STATE_DEAD; 1321 to_put = x; 1322 x = NULL; 1323 goto out; 1324 } 1325 #ifdef CONFIG_XFRM_OFFLOAD 1326 if (pol->xdo.type == XFRM_DEV_OFFLOAD_PACKET) { 1327 struct xfrm_dev_offload *xdo = &pol->xdo; 1328 struct xfrm_dev_offload *xso = &x->xso; 1329 1330 xso->type = XFRM_DEV_OFFLOAD_PACKET; 1331 xso->dir = xdo->dir; 1332 xso->dev = xdo->dev; 1333 xso->real_dev = xdo->real_dev; 1334 xso->flags = XFRM_DEV_OFFLOAD_FLAG_ACQ; 1335 netdev_hold(xso->dev, &xso->dev_tracker, GFP_ATOMIC); 1336 error = xso->dev->xfrmdev_ops->xdo_dev_state_add(x, NULL); 1337 if (error) { 1338 xso->dir = 0; 1339 netdev_put(xso->dev, &xso->dev_tracker); 1340 xso->dev = NULL; 1341 xso->real_dev = NULL; 1342 xso->type = XFRM_DEV_OFFLOAD_UNSPECIFIED; 1343 x->km.state = XFRM_STATE_DEAD; 1344 to_put = x; 1345 x = NULL; 1346 goto out; 1347 } 1348 } 1349 #endif 1350 if (km_query(x, tmpl, pol) == 0) { 1351 spin_lock_bh(&net->xfrm.xfrm_state_lock); 1352 x->km.state = XFRM_STATE_ACQ; 1353 x->dir = XFRM_SA_DIR_OUT; 1354 list_add(&x->km.all, &net->xfrm.state_all); 1355 XFRM_STATE_INSERT(bydst, &x->bydst, 1356 net->xfrm.state_bydst + h, 1357 x->xso.type); 1358 h = xfrm_src_hash(net, daddr, saddr, encap_family); 1359 XFRM_STATE_INSERT(bysrc, &x->bysrc, 1360 net->xfrm.state_bysrc + h, 1361 x->xso.type); 1362 if (x->id.spi) { 1363 h = xfrm_spi_hash(net, &x->id.daddr, x->id.spi, x->id.proto, encap_family); 1364 XFRM_STATE_INSERT(byspi, &x->byspi, 1365 net->xfrm.state_byspi + h, 1366 x->xso.type); 1367 } 1368 if (x->km.seq) { 1369 h = xfrm_seq_hash(net, x->km.seq); 1370 XFRM_STATE_INSERT(byseq, &x->byseq, 1371 net->xfrm.state_byseq + h, 1372 x->xso.type); 1373 } 1374 x->lft.hard_add_expires_seconds = net->xfrm.sysctl_acq_expires; 1375 hrtimer_start(&x->mtimer, 1376 ktime_set(net->xfrm.sysctl_acq_expires, 0), 1377 HRTIMER_MODE_REL_SOFT); 1378 net->xfrm.state_num++; 1379 xfrm_hash_grow_check(net, x->bydst.next != NULL); 1380 spin_unlock_bh(&net->xfrm.xfrm_state_lock); 1381 } else { 1382 #ifdef CONFIG_XFRM_OFFLOAD 1383 struct xfrm_dev_offload *xso = &x->xso; 1384 1385 if (xso->type == XFRM_DEV_OFFLOAD_PACKET) { 1386 xfrm_dev_state_delete(x); 1387 xfrm_dev_state_free(x); 1388 } 1389 #endif 1390 x->km.state = XFRM_STATE_DEAD; 1391 to_put = x; 1392 x = NULL; 1393 error = -ESRCH; 1394 } 1395 } 1396 out: 1397 if (x) { 1398 if (!xfrm_state_hold_rcu(x)) { 1399 *err = -EAGAIN; 1400 x = NULL; 1401 } 1402 } else { 1403 *err = acquire_in_progress ? -EAGAIN : error; 1404 } 1405 rcu_read_unlock(); 1406 if (to_put) 1407 xfrm_state_put(to_put); 1408 1409 if (read_seqcount_retry(&net->xfrm.xfrm_state_hash_generation, sequence)) { 1410 *err = -EAGAIN; 1411 if (x) { 1412 xfrm_state_put(x); 1413 x = NULL; 1414 } 1415 } 1416 1417 return x; 1418 } 1419 1420 struct xfrm_state * 1421 xfrm_stateonly_find(struct net *net, u32 mark, u32 if_id, 1422 xfrm_address_t *daddr, xfrm_address_t *saddr, 1423 unsigned short family, u8 mode, u8 proto, u32 reqid) 1424 { 1425 unsigned int h; 1426 struct xfrm_state *rx = NULL, *x = NULL; 1427 1428 spin_lock_bh(&net->xfrm.xfrm_state_lock); 1429 h = xfrm_dst_hash(net, daddr, saddr, reqid, family); 1430 hlist_for_each_entry(x, net->xfrm.state_bydst+h, bydst) { 1431 if (x->props.family == family && 1432 x->props.reqid == reqid && 1433 (mark & x->mark.m) == x->mark.v && 1434 x->if_id == if_id && 1435 !(x->props.flags & XFRM_STATE_WILDRECV) && 1436 xfrm_state_addr_check(x, daddr, saddr, family) && 1437 mode == x->props.mode && 1438 proto == x->id.proto && 1439 x->km.state == XFRM_STATE_VALID) { 1440 rx = x; 1441 break; 1442 } 1443 } 1444 1445 if (rx) 1446 xfrm_state_hold(rx); 1447 spin_unlock_bh(&net->xfrm.xfrm_state_lock); 1448 1449 1450 return rx; 1451 } 1452 EXPORT_SYMBOL(xfrm_stateonly_find); 1453 1454 struct xfrm_state *xfrm_state_lookup_byspi(struct net *net, __be32 spi, 1455 unsigned short family) 1456 { 1457 struct xfrm_state *x; 1458 struct xfrm_state_walk *w; 1459 1460 spin_lock_bh(&net->xfrm.xfrm_state_lock); 1461 list_for_each_entry(w, &net->xfrm.state_all, all) { 1462 x = container_of(w, struct xfrm_state, km); 1463 if (x->props.family != family || 1464 x->id.spi != spi) 1465 continue; 1466 1467 xfrm_state_hold(x); 1468 spin_unlock_bh(&net->xfrm.xfrm_state_lock); 1469 return x; 1470 } 1471 spin_unlock_bh(&net->xfrm.xfrm_state_lock); 1472 return NULL; 1473 } 1474 EXPORT_SYMBOL(xfrm_state_lookup_byspi); 1475 1476 static void __xfrm_state_insert(struct xfrm_state *x) 1477 { 1478 struct net *net = xs_net(x); 1479 unsigned int h; 1480 1481 list_add(&x->km.all, &net->xfrm.state_all); 1482 1483 h = xfrm_dst_hash(net, &x->id.daddr, &x->props.saddr, 1484 x->props.reqid, x->props.family); 1485 XFRM_STATE_INSERT(bydst, &x->bydst, net->xfrm.state_bydst + h, 1486 x->xso.type); 1487 1488 h = xfrm_src_hash(net, &x->id.daddr, &x->props.saddr, x->props.family); 1489 XFRM_STATE_INSERT(bysrc, &x->bysrc, net->xfrm.state_bysrc + h, 1490 x->xso.type); 1491 1492 if (x->id.spi) { 1493 h = xfrm_spi_hash(net, &x->id.daddr, x->id.spi, x->id.proto, 1494 x->props.family); 1495 1496 XFRM_STATE_INSERT(byspi, &x->byspi, net->xfrm.state_byspi + h, 1497 x->xso.type); 1498 } 1499 1500 if (x->km.seq) { 1501 h = xfrm_seq_hash(net, x->km.seq); 1502 1503 XFRM_STATE_INSERT(byseq, &x->byseq, net->xfrm.state_byseq + h, 1504 x->xso.type); 1505 } 1506 1507 hrtimer_start(&x->mtimer, ktime_set(1, 0), HRTIMER_MODE_REL_SOFT); 1508 if (x->replay_maxage) 1509 mod_timer(&x->rtimer, jiffies + x->replay_maxage); 1510 1511 net->xfrm.state_num++; 1512 1513 xfrm_hash_grow_check(net, x->bydst.next != NULL); 1514 xfrm_nat_keepalive_state_updated(x); 1515 } 1516 1517 /* net->xfrm.xfrm_state_lock is held */ 1518 static void __xfrm_state_bump_genids(struct xfrm_state *xnew) 1519 { 1520 struct net *net = xs_net(xnew); 1521 unsigned short family = xnew->props.family; 1522 u32 reqid = xnew->props.reqid; 1523 struct xfrm_state *x; 1524 unsigned int h; 1525 u32 mark = xnew->mark.v & xnew->mark.m; 1526 u32 if_id = xnew->if_id; 1527 1528 h = xfrm_dst_hash(net, &xnew->id.daddr, &xnew->props.saddr, reqid, family); 1529 hlist_for_each_entry(x, net->xfrm.state_bydst+h, bydst) { 1530 if (x->props.family == family && 1531 x->props.reqid == reqid && 1532 x->if_id == if_id && 1533 (mark & x->mark.m) == x->mark.v && 1534 xfrm_addr_equal(&x->id.daddr, &xnew->id.daddr, family) && 1535 xfrm_addr_equal(&x->props.saddr, &xnew->props.saddr, family)) 1536 x->genid++; 1537 } 1538 } 1539 1540 void xfrm_state_insert(struct xfrm_state *x) 1541 { 1542 struct net *net = xs_net(x); 1543 1544 spin_lock_bh(&net->xfrm.xfrm_state_lock); 1545 __xfrm_state_bump_genids(x); 1546 __xfrm_state_insert(x); 1547 spin_unlock_bh(&net->xfrm.xfrm_state_lock); 1548 } 1549 EXPORT_SYMBOL(xfrm_state_insert); 1550 1551 /* net->xfrm.xfrm_state_lock is held */ 1552 static struct xfrm_state *__find_acq_core(struct net *net, 1553 const struct xfrm_mark *m, 1554 unsigned short family, u8 mode, 1555 u32 reqid, u32 if_id, u8 proto, 1556 const xfrm_address_t *daddr, 1557 const xfrm_address_t *saddr, 1558 int create) 1559 { 1560 unsigned int h = xfrm_dst_hash(net, daddr, saddr, reqid, family); 1561 struct xfrm_state *x; 1562 u32 mark = m->v & m->m; 1563 1564 hlist_for_each_entry(x, net->xfrm.state_bydst+h, bydst) { 1565 if (x->props.reqid != reqid || 1566 x->props.mode != mode || 1567 x->props.family != family || 1568 x->km.state != XFRM_STATE_ACQ || 1569 x->id.spi != 0 || 1570 x->id.proto != proto || 1571 (mark & x->mark.m) != x->mark.v || 1572 !xfrm_addr_equal(&x->id.daddr, daddr, family) || 1573 !xfrm_addr_equal(&x->props.saddr, saddr, family)) 1574 continue; 1575 1576 xfrm_state_hold(x); 1577 return x; 1578 } 1579 1580 if (!create) 1581 return NULL; 1582 1583 x = xfrm_state_alloc(net); 1584 if (likely(x)) { 1585 switch (family) { 1586 case AF_INET: 1587 x->sel.daddr.a4 = daddr->a4; 1588 x->sel.saddr.a4 = saddr->a4; 1589 x->sel.prefixlen_d = 32; 1590 x->sel.prefixlen_s = 32; 1591 x->props.saddr.a4 = saddr->a4; 1592 x->id.daddr.a4 = daddr->a4; 1593 break; 1594 1595 case AF_INET6: 1596 x->sel.daddr.in6 = daddr->in6; 1597 x->sel.saddr.in6 = saddr->in6; 1598 x->sel.prefixlen_d = 128; 1599 x->sel.prefixlen_s = 128; 1600 x->props.saddr.in6 = saddr->in6; 1601 x->id.daddr.in6 = daddr->in6; 1602 break; 1603 } 1604 1605 x->km.state = XFRM_STATE_ACQ; 1606 x->id.proto = proto; 1607 x->props.family = family; 1608 x->props.mode = mode; 1609 x->props.reqid = reqid; 1610 x->if_id = if_id; 1611 x->mark.v = m->v; 1612 x->mark.m = m->m; 1613 x->lft.hard_add_expires_seconds = net->xfrm.sysctl_acq_expires; 1614 xfrm_state_hold(x); 1615 hrtimer_start(&x->mtimer, 1616 ktime_set(net->xfrm.sysctl_acq_expires, 0), 1617 HRTIMER_MODE_REL_SOFT); 1618 list_add(&x->km.all, &net->xfrm.state_all); 1619 XFRM_STATE_INSERT(bydst, &x->bydst, net->xfrm.state_bydst + h, 1620 x->xso.type); 1621 h = xfrm_src_hash(net, daddr, saddr, family); 1622 XFRM_STATE_INSERT(bysrc, &x->bysrc, net->xfrm.state_bysrc + h, 1623 x->xso.type); 1624 1625 net->xfrm.state_num++; 1626 1627 xfrm_hash_grow_check(net, x->bydst.next != NULL); 1628 } 1629 1630 return x; 1631 } 1632 1633 static struct xfrm_state *__xfrm_find_acq_byseq(struct net *net, u32 mark, u32 seq); 1634 1635 int xfrm_state_add(struct xfrm_state *x) 1636 { 1637 struct net *net = xs_net(x); 1638 struct xfrm_state *x1, *to_put; 1639 int family; 1640 int err; 1641 u32 mark = x->mark.v & x->mark.m; 1642 int use_spi = xfrm_id_proto_match(x->id.proto, IPSEC_PROTO_ANY); 1643 1644 family = x->props.family; 1645 1646 to_put = NULL; 1647 1648 spin_lock_bh(&net->xfrm.xfrm_state_lock); 1649 1650 x1 = __xfrm_state_locate(x, use_spi, family); 1651 if (x1) { 1652 to_put = x1; 1653 x1 = NULL; 1654 err = -EEXIST; 1655 goto out; 1656 } 1657 1658 if (use_spi && x->km.seq) { 1659 x1 = __xfrm_find_acq_byseq(net, mark, x->km.seq); 1660 if (x1 && ((x1->id.proto != x->id.proto) || 1661 !xfrm_addr_equal(&x1->id.daddr, &x->id.daddr, family))) { 1662 to_put = x1; 1663 x1 = NULL; 1664 } 1665 } 1666 1667 if (use_spi && !x1) 1668 x1 = __find_acq_core(net, &x->mark, family, x->props.mode, 1669 x->props.reqid, x->if_id, x->id.proto, 1670 &x->id.daddr, &x->props.saddr, 0); 1671 1672 __xfrm_state_bump_genids(x); 1673 __xfrm_state_insert(x); 1674 err = 0; 1675 1676 out: 1677 spin_unlock_bh(&net->xfrm.xfrm_state_lock); 1678 1679 if (x1) { 1680 xfrm_state_delete(x1); 1681 xfrm_state_put(x1); 1682 } 1683 1684 if (to_put) 1685 xfrm_state_put(to_put); 1686 1687 return err; 1688 } 1689 EXPORT_SYMBOL(xfrm_state_add); 1690 1691 #ifdef CONFIG_XFRM_MIGRATE 1692 static inline int clone_security(struct xfrm_state *x, struct xfrm_sec_ctx *security) 1693 { 1694 struct xfrm_user_sec_ctx *uctx; 1695 int size = sizeof(*uctx) + security->ctx_len; 1696 int err; 1697 1698 uctx = kmalloc(size, GFP_KERNEL); 1699 if (!uctx) 1700 return -ENOMEM; 1701 1702 uctx->exttype = XFRMA_SEC_CTX; 1703 uctx->len = size; 1704 uctx->ctx_doi = security->ctx_doi; 1705 uctx->ctx_alg = security->ctx_alg; 1706 uctx->ctx_len = security->ctx_len; 1707 memcpy(uctx + 1, security->ctx_str, security->ctx_len); 1708 err = security_xfrm_state_alloc(x, uctx); 1709 kfree(uctx); 1710 if (err) 1711 return err; 1712 1713 return 0; 1714 } 1715 1716 static struct xfrm_state *xfrm_state_clone(struct xfrm_state *orig, 1717 struct xfrm_encap_tmpl *encap) 1718 { 1719 struct net *net = xs_net(orig); 1720 struct xfrm_state *x = xfrm_state_alloc(net); 1721 if (!x) 1722 goto out; 1723 1724 memcpy(&x->id, &orig->id, sizeof(x->id)); 1725 memcpy(&x->sel, &orig->sel, sizeof(x->sel)); 1726 memcpy(&x->lft, &orig->lft, sizeof(x->lft)); 1727 x->props.mode = orig->props.mode; 1728 x->props.replay_window = orig->props.replay_window; 1729 x->props.reqid = orig->props.reqid; 1730 x->props.family = orig->props.family; 1731 x->props.saddr = orig->props.saddr; 1732 1733 if (orig->aalg) { 1734 x->aalg = xfrm_algo_auth_clone(orig->aalg); 1735 if (!x->aalg) 1736 goto error; 1737 } 1738 x->props.aalgo = orig->props.aalgo; 1739 1740 if (orig->aead) { 1741 x->aead = xfrm_algo_aead_clone(orig->aead); 1742 x->geniv = orig->geniv; 1743 if (!x->aead) 1744 goto error; 1745 } 1746 if (orig->ealg) { 1747 x->ealg = xfrm_algo_clone(orig->ealg); 1748 if (!x->ealg) 1749 goto error; 1750 } 1751 x->props.ealgo = orig->props.ealgo; 1752 1753 if (orig->calg) { 1754 x->calg = xfrm_algo_clone(orig->calg); 1755 if (!x->calg) 1756 goto error; 1757 } 1758 x->props.calgo = orig->props.calgo; 1759 1760 if (encap || orig->encap) { 1761 if (encap) 1762 x->encap = kmemdup(encap, sizeof(*x->encap), 1763 GFP_KERNEL); 1764 else 1765 x->encap = kmemdup(orig->encap, sizeof(*x->encap), 1766 GFP_KERNEL); 1767 1768 if (!x->encap) 1769 goto error; 1770 } 1771 1772 if (orig->security) 1773 if (clone_security(x, orig->security)) 1774 goto error; 1775 1776 if (orig->coaddr) { 1777 x->coaddr = kmemdup(orig->coaddr, sizeof(*x->coaddr), 1778 GFP_KERNEL); 1779 if (!x->coaddr) 1780 goto error; 1781 } 1782 1783 if (orig->replay_esn) { 1784 if (xfrm_replay_clone(x, orig)) 1785 goto error; 1786 } 1787 1788 memcpy(&x->mark, &orig->mark, sizeof(x->mark)); 1789 memcpy(&x->props.smark, &orig->props.smark, sizeof(x->props.smark)); 1790 1791 x->props.flags = orig->props.flags; 1792 x->props.extra_flags = orig->props.extra_flags; 1793 1794 x->if_id = orig->if_id; 1795 x->tfcpad = orig->tfcpad; 1796 x->replay_maxdiff = orig->replay_maxdiff; 1797 x->replay_maxage = orig->replay_maxage; 1798 memcpy(&x->curlft, &orig->curlft, sizeof(x->curlft)); 1799 x->km.state = orig->km.state; 1800 x->km.seq = orig->km.seq; 1801 x->replay = orig->replay; 1802 x->preplay = orig->preplay; 1803 x->mapping_maxage = orig->mapping_maxage; 1804 x->lastused = orig->lastused; 1805 x->new_mapping = 0; 1806 x->new_mapping_sport = 0; 1807 x->dir = orig->dir; 1808 1809 return x; 1810 1811 error: 1812 xfrm_state_put(x); 1813 out: 1814 return NULL; 1815 } 1816 1817 struct xfrm_state *xfrm_migrate_state_find(struct xfrm_migrate *m, struct net *net, 1818 u32 if_id) 1819 { 1820 unsigned int h; 1821 struct xfrm_state *x = NULL; 1822 1823 spin_lock_bh(&net->xfrm.xfrm_state_lock); 1824 1825 if (m->reqid) { 1826 h = xfrm_dst_hash(net, &m->old_daddr, &m->old_saddr, 1827 m->reqid, m->old_family); 1828 hlist_for_each_entry(x, net->xfrm.state_bydst+h, bydst) { 1829 if (x->props.mode != m->mode || 1830 x->id.proto != m->proto) 1831 continue; 1832 if (m->reqid && x->props.reqid != m->reqid) 1833 continue; 1834 if (if_id != 0 && x->if_id != if_id) 1835 continue; 1836 if (!xfrm_addr_equal(&x->id.daddr, &m->old_daddr, 1837 m->old_family) || 1838 !xfrm_addr_equal(&x->props.saddr, &m->old_saddr, 1839 m->old_family)) 1840 continue; 1841 xfrm_state_hold(x); 1842 break; 1843 } 1844 } else { 1845 h = xfrm_src_hash(net, &m->old_daddr, &m->old_saddr, 1846 m->old_family); 1847 hlist_for_each_entry(x, net->xfrm.state_bysrc+h, bysrc) { 1848 if (x->props.mode != m->mode || 1849 x->id.proto != m->proto) 1850 continue; 1851 if (if_id != 0 && x->if_id != if_id) 1852 continue; 1853 if (!xfrm_addr_equal(&x->id.daddr, &m->old_daddr, 1854 m->old_family) || 1855 !xfrm_addr_equal(&x->props.saddr, &m->old_saddr, 1856 m->old_family)) 1857 continue; 1858 xfrm_state_hold(x); 1859 break; 1860 } 1861 } 1862 1863 spin_unlock_bh(&net->xfrm.xfrm_state_lock); 1864 1865 return x; 1866 } 1867 EXPORT_SYMBOL(xfrm_migrate_state_find); 1868 1869 struct xfrm_state *xfrm_state_migrate(struct xfrm_state *x, 1870 struct xfrm_migrate *m, 1871 struct xfrm_encap_tmpl *encap) 1872 { 1873 struct xfrm_state *xc; 1874 1875 xc = xfrm_state_clone(x, encap); 1876 if (!xc) 1877 return NULL; 1878 1879 xc->props.family = m->new_family; 1880 1881 if (xfrm_init_state(xc) < 0) 1882 goto error; 1883 1884 memcpy(&xc->id.daddr, &m->new_daddr, sizeof(xc->id.daddr)); 1885 memcpy(&xc->props.saddr, &m->new_saddr, sizeof(xc->props.saddr)); 1886 1887 /* add state */ 1888 if (xfrm_addr_equal(&x->id.daddr, &m->new_daddr, m->new_family)) { 1889 /* a care is needed when the destination address of the 1890 state is to be updated as it is a part of triplet */ 1891 xfrm_state_insert(xc); 1892 } else { 1893 if (xfrm_state_add(xc) < 0) 1894 goto error; 1895 } 1896 1897 return xc; 1898 error: 1899 xfrm_state_put(xc); 1900 return NULL; 1901 } 1902 EXPORT_SYMBOL(xfrm_state_migrate); 1903 #endif 1904 1905 int xfrm_state_update(struct xfrm_state *x) 1906 { 1907 struct xfrm_state *x1, *to_put; 1908 int err; 1909 int use_spi = xfrm_id_proto_match(x->id.proto, IPSEC_PROTO_ANY); 1910 struct net *net = xs_net(x); 1911 1912 to_put = NULL; 1913 1914 spin_lock_bh(&net->xfrm.xfrm_state_lock); 1915 x1 = __xfrm_state_locate(x, use_spi, x->props.family); 1916 1917 err = -ESRCH; 1918 if (!x1) 1919 goto out; 1920 1921 if (xfrm_state_kern(x1)) { 1922 to_put = x1; 1923 err = -EEXIST; 1924 goto out; 1925 } 1926 1927 if (x1->km.state == XFRM_STATE_ACQ) { 1928 if (x->dir && x1->dir != x->dir) 1929 goto out; 1930 1931 __xfrm_state_insert(x); 1932 x = NULL; 1933 } else { 1934 if (x1->dir != x->dir) 1935 goto out; 1936 } 1937 err = 0; 1938 1939 out: 1940 spin_unlock_bh(&net->xfrm.xfrm_state_lock); 1941 1942 if (to_put) 1943 xfrm_state_put(to_put); 1944 1945 if (err) 1946 return err; 1947 1948 if (!x) { 1949 xfrm_state_delete(x1); 1950 xfrm_state_put(x1); 1951 return 0; 1952 } 1953 1954 err = -EINVAL; 1955 spin_lock_bh(&x1->lock); 1956 if (likely(x1->km.state == XFRM_STATE_VALID)) { 1957 if (x->encap && x1->encap && 1958 x->encap->encap_type == x1->encap->encap_type) 1959 memcpy(x1->encap, x->encap, sizeof(*x1->encap)); 1960 else if (x->encap || x1->encap) 1961 goto fail; 1962 1963 if (x->coaddr && x1->coaddr) { 1964 memcpy(x1->coaddr, x->coaddr, sizeof(*x1->coaddr)); 1965 } 1966 if (!use_spi && memcmp(&x1->sel, &x->sel, sizeof(x1->sel))) 1967 memcpy(&x1->sel, &x->sel, sizeof(x1->sel)); 1968 memcpy(&x1->lft, &x->lft, sizeof(x1->lft)); 1969 x1->km.dying = 0; 1970 1971 hrtimer_start(&x1->mtimer, ktime_set(1, 0), 1972 HRTIMER_MODE_REL_SOFT); 1973 if (READ_ONCE(x1->curlft.use_time)) 1974 xfrm_state_check_expire(x1); 1975 1976 if (x->props.smark.m || x->props.smark.v || x->if_id) { 1977 spin_lock_bh(&net->xfrm.xfrm_state_lock); 1978 1979 if (x->props.smark.m || x->props.smark.v) 1980 x1->props.smark = x->props.smark; 1981 1982 if (x->if_id) 1983 x1->if_id = x->if_id; 1984 1985 __xfrm_state_bump_genids(x1); 1986 spin_unlock_bh(&net->xfrm.xfrm_state_lock); 1987 } 1988 1989 err = 0; 1990 x->km.state = XFRM_STATE_DEAD; 1991 __xfrm_state_put(x); 1992 } 1993 1994 fail: 1995 spin_unlock_bh(&x1->lock); 1996 1997 xfrm_state_put(x1); 1998 1999 return err; 2000 } 2001 EXPORT_SYMBOL(xfrm_state_update); 2002 2003 int xfrm_state_check_expire(struct xfrm_state *x) 2004 { 2005 xfrm_dev_state_update_stats(x); 2006 2007 if (!READ_ONCE(x->curlft.use_time)) 2008 WRITE_ONCE(x->curlft.use_time, ktime_get_real_seconds()); 2009 2010 if (x->curlft.bytes >= x->lft.hard_byte_limit || 2011 x->curlft.packets >= x->lft.hard_packet_limit) { 2012 x->km.state = XFRM_STATE_EXPIRED; 2013 hrtimer_start(&x->mtimer, 0, HRTIMER_MODE_REL_SOFT); 2014 return -EINVAL; 2015 } 2016 2017 if (!x->km.dying && 2018 (x->curlft.bytes >= x->lft.soft_byte_limit || 2019 x->curlft.packets >= x->lft.soft_packet_limit)) { 2020 x->km.dying = 1; 2021 km_state_expired(x, 0, 0); 2022 } 2023 return 0; 2024 } 2025 EXPORT_SYMBOL(xfrm_state_check_expire); 2026 2027 void xfrm_state_update_stats(struct net *net) 2028 { 2029 struct xfrm_state *x; 2030 int i; 2031 2032 spin_lock_bh(&net->xfrm.xfrm_state_lock); 2033 for (i = 0; i <= net->xfrm.state_hmask; i++) { 2034 hlist_for_each_entry(x, net->xfrm.state_bydst + i, bydst) 2035 xfrm_dev_state_update_stats(x); 2036 } 2037 spin_unlock_bh(&net->xfrm.xfrm_state_lock); 2038 } 2039 2040 struct xfrm_state * 2041 xfrm_state_lookup(struct net *net, u32 mark, const xfrm_address_t *daddr, __be32 spi, 2042 u8 proto, unsigned short family) 2043 { 2044 struct xfrm_state *x; 2045 2046 rcu_read_lock(); 2047 x = __xfrm_state_lookup(net, mark, daddr, spi, proto, family); 2048 rcu_read_unlock(); 2049 return x; 2050 } 2051 EXPORT_SYMBOL(xfrm_state_lookup); 2052 2053 struct xfrm_state * 2054 xfrm_state_lookup_byaddr(struct net *net, u32 mark, 2055 const xfrm_address_t *daddr, const xfrm_address_t *saddr, 2056 u8 proto, unsigned short family) 2057 { 2058 struct xfrm_state *x; 2059 2060 spin_lock_bh(&net->xfrm.xfrm_state_lock); 2061 x = __xfrm_state_lookup_byaddr(net, mark, daddr, saddr, proto, family); 2062 spin_unlock_bh(&net->xfrm.xfrm_state_lock); 2063 return x; 2064 } 2065 EXPORT_SYMBOL(xfrm_state_lookup_byaddr); 2066 2067 struct xfrm_state * 2068 xfrm_find_acq(struct net *net, const struct xfrm_mark *mark, u8 mode, u32 reqid, 2069 u32 if_id, u8 proto, const xfrm_address_t *daddr, 2070 const xfrm_address_t *saddr, int create, unsigned short family) 2071 { 2072 struct xfrm_state *x; 2073 2074 spin_lock_bh(&net->xfrm.xfrm_state_lock); 2075 x = __find_acq_core(net, mark, family, mode, reqid, if_id, proto, daddr, saddr, create); 2076 spin_unlock_bh(&net->xfrm.xfrm_state_lock); 2077 2078 return x; 2079 } 2080 EXPORT_SYMBOL(xfrm_find_acq); 2081 2082 #ifdef CONFIG_XFRM_SUB_POLICY 2083 #if IS_ENABLED(CONFIG_IPV6) 2084 /* distribution counting sort function for xfrm_state and xfrm_tmpl */ 2085 static void 2086 __xfrm6_sort(void **dst, void **src, int n, 2087 int (*cmp)(const void *p), int maxclass) 2088 { 2089 int count[XFRM_MAX_DEPTH] = { }; 2090 int class[XFRM_MAX_DEPTH]; 2091 int i; 2092 2093 for (i = 0; i < n; i++) { 2094 int c = cmp(src[i]); 2095 2096 class[i] = c; 2097 count[c]++; 2098 } 2099 2100 for (i = 2; i < maxclass; i++) 2101 count[i] += count[i - 1]; 2102 2103 for (i = 0; i < n; i++) { 2104 dst[count[class[i] - 1]++] = src[i]; 2105 src[i] = NULL; 2106 } 2107 } 2108 2109 /* Rule for xfrm_state: 2110 * 2111 * rule 1: select IPsec transport except AH 2112 * rule 2: select MIPv6 RO or inbound trigger 2113 * rule 3: select IPsec transport AH 2114 * rule 4: select IPsec tunnel 2115 * rule 5: others 2116 */ 2117 static int __xfrm6_state_sort_cmp(const void *p) 2118 { 2119 const struct xfrm_state *v = p; 2120 2121 switch (v->props.mode) { 2122 case XFRM_MODE_TRANSPORT: 2123 if (v->id.proto != IPPROTO_AH) 2124 return 1; 2125 else 2126 return 3; 2127 #if IS_ENABLED(CONFIG_IPV6_MIP6) 2128 case XFRM_MODE_ROUTEOPTIMIZATION: 2129 case XFRM_MODE_IN_TRIGGER: 2130 return 2; 2131 #endif 2132 case XFRM_MODE_TUNNEL: 2133 case XFRM_MODE_BEET: 2134 return 4; 2135 } 2136 return 5; 2137 } 2138 2139 /* Rule for xfrm_tmpl: 2140 * 2141 * rule 1: select IPsec transport 2142 * rule 2: select MIPv6 RO or inbound trigger 2143 * rule 3: select IPsec tunnel 2144 * rule 4: others 2145 */ 2146 static int __xfrm6_tmpl_sort_cmp(const void *p) 2147 { 2148 const struct xfrm_tmpl *v = p; 2149 2150 switch (v->mode) { 2151 case XFRM_MODE_TRANSPORT: 2152 return 1; 2153 #if IS_ENABLED(CONFIG_IPV6_MIP6) 2154 case XFRM_MODE_ROUTEOPTIMIZATION: 2155 case XFRM_MODE_IN_TRIGGER: 2156 return 2; 2157 #endif 2158 case XFRM_MODE_TUNNEL: 2159 case XFRM_MODE_BEET: 2160 return 3; 2161 } 2162 return 4; 2163 } 2164 #else 2165 static inline int __xfrm6_state_sort_cmp(const void *p) { return 5; } 2166 static inline int __xfrm6_tmpl_sort_cmp(const void *p) { return 4; } 2167 2168 static inline void 2169 __xfrm6_sort(void **dst, void **src, int n, 2170 int (*cmp)(const void *p), int maxclass) 2171 { 2172 int i; 2173 2174 for (i = 0; i < n; i++) 2175 dst[i] = src[i]; 2176 } 2177 #endif /* CONFIG_IPV6 */ 2178 2179 void 2180 xfrm_tmpl_sort(struct xfrm_tmpl **dst, struct xfrm_tmpl **src, int n, 2181 unsigned short family) 2182 { 2183 int i; 2184 2185 if (family == AF_INET6) 2186 __xfrm6_sort((void **)dst, (void **)src, n, 2187 __xfrm6_tmpl_sort_cmp, 5); 2188 else 2189 for (i = 0; i < n; i++) 2190 dst[i] = src[i]; 2191 } 2192 2193 void 2194 xfrm_state_sort(struct xfrm_state **dst, struct xfrm_state **src, int n, 2195 unsigned short family) 2196 { 2197 int i; 2198 2199 if (family == AF_INET6) 2200 __xfrm6_sort((void **)dst, (void **)src, n, 2201 __xfrm6_state_sort_cmp, 6); 2202 else 2203 for (i = 0; i < n; i++) 2204 dst[i] = src[i]; 2205 } 2206 #endif 2207 2208 /* Silly enough, but I'm lazy to build resolution list */ 2209 2210 static struct xfrm_state *__xfrm_find_acq_byseq(struct net *net, u32 mark, u32 seq) 2211 { 2212 unsigned int h = xfrm_seq_hash(net, seq); 2213 struct xfrm_state *x; 2214 2215 hlist_for_each_entry_rcu(x, net->xfrm.state_byseq + h, byseq) { 2216 if (x->km.seq == seq && 2217 (mark & x->mark.m) == x->mark.v && 2218 x->km.state == XFRM_STATE_ACQ) { 2219 xfrm_state_hold(x); 2220 return x; 2221 } 2222 } 2223 2224 return NULL; 2225 } 2226 2227 struct xfrm_state *xfrm_find_acq_byseq(struct net *net, u32 mark, u32 seq) 2228 { 2229 struct xfrm_state *x; 2230 2231 spin_lock_bh(&net->xfrm.xfrm_state_lock); 2232 x = __xfrm_find_acq_byseq(net, mark, seq); 2233 spin_unlock_bh(&net->xfrm.xfrm_state_lock); 2234 return x; 2235 } 2236 EXPORT_SYMBOL(xfrm_find_acq_byseq); 2237 2238 u32 xfrm_get_acqseq(void) 2239 { 2240 u32 res; 2241 static atomic_t acqseq; 2242 2243 do { 2244 res = atomic_inc_return(&acqseq); 2245 } while (!res); 2246 2247 return res; 2248 } 2249 EXPORT_SYMBOL(xfrm_get_acqseq); 2250 2251 int verify_spi_info(u8 proto, u32 min, u32 max, struct netlink_ext_ack *extack) 2252 { 2253 switch (proto) { 2254 case IPPROTO_AH: 2255 case IPPROTO_ESP: 2256 break; 2257 2258 case IPPROTO_COMP: 2259 /* IPCOMP spi is 16-bits. */ 2260 if (max >= 0x10000) { 2261 NL_SET_ERR_MSG(extack, "IPCOMP SPI must be <= 65535"); 2262 return -EINVAL; 2263 } 2264 break; 2265 2266 default: 2267 NL_SET_ERR_MSG(extack, "Invalid protocol, must be one of AH, ESP, IPCOMP"); 2268 return -EINVAL; 2269 } 2270 2271 if (min > max) { 2272 NL_SET_ERR_MSG(extack, "Invalid SPI range: min > max"); 2273 return -EINVAL; 2274 } 2275 2276 return 0; 2277 } 2278 EXPORT_SYMBOL(verify_spi_info); 2279 2280 int xfrm_alloc_spi(struct xfrm_state *x, u32 low, u32 high, 2281 struct netlink_ext_ack *extack) 2282 { 2283 struct net *net = xs_net(x); 2284 unsigned int h; 2285 struct xfrm_state *x0; 2286 int err = -ENOENT; 2287 __be32 minspi = htonl(low); 2288 __be32 maxspi = htonl(high); 2289 __be32 newspi = 0; 2290 u32 mark = x->mark.v & x->mark.m; 2291 2292 spin_lock_bh(&x->lock); 2293 if (x->km.state == XFRM_STATE_DEAD) { 2294 NL_SET_ERR_MSG(extack, "Target ACQUIRE is in DEAD state"); 2295 goto unlock; 2296 } 2297 2298 err = 0; 2299 if (x->id.spi) 2300 goto unlock; 2301 2302 err = -ENOENT; 2303 2304 if (minspi == maxspi) { 2305 x0 = xfrm_state_lookup(net, mark, &x->id.daddr, minspi, x->id.proto, x->props.family); 2306 if (x0) { 2307 NL_SET_ERR_MSG(extack, "Requested SPI is already in use"); 2308 xfrm_state_put(x0); 2309 goto unlock; 2310 } 2311 newspi = minspi; 2312 } else { 2313 u32 spi = 0; 2314 for (h = 0; h < high-low+1; h++) { 2315 spi = get_random_u32_inclusive(low, high); 2316 x0 = xfrm_state_lookup(net, mark, &x->id.daddr, htonl(spi), x->id.proto, x->props.family); 2317 if (x0 == NULL) { 2318 newspi = htonl(spi); 2319 break; 2320 } 2321 xfrm_state_put(x0); 2322 } 2323 } 2324 if (newspi) { 2325 spin_lock_bh(&net->xfrm.xfrm_state_lock); 2326 x->id.spi = newspi; 2327 h = xfrm_spi_hash(net, &x->id.daddr, x->id.spi, x->id.proto, x->props.family); 2328 XFRM_STATE_INSERT(byspi, &x->byspi, net->xfrm.state_byspi + h, 2329 x->xso.type); 2330 spin_unlock_bh(&net->xfrm.xfrm_state_lock); 2331 2332 err = 0; 2333 } else { 2334 NL_SET_ERR_MSG(extack, "No SPI available in the requested range"); 2335 } 2336 2337 unlock: 2338 spin_unlock_bh(&x->lock); 2339 2340 return err; 2341 } 2342 EXPORT_SYMBOL(xfrm_alloc_spi); 2343 2344 static bool __xfrm_state_filter_match(struct xfrm_state *x, 2345 struct xfrm_address_filter *filter) 2346 { 2347 if (filter) { 2348 if ((filter->family == AF_INET || 2349 filter->family == AF_INET6) && 2350 x->props.family != filter->family) 2351 return false; 2352 2353 return addr_match(&x->props.saddr, &filter->saddr, 2354 filter->splen) && 2355 addr_match(&x->id.daddr, &filter->daddr, 2356 filter->dplen); 2357 } 2358 return true; 2359 } 2360 2361 int xfrm_state_walk(struct net *net, struct xfrm_state_walk *walk, 2362 int (*func)(struct xfrm_state *, int, void*), 2363 void *data) 2364 { 2365 struct xfrm_state *state; 2366 struct xfrm_state_walk *x; 2367 int err = 0; 2368 2369 if (walk->seq != 0 && list_empty(&walk->all)) 2370 return 0; 2371 2372 spin_lock_bh(&net->xfrm.xfrm_state_lock); 2373 if (list_empty(&walk->all)) 2374 x = list_first_entry(&net->xfrm.state_all, struct xfrm_state_walk, all); 2375 else 2376 x = list_first_entry(&walk->all, struct xfrm_state_walk, all); 2377 list_for_each_entry_from(x, &net->xfrm.state_all, all) { 2378 if (x->state == XFRM_STATE_DEAD) 2379 continue; 2380 state = container_of(x, struct xfrm_state, km); 2381 if (!xfrm_id_proto_match(state->id.proto, walk->proto)) 2382 continue; 2383 if (!__xfrm_state_filter_match(state, walk->filter)) 2384 continue; 2385 err = func(state, walk->seq, data); 2386 if (err) { 2387 list_move_tail(&walk->all, &x->all); 2388 goto out; 2389 } 2390 walk->seq++; 2391 } 2392 if (walk->seq == 0) { 2393 err = -ENOENT; 2394 goto out; 2395 } 2396 list_del_init(&walk->all); 2397 out: 2398 spin_unlock_bh(&net->xfrm.xfrm_state_lock); 2399 return err; 2400 } 2401 EXPORT_SYMBOL(xfrm_state_walk); 2402 2403 void xfrm_state_walk_init(struct xfrm_state_walk *walk, u8 proto, 2404 struct xfrm_address_filter *filter) 2405 { 2406 INIT_LIST_HEAD(&walk->all); 2407 walk->proto = proto; 2408 walk->state = XFRM_STATE_DEAD; 2409 walk->seq = 0; 2410 walk->filter = filter; 2411 } 2412 EXPORT_SYMBOL(xfrm_state_walk_init); 2413 2414 void xfrm_state_walk_done(struct xfrm_state_walk *walk, struct net *net) 2415 { 2416 kfree(walk->filter); 2417 2418 if (list_empty(&walk->all)) 2419 return; 2420 2421 spin_lock_bh(&net->xfrm.xfrm_state_lock); 2422 list_del(&walk->all); 2423 spin_unlock_bh(&net->xfrm.xfrm_state_lock); 2424 } 2425 EXPORT_SYMBOL(xfrm_state_walk_done); 2426 2427 static void xfrm_replay_timer_handler(struct timer_list *t) 2428 { 2429 struct xfrm_state *x = from_timer(x, t, rtimer); 2430 2431 spin_lock(&x->lock); 2432 2433 if (x->km.state == XFRM_STATE_VALID) { 2434 if (xfrm_aevent_is_on(xs_net(x))) 2435 xfrm_replay_notify(x, XFRM_REPLAY_TIMEOUT); 2436 else 2437 x->xflags |= XFRM_TIME_DEFER; 2438 } 2439 2440 spin_unlock(&x->lock); 2441 } 2442 2443 static LIST_HEAD(xfrm_km_list); 2444 2445 void km_policy_notify(struct xfrm_policy *xp, int dir, const struct km_event *c) 2446 { 2447 struct xfrm_mgr *km; 2448 2449 rcu_read_lock(); 2450 list_for_each_entry_rcu(km, &xfrm_km_list, list) 2451 if (km->notify_policy) 2452 km->notify_policy(xp, dir, c); 2453 rcu_read_unlock(); 2454 } 2455 2456 void km_state_notify(struct xfrm_state *x, const struct km_event *c) 2457 { 2458 struct xfrm_mgr *km; 2459 rcu_read_lock(); 2460 list_for_each_entry_rcu(km, &xfrm_km_list, list) 2461 if (km->notify) 2462 km->notify(x, c); 2463 rcu_read_unlock(); 2464 } 2465 2466 EXPORT_SYMBOL(km_policy_notify); 2467 EXPORT_SYMBOL(km_state_notify); 2468 2469 void km_state_expired(struct xfrm_state *x, int hard, u32 portid) 2470 { 2471 struct km_event c; 2472 2473 c.data.hard = hard; 2474 c.portid = portid; 2475 c.event = XFRM_MSG_EXPIRE; 2476 km_state_notify(x, &c); 2477 } 2478 2479 EXPORT_SYMBOL(km_state_expired); 2480 /* 2481 * We send to all registered managers regardless of failure 2482 * We are happy with one success 2483 */ 2484 int km_query(struct xfrm_state *x, struct xfrm_tmpl *t, struct xfrm_policy *pol) 2485 { 2486 int err = -EINVAL, acqret; 2487 struct xfrm_mgr *km; 2488 2489 rcu_read_lock(); 2490 list_for_each_entry_rcu(km, &xfrm_km_list, list) { 2491 acqret = km->acquire(x, t, pol); 2492 if (!acqret) 2493 err = acqret; 2494 } 2495 rcu_read_unlock(); 2496 return err; 2497 } 2498 EXPORT_SYMBOL(km_query); 2499 2500 static int __km_new_mapping(struct xfrm_state *x, xfrm_address_t *ipaddr, __be16 sport) 2501 { 2502 int err = -EINVAL; 2503 struct xfrm_mgr *km; 2504 2505 rcu_read_lock(); 2506 list_for_each_entry_rcu(km, &xfrm_km_list, list) { 2507 if (km->new_mapping) 2508 err = km->new_mapping(x, ipaddr, sport); 2509 if (!err) 2510 break; 2511 } 2512 rcu_read_unlock(); 2513 return err; 2514 } 2515 2516 int km_new_mapping(struct xfrm_state *x, xfrm_address_t *ipaddr, __be16 sport) 2517 { 2518 int ret = 0; 2519 2520 if (x->mapping_maxage) { 2521 if ((jiffies / HZ - x->new_mapping) > x->mapping_maxage || 2522 x->new_mapping_sport != sport) { 2523 x->new_mapping_sport = sport; 2524 x->new_mapping = jiffies / HZ; 2525 ret = __km_new_mapping(x, ipaddr, sport); 2526 } 2527 } else { 2528 ret = __km_new_mapping(x, ipaddr, sport); 2529 } 2530 2531 return ret; 2532 } 2533 EXPORT_SYMBOL(km_new_mapping); 2534 2535 void km_policy_expired(struct xfrm_policy *pol, int dir, int hard, u32 portid) 2536 { 2537 struct km_event c; 2538 2539 c.data.hard = hard; 2540 c.portid = portid; 2541 c.event = XFRM_MSG_POLEXPIRE; 2542 km_policy_notify(pol, dir, &c); 2543 } 2544 EXPORT_SYMBOL(km_policy_expired); 2545 2546 #ifdef CONFIG_XFRM_MIGRATE 2547 int km_migrate(const struct xfrm_selector *sel, u8 dir, u8 type, 2548 const struct xfrm_migrate *m, int num_migrate, 2549 const struct xfrm_kmaddress *k, 2550 const struct xfrm_encap_tmpl *encap) 2551 { 2552 int err = -EINVAL; 2553 int ret; 2554 struct xfrm_mgr *km; 2555 2556 rcu_read_lock(); 2557 list_for_each_entry_rcu(km, &xfrm_km_list, list) { 2558 if (km->migrate) { 2559 ret = km->migrate(sel, dir, type, m, num_migrate, k, 2560 encap); 2561 if (!ret) 2562 err = ret; 2563 } 2564 } 2565 rcu_read_unlock(); 2566 return err; 2567 } 2568 EXPORT_SYMBOL(km_migrate); 2569 #endif 2570 2571 int km_report(struct net *net, u8 proto, struct xfrm_selector *sel, xfrm_address_t *addr) 2572 { 2573 int err = -EINVAL; 2574 int ret; 2575 struct xfrm_mgr *km; 2576 2577 rcu_read_lock(); 2578 list_for_each_entry_rcu(km, &xfrm_km_list, list) { 2579 if (km->report) { 2580 ret = km->report(net, proto, sel, addr); 2581 if (!ret) 2582 err = ret; 2583 } 2584 } 2585 rcu_read_unlock(); 2586 return err; 2587 } 2588 EXPORT_SYMBOL(km_report); 2589 2590 static bool km_is_alive(const struct km_event *c) 2591 { 2592 struct xfrm_mgr *km; 2593 bool is_alive = false; 2594 2595 rcu_read_lock(); 2596 list_for_each_entry_rcu(km, &xfrm_km_list, list) { 2597 if (km->is_alive && km->is_alive(c)) { 2598 is_alive = true; 2599 break; 2600 } 2601 } 2602 rcu_read_unlock(); 2603 2604 return is_alive; 2605 } 2606 2607 #if IS_ENABLED(CONFIG_XFRM_USER_COMPAT) 2608 static DEFINE_SPINLOCK(xfrm_translator_lock); 2609 static struct xfrm_translator __rcu *xfrm_translator; 2610 2611 struct xfrm_translator *xfrm_get_translator(void) 2612 { 2613 struct xfrm_translator *xtr; 2614 2615 rcu_read_lock(); 2616 xtr = rcu_dereference(xfrm_translator); 2617 if (unlikely(!xtr)) 2618 goto out; 2619 if (!try_module_get(xtr->owner)) 2620 xtr = NULL; 2621 out: 2622 rcu_read_unlock(); 2623 return xtr; 2624 } 2625 EXPORT_SYMBOL_GPL(xfrm_get_translator); 2626 2627 void xfrm_put_translator(struct xfrm_translator *xtr) 2628 { 2629 module_put(xtr->owner); 2630 } 2631 EXPORT_SYMBOL_GPL(xfrm_put_translator); 2632 2633 int xfrm_register_translator(struct xfrm_translator *xtr) 2634 { 2635 int err = 0; 2636 2637 spin_lock_bh(&xfrm_translator_lock); 2638 if (unlikely(xfrm_translator != NULL)) 2639 err = -EEXIST; 2640 else 2641 rcu_assign_pointer(xfrm_translator, xtr); 2642 spin_unlock_bh(&xfrm_translator_lock); 2643 2644 return err; 2645 } 2646 EXPORT_SYMBOL_GPL(xfrm_register_translator); 2647 2648 int xfrm_unregister_translator(struct xfrm_translator *xtr) 2649 { 2650 int err = 0; 2651 2652 spin_lock_bh(&xfrm_translator_lock); 2653 if (likely(xfrm_translator != NULL)) { 2654 if (rcu_access_pointer(xfrm_translator) != xtr) 2655 err = -EINVAL; 2656 else 2657 RCU_INIT_POINTER(xfrm_translator, NULL); 2658 } 2659 spin_unlock_bh(&xfrm_translator_lock); 2660 synchronize_rcu(); 2661 2662 return err; 2663 } 2664 EXPORT_SYMBOL_GPL(xfrm_unregister_translator); 2665 #endif 2666 2667 int xfrm_user_policy(struct sock *sk, int optname, sockptr_t optval, int optlen) 2668 { 2669 int err; 2670 u8 *data; 2671 struct xfrm_mgr *km; 2672 struct xfrm_policy *pol = NULL; 2673 2674 if (sockptr_is_null(optval) && !optlen) { 2675 xfrm_sk_policy_insert(sk, XFRM_POLICY_IN, NULL); 2676 xfrm_sk_policy_insert(sk, XFRM_POLICY_OUT, NULL); 2677 __sk_dst_reset(sk); 2678 return 0; 2679 } 2680 2681 if (optlen <= 0 || optlen > PAGE_SIZE) 2682 return -EMSGSIZE; 2683 2684 data = memdup_sockptr(optval, optlen); 2685 if (IS_ERR(data)) 2686 return PTR_ERR(data); 2687 2688 if (in_compat_syscall()) { 2689 struct xfrm_translator *xtr = xfrm_get_translator(); 2690 2691 if (!xtr) { 2692 kfree(data); 2693 return -EOPNOTSUPP; 2694 } 2695 2696 err = xtr->xlate_user_policy_sockptr(&data, optlen); 2697 xfrm_put_translator(xtr); 2698 if (err) { 2699 kfree(data); 2700 return err; 2701 } 2702 } 2703 2704 err = -EINVAL; 2705 rcu_read_lock(); 2706 list_for_each_entry_rcu(km, &xfrm_km_list, list) { 2707 pol = km->compile_policy(sk, optname, data, 2708 optlen, &err); 2709 if (err >= 0) 2710 break; 2711 } 2712 rcu_read_unlock(); 2713 2714 if (err >= 0) { 2715 xfrm_sk_policy_insert(sk, err, pol); 2716 xfrm_pol_put(pol); 2717 __sk_dst_reset(sk); 2718 err = 0; 2719 } 2720 2721 kfree(data); 2722 return err; 2723 } 2724 EXPORT_SYMBOL(xfrm_user_policy); 2725 2726 static DEFINE_SPINLOCK(xfrm_km_lock); 2727 2728 void xfrm_register_km(struct xfrm_mgr *km) 2729 { 2730 spin_lock_bh(&xfrm_km_lock); 2731 list_add_tail_rcu(&km->list, &xfrm_km_list); 2732 spin_unlock_bh(&xfrm_km_lock); 2733 } 2734 EXPORT_SYMBOL(xfrm_register_km); 2735 2736 void xfrm_unregister_km(struct xfrm_mgr *km) 2737 { 2738 spin_lock_bh(&xfrm_km_lock); 2739 list_del_rcu(&km->list); 2740 spin_unlock_bh(&xfrm_km_lock); 2741 synchronize_rcu(); 2742 } 2743 EXPORT_SYMBOL(xfrm_unregister_km); 2744 2745 int xfrm_state_register_afinfo(struct xfrm_state_afinfo *afinfo) 2746 { 2747 int err = 0; 2748 2749 if (WARN_ON(afinfo->family >= NPROTO)) 2750 return -EAFNOSUPPORT; 2751 2752 spin_lock_bh(&xfrm_state_afinfo_lock); 2753 if (unlikely(xfrm_state_afinfo[afinfo->family] != NULL)) 2754 err = -EEXIST; 2755 else 2756 rcu_assign_pointer(xfrm_state_afinfo[afinfo->family], afinfo); 2757 spin_unlock_bh(&xfrm_state_afinfo_lock); 2758 return err; 2759 } 2760 EXPORT_SYMBOL(xfrm_state_register_afinfo); 2761 2762 int xfrm_state_unregister_afinfo(struct xfrm_state_afinfo *afinfo) 2763 { 2764 int err = 0, family = afinfo->family; 2765 2766 if (WARN_ON(family >= NPROTO)) 2767 return -EAFNOSUPPORT; 2768 2769 spin_lock_bh(&xfrm_state_afinfo_lock); 2770 if (likely(xfrm_state_afinfo[afinfo->family] != NULL)) { 2771 if (rcu_access_pointer(xfrm_state_afinfo[family]) != afinfo) 2772 err = -EINVAL; 2773 else 2774 RCU_INIT_POINTER(xfrm_state_afinfo[afinfo->family], NULL); 2775 } 2776 spin_unlock_bh(&xfrm_state_afinfo_lock); 2777 synchronize_rcu(); 2778 return err; 2779 } 2780 EXPORT_SYMBOL(xfrm_state_unregister_afinfo); 2781 2782 struct xfrm_state_afinfo *xfrm_state_afinfo_get_rcu(unsigned int family) 2783 { 2784 if (unlikely(family >= NPROTO)) 2785 return NULL; 2786 2787 return rcu_dereference(xfrm_state_afinfo[family]); 2788 } 2789 EXPORT_SYMBOL_GPL(xfrm_state_afinfo_get_rcu); 2790 2791 struct xfrm_state_afinfo *xfrm_state_get_afinfo(unsigned int family) 2792 { 2793 struct xfrm_state_afinfo *afinfo; 2794 if (unlikely(family >= NPROTO)) 2795 return NULL; 2796 rcu_read_lock(); 2797 afinfo = rcu_dereference(xfrm_state_afinfo[family]); 2798 if (unlikely(!afinfo)) 2799 rcu_read_unlock(); 2800 return afinfo; 2801 } 2802 2803 void xfrm_flush_gc(void) 2804 { 2805 flush_work(&xfrm_state_gc_work); 2806 } 2807 EXPORT_SYMBOL(xfrm_flush_gc); 2808 2809 /* Temporarily located here until net/xfrm/xfrm_tunnel.c is created */ 2810 void xfrm_state_delete_tunnel(struct xfrm_state *x) 2811 { 2812 if (x->tunnel) { 2813 struct xfrm_state *t = x->tunnel; 2814 2815 if (atomic_read(&t->tunnel_users) == 2) 2816 xfrm_state_delete(t); 2817 atomic_dec(&t->tunnel_users); 2818 xfrm_state_put_sync(t); 2819 x->tunnel = NULL; 2820 } 2821 } 2822 EXPORT_SYMBOL(xfrm_state_delete_tunnel); 2823 2824 u32 xfrm_state_mtu(struct xfrm_state *x, int mtu) 2825 { 2826 const struct xfrm_type *type = READ_ONCE(x->type); 2827 struct crypto_aead *aead; 2828 u32 blksize, net_adj = 0; 2829 2830 if (x->km.state != XFRM_STATE_VALID || 2831 !type || type->proto != IPPROTO_ESP) 2832 return mtu - x->props.header_len; 2833 2834 aead = x->data; 2835 blksize = ALIGN(crypto_aead_blocksize(aead), 4); 2836 2837 switch (x->props.mode) { 2838 case XFRM_MODE_TRANSPORT: 2839 case XFRM_MODE_BEET: 2840 if (x->props.family == AF_INET) 2841 net_adj = sizeof(struct iphdr); 2842 else if (x->props.family == AF_INET6) 2843 net_adj = sizeof(struct ipv6hdr); 2844 break; 2845 case XFRM_MODE_TUNNEL: 2846 break; 2847 default: 2848 WARN_ON_ONCE(1); 2849 break; 2850 } 2851 2852 return ((mtu - x->props.header_len - crypto_aead_authsize(aead) - 2853 net_adj) & ~(blksize - 1)) + net_adj - 2; 2854 } 2855 EXPORT_SYMBOL_GPL(xfrm_state_mtu); 2856 2857 int __xfrm_init_state(struct xfrm_state *x, bool init_replay, bool offload, 2858 struct netlink_ext_ack *extack) 2859 { 2860 const struct xfrm_mode *inner_mode; 2861 const struct xfrm_mode *outer_mode; 2862 int family = x->props.family; 2863 int err; 2864 2865 if (family == AF_INET && 2866 READ_ONCE(xs_net(x)->ipv4.sysctl_ip_no_pmtu_disc)) 2867 x->props.flags |= XFRM_STATE_NOPMTUDISC; 2868 2869 err = -EPROTONOSUPPORT; 2870 2871 if (x->sel.family != AF_UNSPEC) { 2872 inner_mode = xfrm_get_mode(x->props.mode, x->sel.family); 2873 if (inner_mode == NULL) { 2874 NL_SET_ERR_MSG(extack, "Requested mode not found"); 2875 goto error; 2876 } 2877 2878 if (!(inner_mode->flags & XFRM_MODE_FLAG_TUNNEL) && 2879 family != x->sel.family) { 2880 NL_SET_ERR_MSG(extack, "Only tunnel modes can accommodate a change of family"); 2881 goto error; 2882 } 2883 2884 x->inner_mode = *inner_mode; 2885 } else { 2886 const struct xfrm_mode *inner_mode_iaf; 2887 int iafamily = AF_INET; 2888 2889 inner_mode = xfrm_get_mode(x->props.mode, x->props.family); 2890 if (inner_mode == NULL) { 2891 NL_SET_ERR_MSG(extack, "Requested mode not found"); 2892 goto error; 2893 } 2894 2895 x->inner_mode = *inner_mode; 2896 2897 if (x->props.family == AF_INET) 2898 iafamily = AF_INET6; 2899 2900 inner_mode_iaf = xfrm_get_mode(x->props.mode, iafamily); 2901 if (inner_mode_iaf) { 2902 if (inner_mode_iaf->flags & XFRM_MODE_FLAG_TUNNEL) 2903 x->inner_mode_iaf = *inner_mode_iaf; 2904 } 2905 } 2906 2907 x->type = xfrm_get_type(x->id.proto, family); 2908 if (x->type == NULL) { 2909 NL_SET_ERR_MSG(extack, "Requested type not found"); 2910 goto error; 2911 } 2912 2913 x->type_offload = xfrm_get_type_offload(x->id.proto, family, offload); 2914 2915 err = x->type->init_state(x, extack); 2916 if (err) 2917 goto error; 2918 2919 outer_mode = xfrm_get_mode(x->props.mode, family); 2920 if (!outer_mode) { 2921 NL_SET_ERR_MSG(extack, "Requested mode not found"); 2922 err = -EPROTONOSUPPORT; 2923 goto error; 2924 } 2925 2926 x->outer_mode = *outer_mode; 2927 if (init_replay) { 2928 err = xfrm_init_replay(x, extack); 2929 if (err) 2930 goto error; 2931 } 2932 2933 if (x->nat_keepalive_interval) { 2934 if (x->dir != XFRM_SA_DIR_OUT) { 2935 NL_SET_ERR_MSG(extack, "NAT keepalive is only supported for outbound SAs"); 2936 err = -EINVAL; 2937 goto error; 2938 } 2939 2940 if (!x->encap || x->encap->encap_type != UDP_ENCAP_ESPINUDP) { 2941 NL_SET_ERR_MSG(extack, 2942 "NAT keepalive is only supported for UDP encapsulation"); 2943 err = -EINVAL; 2944 goto error; 2945 } 2946 } 2947 2948 error: 2949 return err; 2950 } 2951 2952 EXPORT_SYMBOL(__xfrm_init_state); 2953 2954 int xfrm_init_state(struct xfrm_state *x) 2955 { 2956 int err; 2957 2958 err = __xfrm_init_state(x, true, false, NULL); 2959 if (!err) 2960 x->km.state = XFRM_STATE_VALID; 2961 2962 return err; 2963 } 2964 2965 EXPORT_SYMBOL(xfrm_init_state); 2966 2967 int __net_init xfrm_state_init(struct net *net) 2968 { 2969 unsigned int sz; 2970 2971 if (net_eq(net, &init_net)) 2972 xfrm_state_cache = KMEM_CACHE(xfrm_state, 2973 SLAB_HWCACHE_ALIGN | SLAB_PANIC); 2974 2975 INIT_LIST_HEAD(&net->xfrm.state_all); 2976 2977 sz = sizeof(struct hlist_head) * 8; 2978 2979 net->xfrm.state_bydst = xfrm_hash_alloc(sz); 2980 if (!net->xfrm.state_bydst) 2981 goto out_bydst; 2982 net->xfrm.state_bysrc = xfrm_hash_alloc(sz); 2983 if (!net->xfrm.state_bysrc) 2984 goto out_bysrc; 2985 net->xfrm.state_byspi = xfrm_hash_alloc(sz); 2986 if (!net->xfrm.state_byspi) 2987 goto out_byspi; 2988 net->xfrm.state_byseq = xfrm_hash_alloc(sz); 2989 if (!net->xfrm.state_byseq) 2990 goto out_byseq; 2991 net->xfrm.state_hmask = ((sz / sizeof(struct hlist_head)) - 1); 2992 2993 net->xfrm.state_num = 0; 2994 INIT_WORK(&net->xfrm.state_hash_work, xfrm_hash_resize); 2995 spin_lock_init(&net->xfrm.xfrm_state_lock); 2996 seqcount_spinlock_init(&net->xfrm.xfrm_state_hash_generation, 2997 &net->xfrm.xfrm_state_lock); 2998 return 0; 2999 3000 out_byseq: 3001 xfrm_hash_free(net->xfrm.state_byspi, sz); 3002 out_byspi: 3003 xfrm_hash_free(net->xfrm.state_bysrc, sz); 3004 out_bysrc: 3005 xfrm_hash_free(net->xfrm.state_bydst, sz); 3006 out_bydst: 3007 return -ENOMEM; 3008 } 3009 3010 void xfrm_state_fini(struct net *net) 3011 { 3012 unsigned int sz; 3013 3014 flush_work(&net->xfrm.state_hash_work); 3015 flush_work(&xfrm_state_gc_work); 3016 xfrm_state_flush(net, 0, false, true); 3017 3018 WARN_ON(!list_empty(&net->xfrm.state_all)); 3019 3020 sz = (net->xfrm.state_hmask + 1) * sizeof(struct hlist_head); 3021 WARN_ON(!hlist_empty(net->xfrm.state_byseq)); 3022 xfrm_hash_free(net->xfrm.state_byseq, sz); 3023 WARN_ON(!hlist_empty(net->xfrm.state_byspi)); 3024 xfrm_hash_free(net->xfrm.state_byspi, sz); 3025 WARN_ON(!hlist_empty(net->xfrm.state_bysrc)); 3026 xfrm_hash_free(net->xfrm.state_bysrc, sz); 3027 WARN_ON(!hlist_empty(net->xfrm.state_bydst)); 3028 xfrm_hash_free(net->xfrm.state_bydst, sz); 3029 } 3030 3031 #ifdef CONFIG_AUDITSYSCALL 3032 static void xfrm_audit_helper_sainfo(struct xfrm_state *x, 3033 struct audit_buffer *audit_buf) 3034 { 3035 struct xfrm_sec_ctx *ctx = x->security; 3036 u32 spi = ntohl(x->id.spi); 3037 3038 if (ctx) 3039 audit_log_format(audit_buf, " sec_alg=%u sec_doi=%u sec_obj=%s", 3040 ctx->ctx_alg, ctx->ctx_doi, ctx->ctx_str); 3041 3042 switch (x->props.family) { 3043 case AF_INET: 3044 audit_log_format(audit_buf, " src=%pI4 dst=%pI4", 3045 &x->props.saddr.a4, &x->id.daddr.a4); 3046 break; 3047 case AF_INET6: 3048 audit_log_format(audit_buf, " src=%pI6 dst=%pI6", 3049 x->props.saddr.a6, x->id.daddr.a6); 3050 break; 3051 } 3052 3053 audit_log_format(audit_buf, " spi=%u(0x%x)", spi, spi); 3054 } 3055 3056 static void xfrm_audit_helper_pktinfo(struct sk_buff *skb, u16 family, 3057 struct audit_buffer *audit_buf) 3058 { 3059 const struct iphdr *iph4; 3060 const struct ipv6hdr *iph6; 3061 3062 switch (family) { 3063 case AF_INET: 3064 iph4 = ip_hdr(skb); 3065 audit_log_format(audit_buf, " src=%pI4 dst=%pI4", 3066 &iph4->saddr, &iph4->daddr); 3067 break; 3068 case AF_INET6: 3069 iph6 = ipv6_hdr(skb); 3070 audit_log_format(audit_buf, 3071 " src=%pI6 dst=%pI6 flowlbl=0x%x%02x%02x", 3072 &iph6->saddr, &iph6->daddr, 3073 iph6->flow_lbl[0] & 0x0f, 3074 iph6->flow_lbl[1], 3075 iph6->flow_lbl[2]); 3076 break; 3077 } 3078 } 3079 3080 void xfrm_audit_state_add(struct xfrm_state *x, int result, bool task_valid) 3081 { 3082 struct audit_buffer *audit_buf; 3083 3084 audit_buf = xfrm_audit_start("SAD-add"); 3085 if (audit_buf == NULL) 3086 return; 3087 xfrm_audit_helper_usrinfo(task_valid, audit_buf); 3088 xfrm_audit_helper_sainfo(x, audit_buf); 3089 audit_log_format(audit_buf, " res=%u", result); 3090 audit_log_end(audit_buf); 3091 } 3092 EXPORT_SYMBOL_GPL(xfrm_audit_state_add); 3093 3094 void xfrm_audit_state_delete(struct xfrm_state *x, int result, bool task_valid) 3095 { 3096 struct audit_buffer *audit_buf; 3097 3098 audit_buf = xfrm_audit_start("SAD-delete"); 3099 if (audit_buf == NULL) 3100 return; 3101 xfrm_audit_helper_usrinfo(task_valid, audit_buf); 3102 xfrm_audit_helper_sainfo(x, audit_buf); 3103 audit_log_format(audit_buf, " res=%u", result); 3104 audit_log_end(audit_buf); 3105 } 3106 EXPORT_SYMBOL_GPL(xfrm_audit_state_delete); 3107 3108 void xfrm_audit_state_replay_overflow(struct xfrm_state *x, 3109 struct sk_buff *skb) 3110 { 3111 struct audit_buffer *audit_buf; 3112 u32 spi; 3113 3114 audit_buf = xfrm_audit_start("SA-replay-overflow"); 3115 if (audit_buf == NULL) 3116 return; 3117 xfrm_audit_helper_pktinfo(skb, x->props.family, audit_buf); 3118 /* don't record the sequence number because it's inherent in this kind 3119 * of audit message */ 3120 spi = ntohl(x->id.spi); 3121 audit_log_format(audit_buf, " spi=%u(0x%x)", spi, spi); 3122 audit_log_end(audit_buf); 3123 } 3124 EXPORT_SYMBOL_GPL(xfrm_audit_state_replay_overflow); 3125 3126 void xfrm_audit_state_replay(struct xfrm_state *x, 3127 struct sk_buff *skb, __be32 net_seq) 3128 { 3129 struct audit_buffer *audit_buf; 3130 u32 spi; 3131 3132 audit_buf = xfrm_audit_start("SA-replayed-pkt"); 3133 if (audit_buf == NULL) 3134 return; 3135 xfrm_audit_helper_pktinfo(skb, x->props.family, audit_buf); 3136 spi = ntohl(x->id.spi); 3137 audit_log_format(audit_buf, " spi=%u(0x%x) seqno=%u", 3138 spi, spi, ntohl(net_seq)); 3139 audit_log_end(audit_buf); 3140 } 3141 EXPORT_SYMBOL_GPL(xfrm_audit_state_replay); 3142 3143 void xfrm_audit_state_notfound_simple(struct sk_buff *skb, u16 family) 3144 { 3145 struct audit_buffer *audit_buf; 3146 3147 audit_buf = xfrm_audit_start("SA-notfound"); 3148 if (audit_buf == NULL) 3149 return; 3150 xfrm_audit_helper_pktinfo(skb, family, audit_buf); 3151 audit_log_end(audit_buf); 3152 } 3153 EXPORT_SYMBOL_GPL(xfrm_audit_state_notfound_simple); 3154 3155 void xfrm_audit_state_notfound(struct sk_buff *skb, u16 family, 3156 __be32 net_spi, __be32 net_seq) 3157 { 3158 struct audit_buffer *audit_buf; 3159 u32 spi; 3160 3161 audit_buf = xfrm_audit_start("SA-notfound"); 3162 if (audit_buf == NULL) 3163 return; 3164 xfrm_audit_helper_pktinfo(skb, family, audit_buf); 3165 spi = ntohl(net_spi); 3166 audit_log_format(audit_buf, " spi=%u(0x%x) seqno=%u", 3167 spi, spi, ntohl(net_seq)); 3168 audit_log_end(audit_buf); 3169 } 3170 EXPORT_SYMBOL_GPL(xfrm_audit_state_notfound); 3171 3172 void xfrm_audit_state_icvfail(struct xfrm_state *x, 3173 struct sk_buff *skb, u8 proto) 3174 { 3175 struct audit_buffer *audit_buf; 3176 __be32 net_spi; 3177 __be32 net_seq; 3178 3179 audit_buf = xfrm_audit_start("SA-icv-failure"); 3180 if (audit_buf == NULL) 3181 return; 3182 xfrm_audit_helper_pktinfo(skb, x->props.family, audit_buf); 3183 if (xfrm_parse_spi(skb, proto, &net_spi, &net_seq) == 0) { 3184 u32 spi = ntohl(net_spi); 3185 audit_log_format(audit_buf, " spi=%u(0x%x) seqno=%u", 3186 spi, spi, ntohl(net_seq)); 3187 } 3188 audit_log_end(audit_buf); 3189 } 3190 EXPORT_SYMBOL_GPL(xfrm_audit_state_icvfail); 3191 #endif /* CONFIG_AUDITSYSCALL */ 3192