1 // SPDX-License-Identifier: GPL-2.0 2 #include <linux/jhash.h> 3 #include <linux/netfilter.h> 4 #include <linux/rcupdate.h> 5 #include <linux/rhashtable.h> 6 #include <linux/vmalloc.h> 7 #include <net/genetlink.h> 8 #include <net/netns/generic.h> 9 #include <uapi/linux/genetlink.h> 10 #include "ila.h" 11 12 struct ila_xlat_params { 13 struct ila_params ip; 14 int ifindex; 15 }; 16 17 struct ila_map { 18 struct ila_xlat_params xp; 19 struct rhash_head node; 20 struct ila_map __rcu *next; 21 struct rcu_head rcu; 22 }; 23 24 #define MAX_LOCKS 1024 25 #define LOCKS_PER_CPU 10 26 27 static int alloc_ila_locks(struct ila_net *ilan) 28 { 29 return alloc_bucket_spinlocks(&ilan->xlat.locks, &ilan->xlat.locks_mask, 30 MAX_LOCKS, LOCKS_PER_CPU, 31 GFP_KERNEL); 32 } 33 34 static u32 hashrnd __read_mostly; 35 static __always_inline void __ila_hash_secret_init(void) 36 { 37 net_get_random_once(&hashrnd, sizeof(hashrnd)); 38 } 39 40 static inline u32 ila_locator_hash(struct ila_locator loc) 41 { 42 u32 *v = (u32 *)loc.v32; 43 44 __ila_hash_secret_init(); 45 return jhash_2words(v[0], v[1], hashrnd); 46 } 47 48 static inline spinlock_t *ila_get_lock(struct ila_net *ilan, 49 struct ila_locator loc) 50 { 51 return &ilan->xlat.locks[ila_locator_hash(loc) & ilan->xlat.locks_mask]; 52 } 53 54 static inline int ila_cmp_wildcards(struct ila_map *ila, 55 struct ila_addr *iaddr, int ifindex) 56 { 57 return (ila->xp.ifindex && ila->xp.ifindex != ifindex); 58 } 59 60 static inline int ila_cmp_params(struct ila_map *ila, 61 struct ila_xlat_params *xp) 62 { 63 return (ila->xp.ifindex != xp->ifindex); 64 } 65 66 static int ila_cmpfn(struct rhashtable_compare_arg *arg, 67 const void *obj) 68 { 69 const struct ila_map *ila = obj; 70 71 return (ila->xp.ip.locator_match.v64 != *(__be64 *)arg->key); 72 } 73 74 static inline int ila_order(struct ila_map *ila) 75 { 76 int score = 0; 77 78 if (ila->xp.ifindex) 79 score += 1 << 1; 80 81 return score; 82 } 83 84 static const struct rhashtable_params rht_params = { 85 .nelem_hint = 1024, 86 .head_offset = offsetof(struct ila_map, node), 87 .key_offset = offsetof(struct ila_map, xp.ip.locator_match), 88 .key_len = sizeof(u64), /* identifier */ 89 .max_size = 1048576, 90 .min_size = 256, 91 .automatic_shrinking = true, 92 .obj_cmpfn = ila_cmpfn, 93 }; 94 95 static int parse_nl_config(struct genl_info *info, 96 struct ila_xlat_params *xp) 97 { 98 memset(xp, 0, sizeof(*xp)); 99 100 if (info->attrs[ILA_ATTR_LOCATOR]) 101 xp->ip.locator.v64 = (__force __be64)nla_get_u64( 102 info->attrs[ILA_ATTR_LOCATOR]); 103 104 if (info->attrs[ILA_ATTR_LOCATOR_MATCH]) 105 xp->ip.locator_match.v64 = (__force __be64)nla_get_u64( 106 info->attrs[ILA_ATTR_LOCATOR_MATCH]); 107 108 xp->ip.csum_mode = nla_get_u8_default(info->attrs[ILA_ATTR_CSUM_MODE], 109 ILA_CSUM_NO_ACTION); 110 111 xp->ip.ident_type = nla_get_u8_default(info->attrs[ILA_ATTR_IDENT_TYPE], 112 ILA_ATYPE_USE_FORMAT); 113 114 if (info->attrs[ILA_ATTR_IFINDEX]) 115 xp->ifindex = nla_get_s32(info->attrs[ILA_ATTR_IFINDEX]); 116 117 return 0; 118 } 119 120 /* Must be called with rcu readlock */ 121 static inline struct ila_map *ila_lookup_wildcards(struct ila_addr *iaddr, 122 int ifindex, 123 struct ila_net *ilan) 124 { 125 struct ila_map *ila; 126 127 ila = rhashtable_lookup_fast(&ilan->xlat.rhash_table, &iaddr->loc, 128 rht_params); 129 while (ila) { 130 if (!ila_cmp_wildcards(ila, iaddr, ifindex)) 131 return ila; 132 ila = rcu_access_pointer(ila->next); 133 } 134 135 return NULL; 136 } 137 138 /* Must be called with rcu readlock */ 139 static inline struct ila_map *ila_lookup_by_params(struct ila_xlat_params *xp, 140 struct ila_net *ilan) 141 { 142 struct ila_map *ila; 143 144 ila = rhashtable_lookup_fast(&ilan->xlat.rhash_table, 145 &xp->ip.locator_match, 146 rht_params); 147 while (ila) { 148 if (!ila_cmp_params(ila, xp)) 149 return ila; 150 ila = rcu_access_pointer(ila->next); 151 } 152 153 return NULL; 154 } 155 156 static inline void ila_release(struct ila_map *ila) 157 { 158 kfree_rcu(ila, rcu); 159 } 160 161 static void ila_free_node(struct ila_map *ila) 162 { 163 struct ila_map *next; 164 165 /* Assume rcu_readlock held */ 166 while (ila) { 167 next = rcu_access_pointer(ila->next); 168 ila_release(ila); 169 ila = next; 170 } 171 } 172 173 static void ila_free_cb(void *ptr, void *arg) 174 { 175 ila_free_node((struct ila_map *)ptr); 176 } 177 178 static int ila_xlat_addr(struct sk_buff *skb, bool sir2ila); 179 180 static unsigned int 181 ila_nf_input(void *priv, 182 struct sk_buff *skb, 183 const struct nf_hook_state *state) 184 { 185 ila_xlat_addr(skb, false); 186 return NF_ACCEPT; 187 } 188 189 static const struct nf_hook_ops ila_nf_hook_ops[] = { 190 { 191 .hook = ila_nf_input, 192 .pf = NFPROTO_IPV6, 193 .hooknum = NF_INET_PRE_ROUTING, 194 .priority = -1, 195 }, 196 }; 197 198 static int ila_add_mapping(struct net *net, struct ila_xlat_params *xp) 199 { 200 struct ila_net *ilan = net_generic(net, ila_net_id); 201 struct ila_map *ila, *head; 202 spinlock_t *lock = ila_get_lock(ilan, xp->ip.locator_match); 203 int err = 0, order; 204 205 if (!ilan->xlat.hooks_registered) { 206 /* We defer registering net hooks in the namespace until the 207 * first mapping is added. 208 */ 209 err = nf_register_net_hooks(net, ila_nf_hook_ops, 210 ARRAY_SIZE(ila_nf_hook_ops)); 211 if (err) 212 return err; 213 214 ilan->xlat.hooks_registered = true; 215 } 216 217 ila = kzalloc(sizeof(*ila), GFP_KERNEL); 218 if (!ila) 219 return -ENOMEM; 220 221 ila_init_saved_csum(&xp->ip); 222 223 ila->xp = *xp; 224 225 order = ila_order(ila); 226 227 spin_lock(lock); 228 229 head = rhashtable_lookup_fast(&ilan->xlat.rhash_table, 230 &xp->ip.locator_match, 231 rht_params); 232 if (!head) { 233 /* New entry for the rhash_table */ 234 err = rhashtable_lookup_insert_fast(&ilan->xlat.rhash_table, 235 &ila->node, rht_params); 236 } else { 237 struct ila_map *tila = head, *prev = NULL; 238 239 do { 240 if (!ila_cmp_params(tila, xp)) { 241 err = -EEXIST; 242 goto out; 243 } 244 245 if (order > ila_order(tila)) 246 break; 247 248 prev = tila; 249 tila = rcu_dereference_protected(tila->next, 250 lockdep_is_held(lock)); 251 } while (tila); 252 253 if (prev) { 254 /* Insert in sub list of head */ 255 RCU_INIT_POINTER(ila->next, tila); 256 rcu_assign_pointer(prev->next, ila); 257 } else { 258 /* Make this ila new head */ 259 RCU_INIT_POINTER(ila->next, head); 260 err = rhashtable_replace_fast(&ilan->xlat.rhash_table, 261 &head->node, 262 &ila->node, rht_params); 263 if (err) 264 goto out; 265 } 266 } 267 268 out: 269 spin_unlock(lock); 270 271 if (err) 272 kfree(ila); 273 274 return err; 275 } 276 277 static int ila_del_mapping(struct net *net, struct ila_xlat_params *xp) 278 { 279 struct ila_net *ilan = net_generic(net, ila_net_id); 280 struct ila_map *ila, *head, *prev; 281 spinlock_t *lock = ila_get_lock(ilan, xp->ip.locator_match); 282 int err = -ENOENT; 283 284 spin_lock(lock); 285 286 head = rhashtable_lookup_fast(&ilan->xlat.rhash_table, 287 &xp->ip.locator_match, rht_params); 288 ila = head; 289 290 prev = NULL; 291 292 while (ila) { 293 if (ila_cmp_params(ila, xp)) { 294 prev = ila; 295 ila = rcu_dereference_protected(ila->next, 296 lockdep_is_held(lock)); 297 continue; 298 } 299 300 err = 0; 301 302 if (prev) { 303 /* Not head, just delete from list */ 304 rcu_assign_pointer(prev->next, ila->next); 305 } else { 306 /* It is the head. If there is something in the 307 * sublist we need to make a new head. 308 */ 309 head = rcu_dereference_protected(ila->next, 310 lockdep_is_held(lock)); 311 if (head) { 312 /* Put first entry in the sublist into the 313 * table 314 */ 315 err = rhashtable_replace_fast( 316 &ilan->xlat.rhash_table, &ila->node, 317 &head->node, rht_params); 318 if (err) 319 goto out; 320 } else { 321 /* Entry no longer used */ 322 err = rhashtable_remove_fast( 323 &ilan->xlat.rhash_table, 324 &ila->node, rht_params); 325 } 326 } 327 328 ila_release(ila); 329 330 break; 331 } 332 333 out: 334 spin_unlock(lock); 335 336 return err; 337 } 338 339 int ila_xlat_nl_cmd_add_mapping(struct sk_buff *skb, struct genl_info *info) 340 { 341 struct net *net = genl_info_net(info); 342 struct ila_xlat_params p; 343 int err; 344 345 err = parse_nl_config(info, &p); 346 if (err) 347 return err; 348 349 return ila_add_mapping(net, &p); 350 } 351 352 int ila_xlat_nl_cmd_del_mapping(struct sk_buff *skb, struct genl_info *info) 353 { 354 struct net *net = genl_info_net(info); 355 struct ila_xlat_params xp; 356 int err; 357 358 err = parse_nl_config(info, &xp); 359 if (err) 360 return err; 361 362 ila_del_mapping(net, &xp); 363 364 return 0; 365 } 366 367 static inline spinlock_t *lock_from_ila_map(struct ila_net *ilan, 368 struct ila_map *ila) 369 { 370 return ila_get_lock(ilan, ila->xp.ip.locator_match); 371 } 372 373 int ila_xlat_nl_cmd_flush(struct sk_buff *skb, struct genl_info *info) 374 { 375 struct net *net = genl_info_net(info); 376 struct ila_net *ilan = net_generic(net, ila_net_id); 377 struct rhashtable_iter iter; 378 struct ila_map *ila; 379 spinlock_t *lock; 380 int ret = 0; 381 382 rhashtable_walk_enter(&ilan->xlat.rhash_table, &iter); 383 rhashtable_walk_start(&iter); 384 385 for (;;) { 386 ila = rhashtable_walk_next(&iter); 387 388 if (IS_ERR(ila)) { 389 if (PTR_ERR(ila) == -EAGAIN) 390 continue; 391 ret = PTR_ERR(ila); 392 goto done; 393 } else if (!ila) { 394 break; 395 } 396 397 lock = lock_from_ila_map(ilan, ila); 398 399 spin_lock(lock); 400 401 ret = rhashtable_remove_fast(&ilan->xlat.rhash_table, 402 &ila->node, rht_params); 403 if (!ret) 404 ila_free_node(ila); 405 406 spin_unlock(lock); 407 408 if (ret) 409 break; 410 } 411 412 done: 413 rhashtable_walk_stop(&iter); 414 rhashtable_walk_exit(&iter); 415 return ret; 416 } 417 418 static int ila_fill_info(struct ila_map *ila, struct sk_buff *msg) 419 { 420 if (nla_put_u64_64bit(msg, ILA_ATTR_LOCATOR, 421 (__force u64)ila->xp.ip.locator.v64, 422 ILA_ATTR_PAD) || 423 nla_put_u64_64bit(msg, ILA_ATTR_LOCATOR_MATCH, 424 (__force u64)ila->xp.ip.locator_match.v64, 425 ILA_ATTR_PAD) || 426 nla_put_s32(msg, ILA_ATTR_IFINDEX, ila->xp.ifindex) || 427 nla_put_u8(msg, ILA_ATTR_CSUM_MODE, ila->xp.ip.csum_mode) || 428 nla_put_u8(msg, ILA_ATTR_IDENT_TYPE, ila->xp.ip.ident_type)) 429 return -1; 430 431 return 0; 432 } 433 434 static int ila_dump_info(struct ila_map *ila, 435 u32 portid, u32 seq, u32 flags, 436 struct sk_buff *skb, u8 cmd) 437 { 438 void *hdr; 439 440 hdr = genlmsg_put(skb, portid, seq, &ila_nl_family, flags, cmd); 441 if (!hdr) 442 return -ENOMEM; 443 444 if (ila_fill_info(ila, skb) < 0) 445 goto nla_put_failure; 446 447 genlmsg_end(skb, hdr); 448 return 0; 449 450 nla_put_failure: 451 genlmsg_cancel(skb, hdr); 452 return -EMSGSIZE; 453 } 454 455 int ila_xlat_nl_cmd_get_mapping(struct sk_buff *skb, struct genl_info *info) 456 { 457 struct net *net = genl_info_net(info); 458 struct ila_net *ilan = net_generic(net, ila_net_id); 459 struct sk_buff *msg; 460 struct ila_xlat_params xp; 461 struct ila_map *ila; 462 int ret; 463 464 ret = parse_nl_config(info, &xp); 465 if (ret) 466 return ret; 467 468 msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL); 469 if (!msg) 470 return -ENOMEM; 471 472 rcu_read_lock(); 473 474 ret = -ESRCH; 475 ila = ila_lookup_by_params(&xp, ilan); 476 if (ila) { 477 ret = ila_dump_info(ila, 478 info->snd_portid, 479 info->snd_seq, 0, msg, 480 info->genlhdr->cmd); 481 } 482 483 rcu_read_unlock(); 484 485 if (ret < 0) 486 goto out_free; 487 488 return genlmsg_reply(msg, info); 489 490 out_free: 491 nlmsg_free(msg); 492 return ret; 493 } 494 495 struct ila_dump_iter { 496 struct rhashtable_iter rhiter; 497 int skip; 498 }; 499 500 int ila_xlat_nl_dump_start(struct netlink_callback *cb) 501 { 502 struct net *net = sock_net(cb->skb->sk); 503 struct ila_net *ilan = net_generic(net, ila_net_id); 504 struct ila_dump_iter *iter; 505 506 iter = kmalloc(sizeof(*iter), GFP_KERNEL); 507 if (!iter) 508 return -ENOMEM; 509 510 rhashtable_walk_enter(&ilan->xlat.rhash_table, &iter->rhiter); 511 512 iter->skip = 0; 513 cb->args[0] = (long)iter; 514 515 return 0; 516 } 517 518 int ila_xlat_nl_dump_done(struct netlink_callback *cb) 519 { 520 struct ila_dump_iter *iter = (struct ila_dump_iter *)cb->args[0]; 521 522 rhashtable_walk_exit(&iter->rhiter); 523 524 kfree(iter); 525 526 return 0; 527 } 528 529 int ila_xlat_nl_dump(struct sk_buff *skb, struct netlink_callback *cb) 530 { 531 struct ila_dump_iter *iter = (struct ila_dump_iter *)cb->args[0]; 532 struct rhashtable_iter *rhiter = &iter->rhiter; 533 int skip = iter->skip; 534 struct ila_map *ila; 535 int ret; 536 537 rhashtable_walk_start(rhiter); 538 539 /* Get first entry */ 540 ila = rhashtable_walk_peek(rhiter); 541 542 if (ila && !IS_ERR(ila) && skip) { 543 /* Skip over visited entries */ 544 545 while (ila && skip) { 546 /* Skip over any ila entries in this list that we 547 * have already dumped. 548 */ 549 ila = rcu_access_pointer(ila->next); 550 skip--; 551 } 552 } 553 554 skip = 0; 555 556 for (;;) { 557 if (IS_ERR(ila)) { 558 ret = PTR_ERR(ila); 559 if (ret == -EAGAIN) { 560 /* Table has changed and iter has reset. Return 561 * -EAGAIN to the application even if we have 562 * written data to the skb. The application 563 * needs to deal with this. 564 */ 565 566 goto out_ret; 567 } else { 568 break; 569 } 570 } else if (!ila) { 571 ret = 0; 572 break; 573 } 574 575 while (ila) { 576 ret = ila_dump_info(ila, NETLINK_CB(cb->skb).portid, 577 cb->nlh->nlmsg_seq, NLM_F_MULTI, 578 skb, ILA_CMD_GET); 579 if (ret) 580 goto out; 581 582 skip++; 583 ila = rcu_access_pointer(ila->next); 584 } 585 586 skip = 0; 587 ila = rhashtable_walk_next(rhiter); 588 } 589 590 out: 591 iter->skip = skip; 592 ret = (skb->len ? : ret); 593 594 out_ret: 595 rhashtable_walk_stop(rhiter); 596 return ret; 597 } 598 599 int ila_xlat_init_net(struct net *net) 600 { 601 struct ila_net *ilan = net_generic(net, ila_net_id); 602 int err; 603 604 err = alloc_ila_locks(ilan); 605 if (err) 606 return err; 607 608 err = rhashtable_init(&ilan->xlat.rhash_table, &rht_params); 609 if (err) { 610 free_bucket_spinlocks(ilan->xlat.locks); 611 return err; 612 } 613 614 return 0; 615 } 616 617 void ila_xlat_pre_exit_net(struct net *net) 618 { 619 struct ila_net *ilan = net_generic(net, ila_net_id); 620 621 if (ilan->xlat.hooks_registered) 622 nf_unregister_net_hooks(net, ila_nf_hook_ops, 623 ARRAY_SIZE(ila_nf_hook_ops)); 624 } 625 626 void ila_xlat_exit_net(struct net *net) 627 { 628 struct ila_net *ilan = net_generic(net, ila_net_id); 629 630 rhashtable_free_and_destroy(&ilan->xlat.rhash_table, ila_free_cb, NULL); 631 632 free_bucket_spinlocks(ilan->xlat.locks); 633 } 634 635 static int ila_xlat_addr(struct sk_buff *skb, bool sir2ila) 636 { 637 struct ila_map *ila; 638 struct ipv6hdr *ip6h = ipv6_hdr(skb); 639 struct net *net = dev_net(skb->dev); 640 struct ila_net *ilan = net_generic(net, ila_net_id); 641 struct ila_addr *iaddr = ila_a2i(&ip6h->daddr); 642 643 /* Assumes skb contains a valid IPv6 header that is pulled */ 644 645 /* No check here that ILA type in the mapping matches what is in the 646 * address. We assume that whatever sender gaves us can be translated. 647 * The checksum mode however is relevant. 648 */ 649 650 rcu_read_lock(); 651 652 ila = ila_lookup_wildcards(iaddr, skb->dev->ifindex, ilan); 653 if (ila) 654 ila_update_ipv6_locator(skb, &ila->xp.ip, sir2ila); 655 656 rcu_read_unlock(); 657 658 return 0; 659 } 660