1 // SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0 2 /* Copyright (c) 2019-2021 Marvell International Ltd. All rights reserved */ 3 4 #include <linux/rhashtable.h> 5 6 #include "prestera.h" 7 #include "prestera_hw.h" 8 #include "prestera_router_hw.h" 9 #include "prestera_acl.h" 10 11 /* Nexthop is pointed 12 * to port (not rif) 13 * +-------+ 14 * +>|nexthop| 15 * | +-------+ 16 * | 17 * +--+ +-----++ 18 * +------->|vr|<-+ +>|nh_grp| 19 * | +--+ | | +------+ 20 * | | | 21 * +-+-------+ +--+---+-+ 22 * |rif_entry| |fib_node| 23 * +---------+ +--------+ 24 * Rif is Fib - is exit point 25 * used as 26 * entry point 27 * for vr in hw 28 */ 29 30 #define PRESTERA_NHGR_UNUSED (0) 31 #define PRESTERA_NHGR_DROP (0xFFFFFFFF) 32 /* Need to merge it with router_manager */ 33 #define PRESTERA_NH_ACTIVE_JIFFER_FILTER 3000 /* ms */ 34 35 static const struct rhashtable_params __prestera_fib_ht_params = { 36 .key_offset = offsetof(struct prestera_fib_node, key), 37 .head_offset = offsetof(struct prestera_fib_node, ht_node), 38 .key_len = sizeof(struct prestera_fib_key), 39 .automatic_shrinking = true, 40 }; 41 42 static const struct rhashtable_params __prestera_nh_neigh_ht_params = { 43 .key_offset = offsetof(struct prestera_nh_neigh, key), 44 .key_len = sizeof(struct prestera_nh_neigh_key), 45 .head_offset = offsetof(struct prestera_nh_neigh, ht_node), 46 }; 47 48 static const struct rhashtable_params __prestera_nexthop_group_ht_params = { 49 .key_offset = offsetof(struct prestera_nexthop_group, key), 50 .key_len = sizeof(struct prestera_nexthop_group_key), 51 .head_offset = offsetof(struct prestera_nexthop_group, ht_node), 52 }; 53 54 static int prestera_nexthop_group_set(struct prestera_switch *sw, 55 struct prestera_nexthop_group *nh_grp); 56 static bool 57 prestera_nexthop_group_util_hw_state(struct prestera_switch *sw, 58 struct prestera_nexthop_group *nh_grp); 59 static void prestera_fib_node_destroy_ht_cb(void *ptr, void *arg); 60 61 /* TODO: move to router.h as macros */ 62 static bool prestera_nh_neigh_key_is_valid(struct prestera_nh_neigh_key *key) 63 { 64 return memchr_inv(key, 0, sizeof(*key)) ? true : false; 65 } 66 67 int prestera_router_hw_init(struct prestera_switch *sw) 68 { 69 int err; 70 71 err = rhashtable_init(&sw->router->nh_neigh_ht, 72 &__prestera_nh_neigh_ht_params); 73 if (err) 74 goto err_nh_neigh_ht_init; 75 76 err = rhashtable_init(&sw->router->nexthop_group_ht, 77 &__prestera_nexthop_group_ht_params); 78 if (err) 79 goto err_nexthop_grp_ht_init; 80 81 err = rhashtable_init(&sw->router->fib_ht, 82 &__prestera_fib_ht_params); 83 if (err) 84 goto err_fib_ht_init; 85 86 INIT_LIST_HEAD(&sw->router->vr_list); 87 INIT_LIST_HEAD(&sw->router->rif_entry_list); 88 89 return 0; 90 91 err_fib_ht_init: 92 rhashtable_destroy(&sw->router->nexthop_group_ht); 93 err_nexthop_grp_ht_init: 94 rhashtable_destroy(&sw->router->nh_neigh_ht); 95 err_nh_neigh_ht_init: 96 return 0; 97 } 98 99 void prestera_router_hw_fini(struct prestera_switch *sw) 100 { 101 rhashtable_free_and_destroy(&sw->router->fib_ht, 102 prestera_fib_node_destroy_ht_cb, sw); 103 WARN_ON(!list_empty(&sw->router->vr_list)); 104 WARN_ON(!list_empty(&sw->router->rif_entry_list)); 105 rhashtable_destroy(&sw->router->fib_ht); 106 rhashtable_destroy(&sw->router->nexthop_group_ht); 107 rhashtable_destroy(&sw->router->nh_neigh_ht); 108 } 109 110 static struct prestera_vr *__prestera_vr_find(struct prestera_switch *sw, 111 u32 tb_id) 112 { 113 struct prestera_vr *vr; 114 115 list_for_each_entry(vr, &sw->router->vr_list, router_node) { 116 if (vr->tb_id == tb_id) 117 return vr; 118 } 119 120 return NULL; 121 } 122 123 static struct prestera_vr *__prestera_vr_create(struct prestera_switch *sw, 124 u32 tb_id, 125 struct netlink_ext_ack *extack) 126 { 127 struct prestera_vr *vr; 128 int err; 129 130 vr = kzalloc(sizeof(*vr), GFP_KERNEL); 131 if (!vr) { 132 err = -ENOMEM; 133 goto err_alloc_vr; 134 } 135 136 vr->tb_id = tb_id; 137 138 err = prestera_hw_vr_create(sw, &vr->hw_vr_id); 139 if (err) 140 goto err_hw_create; 141 142 list_add(&vr->router_node, &sw->router->vr_list); 143 144 return vr; 145 146 err_hw_create: 147 kfree(vr); 148 err_alloc_vr: 149 return ERR_PTR(err); 150 } 151 152 static void __prestera_vr_destroy(struct prestera_switch *sw, 153 struct prestera_vr *vr) 154 { 155 list_del(&vr->router_node); 156 prestera_hw_vr_delete(sw, vr->hw_vr_id); 157 kfree(vr); 158 } 159 160 static struct prestera_vr *prestera_vr_get(struct prestera_switch *sw, u32 tb_id, 161 struct netlink_ext_ack *extack) 162 { 163 struct prestera_vr *vr; 164 165 vr = __prestera_vr_find(sw, tb_id); 166 if (vr) { 167 refcount_inc(&vr->refcount); 168 } else { 169 vr = __prestera_vr_create(sw, tb_id, extack); 170 if (IS_ERR(vr)) 171 return ERR_CAST(vr); 172 173 refcount_set(&vr->refcount, 1); 174 } 175 176 return vr; 177 } 178 179 static void prestera_vr_put(struct prestera_switch *sw, struct prestera_vr *vr) 180 { 181 if (refcount_dec_and_test(&vr->refcount)) 182 __prestera_vr_destroy(sw, vr); 183 } 184 185 /* iface is overhead struct. vr_id also can be removed. */ 186 static int 187 __prestera_rif_entry_key_copy(const struct prestera_rif_entry_key *in, 188 struct prestera_rif_entry_key *out) 189 { 190 memset(out, 0, sizeof(*out)); 191 192 switch (in->iface.type) { 193 case PRESTERA_IF_PORT_E: 194 out->iface.dev_port.hw_dev_num = in->iface.dev_port.hw_dev_num; 195 out->iface.dev_port.port_num = in->iface.dev_port.port_num; 196 break; 197 case PRESTERA_IF_LAG_E: 198 out->iface.lag_id = in->iface.lag_id; 199 break; 200 case PRESTERA_IF_VID_E: 201 out->iface.vlan_id = in->iface.vlan_id; 202 break; 203 default: 204 WARN(1, "Unsupported iface type"); 205 return -EINVAL; 206 } 207 208 out->iface.type = in->iface.type; 209 return 0; 210 } 211 212 struct prestera_rif_entry * 213 prestera_rif_entry_find(const struct prestera_switch *sw, 214 const struct prestera_rif_entry_key *k) 215 { 216 struct prestera_rif_entry *rif_entry; 217 struct prestera_rif_entry_key lk; /* lookup key */ 218 219 if (__prestera_rif_entry_key_copy(k, &lk)) 220 return NULL; 221 222 list_for_each_entry(rif_entry, &sw->router->rif_entry_list, 223 router_node) { 224 if (!memcmp(k, &rif_entry->key, sizeof(*k))) 225 return rif_entry; 226 } 227 228 return NULL; 229 } 230 231 void prestera_rif_entry_destroy(struct prestera_switch *sw, 232 struct prestera_rif_entry *e) 233 { 234 struct prestera_iface iface; 235 236 list_del(&e->router_node); 237 238 memcpy(&iface, &e->key.iface, sizeof(iface)); 239 iface.vr_id = e->vr->hw_vr_id; 240 prestera_hw_rif_delete(sw, e->hw_id, &iface); 241 242 prestera_vr_put(sw, e->vr); 243 kfree(e); 244 } 245 246 struct prestera_rif_entry * 247 prestera_rif_entry_create(struct prestera_switch *sw, 248 struct prestera_rif_entry_key *k, 249 u32 tb_id, const unsigned char *addr) 250 { 251 int err; 252 struct prestera_rif_entry *e; 253 struct prestera_iface iface; 254 255 e = kzalloc(sizeof(*e), GFP_KERNEL); 256 if (!e) 257 goto err_kzalloc; 258 259 if (__prestera_rif_entry_key_copy(k, &e->key)) 260 goto err_key_copy; 261 262 e->vr = prestera_vr_get(sw, tb_id, NULL); 263 if (IS_ERR(e->vr)) 264 goto err_vr_get; 265 266 memcpy(&e->addr, addr, sizeof(e->addr)); 267 268 /* HW */ 269 memcpy(&iface, &e->key.iface, sizeof(iface)); 270 iface.vr_id = e->vr->hw_vr_id; 271 err = prestera_hw_rif_create(sw, &iface, e->addr, &e->hw_id); 272 if (err) 273 goto err_hw_create; 274 275 list_add(&e->router_node, &sw->router->rif_entry_list); 276 277 return e; 278 279 err_hw_create: 280 prestera_vr_put(sw, e->vr); 281 err_vr_get: 282 err_key_copy: 283 kfree(e); 284 err_kzalloc: 285 return NULL; 286 } 287 288 static void __prestera_nh_neigh_destroy(struct prestera_switch *sw, 289 struct prestera_nh_neigh *neigh) 290 { 291 rhashtable_remove_fast(&sw->router->nh_neigh_ht, 292 &neigh->ht_node, 293 __prestera_nh_neigh_ht_params); 294 kfree(neigh); 295 } 296 297 static struct prestera_nh_neigh * 298 __prestera_nh_neigh_create(struct prestera_switch *sw, 299 struct prestera_nh_neigh_key *key) 300 { 301 struct prestera_nh_neigh *neigh; 302 int err; 303 304 neigh = kzalloc(sizeof(*neigh), GFP_KERNEL); 305 if (!neigh) 306 goto err_kzalloc; 307 308 memcpy(&neigh->key, key, sizeof(*key)); 309 neigh->info.connected = false; 310 INIT_LIST_HEAD(&neigh->nexthop_group_list); 311 err = rhashtable_insert_fast(&sw->router->nh_neigh_ht, 312 &neigh->ht_node, 313 __prestera_nh_neigh_ht_params); 314 if (err) 315 goto err_rhashtable_insert; 316 317 return neigh; 318 319 err_rhashtable_insert: 320 kfree(neigh); 321 err_kzalloc: 322 return NULL; 323 } 324 325 struct prestera_nh_neigh * 326 prestera_nh_neigh_find(struct prestera_switch *sw, 327 struct prestera_nh_neigh_key *key) 328 { 329 struct prestera_nh_neigh *nh_neigh; 330 331 nh_neigh = rhashtable_lookup_fast(&sw->router->nh_neigh_ht, 332 key, __prestera_nh_neigh_ht_params); 333 return nh_neigh; 334 } 335 336 struct prestera_nh_neigh * 337 prestera_nh_neigh_get(struct prestera_switch *sw, 338 struct prestera_nh_neigh_key *key) 339 { 340 struct prestera_nh_neigh *neigh; 341 342 neigh = prestera_nh_neigh_find(sw, key); 343 if (!neigh) 344 return __prestera_nh_neigh_create(sw, key); 345 346 return neigh; 347 } 348 349 void prestera_nh_neigh_put(struct prestera_switch *sw, 350 struct prestera_nh_neigh *neigh) 351 { 352 if (list_empty(&neigh->nexthop_group_list)) 353 __prestera_nh_neigh_destroy(sw, neigh); 354 } 355 356 /* Updates new prestera_neigh_info */ 357 int prestera_nh_neigh_set(struct prestera_switch *sw, 358 struct prestera_nh_neigh *neigh) 359 { 360 struct prestera_nh_neigh_head *nh_head; 361 struct prestera_nexthop_group *nh_grp; 362 int err; 363 364 list_for_each_entry(nh_head, &neigh->nexthop_group_list, head) { 365 nh_grp = nh_head->this; 366 err = prestera_nexthop_group_set(sw, nh_grp); 367 if (err) 368 return err; 369 } 370 371 return 0; 372 } 373 374 bool prestera_nh_neigh_util_hw_state(struct prestera_switch *sw, 375 struct prestera_nh_neigh *nh_neigh) 376 { 377 bool state; 378 struct prestera_nh_neigh_head *nh_head, *tmp; 379 380 state = false; 381 list_for_each_entry_safe(nh_head, tmp, 382 &nh_neigh->nexthop_group_list, head) { 383 state = prestera_nexthop_group_util_hw_state(sw, nh_head->this); 384 if (state) 385 goto out; 386 } 387 388 out: 389 return state; 390 } 391 392 static struct prestera_nexthop_group * 393 __prestera_nexthop_group_create(struct prestera_switch *sw, 394 struct prestera_nexthop_group_key *key) 395 { 396 struct prestera_nexthop_group *nh_grp; 397 struct prestera_nh_neigh *nh_neigh; 398 int nh_cnt, err, gid; 399 400 nh_grp = kzalloc(sizeof(*nh_grp), GFP_KERNEL); 401 if (!nh_grp) 402 goto err_kzalloc; 403 404 memcpy(&nh_grp->key, key, sizeof(*key)); 405 for (nh_cnt = 0; nh_cnt < PRESTERA_NHGR_SIZE_MAX; nh_cnt++) { 406 if (!prestera_nh_neigh_key_is_valid(&nh_grp->key.neigh[nh_cnt])) 407 break; 408 409 nh_neigh = prestera_nh_neigh_get(sw, 410 &nh_grp->key.neigh[nh_cnt]); 411 if (!nh_neigh) 412 goto err_nh_neigh_get; 413 414 nh_grp->nh_neigh_head[nh_cnt].neigh = nh_neigh; 415 nh_grp->nh_neigh_head[nh_cnt].this = nh_grp; 416 list_add(&nh_grp->nh_neigh_head[nh_cnt].head, 417 &nh_neigh->nexthop_group_list); 418 } 419 420 err = prestera_hw_nh_group_create(sw, nh_cnt, &nh_grp->grp_id); 421 if (err) 422 goto err_nh_group_create; 423 424 err = prestera_nexthop_group_set(sw, nh_grp); 425 if (err) 426 goto err_nexthop_group_set; 427 428 err = rhashtable_insert_fast(&sw->router->nexthop_group_ht, 429 &nh_grp->ht_node, 430 __prestera_nexthop_group_ht_params); 431 if (err) 432 goto err_ht_insert; 433 434 /* reset cache for created group */ 435 gid = nh_grp->grp_id; 436 sw->router->nhgrp_hw_state_cache[gid / 8] &= ~BIT(gid % 8); 437 438 return nh_grp; 439 440 err_ht_insert: 441 err_nexthop_group_set: 442 prestera_hw_nh_group_delete(sw, nh_cnt, nh_grp->grp_id); 443 err_nh_group_create: 444 err_nh_neigh_get: 445 for (nh_cnt--; nh_cnt >= 0; nh_cnt--) { 446 list_del(&nh_grp->nh_neigh_head[nh_cnt].head); 447 prestera_nh_neigh_put(sw, nh_grp->nh_neigh_head[nh_cnt].neigh); 448 } 449 450 kfree(nh_grp); 451 err_kzalloc: 452 return NULL; 453 } 454 455 static void 456 __prestera_nexthop_group_destroy(struct prestera_switch *sw, 457 struct prestera_nexthop_group *nh_grp) 458 { 459 struct prestera_nh_neigh *nh_neigh; 460 int nh_cnt; 461 462 rhashtable_remove_fast(&sw->router->nexthop_group_ht, 463 &nh_grp->ht_node, 464 __prestera_nexthop_group_ht_params); 465 466 for (nh_cnt = 0; nh_cnt < PRESTERA_NHGR_SIZE_MAX; nh_cnt++) { 467 nh_neigh = nh_grp->nh_neigh_head[nh_cnt].neigh; 468 if (!nh_neigh) 469 break; 470 471 list_del(&nh_grp->nh_neigh_head[nh_cnt].head); 472 prestera_nh_neigh_put(sw, nh_neigh); 473 } 474 475 prestera_hw_nh_group_delete(sw, nh_cnt, nh_grp->grp_id); 476 kfree(nh_grp); 477 } 478 479 static struct prestera_nexthop_group * 480 __prestera_nexthop_group_find(struct prestera_switch *sw, 481 struct prestera_nexthop_group_key *key) 482 { 483 struct prestera_nexthop_group *nh_grp; 484 485 nh_grp = rhashtable_lookup_fast(&sw->router->nexthop_group_ht, 486 key, __prestera_nexthop_group_ht_params); 487 return nh_grp; 488 } 489 490 static struct prestera_nexthop_group * 491 prestera_nexthop_group_get(struct prestera_switch *sw, 492 struct prestera_nexthop_group_key *key) 493 { 494 struct prestera_nexthop_group *nh_grp; 495 496 nh_grp = __prestera_nexthop_group_find(sw, key); 497 if (nh_grp) { 498 refcount_inc(&nh_grp->refcount); 499 } else { 500 nh_grp = __prestera_nexthop_group_create(sw, key); 501 if (!nh_grp) 502 return ERR_PTR(-ENOMEM); 503 504 refcount_set(&nh_grp->refcount, 1); 505 } 506 507 return nh_grp; 508 } 509 510 static void prestera_nexthop_group_put(struct prestera_switch *sw, 511 struct prestera_nexthop_group *nh_grp) 512 { 513 if (refcount_dec_and_test(&nh_grp->refcount)) 514 __prestera_nexthop_group_destroy(sw, nh_grp); 515 } 516 517 /* Updates with new nh_neigh's info */ 518 static int prestera_nexthop_group_set(struct prestera_switch *sw, 519 struct prestera_nexthop_group *nh_grp) 520 { 521 struct prestera_neigh_info info[PRESTERA_NHGR_SIZE_MAX]; 522 struct prestera_nh_neigh *neigh; 523 int nh_cnt; 524 525 memset(&info[0], 0, sizeof(info)); 526 for (nh_cnt = 0; nh_cnt < PRESTERA_NHGR_SIZE_MAX; nh_cnt++) { 527 neigh = nh_grp->nh_neigh_head[nh_cnt].neigh; 528 if (!neigh) 529 break; 530 531 memcpy(&info[nh_cnt], &neigh->info, sizeof(neigh->info)); 532 } 533 534 return prestera_hw_nh_entries_set(sw, nh_cnt, &info[0], nh_grp->grp_id); 535 } 536 537 static bool 538 prestera_nexthop_group_util_hw_state(struct prestera_switch *sw, 539 struct prestera_nexthop_group *nh_grp) 540 { 541 int err; 542 u32 buf_size = sw->size_tbl_router_nexthop / 8 + 1; 543 u32 gid = nh_grp->grp_id; 544 u8 *cache = sw->router->nhgrp_hw_state_cache; 545 546 /* Antijitter 547 * Prevent situation, when we read state of nh_grp twice in short time, 548 * and state bit is still cleared on second call. So just stuck active 549 * state for PRESTERA_NH_ACTIVE_JIFFER_FILTER, after last occurred. 550 */ 551 if (!time_before(jiffies, sw->router->nhgrp_hw_cache_kick + 552 msecs_to_jiffies(PRESTERA_NH_ACTIVE_JIFFER_FILTER))) { 553 err = prestera_hw_nhgrp_blk_get(sw, cache, buf_size); 554 if (err) { 555 pr_err("Failed to get hw state nh_grp's"); 556 return false; 557 } 558 559 sw->router->nhgrp_hw_cache_kick = jiffies; 560 } 561 562 if (cache[gid / 8] & BIT(gid % 8)) 563 return true; 564 565 return false; 566 } 567 568 struct prestera_fib_node * 569 prestera_fib_node_find(struct prestera_switch *sw, struct prestera_fib_key *key) 570 { 571 struct prestera_fib_node *fib_node; 572 573 fib_node = rhashtable_lookup_fast(&sw->router->fib_ht, key, 574 __prestera_fib_ht_params); 575 return fib_node; 576 } 577 578 static void __prestera_fib_node_destruct(struct prestera_switch *sw, 579 struct prestera_fib_node *fib_node) 580 { 581 struct prestera_vr *vr; 582 583 vr = fib_node->info.vr; 584 prestera_hw_lpm_del(sw, vr->hw_vr_id, fib_node->key.addr.u.ipv4, 585 fib_node->key.prefix_len); 586 switch (fib_node->info.type) { 587 case PRESTERA_FIB_TYPE_UC_NH: 588 prestera_nexthop_group_put(sw, fib_node->info.nh_grp); 589 break; 590 case PRESTERA_FIB_TYPE_TRAP: 591 break; 592 case PRESTERA_FIB_TYPE_DROP: 593 break; 594 default: 595 pr_err("Unknown fib_node->info.type = %d", 596 fib_node->info.type); 597 } 598 599 prestera_vr_put(sw, vr); 600 } 601 602 void prestera_fib_node_destroy(struct prestera_switch *sw, 603 struct prestera_fib_node *fib_node) 604 { 605 __prestera_fib_node_destruct(sw, fib_node); 606 rhashtable_remove_fast(&sw->router->fib_ht, &fib_node->ht_node, 607 __prestera_fib_ht_params); 608 kfree(fib_node); 609 } 610 611 static void prestera_fib_node_destroy_ht_cb(void *ptr, void *arg) 612 { 613 struct prestera_fib_node *node = ptr; 614 struct prestera_switch *sw = arg; 615 616 __prestera_fib_node_destruct(sw, node); 617 kfree(node); 618 } 619 620 struct prestera_fib_node * 621 prestera_fib_node_create(struct prestera_switch *sw, 622 struct prestera_fib_key *key, 623 enum prestera_fib_type fib_type, 624 struct prestera_nexthop_group_key *nh_grp_key) 625 { 626 struct prestera_fib_node *fib_node; 627 u32 grp_id; 628 struct prestera_vr *vr; 629 int err; 630 631 fib_node = kzalloc(sizeof(*fib_node), GFP_KERNEL); 632 if (!fib_node) 633 goto err_kzalloc; 634 635 memcpy(&fib_node->key, key, sizeof(*key)); 636 fib_node->info.type = fib_type; 637 638 vr = prestera_vr_get(sw, key->tb_id, NULL); 639 if (IS_ERR(vr)) 640 goto err_vr_get; 641 642 fib_node->info.vr = vr; 643 644 switch (fib_type) { 645 case PRESTERA_FIB_TYPE_TRAP: 646 grp_id = PRESTERA_NHGR_UNUSED; 647 break; 648 case PRESTERA_FIB_TYPE_DROP: 649 grp_id = PRESTERA_NHGR_DROP; 650 break; 651 case PRESTERA_FIB_TYPE_UC_NH: 652 fib_node->info.nh_grp = prestera_nexthop_group_get(sw, 653 nh_grp_key); 654 if (IS_ERR(fib_node->info.nh_grp)) 655 goto err_nh_grp_get; 656 657 grp_id = fib_node->info.nh_grp->grp_id; 658 break; 659 default: 660 pr_err("Unsupported fib_type %d", fib_type); 661 goto err_nh_grp_get; 662 } 663 664 err = prestera_hw_lpm_add(sw, vr->hw_vr_id, key->addr.u.ipv4, 665 key->prefix_len, grp_id); 666 if (err) 667 goto err_lpm_add; 668 669 err = rhashtable_insert_fast(&sw->router->fib_ht, &fib_node->ht_node, 670 __prestera_fib_ht_params); 671 if (err) 672 goto err_ht_insert; 673 674 return fib_node; 675 676 err_ht_insert: 677 prestera_hw_lpm_del(sw, vr->hw_vr_id, key->addr.u.ipv4, 678 key->prefix_len); 679 err_lpm_add: 680 if (fib_type == PRESTERA_FIB_TYPE_UC_NH) 681 prestera_nexthop_group_put(sw, fib_node->info.nh_grp); 682 err_nh_grp_get: 683 prestera_vr_put(sw, vr); 684 err_vr_get: 685 kfree(fib_node); 686 err_kzalloc: 687 return NULL; 688 } 689