1 /* 2 * drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c 3 * Copyright (c) 2016 Mellanox Technologies. All rights reserved. 4 * Copyright (c) 2016 Jiri Pirko <jiri@mellanox.com> 5 * Copyright (c) 2016 Ido Schimmel <idosch@mellanox.com> 6 * Copyright (c) 2016 Yotam Gigi <yotamg@mellanox.com> 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions are met: 10 * 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 3. Neither the names of the copyright holders nor the names of its 17 * contributors may be used to endorse or promote products derived from 18 * this software without specific prior written permission. 19 * 20 * Alternatively, this software may be distributed under the terms of the 21 * GNU General Public License ("GPL") version 2 as published by the Free 22 * Software Foundation. 23 * 24 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 25 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 26 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 27 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 28 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 29 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 30 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 31 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 32 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 33 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 34 * POSSIBILITY OF SUCH DAMAGE. 35 */ 36 37 #include <linux/kernel.h> 38 #include <linux/types.h> 39 #include <linux/rhashtable.h> 40 #include <linux/bitops.h> 41 #include <linux/in6.h> 42 #include <linux/notifier.h> 43 #include <linux/inetdevice.h> 44 #include <linux/netdevice.h> 45 #include <linux/if_bridge.h> 46 #include <net/netevent.h> 47 #include <net/neighbour.h> 48 #include <net/arp.h> 49 #include <net/ip_fib.h> 50 #include <net/fib_rules.h> 51 #include <net/l3mdev.h> 52 53 #include "spectrum.h" 54 #include "core.h" 55 #include "reg.h" 56 #include "spectrum_cnt.h" 57 #include "spectrum_dpipe.h" 58 #include "spectrum_router.h" 59 60 struct mlxsw_sp_vr; 61 struct mlxsw_sp_lpm_tree; 62 struct mlxsw_sp_rif_ops; 63 64 struct mlxsw_sp_router { 65 struct mlxsw_sp *mlxsw_sp; 66 struct mlxsw_sp_rif **rifs; 67 struct mlxsw_sp_vr *vrs; 68 struct rhashtable neigh_ht; 69 struct rhashtable nexthop_group_ht; 70 struct rhashtable nexthop_ht; 71 struct { 72 struct mlxsw_sp_lpm_tree *trees; 73 unsigned int tree_count; 74 } lpm; 75 struct { 76 struct delayed_work dw; 77 unsigned long interval; /* ms */ 78 } neighs_update; 79 struct delayed_work nexthop_probe_dw; 80 #define MLXSW_SP_UNRESOLVED_NH_PROBE_INTERVAL 5000 /* ms */ 81 struct list_head nexthop_neighs_list; 82 bool aborted; 83 struct notifier_block fib_nb; 84 const struct mlxsw_sp_rif_ops **rif_ops_arr; 85 }; 86 87 struct mlxsw_sp_rif { 88 struct list_head nexthop_list; 89 struct list_head neigh_list; 90 struct net_device *dev; 91 struct mlxsw_sp_fid *fid; 92 unsigned char addr[ETH_ALEN]; 93 int mtu; 94 u16 rif_index; 95 u16 vr_id; 96 const struct mlxsw_sp_rif_ops *ops; 97 struct mlxsw_sp *mlxsw_sp; 98 99 unsigned int counter_ingress; 100 bool counter_ingress_valid; 101 unsigned int counter_egress; 102 bool counter_egress_valid; 103 }; 104 105 struct mlxsw_sp_rif_params { 106 struct net_device *dev; 107 union { 108 u16 system_port; 109 u16 lag_id; 110 }; 111 u16 vid; 112 bool lag; 113 }; 114 115 struct mlxsw_sp_rif_subport { 116 struct mlxsw_sp_rif common; 117 union { 118 u16 system_port; 119 u16 lag_id; 120 }; 121 u16 vid; 122 bool lag; 123 }; 124 125 struct mlxsw_sp_rif_ops { 126 enum mlxsw_sp_rif_type type; 127 size_t rif_size; 128 129 void (*setup)(struct mlxsw_sp_rif *rif, 130 const struct mlxsw_sp_rif_params *params); 131 int (*configure)(struct mlxsw_sp_rif *rif); 132 void (*deconfigure)(struct mlxsw_sp_rif *rif); 133 struct mlxsw_sp_fid * (*fid_get)(struct mlxsw_sp_rif *rif); 134 }; 135 136 static unsigned int * 137 mlxsw_sp_rif_p_counter_get(struct mlxsw_sp_rif *rif, 138 enum mlxsw_sp_rif_counter_dir dir) 139 { 140 switch (dir) { 141 case MLXSW_SP_RIF_COUNTER_EGRESS: 142 return &rif->counter_egress; 143 case MLXSW_SP_RIF_COUNTER_INGRESS: 144 return &rif->counter_ingress; 145 } 146 return NULL; 147 } 148 149 static bool 150 mlxsw_sp_rif_counter_valid_get(struct mlxsw_sp_rif *rif, 151 enum mlxsw_sp_rif_counter_dir dir) 152 { 153 switch (dir) { 154 case MLXSW_SP_RIF_COUNTER_EGRESS: 155 return rif->counter_egress_valid; 156 case MLXSW_SP_RIF_COUNTER_INGRESS: 157 return rif->counter_ingress_valid; 158 } 159 return false; 160 } 161 162 static void 163 mlxsw_sp_rif_counter_valid_set(struct mlxsw_sp_rif *rif, 164 enum mlxsw_sp_rif_counter_dir dir, 165 bool valid) 166 { 167 switch (dir) { 168 case MLXSW_SP_RIF_COUNTER_EGRESS: 169 rif->counter_egress_valid = valid; 170 break; 171 case MLXSW_SP_RIF_COUNTER_INGRESS: 172 rif->counter_ingress_valid = valid; 173 break; 174 } 175 } 176 177 static int mlxsw_sp_rif_counter_edit(struct mlxsw_sp *mlxsw_sp, u16 rif_index, 178 unsigned int counter_index, bool enable, 179 enum mlxsw_sp_rif_counter_dir dir) 180 { 181 char ritr_pl[MLXSW_REG_RITR_LEN]; 182 bool is_egress = false; 183 int err; 184 185 if (dir == MLXSW_SP_RIF_COUNTER_EGRESS) 186 is_egress = true; 187 mlxsw_reg_ritr_rif_pack(ritr_pl, rif_index); 188 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 189 if (err) 190 return err; 191 192 mlxsw_reg_ritr_counter_pack(ritr_pl, counter_index, enable, 193 is_egress); 194 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 195 } 196 197 int mlxsw_sp_rif_counter_value_get(struct mlxsw_sp *mlxsw_sp, 198 struct mlxsw_sp_rif *rif, 199 enum mlxsw_sp_rif_counter_dir dir, u64 *cnt) 200 { 201 char ricnt_pl[MLXSW_REG_RICNT_LEN]; 202 unsigned int *p_counter_index; 203 bool valid; 204 int err; 205 206 valid = mlxsw_sp_rif_counter_valid_get(rif, dir); 207 if (!valid) 208 return -EINVAL; 209 210 p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir); 211 if (!p_counter_index) 212 return -EINVAL; 213 mlxsw_reg_ricnt_pack(ricnt_pl, *p_counter_index, 214 MLXSW_REG_RICNT_OPCODE_NOP); 215 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ricnt), ricnt_pl); 216 if (err) 217 return err; 218 *cnt = mlxsw_reg_ricnt_good_unicast_packets_get(ricnt_pl); 219 return 0; 220 } 221 222 static int mlxsw_sp_rif_counter_clear(struct mlxsw_sp *mlxsw_sp, 223 unsigned int counter_index) 224 { 225 char ricnt_pl[MLXSW_REG_RICNT_LEN]; 226 227 mlxsw_reg_ricnt_pack(ricnt_pl, counter_index, 228 MLXSW_REG_RICNT_OPCODE_CLEAR); 229 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ricnt), ricnt_pl); 230 } 231 232 int mlxsw_sp_rif_counter_alloc(struct mlxsw_sp *mlxsw_sp, 233 struct mlxsw_sp_rif *rif, 234 enum mlxsw_sp_rif_counter_dir dir) 235 { 236 unsigned int *p_counter_index; 237 int err; 238 239 p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir); 240 if (!p_counter_index) 241 return -EINVAL; 242 err = mlxsw_sp_counter_alloc(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF, 243 p_counter_index); 244 if (err) 245 return err; 246 247 err = mlxsw_sp_rif_counter_clear(mlxsw_sp, *p_counter_index); 248 if (err) 249 goto err_counter_clear; 250 251 err = mlxsw_sp_rif_counter_edit(mlxsw_sp, rif->rif_index, 252 *p_counter_index, true, dir); 253 if (err) 254 goto err_counter_edit; 255 mlxsw_sp_rif_counter_valid_set(rif, dir, true); 256 return 0; 257 258 err_counter_edit: 259 err_counter_clear: 260 mlxsw_sp_counter_free(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF, 261 *p_counter_index); 262 return err; 263 } 264 265 void mlxsw_sp_rif_counter_free(struct mlxsw_sp *mlxsw_sp, 266 struct mlxsw_sp_rif *rif, 267 enum mlxsw_sp_rif_counter_dir dir) 268 { 269 unsigned int *p_counter_index; 270 271 if (!mlxsw_sp_rif_counter_valid_get(rif, dir)) 272 return; 273 274 p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir); 275 if (WARN_ON(!p_counter_index)) 276 return; 277 mlxsw_sp_rif_counter_edit(mlxsw_sp, rif->rif_index, 278 *p_counter_index, false, dir); 279 mlxsw_sp_counter_free(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF, 280 *p_counter_index); 281 mlxsw_sp_rif_counter_valid_set(rif, dir, false); 282 } 283 284 static void mlxsw_sp_rif_counters_alloc(struct mlxsw_sp_rif *rif) 285 { 286 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 287 struct devlink *devlink; 288 289 devlink = priv_to_devlink(mlxsw_sp->core); 290 if (!devlink_dpipe_table_counter_enabled(devlink, 291 MLXSW_SP_DPIPE_TABLE_NAME_ERIF)) 292 return; 293 mlxsw_sp_rif_counter_alloc(mlxsw_sp, rif, MLXSW_SP_RIF_COUNTER_EGRESS); 294 } 295 296 static void mlxsw_sp_rif_counters_free(struct mlxsw_sp_rif *rif) 297 { 298 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 299 300 mlxsw_sp_rif_counter_free(mlxsw_sp, rif, MLXSW_SP_RIF_COUNTER_EGRESS); 301 } 302 303 static struct mlxsw_sp_rif * 304 mlxsw_sp_rif_find_by_dev(const struct mlxsw_sp *mlxsw_sp, 305 const struct net_device *dev); 306 307 #define MLXSW_SP_PREFIX_COUNT (sizeof(struct in6_addr) * BITS_PER_BYTE) 308 309 struct mlxsw_sp_prefix_usage { 310 DECLARE_BITMAP(b, MLXSW_SP_PREFIX_COUNT); 311 }; 312 313 #define mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) \ 314 for_each_set_bit(prefix, (prefix_usage)->b, MLXSW_SP_PREFIX_COUNT) 315 316 static bool 317 mlxsw_sp_prefix_usage_subset(struct mlxsw_sp_prefix_usage *prefix_usage1, 318 struct mlxsw_sp_prefix_usage *prefix_usage2) 319 { 320 unsigned char prefix; 321 322 mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage1) { 323 if (!test_bit(prefix, prefix_usage2->b)) 324 return false; 325 } 326 return true; 327 } 328 329 static bool 330 mlxsw_sp_prefix_usage_eq(struct mlxsw_sp_prefix_usage *prefix_usage1, 331 struct mlxsw_sp_prefix_usage *prefix_usage2) 332 { 333 return !memcmp(prefix_usage1, prefix_usage2, sizeof(*prefix_usage1)); 334 } 335 336 static bool 337 mlxsw_sp_prefix_usage_none(struct mlxsw_sp_prefix_usage *prefix_usage) 338 { 339 struct mlxsw_sp_prefix_usage prefix_usage_none = {{ 0 } }; 340 341 return mlxsw_sp_prefix_usage_eq(prefix_usage, &prefix_usage_none); 342 } 343 344 static void 345 mlxsw_sp_prefix_usage_cpy(struct mlxsw_sp_prefix_usage *prefix_usage1, 346 struct mlxsw_sp_prefix_usage *prefix_usage2) 347 { 348 memcpy(prefix_usage1, prefix_usage2, sizeof(*prefix_usage1)); 349 } 350 351 static void 352 mlxsw_sp_prefix_usage_set(struct mlxsw_sp_prefix_usage *prefix_usage, 353 unsigned char prefix_len) 354 { 355 set_bit(prefix_len, prefix_usage->b); 356 } 357 358 static void 359 mlxsw_sp_prefix_usage_clear(struct mlxsw_sp_prefix_usage *prefix_usage, 360 unsigned char prefix_len) 361 { 362 clear_bit(prefix_len, prefix_usage->b); 363 } 364 365 struct mlxsw_sp_fib_key { 366 unsigned char addr[sizeof(struct in6_addr)]; 367 unsigned char prefix_len; 368 }; 369 370 enum mlxsw_sp_fib_entry_type { 371 MLXSW_SP_FIB_ENTRY_TYPE_REMOTE, 372 MLXSW_SP_FIB_ENTRY_TYPE_LOCAL, 373 MLXSW_SP_FIB_ENTRY_TYPE_TRAP, 374 }; 375 376 struct mlxsw_sp_nexthop_group; 377 struct mlxsw_sp_fib; 378 379 struct mlxsw_sp_fib_node { 380 struct list_head entry_list; 381 struct list_head list; 382 struct rhash_head ht_node; 383 struct mlxsw_sp_fib *fib; 384 struct mlxsw_sp_fib_key key; 385 }; 386 387 struct mlxsw_sp_fib_entry_params { 388 u32 tb_id; 389 u32 prio; 390 u8 tos; 391 u8 type; 392 }; 393 394 struct mlxsw_sp_fib_entry { 395 struct list_head list; 396 struct mlxsw_sp_fib_node *fib_node; 397 enum mlxsw_sp_fib_entry_type type; 398 struct list_head nexthop_group_node; 399 struct mlxsw_sp_nexthop_group *nh_group; 400 struct mlxsw_sp_fib_entry_params params; 401 bool offloaded; 402 }; 403 404 enum mlxsw_sp_l3proto { 405 MLXSW_SP_L3_PROTO_IPV4, 406 MLXSW_SP_L3_PROTO_IPV6, 407 }; 408 409 struct mlxsw_sp_lpm_tree { 410 u8 id; /* tree ID */ 411 unsigned int ref_count; 412 enum mlxsw_sp_l3proto proto; 413 struct mlxsw_sp_prefix_usage prefix_usage; 414 }; 415 416 struct mlxsw_sp_fib { 417 struct rhashtable ht; 418 struct list_head node_list; 419 struct mlxsw_sp_vr *vr; 420 struct mlxsw_sp_lpm_tree *lpm_tree; 421 unsigned long prefix_ref_count[MLXSW_SP_PREFIX_COUNT]; 422 struct mlxsw_sp_prefix_usage prefix_usage; 423 enum mlxsw_sp_l3proto proto; 424 }; 425 426 struct mlxsw_sp_vr { 427 u16 id; /* virtual router ID */ 428 u32 tb_id; /* kernel fib table id */ 429 unsigned int rif_count; 430 struct mlxsw_sp_fib *fib4; 431 }; 432 433 static const struct rhashtable_params mlxsw_sp_fib_ht_params; 434 435 static struct mlxsw_sp_fib *mlxsw_sp_fib_create(struct mlxsw_sp_vr *vr, 436 enum mlxsw_sp_l3proto proto) 437 { 438 struct mlxsw_sp_fib *fib; 439 int err; 440 441 fib = kzalloc(sizeof(*fib), GFP_KERNEL); 442 if (!fib) 443 return ERR_PTR(-ENOMEM); 444 err = rhashtable_init(&fib->ht, &mlxsw_sp_fib_ht_params); 445 if (err) 446 goto err_rhashtable_init; 447 INIT_LIST_HEAD(&fib->node_list); 448 fib->proto = proto; 449 fib->vr = vr; 450 return fib; 451 452 err_rhashtable_init: 453 kfree(fib); 454 return ERR_PTR(err); 455 } 456 457 static void mlxsw_sp_fib_destroy(struct mlxsw_sp_fib *fib) 458 { 459 WARN_ON(!list_empty(&fib->node_list)); 460 WARN_ON(fib->lpm_tree); 461 rhashtable_destroy(&fib->ht); 462 kfree(fib); 463 } 464 465 static struct mlxsw_sp_lpm_tree * 466 mlxsw_sp_lpm_tree_find_unused(struct mlxsw_sp *mlxsw_sp) 467 { 468 static struct mlxsw_sp_lpm_tree *lpm_tree; 469 int i; 470 471 for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) { 472 lpm_tree = &mlxsw_sp->router->lpm.trees[i]; 473 if (lpm_tree->ref_count == 0) 474 return lpm_tree; 475 } 476 return NULL; 477 } 478 479 static int mlxsw_sp_lpm_tree_alloc(struct mlxsw_sp *mlxsw_sp, 480 struct mlxsw_sp_lpm_tree *lpm_tree) 481 { 482 char ralta_pl[MLXSW_REG_RALTA_LEN]; 483 484 mlxsw_reg_ralta_pack(ralta_pl, true, 485 (enum mlxsw_reg_ralxx_protocol) lpm_tree->proto, 486 lpm_tree->id); 487 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), ralta_pl); 488 } 489 490 static int mlxsw_sp_lpm_tree_free(struct mlxsw_sp *mlxsw_sp, 491 struct mlxsw_sp_lpm_tree *lpm_tree) 492 { 493 char ralta_pl[MLXSW_REG_RALTA_LEN]; 494 495 mlxsw_reg_ralta_pack(ralta_pl, false, 496 (enum mlxsw_reg_ralxx_protocol) lpm_tree->proto, 497 lpm_tree->id); 498 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), ralta_pl); 499 } 500 501 static int 502 mlxsw_sp_lpm_tree_left_struct_set(struct mlxsw_sp *mlxsw_sp, 503 struct mlxsw_sp_prefix_usage *prefix_usage, 504 struct mlxsw_sp_lpm_tree *lpm_tree) 505 { 506 char ralst_pl[MLXSW_REG_RALST_LEN]; 507 u8 root_bin = 0; 508 u8 prefix; 509 u8 last_prefix = MLXSW_REG_RALST_BIN_NO_CHILD; 510 511 mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) 512 root_bin = prefix; 513 514 mlxsw_reg_ralst_pack(ralst_pl, root_bin, lpm_tree->id); 515 mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) { 516 if (prefix == 0) 517 continue; 518 mlxsw_reg_ralst_bin_pack(ralst_pl, prefix, last_prefix, 519 MLXSW_REG_RALST_BIN_NO_CHILD); 520 last_prefix = prefix; 521 } 522 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralst), ralst_pl); 523 } 524 525 static struct mlxsw_sp_lpm_tree * 526 mlxsw_sp_lpm_tree_create(struct mlxsw_sp *mlxsw_sp, 527 struct mlxsw_sp_prefix_usage *prefix_usage, 528 enum mlxsw_sp_l3proto proto) 529 { 530 struct mlxsw_sp_lpm_tree *lpm_tree; 531 int err; 532 533 lpm_tree = mlxsw_sp_lpm_tree_find_unused(mlxsw_sp); 534 if (!lpm_tree) 535 return ERR_PTR(-EBUSY); 536 lpm_tree->proto = proto; 537 err = mlxsw_sp_lpm_tree_alloc(mlxsw_sp, lpm_tree); 538 if (err) 539 return ERR_PTR(err); 540 541 err = mlxsw_sp_lpm_tree_left_struct_set(mlxsw_sp, prefix_usage, 542 lpm_tree); 543 if (err) 544 goto err_left_struct_set; 545 memcpy(&lpm_tree->prefix_usage, prefix_usage, 546 sizeof(lpm_tree->prefix_usage)); 547 return lpm_tree; 548 549 err_left_struct_set: 550 mlxsw_sp_lpm_tree_free(mlxsw_sp, lpm_tree); 551 return ERR_PTR(err); 552 } 553 554 static int mlxsw_sp_lpm_tree_destroy(struct mlxsw_sp *mlxsw_sp, 555 struct mlxsw_sp_lpm_tree *lpm_tree) 556 { 557 return mlxsw_sp_lpm_tree_free(mlxsw_sp, lpm_tree); 558 } 559 560 static struct mlxsw_sp_lpm_tree * 561 mlxsw_sp_lpm_tree_get(struct mlxsw_sp *mlxsw_sp, 562 struct mlxsw_sp_prefix_usage *prefix_usage, 563 enum mlxsw_sp_l3proto proto) 564 { 565 struct mlxsw_sp_lpm_tree *lpm_tree; 566 int i; 567 568 for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) { 569 lpm_tree = &mlxsw_sp->router->lpm.trees[i]; 570 if (lpm_tree->ref_count != 0 && 571 lpm_tree->proto == proto && 572 mlxsw_sp_prefix_usage_eq(&lpm_tree->prefix_usage, 573 prefix_usage)) 574 goto inc_ref_count; 575 } 576 lpm_tree = mlxsw_sp_lpm_tree_create(mlxsw_sp, prefix_usage, 577 proto); 578 if (IS_ERR(lpm_tree)) 579 return lpm_tree; 580 581 inc_ref_count: 582 lpm_tree->ref_count++; 583 return lpm_tree; 584 } 585 586 static int mlxsw_sp_lpm_tree_put(struct mlxsw_sp *mlxsw_sp, 587 struct mlxsw_sp_lpm_tree *lpm_tree) 588 { 589 if (--lpm_tree->ref_count == 0) 590 return mlxsw_sp_lpm_tree_destroy(mlxsw_sp, lpm_tree); 591 return 0; 592 } 593 594 #define MLXSW_SP_LPM_TREE_MIN 1 /* tree 0 is reserved */ 595 596 static int mlxsw_sp_lpm_init(struct mlxsw_sp *mlxsw_sp) 597 { 598 struct mlxsw_sp_lpm_tree *lpm_tree; 599 u64 max_trees; 600 int i; 601 602 if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_LPM_TREES)) 603 return -EIO; 604 605 max_trees = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_LPM_TREES); 606 mlxsw_sp->router->lpm.tree_count = max_trees - MLXSW_SP_LPM_TREE_MIN; 607 mlxsw_sp->router->lpm.trees = kcalloc(mlxsw_sp->router->lpm.tree_count, 608 sizeof(struct mlxsw_sp_lpm_tree), 609 GFP_KERNEL); 610 if (!mlxsw_sp->router->lpm.trees) 611 return -ENOMEM; 612 613 for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) { 614 lpm_tree = &mlxsw_sp->router->lpm.trees[i]; 615 lpm_tree->id = i + MLXSW_SP_LPM_TREE_MIN; 616 } 617 618 return 0; 619 } 620 621 static void mlxsw_sp_lpm_fini(struct mlxsw_sp *mlxsw_sp) 622 { 623 kfree(mlxsw_sp->router->lpm.trees); 624 } 625 626 static bool mlxsw_sp_vr_is_used(const struct mlxsw_sp_vr *vr) 627 { 628 return !!vr->fib4; 629 } 630 631 static struct mlxsw_sp_vr *mlxsw_sp_vr_find_unused(struct mlxsw_sp *mlxsw_sp) 632 { 633 struct mlxsw_sp_vr *vr; 634 int i; 635 636 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) { 637 vr = &mlxsw_sp->router->vrs[i]; 638 if (!mlxsw_sp_vr_is_used(vr)) 639 return vr; 640 } 641 return NULL; 642 } 643 644 static int mlxsw_sp_vr_lpm_tree_bind(struct mlxsw_sp *mlxsw_sp, 645 const struct mlxsw_sp_fib *fib) 646 { 647 char raltb_pl[MLXSW_REG_RALTB_LEN]; 648 649 mlxsw_reg_raltb_pack(raltb_pl, fib->vr->id, 650 (enum mlxsw_reg_ralxx_protocol) fib->proto, 651 fib->lpm_tree->id); 652 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb), raltb_pl); 653 } 654 655 static int mlxsw_sp_vr_lpm_tree_unbind(struct mlxsw_sp *mlxsw_sp, 656 const struct mlxsw_sp_fib *fib) 657 { 658 char raltb_pl[MLXSW_REG_RALTB_LEN]; 659 660 /* Bind to tree 0 which is default */ 661 mlxsw_reg_raltb_pack(raltb_pl, fib->vr->id, 662 (enum mlxsw_reg_ralxx_protocol) fib->proto, 0); 663 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb), raltb_pl); 664 } 665 666 static u32 mlxsw_sp_fix_tb_id(u32 tb_id) 667 { 668 /* For our purpose, squash main and local table into one */ 669 if (tb_id == RT_TABLE_LOCAL) 670 tb_id = RT_TABLE_MAIN; 671 return tb_id; 672 } 673 674 static struct mlxsw_sp_vr *mlxsw_sp_vr_find(struct mlxsw_sp *mlxsw_sp, 675 u32 tb_id) 676 { 677 struct mlxsw_sp_vr *vr; 678 int i; 679 680 tb_id = mlxsw_sp_fix_tb_id(tb_id); 681 682 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) { 683 vr = &mlxsw_sp->router->vrs[i]; 684 if (mlxsw_sp_vr_is_used(vr) && vr->tb_id == tb_id) 685 return vr; 686 } 687 return NULL; 688 } 689 690 static struct mlxsw_sp_fib *mlxsw_sp_vr_fib(const struct mlxsw_sp_vr *vr, 691 enum mlxsw_sp_l3proto proto) 692 { 693 switch (proto) { 694 case MLXSW_SP_L3_PROTO_IPV4: 695 return vr->fib4; 696 case MLXSW_SP_L3_PROTO_IPV6: 697 BUG_ON(1); 698 } 699 return NULL; 700 } 701 702 static struct mlxsw_sp_vr *mlxsw_sp_vr_create(struct mlxsw_sp *mlxsw_sp, 703 u32 tb_id) 704 { 705 struct mlxsw_sp_vr *vr; 706 707 vr = mlxsw_sp_vr_find_unused(mlxsw_sp); 708 if (!vr) 709 return ERR_PTR(-EBUSY); 710 vr->fib4 = mlxsw_sp_fib_create(vr, MLXSW_SP_L3_PROTO_IPV4); 711 if (IS_ERR(vr->fib4)) 712 return ERR_CAST(vr->fib4); 713 vr->tb_id = tb_id; 714 return vr; 715 } 716 717 static void mlxsw_sp_vr_destroy(struct mlxsw_sp_vr *vr) 718 { 719 mlxsw_sp_fib_destroy(vr->fib4); 720 vr->fib4 = NULL; 721 } 722 723 static int 724 mlxsw_sp_vr_lpm_tree_check(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp_fib *fib, 725 struct mlxsw_sp_prefix_usage *req_prefix_usage) 726 { 727 struct mlxsw_sp_lpm_tree *lpm_tree = fib->lpm_tree; 728 struct mlxsw_sp_lpm_tree *new_tree; 729 int err; 730 731 if (mlxsw_sp_prefix_usage_eq(req_prefix_usage, &lpm_tree->prefix_usage)) 732 return 0; 733 734 new_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, req_prefix_usage, 735 fib->proto); 736 if (IS_ERR(new_tree)) { 737 /* We failed to get a tree according to the required 738 * prefix usage. However, the current tree might be still good 739 * for us if our requirement is subset of the prefixes used 740 * in the tree. 741 */ 742 if (mlxsw_sp_prefix_usage_subset(req_prefix_usage, 743 &lpm_tree->prefix_usage)) 744 return 0; 745 return PTR_ERR(new_tree); 746 } 747 748 /* Prevent packet loss by overwriting existing binding */ 749 fib->lpm_tree = new_tree; 750 err = mlxsw_sp_vr_lpm_tree_bind(mlxsw_sp, fib); 751 if (err) 752 goto err_tree_bind; 753 mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree); 754 755 return 0; 756 757 err_tree_bind: 758 fib->lpm_tree = lpm_tree; 759 mlxsw_sp_lpm_tree_put(mlxsw_sp, new_tree); 760 return err; 761 } 762 763 static struct mlxsw_sp_vr *mlxsw_sp_vr_get(struct mlxsw_sp *mlxsw_sp, u32 tb_id) 764 { 765 struct mlxsw_sp_vr *vr; 766 767 tb_id = mlxsw_sp_fix_tb_id(tb_id); 768 vr = mlxsw_sp_vr_find(mlxsw_sp, tb_id); 769 if (!vr) 770 vr = mlxsw_sp_vr_create(mlxsw_sp, tb_id); 771 return vr; 772 } 773 774 static void mlxsw_sp_vr_put(struct mlxsw_sp_vr *vr) 775 { 776 if (!vr->rif_count && list_empty(&vr->fib4->node_list)) 777 mlxsw_sp_vr_destroy(vr); 778 } 779 780 static int mlxsw_sp_vrs_init(struct mlxsw_sp *mlxsw_sp) 781 { 782 struct mlxsw_sp_vr *vr; 783 u64 max_vrs; 784 int i; 785 786 if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_VRS)) 787 return -EIO; 788 789 max_vrs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); 790 mlxsw_sp->router->vrs = kcalloc(max_vrs, sizeof(struct mlxsw_sp_vr), 791 GFP_KERNEL); 792 if (!mlxsw_sp->router->vrs) 793 return -ENOMEM; 794 795 for (i = 0; i < max_vrs; i++) { 796 vr = &mlxsw_sp->router->vrs[i]; 797 vr->id = i; 798 } 799 800 return 0; 801 } 802 803 static void mlxsw_sp_router_fib_flush(struct mlxsw_sp *mlxsw_sp); 804 805 static void mlxsw_sp_vrs_fini(struct mlxsw_sp *mlxsw_sp) 806 { 807 /* At this stage we're guaranteed not to have new incoming 808 * FIB notifications and the work queue is free from FIBs 809 * sitting on top of mlxsw netdevs. However, we can still 810 * have other FIBs queued. Flush the queue before flushing 811 * the device's tables. No need for locks, as we're the only 812 * writer. 813 */ 814 mlxsw_core_flush_owq(); 815 mlxsw_sp_router_fib_flush(mlxsw_sp); 816 kfree(mlxsw_sp->router->vrs); 817 } 818 819 struct mlxsw_sp_neigh_key { 820 struct neighbour *n; 821 }; 822 823 struct mlxsw_sp_neigh_entry { 824 struct list_head rif_list_node; 825 struct rhash_head ht_node; 826 struct mlxsw_sp_neigh_key key; 827 u16 rif; 828 bool connected; 829 unsigned char ha[ETH_ALEN]; 830 struct list_head nexthop_list; /* list of nexthops using 831 * this neigh entry 832 */ 833 struct list_head nexthop_neighs_list_node; 834 }; 835 836 static const struct rhashtable_params mlxsw_sp_neigh_ht_params = { 837 .key_offset = offsetof(struct mlxsw_sp_neigh_entry, key), 838 .head_offset = offsetof(struct mlxsw_sp_neigh_entry, ht_node), 839 .key_len = sizeof(struct mlxsw_sp_neigh_key), 840 }; 841 842 static struct mlxsw_sp_neigh_entry * 843 mlxsw_sp_neigh_entry_alloc(struct mlxsw_sp *mlxsw_sp, struct neighbour *n, 844 u16 rif) 845 { 846 struct mlxsw_sp_neigh_entry *neigh_entry; 847 848 neigh_entry = kzalloc(sizeof(*neigh_entry), GFP_KERNEL); 849 if (!neigh_entry) 850 return NULL; 851 852 neigh_entry->key.n = n; 853 neigh_entry->rif = rif; 854 INIT_LIST_HEAD(&neigh_entry->nexthop_list); 855 856 return neigh_entry; 857 } 858 859 static void mlxsw_sp_neigh_entry_free(struct mlxsw_sp_neigh_entry *neigh_entry) 860 { 861 kfree(neigh_entry); 862 } 863 864 static int 865 mlxsw_sp_neigh_entry_insert(struct mlxsw_sp *mlxsw_sp, 866 struct mlxsw_sp_neigh_entry *neigh_entry) 867 { 868 return rhashtable_insert_fast(&mlxsw_sp->router->neigh_ht, 869 &neigh_entry->ht_node, 870 mlxsw_sp_neigh_ht_params); 871 } 872 873 static void 874 mlxsw_sp_neigh_entry_remove(struct mlxsw_sp *mlxsw_sp, 875 struct mlxsw_sp_neigh_entry *neigh_entry) 876 { 877 rhashtable_remove_fast(&mlxsw_sp->router->neigh_ht, 878 &neigh_entry->ht_node, 879 mlxsw_sp_neigh_ht_params); 880 } 881 882 static struct mlxsw_sp_neigh_entry * 883 mlxsw_sp_neigh_entry_create(struct mlxsw_sp *mlxsw_sp, struct neighbour *n) 884 { 885 struct mlxsw_sp_neigh_entry *neigh_entry; 886 struct mlxsw_sp_rif *rif; 887 int err; 888 889 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, n->dev); 890 if (!rif) 891 return ERR_PTR(-EINVAL); 892 893 neigh_entry = mlxsw_sp_neigh_entry_alloc(mlxsw_sp, n, rif->rif_index); 894 if (!neigh_entry) 895 return ERR_PTR(-ENOMEM); 896 897 err = mlxsw_sp_neigh_entry_insert(mlxsw_sp, neigh_entry); 898 if (err) 899 goto err_neigh_entry_insert; 900 901 list_add(&neigh_entry->rif_list_node, &rif->neigh_list); 902 903 return neigh_entry; 904 905 err_neigh_entry_insert: 906 mlxsw_sp_neigh_entry_free(neigh_entry); 907 return ERR_PTR(err); 908 } 909 910 static void 911 mlxsw_sp_neigh_entry_destroy(struct mlxsw_sp *mlxsw_sp, 912 struct mlxsw_sp_neigh_entry *neigh_entry) 913 { 914 list_del(&neigh_entry->rif_list_node); 915 mlxsw_sp_neigh_entry_remove(mlxsw_sp, neigh_entry); 916 mlxsw_sp_neigh_entry_free(neigh_entry); 917 } 918 919 static struct mlxsw_sp_neigh_entry * 920 mlxsw_sp_neigh_entry_lookup(struct mlxsw_sp *mlxsw_sp, struct neighbour *n) 921 { 922 struct mlxsw_sp_neigh_key key; 923 924 key.n = n; 925 return rhashtable_lookup_fast(&mlxsw_sp->router->neigh_ht, 926 &key, mlxsw_sp_neigh_ht_params); 927 } 928 929 static void 930 mlxsw_sp_router_neighs_update_interval_init(struct mlxsw_sp *mlxsw_sp) 931 { 932 unsigned long interval = NEIGH_VAR(&arp_tbl.parms, DELAY_PROBE_TIME); 933 934 mlxsw_sp->router->neighs_update.interval = jiffies_to_msecs(interval); 935 } 936 937 static void mlxsw_sp_router_neigh_ent_ipv4_process(struct mlxsw_sp *mlxsw_sp, 938 char *rauhtd_pl, 939 int ent_index) 940 { 941 struct net_device *dev; 942 struct neighbour *n; 943 __be32 dipn; 944 u32 dip; 945 u16 rif; 946 947 mlxsw_reg_rauhtd_ent_ipv4_unpack(rauhtd_pl, ent_index, &rif, &dip); 948 949 if (!mlxsw_sp->router->rifs[rif]) { 950 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Incorrect RIF in neighbour entry\n"); 951 return; 952 } 953 954 dipn = htonl(dip); 955 dev = mlxsw_sp->router->rifs[rif]->dev; 956 n = neigh_lookup(&arp_tbl, &dipn, dev); 957 if (!n) { 958 netdev_err(dev, "Failed to find matching neighbour for IP=%pI4h\n", 959 &dip); 960 return; 961 } 962 963 netdev_dbg(dev, "Updating neighbour with IP=%pI4h\n", &dip); 964 neigh_event_send(n, NULL); 965 neigh_release(n); 966 } 967 968 static void mlxsw_sp_router_neigh_rec_ipv4_process(struct mlxsw_sp *mlxsw_sp, 969 char *rauhtd_pl, 970 int rec_index) 971 { 972 u8 num_entries; 973 int i; 974 975 num_entries = mlxsw_reg_rauhtd_ipv4_rec_num_entries_get(rauhtd_pl, 976 rec_index); 977 /* Hardware starts counting at 0, so add 1. */ 978 num_entries++; 979 980 /* Each record consists of several neighbour entries. */ 981 for (i = 0; i < num_entries; i++) { 982 int ent_index; 983 984 ent_index = rec_index * MLXSW_REG_RAUHTD_IPV4_ENT_PER_REC + i; 985 mlxsw_sp_router_neigh_ent_ipv4_process(mlxsw_sp, rauhtd_pl, 986 ent_index); 987 } 988 989 } 990 991 static void mlxsw_sp_router_neigh_rec_process(struct mlxsw_sp *mlxsw_sp, 992 char *rauhtd_pl, int rec_index) 993 { 994 switch (mlxsw_reg_rauhtd_rec_type_get(rauhtd_pl, rec_index)) { 995 case MLXSW_REG_RAUHTD_TYPE_IPV4: 996 mlxsw_sp_router_neigh_rec_ipv4_process(mlxsw_sp, rauhtd_pl, 997 rec_index); 998 break; 999 case MLXSW_REG_RAUHTD_TYPE_IPV6: 1000 WARN_ON_ONCE(1); 1001 break; 1002 } 1003 } 1004 1005 static bool mlxsw_sp_router_rauhtd_is_full(char *rauhtd_pl) 1006 { 1007 u8 num_rec, last_rec_index, num_entries; 1008 1009 num_rec = mlxsw_reg_rauhtd_num_rec_get(rauhtd_pl); 1010 last_rec_index = num_rec - 1; 1011 1012 if (num_rec < MLXSW_REG_RAUHTD_REC_MAX_NUM) 1013 return false; 1014 if (mlxsw_reg_rauhtd_rec_type_get(rauhtd_pl, last_rec_index) == 1015 MLXSW_REG_RAUHTD_TYPE_IPV6) 1016 return true; 1017 1018 num_entries = mlxsw_reg_rauhtd_ipv4_rec_num_entries_get(rauhtd_pl, 1019 last_rec_index); 1020 if (++num_entries == MLXSW_REG_RAUHTD_IPV4_ENT_PER_REC) 1021 return true; 1022 return false; 1023 } 1024 1025 static int mlxsw_sp_router_neighs_update_rauhtd(struct mlxsw_sp *mlxsw_sp) 1026 { 1027 char *rauhtd_pl; 1028 u8 num_rec; 1029 int i, err; 1030 1031 rauhtd_pl = kmalloc(MLXSW_REG_RAUHTD_LEN, GFP_KERNEL); 1032 if (!rauhtd_pl) 1033 return -ENOMEM; 1034 1035 /* Make sure the neighbour's netdev isn't removed in the 1036 * process. 1037 */ 1038 rtnl_lock(); 1039 do { 1040 mlxsw_reg_rauhtd_pack(rauhtd_pl, MLXSW_REG_RAUHTD_TYPE_IPV4); 1041 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(rauhtd), 1042 rauhtd_pl); 1043 if (err) { 1044 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Failed to dump neighbour talbe\n"); 1045 break; 1046 } 1047 num_rec = mlxsw_reg_rauhtd_num_rec_get(rauhtd_pl); 1048 for (i = 0; i < num_rec; i++) 1049 mlxsw_sp_router_neigh_rec_process(mlxsw_sp, rauhtd_pl, 1050 i); 1051 } while (mlxsw_sp_router_rauhtd_is_full(rauhtd_pl)); 1052 rtnl_unlock(); 1053 1054 kfree(rauhtd_pl); 1055 return err; 1056 } 1057 1058 static void mlxsw_sp_router_neighs_update_nh(struct mlxsw_sp *mlxsw_sp) 1059 { 1060 struct mlxsw_sp_neigh_entry *neigh_entry; 1061 1062 /* Take RTNL mutex here to prevent lists from changes */ 1063 rtnl_lock(); 1064 list_for_each_entry(neigh_entry, &mlxsw_sp->router->nexthop_neighs_list, 1065 nexthop_neighs_list_node) 1066 /* If this neigh have nexthops, make the kernel think this neigh 1067 * is active regardless of the traffic. 1068 */ 1069 neigh_event_send(neigh_entry->key.n, NULL); 1070 rtnl_unlock(); 1071 } 1072 1073 static void 1074 mlxsw_sp_router_neighs_update_work_schedule(struct mlxsw_sp *mlxsw_sp) 1075 { 1076 unsigned long interval = mlxsw_sp->router->neighs_update.interval; 1077 1078 mlxsw_core_schedule_dw(&mlxsw_sp->router->neighs_update.dw, 1079 msecs_to_jiffies(interval)); 1080 } 1081 1082 static void mlxsw_sp_router_neighs_update_work(struct work_struct *work) 1083 { 1084 struct mlxsw_sp_router *router; 1085 int err; 1086 1087 router = container_of(work, struct mlxsw_sp_router, 1088 neighs_update.dw.work); 1089 err = mlxsw_sp_router_neighs_update_rauhtd(router->mlxsw_sp); 1090 if (err) 1091 dev_err(router->mlxsw_sp->bus_info->dev, "Could not update kernel for neigh activity"); 1092 1093 mlxsw_sp_router_neighs_update_nh(router->mlxsw_sp); 1094 1095 mlxsw_sp_router_neighs_update_work_schedule(router->mlxsw_sp); 1096 } 1097 1098 static void mlxsw_sp_router_probe_unresolved_nexthops(struct work_struct *work) 1099 { 1100 struct mlxsw_sp_neigh_entry *neigh_entry; 1101 struct mlxsw_sp_router *router; 1102 1103 router = container_of(work, struct mlxsw_sp_router, 1104 nexthop_probe_dw.work); 1105 /* Iterate over nexthop neighbours, find those who are unresolved and 1106 * send arp on them. This solves the chicken-egg problem when 1107 * the nexthop wouldn't get offloaded until the neighbor is resolved 1108 * but it wouldn't get resolved ever in case traffic is flowing in HW 1109 * using different nexthop. 1110 * 1111 * Take RTNL mutex here to prevent lists from changes. 1112 */ 1113 rtnl_lock(); 1114 list_for_each_entry(neigh_entry, &router->nexthop_neighs_list, 1115 nexthop_neighs_list_node) 1116 if (!neigh_entry->connected) 1117 neigh_event_send(neigh_entry->key.n, NULL); 1118 rtnl_unlock(); 1119 1120 mlxsw_core_schedule_dw(&router->nexthop_probe_dw, 1121 MLXSW_SP_UNRESOLVED_NH_PROBE_INTERVAL); 1122 } 1123 1124 static void 1125 mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp *mlxsw_sp, 1126 struct mlxsw_sp_neigh_entry *neigh_entry, 1127 bool removing); 1128 1129 static enum mlxsw_reg_rauht_op mlxsw_sp_rauht_op(bool adding) 1130 { 1131 return adding ? MLXSW_REG_RAUHT_OP_WRITE_ADD : 1132 MLXSW_REG_RAUHT_OP_WRITE_DELETE; 1133 } 1134 1135 static void 1136 mlxsw_sp_router_neigh_entry_op4(struct mlxsw_sp *mlxsw_sp, 1137 struct mlxsw_sp_neigh_entry *neigh_entry, 1138 enum mlxsw_reg_rauht_op op) 1139 { 1140 struct neighbour *n = neigh_entry->key.n; 1141 u32 dip = ntohl(*((__be32 *) n->primary_key)); 1142 char rauht_pl[MLXSW_REG_RAUHT_LEN]; 1143 1144 mlxsw_reg_rauht_pack4(rauht_pl, op, neigh_entry->rif, neigh_entry->ha, 1145 dip); 1146 mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rauht), rauht_pl); 1147 } 1148 1149 static void 1150 mlxsw_sp_neigh_entry_update(struct mlxsw_sp *mlxsw_sp, 1151 struct mlxsw_sp_neigh_entry *neigh_entry, 1152 bool adding) 1153 { 1154 if (!adding && !neigh_entry->connected) 1155 return; 1156 neigh_entry->connected = adding; 1157 if (neigh_entry->key.n->tbl == &arp_tbl) 1158 mlxsw_sp_router_neigh_entry_op4(mlxsw_sp, neigh_entry, 1159 mlxsw_sp_rauht_op(adding)); 1160 else 1161 WARN_ON_ONCE(1); 1162 } 1163 1164 struct mlxsw_sp_neigh_event_work { 1165 struct work_struct work; 1166 struct mlxsw_sp *mlxsw_sp; 1167 struct neighbour *n; 1168 }; 1169 1170 static void mlxsw_sp_router_neigh_event_work(struct work_struct *work) 1171 { 1172 struct mlxsw_sp_neigh_event_work *neigh_work = 1173 container_of(work, struct mlxsw_sp_neigh_event_work, work); 1174 struct mlxsw_sp *mlxsw_sp = neigh_work->mlxsw_sp; 1175 struct mlxsw_sp_neigh_entry *neigh_entry; 1176 struct neighbour *n = neigh_work->n; 1177 unsigned char ha[ETH_ALEN]; 1178 bool entry_connected; 1179 u8 nud_state, dead; 1180 1181 /* If these parameters are changed after we release the lock, 1182 * then we are guaranteed to receive another event letting us 1183 * know about it. 1184 */ 1185 read_lock_bh(&n->lock); 1186 memcpy(ha, n->ha, ETH_ALEN); 1187 nud_state = n->nud_state; 1188 dead = n->dead; 1189 read_unlock_bh(&n->lock); 1190 1191 rtnl_lock(); 1192 entry_connected = nud_state & NUD_VALID && !dead; 1193 neigh_entry = mlxsw_sp_neigh_entry_lookup(mlxsw_sp, n); 1194 if (!entry_connected && !neigh_entry) 1195 goto out; 1196 if (!neigh_entry) { 1197 neigh_entry = mlxsw_sp_neigh_entry_create(mlxsw_sp, n); 1198 if (IS_ERR(neigh_entry)) 1199 goto out; 1200 } 1201 1202 memcpy(neigh_entry->ha, ha, ETH_ALEN); 1203 mlxsw_sp_neigh_entry_update(mlxsw_sp, neigh_entry, entry_connected); 1204 mlxsw_sp_nexthop_neigh_update(mlxsw_sp, neigh_entry, !entry_connected); 1205 1206 if (!neigh_entry->connected && list_empty(&neigh_entry->nexthop_list)) 1207 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry); 1208 1209 out: 1210 rtnl_unlock(); 1211 neigh_release(n); 1212 kfree(neigh_work); 1213 } 1214 1215 int mlxsw_sp_router_netevent_event(struct notifier_block *unused, 1216 unsigned long event, void *ptr) 1217 { 1218 struct mlxsw_sp_neigh_event_work *neigh_work; 1219 struct mlxsw_sp_port *mlxsw_sp_port; 1220 struct mlxsw_sp *mlxsw_sp; 1221 unsigned long interval; 1222 struct neigh_parms *p; 1223 struct neighbour *n; 1224 1225 switch (event) { 1226 case NETEVENT_DELAY_PROBE_TIME_UPDATE: 1227 p = ptr; 1228 1229 /* We don't care about changes in the default table. */ 1230 if (!p->dev || p->tbl != &arp_tbl) 1231 return NOTIFY_DONE; 1232 1233 /* We are in atomic context and can't take RTNL mutex, 1234 * so use RCU variant to walk the device chain. 1235 */ 1236 mlxsw_sp_port = mlxsw_sp_port_lower_dev_hold(p->dev); 1237 if (!mlxsw_sp_port) 1238 return NOTIFY_DONE; 1239 1240 mlxsw_sp = mlxsw_sp_port->mlxsw_sp; 1241 interval = jiffies_to_msecs(NEIGH_VAR(p, DELAY_PROBE_TIME)); 1242 mlxsw_sp->router->neighs_update.interval = interval; 1243 1244 mlxsw_sp_port_dev_put(mlxsw_sp_port); 1245 break; 1246 case NETEVENT_NEIGH_UPDATE: 1247 n = ptr; 1248 1249 if (n->tbl != &arp_tbl) 1250 return NOTIFY_DONE; 1251 1252 mlxsw_sp_port = mlxsw_sp_port_lower_dev_hold(n->dev); 1253 if (!mlxsw_sp_port) 1254 return NOTIFY_DONE; 1255 1256 neigh_work = kzalloc(sizeof(*neigh_work), GFP_ATOMIC); 1257 if (!neigh_work) { 1258 mlxsw_sp_port_dev_put(mlxsw_sp_port); 1259 return NOTIFY_BAD; 1260 } 1261 1262 INIT_WORK(&neigh_work->work, mlxsw_sp_router_neigh_event_work); 1263 neigh_work->mlxsw_sp = mlxsw_sp_port->mlxsw_sp; 1264 neigh_work->n = n; 1265 1266 /* Take a reference to ensure the neighbour won't be 1267 * destructed until we drop the reference in delayed 1268 * work. 1269 */ 1270 neigh_clone(n); 1271 mlxsw_core_schedule_work(&neigh_work->work); 1272 mlxsw_sp_port_dev_put(mlxsw_sp_port); 1273 break; 1274 } 1275 1276 return NOTIFY_DONE; 1277 } 1278 1279 static int mlxsw_sp_neigh_init(struct mlxsw_sp *mlxsw_sp) 1280 { 1281 int err; 1282 1283 err = rhashtable_init(&mlxsw_sp->router->neigh_ht, 1284 &mlxsw_sp_neigh_ht_params); 1285 if (err) 1286 return err; 1287 1288 /* Initialize the polling interval according to the default 1289 * table. 1290 */ 1291 mlxsw_sp_router_neighs_update_interval_init(mlxsw_sp); 1292 1293 /* Create the delayed works for the activity_update */ 1294 INIT_DELAYED_WORK(&mlxsw_sp->router->neighs_update.dw, 1295 mlxsw_sp_router_neighs_update_work); 1296 INIT_DELAYED_WORK(&mlxsw_sp->router->nexthop_probe_dw, 1297 mlxsw_sp_router_probe_unresolved_nexthops); 1298 mlxsw_core_schedule_dw(&mlxsw_sp->router->neighs_update.dw, 0); 1299 mlxsw_core_schedule_dw(&mlxsw_sp->router->nexthop_probe_dw, 0); 1300 return 0; 1301 } 1302 1303 static void mlxsw_sp_neigh_fini(struct mlxsw_sp *mlxsw_sp) 1304 { 1305 cancel_delayed_work_sync(&mlxsw_sp->router->neighs_update.dw); 1306 cancel_delayed_work_sync(&mlxsw_sp->router->nexthop_probe_dw); 1307 rhashtable_destroy(&mlxsw_sp->router->neigh_ht); 1308 } 1309 1310 static int mlxsw_sp_neigh_rif_flush(struct mlxsw_sp *mlxsw_sp, 1311 const struct mlxsw_sp_rif *rif) 1312 { 1313 char rauht_pl[MLXSW_REG_RAUHT_LEN]; 1314 1315 mlxsw_reg_rauht_pack(rauht_pl, MLXSW_REG_RAUHT_OP_WRITE_DELETE_ALL, 1316 rif->rif_index, rif->addr); 1317 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rauht), rauht_pl); 1318 } 1319 1320 static void mlxsw_sp_neigh_rif_gone_sync(struct mlxsw_sp *mlxsw_sp, 1321 struct mlxsw_sp_rif *rif) 1322 { 1323 struct mlxsw_sp_neigh_entry *neigh_entry, *tmp; 1324 1325 mlxsw_sp_neigh_rif_flush(mlxsw_sp, rif); 1326 list_for_each_entry_safe(neigh_entry, tmp, &rif->neigh_list, 1327 rif_list_node) 1328 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry); 1329 } 1330 1331 struct mlxsw_sp_nexthop_key { 1332 struct fib_nh *fib_nh; 1333 }; 1334 1335 struct mlxsw_sp_nexthop { 1336 struct list_head neigh_list_node; /* member of neigh entry list */ 1337 struct list_head rif_list_node; 1338 struct mlxsw_sp_nexthop_group *nh_grp; /* pointer back to the group 1339 * this belongs to 1340 */ 1341 struct rhash_head ht_node; 1342 struct mlxsw_sp_nexthop_key key; 1343 struct mlxsw_sp_rif *rif; 1344 u8 should_offload:1, /* set indicates this neigh is connected and 1345 * should be put to KVD linear area of this group. 1346 */ 1347 offloaded:1, /* set in case the neigh is actually put into 1348 * KVD linear area of this group. 1349 */ 1350 update:1; /* set indicates that MAC of this neigh should be 1351 * updated in HW 1352 */ 1353 struct mlxsw_sp_neigh_entry *neigh_entry; 1354 }; 1355 1356 struct mlxsw_sp_nexthop_group_key { 1357 struct fib_info *fi; 1358 }; 1359 1360 struct mlxsw_sp_nexthop_group { 1361 struct rhash_head ht_node; 1362 struct list_head fib_list; /* list of fib entries that use this group */ 1363 struct mlxsw_sp_nexthop_group_key key; 1364 u8 adj_index_valid:1, 1365 gateway:1; /* routes using the group use a gateway */ 1366 u32 adj_index; 1367 u16 ecmp_size; 1368 u16 count; 1369 struct mlxsw_sp_nexthop nexthops[0]; 1370 #define nh_rif nexthops[0].rif 1371 }; 1372 1373 static const struct rhashtable_params mlxsw_sp_nexthop_group_ht_params = { 1374 .key_offset = offsetof(struct mlxsw_sp_nexthop_group, key), 1375 .head_offset = offsetof(struct mlxsw_sp_nexthop_group, ht_node), 1376 .key_len = sizeof(struct mlxsw_sp_nexthop_group_key), 1377 }; 1378 1379 static int mlxsw_sp_nexthop_group_insert(struct mlxsw_sp *mlxsw_sp, 1380 struct mlxsw_sp_nexthop_group *nh_grp) 1381 { 1382 return rhashtable_insert_fast(&mlxsw_sp->router->nexthop_group_ht, 1383 &nh_grp->ht_node, 1384 mlxsw_sp_nexthop_group_ht_params); 1385 } 1386 1387 static void mlxsw_sp_nexthop_group_remove(struct mlxsw_sp *mlxsw_sp, 1388 struct mlxsw_sp_nexthop_group *nh_grp) 1389 { 1390 rhashtable_remove_fast(&mlxsw_sp->router->nexthop_group_ht, 1391 &nh_grp->ht_node, 1392 mlxsw_sp_nexthop_group_ht_params); 1393 } 1394 1395 static struct mlxsw_sp_nexthop_group * 1396 mlxsw_sp_nexthop_group_lookup(struct mlxsw_sp *mlxsw_sp, 1397 struct mlxsw_sp_nexthop_group_key key) 1398 { 1399 return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_group_ht, &key, 1400 mlxsw_sp_nexthop_group_ht_params); 1401 } 1402 1403 static const struct rhashtable_params mlxsw_sp_nexthop_ht_params = { 1404 .key_offset = offsetof(struct mlxsw_sp_nexthop, key), 1405 .head_offset = offsetof(struct mlxsw_sp_nexthop, ht_node), 1406 .key_len = sizeof(struct mlxsw_sp_nexthop_key), 1407 }; 1408 1409 static int mlxsw_sp_nexthop_insert(struct mlxsw_sp *mlxsw_sp, 1410 struct mlxsw_sp_nexthop *nh) 1411 { 1412 return rhashtable_insert_fast(&mlxsw_sp->router->nexthop_ht, 1413 &nh->ht_node, mlxsw_sp_nexthop_ht_params); 1414 } 1415 1416 static void mlxsw_sp_nexthop_remove(struct mlxsw_sp *mlxsw_sp, 1417 struct mlxsw_sp_nexthop *nh) 1418 { 1419 rhashtable_remove_fast(&mlxsw_sp->router->nexthop_ht, &nh->ht_node, 1420 mlxsw_sp_nexthop_ht_params); 1421 } 1422 1423 static struct mlxsw_sp_nexthop * 1424 mlxsw_sp_nexthop_lookup(struct mlxsw_sp *mlxsw_sp, 1425 struct mlxsw_sp_nexthop_key key) 1426 { 1427 return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_ht, &key, 1428 mlxsw_sp_nexthop_ht_params); 1429 } 1430 1431 static int mlxsw_sp_adj_index_mass_update_vr(struct mlxsw_sp *mlxsw_sp, 1432 const struct mlxsw_sp_fib *fib, 1433 u32 adj_index, u16 ecmp_size, 1434 u32 new_adj_index, 1435 u16 new_ecmp_size) 1436 { 1437 char raleu_pl[MLXSW_REG_RALEU_LEN]; 1438 1439 mlxsw_reg_raleu_pack(raleu_pl, 1440 (enum mlxsw_reg_ralxx_protocol) fib->proto, 1441 fib->vr->id, adj_index, ecmp_size, new_adj_index, 1442 new_ecmp_size); 1443 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raleu), raleu_pl); 1444 } 1445 1446 static int mlxsw_sp_adj_index_mass_update(struct mlxsw_sp *mlxsw_sp, 1447 struct mlxsw_sp_nexthop_group *nh_grp, 1448 u32 old_adj_index, u16 old_ecmp_size) 1449 { 1450 struct mlxsw_sp_fib_entry *fib_entry; 1451 struct mlxsw_sp_fib *fib = NULL; 1452 int err; 1453 1454 list_for_each_entry(fib_entry, &nh_grp->fib_list, nexthop_group_node) { 1455 if (fib == fib_entry->fib_node->fib) 1456 continue; 1457 fib = fib_entry->fib_node->fib; 1458 err = mlxsw_sp_adj_index_mass_update_vr(mlxsw_sp, fib, 1459 old_adj_index, 1460 old_ecmp_size, 1461 nh_grp->adj_index, 1462 nh_grp->ecmp_size); 1463 if (err) 1464 return err; 1465 } 1466 return 0; 1467 } 1468 1469 static int mlxsw_sp_nexthop_mac_update(struct mlxsw_sp *mlxsw_sp, u32 adj_index, 1470 struct mlxsw_sp_nexthop *nh) 1471 { 1472 struct mlxsw_sp_neigh_entry *neigh_entry = nh->neigh_entry; 1473 char ratr_pl[MLXSW_REG_RATR_LEN]; 1474 1475 mlxsw_reg_ratr_pack(ratr_pl, MLXSW_REG_RATR_OP_WRITE_WRITE_ENTRY, 1476 true, adj_index, neigh_entry->rif); 1477 mlxsw_reg_ratr_eth_entry_pack(ratr_pl, neigh_entry->ha); 1478 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ratr), ratr_pl); 1479 } 1480 1481 static int 1482 mlxsw_sp_nexthop_group_mac_update(struct mlxsw_sp *mlxsw_sp, 1483 struct mlxsw_sp_nexthop_group *nh_grp, 1484 bool reallocate) 1485 { 1486 u32 adj_index = nh_grp->adj_index; /* base */ 1487 struct mlxsw_sp_nexthop *nh; 1488 int i; 1489 int err; 1490 1491 for (i = 0; i < nh_grp->count; i++) { 1492 nh = &nh_grp->nexthops[i]; 1493 1494 if (!nh->should_offload) { 1495 nh->offloaded = 0; 1496 continue; 1497 } 1498 1499 if (nh->update || reallocate) { 1500 err = mlxsw_sp_nexthop_mac_update(mlxsw_sp, 1501 adj_index, nh); 1502 if (err) 1503 return err; 1504 nh->update = 0; 1505 nh->offloaded = 1; 1506 } 1507 adj_index++; 1508 } 1509 return 0; 1510 } 1511 1512 static int mlxsw_sp_fib_entry_update(struct mlxsw_sp *mlxsw_sp, 1513 struct mlxsw_sp_fib_entry *fib_entry); 1514 1515 static int 1516 mlxsw_sp_nexthop_fib_entries_update(struct mlxsw_sp *mlxsw_sp, 1517 struct mlxsw_sp_nexthop_group *nh_grp) 1518 { 1519 struct mlxsw_sp_fib_entry *fib_entry; 1520 int err; 1521 1522 list_for_each_entry(fib_entry, &nh_grp->fib_list, nexthop_group_node) { 1523 err = mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry); 1524 if (err) 1525 return err; 1526 } 1527 return 0; 1528 } 1529 1530 static void 1531 mlxsw_sp_nexthop_group_refresh(struct mlxsw_sp *mlxsw_sp, 1532 struct mlxsw_sp_nexthop_group *nh_grp) 1533 { 1534 struct mlxsw_sp_nexthop *nh; 1535 bool offload_change = false; 1536 u32 adj_index; 1537 u16 ecmp_size = 0; 1538 bool old_adj_index_valid; 1539 u32 old_adj_index; 1540 u16 old_ecmp_size; 1541 int i; 1542 int err; 1543 1544 if (!nh_grp->gateway) { 1545 mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp); 1546 return; 1547 } 1548 1549 for (i = 0; i < nh_grp->count; i++) { 1550 nh = &nh_grp->nexthops[i]; 1551 1552 if (nh->should_offload ^ nh->offloaded) { 1553 offload_change = true; 1554 if (nh->should_offload) 1555 nh->update = 1; 1556 } 1557 if (nh->should_offload) 1558 ecmp_size++; 1559 } 1560 if (!offload_change) { 1561 /* Nothing was added or removed, so no need to reallocate. Just 1562 * update MAC on existing adjacency indexes. 1563 */ 1564 err = mlxsw_sp_nexthop_group_mac_update(mlxsw_sp, nh_grp, 1565 false); 1566 if (err) { 1567 dev_warn(mlxsw_sp->bus_info->dev, "Failed to update neigh MAC in adjacency table.\n"); 1568 goto set_trap; 1569 } 1570 return; 1571 } 1572 if (!ecmp_size) 1573 /* No neigh of this group is connected so we just set 1574 * the trap and let everthing flow through kernel. 1575 */ 1576 goto set_trap; 1577 1578 err = mlxsw_sp_kvdl_alloc(mlxsw_sp, ecmp_size, &adj_index); 1579 if (err) { 1580 /* We ran out of KVD linear space, just set the 1581 * trap and let everything flow through kernel. 1582 */ 1583 dev_warn(mlxsw_sp->bus_info->dev, "Failed to allocate KVD linear area for nexthop group.\n"); 1584 goto set_trap; 1585 } 1586 old_adj_index_valid = nh_grp->adj_index_valid; 1587 old_adj_index = nh_grp->adj_index; 1588 old_ecmp_size = nh_grp->ecmp_size; 1589 nh_grp->adj_index_valid = 1; 1590 nh_grp->adj_index = adj_index; 1591 nh_grp->ecmp_size = ecmp_size; 1592 err = mlxsw_sp_nexthop_group_mac_update(mlxsw_sp, nh_grp, true); 1593 if (err) { 1594 dev_warn(mlxsw_sp->bus_info->dev, "Failed to update neigh MAC in adjacency table.\n"); 1595 goto set_trap; 1596 } 1597 1598 if (!old_adj_index_valid) { 1599 /* The trap was set for fib entries, so we have to call 1600 * fib entry update to unset it and use adjacency index. 1601 */ 1602 err = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp); 1603 if (err) { 1604 dev_warn(mlxsw_sp->bus_info->dev, "Failed to add adjacency index to fib entries.\n"); 1605 goto set_trap; 1606 } 1607 return; 1608 } 1609 1610 err = mlxsw_sp_adj_index_mass_update(mlxsw_sp, nh_grp, 1611 old_adj_index, old_ecmp_size); 1612 mlxsw_sp_kvdl_free(mlxsw_sp, old_adj_index); 1613 if (err) { 1614 dev_warn(mlxsw_sp->bus_info->dev, "Failed to mass-update adjacency index for nexthop group.\n"); 1615 goto set_trap; 1616 } 1617 return; 1618 1619 set_trap: 1620 old_adj_index_valid = nh_grp->adj_index_valid; 1621 nh_grp->adj_index_valid = 0; 1622 for (i = 0; i < nh_grp->count; i++) { 1623 nh = &nh_grp->nexthops[i]; 1624 nh->offloaded = 0; 1625 } 1626 err = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp); 1627 if (err) 1628 dev_warn(mlxsw_sp->bus_info->dev, "Failed to set traps for fib entries.\n"); 1629 if (old_adj_index_valid) 1630 mlxsw_sp_kvdl_free(mlxsw_sp, nh_grp->adj_index); 1631 } 1632 1633 static void __mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp_nexthop *nh, 1634 bool removing) 1635 { 1636 if (!removing && !nh->should_offload) 1637 nh->should_offload = 1; 1638 else if (removing && nh->offloaded) 1639 nh->should_offload = 0; 1640 nh->update = 1; 1641 } 1642 1643 static void 1644 mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp *mlxsw_sp, 1645 struct mlxsw_sp_neigh_entry *neigh_entry, 1646 bool removing) 1647 { 1648 struct mlxsw_sp_nexthop *nh; 1649 1650 list_for_each_entry(nh, &neigh_entry->nexthop_list, 1651 neigh_list_node) { 1652 __mlxsw_sp_nexthop_neigh_update(nh, removing); 1653 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nh_grp); 1654 } 1655 } 1656 1657 static void mlxsw_sp_nexthop_rif_init(struct mlxsw_sp_nexthop *nh, 1658 struct mlxsw_sp_rif *rif) 1659 { 1660 if (nh->rif) 1661 return; 1662 1663 nh->rif = rif; 1664 list_add(&nh->rif_list_node, &rif->nexthop_list); 1665 } 1666 1667 static void mlxsw_sp_nexthop_rif_fini(struct mlxsw_sp_nexthop *nh) 1668 { 1669 if (!nh->rif) 1670 return; 1671 1672 list_del(&nh->rif_list_node); 1673 nh->rif = NULL; 1674 } 1675 1676 static int mlxsw_sp_nexthop_neigh_init(struct mlxsw_sp *mlxsw_sp, 1677 struct mlxsw_sp_nexthop *nh) 1678 { 1679 struct mlxsw_sp_neigh_entry *neigh_entry; 1680 struct fib_nh *fib_nh = nh->key.fib_nh; 1681 struct neighbour *n; 1682 u8 nud_state, dead; 1683 int err; 1684 1685 if (!nh->nh_grp->gateway || nh->neigh_entry) 1686 return 0; 1687 1688 /* Take a reference of neigh here ensuring that neigh would 1689 * not be detructed before the nexthop entry is finished. 1690 * The reference is taken either in neigh_lookup() or 1691 * in neigh_create() in case n is not found. 1692 */ 1693 n = neigh_lookup(&arp_tbl, &fib_nh->nh_gw, fib_nh->nh_dev); 1694 if (!n) { 1695 n = neigh_create(&arp_tbl, &fib_nh->nh_gw, fib_nh->nh_dev); 1696 if (IS_ERR(n)) 1697 return PTR_ERR(n); 1698 neigh_event_send(n, NULL); 1699 } 1700 neigh_entry = mlxsw_sp_neigh_entry_lookup(mlxsw_sp, n); 1701 if (!neigh_entry) { 1702 neigh_entry = mlxsw_sp_neigh_entry_create(mlxsw_sp, n); 1703 if (IS_ERR(neigh_entry)) { 1704 err = -EINVAL; 1705 goto err_neigh_entry_create; 1706 } 1707 } 1708 1709 /* If that is the first nexthop connected to that neigh, add to 1710 * nexthop_neighs_list 1711 */ 1712 if (list_empty(&neigh_entry->nexthop_list)) 1713 list_add_tail(&neigh_entry->nexthop_neighs_list_node, 1714 &mlxsw_sp->router->nexthop_neighs_list); 1715 1716 nh->neigh_entry = neigh_entry; 1717 list_add_tail(&nh->neigh_list_node, &neigh_entry->nexthop_list); 1718 read_lock_bh(&n->lock); 1719 nud_state = n->nud_state; 1720 dead = n->dead; 1721 read_unlock_bh(&n->lock); 1722 __mlxsw_sp_nexthop_neigh_update(nh, !(nud_state & NUD_VALID && !dead)); 1723 1724 return 0; 1725 1726 err_neigh_entry_create: 1727 neigh_release(n); 1728 return err; 1729 } 1730 1731 static void mlxsw_sp_nexthop_neigh_fini(struct mlxsw_sp *mlxsw_sp, 1732 struct mlxsw_sp_nexthop *nh) 1733 { 1734 struct mlxsw_sp_neigh_entry *neigh_entry = nh->neigh_entry; 1735 struct neighbour *n; 1736 1737 if (!neigh_entry) 1738 return; 1739 n = neigh_entry->key.n; 1740 1741 __mlxsw_sp_nexthop_neigh_update(nh, true); 1742 list_del(&nh->neigh_list_node); 1743 nh->neigh_entry = NULL; 1744 1745 /* If that is the last nexthop connected to that neigh, remove from 1746 * nexthop_neighs_list 1747 */ 1748 if (list_empty(&neigh_entry->nexthop_list)) 1749 list_del(&neigh_entry->nexthop_neighs_list_node); 1750 1751 if (!neigh_entry->connected && list_empty(&neigh_entry->nexthop_list)) 1752 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry); 1753 1754 neigh_release(n); 1755 } 1756 1757 static int mlxsw_sp_nexthop_init(struct mlxsw_sp *mlxsw_sp, 1758 struct mlxsw_sp_nexthop_group *nh_grp, 1759 struct mlxsw_sp_nexthop *nh, 1760 struct fib_nh *fib_nh) 1761 { 1762 struct net_device *dev = fib_nh->nh_dev; 1763 struct in_device *in_dev; 1764 struct mlxsw_sp_rif *rif; 1765 int err; 1766 1767 nh->nh_grp = nh_grp; 1768 nh->key.fib_nh = fib_nh; 1769 err = mlxsw_sp_nexthop_insert(mlxsw_sp, nh); 1770 if (err) 1771 return err; 1772 1773 if (!dev) 1774 return 0; 1775 1776 in_dev = __in_dev_get_rtnl(dev); 1777 if (in_dev && IN_DEV_IGNORE_ROUTES_WITH_LINKDOWN(in_dev) && 1778 fib_nh->nh_flags & RTNH_F_LINKDOWN) 1779 return 0; 1780 1781 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev); 1782 if (!rif) 1783 return 0; 1784 mlxsw_sp_nexthop_rif_init(nh, rif); 1785 1786 err = mlxsw_sp_nexthop_neigh_init(mlxsw_sp, nh); 1787 if (err) 1788 goto err_nexthop_neigh_init; 1789 1790 return 0; 1791 1792 err_nexthop_neigh_init: 1793 mlxsw_sp_nexthop_remove(mlxsw_sp, nh); 1794 return err; 1795 } 1796 1797 static void mlxsw_sp_nexthop_fini(struct mlxsw_sp *mlxsw_sp, 1798 struct mlxsw_sp_nexthop *nh) 1799 { 1800 mlxsw_sp_nexthop_neigh_fini(mlxsw_sp, nh); 1801 mlxsw_sp_nexthop_rif_fini(nh); 1802 mlxsw_sp_nexthop_remove(mlxsw_sp, nh); 1803 } 1804 1805 static void mlxsw_sp_nexthop_event(struct mlxsw_sp *mlxsw_sp, 1806 unsigned long event, struct fib_nh *fib_nh) 1807 { 1808 struct mlxsw_sp_nexthop_key key; 1809 struct mlxsw_sp_nexthop *nh; 1810 struct mlxsw_sp_rif *rif; 1811 1812 if (mlxsw_sp->router->aborted) 1813 return; 1814 1815 key.fib_nh = fib_nh; 1816 nh = mlxsw_sp_nexthop_lookup(mlxsw_sp, key); 1817 if (WARN_ON_ONCE(!nh)) 1818 return; 1819 1820 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, fib_nh->nh_dev); 1821 if (!rif) 1822 return; 1823 1824 switch (event) { 1825 case FIB_EVENT_NH_ADD: 1826 mlxsw_sp_nexthop_rif_init(nh, rif); 1827 mlxsw_sp_nexthop_neigh_init(mlxsw_sp, nh); 1828 break; 1829 case FIB_EVENT_NH_DEL: 1830 mlxsw_sp_nexthop_neigh_fini(mlxsw_sp, nh); 1831 mlxsw_sp_nexthop_rif_fini(nh); 1832 break; 1833 } 1834 1835 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nh_grp); 1836 } 1837 1838 static void mlxsw_sp_nexthop_rif_gone_sync(struct mlxsw_sp *mlxsw_sp, 1839 struct mlxsw_sp_rif *rif) 1840 { 1841 struct mlxsw_sp_nexthop *nh, *tmp; 1842 1843 list_for_each_entry_safe(nh, tmp, &rif->nexthop_list, rif_list_node) { 1844 mlxsw_sp_nexthop_neigh_fini(mlxsw_sp, nh); 1845 mlxsw_sp_nexthop_rif_fini(nh); 1846 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nh_grp); 1847 } 1848 } 1849 1850 static struct mlxsw_sp_nexthop_group * 1851 mlxsw_sp_nexthop_group_create(struct mlxsw_sp *mlxsw_sp, struct fib_info *fi) 1852 { 1853 struct mlxsw_sp_nexthop_group *nh_grp; 1854 struct mlxsw_sp_nexthop *nh; 1855 struct fib_nh *fib_nh; 1856 size_t alloc_size; 1857 int i; 1858 int err; 1859 1860 alloc_size = sizeof(*nh_grp) + 1861 fi->fib_nhs * sizeof(struct mlxsw_sp_nexthop); 1862 nh_grp = kzalloc(alloc_size, GFP_KERNEL); 1863 if (!nh_grp) 1864 return ERR_PTR(-ENOMEM); 1865 INIT_LIST_HEAD(&nh_grp->fib_list); 1866 nh_grp->gateway = fi->fib_nh->nh_scope == RT_SCOPE_LINK; 1867 nh_grp->count = fi->fib_nhs; 1868 nh_grp->key.fi = fi; 1869 for (i = 0; i < nh_grp->count; i++) { 1870 nh = &nh_grp->nexthops[i]; 1871 fib_nh = &fi->fib_nh[i]; 1872 err = mlxsw_sp_nexthop_init(mlxsw_sp, nh_grp, nh, fib_nh); 1873 if (err) 1874 goto err_nexthop_init; 1875 } 1876 err = mlxsw_sp_nexthop_group_insert(mlxsw_sp, nh_grp); 1877 if (err) 1878 goto err_nexthop_group_insert; 1879 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp); 1880 return nh_grp; 1881 1882 err_nexthop_group_insert: 1883 err_nexthop_init: 1884 for (i--; i >= 0; i--) { 1885 nh = &nh_grp->nexthops[i]; 1886 mlxsw_sp_nexthop_fini(mlxsw_sp, nh); 1887 } 1888 kfree(nh_grp); 1889 return ERR_PTR(err); 1890 } 1891 1892 static void 1893 mlxsw_sp_nexthop_group_destroy(struct mlxsw_sp *mlxsw_sp, 1894 struct mlxsw_sp_nexthop_group *nh_grp) 1895 { 1896 struct mlxsw_sp_nexthop *nh; 1897 int i; 1898 1899 mlxsw_sp_nexthop_group_remove(mlxsw_sp, nh_grp); 1900 for (i = 0; i < nh_grp->count; i++) { 1901 nh = &nh_grp->nexthops[i]; 1902 mlxsw_sp_nexthop_fini(mlxsw_sp, nh); 1903 } 1904 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp); 1905 WARN_ON_ONCE(nh_grp->adj_index_valid); 1906 kfree(nh_grp); 1907 } 1908 1909 static int mlxsw_sp_nexthop_group_get(struct mlxsw_sp *mlxsw_sp, 1910 struct mlxsw_sp_fib_entry *fib_entry, 1911 struct fib_info *fi) 1912 { 1913 struct mlxsw_sp_nexthop_group_key key; 1914 struct mlxsw_sp_nexthop_group *nh_grp; 1915 1916 key.fi = fi; 1917 nh_grp = mlxsw_sp_nexthop_group_lookup(mlxsw_sp, key); 1918 if (!nh_grp) { 1919 nh_grp = mlxsw_sp_nexthop_group_create(mlxsw_sp, fi); 1920 if (IS_ERR(nh_grp)) 1921 return PTR_ERR(nh_grp); 1922 } 1923 list_add_tail(&fib_entry->nexthop_group_node, &nh_grp->fib_list); 1924 fib_entry->nh_group = nh_grp; 1925 return 0; 1926 } 1927 1928 static void mlxsw_sp_nexthop_group_put(struct mlxsw_sp *mlxsw_sp, 1929 struct mlxsw_sp_fib_entry *fib_entry) 1930 { 1931 struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group; 1932 1933 list_del(&fib_entry->nexthop_group_node); 1934 if (!list_empty(&nh_grp->fib_list)) 1935 return; 1936 mlxsw_sp_nexthop_group_destroy(mlxsw_sp, nh_grp); 1937 } 1938 1939 static bool 1940 mlxsw_sp_fib_entry_should_offload(const struct mlxsw_sp_fib_entry *fib_entry) 1941 { 1942 struct mlxsw_sp_nexthop_group *nh_group = fib_entry->nh_group; 1943 1944 if (fib_entry->params.tos) 1945 return false; 1946 1947 switch (fib_entry->type) { 1948 case MLXSW_SP_FIB_ENTRY_TYPE_REMOTE: 1949 return !!nh_group->adj_index_valid; 1950 case MLXSW_SP_FIB_ENTRY_TYPE_LOCAL: 1951 return !!nh_group->nh_rif; 1952 default: 1953 return false; 1954 } 1955 } 1956 1957 static void mlxsw_sp_fib_entry_offload_set(struct mlxsw_sp_fib_entry *fib_entry) 1958 { 1959 fib_entry->offloaded = true; 1960 1961 switch (fib_entry->fib_node->fib->proto) { 1962 case MLXSW_SP_L3_PROTO_IPV4: 1963 fib_info_offload_inc(fib_entry->nh_group->key.fi); 1964 break; 1965 case MLXSW_SP_L3_PROTO_IPV6: 1966 WARN_ON_ONCE(1); 1967 } 1968 } 1969 1970 static void 1971 mlxsw_sp_fib_entry_offload_unset(struct mlxsw_sp_fib_entry *fib_entry) 1972 { 1973 switch (fib_entry->fib_node->fib->proto) { 1974 case MLXSW_SP_L3_PROTO_IPV4: 1975 fib_info_offload_dec(fib_entry->nh_group->key.fi); 1976 break; 1977 case MLXSW_SP_L3_PROTO_IPV6: 1978 WARN_ON_ONCE(1); 1979 } 1980 1981 fib_entry->offloaded = false; 1982 } 1983 1984 static void 1985 mlxsw_sp_fib_entry_offload_refresh(struct mlxsw_sp_fib_entry *fib_entry, 1986 enum mlxsw_reg_ralue_op op, int err) 1987 { 1988 switch (op) { 1989 case MLXSW_REG_RALUE_OP_WRITE_DELETE: 1990 if (!fib_entry->offloaded) 1991 return; 1992 return mlxsw_sp_fib_entry_offload_unset(fib_entry); 1993 case MLXSW_REG_RALUE_OP_WRITE_WRITE: 1994 if (err) 1995 return; 1996 if (mlxsw_sp_fib_entry_should_offload(fib_entry) && 1997 !fib_entry->offloaded) 1998 mlxsw_sp_fib_entry_offload_set(fib_entry); 1999 else if (!mlxsw_sp_fib_entry_should_offload(fib_entry) && 2000 fib_entry->offloaded) 2001 mlxsw_sp_fib_entry_offload_unset(fib_entry); 2002 return; 2003 default: 2004 return; 2005 } 2006 } 2007 2008 static int mlxsw_sp_fib_entry_op4_remote(struct mlxsw_sp *mlxsw_sp, 2009 struct mlxsw_sp_fib_entry *fib_entry, 2010 enum mlxsw_reg_ralue_op op) 2011 { 2012 char ralue_pl[MLXSW_REG_RALUE_LEN]; 2013 struct mlxsw_sp_fib *fib = fib_entry->fib_node->fib; 2014 u32 *p_dip = (u32 *) fib_entry->fib_node->key.addr; 2015 enum mlxsw_reg_ralue_trap_action trap_action; 2016 u16 trap_id = 0; 2017 u32 adjacency_index = 0; 2018 u16 ecmp_size = 0; 2019 2020 /* In case the nexthop group adjacency index is valid, use it 2021 * with provided ECMP size. Otherwise, setup trap and pass 2022 * traffic to kernel. 2023 */ 2024 if (mlxsw_sp_fib_entry_should_offload(fib_entry)) { 2025 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_NOP; 2026 adjacency_index = fib_entry->nh_group->adj_index; 2027 ecmp_size = fib_entry->nh_group->ecmp_size; 2028 } else { 2029 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_TRAP; 2030 trap_id = MLXSW_TRAP_ID_RTR_INGRESS0; 2031 } 2032 2033 mlxsw_reg_ralue_pack4(ralue_pl, 2034 (enum mlxsw_reg_ralxx_protocol) fib->proto, op, 2035 fib->vr->id, fib_entry->fib_node->key.prefix_len, 2036 *p_dip); 2037 mlxsw_reg_ralue_act_remote_pack(ralue_pl, trap_action, trap_id, 2038 adjacency_index, ecmp_size); 2039 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl); 2040 } 2041 2042 static int mlxsw_sp_fib_entry_op4_local(struct mlxsw_sp *mlxsw_sp, 2043 struct mlxsw_sp_fib_entry *fib_entry, 2044 enum mlxsw_reg_ralue_op op) 2045 { 2046 struct mlxsw_sp_rif *rif = fib_entry->nh_group->nh_rif; 2047 struct mlxsw_sp_fib *fib = fib_entry->fib_node->fib; 2048 enum mlxsw_reg_ralue_trap_action trap_action; 2049 char ralue_pl[MLXSW_REG_RALUE_LEN]; 2050 u32 *p_dip = (u32 *) fib_entry->fib_node->key.addr; 2051 u16 trap_id = 0; 2052 u16 rif_index = 0; 2053 2054 if (mlxsw_sp_fib_entry_should_offload(fib_entry)) { 2055 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_NOP; 2056 rif_index = rif->rif_index; 2057 } else { 2058 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_TRAP; 2059 trap_id = MLXSW_TRAP_ID_RTR_INGRESS0; 2060 } 2061 2062 mlxsw_reg_ralue_pack4(ralue_pl, 2063 (enum mlxsw_reg_ralxx_protocol) fib->proto, op, 2064 fib->vr->id, fib_entry->fib_node->key.prefix_len, 2065 *p_dip); 2066 mlxsw_reg_ralue_act_local_pack(ralue_pl, trap_action, trap_id, 2067 rif_index); 2068 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl); 2069 } 2070 2071 static int mlxsw_sp_fib_entry_op4_trap(struct mlxsw_sp *mlxsw_sp, 2072 struct mlxsw_sp_fib_entry *fib_entry, 2073 enum mlxsw_reg_ralue_op op) 2074 { 2075 struct mlxsw_sp_fib *fib = fib_entry->fib_node->fib; 2076 char ralue_pl[MLXSW_REG_RALUE_LEN]; 2077 u32 *p_dip = (u32 *) fib_entry->fib_node->key.addr; 2078 2079 mlxsw_reg_ralue_pack4(ralue_pl, 2080 (enum mlxsw_reg_ralxx_protocol) fib->proto, op, 2081 fib->vr->id, fib_entry->fib_node->key.prefix_len, 2082 *p_dip); 2083 mlxsw_reg_ralue_act_ip2me_pack(ralue_pl); 2084 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl); 2085 } 2086 2087 static int mlxsw_sp_fib_entry_op4(struct mlxsw_sp *mlxsw_sp, 2088 struct mlxsw_sp_fib_entry *fib_entry, 2089 enum mlxsw_reg_ralue_op op) 2090 { 2091 switch (fib_entry->type) { 2092 case MLXSW_SP_FIB_ENTRY_TYPE_REMOTE: 2093 return mlxsw_sp_fib_entry_op4_remote(mlxsw_sp, fib_entry, op); 2094 case MLXSW_SP_FIB_ENTRY_TYPE_LOCAL: 2095 return mlxsw_sp_fib_entry_op4_local(mlxsw_sp, fib_entry, op); 2096 case MLXSW_SP_FIB_ENTRY_TYPE_TRAP: 2097 return mlxsw_sp_fib_entry_op4_trap(mlxsw_sp, fib_entry, op); 2098 } 2099 return -EINVAL; 2100 } 2101 2102 static int mlxsw_sp_fib_entry_op(struct mlxsw_sp *mlxsw_sp, 2103 struct mlxsw_sp_fib_entry *fib_entry, 2104 enum mlxsw_reg_ralue_op op) 2105 { 2106 int err = -EINVAL; 2107 2108 switch (fib_entry->fib_node->fib->proto) { 2109 case MLXSW_SP_L3_PROTO_IPV4: 2110 err = mlxsw_sp_fib_entry_op4(mlxsw_sp, fib_entry, op); 2111 break; 2112 case MLXSW_SP_L3_PROTO_IPV6: 2113 return err; 2114 } 2115 mlxsw_sp_fib_entry_offload_refresh(fib_entry, op, err); 2116 return err; 2117 } 2118 2119 static int mlxsw_sp_fib_entry_update(struct mlxsw_sp *mlxsw_sp, 2120 struct mlxsw_sp_fib_entry *fib_entry) 2121 { 2122 return mlxsw_sp_fib_entry_op(mlxsw_sp, fib_entry, 2123 MLXSW_REG_RALUE_OP_WRITE_WRITE); 2124 } 2125 2126 static int mlxsw_sp_fib_entry_del(struct mlxsw_sp *mlxsw_sp, 2127 struct mlxsw_sp_fib_entry *fib_entry) 2128 { 2129 return mlxsw_sp_fib_entry_op(mlxsw_sp, fib_entry, 2130 MLXSW_REG_RALUE_OP_WRITE_DELETE); 2131 } 2132 2133 static int 2134 mlxsw_sp_fib4_entry_type_set(struct mlxsw_sp *mlxsw_sp, 2135 const struct fib_entry_notifier_info *fen_info, 2136 struct mlxsw_sp_fib_entry *fib_entry) 2137 { 2138 struct fib_info *fi = fen_info->fi; 2139 2140 switch (fen_info->type) { 2141 case RTN_BROADCAST: /* fall through */ 2142 case RTN_LOCAL: 2143 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP; 2144 return 0; 2145 case RTN_UNREACHABLE: /* fall through */ 2146 case RTN_BLACKHOLE: /* fall through */ 2147 case RTN_PROHIBIT: 2148 /* Packets hitting these routes need to be trapped, but 2149 * can do so with a lower priority than packets directed 2150 * at the host, so use action type local instead of trap. 2151 */ 2152 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL; 2153 return 0; 2154 case RTN_UNICAST: 2155 if (fi->fib_nh->nh_scope != RT_SCOPE_LINK) 2156 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL; 2157 else 2158 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_REMOTE; 2159 return 0; 2160 default: 2161 return -EINVAL; 2162 } 2163 } 2164 2165 static struct mlxsw_sp_fib_entry * 2166 mlxsw_sp_fib4_entry_create(struct mlxsw_sp *mlxsw_sp, 2167 struct mlxsw_sp_fib_node *fib_node, 2168 const struct fib_entry_notifier_info *fen_info) 2169 { 2170 struct mlxsw_sp_fib_entry *fib_entry; 2171 int err; 2172 2173 fib_entry = kzalloc(sizeof(*fib_entry), GFP_KERNEL); 2174 if (!fib_entry) { 2175 err = -ENOMEM; 2176 goto err_fib_entry_alloc; 2177 } 2178 2179 err = mlxsw_sp_fib4_entry_type_set(mlxsw_sp, fen_info, fib_entry); 2180 if (err) 2181 goto err_fib4_entry_type_set; 2182 2183 err = mlxsw_sp_nexthop_group_get(mlxsw_sp, fib_entry, fen_info->fi); 2184 if (err) 2185 goto err_nexthop_group_get; 2186 2187 fib_entry->params.prio = fen_info->fi->fib_priority; 2188 fib_entry->params.tb_id = fen_info->tb_id; 2189 fib_entry->params.type = fen_info->type; 2190 fib_entry->params.tos = fen_info->tos; 2191 2192 fib_entry->fib_node = fib_node; 2193 2194 return fib_entry; 2195 2196 err_nexthop_group_get: 2197 err_fib4_entry_type_set: 2198 kfree(fib_entry); 2199 err_fib_entry_alloc: 2200 return ERR_PTR(err); 2201 } 2202 2203 static void mlxsw_sp_fib4_entry_destroy(struct mlxsw_sp *mlxsw_sp, 2204 struct mlxsw_sp_fib_entry *fib_entry) 2205 { 2206 mlxsw_sp_nexthop_group_put(mlxsw_sp, fib_entry); 2207 kfree(fib_entry); 2208 } 2209 2210 static struct mlxsw_sp_fib_node * 2211 mlxsw_sp_fib4_node_get(struct mlxsw_sp *mlxsw_sp, 2212 const struct fib_entry_notifier_info *fen_info); 2213 2214 static struct mlxsw_sp_fib_entry * 2215 mlxsw_sp_fib4_entry_lookup(struct mlxsw_sp *mlxsw_sp, 2216 const struct fib_entry_notifier_info *fen_info) 2217 { 2218 struct mlxsw_sp_fib_entry *fib_entry; 2219 struct mlxsw_sp_fib_node *fib_node; 2220 2221 fib_node = mlxsw_sp_fib4_node_get(mlxsw_sp, fen_info); 2222 if (IS_ERR(fib_node)) 2223 return NULL; 2224 2225 list_for_each_entry(fib_entry, &fib_node->entry_list, list) { 2226 if (fib_entry->params.tb_id == fen_info->tb_id && 2227 fib_entry->params.tos == fen_info->tos && 2228 fib_entry->params.type == fen_info->type && 2229 fib_entry->nh_group->key.fi == fen_info->fi) { 2230 return fib_entry; 2231 } 2232 } 2233 2234 return NULL; 2235 } 2236 2237 static const struct rhashtable_params mlxsw_sp_fib_ht_params = { 2238 .key_offset = offsetof(struct mlxsw_sp_fib_node, key), 2239 .head_offset = offsetof(struct mlxsw_sp_fib_node, ht_node), 2240 .key_len = sizeof(struct mlxsw_sp_fib_key), 2241 .automatic_shrinking = true, 2242 }; 2243 2244 static int mlxsw_sp_fib_node_insert(struct mlxsw_sp_fib *fib, 2245 struct mlxsw_sp_fib_node *fib_node) 2246 { 2247 return rhashtable_insert_fast(&fib->ht, &fib_node->ht_node, 2248 mlxsw_sp_fib_ht_params); 2249 } 2250 2251 static void mlxsw_sp_fib_node_remove(struct mlxsw_sp_fib *fib, 2252 struct mlxsw_sp_fib_node *fib_node) 2253 { 2254 rhashtable_remove_fast(&fib->ht, &fib_node->ht_node, 2255 mlxsw_sp_fib_ht_params); 2256 } 2257 2258 static struct mlxsw_sp_fib_node * 2259 mlxsw_sp_fib_node_lookup(struct mlxsw_sp_fib *fib, const void *addr, 2260 size_t addr_len, unsigned char prefix_len) 2261 { 2262 struct mlxsw_sp_fib_key key; 2263 2264 memset(&key, 0, sizeof(key)); 2265 memcpy(key.addr, addr, addr_len); 2266 key.prefix_len = prefix_len; 2267 return rhashtable_lookup_fast(&fib->ht, &key, mlxsw_sp_fib_ht_params); 2268 } 2269 2270 static struct mlxsw_sp_fib_node * 2271 mlxsw_sp_fib_node_create(struct mlxsw_sp_fib *fib, const void *addr, 2272 size_t addr_len, unsigned char prefix_len) 2273 { 2274 struct mlxsw_sp_fib_node *fib_node; 2275 2276 fib_node = kzalloc(sizeof(*fib_node), GFP_KERNEL); 2277 if (!fib_node) 2278 return NULL; 2279 2280 INIT_LIST_HEAD(&fib_node->entry_list); 2281 list_add(&fib_node->list, &fib->node_list); 2282 memcpy(fib_node->key.addr, addr, addr_len); 2283 fib_node->key.prefix_len = prefix_len; 2284 2285 return fib_node; 2286 } 2287 2288 static void mlxsw_sp_fib_node_destroy(struct mlxsw_sp_fib_node *fib_node) 2289 { 2290 list_del(&fib_node->list); 2291 WARN_ON(!list_empty(&fib_node->entry_list)); 2292 kfree(fib_node); 2293 } 2294 2295 static bool 2296 mlxsw_sp_fib_node_entry_is_first(const struct mlxsw_sp_fib_node *fib_node, 2297 const struct mlxsw_sp_fib_entry *fib_entry) 2298 { 2299 return list_first_entry(&fib_node->entry_list, 2300 struct mlxsw_sp_fib_entry, list) == fib_entry; 2301 } 2302 2303 static void mlxsw_sp_fib_node_prefix_inc(struct mlxsw_sp_fib_node *fib_node) 2304 { 2305 unsigned char prefix_len = fib_node->key.prefix_len; 2306 struct mlxsw_sp_fib *fib = fib_node->fib; 2307 2308 if (fib->prefix_ref_count[prefix_len]++ == 0) 2309 mlxsw_sp_prefix_usage_set(&fib->prefix_usage, prefix_len); 2310 } 2311 2312 static void mlxsw_sp_fib_node_prefix_dec(struct mlxsw_sp_fib_node *fib_node) 2313 { 2314 unsigned char prefix_len = fib_node->key.prefix_len; 2315 struct mlxsw_sp_fib *fib = fib_node->fib; 2316 2317 if (--fib->prefix_ref_count[prefix_len] == 0) 2318 mlxsw_sp_prefix_usage_clear(&fib->prefix_usage, prefix_len); 2319 } 2320 2321 static int mlxsw_sp_fib_node_init(struct mlxsw_sp *mlxsw_sp, 2322 struct mlxsw_sp_fib_node *fib_node, 2323 struct mlxsw_sp_fib *fib) 2324 { 2325 struct mlxsw_sp_prefix_usage req_prefix_usage; 2326 struct mlxsw_sp_lpm_tree *lpm_tree; 2327 int err; 2328 2329 err = mlxsw_sp_fib_node_insert(fib, fib_node); 2330 if (err) 2331 return err; 2332 fib_node->fib = fib; 2333 2334 mlxsw_sp_prefix_usage_cpy(&req_prefix_usage, &fib->prefix_usage); 2335 mlxsw_sp_prefix_usage_set(&req_prefix_usage, fib_node->key.prefix_len); 2336 2337 if (!mlxsw_sp_prefix_usage_none(&fib->prefix_usage)) { 2338 err = mlxsw_sp_vr_lpm_tree_check(mlxsw_sp, fib, 2339 &req_prefix_usage); 2340 if (err) 2341 goto err_tree_check; 2342 } else { 2343 lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage, 2344 fib->proto); 2345 if (IS_ERR(lpm_tree)) 2346 return PTR_ERR(lpm_tree); 2347 fib->lpm_tree = lpm_tree; 2348 err = mlxsw_sp_vr_lpm_tree_bind(mlxsw_sp, fib); 2349 if (err) 2350 goto err_tree_bind; 2351 } 2352 2353 mlxsw_sp_fib_node_prefix_inc(fib_node); 2354 2355 return 0; 2356 2357 err_tree_bind: 2358 fib->lpm_tree = NULL; 2359 mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree); 2360 err_tree_check: 2361 fib_node->fib = NULL; 2362 mlxsw_sp_fib_node_remove(fib, fib_node); 2363 return err; 2364 } 2365 2366 static void mlxsw_sp_fib_node_fini(struct mlxsw_sp *mlxsw_sp, 2367 struct mlxsw_sp_fib_node *fib_node) 2368 { 2369 struct mlxsw_sp_lpm_tree *lpm_tree = fib_node->fib->lpm_tree; 2370 struct mlxsw_sp_fib *fib = fib_node->fib; 2371 2372 mlxsw_sp_fib_node_prefix_dec(fib_node); 2373 2374 if (mlxsw_sp_prefix_usage_none(&fib->prefix_usage)) { 2375 mlxsw_sp_vr_lpm_tree_unbind(mlxsw_sp, fib); 2376 fib->lpm_tree = NULL; 2377 mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree); 2378 } else { 2379 mlxsw_sp_vr_lpm_tree_check(mlxsw_sp, fib, &fib->prefix_usage); 2380 } 2381 2382 fib_node->fib = NULL; 2383 mlxsw_sp_fib_node_remove(fib, fib_node); 2384 } 2385 2386 static struct mlxsw_sp_fib_node * 2387 mlxsw_sp_fib4_node_get(struct mlxsw_sp *mlxsw_sp, 2388 const struct fib_entry_notifier_info *fen_info) 2389 { 2390 struct mlxsw_sp_fib_node *fib_node; 2391 struct mlxsw_sp_fib *fib; 2392 struct mlxsw_sp_vr *vr; 2393 int err; 2394 2395 vr = mlxsw_sp_vr_get(mlxsw_sp, fen_info->tb_id); 2396 if (IS_ERR(vr)) 2397 return ERR_CAST(vr); 2398 fib = mlxsw_sp_vr_fib(vr, MLXSW_SP_L3_PROTO_IPV4); 2399 2400 fib_node = mlxsw_sp_fib_node_lookup(fib, &fen_info->dst, 2401 sizeof(fen_info->dst), 2402 fen_info->dst_len); 2403 if (fib_node) 2404 return fib_node; 2405 2406 fib_node = mlxsw_sp_fib_node_create(fib, &fen_info->dst, 2407 sizeof(fen_info->dst), 2408 fen_info->dst_len); 2409 if (!fib_node) { 2410 err = -ENOMEM; 2411 goto err_fib_node_create; 2412 } 2413 2414 err = mlxsw_sp_fib_node_init(mlxsw_sp, fib_node, fib); 2415 if (err) 2416 goto err_fib_node_init; 2417 2418 return fib_node; 2419 2420 err_fib_node_init: 2421 mlxsw_sp_fib_node_destroy(fib_node); 2422 err_fib_node_create: 2423 mlxsw_sp_vr_put(vr); 2424 return ERR_PTR(err); 2425 } 2426 2427 static void mlxsw_sp_fib4_node_put(struct mlxsw_sp *mlxsw_sp, 2428 struct mlxsw_sp_fib_node *fib_node) 2429 { 2430 struct mlxsw_sp_vr *vr = fib_node->fib->vr; 2431 2432 if (!list_empty(&fib_node->entry_list)) 2433 return; 2434 mlxsw_sp_fib_node_fini(mlxsw_sp, fib_node); 2435 mlxsw_sp_fib_node_destroy(fib_node); 2436 mlxsw_sp_vr_put(vr); 2437 } 2438 2439 static struct mlxsw_sp_fib_entry * 2440 mlxsw_sp_fib4_node_entry_find(const struct mlxsw_sp_fib_node *fib_node, 2441 const struct mlxsw_sp_fib_entry_params *params) 2442 { 2443 struct mlxsw_sp_fib_entry *fib_entry; 2444 2445 list_for_each_entry(fib_entry, &fib_node->entry_list, list) { 2446 if (fib_entry->params.tb_id > params->tb_id) 2447 continue; 2448 if (fib_entry->params.tb_id != params->tb_id) 2449 break; 2450 if (fib_entry->params.tos > params->tos) 2451 continue; 2452 if (fib_entry->params.prio >= params->prio || 2453 fib_entry->params.tos < params->tos) 2454 return fib_entry; 2455 } 2456 2457 return NULL; 2458 } 2459 2460 static int mlxsw_sp_fib4_node_list_append(struct mlxsw_sp_fib_entry *fib_entry, 2461 struct mlxsw_sp_fib_entry *new_entry) 2462 { 2463 struct mlxsw_sp_fib_node *fib_node; 2464 2465 if (WARN_ON(!fib_entry)) 2466 return -EINVAL; 2467 2468 fib_node = fib_entry->fib_node; 2469 list_for_each_entry_from(fib_entry, &fib_node->entry_list, list) { 2470 if (fib_entry->params.tb_id != new_entry->params.tb_id || 2471 fib_entry->params.tos != new_entry->params.tos || 2472 fib_entry->params.prio != new_entry->params.prio) 2473 break; 2474 } 2475 2476 list_add_tail(&new_entry->list, &fib_entry->list); 2477 return 0; 2478 } 2479 2480 static int 2481 mlxsw_sp_fib4_node_list_insert(struct mlxsw_sp_fib_node *fib_node, 2482 struct mlxsw_sp_fib_entry *new_entry, 2483 bool replace, bool append) 2484 { 2485 struct mlxsw_sp_fib_entry *fib_entry; 2486 2487 fib_entry = mlxsw_sp_fib4_node_entry_find(fib_node, &new_entry->params); 2488 2489 if (append) 2490 return mlxsw_sp_fib4_node_list_append(fib_entry, new_entry); 2491 if (replace && WARN_ON(!fib_entry)) 2492 return -EINVAL; 2493 2494 /* Insert new entry before replaced one, so that we can later 2495 * remove the second. 2496 */ 2497 if (fib_entry) { 2498 list_add_tail(&new_entry->list, &fib_entry->list); 2499 } else { 2500 struct mlxsw_sp_fib_entry *last; 2501 2502 list_for_each_entry(last, &fib_node->entry_list, list) { 2503 if (new_entry->params.tb_id > last->params.tb_id) 2504 break; 2505 fib_entry = last; 2506 } 2507 2508 if (fib_entry) 2509 list_add(&new_entry->list, &fib_entry->list); 2510 else 2511 list_add(&new_entry->list, &fib_node->entry_list); 2512 } 2513 2514 return 0; 2515 } 2516 2517 static void 2518 mlxsw_sp_fib4_node_list_remove(struct mlxsw_sp_fib_entry *fib_entry) 2519 { 2520 list_del(&fib_entry->list); 2521 } 2522 2523 static int 2524 mlxsw_sp_fib4_node_entry_add(struct mlxsw_sp *mlxsw_sp, 2525 const struct mlxsw_sp_fib_node *fib_node, 2526 struct mlxsw_sp_fib_entry *fib_entry) 2527 { 2528 if (!mlxsw_sp_fib_node_entry_is_first(fib_node, fib_entry)) 2529 return 0; 2530 2531 /* To prevent packet loss, overwrite the previously offloaded 2532 * entry. 2533 */ 2534 if (!list_is_singular(&fib_node->entry_list)) { 2535 enum mlxsw_reg_ralue_op op = MLXSW_REG_RALUE_OP_WRITE_DELETE; 2536 struct mlxsw_sp_fib_entry *n = list_next_entry(fib_entry, list); 2537 2538 mlxsw_sp_fib_entry_offload_refresh(n, op, 0); 2539 } 2540 2541 return mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry); 2542 } 2543 2544 static void 2545 mlxsw_sp_fib4_node_entry_del(struct mlxsw_sp *mlxsw_sp, 2546 const struct mlxsw_sp_fib_node *fib_node, 2547 struct mlxsw_sp_fib_entry *fib_entry) 2548 { 2549 if (!mlxsw_sp_fib_node_entry_is_first(fib_node, fib_entry)) 2550 return; 2551 2552 /* Promote the next entry by overwriting the deleted entry */ 2553 if (!list_is_singular(&fib_node->entry_list)) { 2554 struct mlxsw_sp_fib_entry *n = list_next_entry(fib_entry, list); 2555 enum mlxsw_reg_ralue_op op = MLXSW_REG_RALUE_OP_WRITE_DELETE; 2556 2557 mlxsw_sp_fib_entry_update(mlxsw_sp, n); 2558 mlxsw_sp_fib_entry_offload_refresh(fib_entry, op, 0); 2559 return; 2560 } 2561 2562 mlxsw_sp_fib_entry_del(mlxsw_sp, fib_entry); 2563 } 2564 2565 static int mlxsw_sp_fib4_node_entry_link(struct mlxsw_sp *mlxsw_sp, 2566 struct mlxsw_sp_fib_entry *fib_entry, 2567 bool replace, bool append) 2568 { 2569 struct mlxsw_sp_fib_node *fib_node = fib_entry->fib_node; 2570 int err; 2571 2572 err = mlxsw_sp_fib4_node_list_insert(fib_node, fib_entry, replace, 2573 append); 2574 if (err) 2575 return err; 2576 2577 err = mlxsw_sp_fib4_node_entry_add(mlxsw_sp, fib_node, fib_entry); 2578 if (err) 2579 goto err_fib4_node_entry_add; 2580 2581 return 0; 2582 2583 err_fib4_node_entry_add: 2584 mlxsw_sp_fib4_node_list_remove(fib_entry); 2585 return err; 2586 } 2587 2588 static void 2589 mlxsw_sp_fib4_node_entry_unlink(struct mlxsw_sp *mlxsw_sp, 2590 struct mlxsw_sp_fib_entry *fib_entry) 2591 { 2592 struct mlxsw_sp_fib_node *fib_node = fib_entry->fib_node; 2593 2594 mlxsw_sp_fib4_node_entry_del(mlxsw_sp, fib_node, fib_entry); 2595 mlxsw_sp_fib4_node_list_remove(fib_entry); 2596 } 2597 2598 static void mlxsw_sp_fib4_entry_replace(struct mlxsw_sp *mlxsw_sp, 2599 struct mlxsw_sp_fib_entry *fib_entry, 2600 bool replace) 2601 { 2602 struct mlxsw_sp_fib_node *fib_node = fib_entry->fib_node; 2603 struct mlxsw_sp_fib_entry *replaced; 2604 2605 if (!replace) 2606 return; 2607 2608 /* We inserted the new entry before replaced one */ 2609 replaced = list_next_entry(fib_entry, list); 2610 2611 mlxsw_sp_fib4_node_entry_unlink(mlxsw_sp, replaced); 2612 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, replaced); 2613 mlxsw_sp_fib4_node_put(mlxsw_sp, fib_node); 2614 } 2615 2616 static int 2617 mlxsw_sp_router_fib4_add(struct mlxsw_sp *mlxsw_sp, 2618 const struct fib_entry_notifier_info *fen_info, 2619 bool replace, bool append) 2620 { 2621 struct mlxsw_sp_fib_entry *fib_entry; 2622 struct mlxsw_sp_fib_node *fib_node; 2623 int err; 2624 2625 if (mlxsw_sp->router->aborted) 2626 return 0; 2627 2628 fib_node = mlxsw_sp_fib4_node_get(mlxsw_sp, fen_info); 2629 if (IS_ERR(fib_node)) { 2630 dev_warn(mlxsw_sp->bus_info->dev, "Failed to get FIB node\n"); 2631 return PTR_ERR(fib_node); 2632 } 2633 2634 fib_entry = mlxsw_sp_fib4_entry_create(mlxsw_sp, fib_node, fen_info); 2635 if (IS_ERR(fib_entry)) { 2636 dev_warn(mlxsw_sp->bus_info->dev, "Failed to create FIB entry\n"); 2637 err = PTR_ERR(fib_entry); 2638 goto err_fib4_entry_create; 2639 } 2640 2641 err = mlxsw_sp_fib4_node_entry_link(mlxsw_sp, fib_entry, replace, 2642 append); 2643 if (err) { 2644 dev_warn(mlxsw_sp->bus_info->dev, "Failed to link FIB entry to node\n"); 2645 goto err_fib4_node_entry_link; 2646 } 2647 2648 mlxsw_sp_fib4_entry_replace(mlxsw_sp, fib_entry, replace); 2649 2650 return 0; 2651 2652 err_fib4_node_entry_link: 2653 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib_entry); 2654 err_fib4_entry_create: 2655 mlxsw_sp_fib4_node_put(mlxsw_sp, fib_node); 2656 return err; 2657 } 2658 2659 static void mlxsw_sp_router_fib4_del(struct mlxsw_sp *mlxsw_sp, 2660 struct fib_entry_notifier_info *fen_info) 2661 { 2662 struct mlxsw_sp_fib_entry *fib_entry; 2663 struct mlxsw_sp_fib_node *fib_node; 2664 2665 if (mlxsw_sp->router->aborted) 2666 return; 2667 2668 fib_entry = mlxsw_sp_fib4_entry_lookup(mlxsw_sp, fen_info); 2669 if (WARN_ON(!fib_entry)) 2670 return; 2671 fib_node = fib_entry->fib_node; 2672 2673 mlxsw_sp_fib4_node_entry_unlink(mlxsw_sp, fib_entry); 2674 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib_entry); 2675 mlxsw_sp_fib4_node_put(mlxsw_sp, fib_node); 2676 } 2677 2678 static int mlxsw_sp_router_set_abort_trap(struct mlxsw_sp *mlxsw_sp) 2679 { 2680 char ralta_pl[MLXSW_REG_RALTA_LEN]; 2681 char ralst_pl[MLXSW_REG_RALST_LEN]; 2682 int i, err; 2683 2684 mlxsw_reg_ralta_pack(ralta_pl, true, MLXSW_REG_RALXX_PROTOCOL_IPV4, 2685 MLXSW_SP_LPM_TREE_MIN); 2686 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), ralta_pl); 2687 if (err) 2688 return err; 2689 2690 mlxsw_reg_ralst_pack(ralst_pl, 0xff, MLXSW_SP_LPM_TREE_MIN); 2691 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralst), ralst_pl); 2692 if (err) 2693 return err; 2694 2695 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) { 2696 struct mlxsw_sp_vr *vr = &mlxsw_sp->router->vrs[i]; 2697 char raltb_pl[MLXSW_REG_RALTB_LEN]; 2698 char ralue_pl[MLXSW_REG_RALUE_LEN]; 2699 2700 if (!mlxsw_sp_vr_is_used(vr)) 2701 continue; 2702 2703 mlxsw_reg_raltb_pack(raltb_pl, vr->id, 2704 MLXSW_REG_RALXX_PROTOCOL_IPV4, 2705 MLXSW_SP_LPM_TREE_MIN); 2706 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb), 2707 raltb_pl); 2708 if (err) 2709 return err; 2710 2711 mlxsw_reg_ralue_pack4(ralue_pl, MLXSW_SP_L3_PROTO_IPV4, 2712 MLXSW_REG_RALUE_OP_WRITE_WRITE, vr->id, 0, 2713 0); 2714 mlxsw_reg_ralue_act_ip2me_pack(ralue_pl); 2715 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), 2716 ralue_pl); 2717 if (err) 2718 return err; 2719 } 2720 2721 return 0; 2722 } 2723 2724 static void mlxsw_sp_fib4_node_flush(struct mlxsw_sp *mlxsw_sp, 2725 struct mlxsw_sp_fib_node *fib_node) 2726 { 2727 struct mlxsw_sp_fib_entry *fib_entry, *tmp; 2728 2729 list_for_each_entry_safe(fib_entry, tmp, &fib_node->entry_list, list) { 2730 bool do_break = &tmp->list == &fib_node->entry_list; 2731 2732 mlxsw_sp_fib4_node_entry_unlink(mlxsw_sp, fib_entry); 2733 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib_entry); 2734 mlxsw_sp_fib4_node_put(mlxsw_sp, fib_node); 2735 /* Break when entry list is empty and node was freed. 2736 * Otherwise, we'll access freed memory in the next 2737 * iteration. 2738 */ 2739 if (do_break) 2740 break; 2741 } 2742 } 2743 2744 static void mlxsw_sp_fib_node_flush(struct mlxsw_sp *mlxsw_sp, 2745 struct mlxsw_sp_fib_node *fib_node) 2746 { 2747 switch (fib_node->fib->proto) { 2748 case MLXSW_SP_L3_PROTO_IPV4: 2749 mlxsw_sp_fib4_node_flush(mlxsw_sp, fib_node); 2750 break; 2751 case MLXSW_SP_L3_PROTO_IPV6: 2752 WARN_ON_ONCE(1); 2753 break; 2754 } 2755 } 2756 2757 static void mlxsw_sp_vr_fib_flush(struct mlxsw_sp *mlxsw_sp, 2758 struct mlxsw_sp_vr *vr, 2759 enum mlxsw_sp_l3proto proto) 2760 { 2761 struct mlxsw_sp_fib *fib = mlxsw_sp_vr_fib(vr, proto); 2762 struct mlxsw_sp_fib_node *fib_node, *tmp; 2763 2764 list_for_each_entry_safe(fib_node, tmp, &fib->node_list, list) { 2765 bool do_break = &tmp->list == &fib->node_list; 2766 2767 mlxsw_sp_fib_node_flush(mlxsw_sp, fib_node); 2768 if (do_break) 2769 break; 2770 } 2771 } 2772 2773 static void mlxsw_sp_router_fib_flush(struct mlxsw_sp *mlxsw_sp) 2774 { 2775 int i; 2776 2777 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) { 2778 struct mlxsw_sp_vr *vr = &mlxsw_sp->router->vrs[i]; 2779 2780 if (!mlxsw_sp_vr_is_used(vr)) 2781 continue; 2782 mlxsw_sp_vr_fib_flush(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV4); 2783 } 2784 } 2785 2786 static void mlxsw_sp_router_fib4_abort(struct mlxsw_sp *mlxsw_sp) 2787 { 2788 int err; 2789 2790 if (mlxsw_sp->router->aborted) 2791 return; 2792 dev_warn(mlxsw_sp->bus_info->dev, "FIB abort triggered. Note that FIB entries are no longer being offloaded to this device.\n"); 2793 mlxsw_sp_router_fib_flush(mlxsw_sp); 2794 mlxsw_sp->router->aborted = true; 2795 err = mlxsw_sp_router_set_abort_trap(mlxsw_sp); 2796 if (err) 2797 dev_warn(mlxsw_sp->bus_info->dev, "Failed to set abort trap.\n"); 2798 } 2799 2800 struct mlxsw_sp_fib_event_work { 2801 struct work_struct work; 2802 union { 2803 struct fib_entry_notifier_info fen_info; 2804 struct fib_rule_notifier_info fr_info; 2805 struct fib_nh_notifier_info fnh_info; 2806 }; 2807 struct mlxsw_sp *mlxsw_sp; 2808 unsigned long event; 2809 }; 2810 2811 static void mlxsw_sp_router_fib_event_work(struct work_struct *work) 2812 { 2813 struct mlxsw_sp_fib_event_work *fib_work = 2814 container_of(work, struct mlxsw_sp_fib_event_work, work); 2815 struct mlxsw_sp *mlxsw_sp = fib_work->mlxsw_sp; 2816 struct fib_rule *rule; 2817 bool replace, append; 2818 int err; 2819 2820 /* Protect internal structures from changes */ 2821 rtnl_lock(); 2822 switch (fib_work->event) { 2823 case FIB_EVENT_ENTRY_REPLACE: /* fall through */ 2824 case FIB_EVENT_ENTRY_APPEND: /* fall through */ 2825 case FIB_EVENT_ENTRY_ADD: 2826 replace = fib_work->event == FIB_EVENT_ENTRY_REPLACE; 2827 append = fib_work->event == FIB_EVENT_ENTRY_APPEND; 2828 err = mlxsw_sp_router_fib4_add(mlxsw_sp, &fib_work->fen_info, 2829 replace, append); 2830 if (err) 2831 mlxsw_sp_router_fib4_abort(mlxsw_sp); 2832 fib_info_put(fib_work->fen_info.fi); 2833 break; 2834 case FIB_EVENT_ENTRY_DEL: 2835 mlxsw_sp_router_fib4_del(mlxsw_sp, &fib_work->fen_info); 2836 fib_info_put(fib_work->fen_info.fi); 2837 break; 2838 case FIB_EVENT_RULE_ADD: /* fall through */ 2839 case FIB_EVENT_RULE_DEL: 2840 rule = fib_work->fr_info.rule; 2841 if (!fib4_rule_default(rule) && !rule->l3mdev) 2842 mlxsw_sp_router_fib4_abort(mlxsw_sp); 2843 fib_rule_put(rule); 2844 break; 2845 case FIB_EVENT_NH_ADD: /* fall through */ 2846 case FIB_EVENT_NH_DEL: 2847 mlxsw_sp_nexthop_event(mlxsw_sp, fib_work->event, 2848 fib_work->fnh_info.fib_nh); 2849 fib_info_put(fib_work->fnh_info.fib_nh->nh_parent); 2850 break; 2851 } 2852 rtnl_unlock(); 2853 kfree(fib_work); 2854 } 2855 2856 /* Called with rcu_read_lock() */ 2857 static int mlxsw_sp_router_fib_event(struct notifier_block *nb, 2858 unsigned long event, void *ptr) 2859 { 2860 struct mlxsw_sp_fib_event_work *fib_work; 2861 struct fib_notifier_info *info = ptr; 2862 struct mlxsw_sp_router *router; 2863 2864 if (!net_eq(info->net, &init_net)) 2865 return NOTIFY_DONE; 2866 2867 fib_work = kzalloc(sizeof(*fib_work), GFP_ATOMIC); 2868 if (WARN_ON(!fib_work)) 2869 return NOTIFY_BAD; 2870 2871 INIT_WORK(&fib_work->work, mlxsw_sp_router_fib_event_work); 2872 router = container_of(nb, struct mlxsw_sp_router, fib_nb); 2873 fib_work->mlxsw_sp = router->mlxsw_sp; 2874 fib_work->event = event; 2875 2876 switch (event) { 2877 case FIB_EVENT_ENTRY_REPLACE: /* fall through */ 2878 case FIB_EVENT_ENTRY_APPEND: /* fall through */ 2879 case FIB_EVENT_ENTRY_ADD: /* fall through */ 2880 case FIB_EVENT_ENTRY_DEL: 2881 memcpy(&fib_work->fen_info, ptr, sizeof(fib_work->fen_info)); 2882 /* Take referece on fib_info to prevent it from being 2883 * freed while work is queued. Release it afterwards. 2884 */ 2885 fib_info_hold(fib_work->fen_info.fi); 2886 break; 2887 case FIB_EVENT_RULE_ADD: /* fall through */ 2888 case FIB_EVENT_RULE_DEL: 2889 memcpy(&fib_work->fr_info, ptr, sizeof(fib_work->fr_info)); 2890 fib_rule_get(fib_work->fr_info.rule); 2891 break; 2892 case FIB_EVENT_NH_ADD: /* fall through */ 2893 case FIB_EVENT_NH_DEL: 2894 memcpy(&fib_work->fnh_info, ptr, sizeof(fib_work->fnh_info)); 2895 fib_info_hold(fib_work->fnh_info.fib_nh->nh_parent); 2896 break; 2897 } 2898 2899 mlxsw_core_schedule_work(&fib_work->work); 2900 2901 return NOTIFY_DONE; 2902 } 2903 2904 static struct mlxsw_sp_rif * 2905 mlxsw_sp_rif_find_by_dev(const struct mlxsw_sp *mlxsw_sp, 2906 const struct net_device *dev) 2907 { 2908 int i; 2909 2910 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++) 2911 if (mlxsw_sp->router->rifs[i] && 2912 mlxsw_sp->router->rifs[i]->dev == dev) 2913 return mlxsw_sp->router->rifs[i]; 2914 2915 return NULL; 2916 } 2917 2918 static int mlxsw_sp_router_rif_disable(struct mlxsw_sp *mlxsw_sp, u16 rif) 2919 { 2920 char ritr_pl[MLXSW_REG_RITR_LEN]; 2921 int err; 2922 2923 mlxsw_reg_ritr_rif_pack(ritr_pl, rif); 2924 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 2925 if (WARN_ON_ONCE(err)) 2926 return err; 2927 2928 mlxsw_reg_ritr_enable_set(ritr_pl, false); 2929 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 2930 } 2931 2932 static void mlxsw_sp_router_rif_gone_sync(struct mlxsw_sp *mlxsw_sp, 2933 struct mlxsw_sp_rif *rif) 2934 { 2935 mlxsw_sp_router_rif_disable(mlxsw_sp, rif->rif_index); 2936 mlxsw_sp_nexthop_rif_gone_sync(mlxsw_sp, rif); 2937 mlxsw_sp_neigh_rif_gone_sync(mlxsw_sp, rif); 2938 } 2939 2940 static bool mlxsw_sp_rif_should_config(struct mlxsw_sp_rif *rif, 2941 const struct in_device *in_dev, 2942 unsigned long event) 2943 { 2944 switch (event) { 2945 case NETDEV_UP: 2946 if (!rif) 2947 return true; 2948 return false; 2949 case NETDEV_DOWN: 2950 if (rif && !in_dev->ifa_list && 2951 !netif_is_l3_slave(rif->dev)) 2952 return true; 2953 /* It is possible we already removed the RIF ourselves 2954 * if it was assigned to a netdev that is now a bridge 2955 * or LAG slave. 2956 */ 2957 return false; 2958 } 2959 2960 return false; 2961 } 2962 2963 static enum mlxsw_sp_rif_type 2964 mlxsw_sp_dev_rif_type(const struct mlxsw_sp *mlxsw_sp, 2965 const struct net_device *dev) 2966 { 2967 enum mlxsw_sp_fid_type type; 2968 2969 /* RIF type is derived from the type of the underlying FID */ 2970 if (is_vlan_dev(dev) && netif_is_bridge_master(vlan_dev_real_dev(dev))) 2971 type = MLXSW_SP_FID_TYPE_8021Q; 2972 else if (netif_is_bridge_master(dev) && br_vlan_enabled(dev)) 2973 type = MLXSW_SP_FID_TYPE_8021Q; 2974 else if (netif_is_bridge_master(dev)) 2975 type = MLXSW_SP_FID_TYPE_8021D; 2976 else 2977 type = MLXSW_SP_FID_TYPE_RFID; 2978 2979 return mlxsw_sp_fid_type_rif_type(mlxsw_sp, type); 2980 } 2981 2982 static int mlxsw_sp_rif_index_alloc(struct mlxsw_sp *mlxsw_sp, u16 *p_rif_index) 2983 { 2984 int i; 2985 2986 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++) { 2987 if (!mlxsw_sp->router->rifs[i]) { 2988 *p_rif_index = i; 2989 return 0; 2990 } 2991 } 2992 2993 return -ENOBUFS; 2994 } 2995 2996 static struct mlxsw_sp_rif *mlxsw_sp_rif_alloc(size_t rif_size, u16 rif_index, 2997 u16 vr_id, 2998 struct net_device *l3_dev) 2999 { 3000 struct mlxsw_sp_rif *rif; 3001 3002 rif = kzalloc(rif_size, GFP_KERNEL); 3003 if (!rif) 3004 return NULL; 3005 3006 INIT_LIST_HEAD(&rif->nexthop_list); 3007 INIT_LIST_HEAD(&rif->neigh_list); 3008 ether_addr_copy(rif->addr, l3_dev->dev_addr); 3009 rif->mtu = l3_dev->mtu; 3010 rif->vr_id = vr_id; 3011 rif->dev = l3_dev; 3012 rif->rif_index = rif_index; 3013 3014 return rif; 3015 } 3016 3017 struct mlxsw_sp_rif *mlxsw_sp_rif_by_index(const struct mlxsw_sp *mlxsw_sp, 3018 u16 rif_index) 3019 { 3020 return mlxsw_sp->router->rifs[rif_index]; 3021 } 3022 3023 u16 mlxsw_sp_rif_index(const struct mlxsw_sp_rif *rif) 3024 { 3025 return rif->rif_index; 3026 } 3027 3028 int mlxsw_sp_rif_dev_ifindex(const struct mlxsw_sp_rif *rif) 3029 { 3030 return rif->dev->ifindex; 3031 } 3032 3033 static struct mlxsw_sp_rif * 3034 mlxsw_sp_rif_create(struct mlxsw_sp *mlxsw_sp, 3035 const struct mlxsw_sp_rif_params *params) 3036 { 3037 u32 tb_id = l3mdev_fib_table(params->dev); 3038 const struct mlxsw_sp_rif_ops *ops; 3039 enum mlxsw_sp_rif_type type; 3040 struct mlxsw_sp_rif *rif; 3041 struct mlxsw_sp_fid *fid; 3042 struct mlxsw_sp_vr *vr; 3043 u16 rif_index; 3044 int err; 3045 3046 type = mlxsw_sp_dev_rif_type(mlxsw_sp, params->dev); 3047 ops = mlxsw_sp->router->rif_ops_arr[type]; 3048 3049 vr = mlxsw_sp_vr_get(mlxsw_sp, tb_id ? : RT_TABLE_MAIN); 3050 if (IS_ERR(vr)) 3051 return ERR_CAST(vr); 3052 3053 err = mlxsw_sp_rif_index_alloc(mlxsw_sp, &rif_index); 3054 if (err) 3055 goto err_rif_index_alloc; 3056 3057 rif = mlxsw_sp_rif_alloc(ops->rif_size, rif_index, vr->id, params->dev); 3058 if (!rif) { 3059 err = -ENOMEM; 3060 goto err_rif_alloc; 3061 } 3062 rif->mlxsw_sp = mlxsw_sp; 3063 rif->ops = ops; 3064 3065 fid = ops->fid_get(rif); 3066 if (IS_ERR(fid)) { 3067 err = PTR_ERR(fid); 3068 goto err_fid_get; 3069 } 3070 rif->fid = fid; 3071 3072 if (ops->setup) 3073 ops->setup(rif, params); 3074 3075 err = ops->configure(rif); 3076 if (err) 3077 goto err_configure; 3078 3079 err = mlxsw_sp_rif_fdb_op(mlxsw_sp, params->dev->dev_addr, 3080 mlxsw_sp_fid_index(fid), true); 3081 if (err) 3082 goto err_rif_fdb_op; 3083 3084 mlxsw_sp_rif_counters_alloc(rif); 3085 mlxsw_sp_fid_rif_set(fid, rif); 3086 mlxsw_sp->router->rifs[rif_index] = rif; 3087 vr->rif_count++; 3088 3089 return rif; 3090 3091 err_rif_fdb_op: 3092 ops->deconfigure(rif); 3093 err_configure: 3094 mlxsw_sp_fid_put(fid); 3095 err_fid_get: 3096 kfree(rif); 3097 err_rif_alloc: 3098 err_rif_index_alloc: 3099 mlxsw_sp_vr_put(vr); 3100 return ERR_PTR(err); 3101 } 3102 3103 void mlxsw_sp_rif_destroy(struct mlxsw_sp_rif *rif) 3104 { 3105 const struct mlxsw_sp_rif_ops *ops = rif->ops; 3106 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 3107 struct mlxsw_sp_fid *fid = rif->fid; 3108 struct mlxsw_sp_vr *vr; 3109 3110 mlxsw_sp_router_rif_gone_sync(mlxsw_sp, rif); 3111 vr = &mlxsw_sp->router->vrs[rif->vr_id]; 3112 3113 vr->rif_count--; 3114 mlxsw_sp->router->rifs[rif->rif_index] = NULL; 3115 mlxsw_sp_fid_rif_set(fid, NULL); 3116 mlxsw_sp_rif_counters_free(rif); 3117 mlxsw_sp_rif_fdb_op(mlxsw_sp, rif->dev->dev_addr, 3118 mlxsw_sp_fid_index(fid), false); 3119 ops->deconfigure(rif); 3120 mlxsw_sp_fid_put(fid); 3121 kfree(rif); 3122 mlxsw_sp_vr_put(vr); 3123 } 3124 3125 static void 3126 mlxsw_sp_rif_subport_params_init(struct mlxsw_sp_rif_params *params, 3127 struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan) 3128 { 3129 struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port; 3130 3131 params->vid = mlxsw_sp_port_vlan->vid; 3132 params->lag = mlxsw_sp_port->lagged; 3133 if (params->lag) 3134 params->lag_id = mlxsw_sp_port->lag_id; 3135 else 3136 params->system_port = mlxsw_sp_port->local_port; 3137 } 3138 3139 static int 3140 mlxsw_sp_port_vlan_router_join(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan, 3141 struct net_device *l3_dev) 3142 { 3143 struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port; 3144 struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp; 3145 u16 vid = mlxsw_sp_port_vlan->vid; 3146 struct mlxsw_sp_rif *rif; 3147 struct mlxsw_sp_fid *fid; 3148 int err; 3149 3150 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev); 3151 if (!rif) { 3152 struct mlxsw_sp_rif_params params = { 3153 .dev = l3_dev, 3154 }; 3155 3156 mlxsw_sp_rif_subport_params_init(¶ms, mlxsw_sp_port_vlan); 3157 rif = mlxsw_sp_rif_create(mlxsw_sp, ¶ms); 3158 if (IS_ERR(rif)) 3159 return PTR_ERR(rif); 3160 } 3161 3162 /* FID was already created, just take a reference */ 3163 fid = rif->ops->fid_get(rif); 3164 err = mlxsw_sp_fid_port_vid_map(fid, mlxsw_sp_port, vid); 3165 if (err) 3166 goto err_fid_port_vid_map; 3167 3168 err = mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, false); 3169 if (err) 3170 goto err_port_vid_learning_set; 3171 3172 err = mlxsw_sp_port_vid_stp_set(mlxsw_sp_port, vid, 3173 BR_STATE_FORWARDING); 3174 if (err) 3175 goto err_port_vid_stp_set; 3176 3177 mlxsw_sp_port_vlan->fid = fid; 3178 3179 return 0; 3180 3181 err_port_vid_stp_set: 3182 mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, true); 3183 err_port_vid_learning_set: 3184 mlxsw_sp_fid_port_vid_unmap(fid, mlxsw_sp_port, vid); 3185 err_fid_port_vid_map: 3186 mlxsw_sp_fid_put(fid); 3187 return err; 3188 } 3189 3190 void 3191 mlxsw_sp_port_vlan_router_leave(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan) 3192 { 3193 struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port; 3194 struct mlxsw_sp_fid *fid = mlxsw_sp_port_vlan->fid; 3195 u16 vid = mlxsw_sp_port_vlan->vid; 3196 3197 if (WARN_ON(mlxsw_sp_fid_type(fid) != MLXSW_SP_FID_TYPE_RFID)) 3198 return; 3199 3200 mlxsw_sp_port_vlan->fid = NULL; 3201 mlxsw_sp_port_vid_stp_set(mlxsw_sp_port, vid, BR_STATE_BLOCKING); 3202 mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, true); 3203 mlxsw_sp_fid_port_vid_unmap(fid, mlxsw_sp_port, vid); 3204 /* If router port holds the last reference on the rFID, then the 3205 * associated Sub-port RIF will be destroyed. 3206 */ 3207 mlxsw_sp_fid_put(fid); 3208 } 3209 3210 static int mlxsw_sp_inetaddr_port_vlan_event(struct net_device *l3_dev, 3211 struct net_device *port_dev, 3212 unsigned long event, u16 vid) 3213 { 3214 struct mlxsw_sp_port *mlxsw_sp_port = netdev_priv(port_dev); 3215 struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan; 3216 3217 mlxsw_sp_port_vlan = mlxsw_sp_port_vlan_find_by_vid(mlxsw_sp_port, vid); 3218 if (WARN_ON(!mlxsw_sp_port_vlan)) 3219 return -EINVAL; 3220 3221 switch (event) { 3222 case NETDEV_UP: 3223 return mlxsw_sp_port_vlan_router_join(mlxsw_sp_port_vlan, 3224 l3_dev); 3225 case NETDEV_DOWN: 3226 mlxsw_sp_port_vlan_router_leave(mlxsw_sp_port_vlan); 3227 break; 3228 } 3229 3230 return 0; 3231 } 3232 3233 static int mlxsw_sp_inetaddr_port_event(struct net_device *port_dev, 3234 unsigned long event) 3235 { 3236 if (netif_is_bridge_port(port_dev) || 3237 netif_is_lag_port(port_dev) || 3238 netif_is_ovs_port(port_dev)) 3239 return 0; 3240 3241 return mlxsw_sp_inetaddr_port_vlan_event(port_dev, port_dev, event, 1); 3242 } 3243 3244 static int __mlxsw_sp_inetaddr_lag_event(struct net_device *l3_dev, 3245 struct net_device *lag_dev, 3246 unsigned long event, u16 vid) 3247 { 3248 struct net_device *port_dev; 3249 struct list_head *iter; 3250 int err; 3251 3252 netdev_for_each_lower_dev(lag_dev, port_dev, iter) { 3253 if (mlxsw_sp_port_dev_check(port_dev)) { 3254 err = mlxsw_sp_inetaddr_port_vlan_event(l3_dev, 3255 port_dev, 3256 event, vid); 3257 if (err) 3258 return err; 3259 } 3260 } 3261 3262 return 0; 3263 } 3264 3265 static int mlxsw_sp_inetaddr_lag_event(struct net_device *lag_dev, 3266 unsigned long event) 3267 { 3268 if (netif_is_bridge_port(lag_dev)) 3269 return 0; 3270 3271 return __mlxsw_sp_inetaddr_lag_event(lag_dev, lag_dev, event, 1); 3272 } 3273 3274 static int mlxsw_sp_inetaddr_bridge_event(struct net_device *l3_dev, 3275 unsigned long event) 3276 { 3277 struct mlxsw_sp *mlxsw_sp = mlxsw_sp_lower_get(l3_dev); 3278 struct mlxsw_sp_rif_params params = { 3279 .dev = l3_dev, 3280 }; 3281 struct mlxsw_sp_rif *rif; 3282 3283 switch (event) { 3284 case NETDEV_UP: 3285 rif = mlxsw_sp_rif_create(mlxsw_sp, ¶ms); 3286 if (IS_ERR(rif)) 3287 return PTR_ERR(rif); 3288 break; 3289 case NETDEV_DOWN: 3290 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev); 3291 mlxsw_sp_rif_destroy(rif); 3292 break; 3293 } 3294 3295 return 0; 3296 } 3297 3298 static int mlxsw_sp_inetaddr_vlan_event(struct net_device *vlan_dev, 3299 unsigned long event) 3300 { 3301 struct net_device *real_dev = vlan_dev_real_dev(vlan_dev); 3302 u16 vid = vlan_dev_vlan_id(vlan_dev); 3303 3304 if (netif_is_bridge_port(vlan_dev)) 3305 return 0; 3306 3307 if (mlxsw_sp_port_dev_check(real_dev)) 3308 return mlxsw_sp_inetaddr_port_vlan_event(vlan_dev, real_dev, 3309 event, vid); 3310 else if (netif_is_lag_master(real_dev)) 3311 return __mlxsw_sp_inetaddr_lag_event(vlan_dev, real_dev, event, 3312 vid); 3313 else if (netif_is_bridge_master(real_dev) && br_vlan_enabled(real_dev)) 3314 return mlxsw_sp_inetaddr_bridge_event(vlan_dev, event); 3315 3316 return 0; 3317 } 3318 3319 static int __mlxsw_sp_inetaddr_event(struct net_device *dev, 3320 unsigned long event) 3321 { 3322 if (mlxsw_sp_port_dev_check(dev)) 3323 return mlxsw_sp_inetaddr_port_event(dev, event); 3324 else if (netif_is_lag_master(dev)) 3325 return mlxsw_sp_inetaddr_lag_event(dev, event); 3326 else if (netif_is_bridge_master(dev)) 3327 return mlxsw_sp_inetaddr_bridge_event(dev, event); 3328 else if (is_vlan_dev(dev)) 3329 return mlxsw_sp_inetaddr_vlan_event(dev, event); 3330 else 3331 return 0; 3332 } 3333 3334 int mlxsw_sp_inetaddr_event(struct notifier_block *unused, 3335 unsigned long event, void *ptr) 3336 { 3337 struct in_ifaddr *ifa = (struct in_ifaddr *) ptr; 3338 struct net_device *dev = ifa->ifa_dev->dev; 3339 struct mlxsw_sp *mlxsw_sp; 3340 struct mlxsw_sp_rif *rif; 3341 int err = 0; 3342 3343 mlxsw_sp = mlxsw_sp_lower_get(dev); 3344 if (!mlxsw_sp) 3345 goto out; 3346 3347 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev); 3348 if (!mlxsw_sp_rif_should_config(rif, ifa->ifa_dev, event)) 3349 goto out; 3350 3351 err = __mlxsw_sp_inetaddr_event(dev, event); 3352 out: 3353 return notifier_from_errno(err); 3354 } 3355 3356 static int mlxsw_sp_rif_edit(struct mlxsw_sp *mlxsw_sp, u16 rif_index, 3357 const char *mac, int mtu) 3358 { 3359 char ritr_pl[MLXSW_REG_RITR_LEN]; 3360 int err; 3361 3362 mlxsw_reg_ritr_rif_pack(ritr_pl, rif_index); 3363 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 3364 if (err) 3365 return err; 3366 3367 mlxsw_reg_ritr_mtu_set(ritr_pl, mtu); 3368 mlxsw_reg_ritr_if_mac_memcpy_to(ritr_pl, mac); 3369 mlxsw_reg_ritr_op_set(ritr_pl, MLXSW_REG_RITR_RIF_CREATE); 3370 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 3371 } 3372 3373 int mlxsw_sp_netdevice_router_port_event(struct net_device *dev) 3374 { 3375 struct mlxsw_sp *mlxsw_sp; 3376 struct mlxsw_sp_rif *rif; 3377 u16 fid_index; 3378 int err; 3379 3380 mlxsw_sp = mlxsw_sp_lower_get(dev); 3381 if (!mlxsw_sp) 3382 return 0; 3383 3384 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev); 3385 if (!rif) 3386 return 0; 3387 fid_index = mlxsw_sp_fid_index(rif->fid); 3388 3389 err = mlxsw_sp_rif_fdb_op(mlxsw_sp, rif->addr, fid_index, false); 3390 if (err) 3391 return err; 3392 3393 err = mlxsw_sp_rif_edit(mlxsw_sp, rif->rif_index, dev->dev_addr, 3394 dev->mtu); 3395 if (err) 3396 goto err_rif_edit; 3397 3398 err = mlxsw_sp_rif_fdb_op(mlxsw_sp, dev->dev_addr, fid_index, true); 3399 if (err) 3400 goto err_rif_fdb_op; 3401 3402 ether_addr_copy(rif->addr, dev->dev_addr); 3403 rif->mtu = dev->mtu; 3404 3405 netdev_dbg(dev, "Updated RIF=%d\n", rif->rif_index); 3406 3407 return 0; 3408 3409 err_rif_fdb_op: 3410 mlxsw_sp_rif_edit(mlxsw_sp, rif->rif_index, rif->addr, rif->mtu); 3411 err_rif_edit: 3412 mlxsw_sp_rif_fdb_op(mlxsw_sp, rif->addr, fid_index, true); 3413 return err; 3414 } 3415 3416 static int mlxsw_sp_port_vrf_join(struct mlxsw_sp *mlxsw_sp, 3417 struct net_device *l3_dev) 3418 { 3419 struct mlxsw_sp_rif *rif; 3420 3421 /* If netdev is already associated with a RIF, then we need to 3422 * destroy it and create a new one with the new virtual router ID. 3423 */ 3424 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev); 3425 if (rif) 3426 __mlxsw_sp_inetaddr_event(l3_dev, NETDEV_DOWN); 3427 3428 return __mlxsw_sp_inetaddr_event(l3_dev, NETDEV_UP); 3429 } 3430 3431 static void mlxsw_sp_port_vrf_leave(struct mlxsw_sp *mlxsw_sp, 3432 struct net_device *l3_dev) 3433 { 3434 struct mlxsw_sp_rif *rif; 3435 3436 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev); 3437 if (!rif) 3438 return; 3439 __mlxsw_sp_inetaddr_event(l3_dev, NETDEV_DOWN); 3440 } 3441 3442 int mlxsw_sp_netdevice_vrf_event(struct net_device *l3_dev, unsigned long event, 3443 struct netdev_notifier_changeupper_info *info) 3444 { 3445 struct mlxsw_sp *mlxsw_sp = mlxsw_sp_lower_get(l3_dev); 3446 int err = 0; 3447 3448 if (!mlxsw_sp) 3449 return 0; 3450 3451 switch (event) { 3452 case NETDEV_PRECHANGEUPPER: 3453 return 0; 3454 case NETDEV_CHANGEUPPER: 3455 if (info->linking) 3456 err = mlxsw_sp_port_vrf_join(mlxsw_sp, l3_dev); 3457 else 3458 mlxsw_sp_port_vrf_leave(mlxsw_sp, l3_dev); 3459 break; 3460 } 3461 3462 return err; 3463 } 3464 3465 static struct mlxsw_sp_rif_subport * 3466 mlxsw_sp_rif_subport_rif(const struct mlxsw_sp_rif *rif) 3467 { 3468 return container_of(rif, struct mlxsw_sp_rif_subport, common); 3469 } 3470 3471 static void mlxsw_sp_rif_subport_setup(struct mlxsw_sp_rif *rif, 3472 const struct mlxsw_sp_rif_params *params) 3473 { 3474 struct mlxsw_sp_rif_subport *rif_subport; 3475 3476 rif_subport = mlxsw_sp_rif_subport_rif(rif); 3477 rif_subport->vid = params->vid; 3478 rif_subport->lag = params->lag; 3479 if (params->lag) 3480 rif_subport->lag_id = params->lag_id; 3481 else 3482 rif_subport->system_port = params->system_port; 3483 } 3484 3485 static int mlxsw_sp_rif_subport_op(struct mlxsw_sp_rif *rif, bool enable) 3486 { 3487 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 3488 struct mlxsw_sp_rif_subport *rif_subport; 3489 char ritr_pl[MLXSW_REG_RITR_LEN]; 3490 3491 rif_subport = mlxsw_sp_rif_subport_rif(rif); 3492 mlxsw_reg_ritr_pack(ritr_pl, enable, MLXSW_REG_RITR_SP_IF, 3493 rif->rif_index, rif->vr_id, rif->dev->mtu, 3494 rif->dev->dev_addr); 3495 mlxsw_reg_ritr_sp_if_pack(ritr_pl, rif_subport->lag, 3496 rif_subport->lag ? rif_subport->lag_id : 3497 rif_subport->system_port, 3498 rif_subport->vid); 3499 3500 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 3501 } 3502 3503 static int mlxsw_sp_rif_subport_configure(struct mlxsw_sp_rif *rif) 3504 { 3505 return mlxsw_sp_rif_subport_op(rif, true); 3506 } 3507 3508 static void mlxsw_sp_rif_subport_deconfigure(struct mlxsw_sp_rif *rif) 3509 { 3510 mlxsw_sp_rif_subport_op(rif, false); 3511 } 3512 3513 static struct mlxsw_sp_fid * 3514 mlxsw_sp_rif_subport_fid_get(struct mlxsw_sp_rif *rif) 3515 { 3516 return mlxsw_sp_fid_rfid_get(rif->mlxsw_sp, rif->rif_index); 3517 } 3518 3519 static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_subport_ops = { 3520 .type = MLXSW_SP_RIF_TYPE_SUBPORT, 3521 .rif_size = sizeof(struct mlxsw_sp_rif_subport), 3522 .setup = mlxsw_sp_rif_subport_setup, 3523 .configure = mlxsw_sp_rif_subport_configure, 3524 .deconfigure = mlxsw_sp_rif_subport_deconfigure, 3525 .fid_get = mlxsw_sp_rif_subport_fid_get, 3526 }; 3527 3528 static int mlxsw_sp_rif_vlan_fid_op(struct mlxsw_sp_rif *rif, 3529 enum mlxsw_reg_ritr_if_type type, 3530 u16 vid_fid, bool enable) 3531 { 3532 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 3533 char ritr_pl[MLXSW_REG_RITR_LEN]; 3534 3535 mlxsw_reg_ritr_pack(ritr_pl, enable, type, rif->rif_index, rif->vr_id, 3536 rif->dev->mtu, rif->dev->dev_addr); 3537 mlxsw_reg_ritr_fid_set(ritr_pl, type, vid_fid); 3538 3539 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 3540 } 3541 3542 static u8 mlxsw_sp_router_port(const struct mlxsw_sp *mlxsw_sp) 3543 { 3544 return mlxsw_core_max_ports(mlxsw_sp->core) + 1; 3545 } 3546 3547 static int mlxsw_sp_rif_vlan_configure(struct mlxsw_sp_rif *rif) 3548 { 3549 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 3550 u16 vid = mlxsw_sp_fid_8021q_vid(rif->fid); 3551 int err; 3552 3553 err = mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_VLAN_IF, vid, true); 3554 if (err) 3555 return err; 3556 3557 err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC, 3558 mlxsw_sp_router_port(mlxsw_sp), true); 3559 if (err) 3560 goto err_fid_bc_flood_set; 3561 3562 return 0; 3563 3564 err_fid_bc_flood_set: 3565 mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_VLAN_IF, vid, false); 3566 return err; 3567 } 3568 3569 static void mlxsw_sp_rif_vlan_deconfigure(struct mlxsw_sp_rif *rif) 3570 { 3571 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 3572 u16 vid = mlxsw_sp_fid_8021q_vid(rif->fid); 3573 3574 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC, 3575 mlxsw_sp_router_port(mlxsw_sp), false); 3576 mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_VLAN_IF, vid, false); 3577 } 3578 3579 static struct mlxsw_sp_fid * 3580 mlxsw_sp_rif_vlan_fid_get(struct mlxsw_sp_rif *rif) 3581 { 3582 u16 vid = is_vlan_dev(rif->dev) ? vlan_dev_vlan_id(rif->dev) : 1; 3583 3584 return mlxsw_sp_fid_8021q_get(rif->mlxsw_sp, vid); 3585 } 3586 3587 static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_vlan_ops = { 3588 .type = MLXSW_SP_RIF_TYPE_VLAN, 3589 .rif_size = sizeof(struct mlxsw_sp_rif), 3590 .configure = mlxsw_sp_rif_vlan_configure, 3591 .deconfigure = mlxsw_sp_rif_vlan_deconfigure, 3592 .fid_get = mlxsw_sp_rif_vlan_fid_get, 3593 }; 3594 3595 static int mlxsw_sp_rif_fid_configure(struct mlxsw_sp_rif *rif) 3596 { 3597 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 3598 u16 fid_index = mlxsw_sp_fid_index(rif->fid); 3599 int err; 3600 3601 err = mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_FID_IF, fid_index, 3602 true); 3603 if (err) 3604 return err; 3605 3606 err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC, 3607 mlxsw_sp_router_port(mlxsw_sp), true); 3608 if (err) 3609 goto err_fid_bc_flood_set; 3610 3611 return 0; 3612 3613 err_fid_bc_flood_set: 3614 mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_FID_IF, fid_index, false); 3615 return err; 3616 } 3617 3618 static void mlxsw_sp_rif_fid_deconfigure(struct mlxsw_sp_rif *rif) 3619 { 3620 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 3621 u16 fid_index = mlxsw_sp_fid_index(rif->fid); 3622 3623 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC, 3624 mlxsw_sp_router_port(mlxsw_sp), false); 3625 mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_FID_IF, fid_index, false); 3626 } 3627 3628 static struct mlxsw_sp_fid * 3629 mlxsw_sp_rif_fid_fid_get(struct mlxsw_sp_rif *rif) 3630 { 3631 return mlxsw_sp_fid_8021d_get(rif->mlxsw_sp, rif->dev->ifindex); 3632 } 3633 3634 static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_fid_ops = { 3635 .type = MLXSW_SP_RIF_TYPE_FID, 3636 .rif_size = sizeof(struct mlxsw_sp_rif), 3637 .configure = mlxsw_sp_rif_fid_configure, 3638 .deconfigure = mlxsw_sp_rif_fid_deconfigure, 3639 .fid_get = mlxsw_sp_rif_fid_fid_get, 3640 }; 3641 3642 static const struct mlxsw_sp_rif_ops *mlxsw_sp_rif_ops_arr[] = { 3643 [MLXSW_SP_RIF_TYPE_SUBPORT] = &mlxsw_sp_rif_subport_ops, 3644 [MLXSW_SP_RIF_TYPE_VLAN] = &mlxsw_sp_rif_vlan_ops, 3645 [MLXSW_SP_RIF_TYPE_FID] = &mlxsw_sp_rif_fid_ops, 3646 }; 3647 3648 static int mlxsw_sp_rifs_init(struct mlxsw_sp *mlxsw_sp) 3649 { 3650 u64 max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); 3651 3652 mlxsw_sp->router->rifs = kcalloc(max_rifs, 3653 sizeof(struct mlxsw_sp_rif *), 3654 GFP_KERNEL); 3655 if (!mlxsw_sp->router->rifs) 3656 return -ENOMEM; 3657 3658 mlxsw_sp->router->rif_ops_arr = mlxsw_sp_rif_ops_arr; 3659 3660 return 0; 3661 } 3662 3663 static void mlxsw_sp_rifs_fini(struct mlxsw_sp *mlxsw_sp) 3664 { 3665 int i; 3666 3667 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++) 3668 WARN_ON_ONCE(mlxsw_sp->router->rifs[i]); 3669 3670 kfree(mlxsw_sp->router->rifs); 3671 } 3672 3673 static void mlxsw_sp_router_fib_dump_flush(struct notifier_block *nb) 3674 { 3675 struct mlxsw_sp_router *router; 3676 3677 /* Flush pending FIB notifications and then flush the device's 3678 * table before requesting another dump. The FIB notification 3679 * block is unregistered, so no need to take RTNL. 3680 */ 3681 mlxsw_core_flush_owq(); 3682 router = container_of(nb, struct mlxsw_sp_router, fib_nb); 3683 mlxsw_sp_router_fib_flush(router->mlxsw_sp); 3684 } 3685 3686 static int __mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp) 3687 { 3688 char rgcr_pl[MLXSW_REG_RGCR_LEN]; 3689 u64 max_rifs; 3690 int err; 3691 3692 if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_RIFS)) 3693 return -EIO; 3694 max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); 3695 3696 mlxsw_reg_rgcr_pack(rgcr_pl, true); 3697 mlxsw_reg_rgcr_max_router_interfaces_set(rgcr_pl, max_rifs); 3698 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rgcr), rgcr_pl); 3699 if (err) 3700 return err; 3701 return 0; 3702 } 3703 3704 static void __mlxsw_sp_router_fini(struct mlxsw_sp *mlxsw_sp) 3705 { 3706 char rgcr_pl[MLXSW_REG_RGCR_LEN]; 3707 3708 mlxsw_reg_rgcr_pack(rgcr_pl, false); 3709 mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rgcr), rgcr_pl); 3710 } 3711 3712 int mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp) 3713 { 3714 struct mlxsw_sp_router *router; 3715 int err; 3716 3717 router = kzalloc(sizeof(*mlxsw_sp->router), GFP_KERNEL); 3718 if (!router) 3719 return -ENOMEM; 3720 mlxsw_sp->router = router; 3721 router->mlxsw_sp = mlxsw_sp; 3722 3723 INIT_LIST_HEAD(&mlxsw_sp->router->nexthop_neighs_list); 3724 err = __mlxsw_sp_router_init(mlxsw_sp); 3725 if (err) 3726 goto err_router_init; 3727 3728 err = mlxsw_sp_rifs_init(mlxsw_sp); 3729 if (err) 3730 goto err_rifs_init; 3731 3732 err = rhashtable_init(&mlxsw_sp->router->nexthop_ht, 3733 &mlxsw_sp_nexthop_ht_params); 3734 if (err) 3735 goto err_nexthop_ht_init; 3736 3737 err = rhashtable_init(&mlxsw_sp->router->nexthop_group_ht, 3738 &mlxsw_sp_nexthop_group_ht_params); 3739 if (err) 3740 goto err_nexthop_group_ht_init; 3741 3742 err = mlxsw_sp_lpm_init(mlxsw_sp); 3743 if (err) 3744 goto err_lpm_init; 3745 3746 err = mlxsw_sp_vrs_init(mlxsw_sp); 3747 if (err) 3748 goto err_vrs_init; 3749 3750 err = mlxsw_sp_neigh_init(mlxsw_sp); 3751 if (err) 3752 goto err_neigh_init; 3753 3754 mlxsw_sp->router->fib_nb.notifier_call = mlxsw_sp_router_fib_event; 3755 err = register_fib_notifier(&mlxsw_sp->router->fib_nb, 3756 mlxsw_sp_router_fib_dump_flush); 3757 if (err) 3758 goto err_register_fib_notifier; 3759 3760 return 0; 3761 3762 err_register_fib_notifier: 3763 mlxsw_sp_neigh_fini(mlxsw_sp); 3764 err_neigh_init: 3765 mlxsw_sp_vrs_fini(mlxsw_sp); 3766 err_vrs_init: 3767 mlxsw_sp_lpm_fini(mlxsw_sp); 3768 err_lpm_init: 3769 rhashtable_destroy(&mlxsw_sp->router->nexthop_group_ht); 3770 err_nexthop_group_ht_init: 3771 rhashtable_destroy(&mlxsw_sp->router->nexthop_ht); 3772 err_nexthop_ht_init: 3773 mlxsw_sp_rifs_fini(mlxsw_sp); 3774 err_rifs_init: 3775 __mlxsw_sp_router_fini(mlxsw_sp); 3776 err_router_init: 3777 kfree(mlxsw_sp->router); 3778 return err; 3779 } 3780 3781 void mlxsw_sp_router_fini(struct mlxsw_sp *mlxsw_sp) 3782 { 3783 unregister_fib_notifier(&mlxsw_sp->router->fib_nb); 3784 mlxsw_sp_neigh_fini(mlxsw_sp); 3785 mlxsw_sp_vrs_fini(mlxsw_sp); 3786 mlxsw_sp_lpm_fini(mlxsw_sp); 3787 rhashtable_destroy(&mlxsw_sp->router->nexthop_group_ht); 3788 rhashtable_destroy(&mlxsw_sp->router->nexthop_ht); 3789 mlxsw_sp_rifs_fini(mlxsw_sp); 3790 __mlxsw_sp_router_fini(mlxsw_sp); 3791 kfree(mlxsw_sp->router); 3792 } 3793