1 // SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0 2 /* Copyright (c) 2016-2018 Mellanox Technologies. All rights reserved */ 3 4 #include <linux/kernel.h> 5 #include <linux/types.h> 6 #include <linux/rhashtable.h> 7 #include <linux/bitops.h> 8 #include <linux/in6.h> 9 #include <linux/notifier.h> 10 #include <linux/inetdevice.h> 11 #include <linux/netdevice.h> 12 #include <linux/if_bridge.h> 13 #include <linux/socket.h> 14 #include <linux/route.h> 15 #include <linux/gcd.h> 16 #include <linux/random.h> 17 #include <linux/if_macvlan.h> 18 #include <linux/refcount.h> 19 #include <net/netevent.h> 20 #include <net/neighbour.h> 21 #include <net/arp.h> 22 #include <net/ip_fib.h> 23 #include <net/ip6_fib.h> 24 #include <net/fib_rules.h> 25 #include <net/ip_tunnels.h> 26 #include <net/l3mdev.h> 27 #include <net/addrconf.h> 28 #include <net/ndisc.h> 29 #include <net/ipv6.h> 30 #include <net/fib_notifier.h> 31 #include <net/switchdev.h> 32 33 #include "spectrum.h" 34 #include "core.h" 35 #include "reg.h" 36 #include "spectrum_cnt.h" 37 #include "spectrum_dpipe.h" 38 #include "spectrum_ipip.h" 39 #include "spectrum_mr.h" 40 #include "spectrum_mr_tcam.h" 41 #include "spectrum_router.h" 42 #include "spectrum_span.h" 43 44 struct mlxsw_sp_fib; 45 struct mlxsw_sp_vr; 46 struct mlxsw_sp_lpm_tree; 47 struct mlxsw_sp_rif_ops; 48 49 struct mlxsw_sp_router { 50 struct mlxsw_sp *mlxsw_sp; 51 struct mlxsw_sp_rif **rifs; 52 struct mlxsw_sp_vr *vrs; 53 struct rhashtable neigh_ht; 54 struct rhashtable nexthop_group_ht; 55 struct rhashtable nexthop_ht; 56 struct list_head nexthop_list; 57 struct { 58 /* One tree for each protocol: IPv4 and IPv6 */ 59 struct mlxsw_sp_lpm_tree *proto_trees[2]; 60 struct mlxsw_sp_lpm_tree *trees; 61 unsigned int tree_count; 62 } lpm; 63 struct { 64 struct delayed_work dw; 65 unsigned long interval; /* ms */ 66 } neighs_update; 67 struct delayed_work nexthop_probe_dw; 68 #define MLXSW_SP_UNRESOLVED_NH_PROBE_INTERVAL 5000 /* ms */ 69 struct list_head nexthop_neighs_list; 70 struct list_head ipip_list; 71 bool aborted; 72 struct notifier_block fib_nb; 73 struct notifier_block netevent_nb; 74 struct notifier_block inetaddr_nb; 75 struct notifier_block inet6addr_nb; 76 const struct mlxsw_sp_rif_ops **rif_ops_arr; 77 const struct mlxsw_sp_ipip_ops **ipip_ops_arr; 78 }; 79 80 struct mlxsw_sp_rif { 81 struct list_head nexthop_list; 82 struct list_head neigh_list; 83 struct net_device *dev; /* NULL for underlay RIF */ 84 struct mlxsw_sp_fid *fid; 85 unsigned char addr[ETH_ALEN]; 86 int mtu; 87 u16 rif_index; 88 u16 vr_id; 89 const struct mlxsw_sp_rif_ops *ops; 90 struct mlxsw_sp *mlxsw_sp; 91 92 unsigned int counter_ingress; 93 bool counter_ingress_valid; 94 unsigned int counter_egress; 95 bool counter_egress_valid; 96 }; 97 98 struct mlxsw_sp_rif_params { 99 struct net_device *dev; 100 union { 101 u16 system_port; 102 u16 lag_id; 103 }; 104 u16 vid; 105 bool lag; 106 }; 107 108 struct mlxsw_sp_rif_subport { 109 struct mlxsw_sp_rif common; 110 refcount_t ref_count; 111 union { 112 u16 system_port; 113 u16 lag_id; 114 }; 115 u16 vid; 116 bool lag; 117 }; 118 119 struct mlxsw_sp_rif_ipip_lb { 120 struct mlxsw_sp_rif common; 121 struct mlxsw_sp_rif_ipip_lb_config lb_config; 122 u16 ul_vr_id; /* Reserved for Spectrum-2. */ 123 u16 ul_rif_id; /* Reserved for Spectrum. */ 124 }; 125 126 struct mlxsw_sp_rif_params_ipip_lb { 127 struct mlxsw_sp_rif_params common; 128 struct mlxsw_sp_rif_ipip_lb_config lb_config; 129 }; 130 131 struct mlxsw_sp_rif_ops { 132 enum mlxsw_sp_rif_type type; 133 size_t rif_size; 134 135 void (*setup)(struct mlxsw_sp_rif *rif, 136 const struct mlxsw_sp_rif_params *params); 137 int (*configure)(struct mlxsw_sp_rif *rif); 138 void (*deconfigure)(struct mlxsw_sp_rif *rif); 139 struct mlxsw_sp_fid * (*fid_get)(struct mlxsw_sp_rif *rif, 140 struct netlink_ext_ack *extack); 141 void (*fdb_del)(struct mlxsw_sp_rif *rif, const char *mac); 142 }; 143 144 static void mlxsw_sp_rif_destroy(struct mlxsw_sp_rif *rif); 145 static void mlxsw_sp_lpm_tree_hold(struct mlxsw_sp_lpm_tree *lpm_tree); 146 static void mlxsw_sp_lpm_tree_put(struct mlxsw_sp *mlxsw_sp, 147 struct mlxsw_sp_lpm_tree *lpm_tree); 148 static int mlxsw_sp_vr_lpm_tree_bind(struct mlxsw_sp *mlxsw_sp, 149 const struct mlxsw_sp_fib *fib, 150 u8 tree_id); 151 static int mlxsw_sp_vr_lpm_tree_unbind(struct mlxsw_sp *mlxsw_sp, 152 const struct mlxsw_sp_fib *fib); 153 154 static unsigned int * 155 mlxsw_sp_rif_p_counter_get(struct mlxsw_sp_rif *rif, 156 enum mlxsw_sp_rif_counter_dir dir) 157 { 158 switch (dir) { 159 case MLXSW_SP_RIF_COUNTER_EGRESS: 160 return &rif->counter_egress; 161 case MLXSW_SP_RIF_COUNTER_INGRESS: 162 return &rif->counter_ingress; 163 } 164 return NULL; 165 } 166 167 static bool 168 mlxsw_sp_rif_counter_valid_get(struct mlxsw_sp_rif *rif, 169 enum mlxsw_sp_rif_counter_dir dir) 170 { 171 switch (dir) { 172 case MLXSW_SP_RIF_COUNTER_EGRESS: 173 return rif->counter_egress_valid; 174 case MLXSW_SP_RIF_COUNTER_INGRESS: 175 return rif->counter_ingress_valid; 176 } 177 return false; 178 } 179 180 static void 181 mlxsw_sp_rif_counter_valid_set(struct mlxsw_sp_rif *rif, 182 enum mlxsw_sp_rif_counter_dir dir, 183 bool valid) 184 { 185 switch (dir) { 186 case MLXSW_SP_RIF_COUNTER_EGRESS: 187 rif->counter_egress_valid = valid; 188 break; 189 case MLXSW_SP_RIF_COUNTER_INGRESS: 190 rif->counter_ingress_valid = valid; 191 break; 192 } 193 } 194 195 static int mlxsw_sp_rif_counter_edit(struct mlxsw_sp *mlxsw_sp, u16 rif_index, 196 unsigned int counter_index, bool enable, 197 enum mlxsw_sp_rif_counter_dir dir) 198 { 199 char ritr_pl[MLXSW_REG_RITR_LEN]; 200 bool is_egress = false; 201 int err; 202 203 if (dir == MLXSW_SP_RIF_COUNTER_EGRESS) 204 is_egress = true; 205 mlxsw_reg_ritr_rif_pack(ritr_pl, rif_index); 206 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 207 if (err) 208 return err; 209 210 mlxsw_reg_ritr_counter_pack(ritr_pl, counter_index, enable, 211 is_egress); 212 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 213 } 214 215 int mlxsw_sp_rif_counter_value_get(struct mlxsw_sp *mlxsw_sp, 216 struct mlxsw_sp_rif *rif, 217 enum mlxsw_sp_rif_counter_dir dir, u64 *cnt) 218 { 219 char ricnt_pl[MLXSW_REG_RICNT_LEN]; 220 unsigned int *p_counter_index; 221 bool valid; 222 int err; 223 224 valid = mlxsw_sp_rif_counter_valid_get(rif, dir); 225 if (!valid) 226 return -EINVAL; 227 228 p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir); 229 if (!p_counter_index) 230 return -EINVAL; 231 mlxsw_reg_ricnt_pack(ricnt_pl, *p_counter_index, 232 MLXSW_REG_RICNT_OPCODE_NOP); 233 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ricnt), ricnt_pl); 234 if (err) 235 return err; 236 *cnt = mlxsw_reg_ricnt_good_unicast_packets_get(ricnt_pl); 237 return 0; 238 } 239 240 static int mlxsw_sp_rif_counter_clear(struct mlxsw_sp *mlxsw_sp, 241 unsigned int counter_index) 242 { 243 char ricnt_pl[MLXSW_REG_RICNT_LEN]; 244 245 mlxsw_reg_ricnt_pack(ricnt_pl, counter_index, 246 MLXSW_REG_RICNT_OPCODE_CLEAR); 247 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ricnt), ricnt_pl); 248 } 249 250 int mlxsw_sp_rif_counter_alloc(struct mlxsw_sp *mlxsw_sp, 251 struct mlxsw_sp_rif *rif, 252 enum mlxsw_sp_rif_counter_dir dir) 253 { 254 unsigned int *p_counter_index; 255 int err; 256 257 p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir); 258 if (!p_counter_index) 259 return -EINVAL; 260 err = mlxsw_sp_counter_alloc(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF, 261 p_counter_index); 262 if (err) 263 return err; 264 265 err = mlxsw_sp_rif_counter_clear(mlxsw_sp, *p_counter_index); 266 if (err) 267 goto err_counter_clear; 268 269 err = mlxsw_sp_rif_counter_edit(mlxsw_sp, rif->rif_index, 270 *p_counter_index, true, dir); 271 if (err) 272 goto err_counter_edit; 273 mlxsw_sp_rif_counter_valid_set(rif, dir, true); 274 return 0; 275 276 err_counter_edit: 277 err_counter_clear: 278 mlxsw_sp_counter_free(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF, 279 *p_counter_index); 280 return err; 281 } 282 283 void mlxsw_sp_rif_counter_free(struct mlxsw_sp *mlxsw_sp, 284 struct mlxsw_sp_rif *rif, 285 enum mlxsw_sp_rif_counter_dir dir) 286 { 287 unsigned int *p_counter_index; 288 289 if (!mlxsw_sp_rif_counter_valid_get(rif, dir)) 290 return; 291 292 p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir); 293 if (WARN_ON(!p_counter_index)) 294 return; 295 mlxsw_sp_rif_counter_edit(mlxsw_sp, rif->rif_index, 296 *p_counter_index, false, dir); 297 mlxsw_sp_counter_free(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF, 298 *p_counter_index); 299 mlxsw_sp_rif_counter_valid_set(rif, dir, false); 300 } 301 302 static void mlxsw_sp_rif_counters_alloc(struct mlxsw_sp_rif *rif) 303 { 304 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 305 struct devlink *devlink; 306 307 devlink = priv_to_devlink(mlxsw_sp->core); 308 if (!devlink_dpipe_table_counter_enabled(devlink, 309 MLXSW_SP_DPIPE_TABLE_NAME_ERIF)) 310 return; 311 mlxsw_sp_rif_counter_alloc(mlxsw_sp, rif, MLXSW_SP_RIF_COUNTER_EGRESS); 312 } 313 314 static void mlxsw_sp_rif_counters_free(struct mlxsw_sp_rif *rif) 315 { 316 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 317 318 mlxsw_sp_rif_counter_free(mlxsw_sp, rif, MLXSW_SP_RIF_COUNTER_EGRESS); 319 } 320 321 #define MLXSW_SP_PREFIX_COUNT (sizeof(struct in6_addr) * BITS_PER_BYTE + 1) 322 323 struct mlxsw_sp_prefix_usage { 324 DECLARE_BITMAP(b, MLXSW_SP_PREFIX_COUNT); 325 }; 326 327 #define mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) \ 328 for_each_set_bit(prefix, (prefix_usage)->b, MLXSW_SP_PREFIX_COUNT) 329 330 static bool 331 mlxsw_sp_prefix_usage_eq(struct mlxsw_sp_prefix_usage *prefix_usage1, 332 struct mlxsw_sp_prefix_usage *prefix_usage2) 333 { 334 return !memcmp(prefix_usage1, prefix_usage2, sizeof(*prefix_usage1)); 335 } 336 337 static void 338 mlxsw_sp_prefix_usage_cpy(struct mlxsw_sp_prefix_usage *prefix_usage1, 339 struct mlxsw_sp_prefix_usage *prefix_usage2) 340 { 341 memcpy(prefix_usage1, prefix_usage2, sizeof(*prefix_usage1)); 342 } 343 344 static void 345 mlxsw_sp_prefix_usage_set(struct mlxsw_sp_prefix_usage *prefix_usage, 346 unsigned char prefix_len) 347 { 348 set_bit(prefix_len, prefix_usage->b); 349 } 350 351 static void 352 mlxsw_sp_prefix_usage_clear(struct mlxsw_sp_prefix_usage *prefix_usage, 353 unsigned char prefix_len) 354 { 355 clear_bit(prefix_len, prefix_usage->b); 356 } 357 358 struct mlxsw_sp_fib_key { 359 unsigned char addr[sizeof(struct in6_addr)]; 360 unsigned char prefix_len; 361 }; 362 363 enum mlxsw_sp_fib_entry_type { 364 MLXSW_SP_FIB_ENTRY_TYPE_REMOTE, 365 MLXSW_SP_FIB_ENTRY_TYPE_LOCAL, 366 MLXSW_SP_FIB_ENTRY_TYPE_TRAP, 367 MLXSW_SP_FIB_ENTRY_TYPE_BLACKHOLE, 368 369 /* This is a special case of local delivery, where a packet should be 370 * decapsulated on reception. Note that there is no corresponding ENCAP, 371 * because that's a type of next hop, not of FIB entry. (There can be 372 * several next hops in a REMOTE entry, and some of them may be 373 * encapsulating entries.) 374 */ 375 MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP, 376 MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP, 377 }; 378 379 struct mlxsw_sp_nexthop_group; 380 381 struct mlxsw_sp_fib_node { 382 struct list_head entry_list; 383 struct list_head list; 384 struct rhash_head ht_node; 385 struct mlxsw_sp_fib *fib; 386 struct mlxsw_sp_fib_key key; 387 }; 388 389 struct mlxsw_sp_fib_entry_decap { 390 struct mlxsw_sp_ipip_entry *ipip_entry; 391 u32 tunnel_index; 392 }; 393 394 struct mlxsw_sp_fib_entry { 395 struct list_head list; 396 struct mlxsw_sp_fib_node *fib_node; 397 enum mlxsw_sp_fib_entry_type type; 398 struct list_head nexthop_group_node; 399 struct mlxsw_sp_nexthop_group *nh_group; 400 struct mlxsw_sp_fib_entry_decap decap; /* Valid for decap entries. */ 401 }; 402 403 struct mlxsw_sp_fib4_entry { 404 struct mlxsw_sp_fib_entry common; 405 u32 tb_id; 406 u32 prio; 407 u8 tos; 408 u8 type; 409 }; 410 411 struct mlxsw_sp_fib6_entry { 412 struct mlxsw_sp_fib_entry common; 413 struct list_head rt6_list; 414 unsigned int nrt6; 415 }; 416 417 struct mlxsw_sp_rt6 { 418 struct list_head list; 419 struct fib6_info *rt; 420 }; 421 422 struct mlxsw_sp_lpm_tree { 423 u8 id; /* tree ID */ 424 unsigned int ref_count; 425 enum mlxsw_sp_l3proto proto; 426 unsigned long prefix_ref_count[MLXSW_SP_PREFIX_COUNT]; 427 struct mlxsw_sp_prefix_usage prefix_usage; 428 }; 429 430 struct mlxsw_sp_fib { 431 struct rhashtable ht; 432 struct list_head node_list; 433 struct mlxsw_sp_vr *vr; 434 struct mlxsw_sp_lpm_tree *lpm_tree; 435 enum mlxsw_sp_l3proto proto; 436 }; 437 438 struct mlxsw_sp_vr { 439 u16 id; /* virtual router ID */ 440 u32 tb_id; /* kernel fib table id */ 441 unsigned int rif_count; 442 struct mlxsw_sp_fib *fib4; 443 struct mlxsw_sp_fib *fib6; 444 struct mlxsw_sp_mr_table *mr_table[MLXSW_SP_L3_PROTO_MAX]; 445 struct mlxsw_sp_rif *ul_rif; 446 refcount_t ul_rif_refcnt; 447 }; 448 449 static const struct rhashtable_params mlxsw_sp_fib_ht_params; 450 451 static struct mlxsw_sp_fib *mlxsw_sp_fib_create(struct mlxsw_sp *mlxsw_sp, 452 struct mlxsw_sp_vr *vr, 453 enum mlxsw_sp_l3proto proto) 454 { 455 struct mlxsw_sp_lpm_tree *lpm_tree; 456 struct mlxsw_sp_fib *fib; 457 int err; 458 459 lpm_tree = mlxsw_sp->router->lpm.proto_trees[proto]; 460 fib = kzalloc(sizeof(*fib), GFP_KERNEL); 461 if (!fib) 462 return ERR_PTR(-ENOMEM); 463 err = rhashtable_init(&fib->ht, &mlxsw_sp_fib_ht_params); 464 if (err) 465 goto err_rhashtable_init; 466 INIT_LIST_HEAD(&fib->node_list); 467 fib->proto = proto; 468 fib->vr = vr; 469 fib->lpm_tree = lpm_tree; 470 mlxsw_sp_lpm_tree_hold(lpm_tree); 471 err = mlxsw_sp_vr_lpm_tree_bind(mlxsw_sp, fib, lpm_tree->id); 472 if (err) 473 goto err_lpm_tree_bind; 474 return fib; 475 476 err_lpm_tree_bind: 477 mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree); 478 err_rhashtable_init: 479 kfree(fib); 480 return ERR_PTR(err); 481 } 482 483 static void mlxsw_sp_fib_destroy(struct mlxsw_sp *mlxsw_sp, 484 struct mlxsw_sp_fib *fib) 485 { 486 mlxsw_sp_vr_lpm_tree_unbind(mlxsw_sp, fib); 487 mlxsw_sp_lpm_tree_put(mlxsw_sp, fib->lpm_tree); 488 WARN_ON(!list_empty(&fib->node_list)); 489 rhashtable_destroy(&fib->ht); 490 kfree(fib); 491 } 492 493 static struct mlxsw_sp_lpm_tree * 494 mlxsw_sp_lpm_tree_find_unused(struct mlxsw_sp *mlxsw_sp) 495 { 496 static struct mlxsw_sp_lpm_tree *lpm_tree; 497 int i; 498 499 for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) { 500 lpm_tree = &mlxsw_sp->router->lpm.trees[i]; 501 if (lpm_tree->ref_count == 0) 502 return lpm_tree; 503 } 504 return NULL; 505 } 506 507 static int mlxsw_sp_lpm_tree_alloc(struct mlxsw_sp *mlxsw_sp, 508 struct mlxsw_sp_lpm_tree *lpm_tree) 509 { 510 char ralta_pl[MLXSW_REG_RALTA_LEN]; 511 512 mlxsw_reg_ralta_pack(ralta_pl, true, 513 (enum mlxsw_reg_ralxx_protocol) lpm_tree->proto, 514 lpm_tree->id); 515 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), ralta_pl); 516 } 517 518 static void mlxsw_sp_lpm_tree_free(struct mlxsw_sp *mlxsw_sp, 519 struct mlxsw_sp_lpm_tree *lpm_tree) 520 { 521 char ralta_pl[MLXSW_REG_RALTA_LEN]; 522 523 mlxsw_reg_ralta_pack(ralta_pl, false, 524 (enum mlxsw_reg_ralxx_protocol) lpm_tree->proto, 525 lpm_tree->id); 526 mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), ralta_pl); 527 } 528 529 static int 530 mlxsw_sp_lpm_tree_left_struct_set(struct mlxsw_sp *mlxsw_sp, 531 struct mlxsw_sp_prefix_usage *prefix_usage, 532 struct mlxsw_sp_lpm_tree *lpm_tree) 533 { 534 char ralst_pl[MLXSW_REG_RALST_LEN]; 535 u8 root_bin = 0; 536 u8 prefix; 537 u8 last_prefix = MLXSW_REG_RALST_BIN_NO_CHILD; 538 539 mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) 540 root_bin = prefix; 541 542 mlxsw_reg_ralst_pack(ralst_pl, root_bin, lpm_tree->id); 543 mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) { 544 if (prefix == 0) 545 continue; 546 mlxsw_reg_ralst_bin_pack(ralst_pl, prefix, last_prefix, 547 MLXSW_REG_RALST_BIN_NO_CHILD); 548 last_prefix = prefix; 549 } 550 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralst), ralst_pl); 551 } 552 553 static struct mlxsw_sp_lpm_tree * 554 mlxsw_sp_lpm_tree_create(struct mlxsw_sp *mlxsw_sp, 555 struct mlxsw_sp_prefix_usage *prefix_usage, 556 enum mlxsw_sp_l3proto proto) 557 { 558 struct mlxsw_sp_lpm_tree *lpm_tree; 559 int err; 560 561 lpm_tree = mlxsw_sp_lpm_tree_find_unused(mlxsw_sp); 562 if (!lpm_tree) 563 return ERR_PTR(-EBUSY); 564 lpm_tree->proto = proto; 565 err = mlxsw_sp_lpm_tree_alloc(mlxsw_sp, lpm_tree); 566 if (err) 567 return ERR_PTR(err); 568 569 err = mlxsw_sp_lpm_tree_left_struct_set(mlxsw_sp, prefix_usage, 570 lpm_tree); 571 if (err) 572 goto err_left_struct_set; 573 memcpy(&lpm_tree->prefix_usage, prefix_usage, 574 sizeof(lpm_tree->prefix_usage)); 575 memset(&lpm_tree->prefix_ref_count, 0, 576 sizeof(lpm_tree->prefix_ref_count)); 577 lpm_tree->ref_count = 1; 578 return lpm_tree; 579 580 err_left_struct_set: 581 mlxsw_sp_lpm_tree_free(mlxsw_sp, lpm_tree); 582 return ERR_PTR(err); 583 } 584 585 static void mlxsw_sp_lpm_tree_destroy(struct mlxsw_sp *mlxsw_sp, 586 struct mlxsw_sp_lpm_tree *lpm_tree) 587 { 588 mlxsw_sp_lpm_tree_free(mlxsw_sp, lpm_tree); 589 } 590 591 static struct mlxsw_sp_lpm_tree * 592 mlxsw_sp_lpm_tree_get(struct mlxsw_sp *mlxsw_sp, 593 struct mlxsw_sp_prefix_usage *prefix_usage, 594 enum mlxsw_sp_l3proto proto) 595 { 596 struct mlxsw_sp_lpm_tree *lpm_tree; 597 int i; 598 599 for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) { 600 lpm_tree = &mlxsw_sp->router->lpm.trees[i]; 601 if (lpm_tree->ref_count != 0 && 602 lpm_tree->proto == proto && 603 mlxsw_sp_prefix_usage_eq(&lpm_tree->prefix_usage, 604 prefix_usage)) { 605 mlxsw_sp_lpm_tree_hold(lpm_tree); 606 return lpm_tree; 607 } 608 } 609 return mlxsw_sp_lpm_tree_create(mlxsw_sp, prefix_usage, proto); 610 } 611 612 static void mlxsw_sp_lpm_tree_hold(struct mlxsw_sp_lpm_tree *lpm_tree) 613 { 614 lpm_tree->ref_count++; 615 } 616 617 static void mlxsw_sp_lpm_tree_put(struct mlxsw_sp *mlxsw_sp, 618 struct mlxsw_sp_lpm_tree *lpm_tree) 619 { 620 if (--lpm_tree->ref_count == 0) 621 mlxsw_sp_lpm_tree_destroy(mlxsw_sp, lpm_tree); 622 } 623 624 #define MLXSW_SP_LPM_TREE_MIN 1 /* tree 0 is reserved */ 625 626 static int mlxsw_sp_lpm_init(struct mlxsw_sp *mlxsw_sp) 627 { 628 struct mlxsw_sp_prefix_usage req_prefix_usage = {{ 0 } }; 629 struct mlxsw_sp_lpm_tree *lpm_tree; 630 u64 max_trees; 631 int err, i; 632 633 if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_LPM_TREES)) 634 return -EIO; 635 636 max_trees = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_LPM_TREES); 637 mlxsw_sp->router->lpm.tree_count = max_trees - MLXSW_SP_LPM_TREE_MIN; 638 mlxsw_sp->router->lpm.trees = kcalloc(mlxsw_sp->router->lpm.tree_count, 639 sizeof(struct mlxsw_sp_lpm_tree), 640 GFP_KERNEL); 641 if (!mlxsw_sp->router->lpm.trees) 642 return -ENOMEM; 643 644 for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) { 645 lpm_tree = &mlxsw_sp->router->lpm.trees[i]; 646 lpm_tree->id = i + MLXSW_SP_LPM_TREE_MIN; 647 } 648 649 lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage, 650 MLXSW_SP_L3_PROTO_IPV4); 651 if (IS_ERR(lpm_tree)) { 652 err = PTR_ERR(lpm_tree); 653 goto err_ipv4_tree_get; 654 } 655 mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV4] = lpm_tree; 656 657 lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage, 658 MLXSW_SP_L3_PROTO_IPV6); 659 if (IS_ERR(lpm_tree)) { 660 err = PTR_ERR(lpm_tree); 661 goto err_ipv6_tree_get; 662 } 663 mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV6] = lpm_tree; 664 665 return 0; 666 667 err_ipv6_tree_get: 668 lpm_tree = mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV4]; 669 mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree); 670 err_ipv4_tree_get: 671 kfree(mlxsw_sp->router->lpm.trees); 672 return err; 673 } 674 675 static void mlxsw_sp_lpm_fini(struct mlxsw_sp *mlxsw_sp) 676 { 677 struct mlxsw_sp_lpm_tree *lpm_tree; 678 679 lpm_tree = mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV6]; 680 mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree); 681 682 lpm_tree = mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV4]; 683 mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree); 684 685 kfree(mlxsw_sp->router->lpm.trees); 686 } 687 688 static bool mlxsw_sp_vr_is_used(const struct mlxsw_sp_vr *vr) 689 { 690 return !!vr->fib4 || !!vr->fib6 || 691 !!vr->mr_table[MLXSW_SP_L3_PROTO_IPV4] || 692 !!vr->mr_table[MLXSW_SP_L3_PROTO_IPV6]; 693 } 694 695 static struct mlxsw_sp_vr *mlxsw_sp_vr_find_unused(struct mlxsw_sp *mlxsw_sp) 696 { 697 struct mlxsw_sp_vr *vr; 698 int i; 699 700 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) { 701 vr = &mlxsw_sp->router->vrs[i]; 702 if (!mlxsw_sp_vr_is_used(vr)) 703 return vr; 704 } 705 return NULL; 706 } 707 708 static int mlxsw_sp_vr_lpm_tree_bind(struct mlxsw_sp *mlxsw_sp, 709 const struct mlxsw_sp_fib *fib, u8 tree_id) 710 { 711 char raltb_pl[MLXSW_REG_RALTB_LEN]; 712 713 mlxsw_reg_raltb_pack(raltb_pl, fib->vr->id, 714 (enum mlxsw_reg_ralxx_protocol) fib->proto, 715 tree_id); 716 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb), raltb_pl); 717 } 718 719 static int mlxsw_sp_vr_lpm_tree_unbind(struct mlxsw_sp *mlxsw_sp, 720 const struct mlxsw_sp_fib *fib) 721 { 722 char raltb_pl[MLXSW_REG_RALTB_LEN]; 723 724 /* Bind to tree 0 which is default */ 725 mlxsw_reg_raltb_pack(raltb_pl, fib->vr->id, 726 (enum mlxsw_reg_ralxx_protocol) fib->proto, 0); 727 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb), raltb_pl); 728 } 729 730 static u32 mlxsw_sp_fix_tb_id(u32 tb_id) 731 { 732 /* For our purpose, squash main, default and local tables into one */ 733 if (tb_id == RT_TABLE_LOCAL || tb_id == RT_TABLE_DEFAULT) 734 tb_id = RT_TABLE_MAIN; 735 return tb_id; 736 } 737 738 static struct mlxsw_sp_vr *mlxsw_sp_vr_find(struct mlxsw_sp *mlxsw_sp, 739 u32 tb_id) 740 { 741 struct mlxsw_sp_vr *vr; 742 int i; 743 744 tb_id = mlxsw_sp_fix_tb_id(tb_id); 745 746 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) { 747 vr = &mlxsw_sp->router->vrs[i]; 748 if (mlxsw_sp_vr_is_used(vr) && vr->tb_id == tb_id) 749 return vr; 750 } 751 return NULL; 752 } 753 754 int mlxsw_sp_router_tb_id_vr_id(struct mlxsw_sp *mlxsw_sp, u32 tb_id, 755 u16 *vr_id) 756 { 757 struct mlxsw_sp_vr *vr; 758 759 vr = mlxsw_sp_vr_find(mlxsw_sp, tb_id); 760 if (!vr) 761 return -ESRCH; 762 *vr_id = vr->id; 763 764 return 0; 765 } 766 767 static struct mlxsw_sp_fib *mlxsw_sp_vr_fib(const struct mlxsw_sp_vr *vr, 768 enum mlxsw_sp_l3proto proto) 769 { 770 switch (proto) { 771 case MLXSW_SP_L3_PROTO_IPV4: 772 return vr->fib4; 773 case MLXSW_SP_L3_PROTO_IPV6: 774 return vr->fib6; 775 } 776 return NULL; 777 } 778 779 static struct mlxsw_sp_vr *mlxsw_sp_vr_create(struct mlxsw_sp *mlxsw_sp, 780 u32 tb_id, 781 struct netlink_ext_ack *extack) 782 { 783 struct mlxsw_sp_mr_table *mr4_table, *mr6_table; 784 struct mlxsw_sp_fib *fib4; 785 struct mlxsw_sp_fib *fib6; 786 struct mlxsw_sp_vr *vr; 787 int err; 788 789 vr = mlxsw_sp_vr_find_unused(mlxsw_sp); 790 if (!vr) { 791 NL_SET_ERR_MSG_MOD(extack, "Exceeded number of supported virtual routers"); 792 return ERR_PTR(-EBUSY); 793 } 794 fib4 = mlxsw_sp_fib_create(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV4); 795 if (IS_ERR(fib4)) 796 return ERR_CAST(fib4); 797 fib6 = mlxsw_sp_fib_create(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV6); 798 if (IS_ERR(fib6)) { 799 err = PTR_ERR(fib6); 800 goto err_fib6_create; 801 } 802 mr4_table = mlxsw_sp_mr_table_create(mlxsw_sp, vr->id, 803 MLXSW_SP_L3_PROTO_IPV4); 804 if (IS_ERR(mr4_table)) { 805 err = PTR_ERR(mr4_table); 806 goto err_mr4_table_create; 807 } 808 mr6_table = mlxsw_sp_mr_table_create(mlxsw_sp, vr->id, 809 MLXSW_SP_L3_PROTO_IPV6); 810 if (IS_ERR(mr6_table)) { 811 err = PTR_ERR(mr6_table); 812 goto err_mr6_table_create; 813 } 814 815 vr->fib4 = fib4; 816 vr->fib6 = fib6; 817 vr->mr_table[MLXSW_SP_L3_PROTO_IPV4] = mr4_table; 818 vr->mr_table[MLXSW_SP_L3_PROTO_IPV6] = mr6_table; 819 vr->tb_id = tb_id; 820 return vr; 821 822 err_mr6_table_create: 823 mlxsw_sp_mr_table_destroy(mr4_table); 824 err_mr4_table_create: 825 mlxsw_sp_fib_destroy(mlxsw_sp, fib6); 826 err_fib6_create: 827 mlxsw_sp_fib_destroy(mlxsw_sp, fib4); 828 return ERR_PTR(err); 829 } 830 831 static void mlxsw_sp_vr_destroy(struct mlxsw_sp *mlxsw_sp, 832 struct mlxsw_sp_vr *vr) 833 { 834 mlxsw_sp_mr_table_destroy(vr->mr_table[MLXSW_SP_L3_PROTO_IPV6]); 835 vr->mr_table[MLXSW_SP_L3_PROTO_IPV6] = NULL; 836 mlxsw_sp_mr_table_destroy(vr->mr_table[MLXSW_SP_L3_PROTO_IPV4]); 837 vr->mr_table[MLXSW_SP_L3_PROTO_IPV4] = NULL; 838 mlxsw_sp_fib_destroy(mlxsw_sp, vr->fib6); 839 vr->fib6 = NULL; 840 mlxsw_sp_fib_destroy(mlxsw_sp, vr->fib4); 841 vr->fib4 = NULL; 842 } 843 844 static struct mlxsw_sp_vr *mlxsw_sp_vr_get(struct mlxsw_sp *mlxsw_sp, u32 tb_id, 845 struct netlink_ext_ack *extack) 846 { 847 struct mlxsw_sp_vr *vr; 848 849 tb_id = mlxsw_sp_fix_tb_id(tb_id); 850 vr = mlxsw_sp_vr_find(mlxsw_sp, tb_id); 851 if (!vr) 852 vr = mlxsw_sp_vr_create(mlxsw_sp, tb_id, extack); 853 return vr; 854 } 855 856 static void mlxsw_sp_vr_put(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp_vr *vr) 857 { 858 if (!vr->rif_count && list_empty(&vr->fib4->node_list) && 859 list_empty(&vr->fib6->node_list) && 860 mlxsw_sp_mr_table_empty(vr->mr_table[MLXSW_SP_L3_PROTO_IPV4]) && 861 mlxsw_sp_mr_table_empty(vr->mr_table[MLXSW_SP_L3_PROTO_IPV6])) 862 mlxsw_sp_vr_destroy(mlxsw_sp, vr); 863 } 864 865 static bool 866 mlxsw_sp_vr_lpm_tree_should_replace(struct mlxsw_sp_vr *vr, 867 enum mlxsw_sp_l3proto proto, u8 tree_id) 868 { 869 struct mlxsw_sp_fib *fib = mlxsw_sp_vr_fib(vr, proto); 870 871 if (!mlxsw_sp_vr_is_used(vr)) 872 return false; 873 if (fib->lpm_tree->id == tree_id) 874 return true; 875 return false; 876 } 877 878 static int mlxsw_sp_vr_lpm_tree_replace(struct mlxsw_sp *mlxsw_sp, 879 struct mlxsw_sp_fib *fib, 880 struct mlxsw_sp_lpm_tree *new_tree) 881 { 882 struct mlxsw_sp_lpm_tree *old_tree = fib->lpm_tree; 883 int err; 884 885 fib->lpm_tree = new_tree; 886 mlxsw_sp_lpm_tree_hold(new_tree); 887 err = mlxsw_sp_vr_lpm_tree_bind(mlxsw_sp, fib, new_tree->id); 888 if (err) 889 goto err_tree_bind; 890 mlxsw_sp_lpm_tree_put(mlxsw_sp, old_tree); 891 return 0; 892 893 err_tree_bind: 894 mlxsw_sp_lpm_tree_put(mlxsw_sp, new_tree); 895 fib->lpm_tree = old_tree; 896 return err; 897 } 898 899 static int mlxsw_sp_vrs_lpm_tree_replace(struct mlxsw_sp *mlxsw_sp, 900 struct mlxsw_sp_fib *fib, 901 struct mlxsw_sp_lpm_tree *new_tree) 902 { 903 enum mlxsw_sp_l3proto proto = fib->proto; 904 struct mlxsw_sp_lpm_tree *old_tree; 905 u8 old_id, new_id = new_tree->id; 906 struct mlxsw_sp_vr *vr; 907 int i, err; 908 909 old_tree = mlxsw_sp->router->lpm.proto_trees[proto]; 910 old_id = old_tree->id; 911 912 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) { 913 vr = &mlxsw_sp->router->vrs[i]; 914 if (!mlxsw_sp_vr_lpm_tree_should_replace(vr, proto, old_id)) 915 continue; 916 err = mlxsw_sp_vr_lpm_tree_replace(mlxsw_sp, 917 mlxsw_sp_vr_fib(vr, proto), 918 new_tree); 919 if (err) 920 goto err_tree_replace; 921 } 922 923 memcpy(new_tree->prefix_ref_count, old_tree->prefix_ref_count, 924 sizeof(new_tree->prefix_ref_count)); 925 mlxsw_sp->router->lpm.proto_trees[proto] = new_tree; 926 mlxsw_sp_lpm_tree_put(mlxsw_sp, old_tree); 927 928 return 0; 929 930 err_tree_replace: 931 for (i--; i >= 0; i--) { 932 if (!mlxsw_sp_vr_lpm_tree_should_replace(vr, proto, new_id)) 933 continue; 934 mlxsw_sp_vr_lpm_tree_replace(mlxsw_sp, 935 mlxsw_sp_vr_fib(vr, proto), 936 old_tree); 937 } 938 return err; 939 } 940 941 static int mlxsw_sp_vrs_init(struct mlxsw_sp *mlxsw_sp) 942 { 943 struct mlxsw_sp_vr *vr; 944 u64 max_vrs; 945 int i; 946 947 if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_VRS)) 948 return -EIO; 949 950 max_vrs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); 951 mlxsw_sp->router->vrs = kcalloc(max_vrs, sizeof(struct mlxsw_sp_vr), 952 GFP_KERNEL); 953 if (!mlxsw_sp->router->vrs) 954 return -ENOMEM; 955 956 for (i = 0; i < max_vrs; i++) { 957 vr = &mlxsw_sp->router->vrs[i]; 958 vr->id = i; 959 } 960 961 return 0; 962 } 963 964 static void mlxsw_sp_router_fib_flush(struct mlxsw_sp *mlxsw_sp); 965 966 static void mlxsw_sp_vrs_fini(struct mlxsw_sp *mlxsw_sp) 967 { 968 /* At this stage we're guaranteed not to have new incoming 969 * FIB notifications and the work queue is free from FIBs 970 * sitting on top of mlxsw netdevs. However, we can still 971 * have other FIBs queued. Flush the queue before flushing 972 * the device's tables. No need for locks, as we're the only 973 * writer. 974 */ 975 mlxsw_core_flush_owq(); 976 mlxsw_sp_router_fib_flush(mlxsw_sp); 977 kfree(mlxsw_sp->router->vrs); 978 } 979 980 static struct net_device * 981 __mlxsw_sp_ipip_netdev_ul_dev_get(const struct net_device *ol_dev) 982 { 983 struct ip_tunnel *tun = netdev_priv(ol_dev); 984 struct net *net = dev_net(ol_dev); 985 986 return __dev_get_by_index(net, tun->parms.link); 987 } 988 989 u32 mlxsw_sp_ipip_dev_ul_tb_id(const struct net_device *ol_dev) 990 { 991 struct net_device *d = __mlxsw_sp_ipip_netdev_ul_dev_get(ol_dev); 992 993 if (d) 994 return l3mdev_fib_table(d) ? : RT_TABLE_MAIN; 995 else 996 return l3mdev_fib_table(ol_dev) ? : RT_TABLE_MAIN; 997 } 998 999 static struct mlxsw_sp_rif * 1000 mlxsw_sp_rif_create(struct mlxsw_sp *mlxsw_sp, 1001 const struct mlxsw_sp_rif_params *params, 1002 struct netlink_ext_ack *extack); 1003 1004 static struct mlxsw_sp_rif_ipip_lb * 1005 mlxsw_sp_ipip_ol_ipip_lb_create(struct mlxsw_sp *mlxsw_sp, 1006 enum mlxsw_sp_ipip_type ipipt, 1007 struct net_device *ol_dev, 1008 struct netlink_ext_ack *extack) 1009 { 1010 struct mlxsw_sp_rif_params_ipip_lb lb_params; 1011 const struct mlxsw_sp_ipip_ops *ipip_ops; 1012 struct mlxsw_sp_rif *rif; 1013 1014 ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipipt]; 1015 lb_params = (struct mlxsw_sp_rif_params_ipip_lb) { 1016 .common.dev = ol_dev, 1017 .common.lag = false, 1018 .lb_config = ipip_ops->ol_loopback_config(mlxsw_sp, ol_dev), 1019 }; 1020 1021 rif = mlxsw_sp_rif_create(mlxsw_sp, &lb_params.common, extack); 1022 if (IS_ERR(rif)) 1023 return ERR_CAST(rif); 1024 return container_of(rif, struct mlxsw_sp_rif_ipip_lb, common); 1025 } 1026 1027 static struct mlxsw_sp_ipip_entry * 1028 mlxsw_sp_ipip_entry_alloc(struct mlxsw_sp *mlxsw_sp, 1029 enum mlxsw_sp_ipip_type ipipt, 1030 struct net_device *ol_dev) 1031 { 1032 const struct mlxsw_sp_ipip_ops *ipip_ops; 1033 struct mlxsw_sp_ipip_entry *ipip_entry; 1034 struct mlxsw_sp_ipip_entry *ret = NULL; 1035 1036 ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipipt]; 1037 ipip_entry = kzalloc(sizeof(*ipip_entry), GFP_KERNEL); 1038 if (!ipip_entry) 1039 return ERR_PTR(-ENOMEM); 1040 1041 ipip_entry->ol_lb = mlxsw_sp_ipip_ol_ipip_lb_create(mlxsw_sp, ipipt, 1042 ol_dev, NULL); 1043 if (IS_ERR(ipip_entry->ol_lb)) { 1044 ret = ERR_CAST(ipip_entry->ol_lb); 1045 goto err_ol_ipip_lb_create; 1046 } 1047 1048 ipip_entry->ipipt = ipipt; 1049 ipip_entry->ol_dev = ol_dev; 1050 1051 switch (ipip_ops->ul_proto) { 1052 case MLXSW_SP_L3_PROTO_IPV4: 1053 ipip_entry->parms4 = mlxsw_sp_ipip_netdev_parms4(ol_dev); 1054 break; 1055 case MLXSW_SP_L3_PROTO_IPV6: 1056 WARN_ON(1); 1057 break; 1058 } 1059 1060 return ipip_entry; 1061 1062 err_ol_ipip_lb_create: 1063 kfree(ipip_entry); 1064 return ret; 1065 } 1066 1067 static void 1068 mlxsw_sp_ipip_entry_dealloc(struct mlxsw_sp_ipip_entry *ipip_entry) 1069 { 1070 mlxsw_sp_rif_destroy(&ipip_entry->ol_lb->common); 1071 kfree(ipip_entry); 1072 } 1073 1074 static bool 1075 mlxsw_sp_ipip_entry_saddr_matches(struct mlxsw_sp *mlxsw_sp, 1076 const enum mlxsw_sp_l3proto ul_proto, 1077 union mlxsw_sp_l3addr saddr, 1078 u32 ul_tb_id, 1079 struct mlxsw_sp_ipip_entry *ipip_entry) 1080 { 1081 u32 tun_ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(ipip_entry->ol_dev); 1082 enum mlxsw_sp_ipip_type ipipt = ipip_entry->ipipt; 1083 union mlxsw_sp_l3addr tun_saddr; 1084 1085 if (mlxsw_sp->router->ipip_ops_arr[ipipt]->ul_proto != ul_proto) 1086 return false; 1087 1088 tun_saddr = mlxsw_sp_ipip_netdev_saddr(ul_proto, ipip_entry->ol_dev); 1089 return tun_ul_tb_id == ul_tb_id && 1090 mlxsw_sp_l3addr_eq(&tun_saddr, &saddr); 1091 } 1092 1093 static int 1094 mlxsw_sp_fib_entry_decap_init(struct mlxsw_sp *mlxsw_sp, 1095 struct mlxsw_sp_fib_entry *fib_entry, 1096 struct mlxsw_sp_ipip_entry *ipip_entry) 1097 { 1098 u32 tunnel_index; 1099 int err; 1100 1101 err = mlxsw_sp_kvdl_alloc(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 1102 1, &tunnel_index); 1103 if (err) 1104 return err; 1105 1106 ipip_entry->decap_fib_entry = fib_entry; 1107 fib_entry->decap.ipip_entry = ipip_entry; 1108 fib_entry->decap.tunnel_index = tunnel_index; 1109 return 0; 1110 } 1111 1112 static void mlxsw_sp_fib_entry_decap_fini(struct mlxsw_sp *mlxsw_sp, 1113 struct mlxsw_sp_fib_entry *fib_entry) 1114 { 1115 /* Unlink this node from the IPIP entry that it's the decap entry of. */ 1116 fib_entry->decap.ipip_entry->decap_fib_entry = NULL; 1117 fib_entry->decap.ipip_entry = NULL; 1118 mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 1119 1, fib_entry->decap.tunnel_index); 1120 } 1121 1122 static struct mlxsw_sp_fib_node * 1123 mlxsw_sp_fib_node_lookup(struct mlxsw_sp_fib *fib, const void *addr, 1124 size_t addr_len, unsigned char prefix_len); 1125 static int mlxsw_sp_fib_entry_update(struct mlxsw_sp *mlxsw_sp, 1126 struct mlxsw_sp_fib_entry *fib_entry); 1127 1128 static void 1129 mlxsw_sp_ipip_entry_demote_decap(struct mlxsw_sp *mlxsw_sp, 1130 struct mlxsw_sp_ipip_entry *ipip_entry) 1131 { 1132 struct mlxsw_sp_fib_entry *fib_entry = ipip_entry->decap_fib_entry; 1133 1134 mlxsw_sp_fib_entry_decap_fini(mlxsw_sp, fib_entry); 1135 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP; 1136 1137 mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry); 1138 } 1139 1140 static void 1141 mlxsw_sp_ipip_entry_promote_decap(struct mlxsw_sp *mlxsw_sp, 1142 struct mlxsw_sp_ipip_entry *ipip_entry, 1143 struct mlxsw_sp_fib_entry *decap_fib_entry) 1144 { 1145 if (mlxsw_sp_fib_entry_decap_init(mlxsw_sp, decap_fib_entry, 1146 ipip_entry)) 1147 return; 1148 decap_fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP; 1149 1150 if (mlxsw_sp_fib_entry_update(mlxsw_sp, decap_fib_entry)) 1151 mlxsw_sp_ipip_entry_demote_decap(mlxsw_sp, ipip_entry); 1152 } 1153 1154 static struct mlxsw_sp_fib_entry * 1155 mlxsw_sp_router_ip2me_fib_entry_find(struct mlxsw_sp *mlxsw_sp, u32 tb_id, 1156 enum mlxsw_sp_l3proto proto, 1157 const union mlxsw_sp_l3addr *addr, 1158 enum mlxsw_sp_fib_entry_type type) 1159 { 1160 struct mlxsw_sp_fib_entry *fib_entry; 1161 struct mlxsw_sp_fib_node *fib_node; 1162 unsigned char addr_prefix_len; 1163 struct mlxsw_sp_fib *fib; 1164 struct mlxsw_sp_vr *vr; 1165 const void *addrp; 1166 size_t addr_len; 1167 u32 addr4; 1168 1169 vr = mlxsw_sp_vr_find(mlxsw_sp, tb_id); 1170 if (!vr) 1171 return NULL; 1172 fib = mlxsw_sp_vr_fib(vr, proto); 1173 1174 switch (proto) { 1175 case MLXSW_SP_L3_PROTO_IPV4: 1176 addr4 = be32_to_cpu(addr->addr4); 1177 addrp = &addr4; 1178 addr_len = 4; 1179 addr_prefix_len = 32; 1180 break; 1181 case MLXSW_SP_L3_PROTO_IPV6: /* fall through */ 1182 default: 1183 WARN_ON(1); 1184 return NULL; 1185 } 1186 1187 fib_node = mlxsw_sp_fib_node_lookup(fib, addrp, addr_len, 1188 addr_prefix_len); 1189 if (!fib_node || list_empty(&fib_node->entry_list)) 1190 return NULL; 1191 1192 fib_entry = list_first_entry(&fib_node->entry_list, 1193 struct mlxsw_sp_fib_entry, list); 1194 if (fib_entry->type != type) 1195 return NULL; 1196 1197 return fib_entry; 1198 } 1199 1200 /* Given an IPIP entry, find the corresponding decap route. */ 1201 static struct mlxsw_sp_fib_entry * 1202 mlxsw_sp_ipip_entry_find_decap(struct mlxsw_sp *mlxsw_sp, 1203 struct mlxsw_sp_ipip_entry *ipip_entry) 1204 { 1205 static struct mlxsw_sp_fib_node *fib_node; 1206 const struct mlxsw_sp_ipip_ops *ipip_ops; 1207 struct mlxsw_sp_fib_entry *fib_entry; 1208 unsigned char saddr_prefix_len; 1209 union mlxsw_sp_l3addr saddr; 1210 struct mlxsw_sp_fib *ul_fib; 1211 struct mlxsw_sp_vr *ul_vr; 1212 const void *saddrp; 1213 size_t saddr_len; 1214 u32 ul_tb_id; 1215 u32 saddr4; 1216 1217 ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt]; 1218 1219 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(ipip_entry->ol_dev); 1220 ul_vr = mlxsw_sp_vr_find(mlxsw_sp, ul_tb_id); 1221 if (!ul_vr) 1222 return NULL; 1223 1224 ul_fib = mlxsw_sp_vr_fib(ul_vr, ipip_ops->ul_proto); 1225 saddr = mlxsw_sp_ipip_netdev_saddr(ipip_ops->ul_proto, 1226 ipip_entry->ol_dev); 1227 1228 switch (ipip_ops->ul_proto) { 1229 case MLXSW_SP_L3_PROTO_IPV4: 1230 saddr4 = be32_to_cpu(saddr.addr4); 1231 saddrp = &saddr4; 1232 saddr_len = 4; 1233 saddr_prefix_len = 32; 1234 break; 1235 case MLXSW_SP_L3_PROTO_IPV6: 1236 WARN_ON(1); 1237 return NULL; 1238 } 1239 1240 fib_node = mlxsw_sp_fib_node_lookup(ul_fib, saddrp, saddr_len, 1241 saddr_prefix_len); 1242 if (!fib_node || list_empty(&fib_node->entry_list)) 1243 return NULL; 1244 1245 fib_entry = list_first_entry(&fib_node->entry_list, 1246 struct mlxsw_sp_fib_entry, list); 1247 if (fib_entry->type != MLXSW_SP_FIB_ENTRY_TYPE_TRAP) 1248 return NULL; 1249 1250 return fib_entry; 1251 } 1252 1253 static struct mlxsw_sp_ipip_entry * 1254 mlxsw_sp_ipip_entry_create(struct mlxsw_sp *mlxsw_sp, 1255 enum mlxsw_sp_ipip_type ipipt, 1256 struct net_device *ol_dev) 1257 { 1258 struct mlxsw_sp_ipip_entry *ipip_entry; 1259 1260 ipip_entry = mlxsw_sp_ipip_entry_alloc(mlxsw_sp, ipipt, ol_dev); 1261 if (IS_ERR(ipip_entry)) 1262 return ipip_entry; 1263 1264 list_add_tail(&ipip_entry->ipip_list_node, 1265 &mlxsw_sp->router->ipip_list); 1266 1267 return ipip_entry; 1268 } 1269 1270 static void 1271 mlxsw_sp_ipip_entry_destroy(struct mlxsw_sp *mlxsw_sp, 1272 struct mlxsw_sp_ipip_entry *ipip_entry) 1273 { 1274 list_del(&ipip_entry->ipip_list_node); 1275 mlxsw_sp_ipip_entry_dealloc(ipip_entry); 1276 } 1277 1278 static bool 1279 mlxsw_sp_ipip_entry_matches_decap(struct mlxsw_sp *mlxsw_sp, 1280 const struct net_device *ul_dev, 1281 enum mlxsw_sp_l3proto ul_proto, 1282 union mlxsw_sp_l3addr ul_dip, 1283 struct mlxsw_sp_ipip_entry *ipip_entry) 1284 { 1285 u32 ul_tb_id = l3mdev_fib_table(ul_dev) ? : RT_TABLE_MAIN; 1286 enum mlxsw_sp_ipip_type ipipt = ipip_entry->ipipt; 1287 1288 if (mlxsw_sp->router->ipip_ops_arr[ipipt]->ul_proto != ul_proto) 1289 return false; 1290 1291 return mlxsw_sp_ipip_entry_saddr_matches(mlxsw_sp, ul_proto, ul_dip, 1292 ul_tb_id, ipip_entry); 1293 } 1294 1295 /* Given decap parameters, find the corresponding IPIP entry. */ 1296 static struct mlxsw_sp_ipip_entry * 1297 mlxsw_sp_ipip_entry_find_by_decap(struct mlxsw_sp *mlxsw_sp, 1298 const struct net_device *ul_dev, 1299 enum mlxsw_sp_l3proto ul_proto, 1300 union mlxsw_sp_l3addr ul_dip) 1301 { 1302 struct mlxsw_sp_ipip_entry *ipip_entry; 1303 1304 list_for_each_entry(ipip_entry, &mlxsw_sp->router->ipip_list, 1305 ipip_list_node) 1306 if (mlxsw_sp_ipip_entry_matches_decap(mlxsw_sp, ul_dev, 1307 ul_proto, ul_dip, 1308 ipip_entry)) 1309 return ipip_entry; 1310 1311 return NULL; 1312 } 1313 1314 static bool mlxsw_sp_netdev_ipip_type(const struct mlxsw_sp *mlxsw_sp, 1315 const struct net_device *dev, 1316 enum mlxsw_sp_ipip_type *p_type) 1317 { 1318 struct mlxsw_sp_router *router = mlxsw_sp->router; 1319 const struct mlxsw_sp_ipip_ops *ipip_ops; 1320 enum mlxsw_sp_ipip_type ipipt; 1321 1322 for (ipipt = 0; ipipt < MLXSW_SP_IPIP_TYPE_MAX; ++ipipt) { 1323 ipip_ops = router->ipip_ops_arr[ipipt]; 1324 if (dev->type == ipip_ops->dev_type) { 1325 if (p_type) 1326 *p_type = ipipt; 1327 return true; 1328 } 1329 } 1330 return false; 1331 } 1332 1333 bool mlxsw_sp_netdev_is_ipip_ol(const struct mlxsw_sp *mlxsw_sp, 1334 const struct net_device *dev) 1335 { 1336 return mlxsw_sp_netdev_ipip_type(mlxsw_sp, dev, NULL); 1337 } 1338 1339 static struct mlxsw_sp_ipip_entry * 1340 mlxsw_sp_ipip_entry_find_by_ol_dev(struct mlxsw_sp *mlxsw_sp, 1341 const struct net_device *ol_dev) 1342 { 1343 struct mlxsw_sp_ipip_entry *ipip_entry; 1344 1345 list_for_each_entry(ipip_entry, &mlxsw_sp->router->ipip_list, 1346 ipip_list_node) 1347 if (ipip_entry->ol_dev == ol_dev) 1348 return ipip_entry; 1349 1350 return NULL; 1351 } 1352 1353 static struct mlxsw_sp_ipip_entry * 1354 mlxsw_sp_ipip_entry_find_by_ul_dev(const struct mlxsw_sp *mlxsw_sp, 1355 const struct net_device *ul_dev, 1356 struct mlxsw_sp_ipip_entry *start) 1357 { 1358 struct mlxsw_sp_ipip_entry *ipip_entry; 1359 1360 ipip_entry = list_prepare_entry(start, &mlxsw_sp->router->ipip_list, 1361 ipip_list_node); 1362 list_for_each_entry_continue(ipip_entry, &mlxsw_sp->router->ipip_list, 1363 ipip_list_node) { 1364 struct net_device *ipip_ul_dev = 1365 __mlxsw_sp_ipip_netdev_ul_dev_get(ipip_entry->ol_dev); 1366 1367 if (ipip_ul_dev == ul_dev) 1368 return ipip_entry; 1369 } 1370 1371 return NULL; 1372 } 1373 1374 bool mlxsw_sp_netdev_is_ipip_ul(const struct mlxsw_sp *mlxsw_sp, 1375 const struct net_device *dev) 1376 { 1377 return mlxsw_sp_ipip_entry_find_by_ul_dev(mlxsw_sp, dev, NULL); 1378 } 1379 1380 static bool mlxsw_sp_netdevice_ipip_can_offload(struct mlxsw_sp *mlxsw_sp, 1381 const struct net_device *ol_dev, 1382 enum mlxsw_sp_ipip_type ipipt) 1383 { 1384 const struct mlxsw_sp_ipip_ops *ops 1385 = mlxsw_sp->router->ipip_ops_arr[ipipt]; 1386 1387 /* For deciding whether decap should be offloaded, we don't care about 1388 * overlay protocol, so ask whether either one is supported. 1389 */ 1390 return ops->can_offload(mlxsw_sp, ol_dev, MLXSW_SP_L3_PROTO_IPV4) || 1391 ops->can_offload(mlxsw_sp, ol_dev, MLXSW_SP_L3_PROTO_IPV6); 1392 } 1393 1394 static int mlxsw_sp_netdevice_ipip_ol_reg_event(struct mlxsw_sp *mlxsw_sp, 1395 struct net_device *ol_dev) 1396 { 1397 struct mlxsw_sp_ipip_entry *ipip_entry; 1398 enum mlxsw_sp_l3proto ul_proto; 1399 enum mlxsw_sp_ipip_type ipipt; 1400 union mlxsw_sp_l3addr saddr; 1401 u32 ul_tb_id; 1402 1403 mlxsw_sp_netdev_ipip_type(mlxsw_sp, ol_dev, &ipipt); 1404 if (mlxsw_sp_netdevice_ipip_can_offload(mlxsw_sp, ol_dev, ipipt)) { 1405 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(ol_dev); 1406 ul_proto = mlxsw_sp->router->ipip_ops_arr[ipipt]->ul_proto; 1407 saddr = mlxsw_sp_ipip_netdev_saddr(ul_proto, ol_dev); 1408 if (!mlxsw_sp_ipip_demote_tunnel_by_saddr(mlxsw_sp, ul_proto, 1409 saddr, ul_tb_id, 1410 NULL)) { 1411 ipip_entry = mlxsw_sp_ipip_entry_create(mlxsw_sp, ipipt, 1412 ol_dev); 1413 if (IS_ERR(ipip_entry)) 1414 return PTR_ERR(ipip_entry); 1415 } 1416 } 1417 1418 return 0; 1419 } 1420 1421 static void mlxsw_sp_netdevice_ipip_ol_unreg_event(struct mlxsw_sp *mlxsw_sp, 1422 struct net_device *ol_dev) 1423 { 1424 struct mlxsw_sp_ipip_entry *ipip_entry; 1425 1426 ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev); 1427 if (ipip_entry) 1428 mlxsw_sp_ipip_entry_destroy(mlxsw_sp, ipip_entry); 1429 } 1430 1431 static void 1432 mlxsw_sp_ipip_entry_ol_up_event(struct mlxsw_sp *mlxsw_sp, 1433 struct mlxsw_sp_ipip_entry *ipip_entry) 1434 { 1435 struct mlxsw_sp_fib_entry *decap_fib_entry; 1436 1437 decap_fib_entry = mlxsw_sp_ipip_entry_find_decap(mlxsw_sp, ipip_entry); 1438 if (decap_fib_entry) 1439 mlxsw_sp_ipip_entry_promote_decap(mlxsw_sp, ipip_entry, 1440 decap_fib_entry); 1441 } 1442 1443 static int 1444 mlxsw_sp_rif_ipip_lb_op(struct mlxsw_sp_rif_ipip_lb *lb_rif, u16 ul_vr_id, 1445 u16 ul_rif_id, bool enable) 1446 { 1447 struct mlxsw_sp_rif_ipip_lb_config lb_cf = lb_rif->lb_config; 1448 struct mlxsw_sp_rif *rif = &lb_rif->common; 1449 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 1450 char ritr_pl[MLXSW_REG_RITR_LEN]; 1451 u32 saddr4; 1452 1453 switch (lb_cf.ul_protocol) { 1454 case MLXSW_SP_L3_PROTO_IPV4: 1455 saddr4 = be32_to_cpu(lb_cf.saddr.addr4); 1456 mlxsw_reg_ritr_pack(ritr_pl, enable, MLXSW_REG_RITR_LOOPBACK_IF, 1457 rif->rif_index, rif->vr_id, rif->dev->mtu); 1458 mlxsw_reg_ritr_loopback_ipip4_pack(ritr_pl, lb_cf.lb_ipipt, 1459 MLXSW_REG_RITR_LOOPBACK_IPIP_OPTIONS_GRE_KEY_PRESET, 1460 ul_vr_id, ul_rif_id, saddr4, lb_cf.okey); 1461 break; 1462 1463 case MLXSW_SP_L3_PROTO_IPV6: 1464 return -EAFNOSUPPORT; 1465 } 1466 1467 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 1468 } 1469 1470 static int mlxsw_sp_netdevice_ipip_ol_update_mtu(struct mlxsw_sp *mlxsw_sp, 1471 struct net_device *ol_dev) 1472 { 1473 struct mlxsw_sp_ipip_entry *ipip_entry; 1474 struct mlxsw_sp_rif_ipip_lb *lb_rif; 1475 int err = 0; 1476 1477 ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev); 1478 if (ipip_entry) { 1479 lb_rif = ipip_entry->ol_lb; 1480 err = mlxsw_sp_rif_ipip_lb_op(lb_rif, lb_rif->ul_vr_id, 1481 lb_rif->ul_rif_id, true); 1482 if (err) 1483 goto out; 1484 lb_rif->common.mtu = ol_dev->mtu; 1485 } 1486 1487 out: 1488 return err; 1489 } 1490 1491 static void mlxsw_sp_netdevice_ipip_ol_up_event(struct mlxsw_sp *mlxsw_sp, 1492 struct net_device *ol_dev) 1493 { 1494 struct mlxsw_sp_ipip_entry *ipip_entry; 1495 1496 ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev); 1497 if (ipip_entry) 1498 mlxsw_sp_ipip_entry_ol_up_event(mlxsw_sp, ipip_entry); 1499 } 1500 1501 static void 1502 mlxsw_sp_ipip_entry_ol_down_event(struct mlxsw_sp *mlxsw_sp, 1503 struct mlxsw_sp_ipip_entry *ipip_entry) 1504 { 1505 if (ipip_entry->decap_fib_entry) 1506 mlxsw_sp_ipip_entry_demote_decap(mlxsw_sp, ipip_entry); 1507 } 1508 1509 static void mlxsw_sp_netdevice_ipip_ol_down_event(struct mlxsw_sp *mlxsw_sp, 1510 struct net_device *ol_dev) 1511 { 1512 struct mlxsw_sp_ipip_entry *ipip_entry; 1513 1514 ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev); 1515 if (ipip_entry) 1516 mlxsw_sp_ipip_entry_ol_down_event(mlxsw_sp, ipip_entry); 1517 } 1518 1519 static void mlxsw_sp_nexthop_rif_migrate(struct mlxsw_sp *mlxsw_sp, 1520 struct mlxsw_sp_rif *old_rif, 1521 struct mlxsw_sp_rif *new_rif); 1522 static int 1523 mlxsw_sp_ipip_entry_ol_lb_update(struct mlxsw_sp *mlxsw_sp, 1524 struct mlxsw_sp_ipip_entry *ipip_entry, 1525 bool keep_encap, 1526 struct netlink_ext_ack *extack) 1527 { 1528 struct mlxsw_sp_rif_ipip_lb *old_lb_rif = ipip_entry->ol_lb; 1529 struct mlxsw_sp_rif_ipip_lb *new_lb_rif; 1530 1531 new_lb_rif = mlxsw_sp_ipip_ol_ipip_lb_create(mlxsw_sp, 1532 ipip_entry->ipipt, 1533 ipip_entry->ol_dev, 1534 extack); 1535 if (IS_ERR(new_lb_rif)) 1536 return PTR_ERR(new_lb_rif); 1537 ipip_entry->ol_lb = new_lb_rif; 1538 1539 if (keep_encap) 1540 mlxsw_sp_nexthop_rif_migrate(mlxsw_sp, &old_lb_rif->common, 1541 &new_lb_rif->common); 1542 1543 mlxsw_sp_rif_destroy(&old_lb_rif->common); 1544 1545 return 0; 1546 } 1547 1548 static void mlxsw_sp_nexthop_rif_update(struct mlxsw_sp *mlxsw_sp, 1549 struct mlxsw_sp_rif *rif); 1550 1551 /** 1552 * Update the offload related to an IPIP entry. This always updates decap, and 1553 * in addition to that it also: 1554 * @recreate_loopback: recreates the associated loopback RIF 1555 * @keep_encap: updates next hops that use the tunnel netdevice. This is only 1556 * relevant when recreate_loopback is true. 1557 * @update_nexthops: updates next hops, keeping the current loopback RIF. This 1558 * is only relevant when recreate_loopback is false. 1559 */ 1560 int __mlxsw_sp_ipip_entry_update_tunnel(struct mlxsw_sp *mlxsw_sp, 1561 struct mlxsw_sp_ipip_entry *ipip_entry, 1562 bool recreate_loopback, 1563 bool keep_encap, 1564 bool update_nexthops, 1565 struct netlink_ext_ack *extack) 1566 { 1567 int err; 1568 1569 /* RIFs can't be edited, so to update loopback, we need to destroy and 1570 * recreate it. That creates a window of opportunity where RALUE and 1571 * RATR registers end up referencing a RIF that's already gone. RATRs 1572 * are handled in mlxsw_sp_ipip_entry_ol_lb_update(), and to take care 1573 * of RALUE, demote the decap route back. 1574 */ 1575 if (ipip_entry->decap_fib_entry) 1576 mlxsw_sp_ipip_entry_demote_decap(mlxsw_sp, ipip_entry); 1577 1578 if (recreate_loopback) { 1579 err = mlxsw_sp_ipip_entry_ol_lb_update(mlxsw_sp, ipip_entry, 1580 keep_encap, extack); 1581 if (err) 1582 return err; 1583 } else if (update_nexthops) { 1584 mlxsw_sp_nexthop_rif_update(mlxsw_sp, 1585 &ipip_entry->ol_lb->common); 1586 } 1587 1588 if (ipip_entry->ol_dev->flags & IFF_UP) 1589 mlxsw_sp_ipip_entry_ol_up_event(mlxsw_sp, ipip_entry); 1590 1591 return 0; 1592 } 1593 1594 static int mlxsw_sp_netdevice_ipip_ol_vrf_event(struct mlxsw_sp *mlxsw_sp, 1595 struct net_device *ol_dev, 1596 struct netlink_ext_ack *extack) 1597 { 1598 struct mlxsw_sp_ipip_entry *ipip_entry = 1599 mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev); 1600 enum mlxsw_sp_l3proto ul_proto; 1601 union mlxsw_sp_l3addr saddr; 1602 u32 ul_tb_id; 1603 1604 if (!ipip_entry) 1605 return 0; 1606 1607 /* For flat configuration cases, moving overlay to a different VRF might 1608 * cause local address conflict, and the conflicting tunnels need to be 1609 * demoted. 1610 */ 1611 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(ol_dev); 1612 ul_proto = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt]->ul_proto; 1613 saddr = mlxsw_sp_ipip_netdev_saddr(ul_proto, ol_dev); 1614 if (mlxsw_sp_ipip_demote_tunnel_by_saddr(mlxsw_sp, ul_proto, 1615 saddr, ul_tb_id, 1616 ipip_entry)) { 1617 mlxsw_sp_ipip_entry_demote_tunnel(mlxsw_sp, ipip_entry); 1618 return 0; 1619 } 1620 1621 return __mlxsw_sp_ipip_entry_update_tunnel(mlxsw_sp, ipip_entry, 1622 true, false, false, extack); 1623 } 1624 1625 static int 1626 mlxsw_sp_netdevice_ipip_ul_vrf_event(struct mlxsw_sp *mlxsw_sp, 1627 struct mlxsw_sp_ipip_entry *ipip_entry, 1628 struct net_device *ul_dev, 1629 struct netlink_ext_ack *extack) 1630 { 1631 return __mlxsw_sp_ipip_entry_update_tunnel(mlxsw_sp, ipip_entry, 1632 true, true, false, extack); 1633 } 1634 1635 static int 1636 mlxsw_sp_netdevice_ipip_ul_up_event(struct mlxsw_sp *mlxsw_sp, 1637 struct mlxsw_sp_ipip_entry *ipip_entry, 1638 struct net_device *ul_dev) 1639 { 1640 return __mlxsw_sp_ipip_entry_update_tunnel(mlxsw_sp, ipip_entry, 1641 false, false, true, NULL); 1642 } 1643 1644 static int 1645 mlxsw_sp_netdevice_ipip_ul_down_event(struct mlxsw_sp *mlxsw_sp, 1646 struct mlxsw_sp_ipip_entry *ipip_entry, 1647 struct net_device *ul_dev) 1648 { 1649 /* A down underlay device causes encapsulated packets to not be 1650 * forwarded, but decap still works. So refresh next hops without 1651 * touching anything else. 1652 */ 1653 return __mlxsw_sp_ipip_entry_update_tunnel(mlxsw_sp, ipip_entry, 1654 false, false, true, NULL); 1655 } 1656 1657 static int 1658 mlxsw_sp_netdevice_ipip_ol_change_event(struct mlxsw_sp *mlxsw_sp, 1659 struct net_device *ol_dev, 1660 struct netlink_ext_ack *extack) 1661 { 1662 const struct mlxsw_sp_ipip_ops *ipip_ops; 1663 struct mlxsw_sp_ipip_entry *ipip_entry; 1664 int err; 1665 1666 ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev); 1667 if (!ipip_entry) 1668 /* A change might make a tunnel eligible for offloading, but 1669 * that is currently not implemented. What falls to slow path 1670 * stays there. 1671 */ 1672 return 0; 1673 1674 /* A change might make a tunnel not eligible for offloading. */ 1675 if (!mlxsw_sp_netdevice_ipip_can_offload(mlxsw_sp, ol_dev, 1676 ipip_entry->ipipt)) { 1677 mlxsw_sp_ipip_entry_demote_tunnel(mlxsw_sp, ipip_entry); 1678 return 0; 1679 } 1680 1681 ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt]; 1682 err = ipip_ops->ol_netdev_change(mlxsw_sp, ipip_entry, extack); 1683 return err; 1684 } 1685 1686 void mlxsw_sp_ipip_entry_demote_tunnel(struct mlxsw_sp *mlxsw_sp, 1687 struct mlxsw_sp_ipip_entry *ipip_entry) 1688 { 1689 struct net_device *ol_dev = ipip_entry->ol_dev; 1690 1691 if (ol_dev->flags & IFF_UP) 1692 mlxsw_sp_ipip_entry_ol_down_event(mlxsw_sp, ipip_entry); 1693 mlxsw_sp_ipip_entry_destroy(mlxsw_sp, ipip_entry); 1694 } 1695 1696 /* The configuration where several tunnels have the same local address in the 1697 * same underlay table needs special treatment in the HW. That is currently not 1698 * implemented in the driver. This function finds and demotes the first tunnel 1699 * with a given source address, except the one passed in in the argument 1700 * `except'. 1701 */ 1702 bool 1703 mlxsw_sp_ipip_demote_tunnel_by_saddr(struct mlxsw_sp *mlxsw_sp, 1704 enum mlxsw_sp_l3proto ul_proto, 1705 union mlxsw_sp_l3addr saddr, 1706 u32 ul_tb_id, 1707 const struct mlxsw_sp_ipip_entry *except) 1708 { 1709 struct mlxsw_sp_ipip_entry *ipip_entry, *tmp; 1710 1711 list_for_each_entry_safe(ipip_entry, tmp, &mlxsw_sp->router->ipip_list, 1712 ipip_list_node) { 1713 if (ipip_entry != except && 1714 mlxsw_sp_ipip_entry_saddr_matches(mlxsw_sp, ul_proto, saddr, 1715 ul_tb_id, ipip_entry)) { 1716 mlxsw_sp_ipip_entry_demote_tunnel(mlxsw_sp, ipip_entry); 1717 return true; 1718 } 1719 } 1720 1721 return false; 1722 } 1723 1724 static void mlxsw_sp_ipip_demote_tunnel_by_ul_netdev(struct mlxsw_sp *mlxsw_sp, 1725 struct net_device *ul_dev) 1726 { 1727 struct mlxsw_sp_ipip_entry *ipip_entry, *tmp; 1728 1729 list_for_each_entry_safe(ipip_entry, tmp, &mlxsw_sp->router->ipip_list, 1730 ipip_list_node) { 1731 struct net_device *ipip_ul_dev = 1732 __mlxsw_sp_ipip_netdev_ul_dev_get(ipip_entry->ol_dev); 1733 1734 if (ipip_ul_dev == ul_dev) 1735 mlxsw_sp_ipip_entry_demote_tunnel(mlxsw_sp, ipip_entry); 1736 } 1737 } 1738 1739 int mlxsw_sp_netdevice_ipip_ol_event(struct mlxsw_sp *mlxsw_sp, 1740 struct net_device *ol_dev, 1741 unsigned long event, 1742 struct netdev_notifier_info *info) 1743 { 1744 struct netdev_notifier_changeupper_info *chup; 1745 struct netlink_ext_ack *extack; 1746 1747 switch (event) { 1748 case NETDEV_REGISTER: 1749 return mlxsw_sp_netdevice_ipip_ol_reg_event(mlxsw_sp, ol_dev); 1750 case NETDEV_UNREGISTER: 1751 mlxsw_sp_netdevice_ipip_ol_unreg_event(mlxsw_sp, ol_dev); 1752 return 0; 1753 case NETDEV_UP: 1754 mlxsw_sp_netdevice_ipip_ol_up_event(mlxsw_sp, ol_dev); 1755 return 0; 1756 case NETDEV_DOWN: 1757 mlxsw_sp_netdevice_ipip_ol_down_event(mlxsw_sp, ol_dev); 1758 return 0; 1759 case NETDEV_CHANGEUPPER: 1760 chup = container_of(info, typeof(*chup), info); 1761 extack = info->extack; 1762 if (netif_is_l3_master(chup->upper_dev)) 1763 return mlxsw_sp_netdevice_ipip_ol_vrf_event(mlxsw_sp, 1764 ol_dev, 1765 extack); 1766 return 0; 1767 case NETDEV_CHANGE: 1768 extack = info->extack; 1769 return mlxsw_sp_netdevice_ipip_ol_change_event(mlxsw_sp, 1770 ol_dev, extack); 1771 case NETDEV_CHANGEMTU: 1772 return mlxsw_sp_netdevice_ipip_ol_update_mtu(mlxsw_sp, ol_dev); 1773 } 1774 return 0; 1775 } 1776 1777 static int 1778 __mlxsw_sp_netdevice_ipip_ul_event(struct mlxsw_sp *mlxsw_sp, 1779 struct mlxsw_sp_ipip_entry *ipip_entry, 1780 struct net_device *ul_dev, 1781 unsigned long event, 1782 struct netdev_notifier_info *info) 1783 { 1784 struct netdev_notifier_changeupper_info *chup; 1785 struct netlink_ext_ack *extack; 1786 1787 switch (event) { 1788 case NETDEV_CHANGEUPPER: 1789 chup = container_of(info, typeof(*chup), info); 1790 extack = info->extack; 1791 if (netif_is_l3_master(chup->upper_dev)) 1792 return mlxsw_sp_netdevice_ipip_ul_vrf_event(mlxsw_sp, 1793 ipip_entry, 1794 ul_dev, 1795 extack); 1796 break; 1797 1798 case NETDEV_UP: 1799 return mlxsw_sp_netdevice_ipip_ul_up_event(mlxsw_sp, ipip_entry, 1800 ul_dev); 1801 case NETDEV_DOWN: 1802 return mlxsw_sp_netdevice_ipip_ul_down_event(mlxsw_sp, 1803 ipip_entry, 1804 ul_dev); 1805 } 1806 return 0; 1807 } 1808 1809 int 1810 mlxsw_sp_netdevice_ipip_ul_event(struct mlxsw_sp *mlxsw_sp, 1811 struct net_device *ul_dev, 1812 unsigned long event, 1813 struct netdev_notifier_info *info) 1814 { 1815 struct mlxsw_sp_ipip_entry *ipip_entry = NULL; 1816 int err; 1817 1818 while ((ipip_entry = mlxsw_sp_ipip_entry_find_by_ul_dev(mlxsw_sp, 1819 ul_dev, 1820 ipip_entry))) { 1821 err = __mlxsw_sp_netdevice_ipip_ul_event(mlxsw_sp, ipip_entry, 1822 ul_dev, event, info); 1823 if (err) { 1824 mlxsw_sp_ipip_demote_tunnel_by_ul_netdev(mlxsw_sp, 1825 ul_dev); 1826 return err; 1827 } 1828 } 1829 1830 return 0; 1831 } 1832 1833 int mlxsw_sp_router_nve_promote_decap(struct mlxsw_sp *mlxsw_sp, u32 ul_tb_id, 1834 enum mlxsw_sp_l3proto ul_proto, 1835 const union mlxsw_sp_l3addr *ul_sip, 1836 u32 tunnel_index) 1837 { 1838 enum mlxsw_sp_fib_entry_type type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP; 1839 struct mlxsw_sp_fib_entry *fib_entry; 1840 int err; 1841 1842 /* It is valid to create a tunnel with a local IP and only later 1843 * assign this IP address to a local interface 1844 */ 1845 fib_entry = mlxsw_sp_router_ip2me_fib_entry_find(mlxsw_sp, ul_tb_id, 1846 ul_proto, ul_sip, 1847 type); 1848 if (!fib_entry) 1849 return 0; 1850 1851 fib_entry->decap.tunnel_index = tunnel_index; 1852 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP; 1853 1854 err = mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry); 1855 if (err) 1856 goto err_fib_entry_update; 1857 1858 return 0; 1859 1860 err_fib_entry_update: 1861 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP; 1862 mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry); 1863 return err; 1864 } 1865 1866 void mlxsw_sp_router_nve_demote_decap(struct mlxsw_sp *mlxsw_sp, u32 ul_tb_id, 1867 enum mlxsw_sp_l3proto ul_proto, 1868 const union mlxsw_sp_l3addr *ul_sip) 1869 { 1870 enum mlxsw_sp_fib_entry_type type = MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP; 1871 struct mlxsw_sp_fib_entry *fib_entry; 1872 1873 fib_entry = mlxsw_sp_router_ip2me_fib_entry_find(mlxsw_sp, ul_tb_id, 1874 ul_proto, ul_sip, 1875 type); 1876 if (!fib_entry) 1877 return; 1878 1879 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP; 1880 mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry); 1881 } 1882 1883 struct mlxsw_sp_neigh_key { 1884 struct neighbour *n; 1885 }; 1886 1887 struct mlxsw_sp_neigh_entry { 1888 struct list_head rif_list_node; 1889 struct rhash_head ht_node; 1890 struct mlxsw_sp_neigh_key key; 1891 u16 rif; 1892 bool connected; 1893 unsigned char ha[ETH_ALEN]; 1894 struct list_head nexthop_list; /* list of nexthops using 1895 * this neigh entry 1896 */ 1897 struct list_head nexthop_neighs_list_node; 1898 unsigned int counter_index; 1899 bool counter_valid; 1900 }; 1901 1902 static const struct rhashtable_params mlxsw_sp_neigh_ht_params = { 1903 .key_offset = offsetof(struct mlxsw_sp_neigh_entry, key), 1904 .head_offset = offsetof(struct mlxsw_sp_neigh_entry, ht_node), 1905 .key_len = sizeof(struct mlxsw_sp_neigh_key), 1906 }; 1907 1908 struct mlxsw_sp_neigh_entry * 1909 mlxsw_sp_rif_neigh_next(struct mlxsw_sp_rif *rif, 1910 struct mlxsw_sp_neigh_entry *neigh_entry) 1911 { 1912 if (!neigh_entry) { 1913 if (list_empty(&rif->neigh_list)) 1914 return NULL; 1915 else 1916 return list_first_entry(&rif->neigh_list, 1917 typeof(*neigh_entry), 1918 rif_list_node); 1919 } 1920 if (list_is_last(&neigh_entry->rif_list_node, &rif->neigh_list)) 1921 return NULL; 1922 return list_next_entry(neigh_entry, rif_list_node); 1923 } 1924 1925 int mlxsw_sp_neigh_entry_type(struct mlxsw_sp_neigh_entry *neigh_entry) 1926 { 1927 return neigh_entry->key.n->tbl->family; 1928 } 1929 1930 unsigned char * 1931 mlxsw_sp_neigh_entry_ha(struct mlxsw_sp_neigh_entry *neigh_entry) 1932 { 1933 return neigh_entry->ha; 1934 } 1935 1936 u32 mlxsw_sp_neigh4_entry_dip(struct mlxsw_sp_neigh_entry *neigh_entry) 1937 { 1938 struct neighbour *n; 1939 1940 n = neigh_entry->key.n; 1941 return ntohl(*((__be32 *) n->primary_key)); 1942 } 1943 1944 struct in6_addr * 1945 mlxsw_sp_neigh6_entry_dip(struct mlxsw_sp_neigh_entry *neigh_entry) 1946 { 1947 struct neighbour *n; 1948 1949 n = neigh_entry->key.n; 1950 return (struct in6_addr *) &n->primary_key; 1951 } 1952 1953 int mlxsw_sp_neigh_counter_get(struct mlxsw_sp *mlxsw_sp, 1954 struct mlxsw_sp_neigh_entry *neigh_entry, 1955 u64 *p_counter) 1956 { 1957 if (!neigh_entry->counter_valid) 1958 return -EINVAL; 1959 1960 return mlxsw_sp_flow_counter_get(mlxsw_sp, neigh_entry->counter_index, 1961 p_counter, NULL); 1962 } 1963 1964 static struct mlxsw_sp_neigh_entry * 1965 mlxsw_sp_neigh_entry_alloc(struct mlxsw_sp *mlxsw_sp, struct neighbour *n, 1966 u16 rif) 1967 { 1968 struct mlxsw_sp_neigh_entry *neigh_entry; 1969 1970 neigh_entry = kzalloc(sizeof(*neigh_entry), GFP_KERNEL); 1971 if (!neigh_entry) 1972 return NULL; 1973 1974 neigh_entry->key.n = n; 1975 neigh_entry->rif = rif; 1976 INIT_LIST_HEAD(&neigh_entry->nexthop_list); 1977 1978 return neigh_entry; 1979 } 1980 1981 static void mlxsw_sp_neigh_entry_free(struct mlxsw_sp_neigh_entry *neigh_entry) 1982 { 1983 kfree(neigh_entry); 1984 } 1985 1986 static int 1987 mlxsw_sp_neigh_entry_insert(struct mlxsw_sp *mlxsw_sp, 1988 struct mlxsw_sp_neigh_entry *neigh_entry) 1989 { 1990 return rhashtable_insert_fast(&mlxsw_sp->router->neigh_ht, 1991 &neigh_entry->ht_node, 1992 mlxsw_sp_neigh_ht_params); 1993 } 1994 1995 static void 1996 mlxsw_sp_neigh_entry_remove(struct mlxsw_sp *mlxsw_sp, 1997 struct mlxsw_sp_neigh_entry *neigh_entry) 1998 { 1999 rhashtable_remove_fast(&mlxsw_sp->router->neigh_ht, 2000 &neigh_entry->ht_node, 2001 mlxsw_sp_neigh_ht_params); 2002 } 2003 2004 static bool 2005 mlxsw_sp_neigh_counter_should_alloc(struct mlxsw_sp *mlxsw_sp, 2006 struct mlxsw_sp_neigh_entry *neigh_entry) 2007 { 2008 struct devlink *devlink; 2009 const char *table_name; 2010 2011 switch (mlxsw_sp_neigh_entry_type(neigh_entry)) { 2012 case AF_INET: 2013 table_name = MLXSW_SP_DPIPE_TABLE_NAME_HOST4; 2014 break; 2015 case AF_INET6: 2016 table_name = MLXSW_SP_DPIPE_TABLE_NAME_HOST6; 2017 break; 2018 default: 2019 WARN_ON(1); 2020 return false; 2021 } 2022 2023 devlink = priv_to_devlink(mlxsw_sp->core); 2024 return devlink_dpipe_table_counter_enabled(devlink, table_name); 2025 } 2026 2027 static void 2028 mlxsw_sp_neigh_counter_alloc(struct mlxsw_sp *mlxsw_sp, 2029 struct mlxsw_sp_neigh_entry *neigh_entry) 2030 { 2031 if (!mlxsw_sp_neigh_counter_should_alloc(mlxsw_sp, neigh_entry)) 2032 return; 2033 2034 if (mlxsw_sp_flow_counter_alloc(mlxsw_sp, &neigh_entry->counter_index)) 2035 return; 2036 2037 neigh_entry->counter_valid = true; 2038 } 2039 2040 static void 2041 mlxsw_sp_neigh_counter_free(struct mlxsw_sp *mlxsw_sp, 2042 struct mlxsw_sp_neigh_entry *neigh_entry) 2043 { 2044 if (!neigh_entry->counter_valid) 2045 return; 2046 mlxsw_sp_flow_counter_free(mlxsw_sp, 2047 neigh_entry->counter_index); 2048 neigh_entry->counter_valid = false; 2049 } 2050 2051 static struct mlxsw_sp_neigh_entry * 2052 mlxsw_sp_neigh_entry_create(struct mlxsw_sp *mlxsw_sp, struct neighbour *n) 2053 { 2054 struct mlxsw_sp_neigh_entry *neigh_entry; 2055 struct mlxsw_sp_rif *rif; 2056 int err; 2057 2058 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, n->dev); 2059 if (!rif) 2060 return ERR_PTR(-EINVAL); 2061 2062 neigh_entry = mlxsw_sp_neigh_entry_alloc(mlxsw_sp, n, rif->rif_index); 2063 if (!neigh_entry) 2064 return ERR_PTR(-ENOMEM); 2065 2066 err = mlxsw_sp_neigh_entry_insert(mlxsw_sp, neigh_entry); 2067 if (err) 2068 goto err_neigh_entry_insert; 2069 2070 mlxsw_sp_neigh_counter_alloc(mlxsw_sp, neigh_entry); 2071 list_add(&neigh_entry->rif_list_node, &rif->neigh_list); 2072 2073 return neigh_entry; 2074 2075 err_neigh_entry_insert: 2076 mlxsw_sp_neigh_entry_free(neigh_entry); 2077 return ERR_PTR(err); 2078 } 2079 2080 static void 2081 mlxsw_sp_neigh_entry_destroy(struct mlxsw_sp *mlxsw_sp, 2082 struct mlxsw_sp_neigh_entry *neigh_entry) 2083 { 2084 list_del(&neigh_entry->rif_list_node); 2085 mlxsw_sp_neigh_counter_free(mlxsw_sp, neigh_entry); 2086 mlxsw_sp_neigh_entry_remove(mlxsw_sp, neigh_entry); 2087 mlxsw_sp_neigh_entry_free(neigh_entry); 2088 } 2089 2090 static struct mlxsw_sp_neigh_entry * 2091 mlxsw_sp_neigh_entry_lookup(struct mlxsw_sp *mlxsw_sp, struct neighbour *n) 2092 { 2093 struct mlxsw_sp_neigh_key key; 2094 2095 key.n = n; 2096 return rhashtable_lookup_fast(&mlxsw_sp->router->neigh_ht, 2097 &key, mlxsw_sp_neigh_ht_params); 2098 } 2099 2100 static void 2101 mlxsw_sp_router_neighs_update_interval_init(struct mlxsw_sp *mlxsw_sp) 2102 { 2103 unsigned long interval; 2104 2105 #if IS_ENABLED(CONFIG_IPV6) 2106 interval = min_t(unsigned long, 2107 NEIGH_VAR(&arp_tbl.parms, DELAY_PROBE_TIME), 2108 NEIGH_VAR(&nd_tbl.parms, DELAY_PROBE_TIME)); 2109 #else 2110 interval = NEIGH_VAR(&arp_tbl.parms, DELAY_PROBE_TIME); 2111 #endif 2112 mlxsw_sp->router->neighs_update.interval = jiffies_to_msecs(interval); 2113 } 2114 2115 static void mlxsw_sp_router_neigh_ent_ipv4_process(struct mlxsw_sp *mlxsw_sp, 2116 char *rauhtd_pl, 2117 int ent_index) 2118 { 2119 struct net_device *dev; 2120 struct neighbour *n; 2121 __be32 dipn; 2122 u32 dip; 2123 u16 rif; 2124 2125 mlxsw_reg_rauhtd_ent_ipv4_unpack(rauhtd_pl, ent_index, &rif, &dip); 2126 2127 if (!mlxsw_sp->router->rifs[rif]) { 2128 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Incorrect RIF in neighbour entry\n"); 2129 return; 2130 } 2131 2132 dipn = htonl(dip); 2133 dev = mlxsw_sp->router->rifs[rif]->dev; 2134 n = neigh_lookup(&arp_tbl, &dipn, dev); 2135 if (!n) 2136 return; 2137 2138 netdev_dbg(dev, "Updating neighbour with IP=%pI4h\n", &dip); 2139 neigh_event_send(n, NULL); 2140 neigh_release(n); 2141 } 2142 2143 #if IS_ENABLED(CONFIG_IPV6) 2144 static void mlxsw_sp_router_neigh_ent_ipv6_process(struct mlxsw_sp *mlxsw_sp, 2145 char *rauhtd_pl, 2146 int rec_index) 2147 { 2148 struct net_device *dev; 2149 struct neighbour *n; 2150 struct in6_addr dip; 2151 u16 rif; 2152 2153 mlxsw_reg_rauhtd_ent_ipv6_unpack(rauhtd_pl, rec_index, &rif, 2154 (char *) &dip); 2155 2156 if (!mlxsw_sp->router->rifs[rif]) { 2157 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Incorrect RIF in neighbour entry\n"); 2158 return; 2159 } 2160 2161 dev = mlxsw_sp->router->rifs[rif]->dev; 2162 n = neigh_lookup(&nd_tbl, &dip, dev); 2163 if (!n) 2164 return; 2165 2166 netdev_dbg(dev, "Updating neighbour with IP=%pI6c\n", &dip); 2167 neigh_event_send(n, NULL); 2168 neigh_release(n); 2169 } 2170 #else 2171 static void mlxsw_sp_router_neigh_ent_ipv6_process(struct mlxsw_sp *mlxsw_sp, 2172 char *rauhtd_pl, 2173 int rec_index) 2174 { 2175 } 2176 #endif 2177 2178 static void mlxsw_sp_router_neigh_rec_ipv4_process(struct mlxsw_sp *mlxsw_sp, 2179 char *rauhtd_pl, 2180 int rec_index) 2181 { 2182 u8 num_entries; 2183 int i; 2184 2185 num_entries = mlxsw_reg_rauhtd_ipv4_rec_num_entries_get(rauhtd_pl, 2186 rec_index); 2187 /* Hardware starts counting at 0, so add 1. */ 2188 num_entries++; 2189 2190 /* Each record consists of several neighbour entries. */ 2191 for (i = 0; i < num_entries; i++) { 2192 int ent_index; 2193 2194 ent_index = rec_index * MLXSW_REG_RAUHTD_IPV4_ENT_PER_REC + i; 2195 mlxsw_sp_router_neigh_ent_ipv4_process(mlxsw_sp, rauhtd_pl, 2196 ent_index); 2197 } 2198 2199 } 2200 2201 static void mlxsw_sp_router_neigh_rec_ipv6_process(struct mlxsw_sp *mlxsw_sp, 2202 char *rauhtd_pl, 2203 int rec_index) 2204 { 2205 /* One record contains one entry. */ 2206 mlxsw_sp_router_neigh_ent_ipv6_process(mlxsw_sp, rauhtd_pl, 2207 rec_index); 2208 } 2209 2210 static void mlxsw_sp_router_neigh_rec_process(struct mlxsw_sp *mlxsw_sp, 2211 char *rauhtd_pl, int rec_index) 2212 { 2213 switch (mlxsw_reg_rauhtd_rec_type_get(rauhtd_pl, rec_index)) { 2214 case MLXSW_REG_RAUHTD_TYPE_IPV4: 2215 mlxsw_sp_router_neigh_rec_ipv4_process(mlxsw_sp, rauhtd_pl, 2216 rec_index); 2217 break; 2218 case MLXSW_REG_RAUHTD_TYPE_IPV6: 2219 mlxsw_sp_router_neigh_rec_ipv6_process(mlxsw_sp, rauhtd_pl, 2220 rec_index); 2221 break; 2222 } 2223 } 2224 2225 static bool mlxsw_sp_router_rauhtd_is_full(char *rauhtd_pl) 2226 { 2227 u8 num_rec, last_rec_index, num_entries; 2228 2229 num_rec = mlxsw_reg_rauhtd_num_rec_get(rauhtd_pl); 2230 last_rec_index = num_rec - 1; 2231 2232 if (num_rec < MLXSW_REG_RAUHTD_REC_MAX_NUM) 2233 return false; 2234 if (mlxsw_reg_rauhtd_rec_type_get(rauhtd_pl, last_rec_index) == 2235 MLXSW_REG_RAUHTD_TYPE_IPV6) 2236 return true; 2237 2238 num_entries = mlxsw_reg_rauhtd_ipv4_rec_num_entries_get(rauhtd_pl, 2239 last_rec_index); 2240 if (++num_entries == MLXSW_REG_RAUHTD_IPV4_ENT_PER_REC) 2241 return true; 2242 return false; 2243 } 2244 2245 static int 2246 __mlxsw_sp_router_neighs_update_rauhtd(struct mlxsw_sp *mlxsw_sp, 2247 char *rauhtd_pl, 2248 enum mlxsw_reg_rauhtd_type type) 2249 { 2250 int i, num_rec; 2251 int err; 2252 2253 /* Make sure the neighbour's netdev isn't removed in the 2254 * process. 2255 */ 2256 rtnl_lock(); 2257 do { 2258 mlxsw_reg_rauhtd_pack(rauhtd_pl, type); 2259 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(rauhtd), 2260 rauhtd_pl); 2261 if (err) { 2262 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Failed to dump neighbour table\n"); 2263 break; 2264 } 2265 num_rec = mlxsw_reg_rauhtd_num_rec_get(rauhtd_pl); 2266 for (i = 0; i < num_rec; i++) 2267 mlxsw_sp_router_neigh_rec_process(mlxsw_sp, rauhtd_pl, 2268 i); 2269 } while (mlxsw_sp_router_rauhtd_is_full(rauhtd_pl)); 2270 rtnl_unlock(); 2271 2272 return err; 2273 } 2274 2275 static int mlxsw_sp_router_neighs_update_rauhtd(struct mlxsw_sp *mlxsw_sp) 2276 { 2277 enum mlxsw_reg_rauhtd_type type; 2278 char *rauhtd_pl; 2279 int err; 2280 2281 rauhtd_pl = kmalloc(MLXSW_REG_RAUHTD_LEN, GFP_KERNEL); 2282 if (!rauhtd_pl) 2283 return -ENOMEM; 2284 2285 type = MLXSW_REG_RAUHTD_TYPE_IPV4; 2286 err = __mlxsw_sp_router_neighs_update_rauhtd(mlxsw_sp, rauhtd_pl, type); 2287 if (err) 2288 goto out; 2289 2290 type = MLXSW_REG_RAUHTD_TYPE_IPV6; 2291 err = __mlxsw_sp_router_neighs_update_rauhtd(mlxsw_sp, rauhtd_pl, type); 2292 out: 2293 kfree(rauhtd_pl); 2294 return err; 2295 } 2296 2297 static void mlxsw_sp_router_neighs_update_nh(struct mlxsw_sp *mlxsw_sp) 2298 { 2299 struct mlxsw_sp_neigh_entry *neigh_entry; 2300 2301 /* Take RTNL mutex here to prevent lists from changes */ 2302 rtnl_lock(); 2303 list_for_each_entry(neigh_entry, &mlxsw_sp->router->nexthop_neighs_list, 2304 nexthop_neighs_list_node) 2305 /* If this neigh have nexthops, make the kernel think this neigh 2306 * is active regardless of the traffic. 2307 */ 2308 neigh_event_send(neigh_entry->key.n, NULL); 2309 rtnl_unlock(); 2310 } 2311 2312 static void 2313 mlxsw_sp_router_neighs_update_work_schedule(struct mlxsw_sp *mlxsw_sp) 2314 { 2315 unsigned long interval = mlxsw_sp->router->neighs_update.interval; 2316 2317 mlxsw_core_schedule_dw(&mlxsw_sp->router->neighs_update.dw, 2318 msecs_to_jiffies(interval)); 2319 } 2320 2321 static void mlxsw_sp_router_neighs_update_work(struct work_struct *work) 2322 { 2323 struct mlxsw_sp_router *router; 2324 int err; 2325 2326 router = container_of(work, struct mlxsw_sp_router, 2327 neighs_update.dw.work); 2328 err = mlxsw_sp_router_neighs_update_rauhtd(router->mlxsw_sp); 2329 if (err) 2330 dev_err(router->mlxsw_sp->bus_info->dev, "Could not update kernel for neigh activity"); 2331 2332 mlxsw_sp_router_neighs_update_nh(router->mlxsw_sp); 2333 2334 mlxsw_sp_router_neighs_update_work_schedule(router->mlxsw_sp); 2335 } 2336 2337 static void mlxsw_sp_router_probe_unresolved_nexthops(struct work_struct *work) 2338 { 2339 struct mlxsw_sp_neigh_entry *neigh_entry; 2340 struct mlxsw_sp_router *router; 2341 2342 router = container_of(work, struct mlxsw_sp_router, 2343 nexthop_probe_dw.work); 2344 /* Iterate over nexthop neighbours, find those who are unresolved and 2345 * send arp on them. This solves the chicken-egg problem when 2346 * the nexthop wouldn't get offloaded until the neighbor is resolved 2347 * but it wouldn't get resolved ever in case traffic is flowing in HW 2348 * using different nexthop. 2349 * 2350 * Take RTNL mutex here to prevent lists from changes. 2351 */ 2352 rtnl_lock(); 2353 list_for_each_entry(neigh_entry, &router->nexthop_neighs_list, 2354 nexthop_neighs_list_node) 2355 if (!neigh_entry->connected) 2356 neigh_event_send(neigh_entry->key.n, NULL); 2357 rtnl_unlock(); 2358 2359 mlxsw_core_schedule_dw(&router->nexthop_probe_dw, 2360 MLXSW_SP_UNRESOLVED_NH_PROBE_INTERVAL); 2361 } 2362 2363 static void 2364 mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp *mlxsw_sp, 2365 struct mlxsw_sp_neigh_entry *neigh_entry, 2366 bool removing); 2367 2368 static enum mlxsw_reg_rauht_op mlxsw_sp_rauht_op(bool adding) 2369 { 2370 return adding ? MLXSW_REG_RAUHT_OP_WRITE_ADD : 2371 MLXSW_REG_RAUHT_OP_WRITE_DELETE; 2372 } 2373 2374 static void 2375 mlxsw_sp_router_neigh_entry_op4(struct mlxsw_sp *mlxsw_sp, 2376 struct mlxsw_sp_neigh_entry *neigh_entry, 2377 enum mlxsw_reg_rauht_op op) 2378 { 2379 struct neighbour *n = neigh_entry->key.n; 2380 u32 dip = ntohl(*((__be32 *) n->primary_key)); 2381 char rauht_pl[MLXSW_REG_RAUHT_LEN]; 2382 2383 mlxsw_reg_rauht_pack4(rauht_pl, op, neigh_entry->rif, neigh_entry->ha, 2384 dip); 2385 if (neigh_entry->counter_valid) 2386 mlxsw_reg_rauht_pack_counter(rauht_pl, 2387 neigh_entry->counter_index); 2388 mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rauht), rauht_pl); 2389 } 2390 2391 static void 2392 mlxsw_sp_router_neigh_entry_op6(struct mlxsw_sp *mlxsw_sp, 2393 struct mlxsw_sp_neigh_entry *neigh_entry, 2394 enum mlxsw_reg_rauht_op op) 2395 { 2396 struct neighbour *n = neigh_entry->key.n; 2397 char rauht_pl[MLXSW_REG_RAUHT_LEN]; 2398 const char *dip = n->primary_key; 2399 2400 mlxsw_reg_rauht_pack6(rauht_pl, op, neigh_entry->rif, neigh_entry->ha, 2401 dip); 2402 if (neigh_entry->counter_valid) 2403 mlxsw_reg_rauht_pack_counter(rauht_pl, 2404 neigh_entry->counter_index); 2405 mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rauht), rauht_pl); 2406 } 2407 2408 bool mlxsw_sp_neigh_ipv6_ignore(struct mlxsw_sp_neigh_entry *neigh_entry) 2409 { 2410 struct neighbour *n = neigh_entry->key.n; 2411 2412 /* Packets with a link-local destination address are trapped 2413 * after LPM lookup and never reach the neighbour table, so 2414 * there is no need to program such neighbours to the device. 2415 */ 2416 if (ipv6_addr_type((struct in6_addr *) &n->primary_key) & 2417 IPV6_ADDR_LINKLOCAL) 2418 return true; 2419 return false; 2420 } 2421 2422 static void 2423 mlxsw_sp_neigh_entry_update(struct mlxsw_sp *mlxsw_sp, 2424 struct mlxsw_sp_neigh_entry *neigh_entry, 2425 bool adding) 2426 { 2427 if (!adding && !neigh_entry->connected) 2428 return; 2429 neigh_entry->connected = adding; 2430 if (neigh_entry->key.n->tbl->family == AF_INET) { 2431 mlxsw_sp_router_neigh_entry_op4(mlxsw_sp, neigh_entry, 2432 mlxsw_sp_rauht_op(adding)); 2433 } else if (neigh_entry->key.n->tbl->family == AF_INET6) { 2434 if (mlxsw_sp_neigh_ipv6_ignore(neigh_entry)) 2435 return; 2436 mlxsw_sp_router_neigh_entry_op6(mlxsw_sp, neigh_entry, 2437 mlxsw_sp_rauht_op(adding)); 2438 } else { 2439 WARN_ON_ONCE(1); 2440 } 2441 } 2442 2443 void 2444 mlxsw_sp_neigh_entry_counter_update(struct mlxsw_sp *mlxsw_sp, 2445 struct mlxsw_sp_neigh_entry *neigh_entry, 2446 bool adding) 2447 { 2448 if (adding) 2449 mlxsw_sp_neigh_counter_alloc(mlxsw_sp, neigh_entry); 2450 else 2451 mlxsw_sp_neigh_counter_free(mlxsw_sp, neigh_entry); 2452 mlxsw_sp_neigh_entry_update(mlxsw_sp, neigh_entry, true); 2453 } 2454 2455 struct mlxsw_sp_netevent_work { 2456 struct work_struct work; 2457 struct mlxsw_sp *mlxsw_sp; 2458 struct neighbour *n; 2459 }; 2460 2461 static void mlxsw_sp_router_neigh_event_work(struct work_struct *work) 2462 { 2463 struct mlxsw_sp_netevent_work *net_work = 2464 container_of(work, struct mlxsw_sp_netevent_work, work); 2465 struct mlxsw_sp *mlxsw_sp = net_work->mlxsw_sp; 2466 struct mlxsw_sp_neigh_entry *neigh_entry; 2467 struct neighbour *n = net_work->n; 2468 unsigned char ha[ETH_ALEN]; 2469 bool entry_connected; 2470 u8 nud_state, dead; 2471 2472 /* If these parameters are changed after we release the lock, 2473 * then we are guaranteed to receive another event letting us 2474 * know about it. 2475 */ 2476 read_lock_bh(&n->lock); 2477 memcpy(ha, n->ha, ETH_ALEN); 2478 nud_state = n->nud_state; 2479 dead = n->dead; 2480 read_unlock_bh(&n->lock); 2481 2482 rtnl_lock(); 2483 mlxsw_sp_span_respin(mlxsw_sp); 2484 2485 entry_connected = nud_state & NUD_VALID && !dead; 2486 neigh_entry = mlxsw_sp_neigh_entry_lookup(mlxsw_sp, n); 2487 if (!entry_connected && !neigh_entry) 2488 goto out; 2489 if (!neigh_entry) { 2490 neigh_entry = mlxsw_sp_neigh_entry_create(mlxsw_sp, n); 2491 if (IS_ERR(neigh_entry)) 2492 goto out; 2493 } 2494 2495 memcpy(neigh_entry->ha, ha, ETH_ALEN); 2496 mlxsw_sp_neigh_entry_update(mlxsw_sp, neigh_entry, entry_connected); 2497 mlxsw_sp_nexthop_neigh_update(mlxsw_sp, neigh_entry, !entry_connected); 2498 2499 if (!neigh_entry->connected && list_empty(&neigh_entry->nexthop_list)) 2500 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry); 2501 2502 out: 2503 rtnl_unlock(); 2504 neigh_release(n); 2505 kfree(net_work); 2506 } 2507 2508 static int mlxsw_sp_mp_hash_init(struct mlxsw_sp *mlxsw_sp); 2509 2510 static void mlxsw_sp_router_mp_hash_event_work(struct work_struct *work) 2511 { 2512 struct mlxsw_sp_netevent_work *net_work = 2513 container_of(work, struct mlxsw_sp_netevent_work, work); 2514 struct mlxsw_sp *mlxsw_sp = net_work->mlxsw_sp; 2515 2516 mlxsw_sp_mp_hash_init(mlxsw_sp); 2517 kfree(net_work); 2518 } 2519 2520 static int __mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp); 2521 2522 static void mlxsw_sp_router_update_priority_work(struct work_struct *work) 2523 { 2524 struct mlxsw_sp_netevent_work *net_work = 2525 container_of(work, struct mlxsw_sp_netevent_work, work); 2526 struct mlxsw_sp *mlxsw_sp = net_work->mlxsw_sp; 2527 2528 __mlxsw_sp_router_init(mlxsw_sp); 2529 kfree(net_work); 2530 } 2531 2532 static int mlxsw_sp_router_schedule_work(struct net *net, 2533 struct notifier_block *nb, 2534 void (*cb)(struct work_struct *)) 2535 { 2536 struct mlxsw_sp_netevent_work *net_work; 2537 struct mlxsw_sp_router *router; 2538 2539 if (!net_eq(net, &init_net)) 2540 return NOTIFY_DONE; 2541 2542 net_work = kzalloc(sizeof(*net_work), GFP_ATOMIC); 2543 if (!net_work) 2544 return NOTIFY_BAD; 2545 2546 router = container_of(nb, struct mlxsw_sp_router, netevent_nb); 2547 INIT_WORK(&net_work->work, cb); 2548 net_work->mlxsw_sp = router->mlxsw_sp; 2549 mlxsw_core_schedule_work(&net_work->work); 2550 return NOTIFY_DONE; 2551 } 2552 2553 static int mlxsw_sp_router_netevent_event(struct notifier_block *nb, 2554 unsigned long event, void *ptr) 2555 { 2556 struct mlxsw_sp_netevent_work *net_work; 2557 struct mlxsw_sp_port *mlxsw_sp_port; 2558 struct mlxsw_sp *mlxsw_sp; 2559 unsigned long interval; 2560 struct neigh_parms *p; 2561 struct neighbour *n; 2562 2563 switch (event) { 2564 case NETEVENT_DELAY_PROBE_TIME_UPDATE: 2565 p = ptr; 2566 2567 /* We don't care about changes in the default table. */ 2568 if (!p->dev || (p->tbl->family != AF_INET && 2569 p->tbl->family != AF_INET6)) 2570 return NOTIFY_DONE; 2571 2572 /* We are in atomic context and can't take RTNL mutex, 2573 * so use RCU variant to walk the device chain. 2574 */ 2575 mlxsw_sp_port = mlxsw_sp_port_lower_dev_hold(p->dev); 2576 if (!mlxsw_sp_port) 2577 return NOTIFY_DONE; 2578 2579 mlxsw_sp = mlxsw_sp_port->mlxsw_sp; 2580 interval = jiffies_to_msecs(NEIGH_VAR(p, DELAY_PROBE_TIME)); 2581 mlxsw_sp->router->neighs_update.interval = interval; 2582 2583 mlxsw_sp_port_dev_put(mlxsw_sp_port); 2584 break; 2585 case NETEVENT_NEIGH_UPDATE: 2586 n = ptr; 2587 2588 if (n->tbl->family != AF_INET && n->tbl->family != AF_INET6) 2589 return NOTIFY_DONE; 2590 2591 mlxsw_sp_port = mlxsw_sp_port_lower_dev_hold(n->dev); 2592 if (!mlxsw_sp_port) 2593 return NOTIFY_DONE; 2594 2595 net_work = kzalloc(sizeof(*net_work), GFP_ATOMIC); 2596 if (!net_work) { 2597 mlxsw_sp_port_dev_put(mlxsw_sp_port); 2598 return NOTIFY_BAD; 2599 } 2600 2601 INIT_WORK(&net_work->work, mlxsw_sp_router_neigh_event_work); 2602 net_work->mlxsw_sp = mlxsw_sp_port->mlxsw_sp; 2603 net_work->n = n; 2604 2605 /* Take a reference to ensure the neighbour won't be 2606 * destructed until we drop the reference in delayed 2607 * work. 2608 */ 2609 neigh_clone(n); 2610 mlxsw_core_schedule_work(&net_work->work); 2611 mlxsw_sp_port_dev_put(mlxsw_sp_port); 2612 break; 2613 case NETEVENT_IPV4_MPATH_HASH_UPDATE: 2614 case NETEVENT_IPV6_MPATH_HASH_UPDATE: 2615 return mlxsw_sp_router_schedule_work(ptr, nb, 2616 mlxsw_sp_router_mp_hash_event_work); 2617 2618 case NETEVENT_IPV4_FWD_UPDATE_PRIORITY_UPDATE: 2619 return mlxsw_sp_router_schedule_work(ptr, nb, 2620 mlxsw_sp_router_update_priority_work); 2621 } 2622 2623 return NOTIFY_DONE; 2624 } 2625 2626 static int mlxsw_sp_neigh_init(struct mlxsw_sp *mlxsw_sp) 2627 { 2628 int err; 2629 2630 err = rhashtable_init(&mlxsw_sp->router->neigh_ht, 2631 &mlxsw_sp_neigh_ht_params); 2632 if (err) 2633 return err; 2634 2635 /* Initialize the polling interval according to the default 2636 * table. 2637 */ 2638 mlxsw_sp_router_neighs_update_interval_init(mlxsw_sp); 2639 2640 /* Create the delayed works for the activity_update */ 2641 INIT_DELAYED_WORK(&mlxsw_sp->router->neighs_update.dw, 2642 mlxsw_sp_router_neighs_update_work); 2643 INIT_DELAYED_WORK(&mlxsw_sp->router->nexthop_probe_dw, 2644 mlxsw_sp_router_probe_unresolved_nexthops); 2645 mlxsw_core_schedule_dw(&mlxsw_sp->router->neighs_update.dw, 0); 2646 mlxsw_core_schedule_dw(&mlxsw_sp->router->nexthop_probe_dw, 0); 2647 return 0; 2648 } 2649 2650 static void mlxsw_sp_neigh_fini(struct mlxsw_sp *mlxsw_sp) 2651 { 2652 cancel_delayed_work_sync(&mlxsw_sp->router->neighs_update.dw); 2653 cancel_delayed_work_sync(&mlxsw_sp->router->nexthop_probe_dw); 2654 rhashtable_destroy(&mlxsw_sp->router->neigh_ht); 2655 } 2656 2657 static void mlxsw_sp_neigh_rif_gone_sync(struct mlxsw_sp *mlxsw_sp, 2658 struct mlxsw_sp_rif *rif) 2659 { 2660 struct mlxsw_sp_neigh_entry *neigh_entry, *tmp; 2661 2662 list_for_each_entry_safe(neigh_entry, tmp, &rif->neigh_list, 2663 rif_list_node) { 2664 mlxsw_sp_neigh_entry_update(mlxsw_sp, neigh_entry, false); 2665 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry); 2666 } 2667 } 2668 2669 enum mlxsw_sp_nexthop_type { 2670 MLXSW_SP_NEXTHOP_TYPE_ETH, 2671 MLXSW_SP_NEXTHOP_TYPE_IPIP, 2672 }; 2673 2674 struct mlxsw_sp_nexthop_key { 2675 struct fib_nh *fib_nh; 2676 }; 2677 2678 struct mlxsw_sp_nexthop { 2679 struct list_head neigh_list_node; /* member of neigh entry list */ 2680 struct list_head rif_list_node; 2681 struct list_head router_list_node; 2682 struct mlxsw_sp_nexthop_group *nh_grp; /* pointer back to the group 2683 * this belongs to 2684 */ 2685 struct rhash_head ht_node; 2686 struct mlxsw_sp_nexthop_key key; 2687 unsigned char gw_addr[sizeof(struct in6_addr)]; 2688 int ifindex; 2689 int nh_weight; 2690 int norm_nh_weight; 2691 int num_adj_entries; 2692 struct mlxsw_sp_rif *rif; 2693 u8 should_offload:1, /* set indicates this neigh is connected and 2694 * should be put to KVD linear area of this group. 2695 */ 2696 offloaded:1, /* set in case the neigh is actually put into 2697 * KVD linear area of this group. 2698 */ 2699 update:1; /* set indicates that MAC of this neigh should be 2700 * updated in HW 2701 */ 2702 enum mlxsw_sp_nexthop_type type; 2703 union { 2704 struct mlxsw_sp_neigh_entry *neigh_entry; 2705 struct mlxsw_sp_ipip_entry *ipip_entry; 2706 }; 2707 unsigned int counter_index; 2708 bool counter_valid; 2709 }; 2710 2711 struct mlxsw_sp_nexthop_group { 2712 void *priv; 2713 struct rhash_head ht_node; 2714 struct list_head fib_list; /* list of fib entries that use this group */ 2715 struct neigh_table *neigh_tbl; 2716 u8 adj_index_valid:1, 2717 gateway:1; /* routes using the group use a gateway */ 2718 u32 adj_index; 2719 u16 ecmp_size; 2720 u16 count; 2721 int sum_norm_weight; 2722 struct mlxsw_sp_nexthop nexthops[0]; 2723 #define nh_rif nexthops[0].rif 2724 }; 2725 2726 void mlxsw_sp_nexthop_counter_alloc(struct mlxsw_sp *mlxsw_sp, 2727 struct mlxsw_sp_nexthop *nh) 2728 { 2729 struct devlink *devlink; 2730 2731 devlink = priv_to_devlink(mlxsw_sp->core); 2732 if (!devlink_dpipe_table_counter_enabled(devlink, 2733 MLXSW_SP_DPIPE_TABLE_NAME_ADJ)) 2734 return; 2735 2736 if (mlxsw_sp_flow_counter_alloc(mlxsw_sp, &nh->counter_index)) 2737 return; 2738 2739 nh->counter_valid = true; 2740 } 2741 2742 void mlxsw_sp_nexthop_counter_free(struct mlxsw_sp *mlxsw_sp, 2743 struct mlxsw_sp_nexthop *nh) 2744 { 2745 if (!nh->counter_valid) 2746 return; 2747 mlxsw_sp_flow_counter_free(mlxsw_sp, nh->counter_index); 2748 nh->counter_valid = false; 2749 } 2750 2751 int mlxsw_sp_nexthop_counter_get(struct mlxsw_sp *mlxsw_sp, 2752 struct mlxsw_sp_nexthop *nh, u64 *p_counter) 2753 { 2754 if (!nh->counter_valid) 2755 return -EINVAL; 2756 2757 return mlxsw_sp_flow_counter_get(mlxsw_sp, nh->counter_index, 2758 p_counter, NULL); 2759 } 2760 2761 struct mlxsw_sp_nexthop *mlxsw_sp_nexthop_next(struct mlxsw_sp_router *router, 2762 struct mlxsw_sp_nexthop *nh) 2763 { 2764 if (!nh) { 2765 if (list_empty(&router->nexthop_list)) 2766 return NULL; 2767 else 2768 return list_first_entry(&router->nexthop_list, 2769 typeof(*nh), router_list_node); 2770 } 2771 if (list_is_last(&nh->router_list_node, &router->nexthop_list)) 2772 return NULL; 2773 return list_next_entry(nh, router_list_node); 2774 } 2775 2776 bool mlxsw_sp_nexthop_offload(struct mlxsw_sp_nexthop *nh) 2777 { 2778 return nh->offloaded; 2779 } 2780 2781 unsigned char *mlxsw_sp_nexthop_ha(struct mlxsw_sp_nexthop *nh) 2782 { 2783 if (!nh->offloaded) 2784 return NULL; 2785 return nh->neigh_entry->ha; 2786 } 2787 2788 int mlxsw_sp_nexthop_indexes(struct mlxsw_sp_nexthop *nh, u32 *p_adj_index, 2789 u32 *p_adj_size, u32 *p_adj_hash_index) 2790 { 2791 struct mlxsw_sp_nexthop_group *nh_grp = nh->nh_grp; 2792 u32 adj_hash_index = 0; 2793 int i; 2794 2795 if (!nh->offloaded || !nh_grp->adj_index_valid) 2796 return -EINVAL; 2797 2798 *p_adj_index = nh_grp->adj_index; 2799 *p_adj_size = nh_grp->ecmp_size; 2800 2801 for (i = 0; i < nh_grp->count; i++) { 2802 struct mlxsw_sp_nexthop *nh_iter = &nh_grp->nexthops[i]; 2803 2804 if (nh_iter == nh) 2805 break; 2806 if (nh_iter->offloaded) 2807 adj_hash_index += nh_iter->num_adj_entries; 2808 } 2809 2810 *p_adj_hash_index = adj_hash_index; 2811 return 0; 2812 } 2813 2814 struct mlxsw_sp_rif *mlxsw_sp_nexthop_rif(struct mlxsw_sp_nexthop *nh) 2815 { 2816 return nh->rif; 2817 } 2818 2819 bool mlxsw_sp_nexthop_group_has_ipip(struct mlxsw_sp_nexthop *nh) 2820 { 2821 struct mlxsw_sp_nexthop_group *nh_grp = nh->nh_grp; 2822 int i; 2823 2824 for (i = 0; i < nh_grp->count; i++) { 2825 struct mlxsw_sp_nexthop *nh_iter = &nh_grp->nexthops[i]; 2826 2827 if (nh_iter->type == MLXSW_SP_NEXTHOP_TYPE_IPIP) 2828 return true; 2829 } 2830 return false; 2831 } 2832 2833 static struct fib_info * 2834 mlxsw_sp_nexthop4_group_fi(const struct mlxsw_sp_nexthop_group *nh_grp) 2835 { 2836 return nh_grp->priv; 2837 } 2838 2839 struct mlxsw_sp_nexthop_group_cmp_arg { 2840 enum mlxsw_sp_l3proto proto; 2841 union { 2842 struct fib_info *fi; 2843 struct mlxsw_sp_fib6_entry *fib6_entry; 2844 }; 2845 }; 2846 2847 static bool 2848 mlxsw_sp_nexthop6_group_has_nexthop(const struct mlxsw_sp_nexthop_group *nh_grp, 2849 const struct in6_addr *gw, int ifindex, 2850 int weight) 2851 { 2852 int i; 2853 2854 for (i = 0; i < nh_grp->count; i++) { 2855 const struct mlxsw_sp_nexthop *nh; 2856 2857 nh = &nh_grp->nexthops[i]; 2858 if (nh->ifindex == ifindex && nh->nh_weight == weight && 2859 ipv6_addr_equal(gw, (struct in6_addr *) nh->gw_addr)) 2860 return true; 2861 } 2862 2863 return false; 2864 } 2865 2866 static bool 2867 mlxsw_sp_nexthop6_group_cmp(const struct mlxsw_sp_nexthop_group *nh_grp, 2868 const struct mlxsw_sp_fib6_entry *fib6_entry) 2869 { 2870 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 2871 2872 if (nh_grp->count != fib6_entry->nrt6) 2873 return false; 2874 2875 list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) { 2876 struct fib6_nh *fib6_nh = &mlxsw_sp_rt6->rt->fib6_nh; 2877 struct in6_addr *gw; 2878 int ifindex, weight; 2879 2880 ifindex = fib6_nh->fib_nh_dev->ifindex; 2881 weight = fib6_nh->fib_nh_weight; 2882 gw = &fib6_nh->fib_nh_gw6; 2883 if (!mlxsw_sp_nexthop6_group_has_nexthop(nh_grp, gw, ifindex, 2884 weight)) 2885 return false; 2886 } 2887 2888 return true; 2889 } 2890 2891 static int 2892 mlxsw_sp_nexthop_group_cmp(struct rhashtable_compare_arg *arg, const void *ptr) 2893 { 2894 const struct mlxsw_sp_nexthop_group_cmp_arg *cmp_arg = arg->key; 2895 const struct mlxsw_sp_nexthop_group *nh_grp = ptr; 2896 2897 switch (cmp_arg->proto) { 2898 case MLXSW_SP_L3_PROTO_IPV4: 2899 return cmp_arg->fi != mlxsw_sp_nexthop4_group_fi(nh_grp); 2900 case MLXSW_SP_L3_PROTO_IPV6: 2901 return !mlxsw_sp_nexthop6_group_cmp(nh_grp, 2902 cmp_arg->fib6_entry); 2903 default: 2904 WARN_ON(1); 2905 return 1; 2906 } 2907 } 2908 2909 static int 2910 mlxsw_sp_nexthop_group_type(const struct mlxsw_sp_nexthop_group *nh_grp) 2911 { 2912 return nh_grp->neigh_tbl->family; 2913 } 2914 2915 static u32 mlxsw_sp_nexthop_group_hash_obj(const void *data, u32 len, u32 seed) 2916 { 2917 const struct mlxsw_sp_nexthop_group *nh_grp = data; 2918 const struct mlxsw_sp_nexthop *nh; 2919 struct fib_info *fi; 2920 unsigned int val; 2921 int i; 2922 2923 switch (mlxsw_sp_nexthop_group_type(nh_grp)) { 2924 case AF_INET: 2925 fi = mlxsw_sp_nexthop4_group_fi(nh_grp); 2926 return jhash(&fi, sizeof(fi), seed); 2927 case AF_INET6: 2928 val = nh_grp->count; 2929 for (i = 0; i < nh_grp->count; i++) { 2930 nh = &nh_grp->nexthops[i]; 2931 val ^= nh->ifindex; 2932 } 2933 return jhash(&val, sizeof(val), seed); 2934 default: 2935 WARN_ON(1); 2936 return 0; 2937 } 2938 } 2939 2940 static u32 2941 mlxsw_sp_nexthop6_group_hash(struct mlxsw_sp_fib6_entry *fib6_entry, u32 seed) 2942 { 2943 unsigned int val = fib6_entry->nrt6; 2944 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 2945 struct net_device *dev; 2946 2947 list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) { 2948 dev = mlxsw_sp_rt6->rt->fib6_nh.fib_nh_dev; 2949 val ^= dev->ifindex; 2950 } 2951 2952 return jhash(&val, sizeof(val), seed); 2953 } 2954 2955 static u32 2956 mlxsw_sp_nexthop_group_hash(const void *data, u32 len, u32 seed) 2957 { 2958 const struct mlxsw_sp_nexthop_group_cmp_arg *cmp_arg = data; 2959 2960 switch (cmp_arg->proto) { 2961 case MLXSW_SP_L3_PROTO_IPV4: 2962 return jhash(&cmp_arg->fi, sizeof(cmp_arg->fi), seed); 2963 case MLXSW_SP_L3_PROTO_IPV6: 2964 return mlxsw_sp_nexthop6_group_hash(cmp_arg->fib6_entry, seed); 2965 default: 2966 WARN_ON(1); 2967 return 0; 2968 } 2969 } 2970 2971 static const struct rhashtable_params mlxsw_sp_nexthop_group_ht_params = { 2972 .head_offset = offsetof(struct mlxsw_sp_nexthop_group, ht_node), 2973 .hashfn = mlxsw_sp_nexthop_group_hash, 2974 .obj_hashfn = mlxsw_sp_nexthop_group_hash_obj, 2975 .obj_cmpfn = mlxsw_sp_nexthop_group_cmp, 2976 }; 2977 2978 static int mlxsw_sp_nexthop_group_insert(struct mlxsw_sp *mlxsw_sp, 2979 struct mlxsw_sp_nexthop_group *nh_grp) 2980 { 2981 if (mlxsw_sp_nexthop_group_type(nh_grp) == AF_INET6 && 2982 !nh_grp->gateway) 2983 return 0; 2984 2985 return rhashtable_insert_fast(&mlxsw_sp->router->nexthop_group_ht, 2986 &nh_grp->ht_node, 2987 mlxsw_sp_nexthop_group_ht_params); 2988 } 2989 2990 static void mlxsw_sp_nexthop_group_remove(struct mlxsw_sp *mlxsw_sp, 2991 struct mlxsw_sp_nexthop_group *nh_grp) 2992 { 2993 if (mlxsw_sp_nexthop_group_type(nh_grp) == AF_INET6 && 2994 !nh_grp->gateway) 2995 return; 2996 2997 rhashtable_remove_fast(&mlxsw_sp->router->nexthop_group_ht, 2998 &nh_grp->ht_node, 2999 mlxsw_sp_nexthop_group_ht_params); 3000 } 3001 3002 static struct mlxsw_sp_nexthop_group * 3003 mlxsw_sp_nexthop4_group_lookup(struct mlxsw_sp *mlxsw_sp, 3004 struct fib_info *fi) 3005 { 3006 struct mlxsw_sp_nexthop_group_cmp_arg cmp_arg; 3007 3008 cmp_arg.proto = MLXSW_SP_L3_PROTO_IPV4; 3009 cmp_arg.fi = fi; 3010 return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_group_ht, 3011 &cmp_arg, 3012 mlxsw_sp_nexthop_group_ht_params); 3013 } 3014 3015 static struct mlxsw_sp_nexthop_group * 3016 mlxsw_sp_nexthop6_group_lookup(struct mlxsw_sp *mlxsw_sp, 3017 struct mlxsw_sp_fib6_entry *fib6_entry) 3018 { 3019 struct mlxsw_sp_nexthop_group_cmp_arg cmp_arg; 3020 3021 cmp_arg.proto = MLXSW_SP_L3_PROTO_IPV6; 3022 cmp_arg.fib6_entry = fib6_entry; 3023 return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_group_ht, 3024 &cmp_arg, 3025 mlxsw_sp_nexthop_group_ht_params); 3026 } 3027 3028 static const struct rhashtable_params mlxsw_sp_nexthop_ht_params = { 3029 .key_offset = offsetof(struct mlxsw_sp_nexthop, key), 3030 .head_offset = offsetof(struct mlxsw_sp_nexthop, ht_node), 3031 .key_len = sizeof(struct mlxsw_sp_nexthop_key), 3032 }; 3033 3034 static int mlxsw_sp_nexthop_insert(struct mlxsw_sp *mlxsw_sp, 3035 struct mlxsw_sp_nexthop *nh) 3036 { 3037 return rhashtable_insert_fast(&mlxsw_sp->router->nexthop_ht, 3038 &nh->ht_node, mlxsw_sp_nexthop_ht_params); 3039 } 3040 3041 static void mlxsw_sp_nexthop_remove(struct mlxsw_sp *mlxsw_sp, 3042 struct mlxsw_sp_nexthop *nh) 3043 { 3044 rhashtable_remove_fast(&mlxsw_sp->router->nexthop_ht, &nh->ht_node, 3045 mlxsw_sp_nexthop_ht_params); 3046 } 3047 3048 static struct mlxsw_sp_nexthop * 3049 mlxsw_sp_nexthop_lookup(struct mlxsw_sp *mlxsw_sp, 3050 struct mlxsw_sp_nexthop_key key) 3051 { 3052 return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_ht, &key, 3053 mlxsw_sp_nexthop_ht_params); 3054 } 3055 3056 static int mlxsw_sp_adj_index_mass_update_vr(struct mlxsw_sp *mlxsw_sp, 3057 const struct mlxsw_sp_fib *fib, 3058 u32 adj_index, u16 ecmp_size, 3059 u32 new_adj_index, 3060 u16 new_ecmp_size) 3061 { 3062 char raleu_pl[MLXSW_REG_RALEU_LEN]; 3063 3064 mlxsw_reg_raleu_pack(raleu_pl, 3065 (enum mlxsw_reg_ralxx_protocol) fib->proto, 3066 fib->vr->id, adj_index, ecmp_size, new_adj_index, 3067 new_ecmp_size); 3068 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raleu), raleu_pl); 3069 } 3070 3071 static int mlxsw_sp_adj_index_mass_update(struct mlxsw_sp *mlxsw_sp, 3072 struct mlxsw_sp_nexthop_group *nh_grp, 3073 u32 old_adj_index, u16 old_ecmp_size) 3074 { 3075 struct mlxsw_sp_fib_entry *fib_entry; 3076 struct mlxsw_sp_fib *fib = NULL; 3077 int err; 3078 3079 list_for_each_entry(fib_entry, &nh_grp->fib_list, nexthop_group_node) { 3080 if (fib == fib_entry->fib_node->fib) 3081 continue; 3082 fib = fib_entry->fib_node->fib; 3083 err = mlxsw_sp_adj_index_mass_update_vr(mlxsw_sp, fib, 3084 old_adj_index, 3085 old_ecmp_size, 3086 nh_grp->adj_index, 3087 nh_grp->ecmp_size); 3088 if (err) 3089 return err; 3090 } 3091 return 0; 3092 } 3093 3094 static int __mlxsw_sp_nexthop_update(struct mlxsw_sp *mlxsw_sp, u32 adj_index, 3095 struct mlxsw_sp_nexthop *nh) 3096 { 3097 struct mlxsw_sp_neigh_entry *neigh_entry = nh->neigh_entry; 3098 char ratr_pl[MLXSW_REG_RATR_LEN]; 3099 3100 mlxsw_reg_ratr_pack(ratr_pl, MLXSW_REG_RATR_OP_WRITE_WRITE_ENTRY, 3101 true, MLXSW_REG_RATR_TYPE_ETHERNET, 3102 adj_index, neigh_entry->rif); 3103 mlxsw_reg_ratr_eth_entry_pack(ratr_pl, neigh_entry->ha); 3104 if (nh->counter_valid) 3105 mlxsw_reg_ratr_counter_pack(ratr_pl, nh->counter_index, true); 3106 else 3107 mlxsw_reg_ratr_counter_pack(ratr_pl, 0, false); 3108 3109 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ratr), ratr_pl); 3110 } 3111 3112 int mlxsw_sp_nexthop_update(struct mlxsw_sp *mlxsw_sp, u32 adj_index, 3113 struct mlxsw_sp_nexthop *nh) 3114 { 3115 int i; 3116 3117 for (i = 0; i < nh->num_adj_entries; i++) { 3118 int err; 3119 3120 err = __mlxsw_sp_nexthop_update(mlxsw_sp, adj_index + i, nh); 3121 if (err) 3122 return err; 3123 } 3124 3125 return 0; 3126 } 3127 3128 static int __mlxsw_sp_nexthop_ipip_update(struct mlxsw_sp *mlxsw_sp, 3129 u32 adj_index, 3130 struct mlxsw_sp_nexthop *nh) 3131 { 3132 const struct mlxsw_sp_ipip_ops *ipip_ops; 3133 3134 ipip_ops = mlxsw_sp->router->ipip_ops_arr[nh->ipip_entry->ipipt]; 3135 return ipip_ops->nexthop_update(mlxsw_sp, adj_index, nh->ipip_entry); 3136 } 3137 3138 static int mlxsw_sp_nexthop_ipip_update(struct mlxsw_sp *mlxsw_sp, 3139 u32 adj_index, 3140 struct mlxsw_sp_nexthop *nh) 3141 { 3142 int i; 3143 3144 for (i = 0; i < nh->num_adj_entries; i++) { 3145 int err; 3146 3147 err = __mlxsw_sp_nexthop_ipip_update(mlxsw_sp, adj_index + i, 3148 nh); 3149 if (err) 3150 return err; 3151 } 3152 3153 return 0; 3154 } 3155 3156 static int 3157 mlxsw_sp_nexthop_group_update(struct mlxsw_sp *mlxsw_sp, 3158 struct mlxsw_sp_nexthop_group *nh_grp, 3159 bool reallocate) 3160 { 3161 u32 adj_index = nh_grp->adj_index; /* base */ 3162 struct mlxsw_sp_nexthop *nh; 3163 int i; 3164 int err; 3165 3166 for (i = 0; i < nh_grp->count; i++) { 3167 nh = &nh_grp->nexthops[i]; 3168 3169 if (!nh->should_offload) { 3170 nh->offloaded = 0; 3171 continue; 3172 } 3173 3174 if (nh->update || reallocate) { 3175 switch (nh->type) { 3176 case MLXSW_SP_NEXTHOP_TYPE_ETH: 3177 err = mlxsw_sp_nexthop_update 3178 (mlxsw_sp, adj_index, nh); 3179 break; 3180 case MLXSW_SP_NEXTHOP_TYPE_IPIP: 3181 err = mlxsw_sp_nexthop_ipip_update 3182 (mlxsw_sp, adj_index, nh); 3183 break; 3184 } 3185 if (err) 3186 return err; 3187 nh->update = 0; 3188 nh->offloaded = 1; 3189 } 3190 adj_index += nh->num_adj_entries; 3191 } 3192 return 0; 3193 } 3194 3195 static bool 3196 mlxsw_sp_fib_node_entry_is_first(const struct mlxsw_sp_fib_node *fib_node, 3197 const struct mlxsw_sp_fib_entry *fib_entry); 3198 3199 static int 3200 mlxsw_sp_nexthop_fib_entries_update(struct mlxsw_sp *mlxsw_sp, 3201 struct mlxsw_sp_nexthop_group *nh_grp) 3202 { 3203 struct mlxsw_sp_fib_entry *fib_entry; 3204 int err; 3205 3206 list_for_each_entry(fib_entry, &nh_grp->fib_list, nexthop_group_node) { 3207 if (!mlxsw_sp_fib_node_entry_is_first(fib_entry->fib_node, 3208 fib_entry)) 3209 continue; 3210 err = mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry); 3211 if (err) 3212 return err; 3213 } 3214 return 0; 3215 } 3216 3217 static void 3218 mlxsw_sp_fib_entry_offload_refresh(struct mlxsw_sp_fib_entry *fib_entry, 3219 enum mlxsw_reg_ralue_op op, int err); 3220 3221 static void 3222 mlxsw_sp_nexthop_fib_entries_refresh(struct mlxsw_sp_nexthop_group *nh_grp) 3223 { 3224 enum mlxsw_reg_ralue_op op = MLXSW_REG_RALUE_OP_WRITE_WRITE; 3225 struct mlxsw_sp_fib_entry *fib_entry; 3226 3227 list_for_each_entry(fib_entry, &nh_grp->fib_list, nexthop_group_node) { 3228 if (!mlxsw_sp_fib_node_entry_is_first(fib_entry->fib_node, 3229 fib_entry)) 3230 continue; 3231 mlxsw_sp_fib_entry_offload_refresh(fib_entry, op, 0); 3232 } 3233 } 3234 3235 static void mlxsw_sp_adj_grp_size_round_up(u16 *p_adj_grp_size) 3236 { 3237 /* Valid sizes for an adjacency group are: 3238 * 1-64, 512, 1024, 2048 and 4096. 3239 */ 3240 if (*p_adj_grp_size <= 64) 3241 return; 3242 else if (*p_adj_grp_size <= 512) 3243 *p_adj_grp_size = 512; 3244 else if (*p_adj_grp_size <= 1024) 3245 *p_adj_grp_size = 1024; 3246 else if (*p_adj_grp_size <= 2048) 3247 *p_adj_grp_size = 2048; 3248 else 3249 *p_adj_grp_size = 4096; 3250 } 3251 3252 static void mlxsw_sp_adj_grp_size_round_down(u16 *p_adj_grp_size, 3253 unsigned int alloc_size) 3254 { 3255 if (alloc_size >= 4096) 3256 *p_adj_grp_size = 4096; 3257 else if (alloc_size >= 2048) 3258 *p_adj_grp_size = 2048; 3259 else if (alloc_size >= 1024) 3260 *p_adj_grp_size = 1024; 3261 else if (alloc_size >= 512) 3262 *p_adj_grp_size = 512; 3263 } 3264 3265 static int mlxsw_sp_fix_adj_grp_size(struct mlxsw_sp *mlxsw_sp, 3266 u16 *p_adj_grp_size) 3267 { 3268 unsigned int alloc_size; 3269 int err; 3270 3271 /* Round up the requested group size to the next size supported 3272 * by the device and make sure the request can be satisfied. 3273 */ 3274 mlxsw_sp_adj_grp_size_round_up(p_adj_grp_size); 3275 err = mlxsw_sp_kvdl_alloc_count_query(mlxsw_sp, 3276 MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 3277 *p_adj_grp_size, &alloc_size); 3278 if (err) 3279 return err; 3280 /* It is possible the allocation results in more allocated 3281 * entries than requested. Try to use as much of them as 3282 * possible. 3283 */ 3284 mlxsw_sp_adj_grp_size_round_down(p_adj_grp_size, alloc_size); 3285 3286 return 0; 3287 } 3288 3289 static void 3290 mlxsw_sp_nexthop_group_normalize(struct mlxsw_sp_nexthop_group *nh_grp) 3291 { 3292 int i, g = 0, sum_norm_weight = 0; 3293 struct mlxsw_sp_nexthop *nh; 3294 3295 for (i = 0; i < nh_grp->count; i++) { 3296 nh = &nh_grp->nexthops[i]; 3297 3298 if (!nh->should_offload) 3299 continue; 3300 if (g > 0) 3301 g = gcd(nh->nh_weight, g); 3302 else 3303 g = nh->nh_weight; 3304 } 3305 3306 for (i = 0; i < nh_grp->count; i++) { 3307 nh = &nh_grp->nexthops[i]; 3308 3309 if (!nh->should_offload) 3310 continue; 3311 nh->norm_nh_weight = nh->nh_weight / g; 3312 sum_norm_weight += nh->norm_nh_weight; 3313 } 3314 3315 nh_grp->sum_norm_weight = sum_norm_weight; 3316 } 3317 3318 static void 3319 mlxsw_sp_nexthop_group_rebalance(struct mlxsw_sp_nexthop_group *nh_grp) 3320 { 3321 int total = nh_grp->sum_norm_weight; 3322 u16 ecmp_size = nh_grp->ecmp_size; 3323 int i, weight = 0, lower_bound = 0; 3324 3325 for (i = 0; i < nh_grp->count; i++) { 3326 struct mlxsw_sp_nexthop *nh = &nh_grp->nexthops[i]; 3327 int upper_bound; 3328 3329 if (!nh->should_offload) 3330 continue; 3331 weight += nh->norm_nh_weight; 3332 upper_bound = DIV_ROUND_CLOSEST(ecmp_size * weight, total); 3333 nh->num_adj_entries = upper_bound - lower_bound; 3334 lower_bound = upper_bound; 3335 } 3336 } 3337 3338 static void 3339 mlxsw_sp_nexthop_group_refresh(struct mlxsw_sp *mlxsw_sp, 3340 struct mlxsw_sp_nexthop_group *nh_grp) 3341 { 3342 u16 ecmp_size, old_ecmp_size; 3343 struct mlxsw_sp_nexthop *nh; 3344 bool offload_change = false; 3345 u32 adj_index; 3346 bool old_adj_index_valid; 3347 u32 old_adj_index; 3348 int i; 3349 int err; 3350 3351 if (!nh_grp->gateway) { 3352 mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp); 3353 return; 3354 } 3355 3356 for (i = 0; i < nh_grp->count; i++) { 3357 nh = &nh_grp->nexthops[i]; 3358 3359 if (nh->should_offload != nh->offloaded) { 3360 offload_change = true; 3361 if (nh->should_offload) 3362 nh->update = 1; 3363 } 3364 } 3365 if (!offload_change) { 3366 /* Nothing was added or removed, so no need to reallocate. Just 3367 * update MAC on existing adjacency indexes. 3368 */ 3369 err = mlxsw_sp_nexthop_group_update(mlxsw_sp, nh_grp, false); 3370 if (err) { 3371 dev_warn(mlxsw_sp->bus_info->dev, "Failed to update neigh MAC in adjacency table.\n"); 3372 goto set_trap; 3373 } 3374 return; 3375 } 3376 mlxsw_sp_nexthop_group_normalize(nh_grp); 3377 if (!nh_grp->sum_norm_weight) 3378 /* No neigh of this group is connected so we just set 3379 * the trap and let everthing flow through kernel. 3380 */ 3381 goto set_trap; 3382 3383 ecmp_size = nh_grp->sum_norm_weight; 3384 err = mlxsw_sp_fix_adj_grp_size(mlxsw_sp, &ecmp_size); 3385 if (err) 3386 /* No valid allocation size available. */ 3387 goto set_trap; 3388 3389 err = mlxsw_sp_kvdl_alloc(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 3390 ecmp_size, &adj_index); 3391 if (err) { 3392 /* We ran out of KVD linear space, just set the 3393 * trap and let everything flow through kernel. 3394 */ 3395 dev_warn(mlxsw_sp->bus_info->dev, "Failed to allocate KVD linear area for nexthop group.\n"); 3396 goto set_trap; 3397 } 3398 old_adj_index_valid = nh_grp->adj_index_valid; 3399 old_adj_index = nh_grp->adj_index; 3400 old_ecmp_size = nh_grp->ecmp_size; 3401 nh_grp->adj_index_valid = 1; 3402 nh_grp->adj_index = adj_index; 3403 nh_grp->ecmp_size = ecmp_size; 3404 mlxsw_sp_nexthop_group_rebalance(nh_grp); 3405 err = mlxsw_sp_nexthop_group_update(mlxsw_sp, nh_grp, true); 3406 if (err) { 3407 dev_warn(mlxsw_sp->bus_info->dev, "Failed to update neigh MAC in adjacency table.\n"); 3408 goto set_trap; 3409 } 3410 3411 if (!old_adj_index_valid) { 3412 /* The trap was set for fib entries, so we have to call 3413 * fib entry update to unset it and use adjacency index. 3414 */ 3415 err = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp); 3416 if (err) { 3417 dev_warn(mlxsw_sp->bus_info->dev, "Failed to add adjacency index to fib entries.\n"); 3418 goto set_trap; 3419 } 3420 return; 3421 } 3422 3423 err = mlxsw_sp_adj_index_mass_update(mlxsw_sp, nh_grp, 3424 old_adj_index, old_ecmp_size); 3425 mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 3426 old_ecmp_size, old_adj_index); 3427 if (err) { 3428 dev_warn(mlxsw_sp->bus_info->dev, "Failed to mass-update adjacency index for nexthop group.\n"); 3429 goto set_trap; 3430 } 3431 3432 /* Offload state within the group changed, so update the flags. */ 3433 mlxsw_sp_nexthop_fib_entries_refresh(nh_grp); 3434 3435 return; 3436 3437 set_trap: 3438 old_adj_index_valid = nh_grp->adj_index_valid; 3439 nh_grp->adj_index_valid = 0; 3440 for (i = 0; i < nh_grp->count; i++) { 3441 nh = &nh_grp->nexthops[i]; 3442 nh->offloaded = 0; 3443 } 3444 err = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp); 3445 if (err) 3446 dev_warn(mlxsw_sp->bus_info->dev, "Failed to set traps for fib entries.\n"); 3447 if (old_adj_index_valid) 3448 mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 3449 nh_grp->ecmp_size, nh_grp->adj_index); 3450 } 3451 3452 static void __mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp_nexthop *nh, 3453 bool removing) 3454 { 3455 if (!removing) 3456 nh->should_offload = 1; 3457 else 3458 nh->should_offload = 0; 3459 nh->update = 1; 3460 } 3461 3462 static void 3463 mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp *mlxsw_sp, 3464 struct mlxsw_sp_neigh_entry *neigh_entry, 3465 bool removing) 3466 { 3467 struct mlxsw_sp_nexthop *nh; 3468 3469 list_for_each_entry(nh, &neigh_entry->nexthop_list, 3470 neigh_list_node) { 3471 __mlxsw_sp_nexthop_neigh_update(nh, removing); 3472 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nh_grp); 3473 } 3474 } 3475 3476 static void mlxsw_sp_nexthop_rif_init(struct mlxsw_sp_nexthop *nh, 3477 struct mlxsw_sp_rif *rif) 3478 { 3479 if (nh->rif) 3480 return; 3481 3482 nh->rif = rif; 3483 list_add(&nh->rif_list_node, &rif->nexthop_list); 3484 } 3485 3486 static void mlxsw_sp_nexthop_rif_fini(struct mlxsw_sp_nexthop *nh) 3487 { 3488 if (!nh->rif) 3489 return; 3490 3491 list_del(&nh->rif_list_node); 3492 nh->rif = NULL; 3493 } 3494 3495 static int mlxsw_sp_nexthop_neigh_init(struct mlxsw_sp *mlxsw_sp, 3496 struct mlxsw_sp_nexthop *nh) 3497 { 3498 struct mlxsw_sp_neigh_entry *neigh_entry; 3499 struct neighbour *n; 3500 u8 nud_state, dead; 3501 int err; 3502 3503 if (!nh->nh_grp->gateway || nh->neigh_entry) 3504 return 0; 3505 3506 /* Take a reference of neigh here ensuring that neigh would 3507 * not be destructed before the nexthop entry is finished. 3508 * The reference is taken either in neigh_lookup() or 3509 * in neigh_create() in case n is not found. 3510 */ 3511 n = neigh_lookup(nh->nh_grp->neigh_tbl, &nh->gw_addr, nh->rif->dev); 3512 if (!n) { 3513 n = neigh_create(nh->nh_grp->neigh_tbl, &nh->gw_addr, 3514 nh->rif->dev); 3515 if (IS_ERR(n)) 3516 return PTR_ERR(n); 3517 neigh_event_send(n, NULL); 3518 } 3519 neigh_entry = mlxsw_sp_neigh_entry_lookup(mlxsw_sp, n); 3520 if (!neigh_entry) { 3521 neigh_entry = mlxsw_sp_neigh_entry_create(mlxsw_sp, n); 3522 if (IS_ERR(neigh_entry)) { 3523 err = -EINVAL; 3524 goto err_neigh_entry_create; 3525 } 3526 } 3527 3528 /* If that is the first nexthop connected to that neigh, add to 3529 * nexthop_neighs_list 3530 */ 3531 if (list_empty(&neigh_entry->nexthop_list)) 3532 list_add_tail(&neigh_entry->nexthop_neighs_list_node, 3533 &mlxsw_sp->router->nexthop_neighs_list); 3534 3535 nh->neigh_entry = neigh_entry; 3536 list_add_tail(&nh->neigh_list_node, &neigh_entry->nexthop_list); 3537 read_lock_bh(&n->lock); 3538 nud_state = n->nud_state; 3539 dead = n->dead; 3540 read_unlock_bh(&n->lock); 3541 __mlxsw_sp_nexthop_neigh_update(nh, !(nud_state & NUD_VALID && !dead)); 3542 3543 return 0; 3544 3545 err_neigh_entry_create: 3546 neigh_release(n); 3547 return err; 3548 } 3549 3550 static void mlxsw_sp_nexthop_neigh_fini(struct mlxsw_sp *mlxsw_sp, 3551 struct mlxsw_sp_nexthop *nh) 3552 { 3553 struct mlxsw_sp_neigh_entry *neigh_entry = nh->neigh_entry; 3554 struct neighbour *n; 3555 3556 if (!neigh_entry) 3557 return; 3558 n = neigh_entry->key.n; 3559 3560 __mlxsw_sp_nexthop_neigh_update(nh, true); 3561 list_del(&nh->neigh_list_node); 3562 nh->neigh_entry = NULL; 3563 3564 /* If that is the last nexthop connected to that neigh, remove from 3565 * nexthop_neighs_list 3566 */ 3567 if (list_empty(&neigh_entry->nexthop_list)) 3568 list_del(&neigh_entry->nexthop_neighs_list_node); 3569 3570 if (!neigh_entry->connected && list_empty(&neigh_entry->nexthop_list)) 3571 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry); 3572 3573 neigh_release(n); 3574 } 3575 3576 static bool mlxsw_sp_ipip_netdev_ul_up(struct net_device *ol_dev) 3577 { 3578 struct net_device *ul_dev = __mlxsw_sp_ipip_netdev_ul_dev_get(ol_dev); 3579 3580 return ul_dev ? (ul_dev->flags & IFF_UP) : true; 3581 } 3582 3583 static void mlxsw_sp_nexthop_ipip_init(struct mlxsw_sp *mlxsw_sp, 3584 struct mlxsw_sp_nexthop *nh, 3585 struct mlxsw_sp_ipip_entry *ipip_entry) 3586 { 3587 bool removing; 3588 3589 if (!nh->nh_grp->gateway || nh->ipip_entry) 3590 return; 3591 3592 nh->ipip_entry = ipip_entry; 3593 removing = !mlxsw_sp_ipip_netdev_ul_up(ipip_entry->ol_dev); 3594 __mlxsw_sp_nexthop_neigh_update(nh, removing); 3595 mlxsw_sp_nexthop_rif_init(nh, &ipip_entry->ol_lb->common); 3596 } 3597 3598 static void mlxsw_sp_nexthop_ipip_fini(struct mlxsw_sp *mlxsw_sp, 3599 struct mlxsw_sp_nexthop *nh) 3600 { 3601 struct mlxsw_sp_ipip_entry *ipip_entry = nh->ipip_entry; 3602 3603 if (!ipip_entry) 3604 return; 3605 3606 __mlxsw_sp_nexthop_neigh_update(nh, true); 3607 nh->ipip_entry = NULL; 3608 } 3609 3610 static bool mlxsw_sp_nexthop4_ipip_type(const struct mlxsw_sp *mlxsw_sp, 3611 const struct fib_nh *fib_nh, 3612 enum mlxsw_sp_ipip_type *p_ipipt) 3613 { 3614 struct net_device *dev = fib_nh->fib_nh_dev; 3615 3616 return dev && 3617 fib_nh->nh_parent->fib_type == RTN_UNICAST && 3618 mlxsw_sp_netdev_ipip_type(mlxsw_sp, dev, p_ipipt); 3619 } 3620 3621 static void mlxsw_sp_nexthop_type_fini(struct mlxsw_sp *mlxsw_sp, 3622 struct mlxsw_sp_nexthop *nh) 3623 { 3624 switch (nh->type) { 3625 case MLXSW_SP_NEXTHOP_TYPE_ETH: 3626 mlxsw_sp_nexthop_neigh_fini(mlxsw_sp, nh); 3627 mlxsw_sp_nexthop_rif_fini(nh); 3628 break; 3629 case MLXSW_SP_NEXTHOP_TYPE_IPIP: 3630 mlxsw_sp_nexthop_rif_fini(nh); 3631 mlxsw_sp_nexthop_ipip_fini(mlxsw_sp, nh); 3632 break; 3633 } 3634 } 3635 3636 static int mlxsw_sp_nexthop4_type_init(struct mlxsw_sp *mlxsw_sp, 3637 struct mlxsw_sp_nexthop *nh, 3638 struct fib_nh *fib_nh) 3639 { 3640 const struct mlxsw_sp_ipip_ops *ipip_ops; 3641 struct net_device *dev = fib_nh->fib_nh_dev; 3642 struct mlxsw_sp_ipip_entry *ipip_entry; 3643 struct mlxsw_sp_rif *rif; 3644 int err; 3645 3646 ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, dev); 3647 if (ipip_entry) { 3648 ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt]; 3649 if (ipip_ops->can_offload(mlxsw_sp, dev, 3650 MLXSW_SP_L3_PROTO_IPV4)) { 3651 nh->type = MLXSW_SP_NEXTHOP_TYPE_IPIP; 3652 mlxsw_sp_nexthop_ipip_init(mlxsw_sp, nh, ipip_entry); 3653 return 0; 3654 } 3655 } 3656 3657 nh->type = MLXSW_SP_NEXTHOP_TYPE_ETH; 3658 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev); 3659 if (!rif) 3660 return 0; 3661 3662 mlxsw_sp_nexthop_rif_init(nh, rif); 3663 err = mlxsw_sp_nexthop_neigh_init(mlxsw_sp, nh); 3664 if (err) 3665 goto err_neigh_init; 3666 3667 return 0; 3668 3669 err_neigh_init: 3670 mlxsw_sp_nexthop_rif_fini(nh); 3671 return err; 3672 } 3673 3674 static void mlxsw_sp_nexthop4_type_fini(struct mlxsw_sp *mlxsw_sp, 3675 struct mlxsw_sp_nexthop *nh) 3676 { 3677 mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh); 3678 } 3679 3680 static int mlxsw_sp_nexthop4_init(struct mlxsw_sp *mlxsw_sp, 3681 struct mlxsw_sp_nexthop_group *nh_grp, 3682 struct mlxsw_sp_nexthop *nh, 3683 struct fib_nh *fib_nh) 3684 { 3685 struct net_device *dev = fib_nh->fib_nh_dev; 3686 struct in_device *in_dev; 3687 int err; 3688 3689 nh->nh_grp = nh_grp; 3690 nh->key.fib_nh = fib_nh; 3691 #ifdef CONFIG_IP_ROUTE_MULTIPATH 3692 nh->nh_weight = fib_nh->fib_nh_weight; 3693 #else 3694 nh->nh_weight = 1; 3695 #endif 3696 memcpy(&nh->gw_addr, &fib_nh->fib_nh_gw4, sizeof(fib_nh->fib_nh_gw4)); 3697 err = mlxsw_sp_nexthop_insert(mlxsw_sp, nh); 3698 if (err) 3699 return err; 3700 3701 mlxsw_sp_nexthop_counter_alloc(mlxsw_sp, nh); 3702 list_add_tail(&nh->router_list_node, &mlxsw_sp->router->nexthop_list); 3703 3704 if (!dev) 3705 return 0; 3706 3707 in_dev = __in_dev_get_rtnl(dev); 3708 if (in_dev && IN_DEV_IGNORE_ROUTES_WITH_LINKDOWN(in_dev) && 3709 fib_nh->fib_nh_flags & RTNH_F_LINKDOWN) 3710 return 0; 3711 3712 err = mlxsw_sp_nexthop4_type_init(mlxsw_sp, nh, fib_nh); 3713 if (err) 3714 goto err_nexthop_neigh_init; 3715 3716 return 0; 3717 3718 err_nexthop_neigh_init: 3719 mlxsw_sp_nexthop_remove(mlxsw_sp, nh); 3720 return err; 3721 } 3722 3723 static void mlxsw_sp_nexthop4_fini(struct mlxsw_sp *mlxsw_sp, 3724 struct mlxsw_sp_nexthop *nh) 3725 { 3726 mlxsw_sp_nexthop4_type_fini(mlxsw_sp, nh); 3727 list_del(&nh->router_list_node); 3728 mlxsw_sp_nexthop_counter_free(mlxsw_sp, nh); 3729 mlxsw_sp_nexthop_remove(mlxsw_sp, nh); 3730 } 3731 3732 static void mlxsw_sp_nexthop4_event(struct mlxsw_sp *mlxsw_sp, 3733 unsigned long event, struct fib_nh *fib_nh) 3734 { 3735 struct mlxsw_sp_nexthop_key key; 3736 struct mlxsw_sp_nexthop *nh; 3737 3738 if (mlxsw_sp->router->aborted) 3739 return; 3740 3741 key.fib_nh = fib_nh; 3742 nh = mlxsw_sp_nexthop_lookup(mlxsw_sp, key); 3743 if (WARN_ON_ONCE(!nh)) 3744 return; 3745 3746 switch (event) { 3747 case FIB_EVENT_NH_ADD: 3748 mlxsw_sp_nexthop4_type_init(mlxsw_sp, nh, fib_nh); 3749 break; 3750 case FIB_EVENT_NH_DEL: 3751 mlxsw_sp_nexthop4_type_fini(mlxsw_sp, nh); 3752 break; 3753 } 3754 3755 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nh_grp); 3756 } 3757 3758 static void mlxsw_sp_nexthop_rif_update(struct mlxsw_sp *mlxsw_sp, 3759 struct mlxsw_sp_rif *rif) 3760 { 3761 struct mlxsw_sp_nexthop *nh; 3762 bool removing; 3763 3764 list_for_each_entry(nh, &rif->nexthop_list, rif_list_node) { 3765 switch (nh->type) { 3766 case MLXSW_SP_NEXTHOP_TYPE_ETH: 3767 removing = false; 3768 break; 3769 case MLXSW_SP_NEXTHOP_TYPE_IPIP: 3770 removing = !mlxsw_sp_ipip_netdev_ul_up(rif->dev); 3771 break; 3772 default: 3773 WARN_ON(1); 3774 continue; 3775 } 3776 3777 __mlxsw_sp_nexthop_neigh_update(nh, removing); 3778 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nh_grp); 3779 } 3780 } 3781 3782 static void mlxsw_sp_nexthop_rif_migrate(struct mlxsw_sp *mlxsw_sp, 3783 struct mlxsw_sp_rif *old_rif, 3784 struct mlxsw_sp_rif *new_rif) 3785 { 3786 struct mlxsw_sp_nexthop *nh; 3787 3788 list_splice_init(&old_rif->nexthop_list, &new_rif->nexthop_list); 3789 list_for_each_entry(nh, &new_rif->nexthop_list, rif_list_node) 3790 nh->rif = new_rif; 3791 mlxsw_sp_nexthop_rif_update(mlxsw_sp, new_rif); 3792 } 3793 3794 static void mlxsw_sp_nexthop_rif_gone_sync(struct mlxsw_sp *mlxsw_sp, 3795 struct mlxsw_sp_rif *rif) 3796 { 3797 struct mlxsw_sp_nexthop *nh, *tmp; 3798 3799 list_for_each_entry_safe(nh, tmp, &rif->nexthop_list, rif_list_node) { 3800 mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh); 3801 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nh_grp); 3802 } 3803 } 3804 3805 static bool mlxsw_sp_fi_is_gateway(const struct mlxsw_sp *mlxsw_sp, 3806 const struct fib_info *fi) 3807 { 3808 return fi->fib_nh->fib_nh_scope == RT_SCOPE_LINK || 3809 mlxsw_sp_nexthop4_ipip_type(mlxsw_sp, fi->fib_nh, NULL); 3810 } 3811 3812 static struct mlxsw_sp_nexthop_group * 3813 mlxsw_sp_nexthop4_group_create(struct mlxsw_sp *mlxsw_sp, struct fib_info *fi) 3814 { 3815 struct mlxsw_sp_nexthop_group *nh_grp; 3816 struct mlxsw_sp_nexthop *nh; 3817 struct fib_nh *fib_nh; 3818 int i; 3819 int err; 3820 3821 nh_grp = kzalloc(struct_size(nh_grp, nexthops, fi->fib_nhs), 3822 GFP_KERNEL); 3823 if (!nh_grp) 3824 return ERR_PTR(-ENOMEM); 3825 nh_grp->priv = fi; 3826 INIT_LIST_HEAD(&nh_grp->fib_list); 3827 nh_grp->neigh_tbl = &arp_tbl; 3828 3829 nh_grp->gateway = mlxsw_sp_fi_is_gateway(mlxsw_sp, fi); 3830 nh_grp->count = fi->fib_nhs; 3831 fib_info_hold(fi); 3832 for (i = 0; i < nh_grp->count; i++) { 3833 nh = &nh_grp->nexthops[i]; 3834 fib_nh = &fi->fib_nh[i]; 3835 err = mlxsw_sp_nexthop4_init(mlxsw_sp, nh_grp, nh, fib_nh); 3836 if (err) 3837 goto err_nexthop4_init; 3838 } 3839 err = mlxsw_sp_nexthop_group_insert(mlxsw_sp, nh_grp); 3840 if (err) 3841 goto err_nexthop_group_insert; 3842 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp); 3843 return nh_grp; 3844 3845 err_nexthop_group_insert: 3846 err_nexthop4_init: 3847 for (i--; i >= 0; i--) { 3848 nh = &nh_grp->nexthops[i]; 3849 mlxsw_sp_nexthop4_fini(mlxsw_sp, nh); 3850 } 3851 fib_info_put(fi); 3852 kfree(nh_grp); 3853 return ERR_PTR(err); 3854 } 3855 3856 static void 3857 mlxsw_sp_nexthop4_group_destroy(struct mlxsw_sp *mlxsw_sp, 3858 struct mlxsw_sp_nexthop_group *nh_grp) 3859 { 3860 struct mlxsw_sp_nexthop *nh; 3861 int i; 3862 3863 mlxsw_sp_nexthop_group_remove(mlxsw_sp, nh_grp); 3864 for (i = 0; i < nh_grp->count; i++) { 3865 nh = &nh_grp->nexthops[i]; 3866 mlxsw_sp_nexthop4_fini(mlxsw_sp, nh); 3867 } 3868 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp); 3869 WARN_ON_ONCE(nh_grp->adj_index_valid); 3870 fib_info_put(mlxsw_sp_nexthop4_group_fi(nh_grp)); 3871 kfree(nh_grp); 3872 } 3873 3874 static int mlxsw_sp_nexthop4_group_get(struct mlxsw_sp *mlxsw_sp, 3875 struct mlxsw_sp_fib_entry *fib_entry, 3876 struct fib_info *fi) 3877 { 3878 struct mlxsw_sp_nexthop_group *nh_grp; 3879 3880 nh_grp = mlxsw_sp_nexthop4_group_lookup(mlxsw_sp, fi); 3881 if (!nh_grp) { 3882 nh_grp = mlxsw_sp_nexthop4_group_create(mlxsw_sp, fi); 3883 if (IS_ERR(nh_grp)) 3884 return PTR_ERR(nh_grp); 3885 } 3886 list_add_tail(&fib_entry->nexthop_group_node, &nh_grp->fib_list); 3887 fib_entry->nh_group = nh_grp; 3888 return 0; 3889 } 3890 3891 static void mlxsw_sp_nexthop4_group_put(struct mlxsw_sp *mlxsw_sp, 3892 struct mlxsw_sp_fib_entry *fib_entry) 3893 { 3894 struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group; 3895 3896 list_del(&fib_entry->nexthop_group_node); 3897 if (!list_empty(&nh_grp->fib_list)) 3898 return; 3899 mlxsw_sp_nexthop4_group_destroy(mlxsw_sp, nh_grp); 3900 } 3901 3902 static bool 3903 mlxsw_sp_fib4_entry_should_offload(const struct mlxsw_sp_fib_entry *fib_entry) 3904 { 3905 struct mlxsw_sp_fib4_entry *fib4_entry; 3906 3907 fib4_entry = container_of(fib_entry, struct mlxsw_sp_fib4_entry, 3908 common); 3909 return !fib4_entry->tos; 3910 } 3911 3912 static bool 3913 mlxsw_sp_fib_entry_should_offload(const struct mlxsw_sp_fib_entry *fib_entry) 3914 { 3915 struct mlxsw_sp_nexthop_group *nh_group = fib_entry->nh_group; 3916 3917 switch (fib_entry->fib_node->fib->proto) { 3918 case MLXSW_SP_L3_PROTO_IPV4: 3919 if (!mlxsw_sp_fib4_entry_should_offload(fib_entry)) 3920 return false; 3921 break; 3922 case MLXSW_SP_L3_PROTO_IPV6: 3923 break; 3924 } 3925 3926 switch (fib_entry->type) { 3927 case MLXSW_SP_FIB_ENTRY_TYPE_REMOTE: 3928 return !!nh_group->adj_index_valid; 3929 case MLXSW_SP_FIB_ENTRY_TYPE_LOCAL: 3930 return !!nh_group->nh_rif; 3931 case MLXSW_SP_FIB_ENTRY_TYPE_BLACKHOLE: 3932 case MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP: 3933 case MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP: 3934 return true; 3935 default: 3936 return false; 3937 } 3938 } 3939 3940 static struct mlxsw_sp_nexthop * 3941 mlxsw_sp_rt6_nexthop(struct mlxsw_sp_nexthop_group *nh_grp, 3942 const struct mlxsw_sp_rt6 *mlxsw_sp_rt6) 3943 { 3944 int i; 3945 3946 for (i = 0; i < nh_grp->count; i++) { 3947 struct mlxsw_sp_nexthop *nh = &nh_grp->nexthops[i]; 3948 struct fib6_info *rt = mlxsw_sp_rt6->rt; 3949 3950 if (nh->rif && nh->rif->dev == rt->fib6_nh.fib_nh_dev && 3951 ipv6_addr_equal((const struct in6_addr *) &nh->gw_addr, 3952 &rt->fib6_nh.fib_nh_gw6)) 3953 return nh; 3954 continue; 3955 } 3956 3957 return NULL; 3958 } 3959 3960 static void 3961 mlxsw_sp_fib4_entry_offload_set(struct mlxsw_sp_fib_entry *fib_entry) 3962 { 3963 struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group; 3964 int i; 3965 3966 if (fib_entry->type == MLXSW_SP_FIB_ENTRY_TYPE_LOCAL || 3967 fib_entry->type == MLXSW_SP_FIB_ENTRY_TYPE_BLACKHOLE || 3968 fib_entry->type == MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP || 3969 fib_entry->type == MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP) { 3970 nh_grp->nexthops->key.fib_nh->fib_nh_flags |= RTNH_F_OFFLOAD; 3971 return; 3972 } 3973 3974 for (i = 0; i < nh_grp->count; i++) { 3975 struct mlxsw_sp_nexthop *nh = &nh_grp->nexthops[i]; 3976 3977 if (nh->offloaded) 3978 nh->key.fib_nh->fib_nh_flags |= RTNH_F_OFFLOAD; 3979 else 3980 nh->key.fib_nh->fib_nh_flags &= ~RTNH_F_OFFLOAD; 3981 } 3982 } 3983 3984 static void 3985 mlxsw_sp_fib4_entry_offload_unset(struct mlxsw_sp_fib_entry *fib_entry) 3986 { 3987 struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group; 3988 int i; 3989 3990 if (!list_is_singular(&nh_grp->fib_list)) 3991 return; 3992 3993 for (i = 0; i < nh_grp->count; i++) { 3994 struct mlxsw_sp_nexthop *nh = &nh_grp->nexthops[i]; 3995 3996 nh->key.fib_nh->fib_nh_flags &= ~RTNH_F_OFFLOAD; 3997 } 3998 } 3999 4000 static void 4001 mlxsw_sp_fib6_entry_offload_set(struct mlxsw_sp_fib_entry *fib_entry) 4002 { 4003 struct mlxsw_sp_fib6_entry *fib6_entry; 4004 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 4005 4006 fib6_entry = container_of(fib_entry, struct mlxsw_sp_fib6_entry, 4007 common); 4008 4009 if (fib_entry->type == MLXSW_SP_FIB_ENTRY_TYPE_LOCAL || 4010 fib_entry->type == MLXSW_SP_FIB_ENTRY_TYPE_BLACKHOLE) { 4011 list_first_entry(&fib6_entry->rt6_list, struct mlxsw_sp_rt6, 4012 list)->rt->fib6_nh.fib_nh_flags |= RTNH_F_OFFLOAD; 4013 return; 4014 } 4015 4016 list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) { 4017 struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group; 4018 struct fib6_nh *fib6_nh = &mlxsw_sp_rt6->rt->fib6_nh; 4019 struct mlxsw_sp_nexthop *nh; 4020 4021 nh = mlxsw_sp_rt6_nexthop(nh_grp, mlxsw_sp_rt6); 4022 if (nh && nh->offloaded) 4023 fib6_nh->fib_nh_flags |= RTNH_F_OFFLOAD; 4024 else 4025 fib6_nh->fib_nh_flags &= ~RTNH_F_OFFLOAD; 4026 } 4027 } 4028 4029 static void 4030 mlxsw_sp_fib6_entry_offload_unset(struct mlxsw_sp_fib_entry *fib_entry) 4031 { 4032 struct mlxsw_sp_fib6_entry *fib6_entry; 4033 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 4034 4035 fib6_entry = container_of(fib_entry, struct mlxsw_sp_fib6_entry, 4036 common); 4037 list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) { 4038 struct fib6_info *rt = mlxsw_sp_rt6->rt; 4039 4040 rt->fib6_nh.fib_nh_flags &= ~RTNH_F_OFFLOAD; 4041 } 4042 } 4043 4044 static void mlxsw_sp_fib_entry_offload_set(struct mlxsw_sp_fib_entry *fib_entry) 4045 { 4046 switch (fib_entry->fib_node->fib->proto) { 4047 case MLXSW_SP_L3_PROTO_IPV4: 4048 mlxsw_sp_fib4_entry_offload_set(fib_entry); 4049 break; 4050 case MLXSW_SP_L3_PROTO_IPV6: 4051 mlxsw_sp_fib6_entry_offload_set(fib_entry); 4052 break; 4053 } 4054 } 4055 4056 static void 4057 mlxsw_sp_fib_entry_offload_unset(struct mlxsw_sp_fib_entry *fib_entry) 4058 { 4059 switch (fib_entry->fib_node->fib->proto) { 4060 case MLXSW_SP_L3_PROTO_IPV4: 4061 mlxsw_sp_fib4_entry_offload_unset(fib_entry); 4062 break; 4063 case MLXSW_SP_L3_PROTO_IPV6: 4064 mlxsw_sp_fib6_entry_offload_unset(fib_entry); 4065 break; 4066 } 4067 } 4068 4069 static void 4070 mlxsw_sp_fib_entry_offload_refresh(struct mlxsw_sp_fib_entry *fib_entry, 4071 enum mlxsw_reg_ralue_op op, int err) 4072 { 4073 switch (op) { 4074 case MLXSW_REG_RALUE_OP_WRITE_DELETE: 4075 return mlxsw_sp_fib_entry_offload_unset(fib_entry); 4076 case MLXSW_REG_RALUE_OP_WRITE_WRITE: 4077 if (err) 4078 return; 4079 if (mlxsw_sp_fib_entry_should_offload(fib_entry)) 4080 mlxsw_sp_fib_entry_offload_set(fib_entry); 4081 else 4082 mlxsw_sp_fib_entry_offload_unset(fib_entry); 4083 return; 4084 default: 4085 return; 4086 } 4087 } 4088 4089 static void 4090 mlxsw_sp_fib_entry_ralue_pack(char *ralue_pl, 4091 const struct mlxsw_sp_fib_entry *fib_entry, 4092 enum mlxsw_reg_ralue_op op) 4093 { 4094 struct mlxsw_sp_fib *fib = fib_entry->fib_node->fib; 4095 enum mlxsw_reg_ralxx_protocol proto; 4096 u32 *p_dip; 4097 4098 proto = (enum mlxsw_reg_ralxx_protocol) fib->proto; 4099 4100 switch (fib->proto) { 4101 case MLXSW_SP_L3_PROTO_IPV4: 4102 p_dip = (u32 *) fib_entry->fib_node->key.addr; 4103 mlxsw_reg_ralue_pack4(ralue_pl, proto, op, fib->vr->id, 4104 fib_entry->fib_node->key.prefix_len, 4105 *p_dip); 4106 break; 4107 case MLXSW_SP_L3_PROTO_IPV6: 4108 mlxsw_reg_ralue_pack6(ralue_pl, proto, op, fib->vr->id, 4109 fib_entry->fib_node->key.prefix_len, 4110 fib_entry->fib_node->key.addr); 4111 break; 4112 } 4113 } 4114 4115 static int mlxsw_sp_fib_entry_op_remote(struct mlxsw_sp *mlxsw_sp, 4116 struct mlxsw_sp_fib_entry *fib_entry, 4117 enum mlxsw_reg_ralue_op op) 4118 { 4119 char ralue_pl[MLXSW_REG_RALUE_LEN]; 4120 enum mlxsw_reg_ralue_trap_action trap_action; 4121 u16 trap_id = 0; 4122 u32 adjacency_index = 0; 4123 u16 ecmp_size = 0; 4124 4125 /* In case the nexthop group adjacency index is valid, use it 4126 * with provided ECMP size. Otherwise, setup trap and pass 4127 * traffic to kernel. 4128 */ 4129 if (mlxsw_sp_fib_entry_should_offload(fib_entry)) { 4130 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_NOP; 4131 adjacency_index = fib_entry->nh_group->adj_index; 4132 ecmp_size = fib_entry->nh_group->ecmp_size; 4133 } else { 4134 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_TRAP; 4135 trap_id = MLXSW_TRAP_ID_RTR_INGRESS0; 4136 } 4137 4138 mlxsw_sp_fib_entry_ralue_pack(ralue_pl, fib_entry, op); 4139 mlxsw_reg_ralue_act_remote_pack(ralue_pl, trap_action, trap_id, 4140 adjacency_index, ecmp_size); 4141 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl); 4142 } 4143 4144 static int mlxsw_sp_fib_entry_op_local(struct mlxsw_sp *mlxsw_sp, 4145 struct mlxsw_sp_fib_entry *fib_entry, 4146 enum mlxsw_reg_ralue_op op) 4147 { 4148 struct mlxsw_sp_rif *rif = fib_entry->nh_group->nh_rif; 4149 enum mlxsw_reg_ralue_trap_action trap_action; 4150 char ralue_pl[MLXSW_REG_RALUE_LEN]; 4151 u16 trap_id = 0; 4152 u16 rif_index = 0; 4153 4154 if (mlxsw_sp_fib_entry_should_offload(fib_entry)) { 4155 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_NOP; 4156 rif_index = rif->rif_index; 4157 } else { 4158 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_TRAP; 4159 trap_id = MLXSW_TRAP_ID_RTR_INGRESS0; 4160 } 4161 4162 mlxsw_sp_fib_entry_ralue_pack(ralue_pl, fib_entry, op); 4163 mlxsw_reg_ralue_act_local_pack(ralue_pl, trap_action, trap_id, 4164 rif_index); 4165 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl); 4166 } 4167 4168 static int mlxsw_sp_fib_entry_op_trap(struct mlxsw_sp *mlxsw_sp, 4169 struct mlxsw_sp_fib_entry *fib_entry, 4170 enum mlxsw_reg_ralue_op op) 4171 { 4172 char ralue_pl[MLXSW_REG_RALUE_LEN]; 4173 4174 mlxsw_sp_fib_entry_ralue_pack(ralue_pl, fib_entry, op); 4175 mlxsw_reg_ralue_act_ip2me_pack(ralue_pl); 4176 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl); 4177 } 4178 4179 static int mlxsw_sp_fib_entry_op_blackhole(struct mlxsw_sp *mlxsw_sp, 4180 struct mlxsw_sp_fib_entry *fib_entry, 4181 enum mlxsw_reg_ralue_op op) 4182 { 4183 enum mlxsw_reg_ralue_trap_action trap_action; 4184 char ralue_pl[MLXSW_REG_RALUE_LEN]; 4185 4186 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_DISCARD_ERROR; 4187 mlxsw_sp_fib_entry_ralue_pack(ralue_pl, fib_entry, op); 4188 mlxsw_reg_ralue_act_local_pack(ralue_pl, trap_action, 0, 0); 4189 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl); 4190 } 4191 4192 static int 4193 mlxsw_sp_fib_entry_op_ipip_decap(struct mlxsw_sp *mlxsw_sp, 4194 struct mlxsw_sp_fib_entry *fib_entry, 4195 enum mlxsw_reg_ralue_op op) 4196 { 4197 struct mlxsw_sp_ipip_entry *ipip_entry = fib_entry->decap.ipip_entry; 4198 const struct mlxsw_sp_ipip_ops *ipip_ops; 4199 4200 if (WARN_ON(!ipip_entry)) 4201 return -EINVAL; 4202 4203 ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt]; 4204 return ipip_ops->fib_entry_op(mlxsw_sp, ipip_entry, op, 4205 fib_entry->decap.tunnel_index); 4206 } 4207 4208 static int mlxsw_sp_fib_entry_op_nve_decap(struct mlxsw_sp *mlxsw_sp, 4209 struct mlxsw_sp_fib_entry *fib_entry, 4210 enum mlxsw_reg_ralue_op op) 4211 { 4212 char ralue_pl[MLXSW_REG_RALUE_LEN]; 4213 4214 mlxsw_sp_fib_entry_ralue_pack(ralue_pl, fib_entry, op); 4215 mlxsw_reg_ralue_act_ip2me_tun_pack(ralue_pl, 4216 fib_entry->decap.tunnel_index); 4217 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl); 4218 } 4219 4220 static int __mlxsw_sp_fib_entry_op(struct mlxsw_sp *mlxsw_sp, 4221 struct mlxsw_sp_fib_entry *fib_entry, 4222 enum mlxsw_reg_ralue_op op) 4223 { 4224 switch (fib_entry->type) { 4225 case MLXSW_SP_FIB_ENTRY_TYPE_REMOTE: 4226 return mlxsw_sp_fib_entry_op_remote(mlxsw_sp, fib_entry, op); 4227 case MLXSW_SP_FIB_ENTRY_TYPE_LOCAL: 4228 return mlxsw_sp_fib_entry_op_local(mlxsw_sp, fib_entry, op); 4229 case MLXSW_SP_FIB_ENTRY_TYPE_TRAP: 4230 return mlxsw_sp_fib_entry_op_trap(mlxsw_sp, fib_entry, op); 4231 case MLXSW_SP_FIB_ENTRY_TYPE_BLACKHOLE: 4232 return mlxsw_sp_fib_entry_op_blackhole(mlxsw_sp, fib_entry, op); 4233 case MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP: 4234 return mlxsw_sp_fib_entry_op_ipip_decap(mlxsw_sp, 4235 fib_entry, op); 4236 case MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP: 4237 return mlxsw_sp_fib_entry_op_nve_decap(mlxsw_sp, fib_entry, op); 4238 } 4239 return -EINVAL; 4240 } 4241 4242 static int mlxsw_sp_fib_entry_op(struct mlxsw_sp *mlxsw_sp, 4243 struct mlxsw_sp_fib_entry *fib_entry, 4244 enum mlxsw_reg_ralue_op op) 4245 { 4246 int err = __mlxsw_sp_fib_entry_op(mlxsw_sp, fib_entry, op); 4247 4248 mlxsw_sp_fib_entry_offload_refresh(fib_entry, op, err); 4249 4250 return err; 4251 } 4252 4253 static int mlxsw_sp_fib_entry_update(struct mlxsw_sp *mlxsw_sp, 4254 struct mlxsw_sp_fib_entry *fib_entry) 4255 { 4256 return mlxsw_sp_fib_entry_op(mlxsw_sp, fib_entry, 4257 MLXSW_REG_RALUE_OP_WRITE_WRITE); 4258 } 4259 4260 static int mlxsw_sp_fib_entry_del(struct mlxsw_sp *mlxsw_sp, 4261 struct mlxsw_sp_fib_entry *fib_entry) 4262 { 4263 return mlxsw_sp_fib_entry_op(mlxsw_sp, fib_entry, 4264 MLXSW_REG_RALUE_OP_WRITE_DELETE); 4265 } 4266 4267 static int 4268 mlxsw_sp_fib4_entry_type_set(struct mlxsw_sp *mlxsw_sp, 4269 const struct fib_entry_notifier_info *fen_info, 4270 struct mlxsw_sp_fib_entry *fib_entry) 4271 { 4272 union mlxsw_sp_l3addr dip = { .addr4 = htonl(fen_info->dst) }; 4273 u32 tb_id = mlxsw_sp_fix_tb_id(fen_info->tb_id); 4274 struct net_device *dev = fen_info->fi->fib_dev; 4275 struct mlxsw_sp_ipip_entry *ipip_entry; 4276 struct fib_info *fi = fen_info->fi; 4277 4278 switch (fen_info->type) { 4279 case RTN_LOCAL: 4280 ipip_entry = mlxsw_sp_ipip_entry_find_by_decap(mlxsw_sp, dev, 4281 MLXSW_SP_L3_PROTO_IPV4, dip); 4282 if (ipip_entry && ipip_entry->ol_dev->flags & IFF_UP) { 4283 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP; 4284 return mlxsw_sp_fib_entry_decap_init(mlxsw_sp, 4285 fib_entry, 4286 ipip_entry); 4287 } 4288 if (mlxsw_sp_nve_ipv4_route_is_decap(mlxsw_sp, tb_id, 4289 dip.addr4)) { 4290 u32 t_index; 4291 4292 t_index = mlxsw_sp_nve_decap_tunnel_index_get(mlxsw_sp); 4293 fib_entry->decap.tunnel_index = t_index; 4294 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP; 4295 return 0; 4296 } 4297 /* fall through */ 4298 case RTN_BROADCAST: 4299 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP; 4300 return 0; 4301 case RTN_BLACKHOLE: 4302 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_BLACKHOLE; 4303 return 0; 4304 case RTN_UNREACHABLE: /* fall through */ 4305 case RTN_PROHIBIT: 4306 /* Packets hitting these routes need to be trapped, but 4307 * can do so with a lower priority than packets directed 4308 * at the host, so use action type local instead of trap. 4309 */ 4310 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL; 4311 return 0; 4312 case RTN_UNICAST: 4313 if (mlxsw_sp_fi_is_gateway(mlxsw_sp, fi)) 4314 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_REMOTE; 4315 else 4316 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL; 4317 return 0; 4318 default: 4319 return -EINVAL; 4320 } 4321 } 4322 4323 static struct mlxsw_sp_fib4_entry * 4324 mlxsw_sp_fib4_entry_create(struct mlxsw_sp *mlxsw_sp, 4325 struct mlxsw_sp_fib_node *fib_node, 4326 const struct fib_entry_notifier_info *fen_info) 4327 { 4328 struct mlxsw_sp_fib4_entry *fib4_entry; 4329 struct mlxsw_sp_fib_entry *fib_entry; 4330 int err; 4331 4332 fib4_entry = kzalloc(sizeof(*fib4_entry), GFP_KERNEL); 4333 if (!fib4_entry) 4334 return ERR_PTR(-ENOMEM); 4335 fib_entry = &fib4_entry->common; 4336 4337 err = mlxsw_sp_fib4_entry_type_set(mlxsw_sp, fen_info, fib_entry); 4338 if (err) 4339 goto err_fib4_entry_type_set; 4340 4341 err = mlxsw_sp_nexthop4_group_get(mlxsw_sp, fib_entry, fen_info->fi); 4342 if (err) 4343 goto err_nexthop4_group_get; 4344 4345 fib4_entry->prio = fen_info->fi->fib_priority; 4346 fib4_entry->tb_id = fen_info->tb_id; 4347 fib4_entry->type = fen_info->type; 4348 fib4_entry->tos = fen_info->tos; 4349 4350 fib_entry->fib_node = fib_node; 4351 4352 return fib4_entry; 4353 4354 err_nexthop4_group_get: 4355 err_fib4_entry_type_set: 4356 kfree(fib4_entry); 4357 return ERR_PTR(err); 4358 } 4359 4360 static void mlxsw_sp_fib4_entry_destroy(struct mlxsw_sp *mlxsw_sp, 4361 struct mlxsw_sp_fib4_entry *fib4_entry) 4362 { 4363 mlxsw_sp_nexthop4_group_put(mlxsw_sp, &fib4_entry->common); 4364 kfree(fib4_entry); 4365 } 4366 4367 static struct mlxsw_sp_fib4_entry * 4368 mlxsw_sp_fib4_entry_lookup(struct mlxsw_sp *mlxsw_sp, 4369 const struct fib_entry_notifier_info *fen_info) 4370 { 4371 struct mlxsw_sp_fib4_entry *fib4_entry; 4372 struct mlxsw_sp_fib_node *fib_node; 4373 struct mlxsw_sp_fib *fib; 4374 struct mlxsw_sp_vr *vr; 4375 4376 vr = mlxsw_sp_vr_find(mlxsw_sp, fen_info->tb_id); 4377 if (!vr) 4378 return NULL; 4379 fib = mlxsw_sp_vr_fib(vr, MLXSW_SP_L3_PROTO_IPV4); 4380 4381 fib_node = mlxsw_sp_fib_node_lookup(fib, &fen_info->dst, 4382 sizeof(fen_info->dst), 4383 fen_info->dst_len); 4384 if (!fib_node) 4385 return NULL; 4386 4387 list_for_each_entry(fib4_entry, &fib_node->entry_list, common.list) { 4388 if (fib4_entry->tb_id == fen_info->tb_id && 4389 fib4_entry->tos == fen_info->tos && 4390 fib4_entry->type == fen_info->type && 4391 mlxsw_sp_nexthop4_group_fi(fib4_entry->common.nh_group) == 4392 fen_info->fi) { 4393 return fib4_entry; 4394 } 4395 } 4396 4397 return NULL; 4398 } 4399 4400 static const struct rhashtable_params mlxsw_sp_fib_ht_params = { 4401 .key_offset = offsetof(struct mlxsw_sp_fib_node, key), 4402 .head_offset = offsetof(struct mlxsw_sp_fib_node, ht_node), 4403 .key_len = sizeof(struct mlxsw_sp_fib_key), 4404 .automatic_shrinking = true, 4405 }; 4406 4407 static int mlxsw_sp_fib_node_insert(struct mlxsw_sp_fib *fib, 4408 struct mlxsw_sp_fib_node *fib_node) 4409 { 4410 return rhashtable_insert_fast(&fib->ht, &fib_node->ht_node, 4411 mlxsw_sp_fib_ht_params); 4412 } 4413 4414 static void mlxsw_sp_fib_node_remove(struct mlxsw_sp_fib *fib, 4415 struct mlxsw_sp_fib_node *fib_node) 4416 { 4417 rhashtable_remove_fast(&fib->ht, &fib_node->ht_node, 4418 mlxsw_sp_fib_ht_params); 4419 } 4420 4421 static struct mlxsw_sp_fib_node * 4422 mlxsw_sp_fib_node_lookup(struct mlxsw_sp_fib *fib, const void *addr, 4423 size_t addr_len, unsigned char prefix_len) 4424 { 4425 struct mlxsw_sp_fib_key key; 4426 4427 memset(&key, 0, sizeof(key)); 4428 memcpy(key.addr, addr, addr_len); 4429 key.prefix_len = prefix_len; 4430 return rhashtable_lookup_fast(&fib->ht, &key, mlxsw_sp_fib_ht_params); 4431 } 4432 4433 static struct mlxsw_sp_fib_node * 4434 mlxsw_sp_fib_node_create(struct mlxsw_sp_fib *fib, const void *addr, 4435 size_t addr_len, unsigned char prefix_len) 4436 { 4437 struct mlxsw_sp_fib_node *fib_node; 4438 4439 fib_node = kzalloc(sizeof(*fib_node), GFP_KERNEL); 4440 if (!fib_node) 4441 return NULL; 4442 4443 INIT_LIST_HEAD(&fib_node->entry_list); 4444 list_add(&fib_node->list, &fib->node_list); 4445 memcpy(fib_node->key.addr, addr, addr_len); 4446 fib_node->key.prefix_len = prefix_len; 4447 4448 return fib_node; 4449 } 4450 4451 static void mlxsw_sp_fib_node_destroy(struct mlxsw_sp_fib_node *fib_node) 4452 { 4453 list_del(&fib_node->list); 4454 WARN_ON(!list_empty(&fib_node->entry_list)); 4455 kfree(fib_node); 4456 } 4457 4458 static bool 4459 mlxsw_sp_fib_node_entry_is_first(const struct mlxsw_sp_fib_node *fib_node, 4460 const struct mlxsw_sp_fib_entry *fib_entry) 4461 { 4462 return list_first_entry(&fib_node->entry_list, 4463 struct mlxsw_sp_fib_entry, list) == fib_entry; 4464 } 4465 4466 static int mlxsw_sp_fib_lpm_tree_link(struct mlxsw_sp *mlxsw_sp, 4467 struct mlxsw_sp_fib_node *fib_node) 4468 { 4469 struct mlxsw_sp_prefix_usage req_prefix_usage; 4470 struct mlxsw_sp_fib *fib = fib_node->fib; 4471 struct mlxsw_sp_lpm_tree *lpm_tree; 4472 int err; 4473 4474 lpm_tree = mlxsw_sp->router->lpm.proto_trees[fib->proto]; 4475 if (lpm_tree->prefix_ref_count[fib_node->key.prefix_len] != 0) 4476 goto out; 4477 4478 mlxsw_sp_prefix_usage_cpy(&req_prefix_usage, &lpm_tree->prefix_usage); 4479 mlxsw_sp_prefix_usage_set(&req_prefix_usage, fib_node->key.prefix_len); 4480 lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage, 4481 fib->proto); 4482 if (IS_ERR(lpm_tree)) 4483 return PTR_ERR(lpm_tree); 4484 4485 err = mlxsw_sp_vrs_lpm_tree_replace(mlxsw_sp, fib, lpm_tree); 4486 if (err) 4487 goto err_lpm_tree_replace; 4488 4489 out: 4490 lpm_tree->prefix_ref_count[fib_node->key.prefix_len]++; 4491 return 0; 4492 4493 err_lpm_tree_replace: 4494 mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree); 4495 return err; 4496 } 4497 4498 static void mlxsw_sp_fib_lpm_tree_unlink(struct mlxsw_sp *mlxsw_sp, 4499 struct mlxsw_sp_fib_node *fib_node) 4500 { 4501 struct mlxsw_sp_lpm_tree *lpm_tree = fib_node->fib->lpm_tree; 4502 struct mlxsw_sp_prefix_usage req_prefix_usage; 4503 struct mlxsw_sp_fib *fib = fib_node->fib; 4504 int err; 4505 4506 if (--lpm_tree->prefix_ref_count[fib_node->key.prefix_len] != 0) 4507 return; 4508 /* Try to construct a new LPM tree from the current prefix usage 4509 * minus the unused one. If we fail, continue using the old one. 4510 */ 4511 mlxsw_sp_prefix_usage_cpy(&req_prefix_usage, &lpm_tree->prefix_usage); 4512 mlxsw_sp_prefix_usage_clear(&req_prefix_usage, 4513 fib_node->key.prefix_len); 4514 lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage, 4515 fib->proto); 4516 if (IS_ERR(lpm_tree)) 4517 return; 4518 4519 err = mlxsw_sp_vrs_lpm_tree_replace(mlxsw_sp, fib, lpm_tree); 4520 if (err) 4521 goto err_lpm_tree_replace; 4522 4523 return; 4524 4525 err_lpm_tree_replace: 4526 mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree); 4527 } 4528 4529 static int mlxsw_sp_fib_node_init(struct mlxsw_sp *mlxsw_sp, 4530 struct mlxsw_sp_fib_node *fib_node, 4531 struct mlxsw_sp_fib *fib) 4532 { 4533 int err; 4534 4535 err = mlxsw_sp_fib_node_insert(fib, fib_node); 4536 if (err) 4537 return err; 4538 fib_node->fib = fib; 4539 4540 err = mlxsw_sp_fib_lpm_tree_link(mlxsw_sp, fib_node); 4541 if (err) 4542 goto err_fib_lpm_tree_link; 4543 4544 return 0; 4545 4546 err_fib_lpm_tree_link: 4547 fib_node->fib = NULL; 4548 mlxsw_sp_fib_node_remove(fib, fib_node); 4549 return err; 4550 } 4551 4552 static void mlxsw_sp_fib_node_fini(struct mlxsw_sp *mlxsw_sp, 4553 struct mlxsw_sp_fib_node *fib_node) 4554 { 4555 struct mlxsw_sp_fib *fib = fib_node->fib; 4556 4557 mlxsw_sp_fib_lpm_tree_unlink(mlxsw_sp, fib_node); 4558 fib_node->fib = NULL; 4559 mlxsw_sp_fib_node_remove(fib, fib_node); 4560 } 4561 4562 static struct mlxsw_sp_fib_node * 4563 mlxsw_sp_fib_node_get(struct mlxsw_sp *mlxsw_sp, u32 tb_id, const void *addr, 4564 size_t addr_len, unsigned char prefix_len, 4565 enum mlxsw_sp_l3proto proto) 4566 { 4567 struct mlxsw_sp_fib_node *fib_node; 4568 struct mlxsw_sp_fib *fib; 4569 struct mlxsw_sp_vr *vr; 4570 int err; 4571 4572 vr = mlxsw_sp_vr_get(mlxsw_sp, tb_id, NULL); 4573 if (IS_ERR(vr)) 4574 return ERR_CAST(vr); 4575 fib = mlxsw_sp_vr_fib(vr, proto); 4576 4577 fib_node = mlxsw_sp_fib_node_lookup(fib, addr, addr_len, prefix_len); 4578 if (fib_node) 4579 return fib_node; 4580 4581 fib_node = mlxsw_sp_fib_node_create(fib, addr, addr_len, prefix_len); 4582 if (!fib_node) { 4583 err = -ENOMEM; 4584 goto err_fib_node_create; 4585 } 4586 4587 err = mlxsw_sp_fib_node_init(mlxsw_sp, fib_node, fib); 4588 if (err) 4589 goto err_fib_node_init; 4590 4591 return fib_node; 4592 4593 err_fib_node_init: 4594 mlxsw_sp_fib_node_destroy(fib_node); 4595 err_fib_node_create: 4596 mlxsw_sp_vr_put(mlxsw_sp, vr); 4597 return ERR_PTR(err); 4598 } 4599 4600 static void mlxsw_sp_fib_node_put(struct mlxsw_sp *mlxsw_sp, 4601 struct mlxsw_sp_fib_node *fib_node) 4602 { 4603 struct mlxsw_sp_vr *vr = fib_node->fib->vr; 4604 4605 if (!list_empty(&fib_node->entry_list)) 4606 return; 4607 mlxsw_sp_fib_node_fini(mlxsw_sp, fib_node); 4608 mlxsw_sp_fib_node_destroy(fib_node); 4609 mlxsw_sp_vr_put(mlxsw_sp, vr); 4610 } 4611 4612 static struct mlxsw_sp_fib4_entry * 4613 mlxsw_sp_fib4_node_entry_find(const struct mlxsw_sp_fib_node *fib_node, 4614 const struct mlxsw_sp_fib4_entry *new4_entry) 4615 { 4616 struct mlxsw_sp_fib4_entry *fib4_entry; 4617 4618 list_for_each_entry(fib4_entry, &fib_node->entry_list, common.list) { 4619 if (fib4_entry->tb_id > new4_entry->tb_id) 4620 continue; 4621 if (fib4_entry->tb_id != new4_entry->tb_id) 4622 break; 4623 if (fib4_entry->tos > new4_entry->tos) 4624 continue; 4625 if (fib4_entry->prio >= new4_entry->prio || 4626 fib4_entry->tos < new4_entry->tos) 4627 return fib4_entry; 4628 } 4629 4630 return NULL; 4631 } 4632 4633 static int 4634 mlxsw_sp_fib4_node_list_append(struct mlxsw_sp_fib4_entry *fib4_entry, 4635 struct mlxsw_sp_fib4_entry *new4_entry) 4636 { 4637 struct mlxsw_sp_fib_node *fib_node; 4638 4639 if (WARN_ON(!fib4_entry)) 4640 return -EINVAL; 4641 4642 fib_node = fib4_entry->common.fib_node; 4643 list_for_each_entry_from(fib4_entry, &fib_node->entry_list, 4644 common.list) { 4645 if (fib4_entry->tb_id != new4_entry->tb_id || 4646 fib4_entry->tos != new4_entry->tos || 4647 fib4_entry->prio != new4_entry->prio) 4648 break; 4649 } 4650 4651 list_add_tail(&new4_entry->common.list, &fib4_entry->common.list); 4652 return 0; 4653 } 4654 4655 static int 4656 mlxsw_sp_fib4_node_list_insert(struct mlxsw_sp_fib4_entry *new4_entry, 4657 bool replace, bool append) 4658 { 4659 struct mlxsw_sp_fib_node *fib_node = new4_entry->common.fib_node; 4660 struct mlxsw_sp_fib4_entry *fib4_entry; 4661 4662 fib4_entry = mlxsw_sp_fib4_node_entry_find(fib_node, new4_entry); 4663 4664 if (append) 4665 return mlxsw_sp_fib4_node_list_append(fib4_entry, new4_entry); 4666 if (replace && WARN_ON(!fib4_entry)) 4667 return -EINVAL; 4668 4669 /* Insert new entry before replaced one, so that we can later 4670 * remove the second. 4671 */ 4672 if (fib4_entry) { 4673 list_add_tail(&new4_entry->common.list, 4674 &fib4_entry->common.list); 4675 } else { 4676 struct mlxsw_sp_fib4_entry *last; 4677 4678 list_for_each_entry(last, &fib_node->entry_list, common.list) { 4679 if (new4_entry->tb_id > last->tb_id) 4680 break; 4681 fib4_entry = last; 4682 } 4683 4684 if (fib4_entry) 4685 list_add(&new4_entry->common.list, 4686 &fib4_entry->common.list); 4687 else 4688 list_add(&new4_entry->common.list, 4689 &fib_node->entry_list); 4690 } 4691 4692 return 0; 4693 } 4694 4695 static void 4696 mlxsw_sp_fib4_node_list_remove(struct mlxsw_sp_fib4_entry *fib4_entry) 4697 { 4698 list_del(&fib4_entry->common.list); 4699 } 4700 4701 static int mlxsw_sp_fib_node_entry_add(struct mlxsw_sp *mlxsw_sp, 4702 struct mlxsw_sp_fib_entry *fib_entry) 4703 { 4704 struct mlxsw_sp_fib_node *fib_node = fib_entry->fib_node; 4705 4706 if (!mlxsw_sp_fib_node_entry_is_first(fib_node, fib_entry)) 4707 return 0; 4708 4709 /* To prevent packet loss, overwrite the previously offloaded 4710 * entry. 4711 */ 4712 if (!list_is_singular(&fib_node->entry_list)) { 4713 enum mlxsw_reg_ralue_op op = MLXSW_REG_RALUE_OP_WRITE_DELETE; 4714 struct mlxsw_sp_fib_entry *n = list_next_entry(fib_entry, list); 4715 4716 mlxsw_sp_fib_entry_offload_refresh(n, op, 0); 4717 } 4718 4719 return mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry); 4720 } 4721 4722 static void mlxsw_sp_fib_node_entry_del(struct mlxsw_sp *mlxsw_sp, 4723 struct mlxsw_sp_fib_entry *fib_entry) 4724 { 4725 struct mlxsw_sp_fib_node *fib_node = fib_entry->fib_node; 4726 4727 if (!mlxsw_sp_fib_node_entry_is_first(fib_node, fib_entry)) 4728 return; 4729 4730 /* Promote the next entry by overwriting the deleted entry */ 4731 if (!list_is_singular(&fib_node->entry_list)) { 4732 struct mlxsw_sp_fib_entry *n = list_next_entry(fib_entry, list); 4733 enum mlxsw_reg_ralue_op op = MLXSW_REG_RALUE_OP_WRITE_DELETE; 4734 4735 mlxsw_sp_fib_entry_update(mlxsw_sp, n); 4736 mlxsw_sp_fib_entry_offload_refresh(fib_entry, op, 0); 4737 return; 4738 } 4739 4740 mlxsw_sp_fib_entry_del(mlxsw_sp, fib_entry); 4741 } 4742 4743 static int mlxsw_sp_fib4_node_entry_link(struct mlxsw_sp *mlxsw_sp, 4744 struct mlxsw_sp_fib4_entry *fib4_entry, 4745 bool replace, bool append) 4746 { 4747 int err; 4748 4749 err = mlxsw_sp_fib4_node_list_insert(fib4_entry, replace, append); 4750 if (err) 4751 return err; 4752 4753 err = mlxsw_sp_fib_node_entry_add(mlxsw_sp, &fib4_entry->common); 4754 if (err) 4755 goto err_fib_node_entry_add; 4756 4757 return 0; 4758 4759 err_fib_node_entry_add: 4760 mlxsw_sp_fib4_node_list_remove(fib4_entry); 4761 return err; 4762 } 4763 4764 static void 4765 mlxsw_sp_fib4_node_entry_unlink(struct mlxsw_sp *mlxsw_sp, 4766 struct mlxsw_sp_fib4_entry *fib4_entry) 4767 { 4768 mlxsw_sp_fib_node_entry_del(mlxsw_sp, &fib4_entry->common); 4769 mlxsw_sp_fib4_node_list_remove(fib4_entry); 4770 4771 if (fib4_entry->common.type == MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP) 4772 mlxsw_sp_fib_entry_decap_fini(mlxsw_sp, &fib4_entry->common); 4773 } 4774 4775 static void mlxsw_sp_fib4_entry_replace(struct mlxsw_sp *mlxsw_sp, 4776 struct mlxsw_sp_fib4_entry *fib4_entry, 4777 bool replace) 4778 { 4779 struct mlxsw_sp_fib_node *fib_node = fib4_entry->common.fib_node; 4780 struct mlxsw_sp_fib4_entry *replaced; 4781 4782 if (!replace) 4783 return; 4784 4785 /* We inserted the new entry before replaced one */ 4786 replaced = list_next_entry(fib4_entry, common.list); 4787 4788 mlxsw_sp_fib4_node_entry_unlink(mlxsw_sp, replaced); 4789 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, replaced); 4790 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 4791 } 4792 4793 static int 4794 mlxsw_sp_router_fib4_add(struct mlxsw_sp *mlxsw_sp, 4795 const struct fib_entry_notifier_info *fen_info, 4796 bool replace, bool append) 4797 { 4798 struct mlxsw_sp_fib4_entry *fib4_entry; 4799 struct mlxsw_sp_fib_node *fib_node; 4800 int err; 4801 4802 if (mlxsw_sp->router->aborted) 4803 return 0; 4804 4805 fib_node = mlxsw_sp_fib_node_get(mlxsw_sp, fen_info->tb_id, 4806 &fen_info->dst, sizeof(fen_info->dst), 4807 fen_info->dst_len, 4808 MLXSW_SP_L3_PROTO_IPV4); 4809 if (IS_ERR(fib_node)) { 4810 dev_warn(mlxsw_sp->bus_info->dev, "Failed to get FIB node\n"); 4811 return PTR_ERR(fib_node); 4812 } 4813 4814 fib4_entry = mlxsw_sp_fib4_entry_create(mlxsw_sp, fib_node, fen_info); 4815 if (IS_ERR(fib4_entry)) { 4816 dev_warn(mlxsw_sp->bus_info->dev, "Failed to create FIB entry\n"); 4817 err = PTR_ERR(fib4_entry); 4818 goto err_fib4_entry_create; 4819 } 4820 4821 err = mlxsw_sp_fib4_node_entry_link(mlxsw_sp, fib4_entry, replace, 4822 append); 4823 if (err) { 4824 dev_warn(mlxsw_sp->bus_info->dev, "Failed to link FIB entry to node\n"); 4825 goto err_fib4_node_entry_link; 4826 } 4827 4828 mlxsw_sp_fib4_entry_replace(mlxsw_sp, fib4_entry, replace); 4829 4830 return 0; 4831 4832 err_fib4_node_entry_link: 4833 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry); 4834 err_fib4_entry_create: 4835 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 4836 return err; 4837 } 4838 4839 static void mlxsw_sp_router_fib4_del(struct mlxsw_sp *mlxsw_sp, 4840 struct fib_entry_notifier_info *fen_info) 4841 { 4842 struct mlxsw_sp_fib4_entry *fib4_entry; 4843 struct mlxsw_sp_fib_node *fib_node; 4844 4845 if (mlxsw_sp->router->aborted) 4846 return; 4847 4848 fib4_entry = mlxsw_sp_fib4_entry_lookup(mlxsw_sp, fen_info); 4849 if (WARN_ON(!fib4_entry)) 4850 return; 4851 fib_node = fib4_entry->common.fib_node; 4852 4853 mlxsw_sp_fib4_node_entry_unlink(mlxsw_sp, fib4_entry); 4854 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry); 4855 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 4856 } 4857 4858 static bool mlxsw_sp_fib6_rt_should_ignore(const struct fib6_info *rt) 4859 { 4860 /* Packets with link-local destination IP arriving to the router 4861 * are trapped to the CPU, so no need to program specific routes 4862 * for them. 4863 */ 4864 if (ipv6_addr_type(&rt->fib6_dst.addr) & IPV6_ADDR_LINKLOCAL) 4865 return true; 4866 4867 /* Multicast routes aren't supported, so ignore them. Neighbour 4868 * Discovery packets are specifically trapped. 4869 */ 4870 if (ipv6_addr_type(&rt->fib6_dst.addr) & IPV6_ADDR_MULTICAST) 4871 return true; 4872 4873 /* Cloned routes are irrelevant in the forwarding path. */ 4874 if (rt->fib6_flags & RTF_CACHE) 4875 return true; 4876 4877 return false; 4878 } 4879 4880 static struct mlxsw_sp_rt6 *mlxsw_sp_rt6_create(struct fib6_info *rt) 4881 { 4882 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 4883 4884 mlxsw_sp_rt6 = kzalloc(sizeof(*mlxsw_sp_rt6), GFP_KERNEL); 4885 if (!mlxsw_sp_rt6) 4886 return ERR_PTR(-ENOMEM); 4887 4888 /* In case of route replace, replaced route is deleted with 4889 * no notification. Take reference to prevent accessing freed 4890 * memory. 4891 */ 4892 mlxsw_sp_rt6->rt = rt; 4893 fib6_info_hold(rt); 4894 4895 return mlxsw_sp_rt6; 4896 } 4897 4898 #if IS_ENABLED(CONFIG_IPV6) 4899 static void mlxsw_sp_rt6_release(struct fib6_info *rt) 4900 { 4901 fib6_info_release(rt); 4902 } 4903 #else 4904 static void mlxsw_sp_rt6_release(struct fib6_info *rt) 4905 { 4906 } 4907 #endif 4908 4909 static void mlxsw_sp_rt6_destroy(struct mlxsw_sp_rt6 *mlxsw_sp_rt6) 4910 { 4911 mlxsw_sp_rt6_release(mlxsw_sp_rt6->rt); 4912 kfree(mlxsw_sp_rt6); 4913 } 4914 4915 static bool mlxsw_sp_fib6_rt_can_mp(const struct fib6_info *rt) 4916 { 4917 /* RTF_CACHE routes are ignored */ 4918 return !(rt->fib6_flags & RTF_ADDRCONF) && rt->fib6_nh.fib_nh_gw_family; 4919 } 4920 4921 static struct fib6_info * 4922 mlxsw_sp_fib6_entry_rt(const struct mlxsw_sp_fib6_entry *fib6_entry) 4923 { 4924 return list_first_entry(&fib6_entry->rt6_list, struct mlxsw_sp_rt6, 4925 list)->rt; 4926 } 4927 4928 static struct mlxsw_sp_fib6_entry * 4929 mlxsw_sp_fib6_node_mp_entry_find(const struct mlxsw_sp_fib_node *fib_node, 4930 const struct fib6_info *nrt, bool replace) 4931 { 4932 struct mlxsw_sp_fib6_entry *fib6_entry; 4933 4934 if (!mlxsw_sp_fib6_rt_can_mp(nrt) || replace) 4935 return NULL; 4936 4937 list_for_each_entry(fib6_entry, &fib_node->entry_list, common.list) { 4938 struct fib6_info *rt = mlxsw_sp_fib6_entry_rt(fib6_entry); 4939 4940 /* RT6_TABLE_LOCAL and RT6_TABLE_MAIN share the same 4941 * virtual router. 4942 */ 4943 if (rt->fib6_table->tb6_id > nrt->fib6_table->tb6_id) 4944 continue; 4945 if (rt->fib6_table->tb6_id != nrt->fib6_table->tb6_id) 4946 break; 4947 if (rt->fib6_metric < nrt->fib6_metric) 4948 continue; 4949 if (rt->fib6_metric == nrt->fib6_metric && 4950 mlxsw_sp_fib6_rt_can_mp(rt)) 4951 return fib6_entry; 4952 if (rt->fib6_metric > nrt->fib6_metric) 4953 break; 4954 } 4955 4956 return NULL; 4957 } 4958 4959 static struct mlxsw_sp_rt6 * 4960 mlxsw_sp_fib6_entry_rt_find(const struct mlxsw_sp_fib6_entry *fib6_entry, 4961 const struct fib6_info *rt) 4962 { 4963 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 4964 4965 list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) { 4966 if (mlxsw_sp_rt6->rt == rt) 4967 return mlxsw_sp_rt6; 4968 } 4969 4970 return NULL; 4971 } 4972 4973 static bool mlxsw_sp_nexthop6_ipip_type(const struct mlxsw_sp *mlxsw_sp, 4974 const struct fib6_info *rt, 4975 enum mlxsw_sp_ipip_type *ret) 4976 { 4977 return rt->fib6_nh.fib_nh_dev && 4978 mlxsw_sp_netdev_ipip_type(mlxsw_sp, rt->fib6_nh.fib_nh_dev, ret); 4979 } 4980 4981 static int mlxsw_sp_nexthop6_type_init(struct mlxsw_sp *mlxsw_sp, 4982 struct mlxsw_sp_nexthop_group *nh_grp, 4983 struct mlxsw_sp_nexthop *nh, 4984 const struct fib6_info *rt) 4985 { 4986 const struct mlxsw_sp_ipip_ops *ipip_ops; 4987 struct mlxsw_sp_ipip_entry *ipip_entry; 4988 struct net_device *dev = rt->fib6_nh.fib_nh_dev; 4989 struct mlxsw_sp_rif *rif; 4990 int err; 4991 4992 ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, dev); 4993 if (ipip_entry) { 4994 ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt]; 4995 if (ipip_ops->can_offload(mlxsw_sp, dev, 4996 MLXSW_SP_L3_PROTO_IPV6)) { 4997 nh->type = MLXSW_SP_NEXTHOP_TYPE_IPIP; 4998 mlxsw_sp_nexthop_ipip_init(mlxsw_sp, nh, ipip_entry); 4999 return 0; 5000 } 5001 } 5002 5003 nh->type = MLXSW_SP_NEXTHOP_TYPE_ETH; 5004 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev); 5005 if (!rif) 5006 return 0; 5007 mlxsw_sp_nexthop_rif_init(nh, rif); 5008 5009 err = mlxsw_sp_nexthop_neigh_init(mlxsw_sp, nh); 5010 if (err) 5011 goto err_nexthop_neigh_init; 5012 5013 return 0; 5014 5015 err_nexthop_neigh_init: 5016 mlxsw_sp_nexthop_rif_fini(nh); 5017 return err; 5018 } 5019 5020 static void mlxsw_sp_nexthop6_type_fini(struct mlxsw_sp *mlxsw_sp, 5021 struct mlxsw_sp_nexthop *nh) 5022 { 5023 mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh); 5024 } 5025 5026 static int mlxsw_sp_nexthop6_init(struct mlxsw_sp *mlxsw_sp, 5027 struct mlxsw_sp_nexthop_group *nh_grp, 5028 struct mlxsw_sp_nexthop *nh, 5029 const struct fib6_info *rt) 5030 { 5031 struct net_device *dev = rt->fib6_nh.fib_nh_dev; 5032 5033 nh->nh_grp = nh_grp; 5034 nh->nh_weight = rt->fib6_nh.fib_nh_weight; 5035 memcpy(&nh->gw_addr, &rt->fib6_nh.fib_nh_gw6, sizeof(nh->gw_addr)); 5036 mlxsw_sp_nexthop_counter_alloc(mlxsw_sp, nh); 5037 5038 list_add_tail(&nh->router_list_node, &mlxsw_sp->router->nexthop_list); 5039 5040 if (!dev) 5041 return 0; 5042 nh->ifindex = dev->ifindex; 5043 5044 return mlxsw_sp_nexthop6_type_init(mlxsw_sp, nh_grp, nh, rt); 5045 } 5046 5047 static void mlxsw_sp_nexthop6_fini(struct mlxsw_sp *mlxsw_sp, 5048 struct mlxsw_sp_nexthop *nh) 5049 { 5050 mlxsw_sp_nexthop6_type_fini(mlxsw_sp, nh); 5051 list_del(&nh->router_list_node); 5052 mlxsw_sp_nexthop_counter_free(mlxsw_sp, nh); 5053 } 5054 5055 static bool mlxsw_sp_rt6_is_gateway(const struct mlxsw_sp *mlxsw_sp, 5056 const struct fib6_info *rt) 5057 { 5058 return rt->fib6_nh.fib_nh_gw_family || 5059 mlxsw_sp_nexthop6_ipip_type(mlxsw_sp, rt, NULL); 5060 } 5061 5062 static struct mlxsw_sp_nexthop_group * 5063 mlxsw_sp_nexthop6_group_create(struct mlxsw_sp *mlxsw_sp, 5064 struct mlxsw_sp_fib6_entry *fib6_entry) 5065 { 5066 struct mlxsw_sp_nexthop_group *nh_grp; 5067 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 5068 struct mlxsw_sp_nexthop *nh; 5069 int i = 0; 5070 int err; 5071 5072 nh_grp = kzalloc(struct_size(nh_grp, nexthops, fib6_entry->nrt6), 5073 GFP_KERNEL); 5074 if (!nh_grp) 5075 return ERR_PTR(-ENOMEM); 5076 INIT_LIST_HEAD(&nh_grp->fib_list); 5077 #if IS_ENABLED(CONFIG_IPV6) 5078 nh_grp->neigh_tbl = &nd_tbl; 5079 #endif 5080 mlxsw_sp_rt6 = list_first_entry(&fib6_entry->rt6_list, 5081 struct mlxsw_sp_rt6, list); 5082 nh_grp->gateway = mlxsw_sp_rt6_is_gateway(mlxsw_sp, mlxsw_sp_rt6->rt); 5083 nh_grp->count = fib6_entry->nrt6; 5084 for (i = 0; i < nh_grp->count; i++) { 5085 struct fib6_info *rt = mlxsw_sp_rt6->rt; 5086 5087 nh = &nh_grp->nexthops[i]; 5088 err = mlxsw_sp_nexthop6_init(mlxsw_sp, nh_grp, nh, rt); 5089 if (err) 5090 goto err_nexthop6_init; 5091 mlxsw_sp_rt6 = list_next_entry(mlxsw_sp_rt6, list); 5092 } 5093 5094 err = mlxsw_sp_nexthop_group_insert(mlxsw_sp, nh_grp); 5095 if (err) 5096 goto err_nexthop_group_insert; 5097 5098 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp); 5099 return nh_grp; 5100 5101 err_nexthop_group_insert: 5102 err_nexthop6_init: 5103 for (i--; i >= 0; i--) { 5104 nh = &nh_grp->nexthops[i]; 5105 mlxsw_sp_nexthop6_fini(mlxsw_sp, nh); 5106 } 5107 kfree(nh_grp); 5108 return ERR_PTR(err); 5109 } 5110 5111 static void 5112 mlxsw_sp_nexthop6_group_destroy(struct mlxsw_sp *mlxsw_sp, 5113 struct mlxsw_sp_nexthop_group *nh_grp) 5114 { 5115 struct mlxsw_sp_nexthop *nh; 5116 int i = nh_grp->count; 5117 5118 mlxsw_sp_nexthop_group_remove(mlxsw_sp, nh_grp); 5119 for (i--; i >= 0; i--) { 5120 nh = &nh_grp->nexthops[i]; 5121 mlxsw_sp_nexthop6_fini(mlxsw_sp, nh); 5122 } 5123 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp); 5124 WARN_ON(nh_grp->adj_index_valid); 5125 kfree(nh_grp); 5126 } 5127 5128 static int mlxsw_sp_nexthop6_group_get(struct mlxsw_sp *mlxsw_sp, 5129 struct mlxsw_sp_fib6_entry *fib6_entry) 5130 { 5131 struct mlxsw_sp_nexthop_group *nh_grp; 5132 5133 nh_grp = mlxsw_sp_nexthop6_group_lookup(mlxsw_sp, fib6_entry); 5134 if (!nh_grp) { 5135 nh_grp = mlxsw_sp_nexthop6_group_create(mlxsw_sp, fib6_entry); 5136 if (IS_ERR(nh_grp)) 5137 return PTR_ERR(nh_grp); 5138 } 5139 5140 list_add_tail(&fib6_entry->common.nexthop_group_node, 5141 &nh_grp->fib_list); 5142 fib6_entry->common.nh_group = nh_grp; 5143 5144 return 0; 5145 } 5146 5147 static void mlxsw_sp_nexthop6_group_put(struct mlxsw_sp *mlxsw_sp, 5148 struct mlxsw_sp_fib_entry *fib_entry) 5149 { 5150 struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group; 5151 5152 list_del(&fib_entry->nexthop_group_node); 5153 if (!list_empty(&nh_grp->fib_list)) 5154 return; 5155 mlxsw_sp_nexthop6_group_destroy(mlxsw_sp, nh_grp); 5156 } 5157 5158 static int 5159 mlxsw_sp_nexthop6_group_update(struct mlxsw_sp *mlxsw_sp, 5160 struct mlxsw_sp_fib6_entry *fib6_entry) 5161 { 5162 struct mlxsw_sp_nexthop_group *old_nh_grp = fib6_entry->common.nh_group; 5163 int err; 5164 5165 fib6_entry->common.nh_group = NULL; 5166 list_del(&fib6_entry->common.nexthop_group_node); 5167 5168 err = mlxsw_sp_nexthop6_group_get(mlxsw_sp, fib6_entry); 5169 if (err) 5170 goto err_nexthop6_group_get; 5171 5172 /* In case this entry is offloaded, then the adjacency index 5173 * currently associated with it in the device's table is that 5174 * of the old group. Start using the new one instead. 5175 */ 5176 err = mlxsw_sp_fib_node_entry_add(mlxsw_sp, &fib6_entry->common); 5177 if (err) 5178 goto err_fib_node_entry_add; 5179 5180 if (list_empty(&old_nh_grp->fib_list)) 5181 mlxsw_sp_nexthop6_group_destroy(mlxsw_sp, old_nh_grp); 5182 5183 return 0; 5184 5185 err_fib_node_entry_add: 5186 mlxsw_sp_nexthop6_group_put(mlxsw_sp, &fib6_entry->common); 5187 err_nexthop6_group_get: 5188 list_add_tail(&fib6_entry->common.nexthop_group_node, 5189 &old_nh_grp->fib_list); 5190 fib6_entry->common.nh_group = old_nh_grp; 5191 return err; 5192 } 5193 5194 static int 5195 mlxsw_sp_fib6_entry_nexthop_add(struct mlxsw_sp *mlxsw_sp, 5196 struct mlxsw_sp_fib6_entry *fib6_entry, 5197 struct fib6_info *rt) 5198 { 5199 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 5200 int err; 5201 5202 mlxsw_sp_rt6 = mlxsw_sp_rt6_create(rt); 5203 if (IS_ERR(mlxsw_sp_rt6)) 5204 return PTR_ERR(mlxsw_sp_rt6); 5205 5206 list_add_tail(&mlxsw_sp_rt6->list, &fib6_entry->rt6_list); 5207 fib6_entry->nrt6++; 5208 5209 err = mlxsw_sp_nexthop6_group_update(mlxsw_sp, fib6_entry); 5210 if (err) 5211 goto err_nexthop6_group_update; 5212 5213 return 0; 5214 5215 err_nexthop6_group_update: 5216 fib6_entry->nrt6--; 5217 list_del(&mlxsw_sp_rt6->list); 5218 mlxsw_sp_rt6_destroy(mlxsw_sp_rt6); 5219 return err; 5220 } 5221 5222 static void 5223 mlxsw_sp_fib6_entry_nexthop_del(struct mlxsw_sp *mlxsw_sp, 5224 struct mlxsw_sp_fib6_entry *fib6_entry, 5225 struct fib6_info *rt) 5226 { 5227 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 5228 5229 mlxsw_sp_rt6 = mlxsw_sp_fib6_entry_rt_find(fib6_entry, rt); 5230 if (WARN_ON(!mlxsw_sp_rt6)) 5231 return; 5232 5233 fib6_entry->nrt6--; 5234 list_del(&mlxsw_sp_rt6->list); 5235 mlxsw_sp_nexthop6_group_update(mlxsw_sp, fib6_entry); 5236 mlxsw_sp_rt6_destroy(mlxsw_sp_rt6); 5237 } 5238 5239 static void mlxsw_sp_fib6_entry_type_set(struct mlxsw_sp *mlxsw_sp, 5240 struct mlxsw_sp_fib_entry *fib_entry, 5241 const struct fib6_info *rt) 5242 { 5243 /* Packets hitting RTF_REJECT routes need to be discarded by the 5244 * stack. We can rely on their destination device not having a 5245 * RIF (it's the loopback device) and can thus use action type 5246 * local, which will cause them to be trapped with a lower 5247 * priority than packets that need to be locally received. 5248 */ 5249 if (rt->fib6_flags & (RTF_LOCAL | RTF_ANYCAST)) 5250 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP; 5251 else if (rt->fib6_type == RTN_BLACKHOLE) 5252 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_BLACKHOLE; 5253 else if (rt->fib6_flags & RTF_REJECT) 5254 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL; 5255 else if (mlxsw_sp_rt6_is_gateway(mlxsw_sp, rt)) 5256 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_REMOTE; 5257 else 5258 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL; 5259 } 5260 5261 static void 5262 mlxsw_sp_fib6_entry_rt_destroy_all(struct mlxsw_sp_fib6_entry *fib6_entry) 5263 { 5264 struct mlxsw_sp_rt6 *mlxsw_sp_rt6, *tmp; 5265 5266 list_for_each_entry_safe(mlxsw_sp_rt6, tmp, &fib6_entry->rt6_list, 5267 list) { 5268 fib6_entry->nrt6--; 5269 list_del(&mlxsw_sp_rt6->list); 5270 mlxsw_sp_rt6_destroy(mlxsw_sp_rt6); 5271 } 5272 } 5273 5274 static struct mlxsw_sp_fib6_entry * 5275 mlxsw_sp_fib6_entry_create(struct mlxsw_sp *mlxsw_sp, 5276 struct mlxsw_sp_fib_node *fib_node, 5277 struct fib6_info *rt) 5278 { 5279 struct mlxsw_sp_fib6_entry *fib6_entry; 5280 struct mlxsw_sp_fib_entry *fib_entry; 5281 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 5282 int err; 5283 5284 fib6_entry = kzalloc(sizeof(*fib6_entry), GFP_KERNEL); 5285 if (!fib6_entry) 5286 return ERR_PTR(-ENOMEM); 5287 fib_entry = &fib6_entry->common; 5288 5289 mlxsw_sp_rt6 = mlxsw_sp_rt6_create(rt); 5290 if (IS_ERR(mlxsw_sp_rt6)) { 5291 err = PTR_ERR(mlxsw_sp_rt6); 5292 goto err_rt6_create; 5293 } 5294 5295 mlxsw_sp_fib6_entry_type_set(mlxsw_sp, fib_entry, mlxsw_sp_rt6->rt); 5296 5297 INIT_LIST_HEAD(&fib6_entry->rt6_list); 5298 list_add_tail(&mlxsw_sp_rt6->list, &fib6_entry->rt6_list); 5299 fib6_entry->nrt6 = 1; 5300 err = mlxsw_sp_nexthop6_group_get(mlxsw_sp, fib6_entry); 5301 if (err) 5302 goto err_nexthop6_group_get; 5303 5304 fib_entry->fib_node = fib_node; 5305 5306 return fib6_entry; 5307 5308 err_nexthop6_group_get: 5309 list_del(&mlxsw_sp_rt6->list); 5310 mlxsw_sp_rt6_destroy(mlxsw_sp_rt6); 5311 err_rt6_create: 5312 kfree(fib6_entry); 5313 return ERR_PTR(err); 5314 } 5315 5316 static void mlxsw_sp_fib6_entry_destroy(struct mlxsw_sp *mlxsw_sp, 5317 struct mlxsw_sp_fib6_entry *fib6_entry) 5318 { 5319 mlxsw_sp_nexthop6_group_put(mlxsw_sp, &fib6_entry->common); 5320 mlxsw_sp_fib6_entry_rt_destroy_all(fib6_entry); 5321 WARN_ON(fib6_entry->nrt6); 5322 kfree(fib6_entry); 5323 } 5324 5325 static struct mlxsw_sp_fib6_entry * 5326 mlxsw_sp_fib6_node_entry_find(const struct mlxsw_sp_fib_node *fib_node, 5327 const struct fib6_info *nrt, bool replace) 5328 { 5329 struct mlxsw_sp_fib6_entry *fib6_entry, *fallback = NULL; 5330 5331 list_for_each_entry(fib6_entry, &fib_node->entry_list, common.list) { 5332 struct fib6_info *rt = mlxsw_sp_fib6_entry_rt(fib6_entry); 5333 5334 if (rt->fib6_table->tb6_id > nrt->fib6_table->tb6_id) 5335 continue; 5336 if (rt->fib6_table->tb6_id != nrt->fib6_table->tb6_id) 5337 break; 5338 if (replace && rt->fib6_metric == nrt->fib6_metric) { 5339 if (mlxsw_sp_fib6_rt_can_mp(rt) == 5340 mlxsw_sp_fib6_rt_can_mp(nrt)) 5341 return fib6_entry; 5342 if (mlxsw_sp_fib6_rt_can_mp(nrt)) 5343 fallback = fallback ?: fib6_entry; 5344 } 5345 if (rt->fib6_metric > nrt->fib6_metric) 5346 return fallback ?: fib6_entry; 5347 } 5348 5349 return fallback; 5350 } 5351 5352 static int 5353 mlxsw_sp_fib6_node_list_insert(struct mlxsw_sp_fib6_entry *new6_entry, 5354 bool replace) 5355 { 5356 struct mlxsw_sp_fib_node *fib_node = new6_entry->common.fib_node; 5357 struct fib6_info *nrt = mlxsw_sp_fib6_entry_rt(new6_entry); 5358 struct mlxsw_sp_fib6_entry *fib6_entry; 5359 5360 fib6_entry = mlxsw_sp_fib6_node_entry_find(fib_node, nrt, replace); 5361 5362 if (replace && WARN_ON(!fib6_entry)) 5363 return -EINVAL; 5364 5365 if (fib6_entry) { 5366 list_add_tail(&new6_entry->common.list, 5367 &fib6_entry->common.list); 5368 } else { 5369 struct mlxsw_sp_fib6_entry *last; 5370 5371 list_for_each_entry(last, &fib_node->entry_list, common.list) { 5372 struct fib6_info *rt = mlxsw_sp_fib6_entry_rt(last); 5373 5374 if (nrt->fib6_table->tb6_id > rt->fib6_table->tb6_id) 5375 break; 5376 fib6_entry = last; 5377 } 5378 5379 if (fib6_entry) 5380 list_add(&new6_entry->common.list, 5381 &fib6_entry->common.list); 5382 else 5383 list_add(&new6_entry->common.list, 5384 &fib_node->entry_list); 5385 } 5386 5387 return 0; 5388 } 5389 5390 static void 5391 mlxsw_sp_fib6_node_list_remove(struct mlxsw_sp_fib6_entry *fib6_entry) 5392 { 5393 list_del(&fib6_entry->common.list); 5394 } 5395 5396 static int mlxsw_sp_fib6_node_entry_link(struct mlxsw_sp *mlxsw_sp, 5397 struct mlxsw_sp_fib6_entry *fib6_entry, 5398 bool replace) 5399 { 5400 int err; 5401 5402 err = mlxsw_sp_fib6_node_list_insert(fib6_entry, replace); 5403 if (err) 5404 return err; 5405 5406 err = mlxsw_sp_fib_node_entry_add(mlxsw_sp, &fib6_entry->common); 5407 if (err) 5408 goto err_fib_node_entry_add; 5409 5410 return 0; 5411 5412 err_fib_node_entry_add: 5413 mlxsw_sp_fib6_node_list_remove(fib6_entry); 5414 return err; 5415 } 5416 5417 static void 5418 mlxsw_sp_fib6_node_entry_unlink(struct mlxsw_sp *mlxsw_sp, 5419 struct mlxsw_sp_fib6_entry *fib6_entry) 5420 { 5421 mlxsw_sp_fib_node_entry_del(mlxsw_sp, &fib6_entry->common); 5422 mlxsw_sp_fib6_node_list_remove(fib6_entry); 5423 } 5424 5425 static struct mlxsw_sp_fib6_entry * 5426 mlxsw_sp_fib6_entry_lookup(struct mlxsw_sp *mlxsw_sp, 5427 const struct fib6_info *rt) 5428 { 5429 struct mlxsw_sp_fib6_entry *fib6_entry; 5430 struct mlxsw_sp_fib_node *fib_node; 5431 struct mlxsw_sp_fib *fib; 5432 struct mlxsw_sp_vr *vr; 5433 5434 vr = mlxsw_sp_vr_find(mlxsw_sp, rt->fib6_table->tb6_id); 5435 if (!vr) 5436 return NULL; 5437 fib = mlxsw_sp_vr_fib(vr, MLXSW_SP_L3_PROTO_IPV6); 5438 5439 fib_node = mlxsw_sp_fib_node_lookup(fib, &rt->fib6_dst.addr, 5440 sizeof(rt->fib6_dst.addr), 5441 rt->fib6_dst.plen); 5442 if (!fib_node) 5443 return NULL; 5444 5445 list_for_each_entry(fib6_entry, &fib_node->entry_list, common.list) { 5446 struct fib6_info *iter_rt = mlxsw_sp_fib6_entry_rt(fib6_entry); 5447 5448 if (rt->fib6_table->tb6_id == iter_rt->fib6_table->tb6_id && 5449 rt->fib6_metric == iter_rt->fib6_metric && 5450 mlxsw_sp_fib6_entry_rt_find(fib6_entry, rt)) 5451 return fib6_entry; 5452 } 5453 5454 return NULL; 5455 } 5456 5457 static void mlxsw_sp_fib6_entry_replace(struct mlxsw_sp *mlxsw_sp, 5458 struct mlxsw_sp_fib6_entry *fib6_entry, 5459 bool replace) 5460 { 5461 struct mlxsw_sp_fib_node *fib_node = fib6_entry->common.fib_node; 5462 struct mlxsw_sp_fib6_entry *replaced; 5463 5464 if (!replace) 5465 return; 5466 5467 replaced = list_next_entry(fib6_entry, common.list); 5468 5469 mlxsw_sp_fib6_node_entry_unlink(mlxsw_sp, replaced); 5470 mlxsw_sp_fib6_entry_destroy(mlxsw_sp, replaced); 5471 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 5472 } 5473 5474 static int mlxsw_sp_router_fib6_add(struct mlxsw_sp *mlxsw_sp, 5475 struct fib6_info *rt, bool replace) 5476 { 5477 struct mlxsw_sp_fib6_entry *fib6_entry; 5478 struct mlxsw_sp_fib_node *fib_node; 5479 int err; 5480 5481 if (mlxsw_sp->router->aborted) 5482 return 0; 5483 5484 if (rt->fib6_src.plen) 5485 return -EINVAL; 5486 5487 if (mlxsw_sp_fib6_rt_should_ignore(rt)) 5488 return 0; 5489 5490 fib_node = mlxsw_sp_fib_node_get(mlxsw_sp, rt->fib6_table->tb6_id, 5491 &rt->fib6_dst.addr, 5492 sizeof(rt->fib6_dst.addr), 5493 rt->fib6_dst.plen, 5494 MLXSW_SP_L3_PROTO_IPV6); 5495 if (IS_ERR(fib_node)) 5496 return PTR_ERR(fib_node); 5497 5498 /* Before creating a new entry, try to append route to an existing 5499 * multipath entry. 5500 */ 5501 fib6_entry = mlxsw_sp_fib6_node_mp_entry_find(fib_node, rt, replace); 5502 if (fib6_entry) { 5503 err = mlxsw_sp_fib6_entry_nexthop_add(mlxsw_sp, fib6_entry, rt); 5504 if (err) 5505 goto err_fib6_entry_nexthop_add; 5506 return 0; 5507 } 5508 5509 fib6_entry = mlxsw_sp_fib6_entry_create(mlxsw_sp, fib_node, rt); 5510 if (IS_ERR(fib6_entry)) { 5511 err = PTR_ERR(fib6_entry); 5512 goto err_fib6_entry_create; 5513 } 5514 5515 err = mlxsw_sp_fib6_node_entry_link(mlxsw_sp, fib6_entry, replace); 5516 if (err) 5517 goto err_fib6_node_entry_link; 5518 5519 mlxsw_sp_fib6_entry_replace(mlxsw_sp, fib6_entry, replace); 5520 5521 return 0; 5522 5523 err_fib6_node_entry_link: 5524 mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry); 5525 err_fib6_entry_create: 5526 err_fib6_entry_nexthop_add: 5527 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 5528 return err; 5529 } 5530 5531 static void mlxsw_sp_router_fib6_del(struct mlxsw_sp *mlxsw_sp, 5532 struct fib6_info *rt) 5533 { 5534 struct mlxsw_sp_fib6_entry *fib6_entry; 5535 struct mlxsw_sp_fib_node *fib_node; 5536 5537 if (mlxsw_sp->router->aborted) 5538 return; 5539 5540 if (mlxsw_sp_fib6_rt_should_ignore(rt)) 5541 return; 5542 5543 fib6_entry = mlxsw_sp_fib6_entry_lookup(mlxsw_sp, rt); 5544 if (WARN_ON(!fib6_entry)) 5545 return; 5546 5547 /* If route is part of a multipath entry, but not the last one 5548 * removed, then only reduce its nexthop group. 5549 */ 5550 if (!list_is_singular(&fib6_entry->rt6_list)) { 5551 mlxsw_sp_fib6_entry_nexthop_del(mlxsw_sp, fib6_entry, rt); 5552 return; 5553 } 5554 5555 fib_node = fib6_entry->common.fib_node; 5556 5557 mlxsw_sp_fib6_node_entry_unlink(mlxsw_sp, fib6_entry); 5558 mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry); 5559 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 5560 } 5561 5562 static int __mlxsw_sp_router_set_abort_trap(struct mlxsw_sp *mlxsw_sp, 5563 enum mlxsw_reg_ralxx_protocol proto, 5564 u8 tree_id) 5565 { 5566 char ralta_pl[MLXSW_REG_RALTA_LEN]; 5567 char ralst_pl[MLXSW_REG_RALST_LEN]; 5568 int i, err; 5569 5570 mlxsw_reg_ralta_pack(ralta_pl, true, proto, tree_id); 5571 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), ralta_pl); 5572 if (err) 5573 return err; 5574 5575 mlxsw_reg_ralst_pack(ralst_pl, 0xff, tree_id); 5576 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralst), ralst_pl); 5577 if (err) 5578 return err; 5579 5580 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) { 5581 struct mlxsw_sp_vr *vr = &mlxsw_sp->router->vrs[i]; 5582 char raltb_pl[MLXSW_REG_RALTB_LEN]; 5583 char ralue_pl[MLXSW_REG_RALUE_LEN]; 5584 5585 mlxsw_reg_raltb_pack(raltb_pl, vr->id, proto, tree_id); 5586 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb), 5587 raltb_pl); 5588 if (err) 5589 return err; 5590 5591 mlxsw_reg_ralue_pack(ralue_pl, proto, 5592 MLXSW_REG_RALUE_OP_WRITE_WRITE, vr->id, 0); 5593 mlxsw_reg_ralue_act_ip2me_pack(ralue_pl); 5594 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), 5595 ralue_pl); 5596 if (err) 5597 return err; 5598 } 5599 5600 return 0; 5601 } 5602 5603 static struct mlxsw_sp_mr_table * 5604 mlxsw_sp_router_fibmr_family_to_table(struct mlxsw_sp_vr *vr, int family) 5605 { 5606 if (family == RTNL_FAMILY_IPMR) 5607 return vr->mr_table[MLXSW_SP_L3_PROTO_IPV4]; 5608 else 5609 return vr->mr_table[MLXSW_SP_L3_PROTO_IPV6]; 5610 } 5611 5612 static int mlxsw_sp_router_fibmr_add(struct mlxsw_sp *mlxsw_sp, 5613 struct mfc_entry_notifier_info *men_info, 5614 bool replace) 5615 { 5616 struct mlxsw_sp_mr_table *mrt; 5617 struct mlxsw_sp_vr *vr; 5618 5619 if (mlxsw_sp->router->aborted) 5620 return 0; 5621 5622 vr = mlxsw_sp_vr_get(mlxsw_sp, men_info->tb_id, NULL); 5623 if (IS_ERR(vr)) 5624 return PTR_ERR(vr); 5625 5626 mrt = mlxsw_sp_router_fibmr_family_to_table(vr, men_info->info.family); 5627 return mlxsw_sp_mr_route_add(mrt, men_info->mfc, replace); 5628 } 5629 5630 static void mlxsw_sp_router_fibmr_del(struct mlxsw_sp *mlxsw_sp, 5631 struct mfc_entry_notifier_info *men_info) 5632 { 5633 struct mlxsw_sp_mr_table *mrt; 5634 struct mlxsw_sp_vr *vr; 5635 5636 if (mlxsw_sp->router->aborted) 5637 return; 5638 5639 vr = mlxsw_sp_vr_find(mlxsw_sp, men_info->tb_id); 5640 if (WARN_ON(!vr)) 5641 return; 5642 5643 mrt = mlxsw_sp_router_fibmr_family_to_table(vr, men_info->info.family); 5644 mlxsw_sp_mr_route_del(mrt, men_info->mfc); 5645 mlxsw_sp_vr_put(mlxsw_sp, vr); 5646 } 5647 5648 static int 5649 mlxsw_sp_router_fibmr_vif_add(struct mlxsw_sp *mlxsw_sp, 5650 struct vif_entry_notifier_info *ven_info) 5651 { 5652 struct mlxsw_sp_mr_table *mrt; 5653 struct mlxsw_sp_rif *rif; 5654 struct mlxsw_sp_vr *vr; 5655 5656 if (mlxsw_sp->router->aborted) 5657 return 0; 5658 5659 vr = mlxsw_sp_vr_get(mlxsw_sp, ven_info->tb_id, NULL); 5660 if (IS_ERR(vr)) 5661 return PTR_ERR(vr); 5662 5663 mrt = mlxsw_sp_router_fibmr_family_to_table(vr, ven_info->info.family); 5664 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, ven_info->dev); 5665 return mlxsw_sp_mr_vif_add(mrt, ven_info->dev, 5666 ven_info->vif_index, 5667 ven_info->vif_flags, rif); 5668 } 5669 5670 static void 5671 mlxsw_sp_router_fibmr_vif_del(struct mlxsw_sp *mlxsw_sp, 5672 struct vif_entry_notifier_info *ven_info) 5673 { 5674 struct mlxsw_sp_mr_table *mrt; 5675 struct mlxsw_sp_vr *vr; 5676 5677 if (mlxsw_sp->router->aborted) 5678 return; 5679 5680 vr = mlxsw_sp_vr_find(mlxsw_sp, ven_info->tb_id); 5681 if (WARN_ON(!vr)) 5682 return; 5683 5684 mrt = mlxsw_sp_router_fibmr_family_to_table(vr, ven_info->info.family); 5685 mlxsw_sp_mr_vif_del(mrt, ven_info->vif_index); 5686 mlxsw_sp_vr_put(mlxsw_sp, vr); 5687 } 5688 5689 static int mlxsw_sp_router_set_abort_trap(struct mlxsw_sp *mlxsw_sp) 5690 { 5691 enum mlxsw_reg_ralxx_protocol proto = MLXSW_REG_RALXX_PROTOCOL_IPV4; 5692 int err; 5693 5694 err = __mlxsw_sp_router_set_abort_trap(mlxsw_sp, proto, 5695 MLXSW_SP_LPM_TREE_MIN); 5696 if (err) 5697 return err; 5698 5699 /* The multicast router code does not need an abort trap as by default, 5700 * packets that don't match any routes are trapped to the CPU. 5701 */ 5702 5703 proto = MLXSW_REG_RALXX_PROTOCOL_IPV6; 5704 return __mlxsw_sp_router_set_abort_trap(mlxsw_sp, proto, 5705 MLXSW_SP_LPM_TREE_MIN + 1); 5706 } 5707 5708 static void mlxsw_sp_fib4_node_flush(struct mlxsw_sp *mlxsw_sp, 5709 struct mlxsw_sp_fib_node *fib_node) 5710 { 5711 struct mlxsw_sp_fib4_entry *fib4_entry, *tmp; 5712 5713 list_for_each_entry_safe(fib4_entry, tmp, &fib_node->entry_list, 5714 common.list) { 5715 bool do_break = &tmp->common.list == &fib_node->entry_list; 5716 5717 mlxsw_sp_fib4_node_entry_unlink(mlxsw_sp, fib4_entry); 5718 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry); 5719 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 5720 /* Break when entry list is empty and node was freed. 5721 * Otherwise, we'll access freed memory in the next 5722 * iteration. 5723 */ 5724 if (do_break) 5725 break; 5726 } 5727 } 5728 5729 static void mlxsw_sp_fib6_node_flush(struct mlxsw_sp *mlxsw_sp, 5730 struct mlxsw_sp_fib_node *fib_node) 5731 { 5732 struct mlxsw_sp_fib6_entry *fib6_entry, *tmp; 5733 5734 list_for_each_entry_safe(fib6_entry, tmp, &fib_node->entry_list, 5735 common.list) { 5736 bool do_break = &tmp->common.list == &fib_node->entry_list; 5737 5738 mlxsw_sp_fib6_node_entry_unlink(mlxsw_sp, fib6_entry); 5739 mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry); 5740 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 5741 if (do_break) 5742 break; 5743 } 5744 } 5745 5746 static void mlxsw_sp_fib_node_flush(struct mlxsw_sp *mlxsw_sp, 5747 struct mlxsw_sp_fib_node *fib_node) 5748 { 5749 switch (fib_node->fib->proto) { 5750 case MLXSW_SP_L3_PROTO_IPV4: 5751 mlxsw_sp_fib4_node_flush(mlxsw_sp, fib_node); 5752 break; 5753 case MLXSW_SP_L3_PROTO_IPV6: 5754 mlxsw_sp_fib6_node_flush(mlxsw_sp, fib_node); 5755 break; 5756 } 5757 } 5758 5759 static void mlxsw_sp_vr_fib_flush(struct mlxsw_sp *mlxsw_sp, 5760 struct mlxsw_sp_vr *vr, 5761 enum mlxsw_sp_l3proto proto) 5762 { 5763 struct mlxsw_sp_fib *fib = mlxsw_sp_vr_fib(vr, proto); 5764 struct mlxsw_sp_fib_node *fib_node, *tmp; 5765 5766 list_for_each_entry_safe(fib_node, tmp, &fib->node_list, list) { 5767 bool do_break = &tmp->list == &fib->node_list; 5768 5769 mlxsw_sp_fib_node_flush(mlxsw_sp, fib_node); 5770 if (do_break) 5771 break; 5772 } 5773 } 5774 5775 static void mlxsw_sp_router_fib_flush(struct mlxsw_sp *mlxsw_sp) 5776 { 5777 int i, j; 5778 5779 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) { 5780 struct mlxsw_sp_vr *vr = &mlxsw_sp->router->vrs[i]; 5781 5782 if (!mlxsw_sp_vr_is_used(vr)) 5783 continue; 5784 5785 for (j = 0; j < MLXSW_SP_L3_PROTO_MAX; j++) 5786 mlxsw_sp_mr_table_flush(vr->mr_table[j]); 5787 mlxsw_sp_vr_fib_flush(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV4); 5788 5789 /* If virtual router was only used for IPv4, then it's no 5790 * longer used. 5791 */ 5792 if (!mlxsw_sp_vr_is_used(vr)) 5793 continue; 5794 mlxsw_sp_vr_fib_flush(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV6); 5795 } 5796 } 5797 5798 static void mlxsw_sp_router_fib_abort(struct mlxsw_sp *mlxsw_sp) 5799 { 5800 int err; 5801 5802 if (mlxsw_sp->router->aborted) 5803 return; 5804 dev_warn(mlxsw_sp->bus_info->dev, "FIB abort triggered. Note that FIB entries are no longer being offloaded to this device.\n"); 5805 mlxsw_sp_router_fib_flush(mlxsw_sp); 5806 mlxsw_sp->router->aborted = true; 5807 err = mlxsw_sp_router_set_abort_trap(mlxsw_sp); 5808 if (err) 5809 dev_warn(mlxsw_sp->bus_info->dev, "Failed to set abort trap.\n"); 5810 } 5811 5812 struct mlxsw_sp_fib_event_work { 5813 struct work_struct work; 5814 union { 5815 struct fib6_entry_notifier_info fen6_info; 5816 struct fib_entry_notifier_info fen_info; 5817 struct fib_rule_notifier_info fr_info; 5818 struct fib_nh_notifier_info fnh_info; 5819 struct mfc_entry_notifier_info men_info; 5820 struct vif_entry_notifier_info ven_info; 5821 }; 5822 struct mlxsw_sp *mlxsw_sp; 5823 unsigned long event; 5824 }; 5825 5826 static void mlxsw_sp_router_fib4_event_work(struct work_struct *work) 5827 { 5828 struct mlxsw_sp_fib_event_work *fib_work = 5829 container_of(work, struct mlxsw_sp_fib_event_work, work); 5830 struct mlxsw_sp *mlxsw_sp = fib_work->mlxsw_sp; 5831 bool replace, append; 5832 int err; 5833 5834 /* Protect internal structures from changes */ 5835 rtnl_lock(); 5836 mlxsw_sp_span_respin(mlxsw_sp); 5837 5838 switch (fib_work->event) { 5839 case FIB_EVENT_ENTRY_REPLACE: /* fall through */ 5840 case FIB_EVENT_ENTRY_APPEND: /* fall through */ 5841 case FIB_EVENT_ENTRY_ADD: 5842 replace = fib_work->event == FIB_EVENT_ENTRY_REPLACE; 5843 append = fib_work->event == FIB_EVENT_ENTRY_APPEND; 5844 err = mlxsw_sp_router_fib4_add(mlxsw_sp, &fib_work->fen_info, 5845 replace, append); 5846 if (err) 5847 mlxsw_sp_router_fib_abort(mlxsw_sp); 5848 fib_info_put(fib_work->fen_info.fi); 5849 break; 5850 case FIB_EVENT_ENTRY_DEL: 5851 mlxsw_sp_router_fib4_del(mlxsw_sp, &fib_work->fen_info); 5852 fib_info_put(fib_work->fen_info.fi); 5853 break; 5854 case FIB_EVENT_RULE_ADD: 5855 /* if we get here, a rule was added that we do not support. 5856 * just do the fib_abort 5857 */ 5858 mlxsw_sp_router_fib_abort(mlxsw_sp); 5859 break; 5860 case FIB_EVENT_NH_ADD: /* fall through */ 5861 case FIB_EVENT_NH_DEL: 5862 mlxsw_sp_nexthop4_event(mlxsw_sp, fib_work->event, 5863 fib_work->fnh_info.fib_nh); 5864 fib_info_put(fib_work->fnh_info.fib_nh->nh_parent); 5865 break; 5866 } 5867 rtnl_unlock(); 5868 kfree(fib_work); 5869 } 5870 5871 static void mlxsw_sp_router_fib6_event_work(struct work_struct *work) 5872 { 5873 struct mlxsw_sp_fib_event_work *fib_work = 5874 container_of(work, struct mlxsw_sp_fib_event_work, work); 5875 struct mlxsw_sp *mlxsw_sp = fib_work->mlxsw_sp; 5876 bool replace; 5877 int err; 5878 5879 rtnl_lock(); 5880 mlxsw_sp_span_respin(mlxsw_sp); 5881 5882 switch (fib_work->event) { 5883 case FIB_EVENT_ENTRY_REPLACE: /* fall through */ 5884 case FIB_EVENT_ENTRY_APPEND: /* fall through */ 5885 case FIB_EVENT_ENTRY_ADD: 5886 replace = fib_work->event == FIB_EVENT_ENTRY_REPLACE; 5887 err = mlxsw_sp_router_fib6_add(mlxsw_sp, 5888 fib_work->fen6_info.rt, replace); 5889 if (err) 5890 mlxsw_sp_router_fib_abort(mlxsw_sp); 5891 mlxsw_sp_rt6_release(fib_work->fen6_info.rt); 5892 break; 5893 case FIB_EVENT_ENTRY_DEL: 5894 mlxsw_sp_router_fib6_del(mlxsw_sp, fib_work->fen6_info.rt); 5895 mlxsw_sp_rt6_release(fib_work->fen6_info.rt); 5896 break; 5897 case FIB_EVENT_RULE_ADD: 5898 /* if we get here, a rule was added that we do not support. 5899 * just do the fib_abort 5900 */ 5901 mlxsw_sp_router_fib_abort(mlxsw_sp); 5902 break; 5903 } 5904 rtnl_unlock(); 5905 kfree(fib_work); 5906 } 5907 5908 static void mlxsw_sp_router_fibmr_event_work(struct work_struct *work) 5909 { 5910 struct mlxsw_sp_fib_event_work *fib_work = 5911 container_of(work, struct mlxsw_sp_fib_event_work, work); 5912 struct mlxsw_sp *mlxsw_sp = fib_work->mlxsw_sp; 5913 bool replace; 5914 int err; 5915 5916 rtnl_lock(); 5917 switch (fib_work->event) { 5918 case FIB_EVENT_ENTRY_REPLACE: /* fall through */ 5919 case FIB_EVENT_ENTRY_ADD: 5920 replace = fib_work->event == FIB_EVENT_ENTRY_REPLACE; 5921 5922 err = mlxsw_sp_router_fibmr_add(mlxsw_sp, &fib_work->men_info, 5923 replace); 5924 if (err) 5925 mlxsw_sp_router_fib_abort(mlxsw_sp); 5926 mr_cache_put(fib_work->men_info.mfc); 5927 break; 5928 case FIB_EVENT_ENTRY_DEL: 5929 mlxsw_sp_router_fibmr_del(mlxsw_sp, &fib_work->men_info); 5930 mr_cache_put(fib_work->men_info.mfc); 5931 break; 5932 case FIB_EVENT_VIF_ADD: 5933 err = mlxsw_sp_router_fibmr_vif_add(mlxsw_sp, 5934 &fib_work->ven_info); 5935 if (err) 5936 mlxsw_sp_router_fib_abort(mlxsw_sp); 5937 dev_put(fib_work->ven_info.dev); 5938 break; 5939 case FIB_EVENT_VIF_DEL: 5940 mlxsw_sp_router_fibmr_vif_del(mlxsw_sp, 5941 &fib_work->ven_info); 5942 dev_put(fib_work->ven_info.dev); 5943 break; 5944 case FIB_EVENT_RULE_ADD: 5945 /* if we get here, a rule was added that we do not support. 5946 * just do the fib_abort 5947 */ 5948 mlxsw_sp_router_fib_abort(mlxsw_sp); 5949 break; 5950 } 5951 rtnl_unlock(); 5952 kfree(fib_work); 5953 } 5954 5955 static void mlxsw_sp_router_fib4_event(struct mlxsw_sp_fib_event_work *fib_work, 5956 struct fib_notifier_info *info) 5957 { 5958 struct fib_entry_notifier_info *fen_info; 5959 struct fib_nh_notifier_info *fnh_info; 5960 5961 switch (fib_work->event) { 5962 case FIB_EVENT_ENTRY_REPLACE: /* fall through */ 5963 case FIB_EVENT_ENTRY_APPEND: /* fall through */ 5964 case FIB_EVENT_ENTRY_ADD: /* fall through */ 5965 case FIB_EVENT_ENTRY_DEL: 5966 fen_info = container_of(info, struct fib_entry_notifier_info, 5967 info); 5968 fib_work->fen_info = *fen_info; 5969 /* Take reference on fib_info to prevent it from being 5970 * freed while work is queued. Release it afterwards. 5971 */ 5972 fib_info_hold(fib_work->fen_info.fi); 5973 break; 5974 case FIB_EVENT_NH_ADD: /* fall through */ 5975 case FIB_EVENT_NH_DEL: 5976 fnh_info = container_of(info, struct fib_nh_notifier_info, 5977 info); 5978 fib_work->fnh_info = *fnh_info; 5979 fib_info_hold(fib_work->fnh_info.fib_nh->nh_parent); 5980 break; 5981 } 5982 } 5983 5984 static void mlxsw_sp_router_fib6_event(struct mlxsw_sp_fib_event_work *fib_work, 5985 struct fib_notifier_info *info) 5986 { 5987 struct fib6_entry_notifier_info *fen6_info; 5988 5989 switch (fib_work->event) { 5990 case FIB_EVENT_ENTRY_REPLACE: /* fall through */ 5991 case FIB_EVENT_ENTRY_APPEND: /* fall through */ 5992 case FIB_EVENT_ENTRY_ADD: /* fall through */ 5993 case FIB_EVENT_ENTRY_DEL: 5994 fen6_info = container_of(info, struct fib6_entry_notifier_info, 5995 info); 5996 fib_work->fen6_info = *fen6_info; 5997 fib6_info_hold(fib_work->fen6_info.rt); 5998 break; 5999 } 6000 } 6001 6002 static void 6003 mlxsw_sp_router_fibmr_event(struct mlxsw_sp_fib_event_work *fib_work, 6004 struct fib_notifier_info *info) 6005 { 6006 switch (fib_work->event) { 6007 case FIB_EVENT_ENTRY_REPLACE: /* fall through */ 6008 case FIB_EVENT_ENTRY_ADD: /* fall through */ 6009 case FIB_EVENT_ENTRY_DEL: 6010 memcpy(&fib_work->men_info, info, sizeof(fib_work->men_info)); 6011 mr_cache_hold(fib_work->men_info.mfc); 6012 break; 6013 case FIB_EVENT_VIF_ADD: /* fall through */ 6014 case FIB_EVENT_VIF_DEL: 6015 memcpy(&fib_work->ven_info, info, sizeof(fib_work->ven_info)); 6016 dev_hold(fib_work->ven_info.dev); 6017 break; 6018 } 6019 } 6020 6021 static int mlxsw_sp_router_fib_rule_event(unsigned long event, 6022 struct fib_notifier_info *info, 6023 struct mlxsw_sp *mlxsw_sp) 6024 { 6025 struct netlink_ext_ack *extack = info->extack; 6026 struct fib_rule_notifier_info *fr_info; 6027 struct fib_rule *rule; 6028 int err = 0; 6029 6030 /* nothing to do at the moment */ 6031 if (event == FIB_EVENT_RULE_DEL) 6032 return 0; 6033 6034 if (mlxsw_sp->router->aborted) 6035 return 0; 6036 6037 fr_info = container_of(info, struct fib_rule_notifier_info, info); 6038 rule = fr_info->rule; 6039 6040 switch (info->family) { 6041 case AF_INET: 6042 if (!fib4_rule_default(rule) && !rule->l3mdev) 6043 err = -EOPNOTSUPP; 6044 break; 6045 case AF_INET6: 6046 if (!fib6_rule_default(rule) && !rule->l3mdev) 6047 err = -EOPNOTSUPP; 6048 break; 6049 case RTNL_FAMILY_IPMR: 6050 if (!ipmr_rule_default(rule) && !rule->l3mdev) 6051 err = -EOPNOTSUPP; 6052 break; 6053 case RTNL_FAMILY_IP6MR: 6054 if (!ip6mr_rule_default(rule) && !rule->l3mdev) 6055 err = -EOPNOTSUPP; 6056 break; 6057 } 6058 6059 if (err < 0) 6060 NL_SET_ERR_MSG_MOD(extack, "FIB rules not supported"); 6061 6062 return err; 6063 } 6064 6065 /* Called with rcu_read_lock() */ 6066 static int mlxsw_sp_router_fib_event(struct notifier_block *nb, 6067 unsigned long event, void *ptr) 6068 { 6069 struct mlxsw_sp_fib_event_work *fib_work; 6070 struct fib_notifier_info *info = ptr; 6071 struct mlxsw_sp_router *router; 6072 int err; 6073 6074 if (!net_eq(info->net, &init_net) || 6075 (info->family != AF_INET && info->family != AF_INET6 && 6076 info->family != RTNL_FAMILY_IPMR && 6077 info->family != RTNL_FAMILY_IP6MR)) 6078 return NOTIFY_DONE; 6079 6080 router = container_of(nb, struct mlxsw_sp_router, fib_nb); 6081 6082 switch (event) { 6083 case FIB_EVENT_RULE_ADD: /* fall through */ 6084 case FIB_EVENT_RULE_DEL: 6085 err = mlxsw_sp_router_fib_rule_event(event, info, 6086 router->mlxsw_sp); 6087 if (!err || info->extack) 6088 return notifier_from_errno(err); 6089 break; 6090 case FIB_EVENT_ENTRY_ADD: 6091 if (router->aborted) { 6092 NL_SET_ERR_MSG_MOD(info->extack, "FIB offload was aborted. Not configuring route"); 6093 return notifier_from_errno(-EINVAL); 6094 } 6095 if (info->family == AF_INET) { 6096 struct fib_entry_notifier_info *fen_info = ptr; 6097 6098 if (fen_info->fi->fib_nh_is_v6) { 6099 NL_SET_ERR_MSG_MOD(info->extack, "IPv6 gateway with IPv4 route is not supported"); 6100 return notifier_from_errno(-EINVAL); 6101 } 6102 } 6103 break; 6104 } 6105 6106 fib_work = kzalloc(sizeof(*fib_work), GFP_ATOMIC); 6107 if (WARN_ON(!fib_work)) 6108 return NOTIFY_BAD; 6109 6110 fib_work->mlxsw_sp = router->mlxsw_sp; 6111 fib_work->event = event; 6112 6113 switch (info->family) { 6114 case AF_INET: 6115 INIT_WORK(&fib_work->work, mlxsw_sp_router_fib4_event_work); 6116 mlxsw_sp_router_fib4_event(fib_work, info); 6117 break; 6118 case AF_INET6: 6119 INIT_WORK(&fib_work->work, mlxsw_sp_router_fib6_event_work); 6120 mlxsw_sp_router_fib6_event(fib_work, info); 6121 break; 6122 case RTNL_FAMILY_IP6MR: 6123 case RTNL_FAMILY_IPMR: 6124 INIT_WORK(&fib_work->work, mlxsw_sp_router_fibmr_event_work); 6125 mlxsw_sp_router_fibmr_event(fib_work, info); 6126 break; 6127 } 6128 6129 mlxsw_core_schedule_work(&fib_work->work); 6130 6131 return NOTIFY_DONE; 6132 } 6133 6134 struct mlxsw_sp_rif * 6135 mlxsw_sp_rif_find_by_dev(const struct mlxsw_sp *mlxsw_sp, 6136 const struct net_device *dev) 6137 { 6138 int i; 6139 6140 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++) 6141 if (mlxsw_sp->router->rifs[i] && 6142 mlxsw_sp->router->rifs[i]->dev == dev) 6143 return mlxsw_sp->router->rifs[i]; 6144 6145 return NULL; 6146 } 6147 6148 static int mlxsw_sp_router_rif_disable(struct mlxsw_sp *mlxsw_sp, u16 rif) 6149 { 6150 char ritr_pl[MLXSW_REG_RITR_LEN]; 6151 int err; 6152 6153 mlxsw_reg_ritr_rif_pack(ritr_pl, rif); 6154 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 6155 if (err) 6156 return err; 6157 6158 mlxsw_reg_ritr_enable_set(ritr_pl, false); 6159 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 6160 } 6161 6162 static void mlxsw_sp_router_rif_gone_sync(struct mlxsw_sp *mlxsw_sp, 6163 struct mlxsw_sp_rif *rif) 6164 { 6165 mlxsw_sp_router_rif_disable(mlxsw_sp, rif->rif_index); 6166 mlxsw_sp_nexthop_rif_gone_sync(mlxsw_sp, rif); 6167 mlxsw_sp_neigh_rif_gone_sync(mlxsw_sp, rif); 6168 } 6169 6170 static bool 6171 mlxsw_sp_rif_should_config(struct mlxsw_sp_rif *rif, struct net_device *dev, 6172 unsigned long event) 6173 { 6174 struct inet6_dev *inet6_dev; 6175 bool addr_list_empty = true; 6176 struct in_device *idev; 6177 6178 switch (event) { 6179 case NETDEV_UP: 6180 return rif == NULL; 6181 case NETDEV_DOWN: 6182 idev = __in_dev_get_rtnl(dev); 6183 if (idev && idev->ifa_list) 6184 addr_list_empty = false; 6185 6186 inet6_dev = __in6_dev_get(dev); 6187 if (addr_list_empty && inet6_dev && 6188 !list_empty(&inet6_dev->addr_list)) 6189 addr_list_empty = false; 6190 6191 /* macvlans do not have a RIF, but rather piggy back on the 6192 * RIF of their lower device. 6193 */ 6194 if (netif_is_macvlan(dev) && addr_list_empty) 6195 return true; 6196 6197 if (rif && addr_list_empty && 6198 !netif_is_l3_slave(rif->dev)) 6199 return true; 6200 /* It is possible we already removed the RIF ourselves 6201 * if it was assigned to a netdev that is now a bridge 6202 * or LAG slave. 6203 */ 6204 return false; 6205 } 6206 6207 return false; 6208 } 6209 6210 static enum mlxsw_sp_rif_type 6211 mlxsw_sp_dev_rif_type(const struct mlxsw_sp *mlxsw_sp, 6212 const struct net_device *dev) 6213 { 6214 enum mlxsw_sp_fid_type type; 6215 6216 if (mlxsw_sp_netdev_ipip_type(mlxsw_sp, dev, NULL)) 6217 return MLXSW_SP_RIF_TYPE_IPIP_LB; 6218 6219 /* Otherwise RIF type is derived from the type of the underlying FID. */ 6220 if (is_vlan_dev(dev) && netif_is_bridge_master(vlan_dev_real_dev(dev))) 6221 type = MLXSW_SP_FID_TYPE_8021Q; 6222 else if (netif_is_bridge_master(dev) && br_vlan_enabled(dev)) 6223 type = MLXSW_SP_FID_TYPE_8021Q; 6224 else if (netif_is_bridge_master(dev)) 6225 type = MLXSW_SP_FID_TYPE_8021D; 6226 else 6227 type = MLXSW_SP_FID_TYPE_RFID; 6228 6229 return mlxsw_sp_fid_type_rif_type(mlxsw_sp, type); 6230 } 6231 6232 static int mlxsw_sp_rif_index_alloc(struct mlxsw_sp *mlxsw_sp, u16 *p_rif_index) 6233 { 6234 int i; 6235 6236 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++) { 6237 if (!mlxsw_sp->router->rifs[i]) { 6238 *p_rif_index = i; 6239 return 0; 6240 } 6241 } 6242 6243 return -ENOBUFS; 6244 } 6245 6246 static struct mlxsw_sp_rif *mlxsw_sp_rif_alloc(size_t rif_size, u16 rif_index, 6247 u16 vr_id, 6248 struct net_device *l3_dev) 6249 { 6250 struct mlxsw_sp_rif *rif; 6251 6252 rif = kzalloc(rif_size, GFP_KERNEL); 6253 if (!rif) 6254 return NULL; 6255 6256 INIT_LIST_HEAD(&rif->nexthop_list); 6257 INIT_LIST_HEAD(&rif->neigh_list); 6258 if (l3_dev) { 6259 ether_addr_copy(rif->addr, l3_dev->dev_addr); 6260 rif->mtu = l3_dev->mtu; 6261 rif->dev = l3_dev; 6262 } 6263 rif->vr_id = vr_id; 6264 rif->rif_index = rif_index; 6265 6266 return rif; 6267 } 6268 6269 struct mlxsw_sp_rif *mlxsw_sp_rif_by_index(const struct mlxsw_sp *mlxsw_sp, 6270 u16 rif_index) 6271 { 6272 return mlxsw_sp->router->rifs[rif_index]; 6273 } 6274 6275 u16 mlxsw_sp_rif_index(const struct mlxsw_sp_rif *rif) 6276 { 6277 return rif->rif_index; 6278 } 6279 6280 u16 mlxsw_sp_ipip_lb_rif_index(const struct mlxsw_sp_rif_ipip_lb *lb_rif) 6281 { 6282 return lb_rif->common.rif_index; 6283 } 6284 6285 u16 mlxsw_sp_ipip_lb_ul_vr_id(const struct mlxsw_sp_rif_ipip_lb *lb_rif) 6286 { 6287 u32 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(lb_rif->common.dev); 6288 struct mlxsw_sp_vr *ul_vr; 6289 6290 ul_vr = mlxsw_sp_vr_get(lb_rif->common.mlxsw_sp, ul_tb_id, NULL); 6291 if (WARN_ON(IS_ERR(ul_vr))) 6292 return 0; 6293 6294 return ul_vr->id; 6295 } 6296 6297 u16 mlxsw_sp_ipip_lb_ul_rif_id(const struct mlxsw_sp_rif_ipip_lb *lb_rif) 6298 { 6299 return lb_rif->ul_rif_id; 6300 } 6301 6302 int mlxsw_sp_rif_dev_ifindex(const struct mlxsw_sp_rif *rif) 6303 { 6304 return rif->dev->ifindex; 6305 } 6306 6307 const struct net_device *mlxsw_sp_rif_dev(const struct mlxsw_sp_rif *rif) 6308 { 6309 return rif->dev; 6310 } 6311 6312 struct mlxsw_sp_fid *mlxsw_sp_rif_fid(const struct mlxsw_sp_rif *rif) 6313 { 6314 return rif->fid; 6315 } 6316 6317 static struct mlxsw_sp_rif * 6318 mlxsw_sp_rif_create(struct mlxsw_sp *mlxsw_sp, 6319 const struct mlxsw_sp_rif_params *params, 6320 struct netlink_ext_ack *extack) 6321 { 6322 u32 tb_id = l3mdev_fib_table(params->dev); 6323 const struct mlxsw_sp_rif_ops *ops; 6324 struct mlxsw_sp_fid *fid = NULL; 6325 enum mlxsw_sp_rif_type type; 6326 struct mlxsw_sp_rif *rif; 6327 struct mlxsw_sp_vr *vr; 6328 u16 rif_index; 6329 int i, err; 6330 6331 type = mlxsw_sp_dev_rif_type(mlxsw_sp, params->dev); 6332 ops = mlxsw_sp->rif_ops_arr[type]; 6333 6334 vr = mlxsw_sp_vr_get(mlxsw_sp, tb_id ? : RT_TABLE_MAIN, extack); 6335 if (IS_ERR(vr)) 6336 return ERR_CAST(vr); 6337 vr->rif_count++; 6338 6339 err = mlxsw_sp_rif_index_alloc(mlxsw_sp, &rif_index); 6340 if (err) { 6341 NL_SET_ERR_MSG_MOD(extack, "Exceeded number of supported router interfaces"); 6342 goto err_rif_index_alloc; 6343 } 6344 6345 rif = mlxsw_sp_rif_alloc(ops->rif_size, rif_index, vr->id, params->dev); 6346 if (!rif) { 6347 err = -ENOMEM; 6348 goto err_rif_alloc; 6349 } 6350 dev_hold(rif->dev); 6351 mlxsw_sp->router->rifs[rif_index] = rif; 6352 rif->mlxsw_sp = mlxsw_sp; 6353 rif->ops = ops; 6354 6355 if (ops->fid_get) { 6356 fid = ops->fid_get(rif, extack); 6357 if (IS_ERR(fid)) { 6358 err = PTR_ERR(fid); 6359 goto err_fid_get; 6360 } 6361 rif->fid = fid; 6362 } 6363 6364 if (ops->setup) 6365 ops->setup(rif, params); 6366 6367 err = ops->configure(rif); 6368 if (err) 6369 goto err_configure; 6370 6371 for (i = 0; i < MLXSW_SP_L3_PROTO_MAX; i++) { 6372 err = mlxsw_sp_mr_rif_add(vr->mr_table[i], rif); 6373 if (err) 6374 goto err_mr_rif_add; 6375 } 6376 6377 mlxsw_sp_rif_counters_alloc(rif); 6378 6379 return rif; 6380 6381 err_mr_rif_add: 6382 for (i--; i >= 0; i--) 6383 mlxsw_sp_mr_rif_del(vr->mr_table[i], rif); 6384 ops->deconfigure(rif); 6385 err_configure: 6386 if (fid) 6387 mlxsw_sp_fid_put(fid); 6388 err_fid_get: 6389 mlxsw_sp->router->rifs[rif_index] = NULL; 6390 dev_put(rif->dev); 6391 kfree(rif); 6392 err_rif_alloc: 6393 err_rif_index_alloc: 6394 vr->rif_count--; 6395 mlxsw_sp_vr_put(mlxsw_sp, vr); 6396 return ERR_PTR(err); 6397 } 6398 6399 static void mlxsw_sp_rif_destroy(struct mlxsw_sp_rif *rif) 6400 { 6401 const struct mlxsw_sp_rif_ops *ops = rif->ops; 6402 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 6403 struct mlxsw_sp_fid *fid = rif->fid; 6404 struct mlxsw_sp_vr *vr; 6405 int i; 6406 6407 mlxsw_sp_router_rif_gone_sync(mlxsw_sp, rif); 6408 vr = &mlxsw_sp->router->vrs[rif->vr_id]; 6409 6410 mlxsw_sp_rif_counters_free(rif); 6411 for (i = 0; i < MLXSW_SP_L3_PROTO_MAX; i++) 6412 mlxsw_sp_mr_rif_del(vr->mr_table[i], rif); 6413 ops->deconfigure(rif); 6414 if (fid) 6415 /* Loopback RIFs are not associated with a FID. */ 6416 mlxsw_sp_fid_put(fid); 6417 mlxsw_sp->router->rifs[rif->rif_index] = NULL; 6418 dev_put(rif->dev); 6419 kfree(rif); 6420 vr->rif_count--; 6421 mlxsw_sp_vr_put(mlxsw_sp, vr); 6422 } 6423 6424 void mlxsw_sp_rif_destroy_by_dev(struct mlxsw_sp *mlxsw_sp, 6425 struct net_device *dev) 6426 { 6427 struct mlxsw_sp_rif *rif; 6428 6429 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev); 6430 if (!rif) 6431 return; 6432 mlxsw_sp_rif_destroy(rif); 6433 } 6434 6435 static void 6436 mlxsw_sp_rif_subport_params_init(struct mlxsw_sp_rif_params *params, 6437 struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan) 6438 { 6439 struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port; 6440 6441 params->vid = mlxsw_sp_port_vlan->vid; 6442 params->lag = mlxsw_sp_port->lagged; 6443 if (params->lag) 6444 params->lag_id = mlxsw_sp_port->lag_id; 6445 else 6446 params->system_port = mlxsw_sp_port->local_port; 6447 } 6448 6449 static struct mlxsw_sp_rif_subport * 6450 mlxsw_sp_rif_subport_rif(const struct mlxsw_sp_rif *rif) 6451 { 6452 return container_of(rif, struct mlxsw_sp_rif_subport, common); 6453 } 6454 6455 static struct mlxsw_sp_rif * 6456 mlxsw_sp_rif_subport_get(struct mlxsw_sp *mlxsw_sp, 6457 const struct mlxsw_sp_rif_params *params, 6458 struct netlink_ext_ack *extack) 6459 { 6460 struct mlxsw_sp_rif_subport *rif_subport; 6461 struct mlxsw_sp_rif *rif; 6462 6463 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, params->dev); 6464 if (!rif) 6465 return mlxsw_sp_rif_create(mlxsw_sp, params, extack); 6466 6467 rif_subport = mlxsw_sp_rif_subport_rif(rif); 6468 refcount_inc(&rif_subport->ref_count); 6469 return rif; 6470 } 6471 6472 static void mlxsw_sp_rif_subport_put(struct mlxsw_sp_rif *rif) 6473 { 6474 struct mlxsw_sp_rif_subport *rif_subport; 6475 6476 rif_subport = mlxsw_sp_rif_subport_rif(rif); 6477 if (!refcount_dec_and_test(&rif_subport->ref_count)) 6478 return; 6479 6480 mlxsw_sp_rif_destroy(rif); 6481 } 6482 6483 static int 6484 mlxsw_sp_port_vlan_router_join(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan, 6485 struct net_device *l3_dev, 6486 struct netlink_ext_ack *extack) 6487 { 6488 struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port; 6489 struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp; 6490 struct mlxsw_sp_rif_params params = { 6491 .dev = l3_dev, 6492 }; 6493 u16 vid = mlxsw_sp_port_vlan->vid; 6494 struct mlxsw_sp_rif *rif; 6495 struct mlxsw_sp_fid *fid; 6496 int err; 6497 6498 mlxsw_sp_rif_subport_params_init(¶ms, mlxsw_sp_port_vlan); 6499 rif = mlxsw_sp_rif_subport_get(mlxsw_sp, ¶ms, extack); 6500 if (IS_ERR(rif)) 6501 return PTR_ERR(rif); 6502 6503 /* FID was already created, just take a reference */ 6504 fid = rif->ops->fid_get(rif, extack); 6505 err = mlxsw_sp_fid_port_vid_map(fid, mlxsw_sp_port, vid); 6506 if (err) 6507 goto err_fid_port_vid_map; 6508 6509 err = mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, false); 6510 if (err) 6511 goto err_port_vid_learning_set; 6512 6513 err = mlxsw_sp_port_vid_stp_set(mlxsw_sp_port, vid, 6514 BR_STATE_FORWARDING); 6515 if (err) 6516 goto err_port_vid_stp_set; 6517 6518 mlxsw_sp_port_vlan->fid = fid; 6519 6520 return 0; 6521 6522 err_port_vid_stp_set: 6523 mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, true); 6524 err_port_vid_learning_set: 6525 mlxsw_sp_fid_port_vid_unmap(fid, mlxsw_sp_port, vid); 6526 err_fid_port_vid_map: 6527 mlxsw_sp_fid_put(fid); 6528 mlxsw_sp_rif_subport_put(rif); 6529 return err; 6530 } 6531 6532 void 6533 mlxsw_sp_port_vlan_router_leave(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan) 6534 { 6535 struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port; 6536 struct mlxsw_sp_fid *fid = mlxsw_sp_port_vlan->fid; 6537 struct mlxsw_sp_rif *rif = mlxsw_sp_fid_rif(fid); 6538 u16 vid = mlxsw_sp_port_vlan->vid; 6539 6540 if (WARN_ON(mlxsw_sp_fid_type(fid) != MLXSW_SP_FID_TYPE_RFID)) 6541 return; 6542 6543 mlxsw_sp_port_vlan->fid = NULL; 6544 mlxsw_sp_port_vid_stp_set(mlxsw_sp_port, vid, BR_STATE_BLOCKING); 6545 mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, true); 6546 mlxsw_sp_fid_port_vid_unmap(fid, mlxsw_sp_port, vid); 6547 mlxsw_sp_fid_put(fid); 6548 mlxsw_sp_rif_subport_put(rif); 6549 } 6550 6551 static int mlxsw_sp_inetaddr_port_vlan_event(struct net_device *l3_dev, 6552 struct net_device *port_dev, 6553 unsigned long event, u16 vid, 6554 struct netlink_ext_ack *extack) 6555 { 6556 struct mlxsw_sp_port *mlxsw_sp_port = netdev_priv(port_dev); 6557 struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan; 6558 6559 mlxsw_sp_port_vlan = mlxsw_sp_port_vlan_find_by_vid(mlxsw_sp_port, vid); 6560 if (WARN_ON(!mlxsw_sp_port_vlan)) 6561 return -EINVAL; 6562 6563 switch (event) { 6564 case NETDEV_UP: 6565 return mlxsw_sp_port_vlan_router_join(mlxsw_sp_port_vlan, 6566 l3_dev, extack); 6567 case NETDEV_DOWN: 6568 mlxsw_sp_port_vlan_router_leave(mlxsw_sp_port_vlan); 6569 break; 6570 } 6571 6572 return 0; 6573 } 6574 6575 static int mlxsw_sp_inetaddr_port_event(struct net_device *port_dev, 6576 unsigned long event, 6577 struct netlink_ext_ack *extack) 6578 { 6579 if (netif_is_bridge_port(port_dev) || 6580 netif_is_lag_port(port_dev) || 6581 netif_is_ovs_port(port_dev)) 6582 return 0; 6583 6584 return mlxsw_sp_inetaddr_port_vlan_event(port_dev, port_dev, event, 6585 MLXSW_SP_DEFAULT_VID, extack); 6586 } 6587 6588 static int __mlxsw_sp_inetaddr_lag_event(struct net_device *l3_dev, 6589 struct net_device *lag_dev, 6590 unsigned long event, u16 vid, 6591 struct netlink_ext_ack *extack) 6592 { 6593 struct net_device *port_dev; 6594 struct list_head *iter; 6595 int err; 6596 6597 netdev_for_each_lower_dev(lag_dev, port_dev, iter) { 6598 if (mlxsw_sp_port_dev_check(port_dev)) { 6599 err = mlxsw_sp_inetaddr_port_vlan_event(l3_dev, 6600 port_dev, 6601 event, vid, 6602 extack); 6603 if (err) 6604 return err; 6605 } 6606 } 6607 6608 return 0; 6609 } 6610 6611 static int mlxsw_sp_inetaddr_lag_event(struct net_device *lag_dev, 6612 unsigned long event, 6613 struct netlink_ext_ack *extack) 6614 { 6615 if (netif_is_bridge_port(lag_dev)) 6616 return 0; 6617 6618 return __mlxsw_sp_inetaddr_lag_event(lag_dev, lag_dev, event, 6619 MLXSW_SP_DEFAULT_VID, extack); 6620 } 6621 6622 static int mlxsw_sp_inetaddr_bridge_event(struct mlxsw_sp *mlxsw_sp, 6623 struct net_device *l3_dev, 6624 unsigned long event, 6625 struct netlink_ext_ack *extack) 6626 { 6627 struct mlxsw_sp_rif_params params = { 6628 .dev = l3_dev, 6629 }; 6630 struct mlxsw_sp_rif *rif; 6631 6632 switch (event) { 6633 case NETDEV_UP: 6634 rif = mlxsw_sp_rif_create(mlxsw_sp, ¶ms, extack); 6635 if (IS_ERR(rif)) 6636 return PTR_ERR(rif); 6637 break; 6638 case NETDEV_DOWN: 6639 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev); 6640 mlxsw_sp_rif_destroy(rif); 6641 break; 6642 } 6643 6644 return 0; 6645 } 6646 6647 static int mlxsw_sp_inetaddr_vlan_event(struct mlxsw_sp *mlxsw_sp, 6648 struct net_device *vlan_dev, 6649 unsigned long event, 6650 struct netlink_ext_ack *extack) 6651 { 6652 struct net_device *real_dev = vlan_dev_real_dev(vlan_dev); 6653 u16 vid = vlan_dev_vlan_id(vlan_dev); 6654 6655 if (netif_is_bridge_port(vlan_dev)) 6656 return 0; 6657 6658 if (mlxsw_sp_port_dev_check(real_dev)) 6659 return mlxsw_sp_inetaddr_port_vlan_event(vlan_dev, real_dev, 6660 event, vid, extack); 6661 else if (netif_is_lag_master(real_dev)) 6662 return __mlxsw_sp_inetaddr_lag_event(vlan_dev, real_dev, event, 6663 vid, extack); 6664 else if (netif_is_bridge_master(real_dev) && br_vlan_enabled(real_dev)) 6665 return mlxsw_sp_inetaddr_bridge_event(mlxsw_sp, vlan_dev, event, 6666 extack); 6667 6668 return 0; 6669 } 6670 6671 static bool mlxsw_sp_rif_macvlan_is_vrrp4(const u8 *mac) 6672 { 6673 u8 vrrp4[ETH_ALEN] = { 0x00, 0x00, 0x5e, 0x00, 0x01, 0x00 }; 6674 u8 mask[ETH_ALEN] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0x00 }; 6675 6676 return ether_addr_equal_masked(mac, vrrp4, mask); 6677 } 6678 6679 static bool mlxsw_sp_rif_macvlan_is_vrrp6(const u8 *mac) 6680 { 6681 u8 vrrp6[ETH_ALEN] = { 0x00, 0x00, 0x5e, 0x00, 0x02, 0x00 }; 6682 u8 mask[ETH_ALEN] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0x00 }; 6683 6684 return ether_addr_equal_masked(mac, vrrp6, mask); 6685 } 6686 6687 static int mlxsw_sp_rif_vrrp_op(struct mlxsw_sp *mlxsw_sp, u16 rif_index, 6688 const u8 *mac, bool adding) 6689 { 6690 char ritr_pl[MLXSW_REG_RITR_LEN]; 6691 u8 vrrp_id = adding ? mac[5] : 0; 6692 int err; 6693 6694 if (!mlxsw_sp_rif_macvlan_is_vrrp4(mac) && 6695 !mlxsw_sp_rif_macvlan_is_vrrp6(mac)) 6696 return 0; 6697 6698 mlxsw_reg_ritr_rif_pack(ritr_pl, rif_index); 6699 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 6700 if (err) 6701 return err; 6702 6703 if (mlxsw_sp_rif_macvlan_is_vrrp4(mac)) 6704 mlxsw_reg_ritr_if_vrrp_id_ipv4_set(ritr_pl, vrrp_id); 6705 else 6706 mlxsw_reg_ritr_if_vrrp_id_ipv6_set(ritr_pl, vrrp_id); 6707 6708 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 6709 } 6710 6711 static int mlxsw_sp_rif_macvlan_add(struct mlxsw_sp *mlxsw_sp, 6712 const struct net_device *macvlan_dev, 6713 struct netlink_ext_ack *extack) 6714 { 6715 struct macvlan_dev *vlan = netdev_priv(macvlan_dev); 6716 struct mlxsw_sp_rif *rif; 6717 int err; 6718 6719 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, vlan->lowerdev); 6720 if (!rif) { 6721 NL_SET_ERR_MSG_MOD(extack, "macvlan is only supported on top of router interfaces"); 6722 return -EOPNOTSUPP; 6723 } 6724 6725 err = mlxsw_sp_rif_fdb_op(mlxsw_sp, macvlan_dev->dev_addr, 6726 mlxsw_sp_fid_index(rif->fid), true); 6727 if (err) 6728 return err; 6729 6730 err = mlxsw_sp_rif_vrrp_op(mlxsw_sp, rif->rif_index, 6731 macvlan_dev->dev_addr, true); 6732 if (err) 6733 goto err_rif_vrrp_add; 6734 6735 /* Make sure the bridge driver does not have this MAC pointing at 6736 * some other port. 6737 */ 6738 if (rif->ops->fdb_del) 6739 rif->ops->fdb_del(rif, macvlan_dev->dev_addr); 6740 6741 return 0; 6742 6743 err_rif_vrrp_add: 6744 mlxsw_sp_rif_fdb_op(mlxsw_sp, macvlan_dev->dev_addr, 6745 mlxsw_sp_fid_index(rif->fid), false); 6746 return err; 6747 } 6748 6749 void mlxsw_sp_rif_macvlan_del(struct mlxsw_sp *mlxsw_sp, 6750 const struct net_device *macvlan_dev) 6751 { 6752 struct macvlan_dev *vlan = netdev_priv(macvlan_dev); 6753 struct mlxsw_sp_rif *rif; 6754 6755 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, vlan->lowerdev); 6756 /* If we do not have a RIF, then we already took care of 6757 * removing the macvlan's MAC during RIF deletion. 6758 */ 6759 if (!rif) 6760 return; 6761 mlxsw_sp_rif_vrrp_op(mlxsw_sp, rif->rif_index, macvlan_dev->dev_addr, 6762 false); 6763 mlxsw_sp_rif_fdb_op(mlxsw_sp, macvlan_dev->dev_addr, 6764 mlxsw_sp_fid_index(rif->fid), false); 6765 } 6766 6767 static int mlxsw_sp_inetaddr_macvlan_event(struct mlxsw_sp *mlxsw_sp, 6768 struct net_device *macvlan_dev, 6769 unsigned long event, 6770 struct netlink_ext_ack *extack) 6771 { 6772 switch (event) { 6773 case NETDEV_UP: 6774 return mlxsw_sp_rif_macvlan_add(mlxsw_sp, macvlan_dev, extack); 6775 case NETDEV_DOWN: 6776 mlxsw_sp_rif_macvlan_del(mlxsw_sp, macvlan_dev); 6777 break; 6778 } 6779 6780 return 0; 6781 } 6782 6783 static int mlxsw_sp_router_port_check_rif_addr(struct mlxsw_sp *mlxsw_sp, 6784 struct net_device *dev, 6785 const unsigned char *dev_addr, 6786 struct netlink_ext_ack *extack) 6787 { 6788 struct mlxsw_sp_rif *rif; 6789 int i; 6790 6791 /* A RIF is not created for macvlan netdevs. Their MAC is used to 6792 * populate the FDB 6793 */ 6794 if (netif_is_macvlan(dev)) 6795 return 0; 6796 6797 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++) { 6798 rif = mlxsw_sp->router->rifs[i]; 6799 if (rif && rif->dev && rif->dev != dev && 6800 !ether_addr_equal_masked(rif->dev->dev_addr, dev_addr, 6801 mlxsw_sp->mac_mask)) { 6802 NL_SET_ERR_MSG_MOD(extack, "All router interface MAC addresses must have the same prefix"); 6803 return -EINVAL; 6804 } 6805 } 6806 6807 return 0; 6808 } 6809 6810 static int __mlxsw_sp_inetaddr_event(struct mlxsw_sp *mlxsw_sp, 6811 struct net_device *dev, 6812 unsigned long event, 6813 struct netlink_ext_ack *extack) 6814 { 6815 if (mlxsw_sp_port_dev_check(dev)) 6816 return mlxsw_sp_inetaddr_port_event(dev, event, extack); 6817 else if (netif_is_lag_master(dev)) 6818 return mlxsw_sp_inetaddr_lag_event(dev, event, extack); 6819 else if (netif_is_bridge_master(dev)) 6820 return mlxsw_sp_inetaddr_bridge_event(mlxsw_sp, dev, event, 6821 extack); 6822 else if (is_vlan_dev(dev)) 6823 return mlxsw_sp_inetaddr_vlan_event(mlxsw_sp, dev, event, 6824 extack); 6825 else if (netif_is_macvlan(dev)) 6826 return mlxsw_sp_inetaddr_macvlan_event(mlxsw_sp, dev, event, 6827 extack); 6828 else 6829 return 0; 6830 } 6831 6832 static int mlxsw_sp_inetaddr_event(struct notifier_block *nb, 6833 unsigned long event, void *ptr) 6834 { 6835 struct in_ifaddr *ifa = (struct in_ifaddr *) ptr; 6836 struct net_device *dev = ifa->ifa_dev->dev; 6837 struct mlxsw_sp_router *router; 6838 struct mlxsw_sp_rif *rif; 6839 int err = 0; 6840 6841 /* NETDEV_UP event is handled by mlxsw_sp_inetaddr_valid_event */ 6842 if (event == NETDEV_UP) 6843 goto out; 6844 6845 router = container_of(nb, struct mlxsw_sp_router, inetaddr_nb); 6846 rif = mlxsw_sp_rif_find_by_dev(router->mlxsw_sp, dev); 6847 if (!mlxsw_sp_rif_should_config(rif, dev, event)) 6848 goto out; 6849 6850 err = __mlxsw_sp_inetaddr_event(router->mlxsw_sp, dev, event, NULL); 6851 out: 6852 return notifier_from_errno(err); 6853 } 6854 6855 int mlxsw_sp_inetaddr_valid_event(struct notifier_block *unused, 6856 unsigned long event, void *ptr) 6857 { 6858 struct in_validator_info *ivi = (struct in_validator_info *) ptr; 6859 struct net_device *dev = ivi->ivi_dev->dev; 6860 struct mlxsw_sp *mlxsw_sp; 6861 struct mlxsw_sp_rif *rif; 6862 int err = 0; 6863 6864 mlxsw_sp = mlxsw_sp_lower_get(dev); 6865 if (!mlxsw_sp) 6866 goto out; 6867 6868 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev); 6869 if (!mlxsw_sp_rif_should_config(rif, dev, event)) 6870 goto out; 6871 6872 err = mlxsw_sp_router_port_check_rif_addr(mlxsw_sp, dev, dev->dev_addr, 6873 ivi->extack); 6874 if (err) 6875 goto out; 6876 6877 err = __mlxsw_sp_inetaddr_event(mlxsw_sp, dev, event, ivi->extack); 6878 out: 6879 return notifier_from_errno(err); 6880 } 6881 6882 struct mlxsw_sp_inet6addr_event_work { 6883 struct work_struct work; 6884 struct mlxsw_sp *mlxsw_sp; 6885 struct net_device *dev; 6886 unsigned long event; 6887 }; 6888 6889 static void mlxsw_sp_inet6addr_event_work(struct work_struct *work) 6890 { 6891 struct mlxsw_sp_inet6addr_event_work *inet6addr_work = 6892 container_of(work, struct mlxsw_sp_inet6addr_event_work, work); 6893 struct mlxsw_sp *mlxsw_sp = inet6addr_work->mlxsw_sp; 6894 struct net_device *dev = inet6addr_work->dev; 6895 unsigned long event = inet6addr_work->event; 6896 struct mlxsw_sp_rif *rif; 6897 6898 rtnl_lock(); 6899 6900 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev); 6901 if (!mlxsw_sp_rif_should_config(rif, dev, event)) 6902 goto out; 6903 6904 __mlxsw_sp_inetaddr_event(mlxsw_sp, dev, event, NULL); 6905 out: 6906 rtnl_unlock(); 6907 dev_put(dev); 6908 kfree(inet6addr_work); 6909 } 6910 6911 /* Called with rcu_read_lock() */ 6912 static int mlxsw_sp_inet6addr_event(struct notifier_block *nb, 6913 unsigned long event, void *ptr) 6914 { 6915 struct inet6_ifaddr *if6 = (struct inet6_ifaddr *) ptr; 6916 struct mlxsw_sp_inet6addr_event_work *inet6addr_work; 6917 struct net_device *dev = if6->idev->dev; 6918 struct mlxsw_sp_router *router; 6919 6920 /* NETDEV_UP event is handled by mlxsw_sp_inet6addr_valid_event */ 6921 if (event == NETDEV_UP) 6922 return NOTIFY_DONE; 6923 6924 inet6addr_work = kzalloc(sizeof(*inet6addr_work), GFP_ATOMIC); 6925 if (!inet6addr_work) 6926 return NOTIFY_BAD; 6927 6928 router = container_of(nb, struct mlxsw_sp_router, inet6addr_nb); 6929 INIT_WORK(&inet6addr_work->work, mlxsw_sp_inet6addr_event_work); 6930 inet6addr_work->mlxsw_sp = router->mlxsw_sp; 6931 inet6addr_work->dev = dev; 6932 inet6addr_work->event = event; 6933 dev_hold(dev); 6934 mlxsw_core_schedule_work(&inet6addr_work->work); 6935 6936 return NOTIFY_DONE; 6937 } 6938 6939 int mlxsw_sp_inet6addr_valid_event(struct notifier_block *unused, 6940 unsigned long event, void *ptr) 6941 { 6942 struct in6_validator_info *i6vi = (struct in6_validator_info *) ptr; 6943 struct net_device *dev = i6vi->i6vi_dev->dev; 6944 struct mlxsw_sp *mlxsw_sp; 6945 struct mlxsw_sp_rif *rif; 6946 int err = 0; 6947 6948 mlxsw_sp = mlxsw_sp_lower_get(dev); 6949 if (!mlxsw_sp) 6950 goto out; 6951 6952 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev); 6953 if (!mlxsw_sp_rif_should_config(rif, dev, event)) 6954 goto out; 6955 6956 err = mlxsw_sp_router_port_check_rif_addr(mlxsw_sp, dev, dev->dev_addr, 6957 i6vi->extack); 6958 if (err) 6959 goto out; 6960 6961 err = __mlxsw_sp_inetaddr_event(mlxsw_sp, dev, event, i6vi->extack); 6962 out: 6963 return notifier_from_errno(err); 6964 } 6965 6966 static int mlxsw_sp_rif_edit(struct mlxsw_sp *mlxsw_sp, u16 rif_index, 6967 const char *mac, int mtu) 6968 { 6969 char ritr_pl[MLXSW_REG_RITR_LEN]; 6970 int err; 6971 6972 mlxsw_reg_ritr_rif_pack(ritr_pl, rif_index); 6973 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 6974 if (err) 6975 return err; 6976 6977 mlxsw_reg_ritr_mtu_set(ritr_pl, mtu); 6978 mlxsw_reg_ritr_if_mac_memcpy_to(ritr_pl, mac); 6979 mlxsw_reg_ritr_op_set(ritr_pl, MLXSW_REG_RITR_RIF_CREATE); 6980 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 6981 } 6982 6983 static int 6984 mlxsw_sp_router_port_change_event(struct mlxsw_sp *mlxsw_sp, 6985 struct mlxsw_sp_rif *rif) 6986 { 6987 struct net_device *dev = rif->dev; 6988 u16 fid_index; 6989 int err; 6990 6991 fid_index = mlxsw_sp_fid_index(rif->fid); 6992 6993 err = mlxsw_sp_rif_fdb_op(mlxsw_sp, rif->addr, fid_index, false); 6994 if (err) 6995 return err; 6996 6997 err = mlxsw_sp_rif_edit(mlxsw_sp, rif->rif_index, dev->dev_addr, 6998 dev->mtu); 6999 if (err) 7000 goto err_rif_edit; 7001 7002 err = mlxsw_sp_rif_fdb_op(mlxsw_sp, dev->dev_addr, fid_index, true); 7003 if (err) 7004 goto err_rif_fdb_op; 7005 7006 if (rif->mtu != dev->mtu) { 7007 struct mlxsw_sp_vr *vr; 7008 int i; 7009 7010 /* The RIF is relevant only to its mr_table instance, as unlike 7011 * unicast routing, in multicast routing a RIF cannot be shared 7012 * between several multicast routing tables. 7013 */ 7014 vr = &mlxsw_sp->router->vrs[rif->vr_id]; 7015 for (i = 0; i < MLXSW_SP_L3_PROTO_MAX; i++) 7016 mlxsw_sp_mr_rif_mtu_update(vr->mr_table[i], 7017 rif, dev->mtu); 7018 } 7019 7020 ether_addr_copy(rif->addr, dev->dev_addr); 7021 rif->mtu = dev->mtu; 7022 7023 netdev_dbg(dev, "Updated RIF=%d\n", rif->rif_index); 7024 7025 return 0; 7026 7027 err_rif_fdb_op: 7028 mlxsw_sp_rif_edit(mlxsw_sp, rif->rif_index, rif->addr, rif->mtu); 7029 err_rif_edit: 7030 mlxsw_sp_rif_fdb_op(mlxsw_sp, rif->addr, fid_index, true); 7031 return err; 7032 } 7033 7034 static int mlxsw_sp_router_port_pre_changeaddr_event(struct mlxsw_sp_rif *rif, 7035 struct netdev_notifier_pre_changeaddr_info *info) 7036 { 7037 struct netlink_ext_ack *extack; 7038 7039 extack = netdev_notifier_info_to_extack(&info->info); 7040 return mlxsw_sp_router_port_check_rif_addr(rif->mlxsw_sp, rif->dev, 7041 info->dev_addr, extack); 7042 } 7043 7044 int mlxsw_sp_netdevice_router_port_event(struct net_device *dev, 7045 unsigned long event, void *ptr) 7046 { 7047 struct mlxsw_sp *mlxsw_sp; 7048 struct mlxsw_sp_rif *rif; 7049 7050 mlxsw_sp = mlxsw_sp_lower_get(dev); 7051 if (!mlxsw_sp) 7052 return 0; 7053 7054 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev); 7055 if (!rif) 7056 return 0; 7057 7058 switch (event) { 7059 case NETDEV_CHANGEMTU: /* fall through */ 7060 case NETDEV_CHANGEADDR: 7061 return mlxsw_sp_router_port_change_event(mlxsw_sp, rif); 7062 case NETDEV_PRE_CHANGEADDR: 7063 return mlxsw_sp_router_port_pre_changeaddr_event(rif, ptr); 7064 } 7065 7066 return 0; 7067 } 7068 7069 static int mlxsw_sp_port_vrf_join(struct mlxsw_sp *mlxsw_sp, 7070 struct net_device *l3_dev, 7071 struct netlink_ext_ack *extack) 7072 { 7073 struct mlxsw_sp_rif *rif; 7074 7075 /* If netdev is already associated with a RIF, then we need to 7076 * destroy it and create a new one with the new virtual router ID. 7077 */ 7078 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev); 7079 if (rif) 7080 __mlxsw_sp_inetaddr_event(mlxsw_sp, l3_dev, NETDEV_DOWN, 7081 extack); 7082 7083 return __mlxsw_sp_inetaddr_event(mlxsw_sp, l3_dev, NETDEV_UP, extack); 7084 } 7085 7086 static void mlxsw_sp_port_vrf_leave(struct mlxsw_sp *mlxsw_sp, 7087 struct net_device *l3_dev) 7088 { 7089 struct mlxsw_sp_rif *rif; 7090 7091 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev); 7092 if (!rif) 7093 return; 7094 __mlxsw_sp_inetaddr_event(mlxsw_sp, l3_dev, NETDEV_DOWN, NULL); 7095 } 7096 7097 int mlxsw_sp_netdevice_vrf_event(struct net_device *l3_dev, unsigned long event, 7098 struct netdev_notifier_changeupper_info *info) 7099 { 7100 struct mlxsw_sp *mlxsw_sp = mlxsw_sp_lower_get(l3_dev); 7101 int err = 0; 7102 7103 /* We do not create a RIF for a macvlan, but only use it to 7104 * direct more MAC addresses to the router. 7105 */ 7106 if (!mlxsw_sp || netif_is_macvlan(l3_dev)) 7107 return 0; 7108 7109 switch (event) { 7110 case NETDEV_PRECHANGEUPPER: 7111 return 0; 7112 case NETDEV_CHANGEUPPER: 7113 if (info->linking) { 7114 struct netlink_ext_ack *extack; 7115 7116 extack = netdev_notifier_info_to_extack(&info->info); 7117 err = mlxsw_sp_port_vrf_join(mlxsw_sp, l3_dev, extack); 7118 } else { 7119 mlxsw_sp_port_vrf_leave(mlxsw_sp, l3_dev); 7120 } 7121 break; 7122 } 7123 7124 return err; 7125 } 7126 7127 static int __mlxsw_sp_rif_macvlan_flush(struct net_device *dev, void *data) 7128 { 7129 struct mlxsw_sp_rif *rif = data; 7130 7131 if (!netif_is_macvlan(dev)) 7132 return 0; 7133 7134 return mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, dev->dev_addr, 7135 mlxsw_sp_fid_index(rif->fid), false); 7136 } 7137 7138 static int mlxsw_sp_rif_macvlan_flush(struct mlxsw_sp_rif *rif) 7139 { 7140 if (!netif_is_macvlan_port(rif->dev)) 7141 return 0; 7142 7143 netdev_warn(rif->dev, "Router interface is deleted. Upper macvlans will not work\n"); 7144 return netdev_walk_all_upper_dev_rcu(rif->dev, 7145 __mlxsw_sp_rif_macvlan_flush, rif); 7146 } 7147 7148 static void mlxsw_sp_rif_subport_setup(struct mlxsw_sp_rif *rif, 7149 const struct mlxsw_sp_rif_params *params) 7150 { 7151 struct mlxsw_sp_rif_subport *rif_subport; 7152 7153 rif_subport = mlxsw_sp_rif_subport_rif(rif); 7154 refcount_set(&rif_subport->ref_count, 1); 7155 rif_subport->vid = params->vid; 7156 rif_subport->lag = params->lag; 7157 if (params->lag) 7158 rif_subport->lag_id = params->lag_id; 7159 else 7160 rif_subport->system_port = params->system_port; 7161 } 7162 7163 static int mlxsw_sp_rif_subport_op(struct mlxsw_sp_rif *rif, bool enable) 7164 { 7165 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 7166 struct mlxsw_sp_rif_subport *rif_subport; 7167 char ritr_pl[MLXSW_REG_RITR_LEN]; 7168 7169 rif_subport = mlxsw_sp_rif_subport_rif(rif); 7170 mlxsw_reg_ritr_pack(ritr_pl, enable, MLXSW_REG_RITR_SP_IF, 7171 rif->rif_index, rif->vr_id, rif->dev->mtu); 7172 mlxsw_reg_ritr_mac_pack(ritr_pl, rif->dev->dev_addr); 7173 mlxsw_reg_ritr_sp_if_pack(ritr_pl, rif_subport->lag, 7174 rif_subport->lag ? rif_subport->lag_id : 7175 rif_subport->system_port, 7176 rif_subport->vid); 7177 7178 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 7179 } 7180 7181 static int mlxsw_sp_rif_subport_configure(struct mlxsw_sp_rif *rif) 7182 { 7183 int err; 7184 7185 err = mlxsw_sp_rif_subport_op(rif, true); 7186 if (err) 7187 return err; 7188 7189 err = mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr, 7190 mlxsw_sp_fid_index(rif->fid), true); 7191 if (err) 7192 goto err_rif_fdb_op; 7193 7194 mlxsw_sp_fid_rif_set(rif->fid, rif); 7195 return 0; 7196 7197 err_rif_fdb_op: 7198 mlxsw_sp_rif_subport_op(rif, false); 7199 return err; 7200 } 7201 7202 static void mlxsw_sp_rif_subport_deconfigure(struct mlxsw_sp_rif *rif) 7203 { 7204 struct mlxsw_sp_fid *fid = rif->fid; 7205 7206 mlxsw_sp_fid_rif_set(fid, NULL); 7207 mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr, 7208 mlxsw_sp_fid_index(fid), false); 7209 mlxsw_sp_rif_macvlan_flush(rif); 7210 mlxsw_sp_rif_subport_op(rif, false); 7211 } 7212 7213 static struct mlxsw_sp_fid * 7214 mlxsw_sp_rif_subport_fid_get(struct mlxsw_sp_rif *rif, 7215 struct netlink_ext_ack *extack) 7216 { 7217 return mlxsw_sp_fid_rfid_get(rif->mlxsw_sp, rif->rif_index); 7218 } 7219 7220 static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_subport_ops = { 7221 .type = MLXSW_SP_RIF_TYPE_SUBPORT, 7222 .rif_size = sizeof(struct mlxsw_sp_rif_subport), 7223 .setup = mlxsw_sp_rif_subport_setup, 7224 .configure = mlxsw_sp_rif_subport_configure, 7225 .deconfigure = mlxsw_sp_rif_subport_deconfigure, 7226 .fid_get = mlxsw_sp_rif_subport_fid_get, 7227 }; 7228 7229 static int mlxsw_sp_rif_vlan_fid_op(struct mlxsw_sp_rif *rif, 7230 enum mlxsw_reg_ritr_if_type type, 7231 u16 vid_fid, bool enable) 7232 { 7233 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 7234 char ritr_pl[MLXSW_REG_RITR_LEN]; 7235 7236 mlxsw_reg_ritr_pack(ritr_pl, enable, type, rif->rif_index, rif->vr_id, 7237 rif->dev->mtu); 7238 mlxsw_reg_ritr_mac_pack(ritr_pl, rif->dev->dev_addr); 7239 mlxsw_reg_ritr_fid_set(ritr_pl, type, vid_fid); 7240 7241 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 7242 } 7243 7244 u8 mlxsw_sp_router_port(const struct mlxsw_sp *mlxsw_sp) 7245 { 7246 return mlxsw_core_max_ports(mlxsw_sp->core) + 1; 7247 } 7248 7249 static int mlxsw_sp_rif_vlan_configure(struct mlxsw_sp_rif *rif) 7250 { 7251 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 7252 u16 vid = mlxsw_sp_fid_8021q_vid(rif->fid); 7253 int err; 7254 7255 err = mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_VLAN_IF, vid, true); 7256 if (err) 7257 return err; 7258 7259 err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC, 7260 mlxsw_sp_router_port(mlxsw_sp), true); 7261 if (err) 7262 goto err_fid_mc_flood_set; 7263 7264 err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC, 7265 mlxsw_sp_router_port(mlxsw_sp), true); 7266 if (err) 7267 goto err_fid_bc_flood_set; 7268 7269 err = mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr, 7270 mlxsw_sp_fid_index(rif->fid), true); 7271 if (err) 7272 goto err_rif_fdb_op; 7273 7274 mlxsw_sp_fid_rif_set(rif->fid, rif); 7275 return 0; 7276 7277 err_rif_fdb_op: 7278 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC, 7279 mlxsw_sp_router_port(mlxsw_sp), false); 7280 err_fid_bc_flood_set: 7281 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC, 7282 mlxsw_sp_router_port(mlxsw_sp), false); 7283 err_fid_mc_flood_set: 7284 mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_VLAN_IF, vid, false); 7285 return err; 7286 } 7287 7288 static void mlxsw_sp_rif_vlan_deconfigure(struct mlxsw_sp_rif *rif) 7289 { 7290 u16 vid = mlxsw_sp_fid_8021q_vid(rif->fid); 7291 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 7292 struct mlxsw_sp_fid *fid = rif->fid; 7293 7294 mlxsw_sp_fid_rif_set(fid, NULL); 7295 mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr, 7296 mlxsw_sp_fid_index(fid), false); 7297 mlxsw_sp_rif_macvlan_flush(rif); 7298 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC, 7299 mlxsw_sp_router_port(mlxsw_sp), false); 7300 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC, 7301 mlxsw_sp_router_port(mlxsw_sp), false); 7302 mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_VLAN_IF, vid, false); 7303 } 7304 7305 static struct mlxsw_sp_fid * 7306 mlxsw_sp_rif_vlan_fid_get(struct mlxsw_sp_rif *rif, 7307 struct netlink_ext_ack *extack) 7308 { 7309 struct net_device *br_dev = rif->dev; 7310 u16 vid; 7311 int err; 7312 7313 if (is_vlan_dev(rif->dev)) { 7314 vid = vlan_dev_vlan_id(rif->dev); 7315 br_dev = vlan_dev_real_dev(rif->dev); 7316 if (WARN_ON(!netif_is_bridge_master(br_dev))) 7317 return ERR_PTR(-EINVAL); 7318 } else { 7319 err = br_vlan_get_pvid(rif->dev, &vid); 7320 if (err < 0 || !vid) { 7321 NL_SET_ERR_MSG_MOD(extack, "Couldn't determine bridge PVID"); 7322 return ERR_PTR(-EINVAL); 7323 } 7324 } 7325 7326 return mlxsw_sp_bridge_fid_get(rif->mlxsw_sp, br_dev, vid, extack); 7327 } 7328 7329 static void mlxsw_sp_rif_vlan_fdb_del(struct mlxsw_sp_rif *rif, const char *mac) 7330 { 7331 u16 vid = mlxsw_sp_fid_8021q_vid(rif->fid); 7332 struct switchdev_notifier_fdb_info info; 7333 struct net_device *br_dev; 7334 struct net_device *dev; 7335 7336 br_dev = is_vlan_dev(rif->dev) ? vlan_dev_real_dev(rif->dev) : rif->dev; 7337 dev = br_fdb_find_port(br_dev, mac, vid); 7338 if (!dev) 7339 return; 7340 7341 info.addr = mac; 7342 info.vid = vid; 7343 call_switchdev_notifiers(SWITCHDEV_FDB_DEL_TO_BRIDGE, dev, &info.info, 7344 NULL); 7345 } 7346 7347 static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_vlan_ops = { 7348 .type = MLXSW_SP_RIF_TYPE_VLAN, 7349 .rif_size = sizeof(struct mlxsw_sp_rif), 7350 .configure = mlxsw_sp_rif_vlan_configure, 7351 .deconfigure = mlxsw_sp_rif_vlan_deconfigure, 7352 .fid_get = mlxsw_sp_rif_vlan_fid_get, 7353 .fdb_del = mlxsw_sp_rif_vlan_fdb_del, 7354 }; 7355 7356 static int mlxsw_sp_rif_fid_configure(struct mlxsw_sp_rif *rif) 7357 { 7358 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 7359 u16 fid_index = mlxsw_sp_fid_index(rif->fid); 7360 int err; 7361 7362 err = mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_FID_IF, fid_index, 7363 true); 7364 if (err) 7365 return err; 7366 7367 err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC, 7368 mlxsw_sp_router_port(mlxsw_sp), true); 7369 if (err) 7370 goto err_fid_mc_flood_set; 7371 7372 err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC, 7373 mlxsw_sp_router_port(mlxsw_sp), true); 7374 if (err) 7375 goto err_fid_bc_flood_set; 7376 7377 err = mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr, 7378 mlxsw_sp_fid_index(rif->fid), true); 7379 if (err) 7380 goto err_rif_fdb_op; 7381 7382 mlxsw_sp_fid_rif_set(rif->fid, rif); 7383 return 0; 7384 7385 err_rif_fdb_op: 7386 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC, 7387 mlxsw_sp_router_port(mlxsw_sp), false); 7388 err_fid_bc_flood_set: 7389 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC, 7390 mlxsw_sp_router_port(mlxsw_sp), false); 7391 err_fid_mc_flood_set: 7392 mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_FID_IF, fid_index, false); 7393 return err; 7394 } 7395 7396 static void mlxsw_sp_rif_fid_deconfigure(struct mlxsw_sp_rif *rif) 7397 { 7398 u16 fid_index = mlxsw_sp_fid_index(rif->fid); 7399 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 7400 struct mlxsw_sp_fid *fid = rif->fid; 7401 7402 mlxsw_sp_fid_rif_set(fid, NULL); 7403 mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr, 7404 mlxsw_sp_fid_index(fid), false); 7405 mlxsw_sp_rif_macvlan_flush(rif); 7406 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC, 7407 mlxsw_sp_router_port(mlxsw_sp), false); 7408 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC, 7409 mlxsw_sp_router_port(mlxsw_sp), false); 7410 mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_FID_IF, fid_index, false); 7411 } 7412 7413 static struct mlxsw_sp_fid * 7414 mlxsw_sp_rif_fid_fid_get(struct mlxsw_sp_rif *rif, 7415 struct netlink_ext_ack *extack) 7416 { 7417 return mlxsw_sp_bridge_fid_get(rif->mlxsw_sp, rif->dev, 0, extack); 7418 } 7419 7420 static void mlxsw_sp_rif_fid_fdb_del(struct mlxsw_sp_rif *rif, const char *mac) 7421 { 7422 struct switchdev_notifier_fdb_info info; 7423 struct net_device *dev; 7424 7425 dev = br_fdb_find_port(rif->dev, mac, 0); 7426 if (!dev) 7427 return; 7428 7429 info.addr = mac; 7430 info.vid = 0; 7431 call_switchdev_notifiers(SWITCHDEV_FDB_DEL_TO_BRIDGE, dev, &info.info, 7432 NULL); 7433 } 7434 7435 static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_fid_ops = { 7436 .type = MLXSW_SP_RIF_TYPE_FID, 7437 .rif_size = sizeof(struct mlxsw_sp_rif), 7438 .configure = mlxsw_sp_rif_fid_configure, 7439 .deconfigure = mlxsw_sp_rif_fid_deconfigure, 7440 .fid_get = mlxsw_sp_rif_fid_fid_get, 7441 .fdb_del = mlxsw_sp_rif_fid_fdb_del, 7442 }; 7443 7444 static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_vlan_emu_ops = { 7445 .type = MLXSW_SP_RIF_TYPE_VLAN, 7446 .rif_size = sizeof(struct mlxsw_sp_rif), 7447 .configure = mlxsw_sp_rif_fid_configure, 7448 .deconfigure = mlxsw_sp_rif_fid_deconfigure, 7449 .fid_get = mlxsw_sp_rif_vlan_fid_get, 7450 .fdb_del = mlxsw_sp_rif_vlan_fdb_del, 7451 }; 7452 7453 static struct mlxsw_sp_rif_ipip_lb * 7454 mlxsw_sp_rif_ipip_lb_rif(struct mlxsw_sp_rif *rif) 7455 { 7456 return container_of(rif, struct mlxsw_sp_rif_ipip_lb, common); 7457 } 7458 7459 static void 7460 mlxsw_sp_rif_ipip_lb_setup(struct mlxsw_sp_rif *rif, 7461 const struct mlxsw_sp_rif_params *params) 7462 { 7463 struct mlxsw_sp_rif_params_ipip_lb *params_lb; 7464 struct mlxsw_sp_rif_ipip_lb *rif_lb; 7465 7466 params_lb = container_of(params, struct mlxsw_sp_rif_params_ipip_lb, 7467 common); 7468 rif_lb = mlxsw_sp_rif_ipip_lb_rif(rif); 7469 rif_lb->lb_config = params_lb->lb_config; 7470 } 7471 7472 static int 7473 mlxsw_sp1_rif_ipip_lb_configure(struct mlxsw_sp_rif *rif) 7474 { 7475 struct mlxsw_sp_rif_ipip_lb *lb_rif = mlxsw_sp_rif_ipip_lb_rif(rif); 7476 u32 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(rif->dev); 7477 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 7478 struct mlxsw_sp_vr *ul_vr; 7479 int err; 7480 7481 ul_vr = mlxsw_sp_vr_get(mlxsw_sp, ul_tb_id, NULL); 7482 if (IS_ERR(ul_vr)) 7483 return PTR_ERR(ul_vr); 7484 7485 err = mlxsw_sp_rif_ipip_lb_op(lb_rif, ul_vr->id, 0, true); 7486 if (err) 7487 goto err_loopback_op; 7488 7489 lb_rif->ul_vr_id = ul_vr->id; 7490 lb_rif->ul_rif_id = 0; 7491 ++ul_vr->rif_count; 7492 return 0; 7493 7494 err_loopback_op: 7495 mlxsw_sp_vr_put(mlxsw_sp, ul_vr); 7496 return err; 7497 } 7498 7499 static void mlxsw_sp1_rif_ipip_lb_deconfigure(struct mlxsw_sp_rif *rif) 7500 { 7501 struct mlxsw_sp_rif_ipip_lb *lb_rif = mlxsw_sp_rif_ipip_lb_rif(rif); 7502 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 7503 struct mlxsw_sp_vr *ul_vr; 7504 7505 ul_vr = &mlxsw_sp->router->vrs[lb_rif->ul_vr_id]; 7506 mlxsw_sp_rif_ipip_lb_op(lb_rif, ul_vr->id, 0, false); 7507 7508 --ul_vr->rif_count; 7509 mlxsw_sp_vr_put(mlxsw_sp, ul_vr); 7510 } 7511 7512 static const struct mlxsw_sp_rif_ops mlxsw_sp1_rif_ipip_lb_ops = { 7513 .type = MLXSW_SP_RIF_TYPE_IPIP_LB, 7514 .rif_size = sizeof(struct mlxsw_sp_rif_ipip_lb), 7515 .setup = mlxsw_sp_rif_ipip_lb_setup, 7516 .configure = mlxsw_sp1_rif_ipip_lb_configure, 7517 .deconfigure = mlxsw_sp1_rif_ipip_lb_deconfigure, 7518 }; 7519 7520 const struct mlxsw_sp_rif_ops *mlxsw_sp1_rif_ops_arr[] = { 7521 [MLXSW_SP_RIF_TYPE_SUBPORT] = &mlxsw_sp_rif_subport_ops, 7522 [MLXSW_SP_RIF_TYPE_VLAN] = &mlxsw_sp_rif_vlan_emu_ops, 7523 [MLXSW_SP_RIF_TYPE_FID] = &mlxsw_sp_rif_fid_ops, 7524 [MLXSW_SP_RIF_TYPE_IPIP_LB] = &mlxsw_sp1_rif_ipip_lb_ops, 7525 }; 7526 7527 static int 7528 mlxsw_sp_rif_ipip_lb_ul_rif_op(struct mlxsw_sp_rif *ul_rif, bool enable) 7529 { 7530 struct mlxsw_sp *mlxsw_sp = ul_rif->mlxsw_sp; 7531 char ritr_pl[MLXSW_REG_RITR_LEN]; 7532 7533 mlxsw_reg_ritr_pack(ritr_pl, enable, MLXSW_REG_RITR_LOOPBACK_IF, 7534 ul_rif->rif_index, ul_rif->vr_id, IP_MAX_MTU); 7535 mlxsw_reg_ritr_loopback_protocol_set(ritr_pl, 7536 MLXSW_REG_RITR_LOOPBACK_GENERIC); 7537 7538 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 7539 } 7540 7541 static struct mlxsw_sp_rif * 7542 mlxsw_sp_ul_rif_create(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp_vr *vr, 7543 struct netlink_ext_ack *extack) 7544 { 7545 struct mlxsw_sp_rif *ul_rif; 7546 u16 rif_index; 7547 int err; 7548 7549 err = mlxsw_sp_rif_index_alloc(mlxsw_sp, &rif_index); 7550 if (err) { 7551 NL_SET_ERR_MSG_MOD(extack, "Exceeded number of supported router interfaces"); 7552 return ERR_PTR(err); 7553 } 7554 7555 ul_rif = mlxsw_sp_rif_alloc(sizeof(*ul_rif), rif_index, vr->id, NULL); 7556 if (!ul_rif) 7557 return ERR_PTR(-ENOMEM); 7558 7559 mlxsw_sp->router->rifs[rif_index] = ul_rif; 7560 ul_rif->mlxsw_sp = mlxsw_sp; 7561 err = mlxsw_sp_rif_ipip_lb_ul_rif_op(ul_rif, true); 7562 if (err) 7563 goto ul_rif_op_err; 7564 7565 return ul_rif; 7566 7567 ul_rif_op_err: 7568 mlxsw_sp->router->rifs[rif_index] = NULL; 7569 kfree(ul_rif); 7570 return ERR_PTR(err); 7571 } 7572 7573 static void mlxsw_sp_ul_rif_destroy(struct mlxsw_sp_rif *ul_rif) 7574 { 7575 struct mlxsw_sp *mlxsw_sp = ul_rif->mlxsw_sp; 7576 7577 mlxsw_sp_rif_ipip_lb_ul_rif_op(ul_rif, false); 7578 mlxsw_sp->router->rifs[ul_rif->rif_index] = NULL; 7579 kfree(ul_rif); 7580 } 7581 7582 static struct mlxsw_sp_rif * 7583 mlxsw_sp_ul_rif_get(struct mlxsw_sp *mlxsw_sp, u32 tb_id, 7584 struct netlink_ext_ack *extack) 7585 { 7586 struct mlxsw_sp_vr *vr; 7587 int err; 7588 7589 vr = mlxsw_sp_vr_get(mlxsw_sp, tb_id, extack); 7590 if (IS_ERR(vr)) 7591 return ERR_CAST(vr); 7592 7593 if (refcount_inc_not_zero(&vr->ul_rif_refcnt)) 7594 return vr->ul_rif; 7595 7596 vr->ul_rif = mlxsw_sp_ul_rif_create(mlxsw_sp, vr, extack); 7597 if (IS_ERR(vr->ul_rif)) { 7598 err = PTR_ERR(vr->ul_rif); 7599 goto err_ul_rif_create; 7600 } 7601 7602 vr->rif_count++; 7603 refcount_set(&vr->ul_rif_refcnt, 1); 7604 7605 return vr->ul_rif; 7606 7607 err_ul_rif_create: 7608 mlxsw_sp_vr_put(mlxsw_sp, vr); 7609 return ERR_PTR(err); 7610 } 7611 7612 static void mlxsw_sp_ul_rif_put(struct mlxsw_sp_rif *ul_rif) 7613 { 7614 struct mlxsw_sp *mlxsw_sp = ul_rif->mlxsw_sp; 7615 struct mlxsw_sp_vr *vr; 7616 7617 vr = &mlxsw_sp->router->vrs[ul_rif->vr_id]; 7618 7619 if (!refcount_dec_and_test(&vr->ul_rif_refcnt)) 7620 return; 7621 7622 vr->rif_count--; 7623 mlxsw_sp_ul_rif_destroy(ul_rif); 7624 mlxsw_sp_vr_put(mlxsw_sp, vr); 7625 } 7626 7627 int mlxsw_sp_router_ul_rif_get(struct mlxsw_sp *mlxsw_sp, u32 ul_tb_id, 7628 u16 *ul_rif_index) 7629 { 7630 struct mlxsw_sp_rif *ul_rif; 7631 7632 ASSERT_RTNL(); 7633 7634 ul_rif = mlxsw_sp_ul_rif_get(mlxsw_sp, ul_tb_id, NULL); 7635 if (IS_ERR(ul_rif)) 7636 return PTR_ERR(ul_rif); 7637 *ul_rif_index = ul_rif->rif_index; 7638 7639 return 0; 7640 } 7641 7642 void mlxsw_sp_router_ul_rif_put(struct mlxsw_sp *mlxsw_sp, u16 ul_rif_index) 7643 { 7644 struct mlxsw_sp_rif *ul_rif; 7645 7646 ASSERT_RTNL(); 7647 7648 ul_rif = mlxsw_sp->router->rifs[ul_rif_index]; 7649 if (WARN_ON(!ul_rif)) 7650 return; 7651 7652 mlxsw_sp_ul_rif_put(ul_rif); 7653 } 7654 7655 static int 7656 mlxsw_sp2_rif_ipip_lb_configure(struct mlxsw_sp_rif *rif) 7657 { 7658 struct mlxsw_sp_rif_ipip_lb *lb_rif = mlxsw_sp_rif_ipip_lb_rif(rif); 7659 u32 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(rif->dev); 7660 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 7661 struct mlxsw_sp_rif *ul_rif; 7662 int err; 7663 7664 ul_rif = mlxsw_sp_ul_rif_get(mlxsw_sp, ul_tb_id, NULL); 7665 if (IS_ERR(ul_rif)) 7666 return PTR_ERR(ul_rif); 7667 7668 err = mlxsw_sp_rif_ipip_lb_op(lb_rif, 0, ul_rif->rif_index, true); 7669 if (err) 7670 goto err_loopback_op; 7671 7672 lb_rif->ul_vr_id = 0; 7673 lb_rif->ul_rif_id = ul_rif->rif_index; 7674 7675 return 0; 7676 7677 err_loopback_op: 7678 mlxsw_sp_ul_rif_put(ul_rif); 7679 return err; 7680 } 7681 7682 static void mlxsw_sp2_rif_ipip_lb_deconfigure(struct mlxsw_sp_rif *rif) 7683 { 7684 struct mlxsw_sp_rif_ipip_lb *lb_rif = mlxsw_sp_rif_ipip_lb_rif(rif); 7685 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 7686 struct mlxsw_sp_rif *ul_rif; 7687 7688 ul_rif = mlxsw_sp_rif_by_index(mlxsw_sp, lb_rif->ul_rif_id); 7689 mlxsw_sp_rif_ipip_lb_op(lb_rif, 0, lb_rif->ul_rif_id, false); 7690 mlxsw_sp_ul_rif_put(ul_rif); 7691 } 7692 7693 static const struct mlxsw_sp_rif_ops mlxsw_sp2_rif_ipip_lb_ops = { 7694 .type = MLXSW_SP_RIF_TYPE_IPIP_LB, 7695 .rif_size = sizeof(struct mlxsw_sp_rif_ipip_lb), 7696 .setup = mlxsw_sp_rif_ipip_lb_setup, 7697 .configure = mlxsw_sp2_rif_ipip_lb_configure, 7698 .deconfigure = mlxsw_sp2_rif_ipip_lb_deconfigure, 7699 }; 7700 7701 const struct mlxsw_sp_rif_ops *mlxsw_sp2_rif_ops_arr[] = { 7702 [MLXSW_SP_RIF_TYPE_SUBPORT] = &mlxsw_sp_rif_subport_ops, 7703 [MLXSW_SP_RIF_TYPE_VLAN] = &mlxsw_sp_rif_vlan_emu_ops, 7704 [MLXSW_SP_RIF_TYPE_FID] = &mlxsw_sp_rif_fid_ops, 7705 [MLXSW_SP_RIF_TYPE_IPIP_LB] = &mlxsw_sp2_rif_ipip_lb_ops, 7706 }; 7707 7708 static int mlxsw_sp_rifs_init(struct mlxsw_sp *mlxsw_sp) 7709 { 7710 u64 max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); 7711 7712 mlxsw_sp->router->rifs = kcalloc(max_rifs, 7713 sizeof(struct mlxsw_sp_rif *), 7714 GFP_KERNEL); 7715 if (!mlxsw_sp->router->rifs) 7716 return -ENOMEM; 7717 7718 return 0; 7719 } 7720 7721 static void mlxsw_sp_rifs_fini(struct mlxsw_sp *mlxsw_sp) 7722 { 7723 int i; 7724 7725 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++) 7726 WARN_ON_ONCE(mlxsw_sp->router->rifs[i]); 7727 7728 kfree(mlxsw_sp->router->rifs); 7729 } 7730 7731 static int 7732 mlxsw_sp_ipip_config_tigcr(struct mlxsw_sp *mlxsw_sp) 7733 { 7734 char tigcr_pl[MLXSW_REG_TIGCR_LEN]; 7735 7736 mlxsw_reg_tigcr_pack(tigcr_pl, true, 0); 7737 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(tigcr), tigcr_pl); 7738 } 7739 7740 static int mlxsw_sp_ipips_init(struct mlxsw_sp *mlxsw_sp) 7741 { 7742 mlxsw_sp->router->ipip_ops_arr = mlxsw_sp_ipip_ops_arr; 7743 INIT_LIST_HEAD(&mlxsw_sp->router->ipip_list); 7744 return mlxsw_sp_ipip_config_tigcr(mlxsw_sp); 7745 } 7746 7747 static void mlxsw_sp_ipips_fini(struct mlxsw_sp *mlxsw_sp) 7748 { 7749 WARN_ON(!list_empty(&mlxsw_sp->router->ipip_list)); 7750 } 7751 7752 static void mlxsw_sp_router_fib_dump_flush(struct notifier_block *nb) 7753 { 7754 struct mlxsw_sp_router *router; 7755 7756 /* Flush pending FIB notifications and then flush the device's 7757 * table before requesting another dump. The FIB notification 7758 * block is unregistered, so no need to take RTNL. 7759 */ 7760 mlxsw_core_flush_owq(); 7761 router = container_of(nb, struct mlxsw_sp_router, fib_nb); 7762 mlxsw_sp_router_fib_flush(router->mlxsw_sp); 7763 } 7764 7765 #ifdef CONFIG_IP_ROUTE_MULTIPATH 7766 static void mlxsw_sp_mp_hash_header_set(char *recr2_pl, int header) 7767 { 7768 mlxsw_reg_recr2_outer_header_enables_set(recr2_pl, header, true); 7769 } 7770 7771 static void mlxsw_sp_mp_hash_field_set(char *recr2_pl, int field) 7772 { 7773 mlxsw_reg_recr2_outer_header_fields_enable_set(recr2_pl, field, true); 7774 } 7775 7776 static void mlxsw_sp_mp4_hash_init(char *recr2_pl) 7777 { 7778 bool only_l3 = !init_net.ipv4.sysctl_fib_multipath_hash_policy; 7779 7780 mlxsw_sp_mp_hash_header_set(recr2_pl, 7781 MLXSW_REG_RECR2_IPV4_EN_NOT_TCP_NOT_UDP); 7782 mlxsw_sp_mp_hash_header_set(recr2_pl, MLXSW_REG_RECR2_IPV4_EN_TCP_UDP); 7783 mlxsw_reg_recr2_ipv4_sip_enable(recr2_pl); 7784 mlxsw_reg_recr2_ipv4_dip_enable(recr2_pl); 7785 if (only_l3) 7786 return; 7787 mlxsw_sp_mp_hash_header_set(recr2_pl, MLXSW_REG_RECR2_TCP_UDP_EN_IPV4); 7788 mlxsw_sp_mp_hash_field_set(recr2_pl, MLXSW_REG_RECR2_IPV4_PROTOCOL); 7789 mlxsw_sp_mp_hash_field_set(recr2_pl, MLXSW_REG_RECR2_TCP_UDP_SPORT); 7790 mlxsw_sp_mp_hash_field_set(recr2_pl, MLXSW_REG_RECR2_TCP_UDP_DPORT); 7791 } 7792 7793 static void mlxsw_sp_mp6_hash_init(char *recr2_pl) 7794 { 7795 bool only_l3 = !ip6_multipath_hash_policy(&init_net); 7796 7797 mlxsw_sp_mp_hash_header_set(recr2_pl, 7798 MLXSW_REG_RECR2_IPV6_EN_NOT_TCP_NOT_UDP); 7799 mlxsw_sp_mp_hash_header_set(recr2_pl, MLXSW_REG_RECR2_IPV6_EN_TCP_UDP); 7800 mlxsw_reg_recr2_ipv6_sip_enable(recr2_pl); 7801 mlxsw_reg_recr2_ipv6_dip_enable(recr2_pl); 7802 mlxsw_sp_mp_hash_field_set(recr2_pl, MLXSW_REG_RECR2_IPV6_NEXT_HEADER); 7803 if (only_l3) { 7804 mlxsw_sp_mp_hash_field_set(recr2_pl, 7805 MLXSW_REG_RECR2_IPV6_FLOW_LABEL); 7806 } else { 7807 mlxsw_sp_mp_hash_header_set(recr2_pl, 7808 MLXSW_REG_RECR2_TCP_UDP_EN_IPV6); 7809 mlxsw_sp_mp_hash_field_set(recr2_pl, 7810 MLXSW_REG_RECR2_TCP_UDP_SPORT); 7811 mlxsw_sp_mp_hash_field_set(recr2_pl, 7812 MLXSW_REG_RECR2_TCP_UDP_DPORT); 7813 } 7814 } 7815 7816 static int mlxsw_sp_mp_hash_init(struct mlxsw_sp *mlxsw_sp) 7817 { 7818 char recr2_pl[MLXSW_REG_RECR2_LEN]; 7819 u32 seed; 7820 7821 get_random_bytes(&seed, sizeof(seed)); 7822 mlxsw_reg_recr2_pack(recr2_pl, seed); 7823 mlxsw_sp_mp4_hash_init(recr2_pl); 7824 mlxsw_sp_mp6_hash_init(recr2_pl); 7825 7826 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(recr2), recr2_pl); 7827 } 7828 #else 7829 static int mlxsw_sp_mp_hash_init(struct mlxsw_sp *mlxsw_sp) 7830 { 7831 return 0; 7832 } 7833 #endif 7834 7835 static int mlxsw_sp_dscp_init(struct mlxsw_sp *mlxsw_sp) 7836 { 7837 char rdpm_pl[MLXSW_REG_RDPM_LEN]; 7838 unsigned int i; 7839 7840 MLXSW_REG_ZERO(rdpm, rdpm_pl); 7841 7842 /* HW is determining switch priority based on DSCP-bits, but the 7843 * kernel is still doing that based on the ToS. Since there's a 7844 * mismatch in bits we need to make sure to translate the right 7845 * value ToS would observe, skipping the 2 least-significant ECN bits. 7846 */ 7847 for (i = 0; i < MLXSW_REG_RDPM_DSCP_ENTRY_REC_MAX_COUNT; i++) 7848 mlxsw_reg_rdpm_pack(rdpm_pl, i, rt_tos2priority(i << 2)); 7849 7850 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rdpm), rdpm_pl); 7851 } 7852 7853 static int __mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp) 7854 { 7855 bool usp = init_net.ipv4.sysctl_ip_fwd_update_priority; 7856 char rgcr_pl[MLXSW_REG_RGCR_LEN]; 7857 u64 max_rifs; 7858 int err; 7859 7860 if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_RIFS)) 7861 return -EIO; 7862 max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); 7863 7864 mlxsw_reg_rgcr_pack(rgcr_pl, true, true); 7865 mlxsw_reg_rgcr_max_router_interfaces_set(rgcr_pl, max_rifs); 7866 mlxsw_reg_rgcr_usp_set(rgcr_pl, usp); 7867 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rgcr), rgcr_pl); 7868 if (err) 7869 return err; 7870 return 0; 7871 } 7872 7873 static void __mlxsw_sp_router_fini(struct mlxsw_sp *mlxsw_sp) 7874 { 7875 char rgcr_pl[MLXSW_REG_RGCR_LEN]; 7876 7877 mlxsw_reg_rgcr_pack(rgcr_pl, false, false); 7878 mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rgcr), rgcr_pl); 7879 } 7880 7881 int mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp) 7882 { 7883 struct mlxsw_sp_router *router; 7884 int err; 7885 7886 router = kzalloc(sizeof(*mlxsw_sp->router), GFP_KERNEL); 7887 if (!router) 7888 return -ENOMEM; 7889 mlxsw_sp->router = router; 7890 router->mlxsw_sp = mlxsw_sp; 7891 7892 router->inetaddr_nb.notifier_call = mlxsw_sp_inetaddr_event; 7893 err = register_inetaddr_notifier(&router->inetaddr_nb); 7894 if (err) 7895 goto err_register_inetaddr_notifier; 7896 7897 router->inet6addr_nb.notifier_call = mlxsw_sp_inet6addr_event; 7898 err = register_inet6addr_notifier(&router->inet6addr_nb); 7899 if (err) 7900 goto err_register_inet6addr_notifier; 7901 7902 INIT_LIST_HEAD(&mlxsw_sp->router->nexthop_neighs_list); 7903 err = __mlxsw_sp_router_init(mlxsw_sp); 7904 if (err) 7905 goto err_router_init; 7906 7907 err = mlxsw_sp_rifs_init(mlxsw_sp); 7908 if (err) 7909 goto err_rifs_init; 7910 7911 err = mlxsw_sp_ipips_init(mlxsw_sp); 7912 if (err) 7913 goto err_ipips_init; 7914 7915 err = rhashtable_init(&mlxsw_sp->router->nexthop_ht, 7916 &mlxsw_sp_nexthop_ht_params); 7917 if (err) 7918 goto err_nexthop_ht_init; 7919 7920 err = rhashtable_init(&mlxsw_sp->router->nexthop_group_ht, 7921 &mlxsw_sp_nexthop_group_ht_params); 7922 if (err) 7923 goto err_nexthop_group_ht_init; 7924 7925 INIT_LIST_HEAD(&mlxsw_sp->router->nexthop_list); 7926 err = mlxsw_sp_lpm_init(mlxsw_sp); 7927 if (err) 7928 goto err_lpm_init; 7929 7930 err = mlxsw_sp_mr_init(mlxsw_sp, &mlxsw_sp_mr_tcam_ops); 7931 if (err) 7932 goto err_mr_init; 7933 7934 err = mlxsw_sp_vrs_init(mlxsw_sp); 7935 if (err) 7936 goto err_vrs_init; 7937 7938 err = mlxsw_sp_neigh_init(mlxsw_sp); 7939 if (err) 7940 goto err_neigh_init; 7941 7942 mlxsw_sp->router->netevent_nb.notifier_call = 7943 mlxsw_sp_router_netevent_event; 7944 err = register_netevent_notifier(&mlxsw_sp->router->netevent_nb); 7945 if (err) 7946 goto err_register_netevent_notifier; 7947 7948 err = mlxsw_sp_mp_hash_init(mlxsw_sp); 7949 if (err) 7950 goto err_mp_hash_init; 7951 7952 err = mlxsw_sp_dscp_init(mlxsw_sp); 7953 if (err) 7954 goto err_dscp_init; 7955 7956 mlxsw_sp->router->fib_nb.notifier_call = mlxsw_sp_router_fib_event; 7957 err = register_fib_notifier(&mlxsw_sp->router->fib_nb, 7958 mlxsw_sp_router_fib_dump_flush); 7959 if (err) 7960 goto err_register_fib_notifier; 7961 7962 return 0; 7963 7964 err_register_fib_notifier: 7965 err_dscp_init: 7966 err_mp_hash_init: 7967 unregister_netevent_notifier(&mlxsw_sp->router->netevent_nb); 7968 err_register_netevent_notifier: 7969 mlxsw_sp_neigh_fini(mlxsw_sp); 7970 err_neigh_init: 7971 mlxsw_sp_vrs_fini(mlxsw_sp); 7972 err_vrs_init: 7973 mlxsw_sp_mr_fini(mlxsw_sp); 7974 err_mr_init: 7975 mlxsw_sp_lpm_fini(mlxsw_sp); 7976 err_lpm_init: 7977 rhashtable_destroy(&mlxsw_sp->router->nexthop_group_ht); 7978 err_nexthop_group_ht_init: 7979 rhashtable_destroy(&mlxsw_sp->router->nexthop_ht); 7980 err_nexthop_ht_init: 7981 mlxsw_sp_ipips_fini(mlxsw_sp); 7982 err_ipips_init: 7983 mlxsw_sp_rifs_fini(mlxsw_sp); 7984 err_rifs_init: 7985 __mlxsw_sp_router_fini(mlxsw_sp); 7986 err_router_init: 7987 unregister_inet6addr_notifier(&router->inet6addr_nb); 7988 err_register_inet6addr_notifier: 7989 unregister_inetaddr_notifier(&router->inetaddr_nb); 7990 err_register_inetaddr_notifier: 7991 kfree(mlxsw_sp->router); 7992 return err; 7993 } 7994 7995 void mlxsw_sp_router_fini(struct mlxsw_sp *mlxsw_sp) 7996 { 7997 unregister_fib_notifier(&mlxsw_sp->router->fib_nb); 7998 unregister_netevent_notifier(&mlxsw_sp->router->netevent_nb); 7999 mlxsw_sp_neigh_fini(mlxsw_sp); 8000 mlxsw_sp_vrs_fini(mlxsw_sp); 8001 mlxsw_sp_mr_fini(mlxsw_sp); 8002 mlxsw_sp_lpm_fini(mlxsw_sp); 8003 rhashtable_destroy(&mlxsw_sp->router->nexthop_group_ht); 8004 rhashtable_destroy(&mlxsw_sp->router->nexthop_ht); 8005 mlxsw_sp_ipips_fini(mlxsw_sp); 8006 mlxsw_sp_rifs_fini(mlxsw_sp); 8007 __mlxsw_sp_router_fini(mlxsw_sp); 8008 unregister_inet6addr_notifier(&mlxsw_sp->router->inet6addr_nb); 8009 unregister_inetaddr_notifier(&mlxsw_sp->router->inetaddr_nb); 8010 kfree(mlxsw_sp->router); 8011 } 8012