1 // SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0 2 /* Copyright (c) 2016-2018 Mellanox Technologies. All rights reserved */ 3 4 #include <linux/kernel.h> 5 #include <linux/types.h> 6 #include <linux/rhashtable.h> 7 #include <linux/bitops.h> 8 #include <linux/in6.h> 9 #include <linux/notifier.h> 10 #include <linux/inetdevice.h> 11 #include <linux/netdevice.h> 12 #include <linux/if_bridge.h> 13 #include <linux/socket.h> 14 #include <linux/route.h> 15 #include <linux/gcd.h> 16 #include <linux/if_macvlan.h> 17 #include <linux/refcount.h> 18 #include <linux/jhash.h> 19 #include <linux/net_namespace.h> 20 #include <linux/mutex.h> 21 #include <net/netevent.h> 22 #include <net/neighbour.h> 23 #include <net/arp.h> 24 #include <net/ip_fib.h> 25 #include <net/ip6_fib.h> 26 #include <net/nexthop.h> 27 #include <net/fib_rules.h> 28 #include <net/ip_tunnels.h> 29 #include <net/l3mdev.h> 30 #include <net/addrconf.h> 31 #include <net/ndisc.h> 32 #include <net/ipv6.h> 33 #include <net/fib_notifier.h> 34 #include <net/switchdev.h> 35 36 #include "spectrum.h" 37 #include "core.h" 38 #include "reg.h" 39 #include "spectrum_cnt.h" 40 #include "spectrum_dpipe.h" 41 #include "spectrum_ipip.h" 42 #include "spectrum_mr.h" 43 #include "spectrum_mr_tcam.h" 44 #include "spectrum_router.h" 45 #include "spectrum_span.h" 46 47 struct mlxsw_sp_fib; 48 struct mlxsw_sp_vr; 49 struct mlxsw_sp_lpm_tree; 50 struct mlxsw_sp_rif_ops; 51 52 struct mlxsw_sp_rif { 53 struct list_head nexthop_list; 54 struct list_head neigh_list; 55 struct net_device *dev; /* NULL for underlay RIF */ 56 struct mlxsw_sp_fid *fid; 57 unsigned char addr[ETH_ALEN]; 58 int mtu; 59 u16 rif_index; 60 u16 vr_id; 61 const struct mlxsw_sp_rif_ops *ops; 62 struct mlxsw_sp *mlxsw_sp; 63 64 unsigned int counter_ingress; 65 bool counter_ingress_valid; 66 unsigned int counter_egress; 67 bool counter_egress_valid; 68 }; 69 70 struct mlxsw_sp_rif_params { 71 struct net_device *dev; 72 union { 73 u16 system_port; 74 u16 lag_id; 75 }; 76 u16 vid; 77 bool lag; 78 }; 79 80 struct mlxsw_sp_rif_subport { 81 struct mlxsw_sp_rif common; 82 refcount_t ref_count; 83 union { 84 u16 system_port; 85 u16 lag_id; 86 }; 87 u16 vid; 88 bool lag; 89 }; 90 91 struct mlxsw_sp_rif_ipip_lb { 92 struct mlxsw_sp_rif common; 93 struct mlxsw_sp_rif_ipip_lb_config lb_config; 94 u16 ul_vr_id; /* Reserved for Spectrum-2. */ 95 u16 ul_rif_id; /* Reserved for Spectrum. */ 96 }; 97 98 struct mlxsw_sp_rif_params_ipip_lb { 99 struct mlxsw_sp_rif_params common; 100 struct mlxsw_sp_rif_ipip_lb_config lb_config; 101 }; 102 103 struct mlxsw_sp_rif_ops { 104 enum mlxsw_sp_rif_type type; 105 size_t rif_size; 106 107 void (*setup)(struct mlxsw_sp_rif *rif, 108 const struct mlxsw_sp_rif_params *params); 109 int (*configure)(struct mlxsw_sp_rif *rif); 110 void (*deconfigure)(struct mlxsw_sp_rif *rif); 111 struct mlxsw_sp_fid * (*fid_get)(struct mlxsw_sp_rif *rif, 112 struct netlink_ext_ack *extack); 113 void (*fdb_del)(struct mlxsw_sp_rif *rif, const char *mac); 114 }; 115 116 struct mlxsw_sp_router_ops { 117 int (*init)(struct mlxsw_sp *mlxsw_sp); 118 }; 119 120 static struct mlxsw_sp_rif * 121 mlxsw_sp_rif_find_by_dev(const struct mlxsw_sp *mlxsw_sp, 122 const struct net_device *dev); 123 static void mlxsw_sp_rif_destroy(struct mlxsw_sp_rif *rif); 124 static void mlxsw_sp_lpm_tree_hold(struct mlxsw_sp_lpm_tree *lpm_tree); 125 static void mlxsw_sp_lpm_tree_put(struct mlxsw_sp *mlxsw_sp, 126 struct mlxsw_sp_lpm_tree *lpm_tree); 127 static int mlxsw_sp_vr_lpm_tree_bind(struct mlxsw_sp *mlxsw_sp, 128 const struct mlxsw_sp_fib *fib, 129 u8 tree_id); 130 static int mlxsw_sp_vr_lpm_tree_unbind(struct mlxsw_sp *mlxsw_sp, 131 const struct mlxsw_sp_fib *fib); 132 133 static unsigned int * 134 mlxsw_sp_rif_p_counter_get(struct mlxsw_sp_rif *rif, 135 enum mlxsw_sp_rif_counter_dir dir) 136 { 137 switch (dir) { 138 case MLXSW_SP_RIF_COUNTER_EGRESS: 139 return &rif->counter_egress; 140 case MLXSW_SP_RIF_COUNTER_INGRESS: 141 return &rif->counter_ingress; 142 } 143 return NULL; 144 } 145 146 static bool 147 mlxsw_sp_rif_counter_valid_get(struct mlxsw_sp_rif *rif, 148 enum mlxsw_sp_rif_counter_dir dir) 149 { 150 switch (dir) { 151 case MLXSW_SP_RIF_COUNTER_EGRESS: 152 return rif->counter_egress_valid; 153 case MLXSW_SP_RIF_COUNTER_INGRESS: 154 return rif->counter_ingress_valid; 155 } 156 return false; 157 } 158 159 static void 160 mlxsw_sp_rif_counter_valid_set(struct mlxsw_sp_rif *rif, 161 enum mlxsw_sp_rif_counter_dir dir, 162 bool valid) 163 { 164 switch (dir) { 165 case MLXSW_SP_RIF_COUNTER_EGRESS: 166 rif->counter_egress_valid = valid; 167 break; 168 case MLXSW_SP_RIF_COUNTER_INGRESS: 169 rif->counter_ingress_valid = valid; 170 break; 171 } 172 } 173 174 static int mlxsw_sp_rif_counter_edit(struct mlxsw_sp *mlxsw_sp, u16 rif_index, 175 unsigned int counter_index, bool enable, 176 enum mlxsw_sp_rif_counter_dir dir) 177 { 178 char ritr_pl[MLXSW_REG_RITR_LEN]; 179 bool is_egress = false; 180 int err; 181 182 if (dir == MLXSW_SP_RIF_COUNTER_EGRESS) 183 is_egress = true; 184 mlxsw_reg_ritr_rif_pack(ritr_pl, rif_index); 185 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 186 if (err) 187 return err; 188 189 mlxsw_reg_ritr_counter_pack(ritr_pl, counter_index, enable, 190 is_egress); 191 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 192 } 193 194 int mlxsw_sp_rif_counter_value_get(struct mlxsw_sp *mlxsw_sp, 195 struct mlxsw_sp_rif *rif, 196 enum mlxsw_sp_rif_counter_dir dir, u64 *cnt) 197 { 198 char ricnt_pl[MLXSW_REG_RICNT_LEN]; 199 unsigned int *p_counter_index; 200 bool valid; 201 int err; 202 203 valid = mlxsw_sp_rif_counter_valid_get(rif, dir); 204 if (!valid) 205 return -EINVAL; 206 207 p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir); 208 if (!p_counter_index) 209 return -EINVAL; 210 mlxsw_reg_ricnt_pack(ricnt_pl, *p_counter_index, 211 MLXSW_REG_RICNT_OPCODE_NOP); 212 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ricnt), ricnt_pl); 213 if (err) 214 return err; 215 *cnt = mlxsw_reg_ricnt_good_unicast_packets_get(ricnt_pl); 216 return 0; 217 } 218 219 static int mlxsw_sp_rif_counter_clear(struct mlxsw_sp *mlxsw_sp, 220 unsigned int counter_index) 221 { 222 char ricnt_pl[MLXSW_REG_RICNT_LEN]; 223 224 mlxsw_reg_ricnt_pack(ricnt_pl, counter_index, 225 MLXSW_REG_RICNT_OPCODE_CLEAR); 226 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ricnt), ricnt_pl); 227 } 228 229 int mlxsw_sp_rif_counter_alloc(struct mlxsw_sp *mlxsw_sp, 230 struct mlxsw_sp_rif *rif, 231 enum mlxsw_sp_rif_counter_dir dir) 232 { 233 unsigned int *p_counter_index; 234 int err; 235 236 p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir); 237 if (!p_counter_index) 238 return -EINVAL; 239 err = mlxsw_sp_counter_alloc(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF, 240 p_counter_index); 241 if (err) 242 return err; 243 244 err = mlxsw_sp_rif_counter_clear(mlxsw_sp, *p_counter_index); 245 if (err) 246 goto err_counter_clear; 247 248 err = mlxsw_sp_rif_counter_edit(mlxsw_sp, rif->rif_index, 249 *p_counter_index, true, dir); 250 if (err) 251 goto err_counter_edit; 252 mlxsw_sp_rif_counter_valid_set(rif, dir, true); 253 return 0; 254 255 err_counter_edit: 256 err_counter_clear: 257 mlxsw_sp_counter_free(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF, 258 *p_counter_index); 259 return err; 260 } 261 262 void mlxsw_sp_rif_counter_free(struct mlxsw_sp *mlxsw_sp, 263 struct mlxsw_sp_rif *rif, 264 enum mlxsw_sp_rif_counter_dir dir) 265 { 266 unsigned int *p_counter_index; 267 268 if (!mlxsw_sp_rif_counter_valid_get(rif, dir)) 269 return; 270 271 p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir); 272 if (WARN_ON(!p_counter_index)) 273 return; 274 mlxsw_sp_rif_counter_edit(mlxsw_sp, rif->rif_index, 275 *p_counter_index, false, dir); 276 mlxsw_sp_counter_free(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF, 277 *p_counter_index); 278 mlxsw_sp_rif_counter_valid_set(rif, dir, false); 279 } 280 281 static void mlxsw_sp_rif_counters_alloc(struct mlxsw_sp_rif *rif) 282 { 283 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 284 struct devlink *devlink; 285 286 devlink = priv_to_devlink(mlxsw_sp->core); 287 if (!devlink_dpipe_table_counter_enabled(devlink, 288 MLXSW_SP_DPIPE_TABLE_NAME_ERIF)) 289 return; 290 mlxsw_sp_rif_counter_alloc(mlxsw_sp, rif, MLXSW_SP_RIF_COUNTER_EGRESS); 291 } 292 293 static void mlxsw_sp_rif_counters_free(struct mlxsw_sp_rif *rif) 294 { 295 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 296 297 mlxsw_sp_rif_counter_free(mlxsw_sp, rif, MLXSW_SP_RIF_COUNTER_EGRESS); 298 } 299 300 #define MLXSW_SP_PREFIX_COUNT (sizeof(struct in6_addr) * BITS_PER_BYTE + 1) 301 302 struct mlxsw_sp_prefix_usage { 303 DECLARE_BITMAP(b, MLXSW_SP_PREFIX_COUNT); 304 }; 305 306 #define mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) \ 307 for_each_set_bit(prefix, (prefix_usage)->b, MLXSW_SP_PREFIX_COUNT) 308 309 static bool 310 mlxsw_sp_prefix_usage_eq(struct mlxsw_sp_prefix_usage *prefix_usage1, 311 struct mlxsw_sp_prefix_usage *prefix_usage2) 312 { 313 return !memcmp(prefix_usage1, prefix_usage2, sizeof(*prefix_usage1)); 314 } 315 316 static void 317 mlxsw_sp_prefix_usage_cpy(struct mlxsw_sp_prefix_usage *prefix_usage1, 318 struct mlxsw_sp_prefix_usage *prefix_usage2) 319 { 320 memcpy(prefix_usage1, prefix_usage2, sizeof(*prefix_usage1)); 321 } 322 323 static void 324 mlxsw_sp_prefix_usage_set(struct mlxsw_sp_prefix_usage *prefix_usage, 325 unsigned char prefix_len) 326 { 327 set_bit(prefix_len, prefix_usage->b); 328 } 329 330 static void 331 mlxsw_sp_prefix_usage_clear(struct mlxsw_sp_prefix_usage *prefix_usage, 332 unsigned char prefix_len) 333 { 334 clear_bit(prefix_len, prefix_usage->b); 335 } 336 337 struct mlxsw_sp_fib_key { 338 unsigned char addr[sizeof(struct in6_addr)]; 339 unsigned char prefix_len; 340 }; 341 342 enum mlxsw_sp_fib_entry_type { 343 MLXSW_SP_FIB_ENTRY_TYPE_REMOTE, 344 MLXSW_SP_FIB_ENTRY_TYPE_LOCAL, 345 MLXSW_SP_FIB_ENTRY_TYPE_TRAP, 346 MLXSW_SP_FIB_ENTRY_TYPE_BLACKHOLE, 347 MLXSW_SP_FIB_ENTRY_TYPE_UNREACHABLE, 348 349 /* This is a special case of local delivery, where a packet should be 350 * decapsulated on reception. Note that there is no corresponding ENCAP, 351 * because that's a type of next hop, not of FIB entry. (There can be 352 * several next hops in a REMOTE entry, and some of them may be 353 * encapsulating entries.) 354 */ 355 MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP, 356 MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP, 357 }; 358 359 struct mlxsw_sp_nexthop_group_info; 360 struct mlxsw_sp_nexthop_group; 361 struct mlxsw_sp_fib_entry; 362 363 struct mlxsw_sp_fib_node { 364 struct mlxsw_sp_fib_entry *fib_entry; 365 struct list_head list; 366 struct rhash_head ht_node; 367 struct mlxsw_sp_fib *fib; 368 struct mlxsw_sp_fib_key key; 369 }; 370 371 struct mlxsw_sp_fib_entry_decap { 372 struct mlxsw_sp_ipip_entry *ipip_entry; 373 u32 tunnel_index; 374 }; 375 376 static struct mlxsw_sp_fib_entry_priv * 377 mlxsw_sp_fib_entry_priv_create(const struct mlxsw_sp_router_ll_ops *ll_ops) 378 { 379 struct mlxsw_sp_fib_entry_priv *priv; 380 381 if (!ll_ops->fib_entry_priv_size) 382 /* No need to have priv */ 383 return NULL; 384 385 priv = kzalloc(sizeof(*priv) + ll_ops->fib_entry_priv_size, GFP_KERNEL); 386 if (!priv) 387 return ERR_PTR(-ENOMEM); 388 refcount_set(&priv->refcnt, 1); 389 return priv; 390 } 391 392 static void 393 mlxsw_sp_fib_entry_priv_destroy(struct mlxsw_sp_fib_entry_priv *priv) 394 { 395 kfree(priv); 396 } 397 398 static void mlxsw_sp_fib_entry_priv_hold(struct mlxsw_sp_fib_entry_priv *priv) 399 { 400 refcount_inc(&priv->refcnt); 401 } 402 403 static void mlxsw_sp_fib_entry_priv_put(struct mlxsw_sp_fib_entry_priv *priv) 404 { 405 if (!priv || !refcount_dec_and_test(&priv->refcnt)) 406 return; 407 mlxsw_sp_fib_entry_priv_destroy(priv); 408 } 409 410 static void mlxsw_sp_fib_entry_op_ctx_priv_hold(struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 411 struct mlxsw_sp_fib_entry_priv *priv) 412 { 413 if (!priv) 414 return; 415 mlxsw_sp_fib_entry_priv_hold(priv); 416 list_add(&priv->list, &op_ctx->fib_entry_priv_list); 417 } 418 419 static void mlxsw_sp_fib_entry_op_ctx_priv_put_all(struct mlxsw_sp_fib_entry_op_ctx *op_ctx) 420 { 421 struct mlxsw_sp_fib_entry_priv *priv, *tmp; 422 423 list_for_each_entry_safe(priv, tmp, &op_ctx->fib_entry_priv_list, list) 424 mlxsw_sp_fib_entry_priv_put(priv); 425 INIT_LIST_HEAD(&op_ctx->fib_entry_priv_list); 426 } 427 428 struct mlxsw_sp_fib_entry { 429 struct mlxsw_sp_fib_node *fib_node; 430 enum mlxsw_sp_fib_entry_type type; 431 struct list_head nexthop_group_node; 432 struct mlxsw_sp_nexthop_group *nh_group; 433 struct mlxsw_sp_fib_entry_decap decap; /* Valid for decap entries. */ 434 struct mlxsw_sp_fib_entry_priv *priv; 435 }; 436 437 struct mlxsw_sp_fib4_entry { 438 struct mlxsw_sp_fib_entry common; 439 struct fib_info *fi; 440 u32 tb_id; 441 u8 tos; 442 u8 type; 443 }; 444 445 struct mlxsw_sp_fib6_entry { 446 struct mlxsw_sp_fib_entry common; 447 struct list_head rt6_list; 448 unsigned int nrt6; 449 }; 450 451 struct mlxsw_sp_rt6 { 452 struct list_head list; 453 struct fib6_info *rt; 454 }; 455 456 struct mlxsw_sp_lpm_tree { 457 u8 id; /* tree ID */ 458 unsigned int ref_count; 459 enum mlxsw_sp_l3proto proto; 460 unsigned long prefix_ref_count[MLXSW_SP_PREFIX_COUNT]; 461 struct mlxsw_sp_prefix_usage prefix_usage; 462 }; 463 464 struct mlxsw_sp_fib { 465 struct rhashtable ht; 466 struct list_head node_list; 467 struct mlxsw_sp_vr *vr; 468 struct mlxsw_sp_lpm_tree *lpm_tree; 469 enum mlxsw_sp_l3proto proto; 470 const struct mlxsw_sp_router_ll_ops *ll_ops; 471 }; 472 473 struct mlxsw_sp_vr { 474 u16 id; /* virtual router ID */ 475 u32 tb_id; /* kernel fib table id */ 476 unsigned int rif_count; 477 struct mlxsw_sp_fib *fib4; 478 struct mlxsw_sp_fib *fib6; 479 struct mlxsw_sp_mr_table *mr_table[MLXSW_SP_L3_PROTO_MAX]; 480 struct mlxsw_sp_rif *ul_rif; 481 refcount_t ul_rif_refcnt; 482 }; 483 484 static int mlxsw_sp_router_ll_basic_init(struct mlxsw_sp *mlxsw_sp, u16 vr_id, 485 enum mlxsw_sp_l3proto proto) 486 { 487 return 0; 488 } 489 490 static int mlxsw_sp_router_ll_basic_ralta_write(struct mlxsw_sp *mlxsw_sp, char *xralta_pl) 491 { 492 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), 493 xralta_pl + MLXSW_REG_XRALTA_RALTA_OFFSET); 494 } 495 496 static int mlxsw_sp_router_ll_basic_ralst_write(struct mlxsw_sp *mlxsw_sp, char *xralst_pl) 497 { 498 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralst), 499 xralst_pl + MLXSW_REG_XRALST_RALST_OFFSET); 500 } 501 502 static int mlxsw_sp_router_ll_basic_raltb_write(struct mlxsw_sp *mlxsw_sp, char *xraltb_pl) 503 { 504 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb), 505 xraltb_pl + MLXSW_REG_XRALTB_RALTB_OFFSET); 506 } 507 508 static const struct rhashtable_params mlxsw_sp_fib_ht_params; 509 510 static struct mlxsw_sp_fib *mlxsw_sp_fib_create(struct mlxsw_sp *mlxsw_sp, 511 struct mlxsw_sp_vr *vr, 512 enum mlxsw_sp_l3proto proto) 513 { 514 const struct mlxsw_sp_router_ll_ops *ll_ops = mlxsw_sp->router->proto_ll_ops[proto]; 515 struct mlxsw_sp_lpm_tree *lpm_tree; 516 struct mlxsw_sp_fib *fib; 517 int err; 518 519 err = ll_ops->init(mlxsw_sp, vr->id, proto); 520 if (err) 521 return ERR_PTR(err); 522 523 lpm_tree = mlxsw_sp->router->lpm.proto_trees[proto]; 524 fib = kzalloc(sizeof(*fib), GFP_KERNEL); 525 if (!fib) 526 return ERR_PTR(-ENOMEM); 527 err = rhashtable_init(&fib->ht, &mlxsw_sp_fib_ht_params); 528 if (err) 529 goto err_rhashtable_init; 530 INIT_LIST_HEAD(&fib->node_list); 531 fib->proto = proto; 532 fib->vr = vr; 533 fib->lpm_tree = lpm_tree; 534 fib->ll_ops = ll_ops; 535 mlxsw_sp_lpm_tree_hold(lpm_tree); 536 err = mlxsw_sp_vr_lpm_tree_bind(mlxsw_sp, fib, lpm_tree->id); 537 if (err) 538 goto err_lpm_tree_bind; 539 return fib; 540 541 err_lpm_tree_bind: 542 mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree); 543 err_rhashtable_init: 544 kfree(fib); 545 return ERR_PTR(err); 546 } 547 548 static void mlxsw_sp_fib_destroy(struct mlxsw_sp *mlxsw_sp, 549 struct mlxsw_sp_fib *fib) 550 { 551 mlxsw_sp_vr_lpm_tree_unbind(mlxsw_sp, fib); 552 mlxsw_sp_lpm_tree_put(mlxsw_sp, fib->lpm_tree); 553 WARN_ON(!list_empty(&fib->node_list)); 554 rhashtable_destroy(&fib->ht); 555 kfree(fib); 556 } 557 558 static struct mlxsw_sp_lpm_tree * 559 mlxsw_sp_lpm_tree_find_unused(struct mlxsw_sp *mlxsw_sp) 560 { 561 static struct mlxsw_sp_lpm_tree *lpm_tree; 562 int i; 563 564 for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) { 565 lpm_tree = &mlxsw_sp->router->lpm.trees[i]; 566 if (lpm_tree->ref_count == 0) 567 return lpm_tree; 568 } 569 return NULL; 570 } 571 572 static int mlxsw_sp_lpm_tree_alloc(struct mlxsw_sp *mlxsw_sp, 573 const struct mlxsw_sp_router_ll_ops *ll_ops, 574 struct mlxsw_sp_lpm_tree *lpm_tree) 575 { 576 char xralta_pl[MLXSW_REG_XRALTA_LEN]; 577 578 mlxsw_reg_xralta_pack(xralta_pl, true, 579 (enum mlxsw_reg_ralxx_protocol) lpm_tree->proto, 580 lpm_tree->id); 581 return ll_ops->ralta_write(mlxsw_sp, xralta_pl); 582 } 583 584 static void mlxsw_sp_lpm_tree_free(struct mlxsw_sp *mlxsw_sp, 585 const struct mlxsw_sp_router_ll_ops *ll_ops, 586 struct mlxsw_sp_lpm_tree *lpm_tree) 587 { 588 char xralta_pl[MLXSW_REG_XRALTA_LEN]; 589 590 mlxsw_reg_xralta_pack(xralta_pl, false, 591 (enum mlxsw_reg_ralxx_protocol) lpm_tree->proto, 592 lpm_tree->id); 593 ll_ops->ralta_write(mlxsw_sp, xralta_pl); 594 } 595 596 static int 597 mlxsw_sp_lpm_tree_left_struct_set(struct mlxsw_sp *mlxsw_sp, 598 const struct mlxsw_sp_router_ll_ops *ll_ops, 599 struct mlxsw_sp_prefix_usage *prefix_usage, 600 struct mlxsw_sp_lpm_tree *lpm_tree) 601 { 602 char xralst_pl[MLXSW_REG_XRALST_LEN]; 603 u8 root_bin = 0; 604 u8 prefix; 605 u8 last_prefix = MLXSW_REG_RALST_BIN_NO_CHILD; 606 607 mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) 608 root_bin = prefix; 609 610 mlxsw_reg_xralst_pack(xralst_pl, root_bin, lpm_tree->id); 611 mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) { 612 if (prefix == 0) 613 continue; 614 mlxsw_reg_xralst_bin_pack(xralst_pl, prefix, last_prefix, 615 MLXSW_REG_RALST_BIN_NO_CHILD); 616 last_prefix = prefix; 617 } 618 return ll_ops->ralst_write(mlxsw_sp, xralst_pl); 619 } 620 621 static struct mlxsw_sp_lpm_tree * 622 mlxsw_sp_lpm_tree_create(struct mlxsw_sp *mlxsw_sp, 623 const struct mlxsw_sp_router_ll_ops *ll_ops, 624 struct mlxsw_sp_prefix_usage *prefix_usage, 625 enum mlxsw_sp_l3proto proto) 626 { 627 struct mlxsw_sp_lpm_tree *lpm_tree; 628 int err; 629 630 lpm_tree = mlxsw_sp_lpm_tree_find_unused(mlxsw_sp); 631 if (!lpm_tree) 632 return ERR_PTR(-EBUSY); 633 lpm_tree->proto = proto; 634 err = mlxsw_sp_lpm_tree_alloc(mlxsw_sp, ll_ops, lpm_tree); 635 if (err) 636 return ERR_PTR(err); 637 638 err = mlxsw_sp_lpm_tree_left_struct_set(mlxsw_sp, ll_ops, prefix_usage, lpm_tree); 639 if (err) 640 goto err_left_struct_set; 641 memcpy(&lpm_tree->prefix_usage, prefix_usage, 642 sizeof(lpm_tree->prefix_usage)); 643 memset(&lpm_tree->prefix_ref_count, 0, 644 sizeof(lpm_tree->prefix_ref_count)); 645 lpm_tree->ref_count = 1; 646 return lpm_tree; 647 648 err_left_struct_set: 649 mlxsw_sp_lpm_tree_free(mlxsw_sp, ll_ops, lpm_tree); 650 return ERR_PTR(err); 651 } 652 653 static void mlxsw_sp_lpm_tree_destroy(struct mlxsw_sp *mlxsw_sp, 654 const struct mlxsw_sp_router_ll_ops *ll_ops, 655 struct mlxsw_sp_lpm_tree *lpm_tree) 656 { 657 mlxsw_sp_lpm_tree_free(mlxsw_sp, ll_ops, lpm_tree); 658 } 659 660 static struct mlxsw_sp_lpm_tree * 661 mlxsw_sp_lpm_tree_get(struct mlxsw_sp *mlxsw_sp, 662 struct mlxsw_sp_prefix_usage *prefix_usage, 663 enum mlxsw_sp_l3proto proto) 664 { 665 const struct mlxsw_sp_router_ll_ops *ll_ops = mlxsw_sp->router->proto_ll_ops[proto]; 666 struct mlxsw_sp_lpm_tree *lpm_tree; 667 int i; 668 669 for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) { 670 lpm_tree = &mlxsw_sp->router->lpm.trees[i]; 671 if (lpm_tree->ref_count != 0 && 672 lpm_tree->proto == proto && 673 mlxsw_sp_prefix_usage_eq(&lpm_tree->prefix_usage, 674 prefix_usage)) { 675 mlxsw_sp_lpm_tree_hold(lpm_tree); 676 return lpm_tree; 677 } 678 } 679 return mlxsw_sp_lpm_tree_create(mlxsw_sp, ll_ops, prefix_usage, proto); 680 } 681 682 static void mlxsw_sp_lpm_tree_hold(struct mlxsw_sp_lpm_tree *lpm_tree) 683 { 684 lpm_tree->ref_count++; 685 } 686 687 static void mlxsw_sp_lpm_tree_put(struct mlxsw_sp *mlxsw_sp, 688 struct mlxsw_sp_lpm_tree *lpm_tree) 689 { 690 const struct mlxsw_sp_router_ll_ops *ll_ops = 691 mlxsw_sp->router->proto_ll_ops[lpm_tree->proto]; 692 693 if (--lpm_tree->ref_count == 0) 694 mlxsw_sp_lpm_tree_destroy(mlxsw_sp, ll_ops, lpm_tree); 695 } 696 697 #define MLXSW_SP_LPM_TREE_MIN 1 /* tree 0 is reserved */ 698 699 static int mlxsw_sp_lpm_init(struct mlxsw_sp *mlxsw_sp) 700 { 701 struct mlxsw_sp_prefix_usage req_prefix_usage = {{ 0 } }; 702 struct mlxsw_sp_lpm_tree *lpm_tree; 703 u64 max_trees; 704 int err, i; 705 706 if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_LPM_TREES)) 707 return -EIO; 708 709 max_trees = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_LPM_TREES); 710 mlxsw_sp->router->lpm.tree_count = max_trees - MLXSW_SP_LPM_TREE_MIN; 711 mlxsw_sp->router->lpm.trees = kcalloc(mlxsw_sp->router->lpm.tree_count, 712 sizeof(struct mlxsw_sp_lpm_tree), 713 GFP_KERNEL); 714 if (!mlxsw_sp->router->lpm.trees) 715 return -ENOMEM; 716 717 for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) { 718 lpm_tree = &mlxsw_sp->router->lpm.trees[i]; 719 lpm_tree->id = i + MLXSW_SP_LPM_TREE_MIN; 720 } 721 722 lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage, 723 MLXSW_SP_L3_PROTO_IPV4); 724 if (IS_ERR(lpm_tree)) { 725 err = PTR_ERR(lpm_tree); 726 goto err_ipv4_tree_get; 727 } 728 mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV4] = lpm_tree; 729 730 lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage, 731 MLXSW_SP_L3_PROTO_IPV6); 732 if (IS_ERR(lpm_tree)) { 733 err = PTR_ERR(lpm_tree); 734 goto err_ipv6_tree_get; 735 } 736 mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV6] = lpm_tree; 737 738 return 0; 739 740 err_ipv6_tree_get: 741 lpm_tree = mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV4]; 742 mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree); 743 err_ipv4_tree_get: 744 kfree(mlxsw_sp->router->lpm.trees); 745 return err; 746 } 747 748 static void mlxsw_sp_lpm_fini(struct mlxsw_sp *mlxsw_sp) 749 { 750 struct mlxsw_sp_lpm_tree *lpm_tree; 751 752 lpm_tree = mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV6]; 753 mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree); 754 755 lpm_tree = mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV4]; 756 mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree); 757 758 kfree(mlxsw_sp->router->lpm.trees); 759 } 760 761 static bool mlxsw_sp_vr_is_used(const struct mlxsw_sp_vr *vr) 762 { 763 return !!vr->fib4 || !!vr->fib6 || 764 !!vr->mr_table[MLXSW_SP_L3_PROTO_IPV4] || 765 !!vr->mr_table[MLXSW_SP_L3_PROTO_IPV6]; 766 } 767 768 static struct mlxsw_sp_vr *mlxsw_sp_vr_find_unused(struct mlxsw_sp *mlxsw_sp) 769 { 770 struct mlxsw_sp_vr *vr; 771 int i; 772 773 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) { 774 vr = &mlxsw_sp->router->vrs[i]; 775 if (!mlxsw_sp_vr_is_used(vr)) 776 return vr; 777 } 778 return NULL; 779 } 780 781 static int mlxsw_sp_vr_lpm_tree_bind(struct mlxsw_sp *mlxsw_sp, 782 const struct mlxsw_sp_fib *fib, u8 tree_id) 783 { 784 char xraltb_pl[MLXSW_REG_XRALTB_LEN]; 785 786 mlxsw_reg_xraltb_pack(xraltb_pl, fib->vr->id, 787 (enum mlxsw_reg_ralxx_protocol) fib->proto, 788 tree_id); 789 return fib->ll_ops->raltb_write(mlxsw_sp, xraltb_pl); 790 } 791 792 static int mlxsw_sp_vr_lpm_tree_unbind(struct mlxsw_sp *mlxsw_sp, 793 const struct mlxsw_sp_fib *fib) 794 { 795 char xraltb_pl[MLXSW_REG_XRALTB_LEN]; 796 797 /* Bind to tree 0 which is default */ 798 mlxsw_reg_xraltb_pack(xraltb_pl, fib->vr->id, 799 (enum mlxsw_reg_ralxx_protocol) fib->proto, 0); 800 return fib->ll_ops->raltb_write(mlxsw_sp, xraltb_pl); 801 } 802 803 static u32 mlxsw_sp_fix_tb_id(u32 tb_id) 804 { 805 /* For our purpose, squash main, default and local tables into one */ 806 if (tb_id == RT_TABLE_LOCAL || tb_id == RT_TABLE_DEFAULT) 807 tb_id = RT_TABLE_MAIN; 808 return tb_id; 809 } 810 811 static struct mlxsw_sp_vr *mlxsw_sp_vr_find(struct mlxsw_sp *mlxsw_sp, 812 u32 tb_id) 813 { 814 struct mlxsw_sp_vr *vr; 815 int i; 816 817 tb_id = mlxsw_sp_fix_tb_id(tb_id); 818 819 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) { 820 vr = &mlxsw_sp->router->vrs[i]; 821 if (mlxsw_sp_vr_is_used(vr) && vr->tb_id == tb_id) 822 return vr; 823 } 824 return NULL; 825 } 826 827 int mlxsw_sp_router_tb_id_vr_id(struct mlxsw_sp *mlxsw_sp, u32 tb_id, 828 u16 *vr_id) 829 { 830 struct mlxsw_sp_vr *vr; 831 int err = 0; 832 833 mutex_lock(&mlxsw_sp->router->lock); 834 vr = mlxsw_sp_vr_find(mlxsw_sp, tb_id); 835 if (!vr) { 836 err = -ESRCH; 837 goto out; 838 } 839 *vr_id = vr->id; 840 out: 841 mutex_unlock(&mlxsw_sp->router->lock); 842 return err; 843 } 844 845 static struct mlxsw_sp_fib *mlxsw_sp_vr_fib(const struct mlxsw_sp_vr *vr, 846 enum mlxsw_sp_l3proto proto) 847 { 848 switch (proto) { 849 case MLXSW_SP_L3_PROTO_IPV4: 850 return vr->fib4; 851 case MLXSW_SP_L3_PROTO_IPV6: 852 return vr->fib6; 853 } 854 return NULL; 855 } 856 857 static struct mlxsw_sp_vr *mlxsw_sp_vr_create(struct mlxsw_sp *mlxsw_sp, 858 u32 tb_id, 859 struct netlink_ext_ack *extack) 860 { 861 struct mlxsw_sp_mr_table *mr4_table, *mr6_table; 862 struct mlxsw_sp_fib *fib4; 863 struct mlxsw_sp_fib *fib6; 864 struct mlxsw_sp_vr *vr; 865 int err; 866 867 vr = mlxsw_sp_vr_find_unused(mlxsw_sp); 868 if (!vr) { 869 NL_SET_ERR_MSG_MOD(extack, "Exceeded number of supported virtual routers"); 870 return ERR_PTR(-EBUSY); 871 } 872 fib4 = mlxsw_sp_fib_create(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV4); 873 if (IS_ERR(fib4)) 874 return ERR_CAST(fib4); 875 fib6 = mlxsw_sp_fib_create(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV6); 876 if (IS_ERR(fib6)) { 877 err = PTR_ERR(fib6); 878 goto err_fib6_create; 879 } 880 mr4_table = mlxsw_sp_mr_table_create(mlxsw_sp, vr->id, 881 MLXSW_SP_L3_PROTO_IPV4); 882 if (IS_ERR(mr4_table)) { 883 err = PTR_ERR(mr4_table); 884 goto err_mr4_table_create; 885 } 886 mr6_table = mlxsw_sp_mr_table_create(mlxsw_sp, vr->id, 887 MLXSW_SP_L3_PROTO_IPV6); 888 if (IS_ERR(mr6_table)) { 889 err = PTR_ERR(mr6_table); 890 goto err_mr6_table_create; 891 } 892 893 vr->fib4 = fib4; 894 vr->fib6 = fib6; 895 vr->mr_table[MLXSW_SP_L3_PROTO_IPV4] = mr4_table; 896 vr->mr_table[MLXSW_SP_L3_PROTO_IPV6] = mr6_table; 897 vr->tb_id = tb_id; 898 return vr; 899 900 err_mr6_table_create: 901 mlxsw_sp_mr_table_destroy(mr4_table); 902 err_mr4_table_create: 903 mlxsw_sp_fib_destroy(mlxsw_sp, fib6); 904 err_fib6_create: 905 mlxsw_sp_fib_destroy(mlxsw_sp, fib4); 906 return ERR_PTR(err); 907 } 908 909 static void mlxsw_sp_vr_destroy(struct mlxsw_sp *mlxsw_sp, 910 struct mlxsw_sp_vr *vr) 911 { 912 mlxsw_sp_mr_table_destroy(vr->mr_table[MLXSW_SP_L3_PROTO_IPV6]); 913 vr->mr_table[MLXSW_SP_L3_PROTO_IPV6] = NULL; 914 mlxsw_sp_mr_table_destroy(vr->mr_table[MLXSW_SP_L3_PROTO_IPV4]); 915 vr->mr_table[MLXSW_SP_L3_PROTO_IPV4] = NULL; 916 mlxsw_sp_fib_destroy(mlxsw_sp, vr->fib6); 917 vr->fib6 = NULL; 918 mlxsw_sp_fib_destroy(mlxsw_sp, vr->fib4); 919 vr->fib4 = NULL; 920 } 921 922 static struct mlxsw_sp_vr *mlxsw_sp_vr_get(struct mlxsw_sp *mlxsw_sp, u32 tb_id, 923 struct netlink_ext_ack *extack) 924 { 925 struct mlxsw_sp_vr *vr; 926 927 tb_id = mlxsw_sp_fix_tb_id(tb_id); 928 vr = mlxsw_sp_vr_find(mlxsw_sp, tb_id); 929 if (!vr) 930 vr = mlxsw_sp_vr_create(mlxsw_sp, tb_id, extack); 931 return vr; 932 } 933 934 static void mlxsw_sp_vr_put(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp_vr *vr) 935 { 936 if (!vr->rif_count && list_empty(&vr->fib4->node_list) && 937 list_empty(&vr->fib6->node_list) && 938 mlxsw_sp_mr_table_empty(vr->mr_table[MLXSW_SP_L3_PROTO_IPV4]) && 939 mlxsw_sp_mr_table_empty(vr->mr_table[MLXSW_SP_L3_PROTO_IPV6])) 940 mlxsw_sp_vr_destroy(mlxsw_sp, vr); 941 } 942 943 static bool 944 mlxsw_sp_vr_lpm_tree_should_replace(struct mlxsw_sp_vr *vr, 945 enum mlxsw_sp_l3proto proto, u8 tree_id) 946 { 947 struct mlxsw_sp_fib *fib = mlxsw_sp_vr_fib(vr, proto); 948 949 if (!mlxsw_sp_vr_is_used(vr)) 950 return false; 951 if (fib->lpm_tree->id == tree_id) 952 return true; 953 return false; 954 } 955 956 static int mlxsw_sp_vr_lpm_tree_replace(struct mlxsw_sp *mlxsw_sp, 957 struct mlxsw_sp_fib *fib, 958 struct mlxsw_sp_lpm_tree *new_tree) 959 { 960 struct mlxsw_sp_lpm_tree *old_tree = fib->lpm_tree; 961 int err; 962 963 fib->lpm_tree = new_tree; 964 mlxsw_sp_lpm_tree_hold(new_tree); 965 err = mlxsw_sp_vr_lpm_tree_bind(mlxsw_sp, fib, new_tree->id); 966 if (err) 967 goto err_tree_bind; 968 mlxsw_sp_lpm_tree_put(mlxsw_sp, old_tree); 969 return 0; 970 971 err_tree_bind: 972 mlxsw_sp_lpm_tree_put(mlxsw_sp, new_tree); 973 fib->lpm_tree = old_tree; 974 return err; 975 } 976 977 static int mlxsw_sp_vrs_lpm_tree_replace(struct mlxsw_sp *mlxsw_sp, 978 struct mlxsw_sp_fib *fib, 979 struct mlxsw_sp_lpm_tree *new_tree) 980 { 981 enum mlxsw_sp_l3proto proto = fib->proto; 982 struct mlxsw_sp_lpm_tree *old_tree; 983 u8 old_id, new_id = new_tree->id; 984 struct mlxsw_sp_vr *vr; 985 int i, err; 986 987 old_tree = mlxsw_sp->router->lpm.proto_trees[proto]; 988 old_id = old_tree->id; 989 990 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) { 991 vr = &mlxsw_sp->router->vrs[i]; 992 if (!mlxsw_sp_vr_lpm_tree_should_replace(vr, proto, old_id)) 993 continue; 994 err = mlxsw_sp_vr_lpm_tree_replace(mlxsw_sp, 995 mlxsw_sp_vr_fib(vr, proto), 996 new_tree); 997 if (err) 998 goto err_tree_replace; 999 } 1000 1001 memcpy(new_tree->prefix_ref_count, old_tree->prefix_ref_count, 1002 sizeof(new_tree->prefix_ref_count)); 1003 mlxsw_sp->router->lpm.proto_trees[proto] = new_tree; 1004 mlxsw_sp_lpm_tree_put(mlxsw_sp, old_tree); 1005 1006 return 0; 1007 1008 err_tree_replace: 1009 for (i--; i >= 0; i--) { 1010 if (!mlxsw_sp_vr_lpm_tree_should_replace(vr, proto, new_id)) 1011 continue; 1012 mlxsw_sp_vr_lpm_tree_replace(mlxsw_sp, 1013 mlxsw_sp_vr_fib(vr, proto), 1014 old_tree); 1015 } 1016 return err; 1017 } 1018 1019 static int mlxsw_sp_vrs_init(struct mlxsw_sp *mlxsw_sp) 1020 { 1021 struct mlxsw_sp_vr *vr; 1022 u64 max_vrs; 1023 int i; 1024 1025 if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_VRS)) 1026 return -EIO; 1027 1028 max_vrs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); 1029 mlxsw_sp->router->vrs = kcalloc(max_vrs, sizeof(struct mlxsw_sp_vr), 1030 GFP_KERNEL); 1031 if (!mlxsw_sp->router->vrs) 1032 return -ENOMEM; 1033 1034 for (i = 0; i < max_vrs; i++) { 1035 vr = &mlxsw_sp->router->vrs[i]; 1036 vr->id = i; 1037 } 1038 1039 return 0; 1040 } 1041 1042 static void mlxsw_sp_router_fib_flush(struct mlxsw_sp *mlxsw_sp); 1043 1044 static void mlxsw_sp_vrs_fini(struct mlxsw_sp *mlxsw_sp) 1045 { 1046 /* At this stage we're guaranteed not to have new incoming 1047 * FIB notifications and the work queue is free from FIBs 1048 * sitting on top of mlxsw netdevs. However, we can still 1049 * have other FIBs queued. Flush the queue before flushing 1050 * the device's tables. No need for locks, as we're the only 1051 * writer. 1052 */ 1053 mlxsw_core_flush_owq(); 1054 mlxsw_sp_router_fib_flush(mlxsw_sp); 1055 kfree(mlxsw_sp->router->vrs); 1056 } 1057 1058 static struct net_device * 1059 __mlxsw_sp_ipip_netdev_ul_dev_get(const struct net_device *ol_dev) 1060 { 1061 struct ip_tunnel *tun = netdev_priv(ol_dev); 1062 struct net *net = dev_net(ol_dev); 1063 1064 return dev_get_by_index_rcu(net, tun->parms.link); 1065 } 1066 1067 u32 mlxsw_sp_ipip_dev_ul_tb_id(const struct net_device *ol_dev) 1068 { 1069 struct net_device *d; 1070 u32 tb_id; 1071 1072 rcu_read_lock(); 1073 d = __mlxsw_sp_ipip_netdev_ul_dev_get(ol_dev); 1074 if (d) 1075 tb_id = l3mdev_fib_table(d) ? : RT_TABLE_MAIN; 1076 else 1077 tb_id = RT_TABLE_MAIN; 1078 rcu_read_unlock(); 1079 1080 return tb_id; 1081 } 1082 1083 static struct mlxsw_sp_rif * 1084 mlxsw_sp_rif_create(struct mlxsw_sp *mlxsw_sp, 1085 const struct mlxsw_sp_rif_params *params, 1086 struct netlink_ext_ack *extack); 1087 1088 static struct mlxsw_sp_rif_ipip_lb * 1089 mlxsw_sp_ipip_ol_ipip_lb_create(struct mlxsw_sp *mlxsw_sp, 1090 enum mlxsw_sp_ipip_type ipipt, 1091 struct net_device *ol_dev, 1092 struct netlink_ext_ack *extack) 1093 { 1094 struct mlxsw_sp_rif_params_ipip_lb lb_params; 1095 const struct mlxsw_sp_ipip_ops *ipip_ops; 1096 struct mlxsw_sp_rif *rif; 1097 1098 ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipipt]; 1099 lb_params = (struct mlxsw_sp_rif_params_ipip_lb) { 1100 .common.dev = ol_dev, 1101 .common.lag = false, 1102 .lb_config = ipip_ops->ol_loopback_config(mlxsw_sp, ol_dev), 1103 }; 1104 1105 rif = mlxsw_sp_rif_create(mlxsw_sp, &lb_params.common, extack); 1106 if (IS_ERR(rif)) 1107 return ERR_CAST(rif); 1108 return container_of(rif, struct mlxsw_sp_rif_ipip_lb, common); 1109 } 1110 1111 static struct mlxsw_sp_ipip_entry * 1112 mlxsw_sp_ipip_entry_alloc(struct mlxsw_sp *mlxsw_sp, 1113 enum mlxsw_sp_ipip_type ipipt, 1114 struct net_device *ol_dev) 1115 { 1116 const struct mlxsw_sp_ipip_ops *ipip_ops; 1117 struct mlxsw_sp_ipip_entry *ipip_entry; 1118 struct mlxsw_sp_ipip_entry *ret = NULL; 1119 1120 ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipipt]; 1121 ipip_entry = kzalloc(sizeof(*ipip_entry), GFP_KERNEL); 1122 if (!ipip_entry) 1123 return ERR_PTR(-ENOMEM); 1124 1125 ipip_entry->ol_lb = mlxsw_sp_ipip_ol_ipip_lb_create(mlxsw_sp, ipipt, 1126 ol_dev, NULL); 1127 if (IS_ERR(ipip_entry->ol_lb)) { 1128 ret = ERR_CAST(ipip_entry->ol_lb); 1129 goto err_ol_ipip_lb_create; 1130 } 1131 1132 ipip_entry->ipipt = ipipt; 1133 ipip_entry->ol_dev = ol_dev; 1134 1135 switch (ipip_ops->ul_proto) { 1136 case MLXSW_SP_L3_PROTO_IPV4: 1137 ipip_entry->parms4 = mlxsw_sp_ipip_netdev_parms4(ol_dev); 1138 break; 1139 case MLXSW_SP_L3_PROTO_IPV6: 1140 WARN_ON(1); 1141 break; 1142 } 1143 1144 return ipip_entry; 1145 1146 err_ol_ipip_lb_create: 1147 kfree(ipip_entry); 1148 return ret; 1149 } 1150 1151 static void 1152 mlxsw_sp_ipip_entry_dealloc(struct mlxsw_sp_ipip_entry *ipip_entry) 1153 { 1154 mlxsw_sp_rif_destroy(&ipip_entry->ol_lb->common); 1155 kfree(ipip_entry); 1156 } 1157 1158 static bool 1159 mlxsw_sp_ipip_entry_saddr_matches(struct mlxsw_sp *mlxsw_sp, 1160 const enum mlxsw_sp_l3proto ul_proto, 1161 union mlxsw_sp_l3addr saddr, 1162 u32 ul_tb_id, 1163 struct mlxsw_sp_ipip_entry *ipip_entry) 1164 { 1165 u32 tun_ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(ipip_entry->ol_dev); 1166 enum mlxsw_sp_ipip_type ipipt = ipip_entry->ipipt; 1167 union mlxsw_sp_l3addr tun_saddr; 1168 1169 if (mlxsw_sp->router->ipip_ops_arr[ipipt]->ul_proto != ul_proto) 1170 return false; 1171 1172 tun_saddr = mlxsw_sp_ipip_netdev_saddr(ul_proto, ipip_entry->ol_dev); 1173 return tun_ul_tb_id == ul_tb_id && 1174 mlxsw_sp_l3addr_eq(&tun_saddr, &saddr); 1175 } 1176 1177 static int 1178 mlxsw_sp_fib_entry_decap_init(struct mlxsw_sp *mlxsw_sp, 1179 struct mlxsw_sp_fib_entry *fib_entry, 1180 struct mlxsw_sp_ipip_entry *ipip_entry) 1181 { 1182 u32 tunnel_index; 1183 int err; 1184 1185 err = mlxsw_sp_kvdl_alloc(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 1186 1, &tunnel_index); 1187 if (err) 1188 return err; 1189 1190 ipip_entry->decap_fib_entry = fib_entry; 1191 fib_entry->decap.ipip_entry = ipip_entry; 1192 fib_entry->decap.tunnel_index = tunnel_index; 1193 return 0; 1194 } 1195 1196 static void mlxsw_sp_fib_entry_decap_fini(struct mlxsw_sp *mlxsw_sp, 1197 struct mlxsw_sp_fib_entry *fib_entry) 1198 { 1199 /* Unlink this node from the IPIP entry that it's the decap entry of. */ 1200 fib_entry->decap.ipip_entry->decap_fib_entry = NULL; 1201 fib_entry->decap.ipip_entry = NULL; 1202 mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 1203 1, fib_entry->decap.tunnel_index); 1204 } 1205 1206 static struct mlxsw_sp_fib_node * 1207 mlxsw_sp_fib_node_lookup(struct mlxsw_sp_fib *fib, const void *addr, 1208 size_t addr_len, unsigned char prefix_len); 1209 static int mlxsw_sp_fib_entry_update(struct mlxsw_sp *mlxsw_sp, 1210 struct mlxsw_sp_fib_entry *fib_entry); 1211 1212 static void 1213 mlxsw_sp_ipip_entry_demote_decap(struct mlxsw_sp *mlxsw_sp, 1214 struct mlxsw_sp_ipip_entry *ipip_entry) 1215 { 1216 struct mlxsw_sp_fib_entry *fib_entry = ipip_entry->decap_fib_entry; 1217 1218 mlxsw_sp_fib_entry_decap_fini(mlxsw_sp, fib_entry); 1219 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP; 1220 1221 mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry); 1222 } 1223 1224 static void 1225 mlxsw_sp_ipip_entry_promote_decap(struct mlxsw_sp *mlxsw_sp, 1226 struct mlxsw_sp_ipip_entry *ipip_entry, 1227 struct mlxsw_sp_fib_entry *decap_fib_entry) 1228 { 1229 if (mlxsw_sp_fib_entry_decap_init(mlxsw_sp, decap_fib_entry, 1230 ipip_entry)) 1231 return; 1232 decap_fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP; 1233 1234 if (mlxsw_sp_fib_entry_update(mlxsw_sp, decap_fib_entry)) 1235 mlxsw_sp_ipip_entry_demote_decap(mlxsw_sp, ipip_entry); 1236 } 1237 1238 static struct mlxsw_sp_fib_entry * 1239 mlxsw_sp_router_ip2me_fib_entry_find(struct mlxsw_sp *mlxsw_sp, u32 tb_id, 1240 enum mlxsw_sp_l3proto proto, 1241 const union mlxsw_sp_l3addr *addr, 1242 enum mlxsw_sp_fib_entry_type type) 1243 { 1244 struct mlxsw_sp_fib_node *fib_node; 1245 unsigned char addr_prefix_len; 1246 struct mlxsw_sp_fib *fib; 1247 struct mlxsw_sp_vr *vr; 1248 const void *addrp; 1249 size_t addr_len; 1250 u32 addr4; 1251 1252 vr = mlxsw_sp_vr_find(mlxsw_sp, tb_id); 1253 if (!vr) 1254 return NULL; 1255 fib = mlxsw_sp_vr_fib(vr, proto); 1256 1257 switch (proto) { 1258 case MLXSW_SP_L3_PROTO_IPV4: 1259 addr4 = be32_to_cpu(addr->addr4); 1260 addrp = &addr4; 1261 addr_len = 4; 1262 addr_prefix_len = 32; 1263 break; 1264 case MLXSW_SP_L3_PROTO_IPV6: 1265 default: 1266 WARN_ON(1); 1267 return NULL; 1268 } 1269 1270 fib_node = mlxsw_sp_fib_node_lookup(fib, addrp, addr_len, 1271 addr_prefix_len); 1272 if (!fib_node || fib_node->fib_entry->type != type) 1273 return NULL; 1274 1275 return fib_node->fib_entry; 1276 } 1277 1278 /* Given an IPIP entry, find the corresponding decap route. */ 1279 static struct mlxsw_sp_fib_entry * 1280 mlxsw_sp_ipip_entry_find_decap(struct mlxsw_sp *mlxsw_sp, 1281 struct mlxsw_sp_ipip_entry *ipip_entry) 1282 { 1283 static struct mlxsw_sp_fib_node *fib_node; 1284 const struct mlxsw_sp_ipip_ops *ipip_ops; 1285 unsigned char saddr_prefix_len; 1286 union mlxsw_sp_l3addr saddr; 1287 struct mlxsw_sp_fib *ul_fib; 1288 struct mlxsw_sp_vr *ul_vr; 1289 const void *saddrp; 1290 size_t saddr_len; 1291 u32 ul_tb_id; 1292 u32 saddr4; 1293 1294 ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt]; 1295 1296 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(ipip_entry->ol_dev); 1297 ul_vr = mlxsw_sp_vr_find(mlxsw_sp, ul_tb_id); 1298 if (!ul_vr) 1299 return NULL; 1300 1301 ul_fib = mlxsw_sp_vr_fib(ul_vr, ipip_ops->ul_proto); 1302 saddr = mlxsw_sp_ipip_netdev_saddr(ipip_ops->ul_proto, 1303 ipip_entry->ol_dev); 1304 1305 switch (ipip_ops->ul_proto) { 1306 case MLXSW_SP_L3_PROTO_IPV4: 1307 saddr4 = be32_to_cpu(saddr.addr4); 1308 saddrp = &saddr4; 1309 saddr_len = 4; 1310 saddr_prefix_len = 32; 1311 break; 1312 default: 1313 WARN_ON(1); 1314 return NULL; 1315 } 1316 1317 fib_node = mlxsw_sp_fib_node_lookup(ul_fib, saddrp, saddr_len, 1318 saddr_prefix_len); 1319 if (!fib_node || 1320 fib_node->fib_entry->type != MLXSW_SP_FIB_ENTRY_TYPE_TRAP) 1321 return NULL; 1322 1323 return fib_node->fib_entry; 1324 } 1325 1326 static struct mlxsw_sp_ipip_entry * 1327 mlxsw_sp_ipip_entry_create(struct mlxsw_sp *mlxsw_sp, 1328 enum mlxsw_sp_ipip_type ipipt, 1329 struct net_device *ol_dev) 1330 { 1331 struct mlxsw_sp_ipip_entry *ipip_entry; 1332 1333 ipip_entry = mlxsw_sp_ipip_entry_alloc(mlxsw_sp, ipipt, ol_dev); 1334 if (IS_ERR(ipip_entry)) 1335 return ipip_entry; 1336 1337 list_add_tail(&ipip_entry->ipip_list_node, 1338 &mlxsw_sp->router->ipip_list); 1339 1340 return ipip_entry; 1341 } 1342 1343 static void 1344 mlxsw_sp_ipip_entry_destroy(struct mlxsw_sp *mlxsw_sp, 1345 struct mlxsw_sp_ipip_entry *ipip_entry) 1346 { 1347 list_del(&ipip_entry->ipip_list_node); 1348 mlxsw_sp_ipip_entry_dealloc(ipip_entry); 1349 } 1350 1351 static bool 1352 mlxsw_sp_ipip_entry_matches_decap(struct mlxsw_sp *mlxsw_sp, 1353 const struct net_device *ul_dev, 1354 enum mlxsw_sp_l3proto ul_proto, 1355 union mlxsw_sp_l3addr ul_dip, 1356 struct mlxsw_sp_ipip_entry *ipip_entry) 1357 { 1358 u32 ul_tb_id = l3mdev_fib_table(ul_dev) ? : RT_TABLE_MAIN; 1359 enum mlxsw_sp_ipip_type ipipt = ipip_entry->ipipt; 1360 1361 if (mlxsw_sp->router->ipip_ops_arr[ipipt]->ul_proto != ul_proto) 1362 return false; 1363 1364 return mlxsw_sp_ipip_entry_saddr_matches(mlxsw_sp, ul_proto, ul_dip, 1365 ul_tb_id, ipip_entry); 1366 } 1367 1368 /* Given decap parameters, find the corresponding IPIP entry. */ 1369 static struct mlxsw_sp_ipip_entry * 1370 mlxsw_sp_ipip_entry_find_by_decap(struct mlxsw_sp *mlxsw_sp, int ul_dev_ifindex, 1371 enum mlxsw_sp_l3proto ul_proto, 1372 union mlxsw_sp_l3addr ul_dip) 1373 { 1374 struct mlxsw_sp_ipip_entry *ipip_entry = NULL; 1375 struct net_device *ul_dev; 1376 1377 rcu_read_lock(); 1378 1379 ul_dev = dev_get_by_index_rcu(mlxsw_sp_net(mlxsw_sp), ul_dev_ifindex); 1380 if (!ul_dev) 1381 goto out_unlock; 1382 1383 list_for_each_entry(ipip_entry, &mlxsw_sp->router->ipip_list, 1384 ipip_list_node) 1385 if (mlxsw_sp_ipip_entry_matches_decap(mlxsw_sp, ul_dev, 1386 ul_proto, ul_dip, 1387 ipip_entry)) 1388 goto out_unlock; 1389 1390 rcu_read_unlock(); 1391 1392 return NULL; 1393 1394 out_unlock: 1395 rcu_read_unlock(); 1396 return ipip_entry; 1397 } 1398 1399 static bool mlxsw_sp_netdev_ipip_type(const struct mlxsw_sp *mlxsw_sp, 1400 const struct net_device *dev, 1401 enum mlxsw_sp_ipip_type *p_type) 1402 { 1403 struct mlxsw_sp_router *router = mlxsw_sp->router; 1404 const struct mlxsw_sp_ipip_ops *ipip_ops; 1405 enum mlxsw_sp_ipip_type ipipt; 1406 1407 for (ipipt = 0; ipipt < MLXSW_SP_IPIP_TYPE_MAX; ++ipipt) { 1408 ipip_ops = router->ipip_ops_arr[ipipt]; 1409 if (dev->type == ipip_ops->dev_type) { 1410 if (p_type) 1411 *p_type = ipipt; 1412 return true; 1413 } 1414 } 1415 return false; 1416 } 1417 1418 bool mlxsw_sp_netdev_is_ipip_ol(const struct mlxsw_sp *mlxsw_sp, 1419 const struct net_device *dev) 1420 { 1421 return mlxsw_sp_netdev_ipip_type(mlxsw_sp, dev, NULL); 1422 } 1423 1424 static struct mlxsw_sp_ipip_entry * 1425 mlxsw_sp_ipip_entry_find_by_ol_dev(struct mlxsw_sp *mlxsw_sp, 1426 const struct net_device *ol_dev) 1427 { 1428 struct mlxsw_sp_ipip_entry *ipip_entry; 1429 1430 list_for_each_entry(ipip_entry, &mlxsw_sp->router->ipip_list, 1431 ipip_list_node) 1432 if (ipip_entry->ol_dev == ol_dev) 1433 return ipip_entry; 1434 1435 return NULL; 1436 } 1437 1438 static struct mlxsw_sp_ipip_entry * 1439 mlxsw_sp_ipip_entry_find_by_ul_dev(const struct mlxsw_sp *mlxsw_sp, 1440 const struct net_device *ul_dev, 1441 struct mlxsw_sp_ipip_entry *start) 1442 { 1443 struct mlxsw_sp_ipip_entry *ipip_entry; 1444 1445 ipip_entry = list_prepare_entry(start, &mlxsw_sp->router->ipip_list, 1446 ipip_list_node); 1447 list_for_each_entry_continue(ipip_entry, &mlxsw_sp->router->ipip_list, 1448 ipip_list_node) { 1449 struct net_device *ol_dev = ipip_entry->ol_dev; 1450 struct net_device *ipip_ul_dev; 1451 1452 rcu_read_lock(); 1453 ipip_ul_dev = __mlxsw_sp_ipip_netdev_ul_dev_get(ol_dev); 1454 rcu_read_unlock(); 1455 1456 if (ipip_ul_dev == ul_dev) 1457 return ipip_entry; 1458 } 1459 1460 return NULL; 1461 } 1462 1463 bool mlxsw_sp_netdev_is_ipip_ul(struct mlxsw_sp *mlxsw_sp, 1464 const struct net_device *dev) 1465 { 1466 bool is_ipip_ul; 1467 1468 mutex_lock(&mlxsw_sp->router->lock); 1469 is_ipip_ul = mlxsw_sp_ipip_entry_find_by_ul_dev(mlxsw_sp, dev, NULL); 1470 mutex_unlock(&mlxsw_sp->router->lock); 1471 1472 return is_ipip_ul; 1473 } 1474 1475 static bool mlxsw_sp_netdevice_ipip_can_offload(struct mlxsw_sp *mlxsw_sp, 1476 const struct net_device *ol_dev, 1477 enum mlxsw_sp_ipip_type ipipt) 1478 { 1479 const struct mlxsw_sp_ipip_ops *ops 1480 = mlxsw_sp->router->ipip_ops_arr[ipipt]; 1481 1482 return ops->can_offload(mlxsw_sp, ol_dev); 1483 } 1484 1485 static int mlxsw_sp_netdevice_ipip_ol_reg_event(struct mlxsw_sp *mlxsw_sp, 1486 struct net_device *ol_dev) 1487 { 1488 enum mlxsw_sp_ipip_type ipipt = MLXSW_SP_IPIP_TYPE_MAX; 1489 struct mlxsw_sp_ipip_entry *ipip_entry; 1490 enum mlxsw_sp_l3proto ul_proto; 1491 union mlxsw_sp_l3addr saddr; 1492 u32 ul_tb_id; 1493 1494 mlxsw_sp_netdev_ipip_type(mlxsw_sp, ol_dev, &ipipt); 1495 if (mlxsw_sp_netdevice_ipip_can_offload(mlxsw_sp, ol_dev, ipipt)) { 1496 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(ol_dev); 1497 ul_proto = mlxsw_sp->router->ipip_ops_arr[ipipt]->ul_proto; 1498 saddr = mlxsw_sp_ipip_netdev_saddr(ul_proto, ol_dev); 1499 if (!mlxsw_sp_ipip_demote_tunnel_by_saddr(mlxsw_sp, ul_proto, 1500 saddr, ul_tb_id, 1501 NULL)) { 1502 ipip_entry = mlxsw_sp_ipip_entry_create(mlxsw_sp, ipipt, 1503 ol_dev); 1504 if (IS_ERR(ipip_entry)) 1505 return PTR_ERR(ipip_entry); 1506 } 1507 } 1508 1509 return 0; 1510 } 1511 1512 static void mlxsw_sp_netdevice_ipip_ol_unreg_event(struct mlxsw_sp *mlxsw_sp, 1513 struct net_device *ol_dev) 1514 { 1515 struct mlxsw_sp_ipip_entry *ipip_entry; 1516 1517 ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev); 1518 if (ipip_entry) 1519 mlxsw_sp_ipip_entry_destroy(mlxsw_sp, ipip_entry); 1520 } 1521 1522 static void 1523 mlxsw_sp_ipip_entry_ol_up_event(struct mlxsw_sp *mlxsw_sp, 1524 struct mlxsw_sp_ipip_entry *ipip_entry) 1525 { 1526 struct mlxsw_sp_fib_entry *decap_fib_entry; 1527 1528 decap_fib_entry = mlxsw_sp_ipip_entry_find_decap(mlxsw_sp, ipip_entry); 1529 if (decap_fib_entry) 1530 mlxsw_sp_ipip_entry_promote_decap(mlxsw_sp, ipip_entry, 1531 decap_fib_entry); 1532 } 1533 1534 static int 1535 mlxsw_sp_rif_ipip_lb_op(struct mlxsw_sp_rif_ipip_lb *lb_rif, u16 ul_vr_id, 1536 u16 ul_rif_id, bool enable) 1537 { 1538 struct mlxsw_sp_rif_ipip_lb_config lb_cf = lb_rif->lb_config; 1539 struct mlxsw_sp_rif *rif = &lb_rif->common; 1540 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 1541 char ritr_pl[MLXSW_REG_RITR_LEN]; 1542 u32 saddr4; 1543 1544 switch (lb_cf.ul_protocol) { 1545 case MLXSW_SP_L3_PROTO_IPV4: 1546 saddr4 = be32_to_cpu(lb_cf.saddr.addr4); 1547 mlxsw_reg_ritr_pack(ritr_pl, enable, MLXSW_REG_RITR_LOOPBACK_IF, 1548 rif->rif_index, rif->vr_id, rif->dev->mtu); 1549 mlxsw_reg_ritr_loopback_ipip4_pack(ritr_pl, lb_cf.lb_ipipt, 1550 MLXSW_REG_RITR_LOOPBACK_IPIP_OPTIONS_GRE_KEY_PRESET, 1551 ul_vr_id, ul_rif_id, saddr4, lb_cf.okey); 1552 break; 1553 1554 case MLXSW_SP_L3_PROTO_IPV6: 1555 return -EAFNOSUPPORT; 1556 } 1557 1558 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 1559 } 1560 1561 static int mlxsw_sp_netdevice_ipip_ol_update_mtu(struct mlxsw_sp *mlxsw_sp, 1562 struct net_device *ol_dev) 1563 { 1564 struct mlxsw_sp_ipip_entry *ipip_entry; 1565 struct mlxsw_sp_rif_ipip_lb *lb_rif; 1566 int err = 0; 1567 1568 ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev); 1569 if (ipip_entry) { 1570 lb_rif = ipip_entry->ol_lb; 1571 err = mlxsw_sp_rif_ipip_lb_op(lb_rif, lb_rif->ul_vr_id, 1572 lb_rif->ul_rif_id, true); 1573 if (err) 1574 goto out; 1575 lb_rif->common.mtu = ol_dev->mtu; 1576 } 1577 1578 out: 1579 return err; 1580 } 1581 1582 static void mlxsw_sp_netdevice_ipip_ol_up_event(struct mlxsw_sp *mlxsw_sp, 1583 struct net_device *ol_dev) 1584 { 1585 struct mlxsw_sp_ipip_entry *ipip_entry; 1586 1587 ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev); 1588 if (ipip_entry) 1589 mlxsw_sp_ipip_entry_ol_up_event(mlxsw_sp, ipip_entry); 1590 } 1591 1592 static void 1593 mlxsw_sp_ipip_entry_ol_down_event(struct mlxsw_sp *mlxsw_sp, 1594 struct mlxsw_sp_ipip_entry *ipip_entry) 1595 { 1596 if (ipip_entry->decap_fib_entry) 1597 mlxsw_sp_ipip_entry_demote_decap(mlxsw_sp, ipip_entry); 1598 } 1599 1600 static void mlxsw_sp_netdevice_ipip_ol_down_event(struct mlxsw_sp *mlxsw_sp, 1601 struct net_device *ol_dev) 1602 { 1603 struct mlxsw_sp_ipip_entry *ipip_entry; 1604 1605 ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev); 1606 if (ipip_entry) 1607 mlxsw_sp_ipip_entry_ol_down_event(mlxsw_sp, ipip_entry); 1608 } 1609 1610 static void mlxsw_sp_nexthop_rif_migrate(struct mlxsw_sp *mlxsw_sp, 1611 struct mlxsw_sp_rif *old_rif, 1612 struct mlxsw_sp_rif *new_rif); 1613 static int 1614 mlxsw_sp_ipip_entry_ol_lb_update(struct mlxsw_sp *mlxsw_sp, 1615 struct mlxsw_sp_ipip_entry *ipip_entry, 1616 bool keep_encap, 1617 struct netlink_ext_ack *extack) 1618 { 1619 struct mlxsw_sp_rif_ipip_lb *old_lb_rif = ipip_entry->ol_lb; 1620 struct mlxsw_sp_rif_ipip_lb *new_lb_rif; 1621 1622 new_lb_rif = mlxsw_sp_ipip_ol_ipip_lb_create(mlxsw_sp, 1623 ipip_entry->ipipt, 1624 ipip_entry->ol_dev, 1625 extack); 1626 if (IS_ERR(new_lb_rif)) 1627 return PTR_ERR(new_lb_rif); 1628 ipip_entry->ol_lb = new_lb_rif; 1629 1630 if (keep_encap) 1631 mlxsw_sp_nexthop_rif_migrate(mlxsw_sp, &old_lb_rif->common, 1632 &new_lb_rif->common); 1633 1634 mlxsw_sp_rif_destroy(&old_lb_rif->common); 1635 1636 return 0; 1637 } 1638 1639 static void mlxsw_sp_nexthop_rif_update(struct mlxsw_sp *mlxsw_sp, 1640 struct mlxsw_sp_rif *rif); 1641 1642 /** 1643 * __mlxsw_sp_ipip_entry_update_tunnel - Update offload related to IPIP entry. 1644 * @mlxsw_sp: mlxsw_sp. 1645 * @ipip_entry: IPIP entry. 1646 * @recreate_loopback: Recreates the associated loopback RIF. 1647 * @keep_encap: Updates next hops that use the tunnel netdevice. This is only 1648 * relevant when recreate_loopback is true. 1649 * @update_nexthops: Updates next hops, keeping the current loopback RIF. This 1650 * is only relevant when recreate_loopback is false. 1651 * @extack: extack. 1652 * 1653 * Return: Non-zero value on failure. 1654 */ 1655 int __mlxsw_sp_ipip_entry_update_tunnel(struct mlxsw_sp *mlxsw_sp, 1656 struct mlxsw_sp_ipip_entry *ipip_entry, 1657 bool recreate_loopback, 1658 bool keep_encap, 1659 bool update_nexthops, 1660 struct netlink_ext_ack *extack) 1661 { 1662 int err; 1663 1664 /* RIFs can't be edited, so to update loopback, we need to destroy and 1665 * recreate it. That creates a window of opportunity where RALUE and 1666 * RATR registers end up referencing a RIF that's already gone. RATRs 1667 * are handled in mlxsw_sp_ipip_entry_ol_lb_update(), and to take care 1668 * of RALUE, demote the decap route back. 1669 */ 1670 if (ipip_entry->decap_fib_entry) 1671 mlxsw_sp_ipip_entry_demote_decap(mlxsw_sp, ipip_entry); 1672 1673 if (recreate_loopback) { 1674 err = mlxsw_sp_ipip_entry_ol_lb_update(mlxsw_sp, ipip_entry, 1675 keep_encap, extack); 1676 if (err) 1677 return err; 1678 } else if (update_nexthops) { 1679 mlxsw_sp_nexthop_rif_update(mlxsw_sp, 1680 &ipip_entry->ol_lb->common); 1681 } 1682 1683 if (ipip_entry->ol_dev->flags & IFF_UP) 1684 mlxsw_sp_ipip_entry_ol_up_event(mlxsw_sp, ipip_entry); 1685 1686 return 0; 1687 } 1688 1689 static int mlxsw_sp_netdevice_ipip_ol_vrf_event(struct mlxsw_sp *mlxsw_sp, 1690 struct net_device *ol_dev, 1691 struct netlink_ext_ack *extack) 1692 { 1693 struct mlxsw_sp_ipip_entry *ipip_entry = 1694 mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev); 1695 1696 if (!ipip_entry) 1697 return 0; 1698 1699 return __mlxsw_sp_ipip_entry_update_tunnel(mlxsw_sp, ipip_entry, 1700 true, false, false, extack); 1701 } 1702 1703 static int 1704 mlxsw_sp_netdevice_ipip_ul_vrf_event(struct mlxsw_sp *mlxsw_sp, 1705 struct mlxsw_sp_ipip_entry *ipip_entry, 1706 struct net_device *ul_dev, 1707 bool *demote_this, 1708 struct netlink_ext_ack *extack) 1709 { 1710 u32 ul_tb_id = l3mdev_fib_table(ul_dev) ? : RT_TABLE_MAIN; 1711 enum mlxsw_sp_l3proto ul_proto; 1712 union mlxsw_sp_l3addr saddr; 1713 1714 /* Moving underlay to a different VRF might cause local address 1715 * conflict, and the conflicting tunnels need to be demoted. 1716 */ 1717 ul_proto = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt]->ul_proto; 1718 saddr = mlxsw_sp_ipip_netdev_saddr(ul_proto, ipip_entry->ol_dev); 1719 if (mlxsw_sp_ipip_demote_tunnel_by_saddr(mlxsw_sp, ul_proto, 1720 saddr, ul_tb_id, 1721 ipip_entry)) { 1722 *demote_this = true; 1723 return 0; 1724 } 1725 1726 return __mlxsw_sp_ipip_entry_update_tunnel(mlxsw_sp, ipip_entry, 1727 true, true, false, extack); 1728 } 1729 1730 static int 1731 mlxsw_sp_netdevice_ipip_ul_up_event(struct mlxsw_sp *mlxsw_sp, 1732 struct mlxsw_sp_ipip_entry *ipip_entry, 1733 struct net_device *ul_dev) 1734 { 1735 return __mlxsw_sp_ipip_entry_update_tunnel(mlxsw_sp, ipip_entry, 1736 false, false, true, NULL); 1737 } 1738 1739 static int 1740 mlxsw_sp_netdevice_ipip_ul_down_event(struct mlxsw_sp *mlxsw_sp, 1741 struct mlxsw_sp_ipip_entry *ipip_entry, 1742 struct net_device *ul_dev) 1743 { 1744 /* A down underlay device causes encapsulated packets to not be 1745 * forwarded, but decap still works. So refresh next hops without 1746 * touching anything else. 1747 */ 1748 return __mlxsw_sp_ipip_entry_update_tunnel(mlxsw_sp, ipip_entry, 1749 false, false, true, NULL); 1750 } 1751 1752 static int 1753 mlxsw_sp_netdevice_ipip_ol_change_event(struct mlxsw_sp *mlxsw_sp, 1754 struct net_device *ol_dev, 1755 struct netlink_ext_ack *extack) 1756 { 1757 const struct mlxsw_sp_ipip_ops *ipip_ops; 1758 struct mlxsw_sp_ipip_entry *ipip_entry; 1759 int err; 1760 1761 ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev); 1762 if (!ipip_entry) 1763 /* A change might make a tunnel eligible for offloading, but 1764 * that is currently not implemented. What falls to slow path 1765 * stays there. 1766 */ 1767 return 0; 1768 1769 /* A change might make a tunnel not eligible for offloading. */ 1770 if (!mlxsw_sp_netdevice_ipip_can_offload(mlxsw_sp, ol_dev, 1771 ipip_entry->ipipt)) { 1772 mlxsw_sp_ipip_entry_demote_tunnel(mlxsw_sp, ipip_entry); 1773 return 0; 1774 } 1775 1776 ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt]; 1777 err = ipip_ops->ol_netdev_change(mlxsw_sp, ipip_entry, extack); 1778 return err; 1779 } 1780 1781 void mlxsw_sp_ipip_entry_demote_tunnel(struct mlxsw_sp *mlxsw_sp, 1782 struct mlxsw_sp_ipip_entry *ipip_entry) 1783 { 1784 struct net_device *ol_dev = ipip_entry->ol_dev; 1785 1786 if (ol_dev->flags & IFF_UP) 1787 mlxsw_sp_ipip_entry_ol_down_event(mlxsw_sp, ipip_entry); 1788 mlxsw_sp_ipip_entry_destroy(mlxsw_sp, ipip_entry); 1789 } 1790 1791 /* The configuration where several tunnels have the same local address in the 1792 * same underlay table needs special treatment in the HW. That is currently not 1793 * implemented in the driver. This function finds and demotes the first tunnel 1794 * with a given source address, except the one passed in in the argument 1795 * `except'. 1796 */ 1797 bool 1798 mlxsw_sp_ipip_demote_tunnel_by_saddr(struct mlxsw_sp *mlxsw_sp, 1799 enum mlxsw_sp_l3proto ul_proto, 1800 union mlxsw_sp_l3addr saddr, 1801 u32 ul_tb_id, 1802 const struct mlxsw_sp_ipip_entry *except) 1803 { 1804 struct mlxsw_sp_ipip_entry *ipip_entry, *tmp; 1805 1806 list_for_each_entry_safe(ipip_entry, tmp, &mlxsw_sp->router->ipip_list, 1807 ipip_list_node) { 1808 if (ipip_entry != except && 1809 mlxsw_sp_ipip_entry_saddr_matches(mlxsw_sp, ul_proto, saddr, 1810 ul_tb_id, ipip_entry)) { 1811 mlxsw_sp_ipip_entry_demote_tunnel(mlxsw_sp, ipip_entry); 1812 return true; 1813 } 1814 } 1815 1816 return false; 1817 } 1818 1819 static void mlxsw_sp_ipip_demote_tunnel_by_ul_netdev(struct mlxsw_sp *mlxsw_sp, 1820 struct net_device *ul_dev) 1821 { 1822 struct mlxsw_sp_ipip_entry *ipip_entry, *tmp; 1823 1824 list_for_each_entry_safe(ipip_entry, tmp, &mlxsw_sp->router->ipip_list, 1825 ipip_list_node) { 1826 struct net_device *ol_dev = ipip_entry->ol_dev; 1827 struct net_device *ipip_ul_dev; 1828 1829 rcu_read_lock(); 1830 ipip_ul_dev = __mlxsw_sp_ipip_netdev_ul_dev_get(ol_dev); 1831 rcu_read_unlock(); 1832 if (ipip_ul_dev == ul_dev) 1833 mlxsw_sp_ipip_entry_demote_tunnel(mlxsw_sp, ipip_entry); 1834 } 1835 } 1836 1837 int mlxsw_sp_netdevice_ipip_ol_event(struct mlxsw_sp *mlxsw_sp, 1838 struct net_device *ol_dev, 1839 unsigned long event, 1840 struct netdev_notifier_info *info) 1841 { 1842 struct netdev_notifier_changeupper_info *chup; 1843 struct netlink_ext_ack *extack; 1844 int err = 0; 1845 1846 mutex_lock(&mlxsw_sp->router->lock); 1847 switch (event) { 1848 case NETDEV_REGISTER: 1849 err = mlxsw_sp_netdevice_ipip_ol_reg_event(mlxsw_sp, ol_dev); 1850 break; 1851 case NETDEV_UNREGISTER: 1852 mlxsw_sp_netdevice_ipip_ol_unreg_event(mlxsw_sp, ol_dev); 1853 break; 1854 case NETDEV_UP: 1855 mlxsw_sp_netdevice_ipip_ol_up_event(mlxsw_sp, ol_dev); 1856 break; 1857 case NETDEV_DOWN: 1858 mlxsw_sp_netdevice_ipip_ol_down_event(mlxsw_sp, ol_dev); 1859 break; 1860 case NETDEV_CHANGEUPPER: 1861 chup = container_of(info, typeof(*chup), info); 1862 extack = info->extack; 1863 if (netif_is_l3_master(chup->upper_dev)) 1864 err = mlxsw_sp_netdevice_ipip_ol_vrf_event(mlxsw_sp, 1865 ol_dev, 1866 extack); 1867 break; 1868 case NETDEV_CHANGE: 1869 extack = info->extack; 1870 err = mlxsw_sp_netdevice_ipip_ol_change_event(mlxsw_sp, 1871 ol_dev, extack); 1872 break; 1873 case NETDEV_CHANGEMTU: 1874 err = mlxsw_sp_netdevice_ipip_ol_update_mtu(mlxsw_sp, ol_dev); 1875 break; 1876 } 1877 mutex_unlock(&mlxsw_sp->router->lock); 1878 return err; 1879 } 1880 1881 static int 1882 __mlxsw_sp_netdevice_ipip_ul_event(struct mlxsw_sp *mlxsw_sp, 1883 struct mlxsw_sp_ipip_entry *ipip_entry, 1884 struct net_device *ul_dev, 1885 bool *demote_this, 1886 unsigned long event, 1887 struct netdev_notifier_info *info) 1888 { 1889 struct netdev_notifier_changeupper_info *chup; 1890 struct netlink_ext_ack *extack; 1891 1892 switch (event) { 1893 case NETDEV_CHANGEUPPER: 1894 chup = container_of(info, typeof(*chup), info); 1895 extack = info->extack; 1896 if (netif_is_l3_master(chup->upper_dev)) 1897 return mlxsw_sp_netdevice_ipip_ul_vrf_event(mlxsw_sp, 1898 ipip_entry, 1899 ul_dev, 1900 demote_this, 1901 extack); 1902 break; 1903 1904 case NETDEV_UP: 1905 return mlxsw_sp_netdevice_ipip_ul_up_event(mlxsw_sp, ipip_entry, 1906 ul_dev); 1907 case NETDEV_DOWN: 1908 return mlxsw_sp_netdevice_ipip_ul_down_event(mlxsw_sp, 1909 ipip_entry, 1910 ul_dev); 1911 } 1912 return 0; 1913 } 1914 1915 int 1916 mlxsw_sp_netdevice_ipip_ul_event(struct mlxsw_sp *mlxsw_sp, 1917 struct net_device *ul_dev, 1918 unsigned long event, 1919 struct netdev_notifier_info *info) 1920 { 1921 struct mlxsw_sp_ipip_entry *ipip_entry = NULL; 1922 int err = 0; 1923 1924 mutex_lock(&mlxsw_sp->router->lock); 1925 while ((ipip_entry = mlxsw_sp_ipip_entry_find_by_ul_dev(mlxsw_sp, 1926 ul_dev, 1927 ipip_entry))) { 1928 struct mlxsw_sp_ipip_entry *prev; 1929 bool demote_this = false; 1930 1931 err = __mlxsw_sp_netdevice_ipip_ul_event(mlxsw_sp, ipip_entry, 1932 ul_dev, &demote_this, 1933 event, info); 1934 if (err) { 1935 mlxsw_sp_ipip_demote_tunnel_by_ul_netdev(mlxsw_sp, 1936 ul_dev); 1937 break; 1938 } 1939 1940 if (demote_this) { 1941 if (list_is_first(&ipip_entry->ipip_list_node, 1942 &mlxsw_sp->router->ipip_list)) 1943 prev = NULL; 1944 else 1945 /* This can't be cached from previous iteration, 1946 * because that entry could be gone now. 1947 */ 1948 prev = list_prev_entry(ipip_entry, 1949 ipip_list_node); 1950 mlxsw_sp_ipip_entry_demote_tunnel(mlxsw_sp, ipip_entry); 1951 ipip_entry = prev; 1952 } 1953 } 1954 mutex_unlock(&mlxsw_sp->router->lock); 1955 1956 return err; 1957 } 1958 1959 int mlxsw_sp_router_nve_promote_decap(struct mlxsw_sp *mlxsw_sp, u32 ul_tb_id, 1960 enum mlxsw_sp_l3proto ul_proto, 1961 const union mlxsw_sp_l3addr *ul_sip, 1962 u32 tunnel_index) 1963 { 1964 enum mlxsw_sp_fib_entry_type type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP; 1965 struct mlxsw_sp_router *router = mlxsw_sp->router; 1966 struct mlxsw_sp_fib_entry *fib_entry; 1967 int err = 0; 1968 1969 mutex_lock(&mlxsw_sp->router->lock); 1970 1971 if (WARN_ON_ONCE(router->nve_decap_config.valid)) { 1972 err = -EINVAL; 1973 goto out; 1974 } 1975 1976 router->nve_decap_config.ul_tb_id = ul_tb_id; 1977 router->nve_decap_config.tunnel_index = tunnel_index; 1978 router->nve_decap_config.ul_proto = ul_proto; 1979 router->nve_decap_config.ul_sip = *ul_sip; 1980 router->nve_decap_config.valid = true; 1981 1982 /* It is valid to create a tunnel with a local IP and only later 1983 * assign this IP address to a local interface 1984 */ 1985 fib_entry = mlxsw_sp_router_ip2me_fib_entry_find(mlxsw_sp, ul_tb_id, 1986 ul_proto, ul_sip, 1987 type); 1988 if (!fib_entry) 1989 goto out; 1990 1991 fib_entry->decap.tunnel_index = tunnel_index; 1992 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP; 1993 1994 err = mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry); 1995 if (err) 1996 goto err_fib_entry_update; 1997 1998 goto out; 1999 2000 err_fib_entry_update: 2001 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP; 2002 mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry); 2003 out: 2004 mutex_unlock(&mlxsw_sp->router->lock); 2005 return err; 2006 } 2007 2008 void mlxsw_sp_router_nve_demote_decap(struct mlxsw_sp *mlxsw_sp, u32 ul_tb_id, 2009 enum mlxsw_sp_l3proto ul_proto, 2010 const union mlxsw_sp_l3addr *ul_sip) 2011 { 2012 enum mlxsw_sp_fib_entry_type type = MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP; 2013 struct mlxsw_sp_router *router = mlxsw_sp->router; 2014 struct mlxsw_sp_fib_entry *fib_entry; 2015 2016 mutex_lock(&mlxsw_sp->router->lock); 2017 2018 if (WARN_ON_ONCE(!router->nve_decap_config.valid)) 2019 goto out; 2020 2021 router->nve_decap_config.valid = false; 2022 2023 fib_entry = mlxsw_sp_router_ip2me_fib_entry_find(mlxsw_sp, ul_tb_id, 2024 ul_proto, ul_sip, 2025 type); 2026 if (!fib_entry) 2027 goto out; 2028 2029 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP; 2030 mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry); 2031 out: 2032 mutex_unlock(&mlxsw_sp->router->lock); 2033 } 2034 2035 static bool mlxsw_sp_router_nve_is_decap(struct mlxsw_sp *mlxsw_sp, 2036 u32 ul_tb_id, 2037 enum mlxsw_sp_l3proto ul_proto, 2038 const union mlxsw_sp_l3addr *ul_sip) 2039 { 2040 struct mlxsw_sp_router *router = mlxsw_sp->router; 2041 2042 return router->nve_decap_config.valid && 2043 router->nve_decap_config.ul_tb_id == ul_tb_id && 2044 router->nve_decap_config.ul_proto == ul_proto && 2045 !memcmp(&router->nve_decap_config.ul_sip, ul_sip, 2046 sizeof(*ul_sip)); 2047 } 2048 2049 struct mlxsw_sp_neigh_key { 2050 struct neighbour *n; 2051 }; 2052 2053 struct mlxsw_sp_neigh_entry { 2054 struct list_head rif_list_node; 2055 struct rhash_head ht_node; 2056 struct mlxsw_sp_neigh_key key; 2057 u16 rif; 2058 bool connected; 2059 unsigned char ha[ETH_ALEN]; 2060 struct list_head nexthop_list; /* list of nexthops using 2061 * this neigh entry 2062 */ 2063 struct list_head nexthop_neighs_list_node; 2064 unsigned int counter_index; 2065 bool counter_valid; 2066 }; 2067 2068 static const struct rhashtable_params mlxsw_sp_neigh_ht_params = { 2069 .key_offset = offsetof(struct mlxsw_sp_neigh_entry, key), 2070 .head_offset = offsetof(struct mlxsw_sp_neigh_entry, ht_node), 2071 .key_len = sizeof(struct mlxsw_sp_neigh_key), 2072 }; 2073 2074 struct mlxsw_sp_neigh_entry * 2075 mlxsw_sp_rif_neigh_next(struct mlxsw_sp_rif *rif, 2076 struct mlxsw_sp_neigh_entry *neigh_entry) 2077 { 2078 if (!neigh_entry) { 2079 if (list_empty(&rif->neigh_list)) 2080 return NULL; 2081 else 2082 return list_first_entry(&rif->neigh_list, 2083 typeof(*neigh_entry), 2084 rif_list_node); 2085 } 2086 if (list_is_last(&neigh_entry->rif_list_node, &rif->neigh_list)) 2087 return NULL; 2088 return list_next_entry(neigh_entry, rif_list_node); 2089 } 2090 2091 int mlxsw_sp_neigh_entry_type(struct mlxsw_sp_neigh_entry *neigh_entry) 2092 { 2093 return neigh_entry->key.n->tbl->family; 2094 } 2095 2096 unsigned char * 2097 mlxsw_sp_neigh_entry_ha(struct mlxsw_sp_neigh_entry *neigh_entry) 2098 { 2099 return neigh_entry->ha; 2100 } 2101 2102 u32 mlxsw_sp_neigh4_entry_dip(struct mlxsw_sp_neigh_entry *neigh_entry) 2103 { 2104 struct neighbour *n; 2105 2106 n = neigh_entry->key.n; 2107 return ntohl(*((__be32 *) n->primary_key)); 2108 } 2109 2110 struct in6_addr * 2111 mlxsw_sp_neigh6_entry_dip(struct mlxsw_sp_neigh_entry *neigh_entry) 2112 { 2113 struct neighbour *n; 2114 2115 n = neigh_entry->key.n; 2116 return (struct in6_addr *) &n->primary_key; 2117 } 2118 2119 int mlxsw_sp_neigh_counter_get(struct mlxsw_sp *mlxsw_sp, 2120 struct mlxsw_sp_neigh_entry *neigh_entry, 2121 u64 *p_counter) 2122 { 2123 if (!neigh_entry->counter_valid) 2124 return -EINVAL; 2125 2126 return mlxsw_sp_flow_counter_get(mlxsw_sp, neigh_entry->counter_index, 2127 p_counter, NULL); 2128 } 2129 2130 static struct mlxsw_sp_neigh_entry * 2131 mlxsw_sp_neigh_entry_alloc(struct mlxsw_sp *mlxsw_sp, struct neighbour *n, 2132 u16 rif) 2133 { 2134 struct mlxsw_sp_neigh_entry *neigh_entry; 2135 2136 neigh_entry = kzalloc(sizeof(*neigh_entry), GFP_KERNEL); 2137 if (!neigh_entry) 2138 return NULL; 2139 2140 neigh_entry->key.n = n; 2141 neigh_entry->rif = rif; 2142 INIT_LIST_HEAD(&neigh_entry->nexthop_list); 2143 2144 return neigh_entry; 2145 } 2146 2147 static void mlxsw_sp_neigh_entry_free(struct mlxsw_sp_neigh_entry *neigh_entry) 2148 { 2149 kfree(neigh_entry); 2150 } 2151 2152 static int 2153 mlxsw_sp_neigh_entry_insert(struct mlxsw_sp *mlxsw_sp, 2154 struct mlxsw_sp_neigh_entry *neigh_entry) 2155 { 2156 return rhashtable_insert_fast(&mlxsw_sp->router->neigh_ht, 2157 &neigh_entry->ht_node, 2158 mlxsw_sp_neigh_ht_params); 2159 } 2160 2161 static void 2162 mlxsw_sp_neigh_entry_remove(struct mlxsw_sp *mlxsw_sp, 2163 struct mlxsw_sp_neigh_entry *neigh_entry) 2164 { 2165 rhashtable_remove_fast(&mlxsw_sp->router->neigh_ht, 2166 &neigh_entry->ht_node, 2167 mlxsw_sp_neigh_ht_params); 2168 } 2169 2170 static bool 2171 mlxsw_sp_neigh_counter_should_alloc(struct mlxsw_sp *mlxsw_sp, 2172 struct mlxsw_sp_neigh_entry *neigh_entry) 2173 { 2174 struct devlink *devlink; 2175 const char *table_name; 2176 2177 switch (mlxsw_sp_neigh_entry_type(neigh_entry)) { 2178 case AF_INET: 2179 table_name = MLXSW_SP_DPIPE_TABLE_NAME_HOST4; 2180 break; 2181 case AF_INET6: 2182 table_name = MLXSW_SP_DPIPE_TABLE_NAME_HOST6; 2183 break; 2184 default: 2185 WARN_ON(1); 2186 return false; 2187 } 2188 2189 devlink = priv_to_devlink(mlxsw_sp->core); 2190 return devlink_dpipe_table_counter_enabled(devlink, table_name); 2191 } 2192 2193 static void 2194 mlxsw_sp_neigh_counter_alloc(struct mlxsw_sp *mlxsw_sp, 2195 struct mlxsw_sp_neigh_entry *neigh_entry) 2196 { 2197 if (!mlxsw_sp_neigh_counter_should_alloc(mlxsw_sp, neigh_entry)) 2198 return; 2199 2200 if (mlxsw_sp_flow_counter_alloc(mlxsw_sp, &neigh_entry->counter_index)) 2201 return; 2202 2203 neigh_entry->counter_valid = true; 2204 } 2205 2206 static void 2207 mlxsw_sp_neigh_counter_free(struct mlxsw_sp *mlxsw_sp, 2208 struct mlxsw_sp_neigh_entry *neigh_entry) 2209 { 2210 if (!neigh_entry->counter_valid) 2211 return; 2212 mlxsw_sp_flow_counter_free(mlxsw_sp, 2213 neigh_entry->counter_index); 2214 neigh_entry->counter_valid = false; 2215 } 2216 2217 static struct mlxsw_sp_neigh_entry * 2218 mlxsw_sp_neigh_entry_create(struct mlxsw_sp *mlxsw_sp, struct neighbour *n) 2219 { 2220 struct mlxsw_sp_neigh_entry *neigh_entry; 2221 struct mlxsw_sp_rif *rif; 2222 int err; 2223 2224 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, n->dev); 2225 if (!rif) 2226 return ERR_PTR(-EINVAL); 2227 2228 neigh_entry = mlxsw_sp_neigh_entry_alloc(mlxsw_sp, n, rif->rif_index); 2229 if (!neigh_entry) 2230 return ERR_PTR(-ENOMEM); 2231 2232 err = mlxsw_sp_neigh_entry_insert(mlxsw_sp, neigh_entry); 2233 if (err) 2234 goto err_neigh_entry_insert; 2235 2236 mlxsw_sp_neigh_counter_alloc(mlxsw_sp, neigh_entry); 2237 list_add(&neigh_entry->rif_list_node, &rif->neigh_list); 2238 2239 return neigh_entry; 2240 2241 err_neigh_entry_insert: 2242 mlxsw_sp_neigh_entry_free(neigh_entry); 2243 return ERR_PTR(err); 2244 } 2245 2246 static void 2247 mlxsw_sp_neigh_entry_destroy(struct mlxsw_sp *mlxsw_sp, 2248 struct mlxsw_sp_neigh_entry *neigh_entry) 2249 { 2250 list_del(&neigh_entry->rif_list_node); 2251 mlxsw_sp_neigh_counter_free(mlxsw_sp, neigh_entry); 2252 mlxsw_sp_neigh_entry_remove(mlxsw_sp, neigh_entry); 2253 mlxsw_sp_neigh_entry_free(neigh_entry); 2254 } 2255 2256 static struct mlxsw_sp_neigh_entry * 2257 mlxsw_sp_neigh_entry_lookup(struct mlxsw_sp *mlxsw_sp, struct neighbour *n) 2258 { 2259 struct mlxsw_sp_neigh_key key; 2260 2261 key.n = n; 2262 return rhashtable_lookup_fast(&mlxsw_sp->router->neigh_ht, 2263 &key, mlxsw_sp_neigh_ht_params); 2264 } 2265 2266 static void 2267 mlxsw_sp_router_neighs_update_interval_init(struct mlxsw_sp *mlxsw_sp) 2268 { 2269 unsigned long interval; 2270 2271 #if IS_ENABLED(CONFIG_IPV6) 2272 interval = min_t(unsigned long, 2273 NEIGH_VAR(&arp_tbl.parms, DELAY_PROBE_TIME), 2274 NEIGH_VAR(&nd_tbl.parms, DELAY_PROBE_TIME)); 2275 #else 2276 interval = NEIGH_VAR(&arp_tbl.parms, DELAY_PROBE_TIME); 2277 #endif 2278 mlxsw_sp->router->neighs_update.interval = jiffies_to_msecs(interval); 2279 } 2280 2281 static void mlxsw_sp_router_neigh_ent_ipv4_process(struct mlxsw_sp *mlxsw_sp, 2282 char *rauhtd_pl, 2283 int ent_index) 2284 { 2285 struct net_device *dev; 2286 struct neighbour *n; 2287 __be32 dipn; 2288 u32 dip; 2289 u16 rif; 2290 2291 mlxsw_reg_rauhtd_ent_ipv4_unpack(rauhtd_pl, ent_index, &rif, &dip); 2292 2293 if (!mlxsw_sp->router->rifs[rif]) { 2294 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Incorrect RIF in neighbour entry\n"); 2295 return; 2296 } 2297 2298 dipn = htonl(dip); 2299 dev = mlxsw_sp->router->rifs[rif]->dev; 2300 n = neigh_lookup(&arp_tbl, &dipn, dev); 2301 if (!n) 2302 return; 2303 2304 netdev_dbg(dev, "Updating neighbour with IP=%pI4h\n", &dip); 2305 neigh_event_send(n, NULL); 2306 neigh_release(n); 2307 } 2308 2309 #if IS_ENABLED(CONFIG_IPV6) 2310 static void mlxsw_sp_router_neigh_ent_ipv6_process(struct mlxsw_sp *mlxsw_sp, 2311 char *rauhtd_pl, 2312 int rec_index) 2313 { 2314 struct net_device *dev; 2315 struct neighbour *n; 2316 struct in6_addr dip; 2317 u16 rif; 2318 2319 mlxsw_reg_rauhtd_ent_ipv6_unpack(rauhtd_pl, rec_index, &rif, 2320 (char *) &dip); 2321 2322 if (!mlxsw_sp->router->rifs[rif]) { 2323 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Incorrect RIF in neighbour entry\n"); 2324 return; 2325 } 2326 2327 dev = mlxsw_sp->router->rifs[rif]->dev; 2328 n = neigh_lookup(&nd_tbl, &dip, dev); 2329 if (!n) 2330 return; 2331 2332 netdev_dbg(dev, "Updating neighbour with IP=%pI6c\n", &dip); 2333 neigh_event_send(n, NULL); 2334 neigh_release(n); 2335 } 2336 #else 2337 static void mlxsw_sp_router_neigh_ent_ipv6_process(struct mlxsw_sp *mlxsw_sp, 2338 char *rauhtd_pl, 2339 int rec_index) 2340 { 2341 } 2342 #endif 2343 2344 static void mlxsw_sp_router_neigh_rec_ipv4_process(struct mlxsw_sp *mlxsw_sp, 2345 char *rauhtd_pl, 2346 int rec_index) 2347 { 2348 u8 num_entries; 2349 int i; 2350 2351 num_entries = mlxsw_reg_rauhtd_ipv4_rec_num_entries_get(rauhtd_pl, 2352 rec_index); 2353 /* Hardware starts counting at 0, so add 1. */ 2354 num_entries++; 2355 2356 /* Each record consists of several neighbour entries. */ 2357 for (i = 0; i < num_entries; i++) { 2358 int ent_index; 2359 2360 ent_index = rec_index * MLXSW_REG_RAUHTD_IPV4_ENT_PER_REC + i; 2361 mlxsw_sp_router_neigh_ent_ipv4_process(mlxsw_sp, rauhtd_pl, 2362 ent_index); 2363 } 2364 2365 } 2366 2367 static void mlxsw_sp_router_neigh_rec_ipv6_process(struct mlxsw_sp *mlxsw_sp, 2368 char *rauhtd_pl, 2369 int rec_index) 2370 { 2371 /* One record contains one entry. */ 2372 mlxsw_sp_router_neigh_ent_ipv6_process(mlxsw_sp, rauhtd_pl, 2373 rec_index); 2374 } 2375 2376 static void mlxsw_sp_router_neigh_rec_process(struct mlxsw_sp *mlxsw_sp, 2377 char *rauhtd_pl, int rec_index) 2378 { 2379 switch (mlxsw_reg_rauhtd_rec_type_get(rauhtd_pl, rec_index)) { 2380 case MLXSW_REG_RAUHTD_TYPE_IPV4: 2381 mlxsw_sp_router_neigh_rec_ipv4_process(mlxsw_sp, rauhtd_pl, 2382 rec_index); 2383 break; 2384 case MLXSW_REG_RAUHTD_TYPE_IPV6: 2385 mlxsw_sp_router_neigh_rec_ipv6_process(mlxsw_sp, rauhtd_pl, 2386 rec_index); 2387 break; 2388 } 2389 } 2390 2391 static bool mlxsw_sp_router_rauhtd_is_full(char *rauhtd_pl) 2392 { 2393 u8 num_rec, last_rec_index, num_entries; 2394 2395 num_rec = mlxsw_reg_rauhtd_num_rec_get(rauhtd_pl); 2396 last_rec_index = num_rec - 1; 2397 2398 if (num_rec < MLXSW_REG_RAUHTD_REC_MAX_NUM) 2399 return false; 2400 if (mlxsw_reg_rauhtd_rec_type_get(rauhtd_pl, last_rec_index) == 2401 MLXSW_REG_RAUHTD_TYPE_IPV6) 2402 return true; 2403 2404 num_entries = mlxsw_reg_rauhtd_ipv4_rec_num_entries_get(rauhtd_pl, 2405 last_rec_index); 2406 if (++num_entries == MLXSW_REG_RAUHTD_IPV4_ENT_PER_REC) 2407 return true; 2408 return false; 2409 } 2410 2411 static int 2412 __mlxsw_sp_router_neighs_update_rauhtd(struct mlxsw_sp *mlxsw_sp, 2413 char *rauhtd_pl, 2414 enum mlxsw_reg_rauhtd_type type) 2415 { 2416 int i, num_rec; 2417 int err; 2418 2419 /* Ensure the RIF we read from the device does not change mid-dump. */ 2420 mutex_lock(&mlxsw_sp->router->lock); 2421 do { 2422 mlxsw_reg_rauhtd_pack(rauhtd_pl, type); 2423 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(rauhtd), 2424 rauhtd_pl); 2425 if (err) { 2426 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Failed to dump neighbour table\n"); 2427 break; 2428 } 2429 num_rec = mlxsw_reg_rauhtd_num_rec_get(rauhtd_pl); 2430 for (i = 0; i < num_rec; i++) 2431 mlxsw_sp_router_neigh_rec_process(mlxsw_sp, rauhtd_pl, 2432 i); 2433 } while (mlxsw_sp_router_rauhtd_is_full(rauhtd_pl)); 2434 mutex_unlock(&mlxsw_sp->router->lock); 2435 2436 return err; 2437 } 2438 2439 static int mlxsw_sp_router_neighs_update_rauhtd(struct mlxsw_sp *mlxsw_sp) 2440 { 2441 enum mlxsw_reg_rauhtd_type type; 2442 char *rauhtd_pl; 2443 int err; 2444 2445 rauhtd_pl = kmalloc(MLXSW_REG_RAUHTD_LEN, GFP_KERNEL); 2446 if (!rauhtd_pl) 2447 return -ENOMEM; 2448 2449 type = MLXSW_REG_RAUHTD_TYPE_IPV4; 2450 err = __mlxsw_sp_router_neighs_update_rauhtd(mlxsw_sp, rauhtd_pl, type); 2451 if (err) 2452 goto out; 2453 2454 type = MLXSW_REG_RAUHTD_TYPE_IPV6; 2455 err = __mlxsw_sp_router_neighs_update_rauhtd(mlxsw_sp, rauhtd_pl, type); 2456 out: 2457 kfree(rauhtd_pl); 2458 return err; 2459 } 2460 2461 static void mlxsw_sp_router_neighs_update_nh(struct mlxsw_sp *mlxsw_sp) 2462 { 2463 struct mlxsw_sp_neigh_entry *neigh_entry; 2464 2465 mutex_lock(&mlxsw_sp->router->lock); 2466 list_for_each_entry(neigh_entry, &mlxsw_sp->router->nexthop_neighs_list, 2467 nexthop_neighs_list_node) 2468 /* If this neigh have nexthops, make the kernel think this neigh 2469 * is active regardless of the traffic. 2470 */ 2471 neigh_event_send(neigh_entry->key.n, NULL); 2472 mutex_unlock(&mlxsw_sp->router->lock); 2473 } 2474 2475 static void 2476 mlxsw_sp_router_neighs_update_work_schedule(struct mlxsw_sp *mlxsw_sp) 2477 { 2478 unsigned long interval = mlxsw_sp->router->neighs_update.interval; 2479 2480 mlxsw_core_schedule_dw(&mlxsw_sp->router->neighs_update.dw, 2481 msecs_to_jiffies(interval)); 2482 } 2483 2484 static void mlxsw_sp_router_neighs_update_work(struct work_struct *work) 2485 { 2486 struct mlxsw_sp_router *router; 2487 int err; 2488 2489 router = container_of(work, struct mlxsw_sp_router, 2490 neighs_update.dw.work); 2491 err = mlxsw_sp_router_neighs_update_rauhtd(router->mlxsw_sp); 2492 if (err) 2493 dev_err(router->mlxsw_sp->bus_info->dev, "Could not update kernel for neigh activity"); 2494 2495 mlxsw_sp_router_neighs_update_nh(router->mlxsw_sp); 2496 2497 mlxsw_sp_router_neighs_update_work_schedule(router->mlxsw_sp); 2498 } 2499 2500 static void mlxsw_sp_router_probe_unresolved_nexthops(struct work_struct *work) 2501 { 2502 struct mlxsw_sp_neigh_entry *neigh_entry; 2503 struct mlxsw_sp_router *router; 2504 2505 router = container_of(work, struct mlxsw_sp_router, 2506 nexthop_probe_dw.work); 2507 /* Iterate over nexthop neighbours, find those who are unresolved and 2508 * send arp on them. This solves the chicken-egg problem when 2509 * the nexthop wouldn't get offloaded until the neighbor is resolved 2510 * but it wouldn't get resolved ever in case traffic is flowing in HW 2511 * using different nexthop. 2512 */ 2513 mutex_lock(&router->lock); 2514 list_for_each_entry(neigh_entry, &router->nexthop_neighs_list, 2515 nexthop_neighs_list_node) 2516 if (!neigh_entry->connected) 2517 neigh_event_send(neigh_entry->key.n, NULL); 2518 mutex_unlock(&router->lock); 2519 2520 mlxsw_core_schedule_dw(&router->nexthop_probe_dw, 2521 MLXSW_SP_UNRESOLVED_NH_PROBE_INTERVAL); 2522 } 2523 2524 static void 2525 mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp *mlxsw_sp, 2526 struct mlxsw_sp_neigh_entry *neigh_entry, 2527 bool removing, bool dead); 2528 2529 static enum mlxsw_reg_rauht_op mlxsw_sp_rauht_op(bool adding) 2530 { 2531 return adding ? MLXSW_REG_RAUHT_OP_WRITE_ADD : 2532 MLXSW_REG_RAUHT_OP_WRITE_DELETE; 2533 } 2534 2535 static int 2536 mlxsw_sp_router_neigh_entry_op4(struct mlxsw_sp *mlxsw_sp, 2537 struct mlxsw_sp_neigh_entry *neigh_entry, 2538 enum mlxsw_reg_rauht_op op) 2539 { 2540 struct neighbour *n = neigh_entry->key.n; 2541 u32 dip = ntohl(*((__be32 *) n->primary_key)); 2542 char rauht_pl[MLXSW_REG_RAUHT_LEN]; 2543 2544 mlxsw_reg_rauht_pack4(rauht_pl, op, neigh_entry->rif, neigh_entry->ha, 2545 dip); 2546 if (neigh_entry->counter_valid) 2547 mlxsw_reg_rauht_pack_counter(rauht_pl, 2548 neigh_entry->counter_index); 2549 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rauht), rauht_pl); 2550 } 2551 2552 static int 2553 mlxsw_sp_router_neigh_entry_op6(struct mlxsw_sp *mlxsw_sp, 2554 struct mlxsw_sp_neigh_entry *neigh_entry, 2555 enum mlxsw_reg_rauht_op op) 2556 { 2557 struct neighbour *n = neigh_entry->key.n; 2558 char rauht_pl[MLXSW_REG_RAUHT_LEN]; 2559 const char *dip = n->primary_key; 2560 2561 mlxsw_reg_rauht_pack6(rauht_pl, op, neigh_entry->rif, neigh_entry->ha, 2562 dip); 2563 if (neigh_entry->counter_valid) 2564 mlxsw_reg_rauht_pack_counter(rauht_pl, 2565 neigh_entry->counter_index); 2566 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rauht), rauht_pl); 2567 } 2568 2569 bool mlxsw_sp_neigh_ipv6_ignore(struct mlxsw_sp_neigh_entry *neigh_entry) 2570 { 2571 struct neighbour *n = neigh_entry->key.n; 2572 2573 /* Packets with a link-local destination address are trapped 2574 * after LPM lookup and never reach the neighbour table, so 2575 * there is no need to program such neighbours to the device. 2576 */ 2577 if (ipv6_addr_type((struct in6_addr *) &n->primary_key) & 2578 IPV6_ADDR_LINKLOCAL) 2579 return true; 2580 return false; 2581 } 2582 2583 static void 2584 mlxsw_sp_neigh_entry_update(struct mlxsw_sp *mlxsw_sp, 2585 struct mlxsw_sp_neigh_entry *neigh_entry, 2586 bool adding) 2587 { 2588 enum mlxsw_reg_rauht_op op = mlxsw_sp_rauht_op(adding); 2589 int err; 2590 2591 if (!adding && !neigh_entry->connected) 2592 return; 2593 neigh_entry->connected = adding; 2594 if (neigh_entry->key.n->tbl->family == AF_INET) { 2595 err = mlxsw_sp_router_neigh_entry_op4(mlxsw_sp, neigh_entry, 2596 op); 2597 if (err) 2598 return; 2599 } else if (neigh_entry->key.n->tbl->family == AF_INET6) { 2600 if (mlxsw_sp_neigh_ipv6_ignore(neigh_entry)) 2601 return; 2602 err = mlxsw_sp_router_neigh_entry_op6(mlxsw_sp, neigh_entry, 2603 op); 2604 if (err) 2605 return; 2606 } else { 2607 WARN_ON_ONCE(1); 2608 return; 2609 } 2610 2611 if (adding) 2612 neigh_entry->key.n->flags |= NTF_OFFLOADED; 2613 else 2614 neigh_entry->key.n->flags &= ~NTF_OFFLOADED; 2615 } 2616 2617 void 2618 mlxsw_sp_neigh_entry_counter_update(struct mlxsw_sp *mlxsw_sp, 2619 struct mlxsw_sp_neigh_entry *neigh_entry, 2620 bool adding) 2621 { 2622 if (adding) 2623 mlxsw_sp_neigh_counter_alloc(mlxsw_sp, neigh_entry); 2624 else 2625 mlxsw_sp_neigh_counter_free(mlxsw_sp, neigh_entry); 2626 mlxsw_sp_neigh_entry_update(mlxsw_sp, neigh_entry, true); 2627 } 2628 2629 struct mlxsw_sp_netevent_work { 2630 struct work_struct work; 2631 struct mlxsw_sp *mlxsw_sp; 2632 struct neighbour *n; 2633 }; 2634 2635 static void mlxsw_sp_router_neigh_event_work(struct work_struct *work) 2636 { 2637 struct mlxsw_sp_netevent_work *net_work = 2638 container_of(work, struct mlxsw_sp_netevent_work, work); 2639 struct mlxsw_sp *mlxsw_sp = net_work->mlxsw_sp; 2640 struct mlxsw_sp_neigh_entry *neigh_entry; 2641 struct neighbour *n = net_work->n; 2642 unsigned char ha[ETH_ALEN]; 2643 bool entry_connected; 2644 u8 nud_state, dead; 2645 2646 /* If these parameters are changed after we release the lock, 2647 * then we are guaranteed to receive another event letting us 2648 * know about it. 2649 */ 2650 read_lock_bh(&n->lock); 2651 memcpy(ha, n->ha, ETH_ALEN); 2652 nud_state = n->nud_state; 2653 dead = n->dead; 2654 read_unlock_bh(&n->lock); 2655 2656 mutex_lock(&mlxsw_sp->router->lock); 2657 mlxsw_sp_span_respin(mlxsw_sp); 2658 2659 entry_connected = nud_state & NUD_VALID && !dead; 2660 neigh_entry = mlxsw_sp_neigh_entry_lookup(mlxsw_sp, n); 2661 if (!entry_connected && !neigh_entry) 2662 goto out; 2663 if (!neigh_entry) { 2664 neigh_entry = mlxsw_sp_neigh_entry_create(mlxsw_sp, n); 2665 if (IS_ERR(neigh_entry)) 2666 goto out; 2667 } 2668 2669 if (neigh_entry->connected && entry_connected && 2670 !memcmp(neigh_entry->ha, ha, ETH_ALEN)) 2671 goto out; 2672 2673 memcpy(neigh_entry->ha, ha, ETH_ALEN); 2674 mlxsw_sp_neigh_entry_update(mlxsw_sp, neigh_entry, entry_connected); 2675 mlxsw_sp_nexthop_neigh_update(mlxsw_sp, neigh_entry, !entry_connected, 2676 dead); 2677 2678 if (!neigh_entry->connected && list_empty(&neigh_entry->nexthop_list)) 2679 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry); 2680 2681 out: 2682 mutex_unlock(&mlxsw_sp->router->lock); 2683 neigh_release(n); 2684 kfree(net_work); 2685 } 2686 2687 static int mlxsw_sp_mp_hash_init(struct mlxsw_sp *mlxsw_sp); 2688 2689 static void mlxsw_sp_router_mp_hash_event_work(struct work_struct *work) 2690 { 2691 struct mlxsw_sp_netevent_work *net_work = 2692 container_of(work, struct mlxsw_sp_netevent_work, work); 2693 struct mlxsw_sp *mlxsw_sp = net_work->mlxsw_sp; 2694 2695 mlxsw_sp_mp_hash_init(mlxsw_sp); 2696 kfree(net_work); 2697 } 2698 2699 static int __mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp); 2700 2701 static void mlxsw_sp_router_update_priority_work(struct work_struct *work) 2702 { 2703 struct mlxsw_sp_netevent_work *net_work = 2704 container_of(work, struct mlxsw_sp_netevent_work, work); 2705 struct mlxsw_sp *mlxsw_sp = net_work->mlxsw_sp; 2706 2707 __mlxsw_sp_router_init(mlxsw_sp); 2708 kfree(net_work); 2709 } 2710 2711 static int mlxsw_sp_router_schedule_work(struct net *net, 2712 struct notifier_block *nb, 2713 void (*cb)(struct work_struct *)) 2714 { 2715 struct mlxsw_sp_netevent_work *net_work; 2716 struct mlxsw_sp_router *router; 2717 2718 router = container_of(nb, struct mlxsw_sp_router, netevent_nb); 2719 if (!net_eq(net, mlxsw_sp_net(router->mlxsw_sp))) 2720 return NOTIFY_DONE; 2721 2722 net_work = kzalloc(sizeof(*net_work), GFP_ATOMIC); 2723 if (!net_work) 2724 return NOTIFY_BAD; 2725 2726 INIT_WORK(&net_work->work, cb); 2727 net_work->mlxsw_sp = router->mlxsw_sp; 2728 mlxsw_core_schedule_work(&net_work->work); 2729 return NOTIFY_DONE; 2730 } 2731 2732 static int mlxsw_sp_router_netevent_event(struct notifier_block *nb, 2733 unsigned long event, void *ptr) 2734 { 2735 struct mlxsw_sp_netevent_work *net_work; 2736 struct mlxsw_sp_port *mlxsw_sp_port; 2737 struct mlxsw_sp *mlxsw_sp; 2738 unsigned long interval; 2739 struct neigh_parms *p; 2740 struct neighbour *n; 2741 2742 switch (event) { 2743 case NETEVENT_DELAY_PROBE_TIME_UPDATE: 2744 p = ptr; 2745 2746 /* We don't care about changes in the default table. */ 2747 if (!p->dev || (p->tbl->family != AF_INET && 2748 p->tbl->family != AF_INET6)) 2749 return NOTIFY_DONE; 2750 2751 /* We are in atomic context and can't take RTNL mutex, 2752 * so use RCU variant to walk the device chain. 2753 */ 2754 mlxsw_sp_port = mlxsw_sp_port_lower_dev_hold(p->dev); 2755 if (!mlxsw_sp_port) 2756 return NOTIFY_DONE; 2757 2758 mlxsw_sp = mlxsw_sp_port->mlxsw_sp; 2759 interval = jiffies_to_msecs(NEIGH_VAR(p, DELAY_PROBE_TIME)); 2760 mlxsw_sp->router->neighs_update.interval = interval; 2761 2762 mlxsw_sp_port_dev_put(mlxsw_sp_port); 2763 break; 2764 case NETEVENT_NEIGH_UPDATE: 2765 n = ptr; 2766 2767 if (n->tbl->family != AF_INET && n->tbl->family != AF_INET6) 2768 return NOTIFY_DONE; 2769 2770 mlxsw_sp_port = mlxsw_sp_port_lower_dev_hold(n->dev); 2771 if (!mlxsw_sp_port) 2772 return NOTIFY_DONE; 2773 2774 net_work = kzalloc(sizeof(*net_work), GFP_ATOMIC); 2775 if (!net_work) { 2776 mlxsw_sp_port_dev_put(mlxsw_sp_port); 2777 return NOTIFY_BAD; 2778 } 2779 2780 INIT_WORK(&net_work->work, mlxsw_sp_router_neigh_event_work); 2781 net_work->mlxsw_sp = mlxsw_sp_port->mlxsw_sp; 2782 net_work->n = n; 2783 2784 /* Take a reference to ensure the neighbour won't be 2785 * destructed until we drop the reference in delayed 2786 * work. 2787 */ 2788 neigh_clone(n); 2789 mlxsw_core_schedule_work(&net_work->work); 2790 mlxsw_sp_port_dev_put(mlxsw_sp_port); 2791 break; 2792 case NETEVENT_IPV4_MPATH_HASH_UPDATE: 2793 case NETEVENT_IPV6_MPATH_HASH_UPDATE: 2794 return mlxsw_sp_router_schedule_work(ptr, nb, 2795 mlxsw_sp_router_mp_hash_event_work); 2796 2797 case NETEVENT_IPV4_FWD_UPDATE_PRIORITY_UPDATE: 2798 return mlxsw_sp_router_schedule_work(ptr, nb, 2799 mlxsw_sp_router_update_priority_work); 2800 } 2801 2802 return NOTIFY_DONE; 2803 } 2804 2805 static int mlxsw_sp_neigh_init(struct mlxsw_sp *mlxsw_sp) 2806 { 2807 int err; 2808 2809 err = rhashtable_init(&mlxsw_sp->router->neigh_ht, 2810 &mlxsw_sp_neigh_ht_params); 2811 if (err) 2812 return err; 2813 2814 /* Initialize the polling interval according to the default 2815 * table. 2816 */ 2817 mlxsw_sp_router_neighs_update_interval_init(mlxsw_sp); 2818 2819 /* Create the delayed works for the activity_update */ 2820 INIT_DELAYED_WORK(&mlxsw_sp->router->neighs_update.dw, 2821 mlxsw_sp_router_neighs_update_work); 2822 INIT_DELAYED_WORK(&mlxsw_sp->router->nexthop_probe_dw, 2823 mlxsw_sp_router_probe_unresolved_nexthops); 2824 mlxsw_core_schedule_dw(&mlxsw_sp->router->neighs_update.dw, 0); 2825 mlxsw_core_schedule_dw(&mlxsw_sp->router->nexthop_probe_dw, 0); 2826 return 0; 2827 } 2828 2829 static void mlxsw_sp_neigh_fini(struct mlxsw_sp *mlxsw_sp) 2830 { 2831 cancel_delayed_work_sync(&mlxsw_sp->router->neighs_update.dw); 2832 cancel_delayed_work_sync(&mlxsw_sp->router->nexthop_probe_dw); 2833 rhashtable_destroy(&mlxsw_sp->router->neigh_ht); 2834 } 2835 2836 static void mlxsw_sp_neigh_rif_gone_sync(struct mlxsw_sp *mlxsw_sp, 2837 struct mlxsw_sp_rif *rif) 2838 { 2839 struct mlxsw_sp_neigh_entry *neigh_entry, *tmp; 2840 2841 list_for_each_entry_safe(neigh_entry, tmp, &rif->neigh_list, 2842 rif_list_node) { 2843 mlxsw_sp_neigh_entry_update(mlxsw_sp, neigh_entry, false); 2844 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry); 2845 } 2846 } 2847 2848 enum mlxsw_sp_nexthop_type { 2849 MLXSW_SP_NEXTHOP_TYPE_ETH, 2850 MLXSW_SP_NEXTHOP_TYPE_IPIP, 2851 }; 2852 2853 enum mlxsw_sp_nexthop_action { 2854 /* Nexthop forwards packets to an egress RIF */ 2855 MLXSW_SP_NEXTHOP_ACTION_FORWARD, 2856 /* Nexthop discards packets */ 2857 MLXSW_SP_NEXTHOP_ACTION_DISCARD, 2858 /* Nexthop traps packets */ 2859 MLXSW_SP_NEXTHOP_ACTION_TRAP, 2860 }; 2861 2862 struct mlxsw_sp_nexthop_key { 2863 struct fib_nh *fib_nh; 2864 }; 2865 2866 struct mlxsw_sp_nexthop { 2867 struct list_head neigh_list_node; /* member of neigh entry list */ 2868 struct list_head rif_list_node; 2869 struct list_head router_list_node; 2870 struct mlxsw_sp_nexthop_group_info *nhgi; /* pointer back to the group 2871 * this nexthop belongs to 2872 */ 2873 struct rhash_head ht_node; 2874 struct neigh_table *neigh_tbl; 2875 struct mlxsw_sp_nexthop_key key; 2876 unsigned char gw_addr[sizeof(struct in6_addr)]; 2877 int ifindex; 2878 int nh_weight; 2879 int norm_nh_weight; 2880 int num_adj_entries; 2881 struct mlxsw_sp_rif *rif; 2882 u8 should_offload:1, /* set indicates this nexthop should be written 2883 * to the adjacency table. 2884 */ 2885 offloaded:1, /* set indicates this nexthop was written to the 2886 * adjacency table. 2887 */ 2888 update:1; /* set indicates this nexthop should be updated in the 2889 * adjacency table (f.e., its MAC changed). 2890 */ 2891 enum mlxsw_sp_nexthop_action action; 2892 enum mlxsw_sp_nexthop_type type; 2893 union { 2894 struct mlxsw_sp_neigh_entry *neigh_entry; 2895 struct mlxsw_sp_ipip_entry *ipip_entry; 2896 }; 2897 unsigned int counter_index; 2898 bool counter_valid; 2899 }; 2900 2901 enum mlxsw_sp_nexthop_group_type { 2902 MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4, 2903 MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6, 2904 MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ, 2905 }; 2906 2907 struct mlxsw_sp_nexthop_group_info { 2908 struct mlxsw_sp_nexthop_group *nh_grp; 2909 u32 adj_index; 2910 u16 ecmp_size; 2911 u16 count; 2912 int sum_norm_weight; 2913 u8 adj_index_valid:1, 2914 gateway:1; /* routes using the group use a gateway */ 2915 struct mlxsw_sp_nexthop nexthops[0]; 2916 #define nh_rif nexthops[0].rif 2917 }; 2918 2919 struct mlxsw_sp_nexthop_group_vr_key { 2920 u16 vr_id; 2921 enum mlxsw_sp_l3proto proto; 2922 }; 2923 2924 struct mlxsw_sp_nexthop_group_vr_entry { 2925 struct list_head list; /* member in vr_list */ 2926 struct rhash_head ht_node; /* member in vr_ht */ 2927 refcount_t ref_count; 2928 struct mlxsw_sp_nexthop_group_vr_key key; 2929 }; 2930 2931 struct mlxsw_sp_nexthop_group { 2932 struct rhash_head ht_node; 2933 struct list_head fib_list; /* list of fib entries that use this group */ 2934 union { 2935 struct { 2936 struct fib_info *fi; 2937 } ipv4; 2938 struct { 2939 u32 id; 2940 } obj; 2941 }; 2942 struct mlxsw_sp_nexthop_group_info *nhgi; 2943 struct list_head vr_list; 2944 struct rhashtable vr_ht; 2945 enum mlxsw_sp_nexthop_group_type type; 2946 bool can_destroy; 2947 }; 2948 2949 void mlxsw_sp_nexthop_counter_alloc(struct mlxsw_sp *mlxsw_sp, 2950 struct mlxsw_sp_nexthop *nh) 2951 { 2952 struct devlink *devlink; 2953 2954 devlink = priv_to_devlink(mlxsw_sp->core); 2955 if (!devlink_dpipe_table_counter_enabled(devlink, 2956 MLXSW_SP_DPIPE_TABLE_NAME_ADJ)) 2957 return; 2958 2959 if (mlxsw_sp_flow_counter_alloc(mlxsw_sp, &nh->counter_index)) 2960 return; 2961 2962 nh->counter_valid = true; 2963 } 2964 2965 void mlxsw_sp_nexthop_counter_free(struct mlxsw_sp *mlxsw_sp, 2966 struct mlxsw_sp_nexthop *nh) 2967 { 2968 if (!nh->counter_valid) 2969 return; 2970 mlxsw_sp_flow_counter_free(mlxsw_sp, nh->counter_index); 2971 nh->counter_valid = false; 2972 } 2973 2974 int mlxsw_sp_nexthop_counter_get(struct mlxsw_sp *mlxsw_sp, 2975 struct mlxsw_sp_nexthop *nh, u64 *p_counter) 2976 { 2977 if (!nh->counter_valid) 2978 return -EINVAL; 2979 2980 return mlxsw_sp_flow_counter_get(mlxsw_sp, nh->counter_index, 2981 p_counter, NULL); 2982 } 2983 2984 struct mlxsw_sp_nexthop *mlxsw_sp_nexthop_next(struct mlxsw_sp_router *router, 2985 struct mlxsw_sp_nexthop *nh) 2986 { 2987 if (!nh) { 2988 if (list_empty(&router->nexthop_list)) 2989 return NULL; 2990 else 2991 return list_first_entry(&router->nexthop_list, 2992 typeof(*nh), router_list_node); 2993 } 2994 if (list_is_last(&nh->router_list_node, &router->nexthop_list)) 2995 return NULL; 2996 return list_next_entry(nh, router_list_node); 2997 } 2998 2999 bool mlxsw_sp_nexthop_is_forward(const struct mlxsw_sp_nexthop *nh) 3000 { 3001 return nh->offloaded && nh->action == MLXSW_SP_NEXTHOP_ACTION_FORWARD; 3002 } 3003 3004 unsigned char *mlxsw_sp_nexthop_ha(struct mlxsw_sp_nexthop *nh) 3005 { 3006 if (nh->type != MLXSW_SP_NEXTHOP_TYPE_ETH || 3007 !mlxsw_sp_nexthop_is_forward(nh)) 3008 return NULL; 3009 return nh->neigh_entry->ha; 3010 } 3011 3012 int mlxsw_sp_nexthop_indexes(struct mlxsw_sp_nexthop *nh, u32 *p_adj_index, 3013 u32 *p_adj_size, u32 *p_adj_hash_index) 3014 { 3015 struct mlxsw_sp_nexthop_group_info *nhgi = nh->nhgi; 3016 u32 adj_hash_index = 0; 3017 int i; 3018 3019 if (!nh->offloaded || !nhgi->adj_index_valid) 3020 return -EINVAL; 3021 3022 *p_adj_index = nhgi->adj_index; 3023 *p_adj_size = nhgi->ecmp_size; 3024 3025 for (i = 0; i < nhgi->count; i++) { 3026 struct mlxsw_sp_nexthop *nh_iter = &nhgi->nexthops[i]; 3027 3028 if (nh_iter == nh) 3029 break; 3030 if (nh_iter->offloaded) 3031 adj_hash_index += nh_iter->num_adj_entries; 3032 } 3033 3034 *p_adj_hash_index = adj_hash_index; 3035 return 0; 3036 } 3037 3038 struct mlxsw_sp_rif *mlxsw_sp_nexthop_rif(struct mlxsw_sp_nexthop *nh) 3039 { 3040 return nh->rif; 3041 } 3042 3043 bool mlxsw_sp_nexthop_group_has_ipip(struct mlxsw_sp_nexthop *nh) 3044 { 3045 struct mlxsw_sp_nexthop_group_info *nhgi = nh->nhgi; 3046 int i; 3047 3048 for (i = 0; i < nhgi->count; i++) { 3049 struct mlxsw_sp_nexthop *nh_iter = &nhgi->nexthops[i]; 3050 3051 if (nh_iter->type == MLXSW_SP_NEXTHOP_TYPE_IPIP) 3052 return true; 3053 } 3054 return false; 3055 } 3056 3057 static const struct rhashtable_params mlxsw_sp_nexthop_group_vr_ht_params = { 3058 .key_offset = offsetof(struct mlxsw_sp_nexthop_group_vr_entry, key), 3059 .head_offset = offsetof(struct mlxsw_sp_nexthop_group_vr_entry, ht_node), 3060 .key_len = sizeof(struct mlxsw_sp_nexthop_group_vr_key), 3061 .automatic_shrinking = true, 3062 }; 3063 3064 static struct mlxsw_sp_nexthop_group_vr_entry * 3065 mlxsw_sp_nexthop_group_vr_entry_lookup(struct mlxsw_sp_nexthop_group *nh_grp, 3066 const struct mlxsw_sp_fib *fib) 3067 { 3068 struct mlxsw_sp_nexthop_group_vr_key key; 3069 3070 memset(&key, 0, sizeof(key)); 3071 key.vr_id = fib->vr->id; 3072 key.proto = fib->proto; 3073 return rhashtable_lookup_fast(&nh_grp->vr_ht, &key, 3074 mlxsw_sp_nexthop_group_vr_ht_params); 3075 } 3076 3077 static int 3078 mlxsw_sp_nexthop_group_vr_entry_create(struct mlxsw_sp_nexthop_group *nh_grp, 3079 const struct mlxsw_sp_fib *fib) 3080 { 3081 struct mlxsw_sp_nexthop_group_vr_entry *vr_entry; 3082 int err; 3083 3084 vr_entry = kzalloc(sizeof(*vr_entry), GFP_KERNEL); 3085 if (!vr_entry) 3086 return -ENOMEM; 3087 3088 vr_entry->key.vr_id = fib->vr->id; 3089 vr_entry->key.proto = fib->proto; 3090 refcount_set(&vr_entry->ref_count, 1); 3091 3092 err = rhashtable_insert_fast(&nh_grp->vr_ht, &vr_entry->ht_node, 3093 mlxsw_sp_nexthop_group_vr_ht_params); 3094 if (err) 3095 goto err_hashtable_insert; 3096 3097 list_add(&vr_entry->list, &nh_grp->vr_list); 3098 3099 return 0; 3100 3101 err_hashtable_insert: 3102 kfree(vr_entry); 3103 return err; 3104 } 3105 3106 static void 3107 mlxsw_sp_nexthop_group_vr_entry_destroy(struct mlxsw_sp_nexthop_group *nh_grp, 3108 struct mlxsw_sp_nexthop_group_vr_entry *vr_entry) 3109 { 3110 list_del(&vr_entry->list); 3111 rhashtable_remove_fast(&nh_grp->vr_ht, &vr_entry->ht_node, 3112 mlxsw_sp_nexthop_group_vr_ht_params); 3113 kfree(vr_entry); 3114 } 3115 3116 static int 3117 mlxsw_sp_nexthop_group_vr_link(struct mlxsw_sp_nexthop_group *nh_grp, 3118 const struct mlxsw_sp_fib *fib) 3119 { 3120 struct mlxsw_sp_nexthop_group_vr_entry *vr_entry; 3121 3122 vr_entry = mlxsw_sp_nexthop_group_vr_entry_lookup(nh_grp, fib); 3123 if (vr_entry) { 3124 refcount_inc(&vr_entry->ref_count); 3125 return 0; 3126 } 3127 3128 return mlxsw_sp_nexthop_group_vr_entry_create(nh_grp, fib); 3129 } 3130 3131 static void 3132 mlxsw_sp_nexthop_group_vr_unlink(struct mlxsw_sp_nexthop_group *nh_grp, 3133 const struct mlxsw_sp_fib *fib) 3134 { 3135 struct mlxsw_sp_nexthop_group_vr_entry *vr_entry; 3136 3137 vr_entry = mlxsw_sp_nexthop_group_vr_entry_lookup(nh_grp, fib); 3138 if (WARN_ON_ONCE(!vr_entry)) 3139 return; 3140 3141 if (!refcount_dec_and_test(&vr_entry->ref_count)) 3142 return; 3143 3144 mlxsw_sp_nexthop_group_vr_entry_destroy(nh_grp, vr_entry); 3145 } 3146 3147 struct mlxsw_sp_nexthop_group_cmp_arg { 3148 enum mlxsw_sp_nexthop_group_type type; 3149 union { 3150 struct fib_info *fi; 3151 struct mlxsw_sp_fib6_entry *fib6_entry; 3152 u32 id; 3153 }; 3154 }; 3155 3156 static bool 3157 mlxsw_sp_nexthop6_group_has_nexthop(const struct mlxsw_sp_nexthop_group *nh_grp, 3158 const struct in6_addr *gw, int ifindex, 3159 int weight) 3160 { 3161 int i; 3162 3163 for (i = 0; i < nh_grp->nhgi->count; i++) { 3164 const struct mlxsw_sp_nexthop *nh; 3165 3166 nh = &nh_grp->nhgi->nexthops[i]; 3167 if (nh->ifindex == ifindex && nh->nh_weight == weight && 3168 ipv6_addr_equal(gw, (struct in6_addr *) nh->gw_addr)) 3169 return true; 3170 } 3171 3172 return false; 3173 } 3174 3175 static bool 3176 mlxsw_sp_nexthop6_group_cmp(const struct mlxsw_sp_nexthop_group *nh_grp, 3177 const struct mlxsw_sp_fib6_entry *fib6_entry) 3178 { 3179 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 3180 3181 if (nh_grp->nhgi->count != fib6_entry->nrt6) 3182 return false; 3183 3184 list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) { 3185 struct fib6_nh *fib6_nh = mlxsw_sp_rt6->rt->fib6_nh; 3186 struct in6_addr *gw; 3187 int ifindex, weight; 3188 3189 ifindex = fib6_nh->fib_nh_dev->ifindex; 3190 weight = fib6_nh->fib_nh_weight; 3191 gw = &fib6_nh->fib_nh_gw6; 3192 if (!mlxsw_sp_nexthop6_group_has_nexthop(nh_grp, gw, ifindex, 3193 weight)) 3194 return false; 3195 } 3196 3197 return true; 3198 } 3199 3200 static int 3201 mlxsw_sp_nexthop_group_cmp(struct rhashtable_compare_arg *arg, const void *ptr) 3202 { 3203 const struct mlxsw_sp_nexthop_group_cmp_arg *cmp_arg = arg->key; 3204 const struct mlxsw_sp_nexthop_group *nh_grp = ptr; 3205 3206 if (nh_grp->type != cmp_arg->type) 3207 return 1; 3208 3209 switch (cmp_arg->type) { 3210 case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4: 3211 return cmp_arg->fi != nh_grp->ipv4.fi; 3212 case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6: 3213 return !mlxsw_sp_nexthop6_group_cmp(nh_grp, 3214 cmp_arg->fib6_entry); 3215 case MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ: 3216 return cmp_arg->id != nh_grp->obj.id; 3217 default: 3218 WARN_ON(1); 3219 return 1; 3220 } 3221 } 3222 3223 static u32 mlxsw_sp_nexthop_group_hash_obj(const void *data, u32 len, u32 seed) 3224 { 3225 const struct mlxsw_sp_nexthop_group *nh_grp = data; 3226 const struct mlxsw_sp_nexthop *nh; 3227 struct fib_info *fi; 3228 unsigned int val; 3229 int i; 3230 3231 switch (nh_grp->type) { 3232 case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4: 3233 fi = nh_grp->ipv4.fi; 3234 return jhash(&fi, sizeof(fi), seed); 3235 case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6: 3236 val = nh_grp->nhgi->count; 3237 for (i = 0; i < nh_grp->nhgi->count; i++) { 3238 nh = &nh_grp->nhgi->nexthops[i]; 3239 val ^= jhash(&nh->ifindex, sizeof(nh->ifindex), seed); 3240 val ^= jhash(&nh->gw_addr, sizeof(nh->gw_addr), seed); 3241 } 3242 return jhash(&val, sizeof(val), seed); 3243 case MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ: 3244 return jhash(&nh_grp->obj.id, sizeof(nh_grp->obj.id), seed); 3245 default: 3246 WARN_ON(1); 3247 return 0; 3248 } 3249 } 3250 3251 static u32 3252 mlxsw_sp_nexthop6_group_hash(struct mlxsw_sp_fib6_entry *fib6_entry, u32 seed) 3253 { 3254 unsigned int val = fib6_entry->nrt6; 3255 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 3256 3257 list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) { 3258 struct fib6_nh *fib6_nh = mlxsw_sp_rt6->rt->fib6_nh; 3259 struct net_device *dev = fib6_nh->fib_nh_dev; 3260 struct in6_addr *gw = &fib6_nh->fib_nh_gw6; 3261 3262 val ^= jhash(&dev->ifindex, sizeof(dev->ifindex), seed); 3263 val ^= jhash(gw, sizeof(*gw), seed); 3264 } 3265 3266 return jhash(&val, sizeof(val), seed); 3267 } 3268 3269 static u32 3270 mlxsw_sp_nexthop_group_hash(const void *data, u32 len, u32 seed) 3271 { 3272 const struct mlxsw_sp_nexthop_group_cmp_arg *cmp_arg = data; 3273 3274 switch (cmp_arg->type) { 3275 case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4: 3276 return jhash(&cmp_arg->fi, sizeof(cmp_arg->fi), seed); 3277 case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6: 3278 return mlxsw_sp_nexthop6_group_hash(cmp_arg->fib6_entry, seed); 3279 case MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ: 3280 return jhash(&cmp_arg->id, sizeof(cmp_arg->id), seed); 3281 default: 3282 WARN_ON(1); 3283 return 0; 3284 } 3285 } 3286 3287 static const struct rhashtable_params mlxsw_sp_nexthop_group_ht_params = { 3288 .head_offset = offsetof(struct mlxsw_sp_nexthop_group, ht_node), 3289 .hashfn = mlxsw_sp_nexthop_group_hash, 3290 .obj_hashfn = mlxsw_sp_nexthop_group_hash_obj, 3291 .obj_cmpfn = mlxsw_sp_nexthop_group_cmp, 3292 }; 3293 3294 static int mlxsw_sp_nexthop_group_insert(struct mlxsw_sp *mlxsw_sp, 3295 struct mlxsw_sp_nexthop_group *nh_grp) 3296 { 3297 if (nh_grp->type == MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6 && 3298 !nh_grp->nhgi->gateway) 3299 return 0; 3300 3301 return rhashtable_insert_fast(&mlxsw_sp->router->nexthop_group_ht, 3302 &nh_grp->ht_node, 3303 mlxsw_sp_nexthop_group_ht_params); 3304 } 3305 3306 static void mlxsw_sp_nexthop_group_remove(struct mlxsw_sp *mlxsw_sp, 3307 struct mlxsw_sp_nexthop_group *nh_grp) 3308 { 3309 if (nh_grp->type == MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6 && 3310 !nh_grp->nhgi->gateway) 3311 return; 3312 3313 rhashtable_remove_fast(&mlxsw_sp->router->nexthop_group_ht, 3314 &nh_grp->ht_node, 3315 mlxsw_sp_nexthop_group_ht_params); 3316 } 3317 3318 static struct mlxsw_sp_nexthop_group * 3319 mlxsw_sp_nexthop4_group_lookup(struct mlxsw_sp *mlxsw_sp, 3320 struct fib_info *fi) 3321 { 3322 struct mlxsw_sp_nexthop_group_cmp_arg cmp_arg; 3323 3324 cmp_arg.type = MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4; 3325 cmp_arg.fi = fi; 3326 return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_group_ht, 3327 &cmp_arg, 3328 mlxsw_sp_nexthop_group_ht_params); 3329 } 3330 3331 static struct mlxsw_sp_nexthop_group * 3332 mlxsw_sp_nexthop6_group_lookup(struct mlxsw_sp *mlxsw_sp, 3333 struct mlxsw_sp_fib6_entry *fib6_entry) 3334 { 3335 struct mlxsw_sp_nexthop_group_cmp_arg cmp_arg; 3336 3337 cmp_arg.type = MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6; 3338 cmp_arg.fib6_entry = fib6_entry; 3339 return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_group_ht, 3340 &cmp_arg, 3341 mlxsw_sp_nexthop_group_ht_params); 3342 } 3343 3344 static const struct rhashtable_params mlxsw_sp_nexthop_ht_params = { 3345 .key_offset = offsetof(struct mlxsw_sp_nexthop, key), 3346 .head_offset = offsetof(struct mlxsw_sp_nexthop, ht_node), 3347 .key_len = sizeof(struct mlxsw_sp_nexthop_key), 3348 }; 3349 3350 static int mlxsw_sp_nexthop_insert(struct mlxsw_sp *mlxsw_sp, 3351 struct mlxsw_sp_nexthop *nh) 3352 { 3353 return rhashtable_insert_fast(&mlxsw_sp->router->nexthop_ht, 3354 &nh->ht_node, mlxsw_sp_nexthop_ht_params); 3355 } 3356 3357 static void mlxsw_sp_nexthop_remove(struct mlxsw_sp *mlxsw_sp, 3358 struct mlxsw_sp_nexthop *nh) 3359 { 3360 rhashtable_remove_fast(&mlxsw_sp->router->nexthop_ht, &nh->ht_node, 3361 mlxsw_sp_nexthop_ht_params); 3362 } 3363 3364 static struct mlxsw_sp_nexthop * 3365 mlxsw_sp_nexthop_lookup(struct mlxsw_sp *mlxsw_sp, 3366 struct mlxsw_sp_nexthop_key key) 3367 { 3368 return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_ht, &key, 3369 mlxsw_sp_nexthop_ht_params); 3370 } 3371 3372 static int mlxsw_sp_adj_index_mass_update_vr(struct mlxsw_sp *mlxsw_sp, 3373 enum mlxsw_sp_l3proto proto, 3374 u16 vr_id, 3375 u32 adj_index, u16 ecmp_size, 3376 u32 new_adj_index, 3377 u16 new_ecmp_size) 3378 { 3379 char raleu_pl[MLXSW_REG_RALEU_LEN]; 3380 3381 mlxsw_reg_raleu_pack(raleu_pl, 3382 (enum mlxsw_reg_ralxx_protocol) proto, vr_id, 3383 adj_index, ecmp_size, new_adj_index, 3384 new_ecmp_size); 3385 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raleu), raleu_pl); 3386 } 3387 3388 static int mlxsw_sp_adj_index_mass_update(struct mlxsw_sp *mlxsw_sp, 3389 struct mlxsw_sp_nexthop_group *nh_grp, 3390 u32 old_adj_index, u16 old_ecmp_size) 3391 { 3392 struct mlxsw_sp_nexthop_group_info *nhgi = nh_grp->nhgi; 3393 struct mlxsw_sp_nexthop_group_vr_entry *vr_entry; 3394 int err; 3395 3396 list_for_each_entry(vr_entry, &nh_grp->vr_list, list) { 3397 err = mlxsw_sp_adj_index_mass_update_vr(mlxsw_sp, 3398 vr_entry->key.proto, 3399 vr_entry->key.vr_id, 3400 old_adj_index, 3401 old_ecmp_size, 3402 nhgi->adj_index, 3403 nhgi->ecmp_size); 3404 if (err) 3405 goto err_mass_update_vr; 3406 } 3407 return 0; 3408 3409 err_mass_update_vr: 3410 list_for_each_entry_continue_reverse(vr_entry, &nh_grp->vr_list, list) 3411 mlxsw_sp_adj_index_mass_update_vr(mlxsw_sp, vr_entry->key.proto, 3412 vr_entry->key.vr_id, 3413 nhgi->adj_index, 3414 nhgi->ecmp_size, 3415 old_adj_index, old_ecmp_size); 3416 return err; 3417 } 3418 3419 static int __mlxsw_sp_nexthop_eth_update(struct mlxsw_sp *mlxsw_sp, 3420 u32 adj_index, 3421 struct mlxsw_sp_nexthop *nh) 3422 { 3423 struct mlxsw_sp_neigh_entry *neigh_entry = nh->neigh_entry; 3424 char ratr_pl[MLXSW_REG_RATR_LEN]; 3425 u16 rif_index; 3426 3427 rif_index = nh->rif ? nh->rif->rif_index : 3428 mlxsw_sp->router->lb_rif_index; 3429 mlxsw_reg_ratr_pack(ratr_pl, MLXSW_REG_RATR_OP_WRITE_WRITE_ENTRY, 3430 true, MLXSW_REG_RATR_TYPE_ETHERNET, 3431 adj_index, rif_index); 3432 switch (nh->action) { 3433 case MLXSW_SP_NEXTHOP_ACTION_FORWARD: 3434 mlxsw_reg_ratr_eth_entry_pack(ratr_pl, neigh_entry->ha); 3435 break; 3436 case MLXSW_SP_NEXTHOP_ACTION_DISCARD: 3437 mlxsw_reg_ratr_trap_action_set(ratr_pl, 3438 MLXSW_REG_RATR_TRAP_ACTION_DISCARD_ERRORS); 3439 break; 3440 case MLXSW_SP_NEXTHOP_ACTION_TRAP: 3441 mlxsw_reg_ratr_trap_action_set(ratr_pl, 3442 MLXSW_REG_RATR_TRAP_ACTION_TRAP); 3443 mlxsw_reg_ratr_trap_id_set(ratr_pl, MLXSW_TRAP_ID_RTR_EGRESS0); 3444 break; 3445 default: 3446 WARN_ON_ONCE(1); 3447 return -EINVAL; 3448 } 3449 if (nh->counter_valid) 3450 mlxsw_reg_ratr_counter_pack(ratr_pl, nh->counter_index, true); 3451 else 3452 mlxsw_reg_ratr_counter_pack(ratr_pl, 0, false); 3453 3454 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ratr), ratr_pl); 3455 } 3456 3457 int mlxsw_sp_nexthop_eth_update(struct mlxsw_sp *mlxsw_sp, u32 adj_index, 3458 struct mlxsw_sp_nexthop *nh) 3459 { 3460 int i; 3461 3462 for (i = 0; i < nh->num_adj_entries; i++) { 3463 int err; 3464 3465 err = __mlxsw_sp_nexthop_eth_update(mlxsw_sp, adj_index + i, 3466 nh); 3467 if (err) 3468 return err; 3469 } 3470 3471 return 0; 3472 } 3473 3474 static int __mlxsw_sp_nexthop_ipip_update(struct mlxsw_sp *mlxsw_sp, 3475 u32 adj_index, 3476 struct mlxsw_sp_nexthop *nh) 3477 { 3478 const struct mlxsw_sp_ipip_ops *ipip_ops; 3479 3480 ipip_ops = mlxsw_sp->router->ipip_ops_arr[nh->ipip_entry->ipipt]; 3481 return ipip_ops->nexthop_update(mlxsw_sp, adj_index, nh->ipip_entry); 3482 } 3483 3484 static int mlxsw_sp_nexthop_ipip_update(struct mlxsw_sp *mlxsw_sp, 3485 u32 adj_index, 3486 struct mlxsw_sp_nexthop *nh) 3487 { 3488 int i; 3489 3490 for (i = 0; i < nh->num_adj_entries; i++) { 3491 int err; 3492 3493 err = __mlxsw_sp_nexthop_ipip_update(mlxsw_sp, adj_index + i, 3494 nh); 3495 if (err) 3496 return err; 3497 } 3498 3499 return 0; 3500 } 3501 3502 static int mlxsw_sp_nexthop_update(struct mlxsw_sp *mlxsw_sp, u32 adj_index, 3503 struct mlxsw_sp_nexthop *nh) 3504 { 3505 /* When action is discard or trap, the nexthop must be 3506 * programmed as an Ethernet nexthop. 3507 */ 3508 if (nh->type == MLXSW_SP_NEXTHOP_TYPE_ETH || 3509 nh->action == MLXSW_SP_NEXTHOP_ACTION_DISCARD || 3510 nh->action == MLXSW_SP_NEXTHOP_ACTION_TRAP) 3511 return mlxsw_sp_nexthop_eth_update(mlxsw_sp, adj_index, nh); 3512 else 3513 return mlxsw_sp_nexthop_ipip_update(mlxsw_sp, adj_index, nh); 3514 } 3515 3516 static int 3517 mlxsw_sp_nexthop_group_update(struct mlxsw_sp *mlxsw_sp, 3518 struct mlxsw_sp_nexthop_group_info *nhgi, 3519 bool reallocate) 3520 { 3521 u32 adj_index = nhgi->adj_index; /* base */ 3522 struct mlxsw_sp_nexthop *nh; 3523 int i; 3524 3525 for (i = 0; i < nhgi->count; i++) { 3526 nh = &nhgi->nexthops[i]; 3527 3528 if (!nh->should_offload) { 3529 nh->offloaded = 0; 3530 continue; 3531 } 3532 3533 if (nh->update || reallocate) { 3534 int err = 0; 3535 3536 err = mlxsw_sp_nexthop_update(mlxsw_sp, adj_index, nh); 3537 if (err) 3538 return err; 3539 nh->update = 0; 3540 nh->offloaded = 1; 3541 } 3542 adj_index += nh->num_adj_entries; 3543 } 3544 return 0; 3545 } 3546 3547 static int 3548 mlxsw_sp_nexthop_fib_entries_update(struct mlxsw_sp *mlxsw_sp, 3549 struct mlxsw_sp_nexthop_group *nh_grp) 3550 { 3551 struct mlxsw_sp_fib_entry *fib_entry; 3552 int err; 3553 3554 list_for_each_entry(fib_entry, &nh_grp->fib_list, nexthop_group_node) { 3555 err = mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry); 3556 if (err) 3557 return err; 3558 } 3559 return 0; 3560 } 3561 3562 struct mlxsw_sp_adj_grp_size_range { 3563 u16 start; /* Inclusive */ 3564 u16 end; /* Inclusive */ 3565 }; 3566 3567 /* Ordered by range start value */ 3568 static const struct mlxsw_sp_adj_grp_size_range 3569 mlxsw_sp1_adj_grp_size_ranges[] = { 3570 { .start = 1, .end = 64 }, 3571 { .start = 512, .end = 512 }, 3572 { .start = 1024, .end = 1024 }, 3573 { .start = 2048, .end = 2048 }, 3574 { .start = 4096, .end = 4096 }, 3575 }; 3576 3577 /* Ordered by range start value */ 3578 static const struct mlxsw_sp_adj_grp_size_range 3579 mlxsw_sp2_adj_grp_size_ranges[] = { 3580 { .start = 1, .end = 128 }, 3581 { .start = 256, .end = 256 }, 3582 { .start = 512, .end = 512 }, 3583 { .start = 1024, .end = 1024 }, 3584 { .start = 2048, .end = 2048 }, 3585 { .start = 4096, .end = 4096 }, 3586 }; 3587 3588 static void mlxsw_sp_adj_grp_size_round_up(const struct mlxsw_sp *mlxsw_sp, 3589 u16 *p_adj_grp_size) 3590 { 3591 int i; 3592 3593 for (i = 0; i < mlxsw_sp->router->adj_grp_size_ranges_count; i++) { 3594 const struct mlxsw_sp_adj_grp_size_range *size_range; 3595 3596 size_range = &mlxsw_sp->router->adj_grp_size_ranges[i]; 3597 3598 if (*p_adj_grp_size >= size_range->start && 3599 *p_adj_grp_size <= size_range->end) 3600 return; 3601 3602 if (*p_adj_grp_size <= size_range->end) { 3603 *p_adj_grp_size = size_range->end; 3604 return; 3605 } 3606 } 3607 } 3608 3609 static void mlxsw_sp_adj_grp_size_round_down(const struct mlxsw_sp *mlxsw_sp, 3610 u16 *p_adj_grp_size, 3611 unsigned int alloc_size) 3612 { 3613 int i; 3614 3615 for (i = mlxsw_sp->router->adj_grp_size_ranges_count - 1; i >= 0; i--) { 3616 const struct mlxsw_sp_adj_grp_size_range *size_range; 3617 3618 size_range = &mlxsw_sp->router->adj_grp_size_ranges[i]; 3619 3620 if (alloc_size >= size_range->end) { 3621 *p_adj_grp_size = size_range->end; 3622 return; 3623 } 3624 } 3625 } 3626 3627 static int mlxsw_sp_fix_adj_grp_size(struct mlxsw_sp *mlxsw_sp, 3628 u16 *p_adj_grp_size) 3629 { 3630 unsigned int alloc_size; 3631 int err; 3632 3633 /* Round up the requested group size to the next size supported 3634 * by the device and make sure the request can be satisfied. 3635 */ 3636 mlxsw_sp_adj_grp_size_round_up(mlxsw_sp, p_adj_grp_size); 3637 err = mlxsw_sp_kvdl_alloc_count_query(mlxsw_sp, 3638 MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 3639 *p_adj_grp_size, &alloc_size); 3640 if (err) 3641 return err; 3642 /* It is possible the allocation results in more allocated 3643 * entries than requested. Try to use as much of them as 3644 * possible. 3645 */ 3646 mlxsw_sp_adj_grp_size_round_down(mlxsw_sp, p_adj_grp_size, alloc_size); 3647 3648 return 0; 3649 } 3650 3651 static void 3652 mlxsw_sp_nexthop_group_normalize(struct mlxsw_sp_nexthop_group_info *nhgi) 3653 { 3654 int i, g = 0, sum_norm_weight = 0; 3655 struct mlxsw_sp_nexthop *nh; 3656 3657 for (i = 0; i < nhgi->count; i++) { 3658 nh = &nhgi->nexthops[i]; 3659 3660 if (!nh->should_offload) 3661 continue; 3662 if (g > 0) 3663 g = gcd(nh->nh_weight, g); 3664 else 3665 g = nh->nh_weight; 3666 } 3667 3668 for (i = 0; i < nhgi->count; i++) { 3669 nh = &nhgi->nexthops[i]; 3670 3671 if (!nh->should_offload) 3672 continue; 3673 nh->norm_nh_weight = nh->nh_weight / g; 3674 sum_norm_weight += nh->norm_nh_weight; 3675 } 3676 3677 nhgi->sum_norm_weight = sum_norm_weight; 3678 } 3679 3680 static void 3681 mlxsw_sp_nexthop_group_rebalance(struct mlxsw_sp_nexthop_group_info *nhgi) 3682 { 3683 int i, weight = 0, lower_bound = 0; 3684 int total = nhgi->sum_norm_weight; 3685 u16 ecmp_size = nhgi->ecmp_size; 3686 3687 for (i = 0; i < nhgi->count; i++) { 3688 struct mlxsw_sp_nexthop *nh = &nhgi->nexthops[i]; 3689 int upper_bound; 3690 3691 if (!nh->should_offload) 3692 continue; 3693 weight += nh->norm_nh_weight; 3694 upper_bound = DIV_ROUND_CLOSEST(ecmp_size * weight, total); 3695 nh->num_adj_entries = upper_bound - lower_bound; 3696 lower_bound = upper_bound; 3697 } 3698 } 3699 3700 static struct mlxsw_sp_nexthop * 3701 mlxsw_sp_rt6_nexthop(struct mlxsw_sp_nexthop_group *nh_grp, 3702 const struct mlxsw_sp_rt6 *mlxsw_sp_rt6); 3703 3704 static void 3705 mlxsw_sp_nexthop4_group_offload_refresh(struct mlxsw_sp *mlxsw_sp, 3706 struct mlxsw_sp_nexthop_group *nh_grp) 3707 { 3708 int i; 3709 3710 for (i = 0; i < nh_grp->nhgi->count; i++) { 3711 struct mlxsw_sp_nexthop *nh = &nh_grp->nhgi->nexthops[i]; 3712 3713 if (nh->offloaded) 3714 nh->key.fib_nh->fib_nh_flags |= RTNH_F_OFFLOAD; 3715 else 3716 nh->key.fib_nh->fib_nh_flags &= ~RTNH_F_OFFLOAD; 3717 } 3718 } 3719 3720 static void 3721 __mlxsw_sp_nexthop6_group_offload_refresh(struct mlxsw_sp_nexthop_group *nh_grp, 3722 struct mlxsw_sp_fib6_entry *fib6_entry) 3723 { 3724 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 3725 3726 list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) { 3727 struct fib6_nh *fib6_nh = mlxsw_sp_rt6->rt->fib6_nh; 3728 struct mlxsw_sp_nexthop *nh; 3729 3730 nh = mlxsw_sp_rt6_nexthop(nh_grp, mlxsw_sp_rt6); 3731 if (nh && nh->offloaded) 3732 fib6_nh->fib_nh_flags |= RTNH_F_OFFLOAD; 3733 else 3734 fib6_nh->fib_nh_flags &= ~RTNH_F_OFFLOAD; 3735 } 3736 } 3737 3738 static void 3739 mlxsw_sp_nexthop6_group_offload_refresh(struct mlxsw_sp *mlxsw_sp, 3740 struct mlxsw_sp_nexthop_group *nh_grp) 3741 { 3742 struct mlxsw_sp_fib6_entry *fib6_entry; 3743 3744 /* Unfortunately, in IPv6 the route and the nexthop are described by 3745 * the same struct, so we need to iterate over all the routes using the 3746 * nexthop group and set / clear the offload indication for them. 3747 */ 3748 list_for_each_entry(fib6_entry, &nh_grp->fib_list, 3749 common.nexthop_group_node) 3750 __mlxsw_sp_nexthop6_group_offload_refresh(nh_grp, fib6_entry); 3751 } 3752 3753 static void 3754 mlxsw_sp_nexthop_obj_group_offload_refresh(struct mlxsw_sp *mlxsw_sp, 3755 struct mlxsw_sp_nexthop_group *nh_grp) 3756 { 3757 /* Do not update the flags if the nexthop group is being destroyed 3758 * since: 3759 * 1. The nexthop objects is being deleted, in which case the flags are 3760 * irrelevant. 3761 * 2. The nexthop group was replaced by a newer group, in which case 3762 * the flags of the nexthop object were already updated based on the 3763 * new group. 3764 */ 3765 if (nh_grp->can_destroy) 3766 return; 3767 3768 nexthop_set_hw_flags(mlxsw_sp_net(mlxsw_sp), nh_grp->obj.id, 3769 nh_grp->nhgi->adj_index_valid, false); 3770 } 3771 3772 static void 3773 mlxsw_sp_nexthop_group_offload_refresh(struct mlxsw_sp *mlxsw_sp, 3774 struct mlxsw_sp_nexthop_group *nh_grp) 3775 { 3776 switch (nh_grp->type) { 3777 case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4: 3778 mlxsw_sp_nexthop4_group_offload_refresh(mlxsw_sp, nh_grp); 3779 break; 3780 case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6: 3781 mlxsw_sp_nexthop6_group_offload_refresh(mlxsw_sp, nh_grp); 3782 break; 3783 case MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ: 3784 mlxsw_sp_nexthop_obj_group_offload_refresh(mlxsw_sp, nh_grp); 3785 break; 3786 } 3787 } 3788 3789 static int 3790 mlxsw_sp_nexthop_group_refresh(struct mlxsw_sp *mlxsw_sp, 3791 struct mlxsw_sp_nexthop_group *nh_grp) 3792 { 3793 struct mlxsw_sp_nexthop_group_info *nhgi = nh_grp->nhgi; 3794 u16 ecmp_size, old_ecmp_size; 3795 struct mlxsw_sp_nexthop *nh; 3796 bool offload_change = false; 3797 u32 adj_index; 3798 bool old_adj_index_valid; 3799 int i, err2, err = 0; 3800 u32 old_adj_index; 3801 3802 if (!nhgi->gateway) 3803 return mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp); 3804 3805 for (i = 0; i < nhgi->count; i++) { 3806 nh = &nhgi->nexthops[i]; 3807 3808 if (nh->should_offload != nh->offloaded) { 3809 offload_change = true; 3810 if (nh->should_offload) 3811 nh->update = 1; 3812 } 3813 } 3814 if (!offload_change) { 3815 /* Nothing was added or removed, so no need to reallocate. Just 3816 * update MAC on existing adjacency indexes. 3817 */ 3818 err = mlxsw_sp_nexthop_group_update(mlxsw_sp, nhgi, false); 3819 if (err) { 3820 dev_warn(mlxsw_sp->bus_info->dev, "Failed to update neigh MAC in adjacency table.\n"); 3821 goto set_trap; 3822 } 3823 return 0; 3824 } 3825 mlxsw_sp_nexthop_group_normalize(nhgi); 3826 if (!nhgi->sum_norm_weight) 3827 /* No neigh of this group is connected so we just set 3828 * the trap and let everthing flow through kernel. 3829 */ 3830 goto set_trap; 3831 3832 ecmp_size = nhgi->sum_norm_weight; 3833 err = mlxsw_sp_fix_adj_grp_size(mlxsw_sp, &ecmp_size); 3834 if (err) 3835 /* No valid allocation size available. */ 3836 goto set_trap; 3837 3838 err = mlxsw_sp_kvdl_alloc(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 3839 ecmp_size, &adj_index); 3840 if (err) { 3841 /* We ran out of KVD linear space, just set the 3842 * trap and let everything flow through kernel. 3843 */ 3844 dev_warn(mlxsw_sp->bus_info->dev, "Failed to allocate KVD linear area for nexthop group.\n"); 3845 goto set_trap; 3846 } 3847 old_adj_index_valid = nhgi->adj_index_valid; 3848 old_adj_index = nhgi->adj_index; 3849 old_ecmp_size = nhgi->ecmp_size; 3850 nhgi->adj_index_valid = 1; 3851 nhgi->adj_index = adj_index; 3852 nhgi->ecmp_size = ecmp_size; 3853 mlxsw_sp_nexthop_group_rebalance(nhgi); 3854 err = mlxsw_sp_nexthop_group_update(mlxsw_sp, nhgi, true); 3855 if (err) { 3856 dev_warn(mlxsw_sp->bus_info->dev, "Failed to update neigh MAC in adjacency table.\n"); 3857 goto set_trap; 3858 } 3859 3860 mlxsw_sp_nexthop_group_offload_refresh(mlxsw_sp, nh_grp); 3861 3862 if (!old_adj_index_valid) { 3863 /* The trap was set for fib entries, so we have to call 3864 * fib entry update to unset it and use adjacency index. 3865 */ 3866 err = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp); 3867 if (err) { 3868 dev_warn(mlxsw_sp->bus_info->dev, "Failed to add adjacency index to fib entries.\n"); 3869 goto set_trap; 3870 } 3871 return 0; 3872 } 3873 3874 err = mlxsw_sp_adj_index_mass_update(mlxsw_sp, nh_grp, 3875 old_adj_index, old_ecmp_size); 3876 mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 3877 old_ecmp_size, old_adj_index); 3878 if (err) { 3879 dev_warn(mlxsw_sp->bus_info->dev, "Failed to mass-update adjacency index for nexthop group.\n"); 3880 goto set_trap; 3881 } 3882 3883 return 0; 3884 3885 set_trap: 3886 old_adj_index_valid = nhgi->adj_index_valid; 3887 nhgi->adj_index_valid = 0; 3888 for (i = 0; i < nhgi->count; i++) { 3889 nh = &nhgi->nexthops[i]; 3890 nh->offloaded = 0; 3891 } 3892 err2 = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp); 3893 if (err2) 3894 dev_warn(mlxsw_sp->bus_info->dev, "Failed to set traps for fib entries.\n"); 3895 mlxsw_sp_nexthop_group_offload_refresh(mlxsw_sp, nh_grp); 3896 if (old_adj_index_valid) 3897 mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 3898 nhgi->ecmp_size, nhgi->adj_index); 3899 return err; 3900 } 3901 3902 static void __mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp_nexthop *nh, 3903 bool removing) 3904 { 3905 if (!removing) { 3906 nh->action = MLXSW_SP_NEXTHOP_ACTION_FORWARD; 3907 nh->should_offload = 1; 3908 } else { 3909 nh->should_offload = 0; 3910 } 3911 nh->update = 1; 3912 } 3913 3914 static int 3915 mlxsw_sp_nexthop_dead_neigh_replace(struct mlxsw_sp *mlxsw_sp, 3916 struct mlxsw_sp_neigh_entry *neigh_entry) 3917 { 3918 struct neighbour *n, *old_n = neigh_entry->key.n; 3919 struct mlxsw_sp_nexthop *nh; 3920 bool entry_connected; 3921 u8 nud_state, dead; 3922 int err; 3923 3924 nh = list_first_entry(&neigh_entry->nexthop_list, 3925 struct mlxsw_sp_nexthop, neigh_list_node); 3926 3927 n = neigh_lookup(nh->neigh_tbl, &nh->gw_addr, nh->rif->dev); 3928 if (!n) { 3929 n = neigh_create(nh->neigh_tbl, &nh->gw_addr, nh->rif->dev); 3930 if (IS_ERR(n)) 3931 return PTR_ERR(n); 3932 neigh_event_send(n, NULL); 3933 } 3934 3935 mlxsw_sp_neigh_entry_remove(mlxsw_sp, neigh_entry); 3936 neigh_entry->key.n = n; 3937 err = mlxsw_sp_neigh_entry_insert(mlxsw_sp, neigh_entry); 3938 if (err) 3939 goto err_neigh_entry_insert; 3940 3941 read_lock_bh(&n->lock); 3942 nud_state = n->nud_state; 3943 dead = n->dead; 3944 read_unlock_bh(&n->lock); 3945 entry_connected = nud_state & NUD_VALID && !dead; 3946 3947 list_for_each_entry(nh, &neigh_entry->nexthop_list, 3948 neigh_list_node) { 3949 neigh_release(old_n); 3950 neigh_clone(n); 3951 __mlxsw_sp_nexthop_neigh_update(nh, !entry_connected); 3952 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nhgi->nh_grp); 3953 } 3954 3955 neigh_release(n); 3956 3957 return 0; 3958 3959 err_neigh_entry_insert: 3960 neigh_entry->key.n = old_n; 3961 mlxsw_sp_neigh_entry_insert(mlxsw_sp, neigh_entry); 3962 neigh_release(n); 3963 return err; 3964 } 3965 3966 static void 3967 mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp *mlxsw_sp, 3968 struct mlxsw_sp_neigh_entry *neigh_entry, 3969 bool removing, bool dead) 3970 { 3971 struct mlxsw_sp_nexthop *nh; 3972 3973 if (list_empty(&neigh_entry->nexthop_list)) 3974 return; 3975 3976 if (dead) { 3977 int err; 3978 3979 err = mlxsw_sp_nexthop_dead_neigh_replace(mlxsw_sp, 3980 neigh_entry); 3981 if (err) 3982 dev_err(mlxsw_sp->bus_info->dev, "Failed to replace dead neigh\n"); 3983 return; 3984 } 3985 3986 list_for_each_entry(nh, &neigh_entry->nexthop_list, 3987 neigh_list_node) { 3988 __mlxsw_sp_nexthop_neigh_update(nh, removing); 3989 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nhgi->nh_grp); 3990 } 3991 } 3992 3993 static void mlxsw_sp_nexthop_rif_init(struct mlxsw_sp_nexthop *nh, 3994 struct mlxsw_sp_rif *rif) 3995 { 3996 if (nh->rif) 3997 return; 3998 3999 nh->rif = rif; 4000 list_add(&nh->rif_list_node, &rif->nexthop_list); 4001 } 4002 4003 static void mlxsw_sp_nexthop_rif_fini(struct mlxsw_sp_nexthop *nh) 4004 { 4005 if (!nh->rif) 4006 return; 4007 4008 list_del(&nh->rif_list_node); 4009 nh->rif = NULL; 4010 } 4011 4012 static int mlxsw_sp_nexthop_neigh_init(struct mlxsw_sp *mlxsw_sp, 4013 struct mlxsw_sp_nexthop *nh) 4014 { 4015 struct mlxsw_sp_neigh_entry *neigh_entry; 4016 struct neighbour *n; 4017 u8 nud_state, dead; 4018 int err; 4019 4020 if (!nh->nhgi->gateway || nh->neigh_entry) 4021 return 0; 4022 4023 /* Take a reference of neigh here ensuring that neigh would 4024 * not be destructed before the nexthop entry is finished. 4025 * The reference is taken either in neigh_lookup() or 4026 * in neigh_create() in case n is not found. 4027 */ 4028 n = neigh_lookup(nh->neigh_tbl, &nh->gw_addr, nh->rif->dev); 4029 if (!n) { 4030 n = neigh_create(nh->neigh_tbl, &nh->gw_addr, nh->rif->dev); 4031 if (IS_ERR(n)) 4032 return PTR_ERR(n); 4033 neigh_event_send(n, NULL); 4034 } 4035 neigh_entry = mlxsw_sp_neigh_entry_lookup(mlxsw_sp, n); 4036 if (!neigh_entry) { 4037 neigh_entry = mlxsw_sp_neigh_entry_create(mlxsw_sp, n); 4038 if (IS_ERR(neigh_entry)) { 4039 err = -EINVAL; 4040 goto err_neigh_entry_create; 4041 } 4042 } 4043 4044 /* If that is the first nexthop connected to that neigh, add to 4045 * nexthop_neighs_list 4046 */ 4047 if (list_empty(&neigh_entry->nexthop_list)) 4048 list_add_tail(&neigh_entry->nexthop_neighs_list_node, 4049 &mlxsw_sp->router->nexthop_neighs_list); 4050 4051 nh->neigh_entry = neigh_entry; 4052 list_add_tail(&nh->neigh_list_node, &neigh_entry->nexthop_list); 4053 read_lock_bh(&n->lock); 4054 nud_state = n->nud_state; 4055 dead = n->dead; 4056 read_unlock_bh(&n->lock); 4057 __mlxsw_sp_nexthop_neigh_update(nh, !(nud_state & NUD_VALID && !dead)); 4058 4059 return 0; 4060 4061 err_neigh_entry_create: 4062 neigh_release(n); 4063 return err; 4064 } 4065 4066 static void mlxsw_sp_nexthop_neigh_fini(struct mlxsw_sp *mlxsw_sp, 4067 struct mlxsw_sp_nexthop *nh) 4068 { 4069 struct mlxsw_sp_neigh_entry *neigh_entry = nh->neigh_entry; 4070 struct neighbour *n; 4071 4072 if (!neigh_entry) 4073 return; 4074 n = neigh_entry->key.n; 4075 4076 __mlxsw_sp_nexthop_neigh_update(nh, true); 4077 list_del(&nh->neigh_list_node); 4078 nh->neigh_entry = NULL; 4079 4080 /* If that is the last nexthop connected to that neigh, remove from 4081 * nexthop_neighs_list 4082 */ 4083 if (list_empty(&neigh_entry->nexthop_list)) 4084 list_del(&neigh_entry->nexthop_neighs_list_node); 4085 4086 if (!neigh_entry->connected && list_empty(&neigh_entry->nexthop_list)) 4087 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry); 4088 4089 neigh_release(n); 4090 } 4091 4092 static bool mlxsw_sp_ipip_netdev_ul_up(struct net_device *ol_dev) 4093 { 4094 struct net_device *ul_dev; 4095 bool is_up; 4096 4097 rcu_read_lock(); 4098 ul_dev = __mlxsw_sp_ipip_netdev_ul_dev_get(ol_dev); 4099 is_up = ul_dev ? (ul_dev->flags & IFF_UP) : true; 4100 rcu_read_unlock(); 4101 4102 return is_up; 4103 } 4104 4105 static void mlxsw_sp_nexthop_ipip_init(struct mlxsw_sp *mlxsw_sp, 4106 struct mlxsw_sp_nexthop *nh, 4107 struct mlxsw_sp_ipip_entry *ipip_entry) 4108 { 4109 bool removing; 4110 4111 if (!nh->nhgi->gateway || nh->ipip_entry) 4112 return; 4113 4114 nh->ipip_entry = ipip_entry; 4115 removing = !mlxsw_sp_ipip_netdev_ul_up(ipip_entry->ol_dev); 4116 __mlxsw_sp_nexthop_neigh_update(nh, removing); 4117 mlxsw_sp_nexthop_rif_init(nh, &ipip_entry->ol_lb->common); 4118 } 4119 4120 static void mlxsw_sp_nexthop_ipip_fini(struct mlxsw_sp *mlxsw_sp, 4121 struct mlxsw_sp_nexthop *nh) 4122 { 4123 struct mlxsw_sp_ipip_entry *ipip_entry = nh->ipip_entry; 4124 4125 if (!ipip_entry) 4126 return; 4127 4128 __mlxsw_sp_nexthop_neigh_update(nh, true); 4129 nh->ipip_entry = NULL; 4130 } 4131 4132 static bool mlxsw_sp_nexthop4_ipip_type(const struct mlxsw_sp *mlxsw_sp, 4133 const struct fib_nh *fib_nh, 4134 enum mlxsw_sp_ipip_type *p_ipipt) 4135 { 4136 struct net_device *dev = fib_nh->fib_nh_dev; 4137 4138 return dev && 4139 fib_nh->nh_parent->fib_type == RTN_UNICAST && 4140 mlxsw_sp_netdev_ipip_type(mlxsw_sp, dev, p_ipipt); 4141 } 4142 4143 static int mlxsw_sp_nexthop_type_init(struct mlxsw_sp *mlxsw_sp, 4144 struct mlxsw_sp_nexthop *nh, 4145 const struct net_device *dev) 4146 { 4147 const struct mlxsw_sp_ipip_ops *ipip_ops; 4148 struct mlxsw_sp_ipip_entry *ipip_entry; 4149 struct mlxsw_sp_rif *rif; 4150 int err; 4151 4152 ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, dev); 4153 if (ipip_entry) { 4154 ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt]; 4155 if (ipip_ops->can_offload(mlxsw_sp, dev)) { 4156 nh->type = MLXSW_SP_NEXTHOP_TYPE_IPIP; 4157 mlxsw_sp_nexthop_ipip_init(mlxsw_sp, nh, ipip_entry); 4158 return 0; 4159 } 4160 } 4161 4162 nh->type = MLXSW_SP_NEXTHOP_TYPE_ETH; 4163 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev); 4164 if (!rif) 4165 return 0; 4166 4167 mlxsw_sp_nexthop_rif_init(nh, rif); 4168 err = mlxsw_sp_nexthop_neigh_init(mlxsw_sp, nh); 4169 if (err) 4170 goto err_neigh_init; 4171 4172 return 0; 4173 4174 err_neigh_init: 4175 mlxsw_sp_nexthop_rif_fini(nh); 4176 return err; 4177 } 4178 4179 static void mlxsw_sp_nexthop_type_fini(struct mlxsw_sp *mlxsw_sp, 4180 struct mlxsw_sp_nexthop *nh) 4181 { 4182 switch (nh->type) { 4183 case MLXSW_SP_NEXTHOP_TYPE_ETH: 4184 mlxsw_sp_nexthop_neigh_fini(mlxsw_sp, nh); 4185 mlxsw_sp_nexthop_rif_fini(nh); 4186 break; 4187 case MLXSW_SP_NEXTHOP_TYPE_IPIP: 4188 mlxsw_sp_nexthop_rif_fini(nh); 4189 mlxsw_sp_nexthop_ipip_fini(mlxsw_sp, nh); 4190 break; 4191 } 4192 } 4193 4194 static int mlxsw_sp_nexthop4_init(struct mlxsw_sp *mlxsw_sp, 4195 struct mlxsw_sp_nexthop_group *nh_grp, 4196 struct mlxsw_sp_nexthop *nh, 4197 struct fib_nh *fib_nh) 4198 { 4199 struct net_device *dev = fib_nh->fib_nh_dev; 4200 struct in_device *in_dev; 4201 int err; 4202 4203 nh->nhgi = nh_grp->nhgi; 4204 nh->key.fib_nh = fib_nh; 4205 #ifdef CONFIG_IP_ROUTE_MULTIPATH 4206 nh->nh_weight = fib_nh->fib_nh_weight; 4207 #else 4208 nh->nh_weight = 1; 4209 #endif 4210 memcpy(&nh->gw_addr, &fib_nh->fib_nh_gw4, sizeof(fib_nh->fib_nh_gw4)); 4211 nh->neigh_tbl = &arp_tbl; 4212 err = mlxsw_sp_nexthop_insert(mlxsw_sp, nh); 4213 if (err) 4214 return err; 4215 4216 mlxsw_sp_nexthop_counter_alloc(mlxsw_sp, nh); 4217 list_add_tail(&nh->router_list_node, &mlxsw_sp->router->nexthop_list); 4218 4219 if (!dev) 4220 return 0; 4221 nh->ifindex = dev->ifindex; 4222 4223 rcu_read_lock(); 4224 in_dev = __in_dev_get_rcu(dev); 4225 if (in_dev && IN_DEV_IGNORE_ROUTES_WITH_LINKDOWN(in_dev) && 4226 fib_nh->fib_nh_flags & RTNH_F_LINKDOWN) { 4227 rcu_read_unlock(); 4228 return 0; 4229 } 4230 rcu_read_unlock(); 4231 4232 err = mlxsw_sp_nexthop_type_init(mlxsw_sp, nh, dev); 4233 if (err) 4234 goto err_nexthop_neigh_init; 4235 4236 return 0; 4237 4238 err_nexthop_neigh_init: 4239 mlxsw_sp_nexthop_remove(mlxsw_sp, nh); 4240 return err; 4241 } 4242 4243 static void mlxsw_sp_nexthop4_fini(struct mlxsw_sp *mlxsw_sp, 4244 struct mlxsw_sp_nexthop *nh) 4245 { 4246 mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh); 4247 list_del(&nh->router_list_node); 4248 mlxsw_sp_nexthop_counter_free(mlxsw_sp, nh); 4249 mlxsw_sp_nexthop_remove(mlxsw_sp, nh); 4250 } 4251 4252 static void mlxsw_sp_nexthop4_event(struct mlxsw_sp *mlxsw_sp, 4253 unsigned long event, struct fib_nh *fib_nh) 4254 { 4255 struct mlxsw_sp_nexthop_key key; 4256 struct mlxsw_sp_nexthop *nh; 4257 4258 if (mlxsw_sp->router->aborted) 4259 return; 4260 4261 key.fib_nh = fib_nh; 4262 nh = mlxsw_sp_nexthop_lookup(mlxsw_sp, key); 4263 if (!nh) 4264 return; 4265 4266 switch (event) { 4267 case FIB_EVENT_NH_ADD: 4268 mlxsw_sp_nexthop_type_init(mlxsw_sp, nh, fib_nh->fib_nh_dev); 4269 break; 4270 case FIB_EVENT_NH_DEL: 4271 mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh); 4272 break; 4273 } 4274 4275 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nhgi->nh_grp); 4276 } 4277 4278 static void mlxsw_sp_nexthop_rif_update(struct mlxsw_sp *mlxsw_sp, 4279 struct mlxsw_sp_rif *rif) 4280 { 4281 struct mlxsw_sp_nexthop *nh; 4282 bool removing; 4283 4284 list_for_each_entry(nh, &rif->nexthop_list, rif_list_node) { 4285 switch (nh->type) { 4286 case MLXSW_SP_NEXTHOP_TYPE_ETH: 4287 removing = false; 4288 break; 4289 case MLXSW_SP_NEXTHOP_TYPE_IPIP: 4290 removing = !mlxsw_sp_ipip_netdev_ul_up(rif->dev); 4291 break; 4292 default: 4293 WARN_ON(1); 4294 continue; 4295 } 4296 4297 __mlxsw_sp_nexthop_neigh_update(nh, removing); 4298 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nhgi->nh_grp); 4299 } 4300 } 4301 4302 static void mlxsw_sp_nexthop_rif_migrate(struct mlxsw_sp *mlxsw_sp, 4303 struct mlxsw_sp_rif *old_rif, 4304 struct mlxsw_sp_rif *new_rif) 4305 { 4306 struct mlxsw_sp_nexthop *nh; 4307 4308 list_splice_init(&old_rif->nexthop_list, &new_rif->nexthop_list); 4309 list_for_each_entry(nh, &new_rif->nexthop_list, rif_list_node) 4310 nh->rif = new_rif; 4311 mlxsw_sp_nexthop_rif_update(mlxsw_sp, new_rif); 4312 } 4313 4314 static void mlxsw_sp_nexthop_rif_gone_sync(struct mlxsw_sp *mlxsw_sp, 4315 struct mlxsw_sp_rif *rif) 4316 { 4317 struct mlxsw_sp_nexthop *nh, *tmp; 4318 4319 list_for_each_entry_safe(nh, tmp, &rif->nexthop_list, rif_list_node) { 4320 mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh); 4321 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nhgi->nh_grp); 4322 } 4323 } 4324 4325 static int 4326 mlxsw_sp_nexthop_obj_single_validate(struct mlxsw_sp *mlxsw_sp, 4327 const struct nh_notifier_single_info *nh, 4328 struct netlink_ext_ack *extack) 4329 { 4330 int err = -EINVAL; 4331 4332 if (nh->is_fdb) 4333 NL_SET_ERR_MSG_MOD(extack, "FDB nexthops are not supported"); 4334 else if (nh->has_encap) 4335 NL_SET_ERR_MSG_MOD(extack, "Encapsulating nexthops are not supported"); 4336 else 4337 err = 0; 4338 4339 return err; 4340 } 4341 4342 static int 4343 mlxsw_sp_nexthop_obj_group_entry_validate(struct mlxsw_sp *mlxsw_sp, 4344 const struct nh_notifier_single_info *nh, 4345 struct netlink_ext_ack *extack) 4346 { 4347 int err; 4348 4349 err = mlxsw_sp_nexthop_obj_single_validate(mlxsw_sp, nh, extack); 4350 if (err) 4351 return err; 4352 4353 /* Device only nexthops with an IPIP device are programmed as 4354 * encapsulating adjacency entries. 4355 */ 4356 if (!nh->gw_family && !nh->is_reject && 4357 !mlxsw_sp_netdev_ipip_type(mlxsw_sp, nh->dev, NULL)) { 4358 NL_SET_ERR_MSG_MOD(extack, "Nexthop group entry does not have a gateway"); 4359 return -EINVAL; 4360 } 4361 4362 return 0; 4363 } 4364 4365 static int 4366 mlxsw_sp_nexthop_obj_group_validate(struct mlxsw_sp *mlxsw_sp, 4367 const struct nh_notifier_grp_info *nh_grp, 4368 struct netlink_ext_ack *extack) 4369 { 4370 int i; 4371 4372 if (nh_grp->is_fdb) { 4373 NL_SET_ERR_MSG_MOD(extack, "FDB nexthop groups are not supported"); 4374 return -EINVAL; 4375 } 4376 4377 for (i = 0; i < nh_grp->num_nh; i++) { 4378 const struct nh_notifier_single_info *nh; 4379 int err; 4380 4381 nh = &nh_grp->nh_entries[i].nh; 4382 err = mlxsw_sp_nexthop_obj_group_entry_validate(mlxsw_sp, nh, 4383 extack); 4384 if (err) 4385 return err; 4386 } 4387 4388 return 0; 4389 } 4390 4391 static int mlxsw_sp_nexthop_obj_validate(struct mlxsw_sp *mlxsw_sp, 4392 unsigned long event, 4393 struct nh_notifier_info *info) 4394 { 4395 if (event != NEXTHOP_EVENT_REPLACE) 4396 return 0; 4397 4398 switch (info->type) { 4399 case NH_NOTIFIER_INFO_TYPE_SINGLE: 4400 return mlxsw_sp_nexthop_obj_single_validate(mlxsw_sp, info->nh, 4401 info->extack); 4402 case NH_NOTIFIER_INFO_TYPE_GRP: 4403 return mlxsw_sp_nexthop_obj_group_validate(mlxsw_sp, 4404 info->nh_grp, 4405 info->extack); 4406 default: 4407 NL_SET_ERR_MSG_MOD(info->extack, "Unsupported nexthop type"); 4408 return -EOPNOTSUPP; 4409 } 4410 } 4411 4412 static bool mlxsw_sp_nexthop_obj_is_gateway(struct mlxsw_sp *mlxsw_sp, 4413 const struct nh_notifier_info *info) 4414 { 4415 const struct net_device *dev; 4416 4417 switch (info->type) { 4418 case NH_NOTIFIER_INFO_TYPE_SINGLE: 4419 dev = info->nh->dev; 4420 return info->nh->gw_family || info->nh->is_reject || 4421 mlxsw_sp_netdev_ipip_type(mlxsw_sp, dev, NULL); 4422 case NH_NOTIFIER_INFO_TYPE_GRP: 4423 /* Already validated earlier. */ 4424 return true; 4425 default: 4426 return false; 4427 } 4428 } 4429 4430 static void mlxsw_sp_nexthop_obj_blackhole_init(struct mlxsw_sp *mlxsw_sp, 4431 struct mlxsw_sp_nexthop *nh) 4432 { 4433 u16 lb_rif_index = mlxsw_sp->router->lb_rif_index; 4434 4435 nh->action = MLXSW_SP_NEXTHOP_ACTION_DISCARD; 4436 nh->should_offload = 1; 4437 /* While nexthops that discard packets do not forward packets 4438 * via an egress RIF, they still need to be programmed using a 4439 * valid RIF, so use the loopback RIF created during init. 4440 */ 4441 nh->rif = mlxsw_sp->router->rifs[lb_rif_index]; 4442 } 4443 4444 static void mlxsw_sp_nexthop_obj_blackhole_fini(struct mlxsw_sp *mlxsw_sp, 4445 struct mlxsw_sp_nexthop *nh) 4446 { 4447 nh->rif = NULL; 4448 nh->should_offload = 0; 4449 } 4450 4451 static int 4452 mlxsw_sp_nexthop_obj_init(struct mlxsw_sp *mlxsw_sp, 4453 struct mlxsw_sp_nexthop_group *nh_grp, 4454 struct mlxsw_sp_nexthop *nh, 4455 struct nh_notifier_single_info *nh_obj, int weight) 4456 { 4457 struct net_device *dev = nh_obj->dev; 4458 int err; 4459 4460 nh->nhgi = nh_grp->nhgi; 4461 nh->nh_weight = weight; 4462 4463 switch (nh_obj->gw_family) { 4464 case AF_INET: 4465 memcpy(&nh->gw_addr, &nh_obj->ipv4, sizeof(nh_obj->ipv4)); 4466 nh->neigh_tbl = &arp_tbl; 4467 break; 4468 case AF_INET6: 4469 memcpy(&nh->gw_addr, &nh_obj->ipv6, sizeof(nh_obj->ipv6)); 4470 #if IS_ENABLED(CONFIG_IPV6) 4471 nh->neigh_tbl = &nd_tbl; 4472 #endif 4473 break; 4474 } 4475 4476 mlxsw_sp_nexthop_counter_alloc(mlxsw_sp, nh); 4477 list_add_tail(&nh->router_list_node, &mlxsw_sp->router->nexthop_list); 4478 nh->ifindex = dev->ifindex; 4479 4480 err = mlxsw_sp_nexthop_type_init(mlxsw_sp, nh, dev); 4481 if (err) 4482 goto err_type_init; 4483 4484 if (nh_obj->is_reject) 4485 mlxsw_sp_nexthop_obj_blackhole_init(mlxsw_sp, nh); 4486 4487 return 0; 4488 4489 err_type_init: 4490 list_del(&nh->router_list_node); 4491 mlxsw_sp_nexthop_counter_free(mlxsw_sp, nh); 4492 return err; 4493 } 4494 4495 static void mlxsw_sp_nexthop_obj_fini(struct mlxsw_sp *mlxsw_sp, 4496 struct mlxsw_sp_nexthop *nh) 4497 { 4498 if (nh->action == MLXSW_SP_NEXTHOP_ACTION_DISCARD) 4499 mlxsw_sp_nexthop_obj_blackhole_fini(mlxsw_sp, nh); 4500 mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh); 4501 list_del(&nh->router_list_node); 4502 mlxsw_sp_nexthop_counter_free(mlxsw_sp, nh); 4503 } 4504 4505 static int 4506 mlxsw_sp_nexthop_obj_group_info_init(struct mlxsw_sp *mlxsw_sp, 4507 struct mlxsw_sp_nexthop_group *nh_grp, 4508 struct nh_notifier_info *info) 4509 { 4510 struct mlxsw_sp_nexthop_group_info *nhgi; 4511 struct mlxsw_sp_nexthop *nh; 4512 unsigned int nhs; 4513 int err, i; 4514 4515 switch (info->type) { 4516 case NH_NOTIFIER_INFO_TYPE_SINGLE: 4517 nhs = 1; 4518 break; 4519 case NH_NOTIFIER_INFO_TYPE_GRP: 4520 nhs = info->nh_grp->num_nh; 4521 break; 4522 default: 4523 return -EINVAL; 4524 } 4525 4526 nhgi = kzalloc(struct_size(nhgi, nexthops, nhs), GFP_KERNEL); 4527 if (!nhgi) 4528 return -ENOMEM; 4529 nh_grp->nhgi = nhgi; 4530 nhgi->nh_grp = nh_grp; 4531 nhgi->gateway = mlxsw_sp_nexthop_obj_is_gateway(mlxsw_sp, info); 4532 nhgi->count = nhs; 4533 for (i = 0; i < nhgi->count; i++) { 4534 struct nh_notifier_single_info *nh_obj; 4535 int weight; 4536 4537 nh = &nhgi->nexthops[i]; 4538 switch (info->type) { 4539 case NH_NOTIFIER_INFO_TYPE_SINGLE: 4540 nh_obj = info->nh; 4541 weight = 1; 4542 break; 4543 case NH_NOTIFIER_INFO_TYPE_GRP: 4544 nh_obj = &info->nh_grp->nh_entries[i].nh; 4545 weight = info->nh_grp->nh_entries[i].weight; 4546 break; 4547 default: 4548 err = -EINVAL; 4549 goto err_nexthop_obj_init; 4550 } 4551 err = mlxsw_sp_nexthop_obj_init(mlxsw_sp, nh_grp, nh, nh_obj, 4552 weight); 4553 if (err) 4554 goto err_nexthop_obj_init; 4555 } 4556 err = mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp); 4557 if (err) { 4558 NL_SET_ERR_MSG_MOD(info->extack, "Failed to write adjacency entries to the device"); 4559 goto err_group_refresh; 4560 } 4561 4562 return 0; 4563 4564 err_group_refresh: 4565 i = nhgi->count; 4566 err_nexthop_obj_init: 4567 for (i--; i >= 0; i--) { 4568 nh = &nhgi->nexthops[i]; 4569 mlxsw_sp_nexthop_obj_fini(mlxsw_sp, nh); 4570 } 4571 kfree(nhgi); 4572 return err; 4573 } 4574 4575 static void 4576 mlxsw_sp_nexthop_obj_group_info_fini(struct mlxsw_sp *mlxsw_sp, 4577 struct mlxsw_sp_nexthop_group *nh_grp) 4578 { 4579 struct mlxsw_sp_nexthop_group_info *nhgi = nh_grp->nhgi; 4580 int i; 4581 4582 for (i = nhgi->count - 1; i >= 0; i--) { 4583 struct mlxsw_sp_nexthop *nh = &nhgi->nexthops[i]; 4584 4585 mlxsw_sp_nexthop_obj_fini(mlxsw_sp, nh); 4586 } 4587 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp); 4588 WARN_ON_ONCE(nhgi->adj_index_valid); 4589 kfree(nhgi); 4590 } 4591 4592 static struct mlxsw_sp_nexthop_group * 4593 mlxsw_sp_nexthop_obj_group_create(struct mlxsw_sp *mlxsw_sp, 4594 struct nh_notifier_info *info) 4595 { 4596 struct mlxsw_sp_nexthop_group *nh_grp; 4597 int err; 4598 4599 nh_grp = kzalloc(sizeof(*nh_grp), GFP_KERNEL); 4600 if (!nh_grp) 4601 return ERR_PTR(-ENOMEM); 4602 INIT_LIST_HEAD(&nh_grp->vr_list); 4603 err = rhashtable_init(&nh_grp->vr_ht, 4604 &mlxsw_sp_nexthop_group_vr_ht_params); 4605 if (err) 4606 goto err_nexthop_group_vr_ht_init; 4607 INIT_LIST_HEAD(&nh_grp->fib_list); 4608 nh_grp->type = MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ; 4609 nh_grp->obj.id = info->id; 4610 4611 err = mlxsw_sp_nexthop_obj_group_info_init(mlxsw_sp, nh_grp, info); 4612 if (err) 4613 goto err_nexthop_group_info_init; 4614 4615 nh_grp->can_destroy = false; 4616 4617 return nh_grp; 4618 4619 err_nexthop_group_info_init: 4620 rhashtable_destroy(&nh_grp->vr_ht); 4621 err_nexthop_group_vr_ht_init: 4622 kfree(nh_grp); 4623 return ERR_PTR(err); 4624 } 4625 4626 static void 4627 mlxsw_sp_nexthop_obj_group_destroy(struct mlxsw_sp *mlxsw_sp, 4628 struct mlxsw_sp_nexthop_group *nh_grp) 4629 { 4630 if (!nh_grp->can_destroy) 4631 return; 4632 mlxsw_sp_nexthop_obj_group_info_fini(mlxsw_sp, nh_grp); 4633 WARN_ON_ONCE(!list_empty(&nh_grp->fib_list)); 4634 WARN_ON_ONCE(!list_empty(&nh_grp->vr_list)); 4635 rhashtable_destroy(&nh_grp->vr_ht); 4636 kfree(nh_grp); 4637 } 4638 4639 static struct mlxsw_sp_nexthop_group * 4640 mlxsw_sp_nexthop_obj_group_lookup(struct mlxsw_sp *mlxsw_sp, u32 id) 4641 { 4642 struct mlxsw_sp_nexthop_group_cmp_arg cmp_arg; 4643 4644 cmp_arg.type = MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ; 4645 cmp_arg.id = id; 4646 return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_group_ht, 4647 &cmp_arg, 4648 mlxsw_sp_nexthop_group_ht_params); 4649 } 4650 4651 static int mlxsw_sp_nexthop_obj_group_add(struct mlxsw_sp *mlxsw_sp, 4652 struct mlxsw_sp_nexthop_group *nh_grp) 4653 { 4654 return mlxsw_sp_nexthop_group_insert(mlxsw_sp, nh_grp); 4655 } 4656 4657 static int 4658 mlxsw_sp_nexthop_obj_group_replace(struct mlxsw_sp *mlxsw_sp, 4659 struct mlxsw_sp_nexthop_group *nh_grp, 4660 struct mlxsw_sp_nexthop_group *old_nh_grp, 4661 struct netlink_ext_ack *extack) 4662 { 4663 struct mlxsw_sp_nexthop_group_info *old_nhgi = old_nh_grp->nhgi; 4664 struct mlxsw_sp_nexthop_group_info *new_nhgi = nh_grp->nhgi; 4665 int err; 4666 4667 old_nh_grp->nhgi = new_nhgi; 4668 new_nhgi->nh_grp = old_nh_grp; 4669 nh_grp->nhgi = old_nhgi; 4670 old_nhgi->nh_grp = nh_grp; 4671 4672 if (old_nhgi->adj_index_valid && new_nhgi->adj_index_valid) { 4673 /* Both the old adjacency index and the new one are valid. 4674 * Routes are currently using the old one. Tell the device to 4675 * replace the old adjacency index with the new one. 4676 */ 4677 err = mlxsw_sp_adj_index_mass_update(mlxsw_sp, old_nh_grp, 4678 old_nhgi->adj_index, 4679 old_nhgi->ecmp_size); 4680 if (err) { 4681 NL_SET_ERR_MSG_MOD(extack, "Failed to replace old adjacency index with new one"); 4682 goto err_out; 4683 } 4684 } else if (old_nhgi->adj_index_valid && !new_nhgi->adj_index_valid) { 4685 /* The old adjacency index is valid, while the new one is not. 4686 * Iterate over all the routes using the group and change them 4687 * to trap packets to the CPU. 4688 */ 4689 err = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, old_nh_grp); 4690 if (err) { 4691 NL_SET_ERR_MSG_MOD(extack, "Failed to update routes to trap packets"); 4692 goto err_out; 4693 } 4694 } else if (!old_nhgi->adj_index_valid && new_nhgi->adj_index_valid) { 4695 /* The old adjacency index is invalid, while the new one is. 4696 * Iterate over all the routes using the group and change them 4697 * to forward packets using the new valid index. 4698 */ 4699 err = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, old_nh_grp); 4700 if (err) { 4701 NL_SET_ERR_MSG_MOD(extack, "Failed to update routes to forward packets"); 4702 goto err_out; 4703 } 4704 } 4705 4706 /* Make sure the flags are set / cleared based on the new nexthop group 4707 * information. 4708 */ 4709 mlxsw_sp_nexthop_obj_group_offload_refresh(mlxsw_sp, old_nh_grp); 4710 4711 /* At this point 'nh_grp' is just a shell that is not used by anyone 4712 * and its nexthop group info is the old info that was just replaced 4713 * with the new one. Remove it. 4714 */ 4715 nh_grp->can_destroy = true; 4716 mlxsw_sp_nexthop_obj_group_destroy(mlxsw_sp, nh_grp); 4717 4718 return 0; 4719 4720 err_out: 4721 old_nhgi->nh_grp = old_nh_grp; 4722 nh_grp->nhgi = new_nhgi; 4723 new_nhgi->nh_grp = nh_grp; 4724 old_nh_grp->nhgi = old_nhgi; 4725 return err; 4726 } 4727 4728 static int mlxsw_sp_nexthop_obj_new(struct mlxsw_sp *mlxsw_sp, 4729 struct nh_notifier_info *info) 4730 { 4731 struct mlxsw_sp_nexthop_group *nh_grp, *old_nh_grp; 4732 struct netlink_ext_ack *extack = info->extack; 4733 int err; 4734 4735 nh_grp = mlxsw_sp_nexthop_obj_group_create(mlxsw_sp, info); 4736 if (IS_ERR(nh_grp)) 4737 return PTR_ERR(nh_grp); 4738 4739 old_nh_grp = mlxsw_sp_nexthop_obj_group_lookup(mlxsw_sp, info->id); 4740 if (!old_nh_grp) 4741 err = mlxsw_sp_nexthop_obj_group_add(mlxsw_sp, nh_grp); 4742 else 4743 err = mlxsw_sp_nexthop_obj_group_replace(mlxsw_sp, nh_grp, 4744 old_nh_grp, extack); 4745 4746 if (err) { 4747 nh_grp->can_destroy = true; 4748 mlxsw_sp_nexthop_obj_group_destroy(mlxsw_sp, nh_grp); 4749 } 4750 4751 return err; 4752 } 4753 4754 static void mlxsw_sp_nexthop_obj_del(struct mlxsw_sp *mlxsw_sp, 4755 struct nh_notifier_info *info) 4756 { 4757 struct mlxsw_sp_nexthop_group *nh_grp; 4758 4759 nh_grp = mlxsw_sp_nexthop_obj_group_lookup(mlxsw_sp, info->id); 4760 if (!nh_grp) 4761 return; 4762 4763 nh_grp->can_destroy = true; 4764 mlxsw_sp_nexthop_group_remove(mlxsw_sp, nh_grp); 4765 4766 /* If the group still has routes using it, then defer the delete 4767 * operation until the last route using it is deleted. 4768 */ 4769 if (!list_empty(&nh_grp->fib_list)) 4770 return; 4771 mlxsw_sp_nexthop_obj_group_destroy(mlxsw_sp, nh_grp); 4772 } 4773 4774 static int mlxsw_sp_nexthop_obj_event(struct notifier_block *nb, 4775 unsigned long event, void *ptr) 4776 { 4777 struct nh_notifier_info *info = ptr; 4778 struct mlxsw_sp_router *router; 4779 int err = 0; 4780 4781 router = container_of(nb, struct mlxsw_sp_router, nexthop_nb); 4782 err = mlxsw_sp_nexthop_obj_validate(router->mlxsw_sp, event, info); 4783 if (err) 4784 goto out; 4785 4786 mutex_lock(&router->lock); 4787 4788 switch (event) { 4789 case NEXTHOP_EVENT_REPLACE: 4790 err = mlxsw_sp_nexthop_obj_new(router->mlxsw_sp, info); 4791 break; 4792 case NEXTHOP_EVENT_DEL: 4793 mlxsw_sp_nexthop_obj_del(router->mlxsw_sp, info); 4794 break; 4795 default: 4796 break; 4797 } 4798 4799 mutex_unlock(&router->lock); 4800 4801 out: 4802 return notifier_from_errno(err); 4803 } 4804 4805 static bool mlxsw_sp_fi_is_gateway(const struct mlxsw_sp *mlxsw_sp, 4806 struct fib_info *fi) 4807 { 4808 const struct fib_nh *nh = fib_info_nh(fi, 0); 4809 4810 return nh->fib_nh_scope == RT_SCOPE_LINK || 4811 mlxsw_sp_nexthop4_ipip_type(mlxsw_sp, nh, NULL); 4812 } 4813 4814 static int 4815 mlxsw_sp_nexthop4_group_info_init(struct mlxsw_sp *mlxsw_sp, 4816 struct mlxsw_sp_nexthop_group *nh_grp) 4817 { 4818 unsigned int nhs = fib_info_num_path(nh_grp->ipv4.fi); 4819 struct mlxsw_sp_nexthop_group_info *nhgi; 4820 struct mlxsw_sp_nexthop *nh; 4821 int err, i; 4822 4823 nhgi = kzalloc(struct_size(nhgi, nexthops, nhs), GFP_KERNEL); 4824 if (!nhgi) 4825 return -ENOMEM; 4826 nh_grp->nhgi = nhgi; 4827 nhgi->nh_grp = nh_grp; 4828 nhgi->gateway = mlxsw_sp_fi_is_gateway(mlxsw_sp, nh_grp->ipv4.fi); 4829 nhgi->count = nhs; 4830 for (i = 0; i < nhgi->count; i++) { 4831 struct fib_nh *fib_nh; 4832 4833 nh = &nhgi->nexthops[i]; 4834 fib_nh = fib_info_nh(nh_grp->ipv4.fi, i); 4835 err = mlxsw_sp_nexthop4_init(mlxsw_sp, nh_grp, nh, fib_nh); 4836 if (err) 4837 goto err_nexthop4_init; 4838 } 4839 err = mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp); 4840 if (err) 4841 goto err_group_refresh; 4842 4843 return 0; 4844 4845 err_group_refresh: 4846 i = nhgi->count; 4847 err_nexthop4_init: 4848 for (i--; i >= 0; i--) { 4849 nh = &nhgi->nexthops[i]; 4850 mlxsw_sp_nexthop4_fini(mlxsw_sp, nh); 4851 } 4852 kfree(nhgi); 4853 return err; 4854 } 4855 4856 static void 4857 mlxsw_sp_nexthop4_group_info_fini(struct mlxsw_sp *mlxsw_sp, 4858 struct mlxsw_sp_nexthop_group *nh_grp) 4859 { 4860 struct mlxsw_sp_nexthop_group_info *nhgi = nh_grp->nhgi; 4861 int i; 4862 4863 for (i = nhgi->count - 1; i >= 0; i--) { 4864 struct mlxsw_sp_nexthop *nh = &nhgi->nexthops[i]; 4865 4866 mlxsw_sp_nexthop4_fini(mlxsw_sp, nh); 4867 } 4868 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp); 4869 WARN_ON_ONCE(nhgi->adj_index_valid); 4870 kfree(nhgi); 4871 } 4872 4873 static struct mlxsw_sp_nexthop_group * 4874 mlxsw_sp_nexthop4_group_create(struct mlxsw_sp *mlxsw_sp, struct fib_info *fi) 4875 { 4876 struct mlxsw_sp_nexthop_group *nh_grp; 4877 int err; 4878 4879 nh_grp = kzalloc(sizeof(*nh_grp), GFP_KERNEL); 4880 if (!nh_grp) 4881 return ERR_PTR(-ENOMEM); 4882 INIT_LIST_HEAD(&nh_grp->vr_list); 4883 err = rhashtable_init(&nh_grp->vr_ht, 4884 &mlxsw_sp_nexthop_group_vr_ht_params); 4885 if (err) 4886 goto err_nexthop_group_vr_ht_init; 4887 INIT_LIST_HEAD(&nh_grp->fib_list); 4888 nh_grp->type = MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4; 4889 nh_grp->ipv4.fi = fi; 4890 fib_info_hold(fi); 4891 4892 err = mlxsw_sp_nexthop4_group_info_init(mlxsw_sp, nh_grp); 4893 if (err) 4894 goto err_nexthop_group_info_init; 4895 4896 err = mlxsw_sp_nexthop_group_insert(mlxsw_sp, nh_grp); 4897 if (err) 4898 goto err_nexthop_group_insert; 4899 4900 nh_grp->can_destroy = true; 4901 4902 return nh_grp; 4903 4904 err_nexthop_group_insert: 4905 mlxsw_sp_nexthop4_group_info_fini(mlxsw_sp, nh_grp); 4906 err_nexthop_group_info_init: 4907 fib_info_put(fi); 4908 rhashtable_destroy(&nh_grp->vr_ht); 4909 err_nexthop_group_vr_ht_init: 4910 kfree(nh_grp); 4911 return ERR_PTR(err); 4912 } 4913 4914 static void 4915 mlxsw_sp_nexthop4_group_destroy(struct mlxsw_sp *mlxsw_sp, 4916 struct mlxsw_sp_nexthop_group *nh_grp) 4917 { 4918 if (!nh_grp->can_destroy) 4919 return; 4920 mlxsw_sp_nexthop_group_remove(mlxsw_sp, nh_grp); 4921 mlxsw_sp_nexthop4_group_info_fini(mlxsw_sp, nh_grp); 4922 fib_info_put(nh_grp->ipv4.fi); 4923 WARN_ON_ONCE(!list_empty(&nh_grp->vr_list)); 4924 rhashtable_destroy(&nh_grp->vr_ht); 4925 kfree(nh_grp); 4926 } 4927 4928 static int mlxsw_sp_nexthop4_group_get(struct mlxsw_sp *mlxsw_sp, 4929 struct mlxsw_sp_fib_entry *fib_entry, 4930 struct fib_info *fi) 4931 { 4932 struct mlxsw_sp_nexthop_group *nh_grp; 4933 4934 if (fi->nh) { 4935 nh_grp = mlxsw_sp_nexthop_obj_group_lookup(mlxsw_sp, 4936 fi->nh->id); 4937 if (WARN_ON_ONCE(!nh_grp)) 4938 return -EINVAL; 4939 goto out; 4940 } 4941 4942 nh_grp = mlxsw_sp_nexthop4_group_lookup(mlxsw_sp, fi); 4943 if (!nh_grp) { 4944 nh_grp = mlxsw_sp_nexthop4_group_create(mlxsw_sp, fi); 4945 if (IS_ERR(nh_grp)) 4946 return PTR_ERR(nh_grp); 4947 } 4948 out: 4949 list_add_tail(&fib_entry->nexthop_group_node, &nh_grp->fib_list); 4950 fib_entry->nh_group = nh_grp; 4951 return 0; 4952 } 4953 4954 static void mlxsw_sp_nexthop4_group_put(struct mlxsw_sp *mlxsw_sp, 4955 struct mlxsw_sp_fib_entry *fib_entry) 4956 { 4957 struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group; 4958 4959 list_del(&fib_entry->nexthop_group_node); 4960 if (!list_empty(&nh_grp->fib_list)) 4961 return; 4962 4963 if (nh_grp->type == MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ) { 4964 mlxsw_sp_nexthop_obj_group_destroy(mlxsw_sp, nh_grp); 4965 return; 4966 } 4967 4968 mlxsw_sp_nexthop4_group_destroy(mlxsw_sp, nh_grp); 4969 } 4970 4971 static bool 4972 mlxsw_sp_fib4_entry_should_offload(const struct mlxsw_sp_fib_entry *fib_entry) 4973 { 4974 struct mlxsw_sp_fib4_entry *fib4_entry; 4975 4976 fib4_entry = container_of(fib_entry, struct mlxsw_sp_fib4_entry, 4977 common); 4978 return !fib4_entry->tos; 4979 } 4980 4981 static bool 4982 mlxsw_sp_fib_entry_should_offload(const struct mlxsw_sp_fib_entry *fib_entry) 4983 { 4984 struct mlxsw_sp_nexthop_group *nh_group = fib_entry->nh_group; 4985 4986 switch (fib_entry->fib_node->fib->proto) { 4987 case MLXSW_SP_L3_PROTO_IPV4: 4988 if (!mlxsw_sp_fib4_entry_should_offload(fib_entry)) 4989 return false; 4990 break; 4991 case MLXSW_SP_L3_PROTO_IPV6: 4992 break; 4993 } 4994 4995 switch (fib_entry->type) { 4996 case MLXSW_SP_FIB_ENTRY_TYPE_REMOTE: 4997 return !!nh_group->nhgi->adj_index_valid; 4998 case MLXSW_SP_FIB_ENTRY_TYPE_LOCAL: 4999 return !!nh_group->nhgi->nh_rif; 5000 case MLXSW_SP_FIB_ENTRY_TYPE_BLACKHOLE: 5001 case MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP: 5002 case MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP: 5003 return true; 5004 default: 5005 return false; 5006 } 5007 } 5008 5009 static struct mlxsw_sp_nexthop * 5010 mlxsw_sp_rt6_nexthop(struct mlxsw_sp_nexthop_group *nh_grp, 5011 const struct mlxsw_sp_rt6 *mlxsw_sp_rt6) 5012 { 5013 int i; 5014 5015 for (i = 0; i < nh_grp->nhgi->count; i++) { 5016 struct mlxsw_sp_nexthop *nh = &nh_grp->nhgi->nexthops[i]; 5017 struct fib6_info *rt = mlxsw_sp_rt6->rt; 5018 5019 if (nh->rif && nh->rif->dev == rt->fib6_nh->fib_nh_dev && 5020 ipv6_addr_equal((const struct in6_addr *) &nh->gw_addr, 5021 &rt->fib6_nh->fib_nh_gw6)) 5022 return nh; 5023 continue; 5024 } 5025 5026 return NULL; 5027 } 5028 5029 static void 5030 mlxsw_sp_fib4_offload_failed_flag_set(struct mlxsw_sp *mlxsw_sp, 5031 struct fib_entry_notifier_info *fen_info) 5032 { 5033 u32 *p_dst = (u32 *) &fen_info->dst; 5034 struct fib_rt_info fri; 5035 5036 fri.fi = fen_info->fi; 5037 fri.tb_id = fen_info->tb_id; 5038 fri.dst = cpu_to_be32(*p_dst); 5039 fri.dst_len = fen_info->dst_len; 5040 fri.tos = fen_info->tos; 5041 fri.type = fen_info->type; 5042 fri.offload = false; 5043 fri.trap = false; 5044 fri.offload_failed = true; 5045 fib_alias_hw_flags_set(mlxsw_sp_net(mlxsw_sp), &fri); 5046 } 5047 5048 static void 5049 mlxsw_sp_fib4_entry_hw_flags_set(struct mlxsw_sp *mlxsw_sp, 5050 struct mlxsw_sp_fib_entry *fib_entry) 5051 { 5052 u32 *p_dst = (u32 *) fib_entry->fib_node->key.addr; 5053 int dst_len = fib_entry->fib_node->key.prefix_len; 5054 struct mlxsw_sp_fib4_entry *fib4_entry; 5055 struct fib_rt_info fri; 5056 bool should_offload; 5057 5058 should_offload = mlxsw_sp_fib_entry_should_offload(fib_entry); 5059 fib4_entry = container_of(fib_entry, struct mlxsw_sp_fib4_entry, 5060 common); 5061 fri.fi = fib4_entry->fi; 5062 fri.tb_id = fib4_entry->tb_id; 5063 fri.dst = cpu_to_be32(*p_dst); 5064 fri.dst_len = dst_len; 5065 fri.tos = fib4_entry->tos; 5066 fri.type = fib4_entry->type; 5067 fri.offload = should_offload; 5068 fri.trap = !should_offload; 5069 fri.offload_failed = false; 5070 fib_alias_hw_flags_set(mlxsw_sp_net(mlxsw_sp), &fri); 5071 } 5072 5073 static void 5074 mlxsw_sp_fib4_entry_hw_flags_clear(struct mlxsw_sp *mlxsw_sp, 5075 struct mlxsw_sp_fib_entry *fib_entry) 5076 { 5077 u32 *p_dst = (u32 *) fib_entry->fib_node->key.addr; 5078 int dst_len = fib_entry->fib_node->key.prefix_len; 5079 struct mlxsw_sp_fib4_entry *fib4_entry; 5080 struct fib_rt_info fri; 5081 5082 fib4_entry = container_of(fib_entry, struct mlxsw_sp_fib4_entry, 5083 common); 5084 fri.fi = fib4_entry->fi; 5085 fri.tb_id = fib4_entry->tb_id; 5086 fri.dst = cpu_to_be32(*p_dst); 5087 fri.dst_len = dst_len; 5088 fri.tos = fib4_entry->tos; 5089 fri.type = fib4_entry->type; 5090 fri.offload = false; 5091 fri.trap = false; 5092 fri.offload_failed = false; 5093 fib_alias_hw_flags_set(mlxsw_sp_net(mlxsw_sp), &fri); 5094 } 5095 5096 #if IS_ENABLED(CONFIG_IPV6) 5097 static void 5098 mlxsw_sp_fib6_offload_failed_flag_set(struct mlxsw_sp *mlxsw_sp, 5099 struct fib6_info **rt_arr, 5100 unsigned int nrt6) 5101 { 5102 int i; 5103 5104 /* In IPv6 a multipath route is represented using multiple routes, so 5105 * we need to set the flags on all of them. 5106 */ 5107 for (i = 0; i < nrt6; i++) 5108 fib6_info_hw_flags_set(mlxsw_sp_net(mlxsw_sp), rt_arr[i], 5109 false, false, true); 5110 } 5111 #else 5112 static void 5113 mlxsw_sp_fib6_offload_failed_flag_set(struct mlxsw_sp *mlxsw_sp, 5114 struct fib6_info **rt_arr, 5115 unsigned int nrt6) 5116 { 5117 } 5118 #endif 5119 5120 #if IS_ENABLED(CONFIG_IPV6) 5121 static void 5122 mlxsw_sp_fib6_entry_hw_flags_set(struct mlxsw_sp *mlxsw_sp, 5123 struct mlxsw_sp_fib_entry *fib_entry) 5124 { 5125 struct mlxsw_sp_fib6_entry *fib6_entry; 5126 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 5127 bool should_offload; 5128 5129 should_offload = mlxsw_sp_fib_entry_should_offload(fib_entry); 5130 5131 /* In IPv6 a multipath route is represented using multiple routes, so 5132 * we need to set the flags on all of them. 5133 */ 5134 fib6_entry = container_of(fib_entry, struct mlxsw_sp_fib6_entry, 5135 common); 5136 list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) 5137 fib6_info_hw_flags_set(mlxsw_sp_net(mlxsw_sp), mlxsw_sp_rt6->rt, 5138 should_offload, !should_offload, false); 5139 } 5140 #else 5141 static void 5142 mlxsw_sp_fib6_entry_hw_flags_set(struct mlxsw_sp *mlxsw_sp, 5143 struct mlxsw_sp_fib_entry *fib_entry) 5144 { 5145 } 5146 #endif 5147 5148 #if IS_ENABLED(CONFIG_IPV6) 5149 static void 5150 mlxsw_sp_fib6_entry_hw_flags_clear(struct mlxsw_sp *mlxsw_sp, 5151 struct mlxsw_sp_fib_entry *fib_entry) 5152 { 5153 struct mlxsw_sp_fib6_entry *fib6_entry; 5154 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 5155 5156 fib6_entry = container_of(fib_entry, struct mlxsw_sp_fib6_entry, 5157 common); 5158 list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) 5159 fib6_info_hw_flags_set(mlxsw_sp_net(mlxsw_sp), mlxsw_sp_rt6->rt, 5160 false, false, false); 5161 } 5162 #else 5163 static void 5164 mlxsw_sp_fib6_entry_hw_flags_clear(struct mlxsw_sp *mlxsw_sp, 5165 struct mlxsw_sp_fib_entry *fib_entry) 5166 { 5167 } 5168 #endif 5169 5170 static void 5171 mlxsw_sp_fib_entry_hw_flags_set(struct mlxsw_sp *mlxsw_sp, 5172 struct mlxsw_sp_fib_entry *fib_entry) 5173 { 5174 switch (fib_entry->fib_node->fib->proto) { 5175 case MLXSW_SP_L3_PROTO_IPV4: 5176 mlxsw_sp_fib4_entry_hw_flags_set(mlxsw_sp, fib_entry); 5177 break; 5178 case MLXSW_SP_L3_PROTO_IPV6: 5179 mlxsw_sp_fib6_entry_hw_flags_set(mlxsw_sp, fib_entry); 5180 break; 5181 } 5182 } 5183 5184 static void 5185 mlxsw_sp_fib_entry_hw_flags_clear(struct mlxsw_sp *mlxsw_sp, 5186 struct mlxsw_sp_fib_entry *fib_entry) 5187 { 5188 switch (fib_entry->fib_node->fib->proto) { 5189 case MLXSW_SP_L3_PROTO_IPV4: 5190 mlxsw_sp_fib4_entry_hw_flags_clear(mlxsw_sp, fib_entry); 5191 break; 5192 case MLXSW_SP_L3_PROTO_IPV6: 5193 mlxsw_sp_fib6_entry_hw_flags_clear(mlxsw_sp, fib_entry); 5194 break; 5195 } 5196 } 5197 5198 static void 5199 mlxsw_sp_fib_entry_hw_flags_refresh(struct mlxsw_sp *mlxsw_sp, 5200 struct mlxsw_sp_fib_entry *fib_entry, 5201 enum mlxsw_sp_fib_entry_op op) 5202 { 5203 switch (op) { 5204 case MLXSW_SP_FIB_ENTRY_OP_WRITE: 5205 case MLXSW_SP_FIB_ENTRY_OP_UPDATE: 5206 mlxsw_sp_fib_entry_hw_flags_set(mlxsw_sp, fib_entry); 5207 break; 5208 case MLXSW_SP_FIB_ENTRY_OP_DELETE: 5209 mlxsw_sp_fib_entry_hw_flags_clear(mlxsw_sp, fib_entry); 5210 break; 5211 default: 5212 break; 5213 } 5214 } 5215 5216 struct mlxsw_sp_fib_entry_op_ctx_basic { 5217 char ralue_pl[MLXSW_REG_RALUE_LEN]; 5218 }; 5219 5220 static void 5221 mlxsw_sp_router_ll_basic_fib_entry_pack(struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 5222 enum mlxsw_sp_l3proto proto, 5223 enum mlxsw_sp_fib_entry_op op, 5224 u16 virtual_router, u8 prefix_len, 5225 unsigned char *addr, 5226 struct mlxsw_sp_fib_entry_priv *priv) 5227 { 5228 struct mlxsw_sp_fib_entry_op_ctx_basic *op_ctx_basic = (void *) op_ctx->ll_priv; 5229 enum mlxsw_reg_ralxx_protocol ralxx_proto; 5230 char *ralue_pl = op_ctx_basic->ralue_pl; 5231 enum mlxsw_reg_ralue_op ralue_op; 5232 5233 ralxx_proto = (enum mlxsw_reg_ralxx_protocol) proto; 5234 5235 switch (op) { 5236 case MLXSW_SP_FIB_ENTRY_OP_WRITE: 5237 case MLXSW_SP_FIB_ENTRY_OP_UPDATE: 5238 ralue_op = MLXSW_REG_RALUE_OP_WRITE_WRITE; 5239 break; 5240 case MLXSW_SP_FIB_ENTRY_OP_DELETE: 5241 ralue_op = MLXSW_REG_RALUE_OP_WRITE_DELETE; 5242 break; 5243 default: 5244 WARN_ON_ONCE(1); 5245 return; 5246 } 5247 5248 switch (proto) { 5249 case MLXSW_SP_L3_PROTO_IPV4: 5250 mlxsw_reg_ralue_pack4(ralue_pl, ralxx_proto, ralue_op, 5251 virtual_router, prefix_len, (u32 *) addr); 5252 break; 5253 case MLXSW_SP_L3_PROTO_IPV6: 5254 mlxsw_reg_ralue_pack6(ralue_pl, ralxx_proto, ralue_op, 5255 virtual_router, prefix_len, addr); 5256 break; 5257 } 5258 } 5259 5260 static void 5261 mlxsw_sp_router_ll_basic_fib_entry_act_remote_pack(struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 5262 enum mlxsw_reg_ralue_trap_action trap_action, 5263 u16 trap_id, u32 adjacency_index, u16 ecmp_size) 5264 { 5265 struct mlxsw_sp_fib_entry_op_ctx_basic *op_ctx_basic = (void *) op_ctx->ll_priv; 5266 5267 mlxsw_reg_ralue_act_remote_pack(op_ctx_basic->ralue_pl, trap_action, 5268 trap_id, adjacency_index, ecmp_size); 5269 } 5270 5271 static void 5272 mlxsw_sp_router_ll_basic_fib_entry_act_local_pack(struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 5273 enum mlxsw_reg_ralue_trap_action trap_action, 5274 u16 trap_id, u16 local_erif) 5275 { 5276 struct mlxsw_sp_fib_entry_op_ctx_basic *op_ctx_basic = (void *) op_ctx->ll_priv; 5277 5278 mlxsw_reg_ralue_act_local_pack(op_ctx_basic->ralue_pl, trap_action, 5279 trap_id, local_erif); 5280 } 5281 5282 static void 5283 mlxsw_sp_router_ll_basic_fib_entry_act_ip2me_pack(struct mlxsw_sp_fib_entry_op_ctx *op_ctx) 5284 { 5285 struct mlxsw_sp_fib_entry_op_ctx_basic *op_ctx_basic = (void *) op_ctx->ll_priv; 5286 5287 mlxsw_reg_ralue_act_ip2me_pack(op_ctx_basic->ralue_pl); 5288 } 5289 5290 static void 5291 mlxsw_sp_router_ll_basic_fib_entry_act_ip2me_tun_pack(struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 5292 u32 tunnel_ptr) 5293 { 5294 struct mlxsw_sp_fib_entry_op_ctx_basic *op_ctx_basic = (void *) op_ctx->ll_priv; 5295 5296 mlxsw_reg_ralue_act_ip2me_tun_pack(op_ctx_basic->ralue_pl, tunnel_ptr); 5297 } 5298 5299 static int 5300 mlxsw_sp_router_ll_basic_fib_entry_commit(struct mlxsw_sp *mlxsw_sp, 5301 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 5302 bool *postponed_for_bulk) 5303 { 5304 struct mlxsw_sp_fib_entry_op_ctx_basic *op_ctx_basic = (void *) op_ctx->ll_priv; 5305 5306 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), 5307 op_ctx_basic->ralue_pl); 5308 } 5309 5310 static bool 5311 mlxsw_sp_router_ll_basic_fib_entry_is_committed(struct mlxsw_sp_fib_entry_priv *priv) 5312 { 5313 return true; 5314 } 5315 5316 static void mlxsw_sp_fib_entry_pack(struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 5317 struct mlxsw_sp_fib_entry *fib_entry, 5318 enum mlxsw_sp_fib_entry_op op) 5319 { 5320 struct mlxsw_sp_fib *fib = fib_entry->fib_node->fib; 5321 5322 mlxsw_sp_fib_entry_op_ctx_priv_hold(op_ctx, fib_entry->priv); 5323 fib->ll_ops->fib_entry_pack(op_ctx, fib->proto, op, fib->vr->id, 5324 fib_entry->fib_node->key.prefix_len, 5325 fib_entry->fib_node->key.addr, 5326 fib_entry->priv); 5327 } 5328 5329 static int mlxsw_sp_fib_entry_commit(struct mlxsw_sp *mlxsw_sp, 5330 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 5331 const struct mlxsw_sp_router_ll_ops *ll_ops) 5332 { 5333 bool postponed_for_bulk = false; 5334 int err; 5335 5336 err = ll_ops->fib_entry_commit(mlxsw_sp, op_ctx, &postponed_for_bulk); 5337 if (!postponed_for_bulk) 5338 mlxsw_sp_fib_entry_op_ctx_priv_put_all(op_ctx); 5339 return err; 5340 } 5341 5342 static int mlxsw_sp_adj_discard_write(struct mlxsw_sp *mlxsw_sp) 5343 { 5344 enum mlxsw_reg_ratr_trap_action trap_action; 5345 char ratr_pl[MLXSW_REG_RATR_LEN]; 5346 int err; 5347 5348 if (mlxsw_sp->router->adj_discard_index_valid) 5349 return 0; 5350 5351 err = mlxsw_sp_kvdl_alloc(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 1, 5352 &mlxsw_sp->router->adj_discard_index); 5353 if (err) 5354 return err; 5355 5356 trap_action = MLXSW_REG_RATR_TRAP_ACTION_TRAP; 5357 mlxsw_reg_ratr_pack(ratr_pl, MLXSW_REG_RATR_OP_WRITE_WRITE_ENTRY, true, 5358 MLXSW_REG_RATR_TYPE_ETHERNET, 5359 mlxsw_sp->router->adj_discard_index, 5360 mlxsw_sp->router->lb_rif_index); 5361 mlxsw_reg_ratr_trap_action_set(ratr_pl, trap_action); 5362 mlxsw_reg_ratr_trap_id_set(ratr_pl, MLXSW_TRAP_ID_RTR_EGRESS0); 5363 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ratr), ratr_pl); 5364 if (err) 5365 goto err_ratr_write; 5366 5367 mlxsw_sp->router->adj_discard_index_valid = true; 5368 5369 return 0; 5370 5371 err_ratr_write: 5372 mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 1, 5373 mlxsw_sp->router->adj_discard_index); 5374 return err; 5375 } 5376 5377 static int mlxsw_sp_fib_entry_op_remote(struct mlxsw_sp *mlxsw_sp, 5378 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 5379 struct mlxsw_sp_fib_entry *fib_entry, 5380 enum mlxsw_sp_fib_entry_op op) 5381 { 5382 const struct mlxsw_sp_router_ll_ops *ll_ops = fib_entry->fib_node->fib->ll_ops; 5383 struct mlxsw_sp_nexthop_group *nh_group = fib_entry->nh_group; 5384 struct mlxsw_sp_nexthop_group_info *nhgi = nh_group->nhgi; 5385 enum mlxsw_reg_ralue_trap_action trap_action; 5386 u16 trap_id = 0; 5387 u32 adjacency_index = 0; 5388 u16 ecmp_size = 0; 5389 int err; 5390 5391 /* In case the nexthop group adjacency index is valid, use it 5392 * with provided ECMP size. Otherwise, setup trap and pass 5393 * traffic to kernel. 5394 */ 5395 if (mlxsw_sp_fib_entry_should_offload(fib_entry)) { 5396 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_NOP; 5397 adjacency_index = nhgi->adj_index; 5398 ecmp_size = nhgi->ecmp_size; 5399 } else if (!nhgi->adj_index_valid && nhgi->count && nhgi->nh_rif) { 5400 err = mlxsw_sp_adj_discard_write(mlxsw_sp); 5401 if (err) 5402 return err; 5403 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_NOP; 5404 adjacency_index = mlxsw_sp->router->adj_discard_index; 5405 ecmp_size = 1; 5406 } else { 5407 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_TRAP; 5408 trap_id = MLXSW_TRAP_ID_RTR_INGRESS0; 5409 } 5410 5411 mlxsw_sp_fib_entry_pack(op_ctx, fib_entry, op); 5412 ll_ops->fib_entry_act_remote_pack(op_ctx, trap_action, trap_id, 5413 adjacency_index, ecmp_size); 5414 return mlxsw_sp_fib_entry_commit(mlxsw_sp, op_ctx, ll_ops); 5415 } 5416 5417 static int mlxsw_sp_fib_entry_op_local(struct mlxsw_sp *mlxsw_sp, 5418 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 5419 struct mlxsw_sp_fib_entry *fib_entry, 5420 enum mlxsw_sp_fib_entry_op op) 5421 { 5422 const struct mlxsw_sp_router_ll_ops *ll_ops = fib_entry->fib_node->fib->ll_ops; 5423 struct mlxsw_sp_rif *rif = fib_entry->nh_group->nhgi->nh_rif; 5424 enum mlxsw_reg_ralue_trap_action trap_action; 5425 u16 trap_id = 0; 5426 u16 rif_index = 0; 5427 5428 if (mlxsw_sp_fib_entry_should_offload(fib_entry)) { 5429 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_NOP; 5430 rif_index = rif->rif_index; 5431 } else { 5432 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_TRAP; 5433 trap_id = MLXSW_TRAP_ID_RTR_INGRESS0; 5434 } 5435 5436 mlxsw_sp_fib_entry_pack(op_ctx, fib_entry, op); 5437 ll_ops->fib_entry_act_local_pack(op_ctx, trap_action, trap_id, rif_index); 5438 return mlxsw_sp_fib_entry_commit(mlxsw_sp, op_ctx, ll_ops); 5439 } 5440 5441 static int mlxsw_sp_fib_entry_op_trap(struct mlxsw_sp *mlxsw_sp, 5442 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 5443 struct mlxsw_sp_fib_entry *fib_entry, 5444 enum mlxsw_sp_fib_entry_op op) 5445 { 5446 const struct mlxsw_sp_router_ll_ops *ll_ops = fib_entry->fib_node->fib->ll_ops; 5447 5448 mlxsw_sp_fib_entry_pack(op_ctx, fib_entry, op); 5449 ll_ops->fib_entry_act_ip2me_pack(op_ctx); 5450 return mlxsw_sp_fib_entry_commit(mlxsw_sp, op_ctx, ll_ops); 5451 } 5452 5453 static int mlxsw_sp_fib_entry_op_blackhole(struct mlxsw_sp *mlxsw_sp, 5454 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 5455 struct mlxsw_sp_fib_entry *fib_entry, 5456 enum mlxsw_sp_fib_entry_op op) 5457 { 5458 const struct mlxsw_sp_router_ll_ops *ll_ops = fib_entry->fib_node->fib->ll_ops; 5459 enum mlxsw_reg_ralue_trap_action trap_action; 5460 5461 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_DISCARD_ERROR; 5462 mlxsw_sp_fib_entry_pack(op_ctx, fib_entry, op); 5463 ll_ops->fib_entry_act_local_pack(op_ctx, trap_action, 0, 0); 5464 return mlxsw_sp_fib_entry_commit(mlxsw_sp, op_ctx, ll_ops); 5465 } 5466 5467 static int 5468 mlxsw_sp_fib_entry_op_unreachable(struct mlxsw_sp *mlxsw_sp, 5469 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 5470 struct mlxsw_sp_fib_entry *fib_entry, 5471 enum mlxsw_sp_fib_entry_op op) 5472 { 5473 const struct mlxsw_sp_router_ll_ops *ll_ops = fib_entry->fib_node->fib->ll_ops; 5474 enum mlxsw_reg_ralue_trap_action trap_action; 5475 u16 trap_id; 5476 5477 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_TRAP; 5478 trap_id = MLXSW_TRAP_ID_RTR_INGRESS1; 5479 5480 mlxsw_sp_fib_entry_pack(op_ctx, fib_entry, op); 5481 ll_ops->fib_entry_act_local_pack(op_ctx, trap_action, trap_id, 0); 5482 return mlxsw_sp_fib_entry_commit(mlxsw_sp, op_ctx, ll_ops); 5483 } 5484 5485 static int 5486 mlxsw_sp_fib_entry_op_ipip_decap(struct mlxsw_sp *mlxsw_sp, 5487 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 5488 struct mlxsw_sp_fib_entry *fib_entry, 5489 enum mlxsw_sp_fib_entry_op op) 5490 { 5491 const struct mlxsw_sp_router_ll_ops *ll_ops = fib_entry->fib_node->fib->ll_ops; 5492 struct mlxsw_sp_ipip_entry *ipip_entry = fib_entry->decap.ipip_entry; 5493 const struct mlxsw_sp_ipip_ops *ipip_ops; 5494 int err; 5495 5496 if (WARN_ON(!ipip_entry)) 5497 return -EINVAL; 5498 5499 ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt]; 5500 err = ipip_ops->decap_config(mlxsw_sp, ipip_entry, 5501 fib_entry->decap.tunnel_index); 5502 if (err) 5503 return err; 5504 5505 mlxsw_sp_fib_entry_pack(op_ctx, fib_entry, op); 5506 ll_ops->fib_entry_act_ip2me_tun_pack(op_ctx, 5507 fib_entry->decap.tunnel_index); 5508 return mlxsw_sp_fib_entry_commit(mlxsw_sp, op_ctx, ll_ops); 5509 } 5510 5511 static int mlxsw_sp_fib_entry_op_nve_decap(struct mlxsw_sp *mlxsw_sp, 5512 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 5513 struct mlxsw_sp_fib_entry *fib_entry, 5514 enum mlxsw_sp_fib_entry_op op) 5515 { 5516 const struct mlxsw_sp_router_ll_ops *ll_ops = fib_entry->fib_node->fib->ll_ops; 5517 5518 mlxsw_sp_fib_entry_pack(op_ctx, fib_entry, op); 5519 ll_ops->fib_entry_act_ip2me_tun_pack(op_ctx, 5520 fib_entry->decap.tunnel_index); 5521 return mlxsw_sp_fib_entry_commit(mlxsw_sp, op_ctx, ll_ops); 5522 } 5523 5524 static int __mlxsw_sp_fib_entry_op(struct mlxsw_sp *mlxsw_sp, 5525 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 5526 struct mlxsw_sp_fib_entry *fib_entry, 5527 enum mlxsw_sp_fib_entry_op op) 5528 { 5529 switch (fib_entry->type) { 5530 case MLXSW_SP_FIB_ENTRY_TYPE_REMOTE: 5531 return mlxsw_sp_fib_entry_op_remote(mlxsw_sp, op_ctx, fib_entry, op); 5532 case MLXSW_SP_FIB_ENTRY_TYPE_LOCAL: 5533 return mlxsw_sp_fib_entry_op_local(mlxsw_sp, op_ctx, fib_entry, op); 5534 case MLXSW_SP_FIB_ENTRY_TYPE_TRAP: 5535 return mlxsw_sp_fib_entry_op_trap(mlxsw_sp, op_ctx, fib_entry, op); 5536 case MLXSW_SP_FIB_ENTRY_TYPE_BLACKHOLE: 5537 return mlxsw_sp_fib_entry_op_blackhole(mlxsw_sp, op_ctx, fib_entry, op); 5538 case MLXSW_SP_FIB_ENTRY_TYPE_UNREACHABLE: 5539 return mlxsw_sp_fib_entry_op_unreachable(mlxsw_sp, op_ctx, fib_entry, op); 5540 case MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP: 5541 return mlxsw_sp_fib_entry_op_ipip_decap(mlxsw_sp, op_ctx, fib_entry, op); 5542 case MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP: 5543 return mlxsw_sp_fib_entry_op_nve_decap(mlxsw_sp, op_ctx, fib_entry, op); 5544 } 5545 return -EINVAL; 5546 } 5547 5548 static int mlxsw_sp_fib_entry_op(struct mlxsw_sp *mlxsw_sp, 5549 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 5550 struct mlxsw_sp_fib_entry *fib_entry, 5551 enum mlxsw_sp_fib_entry_op op) 5552 { 5553 int err = __mlxsw_sp_fib_entry_op(mlxsw_sp, op_ctx, fib_entry, op); 5554 5555 if (err) 5556 return err; 5557 5558 mlxsw_sp_fib_entry_hw_flags_refresh(mlxsw_sp, fib_entry, op); 5559 5560 return err; 5561 } 5562 5563 static int __mlxsw_sp_fib_entry_update(struct mlxsw_sp *mlxsw_sp, 5564 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 5565 struct mlxsw_sp_fib_entry *fib_entry, 5566 bool is_new) 5567 { 5568 return mlxsw_sp_fib_entry_op(mlxsw_sp, op_ctx, fib_entry, 5569 is_new ? MLXSW_SP_FIB_ENTRY_OP_WRITE : 5570 MLXSW_SP_FIB_ENTRY_OP_UPDATE); 5571 } 5572 5573 static int mlxsw_sp_fib_entry_update(struct mlxsw_sp *mlxsw_sp, 5574 struct mlxsw_sp_fib_entry *fib_entry) 5575 { 5576 struct mlxsw_sp_fib_entry_op_ctx *op_ctx = mlxsw_sp->router->ll_op_ctx; 5577 5578 mlxsw_sp_fib_entry_op_ctx_clear(op_ctx); 5579 return __mlxsw_sp_fib_entry_update(mlxsw_sp, op_ctx, fib_entry, false); 5580 } 5581 5582 static int mlxsw_sp_fib_entry_del(struct mlxsw_sp *mlxsw_sp, 5583 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 5584 struct mlxsw_sp_fib_entry *fib_entry) 5585 { 5586 const struct mlxsw_sp_router_ll_ops *ll_ops = fib_entry->fib_node->fib->ll_ops; 5587 5588 if (!ll_ops->fib_entry_is_committed(fib_entry->priv)) 5589 return 0; 5590 return mlxsw_sp_fib_entry_op(mlxsw_sp, op_ctx, fib_entry, 5591 MLXSW_SP_FIB_ENTRY_OP_DELETE); 5592 } 5593 5594 static int 5595 mlxsw_sp_fib4_entry_type_set(struct mlxsw_sp *mlxsw_sp, 5596 const struct fib_entry_notifier_info *fen_info, 5597 struct mlxsw_sp_fib_entry *fib_entry) 5598 { 5599 struct mlxsw_sp_nexthop_group_info *nhgi = fib_entry->nh_group->nhgi; 5600 union mlxsw_sp_l3addr dip = { .addr4 = htonl(fen_info->dst) }; 5601 struct mlxsw_sp_router *router = mlxsw_sp->router; 5602 u32 tb_id = mlxsw_sp_fix_tb_id(fen_info->tb_id); 5603 int ifindex = nhgi->nexthops[0].ifindex; 5604 struct mlxsw_sp_ipip_entry *ipip_entry; 5605 5606 switch (fen_info->type) { 5607 case RTN_LOCAL: 5608 ipip_entry = mlxsw_sp_ipip_entry_find_by_decap(mlxsw_sp, ifindex, 5609 MLXSW_SP_L3_PROTO_IPV4, dip); 5610 if (ipip_entry && ipip_entry->ol_dev->flags & IFF_UP) { 5611 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP; 5612 return mlxsw_sp_fib_entry_decap_init(mlxsw_sp, 5613 fib_entry, 5614 ipip_entry); 5615 } 5616 if (mlxsw_sp_router_nve_is_decap(mlxsw_sp, tb_id, 5617 MLXSW_SP_L3_PROTO_IPV4, 5618 &dip)) { 5619 u32 tunnel_index; 5620 5621 tunnel_index = router->nve_decap_config.tunnel_index; 5622 fib_entry->decap.tunnel_index = tunnel_index; 5623 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP; 5624 return 0; 5625 } 5626 fallthrough; 5627 case RTN_BROADCAST: 5628 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP; 5629 return 0; 5630 case RTN_BLACKHOLE: 5631 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_BLACKHOLE; 5632 return 0; 5633 case RTN_UNREACHABLE: 5634 case RTN_PROHIBIT: 5635 /* Packets hitting these routes need to be trapped, but 5636 * can do so with a lower priority than packets directed 5637 * at the host, so use action type local instead of trap. 5638 */ 5639 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_UNREACHABLE; 5640 return 0; 5641 case RTN_UNICAST: 5642 if (nhgi->gateway) 5643 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_REMOTE; 5644 else 5645 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL; 5646 return 0; 5647 default: 5648 return -EINVAL; 5649 } 5650 } 5651 5652 static void 5653 mlxsw_sp_fib4_entry_type_unset(struct mlxsw_sp *mlxsw_sp, 5654 struct mlxsw_sp_fib_entry *fib_entry) 5655 { 5656 switch (fib_entry->type) { 5657 case MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP: 5658 mlxsw_sp_fib_entry_decap_fini(mlxsw_sp, fib_entry); 5659 break; 5660 default: 5661 break; 5662 } 5663 } 5664 5665 static struct mlxsw_sp_fib4_entry * 5666 mlxsw_sp_fib4_entry_create(struct mlxsw_sp *mlxsw_sp, 5667 struct mlxsw_sp_fib_node *fib_node, 5668 const struct fib_entry_notifier_info *fen_info) 5669 { 5670 struct mlxsw_sp_fib4_entry *fib4_entry; 5671 struct mlxsw_sp_fib_entry *fib_entry; 5672 int err; 5673 5674 fib4_entry = kzalloc(sizeof(*fib4_entry), GFP_KERNEL); 5675 if (!fib4_entry) 5676 return ERR_PTR(-ENOMEM); 5677 fib_entry = &fib4_entry->common; 5678 5679 fib_entry->priv = mlxsw_sp_fib_entry_priv_create(fib_node->fib->ll_ops); 5680 if (IS_ERR(fib_entry->priv)) { 5681 err = PTR_ERR(fib_entry->priv); 5682 goto err_fib_entry_priv_create; 5683 } 5684 5685 err = mlxsw_sp_nexthop4_group_get(mlxsw_sp, fib_entry, fen_info->fi); 5686 if (err) 5687 goto err_nexthop4_group_get; 5688 5689 err = mlxsw_sp_nexthop_group_vr_link(fib_entry->nh_group, 5690 fib_node->fib); 5691 if (err) 5692 goto err_nexthop_group_vr_link; 5693 5694 err = mlxsw_sp_fib4_entry_type_set(mlxsw_sp, fen_info, fib_entry); 5695 if (err) 5696 goto err_fib4_entry_type_set; 5697 5698 fib4_entry->fi = fen_info->fi; 5699 fib_info_hold(fib4_entry->fi); 5700 fib4_entry->tb_id = fen_info->tb_id; 5701 fib4_entry->type = fen_info->type; 5702 fib4_entry->tos = fen_info->tos; 5703 5704 fib_entry->fib_node = fib_node; 5705 5706 return fib4_entry; 5707 5708 err_fib4_entry_type_set: 5709 mlxsw_sp_nexthop_group_vr_unlink(fib_entry->nh_group, fib_node->fib); 5710 err_nexthop_group_vr_link: 5711 mlxsw_sp_nexthop4_group_put(mlxsw_sp, &fib4_entry->common); 5712 err_nexthop4_group_get: 5713 mlxsw_sp_fib_entry_priv_put(fib_entry->priv); 5714 err_fib_entry_priv_create: 5715 kfree(fib4_entry); 5716 return ERR_PTR(err); 5717 } 5718 5719 static void mlxsw_sp_fib4_entry_destroy(struct mlxsw_sp *mlxsw_sp, 5720 struct mlxsw_sp_fib4_entry *fib4_entry) 5721 { 5722 struct mlxsw_sp_fib_node *fib_node = fib4_entry->common.fib_node; 5723 5724 fib_info_put(fib4_entry->fi); 5725 mlxsw_sp_fib4_entry_type_unset(mlxsw_sp, &fib4_entry->common); 5726 mlxsw_sp_nexthop_group_vr_unlink(fib4_entry->common.nh_group, 5727 fib_node->fib); 5728 mlxsw_sp_nexthop4_group_put(mlxsw_sp, &fib4_entry->common); 5729 mlxsw_sp_fib_entry_priv_put(fib4_entry->common.priv); 5730 kfree(fib4_entry); 5731 } 5732 5733 static struct mlxsw_sp_fib4_entry * 5734 mlxsw_sp_fib4_entry_lookup(struct mlxsw_sp *mlxsw_sp, 5735 const struct fib_entry_notifier_info *fen_info) 5736 { 5737 struct mlxsw_sp_fib4_entry *fib4_entry; 5738 struct mlxsw_sp_fib_node *fib_node; 5739 struct mlxsw_sp_fib *fib; 5740 struct mlxsw_sp_vr *vr; 5741 5742 vr = mlxsw_sp_vr_find(mlxsw_sp, fen_info->tb_id); 5743 if (!vr) 5744 return NULL; 5745 fib = mlxsw_sp_vr_fib(vr, MLXSW_SP_L3_PROTO_IPV4); 5746 5747 fib_node = mlxsw_sp_fib_node_lookup(fib, &fen_info->dst, 5748 sizeof(fen_info->dst), 5749 fen_info->dst_len); 5750 if (!fib_node) 5751 return NULL; 5752 5753 fib4_entry = container_of(fib_node->fib_entry, 5754 struct mlxsw_sp_fib4_entry, common); 5755 if (fib4_entry->tb_id == fen_info->tb_id && 5756 fib4_entry->tos == fen_info->tos && 5757 fib4_entry->type == fen_info->type && 5758 fib4_entry->fi == fen_info->fi) 5759 return fib4_entry; 5760 5761 return NULL; 5762 } 5763 5764 static const struct rhashtable_params mlxsw_sp_fib_ht_params = { 5765 .key_offset = offsetof(struct mlxsw_sp_fib_node, key), 5766 .head_offset = offsetof(struct mlxsw_sp_fib_node, ht_node), 5767 .key_len = sizeof(struct mlxsw_sp_fib_key), 5768 .automatic_shrinking = true, 5769 }; 5770 5771 static int mlxsw_sp_fib_node_insert(struct mlxsw_sp_fib *fib, 5772 struct mlxsw_sp_fib_node *fib_node) 5773 { 5774 return rhashtable_insert_fast(&fib->ht, &fib_node->ht_node, 5775 mlxsw_sp_fib_ht_params); 5776 } 5777 5778 static void mlxsw_sp_fib_node_remove(struct mlxsw_sp_fib *fib, 5779 struct mlxsw_sp_fib_node *fib_node) 5780 { 5781 rhashtable_remove_fast(&fib->ht, &fib_node->ht_node, 5782 mlxsw_sp_fib_ht_params); 5783 } 5784 5785 static struct mlxsw_sp_fib_node * 5786 mlxsw_sp_fib_node_lookup(struct mlxsw_sp_fib *fib, const void *addr, 5787 size_t addr_len, unsigned char prefix_len) 5788 { 5789 struct mlxsw_sp_fib_key key; 5790 5791 memset(&key, 0, sizeof(key)); 5792 memcpy(key.addr, addr, addr_len); 5793 key.prefix_len = prefix_len; 5794 return rhashtable_lookup_fast(&fib->ht, &key, mlxsw_sp_fib_ht_params); 5795 } 5796 5797 static struct mlxsw_sp_fib_node * 5798 mlxsw_sp_fib_node_create(struct mlxsw_sp_fib *fib, const void *addr, 5799 size_t addr_len, unsigned char prefix_len) 5800 { 5801 struct mlxsw_sp_fib_node *fib_node; 5802 5803 fib_node = kzalloc(sizeof(*fib_node), GFP_KERNEL); 5804 if (!fib_node) 5805 return NULL; 5806 5807 list_add(&fib_node->list, &fib->node_list); 5808 memcpy(fib_node->key.addr, addr, addr_len); 5809 fib_node->key.prefix_len = prefix_len; 5810 5811 return fib_node; 5812 } 5813 5814 static void mlxsw_sp_fib_node_destroy(struct mlxsw_sp_fib_node *fib_node) 5815 { 5816 list_del(&fib_node->list); 5817 kfree(fib_node); 5818 } 5819 5820 static int mlxsw_sp_fib_lpm_tree_link(struct mlxsw_sp *mlxsw_sp, 5821 struct mlxsw_sp_fib_node *fib_node) 5822 { 5823 struct mlxsw_sp_prefix_usage req_prefix_usage; 5824 struct mlxsw_sp_fib *fib = fib_node->fib; 5825 struct mlxsw_sp_lpm_tree *lpm_tree; 5826 int err; 5827 5828 lpm_tree = mlxsw_sp->router->lpm.proto_trees[fib->proto]; 5829 if (lpm_tree->prefix_ref_count[fib_node->key.prefix_len] != 0) 5830 goto out; 5831 5832 mlxsw_sp_prefix_usage_cpy(&req_prefix_usage, &lpm_tree->prefix_usage); 5833 mlxsw_sp_prefix_usage_set(&req_prefix_usage, fib_node->key.prefix_len); 5834 lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage, 5835 fib->proto); 5836 if (IS_ERR(lpm_tree)) 5837 return PTR_ERR(lpm_tree); 5838 5839 err = mlxsw_sp_vrs_lpm_tree_replace(mlxsw_sp, fib, lpm_tree); 5840 if (err) 5841 goto err_lpm_tree_replace; 5842 5843 out: 5844 lpm_tree->prefix_ref_count[fib_node->key.prefix_len]++; 5845 return 0; 5846 5847 err_lpm_tree_replace: 5848 mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree); 5849 return err; 5850 } 5851 5852 static void mlxsw_sp_fib_lpm_tree_unlink(struct mlxsw_sp *mlxsw_sp, 5853 struct mlxsw_sp_fib_node *fib_node) 5854 { 5855 struct mlxsw_sp_lpm_tree *lpm_tree = fib_node->fib->lpm_tree; 5856 struct mlxsw_sp_prefix_usage req_prefix_usage; 5857 struct mlxsw_sp_fib *fib = fib_node->fib; 5858 int err; 5859 5860 if (--lpm_tree->prefix_ref_count[fib_node->key.prefix_len] != 0) 5861 return; 5862 /* Try to construct a new LPM tree from the current prefix usage 5863 * minus the unused one. If we fail, continue using the old one. 5864 */ 5865 mlxsw_sp_prefix_usage_cpy(&req_prefix_usage, &lpm_tree->prefix_usage); 5866 mlxsw_sp_prefix_usage_clear(&req_prefix_usage, 5867 fib_node->key.prefix_len); 5868 lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage, 5869 fib->proto); 5870 if (IS_ERR(lpm_tree)) 5871 return; 5872 5873 err = mlxsw_sp_vrs_lpm_tree_replace(mlxsw_sp, fib, lpm_tree); 5874 if (err) 5875 goto err_lpm_tree_replace; 5876 5877 return; 5878 5879 err_lpm_tree_replace: 5880 mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree); 5881 } 5882 5883 static int mlxsw_sp_fib_node_init(struct mlxsw_sp *mlxsw_sp, 5884 struct mlxsw_sp_fib_node *fib_node, 5885 struct mlxsw_sp_fib *fib) 5886 { 5887 int err; 5888 5889 err = mlxsw_sp_fib_node_insert(fib, fib_node); 5890 if (err) 5891 return err; 5892 fib_node->fib = fib; 5893 5894 err = mlxsw_sp_fib_lpm_tree_link(mlxsw_sp, fib_node); 5895 if (err) 5896 goto err_fib_lpm_tree_link; 5897 5898 return 0; 5899 5900 err_fib_lpm_tree_link: 5901 fib_node->fib = NULL; 5902 mlxsw_sp_fib_node_remove(fib, fib_node); 5903 return err; 5904 } 5905 5906 static void mlxsw_sp_fib_node_fini(struct mlxsw_sp *mlxsw_sp, 5907 struct mlxsw_sp_fib_node *fib_node) 5908 { 5909 struct mlxsw_sp_fib *fib = fib_node->fib; 5910 5911 mlxsw_sp_fib_lpm_tree_unlink(mlxsw_sp, fib_node); 5912 fib_node->fib = NULL; 5913 mlxsw_sp_fib_node_remove(fib, fib_node); 5914 } 5915 5916 static struct mlxsw_sp_fib_node * 5917 mlxsw_sp_fib_node_get(struct mlxsw_sp *mlxsw_sp, u32 tb_id, const void *addr, 5918 size_t addr_len, unsigned char prefix_len, 5919 enum mlxsw_sp_l3proto proto) 5920 { 5921 struct mlxsw_sp_fib_node *fib_node; 5922 struct mlxsw_sp_fib *fib; 5923 struct mlxsw_sp_vr *vr; 5924 int err; 5925 5926 vr = mlxsw_sp_vr_get(mlxsw_sp, tb_id, NULL); 5927 if (IS_ERR(vr)) 5928 return ERR_CAST(vr); 5929 fib = mlxsw_sp_vr_fib(vr, proto); 5930 5931 fib_node = mlxsw_sp_fib_node_lookup(fib, addr, addr_len, prefix_len); 5932 if (fib_node) 5933 return fib_node; 5934 5935 fib_node = mlxsw_sp_fib_node_create(fib, addr, addr_len, prefix_len); 5936 if (!fib_node) { 5937 err = -ENOMEM; 5938 goto err_fib_node_create; 5939 } 5940 5941 err = mlxsw_sp_fib_node_init(mlxsw_sp, fib_node, fib); 5942 if (err) 5943 goto err_fib_node_init; 5944 5945 return fib_node; 5946 5947 err_fib_node_init: 5948 mlxsw_sp_fib_node_destroy(fib_node); 5949 err_fib_node_create: 5950 mlxsw_sp_vr_put(mlxsw_sp, vr); 5951 return ERR_PTR(err); 5952 } 5953 5954 static void mlxsw_sp_fib_node_put(struct mlxsw_sp *mlxsw_sp, 5955 struct mlxsw_sp_fib_node *fib_node) 5956 { 5957 struct mlxsw_sp_vr *vr = fib_node->fib->vr; 5958 5959 if (fib_node->fib_entry) 5960 return; 5961 mlxsw_sp_fib_node_fini(mlxsw_sp, fib_node); 5962 mlxsw_sp_fib_node_destroy(fib_node); 5963 mlxsw_sp_vr_put(mlxsw_sp, vr); 5964 } 5965 5966 static int mlxsw_sp_fib_node_entry_link(struct mlxsw_sp *mlxsw_sp, 5967 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 5968 struct mlxsw_sp_fib_entry *fib_entry) 5969 { 5970 struct mlxsw_sp_fib_node *fib_node = fib_entry->fib_node; 5971 bool is_new = !fib_node->fib_entry; 5972 int err; 5973 5974 fib_node->fib_entry = fib_entry; 5975 5976 err = __mlxsw_sp_fib_entry_update(mlxsw_sp, op_ctx, fib_entry, is_new); 5977 if (err) 5978 goto err_fib_entry_update; 5979 5980 return 0; 5981 5982 err_fib_entry_update: 5983 fib_node->fib_entry = NULL; 5984 return err; 5985 } 5986 5987 static int __mlxsw_sp_fib_node_entry_unlink(struct mlxsw_sp *mlxsw_sp, 5988 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 5989 struct mlxsw_sp_fib_entry *fib_entry) 5990 { 5991 struct mlxsw_sp_fib_node *fib_node = fib_entry->fib_node; 5992 int err; 5993 5994 err = mlxsw_sp_fib_entry_del(mlxsw_sp, op_ctx, fib_entry); 5995 fib_node->fib_entry = NULL; 5996 return err; 5997 } 5998 5999 static void mlxsw_sp_fib_node_entry_unlink(struct mlxsw_sp *mlxsw_sp, 6000 struct mlxsw_sp_fib_entry *fib_entry) 6001 { 6002 struct mlxsw_sp_fib_entry_op_ctx *op_ctx = mlxsw_sp->router->ll_op_ctx; 6003 6004 mlxsw_sp_fib_entry_op_ctx_clear(op_ctx); 6005 __mlxsw_sp_fib_node_entry_unlink(mlxsw_sp, op_ctx, fib_entry); 6006 } 6007 6008 static bool mlxsw_sp_fib4_allow_replace(struct mlxsw_sp_fib4_entry *fib4_entry) 6009 { 6010 struct mlxsw_sp_fib_node *fib_node = fib4_entry->common.fib_node; 6011 struct mlxsw_sp_fib4_entry *fib4_replaced; 6012 6013 if (!fib_node->fib_entry) 6014 return true; 6015 6016 fib4_replaced = container_of(fib_node->fib_entry, 6017 struct mlxsw_sp_fib4_entry, common); 6018 if (fib4_entry->tb_id == RT_TABLE_MAIN && 6019 fib4_replaced->tb_id == RT_TABLE_LOCAL) 6020 return false; 6021 6022 return true; 6023 } 6024 6025 static int 6026 mlxsw_sp_router_fib4_replace(struct mlxsw_sp *mlxsw_sp, 6027 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 6028 const struct fib_entry_notifier_info *fen_info) 6029 { 6030 struct mlxsw_sp_fib4_entry *fib4_entry, *fib4_replaced; 6031 struct mlxsw_sp_fib_entry *replaced; 6032 struct mlxsw_sp_fib_node *fib_node; 6033 int err; 6034 6035 if (mlxsw_sp->router->aborted) 6036 return 0; 6037 6038 if (fen_info->fi->nh && 6039 !mlxsw_sp_nexthop_obj_group_lookup(mlxsw_sp, fen_info->fi->nh->id)) 6040 return 0; 6041 6042 fib_node = mlxsw_sp_fib_node_get(mlxsw_sp, fen_info->tb_id, 6043 &fen_info->dst, sizeof(fen_info->dst), 6044 fen_info->dst_len, 6045 MLXSW_SP_L3_PROTO_IPV4); 6046 if (IS_ERR(fib_node)) { 6047 dev_warn(mlxsw_sp->bus_info->dev, "Failed to get FIB node\n"); 6048 return PTR_ERR(fib_node); 6049 } 6050 6051 fib4_entry = mlxsw_sp_fib4_entry_create(mlxsw_sp, fib_node, fen_info); 6052 if (IS_ERR(fib4_entry)) { 6053 dev_warn(mlxsw_sp->bus_info->dev, "Failed to create FIB entry\n"); 6054 err = PTR_ERR(fib4_entry); 6055 goto err_fib4_entry_create; 6056 } 6057 6058 if (!mlxsw_sp_fib4_allow_replace(fib4_entry)) { 6059 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry); 6060 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 6061 return 0; 6062 } 6063 6064 replaced = fib_node->fib_entry; 6065 err = mlxsw_sp_fib_node_entry_link(mlxsw_sp, op_ctx, &fib4_entry->common); 6066 if (err) { 6067 dev_warn(mlxsw_sp->bus_info->dev, "Failed to link FIB entry to node\n"); 6068 goto err_fib_node_entry_link; 6069 } 6070 6071 /* Nothing to replace */ 6072 if (!replaced) 6073 return 0; 6074 6075 mlxsw_sp_fib_entry_hw_flags_clear(mlxsw_sp, replaced); 6076 fib4_replaced = container_of(replaced, struct mlxsw_sp_fib4_entry, 6077 common); 6078 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_replaced); 6079 6080 return 0; 6081 6082 err_fib_node_entry_link: 6083 fib_node->fib_entry = replaced; 6084 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry); 6085 err_fib4_entry_create: 6086 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 6087 return err; 6088 } 6089 6090 static int mlxsw_sp_router_fib4_del(struct mlxsw_sp *mlxsw_sp, 6091 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 6092 struct fib_entry_notifier_info *fen_info) 6093 { 6094 struct mlxsw_sp_fib4_entry *fib4_entry; 6095 struct mlxsw_sp_fib_node *fib_node; 6096 int err; 6097 6098 if (mlxsw_sp->router->aborted) 6099 return 0; 6100 6101 fib4_entry = mlxsw_sp_fib4_entry_lookup(mlxsw_sp, fen_info); 6102 if (!fib4_entry) 6103 return 0; 6104 fib_node = fib4_entry->common.fib_node; 6105 6106 err = __mlxsw_sp_fib_node_entry_unlink(mlxsw_sp, op_ctx, &fib4_entry->common); 6107 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry); 6108 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 6109 return err; 6110 } 6111 6112 static bool mlxsw_sp_fib6_rt_should_ignore(const struct fib6_info *rt) 6113 { 6114 /* Multicast routes aren't supported, so ignore them. Neighbour 6115 * Discovery packets are specifically trapped. 6116 */ 6117 if (ipv6_addr_type(&rt->fib6_dst.addr) & IPV6_ADDR_MULTICAST) 6118 return true; 6119 6120 /* Cloned routes are irrelevant in the forwarding path. */ 6121 if (rt->fib6_flags & RTF_CACHE) 6122 return true; 6123 6124 return false; 6125 } 6126 6127 static struct mlxsw_sp_rt6 *mlxsw_sp_rt6_create(struct fib6_info *rt) 6128 { 6129 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 6130 6131 mlxsw_sp_rt6 = kzalloc(sizeof(*mlxsw_sp_rt6), GFP_KERNEL); 6132 if (!mlxsw_sp_rt6) 6133 return ERR_PTR(-ENOMEM); 6134 6135 /* In case of route replace, replaced route is deleted with 6136 * no notification. Take reference to prevent accessing freed 6137 * memory. 6138 */ 6139 mlxsw_sp_rt6->rt = rt; 6140 fib6_info_hold(rt); 6141 6142 return mlxsw_sp_rt6; 6143 } 6144 6145 #if IS_ENABLED(CONFIG_IPV6) 6146 static void mlxsw_sp_rt6_release(struct fib6_info *rt) 6147 { 6148 fib6_info_release(rt); 6149 } 6150 #else 6151 static void mlxsw_sp_rt6_release(struct fib6_info *rt) 6152 { 6153 } 6154 #endif 6155 6156 static void mlxsw_sp_rt6_destroy(struct mlxsw_sp_rt6 *mlxsw_sp_rt6) 6157 { 6158 struct fib6_nh *fib6_nh = mlxsw_sp_rt6->rt->fib6_nh; 6159 6160 if (!mlxsw_sp_rt6->rt->nh) 6161 fib6_nh->fib_nh_flags &= ~RTNH_F_OFFLOAD; 6162 mlxsw_sp_rt6_release(mlxsw_sp_rt6->rt); 6163 kfree(mlxsw_sp_rt6); 6164 } 6165 6166 static struct fib6_info * 6167 mlxsw_sp_fib6_entry_rt(const struct mlxsw_sp_fib6_entry *fib6_entry) 6168 { 6169 return list_first_entry(&fib6_entry->rt6_list, struct mlxsw_sp_rt6, 6170 list)->rt; 6171 } 6172 6173 static struct mlxsw_sp_rt6 * 6174 mlxsw_sp_fib6_entry_rt_find(const struct mlxsw_sp_fib6_entry *fib6_entry, 6175 const struct fib6_info *rt) 6176 { 6177 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 6178 6179 list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) { 6180 if (mlxsw_sp_rt6->rt == rt) 6181 return mlxsw_sp_rt6; 6182 } 6183 6184 return NULL; 6185 } 6186 6187 static bool mlxsw_sp_nexthop6_ipip_type(const struct mlxsw_sp *mlxsw_sp, 6188 const struct fib6_info *rt, 6189 enum mlxsw_sp_ipip_type *ret) 6190 { 6191 return rt->fib6_nh->fib_nh_dev && 6192 mlxsw_sp_netdev_ipip_type(mlxsw_sp, rt->fib6_nh->fib_nh_dev, ret); 6193 } 6194 6195 static int mlxsw_sp_nexthop6_init(struct mlxsw_sp *mlxsw_sp, 6196 struct mlxsw_sp_nexthop_group *nh_grp, 6197 struct mlxsw_sp_nexthop *nh, 6198 const struct fib6_info *rt) 6199 { 6200 struct net_device *dev = rt->fib6_nh->fib_nh_dev; 6201 6202 nh->nhgi = nh_grp->nhgi; 6203 nh->nh_weight = rt->fib6_nh->fib_nh_weight; 6204 memcpy(&nh->gw_addr, &rt->fib6_nh->fib_nh_gw6, sizeof(nh->gw_addr)); 6205 #if IS_ENABLED(CONFIG_IPV6) 6206 nh->neigh_tbl = &nd_tbl; 6207 #endif 6208 mlxsw_sp_nexthop_counter_alloc(mlxsw_sp, nh); 6209 6210 list_add_tail(&nh->router_list_node, &mlxsw_sp->router->nexthop_list); 6211 6212 if (!dev) 6213 return 0; 6214 nh->ifindex = dev->ifindex; 6215 6216 return mlxsw_sp_nexthop_type_init(mlxsw_sp, nh, dev); 6217 } 6218 6219 static void mlxsw_sp_nexthop6_fini(struct mlxsw_sp *mlxsw_sp, 6220 struct mlxsw_sp_nexthop *nh) 6221 { 6222 mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh); 6223 list_del(&nh->router_list_node); 6224 mlxsw_sp_nexthop_counter_free(mlxsw_sp, nh); 6225 } 6226 6227 static bool mlxsw_sp_rt6_is_gateway(const struct mlxsw_sp *mlxsw_sp, 6228 const struct fib6_info *rt) 6229 { 6230 return rt->fib6_nh->fib_nh_gw_family || 6231 mlxsw_sp_nexthop6_ipip_type(mlxsw_sp, rt, NULL); 6232 } 6233 6234 static int 6235 mlxsw_sp_nexthop6_group_info_init(struct mlxsw_sp *mlxsw_sp, 6236 struct mlxsw_sp_nexthop_group *nh_grp, 6237 struct mlxsw_sp_fib6_entry *fib6_entry) 6238 { 6239 struct mlxsw_sp_nexthop_group_info *nhgi; 6240 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 6241 struct mlxsw_sp_nexthop *nh; 6242 int err, i; 6243 6244 nhgi = kzalloc(struct_size(nhgi, nexthops, fib6_entry->nrt6), 6245 GFP_KERNEL); 6246 if (!nhgi) 6247 return -ENOMEM; 6248 nh_grp->nhgi = nhgi; 6249 nhgi->nh_grp = nh_grp; 6250 mlxsw_sp_rt6 = list_first_entry(&fib6_entry->rt6_list, 6251 struct mlxsw_sp_rt6, list); 6252 nhgi->gateway = mlxsw_sp_rt6_is_gateway(mlxsw_sp, mlxsw_sp_rt6->rt); 6253 nhgi->count = fib6_entry->nrt6; 6254 for (i = 0; i < nhgi->count; i++) { 6255 struct fib6_info *rt = mlxsw_sp_rt6->rt; 6256 6257 nh = &nhgi->nexthops[i]; 6258 err = mlxsw_sp_nexthop6_init(mlxsw_sp, nh_grp, nh, rt); 6259 if (err) 6260 goto err_nexthop6_init; 6261 mlxsw_sp_rt6 = list_next_entry(mlxsw_sp_rt6, list); 6262 } 6263 nh_grp->nhgi = nhgi; 6264 err = mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp); 6265 if (err) 6266 goto err_group_refresh; 6267 6268 return 0; 6269 6270 err_group_refresh: 6271 i = nhgi->count; 6272 err_nexthop6_init: 6273 for (i--; i >= 0; i--) { 6274 nh = &nhgi->nexthops[i]; 6275 mlxsw_sp_nexthop6_fini(mlxsw_sp, nh); 6276 } 6277 kfree(nhgi); 6278 return err; 6279 } 6280 6281 static void 6282 mlxsw_sp_nexthop6_group_info_fini(struct mlxsw_sp *mlxsw_sp, 6283 struct mlxsw_sp_nexthop_group *nh_grp) 6284 { 6285 struct mlxsw_sp_nexthop_group_info *nhgi = nh_grp->nhgi; 6286 int i; 6287 6288 for (i = nhgi->count - 1; i >= 0; i--) { 6289 struct mlxsw_sp_nexthop *nh = &nhgi->nexthops[i]; 6290 6291 mlxsw_sp_nexthop6_fini(mlxsw_sp, nh); 6292 } 6293 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp); 6294 WARN_ON_ONCE(nhgi->adj_index_valid); 6295 kfree(nhgi); 6296 } 6297 6298 static struct mlxsw_sp_nexthop_group * 6299 mlxsw_sp_nexthop6_group_create(struct mlxsw_sp *mlxsw_sp, 6300 struct mlxsw_sp_fib6_entry *fib6_entry) 6301 { 6302 struct mlxsw_sp_nexthop_group *nh_grp; 6303 int err; 6304 6305 nh_grp = kzalloc(sizeof(*nh_grp), GFP_KERNEL); 6306 if (!nh_grp) 6307 return ERR_PTR(-ENOMEM); 6308 INIT_LIST_HEAD(&nh_grp->vr_list); 6309 err = rhashtable_init(&nh_grp->vr_ht, 6310 &mlxsw_sp_nexthop_group_vr_ht_params); 6311 if (err) 6312 goto err_nexthop_group_vr_ht_init; 6313 INIT_LIST_HEAD(&nh_grp->fib_list); 6314 nh_grp->type = MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6; 6315 6316 err = mlxsw_sp_nexthop6_group_info_init(mlxsw_sp, nh_grp, fib6_entry); 6317 if (err) 6318 goto err_nexthop_group_info_init; 6319 6320 err = mlxsw_sp_nexthop_group_insert(mlxsw_sp, nh_grp); 6321 if (err) 6322 goto err_nexthop_group_insert; 6323 6324 nh_grp->can_destroy = true; 6325 6326 return nh_grp; 6327 6328 err_nexthop_group_insert: 6329 mlxsw_sp_nexthop6_group_info_fini(mlxsw_sp, nh_grp); 6330 err_nexthop_group_info_init: 6331 rhashtable_destroy(&nh_grp->vr_ht); 6332 err_nexthop_group_vr_ht_init: 6333 kfree(nh_grp); 6334 return ERR_PTR(err); 6335 } 6336 6337 static void 6338 mlxsw_sp_nexthop6_group_destroy(struct mlxsw_sp *mlxsw_sp, 6339 struct mlxsw_sp_nexthop_group *nh_grp) 6340 { 6341 if (!nh_grp->can_destroy) 6342 return; 6343 mlxsw_sp_nexthop_group_remove(mlxsw_sp, nh_grp); 6344 mlxsw_sp_nexthop6_group_info_fini(mlxsw_sp, nh_grp); 6345 WARN_ON_ONCE(!list_empty(&nh_grp->vr_list)); 6346 rhashtable_destroy(&nh_grp->vr_ht); 6347 kfree(nh_grp); 6348 } 6349 6350 static int mlxsw_sp_nexthop6_group_get(struct mlxsw_sp *mlxsw_sp, 6351 struct mlxsw_sp_fib6_entry *fib6_entry) 6352 { 6353 struct fib6_info *rt = mlxsw_sp_fib6_entry_rt(fib6_entry); 6354 struct mlxsw_sp_nexthop_group *nh_grp; 6355 6356 if (rt->nh) { 6357 nh_grp = mlxsw_sp_nexthop_obj_group_lookup(mlxsw_sp, 6358 rt->nh->id); 6359 if (WARN_ON_ONCE(!nh_grp)) 6360 return -EINVAL; 6361 goto out; 6362 } 6363 6364 nh_grp = mlxsw_sp_nexthop6_group_lookup(mlxsw_sp, fib6_entry); 6365 if (!nh_grp) { 6366 nh_grp = mlxsw_sp_nexthop6_group_create(mlxsw_sp, fib6_entry); 6367 if (IS_ERR(nh_grp)) 6368 return PTR_ERR(nh_grp); 6369 } 6370 6371 /* The route and the nexthop are described by the same struct, so we 6372 * need to the update the nexthop offload indication for the new route. 6373 */ 6374 __mlxsw_sp_nexthop6_group_offload_refresh(nh_grp, fib6_entry); 6375 6376 out: 6377 list_add_tail(&fib6_entry->common.nexthop_group_node, 6378 &nh_grp->fib_list); 6379 fib6_entry->common.nh_group = nh_grp; 6380 6381 return 0; 6382 } 6383 6384 static void mlxsw_sp_nexthop6_group_put(struct mlxsw_sp *mlxsw_sp, 6385 struct mlxsw_sp_fib_entry *fib_entry) 6386 { 6387 struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group; 6388 6389 list_del(&fib_entry->nexthop_group_node); 6390 if (!list_empty(&nh_grp->fib_list)) 6391 return; 6392 6393 if (nh_grp->type == MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ) { 6394 mlxsw_sp_nexthop_obj_group_destroy(mlxsw_sp, nh_grp); 6395 return; 6396 } 6397 6398 mlxsw_sp_nexthop6_group_destroy(mlxsw_sp, nh_grp); 6399 } 6400 6401 static int mlxsw_sp_nexthop6_group_update(struct mlxsw_sp *mlxsw_sp, 6402 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 6403 struct mlxsw_sp_fib6_entry *fib6_entry) 6404 { 6405 struct mlxsw_sp_nexthop_group *old_nh_grp = fib6_entry->common.nh_group; 6406 struct mlxsw_sp_fib_node *fib_node = fib6_entry->common.fib_node; 6407 int err; 6408 6409 mlxsw_sp_nexthop_group_vr_unlink(old_nh_grp, fib_node->fib); 6410 fib6_entry->common.nh_group = NULL; 6411 list_del(&fib6_entry->common.nexthop_group_node); 6412 6413 err = mlxsw_sp_nexthop6_group_get(mlxsw_sp, fib6_entry); 6414 if (err) 6415 goto err_nexthop6_group_get; 6416 6417 err = mlxsw_sp_nexthop_group_vr_link(fib6_entry->common.nh_group, 6418 fib_node->fib); 6419 if (err) 6420 goto err_nexthop_group_vr_link; 6421 6422 /* In case this entry is offloaded, then the adjacency index 6423 * currently associated with it in the device's table is that 6424 * of the old group. Start using the new one instead. 6425 */ 6426 err = __mlxsw_sp_fib_entry_update(mlxsw_sp, op_ctx, 6427 &fib6_entry->common, false); 6428 if (err) 6429 goto err_fib_entry_update; 6430 6431 if (list_empty(&old_nh_grp->fib_list)) 6432 mlxsw_sp_nexthop6_group_destroy(mlxsw_sp, old_nh_grp); 6433 6434 return 0; 6435 6436 err_fib_entry_update: 6437 mlxsw_sp_nexthop_group_vr_unlink(fib6_entry->common.nh_group, 6438 fib_node->fib); 6439 err_nexthop_group_vr_link: 6440 mlxsw_sp_nexthop6_group_put(mlxsw_sp, &fib6_entry->common); 6441 err_nexthop6_group_get: 6442 list_add_tail(&fib6_entry->common.nexthop_group_node, 6443 &old_nh_grp->fib_list); 6444 fib6_entry->common.nh_group = old_nh_grp; 6445 mlxsw_sp_nexthop_group_vr_link(old_nh_grp, fib_node->fib); 6446 return err; 6447 } 6448 6449 static int 6450 mlxsw_sp_fib6_entry_nexthop_add(struct mlxsw_sp *mlxsw_sp, 6451 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 6452 struct mlxsw_sp_fib6_entry *fib6_entry, 6453 struct fib6_info **rt_arr, unsigned int nrt6) 6454 { 6455 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 6456 int err, i; 6457 6458 for (i = 0; i < nrt6; i++) { 6459 mlxsw_sp_rt6 = mlxsw_sp_rt6_create(rt_arr[i]); 6460 if (IS_ERR(mlxsw_sp_rt6)) { 6461 err = PTR_ERR(mlxsw_sp_rt6); 6462 goto err_rt6_create; 6463 } 6464 6465 list_add_tail(&mlxsw_sp_rt6->list, &fib6_entry->rt6_list); 6466 fib6_entry->nrt6++; 6467 } 6468 6469 err = mlxsw_sp_nexthop6_group_update(mlxsw_sp, op_ctx, fib6_entry); 6470 if (err) 6471 goto err_nexthop6_group_update; 6472 6473 return 0; 6474 6475 err_nexthop6_group_update: 6476 i = nrt6; 6477 err_rt6_create: 6478 for (i--; i >= 0; i--) { 6479 fib6_entry->nrt6--; 6480 mlxsw_sp_rt6 = list_last_entry(&fib6_entry->rt6_list, 6481 struct mlxsw_sp_rt6, list); 6482 list_del(&mlxsw_sp_rt6->list); 6483 mlxsw_sp_rt6_destroy(mlxsw_sp_rt6); 6484 } 6485 return err; 6486 } 6487 6488 static void 6489 mlxsw_sp_fib6_entry_nexthop_del(struct mlxsw_sp *mlxsw_sp, 6490 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 6491 struct mlxsw_sp_fib6_entry *fib6_entry, 6492 struct fib6_info **rt_arr, unsigned int nrt6) 6493 { 6494 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 6495 int i; 6496 6497 for (i = 0; i < nrt6; i++) { 6498 mlxsw_sp_rt6 = mlxsw_sp_fib6_entry_rt_find(fib6_entry, 6499 rt_arr[i]); 6500 if (WARN_ON_ONCE(!mlxsw_sp_rt6)) 6501 continue; 6502 6503 fib6_entry->nrt6--; 6504 list_del(&mlxsw_sp_rt6->list); 6505 mlxsw_sp_rt6_destroy(mlxsw_sp_rt6); 6506 } 6507 6508 mlxsw_sp_nexthop6_group_update(mlxsw_sp, op_ctx, fib6_entry); 6509 } 6510 6511 static void mlxsw_sp_fib6_entry_type_set(struct mlxsw_sp *mlxsw_sp, 6512 struct mlxsw_sp_fib_entry *fib_entry, 6513 const struct fib6_info *rt) 6514 { 6515 if (rt->fib6_flags & (RTF_LOCAL | RTF_ANYCAST)) 6516 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP; 6517 else if (rt->fib6_type == RTN_BLACKHOLE) 6518 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_BLACKHOLE; 6519 else if (rt->fib6_flags & RTF_REJECT) 6520 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_UNREACHABLE; 6521 else if (fib_entry->nh_group->nhgi->gateway) 6522 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_REMOTE; 6523 else 6524 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL; 6525 } 6526 6527 static void 6528 mlxsw_sp_fib6_entry_rt_destroy_all(struct mlxsw_sp_fib6_entry *fib6_entry) 6529 { 6530 struct mlxsw_sp_rt6 *mlxsw_sp_rt6, *tmp; 6531 6532 list_for_each_entry_safe(mlxsw_sp_rt6, tmp, &fib6_entry->rt6_list, 6533 list) { 6534 fib6_entry->nrt6--; 6535 list_del(&mlxsw_sp_rt6->list); 6536 mlxsw_sp_rt6_destroy(mlxsw_sp_rt6); 6537 } 6538 } 6539 6540 static struct mlxsw_sp_fib6_entry * 6541 mlxsw_sp_fib6_entry_create(struct mlxsw_sp *mlxsw_sp, 6542 struct mlxsw_sp_fib_node *fib_node, 6543 struct fib6_info **rt_arr, unsigned int nrt6) 6544 { 6545 struct mlxsw_sp_fib6_entry *fib6_entry; 6546 struct mlxsw_sp_fib_entry *fib_entry; 6547 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 6548 int err, i; 6549 6550 fib6_entry = kzalloc(sizeof(*fib6_entry), GFP_KERNEL); 6551 if (!fib6_entry) 6552 return ERR_PTR(-ENOMEM); 6553 fib_entry = &fib6_entry->common; 6554 6555 fib_entry->priv = mlxsw_sp_fib_entry_priv_create(fib_node->fib->ll_ops); 6556 if (IS_ERR(fib_entry->priv)) { 6557 err = PTR_ERR(fib_entry->priv); 6558 goto err_fib_entry_priv_create; 6559 } 6560 6561 INIT_LIST_HEAD(&fib6_entry->rt6_list); 6562 6563 for (i = 0; i < nrt6; i++) { 6564 mlxsw_sp_rt6 = mlxsw_sp_rt6_create(rt_arr[i]); 6565 if (IS_ERR(mlxsw_sp_rt6)) { 6566 err = PTR_ERR(mlxsw_sp_rt6); 6567 goto err_rt6_create; 6568 } 6569 list_add_tail(&mlxsw_sp_rt6->list, &fib6_entry->rt6_list); 6570 fib6_entry->nrt6++; 6571 } 6572 6573 err = mlxsw_sp_nexthop6_group_get(mlxsw_sp, fib6_entry); 6574 if (err) 6575 goto err_nexthop6_group_get; 6576 6577 err = mlxsw_sp_nexthop_group_vr_link(fib_entry->nh_group, 6578 fib_node->fib); 6579 if (err) 6580 goto err_nexthop_group_vr_link; 6581 6582 mlxsw_sp_fib6_entry_type_set(mlxsw_sp, fib_entry, rt_arr[0]); 6583 6584 fib_entry->fib_node = fib_node; 6585 6586 return fib6_entry; 6587 6588 err_nexthop_group_vr_link: 6589 mlxsw_sp_nexthop6_group_put(mlxsw_sp, fib_entry); 6590 err_nexthop6_group_get: 6591 i = nrt6; 6592 err_rt6_create: 6593 for (i--; i >= 0; i--) { 6594 fib6_entry->nrt6--; 6595 mlxsw_sp_rt6 = list_last_entry(&fib6_entry->rt6_list, 6596 struct mlxsw_sp_rt6, list); 6597 list_del(&mlxsw_sp_rt6->list); 6598 mlxsw_sp_rt6_destroy(mlxsw_sp_rt6); 6599 } 6600 mlxsw_sp_fib_entry_priv_put(fib_entry->priv); 6601 err_fib_entry_priv_create: 6602 kfree(fib6_entry); 6603 return ERR_PTR(err); 6604 } 6605 6606 static void mlxsw_sp_fib6_entry_destroy(struct mlxsw_sp *mlxsw_sp, 6607 struct mlxsw_sp_fib6_entry *fib6_entry) 6608 { 6609 struct mlxsw_sp_fib_node *fib_node = fib6_entry->common.fib_node; 6610 6611 mlxsw_sp_nexthop_group_vr_unlink(fib6_entry->common.nh_group, 6612 fib_node->fib); 6613 mlxsw_sp_nexthop6_group_put(mlxsw_sp, &fib6_entry->common); 6614 mlxsw_sp_fib6_entry_rt_destroy_all(fib6_entry); 6615 WARN_ON(fib6_entry->nrt6); 6616 mlxsw_sp_fib_entry_priv_put(fib6_entry->common.priv); 6617 kfree(fib6_entry); 6618 } 6619 6620 static struct mlxsw_sp_fib6_entry * 6621 mlxsw_sp_fib6_entry_lookup(struct mlxsw_sp *mlxsw_sp, 6622 const struct fib6_info *rt) 6623 { 6624 struct mlxsw_sp_fib6_entry *fib6_entry; 6625 struct mlxsw_sp_fib_node *fib_node; 6626 struct mlxsw_sp_fib *fib; 6627 struct fib6_info *cmp_rt; 6628 struct mlxsw_sp_vr *vr; 6629 6630 vr = mlxsw_sp_vr_find(mlxsw_sp, rt->fib6_table->tb6_id); 6631 if (!vr) 6632 return NULL; 6633 fib = mlxsw_sp_vr_fib(vr, MLXSW_SP_L3_PROTO_IPV6); 6634 6635 fib_node = mlxsw_sp_fib_node_lookup(fib, &rt->fib6_dst.addr, 6636 sizeof(rt->fib6_dst.addr), 6637 rt->fib6_dst.plen); 6638 if (!fib_node) 6639 return NULL; 6640 6641 fib6_entry = container_of(fib_node->fib_entry, 6642 struct mlxsw_sp_fib6_entry, common); 6643 cmp_rt = mlxsw_sp_fib6_entry_rt(fib6_entry); 6644 if (rt->fib6_table->tb6_id == cmp_rt->fib6_table->tb6_id && 6645 rt->fib6_metric == cmp_rt->fib6_metric && 6646 mlxsw_sp_fib6_entry_rt_find(fib6_entry, rt)) 6647 return fib6_entry; 6648 6649 return NULL; 6650 } 6651 6652 static bool mlxsw_sp_fib6_allow_replace(struct mlxsw_sp_fib6_entry *fib6_entry) 6653 { 6654 struct mlxsw_sp_fib_node *fib_node = fib6_entry->common.fib_node; 6655 struct mlxsw_sp_fib6_entry *fib6_replaced; 6656 struct fib6_info *rt, *rt_replaced; 6657 6658 if (!fib_node->fib_entry) 6659 return true; 6660 6661 fib6_replaced = container_of(fib_node->fib_entry, 6662 struct mlxsw_sp_fib6_entry, 6663 common); 6664 rt = mlxsw_sp_fib6_entry_rt(fib6_entry); 6665 rt_replaced = mlxsw_sp_fib6_entry_rt(fib6_replaced); 6666 if (rt->fib6_table->tb6_id == RT_TABLE_MAIN && 6667 rt_replaced->fib6_table->tb6_id == RT_TABLE_LOCAL) 6668 return false; 6669 6670 return true; 6671 } 6672 6673 static int mlxsw_sp_router_fib6_replace(struct mlxsw_sp *mlxsw_sp, 6674 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 6675 struct fib6_info **rt_arr, unsigned int nrt6) 6676 { 6677 struct mlxsw_sp_fib6_entry *fib6_entry, *fib6_replaced; 6678 struct mlxsw_sp_fib_entry *replaced; 6679 struct mlxsw_sp_fib_node *fib_node; 6680 struct fib6_info *rt = rt_arr[0]; 6681 int err; 6682 6683 if (mlxsw_sp->router->aborted) 6684 return 0; 6685 6686 if (rt->fib6_src.plen) 6687 return -EINVAL; 6688 6689 if (mlxsw_sp_fib6_rt_should_ignore(rt)) 6690 return 0; 6691 6692 if (rt->nh && !mlxsw_sp_nexthop_obj_group_lookup(mlxsw_sp, rt->nh->id)) 6693 return 0; 6694 6695 fib_node = mlxsw_sp_fib_node_get(mlxsw_sp, rt->fib6_table->tb6_id, 6696 &rt->fib6_dst.addr, 6697 sizeof(rt->fib6_dst.addr), 6698 rt->fib6_dst.plen, 6699 MLXSW_SP_L3_PROTO_IPV6); 6700 if (IS_ERR(fib_node)) 6701 return PTR_ERR(fib_node); 6702 6703 fib6_entry = mlxsw_sp_fib6_entry_create(mlxsw_sp, fib_node, rt_arr, 6704 nrt6); 6705 if (IS_ERR(fib6_entry)) { 6706 err = PTR_ERR(fib6_entry); 6707 goto err_fib6_entry_create; 6708 } 6709 6710 if (!mlxsw_sp_fib6_allow_replace(fib6_entry)) { 6711 mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry); 6712 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 6713 return 0; 6714 } 6715 6716 replaced = fib_node->fib_entry; 6717 err = mlxsw_sp_fib_node_entry_link(mlxsw_sp, op_ctx, &fib6_entry->common); 6718 if (err) 6719 goto err_fib_node_entry_link; 6720 6721 /* Nothing to replace */ 6722 if (!replaced) 6723 return 0; 6724 6725 mlxsw_sp_fib_entry_hw_flags_clear(mlxsw_sp, replaced); 6726 fib6_replaced = container_of(replaced, struct mlxsw_sp_fib6_entry, 6727 common); 6728 mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_replaced); 6729 6730 return 0; 6731 6732 err_fib_node_entry_link: 6733 fib_node->fib_entry = replaced; 6734 mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry); 6735 err_fib6_entry_create: 6736 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 6737 return err; 6738 } 6739 6740 static int mlxsw_sp_router_fib6_append(struct mlxsw_sp *mlxsw_sp, 6741 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 6742 struct fib6_info **rt_arr, unsigned int nrt6) 6743 { 6744 struct mlxsw_sp_fib6_entry *fib6_entry; 6745 struct mlxsw_sp_fib_node *fib_node; 6746 struct fib6_info *rt = rt_arr[0]; 6747 int err; 6748 6749 if (mlxsw_sp->router->aborted) 6750 return 0; 6751 6752 if (rt->fib6_src.plen) 6753 return -EINVAL; 6754 6755 if (mlxsw_sp_fib6_rt_should_ignore(rt)) 6756 return 0; 6757 6758 fib_node = mlxsw_sp_fib_node_get(mlxsw_sp, rt->fib6_table->tb6_id, 6759 &rt->fib6_dst.addr, 6760 sizeof(rt->fib6_dst.addr), 6761 rt->fib6_dst.plen, 6762 MLXSW_SP_L3_PROTO_IPV6); 6763 if (IS_ERR(fib_node)) 6764 return PTR_ERR(fib_node); 6765 6766 if (WARN_ON_ONCE(!fib_node->fib_entry)) { 6767 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 6768 return -EINVAL; 6769 } 6770 6771 fib6_entry = container_of(fib_node->fib_entry, 6772 struct mlxsw_sp_fib6_entry, common); 6773 err = mlxsw_sp_fib6_entry_nexthop_add(mlxsw_sp, op_ctx, fib6_entry, rt_arr, nrt6); 6774 if (err) 6775 goto err_fib6_entry_nexthop_add; 6776 6777 return 0; 6778 6779 err_fib6_entry_nexthop_add: 6780 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 6781 return err; 6782 } 6783 6784 static int mlxsw_sp_router_fib6_del(struct mlxsw_sp *mlxsw_sp, 6785 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 6786 struct fib6_info **rt_arr, unsigned int nrt6) 6787 { 6788 struct mlxsw_sp_fib6_entry *fib6_entry; 6789 struct mlxsw_sp_fib_node *fib_node; 6790 struct fib6_info *rt = rt_arr[0]; 6791 int err; 6792 6793 if (mlxsw_sp->router->aborted) 6794 return 0; 6795 6796 if (mlxsw_sp_fib6_rt_should_ignore(rt)) 6797 return 0; 6798 6799 /* Multipath routes are first added to the FIB trie and only then 6800 * notified. If we vetoed the addition, we will get a delete 6801 * notification for a route we do not have. Therefore, do not warn if 6802 * route was not found. 6803 */ 6804 fib6_entry = mlxsw_sp_fib6_entry_lookup(mlxsw_sp, rt); 6805 if (!fib6_entry) 6806 return 0; 6807 6808 /* If not all the nexthops are deleted, then only reduce the nexthop 6809 * group. 6810 */ 6811 if (nrt6 != fib6_entry->nrt6) { 6812 mlxsw_sp_fib6_entry_nexthop_del(mlxsw_sp, op_ctx, fib6_entry, rt_arr, nrt6); 6813 return 0; 6814 } 6815 6816 fib_node = fib6_entry->common.fib_node; 6817 6818 err = __mlxsw_sp_fib_node_entry_unlink(mlxsw_sp, op_ctx, &fib6_entry->common); 6819 mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry); 6820 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 6821 return err; 6822 } 6823 6824 static int __mlxsw_sp_router_set_abort_trap(struct mlxsw_sp *mlxsw_sp, 6825 enum mlxsw_sp_l3proto proto, 6826 u8 tree_id) 6827 { 6828 const struct mlxsw_sp_router_ll_ops *ll_ops = mlxsw_sp->router->proto_ll_ops[proto]; 6829 enum mlxsw_reg_ralxx_protocol ralxx_proto = 6830 (enum mlxsw_reg_ralxx_protocol) proto; 6831 struct mlxsw_sp_fib_entry_priv *priv; 6832 char xralta_pl[MLXSW_REG_XRALTA_LEN]; 6833 char xralst_pl[MLXSW_REG_XRALST_LEN]; 6834 int i, err; 6835 6836 mlxsw_reg_xralta_pack(xralta_pl, true, ralxx_proto, tree_id); 6837 err = ll_ops->ralta_write(mlxsw_sp, xralta_pl); 6838 if (err) 6839 return err; 6840 6841 mlxsw_reg_xralst_pack(xralst_pl, 0xff, tree_id); 6842 err = ll_ops->ralst_write(mlxsw_sp, xralst_pl); 6843 if (err) 6844 return err; 6845 6846 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) { 6847 struct mlxsw_sp_fib_entry_op_ctx *op_ctx = mlxsw_sp->router->ll_op_ctx; 6848 struct mlxsw_sp_vr *vr = &mlxsw_sp->router->vrs[i]; 6849 char xraltb_pl[MLXSW_REG_XRALTB_LEN]; 6850 6851 mlxsw_sp_fib_entry_op_ctx_clear(op_ctx); 6852 mlxsw_reg_xraltb_pack(xraltb_pl, vr->id, ralxx_proto, tree_id); 6853 err = ll_ops->raltb_write(mlxsw_sp, xraltb_pl); 6854 if (err) 6855 return err; 6856 6857 priv = mlxsw_sp_fib_entry_priv_create(ll_ops); 6858 if (IS_ERR(priv)) 6859 return PTR_ERR(priv); 6860 6861 ll_ops->fib_entry_pack(op_ctx, proto, MLXSW_SP_FIB_ENTRY_OP_WRITE, 6862 vr->id, 0, NULL, priv); 6863 ll_ops->fib_entry_act_ip2me_pack(op_ctx); 6864 err = ll_ops->fib_entry_commit(mlxsw_sp, op_ctx, NULL); 6865 mlxsw_sp_fib_entry_priv_put(priv); 6866 if (err) 6867 return err; 6868 } 6869 6870 return 0; 6871 } 6872 6873 static struct mlxsw_sp_mr_table * 6874 mlxsw_sp_router_fibmr_family_to_table(struct mlxsw_sp_vr *vr, int family) 6875 { 6876 if (family == RTNL_FAMILY_IPMR) 6877 return vr->mr_table[MLXSW_SP_L3_PROTO_IPV4]; 6878 else 6879 return vr->mr_table[MLXSW_SP_L3_PROTO_IPV6]; 6880 } 6881 6882 static int mlxsw_sp_router_fibmr_add(struct mlxsw_sp *mlxsw_sp, 6883 struct mfc_entry_notifier_info *men_info, 6884 bool replace) 6885 { 6886 struct mlxsw_sp_mr_table *mrt; 6887 struct mlxsw_sp_vr *vr; 6888 6889 if (mlxsw_sp->router->aborted) 6890 return 0; 6891 6892 vr = mlxsw_sp_vr_get(mlxsw_sp, men_info->tb_id, NULL); 6893 if (IS_ERR(vr)) 6894 return PTR_ERR(vr); 6895 6896 mrt = mlxsw_sp_router_fibmr_family_to_table(vr, men_info->info.family); 6897 return mlxsw_sp_mr_route_add(mrt, men_info->mfc, replace); 6898 } 6899 6900 static void mlxsw_sp_router_fibmr_del(struct mlxsw_sp *mlxsw_sp, 6901 struct mfc_entry_notifier_info *men_info) 6902 { 6903 struct mlxsw_sp_mr_table *mrt; 6904 struct mlxsw_sp_vr *vr; 6905 6906 if (mlxsw_sp->router->aborted) 6907 return; 6908 6909 vr = mlxsw_sp_vr_find(mlxsw_sp, men_info->tb_id); 6910 if (WARN_ON(!vr)) 6911 return; 6912 6913 mrt = mlxsw_sp_router_fibmr_family_to_table(vr, men_info->info.family); 6914 mlxsw_sp_mr_route_del(mrt, men_info->mfc); 6915 mlxsw_sp_vr_put(mlxsw_sp, vr); 6916 } 6917 6918 static int 6919 mlxsw_sp_router_fibmr_vif_add(struct mlxsw_sp *mlxsw_sp, 6920 struct vif_entry_notifier_info *ven_info) 6921 { 6922 struct mlxsw_sp_mr_table *mrt; 6923 struct mlxsw_sp_rif *rif; 6924 struct mlxsw_sp_vr *vr; 6925 6926 if (mlxsw_sp->router->aborted) 6927 return 0; 6928 6929 vr = mlxsw_sp_vr_get(mlxsw_sp, ven_info->tb_id, NULL); 6930 if (IS_ERR(vr)) 6931 return PTR_ERR(vr); 6932 6933 mrt = mlxsw_sp_router_fibmr_family_to_table(vr, ven_info->info.family); 6934 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, ven_info->dev); 6935 return mlxsw_sp_mr_vif_add(mrt, ven_info->dev, 6936 ven_info->vif_index, 6937 ven_info->vif_flags, rif); 6938 } 6939 6940 static void 6941 mlxsw_sp_router_fibmr_vif_del(struct mlxsw_sp *mlxsw_sp, 6942 struct vif_entry_notifier_info *ven_info) 6943 { 6944 struct mlxsw_sp_mr_table *mrt; 6945 struct mlxsw_sp_vr *vr; 6946 6947 if (mlxsw_sp->router->aborted) 6948 return; 6949 6950 vr = mlxsw_sp_vr_find(mlxsw_sp, ven_info->tb_id); 6951 if (WARN_ON(!vr)) 6952 return; 6953 6954 mrt = mlxsw_sp_router_fibmr_family_to_table(vr, ven_info->info.family); 6955 mlxsw_sp_mr_vif_del(mrt, ven_info->vif_index); 6956 mlxsw_sp_vr_put(mlxsw_sp, vr); 6957 } 6958 6959 static int mlxsw_sp_router_set_abort_trap(struct mlxsw_sp *mlxsw_sp) 6960 { 6961 enum mlxsw_sp_l3proto proto = MLXSW_SP_L3_PROTO_IPV4; 6962 int err; 6963 6964 err = __mlxsw_sp_router_set_abort_trap(mlxsw_sp, proto, 6965 MLXSW_SP_LPM_TREE_MIN); 6966 if (err) 6967 return err; 6968 6969 /* The multicast router code does not need an abort trap as by default, 6970 * packets that don't match any routes are trapped to the CPU. 6971 */ 6972 6973 proto = MLXSW_SP_L3_PROTO_IPV6; 6974 return __mlxsw_sp_router_set_abort_trap(mlxsw_sp, proto, 6975 MLXSW_SP_LPM_TREE_MIN + 1); 6976 } 6977 6978 static void mlxsw_sp_fib4_node_flush(struct mlxsw_sp *mlxsw_sp, 6979 struct mlxsw_sp_fib_node *fib_node) 6980 { 6981 struct mlxsw_sp_fib4_entry *fib4_entry; 6982 6983 fib4_entry = container_of(fib_node->fib_entry, 6984 struct mlxsw_sp_fib4_entry, common); 6985 mlxsw_sp_fib_node_entry_unlink(mlxsw_sp, fib_node->fib_entry); 6986 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry); 6987 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 6988 } 6989 6990 static void mlxsw_sp_fib6_node_flush(struct mlxsw_sp *mlxsw_sp, 6991 struct mlxsw_sp_fib_node *fib_node) 6992 { 6993 struct mlxsw_sp_fib6_entry *fib6_entry; 6994 6995 fib6_entry = container_of(fib_node->fib_entry, 6996 struct mlxsw_sp_fib6_entry, common); 6997 mlxsw_sp_fib_node_entry_unlink(mlxsw_sp, fib_node->fib_entry); 6998 mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry); 6999 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 7000 } 7001 7002 static void mlxsw_sp_fib_node_flush(struct mlxsw_sp *mlxsw_sp, 7003 struct mlxsw_sp_fib_node *fib_node) 7004 { 7005 switch (fib_node->fib->proto) { 7006 case MLXSW_SP_L3_PROTO_IPV4: 7007 mlxsw_sp_fib4_node_flush(mlxsw_sp, fib_node); 7008 break; 7009 case MLXSW_SP_L3_PROTO_IPV6: 7010 mlxsw_sp_fib6_node_flush(mlxsw_sp, fib_node); 7011 break; 7012 } 7013 } 7014 7015 static void mlxsw_sp_vr_fib_flush(struct mlxsw_sp *mlxsw_sp, 7016 struct mlxsw_sp_vr *vr, 7017 enum mlxsw_sp_l3proto proto) 7018 { 7019 struct mlxsw_sp_fib *fib = mlxsw_sp_vr_fib(vr, proto); 7020 struct mlxsw_sp_fib_node *fib_node, *tmp; 7021 7022 list_for_each_entry_safe(fib_node, tmp, &fib->node_list, list) { 7023 bool do_break = &tmp->list == &fib->node_list; 7024 7025 mlxsw_sp_fib_node_flush(mlxsw_sp, fib_node); 7026 if (do_break) 7027 break; 7028 } 7029 } 7030 7031 static void mlxsw_sp_router_fib_flush(struct mlxsw_sp *mlxsw_sp) 7032 { 7033 int i, j; 7034 7035 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) { 7036 struct mlxsw_sp_vr *vr = &mlxsw_sp->router->vrs[i]; 7037 7038 if (!mlxsw_sp_vr_is_used(vr)) 7039 continue; 7040 7041 for (j = 0; j < MLXSW_SP_L3_PROTO_MAX; j++) 7042 mlxsw_sp_mr_table_flush(vr->mr_table[j]); 7043 mlxsw_sp_vr_fib_flush(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV4); 7044 7045 /* If virtual router was only used for IPv4, then it's no 7046 * longer used. 7047 */ 7048 if (!mlxsw_sp_vr_is_used(vr)) 7049 continue; 7050 mlxsw_sp_vr_fib_flush(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV6); 7051 } 7052 7053 /* After flushing all the routes, it is not possible anyone is still 7054 * using the adjacency index that is discarding packets, so free it in 7055 * case it was allocated. 7056 */ 7057 if (!mlxsw_sp->router->adj_discard_index_valid) 7058 return; 7059 mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 1, 7060 mlxsw_sp->router->adj_discard_index); 7061 mlxsw_sp->router->adj_discard_index_valid = false; 7062 } 7063 7064 static void mlxsw_sp_router_fib_abort(struct mlxsw_sp *mlxsw_sp) 7065 { 7066 int err; 7067 7068 if (mlxsw_sp->router->aborted) 7069 return; 7070 dev_warn(mlxsw_sp->bus_info->dev, "FIB abort triggered. Note that FIB entries are no longer being offloaded to this device.\n"); 7071 mlxsw_sp_router_fib_flush(mlxsw_sp); 7072 mlxsw_sp->router->aborted = true; 7073 err = mlxsw_sp_router_set_abort_trap(mlxsw_sp); 7074 if (err) 7075 dev_warn(mlxsw_sp->bus_info->dev, "Failed to set abort trap.\n"); 7076 } 7077 7078 struct mlxsw_sp_fib6_event { 7079 struct fib6_info **rt_arr; 7080 unsigned int nrt6; 7081 }; 7082 7083 struct mlxsw_sp_fib_event { 7084 struct list_head list; /* node in fib queue */ 7085 union { 7086 struct mlxsw_sp_fib6_event fib6_event; 7087 struct fib_entry_notifier_info fen_info; 7088 struct fib_rule_notifier_info fr_info; 7089 struct fib_nh_notifier_info fnh_info; 7090 struct mfc_entry_notifier_info men_info; 7091 struct vif_entry_notifier_info ven_info; 7092 }; 7093 struct mlxsw_sp *mlxsw_sp; 7094 unsigned long event; 7095 int family; 7096 }; 7097 7098 static int 7099 mlxsw_sp_router_fib6_event_init(struct mlxsw_sp_fib6_event *fib6_event, 7100 struct fib6_entry_notifier_info *fen6_info) 7101 { 7102 struct fib6_info *rt = fen6_info->rt; 7103 struct fib6_info **rt_arr; 7104 struct fib6_info *iter; 7105 unsigned int nrt6; 7106 int i = 0; 7107 7108 nrt6 = fen6_info->nsiblings + 1; 7109 7110 rt_arr = kcalloc(nrt6, sizeof(struct fib6_info *), GFP_ATOMIC); 7111 if (!rt_arr) 7112 return -ENOMEM; 7113 7114 fib6_event->rt_arr = rt_arr; 7115 fib6_event->nrt6 = nrt6; 7116 7117 rt_arr[0] = rt; 7118 fib6_info_hold(rt); 7119 7120 if (!fen6_info->nsiblings) 7121 return 0; 7122 7123 list_for_each_entry(iter, &rt->fib6_siblings, fib6_siblings) { 7124 if (i == fen6_info->nsiblings) 7125 break; 7126 7127 rt_arr[i + 1] = iter; 7128 fib6_info_hold(iter); 7129 i++; 7130 } 7131 WARN_ON_ONCE(i != fen6_info->nsiblings); 7132 7133 return 0; 7134 } 7135 7136 static void 7137 mlxsw_sp_router_fib6_event_fini(struct mlxsw_sp_fib6_event *fib6_event) 7138 { 7139 int i; 7140 7141 for (i = 0; i < fib6_event->nrt6; i++) 7142 mlxsw_sp_rt6_release(fib6_event->rt_arr[i]); 7143 kfree(fib6_event->rt_arr); 7144 } 7145 7146 static void mlxsw_sp_router_fib4_event_process(struct mlxsw_sp *mlxsw_sp, 7147 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 7148 struct mlxsw_sp_fib_event *fib_event) 7149 { 7150 int err; 7151 7152 mlxsw_sp_span_respin(mlxsw_sp); 7153 7154 switch (fib_event->event) { 7155 case FIB_EVENT_ENTRY_REPLACE: 7156 err = mlxsw_sp_router_fib4_replace(mlxsw_sp, op_ctx, &fib_event->fen_info); 7157 if (err) { 7158 mlxsw_sp_fib_entry_op_ctx_priv_put_all(op_ctx); 7159 mlxsw_sp_router_fib_abort(mlxsw_sp); 7160 mlxsw_sp_fib4_offload_failed_flag_set(mlxsw_sp, 7161 &fib_event->fen_info); 7162 } 7163 fib_info_put(fib_event->fen_info.fi); 7164 break; 7165 case FIB_EVENT_ENTRY_DEL: 7166 err = mlxsw_sp_router_fib4_del(mlxsw_sp, op_ctx, &fib_event->fen_info); 7167 if (err) 7168 mlxsw_sp_fib_entry_op_ctx_priv_put_all(op_ctx); 7169 fib_info_put(fib_event->fen_info.fi); 7170 break; 7171 case FIB_EVENT_NH_ADD: 7172 case FIB_EVENT_NH_DEL: 7173 mlxsw_sp_nexthop4_event(mlxsw_sp, fib_event->event, fib_event->fnh_info.fib_nh); 7174 fib_info_put(fib_event->fnh_info.fib_nh->nh_parent); 7175 break; 7176 } 7177 } 7178 7179 static void mlxsw_sp_router_fib6_event_process(struct mlxsw_sp *mlxsw_sp, 7180 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 7181 struct mlxsw_sp_fib_event *fib_event) 7182 { 7183 struct mlxsw_sp_fib6_event *fib6_event = &fib_event->fib6_event; 7184 int err; 7185 7186 mlxsw_sp_span_respin(mlxsw_sp); 7187 7188 switch (fib_event->event) { 7189 case FIB_EVENT_ENTRY_REPLACE: 7190 err = mlxsw_sp_router_fib6_replace(mlxsw_sp, op_ctx, fib_event->fib6_event.rt_arr, 7191 fib_event->fib6_event.nrt6); 7192 if (err) { 7193 mlxsw_sp_fib_entry_op_ctx_priv_put_all(op_ctx); 7194 mlxsw_sp_router_fib_abort(mlxsw_sp); 7195 mlxsw_sp_fib6_offload_failed_flag_set(mlxsw_sp, 7196 fib6_event->rt_arr, 7197 fib6_event->nrt6); 7198 } 7199 mlxsw_sp_router_fib6_event_fini(&fib_event->fib6_event); 7200 break; 7201 case FIB_EVENT_ENTRY_APPEND: 7202 err = mlxsw_sp_router_fib6_append(mlxsw_sp, op_ctx, fib_event->fib6_event.rt_arr, 7203 fib_event->fib6_event.nrt6); 7204 if (err) { 7205 mlxsw_sp_fib_entry_op_ctx_priv_put_all(op_ctx); 7206 mlxsw_sp_router_fib_abort(mlxsw_sp); 7207 mlxsw_sp_fib6_offload_failed_flag_set(mlxsw_sp, 7208 fib6_event->rt_arr, 7209 fib6_event->nrt6); 7210 } 7211 mlxsw_sp_router_fib6_event_fini(&fib_event->fib6_event); 7212 break; 7213 case FIB_EVENT_ENTRY_DEL: 7214 err = mlxsw_sp_router_fib6_del(mlxsw_sp, op_ctx, fib_event->fib6_event.rt_arr, 7215 fib_event->fib6_event.nrt6); 7216 if (err) 7217 mlxsw_sp_fib_entry_op_ctx_priv_put_all(op_ctx); 7218 mlxsw_sp_router_fib6_event_fini(&fib_event->fib6_event); 7219 break; 7220 } 7221 } 7222 7223 static void mlxsw_sp_router_fibmr_event_process(struct mlxsw_sp *mlxsw_sp, 7224 struct mlxsw_sp_fib_event *fib_event) 7225 { 7226 bool replace; 7227 int err; 7228 7229 rtnl_lock(); 7230 mutex_lock(&mlxsw_sp->router->lock); 7231 switch (fib_event->event) { 7232 case FIB_EVENT_ENTRY_REPLACE: 7233 case FIB_EVENT_ENTRY_ADD: 7234 replace = fib_event->event == FIB_EVENT_ENTRY_REPLACE; 7235 7236 err = mlxsw_sp_router_fibmr_add(mlxsw_sp, &fib_event->men_info, replace); 7237 if (err) 7238 mlxsw_sp_router_fib_abort(mlxsw_sp); 7239 mr_cache_put(fib_event->men_info.mfc); 7240 break; 7241 case FIB_EVENT_ENTRY_DEL: 7242 mlxsw_sp_router_fibmr_del(mlxsw_sp, &fib_event->men_info); 7243 mr_cache_put(fib_event->men_info.mfc); 7244 break; 7245 case FIB_EVENT_VIF_ADD: 7246 err = mlxsw_sp_router_fibmr_vif_add(mlxsw_sp, 7247 &fib_event->ven_info); 7248 if (err) 7249 mlxsw_sp_router_fib_abort(mlxsw_sp); 7250 dev_put(fib_event->ven_info.dev); 7251 break; 7252 case FIB_EVENT_VIF_DEL: 7253 mlxsw_sp_router_fibmr_vif_del(mlxsw_sp, &fib_event->ven_info); 7254 dev_put(fib_event->ven_info.dev); 7255 break; 7256 } 7257 mutex_unlock(&mlxsw_sp->router->lock); 7258 rtnl_unlock(); 7259 } 7260 7261 static void mlxsw_sp_router_fib_event_work(struct work_struct *work) 7262 { 7263 struct mlxsw_sp_router *router = container_of(work, struct mlxsw_sp_router, fib_event_work); 7264 struct mlxsw_sp_fib_entry_op_ctx *op_ctx = router->ll_op_ctx; 7265 struct mlxsw_sp *mlxsw_sp = router->mlxsw_sp; 7266 struct mlxsw_sp_fib_event *next_fib_event; 7267 struct mlxsw_sp_fib_event *fib_event; 7268 int last_family = AF_UNSPEC; 7269 LIST_HEAD(fib_event_queue); 7270 7271 spin_lock_bh(&router->fib_event_queue_lock); 7272 list_splice_init(&router->fib_event_queue, &fib_event_queue); 7273 spin_unlock_bh(&router->fib_event_queue_lock); 7274 7275 /* Router lock is held here to make sure per-instance 7276 * operation context is not used in between FIB4/6 events 7277 * processing. 7278 */ 7279 mutex_lock(&router->lock); 7280 mlxsw_sp_fib_entry_op_ctx_clear(op_ctx); 7281 list_for_each_entry_safe(fib_event, next_fib_event, 7282 &fib_event_queue, list) { 7283 /* Check if the next entry in the queue exists and it is 7284 * of the same type (family and event) as the currect one. 7285 * In that case it is permitted to do the bulking 7286 * of multiple FIB entries to a single register write. 7287 */ 7288 op_ctx->bulk_ok = !list_is_last(&fib_event->list, &fib_event_queue) && 7289 fib_event->family == next_fib_event->family && 7290 fib_event->event == next_fib_event->event; 7291 op_ctx->event = fib_event->event; 7292 7293 /* In case family of this and the previous entry are different, context 7294 * reinitialization is going to be needed now, indicate that. 7295 * Note that since last_family is initialized to AF_UNSPEC, this is always 7296 * going to happen for the first entry processed in the work. 7297 */ 7298 if (fib_event->family != last_family) 7299 op_ctx->initialized = false; 7300 7301 switch (fib_event->family) { 7302 case AF_INET: 7303 mlxsw_sp_router_fib4_event_process(mlxsw_sp, op_ctx, 7304 fib_event); 7305 break; 7306 case AF_INET6: 7307 mlxsw_sp_router_fib6_event_process(mlxsw_sp, op_ctx, 7308 fib_event); 7309 break; 7310 case RTNL_FAMILY_IP6MR: 7311 case RTNL_FAMILY_IPMR: 7312 /* Unlock here as inside FIBMR the lock is taken again 7313 * under RTNL. The per-instance operation context 7314 * is not used by FIBMR. 7315 */ 7316 mutex_unlock(&router->lock); 7317 mlxsw_sp_router_fibmr_event_process(mlxsw_sp, 7318 fib_event); 7319 mutex_lock(&router->lock); 7320 break; 7321 default: 7322 WARN_ON_ONCE(1); 7323 } 7324 last_family = fib_event->family; 7325 kfree(fib_event); 7326 cond_resched(); 7327 } 7328 WARN_ON_ONCE(!list_empty(&router->ll_op_ctx->fib_entry_priv_list)); 7329 mutex_unlock(&router->lock); 7330 } 7331 7332 static void mlxsw_sp_router_fib4_event(struct mlxsw_sp_fib_event *fib_event, 7333 struct fib_notifier_info *info) 7334 { 7335 struct fib_entry_notifier_info *fen_info; 7336 struct fib_nh_notifier_info *fnh_info; 7337 7338 switch (fib_event->event) { 7339 case FIB_EVENT_ENTRY_REPLACE: 7340 case FIB_EVENT_ENTRY_DEL: 7341 fen_info = container_of(info, struct fib_entry_notifier_info, 7342 info); 7343 fib_event->fen_info = *fen_info; 7344 /* Take reference on fib_info to prevent it from being 7345 * freed while event is queued. Release it afterwards. 7346 */ 7347 fib_info_hold(fib_event->fen_info.fi); 7348 break; 7349 case FIB_EVENT_NH_ADD: 7350 case FIB_EVENT_NH_DEL: 7351 fnh_info = container_of(info, struct fib_nh_notifier_info, 7352 info); 7353 fib_event->fnh_info = *fnh_info; 7354 fib_info_hold(fib_event->fnh_info.fib_nh->nh_parent); 7355 break; 7356 } 7357 } 7358 7359 static int mlxsw_sp_router_fib6_event(struct mlxsw_sp_fib_event *fib_event, 7360 struct fib_notifier_info *info) 7361 { 7362 struct fib6_entry_notifier_info *fen6_info; 7363 int err; 7364 7365 switch (fib_event->event) { 7366 case FIB_EVENT_ENTRY_REPLACE: 7367 case FIB_EVENT_ENTRY_APPEND: 7368 case FIB_EVENT_ENTRY_DEL: 7369 fen6_info = container_of(info, struct fib6_entry_notifier_info, 7370 info); 7371 err = mlxsw_sp_router_fib6_event_init(&fib_event->fib6_event, 7372 fen6_info); 7373 if (err) 7374 return err; 7375 break; 7376 } 7377 7378 return 0; 7379 } 7380 7381 static void 7382 mlxsw_sp_router_fibmr_event(struct mlxsw_sp_fib_event *fib_event, 7383 struct fib_notifier_info *info) 7384 { 7385 switch (fib_event->event) { 7386 case FIB_EVENT_ENTRY_REPLACE: 7387 case FIB_EVENT_ENTRY_ADD: 7388 case FIB_EVENT_ENTRY_DEL: 7389 memcpy(&fib_event->men_info, info, sizeof(fib_event->men_info)); 7390 mr_cache_hold(fib_event->men_info.mfc); 7391 break; 7392 case FIB_EVENT_VIF_ADD: 7393 case FIB_EVENT_VIF_DEL: 7394 memcpy(&fib_event->ven_info, info, sizeof(fib_event->ven_info)); 7395 dev_hold(fib_event->ven_info.dev); 7396 break; 7397 } 7398 } 7399 7400 static int mlxsw_sp_router_fib_rule_event(unsigned long event, 7401 struct fib_notifier_info *info, 7402 struct mlxsw_sp *mlxsw_sp) 7403 { 7404 struct netlink_ext_ack *extack = info->extack; 7405 struct fib_rule_notifier_info *fr_info; 7406 struct fib_rule *rule; 7407 int err = 0; 7408 7409 /* nothing to do at the moment */ 7410 if (event == FIB_EVENT_RULE_DEL) 7411 return 0; 7412 7413 if (mlxsw_sp->router->aborted) 7414 return 0; 7415 7416 fr_info = container_of(info, struct fib_rule_notifier_info, info); 7417 rule = fr_info->rule; 7418 7419 /* Rule only affects locally generated traffic */ 7420 if (rule->iifindex == mlxsw_sp_net(mlxsw_sp)->loopback_dev->ifindex) 7421 return 0; 7422 7423 switch (info->family) { 7424 case AF_INET: 7425 if (!fib4_rule_default(rule) && !rule->l3mdev) 7426 err = -EOPNOTSUPP; 7427 break; 7428 case AF_INET6: 7429 if (!fib6_rule_default(rule) && !rule->l3mdev) 7430 err = -EOPNOTSUPP; 7431 break; 7432 case RTNL_FAMILY_IPMR: 7433 if (!ipmr_rule_default(rule) && !rule->l3mdev) 7434 err = -EOPNOTSUPP; 7435 break; 7436 case RTNL_FAMILY_IP6MR: 7437 if (!ip6mr_rule_default(rule) && !rule->l3mdev) 7438 err = -EOPNOTSUPP; 7439 break; 7440 } 7441 7442 if (err < 0) 7443 NL_SET_ERR_MSG_MOD(extack, "FIB rules not supported"); 7444 7445 return err; 7446 } 7447 7448 /* Called with rcu_read_lock() */ 7449 static int mlxsw_sp_router_fib_event(struct notifier_block *nb, 7450 unsigned long event, void *ptr) 7451 { 7452 struct mlxsw_sp_fib_event *fib_event; 7453 struct fib_notifier_info *info = ptr; 7454 struct mlxsw_sp_router *router; 7455 int err; 7456 7457 if ((info->family != AF_INET && info->family != AF_INET6 && 7458 info->family != RTNL_FAMILY_IPMR && 7459 info->family != RTNL_FAMILY_IP6MR)) 7460 return NOTIFY_DONE; 7461 7462 router = container_of(nb, struct mlxsw_sp_router, fib_nb); 7463 7464 switch (event) { 7465 case FIB_EVENT_RULE_ADD: 7466 case FIB_EVENT_RULE_DEL: 7467 err = mlxsw_sp_router_fib_rule_event(event, info, 7468 router->mlxsw_sp); 7469 return notifier_from_errno(err); 7470 case FIB_EVENT_ENTRY_ADD: 7471 case FIB_EVENT_ENTRY_REPLACE: 7472 case FIB_EVENT_ENTRY_APPEND: 7473 if (router->aborted) { 7474 NL_SET_ERR_MSG_MOD(info->extack, "FIB offload was aborted. Not configuring route"); 7475 return notifier_from_errno(-EINVAL); 7476 } 7477 if (info->family == AF_INET) { 7478 struct fib_entry_notifier_info *fen_info = ptr; 7479 7480 if (fen_info->fi->fib_nh_is_v6) { 7481 NL_SET_ERR_MSG_MOD(info->extack, "IPv6 gateway with IPv4 route is not supported"); 7482 return notifier_from_errno(-EINVAL); 7483 } 7484 } 7485 break; 7486 } 7487 7488 fib_event = kzalloc(sizeof(*fib_event), GFP_ATOMIC); 7489 if (!fib_event) 7490 return NOTIFY_BAD; 7491 7492 fib_event->mlxsw_sp = router->mlxsw_sp; 7493 fib_event->event = event; 7494 fib_event->family = info->family; 7495 7496 switch (info->family) { 7497 case AF_INET: 7498 mlxsw_sp_router_fib4_event(fib_event, info); 7499 break; 7500 case AF_INET6: 7501 err = mlxsw_sp_router_fib6_event(fib_event, info); 7502 if (err) 7503 goto err_fib_event; 7504 break; 7505 case RTNL_FAMILY_IP6MR: 7506 case RTNL_FAMILY_IPMR: 7507 mlxsw_sp_router_fibmr_event(fib_event, info); 7508 break; 7509 } 7510 7511 /* Enqueue the event and trigger the work */ 7512 spin_lock_bh(&router->fib_event_queue_lock); 7513 list_add_tail(&fib_event->list, &router->fib_event_queue); 7514 spin_unlock_bh(&router->fib_event_queue_lock); 7515 mlxsw_core_schedule_work(&router->fib_event_work); 7516 7517 return NOTIFY_DONE; 7518 7519 err_fib_event: 7520 kfree(fib_event); 7521 return NOTIFY_BAD; 7522 } 7523 7524 static struct mlxsw_sp_rif * 7525 mlxsw_sp_rif_find_by_dev(const struct mlxsw_sp *mlxsw_sp, 7526 const struct net_device *dev) 7527 { 7528 int i; 7529 7530 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++) 7531 if (mlxsw_sp->router->rifs[i] && 7532 mlxsw_sp->router->rifs[i]->dev == dev) 7533 return mlxsw_sp->router->rifs[i]; 7534 7535 return NULL; 7536 } 7537 7538 bool mlxsw_sp_rif_exists(struct mlxsw_sp *mlxsw_sp, 7539 const struct net_device *dev) 7540 { 7541 struct mlxsw_sp_rif *rif; 7542 7543 mutex_lock(&mlxsw_sp->router->lock); 7544 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev); 7545 mutex_unlock(&mlxsw_sp->router->lock); 7546 7547 return rif; 7548 } 7549 7550 u16 mlxsw_sp_rif_vid(struct mlxsw_sp *mlxsw_sp, const struct net_device *dev) 7551 { 7552 struct mlxsw_sp_rif *rif; 7553 u16 vid = 0; 7554 7555 mutex_lock(&mlxsw_sp->router->lock); 7556 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev); 7557 if (!rif) 7558 goto out; 7559 7560 /* We only return the VID for VLAN RIFs. Otherwise we return an 7561 * invalid value (0). 7562 */ 7563 if (rif->ops->type != MLXSW_SP_RIF_TYPE_VLAN) 7564 goto out; 7565 7566 vid = mlxsw_sp_fid_8021q_vid(rif->fid); 7567 7568 out: 7569 mutex_unlock(&mlxsw_sp->router->lock); 7570 return vid; 7571 } 7572 7573 static int mlxsw_sp_router_rif_disable(struct mlxsw_sp *mlxsw_sp, u16 rif) 7574 { 7575 char ritr_pl[MLXSW_REG_RITR_LEN]; 7576 int err; 7577 7578 mlxsw_reg_ritr_rif_pack(ritr_pl, rif); 7579 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 7580 if (err) 7581 return err; 7582 7583 mlxsw_reg_ritr_enable_set(ritr_pl, false); 7584 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 7585 } 7586 7587 static void mlxsw_sp_router_rif_gone_sync(struct mlxsw_sp *mlxsw_sp, 7588 struct mlxsw_sp_rif *rif) 7589 { 7590 mlxsw_sp_router_rif_disable(mlxsw_sp, rif->rif_index); 7591 mlxsw_sp_nexthop_rif_gone_sync(mlxsw_sp, rif); 7592 mlxsw_sp_neigh_rif_gone_sync(mlxsw_sp, rif); 7593 } 7594 7595 static bool 7596 mlxsw_sp_rif_should_config(struct mlxsw_sp_rif *rif, struct net_device *dev, 7597 unsigned long event) 7598 { 7599 struct inet6_dev *inet6_dev; 7600 bool addr_list_empty = true; 7601 struct in_device *idev; 7602 7603 switch (event) { 7604 case NETDEV_UP: 7605 return rif == NULL; 7606 case NETDEV_DOWN: 7607 rcu_read_lock(); 7608 idev = __in_dev_get_rcu(dev); 7609 if (idev && idev->ifa_list) 7610 addr_list_empty = false; 7611 7612 inet6_dev = __in6_dev_get(dev); 7613 if (addr_list_empty && inet6_dev && 7614 !list_empty(&inet6_dev->addr_list)) 7615 addr_list_empty = false; 7616 rcu_read_unlock(); 7617 7618 /* macvlans do not have a RIF, but rather piggy back on the 7619 * RIF of their lower device. 7620 */ 7621 if (netif_is_macvlan(dev) && addr_list_empty) 7622 return true; 7623 7624 if (rif && addr_list_empty && 7625 !netif_is_l3_slave(rif->dev)) 7626 return true; 7627 /* It is possible we already removed the RIF ourselves 7628 * if it was assigned to a netdev that is now a bridge 7629 * or LAG slave. 7630 */ 7631 return false; 7632 } 7633 7634 return false; 7635 } 7636 7637 static enum mlxsw_sp_rif_type 7638 mlxsw_sp_dev_rif_type(const struct mlxsw_sp *mlxsw_sp, 7639 const struct net_device *dev) 7640 { 7641 enum mlxsw_sp_fid_type type; 7642 7643 if (mlxsw_sp_netdev_ipip_type(mlxsw_sp, dev, NULL)) 7644 return MLXSW_SP_RIF_TYPE_IPIP_LB; 7645 7646 /* Otherwise RIF type is derived from the type of the underlying FID. */ 7647 if (is_vlan_dev(dev) && netif_is_bridge_master(vlan_dev_real_dev(dev))) 7648 type = MLXSW_SP_FID_TYPE_8021Q; 7649 else if (netif_is_bridge_master(dev) && br_vlan_enabled(dev)) 7650 type = MLXSW_SP_FID_TYPE_8021Q; 7651 else if (netif_is_bridge_master(dev)) 7652 type = MLXSW_SP_FID_TYPE_8021D; 7653 else 7654 type = MLXSW_SP_FID_TYPE_RFID; 7655 7656 return mlxsw_sp_fid_type_rif_type(mlxsw_sp, type); 7657 } 7658 7659 static int mlxsw_sp_rif_index_alloc(struct mlxsw_sp *mlxsw_sp, u16 *p_rif_index) 7660 { 7661 int i; 7662 7663 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++) { 7664 if (!mlxsw_sp->router->rifs[i]) { 7665 *p_rif_index = i; 7666 return 0; 7667 } 7668 } 7669 7670 return -ENOBUFS; 7671 } 7672 7673 static struct mlxsw_sp_rif *mlxsw_sp_rif_alloc(size_t rif_size, u16 rif_index, 7674 u16 vr_id, 7675 struct net_device *l3_dev) 7676 { 7677 struct mlxsw_sp_rif *rif; 7678 7679 rif = kzalloc(rif_size, GFP_KERNEL); 7680 if (!rif) 7681 return NULL; 7682 7683 INIT_LIST_HEAD(&rif->nexthop_list); 7684 INIT_LIST_HEAD(&rif->neigh_list); 7685 if (l3_dev) { 7686 ether_addr_copy(rif->addr, l3_dev->dev_addr); 7687 rif->mtu = l3_dev->mtu; 7688 rif->dev = l3_dev; 7689 } 7690 rif->vr_id = vr_id; 7691 rif->rif_index = rif_index; 7692 7693 return rif; 7694 } 7695 7696 struct mlxsw_sp_rif *mlxsw_sp_rif_by_index(const struct mlxsw_sp *mlxsw_sp, 7697 u16 rif_index) 7698 { 7699 return mlxsw_sp->router->rifs[rif_index]; 7700 } 7701 7702 u16 mlxsw_sp_rif_index(const struct mlxsw_sp_rif *rif) 7703 { 7704 return rif->rif_index; 7705 } 7706 7707 u16 mlxsw_sp_ipip_lb_rif_index(const struct mlxsw_sp_rif_ipip_lb *lb_rif) 7708 { 7709 return lb_rif->common.rif_index; 7710 } 7711 7712 u16 mlxsw_sp_ipip_lb_ul_vr_id(const struct mlxsw_sp_rif_ipip_lb *lb_rif) 7713 { 7714 u32 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(lb_rif->common.dev); 7715 struct mlxsw_sp_vr *ul_vr; 7716 7717 ul_vr = mlxsw_sp_vr_get(lb_rif->common.mlxsw_sp, ul_tb_id, NULL); 7718 if (WARN_ON(IS_ERR(ul_vr))) 7719 return 0; 7720 7721 return ul_vr->id; 7722 } 7723 7724 u16 mlxsw_sp_ipip_lb_ul_rif_id(const struct mlxsw_sp_rif_ipip_lb *lb_rif) 7725 { 7726 return lb_rif->ul_rif_id; 7727 } 7728 7729 int mlxsw_sp_rif_dev_ifindex(const struct mlxsw_sp_rif *rif) 7730 { 7731 return rif->dev->ifindex; 7732 } 7733 7734 const struct net_device *mlxsw_sp_rif_dev(const struct mlxsw_sp_rif *rif) 7735 { 7736 return rif->dev; 7737 } 7738 7739 static struct mlxsw_sp_rif * 7740 mlxsw_sp_rif_create(struct mlxsw_sp *mlxsw_sp, 7741 const struct mlxsw_sp_rif_params *params, 7742 struct netlink_ext_ack *extack) 7743 { 7744 u32 tb_id = l3mdev_fib_table(params->dev); 7745 const struct mlxsw_sp_rif_ops *ops; 7746 struct mlxsw_sp_fid *fid = NULL; 7747 enum mlxsw_sp_rif_type type; 7748 struct mlxsw_sp_rif *rif; 7749 struct mlxsw_sp_vr *vr; 7750 u16 rif_index; 7751 int i, err; 7752 7753 type = mlxsw_sp_dev_rif_type(mlxsw_sp, params->dev); 7754 ops = mlxsw_sp->router->rif_ops_arr[type]; 7755 7756 vr = mlxsw_sp_vr_get(mlxsw_sp, tb_id ? : RT_TABLE_MAIN, extack); 7757 if (IS_ERR(vr)) 7758 return ERR_CAST(vr); 7759 vr->rif_count++; 7760 7761 err = mlxsw_sp_rif_index_alloc(mlxsw_sp, &rif_index); 7762 if (err) { 7763 NL_SET_ERR_MSG_MOD(extack, "Exceeded number of supported router interfaces"); 7764 goto err_rif_index_alloc; 7765 } 7766 7767 rif = mlxsw_sp_rif_alloc(ops->rif_size, rif_index, vr->id, params->dev); 7768 if (!rif) { 7769 err = -ENOMEM; 7770 goto err_rif_alloc; 7771 } 7772 dev_hold(rif->dev); 7773 mlxsw_sp->router->rifs[rif_index] = rif; 7774 rif->mlxsw_sp = mlxsw_sp; 7775 rif->ops = ops; 7776 7777 if (ops->fid_get) { 7778 fid = ops->fid_get(rif, extack); 7779 if (IS_ERR(fid)) { 7780 err = PTR_ERR(fid); 7781 goto err_fid_get; 7782 } 7783 rif->fid = fid; 7784 } 7785 7786 if (ops->setup) 7787 ops->setup(rif, params); 7788 7789 err = ops->configure(rif); 7790 if (err) 7791 goto err_configure; 7792 7793 for (i = 0; i < MLXSW_SP_L3_PROTO_MAX; i++) { 7794 err = mlxsw_sp_mr_rif_add(vr->mr_table[i], rif); 7795 if (err) 7796 goto err_mr_rif_add; 7797 } 7798 7799 mlxsw_sp_rif_counters_alloc(rif); 7800 7801 return rif; 7802 7803 err_mr_rif_add: 7804 for (i--; i >= 0; i--) 7805 mlxsw_sp_mr_rif_del(vr->mr_table[i], rif); 7806 ops->deconfigure(rif); 7807 err_configure: 7808 if (fid) 7809 mlxsw_sp_fid_put(fid); 7810 err_fid_get: 7811 mlxsw_sp->router->rifs[rif_index] = NULL; 7812 dev_put(rif->dev); 7813 kfree(rif); 7814 err_rif_alloc: 7815 err_rif_index_alloc: 7816 vr->rif_count--; 7817 mlxsw_sp_vr_put(mlxsw_sp, vr); 7818 return ERR_PTR(err); 7819 } 7820 7821 static void mlxsw_sp_rif_destroy(struct mlxsw_sp_rif *rif) 7822 { 7823 const struct mlxsw_sp_rif_ops *ops = rif->ops; 7824 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 7825 struct mlxsw_sp_fid *fid = rif->fid; 7826 struct mlxsw_sp_vr *vr; 7827 int i; 7828 7829 mlxsw_sp_router_rif_gone_sync(mlxsw_sp, rif); 7830 vr = &mlxsw_sp->router->vrs[rif->vr_id]; 7831 7832 mlxsw_sp_rif_counters_free(rif); 7833 for (i = 0; i < MLXSW_SP_L3_PROTO_MAX; i++) 7834 mlxsw_sp_mr_rif_del(vr->mr_table[i], rif); 7835 ops->deconfigure(rif); 7836 if (fid) 7837 /* Loopback RIFs are not associated with a FID. */ 7838 mlxsw_sp_fid_put(fid); 7839 mlxsw_sp->router->rifs[rif->rif_index] = NULL; 7840 dev_put(rif->dev); 7841 kfree(rif); 7842 vr->rif_count--; 7843 mlxsw_sp_vr_put(mlxsw_sp, vr); 7844 } 7845 7846 void mlxsw_sp_rif_destroy_by_dev(struct mlxsw_sp *mlxsw_sp, 7847 struct net_device *dev) 7848 { 7849 struct mlxsw_sp_rif *rif; 7850 7851 mutex_lock(&mlxsw_sp->router->lock); 7852 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev); 7853 if (!rif) 7854 goto out; 7855 mlxsw_sp_rif_destroy(rif); 7856 out: 7857 mutex_unlock(&mlxsw_sp->router->lock); 7858 } 7859 7860 static void 7861 mlxsw_sp_rif_subport_params_init(struct mlxsw_sp_rif_params *params, 7862 struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan) 7863 { 7864 struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port; 7865 7866 params->vid = mlxsw_sp_port_vlan->vid; 7867 params->lag = mlxsw_sp_port->lagged; 7868 if (params->lag) 7869 params->lag_id = mlxsw_sp_port->lag_id; 7870 else 7871 params->system_port = mlxsw_sp_port->local_port; 7872 } 7873 7874 static struct mlxsw_sp_rif_subport * 7875 mlxsw_sp_rif_subport_rif(const struct mlxsw_sp_rif *rif) 7876 { 7877 return container_of(rif, struct mlxsw_sp_rif_subport, common); 7878 } 7879 7880 static struct mlxsw_sp_rif * 7881 mlxsw_sp_rif_subport_get(struct mlxsw_sp *mlxsw_sp, 7882 const struct mlxsw_sp_rif_params *params, 7883 struct netlink_ext_ack *extack) 7884 { 7885 struct mlxsw_sp_rif_subport *rif_subport; 7886 struct mlxsw_sp_rif *rif; 7887 7888 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, params->dev); 7889 if (!rif) 7890 return mlxsw_sp_rif_create(mlxsw_sp, params, extack); 7891 7892 rif_subport = mlxsw_sp_rif_subport_rif(rif); 7893 refcount_inc(&rif_subport->ref_count); 7894 return rif; 7895 } 7896 7897 static void mlxsw_sp_rif_subport_put(struct mlxsw_sp_rif *rif) 7898 { 7899 struct mlxsw_sp_rif_subport *rif_subport; 7900 7901 rif_subport = mlxsw_sp_rif_subport_rif(rif); 7902 if (!refcount_dec_and_test(&rif_subport->ref_count)) 7903 return; 7904 7905 mlxsw_sp_rif_destroy(rif); 7906 } 7907 7908 static int 7909 __mlxsw_sp_port_vlan_router_join(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan, 7910 struct net_device *l3_dev, 7911 struct netlink_ext_ack *extack) 7912 { 7913 struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port; 7914 struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp; 7915 struct mlxsw_sp_rif_params params = { 7916 .dev = l3_dev, 7917 }; 7918 u16 vid = mlxsw_sp_port_vlan->vid; 7919 struct mlxsw_sp_rif *rif; 7920 struct mlxsw_sp_fid *fid; 7921 int err; 7922 7923 mlxsw_sp_rif_subport_params_init(¶ms, mlxsw_sp_port_vlan); 7924 rif = mlxsw_sp_rif_subport_get(mlxsw_sp, ¶ms, extack); 7925 if (IS_ERR(rif)) 7926 return PTR_ERR(rif); 7927 7928 /* FID was already created, just take a reference */ 7929 fid = rif->ops->fid_get(rif, extack); 7930 err = mlxsw_sp_fid_port_vid_map(fid, mlxsw_sp_port, vid); 7931 if (err) 7932 goto err_fid_port_vid_map; 7933 7934 err = mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, false); 7935 if (err) 7936 goto err_port_vid_learning_set; 7937 7938 err = mlxsw_sp_port_vid_stp_set(mlxsw_sp_port, vid, 7939 BR_STATE_FORWARDING); 7940 if (err) 7941 goto err_port_vid_stp_set; 7942 7943 mlxsw_sp_port_vlan->fid = fid; 7944 7945 return 0; 7946 7947 err_port_vid_stp_set: 7948 mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, true); 7949 err_port_vid_learning_set: 7950 mlxsw_sp_fid_port_vid_unmap(fid, mlxsw_sp_port, vid); 7951 err_fid_port_vid_map: 7952 mlxsw_sp_fid_put(fid); 7953 mlxsw_sp_rif_subport_put(rif); 7954 return err; 7955 } 7956 7957 static void 7958 __mlxsw_sp_port_vlan_router_leave(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan) 7959 { 7960 struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port; 7961 struct mlxsw_sp_fid *fid = mlxsw_sp_port_vlan->fid; 7962 struct mlxsw_sp_rif *rif = mlxsw_sp_fid_rif(fid); 7963 u16 vid = mlxsw_sp_port_vlan->vid; 7964 7965 if (WARN_ON(mlxsw_sp_fid_type(fid) != MLXSW_SP_FID_TYPE_RFID)) 7966 return; 7967 7968 mlxsw_sp_port_vlan->fid = NULL; 7969 mlxsw_sp_port_vid_stp_set(mlxsw_sp_port, vid, BR_STATE_BLOCKING); 7970 mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, true); 7971 mlxsw_sp_fid_port_vid_unmap(fid, mlxsw_sp_port, vid); 7972 mlxsw_sp_fid_put(fid); 7973 mlxsw_sp_rif_subport_put(rif); 7974 } 7975 7976 int 7977 mlxsw_sp_port_vlan_router_join(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan, 7978 struct net_device *l3_dev, 7979 struct netlink_ext_ack *extack) 7980 { 7981 struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port_vlan->mlxsw_sp_port->mlxsw_sp; 7982 struct mlxsw_sp_rif *rif; 7983 int err = 0; 7984 7985 mutex_lock(&mlxsw_sp->router->lock); 7986 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev); 7987 if (!rif) 7988 goto out; 7989 7990 err = __mlxsw_sp_port_vlan_router_join(mlxsw_sp_port_vlan, l3_dev, 7991 extack); 7992 out: 7993 mutex_unlock(&mlxsw_sp->router->lock); 7994 return err; 7995 } 7996 7997 void 7998 mlxsw_sp_port_vlan_router_leave(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan) 7999 { 8000 struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port_vlan->mlxsw_sp_port->mlxsw_sp; 8001 8002 mutex_lock(&mlxsw_sp->router->lock); 8003 __mlxsw_sp_port_vlan_router_leave(mlxsw_sp_port_vlan); 8004 mutex_unlock(&mlxsw_sp->router->lock); 8005 } 8006 8007 static int mlxsw_sp_inetaddr_port_vlan_event(struct net_device *l3_dev, 8008 struct net_device *port_dev, 8009 unsigned long event, u16 vid, 8010 struct netlink_ext_ack *extack) 8011 { 8012 struct mlxsw_sp_port *mlxsw_sp_port = netdev_priv(port_dev); 8013 struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan; 8014 8015 mlxsw_sp_port_vlan = mlxsw_sp_port_vlan_find_by_vid(mlxsw_sp_port, vid); 8016 if (WARN_ON(!mlxsw_sp_port_vlan)) 8017 return -EINVAL; 8018 8019 switch (event) { 8020 case NETDEV_UP: 8021 return __mlxsw_sp_port_vlan_router_join(mlxsw_sp_port_vlan, 8022 l3_dev, extack); 8023 case NETDEV_DOWN: 8024 __mlxsw_sp_port_vlan_router_leave(mlxsw_sp_port_vlan); 8025 break; 8026 } 8027 8028 return 0; 8029 } 8030 8031 static int mlxsw_sp_inetaddr_port_event(struct net_device *port_dev, 8032 unsigned long event, 8033 struct netlink_ext_ack *extack) 8034 { 8035 if (netif_is_bridge_port(port_dev) || 8036 netif_is_lag_port(port_dev) || 8037 netif_is_ovs_port(port_dev)) 8038 return 0; 8039 8040 return mlxsw_sp_inetaddr_port_vlan_event(port_dev, port_dev, event, 8041 MLXSW_SP_DEFAULT_VID, extack); 8042 } 8043 8044 static int __mlxsw_sp_inetaddr_lag_event(struct net_device *l3_dev, 8045 struct net_device *lag_dev, 8046 unsigned long event, u16 vid, 8047 struct netlink_ext_ack *extack) 8048 { 8049 struct net_device *port_dev; 8050 struct list_head *iter; 8051 int err; 8052 8053 netdev_for_each_lower_dev(lag_dev, port_dev, iter) { 8054 if (mlxsw_sp_port_dev_check(port_dev)) { 8055 err = mlxsw_sp_inetaddr_port_vlan_event(l3_dev, 8056 port_dev, 8057 event, vid, 8058 extack); 8059 if (err) 8060 return err; 8061 } 8062 } 8063 8064 return 0; 8065 } 8066 8067 static int mlxsw_sp_inetaddr_lag_event(struct net_device *lag_dev, 8068 unsigned long event, 8069 struct netlink_ext_ack *extack) 8070 { 8071 if (netif_is_bridge_port(lag_dev)) 8072 return 0; 8073 8074 return __mlxsw_sp_inetaddr_lag_event(lag_dev, lag_dev, event, 8075 MLXSW_SP_DEFAULT_VID, extack); 8076 } 8077 8078 static int mlxsw_sp_inetaddr_bridge_event(struct mlxsw_sp *mlxsw_sp, 8079 struct net_device *l3_dev, 8080 unsigned long event, 8081 struct netlink_ext_ack *extack) 8082 { 8083 struct mlxsw_sp_rif_params params = { 8084 .dev = l3_dev, 8085 }; 8086 struct mlxsw_sp_rif *rif; 8087 8088 switch (event) { 8089 case NETDEV_UP: 8090 if (netif_is_bridge_master(l3_dev) && br_vlan_enabled(l3_dev)) { 8091 u16 proto; 8092 8093 br_vlan_get_proto(l3_dev, &proto); 8094 if (proto == ETH_P_8021AD) { 8095 NL_SET_ERR_MSG_MOD(extack, "Adding an IP address to 802.1ad bridge is not supported"); 8096 return -EOPNOTSUPP; 8097 } 8098 } 8099 rif = mlxsw_sp_rif_create(mlxsw_sp, ¶ms, extack); 8100 if (IS_ERR(rif)) 8101 return PTR_ERR(rif); 8102 break; 8103 case NETDEV_DOWN: 8104 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev); 8105 mlxsw_sp_rif_destroy(rif); 8106 break; 8107 } 8108 8109 return 0; 8110 } 8111 8112 static int mlxsw_sp_inetaddr_vlan_event(struct mlxsw_sp *mlxsw_sp, 8113 struct net_device *vlan_dev, 8114 unsigned long event, 8115 struct netlink_ext_ack *extack) 8116 { 8117 struct net_device *real_dev = vlan_dev_real_dev(vlan_dev); 8118 u16 vid = vlan_dev_vlan_id(vlan_dev); 8119 8120 if (netif_is_bridge_port(vlan_dev)) 8121 return 0; 8122 8123 if (mlxsw_sp_port_dev_check(real_dev)) 8124 return mlxsw_sp_inetaddr_port_vlan_event(vlan_dev, real_dev, 8125 event, vid, extack); 8126 else if (netif_is_lag_master(real_dev)) 8127 return __mlxsw_sp_inetaddr_lag_event(vlan_dev, real_dev, event, 8128 vid, extack); 8129 else if (netif_is_bridge_master(real_dev) && br_vlan_enabled(real_dev)) 8130 return mlxsw_sp_inetaddr_bridge_event(mlxsw_sp, vlan_dev, event, 8131 extack); 8132 8133 return 0; 8134 } 8135 8136 static bool mlxsw_sp_rif_macvlan_is_vrrp4(const u8 *mac) 8137 { 8138 u8 vrrp4[ETH_ALEN] = { 0x00, 0x00, 0x5e, 0x00, 0x01, 0x00 }; 8139 u8 mask[ETH_ALEN] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0x00 }; 8140 8141 return ether_addr_equal_masked(mac, vrrp4, mask); 8142 } 8143 8144 static bool mlxsw_sp_rif_macvlan_is_vrrp6(const u8 *mac) 8145 { 8146 u8 vrrp6[ETH_ALEN] = { 0x00, 0x00, 0x5e, 0x00, 0x02, 0x00 }; 8147 u8 mask[ETH_ALEN] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0x00 }; 8148 8149 return ether_addr_equal_masked(mac, vrrp6, mask); 8150 } 8151 8152 static int mlxsw_sp_rif_vrrp_op(struct mlxsw_sp *mlxsw_sp, u16 rif_index, 8153 const u8 *mac, bool adding) 8154 { 8155 char ritr_pl[MLXSW_REG_RITR_LEN]; 8156 u8 vrrp_id = adding ? mac[5] : 0; 8157 int err; 8158 8159 if (!mlxsw_sp_rif_macvlan_is_vrrp4(mac) && 8160 !mlxsw_sp_rif_macvlan_is_vrrp6(mac)) 8161 return 0; 8162 8163 mlxsw_reg_ritr_rif_pack(ritr_pl, rif_index); 8164 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 8165 if (err) 8166 return err; 8167 8168 if (mlxsw_sp_rif_macvlan_is_vrrp4(mac)) 8169 mlxsw_reg_ritr_if_vrrp_id_ipv4_set(ritr_pl, vrrp_id); 8170 else 8171 mlxsw_reg_ritr_if_vrrp_id_ipv6_set(ritr_pl, vrrp_id); 8172 8173 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 8174 } 8175 8176 static int mlxsw_sp_rif_macvlan_add(struct mlxsw_sp *mlxsw_sp, 8177 const struct net_device *macvlan_dev, 8178 struct netlink_ext_ack *extack) 8179 { 8180 struct macvlan_dev *vlan = netdev_priv(macvlan_dev); 8181 struct mlxsw_sp_rif *rif; 8182 int err; 8183 8184 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, vlan->lowerdev); 8185 if (!rif) { 8186 NL_SET_ERR_MSG_MOD(extack, "macvlan is only supported on top of router interfaces"); 8187 return -EOPNOTSUPP; 8188 } 8189 8190 err = mlxsw_sp_rif_fdb_op(mlxsw_sp, macvlan_dev->dev_addr, 8191 mlxsw_sp_fid_index(rif->fid), true); 8192 if (err) 8193 return err; 8194 8195 err = mlxsw_sp_rif_vrrp_op(mlxsw_sp, rif->rif_index, 8196 macvlan_dev->dev_addr, true); 8197 if (err) 8198 goto err_rif_vrrp_add; 8199 8200 /* Make sure the bridge driver does not have this MAC pointing at 8201 * some other port. 8202 */ 8203 if (rif->ops->fdb_del) 8204 rif->ops->fdb_del(rif, macvlan_dev->dev_addr); 8205 8206 return 0; 8207 8208 err_rif_vrrp_add: 8209 mlxsw_sp_rif_fdb_op(mlxsw_sp, macvlan_dev->dev_addr, 8210 mlxsw_sp_fid_index(rif->fid), false); 8211 return err; 8212 } 8213 8214 static void __mlxsw_sp_rif_macvlan_del(struct mlxsw_sp *mlxsw_sp, 8215 const struct net_device *macvlan_dev) 8216 { 8217 struct macvlan_dev *vlan = netdev_priv(macvlan_dev); 8218 struct mlxsw_sp_rif *rif; 8219 8220 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, vlan->lowerdev); 8221 /* If we do not have a RIF, then we already took care of 8222 * removing the macvlan's MAC during RIF deletion. 8223 */ 8224 if (!rif) 8225 return; 8226 mlxsw_sp_rif_vrrp_op(mlxsw_sp, rif->rif_index, macvlan_dev->dev_addr, 8227 false); 8228 mlxsw_sp_rif_fdb_op(mlxsw_sp, macvlan_dev->dev_addr, 8229 mlxsw_sp_fid_index(rif->fid), false); 8230 } 8231 8232 void mlxsw_sp_rif_macvlan_del(struct mlxsw_sp *mlxsw_sp, 8233 const struct net_device *macvlan_dev) 8234 { 8235 mutex_lock(&mlxsw_sp->router->lock); 8236 __mlxsw_sp_rif_macvlan_del(mlxsw_sp, macvlan_dev); 8237 mutex_unlock(&mlxsw_sp->router->lock); 8238 } 8239 8240 static int mlxsw_sp_inetaddr_macvlan_event(struct mlxsw_sp *mlxsw_sp, 8241 struct net_device *macvlan_dev, 8242 unsigned long event, 8243 struct netlink_ext_ack *extack) 8244 { 8245 switch (event) { 8246 case NETDEV_UP: 8247 return mlxsw_sp_rif_macvlan_add(mlxsw_sp, macvlan_dev, extack); 8248 case NETDEV_DOWN: 8249 __mlxsw_sp_rif_macvlan_del(mlxsw_sp, macvlan_dev); 8250 break; 8251 } 8252 8253 return 0; 8254 } 8255 8256 static int mlxsw_sp_router_port_check_rif_addr(struct mlxsw_sp *mlxsw_sp, 8257 struct net_device *dev, 8258 const unsigned char *dev_addr, 8259 struct netlink_ext_ack *extack) 8260 { 8261 struct mlxsw_sp_rif *rif; 8262 int i; 8263 8264 /* A RIF is not created for macvlan netdevs. Their MAC is used to 8265 * populate the FDB 8266 */ 8267 if (netif_is_macvlan(dev) || netif_is_l3_master(dev)) 8268 return 0; 8269 8270 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++) { 8271 rif = mlxsw_sp->router->rifs[i]; 8272 if (rif && rif->ops && 8273 rif->ops->type == MLXSW_SP_RIF_TYPE_IPIP_LB) 8274 continue; 8275 if (rif && rif->dev && rif->dev != dev && 8276 !ether_addr_equal_masked(rif->dev->dev_addr, dev_addr, 8277 mlxsw_sp->mac_mask)) { 8278 NL_SET_ERR_MSG_MOD(extack, "All router interface MAC addresses must have the same prefix"); 8279 return -EINVAL; 8280 } 8281 } 8282 8283 return 0; 8284 } 8285 8286 static int __mlxsw_sp_inetaddr_event(struct mlxsw_sp *mlxsw_sp, 8287 struct net_device *dev, 8288 unsigned long event, 8289 struct netlink_ext_ack *extack) 8290 { 8291 if (mlxsw_sp_port_dev_check(dev)) 8292 return mlxsw_sp_inetaddr_port_event(dev, event, extack); 8293 else if (netif_is_lag_master(dev)) 8294 return mlxsw_sp_inetaddr_lag_event(dev, event, extack); 8295 else if (netif_is_bridge_master(dev)) 8296 return mlxsw_sp_inetaddr_bridge_event(mlxsw_sp, dev, event, 8297 extack); 8298 else if (is_vlan_dev(dev)) 8299 return mlxsw_sp_inetaddr_vlan_event(mlxsw_sp, dev, event, 8300 extack); 8301 else if (netif_is_macvlan(dev)) 8302 return mlxsw_sp_inetaddr_macvlan_event(mlxsw_sp, dev, event, 8303 extack); 8304 else 8305 return 0; 8306 } 8307 8308 static int mlxsw_sp_inetaddr_event(struct notifier_block *nb, 8309 unsigned long event, void *ptr) 8310 { 8311 struct in_ifaddr *ifa = (struct in_ifaddr *) ptr; 8312 struct net_device *dev = ifa->ifa_dev->dev; 8313 struct mlxsw_sp_router *router; 8314 struct mlxsw_sp_rif *rif; 8315 int err = 0; 8316 8317 /* NETDEV_UP event is handled by mlxsw_sp_inetaddr_valid_event */ 8318 if (event == NETDEV_UP) 8319 return NOTIFY_DONE; 8320 8321 router = container_of(nb, struct mlxsw_sp_router, inetaddr_nb); 8322 mutex_lock(&router->lock); 8323 rif = mlxsw_sp_rif_find_by_dev(router->mlxsw_sp, dev); 8324 if (!mlxsw_sp_rif_should_config(rif, dev, event)) 8325 goto out; 8326 8327 err = __mlxsw_sp_inetaddr_event(router->mlxsw_sp, dev, event, NULL); 8328 out: 8329 mutex_unlock(&router->lock); 8330 return notifier_from_errno(err); 8331 } 8332 8333 int mlxsw_sp_inetaddr_valid_event(struct notifier_block *unused, 8334 unsigned long event, void *ptr) 8335 { 8336 struct in_validator_info *ivi = (struct in_validator_info *) ptr; 8337 struct net_device *dev = ivi->ivi_dev->dev; 8338 struct mlxsw_sp *mlxsw_sp; 8339 struct mlxsw_sp_rif *rif; 8340 int err = 0; 8341 8342 mlxsw_sp = mlxsw_sp_lower_get(dev); 8343 if (!mlxsw_sp) 8344 return NOTIFY_DONE; 8345 8346 mutex_lock(&mlxsw_sp->router->lock); 8347 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev); 8348 if (!mlxsw_sp_rif_should_config(rif, dev, event)) 8349 goto out; 8350 8351 err = mlxsw_sp_router_port_check_rif_addr(mlxsw_sp, dev, dev->dev_addr, 8352 ivi->extack); 8353 if (err) 8354 goto out; 8355 8356 err = __mlxsw_sp_inetaddr_event(mlxsw_sp, dev, event, ivi->extack); 8357 out: 8358 mutex_unlock(&mlxsw_sp->router->lock); 8359 return notifier_from_errno(err); 8360 } 8361 8362 struct mlxsw_sp_inet6addr_event_work { 8363 struct work_struct work; 8364 struct mlxsw_sp *mlxsw_sp; 8365 struct net_device *dev; 8366 unsigned long event; 8367 }; 8368 8369 static void mlxsw_sp_inet6addr_event_work(struct work_struct *work) 8370 { 8371 struct mlxsw_sp_inet6addr_event_work *inet6addr_work = 8372 container_of(work, struct mlxsw_sp_inet6addr_event_work, work); 8373 struct mlxsw_sp *mlxsw_sp = inet6addr_work->mlxsw_sp; 8374 struct net_device *dev = inet6addr_work->dev; 8375 unsigned long event = inet6addr_work->event; 8376 struct mlxsw_sp_rif *rif; 8377 8378 rtnl_lock(); 8379 mutex_lock(&mlxsw_sp->router->lock); 8380 8381 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev); 8382 if (!mlxsw_sp_rif_should_config(rif, dev, event)) 8383 goto out; 8384 8385 __mlxsw_sp_inetaddr_event(mlxsw_sp, dev, event, NULL); 8386 out: 8387 mutex_unlock(&mlxsw_sp->router->lock); 8388 rtnl_unlock(); 8389 dev_put(dev); 8390 kfree(inet6addr_work); 8391 } 8392 8393 /* Called with rcu_read_lock() */ 8394 static int mlxsw_sp_inet6addr_event(struct notifier_block *nb, 8395 unsigned long event, void *ptr) 8396 { 8397 struct inet6_ifaddr *if6 = (struct inet6_ifaddr *) ptr; 8398 struct mlxsw_sp_inet6addr_event_work *inet6addr_work; 8399 struct net_device *dev = if6->idev->dev; 8400 struct mlxsw_sp_router *router; 8401 8402 /* NETDEV_UP event is handled by mlxsw_sp_inet6addr_valid_event */ 8403 if (event == NETDEV_UP) 8404 return NOTIFY_DONE; 8405 8406 inet6addr_work = kzalloc(sizeof(*inet6addr_work), GFP_ATOMIC); 8407 if (!inet6addr_work) 8408 return NOTIFY_BAD; 8409 8410 router = container_of(nb, struct mlxsw_sp_router, inet6addr_nb); 8411 INIT_WORK(&inet6addr_work->work, mlxsw_sp_inet6addr_event_work); 8412 inet6addr_work->mlxsw_sp = router->mlxsw_sp; 8413 inet6addr_work->dev = dev; 8414 inet6addr_work->event = event; 8415 dev_hold(dev); 8416 mlxsw_core_schedule_work(&inet6addr_work->work); 8417 8418 return NOTIFY_DONE; 8419 } 8420 8421 int mlxsw_sp_inet6addr_valid_event(struct notifier_block *unused, 8422 unsigned long event, void *ptr) 8423 { 8424 struct in6_validator_info *i6vi = (struct in6_validator_info *) ptr; 8425 struct net_device *dev = i6vi->i6vi_dev->dev; 8426 struct mlxsw_sp *mlxsw_sp; 8427 struct mlxsw_sp_rif *rif; 8428 int err = 0; 8429 8430 mlxsw_sp = mlxsw_sp_lower_get(dev); 8431 if (!mlxsw_sp) 8432 return NOTIFY_DONE; 8433 8434 mutex_lock(&mlxsw_sp->router->lock); 8435 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev); 8436 if (!mlxsw_sp_rif_should_config(rif, dev, event)) 8437 goto out; 8438 8439 err = mlxsw_sp_router_port_check_rif_addr(mlxsw_sp, dev, dev->dev_addr, 8440 i6vi->extack); 8441 if (err) 8442 goto out; 8443 8444 err = __mlxsw_sp_inetaddr_event(mlxsw_sp, dev, event, i6vi->extack); 8445 out: 8446 mutex_unlock(&mlxsw_sp->router->lock); 8447 return notifier_from_errno(err); 8448 } 8449 8450 static int mlxsw_sp_rif_edit(struct mlxsw_sp *mlxsw_sp, u16 rif_index, 8451 const char *mac, int mtu) 8452 { 8453 char ritr_pl[MLXSW_REG_RITR_LEN]; 8454 int err; 8455 8456 mlxsw_reg_ritr_rif_pack(ritr_pl, rif_index); 8457 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 8458 if (err) 8459 return err; 8460 8461 mlxsw_reg_ritr_mtu_set(ritr_pl, mtu); 8462 mlxsw_reg_ritr_if_mac_memcpy_to(ritr_pl, mac); 8463 mlxsw_reg_ritr_op_set(ritr_pl, MLXSW_REG_RITR_RIF_CREATE); 8464 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 8465 } 8466 8467 static int 8468 mlxsw_sp_router_port_change_event(struct mlxsw_sp *mlxsw_sp, 8469 struct mlxsw_sp_rif *rif) 8470 { 8471 struct net_device *dev = rif->dev; 8472 u16 fid_index; 8473 int err; 8474 8475 fid_index = mlxsw_sp_fid_index(rif->fid); 8476 8477 err = mlxsw_sp_rif_fdb_op(mlxsw_sp, rif->addr, fid_index, false); 8478 if (err) 8479 return err; 8480 8481 err = mlxsw_sp_rif_edit(mlxsw_sp, rif->rif_index, dev->dev_addr, 8482 dev->mtu); 8483 if (err) 8484 goto err_rif_edit; 8485 8486 err = mlxsw_sp_rif_fdb_op(mlxsw_sp, dev->dev_addr, fid_index, true); 8487 if (err) 8488 goto err_rif_fdb_op; 8489 8490 if (rif->mtu != dev->mtu) { 8491 struct mlxsw_sp_vr *vr; 8492 int i; 8493 8494 /* The RIF is relevant only to its mr_table instance, as unlike 8495 * unicast routing, in multicast routing a RIF cannot be shared 8496 * between several multicast routing tables. 8497 */ 8498 vr = &mlxsw_sp->router->vrs[rif->vr_id]; 8499 for (i = 0; i < MLXSW_SP_L3_PROTO_MAX; i++) 8500 mlxsw_sp_mr_rif_mtu_update(vr->mr_table[i], 8501 rif, dev->mtu); 8502 } 8503 8504 ether_addr_copy(rif->addr, dev->dev_addr); 8505 rif->mtu = dev->mtu; 8506 8507 netdev_dbg(dev, "Updated RIF=%d\n", rif->rif_index); 8508 8509 return 0; 8510 8511 err_rif_fdb_op: 8512 mlxsw_sp_rif_edit(mlxsw_sp, rif->rif_index, rif->addr, rif->mtu); 8513 err_rif_edit: 8514 mlxsw_sp_rif_fdb_op(mlxsw_sp, rif->addr, fid_index, true); 8515 return err; 8516 } 8517 8518 static int mlxsw_sp_router_port_pre_changeaddr_event(struct mlxsw_sp_rif *rif, 8519 struct netdev_notifier_pre_changeaddr_info *info) 8520 { 8521 struct netlink_ext_ack *extack; 8522 8523 extack = netdev_notifier_info_to_extack(&info->info); 8524 return mlxsw_sp_router_port_check_rif_addr(rif->mlxsw_sp, rif->dev, 8525 info->dev_addr, extack); 8526 } 8527 8528 int mlxsw_sp_netdevice_router_port_event(struct net_device *dev, 8529 unsigned long event, void *ptr) 8530 { 8531 struct mlxsw_sp *mlxsw_sp; 8532 struct mlxsw_sp_rif *rif; 8533 int err = 0; 8534 8535 mlxsw_sp = mlxsw_sp_lower_get(dev); 8536 if (!mlxsw_sp) 8537 return 0; 8538 8539 mutex_lock(&mlxsw_sp->router->lock); 8540 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev); 8541 if (!rif) 8542 goto out; 8543 8544 switch (event) { 8545 case NETDEV_CHANGEMTU: 8546 case NETDEV_CHANGEADDR: 8547 err = mlxsw_sp_router_port_change_event(mlxsw_sp, rif); 8548 break; 8549 case NETDEV_PRE_CHANGEADDR: 8550 err = mlxsw_sp_router_port_pre_changeaddr_event(rif, ptr); 8551 break; 8552 } 8553 8554 out: 8555 mutex_unlock(&mlxsw_sp->router->lock); 8556 return err; 8557 } 8558 8559 static int mlxsw_sp_port_vrf_join(struct mlxsw_sp *mlxsw_sp, 8560 struct net_device *l3_dev, 8561 struct netlink_ext_ack *extack) 8562 { 8563 struct mlxsw_sp_rif *rif; 8564 8565 /* If netdev is already associated with a RIF, then we need to 8566 * destroy it and create a new one with the new virtual router ID. 8567 */ 8568 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev); 8569 if (rif) 8570 __mlxsw_sp_inetaddr_event(mlxsw_sp, l3_dev, NETDEV_DOWN, 8571 extack); 8572 8573 return __mlxsw_sp_inetaddr_event(mlxsw_sp, l3_dev, NETDEV_UP, extack); 8574 } 8575 8576 static void mlxsw_sp_port_vrf_leave(struct mlxsw_sp *mlxsw_sp, 8577 struct net_device *l3_dev) 8578 { 8579 struct mlxsw_sp_rif *rif; 8580 8581 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev); 8582 if (!rif) 8583 return; 8584 __mlxsw_sp_inetaddr_event(mlxsw_sp, l3_dev, NETDEV_DOWN, NULL); 8585 } 8586 8587 int mlxsw_sp_netdevice_vrf_event(struct net_device *l3_dev, unsigned long event, 8588 struct netdev_notifier_changeupper_info *info) 8589 { 8590 struct mlxsw_sp *mlxsw_sp = mlxsw_sp_lower_get(l3_dev); 8591 int err = 0; 8592 8593 /* We do not create a RIF for a macvlan, but only use it to 8594 * direct more MAC addresses to the router. 8595 */ 8596 if (!mlxsw_sp || netif_is_macvlan(l3_dev)) 8597 return 0; 8598 8599 mutex_lock(&mlxsw_sp->router->lock); 8600 switch (event) { 8601 case NETDEV_PRECHANGEUPPER: 8602 break; 8603 case NETDEV_CHANGEUPPER: 8604 if (info->linking) { 8605 struct netlink_ext_ack *extack; 8606 8607 extack = netdev_notifier_info_to_extack(&info->info); 8608 err = mlxsw_sp_port_vrf_join(mlxsw_sp, l3_dev, extack); 8609 } else { 8610 mlxsw_sp_port_vrf_leave(mlxsw_sp, l3_dev); 8611 } 8612 break; 8613 } 8614 mutex_unlock(&mlxsw_sp->router->lock); 8615 8616 return err; 8617 } 8618 8619 static int __mlxsw_sp_rif_macvlan_flush(struct net_device *dev, 8620 struct netdev_nested_priv *priv) 8621 { 8622 struct mlxsw_sp_rif *rif = (struct mlxsw_sp_rif *)priv->data; 8623 8624 if (!netif_is_macvlan(dev)) 8625 return 0; 8626 8627 return mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, dev->dev_addr, 8628 mlxsw_sp_fid_index(rif->fid), false); 8629 } 8630 8631 static int mlxsw_sp_rif_macvlan_flush(struct mlxsw_sp_rif *rif) 8632 { 8633 struct netdev_nested_priv priv = { 8634 .data = (void *)rif, 8635 }; 8636 8637 if (!netif_is_macvlan_port(rif->dev)) 8638 return 0; 8639 8640 netdev_warn(rif->dev, "Router interface is deleted. Upper macvlans will not work\n"); 8641 return netdev_walk_all_upper_dev_rcu(rif->dev, 8642 __mlxsw_sp_rif_macvlan_flush, &priv); 8643 } 8644 8645 static void mlxsw_sp_rif_subport_setup(struct mlxsw_sp_rif *rif, 8646 const struct mlxsw_sp_rif_params *params) 8647 { 8648 struct mlxsw_sp_rif_subport *rif_subport; 8649 8650 rif_subport = mlxsw_sp_rif_subport_rif(rif); 8651 refcount_set(&rif_subport->ref_count, 1); 8652 rif_subport->vid = params->vid; 8653 rif_subport->lag = params->lag; 8654 if (params->lag) 8655 rif_subport->lag_id = params->lag_id; 8656 else 8657 rif_subport->system_port = params->system_port; 8658 } 8659 8660 static int mlxsw_sp_rif_subport_op(struct mlxsw_sp_rif *rif, bool enable) 8661 { 8662 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 8663 struct mlxsw_sp_rif_subport *rif_subport; 8664 char ritr_pl[MLXSW_REG_RITR_LEN]; 8665 8666 rif_subport = mlxsw_sp_rif_subport_rif(rif); 8667 mlxsw_reg_ritr_pack(ritr_pl, enable, MLXSW_REG_RITR_SP_IF, 8668 rif->rif_index, rif->vr_id, rif->dev->mtu); 8669 mlxsw_reg_ritr_mac_pack(ritr_pl, rif->dev->dev_addr); 8670 mlxsw_reg_ritr_sp_if_pack(ritr_pl, rif_subport->lag, 8671 rif_subport->lag ? rif_subport->lag_id : 8672 rif_subport->system_port, 8673 rif_subport->vid); 8674 8675 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 8676 } 8677 8678 static int mlxsw_sp_rif_subport_configure(struct mlxsw_sp_rif *rif) 8679 { 8680 int err; 8681 8682 err = mlxsw_sp_rif_subport_op(rif, true); 8683 if (err) 8684 return err; 8685 8686 err = mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr, 8687 mlxsw_sp_fid_index(rif->fid), true); 8688 if (err) 8689 goto err_rif_fdb_op; 8690 8691 mlxsw_sp_fid_rif_set(rif->fid, rif); 8692 return 0; 8693 8694 err_rif_fdb_op: 8695 mlxsw_sp_rif_subport_op(rif, false); 8696 return err; 8697 } 8698 8699 static void mlxsw_sp_rif_subport_deconfigure(struct mlxsw_sp_rif *rif) 8700 { 8701 struct mlxsw_sp_fid *fid = rif->fid; 8702 8703 mlxsw_sp_fid_rif_set(fid, NULL); 8704 mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr, 8705 mlxsw_sp_fid_index(fid), false); 8706 mlxsw_sp_rif_macvlan_flush(rif); 8707 mlxsw_sp_rif_subport_op(rif, false); 8708 } 8709 8710 static struct mlxsw_sp_fid * 8711 mlxsw_sp_rif_subport_fid_get(struct mlxsw_sp_rif *rif, 8712 struct netlink_ext_ack *extack) 8713 { 8714 return mlxsw_sp_fid_rfid_get(rif->mlxsw_sp, rif->rif_index); 8715 } 8716 8717 static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_subport_ops = { 8718 .type = MLXSW_SP_RIF_TYPE_SUBPORT, 8719 .rif_size = sizeof(struct mlxsw_sp_rif_subport), 8720 .setup = mlxsw_sp_rif_subport_setup, 8721 .configure = mlxsw_sp_rif_subport_configure, 8722 .deconfigure = mlxsw_sp_rif_subport_deconfigure, 8723 .fid_get = mlxsw_sp_rif_subport_fid_get, 8724 }; 8725 8726 static int mlxsw_sp_rif_vlan_fid_op(struct mlxsw_sp_rif *rif, 8727 enum mlxsw_reg_ritr_if_type type, 8728 u16 vid_fid, bool enable) 8729 { 8730 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 8731 char ritr_pl[MLXSW_REG_RITR_LEN]; 8732 8733 mlxsw_reg_ritr_pack(ritr_pl, enable, type, rif->rif_index, rif->vr_id, 8734 rif->dev->mtu); 8735 mlxsw_reg_ritr_mac_pack(ritr_pl, rif->dev->dev_addr); 8736 mlxsw_reg_ritr_fid_set(ritr_pl, type, vid_fid); 8737 8738 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 8739 } 8740 8741 u8 mlxsw_sp_router_port(const struct mlxsw_sp *mlxsw_sp) 8742 { 8743 return mlxsw_core_max_ports(mlxsw_sp->core) + 1; 8744 } 8745 8746 static int mlxsw_sp_rif_fid_configure(struct mlxsw_sp_rif *rif) 8747 { 8748 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 8749 u16 fid_index = mlxsw_sp_fid_index(rif->fid); 8750 int err; 8751 8752 err = mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_FID_IF, fid_index, 8753 true); 8754 if (err) 8755 return err; 8756 8757 err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC, 8758 mlxsw_sp_router_port(mlxsw_sp), true); 8759 if (err) 8760 goto err_fid_mc_flood_set; 8761 8762 err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC, 8763 mlxsw_sp_router_port(mlxsw_sp), true); 8764 if (err) 8765 goto err_fid_bc_flood_set; 8766 8767 err = mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr, 8768 mlxsw_sp_fid_index(rif->fid), true); 8769 if (err) 8770 goto err_rif_fdb_op; 8771 8772 mlxsw_sp_fid_rif_set(rif->fid, rif); 8773 return 0; 8774 8775 err_rif_fdb_op: 8776 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC, 8777 mlxsw_sp_router_port(mlxsw_sp), false); 8778 err_fid_bc_flood_set: 8779 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC, 8780 mlxsw_sp_router_port(mlxsw_sp), false); 8781 err_fid_mc_flood_set: 8782 mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_FID_IF, fid_index, false); 8783 return err; 8784 } 8785 8786 static void mlxsw_sp_rif_fid_deconfigure(struct mlxsw_sp_rif *rif) 8787 { 8788 u16 fid_index = mlxsw_sp_fid_index(rif->fid); 8789 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 8790 struct mlxsw_sp_fid *fid = rif->fid; 8791 8792 mlxsw_sp_fid_rif_set(fid, NULL); 8793 mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr, 8794 mlxsw_sp_fid_index(fid), false); 8795 mlxsw_sp_rif_macvlan_flush(rif); 8796 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC, 8797 mlxsw_sp_router_port(mlxsw_sp), false); 8798 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC, 8799 mlxsw_sp_router_port(mlxsw_sp), false); 8800 mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_FID_IF, fid_index, false); 8801 } 8802 8803 static struct mlxsw_sp_fid * 8804 mlxsw_sp_rif_fid_fid_get(struct mlxsw_sp_rif *rif, 8805 struct netlink_ext_ack *extack) 8806 { 8807 return mlxsw_sp_fid_8021d_get(rif->mlxsw_sp, rif->dev->ifindex); 8808 } 8809 8810 static void mlxsw_sp_rif_fid_fdb_del(struct mlxsw_sp_rif *rif, const char *mac) 8811 { 8812 struct switchdev_notifier_fdb_info info; 8813 struct net_device *dev; 8814 8815 dev = br_fdb_find_port(rif->dev, mac, 0); 8816 if (!dev) 8817 return; 8818 8819 info.addr = mac; 8820 info.vid = 0; 8821 call_switchdev_notifiers(SWITCHDEV_FDB_DEL_TO_BRIDGE, dev, &info.info, 8822 NULL); 8823 } 8824 8825 static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_fid_ops = { 8826 .type = MLXSW_SP_RIF_TYPE_FID, 8827 .rif_size = sizeof(struct mlxsw_sp_rif), 8828 .configure = mlxsw_sp_rif_fid_configure, 8829 .deconfigure = mlxsw_sp_rif_fid_deconfigure, 8830 .fid_get = mlxsw_sp_rif_fid_fid_get, 8831 .fdb_del = mlxsw_sp_rif_fid_fdb_del, 8832 }; 8833 8834 static struct mlxsw_sp_fid * 8835 mlxsw_sp_rif_vlan_fid_get(struct mlxsw_sp_rif *rif, 8836 struct netlink_ext_ack *extack) 8837 { 8838 struct net_device *br_dev; 8839 u16 vid; 8840 int err; 8841 8842 if (is_vlan_dev(rif->dev)) { 8843 vid = vlan_dev_vlan_id(rif->dev); 8844 br_dev = vlan_dev_real_dev(rif->dev); 8845 if (WARN_ON(!netif_is_bridge_master(br_dev))) 8846 return ERR_PTR(-EINVAL); 8847 } else { 8848 err = br_vlan_get_pvid(rif->dev, &vid); 8849 if (err < 0 || !vid) { 8850 NL_SET_ERR_MSG_MOD(extack, "Couldn't determine bridge PVID"); 8851 return ERR_PTR(-EINVAL); 8852 } 8853 } 8854 8855 return mlxsw_sp_fid_8021q_get(rif->mlxsw_sp, vid); 8856 } 8857 8858 static void mlxsw_sp_rif_vlan_fdb_del(struct mlxsw_sp_rif *rif, const char *mac) 8859 { 8860 u16 vid = mlxsw_sp_fid_8021q_vid(rif->fid); 8861 struct switchdev_notifier_fdb_info info; 8862 struct net_device *br_dev; 8863 struct net_device *dev; 8864 8865 br_dev = is_vlan_dev(rif->dev) ? vlan_dev_real_dev(rif->dev) : rif->dev; 8866 dev = br_fdb_find_port(br_dev, mac, vid); 8867 if (!dev) 8868 return; 8869 8870 info.addr = mac; 8871 info.vid = vid; 8872 call_switchdev_notifiers(SWITCHDEV_FDB_DEL_TO_BRIDGE, dev, &info.info, 8873 NULL); 8874 } 8875 8876 static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_vlan_emu_ops = { 8877 .type = MLXSW_SP_RIF_TYPE_VLAN, 8878 .rif_size = sizeof(struct mlxsw_sp_rif), 8879 .configure = mlxsw_sp_rif_fid_configure, 8880 .deconfigure = mlxsw_sp_rif_fid_deconfigure, 8881 .fid_get = mlxsw_sp_rif_vlan_fid_get, 8882 .fdb_del = mlxsw_sp_rif_vlan_fdb_del, 8883 }; 8884 8885 static struct mlxsw_sp_rif_ipip_lb * 8886 mlxsw_sp_rif_ipip_lb_rif(struct mlxsw_sp_rif *rif) 8887 { 8888 return container_of(rif, struct mlxsw_sp_rif_ipip_lb, common); 8889 } 8890 8891 static void 8892 mlxsw_sp_rif_ipip_lb_setup(struct mlxsw_sp_rif *rif, 8893 const struct mlxsw_sp_rif_params *params) 8894 { 8895 struct mlxsw_sp_rif_params_ipip_lb *params_lb; 8896 struct mlxsw_sp_rif_ipip_lb *rif_lb; 8897 8898 params_lb = container_of(params, struct mlxsw_sp_rif_params_ipip_lb, 8899 common); 8900 rif_lb = mlxsw_sp_rif_ipip_lb_rif(rif); 8901 rif_lb->lb_config = params_lb->lb_config; 8902 } 8903 8904 static int 8905 mlxsw_sp1_rif_ipip_lb_configure(struct mlxsw_sp_rif *rif) 8906 { 8907 struct mlxsw_sp_rif_ipip_lb *lb_rif = mlxsw_sp_rif_ipip_lb_rif(rif); 8908 u32 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(rif->dev); 8909 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 8910 struct mlxsw_sp_vr *ul_vr; 8911 int err; 8912 8913 ul_vr = mlxsw_sp_vr_get(mlxsw_sp, ul_tb_id, NULL); 8914 if (IS_ERR(ul_vr)) 8915 return PTR_ERR(ul_vr); 8916 8917 err = mlxsw_sp_rif_ipip_lb_op(lb_rif, ul_vr->id, 0, true); 8918 if (err) 8919 goto err_loopback_op; 8920 8921 lb_rif->ul_vr_id = ul_vr->id; 8922 lb_rif->ul_rif_id = 0; 8923 ++ul_vr->rif_count; 8924 return 0; 8925 8926 err_loopback_op: 8927 mlxsw_sp_vr_put(mlxsw_sp, ul_vr); 8928 return err; 8929 } 8930 8931 static void mlxsw_sp1_rif_ipip_lb_deconfigure(struct mlxsw_sp_rif *rif) 8932 { 8933 struct mlxsw_sp_rif_ipip_lb *lb_rif = mlxsw_sp_rif_ipip_lb_rif(rif); 8934 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 8935 struct mlxsw_sp_vr *ul_vr; 8936 8937 ul_vr = &mlxsw_sp->router->vrs[lb_rif->ul_vr_id]; 8938 mlxsw_sp_rif_ipip_lb_op(lb_rif, ul_vr->id, 0, false); 8939 8940 --ul_vr->rif_count; 8941 mlxsw_sp_vr_put(mlxsw_sp, ul_vr); 8942 } 8943 8944 static const struct mlxsw_sp_rif_ops mlxsw_sp1_rif_ipip_lb_ops = { 8945 .type = MLXSW_SP_RIF_TYPE_IPIP_LB, 8946 .rif_size = sizeof(struct mlxsw_sp_rif_ipip_lb), 8947 .setup = mlxsw_sp_rif_ipip_lb_setup, 8948 .configure = mlxsw_sp1_rif_ipip_lb_configure, 8949 .deconfigure = mlxsw_sp1_rif_ipip_lb_deconfigure, 8950 }; 8951 8952 static const struct mlxsw_sp_rif_ops *mlxsw_sp1_rif_ops_arr[] = { 8953 [MLXSW_SP_RIF_TYPE_SUBPORT] = &mlxsw_sp_rif_subport_ops, 8954 [MLXSW_SP_RIF_TYPE_VLAN] = &mlxsw_sp_rif_vlan_emu_ops, 8955 [MLXSW_SP_RIF_TYPE_FID] = &mlxsw_sp_rif_fid_ops, 8956 [MLXSW_SP_RIF_TYPE_IPIP_LB] = &mlxsw_sp1_rif_ipip_lb_ops, 8957 }; 8958 8959 static int 8960 mlxsw_sp_rif_ipip_lb_ul_rif_op(struct mlxsw_sp_rif *ul_rif, bool enable) 8961 { 8962 struct mlxsw_sp *mlxsw_sp = ul_rif->mlxsw_sp; 8963 char ritr_pl[MLXSW_REG_RITR_LEN]; 8964 8965 mlxsw_reg_ritr_pack(ritr_pl, enable, MLXSW_REG_RITR_LOOPBACK_IF, 8966 ul_rif->rif_index, ul_rif->vr_id, IP_MAX_MTU); 8967 mlxsw_reg_ritr_loopback_protocol_set(ritr_pl, 8968 MLXSW_REG_RITR_LOOPBACK_GENERIC); 8969 8970 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 8971 } 8972 8973 static struct mlxsw_sp_rif * 8974 mlxsw_sp_ul_rif_create(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp_vr *vr, 8975 struct netlink_ext_ack *extack) 8976 { 8977 struct mlxsw_sp_rif *ul_rif; 8978 u16 rif_index; 8979 int err; 8980 8981 err = mlxsw_sp_rif_index_alloc(mlxsw_sp, &rif_index); 8982 if (err) { 8983 NL_SET_ERR_MSG_MOD(extack, "Exceeded number of supported router interfaces"); 8984 return ERR_PTR(err); 8985 } 8986 8987 ul_rif = mlxsw_sp_rif_alloc(sizeof(*ul_rif), rif_index, vr->id, NULL); 8988 if (!ul_rif) 8989 return ERR_PTR(-ENOMEM); 8990 8991 mlxsw_sp->router->rifs[rif_index] = ul_rif; 8992 ul_rif->mlxsw_sp = mlxsw_sp; 8993 err = mlxsw_sp_rif_ipip_lb_ul_rif_op(ul_rif, true); 8994 if (err) 8995 goto ul_rif_op_err; 8996 8997 return ul_rif; 8998 8999 ul_rif_op_err: 9000 mlxsw_sp->router->rifs[rif_index] = NULL; 9001 kfree(ul_rif); 9002 return ERR_PTR(err); 9003 } 9004 9005 static void mlxsw_sp_ul_rif_destroy(struct mlxsw_sp_rif *ul_rif) 9006 { 9007 struct mlxsw_sp *mlxsw_sp = ul_rif->mlxsw_sp; 9008 9009 mlxsw_sp_rif_ipip_lb_ul_rif_op(ul_rif, false); 9010 mlxsw_sp->router->rifs[ul_rif->rif_index] = NULL; 9011 kfree(ul_rif); 9012 } 9013 9014 static struct mlxsw_sp_rif * 9015 mlxsw_sp_ul_rif_get(struct mlxsw_sp *mlxsw_sp, u32 tb_id, 9016 struct netlink_ext_ack *extack) 9017 { 9018 struct mlxsw_sp_vr *vr; 9019 int err; 9020 9021 vr = mlxsw_sp_vr_get(mlxsw_sp, tb_id, extack); 9022 if (IS_ERR(vr)) 9023 return ERR_CAST(vr); 9024 9025 if (refcount_inc_not_zero(&vr->ul_rif_refcnt)) 9026 return vr->ul_rif; 9027 9028 vr->ul_rif = mlxsw_sp_ul_rif_create(mlxsw_sp, vr, extack); 9029 if (IS_ERR(vr->ul_rif)) { 9030 err = PTR_ERR(vr->ul_rif); 9031 goto err_ul_rif_create; 9032 } 9033 9034 vr->rif_count++; 9035 refcount_set(&vr->ul_rif_refcnt, 1); 9036 9037 return vr->ul_rif; 9038 9039 err_ul_rif_create: 9040 mlxsw_sp_vr_put(mlxsw_sp, vr); 9041 return ERR_PTR(err); 9042 } 9043 9044 static void mlxsw_sp_ul_rif_put(struct mlxsw_sp_rif *ul_rif) 9045 { 9046 struct mlxsw_sp *mlxsw_sp = ul_rif->mlxsw_sp; 9047 struct mlxsw_sp_vr *vr; 9048 9049 vr = &mlxsw_sp->router->vrs[ul_rif->vr_id]; 9050 9051 if (!refcount_dec_and_test(&vr->ul_rif_refcnt)) 9052 return; 9053 9054 vr->rif_count--; 9055 mlxsw_sp_ul_rif_destroy(ul_rif); 9056 mlxsw_sp_vr_put(mlxsw_sp, vr); 9057 } 9058 9059 int mlxsw_sp_router_ul_rif_get(struct mlxsw_sp *mlxsw_sp, u32 ul_tb_id, 9060 u16 *ul_rif_index) 9061 { 9062 struct mlxsw_sp_rif *ul_rif; 9063 int err = 0; 9064 9065 mutex_lock(&mlxsw_sp->router->lock); 9066 ul_rif = mlxsw_sp_ul_rif_get(mlxsw_sp, ul_tb_id, NULL); 9067 if (IS_ERR(ul_rif)) { 9068 err = PTR_ERR(ul_rif); 9069 goto out; 9070 } 9071 *ul_rif_index = ul_rif->rif_index; 9072 out: 9073 mutex_unlock(&mlxsw_sp->router->lock); 9074 return err; 9075 } 9076 9077 void mlxsw_sp_router_ul_rif_put(struct mlxsw_sp *mlxsw_sp, u16 ul_rif_index) 9078 { 9079 struct mlxsw_sp_rif *ul_rif; 9080 9081 mutex_lock(&mlxsw_sp->router->lock); 9082 ul_rif = mlxsw_sp->router->rifs[ul_rif_index]; 9083 if (WARN_ON(!ul_rif)) 9084 goto out; 9085 9086 mlxsw_sp_ul_rif_put(ul_rif); 9087 out: 9088 mutex_unlock(&mlxsw_sp->router->lock); 9089 } 9090 9091 static int 9092 mlxsw_sp2_rif_ipip_lb_configure(struct mlxsw_sp_rif *rif) 9093 { 9094 struct mlxsw_sp_rif_ipip_lb *lb_rif = mlxsw_sp_rif_ipip_lb_rif(rif); 9095 u32 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(rif->dev); 9096 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 9097 struct mlxsw_sp_rif *ul_rif; 9098 int err; 9099 9100 ul_rif = mlxsw_sp_ul_rif_get(mlxsw_sp, ul_tb_id, NULL); 9101 if (IS_ERR(ul_rif)) 9102 return PTR_ERR(ul_rif); 9103 9104 err = mlxsw_sp_rif_ipip_lb_op(lb_rif, 0, ul_rif->rif_index, true); 9105 if (err) 9106 goto err_loopback_op; 9107 9108 lb_rif->ul_vr_id = 0; 9109 lb_rif->ul_rif_id = ul_rif->rif_index; 9110 9111 return 0; 9112 9113 err_loopback_op: 9114 mlxsw_sp_ul_rif_put(ul_rif); 9115 return err; 9116 } 9117 9118 static void mlxsw_sp2_rif_ipip_lb_deconfigure(struct mlxsw_sp_rif *rif) 9119 { 9120 struct mlxsw_sp_rif_ipip_lb *lb_rif = mlxsw_sp_rif_ipip_lb_rif(rif); 9121 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 9122 struct mlxsw_sp_rif *ul_rif; 9123 9124 ul_rif = mlxsw_sp_rif_by_index(mlxsw_sp, lb_rif->ul_rif_id); 9125 mlxsw_sp_rif_ipip_lb_op(lb_rif, 0, lb_rif->ul_rif_id, false); 9126 mlxsw_sp_ul_rif_put(ul_rif); 9127 } 9128 9129 static const struct mlxsw_sp_rif_ops mlxsw_sp2_rif_ipip_lb_ops = { 9130 .type = MLXSW_SP_RIF_TYPE_IPIP_LB, 9131 .rif_size = sizeof(struct mlxsw_sp_rif_ipip_lb), 9132 .setup = mlxsw_sp_rif_ipip_lb_setup, 9133 .configure = mlxsw_sp2_rif_ipip_lb_configure, 9134 .deconfigure = mlxsw_sp2_rif_ipip_lb_deconfigure, 9135 }; 9136 9137 static const struct mlxsw_sp_rif_ops *mlxsw_sp2_rif_ops_arr[] = { 9138 [MLXSW_SP_RIF_TYPE_SUBPORT] = &mlxsw_sp_rif_subport_ops, 9139 [MLXSW_SP_RIF_TYPE_VLAN] = &mlxsw_sp_rif_vlan_emu_ops, 9140 [MLXSW_SP_RIF_TYPE_FID] = &mlxsw_sp_rif_fid_ops, 9141 [MLXSW_SP_RIF_TYPE_IPIP_LB] = &mlxsw_sp2_rif_ipip_lb_ops, 9142 }; 9143 9144 static int mlxsw_sp_rifs_init(struct mlxsw_sp *mlxsw_sp) 9145 { 9146 u64 max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); 9147 9148 mlxsw_sp->router->rifs = kcalloc(max_rifs, 9149 sizeof(struct mlxsw_sp_rif *), 9150 GFP_KERNEL); 9151 if (!mlxsw_sp->router->rifs) 9152 return -ENOMEM; 9153 9154 return 0; 9155 } 9156 9157 static void mlxsw_sp_rifs_fini(struct mlxsw_sp *mlxsw_sp) 9158 { 9159 int i; 9160 9161 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++) 9162 WARN_ON_ONCE(mlxsw_sp->router->rifs[i]); 9163 9164 kfree(mlxsw_sp->router->rifs); 9165 } 9166 9167 static int 9168 mlxsw_sp_ipip_config_tigcr(struct mlxsw_sp *mlxsw_sp) 9169 { 9170 char tigcr_pl[MLXSW_REG_TIGCR_LEN]; 9171 9172 mlxsw_reg_tigcr_pack(tigcr_pl, true, 0); 9173 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(tigcr), tigcr_pl); 9174 } 9175 9176 static int mlxsw_sp_ipips_init(struct mlxsw_sp *mlxsw_sp) 9177 { 9178 int err; 9179 9180 mlxsw_sp->router->ipip_ops_arr = mlxsw_sp_ipip_ops_arr; 9181 INIT_LIST_HEAD(&mlxsw_sp->router->ipip_list); 9182 9183 err = mlxsw_sp_ipip_ecn_encap_init(mlxsw_sp); 9184 if (err) 9185 return err; 9186 err = mlxsw_sp_ipip_ecn_decap_init(mlxsw_sp); 9187 if (err) 9188 return err; 9189 9190 return mlxsw_sp_ipip_config_tigcr(mlxsw_sp); 9191 } 9192 9193 static void mlxsw_sp_ipips_fini(struct mlxsw_sp *mlxsw_sp) 9194 { 9195 WARN_ON(!list_empty(&mlxsw_sp->router->ipip_list)); 9196 } 9197 9198 static void mlxsw_sp_router_fib_dump_flush(struct notifier_block *nb) 9199 { 9200 struct mlxsw_sp_router *router; 9201 9202 /* Flush pending FIB notifications and then flush the device's 9203 * table before requesting another dump. The FIB notification 9204 * block is unregistered, so no need to take RTNL. 9205 */ 9206 mlxsw_core_flush_owq(); 9207 router = container_of(nb, struct mlxsw_sp_router, fib_nb); 9208 mlxsw_sp_router_fib_flush(router->mlxsw_sp); 9209 } 9210 9211 #ifdef CONFIG_IP_ROUTE_MULTIPATH 9212 static void mlxsw_sp_mp_hash_header_set(char *recr2_pl, int header) 9213 { 9214 mlxsw_reg_recr2_outer_header_enables_set(recr2_pl, header, true); 9215 } 9216 9217 static void mlxsw_sp_mp_hash_field_set(char *recr2_pl, int field) 9218 { 9219 mlxsw_reg_recr2_outer_header_fields_enable_set(recr2_pl, field, true); 9220 } 9221 9222 static void mlxsw_sp_mp4_hash_init(struct mlxsw_sp *mlxsw_sp, char *recr2_pl) 9223 { 9224 struct net *net = mlxsw_sp_net(mlxsw_sp); 9225 bool only_l3 = !net->ipv4.sysctl_fib_multipath_hash_policy; 9226 9227 mlxsw_sp_mp_hash_header_set(recr2_pl, 9228 MLXSW_REG_RECR2_IPV4_EN_NOT_TCP_NOT_UDP); 9229 mlxsw_sp_mp_hash_header_set(recr2_pl, MLXSW_REG_RECR2_IPV4_EN_TCP_UDP); 9230 mlxsw_reg_recr2_ipv4_sip_enable(recr2_pl); 9231 mlxsw_reg_recr2_ipv4_dip_enable(recr2_pl); 9232 if (only_l3) 9233 return; 9234 mlxsw_sp_mp_hash_header_set(recr2_pl, MLXSW_REG_RECR2_TCP_UDP_EN_IPV4); 9235 mlxsw_sp_mp_hash_field_set(recr2_pl, MLXSW_REG_RECR2_IPV4_PROTOCOL); 9236 mlxsw_sp_mp_hash_field_set(recr2_pl, MLXSW_REG_RECR2_TCP_UDP_SPORT); 9237 mlxsw_sp_mp_hash_field_set(recr2_pl, MLXSW_REG_RECR2_TCP_UDP_DPORT); 9238 } 9239 9240 static void mlxsw_sp_mp6_hash_init(struct mlxsw_sp *mlxsw_sp, char *recr2_pl) 9241 { 9242 bool only_l3 = !ip6_multipath_hash_policy(mlxsw_sp_net(mlxsw_sp)); 9243 9244 mlxsw_sp_mp_hash_header_set(recr2_pl, 9245 MLXSW_REG_RECR2_IPV6_EN_NOT_TCP_NOT_UDP); 9246 mlxsw_sp_mp_hash_header_set(recr2_pl, MLXSW_REG_RECR2_IPV6_EN_TCP_UDP); 9247 mlxsw_reg_recr2_ipv6_sip_enable(recr2_pl); 9248 mlxsw_reg_recr2_ipv6_dip_enable(recr2_pl); 9249 mlxsw_sp_mp_hash_field_set(recr2_pl, MLXSW_REG_RECR2_IPV6_NEXT_HEADER); 9250 if (only_l3) { 9251 mlxsw_sp_mp_hash_field_set(recr2_pl, 9252 MLXSW_REG_RECR2_IPV6_FLOW_LABEL); 9253 } else { 9254 mlxsw_sp_mp_hash_header_set(recr2_pl, 9255 MLXSW_REG_RECR2_TCP_UDP_EN_IPV6); 9256 mlxsw_sp_mp_hash_field_set(recr2_pl, 9257 MLXSW_REG_RECR2_TCP_UDP_SPORT); 9258 mlxsw_sp_mp_hash_field_set(recr2_pl, 9259 MLXSW_REG_RECR2_TCP_UDP_DPORT); 9260 } 9261 } 9262 9263 static int mlxsw_sp_mp_hash_init(struct mlxsw_sp *mlxsw_sp) 9264 { 9265 char recr2_pl[MLXSW_REG_RECR2_LEN]; 9266 u32 seed; 9267 9268 seed = jhash(mlxsw_sp->base_mac, sizeof(mlxsw_sp->base_mac), 0); 9269 mlxsw_reg_recr2_pack(recr2_pl, seed); 9270 mlxsw_sp_mp4_hash_init(mlxsw_sp, recr2_pl); 9271 mlxsw_sp_mp6_hash_init(mlxsw_sp, recr2_pl); 9272 9273 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(recr2), recr2_pl); 9274 } 9275 #else 9276 static int mlxsw_sp_mp_hash_init(struct mlxsw_sp *mlxsw_sp) 9277 { 9278 return 0; 9279 } 9280 #endif 9281 9282 static int mlxsw_sp_dscp_init(struct mlxsw_sp *mlxsw_sp) 9283 { 9284 char rdpm_pl[MLXSW_REG_RDPM_LEN]; 9285 unsigned int i; 9286 9287 MLXSW_REG_ZERO(rdpm, rdpm_pl); 9288 9289 /* HW is determining switch priority based on DSCP-bits, but the 9290 * kernel is still doing that based on the ToS. Since there's a 9291 * mismatch in bits we need to make sure to translate the right 9292 * value ToS would observe, skipping the 2 least-significant ECN bits. 9293 */ 9294 for (i = 0; i < MLXSW_REG_RDPM_DSCP_ENTRY_REC_MAX_COUNT; i++) 9295 mlxsw_reg_rdpm_pack(rdpm_pl, i, rt_tos2priority(i << 2)); 9296 9297 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rdpm), rdpm_pl); 9298 } 9299 9300 static int __mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp) 9301 { 9302 struct net *net = mlxsw_sp_net(mlxsw_sp); 9303 bool usp = net->ipv4.sysctl_ip_fwd_update_priority; 9304 char rgcr_pl[MLXSW_REG_RGCR_LEN]; 9305 u64 max_rifs; 9306 9307 if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_RIFS)) 9308 return -EIO; 9309 max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); 9310 9311 mlxsw_reg_rgcr_pack(rgcr_pl, true, true); 9312 mlxsw_reg_rgcr_max_router_interfaces_set(rgcr_pl, max_rifs); 9313 mlxsw_reg_rgcr_usp_set(rgcr_pl, usp); 9314 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rgcr), rgcr_pl); 9315 } 9316 9317 static void __mlxsw_sp_router_fini(struct mlxsw_sp *mlxsw_sp) 9318 { 9319 char rgcr_pl[MLXSW_REG_RGCR_LEN]; 9320 9321 mlxsw_reg_rgcr_pack(rgcr_pl, false, false); 9322 mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rgcr), rgcr_pl); 9323 } 9324 9325 static const struct mlxsw_sp_router_ll_ops mlxsw_sp_router_ll_basic_ops = { 9326 .init = mlxsw_sp_router_ll_basic_init, 9327 .ralta_write = mlxsw_sp_router_ll_basic_ralta_write, 9328 .ralst_write = mlxsw_sp_router_ll_basic_ralst_write, 9329 .raltb_write = mlxsw_sp_router_ll_basic_raltb_write, 9330 .fib_entry_op_ctx_size = sizeof(struct mlxsw_sp_fib_entry_op_ctx_basic), 9331 .fib_entry_pack = mlxsw_sp_router_ll_basic_fib_entry_pack, 9332 .fib_entry_act_remote_pack = mlxsw_sp_router_ll_basic_fib_entry_act_remote_pack, 9333 .fib_entry_act_local_pack = mlxsw_sp_router_ll_basic_fib_entry_act_local_pack, 9334 .fib_entry_act_ip2me_pack = mlxsw_sp_router_ll_basic_fib_entry_act_ip2me_pack, 9335 .fib_entry_act_ip2me_tun_pack = mlxsw_sp_router_ll_basic_fib_entry_act_ip2me_tun_pack, 9336 .fib_entry_commit = mlxsw_sp_router_ll_basic_fib_entry_commit, 9337 .fib_entry_is_committed = mlxsw_sp_router_ll_basic_fib_entry_is_committed, 9338 }; 9339 9340 static int mlxsw_sp_router_ll_op_ctx_init(struct mlxsw_sp_router *router) 9341 { 9342 size_t max_size = 0; 9343 int i; 9344 9345 for (i = 0; i < MLXSW_SP_L3_PROTO_MAX; i++) { 9346 size_t size = router->proto_ll_ops[i]->fib_entry_op_ctx_size; 9347 9348 if (size > max_size) 9349 max_size = size; 9350 } 9351 router->ll_op_ctx = kzalloc(sizeof(*router->ll_op_ctx) + max_size, 9352 GFP_KERNEL); 9353 if (!router->ll_op_ctx) 9354 return -ENOMEM; 9355 INIT_LIST_HEAD(&router->ll_op_ctx->fib_entry_priv_list); 9356 return 0; 9357 } 9358 9359 static void mlxsw_sp_router_ll_op_ctx_fini(struct mlxsw_sp_router *router) 9360 { 9361 WARN_ON(!list_empty(&router->ll_op_ctx->fib_entry_priv_list)); 9362 kfree(router->ll_op_ctx); 9363 } 9364 9365 static int mlxsw_sp_lb_rif_init(struct mlxsw_sp *mlxsw_sp) 9366 { 9367 u16 lb_rif_index; 9368 int err; 9369 9370 /* Create a generic loopback RIF associated with the main table 9371 * (default VRF). Any table can be used, but the main table exists 9372 * anyway, so we do not waste resources. 9373 */ 9374 err = mlxsw_sp_router_ul_rif_get(mlxsw_sp, RT_TABLE_MAIN, 9375 &lb_rif_index); 9376 if (err) 9377 return err; 9378 9379 mlxsw_sp->router->lb_rif_index = lb_rif_index; 9380 9381 return 0; 9382 } 9383 9384 static void mlxsw_sp_lb_rif_fini(struct mlxsw_sp *mlxsw_sp) 9385 { 9386 mlxsw_sp_router_ul_rif_put(mlxsw_sp, mlxsw_sp->router->lb_rif_index); 9387 } 9388 9389 static int mlxsw_sp1_router_init(struct mlxsw_sp *mlxsw_sp) 9390 { 9391 size_t size_ranges_count = ARRAY_SIZE(mlxsw_sp1_adj_grp_size_ranges); 9392 9393 mlxsw_sp->router->rif_ops_arr = mlxsw_sp1_rif_ops_arr; 9394 mlxsw_sp->router->adj_grp_size_ranges = mlxsw_sp1_adj_grp_size_ranges; 9395 mlxsw_sp->router->adj_grp_size_ranges_count = size_ranges_count; 9396 9397 return 0; 9398 } 9399 9400 const struct mlxsw_sp_router_ops mlxsw_sp1_router_ops = { 9401 .init = mlxsw_sp1_router_init, 9402 }; 9403 9404 static int mlxsw_sp2_router_init(struct mlxsw_sp *mlxsw_sp) 9405 { 9406 size_t size_ranges_count = ARRAY_SIZE(mlxsw_sp2_adj_grp_size_ranges); 9407 9408 mlxsw_sp->router->rif_ops_arr = mlxsw_sp2_rif_ops_arr; 9409 mlxsw_sp->router->adj_grp_size_ranges = mlxsw_sp2_adj_grp_size_ranges; 9410 mlxsw_sp->router->adj_grp_size_ranges_count = size_ranges_count; 9411 9412 return 0; 9413 } 9414 9415 const struct mlxsw_sp_router_ops mlxsw_sp2_router_ops = { 9416 .init = mlxsw_sp2_router_init, 9417 }; 9418 9419 int mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp, 9420 struct netlink_ext_ack *extack) 9421 { 9422 struct mlxsw_sp_router *router; 9423 int err; 9424 9425 router = kzalloc(sizeof(*mlxsw_sp->router), GFP_KERNEL); 9426 if (!router) 9427 return -ENOMEM; 9428 mutex_init(&router->lock); 9429 mlxsw_sp->router = router; 9430 router->mlxsw_sp = mlxsw_sp; 9431 9432 err = mlxsw_sp->router_ops->init(mlxsw_sp); 9433 if (err) 9434 goto err_router_ops_init; 9435 9436 err = mlxsw_sp_router_xm_init(mlxsw_sp); 9437 if (err) 9438 goto err_xm_init; 9439 9440 router->proto_ll_ops[MLXSW_SP_L3_PROTO_IPV4] = mlxsw_sp_router_xm_ipv4_is_supported(mlxsw_sp) ? 9441 &mlxsw_sp_router_ll_xm_ops : 9442 &mlxsw_sp_router_ll_basic_ops; 9443 router->proto_ll_ops[MLXSW_SP_L3_PROTO_IPV6] = &mlxsw_sp_router_ll_basic_ops; 9444 9445 err = mlxsw_sp_router_ll_op_ctx_init(router); 9446 if (err) 9447 goto err_ll_op_ctx_init; 9448 9449 INIT_LIST_HEAD(&mlxsw_sp->router->nexthop_neighs_list); 9450 err = __mlxsw_sp_router_init(mlxsw_sp); 9451 if (err) 9452 goto err_router_init; 9453 9454 err = mlxsw_sp_rifs_init(mlxsw_sp); 9455 if (err) 9456 goto err_rifs_init; 9457 9458 err = mlxsw_sp_ipips_init(mlxsw_sp); 9459 if (err) 9460 goto err_ipips_init; 9461 9462 err = rhashtable_init(&mlxsw_sp->router->nexthop_ht, 9463 &mlxsw_sp_nexthop_ht_params); 9464 if (err) 9465 goto err_nexthop_ht_init; 9466 9467 err = rhashtable_init(&mlxsw_sp->router->nexthop_group_ht, 9468 &mlxsw_sp_nexthop_group_ht_params); 9469 if (err) 9470 goto err_nexthop_group_ht_init; 9471 9472 INIT_LIST_HEAD(&mlxsw_sp->router->nexthop_list); 9473 err = mlxsw_sp_lpm_init(mlxsw_sp); 9474 if (err) 9475 goto err_lpm_init; 9476 9477 err = mlxsw_sp_mr_init(mlxsw_sp, &mlxsw_sp_mr_tcam_ops); 9478 if (err) 9479 goto err_mr_init; 9480 9481 err = mlxsw_sp_vrs_init(mlxsw_sp); 9482 if (err) 9483 goto err_vrs_init; 9484 9485 err = mlxsw_sp_lb_rif_init(mlxsw_sp); 9486 if (err) 9487 goto err_lb_rif_init; 9488 9489 err = mlxsw_sp_neigh_init(mlxsw_sp); 9490 if (err) 9491 goto err_neigh_init; 9492 9493 err = mlxsw_sp_mp_hash_init(mlxsw_sp); 9494 if (err) 9495 goto err_mp_hash_init; 9496 9497 err = mlxsw_sp_dscp_init(mlxsw_sp); 9498 if (err) 9499 goto err_dscp_init; 9500 9501 INIT_WORK(&router->fib_event_work, mlxsw_sp_router_fib_event_work); 9502 INIT_LIST_HEAD(&router->fib_event_queue); 9503 spin_lock_init(&router->fib_event_queue_lock); 9504 9505 router->inetaddr_nb.notifier_call = mlxsw_sp_inetaddr_event; 9506 err = register_inetaddr_notifier(&router->inetaddr_nb); 9507 if (err) 9508 goto err_register_inetaddr_notifier; 9509 9510 router->inet6addr_nb.notifier_call = mlxsw_sp_inet6addr_event; 9511 err = register_inet6addr_notifier(&router->inet6addr_nb); 9512 if (err) 9513 goto err_register_inet6addr_notifier; 9514 9515 mlxsw_sp->router->netevent_nb.notifier_call = 9516 mlxsw_sp_router_netevent_event; 9517 err = register_netevent_notifier(&mlxsw_sp->router->netevent_nb); 9518 if (err) 9519 goto err_register_netevent_notifier; 9520 9521 mlxsw_sp->router->nexthop_nb.notifier_call = 9522 mlxsw_sp_nexthop_obj_event; 9523 err = register_nexthop_notifier(mlxsw_sp_net(mlxsw_sp), 9524 &mlxsw_sp->router->nexthop_nb, 9525 extack); 9526 if (err) 9527 goto err_register_nexthop_notifier; 9528 9529 mlxsw_sp->router->fib_nb.notifier_call = mlxsw_sp_router_fib_event; 9530 err = register_fib_notifier(mlxsw_sp_net(mlxsw_sp), 9531 &mlxsw_sp->router->fib_nb, 9532 mlxsw_sp_router_fib_dump_flush, extack); 9533 if (err) 9534 goto err_register_fib_notifier; 9535 9536 return 0; 9537 9538 err_register_fib_notifier: 9539 unregister_nexthop_notifier(mlxsw_sp_net(mlxsw_sp), 9540 &mlxsw_sp->router->nexthop_nb); 9541 err_register_nexthop_notifier: 9542 unregister_netevent_notifier(&mlxsw_sp->router->netevent_nb); 9543 err_register_netevent_notifier: 9544 unregister_inet6addr_notifier(&router->inet6addr_nb); 9545 err_register_inet6addr_notifier: 9546 unregister_inetaddr_notifier(&router->inetaddr_nb); 9547 err_register_inetaddr_notifier: 9548 mlxsw_core_flush_owq(); 9549 WARN_ON(!list_empty(&router->fib_event_queue)); 9550 err_dscp_init: 9551 err_mp_hash_init: 9552 mlxsw_sp_neigh_fini(mlxsw_sp); 9553 err_neigh_init: 9554 mlxsw_sp_lb_rif_fini(mlxsw_sp); 9555 err_lb_rif_init: 9556 mlxsw_sp_vrs_fini(mlxsw_sp); 9557 err_vrs_init: 9558 mlxsw_sp_mr_fini(mlxsw_sp); 9559 err_mr_init: 9560 mlxsw_sp_lpm_fini(mlxsw_sp); 9561 err_lpm_init: 9562 rhashtable_destroy(&mlxsw_sp->router->nexthop_group_ht); 9563 err_nexthop_group_ht_init: 9564 rhashtable_destroy(&mlxsw_sp->router->nexthop_ht); 9565 err_nexthop_ht_init: 9566 mlxsw_sp_ipips_fini(mlxsw_sp); 9567 err_ipips_init: 9568 mlxsw_sp_rifs_fini(mlxsw_sp); 9569 err_rifs_init: 9570 __mlxsw_sp_router_fini(mlxsw_sp); 9571 err_router_init: 9572 mlxsw_sp_router_ll_op_ctx_fini(router); 9573 err_ll_op_ctx_init: 9574 mlxsw_sp_router_xm_fini(mlxsw_sp); 9575 err_xm_init: 9576 err_router_ops_init: 9577 mutex_destroy(&mlxsw_sp->router->lock); 9578 kfree(mlxsw_sp->router); 9579 return err; 9580 } 9581 9582 void mlxsw_sp_router_fini(struct mlxsw_sp *mlxsw_sp) 9583 { 9584 unregister_fib_notifier(mlxsw_sp_net(mlxsw_sp), 9585 &mlxsw_sp->router->fib_nb); 9586 unregister_nexthop_notifier(mlxsw_sp_net(mlxsw_sp), 9587 &mlxsw_sp->router->nexthop_nb); 9588 unregister_netevent_notifier(&mlxsw_sp->router->netevent_nb); 9589 unregister_inet6addr_notifier(&mlxsw_sp->router->inet6addr_nb); 9590 unregister_inetaddr_notifier(&mlxsw_sp->router->inetaddr_nb); 9591 mlxsw_core_flush_owq(); 9592 WARN_ON(!list_empty(&mlxsw_sp->router->fib_event_queue)); 9593 mlxsw_sp_neigh_fini(mlxsw_sp); 9594 mlxsw_sp_lb_rif_fini(mlxsw_sp); 9595 mlxsw_sp_vrs_fini(mlxsw_sp); 9596 mlxsw_sp_mr_fini(mlxsw_sp); 9597 mlxsw_sp_lpm_fini(mlxsw_sp); 9598 rhashtable_destroy(&mlxsw_sp->router->nexthop_group_ht); 9599 rhashtable_destroy(&mlxsw_sp->router->nexthop_ht); 9600 mlxsw_sp_ipips_fini(mlxsw_sp); 9601 mlxsw_sp_rifs_fini(mlxsw_sp); 9602 __mlxsw_sp_router_fini(mlxsw_sp); 9603 mlxsw_sp_router_ll_op_ctx_fini(mlxsw_sp->router); 9604 mlxsw_sp_router_xm_fini(mlxsw_sp); 9605 mutex_destroy(&mlxsw_sp->router->lock); 9606 kfree(mlxsw_sp->router); 9607 } 9608