1 // SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0 2 /* Copyright (c) 2016-2018 Mellanox Technologies. All rights reserved */ 3 4 #include <linux/kernel.h> 5 #include <linux/types.h> 6 #include <linux/rhashtable.h> 7 #include <linux/bitops.h> 8 #include <linux/in6.h> 9 #include <linux/notifier.h> 10 #include <linux/inetdevice.h> 11 #include <linux/netdevice.h> 12 #include <linux/if_bridge.h> 13 #include <linux/socket.h> 14 #include <linux/route.h> 15 #include <linux/gcd.h> 16 #include <linux/if_macvlan.h> 17 #include <linux/refcount.h> 18 #include <linux/jhash.h> 19 #include <linux/net_namespace.h> 20 #include <linux/mutex.h> 21 #include <net/netevent.h> 22 #include <net/neighbour.h> 23 #include <net/arp.h> 24 #include <net/ip_fib.h> 25 #include <net/ip6_fib.h> 26 #include <net/nexthop.h> 27 #include <net/fib_rules.h> 28 #include <net/ip_tunnels.h> 29 #include <net/l3mdev.h> 30 #include <net/addrconf.h> 31 #include <net/ndisc.h> 32 #include <net/ipv6.h> 33 #include <net/fib_notifier.h> 34 #include <net/switchdev.h> 35 36 #include "spectrum.h" 37 #include "core.h" 38 #include "reg.h" 39 #include "spectrum_cnt.h" 40 #include "spectrum_dpipe.h" 41 #include "spectrum_ipip.h" 42 #include "spectrum_mr.h" 43 #include "spectrum_mr_tcam.h" 44 #include "spectrum_router.h" 45 #include "spectrum_span.h" 46 47 struct mlxsw_sp_fib; 48 struct mlxsw_sp_vr; 49 struct mlxsw_sp_lpm_tree; 50 struct mlxsw_sp_rif_ops; 51 52 struct mlxsw_sp_rif { 53 struct list_head nexthop_list; 54 struct list_head neigh_list; 55 struct net_device *dev; /* NULL for underlay RIF */ 56 struct mlxsw_sp_fid *fid; 57 unsigned char addr[ETH_ALEN]; 58 int mtu; 59 u16 rif_index; 60 u16 vr_id; 61 const struct mlxsw_sp_rif_ops *ops; 62 struct mlxsw_sp *mlxsw_sp; 63 64 unsigned int counter_ingress; 65 bool counter_ingress_valid; 66 unsigned int counter_egress; 67 bool counter_egress_valid; 68 }; 69 70 struct mlxsw_sp_rif_params { 71 struct net_device *dev; 72 union { 73 u16 system_port; 74 u16 lag_id; 75 }; 76 u16 vid; 77 bool lag; 78 }; 79 80 struct mlxsw_sp_rif_subport { 81 struct mlxsw_sp_rif common; 82 refcount_t ref_count; 83 union { 84 u16 system_port; 85 u16 lag_id; 86 }; 87 u16 vid; 88 bool lag; 89 }; 90 91 struct mlxsw_sp_rif_ipip_lb { 92 struct mlxsw_sp_rif common; 93 struct mlxsw_sp_rif_ipip_lb_config lb_config; 94 u16 ul_vr_id; /* Reserved for Spectrum-2. */ 95 u16 ul_rif_id; /* Reserved for Spectrum. */ 96 }; 97 98 struct mlxsw_sp_rif_params_ipip_lb { 99 struct mlxsw_sp_rif_params common; 100 struct mlxsw_sp_rif_ipip_lb_config lb_config; 101 }; 102 103 struct mlxsw_sp_rif_ops { 104 enum mlxsw_sp_rif_type type; 105 size_t rif_size; 106 107 void (*setup)(struct mlxsw_sp_rif *rif, 108 const struct mlxsw_sp_rif_params *params); 109 int (*configure)(struct mlxsw_sp_rif *rif); 110 void (*deconfigure)(struct mlxsw_sp_rif *rif); 111 struct mlxsw_sp_fid * (*fid_get)(struct mlxsw_sp_rif *rif, 112 struct netlink_ext_ack *extack); 113 void (*fdb_del)(struct mlxsw_sp_rif *rif, const char *mac); 114 }; 115 116 static struct mlxsw_sp_rif * 117 mlxsw_sp_rif_find_by_dev(const struct mlxsw_sp *mlxsw_sp, 118 const struct net_device *dev); 119 static void mlxsw_sp_rif_destroy(struct mlxsw_sp_rif *rif); 120 static void mlxsw_sp_lpm_tree_hold(struct mlxsw_sp_lpm_tree *lpm_tree); 121 static void mlxsw_sp_lpm_tree_put(struct mlxsw_sp *mlxsw_sp, 122 struct mlxsw_sp_lpm_tree *lpm_tree); 123 static int mlxsw_sp_vr_lpm_tree_bind(struct mlxsw_sp *mlxsw_sp, 124 const struct mlxsw_sp_fib *fib, 125 u8 tree_id); 126 static int mlxsw_sp_vr_lpm_tree_unbind(struct mlxsw_sp *mlxsw_sp, 127 const struct mlxsw_sp_fib *fib); 128 129 static unsigned int * 130 mlxsw_sp_rif_p_counter_get(struct mlxsw_sp_rif *rif, 131 enum mlxsw_sp_rif_counter_dir dir) 132 { 133 switch (dir) { 134 case MLXSW_SP_RIF_COUNTER_EGRESS: 135 return &rif->counter_egress; 136 case MLXSW_SP_RIF_COUNTER_INGRESS: 137 return &rif->counter_ingress; 138 } 139 return NULL; 140 } 141 142 static bool 143 mlxsw_sp_rif_counter_valid_get(struct mlxsw_sp_rif *rif, 144 enum mlxsw_sp_rif_counter_dir dir) 145 { 146 switch (dir) { 147 case MLXSW_SP_RIF_COUNTER_EGRESS: 148 return rif->counter_egress_valid; 149 case MLXSW_SP_RIF_COUNTER_INGRESS: 150 return rif->counter_ingress_valid; 151 } 152 return false; 153 } 154 155 static void 156 mlxsw_sp_rif_counter_valid_set(struct mlxsw_sp_rif *rif, 157 enum mlxsw_sp_rif_counter_dir dir, 158 bool valid) 159 { 160 switch (dir) { 161 case MLXSW_SP_RIF_COUNTER_EGRESS: 162 rif->counter_egress_valid = valid; 163 break; 164 case MLXSW_SP_RIF_COUNTER_INGRESS: 165 rif->counter_ingress_valid = valid; 166 break; 167 } 168 } 169 170 static int mlxsw_sp_rif_counter_edit(struct mlxsw_sp *mlxsw_sp, u16 rif_index, 171 unsigned int counter_index, bool enable, 172 enum mlxsw_sp_rif_counter_dir dir) 173 { 174 char ritr_pl[MLXSW_REG_RITR_LEN]; 175 bool is_egress = false; 176 int err; 177 178 if (dir == MLXSW_SP_RIF_COUNTER_EGRESS) 179 is_egress = true; 180 mlxsw_reg_ritr_rif_pack(ritr_pl, rif_index); 181 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 182 if (err) 183 return err; 184 185 mlxsw_reg_ritr_counter_pack(ritr_pl, counter_index, enable, 186 is_egress); 187 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 188 } 189 190 int mlxsw_sp_rif_counter_value_get(struct mlxsw_sp *mlxsw_sp, 191 struct mlxsw_sp_rif *rif, 192 enum mlxsw_sp_rif_counter_dir dir, u64 *cnt) 193 { 194 char ricnt_pl[MLXSW_REG_RICNT_LEN]; 195 unsigned int *p_counter_index; 196 bool valid; 197 int err; 198 199 valid = mlxsw_sp_rif_counter_valid_get(rif, dir); 200 if (!valid) 201 return -EINVAL; 202 203 p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir); 204 if (!p_counter_index) 205 return -EINVAL; 206 mlxsw_reg_ricnt_pack(ricnt_pl, *p_counter_index, 207 MLXSW_REG_RICNT_OPCODE_NOP); 208 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ricnt), ricnt_pl); 209 if (err) 210 return err; 211 *cnt = mlxsw_reg_ricnt_good_unicast_packets_get(ricnt_pl); 212 return 0; 213 } 214 215 static int mlxsw_sp_rif_counter_clear(struct mlxsw_sp *mlxsw_sp, 216 unsigned int counter_index) 217 { 218 char ricnt_pl[MLXSW_REG_RICNT_LEN]; 219 220 mlxsw_reg_ricnt_pack(ricnt_pl, counter_index, 221 MLXSW_REG_RICNT_OPCODE_CLEAR); 222 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ricnt), ricnt_pl); 223 } 224 225 int mlxsw_sp_rif_counter_alloc(struct mlxsw_sp *mlxsw_sp, 226 struct mlxsw_sp_rif *rif, 227 enum mlxsw_sp_rif_counter_dir dir) 228 { 229 unsigned int *p_counter_index; 230 int err; 231 232 p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir); 233 if (!p_counter_index) 234 return -EINVAL; 235 err = mlxsw_sp_counter_alloc(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF, 236 p_counter_index); 237 if (err) 238 return err; 239 240 err = mlxsw_sp_rif_counter_clear(mlxsw_sp, *p_counter_index); 241 if (err) 242 goto err_counter_clear; 243 244 err = mlxsw_sp_rif_counter_edit(mlxsw_sp, rif->rif_index, 245 *p_counter_index, true, dir); 246 if (err) 247 goto err_counter_edit; 248 mlxsw_sp_rif_counter_valid_set(rif, dir, true); 249 return 0; 250 251 err_counter_edit: 252 err_counter_clear: 253 mlxsw_sp_counter_free(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF, 254 *p_counter_index); 255 return err; 256 } 257 258 void mlxsw_sp_rif_counter_free(struct mlxsw_sp *mlxsw_sp, 259 struct mlxsw_sp_rif *rif, 260 enum mlxsw_sp_rif_counter_dir dir) 261 { 262 unsigned int *p_counter_index; 263 264 if (!mlxsw_sp_rif_counter_valid_get(rif, dir)) 265 return; 266 267 p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir); 268 if (WARN_ON(!p_counter_index)) 269 return; 270 mlxsw_sp_rif_counter_edit(mlxsw_sp, rif->rif_index, 271 *p_counter_index, false, dir); 272 mlxsw_sp_counter_free(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF, 273 *p_counter_index); 274 mlxsw_sp_rif_counter_valid_set(rif, dir, false); 275 } 276 277 static void mlxsw_sp_rif_counters_alloc(struct mlxsw_sp_rif *rif) 278 { 279 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 280 struct devlink *devlink; 281 282 devlink = priv_to_devlink(mlxsw_sp->core); 283 if (!devlink_dpipe_table_counter_enabled(devlink, 284 MLXSW_SP_DPIPE_TABLE_NAME_ERIF)) 285 return; 286 mlxsw_sp_rif_counter_alloc(mlxsw_sp, rif, MLXSW_SP_RIF_COUNTER_EGRESS); 287 } 288 289 static void mlxsw_sp_rif_counters_free(struct mlxsw_sp_rif *rif) 290 { 291 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 292 293 mlxsw_sp_rif_counter_free(mlxsw_sp, rif, MLXSW_SP_RIF_COUNTER_EGRESS); 294 } 295 296 #define MLXSW_SP_PREFIX_COUNT (sizeof(struct in6_addr) * BITS_PER_BYTE + 1) 297 298 struct mlxsw_sp_prefix_usage { 299 DECLARE_BITMAP(b, MLXSW_SP_PREFIX_COUNT); 300 }; 301 302 #define mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) \ 303 for_each_set_bit(prefix, (prefix_usage)->b, MLXSW_SP_PREFIX_COUNT) 304 305 static bool 306 mlxsw_sp_prefix_usage_eq(struct mlxsw_sp_prefix_usage *prefix_usage1, 307 struct mlxsw_sp_prefix_usage *prefix_usage2) 308 { 309 return !memcmp(prefix_usage1, prefix_usage2, sizeof(*prefix_usage1)); 310 } 311 312 static void 313 mlxsw_sp_prefix_usage_cpy(struct mlxsw_sp_prefix_usage *prefix_usage1, 314 struct mlxsw_sp_prefix_usage *prefix_usage2) 315 { 316 memcpy(prefix_usage1, prefix_usage2, sizeof(*prefix_usage1)); 317 } 318 319 static void 320 mlxsw_sp_prefix_usage_set(struct mlxsw_sp_prefix_usage *prefix_usage, 321 unsigned char prefix_len) 322 { 323 set_bit(prefix_len, prefix_usage->b); 324 } 325 326 static void 327 mlxsw_sp_prefix_usage_clear(struct mlxsw_sp_prefix_usage *prefix_usage, 328 unsigned char prefix_len) 329 { 330 clear_bit(prefix_len, prefix_usage->b); 331 } 332 333 struct mlxsw_sp_fib_key { 334 unsigned char addr[sizeof(struct in6_addr)]; 335 unsigned char prefix_len; 336 }; 337 338 enum mlxsw_sp_fib_entry_type { 339 MLXSW_SP_FIB_ENTRY_TYPE_REMOTE, 340 MLXSW_SP_FIB_ENTRY_TYPE_LOCAL, 341 MLXSW_SP_FIB_ENTRY_TYPE_TRAP, 342 MLXSW_SP_FIB_ENTRY_TYPE_BLACKHOLE, 343 MLXSW_SP_FIB_ENTRY_TYPE_UNREACHABLE, 344 345 /* This is a special case of local delivery, where a packet should be 346 * decapsulated on reception. Note that there is no corresponding ENCAP, 347 * because that's a type of next hop, not of FIB entry. (There can be 348 * several next hops in a REMOTE entry, and some of them may be 349 * encapsulating entries.) 350 */ 351 MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP, 352 MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP, 353 }; 354 355 struct mlxsw_sp_nexthop_group_info; 356 struct mlxsw_sp_nexthop_group; 357 struct mlxsw_sp_fib_entry; 358 359 struct mlxsw_sp_fib_node { 360 struct mlxsw_sp_fib_entry *fib_entry; 361 struct list_head list; 362 struct rhash_head ht_node; 363 struct mlxsw_sp_fib *fib; 364 struct mlxsw_sp_fib_key key; 365 }; 366 367 struct mlxsw_sp_fib_entry_decap { 368 struct mlxsw_sp_ipip_entry *ipip_entry; 369 u32 tunnel_index; 370 }; 371 372 static struct mlxsw_sp_fib_entry_priv * 373 mlxsw_sp_fib_entry_priv_create(const struct mlxsw_sp_router_ll_ops *ll_ops) 374 { 375 struct mlxsw_sp_fib_entry_priv *priv; 376 377 if (!ll_ops->fib_entry_priv_size) 378 /* No need to have priv */ 379 return NULL; 380 381 priv = kzalloc(sizeof(*priv) + ll_ops->fib_entry_priv_size, GFP_KERNEL); 382 if (!priv) 383 return ERR_PTR(-ENOMEM); 384 refcount_set(&priv->refcnt, 1); 385 return priv; 386 } 387 388 static void 389 mlxsw_sp_fib_entry_priv_destroy(struct mlxsw_sp_fib_entry_priv *priv) 390 { 391 kfree(priv); 392 } 393 394 static void mlxsw_sp_fib_entry_priv_hold(struct mlxsw_sp_fib_entry_priv *priv) 395 { 396 refcount_inc(&priv->refcnt); 397 } 398 399 static void mlxsw_sp_fib_entry_priv_put(struct mlxsw_sp_fib_entry_priv *priv) 400 { 401 if (!priv || !refcount_dec_and_test(&priv->refcnt)) 402 return; 403 mlxsw_sp_fib_entry_priv_destroy(priv); 404 } 405 406 static void mlxsw_sp_fib_entry_op_ctx_priv_hold(struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 407 struct mlxsw_sp_fib_entry_priv *priv) 408 { 409 if (!priv) 410 return; 411 mlxsw_sp_fib_entry_priv_hold(priv); 412 list_add(&priv->list, &op_ctx->fib_entry_priv_list); 413 } 414 415 static void mlxsw_sp_fib_entry_op_ctx_priv_put_all(struct mlxsw_sp_fib_entry_op_ctx *op_ctx) 416 { 417 struct mlxsw_sp_fib_entry_priv *priv, *tmp; 418 419 list_for_each_entry_safe(priv, tmp, &op_ctx->fib_entry_priv_list, list) 420 mlxsw_sp_fib_entry_priv_put(priv); 421 INIT_LIST_HEAD(&op_ctx->fib_entry_priv_list); 422 } 423 424 struct mlxsw_sp_fib_entry { 425 struct mlxsw_sp_fib_node *fib_node; 426 enum mlxsw_sp_fib_entry_type type; 427 struct list_head nexthop_group_node; 428 struct mlxsw_sp_nexthop_group *nh_group; 429 struct mlxsw_sp_fib_entry_decap decap; /* Valid for decap entries. */ 430 struct mlxsw_sp_fib_entry_priv *priv; 431 }; 432 433 struct mlxsw_sp_fib4_entry { 434 struct mlxsw_sp_fib_entry common; 435 struct fib_info *fi; 436 u32 tb_id; 437 u8 tos; 438 u8 type; 439 }; 440 441 struct mlxsw_sp_fib6_entry { 442 struct mlxsw_sp_fib_entry common; 443 struct list_head rt6_list; 444 unsigned int nrt6; 445 }; 446 447 struct mlxsw_sp_rt6 { 448 struct list_head list; 449 struct fib6_info *rt; 450 }; 451 452 struct mlxsw_sp_lpm_tree { 453 u8 id; /* tree ID */ 454 unsigned int ref_count; 455 enum mlxsw_sp_l3proto proto; 456 unsigned long prefix_ref_count[MLXSW_SP_PREFIX_COUNT]; 457 struct mlxsw_sp_prefix_usage prefix_usage; 458 }; 459 460 struct mlxsw_sp_fib { 461 struct rhashtable ht; 462 struct list_head node_list; 463 struct mlxsw_sp_vr *vr; 464 struct mlxsw_sp_lpm_tree *lpm_tree; 465 enum mlxsw_sp_l3proto proto; 466 const struct mlxsw_sp_router_ll_ops *ll_ops; 467 }; 468 469 struct mlxsw_sp_vr { 470 u16 id; /* virtual router ID */ 471 u32 tb_id; /* kernel fib table id */ 472 unsigned int rif_count; 473 struct mlxsw_sp_fib *fib4; 474 struct mlxsw_sp_fib *fib6; 475 struct mlxsw_sp_mr_table *mr_table[MLXSW_SP_L3_PROTO_MAX]; 476 struct mlxsw_sp_rif *ul_rif; 477 refcount_t ul_rif_refcnt; 478 }; 479 480 static int mlxsw_sp_router_ll_basic_init(struct mlxsw_sp *mlxsw_sp, u16 vr_id, 481 enum mlxsw_sp_l3proto proto) 482 { 483 return 0; 484 } 485 486 static int mlxsw_sp_router_ll_basic_ralta_write(struct mlxsw_sp *mlxsw_sp, char *xralta_pl) 487 { 488 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), 489 xralta_pl + MLXSW_REG_XRALTA_RALTA_OFFSET); 490 } 491 492 static int mlxsw_sp_router_ll_basic_ralst_write(struct mlxsw_sp *mlxsw_sp, char *xralst_pl) 493 { 494 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralst), 495 xralst_pl + MLXSW_REG_XRALST_RALST_OFFSET); 496 } 497 498 static int mlxsw_sp_router_ll_basic_raltb_write(struct mlxsw_sp *mlxsw_sp, char *xraltb_pl) 499 { 500 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb), 501 xraltb_pl + MLXSW_REG_XRALTB_RALTB_OFFSET); 502 } 503 504 static const struct rhashtable_params mlxsw_sp_fib_ht_params; 505 506 static struct mlxsw_sp_fib *mlxsw_sp_fib_create(struct mlxsw_sp *mlxsw_sp, 507 struct mlxsw_sp_vr *vr, 508 enum mlxsw_sp_l3proto proto) 509 { 510 const struct mlxsw_sp_router_ll_ops *ll_ops = mlxsw_sp->router->proto_ll_ops[proto]; 511 struct mlxsw_sp_lpm_tree *lpm_tree; 512 struct mlxsw_sp_fib *fib; 513 int err; 514 515 err = ll_ops->init(mlxsw_sp, vr->id, proto); 516 if (err) 517 return ERR_PTR(err); 518 519 lpm_tree = mlxsw_sp->router->lpm.proto_trees[proto]; 520 fib = kzalloc(sizeof(*fib), GFP_KERNEL); 521 if (!fib) 522 return ERR_PTR(-ENOMEM); 523 err = rhashtable_init(&fib->ht, &mlxsw_sp_fib_ht_params); 524 if (err) 525 goto err_rhashtable_init; 526 INIT_LIST_HEAD(&fib->node_list); 527 fib->proto = proto; 528 fib->vr = vr; 529 fib->lpm_tree = lpm_tree; 530 fib->ll_ops = ll_ops; 531 mlxsw_sp_lpm_tree_hold(lpm_tree); 532 err = mlxsw_sp_vr_lpm_tree_bind(mlxsw_sp, fib, lpm_tree->id); 533 if (err) 534 goto err_lpm_tree_bind; 535 return fib; 536 537 err_lpm_tree_bind: 538 mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree); 539 err_rhashtable_init: 540 kfree(fib); 541 return ERR_PTR(err); 542 } 543 544 static void mlxsw_sp_fib_destroy(struct mlxsw_sp *mlxsw_sp, 545 struct mlxsw_sp_fib *fib) 546 { 547 mlxsw_sp_vr_lpm_tree_unbind(mlxsw_sp, fib); 548 mlxsw_sp_lpm_tree_put(mlxsw_sp, fib->lpm_tree); 549 WARN_ON(!list_empty(&fib->node_list)); 550 rhashtable_destroy(&fib->ht); 551 kfree(fib); 552 } 553 554 static struct mlxsw_sp_lpm_tree * 555 mlxsw_sp_lpm_tree_find_unused(struct mlxsw_sp *mlxsw_sp) 556 { 557 static struct mlxsw_sp_lpm_tree *lpm_tree; 558 int i; 559 560 for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) { 561 lpm_tree = &mlxsw_sp->router->lpm.trees[i]; 562 if (lpm_tree->ref_count == 0) 563 return lpm_tree; 564 } 565 return NULL; 566 } 567 568 static int mlxsw_sp_lpm_tree_alloc(struct mlxsw_sp *mlxsw_sp, 569 const struct mlxsw_sp_router_ll_ops *ll_ops, 570 struct mlxsw_sp_lpm_tree *lpm_tree) 571 { 572 char xralta_pl[MLXSW_REG_XRALTA_LEN]; 573 574 mlxsw_reg_xralta_pack(xralta_pl, true, 575 (enum mlxsw_reg_ralxx_protocol) lpm_tree->proto, 576 lpm_tree->id); 577 return ll_ops->ralta_write(mlxsw_sp, xralta_pl); 578 } 579 580 static void mlxsw_sp_lpm_tree_free(struct mlxsw_sp *mlxsw_sp, 581 const struct mlxsw_sp_router_ll_ops *ll_ops, 582 struct mlxsw_sp_lpm_tree *lpm_tree) 583 { 584 char xralta_pl[MLXSW_REG_XRALTA_LEN]; 585 586 mlxsw_reg_xralta_pack(xralta_pl, false, 587 (enum mlxsw_reg_ralxx_protocol) lpm_tree->proto, 588 lpm_tree->id); 589 ll_ops->ralta_write(mlxsw_sp, xralta_pl); 590 } 591 592 static int 593 mlxsw_sp_lpm_tree_left_struct_set(struct mlxsw_sp *mlxsw_sp, 594 const struct mlxsw_sp_router_ll_ops *ll_ops, 595 struct mlxsw_sp_prefix_usage *prefix_usage, 596 struct mlxsw_sp_lpm_tree *lpm_tree) 597 { 598 char xralst_pl[MLXSW_REG_XRALST_LEN]; 599 u8 root_bin = 0; 600 u8 prefix; 601 u8 last_prefix = MLXSW_REG_RALST_BIN_NO_CHILD; 602 603 mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) 604 root_bin = prefix; 605 606 mlxsw_reg_xralst_pack(xralst_pl, root_bin, lpm_tree->id); 607 mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) { 608 if (prefix == 0) 609 continue; 610 mlxsw_reg_xralst_bin_pack(xralst_pl, prefix, last_prefix, 611 MLXSW_REG_RALST_BIN_NO_CHILD); 612 last_prefix = prefix; 613 } 614 return ll_ops->ralst_write(mlxsw_sp, xralst_pl); 615 } 616 617 static struct mlxsw_sp_lpm_tree * 618 mlxsw_sp_lpm_tree_create(struct mlxsw_sp *mlxsw_sp, 619 const struct mlxsw_sp_router_ll_ops *ll_ops, 620 struct mlxsw_sp_prefix_usage *prefix_usage, 621 enum mlxsw_sp_l3proto proto) 622 { 623 struct mlxsw_sp_lpm_tree *lpm_tree; 624 int err; 625 626 lpm_tree = mlxsw_sp_lpm_tree_find_unused(mlxsw_sp); 627 if (!lpm_tree) 628 return ERR_PTR(-EBUSY); 629 lpm_tree->proto = proto; 630 err = mlxsw_sp_lpm_tree_alloc(mlxsw_sp, ll_ops, lpm_tree); 631 if (err) 632 return ERR_PTR(err); 633 634 err = mlxsw_sp_lpm_tree_left_struct_set(mlxsw_sp, ll_ops, prefix_usage, lpm_tree); 635 if (err) 636 goto err_left_struct_set; 637 memcpy(&lpm_tree->prefix_usage, prefix_usage, 638 sizeof(lpm_tree->prefix_usage)); 639 memset(&lpm_tree->prefix_ref_count, 0, 640 sizeof(lpm_tree->prefix_ref_count)); 641 lpm_tree->ref_count = 1; 642 return lpm_tree; 643 644 err_left_struct_set: 645 mlxsw_sp_lpm_tree_free(mlxsw_sp, ll_ops, lpm_tree); 646 return ERR_PTR(err); 647 } 648 649 static void mlxsw_sp_lpm_tree_destroy(struct mlxsw_sp *mlxsw_sp, 650 const struct mlxsw_sp_router_ll_ops *ll_ops, 651 struct mlxsw_sp_lpm_tree *lpm_tree) 652 { 653 mlxsw_sp_lpm_tree_free(mlxsw_sp, ll_ops, lpm_tree); 654 } 655 656 static struct mlxsw_sp_lpm_tree * 657 mlxsw_sp_lpm_tree_get(struct mlxsw_sp *mlxsw_sp, 658 struct mlxsw_sp_prefix_usage *prefix_usage, 659 enum mlxsw_sp_l3proto proto) 660 { 661 const struct mlxsw_sp_router_ll_ops *ll_ops = mlxsw_sp->router->proto_ll_ops[proto]; 662 struct mlxsw_sp_lpm_tree *lpm_tree; 663 int i; 664 665 for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) { 666 lpm_tree = &mlxsw_sp->router->lpm.trees[i]; 667 if (lpm_tree->ref_count != 0 && 668 lpm_tree->proto == proto && 669 mlxsw_sp_prefix_usage_eq(&lpm_tree->prefix_usage, 670 prefix_usage)) { 671 mlxsw_sp_lpm_tree_hold(lpm_tree); 672 return lpm_tree; 673 } 674 } 675 return mlxsw_sp_lpm_tree_create(mlxsw_sp, ll_ops, prefix_usage, proto); 676 } 677 678 static void mlxsw_sp_lpm_tree_hold(struct mlxsw_sp_lpm_tree *lpm_tree) 679 { 680 lpm_tree->ref_count++; 681 } 682 683 static void mlxsw_sp_lpm_tree_put(struct mlxsw_sp *mlxsw_sp, 684 struct mlxsw_sp_lpm_tree *lpm_tree) 685 { 686 const struct mlxsw_sp_router_ll_ops *ll_ops = 687 mlxsw_sp->router->proto_ll_ops[lpm_tree->proto]; 688 689 if (--lpm_tree->ref_count == 0) 690 mlxsw_sp_lpm_tree_destroy(mlxsw_sp, ll_ops, lpm_tree); 691 } 692 693 #define MLXSW_SP_LPM_TREE_MIN 1 /* tree 0 is reserved */ 694 695 static int mlxsw_sp_lpm_init(struct mlxsw_sp *mlxsw_sp) 696 { 697 struct mlxsw_sp_prefix_usage req_prefix_usage = {{ 0 } }; 698 struct mlxsw_sp_lpm_tree *lpm_tree; 699 u64 max_trees; 700 int err, i; 701 702 if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_LPM_TREES)) 703 return -EIO; 704 705 max_trees = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_LPM_TREES); 706 mlxsw_sp->router->lpm.tree_count = max_trees - MLXSW_SP_LPM_TREE_MIN; 707 mlxsw_sp->router->lpm.trees = kcalloc(mlxsw_sp->router->lpm.tree_count, 708 sizeof(struct mlxsw_sp_lpm_tree), 709 GFP_KERNEL); 710 if (!mlxsw_sp->router->lpm.trees) 711 return -ENOMEM; 712 713 for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) { 714 lpm_tree = &mlxsw_sp->router->lpm.trees[i]; 715 lpm_tree->id = i + MLXSW_SP_LPM_TREE_MIN; 716 } 717 718 lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage, 719 MLXSW_SP_L3_PROTO_IPV4); 720 if (IS_ERR(lpm_tree)) { 721 err = PTR_ERR(lpm_tree); 722 goto err_ipv4_tree_get; 723 } 724 mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV4] = lpm_tree; 725 726 lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage, 727 MLXSW_SP_L3_PROTO_IPV6); 728 if (IS_ERR(lpm_tree)) { 729 err = PTR_ERR(lpm_tree); 730 goto err_ipv6_tree_get; 731 } 732 mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV6] = lpm_tree; 733 734 return 0; 735 736 err_ipv6_tree_get: 737 lpm_tree = mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV4]; 738 mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree); 739 err_ipv4_tree_get: 740 kfree(mlxsw_sp->router->lpm.trees); 741 return err; 742 } 743 744 static void mlxsw_sp_lpm_fini(struct mlxsw_sp *mlxsw_sp) 745 { 746 struct mlxsw_sp_lpm_tree *lpm_tree; 747 748 lpm_tree = mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV6]; 749 mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree); 750 751 lpm_tree = mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV4]; 752 mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree); 753 754 kfree(mlxsw_sp->router->lpm.trees); 755 } 756 757 static bool mlxsw_sp_vr_is_used(const struct mlxsw_sp_vr *vr) 758 { 759 return !!vr->fib4 || !!vr->fib6 || 760 !!vr->mr_table[MLXSW_SP_L3_PROTO_IPV4] || 761 !!vr->mr_table[MLXSW_SP_L3_PROTO_IPV6]; 762 } 763 764 static struct mlxsw_sp_vr *mlxsw_sp_vr_find_unused(struct mlxsw_sp *mlxsw_sp) 765 { 766 struct mlxsw_sp_vr *vr; 767 int i; 768 769 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) { 770 vr = &mlxsw_sp->router->vrs[i]; 771 if (!mlxsw_sp_vr_is_used(vr)) 772 return vr; 773 } 774 return NULL; 775 } 776 777 static int mlxsw_sp_vr_lpm_tree_bind(struct mlxsw_sp *mlxsw_sp, 778 const struct mlxsw_sp_fib *fib, u8 tree_id) 779 { 780 char xraltb_pl[MLXSW_REG_XRALTB_LEN]; 781 782 mlxsw_reg_xraltb_pack(xraltb_pl, fib->vr->id, 783 (enum mlxsw_reg_ralxx_protocol) fib->proto, 784 tree_id); 785 return fib->ll_ops->raltb_write(mlxsw_sp, xraltb_pl); 786 } 787 788 static int mlxsw_sp_vr_lpm_tree_unbind(struct mlxsw_sp *mlxsw_sp, 789 const struct mlxsw_sp_fib *fib) 790 { 791 char xraltb_pl[MLXSW_REG_XRALTB_LEN]; 792 793 /* Bind to tree 0 which is default */ 794 mlxsw_reg_xraltb_pack(xraltb_pl, fib->vr->id, 795 (enum mlxsw_reg_ralxx_protocol) fib->proto, 0); 796 return fib->ll_ops->raltb_write(mlxsw_sp, xraltb_pl); 797 } 798 799 static u32 mlxsw_sp_fix_tb_id(u32 tb_id) 800 { 801 /* For our purpose, squash main, default and local tables into one */ 802 if (tb_id == RT_TABLE_LOCAL || tb_id == RT_TABLE_DEFAULT) 803 tb_id = RT_TABLE_MAIN; 804 return tb_id; 805 } 806 807 static struct mlxsw_sp_vr *mlxsw_sp_vr_find(struct mlxsw_sp *mlxsw_sp, 808 u32 tb_id) 809 { 810 struct mlxsw_sp_vr *vr; 811 int i; 812 813 tb_id = mlxsw_sp_fix_tb_id(tb_id); 814 815 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) { 816 vr = &mlxsw_sp->router->vrs[i]; 817 if (mlxsw_sp_vr_is_used(vr) && vr->tb_id == tb_id) 818 return vr; 819 } 820 return NULL; 821 } 822 823 int mlxsw_sp_router_tb_id_vr_id(struct mlxsw_sp *mlxsw_sp, u32 tb_id, 824 u16 *vr_id) 825 { 826 struct mlxsw_sp_vr *vr; 827 int err = 0; 828 829 mutex_lock(&mlxsw_sp->router->lock); 830 vr = mlxsw_sp_vr_find(mlxsw_sp, tb_id); 831 if (!vr) { 832 err = -ESRCH; 833 goto out; 834 } 835 *vr_id = vr->id; 836 out: 837 mutex_unlock(&mlxsw_sp->router->lock); 838 return err; 839 } 840 841 static struct mlxsw_sp_fib *mlxsw_sp_vr_fib(const struct mlxsw_sp_vr *vr, 842 enum mlxsw_sp_l3proto proto) 843 { 844 switch (proto) { 845 case MLXSW_SP_L3_PROTO_IPV4: 846 return vr->fib4; 847 case MLXSW_SP_L3_PROTO_IPV6: 848 return vr->fib6; 849 } 850 return NULL; 851 } 852 853 static struct mlxsw_sp_vr *mlxsw_sp_vr_create(struct mlxsw_sp *mlxsw_sp, 854 u32 tb_id, 855 struct netlink_ext_ack *extack) 856 { 857 struct mlxsw_sp_mr_table *mr4_table, *mr6_table; 858 struct mlxsw_sp_fib *fib4; 859 struct mlxsw_sp_fib *fib6; 860 struct mlxsw_sp_vr *vr; 861 int err; 862 863 vr = mlxsw_sp_vr_find_unused(mlxsw_sp); 864 if (!vr) { 865 NL_SET_ERR_MSG_MOD(extack, "Exceeded number of supported virtual routers"); 866 return ERR_PTR(-EBUSY); 867 } 868 fib4 = mlxsw_sp_fib_create(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV4); 869 if (IS_ERR(fib4)) 870 return ERR_CAST(fib4); 871 fib6 = mlxsw_sp_fib_create(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV6); 872 if (IS_ERR(fib6)) { 873 err = PTR_ERR(fib6); 874 goto err_fib6_create; 875 } 876 mr4_table = mlxsw_sp_mr_table_create(mlxsw_sp, vr->id, 877 MLXSW_SP_L3_PROTO_IPV4); 878 if (IS_ERR(mr4_table)) { 879 err = PTR_ERR(mr4_table); 880 goto err_mr4_table_create; 881 } 882 mr6_table = mlxsw_sp_mr_table_create(mlxsw_sp, vr->id, 883 MLXSW_SP_L3_PROTO_IPV6); 884 if (IS_ERR(mr6_table)) { 885 err = PTR_ERR(mr6_table); 886 goto err_mr6_table_create; 887 } 888 889 vr->fib4 = fib4; 890 vr->fib6 = fib6; 891 vr->mr_table[MLXSW_SP_L3_PROTO_IPV4] = mr4_table; 892 vr->mr_table[MLXSW_SP_L3_PROTO_IPV6] = mr6_table; 893 vr->tb_id = tb_id; 894 return vr; 895 896 err_mr6_table_create: 897 mlxsw_sp_mr_table_destroy(mr4_table); 898 err_mr4_table_create: 899 mlxsw_sp_fib_destroy(mlxsw_sp, fib6); 900 err_fib6_create: 901 mlxsw_sp_fib_destroy(mlxsw_sp, fib4); 902 return ERR_PTR(err); 903 } 904 905 static void mlxsw_sp_vr_destroy(struct mlxsw_sp *mlxsw_sp, 906 struct mlxsw_sp_vr *vr) 907 { 908 mlxsw_sp_mr_table_destroy(vr->mr_table[MLXSW_SP_L3_PROTO_IPV6]); 909 vr->mr_table[MLXSW_SP_L3_PROTO_IPV6] = NULL; 910 mlxsw_sp_mr_table_destroy(vr->mr_table[MLXSW_SP_L3_PROTO_IPV4]); 911 vr->mr_table[MLXSW_SP_L3_PROTO_IPV4] = NULL; 912 mlxsw_sp_fib_destroy(mlxsw_sp, vr->fib6); 913 vr->fib6 = NULL; 914 mlxsw_sp_fib_destroy(mlxsw_sp, vr->fib4); 915 vr->fib4 = NULL; 916 } 917 918 static struct mlxsw_sp_vr *mlxsw_sp_vr_get(struct mlxsw_sp *mlxsw_sp, u32 tb_id, 919 struct netlink_ext_ack *extack) 920 { 921 struct mlxsw_sp_vr *vr; 922 923 tb_id = mlxsw_sp_fix_tb_id(tb_id); 924 vr = mlxsw_sp_vr_find(mlxsw_sp, tb_id); 925 if (!vr) 926 vr = mlxsw_sp_vr_create(mlxsw_sp, tb_id, extack); 927 return vr; 928 } 929 930 static void mlxsw_sp_vr_put(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp_vr *vr) 931 { 932 if (!vr->rif_count && list_empty(&vr->fib4->node_list) && 933 list_empty(&vr->fib6->node_list) && 934 mlxsw_sp_mr_table_empty(vr->mr_table[MLXSW_SP_L3_PROTO_IPV4]) && 935 mlxsw_sp_mr_table_empty(vr->mr_table[MLXSW_SP_L3_PROTO_IPV6])) 936 mlxsw_sp_vr_destroy(mlxsw_sp, vr); 937 } 938 939 static bool 940 mlxsw_sp_vr_lpm_tree_should_replace(struct mlxsw_sp_vr *vr, 941 enum mlxsw_sp_l3proto proto, u8 tree_id) 942 { 943 struct mlxsw_sp_fib *fib = mlxsw_sp_vr_fib(vr, proto); 944 945 if (!mlxsw_sp_vr_is_used(vr)) 946 return false; 947 if (fib->lpm_tree->id == tree_id) 948 return true; 949 return false; 950 } 951 952 static int mlxsw_sp_vr_lpm_tree_replace(struct mlxsw_sp *mlxsw_sp, 953 struct mlxsw_sp_fib *fib, 954 struct mlxsw_sp_lpm_tree *new_tree) 955 { 956 struct mlxsw_sp_lpm_tree *old_tree = fib->lpm_tree; 957 int err; 958 959 fib->lpm_tree = new_tree; 960 mlxsw_sp_lpm_tree_hold(new_tree); 961 err = mlxsw_sp_vr_lpm_tree_bind(mlxsw_sp, fib, new_tree->id); 962 if (err) 963 goto err_tree_bind; 964 mlxsw_sp_lpm_tree_put(mlxsw_sp, old_tree); 965 return 0; 966 967 err_tree_bind: 968 mlxsw_sp_lpm_tree_put(mlxsw_sp, new_tree); 969 fib->lpm_tree = old_tree; 970 return err; 971 } 972 973 static int mlxsw_sp_vrs_lpm_tree_replace(struct mlxsw_sp *mlxsw_sp, 974 struct mlxsw_sp_fib *fib, 975 struct mlxsw_sp_lpm_tree *new_tree) 976 { 977 enum mlxsw_sp_l3proto proto = fib->proto; 978 struct mlxsw_sp_lpm_tree *old_tree; 979 u8 old_id, new_id = new_tree->id; 980 struct mlxsw_sp_vr *vr; 981 int i, err; 982 983 old_tree = mlxsw_sp->router->lpm.proto_trees[proto]; 984 old_id = old_tree->id; 985 986 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) { 987 vr = &mlxsw_sp->router->vrs[i]; 988 if (!mlxsw_sp_vr_lpm_tree_should_replace(vr, proto, old_id)) 989 continue; 990 err = mlxsw_sp_vr_lpm_tree_replace(mlxsw_sp, 991 mlxsw_sp_vr_fib(vr, proto), 992 new_tree); 993 if (err) 994 goto err_tree_replace; 995 } 996 997 memcpy(new_tree->prefix_ref_count, old_tree->prefix_ref_count, 998 sizeof(new_tree->prefix_ref_count)); 999 mlxsw_sp->router->lpm.proto_trees[proto] = new_tree; 1000 mlxsw_sp_lpm_tree_put(mlxsw_sp, old_tree); 1001 1002 return 0; 1003 1004 err_tree_replace: 1005 for (i--; i >= 0; i--) { 1006 if (!mlxsw_sp_vr_lpm_tree_should_replace(vr, proto, new_id)) 1007 continue; 1008 mlxsw_sp_vr_lpm_tree_replace(mlxsw_sp, 1009 mlxsw_sp_vr_fib(vr, proto), 1010 old_tree); 1011 } 1012 return err; 1013 } 1014 1015 static int mlxsw_sp_vrs_init(struct mlxsw_sp *mlxsw_sp) 1016 { 1017 struct mlxsw_sp_vr *vr; 1018 u64 max_vrs; 1019 int i; 1020 1021 if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_VRS)) 1022 return -EIO; 1023 1024 max_vrs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); 1025 mlxsw_sp->router->vrs = kcalloc(max_vrs, sizeof(struct mlxsw_sp_vr), 1026 GFP_KERNEL); 1027 if (!mlxsw_sp->router->vrs) 1028 return -ENOMEM; 1029 1030 for (i = 0; i < max_vrs; i++) { 1031 vr = &mlxsw_sp->router->vrs[i]; 1032 vr->id = i; 1033 } 1034 1035 return 0; 1036 } 1037 1038 static void mlxsw_sp_router_fib_flush(struct mlxsw_sp *mlxsw_sp); 1039 1040 static void mlxsw_sp_vrs_fini(struct mlxsw_sp *mlxsw_sp) 1041 { 1042 /* At this stage we're guaranteed not to have new incoming 1043 * FIB notifications and the work queue is free from FIBs 1044 * sitting on top of mlxsw netdevs. However, we can still 1045 * have other FIBs queued. Flush the queue before flushing 1046 * the device's tables. No need for locks, as we're the only 1047 * writer. 1048 */ 1049 mlxsw_core_flush_owq(); 1050 mlxsw_sp_router_fib_flush(mlxsw_sp); 1051 kfree(mlxsw_sp->router->vrs); 1052 } 1053 1054 static struct net_device * 1055 __mlxsw_sp_ipip_netdev_ul_dev_get(const struct net_device *ol_dev) 1056 { 1057 struct ip_tunnel *tun = netdev_priv(ol_dev); 1058 struct net *net = dev_net(ol_dev); 1059 1060 return dev_get_by_index_rcu(net, tun->parms.link); 1061 } 1062 1063 u32 mlxsw_sp_ipip_dev_ul_tb_id(const struct net_device *ol_dev) 1064 { 1065 struct net_device *d; 1066 u32 tb_id; 1067 1068 rcu_read_lock(); 1069 d = __mlxsw_sp_ipip_netdev_ul_dev_get(ol_dev); 1070 if (d) 1071 tb_id = l3mdev_fib_table(d) ? : RT_TABLE_MAIN; 1072 else 1073 tb_id = RT_TABLE_MAIN; 1074 rcu_read_unlock(); 1075 1076 return tb_id; 1077 } 1078 1079 static struct mlxsw_sp_rif * 1080 mlxsw_sp_rif_create(struct mlxsw_sp *mlxsw_sp, 1081 const struct mlxsw_sp_rif_params *params, 1082 struct netlink_ext_ack *extack); 1083 1084 static struct mlxsw_sp_rif_ipip_lb * 1085 mlxsw_sp_ipip_ol_ipip_lb_create(struct mlxsw_sp *mlxsw_sp, 1086 enum mlxsw_sp_ipip_type ipipt, 1087 struct net_device *ol_dev, 1088 struct netlink_ext_ack *extack) 1089 { 1090 struct mlxsw_sp_rif_params_ipip_lb lb_params; 1091 const struct mlxsw_sp_ipip_ops *ipip_ops; 1092 struct mlxsw_sp_rif *rif; 1093 1094 ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipipt]; 1095 lb_params = (struct mlxsw_sp_rif_params_ipip_lb) { 1096 .common.dev = ol_dev, 1097 .common.lag = false, 1098 .lb_config = ipip_ops->ol_loopback_config(mlxsw_sp, ol_dev), 1099 }; 1100 1101 rif = mlxsw_sp_rif_create(mlxsw_sp, &lb_params.common, extack); 1102 if (IS_ERR(rif)) 1103 return ERR_CAST(rif); 1104 return container_of(rif, struct mlxsw_sp_rif_ipip_lb, common); 1105 } 1106 1107 static struct mlxsw_sp_ipip_entry * 1108 mlxsw_sp_ipip_entry_alloc(struct mlxsw_sp *mlxsw_sp, 1109 enum mlxsw_sp_ipip_type ipipt, 1110 struct net_device *ol_dev) 1111 { 1112 const struct mlxsw_sp_ipip_ops *ipip_ops; 1113 struct mlxsw_sp_ipip_entry *ipip_entry; 1114 struct mlxsw_sp_ipip_entry *ret = NULL; 1115 1116 ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipipt]; 1117 ipip_entry = kzalloc(sizeof(*ipip_entry), GFP_KERNEL); 1118 if (!ipip_entry) 1119 return ERR_PTR(-ENOMEM); 1120 1121 ipip_entry->ol_lb = mlxsw_sp_ipip_ol_ipip_lb_create(mlxsw_sp, ipipt, 1122 ol_dev, NULL); 1123 if (IS_ERR(ipip_entry->ol_lb)) { 1124 ret = ERR_CAST(ipip_entry->ol_lb); 1125 goto err_ol_ipip_lb_create; 1126 } 1127 1128 ipip_entry->ipipt = ipipt; 1129 ipip_entry->ol_dev = ol_dev; 1130 1131 switch (ipip_ops->ul_proto) { 1132 case MLXSW_SP_L3_PROTO_IPV4: 1133 ipip_entry->parms4 = mlxsw_sp_ipip_netdev_parms4(ol_dev); 1134 break; 1135 case MLXSW_SP_L3_PROTO_IPV6: 1136 WARN_ON(1); 1137 break; 1138 } 1139 1140 return ipip_entry; 1141 1142 err_ol_ipip_lb_create: 1143 kfree(ipip_entry); 1144 return ret; 1145 } 1146 1147 static void 1148 mlxsw_sp_ipip_entry_dealloc(struct mlxsw_sp_ipip_entry *ipip_entry) 1149 { 1150 mlxsw_sp_rif_destroy(&ipip_entry->ol_lb->common); 1151 kfree(ipip_entry); 1152 } 1153 1154 static bool 1155 mlxsw_sp_ipip_entry_saddr_matches(struct mlxsw_sp *mlxsw_sp, 1156 const enum mlxsw_sp_l3proto ul_proto, 1157 union mlxsw_sp_l3addr saddr, 1158 u32 ul_tb_id, 1159 struct mlxsw_sp_ipip_entry *ipip_entry) 1160 { 1161 u32 tun_ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(ipip_entry->ol_dev); 1162 enum mlxsw_sp_ipip_type ipipt = ipip_entry->ipipt; 1163 union mlxsw_sp_l3addr tun_saddr; 1164 1165 if (mlxsw_sp->router->ipip_ops_arr[ipipt]->ul_proto != ul_proto) 1166 return false; 1167 1168 tun_saddr = mlxsw_sp_ipip_netdev_saddr(ul_proto, ipip_entry->ol_dev); 1169 return tun_ul_tb_id == ul_tb_id && 1170 mlxsw_sp_l3addr_eq(&tun_saddr, &saddr); 1171 } 1172 1173 static int 1174 mlxsw_sp_fib_entry_decap_init(struct mlxsw_sp *mlxsw_sp, 1175 struct mlxsw_sp_fib_entry *fib_entry, 1176 struct mlxsw_sp_ipip_entry *ipip_entry) 1177 { 1178 u32 tunnel_index; 1179 int err; 1180 1181 err = mlxsw_sp_kvdl_alloc(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 1182 1, &tunnel_index); 1183 if (err) 1184 return err; 1185 1186 ipip_entry->decap_fib_entry = fib_entry; 1187 fib_entry->decap.ipip_entry = ipip_entry; 1188 fib_entry->decap.tunnel_index = tunnel_index; 1189 return 0; 1190 } 1191 1192 static void mlxsw_sp_fib_entry_decap_fini(struct mlxsw_sp *mlxsw_sp, 1193 struct mlxsw_sp_fib_entry *fib_entry) 1194 { 1195 /* Unlink this node from the IPIP entry that it's the decap entry of. */ 1196 fib_entry->decap.ipip_entry->decap_fib_entry = NULL; 1197 fib_entry->decap.ipip_entry = NULL; 1198 mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 1199 1, fib_entry->decap.tunnel_index); 1200 } 1201 1202 static struct mlxsw_sp_fib_node * 1203 mlxsw_sp_fib_node_lookup(struct mlxsw_sp_fib *fib, const void *addr, 1204 size_t addr_len, unsigned char prefix_len); 1205 static int mlxsw_sp_fib_entry_update(struct mlxsw_sp *mlxsw_sp, 1206 struct mlxsw_sp_fib_entry *fib_entry); 1207 1208 static void 1209 mlxsw_sp_ipip_entry_demote_decap(struct mlxsw_sp *mlxsw_sp, 1210 struct mlxsw_sp_ipip_entry *ipip_entry) 1211 { 1212 struct mlxsw_sp_fib_entry *fib_entry = ipip_entry->decap_fib_entry; 1213 1214 mlxsw_sp_fib_entry_decap_fini(mlxsw_sp, fib_entry); 1215 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP; 1216 1217 mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry); 1218 } 1219 1220 static void 1221 mlxsw_sp_ipip_entry_promote_decap(struct mlxsw_sp *mlxsw_sp, 1222 struct mlxsw_sp_ipip_entry *ipip_entry, 1223 struct mlxsw_sp_fib_entry *decap_fib_entry) 1224 { 1225 if (mlxsw_sp_fib_entry_decap_init(mlxsw_sp, decap_fib_entry, 1226 ipip_entry)) 1227 return; 1228 decap_fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP; 1229 1230 if (mlxsw_sp_fib_entry_update(mlxsw_sp, decap_fib_entry)) 1231 mlxsw_sp_ipip_entry_demote_decap(mlxsw_sp, ipip_entry); 1232 } 1233 1234 static struct mlxsw_sp_fib_entry * 1235 mlxsw_sp_router_ip2me_fib_entry_find(struct mlxsw_sp *mlxsw_sp, u32 tb_id, 1236 enum mlxsw_sp_l3proto proto, 1237 const union mlxsw_sp_l3addr *addr, 1238 enum mlxsw_sp_fib_entry_type type) 1239 { 1240 struct mlxsw_sp_fib_node *fib_node; 1241 unsigned char addr_prefix_len; 1242 struct mlxsw_sp_fib *fib; 1243 struct mlxsw_sp_vr *vr; 1244 const void *addrp; 1245 size_t addr_len; 1246 u32 addr4; 1247 1248 vr = mlxsw_sp_vr_find(mlxsw_sp, tb_id); 1249 if (!vr) 1250 return NULL; 1251 fib = mlxsw_sp_vr_fib(vr, proto); 1252 1253 switch (proto) { 1254 case MLXSW_SP_L3_PROTO_IPV4: 1255 addr4 = be32_to_cpu(addr->addr4); 1256 addrp = &addr4; 1257 addr_len = 4; 1258 addr_prefix_len = 32; 1259 break; 1260 case MLXSW_SP_L3_PROTO_IPV6: 1261 default: 1262 WARN_ON(1); 1263 return NULL; 1264 } 1265 1266 fib_node = mlxsw_sp_fib_node_lookup(fib, addrp, addr_len, 1267 addr_prefix_len); 1268 if (!fib_node || fib_node->fib_entry->type != type) 1269 return NULL; 1270 1271 return fib_node->fib_entry; 1272 } 1273 1274 /* Given an IPIP entry, find the corresponding decap route. */ 1275 static struct mlxsw_sp_fib_entry * 1276 mlxsw_sp_ipip_entry_find_decap(struct mlxsw_sp *mlxsw_sp, 1277 struct mlxsw_sp_ipip_entry *ipip_entry) 1278 { 1279 static struct mlxsw_sp_fib_node *fib_node; 1280 const struct mlxsw_sp_ipip_ops *ipip_ops; 1281 unsigned char saddr_prefix_len; 1282 union mlxsw_sp_l3addr saddr; 1283 struct mlxsw_sp_fib *ul_fib; 1284 struct mlxsw_sp_vr *ul_vr; 1285 const void *saddrp; 1286 size_t saddr_len; 1287 u32 ul_tb_id; 1288 u32 saddr4; 1289 1290 ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt]; 1291 1292 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(ipip_entry->ol_dev); 1293 ul_vr = mlxsw_sp_vr_find(mlxsw_sp, ul_tb_id); 1294 if (!ul_vr) 1295 return NULL; 1296 1297 ul_fib = mlxsw_sp_vr_fib(ul_vr, ipip_ops->ul_proto); 1298 saddr = mlxsw_sp_ipip_netdev_saddr(ipip_ops->ul_proto, 1299 ipip_entry->ol_dev); 1300 1301 switch (ipip_ops->ul_proto) { 1302 case MLXSW_SP_L3_PROTO_IPV4: 1303 saddr4 = be32_to_cpu(saddr.addr4); 1304 saddrp = &saddr4; 1305 saddr_len = 4; 1306 saddr_prefix_len = 32; 1307 break; 1308 default: 1309 WARN_ON(1); 1310 return NULL; 1311 } 1312 1313 fib_node = mlxsw_sp_fib_node_lookup(ul_fib, saddrp, saddr_len, 1314 saddr_prefix_len); 1315 if (!fib_node || 1316 fib_node->fib_entry->type != MLXSW_SP_FIB_ENTRY_TYPE_TRAP) 1317 return NULL; 1318 1319 return fib_node->fib_entry; 1320 } 1321 1322 static struct mlxsw_sp_ipip_entry * 1323 mlxsw_sp_ipip_entry_create(struct mlxsw_sp *mlxsw_sp, 1324 enum mlxsw_sp_ipip_type ipipt, 1325 struct net_device *ol_dev) 1326 { 1327 struct mlxsw_sp_ipip_entry *ipip_entry; 1328 1329 ipip_entry = mlxsw_sp_ipip_entry_alloc(mlxsw_sp, ipipt, ol_dev); 1330 if (IS_ERR(ipip_entry)) 1331 return ipip_entry; 1332 1333 list_add_tail(&ipip_entry->ipip_list_node, 1334 &mlxsw_sp->router->ipip_list); 1335 1336 return ipip_entry; 1337 } 1338 1339 static void 1340 mlxsw_sp_ipip_entry_destroy(struct mlxsw_sp *mlxsw_sp, 1341 struct mlxsw_sp_ipip_entry *ipip_entry) 1342 { 1343 list_del(&ipip_entry->ipip_list_node); 1344 mlxsw_sp_ipip_entry_dealloc(ipip_entry); 1345 } 1346 1347 static bool 1348 mlxsw_sp_ipip_entry_matches_decap(struct mlxsw_sp *mlxsw_sp, 1349 const struct net_device *ul_dev, 1350 enum mlxsw_sp_l3proto ul_proto, 1351 union mlxsw_sp_l3addr ul_dip, 1352 struct mlxsw_sp_ipip_entry *ipip_entry) 1353 { 1354 u32 ul_tb_id = l3mdev_fib_table(ul_dev) ? : RT_TABLE_MAIN; 1355 enum mlxsw_sp_ipip_type ipipt = ipip_entry->ipipt; 1356 1357 if (mlxsw_sp->router->ipip_ops_arr[ipipt]->ul_proto != ul_proto) 1358 return false; 1359 1360 return mlxsw_sp_ipip_entry_saddr_matches(mlxsw_sp, ul_proto, ul_dip, 1361 ul_tb_id, ipip_entry); 1362 } 1363 1364 /* Given decap parameters, find the corresponding IPIP entry. */ 1365 static struct mlxsw_sp_ipip_entry * 1366 mlxsw_sp_ipip_entry_find_by_decap(struct mlxsw_sp *mlxsw_sp, int ul_dev_ifindex, 1367 enum mlxsw_sp_l3proto ul_proto, 1368 union mlxsw_sp_l3addr ul_dip) 1369 { 1370 struct mlxsw_sp_ipip_entry *ipip_entry = NULL; 1371 struct net_device *ul_dev; 1372 1373 rcu_read_lock(); 1374 1375 ul_dev = dev_get_by_index_rcu(mlxsw_sp_net(mlxsw_sp), ul_dev_ifindex); 1376 if (!ul_dev) 1377 goto out_unlock; 1378 1379 list_for_each_entry(ipip_entry, &mlxsw_sp->router->ipip_list, 1380 ipip_list_node) 1381 if (mlxsw_sp_ipip_entry_matches_decap(mlxsw_sp, ul_dev, 1382 ul_proto, ul_dip, 1383 ipip_entry)) 1384 goto out_unlock; 1385 1386 rcu_read_unlock(); 1387 1388 return NULL; 1389 1390 out_unlock: 1391 rcu_read_unlock(); 1392 return ipip_entry; 1393 } 1394 1395 static bool mlxsw_sp_netdev_ipip_type(const struct mlxsw_sp *mlxsw_sp, 1396 const struct net_device *dev, 1397 enum mlxsw_sp_ipip_type *p_type) 1398 { 1399 struct mlxsw_sp_router *router = mlxsw_sp->router; 1400 const struct mlxsw_sp_ipip_ops *ipip_ops; 1401 enum mlxsw_sp_ipip_type ipipt; 1402 1403 for (ipipt = 0; ipipt < MLXSW_SP_IPIP_TYPE_MAX; ++ipipt) { 1404 ipip_ops = router->ipip_ops_arr[ipipt]; 1405 if (dev->type == ipip_ops->dev_type) { 1406 if (p_type) 1407 *p_type = ipipt; 1408 return true; 1409 } 1410 } 1411 return false; 1412 } 1413 1414 bool mlxsw_sp_netdev_is_ipip_ol(const struct mlxsw_sp *mlxsw_sp, 1415 const struct net_device *dev) 1416 { 1417 return mlxsw_sp_netdev_ipip_type(mlxsw_sp, dev, NULL); 1418 } 1419 1420 static struct mlxsw_sp_ipip_entry * 1421 mlxsw_sp_ipip_entry_find_by_ol_dev(struct mlxsw_sp *mlxsw_sp, 1422 const struct net_device *ol_dev) 1423 { 1424 struct mlxsw_sp_ipip_entry *ipip_entry; 1425 1426 list_for_each_entry(ipip_entry, &mlxsw_sp->router->ipip_list, 1427 ipip_list_node) 1428 if (ipip_entry->ol_dev == ol_dev) 1429 return ipip_entry; 1430 1431 return NULL; 1432 } 1433 1434 static struct mlxsw_sp_ipip_entry * 1435 mlxsw_sp_ipip_entry_find_by_ul_dev(const struct mlxsw_sp *mlxsw_sp, 1436 const struct net_device *ul_dev, 1437 struct mlxsw_sp_ipip_entry *start) 1438 { 1439 struct mlxsw_sp_ipip_entry *ipip_entry; 1440 1441 ipip_entry = list_prepare_entry(start, &mlxsw_sp->router->ipip_list, 1442 ipip_list_node); 1443 list_for_each_entry_continue(ipip_entry, &mlxsw_sp->router->ipip_list, 1444 ipip_list_node) { 1445 struct net_device *ol_dev = ipip_entry->ol_dev; 1446 struct net_device *ipip_ul_dev; 1447 1448 rcu_read_lock(); 1449 ipip_ul_dev = __mlxsw_sp_ipip_netdev_ul_dev_get(ol_dev); 1450 rcu_read_unlock(); 1451 1452 if (ipip_ul_dev == ul_dev) 1453 return ipip_entry; 1454 } 1455 1456 return NULL; 1457 } 1458 1459 bool mlxsw_sp_netdev_is_ipip_ul(struct mlxsw_sp *mlxsw_sp, 1460 const struct net_device *dev) 1461 { 1462 bool is_ipip_ul; 1463 1464 mutex_lock(&mlxsw_sp->router->lock); 1465 is_ipip_ul = mlxsw_sp_ipip_entry_find_by_ul_dev(mlxsw_sp, dev, NULL); 1466 mutex_unlock(&mlxsw_sp->router->lock); 1467 1468 return is_ipip_ul; 1469 } 1470 1471 static bool mlxsw_sp_netdevice_ipip_can_offload(struct mlxsw_sp *mlxsw_sp, 1472 const struct net_device *ol_dev, 1473 enum mlxsw_sp_ipip_type ipipt) 1474 { 1475 const struct mlxsw_sp_ipip_ops *ops 1476 = mlxsw_sp->router->ipip_ops_arr[ipipt]; 1477 1478 return ops->can_offload(mlxsw_sp, ol_dev); 1479 } 1480 1481 static int mlxsw_sp_netdevice_ipip_ol_reg_event(struct mlxsw_sp *mlxsw_sp, 1482 struct net_device *ol_dev) 1483 { 1484 enum mlxsw_sp_ipip_type ipipt = MLXSW_SP_IPIP_TYPE_MAX; 1485 struct mlxsw_sp_ipip_entry *ipip_entry; 1486 enum mlxsw_sp_l3proto ul_proto; 1487 union mlxsw_sp_l3addr saddr; 1488 u32 ul_tb_id; 1489 1490 mlxsw_sp_netdev_ipip_type(mlxsw_sp, ol_dev, &ipipt); 1491 if (mlxsw_sp_netdevice_ipip_can_offload(mlxsw_sp, ol_dev, ipipt)) { 1492 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(ol_dev); 1493 ul_proto = mlxsw_sp->router->ipip_ops_arr[ipipt]->ul_proto; 1494 saddr = mlxsw_sp_ipip_netdev_saddr(ul_proto, ol_dev); 1495 if (!mlxsw_sp_ipip_demote_tunnel_by_saddr(mlxsw_sp, ul_proto, 1496 saddr, ul_tb_id, 1497 NULL)) { 1498 ipip_entry = mlxsw_sp_ipip_entry_create(mlxsw_sp, ipipt, 1499 ol_dev); 1500 if (IS_ERR(ipip_entry)) 1501 return PTR_ERR(ipip_entry); 1502 } 1503 } 1504 1505 return 0; 1506 } 1507 1508 static void mlxsw_sp_netdevice_ipip_ol_unreg_event(struct mlxsw_sp *mlxsw_sp, 1509 struct net_device *ol_dev) 1510 { 1511 struct mlxsw_sp_ipip_entry *ipip_entry; 1512 1513 ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev); 1514 if (ipip_entry) 1515 mlxsw_sp_ipip_entry_destroy(mlxsw_sp, ipip_entry); 1516 } 1517 1518 static void 1519 mlxsw_sp_ipip_entry_ol_up_event(struct mlxsw_sp *mlxsw_sp, 1520 struct mlxsw_sp_ipip_entry *ipip_entry) 1521 { 1522 struct mlxsw_sp_fib_entry *decap_fib_entry; 1523 1524 decap_fib_entry = mlxsw_sp_ipip_entry_find_decap(mlxsw_sp, ipip_entry); 1525 if (decap_fib_entry) 1526 mlxsw_sp_ipip_entry_promote_decap(mlxsw_sp, ipip_entry, 1527 decap_fib_entry); 1528 } 1529 1530 static int 1531 mlxsw_sp_rif_ipip_lb_op(struct mlxsw_sp_rif_ipip_lb *lb_rif, u16 ul_vr_id, 1532 u16 ul_rif_id, bool enable) 1533 { 1534 struct mlxsw_sp_rif_ipip_lb_config lb_cf = lb_rif->lb_config; 1535 struct mlxsw_sp_rif *rif = &lb_rif->common; 1536 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 1537 char ritr_pl[MLXSW_REG_RITR_LEN]; 1538 u32 saddr4; 1539 1540 switch (lb_cf.ul_protocol) { 1541 case MLXSW_SP_L3_PROTO_IPV4: 1542 saddr4 = be32_to_cpu(lb_cf.saddr.addr4); 1543 mlxsw_reg_ritr_pack(ritr_pl, enable, MLXSW_REG_RITR_LOOPBACK_IF, 1544 rif->rif_index, rif->vr_id, rif->dev->mtu); 1545 mlxsw_reg_ritr_loopback_ipip4_pack(ritr_pl, lb_cf.lb_ipipt, 1546 MLXSW_REG_RITR_LOOPBACK_IPIP_OPTIONS_GRE_KEY_PRESET, 1547 ul_vr_id, ul_rif_id, saddr4, lb_cf.okey); 1548 break; 1549 1550 case MLXSW_SP_L3_PROTO_IPV6: 1551 return -EAFNOSUPPORT; 1552 } 1553 1554 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 1555 } 1556 1557 static int mlxsw_sp_netdevice_ipip_ol_update_mtu(struct mlxsw_sp *mlxsw_sp, 1558 struct net_device *ol_dev) 1559 { 1560 struct mlxsw_sp_ipip_entry *ipip_entry; 1561 struct mlxsw_sp_rif_ipip_lb *lb_rif; 1562 int err = 0; 1563 1564 ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev); 1565 if (ipip_entry) { 1566 lb_rif = ipip_entry->ol_lb; 1567 err = mlxsw_sp_rif_ipip_lb_op(lb_rif, lb_rif->ul_vr_id, 1568 lb_rif->ul_rif_id, true); 1569 if (err) 1570 goto out; 1571 lb_rif->common.mtu = ol_dev->mtu; 1572 } 1573 1574 out: 1575 return err; 1576 } 1577 1578 static void mlxsw_sp_netdevice_ipip_ol_up_event(struct mlxsw_sp *mlxsw_sp, 1579 struct net_device *ol_dev) 1580 { 1581 struct mlxsw_sp_ipip_entry *ipip_entry; 1582 1583 ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev); 1584 if (ipip_entry) 1585 mlxsw_sp_ipip_entry_ol_up_event(mlxsw_sp, ipip_entry); 1586 } 1587 1588 static void 1589 mlxsw_sp_ipip_entry_ol_down_event(struct mlxsw_sp *mlxsw_sp, 1590 struct mlxsw_sp_ipip_entry *ipip_entry) 1591 { 1592 if (ipip_entry->decap_fib_entry) 1593 mlxsw_sp_ipip_entry_demote_decap(mlxsw_sp, ipip_entry); 1594 } 1595 1596 static void mlxsw_sp_netdevice_ipip_ol_down_event(struct mlxsw_sp *mlxsw_sp, 1597 struct net_device *ol_dev) 1598 { 1599 struct mlxsw_sp_ipip_entry *ipip_entry; 1600 1601 ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev); 1602 if (ipip_entry) 1603 mlxsw_sp_ipip_entry_ol_down_event(mlxsw_sp, ipip_entry); 1604 } 1605 1606 static void mlxsw_sp_nexthop_rif_migrate(struct mlxsw_sp *mlxsw_sp, 1607 struct mlxsw_sp_rif *old_rif, 1608 struct mlxsw_sp_rif *new_rif); 1609 static int 1610 mlxsw_sp_ipip_entry_ol_lb_update(struct mlxsw_sp *mlxsw_sp, 1611 struct mlxsw_sp_ipip_entry *ipip_entry, 1612 bool keep_encap, 1613 struct netlink_ext_ack *extack) 1614 { 1615 struct mlxsw_sp_rif_ipip_lb *old_lb_rif = ipip_entry->ol_lb; 1616 struct mlxsw_sp_rif_ipip_lb *new_lb_rif; 1617 1618 new_lb_rif = mlxsw_sp_ipip_ol_ipip_lb_create(mlxsw_sp, 1619 ipip_entry->ipipt, 1620 ipip_entry->ol_dev, 1621 extack); 1622 if (IS_ERR(new_lb_rif)) 1623 return PTR_ERR(new_lb_rif); 1624 ipip_entry->ol_lb = new_lb_rif; 1625 1626 if (keep_encap) 1627 mlxsw_sp_nexthop_rif_migrate(mlxsw_sp, &old_lb_rif->common, 1628 &new_lb_rif->common); 1629 1630 mlxsw_sp_rif_destroy(&old_lb_rif->common); 1631 1632 return 0; 1633 } 1634 1635 static void mlxsw_sp_nexthop_rif_update(struct mlxsw_sp *mlxsw_sp, 1636 struct mlxsw_sp_rif *rif); 1637 1638 /** 1639 * __mlxsw_sp_ipip_entry_update_tunnel - Update offload related to IPIP entry. 1640 * @mlxsw_sp: mlxsw_sp. 1641 * @ipip_entry: IPIP entry. 1642 * @recreate_loopback: Recreates the associated loopback RIF. 1643 * @keep_encap: Updates next hops that use the tunnel netdevice. This is only 1644 * relevant when recreate_loopback is true. 1645 * @update_nexthops: Updates next hops, keeping the current loopback RIF. This 1646 * is only relevant when recreate_loopback is false. 1647 * @extack: extack. 1648 * 1649 * Return: Non-zero value on failure. 1650 */ 1651 int __mlxsw_sp_ipip_entry_update_tunnel(struct mlxsw_sp *mlxsw_sp, 1652 struct mlxsw_sp_ipip_entry *ipip_entry, 1653 bool recreate_loopback, 1654 bool keep_encap, 1655 bool update_nexthops, 1656 struct netlink_ext_ack *extack) 1657 { 1658 int err; 1659 1660 /* RIFs can't be edited, so to update loopback, we need to destroy and 1661 * recreate it. That creates a window of opportunity where RALUE and 1662 * RATR registers end up referencing a RIF that's already gone. RATRs 1663 * are handled in mlxsw_sp_ipip_entry_ol_lb_update(), and to take care 1664 * of RALUE, demote the decap route back. 1665 */ 1666 if (ipip_entry->decap_fib_entry) 1667 mlxsw_sp_ipip_entry_demote_decap(mlxsw_sp, ipip_entry); 1668 1669 if (recreate_loopback) { 1670 err = mlxsw_sp_ipip_entry_ol_lb_update(mlxsw_sp, ipip_entry, 1671 keep_encap, extack); 1672 if (err) 1673 return err; 1674 } else if (update_nexthops) { 1675 mlxsw_sp_nexthop_rif_update(mlxsw_sp, 1676 &ipip_entry->ol_lb->common); 1677 } 1678 1679 if (ipip_entry->ol_dev->flags & IFF_UP) 1680 mlxsw_sp_ipip_entry_ol_up_event(mlxsw_sp, ipip_entry); 1681 1682 return 0; 1683 } 1684 1685 static int mlxsw_sp_netdevice_ipip_ol_vrf_event(struct mlxsw_sp *mlxsw_sp, 1686 struct net_device *ol_dev, 1687 struct netlink_ext_ack *extack) 1688 { 1689 struct mlxsw_sp_ipip_entry *ipip_entry = 1690 mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev); 1691 1692 if (!ipip_entry) 1693 return 0; 1694 1695 return __mlxsw_sp_ipip_entry_update_tunnel(mlxsw_sp, ipip_entry, 1696 true, false, false, extack); 1697 } 1698 1699 static int 1700 mlxsw_sp_netdevice_ipip_ul_vrf_event(struct mlxsw_sp *mlxsw_sp, 1701 struct mlxsw_sp_ipip_entry *ipip_entry, 1702 struct net_device *ul_dev, 1703 bool *demote_this, 1704 struct netlink_ext_ack *extack) 1705 { 1706 u32 ul_tb_id = l3mdev_fib_table(ul_dev) ? : RT_TABLE_MAIN; 1707 enum mlxsw_sp_l3proto ul_proto; 1708 union mlxsw_sp_l3addr saddr; 1709 1710 /* Moving underlay to a different VRF might cause local address 1711 * conflict, and the conflicting tunnels need to be demoted. 1712 */ 1713 ul_proto = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt]->ul_proto; 1714 saddr = mlxsw_sp_ipip_netdev_saddr(ul_proto, ipip_entry->ol_dev); 1715 if (mlxsw_sp_ipip_demote_tunnel_by_saddr(mlxsw_sp, ul_proto, 1716 saddr, ul_tb_id, 1717 ipip_entry)) { 1718 *demote_this = true; 1719 return 0; 1720 } 1721 1722 return __mlxsw_sp_ipip_entry_update_tunnel(mlxsw_sp, ipip_entry, 1723 true, true, false, extack); 1724 } 1725 1726 static int 1727 mlxsw_sp_netdevice_ipip_ul_up_event(struct mlxsw_sp *mlxsw_sp, 1728 struct mlxsw_sp_ipip_entry *ipip_entry, 1729 struct net_device *ul_dev) 1730 { 1731 return __mlxsw_sp_ipip_entry_update_tunnel(mlxsw_sp, ipip_entry, 1732 false, false, true, NULL); 1733 } 1734 1735 static int 1736 mlxsw_sp_netdevice_ipip_ul_down_event(struct mlxsw_sp *mlxsw_sp, 1737 struct mlxsw_sp_ipip_entry *ipip_entry, 1738 struct net_device *ul_dev) 1739 { 1740 /* A down underlay device causes encapsulated packets to not be 1741 * forwarded, but decap still works. So refresh next hops without 1742 * touching anything else. 1743 */ 1744 return __mlxsw_sp_ipip_entry_update_tunnel(mlxsw_sp, ipip_entry, 1745 false, false, true, NULL); 1746 } 1747 1748 static int 1749 mlxsw_sp_netdevice_ipip_ol_change_event(struct mlxsw_sp *mlxsw_sp, 1750 struct net_device *ol_dev, 1751 struct netlink_ext_ack *extack) 1752 { 1753 const struct mlxsw_sp_ipip_ops *ipip_ops; 1754 struct mlxsw_sp_ipip_entry *ipip_entry; 1755 int err; 1756 1757 ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev); 1758 if (!ipip_entry) 1759 /* A change might make a tunnel eligible for offloading, but 1760 * that is currently not implemented. What falls to slow path 1761 * stays there. 1762 */ 1763 return 0; 1764 1765 /* A change might make a tunnel not eligible for offloading. */ 1766 if (!mlxsw_sp_netdevice_ipip_can_offload(mlxsw_sp, ol_dev, 1767 ipip_entry->ipipt)) { 1768 mlxsw_sp_ipip_entry_demote_tunnel(mlxsw_sp, ipip_entry); 1769 return 0; 1770 } 1771 1772 ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt]; 1773 err = ipip_ops->ol_netdev_change(mlxsw_sp, ipip_entry, extack); 1774 return err; 1775 } 1776 1777 void mlxsw_sp_ipip_entry_demote_tunnel(struct mlxsw_sp *mlxsw_sp, 1778 struct mlxsw_sp_ipip_entry *ipip_entry) 1779 { 1780 struct net_device *ol_dev = ipip_entry->ol_dev; 1781 1782 if (ol_dev->flags & IFF_UP) 1783 mlxsw_sp_ipip_entry_ol_down_event(mlxsw_sp, ipip_entry); 1784 mlxsw_sp_ipip_entry_destroy(mlxsw_sp, ipip_entry); 1785 } 1786 1787 /* The configuration where several tunnels have the same local address in the 1788 * same underlay table needs special treatment in the HW. That is currently not 1789 * implemented in the driver. This function finds and demotes the first tunnel 1790 * with a given source address, except the one passed in in the argument 1791 * `except'. 1792 */ 1793 bool 1794 mlxsw_sp_ipip_demote_tunnel_by_saddr(struct mlxsw_sp *mlxsw_sp, 1795 enum mlxsw_sp_l3proto ul_proto, 1796 union mlxsw_sp_l3addr saddr, 1797 u32 ul_tb_id, 1798 const struct mlxsw_sp_ipip_entry *except) 1799 { 1800 struct mlxsw_sp_ipip_entry *ipip_entry, *tmp; 1801 1802 list_for_each_entry_safe(ipip_entry, tmp, &mlxsw_sp->router->ipip_list, 1803 ipip_list_node) { 1804 if (ipip_entry != except && 1805 mlxsw_sp_ipip_entry_saddr_matches(mlxsw_sp, ul_proto, saddr, 1806 ul_tb_id, ipip_entry)) { 1807 mlxsw_sp_ipip_entry_demote_tunnel(mlxsw_sp, ipip_entry); 1808 return true; 1809 } 1810 } 1811 1812 return false; 1813 } 1814 1815 static void mlxsw_sp_ipip_demote_tunnel_by_ul_netdev(struct mlxsw_sp *mlxsw_sp, 1816 struct net_device *ul_dev) 1817 { 1818 struct mlxsw_sp_ipip_entry *ipip_entry, *tmp; 1819 1820 list_for_each_entry_safe(ipip_entry, tmp, &mlxsw_sp->router->ipip_list, 1821 ipip_list_node) { 1822 struct net_device *ol_dev = ipip_entry->ol_dev; 1823 struct net_device *ipip_ul_dev; 1824 1825 rcu_read_lock(); 1826 ipip_ul_dev = __mlxsw_sp_ipip_netdev_ul_dev_get(ol_dev); 1827 rcu_read_unlock(); 1828 if (ipip_ul_dev == ul_dev) 1829 mlxsw_sp_ipip_entry_demote_tunnel(mlxsw_sp, ipip_entry); 1830 } 1831 } 1832 1833 int mlxsw_sp_netdevice_ipip_ol_event(struct mlxsw_sp *mlxsw_sp, 1834 struct net_device *ol_dev, 1835 unsigned long event, 1836 struct netdev_notifier_info *info) 1837 { 1838 struct netdev_notifier_changeupper_info *chup; 1839 struct netlink_ext_ack *extack; 1840 int err = 0; 1841 1842 mutex_lock(&mlxsw_sp->router->lock); 1843 switch (event) { 1844 case NETDEV_REGISTER: 1845 err = mlxsw_sp_netdevice_ipip_ol_reg_event(mlxsw_sp, ol_dev); 1846 break; 1847 case NETDEV_UNREGISTER: 1848 mlxsw_sp_netdevice_ipip_ol_unreg_event(mlxsw_sp, ol_dev); 1849 break; 1850 case NETDEV_UP: 1851 mlxsw_sp_netdevice_ipip_ol_up_event(mlxsw_sp, ol_dev); 1852 break; 1853 case NETDEV_DOWN: 1854 mlxsw_sp_netdevice_ipip_ol_down_event(mlxsw_sp, ol_dev); 1855 break; 1856 case NETDEV_CHANGEUPPER: 1857 chup = container_of(info, typeof(*chup), info); 1858 extack = info->extack; 1859 if (netif_is_l3_master(chup->upper_dev)) 1860 err = mlxsw_sp_netdevice_ipip_ol_vrf_event(mlxsw_sp, 1861 ol_dev, 1862 extack); 1863 break; 1864 case NETDEV_CHANGE: 1865 extack = info->extack; 1866 err = mlxsw_sp_netdevice_ipip_ol_change_event(mlxsw_sp, 1867 ol_dev, extack); 1868 break; 1869 case NETDEV_CHANGEMTU: 1870 err = mlxsw_sp_netdevice_ipip_ol_update_mtu(mlxsw_sp, ol_dev); 1871 break; 1872 } 1873 mutex_unlock(&mlxsw_sp->router->lock); 1874 return err; 1875 } 1876 1877 static int 1878 __mlxsw_sp_netdevice_ipip_ul_event(struct mlxsw_sp *mlxsw_sp, 1879 struct mlxsw_sp_ipip_entry *ipip_entry, 1880 struct net_device *ul_dev, 1881 bool *demote_this, 1882 unsigned long event, 1883 struct netdev_notifier_info *info) 1884 { 1885 struct netdev_notifier_changeupper_info *chup; 1886 struct netlink_ext_ack *extack; 1887 1888 switch (event) { 1889 case NETDEV_CHANGEUPPER: 1890 chup = container_of(info, typeof(*chup), info); 1891 extack = info->extack; 1892 if (netif_is_l3_master(chup->upper_dev)) 1893 return mlxsw_sp_netdevice_ipip_ul_vrf_event(mlxsw_sp, 1894 ipip_entry, 1895 ul_dev, 1896 demote_this, 1897 extack); 1898 break; 1899 1900 case NETDEV_UP: 1901 return mlxsw_sp_netdevice_ipip_ul_up_event(mlxsw_sp, ipip_entry, 1902 ul_dev); 1903 case NETDEV_DOWN: 1904 return mlxsw_sp_netdevice_ipip_ul_down_event(mlxsw_sp, 1905 ipip_entry, 1906 ul_dev); 1907 } 1908 return 0; 1909 } 1910 1911 int 1912 mlxsw_sp_netdevice_ipip_ul_event(struct mlxsw_sp *mlxsw_sp, 1913 struct net_device *ul_dev, 1914 unsigned long event, 1915 struct netdev_notifier_info *info) 1916 { 1917 struct mlxsw_sp_ipip_entry *ipip_entry = NULL; 1918 int err = 0; 1919 1920 mutex_lock(&mlxsw_sp->router->lock); 1921 while ((ipip_entry = mlxsw_sp_ipip_entry_find_by_ul_dev(mlxsw_sp, 1922 ul_dev, 1923 ipip_entry))) { 1924 struct mlxsw_sp_ipip_entry *prev; 1925 bool demote_this = false; 1926 1927 err = __mlxsw_sp_netdevice_ipip_ul_event(mlxsw_sp, ipip_entry, 1928 ul_dev, &demote_this, 1929 event, info); 1930 if (err) { 1931 mlxsw_sp_ipip_demote_tunnel_by_ul_netdev(mlxsw_sp, 1932 ul_dev); 1933 break; 1934 } 1935 1936 if (demote_this) { 1937 if (list_is_first(&ipip_entry->ipip_list_node, 1938 &mlxsw_sp->router->ipip_list)) 1939 prev = NULL; 1940 else 1941 /* This can't be cached from previous iteration, 1942 * because that entry could be gone now. 1943 */ 1944 prev = list_prev_entry(ipip_entry, 1945 ipip_list_node); 1946 mlxsw_sp_ipip_entry_demote_tunnel(mlxsw_sp, ipip_entry); 1947 ipip_entry = prev; 1948 } 1949 } 1950 mutex_unlock(&mlxsw_sp->router->lock); 1951 1952 return err; 1953 } 1954 1955 int mlxsw_sp_router_nve_promote_decap(struct mlxsw_sp *mlxsw_sp, u32 ul_tb_id, 1956 enum mlxsw_sp_l3proto ul_proto, 1957 const union mlxsw_sp_l3addr *ul_sip, 1958 u32 tunnel_index) 1959 { 1960 enum mlxsw_sp_fib_entry_type type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP; 1961 struct mlxsw_sp_router *router = mlxsw_sp->router; 1962 struct mlxsw_sp_fib_entry *fib_entry; 1963 int err = 0; 1964 1965 mutex_lock(&mlxsw_sp->router->lock); 1966 1967 if (WARN_ON_ONCE(router->nve_decap_config.valid)) { 1968 err = -EINVAL; 1969 goto out; 1970 } 1971 1972 router->nve_decap_config.ul_tb_id = ul_tb_id; 1973 router->nve_decap_config.tunnel_index = tunnel_index; 1974 router->nve_decap_config.ul_proto = ul_proto; 1975 router->nve_decap_config.ul_sip = *ul_sip; 1976 router->nve_decap_config.valid = true; 1977 1978 /* It is valid to create a tunnel with a local IP and only later 1979 * assign this IP address to a local interface 1980 */ 1981 fib_entry = mlxsw_sp_router_ip2me_fib_entry_find(mlxsw_sp, ul_tb_id, 1982 ul_proto, ul_sip, 1983 type); 1984 if (!fib_entry) 1985 goto out; 1986 1987 fib_entry->decap.tunnel_index = tunnel_index; 1988 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP; 1989 1990 err = mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry); 1991 if (err) 1992 goto err_fib_entry_update; 1993 1994 goto out; 1995 1996 err_fib_entry_update: 1997 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP; 1998 mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry); 1999 out: 2000 mutex_unlock(&mlxsw_sp->router->lock); 2001 return err; 2002 } 2003 2004 void mlxsw_sp_router_nve_demote_decap(struct mlxsw_sp *mlxsw_sp, u32 ul_tb_id, 2005 enum mlxsw_sp_l3proto ul_proto, 2006 const union mlxsw_sp_l3addr *ul_sip) 2007 { 2008 enum mlxsw_sp_fib_entry_type type = MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP; 2009 struct mlxsw_sp_router *router = mlxsw_sp->router; 2010 struct mlxsw_sp_fib_entry *fib_entry; 2011 2012 mutex_lock(&mlxsw_sp->router->lock); 2013 2014 if (WARN_ON_ONCE(!router->nve_decap_config.valid)) 2015 goto out; 2016 2017 router->nve_decap_config.valid = false; 2018 2019 fib_entry = mlxsw_sp_router_ip2me_fib_entry_find(mlxsw_sp, ul_tb_id, 2020 ul_proto, ul_sip, 2021 type); 2022 if (!fib_entry) 2023 goto out; 2024 2025 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP; 2026 mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry); 2027 out: 2028 mutex_unlock(&mlxsw_sp->router->lock); 2029 } 2030 2031 static bool mlxsw_sp_router_nve_is_decap(struct mlxsw_sp *mlxsw_sp, 2032 u32 ul_tb_id, 2033 enum mlxsw_sp_l3proto ul_proto, 2034 const union mlxsw_sp_l3addr *ul_sip) 2035 { 2036 struct mlxsw_sp_router *router = mlxsw_sp->router; 2037 2038 return router->nve_decap_config.valid && 2039 router->nve_decap_config.ul_tb_id == ul_tb_id && 2040 router->nve_decap_config.ul_proto == ul_proto && 2041 !memcmp(&router->nve_decap_config.ul_sip, ul_sip, 2042 sizeof(*ul_sip)); 2043 } 2044 2045 struct mlxsw_sp_neigh_key { 2046 struct neighbour *n; 2047 }; 2048 2049 struct mlxsw_sp_neigh_entry { 2050 struct list_head rif_list_node; 2051 struct rhash_head ht_node; 2052 struct mlxsw_sp_neigh_key key; 2053 u16 rif; 2054 bool connected; 2055 unsigned char ha[ETH_ALEN]; 2056 struct list_head nexthop_list; /* list of nexthops using 2057 * this neigh entry 2058 */ 2059 struct list_head nexthop_neighs_list_node; 2060 unsigned int counter_index; 2061 bool counter_valid; 2062 }; 2063 2064 static const struct rhashtable_params mlxsw_sp_neigh_ht_params = { 2065 .key_offset = offsetof(struct mlxsw_sp_neigh_entry, key), 2066 .head_offset = offsetof(struct mlxsw_sp_neigh_entry, ht_node), 2067 .key_len = sizeof(struct mlxsw_sp_neigh_key), 2068 }; 2069 2070 struct mlxsw_sp_neigh_entry * 2071 mlxsw_sp_rif_neigh_next(struct mlxsw_sp_rif *rif, 2072 struct mlxsw_sp_neigh_entry *neigh_entry) 2073 { 2074 if (!neigh_entry) { 2075 if (list_empty(&rif->neigh_list)) 2076 return NULL; 2077 else 2078 return list_first_entry(&rif->neigh_list, 2079 typeof(*neigh_entry), 2080 rif_list_node); 2081 } 2082 if (list_is_last(&neigh_entry->rif_list_node, &rif->neigh_list)) 2083 return NULL; 2084 return list_next_entry(neigh_entry, rif_list_node); 2085 } 2086 2087 int mlxsw_sp_neigh_entry_type(struct mlxsw_sp_neigh_entry *neigh_entry) 2088 { 2089 return neigh_entry->key.n->tbl->family; 2090 } 2091 2092 unsigned char * 2093 mlxsw_sp_neigh_entry_ha(struct mlxsw_sp_neigh_entry *neigh_entry) 2094 { 2095 return neigh_entry->ha; 2096 } 2097 2098 u32 mlxsw_sp_neigh4_entry_dip(struct mlxsw_sp_neigh_entry *neigh_entry) 2099 { 2100 struct neighbour *n; 2101 2102 n = neigh_entry->key.n; 2103 return ntohl(*((__be32 *) n->primary_key)); 2104 } 2105 2106 struct in6_addr * 2107 mlxsw_sp_neigh6_entry_dip(struct mlxsw_sp_neigh_entry *neigh_entry) 2108 { 2109 struct neighbour *n; 2110 2111 n = neigh_entry->key.n; 2112 return (struct in6_addr *) &n->primary_key; 2113 } 2114 2115 int mlxsw_sp_neigh_counter_get(struct mlxsw_sp *mlxsw_sp, 2116 struct mlxsw_sp_neigh_entry *neigh_entry, 2117 u64 *p_counter) 2118 { 2119 if (!neigh_entry->counter_valid) 2120 return -EINVAL; 2121 2122 return mlxsw_sp_flow_counter_get(mlxsw_sp, neigh_entry->counter_index, 2123 p_counter, NULL); 2124 } 2125 2126 static struct mlxsw_sp_neigh_entry * 2127 mlxsw_sp_neigh_entry_alloc(struct mlxsw_sp *mlxsw_sp, struct neighbour *n, 2128 u16 rif) 2129 { 2130 struct mlxsw_sp_neigh_entry *neigh_entry; 2131 2132 neigh_entry = kzalloc(sizeof(*neigh_entry), GFP_KERNEL); 2133 if (!neigh_entry) 2134 return NULL; 2135 2136 neigh_entry->key.n = n; 2137 neigh_entry->rif = rif; 2138 INIT_LIST_HEAD(&neigh_entry->nexthop_list); 2139 2140 return neigh_entry; 2141 } 2142 2143 static void mlxsw_sp_neigh_entry_free(struct mlxsw_sp_neigh_entry *neigh_entry) 2144 { 2145 kfree(neigh_entry); 2146 } 2147 2148 static int 2149 mlxsw_sp_neigh_entry_insert(struct mlxsw_sp *mlxsw_sp, 2150 struct mlxsw_sp_neigh_entry *neigh_entry) 2151 { 2152 return rhashtable_insert_fast(&mlxsw_sp->router->neigh_ht, 2153 &neigh_entry->ht_node, 2154 mlxsw_sp_neigh_ht_params); 2155 } 2156 2157 static void 2158 mlxsw_sp_neigh_entry_remove(struct mlxsw_sp *mlxsw_sp, 2159 struct mlxsw_sp_neigh_entry *neigh_entry) 2160 { 2161 rhashtable_remove_fast(&mlxsw_sp->router->neigh_ht, 2162 &neigh_entry->ht_node, 2163 mlxsw_sp_neigh_ht_params); 2164 } 2165 2166 static bool 2167 mlxsw_sp_neigh_counter_should_alloc(struct mlxsw_sp *mlxsw_sp, 2168 struct mlxsw_sp_neigh_entry *neigh_entry) 2169 { 2170 struct devlink *devlink; 2171 const char *table_name; 2172 2173 switch (mlxsw_sp_neigh_entry_type(neigh_entry)) { 2174 case AF_INET: 2175 table_name = MLXSW_SP_DPIPE_TABLE_NAME_HOST4; 2176 break; 2177 case AF_INET6: 2178 table_name = MLXSW_SP_DPIPE_TABLE_NAME_HOST6; 2179 break; 2180 default: 2181 WARN_ON(1); 2182 return false; 2183 } 2184 2185 devlink = priv_to_devlink(mlxsw_sp->core); 2186 return devlink_dpipe_table_counter_enabled(devlink, table_name); 2187 } 2188 2189 static void 2190 mlxsw_sp_neigh_counter_alloc(struct mlxsw_sp *mlxsw_sp, 2191 struct mlxsw_sp_neigh_entry *neigh_entry) 2192 { 2193 if (!mlxsw_sp_neigh_counter_should_alloc(mlxsw_sp, neigh_entry)) 2194 return; 2195 2196 if (mlxsw_sp_flow_counter_alloc(mlxsw_sp, &neigh_entry->counter_index)) 2197 return; 2198 2199 neigh_entry->counter_valid = true; 2200 } 2201 2202 static void 2203 mlxsw_sp_neigh_counter_free(struct mlxsw_sp *mlxsw_sp, 2204 struct mlxsw_sp_neigh_entry *neigh_entry) 2205 { 2206 if (!neigh_entry->counter_valid) 2207 return; 2208 mlxsw_sp_flow_counter_free(mlxsw_sp, 2209 neigh_entry->counter_index); 2210 neigh_entry->counter_valid = false; 2211 } 2212 2213 static struct mlxsw_sp_neigh_entry * 2214 mlxsw_sp_neigh_entry_create(struct mlxsw_sp *mlxsw_sp, struct neighbour *n) 2215 { 2216 struct mlxsw_sp_neigh_entry *neigh_entry; 2217 struct mlxsw_sp_rif *rif; 2218 int err; 2219 2220 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, n->dev); 2221 if (!rif) 2222 return ERR_PTR(-EINVAL); 2223 2224 neigh_entry = mlxsw_sp_neigh_entry_alloc(mlxsw_sp, n, rif->rif_index); 2225 if (!neigh_entry) 2226 return ERR_PTR(-ENOMEM); 2227 2228 err = mlxsw_sp_neigh_entry_insert(mlxsw_sp, neigh_entry); 2229 if (err) 2230 goto err_neigh_entry_insert; 2231 2232 mlxsw_sp_neigh_counter_alloc(mlxsw_sp, neigh_entry); 2233 list_add(&neigh_entry->rif_list_node, &rif->neigh_list); 2234 2235 return neigh_entry; 2236 2237 err_neigh_entry_insert: 2238 mlxsw_sp_neigh_entry_free(neigh_entry); 2239 return ERR_PTR(err); 2240 } 2241 2242 static void 2243 mlxsw_sp_neigh_entry_destroy(struct mlxsw_sp *mlxsw_sp, 2244 struct mlxsw_sp_neigh_entry *neigh_entry) 2245 { 2246 list_del(&neigh_entry->rif_list_node); 2247 mlxsw_sp_neigh_counter_free(mlxsw_sp, neigh_entry); 2248 mlxsw_sp_neigh_entry_remove(mlxsw_sp, neigh_entry); 2249 mlxsw_sp_neigh_entry_free(neigh_entry); 2250 } 2251 2252 static struct mlxsw_sp_neigh_entry * 2253 mlxsw_sp_neigh_entry_lookup(struct mlxsw_sp *mlxsw_sp, struct neighbour *n) 2254 { 2255 struct mlxsw_sp_neigh_key key; 2256 2257 key.n = n; 2258 return rhashtable_lookup_fast(&mlxsw_sp->router->neigh_ht, 2259 &key, mlxsw_sp_neigh_ht_params); 2260 } 2261 2262 static void 2263 mlxsw_sp_router_neighs_update_interval_init(struct mlxsw_sp *mlxsw_sp) 2264 { 2265 unsigned long interval; 2266 2267 #if IS_ENABLED(CONFIG_IPV6) 2268 interval = min_t(unsigned long, 2269 NEIGH_VAR(&arp_tbl.parms, DELAY_PROBE_TIME), 2270 NEIGH_VAR(&nd_tbl.parms, DELAY_PROBE_TIME)); 2271 #else 2272 interval = NEIGH_VAR(&arp_tbl.parms, DELAY_PROBE_TIME); 2273 #endif 2274 mlxsw_sp->router->neighs_update.interval = jiffies_to_msecs(interval); 2275 } 2276 2277 static void mlxsw_sp_router_neigh_ent_ipv4_process(struct mlxsw_sp *mlxsw_sp, 2278 char *rauhtd_pl, 2279 int ent_index) 2280 { 2281 struct net_device *dev; 2282 struct neighbour *n; 2283 __be32 dipn; 2284 u32 dip; 2285 u16 rif; 2286 2287 mlxsw_reg_rauhtd_ent_ipv4_unpack(rauhtd_pl, ent_index, &rif, &dip); 2288 2289 if (!mlxsw_sp->router->rifs[rif]) { 2290 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Incorrect RIF in neighbour entry\n"); 2291 return; 2292 } 2293 2294 dipn = htonl(dip); 2295 dev = mlxsw_sp->router->rifs[rif]->dev; 2296 n = neigh_lookup(&arp_tbl, &dipn, dev); 2297 if (!n) 2298 return; 2299 2300 netdev_dbg(dev, "Updating neighbour with IP=%pI4h\n", &dip); 2301 neigh_event_send(n, NULL); 2302 neigh_release(n); 2303 } 2304 2305 #if IS_ENABLED(CONFIG_IPV6) 2306 static void mlxsw_sp_router_neigh_ent_ipv6_process(struct mlxsw_sp *mlxsw_sp, 2307 char *rauhtd_pl, 2308 int rec_index) 2309 { 2310 struct net_device *dev; 2311 struct neighbour *n; 2312 struct in6_addr dip; 2313 u16 rif; 2314 2315 mlxsw_reg_rauhtd_ent_ipv6_unpack(rauhtd_pl, rec_index, &rif, 2316 (char *) &dip); 2317 2318 if (!mlxsw_sp->router->rifs[rif]) { 2319 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Incorrect RIF in neighbour entry\n"); 2320 return; 2321 } 2322 2323 dev = mlxsw_sp->router->rifs[rif]->dev; 2324 n = neigh_lookup(&nd_tbl, &dip, dev); 2325 if (!n) 2326 return; 2327 2328 netdev_dbg(dev, "Updating neighbour with IP=%pI6c\n", &dip); 2329 neigh_event_send(n, NULL); 2330 neigh_release(n); 2331 } 2332 #else 2333 static void mlxsw_sp_router_neigh_ent_ipv6_process(struct mlxsw_sp *mlxsw_sp, 2334 char *rauhtd_pl, 2335 int rec_index) 2336 { 2337 } 2338 #endif 2339 2340 static void mlxsw_sp_router_neigh_rec_ipv4_process(struct mlxsw_sp *mlxsw_sp, 2341 char *rauhtd_pl, 2342 int rec_index) 2343 { 2344 u8 num_entries; 2345 int i; 2346 2347 num_entries = mlxsw_reg_rauhtd_ipv4_rec_num_entries_get(rauhtd_pl, 2348 rec_index); 2349 /* Hardware starts counting at 0, so add 1. */ 2350 num_entries++; 2351 2352 /* Each record consists of several neighbour entries. */ 2353 for (i = 0; i < num_entries; i++) { 2354 int ent_index; 2355 2356 ent_index = rec_index * MLXSW_REG_RAUHTD_IPV4_ENT_PER_REC + i; 2357 mlxsw_sp_router_neigh_ent_ipv4_process(mlxsw_sp, rauhtd_pl, 2358 ent_index); 2359 } 2360 2361 } 2362 2363 static void mlxsw_sp_router_neigh_rec_ipv6_process(struct mlxsw_sp *mlxsw_sp, 2364 char *rauhtd_pl, 2365 int rec_index) 2366 { 2367 /* One record contains one entry. */ 2368 mlxsw_sp_router_neigh_ent_ipv6_process(mlxsw_sp, rauhtd_pl, 2369 rec_index); 2370 } 2371 2372 static void mlxsw_sp_router_neigh_rec_process(struct mlxsw_sp *mlxsw_sp, 2373 char *rauhtd_pl, int rec_index) 2374 { 2375 switch (mlxsw_reg_rauhtd_rec_type_get(rauhtd_pl, rec_index)) { 2376 case MLXSW_REG_RAUHTD_TYPE_IPV4: 2377 mlxsw_sp_router_neigh_rec_ipv4_process(mlxsw_sp, rauhtd_pl, 2378 rec_index); 2379 break; 2380 case MLXSW_REG_RAUHTD_TYPE_IPV6: 2381 mlxsw_sp_router_neigh_rec_ipv6_process(mlxsw_sp, rauhtd_pl, 2382 rec_index); 2383 break; 2384 } 2385 } 2386 2387 static bool mlxsw_sp_router_rauhtd_is_full(char *rauhtd_pl) 2388 { 2389 u8 num_rec, last_rec_index, num_entries; 2390 2391 num_rec = mlxsw_reg_rauhtd_num_rec_get(rauhtd_pl); 2392 last_rec_index = num_rec - 1; 2393 2394 if (num_rec < MLXSW_REG_RAUHTD_REC_MAX_NUM) 2395 return false; 2396 if (mlxsw_reg_rauhtd_rec_type_get(rauhtd_pl, last_rec_index) == 2397 MLXSW_REG_RAUHTD_TYPE_IPV6) 2398 return true; 2399 2400 num_entries = mlxsw_reg_rauhtd_ipv4_rec_num_entries_get(rauhtd_pl, 2401 last_rec_index); 2402 if (++num_entries == MLXSW_REG_RAUHTD_IPV4_ENT_PER_REC) 2403 return true; 2404 return false; 2405 } 2406 2407 static int 2408 __mlxsw_sp_router_neighs_update_rauhtd(struct mlxsw_sp *mlxsw_sp, 2409 char *rauhtd_pl, 2410 enum mlxsw_reg_rauhtd_type type) 2411 { 2412 int i, num_rec; 2413 int err; 2414 2415 /* Ensure the RIF we read from the device does not change mid-dump. */ 2416 mutex_lock(&mlxsw_sp->router->lock); 2417 do { 2418 mlxsw_reg_rauhtd_pack(rauhtd_pl, type); 2419 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(rauhtd), 2420 rauhtd_pl); 2421 if (err) { 2422 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Failed to dump neighbour table\n"); 2423 break; 2424 } 2425 num_rec = mlxsw_reg_rauhtd_num_rec_get(rauhtd_pl); 2426 for (i = 0; i < num_rec; i++) 2427 mlxsw_sp_router_neigh_rec_process(mlxsw_sp, rauhtd_pl, 2428 i); 2429 } while (mlxsw_sp_router_rauhtd_is_full(rauhtd_pl)); 2430 mutex_unlock(&mlxsw_sp->router->lock); 2431 2432 return err; 2433 } 2434 2435 static int mlxsw_sp_router_neighs_update_rauhtd(struct mlxsw_sp *mlxsw_sp) 2436 { 2437 enum mlxsw_reg_rauhtd_type type; 2438 char *rauhtd_pl; 2439 int err; 2440 2441 rauhtd_pl = kmalloc(MLXSW_REG_RAUHTD_LEN, GFP_KERNEL); 2442 if (!rauhtd_pl) 2443 return -ENOMEM; 2444 2445 type = MLXSW_REG_RAUHTD_TYPE_IPV4; 2446 err = __mlxsw_sp_router_neighs_update_rauhtd(mlxsw_sp, rauhtd_pl, type); 2447 if (err) 2448 goto out; 2449 2450 type = MLXSW_REG_RAUHTD_TYPE_IPV6; 2451 err = __mlxsw_sp_router_neighs_update_rauhtd(mlxsw_sp, rauhtd_pl, type); 2452 out: 2453 kfree(rauhtd_pl); 2454 return err; 2455 } 2456 2457 static void mlxsw_sp_router_neighs_update_nh(struct mlxsw_sp *mlxsw_sp) 2458 { 2459 struct mlxsw_sp_neigh_entry *neigh_entry; 2460 2461 mutex_lock(&mlxsw_sp->router->lock); 2462 list_for_each_entry(neigh_entry, &mlxsw_sp->router->nexthop_neighs_list, 2463 nexthop_neighs_list_node) 2464 /* If this neigh have nexthops, make the kernel think this neigh 2465 * is active regardless of the traffic. 2466 */ 2467 neigh_event_send(neigh_entry->key.n, NULL); 2468 mutex_unlock(&mlxsw_sp->router->lock); 2469 } 2470 2471 static void 2472 mlxsw_sp_router_neighs_update_work_schedule(struct mlxsw_sp *mlxsw_sp) 2473 { 2474 unsigned long interval = mlxsw_sp->router->neighs_update.interval; 2475 2476 mlxsw_core_schedule_dw(&mlxsw_sp->router->neighs_update.dw, 2477 msecs_to_jiffies(interval)); 2478 } 2479 2480 static void mlxsw_sp_router_neighs_update_work(struct work_struct *work) 2481 { 2482 struct mlxsw_sp_router *router; 2483 int err; 2484 2485 router = container_of(work, struct mlxsw_sp_router, 2486 neighs_update.dw.work); 2487 err = mlxsw_sp_router_neighs_update_rauhtd(router->mlxsw_sp); 2488 if (err) 2489 dev_err(router->mlxsw_sp->bus_info->dev, "Could not update kernel for neigh activity"); 2490 2491 mlxsw_sp_router_neighs_update_nh(router->mlxsw_sp); 2492 2493 mlxsw_sp_router_neighs_update_work_schedule(router->mlxsw_sp); 2494 } 2495 2496 static void mlxsw_sp_router_probe_unresolved_nexthops(struct work_struct *work) 2497 { 2498 struct mlxsw_sp_neigh_entry *neigh_entry; 2499 struct mlxsw_sp_router *router; 2500 2501 router = container_of(work, struct mlxsw_sp_router, 2502 nexthop_probe_dw.work); 2503 /* Iterate over nexthop neighbours, find those who are unresolved and 2504 * send arp on them. This solves the chicken-egg problem when 2505 * the nexthop wouldn't get offloaded until the neighbor is resolved 2506 * but it wouldn't get resolved ever in case traffic is flowing in HW 2507 * using different nexthop. 2508 */ 2509 mutex_lock(&router->lock); 2510 list_for_each_entry(neigh_entry, &router->nexthop_neighs_list, 2511 nexthop_neighs_list_node) 2512 if (!neigh_entry->connected) 2513 neigh_event_send(neigh_entry->key.n, NULL); 2514 mutex_unlock(&router->lock); 2515 2516 mlxsw_core_schedule_dw(&router->nexthop_probe_dw, 2517 MLXSW_SP_UNRESOLVED_NH_PROBE_INTERVAL); 2518 } 2519 2520 static void 2521 mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp *mlxsw_sp, 2522 struct mlxsw_sp_neigh_entry *neigh_entry, 2523 bool removing, bool dead); 2524 2525 static enum mlxsw_reg_rauht_op mlxsw_sp_rauht_op(bool adding) 2526 { 2527 return adding ? MLXSW_REG_RAUHT_OP_WRITE_ADD : 2528 MLXSW_REG_RAUHT_OP_WRITE_DELETE; 2529 } 2530 2531 static int 2532 mlxsw_sp_router_neigh_entry_op4(struct mlxsw_sp *mlxsw_sp, 2533 struct mlxsw_sp_neigh_entry *neigh_entry, 2534 enum mlxsw_reg_rauht_op op) 2535 { 2536 struct neighbour *n = neigh_entry->key.n; 2537 u32 dip = ntohl(*((__be32 *) n->primary_key)); 2538 char rauht_pl[MLXSW_REG_RAUHT_LEN]; 2539 2540 mlxsw_reg_rauht_pack4(rauht_pl, op, neigh_entry->rif, neigh_entry->ha, 2541 dip); 2542 if (neigh_entry->counter_valid) 2543 mlxsw_reg_rauht_pack_counter(rauht_pl, 2544 neigh_entry->counter_index); 2545 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rauht), rauht_pl); 2546 } 2547 2548 static int 2549 mlxsw_sp_router_neigh_entry_op6(struct mlxsw_sp *mlxsw_sp, 2550 struct mlxsw_sp_neigh_entry *neigh_entry, 2551 enum mlxsw_reg_rauht_op op) 2552 { 2553 struct neighbour *n = neigh_entry->key.n; 2554 char rauht_pl[MLXSW_REG_RAUHT_LEN]; 2555 const char *dip = n->primary_key; 2556 2557 mlxsw_reg_rauht_pack6(rauht_pl, op, neigh_entry->rif, neigh_entry->ha, 2558 dip); 2559 if (neigh_entry->counter_valid) 2560 mlxsw_reg_rauht_pack_counter(rauht_pl, 2561 neigh_entry->counter_index); 2562 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rauht), rauht_pl); 2563 } 2564 2565 bool mlxsw_sp_neigh_ipv6_ignore(struct mlxsw_sp_neigh_entry *neigh_entry) 2566 { 2567 struct neighbour *n = neigh_entry->key.n; 2568 2569 /* Packets with a link-local destination address are trapped 2570 * after LPM lookup and never reach the neighbour table, so 2571 * there is no need to program such neighbours to the device. 2572 */ 2573 if (ipv6_addr_type((struct in6_addr *) &n->primary_key) & 2574 IPV6_ADDR_LINKLOCAL) 2575 return true; 2576 return false; 2577 } 2578 2579 static void 2580 mlxsw_sp_neigh_entry_update(struct mlxsw_sp *mlxsw_sp, 2581 struct mlxsw_sp_neigh_entry *neigh_entry, 2582 bool adding) 2583 { 2584 enum mlxsw_reg_rauht_op op = mlxsw_sp_rauht_op(adding); 2585 int err; 2586 2587 if (!adding && !neigh_entry->connected) 2588 return; 2589 neigh_entry->connected = adding; 2590 if (neigh_entry->key.n->tbl->family == AF_INET) { 2591 err = mlxsw_sp_router_neigh_entry_op4(mlxsw_sp, neigh_entry, 2592 op); 2593 if (err) 2594 return; 2595 } else if (neigh_entry->key.n->tbl->family == AF_INET6) { 2596 if (mlxsw_sp_neigh_ipv6_ignore(neigh_entry)) 2597 return; 2598 err = mlxsw_sp_router_neigh_entry_op6(mlxsw_sp, neigh_entry, 2599 op); 2600 if (err) 2601 return; 2602 } else { 2603 WARN_ON_ONCE(1); 2604 return; 2605 } 2606 2607 if (adding) 2608 neigh_entry->key.n->flags |= NTF_OFFLOADED; 2609 else 2610 neigh_entry->key.n->flags &= ~NTF_OFFLOADED; 2611 } 2612 2613 void 2614 mlxsw_sp_neigh_entry_counter_update(struct mlxsw_sp *mlxsw_sp, 2615 struct mlxsw_sp_neigh_entry *neigh_entry, 2616 bool adding) 2617 { 2618 if (adding) 2619 mlxsw_sp_neigh_counter_alloc(mlxsw_sp, neigh_entry); 2620 else 2621 mlxsw_sp_neigh_counter_free(mlxsw_sp, neigh_entry); 2622 mlxsw_sp_neigh_entry_update(mlxsw_sp, neigh_entry, true); 2623 } 2624 2625 struct mlxsw_sp_netevent_work { 2626 struct work_struct work; 2627 struct mlxsw_sp *mlxsw_sp; 2628 struct neighbour *n; 2629 }; 2630 2631 static void mlxsw_sp_router_neigh_event_work(struct work_struct *work) 2632 { 2633 struct mlxsw_sp_netevent_work *net_work = 2634 container_of(work, struct mlxsw_sp_netevent_work, work); 2635 struct mlxsw_sp *mlxsw_sp = net_work->mlxsw_sp; 2636 struct mlxsw_sp_neigh_entry *neigh_entry; 2637 struct neighbour *n = net_work->n; 2638 unsigned char ha[ETH_ALEN]; 2639 bool entry_connected; 2640 u8 nud_state, dead; 2641 2642 /* If these parameters are changed after we release the lock, 2643 * then we are guaranteed to receive another event letting us 2644 * know about it. 2645 */ 2646 read_lock_bh(&n->lock); 2647 memcpy(ha, n->ha, ETH_ALEN); 2648 nud_state = n->nud_state; 2649 dead = n->dead; 2650 read_unlock_bh(&n->lock); 2651 2652 mutex_lock(&mlxsw_sp->router->lock); 2653 mlxsw_sp_span_respin(mlxsw_sp); 2654 2655 entry_connected = nud_state & NUD_VALID && !dead; 2656 neigh_entry = mlxsw_sp_neigh_entry_lookup(mlxsw_sp, n); 2657 if (!entry_connected && !neigh_entry) 2658 goto out; 2659 if (!neigh_entry) { 2660 neigh_entry = mlxsw_sp_neigh_entry_create(mlxsw_sp, n); 2661 if (IS_ERR(neigh_entry)) 2662 goto out; 2663 } 2664 2665 memcpy(neigh_entry->ha, ha, ETH_ALEN); 2666 mlxsw_sp_neigh_entry_update(mlxsw_sp, neigh_entry, entry_connected); 2667 mlxsw_sp_nexthop_neigh_update(mlxsw_sp, neigh_entry, !entry_connected, 2668 dead); 2669 2670 if (!neigh_entry->connected && list_empty(&neigh_entry->nexthop_list)) 2671 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry); 2672 2673 out: 2674 mutex_unlock(&mlxsw_sp->router->lock); 2675 neigh_release(n); 2676 kfree(net_work); 2677 } 2678 2679 static int mlxsw_sp_mp_hash_init(struct mlxsw_sp *mlxsw_sp); 2680 2681 static void mlxsw_sp_router_mp_hash_event_work(struct work_struct *work) 2682 { 2683 struct mlxsw_sp_netevent_work *net_work = 2684 container_of(work, struct mlxsw_sp_netevent_work, work); 2685 struct mlxsw_sp *mlxsw_sp = net_work->mlxsw_sp; 2686 2687 mlxsw_sp_mp_hash_init(mlxsw_sp); 2688 kfree(net_work); 2689 } 2690 2691 static int __mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp); 2692 2693 static void mlxsw_sp_router_update_priority_work(struct work_struct *work) 2694 { 2695 struct mlxsw_sp_netevent_work *net_work = 2696 container_of(work, struct mlxsw_sp_netevent_work, work); 2697 struct mlxsw_sp *mlxsw_sp = net_work->mlxsw_sp; 2698 2699 __mlxsw_sp_router_init(mlxsw_sp); 2700 kfree(net_work); 2701 } 2702 2703 static int mlxsw_sp_router_schedule_work(struct net *net, 2704 struct notifier_block *nb, 2705 void (*cb)(struct work_struct *)) 2706 { 2707 struct mlxsw_sp_netevent_work *net_work; 2708 struct mlxsw_sp_router *router; 2709 2710 router = container_of(nb, struct mlxsw_sp_router, netevent_nb); 2711 if (!net_eq(net, mlxsw_sp_net(router->mlxsw_sp))) 2712 return NOTIFY_DONE; 2713 2714 net_work = kzalloc(sizeof(*net_work), GFP_ATOMIC); 2715 if (!net_work) 2716 return NOTIFY_BAD; 2717 2718 INIT_WORK(&net_work->work, cb); 2719 net_work->mlxsw_sp = router->mlxsw_sp; 2720 mlxsw_core_schedule_work(&net_work->work); 2721 return NOTIFY_DONE; 2722 } 2723 2724 static int mlxsw_sp_router_netevent_event(struct notifier_block *nb, 2725 unsigned long event, void *ptr) 2726 { 2727 struct mlxsw_sp_netevent_work *net_work; 2728 struct mlxsw_sp_port *mlxsw_sp_port; 2729 struct mlxsw_sp *mlxsw_sp; 2730 unsigned long interval; 2731 struct neigh_parms *p; 2732 struct neighbour *n; 2733 2734 switch (event) { 2735 case NETEVENT_DELAY_PROBE_TIME_UPDATE: 2736 p = ptr; 2737 2738 /* We don't care about changes in the default table. */ 2739 if (!p->dev || (p->tbl->family != AF_INET && 2740 p->tbl->family != AF_INET6)) 2741 return NOTIFY_DONE; 2742 2743 /* We are in atomic context and can't take RTNL mutex, 2744 * so use RCU variant to walk the device chain. 2745 */ 2746 mlxsw_sp_port = mlxsw_sp_port_lower_dev_hold(p->dev); 2747 if (!mlxsw_sp_port) 2748 return NOTIFY_DONE; 2749 2750 mlxsw_sp = mlxsw_sp_port->mlxsw_sp; 2751 interval = jiffies_to_msecs(NEIGH_VAR(p, DELAY_PROBE_TIME)); 2752 mlxsw_sp->router->neighs_update.interval = interval; 2753 2754 mlxsw_sp_port_dev_put(mlxsw_sp_port); 2755 break; 2756 case NETEVENT_NEIGH_UPDATE: 2757 n = ptr; 2758 2759 if (n->tbl->family != AF_INET && n->tbl->family != AF_INET6) 2760 return NOTIFY_DONE; 2761 2762 mlxsw_sp_port = mlxsw_sp_port_lower_dev_hold(n->dev); 2763 if (!mlxsw_sp_port) 2764 return NOTIFY_DONE; 2765 2766 net_work = kzalloc(sizeof(*net_work), GFP_ATOMIC); 2767 if (!net_work) { 2768 mlxsw_sp_port_dev_put(mlxsw_sp_port); 2769 return NOTIFY_BAD; 2770 } 2771 2772 INIT_WORK(&net_work->work, mlxsw_sp_router_neigh_event_work); 2773 net_work->mlxsw_sp = mlxsw_sp_port->mlxsw_sp; 2774 net_work->n = n; 2775 2776 /* Take a reference to ensure the neighbour won't be 2777 * destructed until we drop the reference in delayed 2778 * work. 2779 */ 2780 neigh_clone(n); 2781 mlxsw_core_schedule_work(&net_work->work); 2782 mlxsw_sp_port_dev_put(mlxsw_sp_port); 2783 break; 2784 case NETEVENT_IPV4_MPATH_HASH_UPDATE: 2785 case NETEVENT_IPV6_MPATH_HASH_UPDATE: 2786 return mlxsw_sp_router_schedule_work(ptr, nb, 2787 mlxsw_sp_router_mp_hash_event_work); 2788 2789 case NETEVENT_IPV4_FWD_UPDATE_PRIORITY_UPDATE: 2790 return mlxsw_sp_router_schedule_work(ptr, nb, 2791 mlxsw_sp_router_update_priority_work); 2792 } 2793 2794 return NOTIFY_DONE; 2795 } 2796 2797 static int mlxsw_sp_neigh_init(struct mlxsw_sp *mlxsw_sp) 2798 { 2799 int err; 2800 2801 err = rhashtable_init(&mlxsw_sp->router->neigh_ht, 2802 &mlxsw_sp_neigh_ht_params); 2803 if (err) 2804 return err; 2805 2806 /* Initialize the polling interval according to the default 2807 * table. 2808 */ 2809 mlxsw_sp_router_neighs_update_interval_init(mlxsw_sp); 2810 2811 /* Create the delayed works for the activity_update */ 2812 INIT_DELAYED_WORK(&mlxsw_sp->router->neighs_update.dw, 2813 mlxsw_sp_router_neighs_update_work); 2814 INIT_DELAYED_WORK(&mlxsw_sp->router->nexthop_probe_dw, 2815 mlxsw_sp_router_probe_unresolved_nexthops); 2816 mlxsw_core_schedule_dw(&mlxsw_sp->router->neighs_update.dw, 0); 2817 mlxsw_core_schedule_dw(&mlxsw_sp->router->nexthop_probe_dw, 0); 2818 return 0; 2819 } 2820 2821 static void mlxsw_sp_neigh_fini(struct mlxsw_sp *mlxsw_sp) 2822 { 2823 cancel_delayed_work_sync(&mlxsw_sp->router->neighs_update.dw); 2824 cancel_delayed_work_sync(&mlxsw_sp->router->nexthop_probe_dw); 2825 rhashtable_destroy(&mlxsw_sp->router->neigh_ht); 2826 } 2827 2828 static void mlxsw_sp_neigh_rif_gone_sync(struct mlxsw_sp *mlxsw_sp, 2829 struct mlxsw_sp_rif *rif) 2830 { 2831 struct mlxsw_sp_neigh_entry *neigh_entry, *tmp; 2832 2833 list_for_each_entry_safe(neigh_entry, tmp, &rif->neigh_list, 2834 rif_list_node) { 2835 mlxsw_sp_neigh_entry_update(mlxsw_sp, neigh_entry, false); 2836 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry); 2837 } 2838 } 2839 2840 enum mlxsw_sp_nexthop_type { 2841 MLXSW_SP_NEXTHOP_TYPE_ETH, 2842 MLXSW_SP_NEXTHOP_TYPE_IPIP, 2843 }; 2844 2845 struct mlxsw_sp_nexthop_key { 2846 struct fib_nh *fib_nh; 2847 }; 2848 2849 struct mlxsw_sp_nexthop { 2850 struct list_head neigh_list_node; /* member of neigh entry list */ 2851 struct list_head rif_list_node; 2852 struct list_head router_list_node; 2853 struct mlxsw_sp_nexthop_group_info *nhgi; /* pointer back to the group 2854 * this nexthop belongs to 2855 */ 2856 struct rhash_head ht_node; 2857 struct neigh_table *neigh_tbl; 2858 struct mlxsw_sp_nexthop_key key; 2859 unsigned char gw_addr[sizeof(struct in6_addr)]; 2860 int ifindex; 2861 int nh_weight; 2862 int norm_nh_weight; 2863 int num_adj_entries; 2864 struct mlxsw_sp_rif *rif; 2865 u8 should_offload:1, /* set indicates this neigh is connected and 2866 * should be put to KVD linear area of this group. 2867 */ 2868 offloaded:1, /* set in case the neigh is actually put into 2869 * KVD linear area of this group. 2870 */ 2871 update:1, /* set indicates that MAC of this neigh should be 2872 * updated in HW 2873 */ 2874 discard:1; /* nexthop is programmed to discard packets */ 2875 enum mlxsw_sp_nexthop_type type; 2876 union { 2877 struct mlxsw_sp_neigh_entry *neigh_entry; 2878 struct mlxsw_sp_ipip_entry *ipip_entry; 2879 }; 2880 unsigned int counter_index; 2881 bool counter_valid; 2882 }; 2883 2884 enum mlxsw_sp_nexthop_group_type { 2885 MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4, 2886 MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6, 2887 MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ, 2888 }; 2889 2890 struct mlxsw_sp_nexthop_group_info { 2891 struct mlxsw_sp_nexthop_group *nh_grp; 2892 u32 adj_index; 2893 u16 ecmp_size; 2894 u16 count; 2895 int sum_norm_weight; 2896 u8 adj_index_valid:1, 2897 gateway:1; /* routes using the group use a gateway */ 2898 struct mlxsw_sp_nexthop nexthops[0]; 2899 #define nh_rif nexthops[0].rif 2900 }; 2901 2902 struct mlxsw_sp_nexthop_group_vr_key { 2903 u16 vr_id; 2904 enum mlxsw_sp_l3proto proto; 2905 }; 2906 2907 struct mlxsw_sp_nexthop_group_vr_entry { 2908 struct list_head list; /* member in vr_list */ 2909 struct rhash_head ht_node; /* member in vr_ht */ 2910 refcount_t ref_count; 2911 struct mlxsw_sp_nexthop_group_vr_key key; 2912 }; 2913 2914 struct mlxsw_sp_nexthop_group { 2915 struct rhash_head ht_node; 2916 struct list_head fib_list; /* list of fib entries that use this group */ 2917 union { 2918 struct { 2919 struct fib_info *fi; 2920 } ipv4; 2921 struct { 2922 u32 id; 2923 } obj; 2924 }; 2925 struct mlxsw_sp_nexthop_group_info *nhgi; 2926 struct list_head vr_list; 2927 struct rhashtable vr_ht; 2928 enum mlxsw_sp_nexthop_group_type type; 2929 bool can_destroy; 2930 }; 2931 2932 void mlxsw_sp_nexthop_counter_alloc(struct mlxsw_sp *mlxsw_sp, 2933 struct mlxsw_sp_nexthop *nh) 2934 { 2935 struct devlink *devlink; 2936 2937 devlink = priv_to_devlink(mlxsw_sp->core); 2938 if (!devlink_dpipe_table_counter_enabled(devlink, 2939 MLXSW_SP_DPIPE_TABLE_NAME_ADJ)) 2940 return; 2941 2942 if (mlxsw_sp_flow_counter_alloc(mlxsw_sp, &nh->counter_index)) 2943 return; 2944 2945 nh->counter_valid = true; 2946 } 2947 2948 void mlxsw_sp_nexthop_counter_free(struct mlxsw_sp *mlxsw_sp, 2949 struct mlxsw_sp_nexthop *nh) 2950 { 2951 if (!nh->counter_valid) 2952 return; 2953 mlxsw_sp_flow_counter_free(mlxsw_sp, nh->counter_index); 2954 nh->counter_valid = false; 2955 } 2956 2957 int mlxsw_sp_nexthop_counter_get(struct mlxsw_sp *mlxsw_sp, 2958 struct mlxsw_sp_nexthop *nh, u64 *p_counter) 2959 { 2960 if (!nh->counter_valid) 2961 return -EINVAL; 2962 2963 return mlxsw_sp_flow_counter_get(mlxsw_sp, nh->counter_index, 2964 p_counter, NULL); 2965 } 2966 2967 struct mlxsw_sp_nexthop *mlxsw_sp_nexthop_next(struct mlxsw_sp_router *router, 2968 struct mlxsw_sp_nexthop *nh) 2969 { 2970 if (!nh) { 2971 if (list_empty(&router->nexthop_list)) 2972 return NULL; 2973 else 2974 return list_first_entry(&router->nexthop_list, 2975 typeof(*nh), router_list_node); 2976 } 2977 if (list_is_last(&nh->router_list_node, &router->nexthop_list)) 2978 return NULL; 2979 return list_next_entry(nh, router_list_node); 2980 } 2981 2982 bool mlxsw_sp_nexthop_offload(struct mlxsw_sp_nexthop *nh) 2983 { 2984 return nh->offloaded; 2985 } 2986 2987 unsigned char *mlxsw_sp_nexthop_ha(struct mlxsw_sp_nexthop *nh) 2988 { 2989 if (!nh->offloaded) 2990 return NULL; 2991 return nh->neigh_entry->ha; 2992 } 2993 2994 int mlxsw_sp_nexthop_indexes(struct mlxsw_sp_nexthop *nh, u32 *p_adj_index, 2995 u32 *p_adj_size, u32 *p_adj_hash_index) 2996 { 2997 struct mlxsw_sp_nexthop_group_info *nhgi = nh->nhgi; 2998 u32 adj_hash_index = 0; 2999 int i; 3000 3001 if (!nh->offloaded || !nhgi->adj_index_valid) 3002 return -EINVAL; 3003 3004 *p_adj_index = nhgi->adj_index; 3005 *p_adj_size = nhgi->ecmp_size; 3006 3007 for (i = 0; i < nhgi->count; i++) { 3008 struct mlxsw_sp_nexthop *nh_iter = &nhgi->nexthops[i]; 3009 3010 if (nh_iter == nh) 3011 break; 3012 if (nh_iter->offloaded) 3013 adj_hash_index += nh_iter->num_adj_entries; 3014 } 3015 3016 *p_adj_hash_index = adj_hash_index; 3017 return 0; 3018 } 3019 3020 struct mlxsw_sp_rif *mlxsw_sp_nexthop_rif(struct mlxsw_sp_nexthop *nh) 3021 { 3022 return nh->rif; 3023 } 3024 3025 bool mlxsw_sp_nexthop_group_has_ipip(struct mlxsw_sp_nexthop *nh) 3026 { 3027 struct mlxsw_sp_nexthop_group_info *nhgi = nh->nhgi; 3028 int i; 3029 3030 for (i = 0; i < nhgi->count; i++) { 3031 struct mlxsw_sp_nexthop *nh_iter = &nhgi->nexthops[i]; 3032 3033 if (nh_iter->type == MLXSW_SP_NEXTHOP_TYPE_IPIP) 3034 return true; 3035 } 3036 return false; 3037 } 3038 3039 bool mlxsw_sp_nexthop_is_discard(const struct mlxsw_sp_nexthop *nh) 3040 { 3041 return nh->discard; 3042 } 3043 3044 static const struct rhashtable_params mlxsw_sp_nexthop_group_vr_ht_params = { 3045 .key_offset = offsetof(struct mlxsw_sp_nexthop_group_vr_entry, key), 3046 .head_offset = offsetof(struct mlxsw_sp_nexthop_group_vr_entry, ht_node), 3047 .key_len = sizeof(struct mlxsw_sp_nexthop_group_vr_key), 3048 .automatic_shrinking = true, 3049 }; 3050 3051 static struct mlxsw_sp_nexthop_group_vr_entry * 3052 mlxsw_sp_nexthop_group_vr_entry_lookup(struct mlxsw_sp_nexthop_group *nh_grp, 3053 const struct mlxsw_sp_fib *fib) 3054 { 3055 struct mlxsw_sp_nexthop_group_vr_key key; 3056 3057 memset(&key, 0, sizeof(key)); 3058 key.vr_id = fib->vr->id; 3059 key.proto = fib->proto; 3060 return rhashtable_lookup_fast(&nh_grp->vr_ht, &key, 3061 mlxsw_sp_nexthop_group_vr_ht_params); 3062 } 3063 3064 static int 3065 mlxsw_sp_nexthop_group_vr_entry_create(struct mlxsw_sp_nexthop_group *nh_grp, 3066 const struct mlxsw_sp_fib *fib) 3067 { 3068 struct mlxsw_sp_nexthop_group_vr_entry *vr_entry; 3069 int err; 3070 3071 vr_entry = kzalloc(sizeof(*vr_entry), GFP_KERNEL); 3072 if (!vr_entry) 3073 return -ENOMEM; 3074 3075 vr_entry->key.vr_id = fib->vr->id; 3076 vr_entry->key.proto = fib->proto; 3077 refcount_set(&vr_entry->ref_count, 1); 3078 3079 err = rhashtable_insert_fast(&nh_grp->vr_ht, &vr_entry->ht_node, 3080 mlxsw_sp_nexthop_group_vr_ht_params); 3081 if (err) 3082 goto err_hashtable_insert; 3083 3084 list_add(&vr_entry->list, &nh_grp->vr_list); 3085 3086 return 0; 3087 3088 err_hashtable_insert: 3089 kfree(vr_entry); 3090 return err; 3091 } 3092 3093 static void 3094 mlxsw_sp_nexthop_group_vr_entry_destroy(struct mlxsw_sp_nexthop_group *nh_grp, 3095 struct mlxsw_sp_nexthop_group_vr_entry *vr_entry) 3096 { 3097 list_del(&vr_entry->list); 3098 rhashtable_remove_fast(&nh_grp->vr_ht, &vr_entry->ht_node, 3099 mlxsw_sp_nexthop_group_vr_ht_params); 3100 kfree(vr_entry); 3101 } 3102 3103 static int 3104 mlxsw_sp_nexthop_group_vr_link(struct mlxsw_sp_nexthop_group *nh_grp, 3105 const struct mlxsw_sp_fib *fib) 3106 { 3107 struct mlxsw_sp_nexthop_group_vr_entry *vr_entry; 3108 3109 vr_entry = mlxsw_sp_nexthop_group_vr_entry_lookup(nh_grp, fib); 3110 if (vr_entry) { 3111 refcount_inc(&vr_entry->ref_count); 3112 return 0; 3113 } 3114 3115 return mlxsw_sp_nexthop_group_vr_entry_create(nh_grp, fib); 3116 } 3117 3118 static void 3119 mlxsw_sp_nexthop_group_vr_unlink(struct mlxsw_sp_nexthop_group *nh_grp, 3120 const struct mlxsw_sp_fib *fib) 3121 { 3122 struct mlxsw_sp_nexthop_group_vr_entry *vr_entry; 3123 3124 vr_entry = mlxsw_sp_nexthop_group_vr_entry_lookup(nh_grp, fib); 3125 if (WARN_ON_ONCE(!vr_entry)) 3126 return; 3127 3128 if (!refcount_dec_and_test(&vr_entry->ref_count)) 3129 return; 3130 3131 mlxsw_sp_nexthop_group_vr_entry_destroy(nh_grp, vr_entry); 3132 } 3133 3134 struct mlxsw_sp_nexthop_group_cmp_arg { 3135 enum mlxsw_sp_nexthop_group_type type; 3136 union { 3137 struct fib_info *fi; 3138 struct mlxsw_sp_fib6_entry *fib6_entry; 3139 u32 id; 3140 }; 3141 }; 3142 3143 static bool 3144 mlxsw_sp_nexthop6_group_has_nexthop(const struct mlxsw_sp_nexthop_group *nh_grp, 3145 const struct in6_addr *gw, int ifindex, 3146 int weight) 3147 { 3148 int i; 3149 3150 for (i = 0; i < nh_grp->nhgi->count; i++) { 3151 const struct mlxsw_sp_nexthop *nh; 3152 3153 nh = &nh_grp->nhgi->nexthops[i]; 3154 if (nh->ifindex == ifindex && nh->nh_weight == weight && 3155 ipv6_addr_equal(gw, (struct in6_addr *) nh->gw_addr)) 3156 return true; 3157 } 3158 3159 return false; 3160 } 3161 3162 static bool 3163 mlxsw_sp_nexthop6_group_cmp(const struct mlxsw_sp_nexthop_group *nh_grp, 3164 const struct mlxsw_sp_fib6_entry *fib6_entry) 3165 { 3166 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 3167 3168 if (nh_grp->nhgi->count != fib6_entry->nrt6) 3169 return false; 3170 3171 list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) { 3172 struct fib6_nh *fib6_nh = mlxsw_sp_rt6->rt->fib6_nh; 3173 struct in6_addr *gw; 3174 int ifindex, weight; 3175 3176 ifindex = fib6_nh->fib_nh_dev->ifindex; 3177 weight = fib6_nh->fib_nh_weight; 3178 gw = &fib6_nh->fib_nh_gw6; 3179 if (!mlxsw_sp_nexthop6_group_has_nexthop(nh_grp, gw, ifindex, 3180 weight)) 3181 return false; 3182 } 3183 3184 return true; 3185 } 3186 3187 static int 3188 mlxsw_sp_nexthop_group_cmp(struct rhashtable_compare_arg *arg, const void *ptr) 3189 { 3190 const struct mlxsw_sp_nexthop_group_cmp_arg *cmp_arg = arg->key; 3191 const struct mlxsw_sp_nexthop_group *nh_grp = ptr; 3192 3193 if (nh_grp->type != cmp_arg->type) 3194 return 1; 3195 3196 switch (cmp_arg->type) { 3197 case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4: 3198 return cmp_arg->fi != nh_grp->ipv4.fi; 3199 case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6: 3200 return !mlxsw_sp_nexthop6_group_cmp(nh_grp, 3201 cmp_arg->fib6_entry); 3202 case MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ: 3203 return cmp_arg->id != nh_grp->obj.id; 3204 default: 3205 WARN_ON(1); 3206 return 1; 3207 } 3208 } 3209 3210 static u32 mlxsw_sp_nexthop_group_hash_obj(const void *data, u32 len, u32 seed) 3211 { 3212 const struct mlxsw_sp_nexthop_group *nh_grp = data; 3213 const struct mlxsw_sp_nexthop *nh; 3214 struct fib_info *fi; 3215 unsigned int val; 3216 int i; 3217 3218 switch (nh_grp->type) { 3219 case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4: 3220 fi = nh_grp->ipv4.fi; 3221 return jhash(&fi, sizeof(fi), seed); 3222 case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6: 3223 val = nh_grp->nhgi->count; 3224 for (i = 0; i < nh_grp->nhgi->count; i++) { 3225 nh = &nh_grp->nhgi->nexthops[i]; 3226 val ^= jhash(&nh->ifindex, sizeof(nh->ifindex), seed); 3227 val ^= jhash(&nh->gw_addr, sizeof(nh->gw_addr), seed); 3228 } 3229 return jhash(&val, sizeof(val), seed); 3230 case MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ: 3231 return jhash(&nh_grp->obj.id, sizeof(nh_grp->obj.id), seed); 3232 default: 3233 WARN_ON(1); 3234 return 0; 3235 } 3236 } 3237 3238 static u32 3239 mlxsw_sp_nexthop6_group_hash(struct mlxsw_sp_fib6_entry *fib6_entry, u32 seed) 3240 { 3241 unsigned int val = fib6_entry->nrt6; 3242 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 3243 3244 list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) { 3245 struct fib6_nh *fib6_nh = mlxsw_sp_rt6->rt->fib6_nh; 3246 struct net_device *dev = fib6_nh->fib_nh_dev; 3247 struct in6_addr *gw = &fib6_nh->fib_nh_gw6; 3248 3249 val ^= jhash(&dev->ifindex, sizeof(dev->ifindex), seed); 3250 val ^= jhash(gw, sizeof(*gw), seed); 3251 } 3252 3253 return jhash(&val, sizeof(val), seed); 3254 } 3255 3256 static u32 3257 mlxsw_sp_nexthop_group_hash(const void *data, u32 len, u32 seed) 3258 { 3259 const struct mlxsw_sp_nexthop_group_cmp_arg *cmp_arg = data; 3260 3261 switch (cmp_arg->type) { 3262 case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4: 3263 return jhash(&cmp_arg->fi, sizeof(cmp_arg->fi), seed); 3264 case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6: 3265 return mlxsw_sp_nexthop6_group_hash(cmp_arg->fib6_entry, seed); 3266 case MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ: 3267 return jhash(&cmp_arg->id, sizeof(cmp_arg->id), seed); 3268 default: 3269 WARN_ON(1); 3270 return 0; 3271 } 3272 } 3273 3274 static const struct rhashtable_params mlxsw_sp_nexthop_group_ht_params = { 3275 .head_offset = offsetof(struct mlxsw_sp_nexthop_group, ht_node), 3276 .hashfn = mlxsw_sp_nexthop_group_hash, 3277 .obj_hashfn = mlxsw_sp_nexthop_group_hash_obj, 3278 .obj_cmpfn = mlxsw_sp_nexthop_group_cmp, 3279 }; 3280 3281 static int mlxsw_sp_nexthop_group_insert(struct mlxsw_sp *mlxsw_sp, 3282 struct mlxsw_sp_nexthop_group *nh_grp) 3283 { 3284 if (nh_grp->type == MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6 && 3285 !nh_grp->nhgi->gateway) 3286 return 0; 3287 3288 return rhashtable_insert_fast(&mlxsw_sp->router->nexthop_group_ht, 3289 &nh_grp->ht_node, 3290 mlxsw_sp_nexthop_group_ht_params); 3291 } 3292 3293 static void mlxsw_sp_nexthop_group_remove(struct mlxsw_sp *mlxsw_sp, 3294 struct mlxsw_sp_nexthop_group *nh_grp) 3295 { 3296 if (nh_grp->type == MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6 && 3297 !nh_grp->nhgi->gateway) 3298 return; 3299 3300 rhashtable_remove_fast(&mlxsw_sp->router->nexthop_group_ht, 3301 &nh_grp->ht_node, 3302 mlxsw_sp_nexthop_group_ht_params); 3303 } 3304 3305 static struct mlxsw_sp_nexthop_group * 3306 mlxsw_sp_nexthop4_group_lookup(struct mlxsw_sp *mlxsw_sp, 3307 struct fib_info *fi) 3308 { 3309 struct mlxsw_sp_nexthop_group_cmp_arg cmp_arg; 3310 3311 cmp_arg.type = MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4; 3312 cmp_arg.fi = fi; 3313 return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_group_ht, 3314 &cmp_arg, 3315 mlxsw_sp_nexthop_group_ht_params); 3316 } 3317 3318 static struct mlxsw_sp_nexthop_group * 3319 mlxsw_sp_nexthop6_group_lookup(struct mlxsw_sp *mlxsw_sp, 3320 struct mlxsw_sp_fib6_entry *fib6_entry) 3321 { 3322 struct mlxsw_sp_nexthop_group_cmp_arg cmp_arg; 3323 3324 cmp_arg.type = MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6; 3325 cmp_arg.fib6_entry = fib6_entry; 3326 return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_group_ht, 3327 &cmp_arg, 3328 mlxsw_sp_nexthop_group_ht_params); 3329 } 3330 3331 static const struct rhashtable_params mlxsw_sp_nexthop_ht_params = { 3332 .key_offset = offsetof(struct mlxsw_sp_nexthop, key), 3333 .head_offset = offsetof(struct mlxsw_sp_nexthop, ht_node), 3334 .key_len = sizeof(struct mlxsw_sp_nexthop_key), 3335 }; 3336 3337 static int mlxsw_sp_nexthop_insert(struct mlxsw_sp *mlxsw_sp, 3338 struct mlxsw_sp_nexthop *nh) 3339 { 3340 return rhashtable_insert_fast(&mlxsw_sp->router->nexthop_ht, 3341 &nh->ht_node, mlxsw_sp_nexthop_ht_params); 3342 } 3343 3344 static void mlxsw_sp_nexthop_remove(struct mlxsw_sp *mlxsw_sp, 3345 struct mlxsw_sp_nexthop *nh) 3346 { 3347 rhashtable_remove_fast(&mlxsw_sp->router->nexthop_ht, &nh->ht_node, 3348 mlxsw_sp_nexthop_ht_params); 3349 } 3350 3351 static struct mlxsw_sp_nexthop * 3352 mlxsw_sp_nexthop_lookup(struct mlxsw_sp *mlxsw_sp, 3353 struct mlxsw_sp_nexthop_key key) 3354 { 3355 return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_ht, &key, 3356 mlxsw_sp_nexthop_ht_params); 3357 } 3358 3359 static int mlxsw_sp_adj_index_mass_update_vr(struct mlxsw_sp *mlxsw_sp, 3360 enum mlxsw_sp_l3proto proto, 3361 u16 vr_id, 3362 u32 adj_index, u16 ecmp_size, 3363 u32 new_adj_index, 3364 u16 new_ecmp_size) 3365 { 3366 char raleu_pl[MLXSW_REG_RALEU_LEN]; 3367 3368 mlxsw_reg_raleu_pack(raleu_pl, 3369 (enum mlxsw_reg_ralxx_protocol) proto, vr_id, 3370 adj_index, ecmp_size, new_adj_index, 3371 new_ecmp_size); 3372 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raleu), raleu_pl); 3373 } 3374 3375 static int mlxsw_sp_adj_index_mass_update(struct mlxsw_sp *mlxsw_sp, 3376 struct mlxsw_sp_nexthop_group *nh_grp, 3377 u32 old_adj_index, u16 old_ecmp_size) 3378 { 3379 struct mlxsw_sp_nexthop_group_info *nhgi = nh_grp->nhgi; 3380 struct mlxsw_sp_nexthop_group_vr_entry *vr_entry; 3381 int err; 3382 3383 list_for_each_entry(vr_entry, &nh_grp->vr_list, list) { 3384 err = mlxsw_sp_adj_index_mass_update_vr(mlxsw_sp, 3385 vr_entry->key.proto, 3386 vr_entry->key.vr_id, 3387 old_adj_index, 3388 old_ecmp_size, 3389 nhgi->adj_index, 3390 nhgi->ecmp_size); 3391 if (err) 3392 goto err_mass_update_vr; 3393 } 3394 return 0; 3395 3396 err_mass_update_vr: 3397 list_for_each_entry_continue_reverse(vr_entry, &nh_grp->vr_list, list) 3398 mlxsw_sp_adj_index_mass_update_vr(mlxsw_sp, vr_entry->key.proto, 3399 vr_entry->key.vr_id, 3400 nhgi->adj_index, 3401 nhgi->ecmp_size, 3402 old_adj_index, old_ecmp_size); 3403 return err; 3404 } 3405 3406 static int __mlxsw_sp_nexthop_update(struct mlxsw_sp *mlxsw_sp, u32 adj_index, 3407 struct mlxsw_sp_nexthop *nh) 3408 { 3409 struct mlxsw_sp_neigh_entry *neigh_entry = nh->neigh_entry; 3410 char ratr_pl[MLXSW_REG_RATR_LEN]; 3411 3412 mlxsw_reg_ratr_pack(ratr_pl, MLXSW_REG_RATR_OP_WRITE_WRITE_ENTRY, 3413 true, MLXSW_REG_RATR_TYPE_ETHERNET, 3414 adj_index, nh->rif->rif_index); 3415 if (nh->discard) 3416 mlxsw_reg_ratr_trap_action_set(ratr_pl, 3417 MLXSW_REG_RATR_TRAP_ACTION_DISCARD_ERRORS); 3418 else 3419 mlxsw_reg_ratr_eth_entry_pack(ratr_pl, neigh_entry->ha); 3420 if (nh->counter_valid) 3421 mlxsw_reg_ratr_counter_pack(ratr_pl, nh->counter_index, true); 3422 else 3423 mlxsw_reg_ratr_counter_pack(ratr_pl, 0, false); 3424 3425 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ratr), ratr_pl); 3426 } 3427 3428 int mlxsw_sp_nexthop_update(struct mlxsw_sp *mlxsw_sp, u32 adj_index, 3429 struct mlxsw_sp_nexthop *nh) 3430 { 3431 int i; 3432 3433 for (i = 0; i < nh->num_adj_entries; i++) { 3434 int err; 3435 3436 err = __mlxsw_sp_nexthop_update(mlxsw_sp, adj_index + i, nh); 3437 if (err) 3438 return err; 3439 } 3440 3441 return 0; 3442 } 3443 3444 static int __mlxsw_sp_nexthop_ipip_update(struct mlxsw_sp *mlxsw_sp, 3445 u32 adj_index, 3446 struct mlxsw_sp_nexthop *nh) 3447 { 3448 const struct mlxsw_sp_ipip_ops *ipip_ops; 3449 3450 ipip_ops = mlxsw_sp->router->ipip_ops_arr[nh->ipip_entry->ipipt]; 3451 return ipip_ops->nexthop_update(mlxsw_sp, adj_index, nh->ipip_entry); 3452 } 3453 3454 static int mlxsw_sp_nexthop_ipip_update(struct mlxsw_sp *mlxsw_sp, 3455 u32 adj_index, 3456 struct mlxsw_sp_nexthop *nh) 3457 { 3458 int i; 3459 3460 for (i = 0; i < nh->num_adj_entries; i++) { 3461 int err; 3462 3463 err = __mlxsw_sp_nexthop_ipip_update(mlxsw_sp, adj_index + i, 3464 nh); 3465 if (err) 3466 return err; 3467 } 3468 3469 return 0; 3470 } 3471 3472 static int 3473 mlxsw_sp_nexthop_group_update(struct mlxsw_sp *mlxsw_sp, 3474 struct mlxsw_sp_nexthop_group_info *nhgi, 3475 bool reallocate) 3476 { 3477 u32 adj_index = nhgi->adj_index; /* base */ 3478 struct mlxsw_sp_nexthop *nh; 3479 int i; 3480 3481 for (i = 0; i < nhgi->count; i++) { 3482 nh = &nhgi->nexthops[i]; 3483 3484 if (!nh->should_offload) { 3485 nh->offloaded = 0; 3486 continue; 3487 } 3488 3489 if (nh->update || reallocate) { 3490 int err = 0; 3491 3492 switch (nh->type) { 3493 case MLXSW_SP_NEXTHOP_TYPE_ETH: 3494 err = mlxsw_sp_nexthop_update 3495 (mlxsw_sp, adj_index, nh); 3496 break; 3497 case MLXSW_SP_NEXTHOP_TYPE_IPIP: 3498 err = mlxsw_sp_nexthop_ipip_update 3499 (mlxsw_sp, adj_index, nh); 3500 break; 3501 } 3502 if (err) 3503 return err; 3504 nh->update = 0; 3505 nh->offloaded = 1; 3506 } 3507 adj_index += nh->num_adj_entries; 3508 } 3509 return 0; 3510 } 3511 3512 static int 3513 mlxsw_sp_nexthop_fib_entries_update(struct mlxsw_sp *mlxsw_sp, 3514 struct mlxsw_sp_nexthop_group *nh_grp) 3515 { 3516 struct mlxsw_sp_fib_entry *fib_entry; 3517 int err; 3518 3519 list_for_each_entry(fib_entry, &nh_grp->fib_list, nexthop_group_node) { 3520 err = mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry); 3521 if (err) 3522 return err; 3523 } 3524 return 0; 3525 } 3526 3527 static void mlxsw_sp_adj_grp_size_round_up(u16 *p_adj_grp_size) 3528 { 3529 /* Valid sizes for an adjacency group are: 3530 * 1-64, 512, 1024, 2048 and 4096. 3531 */ 3532 if (*p_adj_grp_size <= 64) 3533 return; 3534 else if (*p_adj_grp_size <= 512) 3535 *p_adj_grp_size = 512; 3536 else if (*p_adj_grp_size <= 1024) 3537 *p_adj_grp_size = 1024; 3538 else if (*p_adj_grp_size <= 2048) 3539 *p_adj_grp_size = 2048; 3540 else 3541 *p_adj_grp_size = 4096; 3542 } 3543 3544 static void mlxsw_sp_adj_grp_size_round_down(u16 *p_adj_grp_size, 3545 unsigned int alloc_size) 3546 { 3547 if (alloc_size >= 4096) 3548 *p_adj_grp_size = 4096; 3549 else if (alloc_size >= 2048) 3550 *p_adj_grp_size = 2048; 3551 else if (alloc_size >= 1024) 3552 *p_adj_grp_size = 1024; 3553 else if (alloc_size >= 512) 3554 *p_adj_grp_size = 512; 3555 } 3556 3557 static int mlxsw_sp_fix_adj_grp_size(struct mlxsw_sp *mlxsw_sp, 3558 u16 *p_adj_grp_size) 3559 { 3560 unsigned int alloc_size; 3561 int err; 3562 3563 /* Round up the requested group size to the next size supported 3564 * by the device and make sure the request can be satisfied. 3565 */ 3566 mlxsw_sp_adj_grp_size_round_up(p_adj_grp_size); 3567 err = mlxsw_sp_kvdl_alloc_count_query(mlxsw_sp, 3568 MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 3569 *p_adj_grp_size, &alloc_size); 3570 if (err) 3571 return err; 3572 /* It is possible the allocation results in more allocated 3573 * entries than requested. Try to use as much of them as 3574 * possible. 3575 */ 3576 mlxsw_sp_adj_grp_size_round_down(p_adj_grp_size, alloc_size); 3577 3578 return 0; 3579 } 3580 3581 static void 3582 mlxsw_sp_nexthop_group_normalize(struct mlxsw_sp_nexthop_group_info *nhgi) 3583 { 3584 int i, g = 0, sum_norm_weight = 0; 3585 struct mlxsw_sp_nexthop *nh; 3586 3587 for (i = 0; i < nhgi->count; i++) { 3588 nh = &nhgi->nexthops[i]; 3589 3590 if (!nh->should_offload) 3591 continue; 3592 if (g > 0) 3593 g = gcd(nh->nh_weight, g); 3594 else 3595 g = nh->nh_weight; 3596 } 3597 3598 for (i = 0; i < nhgi->count; i++) { 3599 nh = &nhgi->nexthops[i]; 3600 3601 if (!nh->should_offload) 3602 continue; 3603 nh->norm_nh_weight = nh->nh_weight / g; 3604 sum_norm_weight += nh->norm_nh_weight; 3605 } 3606 3607 nhgi->sum_norm_weight = sum_norm_weight; 3608 } 3609 3610 static void 3611 mlxsw_sp_nexthop_group_rebalance(struct mlxsw_sp_nexthop_group_info *nhgi) 3612 { 3613 int i, weight = 0, lower_bound = 0; 3614 int total = nhgi->sum_norm_weight; 3615 u16 ecmp_size = nhgi->ecmp_size; 3616 3617 for (i = 0; i < nhgi->count; i++) { 3618 struct mlxsw_sp_nexthop *nh = &nhgi->nexthops[i]; 3619 int upper_bound; 3620 3621 if (!nh->should_offload) 3622 continue; 3623 weight += nh->norm_nh_weight; 3624 upper_bound = DIV_ROUND_CLOSEST(ecmp_size * weight, total); 3625 nh->num_adj_entries = upper_bound - lower_bound; 3626 lower_bound = upper_bound; 3627 } 3628 } 3629 3630 static struct mlxsw_sp_nexthop * 3631 mlxsw_sp_rt6_nexthop(struct mlxsw_sp_nexthop_group *nh_grp, 3632 const struct mlxsw_sp_rt6 *mlxsw_sp_rt6); 3633 3634 static void 3635 mlxsw_sp_nexthop4_group_offload_refresh(struct mlxsw_sp *mlxsw_sp, 3636 struct mlxsw_sp_nexthop_group *nh_grp) 3637 { 3638 int i; 3639 3640 for (i = 0; i < nh_grp->nhgi->count; i++) { 3641 struct mlxsw_sp_nexthop *nh = &nh_grp->nhgi->nexthops[i]; 3642 3643 if (nh->offloaded) 3644 nh->key.fib_nh->fib_nh_flags |= RTNH_F_OFFLOAD; 3645 else 3646 nh->key.fib_nh->fib_nh_flags &= ~RTNH_F_OFFLOAD; 3647 } 3648 } 3649 3650 static void 3651 __mlxsw_sp_nexthop6_group_offload_refresh(struct mlxsw_sp_nexthop_group *nh_grp, 3652 struct mlxsw_sp_fib6_entry *fib6_entry) 3653 { 3654 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 3655 3656 list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) { 3657 struct fib6_nh *fib6_nh = mlxsw_sp_rt6->rt->fib6_nh; 3658 struct mlxsw_sp_nexthop *nh; 3659 3660 nh = mlxsw_sp_rt6_nexthop(nh_grp, mlxsw_sp_rt6); 3661 if (nh && nh->offloaded) 3662 fib6_nh->fib_nh_flags |= RTNH_F_OFFLOAD; 3663 else 3664 fib6_nh->fib_nh_flags &= ~RTNH_F_OFFLOAD; 3665 } 3666 } 3667 3668 static void 3669 mlxsw_sp_nexthop6_group_offload_refresh(struct mlxsw_sp *mlxsw_sp, 3670 struct mlxsw_sp_nexthop_group *nh_grp) 3671 { 3672 struct mlxsw_sp_fib6_entry *fib6_entry; 3673 3674 /* Unfortunately, in IPv6 the route and the nexthop are described by 3675 * the same struct, so we need to iterate over all the routes using the 3676 * nexthop group and set / clear the offload indication for them. 3677 */ 3678 list_for_each_entry(fib6_entry, &nh_grp->fib_list, 3679 common.nexthop_group_node) 3680 __mlxsw_sp_nexthop6_group_offload_refresh(nh_grp, fib6_entry); 3681 } 3682 3683 static void 3684 mlxsw_sp_nexthop_obj_group_offload_refresh(struct mlxsw_sp *mlxsw_sp, 3685 struct mlxsw_sp_nexthop_group *nh_grp) 3686 { 3687 /* Do not update the flags if the nexthop group is being destroyed 3688 * since: 3689 * 1. The nexthop objects is being deleted, in which case the flags are 3690 * irrelevant. 3691 * 2. The nexthop group was replaced by a newer group, in which case 3692 * the flags of the nexthop object were already updated based on the 3693 * new group. 3694 */ 3695 if (nh_grp->can_destroy) 3696 return; 3697 3698 nexthop_set_hw_flags(mlxsw_sp_net(mlxsw_sp), nh_grp->obj.id, 3699 nh_grp->nhgi->adj_index_valid, false); 3700 } 3701 3702 static void 3703 mlxsw_sp_nexthop_group_offload_refresh(struct mlxsw_sp *mlxsw_sp, 3704 struct mlxsw_sp_nexthop_group *nh_grp) 3705 { 3706 switch (nh_grp->type) { 3707 case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4: 3708 mlxsw_sp_nexthop4_group_offload_refresh(mlxsw_sp, nh_grp); 3709 break; 3710 case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6: 3711 mlxsw_sp_nexthop6_group_offload_refresh(mlxsw_sp, nh_grp); 3712 break; 3713 case MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ: 3714 mlxsw_sp_nexthop_obj_group_offload_refresh(mlxsw_sp, nh_grp); 3715 break; 3716 } 3717 } 3718 3719 static int 3720 mlxsw_sp_nexthop_group_refresh(struct mlxsw_sp *mlxsw_sp, 3721 struct mlxsw_sp_nexthop_group *nh_grp) 3722 { 3723 struct mlxsw_sp_nexthop_group_info *nhgi = nh_grp->nhgi; 3724 u16 ecmp_size, old_ecmp_size; 3725 struct mlxsw_sp_nexthop *nh; 3726 bool offload_change = false; 3727 u32 adj_index; 3728 bool old_adj_index_valid; 3729 int i, err2, err = 0; 3730 u32 old_adj_index; 3731 3732 if (!nhgi->gateway) 3733 return mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp); 3734 3735 for (i = 0; i < nhgi->count; i++) { 3736 nh = &nhgi->nexthops[i]; 3737 3738 if (nh->should_offload != nh->offloaded) { 3739 offload_change = true; 3740 if (nh->should_offload) 3741 nh->update = 1; 3742 } 3743 } 3744 if (!offload_change) { 3745 /* Nothing was added or removed, so no need to reallocate. Just 3746 * update MAC on existing adjacency indexes. 3747 */ 3748 err = mlxsw_sp_nexthop_group_update(mlxsw_sp, nhgi, false); 3749 if (err) { 3750 dev_warn(mlxsw_sp->bus_info->dev, "Failed to update neigh MAC in adjacency table.\n"); 3751 goto set_trap; 3752 } 3753 return 0; 3754 } 3755 mlxsw_sp_nexthop_group_normalize(nhgi); 3756 if (!nhgi->sum_norm_weight) 3757 /* No neigh of this group is connected so we just set 3758 * the trap and let everthing flow through kernel. 3759 */ 3760 goto set_trap; 3761 3762 ecmp_size = nhgi->sum_norm_weight; 3763 err = mlxsw_sp_fix_adj_grp_size(mlxsw_sp, &ecmp_size); 3764 if (err) 3765 /* No valid allocation size available. */ 3766 goto set_trap; 3767 3768 err = mlxsw_sp_kvdl_alloc(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 3769 ecmp_size, &adj_index); 3770 if (err) { 3771 /* We ran out of KVD linear space, just set the 3772 * trap and let everything flow through kernel. 3773 */ 3774 dev_warn(mlxsw_sp->bus_info->dev, "Failed to allocate KVD linear area for nexthop group.\n"); 3775 goto set_trap; 3776 } 3777 old_adj_index_valid = nhgi->adj_index_valid; 3778 old_adj_index = nhgi->adj_index; 3779 old_ecmp_size = nhgi->ecmp_size; 3780 nhgi->adj_index_valid = 1; 3781 nhgi->adj_index = adj_index; 3782 nhgi->ecmp_size = ecmp_size; 3783 mlxsw_sp_nexthop_group_rebalance(nhgi); 3784 err = mlxsw_sp_nexthop_group_update(mlxsw_sp, nhgi, true); 3785 if (err) { 3786 dev_warn(mlxsw_sp->bus_info->dev, "Failed to update neigh MAC in adjacency table.\n"); 3787 goto set_trap; 3788 } 3789 3790 mlxsw_sp_nexthop_group_offload_refresh(mlxsw_sp, nh_grp); 3791 3792 if (!old_adj_index_valid) { 3793 /* The trap was set for fib entries, so we have to call 3794 * fib entry update to unset it and use adjacency index. 3795 */ 3796 err = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp); 3797 if (err) { 3798 dev_warn(mlxsw_sp->bus_info->dev, "Failed to add adjacency index to fib entries.\n"); 3799 goto set_trap; 3800 } 3801 return 0; 3802 } 3803 3804 err = mlxsw_sp_adj_index_mass_update(mlxsw_sp, nh_grp, 3805 old_adj_index, old_ecmp_size); 3806 mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 3807 old_ecmp_size, old_adj_index); 3808 if (err) { 3809 dev_warn(mlxsw_sp->bus_info->dev, "Failed to mass-update adjacency index for nexthop group.\n"); 3810 goto set_trap; 3811 } 3812 3813 return 0; 3814 3815 set_trap: 3816 old_adj_index_valid = nhgi->adj_index_valid; 3817 nhgi->adj_index_valid = 0; 3818 for (i = 0; i < nhgi->count; i++) { 3819 nh = &nhgi->nexthops[i]; 3820 nh->offloaded = 0; 3821 } 3822 err2 = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp); 3823 if (err2) 3824 dev_warn(mlxsw_sp->bus_info->dev, "Failed to set traps for fib entries.\n"); 3825 mlxsw_sp_nexthop_group_offload_refresh(mlxsw_sp, nh_grp); 3826 if (old_adj_index_valid) 3827 mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 3828 nhgi->ecmp_size, nhgi->adj_index); 3829 return err; 3830 } 3831 3832 static void __mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp_nexthop *nh, 3833 bool removing) 3834 { 3835 if (!removing) 3836 nh->should_offload = 1; 3837 else 3838 nh->should_offload = 0; 3839 nh->update = 1; 3840 } 3841 3842 static int 3843 mlxsw_sp_nexthop_dead_neigh_replace(struct mlxsw_sp *mlxsw_sp, 3844 struct mlxsw_sp_neigh_entry *neigh_entry) 3845 { 3846 struct neighbour *n, *old_n = neigh_entry->key.n; 3847 struct mlxsw_sp_nexthop *nh; 3848 bool entry_connected; 3849 u8 nud_state, dead; 3850 int err; 3851 3852 nh = list_first_entry(&neigh_entry->nexthop_list, 3853 struct mlxsw_sp_nexthop, neigh_list_node); 3854 3855 n = neigh_lookup(nh->neigh_tbl, &nh->gw_addr, nh->rif->dev); 3856 if (!n) { 3857 n = neigh_create(nh->neigh_tbl, &nh->gw_addr, nh->rif->dev); 3858 if (IS_ERR(n)) 3859 return PTR_ERR(n); 3860 neigh_event_send(n, NULL); 3861 } 3862 3863 mlxsw_sp_neigh_entry_remove(mlxsw_sp, neigh_entry); 3864 neigh_entry->key.n = n; 3865 err = mlxsw_sp_neigh_entry_insert(mlxsw_sp, neigh_entry); 3866 if (err) 3867 goto err_neigh_entry_insert; 3868 3869 read_lock_bh(&n->lock); 3870 nud_state = n->nud_state; 3871 dead = n->dead; 3872 read_unlock_bh(&n->lock); 3873 entry_connected = nud_state & NUD_VALID && !dead; 3874 3875 list_for_each_entry(nh, &neigh_entry->nexthop_list, 3876 neigh_list_node) { 3877 neigh_release(old_n); 3878 neigh_clone(n); 3879 __mlxsw_sp_nexthop_neigh_update(nh, !entry_connected); 3880 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nhgi->nh_grp); 3881 } 3882 3883 neigh_release(n); 3884 3885 return 0; 3886 3887 err_neigh_entry_insert: 3888 neigh_entry->key.n = old_n; 3889 mlxsw_sp_neigh_entry_insert(mlxsw_sp, neigh_entry); 3890 neigh_release(n); 3891 return err; 3892 } 3893 3894 static void 3895 mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp *mlxsw_sp, 3896 struct mlxsw_sp_neigh_entry *neigh_entry, 3897 bool removing, bool dead) 3898 { 3899 struct mlxsw_sp_nexthop *nh; 3900 3901 if (list_empty(&neigh_entry->nexthop_list)) 3902 return; 3903 3904 if (dead) { 3905 int err; 3906 3907 err = mlxsw_sp_nexthop_dead_neigh_replace(mlxsw_sp, 3908 neigh_entry); 3909 if (err) 3910 dev_err(mlxsw_sp->bus_info->dev, "Failed to replace dead neigh\n"); 3911 return; 3912 } 3913 3914 list_for_each_entry(nh, &neigh_entry->nexthop_list, 3915 neigh_list_node) { 3916 __mlxsw_sp_nexthop_neigh_update(nh, removing); 3917 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nhgi->nh_grp); 3918 } 3919 } 3920 3921 static void mlxsw_sp_nexthop_rif_init(struct mlxsw_sp_nexthop *nh, 3922 struct mlxsw_sp_rif *rif) 3923 { 3924 if (nh->rif) 3925 return; 3926 3927 nh->rif = rif; 3928 list_add(&nh->rif_list_node, &rif->nexthop_list); 3929 } 3930 3931 static void mlxsw_sp_nexthop_rif_fini(struct mlxsw_sp_nexthop *nh) 3932 { 3933 if (!nh->rif) 3934 return; 3935 3936 list_del(&nh->rif_list_node); 3937 nh->rif = NULL; 3938 } 3939 3940 static int mlxsw_sp_nexthop_neigh_init(struct mlxsw_sp *mlxsw_sp, 3941 struct mlxsw_sp_nexthop *nh) 3942 { 3943 struct mlxsw_sp_neigh_entry *neigh_entry; 3944 struct neighbour *n; 3945 u8 nud_state, dead; 3946 int err; 3947 3948 if (!nh->nhgi->gateway || nh->neigh_entry) 3949 return 0; 3950 3951 /* Take a reference of neigh here ensuring that neigh would 3952 * not be destructed before the nexthop entry is finished. 3953 * The reference is taken either in neigh_lookup() or 3954 * in neigh_create() in case n is not found. 3955 */ 3956 n = neigh_lookup(nh->neigh_tbl, &nh->gw_addr, nh->rif->dev); 3957 if (!n) { 3958 n = neigh_create(nh->neigh_tbl, &nh->gw_addr, nh->rif->dev); 3959 if (IS_ERR(n)) 3960 return PTR_ERR(n); 3961 neigh_event_send(n, NULL); 3962 } 3963 neigh_entry = mlxsw_sp_neigh_entry_lookup(mlxsw_sp, n); 3964 if (!neigh_entry) { 3965 neigh_entry = mlxsw_sp_neigh_entry_create(mlxsw_sp, n); 3966 if (IS_ERR(neigh_entry)) { 3967 err = -EINVAL; 3968 goto err_neigh_entry_create; 3969 } 3970 } 3971 3972 /* If that is the first nexthop connected to that neigh, add to 3973 * nexthop_neighs_list 3974 */ 3975 if (list_empty(&neigh_entry->nexthop_list)) 3976 list_add_tail(&neigh_entry->nexthop_neighs_list_node, 3977 &mlxsw_sp->router->nexthop_neighs_list); 3978 3979 nh->neigh_entry = neigh_entry; 3980 list_add_tail(&nh->neigh_list_node, &neigh_entry->nexthop_list); 3981 read_lock_bh(&n->lock); 3982 nud_state = n->nud_state; 3983 dead = n->dead; 3984 read_unlock_bh(&n->lock); 3985 __mlxsw_sp_nexthop_neigh_update(nh, !(nud_state & NUD_VALID && !dead)); 3986 3987 return 0; 3988 3989 err_neigh_entry_create: 3990 neigh_release(n); 3991 return err; 3992 } 3993 3994 static void mlxsw_sp_nexthop_neigh_fini(struct mlxsw_sp *mlxsw_sp, 3995 struct mlxsw_sp_nexthop *nh) 3996 { 3997 struct mlxsw_sp_neigh_entry *neigh_entry = nh->neigh_entry; 3998 struct neighbour *n; 3999 4000 if (!neigh_entry) 4001 return; 4002 n = neigh_entry->key.n; 4003 4004 __mlxsw_sp_nexthop_neigh_update(nh, true); 4005 list_del(&nh->neigh_list_node); 4006 nh->neigh_entry = NULL; 4007 4008 /* If that is the last nexthop connected to that neigh, remove from 4009 * nexthop_neighs_list 4010 */ 4011 if (list_empty(&neigh_entry->nexthop_list)) 4012 list_del(&neigh_entry->nexthop_neighs_list_node); 4013 4014 if (!neigh_entry->connected && list_empty(&neigh_entry->nexthop_list)) 4015 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry); 4016 4017 neigh_release(n); 4018 } 4019 4020 static bool mlxsw_sp_ipip_netdev_ul_up(struct net_device *ol_dev) 4021 { 4022 struct net_device *ul_dev; 4023 bool is_up; 4024 4025 rcu_read_lock(); 4026 ul_dev = __mlxsw_sp_ipip_netdev_ul_dev_get(ol_dev); 4027 is_up = ul_dev ? (ul_dev->flags & IFF_UP) : true; 4028 rcu_read_unlock(); 4029 4030 return is_up; 4031 } 4032 4033 static void mlxsw_sp_nexthop_ipip_init(struct mlxsw_sp *mlxsw_sp, 4034 struct mlxsw_sp_nexthop *nh, 4035 struct mlxsw_sp_ipip_entry *ipip_entry) 4036 { 4037 bool removing; 4038 4039 if (!nh->nhgi->gateway || nh->ipip_entry) 4040 return; 4041 4042 nh->ipip_entry = ipip_entry; 4043 removing = !mlxsw_sp_ipip_netdev_ul_up(ipip_entry->ol_dev); 4044 __mlxsw_sp_nexthop_neigh_update(nh, removing); 4045 mlxsw_sp_nexthop_rif_init(nh, &ipip_entry->ol_lb->common); 4046 } 4047 4048 static void mlxsw_sp_nexthop_ipip_fini(struct mlxsw_sp *mlxsw_sp, 4049 struct mlxsw_sp_nexthop *nh) 4050 { 4051 struct mlxsw_sp_ipip_entry *ipip_entry = nh->ipip_entry; 4052 4053 if (!ipip_entry) 4054 return; 4055 4056 __mlxsw_sp_nexthop_neigh_update(nh, true); 4057 nh->ipip_entry = NULL; 4058 } 4059 4060 static bool mlxsw_sp_nexthop4_ipip_type(const struct mlxsw_sp *mlxsw_sp, 4061 const struct fib_nh *fib_nh, 4062 enum mlxsw_sp_ipip_type *p_ipipt) 4063 { 4064 struct net_device *dev = fib_nh->fib_nh_dev; 4065 4066 return dev && 4067 fib_nh->nh_parent->fib_type == RTN_UNICAST && 4068 mlxsw_sp_netdev_ipip_type(mlxsw_sp, dev, p_ipipt); 4069 } 4070 4071 static int mlxsw_sp_nexthop_type_init(struct mlxsw_sp *mlxsw_sp, 4072 struct mlxsw_sp_nexthop *nh, 4073 const struct net_device *dev) 4074 { 4075 const struct mlxsw_sp_ipip_ops *ipip_ops; 4076 struct mlxsw_sp_ipip_entry *ipip_entry; 4077 struct mlxsw_sp_rif *rif; 4078 int err; 4079 4080 ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, dev); 4081 if (ipip_entry) { 4082 ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt]; 4083 if (ipip_ops->can_offload(mlxsw_sp, dev)) { 4084 nh->type = MLXSW_SP_NEXTHOP_TYPE_IPIP; 4085 mlxsw_sp_nexthop_ipip_init(mlxsw_sp, nh, ipip_entry); 4086 return 0; 4087 } 4088 } 4089 4090 nh->type = MLXSW_SP_NEXTHOP_TYPE_ETH; 4091 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev); 4092 if (!rif) 4093 return 0; 4094 4095 mlxsw_sp_nexthop_rif_init(nh, rif); 4096 err = mlxsw_sp_nexthop_neigh_init(mlxsw_sp, nh); 4097 if (err) 4098 goto err_neigh_init; 4099 4100 return 0; 4101 4102 err_neigh_init: 4103 mlxsw_sp_nexthop_rif_fini(nh); 4104 return err; 4105 } 4106 4107 static void mlxsw_sp_nexthop_type_fini(struct mlxsw_sp *mlxsw_sp, 4108 struct mlxsw_sp_nexthop *nh) 4109 { 4110 switch (nh->type) { 4111 case MLXSW_SP_NEXTHOP_TYPE_ETH: 4112 mlxsw_sp_nexthop_neigh_fini(mlxsw_sp, nh); 4113 mlxsw_sp_nexthop_rif_fini(nh); 4114 break; 4115 case MLXSW_SP_NEXTHOP_TYPE_IPIP: 4116 mlxsw_sp_nexthop_rif_fini(nh); 4117 mlxsw_sp_nexthop_ipip_fini(mlxsw_sp, nh); 4118 break; 4119 } 4120 } 4121 4122 static int mlxsw_sp_nexthop4_init(struct mlxsw_sp *mlxsw_sp, 4123 struct mlxsw_sp_nexthop_group *nh_grp, 4124 struct mlxsw_sp_nexthop *nh, 4125 struct fib_nh *fib_nh) 4126 { 4127 struct net_device *dev = fib_nh->fib_nh_dev; 4128 struct in_device *in_dev; 4129 int err; 4130 4131 nh->nhgi = nh_grp->nhgi; 4132 nh->key.fib_nh = fib_nh; 4133 #ifdef CONFIG_IP_ROUTE_MULTIPATH 4134 nh->nh_weight = fib_nh->fib_nh_weight; 4135 #else 4136 nh->nh_weight = 1; 4137 #endif 4138 memcpy(&nh->gw_addr, &fib_nh->fib_nh_gw4, sizeof(fib_nh->fib_nh_gw4)); 4139 nh->neigh_tbl = &arp_tbl; 4140 err = mlxsw_sp_nexthop_insert(mlxsw_sp, nh); 4141 if (err) 4142 return err; 4143 4144 mlxsw_sp_nexthop_counter_alloc(mlxsw_sp, nh); 4145 list_add_tail(&nh->router_list_node, &mlxsw_sp->router->nexthop_list); 4146 4147 if (!dev) 4148 return 0; 4149 nh->ifindex = dev->ifindex; 4150 4151 rcu_read_lock(); 4152 in_dev = __in_dev_get_rcu(dev); 4153 if (in_dev && IN_DEV_IGNORE_ROUTES_WITH_LINKDOWN(in_dev) && 4154 fib_nh->fib_nh_flags & RTNH_F_LINKDOWN) { 4155 rcu_read_unlock(); 4156 return 0; 4157 } 4158 rcu_read_unlock(); 4159 4160 err = mlxsw_sp_nexthop_type_init(mlxsw_sp, nh, dev); 4161 if (err) 4162 goto err_nexthop_neigh_init; 4163 4164 return 0; 4165 4166 err_nexthop_neigh_init: 4167 mlxsw_sp_nexthop_remove(mlxsw_sp, nh); 4168 return err; 4169 } 4170 4171 static void mlxsw_sp_nexthop4_fini(struct mlxsw_sp *mlxsw_sp, 4172 struct mlxsw_sp_nexthop *nh) 4173 { 4174 mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh); 4175 list_del(&nh->router_list_node); 4176 mlxsw_sp_nexthop_counter_free(mlxsw_sp, nh); 4177 mlxsw_sp_nexthop_remove(mlxsw_sp, nh); 4178 } 4179 4180 static void mlxsw_sp_nexthop4_event(struct mlxsw_sp *mlxsw_sp, 4181 unsigned long event, struct fib_nh *fib_nh) 4182 { 4183 struct mlxsw_sp_nexthop_key key; 4184 struct mlxsw_sp_nexthop *nh; 4185 4186 if (mlxsw_sp->router->aborted) 4187 return; 4188 4189 key.fib_nh = fib_nh; 4190 nh = mlxsw_sp_nexthop_lookup(mlxsw_sp, key); 4191 if (!nh) 4192 return; 4193 4194 switch (event) { 4195 case FIB_EVENT_NH_ADD: 4196 mlxsw_sp_nexthop_type_init(mlxsw_sp, nh, fib_nh->fib_nh_dev); 4197 break; 4198 case FIB_EVENT_NH_DEL: 4199 mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh); 4200 break; 4201 } 4202 4203 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nhgi->nh_grp); 4204 } 4205 4206 static void mlxsw_sp_nexthop_rif_update(struct mlxsw_sp *mlxsw_sp, 4207 struct mlxsw_sp_rif *rif) 4208 { 4209 struct mlxsw_sp_nexthop *nh; 4210 bool removing; 4211 4212 list_for_each_entry(nh, &rif->nexthop_list, rif_list_node) { 4213 switch (nh->type) { 4214 case MLXSW_SP_NEXTHOP_TYPE_ETH: 4215 removing = false; 4216 break; 4217 case MLXSW_SP_NEXTHOP_TYPE_IPIP: 4218 removing = !mlxsw_sp_ipip_netdev_ul_up(rif->dev); 4219 break; 4220 default: 4221 WARN_ON(1); 4222 continue; 4223 } 4224 4225 __mlxsw_sp_nexthop_neigh_update(nh, removing); 4226 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nhgi->nh_grp); 4227 } 4228 } 4229 4230 static void mlxsw_sp_nexthop_rif_migrate(struct mlxsw_sp *mlxsw_sp, 4231 struct mlxsw_sp_rif *old_rif, 4232 struct mlxsw_sp_rif *new_rif) 4233 { 4234 struct mlxsw_sp_nexthop *nh; 4235 4236 list_splice_init(&old_rif->nexthop_list, &new_rif->nexthop_list); 4237 list_for_each_entry(nh, &new_rif->nexthop_list, rif_list_node) 4238 nh->rif = new_rif; 4239 mlxsw_sp_nexthop_rif_update(mlxsw_sp, new_rif); 4240 } 4241 4242 static void mlxsw_sp_nexthop_rif_gone_sync(struct mlxsw_sp *mlxsw_sp, 4243 struct mlxsw_sp_rif *rif) 4244 { 4245 struct mlxsw_sp_nexthop *nh, *tmp; 4246 4247 list_for_each_entry_safe(nh, tmp, &rif->nexthop_list, rif_list_node) { 4248 mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh); 4249 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nhgi->nh_grp); 4250 } 4251 } 4252 4253 static int 4254 mlxsw_sp_nexthop_obj_single_validate(struct mlxsw_sp *mlxsw_sp, 4255 const struct nh_notifier_single_info *nh, 4256 struct netlink_ext_ack *extack) 4257 { 4258 int err = -EINVAL; 4259 4260 if (nh->is_fdb) 4261 NL_SET_ERR_MSG_MOD(extack, "FDB nexthops are not supported"); 4262 else if (nh->has_encap) 4263 NL_SET_ERR_MSG_MOD(extack, "Encapsulating nexthops are not supported"); 4264 else 4265 err = 0; 4266 4267 return err; 4268 } 4269 4270 static int 4271 mlxsw_sp_nexthop_obj_group_validate(struct mlxsw_sp *mlxsw_sp, 4272 const struct nh_notifier_grp_info *nh_grp, 4273 struct netlink_ext_ack *extack) 4274 { 4275 int i; 4276 4277 if (nh_grp->is_fdb) { 4278 NL_SET_ERR_MSG_MOD(extack, "FDB nexthop groups are not supported"); 4279 return -EINVAL; 4280 } 4281 4282 for (i = 0; i < nh_grp->num_nh; i++) { 4283 const struct nh_notifier_single_info *nh; 4284 int err; 4285 4286 nh = &nh_grp->nh_entries[i].nh; 4287 err = mlxsw_sp_nexthop_obj_single_validate(mlxsw_sp, nh, 4288 extack); 4289 if (err) 4290 return err; 4291 4292 /* Device only nexthops with an IPIP device are programmed as 4293 * encapsulating adjacency entries. 4294 */ 4295 if (!nh->gw_family && !nh->is_reject && 4296 !mlxsw_sp_netdev_ipip_type(mlxsw_sp, nh->dev, NULL)) { 4297 NL_SET_ERR_MSG_MOD(extack, "Nexthop group entry does not have a gateway"); 4298 return -EINVAL; 4299 } 4300 } 4301 4302 return 0; 4303 } 4304 4305 static int mlxsw_sp_nexthop_obj_validate(struct mlxsw_sp *mlxsw_sp, 4306 unsigned long event, 4307 struct nh_notifier_info *info) 4308 { 4309 if (event != NEXTHOP_EVENT_REPLACE) 4310 return 0; 4311 4312 switch (info->type) { 4313 case NH_NOTIFIER_INFO_TYPE_SINGLE: 4314 return mlxsw_sp_nexthop_obj_single_validate(mlxsw_sp, info->nh, 4315 info->extack); 4316 case NH_NOTIFIER_INFO_TYPE_GRP: 4317 return mlxsw_sp_nexthop_obj_group_validate(mlxsw_sp, 4318 info->nh_grp, 4319 info->extack); 4320 default: 4321 NL_SET_ERR_MSG_MOD(info->extack, "Unsupported nexthop type"); 4322 return -EOPNOTSUPP; 4323 } 4324 } 4325 4326 static bool mlxsw_sp_nexthop_obj_is_gateway(struct mlxsw_sp *mlxsw_sp, 4327 const struct nh_notifier_info *info) 4328 { 4329 const struct net_device *dev; 4330 4331 switch (info->type) { 4332 case NH_NOTIFIER_INFO_TYPE_SINGLE: 4333 dev = info->nh->dev; 4334 return info->nh->gw_family || info->nh->is_reject || 4335 mlxsw_sp_netdev_ipip_type(mlxsw_sp, dev, NULL); 4336 case NH_NOTIFIER_INFO_TYPE_GRP: 4337 /* Already validated earlier. */ 4338 return true; 4339 default: 4340 return false; 4341 } 4342 } 4343 4344 static void mlxsw_sp_nexthop_obj_blackhole_init(struct mlxsw_sp *mlxsw_sp, 4345 struct mlxsw_sp_nexthop *nh) 4346 { 4347 u16 lb_rif_index = mlxsw_sp->router->lb_rif_index; 4348 4349 nh->discard = 1; 4350 nh->should_offload = 1; 4351 /* While nexthops that discard packets do not forward packets 4352 * via an egress RIF, they still need to be programmed using a 4353 * valid RIF, so use the loopback RIF created during init. 4354 */ 4355 nh->rif = mlxsw_sp->router->rifs[lb_rif_index]; 4356 } 4357 4358 static void mlxsw_sp_nexthop_obj_blackhole_fini(struct mlxsw_sp *mlxsw_sp, 4359 struct mlxsw_sp_nexthop *nh) 4360 { 4361 nh->rif = NULL; 4362 nh->should_offload = 0; 4363 } 4364 4365 static int 4366 mlxsw_sp_nexthop_obj_init(struct mlxsw_sp *mlxsw_sp, 4367 struct mlxsw_sp_nexthop_group *nh_grp, 4368 struct mlxsw_sp_nexthop *nh, 4369 struct nh_notifier_single_info *nh_obj, int weight) 4370 { 4371 struct net_device *dev = nh_obj->dev; 4372 int err; 4373 4374 nh->nhgi = nh_grp->nhgi; 4375 nh->nh_weight = weight; 4376 4377 switch (nh_obj->gw_family) { 4378 case AF_INET: 4379 memcpy(&nh->gw_addr, &nh_obj->ipv4, sizeof(nh_obj->ipv4)); 4380 nh->neigh_tbl = &arp_tbl; 4381 break; 4382 case AF_INET6: 4383 memcpy(&nh->gw_addr, &nh_obj->ipv6, sizeof(nh_obj->ipv6)); 4384 #if IS_ENABLED(CONFIG_IPV6) 4385 nh->neigh_tbl = &nd_tbl; 4386 #endif 4387 break; 4388 } 4389 4390 mlxsw_sp_nexthop_counter_alloc(mlxsw_sp, nh); 4391 list_add_tail(&nh->router_list_node, &mlxsw_sp->router->nexthop_list); 4392 nh->ifindex = dev->ifindex; 4393 4394 err = mlxsw_sp_nexthop_type_init(mlxsw_sp, nh, dev); 4395 if (err) 4396 goto err_type_init; 4397 4398 if (nh_obj->is_reject) 4399 mlxsw_sp_nexthop_obj_blackhole_init(mlxsw_sp, nh); 4400 4401 return 0; 4402 4403 err_type_init: 4404 list_del(&nh->router_list_node); 4405 mlxsw_sp_nexthop_counter_free(mlxsw_sp, nh); 4406 return err; 4407 } 4408 4409 static void mlxsw_sp_nexthop_obj_fini(struct mlxsw_sp *mlxsw_sp, 4410 struct mlxsw_sp_nexthop *nh) 4411 { 4412 if (nh->discard) 4413 mlxsw_sp_nexthop_obj_blackhole_fini(mlxsw_sp, nh); 4414 mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh); 4415 list_del(&nh->router_list_node); 4416 mlxsw_sp_nexthop_counter_free(mlxsw_sp, nh); 4417 } 4418 4419 static int 4420 mlxsw_sp_nexthop_obj_group_info_init(struct mlxsw_sp *mlxsw_sp, 4421 struct mlxsw_sp_nexthop_group *nh_grp, 4422 struct nh_notifier_info *info) 4423 { 4424 struct mlxsw_sp_nexthop_group_info *nhgi; 4425 struct mlxsw_sp_nexthop *nh; 4426 unsigned int nhs; 4427 int err, i; 4428 4429 switch (info->type) { 4430 case NH_NOTIFIER_INFO_TYPE_SINGLE: 4431 nhs = 1; 4432 break; 4433 case NH_NOTIFIER_INFO_TYPE_GRP: 4434 nhs = info->nh_grp->num_nh; 4435 break; 4436 default: 4437 return -EINVAL; 4438 } 4439 4440 nhgi = kzalloc(struct_size(nhgi, nexthops, nhs), GFP_KERNEL); 4441 if (!nhgi) 4442 return -ENOMEM; 4443 nh_grp->nhgi = nhgi; 4444 nhgi->nh_grp = nh_grp; 4445 nhgi->gateway = mlxsw_sp_nexthop_obj_is_gateway(mlxsw_sp, info); 4446 nhgi->count = nhs; 4447 for (i = 0; i < nhgi->count; i++) { 4448 struct nh_notifier_single_info *nh_obj; 4449 int weight; 4450 4451 nh = &nhgi->nexthops[i]; 4452 switch (info->type) { 4453 case NH_NOTIFIER_INFO_TYPE_SINGLE: 4454 nh_obj = info->nh; 4455 weight = 1; 4456 break; 4457 case NH_NOTIFIER_INFO_TYPE_GRP: 4458 nh_obj = &info->nh_grp->nh_entries[i].nh; 4459 weight = info->nh_grp->nh_entries[i].weight; 4460 break; 4461 default: 4462 err = -EINVAL; 4463 goto err_nexthop_obj_init; 4464 } 4465 err = mlxsw_sp_nexthop_obj_init(mlxsw_sp, nh_grp, nh, nh_obj, 4466 weight); 4467 if (err) 4468 goto err_nexthop_obj_init; 4469 } 4470 err = mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp); 4471 if (err) { 4472 NL_SET_ERR_MSG_MOD(info->extack, "Failed to write adjacency entries to the device"); 4473 goto err_group_refresh; 4474 } 4475 4476 return 0; 4477 4478 err_group_refresh: 4479 i = nhgi->count; 4480 err_nexthop_obj_init: 4481 for (i--; i >= 0; i--) { 4482 nh = &nhgi->nexthops[i]; 4483 mlxsw_sp_nexthop_obj_fini(mlxsw_sp, nh); 4484 } 4485 kfree(nhgi); 4486 return err; 4487 } 4488 4489 static void 4490 mlxsw_sp_nexthop_obj_group_info_fini(struct mlxsw_sp *mlxsw_sp, 4491 struct mlxsw_sp_nexthop_group *nh_grp) 4492 { 4493 struct mlxsw_sp_nexthop_group_info *nhgi = nh_grp->nhgi; 4494 int i; 4495 4496 for (i = nhgi->count - 1; i >= 0; i--) { 4497 struct mlxsw_sp_nexthop *nh = &nhgi->nexthops[i]; 4498 4499 mlxsw_sp_nexthop_obj_fini(mlxsw_sp, nh); 4500 } 4501 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp); 4502 WARN_ON_ONCE(nhgi->adj_index_valid); 4503 kfree(nhgi); 4504 } 4505 4506 static struct mlxsw_sp_nexthop_group * 4507 mlxsw_sp_nexthop_obj_group_create(struct mlxsw_sp *mlxsw_sp, 4508 struct nh_notifier_info *info) 4509 { 4510 struct mlxsw_sp_nexthop_group *nh_grp; 4511 int err; 4512 4513 nh_grp = kzalloc(sizeof(*nh_grp), GFP_KERNEL); 4514 if (!nh_grp) 4515 return ERR_PTR(-ENOMEM); 4516 INIT_LIST_HEAD(&nh_grp->vr_list); 4517 err = rhashtable_init(&nh_grp->vr_ht, 4518 &mlxsw_sp_nexthop_group_vr_ht_params); 4519 if (err) 4520 goto err_nexthop_group_vr_ht_init; 4521 INIT_LIST_HEAD(&nh_grp->fib_list); 4522 nh_grp->type = MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ; 4523 nh_grp->obj.id = info->id; 4524 4525 err = mlxsw_sp_nexthop_obj_group_info_init(mlxsw_sp, nh_grp, info); 4526 if (err) 4527 goto err_nexthop_group_info_init; 4528 4529 nh_grp->can_destroy = false; 4530 4531 return nh_grp; 4532 4533 err_nexthop_group_info_init: 4534 rhashtable_destroy(&nh_grp->vr_ht); 4535 err_nexthop_group_vr_ht_init: 4536 kfree(nh_grp); 4537 return ERR_PTR(err); 4538 } 4539 4540 static void 4541 mlxsw_sp_nexthop_obj_group_destroy(struct mlxsw_sp *mlxsw_sp, 4542 struct mlxsw_sp_nexthop_group *nh_grp) 4543 { 4544 if (!nh_grp->can_destroy) 4545 return; 4546 mlxsw_sp_nexthop_obj_group_info_fini(mlxsw_sp, nh_grp); 4547 WARN_ON_ONCE(!list_empty(&nh_grp->fib_list)); 4548 WARN_ON_ONCE(!list_empty(&nh_grp->vr_list)); 4549 rhashtable_destroy(&nh_grp->vr_ht); 4550 kfree(nh_grp); 4551 } 4552 4553 static struct mlxsw_sp_nexthop_group * 4554 mlxsw_sp_nexthop_obj_group_lookup(struct mlxsw_sp *mlxsw_sp, u32 id) 4555 { 4556 struct mlxsw_sp_nexthop_group_cmp_arg cmp_arg; 4557 4558 cmp_arg.type = MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ; 4559 cmp_arg.id = id; 4560 return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_group_ht, 4561 &cmp_arg, 4562 mlxsw_sp_nexthop_group_ht_params); 4563 } 4564 4565 static int mlxsw_sp_nexthop_obj_group_add(struct mlxsw_sp *mlxsw_sp, 4566 struct mlxsw_sp_nexthop_group *nh_grp) 4567 { 4568 return mlxsw_sp_nexthop_group_insert(mlxsw_sp, nh_grp); 4569 } 4570 4571 static int 4572 mlxsw_sp_nexthop_obj_group_replace(struct mlxsw_sp *mlxsw_sp, 4573 struct mlxsw_sp_nexthop_group *nh_grp, 4574 struct mlxsw_sp_nexthop_group *old_nh_grp, 4575 struct netlink_ext_ack *extack) 4576 { 4577 struct mlxsw_sp_nexthop_group_info *old_nhgi = old_nh_grp->nhgi; 4578 struct mlxsw_sp_nexthop_group_info *new_nhgi = nh_grp->nhgi; 4579 int err; 4580 4581 old_nh_grp->nhgi = new_nhgi; 4582 new_nhgi->nh_grp = old_nh_grp; 4583 nh_grp->nhgi = old_nhgi; 4584 old_nhgi->nh_grp = nh_grp; 4585 4586 if (old_nhgi->adj_index_valid && new_nhgi->adj_index_valid) { 4587 /* Both the old adjacency index and the new one are valid. 4588 * Routes are currently using the old one. Tell the device to 4589 * replace the old adjacency index with the new one. 4590 */ 4591 err = mlxsw_sp_adj_index_mass_update(mlxsw_sp, old_nh_grp, 4592 old_nhgi->adj_index, 4593 old_nhgi->ecmp_size); 4594 if (err) { 4595 NL_SET_ERR_MSG_MOD(extack, "Failed to replace old adjacency index with new one"); 4596 goto err_out; 4597 } 4598 } else if (old_nhgi->adj_index_valid && !new_nhgi->adj_index_valid) { 4599 /* The old adjacency index is valid, while the new one is not. 4600 * Iterate over all the routes using the group and change them 4601 * to trap packets to the CPU. 4602 */ 4603 err = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, old_nh_grp); 4604 if (err) { 4605 NL_SET_ERR_MSG_MOD(extack, "Failed to update routes to trap packets"); 4606 goto err_out; 4607 } 4608 } else if (!old_nhgi->adj_index_valid && new_nhgi->adj_index_valid) { 4609 /* The old adjacency index is invalid, while the new one is. 4610 * Iterate over all the routes using the group and change them 4611 * to forward packets using the new valid index. 4612 */ 4613 err = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, old_nh_grp); 4614 if (err) { 4615 NL_SET_ERR_MSG_MOD(extack, "Failed to update routes to forward packets"); 4616 goto err_out; 4617 } 4618 } 4619 4620 /* Make sure the flags are set / cleared based on the new nexthop group 4621 * information. 4622 */ 4623 mlxsw_sp_nexthop_obj_group_offload_refresh(mlxsw_sp, old_nh_grp); 4624 4625 /* At this point 'nh_grp' is just a shell that is not used by anyone 4626 * and its nexthop group info is the old info that was just replaced 4627 * with the new one. Remove it. 4628 */ 4629 nh_grp->can_destroy = true; 4630 mlxsw_sp_nexthop_obj_group_destroy(mlxsw_sp, nh_grp); 4631 4632 return 0; 4633 4634 err_out: 4635 old_nhgi->nh_grp = old_nh_grp; 4636 nh_grp->nhgi = new_nhgi; 4637 new_nhgi->nh_grp = nh_grp; 4638 old_nh_grp->nhgi = old_nhgi; 4639 return err; 4640 } 4641 4642 static int mlxsw_sp_nexthop_obj_new(struct mlxsw_sp *mlxsw_sp, 4643 struct nh_notifier_info *info) 4644 { 4645 struct mlxsw_sp_nexthop_group *nh_grp, *old_nh_grp; 4646 struct netlink_ext_ack *extack = info->extack; 4647 int err; 4648 4649 nh_grp = mlxsw_sp_nexthop_obj_group_create(mlxsw_sp, info); 4650 if (IS_ERR(nh_grp)) 4651 return PTR_ERR(nh_grp); 4652 4653 old_nh_grp = mlxsw_sp_nexthop_obj_group_lookup(mlxsw_sp, info->id); 4654 if (!old_nh_grp) 4655 err = mlxsw_sp_nexthop_obj_group_add(mlxsw_sp, nh_grp); 4656 else 4657 err = mlxsw_sp_nexthop_obj_group_replace(mlxsw_sp, nh_grp, 4658 old_nh_grp, extack); 4659 4660 if (err) { 4661 nh_grp->can_destroy = true; 4662 mlxsw_sp_nexthop_obj_group_destroy(mlxsw_sp, nh_grp); 4663 } 4664 4665 return err; 4666 } 4667 4668 static void mlxsw_sp_nexthop_obj_del(struct mlxsw_sp *mlxsw_sp, 4669 struct nh_notifier_info *info) 4670 { 4671 struct mlxsw_sp_nexthop_group *nh_grp; 4672 4673 nh_grp = mlxsw_sp_nexthop_obj_group_lookup(mlxsw_sp, info->id); 4674 if (!nh_grp) 4675 return; 4676 4677 nh_grp->can_destroy = true; 4678 mlxsw_sp_nexthop_group_remove(mlxsw_sp, nh_grp); 4679 4680 /* If the group still has routes using it, then defer the delete 4681 * operation until the last route using it is deleted. 4682 */ 4683 if (!list_empty(&nh_grp->fib_list)) 4684 return; 4685 mlxsw_sp_nexthop_obj_group_destroy(mlxsw_sp, nh_grp); 4686 } 4687 4688 static int mlxsw_sp_nexthop_obj_event(struct notifier_block *nb, 4689 unsigned long event, void *ptr) 4690 { 4691 struct nh_notifier_info *info = ptr; 4692 struct mlxsw_sp_router *router; 4693 int err = 0; 4694 4695 router = container_of(nb, struct mlxsw_sp_router, nexthop_nb); 4696 err = mlxsw_sp_nexthop_obj_validate(router->mlxsw_sp, event, info); 4697 if (err) 4698 goto out; 4699 4700 mutex_lock(&router->lock); 4701 4702 ASSERT_RTNL(); 4703 4704 switch (event) { 4705 case NEXTHOP_EVENT_REPLACE: 4706 err = mlxsw_sp_nexthop_obj_new(router->mlxsw_sp, info); 4707 break; 4708 case NEXTHOP_EVENT_DEL: 4709 mlxsw_sp_nexthop_obj_del(router->mlxsw_sp, info); 4710 break; 4711 default: 4712 break; 4713 } 4714 4715 mutex_unlock(&router->lock); 4716 4717 out: 4718 return notifier_from_errno(err); 4719 } 4720 4721 static bool mlxsw_sp_fi_is_gateway(const struct mlxsw_sp *mlxsw_sp, 4722 struct fib_info *fi) 4723 { 4724 const struct fib_nh *nh = fib_info_nh(fi, 0); 4725 4726 return nh->fib_nh_scope == RT_SCOPE_LINK || 4727 mlxsw_sp_nexthop4_ipip_type(mlxsw_sp, nh, NULL); 4728 } 4729 4730 static int 4731 mlxsw_sp_nexthop4_group_info_init(struct mlxsw_sp *mlxsw_sp, 4732 struct mlxsw_sp_nexthop_group *nh_grp) 4733 { 4734 unsigned int nhs = fib_info_num_path(nh_grp->ipv4.fi); 4735 struct mlxsw_sp_nexthop_group_info *nhgi; 4736 struct mlxsw_sp_nexthop *nh; 4737 int err, i; 4738 4739 nhgi = kzalloc(struct_size(nhgi, nexthops, nhs), GFP_KERNEL); 4740 if (!nhgi) 4741 return -ENOMEM; 4742 nh_grp->nhgi = nhgi; 4743 nhgi->nh_grp = nh_grp; 4744 nhgi->gateway = mlxsw_sp_fi_is_gateway(mlxsw_sp, nh_grp->ipv4.fi); 4745 nhgi->count = nhs; 4746 for (i = 0; i < nhgi->count; i++) { 4747 struct fib_nh *fib_nh; 4748 4749 nh = &nhgi->nexthops[i]; 4750 fib_nh = fib_info_nh(nh_grp->ipv4.fi, i); 4751 err = mlxsw_sp_nexthop4_init(mlxsw_sp, nh_grp, nh, fib_nh); 4752 if (err) 4753 goto err_nexthop4_init; 4754 } 4755 err = mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp); 4756 if (err) 4757 goto err_group_refresh; 4758 4759 return 0; 4760 4761 err_group_refresh: 4762 i = nhgi->count; 4763 err_nexthop4_init: 4764 for (i--; i >= 0; i--) { 4765 nh = &nhgi->nexthops[i]; 4766 mlxsw_sp_nexthop4_fini(mlxsw_sp, nh); 4767 } 4768 kfree(nhgi); 4769 return err; 4770 } 4771 4772 static void 4773 mlxsw_sp_nexthop4_group_info_fini(struct mlxsw_sp *mlxsw_sp, 4774 struct mlxsw_sp_nexthop_group *nh_grp) 4775 { 4776 struct mlxsw_sp_nexthop_group_info *nhgi = nh_grp->nhgi; 4777 int i; 4778 4779 for (i = nhgi->count - 1; i >= 0; i--) { 4780 struct mlxsw_sp_nexthop *nh = &nhgi->nexthops[i]; 4781 4782 mlxsw_sp_nexthop4_fini(mlxsw_sp, nh); 4783 } 4784 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp); 4785 WARN_ON_ONCE(nhgi->adj_index_valid); 4786 kfree(nhgi); 4787 } 4788 4789 static struct mlxsw_sp_nexthop_group * 4790 mlxsw_sp_nexthop4_group_create(struct mlxsw_sp *mlxsw_sp, struct fib_info *fi) 4791 { 4792 struct mlxsw_sp_nexthop_group *nh_grp; 4793 int err; 4794 4795 nh_grp = kzalloc(sizeof(*nh_grp), GFP_KERNEL); 4796 if (!nh_grp) 4797 return ERR_PTR(-ENOMEM); 4798 INIT_LIST_HEAD(&nh_grp->vr_list); 4799 err = rhashtable_init(&nh_grp->vr_ht, 4800 &mlxsw_sp_nexthop_group_vr_ht_params); 4801 if (err) 4802 goto err_nexthop_group_vr_ht_init; 4803 INIT_LIST_HEAD(&nh_grp->fib_list); 4804 nh_grp->type = MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4; 4805 nh_grp->ipv4.fi = fi; 4806 fib_info_hold(fi); 4807 4808 err = mlxsw_sp_nexthop4_group_info_init(mlxsw_sp, nh_grp); 4809 if (err) 4810 goto err_nexthop_group_info_init; 4811 4812 err = mlxsw_sp_nexthop_group_insert(mlxsw_sp, nh_grp); 4813 if (err) 4814 goto err_nexthop_group_insert; 4815 4816 nh_grp->can_destroy = true; 4817 4818 return nh_grp; 4819 4820 err_nexthop_group_insert: 4821 mlxsw_sp_nexthop4_group_info_fini(mlxsw_sp, nh_grp); 4822 err_nexthop_group_info_init: 4823 fib_info_put(fi); 4824 rhashtable_destroy(&nh_grp->vr_ht); 4825 err_nexthop_group_vr_ht_init: 4826 kfree(nh_grp); 4827 return ERR_PTR(err); 4828 } 4829 4830 static void 4831 mlxsw_sp_nexthop4_group_destroy(struct mlxsw_sp *mlxsw_sp, 4832 struct mlxsw_sp_nexthop_group *nh_grp) 4833 { 4834 if (!nh_grp->can_destroy) 4835 return; 4836 mlxsw_sp_nexthop_group_remove(mlxsw_sp, nh_grp); 4837 mlxsw_sp_nexthop4_group_info_fini(mlxsw_sp, nh_grp); 4838 fib_info_put(nh_grp->ipv4.fi); 4839 WARN_ON_ONCE(!list_empty(&nh_grp->vr_list)); 4840 rhashtable_destroy(&nh_grp->vr_ht); 4841 kfree(nh_grp); 4842 } 4843 4844 static int mlxsw_sp_nexthop4_group_get(struct mlxsw_sp *mlxsw_sp, 4845 struct mlxsw_sp_fib_entry *fib_entry, 4846 struct fib_info *fi) 4847 { 4848 struct mlxsw_sp_nexthop_group *nh_grp; 4849 4850 if (fi->nh) { 4851 nh_grp = mlxsw_sp_nexthop_obj_group_lookup(mlxsw_sp, 4852 fi->nh->id); 4853 if (WARN_ON_ONCE(!nh_grp)) 4854 return -EINVAL; 4855 goto out; 4856 } 4857 4858 nh_grp = mlxsw_sp_nexthop4_group_lookup(mlxsw_sp, fi); 4859 if (!nh_grp) { 4860 nh_grp = mlxsw_sp_nexthop4_group_create(mlxsw_sp, fi); 4861 if (IS_ERR(nh_grp)) 4862 return PTR_ERR(nh_grp); 4863 } 4864 out: 4865 list_add_tail(&fib_entry->nexthop_group_node, &nh_grp->fib_list); 4866 fib_entry->nh_group = nh_grp; 4867 return 0; 4868 } 4869 4870 static void mlxsw_sp_nexthop4_group_put(struct mlxsw_sp *mlxsw_sp, 4871 struct mlxsw_sp_fib_entry *fib_entry) 4872 { 4873 struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group; 4874 4875 list_del(&fib_entry->nexthop_group_node); 4876 if (!list_empty(&nh_grp->fib_list)) 4877 return; 4878 4879 if (nh_grp->type == MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ) { 4880 mlxsw_sp_nexthop_obj_group_destroy(mlxsw_sp, nh_grp); 4881 return; 4882 } 4883 4884 mlxsw_sp_nexthop4_group_destroy(mlxsw_sp, nh_grp); 4885 } 4886 4887 static bool 4888 mlxsw_sp_fib4_entry_should_offload(const struct mlxsw_sp_fib_entry *fib_entry) 4889 { 4890 struct mlxsw_sp_fib4_entry *fib4_entry; 4891 4892 fib4_entry = container_of(fib_entry, struct mlxsw_sp_fib4_entry, 4893 common); 4894 return !fib4_entry->tos; 4895 } 4896 4897 static bool 4898 mlxsw_sp_fib_entry_should_offload(const struct mlxsw_sp_fib_entry *fib_entry) 4899 { 4900 struct mlxsw_sp_nexthop_group *nh_group = fib_entry->nh_group; 4901 4902 switch (fib_entry->fib_node->fib->proto) { 4903 case MLXSW_SP_L3_PROTO_IPV4: 4904 if (!mlxsw_sp_fib4_entry_should_offload(fib_entry)) 4905 return false; 4906 break; 4907 case MLXSW_SP_L3_PROTO_IPV6: 4908 break; 4909 } 4910 4911 switch (fib_entry->type) { 4912 case MLXSW_SP_FIB_ENTRY_TYPE_REMOTE: 4913 return !!nh_group->nhgi->adj_index_valid; 4914 case MLXSW_SP_FIB_ENTRY_TYPE_LOCAL: 4915 return !!nh_group->nhgi->nh_rif; 4916 case MLXSW_SP_FIB_ENTRY_TYPE_BLACKHOLE: 4917 case MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP: 4918 case MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP: 4919 return true; 4920 default: 4921 return false; 4922 } 4923 } 4924 4925 static struct mlxsw_sp_nexthop * 4926 mlxsw_sp_rt6_nexthop(struct mlxsw_sp_nexthop_group *nh_grp, 4927 const struct mlxsw_sp_rt6 *mlxsw_sp_rt6) 4928 { 4929 int i; 4930 4931 for (i = 0; i < nh_grp->nhgi->count; i++) { 4932 struct mlxsw_sp_nexthop *nh = &nh_grp->nhgi->nexthops[i]; 4933 struct fib6_info *rt = mlxsw_sp_rt6->rt; 4934 4935 if (nh->rif && nh->rif->dev == rt->fib6_nh->fib_nh_dev && 4936 ipv6_addr_equal((const struct in6_addr *) &nh->gw_addr, 4937 &rt->fib6_nh->fib_nh_gw6)) 4938 return nh; 4939 continue; 4940 } 4941 4942 return NULL; 4943 } 4944 4945 static void 4946 mlxsw_sp_fib4_offload_failed_flag_set(struct mlxsw_sp *mlxsw_sp, 4947 struct fib_entry_notifier_info *fen_info) 4948 { 4949 u32 *p_dst = (u32 *) &fen_info->dst; 4950 struct fib_rt_info fri; 4951 4952 fri.fi = fen_info->fi; 4953 fri.tb_id = fen_info->tb_id; 4954 fri.dst = cpu_to_be32(*p_dst); 4955 fri.dst_len = fen_info->dst_len; 4956 fri.tos = fen_info->tos; 4957 fri.type = fen_info->type; 4958 fri.offload = false; 4959 fri.trap = false; 4960 fri.offload_failed = true; 4961 fib_alias_hw_flags_set(mlxsw_sp_net(mlxsw_sp), &fri); 4962 } 4963 4964 static void 4965 mlxsw_sp_fib4_entry_hw_flags_set(struct mlxsw_sp *mlxsw_sp, 4966 struct mlxsw_sp_fib_entry *fib_entry) 4967 { 4968 u32 *p_dst = (u32 *) fib_entry->fib_node->key.addr; 4969 int dst_len = fib_entry->fib_node->key.prefix_len; 4970 struct mlxsw_sp_fib4_entry *fib4_entry; 4971 struct fib_rt_info fri; 4972 bool should_offload; 4973 4974 should_offload = mlxsw_sp_fib_entry_should_offload(fib_entry); 4975 fib4_entry = container_of(fib_entry, struct mlxsw_sp_fib4_entry, 4976 common); 4977 fri.fi = fib4_entry->fi; 4978 fri.tb_id = fib4_entry->tb_id; 4979 fri.dst = cpu_to_be32(*p_dst); 4980 fri.dst_len = dst_len; 4981 fri.tos = fib4_entry->tos; 4982 fri.type = fib4_entry->type; 4983 fri.offload = should_offload; 4984 fri.trap = !should_offload; 4985 fri.offload_failed = false; 4986 fib_alias_hw_flags_set(mlxsw_sp_net(mlxsw_sp), &fri); 4987 } 4988 4989 static void 4990 mlxsw_sp_fib4_entry_hw_flags_clear(struct mlxsw_sp *mlxsw_sp, 4991 struct mlxsw_sp_fib_entry *fib_entry) 4992 { 4993 u32 *p_dst = (u32 *) fib_entry->fib_node->key.addr; 4994 int dst_len = fib_entry->fib_node->key.prefix_len; 4995 struct mlxsw_sp_fib4_entry *fib4_entry; 4996 struct fib_rt_info fri; 4997 4998 fib4_entry = container_of(fib_entry, struct mlxsw_sp_fib4_entry, 4999 common); 5000 fri.fi = fib4_entry->fi; 5001 fri.tb_id = fib4_entry->tb_id; 5002 fri.dst = cpu_to_be32(*p_dst); 5003 fri.dst_len = dst_len; 5004 fri.tos = fib4_entry->tos; 5005 fri.type = fib4_entry->type; 5006 fri.offload = false; 5007 fri.trap = false; 5008 fri.offload_failed = false; 5009 fib_alias_hw_flags_set(mlxsw_sp_net(mlxsw_sp), &fri); 5010 } 5011 5012 #if IS_ENABLED(CONFIG_IPV6) 5013 static void 5014 mlxsw_sp_fib6_offload_failed_flag_set(struct mlxsw_sp *mlxsw_sp, 5015 struct fib6_info **rt_arr, 5016 unsigned int nrt6) 5017 { 5018 int i; 5019 5020 /* In IPv6 a multipath route is represented using multiple routes, so 5021 * we need to set the flags on all of them. 5022 */ 5023 for (i = 0; i < nrt6; i++) 5024 fib6_info_hw_flags_set(mlxsw_sp_net(mlxsw_sp), rt_arr[i], 5025 false, false, true); 5026 } 5027 #else 5028 static void 5029 mlxsw_sp_fib6_offload_failed_flag_set(struct mlxsw_sp *mlxsw_sp, 5030 struct fib6_info **rt_arr, 5031 unsigned int nrt6) 5032 { 5033 } 5034 #endif 5035 5036 #if IS_ENABLED(CONFIG_IPV6) 5037 static void 5038 mlxsw_sp_fib6_entry_hw_flags_set(struct mlxsw_sp *mlxsw_sp, 5039 struct mlxsw_sp_fib_entry *fib_entry) 5040 { 5041 struct mlxsw_sp_fib6_entry *fib6_entry; 5042 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 5043 bool should_offload; 5044 5045 should_offload = mlxsw_sp_fib_entry_should_offload(fib_entry); 5046 5047 /* In IPv6 a multipath route is represented using multiple routes, so 5048 * we need to set the flags on all of them. 5049 */ 5050 fib6_entry = container_of(fib_entry, struct mlxsw_sp_fib6_entry, 5051 common); 5052 list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) 5053 fib6_info_hw_flags_set(mlxsw_sp_net(mlxsw_sp), mlxsw_sp_rt6->rt, 5054 should_offload, !should_offload, false); 5055 } 5056 #else 5057 static void 5058 mlxsw_sp_fib6_entry_hw_flags_set(struct mlxsw_sp *mlxsw_sp, 5059 struct mlxsw_sp_fib_entry *fib_entry) 5060 { 5061 } 5062 #endif 5063 5064 #if IS_ENABLED(CONFIG_IPV6) 5065 static void 5066 mlxsw_sp_fib6_entry_hw_flags_clear(struct mlxsw_sp *mlxsw_sp, 5067 struct mlxsw_sp_fib_entry *fib_entry) 5068 { 5069 struct mlxsw_sp_fib6_entry *fib6_entry; 5070 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 5071 5072 fib6_entry = container_of(fib_entry, struct mlxsw_sp_fib6_entry, 5073 common); 5074 list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) 5075 fib6_info_hw_flags_set(mlxsw_sp_net(mlxsw_sp), mlxsw_sp_rt6->rt, 5076 false, false, false); 5077 } 5078 #else 5079 static void 5080 mlxsw_sp_fib6_entry_hw_flags_clear(struct mlxsw_sp *mlxsw_sp, 5081 struct mlxsw_sp_fib_entry *fib_entry) 5082 { 5083 } 5084 #endif 5085 5086 static void 5087 mlxsw_sp_fib_entry_hw_flags_set(struct mlxsw_sp *mlxsw_sp, 5088 struct mlxsw_sp_fib_entry *fib_entry) 5089 { 5090 switch (fib_entry->fib_node->fib->proto) { 5091 case MLXSW_SP_L3_PROTO_IPV4: 5092 mlxsw_sp_fib4_entry_hw_flags_set(mlxsw_sp, fib_entry); 5093 break; 5094 case MLXSW_SP_L3_PROTO_IPV6: 5095 mlxsw_sp_fib6_entry_hw_flags_set(mlxsw_sp, fib_entry); 5096 break; 5097 } 5098 } 5099 5100 static void 5101 mlxsw_sp_fib_entry_hw_flags_clear(struct mlxsw_sp *mlxsw_sp, 5102 struct mlxsw_sp_fib_entry *fib_entry) 5103 { 5104 switch (fib_entry->fib_node->fib->proto) { 5105 case MLXSW_SP_L3_PROTO_IPV4: 5106 mlxsw_sp_fib4_entry_hw_flags_clear(mlxsw_sp, fib_entry); 5107 break; 5108 case MLXSW_SP_L3_PROTO_IPV6: 5109 mlxsw_sp_fib6_entry_hw_flags_clear(mlxsw_sp, fib_entry); 5110 break; 5111 } 5112 } 5113 5114 static void 5115 mlxsw_sp_fib_entry_hw_flags_refresh(struct mlxsw_sp *mlxsw_sp, 5116 struct mlxsw_sp_fib_entry *fib_entry, 5117 enum mlxsw_sp_fib_entry_op op) 5118 { 5119 switch (op) { 5120 case MLXSW_SP_FIB_ENTRY_OP_WRITE: 5121 case MLXSW_SP_FIB_ENTRY_OP_UPDATE: 5122 mlxsw_sp_fib_entry_hw_flags_set(mlxsw_sp, fib_entry); 5123 break; 5124 case MLXSW_SP_FIB_ENTRY_OP_DELETE: 5125 mlxsw_sp_fib_entry_hw_flags_clear(mlxsw_sp, fib_entry); 5126 break; 5127 default: 5128 break; 5129 } 5130 } 5131 5132 struct mlxsw_sp_fib_entry_op_ctx_basic { 5133 char ralue_pl[MLXSW_REG_RALUE_LEN]; 5134 }; 5135 5136 static void 5137 mlxsw_sp_router_ll_basic_fib_entry_pack(struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 5138 enum mlxsw_sp_l3proto proto, 5139 enum mlxsw_sp_fib_entry_op op, 5140 u16 virtual_router, u8 prefix_len, 5141 unsigned char *addr, 5142 struct mlxsw_sp_fib_entry_priv *priv) 5143 { 5144 struct mlxsw_sp_fib_entry_op_ctx_basic *op_ctx_basic = (void *) op_ctx->ll_priv; 5145 enum mlxsw_reg_ralxx_protocol ralxx_proto; 5146 char *ralue_pl = op_ctx_basic->ralue_pl; 5147 enum mlxsw_reg_ralue_op ralue_op; 5148 5149 ralxx_proto = (enum mlxsw_reg_ralxx_protocol) proto; 5150 5151 switch (op) { 5152 case MLXSW_SP_FIB_ENTRY_OP_WRITE: 5153 case MLXSW_SP_FIB_ENTRY_OP_UPDATE: 5154 ralue_op = MLXSW_REG_RALUE_OP_WRITE_WRITE; 5155 break; 5156 case MLXSW_SP_FIB_ENTRY_OP_DELETE: 5157 ralue_op = MLXSW_REG_RALUE_OP_WRITE_DELETE; 5158 break; 5159 default: 5160 WARN_ON_ONCE(1); 5161 return; 5162 } 5163 5164 switch (proto) { 5165 case MLXSW_SP_L3_PROTO_IPV4: 5166 mlxsw_reg_ralue_pack4(ralue_pl, ralxx_proto, ralue_op, 5167 virtual_router, prefix_len, (u32 *) addr); 5168 break; 5169 case MLXSW_SP_L3_PROTO_IPV6: 5170 mlxsw_reg_ralue_pack6(ralue_pl, ralxx_proto, ralue_op, 5171 virtual_router, prefix_len, addr); 5172 break; 5173 } 5174 } 5175 5176 static void 5177 mlxsw_sp_router_ll_basic_fib_entry_act_remote_pack(struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 5178 enum mlxsw_reg_ralue_trap_action trap_action, 5179 u16 trap_id, u32 adjacency_index, u16 ecmp_size) 5180 { 5181 struct mlxsw_sp_fib_entry_op_ctx_basic *op_ctx_basic = (void *) op_ctx->ll_priv; 5182 5183 mlxsw_reg_ralue_act_remote_pack(op_ctx_basic->ralue_pl, trap_action, 5184 trap_id, adjacency_index, ecmp_size); 5185 } 5186 5187 static void 5188 mlxsw_sp_router_ll_basic_fib_entry_act_local_pack(struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 5189 enum mlxsw_reg_ralue_trap_action trap_action, 5190 u16 trap_id, u16 local_erif) 5191 { 5192 struct mlxsw_sp_fib_entry_op_ctx_basic *op_ctx_basic = (void *) op_ctx->ll_priv; 5193 5194 mlxsw_reg_ralue_act_local_pack(op_ctx_basic->ralue_pl, trap_action, 5195 trap_id, local_erif); 5196 } 5197 5198 static void 5199 mlxsw_sp_router_ll_basic_fib_entry_act_ip2me_pack(struct mlxsw_sp_fib_entry_op_ctx *op_ctx) 5200 { 5201 struct mlxsw_sp_fib_entry_op_ctx_basic *op_ctx_basic = (void *) op_ctx->ll_priv; 5202 5203 mlxsw_reg_ralue_act_ip2me_pack(op_ctx_basic->ralue_pl); 5204 } 5205 5206 static void 5207 mlxsw_sp_router_ll_basic_fib_entry_act_ip2me_tun_pack(struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 5208 u32 tunnel_ptr) 5209 { 5210 struct mlxsw_sp_fib_entry_op_ctx_basic *op_ctx_basic = (void *) op_ctx->ll_priv; 5211 5212 mlxsw_reg_ralue_act_ip2me_tun_pack(op_ctx_basic->ralue_pl, tunnel_ptr); 5213 } 5214 5215 static int 5216 mlxsw_sp_router_ll_basic_fib_entry_commit(struct mlxsw_sp *mlxsw_sp, 5217 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 5218 bool *postponed_for_bulk) 5219 { 5220 struct mlxsw_sp_fib_entry_op_ctx_basic *op_ctx_basic = (void *) op_ctx->ll_priv; 5221 5222 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), 5223 op_ctx_basic->ralue_pl); 5224 } 5225 5226 static bool 5227 mlxsw_sp_router_ll_basic_fib_entry_is_committed(struct mlxsw_sp_fib_entry_priv *priv) 5228 { 5229 return true; 5230 } 5231 5232 static void mlxsw_sp_fib_entry_pack(struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 5233 struct mlxsw_sp_fib_entry *fib_entry, 5234 enum mlxsw_sp_fib_entry_op op) 5235 { 5236 struct mlxsw_sp_fib *fib = fib_entry->fib_node->fib; 5237 5238 mlxsw_sp_fib_entry_op_ctx_priv_hold(op_ctx, fib_entry->priv); 5239 fib->ll_ops->fib_entry_pack(op_ctx, fib->proto, op, fib->vr->id, 5240 fib_entry->fib_node->key.prefix_len, 5241 fib_entry->fib_node->key.addr, 5242 fib_entry->priv); 5243 } 5244 5245 static int mlxsw_sp_fib_entry_commit(struct mlxsw_sp *mlxsw_sp, 5246 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 5247 const struct mlxsw_sp_router_ll_ops *ll_ops) 5248 { 5249 bool postponed_for_bulk = false; 5250 int err; 5251 5252 err = ll_ops->fib_entry_commit(mlxsw_sp, op_ctx, &postponed_for_bulk); 5253 if (!postponed_for_bulk) 5254 mlxsw_sp_fib_entry_op_ctx_priv_put_all(op_ctx); 5255 return err; 5256 } 5257 5258 static int mlxsw_sp_adj_discard_write(struct mlxsw_sp *mlxsw_sp) 5259 { 5260 enum mlxsw_reg_ratr_trap_action trap_action; 5261 char ratr_pl[MLXSW_REG_RATR_LEN]; 5262 int err; 5263 5264 if (mlxsw_sp->router->adj_discard_index_valid) 5265 return 0; 5266 5267 err = mlxsw_sp_kvdl_alloc(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 1, 5268 &mlxsw_sp->router->adj_discard_index); 5269 if (err) 5270 return err; 5271 5272 trap_action = MLXSW_REG_RATR_TRAP_ACTION_TRAP; 5273 mlxsw_reg_ratr_pack(ratr_pl, MLXSW_REG_RATR_OP_WRITE_WRITE_ENTRY, true, 5274 MLXSW_REG_RATR_TYPE_ETHERNET, 5275 mlxsw_sp->router->adj_discard_index, 5276 mlxsw_sp->router->lb_rif_index); 5277 mlxsw_reg_ratr_trap_action_set(ratr_pl, trap_action); 5278 mlxsw_reg_ratr_trap_id_set(ratr_pl, MLXSW_TRAP_ID_RTR_EGRESS0); 5279 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ratr), ratr_pl); 5280 if (err) 5281 goto err_ratr_write; 5282 5283 mlxsw_sp->router->adj_discard_index_valid = true; 5284 5285 return 0; 5286 5287 err_ratr_write: 5288 mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 1, 5289 mlxsw_sp->router->adj_discard_index); 5290 return err; 5291 } 5292 5293 static int mlxsw_sp_fib_entry_op_remote(struct mlxsw_sp *mlxsw_sp, 5294 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 5295 struct mlxsw_sp_fib_entry *fib_entry, 5296 enum mlxsw_sp_fib_entry_op op) 5297 { 5298 const struct mlxsw_sp_router_ll_ops *ll_ops = fib_entry->fib_node->fib->ll_ops; 5299 struct mlxsw_sp_nexthop_group *nh_group = fib_entry->nh_group; 5300 struct mlxsw_sp_nexthop_group_info *nhgi = nh_group->nhgi; 5301 enum mlxsw_reg_ralue_trap_action trap_action; 5302 u16 trap_id = 0; 5303 u32 adjacency_index = 0; 5304 u16 ecmp_size = 0; 5305 int err; 5306 5307 /* In case the nexthop group adjacency index is valid, use it 5308 * with provided ECMP size. Otherwise, setup trap and pass 5309 * traffic to kernel. 5310 */ 5311 if (mlxsw_sp_fib_entry_should_offload(fib_entry)) { 5312 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_NOP; 5313 adjacency_index = nhgi->adj_index; 5314 ecmp_size = nhgi->ecmp_size; 5315 } else if (!nhgi->adj_index_valid && nhgi->count && nhgi->nh_rif) { 5316 err = mlxsw_sp_adj_discard_write(mlxsw_sp); 5317 if (err) 5318 return err; 5319 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_NOP; 5320 adjacency_index = mlxsw_sp->router->adj_discard_index; 5321 ecmp_size = 1; 5322 } else { 5323 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_TRAP; 5324 trap_id = MLXSW_TRAP_ID_RTR_INGRESS0; 5325 } 5326 5327 mlxsw_sp_fib_entry_pack(op_ctx, fib_entry, op); 5328 ll_ops->fib_entry_act_remote_pack(op_ctx, trap_action, trap_id, 5329 adjacency_index, ecmp_size); 5330 return mlxsw_sp_fib_entry_commit(mlxsw_sp, op_ctx, ll_ops); 5331 } 5332 5333 static int mlxsw_sp_fib_entry_op_local(struct mlxsw_sp *mlxsw_sp, 5334 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 5335 struct mlxsw_sp_fib_entry *fib_entry, 5336 enum mlxsw_sp_fib_entry_op op) 5337 { 5338 const struct mlxsw_sp_router_ll_ops *ll_ops = fib_entry->fib_node->fib->ll_ops; 5339 struct mlxsw_sp_rif *rif = fib_entry->nh_group->nhgi->nh_rif; 5340 enum mlxsw_reg_ralue_trap_action trap_action; 5341 u16 trap_id = 0; 5342 u16 rif_index = 0; 5343 5344 if (mlxsw_sp_fib_entry_should_offload(fib_entry)) { 5345 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_NOP; 5346 rif_index = rif->rif_index; 5347 } else { 5348 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_TRAP; 5349 trap_id = MLXSW_TRAP_ID_RTR_INGRESS0; 5350 } 5351 5352 mlxsw_sp_fib_entry_pack(op_ctx, fib_entry, op); 5353 ll_ops->fib_entry_act_local_pack(op_ctx, trap_action, trap_id, rif_index); 5354 return mlxsw_sp_fib_entry_commit(mlxsw_sp, op_ctx, ll_ops); 5355 } 5356 5357 static int mlxsw_sp_fib_entry_op_trap(struct mlxsw_sp *mlxsw_sp, 5358 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 5359 struct mlxsw_sp_fib_entry *fib_entry, 5360 enum mlxsw_sp_fib_entry_op op) 5361 { 5362 const struct mlxsw_sp_router_ll_ops *ll_ops = fib_entry->fib_node->fib->ll_ops; 5363 5364 mlxsw_sp_fib_entry_pack(op_ctx, fib_entry, op); 5365 ll_ops->fib_entry_act_ip2me_pack(op_ctx); 5366 return mlxsw_sp_fib_entry_commit(mlxsw_sp, op_ctx, ll_ops); 5367 } 5368 5369 static int mlxsw_sp_fib_entry_op_blackhole(struct mlxsw_sp *mlxsw_sp, 5370 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 5371 struct mlxsw_sp_fib_entry *fib_entry, 5372 enum mlxsw_sp_fib_entry_op op) 5373 { 5374 const struct mlxsw_sp_router_ll_ops *ll_ops = fib_entry->fib_node->fib->ll_ops; 5375 enum mlxsw_reg_ralue_trap_action trap_action; 5376 5377 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_DISCARD_ERROR; 5378 mlxsw_sp_fib_entry_pack(op_ctx, fib_entry, op); 5379 ll_ops->fib_entry_act_local_pack(op_ctx, trap_action, 0, 0); 5380 return mlxsw_sp_fib_entry_commit(mlxsw_sp, op_ctx, ll_ops); 5381 } 5382 5383 static int 5384 mlxsw_sp_fib_entry_op_unreachable(struct mlxsw_sp *mlxsw_sp, 5385 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 5386 struct mlxsw_sp_fib_entry *fib_entry, 5387 enum mlxsw_sp_fib_entry_op op) 5388 { 5389 const struct mlxsw_sp_router_ll_ops *ll_ops = fib_entry->fib_node->fib->ll_ops; 5390 enum mlxsw_reg_ralue_trap_action trap_action; 5391 u16 trap_id; 5392 5393 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_TRAP; 5394 trap_id = MLXSW_TRAP_ID_RTR_INGRESS1; 5395 5396 mlxsw_sp_fib_entry_pack(op_ctx, fib_entry, op); 5397 ll_ops->fib_entry_act_local_pack(op_ctx, trap_action, trap_id, 0); 5398 return mlxsw_sp_fib_entry_commit(mlxsw_sp, op_ctx, ll_ops); 5399 } 5400 5401 static int 5402 mlxsw_sp_fib_entry_op_ipip_decap(struct mlxsw_sp *mlxsw_sp, 5403 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 5404 struct mlxsw_sp_fib_entry *fib_entry, 5405 enum mlxsw_sp_fib_entry_op op) 5406 { 5407 const struct mlxsw_sp_router_ll_ops *ll_ops = fib_entry->fib_node->fib->ll_ops; 5408 struct mlxsw_sp_ipip_entry *ipip_entry = fib_entry->decap.ipip_entry; 5409 const struct mlxsw_sp_ipip_ops *ipip_ops; 5410 int err; 5411 5412 if (WARN_ON(!ipip_entry)) 5413 return -EINVAL; 5414 5415 ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt]; 5416 err = ipip_ops->decap_config(mlxsw_sp, ipip_entry, 5417 fib_entry->decap.tunnel_index); 5418 if (err) 5419 return err; 5420 5421 mlxsw_sp_fib_entry_pack(op_ctx, fib_entry, op); 5422 ll_ops->fib_entry_act_ip2me_tun_pack(op_ctx, 5423 fib_entry->decap.tunnel_index); 5424 return mlxsw_sp_fib_entry_commit(mlxsw_sp, op_ctx, ll_ops); 5425 } 5426 5427 static int mlxsw_sp_fib_entry_op_nve_decap(struct mlxsw_sp *mlxsw_sp, 5428 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 5429 struct mlxsw_sp_fib_entry *fib_entry, 5430 enum mlxsw_sp_fib_entry_op op) 5431 { 5432 const struct mlxsw_sp_router_ll_ops *ll_ops = fib_entry->fib_node->fib->ll_ops; 5433 5434 mlxsw_sp_fib_entry_pack(op_ctx, fib_entry, op); 5435 ll_ops->fib_entry_act_ip2me_tun_pack(op_ctx, 5436 fib_entry->decap.tunnel_index); 5437 return mlxsw_sp_fib_entry_commit(mlxsw_sp, op_ctx, ll_ops); 5438 } 5439 5440 static int __mlxsw_sp_fib_entry_op(struct mlxsw_sp *mlxsw_sp, 5441 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 5442 struct mlxsw_sp_fib_entry *fib_entry, 5443 enum mlxsw_sp_fib_entry_op op) 5444 { 5445 switch (fib_entry->type) { 5446 case MLXSW_SP_FIB_ENTRY_TYPE_REMOTE: 5447 return mlxsw_sp_fib_entry_op_remote(mlxsw_sp, op_ctx, fib_entry, op); 5448 case MLXSW_SP_FIB_ENTRY_TYPE_LOCAL: 5449 return mlxsw_sp_fib_entry_op_local(mlxsw_sp, op_ctx, fib_entry, op); 5450 case MLXSW_SP_FIB_ENTRY_TYPE_TRAP: 5451 return mlxsw_sp_fib_entry_op_trap(mlxsw_sp, op_ctx, fib_entry, op); 5452 case MLXSW_SP_FIB_ENTRY_TYPE_BLACKHOLE: 5453 return mlxsw_sp_fib_entry_op_blackhole(mlxsw_sp, op_ctx, fib_entry, op); 5454 case MLXSW_SP_FIB_ENTRY_TYPE_UNREACHABLE: 5455 return mlxsw_sp_fib_entry_op_unreachable(mlxsw_sp, op_ctx, fib_entry, op); 5456 case MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP: 5457 return mlxsw_sp_fib_entry_op_ipip_decap(mlxsw_sp, op_ctx, fib_entry, op); 5458 case MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP: 5459 return mlxsw_sp_fib_entry_op_nve_decap(mlxsw_sp, op_ctx, fib_entry, op); 5460 } 5461 return -EINVAL; 5462 } 5463 5464 static int mlxsw_sp_fib_entry_op(struct mlxsw_sp *mlxsw_sp, 5465 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 5466 struct mlxsw_sp_fib_entry *fib_entry, 5467 enum mlxsw_sp_fib_entry_op op) 5468 { 5469 int err = __mlxsw_sp_fib_entry_op(mlxsw_sp, op_ctx, fib_entry, op); 5470 5471 if (err) 5472 return err; 5473 5474 mlxsw_sp_fib_entry_hw_flags_refresh(mlxsw_sp, fib_entry, op); 5475 5476 return err; 5477 } 5478 5479 static int __mlxsw_sp_fib_entry_update(struct mlxsw_sp *mlxsw_sp, 5480 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 5481 struct mlxsw_sp_fib_entry *fib_entry, 5482 bool is_new) 5483 { 5484 return mlxsw_sp_fib_entry_op(mlxsw_sp, op_ctx, fib_entry, 5485 is_new ? MLXSW_SP_FIB_ENTRY_OP_WRITE : 5486 MLXSW_SP_FIB_ENTRY_OP_UPDATE); 5487 } 5488 5489 static int mlxsw_sp_fib_entry_update(struct mlxsw_sp *mlxsw_sp, 5490 struct mlxsw_sp_fib_entry *fib_entry) 5491 { 5492 struct mlxsw_sp_fib_entry_op_ctx *op_ctx = mlxsw_sp->router->ll_op_ctx; 5493 5494 mlxsw_sp_fib_entry_op_ctx_clear(op_ctx); 5495 return __mlxsw_sp_fib_entry_update(mlxsw_sp, op_ctx, fib_entry, false); 5496 } 5497 5498 static int mlxsw_sp_fib_entry_del(struct mlxsw_sp *mlxsw_sp, 5499 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 5500 struct mlxsw_sp_fib_entry *fib_entry) 5501 { 5502 const struct mlxsw_sp_router_ll_ops *ll_ops = fib_entry->fib_node->fib->ll_ops; 5503 5504 if (!ll_ops->fib_entry_is_committed(fib_entry->priv)) 5505 return 0; 5506 return mlxsw_sp_fib_entry_op(mlxsw_sp, op_ctx, fib_entry, 5507 MLXSW_SP_FIB_ENTRY_OP_DELETE); 5508 } 5509 5510 static int 5511 mlxsw_sp_fib4_entry_type_set(struct mlxsw_sp *mlxsw_sp, 5512 const struct fib_entry_notifier_info *fen_info, 5513 struct mlxsw_sp_fib_entry *fib_entry) 5514 { 5515 struct mlxsw_sp_nexthop_group_info *nhgi = fib_entry->nh_group->nhgi; 5516 union mlxsw_sp_l3addr dip = { .addr4 = htonl(fen_info->dst) }; 5517 struct mlxsw_sp_router *router = mlxsw_sp->router; 5518 u32 tb_id = mlxsw_sp_fix_tb_id(fen_info->tb_id); 5519 int ifindex = nhgi->nexthops[0].ifindex; 5520 struct mlxsw_sp_ipip_entry *ipip_entry; 5521 5522 switch (fen_info->type) { 5523 case RTN_LOCAL: 5524 ipip_entry = mlxsw_sp_ipip_entry_find_by_decap(mlxsw_sp, ifindex, 5525 MLXSW_SP_L3_PROTO_IPV4, dip); 5526 if (ipip_entry && ipip_entry->ol_dev->flags & IFF_UP) { 5527 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP; 5528 return mlxsw_sp_fib_entry_decap_init(mlxsw_sp, 5529 fib_entry, 5530 ipip_entry); 5531 } 5532 if (mlxsw_sp_router_nve_is_decap(mlxsw_sp, tb_id, 5533 MLXSW_SP_L3_PROTO_IPV4, 5534 &dip)) { 5535 u32 tunnel_index; 5536 5537 tunnel_index = router->nve_decap_config.tunnel_index; 5538 fib_entry->decap.tunnel_index = tunnel_index; 5539 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP; 5540 return 0; 5541 } 5542 fallthrough; 5543 case RTN_BROADCAST: 5544 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP; 5545 return 0; 5546 case RTN_BLACKHOLE: 5547 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_BLACKHOLE; 5548 return 0; 5549 case RTN_UNREACHABLE: 5550 case RTN_PROHIBIT: 5551 /* Packets hitting these routes need to be trapped, but 5552 * can do so with a lower priority than packets directed 5553 * at the host, so use action type local instead of trap. 5554 */ 5555 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_UNREACHABLE; 5556 return 0; 5557 case RTN_UNICAST: 5558 if (nhgi->gateway) 5559 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_REMOTE; 5560 else 5561 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL; 5562 return 0; 5563 default: 5564 return -EINVAL; 5565 } 5566 } 5567 5568 static void 5569 mlxsw_sp_fib4_entry_type_unset(struct mlxsw_sp *mlxsw_sp, 5570 struct mlxsw_sp_fib_entry *fib_entry) 5571 { 5572 switch (fib_entry->type) { 5573 case MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP: 5574 mlxsw_sp_fib_entry_decap_fini(mlxsw_sp, fib_entry); 5575 break; 5576 default: 5577 break; 5578 } 5579 } 5580 5581 static struct mlxsw_sp_fib4_entry * 5582 mlxsw_sp_fib4_entry_create(struct mlxsw_sp *mlxsw_sp, 5583 struct mlxsw_sp_fib_node *fib_node, 5584 const struct fib_entry_notifier_info *fen_info) 5585 { 5586 struct mlxsw_sp_fib4_entry *fib4_entry; 5587 struct mlxsw_sp_fib_entry *fib_entry; 5588 int err; 5589 5590 fib4_entry = kzalloc(sizeof(*fib4_entry), GFP_KERNEL); 5591 if (!fib4_entry) 5592 return ERR_PTR(-ENOMEM); 5593 fib_entry = &fib4_entry->common; 5594 5595 fib_entry->priv = mlxsw_sp_fib_entry_priv_create(fib_node->fib->ll_ops); 5596 if (IS_ERR(fib_entry->priv)) { 5597 err = PTR_ERR(fib_entry->priv); 5598 goto err_fib_entry_priv_create; 5599 } 5600 5601 err = mlxsw_sp_nexthop4_group_get(mlxsw_sp, fib_entry, fen_info->fi); 5602 if (err) 5603 goto err_nexthop4_group_get; 5604 5605 err = mlxsw_sp_nexthop_group_vr_link(fib_entry->nh_group, 5606 fib_node->fib); 5607 if (err) 5608 goto err_nexthop_group_vr_link; 5609 5610 err = mlxsw_sp_fib4_entry_type_set(mlxsw_sp, fen_info, fib_entry); 5611 if (err) 5612 goto err_fib4_entry_type_set; 5613 5614 fib4_entry->fi = fen_info->fi; 5615 fib_info_hold(fib4_entry->fi); 5616 fib4_entry->tb_id = fen_info->tb_id; 5617 fib4_entry->type = fen_info->type; 5618 fib4_entry->tos = fen_info->tos; 5619 5620 fib_entry->fib_node = fib_node; 5621 5622 return fib4_entry; 5623 5624 err_fib4_entry_type_set: 5625 mlxsw_sp_nexthop_group_vr_unlink(fib_entry->nh_group, fib_node->fib); 5626 err_nexthop_group_vr_link: 5627 mlxsw_sp_nexthop4_group_put(mlxsw_sp, &fib4_entry->common); 5628 err_nexthop4_group_get: 5629 mlxsw_sp_fib_entry_priv_put(fib_entry->priv); 5630 err_fib_entry_priv_create: 5631 kfree(fib4_entry); 5632 return ERR_PTR(err); 5633 } 5634 5635 static void mlxsw_sp_fib4_entry_destroy(struct mlxsw_sp *mlxsw_sp, 5636 struct mlxsw_sp_fib4_entry *fib4_entry) 5637 { 5638 struct mlxsw_sp_fib_node *fib_node = fib4_entry->common.fib_node; 5639 5640 fib_info_put(fib4_entry->fi); 5641 mlxsw_sp_fib4_entry_type_unset(mlxsw_sp, &fib4_entry->common); 5642 mlxsw_sp_nexthop_group_vr_unlink(fib4_entry->common.nh_group, 5643 fib_node->fib); 5644 mlxsw_sp_nexthop4_group_put(mlxsw_sp, &fib4_entry->common); 5645 mlxsw_sp_fib_entry_priv_put(fib4_entry->common.priv); 5646 kfree(fib4_entry); 5647 } 5648 5649 static struct mlxsw_sp_fib4_entry * 5650 mlxsw_sp_fib4_entry_lookup(struct mlxsw_sp *mlxsw_sp, 5651 const struct fib_entry_notifier_info *fen_info) 5652 { 5653 struct mlxsw_sp_fib4_entry *fib4_entry; 5654 struct mlxsw_sp_fib_node *fib_node; 5655 struct mlxsw_sp_fib *fib; 5656 struct mlxsw_sp_vr *vr; 5657 5658 vr = mlxsw_sp_vr_find(mlxsw_sp, fen_info->tb_id); 5659 if (!vr) 5660 return NULL; 5661 fib = mlxsw_sp_vr_fib(vr, MLXSW_SP_L3_PROTO_IPV4); 5662 5663 fib_node = mlxsw_sp_fib_node_lookup(fib, &fen_info->dst, 5664 sizeof(fen_info->dst), 5665 fen_info->dst_len); 5666 if (!fib_node) 5667 return NULL; 5668 5669 fib4_entry = container_of(fib_node->fib_entry, 5670 struct mlxsw_sp_fib4_entry, common); 5671 if (fib4_entry->tb_id == fen_info->tb_id && 5672 fib4_entry->tos == fen_info->tos && 5673 fib4_entry->type == fen_info->type && 5674 fib4_entry->fi == fen_info->fi) 5675 return fib4_entry; 5676 5677 return NULL; 5678 } 5679 5680 static const struct rhashtable_params mlxsw_sp_fib_ht_params = { 5681 .key_offset = offsetof(struct mlxsw_sp_fib_node, key), 5682 .head_offset = offsetof(struct mlxsw_sp_fib_node, ht_node), 5683 .key_len = sizeof(struct mlxsw_sp_fib_key), 5684 .automatic_shrinking = true, 5685 }; 5686 5687 static int mlxsw_sp_fib_node_insert(struct mlxsw_sp_fib *fib, 5688 struct mlxsw_sp_fib_node *fib_node) 5689 { 5690 return rhashtable_insert_fast(&fib->ht, &fib_node->ht_node, 5691 mlxsw_sp_fib_ht_params); 5692 } 5693 5694 static void mlxsw_sp_fib_node_remove(struct mlxsw_sp_fib *fib, 5695 struct mlxsw_sp_fib_node *fib_node) 5696 { 5697 rhashtable_remove_fast(&fib->ht, &fib_node->ht_node, 5698 mlxsw_sp_fib_ht_params); 5699 } 5700 5701 static struct mlxsw_sp_fib_node * 5702 mlxsw_sp_fib_node_lookup(struct mlxsw_sp_fib *fib, const void *addr, 5703 size_t addr_len, unsigned char prefix_len) 5704 { 5705 struct mlxsw_sp_fib_key key; 5706 5707 memset(&key, 0, sizeof(key)); 5708 memcpy(key.addr, addr, addr_len); 5709 key.prefix_len = prefix_len; 5710 return rhashtable_lookup_fast(&fib->ht, &key, mlxsw_sp_fib_ht_params); 5711 } 5712 5713 static struct mlxsw_sp_fib_node * 5714 mlxsw_sp_fib_node_create(struct mlxsw_sp_fib *fib, const void *addr, 5715 size_t addr_len, unsigned char prefix_len) 5716 { 5717 struct mlxsw_sp_fib_node *fib_node; 5718 5719 fib_node = kzalloc(sizeof(*fib_node), GFP_KERNEL); 5720 if (!fib_node) 5721 return NULL; 5722 5723 list_add(&fib_node->list, &fib->node_list); 5724 memcpy(fib_node->key.addr, addr, addr_len); 5725 fib_node->key.prefix_len = prefix_len; 5726 5727 return fib_node; 5728 } 5729 5730 static void mlxsw_sp_fib_node_destroy(struct mlxsw_sp_fib_node *fib_node) 5731 { 5732 list_del(&fib_node->list); 5733 kfree(fib_node); 5734 } 5735 5736 static int mlxsw_sp_fib_lpm_tree_link(struct mlxsw_sp *mlxsw_sp, 5737 struct mlxsw_sp_fib_node *fib_node) 5738 { 5739 struct mlxsw_sp_prefix_usage req_prefix_usage; 5740 struct mlxsw_sp_fib *fib = fib_node->fib; 5741 struct mlxsw_sp_lpm_tree *lpm_tree; 5742 int err; 5743 5744 lpm_tree = mlxsw_sp->router->lpm.proto_trees[fib->proto]; 5745 if (lpm_tree->prefix_ref_count[fib_node->key.prefix_len] != 0) 5746 goto out; 5747 5748 mlxsw_sp_prefix_usage_cpy(&req_prefix_usage, &lpm_tree->prefix_usage); 5749 mlxsw_sp_prefix_usage_set(&req_prefix_usage, fib_node->key.prefix_len); 5750 lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage, 5751 fib->proto); 5752 if (IS_ERR(lpm_tree)) 5753 return PTR_ERR(lpm_tree); 5754 5755 err = mlxsw_sp_vrs_lpm_tree_replace(mlxsw_sp, fib, lpm_tree); 5756 if (err) 5757 goto err_lpm_tree_replace; 5758 5759 out: 5760 lpm_tree->prefix_ref_count[fib_node->key.prefix_len]++; 5761 return 0; 5762 5763 err_lpm_tree_replace: 5764 mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree); 5765 return err; 5766 } 5767 5768 static void mlxsw_sp_fib_lpm_tree_unlink(struct mlxsw_sp *mlxsw_sp, 5769 struct mlxsw_sp_fib_node *fib_node) 5770 { 5771 struct mlxsw_sp_lpm_tree *lpm_tree = fib_node->fib->lpm_tree; 5772 struct mlxsw_sp_prefix_usage req_prefix_usage; 5773 struct mlxsw_sp_fib *fib = fib_node->fib; 5774 int err; 5775 5776 if (--lpm_tree->prefix_ref_count[fib_node->key.prefix_len] != 0) 5777 return; 5778 /* Try to construct a new LPM tree from the current prefix usage 5779 * minus the unused one. If we fail, continue using the old one. 5780 */ 5781 mlxsw_sp_prefix_usage_cpy(&req_prefix_usage, &lpm_tree->prefix_usage); 5782 mlxsw_sp_prefix_usage_clear(&req_prefix_usage, 5783 fib_node->key.prefix_len); 5784 lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage, 5785 fib->proto); 5786 if (IS_ERR(lpm_tree)) 5787 return; 5788 5789 err = mlxsw_sp_vrs_lpm_tree_replace(mlxsw_sp, fib, lpm_tree); 5790 if (err) 5791 goto err_lpm_tree_replace; 5792 5793 return; 5794 5795 err_lpm_tree_replace: 5796 mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree); 5797 } 5798 5799 static int mlxsw_sp_fib_node_init(struct mlxsw_sp *mlxsw_sp, 5800 struct mlxsw_sp_fib_node *fib_node, 5801 struct mlxsw_sp_fib *fib) 5802 { 5803 int err; 5804 5805 err = mlxsw_sp_fib_node_insert(fib, fib_node); 5806 if (err) 5807 return err; 5808 fib_node->fib = fib; 5809 5810 err = mlxsw_sp_fib_lpm_tree_link(mlxsw_sp, fib_node); 5811 if (err) 5812 goto err_fib_lpm_tree_link; 5813 5814 return 0; 5815 5816 err_fib_lpm_tree_link: 5817 fib_node->fib = NULL; 5818 mlxsw_sp_fib_node_remove(fib, fib_node); 5819 return err; 5820 } 5821 5822 static void mlxsw_sp_fib_node_fini(struct mlxsw_sp *mlxsw_sp, 5823 struct mlxsw_sp_fib_node *fib_node) 5824 { 5825 struct mlxsw_sp_fib *fib = fib_node->fib; 5826 5827 mlxsw_sp_fib_lpm_tree_unlink(mlxsw_sp, fib_node); 5828 fib_node->fib = NULL; 5829 mlxsw_sp_fib_node_remove(fib, fib_node); 5830 } 5831 5832 static struct mlxsw_sp_fib_node * 5833 mlxsw_sp_fib_node_get(struct mlxsw_sp *mlxsw_sp, u32 tb_id, const void *addr, 5834 size_t addr_len, unsigned char prefix_len, 5835 enum mlxsw_sp_l3proto proto) 5836 { 5837 struct mlxsw_sp_fib_node *fib_node; 5838 struct mlxsw_sp_fib *fib; 5839 struct mlxsw_sp_vr *vr; 5840 int err; 5841 5842 vr = mlxsw_sp_vr_get(mlxsw_sp, tb_id, NULL); 5843 if (IS_ERR(vr)) 5844 return ERR_CAST(vr); 5845 fib = mlxsw_sp_vr_fib(vr, proto); 5846 5847 fib_node = mlxsw_sp_fib_node_lookup(fib, addr, addr_len, prefix_len); 5848 if (fib_node) 5849 return fib_node; 5850 5851 fib_node = mlxsw_sp_fib_node_create(fib, addr, addr_len, prefix_len); 5852 if (!fib_node) { 5853 err = -ENOMEM; 5854 goto err_fib_node_create; 5855 } 5856 5857 err = mlxsw_sp_fib_node_init(mlxsw_sp, fib_node, fib); 5858 if (err) 5859 goto err_fib_node_init; 5860 5861 return fib_node; 5862 5863 err_fib_node_init: 5864 mlxsw_sp_fib_node_destroy(fib_node); 5865 err_fib_node_create: 5866 mlxsw_sp_vr_put(mlxsw_sp, vr); 5867 return ERR_PTR(err); 5868 } 5869 5870 static void mlxsw_sp_fib_node_put(struct mlxsw_sp *mlxsw_sp, 5871 struct mlxsw_sp_fib_node *fib_node) 5872 { 5873 struct mlxsw_sp_vr *vr = fib_node->fib->vr; 5874 5875 if (fib_node->fib_entry) 5876 return; 5877 mlxsw_sp_fib_node_fini(mlxsw_sp, fib_node); 5878 mlxsw_sp_fib_node_destroy(fib_node); 5879 mlxsw_sp_vr_put(mlxsw_sp, vr); 5880 } 5881 5882 static int mlxsw_sp_fib_node_entry_link(struct mlxsw_sp *mlxsw_sp, 5883 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 5884 struct mlxsw_sp_fib_entry *fib_entry) 5885 { 5886 struct mlxsw_sp_fib_node *fib_node = fib_entry->fib_node; 5887 bool is_new = !fib_node->fib_entry; 5888 int err; 5889 5890 fib_node->fib_entry = fib_entry; 5891 5892 err = __mlxsw_sp_fib_entry_update(mlxsw_sp, op_ctx, fib_entry, is_new); 5893 if (err) 5894 goto err_fib_entry_update; 5895 5896 return 0; 5897 5898 err_fib_entry_update: 5899 fib_node->fib_entry = NULL; 5900 return err; 5901 } 5902 5903 static int __mlxsw_sp_fib_node_entry_unlink(struct mlxsw_sp *mlxsw_sp, 5904 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 5905 struct mlxsw_sp_fib_entry *fib_entry) 5906 { 5907 struct mlxsw_sp_fib_node *fib_node = fib_entry->fib_node; 5908 int err; 5909 5910 err = mlxsw_sp_fib_entry_del(mlxsw_sp, op_ctx, fib_entry); 5911 fib_node->fib_entry = NULL; 5912 return err; 5913 } 5914 5915 static void mlxsw_sp_fib_node_entry_unlink(struct mlxsw_sp *mlxsw_sp, 5916 struct mlxsw_sp_fib_entry *fib_entry) 5917 { 5918 struct mlxsw_sp_fib_entry_op_ctx *op_ctx = mlxsw_sp->router->ll_op_ctx; 5919 5920 mlxsw_sp_fib_entry_op_ctx_clear(op_ctx); 5921 __mlxsw_sp_fib_node_entry_unlink(mlxsw_sp, op_ctx, fib_entry); 5922 } 5923 5924 static bool mlxsw_sp_fib4_allow_replace(struct mlxsw_sp_fib4_entry *fib4_entry) 5925 { 5926 struct mlxsw_sp_fib_node *fib_node = fib4_entry->common.fib_node; 5927 struct mlxsw_sp_fib4_entry *fib4_replaced; 5928 5929 if (!fib_node->fib_entry) 5930 return true; 5931 5932 fib4_replaced = container_of(fib_node->fib_entry, 5933 struct mlxsw_sp_fib4_entry, common); 5934 if (fib4_entry->tb_id == RT_TABLE_MAIN && 5935 fib4_replaced->tb_id == RT_TABLE_LOCAL) 5936 return false; 5937 5938 return true; 5939 } 5940 5941 static int 5942 mlxsw_sp_router_fib4_replace(struct mlxsw_sp *mlxsw_sp, 5943 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 5944 const struct fib_entry_notifier_info *fen_info) 5945 { 5946 struct mlxsw_sp_fib4_entry *fib4_entry, *fib4_replaced; 5947 struct mlxsw_sp_fib_entry *replaced; 5948 struct mlxsw_sp_fib_node *fib_node; 5949 int err; 5950 5951 if (mlxsw_sp->router->aborted) 5952 return 0; 5953 5954 fib_node = mlxsw_sp_fib_node_get(mlxsw_sp, fen_info->tb_id, 5955 &fen_info->dst, sizeof(fen_info->dst), 5956 fen_info->dst_len, 5957 MLXSW_SP_L3_PROTO_IPV4); 5958 if (IS_ERR(fib_node)) { 5959 dev_warn(mlxsw_sp->bus_info->dev, "Failed to get FIB node\n"); 5960 return PTR_ERR(fib_node); 5961 } 5962 5963 fib4_entry = mlxsw_sp_fib4_entry_create(mlxsw_sp, fib_node, fen_info); 5964 if (IS_ERR(fib4_entry)) { 5965 dev_warn(mlxsw_sp->bus_info->dev, "Failed to create FIB entry\n"); 5966 err = PTR_ERR(fib4_entry); 5967 goto err_fib4_entry_create; 5968 } 5969 5970 if (!mlxsw_sp_fib4_allow_replace(fib4_entry)) { 5971 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry); 5972 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 5973 return 0; 5974 } 5975 5976 replaced = fib_node->fib_entry; 5977 err = mlxsw_sp_fib_node_entry_link(mlxsw_sp, op_ctx, &fib4_entry->common); 5978 if (err) { 5979 dev_warn(mlxsw_sp->bus_info->dev, "Failed to link FIB entry to node\n"); 5980 goto err_fib_node_entry_link; 5981 } 5982 5983 /* Nothing to replace */ 5984 if (!replaced) 5985 return 0; 5986 5987 mlxsw_sp_fib_entry_hw_flags_clear(mlxsw_sp, replaced); 5988 fib4_replaced = container_of(replaced, struct mlxsw_sp_fib4_entry, 5989 common); 5990 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_replaced); 5991 5992 return 0; 5993 5994 err_fib_node_entry_link: 5995 fib_node->fib_entry = replaced; 5996 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry); 5997 err_fib4_entry_create: 5998 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 5999 return err; 6000 } 6001 6002 static int mlxsw_sp_router_fib4_del(struct mlxsw_sp *mlxsw_sp, 6003 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 6004 struct fib_entry_notifier_info *fen_info) 6005 { 6006 struct mlxsw_sp_fib4_entry *fib4_entry; 6007 struct mlxsw_sp_fib_node *fib_node; 6008 int err; 6009 6010 if (mlxsw_sp->router->aborted) 6011 return 0; 6012 6013 fib4_entry = mlxsw_sp_fib4_entry_lookup(mlxsw_sp, fen_info); 6014 if (!fib4_entry) 6015 return 0; 6016 fib_node = fib4_entry->common.fib_node; 6017 6018 err = __mlxsw_sp_fib_node_entry_unlink(mlxsw_sp, op_ctx, &fib4_entry->common); 6019 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry); 6020 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 6021 return err; 6022 } 6023 6024 static bool mlxsw_sp_fib6_rt_should_ignore(const struct fib6_info *rt) 6025 { 6026 /* Multicast routes aren't supported, so ignore them. Neighbour 6027 * Discovery packets are specifically trapped. 6028 */ 6029 if (ipv6_addr_type(&rt->fib6_dst.addr) & IPV6_ADDR_MULTICAST) 6030 return true; 6031 6032 /* Cloned routes are irrelevant in the forwarding path. */ 6033 if (rt->fib6_flags & RTF_CACHE) 6034 return true; 6035 6036 return false; 6037 } 6038 6039 static struct mlxsw_sp_rt6 *mlxsw_sp_rt6_create(struct fib6_info *rt) 6040 { 6041 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 6042 6043 mlxsw_sp_rt6 = kzalloc(sizeof(*mlxsw_sp_rt6), GFP_KERNEL); 6044 if (!mlxsw_sp_rt6) 6045 return ERR_PTR(-ENOMEM); 6046 6047 /* In case of route replace, replaced route is deleted with 6048 * no notification. Take reference to prevent accessing freed 6049 * memory. 6050 */ 6051 mlxsw_sp_rt6->rt = rt; 6052 fib6_info_hold(rt); 6053 6054 return mlxsw_sp_rt6; 6055 } 6056 6057 #if IS_ENABLED(CONFIG_IPV6) 6058 static void mlxsw_sp_rt6_release(struct fib6_info *rt) 6059 { 6060 fib6_info_release(rt); 6061 } 6062 #else 6063 static void mlxsw_sp_rt6_release(struct fib6_info *rt) 6064 { 6065 } 6066 #endif 6067 6068 static void mlxsw_sp_rt6_destroy(struct mlxsw_sp_rt6 *mlxsw_sp_rt6) 6069 { 6070 struct fib6_nh *fib6_nh = mlxsw_sp_rt6->rt->fib6_nh; 6071 6072 if (!mlxsw_sp_rt6->rt->nh) 6073 fib6_nh->fib_nh_flags &= ~RTNH_F_OFFLOAD; 6074 mlxsw_sp_rt6_release(mlxsw_sp_rt6->rt); 6075 kfree(mlxsw_sp_rt6); 6076 } 6077 6078 static struct fib6_info * 6079 mlxsw_sp_fib6_entry_rt(const struct mlxsw_sp_fib6_entry *fib6_entry) 6080 { 6081 return list_first_entry(&fib6_entry->rt6_list, struct mlxsw_sp_rt6, 6082 list)->rt; 6083 } 6084 6085 static struct mlxsw_sp_rt6 * 6086 mlxsw_sp_fib6_entry_rt_find(const struct mlxsw_sp_fib6_entry *fib6_entry, 6087 const struct fib6_info *rt) 6088 { 6089 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 6090 6091 list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) { 6092 if (mlxsw_sp_rt6->rt == rt) 6093 return mlxsw_sp_rt6; 6094 } 6095 6096 return NULL; 6097 } 6098 6099 static bool mlxsw_sp_nexthop6_ipip_type(const struct mlxsw_sp *mlxsw_sp, 6100 const struct fib6_info *rt, 6101 enum mlxsw_sp_ipip_type *ret) 6102 { 6103 return rt->fib6_nh->fib_nh_dev && 6104 mlxsw_sp_netdev_ipip_type(mlxsw_sp, rt->fib6_nh->fib_nh_dev, ret); 6105 } 6106 6107 static int mlxsw_sp_nexthop6_init(struct mlxsw_sp *mlxsw_sp, 6108 struct mlxsw_sp_nexthop_group *nh_grp, 6109 struct mlxsw_sp_nexthop *nh, 6110 const struct fib6_info *rt) 6111 { 6112 struct net_device *dev = rt->fib6_nh->fib_nh_dev; 6113 6114 nh->nhgi = nh_grp->nhgi; 6115 nh->nh_weight = rt->fib6_nh->fib_nh_weight; 6116 memcpy(&nh->gw_addr, &rt->fib6_nh->fib_nh_gw6, sizeof(nh->gw_addr)); 6117 #if IS_ENABLED(CONFIG_IPV6) 6118 nh->neigh_tbl = &nd_tbl; 6119 #endif 6120 mlxsw_sp_nexthop_counter_alloc(mlxsw_sp, nh); 6121 6122 list_add_tail(&nh->router_list_node, &mlxsw_sp->router->nexthop_list); 6123 6124 if (!dev) 6125 return 0; 6126 nh->ifindex = dev->ifindex; 6127 6128 return mlxsw_sp_nexthop_type_init(mlxsw_sp, nh, dev); 6129 } 6130 6131 static void mlxsw_sp_nexthop6_fini(struct mlxsw_sp *mlxsw_sp, 6132 struct mlxsw_sp_nexthop *nh) 6133 { 6134 mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh); 6135 list_del(&nh->router_list_node); 6136 mlxsw_sp_nexthop_counter_free(mlxsw_sp, nh); 6137 } 6138 6139 static bool mlxsw_sp_rt6_is_gateway(const struct mlxsw_sp *mlxsw_sp, 6140 const struct fib6_info *rt) 6141 { 6142 return rt->fib6_nh->fib_nh_gw_family || 6143 mlxsw_sp_nexthop6_ipip_type(mlxsw_sp, rt, NULL); 6144 } 6145 6146 static int 6147 mlxsw_sp_nexthop6_group_info_init(struct mlxsw_sp *mlxsw_sp, 6148 struct mlxsw_sp_nexthop_group *nh_grp, 6149 struct mlxsw_sp_fib6_entry *fib6_entry) 6150 { 6151 struct mlxsw_sp_nexthop_group_info *nhgi; 6152 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 6153 struct mlxsw_sp_nexthop *nh; 6154 int err, i; 6155 6156 nhgi = kzalloc(struct_size(nhgi, nexthops, fib6_entry->nrt6), 6157 GFP_KERNEL); 6158 if (!nhgi) 6159 return -ENOMEM; 6160 nh_grp->nhgi = nhgi; 6161 nhgi->nh_grp = nh_grp; 6162 mlxsw_sp_rt6 = list_first_entry(&fib6_entry->rt6_list, 6163 struct mlxsw_sp_rt6, list); 6164 nhgi->gateway = mlxsw_sp_rt6_is_gateway(mlxsw_sp, mlxsw_sp_rt6->rt); 6165 nhgi->count = fib6_entry->nrt6; 6166 for (i = 0; i < nhgi->count; i++) { 6167 struct fib6_info *rt = mlxsw_sp_rt6->rt; 6168 6169 nh = &nhgi->nexthops[i]; 6170 err = mlxsw_sp_nexthop6_init(mlxsw_sp, nh_grp, nh, rt); 6171 if (err) 6172 goto err_nexthop6_init; 6173 mlxsw_sp_rt6 = list_next_entry(mlxsw_sp_rt6, list); 6174 } 6175 nh_grp->nhgi = nhgi; 6176 err = mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp); 6177 if (err) 6178 goto err_group_refresh; 6179 6180 return 0; 6181 6182 err_group_refresh: 6183 i = nhgi->count; 6184 err_nexthop6_init: 6185 for (i--; i >= 0; i--) { 6186 nh = &nhgi->nexthops[i]; 6187 mlxsw_sp_nexthop6_fini(mlxsw_sp, nh); 6188 } 6189 kfree(nhgi); 6190 return err; 6191 } 6192 6193 static void 6194 mlxsw_sp_nexthop6_group_info_fini(struct mlxsw_sp *mlxsw_sp, 6195 struct mlxsw_sp_nexthop_group *nh_grp) 6196 { 6197 struct mlxsw_sp_nexthop_group_info *nhgi = nh_grp->nhgi; 6198 int i; 6199 6200 for (i = nhgi->count - 1; i >= 0; i--) { 6201 struct mlxsw_sp_nexthop *nh = &nhgi->nexthops[i]; 6202 6203 mlxsw_sp_nexthop6_fini(mlxsw_sp, nh); 6204 } 6205 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp); 6206 WARN_ON_ONCE(nhgi->adj_index_valid); 6207 kfree(nhgi); 6208 } 6209 6210 static struct mlxsw_sp_nexthop_group * 6211 mlxsw_sp_nexthop6_group_create(struct mlxsw_sp *mlxsw_sp, 6212 struct mlxsw_sp_fib6_entry *fib6_entry) 6213 { 6214 struct mlxsw_sp_nexthop_group *nh_grp; 6215 int err; 6216 6217 nh_grp = kzalloc(sizeof(*nh_grp), GFP_KERNEL); 6218 if (!nh_grp) 6219 return ERR_PTR(-ENOMEM); 6220 INIT_LIST_HEAD(&nh_grp->vr_list); 6221 err = rhashtable_init(&nh_grp->vr_ht, 6222 &mlxsw_sp_nexthop_group_vr_ht_params); 6223 if (err) 6224 goto err_nexthop_group_vr_ht_init; 6225 INIT_LIST_HEAD(&nh_grp->fib_list); 6226 nh_grp->type = MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6; 6227 6228 err = mlxsw_sp_nexthop6_group_info_init(mlxsw_sp, nh_grp, fib6_entry); 6229 if (err) 6230 goto err_nexthop_group_info_init; 6231 6232 err = mlxsw_sp_nexthop_group_insert(mlxsw_sp, nh_grp); 6233 if (err) 6234 goto err_nexthop_group_insert; 6235 6236 nh_grp->can_destroy = true; 6237 6238 return nh_grp; 6239 6240 err_nexthop_group_insert: 6241 mlxsw_sp_nexthop6_group_info_fini(mlxsw_sp, nh_grp); 6242 err_nexthop_group_info_init: 6243 rhashtable_destroy(&nh_grp->vr_ht); 6244 err_nexthop_group_vr_ht_init: 6245 kfree(nh_grp); 6246 return ERR_PTR(err); 6247 } 6248 6249 static void 6250 mlxsw_sp_nexthop6_group_destroy(struct mlxsw_sp *mlxsw_sp, 6251 struct mlxsw_sp_nexthop_group *nh_grp) 6252 { 6253 if (!nh_grp->can_destroy) 6254 return; 6255 mlxsw_sp_nexthop_group_remove(mlxsw_sp, nh_grp); 6256 mlxsw_sp_nexthop6_group_info_fini(mlxsw_sp, nh_grp); 6257 WARN_ON_ONCE(!list_empty(&nh_grp->vr_list)); 6258 rhashtable_destroy(&nh_grp->vr_ht); 6259 kfree(nh_grp); 6260 } 6261 6262 static int mlxsw_sp_nexthop6_group_get(struct mlxsw_sp *mlxsw_sp, 6263 struct mlxsw_sp_fib6_entry *fib6_entry) 6264 { 6265 struct fib6_info *rt = mlxsw_sp_fib6_entry_rt(fib6_entry); 6266 struct mlxsw_sp_nexthop_group *nh_grp; 6267 6268 if (rt->nh) { 6269 nh_grp = mlxsw_sp_nexthop_obj_group_lookup(mlxsw_sp, 6270 rt->nh->id); 6271 if (WARN_ON_ONCE(!nh_grp)) 6272 return -EINVAL; 6273 goto out; 6274 } 6275 6276 nh_grp = mlxsw_sp_nexthop6_group_lookup(mlxsw_sp, fib6_entry); 6277 if (!nh_grp) { 6278 nh_grp = mlxsw_sp_nexthop6_group_create(mlxsw_sp, fib6_entry); 6279 if (IS_ERR(nh_grp)) 6280 return PTR_ERR(nh_grp); 6281 } 6282 6283 /* The route and the nexthop are described by the same struct, so we 6284 * need to the update the nexthop offload indication for the new route. 6285 */ 6286 __mlxsw_sp_nexthop6_group_offload_refresh(nh_grp, fib6_entry); 6287 6288 out: 6289 list_add_tail(&fib6_entry->common.nexthop_group_node, 6290 &nh_grp->fib_list); 6291 fib6_entry->common.nh_group = nh_grp; 6292 6293 return 0; 6294 } 6295 6296 static void mlxsw_sp_nexthop6_group_put(struct mlxsw_sp *mlxsw_sp, 6297 struct mlxsw_sp_fib_entry *fib_entry) 6298 { 6299 struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group; 6300 6301 list_del(&fib_entry->nexthop_group_node); 6302 if (!list_empty(&nh_grp->fib_list)) 6303 return; 6304 6305 if (nh_grp->type == MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ) { 6306 mlxsw_sp_nexthop_obj_group_destroy(mlxsw_sp, nh_grp); 6307 return; 6308 } 6309 6310 mlxsw_sp_nexthop6_group_destroy(mlxsw_sp, nh_grp); 6311 } 6312 6313 static int mlxsw_sp_nexthop6_group_update(struct mlxsw_sp *mlxsw_sp, 6314 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 6315 struct mlxsw_sp_fib6_entry *fib6_entry) 6316 { 6317 struct mlxsw_sp_nexthop_group *old_nh_grp = fib6_entry->common.nh_group; 6318 struct mlxsw_sp_fib_node *fib_node = fib6_entry->common.fib_node; 6319 int err; 6320 6321 mlxsw_sp_nexthop_group_vr_unlink(old_nh_grp, fib_node->fib); 6322 fib6_entry->common.nh_group = NULL; 6323 list_del(&fib6_entry->common.nexthop_group_node); 6324 6325 err = mlxsw_sp_nexthop6_group_get(mlxsw_sp, fib6_entry); 6326 if (err) 6327 goto err_nexthop6_group_get; 6328 6329 err = mlxsw_sp_nexthop_group_vr_link(fib6_entry->common.nh_group, 6330 fib_node->fib); 6331 if (err) 6332 goto err_nexthop_group_vr_link; 6333 6334 /* In case this entry is offloaded, then the adjacency index 6335 * currently associated with it in the device's table is that 6336 * of the old group. Start using the new one instead. 6337 */ 6338 err = __mlxsw_sp_fib_entry_update(mlxsw_sp, op_ctx, 6339 &fib6_entry->common, false); 6340 if (err) 6341 goto err_fib_entry_update; 6342 6343 if (list_empty(&old_nh_grp->fib_list)) 6344 mlxsw_sp_nexthop6_group_destroy(mlxsw_sp, old_nh_grp); 6345 6346 return 0; 6347 6348 err_fib_entry_update: 6349 mlxsw_sp_nexthop_group_vr_unlink(fib6_entry->common.nh_group, 6350 fib_node->fib); 6351 err_nexthop_group_vr_link: 6352 mlxsw_sp_nexthop6_group_put(mlxsw_sp, &fib6_entry->common); 6353 err_nexthop6_group_get: 6354 list_add_tail(&fib6_entry->common.nexthop_group_node, 6355 &old_nh_grp->fib_list); 6356 fib6_entry->common.nh_group = old_nh_grp; 6357 mlxsw_sp_nexthop_group_vr_link(old_nh_grp, fib_node->fib); 6358 return err; 6359 } 6360 6361 static int 6362 mlxsw_sp_fib6_entry_nexthop_add(struct mlxsw_sp *mlxsw_sp, 6363 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 6364 struct mlxsw_sp_fib6_entry *fib6_entry, 6365 struct fib6_info **rt_arr, unsigned int nrt6) 6366 { 6367 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 6368 int err, i; 6369 6370 for (i = 0; i < nrt6; i++) { 6371 mlxsw_sp_rt6 = mlxsw_sp_rt6_create(rt_arr[i]); 6372 if (IS_ERR(mlxsw_sp_rt6)) { 6373 err = PTR_ERR(mlxsw_sp_rt6); 6374 goto err_rt6_create; 6375 } 6376 6377 list_add_tail(&mlxsw_sp_rt6->list, &fib6_entry->rt6_list); 6378 fib6_entry->nrt6++; 6379 } 6380 6381 err = mlxsw_sp_nexthop6_group_update(mlxsw_sp, op_ctx, fib6_entry); 6382 if (err) 6383 goto err_nexthop6_group_update; 6384 6385 return 0; 6386 6387 err_nexthop6_group_update: 6388 i = nrt6; 6389 err_rt6_create: 6390 for (i--; i >= 0; i--) { 6391 fib6_entry->nrt6--; 6392 mlxsw_sp_rt6 = list_last_entry(&fib6_entry->rt6_list, 6393 struct mlxsw_sp_rt6, list); 6394 list_del(&mlxsw_sp_rt6->list); 6395 mlxsw_sp_rt6_destroy(mlxsw_sp_rt6); 6396 } 6397 return err; 6398 } 6399 6400 static void 6401 mlxsw_sp_fib6_entry_nexthop_del(struct mlxsw_sp *mlxsw_sp, 6402 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 6403 struct mlxsw_sp_fib6_entry *fib6_entry, 6404 struct fib6_info **rt_arr, unsigned int nrt6) 6405 { 6406 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 6407 int i; 6408 6409 for (i = 0; i < nrt6; i++) { 6410 mlxsw_sp_rt6 = mlxsw_sp_fib6_entry_rt_find(fib6_entry, 6411 rt_arr[i]); 6412 if (WARN_ON_ONCE(!mlxsw_sp_rt6)) 6413 continue; 6414 6415 fib6_entry->nrt6--; 6416 list_del(&mlxsw_sp_rt6->list); 6417 mlxsw_sp_rt6_destroy(mlxsw_sp_rt6); 6418 } 6419 6420 mlxsw_sp_nexthop6_group_update(mlxsw_sp, op_ctx, fib6_entry); 6421 } 6422 6423 static void mlxsw_sp_fib6_entry_type_set(struct mlxsw_sp *mlxsw_sp, 6424 struct mlxsw_sp_fib_entry *fib_entry, 6425 const struct fib6_info *rt) 6426 { 6427 if (rt->fib6_flags & (RTF_LOCAL | RTF_ANYCAST)) 6428 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP; 6429 else if (rt->fib6_type == RTN_BLACKHOLE) 6430 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_BLACKHOLE; 6431 else if (rt->fib6_flags & RTF_REJECT) 6432 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_UNREACHABLE; 6433 else if (fib_entry->nh_group->nhgi->gateway) 6434 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_REMOTE; 6435 else 6436 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL; 6437 } 6438 6439 static void 6440 mlxsw_sp_fib6_entry_rt_destroy_all(struct mlxsw_sp_fib6_entry *fib6_entry) 6441 { 6442 struct mlxsw_sp_rt6 *mlxsw_sp_rt6, *tmp; 6443 6444 list_for_each_entry_safe(mlxsw_sp_rt6, tmp, &fib6_entry->rt6_list, 6445 list) { 6446 fib6_entry->nrt6--; 6447 list_del(&mlxsw_sp_rt6->list); 6448 mlxsw_sp_rt6_destroy(mlxsw_sp_rt6); 6449 } 6450 } 6451 6452 static struct mlxsw_sp_fib6_entry * 6453 mlxsw_sp_fib6_entry_create(struct mlxsw_sp *mlxsw_sp, 6454 struct mlxsw_sp_fib_node *fib_node, 6455 struct fib6_info **rt_arr, unsigned int nrt6) 6456 { 6457 struct mlxsw_sp_fib6_entry *fib6_entry; 6458 struct mlxsw_sp_fib_entry *fib_entry; 6459 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 6460 int err, i; 6461 6462 fib6_entry = kzalloc(sizeof(*fib6_entry), GFP_KERNEL); 6463 if (!fib6_entry) 6464 return ERR_PTR(-ENOMEM); 6465 fib_entry = &fib6_entry->common; 6466 6467 fib_entry->priv = mlxsw_sp_fib_entry_priv_create(fib_node->fib->ll_ops); 6468 if (IS_ERR(fib_entry->priv)) { 6469 err = PTR_ERR(fib_entry->priv); 6470 goto err_fib_entry_priv_create; 6471 } 6472 6473 INIT_LIST_HEAD(&fib6_entry->rt6_list); 6474 6475 for (i = 0; i < nrt6; i++) { 6476 mlxsw_sp_rt6 = mlxsw_sp_rt6_create(rt_arr[i]); 6477 if (IS_ERR(mlxsw_sp_rt6)) { 6478 err = PTR_ERR(mlxsw_sp_rt6); 6479 goto err_rt6_create; 6480 } 6481 list_add_tail(&mlxsw_sp_rt6->list, &fib6_entry->rt6_list); 6482 fib6_entry->nrt6++; 6483 } 6484 6485 err = mlxsw_sp_nexthop6_group_get(mlxsw_sp, fib6_entry); 6486 if (err) 6487 goto err_nexthop6_group_get; 6488 6489 err = mlxsw_sp_nexthop_group_vr_link(fib_entry->nh_group, 6490 fib_node->fib); 6491 if (err) 6492 goto err_nexthop_group_vr_link; 6493 6494 mlxsw_sp_fib6_entry_type_set(mlxsw_sp, fib_entry, rt_arr[0]); 6495 6496 fib_entry->fib_node = fib_node; 6497 6498 return fib6_entry; 6499 6500 err_nexthop_group_vr_link: 6501 mlxsw_sp_nexthop6_group_put(mlxsw_sp, fib_entry); 6502 err_nexthop6_group_get: 6503 i = nrt6; 6504 err_rt6_create: 6505 for (i--; i >= 0; i--) { 6506 fib6_entry->nrt6--; 6507 mlxsw_sp_rt6 = list_last_entry(&fib6_entry->rt6_list, 6508 struct mlxsw_sp_rt6, list); 6509 list_del(&mlxsw_sp_rt6->list); 6510 mlxsw_sp_rt6_destroy(mlxsw_sp_rt6); 6511 } 6512 mlxsw_sp_fib_entry_priv_put(fib_entry->priv); 6513 err_fib_entry_priv_create: 6514 kfree(fib6_entry); 6515 return ERR_PTR(err); 6516 } 6517 6518 static void mlxsw_sp_fib6_entry_destroy(struct mlxsw_sp *mlxsw_sp, 6519 struct mlxsw_sp_fib6_entry *fib6_entry) 6520 { 6521 struct mlxsw_sp_fib_node *fib_node = fib6_entry->common.fib_node; 6522 6523 mlxsw_sp_nexthop_group_vr_unlink(fib6_entry->common.nh_group, 6524 fib_node->fib); 6525 mlxsw_sp_nexthop6_group_put(mlxsw_sp, &fib6_entry->common); 6526 mlxsw_sp_fib6_entry_rt_destroy_all(fib6_entry); 6527 WARN_ON(fib6_entry->nrt6); 6528 mlxsw_sp_fib_entry_priv_put(fib6_entry->common.priv); 6529 kfree(fib6_entry); 6530 } 6531 6532 static struct mlxsw_sp_fib6_entry * 6533 mlxsw_sp_fib6_entry_lookup(struct mlxsw_sp *mlxsw_sp, 6534 const struct fib6_info *rt) 6535 { 6536 struct mlxsw_sp_fib6_entry *fib6_entry; 6537 struct mlxsw_sp_fib_node *fib_node; 6538 struct mlxsw_sp_fib *fib; 6539 struct fib6_info *cmp_rt; 6540 struct mlxsw_sp_vr *vr; 6541 6542 vr = mlxsw_sp_vr_find(mlxsw_sp, rt->fib6_table->tb6_id); 6543 if (!vr) 6544 return NULL; 6545 fib = mlxsw_sp_vr_fib(vr, MLXSW_SP_L3_PROTO_IPV6); 6546 6547 fib_node = mlxsw_sp_fib_node_lookup(fib, &rt->fib6_dst.addr, 6548 sizeof(rt->fib6_dst.addr), 6549 rt->fib6_dst.plen); 6550 if (!fib_node) 6551 return NULL; 6552 6553 fib6_entry = container_of(fib_node->fib_entry, 6554 struct mlxsw_sp_fib6_entry, common); 6555 cmp_rt = mlxsw_sp_fib6_entry_rt(fib6_entry); 6556 if (rt->fib6_table->tb6_id == cmp_rt->fib6_table->tb6_id && 6557 rt->fib6_metric == cmp_rt->fib6_metric && 6558 mlxsw_sp_fib6_entry_rt_find(fib6_entry, rt)) 6559 return fib6_entry; 6560 6561 return NULL; 6562 } 6563 6564 static bool mlxsw_sp_fib6_allow_replace(struct mlxsw_sp_fib6_entry *fib6_entry) 6565 { 6566 struct mlxsw_sp_fib_node *fib_node = fib6_entry->common.fib_node; 6567 struct mlxsw_sp_fib6_entry *fib6_replaced; 6568 struct fib6_info *rt, *rt_replaced; 6569 6570 if (!fib_node->fib_entry) 6571 return true; 6572 6573 fib6_replaced = container_of(fib_node->fib_entry, 6574 struct mlxsw_sp_fib6_entry, 6575 common); 6576 rt = mlxsw_sp_fib6_entry_rt(fib6_entry); 6577 rt_replaced = mlxsw_sp_fib6_entry_rt(fib6_replaced); 6578 if (rt->fib6_table->tb6_id == RT_TABLE_MAIN && 6579 rt_replaced->fib6_table->tb6_id == RT_TABLE_LOCAL) 6580 return false; 6581 6582 return true; 6583 } 6584 6585 static int mlxsw_sp_router_fib6_replace(struct mlxsw_sp *mlxsw_sp, 6586 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 6587 struct fib6_info **rt_arr, unsigned int nrt6) 6588 { 6589 struct mlxsw_sp_fib6_entry *fib6_entry, *fib6_replaced; 6590 struct mlxsw_sp_fib_entry *replaced; 6591 struct mlxsw_sp_fib_node *fib_node; 6592 struct fib6_info *rt = rt_arr[0]; 6593 int err; 6594 6595 if (mlxsw_sp->router->aborted) 6596 return 0; 6597 6598 if (rt->fib6_src.plen) 6599 return -EINVAL; 6600 6601 if (mlxsw_sp_fib6_rt_should_ignore(rt)) 6602 return 0; 6603 6604 fib_node = mlxsw_sp_fib_node_get(mlxsw_sp, rt->fib6_table->tb6_id, 6605 &rt->fib6_dst.addr, 6606 sizeof(rt->fib6_dst.addr), 6607 rt->fib6_dst.plen, 6608 MLXSW_SP_L3_PROTO_IPV6); 6609 if (IS_ERR(fib_node)) 6610 return PTR_ERR(fib_node); 6611 6612 fib6_entry = mlxsw_sp_fib6_entry_create(mlxsw_sp, fib_node, rt_arr, 6613 nrt6); 6614 if (IS_ERR(fib6_entry)) { 6615 err = PTR_ERR(fib6_entry); 6616 goto err_fib6_entry_create; 6617 } 6618 6619 if (!mlxsw_sp_fib6_allow_replace(fib6_entry)) { 6620 mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry); 6621 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 6622 return 0; 6623 } 6624 6625 replaced = fib_node->fib_entry; 6626 err = mlxsw_sp_fib_node_entry_link(mlxsw_sp, op_ctx, &fib6_entry->common); 6627 if (err) 6628 goto err_fib_node_entry_link; 6629 6630 /* Nothing to replace */ 6631 if (!replaced) 6632 return 0; 6633 6634 mlxsw_sp_fib_entry_hw_flags_clear(mlxsw_sp, replaced); 6635 fib6_replaced = container_of(replaced, struct mlxsw_sp_fib6_entry, 6636 common); 6637 mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_replaced); 6638 6639 return 0; 6640 6641 err_fib_node_entry_link: 6642 fib_node->fib_entry = replaced; 6643 mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry); 6644 err_fib6_entry_create: 6645 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 6646 return err; 6647 } 6648 6649 static int mlxsw_sp_router_fib6_append(struct mlxsw_sp *mlxsw_sp, 6650 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 6651 struct fib6_info **rt_arr, unsigned int nrt6) 6652 { 6653 struct mlxsw_sp_fib6_entry *fib6_entry; 6654 struct mlxsw_sp_fib_node *fib_node; 6655 struct fib6_info *rt = rt_arr[0]; 6656 int err; 6657 6658 if (mlxsw_sp->router->aborted) 6659 return 0; 6660 6661 if (rt->fib6_src.plen) 6662 return -EINVAL; 6663 6664 if (mlxsw_sp_fib6_rt_should_ignore(rt)) 6665 return 0; 6666 6667 fib_node = mlxsw_sp_fib_node_get(mlxsw_sp, rt->fib6_table->tb6_id, 6668 &rt->fib6_dst.addr, 6669 sizeof(rt->fib6_dst.addr), 6670 rt->fib6_dst.plen, 6671 MLXSW_SP_L3_PROTO_IPV6); 6672 if (IS_ERR(fib_node)) 6673 return PTR_ERR(fib_node); 6674 6675 if (WARN_ON_ONCE(!fib_node->fib_entry)) { 6676 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 6677 return -EINVAL; 6678 } 6679 6680 fib6_entry = container_of(fib_node->fib_entry, 6681 struct mlxsw_sp_fib6_entry, common); 6682 err = mlxsw_sp_fib6_entry_nexthop_add(mlxsw_sp, op_ctx, fib6_entry, rt_arr, nrt6); 6683 if (err) 6684 goto err_fib6_entry_nexthop_add; 6685 6686 return 0; 6687 6688 err_fib6_entry_nexthop_add: 6689 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 6690 return err; 6691 } 6692 6693 static int mlxsw_sp_router_fib6_del(struct mlxsw_sp *mlxsw_sp, 6694 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 6695 struct fib6_info **rt_arr, unsigned int nrt6) 6696 { 6697 struct mlxsw_sp_fib6_entry *fib6_entry; 6698 struct mlxsw_sp_fib_node *fib_node; 6699 struct fib6_info *rt = rt_arr[0]; 6700 int err; 6701 6702 if (mlxsw_sp->router->aborted) 6703 return 0; 6704 6705 if (mlxsw_sp_fib6_rt_should_ignore(rt)) 6706 return 0; 6707 6708 /* Multipath routes are first added to the FIB trie and only then 6709 * notified. If we vetoed the addition, we will get a delete 6710 * notification for a route we do not have. Therefore, do not warn if 6711 * route was not found. 6712 */ 6713 fib6_entry = mlxsw_sp_fib6_entry_lookup(mlxsw_sp, rt); 6714 if (!fib6_entry) 6715 return 0; 6716 6717 /* If not all the nexthops are deleted, then only reduce the nexthop 6718 * group. 6719 */ 6720 if (nrt6 != fib6_entry->nrt6) { 6721 mlxsw_sp_fib6_entry_nexthop_del(mlxsw_sp, op_ctx, fib6_entry, rt_arr, nrt6); 6722 return 0; 6723 } 6724 6725 fib_node = fib6_entry->common.fib_node; 6726 6727 err = __mlxsw_sp_fib_node_entry_unlink(mlxsw_sp, op_ctx, &fib6_entry->common); 6728 mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry); 6729 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 6730 return err; 6731 } 6732 6733 static int __mlxsw_sp_router_set_abort_trap(struct mlxsw_sp *mlxsw_sp, 6734 enum mlxsw_sp_l3proto proto, 6735 u8 tree_id) 6736 { 6737 const struct mlxsw_sp_router_ll_ops *ll_ops = mlxsw_sp->router->proto_ll_ops[proto]; 6738 enum mlxsw_reg_ralxx_protocol ralxx_proto = 6739 (enum mlxsw_reg_ralxx_protocol) proto; 6740 struct mlxsw_sp_fib_entry_priv *priv; 6741 char xralta_pl[MLXSW_REG_XRALTA_LEN]; 6742 char xralst_pl[MLXSW_REG_XRALST_LEN]; 6743 int i, err; 6744 6745 mlxsw_reg_xralta_pack(xralta_pl, true, ralxx_proto, tree_id); 6746 err = ll_ops->ralta_write(mlxsw_sp, xralta_pl); 6747 if (err) 6748 return err; 6749 6750 mlxsw_reg_xralst_pack(xralst_pl, 0xff, tree_id); 6751 err = ll_ops->ralst_write(mlxsw_sp, xralst_pl); 6752 if (err) 6753 return err; 6754 6755 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) { 6756 struct mlxsw_sp_fib_entry_op_ctx *op_ctx = mlxsw_sp->router->ll_op_ctx; 6757 struct mlxsw_sp_vr *vr = &mlxsw_sp->router->vrs[i]; 6758 char xraltb_pl[MLXSW_REG_XRALTB_LEN]; 6759 6760 mlxsw_sp_fib_entry_op_ctx_clear(op_ctx); 6761 mlxsw_reg_xraltb_pack(xraltb_pl, vr->id, ralxx_proto, tree_id); 6762 err = ll_ops->raltb_write(mlxsw_sp, xraltb_pl); 6763 if (err) 6764 return err; 6765 6766 priv = mlxsw_sp_fib_entry_priv_create(ll_ops); 6767 if (IS_ERR(priv)) 6768 return PTR_ERR(priv); 6769 6770 ll_ops->fib_entry_pack(op_ctx, proto, MLXSW_SP_FIB_ENTRY_OP_WRITE, 6771 vr->id, 0, NULL, priv); 6772 ll_ops->fib_entry_act_ip2me_pack(op_ctx); 6773 err = ll_ops->fib_entry_commit(mlxsw_sp, op_ctx, NULL); 6774 mlxsw_sp_fib_entry_priv_put(priv); 6775 if (err) 6776 return err; 6777 } 6778 6779 return 0; 6780 } 6781 6782 static struct mlxsw_sp_mr_table * 6783 mlxsw_sp_router_fibmr_family_to_table(struct mlxsw_sp_vr *vr, int family) 6784 { 6785 if (family == RTNL_FAMILY_IPMR) 6786 return vr->mr_table[MLXSW_SP_L3_PROTO_IPV4]; 6787 else 6788 return vr->mr_table[MLXSW_SP_L3_PROTO_IPV6]; 6789 } 6790 6791 static int mlxsw_sp_router_fibmr_add(struct mlxsw_sp *mlxsw_sp, 6792 struct mfc_entry_notifier_info *men_info, 6793 bool replace) 6794 { 6795 struct mlxsw_sp_mr_table *mrt; 6796 struct mlxsw_sp_vr *vr; 6797 6798 if (mlxsw_sp->router->aborted) 6799 return 0; 6800 6801 vr = mlxsw_sp_vr_get(mlxsw_sp, men_info->tb_id, NULL); 6802 if (IS_ERR(vr)) 6803 return PTR_ERR(vr); 6804 6805 mrt = mlxsw_sp_router_fibmr_family_to_table(vr, men_info->info.family); 6806 return mlxsw_sp_mr_route_add(mrt, men_info->mfc, replace); 6807 } 6808 6809 static void mlxsw_sp_router_fibmr_del(struct mlxsw_sp *mlxsw_sp, 6810 struct mfc_entry_notifier_info *men_info) 6811 { 6812 struct mlxsw_sp_mr_table *mrt; 6813 struct mlxsw_sp_vr *vr; 6814 6815 if (mlxsw_sp->router->aborted) 6816 return; 6817 6818 vr = mlxsw_sp_vr_find(mlxsw_sp, men_info->tb_id); 6819 if (WARN_ON(!vr)) 6820 return; 6821 6822 mrt = mlxsw_sp_router_fibmr_family_to_table(vr, men_info->info.family); 6823 mlxsw_sp_mr_route_del(mrt, men_info->mfc); 6824 mlxsw_sp_vr_put(mlxsw_sp, vr); 6825 } 6826 6827 static int 6828 mlxsw_sp_router_fibmr_vif_add(struct mlxsw_sp *mlxsw_sp, 6829 struct vif_entry_notifier_info *ven_info) 6830 { 6831 struct mlxsw_sp_mr_table *mrt; 6832 struct mlxsw_sp_rif *rif; 6833 struct mlxsw_sp_vr *vr; 6834 6835 if (mlxsw_sp->router->aborted) 6836 return 0; 6837 6838 vr = mlxsw_sp_vr_get(mlxsw_sp, ven_info->tb_id, NULL); 6839 if (IS_ERR(vr)) 6840 return PTR_ERR(vr); 6841 6842 mrt = mlxsw_sp_router_fibmr_family_to_table(vr, ven_info->info.family); 6843 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, ven_info->dev); 6844 return mlxsw_sp_mr_vif_add(mrt, ven_info->dev, 6845 ven_info->vif_index, 6846 ven_info->vif_flags, rif); 6847 } 6848 6849 static void 6850 mlxsw_sp_router_fibmr_vif_del(struct mlxsw_sp *mlxsw_sp, 6851 struct vif_entry_notifier_info *ven_info) 6852 { 6853 struct mlxsw_sp_mr_table *mrt; 6854 struct mlxsw_sp_vr *vr; 6855 6856 if (mlxsw_sp->router->aborted) 6857 return; 6858 6859 vr = mlxsw_sp_vr_find(mlxsw_sp, ven_info->tb_id); 6860 if (WARN_ON(!vr)) 6861 return; 6862 6863 mrt = mlxsw_sp_router_fibmr_family_to_table(vr, ven_info->info.family); 6864 mlxsw_sp_mr_vif_del(mrt, ven_info->vif_index); 6865 mlxsw_sp_vr_put(mlxsw_sp, vr); 6866 } 6867 6868 static int mlxsw_sp_router_set_abort_trap(struct mlxsw_sp *mlxsw_sp) 6869 { 6870 enum mlxsw_sp_l3proto proto = MLXSW_SP_L3_PROTO_IPV4; 6871 int err; 6872 6873 err = __mlxsw_sp_router_set_abort_trap(mlxsw_sp, proto, 6874 MLXSW_SP_LPM_TREE_MIN); 6875 if (err) 6876 return err; 6877 6878 /* The multicast router code does not need an abort trap as by default, 6879 * packets that don't match any routes are trapped to the CPU. 6880 */ 6881 6882 proto = MLXSW_SP_L3_PROTO_IPV6; 6883 return __mlxsw_sp_router_set_abort_trap(mlxsw_sp, proto, 6884 MLXSW_SP_LPM_TREE_MIN + 1); 6885 } 6886 6887 static void mlxsw_sp_fib4_node_flush(struct mlxsw_sp *mlxsw_sp, 6888 struct mlxsw_sp_fib_node *fib_node) 6889 { 6890 struct mlxsw_sp_fib4_entry *fib4_entry; 6891 6892 fib4_entry = container_of(fib_node->fib_entry, 6893 struct mlxsw_sp_fib4_entry, common); 6894 mlxsw_sp_fib_node_entry_unlink(mlxsw_sp, fib_node->fib_entry); 6895 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry); 6896 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 6897 } 6898 6899 static void mlxsw_sp_fib6_node_flush(struct mlxsw_sp *mlxsw_sp, 6900 struct mlxsw_sp_fib_node *fib_node) 6901 { 6902 struct mlxsw_sp_fib6_entry *fib6_entry; 6903 6904 fib6_entry = container_of(fib_node->fib_entry, 6905 struct mlxsw_sp_fib6_entry, common); 6906 mlxsw_sp_fib_node_entry_unlink(mlxsw_sp, fib_node->fib_entry); 6907 mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry); 6908 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 6909 } 6910 6911 static void mlxsw_sp_fib_node_flush(struct mlxsw_sp *mlxsw_sp, 6912 struct mlxsw_sp_fib_node *fib_node) 6913 { 6914 switch (fib_node->fib->proto) { 6915 case MLXSW_SP_L3_PROTO_IPV4: 6916 mlxsw_sp_fib4_node_flush(mlxsw_sp, fib_node); 6917 break; 6918 case MLXSW_SP_L3_PROTO_IPV6: 6919 mlxsw_sp_fib6_node_flush(mlxsw_sp, fib_node); 6920 break; 6921 } 6922 } 6923 6924 static void mlxsw_sp_vr_fib_flush(struct mlxsw_sp *mlxsw_sp, 6925 struct mlxsw_sp_vr *vr, 6926 enum mlxsw_sp_l3proto proto) 6927 { 6928 struct mlxsw_sp_fib *fib = mlxsw_sp_vr_fib(vr, proto); 6929 struct mlxsw_sp_fib_node *fib_node, *tmp; 6930 6931 list_for_each_entry_safe(fib_node, tmp, &fib->node_list, list) { 6932 bool do_break = &tmp->list == &fib->node_list; 6933 6934 mlxsw_sp_fib_node_flush(mlxsw_sp, fib_node); 6935 if (do_break) 6936 break; 6937 } 6938 } 6939 6940 static void mlxsw_sp_router_fib_flush(struct mlxsw_sp *mlxsw_sp) 6941 { 6942 int i, j; 6943 6944 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) { 6945 struct mlxsw_sp_vr *vr = &mlxsw_sp->router->vrs[i]; 6946 6947 if (!mlxsw_sp_vr_is_used(vr)) 6948 continue; 6949 6950 for (j = 0; j < MLXSW_SP_L3_PROTO_MAX; j++) 6951 mlxsw_sp_mr_table_flush(vr->mr_table[j]); 6952 mlxsw_sp_vr_fib_flush(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV4); 6953 6954 /* If virtual router was only used for IPv4, then it's no 6955 * longer used. 6956 */ 6957 if (!mlxsw_sp_vr_is_used(vr)) 6958 continue; 6959 mlxsw_sp_vr_fib_flush(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV6); 6960 } 6961 6962 /* After flushing all the routes, it is not possible anyone is still 6963 * using the adjacency index that is discarding packets, so free it in 6964 * case it was allocated. 6965 */ 6966 if (!mlxsw_sp->router->adj_discard_index_valid) 6967 return; 6968 mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 1, 6969 mlxsw_sp->router->adj_discard_index); 6970 mlxsw_sp->router->adj_discard_index_valid = false; 6971 } 6972 6973 static void mlxsw_sp_router_fib_abort(struct mlxsw_sp *mlxsw_sp) 6974 { 6975 int err; 6976 6977 if (mlxsw_sp->router->aborted) 6978 return; 6979 dev_warn(mlxsw_sp->bus_info->dev, "FIB abort triggered. Note that FIB entries are no longer being offloaded to this device.\n"); 6980 mlxsw_sp_router_fib_flush(mlxsw_sp); 6981 mlxsw_sp->router->aborted = true; 6982 err = mlxsw_sp_router_set_abort_trap(mlxsw_sp); 6983 if (err) 6984 dev_warn(mlxsw_sp->bus_info->dev, "Failed to set abort trap.\n"); 6985 } 6986 6987 struct mlxsw_sp_fib6_event { 6988 struct fib6_info **rt_arr; 6989 unsigned int nrt6; 6990 }; 6991 6992 struct mlxsw_sp_fib_event { 6993 struct list_head list; /* node in fib queue */ 6994 union { 6995 struct mlxsw_sp_fib6_event fib6_event; 6996 struct fib_entry_notifier_info fen_info; 6997 struct fib_rule_notifier_info fr_info; 6998 struct fib_nh_notifier_info fnh_info; 6999 struct mfc_entry_notifier_info men_info; 7000 struct vif_entry_notifier_info ven_info; 7001 }; 7002 struct mlxsw_sp *mlxsw_sp; 7003 unsigned long event; 7004 int family; 7005 }; 7006 7007 static int 7008 mlxsw_sp_router_fib6_event_init(struct mlxsw_sp_fib6_event *fib6_event, 7009 struct fib6_entry_notifier_info *fen6_info) 7010 { 7011 struct fib6_info *rt = fen6_info->rt; 7012 struct fib6_info **rt_arr; 7013 struct fib6_info *iter; 7014 unsigned int nrt6; 7015 int i = 0; 7016 7017 nrt6 = fen6_info->nsiblings + 1; 7018 7019 rt_arr = kcalloc(nrt6, sizeof(struct fib6_info *), GFP_ATOMIC); 7020 if (!rt_arr) 7021 return -ENOMEM; 7022 7023 fib6_event->rt_arr = rt_arr; 7024 fib6_event->nrt6 = nrt6; 7025 7026 rt_arr[0] = rt; 7027 fib6_info_hold(rt); 7028 7029 if (!fen6_info->nsiblings) 7030 return 0; 7031 7032 list_for_each_entry(iter, &rt->fib6_siblings, fib6_siblings) { 7033 if (i == fen6_info->nsiblings) 7034 break; 7035 7036 rt_arr[i + 1] = iter; 7037 fib6_info_hold(iter); 7038 i++; 7039 } 7040 WARN_ON_ONCE(i != fen6_info->nsiblings); 7041 7042 return 0; 7043 } 7044 7045 static void 7046 mlxsw_sp_router_fib6_event_fini(struct mlxsw_sp_fib6_event *fib6_event) 7047 { 7048 int i; 7049 7050 for (i = 0; i < fib6_event->nrt6; i++) 7051 mlxsw_sp_rt6_release(fib6_event->rt_arr[i]); 7052 kfree(fib6_event->rt_arr); 7053 } 7054 7055 static void mlxsw_sp_router_fib4_event_process(struct mlxsw_sp *mlxsw_sp, 7056 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 7057 struct mlxsw_sp_fib_event *fib_event) 7058 { 7059 int err; 7060 7061 mlxsw_sp_span_respin(mlxsw_sp); 7062 7063 switch (fib_event->event) { 7064 case FIB_EVENT_ENTRY_REPLACE: 7065 err = mlxsw_sp_router_fib4_replace(mlxsw_sp, op_ctx, &fib_event->fen_info); 7066 if (err) { 7067 mlxsw_sp_fib_entry_op_ctx_priv_put_all(op_ctx); 7068 mlxsw_sp_router_fib_abort(mlxsw_sp); 7069 mlxsw_sp_fib4_offload_failed_flag_set(mlxsw_sp, 7070 &fib_event->fen_info); 7071 } 7072 fib_info_put(fib_event->fen_info.fi); 7073 break; 7074 case FIB_EVENT_ENTRY_DEL: 7075 err = mlxsw_sp_router_fib4_del(mlxsw_sp, op_ctx, &fib_event->fen_info); 7076 if (err) 7077 mlxsw_sp_fib_entry_op_ctx_priv_put_all(op_ctx); 7078 fib_info_put(fib_event->fen_info.fi); 7079 break; 7080 case FIB_EVENT_NH_ADD: 7081 case FIB_EVENT_NH_DEL: 7082 mlxsw_sp_nexthop4_event(mlxsw_sp, fib_event->event, fib_event->fnh_info.fib_nh); 7083 fib_info_put(fib_event->fnh_info.fib_nh->nh_parent); 7084 break; 7085 } 7086 } 7087 7088 static void mlxsw_sp_router_fib6_event_process(struct mlxsw_sp *mlxsw_sp, 7089 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 7090 struct mlxsw_sp_fib_event *fib_event) 7091 { 7092 struct mlxsw_sp_fib6_event *fib6_event = &fib_event->fib6_event; 7093 int err; 7094 7095 mlxsw_sp_span_respin(mlxsw_sp); 7096 7097 switch (fib_event->event) { 7098 case FIB_EVENT_ENTRY_REPLACE: 7099 err = mlxsw_sp_router_fib6_replace(mlxsw_sp, op_ctx, fib_event->fib6_event.rt_arr, 7100 fib_event->fib6_event.nrt6); 7101 if (err) { 7102 mlxsw_sp_fib_entry_op_ctx_priv_put_all(op_ctx); 7103 mlxsw_sp_router_fib_abort(mlxsw_sp); 7104 mlxsw_sp_fib6_offload_failed_flag_set(mlxsw_sp, 7105 fib6_event->rt_arr, 7106 fib6_event->nrt6); 7107 } 7108 mlxsw_sp_router_fib6_event_fini(&fib_event->fib6_event); 7109 break; 7110 case FIB_EVENT_ENTRY_APPEND: 7111 err = mlxsw_sp_router_fib6_append(mlxsw_sp, op_ctx, fib_event->fib6_event.rt_arr, 7112 fib_event->fib6_event.nrt6); 7113 if (err) { 7114 mlxsw_sp_fib_entry_op_ctx_priv_put_all(op_ctx); 7115 mlxsw_sp_router_fib_abort(mlxsw_sp); 7116 mlxsw_sp_fib6_offload_failed_flag_set(mlxsw_sp, 7117 fib6_event->rt_arr, 7118 fib6_event->nrt6); 7119 } 7120 mlxsw_sp_router_fib6_event_fini(&fib_event->fib6_event); 7121 break; 7122 case FIB_EVENT_ENTRY_DEL: 7123 err = mlxsw_sp_router_fib6_del(mlxsw_sp, op_ctx, fib_event->fib6_event.rt_arr, 7124 fib_event->fib6_event.nrt6); 7125 if (err) 7126 mlxsw_sp_fib_entry_op_ctx_priv_put_all(op_ctx); 7127 mlxsw_sp_router_fib6_event_fini(&fib_event->fib6_event); 7128 break; 7129 } 7130 } 7131 7132 static void mlxsw_sp_router_fibmr_event_process(struct mlxsw_sp *mlxsw_sp, 7133 struct mlxsw_sp_fib_event *fib_event) 7134 { 7135 bool replace; 7136 int err; 7137 7138 rtnl_lock(); 7139 mutex_lock(&mlxsw_sp->router->lock); 7140 switch (fib_event->event) { 7141 case FIB_EVENT_ENTRY_REPLACE: 7142 case FIB_EVENT_ENTRY_ADD: 7143 replace = fib_event->event == FIB_EVENT_ENTRY_REPLACE; 7144 7145 err = mlxsw_sp_router_fibmr_add(mlxsw_sp, &fib_event->men_info, replace); 7146 if (err) 7147 mlxsw_sp_router_fib_abort(mlxsw_sp); 7148 mr_cache_put(fib_event->men_info.mfc); 7149 break; 7150 case FIB_EVENT_ENTRY_DEL: 7151 mlxsw_sp_router_fibmr_del(mlxsw_sp, &fib_event->men_info); 7152 mr_cache_put(fib_event->men_info.mfc); 7153 break; 7154 case FIB_EVENT_VIF_ADD: 7155 err = mlxsw_sp_router_fibmr_vif_add(mlxsw_sp, 7156 &fib_event->ven_info); 7157 if (err) 7158 mlxsw_sp_router_fib_abort(mlxsw_sp); 7159 dev_put(fib_event->ven_info.dev); 7160 break; 7161 case FIB_EVENT_VIF_DEL: 7162 mlxsw_sp_router_fibmr_vif_del(mlxsw_sp, &fib_event->ven_info); 7163 dev_put(fib_event->ven_info.dev); 7164 break; 7165 } 7166 mutex_unlock(&mlxsw_sp->router->lock); 7167 rtnl_unlock(); 7168 } 7169 7170 static void mlxsw_sp_router_fib_event_work(struct work_struct *work) 7171 { 7172 struct mlxsw_sp_router *router = container_of(work, struct mlxsw_sp_router, fib_event_work); 7173 struct mlxsw_sp_fib_entry_op_ctx *op_ctx = router->ll_op_ctx; 7174 struct mlxsw_sp *mlxsw_sp = router->mlxsw_sp; 7175 struct mlxsw_sp_fib_event *next_fib_event; 7176 struct mlxsw_sp_fib_event *fib_event; 7177 int last_family = AF_UNSPEC; 7178 LIST_HEAD(fib_event_queue); 7179 7180 spin_lock_bh(&router->fib_event_queue_lock); 7181 list_splice_init(&router->fib_event_queue, &fib_event_queue); 7182 spin_unlock_bh(&router->fib_event_queue_lock); 7183 7184 /* Router lock is held here to make sure per-instance 7185 * operation context is not used in between FIB4/6 events 7186 * processing. 7187 */ 7188 mutex_lock(&router->lock); 7189 mlxsw_sp_fib_entry_op_ctx_clear(op_ctx); 7190 list_for_each_entry_safe(fib_event, next_fib_event, 7191 &fib_event_queue, list) { 7192 /* Check if the next entry in the queue exists and it is 7193 * of the same type (family and event) as the currect one. 7194 * In that case it is permitted to do the bulking 7195 * of multiple FIB entries to a single register write. 7196 */ 7197 op_ctx->bulk_ok = !list_is_last(&fib_event->list, &fib_event_queue) && 7198 fib_event->family == next_fib_event->family && 7199 fib_event->event == next_fib_event->event; 7200 op_ctx->event = fib_event->event; 7201 7202 /* In case family of this and the previous entry are different, context 7203 * reinitialization is going to be needed now, indicate that. 7204 * Note that since last_family is initialized to AF_UNSPEC, this is always 7205 * going to happen for the first entry processed in the work. 7206 */ 7207 if (fib_event->family != last_family) 7208 op_ctx->initialized = false; 7209 7210 switch (fib_event->family) { 7211 case AF_INET: 7212 mlxsw_sp_router_fib4_event_process(mlxsw_sp, op_ctx, 7213 fib_event); 7214 break; 7215 case AF_INET6: 7216 mlxsw_sp_router_fib6_event_process(mlxsw_sp, op_ctx, 7217 fib_event); 7218 break; 7219 case RTNL_FAMILY_IP6MR: 7220 case RTNL_FAMILY_IPMR: 7221 /* Unlock here as inside FIBMR the lock is taken again 7222 * under RTNL. The per-instance operation context 7223 * is not used by FIBMR. 7224 */ 7225 mutex_unlock(&router->lock); 7226 mlxsw_sp_router_fibmr_event_process(mlxsw_sp, 7227 fib_event); 7228 mutex_lock(&router->lock); 7229 break; 7230 default: 7231 WARN_ON_ONCE(1); 7232 } 7233 last_family = fib_event->family; 7234 kfree(fib_event); 7235 cond_resched(); 7236 } 7237 WARN_ON_ONCE(!list_empty(&router->ll_op_ctx->fib_entry_priv_list)); 7238 mutex_unlock(&router->lock); 7239 } 7240 7241 static void mlxsw_sp_router_fib4_event(struct mlxsw_sp_fib_event *fib_event, 7242 struct fib_notifier_info *info) 7243 { 7244 struct fib_entry_notifier_info *fen_info; 7245 struct fib_nh_notifier_info *fnh_info; 7246 7247 switch (fib_event->event) { 7248 case FIB_EVENT_ENTRY_REPLACE: 7249 case FIB_EVENT_ENTRY_DEL: 7250 fen_info = container_of(info, struct fib_entry_notifier_info, 7251 info); 7252 fib_event->fen_info = *fen_info; 7253 /* Take reference on fib_info to prevent it from being 7254 * freed while event is queued. Release it afterwards. 7255 */ 7256 fib_info_hold(fib_event->fen_info.fi); 7257 break; 7258 case FIB_EVENT_NH_ADD: 7259 case FIB_EVENT_NH_DEL: 7260 fnh_info = container_of(info, struct fib_nh_notifier_info, 7261 info); 7262 fib_event->fnh_info = *fnh_info; 7263 fib_info_hold(fib_event->fnh_info.fib_nh->nh_parent); 7264 break; 7265 } 7266 } 7267 7268 static int mlxsw_sp_router_fib6_event(struct mlxsw_sp_fib_event *fib_event, 7269 struct fib_notifier_info *info) 7270 { 7271 struct fib6_entry_notifier_info *fen6_info; 7272 int err; 7273 7274 switch (fib_event->event) { 7275 case FIB_EVENT_ENTRY_REPLACE: 7276 case FIB_EVENT_ENTRY_APPEND: 7277 case FIB_EVENT_ENTRY_DEL: 7278 fen6_info = container_of(info, struct fib6_entry_notifier_info, 7279 info); 7280 err = mlxsw_sp_router_fib6_event_init(&fib_event->fib6_event, 7281 fen6_info); 7282 if (err) 7283 return err; 7284 break; 7285 } 7286 7287 return 0; 7288 } 7289 7290 static void 7291 mlxsw_sp_router_fibmr_event(struct mlxsw_sp_fib_event *fib_event, 7292 struct fib_notifier_info *info) 7293 { 7294 switch (fib_event->event) { 7295 case FIB_EVENT_ENTRY_REPLACE: 7296 case FIB_EVENT_ENTRY_ADD: 7297 case FIB_EVENT_ENTRY_DEL: 7298 memcpy(&fib_event->men_info, info, sizeof(fib_event->men_info)); 7299 mr_cache_hold(fib_event->men_info.mfc); 7300 break; 7301 case FIB_EVENT_VIF_ADD: 7302 case FIB_EVENT_VIF_DEL: 7303 memcpy(&fib_event->ven_info, info, sizeof(fib_event->ven_info)); 7304 dev_hold(fib_event->ven_info.dev); 7305 break; 7306 } 7307 } 7308 7309 static int mlxsw_sp_router_fib_rule_event(unsigned long event, 7310 struct fib_notifier_info *info, 7311 struct mlxsw_sp *mlxsw_sp) 7312 { 7313 struct netlink_ext_ack *extack = info->extack; 7314 struct fib_rule_notifier_info *fr_info; 7315 struct fib_rule *rule; 7316 int err = 0; 7317 7318 /* nothing to do at the moment */ 7319 if (event == FIB_EVENT_RULE_DEL) 7320 return 0; 7321 7322 if (mlxsw_sp->router->aborted) 7323 return 0; 7324 7325 fr_info = container_of(info, struct fib_rule_notifier_info, info); 7326 rule = fr_info->rule; 7327 7328 /* Rule only affects locally generated traffic */ 7329 if (rule->iifindex == mlxsw_sp_net(mlxsw_sp)->loopback_dev->ifindex) 7330 return 0; 7331 7332 switch (info->family) { 7333 case AF_INET: 7334 if (!fib4_rule_default(rule) && !rule->l3mdev) 7335 err = -EOPNOTSUPP; 7336 break; 7337 case AF_INET6: 7338 if (!fib6_rule_default(rule) && !rule->l3mdev) 7339 err = -EOPNOTSUPP; 7340 break; 7341 case RTNL_FAMILY_IPMR: 7342 if (!ipmr_rule_default(rule) && !rule->l3mdev) 7343 err = -EOPNOTSUPP; 7344 break; 7345 case RTNL_FAMILY_IP6MR: 7346 if (!ip6mr_rule_default(rule) && !rule->l3mdev) 7347 err = -EOPNOTSUPP; 7348 break; 7349 } 7350 7351 if (err < 0) 7352 NL_SET_ERR_MSG_MOD(extack, "FIB rules not supported"); 7353 7354 return err; 7355 } 7356 7357 /* Called with rcu_read_lock() */ 7358 static int mlxsw_sp_router_fib_event(struct notifier_block *nb, 7359 unsigned long event, void *ptr) 7360 { 7361 struct mlxsw_sp_fib_event *fib_event; 7362 struct fib_notifier_info *info = ptr; 7363 struct mlxsw_sp_router *router; 7364 int err; 7365 7366 if ((info->family != AF_INET && info->family != AF_INET6 && 7367 info->family != RTNL_FAMILY_IPMR && 7368 info->family != RTNL_FAMILY_IP6MR)) 7369 return NOTIFY_DONE; 7370 7371 router = container_of(nb, struct mlxsw_sp_router, fib_nb); 7372 7373 switch (event) { 7374 case FIB_EVENT_RULE_ADD: 7375 case FIB_EVENT_RULE_DEL: 7376 err = mlxsw_sp_router_fib_rule_event(event, info, 7377 router->mlxsw_sp); 7378 return notifier_from_errno(err); 7379 case FIB_EVENT_ENTRY_ADD: 7380 case FIB_EVENT_ENTRY_REPLACE: 7381 case FIB_EVENT_ENTRY_APPEND: 7382 if (router->aborted) { 7383 NL_SET_ERR_MSG_MOD(info->extack, "FIB offload was aborted. Not configuring route"); 7384 return notifier_from_errno(-EINVAL); 7385 } 7386 if (info->family == AF_INET) { 7387 struct fib_entry_notifier_info *fen_info = ptr; 7388 7389 if (fen_info->fi->fib_nh_is_v6) { 7390 NL_SET_ERR_MSG_MOD(info->extack, "IPv6 gateway with IPv4 route is not supported"); 7391 return notifier_from_errno(-EINVAL); 7392 } 7393 } 7394 break; 7395 } 7396 7397 fib_event = kzalloc(sizeof(*fib_event), GFP_ATOMIC); 7398 if (!fib_event) 7399 return NOTIFY_BAD; 7400 7401 fib_event->mlxsw_sp = router->mlxsw_sp; 7402 fib_event->event = event; 7403 fib_event->family = info->family; 7404 7405 switch (info->family) { 7406 case AF_INET: 7407 mlxsw_sp_router_fib4_event(fib_event, info); 7408 break; 7409 case AF_INET6: 7410 err = mlxsw_sp_router_fib6_event(fib_event, info); 7411 if (err) 7412 goto err_fib_event; 7413 break; 7414 case RTNL_FAMILY_IP6MR: 7415 case RTNL_FAMILY_IPMR: 7416 mlxsw_sp_router_fibmr_event(fib_event, info); 7417 break; 7418 } 7419 7420 /* Enqueue the event and trigger the work */ 7421 spin_lock_bh(&router->fib_event_queue_lock); 7422 list_add_tail(&fib_event->list, &router->fib_event_queue); 7423 spin_unlock_bh(&router->fib_event_queue_lock); 7424 mlxsw_core_schedule_work(&router->fib_event_work); 7425 7426 return NOTIFY_DONE; 7427 7428 err_fib_event: 7429 kfree(fib_event); 7430 return NOTIFY_BAD; 7431 } 7432 7433 static struct mlxsw_sp_rif * 7434 mlxsw_sp_rif_find_by_dev(const struct mlxsw_sp *mlxsw_sp, 7435 const struct net_device *dev) 7436 { 7437 int i; 7438 7439 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++) 7440 if (mlxsw_sp->router->rifs[i] && 7441 mlxsw_sp->router->rifs[i]->dev == dev) 7442 return mlxsw_sp->router->rifs[i]; 7443 7444 return NULL; 7445 } 7446 7447 bool mlxsw_sp_rif_exists(struct mlxsw_sp *mlxsw_sp, 7448 const struct net_device *dev) 7449 { 7450 struct mlxsw_sp_rif *rif; 7451 7452 mutex_lock(&mlxsw_sp->router->lock); 7453 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev); 7454 mutex_unlock(&mlxsw_sp->router->lock); 7455 7456 return rif; 7457 } 7458 7459 u16 mlxsw_sp_rif_vid(struct mlxsw_sp *mlxsw_sp, const struct net_device *dev) 7460 { 7461 struct mlxsw_sp_rif *rif; 7462 u16 vid = 0; 7463 7464 mutex_lock(&mlxsw_sp->router->lock); 7465 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev); 7466 if (!rif) 7467 goto out; 7468 7469 /* We only return the VID for VLAN RIFs. Otherwise we return an 7470 * invalid value (0). 7471 */ 7472 if (rif->ops->type != MLXSW_SP_RIF_TYPE_VLAN) 7473 goto out; 7474 7475 vid = mlxsw_sp_fid_8021q_vid(rif->fid); 7476 7477 out: 7478 mutex_unlock(&mlxsw_sp->router->lock); 7479 return vid; 7480 } 7481 7482 static int mlxsw_sp_router_rif_disable(struct mlxsw_sp *mlxsw_sp, u16 rif) 7483 { 7484 char ritr_pl[MLXSW_REG_RITR_LEN]; 7485 int err; 7486 7487 mlxsw_reg_ritr_rif_pack(ritr_pl, rif); 7488 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 7489 if (err) 7490 return err; 7491 7492 mlxsw_reg_ritr_enable_set(ritr_pl, false); 7493 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 7494 } 7495 7496 static void mlxsw_sp_router_rif_gone_sync(struct mlxsw_sp *mlxsw_sp, 7497 struct mlxsw_sp_rif *rif) 7498 { 7499 mlxsw_sp_router_rif_disable(mlxsw_sp, rif->rif_index); 7500 mlxsw_sp_nexthop_rif_gone_sync(mlxsw_sp, rif); 7501 mlxsw_sp_neigh_rif_gone_sync(mlxsw_sp, rif); 7502 } 7503 7504 static bool 7505 mlxsw_sp_rif_should_config(struct mlxsw_sp_rif *rif, struct net_device *dev, 7506 unsigned long event) 7507 { 7508 struct inet6_dev *inet6_dev; 7509 bool addr_list_empty = true; 7510 struct in_device *idev; 7511 7512 switch (event) { 7513 case NETDEV_UP: 7514 return rif == NULL; 7515 case NETDEV_DOWN: 7516 rcu_read_lock(); 7517 idev = __in_dev_get_rcu(dev); 7518 if (idev && idev->ifa_list) 7519 addr_list_empty = false; 7520 7521 inet6_dev = __in6_dev_get(dev); 7522 if (addr_list_empty && inet6_dev && 7523 !list_empty(&inet6_dev->addr_list)) 7524 addr_list_empty = false; 7525 rcu_read_unlock(); 7526 7527 /* macvlans do not have a RIF, but rather piggy back on the 7528 * RIF of their lower device. 7529 */ 7530 if (netif_is_macvlan(dev) && addr_list_empty) 7531 return true; 7532 7533 if (rif && addr_list_empty && 7534 !netif_is_l3_slave(rif->dev)) 7535 return true; 7536 /* It is possible we already removed the RIF ourselves 7537 * if it was assigned to a netdev that is now a bridge 7538 * or LAG slave. 7539 */ 7540 return false; 7541 } 7542 7543 return false; 7544 } 7545 7546 static enum mlxsw_sp_rif_type 7547 mlxsw_sp_dev_rif_type(const struct mlxsw_sp *mlxsw_sp, 7548 const struct net_device *dev) 7549 { 7550 enum mlxsw_sp_fid_type type; 7551 7552 if (mlxsw_sp_netdev_ipip_type(mlxsw_sp, dev, NULL)) 7553 return MLXSW_SP_RIF_TYPE_IPIP_LB; 7554 7555 /* Otherwise RIF type is derived from the type of the underlying FID. */ 7556 if (is_vlan_dev(dev) && netif_is_bridge_master(vlan_dev_real_dev(dev))) 7557 type = MLXSW_SP_FID_TYPE_8021Q; 7558 else if (netif_is_bridge_master(dev) && br_vlan_enabled(dev)) 7559 type = MLXSW_SP_FID_TYPE_8021Q; 7560 else if (netif_is_bridge_master(dev)) 7561 type = MLXSW_SP_FID_TYPE_8021D; 7562 else 7563 type = MLXSW_SP_FID_TYPE_RFID; 7564 7565 return mlxsw_sp_fid_type_rif_type(mlxsw_sp, type); 7566 } 7567 7568 static int mlxsw_sp_rif_index_alloc(struct mlxsw_sp *mlxsw_sp, u16 *p_rif_index) 7569 { 7570 int i; 7571 7572 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++) { 7573 if (!mlxsw_sp->router->rifs[i]) { 7574 *p_rif_index = i; 7575 return 0; 7576 } 7577 } 7578 7579 return -ENOBUFS; 7580 } 7581 7582 static struct mlxsw_sp_rif *mlxsw_sp_rif_alloc(size_t rif_size, u16 rif_index, 7583 u16 vr_id, 7584 struct net_device *l3_dev) 7585 { 7586 struct mlxsw_sp_rif *rif; 7587 7588 rif = kzalloc(rif_size, GFP_KERNEL); 7589 if (!rif) 7590 return NULL; 7591 7592 INIT_LIST_HEAD(&rif->nexthop_list); 7593 INIT_LIST_HEAD(&rif->neigh_list); 7594 if (l3_dev) { 7595 ether_addr_copy(rif->addr, l3_dev->dev_addr); 7596 rif->mtu = l3_dev->mtu; 7597 rif->dev = l3_dev; 7598 } 7599 rif->vr_id = vr_id; 7600 rif->rif_index = rif_index; 7601 7602 return rif; 7603 } 7604 7605 struct mlxsw_sp_rif *mlxsw_sp_rif_by_index(const struct mlxsw_sp *mlxsw_sp, 7606 u16 rif_index) 7607 { 7608 return mlxsw_sp->router->rifs[rif_index]; 7609 } 7610 7611 u16 mlxsw_sp_rif_index(const struct mlxsw_sp_rif *rif) 7612 { 7613 return rif->rif_index; 7614 } 7615 7616 u16 mlxsw_sp_ipip_lb_rif_index(const struct mlxsw_sp_rif_ipip_lb *lb_rif) 7617 { 7618 return lb_rif->common.rif_index; 7619 } 7620 7621 u16 mlxsw_sp_ipip_lb_ul_vr_id(const struct mlxsw_sp_rif_ipip_lb *lb_rif) 7622 { 7623 u32 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(lb_rif->common.dev); 7624 struct mlxsw_sp_vr *ul_vr; 7625 7626 ul_vr = mlxsw_sp_vr_get(lb_rif->common.mlxsw_sp, ul_tb_id, NULL); 7627 if (WARN_ON(IS_ERR(ul_vr))) 7628 return 0; 7629 7630 return ul_vr->id; 7631 } 7632 7633 u16 mlxsw_sp_ipip_lb_ul_rif_id(const struct mlxsw_sp_rif_ipip_lb *lb_rif) 7634 { 7635 return lb_rif->ul_rif_id; 7636 } 7637 7638 int mlxsw_sp_rif_dev_ifindex(const struct mlxsw_sp_rif *rif) 7639 { 7640 return rif->dev->ifindex; 7641 } 7642 7643 const struct net_device *mlxsw_sp_rif_dev(const struct mlxsw_sp_rif *rif) 7644 { 7645 return rif->dev; 7646 } 7647 7648 static struct mlxsw_sp_rif * 7649 mlxsw_sp_rif_create(struct mlxsw_sp *mlxsw_sp, 7650 const struct mlxsw_sp_rif_params *params, 7651 struct netlink_ext_ack *extack) 7652 { 7653 u32 tb_id = l3mdev_fib_table(params->dev); 7654 const struct mlxsw_sp_rif_ops *ops; 7655 struct mlxsw_sp_fid *fid = NULL; 7656 enum mlxsw_sp_rif_type type; 7657 struct mlxsw_sp_rif *rif; 7658 struct mlxsw_sp_vr *vr; 7659 u16 rif_index; 7660 int i, err; 7661 7662 type = mlxsw_sp_dev_rif_type(mlxsw_sp, params->dev); 7663 ops = mlxsw_sp->rif_ops_arr[type]; 7664 7665 vr = mlxsw_sp_vr_get(mlxsw_sp, tb_id ? : RT_TABLE_MAIN, extack); 7666 if (IS_ERR(vr)) 7667 return ERR_CAST(vr); 7668 vr->rif_count++; 7669 7670 err = mlxsw_sp_rif_index_alloc(mlxsw_sp, &rif_index); 7671 if (err) { 7672 NL_SET_ERR_MSG_MOD(extack, "Exceeded number of supported router interfaces"); 7673 goto err_rif_index_alloc; 7674 } 7675 7676 rif = mlxsw_sp_rif_alloc(ops->rif_size, rif_index, vr->id, params->dev); 7677 if (!rif) { 7678 err = -ENOMEM; 7679 goto err_rif_alloc; 7680 } 7681 dev_hold(rif->dev); 7682 mlxsw_sp->router->rifs[rif_index] = rif; 7683 rif->mlxsw_sp = mlxsw_sp; 7684 rif->ops = ops; 7685 7686 if (ops->fid_get) { 7687 fid = ops->fid_get(rif, extack); 7688 if (IS_ERR(fid)) { 7689 err = PTR_ERR(fid); 7690 goto err_fid_get; 7691 } 7692 rif->fid = fid; 7693 } 7694 7695 if (ops->setup) 7696 ops->setup(rif, params); 7697 7698 err = ops->configure(rif); 7699 if (err) 7700 goto err_configure; 7701 7702 for (i = 0; i < MLXSW_SP_L3_PROTO_MAX; i++) { 7703 err = mlxsw_sp_mr_rif_add(vr->mr_table[i], rif); 7704 if (err) 7705 goto err_mr_rif_add; 7706 } 7707 7708 mlxsw_sp_rif_counters_alloc(rif); 7709 7710 return rif; 7711 7712 err_mr_rif_add: 7713 for (i--; i >= 0; i--) 7714 mlxsw_sp_mr_rif_del(vr->mr_table[i], rif); 7715 ops->deconfigure(rif); 7716 err_configure: 7717 if (fid) 7718 mlxsw_sp_fid_put(fid); 7719 err_fid_get: 7720 mlxsw_sp->router->rifs[rif_index] = NULL; 7721 dev_put(rif->dev); 7722 kfree(rif); 7723 err_rif_alloc: 7724 err_rif_index_alloc: 7725 vr->rif_count--; 7726 mlxsw_sp_vr_put(mlxsw_sp, vr); 7727 return ERR_PTR(err); 7728 } 7729 7730 static void mlxsw_sp_rif_destroy(struct mlxsw_sp_rif *rif) 7731 { 7732 const struct mlxsw_sp_rif_ops *ops = rif->ops; 7733 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 7734 struct mlxsw_sp_fid *fid = rif->fid; 7735 struct mlxsw_sp_vr *vr; 7736 int i; 7737 7738 mlxsw_sp_router_rif_gone_sync(mlxsw_sp, rif); 7739 vr = &mlxsw_sp->router->vrs[rif->vr_id]; 7740 7741 mlxsw_sp_rif_counters_free(rif); 7742 for (i = 0; i < MLXSW_SP_L3_PROTO_MAX; i++) 7743 mlxsw_sp_mr_rif_del(vr->mr_table[i], rif); 7744 ops->deconfigure(rif); 7745 if (fid) 7746 /* Loopback RIFs are not associated with a FID. */ 7747 mlxsw_sp_fid_put(fid); 7748 mlxsw_sp->router->rifs[rif->rif_index] = NULL; 7749 dev_put(rif->dev); 7750 kfree(rif); 7751 vr->rif_count--; 7752 mlxsw_sp_vr_put(mlxsw_sp, vr); 7753 } 7754 7755 void mlxsw_sp_rif_destroy_by_dev(struct mlxsw_sp *mlxsw_sp, 7756 struct net_device *dev) 7757 { 7758 struct mlxsw_sp_rif *rif; 7759 7760 mutex_lock(&mlxsw_sp->router->lock); 7761 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev); 7762 if (!rif) 7763 goto out; 7764 mlxsw_sp_rif_destroy(rif); 7765 out: 7766 mutex_unlock(&mlxsw_sp->router->lock); 7767 } 7768 7769 static void 7770 mlxsw_sp_rif_subport_params_init(struct mlxsw_sp_rif_params *params, 7771 struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan) 7772 { 7773 struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port; 7774 7775 params->vid = mlxsw_sp_port_vlan->vid; 7776 params->lag = mlxsw_sp_port->lagged; 7777 if (params->lag) 7778 params->lag_id = mlxsw_sp_port->lag_id; 7779 else 7780 params->system_port = mlxsw_sp_port->local_port; 7781 } 7782 7783 static struct mlxsw_sp_rif_subport * 7784 mlxsw_sp_rif_subport_rif(const struct mlxsw_sp_rif *rif) 7785 { 7786 return container_of(rif, struct mlxsw_sp_rif_subport, common); 7787 } 7788 7789 static struct mlxsw_sp_rif * 7790 mlxsw_sp_rif_subport_get(struct mlxsw_sp *mlxsw_sp, 7791 const struct mlxsw_sp_rif_params *params, 7792 struct netlink_ext_ack *extack) 7793 { 7794 struct mlxsw_sp_rif_subport *rif_subport; 7795 struct mlxsw_sp_rif *rif; 7796 7797 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, params->dev); 7798 if (!rif) 7799 return mlxsw_sp_rif_create(mlxsw_sp, params, extack); 7800 7801 rif_subport = mlxsw_sp_rif_subport_rif(rif); 7802 refcount_inc(&rif_subport->ref_count); 7803 return rif; 7804 } 7805 7806 static void mlxsw_sp_rif_subport_put(struct mlxsw_sp_rif *rif) 7807 { 7808 struct mlxsw_sp_rif_subport *rif_subport; 7809 7810 rif_subport = mlxsw_sp_rif_subport_rif(rif); 7811 if (!refcount_dec_and_test(&rif_subport->ref_count)) 7812 return; 7813 7814 mlxsw_sp_rif_destroy(rif); 7815 } 7816 7817 static int 7818 __mlxsw_sp_port_vlan_router_join(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan, 7819 struct net_device *l3_dev, 7820 struct netlink_ext_ack *extack) 7821 { 7822 struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port; 7823 struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp; 7824 struct mlxsw_sp_rif_params params = { 7825 .dev = l3_dev, 7826 }; 7827 u16 vid = mlxsw_sp_port_vlan->vid; 7828 struct mlxsw_sp_rif *rif; 7829 struct mlxsw_sp_fid *fid; 7830 int err; 7831 7832 mlxsw_sp_rif_subport_params_init(¶ms, mlxsw_sp_port_vlan); 7833 rif = mlxsw_sp_rif_subport_get(mlxsw_sp, ¶ms, extack); 7834 if (IS_ERR(rif)) 7835 return PTR_ERR(rif); 7836 7837 /* FID was already created, just take a reference */ 7838 fid = rif->ops->fid_get(rif, extack); 7839 err = mlxsw_sp_fid_port_vid_map(fid, mlxsw_sp_port, vid); 7840 if (err) 7841 goto err_fid_port_vid_map; 7842 7843 err = mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, false); 7844 if (err) 7845 goto err_port_vid_learning_set; 7846 7847 err = mlxsw_sp_port_vid_stp_set(mlxsw_sp_port, vid, 7848 BR_STATE_FORWARDING); 7849 if (err) 7850 goto err_port_vid_stp_set; 7851 7852 mlxsw_sp_port_vlan->fid = fid; 7853 7854 return 0; 7855 7856 err_port_vid_stp_set: 7857 mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, true); 7858 err_port_vid_learning_set: 7859 mlxsw_sp_fid_port_vid_unmap(fid, mlxsw_sp_port, vid); 7860 err_fid_port_vid_map: 7861 mlxsw_sp_fid_put(fid); 7862 mlxsw_sp_rif_subport_put(rif); 7863 return err; 7864 } 7865 7866 static void 7867 __mlxsw_sp_port_vlan_router_leave(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan) 7868 { 7869 struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port; 7870 struct mlxsw_sp_fid *fid = mlxsw_sp_port_vlan->fid; 7871 struct mlxsw_sp_rif *rif = mlxsw_sp_fid_rif(fid); 7872 u16 vid = mlxsw_sp_port_vlan->vid; 7873 7874 if (WARN_ON(mlxsw_sp_fid_type(fid) != MLXSW_SP_FID_TYPE_RFID)) 7875 return; 7876 7877 mlxsw_sp_port_vlan->fid = NULL; 7878 mlxsw_sp_port_vid_stp_set(mlxsw_sp_port, vid, BR_STATE_BLOCKING); 7879 mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, true); 7880 mlxsw_sp_fid_port_vid_unmap(fid, mlxsw_sp_port, vid); 7881 mlxsw_sp_fid_put(fid); 7882 mlxsw_sp_rif_subport_put(rif); 7883 } 7884 7885 int 7886 mlxsw_sp_port_vlan_router_join(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan, 7887 struct net_device *l3_dev, 7888 struct netlink_ext_ack *extack) 7889 { 7890 struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port_vlan->mlxsw_sp_port->mlxsw_sp; 7891 struct mlxsw_sp_rif *rif; 7892 int err = 0; 7893 7894 mutex_lock(&mlxsw_sp->router->lock); 7895 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev); 7896 if (!rif) 7897 goto out; 7898 7899 err = __mlxsw_sp_port_vlan_router_join(mlxsw_sp_port_vlan, l3_dev, 7900 extack); 7901 out: 7902 mutex_unlock(&mlxsw_sp->router->lock); 7903 return err; 7904 } 7905 7906 void 7907 mlxsw_sp_port_vlan_router_leave(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan) 7908 { 7909 struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port_vlan->mlxsw_sp_port->mlxsw_sp; 7910 7911 mutex_lock(&mlxsw_sp->router->lock); 7912 __mlxsw_sp_port_vlan_router_leave(mlxsw_sp_port_vlan); 7913 mutex_unlock(&mlxsw_sp->router->lock); 7914 } 7915 7916 static int mlxsw_sp_inetaddr_port_vlan_event(struct net_device *l3_dev, 7917 struct net_device *port_dev, 7918 unsigned long event, u16 vid, 7919 struct netlink_ext_ack *extack) 7920 { 7921 struct mlxsw_sp_port *mlxsw_sp_port = netdev_priv(port_dev); 7922 struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan; 7923 7924 mlxsw_sp_port_vlan = mlxsw_sp_port_vlan_find_by_vid(mlxsw_sp_port, vid); 7925 if (WARN_ON(!mlxsw_sp_port_vlan)) 7926 return -EINVAL; 7927 7928 switch (event) { 7929 case NETDEV_UP: 7930 return __mlxsw_sp_port_vlan_router_join(mlxsw_sp_port_vlan, 7931 l3_dev, extack); 7932 case NETDEV_DOWN: 7933 __mlxsw_sp_port_vlan_router_leave(mlxsw_sp_port_vlan); 7934 break; 7935 } 7936 7937 return 0; 7938 } 7939 7940 static int mlxsw_sp_inetaddr_port_event(struct net_device *port_dev, 7941 unsigned long event, 7942 struct netlink_ext_ack *extack) 7943 { 7944 if (netif_is_bridge_port(port_dev) || 7945 netif_is_lag_port(port_dev) || 7946 netif_is_ovs_port(port_dev)) 7947 return 0; 7948 7949 return mlxsw_sp_inetaddr_port_vlan_event(port_dev, port_dev, event, 7950 MLXSW_SP_DEFAULT_VID, extack); 7951 } 7952 7953 static int __mlxsw_sp_inetaddr_lag_event(struct net_device *l3_dev, 7954 struct net_device *lag_dev, 7955 unsigned long event, u16 vid, 7956 struct netlink_ext_ack *extack) 7957 { 7958 struct net_device *port_dev; 7959 struct list_head *iter; 7960 int err; 7961 7962 netdev_for_each_lower_dev(lag_dev, port_dev, iter) { 7963 if (mlxsw_sp_port_dev_check(port_dev)) { 7964 err = mlxsw_sp_inetaddr_port_vlan_event(l3_dev, 7965 port_dev, 7966 event, vid, 7967 extack); 7968 if (err) 7969 return err; 7970 } 7971 } 7972 7973 return 0; 7974 } 7975 7976 static int mlxsw_sp_inetaddr_lag_event(struct net_device *lag_dev, 7977 unsigned long event, 7978 struct netlink_ext_ack *extack) 7979 { 7980 if (netif_is_bridge_port(lag_dev)) 7981 return 0; 7982 7983 return __mlxsw_sp_inetaddr_lag_event(lag_dev, lag_dev, event, 7984 MLXSW_SP_DEFAULT_VID, extack); 7985 } 7986 7987 static int mlxsw_sp_inetaddr_bridge_event(struct mlxsw_sp *mlxsw_sp, 7988 struct net_device *l3_dev, 7989 unsigned long event, 7990 struct netlink_ext_ack *extack) 7991 { 7992 struct mlxsw_sp_rif_params params = { 7993 .dev = l3_dev, 7994 }; 7995 struct mlxsw_sp_rif *rif; 7996 7997 switch (event) { 7998 case NETDEV_UP: 7999 if (netif_is_bridge_master(l3_dev) && br_vlan_enabled(l3_dev)) { 8000 u16 proto; 8001 8002 br_vlan_get_proto(l3_dev, &proto); 8003 if (proto == ETH_P_8021AD) { 8004 NL_SET_ERR_MSG_MOD(extack, "Adding an IP address to 802.1ad bridge is not supported"); 8005 return -EOPNOTSUPP; 8006 } 8007 } 8008 rif = mlxsw_sp_rif_create(mlxsw_sp, ¶ms, extack); 8009 if (IS_ERR(rif)) 8010 return PTR_ERR(rif); 8011 break; 8012 case NETDEV_DOWN: 8013 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev); 8014 mlxsw_sp_rif_destroy(rif); 8015 break; 8016 } 8017 8018 return 0; 8019 } 8020 8021 static int mlxsw_sp_inetaddr_vlan_event(struct mlxsw_sp *mlxsw_sp, 8022 struct net_device *vlan_dev, 8023 unsigned long event, 8024 struct netlink_ext_ack *extack) 8025 { 8026 struct net_device *real_dev = vlan_dev_real_dev(vlan_dev); 8027 u16 vid = vlan_dev_vlan_id(vlan_dev); 8028 8029 if (netif_is_bridge_port(vlan_dev)) 8030 return 0; 8031 8032 if (mlxsw_sp_port_dev_check(real_dev)) 8033 return mlxsw_sp_inetaddr_port_vlan_event(vlan_dev, real_dev, 8034 event, vid, extack); 8035 else if (netif_is_lag_master(real_dev)) 8036 return __mlxsw_sp_inetaddr_lag_event(vlan_dev, real_dev, event, 8037 vid, extack); 8038 else if (netif_is_bridge_master(real_dev) && br_vlan_enabled(real_dev)) 8039 return mlxsw_sp_inetaddr_bridge_event(mlxsw_sp, vlan_dev, event, 8040 extack); 8041 8042 return 0; 8043 } 8044 8045 static bool mlxsw_sp_rif_macvlan_is_vrrp4(const u8 *mac) 8046 { 8047 u8 vrrp4[ETH_ALEN] = { 0x00, 0x00, 0x5e, 0x00, 0x01, 0x00 }; 8048 u8 mask[ETH_ALEN] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0x00 }; 8049 8050 return ether_addr_equal_masked(mac, vrrp4, mask); 8051 } 8052 8053 static bool mlxsw_sp_rif_macvlan_is_vrrp6(const u8 *mac) 8054 { 8055 u8 vrrp6[ETH_ALEN] = { 0x00, 0x00, 0x5e, 0x00, 0x02, 0x00 }; 8056 u8 mask[ETH_ALEN] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0x00 }; 8057 8058 return ether_addr_equal_masked(mac, vrrp6, mask); 8059 } 8060 8061 static int mlxsw_sp_rif_vrrp_op(struct mlxsw_sp *mlxsw_sp, u16 rif_index, 8062 const u8 *mac, bool adding) 8063 { 8064 char ritr_pl[MLXSW_REG_RITR_LEN]; 8065 u8 vrrp_id = adding ? mac[5] : 0; 8066 int err; 8067 8068 if (!mlxsw_sp_rif_macvlan_is_vrrp4(mac) && 8069 !mlxsw_sp_rif_macvlan_is_vrrp6(mac)) 8070 return 0; 8071 8072 mlxsw_reg_ritr_rif_pack(ritr_pl, rif_index); 8073 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 8074 if (err) 8075 return err; 8076 8077 if (mlxsw_sp_rif_macvlan_is_vrrp4(mac)) 8078 mlxsw_reg_ritr_if_vrrp_id_ipv4_set(ritr_pl, vrrp_id); 8079 else 8080 mlxsw_reg_ritr_if_vrrp_id_ipv6_set(ritr_pl, vrrp_id); 8081 8082 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 8083 } 8084 8085 static int mlxsw_sp_rif_macvlan_add(struct mlxsw_sp *mlxsw_sp, 8086 const struct net_device *macvlan_dev, 8087 struct netlink_ext_ack *extack) 8088 { 8089 struct macvlan_dev *vlan = netdev_priv(macvlan_dev); 8090 struct mlxsw_sp_rif *rif; 8091 int err; 8092 8093 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, vlan->lowerdev); 8094 if (!rif) { 8095 NL_SET_ERR_MSG_MOD(extack, "macvlan is only supported on top of router interfaces"); 8096 return -EOPNOTSUPP; 8097 } 8098 8099 err = mlxsw_sp_rif_fdb_op(mlxsw_sp, macvlan_dev->dev_addr, 8100 mlxsw_sp_fid_index(rif->fid), true); 8101 if (err) 8102 return err; 8103 8104 err = mlxsw_sp_rif_vrrp_op(mlxsw_sp, rif->rif_index, 8105 macvlan_dev->dev_addr, true); 8106 if (err) 8107 goto err_rif_vrrp_add; 8108 8109 /* Make sure the bridge driver does not have this MAC pointing at 8110 * some other port. 8111 */ 8112 if (rif->ops->fdb_del) 8113 rif->ops->fdb_del(rif, macvlan_dev->dev_addr); 8114 8115 return 0; 8116 8117 err_rif_vrrp_add: 8118 mlxsw_sp_rif_fdb_op(mlxsw_sp, macvlan_dev->dev_addr, 8119 mlxsw_sp_fid_index(rif->fid), false); 8120 return err; 8121 } 8122 8123 static void __mlxsw_sp_rif_macvlan_del(struct mlxsw_sp *mlxsw_sp, 8124 const struct net_device *macvlan_dev) 8125 { 8126 struct macvlan_dev *vlan = netdev_priv(macvlan_dev); 8127 struct mlxsw_sp_rif *rif; 8128 8129 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, vlan->lowerdev); 8130 /* If we do not have a RIF, then we already took care of 8131 * removing the macvlan's MAC during RIF deletion. 8132 */ 8133 if (!rif) 8134 return; 8135 mlxsw_sp_rif_vrrp_op(mlxsw_sp, rif->rif_index, macvlan_dev->dev_addr, 8136 false); 8137 mlxsw_sp_rif_fdb_op(mlxsw_sp, macvlan_dev->dev_addr, 8138 mlxsw_sp_fid_index(rif->fid), false); 8139 } 8140 8141 void mlxsw_sp_rif_macvlan_del(struct mlxsw_sp *mlxsw_sp, 8142 const struct net_device *macvlan_dev) 8143 { 8144 mutex_lock(&mlxsw_sp->router->lock); 8145 __mlxsw_sp_rif_macvlan_del(mlxsw_sp, macvlan_dev); 8146 mutex_unlock(&mlxsw_sp->router->lock); 8147 } 8148 8149 static int mlxsw_sp_inetaddr_macvlan_event(struct mlxsw_sp *mlxsw_sp, 8150 struct net_device *macvlan_dev, 8151 unsigned long event, 8152 struct netlink_ext_ack *extack) 8153 { 8154 switch (event) { 8155 case NETDEV_UP: 8156 return mlxsw_sp_rif_macvlan_add(mlxsw_sp, macvlan_dev, extack); 8157 case NETDEV_DOWN: 8158 __mlxsw_sp_rif_macvlan_del(mlxsw_sp, macvlan_dev); 8159 break; 8160 } 8161 8162 return 0; 8163 } 8164 8165 static int mlxsw_sp_router_port_check_rif_addr(struct mlxsw_sp *mlxsw_sp, 8166 struct net_device *dev, 8167 const unsigned char *dev_addr, 8168 struct netlink_ext_ack *extack) 8169 { 8170 struct mlxsw_sp_rif *rif; 8171 int i; 8172 8173 /* A RIF is not created for macvlan netdevs. Their MAC is used to 8174 * populate the FDB 8175 */ 8176 if (netif_is_macvlan(dev) || netif_is_l3_master(dev)) 8177 return 0; 8178 8179 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++) { 8180 rif = mlxsw_sp->router->rifs[i]; 8181 if (rif && rif->ops && 8182 rif->ops->type == MLXSW_SP_RIF_TYPE_IPIP_LB) 8183 continue; 8184 if (rif && rif->dev && rif->dev != dev && 8185 !ether_addr_equal_masked(rif->dev->dev_addr, dev_addr, 8186 mlxsw_sp->mac_mask)) { 8187 NL_SET_ERR_MSG_MOD(extack, "All router interface MAC addresses must have the same prefix"); 8188 return -EINVAL; 8189 } 8190 } 8191 8192 return 0; 8193 } 8194 8195 static int __mlxsw_sp_inetaddr_event(struct mlxsw_sp *mlxsw_sp, 8196 struct net_device *dev, 8197 unsigned long event, 8198 struct netlink_ext_ack *extack) 8199 { 8200 if (mlxsw_sp_port_dev_check(dev)) 8201 return mlxsw_sp_inetaddr_port_event(dev, event, extack); 8202 else if (netif_is_lag_master(dev)) 8203 return mlxsw_sp_inetaddr_lag_event(dev, event, extack); 8204 else if (netif_is_bridge_master(dev)) 8205 return mlxsw_sp_inetaddr_bridge_event(mlxsw_sp, dev, event, 8206 extack); 8207 else if (is_vlan_dev(dev)) 8208 return mlxsw_sp_inetaddr_vlan_event(mlxsw_sp, dev, event, 8209 extack); 8210 else if (netif_is_macvlan(dev)) 8211 return mlxsw_sp_inetaddr_macvlan_event(mlxsw_sp, dev, event, 8212 extack); 8213 else 8214 return 0; 8215 } 8216 8217 static int mlxsw_sp_inetaddr_event(struct notifier_block *nb, 8218 unsigned long event, void *ptr) 8219 { 8220 struct in_ifaddr *ifa = (struct in_ifaddr *) ptr; 8221 struct net_device *dev = ifa->ifa_dev->dev; 8222 struct mlxsw_sp_router *router; 8223 struct mlxsw_sp_rif *rif; 8224 int err = 0; 8225 8226 /* NETDEV_UP event is handled by mlxsw_sp_inetaddr_valid_event */ 8227 if (event == NETDEV_UP) 8228 return NOTIFY_DONE; 8229 8230 router = container_of(nb, struct mlxsw_sp_router, inetaddr_nb); 8231 mutex_lock(&router->lock); 8232 rif = mlxsw_sp_rif_find_by_dev(router->mlxsw_sp, dev); 8233 if (!mlxsw_sp_rif_should_config(rif, dev, event)) 8234 goto out; 8235 8236 err = __mlxsw_sp_inetaddr_event(router->mlxsw_sp, dev, event, NULL); 8237 out: 8238 mutex_unlock(&router->lock); 8239 return notifier_from_errno(err); 8240 } 8241 8242 int mlxsw_sp_inetaddr_valid_event(struct notifier_block *unused, 8243 unsigned long event, void *ptr) 8244 { 8245 struct in_validator_info *ivi = (struct in_validator_info *) ptr; 8246 struct net_device *dev = ivi->ivi_dev->dev; 8247 struct mlxsw_sp *mlxsw_sp; 8248 struct mlxsw_sp_rif *rif; 8249 int err = 0; 8250 8251 mlxsw_sp = mlxsw_sp_lower_get(dev); 8252 if (!mlxsw_sp) 8253 return NOTIFY_DONE; 8254 8255 mutex_lock(&mlxsw_sp->router->lock); 8256 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev); 8257 if (!mlxsw_sp_rif_should_config(rif, dev, event)) 8258 goto out; 8259 8260 err = mlxsw_sp_router_port_check_rif_addr(mlxsw_sp, dev, dev->dev_addr, 8261 ivi->extack); 8262 if (err) 8263 goto out; 8264 8265 err = __mlxsw_sp_inetaddr_event(mlxsw_sp, dev, event, ivi->extack); 8266 out: 8267 mutex_unlock(&mlxsw_sp->router->lock); 8268 return notifier_from_errno(err); 8269 } 8270 8271 struct mlxsw_sp_inet6addr_event_work { 8272 struct work_struct work; 8273 struct mlxsw_sp *mlxsw_sp; 8274 struct net_device *dev; 8275 unsigned long event; 8276 }; 8277 8278 static void mlxsw_sp_inet6addr_event_work(struct work_struct *work) 8279 { 8280 struct mlxsw_sp_inet6addr_event_work *inet6addr_work = 8281 container_of(work, struct mlxsw_sp_inet6addr_event_work, work); 8282 struct mlxsw_sp *mlxsw_sp = inet6addr_work->mlxsw_sp; 8283 struct net_device *dev = inet6addr_work->dev; 8284 unsigned long event = inet6addr_work->event; 8285 struct mlxsw_sp_rif *rif; 8286 8287 rtnl_lock(); 8288 mutex_lock(&mlxsw_sp->router->lock); 8289 8290 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev); 8291 if (!mlxsw_sp_rif_should_config(rif, dev, event)) 8292 goto out; 8293 8294 __mlxsw_sp_inetaddr_event(mlxsw_sp, dev, event, NULL); 8295 out: 8296 mutex_unlock(&mlxsw_sp->router->lock); 8297 rtnl_unlock(); 8298 dev_put(dev); 8299 kfree(inet6addr_work); 8300 } 8301 8302 /* Called with rcu_read_lock() */ 8303 static int mlxsw_sp_inet6addr_event(struct notifier_block *nb, 8304 unsigned long event, void *ptr) 8305 { 8306 struct inet6_ifaddr *if6 = (struct inet6_ifaddr *) ptr; 8307 struct mlxsw_sp_inet6addr_event_work *inet6addr_work; 8308 struct net_device *dev = if6->idev->dev; 8309 struct mlxsw_sp_router *router; 8310 8311 /* NETDEV_UP event is handled by mlxsw_sp_inet6addr_valid_event */ 8312 if (event == NETDEV_UP) 8313 return NOTIFY_DONE; 8314 8315 inet6addr_work = kzalloc(sizeof(*inet6addr_work), GFP_ATOMIC); 8316 if (!inet6addr_work) 8317 return NOTIFY_BAD; 8318 8319 router = container_of(nb, struct mlxsw_sp_router, inet6addr_nb); 8320 INIT_WORK(&inet6addr_work->work, mlxsw_sp_inet6addr_event_work); 8321 inet6addr_work->mlxsw_sp = router->mlxsw_sp; 8322 inet6addr_work->dev = dev; 8323 inet6addr_work->event = event; 8324 dev_hold(dev); 8325 mlxsw_core_schedule_work(&inet6addr_work->work); 8326 8327 return NOTIFY_DONE; 8328 } 8329 8330 int mlxsw_sp_inet6addr_valid_event(struct notifier_block *unused, 8331 unsigned long event, void *ptr) 8332 { 8333 struct in6_validator_info *i6vi = (struct in6_validator_info *) ptr; 8334 struct net_device *dev = i6vi->i6vi_dev->dev; 8335 struct mlxsw_sp *mlxsw_sp; 8336 struct mlxsw_sp_rif *rif; 8337 int err = 0; 8338 8339 mlxsw_sp = mlxsw_sp_lower_get(dev); 8340 if (!mlxsw_sp) 8341 return NOTIFY_DONE; 8342 8343 mutex_lock(&mlxsw_sp->router->lock); 8344 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev); 8345 if (!mlxsw_sp_rif_should_config(rif, dev, event)) 8346 goto out; 8347 8348 err = mlxsw_sp_router_port_check_rif_addr(mlxsw_sp, dev, dev->dev_addr, 8349 i6vi->extack); 8350 if (err) 8351 goto out; 8352 8353 err = __mlxsw_sp_inetaddr_event(mlxsw_sp, dev, event, i6vi->extack); 8354 out: 8355 mutex_unlock(&mlxsw_sp->router->lock); 8356 return notifier_from_errno(err); 8357 } 8358 8359 static int mlxsw_sp_rif_edit(struct mlxsw_sp *mlxsw_sp, u16 rif_index, 8360 const char *mac, int mtu) 8361 { 8362 char ritr_pl[MLXSW_REG_RITR_LEN]; 8363 int err; 8364 8365 mlxsw_reg_ritr_rif_pack(ritr_pl, rif_index); 8366 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 8367 if (err) 8368 return err; 8369 8370 mlxsw_reg_ritr_mtu_set(ritr_pl, mtu); 8371 mlxsw_reg_ritr_if_mac_memcpy_to(ritr_pl, mac); 8372 mlxsw_reg_ritr_op_set(ritr_pl, MLXSW_REG_RITR_RIF_CREATE); 8373 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 8374 } 8375 8376 static int 8377 mlxsw_sp_router_port_change_event(struct mlxsw_sp *mlxsw_sp, 8378 struct mlxsw_sp_rif *rif) 8379 { 8380 struct net_device *dev = rif->dev; 8381 u16 fid_index; 8382 int err; 8383 8384 fid_index = mlxsw_sp_fid_index(rif->fid); 8385 8386 err = mlxsw_sp_rif_fdb_op(mlxsw_sp, rif->addr, fid_index, false); 8387 if (err) 8388 return err; 8389 8390 err = mlxsw_sp_rif_edit(mlxsw_sp, rif->rif_index, dev->dev_addr, 8391 dev->mtu); 8392 if (err) 8393 goto err_rif_edit; 8394 8395 err = mlxsw_sp_rif_fdb_op(mlxsw_sp, dev->dev_addr, fid_index, true); 8396 if (err) 8397 goto err_rif_fdb_op; 8398 8399 if (rif->mtu != dev->mtu) { 8400 struct mlxsw_sp_vr *vr; 8401 int i; 8402 8403 /* The RIF is relevant only to its mr_table instance, as unlike 8404 * unicast routing, in multicast routing a RIF cannot be shared 8405 * between several multicast routing tables. 8406 */ 8407 vr = &mlxsw_sp->router->vrs[rif->vr_id]; 8408 for (i = 0; i < MLXSW_SP_L3_PROTO_MAX; i++) 8409 mlxsw_sp_mr_rif_mtu_update(vr->mr_table[i], 8410 rif, dev->mtu); 8411 } 8412 8413 ether_addr_copy(rif->addr, dev->dev_addr); 8414 rif->mtu = dev->mtu; 8415 8416 netdev_dbg(dev, "Updated RIF=%d\n", rif->rif_index); 8417 8418 return 0; 8419 8420 err_rif_fdb_op: 8421 mlxsw_sp_rif_edit(mlxsw_sp, rif->rif_index, rif->addr, rif->mtu); 8422 err_rif_edit: 8423 mlxsw_sp_rif_fdb_op(mlxsw_sp, rif->addr, fid_index, true); 8424 return err; 8425 } 8426 8427 static int mlxsw_sp_router_port_pre_changeaddr_event(struct mlxsw_sp_rif *rif, 8428 struct netdev_notifier_pre_changeaddr_info *info) 8429 { 8430 struct netlink_ext_ack *extack; 8431 8432 extack = netdev_notifier_info_to_extack(&info->info); 8433 return mlxsw_sp_router_port_check_rif_addr(rif->mlxsw_sp, rif->dev, 8434 info->dev_addr, extack); 8435 } 8436 8437 int mlxsw_sp_netdevice_router_port_event(struct net_device *dev, 8438 unsigned long event, void *ptr) 8439 { 8440 struct mlxsw_sp *mlxsw_sp; 8441 struct mlxsw_sp_rif *rif; 8442 int err = 0; 8443 8444 mlxsw_sp = mlxsw_sp_lower_get(dev); 8445 if (!mlxsw_sp) 8446 return 0; 8447 8448 mutex_lock(&mlxsw_sp->router->lock); 8449 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev); 8450 if (!rif) 8451 goto out; 8452 8453 switch (event) { 8454 case NETDEV_CHANGEMTU: 8455 case NETDEV_CHANGEADDR: 8456 err = mlxsw_sp_router_port_change_event(mlxsw_sp, rif); 8457 break; 8458 case NETDEV_PRE_CHANGEADDR: 8459 err = mlxsw_sp_router_port_pre_changeaddr_event(rif, ptr); 8460 break; 8461 } 8462 8463 out: 8464 mutex_unlock(&mlxsw_sp->router->lock); 8465 return err; 8466 } 8467 8468 static int mlxsw_sp_port_vrf_join(struct mlxsw_sp *mlxsw_sp, 8469 struct net_device *l3_dev, 8470 struct netlink_ext_ack *extack) 8471 { 8472 struct mlxsw_sp_rif *rif; 8473 8474 /* If netdev is already associated with a RIF, then we need to 8475 * destroy it and create a new one with the new virtual router ID. 8476 */ 8477 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev); 8478 if (rif) 8479 __mlxsw_sp_inetaddr_event(mlxsw_sp, l3_dev, NETDEV_DOWN, 8480 extack); 8481 8482 return __mlxsw_sp_inetaddr_event(mlxsw_sp, l3_dev, NETDEV_UP, extack); 8483 } 8484 8485 static void mlxsw_sp_port_vrf_leave(struct mlxsw_sp *mlxsw_sp, 8486 struct net_device *l3_dev) 8487 { 8488 struct mlxsw_sp_rif *rif; 8489 8490 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev); 8491 if (!rif) 8492 return; 8493 __mlxsw_sp_inetaddr_event(mlxsw_sp, l3_dev, NETDEV_DOWN, NULL); 8494 } 8495 8496 int mlxsw_sp_netdevice_vrf_event(struct net_device *l3_dev, unsigned long event, 8497 struct netdev_notifier_changeupper_info *info) 8498 { 8499 struct mlxsw_sp *mlxsw_sp = mlxsw_sp_lower_get(l3_dev); 8500 int err = 0; 8501 8502 /* We do not create a RIF for a macvlan, but only use it to 8503 * direct more MAC addresses to the router. 8504 */ 8505 if (!mlxsw_sp || netif_is_macvlan(l3_dev)) 8506 return 0; 8507 8508 mutex_lock(&mlxsw_sp->router->lock); 8509 switch (event) { 8510 case NETDEV_PRECHANGEUPPER: 8511 break; 8512 case NETDEV_CHANGEUPPER: 8513 if (info->linking) { 8514 struct netlink_ext_ack *extack; 8515 8516 extack = netdev_notifier_info_to_extack(&info->info); 8517 err = mlxsw_sp_port_vrf_join(mlxsw_sp, l3_dev, extack); 8518 } else { 8519 mlxsw_sp_port_vrf_leave(mlxsw_sp, l3_dev); 8520 } 8521 break; 8522 } 8523 mutex_unlock(&mlxsw_sp->router->lock); 8524 8525 return err; 8526 } 8527 8528 static int __mlxsw_sp_rif_macvlan_flush(struct net_device *dev, 8529 struct netdev_nested_priv *priv) 8530 { 8531 struct mlxsw_sp_rif *rif = (struct mlxsw_sp_rif *)priv->data; 8532 8533 if (!netif_is_macvlan(dev)) 8534 return 0; 8535 8536 return mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, dev->dev_addr, 8537 mlxsw_sp_fid_index(rif->fid), false); 8538 } 8539 8540 static int mlxsw_sp_rif_macvlan_flush(struct mlxsw_sp_rif *rif) 8541 { 8542 struct netdev_nested_priv priv = { 8543 .data = (void *)rif, 8544 }; 8545 8546 if (!netif_is_macvlan_port(rif->dev)) 8547 return 0; 8548 8549 netdev_warn(rif->dev, "Router interface is deleted. Upper macvlans will not work\n"); 8550 return netdev_walk_all_upper_dev_rcu(rif->dev, 8551 __mlxsw_sp_rif_macvlan_flush, &priv); 8552 } 8553 8554 static void mlxsw_sp_rif_subport_setup(struct mlxsw_sp_rif *rif, 8555 const struct mlxsw_sp_rif_params *params) 8556 { 8557 struct mlxsw_sp_rif_subport *rif_subport; 8558 8559 rif_subport = mlxsw_sp_rif_subport_rif(rif); 8560 refcount_set(&rif_subport->ref_count, 1); 8561 rif_subport->vid = params->vid; 8562 rif_subport->lag = params->lag; 8563 if (params->lag) 8564 rif_subport->lag_id = params->lag_id; 8565 else 8566 rif_subport->system_port = params->system_port; 8567 } 8568 8569 static int mlxsw_sp_rif_subport_op(struct mlxsw_sp_rif *rif, bool enable) 8570 { 8571 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 8572 struct mlxsw_sp_rif_subport *rif_subport; 8573 char ritr_pl[MLXSW_REG_RITR_LEN]; 8574 8575 rif_subport = mlxsw_sp_rif_subport_rif(rif); 8576 mlxsw_reg_ritr_pack(ritr_pl, enable, MLXSW_REG_RITR_SP_IF, 8577 rif->rif_index, rif->vr_id, rif->dev->mtu); 8578 mlxsw_reg_ritr_mac_pack(ritr_pl, rif->dev->dev_addr); 8579 mlxsw_reg_ritr_sp_if_pack(ritr_pl, rif_subport->lag, 8580 rif_subport->lag ? rif_subport->lag_id : 8581 rif_subport->system_port, 8582 rif_subport->vid); 8583 8584 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 8585 } 8586 8587 static int mlxsw_sp_rif_subport_configure(struct mlxsw_sp_rif *rif) 8588 { 8589 int err; 8590 8591 err = mlxsw_sp_rif_subport_op(rif, true); 8592 if (err) 8593 return err; 8594 8595 err = mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr, 8596 mlxsw_sp_fid_index(rif->fid), true); 8597 if (err) 8598 goto err_rif_fdb_op; 8599 8600 mlxsw_sp_fid_rif_set(rif->fid, rif); 8601 return 0; 8602 8603 err_rif_fdb_op: 8604 mlxsw_sp_rif_subport_op(rif, false); 8605 return err; 8606 } 8607 8608 static void mlxsw_sp_rif_subport_deconfigure(struct mlxsw_sp_rif *rif) 8609 { 8610 struct mlxsw_sp_fid *fid = rif->fid; 8611 8612 mlxsw_sp_fid_rif_set(fid, NULL); 8613 mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr, 8614 mlxsw_sp_fid_index(fid), false); 8615 mlxsw_sp_rif_macvlan_flush(rif); 8616 mlxsw_sp_rif_subport_op(rif, false); 8617 } 8618 8619 static struct mlxsw_sp_fid * 8620 mlxsw_sp_rif_subport_fid_get(struct mlxsw_sp_rif *rif, 8621 struct netlink_ext_ack *extack) 8622 { 8623 return mlxsw_sp_fid_rfid_get(rif->mlxsw_sp, rif->rif_index); 8624 } 8625 8626 static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_subport_ops = { 8627 .type = MLXSW_SP_RIF_TYPE_SUBPORT, 8628 .rif_size = sizeof(struct mlxsw_sp_rif_subport), 8629 .setup = mlxsw_sp_rif_subport_setup, 8630 .configure = mlxsw_sp_rif_subport_configure, 8631 .deconfigure = mlxsw_sp_rif_subport_deconfigure, 8632 .fid_get = mlxsw_sp_rif_subport_fid_get, 8633 }; 8634 8635 static int mlxsw_sp_rif_vlan_fid_op(struct mlxsw_sp_rif *rif, 8636 enum mlxsw_reg_ritr_if_type type, 8637 u16 vid_fid, bool enable) 8638 { 8639 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 8640 char ritr_pl[MLXSW_REG_RITR_LEN]; 8641 8642 mlxsw_reg_ritr_pack(ritr_pl, enable, type, rif->rif_index, rif->vr_id, 8643 rif->dev->mtu); 8644 mlxsw_reg_ritr_mac_pack(ritr_pl, rif->dev->dev_addr); 8645 mlxsw_reg_ritr_fid_set(ritr_pl, type, vid_fid); 8646 8647 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 8648 } 8649 8650 u8 mlxsw_sp_router_port(const struct mlxsw_sp *mlxsw_sp) 8651 { 8652 return mlxsw_core_max_ports(mlxsw_sp->core) + 1; 8653 } 8654 8655 static int mlxsw_sp_rif_fid_configure(struct mlxsw_sp_rif *rif) 8656 { 8657 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 8658 u16 fid_index = mlxsw_sp_fid_index(rif->fid); 8659 int err; 8660 8661 err = mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_FID_IF, fid_index, 8662 true); 8663 if (err) 8664 return err; 8665 8666 err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC, 8667 mlxsw_sp_router_port(mlxsw_sp), true); 8668 if (err) 8669 goto err_fid_mc_flood_set; 8670 8671 err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC, 8672 mlxsw_sp_router_port(mlxsw_sp), true); 8673 if (err) 8674 goto err_fid_bc_flood_set; 8675 8676 err = mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr, 8677 mlxsw_sp_fid_index(rif->fid), true); 8678 if (err) 8679 goto err_rif_fdb_op; 8680 8681 mlxsw_sp_fid_rif_set(rif->fid, rif); 8682 return 0; 8683 8684 err_rif_fdb_op: 8685 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC, 8686 mlxsw_sp_router_port(mlxsw_sp), false); 8687 err_fid_bc_flood_set: 8688 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC, 8689 mlxsw_sp_router_port(mlxsw_sp), false); 8690 err_fid_mc_flood_set: 8691 mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_FID_IF, fid_index, false); 8692 return err; 8693 } 8694 8695 static void mlxsw_sp_rif_fid_deconfigure(struct mlxsw_sp_rif *rif) 8696 { 8697 u16 fid_index = mlxsw_sp_fid_index(rif->fid); 8698 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 8699 struct mlxsw_sp_fid *fid = rif->fid; 8700 8701 mlxsw_sp_fid_rif_set(fid, NULL); 8702 mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr, 8703 mlxsw_sp_fid_index(fid), false); 8704 mlxsw_sp_rif_macvlan_flush(rif); 8705 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC, 8706 mlxsw_sp_router_port(mlxsw_sp), false); 8707 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC, 8708 mlxsw_sp_router_port(mlxsw_sp), false); 8709 mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_FID_IF, fid_index, false); 8710 } 8711 8712 static struct mlxsw_sp_fid * 8713 mlxsw_sp_rif_fid_fid_get(struct mlxsw_sp_rif *rif, 8714 struct netlink_ext_ack *extack) 8715 { 8716 return mlxsw_sp_fid_8021d_get(rif->mlxsw_sp, rif->dev->ifindex); 8717 } 8718 8719 static void mlxsw_sp_rif_fid_fdb_del(struct mlxsw_sp_rif *rif, const char *mac) 8720 { 8721 struct switchdev_notifier_fdb_info info; 8722 struct net_device *dev; 8723 8724 dev = br_fdb_find_port(rif->dev, mac, 0); 8725 if (!dev) 8726 return; 8727 8728 info.addr = mac; 8729 info.vid = 0; 8730 call_switchdev_notifiers(SWITCHDEV_FDB_DEL_TO_BRIDGE, dev, &info.info, 8731 NULL); 8732 } 8733 8734 static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_fid_ops = { 8735 .type = MLXSW_SP_RIF_TYPE_FID, 8736 .rif_size = sizeof(struct mlxsw_sp_rif), 8737 .configure = mlxsw_sp_rif_fid_configure, 8738 .deconfigure = mlxsw_sp_rif_fid_deconfigure, 8739 .fid_get = mlxsw_sp_rif_fid_fid_get, 8740 .fdb_del = mlxsw_sp_rif_fid_fdb_del, 8741 }; 8742 8743 static struct mlxsw_sp_fid * 8744 mlxsw_sp_rif_vlan_fid_get(struct mlxsw_sp_rif *rif, 8745 struct netlink_ext_ack *extack) 8746 { 8747 struct net_device *br_dev; 8748 u16 vid; 8749 int err; 8750 8751 if (is_vlan_dev(rif->dev)) { 8752 vid = vlan_dev_vlan_id(rif->dev); 8753 br_dev = vlan_dev_real_dev(rif->dev); 8754 if (WARN_ON(!netif_is_bridge_master(br_dev))) 8755 return ERR_PTR(-EINVAL); 8756 } else { 8757 err = br_vlan_get_pvid(rif->dev, &vid); 8758 if (err < 0 || !vid) { 8759 NL_SET_ERR_MSG_MOD(extack, "Couldn't determine bridge PVID"); 8760 return ERR_PTR(-EINVAL); 8761 } 8762 } 8763 8764 return mlxsw_sp_fid_8021q_get(rif->mlxsw_sp, vid); 8765 } 8766 8767 static void mlxsw_sp_rif_vlan_fdb_del(struct mlxsw_sp_rif *rif, const char *mac) 8768 { 8769 u16 vid = mlxsw_sp_fid_8021q_vid(rif->fid); 8770 struct switchdev_notifier_fdb_info info; 8771 struct net_device *br_dev; 8772 struct net_device *dev; 8773 8774 br_dev = is_vlan_dev(rif->dev) ? vlan_dev_real_dev(rif->dev) : rif->dev; 8775 dev = br_fdb_find_port(br_dev, mac, vid); 8776 if (!dev) 8777 return; 8778 8779 info.addr = mac; 8780 info.vid = vid; 8781 call_switchdev_notifiers(SWITCHDEV_FDB_DEL_TO_BRIDGE, dev, &info.info, 8782 NULL); 8783 } 8784 8785 static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_vlan_emu_ops = { 8786 .type = MLXSW_SP_RIF_TYPE_VLAN, 8787 .rif_size = sizeof(struct mlxsw_sp_rif), 8788 .configure = mlxsw_sp_rif_fid_configure, 8789 .deconfigure = mlxsw_sp_rif_fid_deconfigure, 8790 .fid_get = mlxsw_sp_rif_vlan_fid_get, 8791 .fdb_del = mlxsw_sp_rif_vlan_fdb_del, 8792 }; 8793 8794 static struct mlxsw_sp_rif_ipip_lb * 8795 mlxsw_sp_rif_ipip_lb_rif(struct mlxsw_sp_rif *rif) 8796 { 8797 return container_of(rif, struct mlxsw_sp_rif_ipip_lb, common); 8798 } 8799 8800 static void 8801 mlxsw_sp_rif_ipip_lb_setup(struct mlxsw_sp_rif *rif, 8802 const struct mlxsw_sp_rif_params *params) 8803 { 8804 struct mlxsw_sp_rif_params_ipip_lb *params_lb; 8805 struct mlxsw_sp_rif_ipip_lb *rif_lb; 8806 8807 params_lb = container_of(params, struct mlxsw_sp_rif_params_ipip_lb, 8808 common); 8809 rif_lb = mlxsw_sp_rif_ipip_lb_rif(rif); 8810 rif_lb->lb_config = params_lb->lb_config; 8811 } 8812 8813 static int 8814 mlxsw_sp1_rif_ipip_lb_configure(struct mlxsw_sp_rif *rif) 8815 { 8816 struct mlxsw_sp_rif_ipip_lb *lb_rif = mlxsw_sp_rif_ipip_lb_rif(rif); 8817 u32 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(rif->dev); 8818 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 8819 struct mlxsw_sp_vr *ul_vr; 8820 int err; 8821 8822 ul_vr = mlxsw_sp_vr_get(mlxsw_sp, ul_tb_id, NULL); 8823 if (IS_ERR(ul_vr)) 8824 return PTR_ERR(ul_vr); 8825 8826 err = mlxsw_sp_rif_ipip_lb_op(lb_rif, ul_vr->id, 0, true); 8827 if (err) 8828 goto err_loopback_op; 8829 8830 lb_rif->ul_vr_id = ul_vr->id; 8831 lb_rif->ul_rif_id = 0; 8832 ++ul_vr->rif_count; 8833 return 0; 8834 8835 err_loopback_op: 8836 mlxsw_sp_vr_put(mlxsw_sp, ul_vr); 8837 return err; 8838 } 8839 8840 static void mlxsw_sp1_rif_ipip_lb_deconfigure(struct mlxsw_sp_rif *rif) 8841 { 8842 struct mlxsw_sp_rif_ipip_lb *lb_rif = mlxsw_sp_rif_ipip_lb_rif(rif); 8843 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 8844 struct mlxsw_sp_vr *ul_vr; 8845 8846 ul_vr = &mlxsw_sp->router->vrs[lb_rif->ul_vr_id]; 8847 mlxsw_sp_rif_ipip_lb_op(lb_rif, ul_vr->id, 0, false); 8848 8849 --ul_vr->rif_count; 8850 mlxsw_sp_vr_put(mlxsw_sp, ul_vr); 8851 } 8852 8853 static const struct mlxsw_sp_rif_ops mlxsw_sp1_rif_ipip_lb_ops = { 8854 .type = MLXSW_SP_RIF_TYPE_IPIP_LB, 8855 .rif_size = sizeof(struct mlxsw_sp_rif_ipip_lb), 8856 .setup = mlxsw_sp_rif_ipip_lb_setup, 8857 .configure = mlxsw_sp1_rif_ipip_lb_configure, 8858 .deconfigure = mlxsw_sp1_rif_ipip_lb_deconfigure, 8859 }; 8860 8861 const struct mlxsw_sp_rif_ops *mlxsw_sp1_rif_ops_arr[] = { 8862 [MLXSW_SP_RIF_TYPE_SUBPORT] = &mlxsw_sp_rif_subport_ops, 8863 [MLXSW_SP_RIF_TYPE_VLAN] = &mlxsw_sp_rif_vlan_emu_ops, 8864 [MLXSW_SP_RIF_TYPE_FID] = &mlxsw_sp_rif_fid_ops, 8865 [MLXSW_SP_RIF_TYPE_IPIP_LB] = &mlxsw_sp1_rif_ipip_lb_ops, 8866 }; 8867 8868 static int 8869 mlxsw_sp_rif_ipip_lb_ul_rif_op(struct mlxsw_sp_rif *ul_rif, bool enable) 8870 { 8871 struct mlxsw_sp *mlxsw_sp = ul_rif->mlxsw_sp; 8872 char ritr_pl[MLXSW_REG_RITR_LEN]; 8873 8874 mlxsw_reg_ritr_pack(ritr_pl, enable, MLXSW_REG_RITR_LOOPBACK_IF, 8875 ul_rif->rif_index, ul_rif->vr_id, IP_MAX_MTU); 8876 mlxsw_reg_ritr_loopback_protocol_set(ritr_pl, 8877 MLXSW_REG_RITR_LOOPBACK_GENERIC); 8878 8879 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 8880 } 8881 8882 static struct mlxsw_sp_rif * 8883 mlxsw_sp_ul_rif_create(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp_vr *vr, 8884 struct netlink_ext_ack *extack) 8885 { 8886 struct mlxsw_sp_rif *ul_rif; 8887 u16 rif_index; 8888 int err; 8889 8890 err = mlxsw_sp_rif_index_alloc(mlxsw_sp, &rif_index); 8891 if (err) { 8892 NL_SET_ERR_MSG_MOD(extack, "Exceeded number of supported router interfaces"); 8893 return ERR_PTR(err); 8894 } 8895 8896 ul_rif = mlxsw_sp_rif_alloc(sizeof(*ul_rif), rif_index, vr->id, NULL); 8897 if (!ul_rif) 8898 return ERR_PTR(-ENOMEM); 8899 8900 mlxsw_sp->router->rifs[rif_index] = ul_rif; 8901 ul_rif->mlxsw_sp = mlxsw_sp; 8902 err = mlxsw_sp_rif_ipip_lb_ul_rif_op(ul_rif, true); 8903 if (err) 8904 goto ul_rif_op_err; 8905 8906 return ul_rif; 8907 8908 ul_rif_op_err: 8909 mlxsw_sp->router->rifs[rif_index] = NULL; 8910 kfree(ul_rif); 8911 return ERR_PTR(err); 8912 } 8913 8914 static void mlxsw_sp_ul_rif_destroy(struct mlxsw_sp_rif *ul_rif) 8915 { 8916 struct mlxsw_sp *mlxsw_sp = ul_rif->mlxsw_sp; 8917 8918 mlxsw_sp_rif_ipip_lb_ul_rif_op(ul_rif, false); 8919 mlxsw_sp->router->rifs[ul_rif->rif_index] = NULL; 8920 kfree(ul_rif); 8921 } 8922 8923 static struct mlxsw_sp_rif * 8924 mlxsw_sp_ul_rif_get(struct mlxsw_sp *mlxsw_sp, u32 tb_id, 8925 struct netlink_ext_ack *extack) 8926 { 8927 struct mlxsw_sp_vr *vr; 8928 int err; 8929 8930 vr = mlxsw_sp_vr_get(mlxsw_sp, tb_id, extack); 8931 if (IS_ERR(vr)) 8932 return ERR_CAST(vr); 8933 8934 if (refcount_inc_not_zero(&vr->ul_rif_refcnt)) 8935 return vr->ul_rif; 8936 8937 vr->ul_rif = mlxsw_sp_ul_rif_create(mlxsw_sp, vr, extack); 8938 if (IS_ERR(vr->ul_rif)) { 8939 err = PTR_ERR(vr->ul_rif); 8940 goto err_ul_rif_create; 8941 } 8942 8943 vr->rif_count++; 8944 refcount_set(&vr->ul_rif_refcnt, 1); 8945 8946 return vr->ul_rif; 8947 8948 err_ul_rif_create: 8949 mlxsw_sp_vr_put(mlxsw_sp, vr); 8950 return ERR_PTR(err); 8951 } 8952 8953 static void mlxsw_sp_ul_rif_put(struct mlxsw_sp_rif *ul_rif) 8954 { 8955 struct mlxsw_sp *mlxsw_sp = ul_rif->mlxsw_sp; 8956 struct mlxsw_sp_vr *vr; 8957 8958 vr = &mlxsw_sp->router->vrs[ul_rif->vr_id]; 8959 8960 if (!refcount_dec_and_test(&vr->ul_rif_refcnt)) 8961 return; 8962 8963 vr->rif_count--; 8964 mlxsw_sp_ul_rif_destroy(ul_rif); 8965 mlxsw_sp_vr_put(mlxsw_sp, vr); 8966 } 8967 8968 int mlxsw_sp_router_ul_rif_get(struct mlxsw_sp *mlxsw_sp, u32 ul_tb_id, 8969 u16 *ul_rif_index) 8970 { 8971 struct mlxsw_sp_rif *ul_rif; 8972 int err = 0; 8973 8974 mutex_lock(&mlxsw_sp->router->lock); 8975 ul_rif = mlxsw_sp_ul_rif_get(mlxsw_sp, ul_tb_id, NULL); 8976 if (IS_ERR(ul_rif)) { 8977 err = PTR_ERR(ul_rif); 8978 goto out; 8979 } 8980 *ul_rif_index = ul_rif->rif_index; 8981 out: 8982 mutex_unlock(&mlxsw_sp->router->lock); 8983 return err; 8984 } 8985 8986 void mlxsw_sp_router_ul_rif_put(struct mlxsw_sp *mlxsw_sp, u16 ul_rif_index) 8987 { 8988 struct mlxsw_sp_rif *ul_rif; 8989 8990 mutex_lock(&mlxsw_sp->router->lock); 8991 ul_rif = mlxsw_sp->router->rifs[ul_rif_index]; 8992 if (WARN_ON(!ul_rif)) 8993 goto out; 8994 8995 mlxsw_sp_ul_rif_put(ul_rif); 8996 out: 8997 mutex_unlock(&mlxsw_sp->router->lock); 8998 } 8999 9000 static int 9001 mlxsw_sp2_rif_ipip_lb_configure(struct mlxsw_sp_rif *rif) 9002 { 9003 struct mlxsw_sp_rif_ipip_lb *lb_rif = mlxsw_sp_rif_ipip_lb_rif(rif); 9004 u32 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(rif->dev); 9005 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 9006 struct mlxsw_sp_rif *ul_rif; 9007 int err; 9008 9009 ul_rif = mlxsw_sp_ul_rif_get(mlxsw_sp, ul_tb_id, NULL); 9010 if (IS_ERR(ul_rif)) 9011 return PTR_ERR(ul_rif); 9012 9013 err = mlxsw_sp_rif_ipip_lb_op(lb_rif, 0, ul_rif->rif_index, true); 9014 if (err) 9015 goto err_loopback_op; 9016 9017 lb_rif->ul_vr_id = 0; 9018 lb_rif->ul_rif_id = ul_rif->rif_index; 9019 9020 return 0; 9021 9022 err_loopback_op: 9023 mlxsw_sp_ul_rif_put(ul_rif); 9024 return err; 9025 } 9026 9027 static void mlxsw_sp2_rif_ipip_lb_deconfigure(struct mlxsw_sp_rif *rif) 9028 { 9029 struct mlxsw_sp_rif_ipip_lb *lb_rif = mlxsw_sp_rif_ipip_lb_rif(rif); 9030 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 9031 struct mlxsw_sp_rif *ul_rif; 9032 9033 ul_rif = mlxsw_sp_rif_by_index(mlxsw_sp, lb_rif->ul_rif_id); 9034 mlxsw_sp_rif_ipip_lb_op(lb_rif, 0, lb_rif->ul_rif_id, false); 9035 mlxsw_sp_ul_rif_put(ul_rif); 9036 } 9037 9038 static const struct mlxsw_sp_rif_ops mlxsw_sp2_rif_ipip_lb_ops = { 9039 .type = MLXSW_SP_RIF_TYPE_IPIP_LB, 9040 .rif_size = sizeof(struct mlxsw_sp_rif_ipip_lb), 9041 .setup = mlxsw_sp_rif_ipip_lb_setup, 9042 .configure = mlxsw_sp2_rif_ipip_lb_configure, 9043 .deconfigure = mlxsw_sp2_rif_ipip_lb_deconfigure, 9044 }; 9045 9046 const struct mlxsw_sp_rif_ops *mlxsw_sp2_rif_ops_arr[] = { 9047 [MLXSW_SP_RIF_TYPE_SUBPORT] = &mlxsw_sp_rif_subport_ops, 9048 [MLXSW_SP_RIF_TYPE_VLAN] = &mlxsw_sp_rif_vlan_emu_ops, 9049 [MLXSW_SP_RIF_TYPE_FID] = &mlxsw_sp_rif_fid_ops, 9050 [MLXSW_SP_RIF_TYPE_IPIP_LB] = &mlxsw_sp2_rif_ipip_lb_ops, 9051 }; 9052 9053 static int mlxsw_sp_rifs_init(struct mlxsw_sp *mlxsw_sp) 9054 { 9055 u64 max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); 9056 9057 mlxsw_sp->router->rifs = kcalloc(max_rifs, 9058 sizeof(struct mlxsw_sp_rif *), 9059 GFP_KERNEL); 9060 if (!mlxsw_sp->router->rifs) 9061 return -ENOMEM; 9062 9063 return 0; 9064 } 9065 9066 static void mlxsw_sp_rifs_fini(struct mlxsw_sp *mlxsw_sp) 9067 { 9068 int i; 9069 9070 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++) 9071 WARN_ON_ONCE(mlxsw_sp->router->rifs[i]); 9072 9073 kfree(mlxsw_sp->router->rifs); 9074 } 9075 9076 static int 9077 mlxsw_sp_ipip_config_tigcr(struct mlxsw_sp *mlxsw_sp) 9078 { 9079 char tigcr_pl[MLXSW_REG_TIGCR_LEN]; 9080 9081 mlxsw_reg_tigcr_pack(tigcr_pl, true, 0); 9082 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(tigcr), tigcr_pl); 9083 } 9084 9085 static int mlxsw_sp_ipips_init(struct mlxsw_sp *mlxsw_sp) 9086 { 9087 int err; 9088 9089 mlxsw_sp->router->ipip_ops_arr = mlxsw_sp_ipip_ops_arr; 9090 INIT_LIST_HEAD(&mlxsw_sp->router->ipip_list); 9091 9092 err = mlxsw_sp_ipip_ecn_encap_init(mlxsw_sp); 9093 if (err) 9094 return err; 9095 err = mlxsw_sp_ipip_ecn_decap_init(mlxsw_sp); 9096 if (err) 9097 return err; 9098 9099 return mlxsw_sp_ipip_config_tigcr(mlxsw_sp); 9100 } 9101 9102 static void mlxsw_sp_ipips_fini(struct mlxsw_sp *mlxsw_sp) 9103 { 9104 WARN_ON(!list_empty(&mlxsw_sp->router->ipip_list)); 9105 } 9106 9107 static void mlxsw_sp_router_fib_dump_flush(struct notifier_block *nb) 9108 { 9109 struct mlxsw_sp_router *router; 9110 9111 /* Flush pending FIB notifications and then flush the device's 9112 * table before requesting another dump. The FIB notification 9113 * block is unregistered, so no need to take RTNL. 9114 */ 9115 mlxsw_core_flush_owq(); 9116 router = container_of(nb, struct mlxsw_sp_router, fib_nb); 9117 mlxsw_sp_router_fib_flush(router->mlxsw_sp); 9118 } 9119 9120 #ifdef CONFIG_IP_ROUTE_MULTIPATH 9121 static void mlxsw_sp_mp_hash_header_set(char *recr2_pl, int header) 9122 { 9123 mlxsw_reg_recr2_outer_header_enables_set(recr2_pl, header, true); 9124 } 9125 9126 static void mlxsw_sp_mp_hash_field_set(char *recr2_pl, int field) 9127 { 9128 mlxsw_reg_recr2_outer_header_fields_enable_set(recr2_pl, field, true); 9129 } 9130 9131 static void mlxsw_sp_mp4_hash_init(struct mlxsw_sp *mlxsw_sp, char *recr2_pl) 9132 { 9133 struct net *net = mlxsw_sp_net(mlxsw_sp); 9134 bool only_l3 = !net->ipv4.sysctl_fib_multipath_hash_policy; 9135 9136 mlxsw_sp_mp_hash_header_set(recr2_pl, 9137 MLXSW_REG_RECR2_IPV4_EN_NOT_TCP_NOT_UDP); 9138 mlxsw_sp_mp_hash_header_set(recr2_pl, MLXSW_REG_RECR2_IPV4_EN_TCP_UDP); 9139 mlxsw_reg_recr2_ipv4_sip_enable(recr2_pl); 9140 mlxsw_reg_recr2_ipv4_dip_enable(recr2_pl); 9141 if (only_l3) 9142 return; 9143 mlxsw_sp_mp_hash_header_set(recr2_pl, MLXSW_REG_RECR2_TCP_UDP_EN_IPV4); 9144 mlxsw_sp_mp_hash_field_set(recr2_pl, MLXSW_REG_RECR2_IPV4_PROTOCOL); 9145 mlxsw_sp_mp_hash_field_set(recr2_pl, MLXSW_REG_RECR2_TCP_UDP_SPORT); 9146 mlxsw_sp_mp_hash_field_set(recr2_pl, MLXSW_REG_RECR2_TCP_UDP_DPORT); 9147 } 9148 9149 static void mlxsw_sp_mp6_hash_init(struct mlxsw_sp *mlxsw_sp, char *recr2_pl) 9150 { 9151 bool only_l3 = !ip6_multipath_hash_policy(mlxsw_sp_net(mlxsw_sp)); 9152 9153 mlxsw_sp_mp_hash_header_set(recr2_pl, 9154 MLXSW_REG_RECR2_IPV6_EN_NOT_TCP_NOT_UDP); 9155 mlxsw_sp_mp_hash_header_set(recr2_pl, MLXSW_REG_RECR2_IPV6_EN_TCP_UDP); 9156 mlxsw_reg_recr2_ipv6_sip_enable(recr2_pl); 9157 mlxsw_reg_recr2_ipv6_dip_enable(recr2_pl); 9158 mlxsw_sp_mp_hash_field_set(recr2_pl, MLXSW_REG_RECR2_IPV6_NEXT_HEADER); 9159 if (only_l3) { 9160 mlxsw_sp_mp_hash_field_set(recr2_pl, 9161 MLXSW_REG_RECR2_IPV6_FLOW_LABEL); 9162 } else { 9163 mlxsw_sp_mp_hash_header_set(recr2_pl, 9164 MLXSW_REG_RECR2_TCP_UDP_EN_IPV6); 9165 mlxsw_sp_mp_hash_field_set(recr2_pl, 9166 MLXSW_REG_RECR2_TCP_UDP_SPORT); 9167 mlxsw_sp_mp_hash_field_set(recr2_pl, 9168 MLXSW_REG_RECR2_TCP_UDP_DPORT); 9169 } 9170 } 9171 9172 static int mlxsw_sp_mp_hash_init(struct mlxsw_sp *mlxsw_sp) 9173 { 9174 char recr2_pl[MLXSW_REG_RECR2_LEN]; 9175 u32 seed; 9176 9177 seed = jhash(mlxsw_sp->base_mac, sizeof(mlxsw_sp->base_mac), 0); 9178 mlxsw_reg_recr2_pack(recr2_pl, seed); 9179 mlxsw_sp_mp4_hash_init(mlxsw_sp, recr2_pl); 9180 mlxsw_sp_mp6_hash_init(mlxsw_sp, recr2_pl); 9181 9182 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(recr2), recr2_pl); 9183 } 9184 #else 9185 static int mlxsw_sp_mp_hash_init(struct mlxsw_sp *mlxsw_sp) 9186 { 9187 return 0; 9188 } 9189 #endif 9190 9191 static int mlxsw_sp_dscp_init(struct mlxsw_sp *mlxsw_sp) 9192 { 9193 char rdpm_pl[MLXSW_REG_RDPM_LEN]; 9194 unsigned int i; 9195 9196 MLXSW_REG_ZERO(rdpm, rdpm_pl); 9197 9198 /* HW is determining switch priority based on DSCP-bits, but the 9199 * kernel is still doing that based on the ToS. Since there's a 9200 * mismatch in bits we need to make sure to translate the right 9201 * value ToS would observe, skipping the 2 least-significant ECN bits. 9202 */ 9203 for (i = 0; i < MLXSW_REG_RDPM_DSCP_ENTRY_REC_MAX_COUNT; i++) 9204 mlxsw_reg_rdpm_pack(rdpm_pl, i, rt_tos2priority(i << 2)); 9205 9206 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rdpm), rdpm_pl); 9207 } 9208 9209 static int __mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp) 9210 { 9211 struct net *net = mlxsw_sp_net(mlxsw_sp); 9212 bool usp = net->ipv4.sysctl_ip_fwd_update_priority; 9213 char rgcr_pl[MLXSW_REG_RGCR_LEN]; 9214 u64 max_rifs; 9215 9216 if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_RIFS)) 9217 return -EIO; 9218 max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); 9219 9220 mlxsw_reg_rgcr_pack(rgcr_pl, true, true); 9221 mlxsw_reg_rgcr_max_router_interfaces_set(rgcr_pl, max_rifs); 9222 mlxsw_reg_rgcr_usp_set(rgcr_pl, usp); 9223 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rgcr), rgcr_pl); 9224 } 9225 9226 static void __mlxsw_sp_router_fini(struct mlxsw_sp *mlxsw_sp) 9227 { 9228 char rgcr_pl[MLXSW_REG_RGCR_LEN]; 9229 9230 mlxsw_reg_rgcr_pack(rgcr_pl, false, false); 9231 mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rgcr), rgcr_pl); 9232 } 9233 9234 static const struct mlxsw_sp_router_ll_ops mlxsw_sp_router_ll_basic_ops = { 9235 .init = mlxsw_sp_router_ll_basic_init, 9236 .ralta_write = mlxsw_sp_router_ll_basic_ralta_write, 9237 .ralst_write = mlxsw_sp_router_ll_basic_ralst_write, 9238 .raltb_write = mlxsw_sp_router_ll_basic_raltb_write, 9239 .fib_entry_op_ctx_size = sizeof(struct mlxsw_sp_fib_entry_op_ctx_basic), 9240 .fib_entry_pack = mlxsw_sp_router_ll_basic_fib_entry_pack, 9241 .fib_entry_act_remote_pack = mlxsw_sp_router_ll_basic_fib_entry_act_remote_pack, 9242 .fib_entry_act_local_pack = mlxsw_sp_router_ll_basic_fib_entry_act_local_pack, 9243 .fib_entry_act_ip2me_pack = mlxsw_sp_router_ll_basic_fib_entry_act_ip2me_pack, 9244 .fib_entry_act_ip2me_tun_pack = mlxsw_sp_router_ll_basic_fib_entry_act_ip2me_tun_pack, 9245 .fib_entry_commit = mlxsw_sp_router_ll_basic_fib_entry_commit, 9246 .fib_entry_is_committed = mlxsw_sp_router_ll_basic_fib_entry_is_committed, 9247 }; 9248 9249 static int mlxsw_sp_router_ll_op_ctx_init(struct mlxsw_sp_router *router) 9250 { 9251 size_t max_size = 0; 9252 int i; 9253 9254 for (i = 0; i < MLXSW_SP_L3_PROTO_MAX; i++) { 9255 size_t size = router->proto_ll_ops[i]->fib_entry_op_ctx_size; 9256 9257 if (size > max_size) 9258 max_size = size; 9259 } 9260 router->ll_op_ctx = kzalloc(sizeof(*router->ll_op_ctx) + max_size, 9261 GFP_KERNEL); 9262 if (!router->ll_op_ctx) 9263 return -ENOMEM; 9264 INIT_LIST_HEAD(&router->ll_op_ctx->fib_entry_priv_list); 9265 return 0; 9266 } 9267 9268 static void mlxsw_sp_router_ll_op_ctx_fini(struct mlxsw_sp_router *router) 9269 { 9270 WARN_ON(!list_empty(&router->ll_op_ctx->fib_entry_priv_list)); 9271 kfree(router->ll_op_ctx); 9272 } 9273 9274 static int mlxsw_sp_lb_rif_init(struct mlxsw_sp *mlxsw_sp) 9275 { 9276 u16 lb_rif_index; 9277 int err; 9278 9279 /* Create a generic loopback RIF associated with the main table 9280 * (default VRF). Any table can be used, but the main table exists 9281 * anyway, so we do not waste resources. 9282 */ 9283 err = mlxsw_sp_router_ul_rif_get(mlxsw_sp, RT_TABLE_MAIN, 9284 &lb_rif_index); 9285 if (err) 9286 return err; 9287 9288 mlxsw_sp->router->lb_rif_index = lb_rif_index; 9289 9290 return 0; 9291 } 9292 9293 static void mlxsw_sp_lb_rif_fini(struct mlxsw_sp *mlxsw_sp) 9294 { 9295 mlxsw_sp_router_ul_rif_put(mlxsw_sp, mlxsw_sp->router->lb_rif_index); 9296 } 9297 9298 int mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp, 9299 struct netlink_ext_ack *extack) 9300 { 9301 struct mlxsw_sp_router *router; 9302 int err; 9303 9304 router = kzalloc(sizeof(*mlxsw_sp->router), GFP_KERNEL); 9305 if (!router) 9306 return -ENOMEM; 9307 mutex_init(&router->lock); 9308 mlxsw_sp->router = router; 9309 router->mlxsw_sp = mlxsw_sp; 9310 9311 err = mlxsw_sp_router_xm_init(mlxsw_sp); 9312 if (err) 9313 goto err_xm_init; 9314 9315 router->proto_ll_ops[MLXSW_SP_L3_PROTO_IPV4] = mlxsw_sp_router_xm_ipv4_is_supported(mlxsw_sp) ? 9316 &mlxsw_sp_router_ll_xm_ops : 9317 &mlxsw_sp_router_ll_basic_ops; 9318 router->proto_ll_ops[MLXSW_SP_L3_PROTO_IPV6] = &mlxsw_sp_router_ll_basic_ops; 9319 9320 err = mlxsw_sp_router_ll_op_ctx_init(router); 9321 if (err) 9322 goto err_ll_op_ctx_init; 9323 9324 INIT_LIST_HEAD(&mlxsw_sp->router->nexthop_neighs_list); 9325 err = __mlxsw_sp_router_init(mlxsw_sp); 9326 if (err) 9327 goto err_router_init; 9328 9329 err = mlxsw_sp_rifs_init(mlxsw_sp); 9330 if (err) 9331 goto err_rifs_init; 9332 9333 err = mlxsw_sp_ipips_init(mlxsw_sp); 9334 if (err) 9335 goto err_ipips_init; 9336 9337 err = rhashtable_init(&mlxsw_sp->router->nexthop_ht, 9338 &mlxsw_sp_nexthop_ht_params); 9339 if (err) 9340 goto err_nexthop_ht_init; 9341 9342 err = rhashtable_init(&mlxsw_sp->router->nexthop_group_ht, 9343 &mlxsw_sp_nexthop_group_ht_params); 9344 if (err) 9345 goto err_nexthop_group_ht_init; 9346 9347 INIT_LIST_HEAD(&mlxsw_sp->router->nexthop_list); 9348 err = mlxsw_sp_lpm_init(mlxsw_sp); 9349 if (err) 9350 goto err_lpm_init; 9351 9352 err = mlxsw_sp_mr_init(mlxsw_sp, &mlxsw_sp_mr_tcam_ops); 9353 if (err) 9354 goto err_mr_init; 9355 9356 err = mlxsw_sp_vrs_init(mlxsw_sp); 9357 if (err) 9358 goto err_vrs_init; 9359 9360 err = mlxsw_sp_lb_rif_init(mlxsw_sp); 9361 if (err) 9362 goto err_lb_rif_init; 9363 9364 err = mlxsw_sp_neigh_init(mlxsw_sp); 9365 if (err) 9366 goto err_neigh_init; 9367 9368 err = mlxsw_sp_mp_hash_init(mlxsw_sp); 9369 if (err) 9370 goto err_mp_hash_init; 9371 9372 err = mlxsw_sp_dscp_init(mlxsw_sp); 9373 if (err) 9374 goto err_dscp_init; 9375 9376 INIT_WORK(&router->fib_event_work, mlxsw_sp_router_fib_event_work); 9377 INIT_LIST_HEAD(&router->fib_event_queue); 9378 spin_lock_init(&router->fib_event_queue_lock); 9379 9380 router->inetaddr_nb.notifier_call = mlxsw_sp_inetaddr_event; 9381 err = register_inetaddr_notifier(&router->inetaddr_nb); 9382 if (err) 9383 goto err_register_inetaddr_notifier; 9384 9385 router->inet6addr_nb.notifier_call = mlxsw_sp_inet6addr_event; 9386 err = register_inet6addr_notifier(&router->inet6addr_nb); 9387 if (err) 9388 goto err_register_inet6addr_notifier; 9389 9390 mlxsw_sp->router->netevent_nb.notifier_call = 9391 mlxsw_sp_router_netevent_event; 9392 err = register_netevent_notifier(&mlxsw_sp->router->netevent_nb); 9393 if (err) 9394 goto err_register_netevent_notifier; 9395 9396 mlxsw_sp->router->nexthop_nb.notifier_call = 9397 mlxsw_sp_nexthop_obj_event; 9398 err = register_nexthop_notifier(mlxsw_sp_net(mlxsw_sp), 9399 &mlxsw_sp->router->nexthop_nb, 9400 extack); 9401 if (err) 9402 goto err_register_nexthop_notifier; 9403 9404 mlxsw_sp->router->fib_nb.notifier_call = mlxsw_sp_router_fib_event; 9405 err = register_fib_notifier(mlxsw_sp_net(mlxsw_sp), 9406 &mlxsw_sp->router->fib_nb, 9407 mlxsw_sp_router_fib_dump_flush, extack); 9408 if (err) 9409 goto err_register_fib_notifier; 9410 9411 return 0; 9412 9413 err_register_fib_notifier: 9414 unregister_nexthop_notifier(mlxsw_sp_net(mlxsw_sp), 9415 &mlxsw_sp->router->nexthop_nb); 9416 err_register_nexthop_notifier: 9417 unregister_netevent_notifier(&mlxsw_sp->router->netevent_nb); 9418 err_register_netevent_notifier: 9419 unregister_inet6addr_notifier(&router->inet6addr_nb); 9420 err_register_inet6addr_notifier: 9421 unregister_inetaddr_notifier(&router->inetaddr_nb); 9422 err_register_inetaddr_notifier: 9423 mlxsw_core_flush_owq(); 9424 WARN_ON(!list_empty(&router->fib_event_queue)); 9425 err_dscp_init: 9426 err_mp_hash_init: 9427 mlxsw_sp_neigh_fini(mlxsw_sp); 9428 err_neigh_init: 9429 mlxsw_sp_lb_rif_fini(mlxsw_sp); 9430 err_lb_rif_init: 9431 mlxsw_sp_vrs_fini(mlxsw_sp); 9432 err_vrs_init: 9433 mlxsw_sp_mr_fini(mlxsw_sp); 9434 err_mr_init: 9435 mlxsw_sp_lpm_fini(mlxsw_sp); 9436 err_lpm_init: 9437 rhashtable_destroy(&mlxsw_sp->router->nexthop_group_ht); 9438 err_nexthop_group_ht_init: 9439 rhashtable_destroy(&mlxsw_sp->router->nexthop_ht); 9440 err_nexthop_ht_init: 9441 mlxsw_sp_ipips_fini(mlxsw_sp); 9442 err_ipips_init: 9443 mlxsw_sp_rifs_fini(mlxsw_sp); 9444 err_rifs_init: 9445 __mlxsw_sp_router_fini(mlxsw_sp); 9446 err_router_init: 9447 mlxsw_sp_router_ll_op_ctx_fini(router); 9448 err_ll_op_ctx_init: 9449 mlxsw_sp_router_xm_fini(mlxsw_sp); 9450 err_xm_init: 9451 mutex_destroy(&mlxsw_sp->router->lock); 9452 kfree(mlxsw_sp->router); 9453 return err; 9454 } 9455 9456 void mlxsw_sp_router_fini(struct mlxsw_sp *mlxsw_sp) 9457 { 9458 unregister_fib_notifier(mlxsw_sp_net(mlxsw_sp), 9459 &mlxsw_sp->router->fib_nb); 9460 unregister_nexthop_notifier(mlxsw_sp_net(mlxsw_sp), 9461 &mlxsw_sp->router->nexthop_nb); 9462 unregister_netevent_notifier(&mlxsw_sp->router->netevent_nb); 9463 unregister_inet6addr_notifier(&mlxsw_sp->router->inet6addr_nb); 9464 unregister_inetaddr_notifier(&mlxsw_sp->router->inetaddr_nb); 9465 mlxsw_core_flush_owq(); 9466 WARN_ON(!list_empty(&mlxsw_sp->router->fib_event_queue)); 9467 mlxsw_sp_neigh_fini(mlxsw_sp); 9468 mlxsw_sp_lb_rif_fini(mlxsw_sp); 9469 mlxsw_sp_vrs_fini(mlxsw_sp); 9470 mlxsw_sp_mr_fini(mlxsw_sp); 9471 mlxsw_sp_lpm_fini(mlxsw_sp); 9472 rhashtable_destroy(&mlxsw_sp->router->nexthop_group_ht); 9473 rhashtable_destroy(&mlxsw_sp->router->nexthop_ht); 9474 mlxsw_sp_ipips_fini(mlxsw_sp); 9475 mlxsw_sp_rifs_fini(mlxsw_sp); 9476 __mlxsw_sp_router_fini(mlxsw_sp); 9477 mlxsw_sp_router_ll_op_ctx_fini(mlxsw_sp->router); 9478 mlxsw_sp_router_xm_fini(mlxsw_sp); 9479 mutex_destroy(&mlxsw_sp->router->lock); 9480 kfree(mlxsw_sp->router); 9481 } 9482