1 // SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0 2 /* Copyright (c) 2016-2018 Mellanox Technologies. All rights reserved */ 3 4 #include <linux/kernel.h> 5 #include <linux/types.h> 6 #include <linux/rhashtable.h> 7 #include <linux/bitops.h> 8 #include <linux/in6.h> 9 #include <linux/notifier.h> 10 #include <linux/inetdevice.h> 11 #include <linux/netdevice.h> 12 #include <linux/if_bridge.h> 13 #include <linux/socket.h> 14 #include <linux/route.h> 15 #include <linux/gcd.h> 16 #include <linux/if_macvlan.h> 17 #include <linux/refcount.h> 18 #include <linux/jhash.h> 19 #include <linux/net_namespace.h> 20 #include <linux/mutex.h> 21 #include <linux/genalloc.h> 22 #include <net/netevent.h> 23 #include <net/neighbour.h> 24 #include <net/arp.h> 25 #include <net/inet_dscp.h> 26 #include <net/ip_fib.h> 27 #include <net/ip6_fib.h> 28 #include <net/nexthop.h> 29 #include <net/fib_rules.h> 30 #include <net/ip_tunnels.h> 31 #include <net/l3mdev.h> 32 #include <net/addrconf.h> 33 #include <net/ndisc.h> 34 #include <net/ipv6.h> 35 #include <net/fib_notifier.h> 36 #include <net/switchdev.h> 37 38 #include "spectrum.h" 39 #include "core.h" 40 #include "reg.h" 41 #include "spectrum_cnt.h" 42 #include "spectrum_dpipe.h" 43 #include "spectrum_ipip.h" 44 #include "spectrum_mr.h" 45 #include "spectrum_mr_tcam.h" 46 #include "spectrum_router.h" 47 #include "spectrum_span.h" 48 49 struct mlxsw_sp_fib; 50 struct mlxsw_sp_vr; 51 struct mlxsw_sp_lpm_tree; 52 struct mlxsw_sp_rif_ops; 53 54 struct mlxsw_sp_crif_key { 55 struct net_device *dev; 56 }; 57 58 struct mlxsw_sp_crif { 59 struct mlxsw_sp_crif_key key; 60 struct rhash_head ht_node; 61 bool can_destroy; 62 struct list_head nexthop_list; 63 struct mlxsw_sp_rif *rif; 64 }; 65 66 static const struct rhashtable_params mlxsw_sp_crif_ht_params = { 67 .key_offset = offsetof(struct mlxsw_sp_crif, key), 68 .key_len = sizeof_field(struct mlxsw_sp_crif, key), 69 .head_offset = offsetof(struct mlxsw_sp_crif, ht_node), 70 }; 71 72 struct mlxsw_sp_rif { 73 struct mlxsw_sp_crif *crif; /* NULL for underlay RIF */ 74 netdevice_tracker dev_tracker; 75 struct list_head neigh_list; 76 struct mlxsw_sp_fid *fid; 77 unsigned char addr[ETH_ALEN]; 78 int mtu; 79 u16 rif_index; 80 u8 mac_profile_id; 81 u8 rif_entries; 82 u16 vr_id; 83 const struct mlxsw_sp_rif_ops *ops; 84 struct mlxsw_sp *mlxsw_sp; 85 86 unsigned int counter_ingress; 87 bool counter_ingress_valid; 88 unsigned int counter_egress; 89 bool counter_egress_valid; 90 }; 91 92 static struct net_device *mlxsw_sp_rif_dev(const struct mlxsw_sp_rif *rif) 93 { 94 if (!rif->crif) 95 return NULL; 96 return rif->crif->key.dev; 97 } 98 99 struct mlxsw_sp_rif_params { 100 struct net_device *dev; 101 union { 102 u16 system_port; 103 u16 lag_id; 104 }; 105 u16 vid; 106 bool lag; 107 bool double_entry; 108 }; 109 110 struct mlxsw_sp_rif_subport { 111 struct mlxsw_sp_rif common; 112 refcount_t ref_count; 113 union { 114 u16 system_port; 115 u16 lag_id; 116 }; 117 u16 vid; 118 bool lag; 119 }; 120 121 struct mlxsw_sp_rif_ipip_lb { 122 struct mlxsw_sp_rif common; 123 struct mlxsw_sp_rif_ipip_lb_config lb_config; 124 u16 ul_vr_id; /* Spectrum-1. */ 125 u16 ul_rif_id; /* Spectrum-2+. */ 126 }; 127 128 struct mlxsw_sp_rif_params_ipip_lb { 129 struct mlxsw_sp_rif_params common; 130 struct mlxsw_sp_rif_ipip_lb_config lb_config; 131 }; 132 133 struct mlxsw_sp_rif_ops { 134 enum mlxsw_sp_rif_type type; 135 size_t rif_size; 136 137 void (*setup)(struct mlxsw_sp_rif *rif, 138 const struct mlxsw_sp_rif_params *params); 139 int (*configure)(struct mlxsw_sp_rif *rif, 140 struct netlink_ext_ack *extack); 141 void (*deconfigure)(struct mlxsw_sp_rif *rif); 142 struct mlxsw_sp_fid * (*fid_get)(struct mlxsw_sp_rif *rif, 143 const struct mlxsw_sp_rif_params *params, 144 struct netlink_ext_ack *extack); 145 void (*fdb_del)(struct mlxsw_sp_rif *rif, const char *mac); 146 }; 147 148 struct mlxsw_sp_rif_mac_profile { 149 unsigned char mac_prefix[ETH_ALEN]; 150 refcount_t ref_count; 151 u8 id; 152 }; 153 154 struct mlxsw_sp_router_ops { 155 int (*init)(struct mlxsw_sp *mlxsw_sp); 156 int (*ipips_init)(struct mlxsw_sp *mlxsw_sp); 157 }; 158 159 static struct mlxsw_sp_rif * 160 mlxsw_sp_rif_find_by_dev(const struct mlxsw_sp *mlxsw_sp, 161 const struct net_device *dev); 162 static void mlxsw_sp_rif_destroy(struct mlxsw_sp_rif *rif); 163 static void mlxsw_sp_lpm_tree_hold(struct mlxsw_sp_lpm_tree *lpm_tree); 164 static void mlxsw_sp_lpm_tree_put(struct mlxsw_sp *mlxsw_sp, 165 struct mlxsw_sp_lpm_tree *lpm_tree); 166 static int mlxsw_sp_vr_lpm_tree_bind(struct mlxsw_sp *mlxsw_sp, 167 const struct mlxsw_sp_fib *fib, 168 u8 tree_id); 169 static int mlxsw_sp_vr_lpm_tree_unbind(struct mlxsw_sp *mlxsw_sp, 170 const struct mlxsw_sp_fib *fib); 171 172 static unsigned int * 173 mlxsw_sp_rif_p_counter_get(struct mlxsw_sp_rif *rif, 174 enum mlxsw_sp_rif_counter_dir dir) 175 { 176 switch (dir) { 177 case MLXSW_SP_RIF_COUNTER_EGRESS: 178 return &rif->counter_egress; 179 case MLXSW_SP_RIF_COUNTER_INGRESS: 180 return &rif->counter_ingress; 181 } 182 return NULL; 183 } 184 185 static bool 186 mlxsw_sp_rif_counter_valid_get(struct mlxsw_sp_rif *rif, 187 enum mlxsw_sp_rif_counter_dir dir) 188 { 189 switch (dir) { 190 case MLXSW_SP_RIF_COUNTER_EGRESS: 191 return rif->counter_egress_valid; 192 case MLXSW_SP_RIF_COUNTER_INGRESS: 193 return rif->counter_ingress_valid; 194 } 195 return false; 196 } 197 198 static void 199 mlxsw_sp_rif_counter_valid_set(struct mlxsw_sp_rif *rif, 200 enum mlxsw_sp_rif_counter_dir dir, 201 bool valid) 202 { 203 switch (dir) { 204 case MLXSW_SP_RIF_COUNTER_EGRESS: 205 rif->counter_egress_valid = valid; 206 break; 207 case MLXSW_SP_RIF_COUNTER_INGRESS: 208 rif->counter_ingress_valid = valid; 209 break; 210 } 211 } 212 213 static int mlxsw_sp_rif_counter_edit(struct mlxsw_sp *mlxsw_sp, u16 rif_index, 214 unsigned int counter_index, bool enable, 215 enum mlxsw_sp_rif_counter_dir dir) 216 { 217 char ritr_pl[MLXSW_REG_RITR_LEN]; 218 bool is_egress = false; 219 int err; 220 221 if (dir == MLXSW_SP_RIF_COUNTER_EGRESS) 222 is_egress = true; 223 mlxsw_reg_ritr_rif_pack(ritr_pl, rif_index); 224 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 225 if (err) 226 return err; 227 228 mlxsw_reg_ritr_counter_pack(ritr_pl, counter_index, enable, 229 is_egress); 230 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 231 } 232 233 int mlxsw_sp_rif_counter_value_get(struct mlxsw_sp *mlxsw_sp, 234 struct mlxsw_sp_rif *rif, 235 enum mlxsw_sp_rif_counter_dir dir, u64 *cnt) 236 { 237 char ricnt_pl[MLXSW_REG_RICNT_LEN]; 238 unsigned int *p_counter_index; 239 bool valid; 240 int err; 241 242 valid = mlxsw_sp_rif_counter_valid_get(rif, dir); 243 if (!valid) 244 return -EINVAL; 245 246 p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir); 247 if (!p_counter_index) 248 return -EINVAL; 249 mlxsw_reg_ricnt_pack(ricnt_pl, *p_counter_index, 250 MLXSW_REG_RICNT_OPCODE_NOP); 251 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ricnt), ricnt_pl); 252 if (err) 253 return err; 254 *cnt = mlxsw_reg_ricnt_good_unicast_packets_get(ricnt_pl); 255 return 0; 256 } 257 258 struct mlxsw_sp_rif_counter_set_basic { 259 u64 good_unicast_packets; 260 u64 good_multicast_packets; 261 u64 good_broadcast_packets; 262 u64 good_unicast_bytes; 263 u64 good_multicast_bytes; 264 u64 good_broadcast_bytes; 265 u64 error_packets; 266 u64 discard_packets; 267 u64 error_bytes; 268 u64 discard_bytes; 269 }; 270 271 static int 272 mlxsw_sp_rif_counter_fetch_clear(struct mlxsw_sp_rif *rif, 273 enum mlxsw_sp_rif_counter_dir dir, 274 struct mlxsw_sp_rif_counter_set_basic *set) 275 { 276 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 277 char ricnt_pl[MLXSW_REG_RICNT_LEN]; 278 unsigned int *p_counter_index; 279 int err; 280 281 if (!mlxsw_sp_rif_counter_valid_get(rif, dir)) 282 return -EINVAL; 283 284 p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir); 285 if (!p_counter_index) 286 return -EINVAL; 287 288 mlxsw_reg_ricnt_pack(ricnt_pl, *p_counter_index, 289 MLXSW_REG_RICNT_OPCODE_CLEAR); 290 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ricnt), ricnt_pl); 291 if (err) 292 return err; 293 294 if (!set) 295 return 0; 296 297 #define MLXSW_SP_RIF_COUNTER_EXTRACT(NAME) \ 298 (set->NAME = mlxsw_reg_ricnt_ ## NAME ## _get(ricnt_pl)) 299 300 MLXSW_SP_RIF_COUNTER_EXTRACT(good_unicast_packets); 301 MLXSW_SP_RIF_COUNTER_EXTRACT(good_multicast_packets); 302 MLXSW_SP_RIF_COUNTER_EXTRACT(good_broadcast_packets); 303 MLXSW_SP_RIF_COUNTER_EXTRACT(good_unicast_bytes); 304 MLXSW_SP_RIF_COUNTER_EXTRACT(good_multicast_bytes); 305 MLXSW_SP_RIF_COUNTER_EXTRACT(good_broadcast_bytes); 306 MLXSW_SP_RIF_COUNTER_EXTRACT(error_packets); 307 MLXSW_SP_RIF_COUNTER_EXTRACT(discard_packets); 308 MLXSW_SP_RIF_COUNTER_EXTRACT(error_bytes); 309 MLXSW_SP_RIF_COUNTER_EXTRACT(discard_bytes); 310 311 #undef MLXSW_SP_RIF_COUNTER_EXTRACT 312 313 return 0; 314 } 315 316 static int mlxsw_sp_rif_counter_clear(struct mlxsw_sp *mlxsw_sp, 317 unsigned int counter_index) 318 { 319 char ricnt_pl[MLXSW_REG_RICNT_LEN]; 320 321 mlxsw_reg_ricnt_pack(ricnt_pl, counter_index, 322 MLXSW_REG_RICNT_OPCODE_CLEAR); 323 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ricnt), ricnt_pl); 324 } 325 326 int mlxsw_sp_rif_counter_alloc(struct mlxsw_sp_rif *rif, 327 enum mlxsw_sp_rif_counter_dir dir) 328 { 329 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 330 unsigned int *p_counter_index; 331 int err; 332 333 if (mlxsw_sp_rif_counter_valid_get(rif, dir)) 334 return 0; 335 336 p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir); 337 if (!p_counter_index) 338 return -EINVAL; 339 340 err = mlxsw_sp_counter_alloc(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF, 341 p_counter_index); 342 if (err) 343 return err; 344 345 err = mlxsw_sp_rif_counter_clear(mlxsw_sp, *p_counter_index); 346 if (err) 347 goto err_counter_clear; 348 349 err = mlxsw_sp_rif_counter_edit(mlxsw_sp, rif->rif_index, 350 *p_counter_index, true, dir); 351 if (err) 352 goto err_counter_edit; 353 mlxsw_sp_rif_counter_valid_set(rif, dir, true); 354 return 0; 355 356 err_counter_edit: 357 err_counter_clear: 358 mlxsw_sp_counter_free(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF, 359 *p_counter_index); 360 return err; 361 } 362 363 void mlxsw_sp_rif_counter_free(struct mlxsw_sp_rif *rif, 364 enum mlxsw_sp_rif_counter_dir dir) 365 { 366 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 367 unsigned int *p_counter_index; 368 369 if (!mlxsw_sp_rif_counter_valid_get(rif, dir)) 370 return; 371 372 p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir); 373 if (WARN_ON(!p_counter_index)) 374 return; 375 mlxsw_sp_rif_counter_edit(mlxsw_sp, rif->rif_index, 376 *p_counter_index, false, dir); 377 mlxsw_sp_counter_free(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF, 378 *p_counter_index); 379 mlxsw_sp_rif_counter_valid_set(rif, dir, false); 380 } 381 382 static void mlxsw_sp_rif_counters_alloc(struct mlxsw_sp_rif *rif) 383 { 384 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 385 struct devlink *devlink; 386 387 devlink = priv_to_devlink(mlxsw_sp->core); 388 if (!devlink_dpipe_table_counter_enabled(devlink, 389 MLXSW_SP_DPIPE_TABLE_NAME_ERIF)) 390 return; 391 mlxsw_sp_rif_counter_alloc(rif, MLXSW_SP_RIF_COUNTER_EGRESS); 392 } 393 394 static void mlxsw_sp_rif_counters_free(struct mlxsw_sp_rif *rif) 395 { 396 mlxsw_sp_rif_counter_free(rif, MLXSW_SP_RIF_COUNTER_EGRESS); 397 } 398 399 #define MLXSW_SP_PREFIX_COUNT (sizeof(struct in6_addr) * BITS_PER_BYTE + 1) 400 401 struct mlxsw_sp_prefix_usage { 402 DECLARE_BITMAP(b, MLXSW_SP_PREFIX_COUNT); 403 }; 404 405 #define mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) \ 406 for_each_set_bit(prefix, (prefix_usage)->b, MLXSW_SP_PREFIX_COUNT) 407 408 static bool 409 mlxsw_sp_prefix_usage_eq(struct mlxsw_sp_prefix_usage *prefix_usage1, 410 struct mlxsw_sp_prefix_usage *prefix_usage2) 411 { 412 return !memcmp(prefix_usage1, prefix_usage2, sizeof(*prefix_usage1)); 413 } 414 415 static void 416 mlxsw_sp_prefix_usage_cpy(struct mlxsw_sp_prefix_usage *prefix_usage1, 417 struct mlxsw_sp_prefix_usage *prefix_usage2) 418 { 419 memcpy(prefix_usage1, prefix_usage2, sizeof(*prefix_usage1)); 420 } 421 422 static void 423 mlxsw_sp_prefix_usage_set(struct mlxsw_sp_prefix_usage *prefix_usage, 424 unsigned char prefix_len) 425 { 426 set_bit(prefix_len, prefix_usage->b); 427 } 428 429 static void 430 mlxsw_sp_prefix_usage_clear(struct mlxsw_sp_prefix_usage *prefix_usage, 431 unsigned char prefix_len) 432 { 433 clear_bit(prefix_len, prefix_usage->b); 434 } 435 436 struct mlxsw_sp_fib_key { 437 unsigned char addr[sizeof(struct in6_addr)]; 438 unsigned char prefix_len; 439 }; 440 441 enum mlxsw_sp_fib_entry_type { 442 MLXSW_SP_FIB_ENTRY_TYPE_REMOTE, 443 MLXSW_SP_FIB_ENTRY_TYPE_LOCAL, 444 MLXSW_SP_FIB_ENTRY_TYPE_TRAP, 445 MLXSW_SP_FIB_ENTRY_TYPE_BLACKHOLE, 446 MLXSW_SP_FIB_ENTRY_TYPE_UNREACHABLE, 447 448 /* This is a special case of local delivery, where a packet should be 449 * decapsulated on reception. Note that there is no corresponding ENCAP, 450 * because that's a type of next hop, not of FIB entry. (There can be 451 * several next hops in a REMOTE entry, and some of them may be 452 * encapsulating entries.) 453 */ 454 MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP, 455 MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP, 456 }; 457 458 struct mlxsw_sp_nexthop_group_info; 459 struct mlxsw_sp_nexthop_group; 460 struct mlxsw_sp_fib_entry; 461 462 struct mlxsw_sp_fib_node { 463 struct mlxsw_sp_fib_entry *fib_entry; 464 struct list_head list; 465 struct rhash_head ht_node; 466 struct mlxsw_sp_fib *fib; 467 struct mlxsw_sp_fib_key key; 468 }; 469 470 struct mlxsw_sp_fib_entry_decap { 471 struct mlxsw_sp_ipip_entry *ipip_entry; 472 u32 tunnel_index; 473 }; 474 475 struct mlxsw_sp_fib_entry { 476 struct mlxsw_sp_fib_node *fib_node; 477 enum mlxsw_sp_fib_entry_type type; 478 struct list_head nexthop_group_node; 479 struct mlxsw_sp_nexthop_group *nh_group; 480 struct mlxsw_sp_fib_entry_decap decap; /* Valid for decap entries. */ 481 }; 482 483 struct mlxsw_sp_fib4_entry { 484 struct mlxsw_sp_fib_entry common; 485 struct fib_info *fi; 486 u32 tb_id; 487 dscp_t dscp; 488 u8 type; 489 }; 490 491 struct mlxsw_sp_fib6_entry { 492 struct mlxsw_sp_fib_entry common; 493 struct list_head rt6_list; 494 unsigned int nrt6; 495 }; 496 497 struct mlxsw_sp_rt6 { 498 struct list_head list; 499 struct fib6_info *rt; 500 }; 501 502 struct mlxsw_sp_lpm_tree { 503 u8 id; /* tree ID */ 504 unsigned int ref_count; 505 enum mlxsw_sp_l3proto proto; 506 unsigned long prefix_ref_count[MLXSW_SP_PREFIX_COUNT]; 507 struct mlxsw_sp_prefix_usage prefix_usage; 508 }; 509 510 struct mlxsw_sp_fib { 511 struct rhashtable ht; 512 struct list_head node_list; 513 struct mlxsw_sp_vr *vr; 514 struct mlxsw_sp_lpm_tree *lpm_tree; 515 enum mlxsw_sp_l3proto proto; 516 }; 517 518 struct mlxsw_sp_vr { 519 u16 id; /* virtual router ID */ 520 u32 tb_id; /* kernel fib table id */ 521 unsigned int rif_count; 522 struct mlxsw_sp_fib *fib4; 523 struct mlxsw_sp_fib *fib6; 524 struct mlxsw_sp_mr_table *mr_table[MLXSW_SP_L3_PROTO_MAX]; 525 struct mlxsw_sp_rif *ul_rif; 526 refcount_t ul_rif_refcnt; 527 }; 528 529 static const struct rhashtable_params mlxsw_sp_fib_ht_params; 530 531 static struct mlxsw_sp_fib *mlxsw_sp_fib_create(struct mlxsw_sp *mlxsw_sp, 532 struct mlxsw_sp_vr *vr, 533 enum mlxsw_sp_l3proto proto) 534 { 535 struct mlxsw_sp_lpm_tree *lpm_tree; 536 struct mlxsw_sp_fib *fib; 537 int err; 538 539 lpm_tree = mlxsw_sp->router->lpm.proto_trees[proto]; 540 fib = kzalloc(sizeof(*fib), GFP_KERNEL); 541 if (!fib) 542 return ERR_PTR(-ENOMEM); 543 err = rhashtable_init(&fib->ht, &mlxsw_sp_fib_ht_params); 544 if (err) 545 goto err_rhashtable_init; 546 INIT_LIST_HEAD(&fib->node_list); 547 fib->proto = proto; 548 fib->vr = vr; 549 fib->lpm_tree = lpm_tree; 550 mlxsw_sp_lpm_tree_hold(lpm_tree); 551 err = mlxsw_sp_vr_lpm_tree_bind(mlxsw_sp, fib, lpm_tree->id); 552 if (err) 553 goto err_lpm_tree_bind; 554 return fib; 555 556 err_lpm_tree_bind: 557 mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree); 558 err_rhashtable_init: 559 kfree(fib); 560 return ERR_PTR(err); 561 } 562 563 static void mlxsw_sp_fib_destroy(struct mlxsw_sp *mlxsw_sp, 564 struct mlxsw_sp_fib *fib) 565 { 566 mlxsw_sp_vr_lpm_tree_unbind(mlxsw_sp, fib); 567 mlxsw_sp_lpm_tree_put(mlxsw_sp, fib->lpm_tree); 568 WARN_ON(!list_empty(&fib->node_list)); 569 rhashtable_destroy(&fib->ht); 570 kfree(fib); 571 } 572 573 static struct mlxsw_sp_lpm_tree * 574 mlxsw_sp_lpm_tree_find_unused(struct mlxsw_sp *mlxsw_sp) 575 { 576 static struct mlxsw_sp_lpm_tree *lpm_tree; 577 int i; 578 579 for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) { 580 lpm_tree = &mlxsw_sp->router->lpm.trees[i]; 581 if (lpm_tree->ref_count == 0) 582 return lpm_tree; 583 } 584 return NULL; 585 } 586 587 static int mlxsw_sp_lpm_tree_alloc(struct mlxsw_sp *mlxsw_sp, 588 struct mlxsw_sp_lpm_tree *lpm_tree) 589 { 590 char ralta_pl[MLXSW_REG_RALTA_LEN]; 591 592 mlxsw_reg_ralta_pack(ralta_pl, true, 593 (enum mlxsw_reg_ralxx_protocol) lpm_tree->proto, 594 lpm_tree->id); 595 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), ralta_pl); 596 } 597 598 static void mlxsw_sp_lpm_tree_free(struct mlxsw_sp *mlxsw_sp, 599 struct mlxsw_sp_lpm_tree *lpm_tree) 600 { 601 char ralta_pl[MLXSW_REG_RALTA_LEN]; 602 603 mlxsw_reg_ralta_pack(ralta_pl, false, 604 (enum mlxsw_reg_ralxx_protocol) lpm_tree->proto, 605 lpm_tree->id); 606 mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), ralta_pl); 607 } 608 609 static int 610 mlxsw_sp_lpm_tree_left_struct_set(struct mlxsw_sp *mlxsw_sp, 611 struct mlxsw_sp_prefix_usage *prefix_usage, 612 struct mlxsw_sp_lpm_tree *lpm_tree) 613 { 614 char ralst_pl[MLXSW_REG_RALST_LEN]; 615 u8 root_bin = 0; 616 u8 prefix; 617 u8 last_prefix = MLXSW_REG_RALST_BIN_NO_CHILD; 618 619 mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) 620 root_bin = prefix; 621 622 mlxsw_reg_ralst_pack(ralst_pl, root_bin, lpm_tree->id); 623 mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) { 624 if (prefix == 0) 625 continue; 626 mlxsw_reg_ralst_bin_pack(ralst_pl, prefix, last_prefix, 627 MLXSW_REG_RALST_BIN_NO_CHILD); 628 last_prefix = prefix; 629 } 630 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralst), ralst_pl); 631 } 632 633 static struct mlxsw_sp_lpm_tree * 634 mlxsw_sp_lpm_tree_create(struct mlxsw_sp *mlxsw_sp, 635 struct mlxsw_sp_prefix_usage *prefix_usage, 636 enum mlxsw_sp_l3proto proto) 637 { 638 struct mlxsw_sp_lpm_tree *lpm_tree; 639 int err; 640 641 lpm_tree = mlxsw_sp_lpm_tree_find_unused(mlxsw_sp); 642 if (!lpm_tree) 643 return ERR_PTR(-EBUSY); 644 lpm_tree->proto = proto; 645 err = mlxsw_sp_lpm_tree_alloc(mlxsw_sp, lpm_tree); 646 if (err) 647 return ERR_PTR(err); 648 649 err = mlxsw_sp_lpm_tree_left_struct_set(mlxsw_sp, prefix_usage, 650 lpm_tree); 651 if (err) 652 goto err_left_struct_set; 653 memcpy(&lpm_tree->prefix_usage, prefix_usage, 654 sizeof(lpm_tree->prefix_usage)); 655 memset(&lpm_tree->prefix_ref_count, 0, 656 sizeof(lpm_tree->prefix_ref_count)); 657 lpm_tree->ref_count = 1; 658 return lpm_tree; 659 660 err_left_struct_set: 661 mlxsw_sp_lpm_tree_free(mlxsw_sp, lpm_tree); 662 return ERR_PTR(err); 663 } 664 665 static void mlxsw_sp_lpm_tree_destroy(struct mlxsw_sp *mlxsw_sp, 666 struct mlxsw_sp_lpm_tree *lpm_tree) 667 { 668 mlxsw_sp_lpm_tree_free(mlxsw_sp, lpm_tree); 669 } 670 671 static struct mlxsw_sp_lpm_tree * 672 mlxsw_sp_lpm_tree_get(struct mlxsw_sp *mlxsw_sp, 673 struct mlxsw_sp_prefix_usage *prefix_usage, 674 enum mlxsw_sp_l3proto proto) 675 { 676 struct mlxsw_sp_lpm_tree *lpm_tree; 677 int i; 678 679 for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) { 680 lpm_tree = &mlxsw_sp->router->lpm.trees[i]; 681 if (lpm_tree->ref_count != 0 && 682 lpm_tree->proto == proto && 683 mlxsw_sp_prefix_usage_eq(&lpm_tree->prefix_usage, 684 prefix_usage)) { 685 mlxsw_sp_lpm_tree_hold(lpm_tree); 686 return lpm_tree; 687 } 688 } 689 return mlxsw_sp_lpm_tree_create(mlxsw_sp, prefix_usage, proto); 690 } 691 692 static void mlxsw_sp_lpm_tree_hold(struct mlxsw_sp_lpm_tree *lpm_tree) 693 { 694 lpm_tree->ref_count++; 695 } 696 697 static void mlxsw_sp_lpm_tree_put(struct mlxsw_sp *mlxsw_sp, 698 struct mlxsw_sp_lpm_tree *lpm_tree) 699 { 700 if (--lpm_tree->ref_count == 0) 701 mlxsw_sp_lpm_tree_destroy(mlxsw_sp, lpm_tree); 702 } 703 704 #define MLXSW_SP_LPM_TREE_MIN 1 /* tree 0 is reserved */ 705 706 static int mlxsw_sp_lpm_init(struct mlxsw_sp *mlxsw_sp) 707 { 708 struct mlxsw_sp_prefix_usage req_prefix_usage = {{ 0 } }; 709 struct mlxsw_sp_lpm_tree *lpm_tree; 710 u64 max_trees; 711 int err, i; 712 713 if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_LPM_TREES)) 714 return -EIO; 715 716 max_trees = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_LPM_TREES); 717 mlxsw_sp->router->lpm.tree_count = max_trees - MLXSW_SP_LPM_TREE_MIN; 718 mlxsw_sp->router->lpm.trees = kcalloc(mlxsw_sp->router->lpm.tree_count, 719 sizeof(struct mlxsw_sp_lpm_tree), 720 GFP_KERNEL); 721 if (!mlxsw_sp->router->lpm.trees) 722 return -ENOMEM; 723 724 for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) { 725 lpm_tree = &mlxsw_sp->router->lpm.trees[i]; 726 lpm_tree->id = i + MLXSW_SP_LPM_TREE_MIN; 727 } 728 729 lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage, 730 MLXSW_SP_L3_PROTO_IPV4); 731 if (IS_ERR(lpm_tree)) { 732 err = PTR_ERR(lpm_tree); 733 goto err_ipv4_tree_get; 734 } 735 mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV4] = lpm_tree; 736 737 lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage, 738 MLXSW_SP_L3_PROTO_IPV6); 739 if (IS_ERR(lpm_tree)) { 740 err = PTR_ERR(lpm_tree); 741 goto err_ipv6_tree_get; 742 } 743 mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV6] = lpm_tree; 744 745 return 0; 746 747 err_ipv6_tree_get: 748 lpm_tree = mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV4]; 749 mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree); 750 err_ipv4_tree_get: 751 kfree(mlxsw_sp->router->lpm.trees); 752 return err; 753 } 754 755 static void mlxsw_sp_lpm_fini(struct mlxsw_sp *mlxsw_sp) 756 { 757 struct mlxsw_sp_lpm_tree *lpm_tree; 758 759 lpm_tree = mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV6]; 760 mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree); 761 762 lpm_tree = mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV4]; 763 mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree); 764 765 kfree(mlxsw_sp->router->lpm.trees); 766 } 767 768 static bool mlxsw_sp_vr_is_used(const struct mlxsw_sp_vr *vr) 769 { 770 return !!vr->fib4 || !!vr->fib6 || 771 !!vr->mr_table[MLXSW_SP_L3_PROTO_IPV4] || 772 !!vr->mr_table[MLXSW_SP_L3_PROTO_IPV6]; 773 } 774 775 static struct mlxsw_sp_vr *mlxsw_sp_vr_find_unused(struct mlxsw_sp *mlxsw_sp) 776 { 777 int max_vrs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); 778 struct mlxsw_sp_vr *vr; 779 int i; 780 781 for (i = 0; i < max_vrs; i++) { 782 vr = &mlxsw_sp->router->vrs[i]; 783 if (!mlxsw_sp_vr_is_used(vr)) 784 return vr; 785 } 786 return NULL; 787 } 788 789 static int mlxsw_sp_vr_lpm_tree_bind(struct mlxsw_sp *mlxsw_sp, 790 const struct mlxsw_sp_fib *fib, u8 tree_id) 791 { 792 char raltb_pl[MLXSW_REG_RALTB_LEN]; 793 794 mlxsw_reg_raltb_pack(raltb_pl, fib->vr->id, 795 (enum mlxsw_reg_ralxx_protocol) fib->proto, 796 tree_id); 797 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb), raltb_pl); 798 } 799 800 static int mlxsw_sp_vr_lpm_tree_unbind(struct mlxsw_sp *mlxsw_sp, 801 const struct mlxsw_sp_fib *fib) 802 { 803 char raltb_pl[MLXSW_REG_RALTB_LEN]; 804 805 /* Bind to tree 0 which is default */ 806 mlxsw_reg_raltb_pack(raltb_pl, fib->vr->id, 807 (enum mlxsw_reg_ralxx_protocol) fib->proto, 0); 808 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb), raltb_pl); 809 } 810 811 static u32 mlxsw_sp_fix_tb_id(u32 tb_id) 812 { 813 /* For our purpose, squash main, default and local tables into one */ 814 if (tb_id == RT_TABLE_LOCAL || tb_id == RT_TABLE_DEFAULT) 815 tb_id = RT_TABLE_MAIN; 816 return tb_id; 817 } 818 819 static struct mlxsw_sp_vr *mlxsw_sp_vr_find(struct mlxsw_sp *mlxsw_sp, 820 u32 tb_id) 821 { 822 int max_vrs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); 823 struct mlxsw_sp_vr *vr; 824 int i; 825 826 tb_id = mlxsw_sp_fix_tb_id(tb_id); 827 828 for (i = 0; i < max_vrs; i++) { 829 vr = &mlxsw_sp->router->vrs[i]; 830 if (mlxsw_sp_vr_is_used(vr) && vr->tb_id == tb_id) 831 return vr; 832 } 833 return NULL; 834 } 835 836 int mlxsw_sp_router_tb_id_vr_id(struct mlxsw_sp *mlxsw_sp, u32 tb_id, 837 u16 *vr_id) 838 { 839 struct mlxsw_sp_vr *vr; 840 int err = 0; 841 842 mutex_lock(&mlxsw_sp->router->lock); 843 vr = mlxsw_sp_vr_find(mlxsw_sp, tb_id); 844 if (!vr) { 845 err = -ESRCH; 846 goto out; 847 } 848 *vr_id = vr->id; 849 out: 850 mutex_unlock(&mlxsw_sp->router->lock); 851 return err; 852 } 853 854 static struct mlxsw_sp_fib *mlxsw_sp_vr_fib(const struct mlxsw_sp_vr *vr, 855 enum mlxsw_sp_l3proto proto) 856 { 857 switch (proto) { 858 case MLXSW_SP_L3_PROTO_IPV4: 859 return vr->fib4; 860 case MLXSW_SP_L3_PROTO_IPV6: 861 return vr->fib6; 862 } 863 return NULL; 864 } 865 866 static struct mlxsw_sp_vr *mlxsw_sp_vr_create(struct mlxsw_sp *mlxsw_sp, 867 u32 tb_id, 868 struct netlink_ext_ack *extack) 869 { 870 struct mlxsw_sp_mr_table *mr4_table, *mr6_table; 871 struct mlxsw_sp_fib *fib4; 872 struct mlxsw_sp_fib *fib6; 873 struct mlxsw_sp_vr *vr; 874 int err; 875 876 vr = mlxsw_sp_vr_find_unused(mlxsw_sp); 877 if (!vr) { 878 NL_SET_ERR_MSG_MOD(extack, "Exceeded number of supported virtual routers"); 879 return ERR_PTR(-EBUSY); 880 } 881 fib4 = mlxsw_sp_fib_create(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV4); 882 if (IS_ERR(fib4)) 883 return ERR_CAST(fib4); 884 fib6 = mlxsw_sp_fib_create(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV6); 885 if (IS_ERR(fib6)) { 886 err = PTR_ERR(fib6); 887 goto err_fib6_create; 888 } 889 mr4_table = mlxsw_sp_mr_table_create(mlxsw_sp, vr->id, 890 MLXSW_SP_L3_PROTO_IPV4); 891 if (IS_ERR(mr4_table)) { 892 err = PTR_ERR(mr4_table); 893 goto err_mr4_table_create; 894 } 895 mr6_table = mlxsw_sp_mr_table_create(mlxsw_sp, vr->id, 896 MLXSW_SP_L3_PROTO_IPV6); 897 if (IS_ERR(mr6_table)) { 898 err = PTR_ERR(mr6_table); 899 goto err_mr6_table_create; 900 } 901 902 vr->fib4 = fib4; 903 vr->fib6 = fib6; 904 vr->mr_table[MLXSW_SP_L3_PROTO_IPV4] = mr4_table; 905 vr->mr_table[MLXSW_SP_L3_PROTO_IPV6] = mr6_table; 906 vr->tb_id = tb_id; 907 return vr; 908 909 err_mr6_table_create: 910 mlxsw_sp_mr_table_destroy(mr4_table); 911 err_mr4_table_create: 912 mlxsw_sp_fib_destroy(mlxsw_sp, fib6); 913 err_fib6_create: 914 mlxsw_sp_fib_destroy(mlxsw_sp, fib4); 915 return ERR_PTR(err); 916 } 917 918 static void mlxsw_sp_vr_destroy(struct mlxsw_sp *mlxsw_sp, 919 struct mlxsw_sp_vr *vr) 920 { 921 mlxsw_sp_mr_table_destroy(vr->mr_table[MLXSW_SP_L3_PROTO_IPV6]); 922 vr->mr_table[MLXSW_SP_L3_PROTO_IPV6] = NULL; 923 mlxsw_sp_mr_table_destroy(vr->mr_table[MLXSW_SP_L3_PROTO_IPV4]); 924 vr->mr_table[MLXSW_SP_L3_PROTO_IPV4] = NULL; 925 mlxsw_sp_fib_destroy(mlxsw_sp, vr->fib6); 926 vr->fib6 = NULL; 927 mlxsw_sp_fib_destroy(mlxsw_sp, vr->fib4); 928 vr->fib4 = NULL; 929 } 930 931 static struct mlxsw_sp_vr *mlxsw_sp_vr_get(struct mlxsw_sp *mlxsw_sp, u32 tb_id, 932 struct netlink_ext_ack *extack) 933 { 934 struct mlxsw_sp_vr *vr; 935 936 tb_id = mlxsw_sp_fix_tb_id(tb_id); 937 vr = mlxsw_sp_vr_find(mlxsw_sp, tb_id); 938 if (!vr) 939 vr = mlxsw_sp_vr_create(mlxsw_sp, tb_id, extack); 940 return vr; 941 } 942 943 static void mlxsw_sp_vr_put(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp_vr *vr) 944 { 945 if (!vr->rif_count && list_empty(&vr->fib4->node_list) && 946 list_empty(&vr->fib6->node_list) && 947 mlxsw_sp_mr_table_empty(vr->mr_table[MLXSW_SP_L3_PROTO_IPV4]) && 948 mlxsw_sp_mr_table_empty(vr->mr_table[MLXSW_SP_L3_PROTO_IPV6])) 949 mlxsw_sp_vr_destroy(mlxsw_sp, vr); 950 } 951 952 static bool 953 mlxsw_sp_vr_lpm_tree_should_replace(struct mlxsw_sp_vr *vr, 954 enum mlxsw_sp_l3proto proto, u8 tree_id) 955 { 956 struct mlxsw_sp_fib *fib = mlxsw_sp_vr_fib(vr, proto); 957 958 if (!mlxsw_sp_vr_is_used(vr)) 959 return false; 960 if (fib->lpm_tree->id == tree_id) 961 return true; 962 return false; 963 } 964 965 static int mlxsw_sp_vr_lpm_tree_replace(struct mlxsw_sp *mlxsw_sp, 966 struct mlxsw_sp_fib *fib, 967 struct mlxsw_sp_lpm_tree *new_tree) 968 { 969 struct mlxsw_sp_lpm_tree *old_tree = fib->lpm_tree; 970 int err; 971 972 fib->lpm_tree = new_tree; 973 mlxsw_sp_lpm_tree_hold(new_tree); 974 err = mlxsw_sp_vr_lpm_tree_bind(mlxsw_sp, fib, new_tree->id); 975 if (err) 976 goto err_tree_bind; 977 mlxsw_sp_lpm_tree_put(mlxsw_sp, old_tree); 978 return 0; 979 980 err_tree_bind: 981 mlxsw_sp_lpm_tree_put(mlxsw_sp, new_tree); 982 fib->lpm_tree = old_tree; 983 return err; 984 } 985 986 static int mlxsw_sp_vrs_lpm_tree_replace(struct mlxsw_sp *mlxsw_sp, 987 struct mlxsw_sp_fib *fib, 988 struct mlxsw_sp_lpm_tree *new_tree) 989 { 990 int max_vrs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); 991 enum mlxsw_sp_l3proto proto = fib->proto; 992 struct mlxsw_sp_lpm_tree *old_tree; 993 u8 old_id, new_id = new_tree->id; 994 struct mlxsw_sp_vr *vr; 995 int i, err; 996 997 old_tree = mlxsw_sp->router->lpm.proto_trees[proto]; 998 old_id = old_tree->id; 999 1000 for (i = 0; i < max_vrs; i++) { 1001 vr = &mlxsw_sp->router->vrs[i]; 1002 if (!mlxsw_sp_vr_lpm_tree_should_replace(vr, proto, old_id)) 1003 continue; 1004 err = mlxsw_sp_vr_lpm_tree_replace(mlxsw_sp, 1005 mlxsw_sp_vr_fib(vr, proto), 1006 new_tree); 1007 if (err) 1008 goto err_tree_replace; 1009 } 1010 1011 memcpy(new_tree->prefix_ref_count, old_tree->prefix_ref_count, 1012 sizeof(new_tree->prefix_ref_count)); 1013 mlxsw_sp->router->lpm.proto_trees[proto] = new_tree; 1014 mlxsw_sp_lpm_tree_put(mlxsw_sp, old_tree); 1015 1016 return 0; 1017 1018 err_tree_replace: 1019 for (i--; i >= 0; i--) { 1020 if (!mlxsw_sp_vr_lpm_tree_should_replace(vr, proto, new_id)) 1021 continue; 1022 mlxsw_sp_vr_lpm_tree_replace(mlxsw_sp, 1023 mlxsw_sp_vr_fib(vr, proto), 1024 old_tree); 1025 } 1026 return err; 1027 } 1028 1029 static int mlxsw_sp_vrs_init(struct mlxsw_sp *mlxsw_sp) 1030 { 1031 struct mlxsw_sp_vr *vr; 1032 u64 max_vrs; 1033 int i; 1034 1035 if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_VRS)) 1036 return -EIO; 1037 1038 max_vrs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); 1039 mlxsw_sp->router->vrs = kcalloc(max_vrs, sizeof(struct mlxsw_sp_vr), 1040 GFP_KERNEL); 1041 if (!mlxsw_sp->router->vrs) 1042 return -ENOMEM; 1043 1044 for (i = 0; i < max_vrs; i++) { 1045 vr = &mlxsw_sp->router->vrs[i]; 1046 vr->id = i; 1047 } 1048 1049 return 0; 1050 } 1051 1052 static void mlxsw_sp_router_fib_flush(struct mlxsw_sp *mlxsw_sp); 1053 1054 static void mlxsw_sp_vrs_fini(struct mlxsw_sp *mlxsw_sp) 1055 { 1056 /* At this stage we're guaranteed not to have new incoming 1057 * FIB notifications and the work queue is free from FIBs 1058 * sitting on top of mlxsw netdevs. However, we can still 1059 * have other FIBs queued. Flush the queue before flushing 1060 * the device's tables. No need for locks, as we're the only 1061 * writer. 1062 */ 1063 mlxsw_core_flush_owq(); 1064 mlxsw_sp_router_fib_flush(mlxsw_sp); 1065 kfree(mlxsw_sp->router->vrs); 1066 } 1067 1068 u32 mlxsw_sp_ipip_dev_ul_tb_id(const struct net_device *ol_dev) 1069 { 1070 struct net_device *d; 1071 u32 tb_id; 1072 1073 rcu_read_lock(); 1074 d = mlxsw_sp_ipip_netdev_ul_dev_get(ol_dev); 1075 if (d) 1076 tb_id = l3mdev_fib_table(d) ? : RT_TABLE_MAIN; 1077 else 1078 tb_id = RT_TABLE_MAIN; 1079 rcu_read_unlock(); 1080 1081 return tb_id; 1082 } 1083 1084 static void 1085 mlxsw_sp_crif_init(struct mlxsw_sp_crif *crif, struct net_device *dev) 1086 { 1087 crif->key.dev = dev; 1088 INIT_LIST_HEAD(&crif->nexthop_list); 1089 } 1090 1091 static struct mlxsw_sp_crif * 1092 mlxsw_sp_crif_alloc(struct net_device *dev) 1093 { 1094 struct mlxsw_sp_crif *crif; 1095 1096 crif = kzalloc(sizeof(*crif), GFP_KERNEL); 1097 if (!crif) 1098 return NULL; 1099 1100 mlxsw_sp_crif_init(crif, dev); 1101 return crif; 1102 } 1103 1104 static void mlxsw_sp_crif_free(struct mlxsw_sp_crif *crif) 1105 { 1106 if (WARN_ON(crif->rif)) 1107 return; 1108 1109 WARN_ON(!list_empty(&crif->nexthop_list)); 1110 kfree(crif); 1111 } 1112 1113 static int mlxsw_sp_crif_insert(struct mlxsw_sp_router *router, 1114 struct mlxsw_sp_crif *crif) 1115 { 1116 return rhashtable_insert_fast(&router->crif_ht, &crif->ht_node, 1117 mlxsw_sp_crif_ht_params); 1118 } 1119 1120 static void mlxsw_sp_crif_remove(struct mlxsw_sp_router *router, 1121 struct mlxsw_sp_crif *crif) 1122 { 1123 rhashtable_remove_fast(&router->crif_ht, &crif->ht_node, 1124 mlxsw_sp_crif_ht_params); 1125 } 1126 1127 static struct mlxsw_sp_crif * 1128 mlxsw_sp_crif_lookup(struct mlxsw_sp_router *router, 1129 const struct net_device *dev) 1130 { 1131 struct mlxsw_sp_crif_key key = { 1132 .dev = (struct net_device *)dev, 1133 }; 1134 1135 return rhashtable_lookup_fast(&router->crif_ht, &key, 1136 mlxsw_sp_crif_ht_params); 1137 } 1138 1139 static struct mlxsw_sp_rif * 1140 mlxsw_sp_rif_create(struct mlxsw_sp *mlxsw_sp, 1141 const struct mlxsw_sp_rif_params *params, 1142 struct netlink_ext_ack *extack); 1143 1144 static struct mlxsw_sp_rif_ipip_lb * 1145 mlxsw_sp_ipip_ol_ipip_lb_create(struct mlxsw_sp *mlxsw_sp, 1146 enum mlxsw_sp_ipip_type ipipt, 1147 struct net_device *ol_dev, 1148 struct netlink_ext_ack *extack) 1149 { 1150 struct mlxsw_sp_rif_params_ipip_lb lb_params; 1151 const struct mlxsw_sp_ipip_ops *ipip_ops; 1152 struct mlxsw_sp_rif *rif; 1153 1154 ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipipt]; 1155 lb_params = (struct mlxsw_sp_rif_params_ipip_lb) { 1156 .common.dev = ol_dev, 1157 .common.lag = false, 1158 .common.double_entry = ipip_ops->double_rif_entry, 1159 .lb_config = ipip_ops->ol_loopback_config(mlxsw_sp, ol_dev), 1160 }; 1161 1162 rif = mlxsw_sp_rif_create(mlxsw_sp, &lb_params.common, extack); 1163 if (IS_ERR(rif)) 1164 return ERR_CAST(rif); 1165 return container_of(rif, struct mlxsw_sp_rif_ipip_lb, common); 1166 } 1167 1168 static struct mlxsw_sp_ipip_entry * 1169 mlxsw_sp_ipip_entry_alloc(struct mlxsw_sp *mlxsw_sp, 1170 enum mlxsw_sp_ipip_type ipipt, 1171 struct net_device *ol_dev) 1172 { 1173 const struct mlxsw_sp_ipip_ops *ipip_ops; 1174 struct mlxsw_sp_ipip_entry *ipip_entry; 1175 struct mlxsw_sp_ipip_entry *ret = NULL; 1176 int err; 1177 1178 ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipipt]; 1179 ipip_entry = kzalloc(sizeof(*ipip_entry), GFP_KERNEL); 1180 if (!ipip_entry) 1181 return ERR_PTR(-ENOMEM); 1182 1183 ipip_entry->ol_lb = mlxsw_sp_ipip_ol_ipip_lb_create(mlxsw_sp, ipipt, 1184 ol_dev, NULL); 1185 if (IS_ERR(ipip_entry->ol_lb)) { 1186 ret = ERR_CAST(ipip_entry->ol_lb); 1187 goto err_ol_ipip_lb_create; 1188 } 1189 1190 ipip_entry->ipipt = ipipt; 1191 ipip_entry->ol_dev = ol_dev; 1192 ipip_entry->parms = ipip_ops->parms_init(ol_dev); 1193 1194 err = ipip_ops->rem_ip_addr_set(mlxsw_sp, ipip_entry); 1195 if (err) { 1196 ret = ERR_PTR(err); 1197 goto err_rem_ip_addr_set; 1198 } 1199 1200 return ipip_entry; 1201 1202 err_rem_ip_addr_set: 1203 mlxsw_sp_rif_destroy(&ipip_entry->ol_lb->common); 1204 err_ol_ipip_lb_create: 1205 kfree(ipip_entry); 1206 return ret; 1207 } 1208 1209 static void mlxsw_sp_ipip_entry_dealloc(struct mlxsw_sp *mlxsw_sp, 1210 struct mlxsw_sp_ipip_entry *ipip_entry) 1211 { 1212 const struct mlxsw_sp_ipip_ops *ipip_ops = 1213 mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt]; 1214 1215 ipip_ops->rem_ip_addr_unset(mlxsw_sp, ipip_entry); 1216 mlxsw_sp_rif_destroy(&ipip_entry->ol_lb->common); 1217 kfree(ipip_entry); 1218 } 1219 1220 static bool 1221 mlxsw_sp_ipip_entry_saddr_matches(struct mlxsw_sp *mlxsw_sp, 1222 const enum mlxsw_sp_l3proto ul_proto, 1223 union mlxsw_sp_l3addr saddr, 1224 u32 ul_tb_id, 1225 struct mlxsw_sp_ipip_entry *ipip_entry) 1226 { 1227 u32 tun_ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(ipip_entry->ol_dev); 1228 enum mlxsw_sp_ipip_type ipipt = ipip_entry->ipipt; 1229 union mlxsw_sp_l3addr tun_saddr; 1230 1231 if (mlxsw_sp->router->ipip_ops_arr[ipipt]->ul_proto != ul_proto) 1232 return false; 1233 1234 tun_saddr = mlxsw_sp_ipip_netdev_saddr(ul_proto, ipip_entry->ol_dev); 1235 return tun_ul_tb_id == ul_tb_id && 1236 mlxsw_sp_l3addr_eq(&tun_saddr, &saddr); 1237 } 1238 1239 static int mlxsw_sp_ipip_decap_parsing_depth_inc(struct mlxsw_sp *mlxsw_sp, 1240 enum mlxsw_sp_ipip_type ipipt) 1241 { 1242 const struct mlxsw_sp_ipip_ops *ipip_ops; 1243 1244 ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipipt]; 1245 1246 /* Not all tunnels require to increase the default pasing depth 1247 * (96 bytes). 1248 */ 1249 if (ipip_ops->inc_parsing_depth) 1250 return mlxsw_sp_parsing_depth_inc(mlxsw_sp); 1251 1252 return 0; 1253 } 1254 1255 static void mlxsw_sp_ipip_decap_parsing_depth_dec(struct mlxsw_sp *mlxsw_sp, 1256 enum mlxsw_sp_ipip_type ipipt) 1257 { 1258 const struct mlxsw_sp_ipip_ops *ipip_ops = 1259 mlxsw_sp->router->ipip_ops_arr[ipipt]; 1260 1261 if (ipip_ops->inc_parsing_depth) 1262 mlxsw_sp_parsing_depth_dec(mlxsw_sp); 1263 } 1264 1265 static int 1266 mlxsw_sp_fib_entry_decap_init(struct mlxsw_sp *mlxsw_sp, 1267 struct mlxsw_sp_fib_entry *fib_entry, 1268 struct mlxsw_sp_ipip_entry *ipip_entry) 1269 { 1270 u32 tunnel_index; 1271 int err; 1272 1273 err = mlxsw_sp_kvdl_alloc(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 1274 1, &tunnel_index); 1275 if (err) 1276 return err; 1277 1278 err = mlxsw_sp_ipip_decap_parsing_depth_inc(mlxsw_sp, 1279 ipip_entry->ipipt); 1280 if (err) 1281 goto err_parsing_depth_inc; 1282 1283 ipip_entry->decap_fib_entry = fib_entry; 1284 fib_entry->decap.ipip_entry = ipip_entry; 1285 fib_entry->decap.tunnel_index = tunnel_index; 1286 1287 return 0; 1288 1289 err_parsing_depth_inc: 1290 mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 1, 1291 fib_entry->decap.tunnel_index); 1292 return err; 1293 } 1294 1295 static void mlxsw_sp_fib_entry_decap_fini(struct mlxsw_sp *mlxsw_sp, 1296 struct mlxsw_sp_fib_entry *fib_entry) 1297 { 1298 enum mlxsw_sp_ipip_type ipipt = fib_entry->decap.ipip_entry->ipipt; 1299 1300 /* Unlink this node from the IPIP entry that it's the decap entry of. */ 1301 fib_entry->decap.ipip_entry->decap_fib_entry = NULL; 1302 fib_entry->decap.ipip_entry = NULL; 1303 mlxsw_sp_ipip_decap_parsing_depth_dec(mlxsw_sp, ipipt); 1304 mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 1305 1, fib_entry->decap.tunnel_index); 1306 } 1307 1308 static struct mlxsw_sp_fib_node * 1309 mlxsw_sp_fib_node_lookup(struct mlxsw_sp_fib *fib, const void *addr, 1310 size_t addr_len, unsigned char prefix_len); 1311 static int mlxsw_sp_fib_entry_update(struct mlxsw_sp *mlxsw_sp, 1312 struct mlxsw_sp_fib_entry *fib_entry); 1313 1314 static void 1315 mlxsw_sp_ipip_entry_demote_decap(struct mlxsw_sp *mlxsw_sp, 1316 struct mlxsw_sp_ipip_entry *ipip_entry) 1317 { 1318 struct mlxsw_sp_fib_entry *fib_entry = ipip_entry->decap_fib_entry; 1319 1320 mlxsw_sp_fib_entry_decap_fini(mlxsw_sp, fib_entry); 1321 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP; 1322 1323 mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry); 1324 } 1325 1326 static void 1327 mlxsw_sp_ipip_entry_promote_decap(struct mlxsw_sp *mlxsw_sp, 1328 struct mlxsw_sp_ipip_entry *ipip_entry, 1329 struct mlxsw_sp_fib_entry *decap_fib_entry) 1330 { 1331 if (mlxsw_sp_fib_entry_decap_init(mlxsw_sp, decap_fib_entry, 1332 ipip_entry)) 1333 return; 1334 decap_fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP; 1335 1336 if (mlxsw_sp_fib_entry_update(mlxsw_sp, decap_fib_entry)) 1337 mlxsw_sp_ipip_entry_demote_decap(mlxsw_sp, ipip_entry); 1338 } 1339 1340 static struct mlxsw_sp_fib_entry * 1341 mlxsw_sp_router_ip2me_fib_entry_find(struct mlxsw_sp *mlxsw_sp, u32 tb_id, 1342 enum mlxsw_sp_l3proto proto, 1343 const union mlxsw_sp_l3addr *addr, 1344 enum mlxsw_sp_fib_entry_type type) 1345 { 1346 struct mlxsw_sp_fib_node *fib_node; 1347 unsigned char addr_prefix_len; 1348 struct mlxsw_sp_fib *fib; 1349 struct mlxsw_sp_vr *vr; 1350 const void *addrp; 1351 size_t addr_len; 1352 u32 addr4; 1353 1354 vr = mlxsw_sp_vr_find(mlxsw_sp, tb_id); 1355 if (!vr) 1356 return NULL; 1357 fib = mlxsw_sp_vr_fib(vr, proto); 1358 1359 switch (proto) { 1360 case MLXSW_SP_L3_PROTO_IPV4: 1361 addr4 = be32_to_cpu(addr->addr4); 1362 addrp = &addr4; 1363 addr_len = 4; 1364 addr_prefix_len = 32; 1365 break; 1366 case MLXSW_SP_L3_PROTO_IPV6: 1367 addrp = &addr->addr6; 1368 addr_len = 16; 1369 addr_prefix_len = 128; 1370 break; 1371 default: 1372 WARN_ON(1); 1373 return NULL; 1374 } 1375 1376 fib_node = mlxsw_sp_fib_node_lookup(fib, addrp, addr_len, 1377 addr_prefix_len); 1378 if (!fib_node || fib_node->fib_entry->type != type) 1379 return NULL; 1380 1381 return fib_node->fib_entry; 1382 } 1383 1384 /* Given an IPIP entry, find the corresponding decap route. */ 1385 static struct mlxsw_sp_fib_entry * 1386 mlxsw_sp_ipip_entry_find_decap(struct mlxsw_sp *mlxsw_sp, 1387 struct mlxsw_sp_ipip_entry *ipip_entry) 1388 { 1389 static struct mlxsw_sp_fib_node *fib_node; 1390 const struct mlxsw_sp_ipip_ops *ipip_ops; 1391 unsigned char saddr_prefix_len; 1392 union mlxsw_sp_l3addr saddr; 1393 struct mlxsw_sp_fib *ul_fib; 1394 struct mlxsw_sp_vr *ul_vr; 1395 const void *saddrp; 1396 size_t saddr_len; 1397 u32 ul_tb_id; 1398 u32 saddr4; 1399 1400 ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt]; 1401 1402 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(ipip_entry->ol_dev); 1403 ul_vr = mlxsw_sp_vr_find(mlxsw_sp, ul_tb_id); 1404 if (!ul_vr) 1405 return NULL; 1406 1407 ul_fib = mlxsw_sp_vr_fib(ul_vr, ipip_ops->ul_proto); 1408 saddr = mlxsw_sp_ipip_netdev_saddr(ipip_ops->ul_proto, 1409 ipip_entry->ol_dev); 1410 1411 switch (ipip_ops->ul_proto) { 1412 case MLXSW_SP_L3_PROTO_IPV4: 1413 saddr4 = be32_to_cpu(saddr.addr4); 1414 saddrp = &saddr4; 1415 saddr_len = 4; 1416 saddr_prefix_len = 32; 1417 break; 1418 case MLXSW_SP_L3_PROTO_IPV6: 1419 saddrp = &saddr.addr6; 1420 saddr_len = 16; 1421 saddr_prefix_len = 128; 1422 break; 1423 default: 1424 WARN_ON(1); 1425 return NULL; 1426 } 1427 1428 fib_node = mlxsw_sp_fib_node_lookup(ul_fib, saddrp, saddr_len, 1429 saddr_prefix_len); 1430 if (!fib_node || 1431 fib_node->fib_entry->type != MLXSW_SP_FIB_ENTRY_TYPE_TRAP) 1432 return NULL; 1433 1434 return fib_node->fib_entry; 1435 } 1436 1437 static struct mlxsw_sp_ipip_entry * 1438 mlxsw_sp_ipip_entry_create(struct mlxsw_sp *mlxsw_sp, 1439 enum mlxsw_sp_ipip_type ipipt, 1440 struct net_device *ol_dev) 1441 { 1442 struct mlxsw_sp_ipip_entry *ipip_entry; 1443 1444 ipip_entry = mlxsw_sp_ipip_entry_alloc(mlxsw_sp, ipipt, ol_dev); 1445 if (IS_ERR(ipip_entry)) 1446 return ipip_entry; 1447 1448 list_add_tail(&ipip_entry->ipip_list_node, 1449 &mlxsw_sp->router->ipip_list); 1450 1451 return ipip_entry; 1452 } 1453 1454 static void 1455 mlxsw_sp_ipip_entry_destroy(struct mlxsw_sp *mlxsw_sp, 1456 struct mlxsw_sp_ipip_entry *ipip_entry) 1457 { 1458 list_del(&ipip_entry->ipip_list_node); 1459 mlxsw_sp_ipip_entry_dealloc(mlxsw_sp, ipip_entry); 1460 } 1461 1462 static bool 1463 mlxsw_sp_ipip_entry_matches_decap(struct mlxsw_sp *mlxsw_sp, 1464 const struct net_device *ul_dev, 1465 enum mlxsw_sp_l3proto ul_proto, 1466 union mlxsw_sp_l3addr ul_dip, 1467 struct mlxsw_sp_ipip_entry *ipip_entry) 1468 { 1469 u32 ul_tb_id = l3mdev_fib_table(ul_dev) ? : RT_TABLE_MAIN; 1470 enum mlxsw_sp_ipip_type ipipt = ipip_entry->ipipt; 1471 1472 if (mlxsw_sp->router->ipip_ops_arr[ipipt]->ul_proto != ul_proto) 1473 return false; 1474 1475 return mlxsw_sp_ipip_entry_saddr_matches(mlxsw_sp, ul_proto, ul_dip, 1476 ul_tb_id, ipip_entry); 1477 } 1478 1479 /* Given decap parameters, find the corresponding IPIP entry. */ 1480 static struct mlxsw_sp_ipip_entry * 1481 mlxsw_sp_ipip_entry_find_by_decap(struct mlxsw_sp *mlxsw_sp, int ul_dev_ifindex, 1482 enum mlxsw_sp_l3proto ul_proto, 1483 union mlxsw_sp_l3addr ul_dip) 1484 { 1485 struct mlxsw_sp_ipip_entry *ipip_entry = NULL; 1486 struct net_device *ul_dev; 1487 1488 rcu_read_lock(); 1489 1490 ul_dev = dev_get_by_index_rcu(mlxsw_sp_net(mlxsw_sp), ul_dev_ifindex); 1491 if (!ul_dev) 1492 goto out_unlock; 1493 1494 list_for_each_entry(ipip_entry, &mlxsw_sp->router->ipip_list, 1495 ipip_list_node) 1496 if (mlxsw_sp_ipip_entry_matches_decap(mlxsw_sp, ul_dev, 1497 ul_proto, ul_dip, 1498 ipip_entry)) 1499 goto out_unlock; 1500 1501 rcu_read_unlock(); 1502 1503 return NULL; 1504 1505 out_unlock: 1506 rcu_read_unlock(); 1507 return ipip_entry; 1508 } 1509 1510 static bool mlxsw_sp_netdev_ipip_type(const struct mlxsw_sp *mlxsw_sp, 1511 const struct net_device *dev, 1512 enum mlxsw_sp_ipip_type *p_type) 1513 { 1514 struct mlxsw_sp_router *router = mlxsw_sp->router; 1515 const struct mlxsw_sp_ipip_ops *ipip_ops; 1516 enum mlxsw_sp_ipip_type ipipt; 1517 1518 for (ipipt = 0; ipipt < MLXSW_SP_IPIP_TYPE_MAX; ++ipipt) { 1519 ipip_ops = router->ipip_ops_arr[ipipt]; 1520 if (dev->type == ipip_ops->dev_type) { 1521 if (p_type) 1522 *p_type = ipipt; 1523 return true; 1524 } 1525 } 1526 return false; 1527 } 1528 1529 static bool mlxsw_sp_netdev_is_ipip_ol(const struct mlxsw_sp *mlxsw_sp, 1530 const struct net_device *dev) 1531 { 1532 return mlxsw_sp_netdev_ipip_type(mlxsw_sp, dev, NULL); 1533 } 1534 1535 static struct mlxsw_sp_ipip_entry * 1536 mlxsw_sp_ipip_entry_find_by_ol_dev(struct mlxsw_sp *mlxsw_sp, 1537 const struct net_device *ol_dev) 1538 { 1539 struct mlxsw_sp_ipip_entry *ipip_entry; 1540 1541 list_for_each_entry(ipip_entry, &mlxsw_sp->router->ipip_list, 1542 ipip_list_node) 1543 if (ipip_entry->ol_dev == ol_dev) 1544 return ipip_entry; 1545 1546 return NULL; 1547 } 1548 1549 static struct mlxsw_sp_ipip_entry * 1550 mlxsw_sp_ipip_entry_find_by_ul_dev(const struct mlxsw_sp *mlxsw_sp, 1551 const struct net_device *ul_dev, 1552 struct mlxsw_sp_ipip_entry *start) 1553 { 1554 struct mlxsw_sp_ipip_entry *ipip_entry; 1555 1556 ipip_entry = list_prepare_entry(start, &mlxsw_sp->router->ipip_list, 1557 ipip_list_node); 1558 list_for_each_entry_continue(ipip_entry, &mlxsw_sp->router->ipip_list, 1559 ipip_list_node) { 1560 struct net_device *ol_dev = ipip_entry->ol_dev; 1561 struct net_device *ipip_ul_dev; 1562 1563 rcu_read_lock(); 1564 ipip_ul_dev = mlxsw_sp_ipip_netdev_ul_dev_get(ol_dev); 1565 rcu_read_unlock(); 1566 1567 if (ipip_ul_dev == ul_dev) 1568 return ipip_entry; 1569 } 1570 1571 return NULL; 1572 } 1573 1574 static bool mlxsw_sp_netdev_is_ipip_ul(struct mlxsw_sp *mlxsw_sp, 1575 const struct net_device *dev) 1576 { 1577 return mlxsw_sp_ipip_entry_find_by_ul_dev(mlxsw_sp, dev, NULL); 1578 } 1579 1580 static bool mlxsw_sp_netdevice_ipip_can_offload(struct mlxsw_sp *mlxsw_sp, 1581 const struct net_device *ol_dev, 1582 enum mlxsw_sp_ipip_type ipipt) 1583 { 1584 const struct mlxsw_sp_ipip_ops *ops 1585 = mlxsw_sp->router->ipip_ops_arr[ipipt]; 1586 1587 return ops->can_offload(mlxsw_sp, ol_dev); 1588 } 1589 1590 static int mlxsw_sp_netdevice_ipip_ol_reg_event(struct mlxsw_sp *mlxsw_sp, 1591 struct net_device *ol_dev) 1592 { 1593 enum mlxsw_sp_ipip_type ipipt = MLXSW_SP_IPIP_TYPE_MAX; 1594 struct mlxsw_sp_ipip_entry *ipip_entry; 1595 enum mlxsw_sp_l3proto ul_proto; 1596 union mlxsw_sp_l3addr saddr; 1597 u32 ul_tb_id; 1598 1599 mlxsw_sp_netdev_ipip_type(mlxsw_sp, ol_dev, &ipipt); 1600 if (mlxsw_sp_netdevice_ipip_can_offload(mlxsw_sp, ol_dev, ipipt)) { 1601 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(ol_dev); 1602 ul_proto = mlxsw_sp->router->ipip_ops_arr[ipipt]->ul_proto; 1603 saddr = mlxsw_sp_ipip_netdev_saddr(ul_proto, ol_dev); 1604 if (!mlxsw_sp_ipip_demote_tunnel_by_saddr(mlxsw_sp, ul_proto, 1605 saddr, ul_tb_id, 1606 NULL)) { 1607 ipip_entry = mlxsw_sp_ipip_entry_create(mlxsw_sp, ipipt, 1608 ol_dev); 1609 if (IS_ERR(ipip_entry)) 1610 return PTR_ERR(ipip_entry); 1611 } 1612 } 1613 1614 return 0; 1615 } 1616 1617 static void mlxsw_sp_netdevice_ipip_ol_unreg_event(struct mlxsw_sp *mlxsw_sp, 1618 struct net_device *ol_dev) 1619 { 1620 struct mlxsw_sp_ipip_entry *ipip_entry; 1621 1622 ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev); 1623 if (ipip_entry) 1624 mlxsw_sp_ipip_entry_destroy(mlxsw_sp, ipip_entry); 1625 } 1626 1627 static void 1628 mlxsw_sp_ipip_entry_ol_up_event(struct mlxsw_sp *mlxsw_sp, 1629 struct mlxsw_sp_ipip_entry *ipip_entry) 1630 { 1631 struct mlxsw_sp_fib_entry *decap_fib_entry; 1632 1633 decap_fib_entry = mlxsw_sp_ipip_entry_find_decap(mlxsw_sp, ipip_entry); 1634 if (decap_fib_entry) 1635 mlxsw_sp_ipip_entry_promote_decap(mlxsw_sp, ipip_entry, 1636 decap_fib_entry); 1637 } 1638 1639 static int 1640 mlxsw_sp_rif_ipip_lb_op(struct mlxsw_sp_rif_ipip_lb *lb_rif, u16 ul_vr_id, 1641 u16 ul_rif_id, bool enable) 1642 { 1643 struct mlxsw_sp_rif_ipip_lb_config lb_cf = lb_rif->lb_config; 1644 struct net_device *dev = mlxsw_sp_rif_dev(&lb_rif->common); 1645 enum mlxsw_reg_ritr_loopback_ipip_options ipip_options; 1646 struct mlxsw_sp_rif *rif = &lb_rif->common; 1647 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 1648 char ritr_pl[MLXSW_REG_RITR_LEN]; 1649 struct in6_addr *saddr6; 1650 u32 saddr4; 1651 1652 ipip_options = MLXSW_REG_RITR_LOOPBACK_IPIP_OPTIONS_GRE_KEY_PRESET; 1653 switch (lb_cf.ul_protocol) { 1654 case MLXSW_SP_L3_PROTO_IPV4: 1655 saddr4 = be32_to_cpu(lb_cf.saddr.addr4); 1656 mlxsw_reg_ritr_pack(ritr_pl, enable, MLXSW_REG_RITR_LOOPBACK_IF, 1657 rif->rif_index, rif->vr_id, dev->mtu); 1658 mlxsw_reg_ritr_loopback_ipip4_pack(ritr_pl, lb_cf.lb_ipipt, 1659 ipip_options, ul_vr_id, 1660 ul_rif_id, saddr4, 1661 lb_cf.okey); 1662 break; 1663 1664 case MLXSW_SP_L3_PROTO_IPV6: 1665 saddr6 = &lb_cf.saddr.addr6; 1666 mlxsw_reg_ritr_pack(ritr_pl, enable, MLXSW_REG_RITR_LOOPBACK_IF, 1667 rif->rif_index, rif->vr_id, dev->mtu); 1668 mlxsw_reg_ritr_loopback_ipip6_pack(ritr_pl, lb_cf.lb_ipipt, 1669 ipip_options, ul_vr_id, 1670 ul_rif_id, saddr6, 1671 lb_cf.okey); 1672 break; 1673 } 1674 1675 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 1676 } 1677 1678 static int mlxsw_sp_netdevice_ipip_ol_update_mtu(struct mlxsw_sp *mlxsw_sp, 1679 struct net_device *ol_dev) 1680 { 1681 struct mlxsw_sp_ipip_entry *ipip_entry; 1682 struct mlxsw_sp_rif_ipip_lb *lb_rif; 1683 int err = 0; 1684 1685 ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev); 1686 if (ipip_entry) { 1687 lb_rif = ipip_entry->ol_lb; 1688 err = mlxsw_sp_rif_ipip_lb_op(lb_rif, lb_rif->ul_vr_id, 1689 lb_rif->ul_rif_id, true); 1690 if (err) 1691 goto out; 1692 lb_rif->common.mtu = ol_dev->mtu; 1693 } 1694 1695 out: 1696 return err; 1697 } 1698 1699 static void mlxsw_sp_netdevice_ipip_ol_up_event(struct mlxsw_sp *mlxsw_sp, 1700 struct net_device *ol_dev) 1701 { 1702 struct mlxsw_sp_ipip_entry *ipip_entry; 1703 1704 ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev); 1705 if (ipip_entry) 1706 mlxsw_sp_ipip_entry_ol_up_event(mlxsw_sp, ipip_entry); 1707 } 1708 1709 static void 1710 mlxsw_sp_ipip_entry_ol_down_event(struct mlxsw_sp *mlxsw_sp, 1711 struct mlxsw_sp_ipip_entry *ipip_entry) 1712 { 1713 if (ipip_entry->decap_fib_entry) 1714 mlxsw_sp_ipip_entry_demote_decap(mlxsw_sp, ipip_entry); 1715 } 1716 1717 static void mlxsw_sp_netdevice_ipip_ol_down_event(struct mlxsw_sp *mlxsw_sp, 1718 struct net_device *ol_dev) 1719 { 1720 struct mlxsw_sp_ipip_entry *ipip_entry; 1721 1722 ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev); 1723 if (ipip_entry) 1724 mlxsw_sp_ipip_entry_ol_down_event(mlxsw_sp, ipip_entry); 1725 } 1726 1727 static void mlxsw_sp_nexthop_rif_update(struct mlxsw_sp *mlxsw_sp, 1728 struct mlxsw_sp_rif *rif); 1729 1730 static void mlxsw_sp_rif_migrate_destroy(struct mlxsw_sp *mlxsw_sp, 1731 struct mlxsw_sp_rif *old_rif, 1732 struct mlxsw_sp_rif *new_rif, 1733 bool migrate_nhs) 1734 { 1735 struct mlxsw_sp_crif *crif = old_rif->crif; 1736 struct mlxsw_sp_crif mock_crif = {}; 1737 1738 if (migrate_nhs) 1739 mlxsw_sp_nexthop_rif_update(mlxsw_sp, new_rif); 1740 1741 /* Plant a mock CRIF so that destroying the old RIF doesn't unoffload 1742 * our nexthops and IPIP tunnels, and doesn't sever the crif->rif link. 1743 */ 1744 mlxsw_sp_crif_init(&mock_crif, crif->key.dev); 1745 old_rif->crif = &mock_crif; 1746 mock_crif.rif = old_rif; 1747 mlxsw_sp_rif_destroy(old_rif); 1748 } 1749 1750 static int 1751 mlxsw_sp_ipip_entry_ol_lb_update(struct mlxsw_sp *mlxsw_sp, 1752 struct mlxsw_sp_ipip_entry *ipip_entry, 1753 bool keep_encap, 1754 struct netlink_ext_ack *extack) 1755 { 1756 struct mlxsw_sp_rif_ipip_lb *old_lb_rif = ipip_entry->ol_lb; 1757 struct mlxsw_sp_rif_ipip_lb *new_lb_rif; 1758 1759 new_lb_rif = mlxsw_sp_ipip_ol_ipip_lb_create(mlxsw_sp, 1760 ipip_entry->ipipt, 1761 ipip_entry->ol_dev, 1762 extack); 1763 if (IS_ERR(new_lb_rif)) 1764 return PTR_ERR(new_lb_rif); 1765 ipip_entry->ol_lb = new_lb_rif; 1766 1767 mlxsw_sp_rif_migrate_destroy(mlxsw_sp, &old_lb_rif->common, 1768 &new_lb_rif->common, keep_encap); 1769 return 0; 1770 } 1771 1772 /** 1773 * __mlxsw_sp_ipip_entry_update_tunnel - Update offload related to IPIP entry. 1774 * @mlxsw_sp: mlxsw_sp. 1775 * @ipip_entry: IPIP entry. 1776 * @recreate_loopback: Recreates the associated loopback RIF. 1777 * @keep_encap: Updates next hops that use the tunnel netdevice. This is only 1778 * relevant when recreate_loopback is true. 1779 * @update_nexthops: Updates next hops, keeping the current loopback RIF. This 1780 * is only relevant when recreate_loopback is false. 1781 * @extack: extack. 1782 * 1783 * Return: Non-zero value on failure. 1784 */ 1785 int __mlxsw_sp_ipip_entry_update_tunnel(struct mlxsw_sp *mlxsw_sp, 1786 struct mlxsw_sp_ipip_entry *ipip_entry, 1787 bool recreate_loopback, 1788 bool keep_encap, 1789 bool update_nexthops, 1790 struct netlink_ext_ack *extack) 1791 { 1792 int err; 1793 1794 /* RIFs can't be edited, so to update loopback, we need to destroy and 1795 * recreate it. That creates a window of opportunity where RALUE and 1796 * RATR registers end up referencing a RIF that's already gone. RATRs 1797 * are handled in mlxsw_sp_ipip_entry_ol_lb_update(), and to take care 1798 * of RALUE, demote the decap route back. 1799 */ 1800 if (ipip_entry->decap_fib_entry) 1801 mlxsw_sp_ipip_entry_demote_decap(mlxsw_sp, ipip_entry); 1802 1803 if (recreate_loopback) { 1804 err = mlxsw_sp_ipip_entry_ol_lb_update(mlxsw_sp, ipip_entry, 1805 keep_encap, extack); 1806 if (err) 1807 return err; 1808 } else if (update_nexthops) { 1809 mlxsw_sp_nexthop_rif_update(mlxsw_sp, 1810 &ipip_entry->ol_lb->common); 1811 } 1812 1813 if (ipip_entry->ol_dev->flags & IFF_UP) 1814 mlxsw_sp_ipip_entry_ol_up_event(mlxsw_sp, ipip_entry); 1815 1816 return 0; 1817 } 1818 1819 static int mlxsw_sp_netdevice_ipip_ol_vrf_event(struct mlxsw_sp *mlxsw_sp, 1820 struct net_device *ol_dev, 1821 struct netlink_ext_ack *extack) 1822 { 1823 struct mlxsw_sp_ipip_entry *ipip_entry = 1824 mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev); 1825 1826 if (!ipip_entry) 1827 return 0; 1828 1829 return __mlxsw_sp_ipip_entry_update_tunnel(mlxsw_sp, ipip_entry, 1830 true, false, false, extack); 1831 } 1832 1833 static int 1834 mlxsw_sp_netdevice_ipip_ul_vrf_event(struct mlxsw_sp *mlxsw_sp, 1835 struct mlxsw_sp_ipip_entry *ipip_entry, 1836 struct net_device *ul_dev, 1837 bool *demote_this, 1838 struct netlink_ext_ack *extack) 1839 { 1840 u32 ul_tb_id = l3mdev_fib_table(ul_dev) ? : RT_TABLE_MAIN; 1841 enum mlxsw_sp_l3proto ul_proto; 1842 union mlxsw_sp_l3addr saddr; 1843 1844 /* Moving underlay to a different VRF might cause local address 1845 * conflict, and the conflicting tunnels need to be demoted. 1846 */ 1847 ul_proto = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt]->ul_proto; 1848 saddr = mlxsw_sp_ipip_netdev_saddr(ul_proto, ipip_entry->ol_dev); 1849 if (mlxsw_sp_ipip_demote_tunnel_by_saddr(mlxsw_sp, ul_proto, 1850 saddr, ul_tb_id, 1851 ipip_entry)) { 1852 *demote_this = true; 1853 return 0; 1854 } 1855 1856 return __mlxsw_sp_ipip_entry_update_tunnel(mlxsw_sp, ipip_entry, 1857 true, true, false, extack); 1858 } 1859 1860 static int 1861 mlxsw_sp_netdevice_ipip_ul_up_event(struct mlxsw_sp *mlxsw_sp, 1862 struct mlxsw_sp_ipip_entry *ipip_entry, 1863 struct net_device *ul_dev) 1864 { 1865 return __mlxsw_sp_ipip_entry_update_tunnel(mlxsw_sp, ipip_entry, 1866 false, false, true, NULL); 1867 } 1868 1869 static int 1870 mlxsw_sp_netdevice_ipip_ul_down_event(struct mlxsw_sp *mlxsw_sp, 1871 struct mlxsw_sp_ipip_entry *ipip_entry, 1872 struct net_device *ul_dev) 1873 { 1874 /* A down underlay device causes encapsulated packets to not be 1875 * forwarded, but decap still works. So refresh next hops without 1876 * touching anything else. 1877 */ 1878 return __mlxsw_sp_ipip_entry_update_tunnel(mlxsw_sp, ipip_entry, 1879 false, false, true, NULL); 1880 } 1881 1882 static int 1883 mlxsw_sp_netdevice_ipip_ol_change_event(struct mlxsw_sp *mlxsw_sp, 1884 struct net_device *ol_dev, 1885 struct netlink_ext_ack *extack) 1886 { 1887 const struct mlxsw_sp_ipip_ops *ipip_ops; 1888 struct mlxsw_sp_ipip_entry *ipip_entry; 1889 int err; 1890 1891 ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev); 1892 if (!ipip_entry) 1893 /* A change might make a tunnel eligible for offloading, but 1894 * that is currently not implemented. What falls to slow path 1895 * stays there. 1896 */ 1897 return 0; 1898 1899 /* A change might make a tunnel not eligible for offloading. */ 1900 if (!mlxsw_sp_netdevice_ipip_can_offload(mlxsw_sp, ol_dev, 1901 ipip_entry->ipipt)) { 1902 mlxsw_sp_ipip_entry_demote_tunnel(mlxsw_sp, ipip_entry); 1903 return 0; 1904 } 1905 1906 ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt]; 1907 err = ipip_ops->ol_netdev_change(mlxsw_sp, ipip_entry, extack); 1908 return err; 1909 } 1910 1911 void mlxsw_sp_ipip_entry_demote_tunnel(struct mlxsw_sp *mlxsw_sp, 1912 struct mlxsw_sp_ipip_entry *ipip_entry) 1913 { 1914 struct net_device *ol_dev = ipip_entry->ol_dev; 1915 1916 if (ol_dev->flags & IFF_UP) 1917 mlxsw_sp_ipip_entry_ol_down_event(mlxsw_sp, ipip_entry); 1918 mlxsw_sp_ipip_entry_destroy(mlxsw_sp, ipip_entry); 1919 } 1920 1921 /* The configuration where several tunnels have the same local address in the 1922 * same underlay table needs special treatment in the HW. That is currently not 1923 * implemented in the driver. This function finds and demotes the first tunnel 1924 * with a given source address, except the one passed in the argument 1925 * `except'. 1926 */ 1927 bool 1928 mlxsw_sp_ipip_demote_tunnel_by_saddr(struct mlxsw_sp *mlxsw_sp, 1929 enum mlxsw_sp_l3proto ul_proto, 1930 union mlxsw_sp_l3addr saddr, 1931 u32 ul_tb_id, 1932 const struct mlxsw_sp_ipip_entry *except) 1933 { 1934 struct mlxsw_sp_ipip_entry *ipip_entry, *tmp; 1935 1936 list_for_each_entry_safe(ipip_entry, tmp, &mlxsw_sp->router->ipip_list, 1937 ipip_list_node) { 1938 if (ipip_entry != except && 1939 mlxsw_sp_ipip_entry_saddr_matches(mlxsw_sp, ul_proto, saddr, 1940 ul_tb_id, ipip_entry)) { 1941 mlxsw_sp_ipip_entry_demote_tunnel(mlxsw_sp, ipip_entry); 1942 return true; 1943 } 1944 } 1945 1946 return false; 1947 } 1948 1949 static void mlxsw_sp_ipip_demote_tunnel_by_ul_netdev(struct mlxsw_sp *mlxsw_sp, 1950 struct net_device *ul_dev) 1951 { 1952 struct mlxsw_sp_ipip_entry *ipip_entry, *tmp; 1953 1954 list_for_each_entry_safe(ipip_entry, tmp, &mlxsw_sp->router->ipip_list, 1955 ipip_list_node) { 1956 struct net_device *ol_dev = ipip_entry->ol_dev; 1957 struct net_device *ipip_ul_dev; 1958 1959 rcu_read_lock(); 1960 ipip_ul_dev = mlxsw_sp_ipip_netdev_ul_dev_get(ol_dev); 1961 rcu_read_unlock(); 1962 if (ipip_ul_dev == ul_dev) 1963 mlxsw_sp_ipip_entry_demote_tunnel(mlxsw_sp, ipip_entry); 1964 } 1965 } 1966 1967 static int mlxsw_sp_netdevice_ipip_ol_event(struct mlxsw_sp *mlxsw_sp, 1968 struct net_device *ol_dev, 1969 unsigned long event, 1970 struct netdev_notifier_info *info) 1971 { 1972 struct netdev_notifier_changeupper_info *chup; 1973 struct netlink_ext_ack *extack; 1974 int err = 0; 1975 1976 switch (event) { 1977 case NETDEV_REGISTER: 1978 err = mlxsw_sp_netdevice_ipip_ol_reg_event(mlxsw_sp, ol_dev); 1979 break; 1980 case NETDEV_UNREGISTER: 1981 mlxsw_sp_netdevice_ipip_ol_unreg_event(mlxsw_sp, ol_dev); 1982 break; 1983 case NETDEV_UP: 1984 mlxsw_sp_netdevice_ipip_ol_up_event(mlxsw_sp, ol_dev); 1985 break; 1986 case NETDEV_DOWN: 1987 mlxsw_sp_netdevice_ipip_ol_down_event(mlxsw_sp, ol_dev); 1988 break; 1989 case NETDEV_CHANGEUPPER: 1990 chup = container_of(info, typeof(*chup), info); 1991 extack = info->extack; 1992 if (netif_is_l3_master(chup->upper_dev)) 1993 err = mlxsw_sp_netdevice_ipip_ol_vrf_event(mlxsw_sp, 1994 ol_dev, 1995 extack); 1996 break; 1997 case NETDEV_CHANGE: 1998 extack = info->extack; 1999 err = mlxsw_sp_netdevice_ipip_ol_change_event(mlxsw_sp, 2000 ol_dev, extack); 2001 break; 2002 case NETDEV_CHANGEMTU: 2003 err = mlxsw_sp_netdevice_ipip_ol_update_mtu(mlxsw_sp, ol_dev); 2004 break; 2005 } 2006 return err; 2007 } 2008 2009 static int 2010 __mlxsw_sp_netdevice_ipip_ul_event(struct mlxsw_sp *mlxsw_sp, 2011 struct mlxsw_sp_ipip_entry *ipip_entry, 2012 struct net_device *ul_dev, 2013 bool *demote_this, 2014 unsigned long event, 2015 struct netdev_notifier_info *info) 2016 { 2017 struct netdev_notifier_changeupper_info *chup; 2018 struct netlink_ext_ack *extack; 2019 2020 switch (event) { 2021 case NETDEV_CHANGEUPPER: 2022 chup = container_of(info, typeof(*chup), info); 2023 extack = info->extack; 2024 if (netif_is_l3_master(chup->upper_dev)) 2025 return mlxsw_sp_netdevice_ipip_ul_vrf_event(mlxsw_sp, 2026 ipip_entry, 2027 ul_dev, 2028 demote_this, 2029 extack); 2030 break; 2031 2032 case NETDEV_UP: 2033 return mlxsw_sp_netdevice_ipip_ul_up_event(mlxsw_sp, ipip_entry, 2034 ul_dev); 2035 case NETDEV_DOWN: 2036 return mlxsw_sp_netdevice_ipip_ul_down_event(mlxsw_sp, 2037 ipip_entry, 2038 ul_dev); 2039 } 2040 return 0; 2041 } 2042 2043 static int 2044 mlxsw_sp_netdevice_ipip_ul_event(struct mlxsw_sp *mlxsw_sp, 2045 struct net_device *ul_dev, 2046 unsigned long event, 2047 struct netdev_notifier_info *info) 2048 { 2049 struct mlxsw_sp_ipip_entry *ipip_entry = NULL; 2050 int err; 2051 2052 while ((ipip_entry = mlxsw_sp_ipip_entry_find_by_ul_dev(mlxsw_sp, 2053 ul_dev, 2054 ipip_entry))) { 2055 struct mlxsw_sp_ipip_entry *prev; 2056 bool demote_this = false; 2057 2058 err = __mlxsw_sp_netdevice_ipip_ul_event(mlxsw_sp, ipip_entry, 2059 ul_dev, &demote_this, 2060 event, info); 2061 if (err) { 2062 mlxsw_sp_ipip_demote_tunnel_by_ul_netdev(mlxsw_sp, 2063 ul_dev); 2064 return err; 2065 } 2066 2067 if (demote_this) { 2068 if (list_is_first(&ipip_entry->ipip_list_node, 2069 &mlxsw_sp->router->ipip_list)) 2070 prev = NULL; 2071 else 2072 /* This can't be cached from previous iteration, 2073 * because that entry could be gone now. 2074 */ 2075 prev = list_prev_entry(ipip_entry, 2076 ipip_list_node); 2077 mlxsw_sp_ipip_entry_demote_tunnel(mlxsw_sp, ipip_entry); 2078 ipip_entry = prev; 2079 } 2080 } 2081 2082 return 0; 2083 } 2084 2085 int mlxsw_sp_router_nve_promote_decap(struct mlxsw_sp *mlxsw_sp, u32 ul_tb_id, 2086 enum mlxsw_sp_l3proto ul_proto, 2087 const union mlxsw_sp_l3addr *ul_sip, 2088 u32 tunnel_index) 2089 { 2090 enum mlxsw_sp_fib_entry_type type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP; 2091 struct mlxsw_sp_router *router = mlxsw_sp->router; 2092 struct mlxsw_sp_fib_entry *fib_entry; 2093 int err = 0; 2094 2095 mutex_lock(&mlxsw_sp->router->lock); 2096 2097 if (WARN_ON_ONCE(router->nve_decap_config.valid)) { 2098 err = -EINVAL; 2099 goto out; 2100 } 2101 2102 router->nve_decap_config.ul_tb_id = ul_tb_id; 2103 router->nve_decap_config.tunnel_index = tunnel_index; 2104 router->nve_decap_config.ul_proto = ul_proto; 2105 router->nve_decap_config.ul_sip = *ul_sip; 2106 router->nve_decap_config.valid = true; 2107 2108 /* It is valid to create a tunnel with a local IP and only later 2109 * assign this IP address to a local interface 2110 */ 2111 fib_entry = mlxsw_sp_router_ip2me_fib_entry_find(mlxsw_sp, ul_tb_id, 2112 ul_proto, ul_sip, 2113 type); 2114 if (!fib_entry) 2115 goto out; 2116 2117 fib_entry->decap.tunnel_index = tunnel_index; 2118 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP; 2119 2120 err = mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry); 2121 if (err) 2122 goto err_fib_entry_update; 2123 2124 goto out; 2125 2126 err_fib_entry_update: 2127 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP; 2128 mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry); 2129 out: 2130 mutex_unlock(&mlxsw_sp->router->lock); 2131 return err; 2132 } 2133 2134 void mlxsw_sp_router_nve_demote_decap(struct mlxsw_sp *mlxsw_sp, u32 ul_tb_id, 2135 enum mlxsw_sp_l3proto ul_proto, 2136 const union mlxsw_sp_l3addr *ul_sip) 2137 { 2138 enum mlxsw_sp_fib_entry_type type = MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP; 2139 struct mlxsw_sp_router *router = mlxsw_sp->router; 2140 struct mlxsw_sp_fib_entry *fib_entry; 2141 2142 mutex_lock(&mlxsw_sp->router->lock); 2143 2144 if (WARN_ON_ONCE(!router->nve_decap_config.valid)) 2145 goto out; 2146 2147 router->nve_decap_config.valid = false; 2148 2149 fib_entry = mlxsw_sp_router_ip2me_fib_entry_find(mlxsw_sp, ul_tb_id, 2150 ul_proto, ul_sip, 2151 type); 2152 if (!fib_entry) 2153 goto out; 2154 2155 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP; 2156 mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry); 2157 out: 2158 mutex_unlock(&mlxsw_sp->router->lock); 2159 } 2160 2161 static bool mlxsw_sp_router_nve_is_decap(struct mlxsw_sp *mlxsw_sp, 2162 u32 ul_tb_id, 2163 enum mlxsw_sp_l3proto ul_proto, 2164 const union mlxsw_sp_l3addr *ul_sip) 2165 { 2166 struct mlxsw_sp_router *router = mlxsw_sp->router; 2167 2168 return router->nve_decap_config.valid && 2169 router->nve_decap_config.ul_tb_id == ul_tb_id && 2170 router->nve_decap_config.ul_proto == ul_proto && 2171 !memcmp(&router->nve_decap_config.ul_sip, ul_sip, 2172 sizeof(*ul_sip)); 2173 } 2174 2175 struct mlxsw_sp_neigh_key { 2176 struct neighbour *n; 2177 }; 2178 2179 struct mlxsw_sp_neigh_entry { 2180 struct list_head rif_list_node; 2181 struct rhash_head ht_node; 2182 struct mlxsw_sp_neigh_key key; 2183 u16 rif; 2184 bool connected; 2185 unsigned char ha[ETH_ALEN]; 2186 struct list_head nexthop_list; /* list of nexthops using 2187 * this neigh entry 2188 */ 2189 struct list_head nexthop_neighs_list_node; 2190 unsigned int counter_index; 2191 bool counter_valid; 2192 }; 2193 2194 static const struct rhashtable_params mlxsw_sp_neigh_ht_params = { 2195 .key_offset = offsetof(struct mlxsw_sp_neigh_entry, key), 2196 .head_offset = offsetof(struct mlxsw_sp_neigh_entry, ht_node), 2197 .key_len = sizeof(struct mlxsw_sp_neigh_key), 2198 }; 2199 2200 struct mlxsw_sp_neigh_entry * 2201 mlxsw_sp_rif_neigh_next(struct mlxsw_sp_rif *rif, 2202 struct mlxsw_sp_neigh_entry *neigh_entry) 2203 { 2204 if (!neigh_entry) { 2205 if (list_empty(&rif->neigh_list)) 2206 return NULL; 2207 else 2208 return list_first_entry(&rif->neigh_list, 2209 typeof(*neigh_entry), 2210 rif_list_node); 2211 } 2212 if (list_is_last(&neigh_entry->rif_list_node, &rif->neigh_list)) 2213 return NULL; 2214 return list_next_entry(neigh_entry, rif_list_node); 2215 } 2216 2217 int mlxsw_sp_neigh_entry_type(struct mlxsw_sp_neigh_entry *neigh_entry) 2218 { 2219 return neigh_entry->key.n->tbl->family; 2220 } 2221 2222 unsigned char * 2223 mlxsw_sp_neigh_entry_ha(struct mlxsw_sp_neigh_entry *neigh_entry) 2224 { 2225 return neigh_entry->ha; 2226 } 2227 2228 u32 mlxsw_sp_neigh4_entry_dip(struct mlxsw_sp_neigh_entry *neigh_entry) 2229 { 2230 struct neighbour *n; 2231 2232 n = neigh_entry->key.n; 2233 return ntohl(*((__be32 *) n->primary_key)); 2234 } 2235 2236 struct in6_addr * 2237 mlxsw_sp_neigh6_entry_dip(struct mlxsw_sp_neigh_entry *neigh_entry) 2238 { 2239 struct neighbour *n; 2240 2241 n = neigh_entry->key.n; 2242 return (struct in6_addr *) &n->primary_key; 2243 } 2244 2245 int mlxsw_sp_neigh_counter_get(struct mlxsw_sp *mlxsw_sp, 2246 struct mlxsw_sp_neigh_entry *neigh_entry, 2247 u64 *p_counter) 2248 { 2249 if (!neigh_entry->counter_valid) 2250 return -EINVAL; 2251 2252 return mlxsw_sp_flow_counter_get(mlxsw_sp, neigh_entry->counter_index, 2253 p_counter, NULL); 2254 } 2255 2256 static struct mlxsw_sp_neigh_entry * 2257 mlxsw_sp_neigh_entry_alloc(struct mlxsw_sp *mlxsw_sp, struct neighbour *n, 2258 u16 rif) 2259 { 2260 struct mlxsw_sp_neigh_entry *neigh_entry; 2261 2262 neigh_entry = kzalloc(sizeof(*neigh_entry), GFP_KERNEL); 2263 if (!neigh_entry) 2264 return NULL; 2265 2266 neigh_entry->key.n = n; 2267 neigh_entry->rif = rif; 2268 INIT_LIST_HEAD(&neigh_entry->nexthop_list); 2269 2270 return neigh_entry; 2271 } 2272 2273 static void mlxsw_sp_neigh_entry_free(struct mlxsw_sp_neigh_entry *neigh_entry) 2274 { 2275 kfree(neigh_entry); 2276 } 2277 2278 static int 2279 mlxsw_sp_neigh_entry_insert(struct mlxsw_sp *mlxsw_sp, 2280 struct mlxsw_sp_neigh_entry *neigh_entry) 2281 { 2282 return rhashtable_insert_fast(&mlxsw_sp->router->neigh_ht, 2283 &neigh_entry->ht_node, 2284 mlxsw_sp_neigh_ht_params); 2285 } 2286 2287 static void 2288 mlxsw_sp_neigh_entry_remove(struct mlxsw_sp *mlxsw_sp, 2289 struct mlxsw_sp_neigh_entry *neigh_entry) 2290 { 2291 rhashtable_remove_fast(&mlxsw_sp->router->neigh_ht, 2292 &neigh_entry->ht_node, 2293 mlxsw_sp_neigh_ht_params); 2294 } 2295 2296 static bool 2297 mlxsw_sp_neigh_counter_should_alloc(struct mlxsw_sp *mlxsw_sp, 2298 struct mlxsw_sp_neigh_entry *neigh_entry) 2299 { 2300 struct devlink *devlink; 2301 const char *table_name; 2302 2303 switch (mlxsw_sp_neigh_entry_type(neigh_entry)) { 2304 case AF_INET: 2305 table_name = MLXSW_SP_DPIPE_TABLE_NAME_HOST4; 2306 break; 2307 case AF_INET6: 2308 table_name = MLXSW_SP_DPIPE_TABLE_NAME_HOST6; 2309 break; 2310 default: 2311 WARN_ON(1); 2312 return false; 2313 } 2314 2315 devlink = priv_to_devlink(mlxsw_sp->core); 2316 return devlink_dpipe_table_counter_enabled(devlink, table_name); 2317 } 2318 2319 static void 2320 mlxsw_sp_neigh_counter_alloc(struct mlxsw_sp *mlxsw_sp, 2321 struct mlxsw_sp_neigh_entry *neigh_entry) 2322 { 2323 if (!mlxsw_sp_neigh_counter_should_alloc(mlxsw_sp, neigh_entry)) 2324 return; 2325 2326 if (mlxsw_sp_flow_counter_alloc(mlxsw_sp, &neigh_entry->counter_index)) 2327 return; 2328 2329 neigh_entry->counter_valid = true; 2330 } 2331 2332 static void 2333 mlxsw_sp_neigh_counter_free(struct mlxsw_sp *mlxsw_sp, 2334 struct mlxsw_sp_neigh_entry *neigh_entry) 2335 { 2336 if (!neigh_entry->counter_valid) 2337 return; 2338 mlxsw_sp_flow_counter_free(mlxsw_sp, 2339 neigh_entry->counter_index); 2340 neigh_entry->counter_valid = false; 2341 } 2342 2343 static struct mlxsw_sp_neigh_entry * 2344 mlxsw_sp_neigh_entry_create(struct mlxsw_sp *mlxsw_sp, struct neighbour *n) 2345 { 2346 struct mlxsw_sp_neigh_entry *neigh_entry; 2347 struct mlxsw_sp_rif *rif; 2348 int err; 2349 2350 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, n->dev); 2351 if (!rif) 2352 return ERR_PTR(-EINVAL); 2353 2354 neigh_entry = mlxsw_sp_neigh_entry_alloc(mlxsw_sp, n, rif->rif_index); 2355 if (!neigh_entry) 2356 return ERR_PTR(-ENOMEM); 2357 2358 err = mlxsw_sp_neigh_entry_insert(mlxsw_sp, neigh_entry); 2359 if (err) 2360 goto err_neigh_entry_insert; 2361 2362 mlxsw_sp_neigh_counter_alloc(mlxsw_sp, neigh_entry); 2363 atomic_inc(&mlxsw_sp->router->neighs_update.neigh_count); 2364 list_add(&neigh_entry->rif_list_node, &rif->neigh_list); 2365 2366 return neigh_entry; 2367 2368 err_neigh_entry_insert: 2369 mlxsw_sp_neigh_entry_free(neigh_entry); 2370 return ERR_PTR(err); 2371 } 2372 2373 static void 2374 mlxsw_sp_neigh_entry_destroy(struct mlxsw_sp *mlxsw_sp, 2375 struct mlxsw_sp_neigh_entry *neigh_entry) 2376 { 2377 list_del(&neigh_entry->rif_list_node); 2378 atomic_dec(&mlxsw_sp->router->neighs_update.neigh_count); 2379 mlxsw_sp_neigh_counter_free(mlxsw_sp, neigh_entry); 2380 mlxsw_sp_neigh_entry_remove(mlxsw_sp, neigh_entry); 2381 mlxsw_sp_neigh_entry_free(neigh_entry); 2382 } 2383 2384 static struct mlxsw_sp_neigh_entry * 2385 mlxsw_sp_neigh_entry_lookup(struct mlxsw_sp *mlxsw_sp, struct neighbour *n) 2386 { 2387 struct mlxsw_sp_neigh_key key; 2388 2389 key.n = n; 2390 return rhashtable_lookup_fast(&mlxsw_sp->router->neigh_ht, 2391 &key, mlxsw_sp_neigh_ht_params); 2392 } 2393 2394 static void 2395 mlxsw_sp_router_neighs_update_interval_init(struct mlxsw_sp *mlxsw_sp) 2396 { 2397 unsigned long interval; 2398 2399 #if IS_ENABLED(CONFIG_IPV6) 2400 interval = min_t(unsigned long, 2401 NEIGH_VAR(&arp_tbl.parms, DELAY_PROBE_TIME), 2402 NEIGH_VAR(&nd_tbl.parms, DELAY_PROBE_TIME)); 2403 #else 2404 interval = NEIGH_VAR(&arp_tbl.parms, DELAY_PROBE_TIME); 2405 #endif 2406 mlxsw_sp->router->neighs_update.interval = jiffies_to_msecs(interval); 2407 } 2408 2409 static void mlxsw_sp_router_neigh_ent_ipv4_process(struct mlxsw_sp *mlxsw_sp, 2410 char *rauhtd_pl, 2411 int ent_index) 2412 { 2413 u64 max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); 2414 struct net_device *dev; 2415 struct neighbour *n; 2416 __be32 dipn; 2417 u32 dip; 2418 u16 rif; 2419 2420 mlxsw_reg_rauhtd_ent_ipv4_unpack(rauhtd_pl, ent_index, &rif, &dip); 2421 2422 if (WARN_ON_ONCE(rif >= max_rifs)) 2423 return; 2424 if (!mlxsw_sp->router->rifs[rif]) { 2425 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Incorrect RIF in neighbour entry\n"); 2426 return; 2427 } 2428 2429 dipn = htonl(dip); 2430 dev = mlxsw_sp_rif_dev(mlxsw_sp->router->rifs[rif]); 2431 n = neigh_lookup(&arp_tbl, &dipn, dev); 2432 if (!n) 2433 return; 2434 2435 netdev_dbg(dev, "Updating neighbour with IP=%pI4h\n", &dip); 2436 neigh_event_send(n, NULL); 2437 neigh_release(n); 2438 } 2439 2440 #if IS_ENABLED(CONFIG_IPV6) 2441 static void mlxsw_sp_router_neigh_ent_ipv6_process(struct mlxsw_sp *mlxsw_sp, 2442 char *rauhtd_pl, 2443 int rec_index) 2444 { 2445 struct net_device *dev; 2446 struct neighbour *n; 2447 struct in6_addr dip; 2448 u16 rif; 2449 2450 mlxsw_reg_rauhtd_ent_ipv6_unpack(rauhtd_pl, rec_index, &rif, 2451 (char *) &dip); 2452 2453 if (!mlxsw_sp->router->rifs[rif]) { 2454 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Incorrect RIF in neighbour entry\n"); 2455 return; 2456 } 2457 2458 dev = mlxsw_sp_rif_dev(mlxsw_sp->router->rifs[rif]); 2459 n = neigh_lookup(&nd_tbl, &dip, dev); 2460 if (!n) 2461 return; 2462 2463 netdev_dbg(dev, "Updating neighbour with IP=%pI6c\n", &dip); 2464 neigh_event_send(n, NULL); 2465 neigh_release(n); 2466 } 2467 #else 2468 static void mlxsw_sp_router_neigh_ent_ipv6_process(struct mlxsw_sp *mlxsw_sp, 2469 char *rauhtd_pl, 2470 int rec_index) 2471 { 2472 } 2473 #endif 2474 2475 static void mlxsw_sp_router_neigh_rec_ipv4_process(struct mlxsw_sp *mlxsw_sp, 2476 char *rauhtd_pl, 2477 int rec_index) 2478 { 2479 u8 num_entries; 2480 int i; 2481 2482 num_entries = mlxsw_reg_rauhtd_ipv4_rec_num_entries_get(rauhtd_pl, 2483 rec_index); 2484 /* Hardware starts counting at 0, so add 1. */ 2485 num_entries++; 2486 2487 /* Each record consists of several neighbour entries. */ 2488 for (i = 0; i < num_entries; i++) { 2489 int ent_index; 2490 2491 ent_index = rec_index * MLXSW_REG_RAUHTD_IPV4_ENT_PER_REC + i; 2492 mlxsw_sp_router_neigh_ent_ipv4_process(mlxsw_sp, rauhtd_pl, 2493 ent_index); 2494 } 2495 2496 } 2497 2498 static void mlxsw_sp_router_neigh_rec_ipv6_process(struct mlxsw_sp *mlxsw_sp, 2499 char *rauhtd_pl, 2500 int rec_index) 2501 { 2502 /* One record contains one entry. */ 2503 mlxsw_sp_router_neigh_ent_ipv6_process(mlxsw_sp, rauhtd_pl, 2504 rec_index); 2505 } 2506 2507 static void mlxsw_sp_router_neigh_rec_process(struct mlxsw_sp *mlxsw_sp, 2508 char *rauhtd_pl, int rec_index) 2509 { 2510 switch (mlxsw_reg_rauhtd_rec_type_get(rauhtd_pl, rec_index)) { 2511 case MLXSW_REG_RAUHTD_TYPE_IPV4: 2512 mlxsw_sp_router_neigh_rec_ipv4_process(mlxsw_sp, rauhtd_pl, 2513 rec_index); 2514 break; 2515 case MLXSW_REG_RAUHTD_TYPE_IPV6: 2516 mlxsw_sp_router_neigh_rec_ipv6_process(mlxsw_sp, rauhtd_pl, 2517 rec_index); 2518 break; 2519 } 2520 } 2521 2522 static bool mlxsw_sp_router_rauhtd_is_full(char *rauhtd_pl) 2523 { 2524 u8 num_rec, last_rec_index, num_entries; 2525 2526 num_rec = mlxsw_reg_rauhtd_num_rec_get(rauhtd_pl); 2527 last_rec_index = num_rec - 1; 2528 2529 if (num_rec < MLXSW_REG_RAUHTD_REC_MAX_NUM) 2530 return false; 2531 if (mlxsw_reg_rauhtd_rec_type_get(rauhtd_pl, last_rec_index) == 2532 MLXSW_REG_RAUHTD_TYPE_IPV6) 2533 return true; 2534 2535 num_entries = mlxsw_reg_rauhtd_ipv4_rec_num_entries_get(rauhtd_pl, 2536 last_rec_index); 2537 if (++num_entries == MLXSW_REG_RAUHTD_IPV4_ENT_PER_REC) 2538 return true; 2539 return false; 2540 } 2541 2542 static int 2543 __mlxsw_sp_router_neighs_update_rauhtd(struct mlxsw_sp *mlxsw_sp, 2544 char *rauhtd_pl, 2545 enum mlxsw_reg_rauhtd_type type) 2546 { 2547 int i, num_rec; 2548 int err; 2549 2550 /* Ensure the RIF we read from the device does not change mid-dump. */ 2551 mutex_lock(&mlxsw_sp->router->lock); 2552 do { 2553 mlxsw_reg_rauhtd_pack(rauhtd_pl, type); 2554 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(rauhtd), 2555 rauhtd_pl); 2556 if (err) { 2557 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Failed to dump neighbour table\n"); 2558 break; 2559 } 2560 num_rec = mlxsw_reg_rauhtd_num_rec_get(rauhtd_pl); 2561 for (i = 0; i < num_rec; i++) 2562 mlxsw_sp_router_neigh_rec_process(mlxsw_sp, rauhtd_pl, 2563 i); 2564 } while (mlxsw_sp_router_rauhtd_is_full(rauhtd_pl)); 2565 mutex_unlock(&mlxsw_sp->router->lock); 2566 2567 return err; 2568 } 2569 2570 static int mlxsw_sp_router_neighs_update_rauhtd(struct mlxsw_sp *mlxsw_sp) 2571 { 2572 enum mlxsw_reg_rauhtd_type type; 2573 char *rauhtd_pl; 2574 int err; 2575 2576 if (!atomic_read(&mlxsw_sp->router->neighs_update.neigh_count)) 2577 return 0; 2578 2579 rauhtd_pl = kmalloc(MLXSW_REG_RAUHTD_LEN, GFP_KERNEL); 2580 if (!rauhtd_pl) 2581 return -ENOMEM; 2582 2583 type = MLXSW_REG_RAUHTD_TYPE_IPV4; 2584 err = __mlxsw_sp_router_neighs_update_rauhtd(mlxsw_sp, rauhtd_pl, type); 2585 if (err) 2586 goto out; 2587 2588 type = MLXSW_REG_RAUHTD_TYPE_IPV6; 2589 err = __mlxsw_sp_router_neighs_update_rauhtd(mlxsw_sp, rauhtd_pl, type); 2590 out: 2591 kfree(rauhtd_pl); 2592 return err; 2593 } 2594 2595 static void mlxsw_sp_router_neighs_update_nh(struct mlxsw_sp *mlxsw_sp) 2596 { 2597 struct mlxsw_sp_neigh_entry *neigh_entry; 2598 2599 mutex_lock(&mlxsw_sp->router->lock); 2600 list_for_each_entry(neigh_entry, &mlxsw_sp->router->nexthop_neighs_list, 2601 nexthop_neighs_list_node) 2602 /* If this neigh have nexthops, make the kernel think this neigh 2603 * is active regardless of the traffic. 2604 */ 2605 neigh_event_send(neigh_entry->key.n, NULL); 2606 mutex_unlock(&mlxsw_sp->router->lock); 2607 } 2608 2609 static void 2610 mlxsw_sp_router_neighs_update_work_schedule(struct mlxsw_sp *mlxsw_sp) 2611 { 2612 unsigned long interval = mlxsw_sp->router->neighs_update.interval; 2613 2614 mlxsw_core_schedule_dw(&mlxsw_sp->router->neighs_update.dw, 2615 msecs_to_jiffies(interval)); 2616 } 2617 2618 static void mlxsw_sp_router_neighs_update_work(struct work_struct *work) 2619 { 2620 struct mlxsw_sp_router *router; 2621 int err; 2622 2623 router = container_of(work, struct mlxsw_sp_router, 2624 neighs_update.dw.work); 2625 err = mlxsw_sp_router_neighs_update_rauhtd(router->mlxsw_sp); 2626 if (err) 2627 dev_err(router->mlxsw_sp->bus_info->dev, "Could not update kernel for neigh activity"); 2628 2629 mlxsw_sp_router_neighs_update_nh(router->mlxsw_sp); 2630 2631 mlxsw_sp_router_neighs_update_work_schedule(router->mlxsw_sp); 2632 } 2633 2634 static void mlxsw_sp_router_probe_unresolved_nexthops(struct work_struct *work) 2635 { 2636 struct mlxsw_sp_neigh_entry *neigh_entry; 2637 struct mlxsw_sp_router *router; 2638 2639 router = container_of(work, struct mlxsw_sp_router, 2640 nexthop_probe_dw.work); 2641 /* Iterate over nexthop neighbours, find those who are unresolved and 2642 * send arp on them. This solves the chicken-egg problem when 2643 * the nexthop wouldn't get offloaded until the neighbor is resolved 2644 * but it wouldn't get resolved ever in case traffic is flowing in HW 2645 * using different nexthop. 2646 */ 2647 mutex_lock(&router->lock); 2648 list_for_each_entry(neigh_entry, &router->nexthop_neighs_list, 2649 nexthop_neighs_list_node) 2650 if (!neigh_entry->connected) 2651 neigh_event_send(neigh_entry->key.n, NULL); 2652 mutex_unlock(&router->lock); 2653 2654 mlxsw_core_schedule_dw(&router->nexthop_probe_dw, 2655 MLXSW_SP_UNRESOLVED_NH_PROBE_INTERVAL); 2656 } 2657 2658 static void 2659 mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp *mlxsw_sp, 2660 struct mlxsw_sp_neigh_entry *neigh_entry, 2661 bool removing, bool dead); 2662 2663 static enum mlxsw_reg_rauht_op mlxsw_sp_rauht_op(bool adding) 2664 { 2665 return adding ? MLXSW_REG_RAUHT_OP_WRITE_ADD : 2666 MLXSW_REG_RAUHT_OP_WRITE_DELETE; 2667 } 2668 2669 static int 2670 mlxsw_sp_router_neigh_entry_op4(struct mlxsw_sp *mlxsw_sp, 2671 struct mlxsw_sp_neigh_entry *neigh_entry, 2672 enum mlxsw_reg_rauht_op op) 2673 { 2674 struct neighbour *n = neigh_entry->key.n; 2675 u32 dip = ntohl(*((__be32 *) n->primary_key)); 2676 char rauht_pl[MLXSW_REG_RAUHT_LEN]; 2677 2678 mlxsw_reg_rauht_pack4(rauht_pl, op, neigh_entry->rif, neigh_entry->ha, 2679 dip); 2680 if (neigh_entry->counter_valid) 2681 mlxsw_reg_rauht_pack_counter(rauht_pl, 2682 neigh_entry->counter_index); 2683 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rauht), rauht_pl); 2684 } 2685 2686 static int 2687 mlxsw_sp_router_neigh_entry_op6(struct mlxsw_sp *mlxsw_sp, 2688 struct mlxsw_sp_neigh_entry *neigh_entry, 2689 enum mlxsw_reg_rauht_op op) 2690 { 2691 struct neighbour *n = neigh_entry->key.n; 2692 char rauht_pl[MLXSW_REG_RAUHT_LEN]; 2693 const char *dip = n->primary_key; 2694 2695 mlxsw_reg_rauht_pack6(rauht_pl, op, neigh_entry->rif, neigh_entry->ha, 2696 dip); 2697 if (neigh_entry->counter_valid) 2698 mlxsw_reg_rauht_pack_counter(rauht_pl, 2699 neigh_entry->counter_index); 2700 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rauht), rauht_pl); 2701 } 2702 2703 bool mlxsw_sp_neigh_ipv6_ignore(struct mlxsw_sp_neigh_entry *neigh_entry) 2704 { 2705 struct neighbour *n = neigh_entry->key.n; 2706 2707 /* Packets with a link-local destination address are trapped 2708 * after LPM lookup and never reach the neighbour table, so 2709 * there is no need to program such neighbours to the device. 2710 */ 2711 if (ipv6_addr_type((struct in6_addr *) &n->primary_key) & 2712 IPV6_ADDR_LINKLOCAL) 2713 return true; 2714 return false; 2715 } 2716 2717 static void 2718 mlxsw_sp_neigh_entry_update(struct mlxsw_sp *mlxsw_sp, 2719 struct mlxsw_sp_neigh_entry *neigh_entry, 2720 bool adding) 2721 { 2722 enum mlxsw_reg_rauht_op op = mlxsw_sp_rauht_op(adding); 2723 int err; 2724 2725 if (!adding && !neigh_entry->connected) 2726 return; 2727 neigh_entry->connected = adding; 2728 if (neigh_entry->key.n->tbl->family == AF_INET) { 2729 err = mlxsw_sp_router_neigh_entry_op4(mlxsw_sp, neigh_entry, 2730 op); 2731 if (err) 2732 return; 2733 } else if (neigh_entry->key.n->tbl->family == AF_INET6) { 2734 if (mlxsw_sp_neigh_ipv6_ignore(neigh_entry)) 2735 return; 2736 err = mlxsw_sp_router_neigh_entry_op6(mlxsw_sp, neigh_entry, 2737 op); 2738 if (err) 2739 return; 2740 } else { 2741 WARN_ON_ONCE(1); 2742 return; 2743 } 2744 2745 if (adding) 2746 neigh_entry->key.n->flags |= NTF_OFFLOADED; 2747 else 2748 neigh_entry->key.n->flags &= ~NTF_OFFLOADED; 2749 } 2750 2751 void 2752 mlxsw_sp_neigh_entry_counter_update(struct mlxsw_sp *mlxsw_sp, 2753 struct mlxsw_sp_neigh_entry *neigh_entry, 2754 bool adding) 2755 { 2756 if (adding) 2757 mlxsw_sp_neigh_counter_alloc(mlxsw_sp, neigh_entry); 2758 else 2759 mlxsw_sp_neigh_counter_free(mlxsw_sp, neigh_entry); 2760 mlxsw_sp_neigh_entry_update(mlxsw_sp, neigh_entry, true); 2761 } 2762 2763 struct mlxsw_sp_netevent_work { 2764 struct work_struct work; 2765 struct mlxsw_sp *mlxsw_sp; 2766 struct neighbour *n; 2767 }; 2768 2769 static void mlxsw_sp_router_neigh_event_work(struct work_struct *work) 2770 { 2771 struct mlxsw_sp_netevent_work *net_work = 2772 container_of(work, struct mlxsw_sp_netevent_work, work); 2773 struct mlxsw_sp *mlxsw_sp = net_work->mlxsw_sp; 2774 struct mlxsw_sp_neigh_entry *neigh_entry; 2775 struct neighbour *n = net_work->n; 2776 unsigned char ha[ETH_ALEN]; 2777 bool entry_connected; 2778 u8 nud_state, dead; 2779 2780 /* If these parameters are changed after we release the lock, 2781 * then we are guaranteed to receive another event letting us 2782 * know about it. 2783 */ 2784 read_lock_bh(&n->lock); 2785 memcpy(ha, n->ha, ETH_ALEN); 2786 nud_state = n->nud_state; 2787 dead = n->dead; 2788 read_unlock_bh(&n->lock); 2789 2790 mutex_lock(&mlxsw_sp->router->lock); 2791 mlxsw_sp_span_respin(mlxsw_sp); 2792 2793 entry_connected = nud_state & NUD_VALID && !dead; 2794 neigh_entry = mlxsw_sp_neigh_entry_lookup(mlxsw_sp, n); 2795 if (!entry_connected && !neigh_entry) 2796 goto out; 2797 if (!neigh_entry) { 2798 neigh_entry = mlxsw_sp_neigh_entry_create(mlxsw_sp, n); 2799 if (IS_ERR(neigh_entry)) 2800 goto out; 2801 } 2802 2803 if (neigh_entry->connected && entry_connected && 2804 !memcmp(neigh_entry->ha, ha, ETH_ALEN)) 2805 goto out; 2806 2807 memcpy(neigh_entry->ha, ha, ETH_ALEN); 2808 mlxsw_sp_neigh_entry_update(mlxsw_sp, neigh_entry, entry_connected); 2809 mlxsw_sp_nexthop_neigh_update(mlxsw_sp, neigh_entry, !entry_connected, 2810 dead); 2811 2812 if (!neigh_entry->connected && list_empty(&neigh_entry->nexthop_list)) 2813 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry); 2814 2815 out: 2816 mutex_unlock(&mlxsw_sp->router->lock); 2817 neigh_release(n); 2818 kfree(net_work); 2819 } 2820 2821 static int mlxsw_sp_mp_hash_init(struct mlxsw_sp *mlxsw_sp); 2822 2823 static void mlxsw_sp_router_mp_hash_event_work(struct work_struct *work) 2824 { 2825 struct mlxsw_sp_netevent_work *net_work = 2826 container_of(work, struct mlxsw_sp_netevent_work, work); 2827 struct mlxsw_sp *mlxsw_sp = net_work->mlxsw_sp; 2828 2829 mlxsw_sp_mp_hash_init(mlxsw_sp); 2830 kfree(net_work); 2831 } 2832 2833 static int __mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp); 2834 2835 static void mlxsw_sp_router_update_priority_work(struct work_struct *work) 2836 { 2837 struct mlxsw_sp_netevent_work *net_work = 2838 container_of(work, struct mlxsw_sp_netevent_work, work); 2839 struct mlxsw_sp *mlxsw_sp = net_work->mlxsw_sp; 2840 2841 __mlxsw_sp_router_init(mlxsw_sp); 2842 kfree(net_work); 2843 } 2844 2845 static int mlxsw_sp_router_schedule_work(struct net *net, 2846 struct mlxsw_sp_router *router, 2847 struct neighbour *n, 2848 void (*cb)(struct work_struct *)) 2849 { 2850 struct mlxsw_sp_netevent_work *net_work; 2851 2852 if (!net_eq(net, mlxsw_sp_net(router->mlxsw_sp))) 2853 return NOTIFY_DONE; 2854 2855 net_work = kzalloc(sizeof(*net_work), GFP_ATOMIC); 2856 if (!net_work) 2857 return NOTIFY_BAD; 2858 2859 INIT_WORK(&net_work->work, cb); 2860 net_work->mlxsw_sp = router->mlxsw_sp; 2861 net_work->n = n; 2862 mlxsw_core_schedule_work(&net_work->work); 2863 return NOTIFY_DONE; 2864 } 2865 2866 static bool mlxsw_sp_dev_lower_is_port(struct net_device *dev) 2867 { 2868 struct mlxsw_sp_port *mlxsw_sp_port; 2869 2870 rcu_read_lock(); 2871 mlxsw_sp_port = mlxsw_sp_port_dev_lower_find_rcu(dev); 2872 rcu_read_unlock(); 2873 return !!mlxsw_sp_port; 2874 } 2875 2876 static int mlxsw_sp_router_schedule_neigh_work(struct mlxsw_sp_router *router, 2877 struct neighbour *n) 2878 { 2879 struct net *net; 2880 2881 net = neigh_parms_net(n->parms); 2882 2883 /* Take a reference to ensure the neighbour won't be destructed until we 2884 * drop the reference in delayed work. 2885 */ 2886 neigh_clone(n); 2887 return mlxsw_sp_router_schedule_work(net, router, n, 2888 mlxsw_sp_router_neigh_event_work); 2889 } 2890 2891 static int mlxsw_sp_router_netevent_event(struct notifier_block *nb, 2892 unsigned long event, void *ptr) 2893 { 2894 struct mlxsw_sp_router *router; 2895 unsigned long interval; 2896 struct neigh_parms *p; 2897 struct neighbour *n; 2898 2899 router = container_of(nb, struct mlxsw_sp_router, netevent_nb); 2900 2901 switch (event) { 2902 case NETEVENT_DELAY_PROBE_TIME_UPDATE: 2903 p = ptr; 2904 2905 /* We don't care about changes in the default table. */ 2906 if (!p->dev || (p->tbl->family != AF_INET && 2907 p->tbl->family != AF_INET6)) 2908 return NOTIFY_DONE; 2909 2910 /* We are in atomic context and can't take RTNL mutex, 2911 * so use RCU variant to walk the device chain. 2912 */ 2913 if (!mlxsw_sp_dev_lower_is_port(p->dev)) 2914 return NOTIFY_DONE; 2915 2916 interval = jiffies_to_msecs(NEIGH_VAR(p, DELAY_PROBE_TIME)); 2917 router->neighs_update.interval = interval; 2918 break; 2919 case NETEVENT_NEIGH_UPDATE: 2920 n = ptr; 2921 2922 if (n->tbl->family != AF_INET && n->tbl->family != AF_INET6) 2923 return NOTIFY_DONE; 2924 2925 if (!mlxsw_sp_dev_lower_is_port(n->dev)) 2926 return NOTIFY_DONE; 2927 2928 return mlxsw_sp_router_schedule_neigh_work(router, n); 2929 2930 case NETEVENT_IPV4_MPATH_HASH_UPDATE: 2931 case NETEVENT_IPV6_MPATH_HASH_UPDATE: 2932 return mlxsw_sp_router_schedule_work(ptr, router, NULL, 2933 mlxsw_sp_router_mp_hash_event_work); 2934 2935 case NETEVENT_IPV4_FWD_UPDATE_PRIORITY_UPDATE: 2936 return mlxsw_sp_router_schedule_work(ptr, router, NULL, 2937 mlxsw_sp_router_update_priority_work); 2938 } 2939 2940 return NOTIFY_DONE; 2941 } 2942 2943 static int mlxsw_sp_neigh_init(struct mlxsw_sp *mlxsw_sp) 2944 { 2945 int err; 2946 2947 err = rhashtable_init(&mlxsw_sp->router->neigh_ht, 2948 &mlxsw_sp_neigh_ht_params); 2949 if (err) 2950 return err; 2951 2952 /* Initialize the polling interval according to the default 2953 * table. 2954 */ 2955 mlxsw_sp_router_neighs_update_interval_init(mlxsw_sp); 2956 2957 /* Create the delayed works for the activity_update */ 2958 INIT_DELAYED_WORK(&mlxsw_sp->router->neighs_update.dw, 2959 mlxsw_sp_router_neighs_update_work); 2960 INIT_DELAYED_WORK(&mlxsw_sp->router->nexthop_probe_dw, 2961 mlxsw_sp_router_probe_unresolved_nexthops); 2962 atomic_set(&mlxsw_sp->router->neighs_update.neigh_count, 0); 2963 mlxsw_core_schedule_dw(&mlxsw_sp->router->neighs_update.dw, 0); 2964 mlxsw_core_schedule_dw(&mlxsw_sp->router->nexthop_probe_dw, 0); 2965 return 0; 2966 } 2967 2968 static void mlxsw_sp_neigh_fini(struct mlxsw_sp *mlxsw_sp) 2969 { 2970 cancel_delayed_work_sync(&mlxsw_sp->router->neighs_update.dw); 2971 cancel_delayed_work_sync(&mlxsw_sp->router->nexthop_probe_dw); 2972 rhashtable_destroy(&mlxsw_sp->router->neigh_ht); 2973 } 2974 2975 static void mlxsw_sp_neigh_rif_gone_sync(struct mlxsw_sp *mlxsw_sp, 2976 struct mlxsw_sp_rif *rif) 2977 { 2978 struct mlxsw_sp_neigh_entry *neigh_entry, *tmp; 2979 2980 list_for_each_entry_safe(neigh_entry, tmp, &rif->neigh_list, 2981 rif_list_node) { 2982 mlxsw_sp_neigh_entry_update(mlxsw_sp, neigh_entry, false); 2983 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry); 2984 } 2985 } 2986 2987 struct mlxsw_sp_neigh_rif_made_sync { 2988 struct mlxsw_sp *mlxsw_sp; 2989 struct mlxsw_sp_rif *rif; 2990 int err; 2991 }; 2992 2993 static void mlxsw_sp_neigh_rif_made_sync_each(struct neighbour *n, void *data) 2994 { 2995 struct mlxsw_sp_neigh_rif_made_sync *rms = data; 2996 int rc; 2997 2998 if (rms->err) 2999 return; 3000 if (n->dev != mlxsw_sp_rif_dev(rms->rif)) 3001 return; 3002 rc = mlxsw_sp_router_schedule_neigh_work(rms->mlxsw_sp->router, n); 3003 if (rc != NOTIFY_DONE) 3004 rms->err = -ENOMEM; 3005 } 3006 3007 static int mlxsw_sp_neigh_rif_made_sync(struct mlxsw_sp *mlxsw_sp, 3008 struct mlxsw_sp_rif *rif) 3009 { 3010 struct mlxsw_sp_neigh_rif_made_sync rms = { 3011 .mlxsw_sp = mlxsw_sp, 3012 .rif = rif, 3013 }; 3014 3015 neigh_for_each(&arp_tbl, mlxsw_sp_neigh_rif_made_sync_each, &rms); 3016 if (rms.err) 3017 goto err_arp; 3018 3019 #if IS_ENABLED(CONFIG_IPV6) 3020 neigh_for_each(&nd_tbl, mlxsw_sp_neigh_rif_made_sync_each, &rms); 3021 #endif 3022 if (rms.err) 3023 goto err_nd; 3024 3025 return 0; 3026 3027 err_nd: 3028 err_arp: 3029 mlxsw_sp_neigh_rif_gone_sync(mlxsw_sp, rif); 3030 return rms.err; 3031 } 3032 3033 enum mlxsw_sp_nexthop_type { 3034 MLXSW_SP_NEXTHOP_TYPE_ETH, 3035 MLXSW_SP_NEXTHOP_TYPE_IPIP, 3036 }; 3037 3038 enum mlxsw_sp_nexthop_action { 3039 /* Nexthop forwards packets to an egress RIF */ 3040 MLXSW_SP_NEXTHOP_ACTION_FORWARD, 3041 /* Nexthop discards packets */ 3042 MLXSW_SP_NEXTHOP_ACTION_DISCARD, 3043 /* Nexthop traps packets */ 3044 MLXSW_SP_NEXTHOP_ACTION_TRAP, 3045 }; 3046 3047 struct mlxsw_sp_nexthop_key { 3048 struct fib_nh *fib_nh; 3049 }; 3050 3051 struct mlxsw_sp_nexthop { 3052 struct list_head neigh_list_node; /* member of neigh entry list */ 3053 struct list_head crif_list_node; 3054 struct list_head router_list_node; 3055 struct mlxsw_sp_nexthop_group_info *nhgi; /* pointer back to the group 3056 * this nexthop belongs to 3057 */ 3058 struct rhash_head ht_node; 3059 struct neigh_table *neigh_tbl; 3060 struct mlxsw_sp_nexthop_key key; 3061 unsigned char gw_addr[sizeof(struct in6_addr)]; 3062 int ifindex; 3063 int nh_weight; 3064 int norm_nh_weight; 3065 int num_adj_entries; 3066 struct mlxsw_sp_crif *crif; 3067 u8 should_offload:1, /* set indicates this nexthop should be written 3068 * to the adjacency table. 3069 */ 3070 offloaded:1, /* set indicates this nexthop was written to the 3071 * adjacency table. 3072 */ 3073 update:1; /* set indicates this nexthop should be updated in the 3074 * adjacency table (f.e., its MAC changed). 3075 */ 3076 enum mlxsw_sp_nexthop_action action; 3077 enum mlxsw_sp_nexthop_type type; 3078 union { 3079 struct mlxsw_sp_neigh_entry *neigh_entry; 3080 struct mlxsw_sp_ipip_entry *ipip_entry; 3081 }; 3082 unsigned int counter_index; 3083 bool counter_valid; 3084 }; 3085 3086 static struct net_device * 3087 mlxsw_sp_nexthop_dev(const struct mlxsw_sp_nexthop *nh) 3088 { 3089 if (!nh->crif) 3090 return NULL; 3091 return nh->crif->key.dev; 3092 } 3093 3094 enum mlxsw_sp_nexthop_group_type { 3095 MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4, 3096 MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6, 3097 MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ, 3098 }; 3099 3100 struct mlxsw_sp_nexthop_group_info { 3101 struct mlxsw_sp_nexthop_group *nh_grp; 3102 u32 adj_index; 3103 u16 ecmp_size; 3104 u16 count; 3105 int sum_norm_weight; 3106 u8 adj_index_valid:1, 3107 gateway:1, /* routes using the group use a gateway */ 3108 is_resilient:1; 3109 struct list_head list; /* member in nh_res_grp_list */ 3110 struct mlxsw_sp_nexthop nexthops[] __counted_by(count); 3111 }; 3112 3113 static struct mlxsw_sp_rif * 3114 mlxsw_sp_nhgi_rif(const struct mlxsw_sp_nexthop_group_info *nhgi) 3115 { 3116 struct mlxsw_sp_crif *crif = nhgi->nexthops[0].crif; 3117 3118 if (!crif) 3119 return NULL; 3120 return crif->rif; 3121 } 3122 3123 struct mlxsw_sp_nexthop_group_vr_key { 3124 u16 vr_id; 3125 enum mlxsw_sp_l3proto proto; 3126 }; 3127 3128 struct mlxsw_sp_nexthop_group_vr_entry { 3129 struct list_head list; /* member in vr_list */ 3130 struct rhash_head ht_node; /* member in vr_ht */ 3131 refcount_t ref_count; 3132 struct mlxsw_sp_nexthop_group_vr_key key; 3133 }; 3134 3135 struct mlxsw_sp_nexthop_group { 3136 struct rhash_head ht_node; 3137 struct list_head fib_list; /* list of fib entries that use this group */ 3138 union { 3139 struct { 3140 struct fib_info *fi; 3141 } ipv4; 3142 struct { 3143 u32 id; 3144 } obj; 3145 }; 3146 struct mlxsw_sp_nexthop_group_info *nhgi; 3147 struct list_head vr_list; 3148 struct rhashtable vr_ht; 3149 enum mlxsw_sp_nexthop_group_type type; 3150 bool can_destroy; 3151 }; 3152 3153 void mlxsw_sp_nexthop_counter_alloc(struct mlxsw_sp *mlxsw_sp, 3154 struct mlxsw_sp_nexthop *nh) 3155 { 3156 struct devlink *devlink; 3157 3158 devlink = priv_to_devlink(mlxsw_sp->core); 3159 if (!devlink_dpipe_table_counter_enabled(devlink, 3160 MLXSW_SP_DPIPE_TABLE_NAME_ADJ)) 3161 return; 3162 3163 if (mlxsw_sp_flow_counter_alloc(mlxsw_sp, &nh->counter_index)) 3164 return; 3165 3166 nh->counter_valid = true; 3167 } 3168 3169 void mlxsw_sp_nexthop_counter_free(struct mlxsw_sp *mlxsw_sp, 3170 struct mlxsw_sp_nexthop *nh) 3171 { 3172 if (!nh->counter_valid) 3173 return; 3174 mlxsw_sp_flow_counter_free(mlxsw_sp, nh->counter_index); 3175 nh->counter_valid = false; 3176 } 3177 3178 int mlxsw_sp_nexthop_counter_get(struct mlxsw_sp *mlxsw_sp, 3179 struct mlxsw_sp_nexthop *nh, u64 *p_counter) 3180 { 3181 if (!nh->counter_valid) 3182 return -EINVAL; 3183 3184 return mlxsw_sp_flow_counter_get(mlxsw_sp, nh->counter_index, 3185 p_counter, NULL); 3186 } 3187 3188 struct mlxsw_sp_nexthop *mlxsw_sp_nexthop_next(struct mlxsw_sp_router *router, 3189 struct mlxsw_sp_nexthop *nh) 3190 { 3191 if (!nh) { 3192 if (list_empty(&router->nexthop_list)) 3193 return NULL; 3194 else 3195 return list_first_entry(&router->nexthop_list, 3196 typeof(*nh), router_list_node); 3197 } 3198 if (list_is_last(&nh->router_list_node, &router->nexthop_list)) 3199 return NULL; 3200 return list_next_entry(nh, router_list_node); 3201 } 3202 3203 bool mlxsw_sp_nexthop_is_forward(const struct mlxsw_sp_nexthop *nh) 3204 { 3205 return nh->offloaded && nh->action == MLXSW_SP_NEXTHOP_ACTION_FORWARD; 3206 } 3207 3208 unsigned char *mlxsw_sp_nexthop_ha(struct mlxsw_sp_nexthop *nh) 3209 { 3210 if (nh->type != MLXSW_SP_NEXTHOP_TYPE_ETH || 3211 !mlxsw_sp_nexthop_is_forward(nh)) 3212 return NULL; 3213 return nh->neigh_entry->ha; 3214 } 3215 3216 int mlxsw_sp_nexthop_indexes(struct mlxsw_sp_nexthop *nh, u32 *p_adj_index, 3217 u32 *p_adj_size, u32 *p_adj_hash_index) 3218 { 3219 struct mlxsw_sp_nexthop_group_info *nhgi = nh->nhgi; 3220 u32 adj_hash_index = 0; 3221 int i; 3222 3223 if (!nh->offloaded || !nhgi->adj_index_valid) 3224 return -EINVAL; 3225 3226 *p_adj_index = nhgi->adj_index; 3227 *p_adj_size = nhgi->ecmp_size; 3228 3229 for (i = 0; i < nhgi->count; i++) { 3230 struct mlxsw_sp_nexthop *nh_iter = &nhgi->nexthops[i]; 3231 3232 if (nh_iter == nh) 3233 break; 3234 if (nh_iter->offloaded) 3235 adj_hash_index += nh_iter->num_adj_entries; 3236 } 3237 3238 *p_adj_hash_index = adj_hash_index; 3239 return 0; 3240 } 3241 3242 struct mlxsw_sp_rif *mlxsw_sp_nexthop_rif(struct mlxsw_sp_nexthop *nh) 3243 { 3244 if (WARN_ON(!nh->crif)) 3245 return NULL; 3246 return nh->crif->rif; 3247 } 3248 3249 bool mlxsw_sp_nexthop_group_has_ipip(struct mlxsw_sp_nexthop *nh) 3250 { 3251 struct mlxsw_sp_nexthop_group_info *nhgi = nh->nhgi; 3252 int i; 3253 3254 for (i = 0; i < nhgi->count; i++) { 3255 struct mlxsw_sp_nexthop *nh_iter = &nhgi->nexthops[i]; 3256 3257 if (nh_iter->type == MLXSW_SP_NEXTHOP_TYPE_IPIP) 3258 return true; 3259 } 3260 return false; 3261 } 3262 3263 static const struct rhashtable_params mlxsw_sp_nexthop_group_vr_ht_params = { 3264 .key_offset = offsetof(struct mlxsw_sp_nexthop_group_vr_entry, key), 3265 .head_offset = offsetof(struct mlxsw_sp_nexthop_group_vr_entry, ht_node), 3266 .key_len = sizeof(struct mlxsw_sp_nexthop_group_vr_key), 3267 .automatic_shrinking = true, 3268 }; 3269 3270 static struct mlxsw_sp_nexthop_group_vr_entry * 3271 mlxsw_sp_nexthop_group_vr_entry_lookup(struct mlxsw_sp_nexthop_group *nh_grp, 3272 const struct mlxsw_sp_fib *fib) 3273 { 3274 struct mlxsw_sp_nexthop_group_vr_key key; 3275 3276 memset(&key, 0, sizeof(key)); 3277 key.vr_id = fib->vr->id; 3278 key.proto = fib->proto; 3279 return rhashtable_lookup_fast(&nh_grp->vr_ht, &key, 3280 mlxsw_sp_nexthop_group_vr_ht_params); 3281 } 3282 3283 static int 3284 mlxsw_sp_nexthop_group_vr_entry_create(struct mlxsw_sp_nexthop_group *nh_grp, 3285 const struct mlxsw_sp_fib *fib) 3286 { 3287 struct mlxsw_sp_nexthop_group_vr_entry *vr_entry; 3288 int err; 3289 3290 vr_entry = kzalloc(sizeof(*vr_entry), GFP_KERNEL); 3291 if (!vr_entry) 3292 return -ENOMEM; 3293 3294 vr_entry->key.vr_id = fib->vr->id; 3295 vr_entry->key.proto = fib->proto; 3296 refcount_set(&vr_entry->ref_count, 1); 3297 3298 err = rhashtable_insert_fast(&nh_grp->vr_ht, &vr_entry->ht_node, 3299 mlxsw_sp_nexthop_group_vr_ht_params); 3300 if (err) 3301 goto err_hashtable_insert; 3302 3303 list_add(&vr_entry->list, &nh_grp->vr_list); 3304 3305 return 0; 3306 3307 err_hashtable_insert: 3308 kfree(vr_entry); 3309 return err; 3310 } 3311 3312 static void 3313 mlxsw_sp_nexthop_group_vr_entry_destroy(struct mlxsw_sp_nexthop_group *nh_grp, 3314 struct mlxsw_sp_nexthop_group_vr_entry *vr_entry) 3315 { 3316 list_del(&vr_entry->list); 3317 rhashtable_remove_fast(&nh_grp->vr_ht, &vr_entry->ht_node, 3318 mlxsw_sp_nexthop_group_vr_ht_params); 3319 kfree(vr_entry); 3320 } 3321 3322 static int 3323 mlxsw_sp_nexthop_group_vr_link(struct mlxsw_sp_nexthop_group *nh_grp, 3324 const struct mlxsw_sp_fib *fib) 3325 { 3326 struct mlxsw_sp_nexthop_group_vr_entry *vr_entry; 3327 3328 vr_entry = mlxsw_sp_nexthop_group_vr_entry_lookup(nh_grp, fib); 3329 if (vr_entry) { 3330 refcount_inc(&vr_entry->ref_count); 3331 return 0; 3332 } 3333 3334 return mlxsw_sp_nexthop_group_vr_entry_create(nh_grp, fib); 3335 } 3336 3337 static void 3338 mlxsw_sp_nexthop_group_vr_unlink(struct mlxsw_sp_nexthop_group *nh_grp, 3339 const struct mlxsw_sp_fib *fib) 3340 { 3341 struct mlxsw_sp_nexthop_group_vr_entry *vr_entry; 3342 3343 vr_entry = mlxsw_sp_nexthop_group_vr_entry_lookup(nh_grp, fib); 3344 if (WARN_ON_ONCE(!vr_entry)) 3345 return; 3346 3347 if (!refcount_dec_and_test(&vr_entry->ref_count)) 3348 return; 3349 3350 mlxsw_sp_nexthop_group_vr_entry_destroy(nh_grp, vr_entry); 3351 } 3352 3353 struct mlxsw_sp_nexthop_group_cmp_arg { 3354 enum mlxsw_sp_nexthop_group_type type; 3355 union { 3356 struct fib_info *fi; 3357 struct mlxsw_sp_fib6_entry *fib6_entry; 3358 u32 id; 3359 }; 3360 }; 3361 3362 static bool 3363 mlxsw_sp_nexthop6_group_has_nexthop(const struct mlxsw_sp_nexthop_group *nh_grp, 3364 const struct in6_addr *gw, int ifindex, 3365 int weight) 3366 { 3367 int i; 3368 3369 for (i = 0; i < nh_grp->nhgi->count; i++) { 3370 const struct mlxsw_sp_nexthop *nh; 3371 3372 nh = &nh_grp->nhgi->nexthops[i]; 3373 if (nh->ifindex == ifindex && nh->nh_weight == weight && 3374 ipv6_addr_equal(gw, (struct in6_addr *) nh->gw_addr)) 3375 return true; 3376 } 3377 3378 return false; 3379 } 3380 3381 static bool 3382 mlxsw_sp_nexthop6_group_cmp(const struct mlxsw_sp_nexthop_group *nh_grp, 3383 const struct mlxsw_sp_fib6_entry *fib6_entry) 3384 { 3385 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 3386 3387 if (nh_grp->nhgi->count != fib6_entry->nrt6) 3388 return false; 3389 3390 list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) { 3391 struct fib6_nh *fib6_nh = mlxsw_sp_rt6->rt->fib6_nh; 3392 struct in6_addr *gw; 3393 int ifindex, weight; 3394 3395 ifindex = fib6_nh->fib_nh_dev->ifindex; 3396 weight = fib6_nh->fib_nh_weight; 3397 gw = &fib6_nh->fib_nh_gw6; 3398 if (!mlxsw_sp_nexthop6_group_has_nexthop(nh_grp, gw, ifindex, 3399 weight)) 3400 return false; 3401 } 3402 3403 return true; 3404 } 3405 3406 static int 3407 mlxsw_sp_nexthop_group_cmp(struct rhashtable_compare_arg *arg, const void *ptr) 3408 { 3409 const struct mlxsw_sp_nexthop_group_cmp_arg *cmp_arg = arg->key; 3410 const struct mlxsw_sp_nexthop_group *nh_grp = ptr; 3411 3412 if (nh_grp->type != cmp_arg->type) 3413 return 1; 3414 3415 switch (cmp_arg->type) { 3416 case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4: 3417 return cmp_arg->fi != nh_grp->ipv4.fi; 3418 case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6: 3419 return !mlxsw_sp_nexthop6_group_cmp(nh_grp, 3420 cmp_arg->fib6_entry); 3421 case MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ: 3422 return cmp_arg->id != nh_grp->obj.id; 3423 default: 3424 WARN_ON(1); 3425 return 1; 3426 } 3427 } 3428 3429 static u32 mlxsw_sp_nexthop_group_hash_obj(const void *data, u32 len, u32 seed) 3430 { 3431 const struct mlxsw_sp_nexthop_group *nh_grp = data; 3432 const struct mlxsw_sp_nexthop *nh; 3433 struct fib_info *fi; 3434 unsigned int val; 3435 int i; 3436 3437 switch (nh_grp->type) { 3438 case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4: 3439 fi = nh_grp->ipv4.fi; 3440 return jhash(&fi, sizeof(fi), seed); 3441 case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6: 3442 val = nh_grp->nhgi->count; 3443 for (i = 0; i < nh_grp->nhgi->count; i++) { 3444 nh = &nh_grp->nhgi->nexthops[i]; 3445 val ^= jhash(&nh->ifindex, sizeof(nh->ifindex), seed); 3446 val ^= jhash(&nh->gw_addr, sizeof(nh->gw_addr), seed); 3447 } 3448 return jhash(&val, sizeof(val), seed); 3449 case MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ: 3450 return jhash(&nh_grp->obj.id, sizeof(nh_grp->obj.id), seed); 3451 default: 3452 WARN_ON(1); 3453 return 0; 3454 } 3455 } 3456 3457 static u32 3458 mlxsw_sp_nexthop6_group_hash(struct mlxsw_sp_fib6_entry *fib6_entry, u32 seed) 3459 { 3460 unsigned int val = fib6_entry->nrt6; 3461 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 3462 3463 list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) { 3464 struct fib6_nh *fib6_nh = mlxsw_sp_rt6->rt->fib6_nh; 3465 struct net_device *dev = fib6_nh->fib_nh_dev; 3466 struct in6_addr *gw = &fib6_nh->fib_nh_gw6; 3467 3468 val ^= jhash(&dev->ifindex, sizeof(dev->ifindex), seed); 3469 val ^= jhash(gw, sizeof(*gw), seed); 3470 } 3471 3472 return jhash(&val, sizeof(val), seed); 3473 } 3474 3475 static u32 3476 mlxsw_sp_nexthop_group_hash(const void *data, u32 len, u32 seed) 3477 { 3478 const struct mlxsw_sp_nexthop_group_cmp_arg *cmp_arg = data; 3479 3480 switch (cmp_arg->type) { 3481 case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4: 3482 return jhash(&cmp_arg->fi, sizeof(cmp_arg->fi), seed); 3483 case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6: 3484 return mlxsw_sp_nexthop6_group_hash(cmp_arg->fib6_entry, seed); 3485 case MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ: 3486 return jhash(&cmp_arg->id, sizeof(cmp_arg->id), seed); 3487 default: 3488 WARN_ON(1); 3489 return 0; 3490 } 3491 } 3492 3493 static const struct rhashtable_params mlxsw_sp_nexthop_group_ht_params = { 3494 .head_offset = offsetof(struct mlxsw_sp_nexthop_group, ht_node), 3495 .hashfn = mlxsw_sp_nexthop_group_hash, 3496 .obj_hashfn = mlxsw_sp_nexthop_group_hash_obj, 3497 .obj_cmpfn = mlxsw_sp_nexthop_group_cmp, 3498 }; 3499 3500 static int mlxsw_sp_nexthop_group_insert(struct mlxsw_sp *mlxsw_sp, 3501 struct mlxsw_sp_nexthop_group *nh_grp) 3502 { 3503 if (nh_grp->type == MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6 && 3504 !nh_grp->nhgi->gateway) 3505 return 0; 3506 3507 return rhashtable_insert_fast(&mlxsw_sp->router->nexthop_group_ht, 3508 &nh_grp->ht_node, 3509 mlxsw_sp_nexthop_group_ht_params); 3510 } 3511 3512 static void mlxsw_sp_nexthop_group_remove(struct mlxsw_sp *mlxsw_sp, 3513 struct mlxsw_sp_nexthop_group *nh_grp) 3514 { 3515 if (nh_grp->type == MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6 && 3516 !nh_grp->nhgi->gateway) 3517 return; 3518 3519 rhashtable_remove_fast(&mlxsw_sp->router->nexthop_group_ht, 3520 &nh_grp->ht_node, 3521 mlxsw_sp_nexthop_group_ht_params); 3522 } 3523 3524 static struct mlxsw_sp_nexthop_group * 3525 mlxsw_sp_nexthop4_group_lookup(struct mlxsw_sp *mlxsw_sp, 3526 struct fib_info *fi) 3527 { 3528 struct mlxsw_sp_nexthop_group_cmp_arg cmp_arg; 3529 3530 cmp_arg.type = MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4; 3531 cmp_arg.fi = fi; 3532 return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_group_ht, 3533 &cmp_arg, 3534 mlxsw_sp_nexthop_group_ht_params); 3535 } 3536 3537 static struct mlxsw_sp_nexthop_group * 3538 mlxsw_sp_nexthop6_group_lookup(struct mlxsw_sp *mlxsw_sp, 3539 struct mlxsw_sp_fib6_entry *fib6_entry) 3540 { 3541 struct mlxsw_sp_nexthop_group_cmp_arg cmp_arg; 3542 3543 cmp_arg.type = MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6; 3544 cmp_arg.fib6_entry = fib6_entry; 3545 return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_group_ht, 3546 &cmp_arg, 3547 mlxsw_sp_nexthop_group_ht_params); 3548 } 3549 3550 static const struct rhashtable_params mlxsw_sp_nexthop_ht_params = { 3551 .key_offset = offsetof(struct mlxsw_sp_nexthop, key), 3552 .head_offset = offsetof(struct mlxsw_sp_nexthop, ht_node), 3553 .key_len = sizeof(struct mlxsw_sp_nexthop_key), 3554 }; 3555 3556 static int mlxsw_sp_nexthop_insert(struct mlxsw_sp *mlxsw_sp, 3557 struct mlxsw_sp_nexthop *nh) 3558 { 3559 return rhashtable_insert_fast(&mlxsw_sp->router->nexthop_ht, 3560 &nh->ht_node, mlxsw_sp_nexthop_ht_params); 3561 } 3562 3563 static void mlxsw_sp_nexthop_remove(struct mlxsw_sp *mlxsw_sp, 3564 struct mlxsw_sp_nexthop *nh) 3565 { 3566 rhashtable_remove_fast(&mlxsw_sp->router->nexthop_ht, &nh->ht_node, 3567 mlxsw_sp_nexthop_ht_params); 3568 } 3569 3570 static struct mlxsw_sp_nexthop * 3571 mlxsw_sp_nexthop_lookup(struct mlxsw_sp *mlxsw_sp, 3572 struct mlxsw_sp_nexthop_key key) 3573 { 3574 return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_ht, &key, 3575 mlxsw_sp_nexthop_ht_params); 3576 } 3577 3578 static int mlxsw_sp_adj_index_mass_update_vr(struct mlxsw_sp *mlxsw_sp, 3579 enum mlxsw_sp_l3proto proto, 3580 u16 vr_id, 3581 u32 adj_index, u16 ecmp_size, 3582 u32 new_adj_index, 3583 u16 new_ecmp_size) 3584 { 3585 char raleu_pl[MLXSW_REG_RALEU_LEN]; 3586 3587 mlxsw_reg_raleu_pack(raleu_pl, 3588 (enum mlxsw_reg_ralxx_protocol) proto, vr_id, 3589 adj_index, ecmp_size, new_adj_index, 3590 new_ecmp_size); 3591 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raleu), raleu_pl); 3592 } 3593 3594 static int mlxsw_sp_adj_index_mass_update(struct mlxsw_sp *mlxsw_sp, 3595 struct mlxsw_sp_nexthop_group *nh_grp, 3596 u32 old_adj_index, u16 old_ecmp_size) 3597 { 3598 struct mlxsw_sp_nexthop_group_info *nhgi = nh_grp->nhgi; 3599 struct mlxsw_sp_nexthop_group_vr_entry *vr_entry; 3600 int err; 3601 3602 list_for_each_entry(vr_entry, &nh_grp->vr_list, list) { 3603 err = mlxsw_sp_adj_index_mass_update_vr(mlxsw_sp, 3604 vr_entry->key.proto, 3605 vr_entry->key.vr_id, 3606 old_adj_index, 3607 old_ecmp_size, 3608 nhgi->adj_index, 3609 nhgi->ecmp_size); 3610 if (err) 3611 goto err_mass_update_vr; 3612 } 3613 return 0; 3614 3615 err_mass_update_vr: 3616 list_for_each_entry_continue_reverse(vr_entry, &nh_grp->vr_list, list) 3617 mlxsw_sp_adj_index_mass_update_vr(mlxsw_sp, vr_entry->key.proto, 3618 vr_entry->key.vr_id, 3619 nhgi->adj_index, 3620 nhgi->ecmp_size, 3621 old_adj_index, old_ecmp_size); 3622 return err; 3623 } 3624 3625 static int __mlxsw_sp_nexthop_eth_update(struct mlxsw_sp *mlxsw_sp, 3626 u32 adj_index, 3627 struct mlxsw_sp_nexthop *nh, 3628 bool force, char *ratr_pl) 3629 { 3630 struct mlxsw_sp_neigh_entry *neigh_entry = nh->neigh_entry; 3631 struct mlxsw_sp_rif *rif = mlxsw_sp_nexthop_rif(nh); 3632 enum mlxsw_reg_ratr_op op; 3633 u16 rif_index; 3634 3635 rif_index = rif ? rif->rif_index : 3636 mlxsw_sp->router->lb_crif->rif->rif_index; 3637 op = force ? MLXSW_REG_RATR_OP_WRITE_WRITE_ENTRY : 3638 MLXSW_REG_RATR_OP_WRITE_WRITE_ENTRY_ON_ACTIVITY; 3639 mlxsw_reg_ratr_pack(ratr_pl, op, true, MLXSW_REG_RATR_TYPE_ETHERNET, 3640 adj_index, rif_index); 3641 switch (nh->action) { 3642 case MLXSW_SP_NEXTHOP_ACTION_FORWARD: 3643 mlxsw_reg_ratr_eth_entry_pack(ratr_pl, neigh_entry->ha); 3644 break; 3645 case MLXSW_SP_NEXTHOP_ACTION_DISCARD: 3646 mlxsw_reg_ratr_trap_action_set(ratr_pl, 3647 MLXSW_REG_RATR_TRAP_ACTION_DISCARD_ERRORS); 3648 break; 3649 case MLXSW_SP_NEXTHOP_ACTION_TRAP: 3650 mlxsw_reg_ratr_trap_action_set(ratr_pl, 3651 MLXSW_REG_RATR_TRAP_ACTION_TRAP); 3652 mlxsw_reg_ratr_trap_id_set(ratr_pl, MLXSW_TRAP_ID_RTR_EGRESS0); 3653 break; 3654 default: 3655 WARN_ON_ONCE(1); 3656 return -EINVAL; 3657 } 3658 if (nh->counter_valid) 3659 mlxsw_reg_ratr_counter_pack(ratr_pl, nh->counter_index, true); 3660 else 3661 mlxsw_reg_ratr_counter_pack(ratr_pl, 0, false); 3662 3663 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ratr), ratr_pl); 3664 } 3665 3666 int mlxsw_sp_nexthop_eth_update(struct mlxsw_sp *mlxsw_sp, u32 adj_index, 3667 struct mlxsw_sp_nexthop *nh, bool force, 3668 char *ratr_pl) 3669 { 3670 int i; 3671 3672 for (i = 0; i < nh->num_adj_entries; i++) { 3673 int err; 3674 3675 err = __mlxsw_sp_nexthop_eth_update(mlxsw_sp, adj_index + i, 3676 nh, force, ratr_pl); 3677 if (err) 3678 return err; 3679 } 3680 3681 return 0; 3682 } 3683 3684 static int __mlxsw_sp_nexthop_ipip_update(struct mlxsw_sp *mlxsw_sp, 3685 u32 adj_index, 3686 struct mlxsw_sp_nexthop *nh, 3687 bool force, char *ratr_pl) 3688 { 3689 const struct mlxsw_sp_ipip_ops *ipip_ops; 3690 3691 ipip_ops = mlxsw_sp->router->ipip_ops_arr[nh->ipip_entry->ipipt]; 3692 return ipip_ops->nexthop_update(mlxsw_sp, adj_index, nh->ipip_entry, 3693 force, ratr_pl); 3694 } 3695 3696 static int mlxsw_sp_nexthop_ipip_update(struct mlxsw_sp *mlxsw_sp, 3697 u32 adj_index, 3698 struct mlxsw_sp_nexthop *nh, bool force, 3699 char *ratr_pl) 3700 { 3701 int i; 3702 3703 for (i = 0; i < nh->num_adj_entries; i++) { 3704 int err; 3705 3706 err = __mlxsw_sp_nexthop_ipip_update(mlxsw_sp, adj_index + i, 3707 nh, force, ratr_pl); 3708 if (err) 3709 return err; 3710 } 3711 3712 return 0; 3713 } 3714 3715 static int mlxsw_sp_nexthop_update(struct mlxsw_sp *mlxsw_sp, u32 adj_index, 3716 struct mlxsw_sp_nexthop *nh, bool force, 3717 char *ratr_pl) 3718 { 3719 /* When action is discard or trap, the nexthop must be 3720 * programmed as an Ethernet nexthop. 3721 */ 3722 if (nh->type == MLXSW_SP_NEXTHOP_TYPE_ETH || 3723 nh->action == MLXSW_SP_NEXTHOP_ACTION_DISCARD || 3724 nh->action == MLXSW_SP_NEXTHOP_ACTION_TRAP) 3725 return mlxsw_sp_nexthop_eth_update(mlxsw_sp, adj_index, nh, 3726 force, ratr_pl); 3727 else 3728 return mlxsw_sp_nexthop_ipip_update(mlxsw_sp, adj_index, nh, 3729 force, ratr_pl); 3730 } 3731 3732 static int 3733 mlxsw_sp_nexthop_group_update(struct mlxsw_sp *mlxsw_sp, 3734 struct mlxsw_sp_nexthop_group_info *nhgi, 3735 bool reallocate) 3736 { 3737 char ratr_pl[MLXSW_REG_RATR_LEN]; 3738 u32 adj_index = nhgi->adj_index; /* base */ 3739 struct mlxsw_sp_nexthop *nh; 3740 int i; 3741 3742 for (i = 0; i < nhgi->count; i++) { 3743 nh = &nhgi->nexthops[i]; 3744 3745 if (!nh->should_offload) { 3746 nh->offloaded = 0; 3747 continue; 3748 } 3749 3750 if (nh->update || reallocate) { 3751 int err = 0; 3752 3753 err = mlxsw_sp_nexthop_update(mlxsw_sp, adj_index, nh, 3754 true, ratr_pl); 3755 if (err) 3756 return err; 3757 nh->update = 0; 3758 nh->offloaded = 1; 3759 } 3760 adj_index += nh->num_adj_entries; 3761 } 3762 return 0; 3763 } 3764 3765 static int 3766 mlxsw_sp_nexthop_fib_entries_update(struct mlxsw_sp *mlxsw_sp, 3767 struct mlxsw_sp_nexthop_group *nh_grp) 3768 { 3769 struct mlxsw_sp_fib_entry *fib_entry; 3770 int err; 3771 3772 list_for_each_entry(fib_entry, &nh_grp->fib_list, nexthop_group_node) { 3773 err = mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry); 3774 if (err) 3775 return err; 3776 } 3777 return 0; 3778 } 3779 3780 struct mlxsw_sp_adj_grp_size_range { 3781 u16 start; /* Inclusive */ 3782 u16 end; /* Inclusive */ 3783 }; 3784 3785 /* Ordered by range start value */ 3786 static const struct mlxsw_sp_adj_grp_size_range 3787 mlxsw_sp1_adj_grp_size_ranges[] = { 3788 { .start = 1, .end = 64 }, 3789 { .start = 512, .end = 512 }, 3790 { .start = 1024, .end = 1024 }, 3791 { .start = 2048, .end = 2048 }, 3792 { .start = 4096, .end = 4096 }, 3793 }; 3794 3795 /* Ordered by range start value */ 3796 static const struct mlxsw_sp_adj_grp_size_range 3797 mlxsw_sp2_adj_grp_size_ranges[] = { 3798 { .start = 1, .end = 128 }, 3799 { .start = 256, .end = 256 }, 3800 { .start = 512, .end = 512 }, 3801 { .start = 1024, .end = 1024 }, 3802 { .start = 2048, .end = 2048 }, 3803 { .start = 4096, .end = 4096 }, 3804 }; 3805 3806 static void mlxsw_sp_adj_grp_size_round_up(const struct mlxsw_sp *mlxsw_sp, 3807 u16 *p_adj_grp_size) 3808 { 3809 int i; 3810 3811 for (i = 0; i < mlxsw_sp->router->adj_grp_size_ranges_count; i++) { 3812 const struct mlxsw_sp_adj_grp_size_range *size_range; 3813 3814 size_range = &mlxsw_sp->router->adj_grp_size_ranges[i]; 3815 3816 if (*p_adj_grp_size >= size_range->start && 3817 *p_adj_grp_size <= size_range->end) 3818 return; 3819 3820 if (*p_adj_grp_size <= size_range->end) { 3821 *p_adj_grp_size = size_range->end; 3822 return; 3823 } 3824 } 3825 } 3826 3827 static void mlxsw_sp_adj_grp_size_round_down(const struct mlxsw_sp *mlxsw_sp, 3828 u16 *p_adj_grp_size, 3829 unsigned int alloc_size) 3830 { 3831 int i; 3832 3833 for (i = mlxsw_sp->router->adj_grp_size_ranges_count - 1; i >= 0; i--) { 3834 const struct mlxsw_sp_adj_grp_size_range *size_range; 3835 3836 size_range = &mlxsw_sp->router->adj_grp_size_ranges[i]; 3837 3838 if (alloc_size >= size_range->end) { 3839 *p_adj_grp_size = size_range->end; 3840 return; 3841 } 3842 } 3843 } 3844 3845 static int mlxsw_sp_fix_adj_grp_size(struct mlxsw_sp *mlxsw_sp, 3846 u16 *p_adj_grp_size) 3847 { 3848 unsigned int alloc_size; 3849 int err; 3850 3851 /* Round up the requested group size to the next size supported 3852 * by the device and make sure the request can be satisfied. 3853 */ 3854 mlxsw_sp_adj_grp_size_round_up(mlxsw_sp, p_adj_grp_size); 3855 err = mlxsw_sp_kvdl_alloc_count_query(mlxsw_sp, 3856 MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 3857 *p_adj_grp_size, &alloc_size); 3858 if (err) 3859 return err; 3860 /* It is possible the allocation results in more allocated 3861 * entries than requested. Try to use as much of them as 3862 * possible. 3863 */ 3864 mlxsw_sp_adj_grp_size_round_down(mlxsw_sp, p_adj_grp_size, alloc_size); 3865 3866 return 0; 3867 } 3868 3869 static void 3870 mlxsw_sp_nexthop_group_normalize(struct mlxsw_sp_nexthop_group_info *nhgi) 3871 { 3872 int i, g = 0, sum_norm_weight = 0; 3873 struct mlxsw_sp_nexthop *nh; 3874 3875 for (i = 0; i < nhgi->count; i++) { 3876 nh = &nhgi->nexthops[i]; 3877 3878 if (!nh->should_offload) 3879 continue; 3880 if (g > 0) 3881 g = gcd(nh->nh_weight, g); 3882 else 3883 g = nh->nh_weight; 3884 } 3885 3886 for (i = 0; i < nhgi->count; i++) { 3887 nh = &nhgi->nexthops[i]; 3888 3889 if (!nh->should_offload) 3890 continue; 3891 nh->norm_nh_weight = nh->nh_weight / g; 3892 sum_norm_weight += nh->norm_nh_weight; 3893 } 3894 3895 nhgi->sum_norm_weight = sum_norm_weight; 3896 } 3897 3898 static void 3899 mlxsw_sp_nexthop_group_rebalance(struct mlxsw_sp_nexthop_group_info *nhgi) 3900 { 3901 int i, weight = 0, lower_bound = 0; 3902 int total = nhgi->sum_norm_weight; 3903 u16 ecmp_size = nhgi->ecmp_size; 3904 3905 for (i = 0; i < nhgi->count; i++) { 3906 struct mlxsw_sp_nexthop *nh = &nhgi->nexthops[i]; 3907 int upper_bound; 3908 3909 if (!nh->should_offload) 3910 continue; 3911 weight += nh->norm_nh_weight; 3912 upper_bound = DIV_ROUND_CLOSEST(ecmp_size * weight, total); 3913 nh->num_adj_entries = upper_bound - lower_bound; 3914 lower_bound = upper_bound; 3915 } 3916 } 3917 3918 static struct mlxsw_sp_nexthop * 3919 mlxsw_sp_rt6_nexthop(struct mlxsw_sp_nexthop_group *nh_grp, 3920 const struct mlxsw_sp_rt6 *mlxsw_sp_rt6); 3921 3922 static void 3923 mlxsw_sp_nexthop4_group_offload_refresh(struct mlxsw_sp *mlxsw_sp, 3924 struct mlxsw_sp_nexthop_group *nh_grp) 3925 { 3926 int i; 3927 3928 for (i = 0; i < nh_grp->nhgi->count; i++) { 3929 struct mlxsw_sp_nexthop *nh = &nh_grp->nhgi->nexthops[i]; 3930 3931 if (nh->offloaded) 3932 nh->key.fib_nh->fib_nh_flags |= RTNH_F_OFFLOAD; 3933 else 3934 nh->key.fib_nh->fib_nh_flags &= ~RTNH_F_OFFLOAD; 3935 } 3936 } 3937 3938 static void 3939 __mlxsw_sp_nexthop6_group_offload_refresh(struct mlxsw_sp_nexthop_group *nh_grp, 3940 struct mlxsw_sp_fib6_entry *fib6_entry) 3941 { 3942 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 3943 3944 list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) { 3945 struct fib6_nh *fib6_nh = mlxsw_sp_rt6->rt->fib6_nh; 3946 struct mlxsw_sp_nexthop *nh; 3947 3948 nh = mlxsw_sp_rt6_nexthop(nh_grp, mlxsw_sp_rt6); 3949 if (nh && nh->offloaded) 3950 fib6_nh->fib_nh_flags |= RTNH_F_OFFLOAD; 3951 else 3952 fib6_nh->fib_nh_flags &= ~RTNH_F_OFFLOAD; 3953 } 3954 } 3955 3956 static void 3957 mlxsw_sp_nexthop6_group_offload_refresh(struct mlxsw_sp *mlxsw_sp, 3958 struct mlxsw_sp_nexthop_group *nh_grp) 3959 { 3960 struct mlxsw_sp_fib6_entry *fib6_entry; 3961 3962 /* Unfortunately, in IPv6 the route and the nexthop are described by 3963 * the same struct, so we need to iterate over all the routes using the 3964 * nexthop group and set / clear the offload indication for them. 3965 */ 3966 list_for_each_entry(fib6_entry, &nh_grp->fib_list, 3967 common.nexthop_group_node) 3968 __mlxsw_sp_nexthop6_group_offload_refresh(nh_grp, fib6_entry); 3969 } 3970 3971 static void 3972 mlxsw_sp_nexthop_bucket_offload_refresh(struct mlxsw_sp *mlxsw_sp, 3973 const struct mlxsw_sp_nexthop *nh, 3974 u16 bucket_index) 3975 { 3976 struct mlxsw_sp_nexthop_group *nh_grp = nh->nhgi->nh_grp; 3977 bool offload = false, trap = false; 3978 3979 if (nh->offloaded) { 3980 if (nh->action == MLXSW_SP_NEXTHOP_ACTION_TRAP) 3981 trap = true; 3982 else 3983 offload = true; 3984 } 3985 nexthop_bucket_set_hw_flags(mlxsw_sp_net(mlxsw_sp), nh_grp->obj.id, 3986 bucket_index, offload, trap); 3987 } 3988 3989 static void 3990 mlxsw_sp_nexthop_obj_group_offload_refresh(struct mlxsw_sp *mlxsw_sp, 3991 struct mlxsw_sp_nexthop_group *nh_grp) 3992 { 3993 int i; 3994 3995 /* Do not update the flags if the nexthop group is being destroyed 3996 * since: 3997 * 1. The nexthop objects is being deleted, in which case the flags are 3998 * irrelevant. 3999 * 2. The nexthop group was replaced by a newer group, in which case 4000 * the flags of the nexthop object were already updated based on the 4001 * new group. 4002 */ 4003 if (nh_grp->can_destroy) 4004 return; 4005 4006 nexthop_set_hw_flags(mlxsw_sp_net(mlxsw_sp), nh_grp->obj.id, 4007 nh_grp->nhgi->adj_index_valid, false); 4008 4009 /* Update flags of individual nexthop buckets in case of a resilient 4010 * nexthop group. 4011 */ 4012 if (!nh_grp->nhgi->is_resilient) 4013 return; 4014 4015 for (i = 0; i < nh_grp->nhgi->count; i++) { 4016 struct mlxsw_sp_nexthop *nh = &nh_grp->nhgi->nexthops[i]; 4017 4018 mlxsw_sp_nexthop_bucket_offload_refresh(mlxsw_sp, nh, i); 4019 } 4020 } 4021 4022 static void 4023 mlxsw_sp_nexthop_group_offload_refresh(struct mlxsw_sp *mlxsw_sp, 4024 struct mlxsw_sp_nexthop_group *nh_grp) 4025 { 4026 switch (nh_grp->type) { 4027 case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4: 4028 mlxsw_sp_nexthop4_group_offload_refresh(mlxsw_sp, nh_grp); 4029 break; 4030 case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6: 4031 mlxsw_sp_nexthop6_group_offload_refresh(mlxsw_sp, nh_grp); 4032 break; 4033 case MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ: 4034 mlxsw_sp_nexthop_obj_group_offload_refresh(mlxsw_sp, nh_grp); 4035 break; 4036 } 4037 } 4038 4039 static int 4040 mlxsw_sp_nexthop_group_refresh(struct mlxsw_sp *mlxsw_sp, 4041 struct mlxsw_sp_nexthop_group *nh_grp) 4042 { 4043 struct mlxsw_sp_nexthop_group_info *nhgi = nh_grp->nhgi; 4044 u16 ecmp_size, old_ecmp_size; 4045 struct mlxsw_sp_nexthop *nh; 4046 bool offload_change = false; 4047 u32 adj_index; 4048 bool old_adj_index_valid; 4049 u32 old_adj_index; 4050 int i, err2, err; 4051 4052 if (!nhgi->gateway) 4053 return mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp); 4054 4055 for (i = 0; i < nhgi->count; i++) { 4056 nh = &nhgi->nexthops[i]; 4057 4058 if (nh->should_offload != nh->offloaded) { 4059 offload_change = true; 4060 if (nh->should_offload) 4061 nh->update = 1; 4062 } 4063 } 4064 if (!offload_change) { 4065 /* Nothing was added or removed, so no need to reallocate. Just 4066 * update MAC on existing adjacency indexes. 4067 */ 4068 err = mlxsw_sp_nexthop_group_update(mlxsw_sp, nhgi, false); 4069 if (err) { 4070 dev_warn(mlxsw_sp->bus_info->dev, "Failed to update neigh MAC in adjacency table.\n"); 4071 goto set_trap; 4072 } 4073 /* Flags of individual nexthop buckets might need to be 4074 * updated. 4075 */ 4076 mlxsw_sp_nexthop_group_offload_refresh(mlxsw_sp, nh_grp); 4077 return 0; 4078 } 4079 mlxsw_sp_nexthop_group_normalize(nhgi); 4080 if (!nhgi->sum_norm_weight) { 4081 /* No neigh of this group is connected so we just set 4082 * the trap and let everthing flow through kernel. 4083 */ 4084 err = 0; 4085 goto set_trap; 4086 } 4087 4088 ecmp_size = nhgi->sum_norm_weight; 4089 err = mlxsw_sp_fix_adj_grp_size(mlxsw_sp, &ecmp_size); 4090 if (err) 4091 /* No valid allocation size available. */ 4092 goto set_trap; 4093 4094 err = mlxsw_sp_kvdl_alloc(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 4095 ecmp_size, &adj_index); 4096 if (err) { 4097 /* We ran out of KVD linear space, just set the 4098 * trap and let everything flow through kernel. 4099 */ 4100 dev_warn(mlxsw_sp->bus_info->dev, "Failed to allocate KVD linear area for nexthop group.\n"); 4101 goto set_trap; 4102 } 4103 old_adj_index_valid = nhgi->adj_index_valid; 4104 old_adj_index = nhgi->adj_index; 4105 old_ecmp_size = nhgi->ecmp_size; 4106 nhgi->adj_index_valid = 1; 4107 nhgi->adj_index = adj_index; 4108 nhgi->ecmp_size = ecmp_size; 4109 mlxsw_sp_nexthop_group_rebalance(nhgi); 4110 err = mlxsw_sp_nexthop_group_update(mlxsw_sp, nhgi, true); 4111 if (err) { 4112 dev_warn(mlxsw_sp->bus_info->dev, "Failed to update neigh MAC in adjacency table.\n"); 4113 goto set_trap; 4114 } 4115 4116 mlxsw_sp_nexthop_group_offload_refresh(mlxsw_sp, nh_grp); 4117 4118 if (!old_adj_index_valid) { 4119 /* The trap was set for fib entries, so we have to call 4120 * fib entry update to unset it and use adjacency index. 4121 */ 4122 err = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp); 4123 if (err) { 4124 dev_warn(mlxsw_sp->bus_info->dev, "Failed to add adjacency index to fib entries.\n"); 4125 goto set_trap; 4126 } 4127 return 0; 4128 } 4129 4130 err = mlxsw_sp_adj_index_mass_update(mlxsw_sp, nh_grp, 4131 old_adj_index, old_ecmp_size); 4132 mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 4133 old_ecmp_size, old_adj_index); 4134 if (err) { 4135 dev_warn(mlxsw_sp->bus_info->dev, "Failed to mass-update adjacency index for nexthop group.\n"); 4136 goto set_trap; 4137 } 4138 4139 return 0; 4140 4141 set_trap: 4142 old_adj_index_valid = nhgi->adj_index_valid; 4143 nhgi->adj_index_valid = 0; 4144 for (i = 0; i < nhgi->count; i++) { 4145 nh = &nhgi->nexthops[i]; 4146 nh->offloaded = 0; 4147 } 4148 err2 = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp); 4149 if (err2) 4150 dev_warn(mlxsw_sp->bus_info->dev, "Failed to set traps for fib entries.\n"); 4151 mlxsw_sp_nexthop_group_offload_refresh(mlxsw_sp, nh_grp); 4152 if (old_adj_index_valid) 4153 mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 4154 nhgi->ecmp_size, nhgi->adj_index); 4155 return err; 4156 } 4157 4158 static void __mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp_nexthop *nh, 4159 bool removing) 4160 { 4161 if (!removing) { 4162 nh->action = MLXSW_SP_NEXTHOP_ACTION_FORWARD; 4163 nh->should_offload = 1; 4164 } else if (nh->nhgi->is_resilient) { 4165 nh->action = MLXSW_SP_NEXTHOP_ACTION_TRAP; 4166 nh->should_offload = 1; 4167 } else { 4168 nh->should_offload = 0; 4169 } 4170 nh->update = 1; 4171 } 4172 4173 static int 4174 mlxsw_sp_nexthop_dead_neigh_replace(struct mlxsw_sp *mlxsw_sp, 4175 struct mlxsw_sp_neigh_entry *neigh_entry) 4176 { 4177 struct neighbour *n, *old_n = neigh_entry->key.n; 4178 struct mlxsw_sp_nexthop *nh; 4179 struct net_device *dev; 4180 bool entry_connected; 4181 u8 nud_state, dead; 4182 int err; 4183 4184 nh = list_first_entry(&neigh_entry->nexthop_list, 4185 struct mlxsw_sp_nexthop, neigh_list_node); 4186 dev = mlxsw_sp_nexthop_dev(nh); 4187 4188 n = neigh_lookup(nh->neigh_tbl, &nh->gw_addr, dev); 4189 if (!n) { 4190 n = neigh_create(nh->neigh_tbl, &nh->gw_addr, dev); 4191 if (IS_ERR(n)) 4192 return PTR_ERR(n); 4193 neigh_event_send(n, NULL); 4194 } 4195 4196 mlxsw_sp_neigh_entry_remove(mlxsw_sp, neigh_entry); 4197 neigh_entry->key.n = n; 4198 err = mlxsw_sp_neigh_entry_insert(mlxsw_sp, neigh_entry); 4199 if (err) 4200 goto err_neigh_entry_insert; 4201 4202 read_lock_bh(&n->lock); 4203 nud_state = n->nud_state; 4204 dead = n->dead; 4205 read_unlock_bh(&n->lock); 4206 entry_connected = nud_state & NUD_VALID && !dead; 4207 4208 list_for_each_entry(nh, &neigh_entry->nexthop_list, 4209 neigh_list_node) { 4210 neigh_release(old_n); 4211 neigh_clone(n); 4212 __mlxsw_sp_nexthop_neigh_update(nh, !entry_connected); 4213 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nhgi->nh_grp); 4214 } 4215 4216 neigh_release(n); 4217 4218 return 0; 4219 4220 err_neigh_entry_insert: 4221 neigh_entry->key.n = old_n; 4222 mlxsw_sp_neigh_entry_insert(mlxsw_sp, neigh_entry); 4223 neigh_release(n); 4224 return err; 4225 } 4226 4227 static void 4228 mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp *mlxsw_sp, 4229 struct mlxsw_sp_neigh_entry *neigh_entry, 4230 bool removing, bool dead) 4231 { 4232 struct mlxsw_sp_nexthop *nh; 4233 4234 if (list_empty(&neigh_entry->nexthop_list)) 4235 return; 4236 4237 if (dead) { 4238 int err; 4239 4240 err = mlxsw_sp_nexthop_dead_neigh_replace(mlxsw_sp, 4241 neigh_entry); 4242 if (err) 4243 dev_err(mlxsw_sp->bus_info->dev, "Failed to replace dead neigh\n"); 4244 return; 4245 } 4246 4247 list_for_each_entry(nh, &neigh_entry->nexthop_list, 4248 neigh_list_node) { 4249 __mlxsw_sp_nexthop_neigh_update(nh, removing); 4250 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nhgi->nh_grp); 4251 } 4252 } 4253 4254 static void mlxsw_sp_nexthop_crif_init(struct mlxsw_sp_nexthop *nh, 4255 struct mlxsw_sp_crif *crif) 4256 { 4257 if (nh->crif) 4258 return; 4259 4260 nh->crif = crif; 4261 list_add(&nh->crif_list_node, &crif->nexthop_list); 4262 } 4263 4264 static void mlxsw_sp_nexthop_crif_fini(struct mlxsw_sp_nexthop *nh) 4265 { 4266 if (!nh->crif) 4267 return; 4268 4269 list_del(&nh->crif_list_node); 4270 nh->crif = NULL; 4271 } 4272 4273 static int mlxsw_sp_nexthop_neigh_init(struct mlxsw_sp *mlxsw_sp, 4274 struct mlxsw_sp_nexthop *nh) 4275 { 4276 struct mlxsw_sp_neigh_entry *neigh_entry; 4277 struct net_device *dev; 4278 struct neighbour *n; 4279 u8 nud_state, dead; 4280 int err; 4281 4282 if (WARN_ON(!nh->crif->rif)) 4283 return 0; 4284 4285 if (!nh->nhgi->gateway || nh->neigh_entry) 4286 return 0; 4287 dev = mlxsw_sp_nexthop_dev(nh); 4288 4289 /* Take a reference of neigh here ensuring that neigh would 4290 * not be destructed before the nexthop entry is finished. 4291 * The reference is taken either in neigh_lookup() or 4292 * in neigh_create() in case n is not found. 4293 */ 4294 n = neigh_lookup(nh->neigh_tbl, &nh->gw_addr, dev); 4295 if (!n) { 4296 n = neigh_create(nh->neigh_tbl, &nh->gw_addr, dev); 4297 if (IS_ERR(n)) 4298 return PTR_ERR(n); 4299 neigh_event_send(n, NULL); 4300 } 4301 neigh_entry = mlxsw_sp_neigh_entry_lookup(mlxsw_sp, n); 4302 if (!neigh_entry) { 4303 neigh_entry = mlxsw_sp_neigh_entry_create(mlxsw_sp, n); 4304 if (IS_ERR(neigh_entry)) { 4305 err = -EINVAL; 4306 goto err_neigh_entry_create; 4307 } 4308 } 4309 4310 /* If that is the first nexthop connected to that neigh, add to 4311 * nexthop_neighs_list 4312 */ 4313 if (list_empty(&neigh_entry->nexthop_list)) 4314 list_add_tail(&neigh_entry->nexthop_neighs_list_node, 4315 &mlxsw_sp->router->nexthop_neighs_list); 4316 4317 nh->neigh_entry = neigh_entry; 4318 list_add_tail(&nh->neigh_list_node, &neigh_entry->nexthop_list); 4319 read_lock_bh(&n->lock); 4320 nud_state = n->nud_state; 4321 dead = n->dead; 4322 read_unlock_bh(&n->lock); 4323 __mlxsw_sp_nexthop_neigh_update(nh, !(nud_state & NUD_VALID && !dead)); 4324 4325 return 0; 4326 4327 err_neigh_entry_create: 4328 neigh_release(n); 4329 return err; 4330 } 4331 4332 static void mlxsw_sp_nexthop_neigh_fini(struct mlxsw_sp *mlxsw_sp, 4333 struct mlxsw_sp_nexthop *nh) 4334 { 4335 struct mlxsw_sp_neigh_entry *neigh_entry = nh->neigh_entry; 4336 struct neighbour *n; 4337 4338 if (!neigh_entry) 4339 return; 4340 n = neigh_entry->key.n; 4341 4342 __mlxsw_sp_nexthop_neigh_update(nh, true); 4343 list_del(&nh->neigh_list_node); 4344 nh->neigh_entry = NULL; 4345 4346 /* If that is the last nexthop connected to that neigh, remove from 4347 * nexthop_neighs_list 4348 */ 4349 if (list_empty(&neigh_entry->nexthop_list)) 4350 list_del(&neigh_entry->nexthop_neighs_list_node); 4351 4352 if (!neigh_entry->connected && list_empty(&neigh_entry->nexthop_list)) 4353 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry); 4354 4355 neigh_release(n); 4356 } 4357 4358 static bool mlxsw_sp_ipip_netdev_ul_up(struct net_device *ol_dev) 4359 { 4360 struct net_device *ul_dev; 4361 bool is_up; 4362 4363 rcu_read_lock(); 4364 ul_dev = mlxsw_sp_ipip_netdev_ul_dev_get(ol_dev); 4365 is_up = ul_dev ? (ul_dev->flags & IFF_UP) : true; 4366 rcu_read_unlock(); 4367 4368 return is_up; 4369 } 4370 4371 static void mlxsw_sp_nexthop_ipip_init(struct mlxsw_sp *mlxsw_sp, 4372 struct mlxsw_sp_nexthop *nh, 4373 struct mlxsw_sp_ipip_entry *ipip_entry) 4374 { 4375 struct mlxsw_sp_crif *crif; 4376 bool removing; 4377 4378 if (!nh->nhgi->gateway || nh->ipip_entry) 4379 return; 4380 4381 crif = mlxsw_sp_crif_lookup(mlxsw_sp->router, ipip_entry->ol_dev); 4382 if (WARN_ON(!crif)) 4383 return; 4384 4385 nh->ipip_entry = ipip_entry; 4386 removing = !mlxsw_sp_ipip_netdev_ul_up(ipip_entry->ol_dev); 4387 __mlxsw_sp_nexthop_neigh_update(nh, removing); 4388 mlxsw_sp_nexthop_crif_init(nh, crif); 4389 } 4390 4391 static void mlxsw_sp_nexthop_ipip_fini(struct mlxsw_sp *mlxsw_sp, 4392 struct mlxsw_sp_nexthop *nh) 4393 { 4394 struct mlxsw_sp_ipip_entry *ipip_entry = nh->ipip_entry; 4395 4396 if (!ipip_entry) 4397 return; 4398 4399 __mlxsw_sp_nexthop_neigh_update(nh, true); 4400 nh->ipip_entry = NULL; 4401 } 4402 4403 static bool mlxsw_sp_nexthop4_ipip_type(const struct mlxsw_sp *mlxsw_sp, 4404 const struct fib_nh *fib_nh, 4405 enum mlxsw_sp_ipip_type *p_ipipt) 4406 { 4407 struct net_device *dev = fib_nh->fib_nh_dev; 4408 4409 return dev && 4410 fib_nh->nh_parent->fib_type == RTN_UNICAST && 4411 mlxsw_sp_netdev_ipip_type(mlxsw_sp, dev, p_ipipt); 4412 } 4413 4414 static int mlxsw_sp_nexthop_type_init(struct mlxsw_sp *mlxsw_sp, 4415 struct mlxsw_sp_nexthop *nh, 4416 const struct net_device *dev) 4417 { 4418 const struct mlxsw_sp_ipip_ops *ipip_ops; 4419 struct mlxsw_sp_ipip_entry *ipip_entry; 4420 struct mlxsw_sp_crif *crif; 4421 int err; 4422 4423 ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, dev); 4424 if (ipip_entry) { 4425 ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt]; 4426 if (ipip_ops->can_offload(mlxsw_sp, dev)) { 4427 nh->type = MLXSW_SP_NEXTHOP_TYPE_IPIP; 4428 mlxsw_sp_nexthop_ipip_init(mlxsw_sp, nh, ipip_entry); 4429 return 0; 4430 } 4431 } 4432 4433 nh->type = MLXSW_SP_NEXTHOP_TYPE_ETH; 4434 crif = mlxsw_sp_crif_lookup(mlxsw_sp->router, dev); 4435 if (!crif) 4436 return 0; 4437 4438 mlxsw_sp_nexthop_crif_init(nh, crif); 4439 4440 if (!crif->rif) 4441 return 0; 4442 4443 err = mlxsw_sp_nexthop_neigh_init(mlxsw_sp, nh); 4444 if (err) 4445 goto err_neigh_init; 4446 4447 return 0; 4448 4449 err_neigh_init: 4450 mlxsw_sp_nexthop_crif_fini(nh); 4451 return err; 4452 } 4453 4454 static int mlxsw_sp_nexthop_type_rif_made(struct mlxsw_sp *mlxsw_sp, 4455 struct mlxsw_sp_nexthop *nh) 4456 { 4457 switch (nh->type) { 4458 case MLXSW_SP_NEXTHOP_TYPE_ETH: 4459 return mlxsw_sp_nexthop_neigh_init(mlxsw_sp, nh); 4460 case MLXSW_SP_NEXTHOP_TYPE_IPIP: 4461 break; 4462 } 4463 4464 return 0; 4465 } 4466 4467 static void mlxsw_sp_nexthop_type_rif_gone(struct mlxsw_sp *mlxsw_sp, 4468 struct mlxsw_sp_nexthop *nh) 4469 { 4470 switch (nh->type) { 4471 case MLXSW_SP_NEXTHOP_TYPE_ETH: 4472 mlxsw_sp_nexthop_neigh_fini(mlxsw_sp, nh); 4473 break; 4474 case MLXSW_SP_NEXTHOP_TYPE_IPIP: 4475 mlxsw_sp_nexthop_ipip_fini(mlxsw_sp, nh); 4476 break; 4477 } 4478 } 4479 4480 static void mlxsw_sp_nexthop_type_fini(struct mlxsw_sp *mlxsw_sp, 4481 struct mlxsw_sp_nexthop *nh) 4482 { 4483 mlxsw_sp_nexthop_type_rif_gone(mlxsw_sp, nh); 4484 mlxsw_sp_nexthop_crif_fini(nh); 4485 } 4486 4487 static int mlxsw_sp_nexthop4_init(struct mlxsw_sp *mlxsw_sp, 4488 struct mlxsw_sp_nexthop_group *nh_grp, 4489 struct mlxsw_sp_nexthop *nh, 4490 struct fib_nh *fib_nh) 4491 { 4492 struct net_device *dev = fib_nh->fib_nh_dev; 4493 struct in_device *in_dev; 4494 int err; 4495 4496 nh->nhgi = nh_grp->nhgi; 4497 nh->key.fib_nh = fib_nh; 4498 #ifdef CONFIG_IP_ROUTE_MULTIPATH 4499 nh->nh_weight = fib_nh->fib_nh_weight; 4500 #else 4501 nh->nh_weight = 1; 4502 #endif 4503 memcpy(&nh->gw_addr, &fib_nh->fib_nh_gw4, sizeof(fib_nh->fib_nh_gw4)); 4504 nh->neigh_tbl = &arp_tbl; 4505 err = mlxsw_sp_nexthop_insert(mlxsw_sp, nh); 4506 if (err) 4507 return err; 4508 4509 mlxsw_sp_nexthop_counter_alloc(mlxsw_sp, nh); 4510 list_add_tail(&nh->router_list_node, &mlxsw_sp->router->nexthop_list); 4511 4512 if (!dev) 4513 return 0; 4514 nh->ifindex = dev->ifindex; 4515 4516 rcu_read_lock(); 4517 in_dev = __in_dev_get_rcu(dev); 4518 if (in_dev && IN_DEV_IGNORE_ROUTES_WITH_LINKDOWN(in_dev) && 4519 fib_nh->fib_nh_flags & RTNH_F_LINKDOWN) { 4520 rcu_read_unlock(); 4521 return 0; 4522 } 4523 rcu_read_unlock(); 4524 4525 err = mlxsw_sp_nexthop_type_init(mlxsw_sp, nh, dev); 4526 if (err) 4527 goto err_nexthop_neigh_init; 4528 4529 return 0; 4530 4531 err_nexthop_neigh_init: 4532 list_del(&nh->router_list_node); 4533 mlxsw_sp_nexthop_counter_free(mlxsw_sp, nh); 4534 mlxsw_sp_nexthop_remove(mlxsw_sp, nh); 4535 return err; 4536 } 4537 4538 static void mlxsw_sp_nexthop4_fini(struct mlxsw_sp *mlxsw_sp, 4539 struct mlxsw_sp_nexthop *nh) 4540 { 4541 mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh); 4542 list_del(&nh->router_list_node); 4543 mlxsw_sp_nexthop_counter_free(mlxsw_sp, nh); 4544 mlxsw_sp_nexthop_remove(mlxsw_sp, nh); 4545 } 4546 4547 static void mlxsw_sp_nexthop4_event(struct mlxsw_sp *mlxsw_sp, 4548 unsigned long event, struct fib_nh *fib_nh) 4549 { 4550 struct mlxsw_sp_nexthop_key key; 4551 struct mlxsw_sp_nexthop *nh; 4552 4553 key.fib_nh = fib_nh; 4554 nh = mlxsw_sp_nexthop_lookup(mlxsw_sp, key); 4555 if (!nh) 4556 return; 4557 4558 switch (event) { 4559 case FIB_EVENT_NH_ADD: 4560 mlxsw_sp_nexthop_type_init(mlxsw_sp, nh, fib_nh->fib_nh_dev); 4561 break; 4562 case FIB_EVENT_NH_DEL: 4563 mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh); 4564 break; 4565 } 4566 4567 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nhgi->nh_grp); 4568 } 4569 4570 static void mlxsw_sp_nexthop_rif_update(struct mlxsw_sp *mlxsw_sp, 4571 struct mlxsw_sp_rif *rif) 4572 { 4573 struct net_device *dev = mlxsw_sp_rif_dev(rif); 4574 struct mlxsw_sp_nexthop *nh; 4575 bool removing; 4576 4577 list_for_each_entry(nh, &rif->crif->nexthop_list, crif_list_node) { 4578 switch (nh->type) { 4579 case MLXSW_SP_NEXTHOP_TYPE_ETH: 4580 removing = false; 4581 break; 4582 case MLXSW_SP_NEXTHOP_TYPE_IPIP: 4583 removing = !mlxsw_sp_ipip_netdev_ul_up(dev); 4584 break; 4585 default: 4586 WARN_ON(1); 4587 continue; 4588 } 4589 4590 __mlxsw_sp_nexthop_neigh_update(nh, removing); 4591 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nhgi->nh_grp); 4592 } 4593 } 4594 4595 static int mlxsw_sp_nexthop_rif_made_sync(struct mlxsw_sp *mlxsw_sp, 4596 struct mlxsw_sp_rif *rif) 4597 { 4598 struct mlxsw_sp_nexthop *nh, *tmp; 4599 unsigned int n = 0; 4600 int err; 4601 4602 list_for_each_entry_safe(nh, tmp, &rif->crif->nexthop_list, 4603 crif_list_node) { 4604 err = mlxsw_sp_nexthop_type_rif_made(mlxsw_sp, nh); 4605 if (err) 4606 goto err_nexthop_type_rif; 4607 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nhgi->nh_grp); 4608 n++; 4609 } 4610 4611 return 0; 4612 4613 err_nexthop_type_rif: 4614 list_for_each_entry_safe(nh, tmp, &rif->crif->nexthop_list, 4615 crif_list_node) { 4616 if (!n--) 4617 break; 4618 mlxsw_sp_nexthop_type_rif_gone(mlxsw_sp, nh); 4619 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nhgi->nh_grp); 4620 } 4621 return err; 4622 } 4623 4624 static void mlxsw_sp_nexthop_rif_gone_sync(struct mlxsw_sp *mlxsw_sp, 4625 struct mlxsw_sp_rif *rif) 4626 { 4627 struct mlxsw_sp_nexthop *nh, *tmp; 4628 4629 list_for_each_entry_safe(nh, tmp, &rif->crif->nexthop_list, 4630 crif_list_node) { 4631 mlxsw_sp_nexthop_type_rif_gone(mlxsw_sp, nh); 4632 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nhgi->nh_grp); 4633 } 4634 } 4635 4636 static int mlxsw_sp_adj_trap_entry_init(struct mlxsw_sp *mlxsw_sp) 4637 { 4638 enum mlxsw_reg_ratr_trap_action trap_action; 4639 char ratr_pl[MLXSW_REG_RATR_LEN]; 4640 int err; 4641 4642 err = mlxsw_sp_kvdl_alloc(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 1, 4643 &mlxsw_sp->router->adj_trap_index); 4644 if (err) 4645 return err; 4646 4647 trap_action = MLXSW_REG_RATR_TRAP_ACTION_TRAP; 4648 mlxsw_reg_ratr_pack(ratr_pl, MLXSW_REG_RATR_OP_WRITE_WRITE_ENTRY, true, 4649 MLXSW_REG_RATR_TYPE_ETHERNET, 4650 mlxsw_sp->router->adj_trap_index, 4651 mlxsw_sp->router->lb_crif->rif->rif_index); 4652 mlxsw_reg_ratr_trap_action_set(ratr_pl, trap_action); 4653 mlxsw_reg_ratr_trap_id_set(ratr_pl, MLXSW_TRAP_ID_RTR_EGRESS0); 4654 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ratr), ratr_pl); 4655 if (err) 4656 goto err_ratr_write; 4657 4658 return 0; 4659 4660 err_ratr_write: 4661 mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 1, 4662 mlxsw_sp->router->adj_trap_index); 4663 return err; 4664 } 4665 4666 static void mlxsw_sp_adj_trap_entry_fini(struct mlxsw_sp *mlxsw_sp) 4667 { 4668 mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 1, 4669 mlxsw_sp->router->adj_trap_index); 4670 } 4671 4672 static int mlxsw_sp_nexthop_group_inc(struct mlxsw_sp *mlxsw_sp) 4673 { 4674 int err; 4675 4676 if (refcount_inc_not_zero(&mlxsw_sp->router->num_groups)) 4677 return 0; 4678 4679 err = mlxsw_sp_adj_trap_entry_init(mlxsw_sp); 4680 if (err) 4681 return err; 4682 4683 refcount_set(&mlxsw_sp->router->num_groups, 1); 4684 4685 return 0; 4686 } 4687 4688 static void mlxsw_sp_nexthop_group_dec(struct mlxsw_sp *mlxsw_sp) 4689 { 4690 if (!refcount_dec_and_test(&mlxsw_sp->router->num_groups)) 4691 return; 4692 4693 mlxsw_sp_adj_trap_entry_fini(mlxsw_sp); 4694 } 4695 4696 static void 4697 mlxsw_sp_nh_grp_activity_get(struct mlxsw_sp *mlxsw_sp, 4698 const struct mlxsw_sp_nexthop_group *nh_grp, 4699 unsigned long *activity) 4700 { 4701 char *ratrad_pl; 4702 int i, err; 4703 4704 ratrad_pl = kmalloc(MLXSW_REG_RATRAD_LEN, GFP_KERNEL); 4705 if (!ratrad_pl) 4706 return; 4707 4708 mlxsw_reg_ratrad_pack(ratrad_pl, nh_grp->nhgi->adj_index, 4709 nh_grp->nhgi->count); 4710 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ratrad), ratrad_pl); 4711 if (err) 4712 goto out; 4713 4714 for (i = 0; i < nh_grp->nhgi->count; i++) { 4715 if (!mlxsw_reg_ratrad_activity_vector_get(ratrad_pl, i)) 4716 continue; 4717 bitmap_set(activity, i, 1); 4718 } 4719 4720 out: 4721 kfree(ratrad_pl); 4722 } 4723 4724 #define MLXSW_SP_NH_GRP_ACTIVITY_UPDATE_INTERVAL 1000 /* ms */ 4725 4726 static void 4727 mlxsw_sp_nh_grp_activity_update(struct mlxsw_sp *mlxsw_sp, 4728 const struct mlxsw_sp_nexthop_group *nh_grp) 4729 { 4730 unsigned long *activity; 4731 4732 activity = bitmap_zalloc(nh_grp->nhgi->count, GFP_KERNEL); 4733 if (!activity) 4734 return; 4735 4736 mlxsw_sp_nh_grp_activity_get(mlxsw_sp, nh_grp, activity); 4737 nexthop_res_grp_activity_update(mlxsw_sp_net(mlxsw_sp), nh_grp->obj.id, 4738 nh_grp->nhgi->count, activity); 4739 4740 bitmap_free(activity); 4741 } 4742 4743 static void 4744 mlxsw_sp_nh_grp_activity_work_schedule(struct mlxsw_sp *mlxsw_sp) 4745 { 4746 unsigned int interval = MLXSW_SP_NH_GRP_ACTIVITY_UPDATE_INTERVAL; 4747 4748 mlxsw_core_schedule_dw(&mlxsw_sp->router->nh_grp_activity_dw, 4749 msecs_to_jiffies(interval)); 4750 } 4751 4752 static void mlxsw_sp_nh_grp_activity_work(struct work_struct *work) 4753 { 4754 struct mlxsw_sp_nexthop_group_info *nhgi; 4755 struct mlxsw_sp_router *router; 4756 bool reschedule = false; 4757 4758 router = container_of(work, struct mlxsw_sp_router, 4759 nh_grp_activity_dw.work); 4760 4761 mutex_lock(&router->lock); 4762 4763 list_for_each_entry(nhgi, &router->nh_res_grp_list, list) { 4764 mlxsw_sp_nh_grp_activity_update(router->mlxsw_sp, nhgi->nh_grp); 4765 reschedule = true; 4766 } 4767 4768 mutex_unlock(&router->lock); 4769 4770 if (!reschedule) 4771 return; 4772 mlxsw_sp_nh_grp_activity_work_schedule(router->mlxsw_sp); 4773 } 4774 4775 static int 4776 mlxsw_sp_nexthop_obj_single_validate(struct mlxsw_sp *mlxsw_sp, 4777 const struct nh_notifier_single_info *nh, 4778 struct netlink_ext_ack *extack) 4779 { 4780 int err = -EINVAL; 4781 4782 if (nh->is_fdb) 4783 NL_SET_ERR_MSG_MOD(extack, "FDB nexthops are not supported"); 4784 else if (nh->has_encap) 4785 NL_SET_ERR_MSG_MOD(extack, "Encapsulating nexthops are not supported"); 4786 else 4787 err = 0; 4788 4789 return err; 4790 } 4791 4792 static int 4793 mlxsw_sp_nexthop_obj_group_entry_validate(struct mlxsw_sp *mlxsw_sp, 4794 const struct nh_notifier_single_info *nh, 4795 struct netlink_ext_ack *extack) 4796 { 4797 int err; 4798 4799 err = mlxsw_sp_nexthop_obj_single_validate(mlxsw_sp, nh, extack); 4800 if (err) 4801 return err; 4802 4803 /* Device only nexthops with an IPIP device are programmed as 4804 * encapsulating adjacency entries. 4805 */ 4806 if (!nh->gw_family && !nh->is_reject && 4807 !mlxsw_sp_netdev_ipip_type(mlxsw_sp, nh->dev, NULL)) { 4808 NL_SET_ERR_MSG_MOD(extack, "Nexthop group entry does not have a gateway"); 4809 return -EINVAL; 4810 } 4811 4812 return 0; 4813 } 4814 4815 static int 4816 mlxsw_sp_nexthop_obj_group_validate(struct mlxsw_sp *mlxsw_sp, 4817 const struct nh_notifier_grp_info *nh_grp, 4818 struct netlink_ext_ack *extack) 4819 { 4820 int i; 4821 4822 if (nh_grp->is_fdb) { 4823 NL_SET_ERR_MSG_MOD(extack, "FDB nexthop groups are not supported"); 4824 return -EINVAL; 4825 } 4826 4827 for (i = 0; i < nh_grp->num_nh; i++) { 4828 const struct nh_notifier_single_info *nh; 4829 int err; 4830 4831 nh = &nh_grp->nh_entries[i].nh; 4832 err = mlxsw_sp_nexthop_obj_group_entry_validate(mlxsw_sp, nh, 4833 extack); 4834 if (err) 4835 return err; 4836 } 4837 4838 return 0; 4839 } 4840 4841 static int 4842 mlxsw_sp_nexthop_obj_res_group_size_validate(struct mlxsw_sp *mlxsw_sp, 4843 const struct nh_notifier_res_table_info *nh_res_table, 4844 struct netlink_ext_ack *extack) 4845 { 4846 unsigned int alloc_size; 4847 bool valid_size = false; 4848 int err, i; 4849 4850 if (nh_res_table->num_nh_buckets < 32) { 4851 NL_SET_ERR_MSG_MOD(extack, "Minimum number of buckets is 32"); 4852 return -EINVAL; 4853 } 4854 4855 for (i = 0; i < mlxsw_sp->router->adj_grp_size_ranges_count; i++) { 4856 const struct mlxsw_sp_adj_grp_size_range *size_range; 4857 4858 size_range = &mlxsw_sp->router->adj_grp_size_ranges[i]; 4859 4860 if (nh_res_table->num_nh_buckets >= size_range->start && 4861 nh_res_table->num_nh_buckets <= size_range->end) { 4862 valid_size = true; 4863 break; 4864 } 4865 } 4866 4867 if (!valid_size) { 4868 NL_SET_ERR_MSG_MOD(extack, "Invalid number of buckets"); 4869 return -EINVAL; 4870 } 4871 4872 err = mlxsw_sp_kvdl_alloc_count_query(mlxsw_sp, 4873 MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 4874 nh_res_table->num_nh_buckets, 4875 &alloc_size); 4876 if (err || nh_res_table->num_nh_buckets != alloc_size) { 4877 NL_SET_ERR_MSG_MOD(extack, "Number of buckets does not fit allocation size of any KVDL partition"); 4878 return -EINVAL; 4879 } 4880 4881 return 0; 4882 } 4883 4884 static int 4885 mlxsw_sp_nexthop_obj_res_group_validate(struct mlxsw_sp *mlxsw_sp, 4886 const struct nh_notifier_res_table_info *nh_res_table, 4887 struct netlink_ext_ack *extack) 4888 { 4889 int err; 4890 u16 i; 4891 4892 err = mlxsw_sp_nexthop_obj_res_group_size_validate(mlxsw_sp, 4893 nh_res_table, 4894 extack); 4895 if (err) 4896 return err; 4897 4898 for (i = 0; i < nh_res_table->num_nh_buckets; i++) { 4899 const struct nh_notifier_single_info *nh; 4900 int err; 4901 4902 nh = &nh_res_table->nhs[i]; 4903 err = mlxsw_sp_nexthop_obj_group_entry_validate(mlxsw_sp, nh, 4904 extack); 4905 if (err) 4906 return err; 4907 } 4908 4909 return 0; 4910 } 4911 4912 static int mlxsw_sp_nexthop_obj_validate(struct mlxsw_sp *mlxsw_sp, 4913 unsigned long event, 4914 struct nh_notifier_info *info) 4915 { 4916 struct nh_notifier_single_info *nh; 4917 4918 if (event != NEXTHOP_EVENT_REPLACE && 4919 event != NEXTHOP_EVENT_RES_TABLE_PRE_REPLACE && 4920 event != NEXTHOP_EVENT_BUCKET_REPLACE) 4921 return 0; 4922 4923 switch (info->type) { 4924 case NH_NOTIFIER_INFO_TYPE_SINGLE: 4925 return mlxsw_sp_nexthop_obj_single_validate(mlxsw_sp, info->nh, 4926 info->extack); 4927 case NH_NOTIFIER_INFO_TYPE_GRP: 4928 return mlxsw_sp_nexthop_obj_group_validate(mlxsw_sp, 4929 info->nh_grp, 4930 info->extack); 4931 case NH_NOTIFIER_INFO_TYPE_RES_TABLE: 4932 return mlxsw_sp_nexthop_obj_res_group_validate(mlxsw_sp, 4933 info->nh_res_table, 4934 info->extack); 4935 case NH_NOTIFIER_INFO_TYPE_RES_BUCKET: 4936 nh = &info->nh_res_bucket->new_nh; 4937 return mlxsw_sp_nexthop_obj_group_entry_validate(mlxsw_sp, nh, 4938 info->extack); 4939 default: 4940 NL_SET_ERR_MSG_MOD(info->extack, "Unsupported nexthop type"); 4941 return -EOPNOTSUPP; 4942 } 4943 } 4944 4945 static bool mlxsw_sp_nexthop_obj_is_gateway(struct mlxsw_sp *mlxsw_sp, 4946 const struct nh_notifier_info *info) 4947 { 4948 const struct net_device *dev; 4949 4950 switch (info->type) { 4951 case NH_NOTIFIER_INFO_TYPE_SINGLE: 4952 dev = info->nh->dev; 4953 return info->nh->gw_family || info->nh->is_reject || 4954 mlxsw_sp_netdev_ipip_type(mlxsw_sp, dev, NULL); 4955 case NH_NOTIFIER_INFO_TYPE_GRP: 4956 case NH_NOTIFIER_INFO_TYPE_RES_TABLE: 4957 /* Already validated earlier. */ 4958 return true; 4959 default: 4960 return false; 4961 } 4962 } 4963 4964 static void mlxsw_sp_nexthop_obj_blackhole_init(struct mlxsw_sp *mlxsw_sp, 4965 struct mlxsw_sp_nexthop *nh) 4966 { 4967 nh->action = MLXSW_SP_NEXTHOP_ACTION_DISCARD; 4968 nh->should_offload = 1; 4969 /* While nexthops that discard packets do not forward packets 4970 * via an egress RIF, they still need to be programmed using a 4971 * valid RIF, so use the loopback RIF created during init. 4972 */ 4973 nh->crif = mlxsw_sp->router->lb_crif; 4974 } 4975 4976 static void mlxsw_sp_nexthop_obj_blackhole_fini(struct mlxsw_sp *mlxsw_sp, 4977 struct mlxsw_sp_nexthop *nh) 4978 { 4979 nh->crif = NULL; 4980 nh->should_offload = 0; 4981 } 4982 4983 static int 4984 mlxsw_sp_nexthop_obj_init(struct mlxsw_sp *mlxsw_sp, 4985 struct mlxsw_sp_nexthop_group *nh_grp, 4986 struct mlxsw_sp_nexthop *nh, 4987 struct nh_notifier_single_info *nh_obj, int weight) 4988 { 4989 struct net_device *dev = nh_obj->dev; 4990 int err; 4991 4992 nh->nhgi = nh_grp->nhgi; 4993 nh->nh_weight = weight; 4994 4995 switch (nh_obj->gw_family) { 4996 case AF_INET: 4997 memcpy(&nh->gw_addr, &nh_obj->ipv4, sizeof(nh_obj->ipv4)); 4998 nh->neigh_tbl = &arp_tbl; 4999 break; 5000 case AF_INET6: 5001 memcpy(&nh->gw_addr, &nh_obj->ipv6, sizeof(nh_obj->ipv6)); 5002 #if IS_ENABLED(CONFIG_IPV6) 5003 nh->neigh_tbl = &nd_tbl; 5004 #endif 5005 break; 5006 } 5007 5008 mlxsw_sp_nexthop_counter_alloc(mlxsw_sp, nh); 5009 list_add_tail(&nh->router_list_node, &mlxsw_sp->router->nexthop_list); 5010 nh->ifindex = dev->ifindex; 5011 5012 err = mlxsw_sp_nexthop_type_init(mlxsw_sp, nh, dev); 5013 if (err) 5014 goto err_type_init; 5015 5016 if (nh_obj->is_reject) 5017 mlxsw_sp_nexthop_obj_blackhole_init(mlxsw_sp, nh); 5018 5019 /* In a resilient nexthop group, all the nexthops must be written to 5020 * the adjacency table. Even if they do not have a valid neighbour or 5021 * RIF. 5022 */ 5023 if (nh_grp->nhgi->is_resilient && !nh->should_offload) { 5024 nh->action = MLXSW_SP_NEXTHOP_ACTION_TRAP; 5025 nh->should_offload = 1; 5026 } 5027 5028 return 0; 5029 5030 err_type_init: 5031 list_del(&nh->router_list_node); 5032 mlxsw_sp_nexthop_counter_free(mlxsw_sp, nh); 5033 return err; 5034 } 5035 5036 static void mlxsw_sp_nexthop_obj_fini(struct mlxsw_sp *mlxsw_sp, 5037 struct mlxsw_sp_nexthop *nh) 5038 { 5039 if (nh->action == MLXSW_SP_NEXTHOP_ACTION_DISCARD) 5040 mlxsw_sp_nexthop_obj_blackhole_fini(mlxsw_sp, nh); 5041 mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh); 5042 list_del(&nh->router_list_node); 5043 mlxsw_sp_nexthop_counter_free(mlxsw_sp, nh); 5044 nh->should_offload = 0; 5045 } 5046 5047 static int 5048 mlxsw_sp_nexthop_obj_group_info_init(struct mlxsw_sp *mlxsw_sp, 5049 struct mlxsw_sp_nexthop_group *nh_grp, 5050 struct nh_notifier_info *info) 5051 { 5052 struct mlxsw_sp_nexthop_group_info *nhgi; 5053 struct mlxsw_sp_nexthop *nh; 5054 bool is_resilient = false; 5055 unsigned int nhs; 5056 int err, i; 5057 5058 switch (info->type) { 5059 case NH_NOTIFIER_INFO_TYPE_SINGLE: 5060 nhs = 1; 5061 break; 5062 case NH_NOTIFIER_INFO_TYPE_GRP: 5063 nhs = info->nh_grp->num_nh; 5064 break; 5065 case NH_NOTIFIER_INFO_TYPE_RES_TABLE: 5066 nhs = info->nh_res_table->num_nh_buckets; 5067 is_resilient = true; 5068 break; 5069 default: 5070 return -EINVAL; 5071 } 5072 5073 nhgi = kzalloc(struct_size(nhgi, nexthops, nhs), GFP_KERNEL); 5074 if (!nhgi) 5075 return -ENOMEM; 5076 nh_grp->nhgi = nhgi; 5077 nhgi->nh_grp = nh_grp; 5078 nhgi->gateway = mlxsw_sp_nexthop_obj_is_gateway(mlxsw_sp, info); 5079 nhgi->is_resilient = is_resilient; 5080 nhgi->count = nhs; 5081 for (i = 0; i < nhgi->count; i++) { 5082 struct nh_notifier_single_info *nh_obj; 5083 int weight; 5084 5085 nh = &nhgi->nexthops[i]; 5086 switch (info->type) { 5087 case NH_NOTIFIER_INFO_TYPE_SINGLE: 5088 nh_obj = info->nh; 5089 weight = 1; 5090 break; 5091 case NH_NOTIFIER_INFO_TYPE_GRP: 5092 nh_obj = &info->nh_grp->nh_entries[i].nh; 5093 weight = info->nh_grp->nh_entries[i].weight; 5094 break; 5095 case NH_NOTIFIER_INFO_TYPE_RES_TABLE: 5096 nh_obj = &info->nh_res_table->nhs[i]; 5097 weight = 1; 5098 break; 5099 default: 5100 err = -EINVAL; 5101 goto err_nexthop_obj_init; 5102 } 5103 err = mlxsw_sp_nexthop_obj_init(mlxsw_sp, nh_grp, nh, nh_obj, 5104 weight); 5105 if (err) 5106 goto err_nexthop_obj_init; 5107 } 5108 err = mlxsw_sp_nexthop_group_inc(mlxsw_sp); 5109 if (err) 5110 goto err_group_inc; 5111 err = mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp); 5112 if (err) { 5113 NL_SET_ERR_MSG_MOD(info->extack, "Failed to write adjacency entries to the device"); 5114 goto err_group_refresh; 5115 } 5116 5117 /* Add resilient nexthop groups to a list so that the activity of their 5118 * nexthop buckets will be periodically queried and cleared. 5119 */ 5120 if (nhgi->is_resilient) { 5121 if (list_empty(&mlxsw_sp->router->nh_res_grp_list)) 5122 mlxsw_sp_nh_grp_activity_work_schedule(mlxsw_sp); 5123 list_add(&nhgi->list, &mlxsw_sp->router->nh_res_grp_list); 5124 } 5125 5126 return 0; 5127 5128 err_group_refresh: 5129 mlxsw_sp_nexthop_group_dec(mlxsw_sp); 5130 err_group_inc: 5131 i = nhgi->count; 5132 err_nexthop_obj_init: 5133 for (i--; i >= 0; i--) { 5134 nh = &nhgi->nexthops[i]; 5135 mlxsw_sp_nexthop_obj_fini(mlxsw_sp, nh); 5136 } 5137 kfree(nhgi); 5138 return err; 5139 } 5140 5141 static void 5142 mlxsw_sp_nexthop_obj_group_info_fini(struct mlxsw_sp *mlxsw_sp, 5143 struct mlxsw_sp_nexthop_group *nh_grp) 5144 { 5145 struct mlxsw_sp_nexthop_group_info *nhgi = nh_grp->nhgi; 5146 struct mlxsw_sp_router *router = mlxsw_sp->router; 5147 int i; 5148 5149 if (nhgi->is_resilient) { 5150 list_del(&nhgi->list); 5151 if (list_empty(&mlxsw_sp->router->nh_res_grp_list)) 5152 cancel_delayed_work(&router->nh_grp_activity_dw); 5153 } 5154 5155 mlxsw_sp_nexthop_group_dec(mlxsw_sp); 5156 for (i = nhgi->count - 1; i >= 0; i--) { 5157 struct mlxsw_sp_nexthop *nh = &nhgi->nexthops[i]; 5158 5159 mlxsw_sp_nexthop_obj_fini(mlxsw_sp, nh); 5160 } 5161 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp); 5162 WARN_ON_ONCE(nhgi->adj_index_valid); 5163 kfree(nhgi); 5164 } 5165 5166 static struct mlxsw_sp_nexthop_group * 5167 mlxsw_sp_nexthop_obj_group_create(struct mlxsw_sp *mlxsw_sp, 5168 struct nh_notifier_info *info) 5169 { 5170 struct mlxsw_sp_nexthop_group *nh_grp; 5171 int err; 5172 5173 nh_grp = kzalloc(sizeof(*nh_grp), GFP_KERNEL); 5174 if (!nh_grp) 5175 return ERR_PTR(-ENOMEM); 5176 INIT_LIST_HEAD(&nh_grp->vr_list); 5177 err = rhashtable_init(&nh_grp->vr_ht, 5178 &mlxsw_sp_nexthop_group_vr_ht_params); 5179 if (err) 5180 goto err_nexthop_group_vr_ht_init; 5181 INIT_LIST_HEAD(&nh_grp->fib_list); 5182 nh_grp->type = MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ; 5183 nh_grp->obj.id = info->id; 5184 5185 err = mlxsw_sp_nexthop_obj_group_info_init(mlxsw_sp, nh_grp, info); 5186 if (err) 5187 goto err_nexthop_group_info_init; 5188 5189 nh_grp->can_destroy = false; 5190 5191 return nh_grp; 5192 5193 err_nexthop_group_info_init: 5194 rhashtable_destroy(&nh_grp->vr_ht); 5195 err_nexthop_group_vr_ht_init: 5196 kfree(nh_grp); 5197 return ERR_PTR(err); 5198 } 5199 5200 static void 5201 mlxsw_sp_nexthop_obj_group_destroy(struct mlxsw_sp *mlxsw_sp, 5202 struct mlxsw_sp_nexthop_group *nh_grp) 5203 { 5204 if (!nh_grp->can_destroy) 5205 return; 5206 mlxsw_sp_nexthop_obj_group_info_fini(mlxsw_sp, nh_grp); 5207 WARN_ON_ONCE(!list_empty(&nh_grp->fib_list)); 5208 WARN_ON_ONCE(!list_empty(&nh_grp->vr_list)); 5209 rhashtable_destroy(&nh_grp->vr_ht); 5210 kfree(nh_grp); 5211 } 5212 5213 static struct mlxsw_sp_nexthop_group * 5214 mlxsw_sp_nexthop_obj_group_lookup(struct mlxsw_sp *mlxsw_sp, u32 id) 5215 { 5216 struct mlxsw_sp_nexthop_group_cmp_arg cmp_arg; 5217 5218 cmp_arg.type = MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ; 5219 cmp_arg.id = id; 5220 return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_group_ht, 5221 &cmp_arg, 5222 mlxsw_sp_nexthop_group_ht_params); 5223 } 5224 5225 static int mlxsw_sp_nexthop_obj_group_add(struct mlxsw_sp *mlxsw_sp, 5226 struct mlxsw_sp_nexthop_group *nh_grp) 5227 { 5228 return mlxsw_sp_nexthop_group_insert(mlxsw_sp, nh_grp); 5229 } 5230 5231 static int 5232 mlxsw_sp_nexthop_obj_group_replace(struct mlxsw_sp *mlxsw_sp, 5233 struct mlxsw_sp_nexthop_group *nh_grp, 5234 struct mlxsw_sp_nexthop_group *old_nh_grp, 5235 struct netlink_ext_ack *extack) 5236 { 5237 struct mlxsw_sp_nexthop_group_info *old_nhgi = old_nh_grp->nhgi; 5238 struct mlxsw_sp_nexthop_group_info *new_nhgi = nh_grp->nhgi; 5239 int err; 5240 5241 old_nh_grp->nhgi = new_nhgi; 5242 new_nhgi->nh_grp = old_nh_grp; 5243 nh_grp->nhgi = old_nhgi; 5244 old_nhgi->nh_grp = nh_grp; 5245 5246 if (old_nhgi->adj_index_valid && new_nhgi->adj_index_valid) { 5247 /* Both the old adjacency index and the new one are valid. 5248 * Routes are currently using the old one. Tell the device to 5249 * replace the old adjacency index with the new one. 5250 */ 5251 err = mlxsw_sp_adj_index_mass_update(mlxsw_sp, old_nh_grp, 5252 old_nhgi->adj_index, 5253 old_nhgi->ecmp_size); 5254 if (err) { 5255 NL_SET_ERR_MSG_MOD(extack, "Failed to replace old adjacency index with new one"); 5256 goto err_out; 5257 } 5258 } else if (old_nhgi->adj_index_valid && !new_nhgi->adj_index_valid) { 5259 /* The old adjacency index is valid, while the new one is not. 5260 * Iterate over all the routes using the group and change them 5261 * to trap packets to the CPU. 5262 */ 5263 err = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, old_nh_grp); 5264 if (err) { 5265 NL_SET_ERR_MSG_MOD(extack, "Failed to update routes to trap packets"); 5266 goto err_out; 5267 } 5268 } else if (!old_nhgi->adj_index_valid && new_nhgi->adj_index_valid) { 5269 /* The old adjacency index is invalid, while the new one is. 5270 * Iterate over all the routes using the group and change them 5271 * to forward packets using the new valid index. 5272 */ 5273 err = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, old_nh_grp); 5274 if (err) { 5275 NL_SET_ERR_MSG_MOD(extack, "Failed to update routes to forward packets"); 5276 goto err_out; 5277 } 5278 } 5279 5280 /* Make sure the flags are set / cleared based on the new nexthop group 5281 * information. 5282 */ 5283 mlxsw_sp_nexthop_obj_group_offload_refresh(mlxsw_sp, old_nh_grp); 5284 5285 /* At this point 'nh_grp' is just a shell that is not used by anyone 5286 * and its nexthop group info is the old info that was just replaced 5287 * with the new one. Remove it. 5288 */ 5289 nh_grp->can_destroy = true; 5290 mlxsw_sp_nexthop_obj_group_destroy(mlxsw_sp, nh_grp); 5291 5292 return 0; 5293 5294 err_out: 5295 old_nhgi->nh_grp = old_nh_grp; 5296 nh_grp->nhgi = new_nhgi; 5297 new_nhgi->nh_grp = nh_grp; 5298 old_nh_grp->nhgi = old_nhgi; 5299 return err; 5300 } 5301 5302 static int mlxsw_sp_nexthop_obj_new(struct mlxsw_sp *mlxsw_sp, 5303 struct nh_notifier_info *info) 5304 { 5305 struct mlxsw_sp_nexthop_group *nh_grp, *old_nh_grp; 5306 struct netlink_ext_ack *extack = info->extack; 5307 int err; 5308 5309 nh_grp = mlxsw_sp_nexthop_obj_group_create(mlxsw_sp, info); 5310 if (IS_ERR(nh_grp)) 5311 return PTR_ERR(nh_grp); 5312 5313 old_nh_grp = mlxsw_sp_nexthop_obj_group_lookup(mlxsw_sp, info->id); 5314 if (!old_nh_grp) 5315 err = mlxsw_sp_nexthop_obj_group_add(mlxsw_sp, nh_grp); 5316 else 5317 err = mlxsw_sp_nexthop_obj_group_replace(mlxsw_sp, nh_grp, 5318 old_nh_grp, extack); 5319 5320 if (err) { 5321 nh_grp->can_destroy = true; 5322 mlxsw_sp_nexthop_obj_group_destroy(mlxsw_sp, nh_grp); 5323 } 5324 5325 return err; 5326 } 5327 5328 static void mlxsw_sp_nexthop_obj_del(struct mlxsw_sp *mlxsw_sp, 5329 struct nh_notifier_info *info) 5330 { 5331 struct mlxsw_sp_nexthop_group *nh_grp; 5332 5333 nh_grp = mlxsw_sp_nexthop_obj_group_lookup(mlxsw_sp, info->id); 5334 if (!nh_grp) 5335 return; 5336 5337 nh_grp->can_destroy = true; 5338 mlxsw_sp_nexthop_group_remove(mlxsw_sp, nh_grp); 5339 5340 /* If the group still has routes using it, then defer the delete 5341 * operation until the last route using it is deleted. 5342 */ 5343 if (!list_empty(&nh_grp->fib_list)) 5344 return; 5345 mlxsw_sp_nexthop_obj_group_destroy(mlxsw_sp, nh_grp); 5346 } 5347 5348 static int mlxsw_sp_nexthop_obj_bucket_query(struct mlxsw_sp *mlxsw_sp, 5349 u32 adj_index, char *ratr_pl) 5350 { 5351 MLXSW_REG_ZERO(ratr, ratr_pl); 5352 mlxsw_reg_ratr_op_set(ratr_pl, MLXSW_REG_RATR_OP_QUERY_READ); 5353 mlxsw_reg_ratr_adjacency_index_low_set(ratr_pl, adj_index); 5354 mlxsw_reg_ratr_adjacency_index_high_set(ratr_pl, adj_index >> 16); 5355 5356 return mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ratr), ratr_pl); 5357 } 5358 5359 static int mlxsw_sp_nexthop_obj_bucket_compare(char *ratr_pl, char *ratr_pl_new) 5360 { 5361 /* Clear the opcode and activity on both the old and new payload as 5362 * they are irrelevant for the comparison. 5363 */ 5364 mlxsw_reg_ratr_op_set(ratr_pl, MLXSW_REG_RATR_OP_QUERY_READ); 5365 mlxsw_reg_ratr_a_set(ratr_pl, 0); 5366 mlxsw_reg_ratr_op_set(ratr_pl_new, MLXSW_REG_RATR_OP_QUERY_READ); 5367 mlxsw_reg_ratr_a_set(ratr_pl_new, 0); 5368 5369 /* If the contents of the adjacency entry are consistent with the 5370 * replacement request, then replacement was successful. 5371 */ 5372 if (!memcmp(ratr_pl, ratr_pl_new, MLXSW_REG_RATR_LEN)) 5373 return 0; 5374 5375 return -EINVAL; 5376 } 5377 5378 static int 5379 mlxsw_sp_nexthop_obj_bucket_adj_update(struct mlxsw_sp *mlxsw_sp, 5380 struct mlxsw_sp_nexthop *nh, 5381 struct nh_notifier_info *info) 5382 { 5383 u16 bucket_index = info->nh_res_bucket->bucket_index; 5384 struct netlink_ext_ack *extack = info->extack; 5385 bool force = info->nh_res_bucket->force; 5386 char ratr_pl_new[MLXSW_REG_RATR_LEN]; 5387 char ratr_pl[MLXSW_REG_RATR_LEN]; 5388 u32 adj_index; 5389 int err; 5390 5391 /* No point in trying an atomic replacement if the idle timer interval 5392 * is smaller than the interval in which we query and clear activity. 5393 */ 5394 if (!force && info->nh_res_bucket->idle_timer_ms < 5395 MLXSW_SP_NH_GRP_ACTIVITY_UPDATE_INTERVAL) 5396 force = true; 5397 5398 adj_index = nh->nhgi->adj_index + bucket_index; 5399 err = mlxsw_sp_nexthop_update(mlxsw_sp, adj_index, nh, force, ratr_pl); 5400 if (err) { 5401 NL_SET_ERR_MSG_MOD(extack, "Failed to overwrite nexthop bucket"); 5402 return err; 5403 } 5404 5405 if (!force) { 5406 err = mlxsw_sp_nexthop_obj_bucket_query(mlxsw_sp, adj_index, 5407 ratr_pl_new); 5408 if (err) { 5409 NL_SET_ERR_MSG_MOD(extack, "Failed to query nexthop bucket state after replacement. State might be inconsistent"); 5410 return err; 5411 } 5412 5413 err = mlxsw_sp_nexthop_obj_bucket_compare(ratr_pl, ratr_pl_new); 5414 if (err) { 5415 NL_SET_ERR_MSG_MOD(extack, "Nexthop bucket was not replaced because it was active during replacement"); 5416 return err; 5417 } 5418 } 5419 5420 nh->update = 0; 5421 nh->offloaded = 1; 5422 mlxsw_sp_nexthop_bucket_offload_refresh(mlxsw_sp, nh, bucket_index); 5423 5424 return 0; 5425 } 5426 5427 static int mlxsw_sp_nexthop_obj_bucket_replace(struct mlxsw_sp *mlxsw_sp, 5428 struct nh_notifier_info *info) 5429 { 5430 u16 bucket_index = info->nh_res_bucket->bucket_index; 5431 struct netlink_ext_ack *extack = info->extack; 5432 struct mlxsw_sp_nexthop_group_info *nhgi; 5433 struct nh_notifier_single_info *nh_obj; 5434 struct mlxsw_sp_nexthop_group *nh_grp; 5435 struct mlxsw_sp_nexthop *nh; 5436 int err; 5437 5438 nh_grp = mlxsw_sp_nexthop_obj_group_lookup(mlxsw_sp, info->id); 5439 if (!nh_grp) { 5440 NL_SET_ERR_MSG_MOD(extack, "Nexthop group was not found"); 5441 return -EINVAL; 5442 } 5443 5444 nhgi = nh_grp->nhgi; 5445 5446 if (bucket_index >= nhgi->count) { 5447 NL_SET_ERR_MSG_MOD(extack, "Nexthop bucket index out of range"); 5448 return -EINVAL; 5449 } 5450 5451 nh = &nhgi->nexthops[bucket_index]; 5452 mlxsw_sp_nexthop_obj_fini(mlxsw_sp, nh); 5453 5454 nh_obj = &info->nh_res_bucket->new_nh; 5455 err = mlxsw_sp_nexthop_obj_init(mlxsw_sp, nh_grp, nh, nh_obj, 1); 5456 if (err) { 5457 NL_SET_ERR_MSG_MOD(extack, "Failed to initialize nexthop object for nexthop bucket replacement"); 5458 goto err_nexthop_obj_init; 5459 } 5460 5461 err = mlxsw_sp_nexthop_obj_bucket_adj_update(mlxsw_sp, nh, info); 5462 if (err) 5463 goto err_nexthop_obj_bucket_adj_update; 5464 5465 return 0; 5466 5467 err_nexthop_obj_bucket_adj_update: 5468 mlxsw_sp_nexthop_obj_fini(mlxsw_sp, nh); 5469 err_nexthop_obj_init: 5470 nh_obj = &info->nh_res_bucket->old_nh; 5471 mlxsw_sp_nexthop_obj_init(mlxsw_sp, nh_grp, nh, nh_obj, 1); 5472 /* The old adjacency entry was not overwritten */ 5473 nh->update = 0; 5474 nh->offloaded = 1; 5475 return err; 5476 } 5477 5478 static int mlxsw_sp_nexthop_obj_event(struct notifier_block *nb, 5479 unsigned long event, void *ptr) 5480 { 5481 struct nh_notifier_info *info = ptr; 5482 struct mlxsw_sp_router *router; 5483 int err = 0; 5484 5485 router = container_of(nb, struct mlxsw_sp_router, nexthop_nb); 5486 err = mlxsw_sp_nexthop_obj_validate(router->mlxsw_sp, event, info); 5487 if (err) 5488 goto out; 5489 5490 mutex_lock(&router->lock); 5491 5492 switch (event) { 5493 case NEXTHOP_EVENT_REPLACE: 5494 err = mlxsw_sp_nexthop_obj_new(router->mlxsw_sp, info); 5495 break; 5496 case NEXTHOP_EVENT_DEL: 5497 mlxsw_sp_nexthop_obj_del(router->mlxsw_sp, info); 5498 break; 5499 case NEXTHOP_EVENT_BUCKET_REPLACE: 5500 err = mlxsw_sp_nexthop_obj_bucket_replace(router->mlxsw_sp, 5501 info); 5502 break; 5503 default: 5504 break; 5505 } 5506 5507 mutex_unlock(&router->lock); 5508 5509 out: 5510 return notifier_from_errno(err); 5511 } 5512 5513 static bool mlxsw_sp_fi_is_gateway(const struct mlxsw_sp *mlxsw_sp, 5514 struct fib_info *fi) 5515 { 5516 const struct fib_nh *nh = fib_info_nh(fi, 0); 5517 5518 return nh->fib_nh_gw_family || 5519 mlxsw_sp_nexthop4_ipip_type(mlxsw_sp, nh, NULL); 5520 } 5521 5522 static int 5523 mlxsw_sp_nexthop4_group_info_init(struct mlxsw_sp *mlxsw_sp, 5524 struct mlxsw_sp_nexthop_group *nh_grp) 5525 { 5526 unsigned int nhs = fib_info_num_path(nh_grp->ipv4.fi); 5527 struct mlxsw_sp_nexthop_group_info *nhgi; 5528 struct mlxsw_sp_nexthop *nh; 5529 int err, i; 5530 5531 nhgi = kzalloc(struct_size(nhgi, nexthops, nhs), GFP_KERNEL); 5532 if (!nhgi) 5533 return -ENOMEM; 5534 nh_grp->nhgi = nhgi; 5535 nhgi->nh_grp = nh_grp; 5536 nhgi->gateway = mlxsw_sp_fi_is_gateway(mlxsw_sp, nh_grp->ipv4.fi); 5537 nhgi->count = nhs; 5538 for (i = 0; i < nhgi->count; i++) { 5539 struct fib_nh *fib_nh; 5540 5541 nh = &nhgi->nexthops[i]; 5542 fib_nh = fib_info_nh(nh_grp->ipv4.fi, i); 5543 err = mlxsw_sp_nexthop4_init(mlxsw_sp, nh_grp, nh, fib_nh); 5544 if (err) 5545 goto err_nexthop4_init; 5546 } 5547 err = mlxsw_sp_nexthop_group_inc(mlxsw_sp); 5548 if (err) 5549 goto err_group_inc; 5550 err = mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp); 5551 if (err) 5552 goto err_group_refresh; 5553 5554 return 0; 5555 5556 err_group_refresh: 5557 mlxsw_sp_nexthop_group_dec(mlxsw_sp); 5558 err_group_inc: 5559 i = nhgi->count; 5560 err_nexthop4_init: 5561 for (i--; i >= 0; i--) { 5562 nh = &nhgi->nexthops[i]; 5563 mlxsw_sp_nexthop4_fini(mlxsw_sp, nh); 5564 } 5565 kfree(nhgi); 5566 return err; 5567 } 5568 5569 static void 5570 mlxsw_sp_nexthop4_group_info_fini(struct mlxsw_sp *mlxsw_sp, 5571 struct mlxsw_sp_nexthop_group *nh_grp) 5572 { 5573 struct mlxsw_sp_nexthop_group_info *nhgi = nh_grp->nhgi; 5574 int i; 5575 5576 mlxsw_sp_nexthop_group_dec(mlxsw_sp); 5577 for (i = nhgi->count - 1; i >= 0; i--) { 5578 struct mlxsw_sp_nexthop *nh = &nhgi->nexthops[i]; 5579 5580 mlxsw_sp_nexthop4_fini(mlxsw_sp, nh); 5581 } 5582 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp); 5583 WARN_ON_ONCE(nhgi->adj_index_valid); 5584 kfree(nhgi); 5585 } 5586 5587 static struct mlxsw_sp_nexthop_group * 5588 mlxsw_sp_nexthop4_group_create(struct mlxsw_sp *mlxsw_sp, struct fib_info *fi) 5589 { 5590 struct mlxsw_sp_nexthop_group *nh_grp; 5591 int err; 5592 5593 nh_grp = kzalloc(sizeof(*nh_grp), GFP_KERNEL); 5594 if (!nh_grp) 5595 return ERR_PTR(-ENOMEM); 5596 INIT_LIST_HEAD(&nh_grp->vr_list); 5597 err = rhashtable_init(&nh_grp->vr_ht, 5598 &mlxsw_sp_nexthop_group_vr_ht_params); 5599 if (err) 5600 goto err_nexthop_group_vr_ht_init; 5601 INIT_LIST_HEAD(&nh_grp->fib_list); 5602 nh_grp->type = MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4; 5603 nh_grp->ipv4.fi = fi; 5604 fib_info_hold(fi); 5605 5606 err = mlxsw_sp_nexthop4_group_info_init(mlxsw_sp, nh_grp); 5607 if (err) 5608 goto err_nexthop_group_info_init; 5609 5610 err = mlxsw_sp_nexthop_group_insert(mlxsw_sp, nh_grp); 5611 if (err) 5612 goto err_nexthop_group_insert; 5613 5614 nh_grp->can_destroy = true; 5615 5616 return nh_grp; 5617 5618 err_nexthop_group_insert: 5619 mlxsw_sp_nexthop4_group_info_fini(mlxsw_sp, nh_grp); 5620 err_nexthop_group_info_init: 5621 fib_info_put(fi); 5622 rhashtable_destroy(&nh_grp->vr_ht); 5623 err_nexthop_group_vr_ht_init: 5624 kfree(nh_grp); 5625 return ERR_PTR(err); 5626 } 5627 5628 static void 5629 mlxsw_sp_nexthop4_group_destroy(struct mlxsw_sp *mlxsw_sp, 5630 struct mlxsw_sp_nexthop_group *nh_grp) 5631 { 5632 if (!nh_grp->can_destroy) 5633 return; 5634 mlxsw_sp_nexthop_group_remove(mlxsw_sp, nh_grp); 5635 mlxsw_sp_nexthop4_group_info_fini(mlxsw_sp, nh_grp); 5636 fib_info_put(nh_grp->ipv4.fi); 5637 WARN_ON_ONCE(!list_empty(&nh_grp->vr_list)); 5638 rhashtable_destroy(&nh_grp->vr_ht); 5639 kfree(nh_grp); 5640 } 5641 5642 static int mlxsw_sp_nexthop4_group_get(struct mlxsw_sp *mlxsw_sp, 5643 struct mlxsw_sp_fib_entry *fib_entry, 5644 struct fib_info *fi) 5645 { 5646 struct mlxsw_sp_nexthop_group *nh_grp; 5647 5648 if (fi->nh) { 5649 nh_grp = mlxsw_sp_nexthop_obj_group_lookup(mlxsw_sp, 5650 fi->nh->id); 5651 if (WARN_ON_ONCE(!nh_grp)) 5652 return -EINVAL; 5653 goto out; 5654 } 5655 5656 nh_grp = mlxsw_sp_nexthop4_group_lookup(mlxsw_sp, fi); 5657 if (!nh_grp) { 5658 nh_grp = mlxsw_sp_nexthop4_group_create(mlxsw_sp, fi); 5659 if (IS_ERR(nh_grp)) 5660 return PTR_ERR(nh_grp); 5661 } 5662 out: 5663 list_add_tail(&fib_entry->nexthop_group_node, &nh_grp->fib_list); 5664 fib_entry->nh_group = nh_grp; 5665 return 0; 5666 } 5667 5668 static void mlxsw_sp_nexthop4_group_put(struct mlxsw_sp *mlxsw_sp, 5669 struct mlxsw_sp_fib_entry *fib_entry) 5670 { 5671 struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group; 5672 5673 list_del(&fib_entry->nexthop_group_node); 5674 if (!list_empty(&nh_grp->fib_list)) 5675 return; 5676 5677 if (nh_grp->type == MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ) { 5678 mlxsw_sp_nexthop_obj_group_destroy(mlxsw_sp, nh_grp); 5679 return; 5680 } 5681 5682 mlxsw_sp_nexthop4_group_destroy(mlxsw_sp, nh_grp); 5683 } 5684 5685 static bool 5686 mlxsw_sp_fib4_entry_should_offload(const struct mlxsw_sp_fib_entry *fib_entry) 5687 { 5688 struct mlxsw_sp_fib4_entry *fib4_entry; 5689 5690 fib4_entry = container_of(fib_entry, struct mlxsw_sp_fib4_entry, 5691 common); 5692 return !fib4_entry->dscp; 5693 } 5694 5695 static bool 5696 mlxsw_sp_fib_entry_should_offload(const struct mlxsw_sp_fib_entry *fib_entry) 5697 { 5698 struct mlxsw_sp_nexthop_group *nh_group = fib_entry->nh_group; 5699 5700 switch (fib_entry->fib_node->fib->proto) { 5701 case MLXSW_SP_L3_PROTO_IPV4: 5702 if (!mlxsw_sp_fib4_entry_should_offload(fib_entry)) 5703 return false; 5704 break; 5705 case MLXSW_SP_L3_PROTO_IPV6: 5706 break; 5707 } 5708 5709 switch (fib_entry->type) { 5710 case MLXSW_SP_FIB_ENTRY_TYPE_REMOTE: 5711 return !!nh_group->nhgi->adj_index_valid; 5712 case MLXSW_SP_FIB_ENTRY_TYPE_LOCAL: 5713 return !!mlxsw_sp_nhgi_rif(nh_group->nhgi); 5714 case MLXSW_SP_FIB_ENTRY_TYPE_BLACKHOLE: 5715 case MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP: 5716 case MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP: 5717 return true; 5718 default: 5719 return false; 5720 } 5721 } 5722 5723 static struct mlxsw_sp_nexthop * 5724 mlxsw_sp_rt6_nexthop(struct mlxsw_sp_nexthop_group *nh_grp, 5725 const struct mlxsw_sp_rt6 *mlxsw_sp_rt6) 5726 { 5727 int i; 5728 5729 for (i = 0; i < nh_grp->nhgi->count; i++) { 5730 struct mlxsw_sp_nexthop *nh = &nh_grp->nhgi->nexthops[i]; 5731 struct net_device *dev = mlxsw_sp_nexthop_dev(nh); 5732 struct fib6_info *rt = mlxsw_sp_rt6->rt; 5733 5734 if (dev && dev == rt->fib6_nh->fib_nh_dev && 5735 ipv6_addr_equal((const struct in6_addr *) &nh->gw_addr, 5736 &rt->fib6_nh->fib_nh_gw6)) 5737 return nh; 5738 } 5739 5740 return NULL; 5741 } 5742 5743 static void 5744 mlxsw_sp_fib4_offload_failed_flag_set(struct mlxsw_sp *mlxsw_sp, 5745 struct fib_entry_notifier_info *fen_info) 5746 { 5747 u32 *p_dst = (u32 *) &fen_info->dst; 5748 struct fib_rt_info fri; 5749 5750 fri.fi = fen_info->fi; 5751 fri.tb_id = fen_info->tb_id; 5752 fri.dst = cpu_to_be32(*p_dst); 5753 fri.dst_len = fen_info->dst_len; 5754 fri.dscp = fen_info->dscp; 5755 fri.type = fen_info->type; 5756 fri.offload = false; 5757 fri.trap = false; 5758 fri.offload_failed = true; 5759 fib_alias_hw_flags_set(mlxsw_sp_net(mlxsw_sp), &fri); 5760 } 5761 5762 static void 5763 mlxsw_sp_fib4_entry_hw_flags_set(struct mlxsw_sp *mlxsw_sp, 5764 struct mlxsw_sp_fib_entry *fib_entry) 5765 { 5766 u32 *p_dst = (u32 *) fib_entry->fib_node->key.addr; 5767 int dst_len = fib_entry->fib_node->key.prefix_len; 5768 struct mlxsw_sp_fib4_entry *fib4_entry; 5769 struct fib_rt_info fri; 5770 bool should_offload; 5771 5772 should_offload = mlxsw_sp_fib_entry_should_offload(fib_entry); 5773 fib4_entry = container_of(fib_entry, struct mlxsw_sp_fib4_entry, 5774 common); 5775 fri.fi = fib4_entry->fi; 5776 fri.tb_id = fib4_entry->tb_id; 5777 fri.dst = cpu_to_be32(*p_dst); 5778 fri.dst_len = dst_len; 5779 fri.dscp = fib4_entry->dscp; 5780 fri.type = fib4_entry->type; 5781 fri.offload = should_offload; 5782 fri.trap = !should_offload; 5783 fri.offload_failed = false; 5784 fib_alias_hw_flags_set(mlxsw_sp_net(mlxsw_sp), &fri); 5785 } 5786 5787 static void 5788 mlxsw_sp_fib4_entry_hw_flags_clear(struct mlxsw_sp *mlxsw_sp, 5789 struct mlxsw_sp_fib_entry *fib_entry) 5790 { 5791 u32 *p_dst = (u32 *) fib_entry->fib_node->key.addr; 5792 int dst_len = fib_entry->fib_node->key.prefix_len; 5793 struct mlxsw_sp_fib4_entry *fib4_entry; 5794 struct fib_rt_info fri; 5795 5796 fib4_entry = container_of(fib_entry, struct mlxsw_sp_fib4_entry, 5797 common); 5798 fri.fi = fib4_entry->fi; 5799 fri.tb_id = fib4_entry->tb_id; 5800 fri.dst = cpu_to_be32(*p_dst); 5801 fri.dst_len = dst_len; 5802 fri.dscp = fib4_entry->dscp; 5803 fri.type = fib4_entry->type; 5804 fri.offload = false; 5805 fri.trap = false; 5806 fri.offload_failed = false; 5807 fib_alias_hw_flags_set(mlxsw_sp_net(mlxsw_sp), &fri); 5808 } 5809 5810 #if IS_ENABLED(CONFIG_IPV6) 5811 static void 5812 mlxsw_sp_fib6_offload_failed_flag_set(struct mlxsw_sp *mlxsw_sp, 5813 struct fib6_info **rt_arr, 5814 unsigned int nrt6) 5815 { 5816 int i; 5817 5818 /* In IPv6 a multipath route is represented using multiple routes, so 5819 * we need to set the flags on all of them. 5820 */ 5821 for (i = 0; i < nrt6; i++) 5822 fib6_info_hw_flags_set(mlxsw_sp_net(mlxsw_sp), rt_arr[i], 5823 false, false, true); 5824 } 5825 #else 5826 static void 5827 mlxsw_sp_fib6_offload_failed_flag_set(struct mlxsw_sp *mlxsw_sp, 5828 struct fib6_info **rt_arr, 5829 unsigned int nrt6) 5830 { 5831 } 5832 #endif 5833 5834 #if IS_ENABLED(CONFIG_IPV6) 5835 static void 5836 mlxsw_sp_fib6_entry_hw_flags_set(struct mlxsw_sp *mlxsw_sp, 5837 struct mlxsw_sp_fib_entry *fib_entry) 5838 { 5839 struct mlxsw_sp_fib6_entry *fib6_entry; 5840 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 5841 bool should_offload; 5842 5843 should_offload = mlxsw_sp_fib_entry_should_offload(fib_entry); 5844 5845 /* In IPv6 a multipath route is represented using multiple routes, so 5846 * we need to set the flags on all of them. 5847 */ 5848 fib6_entry = container_of(fib_entry, struct mlxsw_sp_fib6_entry, 5849 common); 5850 list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) 5851 fib6_info_hw_flags_set(mlxsw_sp_net(mlxsw_sp), mlxsw_sp_rt6->rt, 5852 should_offload, !should_offload, false); 5853 } 5854 #else 5855 static void 5856 mlxsw_sp_fib6_entry_hw_flags_set(struct mlxsw_sp *mlxsw_sp, 5857 struct mlxsw_sp_fib_entry *fib_entry) 5858 { 5859 } 5860 #endif 5861 5862 #if IS_ENABLED(CONFIG_IPV6) 5863 static void 5864 mlxsw_sp_fib6_entry_hw_flags_clear(struct mlxsw_sp *mlxsw_sp, 5865 struct mlxsw_sp_fib_entry *fib_entry) 5866 { 5867 struct mlxsw_sp_fib6_entry *fib6_entry; 5868 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 5869 5870 fib6_entry = container_of(fib_entry, struct mlxsw_sp_fib6_entry, 5871 common); 5872 list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) 5873 fib6_info_hw_flags_set(mlxsw_sp_net(mlxsw_sp), mlxsw_sp_rt6->rt, 5874 false, false, false); 5875 } 5876 #else 5877 static void 5878 mlxsw_sp_fib6_entry_hw_flags_clear(struct mlxsw_sp *mlxsw_sp, 5879 struct mlxsw_sp_fib_entry *fib_entry) 5880 { 5881 } 5882 #endif 5883 5884 static void 5885 mlxsw_sp_fib_entry_hw_flags_set(struct mlxsw_sp *mlxsw_sp, 5886 struct mlxsw_sp_fib_entry *fib_entry) 5887 { 5888 switch (fib_entry->fib_node->fib->proto) { 5889 case MLXSW_SP_L3_PROTO_IPV4: 5890 mlxsw_sp_fib4_entry_hw_flags_set(mlxsw_sp, fib_entry); 5891 break; 5892 case MLXSW_SP_L3_PROTO_IPV6: 5893 mlxsw_sp_fib6_entry_hw_flags_set(mlxsw_sp, fib_entry); 5894 break; 5895 } 5896 } 5897 5898 static void 5899 mlxsw_sp_fib_entry_hw_flags_clear(struct mlxsw_sp *mlxsw_sp, 5900 struct mlxsw_sp_fib_entry *fib_entry) 5901 { 5902 switch (fib_entry->fib_node->fib->proto) { 5903 case MLXSW_SP_L3_PROTO_IPV4: 5904 mlxsw_sp_fib4_entry_hw_flags_clear(mlxsw_sp, fib_entry); 5905 break; 5906 case MLXSW_SP_L3_PROTO_IPV6: 5907 mlxsw_sp_fib6_entry_hw_flags_clear(mlxsw_sp, fib_entry); 5908 break; 5909 } 5910 } 5911 5912 static void 5913 mlxsw_sp_fib_entry_hw_flags_refresh(struct mlxsw_sp *mlxsw_sp, 5914 struct mlxsw_sp_fib_entry *fib_entry, 5915 enum mlxsw_reg_ralue_op op) 5916 { 5917 switch (op) { 5918 case MLXSW_REG_RALUE_OP_WRITE_WRITE: 5919 mlxsw_sp_fib_entry_hw_flags_set(mlxsw_sp, fib_entry); 5920 break; 5921 case MLXSW_REG_RALUE_OP_WRITE_DELETE: 5922 mlxsw_sp_fib_entry_hw_flags_clear(mlxsw_sp, fib_entry); 5923 break; 5924 default: 5925 break; 5926 } 5927 } 5928 5929 static void 5930 mlxsw_sp_fib_entry_ralue_pack(char *ralue_pl, 5931 const struct mlxsw_sp_fib_entry *fib_entry, 5932 enum mlxsw_reg_ralue_op op) 5933 { 5934 struct mlxsw_sp_fib *fib = fib_entry->fib_node->fib; 5935 enum mlxsw_reg_ralxx_protocol proto; 5936 u32 *p_dip; 5937 5938 proto = (enum mlxsw_reg_ralxx_protocol) fib->proto; 5939 5940 switch (fib->proto) { 5941 case MLXSW_SP_L3_PROTO_IPV4: 5942 p_dip = (u32 *) fib_entry->fib_node->key.addr; 5943 mlxsw_reg_ralue_pack4(ralue_pl, proto, op, fib->vr->id, 5944 fib_entry->fib_node->key.prefix_len, 5945 *p_dip); 5946 break; 5947 case MLXSW_SP_L3_PROTO_IPV6: 5948 mlxsw_reg_ralue_pack6(ralue_pl, proto, op, fib->vr->id, 5949 fib_entry->fib_node->key.prefix_len, 5950 fib_entry->fib_node->key.addr); 5951 break; 5952 } 5953 } 5954 5955 static int mlxsw_sp_fib_entry_op_remote(struct mlxsw_sp *mlxsw_sp, 5956 struct mlxsw_sp_fib_entry *fib_entry, 5957 enum mlxsw_reg_ralue_op op) 5958 { 5959 struct mlxsw_sp_nexthop_group *nh_group = fib_entry->nh_group; 5960 struct mlxsw_sp_nexthop_group_info *nhgi = nh_group->nhgi; 5961 char ralue_pl[MLXSW_REG_RALUE_LEN]; 5962 enum mlxsw_reg_ralue_trap_action trap_action; 5963 u16 trap_id = 0; 5964 u32 adjacency_index = 0; 5965 u16 ecmp_size = 0; 5966 5967 /* In case the nexthop group adjacency index is valid, use it 5968 * with provided ECMP size. Otherwise, setup trap and pass 5969 * traffic to kernel. 5970 */ 5971 if (mlxsw_sp_fib_entry_should_offload(fib_entry)) { 5972 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_NOP; 5973 adjacency_index = nhgi->adj_index; 5974 ecmp_size = nhgi->ecmp_size; 5975 } else if (!nhgi->adj_index_valid && nhgi->count && 5976 mlxsw_sp_nhgi_rif(nhgi)) { 5977 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_NOP; 5978 adjacency_index = mlxsw_sp->router->adj_trap_index; 5979 ecmp_size = 1; 5980 } else { 5981 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_TRAP; 5982 trap_id = MLXSW_TRAP_ID_RTR_INGRESS0; 5983 } 5984 5985 mlxsw_sp_fib_entry_ralue_pack(ralue_pl, fib_entry, op); 5986 mlxsw_reg_ralue_act_remote_pack(ralue_pl, trap_action, trap_id, 5987 adjacency_index, ecmp_size); 5988 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl); 5989 } 5990 5991 static int mlxsw_sp_fib_entry_op_local(struct mlxsw_sp *mlxsw_sp, 5992 struct mlxsw_sp_fib_entry *fib_entry, 5993 enum mlxsw_reg_ralue_op op) 5994 { 5995 struct mlxsw_sp_rif *rif = mlxsw_sp_nhgi_rif(fib_entry->nh_group->nhgi); 5996 enum mlxsw_reg_ralue_trap_action trap_action; 5997 char ralue_pl[MLXSW_REG_RALUE_LEN]; 5998 u16 trap_id = 0; 5999 u16 rif_index = 0; 6000 6001 if (mlxsw_sp_fib_entry_should_offload(fib_entry)) { 6002 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_NOP; 6003 rif_index = rif->rif_index; 6004 } else { 6005 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_TRAP; 6006 trap_id = MLXSW_TRAP_ID_RTR_INGRESS0; 6007 } 6008 6009 mlxsw_sp_fib_entry_ralue_pack(ralue_pl, fib_entry, op); 6010 mlxsw_reg_ralue_act_local_pack(ralue_pl, trap_action, trap_id, 6011 rif_index); 6012 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl); 6013 } 6014 6015 static int mlxsw_sp_fib_entry_op_trap(struct mlxsw_sp *mlxsw_sp, 6016 struct mlxsw_sp_fib_entry *fib_entry, 6017 enum mlxsw_reg_ralue_op op) 6018 { 6019 char ralue_pl[MLXSW_REG_RALUE_LEN]; 6020 6021 mlxsw_sp_fib_entry_ralue_pack(ralue_pl, fib_entry, op); 6022 mlxsw_reg_ralue_act_ip2me_pack(ralue_pl); 6023 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl); 6024 } 6025 6026 static int mlxsw_sp_fib_entry_op_blackhole(struct mlxsw_sp *mlxsw_sp, 6027 struct mlxsw_sp_fib_entry *fib_entry, 6028 enum mlxsw_reg_ralue_op op) 6029 { 6030 enum mlxsw_reg_ralue_trap_action trap_action; 6031 char ralue_pl[MLXSW_REG_RALUE_LEN]; 6032 6033 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_DISCARD_ERROR; 6034 mlxsw_sp_fib_entry_ralue_pack(ralue_pl, fib_entry, op); 6035 mlxsw_reg_ralue_act_local_pack(ralue_pl, trap_action, 0, 0); 6036 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl); 6037 } 6038 6039 static int 6040 mlxsw_sp_fib_entry_op_unreachable(struct mlxsw_sp *mlxsw_sp, 6041 struct mlxsw_sp_fib_entry *fib_entry, 6042 enum mlxsw_reg_ralue_op op) 6043 { 6044 enum mlxsw_reg_ralue_trap_action trap_action; 6045 char ralue_pl[MLXSW_REG_RALUE_LEN]; 6046 u16 trap_id; 6047 6048 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_TRAP; 6049 trap_id = MLXSW_TRAP_ID_RTR_INGRESS1; 6050 6051 mlxsw_sp_fib_entry_ralue_pack(ralue_pl, fib_entry, op); 6052 mlxsw_reg_ralue_act_local_pack(ralue_pl, trap_action, trap_id, 0); 6053 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl); 6054 } 6055 6056 static int 6057 mlxsw_sp_fib_entry_op_ipip_decap(struct mlxsw_sp *mlxsw_sp, 6058 struct mlxsw_sp_fib_entry *fib_entry, 6059 enum mlxsw_reg_ralue_op op) 6060 { 6061 struct mlxsw_sp_ipip_entry *ipip_entry = fib_entry->decap.ipip_entry; 6062 const struct mlxsw_sp_ipip_ops *ipip_ops; 6063 char ralue_pl[MLXSW_REG_RALUE_LEN]; 6064 int err; 6065 6066 if (WARN_ON(!ipip_entry)) 6067 return -EINVAL; 6068 6069 ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt]; 6070 err = ipip_ops->decap_config(mlxsw_sp, ipip_entry, 6071 fib_entry->decap.tunnel_index); 6072 if (err) 6073 return err; 6074 6075 mlxsw_sp_fib_entry_ralue_pack(ralue_pl, fib_entry, op); 6076 mlxsw_reg_ralue_act_ip2me_tun_pack(ralue_pl, 6077 fib_entry->decap.tunnel_index); 6078 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl); 6079 } 6080 6081 static int mlxsw_sp_fib_entry_op_nve_decap(struct mlxsw_sp *mlxsw_sp, 6082 struct mlxsw_sp_fib_entry *fib_entry, 6083 enum mlxsw_reg_ralue_op op) 6084 { 6085 char ralue_pl[MLXSW_REG_RALUE_LEN]; 6086 6087 mlxsw_sp_fib_entry_ralue_pack(ralue_pl, fib_entry, op); 6088 mlxsw_reg_ralue_act_ip2me_tun_pack(ralue_pl, 6089 fib_entry->decap.tunnel_index); 6090 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl); 6091 } 6092 6093 static int __mlxsw_sp_fib_entry_op(struct mlxsw_sp *mlxsw_sp, 6094 struct mlxsw_sp_fib_entry *fib_entry, 6095 enum mlxsw_reg_ralue_op op) 6096 { 6097 switch (fib_entry->type) { 6098 case MLXSW_SP_FIB_ENTRY_TYPE_REMOTE: 6099 return mlxsw_sp_fib_entry_op_remote(mlxsw_sp, fib_entry, op); 6100 case MLXSW_SP_FIB_ENTRY_TYPE_LOCAL: 6101 return mlxsw_sp_fib_entry_op_local(mlxsw_sp, fib_entry, op); 6102 case MLXSW_SP_FIB_ENTRY_TYPE_TRAP: 6103 return mlxsw_sp_fib_entry_op_trap(mlxsw_sp, fib_entry, op); 6104 case MLXSW_SP_FIB_ENTRY_TYPE_BLACKHOLE: 6105 return mlxsw_sp_fib_entry_op_blackhole(mlxsw_sp, fib_entry, op); 6106 case MLXSW_SP_FIB_ENTRY_TYPE_UNREACHABLE: 6107 return mlxsw_sp_fib_entry_op_unreachable(mlxsw_sp, fib_entry, 6108 op); 6109 case MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP: 6110 return mlxsw_sp_fib_entry_op_ipip_decap(mlxsw_sp, 6111 fib_entry, op); 6112 case MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP: 6113 return mlxsw_sp_fib_entry_op_nve_decap(mlxsw_sp, fib_entry, op); 6114 } 6115 return -EINVAL; 6116 } 6117 6118 static int mlxsw_sp_fib_entry_op(struct mlxsw_sp *mlxsw_sp, 6119 struct mlxsw_sp_fib_entry *fib_entry, 6120 enum mlxsw_reg_ralue_op op) 6121 { 6122 int err = __mlxsw_sp_fib_entry_op(mlxsw_sp, fib_entry, op); 6123 6124 if (err) 6125 return err; 6126 6127 mlxsw_sp_fib_entry_hw_flags_refresh(mlxsw_sp, fib_entry, op); 6128 6129 return err; 6130 } 6131 6132 static int mlxsw_sp_fib_entry_update(struct mlxsw_sp *mlxsw_sp, 6133 struct mlxsw_sp_fib_entry *fib_entry) 6134 { 6135 return mlxsw_sp_fib_entry_op(mlxsw_sp, fib_entry, 6136 MLXSW_REG_RALUE_OP_WRITE_WRITE); 6137 } 6138 6139 static int mlxsw_sp_fib_entry_del(struct mlxsw_sp *mlxsw_sp, 6140 struct mlxsw_sp_fib_entry *fib_entry) 6141 { 6142 return mlxsw_sp_fib_entry_op(mlxsw_sp, fib_entry, 6143 MLXSW_REG_RALUE_OP_WRITE_DELETE); 6144 } 6145 6146 static int 6147 mlxsw_sp_fib4_entry_type_set(struct mlxsw_sp *mlxsw_sp, 6148 const struct fib_entry_notifier_info *fen_info, 6149 struct mlxsw_sp_fib_entry *fib_entry) 6150 { 6151 struct mlxsw_sp_nexthop_group_info *nhgi = fib_entry->nh_group->nhgi; 6152 union mlxsw_sp_l3addr dip = { .addr4 = htonl(fen_info->dst) }; 6153 struct mlxsw_sp_router *router = mlxsw_sp->router; 6154 u32 tb_id = mlxsw_sp_fix_tb_id(fen_info->tb_id); 6155 int ifindex = nhgi->nexthops[0].ifindex; 6156 struct mlxsw_sp_ipip_entry *ipip_entry; 6157 6158 switch (fen_info->type) { 6159 case RTN_LOCAL: 6160 ipip_entry = mlxsw_sp_ipip_entry_find_by_decap(mlxsw_sp, ifindex, 6161 MLXSW_SP_L3_PROTO_IPV4, dip); 6162 if (ipip_entry && ipip_entry->ol_dev->flags & IFF_UP) { 6163 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP; 6164 return mlxsw_sp_fib_entry_decap_init(mlxsw_sp, 6165 fib_entry, 6166 ipip_entry); 6167 } 6168 if (mlxsw_sp_router_nve_is_decap(mlxsw_sp, tb_id, 6169 MLXSW_SP_L3_PROTO_IPV4, 6170 &dip)) { 6171 u32 tunnel_index; 6172 6173 tunnel_index = router->nve_decap_config.tunnel_index; 6174 fib_entry->decap.tunnel_index = tunnel_index; 6175 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP; 6176 return 0; 6177 } 6178 fallthrough; 6179 case RTN_BROADCAST: 6180 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP; 6181 return 0; 6182 case RTN_BLACKHOLE: 6183 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_BLACKHOLE; 6184 return 0; 6185 case RTN_UNREACHABLE: 6186 case RTN_PROHIBIT: 6187 /* Packets hitting these routes need to be trapped, but 6188 * can do so with a lower priority than packets directed 6189 * at the host, so use action type local instead of trap. 6190 */ 6191 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_UNREACHABLE; 6192 return 0; 6193 case RTN_UNICAST: 6194 if (nhgi->gateway) 6195 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_REMOTE; 6196 else 6197 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL; 6198 return 0; 6199 default: 6200 return -EINVAL; 6201 } 6202 } 6203 6204 static void 6205 mlxsw_sp_fib_entry_type_unset(struct mlxsw_sp *mlxsw_sp, 6206 struct mlxsw_sp_fib_entry *fib_entry) 6207 { 6208 switch (fib_entry->type) { 6209 case MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP: 6210 mlxsw_sp_fib_entry_decap_fini(mlxsw_sp, fib_entry); 6211 break; 6212 default: 6213 break; 6214 } 6215 } 6216 6217 static void 6218 mlxsw_sp_fib4_entry_type_unset(struct mlxsw_sp *mlxsw_sp, 6219 struct mlxsw_sp_fib4_entry *fib4_entry) 6220 { 6221 mlxsw_sp_fib_entry_type_unset(mlxsw_sp, &fib4_entry->common); 6222 } 6223 6224 static struct mlxsw_sp_fib4_entry * 6225 mlxsw_sp_fib4_entry_create(struct mlxsw_sp *mlxsw_sp, 6226 struct mlxsw_sp_fib_node *fib_node, 6227 const struct fib_entry_notifier_info *fen_info) 6228 { 6229 struct mlxsw_sp_fib4_entry *fib4_entry; 6230 struct mlxsw_sp_fib_entry *fib_entry; 6231 int err; 6232 6233 fib4_entry = kzalloc(sizeof(*fib4_entry), GFP_KERNEL); 6234 if (!fib4_entry) 6235 return ERR_PTR(-ENOMEM); 6236 fib_entry = &fib4_entry->common; 6237 6238 err = mlxsw_sp_nexthop4_group_get(mlxsw_sp, fib_entry, fen_info->fi); 6239 if (err) 6240 goto err_nexthop4_group_get; 6241 6242 err = mlxsw_sp_nexthop_group_vr_link(fib_entry->nh_group, 6243 fib_node->fib); 6244 if (err) 6245 goto err_nexthop_group_vr_link; 6246 6247 err = mlxsw_sp_fib4_entry_type_set(mlxsw_sp, fen_info, fib_entry); 6248 if (err) 6249 goto err_fib4_entry_type_set; 6250 6251 fib4_entry->fi = fen_info->fi; 6252 fib_info_hold(fib4_entry->fi); 6253 fib4_entry->tb_id = fen_info->tb_id; 6254 fib4_entry->type = fen_info->type; 6255 fib4_entry->dscp = fen_info->dscp; 6256 6257 fib_entry->fib_node = fib_node; 6258 6259 return fib4_entry; 6260 6261 err_fib4_entry_type_set: 6262 mlxsw_sp_nexthop_group_vr_unlink(fib_entry->nh_group, fib_node->fib); 6263 err_nexthop_group_vr_link: 6264 mlxsw_sp_nexthop4_group_put(mlxsw_sp, &fib4_entry->common); 6265 err_nexthop4_group_get: 6266 kfree(fib4_entry); 6267 return ERR_PTR(err); 6268 } 6269 6270 static void mlxsw_sp_fib4_entry_destroy(struct mlxsw_sp *mlxsw_sp, 6271 struct mlxsw_sp_fib4_entry *fib4_entry) 6272 { 6273 struct mlxsw_sp_fib_node *fib_node = fib4_entry->common.fib_node; 6274 6275 fib_info_put(fib4_entry->fi); 6276 mlxsw_sp_fib4_entry_type_unset(mlxsw_sp, fib4_entry); 6277 mlxsw_sp_nexthop_group_vr_unlink(fib4_entry->common.nh_group, 6278 fib_node->fib); 6279 mlxsw_sp_nexthop4_group_put(mlxsw_sp, &fib4_entry->common); 6280 kfree(fib4_entry); 6281 } 6282 6283 static struct mlxsw_sp_fib4_entry * 6284 mlxsw_sp_fib4_entry_lookup(struct mlxsw_sp *mlxsw_sp, 6285 const struct fib_entry_notifier_info *fen_info) 6286 { 6287 struct mlxsw_sp_fib4_entry *fib4_entry; 6288 struct mlxsw_sp_fib_node *fib_node; 6289 struct mlxsw_sp_fib *fib; 6290 struct mlxsw_sp_vr *vr; 6291 6292 vr = mlxsw_sp_vr_find(mlxsw_sp, fen_info->tb_id); 6293 if (!vr) 6294 return NULL; 6295 fib = mlxsw_sp_vr_fib(vr, MLXSW_SP_L3_PROTO_IPV4); 6296 6297 fib_node = mlxsw_sp_fib_node_lookup(fib, &fen_info->dst, 6298 sizeof(fen_info->dst), 6299 fen_info->dst_len); 6300 if (!fib_node) 6301 return NULL; 6302 6303 fib4_entry = container_of(fib_node->fib_entry, 6304 struct mlxsw_sp_fib4_entry, common); 6305 if (fib4_entry->tb_id == fen_info->tb_id && 6306 fib4_entry->dscp == fen_info->dscp && 6307 fib4_entry->type == fen_info->type && 6308 fib4_entry->fi == fen_info->fi) 6309 return fib4_entry; 6310 6311 return NULL; 6312 } 6313 6314 static const struct rhashtable_params mlxsw_sp_fib_ht_params = { 6315 .key_offset = offsetof(struct mlxsw_sp_fib_node, key), 6316 .head_offset = offsetof(struct mlxsw_sp_fib_node, ht_node), 6317 .key_len = sizeof(struct mlxsw_sp_fib_key), 6318 .automatic_shrinking = true, 6319 }; 6320 6321 static int mlxsw_sp_fib_node_insert(struct mlxsw_sp_fib *fib, 6322 struct mlxsw_sp_fib_node *fib_node) 6323 { 6324 return rhashtable_insert_fast(&fib->ht, &fib_node->ht_node, 6325 mlxsw_sp_fib_ht_params); 6326 } 6327 6328 static void mlxsw_sp_fib_node_remove(struct mlxsw_sp_fib *fib, 6329 struct mlxsw_sp_fib_node *fib_node) 6330 { 6331 rhashtable_remove_fast(&fib->ht, &fib_node->ht_node, 6332 mlxsw_sp_fib_ht_params); 6333 } 6334 6335 static struct mlxsw_sp_fib_node * 6336 mlxsw_sp_fib_node_lookup(struct mlxsw_sp_fib *fib, const void *addr, 6337 size_t addr_len, unsigned char prefix_len) 6338 { 6339 struct mlxsw_sp_fib_key key; 6340 6341 memset(&key, 0, sizeof(key)); 6342 memcpy(key.addr, addr, addr_len); 6343 key.prefix_len = prefix_len; 6344 return rhashtable_lookup_fast(&fib->ht, &key, mlxsw_sp_fib_ht_params); 6345 } 6346 6347 static struct mlxsw_sp_fib_node * 6348 mlxsw_sp_fib_node_create(struct mlxsw_sp_fib *fib, const void *addr, 6349 size_t addr_len, unsigned char prefix_len) 6350 { 6351 struct mlxsw_sp_fib_node *fib_node; 6352 6353 fib_node = kzalloc(sizeof(*fib_node), GFP_KERNEL); 6354 if (!fib_node) 6355 return NULL; 6356 6357 list_add(&fib_node->list, &fib->node_list); 6358 memcpy(fib_node->key.addr, addr, addr_len); 6359 fib_node->key.prefix_len = prefix_len; 6360 6361 return fib_node; 6362 } 6363 6364 static void mlxsw_sp_fib_node_destroy(struct mlxsw_sp_fib_node *fib_node) 6365 { 6366 list_del(&fib_node->list); 6367 kfree(fib_node); 6368 } 6369 6370 static int mlxsw_sp_fib_lpm_tree_link(struct mlxsw_sp *mlxsw_sp, 6371 struct mlxsw_sp_fib_node *fib_node) 6372 { 6373 struct mlxsw_sp_prefix_usage req_prefix_usage; 6374 struct mlxsw_sp_fib *fib = fib_node->fib; 6375 struct mlxsw_sp_lpm_tree *lpm_tree; 6376 int err; 6377 6378 lpm_tree = mlxsw_sp->router->lpm.proto_trees[fib->proto]; 6379 if (lpm_tree->prefix_ref_count[fib_node->key.prefix_len] != 0) 6380 goto out; 6381 6382 mlxsw_sp_prefix_usage_cpy(&req_prefix_usage, &lpm_tree->prefix_usage); 6383 mlxsw_sp_prefix_usage_set(&req_prefix_usage, fib_node->key.prefix_len); 6384 lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage, 6385 fib->proto); 6386 if (IS_ERR(lpm_tree)) 6387 return PTR_ERR(lpm_tree); 6388 6389 err = mlxsw_sp_vrs_lpm_tree_replace(mlxsw_sp, fib, lpm_tree); 6390 if (err) 6391 goto err_lpm_tree_replace; 6392 6393 out: 6394 lpm_tree->prefix_ref_count[fib_node->key.prefix_len]++; 6395 return 0; 6396 6397 err_lpm_tree_replace: 6398 mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree); 6399 return err; 6400 } 6401 6402 static void mlxsw_sp_fib_lpm_tree_unlink(struct mlxsw_sp *mlxsw_sp, 6403 struct mlxsw_sp_fib_node *fib_node) 6404 { 6405 struct mlxsw_sp_lpm_tree *lpm_tree = fib_node->fib->lpm_tree; 6406 struct mlxsw_sp_prefix_usage req_prefix_usage; 6407 struct mlxsw_sp_fib *fib = fib_node->fib; 6408 int err; 6409 6410 if (--lpm_tree->prefix_ref_count[fib_node->key.prefix_len] != 0) 6411 return; 6412 /* Try to construct a new LPM tree from the current prefix usage 6413 * minus the unused one. If we fail, continue using the old one. 6414 */ 6415 mlxsw_sp_prefix_usage_cpy(&req_prefix_usage, &lpm_tree->prefix_usage); 6416 mlxsw_sp_prefix_usage_clear(&req_prefix_usage, 6417 fib_node->key.prefix_len); 6418 lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage, 6419 fib->proto); 6420 if (IS_ERR(lpm_tree)) 6421 return; 6422 6423 err = mlxsw_sp_vrs_lpm_tree_replace(mlxsw_sp, fib, lpm_tree); 6424 if (err) 6425 goto err_lpm_tree_replace; 6426 6427 return; 6428 6429 err_lpm_tree_replace: 6430 mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree); 6431 } 6432 6433 static int mlxsw_sp_fib_node_init(struct mlxsw_sp *mlxsw_sp, 6434 struct mlxsw_sp_fib_node *fib_node, 6435 struct mlxsw_sp_fib *fib) 6436 { 6437 int err; 6438 6439 err = mlxsw_sp_fib_node_insert(fib, fib_node); 6440 if (err) 6441 return err; 6442 fib_node->fib = fib; 6443 6444 err = mlxsw_sp_fib_lpm_tree_link(mlxsw_sp, fib_node); 6445 if (err) 6446 goto err_fib_lpm_tree_link; 6447 6448 return 0; 6449 6450 err_fib_lpm_tree_link: 6451 fib_node->fib = NULL; 6452 mlxsw_sp_fib_node_remove(fib, fib_node); 6453 return err; 6454 } 6455 6456 static void mlxsw_sp_fib_node_fini(struct mlxsw_sp *mlxsw_sp, 6457 struct mlxsw_sp_fib_node *fib_node) 6458 { 6459 struct mlxsw_sp_fib *fib = fib_node->fib; 6460 6461 mlxsw_sp_fib_lpm_tree_unlink(mlxsw_sp, fib_node); 6462 fib_node->fib = NULL; 6463 mlxsw_sp_fib_node_remove(fib, fib_node); 6464 } 6465 6466 static struct mlxsw_sp_fib_node * 6467 mlxsw_sp_fib_node_get(struct mlxsw_sp *mlxsw_sp, u32 tb_id, const void *addr, 6468 size_t addr_len, unsigned char prefix_len, 6469 enum mlxsw_sp_l3proto proto) 6470 { 6471 struct mlxsw_sp_fib_node *fib_node; 6472 struct mlxsw_sp_fib *fib; 6473 struct mlxsw_sp_vr *vr; 6474 int err; 6475 6476 vr = mlxsw_sp_vr_get(mlxsw_sp, tb_id, NULL); 6477 if (IS_ERR(vr)) 6478 return ERR_CAST(vr); 6479 fib = mlxsw_sp_vr_fib(vr, proto); 6480 6481 fib_node = mlxsw_sp_fib_node_lookup(fib, addr, addr_len, prefix_len); 6482 if (fib_node) 6483 return fib_node; 6484 6485 fib_node = mlxsw_sp_fib_node_create(fib, addr, addr_len, prefix_len); 6486 if (!fib_node) { 6487 err = -ENOMEM; 6488 goto err_fib_node_create; 6489 } 6490 6491 err = mlxsw_sp_fib_node_init(mlxsw_sp, fib_node, fib); 6492 if (err) 6493 goto err_fib_node_init; 6494 6495 return fib_node; 6496 6497 err_fib_node_init: 6498 mlxsw_sp_fib_node_destroy(fib_node); 6499 err_fib_node_create: 6500 mlxsw_sp_vr_put(mlxsw_sp, vr); 6501 return ERR_PTR(err); 6502 } 6503 6504 static void mlxsw_sp_fib_node_put(struct mlxsw_sp *mlxsw_sp, 6505 struct mlxsw_sp_fib_node *fib_node) 6506 { 6507 struct mlxsw_sp_vr *vr = fib_node->fib->vr; 6508 6509 if (fib_node->fib_entry) 6510 return; 6511 mlxsw_sp_fib_node_fini(mlxsw_sp, fib_node); 6512 mlxsw_sp_fib_node_destroy(fib_node); 6513 mlxsw_sp_vr_put(mlxsw_sp, vr); 6514 } 6515 6516 static int mlxsw_sp_fib_node_entry_link(struct mlxsw_sp *mlxsw_sp, 6517 struct mlxsw_sp_fib_entry *fib_entry) 6518 { 6519 struct mlxsw_sp_fib_node *fib_node = fib_entry->fib_node; 6520 int err; 6521 6522 fib_node->fib_entry = fib_entry; 6523 6524 err = mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry); 6525 if (err) 6526 goto err_fib_entry_update; 6527 6528 return 0; 6529 6530 err_fib_entry_update: 6531 fib_node->fib_entry = NULL; 6532 return err; 6533 } 6534 6535 static void 6536 mlxsw_sp_fib_node_entry_unlink(struct mlxsw_sp *mlxsw_sp, 6537 struct mlxsw_sp_fib_entry *fib_entry) 6538 { 6539 struct mlxsw_sp_fib_node *fib_node = fib_entry->fib_node; 6540 6541 mlxsw_sp_fib_entry_del(mlxsw_sp, fib_entry); 6542 fib_node->fib_entry = NULL; 6543 } 6544 6545 static bool mlxsw_sp_fib4_allow_replace(struct mlxsw_sp_fib4_entry *fib4_entry) 6546 { 6547 struct mlxsw_sp_fib_node *fib_node = fib4_entry->common.fib_node; 6548 struct mlxsw_sp_fib4_entry *fib4_replaced; 6549 6550 if (!fib_node->fib_entry) 6551 return true; 6552 6553 fib4_replaced = container_of(fib_node->fib_entry, 6554 struct mlxsw_sp_fib4_entry, common); 6555 if (fib4_entry->tb_id == RT_TABLE_MAIN && 6556 fib4_replaced->tb_id == RT_TABLE_LOCAL) 6557 return false; 6558 6559 return true; 6560 } 6561 6562 static int 6563 mlxsw_sp_router_fib4_replace(struct mlxsw_sp *mlxsw_sp, 6564 const struct fib_entry_notifier_info *fen_info) 6565 { 6566 struct mlxsw_sp_fib4_entry *fib4_entry, *fib4_replaced; 6567 struct mlxsw_sp_fib_entry *replaced; 6568 struct mlxsw_sp_fib_node *fib_node; 6569 int err; 6570 6571 if (fen_info->fi->nh && 6572 !mlxsw_sp_nexthop_obj_group_lookup(mlxsw_sp, fen_info->fi->nh->id)) 6573 return 0; 6574 6575 fib_node = mlxsw_sp_fib_node_get(mlxsw_sp, fen_info->tb_id, 6576 &fen_info->dst, sizeof(fen_info->dst), 6577 fen_info->dst_len, 6578 MLXSW_SP_L3_PROTO_IPV4); 6579 if (IS_ERR(fib_node)) { 6580 dev_warn(mlxsw_sp->bus_info->dev, "Failed to get FIB node\n"); 6581 return PTR_ERR(fib_node); 6582 } 6583 6584 fib4_entry = mlxsw_sp_fib4_entry_create(mlxsw_sp, fib_node, fen_info); 6585 if (IS_ERR(fib4_entry)) { 6586 dev_warn(mlxsw_sp->bus_info->dev, "Failed to create FIB entry\n"); 6587 err = PTR_ERR(fib4_entry); 6588 goto err_fib4_entry_create; 6589 } 6590 6591 if (!mlxsw_sp_fib4_allow_replace(fib4_entry)) { 6592 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry); 6593 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 6594 return 0; 6595 } 6596 6597 replaced = fib_node->fib_entry; 6598 err = mlxsw_sp_fib_node_entry_link(mlxsw_sp, &fib4_entry->common); 6599 if (err) { 6600 dev_warn(mlxsw_sp->bus_info->dev, "Failed to link FIB entry to node\n"); 6601 goto err_fib_node_entry_link; 6602 } 6603 6604 /* Nothing to replace */ 6605 if (!replaced) 6606 return 0; 6607 6608 mlxsw_sp_fib_entry_hw_flags_clear(mlxsw_sp, replaced); 6609 fib4_replaced = container_of(replaced, struct mlxsw_sp_fib4_entry, 6610 common); 6611 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_replaced); 6612 6613 return 0; 6614 6615 err_fib_node_entry_link: 6616 fib_node->fib_entry = replaced; 6617 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry); 6618 err_fib4_entry_create: 6619 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 6620 return err; 6621 } 6622 6623 static void mlxsw_sp_router_fib4_del(struct mlxsw_sp *mlxsw_sp, 6624 struct fib_entry_notifier_info *fen_info) 6625 { 6626 struct mlxsw_sp_fib4_entry *fib4_entry; 6627 struct mlxsw_sp_fib_node *fib_node; 6628 6629 fib4_entry = mlxsw_sp_fib4_entry_lookup(mlxsw_sp, fen_info); 6630 if (!fib4_entry) 6631 return; 6632 fib_node = fib4_entry->common.fib_node; 6633 6634 mlxsw_sp_fib_node_entry_unlink(mlxsw_sp, &fib4_entry->common); 6635 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry); 6636 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 6637 } 6638 6639 static bool mlxsw_sp_fib6_rt_should_ignore(const struct fib6_info *rt) 6640 { 6641 /* Multicast routes aren't supported, so ignore them. Neighbour 6642 * Discovery packets are specifically trapped. 6643 */ 6644 if (ipv6_addr_type(&rt->fib6_dst.addr) & IPV6_ADDR_MULTICAST) 6645 return true; 6646 6647 /* Cloned routes are irrelevant in the forwarding path. */ 6648 if (rt->fib6_flags & RTF_CACHE) 6649 return true; 6650 6651 return false; 6652 } 6653 6654 static struct mlxsw_sp_rt6 *mlxsw_sp_rt6_create(struct fib6_info *rt) 6655 { 6656 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 6657 6658 mlxsw_sp_rt6 = kzalloc(sizeof(*mlxsw_sp_rt6), GFP_KERNEL); 6659 if (!mlxsw_sp_rt6) 6660 return ERR_PTR(-ENOMEM); 6661 6662 /* In case of route replace, replaced route is deleted with 6663 * no notification. Take reference to prevent accessing freed 6664 * memory. 6665 */ 6666 mlxsw_sp_rt6->rt = rt; 6667 fib6_info_hold(rt); 6668 6669 return mlxsw_sp_rt6; 6670 } 6671 6672 #if IS_ENABLED(CONFIG_IPV6) 6673 static void mlxsw_sp_rt6_release(struct fib6_info *rt) 6674 { 6675 fib6_info_release(rt); 6676 } 6677 #else 6678 static void mlxsw_sp_rt6_release(struct fib6_info *rt) 6679 { 6680 } 6681 #endif 6682 6683 static void mlxsw_sp_rt6_destroy(struct mlxsw_sp_rt6 *mlxsw_sp_rt6) 6684 { 6685 struct fib6_nh *fib6_nh = mlxsw_sp_rt6->rt->fib6_nh; 6686 6687 if (!mlxsw_sp_rt6->rt->nh) 6688 fib6_nh->fib_nh_flags &= ~RTNH_F_OFFLOAD; 6689 mlxsw_sp_rt6_release(mlxsw_sp_rt6->rt); 6690 kfree(mlxsw_sp_rt6); 6691 } 6692 6693 static struct fib6_info * 6694 mlxsw_sp_fib6_entry_rt(const struct mlxsw_sp_fib6_entry *fib6_entry) 6695 { 6696 return list_first_entry(&fib6_entry->rt6_list, struct mlxsw_sp_rt6, 6697 list)->rt; 6698 } 6699 6700 static struct mlxsw_sp_rt6 * 6701 mlxsw_sp_fib6_entry_rt_find(const struct mlxsw_sp_fib6_entry *fib6_entry, 6702 const struct fib6_info *rt) 6703 { 6704 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 6705 6706 list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) { 6707 if (mlxsw_sp_rt6->rt == rt) 6708 return mlxsw_sp_rt6; 6709 } 6710 6711 return NULL; 6712 } 6713 6714 static bool mlxsw_sp_nexthop6_ipip_type(const struct mlxsw_sp *mlxsw_sp, 6715 const struct fib6_info *rt, 6716 enum mlxsw_sp_ipip_type *ret) 6717 { 6718 return rt->fib6_nh->fib_nh_dev && 6719 mlxsw_sp_netdev_ipip_type(mlxsw_sp, rt->fib6_nh->fib_nh_dev, ret); 6720 } 6721 6722 static int mlxsw_sp_nexthop6_init(struct mlxsw_sp *mlxsw_sp, 6723 struct mlxsw_sp_nexthop_group *nh_grp, 6724 struct mlxsw_sp_nexthop *nh, 6725 const struct fib6_info *rt) 6726 { 6727 struct net_device *dev = rt->fib6_nh->fib_nh_dev; 6728 int err; 6729 6730 nh->nhgi = nh_grp->nhgi; 6731 nh->nh_weight = rt->fib6_nh->fib_nh_weight; 6732 memcpy(&nh->gw_addr, &rt->fib6_nh->fib_nh_gw6, sizeof(nh->gw_addr)); 6733 #if IS_ENABLED(CONFIG_IPV6) 6734 nh->neigh_tbl = &nd_tbl; 6735 #endif 6736 mlxsw_sp_nexthop_counter_alloc(mlxsw_sp, nh); 6737 6738 list_add_tail(&nh->router_list_node, &mlxsw_sp->router->nexthop_list); 6739 6740 if (!dev) 6741 return 0; 6742 nh->ifindex = dev->ifindex; 6743 6744 err = mlxsw_sp_nexthop_type_init(mlxsw_sp, nh, dev); 6745 if (err) 6746 goto err_nexthop_type_init; 6747 6748 return 0; 6749 6750 err_nexthop_type_init: 6751 list_del(&nh->router_list_node); 6752 mlxsw_sp_nexthop_counter_free(mlxsw_sp, nh); 6753 return err; 6754 } 6755 6756 static void mlxsw_sp_nexthop6_fini(struct mlxsw_sp *mlxsw_sp, 6757 struct mlxsw_sp_nexthop *nh) 6758 { 6759 mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh); 6760 list_del(&nh->router_list_node); 6761 mlxsw_sp_nexthop_counter_free(mlxsw_sp, nh); 6762 } 6763 6764 static bool mlxsw_sp_rt6_is_gateway(const struct mlxsw_sp *mlxsw_sp, 6765 const struct fib6_info *rt) 6766 { 6767 return rt->fib6_nh->fib_nh_gw_family || 6768 mlxsw_sp_nexthop6_ipip_type(mlxsw_sp, rt, NULL); 6769 } 6770 6771 static int 6772 mlxsw_sp_nexthop6_group_info_init(struct mlxsw_sp *mlxsw_sp, 6773 struct mlxsw_sp_nexthop_group *nh_grp, 6774 struct mlxsw_sp_fib6_entry *fib6_entry) 6775 { 6776 struct mlxsw_sp_nexthop_group_info *nhgi; 6777 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 6778 struct mlxsw_sp_nexthop *nh; 6779 int err, i; 6780 6781 nhgi = kzalloc(struct_size(nhgi, nexthops, fib6_entry->nrt6), 6782 GFP_KERNEL); 6783 if (!nhgi) 6784 return -ENOMEM; 6785 nh_grp->nhgi = nhgi; 6786 nhgi->nh_grp = nh_grp; 6787 mlxsw_sp_rt6 = list_first_entry(&fib6_entry->rt6_list, 6788 struct mlxsw_sp_rt6, list); 6789 nhgi->gateway = mlxsw_sp_rt6_is_gateway(mlxsw_sp, mlxsw_sp_rt6->rt); 6790 nhgi->count = fib6_entry->nrt6; 6791 for (i = 0; i < nhgi->count; i++) { 6792 struct fib6_info *rt = mlxsw_sp_rt6->rt; 6793 6794 nh = &nhgi->nexthops[i]; 6795 err = mlxsw_sp_nexthop6_init(mlxsw_sp, nh_grp, nh, rt); 6796 if (err) 6797 goto err_nexthop6_init; 6798 mlxsw_sp_rt6 = list_next_entry(mlxsw_sp_rt6, list); 6799 } 6800 nh_grp->nhgi = nhgi; 6801 err = mlxsw_sp_nexthop_group_inc(mlxsw_sp); 6802 if (err) 6803 goto err_group_inc; 6804 err = mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp); 6805 if (err) 6806 goto err_group_refresh; 6807 6808 return 0; 6809 6810 err_group_refresh: 6811 mlxsw_sp_nexthop_group_dec(mlxsw_sp); 6812 err_group_inc: 6813 i = nhgi->count; 6814 err_nexthop6_init: 6815 for (i--; i >= 0; i--) { 6816 nh = &nhgi->nexthops[i]; 6817 mlxsw_sp_nexthop6_fini(mlxsw_sp, nh); 6818 } 6819 kfree(nhgi); 6820 return err; 6821 } 6822 6823 static void 6824 mlxsw_sp_nexthop6_group_info_fini(struct mlxsw_sp *mlxsw_sp, 6825 struct mlxsw_sp_nexthop_group *nh_grp) 6826 { 6827 struct mlxsw_sp_nexthop_group_info *nhgi = nh_grp->nhgi; 6828 int i; 6829 6830 mlxsw_sp_nexthop_group_dec(mlxsw_sp); 6831 for (i = nhgi->count - 1; i >= 0; i--) { 6832 struct mlxsw_sp_nexthop *nh = &nhgi->nexthops[i]; 6833 6834 mlxsw_sp_nexthop6_fini(mlxsw_sp, nh); 6835 } 6836 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp); 6837 WARN_ON_ONCE(nhgi->adj_index_valid); 6838 kfree(nhgi); 6839 } 6840 6841 static struct mlxsw_sp_nexthop_group * 6842 mlxsw_sp_nexthop6_group_create(struct mlxsw_sp *mlxsw_sp, 6843 struct mlxsw_sp_fib6_entry *fib6_entry) 6844 { 6845 struct mlxsw_sp_nexthop_group *nh_grp; 6846 int err; 6847 6848 nh_grp = kzalloc(sizeof(*nh_grp), GFP_KERNEL); 6849 if (!nh_grp) 6850 return ERR_PTR(-ENOMEM); 6851 INIT_LIST_HEAD(&nh_grp->vr_list); 6852 err = rhashtable_init(&nh_grp->vr_ht, 6853 &mlxsw_sp_nexthop_group_vr_ht_params); 6854 if (err) 6855 goto err_nexthop_group_vr_ht_init; 6856 INIT_LIST_HEAD(&nh_grp->fib_list); 6857 nh_grp->type = MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6; 6858 6859 err = mlxsw_sp_nexthop6_group_info_init(mlxsw_sp, nh_grp, fib6_entry); 6860 if (err) 6861 goto err_nexthop_group_info_init; 6862 6863 err = mlxsw_sp_nexthop_group_insert(mlxsw_sp, nh_grp); 6864 if (err) 6865 goto err_nexthop_group_insert; 6866 6867 nh_grp->can_destroy = true; 6868 6869 return nh_grp; 6870 6871 err_nexthop_group_insert: 6872 mlxsw_sp_nexthop6_group_info_fini(mlxsw_sp, nh_grp); 6873 err_nexthop_group_info_init: 6874 rhashtable_destroy(&nh_grp->vr_ht); 6875 err_nexthop_group_vr_ht_init: 6876 kfree(nh_grp); 6877 return ERR_PTR(err); 6878 } 6879 6880 static void 6881 mlxsw_sp_nexthop6_group_destroy(struct mlxsw_sp *mlxsw_sp, 6882 struct mlxsw_sp_nexthop_group *nh_grp) 6883 { 6884 if (!nh_grp->can_destroy) 6885 return; 6886 mlxsw_sp_nexthop_group_remove(mlxsw_sp, nh_grp); 6887 mlxsw_sp_nexthop6_group_info_fini(mlxsw_sp, nh_grp); 6888 WARN_ON_ONCE(!list_empty(&nh_grp->vr_list)); 6889 rhashtable_destroy(&nh_grp->vr_ht); 6890 kfree(nh_grp); 6891 } 6892 6893 static int mlxsw_sp_nexthop6_group_get(struct mlxsw_sp *mlxsw_sp, 6894 struct mlxsw_sp_fib6_entry *fib6_entry) 6895 { 6896 struct fib6_info *rt = mlxsw_sp_fib6_entry_rt(fib6_entry); 6897 struct mlxsw_sp_nexthop_group *nh_grp; 6898 6899 if (rt->nh) { 6900 nh_grp = mlxsw_sp_nexthop_obj_group_lookup(mlxsw_sp, 6901 rt->nh->id); 6902 if (WARN_ON_ONCE(!nh_grp)) 6903 return -EINVAL; 6904 goto out; 6905 } 6906 6907 nh_grp = mlxsw_sp_nexthop6_group_lookup(mlxsw_sp, fib6_entry); 6908 if (!nh_grp) { 6909 nh_grp = mlxsw_sp_nexthop6_group_create(mlxsw_sp, fib6_entry); 6910 if (IS_ERR(nh_grp)) 6911 return PTR_ERR(nh_grp); 6912 } 6913 6914 /* The route and the nexthop are described by the same struct, so we 6915 * need to the update the nexthop offload indication for the new route. 6916 */ 6917 __mlxsw_sp_nexthop6_group_offload_refresh(nh_grp, fib6_entry); 6918 6919 out: 6920 list_add_tail(&fib6_entry->common.nexthop_group_node, 6921 &nh_grp->fib_list); 6922 fib6_entry->common.nh_group = nh_grp; 6923 6924 return 0; 6925 } 6926 6927 static void mlxsw_sp_nexthop6_group_put(struct mlxsw_sp *mlxsw_sp, 6928 struct mlxsw_sp_fib_entry *fib_entry) 6929 { 6930 struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group; 6931 6932 list_del(&fib_entry->nexthop_group_node); 6933 if (!list_empty(&nh_grp->fib_list)) 6934 return; 6935 6936 if (nh_grp->type == MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ) { 6937 mlxsw_sp_nexthop_obj_group_destroy(mlxsw_sp, nh_grp); 6938 return; 6939 } 6940 6941 mlxsw_sp_nexthop6_group_destroy(mlxsw_sp, nh_grp); 6942 } 6943 6944 static int 6945 mlxsw_sp_nexthop6_group_update(struct mlxsw_sp *mlxsw_sp, 6946 struct mlxsw_sp_fib6_entry *fib6_entry) 6947 { 6948 struct mlxsw_sp_nexthop_group *old_nh_grp = fib6_entry->common.nh_group; 6949 struct mlxsw_sp_fib_node *fib_node = fib6_entry->common.fib_node; 6950 int err; 6951 6952 mlxsw_sp_nexthop_group_vr_unlink(old_nh_grp, fib_node->fib); 6953 fib6_entry->common.nh_group = NULL; 6954 list_del(&fib6_entry->common.nexthop_group_node); 6955 6956 err = mlxsw_sp_nexthop6_group_get(mlxsw_sp, fib6_entry); 6957 if (err) 6958 goto err_nexthop6_group_get; 6959 6960 err = mlxsw_sp_nexthop_group_vr_link(fib6_entry->common.nh_group, 6961 fib_node->fib); 6962 if (err) 6963 goto err_nexthop_group_vr_link; 6964 6965 /* In case this entry is offloaded, then the adjacency index 6966 * currently associated with it in the device's table is that 6967 * of the old group. Start using the new one instead. 6968 */ 6969 err = mlxsw_sp_fib_entry_update(mlxsw_sp, &fib6_entry->common); 6970 if (err) 6971 goto err_fib_entry_update; 6972 6973 if (list_empty(&old_nh_grp->fib_list)) 6974 mlxsw_sp_nexthop6_group_destroy(mlxsw_sp, old_nh_grp); 6975 6976 return 0; 6977 6978 err_fib_entry_update: 6979 mlxsw_sp_nexthop_group_vr_unlink(fib6_entry->common.nh_group, 6980 fib_node->fib); 6981 err_nexthop_group_vr_link: 6982 mlxsw_sp_nexthop6_group_put(mlxsw_sp, &fib6_entry->common); 6983 err_nexthop6_group_get: 6984 list_add_tail(&fib6_entry->common.nexthop_group_node, 6985 &old_nh_grp->fib_list); 6986 fib6_entry->common.nh_group = old_nh_grp; 6987 mlxsw_sp_nexthop_group_vr_link(old_nh_grp, fib_node->fib); 6988 return err; 6989 } 6990 6991 static int 6992 mlxsw_sp_fib6_entry_nexthop_add(struct mlxsw_sp *mlxsw_sp, 6993 struct mlxsw_sp_fib6_entry *fib6_entry, 6994 struct fib6_info **rt_arr, unsigned int nrt6) 6995 { 6996 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 6997 int err, i; 6998 6999 for (i = 0; i < nrt6; i++) { 7000 mlxsw_sp_rt6 = mlxsw_sp_rt6_create(rt_arr[i]); 7001 if (IS_ERR(mlxsw_sp_rt6)) { 7002 err = PTR_ERR(mlxsw_sp_rt6); 7003 goto err_rt6_unwind; 7004 } 7005 7006 list_add_tail(&mlxsw_sp_rt6->list, &fib6_entry->rt6_list); 7007 fib6_entry->nrt6++; 7008 } 7009 7010 err = mlxsw_sp_nexthop6_group_update(mlxsw_sp, fib6_entry); 7011 if (err) 7012 goto err_rt6_unwind; 7013 7014 return 0; 7015 7016 err_rt6_unwind: 7017 for (; i > 0; i--) { 7018 fib6_entry->nrt6--; 7019 mlxsw_sp_rt6 = list_last_entry(&fib6_entry->rt6_list, 7020 struct mlxsw_sp_rt6, list); 7021 list_del(&mlxsw_sp_rt6->list); 7022 mlxsw_sp_rt6_destroy(mlxsw_sp_rt6); 7023 } 7024 return err; 7025 } 7026 7027 static void 7028 mlxsw_sp_fib6_entry_nexthop_del(struct mlxsw_sp *mlxsw_sp, 7029 struct mlxsw_sp_fib6_entry *fib6_entry, 7030 struct fib6_info **rt_arr, unsigned int nrt6) 7031 { 7032 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 7033 int i; 7034 7035 for (i = 0; i < nrt6; i++) { 7036 mlxsw_sp_rt6 = mlxsw_sp_fib6_entry_rt_find(fib6_entry, 7037 rt_arr[i]); 7038 if (WARN_ON_ONCE(!mlxsw_sp_rt6)) 7039 continue; 7040 7041 fib6_entry->nrt6--; 7042 list_del(&mlxsw_sp_rt6->list); 7043 mlxsw_sp_rt6_destroy(mlxsw_sp_rt6); 7044 } 7045 7046 mlxsw_sp_nexthop6_group_update(mlxsw_sp, fib6_entry); 7047 } 7048 7049 static int 7050 mlxsw_sp_fib6_entry_type_set_local(struct mlxsw_sp *mlxsw_sp, 7051 struct mlxsw_sp_fib_entry *fib_entry, 7052 const struct fib6_info *rt) 7053 { 7054 struct mlxsw_sp_nexthop_group_info *nhgi = fib_entry->nh_group->nhgi; 7055 union mlxsw_sp_l3addr dip = { .addr6 = rt->fib6_dst.addr }; 7056 u32 tb_id = mlxsw_sp_fix_tb_id(rt->fib6_table->tb6_id); 7057 struct mlxsw_sp_router *router = mlxsw_sp->router; 7058 int ifindex = nhgi->nexthops[0].ifindex; 7059 struct mlxsw_sp_ipip_entry *ipip_entry; 7060 7061 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP; 7062 ipip_entry = mlxsw_sp_ipip_entry_find_by_decap(mlxsw_sp, ifindex, 7063 MLXSW_SP_L3_PROTO_IPV6, 7064 dip); 7065 7066 if (ipip_entry && ipip_entry->ol_dev->flags & IFF_UP) { 7067 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP; 7068 return mlxsw_sp_fib_entry_decap_init(mlxsw_sp, fib_entry, 7069 ipip_entry); 7070 } 7071 if (mlxsw_sp_router_nve_is_decap(mlxsw_sp, tb_id, 7072 MLXSW_SP_L3_PROTO_IPV6, &dip)) { 7073 u32 tunnel_index; 7074 7075 tunnel_index = router->nve_decap_config.tunnel_index; 7076 fib_entry->decap.tunnel_index = tunnel_index; 7077 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP; 7078 } 7079 7080 return 0; 7081 } 7082 7083 static int mlxsw_sp_fib6_entry_type_set(struct mlxsw_sp *mlxsw_sp, 7084 struct mlxsw_sp_fib_entry *fib_entry, 7085 const struct fib6_info *rt) 7086 { 7087 if (rt->fib6_flags & RTF_LOCAL) 7088 return mlxsw_sp_fib6_entry_type_set_local(mlxsw_sp, fib_entry, 7089 rt); 7090 if (rt->fib6_flags & RTF_ANYCAST) 7091 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP; 7092 else if (rt->fib6_type == RTN_BLACKHOLE) 7093 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_BLACKHOLE; 7094 else if (rt->fib6_flags & RTF_REJECT) 7095 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_UNREACHABLE; 7096 else if (fib_entry->nh_group->nhgi->gateway) 7097 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_REMOTE; 7098 else 7099 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL; 7100 7101 return 0; 7102 } 7103 7104 static void 7105 mlxsw_sp_fib6_entry_rt_destroy_all(struct mlxsw_sp_fib6_entry *fib6_entry) 7106 { 7107 struct mlxsw_sp_rt6 *mlxsw_sp_rt6, *tmp; 7108 7109 list_for_each_entry_safe(mlxsw_sp_rt6, tmp, &fib6_entry->rt6_list, 7110 list) { 7111 fib6_entry->nrt6--; 7112 list_del(&mlxsw_sp_rt6->list); 7113 mlxsw_sp_rt6_destroy(mlxsw_sp_rt6); 7114 } 7115 } 7116 7117 static struct mlxsw_sp_fib6_entry * 7118 mlxsw_sp_fib6_entry_create(struct mlxsw_sp *mlxsw_sp, 7119 struct mlxsw_sp_fib_node *fib_node, 7120 struct fib6_info **rt_arr, unsigned int nrt6) 7121 { 7122 struct mlxsw_sp_fib6_entry *fib6_entry; 7123 struct mlxsw_sp_fib_entry *fib_entry; 7124 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 7125 int err, i; 7126 7127 fib6_entry = kzalloc(sizeof(*fib6_entry), GFP_KERNEL); 7128 if (!fib6_entry) 7129 return ERR_PTR(-ENOMEM); 7130 fib_entry = &fib6_entry->common; 7131 7132 INIT_LIST_HEAD(&fib6_entry->rt6_list); 7133 7134 for (i = 0; i < nrt6; i++) { 7135 mlxsw_sp_rt6 = mlxsw_sp_rt6_create(rt_arr[i]); 7136 if (IS_ERR(mlxsw_sp_rt6)) { 7137 err = PTR_ERR(mlxsw_sp_rt6); 7138 goto err_rt6_unwind; 7139 } 7140 list_add_tail(&mlxsw_sp_rt6->list, &fib6_entry->rt6_list); 7141 fib6_entry->nrt6++; 7142 } 7143 7144 err = mlxsw_sp_nexthop6_group_get(mlxsw_sp, fib6_entry); 7145 if (err) 7146 goto err_rt6_unwind; 7147 7148 err = mlxsw_sp_nexthop_group_vr_link(fib_entry->nh_group, 7149 fib_node->fib); 7150 if (err) 7151 goto err_nexthop_group_vr_link; 7152 7153 err = mlxsw_sp_fib6_entry_type_set(mlxsw_sp, fib_entry, rt_arr[0]); 7154 if (err) 7155 goto err_fib6_entry_type_set; 7156 7157 fib_entry->fib_node = fib_node; 7158 7159 return fib6_entry; 7160 7161 err_fib6_entry_type_set: 7162 mlxsw_sp_nexthop_group_vr_unlink(fib_entry->nh_group, fib_node->fib); 7163 err_nexthop_group_vr_link: 7164 mlxsw_sp_nexthop6_group_put(mlxsw_sp, fib_entry); 7165 err_rt6_unwind: 7166 for (; i > 0; i--) { 7167 fib6_entry->nrt6--; 7168 mlxsw_sp_rt6 = list_last_entry(&fib6_entry->rt6_list, 7169 struct mlxsw_sp_rt6, list); 7170 list_del(&mlxsw_sp_rt6->list); 7171 mlxsw_sp_rt6_destroy(mlxsw_sp_rt6); 7172 } 7173 kfree(fib6_entry); 7174 return ERR_PTR(err); 7175 } 7176 7177 static void 7178 mlxsw_sp_fib6_entry_type_unset(struct mlxsw_sp *mlxsw_sp, 7179 struct mlxsw_sp_fib6_entry *fib6_entry) 7180 { 7181 mlxsw_sp_fib_entry_type_unset(mlxsw_sp, &fib6_entry->common); 7182 } 7183 7184 static void mlxsw_sp_fib6_entry_destroy(struct mlxsw_sp *mlxsw_sp, 7185 struct mlxsw_sp_fib6_entry *fib6_entry) 7186 { 7187 struct mlxsw_sp_fib_node *fib_node = fib6_entry->common.fib_node; 7188 7189 mlxsw_sp_fib6_entry_type_unset(mlxsw_sp, fib6_entry); 7190 mlxsw_sp_nexthop_group_vr_unlink(fib6_entry->common.nh_group, 7191 fib_node->fib); 7192 mlxsw_sp_nexthop6_group_put(mlxsw_sp, &fib6_entry->common); 7193 mlxsw_sp_fib6_entry_rt_destroy_all(fib6_entry); 7194 WARN_ON(fib6_entry->nrt6); 7195 kfree(fib6_entry); 7196 } 7197 7198 static struct mlxsw_sp_fib6_entry * 7199 mlxsw_sp_fib6_entry_lookup(struct mlxsw_sp *mlxsw_sp, 7200 const struct fib6_info *rt) 7201 { 7202 struct mlxsw_sp_fib6_entry *fib6_entry; 7203 struct mlxsw_sp_fib_node *fib_node; 7204 struct mlxsw_sp_fib *fib; 7205 struct fib6_info *cmp_rt; 7206 struct mlxsw_sp_vr *vr; 7207 7208 vr = mlxsw_sp_vr_find(mlxsw_sp, rt->fib6_table->tb6_id); 7209 if (!vr) 7210 return NULL; 7211 fib = mlxsw_sp_vr_fib(vr, MLXSW_SP_L3_PROTO_IPV6); 7212 7213 fib_node = mlxsw_sp_fib_node_lookup(fib, &rt->fib6_dst.addr, 7214 sizeof(rt->fib6_dst.addr), 7215 rt->fib6_dst.plen); 7216 if (!fib_node) 7217 return NULL; 7218 7219 fib6_entry = container_of(fib_node->fib_entry, 7220 struct mlxsw_sp_fib6_entry, common); 7221 cmp_rt = mlxsw_sp_fib6_entry_rt(fib6_entry); 7222 if (rt->fib6_table->tb6_id == cmp_rt->fib6_table->tb6_id && 7223 rt->fib6_metric == cmp_rt->fib6_metric && 7224 mlxsw_sp_fib6_entry_rt_find(fib6_entry, rt)) 7225 return fib6_entry; 7226 7227 return NULL; 7228 } 7229 7230 static bool mlxsw_sp_fib6_allow_replace(struct mlxsw_sp_fib6_entry *fib6_entry) 7231 { 7232 struct mlxsw_sp_fib_node *fib_node = fib6_entry->common.fib_node; 7233 struct mlxsw_sp_fib6_entry *fib6_replaced; 7234 struct fib6_info *rt, *rt_replaced; 7235 7236 if (!fib_node->fib_entry) 7237 return true; 7238 7239 fib6_replaced = container_of(fib_node->fib_entry, 7240 struct mlxsw_sp_fib6_entry, 7241 common); 7242 rt = mlxsw_sp_fib6_entry_rt(fib6_entry); 7243 rt_replaced = mlxsw_sp_fib6_entry_rt(fib6_replaced); 7244 if (rt->fib6_table->tb6_id == RT_TABLE_MAIN && 7245 rt_replaced->fib6_table->tb6_id == RT_TABLE_LOCAL) 7246 return false; 7247 7248 return true; 7249 } 7250 7251 static int mlxsw_sp_router_fib6_replace(struct mlxsw_sp *mlxsw_sp, 7252 struct fib6_info **rt_arr, 7253 unsigned int nrt6) 7254 { 7255 struct mlxsw_sp_fib6_entry *fib6_entry, *fib6_replaced; 7256 struct mlxsw_sp_fib_entry *replaced; 7257 struct mlxsw_sp_fib_node *fib_node; 7258 struct fib6_info *rt = rt_arr[0]; 7259 int err; 7260 7261 if (rt->fib6_src.plen) 7262 return -EINVAL; 7263 7264 if (mlxsw_sp_fib6_rt_should_ignore(rt)) 7265 return 0; 7266 7267 if (rt->nh && !mlxsw_sp_nexthop_obj_group_lookup(mlxsw_sp, rt->nh->id)) 7268 return 0; 7269 7270 fib_node = mlxsw_sp_fib_node_get(mlxsw_sp, rt->fib6_table->tb6_id, 7271 &rt->fib6_dst.addr, 7272 sizeof(rt->fib6_dst.addr), 7273 rt->fib6_dst.plen, 7274 MLXSW_SP_L3_PROTO_IPV6); 7275 if (IS_ERR(fib_node)) 7276 return PTR_ERR(fib_node); 7277 7278 fib6_entry = mlxsw_sp_fib6_entry_create(mlxsw_sp, fib_node, rt_arr, 7279 nrt6); 7280 if (IS_ERR(fib6_entry)) { 7281 err = PTR_ERR(fib6_entry); 7282 goto err_fib6_entry_create; 7283 } 7284 7285 if (!mlxsw_sp_fib6_allow_replace(fib6_entry)) { 7286 mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry); 7287 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 7288 return 0; 7289 } 7290 7291 replaced = fib_node->fib_entry; 7292 err = mlxsw_sp_fib_node_entry_link(mlxsw_sp, &fib6_entry->common); 7293 if (err) 7294 goto err_fib_node_entry_link; 7295 7296 /* Nothing to replace */ 7297 if (!replaced) 7298 return 0; 7299 7300 mlxsw_sp_fib_entry_hw_flags_clear(mlxsw_sp, replaced); 7301 fib6_replaced = container_of(replaced, struct mlxsw_sp_fib6_entry, 7302 common); 7303 mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_replaced); 7304 7305 return 0; 7306 7307 err_fib_node_entry_link: 7308 fib_node->fib_entry = replaced; 7309 mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry); 7310 err_fib6_entry_create: 7311 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 7312 return err; 7313 } 7314 7315 static int mlxsw_sp_router_fib6_append(struct mlxsw_sp *mlxsw_sp, 7316 struct fib6_info **rt_arr, 7317 unsigned int nrt6) 7318 { 7319 struct mlxsw_sp_fib6_entry *fib6_entry; 7320 struct mlxsw_sp_fib_node *fib_node; 7321 struct fib6_info *rt = rt_arr[0]; 7322 int err; 7323 7324 if (rt->fib6_src.plen) 7325 return -EINVAL; 7326 7327 if (mlxsw_sp_fib6_rt_should_ignore(rt)) 7328 return 0; 7329 7330 fib_node = mlxsw_sp_fib_node_get(mlxsw_sp, rt->fib6_table->tb6_id, 7331 &rt->fib6_dst.addr, 7332 sizeof(rt->fib6_dst.addr), 7333 rt->fib6_dst.plen, 7334 MLXSW_SP_L3_PROTO_IPV6); 7335 if (IS_ERR(fib_node)) 7336 return PTR_ERR(fib_node); 7337 7338 if (WARN_ON_ONCE(!fib_node->fib_entry)) { 7339 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 7340 return -EINVAL; 7341 } 7342 7343 fib6_entry = container_of(fib_node->fib_entry, 7344 struct mlxsw_sp_fib6_entry, common); 7345 err = mlxsw_sp_fib6_entry_nexthop_add(mlxsw_sp, fib6_entry, rt_arr, 7346 nrt6); 7347 if (err) 7348 goto err_fib6_entry_nexthop_add; 7349 7350 return 0; 7351 7352 err_fib6_entry_nexthop_add: 7353 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 7354 return err; 7355 } 7356 7357 static void mlxsw_sp_router_fib6_del(struct mlxsw_sp *mlxsw_sp, 7358 struct fib6_info **rt_arr, 7359 unsigned int nrt6) 7360 { 7361 struct mlxsw_sp_fib6_entry *fib6_entry; 7362 struct mlxsw_sp_fib_node *fib_node; 7363 struct fib6_info *rt = rt_arr[0]; 7364 7365 if (mlxsw_sp_fib6_rt_should_ignore(rt)) 7366 return; 7367 7368 /* Multipath routes are first added to the FIB trie and only then 7369 * notified. If we vetoed the addition, we will get a delete 7370 * notification for a route we do not have. Therefore, do not warn if 7371 * route was not found. 7372 */ 7373 fib6_entry = mlxsw_sp_fib6_entry_lookup(mlxsw_sp, rt); 7374 if (!fib6_entry) 7375 return; 7376 7377 /* If not all the nexthops are deleted, then only reduce the nexthop 7378 * group. 7379 */ 7380 if (nrt6 != fib6_entry->nrt6) { 7381 mlxsw_sp_fib6_entry_nexthop_del(mlxsw_sp, fib6_entry, rt_arr, 7382 nrt6); 7383 return; 7384 } 7385 7386 fib_node = fib6_entry->common.fib_node; 7387 7388 mlxsw_sp_fib_node_entry_unlink(mlxsw_sp, &fib6_entry->common); 7389 mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry); 7390 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 7391 } 7392 7393 static struct mlxsw_sp_mr_table * 7394 mlxsw_sp_router_fibmr_family_to_table(struct mlxsw_sp_vr *vr, int family) 7395 { 7396 if (family == RTNL_FAMILY_IPMR) 7397 return vr->mr_table[MLXSW_SP_L3_PROTO_IPV4]; 7398 else 7399 return vr->mr_table[MLXSW_SP_L3_PROTO_IPV6]; 7400 } 7401 7402 static int mlxsw_sp_router_fibmr_add(struct mlxsw_sp *mlxsw_sp, 7403 struct mfc_entry_notifier_info *men_info, 7404 bool replace) 7405 { 7406 struct mlxsw_sp_mr_table *mrt; 7407 struct mlxsw_sp_vr *vr; 7408 7409 vr = mlxsw_sp_vr_get(mlxsw_sp, men_info->tb_id, NULL); 7410 if (IS_ERR(vr)) 7411 return PTR_ERR(vr); 7412 7413 mrt = mlxsw_sp_router_fibmr_family_to_table(vr, men_info->info.family); 7414 return mlxsw_sp_mr_route_add(mrt, men_info->mfc, replace); 7415 } 7416 7417 static void mlxsw_sp_router_fibmr_del(struct mlxsw_sp *mlxsw_sp, 7418 struct mfc_entry_notifier_info *men_info) 7419 { 7420 struct mlxsw_sp_mr_table *mrt; 7421 struct mlxsw_sp_vr *vr; 7422 7423 vr = mlxsw_sp_vr_find(mlxsw_sp, men_info->tb_id); 7424 if (WARN_ON(!vr)) 7425 return; 7426 7427 mrt = mlxsw_sp_router_fibmr_family_to_table(vr, men_info->info.family); 7428 mlxsw_sp_mr_route_del(mrt, men_info->mfc); 7429 mlxsw_sp_vr_put(mlxsw_sp, vr); 7430 } 7431 7432 static int 7433 mlxsw_sp_router_fibmr_vif_add(struct mlxsw_sp *mlxsw_sp, 7434 struct vif_entry_notifier_info *ven_info) 7435 { 7436 struct mlxsw_sp_mr_table *mrt; 7437 struct mlxsw_sp_rif *rif; 7438 struct mlxsw_sp_vr *vr; 7439 7440 vr = mlxsw_sp_vr_get(mlxsw_sp, ven_info->tb_id, NULL); 7441 if (IS_ERR(vr)) 7442 return PTR_ERR(vr); 7443 7444 mrt = mlxsw_sp_router_fibmr_family_to_table(vr, ven_info->info.family); 7445 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, ven_info->dev); 7446 return mlxsw_sp_mr_vif_add(mrt, ven_info->dev, 7447 ven_info->vif_index, 7448 ven_info->vif_flags, rif); 7449 } 7450 7451 static void 7452 mlxsw_sp_router_fibmr_vif_del(struct mlxsw_sp *mlxsw_sp, 7453 struct vif_entry_notifier_info *ven_info) 7454 { 7455 struct mlxsw_sp_mr_table *mrt; 7456 struct mlxsw_sp_vr *vr; 7457 7458 vr = mlxsw_sp_vr_find(mlxsw_sp, ven_info->tb_id); 7459 if (WARN_ON(!vr)) 7460 return; 7461 7462 mrt = mlxsw_sp_router_fibmr_family_to_table(vr, ven_info->info.family); 7463 mlxsw_sp_mr_vif_del(mrt, ven_info->vif_index); 7464 mlxsw_sp_vr_put(mlxsw_sp, vr); 7465 } 7466 7467 static void mlxsw_sp_fib4_node_flush(struct mlxsw_sp *mlxsw_sp, 7468 struct mlxsw_sp_fib_node *fib_node) 7469 { 7470 struct mlxsw_sp_fib4_entry *fib4_entry; 7471 7472 fib4_entry = container_of(fib_node->fib_entry, 7473 struct mlxsw_sp_fib4_entry, common); 7474 mlxsw_sp_fib_node_entry_unlink(mlxsw_sp, fib_node->fib_entry); 7475 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry); 7476 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 7477 } 7478 7479 static void mlxsw_sp_fib6_node_flush(struct mlxsw_sp *mlxsw_sp, 7480 struct mlxsw_sp_fib_node *fib_node) 7481 { 7482 struct mlxsw_sp_fib6_entry *fib6_entry; 7483 7484 fib6_entry = container_of(fib_node->fib_entry, 7485 struct mlxsw_sp_fib6_entry, common); 7486 mlxsw_sp_fib_node_entry_unlink(mlxsw_sp, fib_node->fib_entry); 7487 mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry); 7488 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 7489 } 7490 7491 static void mlxsw_sp_fib_node_flush(struct mlxsw_sp *mlxsw_sp, 7492 struct mlxsw_sp_fib_node *fib_node) 7493 { 7494 switch (fib_node->fib->proto) { 7495 case MLXSW_SP_L3_PROTO_IPV4: 7496 mlxsw_sp_fib4_node_flush(mlxsw_sp, fib_node); 7497 break; 7498 case MLXSW_SP_L3_PROTO_IPV6: 7499 mlxsw_sp_fib6_node_flush(mlxsw_sp, fib_node); 7500 break; 7501 } 7502 } 7503 7504 static void mlxsw_sp_vr_fib_flush(struct mlxsw_sp *mlxsw_sp, 7505 struct mlxsw_sp_vr *vr, 7506 enum mlxsw_sp_l3proto proto) 7507 { 7508 struct mlxsw_sp_fib *fib = mlxsw_sp_vr_fib(vr, proto); 7509 struct mlxsw_sp_fib_node *fib_node, *tmp; 7510 7511 list_for_each_entry_safe(fib_node, tmp, &fib->node_list, list) { 7512 bool do_break = &tmp->list == &fib->node_list; 7513 7514 mlxsw_sp_fib_node_flush(mlxsw_sp, fib_node); 7515 if (do_break) 7516 break; 7517 } 7518 } 7519 7520 static void mlxsw_sp_router_fib_flush(struct mlxsw_sp *mlxsw_sp) 7521 { 7522 int max_vrs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); 7523 int i, j; 7524 7525 for (i = 0; i < max_vrs; i++) { 7526 struct mlxsw_sp_vr *vr = &mlxsw_sp->router->vrs[i]; 7527 7528 if (!mlxsw_sp_vr_is_used(vr)) 7529 continue; 7530 7531 for (j = 0; j < MLXSW_SP_L3_PROTO_MAX; j++) 7532 mlxsw_sp_mr_table_flush(vr->mr_table[j]); 7533 mlxsw_sp_vr_fib_flush(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV4); 7534 7535 /* If virtual router was only used for IPv4, then it's no 7536 * longer used. 7537 */ 7538 if (!mlxsw_sp_vr_is_used(vr)) 7539 continue; 7540 mlxsw_sp_vr_fib_flush(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV6); 7541 } 7542 } 7543 7544 struct mlxsw_sp_fib6_event_work { 7545 struct fib6_info **rt_arr; 7546 unsigned int nrt6; 7547 }; 7548 7549 struct mlxsw_sp_fib_event_work { 7550 struct work_struct work; 7551 netdevice_tracker dev_tracker; 7552 union { 7553 struct mlxsw_sp_fib6_event_work fib6_work; 7554 struct fib_entry_notifier_info fen_info; 7555 struct fib_rule_notifier_info fr_info; 7556 struct fib_nh_notifier_info fnh_info; 7557 struct mfc_entry_notifier_info men_info; 7558 struct vif_entry_notifier_info ven_info; 7559 }; 7560 struct mlxsw_sp *mlxsw_sp; 7561 unsigned long event; 7562 }; 7563 7564 static int 7565 mlxsw_sp_router_fib6_work_init(struct mlxsw_sp_fib6_event_work *fib6_work, 7566 struct fib6_entry_notifier_info *fen6_info) 7567 { 7568 struct fib6_info *rt = fen6_info->rt; 7569 struct fib6_info **rt_arr; 7570 struct fib6_info *iter; 7571 unsigned int nrt6; 7572 int i = 0; 7573 7574 nrt6 = fen6_info->nsiblings + 1; 7575 7576 rt_arr = kcalloc(nrt6, sizeof(struct fib6_info *), GFP_ATOMIC); 7577 if (!rt_arr) 7578 return -ENOMEM; 7579 7580 fib6_work->rt_arr = rt_arr; 7581 fib6_work->nrt6 = nrt6; 7582 7583 rt_arr[0] = rt; 7584 fib6_info_hold(rt); 7585 7586 if (!fen6_info->nsiblings) 7587 return 0; 7588 7589 list_for_each_entry(iter, &rt->fib6_siblings, fib6_siblings) { 7590 if (i == fen6_info->nsiblings) 7591 break; 7592 7593 rt_arr[i + 1] = iter; 7594 fib6_info_hold(iter); 7595 i++; 7596 } 7597 WARN_ON_ONCE(i != fen6_info->nsiblings); 7598 7599 return 0; 7600 } 7601 7602 static void 7603 mlxsw_sp_router_fib6_work_fini(struct mlxsw_sp_fib6_event_work *fib6_work) 7604 { 7605 int i; 7606 7607 for (i = 0; i < fib6_work->nrt6; i++) 7608 mlxsw_sp_rt6_release(fib6_work->rt_arr[i]); 7609 kfree(fib6_work->rt_arr); 7610 } 7611 7612 static void mlxsw_sp_router_fib4_event_work(struct work_struct *work) 7613 { 7614 struct mlxsw_sp_fib_event_work *fib_work = 7615 container_of(work, struct mlxsw_sp_fib_event_work, work); 7616 struct mlxsw_sp *mlxsw_sp = fib_work->mlxsw_sp; 7617 int err; 7618 7619 mutex_lock(&mlxsw_sp->router->lock); 7620 mlxsw_sp_span_respin(mlxsw_sp); 7621 7622 switch (fib_work->event) { 7623 case FIB_EVENT_ENTRY_REPLACE: 7624 err = mlxsw_sp_router_fib4_replace(mlxsw_sp, 7625 &fib_work->fen_info); 7626 if (err) { 7627 dev_warn(mlxsw_sp->bus_info->dev, "FIB replace failed.\n"); 7628 mlxsw_sp_fib4_offload_failed_flag_set(mlxsw_sp, 7629 &fib_work->fen_info); 7630 } 7631 fib_info_put(fib_work->fen_info.fi); 7632 break; 7633 case FIB_EVENT_ENTRY_DEL: 7634 mlxsw_sp_router_fib4_del(mlxsw_sp, &fib_work->fen_info); 7635 fib_info_put(fib_work->fen_info.fi); 7636 break; 7637 case FIB_EVENT_NH_ADD: 7638 case FIB_EVENT_NH_DEL: 7639 mlxsw_sp_nexthop4_event(mlxsw_sp, fib_work->event, 7640 fib_work->fnh_info.fib_nh); 7641 fib_info_put(fib_work->fnh_info.fib_nh->nh_parent); 7642 break; 7643 } 7644 mutex_unlock(&mlxsw_sp->router->lock); 7645 kfree(fib_work); 7646 } 7647 7648 static void mlxsw_sp_router_fib6_event_work(struct work_struct *work) 7649 { 7650 struct mlxsw_sp_fib_event_work *fib_work = 7651 container_of(work, struct mlxsw_sp_fib_event_work, work); 7652 struct mlxsw_sp_fib6_event_work *fib6_work = &fib_work->fib6_work; 7653 struct mlxsw_sp *mlxsw_sp = fib_work->mlxsw_sp; 7654 int err; 7655 7656 mutex_lock(&mlxsw_sp->router->lock); 7657 mlxsw_sp_span_respin(mlxsw_sp); 7658 7659 switch (fib_work->event) { 7660 case FIB_EVENT_ENTRY_REPLACE: 7661 err = mlxsw_sp_router_fib6_replace(mlxsw_sp, 7662 fib6_work->rt_arr, 7663 fib6_work->nrt6); 7664 if (err) { 7665 dev_warn(mlxsw_sp->bus_info->dev, "FIB replace failed.\n"); 7666 mlxsw_sp_fib6_offload_failed_flag_set(mlxsw_sp, 7667 fib6_work->rt_arr, 7668 fib6_work->nrt6); 7669 } 7670 mlxsw_sp_router_fib6_work_fini(fib6_work); 7671 break; 7672 case FIB_EVENT_ENTRY_APPEND: 7673 err = mlxsw_sp_router_fib6_append(mlxsw_sp, 7674 fib6_work->rt_arr, 7675 fib6_work->nrt6); 7676 if (err) { 7677 dev_warn(mlxsw_sp->bus_info->dev, "FIB append failed.\n"); 7678 mlxsw_sp_fib6_offload_failed_flag_set(mlxsw_sp, 7679 fib6_work->rt_arr, 7680 fib6_work->nrt6); 7681 } 7682 mlxsw_sp_router_fib6_work_fini(fib6_work); 7683 break; 7684 case FIB_EVENT_ENTRY_DEL: 7685 mlxsw_sp_router_fib6_del(mlxsw_sp, 7686 fib6_work->rt_arr, 7687 fib6_work->nrt6); 7688 mlxsw_sp_router_fib6_work_fini(fib6_work); 7689 break; 7690 } 7691 mutex_unlock(&mlxsw_sp->router->lock); 7692 kfree(fib_work); 7693 } 7694 7695 static void mlxsw_sp_router_fibmr_event_work(struct work_struct *work) 7696 { 7697 struct mlxsw_sp_fib_event_work *fib_work = 7698 container_of(work, struct mlxsw_sp_fib_event_work, work); 7699 struct mlxsw_sp *mlxsw_sp = fib_work->mlxsw_sp; 7700 bool replace; 7701 int err; 7702 7703 rtnl_lock(); 7704 mutex_lock(&mlxsw_sp->router->lock); 7705 switch (fib_work->event) { 7706 case FIB_EVENT_ENTRY_REPLACE: 7707 case FIB_EVENT_ENTRY_ADD: 7708 replace = fib_work->event == FIB_EVENT_ENTRY_REPLACE; 7709 7710 err = mlxsw_sp_router_fibmr_add(mlxsw_sp, &fib_work->men_info, 7711 replace); 7712 if (err) 7713 dev_warn(mlxsw_sp->bus_info->dev, "MR entry add failed.\n"); 7714 mr_cache_put(fib_work->men_info.mfc); 7715 break; 7716 case FIB_EVENT_ENTRY_DEL: 7717 mlxsw_sp_router_fibmr_del(mlxsw_sp, &fib_work->men_info); 7718 mr_cache_put(fib_work->men_info.mfc); 7719 break; 7720 case FIB_EVENT_VIF_ADD: 7721 err = mlxsw_sp_router_fibmr_vif_add(mlxsw_sp, 7722 &fib_work->ven_info); 7723 if (err) 7724 dev_warn(mlxsw_sp->bus_info->dev, "MR VIF add failed.\n"); 7725 netdev_put(fib_work->ven_info.dev, &fib_work->dev_tracker); 7726 break; 7727 case FIB_EVENT_VIF_DEL: 7728 mlxsw_sp_router_fibmr_vif_del(mlxsw_sp, 7729 &fib_work->ven_info); 7730 netdev_put(fib_work->ven_info.dev, &fib_work->dev_tracker); 7731 break; 7732 } 7733 mutex_unlock(&mlxsw_sp->router->lock); 7734 rtnl_unlock(); 7735 kfree(fib_work); 7736 } 7737 7738 static void mlxsw_sp_router_fib4_event(struct mlxsw_sp_fib_event_work *fib_work, 7739 struct fib_notifier_info *info) 7740 { 7741 struct fib_entry_notifier_info *fen_info; 7742 struct fib_nh_notifier_info *fnh_info; 7743 7744 switch (fib_work->event) { 7745 case FIB_EVENT_ENTRY_REPLACE: 7746 case FIB_EVENT_ENTRY_DEL: 7747 fen_info = container_of(info, struct fib_entry_notifier_info, 7748 info); 7749 fib_work->fen_info = *fen_info; 7750 /* Take reference on fib_info to prevent it from being 7751 * freed while work is queued. Release it afterwards. 7752 */ 7753 fib_info_hold(fib_work->fen_info.fi); 7754 break; 7755 case FIB_EVENT_NH_ADD: 7756 case FIB_EVENT_NH_DEL: 7757 fnh_info = container_of(info, struct fib_nh_notifier_info, 7758 info); 7759 fib_work->fnh_info = *fnh_info; 7760 fib_info_hold(fib_work->fnh_info.fib_nh->nh_parent); 7761 break; 7762 } 7763 } 7764 7765 static int mlxsw_sp_router_fib6_event(struct mlxsw_sp_fib_event_work *fib_work, 7766 struct fib_notifier_info *info) 7767 { 7768 struct fib6_entry_notifier_info *fen6_info; 7769 int err; 7770 7771 switch (fib_work->event) { 7772 case FIB_EVENT_ENTRY_REPLACE: 7773 case FIB_EVENT_ENTRY_APPEND: 7774 case FIB_EVENT_ENTRY_DEL: 7775 fen6_info = container_of(info, struct fib6_entry_notifier_info, 7776 info); 7777 err = mlxsw_sp_router_fib6_work_init(&fib_work->fib6_work, 7778 fen6_info); 7779 if (err) 7780 return err; 7781 break; 7782 } 7783 7784 return 0; 7785 } 7786 7787 static void 7788 mlxsw_sp_router_fibmr_event(struct mlxsw_sp_fib_event_work *fib_work, 7789 struct fib_notifier_info *info) 7790 { 7791 switch (fib_work->event) { 7792 case FIB_EVENT_ENTRY_REPLACE: 7793 case FIB_EVENT_ENTRY_ADD: 7794 case FIB_EVENT_ENTRY_DEL: 7795 memcpy(&fib_work->men_info, info, sizeof(fib_work->men_info)); 7796 mr_cache_hold(fib_work->men_info.mfc); 7797 break; 7798 case FIB_EVENT_VIF_ADD: 7799 case FIB_EVENT_VIF_DEL: 7800 memcpy(&fib_work->ven_info, info, sizeof(fib_work->ven_info)); 7801 netdev_hold(fib_work->ven_info.dev, &fib_work->dev_tracker, 7802 GFP_ATOMIC); 7803 break; 7804 } 7805 } 7806 7807 static int mlxsw_sp_router_fib_rule_event(unsigned long event, 7808 struct fib_notifier_info *info, 7809 struct mlxsw_sp *mlxsw_sp) 7810 { 7811 struct netlink_ext_ack *extack = info->extack; 7812 struct fib_rule_notifier_info *fr_info; 7813 struct fib_rule *rule; 7814 int err = 0; 7815 7816 /* nothing to do at the moment */ 7817 if (event == FIB_EVENT_RULE_DEL) 7818 return 0; 7819 7820 fr_info = container_of(info, struct fib_rule_notifier_info, info); 7821 rule = fr_info->rule; 7822 7823 /* Rule only affects locally generated traffic */ 7824 if (rule->iifindex == mlxsw_sp_net(mlxsw_sp)->loopback_dev->ifindex) 7825 return 0; 7826 7827 switch (info->family) { 7828 case AF_INET: 7829 if (!fib4_rule_default(rule) && !rule->l3mdev) 7830 err = -EOPNOTSUPP; 7831 break; 7832 case AF_INET6: 7833 if (!fib6_rule_default(rule) && !rule->l3mdev) 7834 err = -EOPNOTSUPP; 7835 break; 7836 case RTNL_FAMILY_IPMR: 7837 if (!ipmr_rule_default(rule) && !rule->l3mdev) 7838 err = -EOPNOTSUPP; 7839 break; 7840 case RTNL_FAMILY_IP6MR: 7841 if (!ip6mr_rule_default(rule) && !rule->l3mdev) 7842 err = -EOPNOTSUPP; 7843 break; 7844 } 7845 7846 if (err < 0) 7847 NL_SET_ERR_MSG_MOD(extack, "FIB rules not supported"); 7848 7849 return err; 7850 } 7851 7852 /* Called with rcu_read_lock() */ 7853 static int mlxsw_sp_router_fib_event(struct notifier_block *nb, 7854 unsigned long event, void *ptr) 7855 { 7856 struct mlxsw_sp_fib_event_work *fib_work; 7857 struct fib_notifier_info *info = ptr; 7858 struct mlxsw_sp_router *router; 7859 int err; 7860 7861 if ((info->family != AF_INET && info->family != AF_INET6 && 7862 info->family != RTNL_FAMILY_IPMR && 7863 info->family != RTNL_FAMILY_IP6MR)) 7864 return NOTIFY_DONE; 7865 7866 router = container_of(nb, struct mlxsw_sp_router, fib_nb); 7867 7868 switch (event) { 7869 case FIB_EVENT_RULE_ADD: 7870 case FIB_EVENT_RULE_DEL: 7871 err = mlxsw_sp_router_fib_rule_event(event, info, 7872 router->mlxsw_sp); 7873 return notifier_from_errno(err); 7874 case FIB_EVENT_ENTRY_ADD: 7875 case FIB_EVENT_ENTRY_REPLACE: 7876 case FIB_EVENT_ENTRY_APPEND: 7877 if (info->family == AF_INET) { 7878 struct fib_entry_notifier_info *fen_info = ptr; 7879 7880 if (fen_info->fi->fib_nh_is_v6) { 7881 NL_SET_ERR_MSG_MOD(info->extack, "IPv6 gateway with IPv4 route is not supported"); 7882 return notifier_from_errno(-EINVAL); 7883 } 7884 } 7885 break; 7886 } 7887 7888 fib_work = kzalloc(sizeof(*fib_work), GFP_ATOMIC); 7889 if (!fib_work) 7890 return NOTIFY_BAD; 7891 7892 fib_work->mlxsw_sp = router->mlxsw_sp; 7893 fib_work->event = event; 7894 7895 switch (info->family) { 7896 case AF_INET: 7897 INIT_WORK(&fib_work->work, mlxsw_sp_router_fib4_event_work); 7898 mlxsw_sp_router_fib4_event(fib_work, info); 7899 break; 7900 case AF_INET6: 7901 INIT_WORK(&fib_work->work, mlxsw_sp_router_fib6_event_work); 7902 err = mlxsw_sp_router_fib6_event(fib_work, info); 7903 if (err) 7904 goto err_fib_event; 7905 break; 7906 case RTNL_FAMILY_IP6MR: 7907 case RTNL_FAMILY_IPMR: 7908 INIT_WORK(&fib_work->work, mlxsw_sp_router_fibmr_event_work); 7909 mlxsw_sp_router_fibmr_event(fib_work, info); 7910 break; 7911 } 7912 7913 mlxsw_core_schedule_work(&fib_work->work); 7914 7915 return NOTIFY_DONE; 7916 7917 err_fib_event: 7918 kfree(fib_work); 7919 return NOTIFY_BAD; 7920 } 7921 7922 static struct mlxsw_sp_rif * 7923 mlxsw_sp_rif_find_by_dev(const struct mlxsw_sp *mlxsw_sp, 7924 const struct net_device *dev) 7925 { 7926 int max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); 7927 int i; 7928 7929 for (i = 0; i < max_rifs; i++) 7930 if (mlxsw_sp->router->rifs[i] && 7931 mlxsw_sp_rif_dev_is(mlxsw_sp->router->rifs[i], dev)) 7932 return mlxsw_sp->router->rifs[i]; 7933 7934 return NULL; 7935 } 7936 7937 bool mlxsw_sp_rif_exists(struct mlxsw_sp *mlxsw_sp, 7938 const struct net_device *dev) 7939 { 7940 struct mlxsw_sp_rif *rif; 7941 7942 mutex_lock(&mlxsw_sp->router->lock); 7943 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev); 7944 mutex_unlock(&mlxsw_sp->router->lock); 7945 7946 return rif; 7947 } 7948 7949 u16 mlxsw_sp_rif_vid(struct mlxsw_sp *mlxsw_sp, const struct net_device *dev) 7950 { 7951 struct mlxsw_sp_rif *rif; 7952 u16 vid = 0; 7953 7954 mutex_lock(&mlxsw_sp->router->lock); 7955 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev); 7956 if (!rif) 7957 goto out; 7958 7959 /* We only return the VID for VLAN RIFs. Otherwise we return an 7960 * invalid value (0). 7961 */ 7962 if (rif->ops->type != MLXSW_SP_RIF_TYPE_VLAN) 7963 goto out; 7964 7965 vid = mlxsw_sp_fid_8021q_vid(rif->fid); 7966 7967 out: 7968 mutex_unlock(&mlxsw_sp->router->lock); 7969 return vid; 7970 } 7971 7972 static int mlxsw_sp_router_rif_disable(struct mlxsw_sp *mlxsw_sp, u16 rif) 7973 { 7974 char ritr_pl[MLXSW_REG_RITR_LEN]; 7975 int err; 7976 7977 mlxsw_reg_ritr_rif_pack(ritr_pl, rif); 7978 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 7979 if (err) 7980 return err; 7981 7982 mlxsw_reg_ritr_enable_set(ritr_pl, false); 7983 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 7984 } 7985 7986 static int mlxsw_sp_router_rif_made_sync(struct mlxsw_sp *mlxsw_sp, 7987 struct mlxsw_sp_rif *rif) 7988 { 7989 int err; 7990 7991 err = mlxsw_sp_neigh_rif_made_sync(mlxsw_sp, rif); 7992 if (err) 7993 return err; 7994 7995 err = mlxsw_sp_nexthop_rif_made_sync(mlxsw_sp, rif); 7996 if (err) 7997 goto err_nexthop; 7998 7999 return 0; 8000 8001 err_nexthop: 8002 mlxsw_sp_neigh_rif_gone_sync(mlxsw_sp, rif); 8003 return err; 8004 } 8005 8006 static void mlxsw_sp_router_rif_gone_sync(struct mlxsw_sp *mlxsw_sp, 8007 struct mlxsw_sp_rif *rif) 8008 { 8009 /* Signal to nexthop cleanup that the RIF is going away. */ 8010 rif->crif->rif = NULL; 8011 8012 mlxsw_sp_router_rif_disable(mlxsw_sp, rif->rif_index); 8013 mlxsw_sp_nexthop_rif_gone_sync(mlxsw_sp, rif); 8014 mlxsw_sp_neigh_rif_gone_sync(mlxsw_sp, rif); 8015 } 8016 8017 static bool __mlxsw_sp_dev_addr_list_empty(const struct net_device *dev) 8018 { 8019 struct inet6_dev *inet6_dev; 8020 struct in_device *idev; 8021 8022 idev = __in_dev_get_rcu(dev); 8023 if (idev && idev->ifa_list) 8024 return false; 8025 8026 inet6_dev = __in6_dev_get(dev); 8027 if (inet6_dev && !list_empty(&inet6_dev->addr_list)) 8028 return false; 8029 8030 return true; 8031 } 8032 8033 static bool mlxsw_sp_dev_addr_list_empty(const struct net_device *dev) 8034 { 8035 bool addr_list_empty; 8036 8037 rcu_read_lock(); 8038 addr_list_empty = __mlxsw_sp_dev_addr_list_empty(dev); 8039 rcu_read_unlock(); 8040 8041 return addr_list_empty; 8042 } 8043 8044 static bool 8045 mlxsw_sp_rif_should_config(struct mlxsw_sp_rif *rif, struct net_device *dev, 8046 unsigned long event) 8047 { 8048 bool addr_list_empty; 8049 8050 switch (event) { 8051 case NETDEV_UP: 8052 return rif == NULL; 8053 case NETDEV_DOWN: 8054 addr_list_empty = mlxsw_sp_dev_addr_list_empty(dev); 8055 8056 /* macvlans do not have a RIF, but rather piggy back on the 8057 * RIF of their lower device. 8058 */ 8059 if (netif_is_macvlan(dev) && addr_list_empty) 8060 return true; 8061 8062 if (rif && addr_list_empty && 8063 !netif_is_l3_slave(mlxsw_sp_rif_dev(rif))) 8064 return true; 8065 /* It is possible we already removed the RIF ourselves 8066 * if it was assigned to a netdev that is now a bridge 8067 * or LAG slave. 8068 */ 8069 return false; 8070 } 8071 8072 return false; 8073 } 8074 8075 static enum mlxsw_sp_rif_type 8076 mlxsw_sp_dev_rif_type(const struct mlxsw_sp *mlxsw_sp, 8077 const struct net_device *dev) 8078 { 8079 enum mlxsw_sp_fid_type type; 8080 8081 if (mlxsw_sp_netdev_ipip_type(mlxsw_sp, dev, NULL)) 8082 return MLXSW_SP_RIF_TYPE_IPIP_LB; 8083 8084 /* Otherwise RIF type is derived from the type of the underlying FID. */ 8085 if (is_vlan_dev(dev) && netif_is_bridge_master(vlan_dev_real_dev(dev))) 8086 type = MLXSW_SP_FID_TYPE_8021Q; 8087 else if (netif_is_bridge_master(dev) && br_vlan_enabled(dev)) 8088 type = MLXSW_SP_FID_TYPE_8021Q; 8089 else if (netif_is_bridge_master(dev)) 8090 type = MLXSW_SP_FID_TYPE_8021D; 8091 else 8092 type = MLXSW_SP_FID_TYPE_RFID; 8093 8094 return mlxsw_sp_fid_type_rif_type(mlxsw_sp, type); 8095 } 8096 8097 static int mlxsw_sp_rif_index_alloc(struct mlxsw_sp *mlxsw_sp, u16 *p_rif_index, 8098 u8 rif_entries) 8099 { 8100 *p_rif_index = gen_pool_alloc(mlxsw_sp->router->rifs_table, 8101 rif_entries); 8102 if (*p_rif_index == 0) 8103 return -ENOBUFS; 8104 *p_rif_index -= MLXSW_SP_ROUTER_GENALLOC_OFFSET; 8105 8106 /* RIF indexes must be aligned to the allocation size. */ 8107 WARN_ON_ONCE(*p_rif_index % rif_entries); 8108 8109 return 0; 8110 } 8111 8112 static void mlxsw_sp_rif_index_free(struct mlxsw_sp *mlxsw_sp, u16 rif_index, 8113 u8 rif_entries) 8114 { 8115 gen_pool_free(mlxsw_sp->router->rifs_table, 8116 MLXSW_SP_ROUTER_GENALLOC_OFFSET + rif_index, rif_entries); 8117 } 8118 8119 static struct mlxsw_sp_rif *mlxsw_sp_rif_alloc(size_t rif_size, u16 rif_index, 8120 u16 vr_id, 8121 struct mlxsw_sp_crif *crif) 8122 { 8123 struct net_device *l3_dev = crif ? crif->key.dev : NULL; 8124 struct mlxsw_sp_rif *rif; 8125 8126 rif = kzalloc(rif_size, GFP_KERNEL); 8127 if (!rif) 8128 return NULL; 8129 8130 INIT_LIST_HEAD(&rif->neigh_list); 8131 if (l3_dev) { 8132 ether_addr_copy(rif->addr, l3_dev->dev_addr); 8133 rif->mtu = l3_dev->mtu; 8134 } 8135 rif->vr_id = vr_id; 8136 rif->rif_index = rif_index; 8137 if (crif) { 8138 rif->crif = crif; 8139 crif->rif = rif; 8140 } 8141 8142 return rif; 8143 } 8144 8145 static void mlxsw_sp_rif_free(struct mlxsw_sp_rif *rif) 8146 { 8147 WARN_ON(!list_empty(&rif->neigh_list)); 8148 8149 if (rif->crif) 8150 rif->crif->rif = NULL; 8151 kfree(rif); 8152 } 8153 8154 struct mlxsw_sp_rif *mlxsw_sp_rif_by_index(const struct mlxsw_sp *mlxsw_sp, 8155 u16 rif_index) 8156 { 8157 return mlxsw_sp->router->rifs[rif_index]; 8158 } 8159 8160 u16 mlxsw_sp_rif_index(const struct mlxsw_sp_rif *rif) 8161 { 8162 return rif->rif_index; 8163 } 8164 8165 u16 mlxsw_sp_ipip_lb_rif_index(const struct mlxsw_sp_rif_ipip_lb *lb_rif) 8166 { 8167 return lb_rif->common.rif_index; 8168 } 8169 8170 u16 mlxsw_sp_ipip_lb_ul_vr_id(const struct mlxsw_sp_rif_ipip_lb *lb_rif) 8171 { 8172 struct net_device *dev = mlxsw_sp_rif_dev(&lb_rif->common); 8173 u32 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(dev); 8174 struct mlxsw_sp_vr *ul_vr; 8175 8176 ul_vr = mlxsw_sp_vr_get(lb_rif->common.mlxsw_sp, ul_tb_id, NULL); 8177 if (WARN_ON(IS_ERR(ul_vr))) 8178 return 0; 8179 8180 return ul_vr->id; 8181 } 8182 8183 u16 mlxsw_sp_ipip_lb_ul_rif_id(const struct mlxsw_sp_rif_ipip_lb *lb_rif) 8184 { 8185 return lb_rif->ul_rif_id; 8186 } 8187 8188 static bool 8189 mlxsw_sp_router_port_l3_stats_enabled(struct mlxsw_sp_rif *rif) 8190 { 8191 return mlxsw_sp_rif_counter_valid_get(rif, 8192 MLXSW_SP_RIF_COUNTER_EGRESS) && 8193 mlxsw_sp_rif_counter_valid_get(rif, 8194 MLXSW_SP_RIF_COUNTER_INGRESS); 8195 } 8196 8197 static int 8198 mlxsw_sp_router_port_l3_stats_enable(struct mlxsw_sp_rif *rif) 8199 { 8200 int err; 8201 8202 err = mlxsw_sp_rif_counter_alloc(rif, MLXSW_SP_RIF_COUNTER_INGRESS); 8203 if (err) 8204 return err; 8205 8206 /* Clear stale data. */ 8207 err = mlxsw_sp_rif_counter_fetch_clear(rif, 8208 MLXSW_SP_RIF_COUNTER_INGRESS, 8209 NULL); 8210 if (err) 8211 goto err_clear_ingress; 8212 8213 err = mlxsw_sp_rif_counter_alloc(rif, MLXSW_SP_RIF_COUNTER_EGRESS); 8214 if (err) 8215 goto err_alloc_egress; 8216 8217 /* Clear stale data. */ 8218 err = mlxsw_sp_rif_counter_fetch_clear(rif, 8219 MLXSW_SP_RIF_COUNTER_EGRESS, 8220 NULL); 8221 if (err) 8222 goto err_clear_egress; 8223 8224 return 0; 8225 8226 err_clear_egress: 8227 mlxsw_sp_rif_counter_free(rif, MLXSW_SP_RIF_COUNTER_EGRESS); 8228 err_alloc_egress: 8229 err_clear_ingress: 8230 mlxsw_sp_rif_counter_free(rif, MLXSW_SP_RIF_COUNTER_INGRESS); 8231 return err; 8232 } 8233 8234 static void 8235 mlxsw_sp_router_port_l3_stats_disable(struct mlxsw_sp_rif *rif) 8236 { 8237 mlxsw_sp_rif_counter_free(rif, MLXSW_SP_RIF_COUNTER_EGRESS); 8238 mlxsw_sp_rif_counter_free(rif, MLXSW_SP_RIF_COUNTER_INGRESS); 8239 } 8240 8241 static void 8242 mlxsw_sp_router_port_l3_stats_report_used(struct mlxsw_sp_rif *rif, 8243 struct netdev_notifier_offload_xstats_info *info) 8244 { 8245 if (!mlxsw_sp_router_port_l3_stats_enabled(rif)) 8246 return; 8247 netdev_offload_xstats_report_used(info->report_used); 8248 } 8249 8250 static int 8251 mlxsw_sp_router_port_l3_stats_fetch(struct mlxsw_sp_rif *rif, 8252 struct rtnl_hw_stats64 *p_stats) 8253 { 8254 struct mlxsw_sp_rif_counter_set_basic ingress; 8255 struct mlxsw_sp_rif_counter_set_basic egress; 8256 int err; 8257 8258 err = mlxsw_sp_rif_counter_fetch_clear(rif, 8259 MLXSW_SP_RIF_COUNTER_INGRESS, 8260 &ingress); 8261 if (err) 8262 return err; 8263 8264 err = mlxsw_sp_rif_counter_fetch_clear(rif, 8265 MLXSW_SP_RIF_COUNTER_EGRESS, 8266 &egress); 8267 if (err) 8268 return err; 8269 8270 #define MLXSW_SP_ROUTER_ALL_GOOD(SET, SFX) \ 8271 ((SET.good_unicast_ ## SFX) + \ 8272 (SET.good_multicast_ ## SFX) + \ 8273 (SET.good_broadcast_ ## SFX)) 8274 8275 p_stats->rx_packets = MLXSW_SP_ROUTER_ALL_GOOD(ingress, packets); 8276 p_stats->tx_packets = MLXSW_SP_ROUTER_ALL_GOOD(egress, packets); 8277 p_stats->rx_bytes = MLXSW_SP_ROUTER_ALL_GOOD(ingress, bytes); 8278 p_stats->tx_bytes = MLXSW_SP_ROUTER_ALL_GOOD(egress, bytes); 8279 p_stats->rx_errors = ingress.error_packets; 8280 p_stats->tx_errors = egress.error_packets; 8281 p_stats->rx_dropped = ingress.discard_packets; 8282 p_stats->tx_dropped = egress.discard_packets; 8283 p_stats->multicast = ingress.good_multicast_packets + 8284 ingress.good_broadcast_packets; 8285 8286 #undef MLXSW_SP_ROUTER_ALL_GOOD 8287 8288 return 0; 8289 } 8290 8291 static int 8292 mlxsw_sp_router_port_l3_stats_report_delta(struct mlxsw_sp_rif *rif, 8293 struct netdev_notifier_offload_xstats_info *info) 8294 { 8295 struct rtnl_hw_stats64 stats = {}; 8296 int err; 8297 8298 if (!mlxsw_sp_router_port_l3_stats_enabled(rif)) 8299 return 0; 8300 8301 err = mlxsw_sp_router_port_l3_stats_fetch(rif, &stats); 8302 if (err) 8303 return err; 8304 8305 netdev_offload_xstats_report_delta(info->report_delta, &stats); 8306 return 0; 8307 } 8308 8309 struct mlxsw_sp_router_hwstats_notify_work { 8310 struct work_struct work; 8311 struct net_device *dev; 8312 netdevice_tracker dev_tracker; 8313 }; 8314 8315 static void mlxsw_sp_router_hwstats_notify_work(struct work_struct *work) 8316 { 8317 struct mlxsw_sp_router_hwstats_notify_work *hws_work = 8318 container_of(work, struct mlxsw_sp_router_hwstats_notify_work, 8319 work); 8320 8321 rtnl_lock(); 8322 rtnl_offload_xstats_notify(hws_work->dev); 8323 rtnl_unlock(); 8324 netdev_put(hws_work->dev, &hws_work->dev_tracker); 8325 kfree(hws_work); 8326 } 8327 8328 static void 8329 mlxsw_sp_router_hwstats_notify_schedule(struct net_device *dev) 8330 { 8331 struct mlxsw_sp_router_hwstats_notify_work *hws_work; 8332 8333 /* To collect notification payload, the core ends up sending another 8334 * notifier block message, which would deadlock on the attempt to 8335 * acquire the router lock again. Just postpone the notification until 8336 * later. 8337 */ 8338 8339 hws_work = kzalloc(sizeof(*hws_work), GFP_KERNEL); 8340 if (!hws_work) 8341 return; 8342 8343 INIT_WORK(&hws_work->work, mlxsw_sp_router_hwstats_notify_work); 8344 netdev_hold(dev, &hws_work->dev_tracker, GFP_KERNEL); 8345 hws_work->dev = dev; 8346 mlxsw_core_schedule_work(&hws_work->work); 8347 } 8348 8349 int mlxsw_sp_rif_dev_ifindex(const struct mlxsw_sp_rif *rif) 8350 { 8351 return mlxsw_sp_rif_dev(rif)->ifindex; 8352 } 8353 8354 bool mlxsw_sp_rif_has_dev(const struct mlxsw_sp_rif *rif) 8355 { 8356 return !!mlxsw_sp_rif_dev(rif); 8357 } 8358 8359 bool mlxsw_sp_rif_dev_is(const struct mlxsw_sp_rif *rif, 8360 const struct net_device *dev) 8361 { 8362 return mlxsw_sp_rif_dev(rif) == dev; 8363 } 8364 8365 static void mlxsw_sp_rif_push_l3_stats(struct mlxsw_sp_rif *rif) 8366 { 8367 struct rtnl_hw_stats64 stats = {}; 8368 8369 if (!mlxsw_sp_router_port_l3_stats_fetch(rif, &stats)) 8370 netdev_offload_xstats_push_delta(mlxsw_sp_rif_dev(rif), 8371 NETDEV_OFFLOAD_XSTATS_TYPE_L3, 8372 &stats); 8373 } 8374 8375 static struct mlxsw_sp_rif * 8376 mlxsw_sp_rif_create(struct mlxsw_sp *mlxsw_sp, 8377 const struct mlxsw_sp_rif_params *params, 8378 struct netlink_ext_ack *extack) 8379 { 8380 u8 rif_entries = params->double_entry ? 2 : 1; 8381 u32 tb_id = l3mdev_fib_table(params->dev); 8382 const struct mlxsw_sp_rif_ops *ops; 8383 struct mlxsw_sp_fid *fid = NULL; 8384 enum mlxsw_sp_rif_type type; 8385 struct mlxsw_sp_crif *crif; 8386 struct mlxsw_sp_rif *rif; 8387 struct mlxsw_sp_vr *vr; 8388 u16 rif_index; 8389 int i, err; 8390 8391 type = mlxsw_sp_dev_rif_type(mlxsw_sp, params->dev); 8392 ops = mlxsw_sp->router->rif_ops_arr[type]; 8393 8394 vr = mlxsw_sp_vr_get(mlxsw_sp, tb_id ? : RT_TABLE_MAIN, extack); 8395 if (IS_ERR(vr)) 8396 return ERR_CAST(vr); 8397 vr->rif_count++; 8398 8399 err = mlxsw_sp_rif_index_alloc(mlxsw_sp, &rif_index, rif_entries); 8400 if (err) { 8401 NL_SET_ERR_MSG_MOD(extack, "Exceeded number of supported router interfaces"); 8402 goto err_rif_index_alloc; 8403 } 8404 8405 crif = mlxsw_sp_crif_lookup(mlxsw_sp->router, params->dev); 8406 if (WARN_ON(!crif)) { 8407 err = -ENOENT; 8408 goto err_crif_lookup; 8409 } 8410 8411 rif = mlxsw_sp_rif_alloc(ops->rif_size, rif_index, vr->id, crif); 8412 if (!rif) { 8413 err = -ENOMEM; 8414 goto err_rif_alloc; 8415 } 8416 netdev_hold(params->dev, &rif->dev_tracker, GFP_KERNEL); 8417 mlxsw_sp->router->rifs[rif_index] = rif; 8418 rif->mlxsw_sp = mlxsw_sp; 8419 rif->ops = ops; 8420 rif->rif_entries = rif_entries; 8421 8422 if (ops->fid_get) { 8423 fid = ops->fid_get(rif, params, extack); 8424 if (IS_ERR(fid)) { 8425 err = PTR_ERR(fid); 8426 goto err_fid_get; 8427 } 8428 rif->fid = fid; 8429 } 8430 8431 if (ops->setup) 8432 ops->setup(rif, params); 8433 8434 err = ops->configure(rif, extack); 8435 if (err) 8436 goto err_configure; 8437 8438 for (i = 0; i < MLXSW_SP_L3_PROTO_MAX; i++) { 8439 err = mlxsw_sp_mr_rif_add(vr->mr_table[i], rif); 8440 if (err) 8441 goto err_mr_rif_add; 8442 } 8443 8444 err = mlxsw_sp_router_rif_made_sync(mlxsw_sp, rif); 8445 if (err) 8446 goto err_rif_made_sync; 8447 8448 if (netdev_offload_xstats_enabled(params->dev, 8449 NETDEV_OFFLOAD_XSTATS_TYPE_L3)) { 8450 err = mlxsw_sp_router_port_l3_stats_enable(rif); 8451 if (err) 8452 goto err_stats_enable; 8453 mlxsw_sp_router_hwstats_notify_schedule(params->dev); 8454 } else { 8455 mlxsw_sp_rif_counters_alloc(rif); 8456 } 8457 8458 atomic_add(rif_entries, &mlxsw_sp->router->rifs_count); 8459 return rif; 8460 8461 err_stats_enable: 8462 mlxsw_sp_router_rif_gone_sync(mlxsw_sp, rif); 8463 err_rif_made_sync: 8464 err_mr_rif_add: 8465 for (i--; i >= 0; i--) 8466 mlxsw_sp_mr_rif_del(vr->mr_table[i], rif); 8467 ops->deconfigure(rif); 8468 err_configure: 8469 if (fid) 8470 mlxsw_sp_fid_put(fid); 8471 err_fid_get: 8472 mlxsw_sp->router->rifs[rif_index] = NULL; 8473 netdev_put(params->dev, &rif->dev_tracker); 8474 mlxsw_sp_rif_free(rif); 8475 err_rif_alloc: 8476 err_crif_lookup: 8477 mlxsw_sp_rif_index_free(mlxsw_sp, rif_index, rif_entries); 8478 err_rif_index_alloc: 8479 vr->rif_count--; 8480 mlxsw_sp_vr_put(mlxsw_sp, vr); 8481 return ERR_PTR(err); 8482 } 8483 8484 static void mlxsw_sp_rif_destroy(struct mlxsw_sp_rif *rif) 8485 { 8486 struct net_device *dev = mlxsw_sp_rif_dev(rif); 8487 const struct mlxsw_sp_rif_ops *ops = rif->ops; 8488 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 8489 struct mlxsw_sp_crif *crif = rif->crif; 8490 struct mlxsw_sp_fid *fid = rif->fid; 8491 u8 rif_entries = rif->rif_entries; 8492 u16 rif_index = rif->rif_index; 8493 struct mlxsw_sp_vr *vr; 8494 int i; 8495 8496 atomic_sub(rif_entries, &mlxsw_sp->router->rifs_count); 8497 mlxsw_sp_router_rif_gone_sync(mlxsw_sp, rif); 8498 vr = &mlxsw_sp->router->vrs[rif->vr_id]; 8499 8500 if (netdev_offload_xstats_enabled(dev, NETDEV_OFFLOAD_XSTATS_TYPE_L3)) { 8501 mlxsw_sp_rif_push_l3_stats(rif); 8502 mlxsw_sp_router_port_l3_stats_disable(rif); 8503 mlxsw_sp_router_hwstats_notify_schedule(dev); 8504 } else { 8505 mlxsw_sp_rif_counters_free(rif); 8506 } 8507 8508 for (i = 0; i < MLXSW_SP_L3_PROTO_MAX; i++) 8509 mlxsw_sp_mr_rif_del(vr->mr_table[i], rif); 8510 ops->deconfigure(rif); 8511 if (fid) 8512 /* Loopback RIFs are not associated with a FID. */ 8513 mlxsw_sp_fid_put(fid); 8514 mlxsw_sp->router->rifs[rif->rif_index] = NULL; 8515 netdev_put(dev, &rif->dev_tracker); 8516 mlxsw_sp_rif_free(rif); 8517 mlxsw_sp_rif_index_free(mlxsw_sp, rif_index, rif_entries); 8518 vr->rif_count--; 8519 mlxsw_sp_vr_put(mlxsw_sp, vr); 8520 8521 if (crif->can_destroy) 8522 mlxsw_sp_crif_free(crif); 8523 } 8524 8525 void mlxsw_sp_rif_destroy_by_dev(struct mlxsw_sp *mlxsw_sp, 8526 struct net_device *dev) 8527 { 8528 struct mlxsw_sp_rif *rif; 8529 8530 mutex_lock(&mlxsw_sp->router->lock); 8531 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev); 8532 if (!rif) 8533 goto out; 8534 mlxsw_sp_rif_destroy(rif); 8535 out: 8536 mutex_unlock(&mlxsw_sp->router->lock); 8537 } 8538 8539 static void mlxsw_sp_rif_destroy_vlan_upper(struct mlxsw_sp *mlxsw_sp, 8540 struct net_device *br_dev, 8541 u16 vid) 8542 { 8543 struct net_device *upper_dev; 8544 struct mlxsw_sp_crif *crif; 8545 8546 rcu_read_lock(); 8547 upper_dev = __vlan_find_dev_deep_rcu(br_dev, htons(ETH_P_8021Q), vid); 8548 rcu_read_unlock(); 8549 8550 if (!upper_dev) 8551 return; 8552 8553 crif = mlxsw_sp_crif_lookup(mlxsw_sp->router, upper_dev); 8554 if (!crif || !crif->rif) 8555 return; 8556 8557 mlxsw_sp_rif_destroy(crif->rif); 8558 } 8559 8560 static int mlxsw_sp_inetaddr_bridge_event(struct mlxsw_sp *mlxsw_sp, 8561 struct net_device *l3_dev, 8562 int lower_pvid, 8563 unsigned long event, 8564 struct netlink_ext_ack *extack); 8565 8566 int mlxsw_sp_router_bridge_vlan_add(struct mlxsw_sp *mlxsw_sp, 8567 struct net_device *br_dev, 8568 u16 new_vid, bool is_pvid, 8569 struct netlink_ext_ack *extack) 8570 { 8571 struct mlxsw_sp_rif *old_rif; 8572 struct mlxsw_sp_rif *new_rif; 8573 struct net_device *upper_dev; 8574 u16 old_pvid = 0; 8575 u16 new_pvid; 8576 int err = 0; 8577 8578 mutex_lock(&mlxsw_sp->router->lock); 8579 old_rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, br_dev); 8580 if (old_rif) { 8581 /* If the RIF on the bridge is not a VLAN RIF, we shouldn't have 8582 * gotten a PVID notification. 8583 */ 8584 if (WARN_ON(old_rif->ops->type != MLXSW_SP_RIF_TYPE_VLAN)) 8585 old_rif = NULL; 8586 else 8587 old_pvid = mlxsw_sp_fid_8021q_vid(old_rif->fid); 8588 } 8589 8590 if (is_pvid) 8591 new_pvid = new_vid; 8592 else if (old_pvid == new_vid) 8593 new_pvid = 0; 8594 else 8595 goto out; 8596 8597 if (old_pvid == new_pvid) 8598 goto out; 8599 8600 if (new_pvid) { 8601 struct mlxsw_sp_rif_params params = { 8602 .dev = br_dev, 8603 .vid = new_pvid, 8604 }; 8605 8606 /* If there is a VLAN upper with the same VID as the new PVID, 8607 * kill its RIF, if there is one. 8608 */ 8609 mlxsw_sp_rif_destroy_vlan_upper(mlxsw_sp, br_dev, new_pvid); 8610 8611 if (mlxsw_sp_dev_addr_list_empty(br_dev)) 8612 goto out; 8613 new_rif = mlxsw_sp_rif_create(mlxsw_sp, ¶ms, extack); 8614 if (IS_ERR(new_rif)) { 8615 err = PTR_ERR(new_rif); 8616 goto out; 8617 } 8618 8619 if (old_pvid) 8620 mlxsw_sp_rif_migrate_destroy(mlxsw_sp, old_rif, new_rif, 8621 true); 8622 } else { 8623 mlxsw_sp_rif_destroy(old_rif); 8624 } 8625 8626 if (old_pvid) { 8627 rcu_read_lock(); 8628 upper_dev = __vlan_find_dev_deep_rcu(br_dev, htons(ETH_P_8021Q), 8629 old_pvid); 8630 rcu_read_unlock(); 8631 if (upper_dev) 8632 err = mlxsw_sp_inetaddr_bridge_event(mlxsw_sp, 8633 upper_dev, 8634 new_pvid, 8635 NETDEV_UP, extack); 8636 } 8637 8638 out: 8639 mutex_unlock(&mlxsw_sp->router->lock); 8640 return err; 8641 } 8642 8643 static void 8644 mlxsw_sp_rif_subport_params_init(struct mlxsw_sp_rif_params *params, 8645 struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan) 8646 { 8647 struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port; 8648 8649 params->vid = mlxsw_sp_port_vlan->vid; 8650 params->lag = mlxsw_sp_port->lagged; 8651 if (params->lag) 8652 params->lag_id = mlxsw_sp_port->lag_id; 8653 else 8654 params->system_port = mlxsw_sp_port->local_port; 8655 } 8656 8657 static struct mlxsw_sp_rif_subport * 8658 mlxsw_sp_rif_subport_rif(const struct mlxsw_sp_rif *rif) 8659 { 8660 return container_of(rif, struct mlxsw_sp_rif_subport, common); 8661 } 8662 8663 static struct mlxsw_sp_rif * 8664 mlxsw_sp_rif_subport_get(struct mlxsw_sp *mlxsw_sp, 8665 const struct mlxsw_sp_rif_params *params, 8666 struct netlink_ext_ack *extack) 8667 { 8668 struct mlxsw_sp_rif_subport *rif_subport; 8669 struct mlxsw_sp_rif *rif; 8670 8671 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, params->dev); 8672 if (!rif) 8673 return mlxsw_sp_rif_create(mlxsw_sp, params, extack); 8674 8675 rif_subport = mlxsw_sp_rif_subport_rif(rif); 8676 refcount_inc(&rif_subport->ref_count); 8677 return rif; 8678 } 8679 8680 static void mlxsw_sp_rif_subport_put(struct mlxsw_sp_rif *rif) 8681 { 8682 struct mlxsw_sp_rif_subport *rif_subport; 8683 8684 rif_subport = mlxsw_sp_rif_subport_rif(rif); 8685 if (!refcount_dec_and_test(&rif_subport->ref_count)) 8686 return; 8687 8688 mlxsw_sp_rif_destroy(rif); 8689 } 8690 8691 static int mlxsw_sp_rif_mac_profile_index_alloc(struct mlxsw_sp *mlxsw_sp, 8692 struct mlxsw_sp_rif_mac_profile *profile, 8693 struct netlink_ext_ack *extack) 8694 { 8695 u8 max_rif_mac_profiles = mlxsw_sp->router->max_rif_mac_profile; 8696 struct mlxsw_sp_router *router = mlxsw_sp->router; 8697 int id; 8698 8699 id = idr_alloc(&router->rif_mac_profiles_idr, profile, 0, 8700 max_rif_mac_profiles, GFP_KERNEL); 8701 8702 if (id >= 0) { 8703 profile->id = id; 8704 return 0; 8705 } 8706 8707 if (id == -ENOSPC) 8708 NL_SET_ERR_MSG_MOD(extack, 8709 "Exceeded number of supported router interface MAC profiles"); 8710 8711 return id; 8712 } 8713 8714 static struct mlxsw_sp_rif_mac_profile * 8715 mlxsw_sp_rif_mac_profile_index_free(struct mlxsw_sp *mlxsw_sp, u8 mac_profile) 8716 { 8717 struct mlxsw_sp_rif_mac_profile *profile; 8718 8719 profile = idr_remove(&mlxsw_sp->router->rif_mac_profiles_idr, 8720 mac_profile); 8721 WARN_ON(!profile); 8722 return profile; 8723 } 8724 8725 static struct mlxsw_sp_rif_mac_profile * 8726 mlxsw_sp_rif_mac_profile_alloc(const char *mac) 8727 { 8728 struct mlxsw_sp_rif_mac_profile *profile; 8729 8730 profile = kzalloc(sizeof(*profile), GFP_KERNEL); 8731 if (!profile) 8732 return NULL; 8733 8734 ether_addr_copy(profile->mac_prefix, mac); 8735 refcount_set(&profile->ref_count, 1); 8736 return profile; 8737 } 8738 8739 static struct mlxsw_sp_rif_mac_profile * 8740 mlxsw_sp_rif_mac_profile_find(const struct mlxsw_sp *mlxsw_sp, const char *mac) 8741 { 8742 struct mlxsw_sp_router *router = mlxsw_sp->router; 8743 struct mlxsw_sp_rif_mac_profile *profile; 8744 int id; 8745 8746 idr_for_each_entry(&router->rif_mac_profiles_idr, profile, id) { 8747 if (ether_addr_equal_masked(profile->mac_prefix, mac, 8748 mlxsw_sp->mac_mask)) 8749 return profile; 8750 } 8751 8752 return NULL; 8753 } 8754 8755 static u64 mlxsw_sp_rif_mac_profiles_occ_get(void *priv) 8756 { 8757 const struct mlxsw_sp *mlxsw_sp = priv; 8758 8759 return atomic_read(&mlxsw_sp->router->rif_mac_profiles_count); 8760 } 8761 8762 static u64 mlxsw_sp_rifs_occ_get(void *priv) 8763 { 8764 const struct mlxsw_sp *mlxsw_sp = priv; 8765 8766 return atomic_read(&mlxsw_sp->router->rifs_count); 8767 } 8768 8769 static struct mlxsw_sp_rif_mac_profile * 8770 mlxsw_sp_rif_mac_profile_create(struct mlxsw_sp *mlxsw_sp, const char *mac, 8771 struct netlink_ext_ack *extack) 8772 { 8773 struct mlxsw_sp_rif_mac_profile *profile; 8774 int err; 8775 8776 profile = mlxsw_sp_rif_mac_profile_alloc(mac); 8777 if (!profile) 8778 return ERR_PTR(-ENOMEM); 8779 8780 err = mlxsw_sp_rif_mac_profile_index_alloc(mlxsw_sp, profile, extack); 8781 if (err) 8782 goto profile_index_alloc_err; 8783 8784 atomic_inc(&mlxsw_sp->router->rif_mac_profiles_count); 8785 return profile; 8786 8787 profile_index_alloc_err: 8788 kfree(profile); 8789 return ERR_PTR(err); 8790 } 8791 8792 static void mlxsw_sp_rif_mac_profile_destroy(struct mlxsw_sp *mlxsw_sp, 8793 u8 mac_profile) 8794 { 8795 struct mlxsw_sp_rif_mac_profile *profile; 8796 8797 atomic_dec(&mlxsw_sp->router->rif_mac_profiles_count); 8798 profile = mlxsw_sp_rif_mac_profile_index_free(mlxsw_sp, mac_profile); 8799 kfree(profile); 8800 } 8801 8802 static int mlxsw_sp_rif_mac_profile_get(struct mlxsw_sp *mlxsw_sp, 8803 const char *mac, u8 *p_mac_profile, 8804 struct netlink_ext_ack *extack) 8805 { 8806 struct mlxsw_sp_rif_mac_profile *profile; 8807 8808 profile = mlxsw_sp_rif_mac_profile_find(mlxsw_sp, mac); 8809 if (profile) { 8810 refcount_inc(&profile->ref_count); 8811 goto out; 8812 } 8813 8814 profile = mlxsw_sp_rif_mac_profile_create(mlxsw_sp, mac, extack); 8815 if (IS_ERR(profile)) 8816 return PTR_ERR(profile); 8817 8818 out: 8819 *p_mac_profile = profile->id; 8820 return 0; 8821 } 8822 8823 static void mlxsw_sp_rif_mac_profile_put(struct mlxsw_sp *mlxsw_sp, 8824 u8 mac_profile) 8825 { 8826 struct mlxsw_sp_rif_mac_profile *profile; 8827 8828 profile = idr_find(&mlxsw_sp->router->rif_mac_profiles_idr, 8829 mac_profile); 8830 if (WARN_ON(!profile)) 8831 return; 8832 8833 if (!refcount_dec_and_test(&profile->ref_count)) 8834 return; 8835 8836 mlxsw_sp_rif_mac_profile_destroy(mlxsw_sp, mac_profile); 8837 } 8838 8839 static bool mlxsw_sp_rif_mac_profile_is_shared(const struct mlxsw_sp_rif *rif) 8840 { 8841 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 8842 struct mlxsw_sp_rif_mac_profile *profile; 8843 8844 profile = idr_find(&mlxsw_sp->router->rif_mac_profiles_idr, 8845 rif->mac_profile_id); 8846 if (WARN_ON(!profile)) 8847 return false; 8848 8849 return refcount_read(&profile->ref_count) > 1; 8850 } 8851 8852 static int mlxsw_sp_rif_mac_profile_edit(struct mlxsw_sp_rif *rif, 8853 const char *new_mac) 8854 { 8855 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 8856 struct mlxsw_sp_rif_mac_profile *profile; 8857 8858 profile = idr_find(&mlxsw_sp->router->rif_mac_profiles_idr, 8859 rif->mac_profile_id); 8860 if (WARN_ON(!profile)) 8861 return -EINVAL; 8862 8863 ether_addr_copy(profile->mac_prefix, new_mac); 8864 return 0; 8865 } 8866 8867 static int 8868 mlxsw_sp_rif_mac_profile_replace(struct mlxsw_sp *mlxsw_sp, 8869 struct mlxsw_sp_rif *rif, 8870 const char *new_mac, 8871 struct netlink_ext_ack *extack) 8872 { 8873 u8 mac_profile; 8874 int err; 8875 8876 if (!mlxsw_sp_rif_mac_profile_is_shared(rif) && 8877 !mlxsw_sp_rif_mac_profile_find(mlxsw_sp, new_mac)) 8878 return mlxsw_sp_rif_mac_profile_edit(rif, new_mac); 8879 8880 err = mlxsw_sp_rif_mac_profile_get(mlxsw_sp, new_mac, 8881 &mac_profile, extack); 8882 if (err) 8883 return err; 8884 8885 mlxsw_sp_rif_mac_profile_put(mlxsw_sp, rif->mac_profile_id); 8886 rif->mac_profile_id = mac_profile; 8887 return 0; 8888 } 8889 8890 static int 8891 __mlxsw_sp_port_vlan_router_join(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan, 8892 struct net_device *l3_dev, 8893 struct netlink_ext_ack *extack) 8894 { 8895 struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port; 8896 struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp; 8897 struct mlxsw_sp_rif_params params; 8898 u16 vid = mlxsw_sp_port_vlan->vid; 8899 struct mlxsw_sp_rif *rif; 8900 struct mlxsw_sp_fid *fid; 8901 int err; 8902 8903 params = (struct mlxsw_sp_rif_params) { 8904 .dev = l3_dev, 8905 .vid = vid, 8906 }; 8907 8908 mlxsw_sp_rif_subport_params_init(¶ms, mlxsw_sp_port_vlan); 8909 rif = mlxsw_sp_rif_subport_get(mlxsw_sp, ¶ms, extack); 8910 if (IS_ERR(rif)) 8911 return PTR_ERR(rif); 8912 8913 /* FID was already created, just take a reference */ 8914 fid = rif->ops->fid_get(rif, ¶ms, extack); 8915 err = mlxsw_sp_fid_port_vid_map(fid, mlxsw_sp_port, vid); 8916 if (err) 8917 goto err_fid_port_vid_map; 8918 8919 err = mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, false); 8920 if (err) 8921 goto err_port_vid_learning_set; 8922 8923 err = mlxsw_sp_port_vid_stp_set(mlxsw_sp_port, vid, 8924 BR_STATE_FORWARDING); 8925 if (err) 8926 goto err_port_vid_stp_set; 8927 8928 mlxsw_sp_port_vlan->fid = fid; 8929 8930 return 0; 8931 8932 err_port_vid_stp_set: 8933 mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, true); 8934 err_port_vid_learning_set: 8935 mlxsw_sp_fid_port_vid_unmap(fid, mlxsw_sp_port, vid); 8936 err_fid_port_vid_map: 8937 mlxsw_sp_fid_put(fid); 8938 mlxsw_sp_rif_subport_put(rif); 8939 return err; 8940 } 8941 8942 static void 8943 __mlxsw_sp_port_vlan_router_leave(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan) 8944 { 8945 struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port; 8946 struct mlxsw_sp_fid *fid = mlxsw_sp_port_vlan->fid; 8947 struct mlxsw_sp_rif *rif = mlxsw_sp_fid_rif(fid); 8948 u16 vid = mlxsw_sp_port_vlan->vid; 8949 8950 if (WARN_ON(mlxsw_sp_fid_type(fid) != MLXSW_SP_FID_TYPE_RFID)) 8951 return; 8952 8953 mlxsw_sp_port_vlan->fid = NULL; 8954 mlxsw_sp_port_vid_stp_set(mlxsw_sp_port, vid, BR_STATE_BLOCKING); 8955 mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, true); 8956 mlxsw_sp_fid_port_vid_unmap(fid, mlxsw_sp_port, vid); 8957 mlxsw_sp_fid_put(fid); 8958 mlxsw_sp_rif_subport_put(rif); 8959 } 8960 8961 static int 8962 mlxsw_sp_port_vlan_router_join_existing(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan, 8963 struct net_device *l3_dev, 8964 struct netlink_ext_ack *extack) 8965 { 8966 struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port_vlan->mlxsw_sp_port->mlxsw_sp; 8967 8968 lockdep_assert_held(&mlxsw_sp->router->lock); 8969 8970 if (!mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev)) 8971 return 0; 8972 8973 return __mlxsw_sp_port_vlan_router_join(mlxsw_sp_port_vlan, l3_dev, 8974 extack); 8975 } 8976 8977 void 8978 mlxsw_sp_port_vlan_router_leave(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan) 8979 { 8980 struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port_vlan->mlxsw_sp_port->mlxsw_sp; 8981 8982 mutex_lock(&mlxsw_sp->router->lock); 8983 __mlxsw_sp_port_vlan_router_leave(mlxsw_sp_port_vlan); 8984 mutex_unlock(&mlxsw_sp->router->lock); 8985 } 8986 8987 static int mlxsw_sp_inetaddr_port_vlan_event(struct net_device *l3_dev, 8988 struct net_device *port_dev, 8989 unsigned long event, u16 vid, 8990 struct netlink_ext_ack *extack) 8991 { 8992 struct mlxsw_sp_port *mlxsw_sp_port = netdev_priv(port_dev); 8993 struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan; 8994 8995 mlxsw_sp_port_vlan = mlxsw_sp_port_vlan_find_by_vid(mlxsw_sp_port, vid); 8996 if (WARN_ON(!mlxsw_sp_port_vlan)) 8997 return -EINVAL; 8998 8999 switch (event) { 9000 case NETDEV_UP: 9001 return __mlxsw_sp_port_vlan_router_join(mlxsw_sp_port_vlan, 9002 l3_dev, extack); 9003 case NETDEV_DOWN: 9004 __mlxsw_sp_port_vlan_router_leave(mlxsw_sp_port_vlan); 9005 break; 9006 } 9007 9008 return 0; 9009 } 9010 9011 static int mlxsw_sp_inetaddr_port_event(struct net_device *port_dev, 9012 unsigned long event, bool nomaster, 9013 struct netlink_ext_ack *extack) 9014 { 9015 if (!nomaster && (netif_is_any_bridge_port(port_dev) || 9016 netif_is_lag_port(port_dev))) 9017 return 0; 9018 9019 return mlxsw_sp_inetaddr_port_vlan_event(port_dev, port_dev, event, 9020 MLXSW_SP_DEFAULT_VID, extack); 9021 } 9022 9023 static int __mlxsw_sp_inetaddr_lag_event(struct net_device *l3_dev, 9024 struct net_device *lag_dev, 9025 unsigned long event, u16 vid, 9026 struct netlink_ext_ack *extack) 9027 { 9028 struct net_device *port_dev; 9029 struct list_head *iter; 9030 int err; 9031 9032 netdev_for_each_lower_dev(lag_dev, port_dev, iter) { 9033 if (mlxsw_sp_port_dev_check(port_dev)) { 9034 err = mlxsw_sp_inetaddr_port_vlan_event(l3_dev, 9035 port_dev, 9036 event, vid, 9037 extack); 9038 if (err) 9039 return err; 9040 } 9041 } 9042 9043 return 0; 9044 } 9045 9046 static int mlxsw_sp_inetaddr_lag_event(struct net_device *lag_dev, 9047 unsigned long event, bool nomaster, 9048 struct netlink_ext_ack *extack) 9049 { 9050 if (!nomaster && netif_is_bridge_port(lag_dev)) 9051 return 0; 9052 9053 return __mlxsw_sp_inetaddr_lag_event(lag_dev, lag_dev, event, 9054 MLXSW_SP_DEFAULT_VID, extack); 9055 } 9056 9057 static int mlxsw_sp_inetaddr_bridge_event(struct mlxsw_sp *mlxsw_sp, 9058 struct net_device *l3_dev, 9059 int lower_pvid, 9060 unsigned long event, 9061 struct netlink_ext_ack *extack) 9062 { 9063 struct mlxsw_sp_rif_params params = { 9064 .dev = l3_dev, 9065 }; 9066 struct mlxsw_sp_rif *rif; 9067 int err; 9068 9069 switch (event) { 9070 case NETDEV_UP: 9071 if (netif_is_bridge_master(l3_dev) && br_vlan_enabled(l3_dev)) { 9072 u16 proto; 9073 9074 br_vlan_get_proto(l3_dev, &proto); 9075 if (proto == ETH_P_8021AD) { 9076 NL_SET_ERR_MSG_MOD(extack, "Adding an IP address to 802.1ad bridge is not supported"); 9077 return -EOPNOTSUPP; 9078 } 9079 err = br_vlan_get_pvid(l3_dev, ¶ms.vid); 9080 if (err) 9081 return err; 9082 if (!params.vid) 9083 return 0; 9084 } else if (is_vlan_dev(l3_dev)) { 9085 params.vid = vlan_dev_vlan_id(l3_dev); 9086 9087 /* If the VID matches PVID of the bridge below, the 9088 * bridge owns the RIF for this VLAN. Don't do anything. 9089 */ 9090 if ((int)params.vid == lower_pvid) 9091 return 0; 9092 } 9093 9094 rif = mlxsw_sp_rif_create(mlxsw_sp, ¶ms, extack); 9095 if (IS_ERR(rif)) 9096 return PTR_ERR(rif); 9097 break; 9098 case NETDEV_DOWN: 9099 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev); 9100 mlxsw_sp_rif_destroy(rif); 9101 break; 9102 } 9103 9104 return 0; 9105 } 9106 9107 static int mlxsw_sp_inetaddr_vlan_event(struct mlxsw_sp *mlxsw_sp, 9108 struct net_device *vlan_dev, 9109 unsigned long event, bool nomaster, 9110 struct netlink_ext_ack *extack) 9111 { 9112 struct net_device *real_dev = vlan_dev_real_dev(vlan_dev); 9113 u16 vid = vlan_dev_vlan_id(vlan_dev); 9114 u16 lower_pvid; 9115 int err; 9116 9117 if (!nomaster && netif_is_bridge_port(vlan_dev)) 9118 return 0; 9119 9120 if (mlxsw_sp_port_dev_check(real_dev)) { 9121 return mlxsw_sp_inetaddr_port_vlan_event(vlan_dev, real_dev, 9122 event, vid, extack); 9123 } else if (netif_is_lag_master(real_dev)) { 9124 return __mlxsw_sp_inetaddr_lag_event(vlan_dev, real_dev, event, 9125 vid, extack); 9126 } else if (netif_is_bridge_master(real_dev) && 9127 br_vlan_enabled(real_dev)) { 9128 err = br_vlan_get_pvid(real_dev, &lower_pvid); 9129 if (err) 9130 return err; 9131 return mlxsw_sp_inetaddr_bridge_event(mlxsw_sp, vlan_dev, 9132 lower_pvid, event, 9133 extack); 9134 } 9135 9136 return 0; 9137 } 9138 9139 static bool mlxsw_sp_rif_macvlan_is_vrrp4(const u8 *mac) 9140 { 9141 u8 vrrp4[ETH_ALEN] = { 0x00, 0x00, 0x5e, 0x00, 0x01, 0x00 }; 9142 u8 mask[ETH_ALEN] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0x00 }; 9143 9144 return ether_addr_equal_masked(mac, vrrp4, mask); 9145 } 9146 9147 static bool mlxsw_sp_rif_macvlan_is_vrrp6(const u8 *mac) 9148 { 9149 u8 vrrp6[ETH_ALEN] = { 0x00, 0x00, 0x5e, 0x00, 0x02, 0x00 }; 9150 u8 mask[ETH_ALEN] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0x00 }; 9151 9152 return ether_addr_equal_masked(mac, vrrp6, mask); 9153 } 9154 9155 static int mlxsw_sp_rif_vrrp_op(struct mlxsw_sp *mlxsw_sp, u16 rif_index, 9156 const u8 *mac, bool adding) 9157 { 9158 char ritr_pl[MLXSW_REG_RITR_LEN]; 9159 u8 vrrp_id = adding ? mac[5] : 0; 9160 int err; 9161 9162 if (!mlxsw_sp_rif_macvlan_is_vrrp4(mac) && 9163 !mlxsw_sp_rif_macvlan_is_vrrp6(mac)) 9164 return 0; 9165 9166 mlxsw_reg_ritr_rif_pack(ritr_pl, rif_index); 9167 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 9168 if (err) 9169 return err; 9170 9171 if (mlxsw_sp_rif_macvlan_is_vrrp4(mac)) 9172 mlxsw_reg_ritr_if_vrrp_id_ipv4_set(ritr_pl, vrrp_id); 9173 else 9174 mlxsw_reg_ritr_if_vrrp_id_ipv6_set(ritr_pl, vrrp_id); 9175 9176 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 9177 } 9178 9179 static int mlxsw_sp_rif_macvlan_add(struct mlxsw_sp *mlxsw_sp, 9180 const struct net_device *macvlan_dev, 9181 struct netlink_ext_ack *extack) 9182 { 9183 struct macvlan_dev *vlan = netdev_priv(macvlan_dev); 9184 struct mlxsw_sp_rif *rif; 9185 int err; 9186 9187 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, vlan->lowerdev); 9188 if (!rif) 9189 return 0; 9190 9191 err = mlxsw_sp_rif_fdb_op(mlxsw_sp, macvlan_dev->dev_addr, 9192 mlxsw_sp_fid_index(rif->fid), true); 9193 if (err) 9194 return err; 9195 9196 err = mlxsw_sp_rif_vrrp_op(mlxsw_sp, rif->rif_index, 9197 macvlan_dev->dev_addr, true); 9198 if (err) 9199 goto err_rif_vrrp_add; 9200 9201 /* Make sure the bridge driver does not have this MAC pointing at 9202 * some other port. 9203 */ 9204 if (rif->ops->fdb_del) 9205 rif->ops->fdb_del(rif, macvlan_dev->dev_addr); 9206 9207 return 0; 9208 9209 err_rif_vrrp_add: 9210 mlxsw_sp_rif_fdb_op(mlxsw_sp, macvlan_dev->dev_addr, 9211 mlxsw_sp_fid_index(rif->fid), false); 9212 return err; 9213 } 9214 9215 static void __mlxsw_sp_rif_macvlan_del(struct mlxsw_sp *mlxsw_sp, 9216 const struct net_device *macvlan_dev) 9217 { 9218 struct macvlan_dev *vlan = netdev_priv(macvlan_dev); 9219 struct mlxsw_sp_rif *rif; 9220 9221 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, vlan->lowerdev); 9222 /* If we do not have a RIF, then we already took care of 9223 * removing the macvlan's MAC during RIF deletion. 9224 */ 9225 if (!rif) 9226 return; 9227 mlxsw_sp_rif_vrrp_op(mlxsw_sp, rif->rif_index, macvlan_dev->dev_addr, 9228 false); 9229 mlxsw_sp_rif_fdb_op(mlxsw_sp, macvlan_dev->dev_addr, 9230 mlxsw_sp_fid_index(rif->fid), false); 9231 } 9232 9233 void mlxsw_sp_rif_macvlan_del(struct mlxsw_sp *mlxsw_sp, 9234 const struct net_device *macvlan_dev) 9235 { 9236 mutex_lock(&mlxsw_sp->router->lock); 9237 __mlxsw_sp_rif_macvlan_del(mlxsw_sp, macvlan_dev); 9238 mutex_unlock(&mlxsw_sp->router->lock); 9239 } 9240 9241 static int mlxsw_sp_inetaddr_macvlan_event(struct mlxsw_sp *mlxsw_sp, 9242 struct net_device *macvlan_dev, 9243 unsigned long event, 9244 struct netlink_ext_ack *extack) 9245 { 9246 switch (event) { 9247 case NETDEV_UP: 9248 return mlxsw_sp_rif_macvlan_add(mlxsw_sp, macvlan_dev, extack); 9249 case NETDEV_DOWN: 9250 __mlxsw_sp_rif_macvlan_del(mlxsw_sp, macvlan_dev); 9251 break; 9252 } 9253 9254 return 0; 9255 } 9256 9257 static int __mlxsw_sp_inetaddr_event(struct mlxsw_sp *mlxsw_sp, 9258 struct net_device *dev, 9259 unsigned long event, bool nomaster, 9260 struct netlink_ext_ack *extack) 9261 { 9262 if (mlxsw_sp_port_dev_check(dev)) 9263 return mlxsw_sp_inetaddr_port_event(dev, event, nomaster, 9264 extack); 9265 else if (netif_is_lag_master(dev)) 9266 return mlxsw_sp_inetaddr_lag_event(dev, event, nomaster, 9267 extack); 9268 else if (netif_is_bridge_master(dev)) 9269 return mlxsw_sp_inetaddr_bridge_event(mlxsw_sp, dev, -1, event, 9270 extack); 9271 else if (is_vlan_dev(dev)) 9272 return mlxsw_sp_inetaddr_vlan_event(mlxsw_sp, dev, event, 9273 nomaster, extack); 9274 else if (netif_is_macvlan(dev)) 9275 return mlxsw_sp_inetaddr_macvlan_event(mlxsw_sp, dev, event, 9276 extack); 9277 else 9278 return 0; 9279 } 9280 9281 static int mlxsw_sp_inetaddr_event(struct notifier_block *nb, 9282 unsigned long event, void *ptr) 9283 { 9284 struct in_ifaddr *ifa = (struct in_ifaddr *) ptr; 9285 struct net_device *dev = ifa->ifa_dev->dev; 9286 struct mlxsw_sp_router *router; 9287 struct mlxsw_sp_rif *rif; 9288 int err = 0; 9289 9290 /* NETDEV_UP event is handled by mlxsw_sp_inetaddr_valid_event */ 9291 if (event == NETDEV_UP) 9292 return NOTIFY_DONE; 9293 9294 router = container_of(nb, struct mlxsw_sp_router, inetaddr_nb); 9295 mutex_lock(&router->lock); 9296 rif = mlxsw_sp_rif_find_by_dev(router->mlxsw_sp, dev); 9297 if (!mlxsw_sp_rif_should_config(rif, dev, event)) 9298 goto out; 9299 9300 err = __mlxsw_sp_inetaddr_event(router->mlxsw_sp, dev, event, false, 9301 NULL); 9302 out: 9303 mutex_unlock(&router->lock); 9304 return notifier_from_errno(err); 9305 } 9306 9307 static int mlxsw_sp_inetaddr_valid_event(struct notifier_block *unused, 9308 unsigned long event, void *ptr) 9309 { 9310 struct in_validator_info *ivi = (struct in_validator_info *) ptr; 9311 struct net_device *dev = ivi->ivi_dev->dev; 9312 struct mlxsw_sp *mlxsw_sp; 9313 struct mlxsw_sp_rif *rif; 9314 int err = 0; 9315 9316 mlxsw_sp = mlxsw_sp_lower_get(dev); 9317 if (!mlxsw_sp) 9318 return NOTIFY_DONE; 9319 9320 mutex_lock(&mlxsw_sp->router->lock); 9321 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev); 9322 if (!mlxsw_sp_rif_should_config(rif, dev, event)) 9323 goto out; 9324 9325 err = __mlxsw_sp_inetaddr_event(mlxsw_sp, dev, event, false, 9326 ivi->extack); 9327 out: 9328 mutex_unlock(&mlxsw_sp->router->lock); 9329 return notifier_from_errno(err); 9330 } 9331 9332 struct mlxsw_sp_inet6addr_event_work { 9333 struct work_struct work; 9334 struct mlxsw_sp *mlxsw_sp; 9335 struct net_device *dev; 9336 netdevice_tracker dev_tracker; 9337 unsigned long event; 9338 }; 9339 9340 static void mlxsw_sp_inet6addr_event_work(struct work_struct *work) 9341 { 9342 struct mlxsw_sp_inet6addr_event_work *inet6addr_work = 9343 container_of(work, struct mlxsw_sp_inet6addr_event_work, work); 9344 struct mlxsw_sp *mlxsw_sp = inet6addr_work->mlxsw_sp; 9345 struct net_device *dev = inet6addr_work->dev; 9346 unsigned long event = inet6addr_work->event; 9347 struct mlxsw_sp_rif *rif; 9348 9349 rtnl_lock(); 9350 mutex_lock(&mlxsw_sp->router->lock); 9351 9352 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev); 9353 if (!mlxsw_sp_rif_should_config(rif, dev, event)) 9354 goto out; 9355 9356 __mlxsw_sp_inetaddr_event(mlxsw_sp, dev, event, false, NULL); 9357 out: 9358 mutex_unlock(&mlxsw_sp->router->lock); 9359 rtnl_unlock(); 9360 netdev_put(dev, &inet6addr_work->dev_tracker); 9361 kfree(inet6addr_work); 9362 } 9363 9364 /* Called with rcu_read_lock() */ 9365 static int mlxsw_sp_inet6addr_event(struct notifier_block *nb, 9366 unsigned long event, void *ptr) 9367 { 9368 struct inet6_ifaddr *if6 = (struct inet6_ifaddr *) ptr; 9369 struct mlxsw_sp_inet6addr_event_work *inet6addr_work; 9370 struct net_device *dev = if6->idev->dev; 9371 struct mlxsw_sp_router *router; 9372 9373 /* NETDEV_UP event is handled by mlxsw_sp_inet6addr_valid_event */ 9374 if (event == NETDEV_UP) 9375 return NOTIFY_DONE; 9376 9377 inet6addr_work = kzalloc(sizeof(*inet6addr_work), GFP_ATOMIC); 9378 if (!inet6addr_work) 9379 return NOTIFY_BAD; 9380 9381 router = container_of(nb, struct mlxsw_sp_router, inet6addr_nb); 9382 INIT_WORK(&inet6addr_work->work, mlxsw_sp_inet6addr_event_work); 9383 inet6addr_work->mlxsw_sp = router->mlxsw_sp; 9384 inet6addr_work->dev = dev; 9385 inet6addr_work->event = event; 9386 netdev_hold(dev, &inet6addr_work->dev_tracker, GFP_ATOMIC); 9387 mlxsw_core_schedule_work(&inet6addr_work->work); 9388 9389 return NOTIFY_DONE; 9390 } 9391 9392 static int mlxsw_sp_inet6addr_valid_event(struct notifier_block *unused, 9393 unsigned long event, void *ptr) 9394 { 9395 struct in6_validator_info *i6vi = (struct in6_validator_info *) ptr; 9396 struct net_device *dev = i6vi->i6vi_dev->dev; 9397 struct mlxsw_sp *mlxsw_sp; 9398 struct mlxsw_sp_rif *rif; 9399 int err = 0; 9400 9401 mlxsw_sp = mlxsw_sp_lower_get(dev); 9402 if (!mlxsw_sp) 9403 return NOTIFY_DONE; 9404 9405 mutex_lock(&mlxsw_sp->router->lock); 9406 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev); 9407 if (!mlxsw_sp_rif_should_config(rif, dev, event)) 9408 goto out; 9409 9410 err = __mlxsw_sp_inetaddr_event(mlxsw_sp, dev, event, false, 9411 i6vi->extack); 9412 out: 9413 mutex_unlock(&mlxsw_sp->router->lock); 9414 return notifier_from_errno(err); 9415 } 9416 9417 static int mlxsw_sp_rif_edit(struct mlxsw_sp *mlxsw_sp, u16 rif_index, 9418 const char *mac, int mtu, u8 mac_profile) 9419 { 9420 char ritr_pl[MLXSW_REG_RITR_LEN]; 9421 int err; 9422 9423 mlxsw_reg_ritr_rif_pack(ritr_pl, rif_index); 9424 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 9425 if (err) 9426 return err; 9427 9428 mlxsw_reg_ritr_mtu_set(ritr_pl, mtu); 9429 mlxsw_reg_ritr_if_mac_memcpy_to(ritr_pl, mac); 9430 mlxsw_reg_ritr_if_mac_profile_id_set(ritr_pl, mac_profile); 9431 mlxsw_reg_ritr_op_set(ritr_pl, MLXSW_REG_RITR_RIF_CREATE); 9432 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 9433 } 9434 9435 static int 9436 mlxsw_sp_router_port_change_event(struct mlxsw_sp *mlxsw_sp, 9437 struct mlxsw_sp_rif *rif, 9438 struct netlink_ext_ack *extack) 9439 { 9440 struct net_device *dev = mlxsw_sp_rif_dev(rif); 9441 u8 old_mac_profile; 9442 u16 fid_index; 9443 int err; 9444 9445 fid_index = mlxsw_sp_fid_index(rif->fid); 9446 9447 err = mlxsw_sp_rif_fdb_op(mlxsw_sp, rif->addr, fid_index, false); 9448 if (err) 9449 return err; 9450 9451 old_mac_profile = rif->mac_profile_id; 9452 err = mlxsw_sp_rif_mac_profile_replace(mlxsw_sp, rif, dev->dev_addr, 9453 extack); 9454 if (err) 9455 goto err_rif_mac_profile_replace; 9456 9457 err = mlxsw_sp_rif_edit(mlxsw_sp, rif->rif_index, dev->dev_addr, 9458 dev->mtu, rif->mac_profile_id); 9459 if (err) 9460 goto err_rif_edit; 9461 9462 err = mlxsw_sp_rif_fdb_op(mlxsw_sp, dev->dev_addr, fid_index, true); 9463 if (err) 9464 goto err_rif_fdb_op; 9465 9466 if (rif->mtu != dev->mtu) { 9467 struct mlxsw_sp_vr *vr; 9468 int i; 9469 9470 /* The RIF is relevant only to its mr_table instance, as unlike 9471 * unicast routing, in multicast routing a RIF cannot be shared 9472 * between several multicast routing tables. 9473 */ 9474 vr = &mlxsw_sp->router->vrs[rif->vr_id]; 9475 for (i = 0; i < MLXSW_SP_L3_PROTO_MAX; i++) 9476 mlxsw_sp_mr_rif_mtu_update(vr->mr_table[i], 9477 rif, dev->mtu); 9478 } 9479 9480 ether_addr_copy(rif->addr, dev->dev_addr); 9481 rif->mtu = dev->mtu; 9482 9483 netdev_dbg(dev, "Updated RIF=%d\n", rif->rif_index); 9484 9485 return 0; 9486 9487 err_rif_fdb_op: 9488 mlxsw_sp_rif_edit(mlxsw_sp, rif->rif_index, rif->addr, rif->mtu, 9489 old_mac_profile); 9490 err_rif_edit: 9491 mlxsw_sp_rif_mac_profile_replace(mlxsw_sp, rif, rif->addr, extack); 9492 err_rif_mac_profile_replace: 9493 mlxsw_sp_rif_fdb_op(mlxsw_sp, rif->addr, fid_index, true); 9494 return err; 9495 } 9496 9497 static int mlxsw_sp_router_port_pre_changeaddr_event(struct mlxsw_sp_rif *rif, 9498 struct netdev_notifier_pre_changeaddr_info *info) 9499 { 9500 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 9501 struct mlxsw_sp_rif_mac_profile *profile; 9502 struct netlink_ext_ack *extack; 9503 u8 max_rif_mac_profiles; 9504 u64 occ; 9505 9506 extack = netdev_notifier_info_to_extack(&info->info); 9507 9508 profile = mlxsw_sp_rif_mac_profile_find(mlxsw_sp, info->dev_addr); 9509 if (profile) 9510 return 0; 9511 9512 max_rif_mac_profiles = mlxsw_sp->router->max_rif_mac_profile; 9513 occ = mlxsw_sp_rif_mac_profiles_occ_get(mlxsw_sp); 9514 if (occ < max_rif_mac_profiles) 9515 return 0; 9516 9517 if (!mlxsw_sp_rif_mac_profile_is_shared(rif)) 9518 return 0; 9519 9520 NL_SET_ERR_MSG_MOD(extack, "Exceeded number of supported router interface MAC profiles"); 9521 return -ENOBUFS; 9522 } 9523 9524 static bool mlxsw_sp_router_netdevice_interesting(struct mlxsw_sp *mlxsw_sp, 9525 struct net_device *dev) 9526 { 9527 struct vlan_dev_priv *vlan; 9528 9529 if (netif_is_lag_master(dev) || 9530 netif_is_bridge_master(dev) || 9531 mlxsw_sp_port_dev_check(dev) || 9532 mlxsw_sp_netdev_is_ipip_ol(mlxsw_sp, dev) || 9533 netif_is_l3_master(dev)) 9534 return true; 9535 9536 if (!is_vlan_dev(dev)) 9537 return false; 9538 9539 vlan = vlan_dev_priv(dev); 9540 return netif_is_lag_master(vlan->real_dev) || 9541 netif_is_bridge_master(vlan->real_dev) || 9542 mlxsw_sp_port_dev_check(vlan->real_dev); 9543 } 9544 9545 static struct mlxsw_sp_crif * 9546 mlxsw_sp_crif_register(struct mlxsw_sp_router *router, struct net_device *dev) 9547 { 9548 struct mlxsw_sp_crif *crif; 9549 int err; 9550 9551 if (WARN_ON(mlxsw_sp_crif_lookup(router, dev))) 9552 return NULL; 9553 9554 crif = mlxsw_sp_crif_alloc(dev); 9555 if (!crif) 9556 return ERR_PTR(-ENOMEM); 9557 9558 err = mlxsw_sp_crif_insert(router, crif); 9559 if (err) 9560 goto err_netdev_insert; 9561 9562 return crif; 9563 9564 err_netdev_insert: 9565 mlxsw_sp_crif_free(crif); 9566 return ERR_PTR(err); 9567 } 9568 9569 static void mlxsw_sp_crif_unregister(struct mlxsw_sp_router *router, 9570 struct mlxsw_sp_crif *crif) 9571 { 9572 struct mlxsw_sp_nexthop *nh, *tmp; 9573 9574 mlxsw_sp_crif_remove(router, crif); 9575 9576 list_for_each_entry_safe(nh, tmp, &crif->nexthop_list, crif_list_node) 9577 mlxsw_sp_nexthop_type_fini(router->mlxsw_sp, nh); 9578 9579 if (crif->rif) 9580 crif->can_destroy = true; 9581 else 9582 mlxsw_sp_crif_free(crif); 9583 } 9584 9585 static int mlxsw_sp_netdevice_register(struct mlxsw_sp_router *router, 9586 struct net_device *dev) 9587 { 9588 struct mlxsw_sp_crif *crif; 9589 9590 if (!mlxsw_sp_router_netdevice_interesting(router->mlxsw_sp, dev)) 9591 return 0; 9592 9593 crif = mlxsw_sp_crif_register(router, dev); 9594 return PTR_ERR_OR_ZERO(crif); 9595 } 9596 9597 static void mlxsw_sp_netdevice_unregister(struct mlxsw_sp_router *router, 9598 struct net_device *dev) 9599 { 9600 struct mlxsw_sp_crif *crif; 9601 9602 if (!mlxsw_sp_router_netdevice_interesting(router->mlxsw_sp, dev)) 9603 return; 9604 9605 /* netdev_run_todo(), by way of netdev_wait_allrefs_any(), rebroadcasts 9606 * the NETDEV_UNREGISTER message, so we can get here twice. If that's 9607 * what happened, the netdevice state is NETREG_UNREGISTERED. In that 9608 * case, we expect to have collected the CRIF already, and warn if it 9609 * still exists. Otherwise we expect the CRIF to exist. 9610 */ 9611 crif = mlxsw_sp_crif_lookup(router, dev); 9612 if (dev->reg_state == NETREG_UNREGISTERED) { 9613 if (!WARN_ON(crif)) 9614 return; 9615 } 9616 if (WARN_ON(!crif)) 9617 return; 9618 9619 mlxsw_sp_crif_unregister(router, crif); 9620 } 9621 9622 static bool mlxsw_sp_is_offload_xstats_event(unsigned long event) 9623 { 9624 switch (event) { 9625 case NETDEV_OFFLOAD_XSTATS_ENABLE: 9626 case NETDEV_OFFLOAD_XSTATS_DISABLE: 9627 case NETDEV_OFFLOAD_XSTATS_REPORT_USED: 9628 case NETDEV_OFFLOAD_XSTATS_REPORT_DELTA: 9629 return true; 9630 } 9631 9632 return false; 9633 } 9634 9635 static int 9636 mlxsw_sp_router_port_offload_xstats_cmd(struct mlxsw_sp_rif *rif, 9637 unsigned long event, 9638 struct netdev_notifier_offload_xstats_info *info) 9639 { 9640 switch (info->type) { 9641 case NETDEV_OFFLOAD_XSTATS_TYPE_L3: 9642 break; 9643 default: 9644 return 0; 9645 } 9646 9647 switch (event) { 9648 case NETDEV_OFFLOAD_XSTATS_ENABLE: 9649 return mlxsw_sp_router_port_l3_stats_enable(rif); 9650 case NETDEV_OFFLOAD_XSTATS_DISABLE: 9651 mlxsw_sp_router_port_l3_stats_disable(rif); 9652 return 0; 9653 case NETDEV_OFFLOAD_XSTATS_REPORT_USED: 9654 mlxsw_sp_router_port_l3_stats_report_used(rif, info); 9655 return 0; 9656 case NETDEV_OFFLOAD_XSTATS_REPORT_DELTA: 9657 return mlxsw_sp_router_port_l3_stats_report_delta(rif, info); 9658 } 9659 9660 WARN_ON_ONCE(1); 9661 return 0; 9662 } 9663 9664 static int 9665 mlxsw_sp_netdevice_offload_xstats_cmd(struct mlxsw_sp *mlxsw_sp, 9666 struct net_device *dev, 9667 unsigned long event, 9668 struct netdev_notifier_offload_xstats_info *info) 9669 { 9670 struct mlxsw_sp_rif *rif; 9671 9672 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev); 9673 if (!rif) 9674 return 0; 9675 9676 return mlxsw_sp_router_port_offload_xstats_cmd(rif, event, info); 9677 } 9678 9679 static bool mlxsw_sp_is_router_event(unsigned long event) 9680 { 9681 switch (event) { 9682 case NETDEV_PRE_CHANGEADDR: 9683 case NETDEV_CHANGEADDR: 9684 case NETDEV_CHANGEMTU: 9685 return true; 9686 default: 9687 return false; 9688 } 9689 } 9690 9691 static int mlxsw_sp_netdevice_router_port_event(struct net_device *dev, 9692 unsigned long event, void *ptr) 9693 { 9694 struct netlink_ext_ack *extack = netdev_notifier_info_to_extack(ptr); 9695 struct mlxsw_sp *mlxsw_sp; 9696 struct mlxsw_sp_rif *rif; 9697 9698 mlxsw_sp = mlxsw_sp_lower_get(dev); 9699 if (!mlxsw_sp) 9700 return 0; 9701 9702 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev); 9703 if (!rif) 9704 return 0; 9705 9706 switch (event) { 9707 case NETDEV_CHANGEMTU: 9708 case NETDEV_CHANGEADDR: 9709 return mlxsw_sp_router_port_change_event(mlxsw_sp, rif, extack); 9710 case NETDEV_PRE_CHANGEADDR: 9711 return mlxsw_sp_router_port_pre_changeaddr_event(rif, ptr); 9712 default: 9713 WARN_ON_ONCE(1); 9714 break; 9715 } 9716 9717 return 0; 9718 } 9719 9720 static int mlxsw_sp_port_vrf_join(struct mlxsw_sp *mlxsw_sp, 9721 struct net_device *l3_dev, 9722 struct netlink_ext_ack *extack) 9723 { 9724 struct mlxsw_sp_rif *rif; 9725 9726 /* If netdev is already associated with a RIF, then we need to 9727 * destroy it and create a new one with the new virtual router ID. 9728 */ 9729 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev); 9730 if (rif) 9731 __mlxsw_sp_inetaddr_event(mlxsw_sp, l3_dev, NETDEV_DOWN, false, 9732 extack); 9733 9734 return __mlxsw_sp_inetaddr_event(mlxsw_sp, l3_dev, NETDEV_UP, false, 9735 extack); 9736 } 9737 9738 static void mlxsw_sp_port_vrf_leave(struct mlxsw_sp *mlxsw_sp, 9739 struct net_device *l3_dev) 9740 { 9741 struct mlxsw_sp_rif *rif; 9742 9743 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev); 9744 if (!rif) 9745 return; 9746 __mlxsw_sp_inetaddr_event(mlxsw_sp, l3_dev, NETDEV_DOWN, false, NULL); 9747 } 9748 9749 static bool mlxsw_sp_is_vrf_event(unsigned long event, void *ptr) 9750 { 9751 struct netdev_notifier_changeupper_info *info = ptr; 9752 9753 if (event != NETDEV_PRECHANGEUPPER && event != NETDEV_CHANGEUPPER) 9754 return false; 9755 return netif_is_l3_master(info->upper_dev); 9756 } 9757 9758 static int 9759 mlxsw_sp_netdevice_vrf_event(struct net_device *l3_dev, unsigned long event, 9760 struct netdev_notifier_changeupper_info *info) 9761 { 9762 struct mlxsw_sp *mlxsw_sp = mlxsw_sp_lower_get(l3_dev); 9763 int err = 0; 9764 9765 /* We do not create a RIF for a macvlan, but only use it to 9766 * direct more MAC addresses to the router. 9767 */ 9768 if (!mlxsw_sp || netif_is_macvlan(l3_dev)) 9769 return 0; 9770 9771 switch (event) { 9772 case NETDEV_PRECHANGEUPPER: 9773 break; 9774 case NETDEV_CHANGEUPPER: 9775 if (info->linking) { 9776 struct netlink_ext_ack *extack; 9777 9778 extack = netdev_notifier_info_to_extack(&info->info); 9779 err = mlxsw_sp_port_vrf_join(mlxsw_sp, l3_dev, extack); 9780 } else { 9781 mlxsw_sp_port_vrf_leave(mlxsw_sp, l3_dev); 9782 } 9783 break; 9784 } 9785 9786 return err; 9787 } 9788 9789 struct mlxsw_sp_router_replay_inetaddr_up { 9790 struct mlxsw_sp *mlxsw_sp; 9791 struct netlink_ext_ack *extack; 9792 unsigned int done; 9793 bool deslavement; 9794 }; 9795 9796 static int mlxsw_sp_router_replay_inetaddr_up(struct net_device *dev, 9797 struct netdev_nested_priv *priv) 9798 { 9799 struct mlxsw_sp_router_replay_inetaddr_up *ctx = priv->data; 9800 bool nomaster = ctx->deslavement; 9801 struct mlxsw_sp_crif *crif; 9802 int err; 9803 9804 if (mlxsw_sp_dev_addr_list_empty(dev)) 9805 return 0; 9806 9807 crif = mlxsw_sp_crif_lookup(ctx->mlxsw_sp->router, dev); 9808 if (!crif || crif->rif) 9809 return 0; 9810 9811 if (!mlxsw_sp_rif_should_config(crif->rif, dev, NETDEV_UP)) 9812 return 0; 9813 9814 err = __mlxsw_sp_inetaddr_event(ctx->mlxsw_sp, dev, NETDEV_UP, 9815 nomaster, ctx->extack); 9816 if (err) 9817 return err; 9818 9819 ctx->done++; 9820 return 0; 9821 } 9822 9823 static int mlxsw_sp_router_unreplay_inetaddr_up(struct net_device *dev, 9824 struct netdev_nested_priv *priv) 9825 { 9826 struct mlxsw_sp_router_replay_inetaddr_up *ctx = priv->data; 9827 bool nomaster = ctx->deslavement; 9828 struct mlxsw_sp_crif *crif; 9829 9830 if (!ctx->done) 9831 return 0; 9832 9833 if (mlxsw_sp_dev_addr_list_empty(dev)) 9834 return 0; 9835 9836 crif = mlxsw_sp_crif_lookup(ctx->mlxsw_sp->router, dev); 9837 if (!crif || !crif->rif) 9838 return 0; 9839 9840 /* We are rolling back NETDEV_UP, so ask for that. */ 9841 if (!mlxsw_sp_rif_should_config(crif->rif, dev, NETDEV_UP)) 9842 return 0; 9843 9844 __mlxsw_sp_inetaddr_event(ctx->mlxsw_sp, dev, NETDEV_DOWN, nomaster, 9845 NULL); 9846 9847 ctx->done--; 9848 return 0; 9849 } 9850 9851 int mlxsw_sp_netdevice_enslavement_replay(struct mlxsw_sp *mlxsw_sp, 9852 struct net_device *upper_dev, 9853 struct netlink_ext_ack *extack) 9854 { 9855 struct mlxsw_sp_router_replay_inetaddr_up ctx = { 9856 .mlxsw_sp = mlxsw_sp, 9857 .extack = extack, 9858 .deslavement = false, 9859 }; 9860 struct netdev_nested_priv priv = { 9861 .data = &ctx, 9862 }; 9863 int err; 9864 9865 err = mlxsw_sp_router_replay_inetaddr_up(upper_dev, &priv); 9866 if (err) 9867 return err; 9868 9869 err = netdev_walk_all_upper_dev_rcu(upper_dev, 9870 mlxsw_sp_router_replay_inetaddr_up, 9871 &priv); 9872 if (err) 9873 goto err_replay_up; 9874 9875 return 0; 9876 9877 err_replay_up: 9878 netdev_walk_all_upper_dev_rcu(upper_dev, 9879 mlxsw_sp_router_unreplay_inetaddr_up, 9880 &priv); 9881 mlxsw_sp_router_unreplay_inetaddr_up(upper_dev, &priv); 9882 return err; 9883 } 9884 9885 void mlxsw_sp_netdevice_deslavement_replay(struct mlxsw_sp *mlxsw_sp, 9886 struct net_device *dev) 9887 { 9888 struct mlxsw_sp_router_replay_inetaddr_up ctx = { 9889 .mlxsw_sp = mlxsw_sp, 9890 .deslavement = true, 9891 }; 9892 struct netdev_nested_priv priv = { 9893 .data = &ctx, 9894 }; 9895 9896 mlxsw_sp_router_replay_inetaddr_up(dev, &priv); 9897 } 9898 9899 static int 9900 mlxsw_sp_port_vid_router_join_existing(struct mlxsw_sp_port *mlxsw_sp_port, 9901 u16 vid, struct net_device *dev, 9902 struct netlink_ext_ack *extack) 9903 { 9904 struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan; 9905 9906 mlxsw_sp_port_vlan = mlxsw_sp_port_vlan_find_by_vid(mlxsw_sp_port, 9907 vid); 9908 if (WARN_ON(!mlxsw_sp_port_vlan)) 9909 return -EINVAL; 9910 9911 return mlxsw_sp_port_vlan_router_join_existing(mlxsw_sp_port_vlan, 9912 dev, extack); 9913 } 9914 9915 static void 9916 mlxsw_sp_port_vid_router_leave(struct mlxsw_sp_port *mlxsw_sp_port, u16 vid, 9917 struct net_device *dev) 9918 { 9919 struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan; 9920 9921 mlxsw_sp_port_vlan = mlxsw_sp_port_vlan_find_by_vid(mlxsw_sp_port, 9922 vid); 9923 if (WARN_ON(!mlxsw_sp_port_vlan)) 9924 return; 9925 9926 __mlxsw_sp_port_vlan_router_leave(mlxsw_sp_port_vlan); 9927 } 9928 9929 static int __mlxsw_sp_router_port_join_lag(struct mlxsw_sp_port *mlxsw_sp_port, 9930 struct net_device *lag_dev, 9931 struct netlink_ext_ack *extack) 9932 { 9933 u16 default_vid = MLXSW_SP_DEFAULT_VID; 9934 struct net_device *upper_dev; 9935 struct list_head *iter; 9936 int done = 0; 9937 u16 vid; 9938 int err; 9939 9940 err = mlxsw_sp_port_vid_router_join_existing(mlxsw_sp_port, default_vid, 9941 lag_dev, extack); 9942 if (err) 9943 return err; 9944 9945 netdev_for_each_upper_dev_rcu(lag_dev, upper_dev, iter) { 9946 if (!is_vlan_dev(upper_dev)) 9947 continue; 9948 9949 vid = vlan_dev_vlan_id(upper_dev); 9950 err = mlxsw_sp_port_vid_router_join_existing(mlxsw_sp_port, vid, 9951 upper_dev, extack); 9952 if (err) 9953 goto err_router_join_dev; 9954 9955 ++done; 9956 } 9957 9958 return 0; 9959 9960 err_router_join_dev: 9961 netdev_for_each_upper_dev_rcu(lag_dev, upper_dev, iter) { 9962 if (!is_vlan_dev(upper_dev)) 9963 continue; 9964 if (!done--) 9965 break; 9966 9967 vid = vlan_dev_vlan_id(upper_dev); 9968 mlxsw_sp_port_vid_router_leave(mlxsw_sp_port, vid, upper_dev); 9969 } 9970 9971 mlxsw_sp_port_vid_router_leave(mlxsw_sp_port, default_vid, lag_dev); 9972 return err; 9973 } 9974 9975 static void 9976 __mlxsw_sp_router_port_leave_lag(struct mlxsw_sp_port *mlxsw_sp_port, 9977 struct net_device *lag_dev) 9978 { 9979 u16 default_vid = MLXSW_SP_DEFAULT_VID; 9980 struct net_device *upper_dev; 9981 struct list_head *iter; 9982 u16 vid; 9983 9984 netdev_for_each_upper_dev_rcu(lag_dev, upper_dev, iter) { 9985 if (!is_vlan_dev(upper_dev)) 9986 continue; 9987 9988 vid = vlan_dev_vlan_id(upper_dev); 9989 mlxsw_sp_port_vid_router_leave(mlxsw_sp_port, vid, upper_dev); 9990 } 9991 9992 mlxsw_sp_port_vid_router_leave(mlxsw_sp_port, default_vid, lag_dev); 9993 } 9994 9995 int mlxsw_sp_router_port_join_lag(struct mlxsw_sp_port *mlxsw_sp_port, 9996 struct net_device *lag_dev, 9997 struct netlink_ext_ack *extack) 9998 { 9999 int err; 10000 10001 mutex_lock(&mlxsw_sp_port->mlxsw_sp->router->lock); 10002 err = __mlxsw_sp_router_port_join_lag(mlxsw_sp_port, lag_dev, extack); 10003 mutex_unlock(&mlxsw_sp_port->mlxsw_sp->router->lock); 10004 10005 return err; 10006 } 10007 10008 void mlxsw_sp_router_port_leave_lag(struct mlxsw_sp_port *mlxsw_sp_port, 10009 struct net_device *lag_dev) 10010 { 10011 mutex_lock(&mlxsw_sp_port->mlxsw_sp->router->lock); 10012 __mlxsw_sp_router_port_leave_lag(mlxsw_sp_port, lag_dev); 10013 mutex_unlock(&mlxsw_sp_port->mlxsw_sp->router->lock); 10014 } 10015 10016 static int mlxsw_sp_router_netdevice_event(struct notifier_block *nb, 10017 unsigned long event, void *ptr) 10018 { 10019 struct net_device *dev = netdev_notifier_info_to_dev(ptr); 10020 struct mlxsw_sp_router *router; 10021 struct mlxsw_sp *mlxsw_sp; 10022 int err = 0; 10023 10024 router = container_of(nb, struct mlxsw_sp_router, netdevice_nb); 10025 mlxsw_sp = router->mlxsw_sp; 10026 10027 mutex_lock(&mlxsw_sp->router->lock); 10028 10029 if (event == NETDEV_REGISTER) { 10030 err = mlxsw_sp_netdevice_register(router, dev); 10031 if (err) 10032 /* No need to roll this back, UNREGISTER will collect it 10033 * anyhow. 10034 */ 10035 goto out; 10036 } 10037 10038 if (mlxsw_sp_is_offload_xstats_event(event)) 10039 err = mlxsw_sp_netdevice_offload_xstats_cmd(mlxsw_sp, dev, 10040 event, ptr); 10041 else if (mlxsw_sp_netdev_is_ipip_ol(mlxsw_sp, dev)) 10042 err = mlxsw_sp_netdevice_ipip_ol_event(mlxsw_sp, dev, 10043 event, ptr); 10044 else if (mlxsw_sp_netdev_is_ipip_ul(mlxsw_sp, dev)) 10045 err = mlxsw_sp_netdevice_ipip_ul_event(mlxsw_sp, dev, 10046 event, ptr); 10047 else if (mlxsw_sp_is_router_event(event)) 10048 err = mlxsw_sp_netdevice_router_port_event(dev, event, ptr); 10049 else if (mlxsw_sp_is_vrf_event(event, ptr)) 10050 err = mlxsw_sp_netdevice_vrf_event(dev, event, ptr); 10051 10052 if (event == NETDEV_UNREGISTER) 10053 mlxsw_sp_netdevice_unregister(router, dev); 10054 10055 out: 10056 mutex_unlock(&mlxsw_sp->router->lock); 10057 10058 return notifier_from_errno(err); 10059 } 10060 10061 struct mlxsw_sp_macvlan_replay { 10062 struct mlxsw_sp *mlxsw_sp; 10063 struct netlink_ext_ack *extack; 10064 }; 10065 10066 static int mlxsw_sp_macvlan_replay_upper(struct net_device *dev, 10067 struct netdev_nested_priv *priv) 10068 { 10069 const struct mlxsw_sp_macvlan_replay *rms = priv->data; 10070 struct netlink_ext_ack *extack = rms->extack; 10071 struct mlxsw_sp *mlxsw_sp = rms->mlxsw_sp; 10072 10073 if (!netif_is_macvlan(dev)) 10074 return 0; 10075 10076 return mlxsw_sp_rif_macvlan_add(mlxsw_sp, dev, extack); 10077 } 10078 10079 static int mlxsw_sp_macvlan_replay(struct mlxsw_sp_rif *rif, 10080 struct netlink_ext_ack *extack) 10081 { 10082 struct mlxsw_sp_macvlan_replay rms = { 10083 .mlxsw_sp = rif->mlxsw_sp, 10084 .extack = extack, 10085 }; 10086 struct netdev_nested_priv priv = { 10087 .data = &rms, 10088 }; 10089 10090 return netdev_walk_all_upper_dev_rcu(mlxsw_sp_rif_dev(rif), 10091 mlxsw_sp_macvlan_replay_upper, 10092 &priv); 10093 } 10094 10095 static int __mlxsw_sp_rif_macvlan_flush(struct net_device *dev, 10096 struct netdev_nested_priv *priv) 10097 { 10098 struct mlxsw_sp_rif *rif = (struct mlxsw_sp_rif *)priv->data; 10099 10100 if (!netif_is_macvlan(dev)) 10101 return 0; 10102 10103 return mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, dev->dev_addr, 10104 mlxsw_sp_fid_index(rif->fid), false); 10105 } 10106 10107 static int mlxsw_sp_rif_macvlan_flush(struct mlxsw_sp_rif *rif) 10108 { 10109 struct net_device *dev = mlxsw_sp_rif_dev(rif); 10110 struct netdev_nested_priv priv = { 10111 .data = (void *)rif, 10112 }; 10113 10114 if (!netif_is_macvlan_port(dev)) 10115 return 0; 10116 10117 return netdev_walk_all_upper_dev_rcu(dev, 10118 __mlxsw_sp_rif_macvlan_flush, &priv); 10119 } 10120 10121 static void mlxsw_sp_rif_subport_setup(struct mlxsw_sp_rif *rif, 10122 const struct mlxsw_sp_rif_params *params) 10123 { 10124 struct mlxsw_sp_rif_subport *rif_subport; 10125 10126 rif_subport = mlxsw_sp_rif_subport_rif(rif); 10127 refcount_set(&rif_subport->ref_count, 1); 10128 rif_subport->vid = params->vid; 10129 rif_subport->lag = params->lag; 10130 if (params->lag) 10131 rif_subport->lag_id = params->lag_id; 10132 else 10133 rif_subport->system_port = params->system_port; 10134 } 10135 10136 static int mlxsw_sp_rif_subport_op(struct mlxsw_sp_rif *rif, bool enable) 10137 { 10138 struct net_device *dev = mlxsw_sp_rif_dev(rif); 10139 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 10140 struct mlxsw_sp_rif_subport *rif_subport; 10141 char ritr_pl[MLXSW_REG_RITR_LEN]; 10142 u16 efid; 10143 10144 rif_subport = mlxsw_sp_rif_subport_rif(rif); 10145 mlxsw_reg_ritr_pack(ritr_pl, enable, MLXSW_REG_RITR_SP_IF, 10146 rif->rif_index, rif->vr_id, dev->mtu); 10147 mlxsw_reg_ritr_mac_pack(ritr_pl, dev->dev_addr); 10148 mlxsw_reg_ritr_if_mac_profile_id_set(ritr_pl, rif->mac_profile_id); 10149 efid = mlxsw_sp_fid_index(rif->fid); 10150 mlxsw_reg_ritr_sp_if_pack(ritr_pl, rif_subport->lag, 10151 rif_subport->lag ? rif_subport->lag_id : 10152 rif_subport->system_port, 10153 efid, 0); 10154 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 10155 } 10156 10157 static int mlxsw_sp_rif_subport_configure(struct mlxsw_sp_rif *rif, 10158 struct netlink_ext_ack *extack) 10159 { 10160 struct net_device *dev = mlxsw_sp_rif_dev(rif); 10161 u8 mac_profile; 10162 int err; 10163 10164 err = mlxsw_sp_rif_mac_profile_get(rif->mlxsw_sp, rif->addr, 10165 &mac_profile, extack); 10166 if (err) 10167 return err; 10168 rif->mac_profile_id = mac_profile; 10169 10170 err = mlxsw_sp_rif_subport_op(rif, true); 10171 if (err) 10172 goto err_rif_subport_op; 10173 10174 err = mlxsw_sp_macvlan_replay(rif, extack); 10175 if (err) 10176 goto err_macvlan_replay; 10177 10178 err = mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, dev->dev_addr, 10179 mlxsw_sp_fid_index(rif->fid), true); 10180 if (err) 10181 goto err_rif_fdb_op; 10182 10183 err = mlxsw_sp_fid_rif_set(rif->fid, rif); 10184 if (err) 10185 goto err_fid_rif_set; 10186 10187 return 0; 10188 10189 err_fid_rif_set: 10190 mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, dev->dev_addr, 10191 mlxsw_sp_fid_index(rif->fid), false); 10192 err_rif_fdb_op: 10193 mlxsw_sp_rif_macvlan_flush(rif); 10194 err_macvlan_replay: 10195 mlxsw_sp_rif_subport_op(rif, false); 10196 err_rif_subport_op: 10197 mlxsw_sp_rif_mac_profile_put(rif->mlxsw_sp, mac_profile); 10198 return err; 10199 } 10200 10201 static void mlxsw_sp_rif_subport_deconfigure(struct mlxsw_sp_rif *rif) 10202 { 10203 struct net_device *dev = mlxsw_sp_rif_dev(rif); 10204 struct mlxsw_sp_fid *fid = rif->fid; 10205 10206 mlxsw_sp_fid_rif_unset(fid); 10207 mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, dev->dev_addr, 10208 mlxsw_sp_fid_index(fid), false); 10209 mlxsw_sp_rif_macvlan_flush(rif); 10210 mlxsw_sp_rif_subport_op(rif, false); 10211 mlxsw_sp_rif_mac_profile_put(rif->mlxsw_sp, rif->mac_profile_id); 10212 } 10213 10214 static struct mlxsw_sp_fid * 10215 mlxsw_sp_rif_subport_fid_get(struct mlxsw_sp_rif *rif, 10216 const struct mlxsw_sp_rif_params *params, 10217 struct netlink_ext_ack *extack) 10218 { 10219 return mlxsw_sp_fid_rfid_get(rif->mlxsw_sp, rif->rif_index); 10220 } 10221 10222 static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_subport_ops = { 10223 .type = MLXSW_SP_RIF_TYPE_SUBPORT, 10224 .rif_size = sizeof(struct mlxsw_sp_rif_subport), 10225 .setup = mlxsw_sp_rif_subport_setup, 10226 .configure = mlxsw_sp_rif_subport_configure, 10227 .deconfigure = mlxsw_sp_rif_subport_deconfigure, 10228 .fid_get = mlxsw_sp_rif_subport_fid_get, 10229 }; 10230 10231 static int mlxsw_sp_rif_fid_op(struct mlxsw_sp_rif *rif, u16 fid, bool enable) 10232 { 10233 enum mlxsw_reg_ritr_if_type type = MLXSW_REG_RITR_FID_IF; 10234 struct net_device *dev = mlxsw_sp_rif_dev(rif); 10235 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 10236 char ritr_pl[MLXSW_REG_RITR_LEN]; 10237 10238 mlxsw_reg_ritr_pack(ritr_pl, enable, type, rif->rif_index, rif->vr_id, 10239 dev->mtu); 10240 mlxsw_reg_ritr_mac_pack(ritr_pl, dev->dev_addr); 10241 mlxsw_reg_ritr_if_mac_profile_id_set(ritr_pl, rif->mac_profile_id); 10242 mlxsw_reg_ritr_fid_if_fid_set(ritr_pl, fid); 10243 10244 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 10245 } 10246 10247 u16 mlxsw_sp_router_port(const struct mlxsw_sp *mlxsw_sp) 10248 { 10249 return mlxsw_core_max_ports(mlxsw_sp->core) + 1; 10250 } 10251 10252 static int mlxsw_sp_rif_fid_configure(struct mlxsw_sp_rif *rif, 10253 struct netlink_ext_ack *extack) 10254 { 10255 struct net_device *dev = mlxsw_sp_rif_dev(rif); 10256 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 10257 u16 fid_index = mlxsw_sp_fid_index(rif->fid); 10258 u8 mac_profile; 10259 int err; 10260 10261 err = mlxsw_sp_rif_mac_profile_get(mlxsw_sp, rif->addr, 10262 &mac_profile, extack); 10263 if (err) 10264 return err; 10265 rif->mac_profile_id = mac_profile; 10266 10267 err = mlxsw_sp_rif_fid_op(rif, fid_index, true); 10268 if (err) 10269 goto err_rif_fid_op; 10270 10271 err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC, 10272 mlxsw_sp_router_port(mlxsw_sp), true); 10273 if (err) 10274 goto err_fid_mc_flood_set; 10275 10276 err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC, 10277 mlxsw_sp_router_port(mlxsw_sp), true); 10278 if (err) 10279 goto err_fid_bc_flood_set; 10280 10281 err = mlxsw_sp_macvlan_replay(rif, extack); 10282 if (err) 10283 goto err_macvlan_replay; 10284 10285 err = mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, dev->dev_addr, 10286 mlxsw_sp_fid_index(rif->fid), true); 10287 if (err) 10288 goto err_rif_fdb_op; 10289 10290 err = mlxsw_sp_fid_rif_set(rif->fid, rif); 10291 if (err) 10292 goto err_fid_rif_set; 10293 10294 return 0; 10295 10296 err_fid_rif_set: 10297 mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, dev->dev_addr, 10298 mlxsw_sp_fid_index(rif->fid), false); 10299 err_rif_fdb_op: 10300 mlxsw_sp_rif_macvlan_flush(rif); 10301 err_macvlan_replay: 10302 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC, 10303 mlxsw_sp_router_port(mlxsw_sp), false); 10304 err_fid_bc_flood_set: 10305 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC, 10306 mlxsw_sp_router_port(mlxsw_sp), false); 10307 err_fid_mc_flood_set: 10308 mlxsw_sp_rif_fid_op(rif, fid_index, false); 10309 err_rif_fid_op: 10310 mlxsw_sp_rif_mac_profile_put(mlxsw_sp, mac_profile); 10311 return err; 10312 } 10313 10314 static void mlxsw_sp_rif_fid_deconfigure(struct mlxsw_sp_rif *rif) 10315 { 10316 struct net_device *dev = mlxsw_sp_rif_dev(rif); 10317 u16 fid_index = mlxsw_sp_fid_index(rif->fid); 10318 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 10319 struct mlxsw_sp_fid *fid = rif->fid; 10320 10321 mlxsw_sp_fid_rif_unset(fid); 10322 mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, dev->dev_addr, 10323 mlxsw_sp_fid_index(fid), false); 10324 mlxsw_sp_rif_macvlan_flush(rif); 10325 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC, 10326 mlxsw_sp_router_port(mlxsw_sp), false); 10327 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC, 10328 mlxsw_sp_router_port(mlxsw_sp), false); 10329 mlxsw_sp_rif_fid_op(rif, fid_index, false); 10330 mlxsw_sp_rif_mac_profile_put(rif->mlxsw_sp, rif->mac_profile_id); 10331 } 10332 10333 static struct mlxsw_sp_fid * 10334 mlxsw_sp_rif_fid_fid_get(struct mlxsw_sp_rif *rif, 10335 const struct mlxsw_sp_rif_params *params, 10336 struct netlink_ext_ack *extack) 10337 { 10338 int rif_ifindex = mlxsw_sp_rif_dev_ifindex(rif); 10339 10340 return mlxsw_sp_fid_8021d_get(rif->mlxsw_sp, rif_ifindex); 10341 } 10342 10343 static void mlxsw_sp_rif_fid_fdb_del(struct mlxsw_sp_rif *rif, const char *mac) 10344 { 10345 struct switchdev_notifier_fdb_info info = {}; 10346 struct net_device *dev; 10347 10348 dev = br_fdb_find_port(mlxsw_sp_rif_dev(rif), mac, 0); 10349 if (!dev) 10350 return; 10351 10352 info.addr = mac; 10353 info.vid = 0; 10354 call_switchdev_notifiers(SWITCHDEV_FDB_DEL_TO_BRIDGE, dev, &info.info, 10355 NULL); 10356 } 10357 10358 static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_fid_ops = { 10359 .type = MLXSW_SP_RIF_TYPE_FID, 10360 .rif_size = sizeof(struct mlxsw_sp_rif), 10361 .configure = mlxsw_sp_rif_fid_configure, 10362 .deconfigure = mlxsw_sp_rif_fid_deconfigure, 10363 .fid_get = mlxsw_sp_rif_fid_fid_get, 10364 .fdb_del = mlxsw_sp_rif_fid_fdb_del, 10365 }; 10366 10367 static struct mlxsw_sp_fid * 10368 mlxsw_sp_rif_vlan_fid_get(struct mlxsw_sp_rif *rif, 10369 const struct mlxsw_sp_rif_params *params, 10370 struct netlink_ext_ack *extack) 10371 { 10372 struct net_device *dev = mlxsw_sp_rif_dev(rif); 10373 struct net_device *br_dev; 10374 10375 if (WARN_ON(!params->vid)) 10376 return ERR_PTR(-EINVAL); 10377 10378 if (is_vlan_dev(dev)) { 10379 br_dev = vlan_dev_real_dev(dev); 10380 if (WARN_ON(!netif_is_bridge_master(br_dev))) 10381 return ERR_PTR(-EINVAL); 10382 } 10383 10384 return mlxsw_sp_fid_8021q_get(rif->mlxsw_sp, params->vid); 10385 } 10386 10387 static void mlxsw_sp_rif_vlan_fdb_del(struct mlxsw_sp_rif *rif, const char *mac) 10388 { 10389 struct net_device *rif_dev = mlxsw_sp_rif_dev(rif); 10390 struct switchdev_notifier_fdb_info info = {}; 10391 u16 vid = mlxsw_sp_fid_8021q_vid(rif->fid); 10392 struct net_device *br_dev; 10393 struct net_device *dev; 10394 10395 br_dev = is_vlan_dev(rif_dev) ? vlan_dev_real_dev(rif_dev) : rif_dev; 10396 dev = br_fdb_find_port(br_dev, mac, vid); 10397 if (!dev) 10398 return; 10399 10400 info.addr = mac; 10401 info.vid = vid; 10402 call_switchdev_notifiers(SWITCHDEV_FDB_DEL_TO_BRIDGE, dev, &info.info, 10403 NULL); 10404 } 10405 10406 static int mlxsw_sp_rif_vlan_op(struct mlxsw_sp_rif *rif, u16 vid, u16 efid, 10407 bool enable) 10408 { 10409 struct net_device *dev = mlxsw_sp_rif_dev(rif); 10410 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 10411 char ritr_pl[MLXSW_REG_RITR_LEN]; 10412 10413 mlxsw_reg_ritr_vlan_if_pack(ritr_pl, enable, rif->rif_index, rif->vr_id, 10414 dev->mtu, dev->dev_addr, 10415 rif->mac_profile_id, vid, efid); 10416 10417 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 10418 } 10419 10420 static int mlxsw_sp_rif_vlan_configure(struct mlxsw_sp_rif *rif, u16 efid, 10421 struct netlink_ext_ack *extack) 10422 { 10423 struct net_device *dev = mlxsw_sp_rif_dev(rif); 10424 u16 vid = mlxsw_sp_fid_8021q_vid(rif->fid); 10425 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 10426 u8 mac_profile; 10427 int err; 10428 10429 err = mlxsw_sp_rif_mac_profile_get(mlxsw_sp, rif->addr, 10430 &mac_profile, extack); 10431 if (err) 10432 return err; 10433 rif->mac_profile_id = mac_profile; 10434 10435 err = mlxsw_sp_rif_vlan_op(rif, vid, efid, true); 10436 if (err) 10437 goto err_rif_vlan_fid_op; 10438 10439 err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC, 10440 mlxsw_sp_router_port(mlxsw_sp), true); 10441 if (err) 10442 goto err_fid_mc_flood_set; 10443 10444 err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC, 10445 mlxsw_sp_router_port(mlxsw_sp), true); 10446 if (err) 10447 goto err_fid_bc_flood_set; 10448 10449 err = mlxsw_sp_macvlan_replay(rif, extack); 10450 if (err) 10451 goto err_macvlan_replay; 10452 10453 err = mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, dev->dev_addr, 10454 mlxsw_sp_fid_index(rif->fid), true); 10455 if (err) 10456 goto err_rif_fdb_op; 10457 10458 err = mlxsw_sp_fid_rif_set(rif->fid, rif); 10459 if (err) 10460 goto err_fid_rif_set; 10461 10462 return 0; 10463 10464 err_fid_rif_set: 10465 mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, dev->dev_addr, 10466 mlxsw_sp_fid_index(rif->fid), false); 10467 err_rif_fdb_op: 10468 mlxsw_sp_rif_macvlan_flush(rif); 10469 err_macvlan_replay: 10470 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC, 10471 mlxsw_sp_router_port(mlxsw_sp), false); 10472 err_fid_bc_flood_set: 10473 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC, 10474 mlxsw_sp_router_port(mlxsw_sp), false); 10475 err_fid_mc_flood_set: 10476 mlxsw_sp_rif_vlan_op(rif, vid, 0, false); 10477 err_rif_vlan_fid_op: 10478 mlxsw_sp_rif_mac_profile_put(mlxsw_sp, mac_profile); 10479 return err; 10480 } 10481 10482 static void mlxsw_sp_rif_vlan_deconfigure(struct mlxsw_sp_rif *rif) 10483 { 10484 struct net_device *dev = mlxsw_sp_rif_dev(rif); 10485 u16 vid = mlxsw_sp_fid_8021q_vid(rif->fid); 10486 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 10487 10488 mlxsw_sp_fid_rif_unset(rif->fid); 10489 mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, dev->dev_addr, 10490 mlxsw_sp_fid_index(rif->fid), false); 10491 mlxsw_sp_rif_macvlan_flush(rif); 10492 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC, 10493 mlxsw_sp_router_port(mlxsw_sp), false); 10494 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC, 10495 mlxsw_sp_router_port(mlxsw_sp), false); 10496 mlxsw_sp_rif_vlan_op(rif, vid, 0, false); 10497 mlxsw_sp_rif_mac_profile_put(rif->mlxsw_sp, rif->mac_profile_id); 10498 } 10499 10500 static int mlxsw_sp1_rif_vlan_configure(struct mlxsw_sp_rif *rif, 10501 struct netlink_ext_ack *extack) 10502 { 10503 return mlxsw_sp_rif_vlan_configure(rif, 0, extack); 10504 } 10505 10506 static const struct mlxsw_sp_rif_ops mlxsw_sp1_rif_vlan_ops = { 10507 .type = MLXSW_SP_RIF_TYPE_VLAN, 10508 .rif_size = sizeof(struct mlxsw_sp_rif), 10509 .configure = mlxsw_sp1_rif_vlan_configure, 10510 .deconfigure = mlxsw_sp_rif_vlan_deconfigure, 10511 .fid_get = mlxsw_sp_rif_vlan_fid_get, 10512 .fdb_del = mlxsw_sp_rif_vlan_fdb_del, 10513 }; 10514 10515 static int mlxsw_sp2_rif_vlan_configure(struct mlxsw_sp_rif *rif, 10516 struct netlink_ext_ack *extack) 10517 { 10518 u16 efid = mlxsw_sp_fid_index(rif->fid); 10519 10520 return mlxsw_sp_rif_vlan_configure(rif, efid, extack); 10521 } 10522 10523 static const struct mlxsw_sp_rif_ops mlxsw_sp2_rif_vlan_ops = { 10524 .type = MLXSW_SP_RIF_TYPE_VLAN, 10525 .rif_size = sizeof(struct mlxsw_sp_rif), 10526 .configure = mlxsw_sp2_rif_vlan_configure, 10527 .deconfigure = mlxsw_sp_rif_vlan_deconfigure, 10528 .fid_get = mlxsw_sp_rif_vlan_fid_get, 10529 .fdb_del = mlxsw_sp_rif_vlan_fdb_del, 10530 }; 10531 10532 static struct mlxsw_sp_rif_ipip_lb * 10533 mlxsw_sp_rif_ipip_lb_rif(struct mlxsw_sp_rif *rif) 10534 { 10535 return container_of(rif, struct mlxsw_sp_rif_ipip_lb, common); 10536 } 10537 10538 static void 10539 mlxsw_sp_rif_ipip_lb_setup(struct mlxsw_sp_rif *rif, 10540 const struct mlxsw_sp_rif_params *params) 10541 { 10542 struct mlxsw_sp_rif_params_ipip_lb *params_lb; 10543 struct mlxsw_sp_rif_ipip_lb *rif_lb; 10544 10545 params_lb = container_of(params, struct mlxsw_sp_rif_params_ipip_lb, 10546 common); 10547 rif_lb = mlxsw_sp_rif_ipip_lb_rif(rif); 10548 rif_lb->lb_config = params_lb->lb_config; 10549 } 10550 10551 static int 10552 mlxsw_sp1_rif_ipip_lb_configure(struct mlxsw_sp_rif *rif, 10553 struct netlink_ext_ack *extack) 10554 { 10555 struct mlxsw_sp_rif_ipip_lb *lb_rif = mlxsw_sp_rif_ipip_lb_rif(rif); 10556 struct net_device *dev = mlxsw_sp_rif_dev(rif); 10557 u32 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(dev); 10558 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 10559 struct mlxsw_sp_vr *ul_vr; 10560 int err; 10561 10562 ul_vr = mlxsw_sp_vr_get(mlxsw_sp, ul_tb_id, extack); 10563 if (IS_ERR(ul_vr)) 10564 return PTR_ERR(ul_vr); 10565 10566 err = mlxsw_sp_rif_ipip_lb_op(lb_rif, ul_vr->id, 0, true); 10567 if (err) 10568 goto err_loopback_op; 10569 10570 lb_rif->ul_vr_id = ul_vr->id; 10571 lb_rif->ul_rif_id = 0; 10572 ++ul_vr->rif_count; 10573 return 0; 10574 10575 err_loopback_op: 10576 mlxsw_sp_vr_put(mlxsw_sp, ul_vr); 10577 return err; 10578 } 10579 10580 static void mlxsw_sp1_rif_ipip_lb_deconfigure(struct mlxsw_sp_rif *rif) 10581 { 10582 struct mlxsw_sp_rif_ipip_lb *lb_rif = mlxsw_sp_rif_ipip_lb_rif(rif); 10583 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 10584 struct mlxsw_sp_vr *ul_vr; 10585 10586 ul_vr = &mlxsw_sp->router->vrs[lb_rif->ul_vr_id]; 10587 mlxsw_sp_rif_ipip_lb_op(lb_rif, ul_vr->id, 0, false); 10588 10589 --ul_vr->rif_count; 10590 mlxsw_sp_vr_put(mlxsw_sp, ul_vr); 10591 } 10592 10593 static const struct mlxsw_sp_rif_ops mlxsw_sp1_rif_ipip_lb_ops = { 10594 .type = MLXSW_SP_RIF_TYPE_IPIP_LB, 10595 .rif_size = sizeof(struct mlxsw_sp_rif_ipip_lb), 10596 .setup = mlxsw_sp_rif_ipip_lb_setup, 10597 .configure = mlxsw_sp1_rif_ipip_lb_configure, 10598 .deconfigure = mlxsw_sp1_rif_ipip_lb_deconfigure, 10599 }; 10600 10601 static const struct mlxsw_sp_rif_ops *mlxsw_sp1_rif_ops_arr[] = { 10602 [MLXSW_SP_RIF_TYPE_SUBPORT] = &mlxsw_sp_rif_subport_ops, 10603 [MLXSW_SP_RIF_TYPE_VLAN] = &mlxsw_sp1_rif_vlan_ops, 10604 [MLXSW_SP_RIF_TYPE_FID] = &mlxsw_sp_rif_fid_ops, 10605 [MLXSW_SP_RIF_TYPE_IPIP_LB] = &mlxsw_sp1_rif_ipip_lb_ops, 10606 }; 10607 10608 static int 10609 mlxsw_sp_rif_ipip_lb_ul_rif_op(struct mlxsw_sp_rif *ul_rif, bool enable) 10610 { 10611 struct mlxsw_sp *mlxsw_sp = ul_rif->mlxsw_sp; 10612 char ritr_pl[MLXSW_REG_RITR_LEN]; 10613 10614 mlxsw_reg_ritr_pack(ritr_pl, enable, MLXSW_REG_RITR_LOOPBACK_IF, 10615 ul_rif->rif_index, ul_rif->vr_id, IP_MAX_MTU); 10616 mlxsw_reg_ritr_loopback_protocol_set(ritr_pl, 10617 MLXSW_REG_RITR_LOOPBACK_GENERIC); 10618 10619 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 10620 } 10621 10622 static struct mlxsw_sp_rif * 10623 mlxsw_sp_ul_rif_create(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp_vr *vr, 10624 struct mlxsw_sp_crif *ul_crif, 10625 struct netlink_ext_ack *extack) 10626 { 10627 struct mlxsw_sp_rif *ul_rif; 10628 u8 rif_entries = 1; 10629 u16 rif_index; 10630 int err; 10631 10632 err = mlxsw_sp_rif_index_alloc(mlxsw_sp, &rif_index, rif_entries); 10633 if (err) { 10634 NL_SET_ERR_MSG_MOD(extack, "Exceeded number of supported router interfaces"); 10635 return ERR_PTR(err); 10636 } 10637 10638 ul_rif = mlxsw_sp_rif_alloc(sizeof(*ul_rif), rif_index, vr->id, 10639 ul_crif); 10640 if (!ul_rif) { 10641 err = -ENOMEM; 10642 goto err_rif_alloc; 10643 } 10644 10645 mlxsw_sp->router->rifs[rif_index] = ul_rif; 10646 ul_rif->mlxsw_sp = mlxsw_sp; 10647 ul_rif->rif_entries = rif_entries; 10648 err = mlxsw_sp_rif_ipip_lb_ul_rif_op(ul_rif, true); 10649 if (err) 10650 goto ul_rif_op_err; 10651 10652 atomic_add(rif_entries, &mlxsw_sp->router->rifs_count); 10653 return ul_rif; 10654 10655 ul_rif_op_err: 10656 mlxsw_sp->router->rifs[rif_index] = NULL; 10657 mlxsw_sp_rif_free(ul_rif); 10658 err_rif_alloc: 10659 mlxsw_sp_rif_index_free(mlxsw_sp, rif_index, rif_entries); 10660 return ERR_PTR(err); 10661 } 10662 10663 static void mlxsw_sp_ul_rif_destroy(struct mlxsw_sp_rif *ul_rif) 10664 { 10665 struct mlxsw_sp *mlxsw_sp = ul_rif->mlxsw_sp; 10666 u8 rif_entries = ul_rif->rif_entries; 10667 u16 rif_index = ul_rif->rif_index; 10668 10669 atomic_sub(rif_entries, &mlxsw_sp->router->rifs_count); 10670 mlxsw_sp_rif_ipip_lb_ul_rif_op(ul_rif, false); 10671 mlxsw_sp->router->rifs[ul_rif->rif_index] = NULL; 10672 mlxsw_sp_rif_free(ul_rif); 10673 mlxsw_sp_rif_index_free(mlxsw_sp, rif_index, rif_entries); 10674 } 10675 10676 static struct mlxsw_sp_rif * 10677 mlxsw_sp_ul_rif_get(struct mlxsw_sp *mlxsw_sp, u32 tb_id, 10678 struct mlxsw_sp_crif *ul_crif, 10679 struct netlink_ext_ack *extack) 10680 { 10681 struct mlxsw_sp_vr *vr; 10682 int err; 10683 10684 vr = mlxsw_sp_vr_get(mlxsw_sp, tb_id, extack); 10685 if (IS_ERR(vr)) 10686 return ERR_CAST(vr); 10687 10688 if (refcount_inc_not_zero(&vr->ul_rif_refcnt)) 10689 return vr->ul_rif; 10690 10691 vr->ul_rif = mlxsw_sp_ul_rif_create(mlxsw_sp, vr, ul_crif, extack); 10692 if (IS_ERR(vr->ul_rif)) { 10693 err = PTR_ERR(vr->ul_rif); 10694 goto err_ul_rif_create; 10695 } 10696 10697 vr->rif_count++; 10698 refcount_set(&vr->ul_rif_refcnt, 1); 10699 10700 return vr->ul_rif; 10701 10702 err_ul_rif_create: 10703 mlxsw_sp_vr_put(mlxsw_sp, vr); 10704 return ERR_PTR(err); 10705 } 10706 10707 static void mlxsw_sp_ul_rif_put(struct mlxsw_sp_rif *ul_rif) 10708 { 10709 struct mlxsw_sp *mlxsw_sp = ul_rif->mlxsw_sp; 10710 struct mlxsw_sp_vr *vr; 10711 10712 vr = &mlxsw_sp->router->vrs[ul_rif->vr_id]; 10713 10714 if (!refcount_dec_and_test(&vr->ul_rif_refcnt)) 10715 return; 10716 10717 vr->rif_count--; 10718 mlxsw_sp_ul_rif_destroy(ul_rif); 10719 mlxsw_sp_vr_put(mlxsw_sp, vr); 10720 } 10721 10722 int mlxsw_sp_router_ul_rif_get(struct mlxsw_sp *mlxsw_sp, u32 ul_tb_id, 10723 u16 *ul_rif_index) 10724 { 10725 struct mlxsw_sp_rif *ul_rif; 10726 int err = 0; 10727 10728 mutex_lock(&mlxsw_sp->router->lock); 10729 ul_rif = mlxsw_sp_ul_rif_get(mlxsw_sp, ul_tb_id, NULL, NULL); 10730 if (IS_ERR(ul_rif)) { 10731 err = PTR_ERR(ul_rif); 10732 goto out; 10733 } 10734 *ul_rif_index = ul_rif->rif_index; 10735 out: 10736 mutex_unlock(&mlxsw_sp->router->lock); 10737 return err; 10738 } 10739 10740 void mlxsw_sp_router_ul_rif_put(struct mlxsw_sp *mlxsw_sp, u16 ul_rif_index) 10741 { 10742 struct mlxsw_sp_rif *ul_rif; 10743 10744 mutex_lock(&mlxsw_sp->router->lock); 10745 ul_rif = mlxsw_sp->router->rifs[ul_rif_index]; 10746 if (WARN_ON(!ul_rif)) 10747 goto out; 10748 10749 mlxsw_sp_ul_rif_put(ul_rif); 10750 out: 10751 mutex_unlock(&mlxsw_sp->router->lock); 10752 } 10753 10754 static int 10755 mlxsw_sp2_rif_ipip_lb_configure(struct mlxsw_sp_rif *rif, 10756 struct netlink_ext_ack *extack) 10757 { 10758 struct mlxsw_sp_rif_ipip_lb *lb_rif = mlxsw_sp_rif_ipip_lb_rif(rif); 10759 struct net_device *dev = mlxsw_sp_rif_dev(rif); 10760 u32 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(dev); 10761 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 10762 struct mlxsw_sp_rif *ul_rif; 10763 int err; 10764 10765 ul_rif = mlxsw_sp_ul_rif_get(mlxsw_sp, ul_tb_id, NULL, extack); 10766 if (IS_ERR(ul_rif)) 10767 return PTR_ERR(ul_rif); 10768 10769 err = mlxsw_sp_rif_ipip_lb_op(lb_rif, 0, ul_rif->rif_index, true); 10770 if (err) 10771 goto err_loopback_op; 10772 10773 lb_rif->ul_vr_id = 0; 10774 lb_rif->ul_rif_id = ul_rif->rif_index; 10775 10776 return 0; 10777 10778 err_loopback_op: 10779 mlxsw_sp_ul_rif_put(ul_rif); 10780 return err; 10781 } 10782 10783 static void mlxsw_sp2_rif_ipip_lb_deconfigure(struct mlxsw_sp_rif *rif) 10784 { 10785 struct mlxsw_sp_rif_ipip_lb *lb_rif = mlxsw_sp_rif_ipip_lb_rif(rif); 10786 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 10787 struct mlxsw_sp_rif *ul_rif; 10788 10789 ul_rif = mlxsw_sp_rif_by_index(mlxsw_sp, lb_rif->ul_rif_id); 10790 mlxsw_sp_rif_ipip_lb_op(lb_rif, 0, lb_rif->ul_rif_id, false); 10791 mlxsw_sp_ul_rif_put(ul_rif); 10792 } 10793 10794 static const struct mlxsw_sp_rif_ops mlxsw_sp2_rif_ipip_lb_ops = { 10795 .type = MLXSW_SP_RIF_TYPE_IPIP_LB, 10796 .rif_size = sizeof(struct mlxsw_sp_rif_ipip_lb), 10797 .setup = mlxsw_sp_rif_ipip_lb_setup, 10798 .configure = mlxsw_sp2_rif_ipip_lb_configure, 10799 .deconfigure = mlxsw_sp2_rif_ipip_lb_deconfigure, 10800 }; 10801 10802 static const struct mlxsw_sp_rif_ops *mlxsw_sp2_rif_ops_arr[] = { 10803 [MLXSW_SP_RIF_TYPE_SUBPORT] = &mlxsw_sp_rif_subport_ops, 10804 [MLXSW_SP_RIF_TYPE_VLAN] = &mlxsw_sp2_rif_vlan_ops, 10805 [MLXSW_SP_RIF_TYPE_FID] = &mlxsw_sp_rif_fid_ops, 10806 [MLXSW_SP_RIF_TYPE_IPIP_LB] = &mlxsw_sp2_rif_ipip_lb_ops, 10807 }; 10808 10809 static int mlxsw_sp_rifs_table_init(struct mlxsw_sp *mlxsw_sp) 10810 { 10811 struct gen_pool *rifs_table; 10812 int err; 10813 10814 rifs_table = gen_pool_create(0, -1); 10815 if (!rifs_table) 10816 return -ENOMEM; 10817 10818 gen_pool_set_algo(rifs_table, gen_pool_first_fit_order_align, 10819 NULL); 10820 10821 err = gen_pool_add(rifs_table, MLXSW_SP_ROUTER_GENALLOC_OFFSET, 10822 MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS), -1); 10823 if (err) 10824 goto err_gen_pool_add; 10825 10826 mlxsw_sp->router->rifs_table = rifs_table; 10827 10828 return 0; 10829 10830 err_gen_pool_add: 10831 gen_pool_destroy(rifs_table); 10832 return err; 10833 } 10834 10835 static void mlxsw_sp_rifs_table_fini(struct mlxsw_sp *mlxsw_sp) 10836 { 10837 gen_pool_destroy(mlxsw_sp->router->rifs_table); 10838 } 10839 10840 static int mlxsw_sp_rifs_init(struct mlxsw_sp *mlxsw_sp) 10841 { 10842 u64 max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); 10843 struct devlink *devlink = priv_to_devlink(mlxsw_sp->core); 10844 struct mlxsw_core *core = mlxsw_sp->core; 10845 int err; 10846 10847 if (!MLXSW_CORE_RES_VALID(core, MAX_RIF_MAC_PROFILES)) 10848 return -EIO; 10849 mlxsw_sp->router->max_rif_mac_profile = 10850 MLXSW_CORE_RES_GET(core, MAX_RIF_MAC_PROFILES); 10851 10852 mlxsw_sp->router->rifs = kcalloc(max_rifs, 10853 sizeof(struct mlxsw_sp_rif *), 10854 GFP_KERNEL); 10855 if (!mlxsw_sp->router->rifs) 10856 return -ENOMEM; 10857 10858 err = mlxsw_sp_rifs_table_init(mlxsw_sp); 10859 if (err) 10860 goto err_rifs_table_init; 10861 10862 idr_init(&mlxsw_sp->router->rif_mac_profiles_idr); 10863 atomic_set(&mlxsw_sp->router->rif_mac_profiles_count, 0); 10864 atomic_set(&mlxsw_sp->router->rifs_count, 0); 10865 devl_resource_occ_get_register(devlink, 10866 MLXSW_SP_RESOURCE_RIF_MAC_PROFILES, 10867 mlxsw_sp_rif_mac_profiles_occ_get, 10868 mlxsw_sp); 10869 devl_resource_occ_get_register(devlink, 10870 MLXSW_SP_RESOURCE_RIFS, 10871 mlxsw_sp_rifs_occ_get, 10872 mlxsw_sp); 10873 10874 return 0; 10875 10876 err_rifs_table_init: 10877 kfree(mlxsw_sp->router->rifs); 10878 return err; 10879 } 10880 10881 static void mlxsw_sp_rifs_fini(struct mlxsw_sp *mlxsw_sp) 10882 { 10883 int max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); 10884 struct devlink *devlink = priv_to_devlink(mlxsw_sp->core); 10885 int i; 10886 10887 WARN_ON_ONCE(atomic_read(&mlxsw_sp->router->rifs_count)); 10888 for (i = 0; i < max_rifs; i++) 10889 WARN_ON_ONCE(mlxsw_sp->router->rifs[i]); 10890 10891 devl_resource_occ_get_unregister(devlink, MLXSW_SP_RESOURCE_RIFS); 10892 devl_resource_occ_get_unregister(devlink, 10893 MLXSW_SP_RESOURCE_RIF_MAC_PROFILES); 10894 WARN_ON(!idr_is_empty(&mlxsw_sp->router->rif_mac_profiles_idr)); 10895 idr_destroy(&mlxsw_sp->router->rif_mac_profiles_idr); 10896 mlxsw_sp_rifs_table_fini(mlxsw_sp); 10897 kfree(mlxsw_sp->router->rifs); 10898 } 10899 10900 static int 10901 mlxsw_sp_ipip_config_tigcr(struct mlxsw_sp *mlxsw_sp) 10902 { 10903 char tigcr_pl[MLXSW_REG_TIGCR_LEN]; 10904 10905 mlxsw_reg_tigcr_pack(tigcr_pl, true, 0); 10906 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(tigcr), tigcr_pl); 10907 } 10908 10909 static int mlxsw_sp_ipips_init(struct mlxsw_sp *mlxsw_sp) 10910 { 10911 int err; 10912 10913 INIT_LIST_HEAD(&mlxsw_sp->router->ipip_list); 10914 10915 err = mlxsw_sp_ipip_ecn_encap_init(mlxsw_sp); 10916 if (err) 10917 return err; 10918 err = mlxsw_sp_ipip_ecn_decap_init(mlxsw_sp); 10919 if (err) 10920 return err; 10921 10922 return mlxsw_sp_ipip_config_tigcr(mlxsw_sp); 10923 } 10924 10925 static int mlxsw_sp1_ipips_init(struct mlxsw_sp *mlxsw_sp) 10926 { 10927 mlxsw_sp->router->ipip_ops_arr = mlxsw_sp1_ipip_ops_arr; 10928 return mlxsw_sp_ipips_init(mlxsw_sp); 10929 } 10930 10931 static int mlxsw_sp2_ipips_init(struct mlxsw_sp *mlxsw_sp) 10932 { 10933 mlxsw_sp->router->ipip_ops_arr = mlxsw_sp2_ipip_ops_arr; 10934 return mlxsw_sp_ipips_init(mlxsw_sp); 10935 } 10936 10937 static void mlxsw_sp_ipips_fini(struct mlxsw_sp *mlxsw_sp) 10938 { 10939 WARN_ON(!list_empty(&mlxsw_sp->router->ipip_list)); 10940 } 10941 10942 static void mlxsw_sp_router_fib_dump_flush(struct notifier_block *nb) 10943 { 10944 struct mlxsw_sp_router *router; 10945 10946 /* Flush pending FIB notifications and then flush the device's 10947 * table before requesting another dump. The FIB notification 10948 * block is unregistered, so no need to take RTNL. 10949 */ 10950 mlxsw_core_flush_owq(); 10951 router = container_of(nb, struct mlxsw_sp_router, fib_nb); 10952 mlxsw_sp_router_fib_flush(router->mlxsw_sp); 10953 } 10954 10955 #ifdef CONFIG_IP_ROUTE_MULTIPATH 10956 struct mlxsw_sp_mp_hash_config { 10957 DECLARE_BITMAP(headers, __MLXSW_REG_RECR2_HEADER_CNT); 10958 DECLARE_BITMAP(fields, __MLXSW_REG_RECR2_FIELD_CNT); 10959 DECLARE_BITMAP(inner_headers, __MLXSW_REG_RECR2_HEADER_CNT); 10960 DECLARE_BITMAP(inner_fields, __MLXSW_REG_RECR2_INNER_FIELD_CNT); 10961 bool inc_parsing_depth; 10962 }; 10963 10964 #define MLXSW_SP_MP_HASH_HEADER_SET(_headers, _header) \ 10965 bitmap_set(_headers, MLXSW_REG_RECR2_##_header, 1) 10966 10967 #define MLXSW_SP_MP_HASH_FIELD_SET(_fields, _field) \ 10968 bitmap_set(_fields, MLXSW_REG_RECR2_##_field, 1) 10969 10970 #define MLXSW_SP_MP_HASH_FIELD_RANGE_SET(_fields, _field, _nr) \ 10971 bitmap_set(_fields, MLXSW_REG_RECR2_##_field, _nr) 10972 10973 static void mlxsw_sp_mp_hash_inner_l3(struct mlxsw_sp_mp_hash_config *config) 10974 { 10975 unsigned long *inner_headers = config->inner_headers; 10976 unsigned long *inner_fields = config->inner_fields; 10977 10978 /* IPv4 inner */ 10979 MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, IPV4_EN_NOT_TCP_NOT_UDP); 10980 MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, IPV4_EN_TCP_UDP); 10981 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(inner_fields, INNER_IPV4_SIP0, 4); 10982 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(inner_fields, INNER_IPV4_DIP0, 4); 10983 /* IPv6 inner */ 10984 MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, IPV6_EN_NOT_TCP_NOT_UDP); 10985 MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, IPV6_EN_TCP_UDP); 10986 MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV6_SIP0_7); 10987 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(inner_fields, INNER_IPV6_SIP8, 8); 10988 MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV6_DIP0_7); 10989 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(inner_fields, INNER_IPV6_DIP8, 8); 10990 MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV6_NEXT_HEADER); 10991 MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV6_FLOW_LABEL); 10992 } 10993 10994 static void mlxsw_sp_mp4_hash_outer_addr(struct mlxsw_sp_mp_hash_config *config) 10995 { 10996 unsigned long *headers = config->headers; 10997 unsigned long *fields = config->fields; 10998 10999 MLXSW_SP_MP_HASH_HEADER_SET(headers, IPV4_EN_NOT_TCP_NOT_UDP); 11000 MLXSW_SP_MP_HASH_HEADER_SET(headers, IPV4_EN_TCP_UDP); 11001 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(fields, IPV4_SIP0, 4); 11002 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(fields, IPV4_DIP0, 4); 11003 } 11004 11005 static void 11006 mlxsw_sp_mp_hash_inner_custom(struct mlxsw_sp_mp_hash_config *config, 11007 u32 hash_fields) 11008 { 11009 unsigned long *inner_headers = config->inner_headers; 11010 unsigned long *inner_fields = config->inner_fields; 11011 11012 /* IPv4 Inner */ 11013 MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, IPV4_EN_NOT_TCP_NOT_UDP); 11014 MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, IPV4_EN_TCP_UDP); 11015 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_SRC_IP) 11016 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(inner_fields, INNER_IPV4_SIP0, 4); 11017 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_DST_IP) 11018 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(inner_fields, INNER_IPV4_DIP0, 4); 11019 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_IP_PROTO) 11020 MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV4_PROTOCOL); 11021 /* IPv6 inner */ 11022 MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, IPV6_EN_NOT_TCP_NOT_UDP); 11023 MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, IPV6_EN_TCP_UDP); 11024 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_SRC_IP) { 11025 MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV6_SIP0_7); 11026 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(inner_fields, INNER_IPV6_SIP8, 8); 11027 } 11028 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_DST_IP) { 11029 MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV6_DIP0_7); 11030 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(inner_fields, INNER_IPV6_DIP8, 8); 11031 } 11032 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_IP_PROTO) 11033 MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV6_NEXT_HEADER); 11034 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_FLOWLABEL) 11035 MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV6_FLOW_LABEL); 11036 /* L4 inner */ 11037 MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, TCP_UDP_EN_IPV4); 11038 MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, TCP_UDP_EN_IPV6); 11039 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_SRC_PORT) 11040 MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_TCP_UDP_SPORT); 11041 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_DST_PORT) 11042 MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_TCP_UDP_DPORT); 11043 } 11044 11045 static void mlxsw_sp_mp4_hash_init(struct mlxsw_sp *mlxsw_sp, 11046 struct mlxsw_sp_mp_hash_config *config) 11047 { 11048 struct net *net = mlxsw_sp_net(mlxsw_sp); 11049 unsigned long *headers = config->headers; 11050 unsigned long *fields = config->fields; 11051 u32 hash_fields; 11052 11053 switch (READ_ONCE(net->ipv4.sysctl_fib_multipath_hash_policy)) { 11054 case 0: 11055 mlxsw_sp_mp4_hash_outer_addr(config); 11056 break; 11057 case 1: 11058 mlxsw_sp_mp4_hash_outer_addr(config); 11059 MLXSW_SP_MP_HASH_HEADER_SET(headers, TCP_UDP_EN_IPV4); 11060 MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV4_PROTOCOL); 11061 MLXSW_SP_MP_HASH_FIELD_SET(fields, TCP_UDP_SPORT); 11062 MLXSW_SP_MP_HASH_FIELD_SET(fields, TCP_UDP_DPORT); 11063 break; 11064 case 2: 11065 /* Outer */ 11066 mlxsw_sp_mp4_hash_outer_addr(config); 11067 /* Inner */ 11068 mlxsw_sp_mp_hash_inner_l3(config); 11069 break; 11070 case 3: 11071 hash_fields = READ_ONCE(net->ipv4.sysctl_fib_multipath_hash_fields); 11072 /* Outer */ 11073 MLXSW_SP_MP_HASH_HEADER_SET(headers, IPV4_EN_NOT_TCP_NOT_UDP); 11074 MLXSW_SP_MP_HASH_HEADER_SET(headers, IPV4_EN_TCP_UDP); 11075 MLXSW_SP_MP_HASH_HEADER_SET(headers, TCP_UDP_EN_IPV4); 11076 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_SRC_IP) 11077 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(fields, IPV4_SIP0, 4); 11078 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_DST_IP) 11079 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(fields, IPV4_DIP0, 4); 11080 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_IP_PROTO) 11081 MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV4_PROTOCOL); 11082 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_SRC_PORT) 11083 MLXSW_SP_MP_HASH_FIELD_SET(fields, TCP_UDP_SPORT); 11084 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_DST_PORT) 11085 MLXSW_SP_MP_HASH_FIELD_SET(fields, TCP_UDP_DPORT); 11086 /* Inner */ 11087 mlxsw_sp_mp_hash_inner_custom(config, hash_fields); 11088 break; 11089 } 11090 } 11091 11092 static void mlxsw_sp_mp6_hash_outer_addr(struct mlxsw_sp_mp_hash_config *config) 11093 { 11094 unsigned long *headers = config->headers; 11095 unsigned long *fields = config->fields; 11096 11097 MLXSW_SP_MP_HASH_HEADER_SET(headers, IPV6_EN_NOT_TCP_NOT_UDP); 11098 MLXSW_SP_MP_HASH_HEADER_SET(headers, IPV6_EN_TCP_UDP); 11099 MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_SIP0_7); 11100 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(fields, IPV6_SIP8, 8); 11101 MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_DIP0_7); 11102 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(fields, IPV6_DIP8, 8); 11103 } 11104 11105 static void mlxsw_sp_mp6_hash_init(struct mlxsw_sp *mlxsw_sp, 11106 struct mlxsw_sp_mp_hash_config *config) 11107 { 11108 u32 hash_fields = ip6_multipath_hash_fields(mlxsw_sp_net(mlxsw_sp)); 11109 unsigned long *headers = config->headers; 11110 unsigned long *fields = config->fields; 11111 11112 switch (ip6_multipath_hash_policy(mlxsw_sp_net(mlxsw_sp))) { 11113 case 0: 11114 mlxsw_sp_mp6_hash_outer_addr(config); 11115 MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_NEXT_HEADER); 11116 MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_FLOW_LABEL); 11117 break; 11118 case 1: 11119 mlxsw_sp_mp6_hash_outer_addr(config); 11120 MLXSW_SP_MP_HASH_HEADER_SET(headers, TCP_UDP_EN_IPV6); 11121 MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_NEXT_HEADER); 11122 MLXSW_SP_MP_HASH_FIELD_SET(fields, TCP_UDP_SPORT); 11123 MLXSW_SP_MP_HASH_FIELD_SET(fields, TCP_UDP_DPORT); 11124 break; 11125 case 2: 11126 /* Outer */ 11127 mlxsw_sp_mp6_hash_outer_addr(config); 11128 MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_NEXT_HEADER); 11129 MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_FLOW_LABEL); 11130 /* Inner */ 11131 mlxsw_sp_mp_hash_inner_l3(config); 11132 config->inc_parsing_depth = true; 11133 break; 11134 case 3: 11135 /* Outer */ 11136 MLXSW_SP_MP_HASH_HEADER_SET(headers, IPV6_EN_NOT_TCP_NOT_UDP); 11137 MLXSW_SP_MP_HASH_HEADER_SET(headers, IPV6_EN_TCP_UDP); 11138 MLXSW_SP_MP_HASH_HEADER_SET(headers, TCP_UDP_EN_IPV6); 11139 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_SRC_IP) { 11140 MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_SIP0_7); 11141 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(fields, IPV6_SIP8, 8); 11142 } 11143 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_DST_IP) { 11144 MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_DIP0_7); 11145 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(fields, IPV6_DIP8, 8); 11146 } 11147 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_IP_PROTO) 11148 MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_NEXT_HEADER); 11149 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_FLOWLABEL) 11150 MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_FLOW_LABEL); 11151 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_SRC_PORT) 11152 MLXSW_SP_MP_HASH_FIELD_SET(fields, TCP_UDP_SPORT); 11153 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_DST_PORT) 11154 MLXSW_SP_MP_HASH_FIELD_SET(fields, TCP_UDP_DPORT); 11155 /* Inner */ 11156 mlxsw_sp_mp_hash_inner_custom(config, hash_fields); 11157 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_MASK) 11158 config->inc_parsing_depth = true; 11159 break; 11160 } 11161 } 11162 11163 static int mlxsw_sp_mp_hash_parsing_depth_adjust(struct mlxsw_sp *mlxsw_sp, 11164 bool old_inc_parsing_depth, 11165 bool new_inc_parsing_depth) 11166 { 11167 int err; 11168 11169 if (!old_inc_parsing_depth && new_inc_parsing_depth) { 11170 err = mlxsw_sp_parsing_depth_inc(mlxsw_sp); 11171 if (err) 11172 return err; 11173 mlxsw_sp->router->inc_parsing_depth = true; 11174 } else if (old_inc_parsing_depth && !new_inc_parsing_depth) { 11175 mlxsw_sp_parsing_depth_dec(mlxsw_sp); 11176 mlxsw_sp->router->inc_parsing_depth = false; 11177 } 11178 11179 return 0; 11180 } 11181 11182 static int mlxsw_sp_mp_hash_init(struct mlxsw_sp *mlxsw_sp) 11183 { 11184 bool old_inc_parsing_depth, new_inc_parsing_depth; 11185 struct mlxsw_sp_mp_hash_config config = {}; 11186 char recr2_pl[MLXSW_REG_RECR2_LEN]; 11187 unsigned long bit; 11188 u32 seed; 11189 int err; 11190 11191 seed = jhash(mlxsw_sp->base_mac, sizeof(mlxsw_sp->base_mac), 0); 11192 mlxsw_reg_recr2_pack(recr2_pl, seed); 11193 mlxsw_sp_mp4_hash_init(mlxsw_sp, &config); 11194 mlxsw_sp_mp6_hash_init(mlxsw_sp, &config); 11195 11196 old_inc_parsing_depth = mlxsw_sp->router->inc_parsing_depth; 11197 new_inc_parsing_depth = config.inc_parsing_depth; 11198 err = mlxsw_sp_mp_hash_parsing_depth_adjust(mlxsw_sp, 11199 old_inc_parsing_depth, 11200 new_inc_parsing_depth); 11201 if (err) 11202 return err; 11203 11204 for_each_set_bit(bit, config.headers, __MLXSW_REG_RECR2_HEADER_CNT) 11205 mlxsw_reg_recr2_outer_header_enables_set(recr2_pl, bit, 1); 11206 for_each_set_bit(bit, config.fields, __MLXSW_REG_RECR2_FIELD_CNT) 11207 mlxsw_reg_recr2_outer_header_fields_enable_set(recr2_pl, bit, 1); 11208 for_each_set_bit(bit, config.inner_headers, __MLXSW_REG_RECR2_HEADER_CNT) 11209 mlxsw_reg_recr2_inner_header_enables_set(recr2_pl, bit, 1); 11210 for_each_set_bit(bit, config.inner_fields, __MLXSW_REG_RECR2_INNER_FIELD_CNT) 11211 mlxsw_reg_recr2_inner_header_fields_enable_set(recr2_pl, bit, 1); 11212 11213 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(recr2), recr2_pl); 11214 if (err) 11215 goto err_reg_write; 11216 11217 return 0; 11218 11219 err_reg_write: 11220 mlxsw_sp_mp_hash_parsing_depth_adjust(mlxsw_sp, new_inc_parsing_depth, 11221 old_inc_parsing_depth); 11222 return err; 11223 } 11224 11225 static void mlxsw_sp_mp_hash_fini(struct mlxsw_sp *mlxsw_sp) 11226 { 11227 bool old_inc_parsing_depth = mlxsw_sp->router->inc_parsing_depth; 11228 11229 mlxsw_sp_mp_hash_parsing_depth_adjust(mlxsw_sp, old_inc_parsing_depth, 11230 false); 11231 } 11232 #else 11233 static int mlxsw_sp_mp_hash_init(struct mlxsw_sp *mlxsw_sp) 11234 { 11235 return 0; 11236 } 11237 11238 static void mlxsw_sp_mp_hash_fini(struct mlxsw_sp *mlxsw_sp) 11239 { 11240 } 11241 #endif 11242 11243 static int mlxsw_sp_dscp_init(struct mlxsw_sp *mlxsw_sp) 11244 { 11245 char rdpm_pl[MLXSW_REG_RDPM_LEN]; 11246 unsigned int i; 11247 11248 MLXSW_REG_ZERO(rdpm, rdpm_pl); 11249 11250 /* HW is determining switch priority based on DSCP-bits, but the 11251 * kernel is still doing that based on the ToS. Since there's a 11252 * mismatch in bits we need to make sure to translate the right 11253 * value ToS would observe, skipping the 2 least-significant ECN bits. 11254 */ 11255 for (i = 0; i < MLXSW_REG_RDPM_DSCP_ENTRY_REC_MAX_COUNT; i++) 11256 mlxsw_reg_rdpm_pack(rdpm_pl, i, rt_tos2priority(i << 2)); 11257 11258 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rdpm), rdpm_pl); 11259 } 11260 11261 static int __mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp) 11262 { 11263 struct net *net = mlxsw_sp_net(mlxsw_sp); 11264 char rgcr_pl[MLXSW_REG_RGCR_LEN]; 11265 u64 max_rifs; 11266 bool usp; 11267 11268 if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_RIFS)) 11269 return -EIO; 11270 max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); 11271 usp = READ_ONCE(net->ipv4.sysctl_ip_fwd_update_priority); 11272 11273 mlxsw_reg_rgcr_pack(rgcr_pl, true, true); 11274 mlxsw_reg_rgcr_max_router_interfaces_set(rgcr_pl, max_rifs); 11275 mlxsw_reg_rgcr_usp_set(rgcr_pl, usp); 11276 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rgcr), rgcr_pl); 11277 } 11278 11279 static void __mlxsw_sp_router_fini(struct mlxsw_sp *mlxsw_sp) 11280 { 11281 char rgcr_pl[MLXSW_REG_RGCR_LEN]; 11282 11283 mlxsw_reg_rgcr_pack(rgcr_pl, false, false); 11284 mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rgcr), rgcr_pl); 11285 } 11286 11287 static int mlxsw_sp_lb_rif_init(struct mlxsw_sp *mlxsw_sp, 11288 struct netlink_ext_ack *extack) 11289 { 11290 struct mlxsw_sp_router *router = mlxsw_sp->router; 11291 struct mlxsw_sp_rif *lb_rif; 11292 int err; 11293 11294 router->lb_crif = mlxsw_sp_crif_alloc(NULL); 11295 if (!router->lb_crif) 11296 return -ENOMEM; 11297 11298 /* Create a generic loopback RIF associated with the main table 11299 * (default VRF). Any table can be used, but the main table exists 11300 * anyway, so we do not waste resources. Loopback RIFs are usually 11301 * created with a NULL CRIF, but this RIF is used as a fallback RIF 11302 * for blackhole nexthops, and nexthops expect to have a valid CRIF. 11303 */ 11304 lb_rif = mlxsw_sp_ul_rif_get(mlxsw_sp, RT_TABLE_MAIN, router->lb_crif, 11305 extack); 11306 if (IS_ERR(lb_rif)) { 11307 err = PTR_ERR(lb_rif); 11308 goto err_ul_rif_get; 11309 } 11310 11311 return 0; 11312 11313 err_ul_rif_get: 11314 mlxsw_sp_crif_free(router->lb_crif); 11315 return err; 11316 } 11317 11318 static void mlxsw_sp_lb_rif_fini(struct mlxsw_sp *mlxsw_sp) 11319 { 11320 mlxsw_sp_ul_rif_put(mlxsw_sp->router->lb_crif->rif); 11321 mlxsw_sp_crif_free(mlxsw_sp->router->lb_crif); 11322 } 11323 11324 static int mlxsw_sp1_router_init(struct mlxsw_sp *mlxsw_sp) 11325 { 11326 size_t size_ranges_count = ARRAY_SIZE(mlxsw_sp1_adj_grp_size_ranges); 11327 11328 mlxsw_sp->router->rif_ops_arr = mlxsw_sp1_rif_ops_arr; 11329 mlxsw_sp->router->adj_grp_size_ranges = mlxsw_sp1_adj_grp_size_ranges; 11330 mlxsw_sp->router->adj_grp_size_ranges_count = size_ranges_count; 11331 11332 return 0; 11333 } 11334 11335 const struct mlxsw_sp_router_ops mlxsw_sp1_router_ops = { 11336 .init = mlxsw_sp1_router_init, 11337 .ipips_init = mlxsw_sp1_ipips_init, 11338 }; 11339 11340 static int mlxsw_sp2_router_init(struct mlxsw_sp *mlxsw_sp) 11341 { 11342 size_t size_ranges_count = ARRAY_SIZE(mlxsw_sp2_adj_grp_size_ranges); 11343 11344 mlxsw_sp->router->rif_ops_arr = mlxsw_sp2_rif_ops_arr; 11345 mlxsw_sp->router->adj_grp_size_ranges = mlxsw_sp2_adj_grp_size_ranges; 11346 mlxsw_sp->router->adj_grp_size_ranges_count = size_ranges_count; 11347 11348 return 0; 11349 } 11350 11351 const struct mlxsw_sp_router_ops mlxsw_sp2_router_ops = { 11352 .init = mlxsw_sp2_router_init, 11353 .ipips_init = mlxsw_sp2_ipips_init, 11354 }; 11355 11356 int mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp, 11357 struct netlink_ext_ack *extack) 11358 { 11359 struct mlxsw_sp_router *router; 11360 struct notifier_block *nb; 11361 int err; 11362 11363 router = kzalloc(sizeof(*mlxsw_sp->router), GFP_KERNEL); 11364 if (!router) 11365 return -ENOMEM; 11366 mutex_init(&router->lock); 11367 mlxsw_sp->router = router; 11368 router->mlxsw_sp = mlxsw_sp; 11369 11370 err = mlxsw_sp->router_ops->init(mlxsw_sp); 11371 if (err) 11372 goto err_router_ops_init; 11373 11374 INIT_LIST_HEAD(&mlxsw_sp->router->nh_res_grp_list); 11375 INIT_DELAYED_WORK(&mlxsw_sp->router->nh_grp_activity_dw, 11376 mlxsw_sp_nh_grp_activity_work); 11377 INIT_LIST_HEAD(&mlxsw_sp->router->nexthop_neighs_list); 11378 err = __mlxsw_sp_router_init(mlxsw_sp); 11379 if (err) 11380 goto err_router_init; 11381 11382 err = mlxsw_sp->router_ops->ipips_init(mlxsw_sp); 11383 if (err) 11384 goto err_ipips_init; 11385 11386 err = rhashtable_init(&mlxsw_sp->router->crif_ht, 11387 &mlxsw_sp_crif_ht_params); 11388 if (err) 11389 goto err_crif_ht_init; 11390 11391 err = mlxsw_sp_rifs_init(mlxsw_sp); 11392 if (err) 11393 goto err_rifs_init; 11394 11395 err = rhashtable_init(&mlxsw_sp->router->nexthop_ht, 11396 &mlxsw_sp_nexthop_ht_params); 11397 if (err) 11398 goto err_nexthop_ht_init; 11399 11400 err = rhashtable_init(&mlxsw_sp->router->nexthop_group_ht, 11401 &mlxsw_sp_nexthop_group_ht_params); 11402 if (err) 11403 goto err_nexthop_group_ht_init; 11404 11405 INIT_LIST_HEAD(&mlxsw_sp->router->nexthop_list); 11406 err = mlxsw_sp_lpm_init(mlxsw_sp); 11407 if (err) 11408 goto err_lpm_init; 11409 11410 err = mlxsw_sp_mr_init(mlxsw_sp, &mlxsw_sp_mr_tcam_ops); 11411 if (err) 11412 goto err_mr_init; 11413 11414 err = mlxsw_sp_vrs_init(mlxsw_sp); 11415 if (err) 11416 goto err_vrs_init; 11417 11418 err = mlxsw_sp_lb_rif_init(mlxsw_sp, extack); 11419 if (err) 11420 goto err_lb_rif_init; 11421 11422 err = mlxsw_sp_neigh_init(mlxsw_sp); 11423 if (err) 11424 goto err_neigh_init; 11425 11426 err = mlxsw_sp_mp_hash_init(mlxsw_sp); 11427 if (err) 11428 goto err_mp_hash_init; 11429 11430 err = mlxsw_sp_dscp_init(mlxsw_sp); 11431 if (err) 11432 goto err_dscp_init; 11433 11434 router->inetaddr_nb.notifier_call = mlxsw_sp_inetaddr_event; 11435 err = register_inetaddr_notifier(&router->inetaddr_nb); 11436 if (err) 11437 goto err_register_inetaddr_notifier; 11438 11439 router->inet6addr_nb.notifier_call = mlxsw_sp_inet6addr_event; 11440 err = register_inet6addr_notifier(&router->inet6addr_nb); 11441 if (err) 11442 goto err_register_inet6addr_notifier; 11443 11444 router->inetaddr_valid_nb.notifier_call = mlxsw_sp_inetaddr_valid_event; 11445 err = register_inetaddr_validator_notifier(&router->inetaddr_valid_nb); 11446 if (err) 11447 goto err_register_inetaddr_valid_notifier; 11448 11449 nb = &router->inet6addr_valid_nb; 11450 nb->notifier_call = mlxsw_sp_inet6addr_valid_event; 11451 err = register_inet6addr_validator_notifier(nb); 11452 if (err) 11453 goto err_register_inet6addr_valid_notifier; 11454 11455 mlxsw_sp->router->netevent_nb.notifier_call = 11456 mlxsw_sp_router_netevent_event; 11457 err = register_netevent_notifier(&mlxsw_sp->router->netevent_nb); 11458 if (err) 11459 goto err_register_netevent_notifier; 11460 11461 mlxsw_sp->router->nexthop_nb.notifier_call = 11462 mlxsw_sp_nexthop_obj_event; 11463 err = register_nexthop_notifier(mlxsw_sp_net(mlxsw_sp), 11464 &mlxsw_sp->router->nexthop_nb, 11465 extack); 11466 if (err) 11467 goto err_register_nexthop_notifier; 11468 11469 mlxsw_sp->router->fib_nb.notifier_call = mlxsw_sp_router_fib_event; 11470 err = register_fib_notifier(mlxsw_sp_net(mlxsw_sp), 11471 &mlxsw_sp->router->fib_nb, 11472 mlxsw_sp_router_fib_dump_flush, extack); 11473 if (err) 11474 goto err_register_fib_notifier; 11475 11476 mlxsw_sp->router->netdevice_nb.notifier_call = 11477 mlxsw_sp_router_netdevice_event; 11478 err = register_netdevice_notifier_net(mlxsw_sp_net(mlxsw_sp), 11479 &mlxsw_sp->router->netdevice_nb); 11480 if (err) 11481 goto err_register_netdev_notifier; 11482 11483 return 0; 11484 11485 err_register_netdev_notifier: 11486 unregister_fib_notifier(mlxsw_sp_net(mlxsw_sp), 11487 &mlxsw_sp->router->fib_nb); 11488 err_register_fib_notifier: 11489 unregister_nexthop_notifier(mlxsw_sp_net(mlxsw_sp), 11490 &mlxsw_sp->router->nexthop_nb); 11491 err_register_nexthop_notifier: 11492 unregister_netevent_notifier(&mlxsw_sp->router->netevent_nb); 11493 err_register_netevent_notifier: 11494 unregister_inet6addr_validator_notifier(&router->inet6addr_valid_nb); 11495 err_register_inet6addr_valid_notifier: 11496 unregister_inetaddr_validator_notifier(&router->inetaddr_valid_nb); 11497 err_register_inetaddr_valid_notifier: 11498 unregister_inet6addr_notifier(&router->inet6addr_nb); 11499 err_register_inet6addr_notifier: 11500 unregister_inetaddr_notifier(&router->inetaddr_nb); 11501 err_register_inetaddr_notifier: 11502 mlxsw_core_flush_owq(); 11503 err_dscp_init: 11504 mlxsw_sp_mp_hash_fini(mlxsw_sp); 11505 err_mp_hash_init: 11506 mlxsw_sp_neigh_fini(mlxsw_sp); 11507 err_neigh_init: 11508 mlxsw_sp_lb_rif_fini(mlxsw_sp); 11509 err_lb_rif_init: 11510 mlxsw_sp_vrs_fini(mlxsw_sp); 11511 err_vrs_init: 11512 mlxsw_sp_mr_fini(mlxsw_sp); 11513 err_mr_init: 11514 mlxsw_sp_lpm_fini(mlxsw_sp); 11515 err_lpm_init: 11516 rhashtable_destroy(&mlxsw_sp->router->nexthop_group_ht); 11517 err_nexthop_group_ht_init: 11518 rhashtable_destroy(&mlxsw_sp->router->nexthop_ht); 11519 err_nexthop_ht_init: 11520 mlxsw_sp_rifs_fini(mlxsw_sp); 11521 err_rifs_init: 11522 rhashtable_destroy(&mlxsw_sp->router->crif_ht); 11523 err_crif_ht_init: 11524 mlxsw_sp_ipips_fini(mlxsw_sp); 11525 err_ipips_init: 11526 __mlxsw_sp_router_fini(mlxsw_sp); 11527 err_router_init: 11528 cancel_delayed_work_sync(&mlxsw_sp->router->nh_grp_activity_dw); 11529 err_router_ops_init: 11530 mutex_destroy(&mlxsw_sp->router->lock); 11531 kfree(mlxsw_sp->router); 11532 return err; 11533 } 11534 11535 void mlxsw_sp_router_fini(struct mlxsw_sp *mlxsw_sp) 11536 { 11537 struct mlxsw_sp_router *router = mlxsw_sp->router; 11538 11539 unregister_netdevice_notifier_net(mlxsw_sp_net(mlxsw_sp), 11540 &router->netdevice_nb); 11541 unregister_fib_notifier(mlxsw_sp_net(mlxsw_sp), &router->fib_nb); 11542 unregister_nexthop_notifier(mlxsw_sp_net(mlxsw_sp), 11543 &router->nexthop_nb); 11544 unregister_netevent_notifier(&router->netevent_nb); 11545 unregister_inet6addr_validator_notifier(&router->inet6addr_valid_nb); 11546 unregister_inetaddr_validator_notifier(&router->inetaddr_valid_nb); 11547 unregister_inet6addr_notifier(&router->inet6addr_nb); 11548 unregister_inetaddr_notifier(&router->inetaddr_nb); 11549 mlxsw_core_flush_owq(); 11550 mlxsw_sp_mp_hash_fini(mlxsw_sp); 11551 mlxsw_sp_neigh_fini(mlxsw_sp); 11552 mlxsw_sp_lb_rif_fini(mlxsw_sp); 11553 mlxsw_sp_vrs_fini(mlxsw_sp); 11554 mlxsw_sp_mr_fini(mlxsw_sp); 11555 mlxsw_sp_lpm_fini(mlxsw_sp); 11556 rhashtable_destroy(&router->nexthop_group_ht); 11557 rhashtable_destroy(&router->nexthop_ht); 11558 mlxsw_sp_rifs_fini(mlxsw_sp); 11559 rhashtable_destroy(&mlxsw_sp->router->crif_ht); 11560 mlxsw_sp_ipips_fini(mlxsw_sp); 11561 __mlxsw_sp_router_fini(mlxsw_sp); 11562 cancel_delayed_work_sync(&router->nh_grp_activity_dw); 11563 mutex_destroy(&router->lock); 11564 kfree(router); 11565 } 11566