1 // SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0 2 /* Copyright (c) 2016-2018 Mellanox Technologies. All rights reserved */ 3 4 #include <linux/kernel.h> 5 #include <linux/types.h> 6 #include <linux/rhashtable.h> 7 #include <linux/bitops.h> 8 #include <linux/in6.h> 9 #include <linux/notifier.h> 10 #include <linux/inetdevice.h> 11 #include <linux/netdevice.h> 12 #include <linux/if_bridge.h> 13 #include <linux/socket.h> 14 #include <linux/route.h> 15 #include <linux/gcd.h> 16 #include <linux/if_macvlan.h> 17 #include <linux/refcount.h> 18 #include <linux/jhash.h> 19 #include <linux/net_namespace.h> 20 #include <linux/mutex.h> 21 #include <linux/genalloc.h> 22 #include <linux/xarray.h> 23 #include <net/netevent.h> 24 #include <net/neighbour.h> 25 #include <net/arp.h> 26 #include <net/inet_dscp.h> 27 #include <net/ip_fib.h> 28 #include <net/ip6_fib.h> 29 #include <net/nexthop.h> 30 #include <net/fib_rules.h> 31 #include <net/ip_tunnels.h> 32 #include <net/l3mdev.h> 33 #include <net/addrconf.h> 34 #include <net/ndisc.h> 35 #include <net/ipv6.h> 36 #include <net/fib_notifier.h> 37 #include <net/switchdev.h> 38 39 #include "spectrum.h" 40 #include "core.h" 41 #include "reg.h" 42 #include "spectrum_cnt.h" 43 #include "spectrum_dpipe.h" 44 #include "spectrum_ipip.h" 45 #include "spectrum_mr.h" 46 #include "spectrum_mr_tcam.h" 47 #include "spectrum_router.h" 48 #include "spectrum_span.h" 49 50 struct mlxsw_sp_fib; 51 struct mlxsw_sp_vr; 52 struct mlxsw_sp_lpm_tree; 53 struct mlxsw_sp_rif_ops; 54 55 struct mlxsw_sp_crif_key { 56 struct net_device *dev; 57 }; 58 59 struct mlxsw_sp_crif { 60 struct mlxsw_sp_crif_key key; 61 struct rhash_head ht_node; 62 bool can_destroy; 63 struct list_head nexthop_list; 64 struct mlxsw_sp_rif *rif; 65 }; 66 67 static const struct rhashtable_params mlxsw_sp_crif_ht_params = { 68 .key_offset = offsetof(struct mlxsw_sp_crif, key), 69 .key_len = sizeof_field(struct mlxsw_sp_crif, key), 70 .head_offset = offsetof(struct mlxsw_sp_crif, ht_node), 71 }; 72 73 struct mlxsw_sp_rif { 74 struct mlxsw_sp_crif *crif; /* NULL for underlay RIF */ 75 netdevice_tracker dev_tracker; 76 struct list_head neigh_list; 77 struct mlxsw_sp_fid *fid; 78 unsigned char addr[ETH_ALEN]; 79 int mtu; 80 u16 rif_index; 81 u8 mac_profile_id; 82 u8 rif_entries; 83 u16 vr_id; 84 const struct mlxsw_sp_rif_ops *ops; 85 struct mlxsw_sp *mlxsw_sp; 86 87 unsigned int counter_ingress; 88 bool counter_ingress_valid; 89 unsigned int counter_egress; 90 bool counter_egress_valid; 91 }; 92 93 static struct net_device *mlxsw_sp_rif_dev(const struct mlxsw_sp_rif *rif) 94 { 95 if (!rif->crif) 96 return NULL; 97 return rif->crif->key.dev; 98 } 99 100 struct mlxsw_sp_rif_params { 101 struct net_device *dev; 102 union { 103 u16 system_port; 104 u16 lag_id; 105 }; 106 u16 vid; 107 bool lag; 108 bool double_entry; 109 }; 110 111 struct mlxsw_sp_rif_subport { 112 struct mlxsw_sp_rif common; 113 refcount_t ref_count; 114 union { 115 u16 system_port; 116 u16 lag_id; 117 }; 118 u16 vid; 119 bool lag; 120 }; 121 122 struct mlxsw_sp_rif_ipip_lb { 123 struct mlxsw_sp_rif common; 124 struct mlxsw_sp_rif_ipip_lb_config lb_config; 125 u16 ul_vr_id; /* Spectrum-1. */ 126 u16 ul_rif_id; /* Spectrum-2+. */ 127 }; 128 129 struct mlxsw_sp_rif_params_ipip_lb { 130 struct mlxsw_sp_rif_params common; 131 struct mlxsw_sp_rif_ipip_lb_config lb_config; 132 }; 133 134 struct mlxsw_sp_rif_ops { 135 enum mlxsw_sp_rif_type type; 136 size_t rif_size; 137 138 void (*setup)(struct mlxsw_sp_rif *rif, 139 const struct mlxsw_sp_rif_params *params); 140 int (*configure)(struct mlxsw_sp_rif *rif, 141 struct netlink_ext_ack *extack); 142 void (*deconfigure)(struct mlxsw_sp_rif *rif); 143 struct mlxsw_sp_fid * (*fid_get)(struct mlxsw_sp_rif *rif, 144 const struct mlxsw_sp_rif_params *params, 145 struct netlink_ext_ack *extack); 146 void (*fdb_del)(struct mlxsw_sp_rif *rif, const char *mac); 147 }; 148 149 struct mlxsw_sp_rif_mac_profile { 150 unsigned char mac_prefix[ETH_ALEN]; 151 refcount_t ref_count; 152 u8 id; 153 }; 154 155 struct mlxsw_sp_router_ops { 156 int (*init)(struct mlxsw_sp *mlxsw_sp); 157 int (*ipips_init)(struct mlxsw_sp *mlxsw_sp); 158 }; 159 160 static struct mlxsw_sp_rif * 161 mlxsw_sp_rif_find_by_dev(const struct mlxsw_sp *mlxsw_sp, 162 const struct net_device *dev); 163 static void mlxsw_sp_rif_destroy(struct mlxsw_sp_rif *rif); 164 static void mlxsw_sp_lpm_tree_hold(struct mlxsw_sp_lpm_tree *lpm_tree); 165 static void mlxsw_sp_lpm_tree_put(struct mlxsw_sp *mlxsw_sp, 166 struct mlxsw_sp_lpm_tree *lpm_tree); 167 static int mlxsw_sp_vr_lpm_tree_bind(struct mlxsw_sp *mlxsw_sp, 168 const struct mlxsw_sp_fib *fib, 169 u8 tree_id); 170 static int mlxsw_sp_vr_lpm_tree_unbind(struct mlxsw_sp *mlxsw_sp, 171 const struct mlxsw_sp_fib *fib); 172 173 static unsigned int * 174 mlxsw_sp_rif_p_counter_get(struct mlxsw_sp_rif *rif, 175 enum mlxsw_sp_rif_counter_dir dir) 176 { 177 switch (dir) { 178 case MLXSW_SP_RIF_COUNTER_EGRESS: 179 return &rif->counter_egress; 180 case MLXSW_SP_RIF_COUNTER_INGRESS: 181 return &rif->counter_ingress; 182 } 183 return NULL; 184 } 185 186 static bool 187 mlxsw_sp_rif_counter_valid_get(struct mlxsw_sp_rif *rif, 188 enum mlxsw_sp_rif_counter_dir dir) 189 { 190 switch (dir) { 191 case MLXSW_SP_RIF_COUNTER_EGRESS: 192 return rif->counter_egress_valid; 193 case MLXSW_SP_RIF_COUNTER_INGRESS: 194 return rif->counter_ingress_valid; 195 } 196 return false; 197 } 198 199 static void 200 mlxsw_sp_rif_counter_valid_set(struct mlxsw_sp_rif *rif, 201 enum mlxsw_sp_rif_counter_dir dir, 202 bool valid) 203 { 204 switch (dir) { 205 case MLXSW_SP_RIF_COUNTER_EGRESS: 206 rif->counter_egress_valid = valid; 207 break; 208 case MLXSW_SP_RIF_COUNTER_INGRESS: 209 rif->counter_ingress_valid = valid; 210 break; 211 } 212 } 213 214 static int mlxsw_sp_rif_counter_edit(struct mlxsw_sp *mlxsw_sp, u16 rif_index, 215 unsigned int counter_index, bool enable, 216 enum mlxsw_sp_rif_counter_dir dir) 217 { 218 char ritr_pl[MLXSW_REG_RITR_LEN]; 219 bool is_egress = false; 220 int err; 221 222 if (dir == MLXSW_SP_RIF_COUNTER_EGRESS) 223 is_egress = true; 224 mlxsw_reg_ritr_rif_pack(ritr_pl, rif_index); 225 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 226 if (err) 227 return err; 228 229 mlxsw_reg_ritr_counter_pack(ritr_pl, counter_index, enable, 230 is_egress); 231 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 232 } 233 234 int mlxsw_sp_rif_counter_value_get(struct mlxsw_sp *mlxsw_sp, 235 struct mlxsw_sp_rif *rif, 236 enum mlxsw_sp_rif_counter_dir dir, u64 *cnt) 237 { 238 char ricnt_pl[MLXSW_REG_RICNT_LEN]; 239 unsigned int *p_counter_index; 240 bool valid; 241 int err; 242 243 valid = mlxsw_sp_rif_counter_valid_get(rif, dir); 244 if (!valid) 245 return -EINVAL; 246 247 p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir); 248 if (!p_counter_index) 249 return -EINVAL; 250 mlxsw_reg_ricnt_pack(ricnt_pl, *p_counter_index, 251 MLXSW_REG_RICNT_OPCODE_NOP); 252 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ricnt), ricnt_pl); 253 if (err) 254 return err; 255 *cnt = mlxsw_reg_ricnt_good_unicast_packets_get(ricnt_pl); 256 return 0; 257 } 258 259 struct mlxsw_sp_rif_counter_set_basic { 260 u64 good_unicast_packets; 261 u64 good_multicast_packets; 262 u64 good_broadcast_packets; 263 u64 good_unicast_bytes; 264 u64 good_multicast_bytes; 265 u64 good_broadcast_bytes; 266 u64 error_packets; 267 u64 discard_packets; 268 u64 error_bytes; 269 u64 discard_bytes; 270 }; 271 272 static int 273 mlxsw_sp_rif_counter_fetch_clear(struct mlxsw_sp_rif *rif, 274 enum mlxsw_sp_rif_counter_dir dir, 275 struct mlxsw_sp_rif_counter_set_basic *set) 276 { 277 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 278 char ricnt_pl[MLXSW_REG_RICNT_LEN]; 279 unsigned int *p_counter_index; 280 int err; 281 282 if (!mlxsw_sp_rif_counter_valid_get(rif, dir)) 283 return -EINVAL; 284 285 p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir); 286 if (!p_counter_index) 287 return -EINVAL; 288 289 mlxsw_reg_ricnt_pack(ricnt_pl, *p_counter_index, 290 MLXSW_REG_RICNT_OPCODE_CLEAR); 291 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ricnt), ricnt_pl); 292 if (err) 293 return err; 294 295 if (!set) 296 return 0; 297 298 #define MLXSW_SP_RIF_COUNTER_EXTRACT(NAME) \ 299 (set->NAME = mlxsw_reg_ricnt_ ## NAME ## _get(ricnt_pl)) 300 301 MLXSW_SP_RIF_COUNTER_EXTRACT(good_unicast_packets); 302 MLXSW_SP_RIF_COUNTER_EXTRACT(good_multicast_packets); 303 MLXSW_SP_RIF_COUNTER_EXTRACT(good_broadcast_packets); 304 MLXSW_SP_RIF_COUNTER_EXTRACT(good_unicast_bytes); 305 MLXSW_SP_RIF_COUNTER_EXTRACT(good_multicast_bytes); 306 MLXSW_SP_RIF_COUNTER_EXTRACT(good_broadcast_bytes); 307 MLXSW_SP_RIF_COUNTER_EXTRACT(error_packets); 308 MLXSW_SP_RIF_COUNTER_EXTRACT(discard_packets); 309 MLXSW_SP_RIF_COUNTER_EXTRACT(error_bytes); 310 MLXSW_SP_RIF_COUNTER_EXTRACT(discard_bytes); 311 312 #undef MLXSW_SP_RIF_COUNTER_EXTRACT 313 314 return 0; 315 } 316 317 static int mlxsw_sp_rif_counter_clear(struct mlxsw_sp *mlxsw_sp, 318 unsigned int counter_index) 319 { 320 char ricnt_pl[MLXSW_REG_RICNT_LEN]; 321 322 mlxsw_reg_ricnt_pack(ricnt_pl, counter_index, 323 MLXSW_REG_RICNT_OPCODE_CLEAR); 324 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ricnt), ricnt_pl); 325 } 326 327 int mlxsw_sp_rif_counter_alloc(struct mlxsw_sp_rif *rif, 328 enum mlxsw_sp_rif_counter_dir dir) 329 { 330 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 331 unsigned int *p_counter_index; 332 int err; 333 334 if (mlxsw_sp_rif_counter_valid_get(rif, dir)) 335 return 0; 336 337 p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir); 338 if (!p_counter_index) 339 return -EINVAL; 340 341 err = mlxsw_sp_counter_alloc(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF, 342 p_counter_index); 343 if (err) 344 return err; 345 346 err = mlxsw_sp_rif_counter_clear(mlxsw_sp, *p_counter_index); 347 if (err) 348 goto err_counter_clear; 349 350 err = mlxsw_sp_rif_counter_edit(mlxsw_sp, rif->rif_index, 351 *p_counter_index, true, dir); 352 if (err) 353 goto err_counter_edit; 354 mlxsw_sp_rif_counter_valid_set(rif, dir, true); 355 return 0; 356 357 err_counter_edit: 358 err_counter_clear: 359 mlxsw_sp_counter_free(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF, 360 *p_counter_index); 361 return err; 362 } 363 364 void mlxsw_sp_rif_counter_free(struct mlxsw_sp_rif *rif, 365 enum mlxsw_sp_rif_counter_dir dir) 366 { 367 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 368 unsigned int *p_counter_index; 369 370 if (!mlxsw_sp_rif_counter_valid_get(rif, dir)) 371 return; 372 373 p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir); 374 if (WARN_ON(!p_counter_index)) 375 return; 376 mlxsw_sp_rif_counter_edit(mlxsw_sp, rif->rif_index, 377 *p_counter_index, false, dir); 378 mlxsw_sp_counter_free(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF, 379 *p_counter_index); 380 mlxsw_sp_rif_counter_valid_set(rif, dir, false); 381 } 382 383 static void mlxsw_sp_rif_counters_alloc(struct mlxsw_sp_rif *rif) 384 { 385 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 386 struct devlink *devlink; 387 388 devlink = priv_to_devlink(mlxsw_sp->core); 389 if (!devlink_dpipe_table_counter_enabled(devlink, 390 MLXSW_SP_DPIPE_TABLE_NAME_ERIF)) 391 return; 392 mlxsw_sp_rif_counter_alloc(rif, MLXSW_SP_RIF_COUNTER_EGRESS); 393 } 394 395 static void mlxsw_sp_rif_counters_free(struct mlxsw_sp_rif *rif) 396 { 397 mlxsw_sp_rif_counter_free(rif, MLXSW_SP_RIF_COUNTER_EGRESS); 398 } 399 400 #define MLXSW_SP_PREFIX_COUNT (sizeof(struct in6_addr) * BITS_PER_BYTE + 1) 401 402 struct mlxsw_sp_prefix_usage { 403 DECLARE_BITMAP(b, MLXSW_SP_PREFIX_COUNT); 404 }; 405 406 #define mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) \ 407 for_each_set_bit(prefix, (prefix_usage)->b, MLXSW_SP_PREFIX_COUNT) 408 409 static bool 410 mlxsw_sp_prefix_usage_eq(struct mlxsw_sp_prefix_usage *prefix_usage1, 411 struct mlxsw_sp_prefix_usage *prefix_usage2) 412 { 413 return !memcmp(prefix_usage1, prefix_usage2, sizeof(*prefix_usage1)); 414 } 415 416 static void 417 mlxsw_sp_prefix_usage_cpy(struct mlxsw_sp_prefix_usage *prefix_usage1, 418 struct mlxsw_sp_prefix_usage *prefix_usage2) 419 { 420 memcpy(prefix_usage1, prefix_usage2, sizeof(*prefix_usage1)); 421 } 422 423 static void 424 mlxsw_sp_prefix_usage_set(struct mlxsw_sp_prefix_usage *prefix_usage, 425 unsigned char prefix_len) 426 { 427 set_bit(prefix_len, prefix_usage->b); 428 } 429 430 static void 431 mlxsw_sp_prefix_usage_clear(struct mlxsw_sp_prefix_usage *prefix_usage, 432 unsigned char prefix_len) 433 { 434 clear_bit(prefix_len, prefix_usage->b); 435 } 436 437 struct mlxsw_sp_fib_key { 438 unsigned char addr[sizeof(struct in6_addr)]; 439 unsigned char prefix_len; 440 }; 441 442 enum mlxsw_sp_fib_entry_type { 443 MLXSW_SP_FIB_ENTRY_TYPE_REMOTE, 444 MLXSW_SP_FIB_ENTRY_TYPE_LOCAL, 445 MLXSW_SP_FIB_ENTRY_TYPE_TRAP, 446 MLXSW_SP_FIB_ENTRY_TYPE_BLACKHOLE, 447 MLXSW_SP_FIB_ENTRY_TYPE_UNREACHABLE, 448 449 /* This is a special case of local delivery, where a packet should be 450 * decapsulated on reception. Note that there is no corresponding ENCAP, 451 * because that's a type of next hop, not of FIB entry. (There can be 452 * several next hops in a REMOTE entry, and some of them may be 453 * encapsulating entries.) 454 */ 455 MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP, 456 MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP, 457 }; 458 459 struct mlxsw_sp_nexthop_group_info; 460 struct mlxsw_sp_nexthop_group; 461 struct mlxsw_sp_fib_entry; 462 463 struct mlxsw_sp_fib_node { 464 struct mlxsw_sp_fib_entry *fib_entry; 465 struct list_head list; 466 struct rhash_head ht_node; 467 struct mlxsw_sp_fib *fib; 468 struct mlxsw_sp_fib_key key; 469 }; 470 471 struct mlxsw_sp_fib_entry_decap { 472 struct mlxsw_sp_ipip_entry *ipip_entry; 473 u32 tunnel_index; 474 }; 475 476 struct mlxsw_sp_fib_entry { 477 struct mlxsw_sp_fib_node *fib_node; 478 enum mlxsw_sp_fib_entry_type type; 479 struct list_head nexthop_group_node; 480 struct mlxsw_sp_nexthop_group *nh_group; 481 struct mlxsw_sp_fib_entry_decap decap; /* Valid for decap entries. */ 482 }; 483 484 struct mlxsw_sp_fib4_entry { 485 struct mlxsw_sp_fib_entry common; 486 struct fib_info *fi; 487 u32 tb_id; 488 dscp_t dscp; 489 u8 type; 490 }; 491 492 struct mlxsw_sp_fib6_entry { 493 struct mlxsw_sp_fib_entry common; 494 struct list_head rt6_list; 495 unsigned int nrt6; 496 }; 497 498 struct mlxsw_sp_rt6 { 499 struct list_head list; 500 struct fib6_info *rt; 501 }; 502 503 struct mlxsw_sp_lpm_tree { 504 u8 id; /* tree ID */ 505 refcount_t ref_count; 506 enum mlxsw_sp_l3proto proto; 507 unsigned long prefix_ref_count[MLXSW_SP_PREFIX_COUNT]; 508 struct mlxsw_sp_prefix_usage prefix_usage; 509 }; 510 511 struct mlxsw_sp_fib { 512 struct rhashtable ht; 513 struct list_head node_list; 514 struct mlxsw_sp_vr *vr; 515 struct mlxsw_sp_lpm_tree *lpm_tree; 516 enum mlxsw_sp_l3proto proto; 517 }; 518 519 struct mlxsw_sp_vr { 520 u16 id; /* virtual router ID */ 521 u32 tb_id; /* kernel fib table id */ 522 unsigned int rif_count; 523 struct mlxsw_sp_fib *fib4; 524 struct mlxsw_sp_fib *fib6; 525 struct mlxsw_sp_mr_table *mr_table[MLXSW_SP_L3_PROTO_MAX]; 526 struct mlxsw_sp_rif *ul_rif; 527 refcount_t ul_rif_refcnt; 528 }; 529 530 static const struct rhashtable_params mlxsw_sp_fib_ht_params; 531 532 static struct mlxsw_sp_fib *mlxsw_sp_fib_create(struct mlxsw_sp *mlxsw_sp, 533 struct mlxsw_sp_vr *vr, 534 enum mlxsw_sp_l3proto proto) 535 { 536 struct mlxsw_sp_lpm_tree *lpm_tree; 537 struct mlxsw_sp_fib *fib; 538 int err; 539 540 lpm_tree = mlxsw_sp->router->lpm.proto_trees[proto]; 541 fib = kzalloc_obj(*fib); 542 if (!fib) 543 return ERR_PTR(-ENOMEM); 544 err = rhashtable_init(&fib->ht, &mlxsw_sp_fib_ht_params); 545 if (err) 546 goto err_rhashtable_init; 547 INIT_LIST_HEAD(&fib->node_list); 548 fib->proto = proto; 549 fib->vr = vr; 550 fib->lpm_tree = lpm_tree; 551 mlxsw_sp_lpm_tree_hold(lpm_tree); 552 err = mlxsw_sp_vr_lpm_tree_bind(mlxsw_sp, fib, lpm_tree->id); 553 if (err) 554 goto err_lpm_tree_bind; 555 return fib; 556 557 err_lpm_tree_bind: 558 mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree); 559 err_rhashtable_init: 560 kfree(fib); 561 return ERR_PTR(err); 562 } 563 564 static void mlxsw_sp_fib_destroy(struct mlxsw_sp *mlxsw_sp, 565 struct mlxsw_sp_fib *fib) 566 { 567 mlxsw_sp_vr_lpm_tree_unbind(mlxsw_sp, fib); 568 mlxsw_sp_lpm_tree_put(mlxsw_sp, fib->lpm_tree); 569 WARN_ON(!list_empty(&fib->node_list)); 570 rhashtable_destroy(&fib->ht); 571 kfree(fib); 572 } 573 574 static struct mlxsw_sp_lpm_tree * 575 mlxsw_sp_lpm_tree_find_unused(struct mlxsw_sp *mlxsw_sp) 576 { 577 static struct mlxsw_sp_lpm_tree *lpm_tree; 578 int i; 579 580 for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) { 581 lpm_tree = &mlxsw_sp->router->lpm.trees[i]; 582 if (refcount_read(&lpm_tree->ref_count) == 0) 583 return lpm_tree; 584 } 585 return NULL; 586 } 587 588 static int mlxsw_sp_lpm_tree_alloc(struct mlxsw_sp *mlxsw_sp, 589 struct mlxsw_sp_lpm_tree *lpm_tree) 590 { 591 char ralta_pl[MLXSW_REG_RALTA_LEN]; 592 593 mlxsw_reg_ralta_pack(ralta_pl, true, 594 (enum mlxsw_reg_ralxx_protocol) lpm_tree->proto, 595 lpm_tree->id); 596 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), ralta_pl); 597 } 598 599 static void mlxsw_sp_lpm_tree_free(struct mlxsw_sp *mlxsw_sp, 600 struct mlxsw_sp_lpm_tree *lpm_tree) 601 { 602 char ralta_pl[MLXSW_REG_RALTA_LEN]; 603 604 mlxsw_reg_ralta_pack(ralta_pl, false, 605 (enum mlxsw_reg_ralxx_protocol) lpm_tree->proto, 606 lpm_tree->id); 607 mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), ralta_pl); 608 } 609 610 static int 611 mlxsw_sp_lpm_tree_left_struct_set(struct mlxsw_sp *mlxsw_sp, 612 struct mlxsw_sp_prefix_usage *prefix_usage, 613 struct mlxsw_sp_lpm_tree *lpm_tree) 614 { 615 char ralst_pl[MLXSW_REG_RALST_LEN]; 616 u8 root_bin = 0; 617 u8 prefix; 618 u8 last_prefix = MLXSW_REG_RALST_BIN_NO_CHILD; 619 620 mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) 621 root_bin = prefix; 622 623 mlxsw_reg_ralst_pack(ralst_pl, root_bin, lpm_tree->id); 624 mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) { 625 if (prefix == 0) 626 continue; 627 mlxsw_reg_ralst_bin_pack(ralst_pl, prefix, last_prefix, 628 MLXSW_REG_RALST_BIN_NO_CHILD); 629 last_prefix = prefix; 630 } 631 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralst), ralst_pl); 632 } 633 634 static struct mlxsw_sp_lpm_tree * 635 mlxsw_sp_lpm_tree_create(struct mlxsw_sp *mlxsw_sp, 636 struct mlxsw_sp_prefix_usage *prefix_usage, 637 enum mlxsw_sp_l3proto proto) 638 { 639 struct mlxsw_sp_lpm_tree *lpm_tree; 640 int err; 641 642 lpm_tree = mlxsw_sp_lpm_tree_find_unused(mlxsw_sp); 643 if (!lpm_tree) 644 return ERR_PTR(-EBUSY); 645 lpm_tree->proto = proto; 646 err = mlxsw_sp_lpm_tree_alloc(mlxsw_sp, lpm_tree); 647 if (err) 648 return ERR_PTR(err); 649 650 err = mlxsw_sp_lpm_tree_left_struct_set(mlxsw_sp, prefix_usage, 651 lpm_tree); 652 if (err) 653 goto err_left_struct_set; 654 memcpy(&lpm_tree->prefix_usage, prefix_usage, 655 sizeof(lpm_tree->prefix_usage)); 656 memset(&lpm_tree->prefix_ref_count, 0, 657 sizeof(lpm_tree->prefix_ref_count)); 658 refcount_set(&lpm_tree->ref_count, 1); 659 return lpm_tree; 660 661 err_left_struct_set: 662 mlxsw_sp_lpm_tree_free(mlxsw_sp, lpm_tree); 663 return ERR_PTR(err); 664 } 665 666 static void mlxsw_sp_lpm_tree_destroy(struct mlxsw_sp *mlxsw_sp, 667 struct mlxsw_sp_lpm_tree *lpm_tree) 668 { 669 mlxsw_sp_lpm_tree_free(mlxsw_sp, lpm_tree); 670 } 671 672 static struct mlxsw_sp_lpm_tree * 673 mlxsw_sp_lpm_tree_get(struct mlxsw_sp *mlxsw_sp, 674 struct mlxsw_sp_prefix_usage *prefix_usage, 675 enum mlxsw_sp_l3proto proto) 676 { 677 struct mlxsw_sp_lpm_tree *lpm_tree; 678 int i; 679 680 for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) { 681 lpm_tree = &mlxsw_sp->router->lpm.trees[i]; 682 if (refcount_read(&lpm_tree->ref_count) && 683 lpm_tree->proto == proto && 684 mlxsw_sp_prefix_usage_eq(&lpm_tree->prefix_usage, 685 prefix_usage)) { 686 mlxsw_sp_lpm_tree_hold(lpm_tree); 687 return lpm_tree; 688 } 689 } 690 return mlxsw_sp_lpm_tree_create(mlxsw_sp, prefix_usage, proto); 691 } 692 693 static void mlxsw_sp_lpm_tree_hold(struct mlxsw_sp_lpm_tree *lpm_tree) 694 { 695 refcount_inc(&lpm_tree->ref_count); 696 } 697 698 static void mlxsw_sp_lpm_tree_put(struct mlxsw_sp *mlxsw_sp, 699 struct mlxsw_sp_lpm_tree *lpm_tree) 700 { 701 if (!refcount_dec_and_test(&lpm_tree->ref_count)) 702 return; 703 mlxsw_sp_lpm_tree_destroy(mlxsw_sp, lpm_tree); 704 } 705 706 #define MLXSW_SP_LPM_TREE_MIN 1 /* tree 0 is reserved */ 707 708 static int mlxsw_sp_lpm_init(struct mlxsw_sp *mlxsw_sp) 709 { 710 struct mlxsw_sp_prefix_usage req_prefix_usage = {{ 0 } }; 711 struct mlxsw_sp_lpm_tree *lpm_tree; 712 u64 max_trees; 713 int err, i; 714 715 if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_LPM_TREES)) 716 return -EIO; 717 718 max_trees = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_LPM_TREES); 719 mlxsw_sp->router->lpm.tree_count = max_trees - MLXSW_SP_LPM_TREE_MIN; 720 mlxsw_sp->router->lpm.trees = kzalloc_objs(struct mlxsw_sp_lpm_tree, 721 mlxsw_sp->router->lpm.tree_count); 722 if (!mlxsw_sp->router->lpm.trees) 723 return -ENOMEM; 724 725 for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) { 726 lpm_tree = &mlxsw_sp->router->lpm.trees[i]; 727 lpm_tree->id = i + MLXSW_SP_LPM_TREE_MIN; 728 } 729 730 lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage, 731 MLXSW_SP_L3_PROTO_IPV4); 732 if (IS_ERR(lpm_tree)) { 733 err = PTR_ERR(lpm_tree); 734 goto err_ipv4_tree_get; 735 } 736 mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV4] = lpm_tree; 737 738 lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage, 739 MLXSW_SP_L3_PROTO_IPV6); 740 if (IS_ERR(lpm_tree)) { 741 err = PTR_ERR(lpm_tree); 742 goto err_ipv6_tree_get; 743 } 744 mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV6] = lpm_tree; 745 746 return 0; 747 748 err_ipv6_tree_get: 749 lpm_tree = mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV4]; 750 mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree); 751 err_ipv4_tree_get: 752 kfree(mlxsw_sp->router->lpm.trees); 753 return err; 754 } 755 756 static void mlxsw_sp_lpm_fini(struct mlxsw_sp *mlxsw_sp) 757 { 758 struct mlxsw_sp_lpm_tree *lpm_tree; 759 760 lpm_tree = mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV6]; 761 mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree); 762 763 lpm_tree = mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV4]; 764 mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree); 765 766 kfree(mlxsw_sp->router->lpm.trees); 767 } 768 769 static bool mlxsw_sp_vr_is_used(const struct mlxsw_sp_vr *vr) 770 { 771 return !!vr->fib4 || !!vr->fib6 || 772 !!vr->mr_table[MLXSW_SP_L3_PROTO_IPV4] || 773 !!vr->mr_table[MLXSW_SP_L3_PROTO_IPV6]; 774 } 775 776 static struct mlxsw_sp_vr *mlxsw_sp_vr_find_unused(struct mlxsw_sp *mlxsw_sp) 777 { 778 int max_vrs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); 779 struct mlxsw_sp_vr *vr; 780 int i; 781 782 for (i = 0; i < max_vrs; i++) { 783 vr = &mlxsw_sp->router->vrs[i]; 784 if (!mlxsw_sp_vr_is_used(vr)) 785 return vr; 786 } 787 return NULL; 788 } 789 790 static int mlxsw_sp_vr_lpm_tree_bind(struct mlxsw_sp *mlxsw_sp, 791 const struct mlxsw_sp_fib *fib, u8 tree_id) 792 { 793 char raltb_pl[MLXSW_REG_RALTB_LEN]; 794 795 mlxsw_reg_raltb_pack(raltb_pl, fib->vr->id, 796 (enum mlxsw_reg_ralxx_protocol) fib->proto, 797 tree_id); 798 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb), raltb_pl); 799 } 800 801 static int mlxsw_sp_vr_lpm_tree_unbind(struct mlxsw_sp *mlxsw_sp, 802 const struct mlxsw_sp_fib *fib) 803 { 804 char raltb_pl[MLXSW_REG_RALTB_LEN]; 805 806 /* Bind to tree 0 which is default */ 807 mlxsw_reg_raltb_pack(raltb_pl, fib->vr->id, 808 (enum mlxsw_reg_ralxx_protocol) fib->proto, 0); 809 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb), raltb_pl); 810 } 811 812 static u32 mlxsw_sp_fix_tb_id(u32 tb_id) 813 { 814 /* For our purpose, squash main, default and local tables into one */ 815 if (tb_id == RT_TABLE_LOCAL || tb_id == RT_TABLE_DEFAULT) 816 tb_id = RT_TABLE_MAIN; 817 return tb_id; 818 } 819 820 static struct mlxsw_sp_vr *mlxsw_sp_vr_find(struct mlxsw_sp *mlxsw_sp, 821 u32 tb_id) 822 { 823 int max_vrs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); 824 struct mlxsw_sp_vr *vr; 825 int i; 826 827 tb_id = mlxsw_sp_fix_tb_id(tb_id); 828 829 for (i = 0; i < max_vrs; i++) { 830 vr = &mlxsw_sp->router->vrs[i]; 831 if (mlxsw_sp_vr_is_used(vr) && vr->tb_id == tb_id) 832 return vr; 833 } 834 return NULL; 835 } 836 837 int mlxsw_sp_router_tb_id_vr_id(struct mlxsw_sp *mlxsw_sp, u32 tb_id, 838 u16 *vr_id) 839 { 840 struct mlxsw_sp_vr *vr; 841 int err = 0; 842 843 mutex_lock(&mlxsw_sp->router->lock); 844 vr = mlxsw_sp_vr_find(mlxsw_sp, tb_id); 845 if (!vr) { 846 err = -ESRCH; 847 goto out; 848 } 849 *vr_id = vr->id; 850 out: 851 mutex_unlock(&mlxsw_sp->router->lock); 852 return err; 853 } 854 855 static struct mlxsw_sp_fib *mlxsw_sp_vr_fib(const struct mlxsw_sp_vr *vr, 856 enum mlxsw_sp_l3proto proto) 857 { 858 switch (proto) { 859 case MLXSW_SP_L3_PROTO_IPV4: 860 return vr->fib4; 861 case MLXSW_SP_L3_PROTO_IPV6: 862 return vr->fib6; 863 } 864 return NULL; 865 } 866 867 static struct mlxsw_sp_vr *mlxsw_sp_vr_create(struct mlxsw_sp *mlxsw_sp, 868 u32 tb_id, 869 struct netlink_ext_ack *extack) 870 { 871 struct mlxsw_sp_mr_table *mr4_table, *mr6_table; 872 struct mlxsw_sp_fib *fib4; 873 struct mlxsw_sp_fib *fib6; 874 struct mlxsw_sp_vr *vr; 875 int err; 876 877 vr = mlxsw_sp_vr_find_unused(mlxsw_sp); 878 if (!vr) { 879 NL_SET_ERR_MSG_MOD(extack, "Exceeded number of supported virtual routers"); 880 return ERR_PTR(-EBUSY); 881 } 882 fib4 = mlxsw_sp_fib_create(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV4); 883 if (IS_ERR(fib4)) 884 return ERR_CAST(fib4); 885 fib6 = mlxsw_sp_fib_create(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV6); 886 if (IS_ERR(fib6)) { 887 err = PTR_ERR(fib6); 888 goto err_fib6_create; 889 } 890 mr4_table = mlxsw_sp_mr_table_create(mlxsw_sp, vr->id, 891 MLXSW_SP_L3_PROTO_IPV4); 892 if (IS_ERR(mr4_table)) { 893 err = PTR_ERR(mr4_table); 894 goto err_mr4_table_create; 895 } 896 mr6_table = mlxsw_sp_mr_table_create(mlxsw_sp, vr->id, 897 MLXSW_SP_L3_PROTO_IPV6); 898 if (IS_ERR(mr6_table)) { 899 err = PTR_ERR(mr6_table); 900 goto err_mr6_table_create; 901 } 902 903 vr->fib4 = fib4; 904 vr->fib6 = fib6; 905 vr->mr_table[MLXSW_SP_L3_PROTO_IPV4] = mr4_table; 906 vr->mr_table[MLXSW_SP_L3_PROTO_IPV6] = mr6_table; 907 vr->tb_id = tb_id; 908 return vr; 909 910 err_mr6_table_create: 911 mlxsw_sp_mr_table_destroy(mr4_table); 912 err_mr4_table_create: 913 mlxsw_sp_fib_destroy(mlxsw_sp, fib6); 914 err_fib6_create: 915 mlxsw_sp_fib_destroy(mlxsw_sp, fib4); 916 return ERR_PTR(err); 917 } 918 919 static void mlxsw_sp_vr_destroy(struct mlxsw_sp *mlxsw_sp, 920 struct mlxsw_sp_vr *vr) 921 { 922 mlxsw_sp_mr_table_destroy(vr->mr_table[MLXSW_SP_L3_PROTO_IPV6]); 923 vr->mr_table[MLXSW_SP_L3_PROTO_IPV6] = NULL; 924 mlxsw_sp_mr_table_destroy(vr->mr_table[MLXSW_SP_L3_PROTO_IPV4]); 925 vr->mr_table[MLXSW_SP_L3_PROTO_IPV4] = NULL; 926 mlxsw_sp_fib_destroy(mlxsw_sp, vr->fib6); 927 vr->fib6 = NULL; 928 mlxsw_sp_fib_destroy(mlxsw_sp, vr->fib4); 929 vr->fib4 = NULL; 930 } 931 932 static struct mlxsw_sp_vr *mlxsw_sp_vr_get(struct mlxsw_sp *mlxsw_sp, u32 tb_id, 933 struct netlink_ext_ack *extack) 934 { 935 struct mlxsw_sp_vr *vr; 936 937 tb_id = mlxsw_sp_fix_tb_id(tb_id); 938 vr = mlxsw_sp_vr_find(mlxsw_sp, tb_id); 939 if (!vr) 940 vr = mlxsw_sp_vr_create(mlxsw_sp, tb_id, extack); 941 return vr; 942 } 943 944 static void mlxsw_sp_vr_put(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp_vr *vr) 945 { 946 if (!vr->rif_count && list_empty(&vr->fib4->node_list) && 947 list_empty(&vr->fib6->node_list) && 948 mlxsw_sp_mr_table_empty(vr->mr_table[MLXSW_SP_L3_PROTO_IPV4]) && 949 mlxsw_sp_mr_table_empty(vr->mr_table[MLXSW_SP_L3_PROTO_IPV6])) 950 mlxsw_sp_vr_destroy(mlxsw_sp, vr); 951 } 952 953 static bool 954 mlxsw_sp_vr_lpm_tree_should_replace(struct mlxsw_sp_vr *vr, 955 enum mlxsw_sp_l3proto proto, u8 tree_id) 956 { 957 struct mlxsw_sp_fib *fib = mlxsw_sp_vr_fib(vr, proto); 958 959 if (!mlxsw_sp_vr_is_used(vr)) 960 return false; 961 if (fib->lpm_tree->id == tree_id) 962 return true; 963 return false; 964 } 965 966 static int mlxsw_sp_vr_lpm_tree_replace(struct mlxsw_sp *mlxsw_sp, 967 struct mlxsw_sp_fib *fib, 968 struct mlxsw_sp_lpm_tree *new_tree) 969 { 970 struct mlxsw_sp_lpm_tree *old_tree = fib->lpm_tree; 971 int err; 972 973 fib->lpm_tree = new_tree; 974 mlxsw_sp_lpm_tree_hold(new_tree); 975 err = mlxsw_sp_vr_lpm_tree_bind(mlxsw_sp, fib, new_tree->id); 976 if (err) 977 goto err_tree_bind; 978 mlxsw_sp_lpm_tree_put(mlxsw_sp, old_tree); 979 return 0; 980 981 err_tree_bind: 982 mlxsw_sp_lpm_tree_put(mlxsw_sp, new_tree); 983 fib->lpm_tree = old_tree; 984 return err; 985 } 986 987 static int mlxsw_sp_vrs_lpm_tree_replace(struct mlxsw_sp *mlxsw_sp, 988 struct mlxsw_sp_fib *fib, 989 struct mlxsw_sp_lpm_tree *new_tree) 990 { 991 int max_vrs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); 992 enum mlxsw_sp_l3proto proto = fib->proto; 993 struct mlxsw_sp_lpm_tree *old_tree; 994 u8 old_id, new_id = new_tree->id; 995 struct mlxsw_sp_vr *vr; 996 int i, err; 997 998 old_tree = mlxsw_sp->router->lpm.proto_trees[proto]; 999 old_id = old_tree->id; 1000 1001 for (i = 0; i < max_vrs; i++) { 1002 vr = &mlxsw_sp->router->vrs[i]; 1003 if (!mlxsw_sp_vr_lpm_tree_should_replace(vr, proto, old_id)) 1004 continue; 1005 err = mlxsw_sp_vr_lpm_tree_replace(mlxsw_sp, 1006 mlxsw_sp_vr_fib(vr, proto), 1007 new_tree); 1008 if (err) 1009 goto err_tree_replace; 1010 } 1011 1012 memcpy(new_tree->prefix_ref_count, old_tree->prefix_ref_count, 1013 sizeof(new_tree->prefix_ref_count)); 1014 mlxsw_sp->router->lpm.proto_trees[proto] = new_tree; 1015 mlxsw_sp_lpm_tree_put(mlxsw_sp, old_tree); 1016 1017 return 0; 1018 1019 err_tree_replace: 1020 for (i--; i >= 0; i--) { 1021 vr = &mlxsw_sp->router->vrs[i]; 1022 if (!mlxsw_sp_vr_lpm_tree_should_replace(vr, proto, new_id)) 1023 continue; 1024 mlxsw_sp_vr_lpm_tree_replace(mlxsw_sp, 1025 mlxsw_sp_vr_fib(vr, proto), 1026 old_tree); 1027 } 1028 return err; 1029 } 1030 1031 static int mlxsw_sp_vrs_init(struct mlxsw_sp *mlxsw_sp) 1032 { 1033 struct mlxsw_sp_vr *vr; 1034 u64 max_vrs; 1035 int i; 1036 1037 if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_VRS)) 1038 return -EIO; 1039 1040 max_vrs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); 1041 mlxsw_sp->router->vrs = kzalloc_objs(struct mlxsw_sp_vr, max_vrs); 1042 if (!mlxsw_sp->router->vrs) 1043 return -ENOMEM; 1044 1045 for (i = 0; i < max_vrs; i++) { 1046 vr = &mlxsw_sp->router->vrs[i]; 1047 vr->id = i; 1048 } 1049 1050 return 0; 1051 } 1052 1053 static void mlxsw_sp_router_fib_flush(struct mlxsw_sp *mlxsw_sp); 1054 1055 static void mlxsw_sp_vrs_fini(struct mlxsw_sp *mlxsw_sp) 1056 { 1057 /* At this stage we're guaranteed not to have new incoming 1058 * FIB notifications and the work queue is free from FIBs 1059 * sitting on top of mlxsw netdevs. However, we can still 1060 * have other FIBs queued. Flush the queue before flushing 1061 * the device's tables. No need for locks, as we're the only 1062 * writer. 1063 */ 1064 mlxsw_core_flush_owq(); 1065 mlxsw_sp_router_fib_flush(mlxsw_sp); 1066 kfree(mlxsw_sp->router->vrs); 1067 } 1068 1069 u32 mlxsw_sp_ipip_dev_ul_tb_id(const struct net_device *ol_dev) 1070 { 1071 struct net_device *d; 1072 u32 tb_id; 1073 1074 rcu_read_lock(); 1075 d = mlxsw_sp_ipip_netdev_ul_dev_get(ol_dev); 1076 if (d) 1077 tb_id = l3mdev_fib_table(d) ? : RT_TABLE_MAIN; 1078 else 1079 tb_id = RT_TABLE_MAIN; 1080 rcu_read_unlock(); 1081 1082 return tb_id; 1083 } 1084 1085 static void 1086 mlxsw_sp_crif_init(struct mlxsw_sp_crif *crif, struct net_device *dev) 1087 { 1088 crif->key.dev = dev; 1089 INIT_LIST_HEAD(&crif->nexthop_list); 1090 } 1091 1092 static struct mlxsw_sp_crif * 1093 mlxsw_sp_crif_alloc(struct net_device *dev) 1094 { 1095 struct mlxsw_sp_crif *crif; 1096 1097 crif = kzalloc_obj(*crif); 1098 if (!crif) 1099 return NULL; 1100 1101 mlxsw_sp_crif_init(crif, dev); 1102 return crif; 1103 } 1104 1105 static void mlxsw_sp_crif_free(struct mlxsw_sp_crif *crif) 1106 { 1107 if (WARN_ON(crif->rif)) 1108 return; 1109 1110 WARN_ON(!list_empty(&crif->nexthop_list)); 1111 kfree(crif); 1112 } 1113 1114 static int mlxsw_sp_crif_insert(struct mlxsw_sp_router *router, 1115 struct mlxsw_sp_crif *crif) 1116 { 1117 return rhashtable_insert_fast(&router->crif_ht, &crif->ht_node, 1118 mlxsw_sp_crif_ht_params); 1119 } 1120 1121 static void mlxsw_sp_crif_remove(struct mlxsw_sp_router *router, 1122 struct mlxsw_sp_crif *crif) 1123 { 1124 rhashtable_remove_fast(&router->crif_ht, &crif->ht_node, 1125 mlxsw_sp_crif_ht_params); 1126 } 1127 1128 static struct mlxsw_sp_crif * 1129 mlxsw_sp_crif_lookup(struct mlxsw_sp_router *router, 1130 const struct net_device *dev) 1131 { 1132 struct mlxsw_sp_crif_key key = { 1133 .dev = (struct net_device *)dev, 1134 }; 1135 1136 return rhashtable_lookup_fast(&router->crif_ht, &key, 1137 mlxsw_sp_crif_ht_params); 1138 } 1139 1140 static struct mlxsw_sp_rif * 1141 mlxsw_sp_rif_create(struct mlxsw_sp *mlxsw_sp, 1142 const struct mlxsw_sp_rif_params *params, 1143 struct netlink_ext_ack *extack); 1144 1145 static struct mlxsw_sp_rif_ipip_lb * 1146 mlxsw_sp_ipip_ol_ipip_lb_create(struct mlxsw_sp *mlxsw_sp, 1147 enum mlxsw_sp_ipip_type ipipt, 1148 struct net_device *ol_dev, 1149 struct netlink_ext_ack *extack) 1150 { 1151 struct mlxsw_sp_rif_params_ipip_lb lb_params; 1152 const struct mlxsw_sp_ipip_ops *ipip_ops; 1153 struct mlxsw_sp_rif *rif; 1154 1155 ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipipt]; 1156 lb_params = (struct mlxsw_sp_rif_params_ipip_lb) { 1157 .common.dev = ol_dev, 1158 .common.lag = false, 1159 .common.double_entry = ipip_ops->double_rif_entry, 1160 .lb_config = ipip_ops->ol_loopback_config(mlxsw_sp, ol_dev), 1161 }; 1162 1163 rif = mlxsw_sp_rif_create(mlxsw_sp, &lb_params.common, extack); 1164 if (IS_ERR(rif)) 1165 return ERR_CAST(rif); 1166 return container_of(rif, struct mlxsw_sp_rif_ipip_lb, common); 1167 } 1168 1169 static struct mlxsw_sp_ipip_entry * 1170 mlxsw_sp_ipip_entry_alloc(struct mlxsw_sp *mlxsw_sp, 1171 enum mlxsw_sp_ipip_type ipipt, 1172 struct net_device *ol_dev) 1173 { 1174 const struct mlxsw_sp_ipip_ops *ipip_ops; 1175 struct mlxsw_sp_ipip_entry *ipip_entry; 1176 struct mlxsw_sp_ipip_entry *ret = NULL; 1177 int err; 1178 1179 ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipipt]; 1180 ipip_entry = kzalloc_obj(*ipip_entry); 1181 if (!ipip_entry) 1182 return ERR_PTR(-ENOMEM); 1183 1184 ipip_entry->ol_lb = mlxsw_sp_ipip_ol_ipip_lb_create(mlxsw_sp, ipipt, 1185 ol_dev, NULL); 1186 if (IS_ERR(ipip_entry->ol_lb)) { 1187 ret = ERR_CAST(ipip_entry->ol_lb); 1188 goto err_ol_ipip_lb_create; 1189 } 1190 1191 ipip_entry->ipipt = ipipt; 1192 ipip_entry->ol_dev = ol_dev; 1193 ipip_entry->parms = ipip_ops->parms_init(ol_dev); 1194 1195 err = ipip_ops->rem_ip_addr_set(mlxsw_sp, ipip_entry); 1196 if (err) { 1197 ret = ERR_PTR(err); 1198 goto err_rem_ip_addr_set; 1199 } 1200 1201 return ipip_entry; 1202 1203 err_rem_ip_addr_set: 1204 mlxsw_sp_rif_destroy(&ipip_entry->ol_lb->common); 1205 err_ol_ipip_lb_create: 1206 kfree(ipip_entry); 1207 return ret; 1208 } 1209 1210 static void mlxsw_sp_ipip_entry_dealloc(struct mlxsw_sp *mlxsw_sp, 1211 struct mlxsw_sp_ipip_entry *ipip_entry) 1212 { 1213 const struct mlxsw_sp_ipip_ops *ipip_ops = 1214 mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt]; 1215 1216 ipip_ops->rem_ip_addr_unset(mlxsw_sp, ipip_entry); 1217 mlxsw_sp_rif_destroy(&ipip_entry->ol_lb->common); 1218 kfree(ipip_entry); 1219 } 1220 1221 static bool 1222 mlxsw_sp_ipip_entry_saddr_matches(struct mlxsw_sp *mlxsw_sp, 1223 const enum mlxsw_sp_l3proto ul_proto, 1224 union mlxsw_sp_l3addr saddr, 1225 u32 ul_tb_id, 1226 struct mlxsw_sp_ipip_entry *ipip_entry) 1227 { 1228 u32 tun_ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(ipip_entry->ol_dev); 1229 enum mlxsw_sp_ipip_type ipipt = ipip_entry->ipipt; 1230 union mlxsw_sp_l3addr tun_saddr; 1231 1232 if (mlxsw_sp->router->ipip_ops_arr[ipipt]->ul_proto != ul_proto) 1233 return false; 1234 1235 tun_saddr = mlxsw_sp_ipip_netdev_saddr(ul_proto, ipip_entry->ol_dev); 1236 return tun_ul_tb_id == ul_tb_id && 1237 mlxsw_sp_l3addr_eq(&tun_saddr, &saddr); 1238 } 1239 1240 static int mlxsw_sp_ipip_decap_parsing_depth_inc(struct mlxsw_sp *mlxsw_sp, 1241 enum mlxsw_sp_ipip_type ipipt) 1242 { 1243 const struct mlxsw_sp_ipip_ops *ipip_ops; 1244 1245 ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipipt]; 1246 1247 /* Not all tunnels require to increase the default pasing depth 1248 * (96 bytes). 1249 */ 1250 if (ipip_ops->inc_parsing_depth) 1251 return mlxsw_sp_parsing_depth_inc(mlxsw_sp); 1252 1253 return 0; 1254 } 1255 1256 static void mlxsw_sp_ipip_decap_parsing_depth_dec(struct mlxsw_sp *mlxsw_sp, 1257 enum mlxsw_sp_ipip_type ipipt) 1258 { 1259 const struct mlxsw_sp_ipip_ops *ipip_ops = 1260 mlxsw_sp->router->ipip_ops_arr[ipipt]; 1261 1262 if (ipip_ops->inc_parsing_depth) 1263 mlxsw_sp_parsing_depth_dec(mlxsw_sp); 1264 } 1265 1266 static int 1267 mlxsw_sp_fib_entry_decap_init(struct mlxsw_sp *mlxsw_sp, 1268 struct mlxsw_sp_fib_entry *fib_entry, 1269 struct mlxsw_sp_ipip_entry *ipip_entry) 1270 { 1271 u32 tunnel_index; 1272 int err; 1273 1274 err = mlxsw_sp_kvdl_alloc(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 1275 1, &tunnel_index); 1276 if (err) 1277 return err; 1278 1279 err = mlxsw_sp_ipip_decap_parsing_depth_inc(mlxsw_sp, 1280 ipip_entry->ipipt); 1281 if (err) 1282 goto err_parsing_depth_inc; 1283 1284 ipip_entry->decap_fib_entry = fib_entry; 1285 fib_entry->decap.ipip_entry = ipip_entry; 1286 fib_entry->decap.tunnel_index = tunnel_index; 1287 1288 return 0; 1289 1290 err_parsing_depth_inc: 1291 mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 1, 1292 fib_entry->decap.tunnel_index); 1293 return err; 1294 } 1295 1296 static void mlxsw_sp_fib_entry_decap_fini(struct mlxsw_sp *mlxsw_sp, 1297 struct mlxsw_sp_fib_entry *fib_entry) 1298 { 1299 enum mlxsw_sp_ipip_type ipipt = fib_entry->decap.ipip_entry->ipipt; 1300 1301 /* Unlink this node from the IPIP entry that it's the decap entry of. */ 1302 fib_entry->decap.ipip_entry->decap_fib_entry = NULL; 1303 fib_entry->decap.ipip_entry = NULL; 1304 mlxsw_sp_ipip_decap_parsing_depth_dec(mlxsw_sp, ipipt); 1305 mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 1306 1, fib_entry->decap.tunnel_index); 1307 } 1308 1309 static struct mlxsw_sp_fib_node * 1310 mlxsw_sp_fib_node_lookup(struct mlxsw_sp_fib *fib, const void *addr, 1311 size_t addr_len, unsigned char prefix_len); 1312 static int mlxsw_sp_fib_entry_update(struct mlxsw_sp *mlxsw_sp, 1313 struct mlxsw_sp_fib_entry *fib_entry); 1314 1315 static void 1316 mlxsw_sp_ipip_entry_demote_decap(struct mlxsw_sp *mlxsw_sp, 1317 struct mlxsw_sp_ipip_entry *ipip_entry) 1318 { 1319 struct mlxsw_sp_fib_entry *fib_entry = ipip_entry->decap_fib_entry; 1320 1321 mlxsw_sp_fib_entry_decap_fini(mlxsw_sp, fib_entry); 1322 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP; 1323 1324 mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry); 1325 } 1326 1327 static void 1328 mlxsw_sp_ipip_entry_promote_decap(struct mlxsw_sp *mlxsw_sp, 1329 struct mlxsw_sp_ipip_entry *ipip_entry, 1330 struct mlxsw_sp_fib_entry *decap_fib_entry) 1331 { 1332 if (mlxsw_sp_fib_entry_decap_init(mlxsw_sp, decap_fib_entry, 1333 ipip_entry)) 1334 return; 1335 decap_fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP; 1336 1337 if (mlxsw_sp_fib_entry_update(mlxsw_sp, decap_fib_entry)) 1338 mlxsw_sp_ipip_entry_demote_decap(mlxsw_sp, ipip_entry); 1339 } 1340 1341 static struct mlxsw_sp_fib_entry * 1342 mlxsw_sp_router_ip2me_fib_entry_find(struct mlxsw_sp *mlxsw_sp, u32 tb_id, 1343 enum mlxsw_sp_l3proto proto, 1344 const union mlxsw_sp_l3addr *addr, 1345 enum mlxsw_sp_fib_entry_type type) 1346 { 1347 struct mlxsw_sp_fib_node *fib_node; 1348 unsigned char addr_prefix_len; 1349 struct mlxsw_sp_fib *fib; 1350 struct mlxsw_sp_vr *vr; 1351 const void *addrp; 1352 size_t addr_len; 1353 u32 addr4; 1354 1355 vr = mlxsw_sp_vr_find(mlxsw_sp, tb_id); 1356 if (!vr) 1357 return NULL; 1358 fib = mlxsw_sp_vr_fib(vr, proto); 1359 1360 switch (proto) { 1361 case MLXSW_SP_L3_PROTO_IPV4: 1362 addr4 = be32_to_cpu(addr->addr4); 1363 addrp = &addr4; 1364 addr_len = 4; 1365 addr_prefix_len = 32; 1366 break; 1367 case MLXSW_SP_L3_PROTO_IPV6: 1368 addrp = &addr->addr6; 1369 addr_len = 16; 1370 addr_prefix_len = 128; 1371 break; 1372 default: 1373 WARN_ON(1); 1374 return NULL; 1375 } 1376 1377 fib_node = mlxsw_sp_fib_node_lookup(fib, addrp, addr_len, 1378 addr_prefix_len); 1379 if (!fib_node || fib_node->fib_entry->type != type) 1380 return NULL; 1381 1382 return fib_node->fib_entry; 1383 } 1384 1385 /* Given an IPIP entry, find the corresponding decap route. */ 1386 static struct mlxsw_sp_fib_entry * 1387 mlxsw_sp_ipip_entry_find_decap(struct mlxsw_sp *mlxsw_sp, 1388 struct mlxsw_sp_ipip_entry *ipip_entry) 1389 { 1390 static struct mlxsw_sp_fib_node *fib_node; 1391 const struct mlxsw_sp_ipip_ops *ipip_ops; 1392 unsigned char saddr_prefix_len; 1393 union mlxsw_sp_l3addr saddr; 1394 struct mlxsw_sp_fib *ul_fib; 1395 struct mlxsw_sp_vr *ul_vr; 1396 const void *saddrp; 1397 size_t saddr_len; 1398 u32 ul_tb_id; 1399 u32 saddr4; 1400 1401 ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt]; 1402 1403 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(ipip_entry->ol_dev); 1404 ul_vr = mlxsw_sp_vr_find(mlxsw_sp, ul_tb_id); 1405 if (!ul_vr) 1406 return NULL; 1407 1408 ul_fib = mlxsw_sp_vr_fib(ul_vr, ipip_ops->ul_proto); 1409 saddr = mlxsw_sp_ipip_netdev_saddr(ipip_ops->ul_proto, 1410 ipip_entry->ol_dev); 1411 1412 switch (ipip_ops->ul_proto) { 1413 case MLXSW_SP_L3_PROTO_IPV4: 1414 saddr4 = be32_to_cpu(saddr.addr4); 1415 saddrp = &saddr4; 1416 saddr_len = 4; 1417 saddr_prefix_len = 32; 1418 break; 1419 case MLXSW_SP_L3_PROTO_IPV6: 1420 saddrp = &saddr.addr6; 1421 saddr_len = 16; 1422 saddr_prefix_len = 128; 1423 break; 1424 default: 1425 WARN_ON(1); 1426 return NULL; 1427 } 1428 1429 fib_node = mlxsw_sp_fib_node_lookup(ul_fib, saddrp, saddr_len, 1430 saddr_prefix_len); 1431 if (!fib_node || 1432 fib_node->fib_entry->type != MLXSW_SP_FIB_ENTRY_TYPE_TRAP) 1433 return NULL; 1434 1435 return fib_node->fib_entry; 1436 } 1437 1438 static struct mlxsw_sp_ipip_entry * 1439 mlxsw_sp_ipip_entry_create(struct mlxsw_sp *mlxsw_sp, 1440 enum mlxsw_sp_ipip_type ipipt, 1441 struct net_device *ol_dev) 1442 { 1443 struct mlxsw_sp_ipip_entry *ipip_entry; 1444 1445 ipip_entry = mlxsw_sp_ipip_entry_alloc(mlxsw_sp, ipipt, ol_dev); 1446 if (IS_ERR(ipip_entry)) 1447 return ipip_entry; 1448 1449 list_add_tail(&ipip_entry->ipip_list_node, 1450 &mlxsw_sp->router->ipip_list); 1451 1452 return ipip_entry; 1453 } 1454 1455 static void 1456 mlxsw_sp_ipip_entry_destroy(struct mlxsw_sp *mlxsw_sp, 1457 struct mlxsw_sp_ipip_entry *ipip_entry) 1458 { 1459 list_del(&ipip_entry->ipip_list_node); 1460 mlxsw_sp_ipip_entry_dealloc(mlxsw_sp, ipip_entry); 1461 } 1462 1463 static bool 1464 mlxsw_sp_ipip_entry_matches_decap(struct mlxsw_sp *mlxsw_sp, 1465 const struct net_device *ul_dev, 1466 enum mlxsw_sp_l3proto ul_proto, 1467 union mlxsw_sp_l3addr ul_dip, 1468 struct mlxsw_sp_ipip_entry *ipip_entry) 1469 { 1470 u32 ul_tb_id = l3mdev_fib_table(ul_dev) ? : RT_TABLE_MAIN; 1471 enum mlxsw_sp_ipip_type ipipt = ipip_entry->ipipt; 1472 1473 if (mlxsw_sp->router->ipip_ops_arr[ipipt]->ul_proto != ul_proto) 1474 return false; 1475 1476 return mlxsw_sp_ipip_entry_saddr_matches(mlxsw_sp, ul_proto, ul_dip, 1477 ul_tb_id, ipip_entry); 1478 } 1479 1480 /* Given decap parameters, find the corresponding IPIP entry. */ 1481 static struct mlxsw_sp_ipip_entry * 1482 mlxsw_sp_ipip_entry_find_by_decap(struct mlxsw_sp *mlxsw_sp, int ul_dev_ifindex, 1483 enum mlxsw_sp_l3proto ul_proto, 1484 union mlxsw_sp_l3addr ul_dip) 1485 { 1486 struct mlxsw_sp_ipip_entry *ipip_entry = NULL; 1487 struct net_device *ul_dev; 1488 1489 rcu_read_lock(); 1490 1491 ul_dev = dev_get_by_index_rcu(mlxsw_sp_net(mlxsw_sp), ul_dev_ifindex); 1492 if (!ul_dev) 1493 goto out_unlock; 1494 1495 list_for_each_entry(ipip_entry, &mlxsw_sp->router->ipip_list, 1496 ipip_list_node) 1497 if (mlxsw_sp_ipip_entry_matches_decap(mlxsw_sp, ul_dev, 1498 ul_proto, ul_dip, 1499 ipip_entry)) 1500 goto out_unlock; 1501 1502 rcu_read_unlock(); 1503 1504 return NULL; 1505 1506 out_unlock: 1507 rcu_read_unlock(); 1508 return ipip_entry; 1509 } 1510 1511 static bool mlxsw_sp_netdev_ipip_type(const struct mlxsw_sp *mlxsw_sp, 1512 const struct net_device *dev, 1513 enum mlxsw_sp_ipip_type *p_type) 1514 { 1515 struct mlxsw_sp_router *router = mlxsw_sp->router; 1516 const struct mlxsw_sp_ipip_ops *ipip_ops; 1517 enum mlxsw_sp_ipip_type ipipt; 1518 1519 for (ipipt = 0; ipipt < MLXSW_SP_IPIP_TYPE_MAX; ++ipipt) { 1520 ipip_ops = router->ipip_ops_arr[ipipt]; 1521 if (dev->type == ipip_ops->dev_type) { 1522 if (p_type) 1523 *p_type = ipipt; 1524 return true; 1525 } 1526 } 1527 return false; 1528 } 1529 1530 static bool mlxsw_sp_netdev_is_ipip_ol(const struct mlxsw_sp *mlxsw_sp, 1531 const struct net_device *dev) 1532 { 1533 return mlxsw_sp_netdev_ipip_type(mlxsw_sp, dev, NULL); 1534 } 1535 1536 static struct mlxsw_sp_ipip_entry * 1537 mlxsw_sp_ipip_entry_find_by_ol_dev(struct mlxsw_sp *mlxsw_sp, 1538 const struct net_device *ol_dev) 1539 { 1540 struct mlxsw_sp_ipip_entry *ipip_entry; 1541 1542 list_for_each_entry(ipip_entry, &mlxsw_sp->router->ipip_list, 1543 ipip_list_node) 1544 if (ipip_entry->ol_dev == ol_dev) 1545 return ipip_entry; 1546 1547 return NULL; 1548 } 1549 1550 static struct mlxsw_sp_ipip_entry * 1551 mlxsw_sp_ipip_entry_find_by_ul_dev(const struct mlxsw_sp *mlxsw_sp, 1552 const struct net_device *ul_dev, 1553 struct mlxsw_sp_ipip_entry *start) 1554 { 1555 struct mlxsw_sp_ipip_entry *ipip_entry; 1556 1557 ipip_entry = list_prepare_entry(start, &mlxsw_sp->router->ipip_list, 1558 ipip_list_node); 1559 list_for_each_entry_continue(ipip_entry, &mlxsw_sp->router->ipip_list, 1560 ipip_list_node) { 1561 struct net_device *ol_dev = ipip_entry->ol_dev; 1562 struct net_device *ipip_ul_dev; 1563 1564 rcu_read_lock(); 1565 ipip_ul_dev = mlxsw_sp_ipip_netdev_ul_dev_get(ol_dev); 1566 rcu_read_unlock(); 1567 1568 if (ipip_ul_dev == ul_dev) 1569 return ipip_entry; 1570 } 1571 1572 return NULL; 1573 } 1574 1575 static bool mlxsw_sp_netdev_is_ipip_ul(struct mlxsw_sp *mlxsw_sp, 1576 const struct net_device *dev) 1577 { 1578 return mlxsw_sp_ipip_entry_find_by_ul_dev(mlxsw_sp, dev, NULL); 1579 } 1580 1581 static bool mlxsw_sp_netdevice_ipip_can_offload(struct mlxsw_sp *mlxsw_sp, 1582 const struct net_device *ol_dev, 1583 enum mlxsw_sp_ipip_type ipipt) 1584 { 1585 const struct mlxsw_sp_ipip_ops *ops 1586 = mlxsw_sp->router->ipip_ops_arr[ipipt]; 1587 1588 return ops->can_offload(mlxsw_sp, ol_dev); 1589 } 1590 1591 static int mlxsw_sp_netdevice_ipip_ol_reg_event(struct mlxsw_sp *mlxsw_sp, 1592 struct net_device *ol_dev) 1593 { 1594 enum mlxsw_sp_ipip_type ipipt = MLXSW_SP_IPIP_TYPE_MAX; 1595 struct mlxsw_sp_ipip_entry *ipip_entry; 1596 enum mlxsw_sp_l3proto ul_proto; 1597 union mlxsw_sp_l3addr saddr; 1598 u32 ul_tb_id; 1599 1600 mlxsw_sp_netdev_ipip_type(mlxsw_sp, ol_dev, &ipipt); 1601 if (mlxsw_sp_netdevice_ipip_can_offload(mlxsw_sp, ol_dev, ipipt)) { 1602 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(ol_dev); 1603 ul_proto = mlxsw_sp->router->ipip_ops_arr[ipipt]->ul_proto; 1604 saddr = mlxsw_sp_ipip_netdev_saddr(ul_proto, ol_dev); 1605 if (!mlxsw_sp_ipip_demote_tunnel_by_saddr(mlxsw_sp, ul_proto, 1606 saddr, ul_tb_id, 1607 NULL)) { 1608 ipip_entry = mlxsw_sp_ipip_entry_create(mlxsw_sp, ipipt, 1609 ol_dev); 1610 if (IS_ERR(ipip_entry)) 1611 return PTR_ERR(ipip_entry); 1612 } 1613 } 1614 1615 return 0; 1616 } 1617 1618 static void mlxsw_sp_netdevice_ipip_ol_unreg_event(struct mlxsw_sp *mlxsw_sp, 1619 struct net_device *ol_dev) 1620 { 1621 struct mlxsw_sp_ipip_entry *ipip_entry; 1622 1623 ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev); 1624 if (ipip_entry) 1625 mlxsw_sp_ipip_entry_destroy(mlxsw_sp, ipip_entry); 1626 } 1627 1628 static void 1629 mlxsw_sp_ipip_entry_ol_up_event(struct mlxsw_sp *mlxsw_sp, 1630 struct mlxsw_sp_ipip_entry *ipip_entry) 1631 { 1632 struct mlxsw_sp_fib_entry *decap_fib_entry; 1633 1634 decap_fib_entry = mlxsw_sp_ipip_entry_find_decap(mlxsw_sp, ipip_entry); 1635 if (decap_fib_entry) 1636 mlxsw_sp_ipip_entry_promote_decap(mlxsw_sp, ipip_entry, 1637 decap_fib_entry); 1638 } 1639 1640 static int 1641 mlxsw_sp_rif_ipip_lb_op(struct mlxsw_sp_rif_ipip_lb *lb_rif, u16 ul_vr_id, 1642 u16 ul_rif_id, bool enable) 1643 { 1644 struct mlxsw_sp_rif_ipip_lb_config lb_cf = lb_rif->lb_config; 1645 struct net_device *dev = mlxsw_sp_rif_dev(&lb_rif->common); 1646 enum mlxsw_reg_ritr_loopback_ipip_options ipip_options; 1647 struct mlxsw_sp_rif *rif = &lb_rif->common; 1648 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 1649 char ritr_pl[MLXSW_REG_RITR_LEN]; 1650 struct in6_addr *saddr6; 1651 u32 saddr4; 1652 1653 ipip_options = MLXSW_REG_RITR_LOOPBACK_IPIP_OPTIONS_GRE_KEY_PRESET; 1654 switch (lb_cf.ul_protocol) { 1655 case MLXSW_SP_L3_PROTO_IPV4: 1656 saddr4 = be32_to_cpu(lb_cf.saddr.addr4); 1657 mlxsw_reg_ritr_pack(ritr_pl, enable, MLXSW_REG_RITR_LOOPBACK_IF, 1658 rif->rif_index, rif->vr_id, dev->mtu); 1659 mlxsw_reg_ritr_loopback_ipip4_pack(ritr_pl, lb_cf.lb_ipipt, 1660 ipip_options, ul_vr_id, 1661 ul_rif_id, saddr4, 1662 lb_cf.okey); 1663 break; 1664 1665 case MLXSW_SP_L3_PROTO_IPV6: 1666 saddr6 = &lb_cf.saddr.addr6; 1667 mlxsw_reg_ritr_pack(ritr_pl, enable, MLXSW_REG_RITR_LOOPBACK_IF, 1668 rif->rif_index, rif->vr_id, dev->mtu); 1669 mlxsw_reg_ritr_loopback_ipip6_pack(ritr_pl, lb_cf.lb_ipipt, 1670 ipip_options, ul_vr_id, 1671 ul_rif_id, saddr6, 1672 lb_cf.okey); 1673 break; 1674 } 1675 1676 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 1677 } 1678 1679 static int mlxsw_sp_netdevice_ipip_ol_update_mtu(struct mlxsw_sp *mlxsw_sp, 1680 struct net_device *ol_dev) 1681 { 1682 struct mlxsw_sp_ipip_entry *ipip_entry; 1683 struct mlxsw_sp_rif_ipip_lb *lb_rif; 1684 int err = 0; 1685 1686 ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev); 1687 if (ipip_entry) { 1688 lb_rif = ipip_entry->ol_lb; 1689 err = mlxsw_sp_rif_ipip_lb_op(lb_rif, lb_rif->ul_vr_id, 1690 lb_rif->ul_rif_id, true); 1691 if (err) 1692 goto out; 1693 lb_rif->common.mtu = ol_dev->mtu; 1694 } 1695 1696 out: 1697 return err; 1698 } 1699 1700 static void mlxsw_sp_netdevice_ipip_ol_up_event(struct mlxsw_sp *mlxsw_sp, 1701 struct net_device *ol_dev) 1702 { 1703 struct mlxsw_sp_ipip_entry *ipip_entry; 1704 1705 ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev); 1706 if (ipip_entry) 1707 mlxsw_sp_ipip_entry_ol_up_event(mlxsw_sp, ipip_entry); 1708 } 1709 1710 static void 1711 mlxsw_sp_ipip_entry_ol_down_event(struct mlxsw_sp *mlxsw_sp, 1712 struct mlxsw_sp_ipip_entry *ipip_entry) 1713 { 1714 if (ipip_entry->decap_fib_entry) 1715 mlxsw_sp_ipip_entry_demote_decap(mlxsw_sp, ipip_entry); 1716 } 1717 1718 static void mlxsw_sp_netdevice_ipip_ol_down_event(struct mlxsw_sp *mlxsw_sp, 1719 struct net_device *ol_dev) 1720 { 1721 struct mlxsw_sp_ipip_entry *ipip_entry; 1722 1723 ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev); 1724 if (ipip_entry) 1725 mlxsw_sp_ipip_entry_ol_down_event(mlxsw_sp, ipip_entry); 1726 } 1727 1728 static void mlxsw_sp_nexthop_rif_update(struct mlxsw_sp *mlxsw_sp, 1729 struct mlxsw_sp_rif *rif); 1730 1731 static void mlxsw_sp_rif_migrate_destroy(struct mlxsw_sp *mlxsw_sp, 1732 struct mlxsw_sp_rif *old_rif, 1733 struct mlxsw_sp_rif *new_rif, 1734 bool migrate_nhs) 1735 { 1736 struct mlxsw_sp_crif *crif = old_rif->crif; 1737 struct mlxsw_sp_crif mock_crif = {}; 1738 1739 if (migrate_nhs) 1740 mlxsw_sp_nexthop_rif_update(mlxsw_sp, new_rif); 1741 1742 /* Plant a mock CRIF so that destroying the old RIF doesn't unoffload 1743 * our nexthops and IPIP tunnels, and doesn't sever the crif->rif link. 1744 */ 1745 mlxsw_sp_crif_init(&mock_crif, crif->key.dev); 1746 old_rif->crif = &mock_crif; 1747 mock_crif.rif = old_rif; 1748 mlxsw_sp_rif_destroy(old_rif); 1749 } 1750 1751 static int 1752 mlxsw_sp_ipip_entry_ol_lb_update(struct mlxsw_sp *mlxsw_sp, 1753 struct mlxsw_sp_ipip_entry *ipip_entry, 1754 bool keep_encap, 1755 struct netlink_ext_ack *extack) 1756 { 1757 struct mlxsw_sp_rif_ipip_lb *old_lb_rif = ipip_entry->ol_lb; 1758 struct mlxsw_sp_rif_ipip_lb *new_lb_rif; 1759 1760 new_lb_rif = mlxsw_sp_ipip_ol_ipip_lb_create(mlxsw_sp, 1761 ipip_entry->ipipt, 1762 ipip_entry->ol_dev, 1763 extack); 1764 if (IS_ERR(new_lb_rif)) 1765 return PTR_ERR(new_lb_rif); 1766 ipip_entry->ol_lb = new_lb_rif; 1767 1768 mlxsw_sp_rif_migrate_destroy(mlxsw_sp, &old_lb_rif->common, 1769 &new_lb_rif->common, keep_encap); 1770 return 0; 1771 } 1772 1773 /** 1774 * __mlxsw_sp_ipip_entry_update_tunnel - Update offload related to IPIP entry. 1775 * @mlxsw_sp: mlxsw_sp. 1776 * @ipip_entry: IPIP entry. 1777 * @recreate_loopback: Recreates the associated loopback RIF. 1778 * @keep_encap: Updates next hops that use the tunnel netdevice. This is only 1779 * relevant when recreate_loopback is true. 1780 * @update_nexthops: Updates next hops, keeping the current loopback RIF. This 1781 * is only relevant when recreate_loopback is false. 1782 * @extack: extack. 1783 * 1784 * Return: Non-zero value on failure. 1785 */ 1786 int __mlxsw_sp_ipip_entry_update_tunnel(struct mlxsw_sp *mlxsw_sp, 1787 struct mlxsw_sp_ipip_entry *ipip_entry, 1788 bool recreate_loopback, 1789 bool keep_encap, 1790 bool update_nexthops, 1791 struct netlink_ext_ack *extack) 1792 { 1793 int err; 1794 1795 /* RIFs can't be edited, so to update loopback, we need to destroy and 1796 * recreate it. That creates a window of opportunity where RALUE and 1797 * RATR registers end up referencing a RIF that's already gone. RATRs 1798 * are handled in mlxsw_sp_ipip_entry_ol_lb_update(), and to take care 1799 * of RALUE, demote the decap route back. 1800 */ 1801 if (ipip_entry->decap_fib_entry) 1802 mlxsw_sp_ipip_entry_demote_decap(mlxsw_sp, ipip_entry); 1803 1804 if (recreate_loopback) { 1805 err = mlxsw_sp_ipip_entry_ol_lb_update(mlxsw_sp, ipip_entry, 1806 keep_encap, extack); 1807 if (err) 1808 return err; 1809 } else if (update_nexthops) { 1810 mlxsw_sp_nexthop_rif_update(mlxsw_sp, 1811 &ipip_entry->ol_lb->common); 1812 } 1813 1814 if (ipip_entry->ol_dev->flags & IFF_UP) 1815 mlxsw_sp_ipip_entry_ol_up_event(mlxsw_sp, ipip_entry); 1816 1817 return 0; 1818 } 1819 1820 static int mlxsw_sp_netdevice_ipip_ol_vrf_event(struct mlxsw_sp *mlxsw_sp, 1821 struct net_device *ol_dev, 1822 struct netlink_ext_ack *extack) 1823 { 1824 struct mlxsw_sp_ipip_entry *ipip_entry = 1825 mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev); 1826 1827 if (!ipip_entry) 1828 return 0; 1829 1830 return __mlxsw_sp_ipip_entry_update_tunnel(mlxsw_sp, ipip_entry, 1831 true, false, false, extack); 1832 } 1833 1834 static int 1835 mlxsw_sp_netdevice_ipip_ul_vrf_event(struct mlxsw_sp *mlxsw_sp, 1836 struct mlxsw_sp_ipip_entry *ipip_entry, 1837 struct net_device *ul_dev, 1838 bool *demote_this, 1839 struct netlink_ext_ack *extack) 1840 { 1841 u32 ul_tb_id = l3mdev_fib_table(ul_dev) ? : RT_TABLE_MAIN; 1842 enum mlxsw_sp_l3proto ul_proto; 1843 union mlxsw_sp_l3addr saddr; 1844 1845 /* Moving underlay to a different VRF might cause local address 1846 * conflict, and the conflicting tunnels need to be demoted. 1847 */ 1848 ul_proto = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt]->ul_proto; 1849 saddr = mlxsw_sp_ipip_netdev_saddr(ul_proto, ipip_entry->ol_dev); 1850 if (mlxsw_sp_ipip_demote_tunnel_by_saddr(mlxsw_sp, ul_proto, 1851 saddr, ul_tb_id, 1852 ipip_entry)) { 1853 *demote_this = true; 1854 return 0; 1855 } 1856 1857 return __mlxsw_sp_ipip_entry_update_tunnel(mlxsw_sp, ipip_entry, 1858 true, true, false, extack); 1859 } 1860 1861 static int 1862 mlxsw_sp_netdevice_ipip_ul_up_event(struct mlxsw_sp *mlxsw_sp, 1863 struct mlxsw_sp_ipip_entry *ipip_entry, 1864 struct net_device *ul_dev) 1865 { 1866 return __mlxsw_sp_ipip_entry_update_tunnel(mlxsw_sp, ipip_entry, 1867 false, false, true, NULL); 1868 } 1869 1870 static int 1871 mlxsw_sp_netdevice_ipip_ul_down_event(struct mlxsw_sp *mlxsw_sp, 1872 struct mlxsw_sp_ipip_entry *ipip_entry, 1873 struct net_device *ul_dev) 1874 { 1875 /* A down underlay device causes encapsulated packets to not be 1876 * forwarded, but decap still works. So refresh next hops without 1877 * touching anything else. 1878 */ 1879 return __mlxsw_sp_ipip_entry_update_tunnel(mlxsw_sp, ipip_entry, 1880 false, false, true, NULL); 1881 } 1882 1883 static int 1884 mlxsw_sp_netdevice_ipip_ol_change_event(struct mlxsw_sp *mlxsw_sp, 1885 struct net_device *ol_dev, 1886 struct netlink_ext_ack *extack) 1887 { 1888 const struct mlxsw_sp_ipip_ops *ipip_ops; 1889 struct mlxsw_sp_ipip_entry *ipip_entry; 1890 int err; 1891 1892 ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev); 1893 if (!ipip_entry) 1894 /* A change might make a tunnel eligible for offloading, but 1895 * that is currently not implemented. What falls to slow path 1896 * stays there. 1897 */ 1898 return 0; 1899 1900 /* A change might make a tunnel not eligible for offloading. */ 1901 if (!mlxsw_sp_netdevice_ipip_can_offload(mlxsw_sp, ol_dev, 1902 ipip_entry->ipipt)) { 1903 mlxsw_sp_ipip_entry_demote_tunnel(mlxsw_sp, ipip_entry); 1904 return 0; 1905 } 1906 1907 ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt]; 1908 err = ipip_ops->ol_netdev_change(mlxsw_sp, ipip_entry, extack); 1909 return err; 1910 } 1911 1912 void mlxsw_sp_ipip_entry_demote_tunnel(struct mlxsw_sp *mlxsw_sp, 1913 struct mlxsw_sp_ipip_entry *ipip_entry) 1914 { 1915 struct net_device *ol_dev = ipip_entry->ol_dev; 1916 1917 if (ol_dev->flags & IFF_UP) 1918 mlxsw_sp_ipip_entry_ol_down_event(mlxsw_sp, ipip_entry); 1919 mlxsw_sp_ipip_entry_destroy(mlxsw_sp, ipip_entry); 1920 } 1921 1922 /* The configuration where several tunnels have the same local address in the 1923 * same underlay table needs special treatment in the HW. That is currently not 1924 * implemented in the driver. This function finds and demotes the first tunnel 1925 * with a given source address, except the one passed in the argument 1926 * `except'. 1927 */ 1928 bool 1929 mlxsw_sp_ipip_demote_tunnel_by_saddr(struct mlxsw_sp *mlxsw_sp, 1930 enum mlxsw_sp_l3proto ul_proto, 1931 union mlxsw_sp_l3addr saddr, 1932 u32 ul_tb_id, 1933 const struct mlxsw_sp_ipip_entry *except) 1934 { 1935 struct mlxsw_sp_ipip_entry *ipip_entry, *tmp; 1936 1937 list_for_each_entry_safe(ipip_entry, tmp, &mlxsw_sp->router->ipip_list, 1938 ipip_list_node) { 1939 if (ipip_entry != except && 1940 mlxsw_sp_ipip_entry_saddr_matches(mlxsw_sp, ul_proto, saddr, 1941 ul_tb_id, ipip_entry)) { 1942 mlxsw_sp_ipip_entry_demote_tunnel(mlxsw_sp, ipip_entry); 1943 return true; 1944 } 1945 } 1946 1947 return false; 1948 } 1949 1950 static void mlxsw_sp_ipip_demote_tunnel_by_ul_netdev(struct mlxsw_sp *mlxsw_sp, 1951 struct net_device *ul_dev) 1952 { 1953 struct mlxsw_sp_ipip_entry *ipip_entry, *tmp; 1954 1955 list_for_each_entry_safe(ipip_entry, tmp, &mlxsw_sp->router->ipip_list, 1956 ipip_list_node) { 1957 struct net_device *ol_dev = ipip_entry->ol_dev; 1958 struct net_device *ipip_ul_dev; 1959 1960 rcu_read_lock(); 1961 ipip_ul_dev = mlxsw_sp_ipip_netdev_ul_dev_get(ol_dev); 1962 rcu_read_unlock(); 1963 if (ipip_ul_dev == ul_dev) 1964 mlxsw_sp_ipip_entry_demote_tunnel(mlxsw_sp, ipip_entry); 1965 } 1966 } 1967 1968 static int mlxsw_sp_netdevice_ipip_ol_event(struct mlxsw_sp *mlxsw_sp, 1969 struct net_device *ol_dev, 1970 unsigned long event, 1971 struct netdev_notifier_info *info) 1972 { 1973 struct netdev_notifier_changeupper_info *chup; 1974 struct netlink_ext_ack *extack; 1975 int err = 0; 1976 1977 switch (event) { 1978 case NETDEV_REGISTER: 1979 err = mlxsw_sp_netdevice_ipip_ol_reg_event(mlxsw_sp, ol_dev); 1980 break; 1981 case NETDEV_UNREGISTER: 1982 mlxsw_sp_netdevice_ipip_ol_unreg_event(mlxsw_sp, ol_dev); 1983 break; 1984 case NETDEV_UP: 1985 mlxsw_sp_netdevice_ipip_ol_up_event(mlxsw_sp, ol_dev); 1986 break; 1987 case NETDEV_DOWN: 1988 mlxsw_sp_netdevice_ipip_ol_down_event(mlxsw_sp, ol_dev); 1989 break; 1990 case NETDEV_CHANGEUPPER: 1991 chup = container_of(info, typeof(*chup), info); 1992 extack = info->extack; 1993 if (netif_is_l3_master(chup->upper_dev)) 1994 err = mlxsw_sp_netdevice_ipip_ol_vrf_event(mlxsw_sp, 1995 ol_dev, 1996 extack); 1997 break; 1998 case NETDEV_CHANGE: 1999 extack = info->extack; 2000 err = mlxsw_sp_netdevice_ipip_ol_change_event(mlxsw_sp, 2001 ol_dev, extack); 2002 break; 2003 case NETDEV_CHANGEMTU: 2004 err = mlxsw_sp_netdevice_ipip_ol_update_mtu(mlxsw_sp, ol_dev); 2005 break; 2006 } 2007 return err; 2008 } 2009 2010 static int 2011 __mlxsw_sp_netdevice_ipip_ul_event(struct mlxsw_sp *mlxsw_sp, 2012 struct mlxsw_sp_ipip_entry *ipip_entry, 2013 struct net_device *ul_dev, 2014 bool *demote_this, 2015 unsigned long event, 2016 struct netdev_notifier_info *info) 2017 { 2018 struct netdev_notifier_changeupper_info *chup; 2019 struct netlink_ext_ack *extack; 2020 2021 switch (event) { 2022 case NETDEV_CHANGEUPPER: 2023 chup = container_of(info, typeof(*chup), info); 2024 extack = info->extack; 2025 if (netif_is_l3_master(chup->upper_dev)) 2026 return mlxsw_sp_netdevice_ipip_ul_vrf_event(mlxsw_sp, 2027 ipip_entry, 2028 ul_dev, 2029 demote_this, 2030 extack); 2031 break; 2032 2033 case NETDEV_UP: 2034 return mlxsw_sp_netdevice_ipip_ul_up_event(mlxsw_sp, ipip_entry, 2035 ul_dev); 2036 case NETDEV_DOWN: 2037 return mlxsw_sp_netdevice_ipip_ul_down_event(mlxsw_sp, 2038 ipip_entry, 2039 ul_dev); 2040 } 2041 return 0; 2042 } 2043 2044 static int 2045 mlxsw_sp_netdevice_ipip_ul_event(struct mlxsw_sp *mlxsw_sp, 2046 struct net_device *ul_dev, 2047 unsigned long event, 2048 struct netdev_notifier_info *info) 2049 { 2050 struct mlxsw_sp_ipip_entry *ipip_entry = NULL; 2051 int err; 2052 2053 while ((ipip_entry = mlxsw_sp_ipip_entry_find_by_ul_dev(mlxsw_sp, 2054 ul_dev, 2055 ipip_entry))) { 2056 struct mlxsw_sp_ipip_entry *prev; 2057 bool demote_this = false; 2058 2059 err = __mlxsw_sp_netdevice_ipip_ul_event(mlxsw_sp, ipip_entry, 2060 ul_dev, &demote_this, 2061 event, info); 2062 if (err) { 2063 mlxsw_sp_ipip_demote_tunnel_by_ul_netdev(mlxsw_sp, 2064 ul_dev); 2065 return err; 2066 } 2067 2068 if (demote_this) { 2069 if (list_is_first(&ipip_entry->ipip_list_node, 2070 &mlxsw_sp->router->ipip_list)) 2071 prev = NULL; 2072 else 2073 /* This can't be cached from previous iteration, 2074 * because that entry could be gone now. 2075 */ 2076 prev = list_prev_entry(ipip_entry, 2077 ipip_list_node); 2078 mlxsw_sp_ipip_entry_demote_tunnel(mlxsw_sp, ipip_entry); 2079 ipip_entry = prev; 2080 } 2081 } 2082 2083 return 0; 2084 } 2085 2086 int mlxsw_sp_router_nve_promote_decap(struct mlxsw_sp *mlxsw_sp, u32 ul_tb_id, 2087 enum mlxsw_sp_l3proto ul_proto, 2088 const union mlxsw_sp_l3addr *ul_sip, 2089 u32 tunnel_index) 2090 { 2091 enum mlxsw_sp_fib_entry_type type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP; 2092 struct mlxsw_sp_router *router = mlxsw_sp->router; 2093 struct mlxsw_sp_fib_entry *fib_entry; 2094 int err = 0; 2095 2096 mutex_lock(&mlxsw_sp->router->lock); 2097 2098 if (WARN_ON_ONCE(router->nve_decap_config.valid)) { 2099 err = -EINVAL; 2100 goto out; 2101 } 2102 2103 router->nve_decap_config.ul_tb_id = ul_tb_id; 2104 router->nve_decap_config.tunnel_index = tunnel_index; 2105 router->nve_decap_config.ul_proto = ul_proto; 2106 router->nve_decap_config.ul_sip = *ul_sip; 2107 router->nve_decap_config.valid = true; 2108 2109 /* It is valid to create a tunnel with a local IP and only later 2110 * assign this IP address to a local interface 2111 */ 2112 fib_entry = mlxsw_sp_router_ip2me_fib_entry_find(mlxsw_sp, ul_tb_id, 2113 ul_proto, ul_sip, 2114 type); 2115 if (!fib_entry) 2116 goto out; 2117 2118 fib_entry->decap.tunnel_index = tunnel_index; 2119 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP; 2120 2121 err = mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry); 2122 if (err) 2123 goto err_fib_entry_update; 2124 2125 goto out; 2126 2127 err_fib_entry_update: 2128 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP; 2129 mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry); 2130 out: 2131 mutex_unlock(&mlxsw_sp->router->lock); 2132 return err; 2133 } 2134 2135 void mlxsw_sp_router_nve_demote_decap(struct mlxsw_sp *mlxsw_sp, u32 ul_tb_id, 2136 enum mlxsw_sp_l3proto ul_proto, 2137 const union mlxsw_sp_l3addr *ul_sip) 2138 { 2139 enum mlxsw_sp_fib_entry_type type = MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP; 2140 struct mlxsw_sp_router *router = mlxsw_sp->router; 2141 struct mlxsw_sp_fib_entry *fib_entry; 2142 2143 mutex_lock(&mlxsw_sp->router->lock); 2144 2145 if (WARN_ON_ONCE(!router->nve_decap_config.valid)) 2146 goto out; 2147 2148 router->nve_decap_config.valid = false; 2149 2150 fib_entry = mlxsw_sp_router_ip2me_fib_entry_find(mlxsw_sp, ul_tb_id, 2151 ul_proto, ul_sip, 2152 type); 2153 if (!fib_entry) 2154 goto out; 2155 2156 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP; 2157 mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry); 2158 out: 2159 mutex_unlock(&mlxsw_sp->router->lock); 2160 } 2161 2162 static bool mlxsw_sp_router_nve_is_decap(struct mlxsw_sp *mlxsw_sp, 2163 u32 ul_tb_id, 2164 enum mlxsw_sp_l3proto ul_proto, 2165 const union mlxsw_sp_l3addr *ul_sip) 2166 { 2167 struct mlxsw_sp_router *router = mlxsw_sp->router; 2168 2169 return router->nve_decap_config.valid && 2170 router->nve_decap_config.ul_tb_id == ul_tb_id && 2171 router->nve_decap_config.ul_proto == ul_proto && 2172 !memcmp(&router->nve_decap_config.ul_sip, ul_sip, 2173 sizeof(*ul_sip)); 2174 } 2175 2176 struct mlxsw_sp_neigh_key { 2177 struct neighbour *n; 2178 }; 2179 2180 struct mlxsw_sp_neigh_entry { 2181 struct list_head rif_list_node; 2182 struct rhash_head ht_node; 2183 struct mlxsw_sp_neigh_key key; 2184 u16 rif; 2185 bool connected; 2186 unsigned char ha[ETH_ALEN]; 2187 struct list_head nexthop_list; /* list of nexthops using 2188 * this neigh entry 2189 */ 2190 struct list_head nexthop_neighs_list_node; 2191 unsigned int counter_index; 2192 bool counter_valid; 2193 }; 2194 2195 static const struct rhashtable_params mlxsw_sp_neigh_ht_params = { 2196 .key_offset = offsetof(struct mlxsw_sp_neigh_entry, key), 2197 .head_offset = offsetof(struct mlxsw_sp_neigh_entry, ht_node), 2198 .key_len = sizeof(struct mlxsw_sp_neigh_key), 2199 }; 2200 2201 struct mlxsw_sp_neigh_entry * 2202 mlxsw_sp_rif_neigh_next(struct mlxsw_sp_rif *rif, 2203 struct mlxsw_sp_neigh_entry *neigh_entry) 2204 { 2205 if (!neigh_entry) { 2206 if (list_empty(&rif->neigh_list)) 2207 return NULL; 2208 else 2209 return list_first_entry(&rif->neigh_list, 2210 typeof(*neigh_entry), 2211 rif_list_node); 2212 } 2213 if (list_is_last(&neigh_entry->rif_list_node, &rif->neigh_list)) 2214 return NULL; 2215 return list_next_entry(neigh_entry, rif_list_node); 2216 } 2217 2218 int mlxsw_sp_neigh_entry_type(struct mlxsw_sp_neigh_entry *neigh_entry) 2219 { 2220 return neigh_entry->key.n->tbl->family; 2221 } 2222 2223 unsigned char * 2224 mlxsw_sp_neigh_entry_ha(struct mlxsw_sp_neigh_entry *neigh_entry) 2225 { 2226 return neigh_entry->ha; 2227 } 2228 2229 u32 mlxsw_sp_neigh4_entry_dip(struct mlxsw_sp_neigh_entry *neigh_entry) 2230 { 2231 struct neighbour *n; 2232 2233 n = neigh_entry->key.n; 2234 return ntohl(*((__be32 *) n->primary_key)); 2235 } 2236 2237 struct in6_addr * 2238 mlxsw_sp_neigh6_entry_dip(struct mlxsw_sp_neigh_entry *neigh_entry) 2239 { 2240 struct neighbour *n; 2241 2242 n = neigh_entry->key.n; 2243 return (struct in6_addr *) &n->primary_key; 2244 } 2245 2246 int mlxsw_sp_neigh_counter_get(struct mlxsw_sp *mlxsw_sp, 2247 struct mlxsw_sp_neigh_entry *neigh_entry, 2248 u64 *p_counter) 2249 { 2250 if (!neigh_entry->counter_valid) 2251 return -EINVAL; 2252 2253 return mlxsw_sp_flow_counter_get(mlxsw_sp, neigh_entry->counter_index, 2254 false, p_counter, NULL); 2255 } 2256 2257 static struct mlxsw_sp_neigh_entry * 2258 mlxsw_sp_neigh_entry_alloc(struct mlxsw_sp *mlxsw_sp, struct neighbour *n, 2259 u16 rif) 2260 { 2261 struct mlxsw_sp_neigh_entry *neigh_entry; 2262 2263 neigh_entry = kzalloc_obj(*neigh_entry); 2264 if (!neigh_entry) 2265 return NULL; 2266 2267 neigh_hold(n); 2268 neigh_entry->key.n = n; 2269 neigh_entry->rif = rif; 2270 INIT_LIST_HEAD(&neigh_entry->nexthop_list); 2271 2272 return neigh_entry; 2273 } 2274 2275 static void mlxsw_sp_neigh_entry_free(struct mlxsw_sp_neigh_entry *neigh_entry) 2276 { 2277 neigh_release(neigh_entry->key.n); 2278 kfree(neigh_entry); 2279 } 2280 2281 static int 2282 mlxsw_sp_neigh_entry_insert(struct mlxsw_sp *mlxsw_sp, 2283 struct mlxsw_sp_neigh_entry *neigh_entry) 2284 { 2285 return rhashtable_insert_fast(&mlxsw_sp->router->neigh_ht, 2286 &neigh_entry->ht_node, 2287 mlxsw_sp_neigh_ht_params); 2288 } 2289 2290 static void 2291 mlxsw_sp_neigh_entry_remove(struct mlxsw_sp *mlxsw_sp, 2292 struct mlxsw_sp_neigh_entry *neigh_entry) 2293 { 2294 rhashtable_remove_fast(&mlxsw_sp->router->neigh_ht, 2295 &neigh_entry->ht_node, 2296 mlxsw_sp_neigh_ht_params); 2297 } 2298 2299 static bool 2300 mlxsw_sp_neigh_counter_should_alloc(struct mlxsw_sp *mlxsw_sp, 2301 struct mlxsw_sp_neigh_entry *neigh_entry) 2302 { 2303 struct devlink *devlink; 2304 const char *table_name; 2305 2306 switch (mlxsw_sp_neigh_entry_type(neigh_entry)) { 2307 case AF_INET: 2308 table_name = MLXSW_SP_DPIPE_TABLE_NAME_HOST4; 2309 break; 2310 case AF_INET6: 2311 table_name = MLXSW_SP_DPIPE_TABLE_NAME_HOST6; 2312 break; 2313 default: 2314 WARN_ON(1); 2315 return false; 2316 } 2317 2318 devlink = priv_to_devlink(mlxsw_sp->core); 2319 return devlink_dpipe_table_counter_enabled(devlink, table_name); 2320 } 2321 2322 static void 2323 mlxsw_sp_neigh_counter_alloc(struct mlxsw_sp *mlxsw_sp, 2324 struct mlxsw_sp_neigh_entry *neigh_entry) 2325 { 2326 if (!mlxsw_sp_neigh_counter_should_alloc(mlxsw_sp, neigh_entry)) 2327 return; 2328 2329 if (mlxsw_sp_flow_counter_alloc(mlxsw_sp, &neigh_entry->counter_index)) 2330 return; 2331 2332 neigh_entry->counter_valid = true; 2333 } 2334 2335 static void 2336 mlxsw_sp_neigh_counter_free(struct mlxsw_sp *mlxsw_sp, 2337 struct mlxsw_sp_neigh_entry *neigh_entry) 2338 { 2339 if (!neigh_entry->counter_valid) 2340 return; 2341 mlxsw_sp_flow_counter_free(mlxsw_sp, 2342 neigh_entry->counter_index); 2343 neigh_entry->counter_valid = false; 2344 } 2345 2346 static struct mlxsw_sp_neigh_entry * 2347 mlxsw_sp_neigh_entry_create(struct mlxsw_sp *mlxsw_sp, struct neighbour *n) 2348 { 2349 struct mlxsw_sp_neigh_entry *neigh_entry; 2350 struct mlxsw_sp_rif *rif; 2351 int err; 2352 2353 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, n->dev); 2354 if (!rif) 2355 return ERR_PTR(-EINVAL); 2356 2357 neigh_entry = mlxsw_sp_neigh_entry_alloc(mlxsw_sp, n, rif->rif_index); 2358 if (!neigh_entry) 2359 return ERR_PTR(-ENOMEM); 2360 2361 err = mlxsw_sp_neigh_entry_insert(mlxsw_sp, neigh_entry); 2362 if (err) 2363 goto err_neigh_entry_insert; 2364 2365 mlxsw_sp_neigh_counter_alloc(mlxsw_sp, neigh_entry); 2366 atomic_inc(&mlxsw_sp->router->neighs_update.neigh_count); 2367 list_add(&neigh_entry->rif_list_node, &rif->neigh_list); 2368 2369 return neigh_entry; 2370 2371 err_neigh_entry_insert: 2372 mlxsw_sp_neigh_entry_free(neigh_entry); 2373 return ERR_PTR(err); 2374 } 2375 2376 static void 2377 mlxsw_sp_neigh_entry_destroy(struct mlxsw_sp *mlxsw_sp, 2378 struct mlxsw_sp_neigh_entry *neigh_entry) 2379 { 2380 list_del(&neigh_entry->rif_list_node); 2381 atomic_dec(&mlxsw_sp->router->neighs_update.neigh_count); 2382 mlxsw_sp_neigh_counter_free(mlxsw_sp, neigh_entry); 2383 mlxsw_sp_neigh_entry_remove(mlxsw_sp, neigh_entry); 2384 mlxsw_sp_neigh_entry_free(neigh_entry); 2385 } 2386 2387 static struct mlxsw_sp_neigh_entry * 2388 mlxsw_sp_neigh_entry_lookup(struct mlxsw_sp *mlxsw_sp, struct neighbour *n) 2389 { 2390 struct mlxsw_sp_neigh_key key; 2391 2392 key.n = n; 2393 return rhashtable_lookup_fast(&mlxsw_sp->router->neigh_ht, 2394 &key, mlxsw_sp_neigh_ht_params); 2395 } 2396 2397 static void 2398 mlxsw_sp_router_neighs_update_interval_init(struct mlxsw_sp *mlxsw_sp) 2399 { 2400 unsigned long interval; 2401 2402 #if IS_ENABLED(CONFIG_IPV6) 2403 interval = min_t(unsigned long, 2404 NEIGH_VAR(&arp_tbl.parms, DELAY_PROBE_TIME), 2405 NEIGH_VAR(&nd_tbl.parms, DELAY_PROBE_TIME)); 2406 #else 2407 interval = NEIGH_VAR(&arp_tbl.parms, DELAY_PROBE_TIME); 2408 #endif 2409 mlxsw_sp->router->neighs_update.interval = jiffies_to_msecs(interval); 2410 } 2411 2412 static void mlxsw_sp_router_neigh_ent_ipv4_process(struct mlxsw_sp *mlxsw_sp, 2413 char *rauhtd_pl, 2414 int ent_index) 2415 { 2416 u64 max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); 2417 struct net_device *dev; 2418 struct neighbour *n; 2419 __be32 dipn; 2420 u32 dip; 2421 u16 rif; 2422 2423 mlxsw_reg_rauhtd_ent_ipv4_unpack(rauhtd_pl, ent_index, &rif, &dip); 2424 2425 if (WARN_ON_ONCE(rif >= max_rifs)) 2426 return; 2427 if (!mlxsw_sp->router->rifs[rif]) { 2428 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Incorrect RIF in neighbour entry\n"); 2429 return; 2430 } 2431 2432 dipn = htonl(dip); 2433 dev = mlxsw_sp_rif_dev(mlxsw_sp->router->rifs[rif]); 2434 n = neigh_lookup(&arp_tbl, &dipn, dev); 2435 if (!n) 2436 return; 2437 2438 netdev_dbg(dev, "Updating neighbour with IP=%pI4h\n", &dip); 2439 neigh_event_send(n, NULL); 2440 neigh_release(n); 2441 } 2442 2443 #if IS_ENABLED(CONFIG_IPV6) 2444 static void mlxsw_sp_router_neigh_ent_ipv6_process(struct mlxsw_sp *mlxsw_sp, 2445 char *rauhtd_pl, 2446 int rec_index) 2447 { 2448 struct net_device *dev; 2449 struct neighbour *n; 2450 struct in6_addr dip; 2451 u16 rif; 2452 2453 mlxsw_reg_rauhtd_ent_ipv6_unpack(rauhtd_pl, rec_index, &rif, 2454 (char *) &dip); 2455 2456 if (!mlxsw_sp->router->rifs[rif]) { 2457 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Incorrect RIF in neighbour entry\n"); 2458 return; 2459 } 2460 2461 dev = mlxsw_sp_rif_dev(mlxsw_sp->router->rifs[rif]); 2462 n = neigh_lookup(&nd_tbl, &dip, dev); 2463 if (!n) 2464 return; 2465 2466 netdev_dbg(dev, "Updating neighbour with IP=%pI6c\n", &dip); 2467 neigh_event_send(n, NULL); 2468 neigh_release(n); 2469 } 2470 #else 2471 static void mlxsw_sp_router_neigh_ent_ipv6_process(struct mlxsw_sp *mlxsw_sp, 2472 char *rauhtd_pl, 2473 int rec_index) 2474 { 2475 } 2476 #endif 2477 2478 static void mlxsw_sp_router_neigh_rec_ipv4_process(struct mlxsw_sp *mlxsw_sp, 2479 char *rauhtd_pl, 2480 int rec_index) 2481 { 2482 u8 num_entries; 2483 int i; 2484 2485 num_entries = mlxsw_reg_rauhtd_ipv4_rec_num_entries_get(rauhtd_pl, 2486 rec_index); 2487 /* Hardware starts counting at 0, so add 1. */ 2488 num_entries++; 2489 2490 /* Each record consists of several neighbour entries. */ 2491 for (i = 0; i < num_entries; i++) { 2492 int ent_index; 2493 2494 ent_index = rec_index * MLXSW_REG_RAUHTD_IPV4_ENT_PER_REC + i; 2495 mlxsw_sp_router_neigh_ent_ipv4_process(mlxsw_sp, rauhtd_pl, 2496 ent_index); 2497 } 2498 2499 } 2500 2501 static void mlxsw_sp_router_neigh_rec_ipv6_process(struct mlxsw_sp *mlxsw_sp, 2502 char *rauhtd_pl, 2503 int rec_index) 2504 { 2505 /* One record contains one entry. */ 2506 mlxsw_sp_router_neigh_ent_ipv6_process(mlxsw_sp, rauhtd_pl, 2507 rec_index); 2508 } 2509 2510 static void mlxsw_sp_router_neigh_rec_process(struct mlxsw_sp *mlxsw_sp, 2511 char *rauhtd_pl, int rec_index) 2512 { 2513 switch (mlxsw_reg_rauhtd_rec_type_get(rauhtd_pl, rec_index)) { 2514 case MLXSW_REG_RAUHTD_TYPE_IPV4: 2515 mlxsw_sp_router_neigh_rec_ipv4_process(mlxsw_sp, rauhtd_pl, 2516 rec_index); 2517 break; 2518 case MLXSW_REG_RAUHTD_TYPE_IPV6: 2519 mlxsw_sp_router_neigh_rec_ipv6_process(mlxsw_sp, rauhtd_pl, 2520 rec_index); 2521 break; 2522 } 2523 } 2524 2525 static bool mlxsw_sp_router_rauhtd_is_full(char *rauhtd_pl) 2526 { 2527 u8 num_rec, last_rec_index, num_entries; 2528 2529 num_rec = mlxsw_reg_rauhtd_num_rec_get(rauhtd_pl); 2530 last_rec_index = num_rec - 1; 2531 2532 if (num_rec < MLXSW_REG_RAUHTD_REC_MAX_NUM) 2533 return false; 2534 if (mlxsw_reg_rauhtd_rec_type_get(rauhtd_pl, last_rec_index) == 2535 MLXSW_REG_RAUHTD_TYPE_IPV6) 2536 return true; 2537 2538 num_entries = mlxsw_reg_rauhtd_ipv4_rec_num_entries_get(rauhtd_pl, 2539 last_rec_index); 2540 if (++num_entries == MLXSW_REG_RAUHTD_IPV4_ENT_PER_REC) 2541 return true; 2542 return false; 2543 } 2544 2545 static int 2546 __mlxsw_sp_router_neighs_update_rauhtd(struct mlxsw_sp *mlxsw_sp, 2547 char *rauhtd_pl, 2548 enum mlxsw_reg_rauhtd_type type) 2549 { 2550 int i, num_rec; 2551 int err; 2552 2553 /* Ensure the RIF we read from the device does not change mid-dump. */ 2554 mutex_lock(&mlxsw_sp->router->lock); 2555 do { 2556 mlxsw_reg_rauhtd_pack(rauhtd_pl, type); 2557 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(rauhtd), 2558 rauhtd_pl); 2559 if (err) { 2560 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Failed to dump neighbour table\n"); 2561 break; 2562 } 2563 num_rec = mlxsw_reg_rauhtd_num_rec_get(rauhtd_pl); 2564 for (i = 0; i < num_rec; i++) 2565 mlxsw_sp_router_neigh_rec_process(mlxsw_sp, rauhtd_pl, 2566 i); 2567 } while (mlxsw_sp_router_rauhtd_is_full(rauhtd_pl)); 2568 mutex_unlock(&mlxsw_sp->router->lock); 2569 2570 return err; 2571 } 2572 2573 static int mlxsw_sp_router_neighs_update_rauhtd(struct mlxsw_sp *mlxsw_sp) 2574 { 2575 enum mlxsw_reg_rauhtd_type type; 2576 char *rauhtd_pl; 2577 int err; 2578 2579 if (!atomic_read(&mlxsw_sp->router->neighs_update.neigh_count)) 2580 return 0; 2581 2582 rauhtd_pl = kmalloc(MLXSW_REG_RAUHTD_LEN, GFP_KERNEL); 2583 if (!rauhtd_pl) 2584 return -ENOMEM; 2585 2586 type = MLXSW_REG_RAUHTD_TYPE_IPV4; 2587 err = __mlxsw_sp_router_neighs_update_rauhtd(mlxsw_sp, rauhtd_pl, type); 2588 if (err) 2589 goto out; 2590 2591 type = MLXSW_REG_RAUHTD_TYPE_IPV6; 2592 err = __mlxsw_sp_router_neighs_update_rauhtd(mlxsw_sp, rauhtd_pl, type); 2593 out: 2594 kfree(rauhtd_pl); 2595 return err; 2596 } 2597 2598 static void mlxsw_sp_router_neighs_update_nh(struct mlxsw_sp *mlxsw_sp) 2599 { 2600 struct mlxsw_sp_neigh_entry *neigh_entry; 2601 2602 mutex_lock(&mlxsw_sp->router->lock); 2603 list_for_each_entry(neigh_entry, &mlxsw_sp->router->nexthop_neighs_list, 2604 nexthop_neighs_list_node) 2605 /* If this neigh have nexthops, make the kernel think this neigh 2606 * is active regardless of the traffic. 2607 */ 2608 neigh_event_send(neigh_entry->key.n, NULL); 2609 mutex_unlock(&mlxsw_sp->router->lock); 2610 } 2611 2612 static void 2613 mlxsw_sp_router_neighs_update_work_schedule(struct mlxsw_sp *mlxsw_sp) 2614 { 2615 unsigned long interval = mlxsw_sp->router->neighs_update.interval; 2616 2617 mlxsw_core_schedule_dw(&mlxsw_sp->router->neighs_update.dw, 2618 msecs_to_jiffies(interval)); 2619 } 2620 2621 static void mlxsw_sp_router_neighs_update_work(struct work_struct *work) 2622 { 2623 struct mlxsw_sp_router *router; 2624 int err; 2625 2626 router = container_of(work, struct mlxsw_sp_router, 2627 neighs_update.dw.work); 2628 err = mlxsw_sp_router_neighs_update_rauhtd(router->mlxsw_sp); 2629 if (err) 2630 dev_err(router->mlxsw_sp->bus_info->dev, "Could not update kernel for neigh activity"); 2631 2632 mlxsw_sp_router_neighs_update_nh(router->mlxsw_sp); 2633 2634 mlxsw_sp_router_neighs_update_work_schedule(router->mlxsw_sp); 2635 } 2636 2637 static void mlxsw_sp_router_probe_unresolved_nexthops(struct work_struct *work) 2638 { 2639 struct mlxsw_sp_neigh_entry *neigh_entry; 2640 struct mlxsw_sp_router *router; 2641 2642 router = container_of(work, struct mlxsw_sp_router, 2643 nexthop_probe_dw.work); 2644 /* Iterate over nexthop neighbours, find those who are unresolved and 2645 * send arp on them. This solves the chicken-egg problem when 2646 * the nexthop wouldn't get offloaded until the neighbor is resolved 2647 * but it wouldn't get resolved ever in case traffic is flowing in HW 2648 * using different nexthop. 2649 */ 2650 mutex_lock(&router->lock); 2651 list_for_each_entry(neigh_entry, &router->nexthop_neighs_list, 2652 nexthop_neighs_list_node) 2653 if (!neigh_entry->connected) 2654 neigh_event_send(neigh_entry->key.n, NULL); 2655 mutex_unlock(&router->lock); 2656 2657 mlxsw_core_schedule_dw(&router->nexthop_probe_dw, 2658 MLXSW_SP_UNRESOLVED_NH_PROBE_INTERVAL); 2659 } 2660 2661 static void 2662 mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp *mlxsw_sp, 2663 struct mlxsw_sp_neigh_entry *neigh_entry, 2664 bool removing, bool dead); 2665 2666 static enum mlxsw_reg_rauht_op mlxsw_sp_rauht_op(bool adding) 2667 { 2668 return adding ? MLXSW_REG_RAUHT_OP_WRITE_ADD : 2669 MLXSW_REG_RAUHT_OP_WRITE_DELETE; 2670 } 2671 2672 static int 2673 mlxsw_sp_router_neigh_entry_op4(struct mlxsw_sp *mlxsw_sp, 2674 struct mlxsw_sp_neigh_entry *neigh_entry, 2675 enum mlxsw_reg_rauht_op op) 2676 { 2677 struct neighbour *n = neigh_entry->key.n; 2678 u32 dip = ntohl(*((__be32 *) n->primary_key)); 2679 char rauht_pl[MLXSW_REG_RAUHT_LEN]; 2680 2681 mlxsw_reg_rauht_pack4(rauht_pl, op, neigh_entry->rif, neigh_entry->ha, 2682 dip); 2683 if (neigh_entry->counter_valid) 2684 mlxsw_reg_rauht_pack_counter(rauht_pl, 2685 neigh_entry->counter_index); 2686 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rauht), rauht_pl); 2687 } 2688 2689 static int 2690 mlxsw_sp_router_neigh_entry_op6(struct mlxsw_sp *mlxsw_sp, 2691 struct mlxsw_sp_neigh_entry *neigh_entry, 2692 enum mlxsw_reg_rauht_op op) 2693 { 2694 struct neighbour *n = neigh_entry->key.n; 2695 char rauht_pl[MLXSW_REG_RAUHT_LEN]; 2696 const char *dip = n->primary_key; 2697 2698 mlxsw_reg_rauht_pack6(rauht_pl, op, neigh_entry->rif, neigh_entry->ha, 2699 dip); 2700 if (neigh_entry->counter_valid) 2701 mlxsw_reg_rauht_pack_counter(rauht_pl, 2702 neigh_entry->counter_index); 2703 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rauht), rauht_pl); 2704 } 2705 2706 bool mlxsw_sp_neigh_ipv6_ignore(struct mlxsw_sp_neigh_entry *neigh_entry) 2707 { 2708 struct neighbour *n = neigh_entry->key.n; 2709 2710 /* Packets with a link-local destination address are trapped 2711 * after LPM lookup and never reach the neighbour table, so 2712 * there is no need to program such neighbours to the device. 2713 */ 2714 if (ipv6_addr_type((struct in6_addr *) &n->primary_key) & 2715 IPV6_ADDR_LINKLOCAL) 2716 return true; 2717 return false; 2718 } 2719 2720 static void 2721 mlxsw_sp_neigh_entry_update(struct mlxsw_sp *mlxsw_sp, 2722 struct mlxsw_sp_neigh_entry *neigh_entry, 2723 bool adding) 2724 { 2725 enum mlxsw_reg_rauht_op op = mlxsw_sp_rauht_op(adding); 2726 int err; 2727 2728 if (!adding && !neigh_entry->connected) 2729 return; 2730 neigh_entry->connected = adding; 2731 if (neigh_entry->key.n->tbl->family == AF_INET) { 2732 err = mlxsw_sp_router_neigh_entry_op4(mlxsw_sp, neigh_entry, 2733 op); 2734 if (err) 2735 return; 2736 } else if (neigh_entry->key.n->tbl->family == AF_INET6) { 2737 if (mlxsw_sp_neigh_ipv6_ignore(neigh_entry)) 2738 return; 2739 err = mlxsw_sp_router_neigh_entry_op6(mlxsw_sp, neigh_entry, 2740 op); 2741 if (err) 2742 return; 2743 } else { 2744 WARN_ON_ONCE(1); 2745 return; 2746 } 2747 2748 if (adding) 2749 neigh_entry->key.n->flags |= NTF_OFFLOADED; 2750 else 2751 neigh_entry->key.n->flags &= ~NTF_OFFLOADED; 2752 } 2753 2754 void 2755 mlxsw_sp_neigh_entry_counter_update(struct mlxsw_sp *mlxsw_sp, 2756 struct mlxsw_sp_neigh_entry *neigh_entry, 2757 bool adding) 2758 { 2759 if (adding) 2760 mlxsw_sp_neigh_counter_alloc(mlxsw_sp, neigh_entry); 2761 else 2762 mlxsw_sp_neigh_counter_free(mlxsw_sp, neigh_entry); 2763 mlxsw_sp_neigh_entry_update(mlxsw_sp, neigh_entry, true); 2764 } 2765 2766 struct mlxsw_sp_netevent_work { 2767 struct work_struct work; 2768 struct mlxsw_sp *mlxsw_sp; 2769 struct neighbour *n; 2770 }; 2771 2772 static void mlxsw_sp_router_neigh_event_work(struct work_struct *work) 2773 { 2774 struct mlxsw_sp_netevent_work *net_work = 2775 container_of(work, struct mlxsw_sp_netevent_work, work); 2776 struct mlxsw_sp *mlxsw_sp = net_work->mlxsw_sp; 2777 struct mlxsw_sp_neigh_entry *neigh_entry; 2778 struct neighbour *n = net_work->n; 2779 unsigned char ha[ETH_ALEN]; 2780 bool entry_connected; 2781 u8 nud_state, dead; 2782 2783 /* If these parameters are changed after we release the lock, 2784 * then we are guaranteed to receive another event letting us 2785 * know about it. 2786 */ 2787 read_lock_bh(&n->lock); 2788 memcpy(ha, n->ha, ETH_ALEN); 2789 nud_state = n->nud_state; 2790 dead = n->dead; 2791 read_unlock_bh(&n->lock); 2792 2793 mutex_lock(&mlxsw_sp->router->lock); 2794 mlxsw_sp_span_respin(mlxsw_sp); 2795 2796 entry_connected = nud_state & NUD_VALID && !dead; 2797 neigh_entry = mlxsw_sp_neigh_entry_lookup(mlxsw_sp, n); 2798 if (!entry_connected && !neigh_entry) 2799 goto out; 2800 if (!neigh_entry) { 2801 neigh_entry = mlxsw_sp_neigh_entry_create(mlxsw_sp, n); 2802 if (IS_ERR(neigh_entry)) 2803 goto out; 2804 } 2805 2806 if (neigh_entry->connected && entry_connected && 2807 !memcmp(neigh_entry->ha, ha, ETH_ALEN)) 2808 goto out; 2809 2810 memcpy(neigh_entry->ha, ha, ETH_ALEN); 2811 mlxsw_sp_neigh_entry_update(mlxsw_sp, neigh_entry, entry_connected); 2812 mlxsw_sp_nexthop_neigh_update(mlxsw_sp, neigh_entry, !entry_connected, 2813 dead); 2814 2815 if (!neigh_entry->connected && list_empty(&neigh_entry->nexthop_list)) 2816 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry); 2817 2818 out: 2819 mutex_unlock(&mlxsw_sp->router->lock); 2820 neigh_release(n); 2821 kfree(net_work); 2822 } 2823 2824 static int mlxsw_sp_mp_hash_init(struct mlxsw_sp *mlxsw_sp); 2825 2826 static void mlxsw_sp_router_mp_hash_event_work(struct work_struct *work) 2827 { 2828 struct mlxsw_sp_netevent_work *net_work = 2829 container_of(work, struct mlxsw_sp_netevent_work, work); 2830 struct mlxsw_sp *mlxsw_sp = net_work->mlxsw_sp; 2831 2832 mlxsw_sp_mp_hash_init(mlxsw_sp); 2833 kfree(net_work); 2834 } 2835 2836 static int __mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp); 2837 2838 static void mlxsw_sp_router_update_priority_work(struct work_struct *work) 2839 { 2840 struct mlxsw_sp_netevent_work *net_work = 2841 container_of(work, struct mlxsw_sp_netevent_work, work); 2842 struct mlxsw_sp *mlxsw_sp = net_work->mlxsw_sp; 2843 2844 __mlxsw_sp_router_init(mlxsw_sp); 2845 kfree(net_work); 2846 } 2847 2848 static int mlxsw_sp_router_schedule_work(struct net *net, 2849 struct mlxsw_sp_router *router, 2850 struct neighbour *n, 2851 void (*cb)(struct work_struct *)) 2852 { 2853 struct mlxsw_sp_netevent_work *net_work; 2854 2855 if (!net_eq(net, mlxsw_sp_net(router->mlxsw_sp))) 2856 return NOTIFY_DONE; 2857 2858 net_work = kzalloc_obj(*net_work, GFP_ATOMIC); 2859 if (!net_work) 2860 return NOTIFY_BAD; 2861 2862 /* Take a reference to ensure the neighbour won't be destructed until 2863 * we drop the reference in the work item. 2864 */ 2865 neigh_clone(n); 2866 2867 INIT_WORK(&net_work->work, cb); 2868 net_work->mlxsw_sp = router->mlxsw_sp; 2869 net_work->n = n; 2870 mlxsw_core_schedule_work(&net_work->work); 2871 return NOTIFY_DONE; 2872 } 2873 2874 static bool mlxsw_sp_dev_lower_is_port(struct net_device *dev) 2875 { 2876 struct mlxsw_sp_port *mlxsw_sp_port; 2877 2878 rcu_read_lock(); 2879 mlxsw_sp_port = mlxsw_sp_port_dev_lower_find_rcu(dev); 2880 rcu_read_unlock(); 2881 return !!mlxsw_sp_port; 2882 } 2883 2884 static int mlxsw_sp_router_schedule_neigh_work(struct mlxsw_sp_router *router, 2885 struct neighbour *n) 2886 { 2887 struct net *net; 2888 2889 net = neigh_parms_net(n->parms); 2890 return mlxsw_sp_router_schedule_work(net, router, n, 2891 mlxsw_sp_router_neigh_event_work); 2892 } 2893 2894 static int mlxsw_sp_router_netevent_event(struct notifier_block *nb, 2895 unsigned long event, void *ptr) 2896 { 2897 struct mlxsw_sp_router *router; 2898 unsigned long interval; 2899 struct neigh_parms *p; 2900 struct neighbour *n; 2901 2902 router = container_of(nb, struct mlxsw_sp_router, netevent_nb); 2903 2904 switch (event) { 2905 case NETEVENT_DELAY_PROBE_TIME_UPDATE: 2906 p = ptr; 2907 2908 /* We don't care about changes in the default table. */ 2909 if (!p->dev || (p->tbl->family != AF_INET && 2910 p->tbl->family != AF_INET6)) 2911 return NOTIFY_DONE; 2912 2913 /* We are in atomic context and can't take RTNL mutex, 2914 * so use RCU variant to walk the device chain. 2915 */ 2916 if (!mlxsw_sp_dev_lower_is_port(p->dev)) 2917 return NOTIFY_DONE; 2918 2919 interval = jiffies_to_msecs(NEIGH_VAR(p, DELAY_PROBE_TIME)); 2920 router->neighs_update.interval = interval; 2921 break; 2922 case NETEVENT_NEIGH_UPDATE: 2923 n = ptr; 2924 2925 if (n->tbl->family != AF_INET && n->tbl->family != AF_INET6) 2926 return NOTIFY_DONE; 2927 2928 if (!mlxsw_sp_dev_lower_is_port(n->dev)) 2929 return NOTIFY_DONE; 2930 2931 return mlxsw_sp_router_schedule_neigh_work(router, n); 2932 2933 case NETEVENT_IPV4_MPATH_HASH_UPDATE: 2934 case NETEVENT_IPV6_MPATH_HASH_UPDATE: 2935 return mlxsw_sp_router_schedule_work(ptr, router, NULL, 2936 mlxsw_sp_router_mp_hash_event_work); 2937 2938 case NETEVENT_IPV4_FWD_UPDATE_PRIORITY_UPDATE: 2939 return mlxsw_sp_router_schedule_work(ptr, router, NULL, 2940 mlxsw_sp_router_update_priority_work); 2941 } 2942 2943 return NOTIFY_DONE; 2944 } 2945 2946 static int mlxsw_sp_neigh_init(struct mlxsw_sp *mlxsw_sp) 2947 { 2948 int err; 2949 2950 err = rhashtable_init(&mlxsw_sp->router->neigh_ht, 2951 &mlxsw_sp_neigh_ht_params); 2952 if (err) 2953 return err; 2954 2955 /* Initialize the polling interval according to the default 2956 * table. 2957 */ 2958 mlxsw_sp_router_neighs_update_interval_init(mlxsw_sp); 2959 2960 /* Create the delayed works for the activity_update */ 2961 INIT_DELAYED_WORK(&mlxsw_sp->router->neighs_update.dw, 2962 mlxsw_sp_router_neighs_update_work); 2963 INIT_DELAYED_WORK(&mlxsw_sp->router->nexthop_probe_dw, 2964 mlxsw_sp_router_probe_unresolved_nexthops); 2965 atomic_set(&mlxsw_sp->router->neighs_update.neigh_count, 0); 2966 mlxsw_core_schedule_dw(&mlxsw_sp->router->neighs_update.dw, 0); 2967 mlxsw_core_schedule_dw(&mlxsw_sp->router->nexthop_probe_dw, 0); 2968 return 0; 2969 } 2970 2971 static void mlxsw_sp_neigh_fini(struct mlxsw_sp *mlxsw_sp) 2972 { 2973 cancel_delayed_work_sync(&mlxsw_sp->router->neighs_update.dw); 2974 cancel_delayed_work_sync(&mlxsw_sp->router->nexthop_probe_dw); 2975 rhashtable_destroy(&mlxsw_sp->router->neigh_ht); 2976 } 2977 2978 static void mlxsw_sp_neigh_rif_gone_sync(struct mlxsw_sp *mlxsw_sp, 2979 struct mlxsw_sp_rif *rif) 2980 { 2981 struct mlxsw_sp_neigh_entry *neigh_entry, *tmp; 2982 2983 list_for_each_entry_safe(neigh_entry, tmp, &rif->neigh_list, 2984 rif_list_node) { 2985 mlxsw_sp_neigh_entry_update(mlxsw_sp, neigh_entry, false); 2986 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry); 2987 } 2988 } 2989 2990 struct mlxsw_sp_neigh_rif_made_sync { 2991 struct mlxsw_sp *mlxsw_sp; 2992 struct mlxsw_sp_rif *rif; 2993 int err; 2994 }; 2995 2996 static void mlxsw_sp_neigh_rif_made_sync_each(struct neighbour *n, void *data) 2997 { 2998 struct mlxsw_sp_neigh_rif_made_sync *rms = data; 2999 int rc; 3000 3001 if (rms->err) 3002 return; 3003 if (n->dev != mlxsw_sp_rif_dev(rms->rif)) 3004 return; 3005 rc = mlxsw_sp_router_schedule_neigh_work(rms->mlxsw_sp->router, n); 3006 if (rc != NOTIFY_DONE) 3007 rms->err = -ENOMEM; 3008 } 3009 3010 static int mlxsw_sp_neigh_rif_made_sync(struct mlxsw_sp *mlxsw_sp, 3011 struct mlxsw_sp_rif *rif) 3012 { 3013 struct mlxsw_sp_neigh_rif_made_sync rms = { 3014 .mlxsw_sp = mlxsw_sp, 3015 .rif = rif, 3016 }; 3017 3018 if (!mlxsw_sp_dev_lower_is_port(mlxsw_sp_rif_dev(rif))) 3019 return 0; 3020 3021 neigh_for_each(&arp_tbl, mlxsw_sp_neigh_rif_made_sync_each, &rms); 3022 if (rms.err) 3023 goto err_arp; 3024 3025 #if IS_ENABLED(CONFIG_IPV6) 3026 neigh_for_each(&nd_tbl, mlxsw_sp_neigh_rif_made_sync_each, &rms); 3027 #endif 3028 if (rms.err) 3029 goto err_nd; 3030 3031 return 0; 3032 3033 err_nd: 3034 err_arp: 3035 mlxsw_sp_neigh_rif_gone_sync(mlxsw_sp, rif); 3036 return rms.err; 3037 } 3038 3039 enum mlxsw_sp_nexthop_type { 3040 MLXSW_SP_NEXTHOP_TYPE_ETH, 3041 MLXSW_SP_NEXTHOP_TYPE_IPIP, 3042 }; 3043 3044 enum mlxsw_sp_nexthop_action { 3045 /* Nexthop forwards packets to an egress RIF */ 3046 MLXSW_SP_NEXTHOP_ACTION_FORWARD, 3047 /* Nexthop discards packets */ 3048 MLXSW_SP_NEXTHOP_ACTION_DISCARD, 3049 /* Nexthop traps packets */ 3050 MLXSW_SP_NEXTHOP_ACTION_TRAP, 3051 }; 3052 3053 struct mlxsw_sp_nexthop_key { 3054 struct fib_nh *fib_nh; 3055 }; 3056 3057 struct mlxsw_sp_nexthop_counter; 3058 3059 struct mlxsw_sp_nexthop { 3060 struct list_head neigh_list_node; /* member of neigh entry list */ 3061 struct list_head crif_list_node; 3062 struct list_head router_list_node; 3063 struct mlxsw_sp_nexthop_group_info *nhgi; /* pointer back to the group 3064 * this nexthop belongs to 3065 */ 3066 struct rhash_head ht_node; 3067 struct neigh_table *neigh_tbl; 3068 struct mlxsw_sp_nexthop_key key; 3069 unsigned char gw_addr[sizeof(struct in6_addr)]; 3070 int ifindex; 3071 int nh_weight; 3072 int norm_nh_weight; 3073 int num_adj_entries; 3074 struct mlxsw_sp_crif *crif; 3075 u8 should_offload:1, /* set indicates this nexthop should be written 3076 * to the adjacency table. 3077 */ 3078 offloaded:1, /* set indicates this nexthop was written to the 3079 * adjacency table. 3080 */ 3081 update:1; /* set indicates this nexthop should be updated in the 3082 * adjacency table (f.e., its MAC changed). 3083 */ 3084 enum mlxsw_sp_nexthop_action action; 3085 enum mlxsw_sp_nexthop_type type; 3086 union { 3087 struct mlxsw_sp_neigh_entry *neigh_entry; 3088 struct mlxsw_sp_ipip_entry *ipip_entry; 3089 }; 3090 struct mlxsw_sp_nexthop_counter *counter; 3091 u32 id; /* NH ID for members of a NH object group. */ 3092 }; 3093 3094 static struct net_device * 3095 mlxsw_sp_nexthop_dev(const struct mlxsw_sp_nexthop *nh) 3096 { 3097 if (!nh->crif) 3098 return NULL; 3099 return nh->crif->key.dev; 3100 } 3101 3102 enum mlxsw_sp_nexthop_group_type { 3103 MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4, 3104 MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6, 3105 MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ, 3106 }; 3107 3108 struct mlxsw_sp_nexthop_group_info { 3109 struct mlxsw_sp_nexthop_group *nh_grp; 3110 u32 adj_index; 3111 u16 ecmp_size; 3112 u16 count; 3113 int sum_norm_weight; 3114 u8 adj_index_valid:1, 3115 gateway:1, /* routes using the group use a gateway */ 3116 is_resilient:1, 3117 hw_stats:1; 3118 struct list_head list; /* member in nh_res_grp_list */ 3119 struct xarray nexthop_counters; 3120 struct mlxsw_sp_nexthop nexthops[] __counted_by(count); 3121 }; 3122 3123 static struct mlxsw_sp_rif * 3124 mlxsw_sp_nhgi_rif(const struct mlxsw_sp_nexthop_group_info *nhgi) 3125 { 3126 struct mlxsw_sp_crif *crif = nhgi->nexthops[0].crif; 3127 3128 if (!crif) 3129 return NULL; 3130 return crif->rif; 3131 } 3132 3133 struct mlxsw_sp_nexthop_group_vr_key { 3134 u16 vr_id; 3135 enum mlxsw_sp_l3proto proto; 3136 }; 3137 3138 struct mlxsw_sp_nexthop_group_vr_entry { 3139 struct list_head list; /* member in vr_list */ 3140 struct rhash_head ht_node; /* member in vr_ht */ 3141 refcount_t ref_count; 3142 struct mlxsw_sp_nexthop_group_vr_key key; 3143 }; 3144 3145 struct mlxsw_sp_nexthop_group { 3146 struct rhash_head ht_node; 3147 struct list_head fib_list; /* list of fib entries that use this group */ 3148 union { 3149 struct { 3150 struct fib_info *fi; 3151 } ipv4; 3152 struct { 3153 u32 id; 3154 } obj; 3155 }; 3156 struct mlxsw_sp_nexthop_group_info *nhgi; 3157 struct list_head vr_list; 3158 struct rhashtable vr_ht; 3159 enum mlxsw_sp_nexthop_group_type type; 3160 bool can_destroy; 3161 }; 3162 3163 struct mlxsw_sp_nexthop_counter { 3164 unsigned int counter_index; 3165 refcount_t ref_count; 3166 }; 3167 3168 static struct mlxsw_sp_nexthop_counter * 3169 mlxsw_sp_nexthop_counter_alloc(struct mlxsw_sp *mlxsw_sp) 3170 { 3171 struct mlxsw_sp_nexthop_counter *nhct; 3172 int err; 3173 3174 nhct = kzalloc_obj(*nhct); 3175 if (!nhct) 3176 return ERR_PTR(-ENOMEM); 3177 3178 err = mlxsw_sp_flow_counter_alloc(mlxsw_sp, &nhct->counter_index); 3179 if (err) 3180 goto err_counter_alloc; 3181 3182 refcount_set(&nhct->ref_count, 1); 3183 return nhct; 3184 3185 err_counter_alloc: 3186 kfree(nhct); 3187 return ERR_PTR(err); 3188 } 3189 3190 static void 3191 mlxsw_sp_nexthop_counter_free(struct mlxsw_sp *mlxsw_sp, 3192 struct mlxsw_sp_nexthop_counter *nhct) 3193 { 3194 mlxsw_sp_flow_counter_free(mlxsw_sp, nhct->counter_index); 3195 kfree(nhct); 3196 } 3197 3198 static struct mlxsw_sp_nexthop_counter * 3199 mlxsw_sp_nexthop_sh_counter_get(struct mlxsw_sp *mlxsw_sp, 3200 struct mlxsw_sp_nexthop *nh) 3201 { 3202 struct mlxsw_sp_nexthop_group *nh_grp = nh->nhgi->nh_grp; 3203 struct mlxsw_sp_nexthop_counter *nhct; 3204 int err; 3205 3206 nhct = xa_load(&nh_grp->nhgi->nexthop_counters, nh->id); 3207 if (nhct) { 3208 refcount_inc(&nhct->ref_count); 3209 return nhct; 3210 } 3211 3212 nhct = mlxsw_sp_nexthop_counter_alloc(mlxsw_sp); 3213 if (IS_ERR(nhct)) 3214 return nhct; 3215 3216 err = xa_err(xa_store(&nh_grp->nhgi->nexthop_counters, nh->id, nhct, 3217 GFP_KERNEL)); 3218 if (err) 3219 goto err_store; 3220 3221 return nhct; 3222 3223 err_store: 3224 mlxsw_sp_nexthop_counter_free(mlxsw_sp, nhct); 3225 return ERR_PTR(err); 3226 } 3227 3228 static void mlxsw_sp_nexthop_sh_counter_put(struct mlxsw_sp *mlxsw_sp, 3229 struct mlxsw_sp_nexthop *nh) 3230 { 3231 struct mlxsw_sp_nexthop_group *nh_grp = nh->nhgi->nh_grp; 3232 struct mlxsw_sp_nexthop_counter *nhct; 3233 3234 nhct = xa_load(&nh_grp->nhgi->nexthop_counters, nh->id); 3235 if (WARN_ON(!nhct)) 3236 return; 3237 3238 if (!refcount_dec_and_test(&nhct->ref_count)) 3239 return; 3240 3241 xa_erase(&nh_grp->nhgi->nexthop_counters, nh->id); 3242 mlxsw_sp_nexthop_counter_free(mlxsw_sp, nhct); 3243 } 3244 3245 int mlxsw_sp_nexthop_counter_enable(struct mlxsw_sp *mlxsw_sp, 3246 struct mlxsw_sp_nexthop *nh) 3247 { 3248 const char *table_adj = MLXSW_SP_DPIPE_TABLE_NAME_ADJ; 3249 struct mlxsw_sp_nexthop_counter *nhct; 3250 struct devlink *devlink; 3251 bool dpipe_stats; 3252 3253 if (nh->counter) 3254 return 0; 3255 3256 devlink = priv_to_devlink(mlxsw_sp->core); 3257 dpipe_stats = devlink_dpipe_table_counter_enabled(devlink, table_adj); 3258 if (!(nh->nhgi->hw_stats || dpipe_stats)) 3259 return 0; 3260 3261 if (nh->id) 3262 nhct = mlxsw_sp_nexthop_sh_counter_get(mlxsw_sp, nh); 3263 else 3264 nhct = mlxsw_sp_nexthop_counter_alloc(mlxsw_sp); 3265 if (IS_ERR(nhct)) 3266 return PTR_ERR(nhct); 3267 3268 nh->counter = nhct; 3269 return 0; 3270 } 3271 3272 void mlxsw_sp_nexthop_counter_disable(struct mlxsw_sp *mlxsw_sp, 3273 struct mlxsw_sp_nexthop *nh) 3274 { 3275 if (!nh->counter) 3276 return; 3277 3278 if (nh->id) 3279 mlxsw_sp_nexthop_sh_counter_put(mlxsw_sp, nh); 3280 else 3281 mlxsw_sp_nexthop_counter_free(mlxsw_sp, nh->counter); 3282 nh->counter = NULL; 3283 } 3284 3285 static int mlxsw_sp_nexthop_counter_update(struct mlxsw_sp *mlxsw_sp, 3286 struct mlxsw_sp_nexthop *nh) 3287 { 3288 if (nh->nhgi->hw_stats) 3289 return mlxsw_sp_nexthop_counter_enable(mlxsw_sp, nh); 3290 mlxsw_sp_nexthop_counter_disable(mlxsw_sp, nh); 3291 return 0; 3292 } 3293 3294 int mlxsw_sp_nexthop_counter_get(struct mlxsw_sp *mlxsw_sp, 3295 struct mlxsw_sp_nexthop *nh, u64 *p_counter) 3296 { 3297 if (!nh->counter) 3298 return -EINVAL; 3299 3300 return mlxsw_sp_flow_counter_get(mlxsw_sp, nh->counter->counter_index, 3301 true, p_counter, NULL); 3302 } 3303 3304 struct mlxsw_sp_nexthop *mlxsw_sp_nexthop_next(struct mlxsw_sp_router *router, 3305 struct mlxsw_sp_nexthop *nh) 3306 { 3307 if (!nh) { 3308 if (list_empty(&router->nexthop_list)) 3309 return NULL; 3310 else 3311 return list_first_entry(&router->nexthop_list, 3312 typeof(*nh), router_list_node); 3313 } 3314 if (list_is_last(&nh->router_list_node, &router->nexthop_list)) 3315 return NULL; 3316 return list_next_entry(nh, router_list_node); 3317 } 3318 3319 bool mlxsw_sp_nexthop_is_forward(const struct mlxsw_sp_nexthop *nh) 3320 { 3321 return nh->offloaded && nh->action == MLXSW_SP_NEXTHOP_ACTION_FORWARD; 3322 } 3323 3324 unsigned char *mlxsw_sp_nexthop_ha(struct mlxsw_sp_nexthop *nh) 3325 { 3326 if (nh->type != MLXSW_SP_NEXTHOP_TYPE_ETH || 3327 !mlxsw_sp_nexthop_is_forward(nh)) 3328 return NULL; 3329 return nh->neigh_entry->ha; 3330 } 3331 3332 int mlxsw_sp_nexthop_indexes(struct mlxsw_sp_nexthop *nh, u32 *p_adj_index, 3333 u32 *p_adj_size, u32 *p_adj_hash_index) 3334 { 3335 struct mlxsw_sp_nexthop_group_info *nhgi = nh->nhgi; 3336 u32 adj_hash_index = 0; 3337 int i; 3338 3339 if (!nh->offloaded || !nhgi->adj_index_valid) 3340 return -EINVAL; 3341 3342 *p_adj_index = nhgi->adj_index; 3343 *p_adj_size = nhgi->ecmp_size; 3344 3345 for (i = 0; i < nhgi->count; i++) { 3346 struct mlxsw_sp_nexthop *nh_iter = &nhgi->nexthops[i]; 3347 3348 if (nh_iter == nh) 3349 break; 3350 if (nh_iter->offloaded) 3351 adj_hash_index += nh_iter->num_adj_entries; 3352 } 3353 3354 *p_adj_hash_index = adj_hash_index; 3355 return 0; 3356 } 3357 3358 struct mlxsw_sp_rif *mlxsw_sp_nexthop_rif(struct mlxsw_sp_nexthop *nh) 3359 { 3360 if (WARN_ON(!nh->crif)) 3361 return NULL; 3362 return nh->crif->rif; 3363 } 3364 3365 bool mlxsw_sp_nexthop_group_has_ipip(struct mlxsw_sp_nexthop *nh) 3366 { 3367 struct mlxsw_sp_nexthop_group_info *nhgi = nh->nhgi; 3368 int i; 3369 3370 for (i = 0; i < nhgi->count; i++) { 3371 struct mlxsw_sp_nexthop *nh_iter = &nhgi->nexthops[i]; 3372 3373 if (nh_iter->type == MLXSW_SP_NEXTHOP_TYPE_IPIP) 3374 return true; 3375 } 3376 return false; 3377 } 3378 3379 static const struct rhashtable_params mlxsw_sp_nexthop_group_vr_ht_params = { 3380 .key_offset = offsetof(struct mlxsw_sp_nexthop_group_vr_entry, key), 3381 .head_offset = offsetof(struct mlxsw_sp_nexthop_group_vr_entry, ht_node), 3382 .key_len = sizeof(struct mlxsw_sp_nexthop_group_vr_key), 3383 .automatic_shrinking = true, 3384 }; 3385 3386 static struct mlxsw_sp_nexthop_group_vr_entry * 3387 mlxsw_sp_nexthop_group_vr_entry_lookup(struct mlxsw_sp_nexthop_group *nh_grp, 3388 const struct mlxsw_sp_fib *fib) 3389 { 3390 struct mlxsw_sp_nexthop_group_vr_key key; 3391 3392 memset(&key, 0, sizeof(key)); 3393 key.vr_id = fib->vr->id; 3394 key.proto = fib->proto; 3395 return rhashtable_lookup_fast(&nh_grp->vr_ht, &key, 3396 mlxsw_sp_nexthop_group_vr_ht_params); 3397 } 3398 3399 static int 3400 mlxsw_sp_nexthop_group_vr_entry_create(struct mlxsw_sp_nexthop_group *nh_grp, 3401 const struct mlxsw_sp_fib *fib) 3402 { 3403 struct mlxsw_sp_nexthop_group_vr_entry *vr_entry; 3404 int err; 3405 3406 vr_entry = kzalloc_obj(*vr_entry); 3407 if (!vr_entry) 3408 return -ENOMEM; 3409 3410 vr_entry->key.vr_id = fib->vr->id; 3411 vr_entry->key.proto = fib->proto; 3412 refcount_set(&vr_entry->ref_count, 1); 3413 3414 err = rhashtable_insert_fast(&nh_grp->vr_ht, &vr_entry->ht_node, 3415 mlxsw_sp_nexthop_group_vr_ht_params); 3416 if (err) 3417 goto err_hashtable_insert; 3418 3419 list_add(&vr_entry->list, &nh_grp->vr_list); 3420 3421 return 0; 3422 3423 err_hashtable_insert: 3424 kfree(vr_entry); 3425 return err; 3426 } 3427 3428 static void 3429 mlxsw_sp_nexthop_group_vr_entry_destroy(struct mlxsw_sp_nexthop_group *nh_grp, 3430 struct mlxsw_sp_nexthop_group_vr_entry *vr_entry) 3431 { 3432 list_del(&vr_entry->list); 3433 rhashtable_remove_fast(&nh_grp->vr_ht, &vr_entry->ht_node, 3434 mlxsw_sp_nexthop_group_vr_ht_params); 3435 kfree(vr_entry); 3436 } 3437 3438 static int 3439 mlxsw_sp_nexthop_group_vr_link(struct mlxsw_sp_nexthop_group *nh_grp, 3440 const struct mlxsw_sp_fib *fib) 3441 { 3442 struct mlxsw_sp_nexthop_group_vr_entry *vr_entry; 3443 3444 vr_entry = mlxsw_sp_nexthop_group_vr_entry_lookup(nh_grp, fib); 3445 if (vr_entry) { 3446 refcount_inc(&vr_entry->ref_count); 3447 return 0; 3448 } 3449 3450 return mlxsw_sp_nexthop_group_vr_entry_create(nh_grp, fib); 3451 } 3452 3453 static void 3454 mlxsw_sp_nexthop_group_vr_unlink(struct mlxsw_sp_nexthop_group *nh_grp, 3455 const struct mlxsw_sp_fib *fib) 3456 { 3457 struct mlxsw_sp_nexthop_group_vr_entry *vr_entry; 3458 3459 vr_entry = mlxsw_sp_nexthop_group_vr_entry_lookup(nh_grp, fib); 3460 if (WARN_ON_ONCE(!vr_entry)) 3461 return; 3462 3463 if (!refcount_dec_and_test(&vr_entry->ref_count)) 3464 return; 3465 3466 mlxsw_sp_nexthop_group_vr_entry_destroy(nh_grp, vr_entry); 3467 } 3468 3469 struct mlxsw_sp_nexthop_group_cmp_arg { 3470 enum mlxsw_sp_nexthop_group_type type; 3471 union { 3472 struct fib_info *fi; 3473 struct mlxsw_sp_fib6_entry *fib6_entry; 3474 u32 id; 3475 }; 3476 }; 3477 3478 static bool 3479 mlxsw_sp_nexthop6_group_has_nexthop(const struct mlxsw_sp_nexthop_group *nh_grp, 3480 const struct in6_addr *gw, int ifindex, 3481 int weight) 3482 { 3483 int i; 3484 3485 for (i = 0; i < nh_grp->nhgi->count; i++) { 3486 const struct mlxsw_sp_nexthop *nh; 3487 3488 nh = &nh_grp->nhgi->nexthops[i]; 3489 if (nh->ifindex == ifindex && nh->nh_weight == weight && 3490 ipv6_addr_equal(gw, (struct in6_addr *) nh->gw_addr)) 3491 return true; 3492 } 3493 3494 return false; 3495 } 3496 3497 static bool 3498 mlxsw_sp_nexthop6_group_cmp(const struct mlxsw_sp_nexthop_group *nh_grp, 3499 const struct mlxsw_sp_fib6_entry *fib6_entry) 3500 { 3501 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 3502 3503 if (nh_grp->nhgi->count != fib6_entry->nrt6) 3504 return false; 3505 3506 list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) { 3507 struct fib6_nh *fib6_nh = mlxsw_sp_rt6->rt->fib6_nh; 3508 struct in6_addr *gw; 3509 int ifindex, weight; 3510 3511 ifindex = fib6_nh->fib_nh_dev->ifindex; 3512 weight = fib6_nh->fib_nh_weight; 3513 gw = &fib6_nh->fib_nh_gw6; 3514 if (!mlxsw_sp_nexthop6_group_has_nexthop(nh_grp, gw, ifindex, 3515 weight)) 3516 return false; 3517 } 3518 3519 return true; 3520 } 3521 3522 static int 3523 mlxsw_sp_nexthop_group_cmp(struct rhashtable_compare_arg *arg, const void *ptr) 3524 { 3525 const struct mlxsw_sp_nexthop_group_cmp_arg *cmp_arg = arg->key; 3526 const struct mlxsw_sp_nexthop_group *nh_grp = ptr; 3527 3528 if (nh_grp->type != cmp_arg->type) 3529 return 1; 3530 3531 switch (cmp_arg->type) { 3532 case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4: 3533 return cmp_arg->fi != nh_grp->ipv4.fi; 3534 case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6: 3535 return !mlxsw_sp_nexthop6_group_cmp(nh_grp, 3536 cmp_arg->fib6_entry); 3537 case MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ: 3538 return cmp_arg->id != nh_grp->obj.id; 3539 default: 3540 WARN_ON(1); 3541 return 1; 3542 } 3543 } 3544 3545 static u32 mlxsw_sp_nexthop_group_hash_obj(const void *data, u32 len, u32 seed) 3546 { 3547 const struct mlxsw_sp_nexthop_group *nh_grp = data; 3548 const struct mlxsw_sp_nexthop *nh; 3549 struct fib_info *fi; 3550 unsigned int val; 3551 int i; 3552 3553 switch (nh_grp->type) { 3554 case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4: 3555 fi = nh_grp->ipv4.fi; 3556 return jhash(&fi, sizeof(fi), seed); 3557 case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6: 3558 val = nh_grp->nhgi->count; 3559 for (i = 0; i < nh_grp->nhgi->count; i++) { 3560 nh = &nh_grp->nhgi->nexthops[i]; 3561 val ^= jhash(&nh->ifindex, sizeof(nh->ifindex), seed); 3562 val ^= jhash(&nh->gw_addr, sizeof(nh->gw_addr), seed); 3563 } 3564 return jhash(&val, sizeof(val), seed); 3565 case MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ: 3566 return jhash(&nh_grp->obj.id, sizeof(nh_grp->obj.id), seed); 3567 default: 3568 WARN_ON(1); 3569 return 0; 3570 } 3571 } 3572 3573 static u32 3574 mlxsw_sp_nexthop6_group_hash(struct mlxsw_sp_fib6_entry *fib6_entry, u32 seed) 3575 { 3576 unsigned int val = fib6_entry->nrt6; 3577 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 3578 3579 list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) { 3580 struct fib6_nh *fib6_nh = mlxsw_sp_rt6->rt->fib6_nh; 3581 struct net_device *dev = fib6_nh->fib_nh_dev; 3582 struct in6_addr *gw = &fib6_nh->fib_nh_gw6; 3583 3584 val ^= jhash(&dev->ifindex, sizeof(dev->ifindex), seed); 3585 val ^= jhash(gw, sizeof(*gw), seed); 3586 } 3587 3588 return jhash(&val, sizeof(val), seed); 3589 } 3590 3591 static u32 3592 mlxsw_sp_nexthop_group_hash(const void *data, u32 len, u32 seed) 3593 { 3594 const struct mlxsw_sp_nexthop_group_cmp_arg *cmp_arg = data; 3595 3596 switch (cmp_arg->type) { 3597 case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4: 3598 return jhash(&cmp_arg->fi, sizeof(cmp_arg->fi), seed); 3599 case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6: 3600 return mlxsw_sp_nexthop6_group_hash(cmp_arg->fib6_entry, seed); 3601 case MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ: 3602 return jhash(&cmp_arg->id, sizeof(cmp_arg->id), seed); 3603 default: 3604 WARN_ON(1); 3605 return 0; 3606 } 3607 } 3608 3609 static const struct rhashtable_params mlxsw_sp_nexthop_group_ht_params = { 3610 .head_offset = offsetof(struct mlxsw_sp_nexthop_group, ht_node), 3611 .hashfn = mlxsw_sp_nexthop_group_hash, 3612 .obj_hashfn = mlxsw_sp_nexthop_group_hash_obj, 3613 .obj_cmpfn = mlxsw_sp_nexthop_group_cmp, 3614 }; 3615 3616 static int mlxsw_sp_nexthop_group_insert(struct mlxsw_sp *mlxsw_sp, 3617 struct mlxsw_sp_nexthop_group *nh_grp) 3618 { 3619 if (nh_grp->type == MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6 && 3620 !nh_grp->nhgi->gateway) 3621 return 0; 3622 3623 return rhashtable_insert_fast(&mlxsw_sp->router->nexthop_group_ht, 3624 &nh_grp->ht_node, 3625 mlxsw_sp_nexthop_group_ht_params); 3626 } 3627 3628 static void mlxsw_sp_nexthop_group_remove(struct mlxsw_sp *mlxsw_sp, 3629 struct mlxsw_sp_nexthop_group *nh_grp) 3630 { 3631 if (nh_grp->type == MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6 && 3632 !nh_grp->nhgi->gateway) 3633 return; 3634 3635 rhashtable_remove_fast(&mlxsw_sp->router->nexthop_group_ht, 3636 &nh_grp->ht_node, 3637 mlxsw_sp_nexthop_group_ht_params); 3638 } 3639 3640 static struct mlxsw_sp_nexthop_group * 3641 mlxsw_sp_nexthop4_group_lookup(struct mlxsw_sp *mlxsw_sp, 3642 struct fib_info *fi) 3643 { 3644 struct mlxsw_sp_nexthop_group_cmp_arg cmp_arg; 3645 3646 cmp_arg.type = MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4; 3647 cmp_arg.fi = fi; 3648 return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_group_ht, 3649 &cmp_arg, 3650 mlxsw_sp_nexthop_group_ht_params); 3651 } 3652 3653 static struct mlxsw_sp_nexthop_group * 3654 mlxsw_sp_nexthop6_group_lookup(struct mlxsw_sp *mlxsw_sp, 3655 struct mlxsw_sp_fib6_entry *fib6_entry) 3656 { 3657 struct mlxsw_sp_nexthop_group_cmp_arg cmp_arg; 3658 3659 cmp_arg.type = MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6; 3660 cmp_arg.fib6_entry = fib6_entry; 3661 return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_group_ht, 3662 &cmp_arg, 3663 mlxsw_sp_nexthop_group_ht_params); 3664 } 3665 3666 static const struct rhashtable_params mlxsw_sp_nexthop_ht_params = { 3667 .key_offset = offsetof(struct mlxsw_sp_nexthop, key), 3668 .head_offset = offsetof(struct mlxsw_sp_nexthop, ht_node), 3669 .key_len = sizeof(struct mlxsw_sp_nexthop_key), 3670 }; 3671 3672 static int mlxsw_sp_nexthop_insert(struct mlxsw_sp *mlxsw_sp, 3673 struct mlxsw_sp_nexthop *nh) 3674 { 3675 return rhashtable_insert_fast(&mlxsw_sp->router->nexthop_ht, 3676 &nh->ht_node, mlxsw_sp_nexthop_ht_params); 3677 } 3678 3679 static void mlxsw_sp_nexthop_remove(struct mlxsw_sp *mlxsw_sp, 3680 struct mlxsw_sp_nexthop *nh) 3681 { 3682 rhashtable_remove_fast(&mlxsw_sp->router->nexthop_ht, &nh->ht_node, 3683 mlxsw_sp_nexthop_ht_params); 3684 } 3685 3686 static struct mlxsw_sp_nexthop * 3687 mlxsw_sp_nexthop_lookup(struct mlxsw_sp *mlxsw_sp, 3688 struct mlxsw_sp_nexthop_key key) 3689 { 3690 return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_ht, &key, 3691 mlxsw_sp_nexthop_ht_params); 3692 } 3693 3694 static int mlxsw_sp_adj_index_mass_update_vr(struct mlxsw_sp *mlxsw_sp, 3695 enum mlxsw_sp_l3proto proto, 3696 u16 vr_id, 3697 u32 adj_index, u16 ecmp_size, 3698 u32 new_adj_index, 3699 u16 new_ecmp_size) 3700 { 3701 char raleu_pl[MLXSW_REG_RALEU_LEN]; 3702 3703 mlxsw_reg_raleu_pack(raleu_pl, 3704 (enum mlxsw_reg_ralxx_protocol) proto, vr_id, 3705 adj_index, ecmp_size, new_adj_index, 3706 new_ecmp_size); 3707 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raleu), raleu_pl); 3708 } 3709 3710 static int mlxsw_sp_adj_index_mass_update(struct mlxsw_sp *mlxsw_sp, 3711 struct mlxsw_sp_nexthop_group *nh_grp, 3712 u32 old_adj_index, u16 old_ecmp_size) 3713 { 3714 struct mlxsw_sp_nexthop_group_info *nhgi = nh_grp->nhgi; 3715 struct mlxsw_sp_nexthop_group_vr_entry *vr_entry; 3716 int err; 3717 3718 list_for_each_entry(vr_entry, &nh_grp->vr_list, list) { 3719 err = mlxsw_sp_adj_index_mass_update_vr(mlxsw_sp, 3720 vr_entry->key.proto, 3721 vr_entry->key.vr_id, 3722 old_adj_index, 3723 old_ecmp_size, 3724 nhgi->adj_index, 3725 nhgi->ecmp_size); 3726 if (err) 3727 goto err_mass_update_vr; 3728 } 3729 return 0; 3730 3731 err_mass_update_vr: 3732 list_for_each_entry_continue_reverse(vr_entry, &nh_grp->vr_list, list) 3733 mlxsw_sp_adj_index_mass_update_vr(mlxsw_sp, vr_entry->key.proto, 3734 vr_entry->key.vr_id, 3735 nhgi->adj_index, 3736 nhgi->ecmp_size, 3737 old_adj_index, old_ecmp_size); 3738 return err; 3739 } 3740 3741 static int __mlxsw_sp_nexthop_eth_update(struct mlxsw_sp *mlxsw_sp, 3742 u32 adj_index, 3743 struct mlxsw_sp_nexthop *nh, 3744 bool force, char *ratr_pl) 3745 { 3746 struct mlxsw_sp_neigh_entry *neigh_entry = nh->neigh_entry; 3747 struct mlxsw_sp_rif *rif = mlxsw_sp_nexthop_rif(nh); 3748 enum mlxsw_reg_ratr_op op; 3749 u16 rif_index; 3750 3751 rif_index = rif ? rif->rif_index : 3752 mlxsw_sp->router->lb_crif->rif->rif_index; 3753 op = force ? MLXSW_REG_RATR_OP_WRITE_WRITE_ENTRY : 3754 MLXSW_REG_RATR_OP_WRITE_WRITE_ENTRY_ON_ACTIVITY; 3755 mlxsw_reg_ratr_pack(ratr_pl, op, true, MLXSW_REG_RATR_TYPE_ETHERNET, 3756 adj_index, rif_index); 3757 switch (nh->action) { 3758 case MLXSW_SP_NEXTHOP_ACTION_FORWARD: 3759 mlxsw_reg_ratr_eth_entry_pack(ratr_pl, neigh_entry->ha); 3760 break; 3761 case MLXSW_SP_NEXTHOP_ACTION_DISCARD: 3762 mlxsw_reg_ratr_trap_action_set(ratr_pl, 3763 MLXSW_REG_RATR_TRAP_ACTION_DISCARD_ERRORS); 3764 break; 3765 case MLXSW_SP_NEXTHOP_ACTION_TRAP: 3766 mlxsw_reg_ratr_trap_action_set(ratr_pl, 3767 MLXSW_REG_RATR_TRAP_ACTION_TRAP); 3768 mlxsw_reg_ratr_trap_id_set(ratr_pl, MLXSW_TRAP_ID_RTR_EGRESS0); 3769 break; 3770 default: 3771 WARN_ON_ONCE(1); 3772 return -EINVAL; 3773 } 3774 if (nh->counter) 3775 mlxsw_reg_ratr_counter_pack(ratr_pl, nh->counter->counter_index, 3776 true); 3777 else 3778 mlxsw_reg_ratr_counter_pack(ratr_pl, 0, false); 3779 3780 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ratr), ratr_pl); 3781 } 3782 3783 int mlxsw_sp_nexthop_eth_update(struct mlxsw_sp *mlxsw_sp, u32 adj_index, 3784 struct mlxsw_sp_nexthop *nh, bool force, 3785 char *ratr_pl) 3786 { 3787 int i; 3788 3789 for (i = 0; i < nh->num_adj_entries; i++) { 3790 int err; 3791 3792 err = __mlxsw_sp_nexthop_eth_update(mlxsw_sp, adj_index + i, 3793 nh, force, ratr_pl); 3794 if (err) 3795 return err; 3796 } 3797 3798 return 0; 3799 } 3800 3801 static int __mlxsw_sp_nexthop_ipip_update(struct mlxsw_sp *mlxsw_sp, 3802 u32 adj_index, 3803 struct mlxsw_sp_nexthop *nh, 3804 bool force, char *ratr_pl) 3805 { 3806 const struct mlxsw_sp_ipip_ops *ipip_ops; 3807 3808 ipip_ops = mlxsw_sp->router->ipip_ops_arr[nh->ipip_entry->ipipt]; 3809 return ipip_ops->nexthop_update(mlxsw_sp, adj_index, nh->ipip_entry, 3810 force, ratr_pl); 3811 } 3812 3813 static int mlxsw_sp_nexthop_ipip_update(struct mlxsw_sp *mlxsw_sp, 3814 u32 adj_index, 3815 struct mlxsw_sp_nexthop *nh, bool force, 3816 char *ratr_pl) 3817 { 3818 int i; 3819 3820 for (i = 0; i < nh->num_adj_entries; i++) { 3821 int err; 3822 3823 err = __mlxsw_sp_nexthop_ipip_update(mlxsw_sp, adj_index + i, 3824 nh, force, ratr_pl); 3825 if (err) 3826 return err; 3827 } 3828 3829 return 0; 3830 } 3831 3832 static int mlxsw_sp_nexthop_update(struct mlxsw_sp *mlxsw_sp, u32 adj_index, 3833 struct mlxsw_sp_nexthop *nh, bool force, 3834 char *ratr_pl) 3835 { 3836 /* When action is discard or trap, the nexthop must be 3837 * programmed as an Ethernet nexthop. 3838 */ 3839 if (nh->type == MLXSW_SP_NEXTHOP_TYPE_ETH || 3840 nh->action == MLXSW_SP_NEXTHOP_ACTION_DISCARD || 3841 nh->action == MLXSW_SP_NEXTHOP_ACTION_TRAP) 3842 return mlxsw_sp_nexthop_eth_update(mlxsw_sp, adj_index, nh, 3843 force, ratr_pl); 3844 else 3845 return mlxsw_sp_nexthop_ipip_update(mlxsw_sp, adj_index, nh, 3846 force, ratr_pl); 3847 } 3848 3849 static int 3850 mlxsw_sp_nexthop_group_update(struct mlxsw_sp *mlxsw_sp, 3851 struct mlxsw_sp_nexthop_group_info *nhgi, 3852 bool reallocate) 3853 { 3854 char ratr_pl[MLXSW_REG_RATR_LEN]; 3855 u32 adj_index = nhgi->adj_index; /* base */ 3856 struct mlxsw_sp_nexthop *nh; 3857 int i; 3858 3859 for (i = 0; i < nhgi->count; i++) { 3860 nh = &nhgi->nexthops[i]; 3861 3862 if (!nh->should_offload) { 3863 mlxsw_sp_nexthop_counter_disable(mlxsw_sp, nh); 3864 nh->offloaded = 0; 3865 continue; 3866 } 3867 3868 if (nh->update || reallocate) { 3869 int err = 0; 3870 3871 err = mlxsw_sp_nexthop_counter_update(mlxsw_sp, nh); 3872 if (err) 3873 return err; 3874 3875 err = mlxsw_sp_nexthop_update(mlxsw_sp, adj_index, nh, 3876 true, ratr_pl); 3877 if (err) 3878 return err; 3879 nh->update = 0; 3880 nh->offloaded = 1; 3881 } 3882 adj_index += nh->num_adj_entries; 3883 } 3884 return 0; 3885 } 3886 3887 static int 3888 mlxsw_sp_nexthop_fib_entries_update(struct mlxsw_sp *mlxsw_sp, 3889 struct mlxsw_sp_nexthop_group *nh_grp) 3890 { 3891 struct mlxsw_sp_fib_entry *fib_entry; 3892 int err; 3893 3894 list_for_each_entry(fib_entry, &nh_grp->fib_list, nexthop_group_node) { 3895 err = mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry); 3896 if (err) 3897 return err; 3898 } 3899 return 0; 3900 } 3901 3902 struct mlxsw_sp_adj_grp_size_range { 3903 u16 start; /* Inclusive */ 3904 u16 end; /* Inclusive */ 3905 }; 3906 3907 /* Ordered by range start value */ 3908 static const struct mlxsw_sp_adj_grp_size_range 3909 mlxsw_sp1_adj_grp_size_ranges[] = { 3910 { .start = 1, .end = 64 }, 3911 { .start = 512, .end = 512 }, 3912 { .start = 1024, .end = 1024 }, 3913 { .start = 2048, .end = 2048 }, 3914 { .start = 4096, .end = 4096 }, 3915 }; 3916 3917 /* Ordered by range start value */ 3918 static const struct mlxsw_sp_adj_grp_size_range 3919 mlxsw_sp2_adj_grp_size_ranges[] = { 3920 { .start = 1, .end = 128 }, 3921 { .start = 256, .end = 256 }, 3922 { .start = 512, .end = 512 }, 3923 { .start = 1024, .end = 1024 }, 3924 { .start = 2048, .end = 2048 }, 3925 { .start = 4096, .end = 4096 }, 3926 }; 3927 3928 static void mlxsw_sp_adj_grp_size_round_up(const struct mlxsw_sp *mlxsw_sp, 3929 u16 *p_adj_grp_size) 3930 { 3931 int i; 3932 3933 for (i = 0; i < mlxsw_sp->router->adj_grp_size_ranges_count; i++) { 3934 const struct mlxsw_sp_adj_grp_size_range *size_range; 3935 3936 size_range = &mlxsw_sp->router->adj_grp_size_ranges[i]; 3937 3938 if (*p_adj_grp_size >= size_range->start && 3939 *p_adj_grp_size <= size_range->end) 3940 return; 3941 3942 if (*p_adj_grp_size <= size_range->end) { 3943 *p_adj_grp_size = size_range->end; 3944 return; 3945 } 3946 } 3947 } 3948 3949 static void mlxsw_sp_adj_grp_size_round_down(const struct mlxsw_sp *mlxsw_sp, 3950 u16 *p_adj_grp_size, 3951 unsigned int alloc_size) 3952 { 3953 int i; 3954 3955 for (i = mlxsw_sp->router->adj_grp_size_ranges_count - 1; i >= 0; i--) { 3956 const struct mlxsw_sp_adj_grp_size_range *size_range; 3957 3958 size_range = &mlxsw_sp->router->adj_grp_size_ranges[i]; 3959 3960 if (alloc_size >= size_range->end) { 3961 *p_adj_grp_size = size_range->end; 3962 return; 3963 } 3964 } 3965 } 3966 3967 static int mlxsw_sp_fix_adj_grp_size(struct mlxsw_sp *mlxsw_sp, 3968 u16 *p_adj_grp_size) 3969 { 3970 unsigned int alloc_size; 3971 int err; 3972 3973 /* Round up the requested group size to the next size supported 3974 * by the device and make sure the request can be satisfied. 3975 */ 3976 mlxsw_sp_adj_grp_size_round_up(mlxsw_sp, p_adj_grp_size); 3977 err = mlxsw_sp_kvdl_alloc_count_query(mlxsw_sp, 3978 MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 3979 *p_adj_grp_size, &alloc_size); 3980 if (err) 3981 return err; 3982 /* It is possible the allocation results in more allocated 3983 * entries than requested. Try to use as much of them as 3984 * possible. 3985 */ 3986 mlxsw_sp_adj_grp_size_round_down(mlxsw_sp, p_adj_grp_size, alloc_size); 3987 3988 return 0; 3989 } 3990 3991 static void 3992 mlxsw_sp_nexthop_group_normalize(struct mlxsw_sp_nexthop_group_info *nhgi) 3993 { 3994 int i, g = 0, sum_norm_weight = 0; 3995 struct mlxsw_sp_nexthop *nh; 3996 3997 for (i = 0; i < nhgi->count; i++) { 3998 nh = &nhgi->nexthops[i]; 3999 4000 if (!nh->should_offload) 4001 continue; 4002 if (g > 0) 4003 g = gcd(nh->nh_weight, g); 4004 else 4005 g = nh->nh_weight; 4006 } 4007 4008 for (i = 0; i < nhgi->count; i++) { 4009 nh = &nhgi->nexthops[i]; 4010 4011 if (!nh->should_offload) 4012 continue; 4013 nh->norm_nh_weight = nh->nh_weight / g; 4014 sum_norm_weight += nh->norm_nh_weight; 4015 } 4016 4017 nhgi->sum_norm_weight = sum_norm_weight; 4018 } 4019 4020 static void 4021 mlxsw_sp_nexthop_group_rebalance(struct mlxsw_sp_nexthop_group_info *nhgi) 4022 { 4023 int i, weight = 0, lower_bound = 0; 4024 int total = nhgi->sum_norm_weight; 4025 u16 ecmp_size = nhgi->ecmp_size; 4026 4027 for (i = 0; i < nhgi->count; i++) { 4028 struct mlxsw_sp_nexthop *nh = &nhgi->nexthops[i]; 4029 int upper_bound; 4030 4031 if (!nh->should_offload) 4032 continue; 4033 weight += nh->norm_nh_weight; 4034 upper_bound = DIV_ROUND_CLOSEST(ecmp_size * weight, total); 4035 nh->num_adj_entries = upper_bound - lower_bound; 4036 lower_bound = upper_bound; 4037 } 4038 } 4039 4040 static struct mlxsw_sp_nexthop * 4041 mlxsw_sp_rt6_nexthop(struct mlxsw_sp_nexthop_group *nh_grp, 4042 const struct mlxsw_sp_rt6 *mlxsw_sp_rt6); 4043 4044 static void 4045 mlxsw_sp_nexthop4_group_offload_refresh(struct mlxsw_sp *mlxsw_sp, 4046 struct mlxsw_sp_nexthop_group *nh_grp) 4047 { 4048 int i; 4049 4050 for (i = 0; i < nh_grp->nhgi->count; i++) { 4051 struct mlxsw_sp_nexthop *nh = &nh_grp->nhgi->nexthops[i]; 4052 4053 if (nh->offloaded) 4054 nh->key.fib_nh->fib_nh_flags |= RTNH_F_OFFLOAD; 4055 else 4056 nh->key.fib_nh->fib_nh_flags &= ~RTNH_F_OFFLOAD; 4057 } 4058 } 4059 4060 static void 4061 __mlxsw_sp_nexthop6_group_offload_refresh(struct mlxsw_sp_nexthop_group *nh_grp, 4062 struct mlxsw_sp_fib6_entry *fib6_entry) 4063 { 4064 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 4065 4066 list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) { 4067 struct fib6_nh *fib6_nh = mlxsw_sp_rt6->rt->fib6_nh; 4068 struct mlxsw_sp_nexthop *nh; 4069 4070 nh = mlxsw_sp_rt6_nexthop(nh_grp, mlxsw_sp_rt6); 4071 if (nh && nh->offloaded) 4072 fib6_nh->fib_nh_flags |= RTNH_F_OFFLOAD; 4073 else 4074 fib6_nh->fib_nh_flags &= ~RTNH_F_OFFLOAD; 4075 } 4076 } 4077 4078 static void 4079 mlxsw_sp_nexthop6_group_offload_refresh(struct mlxsw_sp *mlxsw_sp, 4080 struct mlxsw_sp_nexthop_group *nh_grp) 4081 { 4082 struct mlxsw_sp_fib6_entry *fib6_entry; 4083 4084 /* Unfortunately, in IPv6 the route and the nexthop are described by 4085 * the same struct, so we need to iterate over all the routes using the 4086 * nexthop group and set / clear the offload indication for them. 4087 */ 4088 list_for_each_entry(fib6_entry, &nh_grp->fib_list, 4089 common.nexthop_group_node) 4090 __mlxsw_sp_nexthop6_group_offload_refresh(nh_grp, fib6_entry); 4091 } 4092 4093 static void 4094 mlxsw_sp_nexthop_bucket_offload_refresh(struct mlxsw_sp *mlxsw_sp, 4095 const struct mlxsw_sp_nexthop *nh, 4096 u16 bucket_index) 4097 { 4098 struct mlxsw_sp_nexthop_group *nh_grp = nh->nhgi->nh_grp; 4099 bool offload = false, trap = false; 4100 4101 if (nh->offloaded) { 4102 if (nh->action == MLXSW_SP_NEXTHOP_ACTION_TRAP) 4103 trap = true; 4104 else 4105 offload = true; 4106 } 4107 nexthop_bucket_set_hw_flags(mlxsw_sp_net(mlxsw_sp), nh_grp->obj.id, 4108 bucket_index, offload, trap); 4109 } 4110 4111 static void 4112 mlxsw_sp_nexthop_obj_group_offload_refresh(struct mlxsw_sp *mlxsw_sp, 4113 struct mlxsw_sp_nexthop_group *nh_grp) 4114 { 4115 int i; 4116 4117 /* Do not update the flags if the nexthop group is being destroyed 4118 * since: 4119 * 1. The nexthop objects is being deleted, in which case the flags are 4120 * irrelevant. 4121 * 2. The nexthop group was replaced by a newer group, in which case 4122 * the flags of the nexthop object were already updated based on the 4123 * new group. 4124 */ 4125 if (nh_grp->can_destroy) 4126 return; 4127 4128 nexthop_set_hw_flags(mlxsw_sp_net(mlxsw_sp), nh_grp->obj.id, 4129 nh_grp->nhgi->adj_index_valid, false); 4130 4131 /* Update flags of individual nexthop buckets in case of a resilient 4132 * nexthop group. 4133 */ 4134 if (!nh_grp->nhgi->is_resilient) 4135 return; 4136 4137 for (i = 0; i < nh_grp->nhgi->count; i++) { 4138 struct mlxsw_sp_nexthop *nh = &nh_grp->nhgi->nexthops[i]; 4139 4140 mlxsw_sp_nexthop_bucket_offload_refresh(mlxsw_sp, nh, i); 4141 } 4142 } 4143 4144 static void 4145 mlxsw_sp_nexthop_group_offload_refresh(struct mlxsw_sp *mlxsw_sp, 4146 struct mlxsw_sp_nexthop_group *nh_grp) 4147 { 4148 switch (nh_grp->type) { 4149 case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4: 4150 mlxsw_sp_nexthop4_group_offload_refresh(mlxsw_sp, nh_grp); 4151 break; 4152 case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6: 4153 mlxsw_sp_nexthop6_group_offload_refresh(mlxsw_sp, nh_grp); 4154 break; 4155 case MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ: 4156 mlxsw_sp_nexthop_obj_group_offload_refresh(mlxsw_sp, nh_grp); 4157 break; 4158 } 4159 } 4160 4161 static int 4162 mlxsw_sp_nexthop_group_refresh(struct mlxsw_sp *mlxsw_sp, 4163 struct mlxsw_sp_nexthop_group *nh_grp) 4164 { 4165 struct mlxsw_sp_nexthop_group_info *nhgi = nh_grp->nhgi; 4166 u16 ecmp_size, old_ecmp_size; 4167 struct mlxsw_sp_nexthop *nh; 4168 bool offload_change = false; 4169 u32 adj_index; 4170 bool old_adj_index_valid; 4171 u32 old_adj_index; 4172 int i, err2, err; 4173 4174 if (!nhgi->gateway) 4175 return mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp); 4176 4177 for (i = 0; i < nhgi->count; i++) { 4178 nh = &nhgi->nexthops[i]; 4179 4180 if (nh->should_offload != nh->offloaded) { 4181 offload_change = true; 4182 if (nh->should_offload) 4183 nh->update = 1; 4184 } 4185 } 4186 if (!offload_change) { 4187 /* Nothing was added or removed, so no need to reallocate. Just 4188 * update MAC on existing adjacency indexes. 4189 */ 4190 err = mlxsw_sp_nexthop_group_update(mlxsw_sp, nhgi, false); 4191 if (err) { 4192 dev_warn(mlxsw_sp->bus_info->dev, "Failed to update neigh MAC in adjacency table.\n"); 4193 goto set_trap; 4194 } 4195 /* Flags of individual nexthop buckets might need to be 4196 * updated. 4197 */ 4198 mlxsw_sp_nexthop_group_offload_refresh(mlxsw_sp, nh_grp); 4199 return 0; 4200 } 4201 mlxsw_sp_nexthop_group_normalize(nhgi); 4202 if (!nhgi->sum_norm_weight) { 4203 /* No neigh of this group is connected so we just set 4204 * the trap and let everthing flow through kernel. 4205 */ 4206 err = 0; 4207 goto set_trap; 4208 } 4209 4210 ecmp_size = nhgi->sum_norm_weight; 4211 err = mlxsw_sp_fix_adj_grp_size(mlxsw_sp, &ecmp_size); 4212 if (err) 4213 /* No valid allocation size available. */ 4214 goto set_trap; 4215 4216 err = mlxsw_sp_kvdl_alloc(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 4217 ecmp_size, &adj_index); 4218 if (err) { 4219 /* We ran out of KVD linear space, just set the 4220 * trap and let everything flow through kernel. 4221 */ 4222 dev_warn(mlxsw_sp->bus_info->dev, "Failed to allocate KVD linear area for nexthop group.\n"); 4223 goto set_trap; 4224 } 4225 old_adj_index_valid = nhgi->adj_index_valid; 4226 old_adj_index = nhgi->adj_index; 4227 old_ecmp_size = nhgi->ecmp_size; 4228 nhgi->adj_index_valid = 1; 4229 nhgi->adj_index = adj_index; 4230 nhgi->ecmp_size = ecmp_size; 4231 mlxsw_sp_nexthop_group_rebalance(nhgi); 4232 err = mlxsw_sp_nexthop_group_update(mlxsw_sp, nhgi, true); 4233 if (err) { 4234 dev_warn(mlxsw_sp->bus_info->dev, "Failed to update neigh MAC in adjacency table.\n"); 4235 goto set_trap; 4236 } 4237 4238 mlxsw_sp_nexthop_group_offload_refresh(mlxsw_sp, nh_grp); 4239 4240 if (!old_adj_index_valid) { 4241 /* The trap was set for fib entries, so we have to call 4242 * fib entry update to unset it and use adjacency index. 4243 */ 4244 err = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp); 4245 if (err) { 4246 dev_warn(mlxsw_sp->bus_info->dev, "Failed to add adjacency index to fib entries.\n"); 4247 goto set_trap; 4248 } 4249 return 0; 4250 } 4251 4252 err = mlxsw_sp_adj_index_mass_update(mlxsw_sp, nh_grp, 4253 old_adj_index, old_ecmp_size); 4254 mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 4255 old_ecmp_size, old_adj_index); 4256 if (err) { 4257 dev_warn(mlxsw_sp->bus_info->dev, "Failed to mass-update adjacency index for nexthop group.\n"); 4258 goto set_trap; 4259 } 4260 4261 return 0; 4262 4263 set_trap: 4264 old_adj_index_valid = nhgi->adj_index_valid; 4265 nhgi->adj_index_valid = 0; 4266 for (i = 0; i < nhgi->count; i++) { 4267 nh = &nhgi->nexthops[i]; 4268 nh->offloaded = 0; 4269 } 4270 err2 = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp); 4271 if (err2) 4272 dev_warn(mlxsw_sp->bus_info->dev, "Failed to set traps for fib entries.\n"); 4273 mlxsw_sp_nexthop_group_offload_refresh(mlxsw_sp, nh_grp); 4274 if (old_adj_index_valid) 4275 mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 4276 nhgi->ecmp_size, nhgi->adj_index); 4277 return err; 4278 } 4279 4280 static void __mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp_nexthop *nh, 4281 bool removing) 4282 { 4283 if (!removing) { 4284 nh->action = MLXSW_SP_NEXTHOP_ACTION_FORWARD; 4285 nh->should_offload = 1; 4286 } else if (nh->nhgi->is_resilient) { 4287 nh->action = MLXSW_SP_NEXTHOP_ACTION_TRAP; 4288 nh->should_offload = 1; 4289 } else { 4290 nh->should_offload = 0; 4291 } 4292 nh->update = 1; 4293 } 4294 4295 static int 4296 mlxsw_sp_nexthop_dead_neigh_replace(struct mlxsw_sp *mlxsw_sp, 4297 struct mlxsw_sp_neigh_entry *neigh_entry) 4298 { 4299 struct neighbour *n, *old_n = neigh_entry->key.n; 4300 struct mlxsw_sp_nexthop *nh; 4301 struct net_device *dev; 4302 bool entry_connected; 4303 u8 nud_state, dead; 4304 int err; 4305 4306 nh = list_first_entry(&neigh_entry->nexthop_list, 4307 struct mlxsw_sp_nexthop, neigh_list_node); 4308 dev = mlxsw_sp_nexthop_dev(nh); 4309 4310 n = neigh_lookup(nh->neigh_tbl, &nh->gw_addr, dev); 4311 if (!n) { 4312 n = neigh_create(nh->neigh_tbl, &nh->gw_addr, dev); 4313 if (IS_ERR(n)) 4314 return PTR_ERR(n); 4315 neigh_event_send(n, NULL); 4316 } 4317 4318 mlxsw_sp_neigh_entry_remove(mlxsw_sp, neigh_entry); 4319 neigh_entry->key.n = n; 4320 err = mlxsw_sp_neigh_entry_insert(mlxsw_sp, neigh_entry); 4321 if (err) 4322 goto err_neigh_entry_insert; 4323 4324 neigh_release(old_n); 4325 4326 read_lock_bh(&n->lock); 4327 nud_state = n->nud_state; 4328 dead = n->dead; 4329 read_unlock_bh(&n->lock); 4330 entry_connected = nud_state & NUD_VALID && !dead; 4331 4332 list_for_each_entry(nh, &neigh_entry->nexthop_list, 4333 neigh_list_node) { 4334 __mlxsw_sp_nexthop_neigh_update(nh, !entry_connected); 4335 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nhgi->nh_grp); 4336 } 4337 4338 return 0; 4339 4340 err_neigh_entry_insert: 4341 neigh_entry->key.n = old_n; 4342 mlxsw_sp_neigh_entry_insert(mlxsw_sp, neigh_entry); 4343 neigh_release(n); 4344 return err; 4345 } 4346 4347 static void 4348 mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp *mlxsw_sp, 4349 struct mlxsw_sp_neigh_entry *neigh_entry, 4350 bool removing, bool dead) 4351 { 4352 struct mlxsw_sp_nexthop *nh; 4353 4354 if (list_empty(&neigh_entry->nexthop_list)) 4355 return; 4356 4357 if (dead) { 4358 int err; 4359 4360 err = mlxsw_sp_nexthop_dead_neigh_replace(mlxsw_sp, 4361 neigh_entry); 4362 if (err) 4363 dev_err(mlxsw_sp->bus_info->dev, "Failed to replace dead neigh\n"); 4364 return; 4365 } 4366 4367 list_for_each_entry(nh, &neigh_entry->nexthop_list, 4368 neigh_list_node) { 4369 __mlxsw_sp_nexthop_neigh_update(nh, removing); 4370 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nhgi->nh_grp); 4371 } 4372 } 4373 4374 static void mlxsw_sp_nexthop_crif_init(struct mlxsw_sp_nexthop *nh, 4375 struct mlxsw_sp_crif *crif) 4376 { 4377 if (nh->crif) 4378 return; 4379 4380 nh->crif = crif; 4381 list_add(&nh->crif_list_node, &crif->nexthop_list); 4382 } 4383 4384 static void mlxsw_sp_nexthop_crif_fini(struct mlxsw_sp_nexthop *nh) 4385 { 4386 if (!nh->crif) 4387 return; 4388 4389 list_del(&nh->crif_list_node); 4390 nh->crif = NULL; 4391 } 4392 4393 static int mlxsw_sp_nexthop_neigh_init(struct mlxsw_sp *mlxsw_sp, 4394 struct mlxsw_sp_nexthop *nh) 4395 { 4396 struct mlxsw_sp_neigh_entry *neigh_entry; 4397 struct net_device *dev; 4398 struct neighbour *n; 4399 u8 nud_state, dead; 4400 int err; 4401 4402 if (WARN_ON(!nh->crif->rif)) 4403 return 0; 4404 4405 if (!nh->nhgi->gateway || nh->neigh_entry) 4406 return 0; 4407 dev = mlxsw_sp_nexthop_dev(nh); 4408 4409 /* Take a reference of neigh here ensuring that neigh would 4410 * not be destructed before the nexthop entry is finished. 4411 * The reference is taken either in neigh_lookup() or 4412 * in neigh_create() in case n is not found. 4413 */ 4414 n = neigh_lookup(nh->neigh_tbl, &nh->gw_addr, dev); 4415 if (!n) { 4416 n = neigh_create(nh->neigh_tbl, &nh->gw_addr, dev); 4417 if (IS_ERR(n)) 4418 return PTR_ERR(n); 4419 neigh_event_send(n, NULL); 4420 } 4421 neigh_entry = mlxsw_sp_neigh_entry_lookup(mlxsw_sp, n); 4422 if (!neigh_entry) { 4423 neigh_entry = mlxsw_sp_neigh_entry_create(mlxsw_sp, n); 4424 if (IS_ERR(neigh_entry)) { 4425 err = -EINVAL; 4426 goto err_neigh_entry_create; 4427 } 4428 } 4429 4430 /* Release the reference taken by neigh_lookup() / neigh_create() since 4431 * neigh_entry already holds one. 4432 */ 4433 neigh_release(n); 4434 4435 /* If that is the first nexthop connected to that neigh, add to 4436 * nexthop_neighs_list 4437 */ 4438 if (list_empty(&neigh_entry->nexthop_list)) 4439 list_add_tail(&neigh_entry->nexthop_neighs_list_node, 4440 &mlxsw_sp->router->nexthop_neighs_list); 4441 4442 nh->neigh_entry = neigh_entry; 4443 list_add_tail(&nh->neigh_list_node, &neigh_entry->nexthop_list); 4444 read_lock_bh(&n->lock); 4445 nud_state = n->nud_state; 4446 dead = n->dead; 4447 read_unlock_bh(&n->lock); 4448 __mlxsw_sp_nexthop_neigh_update(nh, !(nud_state & NUD_VALID && !dead)); 4449 4450 return 0; 4451 4452 err_neigh_entry_create: 4453 neigh_release(n); 4454 return err; 4455 } 4456 4457 static void mlxsw_sp_nexthop_neigh_fini(struct mlxsw_sp *mlxsw_sp, 4458 struct mlxsw_sp_nexthop *nh) 4459 { 4460 struct mlxsw_sp_neigh_entry *neigh_entry = nh->neigh_entry; 4461 4462 if (!neigh_entry) 4463 return; 4464 4465 __mlxsw_sp_nexthop_neigh_update(nh, true); 4466 list_del(&nh->neigh_list_node); 4467 nh->neigh_entry = NULL; 4468 4469 /* If that is the last nexthop connected to that neigh, remove from 4470 * nexthop_neighs_list 4471 */ 4472 if (list_empty(&neigh_entry->nexthop_list)) 4473 list_del(&neigh_entry->nexthop_neighs_list_node); 4474 4475 if (!neigh_entry->connected && list_empty(&neigh_entry->nexthop_list)) 4476 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry); 4477 } 4478 4479 static bool mlxsw_sp_ipip_netdev_ul_up(struct net_device *ol_dev) 4480 { 4481 struct net_device *ul_dev; 4482 bool is_up; 4483 4484 rcu_read_lock(); 4485 ul_dev = mlxsw_sp_ipip_netdev_ul_dev_get(ol_dev); 4486 is_up = ul_dev ? (ul_dev->flags & IFF_UP) : true; 4487 rcu_read_unlock(); 4488 4489 return is_up; 4490 } 4491 4492 static void mlxsw_sp_nexthop_ipip_init(struct mlxsw_sp *mlxsw_sp, 4493 struct mlxsw_sp_nexthop *nh, 4494 struct mlxsw_sp_ipip_entry *ipip_entry) 4495 { 4496 struct mlxsw_sp_crif *crif; 4497 bool removing; 4498 4499 if (!nh->nhgi->gateway || nh->ipip_entry) 4500 return; 4501 4502 crif = mlxsw_sp_crif_lookup(mlxsw_sp->router, ipip_entry->ol_dev); 4503 if (WARN_ON(!crif)) 4504 return; 4505 4506 nh->ipip_entry = ipip_entry; 4507 removing = !mlxsw_sp_ipip_netdev_ul_up(ipip_entry->ol_dev); 4508 __mlxsw_sp_nexthop_neigh_update(nh, removing); 4509 mlxsw_sp_nexthop_crif_init(nh, crif); 4510 } 4511 4512 static void mlxsw_sp_nexthop_ipip_fini(struct mlxsw_sp *mlxsw_sp, 4513 struct mlxsw_sp_nexthop *nh) 4514 { 4515 struct mlxsw_sp_ipip_entry *ipip_entry = nh->ipip_entry; 4516 4517 if (!ipip_entry) 4518 return; 4519 4520 __mlxsw_sp_nexthop_neigh_update(nh, true); 4521 nh->ipip_entry = NULL; 4522 } 4523 4524 static bool mlxsw_sp_nexthop4_ipip_type(const struct mlxsw_sp *mlxsw_sp, 4525 const struct fib_nh *fib_nh, 4526 enum mlxsw_sp_ipip_type *p_ipipt) 4527 { 4528 struct net_device *dev = fib_nh->fib_nh_dev; 4529 4530 return dev && 4531 fib_nh->nh_parent->fib_type == RTN_UNICAST && 4532 mlxsw_sp_netdev_ipip_type(mlxsw_sp, dev, p_ipipt); 4533 } 4534 4535 static int mlxsw_sp_nexthop_type_init(struct mlxsw_sp *mlxsw_sp, 4536 struct mlxsw_sp_nexthop *nh, 4537 const struct net_device *dev) 4538 { 4539 const struct mlxsw_sp_ipip_ops *ipip_ops; 4540 struct mlxsw_sp_ipip_entry *ipip_entry; 4541 struct mlxsw_sp_crif *crif; 4542 int err; 4543 4544 ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, dev); 4545 if (ipip_entry) { 4546 ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt]; 4547 if (ipip_ops->can_offload(mlxsw_sp, dev)) { 4548 nh->type = MLXSW_SP_NEXTHOP_TYPE_IPIP; 4549 mlxsw_sp_nexthop_ipip_init(mlxsw_sp, nh, ipip_entry); 4550 return 0; 4551 } 4552 } 4553 4554 nh->type = MLXSW_SP_NEXTHOP_TYPE_ETH; 4555 crif = mlxsw_sp_crif_lookup(mlxsw_sp->router, dev); 4556 if (!crif) 4557 return 0; 4558 4559 mlxsw_sp_nexthop_crif_init(nh, crif); 4560 4561 if (!crif->rif) 4562 return 0; 4563 4564 err = mlxsw_sp_nexthop_neigh_init(mlxsw_sp, nh); 4565 if (err) 4566 goto err_neigh_init; 4567 4568 return 0; 4569 4570 err_neigh_init: 4571 mlxsw_sp_nexthop_crif_fini(nh); 4572 return err; 4573 } 4574 4575 static int mlxsw_sp_nexthop_type_rif_made(struct mlxsw_sp *mlxsw_sp, 4576 struct mlxsw_sp_nexthop *nh) 4577 { 4578 switch (nh->type) { 4579 case MLXSW_SP_NEXTHOP_TYPE_ETH: 4580 return mlxsw_sp_nexthop_neigh_init(mlxsw_sp, nh); 4581 case MLXSW_SP_NEXTHOP_TYPE_IPIP: 4582 break; 4583 } 4584 4585 return 0; 4586 } 4587 4588 static void mlxsw_sp_nexthop_type_rif_gone(struct mlxsw_sp *mlxsw_sp, 4589 struct mlxsw_sp_nexthop *nh) 4590 { 4591 switch (nh->type) { 4592 case MLXSW_SP_NEXTHOP_TYPE_ETH: 4593 mlxsw_sp_nexthop_neigh_fini(mlxsw_sp, nh); 4594 break; 4595 case MLXSW_SP_NEXTHOP_TYPE_IPIP: 4596 mlxsw_sp_nexthop_ipip_fini(mlxsw_sp, nh); 4597 break; 4598 } 4599 } 4600 4601 static void mlxsw_sp_nexthop_type_fini(struct mlxsw_sp *mlxsw_sp, 4602 struct mlxsw_sp_nexthop *nh) 4603 { 4604 mlxsw_sp_nexthop_type_rif_gone(mlxsw_sp, nh); 4605 mlxsw_sp_nexthop_crif_fini(nh); 4606 } 4607 4608 static int mlxsw_sp_nexthop4_init(struct mlxsw_sp *mlxsw_sp, 4609 struct mlxsw_sp_nexthop_group *nh_grp, 4610 struct mlxsw_sp_nexthop *nh, 4611 struct fib_nh *fib_nh) 4612 { 4613 struct net_device *dev = fib_nh->fib_nh_dev; 4614 struct in_device *in_dev; 4615 int err; 4616 4617 nh->nhgi = nh_grp->nhgi; 4618 nh->key.fib_nh = fib_nh; 4619 #ifdef CONFIG_IP_ROUTE_MULTIPATH 4620 nh->nh_weight = fib_nh->fib_nh_weight; 4621 #else 4622 nh->nh_weight = 1; 4623 #endif 4624 memcpy(&nh->gw_addr, &fib_nh->fib_nh_gw4, sizeof(fib_nh->fib_nh_gw4)); 4625 nh->neigh_tbl = &arp_tbl; 4626 err = mlxsw_sp_nexthop_insert(mlxsw_sp, nh); 4627 if (err) 4628 return err; 4629 4630 err = mlxsw_sp_nexthop_counter_enable(mlxsw_sp, nh); 4631 if (err) 4632 goto err_counter_enable; 4633 4634 list_add_tail(&nh->router_list_node, &mlxsw_sp->router->nexthop_list); 4635 4636 if (!dev) 4637 return 0; 4638 nh->ifindex = dev->ifindex; 4639 4640 rcu_read_lock(); 4641 in_dev = __in_dev_get_rcu(dev); 4642 if (in_dev && IN_DEV_IGNORE_ROUTES_WITH_LINKDOWN(in_dev) && 4643 fib_nh->fib_nh_flags & RTNH_F_LINKDOWN) { 4644 rcu_read_unlock(); 4645 return 0; 4646 } 4647 rcu_read_unlock(); 4648 4649 err = mlxsw_sp_nexthop_type_init(mlxsw_sp, nh, dev); 4650 if (err) 4651 goto err_nexthop_neigh_init; 4652 4653 return 0; 4654 4655 err_nexthop_neigh_init: 4656 list_del(&nh->router_list_node); 4657 mlxsw_sp_nexthop_counter_disable(mlxsw_sp, nh); 4658 err_counter_enable: 4659 mlxsw_sp_nexthop_remove(mlxsw_sp, nh); 4660 return err; 4661 } 4662 4663 static void mlxsw_sp_nexthop4_fini(struct mlxsw_sp *mlxsw_sp, 4664 struct mlxsw_sp_nexthop *nh) 4665 { 4666 mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh); 4667 list_del(&nh->router_list_node); 4668 mlxsw_sp_nexthop_counter_disable(mlxsw_sp, nh); 4669 mlxsw_sp_nexthop_remove(mlxsw_sp, nh); 4670 } 4671 4672 static void mlxsw_sp_nexthop4_event(struct mlxsw_sp *mlxsw_sp, 4673 unsigned long event, struct fib_nh *fib_nh) 4674 { 4675 struct mlxsw_sp_nexthop_key key; 4676 struct mlxsw_sp_nexthop *nh; 4677 4678 key.fib_nh = fib_nh; 4679 nh = mlxsw_sp_nexthop_lookup(mlxsw_sp, key); 4680 if (!nh) 4681 return; 4682 4683 switch (event) { 4684 case FIB_EVENT_NH_ADD: 4685 mlxsw_sp_nexthop_type_init(mlxsw_sp, nh, fib_nh->fib_nh_dev); 4686 break; 4687 case FIB_EVENT_NH_DEL: 4688 mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh); 4689 break; 4690 } 4691 4692 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nhgi->nh_grp); 4693 } 4694 4695 static void mlxsw_sp_nexthop_rif_update(struct mlxsw_sp *mlxsw_sp, 4696 struct mlxsw_sp_rif *rif) 4697 { 4698 struct net_device *dev = mlxsw_sp_rif_dev(rif); 4699 struct mlxsw_sp_nexthop *nh; 4700 bool removing; 4701 4702 list_for_each_entry(nh, &rif->crif->nexthop_list, crif_list_node) { 4703 switch (nh->type) { 4704 case MLXSW_SP_NEXTHOP_TYPE_ETH: 4705 removing = false; 4706 break; 4707 case MLXSW_SP_NEXTHOP_TYPE_IPIP: 4708 removing = !mlxsw_sp_ipip_netdev_ul_up(dev); 4709 break; 4710 default: 4711 WARN_ON(1); 4712 continue; 4713 } 4714 4715 __mlxsw_sp_nexthop_neigh_update(nh, removing); 4716 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nhgi->nh_grp); 4717 } 4718 } 4719 4720 static int mlxsw_sp_nexthop_rif_made_sync(struct mlxsw_sp *mlxsw_sp, 4721 struct mlxsw_sp_rif *rif) 4722 { 4723 struct mlxsw_sp_nexthop *nh, *tmp; 4724 unsigned int n = 0; 4725 int err; 4726 4727 list_for_each_entry_safe(nh, tmp, &rif->crif->nexthop_list, 4728 crif_list_node) { 4729 err = mlxsw_sp_nexthop_type_rif_made(mlxsw_sp, nh); 4730 if (err) 4731 goto err_nexthop_type_rif; 4732 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nhgi->nh_grp); 4733 n++; 4734 } 4735 4736 return 0; 4737 4738 err_nexthop_type_rif: 4739 list_for_each_entry_safe(nh, tmp, &rif->crif->nexthop_list, 4740 crif_list_node) { 4741 if (!n--) 4742 break; 4743 mlxsw_sp_nexthop_type_rif_gone(mlxsw_sp, nh); 4744 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nhgi->nh_grp); 4745 } 4746 return err; 4747 } 4748 4749 static void mlxsw_sp_nexthop_rif_gone_sync(struct mlxsw_sp *mlxsw_sp, 4750 struct mlxsw_sp_rif *rif) 4751 { 4752 struct mlxsw_sp_nexthop *nh, *tmp; 4753 4754 list_for_each_entry_safe(nh, tmp, &rif->crif->nexthop_list, 4755 crif_list_node) { 4756 mlxsw_sp_nexthop_type_rif_gone(mlxsw_sp, nh); 4757 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nhgi->nh_grp); 4758 } 4759 } 4760 4761 static int mlxsw_sp_adj_trap_entry_init(struct mlxsw_sp *mlxsw_sp) 4762 { 4763 enum mlxsw_reg_ratr_trap_action trap_action; 4764 char ratr_pl[MLXSW_REG_RATR_LEN]; 4765 int err; 4766 4767 err = mlxsw_sp_kvdl_alloc(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 1, 4768 &mlxsw_sp->router->adj_trap_index); 4769 if (err) 4770 return err; 4771 4772 trap_action = MLXSW_REG_RATR_TRAP_ACTION_TRAP; 4773 mlxsw_reg_ratr_pack(ratr_pl, MLXSW_REG_RATR_OP_WRITE_WRITE_ENTRY, true, 4774 MLXSW_REG_RATR_TYPE_ETHERNET, 4775 mlxsw_sp->router->adj_trap_index, 4776 mlxsw_sp->router->lb_crif->rif->rif_index); 4777 mlxsw_reg_ratr_trap_action_set(ratr_pl, trap_action); 4778 mlxsw_reg_ratr_trap_id_set(ratr_pl, MLXSW_TRAP_ID_RTR_EGRESS0); 4779 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ratr), ratr_pl); 4780 if (err) 4781 goto err_ratr_write; 4782 4783 return 0; 4784 4785 err_ratr_write: 4786 mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 1, 4787 mlxsw_sp->router->adj_trap_index); 4788 return err; 4789 } 4790 4791 static void mlxsw_sp_adj_trap_entry_fini(struct mlxsw_sp *mlxsw_sp) 4792 { 4793 mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 1, 4794 mlxsw_sp->router->adj_trap_index); 4795 } 4796 4797 static int mlxsw_sp_nexthop_group_inc(struct mlxsw_sp *mlxsw_sp) 4798 { 4799 int err; 4800 4801 if (refcount_inc_not_zero(&mlxsw_sp->router->num_groups)) 4802 return 0; 4803 4804 err = mlxsw_sp_adj_trap_entry_init(mlxsw_sp); 4805 if (err) 4806 return err; 4807 4808 refcount_set(&mlxsw_sp->router->num_groups, 1); 4809 4810 return 0; 4811 } 4812 4813 static void mlxsw_sp_nexthop_group_dec(struct mlxsw_sp *mlxsw_sp) 4814 { 4815 if (!refcount_dec_and_test(&mlxsw_sp->router->num_groups)) 4816 return; 4817 4818 mlxsw_sp_adj_trap_entry_fini(mlxsw_sp); 4819 } 4820 4821 static void 4822 mlxsw_sp_nh_grp_activity_get(struct mlxsw_sp *mlxsw_sp, 4823 const struct mlxsw_sp_nexthop_group *nh_grp, 4824 unsigned long *activity) 4825 { 4826 char *ratrad_pl; 4827 int i, err; 4828 4829 ratrad_pl = kmalloc(MLXSW_REG_RATRAD_LEN, GFP_KERNEL); 4830 if (!ratrad_pl) 4831 return; 4832 4833 mlxsw_reg_ratrad_pack(ratrad_pl, nh_grp->nhgi->adj_index, 4834 nh_grp->nhgi->count); 4835 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ratrad), ratrad_pl); 4836 if (err) 4837 goto out; 4838 4839 for (i = 0; i < nh_grp->nhgi->count; i++) { 4840 if (!mlxsw_reg_ratrad_activity_vector_get(ratrad_pl, i)) 4841 continue; 4842 bitmap_set(activity, i, 1); 4843 } 4844 4845 out: 4846 kfree(ratrad_pl); 4847 } 4848 4849 #define MLXSW_SP_NH_GRP_ACTIVITY_UPDATE_INTERVAL 1000 /* ms */ 4850 4851 static void 4852 mlxsw_sp_nh_grp_activity_update(struct mlxsw_sp *mlxsw_sp, 4853 const struct mlxsw_sp_nexthop_group *nh_grp) 4854 { 4855 unsigned long *activity; 4856 4857 activity = bitmap_zalloc(nh_grp->nhgi->count, GFP_KERNEL); 4858 if (!activity) 4859 return; 4860 4861 mlxsw_sp_nh_grp_activity_get(mlxsw_sp, nh_grp, activity); 4862 nexthop_res_grp_activity_update(mlxsw_sp_net(mlxsw_sp), nh_grp->obj.id, 4863 nh_grp->nhgi->count, activity); 4864 4865 bitmap_free(activity); 4866 } 4867 4868 static void 4869 mlxsw_sp_nh_grp_activity_work_schedule(struct mlxsw_sp *mlxsw_sp) 4870 { 4871 unsigned int interval = MLXSW_SP_NH_GRP_ACTIVITY_UPDATE_INTERVAL; 4872 4873 mlxsw_core_schedule_dw(&mlxsw_sp->router->nh_grp_activity_dw, 4874 msecs_to_jiffies(interval)); 4875 } 4876 4877 static void mlxsw_sp_nh_grp_activity_work(struct work_struct *work) 4878 { 4879 struct mlxsw_sp_nexthop_group_info *nhgi; 4880 struct mlxsw_sp_router *router; 4881 bool reschedule = false; 4882 4883 router = container_of(work, struct mlxsw_sp_router, 4884 nh_grp_activity_dw.work); 4885 4886 mutex_lock(&router->lock); 4887 4888 list_for_each_entry(nhgi, &router->nh_res_grp_list, list) { 4889 mlxsw_sp_nh_grp_activity_update(router->mlxsw_sp, nhgi->nh_grp); 4890 reschedule = true; 4891 } 4892 4893 mutex_unlock(&router->lock); 4894 4895 if (!reschedule) 4896 return; 4897 mlxsw_sp_nh_grp_activity_work_schedule(router->mlxsw_sp); 4898 } 4899 4900 static int 4901 mlxsw_sp_nexthop_obj_single_validate(struct mlxsw_sp *mlxsw_sp, 4902 const struct nh_notifier_single_info *nh, 4903 struct netlink_ext_ack *extack) 4904 { 4905 int err = -EINVAL; 4906 4907 if (nh->is_fdb) 4908 NL_SET_ERR_MSG_MOD(extack, "FDB nexthops are not supported"); 4909 else if (nh->has_encap) 4910 NL_SET_ERR_MSG_MOD(extack, "Encapsulating nexthops are not supported"); 4911 else 4912 err = 0; 4913 4914 return err; 4915 } 4916 4917 static int 4918 mlxsw_sp_nexthop_obj_group_entry_validate(struct mlxsw_sp *mlxsw_sp, 4919 const struct nh_notifier_single_info *nh, 4920 struct netlink_ext_ack *extack) 4921 { 4922 int err; 4923 4924 err = mlxsw_sp_nexthop_obj_single_validate(mlxsw_sp, nh, extack); 4925 if (err) 4926 return err; 4927 4928 /* Device only nexthops with an IPIP device are programmed as 4929 * encapsulating adjacency entries. 4930 */ 4931 if (!nh->gw_family && !nh->is_reject && 4932 !mlxsw_sp_netdev_ipip_type(mlxsw_sp, nh->dev, NULL)) { 4933 NL_SET_ERR_MSG_MOD(extack, "Nexthop group entry does not have a gateway"); 4934 return -EINVAL; 4935 } 4936 4937 return 0; 4938 } 4939 4940 static int 4941 mlxsw_sp_nexthop_obj_group_validate(struct mlxsw_sp *mlxsw_sp, 4942 const struct nh_notifier_grp_info *nh_grp, 4943 struct netlink_ext_ack *extack) 4944 { 4945 int i; 4946 4947 if (nh_grp->is_fdb) { 4948 NL_SET_ERR_MSG_MOD(extack, "FDB nexthop groups are not supported"); 4949 return -EINVAL; 4950 } 4951 4952 for (i = 0; i < nh_grp->num_nh; i++) { 4953 const struct nh_notifier_single_info *nh; 4954 int err; 4955 4956 nh = &nh_grp->nh_entries[i].nh; 4957 err = mlxsw_sp_nexthop_obj_group_entry_validate(mlxsw_sp, nh, 4958 extack); 4959 if (err) 4960 return err; 4961 } 4962 4963 return 0; 4964 } 4965 4966 static int 4967 mlxsw_sp_nexthop_obj_res_group_size_validate(struct mlxsw_sp *mlxsw_sp, 4968 const struct nh_notifier_res_table_info *nh_res_table, 4969 struct netlink_ext_ack *extack) 4970 { 4971 unsigned int alloc_size; 4972 bool valid_size = false; 4973 int err, i; 4974 4975 if (nh_res_table->num_nh_buckets < 32) { 4976 NL_SET_ERR_MSG_MOD(extack, "Minimum number of buckets is 32"); 4977 return -EINVAL; 4978 } 4979 4980 for (i = 0; i < mlxsw_sp->router->adj_grp_size_ranges_count; i++) { 4981 const struct mlxsw_sp_adj_grp_size_range *size_range; 4982 4983 size_range = &mlxsw_sp->router->adj_grp_size_ranges[i]; 4984 4985 if (nh_res_table->num_nh_buckets >= size_range->start && 4986 nh_res_table->num_nh_buckets <= size_range->end) { 4987 valid_size = true; 4988 break; 4989 } 4990 } 4991 4992 if (!valid_size) { 4993 NL_SET_ERR_MSG_MOD(extack, "Invalid number of buckets"); 4994 return -EINVAL; 4995 } 4996 4997 err = mlxsw_sp_kvdl_alloc_count_query(mlxsw_sp, 4998 MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 4999 nh_res_table->num_nh_buckets, 5000 &alloc_size); 5001 if (err || nh_res_table->num_nh_buckets != alloc_size) { 5002 NL_SET_ERR_MSG_MOD(extack, "Number of buckets does not fit allocation size of any KVDL partition"); 5003 return -EINVAL; 5004 } 5005 5006 return 0; 5007 } 5008 5009 static int 5010 mlxsw_sp_nexthop_obj_res_group_validate(struct mlxsw_sp *mlxsw_sp, 5011 const struct nh_notifier_res_table_info *nh_res_table, 5012 struct netlink_ext_ack *extack) 5013 { 5014 int err; 5015 u16 i; 5016 5017 err = mlxsw_sp_nexthop_obj_res_group_size_validate(mlxsw_sp, 5018 nh_res_table, 5019 extack); 5020 if (err) 5021 return err; 5022 5023 for (i = 0; i < nh_res_table->num_nh_buckets; i++) { 5024 const struct nh_notifier_single_info *nh; 5025 int err; 5026 5027 nh = &nh_res_table->nhs[i]; 5028 err = mlxsw_sp_nexthop_obj_group_entry_validate(mlxsw_sp, nh, 5029 extack); 5030 if (err) 5031 return err; 5032 } 5033 5034 return 0; 5035 } 5036 5037 static int mlxsw_sp_nexthop_obj_validate(struct mlxsw_sp *mlxsw_sp, 5038 unsigned long event, 5039 struct nh_notifier_info *info) 5040 { 5041 struct nh_notifier_single_info *nh; 5042 5043 if (event != NEXTHOP_EVENT_REPLACE && 5044 event != NEXTHOP_EVENT_RES_TABLE_PRE_REPLACE && 5045 event != NEXTHOP_EVENT_BUCKET_REPLACE) 5046 return 0; 5047 5048 switch (info->type) { 5049 case NH_NOTIFIER_INFO_TYPE_SINGLE: 5050 return mlxsw_sp_nexthop_obj_single_validate(mlxsw_sp, info->nh, 5051 info->extack); 5052 case NH_NOTIFIER_INFO_TYPE_GRP: 5053 return mlxsw_sp_nexthop_obj_group_validate(mlxsw_sp, 5054 info->nh_grp, 5055 info->extack); 5056 case NH_NOTIFIER_INFO_TYPE_RES_TABLE: 5057 return mlxsw_sp_nexthop_obj_res_group_validate(mlxsw_sp, 5058 info->nh_res_table, 5059 info->extack); 5060 case NH_NOTIFIER_INFO_TYPE_RES_BUCKET: 5061 nh = &info->nh_res_bucket->new_nh; 5062 return mlxsw_sp_nexthop_obj_group_entry_validate(mlxsw_sp, nh, 5063 info->extack); 5064 default: 5065 NL_SET_ERR_MSG_MOD(info->extack, "Unsupported nexthop type"); 5066 return -EOPNOTSUPP; 5067 } 5068 } 5069 5070 static bool mlxsw_sp_nexthop_obj_is_gateway(struct mlxsw_sp *mlxsw_sp, 5071 const struct nh_notifier_info *info) 5072 { 5073 const struct net_device *dev; 5074 5075 switch (info->type) { 5076 case NH_NOTIFIER_INFO_TYPE_SINGLE: 5077 dev = info->nh->dev; 5078 return info->nh->gw_family || info->nh->is_reject || 5079 mlxsw_sp_netdev_ipip_type(mlxsw_sp, dev, NULL); 5080 case NH_NOTIFIER_INFO_TYPE_GRP: 5081 case NH_NOTIFIER_INFO_TYPE_RES_TABLE: 5082 /* Already validated earlier. */ 5083 return true; 5084 default: 5085 return false; 5086 } 5087 } 5088 5089 static void mlxsw_sp_nexthop_obj_blackhole_init(struct mlxsw_sp *mlxsw_sp, 5090 struct mlxsw_sp_nexthop *nh) 5091 { 5092 nh->action = MLXSW_SP_NEXTHOP_ACTION_DISCARD; 5093 nh->should_offload = 1; 5094 /* While nexthops that discard packets do not forward packets 5095 * via an egress RIF, they still need to be programmed using a 5096 * valid RIF, so use the loopback RIF created during init. 5097 */ 5098 nh->crif = mlxsw_sp->router->lb_crif; 5099 } 5100 5101 static void mlxsw_sp_nexthop_obj_blackhole_fini(struct mlxsw_sp *mlxsw_sp, 5102 struct mlxsw_sp_nexthop *nh) 5103 { 5104 nh->crif = NULL; 5105 nh->should_offload = 0; 5106 } 5107 5108 static int 5109 mlxsw_sp_nexthop_obj_init(struct mlxsw_sp *mlxsw_sp, 5110 struct mlxsw_sp_nexthop_group *nh_grp, 5111 struct mlxsw_sp_nexthop *nh, 5112 struct nh_notifier_single_info *nh_obj, int weight) 5113 { 5114 struct net_device *dev = nh_obj->dev; 5115 int err; 5116 5117 nh->nhgi = nh_grp->nhgi; 5118 nh->nh_weight = weight; 5119 5120 switch (nh_obj->gw_family) { 5121 case AF_INET: 5122 memcpy(&nh->gw_addr, &nh_obj->ipv4, sizeof(nh_obj->ipv4)); 5123 nh->neigh_tbl = &arp_tbl; 5124 break; 5125 case AF_INET6: 5126 memcpy(&nh->gw_addr, &nh_obj->ipv6, sizeof(nh_obj->ipv6)); 5127 #if IS_ENABLED(CONFIG_IPV6) 5128 nh->neigh_tbl = &nd_tbl; 5129 #endif 5130 break; 5131 } 5132 5133 list_add_tail(&nh->router_list_node, &mlxsw_sp->router->nexthop_list); 5134 nh->ifindex = dev->ifindex; 5135 nh->id = nh_obj->id; 5136 5137 err = mlxsw_sp_nexthop_type_init(mlxsw_sp, nh, dev); 5138 if (err) 5139 goto err_type_init; 5140 5141 if (nh_obj->is_reject) 5142 mlxsw_sp_nexthop_obj_blackhole_init(mlxsw_sp, nh); 5143 5144 /* In a resilient nexthop group, all the nexthops must be written to 5145 * the adjacency table. Even if they do not have a valid neighbour or 5146 * RIF. 5147 */ 5148 if (nh_grp->nhgi->is_resilient && !nh->should_offload) { 5149 nh->action = MLXSW_SP_NEXTHOP_ACTION_TRAP; 5150 nh->should_offload = 1; 5151 } 5152 5153 return 0; 5154 5155 err_type_init: 5156 list_del(&nh->router_list_node); 5157 return err; 5158 } 5159 5160 static void mlxsw_sp_nexthop_obj_fini(struct mlxsw_sp *mlxsw_sp, 5161 struct mlxsw_sp_nexthop *nh) 5162 { 5163 if (nh->action == MLXSW_SP_NEXTHOP_ACTION_DISCARD) 5164 mlxsw_sp_nexthop_obj_blackhole_fini(mlxsw_sp, nh); 5165 mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh); 5166 list_del(&nh->router_list_node); 5167 mlxsw_sp_nexthop_counter_disable(mlxsw_sp, nh); 5168 nh->should_offload = 0; 5169 } 5170 5171 static int 5172 mlxsw_sp_nexthop_obj_group_info_init(struct mlxsw_sp *mlxsw_sp, 5173 struct mlxsw_sp_nexthop_group *nh_grp, 5174 struct nh_notifier_info *info) 5175 { 5176 struct mlxsw_sp_nexthop_group_info *nhgi; 5177 struct mlxsw_sp_nexthop *nh; 5178 bool is_resilient = false; 5179 bool hw_stats = false; 5180 unsigned int nhs; 5181 int err, i; 5182 5183 switch (info->type) { 5184 case NH_NOTIFIER_INFO_TYPE_SINGLE: 5185 nhs = 1; 5186 break; 5187 case NH_NOTIFIER_INFO_TYPE_GRP: 5188 nhs = info->nh_grp->num_nh; 5189 hw_stats = info->nh_grp->hw_stats; 5190 break; 5191 case NH_NOTIFIER_INFO_TYPE_RES_TABLE: 5192 nhs = info->nh_res_table->num_nh_buckets; 5193 hw_stats = info->nh_res_table->hw_stats; 5194 is_resilient = true; 5195 break; 5196 default: 5197 return -EINVAL; 5198 } 5199 5200 nhgi = kzalloc_flex(*nhgi, nexthops, nhs); 5201 if (!nhgi) 5202 return -ENOMEM; 5203 nh_grp->nhgi = nhgi; 5204 nhgi->nh_grp = nh_grp; 5205 nhgi->gateway = mlxsw_sp_nexthop_obj_is_gateway(mlxsw_sp, info); 5206 nhgi->is_resilient = is_resilient; 5207 nhgi->count = nhs; 5208 nhgi->hw_stats = hw_stats; 5209 5210 xa_init_flags(&nhgi->nexthop_counters, XA_FLAGS_ALLOC1); 5211 5212 for (i = 0; i < nhgi->count; i++) { 5213 struct nh_notifier_single_info *nh_obj; 5214 int weight; 5215 5216 nh = &nhgi->nexthops[i]; 5217 switch (info->type) { 5218 case NH_NOTIFIER_INFO_TYPE_SINGLE: 5219 nh_obj = info->nh; 5220 weight = 1; 5221 break; 5222 case NH_NOTIFIER_INFO_TYPE_GRP: 5223 nh_obj = &info->nh_grp->nh_entries[i].nh; 5224 weight = info->nh_grp->nh_entries[i].weight; 5225 break; 5226 case NH_NOTIFIER_INFO_TYPE_RES_TABLE: 5227 nh_obj = &info->nh_res_table->nhs[i]; 5228 weight = 1; 5229 break; 5230 default: 5231 err = -EINVAL; 5232 goto err_nexthop_obj_init; 5233 } 5234 err = mlxsw_sp_nexthop_obj_init(mlxsw_sp, nh_grp, nh, nh_obj, 5235 weight); 5236 if (err) 5237 goto err_nexthop_obj_init; 5238 } 5239 err = mlxsw_sp_nexthop_group_inc(mlxsw_sp); 5240 if (err) 5241 goto err_group_inc; 5242 err = mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp); 5243 if (err) { 5244 NL_SET_ERR_MSG_MOD(info->extack, "Failed to write adjacency entries to the device"); 5245 goto err_group_refresh; 5246 } 5247 5248 /* Add resilient nexthop groups to a list so that the activity of their 5249 * nexthop buckets will be periodically queried and cleared. 5250 */ 5251 if (nhgi->is_resilient) { 5252 if (list_empty(&mlxsw_sp->router->nh_res_grp_list)) 5253 mlxsw_sp_nh_grp_activity_work_schedule(mlxsw_sp); 5254 list_add(&nhgi->list, &mlxsw_sp->router->nh_res_grp_list); 5255 } 5256 5257 return 0; 5258 5259 err_group_refresh: 5260 mlxsw_sp_nexthop_group_dec(mlxsw_sp); 5261 err_group_inc: 5262 i = nhgi->count; 5263 err_nexthop_obj_init: 5264 for (i--; i >= 0; i--) { 5265 nh = &nhgi->nexthops[i]; 5266 mlxsw_sp_nexthop_obj_fini(mlxsw_sp, nh); 5267 } 5268 kfree(nhgi); 5269 return err; 5270 } 5271 5272 static void 5273 mlxsw_sp_nexthop_obj_group_info_fini(struct mlxsw_sp *mlxsw_sp, 5274 struct mlxsw_sp_nexthop_group *nh_grp) 5275 { 5276 struct mlxsw_sp_nexthop_group_info *nhgi = nh_grp->nhgi; 5277 struct mlxsw_sp_router *router = mlxsw_sp->router; 5278 int i; 5279 5280 if (nhgi->is_resilient) { 5281 list_del(&nhgi->list); 5282 if (list_empty(&mlxsw_sp->router->nh_res_grp_list)) 5283 cancel_delayed_work(&router->nh_grp_activity_dw); 5284 } 5285 5286 mlxsw_sp_nexthop_group_dec(mlxsw_sp); 5287 for (i = nhgi->count - 1; i >= 0; i--) { 5288 struct mlxsw_sp_nexthop *nh = &nhgi->nexthops[i]; 5289 5290 mlxsw_sp_nexthop_obj_fini(mlxsw_sp, nh); 5291 } 5292 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp); 5293 WARN_ON_ONCE(nhgi->adj_index_valid); 5294 WARN_ON(!xa_empty(&nhgi->nexthop_counters)); 5295 xa_destroy(&nhgi->nexthop_counters); 5296 kfree(nhgi); 5297 } 5298 5299 static struct mlxsw_sp_nexthop_group * 5300 mlxsw_sp_nexthop_obj_group_create(struct mlxsw_sp *mlxsw_sp, 5301 struct nh_notifier_info *info) 5302 { 5303 struct mlxsw_sp_nexthop_group *nh_grp; 5304 int err; 5305 5306 nh_grp = kzalloc_obj(*nh_grp); 5307 if (!nh_grp) 5308 return ERR_PTR(-ENOMEM); 5309 INIT_LIST_HEAD(&nh_grp->vr_list); 5310 err = rhashtable_init(&nh_grp->vr_ht, 5311 &mlxsw_sp_nexthop_group_vr_ht_params); 5312 if (err) 5313 goto err_nexthop_group_vr_ht_init; 5314 INIT_LIST_HEAD(&nh_grp->fib_list); 5315 nh_grp->type = MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ; 5316 nh_grp->obj.id = info->id; 5317 5318 err = mlxsw_sp_nexthop_obj_group_info_init(mlxsw_sp, nh_grp, info); 5319 if (err) 5320 goto err_nexthop_group_info_init; 5321 5322 nh_grp->can_destroy = false; 5323 5324 return nh_grp; 5325 5326 err_nexthop_group_info_init: 5327 rhashtable_destroy(&nh_grp->vr_ht); 5328 err_nexthop_group_vr_ht_init: 5329 kfree(nh_grp); 5330 return ERR_PTR(err); 5331 } 5332 5333 static void 5334 mlxsw_sp_nexthop_obj_group_destroy(struct mlxsw_sp *mlxsw_sp, 5335 struct mlxsw_sp_nexthop_group *nh_grp) 5336 { 5337 if (!nh_grp->can_destroy) 5338 return; 5339 mlxsw_sp_nexthop_obj_group_info_fini(mlxsw_sp, nh_grp); 5340 WARN_ON_ONCE(!list_empty(&nh_grp->fib_list)); 5341 WARN_ON_ONCE(!list_empty(&nh_grp->vr_list)); 5342 rhashtable_destroy(&nh_grp->vr_ht); 5343 kfree(nh_grp); 5344 } 5345 5346 static struct mlxsw_sp_nexthop_group * 5347 mlxsw_sp_nexthop_obj_group_lookup(struct mlxsw_sp *mlxsw_sp, u32 id) 5348 { 5349 struct mlxsw_sp_nexthop_group_cmp_arg cmp_arg; 5350 5351 cmp_arg.type = MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ; 5352 cmp_arg.id = id; 5353 return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_group_ht, 5354 &cmp_arg, 5355 mlxsw_sp_nexthop_group_ht_params); 5356 } 5357 5358 static int mlxsw_sp_nexthop_obj_group_add(struct mlxsw_sp *mlxsw_sp, 5359 struct mlxsw_sp_nexthop_group *nh_grp) 5360 { 5361 return mlxsw_sp_nexthop_group_insert(mlxsw_sp, nh_grp); 5362 } 5363 5364 static int 5365 mlxsw_sp_nexthop_obj_group_replace(struct mlxsw_sp *mlxsw_sp, 5366 struct mlxsw_sp_nexthop_group *nh_grp, 5367 struct mlxsw_sp_nexthop_group *old_nh_grp, 5368 struct netlink_ext_ack *extack) 5369 { 5370 struct mlxsw_sp_nexthop_group_info *old_nhgi = old_nh_grp->nhgi; 5371 struct mlxsw_sp_nexthop_group_info *new_nhgi = nh_grp->nhgi; 5372 int err; 5373 5374 old_nh_grp->nhgi = new_nhgi; 5375 new_nhgi->nh_grp = old_nh_grp; 5376 nh_grp->nhgi = old_nhgi; 5377 old_nhgi->nh_grp = nh_grp; 5378 5379 if (old_nhgi->adj_index_valid && new_nhgi->adj_index_valid) { 5380 /* Both the old adjacency index and the new one are valid. 5381 * Routes are currently using the old one. Tell the device to 5382 * replace the old adjacency index with the new one. 5383 */ 5384 err = mlxsw_sp_adj_index_mass_update(mlxsw_sp, old_nh_grp, 5385 old_nhgi->adj_index, 5386 old_nhgi->ecmp_size); 5387 if (err) { 5388 NL_SET_ERR_MSG_MOD(extack, "Failed to replace old adjacency index with new one"); 5389 goto err_out; 5390 } 5391 } else if (old_nhgi->adj_index_valid && !new_nhgi->adj_index_valid) { 5392 /* The old adjacency index is valid, while the new one is not. 5393 * Iterate over all the routes using the group and change them 5394 * to trap packets to the CPU. 5395 */ 5396 err = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, old_nh_grp); 5397 if (err) { 5398 NL_SET_ERR_MSG_MOD(extack, "Failed to update routes to trap packets"); 5399 goto err_out; 5400 } 5401 } else if (!old_nhgi->adj_index_valid && new_nhgi->adj_index_valid) { 5402 /* The old adjacency index is invalid, while the new one is. 5403 * Iterate over all the routes using the group and change them 5404 * to forward packets using the new valid index. 5405 */ 5406 err = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, old_nh_grp); 5407 if (err) { 5408 NL_SET_ERR_MSG_MOD(extack, "Failed to update routes to forward packets"); 5409 goto err_out; 5410 } 5411 } 5412 5413 /* Make sure the flags are set / cleared based on the new nexthop group 5414 * information. 5415 */ 5416 mlxsw_sp_nexthop_obj_group_offload_refresh(mlxsw_sp, old_nh_grp); 5417 5418 /* At this point 'nh_grp' is just a shell that is not used by anyone 5419 * and its nexthop group info is the old info that was just replaced 5420 * with the new one. Remove it. 5421 */ 5422 nh_grp->can_destroy = true; 5423 mlxsw_sp_nexthop_obj_group_destroy(mlxsw_sp, nh_grp); 5424 5425 return 0; 5426 5427 err_out: 5428 old_nhgi->nh_grp = old_nh_grp; 5429 nh_grp->nhgi = new_nhgi; 5430 new_nhgi->nh_grp = nh_grp; 5431 old_nh_grp->nhgi = old_nhgi; 5432 return err; 5433 } 5434 5435 static int mlxsw_sp_nexthop_obj_res_group_pre(struct mlxsw_sp *mlxsw_sp, 5436 struct nh_notifier_info *info) 5437 { 5438 struct nh_notifier_grp_info *grp_info = info->nh_grp; 5439 struct mlxsw_sp_nexthop_group_info *nhgi; 5440 struct mlxsw_sp_nexthop_group *nh_grp; 5441 int err; 5442 int i; 5443 5444 nh_grp = mlxsw_sp_nexthop_obj_group_lookup(mlxsw_sp, info->id); 5445 if (!nh_grp) 5446 return 0; 5447 nhgi = nh_grp->nhgi; 5448 5449 if (nhgi->hw_stats == grp_info->hw_stats) 5450 return 0; 5451 5452 nhgi->hw_stats = grp_info->hw_stats; 5453 5454 for (i = 0; i < nhgi->count; i++) { 5455 struct mlxsw_sp_nexthop *nh = &nhgi->nexthops[i]; 5456 5457 if (nh->offloaded) 5458 nh->update = 1; 5459 } 5460 5461 err = mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp); 5462 if (err) 5463 goto err_group_refresh; 5464 5465 return 0; 5466 5467 err_group_refresh: 5468 nhgi->hw_stats = !grp_info->hw_stats; 5469 return err; 5470 } 5471 5472 static int mlxsw_sp_nexthop_obj_new(struct mlxsw_sp *mlxsw_sp, 5473 struct nh_notifier_info *info) 5474 { 5475 struct mlxsw_sp_nexthop_group *nh_grp, *old_nh_grp; 5476 struct netlink_ext_ack *extack = info->extack; 5477 int err; 5478 5479 nh_grp = mlxsw_sp_nexthop_obj_group_create(mlxsw_sp, info); 5480 if (IS_ERR(nh_grp)) 5481 return PTR_ERR(nh_grp); 5482 5483 old_nh_grp = mlxsw_sp_nexthop_obj_group_lookup(mlxsw_sp, info->id); 5484 if (!old_nh_grp) 5485 err = mlxsw_sp_nexthop_obj_group_add(mlxsw_sp, nh_grp); 5486 else 5487 err = mlxsw_sp_nexthop_obj_group_replace(mlxsw_sp, nh_grp, 5488 old_nh_grp, extack); 5489 5490 if (err) { 5491 nh_grp->can_destroy = true; 5492 mlxsw_sp_nexthop_obj_group_destroy(mlxsw_sp, nh_grp); 5493 } 5494 5495 return err; 5496 } 5497 5498 static void mlxsw_sp_nexthop_obj_del(struct mlxsw_sp *mlxsw_sp, 5499 struct nh_notifier_info *info) 5500 { 5501 struct mlxsw_sp_nexthop_group *nh_grp; 5502 5503 nh_grp = mlxsw_sp_nexthop_obj_group_lookup(mlxsw_sp, info->id); 5504 if (!nh_grp) 5505 return; 5506 5507 nh_grp->can_destroy = true; 5508 mlxsw_sp_nexthop_group_remove(mlxsw_sp, nh_grp); 5509 5510 /* If the group still has routes using it, then defer the delete 5511 * operation until the last route using it is deleted. 5512 */ 5513 if (!list_empty(&nh_grp->fib_list)) 5514 return; 5515 mlxsw_sp_nexthop_obj_group_destroy(mlxsw_sp, nh_grp); 5516 } 5517 5518 static int mlxsw_sp_nexthop_obj_bucket_query(struct mlxsw_sp *mlxsw_sp, 5519 u32 adj_index, char *ratr_pl) 5520 { 5521 MLXSW_REG_ZERO(ratr, ratr_pl); 5522 mlxsw_reg_ratr_op_set(ratr_pl, MLXSW_REG_RATR_OP_QUERY_READ); 5523 mlxsw_reg_ratr_adjacency_index_low_set(ratr_pl, adj_index); 5524 mlxsw_reg_ratr_adjacency_index_high_set(ratr_pl, adj_index >> 16); 5525 5526 return mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ratr), ratr_pl); 5527 } 5528 5529 static int mlxsw_sp_nexthop_obj_bucket_compare(char *ratr_pl, char *ratr_pl_new) 5530 { 5531 /* Clear the opcode and activity on both the old and new payload as 5532 * they are irrelevant for the comparison. 5533 */ 5534 mlxsw_reg_ratr_op_set(ratr_pl, MLXSW_REG_RATR_OP_QUERY_READ); 5535 mlxsw_reg_ratr_a_set(ratr_pl, 0); 5536 mlxsw_reg_ratr_op_set(ratr_pl_new, MLXSW_REG_RATR_OP_QUERY_READ); 5537 mlxsw_reg_ratr_a_set(ratr_pl_new, 0); 5538 5539 /* If the contents of the adjacency entry are consistent with the 5540 * replacement request, then replacement was successful. 5541 */ 5542 if (!memcmp(ratr_pl, ratr_pl_new, MLXSW_REG_RATR_LEN)) 5543 return 0; 5544 5545 return -EINVAL; 5546 } 5547 5548 static int 5549 mlxsw_sp_nexthop_obj_bucket_adj_update(struct mlxsw_sp *mlxsw_sp, 5550 struct mlxsw_sp_nexthop *nh, 5551 struct nh_notifier_info *info) 5552 { 5553 u16 bucket_index = info->nh_res_bucket->bucket_index; 5554 struct netlink_ext_ack *extack = info->extack; 5555 bool force = info->nh_res_bucket->force; 5556 char ratr_pl_new[MLXSW_REG_RATR_LEN]; 5557 char ratr_pl[MLXSW_REG_RATR_LEN]; 5558 u32 adj_index; 5559 int err; 5560 5561 /* No point in trying an atomic replacement if the idle timer interval 5562 * is smaller than the interval in which we query and clear activity. 5563 */ 5564 if (!force && info->nh_res_bucket->idle_timer_ms < 5565 MLXSW_SP_NH_GRP_ACTIVITY_UPDATE_INTERVAL) 5566 force = true; 5567 5568 adj_index = nh->nhgi->adj_index + bucket_index; 5569 err = mlxsw_sp_nexthop_update(mlxsw_sp, adj_index, nh, force, ratr_pl); 5570 if (err) { 5571 NL_SET_ERR_MSG_MOD(extack, "Failed to overwrite nexthop bucket"); 5572 return err; 5573 } 5574 5575 if (!force) { 5576 err = mlxsw_sp_nexthop_obj_bucket_query(mlxsw_sp, adj_index, 5577 ratr_pl_new); 5578 if (err) { 5579 NL_SET_ERR_MSG_MOD(extack, "Failed to query nexthop bucket state after replacement. State might be inconsistent"); 5580 return err; 5581 } 5582 5583 err = mlxsw_sp_nexthop_obj_bucket_compare(ratr_pl, ratr_pl_new); 5584 if (err) { 5585 NL_SET_ERR_MSG_MOD(extack, "Nexthop bucket was not replaced because it was active during replacement"); 5586 return err; 5587 } 5588 } 5589 5590 nh->update = 0; 5591 nh->offloaded = 1; 5592 mlxsw_sp_nexthop_bucket_offload_refresh(mlxsw_sp, nh, bucket_index); 5593 5594 return 0; 5595 } 5596 5597 static int mlxsw_sp_nexthop_obj_bucket_replace(struct mlxsw_sp *mlxsw_sp, 5598 struct nh_notifier_info *info) 5599 { 5600 u16 bucket_index = info->nh_res_bucket->bucket_index; 5601 struct netlink_ext_ack *extack = info->extack; 5602 struct mlxsw_sp_nexthop_group_info *nhgi; 5603 struct nh_notifier_single_info *nh_obj; 5604 struct mlxsw_sp_nexthop_group *nh_grp; 5605 struct mlxsw_sp_nexthop *nh; 5606 int err; 5607 5608 nh_grp = mlxsw_sp_nexthop_obj_group_lookup(mlxsw_sp, info->id); 5609 if (!nh_grp) { 5610 NL_SET_ERR_MSG_MOD(extack, "Nexthop group was not found"); 5611 return -EINVAL; 5612 } 5613 5614 nhgi = nh_grp->nhgi; 5615 5616 if (bucket_index >= nhgi->count) { 5617 NL_SET_ERR_MSG_MOD(extack, "Nexthop bucket index out of range"); 5618 return -EINVAL; 5619 } 5620 5621 nh = &nhgi->nexthops[bucket_index]; 5622 mlxsw_sp_nexthop_obj_fini(mlxsw_sp, nh); 5623 5624 nh_obj = &info->nh_res_bucket->new_nh; 5625 err = mlxsw_sp_nexthop_obj_init(mlxsw_sp, nh_grp, nh, nh_obj, 1); 5626 if (err) { 5627 NL_SET_ERR_MSG_MOD(extack, "Failed to initialize nexthop object for nexthop bucket replacement"); 5628 goto err_nexthop_obj_init; 5629 } 5630 5631 err = mlxsw_sp_nexthop_obj_bucket_adj_update(mlxsw_sp, nh, info); 5632 if (err) 5633 goto err_nexthop_obj_bucket_adj_update; 5634 5635 return 0; 5636 5637 err_nexthop_obj_bucket_adj_update: 5638 mlxsw_sp_nexthop_obj_fini(mlxsw_sp, nh); 5639 err_nexthop_obj_init: 5640 nh_obj = &info->nh_res_bucket->old_nh; 5641 mlxsw_sp_nexthop_obj_init(mlxsw_sp, nh_grp, nh, nh_obj, 1); 5642 /* The old adjacency entry was not overwritten */ 5643 nh->update = 0; 5644 nh->offloaded = 1; 5645 return err; 5646 } 5647 5648 static void 5649 mlxsw_sp_nexthop_obj_mp_hw_stats_get(struct mlxsw_sp *mlxsw_sp, 5650 struct mlxsw_sp_nexthop_group_info *nhgi, 5651 struct nh_notifier_grp_hw_stats_info *info) 5652 { 5653 int nhi; 5654 5655 for (nhi = 0; nhi < info->num_nh; nhi++) { 5656 struct mlxsw_sp_nexthop *nh = &nhgi->nexthops[nhi]; 5657 u64 packets; 5658 int err; 5659 5660 err = mlxsw_sp_nexthop_counter_get(mlxsw_sp, nh, &packets); 5661 if (err) 5662 continue; 5663 5664 nh_grp_hw_stats_report_delta(info, nhi, packets); 5665 } 5666 } 5667 5668 static void 5669 mlxsw_sp_nexthop_obj_res_hw_stats_get(struct mlxsw_sp *mlxsw_sp, 5670 struct mlxsw_sp_nexthop_group_info *nhgi, 5671 struct nh_notifier_grp_hw_stats_info *info) 5672 { 5673 int nhi = -1; 5674 int bucket; 5675 5676 for (bucket = 0; bucket < nhgi->count; bucket++) { 5677 struct mlxsw_sp_nexthop *nh = &nhgi->nexthops[bucket]; 5678 u64 packets; 5679 int err; 5680 5681 if (nhi == -1 || info->stats[nhi].id != nh->id) { 5682 for (nhi = 0; nhi < info->num_nh; nhi++) 5683 if (info->stats[nhi].id == nh->id) 5684 break; 5685 if (WARN_ON_ONCE(nhi == info->num_nh)) { 5686 nhi = -1; 5687 continue; 5688 } 5689 } 5690 5691 err = mlxsw_sp_nexthop_counter_get(mlxsw_sp, nh, &packets); 5692 if (err) 5693 continue; 5694 5695 nh_grp_hw_stats_report_delta(info, nhi, packets); 5696 } 5697 } 5698 5699 static void mlxsw_sp_nexthop_obj_hw_stats_get(struct mlxsw_sp *mlxsw_sp, 5700 struct nh_notifier_info *info) 5701 { 5702 struct mlxsw_sp_nexthop_group_info *nhgi; 5703 struct mlxsw_sp_nexthop_group *nh_grp; 5704 5705 if (info->type != NH_NOTIFIER_INFO_TYPE_GRP_HW_STATS) 5706 return; 5707 5708 nh_grp = mlxsw_sp_nexthop_obj_group_lookup(mlxsw_sp, info->id); 5709 if (!nh_grp) 5710 return; 5711 nhgi = nh_grp->nhgi; 5712 5713 if (nhgi->is_resilient) 5714 mlxsw_sp_nexthop_obj_res_hw_stats_get(mlxsw_sp, nhgi, 5715 info->nh_grp_hw_stats); 5716 else 5717 mlxsw_sp_nexthop_obj_mp_hw_stats_get(mlxsw_sp, nhgi, 5718 info->nh_grp_hw_stats); 5719 } 5720 5721 static int mlxsw_sp_nexthop_obj_event(struct notifier_block *nb, 5722 unsigned long event, void *ptr) 5723 { 5724 struct nh_notifier_info *info = ptr; 5725 struct mlxsw_sp_router *router; 5726 int err = 0; 5727 5728 router = container_of(nb, struct mlxsw_sp_router, nexthop_nb); 5729 err = mlxsw_sp_nexthop_obj_validate(router->mlxsw_sp, event, info); 5730 if (err) 5731 goto out; 5732 5733 mutex_lock(&router->lock); 5734 5735 switch (event) { 5736 case NEXTHOP_EVENT_RES_TABLE_PRE_REPLACE: 5737 err = mlxsw_sp_nexthop_obj_res_group_pre(router->mlxsw_sp, 5738 info); 5739 break; 5740 case NEXTHOP_EVENT_REPLACE: 5741 err = mlxsw_sp_nexthop_obj_new(router->mlxsw_sp, info); 5742 break; 5743 case NEXTHOP_EVENT_DEL: 5744 mlxsw_sp_nexthop_obj_del(router->mlxsw_sp, info); 5745 break; 5746 case NEXTHOP_EVENT_BUCKET_REPLACE: 5747 err = mlxsw_sp_nexthop_obj_bucket_replace(router->mlxsw_sp, 5748 info); 5749 break; 5750 case NEXTHOP_EVENT_HW_STATS_REPORT_DELTA: 5751 mlxsw_sp_nexthop_obj_hw_stats_get(router->mlxsw_sp, info); 5752 break; 5753 default: 5754 break; 5755 } 5756 5757 mutex_unlock(&router->lock); 5758 5759 out: 5760 return notifier_from_errno(err); 5761 } 5762 5763 static bool mlxsw_sp_fi_is_gateway(const struct mlxsw_sp *mlxsw_sp, 5764 struct fib_info *fi) 5765 { 5766 const struct fib_nh *nh = fib_info_nh(fi, 0); 5767 5768 return nh->fib_nh_gw_family || 5769 mlxsw_sp_nexthop4_ipip_type(mlxsw_sp, nh, NULL); 5770 } 5771 5772 static int 5773 mlxsw_sp_nexthop4_group_info_init(struct mlxsw_sp *mlxsw_sp, 5774 struct mlxsw_sp_nexthop_group *nh_grp) 5775 { 5776 unsigned int nhs = fib_info_num_path(nh_grp->ipv4.fi); 5777 struct mlxsw_sp_nexthop_group_info *nhgi; 5778 struct mlxsw_sp_nexthop *nh; 5779 int err, i; 5780 5781 nhgi = kzalloc_flex(*nhgi, nexthops, nhs); 5782 if (!nhgi) 5783 return -ENOMEM; 5784 nh_grp->nhgi = nhgi; 5785 nhgi->nh_grp = nh_grp; 5786 nhgi->gateway = mlxsw_sp_fi_is_gateway(mlxsw_sp, nh_grp->ipv4.fi); 5787 nhgi->count = nhs; 5788 for (i = 0; i < nhgi->count; i++) { 5789 struct fib_nh *fib_nh; 5790 5791 nh = &nhgi->nexthops[i]; 5792 fib_nh = fib_info_nh(nh_grp->ipv4.fi, i); 5793 err = mlxsw_sp_nexthop4_init(mlxsw_sp, nh_grp, nh, fib_nh); 5794 if (err) 5795 goto err_nexthop4_init; 5796 } 5797 err = mlxsw_sp_nexthop_group_inc(mlxsw_sp); 5798 if (err) 5799 goto err_group_inc; 5800 err = mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp); 5801 if (err) 5802 goto err_group_refresh; 5803 5804 return 0; 5805 5806 err_group_refresh: 5807 mlxsw_sp_nexthop_group_dec(mlxsw_sp); 5808 err_group_inc: 5809 i = nhgi->count; 5810 err_nexthop4_init: 5811 for (i--; i >= 0; i--) { 5812 nh = &nhgi->nexthops[i]; 5813 mlxsw_sp_nexthop4_fini(mlxsw_sp, nh); 5814 } 5815 kfree(nhgi); 5816 return err; 5817 } 5818 5819 static void 5820 mlxsw_sp_nexthop4_group_info_fini(struct mlxsw_sp *mlxsw_sp, 5821 struct mlxsw_sp_nexthop_group *nh_grp) 5822 { 5823 struct mlxsw_sp_nexthop_group_info *nhgi = nh_grp->nhgi; 5824 int i; 5825 5826 mlxsw_sp_nexthop_group_dec(mlxsw_sp); 5827 for (i = nhgi->count - 1; i >= 0; i--) { 5828 struct mlxsw_sp_nexthop *nh = &nhgi->nexthops[i]; 5829 5830 mlxsw_sp_nexthop4_fini(mlxsw_sp, nh); 5831 } 5832 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp); 5833 WARN_ON_ONCE(nhgi->adj_index_valid); 5834 kfree(nhgi); 5835 } 5836 5837 static struct mlxsw_sp_nexthop_group * 5838 mlxsw_sp_nexthop4_group_create(struct mlxsw_sp *mlxsw_sp, struct fib_info *fi) 5839 { 5840 struct mlxsw_sp_nexthop_group *nh_grp; 5841 int err; 5842 5843 nh_grp = kzalloc_obj(*nh_grp); 5844 if (!nh_grp) 5845 return ERR_PTR(-ENOMEM); 5846 INIT_LIST_HEAD(&nh_grp->vr_list); 5847 err = rhashtable_init(&nh_grp->vr_ht, 5848 &mlxsw_sp_nexthop_group_vr_ht_params); 5849 if (err) 5850 goto err_nexthop_group_vr_ht_init; 5851 INIT_LIST_HEAD(&nh_grp->fib_list); 5852 nh_grp->type = MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4; 5853 nh_grp->ipv4.fi = fi; 5854 fib_info_hold(fi); 5855 5856 err = mlxsw_sp_nexthop4_group_info_init(mlxsw_sp, nh_grp); 5857 if (err) 5858 goto err_nexthop_group_info_init; 5859 5860 err = mlxsw_sp_nexthop_group_insert(mlxsw_sp, nh_grp); 5861 if (err) 5862 goto err_nexthop_group_insert; 5863 5864 nh_grp->can_destroy = true; 5865 5866 return nh_grp; 5867 5868 err_nexthop_group_insert: 5869 mlxsw_sp_nexthop4_group_info_fini(mlxsw_sp, nh_grp); 5870 err_nexthop_group_info_init: 5871 fib_info_put(fi); 5872 rhashtable_destroy(&nh_grp->vr_ht); 5873 err_nexthop_group_vr_ht_init: 5874 kfree(nh_grp); 5875 return ERR_PTR(err); 5876 } 5877 5878 static void 5879 mlxsw_sp_nexthop4_group_destroy(struct mlxsw_sp *mlxsw_sp, 5880 struct mlxsw_sp_nexthop_group *nh_grp) 5881 { 5882 if (!nh_grp->can_destroy) 5883 return; 5884 mlxsw_sp_nexthop_group_remove(mlxsw_sp, nh_grp); 5885 mlxsw_sp_nexthop4_group_info_fini(mlxsw_sp, nh_grp); 5886 fib_info_put(nh_grp->ipv4.fi); 5887 WARN_ON_ONCE(!list_empty(&nh_grp->vr_list)); 5888 rhashtable_destroy(&nh_grp->vr_ht); 5889 kfree(nh_grp); 5890 } 5891 5892 static int mlxsw_sp_nexthop4_group_get(struct mlxsw_sp *mlxsw_sp, 5893 struct mlxsw_sp_fib_entry *fib_entry, 5894 struct fib_info *fi) 5895 { 5896 struct mlxsw_sp_nexthop_group *nh_grp; 5897 5898 if (fi->nh) { 5899 nh_grp = mlxsw_sp_nexthop_obj_group_lookup(mlxsw_sp, 5900 fi->nh->id); 5901 if (WARN_ON_ONCE(!nh_grp)) 5902 return -EINVAL; 5903 goto out; 5904 } 5905 5906 nh_grp = mlxsw_sp_nexthop4_group_lookup(mlxsw_sp, fi); 5907 if (!nh_grp) { 5908 nh_grp = mlxsw_sp_nexthop4_group_create(mlxsw_sp, fi); 5909 if (IS_ERR(nh_grp)) 5910 return PTR_ERR(nh_grp); 5911 } 5912 out: 5913 list_add_tail(&fib_entry->nexthop_group_node, &nh_grp->fib_list); 5914 fib_entry->nh_group = nh_grp; 5915 return 0; 5916 } 5917 5918 static void mlxsw_sp_nexthop4_group_put(struct mlxsw_sp *mlxsw_sp, 5919 struct mlxsw_sp_fib_entry *fib_entry) 5920 { 5921 struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group; 5922 5923 list_del(&fib_entry->nexthop_group_node); 5924 if (!list_empty(&nh_grp->fib_list)) 5925 return; 5926 5927 if (nh_grp->type == MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ) { 5928 mlxsw_sp_nexthop_obj_group_destroy(mlxsw_sp, nh_grp); 5929 return; 5930 } 5931 5932 mlxsw_sp_nexthop4_group_destroy(mlxsw_sp, nh_grp); 5933 } 5934 5935 static bool 5936 mlxsw_sp_fib4_entry_should_offload(const struct mlxsw_sp_fib_entry *fib_entry) 5937 { 5938 struct mlxsw_sp_fib4_entry *fib4_entry; 5939 5940 fib4_entry = container_of(fib_entry, struct mlxsw_sp_fib4_entry, 5941 common); 5942 return !fib4_entry->dscp; 5943 } 5944 5945 static bool 5946 mlxsw_sp_fib_entry_should_offload(const struct mlxsw_sp_fib_entry *fib_entry) 5947 { 5948 struct mlxsw_sp_nexthop_group *nh_group = fib_entry->nh_group; 5949 5950 switch (fib_entry->fib_node->fib->proto) { 5951 case MLXSW_SP_L3_PROTO_IPV4: 5952 if (!mlxsw_sp_fib4_entry_should_offload(fib_entry)) 5953 return false; 5954 break; 5955 case MLXSW_SP_L3_PROTO_IPV6: 5956 break; 5957 } 5958 5959 switch (fib_entry->type) { 5960 case MLXSW_SP_FIB_ENTRY_TYPE_REMOTE: 5961 return !!nh_group->nhgi->adj_index_valid; 5962 case MLXSW_SP_FIB_ENTRY_TYPE_LOCAL: 5963 return !!mlxsw_sp_nhgi_rif(nh_group->nhgi); 5964 case MLXSW_SP_FIB_ENTRY_TYPE_BLACKHOLE: 5965 case MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP: 5966 case MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP: 5967 return true; 5968 default: 5969 return false; 5970 } 5971 } 5972 5973 static struct mlxsw_sp_nexthop * 5974 mlxsw_sp_rt6_nexthop(struct mlxsw_sp_nexthop_group *nh_grp, 5975 const struct mlxsw_sp_rt6 *mlxsw_sp_rt6) 5976 { 5977 int i; 5978 5979 for (i = 0; i < nh_grp->nhgi->count; i++) { 5980 struct mlxsw_sp_nexthop *nh = &nh_grp->nhgi->nexthops[i]; 5981 struct net_device *dev = mlxsw_sp_nexthop_dev(nh); 5982 struct fib6_info *rt = mlxsw_sp_rt6->rt; 5983 5984 if (dev && dev == rt->fib6_nh->fib_nh_dev && 5985 ipv6_addr_equal((const struct in6_addr *) &nh->gw_addr, 5986 &rt->fib6_nh->fib_nh_gw6)) 5987 return nh; 5988 } 5989 5990 return NULL; 5991 } 5992 5993 static void 5994 mlxsw_sp_fib4_offload_failed_flag_set(struct mlxsw_sp *mlxsw_sp, 5995 struct fib_entry_notifier_info *fen_info) 5996 { 5997 u32 *p_dst = (u32 *) &fen_info->dst; 5998 struct fib_rt_info fri; 5999 6000 fri.fi = fen_info->fi; 6001 fri.tb_id = fen_info->tb_id; 6002 fri.dst = cpu_to_be32(*p_dst); 6003 fri.dst_len = fen_info->dst_len; 6004 fri.dscp = fen_info->dscp; 6005 fri.type = fen_info->type; 6006 fri.offload = false; 6007 fri.trap = false; 6008 fri.offload_failed = true; 6009 fib_alias_hw_flags_set(mlxsw_sp_net(mlxsw_sp), &fri); 6010 } 6011 6012 static void 6013 mlxsw_sp_fib4_entry_hw_flags_set(struct mlxsw_sp *mlxsw_sp, 6014 struct mlxsw_sp_fib_entry *fib_entry) 6015 { 6016 u32 *p_dst = (u32 *) fib_entry->fib_node->key.addr; 6017 int dst_len = fib_entry->fib_node->key.prefix_len; 6018 struct mlxsw_sp_fib4_entry *fib4_entry; 6019 struct fib_rt_info fri; 6020 bool should_offload; 6021 6022 should_offload = mlxsw_sp_fib_entry_should_offload(fib_entry); 6023 fib4_entry = container_of(fib_entry, struct mlxsw_sp_fib4_entry, 6024 common); 6025 fri.fi = fib4_entry->fi; 6026 fri.tb_id = fib4_entry->tb_id; 6027 fri.dst = cpu_to_be32(*p_dst); 6028 fri.dst_len = dst_len; 6029 fri.dscp = fib4_entry->dscp; 6030 fri.type = fib4_entry->type; 6031 fri.offload = should_offload; 6032 fri.trap = !should_offload; 6033 fri.offload_failed = false; 6034 fib_alias_hw_flags_set(mlxsw_sp_net(mlxsw_sp), &fri); 6035 } 6036 6037 static void 6038 mlxsw_sp_fib4_entry_hw_flags_clear(struct mlxsw_sp *mlxsw_sp, 6039 struct mlxsw_sp_fib_entry *fib_entry) 6040 { 6041 u32 *p_dst = (u32 *) fib_entry->fib_node->key.addr; 6042 int dst_len = fib_entry->fib_node->key.prefix_len; 6043 struct mlxsw_sp_fib4_entry *fib4_entry; 6044 struct fib_rt_info fri; 6045 6046 fib4_entry = container_of(fib_entry, struct mlxsw_sp_fib4_entry, 6047 common); 6048 fri.fi = fib4_entry->fi; 6049 fri.tb_id = fib4_entry->tb_id; 6050 fri.dst = cpu_to_be32(*p_dst); 6051 fri.dst_len = dst_len; 6052 fri.dscp = fib4_entry->dscp; 6053 fri.type = fib4_entry->type; 6054 fri.offload = false; 6055 fri.trap = false; 6056 fri.offload_failed = false; 6057 fib_alias_hw_flags_set(mlxsw_sp_net(mlxsw_sp), &fri); 6058 } 6059 6060 #if IS_ENABLED(CONFIG_IPV6) 6061 static void 6062 mlxsw_sp_fib6_offload_failed_flag_set(struct mlxsw_sp *mlxsw_sp, 6063 struct fib6_info **rt_arr, 6064 unsigned int nrt6) 6065 { 6066 int i; 6067 6068 /* In IPv6 a multipath route is represented using multiple routes, so 6069 * we need to set the flags on all of them. 6070 */ 6071 for (i = 0; i < nrt6; i++) 6072 fib6_info_hw_flags_set(mlxsw_sp_net(mlxsw_sp), rt_arr[i], 6073 false, false, true); 6074 } 6075 #else 6076 static void 6077 mlxsw_sp_fib6_offload_failed_flag_set(struct mlxsw_sp *mlxsw_sp, 6078 struct fib6_info **rt_arr, 6079 unsigned int nrt6) 6080 { 6081 } 6082 #endif 6083 6084 #if IS_ENABLED(CONFIG_IPV6) 6085 static void 6086 mlxsw_sp_fib6_entry_hw_flags_set(struct mlxsw_sp *mlxsw_sp, 6087 struct mlxsw_sp_fib_entry *fib_entry) 6088 { 6089 struct mlxsw_sp_fib6_entry *fib6_entry; 6090 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 6091 bool should_offload; 6092 6093 should_offload = mlxsw_sp_fib_entry_should_offload(fib_entry); 6094 6095 /* In IPv6 a multipath route is represented using multiple routes, so 6096 * we need to set the flags on all of them. 6097 */ 6098 fib6_entry = container_of(fib_entry, struct mlxsw_sp_fib6_entry, 6099 common); 6100 list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) 6101 fib6_info_hw_flags_set(mlxsw_sp_net(mlxsw_sp), mlxsw_sp_rt6->rt, 6102 should_offload, !should_offload, false); 6103 } 6104 #else 6105 static void 6106 mlxsw_sp_fib6_entry_hw_flags_set(struct mlxsw_sp *mlxsw_sp, 6107 struct mlxsw_sp_fib_entry *fib_entry) 6108 { 6109 } 6110 #endif 6111 6112 #if IS_ENABLED(CONFIG_IPV6) 6113 static void 6114 mlxsw_sp_fib6_entry_hw_flags_clear(struct mlxsw_sp *mlxsw_sp, 6115 struct mlxsw_sp_fib_entry *fib_entry) 6116 { 6117 struct mlxsw_sp_fib6_entry *fib6_entry; 6118 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 6119 6120 fib6_entry = container_of(fib_entry, struct mlxsw_sp_fib6_entry, 6121 common); 6122 list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) 6123 fib6_info_hw_flags_set(mlxsw_sp_net(mlxsw_sp), mlxsw_sp_rt6->rt, 6124 false, false, false); 6125 } 6126 #else 6127 static void 6128 mlxsw_sp_fib6_entry_hw_flags_clear(struct mlxsw_sp *mlxsw_sp, 6129 struct mlxsw_sp_fib_entry *fib_entry) 6130 { 6131 } 6132 #endif 6133 6134 static void 6135 mlxsw_sp_fib_entry_hw_flags_set(struct mlxsw_sp *mlxsw_sp, 6136 struct mlxsw_sp_fib_entry *fib_entry) 6137 { 6138 switch (fib_entry->fib_node->fib->proto) { 6139 case MLXSW_SP_L3_PROTO_IPV4: 6140 mlxsw_sp_fib4_entry_hw_flags_set(mlxsw_sp, fib_entry); 6141 break; 6142 case MLXSW_SP_L3_PROTO_IPV6: 6143 mlxsw_sp_fib6_entry_hw_flags_set(mlxsw_sp, fib_entry); 6144 break; 6145 } 6146 } 6147 6148 static void 6149 mlxsw_sp_fib_entry_hw_flags_clear(struct mlxsw_sp *mlxsw_sp, 6150 struct mlxsw_sp_fib_entry *fib_entry) 6151 { 6152 switch (fib_entry->fib_node->fib->proto) { 6153 case MLXSW_SP_L3_PROTO_IPV4: 6154 mlxsw_sp_fib4_entry_hw_flags_clear(mlxsw_sp, fib_entry); 6155 break; 6156 case MLXSW_SP_L3_PROTO_IPV6: 6157 mlxsw_sp_fib6_entry_hw_flags_clear(mlxsw_sp, fib_entry); 6158 break; 6159 } 6160 } 6161 6162 static void 6163 mlxsw_sp_fib_entry_hw_flags_refresh(struct mlxsw_sp *mlxsw_sp, 6164 struct mlxsw_sp_fib_entry *fib_entry, 6165 enum mlxsw_reg_ralue_op op) 6166 { 6167 switch (op) { 6168 case MLXSW_REG_RALUE_OP_WRITE_WRITE: 6169 mlxsw_sp_fib_entry_hw_flags_set(mlxsw_sp, fib_entry); 6170 break; 6171 case MLXSW_REG_RALUE_OP_WRITE_DELETE: 6172 mlxsw_sp_fib_entry_hw_flags_clear(mlxsw_sp, fib_entry); 6173 break; 6174 default: 6175 break; 6176 } 6177 } 6178 6179 static void 6180 mlxsw_sp_fib_entry_ralue_pack(char *ralue_pl, 6181 const struct mlxsw_sp_fib_entry *fib_entry, 6182 enum mlxsw_reg_ralue_op op) 6183 { 6184 struct mlxsw_sp_fib *fib = fib_entry->fib_node->fib; 6185 enum mlxsw_reg_ralxx_protocol proto; 6186 u32 *p_dip; 6187 6188 proto = (enum mlxsw_reg_ralxx_protocol) fib->proto; 6189 6190 switch (fib->proto) { 6191 case MLXSW_SP_L3_PROTO_IPV4: 6192 p_dip = (u32 *) fib_entry->fib_node->key.addr; 6193 mlxsw_reg_ralue_pack4(ralue_pl, proto, op, fib->vr->id, 6194 fib_entry->fib_node->key.prefix_len, 6195 *p_dip); 6196 break; 6197 case MLXSW_SP_L3_PROTO_IPV6: 6198 mlxsw_reg_ralue_pack6(ralue_pl, proto, op, fib->vr->id, 6199 fib_entry->fib_node->key.prefix_len, 6200 fib_entry->fib_node->key.addr); 6201 break; 6202 } 6203 } 6204 6205 static int mlxsw_sp_fib_entry_op_remote(struct mlxsw_sp *mlxsw_sp, 6206 struct mlxsw_sp_fib_entry *fib_entry, 6207 enum mlxsw_reg_ralue_op op) 6208 { 6209 struct mlxsw_sp_nexthop_group *nh_group = fib_entry->nh_group; 6210 struct mlxsw_sp_nexthop_group_info *nhgi = nh_group->nhgi; 6211 char ralue_pl[MLXSW_REG_RALUE_LEN]; 6212 enum mlxsw_reg_ralue_trap_action trap_action; 6213 u16 trap_id = 0; 6214 u32 adjacency_index = 0; 6215 u16 ecmp_size = 0; 6216 6217 /* In case the nexthop group adjacency index is valid, use it 6218 * with provided ECMP size. Otherwise, setup trap and pass 6219 * traffic to kernel. 6220 */ 6221 if (mlxsw_sp_fib_entry_should_offload(fib_entry)) { 6222 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_NOP; 6223 adjacency_index = nhgi->adj_index; 6224 ecmp_size = nhgi->ecmp_size; 6225 } else if (!nhgi->adj_index_valid && nhgi->count && 6226 mlxsw_sp_nhgi_rif(nhgi)) { 6227 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_NOP; 6228 adjacency_index = mlxsw_sp->router->adj_trap_index; 6229 ecmp_size = 1; 6230 } else { 6231 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_TRAP; 6232 trap_id = MLXSW_TRAP_ID_RTR_INGRESS0; 6233 } 6234 6235 mlxsw_sp_fib_entry_ralue_pack(ralue_pl, fib_entry, op); 6236 mlxsw_reg_ralue_act_remote_pack(ralue_pl, trap_action, trap_id, 6237 adjacency_index, ecmp_size); 6238 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl); 6239 } 6240 6241 static int mlxsw_sp_fib_entry_op_local(struct mlxsw_sp *mlxsw_sp, 6242 struct mlxsw_sp_fib_entry *fib_entry, 6243 enum mlxsw_reg_ralue_op op) 6244 { 6245 struct mlxsw_sp_rif *rif = mlxsw_sp_nhgi_rif(fib_entry->nh_group->nhgi); 6246 enum mlxsw_reg_ralue_trap_action trap_action; 6247 char ralue_pl[MLXSW_REG_RALUE_LEN]; 6248 u16 trap_id = 0; 6249 u16 rif_index = 0; 6250 6251 if (mlxsw_sp_fib_entry_should_offload(fib_entry)) { 6252 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_NOP; 6253 rif_index = rif->rif_index; 6254 } else { 6255 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_TRAP; 6256 trap_id = MLXSW_TRAP_ID_RTR_INGRESS0; 6257 } 6258 6259 mlxsw_sp_fib_entry_ralue_pack(ralue_pl, fib_entry, op); 6260 mlxsw_reg_ralue_act_local_pack(ralue_pl, trap_action, trap_id, 6261 rif_index); 6262 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl); 6263 } 6264 6265 static int mlxsw_sp_fib_entry_op_trap(struct mlxsw_sp *mlxsw_sp, 6266 struct mlxsw_sp_fib_entry *fib_entry, 6267 enum mlxsw_reg_ralue_op op) 6268 { 6269 char ralue_pl[MLXSW_REG_RALUE_LEN]; 6270 6271 mlxsw_sp_fib_entry_ralue_pack(ralue_pl, fib_entry, op); 6272 mlxsw_reg_ralue_act_ip2me_pack(ralue_pl); 6273 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl); 6274 } 6275 6276 static int mlxsw_sp_fib_entry_op_blackhole(struct mlxsw_sp *mlxsw_sp, 6277 struct mlxsw_sp_fib_entry *fib_entry, 6278 enum mlxsw_reg_ralue_op op) 6279 { 6280 enum mlxsw_reg_ralue_trap_action trap_action; 6281 char ralue_pl[MLXSW_REG_RALUE_LEN]; 6282 6283 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_DISCARD_ERROR; 6284 mlxsw_sp_fib_entry_ralue_pack(ralue_pl, fib_entry, op); 6285 mlxsw_reg_ralue_act_local_pack(ralue_pl, trap_action, 0, 0); 6286 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl); 6287 } 6288 6289 static int 6290 mlxsw_sp_fib_entry_op_unreachable(struct mlxsw_sp *mlxsw_sp, 6291 struct mlxsw_sp_fib_entry *fib_entry, 6292 enum mlxsw_reg_ralue_op op) 6293 { 6294 enum mlxsw_reg_ralue_trap_action trap_action; 6295 char ralue_pl[MLXSW_REG_RALUE_LEN]; 6296 u16 trap_id; 6297 6298 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_TRAP; 6299 trap_id = MLXSW_TRAP_ID_RTR_INGRESS1; 6300 6301 mlxsw_sp_fib_entry_ralue_pack(ralue_pl, fib_entry, op); 6302 mlxsw_reg_ralue_act_local_pack(ralue_pl, trap_action, trap_id, 0); 6303 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl); 6304 } 6305 6306 static int 6307 mlxsw_sp_fib_entry_op_ipip_decap(struct mlxsw_sp *mlxsw_sp, 6308 struct mlxsw_sp_fib_entry *fib_entry, 6309 enum mlxsw_reg_ralue_op op) 6310 { 6311 struct mlxsw_sp_ipip_entry *ipip_entry = fib_entry->decap.ipip_entry; 6312 const struct mlxsw_sp_ipip_ops *ipip_ops; 6313 char ralue_pl[MLXSW_REG_RALUE_LEN]; 6314 int err; 6315 6316 if (WARN_ON(!ipip_entry)) 6317 return -EINVAL; 6318 6319 ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt]; 6320 err = ipip_ops->decap_config(mlxsw_sp, ipip_entry, 6321 fib_entry->decap.tunnel_index); 6322 if (err) 6323 return err; 6324 6325 mlxsw_sp_fib_entry_ralue_pack(ralue_pl, fib_entry, op); 6326 mlxsw_reg_ralue_act_ip2me_tun_pack(ralue_pl, 6327 fib_entry->decap.tunnel_index); 6328 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl); 6329 } 6330 6331 static int mlxsw_sp_fib_entry_op_nve_decap(struct mlxsw_sp *mlxsw_sp, 6332 struct mlxsw_sp_fib_entry *fib_entry, 6333 enum mlxsw_reg_ralue_op op) 6334 { 6335 char ralue_pl[MLXSW_REG_RALUE_LEN]; 6336 6337 mlxsw_sp_fib_entry_ralue_pack(ralue_pl, fib_entry, op); 6338 mlxsw_reg_ralue_act_ip2me_tun_pack(ralue_pl, 6339 fib_entry->decap.tunnel_index); 6340 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl); 6341 } 6342 6343 static int __mlxsw_sp_fib_entry_op(struct mlxsw_sp *mlxsw_sp, 6344 struct mlxsw_sp_fib_entry *fib_entry, 6345 enum mlxsw_reg_ralue_op op) 6346 { 6347 switch (fib_entry->type) { 6348 case MLXSW_SP_FIB_ENTRY_TYPE_REMOTE: 6349 return mlxsw_sp_fib_entry_op_remote(mlxsw_sp, fib_entry, op); 6350 case MLXSW_SP_FIB_ENTRY_TYPE_LOCAL: 6351 return mlxsw_sp_fib_entry_op_local(mlxsw_sp, fib_entry, op); 6352 case MLXSW_SP_FIB_ENTRY_TYPE_TRAP: 6353 return mlxsw_sp_fib_entry_op_trap(mlxsw_sp, fib_entry, op); 6354 case MLXSW_SP_FIB_ENTRY_TYPE_BLACKHOLE: 6355 return mlxsw_sp_fib_entry_op_blackhole(mlxsw_sp, fib_entry, op); 6356 case MLXSW_SP_FIB_ENTRY_TYPE_UNREACHABLE: 6357 return mlxsw_sp_fib_entry_op_unreachable(mlxsw_sp, fib_entry, 6358 op); 6359 case MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP: 6360 return mlxsw_sp_fib_entry_op_ipip_decap(mlxsw_sp, 6361 fib_entry, op); 6362 case MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP: 6363 return mlxsw_sp_fib_entry_op_nve_decap(mlxsw_sp, fib_entry, op); 6364 } 6365 return -EINVAL; 6366 } 6367 6368 static int mlxsw_sp_fib_entry_op(struct mlxsw_sp *mlxsw_sp, 6369 struct mlxsw_sp_fib_entry *fib_entry, 6370 enum mlxsw_reg_ralue_op op) 6371 { 6372 int err = __mlxsw_sp_fib_entry_op(mlxsw_sp, fib_entry, op); 6373 6374 if (err) 6375 return err; 6376 6377 mlxsw_sp_fib_entry_hw_flags_refresh(mlxsw_sp, fib_entry, op); 6378 6379 return err; 6380 } 6381 6382 static int mlxsw_sp_fib_entry_update(struct mlxsw_sp *mlxsw_sp, 6383 struct mlxsw_sp_fib_entry *fib_entry) 6384 { 6385 return mlxsw_sp_fib_entry_op(mlxsw_sp, fib_entry, 6386 MLXSW_REG_RALUE_OP_WRITE_WRITE); 6387 } 6388 6389 static int mlxsw_sp_fib_entry_del(struct mlxsw_sp *mlxsw_sp, 6390 struct mlxsw_sp_fib_entry *fib_entry) 6391 { 6392 return mlxsw_sp_fib_entry_op(mlxsw_sp, fib_entry, 6393 MLXSW_REG_RALUE_OP_WRITE_DELETE); 6394 } 6395 6396 static int 6397 mlxsw_sp_fib4_entry_type_set(struct mlxsw_sp *mlxsw_sp, 6398 const struct fib_entry_notifier_info *fen_info, 6399 struct mlxsw_sp_fib_entry *fib_entry) 6400 { 6401 struct mlxsw_sp_nexthop_group_info *nhgi = fib_entry->nh_group->nhgi; 6402 union mlxsw_sp_l3addr dip = { .addr4 = htonl(fen_info->dst) }; 6403 struct mlxsw_sp_router *router = mlxsw_sp->router; 6404 u32 tb_id = mlxsw_sp_fix_tb_id(fen_info->tb_id); 6405 int ifindex = nhgi->nexthops[0].ifindex; 6406 struct mlxsw_sp_ipip_entry *ipip_entry; 6407 6408 switch (fen_info->type) { 6409 case RTN_LOCAL: 6410 ipip_entry = mlxsw_sp_ipip_entry_find_by_decap(mlxsw_sp, ifindex, 6411 MLXSW_SP_L3_PROTO_IPV4, dip); 6412 if (ipip_entry && ipip_entry->ol_dev->flags & IFF_UP) { 6413 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP; 6414 return mlxsw_sp_fib_entry_decap_init(mlxsw_sp, 6415 fib_entry, 6416 ipip_entry); 6417 } 6418 if (mlxsw_sp_router_nve_is_decap(mlxsw_sp, tb_id, 6419 MLXSW_SP_L3_PROTO_IPV4, 6420 &dip)) { 6421 u32 tunnel_index; 6422 6423 tunnel_index = router->nve_decap_config.tunnel_index; 6424 fib_entry->decap.tunnel_index = tunnel_index; 6425 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP; 6426 return 0; 6427 } 6428 fallthrough; 6429 case RTN_BROADCAST: 6430 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP; 6431 return 0; 6432 case RTN_BLACKHOLE: 6433 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_BLACKHOLE; 6434 return 0; 6435 case RTN_UNREACHABLE: 6436 case RTN_PROHIBIT: 6437 /* Packets hitting these routes need to be trapped, but 6438 * can do so with a lower priority than packets directed 6439 * at the host, so use action type local instead of trap. 6440 */ 6441 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_UNREACHABLE; 6442 return 0; 6443 case RTN_UNICAST: 6444 if (nhgi->gateway) 6445 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_REMOTE; 6446 else 6447 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL; 6448 return 0; 6449 default: 6450 return -EINVAL; 6451 } 6452 } 6453 6454 static void 6455 mlxsw_sp_fib_entry_type_unset(struct mlxsw_sp *mlxsw_sp, 6456 struct mlxsw_sp_fib_entry *fib_entry) 6457 { 6458 switch (fib_entry->type) { 6459 case MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP: 6460 mlxsw_sp_fib_entry_decap_fini(mlxsw_sp, fib_entry); 6461 break; 6462 default: 6463 break; 6464 } 6465 } 6466 6467 static void 6468 mlxsw_sp_fib4_entry_type_unset(struct mlxsw_sp *mlxsw_sp, 6469 struct mlxsw_sp_fib4_entry *fib4_entry) 6470 { 6471 mlxsw_sp_fib_entry_type_unset(mlxsw_sp, &fib4_entry->common); 6472 } 6473 6474 static struct mlxsw_sp_fib4_entry * 6475 mlxsw_sp_fib4_entry_create(struct mlxsw_sp *mlxsw_sp, 6476 struct mlxsw_sp_fib_node *fib_node, 6477 const struct fib_entry_notifier_info *fen_info) 6478 { 6479 struct mlxsw_sp_fib4_entry *fib4_entry; 6480 struct mlxsw_sp_fib_entry *fib_entry; 6481 int err; 6482 6483 fib4_entry = kzalloc_obj(*fib4_entry); 6484 if (!fib4_entry) 6485 return ERR_PTR(-ENOMEM); 6486 fib_entry = &fib4_entry->common; 6487 6488 err = mlxsw_sp_nexthop4_group_get(mlxsw_sp, fib_entry, fen_info->fi); 6489 if (err) 6490 goto err_nexthop4_group_get; 6491 6492 err = mlxsw_sp_nexthop_group_vr_link(fib_entry->nh_group, 6493 fib_node->fib); 6494 if (err) 6495 goto err_nexthop_group_vr_link; 6496 6497 err = mlxsw_sp_fib4_entry_type_set(mlxsw_sp, fen_info, fib_entry); 6498 if (err) 6499 goto err_fib4_entry_type_set; 6500 6501 fib4_entry->fi = fen_info->fi; 6502 fib_info_hold(fib4_entry->fi); 6503 fib4_entry->tb_id = fen_info->tb_id; 6504 fib4_entry->type = fen_info->type; 6505 fib4_entry->dscp = fen_info->dscp; 6506 6507 fib_entry->fib_node = fib_node; 6508 6509 return fib4_entry; 6510 6511 err_fib4_entry_type_set: 6512 mlxsw_sp_nexthop_group_vr_unlink(fib_entry->nh_group, fib_node->fib); 6513 err_nexthop_group_vr_link: 6514 mlxsw_sp_nexthop4_group_put(mlxsw_sp, &fib4_entry->common); 6515 err_nexthop4_group_get: 6516 kfree(fib4_entry); 6517 return ERR_PTR(err); 6518 } 6519 6520 static void mlxsw_sp_fib4_entry_destroy(struct mlxsw_sp *mlxsw_sp, 6521 struct mlxsw_sp_fib4_entry *fib4_entry) 6522 { 6523 struct mlxsw_sp_fib_node *fib_node = fib4_entry->common.fib_node; 6524 6525 fib_info_put(fib4_entry->fi); 6526 mlxsw_sp_fib4_entry_type_unset(mlxsw_sp, fib4_entry); 6527 mlxsw_sp_nexthop_group_vr_unlink(fib4_entry->common.nh_group, 6528 fib_node->fib); 6529 mlxsw_sp_nexthop4_group_put(mlxsw_sp, &fib4_entry->common); 6530 kfree(fib4_entry); 6531 } 6532 6533 static struct mlxsw_sp_fib4_entry * 6534 mlxsw_sp_fib4_entry_lookup(struct mlxsw_sp *mlxsw_sp, 6535 const struct fib_entry_notifier_info *fen_info) 6536 { 6537 struct mlxsw_sp_fib4_entry *fib4_entry; 6538 struct mlxsw_sp_fib_node *fib_node; 6539 struct mlxsw_sp_fib *fib; 6540 struct mlxsw_sp_vr *vr; 6541 6542 vr = mlxsw_sp_vr_find(mlxsw_sp, fen_info->tb_id); 6543 if (!vr) 6544 return NULL; 6545 fib = mlxsw_sp_vr_fib(vr, MLXSW_SP_L3_PROTO_IPV4); 6546 6547 fib_node = mlxsw_sp_fib_node_lookup(fib, &fen_info->dst, 6548 sizeof(fen_info->dst), 6549 fen_info->dst_len); 6550 if (!fib_node) 6551 return NULL; 6552 6553 fib4_entry = container_of(fib_node->fib_entry, 6554 struct mlxsw_sp_fib4_entry, common); 6555 if (fib4_entry->tb_id == fen_info->tb_id && 6556 fib4_entry->dscp == fen_info->dscp && 6557 fib4_entry->type == fen_info->type && 6558 fib4_entry->fi == fen_info->fi) 6559 return fib4_entry; 6560 6561 return NULL; 6562 } 6563 6564 static const struct rhashtable_params mlxsw_sp_fib_ht_params = { 6565 .key_offset = offsetof(struct mlxsw_sp_fib_node, key), 6566 .head_offset = offsetof(struct mlxsw_sp_fib_node, ht_node), 6567 .key_len = sizeof(struct mlxsw_sp_fib_key), 6568 .automatic_shrinking = true, 6569 }; 6570 6571 static int mlxsw_sp_fib_node_insert(struct mlxsw_sp_fib *fib, 6572 struct mlxsw_sp_fib_node *fib_node) 6573 { 6574 return rhashtable_insert_fast(&fib->ht, &fib_node->ht_node, 6575 mlxsw_sp_fib_ht_params); 6576 } 6577 6578 static void mlxsw_sp_fib_node_remove(struct mlxsw_sp_fib *fib, 6579 struct mlxsw_sp_fib_node *fib_node) 6580 { 6581 rhashtable_remove_fast(&fib->ht, &fib_node->ht_node, 6582 mlxsw_sp_fib_ht_params); 6583 } 6584 6585 static struct mlxsw_sp_fib_node * 6586 mlxsw_sp_fib_node_lookup(struct mlxsw_sp_fib *fib, const void *addr, 6587 size_t addr_len, unsigned char prefix_len) 6588 { 6589 struct mlxsw_sp_fib_key key; 6590 6591 memset(&key, 0, sizeof(key)); 6592 memcpy(key.addr, addr, addr_len); 6593 key.prefix_len = prefix_len; 6594 return rhashtable_lookup_fast(&fib->ht, &key, mlxsw_sp_fib_ht_params); 6595 } 6596 6597 static struct mlxsw_sp_fib_node * 6598 mlxsw_sp_fib_node_create(struct mlxsw_sp_fib *fib, const void *addr, 6599 size_t addr_len, unsigned char prefix_len) 6600 { 6601 struct mlxsw_sp_fib_node *fib_node; 6602 6603 fib_node = kzalloc_obj(*fib_node); 6604 if (!fib_node) 6605 return NULL; 6606 6607 list_add(&fib_node->list, &fib->node_list); 6608 memcpy(fib_node->key.addr, addr, addr_len); 6609 fib_node->key.prefix_len = prefix_len; 6610 6611 return fib_node; 6612 } 6613 6614 static void mlxsw_sp_fib_node_destroy(struct mlxsw_sp_fib_node *fib_node) 6615 { 6616 list_del(&fib_node->list); 6617 kfree(fib_node); 6618 } 6619 6620 static int mlxsw_sp_fib_lpm_tree_link(struct mlxsw_sp *mlxsw_sp, 6621 struct mlxsw_sp_fib_node *fib_node) 6622 { 6623 struct mlxsw_sp_prefix_usage req_prefix_usage; 6624 struct mlxsw_sp_fib *fib = fib_node->fib; 6625 struct mlxsw_sp_lpm_tree *lpm_tree; 6626 int err; 6627 6628 lpm_tree = mlxsw_sp->router->lpm.proto_trees[fib->proto]; 6629 if (lpm_tree->prefix_ref_count[fib_node->key.prefix_len] != 0) 6630 goto out; 6631 6632 mlxsw_sp_prefix_usage_cpy(&req_prefix_usage, &lpm_tree->prefix_usage); 6633 mlxsw_sp_prefix_usage_set(&req_prefix_usage, fib_node->key.prefix_len); 6634 lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage, 6635 fib->proto); 6636 if (IS_ERR(lpm_tree)) 6637 return PTR_ERR(lpm_tree); 6638 6639 err = mlxsw_sp_vrs_lpm_tree_replace(mlxsw_sp, fib, lpm_tree); 6640 if (err) 6641 goto err_lpm_tree_replace; 6642 6643 out: 6644 lpm_tree->prefix_ref_count[fib_node->key.prefix_len]++; 6645 return 0; 6646 6647 err_lpm_tree_replace: 6648 mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree); 6649 return err; 6650 } 6651 6652 static void mlxsw_sp_fib_lpm_tree_unlink(struct mlxsw_sp *mlxsw_sp, 6653 struct mlxsw_sp_fib_node *fib_node) 6654 { 6655 struct mlxsw_sp_lpm_tree *lpm_tree = fib_node->fib->lpm_tree; 6656 struct mlxsw_sp_prefix_usage req_prefix_usage; 6657 struct mlxsw_sp_fib *fib = fib_node->fib; 6658 int err; 6659 6660 if (--lpm_tree->prefix_ref_count[fib_node->key.prefix_len] != 0) 6661 return; 6662 /* Try to construct a new LPM tree from the current prefix usage 6663 * minus the unused one. If we fail, continue using the old one. 6664 */ 6665 mlxsw_sp_prefix_usage_cpy(&req_prefix_usage, &lpm_tree->prefix_usage); 6666 mlxsw_sp_prefix_usage_clear(&req_prefix_usage, 6667 fib_node->key.prefix_len); 6668 lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage, 6669 fib->proto); 6670 if (IS_ERR(lpm_tree)) 6671 return; 6672 6673 err = mlxsw_sp_vrs_lpm_tree_replace(mlxsw_sp, fib, lpm_tree); 6674 if (err) 6675 goto err_lpm_tree_replace; 6676 6677 return; 6678 6679 err_lpm_tree_replace: 6680 mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree); 6681 } 6682 6683 static int mlxsw_sp_fib_node_init(struct mlxsw_sp *mlxsw_sp, 6684 struct mlxsw_sp_fib_node *fib_node, 6685 struct mlxsw_sp_fib *fib) 6686 { 6687 int err; 6688 6689 err = mlxsw_sp_fib_node_insert(fib, fib_node); 6690 if (err) 6691 return err; 6692 fib_node->fib = fib; 6693 6694 err = mlxsw_sp_fib_lpm_tree_link(mlxsw_sp, fib_node); 6695 if (err) 6696 goto err_fib_lpm_tree_link; 6697 6698 return 0; 6699 6700 err_fib_lpm_tree_link: 6701 fib_node->fib = NULL; 6702 mlxsw_sp_fib_node_remove(fib, fib_node); 6703 return err; 6704 } 6705 6706 static void mlxsw_sp_fib_node_fini(struct mlxsw_sp *mlxsw_sp, 6707 struct mlxsw_sp_fib_node *fib_node) 6708 { 6709 struct mlxsw_sp_fib *fib = fib_node->fib; 6710 6711 mlxsw_sp_fib_lpm_tree_unlink(mlxsw_sp, fib_node); 6712 fib_node->fib = NULL; 6713 mlxsw_sp_fib_node_remove(fib, fib_node); 6714 } 6715 6716 static struct mlxsw_sp_fib_node * 6717 mlxsw_sp_fib_node_get(struct mlxsw_sp *mlxsw_sp, u32 tb_id, const void *addr, 6718 size_t addr_len, unsigned char prefix_len, 6719 enum mlxsw_sp_l3proto proto) 6720 { 6721 struct mlxsw_sp_fib_node *fib_node; 6722 struct mlxsw_sp_fib *fib; 6723 struct mlxsw_sp_vr *vr; 6724 int err; 6725 6726 vr = mlxsw_sp_vr_get(mlxsw_sp, tb_id, NULL); 6727 if (IS_ERR(vr)) 6728 return ERR_CAST(vr); 6729 fib = mlxsw_sp_vr_fib(vr, proto); 6730 6731 fib_node = mlxsw_sp_fib_node_lookup(fib, addr, addr_len, prefix_len); 6732 if (fib_node) 6733 return fib_node; 6734 6735 fib_node = mlxsw_sp_fib_node_create(fib, addr, addr_len, prefix_len); 6736 if (!fib_node) { 6737 err = -ENOMEM; 6738 goto err_fib_node_create; 6739 } 6740 6741 err = mlxsw_sp_fib_node_init(mlxsw_sp, fib_node, fib); 6742 if (err) 6743 goto err_fib_node_init; 6744 6745 return fib_node; 6746 6747 err_fib_node_init: 6748 mlxsw_sp_fib_node_destroy(fib_node); 6749 err_fib_node_create: 6750 mlxsw_sp_vr_put(mlxsw_sp, vr); 6751 return ERR_PTR(err); 6752 } 6753 6754 static void mlxsw_sp_fib_node_put(struct mlxsw_sp *mlxsw_sp, 6755 struct mlxsw_sp_fib_node *fib_node) 6756 { 6757 struct mlxsw_sp_vr *vr = fib_node->fib->vr; 6758 6759 if (fib_node->fib_entry) 6760 return; 6761 mlxsw_sp_fib_node_fini(mlxsw_sp, fib_node); 6762 mlxsw_sp_fib_node_destroy(fib_node); 6763 mlxsw_sp_vr_put(mlxsw_sp, vr); 6764 } 6765 6766 static int mlxsw_sp_fib_node_entry_link(struct mlxsw_sp *mlxsw_sp, 6767 struct mlxsw_sp_fib_entry *fib_entry) 6768 { 6769 struct mlxsw_sp_fib_node *fib_node = fib_entry->fib_node; 6770 int err; 6771 6772 fib_node->fib_entry = fib_entry; 6773 6774 err = mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry); 6775 if (err) 6776 goto err_fib_entry_update; 6777 6778 return 0; 6779 6780 err_fib_entry_update: 6781 fib_node->fib_entry = NULL; 6782 return err; 6783 } 6784 6785 static void 6786 mlxsw_sp_fib_node_entry_unlink(struct mlxsw_sp *mlxsw_sp, 6787 struct mlxsw_sp_fib_entry *fib_entry) 6788 { 6789 struct mlxsw_sp_fib_node *fib_node = fib_entry->fib_node; 6790 6791 mlxsw_sp_fib_entry_del(mlxsw_sp, fib_entry); 6792 fib_node->fib_entry = NULL; 6793 } 6794 6795 static bool mlxsw_sp_fib4_allow_replace(struct mlxsw_sp_fib4_entry *fib4_entry) 6796 { 6797 struct mlxsw_sp_fib_node *fib_node = fib4_entry->common.fib_node; 6798 struct mlxsw_sp_fib4_entry *fib4_replaced; 6799 6800 if (!fib_node->fib_entry) 6801 return true; 6802 6803 fib4_replaced = container_of(fib_node->fib_entry, 6804 struct mlxsw_sp_fib4_entry, common); 6805 if (fib4_entry->tb_id == RT_TABLE_MAIN && 6806 fib4_replaced->tb_id == RT_TABLE_LOCAL) 6807 return false; 6808 6809 return true; 6810 } 6811 6812 static int 6813 mlxsw_sp_router_fib4_replace(struct mlxsw_sp *mlxsw_sp, 6814 const struct fib_entry_notifier_info *fen_info) 6815 { 6816 struct mlxsw_sp_fib4_entry *fib4_entry, *fib4_replaced; 6817 struct mlxsw_sp_fib_entry *replaced; 6818 struct mlxsw_sp_fib_node *fib_node; 6819 int err; 6820 6821 if (fen_info->fi->nh && 6822 !mlxsw_sp_nexthop_obj_group_lookup(mlxsw_sp, fen_info->fi->nh->id)) 6823 return 0; 6824 6825 fib_node = mlxsw_sp_fib_node_get(mlxsw_sp, fen_info->tb_id, 6826 &fen_info->dst, sizeof(fen_info->dst), 6827 fen_info->dst_len, 6828 MLXSW_SP_L3_PROTO_IPV4); 6829 if (IS_ERR(fib_node)) { 6830 dev_warn(mlxsw_sp->bus_info->dev, "Failed to get FIB node\n"); 6831 return PTR_ERR(fib_node); 6832 } 6833 6834 fib4_entry = mlxsw_sp_fib4_entry_create(mlxsw_sp, fib_node, fen_info); 6835 if (IS_ERR(fib4_entry)) { 6836 dev_warn(mlxsw_sp->bus_info->dev, "Failed to create FIB entry\n"); 6837 err = PTR_ERR(fib4_entry); 6838 goto err_fib4_entry_create; 6839 } 6840 6841 if (!mlxsw_sp_fib4_allow_replace(fib4_entry)) { 6842 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry); 6843 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 6844 return 0; 6845 } 6846 6847 replaced = fib_node->fib_entry; 6848 err = mlxsw_sp_fib_node_entry_link(mlxsw_sp, &fib4_entry->common); 6849 if (err) { 6850 dev_warn(mlxsw_sp->bus_info->dev, "Failed to link FIB entry to node\n"); 6851 goto err_fib_node_entry_link; 6852 } 6853 6854 /* Nothing to replace */ 6855 if (!replaced) 6856 return 0; 6857 6858 mlxsw_sp_fib_entry_hw_flags_clear(mlxsw_sp, replaced); 6859 fib4_replaced = container_of(replaced, struct mlxsw_sp_fib4_entry, 6860 common); 6861 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_replaced); 6862 6863 return 0; 6864 6865 err_fib_node_entry_link: 6866 fib_node->fib_entry = replaced; 6867 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry); 6868 err_fib4_entry_create: 6869 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 6870 return err; 6871 } 6872 6873 static void mlxsw_sp_router_fib4_del(struct mlxsw_sp *mlxsw_sp, 6874 struct fib_entry_notifier_info *fen_info) 6875 { 6876 struct mlxsw_sp_fib4_entry *fib4_entry; 6877 struct mlxsw_sp_fib_node *fib_node; 6878 6879 fib4_entry = mlxsw_sp_fib4_entry_lookup(mlxsw_sp, fen_info); 6880 if (!fib4_entry) 6881 return; 6882 fib_node = fib4_entry->common.fib_node; 6883 6884 mlxsw_sp_fib_node_entry_unlink(mlxsw_sp, &fib4_entry->common); 6885 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry); 6886 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 6887 } 6888 6889 static bool mlxsw_sp_fib6_rt_should_ignore(const struct fib6_info *rt) 6890 { 6891 /* Multicast routes aren't supported, so ignore them. Neighbour 6892 * Discovery packets are specifically trapped. 6893 */ 6894 if (ipv6_addr_type(&rt->fib6_dst.addr) & IPV6_ADDR_MULTICAST) 6895 return true; 6896 6897 /* Cloned routes are irrelevant in the forwarding path. */ 6898 if (rt->fib6_flags & RTF_CACHE) 6899 return true; 6900 6901 return false; 6902 } 6903 6904 static struct mlxsw_sp_rt6 *mlxsw_sp_rt6_create(struct fib6_info *rt) 6905 { 6906 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 6907 6908 mlxsw_sp_rt6 = kzalloc_obj(*mlxsw_sp_rt6); 6909 if (!mlxsw_sp_rt6) 6910 return ERR_PTR(-ENOMEM); 6911 6912 /* In case of route replace, replaced route is deleted with 6913 * no notification. Take reference to prevent accessing freed 6914 * memory. 6915 */ 6916 mlxsw_sp_rt6->rt = rt; 6917 fib6_info_hold(rt); 6918 6919 return mlxsw_sp_rt6; 6920 } 6921 6922 #if IS_ENABLED(CONFIG_IPV6) 6923 static void mlxsw_sp_rt6_release(struct fib6_info *rt) 6924 { 6925 fib6_info_release(rt); 6926 } 6927 #else 6928 static void mlxsw_sp_rt6_release(struct fib6_info *rt) 6929 { 6930 } 6931 #endif 6932 6933 static void mlxsw_sp_rt6_destroy(struct mlxsw_sp_rt6 *mlxsw_sp_rt6) 6934 { 6935 struct fib6_nh *fib6_nh = mlxsw_sp_rt6->rt->fib6_nh; 6936 6937 if (!mlxsw_sp_rt6->rt->nh) 6938 fib6_nh->fib_nh_flags &= ~RTNH_F_OFFLOAD; 6939 mlxsw_sp_rt6_release(mlxsw_sp_rt6->rt); 6940 kfree(mlxsw_sp_rt6); 6941 } 6942 6943 static struct fib6_info * 6944 mlxsw_sp_fib6_entry_rt(const struct mlxsw_sp_fib6_entry *fib6_entry) 6945 { 6946 return list_first_entry(&fib6_entry->rt6_list, struct mlxsw_sp_rt6, 6947 list)->rt; 6948 } 6949 6950 static struct mlxsw_sp_rt6 * 6951 mlxsw_sp_fib6_entry_rt_find(const struct mlxsw_sp_fib6_entry *fib6_entry, 6952 const struct fib6_info *rt) 6953 { 6954 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 6955 6956 list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) { 6957 if (mlxsw_sp_rt6->rt == rt) 6958 return mlxsw_sp_rt6; 6959 } 6960 6961 return NULL; 6962 } 6963 6964 static bool mlxsw_sp_nexthop6_ipip_type(const struct mlxsw_sp *mlxsw_sp, 6965 const struct fib6_info *rt, 6966 enum mlxsw_sp_ipip_type *ret) 6967 { 6968 return rt->fib6_nh->fib_nh_dev && 6969 mlxsw_sp_netdev_ipip_type(mlxsw_sp, rt->fib6_nh->fib_nh_dev, ret); 6970 } 6971 6972 static int mlxsw_sp_nexthop6_init(struct mlxsw_sp *mlxsw_sp, 6973 struct mlxsw_sp_nexthop_group *nh_grp, 6974 struct mlxsw_sp_nexthop *nh, 6975 const struct fib6_info *rt) 6976 { 6977 struct net_device *dev = rt->fib6_nh->fib_nh_dev; 6978 int err; 6979 6980 nh->nhgi = nh_grp->nhgi; 6981 nh->nh_weight = rt->fib6_nh->fib_nh_weight; 6982 memcpy(&nh->gw_addr, &rt->fib6_nh->fib_nh_gw6, sizeof(nh->gw_addr)); 6983 #if IS_ENABLED(CONFIG_IPV6) 6984 nh->neigh_tbl = &nd_tbl; 6985 #endif 6986 6987 err = mlxsw_sp_nexthop_counter_enable(mlxsw_sp, nh); 6988 if (err) 6989 return err; 6990 6991 list_add_tail(&nh->router_list_node, &mlxsw_sp->router->nexthop_list); 6992 6993 if (!dev) 6994 return 0; 6995 nh->ifindex = dev->ifindex; 6996 6997 err = mlxsw_sp_nexthop_type_init(mlxsw_sp, nh, dev); 6998 if (err) 6999 goto err_nexthop_type_init; 7000 7001 return 0; 7002 7003 err_nexthop_type_init: 7004 list_del(&nh->router_list_node); 7005 mlxsw_sp_nexthop_counter_disable(mlxsw_sp, nh); 7006 return err; 7007 } 7008 7009 static void mlxsw_sp_nexthop6_fini(struct mlxsw_sp *mlxsw_sp, 7010 struct mlxsw_sp_nexthop *nh) 7011 { 7012 mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh); 7013 list_del(&nh->router_list_node); 7014 mlxsw_sp_nexthop_counter_disable(mlxsw_sp, nh); 7015 } 7016 7017 static bool mlxsw_sp_rt6_is_gateway(const struct mlxsw_sp *mlxsw_sp, 7018 const struct fib6_info *rt) 7019 { 7020 return rt->fib6_nh->fib_nh_gw_family || 7021 mlxsw_sp_nexthop6_ipip_type(mlxsw_sp, rt, NULL); 7022 } 7023 7024 static int 7025 mlxsw_sp_nexthop6_group_info_init(struct mlxsw_sp *mlxsw_sp, 7026 struct mlxsw_sp_nexthop_group *nh_grp, 7027 struct mlxsw_sp_fib6_entry *fib6_entry) 7028 { 7029 struct mlxsw_sp_nexthop_group_info *nhgi; 7030 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 7031 struct mlxsw_sp_nexthop *nh; 7032 int err, i; 7033 7034 nhgi = kzalloc_flex(*nhgi, nexthops, fib6_entry->nrt6); 7035 if (!nhgi) 7036 return -ENOMEM; 7037 nh_grp->nhgi = nhgi; 7038 nhgi->nh_grp = nh_grp; 7039 mlxsw_sp_rt6 = list_first_entry(&fib6_entry->rt6_list, 7040 struct mlxsw_sp_rt6, list); 7041 nhgi->gateway = mlxsw_sp_rt6_is_gateway(mlxsw_sp, mlxsw_sp_rt6->rt); 7042 nhgi->count = fib6_entry->nrt6; 7043 for (i = 0; i < nhgi->count; i++) { 7044 struct fib6_info *rt = mlxsw_sp_rt6->rt; 7045 7046 nh = &nhgi->nexthops[i]; 7047 err = mlxsw_sp_nexthop6_init(mlxsw_sp, nh_grp, nh, rt); 7048 if (err) 7049 goto err_nexthop6_init; 7050 mlxsw_sp_rt6 = list_next_entry(mlxsw_sp_rt6, list); 7051 } 7052 nh_grp->nhgi = nhgi; 7053 err = mlxsw_sp_nexthop_group_inc(mlxsw_sp); 7054 if (err) 7055 goto err_group_inc; 7056 err = mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp); 7057 if (err) 7058 goto err_group_refresh; 7059 7060 return 0; 7061 7062 err_group_refresh: 7063 mlxsw_sp_nexthop_group_dec(mlxsw_sp); 7064 err_group_inc: 7065 i = nhgi->count; 7066 err_nexthop6_init: 7067 for (i--; i >= 0; i--) { 7068 nh = &nhgi->nexthops[i]; 7069 mlxsw_sp_nexthop6_fini(mlxsw_sp, nh); 7070 } 7071 kfree(nhgi); 7072 return err; 7073 } 7074 7075 static void 7076 mlxsw_sp_nexthop6_group_info_fini(struct mlxsw_sp *mlxsw_sp, 7077 struct mlxsw_sp_nexthop_group *nh_grp) 7078 { 7079 struct mlxsw_sp_nexthop_group_info *nhgi = nh_grp->nhgi; 7080 int i; 7081 7082 mlxsw_sp_nexthop_group_dec(mlxsw_sp); 7083 for (i = nhgi->count - 1; i >= 0; i--) { 7084 struct mlxsw_sp_nexthop *nh = &nhgi->nexthops[i]; 7085 7086 mlxsw_sp_nexthop6_fini(mlxsw_sp, nh); 7087 } 7088 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp); 7089 WARN_ON_ONCE(nhgi->adj_index_valid); 7090 kfree(nhgi); 7091 } 7092 7093 static struct mlxsw_sp_nexthop_group * 7094 mlxsw_sp_nexthop6_group_create(struct mlxsw_sp *mlxsw_sp, 7095 struct mlxsw_sp_fib6_entry *fib6_entry) 7096 { 7097 struct mlxsw_sp_nexthop_group *nh_grp; 7098 int err; 7099 7100 nh_grp = kzalloc_obj(*nh_grp); 7101 if (!nh_grp) 7102 return ERR_PTR(-ENOMEM); 7103 INIT_LIST_HEAD(&nh_grp->vr_list); 7104 err = rhashtable_init(&nh_grp->vr_ht, 7105 &mlxsw_sp_nexthop_group_vr_ht_params); 7106 if (err) 7107 goto err_nexthop_group_vr_ht_init; 7108 INIT_LIST_HEAD(&nh_grp->fib_list); 7109 nh_grp->type = MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6; 7110 7111 err = mlxsw_sp_nexthop6_group_info_init(mlxsw_sp, nh_grp, fib6_entry); 7112 if (err) 7113 goto err_nexthop_group_info_init; 7114 7115 err = mlxsw_sp_nexthop_group_insert(mlxsw_sp, nh_grp); 7116 if (err) 7117 goto err_nexthop_group_insert; 7118 7119 nh_grp->can_destroy = true; 7120 7121 return nh_grp; 7122 7123 err_nexthop_group_insert: 7124 mlxsw_sp_nexthop6_group_info_fini(mlxsw_sp, nh_grp); 7125 err_nexthop_group_info_init: 7126 rhashtable_destroy(&nh_grp->vr_ht); 7127 err_nexthop_group_vr_ht_init: 7128 kfree(nh_grp); 7129 return ERR_PTR(err); 7130 } 7131 7132 static void 7133 mlxsw_sp_nexthop6_group_destroy(struct mlxsw_sp *mlxsw_sp, 7134 struct mlxsw_sp_nexthop_group *nh_grp) 7135 { 7136 if (!nh_grp->can_destroy) 7137 return; 7138 mlxsw_sp_nexthop_group_remove(mlxsw_sp, nh_grp); 7139 mlxsw_sp_nexthop6_group_info_fini(mlxsw_sp, nh_grp); 7140 WARN_ON_ONCE(!list_empty(&nh_grp->vr_list)); 7141 rhashtable_destroy(&nh_grp->vr_ht); 7142 kfree(nh_grp); 7143 } 7144 7145 static int mlxsw_sp_nexthop6_group_get(struct mlxsw_sp *mlxsw_sp, 7146 struct mlxsw_sp_fib6_entry *fib6_entry) 7147 { 7148 struct fib6_info *rt = mlxsw_sp_fib6_entry_rt(fib6_entry); 7149 struct mlxsw_sp_nexthop_group *nh_grp; 7150 7151 if (rt->nh) { 7152 nh_grp = mlxsw_sp_nexthop_obj_group_lookup(mlxsw_sp, 7153 rt->nh->id); 7154 if (WARN_ON_ONCE(!nh_grp)) 7155 return -EINVAL; 7156 goto out; 7157 } 7158 7159 nh_grp = mlxsw_sp_nexthop6_group_lookup(mlxsw_sp, fib6_entry); 7160 if (!nh_grp) { 7161 nh_grp = mlxsw_sp_nexthop6_group_create(mlxsw_sp, fib6_entry); 7162 if (IS_ERR(nh_grp)) 7163 return PTR_ERR(nh_grp); 7164 } 7165 7166 /* The route and the nexthop are described by the same struct, so we 7167 * need to the update the nexthop offload indication for the new route. 7168 */ 7169 __mlxsw_sp_nexthop6_group_offload_refresh(nh_grp, fib6_entry); 7170 7171 out: 7172 list_add_tail(&fib6_entry->common.nexthop_group_node, 7173 &nh_grp->fib_list); 7174 fib6_entry->common.nh_group = nh_grp; 7175 7176 return 0; 7177 } 7178 7179 static void mlxsw_sp_nexthop6_group_put(struct mlxsw_sp *mlxsw_sp, 7180 struct mlxsw_sp_fib_entry *fib_entry) 7181 { 7182 struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group; 7183 7184 list_del(&fib_entry->nexthop_group_node); 7185 if (!list_empty(&nh_grp->fib_list)) 7186 return; 7187 7188 if (nh_grp->type == MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ) { 7189 mlxsw_sp_nexthop_obj_group_destroy(mlxsw_sp, nh_grp); 7190 return; 7191 } 7192 7193 mlxsw_sp_nexthop6_group_destroy(mlxsw_sp, nh_grp); 7194 } 7195 7196 static int 7197 mlxsw_sp_nexthop6_group_update(struct mlxsw_sp *mlxsw_sp, 7198 struct mlxsw_sp_fib6_entry *fib6_entry) 7199 { 7200 struct mlxsw_sp_nexthop_group *old_nh_grp = fib6_entry->common.nh_group; 7201 struct mlxsw_sp_fib_node *fib_node = fib6_entry->common.fib_node; 7202 int err; 7203 7204 mlxsw_sp_nexthop_group_vr_unlink(old_nh_grp, fib_node->fib); 7205 fib6_entry->common.nh_group = NULL; 7206 list_del(&fib6_entry->common.nexthop_group_node); 7207 7208 err = mlxsw_sp_nexthop6_group_get(mlxsw_sp, fib6_entry); 7209 if (err) 7210 goto err_nexthop6_group_get; 7211 7212 err = mlxsw_sp_nexthop_group_vr_link(fib6_entry->common.nh_group, 7213 fib_node->fib); 7214 if (err) 7215 goto err_nexthop_group_vr_link; 7216 7217 /* In case this entry is offloaded, then the adjacency index 7218 * currently associated with it in the device's table is that 7219 * of the old group. Start using the new one instead. 7220 */ 7221 err = mlxsw_sp_fib_entry_update(mlxsw_sp, &fib6_entry->common); 7222 if (err) 7223 goto err_fib_entry_update; 7224 7225 if (list_empty(&old_nh_grp->fib_list)) 7226 mlxsw_sp_nexthop6_group_destroy(mlxsw_sp, old_nh_grp); 7227 7228 return 0; 7229 7230 err_fib_entry_update: 7231 mlxsw_sp_nexthop_group_vr_unlink(fib6_entry->common.nh_group, 7232 fib_node->fib); 7233 err_nexthop_group_vr_link: 7234 mlxsw_sp_nexthop6_group_put(mlxsw_sp, &fib6_entry->common); 7235 err_nexthop6_group_get: 7236 list_add_tail(&fib6_entry->common.nexthop_group_node, 7237 &old_nh_grp->fib_list); 7238 fib6_entry->common.nh_group = old_nh_grp; 7239 mlxsw_sp_nexthop_group_vr_link(old_nh_grp, fib_node->fib); 7240 return err; 7241 } 7242 7243 static int 7244 mlxsw_sp_fib6_entry_nexthop_add(struct mlxsw_sp *mlxsw_sp, 7245 struct mlxsw_sp_fib6_entry *fib6_entry, 7246 struct fib6_info **rt_arr, unsigned int nrt6) 7247 { 7248 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 7249 int err, i; 7250 7251 for (i = 0; i < nrt6; i++) { 7252 mlxsw_sp_rt6 = mlxsw_sp_rt6_create(rt_arr[i]); 7253 if (IS_ERR(mlxsw_sp_rt6)) { 7254 err = PTR_ERR(mlxsw_sp_rt6); 7255 goto err_rt6_unwind; 7256 } 7257 7258 list_add_tail(&mlxsw_sp_rt6->list, &fib6_entry->rt6_list); 7259 fib6_entry->nrt6++; 7260 } 7261 7262 err = mlxsw_sp_nexthop6_group_update(mlxsw_sp, fib6_entry); 7263 if (err) 7264 goto err_rt6_unwind; 7265 7266 return 0; 7267 7268 err_rt6_unwind: 7269 for (; i > 0; i--) { 7270 fib6_entry->nrt6--; 7271 mlxsw_sp_rt6 = list_last_entry(&fib6_entry->rt6_list, 7272 struct mlxsw_sp_rt6, list); 7273 list_del(&mlxsw_sp_rt6->list); 7274 mlxsw_sp_rt6_destroy(mlxsw_sp_rt6); 7275 } 7276 return err; 7277 } 7278 7279 static void 7280 mlxsw_sp_fib6_entry_nexthop_del(struct mlxsw_sp *mlxsw_sp, 7281 struct mlxsw_sp_fib6_entry *fib6_entry, 7282 struct fib6_info **rt_arr, unsigned int nrt6) 7283 { 7284 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 7285 int i; 7286 7287 for (i = 0; i < nrt6; i++) { 7288 mlxsw_sp_rt6 = mlxsw_sp_fib6_entry_rt_find(fib6_entry, 7289 rt_arr[i]); 7290 if (WARN_ON_ONCE(!mlxsw_sp_rt6)) 7291 continue; 7292 7293 fib6_entry->nrt6--; 7294 list_del(&mlxsw_sp_rt6->list); 7295 mlxsw_sp_rt6_destroy(mlxsw_sp_rt6); 7296 } 7297 7298 mlxsw_sp_nexthop6_group_update(mlxsw_sp, fib6_entry); 7299 } 7300 7301 static int 7302 mlxsw_sp_fib6_entry_type_set_local(struct mlxsw_sp *mlxsw_sp, 7303 struct mlxsw_sp_fib_entry *fib_entry, 7304 const struct fib6_info *rt) 7305 { 7306 struct mlxsw_sp_nexthop_group_info *nhgi = fib_entry->nh_group->nhgi; 7307 union mlxsw_sp_l3addr dip = { .addr6 = rt->fib6_dst.addr }; 7308 u32 tb_id = mlxsw_sp_fix_tb_id(rt->fib6_table->tb6_id); 7309 struct mlxsw_sp_router *router = mlxsw_sp->router; 7310 int ifindex = nhgi->nexthops[0].ifindex; 7311 struct mlxsw_sp_ipip_entry *ipip_entry; 7312 7313 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP; 7314 ipip_entry = mlxsw_sp_ipip_entry_find_by_decap(mlxsw_sp, ifindex, 7315 MLXSW_SP_L3_PROTO_IPV6, 7316 dip); 7317 7318 if (ipip_entry && ipip_entry->ol_dev->flags & IFF_UP) { 7319 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP; 7320 return mlxsw_sp_fib_entry_decap_init(mlxsw_sp, fib_entry, 7321 ipip_entry); 7322 } 7323 if (mlxsw_sp_router_nve_is_decap(mlxsw_sp, tb_id, 7324 MLXSW_SP_L3_PROTO_IPV6, &dip)) { 7325 u32 tunnel_index; 7326 7327 tunnel_index = router->nve_decap_config.tunnel_index; 7328 fib_entry->decap.tunnel_index = tunnel_index; 7329 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP; 7330 } 7331 7332 return 0; 7333 } 7334 7335 static int mlxsw_sp_fib6_entry_type_set(struct mlxsw_sp *mlxsw_sp, 7336 struct mlxsw_sp_fib_entry *fib_entry, 7337 const struct fib6_info *rt) 7338 { 7339 if (rt->fib6_flags & RTF_LOCAL) 7340 return mlxsw_sp_fib6_entry_type_set_local(mlxsw_sp, fib_entry, 7341 rt); 7342 if (rt->fib6_flags & RTF_ANYCAST) 7343 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP; 7344 else if (rt->fib6_type == RTN_BLACKHOLE) 7345 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_BLACKHOLE; 7346 else if (rt->fib6_flags & RTF_REJECT) 7347 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_UNREACHABLE; 7348 else if (fib_entry->nh_group->nhgi->gateway) 7349 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_REMOTE; 7350 else 7351 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL; 7352 7353 return 0; 7354 } 7355 7356 static void 7357 mlxsw_sp_fib6_entry_rt_destroy_all(struct mlxsw_sp_fib6_entry *fib6_entry) 7358 { 7359 struct mlxsw_sp_rt6 *mlxsw_sp_rt6, *tmp; 7360 7361 list_for_each_entry_safe(mlxsw_sp_rt6, tmp, &fib6_entry->rt6_list, 7362 list) { 7363 fib6_entry->nrt6--; 7364 list_del(&mlxsw_sp_rt6->list); 7365 mlxsw_sp_rt6_destroy(mlxsw_sp_rt6); 7366 } 7367 } 7368 7369 static struct mlxsw_sp_fib6_entry * 7370 mlxsw_sp_fib6_entry_create(struct mlxsw_sp *mlxsw_sp, 7371 struct mlxsw_sp_fib_node *fib_node, 7372 struct fib6_info **rt_arr, unsigned int nrt6) 7373 { 7374 struct mlxsw_sp_fib6_entry *fib6_entry; 7375 struct mlxsw_sp_fib_entry *fib_entry; 7376 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 7377 int err, i; 7378 7379 fib6_entry = kzalloc_obj(*fib6_entry); 7380 if (!fib6_entry) 7381 return ERR_PTR(-ENOMEM); 7382 fib_entry = &fib6_entry->common; 7383 7384 INIT_LIST_HEAD(&fib6_entry->rt6_list); 7385 7386 for (i = 0; i < nrt6; i++) { 7387 mlxsw_sp_rt6 = mlxsw_sp_rt6_create(rt_arr[i]); 7388 if (IS_ERR(mlxsw_sp_rt6)) { 7389 err = PTR_ERR(mlxsw_sp_rt6); 7390 goto err_rt6_unwind; 7391 } 7392 list_add_tail(&mlxsw_sp_rt6->list, &fib6_entry->rt6_list); 7393 fib6_entry->nrt6++; 7394 } 7395 7396 err = mlxsw_sp_nexthop6_group_get(mlxsw_sp, fib6_entry); 7397 if (err) 7398 goto err_rt6_unwind; 7399 7400 err = mlxsw_sp_nexthop_group_vr_link(fib_entry->nh_group, 7401 fib_node->fib); 7402 if (err) 7403 goto err_nexthop_group_vr_link; 7404 7405 err = mlxsw_sp_fib6_entry_type_set(mlxsw_sp, fib_entry, rt_arr[0]); 7406 if (err) 7407 goto err_fib6_entry_type_set; 7408 7409 fib_entry->fib_node = fib_node; 7410 7411 return fib6_entry; 7412 7413 err_fib6_entry_type_set: 7414 mlxsw_sp_nexthop_group_vr_unlink(fib_entry->nh_group, fib_node->fib); 7415 err_nexthop_group_vr_link: 7416 mlxsw_sp_nexthop6_group_put(mlxsw_sp, fib_entry); 7417 err_rt6_unwind: 7418 for (; i > 0; i--) { 7419 fib6_entry->nrt6--; 7420 mlxsw_sp_rt6 = list_last_entry(&fib6_entry->rt6_list, 7421 struct mlxsw_sp_rt6, list); 7422 list_del(&mlxsw_sp_rt6->list); 7423 mlxsw_sp_rt6_destroy(mlxsw_sp_rt6); 7424 } 7425 kfree(fib6_entry); 7426 return ERR_PTR(err); 7427 } 7428 7429 static void 7430 mlxsw_sp_fib6_entry_type_unset(struct mlxsw_sp *mlxsw_sp, 7431 struct mlxsw_sp_fib6_entry *fib6_entry) 7432 { 7433 mlxsw_sp_fib_entry_type_unset(mlxsw_sp, &fib6_entry->common); 7434 } 7435 7436 static void mlxsw_sp_fib6_entry_destroy(struct mlxsw_sp *mlxsw_sp, 7437 struct mlxsw_sp_fib6_entry *fib6_entry) 7438 { 7439 struct mlxsw_sp_fib_node *fib_node = fib6_entry->common.fib_node; 7440 7441 mlxsw_sp_fib6_entry_type_unset(mlxsw_sp, fib6_entry); 7442 mlxsw_sp_nexthop_group_vr_unlink(fib6_entry->common.nh_group, 7443 fib_node->fib); 7444 mlxsw_sp_nexthop6_group_put(mlxsw_sp, &fib6_entry->common); 7445 mlxsw_sp_fib6_entry_rt_destroy_all(fib6_entry); 7446 WARN_ON(fib6_entry->nrt6); 7447 kfree(fib6_entry); 7448 } 7449 7450 static struct mlxsw_sp_fib6_entry * 7451 mlxsw_sp_fib6_entry_lookup(struct mlxsw_sp *mlxsw_sp, 7452 const struct fib6_info *rt) 7453 { 7454 struct mlxsw_sp_fib6_entry *fib6_entry; 7455 struct mlxsw_sp_fib_node *fib_node; 7456 struct mlxsw_sp_fib *fib; 7457 struct fib6_info *cmp_rt; 7458 struct mlxsw_sp_vr *vr; 7459 7460 vr = mlxsw_sp_vr_find(mlxsw_sp, rt->fib6_table->tb6_id); 7461 if (!vr) 7462 return NULL; 7463 fib = mlxsw_sp_vr_fib(vr, MLXSW_SP_L3_PROTO_IPV6); 7464 7465 fib_node = mlxsw_sp_fib_node_lookup(fib, &rt->fib6_dst.addr, 7466 sizeof(rt->fib6_dst.addr), 7467 rt->fib6_dst.plen); 7468 if (!fib_node) 7469 return NULL; 7470 7471 fib6_entry = container_of(fib_node->fib_entry, 7472 struct mlxsw_sp_fib6_entry, common); 7473 cmp_rt = mlxsw_sp_fib6_entry_rt(fib6_entry); 7474 if (rt->fib6_table->tb6_id == cmp_rt->fib6_table->tb6_id && 7475 rt->fib6_metric == cmp_rt->fib6_metric && 7476 mlxsw_sp_fib6_entry_rt_find(fib6_entry, rt)) 7477 return fib6_entry; 7478 7479 return NULL; 7480 } 7481 7482 static bool mlxsw_sp_fib6_allow_replace(struct mlxsw_sp_fib6_entry *fib6_entry) 7483 { 7484 struct mlxsw_sp_fib_node *fib_node = fib6_entry->common.fib_node; 7485 struct mlxsw_sp_fib6_entry *fib6_replaced; 7486 struct fib6_info *rt, *rt_replaced; 7487 7488 if (!fib_node->fib_entry) 7489 return true; 7490 7491 fib6_replaced = container_of(fib_node->fib_entry, 7492 struct mlxsw_sp_fib6_entry, 7493 common); 7494 rt = mlxsw_sp_fib6_entry_rt(fib6_entry); 7495 rt_replaced = mlxsw_sp_fib6_entry_rt(fib6_replaced); 7496 if (rt->fib6_table->tb6_id == RT_TABLE_MAIN && 7497 rt_replaced->fib6_table->tb6_id == RT_TABLE_LOCAL) 7498 return false; 7499 7500 return true; 7501 } 7502 7503 static int mlxsw_sp_router_fib6_replace(struct mlxsw_sp *mlxsw_sp, 7504 struct fib6_info **rt_arr, 7505 unsigned int nrt6) 7506 { 7507 struct mlxsw_sp_fib6_entry *fib6_entry, *fib6_replaced; 7508 struct mlxsw_sp_fib_entry *replaced; 7509 struct mlxsw_sp_fib_node *fib_node; 7510 struct fib6_info *rt = rt_arr[0]; 7511 int err; 7512 7513 if (rt->fib6_src.plen) 7514 return -EINVAL; 7515 7516 if (mlxsw_sp_fib6_rt_should_ignore(rt)) 7517 return 0; 7518 7519 if (rt->nh && !mlxsw_sp_nexthop_obj_group_lookup(mlxsw_sp, rt->nh->id)) 7520 return 0; 7521 7522 fib_node = mlxsw_sp_fib_node_get(mlxsw_sp, rt->fib6_table->tb6_id, 7523 &rt->fib6_dst.addr, 7524 sizeof(rt->fib6_dst.addr), 7525 rt->fib6_dst.plen, 7526 MLXSW_SP_L3_PROTO_IPV6); 7527 if (IS_ERR(fib_node)) 7528 return PTR_ERR(fib_node); 7529 7530 fib6_entry = mlxsw_sp_fib6_entry_create(mlxsw_sp, fib_node, rt_arr, 7531 nrt6); 7532 if (IS_ERR(fib6_entry)) { 7533 err = PTR_ERR(fib6_entry); 7534 goto err_fib6_entry_create; 7535 } 7536 7537 if (!mlxsw_sp_fib6_allow_replace(fib6_entry)) { 7538 mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry); 7539 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 7540 return 0; 7541 } 7542 7543 replaced = fib_node->fib_entry; 7544 err = mlxsw_sp_fib_node_entry_link(mlxsw_sp, &fib6_entry->common); 7545 if (err) 7546 goto err_fib_node_entry_link; 7547 7548 /* Nothing to replace */ 7549 if (!replaced) 7550 return 0; 7551 7552 mlxsw_sp_fib_entry_hw_flags_clear(mlxsw_sp, replaced); 7553 fib6_replaced = container_of(replaced, struct mlxsw_sp_fib6_entry, 7554 common); 7555 mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_replaced); 7556 7557 return 0; 7558 7559 err_fib_node_entry_link: 7560 fib_node->fib_entry = replaced; 7561 mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry); 7562 err_fib6_entry_create: 7563 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 7564 return err; 7565 } 7566 7567 static int mlxsw_sp_router_fib6_append(struct mlxsw_sp *mlxsw_sp, 7568 struct fib6_info **rt_arr, 7569 unsigned int nrt6) 7570 { 7571 struct mlxsw_sp_fib6_entry *fib6_entry; 7572 struct mlxsw_sp_fib_node *fib_node; 7573 struct fib6_info *rt = rt_arr[0]; 7574 int err; 7575 7576 if (rt->fib6_src.plen) 7577 return -EINVAL; 7578 7579 if (mlxsw_sp_fib6_rt_should_ignore(rt)) 7580 return 0; 7581 7582 fib_node = mlxsw_sp_fib_node_get(mlxsw_sp, rt->fib6_table->tb6_id, 7583 &rt->fib6_dst.addr, 7584 sizeof(rt->fib6_dst.addr), 7585 rt->fib6_dst.plen, 7586 MLXSW_SP_L3_PROTO_IPV6); 7587 if (IS_ERR(fib_node)) 7588 return PTR_ERR(fib_node); 7589 7590 if (WARN_ON_ONCE(!fib_node->fib_entry)) { 7591 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 7592 return -EINVAL; 7593 } 7594 7595 fib6_entry = container_of(fib_node->fib_entry, 7596 struct mlxsw_sp_fib6_entry, common); 7597 err = mlxsw_sp_fib6_entry_nexthop_add(mlxsw_sp, fib6_entry, rt_arr, 7598 nrt6); 7599 if (err) 7600 goto err_fib6_entry_nexthop_add; 7601 7602 return 0; 7603 7604 err_fib6_entry_nexthop_add: 7605 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 7606 return err; 7607 } 7608 7609 static void mlxsw_sp_router_fib6_del(struct mlxsw_sp *mlxsw_sp, 7610 struct fib6_info **rt_arr, 7611 unsigned int nrt6) 7612 { 7613 struct mlxsw_sp_fib6_entry *fib6_entry; 7614 struct mlxsw_sp_fib_node *fib_node; 7615 struct fib6_info *rt = rt_arr[0]; 7616 7617 if (mlxsw_sp_fib6_rt_should_ignore(rt)) 7618 return; 7619 7620 /* Multipath routes are first added to the FIB trie and only then 7621 * notified. If we vetoed the addition, we will get a delete 7622 * notification for a route we do not have. Therefore, do not warn if 7623 * route was not found. 7624 */ 7625 fib6_entry = mlxsw_sp_fib6_entry_lookup(mlxsw_sp, rt); 7626 if (!fib6_entry) 7627 return; 7628 7629 /* If not all the nexthops are deleted, then only reduce the nexthop 7630 * group. 7631 */ 7632 if (nrt6 != fib6_entry->nrt6) { 7633 mlxsw_sp_fib6_entry_nexthop_del(mlxsw_sp, fib6_entry, rt_arr, 7634 nrt6); 7635 return; 7636 } 7637 7638 fib_node = fib6_entry->common.fib_node; 7639 7640 mlxsw_sp_fib_node_entry_unlink(mlxsw_sp, &fib6_entry->common); 7641 mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry); 7642 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 7643 } 7644 7645 static struct mlxsw_sp_mr_table * 7646 mlxsw_sp_router_fibmr_family_to_table(struct mlxsw_sp_vr *vr, int family) 7647 { 7648 if (family == RTNL_FAMILY_IPMR) 7649 return vr->mr_table[MLXSW_SP_L3_PROTO_IPV4]; 7650 else 7651 return vr->mr_table[MLXSW_SP_L3_PROTO_IPV6]; 7652 } 7653 7654 static int mlxsw_sp_router_fibmr_add(struct mlxsw_sp *mlxsw_sp, 7655 struct mfc_entry_notifier_info *men_info, 7656 bool replace) 7657 { 7658 struct mlxsw_sp_mr_table *mrt; 7659 struct mlxsw_sp_vr *vr; 7660 7661 vr = mlxsw_sp_vr_get(mlxsw_sp, men_info->tb_id, NULL); 7662 if (IS_ERR(vr)) 7663 return PTR_ERR(vr); 7664 7665 mrt = mlxsw_sp_router_fibmr_family_to_table(vr, men_info->info.family); 7666 return mlxsw_sp_mr_route_add(mrt, men_info->mfc, replace); 7667 } 7668 7669 static void mlxsw_sp_router_fibmr_del(struct mlxsw_sp *mlxsw_sp, 7670 struct mfc_entry_notifier_info *men_info) 7671 { 7672 struct mlxsw_sp_mr_table *mrt; 7673 struct mlxsw_sp_vr *vr; 7674 7675 vr = mlxsw_sp_vr_find(mlxsw_sp, men_info->tb_id); 7676 if (WARN_ON(!vr)) 7677 return; 7678 7679 mrt = mlxsw_sp_router_fibmr_family_to_table(vr, men_info->info.family); 7680 mlxsw_sp_mr_route_del(mrt, men_info->mfc); 7681 mlxsw_sp_vr_put(mlxsw_sp, vr); 7682 } 7683 7684 static int 7685 mlxsw_sp_router_fibmr_vif_add(struct mlxsw_sp *mlxsw_sp, 7686 struct vif_entry_notifier_info *ven_info) 7687 { 7688 struct mlxsw_sp_mr_table *mrt; 7689 struct mlxsw_sp_rif *rif; 7690 struct mlxsw_sp_vr *vr; 7691 7692 vr = mlxsw_sp_vr_get(mlxsw_sp, ven_info->tb_id, NULL); 7693 if (IS_ERR(vr)) 7694 return PTR_ERR(vr); 7695 7696 mrt = mlxsw_sp_router_fibmr_family_to_table(vr, ven_info->info.family); 7697 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, ven_info->dev); 7698 return mlxsw_sp_mr_vif_add(mrt, ven_info->dev, 7699 ven_info->vif_index, 7700 ven_info->vif_flags, rif); 7701 } 7702 7703 static void 7704 mlxsw_sp_router_fibmr_vif_del(struct mlxsw_sp *mlxsw_sp, 7705 struct vif_entry_notifier_info *ven_info) 7706 { 7707 struct mlxsw_sp_mr_table *mrt; 7708 struct mlxsw_sp_vr *vr; 7709 7710 vr = mlxsw_sp_vr_find(mlxsw_sp, ven_info->tb_id); 7711 if (WARN_ON(!vr)) 7712 return; 7713 7714 mrt = mlxsw_sp_router_fibmr_family_to_table(vr, ven_info->info.family); 7715 mlxsw_sp_mr_vif_del(mrt, ven_info->vif_index); 7716 mlxsw_sp_vr_put(mlxsw_sp, vr); 7717 } 7718 7719 static void mlxsw_sp_fib4_node_flush(struct mlxsw_sp *mlxsw_sp, 7720 struct mlxsw_sp_fib_node *fib_node) 7721 { 7722 struct mlxsw_sp_fib4_entry *fib4_entry; 7723 7724 fib4_entry = container_of(fib_node->fib_entry, 7725 struct mlxsw_sp_fib4_entry, common); 7726 mlxsw_sp_fib_node_entry_unlink(mlxsw_sp, fib_node->fib_entry); 7727 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry); 7728 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 7729 } 7730 7731 static void mlxsw_sp_fib6_node_flush(struct mlxsw_sp *mlxsw_sp, 7732 struct mlxsw_sp_fib_node *fib_node) 7733 { 7734 struct mlxsw_sp_fib6_entry *fib6_entry; 7735 7736 fib6_entry = container_of(fib_node->fib_entry, 7737 struct mlxsw_sp_fib6_entry, common); 7738 mlxsw_sp_fib_node_entry_unlink(mlxsw_sp, fib_node->fib_entry); 7739 mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry); 7740 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 7741 } 7742 7743 static void mlxsw_sp_fib_node_flush(struct mlxsw_sp *mlxsw_sp, 7744 struct mlxsw_sp_fib_node *fib_node) 7745 { 7746 switch (fib_node->fib->proto) { 7747 case MLXSW_SP_L3_PROTO_IPV4: 7748 mlxsw_sp_fib4_node_flush(mlxsw_sp, fib_node); 7749 break; 7750 case MLXSW_SP_L3_PROTO_IPV6: 7751 mlxsw_sp_fib6_node_flush(mlxsw_sp, fib_node); 7752 break; 7753 } 7754 } 7755 7756 static void mlxsw_sp_vr_fib_flush(struct mlxsw_sp *mlxsw_sp, 7757 struct mlxsw_sp_vr *vr, 7758 enum mlxsw_sp_l3proto proto) 7759 { 7760 struct mlxsw_sp_fib *fib = mlxsw_sp_vr_fib(vr, proto); 7761 struct mlxsw_sp_fib_node *fib_node, *tmp; 7762 7763 list_for_each_entry_safe(fib_node, tmp, &fib->node_list, list) { 7764 bool do_break = &tmp->list == &fib->node_list; 7765 7766 mlxsw_sp_fib_node_flush(mlxsw_sp, fib_node); 7767 if (do_break) 7768 break; 7769 } 7770 } 7771 7772 static void mlxsw_sp_router_fib_flush(struct mlxsw_sp *mlxsw_sp) 7773 { 7774 int max_vrs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); 7775 int i, j; 7776 7777 for (i = 0; i < max_vrs; i++) { 7778 struct mlxsw_sp_vr *vr = &mlxsw_sp->router->vrs[i]; 7779 7780 if (!mlxsw_sp_vr_is_used(vr)) 7781 continue; 7782 7783 for (j = 0; j < MLXSW_SP_L3_PROTO_MAX; j++) 7784 mlxsw_sp_mr_table_flush(vr->mr_table[j]); 7785 mlxsw_sp_vr_fib_flush(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV4); 7786 7787 /* If virtual router was only used for IPv4, then it's no 7788 * longer used. 7789 */ 7790 if (!mlxsw_sp_vr_is_used(vr)) 7791 continue; 7792 mlxsw_sp_vr_fib_flush(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV6); 7793 } 7794 } 7795 7796 struct mlxsw_sp_fib6_event_work { 7797 struct fib6_info **rt_arr; 7798 unsigned int nrt6; 7799 }; 7800 7801 struct mlxsw_sp_fib_event_work { 7802 struct work_struct work; 7803 netdevice_tracker dev_tracker; 7804 union { 7805 struct mlxsw_sp_fib6_event_work fib6_work; 7806 struct fib_entry_notifier_info fen_info; 7807 struct fib_rule_notifier_info fr_info; 7808 struct fib_nh_notifier_info fnh_info; 7809 struct mfc_entry_notifier_info men_info; 7810 struct vif_entry_notifier_info ven_info; 7811 }; 7812 struct mlxsw_sp *mlxsw_sp; 7813 unsigned long event; 7814 }; 7815 7816 static int 7817 mlxsw_sp_router_fib6_work_init(struct mlxsw_sp_fib6_event_work *fib6_work, 7818 struct fib6_entry_notifier_info *fen6_info) 7819 { 7820 struct fib6_info *rt = fen6_info->rt; 7821 struct fib6_info **rt_arr; 7822 struct fib6_info *iter; 7823 unsigned int nrt6; 7824 int i = 0; 7825 7826 nrt6 = fen6_info->nsiblings + 1; 7827 7828 rt_arr = kzalloc_objs(struct fib6_info *, nrt6, GFP_ATOMIC); 7829 if (!rt_arr) 7830 return -ENOMEM; 7831 7832 fib6_work->rt_arr = rt_arr; 7833 fib6_work->nrt6 = nrt6; 7834 7835 rt_arr[0] = rt; 7836 fib6_info_hold(rt); 7837 7838 if (!fen6_info->nsiblings) 7839 return 0; 7840 7841 list_for_each_entry(iter, &rt->fib6_siblings, fib6_siblings) { 7842 if (i == fen6_info->nsiblings) 7843 break; 7844 7845 rt_arr[i + 1] = iter; 7846 fib6_info_hold(iter); 7847 i++; 7848 } 7849 WARN_ON_ONCE(i != fen6_info->nsiblings); 7850 7851 return 0; 7852 } 7853 7854 static void 7855 mlxsw_sp_router_fib6_work_fini(struct mlxsw_sp_fib6_event_work *fib6_work) 7856 { 7857 int i; 7858 7859 for (i = 0; i < fib6_work->nrt6; i++) 7860 mlxsw_sp_rt6_release(fib6_work->rt_arr[i]); 7861 kfree(fib6_work->rt_arr); 7862 } 7863 7864 static void mlxsw_sp_router_fib4_event_work(struct work_struct *work) 7865 { 7866 struct mlxsw_sp_fib_event_work *fib_work = 7867 container_of(work, struct mlxsw_sp_fib_event_work, work); 7868 struct mlxsw_sp *mlxsw_sp = fib_work->mlxsw_sp; 7869 int err; 7870 7871 mutex_lock(&mlxsw_sp->router->lock); 7872 mlxsw_sp_span_respin(mlxsw_sp); 7873 7874 switch (fib_work->event) { 7875 case FIB_EVENT_ENTRY_REPLACE: 7876 err = mlxsw_sp_router_fib4_replace(mlxsw_sp, 7877 &fib_work->fen_info); 7878 if (err) { 7879 dev_warn(mlxsw_sp->bus_info->dev, "FIB replace failed.\n"); 7880 mlxsw_sp_fib4_offload_failed_flag_set(mlxsw_sp, 7881 &fib_work->fen_info); 7882 } 7883 fib_info_put(fib_work->fen_info.fi); 7884 break; 7885 case FIB_EVENT_ENTRY_DEL: 7886 mlxsw_sp_router_fib4_del(mlxsw_sp, &fib_work->fen_info); 7887 fib_info_put(fib_work->fen_info.fi); 7888 break; 7889 case FIB_EVENT_NH_ADD: 7890 case FIB_EVENT_NH_DEL: 7891 mlxsw_sp_nexthop4_event(mlxsw_sp, fib_work->event, 7892 fib_work->fnh_info.fib_nh); 7893 fib_info_put(fib_work->fnh_info.fib_nh->nh_parent); 7894 break; 7895 } 7896 mutex_unlock(&mlxsw_sp->router->lock); 7897 kfree(fib_work); 7898 } 7899 7900 static void mlxsw_sp_router_fib6_event_work(struct work_struct *work) 7901 { 7902 struct mlxsw_sp_fib_event_work *fib_work = 7903 container_of(work, struct mlxsw_sp_fib_event_work, work); 7904 struct mlxsw_sp_fib6_event_work *fib6_work = &fib_work->fib6_work; 7905 struct mlxsw_sp *mlxsw_sp = fib_work->mlxsw_sp; 7906 int err; 7907 7908 mutex_lock(&mlxsw_sp->router->lock); 7909 mlxsw_sp_span_respin(mlxsw_sp); 7910 7911 switch (fib_work->event) { 7912 case FIB_EVENT_ENTRY_REPLACE: 7913 err = mlxsw_sp_router_fib6_replace(mlxsw_sp, 7914 fib6_work->rt_arr, 7915 fib6_work->nrt6); 7916 if (err) { 7917 dev_warn(mlxsw_sp->bus_info->dev, "FIB replace failed.\n"); 7918 mlxsw_sp_fib6_offload_failed_flag_set(mlxsw_sp, 7919 fib6_work->rt_arr, 7920 fib6_work->nrt6); 7921 } 7922 mlxsw_sp_router_fib6_work_fini(fib6_work); 7923 break; 7924 case FIB_EVENT_ENTRY_APPEND: 7925 err = mlxsw_sp_router_fib6_append(mlxsw_sp, 7926 fib6_work->rt_arr, 7927 fib6_work->nrt6); 7928 if (err) { 7929 dev_warn(mlxsw_sp->bus_info->dev, "FIB append failed.\n"); 7930 mlxsw_sp_fib6_offload_failed_flag_set(mlxsw_sp, 7931 fib6_work->rt_arr, 7932 fib6_work->nrt6); 7933 } 7934 mlxsw_sp_router_fib6_work_fini(fib6_work); 7935 break; 7936 case FIB_EVENT_ENTRY_DEL: 7937 mlxsw_sp_router_fib6_del(mlxsw_sp, 7938 fib6_work->rt_arr, 7939 fib6_work->nrt6); 7940 mlxsw_sp_router_fib6_work_fini(fib6_work); 7941 break; 7942 } 7943 mutex_unlock(&mlxsw_sp->router->lock); 7944 kfree(fib_work); 7945 } 7946 7947 static void mlxsw_sp_router_fibmr_event_work(struct work_struct *work) 7948 { 7949 struct mlxsw_sp_fib_event_work *fib_work = 7950 container_of(work, struct mlxsw_sp_fib_event_work, work); 7951 struct mlxsw_sp *mlxsw_sp = fib_work->mlxsw_sp; 7952 bool replace; 7953 int err; 7954 7955 rtnl_lock(); 7956 mutex_lock(&mlxsw_sp->router->lock); 7957 switch (fib_work->event) { 7958 case FIB_EVENT_ENTRY_REPLACE: 7959 case FIB_EVENT_ENTRY_ADD: 7960 replace = fib_work->event == FIB_EVENT_ENTRY_REPLACE; 7961 7962 err = mlxsw_sp_router_fibmr_add(mlxsw_sp, &fib_work->men_info, 7963 replace); 7964 if (err) 7965 dev_warn(mlxsw_sp->bus_info->dev, "MR entry add failed.\n"); 7966 mr_cache_put(fib_work->men_info.mfc); 7967 break; 7968 case FIB_EVENT_ENTRY_DEL: 7969 mlxsw_sp_router_fibmr_del(mlxsw_sp, &fib_work->men_info); 7970 mr_cache_put(fib_work->men_info.mfc); 7971 break; 7972 case FIB_EVENT_VIF_ADD: 7973 err = mlxsw_sp_router_fibmr_vif_add(mlxsw_sp, 7974 &fib_work->ven_info); 7975 if (err) 7976 dev_warn(mlxsw_sp->bus_info->dev, "MR VIF add failed.\n"); 7977 netdev_put(fib_work->ven_info.dev, &fib_work->dev_tracker); 7978 break; 7979 case FIB_EVENT_VIF_DEL: 7980 mlxsw_sp_router_fibmr_vif_del(mlxsw_sp, 7981 &fib_work->ven_info); 7982 netdev_put(fib_work->ven_info.dev, &fib_work->dev_tracker); 7983 break; 7984 } 7985 mutex_unlock(&mlxsw_sp->router->lock); 7986 rtnl_unlock(); 7987 kfree(fib_work); 7988 } 7989 7990 static void mlxsw_sp_router_fib4_event(struct mlxsw_sp_fib_event_work *fib_work, 7991 struct fib_notifier_info *info) 7992 { 7993 struct fib_entry_notifier_info *fen_info; 7994 struct fib_nh_notifier_info *fnh_info; 7995 7996 switch (fib_work->event) { 7997 case FIB_EVENT_ENTRY_REPLACE: 7998 case FIB_EVENT_ENTRY_DEL: 7999 fen_info = container_of(info, struct fib_entry_notifier_info, 8000 info); 8001 fib_work->fen_info = *fen_info; 8002 /* Take reference on fib_info to prevent it from being 8003 * freed while work is queued. Release it afterwards. 8004 */ 8005 fib_info_hold(fib_work->fen_info.fi); 8006 break; 8007 case FIB_EVENT_NH_ADD: 8008 case FIB_EVENT_NH_DEL: 8009 fnh_info = container_of(info, struct fib_nh_notifier_info, 8010 info); 8011 fib_work->fnh_info = *fnh_info; 8012 fib_info_hold(fib_work->fnh_info.fib_nh->nh_parent); 8013 break; 8014 } 8015 } 8016 8017 static int mlxsw_sp_router_fib6_event(struct mlxsw_sp_fib_event_work *fib_work, 8018 struct fib_notifier_info *info) 8019 { 8020 struct fib6_entry_notifier_info *fen6_info; 8021 int err; 8022 8023 switch (fib_work->event) { 8024 case FIB_EVENT_ENTRY_REPLACE: 8025 case FIB_EVENT_ENTRY_APPEND: 8026 case FIB_EVENT_ENTRY_DEL: 8027 fen6_info = container_of(info, struct fib6_entry_notifier_info, 8028 info); 8029 err = mlxsw_sp_router_fib6_work_init(&fib_work->fib6_work, 8030 fen6_info); 8031 if (err) 8032 return err; 8033 break; 8034 } 8035 8036 return 0; 8037 } 8038 8039 static void 8040 mlxsw_sp_router_fibmr_event(struct mlxsw_sp_fib_event_work *fib_work, 8041 struct fib_notifier_info *info) 8042 { 8043 switch (fib_work->event) { 8044 case FIB_EVENT_ENTRY_REPLACE: 8045 case FIB_EVENT_ENTRY_ADD: 8046 case FIB_EVENT_ENTRY_DEL: 8047 memcpy(&fib_work->men_info, info, sizeof(fib_work->men_info)); 8048 mr_cache_hold(fib_work->men_info.mfc); 8049 break; 8050 case FIB_EVENT_VIF_ADD: 8051 case FIB_EVENT_VIF_DEL: 8052 memcpy(&fib_work->ven_info, info, sizeof(fib_work->ven_info)); 8053 netdev_hold(fib_work->ven_info.dev, &fib_work->dev_tracker, 8054 GFP_ATOMIC); 8055 break; 8056 } 8057 } 8058 8059 static int mlxsw_sp_router_fib_rule_event(unsigned long event, 8060 struct fib_notifier_info *info, 8061 struct mlxsw_sp *mlxsw_sp) 8062 { 8063 struct netlink_ext_ack *extack = info->extack; 8064 struct fib_rule_notifier_info *fr_info; 8065 struct fib_rule *rule; 8066 int err = 0; 8067 8068 /* nothing to do at the moment */ 8069 if (event == FIB_EVENT_RULE_DEL) 8070 return 0; 8071 8072 fr_info = container_of(info, struct fib_rule_notifier_info, info); 8073 rule = fr_info->rule; 8074 8075 /* Rule only affects locally generated traffic */ 8076 if (rule->iifindex == mlxsw_sp_net(mlxsw_sp)->loopback_dev->ifindex) 8077 return 0; 8078 8079 switch (info->family) { 8080 case AF_INET: 8081 if (!fib4_rule_default(rule) && !rule->l3mdev) 8082 err = -EOPNOTSUPP; 8083 break; 8084 case AF_INET6: 8085 if (!fib6_rule_default(rule) && !rule->l3mdev) 8086 err = -EOPNOTSUPP; 8087 break; 8088 case RTNL_FAMILY_IPMR: 8089 if (!ipmr_rule_default(rule) && !rule->l3mdev) 8090 err = -EOPNOTSUPP; 8091 break; 8092 case RTNL_FAMILY_IP6MR: 8093 if (!ip6mr_rule_default(rule) && !rule->l3mdev) 8094 err = -EOPNOTSUPP; 8095 break; 8096 } 8097 8098 if (err < 0) 8099 NL_SET_ERR_MSG_MOD(extack, "FIB rules not supported"); 8100 8101 return err; 8102 } 8103 8104 /* Called with rcu_read_lock() */ 8105 static int mlxsw_sp_router_fib_event(struct notifier_block *nb, 8106 unsigned long event, void *ptr) 8107 { 8108 struct mlxsw_sp_fib_event_work *fib_work; 8109 struct fib_notifier_info *info = ptr; 8110 struct mlxsw_sp_router *router; 8111 int err; 8112 8113 if ((info->family != AF_INET && info->family != AF_INET6 && 8114 info->family != RTNL_FAMILY_IPMR && 8115 info->family != RTNL_FAMILY_IP6MR)) 8116 return NOTIFY_DONE; 8117 8118 router = container_of(nb, struct mlxsw_sp_router, fib_nb); 8119 8120 switch (event) { 8121 case FIB_EVENT_RULE_ADD: 8122 case FIB_EVENT_RULE_DEL: 8123 err = mlxsw_sp_router_fib_rule_event(event, info, 8124 router->mlxsw_sp); 8125 return notifier_from_errno(err); 8126 case FIB_EVENT_ENTRY_ADD: 8127 case FIB_EVENT_ENTRY_REPLACE: 8128 case FIB_EVENT_ENTRY_APPEND: 8129 if (info->family == AF_INET) { 8130 struct fib_entry_notifier_info *fen_info = ptr; 8131 8132 if (fen_info->fi->fib_nh_is_v6) { 8133 NL_SET_ERR_MSG_MOD(info->extack, "IPv6 gateway with IPv4 route is not supported"); 8134 return notifier_from_errno(-EINVAL); 8135 } 8136 } 8137 break; 8138 } 8139 8140 fib_work = kzalloc_obj(*fib_work, GFP_ATOMIC); 8141 if (!fib_work) 8142 return NOTIFY_BAD; 8143 8144 fib_work->mlxsw_sp = router->mlxsw_sp; 8145 fib_work->event = event; 8146 8147 switch (info->family) { 8148 case AF_INET: 8149 INIT_WORK(&fib_work->work, mlxsw_sp_router_fib4_event_work); 8150 mlxsw_sp_router_fib4_event(fib_work, info); 8151 break; 8152 case AF_INET6: 8153 INIT_WORK(&fib_work->work, mlxsw_sp_router_fib6_event_work); 8154 err = mlxsw_sp_router_fib6_event(fib_work, info); 8155 if (err) 8156 goto err_fib_event; 8157 break; 8158 case RTNL_FAMILY_IP6MR: 8159 case RTNL_FAMILY_IPMR: 8160 INIT_WORK(&fib_work->work, mlxsw_sp_router_fibmr_event_work); 8161 mlxsw_sp_router_fibmr_event(fib_work, info); 8162 break; 8163 } 8164 8165 mlxsw_core_schedule_work(&fib_work->work); 8166 8167 return NOTIFY_DONE; 8168 8169 err_fib_event: 8170 kfree(fib_work); 8171 return NOTIFY_BAD; 8172 } 8173 8174 static struct mlxsw_sp_rif * 8175 mlxsw_sp_rif_find_by_dev(const struct mlxsw_sp *mlxsw_sp, 8176 const struct net_device *dev) 8177 { 8178 int max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); 8179 int i; 8180 8181 for (i = 0; i < max_rifs; i++) 8182 if (mlxsw_sp->router->rifs[i] && 8183 mlxsw_sp_rif_dev_is(mlxsw_sp->router->rifs[i], dev)) 8184 return mlxsw_sp->router->rifs[i]; 8185 8186 return NULL; 8187 } 8188 8189 static int mlxsw_sp_router_rif_disable(struct mlxsw_sp *mlxsw_sp, u16 rif) 8190 { 8191 char ritr_pl[MLXSW_REG_RITR_LEN]; 8192 int err; 8193 8194 mlxsw_reg_ritr_rif_pack(ritr_pl, rif); 8195 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 8196 if (err) 8197 return err; 8198 8199 mlxsw_reg_ritr_enable_set(ritr_pl, false); 8200 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 8201 } 8202 8203 static int mlxsw_sp_router_rif_made_sync(struct mlxsw_sp *mlxsw_sp, 8204 struct mlxsw_sp_rif *rif) 8205 { 8206 int err; 8207 8208 err = mlxsw_sp_neigh_rif_made_sync(mlxsw_sp, rif); 8209 if (err) 8210 return err; 8211 8212 err = mlxsw_sp_nexthop_rif_made_sync(mlxsw_sp, rif); 8213 if (err) 8214 goto err_nexthop; 8215 8216 return 0; 8217 8218 err_nexthop: 8219 mlxsw_sp_neigh_rif_gone_sync(mlxsw_sp, rif); 8220 return err; 8221 } 8222 8223 static void mlxsw_sp_router_rif_gone_sync(struct mlxsw_sp *mlxsw_sp, 8224 struct mlxsw_sp_rif *rif) 8225 { 8226 /* Signal to nexthop cleanup that the RIF is going away. */ 8227 rif->crif->rif = NULL; 8228 8229 mlxsw_sp_router_rif_disable(mlxsw_sp, rif->rif_index); 8230 mlxsw_sp_nexthop_rif_gone_sync(mlxsw_sp, rif); 8231 mlxsw_sp_neigh_rif_gone_sync(mlxsw_sp, rif); 8232 } 8233 8234 static bool __mlxsw_sp_dev_addr_list_empty(const struct net_device *dev) 8235 { 8236 struct inet6_dev *inet6_dev; 8237 struct in_device *idev; 8238 8239 idev = __in_dev_get_rcu(dev); 8240 if (idev && idev->ifa_list) 8241 return false; 8242 8243 inet6_dev = __in6_dev_get(dev); 8244 if (inet6_dev && !list_empty(&inet6_dev->addr_list)) 8245 return false; 8246 8247 return true; 8248 } 8249 8250 static bool mlxsw_sp_dev_addr_list_empty(const struct net_device *dev) 8251 { 8252 bool addr_list_empty; 8253 8254 rcu_read_lock(); 8255 addr_list_empty = __mlxsw_sp_dev_addr_list_empty(dev); 8256 rcu_read_unlock(); 8257 8258 return addr_list_empty; 8259 } 8260 8261 static bool 8262 mlxsw_sp_rif_should_config(struct mlxsw_sp_rif *rif, struct net_device *dev, 8263 unsigned long event) 8264 { 8265 bool addr_list_empty; 8266 8267 switch (event) { 8268 case NETDEV_UP: 8269 return rif == NULL; 8270 case NETDEV_DOWN: 8271 addr_list_empty = mlxsw_sp_dev_addr_list_empty(dev); 8272 8273 /* macvlans do not have a RIF, but rather piggy back on the 8274 * RIF of their lower device. 8275 */ 8276 if (netif_is_macvlan(dev) && addr_list_empty) 8277 return true; 8278 8279 if (rif && addr_list_empty && 8280 !netif_is_l3_slave(mlxsw_sp_rif_dev(rif))) 8281 return true; 8282 /* It is possible we already removed the RIF ourselves 8283 * if it was assigned to a netdev that is now a bridge 8284 * or LAG slave. 8285 */ 8286 return false; 8287 } 8288 8289 return false; 8290 } 8291 8292 static enum mlxsw_sp_rif_type 8293 mlxsw_sp_dev_rif_type(const struct mlxsw_sp *mlxsw_sp, 8294 const struct net_device *dev) 8295 { 8296 enum mlxsw_sp_fid_type type; 8297 8298 if (mlxsw_sp_netdev_ipip_type(mlxsw_sp, dev, NULL)) 8299 return MLXSW_SP_RIF_TYPE_IPIP_LB; 8300 8301 /* Otherwise RIF type is derived from the type of the underlying FID. */ 8302 if (is_vlan_dev(dev) && netif_is_bridge_master(vlan_dev_real_dev(dev))) 8303 type = MLXSW_SP_FID_TYPE_8021Q; 8304 else if (netif_is_bridge_master(dev) && br_vlan_enabled(dev)) 8305 type = MLXSW_SP_FID_TYPE_8021Q; 8306 else if (netif_is_bridge_master(dev)) 8307 type = MLXSW_SP_FID_TYPE_8021D; 8308 else 8309 type = MLXSW_SP_FID_TYPE_RFID; 8310 8311 return mlxsw_sp_fid_type_rif_type(mlxsw_sp, type); 8312 } 8313 8314 static int mlxsw_sp_rif_index_alloc(struct mlxsw_sp *mlxsw_sp, u16 *p_rif_index, 8315 u8 rif_entries) 8316 { 8317 *p_rif_index = gen_pool_alloc(mlxsw_sp->router->rifs_table, 8318 rif_entries); 8319 if (*p_rif_index == 0) 8320 return -ENOBUFS; 8321 *p_rif_index -= MLXSW_SP_ROUTER_GENALLOC_OFFSET; 8322 8323 /* RIF indexes must be aligned to the allocation size. */ 8324 WARN_ON_ONCE(*p_rif_index % rif_entries); 8325 8326 return 0; 8327 } 8328 8329 static void mlxsw_sp_rif_index_free(struct mlxsw_sp *mlxsw_sp, u16 rif_index, 8330 u8 rif_entries) 8331 { 8332 gen_pool_free(mlxsw_sp->router->rifs_table, 8333 MLXSW_SP_ROUTER_GENALLOC_OFFSET + rif_index, rif_entries); 8334 } 8335 8336 static struct mlxsw_sp_rif *mlxsw_sp_rif_alloc(size_t rif_size, u16 rif_index, 8337 u16 vr_id, 8338 struct mlxsw_sp_crif *crif) 8339 { 8340 struct net_device *l3_dev = crif ? crif->key.dev : NULL; 8341 struct mlxsw_sp_rif *rif; 8342 8343 rif = kzalloc(rif_size, GFP_KERNEL); 8344 if (!rif) 8345 return NULL; 8346 8347 INIT_LIST_HEAD(&rif->neigh_list); 8348 if (l3_dev) { 8349 ether_addr_copy(rif->addr, l3_dev->dev_addr); 8350 rif->mtu = l3_dev->mtu; 8351 } 8352 rif->vr_id = vr_id; 8353 rif->rif_index = rif_index; 8354 if (crif) { 8355 rif->crif = crif; 8356 crif->rif = rif; 8357 } 8358 8359 return rif; 8360 } 8361 8362 static void mlxsw_sp_rif_free(struct mlxsw_sp_rif *rif) 8363 { 8364 WARN_ON(!list_empty(&rif->neigh_list)); 8365 8366 if (rif->crif) 8367 rif->crif->rif = NULL; 8368 kfree(rif); 8369 } 8370 8371 struct mlxsw_sp_rif *mlxsw_sp_rif_by_index(const struct mlxsw_sp *mlxsw_sp, 8372 u16 rif_index) 8373 { 8374 return mlxsw_sp->router->rifs[rif_index]; 8375 } 8376 8377 u16 mlxsw_sp_rif_index(const struct mlxsw_sp_rif *rif) 8378 { 8379 return rif->rif_index; 8380 } 8381 8382 u16 mlxsw_sp_ipip_lb_rif_index(const struct mlxsw_sp_rif_ipip_lb *lb_rif) 8383 { 8384 return lb_rif->common.rif_index; 8385 } 8386 8387 u16 mlxsw_sp_ipip_lb_ul_rif_id(const struct mlxsw_sp_rif_ipip_lb *lb_rif) 8388 { 8389 return lb_rif->ul_rif_id; 8390 } 8391 8392 static bool 8393 mlxsw_sp_router_port_l3_stats_enabled(struct mlxsw_sp_rif *rif) 8394 { 8395 return mlxsw_sp_rif_counter_valid_get(rif, 8396 MLXSW_SP_RIF_COUNTER_EGRESS) && 8397 mlxsw_sp_rif_counter_valid_get(rif, 8398 MLXSW_SP_RIF_COUNTER_INGRESS); 8399 } 8400 8401 static int 8402 mlxsw_sp_router_port_l3_stats_enable(struct mlxsw_sp_rif *rif) 8403 { 8404 int err; 8405 8406 err = mlxsw_sp_rif_counter_alloc(rif, MLXSW_SP_RIF_COUNTER_INGRESS); 8407 if (err) 8408 return err; 8409 8410 /* Clear stale data. */ 8411 err = mlxsw_sp_rif_counter_fetch_clear(rif, 8412 MLXSW_SP_RIF_COUNTER_INGRESS, 8413 NULL); 8414 if (err) 8415 goto err_clear_ingress; 8416 8417 err = mlxsw_sp_rif_counter_alloc(rif, MLXSW_SP_RIF_COUNTER_EGRESS); 8418 if (err) 8419 goto err_alloc_egress; 8420 8421 /* Clear stale data. */ 8422 err = mlxsw_sp_rif_counter_fetch_clear(rif, 8423 MLXSW_SP_RIF_COUNTER_EGRESS, 8424 NULL); 8425 if (err) 8426 goto err_clear_egress; 8427 8428 return 0; 8429 8430 err_clear_egress: 8431 mlxsw_sp_rif_counter_free(rif, MLXSW_SP_RIF_COUNTER_EGRESS); 8432 err_alloc_egress: 8433 err_clear_ingress: 8434 mlxsw_sp_rif_counter_free(rif, MLXSW_SP_RIF_COUNTER_INGRESS); 8435 return err; 8436 } 8437 8438 static void 8439 mlxsw_sp_router_port_l3_stats_disable(struct mlxsw_sp_rif *rif) 8440 { 8441 mlxsw_sp_rif_counter_free(rif, MLXSW_SP_RIF_COUNTER_EGRESS); 8442 mlxsw_sp_rif_counter_free(rif, MLXSW_SP_RIF_COUNTER_INGRESS); 8443 } 8444 8445 static void 8446 mlxsw_sp_router_port_l3_stats_report_used(struct mlxsw_sp_rif *rif, 8447 struct netdev_notifier_offload_xstats_info *info) 8448 { 8449 if (!mlxsw_sp_router_port_l3_stats_enabled(rif)) 8450 return; 8451 netdev_offload_xstats_report_used(info->report_used); 8452 } 8453 8454 static int 8455 mlxsw_sp_router_port_l3_stats_fetch(struct mlxsw_sp_rif *rif, 8456 struct rtnl_hw_stats64 *p_stats) 8457 { 8458 struct mlxsw_sp_rif_counter_set_basic ingress; 8459 struct mlxsw_sp_rif_counter_set_basic egress; 8460 int err; 8461 8462 err = mlxsw_sp_rif_counter_fetch_clear(rif, 8463 MLXSW_SP_RIF_COUNTER_INGRESS, 8464 &ingress); 8465 if (err) 8466 return err; 8467 8468 err = mlxsw_sp_rif_counter_fetch_clear(rif, 8469 MLXSW_SP_RIF_COUNTER_EGRESS, 8470 &egress); 8471 if (err) 8472 return err; 8473 8474 #define MLXSW_SP_ROUTER_ALL_GOOD(SET, SFX) \ 8475 ((SET.good_unicast_ ## SFX) + \ 8476 (SET.good_multicast_ ## SFX) + \ 8477 (SET.good_broadcast_ ## SFX)) 8478 8479 p_stats->rx_packets = MLXSW_SP_ROUTER_ALL_GOOD(ingress, packets); 8480 p_stats->tx_packets = MLXSW_SP_ROUTER_ALL_GOOD(egress, packets); 8481 p_stats->rx_bytes = MLXSW_SP_ROUTER_ALL_GOOD(ingress, bytes); 8482 p_stats->tx_bytes = MLXSW_SP_ROUTER_ALL_GOOD(egress, bytes); 8483 p_stats->rx_errors = ingress.error_packets; 8484 p_stats->tx_errors = egress.error_packets; 8485 p_stats->rx_dropped = ingress.discard_packets; 8486 p_stats->tx_dropped = egress.discard_packets; 8487 p_stats->multicast = ingress.good_multicast_packets + 8488 ingress.good_broadcast_packets; 8489 8490 #undef MLXSW_SP_ROUTER_ALL_GOOD 8491 8492 return 0; 8493 } 8494 8495 static int 8496 mlxsw_sp_router_port_l3_stats_report_delta(struct mlxsw_sp_rif *rif, 8497 struct netdev_notifier_offload_xstats_info *info) 8498 { 8499 struct rtnl_hw_stats64 stats = {}; 8500 int err; 8501 8502 if (!mlxsw_sp_router_port_l3_stats_enabled(rif)) 8503 return 0; 8504 8505 err = mlxsw_sp_router_port_l3_stats_fetch(rif, &stats); 8506 if (err) 8507 return err; 8508 8509 netdev_offload_xstats_report_delta(info->report_delta, &stats); 8510 return 0; 8511 } 8512 8513 struct mlxsw_sp_router_hwstats_notify_work { 8514 struct work_struct work; 8515 struct net_device *dev; 8516 netdevice_tracker dev_tracker; 8517 }; 8518 8519 static void mlxsw_sp_router_hwstats_notify_work(struct work_struct *work) 8520 { 8521 struct mlxsw_sp_router_hwstats_notify_work *hws_work = 8522 container_of(work, struct mlxsw_sp_router_hwstats_notify_work, 8523 work); 8524 8525 rtnl_lock(); 8526 rtnl_offload_xstats_notify(hws_work->dev); 8527 rtnl_unlock(); 8528 netdev_put(hws_work->dev, &hws_work->dev_tracker); 8529 kfree(hws_work); 8530 } 8531 8532 static void 8533 mlxsw_sp_router_hwstats_notify_schedule(struct net_device *dev) 8534 { 8535 struct mlxsw_sp_router_hwstats_notify_work *hws_work; 8536 8537 /* To collect notification payload, the core ends up sending another 8538 * notifier block message, which would deadlock on the attempt to 8539 * acquire the router lock again. Just postpone the notification until 8540 * later. 8541 */ 8542 8543 hws_work = kzalloc_obj(*hws_work); 8544 if (!hws_work) 8545 return; 8546 8547 INIT_WORK(&hws_work->work, mlxsw_sp_router_hwstats_notify_work); 8548 netdev_hold(dev, &hws_work->dev_tracker, GFP_KERNEL); 8549 hws_work->dev = dev; 8550 mlxsw_core_schedule_work(&hws_work->work); 8551 } 8552 8553 int mlxsw_sp_rif_dev_ifindex(const struct mlxsw_sp_rif *rif) 8554 { 8555 return mlxsw_sp_rif_dev(rif)->ifindex; 8556 } 8557 8558 bool mlxsw_sp_rif_has_dev(const struct mlxsw_sp_rif *rif) 8559 { 8560 return !!mlxsw_sp_rif_dev(rif); 8561 } 8562 8563 bool mlxsw_sp_rif_dev_is(const struct mlxsw_sp_rif *rif, 8564 const struct net_device *dev) 8565 { 8566 return mlxsw_sp_rif_dev(rif) == dev; 8567 } 8568 8569 static void mlxsw_sp_rif_push_l3_stats(struct mlxsw_sp_rif *rif) 8570 { 8571 struct rtnl_hw_stats64 stats = {}; 8572 8573 if (!mlxsw_sp_router_port_l3_stats_fetch(rif, &stats)) 8574 netdev_offload_xstats_push_delta(mlxsw_sp_rif_dev(rif), 8575 NETDEV_OFFLOAD_XSTATS_TYPE_L3, 8576 &stats); 8577 } 8578 8579 static struct mlxsw_sp_rif * 8580 mlxsw_sp_rif_create(struct mlxsw_sp *mlxsw_sp, 8581 const struct mlxsw_sp_rif_params *params, 8582 struct netlink_ext_ack *extack) 8583 { 8584 u8 rif_entries = params->double_entry ? 2 : 1; 8585 u32 tb_id = l3mdev_fib_table(params->dev); 8586 const struct mlxsw_sp_rif_ops *ops; 8587 struct mlxsw_sp_fid *fid = NULL; 8588 enum mlxsw_sp_rif_type type; 8589 struct mlxsw_sp_crif *crif; 8590 struct mlxsw_sp_rif *rif; 8591 struct mlxsw_sp_vr *vr; 8592 u16 rif_index; 8593 int i, err; 8594 8595 type = mlxsw_sp_dev_rif_type(mlxsw_sp, params->dev); 8596 ops = mlxsw_sp->router->rif_ops_arr[type]; 8597 8598 vr = mlxsw_sp_vr_get(mlxsw_sp, tb_id ? : RT_TABLE_MAIN, extack); 8599 if (IS_ERR(vr)) 8600 return ERR_CAST(vr); 8601 vr->rif_count++; 8602 8603 err = mlxsw_sp_rif_index_alloc(mlxsw_sp, &rif_index, rif_entries); 8604 if (err) { 8605 NL_SET_ERR_MSG_MOD(extack, "Exceeded number of supported router interfaces"); 8606 goto err_rif_index_alloc; 8607 } 8608 8609 crif = mlxsw_sp_crif_lookup(mlxsw_sp->router, params->dev); 8610 if (WARN_ON(!crif)) { 8611 err = -ENOENT; 8612 goto err_crif_lookup; 8613 } 8614 8615 rif = mlxsw_sp_rif_alloc(ops->rif_size, rif_index, vr->id, crif); 8616 if (!rif) { 8617 err = -ENOMEM; 8618 goto err_rif_alloc; 8619 } 8620 netdev_hold(params->dev, &rif->dev_tracker, GFP_KERNEL); 8621 mlxsw_sp->router->rifs[rif_index] = rif; 8622 rif->mlxsw_sp = mlxsw_sp; 8623 rif->ops = ops; 8624 rif->rif_entries = rif_entries; 8625 8626 if (ops->setup) 8627 ops->setup(rif, params); 8628 8629 if (ops->fid_get) { 8630 fid = ops->fid_get(rif, params, extack); 8631 if (IS_ERR(fid)) { 8632 err = PTR_ERR(fid); 8633 goto err_fid_get; 8634 } 8635 rif->fid = fid; 8636 } 8637 8638 err = ops->configure(rif, extack); 8639 if (err) 8640 goto err_configure; 8641 8642 for (i = 0; i < MLXSW_SP_L3_PROTO_MAX; i++) { 8643 err = mlxsw_sp_mr_rif_add(vr->mr_table[i], rif); 8644 if (err) 8645 goto err_mr_rif_add; 8646 } 8647 8648 err = mlxsw_sp_router_rif_made_sync(mlxsw_sp, rif); 8649 if (err) 8650 goto err_rif_made_sync; 8651 8652 if (netdev_offload_xstats_enabled(params->dev, 8653 NETDEV_OFFLOAD_XSTATS_TYPE_L3)) { 8654 err = mlxsw_sp_router_port_l3_stats_enable(rif); 8655 if (err) 8656 goto err_stats_enable; 8657 mlxsw_sp_router_hwstats_notify_schedule(params->dev); 8658 } else { 8659 mlxsw_sp_rif_counters_alloc(rif); 8660 } 8661 8662 atomic_add(rif_entries, &mlxsw_sp->router->rifs_count); 8663 return rif; 8664 8665 err_stats_enable: 8666 mlxsw_sp_router_rif_gone_sync(mlxsw_sp, rif); 8667 err_rif_made_sync: 8668 err_mr_rif_add: 8669 for (i--; i >= 0; i--) 8670 mlxsw_sp_mr_rif_del(vr->mr_table[i], rif); 8671 ops->deconfigure(rif); 8672 err_configure: 8673 if (fid) 8674 mlxsw_sp_fid_put(fid); 8675 err_fid_get: 8676 mlxsw_sp->router->rifs[rif_index] = NULL; 8677 netdev_put(params->dev, &rif->dev_tracker); 8678 mlxsw_sp_rif_free(rif); 8679 err_rif_alloc: 8680 err_crif_lookup: 8681 mlxsw_sp_rif_index_free(mlxsw_sp, rif_index, rif_entries); 8682 err_rif_index_alloc: 8683 vr->rif_count--; 8684 mlxsw_sp_vr_put(mlxsw_sp, vr); 8685 return ERR_PTR(err); 8686 } 8687 8688 static void mlxsw_sp_rif_destroy(struct mlxsw_sp_rif *rif) 8689 { 8690 struct net_device *dev = mlxsw_sp_rif_dev(rif); 8691 const struct mlxsw_sp_rif_ops *ops = rif->ops; 8692 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 8693 struct mlxsw_sp_crif *crif = rif->crif; 8694 struct mlxsw_sp_fid *fid = rif->fid; 8695 u8 rif_entries = rif->rif_entries; 8696 u16 rif_index = rif->rif_index; 8697 struct mlxsw_sp_vr *vr; 8698 int i; 8699 8700 atomic_sub(rif_entries, &mlxsw_sp->router->rifs_count); 8701 mlxsw_sp_router_rif_gone_sync(mlxsw_sp, rif); 8702 vr = &mlxsw_sp->router->vrs[rif->vr_id]; 8703 8704 if (netdev_offload_xstats_enabled(dev, NETDEV_OFFLOAD_XSTATS_TYPE_L3)) { 8705 mlxsw_sp_rif_push_l3_stats(rif); 8706 mlxsw_sp_router_port_l3_stats_disable(rif); 8707 mlxsw_sp_router_hwstats_notify_schedule(dev); 8708 } else { 8709 mlxsw_sp_rif_counters_free(rif); 8710 } 8711 8712 for (i = 0; i < MLXSW_SP_L3_PROTO_MAX; i++) 8713 mlxsw_sp_mr_rif_del(vr->mr_table[i], rif); 8714 ops->deconfigure(rif); 8715 if (fid) 8716 /* Loopback RIFs are not associated with a FID. */ 8717 mlxsw_sp_fid_put(fid); 8718 mlxsw_sp->router->rifs[rif->rif_index] = NULL; 8719 netdev_put(dev, &rif->dev_tracker); 8720 mlxsw_sp_rif_free(rif); 8721 mlxsw_sp_rif_index_free(mlxsw_sp, rif_index, rif_entries); 8722 vr->rif_count--; 8723 mlxsw_sp_vr_put(mlxsw_sp, vr); 8724 8725 if (crif->can_destroy) 8726 mlxsw_sp_crif_free(crif); 8727 } 8728 8729 void mlxsw_sp_rif_destroy_by_dev(struct mlxsw_sp *mlxsw_sp, 8730 struct net_device *dev) 8731 { 8732 struct mlxsw_sp_rif *rif; 8733 8734 mutex_lock(&mlxsw_sp->router->lock); 8735 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev); 8736 if (!rif) 8737 goto out; 8738 mlxsw_sp_rif_destroy(rif); 8739 out: 8740 mutex_unlock(&mlxsw_sp->router->lock); 8741 } 8742 8743 static void mlxsw_sp_rif_destroy_vlan_upper(struct mlxsw_sp *mlxsw_sp, 8744 struct net_device *br_dev, 8745 u16 vid) 8746 { 8747 struct net_device *upper_dev; 8748 struct mlxsw_sp_crif *crif; 8749 8750 rcu_read_lock(); 8751 upper_dev = __vlan_find_dev_deep_rcu(br_dev, htons(ETH_P_8021Q), vid); 8752 rcu_read_unlock(); 8753 8754 if (!upper_dev) 8755 return; 8756 8757 crif = mlxsw_sp_crif_lookup(mlxsw_sp->router, upper_dev); 8758 if (!crif || !crif->rif) 8759 return; 8760 8761 mlxsw_sp_rif_destroy(crif->rif); 8762 } 8763 8764 static int mlxsw_sp_inetaddr_bridge_event(struct mlxsw_sp *mlxsw_sp, 8765 struct net_device *l3_dev, 8766 int lower_pvid, 8767 unsigned long event, 8768 struct netlink_ext_ack *extack); 8769 8770 int mlxsw_sp_router_bridge_vlan_add(struct mlxsw_sp *mlxsw_sp, 8771 struct net_device *br_dev, 8772 u16 new_vid, bool is_pvid, 8773 struct netlink_ext_ack *extack) 8774 { 8775 struct mlxsw_sp_rif *old_rif; 8776 struct mlxsw_sp_rif *new_rif; 8777 struct net_device *upper_dev; 8778 u16 old_pvid = 0; 8779 u16 new_pvid; 8780 int err = 0; 8781 8782 mutex_lock(&mlxsw_sp->router->lock); 8783 old_rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, br_dev); 8784 if (old_rif) { 8785 /* If the RIF on the bridge is not a VLAN RIF, we shouldn't have 8786 * gotten a PVID notification. 8787 */ 8788 if (WARN_ON(old_rif->ops->type != MLXSW_SP_RIF_TYPE_VLAN)) 8789 old_rif = NULL; 8790 else 8791 old_pvid = mlxsw_sp_fid_8021q_vid(old_rif->fid); 8792 } 8793 8794 if (is_pvid) 8795 new_pvid = new_vid; 8796 else if (old_pvid == new_vid) 8797 new_pvid = 0; 8798 else 8799 goto out; 8800 8801 if (old_pvid == new_pvid) 8802 goto out; 8803 8804 if (new_pvid) { 8805 struct mlxsw_sp_rif_params params = { 8806 .dev = br_dev, 8807 .vid = new_pvid, 8808 }; 8809 8810 /* If there is a VLAN upper with the same VID as the new PVID, 8811 * kill its RIF, if there is one. 8812 */ 8813 mlxsw_sp_rif_destroy_vlan_upper(mlxsw_sp, br_dev, new_pvid); 8814 8815 if (mlxsw_sp_dev_addr_list_empty(br_dev)) 8816 goto out; 8817 new_rif = mlxsw_sp_rif_create(mlxsw_sp, ¶ms, extack); 8818 if (IS_ERR(new_rif)) { 8819 err = PTR_ERR(new_rif); 8820 goto out; 8821 } 8822 8823 if (old_pvid) 8824 mlxsw_sp_rif_migrate_destroy(mlxsw_sp, old_rif, new_rif, 8825 true); 8826 } else { 8827 mlxsw_sp_rif_destroy(old_rif); 8828 } 8829 8830 if (old_pvid) { 8831 rcu_read_lock(); 8832 upper_dev = __vlan_find_dev_deep_rcu(br_dev, htons(ETH_P_8021Q), 8833 old_pvid); 8834 rcu_read_unlock(); 8835 if (upper_dev) 8836 err = mlxsw_sp_inetaddr_bridge_event(mlxsw_sp, 8837 upper_dev, 8838 new_pvid, 8839 NETDEV_UP, extack); 8840 } 8841 8842 out: 8843 mutex_unlock(&mlxsw_sp->router->lock); 8844 return err; 8845 } 8846 8847 static void 8848 mlxsw_sp_rif_subport_params_init(struct mlxsw_sp_rif_params *params, 8849 struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan) 8850 { 8851 struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port; 8852 8853 params->vid = mlxsw_sp_port_vlan->vid; 8854 params->lag = mlxsw_sp_port->lagged; 8855 if (params->lag) 8856 params->lag_id = mlxsw_sp_port->lag_id; 8857 else 8858 params->system_port = mlxsw_sp_port->local_port; 8859 } 8860 8861 static struct mlxsw_sp_rif_subport * 8862 mlxsw_sp_rif_subport_rif(const struct mlxsw_sp_rif *rif) 8863 { 8864 return container_of(rif, struct mlxsw_sp_rif_subport, common); 8865 } 8866 8867 int mlxsw_sp_rif_subport_port(const struct mlxsw_sp_rif *rif, 8868 u16 *port, bool *is_lag) 8869 { 8870 struct mlxsw_sp_rif_subport *rif_subport; 8871 8872 if (WARN_ON(rif->ops->type != MLXSW_SP_RIF_TYPE_SUBPORT)) 8873 return -EINVAL; 8874 8875 rif_subport = mlxsw_sp_rif_subport_rif(rif); 8876 *is_lag = rif_subport->lag; 8877 *port = *is_lag ? rif_subport->lag_id : rif_subport->system_port; 8878 return 0; 8879 } 8880 8881 static struct mlxsw_sp_rif * 8882 mlxsw_sp_rif_subport_get(struct mlxsw_sp *mlxsw_sp, 8883 const struct mlxsw_sp_rif_params *params, 8884 struct netlink_ext_ack *extack) 8885 { 8886 struct mlxsw_sp_rif_subport *rif_subport; 8887 struct mlxsw_sp_rif *rif; 8888 8889 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, params->dev); 8890 if (!rif) 8891 return mlxsw_sp_rif_create(mlxsw_sp, params, extack); 8892 8893 rif_subport = mlxsw_sp_rif_subport_rif(rif); 8894 refcount_inc(&rif_subport->ref_count); 8895 return rif; 8896 } 8897 8898 static void mlxsw_sp_rif_subport_put(struct mlxsw_sp_rif *rif) 8899 { 8900 struct mlxsw_sp_rif_subport *rif_subport; 8901 8902 rif_subport = mlxsw_sp_rif_subport_rif(rif); 8903 if (!refcount_dec_and_test(&rif_subport->ref_count)) 8904 return; 8905 8906 mlxsw_sp_rif_destroy(rif); 8907 } 8908 8909 static int mlxsw_sp_rif_mac_profile_index_alloc(struct mlxsw_sp *mlxsw_sp, 8910 struct mlxsw_sp_rif_mac_profile *profile, 8911 struct netlink_ext_ack *extack) 8912 { 8913 u8 max_rif_mac_profiles = mlxsw_sp->router->max_rif_mac_profile; 8914 struct mlxsw_sp_router *router = mlxsw_sp->router; 8915 int id; 8916 8917 id = idr_alloc(&router->rif_mac_profiles_idr, profile, 0, 8918 max_rif_mac_profiles, GFP_KERNEL); 8919 8920 if (id >= 0) { 8921 profile->id = id; 8922 return 0; 8923 } 8924 8925 if (id == -ENOSPC) 8926 NL_SET_ERR_MSG_MOD(extack, 8927 "Exceeded number of supported router interface MAC profiles"); 8928 8929 return id; 8930 } 8931 8932 static struct mlxsw_sp_rif_mac_profile * 8933 mlxsw_sp_rif_mac_profile_index_free(struct mlxsw_sp *mlxsw_sp, u8 mac_profile) 8934 { 8935 struct mlxsw_sp_rif_mac_profile *profile; 8936 8937 profile = idr_remove(&mlxsw_sp->router->rif_mac_profiles_idr, 8938 mac_profile); 8939 WARN_ON(!profile); 8940 return profile; 8941 } 8942 8943 static struct mlxsw_sp_rif_mac_profile * 8944 mlxsw_sp_rif_mac_profile_alloc(const char *mac) 8945 { 8946 struct mlxsw_sp_rif_mac_profile *profile; 8947 8948 profile = kzalloc_obj(*profile); 8949 if (!profile) 8950 return NULL; 8951 8952 ether_addr_copy(profile->mac_prefix, mac); 8953 refcount_set(&profile->ref_count, 1); 8954 return profile; 8955 } 8956 8957 static struct mlxsw_sp_rif_mac_profile * 8958 mlxsw_sp_rif_mac_profile_find(const struct mlxsw_sp *mlxsw_sp, const char *mac) 8959 { 8960 struct mlxsw_sp_router *router = mlxsw_sp->router; 8961 struct mlxsw_sp_rif_mac_profile *profile; 8962 int id; 8963 8964 idr_for_each_entry(&router->rif_mac_profiles_idr, profile, id) { 8965 if (ether_addr_equal_masked(profile->mac_prefix, mac, 8966 mlxsw_sp->mac_mask)) 8967 return profile; 8968 } 8969 8970 return NULL; 8971 } 8972 8973 static u64 mlxsw_sp_rif_mac_profiles_occ_get(void *priv) 8974 { 8975 const struct mlxsw_sp *mlxsw_sp = priv; 8976 8977 return atomic_read(&mlxsw_sp->router->rif_mac_profiles_count); 8978 } 8979 8980 static u64 mlxsw_sp_rifs_occ_get(void *priv) 8981 { 8982 const struct mlxsw_sp *mlxsw_sp = priv; 8983 8984 return atomic_read(&mlxsw_sp->router->rifs_count); 8985 } 8986 8987 static struct mlxsw_sp_rif_mac_profile * 8988 mlxsw_sp_rif_mac_profile_create(struct mlxsw_sp *mlxsw_sp, const char *mac, 8989 struct netlink_ext_ack *extack) 8990 { 8991 struct mlxsw_sp_rif_mac_profile *profile; 8992 int err; 8993 8994 profile = mlxsw_sp_rif_mac_profile_alloc(mac); 8995 if (!profile) 8996 return ERR_PTR(-ENOMEM); 8997 8998 err = mlxsw_sp_rif_mac_profile_index_alloc(mlxsw_sp, profile, extack); 8999 if (err) 9000 goto profile_index_alloc_err; 9001 9002 atomic_inc(&mlxsw_sp->router->rif_mac_profiles_count); 9003 return profile; 9004 9005 profile_index_alloc_err: 9006 kfree(profile); 9007 return ERR_PTR(err); 9008 } 9009 9010 static void mlxsw_sp_rif_mac_profile_destroy(struct mlxsw_sp *mlxsw_sp, 9011 u8 mac_profile) 9012 { 9013 struct mlxsw_sp_rif_mac_profile *profile; 9014 9015 atomic_dec(&mlxsw_sp->router->rif_mac_profiles_count); 9016 profile = mlxsw_sp_rif_mac_profile_index_free(mlxsw_sp, mac_profile); 9017 kfree(profile); 9018 } 9019 9020 static int mlxsw_sp_rif_mac_profile_get(struct mlxsw_sp *mlxsw_sp, 9021 const char *mac, u8 *p_mac_profile, 9022 struct netlink_ext_ack *extack) 9023 { 9024 struct mlxsw_sp_rif_mac_profile *profile; 9025 9026 profile = mlxsw_sp_rif_mac_profile_find(mlxsw_sp, mac); 9027 if (profile) { 9028 refcount_inc(&profile->ref_count); 9029 goto out; 9030 } 9031 9032 profile = mlxsw_sp_rif_mac_profile_create(mlxsw_sp, mac, extack); 9033 if (IS_ERR(profile)) 9034 return PTR_ERR(profile); 9035 9036 out: 9037 *p_mac_profile = profile->id; 9038 return 0; 9039 } 9040 9041 static void mlxsw_sp_rif_mac_profile_put(struct mlxsw_sp *mlxsw_sp, 9042 u8 mac_profile) 9043 { 9044 struct mlxsw_sp_rif_mac_profile *profile; 9045 9046 profile = idr_find(&mlxsw_sp->router->rif_mac_profiles_idr, 9047 mac_profile); 9048 if (WARN_ON(!profile)) 9049 return; 9050 9051 if (!refcount_dec_and_test(&profile->ref_count)) 9052 return; 9053 9054 mlxsw_sp_rif_mac_profile_destroy(mlxsw_sp, mac_profile); 9055 } 9056 9057 static bool mlxsw_sp_rif_mac_profile_is_shared(const struct mlxsw_sp_rif *rif) 9058 { 9059 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 9060 struct mlxsw_sp_rif_mac_profile *profile; 9061 9062 profile = idr_find(&mlxsw_sp->router->rif_mac_profiles_idr, 9063 rif->mac_profile_id); 9064 if (WARN_ON(!profile)) 9065 return false; 9066 9067 return refcount_read(&profile->ref_count) > 1; 9068 } 9069 9070 static int mlxsw_sp_rif_mac_profile_edit(struct mlxsw_sp_rif *rif, 9071 const char *new_mac) 9072 { 9073 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 9074 struct mlxsw_sp_rif_mac_profile *profile; 9075 9076 profile = idr_find(&mlxsw_sp->router->rif_mac_profiles_idr, 9077 rif->mac_profile_id); 9078 if (WARN_ON(!profile)) 9079 return -EINVAL; 9080 9081 ether_addr_copy(profile->mac_prefix, new_mac); 9082 return 0; 9083 } 9084 9085 static int 9086 mlxsw_sp_rif_mac_profile_replace(struct mlxsw_sp *mlxsw_sp, 9087 struct mlxsw_sp_rif *rif, 9088 const char *new_mac, 9089 struct netlink_ext_ack *extack) 9090 { 9091 u8 mac_profile; 9092 int err; 9093 9094 if (!mlxsw_sp_rif_mac_profile_is_shared(rif) && 9095 !mlxsw_sp_rif_mac_profile_find(mlxsw_sp, new_mac)) 9096 return mlxsw_sp_rif_mac_profile_edit(rif, new_mac); 9097 9098 err = mlxsw_sp_rif_mac_profile_get(mlxsw_sp, new_mac, 9099 &mac_profile, extack); 9100 if (err) 9101 return err; 9102 9103 mlxsw_sp_rif_mac_profile_put(mlxsw_sp, rif->mac_profile_id); 9104 rif->mac_profile_id = mac_profile; 9105 return 0; 9106 } 9107 9108 static int 9109 __mlxsw_sp_port_vlan_router_join(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan, 9110 struct net_device *l3_dev, 9111 struct netlink_ext_ack *extack) 9112 { 9113 struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port; 9114 struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp; 9115 struct mlxsw_sp_rif_params params; 9116 u16 vid = mlxsw_sp_port_vlan->vid; 9117 struct mlxsw_sp_rif *rif; 9118 struct mlxsw_sp_fid *fid; 9119 int err; 9120 9121 params = (struct mlxsw_sp_rif_params) { 9122 .dev = l3_dev, 9123 .vid = vid, 9124 }; 9125 9126 mlxsw_sp_rif_subport_params_init(¶ms, mlxsw_sp_port_vlan); 9127 rif = mlxsw_sp_rif_subport_get(mlxsw_sp, ¶ms, extack); 9128 if (IS_ERR(rif)) 9129 return PTR_ERR(rif); 9130 9131 /* FID was already created, just take a reference */ 9132 fid = rif->ops->fid_get(rif, ¶ms, extack); 9133 err = mlxsw_sp_fid_port_vid_map(fid, mlxsw_sp_port, vid); 9134 if (err) 9135 goto err_fid_port_vid_map; 9136 9137 err = mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, false); 9138 if (err) 9139 goto err_port_vid_learning_set; 9140 9141 err = mlxsw_sp_port_vid_stp_set(mlxsw_sp_port, vid, 9142 BR_STATE_FORWARDING); 9143 if (err) 9144 goto err_port_vid_stp_set; 9145 9146 mlxsw_sp_port_vlan->fid = fid; 9147 9148 return 0; 9149 9150 err_port_vid_stp_set: 9151 mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, true); 9152 err_port_vid_learning_set: 9153 mlxsw_sp_fid_port_vid_unmap(fid, mlxsw_sp_port, vid); 9154 err_fid_port_vid_map: 9155 mlxsw_sp_fid_put(fid); 9156 mlxsw_sp_rif_subport_put(rif); 9157 return err; 9158 } 9159 9160 static void 9161 __mlxsw_sp_port_vlan_router_leave(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan) 9162 { 9163 struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port; 9164 struct mlxsw_sp_fid *fid = mlxsw_sp_port_vlan->fid; 9165 struct mlxsw_sp_rif *rif = mlxsw_sp_fid_rif(fid); 9166 u16 vid = mlxsw_sp_port_vlan->vid; 9167 9168 if (WARN_ON(mlxsw_sp_fid_type(fid) != MLXSW_SP_FID_TYPE_RFID)) 9169 return; 9170 9171 mlxsw_sp_port_vlan->fid = NULL; 9172 mlxsw_sp_port_vid_stp_set(mlxsw_sp_port, vid, BR_STATE_BLOCKING); 9173 mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, true); 9174 mlxsw_sp_fid_port_vid_unmap(fid, mlxsw_sp_port, vid); 9175 mlxsw_sp_fid_put(fid); 9176 mlxsw_sp_rif_subport_put(rif); 9177 } 9178 9179 static int 9180 mlxsw_sp_port_vlan_router_join_existing(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan, 9181 struct net_device *l3_dev, 9182 struct netlink_ext_ack *extack) 9183 { 9184 struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port_vlan->mlxsw_sp_port->mlxsw_sp; 9185 9186 lockdep_assert_held(&mlxsw_sp->router->lock); 9187 9188 if (!mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev)) 9189 return 0; 9190 9191 return __mlxsw_sp_port_vlan_router_join(mlxsw_sp_port_vlan, l3_dev, 9192 extack); 9193 } 9194 9195 void 9196 mlxsw_sp_port_vlan_router_leave(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan) 9197 { 9198 struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port_vlan->mlxsw_sp_port->mlxsw_sp; 9199 9200 mutex_lock(&mlxsw_sp->router->lock); 9201 __mlxsw_sp_port_vlan_router_leave(mlxsw_sp_port_vlan); 9202 mutex_unlock(&mlxsw_sp->router->lock); 9203 } 9204 9205 static int mlxsw_sp_inetaddr_port_vlan_event(struct net_device *l3_dev, 9206 struct net_device *port_dev, 9207 unsigned long event, u16 vid, 9208 struct netlink_ext_ack *extack) 9209 { 9210 struct mlxsw_sp_port *mlxsw_sp_port = netdev_priv(port_dev); 9211 struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan; 9212 9213 mlxsw_sp_port_vlan = mlxsw_sp_port_vlan_find_by_vid(mlxsw_sp_port, vid); 9214 if (WARN_ON(!mlxsw_sp_port_vlan)) 9215 return -EINVAL; 9216 9217 switch (event) { 9218 case NETDEV_UP: 9219 return __mlxsw_sp_port_vlan_router_join(mlxsw_sp_port_vlan, 9220 l3_dev, extack); 9221 case NETDEV_DOWN: 9222 __mlxsw_sp_port_vlan_router_leave(mlxsw_sp_port_vlan); 9223 break; 9224 } 9225 9226 return 0; 9227 } 9228 9229 static int mlxsw_sp_inetaddr_port_event(struct net_device *port_dev, 9230 unsigned long event, bool nomaster, 9231 struct netlink_ext_ack *extack) 9232 { 9233 if (!nomaster && (netif_is_any_bridge_port(port_dev) || 9234 netif_is_lag_port(port_dev))) 9235 return 0; 9236 9237 return mlxsw_sp_inetaddr_port_vlan_event(port_dev, port_dev, event, 9238 MLXSW_SP_DEFAULT_VID, extack); 9239 } 9240 9241 static int __mlxsw_sp_inetaddr_lag_event(struct net_device *l3_dev, 9242 struct net_device *lag_dev, 9243 unsigned long event, u16 vid, 9244 struct netlink_ext_ack *extack) 9245 { 9246 struct net_device *port_dev; 9247 struct list_head *iter; 9248 int err; 9249 9250 netdev_for_each_lower_dev(lag_dev, port_dev, iter) { 9251 if (mlxsw_sp_port_dev_check(port_dev)) { 9252 err = mlxsw_sp_inetaddr_port_vlan_event(l3_dev, 9253 port_dev, 9254 event, vid, 9255 extack); 9256 if (err) 9257 return err; 9258 } 9259 } 9260 9261 return 0; 9262 } 9263 9264 static int mlxsw_sp_inetaddr_lag_event(struct net_device *lag_dev, 9265 unsigned long event, bool nomaster, 9266 struct netlink_ext_ack *extack) 9267 { 9268 if (!nomaster && netif_is_bridge_port(lag_dev)) 9269 return 0; 9270 9271 return __mlxsw_sp_inetaddr_lag_event(lag_dev, lag_dev, event, 9272 MLXSW_SP_DEFAULT_VID, extack); 9273 } 9274 9275 static int mlxsw_sp_inetaddr_bridge_event(struct mlxsw_sp *mlxsw_sp, 9276 struct net_device *l3_dev, 9277 int lower_pvid, 9278 unsigned long event, 9279 struct netlink_ext_ack *extack) 9280 { 9281 struct mlxsw_sp_rif_params params = { 9282 .dev = l3_dev, 9283 }; 9284 struct mlxsw_sp_rif *rif; 9285 int err; 9286 9287 switch (event) { 9288 case NETDEV_UP: 9289 if (netif_is_bridge_master(l3_dev) && br_vlan_enabled(l3_dev)) { 9290 u16 proto; 9291 9292 br_vlan_get_proto(l3_dev, &proto); 9293 if (proto == ETH_P_8021AD) { 9294 NL_SET_ERR_MSG_MOD(extack, "Adding an IP address to 802.1ad bridge is not supported"); 9295 return -EOPNOTSUPP; 9296 } 9297 err = br_vlan_get_pvid(l3_dev, ¶ms.vid); 9298 if (err) 9299 return err; 9300 if (!params.vid) 9301 return 0; 9302 } else if (is_vlan_dev(l3_dev)) { 9303 params.vid = vlan_dev_vlan_id(l3_dev); 9304 9305 /* If the VID matches PVID of the bridge below, the 9306 * bridge owns the RIF for this VLAN. Don't do anything. 9307 */ 9308 if ((int)params.vid == lower_pvid) 9309 return 0; 9310 } 9311 9312 rif = mlxsw_sp_rif_create(mlxsw_sp, ¶ms, extack); 9313 if (IS_ERR(rif)) 9314 return PTR_ERR(rif); 9315 break; 9316 case NETDEV_DOWN: 9317 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev); 9318 mlxsw_sp_rif_destroy(rif); 9319 break; 9320 } 9321 9322 return 0; 9323 } 9324 9325 static int mlxsw_sp_inetaddr_vlan_event(struct mlxsw_sp *mlxsw_sp, 9326 struct net_device *vlan_dev, 9327 unsigned long event, bool nomaster, 9328 struct netlink_ext_ack *extack) 9329 { 9330 struct net_device *real_dev = vlan_dev_real_dev(vlan_dev); 9331 u16 vid = vlan_dev_vlan_id(vlan_dev); 9332 u16 lower_pvid; 9333 int err; 9334 9335 if (!nomaster && netif_is_bridge_port(vlan_dev)) 9336 return 0; 9337 9338 if (mlxsw_sp_port_dev_check(real_dev)) { 9339 return mlxsw_sp_inetaddr_port_vlan_event(vlan_dev, real_dev, 9340 event, vid, extack); 9341 } else if (netif_is_lag_master(real_dev)) { 9342 return __mlxsw_sp_inetaddr_lag_event(vlan_dev, real_dev, event, 9343 vid, extack); 9344 } else if (netif_is_bridge_master(real_dev) && 9345 br_vlan_enabled(real_dev)) { 9346 err = br_vlan_get_pvid(real_dev, &lower_pvid); 9347 if (err) 9348 return err; 9349 return mlxsw_sp_inetaddr_bridge_event(mlxsw_sp, vlan_dev, 9350 lower_pvid, event, 9351 extack); 9352 } 9353 9354 return 0; 9355 } 9356 9357 static bool mlxsw_sp_rif_macvlan_is_vrrp4(const u8 *mac) 9358 { 9359 u8 vrrp4[ETH_ALEN] = { 0x00, 0x00, 0x5e, 0x00, 0x01, 0x00 }; 9360 u8 mask[ETH_ALEN] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0x00 }; 9361 9362 return ether_addr_equal_masked(mac, vrrp4, mask); 9363 } 9364 9365 static bool mlxsw_sp_rif_macvlan_is_vrrp6(const u8 *mac) 9366 { 9367 u8 vrrp6[ETH_ALEN] = { 0x00, 0x00, 0x5e, 0x00, 0x02, 0x00 }; 9368 u8 mask[ETH_ALEN] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0x00 }; 9369 9370 return ether_addr_equal_masked(mac, vrrp6, mask); 9371 } 9372 9373 static int mlxsw_sp_rif_vrrp_op(struct mlxsw_sp *mlxsw_sp, u16 rif_index, 9374 const u8 *mac, bool adding) 9375 { 9376 char ritr_pl[MLXSW_REG_RITR_LEN]; 9377 u8 vrrp_id = adding ? mac[5] : 0; 9378 int err; 9379 9380 if (!mlxsw_sp_rif_macvlan_is_vrrp4(mac) && 9381 !mlxsw_sp_rif_macvlan_is_vrrp6(mac)) 9382 return 0; 9383 9384 mlxsw_reg_ritr_rif_pack(ritr_pl, rif_index); 9385 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 9386 if (err) 9387 return err; 9388 9389 if (mlxsw_sp_rif_macvlan_is_vrrp4(mac)) 9390 mlxsw_reg_ritr_if_vrrp_id_ipv4_set(ritr_pl, vrrp_id); 9391 else 9392 mlxsw_reg_ritr_if_vrrp_id_ipv6_set(ritr_pl, vrrp_id); 9393 9394 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 9395 } 9396 9397 static int mlxsw_sp_rif_macvlan_add(struct mlxsw_sp *mlxsw_sp, 9398 const struct net_device *macvlan_dev, 9399 struct netlink_ext_ack *extack) 9400 { 9401 struct macvlan_dev *vlan = netdev_priv(macvlan_dev); 9402 struct mlxsw_sp_rif *rif; 9403 int err; 9404 9405 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, vlan->lowerdev); 9406 if (!rif) 9407 return 0; 9408 9409 err = mlxsw_sp_rif_fdb_op(mlxsw_sp, macvlan_dev->dev_addr, 9410 mlxsw_sp_fid_index(rif->fid), true); 9411 if (err) 9412 return err; 9413 9414 err = mlxsw_sp_rif_vrrp_op(mlxsw_sp, rif->rif_index, 9415 macvlan_dev->dev_addr, true); 9416 if (err) 9417 goto err_rif_vrrp_add; 9418 9419 /* Make sure the bridge driver does not have this MAC pointing at 9420 * some other port. 9421 */ 9422 if (rif->ops->fdb_del) 9423 rif->ops->fdb_del(rif, macvlan_dev->dev_addr); 9424 9425 return 0; 9426 9427 err_rif_vrrp_add: 9428 mlxsw_sp_rif_fdb_op(mlxsw_sp, macvlan_dev->dev_addr, 9429 mlxsw_sp_fid_index(rif->fid), false); 9430 return err; 9431 } 9432 9433 static void __mlxsw_sp_rif_macvlan_del(struct mlxsw_sp *mlxsw_sp, 9434 const struct net_device *macvlan_dev) 9435 { 9436 struct macvlan_dev *vlan = netdev_priv(macvlan_dev); 9437 struct mlxsw_sp_rif *rif; 9438 9439 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, vlan->lowerdev); 9440 /* If we do not have a RIF, then we already took care of 9441 * removing the macvlan's MAC during RIF deletion. 9442 */ 9443 if (!rif) 9444 return; 9445 mlxsw_sp_rif_vrrp_op(mlxsw_sp, rif->rif_index, macvlan_dev->dev_addr, 9446 false); 9447 mlxsw_sp_rif_fdb_op(mlxsw_sp, macvlan_dev->dev_addr, 9448 mlxsw_sp_fid_index(rif->fid), false); 9449 } 9450 9451 void mlxsw_sp_rif_macvlan_del(struct mlxsw_sp *mlxsw_sp, 9452 const struct net_device *macvlan_dev) 9453 { 9454 mutex_lock(&mlxsw_sp->router->lock); 9455 __mlxsw_sp_rif_macvlan_del(mlxsw_sp, macvlan_dev); 9456 mutex_unlock(&mlxsw_sp->router->lock); 9457 } 9458 9459 static int mlxsw_sp_inetaddr_macvlan_event(struct mlxsw_sp *mlxsw_sp, 9460 struct net_device *macvlan_dev, 9461 unsigned long event, 9462 struct netlink_ext_ack *extack) 9463 { 9464 switch (event) { 9465 case NETDEV_UP: 9466 return mlxsw_sp_rif_macvlan_add(mlxsw_sp, macvlan_dev, extack); 9467 case NETDEV_DOWN: 9468 __mlxsw_sp_rif_macvlan_del(mlxsw_sp, macvlan_dev); 9469 break; 9470 } 9471 9472 return 0; 9473 } 9474 9475 static int __mlxsw_sp_inetaddr_event(struct mlxsw_sp *mlxsw_sp, 9476 struct net_device *dev, 9477 unsigned long event, bool nomaster, 9478 struct netlink_ext_ack *extack) 9479 { 9480 if (mlxsw_sp_port_dev_check(dev)) 9481 return mlxsw_sp_inetaddr_port_event(dev, event, nomaster, 9482 extack); 9483 else if (netif_is_lag_master(dev)) 9484 return mlxsw_sp_inetaddr_lag_event(dev, event, nomaster, 9485 extack); 9486 else if (netif_is_bridge_master(dev)) 9487 return mlxsw_sp_inetaddr_bridge_event(mlxsw_sp, dev, -1, event, 9488 extack); 9489 else if (is_vlan_dev(dev)) 9490 return mlxsw_sp_inetaddr_vlan_event(mlxsw_sp, dev, event, 9491 nomaster, extack); 9492 else if (netif_is_macvlan(dev)) 9493 return mlxsw_sp_inetaddr_macvlan_event(mlxsw_sp, dev, event, 9494 extack); 9495 else 9496 return 0; 9497 } 9498 9499 static int mlxsw_sp_inetaddr_event(struct notifier_block *nb, 9500 unsigned long event, void *ptr) 9501 { 9502 struct in_ifaddr *ifa = (struct in_ifaddr *) ptr; 9503 struct net_device *dev = ifa->ifa_dev->dev; 9504 struct mlxsw_sp_router *router; 9505 struct mlxsw_sp_rif *rif; 9506 int err = 0; 9507 9508 /* NETDEV_UP event is handled by mlxsw_sp_inetaddr_valid_event */ 9509 if (event == NETDEV_UP) 9510 return NOTIFY_DONE; 9511 9512 router = container_of(nb, struct mlxsw_sp_router, inetaddr_nb); 9513 mutex_lock(&router->lock); 9514 rif = mlxsw_sp_rif_find_by_dev(router->mlxsw_sp, dev); 9515 if (!mlxsw_sp_rif_should_config(rif, dev, event)) 9516 goto out; 9517 9518 err = __mlxsw_sp_inetaddr_event(router->mlxsw_sp, dev, event, false, 9519 NULL); 9520 out: 9521 mutex_unlock(&router->lock); 9522 return notifier_from_errno(err); 9523 } 9524 9525 static int mlxsw_sp_inetaddr_valid_event(struct notifier_block *unused, 9526 unsigned long event, void *ptr) 9527 { 9528 struct in_validator_info *ivi = (struct in_validator_info *) ptr; 9529 struct net_device *dev = ivi->ivi_dev->dev; 9530 struct mlxsw_sp *mlxsw_sp; 9531 struct mlxsw_sp_rif *rif; 9532 int err = 0; 9533 9534 mlxsw_sp = mlxsw_sp_lower_get(dev); 9535 if (!mlxsw_sp) 9536 return NOTIFY_DONE; 9537 9538 mutex_lock(&mlxsw_sp->router->lock); 9539 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev); 9540 if (!mlxsw_sp_rif_should_config(rif, dev, event)) 9541 goto out; 9542 9543 err = __mlxsw_sp_inetaddr_event(mlxsw_sp, dev, event, false, 9544 ivi->extack); 9545 out: 9546 mutex_unlock(&mlxsw_sp->router->lock); 9547 return notifier_from_errno(err); 9548 } 9549 9550 struct mlxsw_sp_inet6addr_event_work { 9551 struct work_struct work; 9552 struct mlxsw_sp *mlxsw_sp; 9553 struct net_device *dev; 9554 netdevice_tracker dev_tracker; 9555 unsigned long event; 9556 }; 9557 9558 static void mlxsw_sp_inet6addr_event_work(struct work_struct *work) 9559 { 9560 struct mlxsw_sp_inet6addr_event_work *inet6addr_work = 9561 container_of(work, struct mlxsw_sp_inet6addr_event_work, work); 9562 struct mlxsw_sp *mlxsw_sp = inet6addr_work->mlxsw_sp; 9563 struct net_device *dev = inet6addr_work->dev; 9564 unsigned long event = inet6addr_work->event; 9565 struct mlxsw_sp_rif *rif; 9566 9567 rtnl_lock(); 9568 mutex_lock(&mlxsw_sp->router->lock); 9569 9570 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev); 9571 if (!mlxsw_sp_rif_should_config(rif, dev, event)) 9572 goto out; 9573 9574 __mlxsw_sp_inetaddr_event(mlxsw_sp, dev, event, false, NULL); 9575 out: 9576 mutex_unlock(&mlxsw_sp->router->lock); 9577 rtnl_unlock(); 9578 netdev_put(dev, &inet6addr_work->dev_tracker); 9579 kfree(inet6addr_work); 9580 } 9581 9582 /* Called with rcu_read_lock() */ 9583 static int mlxsw_sp_inet6addr_event(struct notifier_block *nb, 9584 unsigned long event, void *ptr) 9585 { 9586 struct inet6_ifaddr *if6 = (struct inet6_ifaddr *) ptr; 9587 struct mlxsw_sp_inet6addr_event_work *inet6addr_work; 9588 struct net_device *dev = if6->idev->dev; 9589 struct mlxsw_sp_router *router; 9590 9591 /* NETDEV_UP event is handled by mlxsw_sp_inet6addr_valid_event */ 9592 if (event == NETDEV_UP) 9593 return NOTIFY_DONE; 9594 9595 inet6addr_work = kzalloc_obj(*inet6addr_work, GFP_ATOMIC); 9596 if (!inet6addr_work) 9597 return NOTIFY_BAD; 9598 9599 router = container_of(nb, struct mlxsw_sp_router, inet6addr_nb); 9600 INIT_WORK(&inet6addr_work->work, mlxsw_sp_inet6addr_event_work); 9601 inet6addr_work->mlxsw_sp = router->mlxsw_sp; 9602 inet6addr_work->dev = dev; 9603 inet6addr_work->event = event; 9604 netdev_hold(dev, &inet6addr_work->dev_tracker, GFP_ATOMIC); 9605 mlxsw_core_schedule_work(&inet6addr_work->work); 9606 9607 return NOTIFY_DONE; 9608 } 9609 9610 static int mlxsw_sp_inet6addr_valid_event(struct notifier_block *unused, 9611 unsigned long event, void *ptr) 9612 { 9613 struct in6_validator_info *i6vi = (struct in6_validator_info *) ptr; 9614 struct net_device *dev = i6vi->i6vi_dev->dev; 9615 struct mlxsw_sp *mlxsw_sp; 9616 struct mlxsw_sp_rif *rif; 9617 int err = 0; 9618 9619 mlxsw_sp = mlxsw_sp_lower_get(dev); 9620 if (!mlxsw_sp) 9621 return NOTIFY_DONE; 9622 9623 mutex_lock(&mlxsw_sp->router->lock); 9624 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev); 9625 if (!mlxsw_sp_rif_should_config(rif, dev, event)) 9626 goto out; 9627 9628 err = __mlxsw_sp_inetaddr_event(mlxsw_sp, dev, event, false, 9629 i6vi->extack); 9630 out: 9631 mutex_unlock(&mlxsw_sp->router->lock); 9632 return notifier_from_errno(err); 9633 } 9634 9635 static int mlxsw_sp_rif_edit(struct mlxsw_sp *mlxsw_sp, u16 rif_index, 9636 const char *mac, int mtu, u8 mac_profile) 9637 { 9638 char ritr_pl[MLXSW_REG_RITR_LEN]; 9639 int err; 9640 9641 mlxsw_reg_ritr_rif_pack(ritr_pl, rif_index); 9642 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 9643 if (err) 9644 return err; 9645 9646 mlxsw_reg_ritr_mtu_set(ritr_pl, mtu); 9647 mlxsw_reg_ritr_if_mac_memcpy_to(ritr_pl, mac); 9648 mlxsw_reg_ritr_if_mac_profile_id_set(ritr_pl, mac_profile); 9649 mlxsw_reg_ritr_op_set(ritr_pl, MLXSW_REG_RITR_RIF_CREATE); 9650 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 9651 } 9652 9653 static int 9654 mlxsw_sp_router_port_change_event(struct mlxsw_sp *mlxsw_sp, 9655 struct mlxsw_sp_rif *rif, 9656 struct netlink_ext_ack *extack) 9657 { 9658 struct net_device *dev = mlxsw_sp_rif_dev(rif); 9659 u8 old_mac_profile; 9660 u16 fid_index; 9661 int err; 9662 9663 fid_index = mlxsw_sp_fid_index(rif->fid); 9664 9665 err = mlxsw_sp_rif_fdb_op(mlxsw_sp, rif->addr, fid_index, false); 9666 if (err) 9667 return err; 9668 9669 old_mac_profile = rif->mac_profile_id; 9670 err = mlxsw_sp_rif_mac_profile_replace(mlxsw_sp, rif, dev->dev_addr, 9671 extack); 9672 if (err) 9673 goto err_rif_mac_profile_replace; 9674 9675 err = mlxsw_sp_rif_edit(mlxsw_sp, rif->rif_index, dev->dev_addr, 9676 dev->mtu, rif->mac_profile_id); 9677 if (err) 9678 goto err_rif_edit; 9679 9680 err = mlxsw_sp_rif_fdb_op(mlxsw_sp, dev->dev_addr, fid_index, true); 9681 if (err) 9682 goto err_rif_fdb_op; 9683 9684 if (rif->mtu != dev->mtu) { 9685 struct mlxsw_sp_vr *vr; 9686 int i; 9687 9688 /* The RIF is relevant only to its mr_table instance, as unlike 9689 * unicast routing, in multicast routing a RIF cannot be shared 9690 * between several multicast routing tables. 9691 */ 9692 vr = &mlxsw_sp->router->vrs[rif->vr_id]; 9693 for (i = 0; i < MLXSW_SP_L3_PROTO_MAX; i++) 9694 mlxsw_sp_mr_rif_mtu_update(vr->mr_table[i], 9695 rif, dev->mtu); 9696 } 9697 9698 ether_addr_copy(rif->addr, dev->dev_addr); 9699 rif->mtu = dev->mtu; 9700 9701 netdev_dbg(dev, "Updated RIF=%d\n", rif->rif_index); 9702 9703 return 0; 9704 9705 err_rif_fdb_op: 9706 mlxsw_sp_rif_edit(mlxsw_sp, rif->rif_index, rif->addr, rif->mtu, 9707 old_mac_profile); 9708 err_rif_edit: 9709 mlxsw_sp_rif_mac_profile_replace(mlxsw_sp, rif, rif->addr, extack); 9710 err_rif_mac_profile_replace: 9711 mlxsw_sp_rif_fdb_op(mlxsw_sp, rif->addr, fid_index, true); 9712 return err; 9713 } 9714 9715 static int mlxsw_sp_router_port_pre_changeaddr_event(struct mlxsw_sp_rif *rif, 9716 struct netdev_notifier_pre_changeaddr_info *info) 9717 { 9718 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 9719 struct mlxsw_sp_rif_mac_profile *profile; 9720 struct netlink_ext_ack *extack; 9721 u8 max_rif_mac_profiles; 9722 u64 occ; 9723 9724 extack = netdev_notifier_info_to_extack(&info->info); 9725 9726 profile = mlxsw_sp_rif_mac_profile_find(mlxsw_sp, info->dev_addr); 9727 if (profile) 9728 return 0; 9729 9730 max_rif_mac_profiles = mlxsw_sp->router->max_rif_mac_profile; 9731 occ = mlxsw_sp_rif_mac_profiles_occ_get(mlxsw_sp); 9732 if (occ < max_rif_mac_profiles) 9733 return 0; 9734 9735 if (!mlxsw_sp_rif_mac_profile_is_shared(rif)) 9736 return 0; 9737 9738 NL_SET_ERR_MSG_MOD(extack, "Exceeded number of supported router interface MAC profiles"); 9739 return -ENOBUFS; 9740 } 9741 9742 static bool mlxsw_sp_router_netdevice_interesting(struct mlxsw_sp *mlxsw_sp, 9743 struct net_device *dev) 9744 { 9745 struct vlan_dev_priv *vlan; 9746 9747 if (netif_is_lag_master(dev) || 9748 netif_is_bridge_master(dev) || 9749 mlxsw_sp_port_dev_check(dev) || 9750 mlxsw_sp_netdev_is_ipip_ol(mlxsw_sp, dev) || 9751 netif_is_l3_master(dev)) 9752 return true; 9753 9754 if (!is_vlan_dev(dev)) 9755 return false; 9756 9757 vlan = vlan_dev_priv(dev); 9758 return netif_is_lag_master(vlan->real_dev) || 9759 netif_is_bridge_master(vlan->real_dev) || 9760 mlxsw_sp_port_dev_check(vlan->real_dev); 9761 } 9762 9763 static struct mlxsw_sp_crif * 9764 mlxsw_sp_crif_register(struct mlxsw_sp_router *router, struct net_device *dev) 9765 { 9766 struct mlxsw_sp_crif *crif; 9767 int err; 9768 9769 if (WARN_ON(mlxsw_sp_crif_lookup(router, dev))) 9770 return NULL; 9771 9772 crif = mlxsw_sp_crif_alloc(dev); 9773 if (!crif) 9774 return ERR_PTR(-ENOMEM); 9775 9776 err = mlxsw_sp_crif_insert(router, crif); 9777 if (err) 9778 goto err_netdev_insert; 9779 9780 return crif; 9781 9782 err_netdev_insert: 9783 mlxsw_sp_crif_free(crif); 9784 return ERR_PTR(err); 9785 } 9786 9787 static void mlxsw_sp_crif_unregister(struct mlxsw_sp_router *router, 9788 struct mlxsw_sp_crif *crif) 9789 { 9790 struct mlxsw_sp_nexthop *nh, *tmp; 9791 9792 mlxsw_sp_crif_remove(router, crif); 9793 9794 list_for_each_entry_safe(nh, tmp, &crif->nexthop_list, crif_list_node) 9795 mlxsw_sp_nexthop_type_fini(router->mlxsw_sp, nh); 9796 9797 if (crif->rif) 9798 crif->can_destroy = true; 9799 else 9800 mlxsw_sp_crif_free(crif); 9801 } 9802 9803 static int mlxsw_sp_netdevice_register(struct mlxsw_sp_router *router, 9804 struct net_device *dev) 9805 { 9806 struct mlxsw_sp_crif *crif; 9807 9808 if (!mlxsw_sp_router_netdevice_interesting(router->mlxsw_sp, dev)) 9809 return 0; 9810 9811 crif = mlxsw_sp_crif_register(router, dev); 9812 return PTR_ERR_OR_ZERO(crif); 9813 } 9814 9815 static void mlxsw_sp_netdevice_unregister(struct mlxsw_sp_router *router, 9816 struct net_device *dev) 9817 { 9818 struct mlxsw_sp_crif *crif; 9819 9820 if (!mlxsw_sp_router_netdevice_interesting(router->mlxsw_sp, dev)) 9821 return; 9822 9823 /* netdev_run_todo(), by way of netdev_wait_allrefs_any(), rebroadcasts 9824 * the NETDEV_UNREGISTER message, so we can get here twice. If that's 9825 * what happened, the netdevice state is NETREG_UNREGISTERED. In that 9826 * case, we expect to have collected the CRIF already, and warn if it 9827 * still exists. Otherwise we expect the CRIF to exist. 9828 */ 9829 crif = mlxsw_sp_crif_lookup(router, dev); 9830 if (dev->reg_state == NETREG_UNREGISTERED) { 9831 if (!WARN_ON(crif)) 9832 return; 9833 } 9834 if (WARN_ON(!crif)) 9835 return; 9836 9837 mlxsw_sp_crif_unregister(router, crif); 9838 } 9839 9840 static bool mlxsw_sp_is_offload_xstats_event(unsigned long event) 9841 { 9842 switch (event) { 9843 case NETDEV_OFFLOAD_XSTATS_ENABLE: 9844 case NETDEV_OFFLOAD_XSTATS_DISABLE: 9845 case NETDEV_OFFLOAD_XSTATS_REPORT_USED: 9846 case NETDEV_OFFLOAD_XSTATS_REPORT_DELTA: 9847 return true; 9848 } 9849 9850 return false; 9851 } 9852 9853 static int 9854 mlxsw_sp_router_port_offload_xstats_cmd(struct mlxsw_sp_rif *rif, 9855 unsigned long event, 9856 struct netdev_notifier_offload_xstats_info *info) 9857 { 9858 switch (info->type) { 9859 case NETDEV_OFFLOAD_XSTATS_TYPE_L3: 9860 break; 9861 default: 9862 return 0; 9863 } 9864 9865 switch (event) { 9866 case NETDEV_OFFLOAD_XSTATS_ENABLE: 9867 return mlxsw_sp_router_port_l3_stats_enable(rif); 9868 case NETDEV_OFFLOAD_XSTATS_DISABLE: 9869 mlxsw_sp_router_port_l3_stats_disable(rif); 9870 return 0; 9871 case NETDEV_OFFLOAD_XSTATS_REPORT_USED: 9872 mlxsw_sp_router_port_l3_stats_report_used(rif, info); 9873 return 0; 9874 case NETDEV_OFFLOAD_XSTATS_REPORT_DELTA: 9875 return mlxsw_sp_router_port_l3_stats_report_delta(rif, info); 9876 } 9877 9878 WARN_ON_ONCE(1); 9879 return 0; 9880 } 9881 9882 static int 9883 mlxsw_sp_netdevice_offload_xstats_cmd(struct mlxsw_sp *mlxsw_sp, 9884 struct net_device *dev, 9885 unsigned long event, 9886 struct netdev_notifier_offload_xstats_info *info) 9887 { 9888 struct mlxsw_sp_rif *rif; 9889 9890 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev); 9891 if (!rif) 9892 return 0; 9893 9894 return mlxsw_sp_router_port_offload_xstats_cmd(rif, event, info); 9895 } 9896 9897 static bool mlxsw_sp_is_router_event(unsigned long event) 9898 { 9899 switch (event) { 9900 case NETDEV_PRE_CHANGEADDR: 9901 case NETDEV_CHANGEADDR: 9902 case NETDEV_CHANGEMTU: 9903 return true; 9904 default: 9905 return false; 9906 } 9907 } 9908 9909 static int mlxsw_sp_netdevice_router_port_event(struct net_device *dev, 9910 unsigned long event, void *ptr) 9911 { 9912 struct netlink_ext_ack *extack = netdev_notifier_info_to_extack(ptr); 9913 struct mlxsw_sp *mlxsw_sp; 9914 struct mlxsw_sp_rif *rif; 9915 9916 mlxsw_sp = mlxsw_sp_lower_get(dev); 9917 if (!mlxsw_sp) 9918 return 0; 9919 9920 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev); 9921 if (!rif) 9922 return 0; 9923 9924 switch (event) { 9925 case NETDEV_CHANGEMTU: 9926 case NETDEV_CHANGEADDR: 9927 return mlxsw_sp_router_port_change_event(mlxsw_sp, rif, extack); 9928 case NETDEV_PRE_CHANGEADDR: 9929 return mlxsw_sp_router_port_pre_changeaddr_event(rif, ptr); 9930 default: 9931 WARN_ON_ONCE(1); 9932 break; 9933 } 9934 9935 return 0; 9936 } 9937 9938 static int mlxsw_sp_port_vrf_join(struct mlxsw_sp *mlxsw_sp, 9939 struct net_device *l3_dev, 9940 struct netlink_ext_ack *extack) 9941 { 9942 struct mlxsw_sp_rif *rif; 9943 9944 /* If netdev is already associated with a RIF, then we need to 9945 * destroy it and create a new one with the new virtual router ID. 9946 */ 9947 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev); 9948 if (rif) 9949 __mlxsw_sp_inetaddr_event(mlxsw_sp, l3_dev, NETDEV_DOWN, false, 9950 extack); 9951 9952 return __mlxsw_sp_inetaddr_event(mlxsw_sp, l3_dev, NETDEV_UP, false, 9953 extack); 9954 } 9955 9956 static void mlxsw_sp_port_vrf_leave(struct mlxsw_sp *mlxsw_sp, 9957 struct net_device *l3_dev) 9958 { 9959 struct mlxsw_sp_rif *rif; 9960 9961 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev); 9962 if (!rif) 9963 return; 9964 __mlxsw_sp_inetaddr_event(mlxsw_sp, l3_dev, NETDEV_DOWN, false, NULL); 9965 } 9966 9967 static bool mlxsw_sp_is_vrf_event(unsigned long event, void *ptr) 9968 { 9969 struct netdev_notifier_changeupper_info *info = ptr; 9970 9971 if (event != NETDEV_PRECHANGEUPPER && event != NETDEV_CHANGEUPPER) 9972 return false; 9973 return netif_is_l3_master(info->upper_dev); 9974 } 9975 9976 static int 9977 mlxsw_sp_netdevice_vrf_event(struct net_device *l3_dev, unsigned long event, 9978 struct netdev_notifier_changeupper_info *info) 9979 { 9980 struct mlxsw_sp *mlxsw_sp = mlxsw_sp_lower_get(l3_dev); 9981 int err = 0; 9982 9983 /* We do not create a RIF for a macvlan, but only use it to 9984 * direct more MAC addresses to the router. 9985 */ 9986 if (!mlxsw_sp || netif_is_macvlan(l3_dev)) 9987 return 0; 9988 9989 switch (event) { 9990 case NETDEV_PRECHANGEUPPER: 9991 break; 9992 case NETDEV_CHANGEUPPER: 9993 if (info->linking) { 9994 struct netlink_ext_ack *extack; 9995 9996 extack = netdev_notifier_info_to_extack(&info->info); 9997 err = mlxsw_sp_port_vrf_join(mlxsw_sp, l3_dev, extack); 9998 } else { 9999 mlxsw_sp_port_vrf_leave(mlxsw_sp, l3_dev); 10000 } 10001 break; 10002 } 10003 10004 return err; 10005 } 10006 10007 struct mlxsw_sp_router_replay_inetaddr_up { 10008 struct mlxsw_sp *mlxsw_sp; 10009 struct netlink_ext_ack *extack; 10010 unsigned int done; 10011 bool deslavement; 10012 }; 10013 10014 static int mlxsw_sp_router_replay_inetaddr_up(struct net_device *dev, 10015 struct netdev_nested_priv *priv) 10016 { 10017 struct mlxsw_sp_router_replay_inetaddr_up *ctx = priv->data; 10018 bool nomaster = ctx->deslavement; 10019 struct mlxsw_sp_crif *crif; 10020 int err; 10021 10022 if (mlxsw_sp_dev_addr_list_empty(dev)) 10023 return 0; 10024 10025 crif = mlxsw_sp_crif_lookup(ctx->mlxsw_sp->router, dev); 10026 if (!crif || crif->rif) 10027 return 0; 10028 10029 if (!mlxsw_sp_rif_should_config(crif->rif, dev, NETDEV_UP)) 10030 return 0; 10031 10032 err = __mlxsw_sp_inetaddr_event(ctx->mlxsw_sp, dev, NETDEV_UP, 10033 nomaster, ctx->extack); 10034 if (err) 10035 return err; 10036 10037 ctx->done++; 10038 return 0; 10039 } 10040 10041 static int mlxsw_sp_router_unreplay_inetaddr_up(struct net_device *dev, 10042 struct netdev_nested_priv *priv) 10043 { 10044 struct mlxsw_sp_router_replay_inetaddr_up *ctx = priv->data; 10045 bool nomaster = ctx->deslavement; 10046 struct mlxsw_sp_crif *crif; 10047 10048 if (!ctx->done) 10049 return 0; 10050 10051 if (mlxsw_sp_dev_addr_list_empty(dev)) 10052 return 0; 10053 10054 crif = mlxsw_sp_crif_lookup(ctx->mlxsw_sp->router, dev); 10055 if (!crif || !crif->rif) 10056 return 0; 10057 10058 /* We are rolling back NETDEV_UP, so ask for that. */ 10059 if (!mlxsw_sp_rif_should_config(crif->rif, dev, NETDEV_UP)) 10060 return 0; 10061 10062 __mlxsw_sp_inetaddr_event(ctx->mlxsw_sp, dev, NETDEV_DOWN, nomaster, 10063 NULL); 10064 10065 ctx->done--; 10066 return 0; 10067 } 10068 10069 int mlxsw_sp_netdevice_enslavement_replay(struct mlxsw_sp *mlxsw_sp, 10070 struct net_device *upper_dev, 10071 struct netlink_ext_ack *extack) 10072 { 10073 struct mlxsw_sp_router_replay_inetaddr_up ctx = { 10074 .mlxsw_sp = mlxsw_sp, 10075 .extack = extack, 10076 .deslavement = false, 10077 }; 10078 struct netdev_nested_priv priv = { 10079 .data = &ctx, 10080 }; 10081 int err; 10082 10083 err = mlxsw_sp_router_replay_inetaddr_up(upper_dev, &priv); 10084 if (err) 10085 return err; 10086 10087 err = netdev_walk_all_upper_dev_rcu(upper_dev, 10088 mlxsw_sp_router_replay_inetaddr_up, 10089 &priv); 10090 if (err) 10091 goto err_replay_up; 10092 10093 return 0; 10094 10095 err_replay_up: 10096 netdev_walk_all_upper_dev_rcu(upper_dev, 10097 mlxsw_sp_router_unreplay_inetaddr_up, 10098 &priv); 10099 mlxsw_sp_router_unreplay_inetaddr_up(upper_dev, &priv); 10100 return err; 10101 } 10102 10103 void mlxsw_sp_netdevice_deslavement_replay(struct mlxsw_sp *mlxsw_sp, 10104 struct net_device *dev) 10105 { 10106 struct mlxsw_sp_router_replay_inetaddr_up ctx = { 10107 .mlxsw_sp = mlxsw_sp, 10108 .deslavement = true, 10109 }; 10110 struct netdev_nested_priv priv = { 10111 .data = &ctx, 10112 }; 10113 10114 mlxsw_sp_router_replay_inetaddr_up(dev, &priv); 10115 } 10116 10117 static int 10118 mlxsw_sp_port_vid_router_join_existing(struct mlxsw_sp_port *mlxsw_sp_port, 10119 u16 vid, struct net_device *dev, 10120 struct netlink_ext_ack *extack) 10121 { 10122 struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan; 10123 10124 mlxsw_sp_port_vlan = mlxsw_sp_port_vlan_find_by_vid(mlxsw_sp_port, 10125 vid); 10126 if (WARN_ON(!mlxsw_sp_port_vlan)) 10127 return -EINVAL; 10128 10129 return mlxsw_sp_port_vlan_router_join_existing(mlxsw_sp_port_vlan, 10130 dev, extack); 10131 } 10132 10133 static void 10134 mlxsw_sp_port_vid_router_leave(struct mlxsw_sp_port *mlxsw_sp_port, u16 vid, 10135 struct net_device *dev) 10136 { 10137 struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan; 10138 10139 mlxsw_sp_port_vlan = mlxsw_sp_port_vlan_find_by_vid(mlxsw_sp_port, 10140 vid); 10141 if (WARN_ON(!mlxsw_sp_port_vlan)) 10142 return; 10143 10144 __mlxsw_sp_port_vlan_router_leave(mlxsw_sp_port_vlan); 10145 } 10146 10147 static int __mlxsw_sp_router_port_join_lag(struct mlxsw_sp_port *mlxsw_sp_port, 10148 struct net_device *lag_dev, 10149 struct netlink_ext_ack *extack) 10150 { 10151 u16 default_vid = MLXSW_SP_DEFAULT_VID; 10152 struct net_device *upper_dev; 10153 struct list_head *iter; 10154 int done = 0; 10155 u16 vid; 10156 int err; 10157 10158 err = mlxsw_sp_port_vid_router_join_existing(mlxsw_sp_port, default_vid, 10159 lag_dev, extack); 10160 if (err) 10161 return err; 10162 10163 netdev_for_each_upper_dev_rcu(lag_dev, upper_dev, iter) { 10164 if (!is_vlan_dev(upper_dev)) 10165 continue; 10166 10167 vid = vlan_dev_vlan_id(upper_dev); 10168 err = mlxsw_sp_port_vid_router_join_existing(mlxsw_sp_port, vid, 10169 upper_dev, extack); 10170 if (err) 10171 goto err_router_join_dev; 10172 10173 ++done; 10174 } 10175 10176 return 0; 10177 10178 err_router_join_dev: 10179 netdev_for_each_upper_dev_rcu(lag_dev, upper_dev, iter) { 10180 if (!is_vlan_dev(upper_dev)) 10181 continue; 10182 if (!done--) 10183 break; 10184 10185 vid = vlan_dev_vlan_id(upper_dev); 10186 mlxsw_sp_port_vid_router_leave(mlxsw_sp_port, vid, upper_dev); 10187 } 10188 10189 mlxsw_sp_port_vid_router_leave(mlxsw_sp_port, default_vid, lag_dev); 10190 return err; 10191 } 10192 10193 static void 10194 __mlxsw_sp_router_port_leave_lag(struct mlxsw_sp_port *mlxsw_sp_port, 10195 struct net_device *lag_dev) 10196 { 10197 u16 default_vid = MLXSW_SP_DEFAULT_VID; 10198 struct net_device *upper_dev; 10199 struct list_head *iter; 10200 u16 vid; 10201 10202 netdev_for_each_upper_dev_rcu(lag_dev, upper_dev, iter) { 10203 if (!is_vlan_dev(upper_dev)) 10204 continue; 10205 10206 vid = vlan_dev_vlan_id(upper_dev); 10207 mlxsw_sp_port_vid_router_leave(mlxsw_sp_port, vid, upper_dev); 10208 } 10209 10210 mlxsw_sp_port_vid_router_leave(mlxsw_sp_port, default_vid, lag_dev); 10211 } 10212 10213 int mlxsw_sp_router_port_join_lag(struct mlxsw_sp_port *mlxsw_sp_port, 10214 struct net_device *lag_dev, 10215 struct netlink_ext_ack *extack) 10216 { 10217 int err; 10218 10219 mutex_lock(&mlxsw_sp_port->mlxsw_sp->router->lock); 10220 err = __mlxsw_sp_router_port_join_lag(mlxsw_sp_port, lag_dev, extack); 10221 mutex_unlock(&mlxsw_sp_port->mlxsw_sp->router->lock); 10222 10223 return err; 10224 } 10225 10226 void mlxsw_sp_router_port_leave_lag(struct mlxsw_sp_port *mlxsw_sp_port, 10227 struct net_device *lag_dev) 10228 { 10229 mutex_lock(&mlxsw_sp_port->mlxsw_sp->router->lock); 10230 __mlxsw_sp_router_port_leave_lag(mlxsw_sp_port, lag_dev); 10231 mutex_unlock(&mlxsw_sp_port->mlxsw_sp->router->lock); 10232 } 10233 10234 static int mlxsw_sp_router_netdevice_event(struct notifier_block *nb, 10235 unsigned long event, void *ptr) 10236 { 10237 struct net_device *dev = netdev_notifier_info_to_dev(ptr); 10238 struct mlxsw_sp_router *router; 10239 struct mlxsw_sp *mlxsw_sp; 10240 int err = 0; 10241 10242 router = container_of(nb, struct mlxsw_sp_router, netdevice_nb); 10243 mlxsw_sp = router->mlxsw_sp; 10244 10245 mutex_lock(&mlxsw_sp->router->lock); 10246 10247 if (event == NETDEV_REGISTER) { 10248 err = mlxsw_sp_netdevice_register(router, dev); 10249 if (err) 10250 /* No need to roll this back, UNREGISTER will collect it 10251 * anyhow. 10252 */ 10253 goto out; 10254 } 10255 10256 if (mlxsw_sp_is_offload_xstats_event(event)) 10257 err = mlxsw_sp_netdevice_offload_xstats_cmd(mlxsw_sp, dev, 10258 event, ptr); 10259 else if (mlxsw_sp_netdev_is_ipip_ol(mlxsw_sp, dev)) 10260 err = mlxsw_sp_netdevice_ipip_ol_event(mlxsw_sp, dev, 10261 event, ptr); 10262 else if (mlxsw_sp_netdev_is_ipip_ul(mlxsw_sp, dev)) 10263 err = mlxsw_sp_netdevice_ipip_ul_event(mlxsw_sp, dev, 10264 event, ptr); 10265 else if (mlxsw_sp_is_router_event(event)) 10266 err = mlxsw_sp_netdevice_router_port_event(dev, event, ptr); 10267 else if (mlxsw_sp_is_vrf_event(event, ptr)) 10268 err = mlxsw_sp_netdevice_vrf_event(dev, event, ptr); 10269 10270 if (event == NETDEV_UNREGISTER) 10271 mlxsw_sp_netdevice_unregister(router, dev); 10272 10273 out: 10274 mutex_unlock(&mlxsw_sp->router->lock); 10275 10276 return notifier_from_errno(err); 10277 } 10278 10279 struct mlxsw_sp_macvlan_replay { 10280 struct mlxsw_sp *mlxsw_sp; 10281 struct netlink_ext_ack *extack; 10282 }; 10283 10284 static int mlxsw_sp_macvlan_replay_upper(struct net_device *dev, 10285 struct netdev_nested_priv *priv) 10286 { 10287 const struct mlxsw_sp_macvlan_replay *rms = priv->data; 10288 struct netlink_ext_ack *extack = rms->extack; 10289 struct mlxsw_sp *mlxsw_sp = rms->mlxsw_sp; 10290 10291 if (!netif_is_macvlan(dev)) 10292 return 0; 10293 10294 return mlxsw_sp_rif_macvlan_add(mlxsw_sp, dev, extack); 10295 } 10296 10297 static int mlxsw_sp_macvlan_replay(struct mlxsw_sp_rif *rif, 10298 struct netlink_ext_ack *extack) 10299 { 10300 struct mlxsw_sp_macvlan_replay rms = { 10301 .mlxsw_sp = rif->mlxsw_sp, 10302 .extack = extack, 10303 }; 10304 struct netdev_nested_priv priv = { 10305 .data = &rms, 10306 }; 10307 10308 return netdev_walk_all_upper_dev_rcu(mlxsw_sp_rif_dev(rif), 10309 mlxsw_sp_macvlan_replay_upper, 10310 &priv); 10311 } 10312 10313 static int __mlxsw_sp_rif_macvlan_flush(struct net_device *dev, 10314 struct netdev_nested_priv *priv) 10315 { 10316 struct mlxsw_sp_rif *rif = (struct mlxsw_sp_rif *)priv->data; 10317 10318 if (!netif_is_macvlan(dev)) 10319 return 0; 10320 10321 return mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, dev->dev_addr, 10322 mlxsw_sp_fid_index(rif->fid), false); 10323 } 10324 10325 static int mlxsw_sp_rif_macvlan_flush(struct mlxsw_sp_rif *rif) 10326 { 10327 struct net_device *dev = mlxsw_sp_rif_dev(rif); 10328 struct netdev_nested_priv priv = { 10329 .data = (void *)rif, 10330 }; 10331 10332 if (!netif_is_macvlan_port(dev)) 10333 return 0; 10334 10335 return netdev_walk_all_upper_dev_rcu(dev, 10336 __mlxsw_sp_rif_macvlan_flush, &priv); 10337 } 10338 10339 static void mlxsw_sp_rif_subport_setup(struct mlxsw_sp_rif *rif, 10340 const struct mlxsw_sp_rif_params *params) 10341 { 10342 struct mlxsw_sp_rif_subport *rif_subport; 10343 10344 rif_subport = mlxsw_sp_rif_subport_rif(rif); 10345 refcount_set(&rif_subport->ref_count, 1); 10346 rif_subport->vid = params->vid; 10347 rif_subport->lag = params->lag; 10348 if (params->lag) 10349 rif_subport->lag_id = params->lag_id; 10350 else 10351 rif_subport->system_port = params->system_port; 10352 } 10353 10354 static int mlxsw_sp_rif_subport_op(struct mlxsw_sp_rif *rif, bool enable) 10355 { 10356 struct net_device *dev = mlxsw_sp_rif_dev(rif); 10357 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 10358 struct mlxsw_sp_rif_subport *rif_subport; 10359 char ritr_pl[MLXSW_REG_RITR_LEN]; 10360 u16 efid; 10361 10362 rif_subport = mlxsw_sp_rif_subport_rif(rif); 10363 mlxsw_reg_ritr_pack(ritr_pl, enable, MLXSW_REG_RITR_SP_IF, 10364 rif->rif_index, rif->vr_id, dev->mtu); 10365 mlxsw_reg_ritr_mac_pack(ritr_pl, dev->dev_addr); 10366 mlxsw_reg_ritr_if_mac_profile_id_set(ritr_pl, rif->mac_profile_id); 10367 efid = mlxsw_sp_fid_index(rif->fid); 10368 mlxsw_reg_ritr_sp_if_pack(ritr_pl, rif_subport->lag, 10369 rif_subport->lag ? rif_subport->lag_id : 10370 rif_subport->system_port, 10371 efid, 0); 10372 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 10373 } 10374 10375 static int mlxsw_sp_rif_subport_configure(struct mlxsw_sp_rif *rif, 10376 struct netlink_ext_ack *extack) 10377 { 10378 struct net_device *dev = mlxsw_sp_rif_dev(rif); 10379 u8 mac_profile; 10380 int err; 10381 10382 err = mlxsw_sp_rif_mac_profile_get(rif->mlxsw_sp, rif->addr, 10383 &mac_profile, extack); 10384 if (err) 10385 return err; 10386 rif->mac_profile_id = mac_profile; 10387 10388 err = mlxsw_sp_rif_subport_op(rif, true); 10389 if (err) 10390 goto err_rif_subport_op; 10391 10392 err = mlxsw_sp_macvlan_replay(rif, extack); 10393 if (err) 10394 goto err_macvlan_replay; 10395 10396 err = mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, dev->dev_addr, 10397 mlxsw_sp_fid_index(rif->fid), true); 10398 if (err) 10399 goto err_rif_fdb_op; 10400 10401 err = mlxsw_sp_fid_rif_set(rif->fid, rif); 10402 if (err) 10403 goto err_fid_rif_set; 10404 10405 return 0; 10406 10407 err_fid_rif_set: 10408 mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, dev->dev_addr, 10409 mlxsw_sp_fid_index(rif->fid), false); 10410 err_rif_fdb_op: 10411 mlxsw_sp_rif_macvlan_flush(rif); 10412 err_macvlan_replay: 10413 mlxsw_sp_rif_subport_op(rif, false); 10414 err_rif_subport_op: 10415 mlxsw_sp_rif_mac_profile_put(rif->mlxsw_sp, mac_profile); 10416 return err; 10417 } 10418 10419 static void mlxsw_sp_rif_subport_deconfigure(struct mlxsw_sp_rif *rif) 10420 { 10421 struct net_device *dev = mlxsw_sp_rif_dev(rif); 10422 struct mlxsw_sp_fid *fid = rif->fid; 10423 10424 mlxsw_sp_fid_rif_unset(fid); 10425 mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, dev->dev_addr, 10426 mlxsw_sp_fid_index(fid), false); 10427 mlxsw_sp_rif_macvlan_flush(rif); 10428 mlxsw_sp_rif_subport_op(rif, false); 10429 mlxsw_sp_rif_mac_profile_put(rif->mlxsw_sp, rif->mac_profile_id); 10430 } 10431 10432 static struct mlxsw_sp_fid * 10433 mlxsw_sp_rif_subport_fid_get(struct mlxsw_sp_rif *rif, 10434 const struct mlxsw_sp_rif_params *params, 10435 struct netlink_ext_ack *extack) 10436 { 10437 return mlxsw_sp_fid_rfid_get(rif->mlxsw_sp, rif->rif_index); 10438 } 10439 10440 static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_subport_ops = { 10441 .type = MLXSW_SP_RIF_TYPE_SUBPORT, 10442 .rif_size = sizeof(struct mlxsw_sp_rif_subport), 10443 .setup = mlxsw_sp_rif_subport_setup, 10444 .configure = mlxsw_sp_rif_subport_configure, 10445 .deconfigure = mlxsw_sp_rif_subport_deconfigure, 10446 .fid_get = mlxsw_sp_rif_subport_fid_get, 10447 }; 10448 10449 static int mlxsw_sp_rif_fid_op(struct mlxsw_sp_rif *rif, u16 fid, bool enable) 10450 { 10451 enum mlxsw_reg_ritr_if_type type = MLXSW_REG_RITR_FID_IF; 10452 struct net_device *dev = mlxsw_sp_rif_dev(rif); 10453 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 10454 char ritr_pl[MLXSW_REG_RITR_LEN]; 10455 10456 mlxsw_reg_ritr_pack(ritr_pl, enable, type, rif->rif_index, rif->vr_id, 10457 dev->mtu); 10458 mlxsw_reg_ritr_mac_pack(ritr_pl, dev->dev_addr); 10459 mlxsw_reg_ritr_if_mac_profile_id_set(ritr_pl, rif->mac_profile_id); 10460 mlxsw_reg_ritr_fid_if_fid_set(ritr_pl, fid); 10461 10462 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 10463 } 10464 10465 u16 mlxsw_sp_router_port(const struct mlxsw_sp *mlxsw_sp) 10466 { 10467 return mlxsw_core_max_ports(mlxsw_sp->core) + 1; 10468 } 10469 10470 static int mlxsw_sp_rif_fid_configure(struct mlxsw_sp_rif *rif, 10471 struct netlink_ext_ack *extack) 10472 { 10473 struct net_device *dev = mlxsw_sp_rif_dev(rif); 10474 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 10475 u16 fid_index = mlxsw_sp_fid_index(rif->fid); 10476 u8 mac_profile; 10477 int err; 10478 10479 err = mlxsw_sp_rif_mac_profile_get(mlxsw_sp, rif->addr, 10480 &mac_profile, extack); 10481 if (err) 10482 return err; 10483 rif->mac_profile_id = mac_profile; 10484 10485 err = mlxsw_sp_rif_fid_op(rif, fid_index, true); 10486 if (err) 10487 goto err_rif_fid_op; 10488 10489 err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC, 10490 mlxsw_sp_router_port(mlxsw_sp), true); 10491 if (err) 10492 goto err_fid_mc_flood_set; 10493 10494 err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC, 10495 mlxsw_sp_router_port(mlxsw_sp), true); 10496 if (err) 10497 goto err_fid_bc_flood_set; 10498 10499 err = mlxsw_sp_macvlan_replay(rif, extack); 10500 if (err) 10501 goto err_macvlan_replay; 10502 10503 err = mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, dev->dev_addr, 10504 mlxsw_sp_fid_index(rif->fid), true); 10505 if (err) 10506 goto err_rif_fdb_op; 10507 10508 err = mlxsw_sp_fid_rif_set(rif->fid, rif); 10509 if (err) 10510 goto err_fid_rif_set; 10511 10512 return 0; 10513 10514 err_fid_rif_set: 10515 mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, dev->dev_addr, 10516 mlxsw_sp_fid_index(rif->fid), false); 10517 err_rif_fdb_op: 10518 mlxsw_sp_rif_macvlan_flush(rif); 10519 err_macvlan_replay: 10520 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC, 10521 mlxsw_sp_router_port(mlxsw_sp), false); 10522 err_fid_bc_flood_set: 10523 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC, 10524 mlxsw_sp_router_port(mlxsw_sp), false); 10525 err_fid_mc_flood_set: 10526 mlxsw_sp_rif_fid_op(rif, fid_index, false); 10527 err_rif_fid_op: 10528 mlxsw_sp_rif_mac_profile_put(mlxsw_sp, mac_profile); 10529 return err; 10530 } 10531 10532 static void mlxsw_sp_rif_fid_deconfigure(struct mlxsw_sp_rif *rif) 10533 { 10534 struct net_device *dev = mlxsw_sp_rif_dev(rif); 10535 u16 fid_index = mlxsw_sp_fid_index(rif->fid); 10536 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 10537 struct mlxsw_sp_fid *fid = rif->fid; 10538 10539 mlxsw_sp_fid_rif_unset(fid); 10540 mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, dev->dev_addr, 10541 mlxsw_sp_fid_index(fid), false); 10542 mlxsw_sp_rif_macvlan_flush(rif); 10543 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC, 10544 mlxsw_sp_router_port(mlxsw_sp), false); 10545 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC, 10546 mlxsw_sp_router_port(mlxsw_sp), false); 10547 mlxsw_sp_rif_fid_op(rif, fid_index, false); 10548 mlxsw_sp_rif_mac_profile_put(rif->mlxsw_sp, rif->mac_profile_id); 10549 } 10550 10551 static struct mlxsw_sp_fid * 10552 mlxsw_sp_rif_fid_fid_get(struct mlxsw_sp_rif *rif, 10553 const struct mlxsw_sp_rif_params *params, 10554 struct netlink_ext_ack *extack) 10555 { 10556 int rif_ifindex = mlxsw_sp_rif_dev_ifindex(rif); 10557 10558 return mlxsw_sp_fid_8021d_get(rif->mlxsw_sp, rif_ifindex); 10559 } 10560 10561 static void mlxsw_sp_rif_fid_fdb_del(struct mlxsw_sp_rif *rif, const char *mac) 10562 { 10563 struct switchdev_notifier_fdb_info info = {}; 10564 struct net_device *dev; 10565 10566 dev = br_fdb_find_port(mlxsw_sp_rif_dev(rif), mac, 0); 10567 if (!dev) 10568 return; 10569 10570 info.addr = mac; 10571 info.vid = 0; 10572 call_switchdev_notifiers(SWITCHDEV_FDB_DEL_TO_BRIDGE, dev, &info.info, 10573 NULL); 10574 } 10575 10576 static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_fid_ops = { 10577 .type = MLXSW_SP_RIF_TYPE_FID, 10578 .rif_size = sizeof(struct mlxsw_sp_rif), 10579 .configure = mlxsw_sp_rif_fid_configure, 10580 .deconfigure = mlxsw_sp_rif_fid_deconfigure, 10581 .fid_get = mlxsw_sp_rif_fid_fid_get, 10582 .fdb_del = mlxsw_sp_rif_fid_fdb_del, 10583 }; 10584 10585 static struct mlxsw_sp_fid * 10586 mlxsw_sp_rif_vlan_fid_get(struct mlxsw_sp_rif *rif, 10587 const struct mlxsw_sp_rif_params *params, 10588 struct netlink_ext_ack *extack) 10589 { 10590 struct net_device *dev = mlxsw_sp_rif_dev(rif); 10591 struct net_device *br_dev; 10592 10593 if (WARN_ON(!params->vid)) 10594 return ERR_PTR(-EINVAL); 10595 10596 if (is_vlan_dev(dev)) { 10597 br_dev = vlan_dev_real_dev(dev); 10598 if (WARN_ON(!netif_is_bridge_master(br_dev))) 10599 return ERR_PTR(-EINVAL); 10600 } 10601 10602 return mlxsw_sp_fid_8021q_get(rif->mlxsw_sp, params->vid); 10603 } 10604 10605 static void mlxsw_sp_rif_vlan_fdb_del(struct mlxsw_sp_rif *rif, const char *mac) 10606 { 10607 struct net_device *rif_dev = mlxsw_sp_rif_dev(rif); 10608 struct switchdev_notifier_fdb_info info = {}; 10609 u16 vid = mlxsw_sp_fid_8021q_vid(rif->fid); 10610 struct net_device *br_dev; 10611 struct net_device *dev; 10612 10613 br_dev = is_vlan_dev(rif_dev) ? vlan_dev_real_dev(rif_dev) : rif_dev; 10614 dev = br_fdb_find_port(br_dev, mac, vid); 10615 if (!dev) 10616 return; 10617 10618 info.addr = mac; 10619 info.vid = vid; 10620 call_switchdev_notifiers(SWITCHDEV_FDB_DEL_TO_BRIDGE, dev, &info.info, 10621 NULL); 10622 } 10623 10624 static int mlxsw_sp_rif_vlan_op(struct mlxsw_sp_rif *rif, u16 vid, u16 efid, 10625 bool enable) 10626 { 10627 struct net_device *dev = mlxsw_sp_rif_dev(rif); 10628 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 10629 char ritr_pl[MLXSW_REG_RITR_LEN]; 10630 10631 mlxsw_reg_ritr_vlan_if_pack(ritr_pl, enable, rif->rif_index, rif->vr_id, 10632 dev->mtu, dev->dev_addr, 10633 rif->mac_profile_id, vid, efid); 10634 10635 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 10636 } 10637 10638 static int mlxsw_sp_rif_vlan_configure(struct mlxsw_sp_rif *rif, u16 efid, 10639 struct netlink_ext_ack *extack) 10640 { 10641 struct net_device *dev = mlxsw_sp_rif_dev(rif); 10642 u16 vid = mlxsw_sp_fid_8021q_vid(rif->fid); 10643 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 10644 u8 mac_profile; 10645 int err; 10646 10647 err = mlxsw_sp_rif_mac_profile_get(mlxsw_sp, rif->addr, 10648 &mac_profile, extack); 10649 if (err) 10650 return err; 10651 rif->mac_profile_id = mac_profile; 10652 10653 err = mlxsw_sp_rif_vlan_op(rif, vid, efid, true); 10654 if (err) 10655 goto err_rif_vlan_fid_op; 10656 10657 err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC, 10658 mlxsw_sp_router_port(mlxsw_sp), true); 10659 if (err) 10660 goto err_fid_mc_flood_set; 10661 10662 err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC, 10663 mlxsw_sp_router_port(mlxsw_sp), true); 10664 if (err) 10665 goto err_fid_bc_flood_set; 10666 10667 err = mlxsw_sp_macvlan_replay(rif, extack); 10668 if (err) 10669 goto err_macvlan_replay; 10670 10671 err = mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, dev->dev_addr, 10672 mlxsw_sp_fid_index(rif->fid), true); 10673 if (err) 10674 goto err_rif_fdb_op; 10675 10676 err = mlxsw_sp_fid_rif_set(rif->fid, rif); 10677 if (err) 10678 goto err_fid_rif_set; 10679 10680 return 0; 10681 10682 err_fid_rif_set: 10683 mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, dev->dev_addr, 10684 mlxsw_sp_fid_index(rif->fid), false); 10685 err_rif_fdb_op: 10686 mlxsw_sp_rif_macvlan_flush(rif); 10687 err_macvlan_replay: 10688 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC, 10689 mlxsw_sp_router_port(mlxsw_sp), false); 10690 err_fid_bc_flood_set: 10691 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC, 10692 mlxsw_sp_router_port(mlxsw_sp), false); 10693 err_fid_mc_flood_set: 10694 mlxsw_sp_rif_vlan_op(rif, vid, 0, false); 10695 err_rif_vlan_fid_op: 10696 mlxsw_sp_rif_mac_profile_put(mlxsw_sp, mac_profile); 10697 return err; 10698 } 10699 10700 static void mlxsw_sp_rif_vlan_deconfigure(struct mlxsw_sp_rif *rif) 10701 { 10702 struct net_device *dev = mlxsw_sp_rif_dev(rif); 10703 u16 vid = mlxsw_sp_fid_8021q_vid(rif->fid); 10704 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 10705 10706 mlxsw_sp_fid_rif_unset(rif->fid); 10707 mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, dev->dev_addr, 10708 mlxsw_sp_fid_index(rif->fid), false); 10709 mlxsw_sp_rif_macvlan_flush(rif); 10710 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC, 10711 mlxsw_sp_router_port(mlxsw_sp), false); 10712 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC, 10713 mlxsw_sp_router_port(mlxsw_sp), false); 10714 mlxsw_sp_rif_vlan_op(rif, vid, 0, false); 10715 mlxsw_sp_rif_mac_profile_put(rif->mlxsw_sp, rif->mac_profile_id); 10716 } 10717 10718 static int mlxsw_sp1_rif_vlan_configure(struct mlxsw_sp_rif *rif, 10719 struct netlink_ext_ack *extack) 10720 { 10721 return mlxsw_sp_rif_vlan_configure(rif, 0, extack); 10722 } 10723 10724 static const struct mlxsw_sp_rif_ops mlxsw_sp1_rif_vlan_ops = { 10725 .type = MLXSW_SP_RIF_TYPE_VLAN, 10726 .rif_size = sizeof(struct mlxsw_sp_rif), 10727 .configure = mlxsw_sp1_rif_vlan_configure, 10728 .deconfigure = mlxsw_sp_rif_vlan_deconfigure, 10729 .fid_get = mlxsw_sp_rif_vlan_fid_get, 10730 .fdb_del = mlxsw_sp_rif_vlan_fdb_del, 10731 }; 10732 10733 static int mlxsw_sp2_rif_vlan_configure(struct mlxsw_sp_rif *rif, 10734 struct netlink_ext_ack *extack) 10735 { 10736 u16 efid = mlxsw_sp_fid_index(rif->fid); 10737 10738 return mlxsw_sp_rif_vlan_configure(rif, efid, extack); 10739 } 10740 10741 static const struct mlxsw_sp_rif_ops mlxsw_sp2_rif_vlan_ops = { 10742 .type = MLXSW_SP_RIF_TYPE_VLAN, 10743 .rif_size = sizeof(struct mlxsw_sp_rif), 10744 .configure = mlxsw_sp2_rif_vlan_configure, 10745 .deconfigure = mlxsw_sp_rif_vlan_deconfigure, 10746 .fid_get = mlxsw_sp_rif_vlan_fid_get, 10747 .fdb_del = mlxsw_sp_rif_vlan_fdb_del, 10748 }; 10749 10750 static struct mlxsw_sp_rif_ipip_lb * 10751 mlxsw_sp_rif_ipip_lb_rif(struct mlxsw_sp_rif *rif) 10752 { 10753 return container_of(rif, struct mlxsw_sp_rif_ipip_lb, common); 10754 } 10755 10756 static void 10757 mlxsw_sp_rif_ipip_lb_setup(struct mlxsw_sp_rif *rif, 10758 const struct mlxsw_sp_rif_params *params) 10759 { 10760 struct mlxsw_sp_rif_params_ipip_lb *params_lb; 10761 struct mlxsw_sp_rif_ipip_lb *rif_lb; 10762 10763 params_lb = container_of(params, struct mlxsw_sp_rif_params_ipip_lb, 10764 common); 10765 rif_lb = mlxsw_sp_rif_ipip_lb_rif(rif); 10766 rif_lb->lb_config = params_lb->lb_config; 10767 } 10768 10769 static int 10770 mlxsw_sp1_rif_ipip_lb_configure(struct mlxsw_sp_rif *rif, 10771 struct netlink_ext_ack *extack) 10772 { 10773 struct mlxsw_sp_rif_ipip_lb *lb_rif = mlxsw_sp_rif_ipip_lb_rif(rif); 10774 struct net_device *dev = mlxsw_sp_rif_dev(rif); 10775 u32 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(dev); 10776 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 10777 struct mlxsw_sp_vr *ul_vr; 10778 int err; 10779 10780 ul_vr = mlxsw_sp_vr_get(mlxsw_sp, ul_tb_id, extack); 10781 if (IS_ERR(ul_vr)) 10782 return PTR_ERR(ul_vr); 10783 10784 err = mlxsw_sp_rif_ipip_lb_op(lb_rif, ul_vr->id, 0, true); 10785 if (err) 10786 goto err_loopback_op; 10787 10788 lb_rif->ul_vr_id = ul_vr->id; 10789 lb_rif->ul_rif_id = 0; 10790 ++ul_vr->rif_count; 10791 return 0; 10792 10793 err_loopback_op: 10794 mlxsw_sp_vr_put(mlxsw_sp, ul_vr); 10795 return err; 10796 } 10797 10798 static void mlxsw_sp1_rif_ipip_lb_deconfigure(struct mlxsw_sp_rif *rif) 10799 { 10800 struct mlxsw_sp_rif_ipip_lb *lb_rif = mlxsw_sp_rif_ipip_lb_rif(rif); 10801 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 10802 struct mlxsw_sp_vr *ul_vr; 10803 10804 ul_vr = &mlxsw_sp->router->vrs[lb_rif->ul_vr_id]; 10805 mlxsw_sp_rif_ipip_lb_op(lb_rif, ul_vr->id, 0, false); 10806 10807 --ul_vr->rif_count; 10808 mlxsw_sp_vr_put(mlxsw_sp, ul_vr); 10809 } 10810 10811 static const struct mlxsw_sp_rif_ops mlxsw_sp1_rif_ipip_lb_ops = { 10812 .type = MLXSW_SP_RIF_TYPE_IPIP_LB, 10813 .rif_size = sizeof(struct mlxsw_sp_rif_ipip_lb), 10814 .setup = mlxsw_sp_rif_ipip_lb_setup, 10815 .configure = mlxsw_sp1_rif_ipip_lb_configure, 10816 .deconfigure = mlxsw_sp1_rif_ipip_lb_deconfigure, 10817 }; 10818 10819 static const struct mlxsw_sp_rif_ops *mlxsw_sp1_rif_ops_arr[] = { 10820 [MLXSW_SP_RIF_TYPE_SUBPORT] = &mlxsw_sp_rif_subport_ops, 10821 [MLXSW_SP_RIF_TYPE_VLAN] = &mlxsw_sp1_rif_vlan_ops, 10822 [MLXSW_SP_RIF_TYPE_FID] = &mlxsw_sp_rif_fid_ops, 10823 [MLXSW_SP_RIF_TYPE_IPIP_LB] = &mlxsw_sp1_rif_ipip_lb_ops, 10824 }; 10825 10826 static int 10827 mlxsw_sp_rif_ipip_lb_ul_rif_op(struct mlxsw_sp_rif *ul_rif, bool enable) 10828 { 10829 struct mlxsw_sp *mlxsw_sp = ul_rif->mlxsw_sp; 10830 char ritr_pl[MLXSW_REG_RITR_LEN]; 10831 10832 mlxsw_reg_ritr_pack(ritr_pl, enable, MLXSW_REG_RITR_LOOPBACK_IF, 10833 ul_rif->rif_index, ul_rif->vr_id, IP_MAX_MTU); 10834 mlxsw_reg_ritr_loopback_protocol_set(ritr_pl, 10835 MLXSW_REG_RITR_LOOPBACK_GENERIC); 10836 10837 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 10838 } 10839 10840 static struct mlxsw_sp_rif * 10841 mlxsw_sp_ul_rif_create(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp_vr *vr, 10842 struct mlxsw_sp_crif *ul_crif, 10843 struct netlink_ext_ack *extack) 10844 { 10845 struct mlxsw_sp_rif *ul_rif; 10846 u8 rif_entries = 1; 10847 u16 rif_index; 10848 int err; 10849 10850 err = mlxsw_sp_rif_index_alloc(mlxsw_sp, &rif_index, rif_entries); 10851 if (err) { 10852 NL_SET_ERR_MSG_MOD(extack, "Exceeded number of supported router interfaces"); 10853 return ERR_PTR(err); 10854 } 10855 10856 ul_rif = mlxsw_sp_rif_alloc(sizeof(*ul_rif), rif_index, vr->id, 10857 ul_crif); 10858 if (!ul_rif) { 10859 err = -ENOMEM; 10860 goto err_rif_alloc; 10861 } 10862 10863 mlxsw_sp->router->rifs[rif_index] = ul_rif; 10864 ul_rif->mlxsw_sp = mlxsw_sp; 10865 ul_rif->rif_entries = rif_entries; 10866 err = mlxsw_sp_rif_ipip_lb_ul_rif_op(ul_rif, true); 10867 if (err) 10868 goto ul_rif_op_err; 10869 10870 atomic_add(rif_entries, &mlxsw_sp->router->rifs_count); 10871 return ul_rif; 10872 10873 ul_rif_op_err: 10874 mlxsw_sp->router->rifs[rif_index] = NULL; 10875 mlxsw_sp_rif_free(ul_rif); 10876 err_rif_alloc: 10877 mlxsw_sp_rif_index_free(mlxsw_sp, rif_index, rif_entries); 10878 return ERR_PTR(err); 10879 } 10880 10881 static void mlxsw_sp_ul_rif_destroy(struct mlxsw_sp_rif *ul_rif) 10882 { 10883 struct mlxsw_sp *mlxsw_sp = ul_rif->mlxsw_sp; 10884 u8 rif_entries = ul_rif->rif_entries; 10885 u16 rif_index = ul_rif->rif_index; 10886 10887 atomic_sub(rif_entries, &mlxsw_sp->router->rifs_count); 10888 mlxsw_sp_rif_ipip_lb_ul_rif_op(ul_rif, false); 10889 mlxsw_sp->router->rifs[ul_rif->rif_index] = NULL; 10890 mlxsw_sp_rif_free(ul_rif); 10891 mlxsw_sp_rif_index_free(mlxsw_sp, rif_index, rif_entries); 10892 } 10893 10894 static struct mlxsw_sp_rif * 10895 mlxsw_sp_ul_rif_get(struct mlxsw_sp *mlxsw_sp, u32 tb_id, 10896 struct mlxsw_sp_crif *ul_crif, 10897 struct netlink_ext_ack *extack) 10898 { 10899 struct mlxsw_sp_vr *vr; 10900 int err; 10901 10902 vr = mlxsw_sp_vr_get(mlxsw_sp, tb_id, extack); 10903 if (IS_ERR(vr)) 10904 return ERR_CAST(vr); 10905 10906 if (refcount_inc_not_zero(&vr->ul_rif_refcnt)) 10907 return vr->ul_rif; 10908 10909 vr->ul_rif = mlxsw_sp_ul_rif_create(mlxsw_sp, vr, ul_crif, extack); 10910 if (IS_ERR(vr->ul_rif)) { 10911 err = PTR_ERR(vr->ul_rif); 10912 goto err_ul_rif_create; 10913 } 10914 10915 vr->rif_count++; 10916 refcount_set(&vr->ul_rif_refcnt, 1); 10917 10918 return vr->ul_rif; 10919 10920 err_ul_rif_create: 10921 mlxsw_sp_vr_put(mlxsw_sp, vr); 10922 return ERR_PTR(err); 10923 } 10924 10925 static void mlxsw_sp_ul_rif_put(struct mlxsw_sp_rif *ul_rif) 10926 { 10927 struct mlxsw_sp *mlxsw_sp = ul_rif->mlxsw_sp; 10928 struct mlxsw_sp_vr *vr; 10929 10930 vr = &mlxsw_sp->router->vrs[ul_rif->vr_id]; 10931 10932 if (!refcount_dec_and_test(&vr->ul_rif_refcnt)) 10933 return; 10934 10935 vr->rif_count--; 10936 mlxsw_sp_ul_rif_destroy(ul_rif); 10937 mlxsw_sp_vr_put(mlxsw_sp, vr); 10938 } 10939 10940 int mlxsw_sp_router_ul_rif_get(struct mlxsw_sp *mlxsw_sp, u32 ul_tb_id, 10941 u16 *ul_rif_index) 10942 { 10943 struct mlxsw_sp_rif *ul_rif; 10944 int err = 0; 10945 10946 mutex_lock(&mlxsw_sp->router->lock); 10947 ul_rif = mlxsw_sp_ul_rif_get(mlxsw_sp, ul_tb_id, NULL, NULL); 10948 if (IS_ERR(ul_rif)) { 10949 err = PTR_ERR(ul_rif); 10950 goto out; 10951 } 10952 *ul_rif_index = ul_rif->rif_index; 10953 out: 10954 mutex_unlock(&mlxsw_sp->router->lock); 10955 return err; 10956 } 10957 10958 void mlxsw_sp_router_ul_rif_put(struct mlxsw_sp *mlxsw_sp, u16 ul_rif_index) 10959 { 10960 struct mlxsw_sp_rif *ul_rif; 10961 10962 mutex_lock(&mlxsw_sp->router->lock); 10963 ul_rif = mlxsw_sp->router->rifs[ul_rif_index]; 10964 if (WARN_ON(!ul_rif)) 10965 goto out; 10966 10967 mlxsw_sp_ul_rif_put(ul_rif); 10968 out: 10969 mutex_unlock(&mlxsw_sp->router->lock); 10970 } 10971 10972 static int 10973 mlxsw_sp2_rif_ipip_lb_configure(struct mlxsw_sp_rif *rif, 10974 struct netlink_ext_ack *extack) 10975 { 10976 struct mlxsw_sp_rif_ipip_lb *lb_rif = mlxsw_sp_rif_ipip_lb_rif(rif); 10977 struct net_device *dev = mlxsw_sp_rif_dev(rif); 10978 u32 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(dev); 10979 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 10980 struct mlxsw_sp_rif *ul_rif; 10981 int err; 10982 10983 ul_rif = mlxsw_sp_ul_rif_get(mlxsw_sp, ul_tb_id, NULL, extack); 10984 if (IS_ERR(ul_rif)) 10985 return PTR_ERR(ul_rif); 10986 10987 err = mlxsw_sp_rif_ipip_lb_op(lb_rif, 0, ul_rif->rif_index, true); 10988 if (err) 10989 goto err_loopback_op; 10990 10991 lb_rif->ul_vr_id = 0; 10992 lb_rif->ul_rif_id = ul_rif->rif_index; 10993 10994 return 0; 10995 10996 err_loopback_op: 10997 mlxsw_sp_ul_rif_put(ul_rif); 10998 return err; 10999 } 11000 11001 static void mlxsw_sp2_rif_ipip_lb_deconfigure(struct mlxsw_sp_rif *rif) 11002 { 11003 struct mlxsw_sp_rif_ipip_lb *lb_rif = mlxsw_sp_rif_ipip_lb_rif(rif); 11004 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 11005 struct mlxsw_sp_rif *ul_rif; 11006 11007 ul_rif = mlxsw_sp_rif_by_index(mlxsw_sp, lb_rif->ul_rif_id); 11008 mlxsw_sp_rif_ipip_lb_op(lb_rif, 0, lb_rif->ul_rif_id, false); 11009 mlxsw_sp_ul_rif_put(ul_rif); 11010 } 11011 11012 static const struct mlxsw_sp_rif_ops mlxsw_sp2_rif_ipip_lb_ops = { 11013 .type = MLXSW_SP_RIF_TYPE_IPIP_LB, 11014 .rif_size = sizeof(struct mlxsw_sp_rif_ipip_lb), 11015 .setup = mlxsw_sp_rif_ipip_lb_setup, 11016 .configure = mlxsw_sp2_rif_ipip_lb_configure, 11017 .deconfigure = mlxsw_sp2_rif_ipip_lb_deconfigure, 11018 }; 11019 11020 static const struct mlxsw_sp_rif_ops *mlxsw_sp2_rif_ops_arr[] = { 11021 [MLXSW_SP_RIF_TYPE_SUBPORT] = &mlxsw_sp_rif_subport_ops, 11022 [MLXSW_SP_RIF_TYPE_VLAN] = &mlxsw_sp2_rif_vlan_ops, 11023 [MLXSW_SP_RIF_TYPE_FID] = &mlxsw_sp_rif_fid_ops, 11024 [MLXSW_SP_RIF_TYPE_IPIP_LB] = &mlxsw_sp2_rif_ipip_lb_ops, 11025 }; 11026 11027 static int mlxsw_sp_rifs_table_init(struct mlxsw_sp *mlxsw_sp) 11028 { 11029 struct gen_pool *rifs_table; 11030 int err; 11031 11032 rifs_table = gen_pool_create(0, -1); 11033 if (!rifs_table) 11034 return -ENOMEM; 11035 11036 gen_pool_set_algo(rifs_table, gen_pool_first_fit_order_align, 11037 NULL); 11038 11039 err = gen_pool_add(rifs_table, MLXSW_SP_ROUTER_GENALLOC_OFFSET, 11040 MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS), -1); 11041 if (err) 11042 goto err_gen_pool_add; 11043 11044 mlxsw_sp->router->rifs_table = rifs_table; 11045 11046 return 0; 11047 11048 err_gen_pool_add: 11049 gen_pool_destroy(rifs_table); 11050 return err; 11051 } 11052 11053 static void mlxsw_sp_rifs_table_fini(struct mlxsw_sp *mlxsw_sp) 11054 { 11055 gen_pool_destroy(mlxsw_sp->router->rifs_table); 11056 } 11057 11058 static int mlxsw_sp_rifs_init(struct mlxsw_sp *mlxsw_sp) 11059 { 11060 u64 max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); 11061 struct devlink *devlink = priv_to_devlink(mlxsw_sp->core); 11062 struct mlxsw_core *core = mlxsw_sp->core; 11063 int err; 11064 11065 if (!MLXSW_CORE_RES_VALID(core, MAX_RIF_MAC_PROFILES)) 11066 return -EIO; 11067 mlxsw_sp->router->max_rif_mac_profile = 11068 MLXSW_CORE_RES_GET(core, MAX_RIF_MAC_PROFILES); 11069 11070 mlxsw_sp->router->rifs = kzalloc_objs(struct mlxsw_sp_rif *, max_rifs); 11071 if (!mlxsw_sp->router->rifs) 11072 return -ENOMEM; 11073 11074 err = mlxsw_sp_rifs_table_init(mlxsw_sp); 11075 if (err) 11076 goto err_rifs_table_init; 11077 11078 idr_init(&mlxsw_sp->router->rif_mac_profiles_idr); 11079 atomic_set(&mlxsw_sp->router->rif_mac_profiles_count, 0); 11080 atomic_set(&mlxsw_sp->router->rifs_count, 0); 11081 devl_resource_occ_get_register(devlink, 11082 MLXSW_SP_RESOURCE_RIF_MAC_PROFILES, 11083 mlxsw_sp_rif_mac_profiles_occ_get, 11084 mlxsw_sp); 11085 devl_resource_occ_get_register(devlink, 11086 MLXSW_SP_RESOURCE_RIFS, 11087 mlxsw_sp_rifs_occ_get, 11088 mlxsw_sp); 11089 11090 return 0; 11091 11092 err_rifs_table_init: 11093 kfree(mlxsw_sp->router->rifs); 11094 return err; 11095 } 11096 11097 static void mlxsw_sp_rifs_fini(struct mlxsw_sp *mlxsw_sp) 11098 { 11099 int max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); 11100 struct devlink *devlink = priv_to_devlink(mlxsw_sp->core); 11101 int i; 11102 11103 WARN_ON_ONCE(atomic_read(&mlxsw_sp->router->rifs_count)); 11104 for (i = 0; i < max_rifs; i++) 11105 WARN_ON_ONCE(mlxsw_sp->router->rifs[i]); 11106 11107 devl_resource_occ_get_unregister(devlink, MLXSW_SP_RESOURCE_RIFS); 11108 devl_resource_occ_get_unregister(devlink, 11109 MLXSW_SP_RESOURCE_RIF_MAC_PROFILES); 11110 WARN_ON(!idr_is_empty(&mlxsw_sp->router->rif_mac_profiles_idr)); 11111 idr_destroy(&mlxsw_sp->router->rif_mac_profiles_idr); 11112 mlxsw_sp_rifs_table_fini(mlxsw_sp); 11113 kfree(mlxsw_sp->router->rifs); 11114 } 11115 11116 static int 11117 mlxsw_sp_ipip_config_tigcr(struct mlxsw_sp *mlxsw_sp) 11118 { 11119 char tigcr_pl[MLXSW_REG_TIGCR_LEN]; 11120 11121 mlxsw_reg_tigcr_pack(tigcr_pl, true, 0); 11122 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(tigcr), tigcr_pl); 11123 } 11124 11125 static int mlxsw_sp_ipips_init(struct mlxsw_sp *mlxsw_sp) 11126 { 11127 int err; 11128 11129 INIT_LIST_HEAD(&mlxsw_sp->router->ipip_list); 11130 11131 err = mlxsw_sp_ipip_ecn_encap_init(mlxsw_sp); 11132 if (err) 11133 return err; 11134 err = mlxsw_sp_ipip_ecn_decap_init(mlxsw_sp); 11135 if (err) 11136 return err; 11137 11138 return mlxsw_sp_ipip_config_tigcr(mlxsw_sp); 11139 } 11140 11141 static int mlxsw_sp1_ipips_init(struct mlxsw_sp *mlxsw_sp) 11142 { 11143 mlxsw_sp->router->ipip_ops_arr = mlxsw_sp1_ipip_ops_arr; 11144 return mlxsw_sp_ipips_init(mlxsw_sp); 11145 } 11146 11147 static int mlxsw_sp2_ipips_init(struct mlxsw_sp *mlxsw_sp) 11148 { 11149 mlxsw_sp->router->ipip_ops_arr = mlxsw_sp2_ipip_ops_arr; 11150 return mlxsw_sp_ipips_init(mlxsw_sp); 11151 } 11152 11153 static void mlxsw_sp_ipips_fini(struct mlxsw_sp *mlxsw_sp) 11154 { 11155 WARN_ON(!list_empty(&mlxsw_sp->router->ipip_list)); 11156 } 11157 11158 static void mlxsw_sp_router_fib_dump_flush(struct notifier_block *nb) 11159 { 11160 struct mlxsw_sp_router *router; 11161 11162 /* Flush pending FIB notifications and then flush the device's 11163 * table before requesting another dump. The FIB notification 11164 * block is unregistered, so no need to take RTNL. 11165 */ 11166 mlxsw_core_flush_owq(); 11167 router = container_of(nb, struct mlxsw_sp_router, fib_nb); 11168 mlxsw_sp_router_fib_flush(router->mlxsw_sp); 11169 } 11170 11171 #ifdef CONFIG_IP_ROUTE_MULTIPATH 11172 struct mlxsw_sp_mp_hash_config { 11173 DECLARE_BITMAP(headers, __MLXSW_REG_RECR2_HEADER_CNT); 11174 DECLARE_BITMAP(fields, __MLXSW_REG_RECR2_FIELD_CNT); 11175 DECLARE_BITMAP(inner_headers, __MLXSW_REG_RECR2_HEADER_CNT); 11176 DECLARE_BITMAP(inner_fields, __MLXSW_REG_RECR2_INNER_FIELD_CNT); 11177 bool inc_parsing_depth; 11178 }; 11179 11180 #define MLXSW_SP_MP_HASH_HEADER_SET(_headers, _header) \ 11181 bitmap_set(_headers, MLXSW_REG_RECR2_##_header, 1) 11182 11183 #define MLXSW_SP_MP_HASH_FIELD_SET(_fields, _field) \ 11184 bitmap_set(_fields, MLXSW_REG_RECR2_##_field, 1) 11185 11186 #define MLXSW_SP_MP_HASH_FIELD_RANGE_SET(_fields, _field, _nr) \ 11187 bitmap_set(_fields, MLXSW_REG_RECR2_##_field, _nr) 11188 11189 static void mlxsw_sp_mp_hash_inner_l3(struct mlxsw_sp_mp_hash_config *config) 11190 { 11191 unsigned long *inner_headers = config->inner_headers; 11192 unsigned long *inner_fields = config->inner_fields; 11193 11194 /* IPv4 inner */ 11195 MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, IPV4_EN_NOT_TCP_NOT_UDP); 11196 MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, IPV4_EN_TCP_UDP); 11197 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(inner_fields, INNER_IPV4_SIP0, 4); 11198 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(inner_fields, INNER_IPV4_DIP0, 4); 11199 /* IPv6 inner */ 11200 MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, IPV6_EN_NOT_TCP_NOT_UDP); 11201 MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, IPV6_EN_TCP_UDP); 11202 MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV6_SIP0_7); 11203 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(inner_fields, INNER_IPV6_SIP8, 8); 11204 MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV6_DIP0_7); 11205 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(inner_fields, INNER_IPV6_DIP8, 8); 11206 MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV6_NEXT_HEADER); 11207 MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV6_FLOW_LABEL); 11208 } 11209 11210 static void mlxsw_sp_mp4_hash_outer_addr(struct mlxsw_sp_mp_hash_config *config) 11211 { 11212 unsigned long *headers = config->headers; 11213 unsigned long *fields = config->fields; 11214 11215 MLXSW_SP_MP_HASH_HEADER_SET(headers, IPV4_EN_NOT_TCP_NOT_UDP); 11216 MLXSW_SP_MP_HASH_HEADER_SET(headers, IPV4_EN_TCP_UDP); 11217 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(fields, IPV4_SIP0, 4); 11218 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(fields, IPV4_DIP0, 4); 11219 } 11220 11221 static void 11222 mlxsw_sp_mp_hash_inner_custom(struct mlxsw_sp_mp_hash_config *config, 11223 u32 hash_fields) 11224 { 11225 unsigned long *inner_headers = config->inner_headers; 11226 unsigned long *inner_fields = config->inner_fields; 11227 11228 /* IPv4 Inner */ 11229 MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, IPV4_EN_NOT_TCP_NOT_UDP); 11230 MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, IPV4_EN_TCP_UDP); 11231 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_SRC_IP) 11232 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(inner_fields, INNER_IPV4_SIP0, 4); 11233 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_DST_IP) 11234 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(inner_fields, INNER_IPV4_DIP0, 4); 11235 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_IP_PROTO) 11236 MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV4_PROTOCOL); 11237 /* IPv6 inner */ 11238 MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, IPV6_EN_NOT_TCP_NOT_UDP); 11239 MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, IPV6_EN_TCP_UDP); 11240 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_SRC_IP) { 11241 MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV6_SIP0_7); 11242 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(inner_fields, INNER_IPV6_SIP8, 8); 11243 } 11244 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_DST_IP) { 11245 MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV6_DIP0_7); 11246 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(inner_fields, INNER_IPV6_DIP8, 8); 11247 } 11248 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_IP_PROTO) 11249 MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV6_NEXT_HEADER); 11250 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_FLOWLABEL) 11251 MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV6_FLOW_LABEL); 11252 /* L4 inner */ 11253 MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, TCP_UDP_EN_IPV4); 11254 MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, TCP_UDP_EN_IPV6); 11255 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_SRC_PORT) 11256 MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_TCP_UDP_SPORT); 11257 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_DST_PORT) 11258 MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_TCP_UDP_DPORT); 11259 } 11260 11261 static void mlxsw_sp_mp4_hash_init(struct mlxsw_sp *mlxsw_sp, 11262 struct mlxsw_sp_mp_hash_config *config) 11263 { 11264 struct net *net = mlxsw_sp_net(mlxsw_sp); 11265 unsigned long *headers = config->headers; 11266 unsigned long *fields = config->fields; 11267 u32 hash_fields; 11268 11269 switch (READ_ONCE(net->ipv4.sysctl_fib_multipath_hash_policy)) { 11270 case 0: 11271 mlxsw_sp_mp4_hash_outer_addr(config); 11272 break; 11273 case 1: 11274 mlxsw_sp_mp4_hash_outer_addr(config); 11275 MLXSW_SP_MP_HASH_HEADER_SET(headers, TCP_UDP_EN_IPV4); 11276 MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV4_PROTOCOL); 11277 MLXSW_SP_MP_HASH_FIELD_SET(fields, TCP_UDP_SPORT); 11278 MLXSW_SP_MP_HASH_FIELD_SET(fields, TCP_UDP_DPORT); 11279 break; 11280 case 2: 11281 /* Outer */ 11282 mlxsw_sp_mp4_hash_outer_addr(config); 11283 /* Inner */ 11284 mlxsw_sp_mp_hash_inner_l3(config); 11285 break; 11286 case 3: 11287 hash_fields = READ_ONCE(net->ipv4.sysctl_fib_multipath_hash_fields); 11288 /* Outer */ 11289 MLXSW_SP_MP_HASH_HEADER_SET(headers, IPV4_EN_NOT_TCP_NOT_UDP); 11290 MLXSW_SP_MP_HASH_HEADER_SET(headers, IPV4_EN_TCP_UDP); 11291 MLXSW_SP_MP_HASH_HEADER_SET(headers, TCP_UDP_EN_IPV4); 11292 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_SRC_IP) 11293 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(fields, IPV4_SIP0, 4); 11294 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_DST_IP) 11295 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(fields, IPV4_DIP0, 4); 11296 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_IP_PROTO) 11297 MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV4_PROTOCOL); 11298 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_SRC_PORT) 11299 MLXSW_SP_MP_HASH_FIELD_SET(fields, TCP_UDP_SPORT); 11300 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_DST_PORT) 11301 MLXSW_SP_MP_HASH_FIELD_SET(fields, TCP_UDP_DPORT); 11302 /* Inner */ 11303 mlxsw_sp_mp_hash_inner_custom(config, hash_fields); 11304 break; 11305 } 11306 } 11307 11308 static void mlxsw_sp_mp6_hash_outer_addr(struct mlxsw_sp_mp_hash_config *config) 11309 { 11310 unsigned long *headers = config->headers; 11311 unsigned long *fields = config->fields; 11312 11313 MLXSW_SP_MP_HASH_HEADER_SET(headers, IPV6_EN_NOT_TCP_NOT_UDP); 11314 MLXSW_SP_MP_HASH_HEADER_SET(headers, IPV6_EN_TCP_UDP); 11315 MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_SIP0_7); 11316 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(fields, IPV6_SIP8, 8); 11317 MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_DIP0_7); 11318 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(fields, IPV6_DIP8, 8); 11319 } 11320 11321 static void mlxsw_sp_mp6_hash_init(struct mlxsw_sp *mlxsw_sp, 11322 struct mlxsw_sp_mp_hash_config *config) 11323 { 11324 u32 hash_fields = ip6_multipath_hash_fields(mlxsw_sp_net(mlxsw_sp)); 11325 unsigned long *headers = config->headers; 11326 unsigned long *fields = config->fields; 11327 11328 switch (ip6_multipath_hash_policy(mlxsw_sp_net(mlxsw_sp))) { 11329 case 0: 11330 mlxsw_sp_mp6_hash_outer_addr(config); 11331 MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_NEXT_HEADER); 11332 MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_FLOW_LABEL); 11333 break; 11334 case 1: 11335 mlxsw_sp_mp6_hash_outer_addr(config); 11336 MLXSW_SP_MP_HASH_HEADER_SET(headers, TCP_UDP_EN_IPV6); 11337 MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_NEXT_HEADER); 11338 MLXSW_SP_MP_HASH_FIELD_SET(fields, TCP_UDP_SPORT); 11339 MLXSW_SP_MP_HASH_FIELD_SET(fields, TCP_UDP_DPORT); 11340 break; 11341 case 2: 11342 /* Outer */ 11343 mlxsw_sp_mp6_hash_outer_addr(config); 11344 MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_NEXT_HEADER); 11345 MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_FLOW_LABEL); 11346 /* Inner */ 11347 mlxsw_sp_mp_hash_inner_l3(config); 11348 config->inc_parsing_depth = true; 11349 break; 11350 case 3: 11351 /* Outer */ 11352 MLXSW_SP_MP_HASH_HEADER_SET(headers, IPV6_EN_NOT_TCP_NOT_UDP); 11353 MLXSW_SP_MP_HASH_HEADER_SET(headers, IPV6_EN_TCP_UDP); 11354 MLXSW_SP_MP_HASH_HEADER_SET(headers, TCP_UDP_EN_IPV6); 11355 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_SRC_IP) { 11356 MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_SIP0_7); 11357 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(fields, IPV6_SIP8, 8); 11358 } 11359 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_DST_IP) { 11360 MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_DIP0_7); 11361 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(fields, IPV6_DIP8, 8); 11362 } 11363 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_IP_PROTO) 11364 MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_NEXT_HEADER); 11365 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_FLOWLABEL) 11366 MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_FLOW_LABEL); 11367 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_SRC_PORT) 11368 MLXSW_SP_MP_HASH_FIELD_SET(fields, TCP_UDP_SPORT); 11369 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_DST_PORT) 11370 MLXSW_SP_MP_HASH_FIELD_SET(fields, TCP_UDP_DPORT); 11371 /* Inner */ 11372 mlxsw_sp_mp_hash_inner_custom(config, hash_fields); 11373 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_MASK) 11374 config->inc_parsing_depth = true; 11375 break; 11376 } 11377 } 11378 11379 static int mlxsw_sp_mp_hash_parsing_depth_adjust(struct mlxsw_sp *mlxsw_sp, 11380 bool old_inc_parsing_depth, 11381 bool new_inc_parsing_depth) 11382 { 11383 int err; 11384 11385 if (!old_inc_parsing_depth && new_inc_parsing_depth) { 11386 err = mlxsw_sp_parsing_depth_inc(mlxsw_sp); 11387 if (err) 11388 return err; 11389 mlxsw_sp->router->inc_parsing_depth = true; 11390 } else if (old_inc_parsing_depth && !new_inc_parsing_depth) { 11391 mlxsw_sp_parsing_depth_dec(mlxsw_sp); 11392 mlxsw_sp->router->inc_parsing_depth = false; 11393 } 11394 11395 return 0; 11396 } 11397 11398 static int mlxsw_sp_mp_hash_init(struct mlxsw_sp *mlxsw_sp) 11399 { 11400 bool old_inc_parsing_depth, new_inc_parsing_depth; 11401 struct mlxsw_sp_mp_hash_config config = {}; 11402 struct net *net = mlxsw_sp_net(mlxsw_sp); 11403 char recr2_pl[MLXSW_REG_RECR2_LEN]; 11404 unsigned long bit; 11405 u32 seed; 11406 int err; 11407 11408 seed = READ_ONCE(net->ipv4.sysctl_fib_multipath_hash_seed).user_seed; 11409 if (!seed) 11410 seed = jhash(mlxsw_sp->base_mac, sizeof(mlxsw_sp->base_mac), 0); 11411 11412 mlxsw_reg_recr2_pack(recr2_pl, seed); 11413 mlxsw_sp_mp4_hash_init(mlxsw_sp, &config); 11414 mlxsw_sp_mp6_hash_init(mlxsw_sp, &config); 11415 11416 old_inc_parsing_depth = mlxsw_sp->router->inc_parsing_depth; 11417 new_inc_parsing_depth = config.inc_parsing_depth; 11418 err = mlxsw_sp_mp_hash_parsing_depth_adjust(mlxsw_sp, 11419 old_inc_parsing_depth, 11420 new_inc_parsing_depth); 11421 if (err) 11422 return err; 11423 11424 for_each_set_bit(bit, config.headers, __MLXSW_REG_RECR2_HEADER_CNT) 11425 mlxsw_reg_recr2_outer_header_enables_set(recr2_pl, bit, 1); 11426 for_each_set_bit(bit, config.fields, __MLXSW_REG_RECR2_FIELD_CNT) 11427 mlxsw_reg_recr2_outer_header_fields_enable_set(recr2_pl, bit, 1); 11428 for_each_set_bit(bit, config.inner_headers, __MLXSW_REG_RECR2_HEADER_CNT) 11429 mlxsw_reg_recr2_inner_header_enables_set(recr2_pl, bit, 1); 11430 for_each_set_bit(bit, config.inner_fields, __MLXSW_REG_RECR2_INNER_FIELD_CNT) 11431 mlxsw_reg_recr2_inner_header_fields_enable_set(recr2_pl, bit, 1); 11432 11433 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(recr2), recr2_pl); 11434 if (err) 11435 goto err_reg_write; 11436 11437 return 0; 11438 11439 err_reg_write: 11440 mlxsw_sp_mp_hash_parsing_depth_adjust(mlxsw_sp, new_inc_parsing_depth, 11441 old_inc_parsing_depth); 11442 return err; 11443 } 11444 11445 static void mlxsw_sp_mp_hash_fini(struct mlxsw_sp *mlxsw_sp) 11446 { 11447 bool old_inc_parsing_depth = mlxsw_sp->router->inc_parsing_depth; 11448 11449 mlxsw_sp_mp_hash_parsing_depth_adjust(mlxsw_sp, old_inc_parsing_depth, 11450 false); 11451 } 11452 #else 11453 static int mlxsw_sp_mp_hash_init(struct mlxsw_sp *mlxsw_sp) 11454 { 11455 return 0; 11456 } 11457 11458 static void mlxsw_sp_mp_hash_fini(struct mlxsw_sp *mlxsw_sp) 11459 { 11460 } 11461 #endif 11462 11463 static int mlxsw_sp_dscp_init(struct mlxsw_sp *mlxsw_sp) 11464 { 11465 char rdpm_pl[MLXSW_REG_RDPM_LEN]; 11466 unsigned int i; 11467 11468 MLXSW_REG_ZERO(rdpm, rdpm_pl); 11469 11470 /* HW is determining switch priority based on DSCP-bits, but the 11471 * kernel is still doing that based on the ToS. Since there's a 11472 * mismatch in bits we need to make sure to translate the right 11473 * value ToS would observe, skipping the 2 least-significant ECN bits. 11474 */ 11475 for (i = 0; i < MLXSW_REG_RDPM_DSCP_ENTRY_REC_MAX_COUNT; i++) 11476 mlxsw_reg_rdpm_pack(rdpm_pl, i, rt_tos2priority(i << 2)); 11477 11478 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rdpm), rdpm_pl); 11479 } 11480 11481 static int __mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp) 11482 { 11483 struct net *net = mlxsw_sp_net(mlxsw_sp); 11484 char rgcr_pl[MLXSW_REG_RGCR_LEN]; 11485 u64 max_rifs; 11486 bool usp; 11487 11488 if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_RIFS)) 11489 return -EIO; 11490 max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); 11491 usp = READ_ONCE(net->ipv4.sysctl_ip_fwd_update_priority); 11492 11493 mlxsw_reg_rgcr_pack(rgcr_pl, true, true); 11494 mlxsw_reg_rgcr_max_router_interfaces_set(rgcr_pl, max_rifs); 11495 mlxsw_reg_rgcr_usp_set(rgcr_pl, usp); 11496 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rgcr), rgcr_pl); 11497 } 11498 11499 static void __mlxsw_sp_router_fini(struct mlxsw_sp *mlxsw_sp) 11500 { 11501 char rgcr_pl[MLXSW_REG_RGCR_LEN]; 11502 11503 mlxsw_reg_rgcr_pack(rgcr_pl, false, false); 11504 mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rgcr), rgcr_pl); 11505 } 11506 11507 static int mlxsw_sp_lb_rif_init(struct mlxsw_sp *mlxsw_sp, 11508 struct netlink_ext_ack *extack) 11509 { 11510 struct mlxsw_sp_router *router = mlxsw_sp->router; 11511 struct mlxsw_sp_rif *lb_rif; 11512 int err; 11513 11514 router->lb_crif = mlxsw_sp_crif_alloc(NULL); 11515 if (!router->lb_crif) 11516 return -ENOMEM; 11517 11518 /* Create a generic loopback RIF associated with the main table 11519 * (default VRF). Any table can be used, but the main table exists 11520 * anyway, so we do not waste resources. Loopback RIFs are usually 11521 * created with a NULL CRIF, but this RIF is used as a fallback RIF 11522 * for blackhole nexthops, and nexthops expect to have a valid CRIF. 11523 */ 11524 lb_rif = mlxsw_sp_ul_rif_get(mlxsw_sp, RT_TABLE_MAIN, router->lb_crif, 11525 extack); 11526 if (IS_ERR(lb_rif)) { 11527 err = PTR_ERR(lb_rif); 11528 goto err_ul_rif_get; 11529 } 11530 11531 return 0; 11532 11533 err_ul_rif_get: 11534 mlxsw_sp_crif_free(router->lb_crif); 11535 return err; 11536 } 11537 11538 static void mlxsw_sp_lb_rif_fini(struct mlxsw_sp *mlxsw_sp) 11539 { 11540 mlxsw_sp_ul_rif_put(mlxsw_sp->router->lb_crif->rif); 11541 mlxsw_sp_crif_free(mlxsw_sp->router->lb_crif); 11542 } 11543 11544 static int mlxsw_sp1_router_init(struct mlxsw_sp *mlxsw_sp) 11545 { 11546 size_t size_ranges_count = ARRAY_SIZE(mlxsw_sp1_adj_grp_size_ranges); 11547 11548 mlxsw_sp->router->rif_ops_arr = mlxsw_sp1_rif_ops_arr; 11549 mlxsw_sp->router->adj_grp_size_ranges = mlxsw_sp1_adj_grp_size_ranges; 11550 mlxsw_sp->router->adj_grp_size_ranges_count = size_ranges_count; 11551 11552 return 0; 11553 } 11554 11555 const struct mlxsw_sp_router_ops mlxsw_sp1_router_ops = { 11556 .init = mlxsw_sp1_router_init, 11557 .ipips_init = mlxsw_sp1_ipips_init, 11558 }; 11559 11560 static int mlxsw_sp2_router_init(struct mlxsw_sp *mlxsw_sp) 11561 { 11562 size_t size_ranges_count = ARRAY_SIZE(mlxsw_sp2_adj_grp_size_ranges); 11563 11564 mlxsw_sp->router->rif_ops_arr = mlxsw_sp2_rif_ops_arr; 11565 mlxsw_sp->router->adj_grp_size_ranges = mlxsw_sp2_adj_grp_size_ranges; 11566 mlxsw_sp->router->adj_grp_size_ranges_count = size_ranges_count; 11567 11568 return 0; 11569 } 11570 11571 const struct mlxsw_sp_router_ops mlxsw_sp2_router_ops = { 11572 .init = mlxsw_sp2_router_init, 11573 .ipips_init = mlxsw_sp2_ipips_init, 11574 }; 11575 11576 int mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp, 11577 struct netlink_ext_ack *extack) 11578 { 11579 struct mlxsw_sp_router *router; 11580 struct notifier_block *nb; 11581 int err; 11582 11583 router = kzalloc_obj(*mlxsw_sp->router); 11584 if (!router) 11585 return -ENOMEM; 11586 mutex_init(&router->lock); 11587 mlxsw_sp->router = router; 11588 router->mlxsw_sp = mlxsw_sp; 11589 11590 err = mlxsw_sp->router_ops->init(mlxsw_sp); 11591 if (err) 11592 goto err_router_ops_init; 11593 11594 INIT_LIST_HEAD(&mlxsw_sp->router->nh_res_grp_list); 11595 INIT_DELAYED_WORK(&mlxsw_sp->router->nh_grp_activity_dw, 11596 mlxsw_sp_nh_grp_activity_work); 11597 INIT_LIST_HEAD(&mlxsw_sp->router->nexthop_neighs_list); 11598 err = __mlxsw_sp_router_init(mlxsw_sp); 11599 if (err) 11600 goto err_router_init; 11601 11602 err = mlxsw_sp->router_ops->ipips_init(mlxsw_sp); 11603 if (err) 11604 goto err_ipips_init; 11605 11606 err = rhashtable_init(&mlxsw_sp->router->crif_ht, 11607 &mlxsw_sp_crif_ht_params); 11608 if (err) 11609 goto err_crif_ht_init; 11610 11611 err = mlxsw_sp_rifs_init(mlxsw_sp); 11612 if (err) 11613 goto err_rifs_init; 11614 11615 err = rhashtable_init(&mlxsw_sp->router->nexthop_ht, 11616 &mlxsw_sp_nexthop_ht_params); 11617 if (err) 11618 goto err_nexthop_ht_init; 11619 11620 err = rhashtable_init(&mlxsw_sp->router->nexthop_group_ht, 11621 &mlxsw_sp_nexthop_group_ht_params); 11622 if (err) 11623 goto err_nexthop_group_ht_init; 11624 11625 INIT_LIST_HEAD(&mlxsw_sp->router->nexthop_list); 11626 err = mlxsw_sp_lpm_init(mlxsw_sp); 11627 if (err) 11628 goto err_lpm_init; 11629 11630 err = mlxsw_sp_mr_init(mlxsw_sp, &mlxsw_sp_mr_tcam_ops); 11631 if (err) 11632 goto err_mr_init; 11633 11634 err = mlxsw_sp_vrs_init(mlxsw_sp); 11635 if (err) 11636 goto err_vrs_init; 11637 11638 err = mlxsw_sp_lb_rif_init(mlxsw_sp, extack); 11639 if (err) 11640 goto err_lb_rif_init; 11641 11642 err = mlxsw_sp_neigh_init(mlxsw_sp); 11643 if (err) 11644 goto err_neigh_init; 11645 11646 err = mlxsw_sp_mp_hash_init(mlxsw_sp); 11647 if (err) 11648 goto err_mp_hash_init; 11649 11650 err = mlxsw_sp_dscp_init(mlxsw_sp); 11651 if (err) 11652 goto err_dscp_init; 11653 11654 router->inetaddr_nb.notifier_call = mlxsw_sp_inetaddr_event; 11655 err = register_inetaddr_notifier(&router->inetaddr_nb); 11656 if (err) 11657 goto err_register_inetaddr_notifier; 11658 11659 router->inet6addr_nb.notifier_call = mlxsw_sp_inet6addr_event; 11660 err = register_inet6addr_notifier(&router->inet6addr_nb); 11661 if (err) 11662 goto err_register_inet6addr_notifier; 11663 11664 router->inetaddr_valid_nb.notifier_call = mlxsw_sp_inetaddr_valid_event; 11665 err = register_inetaddr_validator_notifier(&router->inetaddr_valid_nb); 11666 if (err) 11667 goto err_register_inetaddr_valid_notifier; 11668 11669 nb = &router->inet6addr_valid_nb; 11670 nb->notifier_call = mlxsw_sp_inet6addr_valid_event; 11671 err = register_inet6addr_validator_notifier(nb); 11672 if (err) 11673 goto err_register_inet6addr_valid_notifier; 11674 11675 mlxsw_sp->router->netevent_nb.notifier_call = 11676 mlxsw_sp_router_netevent_event; 11677 err = register_netevent_notifier(&mlxsw_sp->router->netevent_nb); 11678 if (err) 11679 goto err_register_netevent_notifier; 11680 11681 mlxsw_sp->router->netdevice_nb.notifier_call = 11682 mlxsw_sp_router_netdevice_event; 11683 err = register_netdevice_notifier_net(mlxsw_sp_net(mlxsw_sp), 11684 &mlxsw_sp->router->netdevice_nb); 11685 if (err) 11686 goto err_register_netdev_notifier; 11687 11688 mlxsw_sp->router->nexthop_nb.notifier_call = 11689 mlxsw_sp_nexthop_obj_event; 11690 err = register_nexthop_notifier(mlxsw_sp_net(mlxsw_sp), 11691 &mlxsw_sp->router->nexthop_nb, 11692 extack); 11693 if (err) 11694 goto err_register_nexthop_notifier; 11695 11696 mlxsw_sp->router->fib_nb.notifier_call = mlxsw_sp_router_fib_event; 11697 err = register_fib_notifier(mlxsw_sp_net(mlxsw_sp), 11698 &mlxsw_sp->router->fib_nb, 11699 mlxsw_sp_router_fib_dump_flush, extack); 11700 if (err) 11701 goto err_register_fib_notifier; 11702 11703 return 0; 11704 11705 err_register_fib_notifier: 11706 unregister_nexthop_notifier(mlxsw_sp_net(mlxsw_sp), 11707 &mlxsw_sp->router->nexthop_nb); 11708 err_register_nexthop_notifier: 11709 unregister_netdevice_notifier_net(mlxsw_sp_net(mlxsw_sp), 11710 &router->netdevice_nb); 11711 err_register_netdev_notifier: 11712 unregister_netevent_notifier(&mlxsw_sp->router->netevent_nb); 11713 err_register_netevent_notifier: 11714 unregister_inet6addr_validator_notifier(&router->inet6addr_valid_nb); 11715 err_register_inet6addr_valid_notifier: 11716 unregister_inetaddr_validator_notifier(&router->inetaddr_valid_nb); 11717 err_register_inetaddr_valid_notifier: 11718 unregister_inet6addr_notifier(&router->inet6addr_nb); 11719 err_register_inet6addr_notifier: 11720 unregister_inetaddr_notifier(&router->inetaddr_nb); 11721 err_register_inetaddr_notifier: 11722 mlxsw_core_flush_owq(); 11723 err_dscp_init: 11724 mlxsw_sp_mp_hash_fini(mlxsw_sp); 11725 err_mp_hash_init: 11726 mlxsw_sp_neigh_fini(mlxsw_sp); 11727 err_neigh_init: 11728 mlxsw_sp_lb_rif_fini(mlxsw_sp); 11729 err_lb_rif_init: 11730 mlxsw_sp_vrs_fini(mlxsw_sp); 11731 err_vrs_init: 11732 mlxsw_sp_mr_fini(mlxsw_sp); 11733 err_mr_init: 11734 mlxsw_sp_lpm_fini(mlxsw_sp); 11735 err_lpm_init: 11736 rhashtable_destroy(&mlxsw_sp->router->nexthop_group_ht); 11737 err_nexthop_group_ht_init: 11738 rhashtable_destroy(&mlxsw_sp->router->nexthop_ht); 11739 err_nexthop_ht_init: 11740 mlxsw_sp_rifs_fini(mlxsw_sp); 11741 err_rifs_init: 11742 rhashtable_destroy(&mlxsw_sp->router->crif_ht); 11743 err_crif_ht_init: 11744 mlxsw_sp_ipips_fini(mlxsw_sp); 11745 err_ipips_init: 11746 __mlxsw_sp_router_fini(mlxsw_sp); 11747 err_router_init: 11748 cancel_delayed_work_sync(&mlxsw_sp->router->nh_grp_activity_dw); 11749 err_router_ops_init: 11750 mutex_destroy(&mlxsw_sp->router->lock); 11751 kfree(mlxsw_sp->router); 11752 return err; 11753 } 11754 11755 void mlxsw_sp_router_fini(struct mlxsw_sp *mlxsw_sp) 11756 { 11757 struct mlxsw_sp_router *router = mlxsw_sp->router; 11758 11759 unregister_fib_notifier(mlxsw_sp_net(mlxsw_sp), &router->fib_nb); 11760 unregister_nexthop_notifier(mlxsw_sp_net(mlxsw_sp), 11761 &router->nexthop_nb); 11762 unregister_netdevice_notifier_net(mlxsw_sp_net(mlxsw_sp), 11763 &router->netdevice_nb); 11764 unregister_netevent_notifier(&router->netevent_nb); 11765 unregister_inet6addr_validator_notifier(&router->inet6addr_valid_nb); 11766 unregister_inetaddr_validator_notifier(&router->inetaddr_valid_nb); 11767 unregister_inet6addr_notifier(&router->inet6addr_nb); 11768 unregister_inetaddr_notifier(&router->inetaddr_nb); 11769 mlxsw_core_flush_owq(); 11770 mlxsw_sp_mp_hash_fini(mlxsw_sp); 11771 mlxsw_sp_neigh_fini(mlxsw_sp); 11772 mlxsw_sp_lb_rif_fini(mlxsw_sp); 11773 mlxsw_sp_vrs_fini(mlxsw_sp); 11774 mlxsw_sp_mr_fini(mlxsw_sp); 11775 mlxsw_sp_lpm_fini(mlxsw_sp); 11776 rhashtable_destroy(&router->nexthop_group_ht); 11777 rhashtable_destroy(&router->nexthop_ht); 11778 mlxsw_sp_rifs_fini(mlxsw_sp); 11779 rhashtable_destroy(&mlxsw_sp->router->crif_ht); 11780 mlxsw_sp_ipips_fini(mlxsw_sp); 11781 __mlxsw_sp_router_fini(mlxsw_sp); 11782 cancel_delayed_work_sync(&router->nh_grp_activity_dw); 11783 mutex_destroy(&router->lock); 11784 kfree(router); 11785 } 11786