1 // SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0 2 /* Copyright (c) 2018 Mellanox Technologies. All rights reserved */ 3 4 #include <linux/kernel.h> 5 #include <linux/err.h> 6 #include <linux/errno.h> 7 #include <linux/gfp.h> 8 #include <linux/refcount.h> 9 #include <linux/rhashtable.h> 10 #define CREATE_TRACE_POINTS 11 #include <trace/events/mlxsw.h> 12 13 #include "reg.h" 14 #include "core.h" 15 #include "spectrum.h" 16 #include "spectrum_acl_tcam.h" 17 #include "core_acl_flex_keys.h" 18 19 #define MLXSW_SP_ACL_ATCAM_LKEY_ID_BLOCK_CLEAR_START 0 20 #define MLXSW_SP_ACL_ATCAM_LKEY_ID_BLOCK_CLEAR_END 5 21 22 struct mlxsw_sp_acl_atcam_lkey_id_ht_key { 23 char enc_key[MLXSW_REG_PTCEX_FLEX_KEY_BLOCKS_LEN]; /* MSB blocks */ 24 u8 erp_id; 25 }; 26 27 struct mlxsw_sp_acl_atcam_lkey_id { 28 struct rhash_head ht_node; 29 struct mlxsw_sp_acl_atcam_lkey_id_ht_key ht_key; 30 refcount_t refcnt; 31 u32 id; 32 }; 33 34 struct mlxsw_sp_acl_atcam_region_ops { 35 int (*init)(struct mlxsw_sp_acl_atcam_region *aregion); 36 void (*fini)(struct mlxsw_sp_acl_atcam_region *aregion); 37 struct mlxsw_sp_acl_atcam_lkey_id * 38 (*lkey_id_get)(struct mlxsw_sp_acl_atcam_region *aregion, 39 char *enc_key, u8 erp_id); 40 void (*lkey_id_put)(struct mlxsw_sp_acl_atcam_region *aregion, 41 struct mlxsw_sp_acl_atcam_lkey_id *lkey_id); 42 }; 43 44 struct mlxsw_sp_acl_atcam_region_generic { 45 struct mlxsw_sp_acl_atcam_lkey_id dummy_lkey_id; 46 }; 47 48 struct mlxsw_sp_acl_atcam_region_12kb { 49 struct rhashtable lkey_ht; 50 unsigned int max_lkey_id; 51 unsigned long *used_lkey_id; 52 }; 53 54 static const struct rhashtable_params mlxsw_sp_acl_atcam_lkey_id_ht_params = { 55 .key_len = sizeof(struct mlxsw_sp_acl_atcam_lkey_id_ht_key), 56 .key_offset = offsetof(struct mlxsw_sp_acl_atcam_lkey_id, ht_key), 57 .head_offset = offsetof(struct mlxsw_sp_acl_atcam_lkey_id, ht_node), 58 }; 59 60 static const struct rhashtable_params mlxsw_sp_acl_atcam_entries_ht_params = { 61 .key_len = sizeof(struct mlxsw_sp_acl_atcam_entry_ht_key), 62 .key_offset = offsetof(struct mlxsw_sp_acl_atcam_entry, ht_key), 63 .head_offset = offsetof(struct mlxsw_sp_acl_atcam_entry, ht_node), 64 }; 65 66 static bool 67 mlxsw_sp_acl_atcam_is_centry(const struct mlxsw_sp_acl_atcam_entry *aentry) 68 { 69 return mlxsw_sp_acl_erp_mask_is_ctcam(aentry->erp_mask); 70 } 71 72 static int 73 mlxsw_sp_acl_atcam_region_generic_init(struct mlxsw_sp_acl_atcam_region *aregion) 74 { 75 struct mlxsw_sp_acl_atcam_region_generic *region_generic; 76 77 region_generic = kzalloc(sizeof(*region_generic), GFP_KERNEL); 78 if (!region_generic) 79 return -ENOMEM; 80 81 refcount_set(®ion_generic->dummy_lkey_id.refcnt, 1); 82 aregion->priv = region_generic; 83 84 return 0; 85 } 86 87 static void 88 mlxsw_sp_acl_atcam_region_generic_fini(struct mlxsw_sp_acl_atcam_region *aregion) 89 { 90 kfree(aregion->priv); 91 } 92 93 static struct mlxsw_sp_acl_atcam_lkey_id * 94 mlxsw_sp_acl_atcam_generic_lkey_id_get(struct mlxsw_sp_acl_atcam_region *aregion, 95 char *enc_key, u8 erp_id) 96 { 97 struct mlxsw_sp_acl_atcam_region_generic *region_generic; 98 99 region_generic = aregion->priv; 100 return ®ion_generic->dummy_lkey_id; 101 } 102 103 static void 104 mlxsw_sp_acl_atcam_generic_lkey_id_put(struct mlxsw_sp_acl_atcam_region *aregion, 105 struct mlxsw_sp_acl_atcam_lkey_id *lkey_id) 106 { 107 } 108 109 static const struct mlxsw_sp_acl_atcam_region_ops 110 mlxsw_sp_acl_atcam_region_generic_ops = { 111 .init = mlxsw_sp_acl_atcam_region_generic_init, 112 .fini = mlxsw_sp_acl_atcam_region_generic_fini, 113 .lkey_id_get = mlxsw_sp_acl_atcam_generic_lkey_id_get, 114 .lkey_id_put = mlxsw_sp_acl_atcam_generic_lkey_id_put, 115 }; 116 117 static int 118 mlxsw_sp_acl_atcam_region_12kb_init(struct mlxsw_sp_acl_atcam_region *aregion) 119 { 120 struct mlxsw_sp *mlxsw_sp = aregion->region->mlxsw_sp; 121 struct mlxsw_sp_acl_atcam_region_12kb *region_12kb; 122 u64 max_lkey_id; 123 int err; 124 125 if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, ACL_MAX_LARGE_KEY_ID)) 126 return -EIO; 127 128 max_lkey_id = MLXSW_CORE_RES_GET(mlxsw_sp->core, ACL_MAX_LARGE_KEY_ID); 129 region_12kb = kzalloc(sizeof(*region_12kb), GFP_KERNEL); 130 if (!region_12kb) 131 return -ENOMEM; 132 133 region_12kb->used_lkey_id = bitmap_zalloc(max_lkey_id, GFP_KERNEL); 134 if (!region_12kb->used_lkey_id) { 135 err = -ENOMEM; 136 goto err_used_lkey_id_alloc; 137 } 138 139 err = rhashtable_init(®ion_12kb->lkey_ht, 140 &mlxsw_sp_acl_atcam_lkey_id_ht_params); 141 if (err) 142 goto err_rhashtable_init; 143 144 region_12kb->max_lkey_id = max_lkey_id; 145 aregion->priv = region_12kb; 146 147 return 0; 148 149 err_rhashtable_init: 150 bitmap_free(region_12kb->used_lkey_id); 151 err_used_lkey_id_alloc: 152 kfree(region_12kb); 153 return err; 154 } 155 156 static void 157 mlxsw_sp_acl_atcam_region_12kb_fini(struct mlxsw_sp_acl_atcam_region *aregion) 158 { 159 struct mlxsw_sp_acl_atcam_region_12kb *region_12kb = aregion->priv; 160 161 rhashtable_destroy(®ion_12kb->lkey_ht); 162 bitmap_free(region_12kb->used_lkey_id); 163 kfree(region_12kb); 164 } 165 166 static struct mlxsw_sp_acl_atcam_lkey_id * 167 mlxsw_sp_acl_atcam_lkey_id_create(struct mlxsw_sp_acl_atcam_region *aregion, 168 struct mlxsw_sp_acl_atcam_lkey_id_ht_key *ht_key) 169 { 170 struct mlxsw_sp_acl_atcam_region_12kb *region_12kb = aregion->priv; 171 struct mlxsw_sp_acl_atcam_lkey_id *lkey_id; 172 u32 id; 173 int err; 174 175 id = find_first_zero_bit(region_12kb->used_lkey_id, 176 region_12kb->max_lkey_id); 177 if (id < region_12kb->max_lkey_id) 178 __set_bit(id, region_12kb->used_lkey_id); 179 else 180 return ERR_PTR(-ENOBUFS); 181 182 lkey_id = kzalloc(sizeof(*lkey_id), GFP_KERNEL); 183 if (!lkey_id) { 184 err = -ENOMEM; 185 goto err_lkey_id_alloc; 186 } 187 188 lkey_id->id = id; 189 memcpy(&lkey_id->ht_key, ht_key, sizeof(*ht_key)); 190 refcount_set(&lkey_id->refcnt, 1); 191 192 err = rhashtable_insert_fast(®ion_12kb->lkey_ht, 193 &lkey_id->ht_node, 194 mlxsw_sp_acl_atcam_lkey_id_ht_params); 195 if (err) 196 goto err_rhashtable_insert; 197 198 return lkey_id; 199 200 err_rhashtable_insert: 201 kfree(lkey_id); 202 err_lkey_id_alloc: 203 __clear_bit(id, region_12kb->used_lkey_id); 204 return ERR_PTR(err); 205 } 206 207 static void 208 mlxsw_sp_acl_atcam_lkey_id_destroy(struct mlxsw_sp_acl_atcam_region *aregion, 209 struct mlxsw_sp_acl_atcam_lkey_id *lkey_id) 210 { 211 struct mlxsw_sp_acl_atcam_region_12kb *region_12kb = aregion->priv; 212 u32 id = lkey_id->id; 213 214 rhashtable_remove_fast(®ion_12kb->lkey_ht, &lkey_id->ht_node, 215 mlxsw_sp_acl_atcam_lkey_id_ht_params); 216 kfree(lkey_id); 217 __clear_bit(id, region_12kb->used_lkey_id); 218 } 219 220 static struct mlxsw_sp_acl_atcam_lkey_id * 221 mlxsw_sp_acl_atcam_12kb_lkey_id_get(struct mlxsw_sp_acl_atcam_region *aregion, 222 char *enc_key, u8 erp_id) 223 { 224 struct mlxsw_sp_acl_atcam_region_12kb *region_12kb = aregion->priv; 225 struct mlxsw_sp_acl_tcam_region *region = aregion->region; 226 struct mlxsw_sp_acl_atcam_lkey_id_ht_key ht_key = {{ 0 } }; 227 struct mlxsw_sp *mlxsw_sp = region->mlxsw_sp; 228 struct mlxsw_afk *afk = mlxsw_sp_acl_afk(mlxsw_sp->acl); 229 struct mlxsw_sp_acl_atcam_lkey_id *lkey_id; 230 231 memcpy(ht_key.enc_key, enc_key, sizeof(ht_key.enc_key)); 232 mlxsw_afk_clear(afk, ht_key.enc_key, 233 MLXSW_SP_ACL_ATCAM_LKEY_ID_BLOCK_CLEAR_START, 234 MLXSW_SP_ACL_ATCAM_LKEY_ID_BLOCK_CLEAR_END); 235 ht_key.erp_id = erp_id; 236 lkey_id = rhashtable_lookup_fast(®ion_12kb->lkey_ht, &ht_key, 237 mlxsw_sp_acl_atcam_lkey_id_ht_params); 238 if (lkey_id) { 239 refcount_inc(&lkey_id->refcnt); 240 return lkey_id; 241 } 242 243 return mlxsw_sp_acl_atcam_lkey_id_create(aregion, &ht_key); 244 } 245 246 static void 247 mlxsw_sp_acl_atcam_12kb_lkey_id_put(struct mlxsw_sp_acl_atcam_region *aregion, 248 struct mlxsw_sp_acl_atcam_lkey_id *lkey_id) 249 { 250 if (refcount_dec_and_test(&lkey_id->refcnt)) 251 mlxsw_sp_acl_atcam_lkey_id_destroy(aregion, lkey_id); 252 } 253 254 static const struct mlxsw_sp_acl_atcam_region_ops 255 mlxsw_sp_acl_atcam_region_12kb_ops = { 256 .init = mlxsw_sp_acl_atcam_region_12kb_init, 257 .fini = mlxsw_sp_acl_atcam_region_12kb_fini, 258 .lkey_id_get = mlxsw_sp_acl_atcam_12kb_lkey_id_get, 259 .lkey_id_put = mlxsw_sp_acl_atcam_12kb_lkey_id_put, 260 }; 261 262 static const struct mlxsw_sp_acl_atcam_region_ops * 263 mlxsw_sp_acl_atcam_region_ops_arr[] = { 264 [MLXSW_SP_ACL_ATCAM_REGION_TYPE_2KB] = 265 &mlxsw_sp_acl_atcam_region_generic_ops, 266 [MLXSW_SP_ACL_ATCAM_REGION_TYPE_4KB] = 267 &mlxsw_sp_acl_atcam_region_generic_ops, 268 [MLXSW_SP_ACL_ATCAM_REGION_TYPE_8KB] = 269 &mlxsw_sp_acl_atcam_region_generic_ops, 270 [MLXSW_SP_ACL_ATCAM_REGION_TYPE_12KB] = 271 &mlxsw_sp_acl_atcam_region_12kb_ops, 272 }; 273 274 int mlxsw_sp_acl_atcam_region_associate(struct mlxsw_sp *mlxsw_sp, 275 u16 region_id) 276 { 277 char perar_pl[MLXSW_REG_PERAR_LEN]; 278 /* For now, just assume that every region has 12 key blocks */ 279 u16 hw_region = region_id * 3; 280 u64 max_regions; 281 282 max_regions = MLXSW_CORE_RES_GET(mlxsw_sp->core, ACL_MAX_REGIONS); 283 if (hw_region >= max_regions) 284 return -ENOBUFS; 285 286 mlxsw_reg_perar_pack(perar_pl, region_id, hw_region); 287 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(perar), perar_pl); 288 } 289 290 static void 291 mlxsw_sp_acl_atcam_region_type_init(struct mlxsw_sp_acl_atcam_region *aregion) 292 { 293 struct mlxsw_sp_acl_tcam_region *region = aregion->region; 294 enum mlxsw_sp_acl_atcam_region_type region_type; 295 unsigned int blocks_count; 296 297 /* We already know the blocks count can not exceed the maximum 298 * blocks count. 299 */ 300 blocks_count = mlxsw_afk_key_info_blocks_count_get(region->key_info); 301 if (blocks_count <= 2) 302 region_type = MLXSW_SP_ACL_ATCAM_REGION_TYPE_2KB; 303 else if (blocks_count <= 4) 304 region_type = MLXSW_SP_ACL_ATCAM_REGION_TYPE_4KB; 305 else if (blocks_count <= 8) 306 region_type = MLXSW_SP_ACL_ATCAM_REGION_TYPE_8KB; 307 else 308 region_type = MLXSW_SP_ACL_ATCAM_REGION_TYPE_12KB; 309 310 aregion->type = region_type; 311 aregion->ops = mlxsw_sp_acl_atcam_region_ops_arr[region_type]; 312 } 313 314 int 315 mlxsw_sp_acl_atcam_region_init(struct mlxsw_sp *mlxsw_sp, 316 struct mlxsw_sp_acl_atcam *atcam, 317 struct mlxsw_sp_acl_atcam_region *aregion, 318 struct mlxsw_sp_acl_tcam_region *region, 319 void *hints_priv, 320 const struct mlxsw_sp_acl_ctcam_region_ops *ops) 321 { 322 int err; 323 324 aregion->region = region; 325 aregion->atcam = atcam; 326 mlxsw_sp_acl_atcam_region_type_init(aregion); 327 INIT_LIST_HEAD(&aregion->entries_list); 328 329 err = rhashtable_init(&aregion->entries_ht, 330 &mlxsw_sp_acl_atcam_entries_ht_params); 331 if (err) 332 return err; 333 err = aregion->ops->init(aregion); 334 if (err) 335 goto err_ops_init; 336 err = mlxsw_sp_acl_erp_region_init(aregion, hints_priv); 337 if (err) 338 goto err_erp_region_init; 339 err = mlxsw_sp_acl_ctcam_region_init(mlxsw_sp, &aregion->cregion, 340 region, ops); 341 if (err) 342 goto err_ctcam_region_init; 343 344 return 0; 345 346 err_ctcam_region_init: 347 mlxsw_sp_acl_erp_region_fini(aregion); 348 err_erp_region_init: 349 aregion->ops->fini(aregion); 350 err_ops_init: 351 rhashtable_destroy(&aregion->entries_ht); 352 return err; 353 } 354 355 void mlxsw_sp_acl_atcam_region_fini(struct mlxsw_sp_acl_atcam_region *aregion) 356 { 357 mlxsw_sp_acl_ctcam_region_fini(&aregion->cregion); 358 mlxsw_sp_acl_erp_region_fini(aregion); 359 aregion->ops->fini(aregion); 360 rhashtable_destroy(&aregion->entries_ht); 361 WARN_ON(!list_empty(&aregion->entries_list)); 362 } 363 364 void mlxsw_sp_acl_atcam_chunk_init(struct mlxsw_sp_acl_atcam_region *aregion, 365 struct mlxsw_sp_acl_atcam_chunk *achunk, 366 unsigned int priority) 367 { 368 mlxsw_sp_acl_ctcam_chunk_init(&aregion->cregion, &achunk->cchunk, 369 priority); 370 } 371 372 void mlxsw_sp_acl_atcam_chunk_fini(struct mlxsw_sp_acl_atcam_chunk *achunk) 373 { 374 mlxsw_sp_acl_ctcam_chunk_fini(&achunk->cchunk); 375 } 376 377 static int 378 mlxsw_sp_acl_atcam_region_entry_insert(struct mlxsw_sp *mlxsw_sp, 379 struct mlxsw_sp_acl_atcam_region *aregion, 380 struct mlxsw_sp_acl_atcam_entry *aentry, 381 struct mlxsw_sp_acl_rule_info *rulei) 382 { 383 struct mlxsw_sp_acl_tcam_region *region = aregion->region; 384 u8 erp_id = mlxsw_sp_acl_erp_mask_erp_id(aentry->erp_mask); 385 struct mlxsw_sp_acl_atcam_lkey_id *lkey_id; 386 char ptce3_pl[MLXSW_REG_PTCE3_LEN]; 387 u32 kvdl_index, priority; 388 int err; 389 390 err = mlxsw_sp_acl_tcam_priority_get(mlxsw_sp, rulei, &priority, true); 391 if (err) 392 return err; 393 394 lkey_id = aregion->ops->lkey_id_get(aregion, aentry->enc_key, erp_id); 395 if (IS_ERR(lkey_id)) 396 return PTR_ERR(lkey_id); 397 aentry->lkey_id = lkey_id; 398 399 kvdl_index = mlxsw_afa_block_first_kvdl_index(rulei->act_block); 400 mlxsw_reg_ptce3_pack(ptce3_pl, true, MLXSW_REG_PTCE3_OP_WRITE_WRITE, 401 priority, region->tcam_region_info, 402 aentry->enc_key, erp_id, 403 aentry->delta_info.start, 404 aentry->delta_info.mask, 405 aentry->delta_info.value, 406 refcount_read(&lkey_id->refcnt) != 1, lkey_id->id, 407 kvdl_index); 408 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ptce3), ptce3_pl); 409 if (err) 410 goto err_ptce3_write; 411 412 return 0; 413 414 err_ptce3_write: 415 aregion->ops->lkey_id_put(aregion, lkey_id); 416 return err; 417 } 418 419 static void 420 mlxsw_sp_acl_atcam_region_entry_remove(struct mlxsw_sp *mlxsw_sp, 421 struct mlxsw_sp_acl_atcam_region *aregion, 422 struct mlxsw_sp_acl_atcam_entry *aentry) 423 { 424 struct mlxsw_sp_acl_atcam_lkey_id *lkey_id = aentry->lkey_id; 425 struct mlxsw_sp_acl_tcam_region *region = aregion->region; 426 u8 erp_id = mlxsw_sp_acl_erp_mask_erp_id(aentry->erp_mask); 427 char ptce3_pl[MLXSW_REG_PTCE3_LEN]; 428 429 mlxsw_reg_ptce3_pack(ptce3_pl, false, MLXSW_REG_PTCE3_OP_WRITE_WRITE, 0, 430 region->tcam_region_info, 431 aentry->enc_key, erp_id, 432 aentry->delta_info.start, 433 aentry->delta_info.mask, 434 aentry->delta_info.value, 435 refcount_read(&lkey_id->refcnt) != 1, 436 lkey_id->id, 0); 437 mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ptce3), ptce3_pl); 438 aregion->ops->lkey_id_put(aregion, lkey_id); 439 } 440 441 static int 442 mlxsw_sp_acl_atcam_region_entry_action_replace(struct mlxsw_sp *mlxsw_sp, 443 struct mlxsw_sp_acl_atcam_region *aregion, 444 struct mlxsw_sp_acl_atcam_entry *aentry, 445 struct mlxsw_sp_acl_rule_info *rulei) 446 { 447 struct mlxsw_sp_acl_atcam_lkey_id *lkey_id = aentry->lkey_id; 448 u8 erp_id = mlxsw_sp_acl_erp_mask_erp_id(aentry->erp_mask); 449 struct mlxsw_sp_acl_tcam_region *region = aregion->region; 450 char ptce3_pl[MLXSW_REG_PTCE3_LEN]; 451 u32 kvdl_index, priority; 452 int err; 453 454 err = mlxsw_sp_acl_tcam_priority_get(mlxsw_sp, rulei, &priority, true); 455 if (err) 456 return err; 457 kvdl_index = mlxsw_afa_block_first_kvdl_index(rulei->act_block); 458 mlxsw_reg_ptce3_pack(ptce3_pl, true, MLXSW_REG_PTCE3_OP_WRITE_UPDATE, 459 priority, region->tcam_region_info, 460 aentry->enc_key, erp_id, 461 aentry->delta_info.start, 462 aentry->delta_info.mask, 463 aentry->delta_info.value, 464 refcount_read(&lkey_id->refcnt) != 1, lkey_id->id, 465 kvdl_index); 466 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ptce3), ptce3_pl); 467 } 468 469 static int 470 __mlxsw_sp_acl_atcam_entry_add(struct mlxsw_sp *mlxsw_sp, 471 struct mlxsw_sp_acl_atcam_region *aregion, 472 struct mlxsw_sp_acl_atcam_entry *aentry, 473 struct mlxsw_sp_acl_rule_info *rulei) 474 { 475 struct mlxsw_sp_acl_tcam_region *region = aregion->region; 476 char mask[MLXSW_REG_PTCEX_FLEX_KEY_BLOCKS_LEN] = { 0 }; 477 struct mlxsw_afk *afk = mlxsw_sp_acl_afk(mlxsw_sp->acl); 478 const struct mlxsw_sp_acl_erp_delta *delta; 479 struct mlxsw_sp_acl_erp_mask *erp_mask; 480 int err; 481 482 mlxsw_afk_encode(afk, region->key_info, &rulei->values, 483 aentry->ht_key.full_enc_key, mask); 484 485 erp_mask = mlxsw_sp_acl_erp_mask_get(aregion, mask, false); 486 if (IS_ERR(erp_mask)) 487 return PTR_ERR(erp_mask); 488 aentry->erp_mask = erp_mask; 489 aentry->ht_key.erp_id = mlxsw_sp_acl_erp_mask_erp_id(erp_mask); 490 memcpy(aentry->enc_key, aentry->ht_key.full_enc_key, 491 sizeof(aentry->enc_key)); 492 493 /* Compute all needed delta information and clear the delta bits 494 * from the encrypted key. 495 */ 496 delta = mlxsw_sp_acl_erp_delta(aentry->erp_mask); 497 aentry->delta_info.start = mlxsw_sp_acl_erp_delta_start(delta); 498 aentry->delta_info.mask = mlxsw_sp_acl_erp_delta_mask(delta); 499 aentry->delta_info.value = 500 mlxsw_sp_acl_erp_delta_value(delta, 501 aentry->ht_key.full_enc_key); 502 mlxsw_sp_acl_erp_delta_clear(delta, aentry->enc_key); 503 504 /* Add rule to the list of A-TCAM rules, assuming this 505 * rule is intended to A-TCAM. In case this rule does 506 * not fit into A-TCAM it will be removed from the list. 507 */ 508 list_add(&aentry->list, &aregion->entries_list); 509 510 /* We can't insert identical rules into the A-TCAM, so fail and 511 * let the rule spill into C-TCAM 512 */ 513 err = rhashtable_lookup_insert_fast(&aregion->entries_ht, 514 &aentry->ht_node, 515 mlxsw_sp_acl_atcam_entries_ht_params); 516 if (err) 517 goto err_rhashtable_insert; 518 519 /* Bloom filter must be updated here, before inserting the rule into 520 * the A-TCAM. 521 */ 522 err = mlxsw_sp_acl_erp_bf_insert(mlxsw_sp, aregion, erp_mask, aentry); 523 if (err) 524 goto err_bf_insert; 525 526 err = mlxsw_sp_acl_atcam_region_entry_insert(mlxsw_sp, aregion, aentry, 527 rulei); 528 if (err) 529 goto err_rule_insert; 530 531 return 0; 532 533 err_rule_insert: 534 mlxsw_sp_acl_erp_bf_remove(mlxsw_sp, aregion, erp_mask, aentry); 535 err_bf_insert: 536 rhashtable_remove_fast(&aregion->entries_ht, &aentry->ht_node, 537 mlxsw_sp_acl_atcam_entries_ht_params); 538 err_rhashtable_insert: 539 list_del(&aentry->list); 540 mlxsw_sp_acl_erp_mask_put(aregion, erp_mask); 541 return err; 542 } 543 544 static void 545 __mlxsw_sp_acl_atcam_entry_del(struct mlxsw_sp *mlxsw_sp, 546 struct mlxsw_sp_acl_atcam_region *aregion, 547 struct mlxsw_sp_acl_atcam_entry *aentry) 548 { 549 mlxsw_sp_acl_atcam_region_entry_remove(mlxsw_sp, aregion, aentry); 550 mlxsw_sp_acl_erp_bf_remove(mlxsw_sp, aregion, aentry->erp_mask, aentry); 551 rhashtable_remove_fast(&aregion->entries_ht, &aentry->ht_node, 552 mlxsw_sp_acl_atcam_entries_ht_params); 553 list_del(&aentry->list); 554 mlxsw_sp_acl_erp_mask_put(aregion, aentry->erp_mask); 555 } 556 557 static int 558 __mlxsw_sp_acl_atcam_entry_action_replace(struct mlxsw_sp *mlxsw_sp, 559 struct mlxsw_sp_acl_atcam_region *aregion, 560 struct mlxsw_sp_acl_atcam_entry *aentry, 561 struct mlxsw_sp_acl_rule_info *rulei) 562 { 563 return mlxsw_sp_acl_atcam_region_entry_action_replace(mlxsw_sp, aregion, 564 aentry, rulei); 565 } 566 567 int mlxsw_sp_acl_atcam_entry_add(struct mlxsw_sp *mlxsw_sp, 568 struct mlxsw_sp_acl_atcam_region *aregion, 569 struct mlxsw_sp_acl_atcam_chunk *achunk, 570 struct mlxsw_sp_acl_atcam_entry *aentry, 571 struct mlxsw_sp_acl_rule_info *rulei) 572 { 573 int err; 574 575 err = __mlxsw_sp_acl_atcam_entry_add(mlxsw_sp, aregion, aentry, rulei); 576 if (!err) 577 return 0; 578 579 /* It is possible we failed to add the rule to the A-TCAM due to 580 * exceeded number of masks. Try to spill into C-TCAM. 581 */ 582 trace_mlxsw_sp_acl_atcam_entry_add_ctcam_spill(mlxsw_sp, aregion); 583 err = mlxsw_sp_acl_ctcam_entry_add(mlxsw_sp, &aregion->cregion, 584 &achunk->cchunk, &aentry->centry, 585 rulei, true); 586 if (!err) 587 return 0; 588 589 return err; 590 } 591 592 void mlxsw_sp_acl_atcam_entry_del(struct mlxsw_sp *mlxsw_sp, 593 struct mlxsw_sp_acl_atcam_region *aregion, 594 struct mlxsw_sp_acl_atcam_chunk *achunk, 595 struct mlxsw_sp_acl_atcam_entry *aentry) 596 { 597 if (mlxsw_sp_acl_atcam_is_centry(aentry)) 598 mlxsw_sp_acl_ctcam_entry_del(mlxsw_sp, &aregion->cregion, 599 &achunk->cchunk, &aentry->centry); 600 else 601 __mlxsw_sp_acl_atcam_entry_del(mlxsw_sp, aregion, aentry); 602 } 603 604 int 605 mlxsw_sp_acl_atcam_entry_action_replace(struct mlxsw_sp *mlxsw_sp, 606 struct mlxsw_sp_acl_atcam_region *aregion, 607 struct mlxsw_sp_acl_atcam_entry *aentry, 608 struct mlxsw_sp_acl_rule_info *rulei) 609 { 610 int err; 611 612 if (mlxsw_sp_acl_atcam_is_centry(aentry)) 613 err = mlxsw_sp_acl_ctcam_entry_action_replace(mlxsw_sp, 614 &aregion->cregion, 615 &aentry->centry, 616 rulei); 617 else 618 err = __mlxsw_sp_acl_atcam_entry_action_replace(mlxsw_sp, 619 aregion, aentry, 620 rulei); 621 622 return err; 623 } 624 625 int mlxsw_sp_acl_atcam_init(struct mlxsw_sp *mlxsw_sp, 626 struct mlxsw_sp_acl_atcam *atcam) 627 { 628 return mlxsw_sp_acl_erps_init(mlxsw_sp, atcam); 629 } 630 631 void mlxsw_sp_acl_atcam_fini(struct mlxsw_sp *mlxsw_sp, 632 struct mlxsw_sp_acl_atcam *atcam) 633 { 634 mlxsw_sp_acl_erps_fini(mlxsw_sp, atcam); 635 } 636 637 void * 638 mlxsw_sp_acl_atcam_rehash_hints_get(struct mlxsw_sp_acl_atcam_region *aregion) 639 { 640 return mlxsw_sp_acl_erp_rehash_hints_get(aregion); 641 } 642 643 void mlxsw_sp_acl_atcam_rehash_hints_put(void *hints_priv) 644 { 645 mlxsw_sp_acl_erp_rehash_hints_put(hints_priv); 646 } 647