1 // SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0 2 /* Copyright (c) 2018 Mellanox Technologies. All rights reserved */ 3 4 #include <linux/bitmap.h> 5 #include <linux/errno.h> 6 #include <linux/genalloc.h> 7 #include <linux/gfp.h> 8 #include <linux/kernel.h> 9 #include <linux/list.h> 10 #include <linux/mutex.h> 11 #include <linux/objagg.h> 12 #include <linux/rtnetlink.h> 13 #include <linux/slab.h> 14 15 #include "core.h" 16 #include "reg.h" 17 #include "spectrum.h" 18 #include "spectrum_acl_tcam.h" 19 20 /* gen_pool_alloc() returns 0 when allocation fails, so use an offset */ 21 #define MLXSW_SP_ACL_ERP_GENALLOC_OFFSET 0x100 22 #define MLXSW_SP_ACL_ERP_MAX_PER_REGION 16 23 24 struct mlxsw_sp_acl_erp_core { 25 unsigned int erpt_entries_size[MLXSW_SP_ACL_ATCAM_REGION_TYPE_MAX + 1]; 26 struct gen_pool *erp_tables; 27 struct mlxsw_sp *mlxsw_sp; 28 struct mlxsw_sp_acl_bf *bf; 29 unsigned int num_erp_banks; 30 }; 31 32 struct mlxsw_sp_acl_erp_key { 33 char mask[MLXSW_REG_PTCEX_FLEX_KEY_BLOCKS_LEN]; 34 #define __MASK_LEN 0x38 35 #define __MASK_IDX(i) (__MASK_LEN - (i) - 1) 36 bool ctcam; 37 }; 38 39 struct mlxsw_sp_acl_erp { 40 struct mlxsw_sp_acl_erp_key key; 41 u8 id; 42 u8 index; 43 DECLARE_BITMAP(mask_bitmap, MLXSW_SP_ACL_TCAM_MASK_LEN); 44 struct list_head list; 45 struct mlxsw_sp_acl_erp_table *erp_table; 46 }; 47 48 struct mlxsw_sp_acl_erp_master_mask { 49 DECLARE_BITMAP(bitmap, MLXSW_SP_ACL_TCAM_MASK_LEN); 50 unsigned int count[MLXSW_SP_ACL_TCAM_MASK_LEN]; 51 }; 52 53 struct mlxsw_sp_acl_erp_table { 54 struct mlxsw_sp_acl_erp_master_mask master_mask; 55 DECLARE_BITMAP(erp_id_bitmap, MLXSW_SP_ACL_ERP_MAX_PER_REGION); 56 DECLARE_BITMAP(erp_index_bitmap, MLXSW_SP_ACL_ERP_MAX_PER_REGION); 57 struct list_head atcam_erps_list; 58 struct mlxsw_sp_acl_erp_core *erp_core; 59 struct mlxsw_sp_acl_atcam_region *aregion; 60 const struct mlxsw_sp_acl_erp_table_ops *ops; 61 unsigned long base_index; 62 unsigned int num_atcam_erps; 63 unsigned int num_max_atcam_erps; 64 unsigned int num_ctcam_erps; 65 unsigned int num_deltas; 66 struct objagg *objagg; 67 struct mutex objagg_lock; /* guards objagg manipulation */ 68 }; 69 70 struct mlxsw_sp_acl_erp_table_ops { 71 struct mlxsw_sp_acl_erp * 72 (*erp_create)(struct mlxsw_sp_acl_erp_table *erp_table, 73 struct mlxsw_sp_acl_erp_key *key); 74 void (*erp_destroy)(struct mlxsw_sp_acl_erp_table *erp_table, 75 struct mlxsw_sp_acl_erp *erp); 76 }; 77 78 static struct mlxsw_sp_acl_erp * 79 mlxsw_sp_acl_erp_mask_create(struct mlxsw_sp_acl_erp_table *erp_table, 80 struct mlxsw_sp_acl_erp_key *key); 81 static void 82 mlxsw_sp_acl_erp_mask_destroy(struct mlxsw_sp_acl_erp_table *erp_table, 83 struct mlxsw_sp_acl_erp *erp); 84 static struct mlxsw_sp_acl_erp * 85 mlxsw_sp_acl_erp_second_mask_create(struct mlxsw_sp_acl_erp_table *erp_table, 86 struct mlxsw_sp_acl_erp_key *key); 87 static void 88 mlxsw_sp_acl_erp_second_mask_destroy(struct mlxsw_sp_acl_erp_table *erp_table, 89 struct mlxsw_sp_acl_erp *erp); 90 static struct mlxsw_sp_acl_erp * 91 mlxsw_sp_acl_erp_first_mask_create(struct mlxsw_sp_acl_erp_table *erp_table, 92 struct mlxsw_sp_acl_erp_key *key); 93 static void 94 mlxsw_sp_acl_erp_first_mask_destroy(struct mlxsw_sp_acl_erp_table *erp_table, 95 struct mlxsw_sp_acl_erp *erp); 96 static void 97 mlxsw_sp_acl_erp_no_mask_destroy(struct mlxsw_sp_acl_erp_table *erp_table, 98 struct mlxsw_sp_acl_erp *erp); 99 100 static const struct mlxsw_sp_acl_erp_table_ops erp_multiple_masks_ops = { 101 .erp_create = mlxsw_sp_acl_erp_mask_create, 102 .erp_destroy = mlxsw_sp_acl_erp_mask_destroy, 103 }; 104 105 static const struct mlxsw_sp_acl_erp_table_ops erp_two_masks_ops = { 106 .erp_create = mlxsw_sp_acl_erp_mask_create, 107 .erp_destroy = mlxsw_sp_acl_erp_second_mask_destroy, 108 }; 109 110 static const struct mlxsw_sp_acl_erp_table_ops erp_single_mask_ops = { 111 .erp_create = mlxsw_sp_acl_erp_second_mask_create, 112 .erp_destroy = mlxsw_sp_acl_erp_first_mask_destroy, 113 }; 114 115 static const struct mlxsw_sp_acl_erp_table_ops erp_no_mask_ops = { 116 .erp_create = mlxsw_sp_acl_erp_first_mask_create, 117 .erp_destroy = mlxsw_sp_acl_erp_no_mask_destroy, 118 }; 119 120 static bool 121 mlxsw_sp_acl_erp_table_is_used(const struct mlxsw_sp_acl_erp_table *erp_table) 122 { 123 return erp_table->ops != &erp_single_mask_ops && 124 erp_table->ops != &erp_no_mask_ops; 125 } 126 127 static unsigned int 128 mlxsw_sp_acl_erp_bank_get(const struct mlxsw_sp_acl_erp *erp) 129 { 130 return erp->index % erp->erp_table->erp_core->num_erp_banks; 131 } 132 133 static unsigned int 134 mlxsw_sp_acl_erp_table_entry_size(const struct mlxsw_sp_acl_erp_table *erp_table) 135 { 136 struct mlxsw_sp_acl_atcam_region *aregion = erp_table->aregion; 137 struct mlxsw_sp_acl_erp_core *erp_core = erp_table->erp_core; 138 139 return erp_core->erpt_entries_size[aregion->type]; 140 } 141 142 static int mlxsw_sp_acl_erp_id_get(struct mlxsw_sp_acl_erp_table *erp_table, 143 u8 *p_id) 144 { 145 u8 id; 146 147 id = find_first_zero_bit(erp_table->erp_id_bitmap, 148 MLXSW_SP_ACL_ERP_MAX_PER_REGION); 149 if (id < MLXSW_SP_ACL_ERP_MAX_PER_REGION) { 150 __set_bit(id, erp_table->erp_id_bitmap); 151 *p_id = id; 152 return 0; 153 } 154 155 return -ENOBUFS; 156 } 157 158 static void mlxsw_sp_acl_erp_id_put(struct mlxsw_sp_acl_erp_table *erp_table, 159 u8 id) 160 { 161 __clear_bit(id, erp_table->erp_id_bitmap); 162 } 163 164 static void 165 mlxsw_sp_acl_erp_master_mask_bit_set(unsigned long bit, 166 struct mlxsw_sp_acl_erp_master_mask *mask) 167 { 168 if (mask->count[bit]++ == 0) 169 __set_bit(bit, mask->bitmap); 170 } 171 172 static void 173 mlxsw_sp_acl_erp_master_mask_bit_clear(unsigned long bit, 174 struct mlxsw_sp_acl_erp_master_mask *mask) 175 { 176 if (--mask->count[bit] == 0) 177 __clear_bit(bit, mask->bitmap); 178 } 179 180 static int 181 mlxsw_sp_acl_erp_master_mask_update(struct mlxsw_sp_acl_erp_table *erp_table) 182 { 183 struct mlxsw_sp_acl_tcam_region *region = erp_table->aregion->region; 184 struct mlxsw_sp *mlxsw_sp = region->mlxsw_sp; 185 char percr_pl[MLXSW_REG_PERCR_LEN]; 186 char *master_mask; 187 188 mlxsw_reg_percr_pack(percr_pl, region->id); 189 master_mask = mlxsw_reg_percr_master_mask_data(percr_pl); 190 bitmap_to_arr32((u32 *) master_mask, erp_table->master_mask.bitmap, 191 MLXSW_SP_ACL_TCAM_MASK_LEN); 192 193 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(percr), percr_pl); 194 } 195 196 static int 197 mlxsw_sp_acl_erp_master_mask_set(struct mlxsw_sp_acl_erp_table *erp_table, 198 struct mlxsw_sp_acl_erp_key *key) 199 { 200 DECLARE_BITMAP(mask_bitmap, MLXSW_SP_ACL_TCAM_MASK_LEN); 201 unsigned long bit; 202 int err; 203 204 bitmap_from_arr32(mask_bitmap, (u32 *) key->mask, 205 MLXSW_SP_ACL_TCAM_MASK_LEN); 206 for_each_set_bit(bit, mask_bitmap, MLXSW_SP_ACL_TCAM_MASK_LEN) 207 mlxsw_sp_acl_erp_master_mask_bit_set(bit, 208 &erp_table->master_mask); 209 210 err = mlxsw_sp_acl_erp_master_mask_update(erp_table); 211 if (err) 212 goto err_master_mask_update; 213 214 return 0; 215 216 err_master_mask_update: 217 for_each_set_bit(bit, mask_bitmap, MLXSW_SP_ACL_TCAM_MASK_LEN) 218 mlxsw_sp_acl_erp_master_mask_bit_clear(bit, 219 &erp_table->master_mask); 220 return err; 221 } 222 223 static int 224 mlxsw_sp_acl_erp_master_mask_clear(struct mlxsw_sp_acl_erp_table *erp_table, 225 struct mlxsw_sp_acl_erp_key *key) 226 { 227 DECLARE_BITMAP(mask_bitmap, MLXSW_SP_ACL_TCAM_MASK_LEN); 228 unsigned long bit; 229 int err; 230 231 bitmap_from_arr32(mask_bitmap, (u32 *) key->mask, 232 MLXSW_SP_ACL_TCAM_MASK_LEN); 233 for_each_set_bit(bit, mask_bitmap, MLXSW_SP_ACL_TCAM_MASK_LEN) 234 mlxsw_sp_acl_erp_master_mask_bit_clear(bit, 235 &erp_table->master_mask); 236 237 err = mlxsw_sp_acl_erp_master_mask_update(erp_table); 238 if (err) 239 goto err_master_mask_update; 240 241 return 0; 242 243 err_master_mask_update: 244 for_each_set_bit(bit, mask_bitmap, MLXSW_SP_ACL_TCAM_MASK_LEN) 245 mlxsw_sp_acl_erp_master_mask_bit_set(bit, 246 &erp_table->master_mask); 247 return err; 248 } 249 250 static struct mlxsw_sp_acl_erp * 251 mlxsw_sp_acl_erp_generic_create(struct mlxsw_sp_acl_erp_table *erp_table, 252 struct mlxsw_sp_acl_erp_key *key) 253 { 254 struct mlxsw_sp_acl_erp *erp; 255 int err; 256 257 erp = kzalloc(sizeof(*erp), GFP_KERNEL); 258 if (!erp) 259 return ERR_PTR(-ENOMEM); 260 261 err = mlxsw_sp_acl_erp_id_get(erp_table, &erp->id); 262 if (err) 263 goto err_erp_id_get; 264 265 memcpy(&erp->key, key, sizeof(*key)); 266 list_add(&erp->list, &erp_table->atcam_erps_list); 267 erp_table->num_atcam_erps++; 268 erp->erp_table = erp_table; 269 270 err = mlxsw_sp_acl_erp_master_mask_set(erp_table, &erp->key); 271 if (err) 272 goto err_master_mask_set; 273 274 return erp; 275 276 err_master_mask_set: 277 erp_table->num_atcam_erps--; 278 list_del(&erp->list); 279 mlxsw_sp_acl_erp_id_put(erp_table, erp->id); 280 err_erp_id_get: 281 kfree(erp); 282 return ERR_PTR(err); 283 } 284 285 static void 286 mlxsw_sp_acl_erp_generic_destroy(struct mlxsw_sp_acl_erp *erp) 287 { 288 struct mlxsw_sp_acl_erp_table *erp_table = erp->erp_table; 289 290 mlxsw_sp_acl_erp_master_mask_clear(erp_table, &erp->key); 291 erp_table->num_atcam_erps--; 292 list_del(&erp->list); 293 mlxsw_sp_acl_erp_id_put(erp_table, erp->id); 294 kfree(erp); 295 } 296 297 static int 298 mlxsw_sp_acl_erp_table_alloc(struct mlxsw_sp_acl_erp_core *erp_core, 299 unsigned int num_erps, 300 enum mlxsw_sp_acl_atcam_region_type region_type, 301 unsigned long *p_index) 302 { 303 unsigned int num_rows, entry_size; 304 unsigned long index; 305 306 /* We only allow allocations of entire rows */ 307 if (num_erps % erp_core->num_erp_banks != 0) 308 return -EINVAL; 309 310 entry_size = erp_core->erpt_entries_size[region_type]; 311 num_rows = num_erps / erp_core->num_erp_banks; 312 313 index = gen_pool_alloc(erp_core->erp_tables, num_rows * entry_size); 314 if (!index) 315 return -ENOBUFS; 316 317 *p_index = index - MLXSW_SP_ACL_ERP_GENALLOC_OFFSET; 318 319 return 0; 320 } 321 322 static void 323 mlxsw_sp_acl_erp_table_free(struct mlxsw_sp_acl_erp_core *erp_core, 324 unsigned int num_erps, 325 enum mlxsw_sp_acl_atcam_region_type region_type, 326 unsigned long index) 327 { 328 unsigned long base_index; 329 unsigned int entry_size; 330 size_t size; 331 332 entry_size = erp_core->erpt_entries_size[region_type]; 333 base_index = index + MLXSW_SP_ACL_ERP_GENALLOC_OFFSET; 334 size = num_erps / erp_core->num_erp_banks * entry_size; 335 gen_pool_free(erp_core->erp_tables, base_index, size); 336 } 337 338 static struct mlxsw_sp_acl_erp * 339 mlxsw_sp_acl_erp_table_master_rp(struct mlxsw_sp_acl_erp_table *erp_table) 340 { 341 if (!list_is_singular(&erp_table->atcam_erps_list)) 342 return NULL; 343 344 return list_first_entry(&erp_table->atcam_erps_list, 345 struct mlxsw_sp_acl_erp, list); 346 } 347 348 static int mlxsw_sp_acl_erp_index_get(struct mlxsw_sp_acl_erp_table *erp_table, 349 u8 *p_index) 350 { 351 u8 index; 352 353 index = find_first_zero_bit(erp_table->erp_index_bitmap, 354 erp_table->num_max_atcam_erps); 355 if (index < erp_table->num_max_atcam_erps) { 356 __set_bit(index, erp_table->erp_index_bitmap); 357 *p_index = index; 358 return 0; 359 } 360 361 return -ENOBUFS; 362 } 363 364 static void mlxsw_sp_acl_erp_index_put(struct mlxsw_sp_acl_erp_table *erp_table, 365 u8 index) 366 { 367 __clear_bit(index, erp_table->erp_index_bitmap); 368 } 369 370 static void 371 mlxsw_sp_acl_erp_table_locate(const struct mlxsw_sp_acl_erp_table *erp_table, 372 const struct mlxsw_sp_acl_erp *erp, 373 u8 *p_erpt_bank, u8 *p_erpt_index) 374 { 375 unsigned int entry_size = mlxsw_sp_acl_erp_table_entry_size(erp_table); 376 struct mlxsw_sp_acl_erp_core *erp_core = erp_table->erp_core; 377 unsigned int row; 378 379 *p_erpt_bank = erp->index % erp_core->num_erp_banks; 380 row = erp->index / erp_core->num_erp_banks; 381 *p_erpt_index = erp_table->base_index + row * entry_size; 382 } 383 384 static int 385 mlxsw_sp_acl_erp_table_erp_add(struct mlxsw_sp_acl_erp_table *erp_table, 386 struct mlxsw_sp_acl_erp *erp) 387 { 388 struct mlxsw_sp *mlxsw_sp = erp_table->erp_core->mlxsw_sp; 389 enum mlxsw_reg_perpt_key_size key_size; 390 char perpt_pl[MLXSW_REG_PERPT_LEN]; 391 u8 erpt_bank, erpt_index; 392 393 mlxsw_sp_acl_erp_table_locate(erp_table, erp, &erpt_bank, &erpt_index); 394 key_size = (enum mlxsw_reg_perpt_key_size) erp_table->aregion->type; 395 mlxsw_reg_perpt_pack(perpt_pl, erpt_bank, erpt_index, key_size, erp->id, 396 0, erp_table->base_index, erp->index, 397 erp->key.mask); 398 mlxsw_reg_perpt_erp_vector_pack(perpt_pl, erp_table->erp_index_bitmap, 399 MLXSW_SP_ACL_ERP_MAX_PER_REGION); 400 mlxsw_reg_perpt_erp_vector_set(perpt_pl, erp->index, true); 401 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(perpt), perpt_pl); 402 } 403 404 static void mlxsw_sp_acl_erp_table_erp_del(struct mlxsw_sp_acl_erp *erp) 405 { 406 char empty_mask[MLXSW_REG_PTCEX_FLEX_KEY_BLOCKS_LEN] = { 0 }; 407 struct mlxsw_sp_acl_erp_table *erp_table = erp->erp_table; 408 struct mlxsw_sp *mlxsw_sp = erp_table->erp_core->mlxsw_sp; 409 enum mlxsw_reg_perpt_key_size key_size; 410 char perpt_pl[MLXSW_REG_PERPT_LEN]; 411 u8 erpt_bank, erpt_index; 412 413 mlxsw_sp_acl_erp_table_locate(erp_table, erp, &erpt_bank, &erpt_index); 414 key_size = (enum mlxsw_reg_perpt_key_size) erp_table->aregion->type; 415 mlxsw_reg_perpt_pack(perpt_pl, erpt_bank, erpt_index, key_size, erp->id, 416 0, erp_table->base_index, erp->index, empty_mask); 417 mlxsw_reg_perpt_erp_vector_pack(perpt_pl, erp_table->erp_index_bitmap, 418 MLXSW_SP_ACL_ERP_MAX_PER_REGION); 419 mlxsw_reg_perpt_erp_vector_set(perpt_pl, erp->index, false); 420 mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(perpt), perpt_pl); 421 } 422 423 static int 424 mlxsw_sp_acl_erp_table_enable(struct mlxsw_sp_acl_erp_table *erp_table, 425 bool ctcam_le) 426 { 427 struct mlxsw_sp_acl_tcam_region *region = erp_table->aregion->region; 428 struct mlxsw_sp *mlxsw_sp = erp_table->erp_core->mlxsw_sp; 429 char pererp_pl[MLXSW_REG_PERERP_LEN]; 430 431 mlxsw_reg_pererp_pack(pererp_pl, region->id, ctcam_le, true, 0, 432 erp_table->base_index, 0); 433 mlxsw_reg_pererp_erp_vector_pack(pererp_pl, erp_table->erp_index_bitmap, 434 MLXSW_SP_ACL_ERP_MAX_PER_REGION); 435 436 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(pererp), pererp_pl); 437 } 438 439 static void 440 mlxsw_sp_acl_erp_table_disable(struct mlxsw_sp_acl_erp_table *erp_table) 441 { 442 struct mlxsw_sp_acl_tcam_region *region = erp_table->aregion->region; 443 struct mlxsw_sp *mlxsw_sp = erp_table->erp_core->mlxsw_sp; 444 char pererp_pl[MLXSW_REG_PERERP_LEN]; 445 struct mlxsw_sp_acl_erp *master_rp; 446 447 master_rp = mlxsw_sp_acl_erp_table_master_rp(erp_table); 448 /* It is possible we do not have a master RP when we disable the 449 * table when there are no rules in the A-TCAM and the last C-TCAM 450 * rule is deleted 451 */ 452 mlxsw_reg_pererp_pack(pererp_pl, region->id, false, false, 0, 0, 453 master_rp ? master_rp->id : 0); 454 mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(pererp), pererp_pl); 455 } 456 457 static int 458 mlxsw_sp_acl_erp_table_relocate(struct mlxsw_sp_acl_erp_table *erp_table) 459 { 460 struct mlxsw_sp_acl_erp *erp; 461 int err; 462 463 list_for_each_entry(erp, &erp_table->atcam_erps_list, list) { 464 err = mlxsw_sp_acl_erp_table_erp_add(erp_table, erp); 465 if (err) 466 goto err_table_erp_add; 467 } 468 469 return 0; 470 471 err_table_erp_add: 472 list_for_each_entry_continue_reverse(erp, &erp_table->atcam_erps_list, 473 list) 474 mlxsw_sp_acl_erp_table_erp_del(erp); 475 return err; 476 } 477 478 static int 479 mlxsw_sp_acl_erp_table_expand(struct mlxsw_sp_acl_erp_table *erp_table) 480 { 481 unsigned int num_erps, old_num_erps = erp_table->num_max_atcam_erps; 482 struct mlxsw_sp_acl_erp_core *erp_core = erp_table->erp_core; 483 unsigned long old_base_index = erp_table->base_index; 484 bool ctcam_le = erp_table->num_ctcam_erps > 0; 485 int err; 486 487 if (erp_table->num_atcam_erps < erp_table->num_max_atcam_erps) 488 return 0; 489 490 if (erp_table->num_max_atcam_erps == MLXSW_SP_ACL_ERP_MAX_PER_REGION) 491 return -ENOBUFS; 492 493 num_erps = old_num_erps + erp_core->num_erp_banks; 494 err = mlxsw_sp_acl_erp_table_alloc(erp_core, num_erps, 495 erp_table->aregion->type, 496 &erp_table->base_index); 497 if (err) 498 return err; 499 erp_table->num_max_atcam_erps = num_erps; 500 501 err = mlxsw_sp_acl_erp_table_relocate(erp_table); 502 if (err) 503 goto err_table_relocate; 504 505 err = mlxsw_sp_acl_erp_table_enable(erp_table, ctcam_le); 506 if (err) 507 goto err_table_enable; 508 509 mlxsw_sp_acl_erp_table_free(erp_core, old_num_erps, 510 erp_table->aregion->type, old_base_index); 511 512 return 0; 513 514 err_table_enable: 515 err_table_relocate: 516 erp_table->num_max_atcam_erps = old_num_erps; 517 mlxsw_sp_acl_erp_table_free(erp_core, num_erps, 518 erp_table->aregion->type, 519 erp_table->base_index); 520 erp_table->base_index = old_base_index; 521 return err; 522 } 523 524 static int 525 mlxsw_acl_erp_table_bf_add(struct mlxsw_sp_acl_erp_table *erp_table, 526 struct mlxsw_sp_acl_erp *erp) 527 { 528 struct mlxsw_sp_acl_atcam_region *aregion = erp_table->aregion; 529 unsigned int erp_bank = mlxsw_sp_acl_erp_bank_get(erp); 530 struct mlxsw_sp_acl_atcam_entry *aentry; 531 int err; 532 533 list_for_each_entry(aentry, &aregion->entries_list, list) { 534 err = mlxsw_sp_acl_bf_entry_add(aregion->region->mlxsw_sp, 535 erp_table->erp_core->bf, 536 aregion, erp_bank, aentry); 537 if (err) 538 goto bf_entry_add_err; 539 } 540 541 return 0; 542 543 bf_entry_add_err: 544 list_for_each_entry_continue_reverse(aentry, &aregion->entries_list, 545 list) 546 mlxsw_sp_acl_bf_entry_del(aregion->region->mlxsw_sp, 547 erp_table->erp_core->bf, 548 aregion, erp_bank, aentry); 549 return err; 550 } 551 552 static void 553 mlxsw_acl_erp_table_bf_del(struct mlxsw_sp_acl_erp_table *erp_table, 554 struct mlxsw_sp_acl_erp *erp) 555 { 556 struct mlxsw_sp_acl_atcam_region *aregion = erp_table->aregion; 557 unsigned int erp_bank = mlxsw_sp_acl_erp_bank_get(erp); 558 struct mlxsw_sp_acl_atcam_entry *aentry; 559 560 list_for_each_entry_reverse(aentry, &aregion->entries_list, list) 561 mlxsw_sp_acl_bf_entry_del(aregion->region->mlxsw_sp, 562 erp_table->erp_core->bf, 563 aregion, erp_bank, aentry); 564 } 565 566 static int 567 mlxsw_sp_acl_erp_region_table_trans(struct mlxsw_sp_acl_erp_table *erp_table) 568 { 569 struct mlxsw_sp_acl_erp_core *erp_core = erp_table->erp_core; 570 struct mlxsw_sp_acl_erp *master_rp; 571 int err; 572 573 /* Initially, allocate a single eRP row. Expand later as needed */ 574 err = mlxsw_sp_acl_erp_table_alloc(erp_core, erp_core->num_erp_banks, 575 erp_table->aregion->type, 576 &erp_table->base_index); 577 if (err) 578 return err; 579 erp_table->num_max_atcam_erps = erp_core->num_erp_banks; 580 581 /* Transition the sole RP currently configured (the master RP) 582 * to the eRP table 583 */ 584 master_rp = mlxsw_sp_acl_erp_table_master_rp(erp_table); 585 if (!master_rp) { 586 err = -EINVAL; 587 goto err_table_master_rp; 588 } 589 590 /* Make sure the master RP is using a valid index, as 591 * only a single eRP row is currently allocated. 592 */ 593 master_rp->index = 0; 594 __set_bit(master_rp->index, erp_table->erp_index_bitmap); 595 596 err = mlxsw_sp_acl_erp_table_erp_add(erp_table, master_rp); 597 if (err) 598 goto err_table_master_rp_add; 599 600 /* Update Bloom filter before enabling eRP table, as rules 601 * on the master RP were not set to Bloom filter up to this 602 * point. 603 */ 604 err = mlxsw_acl_erp_table_bf_add(erp_table, master_rp); 605 if (err) 606 goto err_table_bf_add; 607 608 err = mlxsw_sp_acl_erp_table_enable(erp_table, false); 609 if (err) 610 goto err_table_enable; 611 612 return 0; 613 614 err_table_enable: 615 mlxsw_acl_erp_table_bf_del(erp_table, master_rp); 616 err_table_bf_add: 617 mlxsw_sp_acl_erp_table_erp_del(master_rp); 618 err_table_master_rp_add: 619 __clear_bit(master_rp->index, erp_table->erp_index_bitmap); 620 err_table_master_rp: 621 mlxsw_sp_acl_erp_table_free(erp_core, erp_table->num_max_atcam_erps, 622 erp_table->aregion->type, 623 erp_table->base_index); 624 return err; 625 } 626 627 static void 628 mlxsw_sp_acl_erp_region_master_mask_trans(struct mlxsw_sp_acl_erp_table *erp_table) 629 { 630 struct mlxsw_sp_acl_erp_core *erp_core = erp_table->erp_core; 631 struct mlxsw_sp_acl_erp *master_rp; 632 633 mlxsw_sp_acl_erp_table_disable(erp_table); 634 master_rp = mlxsw_sp_acl_erp_table_master_rp(erp_table); 635 if (!master_rp) 636 return; 637 mlxsw_acl_erp_table_bf_del(erp_table, master_rp); 638 mlxsw_sp_acl_erp_table_erp_del(master_rp); 639 __clear_bit(master_rp->index, erp_table->erp_index_bitmap); 640 mlxsw_sp_acl_erp_table_free(erp_core, erp_table->num_max_atcam_erps, 641 erp_table->aregion->type, 642 erp_table->base_index); 643 } 644 645 static int 646 mlxsw_sp_acl_erp_region_erp_add(struct mlxsw_sp_acl_erp_table *erp_table, 647 struct mlxsw_sp_acl_erp *erp) 648 { 649 struct mlxsw_sp_acl_tcam_region *region = erp_table->aregion->region; 650 struct mlxsw_sp *mlxsw_sp = erp_table->erp_core->mlxsw_sp; 651 bool ctcam_le = erp_table->num_ctcam_erps > 0; 652 char pererp_pl[MLXSW_REG_PERERP_LEN]; 653 654 mlxsw_reg_pererp_pack(pererp_pl, region->id, ctcam_le, true, 0, 655 erp_table->base_index, 0); 656 mlxsw_reg_pererp_erp_vector_pack(pererp_pl, erp_table->erp_index_bitmap, 657 MLXSW_SP_ACL_ERP_MAX_PER_REGION); 658 mlxsw_reg_pererp_erpt_vector_set(pererp_pl, erp->index, true); 659 660 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(pererp), pererp_pl); 661 } 662 663 static void mlxsw_sp_acl_erp_region_erp_del(struct mlxsw_sp_acl_erp *erp) 664 { 665 struct mlxsw_sp_acl_erp_table *erp_table = erp->erp_table; 666 struct mlxsw_sp_acl_tcam_region *region = erp_table->aregion->region; 667 struct mlxsw_sp *mlxsw_sp = erp_table->erp_core->mlxsw_sp; 668 bool ctcam_le = erp_table->num_ctcam_erps > 0; 669 char pererp_pl[MLXSW_REG_PERERP_LEN]; 670 671 mlxsw_reg_pererp_pack(pererp_pl, region->id, ctcam_le, true, 0, 672 erp_table->base_index, 0); 673 mlxsw_reg_pererp_erp_vector_pack(pererp_pl, erp_table->erp_index_bitmap, 674 MLXSW_SP_ACL_ERP_MAX_PER_REGION); 675 mlxsw_reg_pererp_erpt_vector_set(pererp_pl, erp->index, false); 676 677 mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(pererp), pererp_pl); 678 } 679 680 static int 681 mlxsw_sp_acl_erp_region_ctcam_enable(struct mlxsw_sp_acl_erp_table *erp_table) 682 { 683 /* No need to re-enable lookup in the C-TCAM */ 684 if (erp_table->num_ctcam_erps > 1) 685 return 0; 686 687 return mlxsw_sp_acl_erp_table_enable(erp_table, true); 688 } 689 690 static void 691 mlxsw_sp_acl_erp_region_ctcam_disable(struct mlxsw_sp_acl_erp_table *erp_table) 692 { 693 /* Only disable C-TCAM lookup when last C-TCAM eRP is deleted */ 694 if (erp_table->num_ctcam_erps > 1) 695 return; 696 697 mlxsw_sp_acl_erp_table_enable(erp_table, false); 698 } 699 700 static int 701 __mlxsw_sp_acl_erp_table_other_inc(struct mlxsw_sp_acl_erp_table *erp_table, 702 unsigned int *inc_num) 703 { 704 int err; 705 706 /* If there are C-TCAM eRP or deltas in use we need to transition 707 * the region to use eRP table, if it is not already done 708 */ 709 if (!mlxsw_sp_acl_erp_table_is_used(erp_table)) { 710 err = mlxsw_sp_acl_erp_region_table_trans(erp_table); 711 if (err) 712 return err; 713 } 714 715 /* When C-TCAM or deltas are used, the eRP table must be used */ 716 if (erp_table->ops != &erp_multiple_masks_ops) 717 erp_table->ops = &erp_multiple_masks_ops; 718 719 (*inc_num)++; 720 721 return 0; 722 } 723 724 static int mlxsw_sp_acl_erp_ctcam_inc(struct mlxsw_sp_acl_erp_table *erp_table) 725 { 726 return __mlxsw_sp_acl_erp_table_other_inc(erp_table, 727 &erp_table->num_ctcam_erps); 728 } 729 730 static int mlxsw_sp_acl_erp_delta_inc(struct mlxsw_sp_acl_erp_table *erp_table) 731 { 732 return __mlxsw_sp_acl_erp_table_other_inc(erp_table, 733 &erp_table->num_deltas); 734 } 735 736 static void 737 __mlxsw_sp_acl_erp_table_other_dec(struct mlxsw_sp_acl_erp_table *erp_table, 738 unsigned int *dec_num) 739 { 740 (*dec_num)--; 741 742 /* If there are no C-TCAM eRP or deltas in use, the state we 743 * transition to depends on the number of A-TCAM eRPs currently 744 * in use. 745 */ 746 if (erp_table->num_ctcam_erps > 0 || erp_table->num_deltas > 0) 747 return; 748 749 switch (erp_table->num_atcam_erps) { 750 case 2: 751 /* Keep using the eRP table, but correctly set the 752 * operations pointer so that when an A-TCAM eRP is 753 * deleted we will transition to use the master mask 754 */ 755 erp_table->ops = &erp_two_masks_ops; 756 break; 757 case 1: 758 /* We only kept the eRP table because we had C-TCAM 759 * eRPs in use. Now that the last C-TCAM eRP is gone we 760 * can stop using the table and transition to use the 761 * master mask 762 */ 763 mlxsw_sp_acl_erp_region_master_mask_trans(erp_table); 764 erp_table->ops = &erp_single_mask_ops; 765 break; 766 case 0: 767 /* There are no more eRPs of any kind used by the region 768 * so free its eRP table and transition to initial state 769 */ 770 mlxsw_sp_acl_erp_table_disable(erp_table); 771 mlxsw_sp_acl_erp_table_free(erp_table->erp_core, 772 erp_table->num_max_atcam_erps, 773 erp_table->aregion->type, 774 erp_table->base_index); 775 erp_table->ops = &erp_no_mask_ops; 776 break; 777 default: 778 break; 779 } 780 } 781 782 static void mlxsw_sp_acl_erp_ctcam_dec(struct mlxsw_sp_acl_erp_table *erp_table) 783 { 784 __mlxsw_sp_acl_erp_table_other_dec(erp_table, 785 &erp_table->num_ctcam_erps); 786 } 787 788 static void mlxsw_sp_acl_erp_delta_dec(struct mlxsw_sp_acl_erp_table *erp_table) 789 { 790 __mlxsw_sp_acl_erp_table_other_dec(erp_table, 791 &erp_table->num_deltas); 792 } 793 794 static struct mlxsw_sp_acl_erp * 795 mlxsw_sp_acl_erp_ctcam_mask_create(struct mlxsw_sp_acl_erp_table *erp_table, 796 struct mlxsw_sp_acl_erp_key *key) 797 { 798 struct mlxsw_sp_acl_erp *erp; 799 int err; 800 801 erp = kzalloc(sizeof(*erp), GFP_KERNEL); 802 if (!erp) 803 return ERR_PTR(-ENOMEM); 804 805 memcpy(&erp->key, key, sizeof(*key)); 806 bitmap_from_arr32(erp->mask_bitmap, (u32 *) key->mask, 807 MLXSW_SP_ACL_TCAM_MASK_LEN); 808 809 err = mlxsw_sp_acl_erp_ctcam_inc(erp_table); 810 if (err) 811 goto err_erp_ctcam_inc; 812 813 erp->erp_table = erp_table; 814 815 err = mlxsw_sp_acl_erp_master_mask_set(erp_table, &erp->key); 816 if (err) 817 goto err_master_mask_set; 818 819 err = mlxsw_sp_acl_erp_region_ctcam_enable(erp_table); 820 if (err) 821 goto err_erp_region_ctcam_enable; 822 823 return erp; 824 825 err_erp_region_ctcam_enable: 826 mlxsw_sp_acl_erp_master_mask_clear(erp_table, &erp->key); 827 err_master_mask_set: 828 mlxsw_sp_acl_erp_ctcam_dec(erp_table); 829 err_erp_ctcam_inc: 830 kfree(erp); 831 return ERR_PTR(err); 832 } 833 834 static void 835 mlxsw_sp_acl_erp_ctcam_mask_destroy(struct mlxsw_sp_acl_erp *erp) 836 { 837 struct mlxsw_sp_acl_erp_table *erp_table = erp->erp_table; 838 839 mlxsw_sp_acl_erp_region_ctcam_disable(erp_table); 840 mlxsw_sp_acl_erp_master_mask_clear(erp_table, &erp->key); 841 mlxsw_sp_acl_erp_ctcam_dec(erp_table); 842 kfree(erp); 843 } 844 845 static struct mlxsw_sp_acl_erp * 846 mlxsw_sp_acl_erp_mask_create(struct mlxsw_sp_acl_erp_table *erp_table, 847 struct mlxsw_sp_acl_erp_key *key) 848 { 849 struct mlxsw_sp_acl_erp *erp; 850 int err; 851 852 if (key->ctcam) 853 return mlxsw_sp_acl_erp_ctcam_mask_create(erp_table, key); 854 855 /* Expand the eRP table for the new eRP, if needed */ 856 err = mlxsw_sp_acl_erp_table_expand(erp_table); 857 if (err) 858 return ERR_PTR(err); 859 860 erp = mlxsw_sp_acl_erp_generic_create(erp_table, key); 861 if (IS_ERR(erp)) 862 return erp; 863 864 err = mlxsw_sp_acl_erp_index_get(erp_table, &erp->index); 865 if (err) 866 goto err_erp_index_get; 867 868 err = mlxsw_sp_acl_erp_table_erp_add(erp_table, erp); 869 if (err) 870 goto err_table_erp_add; 871 872 err = mlxsw_sp_acl_erp_region_erp_add(erp_table, erp); 873 if (err) 874 goto err_region_erp_add; 875 876 erp_table->ops = &erp_multiple_masks_ops; 877 878 return erp; 879 880 err_region_erp_add: 881 mlxsw_sp_acl_erp_table_erp_del(erp); 882 err_table_erp_add: 883 mlxsw_sp_acl_erp_index_put(erp_table, erp->index); 884 err_erp_index_get: 885 mlxsw_sp_acl_erp_generic_destroy(erp); 886 return ERR_PTR(err); 887 } 888 889 static void 890 mlxsw_sp_acl_erp_mask_destroy(struct mlxsw_sp_acl_erp_table *erp_table, 891 struct mlxsw_sp_acl_erp *erp) 892 { 893 if (erp->key.ctcam) 894 return mlxsw_sp_acl_erp_ctcam_mask_destroy(erp); 895 896 mlxsw_sp_acl_erp_region_erp_del(erp); 897 mlxsw_sp_acl_erp_table_erp_del(erp); 898 mlxsw_sp_acl_erp_index_put(erp_table, erp->index); 899 mlxsw_sp_acl_erp_generic_destroy(erp); 900 901 if (erp_table->num_atcam_erps == 2 && erp_table->num_ctcam_erps == 0 && 902 erp_table->num_deltas == 0) 903 erp_table->ops = &erp_two_masks_ops; 904 } 905 906 static struct mlxsw_sp_acl_erp * 907 mlxsw_sp_acl_erp_second_mask_create(struct mlxsw_sp_acl_erp_table *erp_table, 908 struct mlxsw_sp_acl_erp_key *key) 909 { 910 struct mlxsw_sp_acl_erp *erp; 911 int err; 912 913 if (key->ctcam) 914 return mlxsw_sp_acl_erp_ctcam_mask_create(erp_table, key); 915 916 /* Transition to use eRP table instead of master mask */ 917 err = mlxsw_sp_acl_erp_region_table_trans(erp_table); 918 if (err) 919 return ERR_PTR(err); 920 921 erp = mlxsw_sp_acl_erp_generic_create(erp_table, key); 922 if (IS_ERR(erp)) { 923 err = PTR_ERR(erp); 924 goto err_erp_create; 925 } 926 927 err = mlxsw_sp_acl_erp_index_get(erp_table, &erp->index); 928 if (err) 929 goto err_erp_index_get; 930 931 err = mlxsw_sp_acl_erp_table_erp_add(erp_table, erp); 932 if (err) 933 goto err_table_erp_add; 934 935 err = mlxsw_sp_acl_erp_region_erp_add(erp_table, erp); 936 if (err) 937 goto err_region_erp_add; 938 939 erp_table->ops = &erp_two_masks_ops; 940 941 return erp; 942 943 err_region_erp_add: 944 mlxsw_sp_acl_erp_table_erp_del(erp); 945 err_table_erp_add: 946 mlxsw_sp_acl_erp_index_put(erp_table, erp->index); 947 err_erp_index_get: 948 mlxsw_sp_acl_erp_generic_destroy(erp); 949 err_erp_create: 950 mlxsw_sp_acl_erp_region_master_mask_trans(erp_table); 951 return ERR_PTR(err); 952 } 953 954 static void 955 mlxsw_sp_acl_erp_second_mask_destroy(struct mlxsw_sp_acl_erp_table *erp_table, 956 struct mlxsw_sp_acl_erp *erp) 957 { 958 if (erp->key.ctcam) 959 return mlxsw_sp_acl_erp_ctcam_mask_destroy(erp); 960 961 mlxsw_sp_acl_erp_region_erp_del(erp); 962 mlxsw_sp_acl_erp_table_erp_del(erp); 963 mlxsw_sp_acl_erp_index_put(erp_table, erp->index); 964 mlxsw_sp_acl_erp_generic_destroy(erp); 965 /* Transition to use master mask instead of eRP table */ 966 mlxsw_sp_acl_erp_region_master_mask_trans(erp_table); 967 968 erp_table->ops = &erp_single_mask_ops; 969 } 970 971 static struct mlxsw_sp_acl_erp * 972 mlxsw_sp_acl_erp_first_mask_create(struct mlxsw_sp_acl_erp_table *erp_table, 973 struct mlxsw_sp_acl_erp_key *key) 974 { 975 struct mlxsw_sp_acl_erp *erp; 976 977 if (key->ctcam) 978 return ERR_PTR(-EINVAL); 979 980 erp = mlxsw_sp_acl_erp_generic_create(erp_table, key); 981 if (IS_ERR(erp)) 982 return erp; 983 984 erp_table->ops = &erp_single_mask_ops; 985 986 return erp; 987 } 988 989 static void 990 mlxsw_sp_acl_erp_first_mask_destroy(struct mlxsw_sp_acl_erp_table *erp_table, 991 struct mlxsw_sp_acl_erp *erp) 992 { 993 mlxsw_sp_acl_erp_generic_destroy(erp); 994 erp_table->ops = &erp_no_mask_ops; 995 } 996 997 static void 998 mlxsw_sp_acl_erp_no_mask_destroy(struct mlxsw_sp_acl_erp_table *erp_table, 999 struct mlxsw_sp_acl_erp *erp) 1000 { 1001 WARN_ON(1); 1002 } 1003 1004 struct mlxsw_sp_acl_erp_mask * 1005 mlxsw_sp_acl_erp_mask_get(struct mlxsw_sp_acl_atcam_region *aregion, 1006 const char *mask, bool ctcam) 1007 { 1008 struct mlxsw_sp_acl_erp_table *erp_table = aregion->erp_table; 1009 struct mlxsw_sp_acl_erp_key key; 1010 struct objagg_obj *objagg_obj; 1011 1012 memcpy(key.mask, mask, MLXSW_REG_PTCEX_FLEX_KEY_BLOCKS_LEN); 1013 key.ctcam = ctcam; 1014 mutex_lock(&erp_table->objagg_lock); 1015 objagg_obj = objagg_obj_get(erp_table->objagg, &key); 1016 mutex_unlock(&erp_table->objagg_lock); 1017 if (IS_ERR(objagg_obj)) 1018 return ERR_CAST(objagg_obj); 1019 return (struct mlxsw_sp_acl_erp_mask *) objagg_obj; 1020 } 1021 1022 void mlxsw_sp_acl_erp_mask_put(struct mlxsw_sp_acl_atcam_region *aregion, 1023 struct mlxsw_sp_acl_erp_mask *erp_mask) 1024 { 1025 struct objagg_obj *objagg_obj = (struct objagg_obj *) erp_mask; 1026 struct mlxsw_sp_acl_erp_table *erp_table = aregion->erp_table; 1027 1028 mutex_lock(&erp_table->objagg_lock); 1029 objagg_obj_put(erp_table->objagg, objagg_obj); 1030 mutex_unlock(&erp_table->objagg_lock); 1031 } 1032 1033 int mlxsw_sp_acl_erp_bf_insert(struct mlxsw_sp *mlxsw_sp, 1034 struct mlxsw_sp_acl_atcam_region *aregion, 1035 struct mlxsw_sp_acl_erp_mask *erp_mask, 1036 struct mlxsw_sp_acl_atcam_entry *aentry) 1037 { 1038 struct objagg_obj *objagg_obj = (struct objagg_obj *) erp_mask; 1039 const struct mlxsw_sp_acl_erp *erp = objagg_obj_root_priv(objagg_obj); 1040 unsigned int erp_bank; 1041 1042 if (!mlxsw_sp_acl_erp_table_is_used(erp->erp_table)) 1043 return 0; 1044 1045 erp_bank = mlxsw_sp_acl_erp_bank_get(erp); 1046 return mlxsw_sp_acl_bf_entry_add(mlxsw_sp, 1047 erp->erp_table->erp_core->bf, 1048 aregion, erp_bank, aentry); 1049 } 1050 1051 void mlxsw_sp_acl_erp_bf_remove(struct mlxsw_sp *mlxsw_sp, 1052 struct mlxsw_sp_acl_atcam_region *aregion, 1053 struct mlxsw_sp_acl_erp_mask *erp_mask, 1054 struct mlxsw_sp_acl_atcam_entry *aentry) 1055 { 1056 struct objagg_obj *objagg_obj = (struct objagg_obj *) erp_mask; 1057 const struct mlxsw_sp_acl_erp *erp = objagg_obj_root_priv(objagg_obj); 1058 unsigned int erp_bank; 1059 1060 if (!mlxsw_sp_acl_erp_table_is_used(erp->erp_table)) 1061 return; 1062 1063 erp_bank = mlxsw_sp_acl_erp_bank_get(erp); 1064 mlxsw_sp_acl_bf_entry_del(mlxsw_sp, 1065 erp->erp_table->erp_core->bf, 1066 aregion, erp_bank, aentry); 1067 } 1068 1069 bool 1070 mlxsw_sp_acl_erp_mask_is_ctcam(const struct mlxsw_sp_acl_erp_mask *erp_mask) 1071 { 1072 struct objagg_obj *objagg_obj = (struct objagg_obj *) erp_mask; 1073 const struct mlxsw_sp_acl_erp_key *key = objagg_obj_raw(objagg_obj); 1074 1075 return key->ctcam; 1076 } 1077 1078 u8 mlxsw_sp_acl_erp_mask_erp_id(const struct mlxsw_sp_acl_erp_mask *erp_mask) 1079 { 1080 struct objagg_obj *objagg_obj = (struct objagg_obj *) erp_mask; 1081 const struct mlxsw_sp_acl_erp *erp = objagg_obj_root_priv(objagg_obj); 1082 1083 return erp->id; 1084 } 1085 1086 struct mlxsw_sp_acl_erp_delta { 1087 struct mlxsw_sp_acl_erp_key key; 1088 u16 start; 1089 u8 mask; 1090 }; 1091 1092 u16 mlxsw_sp_acl_erp_delta_start(const struct mlxsw_sp_acl_erp_delta *delta) 1093 { 1094 return delta->start; 1095 } 1096 1097 u8 mlxsw_sp_acl_erp_delta_mask(const struct mlxsw_sp_acl_erp_delta *delta) 1098 { 1099 return delta->mask; 1100 } 1101 1102 u8 mlxsw_sp_acl_erp_delta_value(const struct mlxsw_sp_acl_erp_delta *delta, 1103 const char *enc_key) 1104 { 1105 u16 start = delta->start; 1106 u8 mask = delta->mask; 1107 u16 tmp; 1108 1109 if (!mask) 1110 return 0; 1111 1112 tmp = (unsigned char) enc_key[__MASK_IDX(start / 8)]; 1113 if (start / 8 + 1 < __MASK_LEN) 1114 tmp |= (unsigned char) enc_key[__MASK_IDX(start / 8 + 1)] << 8; 1115 tmp >>= start % 8; 1116 tmp &= mask; 1117 return tmp; 1118 } 1119 1120 void mlxsw_sp_acl_erp_delta_clear(const struct mlxsw_sp_acl_erp_delta *delta, 1121 const char *enc_key) 1122 { 1123 u16 start = delta->start; 1124 u8 mask = delta->mask; 1125 unsigned char *byte; 1126 u16 tmp; 1127 1128 tmp = mask; 1129 tmp <<= start % 8; 1130 tmp = ~tmp; 1131 1132 byte = (unsigned char *) &enc_key[__MASK_IDX(start / 8)]; 1133 *byte &= tmp & 0xff; 1134 if (start / 8 + 1 < __MASK_LEN) { 1135 byte = (unsigned char *) &enc_key[__MASK_IDX(start / 8 + 1)]; 1136 *byte &= (tmp >> 8) & 0xff; 1137 } 1138 } 1139 1140 static const struct mlxsw_sp_acl_erp_delta 1141 mlxsw_sp_acl_erp_delta_default = {}; 1142 1143 const struct mlxsw_sp_acl_erp_delta * 1144 mlxsw_sp_acl_erp_delta(const struct mlxsw_sp_acl_erp_mask *erp_mask) 1145 { 1146 struct objagg_obj *objagg_obj = (struct objagg_obj *) erp_mask; 1147 const struct mlxsw_sp_acl_erp_delta *delta; 1148 1149 delta = objagg_obj_delta_priv(objagg_obj); 1150 if (!delta) 1151 delta = &mlxsw_sp_acl_erp_delta_default; 1152 return delta; 1153 } 1154 1155 static int 1156 mlxsw_sp_acl_erp_delta_fill(const struct mlxsw_sp_acl_erp_key *parent_key, 1157 const struct mlxsw_sp_acl_erp_key *key, 1158 u16 *delta_start, u8 *delta_mask) 1159 { 1160 int offset = 0; 1161 int si = -1; 1162 u16 pmask; 1163 u16 mask; 1164 int i; 1165 1166 /* The difference between 2 masks can be up to 8 consecutive bits. */ 1167 for (i = 0; i < __MASK_LEN; i++) { 1168 if (parent_key->mask[__MASK_IDX(i)] == key->mask[__MASK_IDX(i)]) 1169 continue; 1170 if (si == -1) 1171 si = i; 1172 else if (si != i - 1) 1173 return -EINVAL; 1174 } 1175 if (si == -1) { 1176 /* The masks are the same, this can happen in case eRPs with 1177 * the same mask were created in both A-TCAM and C-TCAM. 1178 * The only possible condition under which this can happen 1179 * is identical rule insertion. Delta is not possible here. 1180 */ 1181 return -EINVAL; 1182 } 1183 pmask = (unsigned char) parent_key->mask[__MASK_IDX(si)]; 1184 mask = (unsigned char) key->mask[__MASK_IDX(si)]; 1185 if (si + 1 < __MASK_LEN) { 1186 pmask |= (unsigned char) parent_key->mask[__MASK_IDX(si + 1)] << 8; 1187 mask |= (unsigned char) key->mask[__MASK_IDX(si + 1)] << 8; 1188 } 1189 1190 if ((pmask ^ mask) & pmask) 1191 return -EINVAL; 1192 mask &= ~pmask; 1193 while (!(mask & (1 << offset))) 1194 offset++; 1195 while (!(mask & 1)) 1196 mask >>= 1; 1197 if (mask & 0xff00) 1198 return -EINVAL; 1199 1200 *delta_start = si * 8 + offset; 1201 *delta_mask = mask; 1202 1203 return 0; 1204 } 1205 1206 static bool mlxsw_sp_acl_erp_delta_check(void *priv, const void *parent_obj, 1207 const void *obj) 1208 { 1209 const struct mlxsw_sp_acl_erp_key *parent_key = parent_obj; 1210 const struct mlxsw_sp_acl_erp_key *key = obj; 1211 u16 delta_start; 1212 u8 delta_mask; 1213 int err; 1214 1215 err = mlxsw_sp_acl_erp_delta_fill(parent_key, key, 1216 &delta_start, &delta_mask); 1217 return err ? false : true; 1218 } 1219 1220 static int mlxsw_sp_acl_erp_hints_obj_cmp(const void *obj1, const void *obj2) 1221 { 1222 const struct mlxsw_sp_acl_erp_key *key1 = obj1; 1223 const struct mlxsw_sp_acl_erp_key *key2 = obj2; 1224 1225 /* For hints purposes, two objects are considered equal 1226 * in case the masks are the same. Does not matter what 1227 * the "ctcam" value is. 1228 */ 1229 return memcmp(key1->mask, key2->mask, sizeof(key1->mask)); 1230 } 1231 1232 static void *mlxsw_sp_acl_erp_delta_create(void *priv, void *parent_obj, 1233 void *obj) 1234 { 1235 struct mlxsw_sp_acl_erp_key *parent_key = parent_obj; 1236 struct mlxsw_sp_acl_atcam_region *aregion = priv; 1237 struct mlxsw_sp_acl_erp_table *erp_table = aregion->erp_table; 1238 struct mlxsw_sp_acl_erp_key *key = obj; 1239 struct mlxsw_sp_acl_erp_delta *delta; 1240 u16 delta_start; 1241 u8 delta_mask; 1242 int err; 1243 1244 if (parent_key->ctcam || key->ctcam) 1245 return ERR_PTR(-EINVAL); 1246 err = mlxsw_sp_acl_erp_delta_fill(parent_key, key, 1247 &delta_start, &delta_mask); 1248 if (err) 1249 return ERR_PTR(-EINVAL); 1250 1251 delta = kzalloc(sizeof(*delta), GFP_KERNEL); 1252 if (!delta) 1253 return ERR_PTR(-ENOMEM); 1254 delta->start = delta_start; 1255 delta->mask = delta_mask; 1256 1257 err = mlxsw_sp_acl_erp_delta_inc(erp_table); 1258 if (err) 1259 goto err_erp_delta_inc; 1260 1261 memcpy(&delta->key, key, sizeof(*key)); 1262 err = mlxsw_sp_acl_erp_master_mask_set(erp_table, &delta->key); 1263 if (err) 1264 goto err_master_mask_set; 1265 1266 return delta; 1267 1268 err_master_mask_set: 1269 mlxsw_sp_acl_erp_delta_dec(erp_table); 1270 err_erp_delta_inc: 1271 kfree(delta); 1272 return ERR_PTR(err); 1273 } 1274 1275 static void mlxsw_sp_acl_erp_delta_destroy(void *priv, void *delta_priv) 1276 { 1277 struct mlxsw_sp_acl_erp_delta *delta = delta_priv; 1278 struct mlxsw_sp_acl_atcam_region *aregion = priv; 1279 struct mlxsw_sp_acl_erp_table *erp_table = aregion->erp_table; 1280 1281 mlxsw_sp_acl_erp_master_mask_clear(erp_table, &delta->key); 1282 mlxsw_sp_acl_erp_delta_dec(erp_table); 1283 kfree(delta); 1284 } 1285 1286 static void *mlxsw_sp_acl_erp_root_create(void *priv, void *obj, 1287 unsigned int root_id) 1288 { 1289 struct mlxsw_sp_acl_atcam_region *aregion = priv; 1290 struct mlxsw_sp_acl_erp_table *erp_table = aregion->erp_table; 1291 struct mlxsw_sp_acl_erp_key *key = obj; 1292 1293 if (!key->ctcam && 1294 root_id != OBJAGG_OBJ_ROOT_ID_INVALID && 1295 root_id >= MLXSW_SP_ACL_ERP_MAX_PER_REGION) 1296 return ERR_PTR(-ENOBUFS); 1297 return erp_table->ops->erp_create(erp_table, key); 1298 } 1299 1300 static void mlxsw_sp_acl_erp_root_destroy(void *priv, void *root_priv) 1301 { 1302 struct mlxsw_sp_acl_atcam_region *aregion = priv; 1303 struct mlxsw_sp_acl_erp_table *erp_table = aregion->erp_table; 1304 1305 erp_table->ops->erp_destroy(erp_table, root_priv); 1306 } 1307 1308 static const struct objagg_ops mlxsw_sp_acl_erp_objagg_ops = { 1309 .obj_size = sizeof(struct mlxsw_sp_acl_erp_key), 1310 .delta_check = mlxsw_sp_acl_erp_delta_check, 1311 .hints_obj_cmp = mlxsw_sp_acl_erp_hints_obj_cmp, 1312 .delta_create = mlxsw_sp_acl_erp_delta_create, 1313 .delta_destroy = mlxsw_sp_acl_erp_delta_destroy, 1314 .root_create = mlxsw_sp_acl_erp_root_create, 1315 .root_destroy = mlxsw_sp_acl_erp_root_destroy, 1316 }; 1317 1318 static struct mlxsw_sp_acl_erp_table * 1319 mlxsw_sp_acl_erp_table_create(struct mlxsw_sp_acl_atcam_region *aregion, 1320 struct objagg_hints *hints) 1321 { 1322 struct mlxsw_sp_acl_erp_table *erp_table; 1323 int err; 1324 1325 erp_table = kzalloc(sizeof(*erp_table), GFP_KERNEL); 1326 if (!erp_table) 1327 return ERR_PTR(-ENOMEM); 1328 1329 erp_table->objagg = objagg_create(&mlxsw_sp_acl_erp_objagg_ops, 1330 hints, aregion); 1331 if (IS_ERR(erp_table->objagg)) { 1332 err = PTR_ERR(erp_table->objagg); 1333 goto err_objagg_create; 1334 } 1335 1336 erp_table->erp_core = aregion->atcam->erp_core; 1337 erp_table->ops = &erp_no_mask_ops; 1338 INIT_LIST_HEAD(&erp_table->atcam_erps_list); 1339 erp_table->aregion = aregion; 1340 mutex_init(&erp_table->objagg_lock); 1341 1342 return erp_table; 1343 1344 err_objagg_create: 1345 kfree(erp_table); 1346 return ERR_PTR(err); 1347 } 1348 1349 static void 1350 mlxsw_sp_acl_erp_table_destroy(struct mlxsw_sp_acl_erp_table *erp_table) 1351 { 1352 WARN_ON(!list_empty(&erp_table->atcam_erps_list)); 1353 mutex_destroy(&erp_table->objagg_lock); 1354 objagg_destroy(erp_table->objagg); 1355 kfree(erp_table); 1356 } 1357 1358 static int 1359 mlxsw_sp_acl_erp_master_mask_init(struct mlxsw_sp_acl_atcam_region *aregion) 1360 { 1361 struct mlxsw_sp *mlxsw_sp = aregion->region->mlxsw_sp; 1362 char percr_pl[MLXSW_REG_PERCR_LEN]; 1363 1364 mlxsw_reg_percr_pack(percr_pl, aregion->region->id); 1365 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(percr), percr_pl); 1366 } 1367 1368 static int 1369 mlxsw_sp_acl_erp_region_param_init(struct mlxsw_sp_acl_atcam_region *aregion) 1370 { 1371 struct mlxsw_sp *mlxsw_sp = aregion->region->mlxsw_sp; 1372 char pererp_pl[MLXSW_REG_PERERP_LEN]; 1373 1374 mlxsw_reg_pererp_pack(pererp_pl, aregion->region->id, false, false, 0, 1375 0, 0); 1376 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(pererp), pererp_pl); 1377 } 1378 1379 static int 1380 mlxsw_sp_acl_erp_hints_check(struct mlxsw_sp *mlxsw_sp, 1381 struct mlxsw_sp_acl_atcam_region *aregion, 1382 struct objagg_hints *hints, bool *p_rehash_needed) 1383 { 1384 struct mlxsw_sp_acl_erp_table *erp_table = aregion->erp_table; 1385 const struct objagg_stats *ostats; 1386 const struct objagg_stats *hstats; 1387 int err; 1388 1389 *p_rehash_needed = false; 1390 1391 mutex_lock(&erp_table->objagg_lock); 1392 ostats = objagg_stats_get(erp_table->objagg); 1393 mutex_unlock(&erp_table->objagg_lock); 1394 if (IS_ERR(ostats)) { 1395 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Failed to get ERP stats\n"); 1396 return PTR_ERR(ostats); 1397 } 1398 1399 hstats = objagg_hints_stats_get(hints); 1400 if (IS_ERR(hstats)) { 1401 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Failed to get ERP hints stats\n"); 1402 err = PTR_ERR(hstats); 1403 goto err_hints_stats_get; 1404 } 1405 1406 /* Very basic criterion for now. */ 1407 if (hstats->root_count < ostats->root_count) 1408 *p_rehash_needed = true; 1409 1410 err = 0; 1411 1412 objagg_stats_put(hstats); 1413 err_hints_stats_get: 1414 objagg_stats_put(ostats); 1415 return err; 1416 } 1417 1418 void * 1419 mlxsw_sp_acl_erp_rehash_hints_get(struct mlxsw_sp_acl_atcam_region *aregion) 1420 { 1421 struct mlxsw_sp_acl_erp_table *erp_table = aregion->erp_table; 1422 struct mlxsw_sp *mlxsw_sp = aregion->region->mlxsw_sp; 1423 struct objagg_hints *hints; 1424 bool rehash_needed; 1425 int err; 1426 1427 mutex_lock(&erp_table->objagg_lock); 1428 hints = objagg_hints_get(erp_table->objagg, 1429 OBJAGG_OPT_ALGO_SIMPLE_GREEDY); 1430 mutex_unlock(&erp_table->objagg_lock); 1431 if (IS_ERR(hints)) { 1432 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Failed to create ERP hints\n"); 1433 return ERR_CAST(hints); 1434 } 1435 err = mlxsw_sp_acl_erp_hints_check(mlxsw_sp, aregion, hints, 1436 &rehash_needed); 1437 if (err) 1438 goto errout; 1439 1440 if (!rehash_needed) { 1441 err = -EAGAIN; 1442 goto errout; 1443 } 1444 return hints; 1445 1446 errout: 1447 objagg_hints_put(hints); 1448 return ERR_PTR(err); 1449 } 1450 1451 void mlxsw_sp_acl_erp_rehash_hints_put(void *hints_priv) 1452 { 1453 struct objagg_hints *hints = hints_priv; 1454 1455 objagg_hints_put(hints); 1456 } 1457 1458 int mlxsw_sp_acl_erp_region_init(struct mlxsw_sp_acl_atcam_region *aregion, 1459 void *hints_priv) 1460 { 1461 struct mlxsw_sp_acl_erp_table *erp_table; 1462 struct objagg_hints *hints = hints_priv; 1463 int err; 1464 1465 erp_table = mlxsw_sp_acl_erp_table_create(aregion, hints); 1466 if (IS_ERR(erp_table)) 1467 return PTR_ERR(erp_table); 1468 aregion->erp_table = erp_table; 1469 1470 /* Initialize the region's master mask to all zeroes */ 1471 err = mlxsw_sp_acl_erp_master_mask_init(aregion); 1472 if (err) 1473 goto err_erp_master_mask_init; 1474 1475 /* Initialize the region to not use the eRP table */ 1476 err = mlxsw_sp_acl_erp_region_param_init(aregion); 1477 if (err) 1478 goto err_erp_region_param_init; 1479 1480 return 0; 1481 1482 err_erp_region_param_init: 1483 err_erp_master_mask_init: 1484 mlxsw_sp_acl_erp_table_destroy(erp_table); 1485 return err; 1486 } 1487 1488 void mlxsw_sp_acl_erp_region_fini(struct mlxsw_sp_acl_atcam_region *aregion) 1489 { 1490 mlxsw_sp_acl_erp_table_destroy(aregion->erp_table); 1491 } 1492 1493 static int 1494 mlxsw_sp_acl_erp_tables_sizes_query(struct mlxsw_sp *mlxsw_sp, 1495 struct mlxsw_sp_acl_erp_core *erp_core) 1496 { 1497 unsigned int size; 1498 1499 if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, ACL_ERPT_ENTRIES_2KB) || 1500 !MLXSW_CORE_RES_VALID(mlxsw_sp->core, ACL_ERPT_ENTRIES_4KB) || 1501 !MLXSW_CORE_RES_VALID(mlxsw_sp->core, ACL_ERPT_ENTRIES_8KB) || 1502 !MLXSW_CORE_RES_VALID(mlxsw_sp->core, ACL_ERPT_ENTRIES_12KB)) 1503 return -EIO; 1504 1505 size = MLXSW_CORE_RES_GET(mlxsw_sp->core, ACL_ERPT_ENTRIES_2KB); 1506 erp_core->erpt_entries_size[MLXSW_SP_ACL_ATCAM_REGION_TYPE_2KB] = size; 1507 1508 size = MLXSW_CORE_RES_GET(mlxsw_sp->core, ACL_ERPT_ENTRIES_4KB); 1509 erp_core->erpt_entries_size[MLXSW_SP_ACL_ATCAM_REGION_TYPE_4KB] = size; 1510 1511 size = MLXSW_CORE_RES_GET(mlxsw_sp->core, ACL_ERPT_ENTRIES_8KB); 1512 erp_core->erpt_entries_size[MLXSW_SP_ACL_ATCAM_REGION_TYPE_8KB] = size; 1513 1514 size = MLXSW_CORE_RES_GET(mlxsw_sp->core, ACL_ERPT_ENTRIES_12KB); 1515 erp_core->erpt_entries_size[MLXSW_SP_ACL_ATCAM_REGION_TYPE_12KB] = size; 1516 1517 return 0; 1518 } 1519 1520 static int mlxsw_sp_acl_erp_tables_init(struct mlxsw_sp *mlxsw_sp, 1521 struct mlxsw_sp_acl_erp_core *erp_core) 1522 { 1523 unsigned int erpt_bank_size; 1524 int err; 1525 1526 if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, ACL_MAX_ERPT_BANK_SIZE) || 1527 !MLXSW_CORE_RES_VALID(mlxsw_sp->core, ACL_MAX_ERPT_BANKS)) 1528 return -EIO; 1529 erpt_bank_size = MLXSW_CORE_RES_GET(mlxsw_sp->core, 1530 ACL_MAX_ERPT_BANK_SIZE); 1531 erp_core->num_erp_banks = MLXSW_CORE_RES_GET(mlxsw_sp->core, 1532 ACL_MAX_ERPT_BANKS); 1533 1534 erp_core->erp_tables = gen_pool_create(0, -1); 1535 if (!erp_core->erp_tables) 1536 return -ENOMEM; 1537 gen_pool_set_algo(erp_core->erp_tables, gen_pool_best_fit, NULL); 1538 1539 err = gen_pool_add(erp_core->erp_tables, 1540 MLXSW_SP_ACL_ERP_GENALLOC_OFFSET, erpt_bank_size, 1541 -1); 1542 if (err) 1543 goto err_gen_pool_add; 1544 1545 erp_core->bf = mlxsw_sp_acl_bf_init(mlxsw_sp, erp_core->num_erp_banks); 1546 if (IS_ERR(erp_core->bf)) { 1547 err = PTR_ERR(erp_core->bf); 1548 goto err_bf_init; 1549 } 1550 1551 /* Different regions require masks of different sizes */ 1552 err = mlxsw_sp_acl_erp_tables_sizes_query(mlxsw_sp, erp_core); 1553 if (err) 1554 goto err_erp_tables_sizes_query; 1555 1556 return 0; 1557 1558 err_erp_tables_sizes_query: 1559 mlxsw_sp_acl_bf_fini(erp_core->bf); 1560 err_bf_init: 1561 err_gen_pool_add: 1562 gen_pool_destroy(erp_core->erp_tables); 1563 return err; 1564 } 1565 1566 static void mlxsw_sp_acl_erp_tables_fini(struct mlxsw_sp *mlxsw_sp, 1567 struct mlxsw_sp_acl_erp_core *erp_core) 1568 { 1569 mlxsw_sp_acl_bf_fini(erp_core->bf); 1570 gen_pool_destroy(erp_core->erp_tables); 1571 } 1572 1573 int mlxsw_sp_acl_erps_init(struct mlxsw_sp *mlxsw_sp, 1574 struct mlxsw_sp_acl_atcam *atcam) 1575 { 1576 struct mlxsw_sp_acl_erp_core *erp_core; 1577 int err; 1578 1579 erp_core = kzalloc(sizeof(*erp_core), GFP_KERNEL); 1580 if (!erp_core) 1581 return -ENOMEM; 1582 erp_core->mlxsw_sp = mlxsw_sp; 1583 atcam->erp_core = erp_core; 1584 1585 err = mlxsw_sp_acl_erp_tables_init(mlxsw_sp, erp_core); 1586 if (err) 1587 goto err_erp_tables_init; 1588 1589 return 0; 1590 1591 err_erp_tables_init: 1592 kfree(erp_core); 1593 return err; 1594 } 1595 1596 void mlxsw_sp_acl_erps_fini(struct mlxsw_sp *mlxsw_sp, 1597 struct mlxsw_sp_acl_atcam *atcam) 1598 { 1599 mlxsw_sp_acl_erp_tables_fini(mlxsw_sp, atcam->erp_core); 1600 kfree(atcam->erp_core); 1601 } 1602