1 // SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0 2 /* Copyright (c) 2018 Mellanox Technologies. All rights reserved */ 3 4 #include <linux/bitmap.h> 5 #include <linux/errno.h> 6 #include <linux/genalloc.h> 7 #include <linux/gfp.h> 8 #include <linux/kernel.h> 9 #include <linux/list.h> 10 #include <linux/mutex.h> 11 #include <linux/objagg.h> 12 #include <linux/rtnetlink.h> 13 #include <linux/slab.h> 14 15 #include "core.h" 16 #include "reg.h" 17 #include "spectrum.h" 18 #include "spectrum_acl_tcam.h" 19 20 /* gen_pool_alloc() returns 0 when allocation fails, so use an offset */ 21 #define MLXSW_SP_ACL_ERP_GENALLOC_OFFSET 0x100 22 #define MLXSW_SP_ACL_ERP_MAX_PER_REGION 16 23 24 struct mlxsw_sp_acl_erp_core { 25 unsigned int erpt_entries_size[MLXSW_SP_ACL_ATCAM_REGION_TYPE_MAX + 1]; 26 struct gen_pool *erp_tables; 27 struct mlxsw_sp *mlxsw_sp; 28 struct mlxsw_sp_acl_bf *bf; 29 unsigned int num_erp_banks; 30 }; 31 32 struct mlxsw_sp_acl_erp_key { 33 char mask[MLXSW_REG_PTCEX_FLEX_KEY_BLOCKS_LEN]; 34 #define __MASK_LEN 0x38 35 #define __MASK_IDX(i) (__MASK_LEN - (i) - 1) 36 bool ctcam; 37 }; 38 39 struct mlxsw_sp_acl_erp { 40 struct mlxsw_sp_acl_erp_key key; 41 u8 id; 42 u8 index; 43 DECLARE_BITMAP(mask_bitmap, MLXSW_SP_ACL_TCAM_MASK_LEN); 44 struct list_head list; 45 struct mlxsw_sp_acl_erp_table *erp_table; 46 }; 47 48 struct mlxsw_sp_acl_erp_master_mask { 49 DECLARE_BITMAP(bitmap, MLXSW_SP_ACL_TCAM_MASK_LEN); 50 unsigned int count[MLXSW_SP_ACL_TCAM_MASK_LEN]; 51 }; 52 53 struct mlxsw_sp_acl_erp_table { 54 struct mlxsw_sp_acl_erp_master_mask master_mask; 55 DECLARE_BITMAP(erp_id_bitmap, MLXSW_SP_ACL_ERP_MAX_PER_REGION); 56 DECLARE_BITMAP(erp_index_bitmap, MLXSW_SP_ACL_ERP_MAX_PER_REGION); 57 struct list_head atcam_erps_list; 58 struct mlxsw_sp_acl_erp_core *erp_core; 59 struct mlxsw_sp_acl_atcam_region *aregion; 60 const struct mlxsw_sp_acl_erp_table_ops *ops; 61 unsigned long base_index; 62 unsigned int num_atcam_erps; 63 unsigned int num_max_atcam_erps; 64 unsigned int num_ctcam_erps; 65 unsigned int num_deltas; 66 struct objagg *objagg; 67 struct mutex objagg_lock; /* guards objagg manipulation */ 68 }; 69 70 struct mlxsw_sp_acl_erp_table_ops { 71 struct mlxsw_sp_acl_erp * 72 (*erp_create)(struct mlxsw_sp_acl_erp_table *erp_table, 73 struct mlxsw_sp_acl_erp_key *key); 74 void (*erp_destroy)(struct mlxsw_sp_acl_erp_table *erp_table, 75 struct mlxsw_sp_acl_erp *erp); 76 }; 77 78 static struct mlxsw_sp_acl_erp * 79 mlxsw_sp_acl_erp_mask_create(struct mlxsw_sp_acl_erp_table *erp_table, 80 struct mlxsw_sp_acl_erp_key *key); 81 static void 82 mlxsw_sp_acl_erp_mask_destroy(struct mlxsw_sp_acl_erp_table *erp_table, 83 struct mlxsw_sp_acl_erp *erp); 84 static struct mlxsw_sp_acl_erp * 85 mlxsw_sp_acl_erp_second_mask_create(struct mlxsw_sp_acl_erp_table *erp_table, 86 struct mlxsw_sp_acl_erp_key *key); 87 static void 88 mlxsw_sp_acl_erp_second_mask_destroy(struct mlxsw_sp_acl_erp_table *erp_table, 89 struct mlxsw_sp_acl_erp *erp); 90 static struct mlxsw_sp_acl_erp * 91 mlxsw_sp_acl_erp_first_mask_create(struct mlxsw_sp_acl_erp_table *erp_table, 92 struct mlxsw_sp_acl_erp_key *key); 93 static void 94 mlxsw_sp_acl_erp_first_mask_destroy(struct mlxsw_sp_acl_erp_table *erp_table, 95 struct mlxsw_sp_acl_erp *erp); 96 static void 97 mlxsw_sp_acl_erp_no_mask_destroy(struct mlxsw_sp_acl_erp_table *erp_table, 98 struct mlxsw_sp_acl_erp *erp); 99 100 static const struct mlxsw_sp_acl_erp_table_ops erp_multiple_masks_ops = { 101 .erp_create = mlxsw_sp_acl_erp_mask_create, 102 .erp_destroy = mlxsw_sp_acl_erp_mask_destroy, 103 }; 104 105 static const struct mlxsw_sp_acl_erp_table_ops erp_two_masks_ops = { 106 .erp_create = mlxsw_sp_acl_erp_mask_create, 107 .erp_destroy = mlxsw_sp_acl_erp_second_mask_destroy, 108 }; 109 110 static const struct mlxsw_sp_acl_erp_table_ops erp_single_mask_ops = { 111 .erp_create = mlxsw_sp_acl_erp_second_mask_create, 112 .erp_destroy = mlxsw_sp_acl_erp_first_mask_destroy, 113 }; 114 115 static const struct mlxsw_sp_acl_erp_table_ops erp_no_mask_ops = { 116 .erp_create = mlxsw_sp_acl_erp_first_mask_create, 117 .erp_destroy = mlxsw_sp_acl_erp_no_mask_destroy, 118 }; 119 120 static bool 121 mlxsw_sp_acl_erp_table_is_used(const struct mlxsw_sp_acl_erp_table *erp_table) 122 { 123 return erp_table->ops != &erp_single_mask_ops && 124 erp_table->ops != &erp_no_mask_ops; 125 } 126 127 static unsigned int 128 mlxsw_sp_acl_erp_bank_get(const struct mlxsw_sp_acl_erp *erp) 129 { 130 return erp->index % erp->erp_table->erp_core->num_erp_banks; 131 } 132 133 static unsigned int 134 mlxsw_sp_acl_erp_table_entry_size(const struct mlxsw_sp_acl_erp_table *erp_table) 135 { 136 struct mlxsw_sp_acl_atcam_region *aregion = erp_table->aregion; 137 struct mlxsw_sp_acl_erp_core *erp_core = erp_table->erp_core; 138 139 return erp_core->erpt_entries_size[aregion->type]; 140 } 141 142 static int mlxsw_sp_acl_erp_id_get(struct mlxsw_sp_acl_erp_table *erp_table, 143 u8 *p_id) 144 { 145 u8 id; 146 147 id = find_first_zero_bit(erp_table->erp_id_bitmap, 148 MLXSW_SP_ACL_ERP_MAX_PER_REGION); 149 if (id < MLXSW_SP_ACL_ERP_MAX_PER_REGION) { 150 __set_bit(id, erp_table->erp_id_bitmap); 151 *p_id = id; 152 return 0; 153 } 154 155 return -ENOBUFS; 156 } 157 158 static void mlxsw_sp_acl_erp_id_put(struct mlxsw_sp_acl_erp_table *erp_table, 159 u8 id) 160 { 161 __clear_bit(id, erp_table->erp_id_bitmap); 162 } 163 164 static void 165 mlxsw_sp_acl_erp_master_mask_bit_set(unsigned long bit, 166 struct mlxsw_sp_acl_erp_master_mask *mask) 167 { 168 if (mask->count[bit]++ == 0) 169 __set_bit(bit, mask->bitmap); 170 } 171 172 static void 173 mlxsw_sp_acl_erp_master_mask_bit_clear(unsigned long bit, 174 struct mlxsw_sp_acl_erp_master_mask *mask) 175 { 176 if (--mask->count[bit] == 0) 177 __clear_bit(bit, mask->bitmap); 178 } 179 180 static int 181 mlxsw_sp_acl_erp_master_mask_update(struct mlxsw_sp_acl_erp_table *erp_table) 182 { 183 struct mlxsw_sp_acl_tcam_region *region = erp_table->aregion->region; 184 struct mlxsw_sp *mlxsw_sp = region->mlxsw_sp; 185 char percr_pl[MLXSW_REG_PERCR_LEN]; 186 char *master_mask; 187 188 mlxsw_reg_percr_pack(percr_pl, region->id); 189 master_mask = mlxsw_reg_percr_master_mask_data(percr_pl); 190 bitmap_to_arr32((u32 *) master_mask, erp_table->master_mask.bitmap, 191 MLXSW_SP_ACL_TCAM_MASK_LEN); 192 193 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(percr), percr_pl); 194 } 195 196 static int 197 mlxsw_sp_acl_erp_master_mask_set(struct mlxsw_sp_acl_erp_table *erp_table, 198 struct mlxsw_sp_acl_erp_key *key) 199 { 200 DECLARE_BITMAP(mask_bitmap, MLXSW_SP_ACL_TCAM_MASK_LEN); 201 unsigned long bit; 202 int err; 203 204 bitmap_from_arr32(mask_bitmap, (u32 *) key->mask, 205 MLXSW_SP_ACL_TCAM_MASK_LEN); 206 for_each_set_bit(bit, mask_bitmap, MLXSW_SP_ACL_TCAM_MASK_LEN) 207 mlxsw_sp_acl_erp_master_mask_bit_set(bit, 208 &erp_table->master_mask); 209 210 err = mlxsw_sp_acl_erp_master_mask_update(erp_table); 211 if (err) 212 goto err_master_mask_update; 213 214 return 0; 215 216 err_master_mask_update: 217 for_each_set_bit(bit, mask_bitmap, MLXSW_SP_ACL_TCAM_MASK_LEN) 218 mlxsw_sp_acl_erp_master_mask_bit_clear(bit, 219 &erp_table->master_mask); 220 return err; 221 } 222 223 static int 224 mlxsw_sp_acl_erp_master_mask_clear(struct mlxsw_sp_acl_erp_table *erp_table, 225 struct mlxsw_sp_acl_erp_key *key) 226 { 227 DECLARE_BITMAP(mask_bitmap, MLXSW_SP_ACL_TCAM_MASK_LEN); 228 unsigned long bit; 229 int err; 230 231 bitmap_from_arr32(mask_bitmap, (u32 *) key->mask, 232 MLXSW_SP_ACL_TCAM_MASK_LEN); 233 for_each_set_bit(bit, mask_bitmap, MLXSW_SP_ACL_TCAM_MASK_LEN) 234 mlxsw_sp_acl_erp_master_mask_bit_clear(bit, 235 &erp_table->master_mask); 236 237 err = mlxsw_sp_acl_erp_master_mask_update(erp_table); 238 if (err) 239 goto err_master_mask_update; 240 241 return 0; 242 243 err_master_mask_update: 244 for_each_set_bit(bit, mask_bitmap, MLXSW_SP_ACL_TCAM_MASK_LEN) 245 mlxsw_sp_acl_erp_master_mask_bit_set(bit, 246 &erp_table->master_mask); 247 return err; 248 } 249 250 static struct mlxsw_sp_acl_erp * 251 mlxsw_sp_acl_erp_generic_create(struct mlxsw_sp_acl_erp_table *erp_table, 252 struct mlxsw_sp_acl_erp_key *key) 253 { 254 struct mlxsw_sp_acl_erp *erp; 255 int err; 256 257 erp = kzalloc(sizeof(*erp), GFP_KERNEL); 258 if (!erp) 259 return ERR_PTR(-ENOMEM); 260 261 err = mlxsw_sp_acl_erp_id_get(erp_table, &erp->id); 262 if (err) 263 goto err_erp_id_get; 264 265 memcpy(&erp->key, key, sizeof(*key)); 266 list_add(&erp->list, &erp_table->atcam_erps_list); 267 erp_table->num_atcam_erps++; 268 erp->erp_table = erp_table; 269 270 err = mlxsw_sp_acl_erp_master_mask_set(erp_table, &erp->key); 271 if (err) 272 goto err_master_mask_set; 273 274 return erp; 275 276 err_master_mask_set: 277 erp_table->num_atcam_erps--; 278 list_del(&erp->list); 279 mlxsw_sp_acl_erp_id_put(erp_table, erp->id); 280 err_erp_id_get: 281 kfree(erp); 282 return ERR_PTR(err); 283 } 284 285 static void 286 mlxsw_sp_acl_erp_generic_destroy(struct mlxsw_sp_acl_erp *erp) 287 { 288 struct mlxsw_sp_acl_erp_table *erp_table = erp->erp_table; 289 290 mlxsw_sp_acl_erp_master_mask_clear(erp_table, &erp->key); 291 erp_table->num_atcam_erps--; 292 list_del(&erp->list); 293 mlxsw_sp_acl_erp_id_put(erp_table, erp->id); 294 kfree(erp); 295 } 296 297 static int 298 mlxsw_sp_acl_erp_table_alloc(struct mlxsw_sp_acl_erp_core *erp_core, 299 unsigned int num_erps, 300 enum mlxsw_sp_acl_atcam_region_type region_type, 301 unsigned long *p_index) 302 { 303 unsigned int num_rows, entry_size; 304 unsigned long index; 305 306 /* We only allow allocations of entire rows */ 307 if (num_erps % erp_core->num_erp_banks != 0) 308 return -EINVAL; 309 310 entry_size = erp_core->erpt_entries_size[region_type]; 311 num_rows = num_erps / erp_core->num_erp_banks; 312 313 index = gen_pool_alloc(erp_core->erp_tables, num_rows * entry_size); 314 if (!index) 315 return -ENOBUFS; 316 317 *p_index = index - MLXSW_SP_ACL_ERP_GENALLOC_OFFSET; 318 319 return 0; 320 } 321 322 static void 323 mlxsw_sp_acl_erp_table_free(struct mlxsw_sp_acl_erp_core *erp_core, 324 unsigned int num_erps, 325 enum mlxsw_sp_acl_atcam_region_type region_type, 326 unsigned long index) 327 { 328 unsigned long base_index; 329 unsigned int entry_size; 330 size_t size; 331 332 entry_size = erp_core->erpt_entries_size[region_type]; 333 base_index = index + MLXSW_SP_ACL_ERP_GENALLOC_OFFSET; 334 size = num_erps / erp_core->num_erp_banks * entry_size; 335 gen_pool_free(erp_core->erp_tables, base_index, size); 336 } 337 338 static struct mlxsw_sp_acl_erp * 339 mlxsw_sp_acl_erp_table_master_rp(struct mlxsw_sp_acl_erp_table *erp_table) 340 { 341 if (!list_is_singular(&erp_table->atcam_erps_list)) 342 return NULL; 343 344 return list_first_entry(&erp_table->atcam_erps_list, 345 struct mlxsw_sp_acl_erp, list); 346 } 347 348 static int mlxsw_sp_acl_erp_index_get(struct mlxsw_sp_acl_erp_table *erp_table, 349 u8 *p_index) 350 { 351 u8 index; 352 353 index = find_first_zero_bit(erp_table->erp_index_bitmap, 354 erp_table->num_max_atcam_erps); 355 if (index < erp_table->num_max_atcam_erps) { 356 __set_bit(index, erp_table->erp_index_bitmap); 357 *p_index = index; 358 return 0; 359 } 360 361 return -ENOBUFS; 362 } 363 364 static void mlxsw_sp_acl_erp_index_put(struct mlxsw_sp_acl_erp_table *erp_table, 365 u8 index) 366 { 367 __clear_bit(index, erp_table->erp_index_bitmap); 368 } 369 370 static void 371 mlxsw_sp_acl_erp_table_locate(const struct mlxsw_sp_acl_erp_table *erp_table, 372 const struct mlxsw_sp_acl_erp *erp, 373 u8 *p_erpt_bank, u8 *p_erpt_index) 374 { 375 unsigned int entry_size = mlxsw_sp_acl_erp_table_entry_size(erp_table); 376 struct mlxsw_sp_acl_erp_core *erp_core = erp_table->erp_core; 377 unsigned int row; 378 379 *p_erpt_bank = erp->index % erp_core->num_erp_banks; 380 row = erp->index / erp_core->num_erp_banks; 381 *p_erpt_index = erp_table->base_index + row * entry_size; 382 } 383 384 static int 385 mlxsw_sp_acl_erp_table_erp_add(struct mlxsw_sp_acl_erp_table *erp_table, 386 struct mlxsw_sp_acl_erp *erp) 387 { 388 struct mlxsw_sp *mlxsw_sp = erp_table->erp_core->mlxsw_sp; 389 enum mlxsw_reg_perpt_key_size key_size; 390 char perpt_pl[MLXSW_REG_PERPT_LEN]; 391 u8 erpt_bank, erpt_index; 392 393 mlxsw_sp_acl_erp_table_locate(erp_table, erp, &erpt_bank, &erpt_index); 394 key_size = (enum mlxsw_reg_perpt_key_size) erp_table->aregion->type; 395 mlxsw_reg_perpt_pack(perpt_pl, erpt_bank, erpt_index, key_size, erp->id, 396 0, erp_table->base_index, erp->index, 397 erp->key.mask); 398 mlxsw_reg_perpt_erp_vector_pack(perpt_pl, erp_table->erp_index_bitmap, 399 MLXSW_SP_ACL_ERP_MAX_PER_REGION); 400 mlxsw_reg_perpt_erp_vector_set(perpt_pl, erp->index, true); 401 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(perpt), perpt_pl); 402 } 403 404 static void mlxsw_sp_acl_erp_table_erp_del(struct mlxsw_sp_acl_erp *erp) 405 { 406 char empty_mask[MLXSW_REG_PTCEX_FLEX_KEY_BLOCKS_LEN] = { 0 }; 407 struct mlxsw_sp_acl_erp_table *erp_table = erp->erp_table; 408 struct mlxsw_sp *mlxsw_sp = erp_table->erp_core->mlxsw_sp; 409 enum mlxsw_reg_perpt_key_size key_size; 410 char perpt_pl[MLXSW_REG_PERPT_LEN]; 411 u8 erpt_bank, erpt_index; 412 413 mlxsw_sp_acl_erp_table_locate(erp_table, erp, &erpt_bank, &erpt_index); 414 key_size = (enum mlxsw_reg_perpt_key_size) erp_table->aregion->type; 415 mlxsw_reg_perpt_pack(perpt_pl, erpt_bank, erpt_index, key_size, erp->id, 416 0, erp_table->base_index, erp->index, empty_mask); 417 mlxsw_reg_perpt_erp_vector_pack(perpt_pl, erp_table->erp_index_bitmap, 418 MLXSW_SP_ACL_ERP_MAX_PER_REGION); 419 mlxsw_reg_perpt_erp_vector_set(perpt_pl, erp->index, false); 420 mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(perpt), perpt_pl); 421 } 422 423 static int 424 mlxsw_sp_acl_erp_table_enable(struct mlxsw_sp_acl_erp_table *erp_table, 425 bool ctcam_le) 426 { 427 struct mlxsw_sp_acl_tcam_region *region = erp_table->aregion->region; 428 struct mlxsw_sp *mlxsw_sp = erp_table->erp_core->mlxsw_sp; 429 char pererp_pl[MLXSW_REG_PERERP_LEN]; 430 431 mlxsw_reg_pererp_pack(pererp_pl, region->id, ctcam_le, true, 0, 432 erp_table->base_index, 0); 433 mlxsw_reg_pererp_erp_vector_pack(pererp_pl, erp_table->erp_index_bitmap, 434 MLXSW_SP_ACL_ERP_MAX_PER_REGION); 435 436 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(pererp), pererp_pl); 437 } 438 439 static void 440 mlxsw_sp_acl_erp_table_disable(struct mlxsw_sp_acl_erp_table *erp_table) 441 { 442 struct mlxsw_sp_acl_tcam_region *region = erp_table->aregion->region; 443 struct mlxsw_sp *mlxsw_sp = erp_table->erp_core->mlxsw_sp; 444 char pererp_pl[MLXSW_REG_PERERP_LEN]; 445 struct mlxsw_sp_acl_erp *master_rp; 446 447 master_rp = mlxsw_sp_acl_erp_table_master_rp(erp_table); 448 /* It is possible we do not have a master RP when we disable the 449 * table when there are no rules in the A-TCAM and the last C-TCAM 450 * rule is deleted 451 */ 452 mlxsw_reg_pererp_pack(pererp_pl, region->id, false, false, 0, 0, 453 master_rp ? master_rp->id : 0); 454 mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(pererp), pererp_pl); 455 } 456 457 static int 458 mlxsw_sp_acl_erp_table_relocate(struct mlxsw_sp_acl_erp_table *erp_table) 459 { 460 struct mlxsw_sp_acl_erp *erp; 461 int err; 462 463 list_for_each_entry(erp, &erp_table->atcam_erps_list, list) { 464 err = mlxsw_sp_acl_erp_table_erp_add(erp_table, erp); 465 if (err) 466 goto err_table_erp_add; 467 } 468 469 return 0; 470 471 err_table_erp_add: 472 list_for_each_entry_continue_reverse(erp, &erp_table->atcam_erps_list, 473 list) 474 mlxsw_sp_acl_erp_table_erp_del(erp); 475 return err; 476 } 477 478 static int 479 mlxsw_sp_acl_erp_table_expand(struct mlxsw_sp_acl_erp_table *erp_table) 480 { 481 unsigned int num_erps, old_num_erps = erp_table->num_max_atcam_erps; 482 struct mlxsw_sp_acl_erp_core *erp_core = erp_table->erp_core; 483 unsigned long old_base_index = erp_table->base_index; 484 bool ctcam_le = erp_table->num_ctcam_erps > 0; 485 int err; 486 487 if (erp_table->num_atcam_erps < erp_table->num_max_atcam_erps) 488 return 0; 489 490 if (erp_table->num_max_atcam_erps == MLXSW_SP_ACL_ERP_MAX_PER_REGION) 491 return -ENOBUFS; 492 493 num_erps = old_num_erps + erp_core->num_erp_banks; 494 err = mlxsw_sp_acl_erp_table_alloc(erp_core, num_erps, 495 erp_table->aregion->type, 496 &erp_table->base_index); 497 if (err) 498 return err; 499 erp_table->num_max_atcam_erps = num_erps; 500 501 err = mlxsw_sp_acl_erp_table_relocate(erp_table); 502 if (err) 503 goto err_table_relocate; 504 505 err = mlxsw_sp_acl_erp_table_enable(erp_table, ctcam_le); 506 if (err) 507 goto err_table_enable; 508 509 mlxsw_sp_acl_erp_table_free(erp_core, old_num_erps, 510 erp_table->aregion->type, old_base_index); 511 512 return 0; 513 514 err_table_enable: 515 err_table_relocate: 516 erp_table->num_max_atcam_erps = old_num_erps; 517 mlxsw_sp_acl_erp_table_free(erp_core, num_erps, 518 erp_table->aregion->type, 519 erp_table->base_index); 520 erp_table->base_index = old_base_index; 521 return err; 522 } 523 524 static int 525 mlxsw_acl_erp_table_bf_add(struct mlxsw_sp_acl_erp_table *erp_table, 526 struct mlxsw_sp_acl_erp *erp) 527 { 528 struct mlxsw_sp_acl_atcam_region *aregion = erp_table->aregion; 529 unsigned int erp_bank = mlxsw_sp_acl_erp_bank_get(erp); 530 struct mlxsw_sp_acl_atcam_entry *aentry; 531 int err; 532 533 list_for_each_entry(aentry, &aregion->entries_list, list) { 534 err = mlxsw_sp_acl_bf_entry_add(aregion->region->mlxsw_sp, 535 erp_table->erp_core->bf, 536 aregion, erp_bank, aentry); 537 if (err) 538 goto bf_entry_add_err; 539 } 540 541 return 0; 542 543 bf_entry_add_err: 544 list_for_each_entry_continue_reverse(aentry, &aregion->entries_list, 545 list) 546 mlxsw_sp_acl_bf_entry_del(aregion->region->mlxsw_sp, 547 erp_table->erp_core->bf, 548 aregion, erp_bank, aentry); 549 return err; 550 } 551 552 static void 553 mlxsw_acl_erp_table_bf_del(struct mlxsw_sp_acl_erp_table *erp_table, 554 struct mlxsw_sp_acl_erp *erp) 555 { 556 struct mlxsw_sp_acl_atcam_region *aregion = erp_table->aregion; 557 unsigned int erp_bank = mlxsw_sp_acl_erp_bank_get(erp); 558 struct mlxsw_sp_acl_atcam_entry *aentry; 559 560 list_for_each_entry_reverse(aentry, &aregion->entries_list, list) 561 mlxsw_sp_acl_bf_entry_del(aregion->region->mlxsw_sp, 562 erp_table->erp_core->bf, 563 aregion, erp_bank, aentry); 564 } 565 566 static int 567 mlxsw_sp_acl_erp_region_table_trans(struct mlxsw_sp_acl_erp_table *erp_table) 568 { 569 struct mlxsw_sp_acl_erp_core *erp_core = erp_table->erp_core; 570 struct mlxsw_sp_acl_erp *master_rp; 571 int err; 572 573 /* Initially, allocate a single eRP row. Expand later as needed */ 574 err = mlxsw_sp_acl_erp_table_alloc(erp_core, erp_core->num_erp_banks, 575 erp_table->aregion->type, 576 &erp_table->base_index); 577 if (err) 578 return err; 579 erp_table->num_max_atcam_erps = erp_core->num_erp_banks; 580 581 /* Transition the sole RP currently configured (the master RP) 582 * to the eRP table 583 */ 584 master_rp = mlxsw_sp_acl_erp_table_master_rp(erp_table); 585 if (!master_rp) { 586 err = -EINVAL; 587 goto err_table_master_rp; 588 } 589 590 /* Make sure the master RP is using a valid index, as 591 * only a single eRP row is currently allocated. 592 */ 593 master_rp->index = 0; 594 __set_bit(master_rp->index, erp_table->erp_index_bitmap); 595 596 err = mlxsw_sp_acl_erp_table_erp_add(erp_table, master_rp); 597 if (err) 598 goto err_table_master_rp_add; 599 600 /* Update Bloom filter before enabling eRP table, as rules 601 * on the master RP were not set to Bloom filter up to this 602 * point. 603 */ 604 err = mlxsw_acl_erp_table_bf_add(erp_table, master_rp); 605 if (err) 606 goto err_table_bf_add; 607 608 err = mlxsw_sp_acl_erp_table_enable(erp_table, false); 609 if (err) 610 goto err_table_enable; 611 612 return 0; 613 614 err_table_enable: 615 mlxsw_acl_erp_table_bf_del(erp_table, master_rp); 616 err_table_bf_add: 617 mlxsw_sp_acl_erp_table_erp_del(master_rp); 618 err_table_master_rp_add: 619 __clear_bit(master_rp->index, erp_table->erp_index_bitmap); 620 err_table_master_rp: 621 mlxsw_sp_acl_erp_table_free(erp_core, erp_table->num_max_atcam_erps, 622 erp_table->aregion->type, 623 erp_table->base_index); 624 return err; 625 } 626 627 static void 628 mlxsw_sp_acl_erp_region_master_mask_trans(struct mlxsw_sp_acl_erp_table *erp_table) 629 { 630 struct mlxsw_sp_acl_erp_core *erp_core = erp_table->erp_core; 631 struct mlxsw_sp_acl_erp *master_rp; 632 633 mlxsw_sp_acl_erp_table_disable(erp_table); 634 master_rp = mlxsw_sp_acl_erp_table_master_rp(erp_table); 635 if (!master_rp) 636 return; 637 mlxsw_acl_erp_table_bf_del(erp_table, master_rp); 638 mlxsw_sp_acl_erp_table_erp_del(master_rp); 639 __clear_bit(master_rp->index, erp_table->erp_index_bitmap); 640 mlxsw_sp_acl_erp_table_free(erp_core, erp_table->num_max_atcam_erps, 641 erp_table->aregion->type, 642 erp_table->base_index); 643 } 644 645 static int 646 mlxsw_sp_acl_erp_region_erp_add(struct mlxsw_sp_acl_erp_table *erp_table, 647 struct mlxsw_sp_acl_erp *erp) 648 { 649 struct mlxsw_sp_acl_tcam_region *region = erp_table->aregion->region; 650 struct mlxsw_sp *mlxsw_sp = erp_table->erp_core->mlxsw_sp; 651 bool ctcam_le = erp_table->num_ctcam_erps > 0; 652 char pererp_pl[MLXSW_REG_PERERP_LEN]; 653 654 mlxsw_reg_pererp_pack(pererp_pl, region->id, ctcam_le, true, 0, 655 erp_table->base_index, 0); 656 mlxsw_reg_pererp_erp_vector_pack(pererp_pl, erp_table->erp_index_bitmap, 657 MLXSW_SP_ACL_ERP_MAX_PER_REGION); 658 mlxsw_reg_pererp_erpt_vector_set(pererp_pl, erp->index, true); 659 660 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(pererp), pererp_pl); 661 } 662 663 static void mlxsw_sp_acl_erp_region_erp_del(struct mlxsw_sp_acl_erp *erp) 664 { 665 struct mlxsw_sp_acl_erp_table *erp_table = erp->erp_table; 666 struct mlxsw_sp_acl_tcam_region *region = erp_table->aregion->region; 667 struct mlxsw_sp *mlxsw_sp = erp_table->erp_core->mlxsw_sp; 668 bool ctcam_le = erp_table->num_ctcam_erps > 0; 669 char pererp_pl[MLXSW_REG_PERERP_LEN]; 670 671 mlxsw_reg_pererp_pack(pererp_pl, region->id, ctcam_le, true, 0, 672 erp_table->base_index, 0); 673 mlxsw_reg_pererp_erp_vector_pack(pererp_pl, erp_table->erp_index_bitmap, 674 MLXSW_SP_ACL_ERP_MAX_PER_REGION); 675 mlxsw_reg_pererp_erpt_vector_set(pererp_pl, erp->index, false); 676 677 mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(pererp), pererp_pl); 678 } 679 680 static int 681 mlxsw_sp_acl_erp_region_ctcam_enable(struct mlxsw_sp_acl_erp_table *erp_table) 682 { 683 /* No need to re-enable lookup in the C-TCAM */ 684 if (erp_table->num_ctcam_erps > 1) 685 return 0; 686 687 return mlxsw_sp_acl_erp_table_enable(erp_table, true); 688 } 689 690 static void 691 mlxsw_sp_acl_erp_region_ctcam_disable(struct mlxsw_sp_acl_erp_table *erp_table) 692 { 693 /* Only disable C-TCAM lookup when last C-TCAM eRP is deleted */ 694 if (erp_table->num_ctcam_erps > 1) 695 return; 696 697 mlxsw_sp_acl_erp_table_enable(erp_table, false); 698 } 699 700 static int 701 __mlxsw_sp_acl_erp_table_other_inc(struct mlxsw_sp_acl_erp_table *erp_table, 702 unsigned int *inc_num) 703 { 704 int err; 705 706 /* If there are C-TCAM eRP or deltas in use we need to transition 707 * the region to use eRP table, if it is not already done 708 */ 709 if (!mlxsw_sp_acl_erp_table_is_used(erp_table)) { 710 err = mlxsw_sp_acl_erp_region_table_trans(erp_table); 711 if (err) 712 return err; 713 } 714 715 /* When C-TCAM or deltas are used, the eRP table must be used */ 716 if (erp_table->ops != &erp_multiple_masks_ops) 717 erp_table->ops = &erp_multiple_masks_ops; 718 719 (*inc_num)++; 720 721 return 0; 722 } 723 724 static int mlxsw_sp_acl_erp_ctcam_inc(struct mlxsw_sp_acl_erp_table *erp_table) 725 { 726 return __mlxsw_sp_acl_erp_table_other_inc(erp_table, 727 &erp_table->num_ctcam_erps); 728 } 729 730 static int mlxsw_sp_acl_erp_delta_inc(struct mlxsw_sp_acl_erp_table *erp_table) 731 { 732 return __mlxsw_sp_acl_erp_table_other_inc(erp_table, 733 &erp_table->num_deltas); 734 } 735 736 static void 737 __mlxsw_sp_acl_erp_table_other_dec(struct mlxsw_sp_acl_erp_table *erp_table, 738 unsigned int *dec_num) 739 { 740 (*dec_num)--; 741 742 /* If there are no C-TCAM eRP or deltas in use, the state we 743 * transition to depends on the number of A-TCAM eRPs currently 744 * in use. 745 */ 746 if (erp_table->num_ctcam_erps > 0 || erp_table->num_deltas > 0) 747 return; 748 749 switch (erp_table->num_atcam_erps) { 750 case 2: 751 /* Keep using the eRP table, but correctly set the 752 * operations pointer so that when an A-TCAM eRP is 753 * deleted we will transition to use the master mask 754 */ 755 erp_table->ops = &erp_two_masks_ops; 756 break; 757 case 1: 758 /* We only kept the eRP table because we had C-TCAM 759 * eRPs in use. Now that the last C-TCAM eRP is gone we 760 * can stop using the table and transition to use the 761 * master mask 762 */ 763 mlxsw_sp_acl_erp_region_master_mask_trans(erp_table); 764 erp_table->ops = &erp_single_mask_ops; 765 break; 766 case 0: 767 /* There are no more eRPs of any kind used by the region 768 * so free its eRP table and transition to initial state 769 */ 770 mlxsw_sp_acl_erp_table_disable(erp_table); 771 mlxsw_sp_acl_erp_table_free(erp_table->erp_core, 772 erp_table->num_max_atcam_erps, 773 erp_table->aregion->type, 774 erp_table->base_index); 775 erp_table->ops = &erp_no_mask_ops; 776 break; 777 default: 778 break; 779 } 780 } 781 782 static void mlxsw_sp_acl_erp_ctcam_dec(struct mlxsw_sp_acl_erp_table *erp_table) 783 { 784 __mlxsw_sp_acl_erp_table_other_dec(erp_table, 785 &erp_table->num_ctcam_erps); 786 } 787 788 static void mlxsw_sp_acl_erp_delta_dec(struct mlxsw_sp_acl_erp_table *erp_table) 789 { 790 __mlxsw_sp_acl_erp_table_other_dec(erp_table, 791 &erp_table->num_deltas); 792 } 793 794 static struct mlxsw_sp_acl_erp * 795 mlxsw_sp_acl_erp_ctcam_mask_create(struct mlxsw_sp_acl_erp_table *erp_table, 796 struct mlxsw_sp_acl_erp_key *key) 797 { 798 struct mlxsw_sp_acl_erp *erp; 799 int err; 800 801 erp = kzalloc(sizeof(*erp), GFP_KERNEL); 802 if (!erp) 803 return ERR_PTR(-ENOMEM); 804 805 memcpy(&erp->key, key, sizeof(*key)); 806 bitmap_from_arr32(erp->mask_bitmap, (u32 *) key->mask, 807 MLXSW_SP_ACL_TCAM_MASK_LEN); 808 809 err = mlxsw_sp_acl_erp_ctcam_inc(erp_table); 810 if (err) 811 goto err_erp_ctcam_inc; 812 813 erp->erp_table = erp_table; 814 815 err = mlxsw_sp_acl_erp_master_mask_set(erp_table, &erp->key); 816 if (err) 817 goto err_master_mask_set; 818 819 err = mlxsw_sp_acl_erp_region_ctcam_enable(erp_table); 820 if (err) 821 goto err_erp_region_ctcam_enable; 822 823 return erp; 824 825 err_erp_region_ctcam_enable: 826 mlxsw_sp_acl_erp_master_mask_clear(erp_table, &erp->key); 827 err_master_mask_set: 828 mlxsw_sp_acl_erp_ctcam_dec(erp_table); 829 err_erp_ctcam_inc: 830 kfree(erp); 831 return ERR_PTR(err); 832 } 833 834 static void 835 mlxsw_sp_acl_erp_ctcam_mask_destroy(struct mlxsw_sp_acl_erp *erp) 836 { 837 struct mlxsw_sp_acl_erp_table *erp_table = erp->erp_table; 838 839 mlxsw_sp_acl_erp_region_ctcam_disable(erp_table); 840 mlxsw_sp_acl_erp_master_mask_clear(erp_table, &erp->key); 841 mlxsw_sp_acl_erp_ctcam_dec(erp_table); 842 kfree(erp); 843 } 844 845 static struct mlxsw_sp_acl_erp * 846 mlxsw_sp_acl_erp_mask_create(struct mlxsw_sp_acl_erp_table *erp_table, 847 struct mlxsw_sp_acl_erp_key *key) 848 { 849 struct mlxsw_sp_acl_erp *erp; 850 int err; 851 852 if (key->ctcam) 853 return mlxsw_sp_acl_erp_ctcam_mask_create(erp_table, key); 854 855 /* Expand the eRP table for the new eRP, if needed */ 856 err = mlxsw_sp_acl_erp_table_expand(erp_table); 857 if (err) 858 return ERR_PTR(err); 859 860 erp = mlxsw_sp_acl_erp_generic_create(erp_table, key); 861 if (IS_ERR(erp)) 862 return erp; 863 864 err = mlxsw_sp_acl_erp_index_get(erp_table, &erp->index); 865 if (err) 866 goto err_erp_index_get; 867 868 err = mlxsw_sp_acl_erp_table_erp_add(erp_table, erp); 869 if (err) 870 goto err_table_erp_add; 871 872 err = mlxsw_sp_acl_erp_region_erp_add(erp_table, erp); 873 if (err) 874 goto err_region_erp_add; 875 876 erp_table->ops = &erp_multiple_masks_ops; 877 878 return erp; 879 880 err_region_erp_add: 881 mlxsw_sp_acl_erp_table_erp_del(erp); 882 err_table_erp_add: 883 mlxsw_sp_acl_erp_index_put(erp_table, erp->index); 884 err_erp_index_get: 885 mlxsw_sp_acl_erp_generic_destroy(erp); 886 return ERR_PTR(err); 887 } 888 889 static void 890 mlxsw_sp_acl_erp_mask_destroy(struct mlxsw_sp_acl_erp_table *erp_table, 891 struct mlxsw_sp_acl_erp *erp) 892 { 893 if (erp->key.ctcam) 894 return mlxsw_sp_acl_erp_ctcam_mask_destroy(erp); 895 896 mlxsw_sp_acl_erp_region_erp_del(erp); 897 mlxsw_sp_acl_erp_table_erp_del(erp); 898 mlxsw_sp_acl_erp_index_put(erp_table, erp->index); 899 mlxsw_sp_acl_erp_generic_destroy(erp); 900 901 if (erp_table->num_atcam_erps == 2 && erp_table->num_ctcam_erps == 0 && 902 erp_table->num_deltas == 0) 903 erp_table->ops = &erp_two_masks_ops; 904 } 905 906 static struct mlxsw_sp_acl_erp * 907 mlxsw_sp_acl_erp_second_mask_create(struct mlxsw_sp_acl_erp_table *erp_table, 908 struct mlxsw_sp_acl_erp_key *key) 909 { 910 struct mlxsw_sp_acl_erp *erp; 911 int err; 912 913 if (key->ctcam) 914 return mlxsw_sp_acl_erp_ctcam_mask_create(erp_table, key); 915 916 /* Transition to use eRP table instead of master mask */ 917 err = mlxsw_sp_acl_erp_region_table_trans(erp_table); 918 if (err) 919 return ERR_PTR(err); 920 921 erp = mlxsw_sp_acl_erp_generic_create(erp_table, key); 922 if (IS_ERR(erp)) { 923 err = PTR_ERR(erp); 924 goto err_erp_create; 925 } 926 927 err = mlxsw_sp_acl_erp_index_get(erp_table, &erp->index); 928 if (err) 929 goto err_erp_index_get; 930 931 err = mlxsw_sp_acl_erp_table_erp_add(erp_table, erp); 932 if (err) 933 goto err_table_erp_add; 934 935 err = mlxsw_sp_acl_erp_region_erp_add(erp_table, erp); 936 if (err) 937 goto err_region_erp_add; 938 939 erp_table->ops = &erp_two_masks_ops; 940 941 return erp; 942 943 err_region_erp_add: 944 mlxsw_sp_acl_erp_table_erp_del(erp); 945 err_table_erp_add: 946 mlxsw_sp_acl_erp_index_put(erp_table, erp->index); 947 err_erp_index_get: 948 mlxsw_sp_acl_erp_generic_destroy(erp); 949 err_erp_create: 950 mlxsw_sp_acl_erp_region_master_mask_trans(erp_table); 951 return ERR_PTR(err); 952 } 953 954 static void 955 mlxsw_sp_acl_erp_second_mask_destroy(struct mlxsw_sp_acl_erp_table *erp_table, 956 struct mlxsw_sp_acl_erp *erp) 957 { 958 if (erp->key.ctcam) 959 return mlxsw_sp_acl_erp_ctcam_mask_destroy(erp); 960 961 mlxsw_sp_acl_erp_region_erp_del(erp); 962 mlxsw_sp_acl_erp_table_erp_del(erp); 963 mlxsw_sp_acl_erp_index_put(erp_table, erp->index); 964 mlxsw_sp_acl_erp_generic_destroy(erp); 965 /* Transition to use master mask instead of eRP table */ 966 mlxsw_sp_acl_erp_region_master_mask_trans(erp_table); 967 968 erp_table->ops = &erp_single_mask_ops; 969 } 970 971 static struct mlxsw_sp_acl_erp * 972 mlxsw_sp_acl_erp_first_mask_create(struct mlxsw_sp_acl_erp_table *erp_table, 973 struct mlxsw_sp_acl_erp_key *key) 974 { 975 struct mlxsw_sp_acl_erp *erp; 976 977 if (key->ctcam) 978 return ERR_PTR(-EINVAL); 979 980 erp = mlxsw_sp_acl_erp_generic_create(erp_table, key); 981 if (IS_ERR(erp)) 982 return erp; 983 984 erp_table->ops = &erp_single_mask_ops; 985 986 return erp; 987 } 988 989 static void 990 mlxsw_sp_acl_erp_first_mask_destroy(struct mlxsw_sp_acl_erp_table *erp_table, 991 struct mlxsw_sp_acl_erp *erp) 992 { 993 mlxsw_sp_acl_erp_generic_destroy(erp); 994 erp_table->ops = &erp_no_mask_ops; 995 } 996 997 static void 998 mlxsw_sp_acl_erp_no_mask_destroy(struct mlxsw_sp_acl_erp_table *erp_table, 999 struct mlxsw_sp_acl_erp *erp) 1000 { 1001 WARN_ON(1); 1002 } 1003 1004 struct mlxsw_sp_acl_erp_mask * 1005 mlxsw_sp_acl_erp_mask_get(struct mlxsw_sp_acl_atcam_region *aregion, 1006 const char *mask, bool ctcam) 1007 { 1008 struct mlxsw_sp_acl_erp_table *erp_table = aregion->erp_table; 1009 struct mlxsw_sp_acl_erp_key key; 1010 struct objagg_obj *objagg_obj; 1011 1012 memcpy(key.mask, mask, MLXSW_REG_PTCEX_FLEX_KEY_BLOCKS_LEN); 1013 key.ctcam = ctcam; 1014 mutex_lock(&erp_table->objagg_lock); 1015 objagg_obj = objagg_obj_get(erp_table->objagg, &key); 1016 mutex_unlock(&erp_table->objagg_lock); 1017 if (IS_ERR(objagg_obj)) 1018 return ERR_CAST(objagg_obj); 1019 return (struct mlxsw_sp_acl_erp_mask *) objagg_obj; 1020 } 1021 1022 void mlxsw_sp_acl_erp_mask_put(struct mlxsw_sp_acl_atcam_region *aregion, 1023 struct mlxsw_sp_acl_erp_mask *erp_mask) 1024 { 1025 struct objagg_obj *objagg_obj = (struct objagg_obj *) erp_mask; 1026 struct mlxsw_sp_acl_erp_table *erp_table = aregion->erp_table; 1027 1028 mutex_lock(&erp_table->objagg_lock); 1029 objagg_obj_put(erp_table->objagg, objagg_obj); 1030 mutex_unlock(&erp_table->objagg_lock); 1031 } 1032 1033 int mlxsw_sp_acl_erp_bf_insert(struct mlxsw_sp *mlxsw_sp, 1034 struct mlxsw_sp_acl_atcam_region *aregion, 1035 struct mlxsw_sp_acl_erp_mask *erp_mask, 1036 struct mlxsw_sp_acl_atcam_entry *aentry) 1037 { 1038 struct objagg_obj *objagg_obj = (struct objagg_obj *) erp_mask; 1039 const struct mlxsw_sp_acl_erp *erp = objagg_obj_root_priv(objagg_obj); 1040 unsigned int erp_bank; 1041 1042 if (!mlxsw_sp_acl_erp_table_is_used(erp->erp_table)) 1043 return 0; 1044 1045 erp_bank = mlxsw_sp_acl_erp_bank_get(erp); 1046 return mlxsw_sp_acl_bf_entry_add(mlxsw_sp, 1047 erp->erp_table->erp_core->bf, 1048 aregion, erp_bank, aentry); 1049 } 1050 1051 void mlxsw_sp_acl_erp_bf_remove(struct mlxsw_sp *mlxsw_sp, 1052 struct mlxsw_sp_acl_atcam_region *aregion, 1053 struct mlxsw_sp_acl_erp_mask *erp_mask, 1054 struct mlxsw_sp_acl_atcam_entry *aentry) 1055 { 1056 struct objagg_obj *objagg_obj = (struct objagg_obj *) erp_mask; 1057 const struct mlxsw_sp_acl_erp *erp = objagg_obj_root_priv(objagg_obj); 1058 unsigned int erp_bank; 1059 1060 if (!mlxsw_sp_acl_erp_table_is_used(erp->erp_table)) 1061 return; 1062 1063 erp_bank = mlxsw_sp_acl_erp_bank_get(erp); 1064 mlxsw_sp_acl_bf_entry_del(mlxsw_sp, 1065 erp->erp_table->erp_core->bf, 1066 aregion, erp_bank, aentry); 1067 } 1068 1069 bool 1070 mlxsw_sp_acl_erp_mask_is_ctcam(const struct mlxsw_sp_acl_erp_mask *erp_mask) 1071 { 1072 struct objagg_obj *objagg_obj = (struct objagg_obj *) erp_mask; 1073 const struct mlxsw_sp_acl_erp_key *key = objagg_obj_raw(objagg_obj); 1074 1075 return key->ctcam; 1076 } 1077 1078 u8 mlxsw_sp_acl_erp_mask_erp_id(const struct mlxsw_sp_acl_erp_mask *erp_mask) 1079 { 1080 struct objagg_obj *objagg_obj = (struct objagg_obj *) erp_mask; 1081 const struct mlxsw_sp_acl_erp *erp = objagg_obj_root_priv(objagg_obj); 1082 1083 return erp->id; 1084 } 1085 1086 struct mlxsw_sp_acl_erp_delta { 1087 struct mlxsw_sp_acl_erp_key key; 1088 u16 start; 1089 u8 mask; 1090 }; 1091 1092 u16 mlxsw_sp_acl_erp_delta_start(const struct mlxsw_sp_acl_erp_delta *delta) 1093 { 1094 return delta->start; 1095 } 1096 1097 u8 mlxsw_sp_acl_erp_delta_mask(const struct mlxsw_sp_acl_erp_delta *delta) 1098 { 1099 return delta->mask; 1100 } 1101 1102 u8 mlxsw_sp_acl_erp_delta_value(const struct mlxsw_sp_acl_erp_delta *delta, 1103 const char *enc_key) 1104 { 1105 u16 start = delta->start; 1106 u8 mask = delta->mask; 1107 u16 tmp; 1108 1109 if (!mask) 1110 return 0; 1111 1112 tmp = (unsigned char) enc_key[__MASK_IDX(start / 8)]; 1113 if (start / 8 + 1 < __MASK_LEN) 1114 tmp |= (unsigned char) enc_key[__MASK_IDX(start / 8 + 1)] << 8; 1115 tmp >>= start % 8; 1116 tmp &= mask; 1117 return tmp; 1118 } 1119 1120 void mlxsw_sp_acl_erp_delta_clear(const struct mlxsw_sp_acl_erp_delta *delta, 1121 const char *enc_key) 1122 { 1123 u16 start = delta->start; 1124 u8 mask = delta->mask; 1125 unsigned char *byte; 1126 u16 tmp; 1127 1128 tmp = mask; 1129 tmp <<= start % 8; 1130 tmp = ~tmp; 1131 1132 byte = (unsigned char *) &enc_key[__MASK_IDX(start / 8)]; 1133 *byte &= tmp & 0xff; 1134 if (start / 8 + 1 < __MASK_LEN) { 1135 byte = (unsigned char *) &enc_key[__MASK_IDX(start / 8 + 1)]; 1136 *byte &= (tmp >> 8) & 0xff; 1137 } 1138 } 1139 1140 static const struct mlxsw_sp_acl_erp_delta 1141 mlxsw_sp_acl_erp_delta_default = {}; 1142 1143 const struct mlxsw_sp_acl_erp_delta * 1144 mlxsw_sp_acl_erp_delta(const struct mlxsw_sp_acl_erp_mask *erp_mask) 1145 { 1146 struct objagg_obj *objagg_obj = (struct objagg_obj *) erp_mask; 1147 const struct mlxsw_sp_acl_erp_delta *delta; 1148 1149 delta = objagg_obj_delta_priv(objagg_obj); 1150 if (!delta) 1151 delta = &mlxsw_sp_acl_erp_delta_default; 1152 return delta; 1153 } 1154 1155 static int 1156 mlxsw_sp_acl_erp_delta_fill(const struct mlxsw_sp_acl_erp_key *parent_key, 1157 const struct mlxsw_sp_acl_erp_key *key, 1158 u16 *delta_start, u8 *delta_mask) 1159 { 1160 int offset = 0; 1161 int si = -1; 1162 u16 pmask; 1163 u16 mask; 1164 int i; 1165 1166 /* The difference between 2 masks can be up to 8 consecutive bits. */ 1167 for (i = 0; i < __MASK_LEN; i++) { 1168 if (parent_key->mask[__MASK_IDX(i)] == key->mask[__MASK_IDX(i)]) 1169 continue; 1170 if (si == -1) 1171 si = i; 1172 else if (si != i - 1) 1173 return -EINVAL; 1174 } 1175 if (si == -1) { 1176 /* The masks are the same, this can happen in case eRPs with 1177 * the same mask were created in both A-TCAM and C-TCAM. 1178 * The only possible condition under which this can happen 1179 * is identical rule insertion. Delta is not possible here. 1180 */ 1181 return -EINVAL; 1182 } 1183 pmask = (unsigned char) parent_key->mask[__MASK_IDX(si)]; 1184 mask = (unsigned char) key->mask[__MASK_IDX(si)]; 1185 if (si + 1 < __MASK_LEN) { 1186 pmask |= (unsigned char) parent_key->mask[__MASK_IDX(si + 1)] << 8; 1187 mask |= (unsigned char) key->mask[__MASK_IDX(si + 1)] << 8; 1188 } 1189 1190 if ((pmask ^ mask) & pmask) 1191 return -EINVAL; 1192 mask &= ~pmask; 1193 while (!(mask & (1 << offset))) 1194 offset++; 1195 while (!(mask & 1)) 1196 mask >>= 1; 1197 if (mask & 0xff00) 1198 return -EINVAL; 1199 1200 *delta_start = si * 8 + offset; 1201 *delta_mask = mask; 1202 1203 return 0; 1204 } 1205 1206 static bool mlxsw_sp_acl_erp_delta_check(void *priv, const void *parent_obj, 1207 const void *obj) 1208 { 1209 const struct mlxsw_sp_acl_erp_key *parent_key = parent_obj; 1210 const struct mlxsw_sp_acl_erp_key *key = obj; 1211 u16 delta_start; 1212 u8 delta_mask; 1213 int err; 1214 1215 err = mlxsw_sp_acl_erp_delta_fill(parent_key, key, 1216 &delta_start, &delta_mask); 1217 return err ? false : true; 1218 } 1219 1220 static void *mlxsw_sp_acl_erp_delta_create(void *priv, void *parent_obj, 1221 void *obj) 1222 { 1223 struct mlxsw_sp_acl_erp_key *parent_key = parent_obj; 1224 struct mlxsw_sp_acl_atcam_region *aregion = priv; 1225 struct mlxsw_sp_acl_erp_table *erp_table = aregion->erp_table; 1226 struct mlxsw_sp_acl_erp_key *key = obj; 1227 struct mlxsw_sp_acl_erp_delta *delta; 1228 u16 delta_start; 1229 u8 delta_mask; 1230 int err; 1231 1232 if (parent_key->ctcam || key->ctcam) 1233 return ERR_PTR(-EINVAL); 1234 err = mlxsw_sp_acl_erp_delta_fill(parent_key, key, 1235 &delta_start, &delta_mask); 1236 if (err) 1237 return ERR_PTR(-EINVAL); 1238 1239 delta = kzalloc(sizeof(*delta), GFP_KERNEL); 1240 if (!delta) 1241 return ERR_PTR(-ENOMEM); 1242 delta->start = delta_start; 1243 delta->mask = delta_mask; 1244 1245 err = mlxsw_sp_acl_erp_delta_inc(erp_table); 1246 if (err) 1247 goto err_erp_delta_inc; 1248 1249 memcpy(&delta->key, key, sizeof(*key)); 1250 err = mlxsw_sp_acl_erp_master_mask_set(erp_table, &delta->key); 1251 if (err) 1252 goto err_master_mask_set; 1253 1254 return delta; 1255 1256 err_master_mask_set: 1257 mlxsw_sp_acl_erp_delta_dec(erp_table); 1258 err_erp_delta_inc: 1259 kfree(delta); 1260 return ERR_PTR(err); 1261 } 1262 1263 static void mlxsw_sp_acl_erp_delta_destroy(void *priv, void *delta_priv) 1264 { 1265 struct mlxsw_sp_acl_erp_delta *delta = delta_priv; 1266 struct mlxsw_sp_acl_atcam_region *aregion = priv; 1267 struct mlxsw_sp_acl_erp_table *erp_table = aregion->erp_table; 1268 1269 mlxsw_sp_acl_erp_master_mask_clear(erp_table, &delta->key); 1270 mlxsw_sp_acl_erp_delta_dec(erp_table); 1271 kfree(delta); 1272 } 1273 1274 static void *mlxsw_sp_acl_erp_root_create(void *priv, void *obj, 1275 unsigned int root_id) 1276 { 1277 struct mlxsw_sp_acl_atcam_region *aregion = priv; 1278 struct mlxsw_sp_acl_erp_table *erp_table = aregion->erp_table; 1279 struct mlxsw_sp_acl_erp_key *key = obj; 1280 1281 if (!key->ctcam && 1282 root_id != OBJAGG_OBJ_ROOT_ID_INVALID && 1283 root_id >= MLXSW_SP_ACL_ERP_MAX_PER_REGION) 1284 return ERR_PTR(-ENOBUFS); 1285 return erp_table->ops->erp_create(erp_table, key); 1286 } 1287 1288 static void mlxsw_sp_acl_erp_root_destroy(void *priv, void *root_priv) 1289 { 1290 struct mlxsw_sp_acl_atcam_region *aregion = priv; 1291 struct mlxsw_sp_acl_erp_table *erp_table = aregion->erp_table; 1292 1293 erp_table->ops->erp_destroy(erp_table, root_priv); 1294 } 1295 1296 static const struct objagg_ops mlxsw_sp_acl_erp_objagg_ops = { 1297 .obj_size = sizeof(struct mlxsw_sp_acl_erp_key), 1298 .delta_check = mlxsw_sp_acl_erp_delta_check, 1299 .delta_create = mlxsw_sp_acl_erp_delta_create, 1300 .delta_destroy = mlxsw_sp_acl_erp_delta_destroy, 1301 .root_create = mlxsw_sp_acl_erp_root_create, 1302 .root_destroy = mlxsw_sp_acl_erp_root_destroy, 1303 }; 1304 1305 static struct mlxsw_sp_acl_erp_table * 1306 mlxsw_sp_acl_erp_table_create(struct mlxsw_sp_acl_atcam_region *aregion, 1307 struct objagg_hints *hints) 1308 { 1309 struct mlxsw_sp_acl_erp_table *erp_table; 1310 int err; 1311 1312 erp_table = kzalloc(sizeof(*erp_table), GFP_KERNEL); 1313 if (!erp_table) 1314 return ERR_PTR(-ENOMEM); 1315 1316 erp_table->objagg = objagg_create(&mlxsw_sp_acl_erp_objagg_ops, 1317 hints, aregion); 1318 if (IS_ERR(erp_table->objagg)) { 1319 err = PTR_ERR(erp_table->objagg); 1320 goto err_objagg_create; 1321 } 1322 1323 erp_table->erp_core = aregion->atcam->erp_core; 1324 erp_table->ops = &erp_no_mask_ops; 1325 INIT_LIST_HEAD(&erp_table->atcam_erps_list); 1326 erp_table->aregion = aregion; 1327 mutex_init(&erp_table->objagg_lock); 1328 1329 return erp_table; 1330 1331 err_objagg_create: 1332 kfree(erp_table); 1333 return ERR_PTR(err); 1334 } 1335 1336 static void 1337 mlxsw_sp_acl_erp_table_destroy(struct mlxsw_sp_acl_erp_table *erp_table) 1338 { 1339 WARN_ON(!list_empty(&erp_table->atcam_erps_list)); 1340 mutex_destroy(&erp_table->objagg_lock); 1341 objagg_destroy(erp_table->objagg); 1342 kfree(erp_table); 1343 } 1344 1345 static int 1346 mlxsw_sp_acl_erp_master_mask_init(struct mlxsw_sp_acl_atcam_region *aregion) 1347 { 1348 struct mlxsw_sp *mlxsw_sp = aregion->region->mlxsw_sp; 1349 char percr_pl[MLXSW_REG_PERCR_LEN]; 1350 1351 mlxsw_reg_percr_pack(percr_pl, aregion->region->id); 1352 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(percr), percr_pl); 1353 } 1354 1355 static int 1356 mlxsw_sp_acl_erp_region_param_init(struct mlxsw_sp_acl_atcam_region *aregion) 1357 { 1358 struct mlxsw_sp *mlxsw_sp = aregion->region->mlxsw_sp; 1359 char pererp_pl[MLXSW_REG_PERERP_LEN]; 1360 1361 mlxsw_reg_pererp_pack(pererp_pl, aregion->region->id, false, false, 0, 1362 0, 0); 1363 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(pererp), pererp_pl); 1364 } 1365 1366 static int 1367 mlxsw_sp_acl_erp_hints_check(struct mlxsw_sp *mlxsw_sp, 1368 struct mlxsw_sp_acl_atcam_region *aregion, 1369 struct objagg_hints *hints, bool *p_rehash_needed) 1370 { 1371 struct mlxsw_sp_acl_erp_table *erp_table = aregion->erp_table; 1372 const struct objagg_stats *ostats; 1373 const struct objagg_stats *hstats; 1374 int err; 1375 1376 *p_rehash_needed = false; 1377 1378 mutex_lock(&erp_table->objagg_lock); 1379 ostats = objagg_stats_get(erp_table->objagg); 1380 mutex_unlock(&erp_table->objagg_lock); 1381 if (IS_ERR(ostats)) { 1382 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Failed to get ERP stats\n"); 1383 return PTR_ERR(ostats); 1384 } 1385 1386 hstats = objagg_hints_stats_get(hints); 1387 if (IS_ERR(hstats)) { 1388 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Failed to get ERP hints stats\n"); 1389 err = PTR_ERR(hstats); 1390 goto err_hints_stats_get; 1391 } 1392 1393 /* Very basic criterion for now. */ 1394 if (hstats->root_count < ostats->root_count) 1395 *p_rehash_needed = true; 1396 1397 err = 0; 1398 1399 objagg_stats_put(hstats); 1400 err_hints_stats_get: 1401 objagg_stats_put(ostats); 1402 return err; 1403 } 1404 1405 void * 1406 mlxsw_sp_acl_erp_rehash_hints_get(struct mlxsw_sp_acl_atcam_region *aregion) 1407 { 1408 struct mlxsw_sp_acl_erp_table *erp_table = aregion->erp_table; 1409 struct mlxsw_sp *mlxsw_sp = aregion->region->mlxsw_sp; 1410 struct objagg_hints *hints; 1411 bool rehash_needed; 1412 int err; 1413 1414 mutex_lock(&erp_table->objagg_lock); 1415 hints = objagg_hints_get(erp_table->objagg, 1416 OBJAGG_OPT_ALGO_SIMPLE_GREEDY); 1417 mutex_unlock(&erp_table->objagg_lock); 1418 if (IS_ERR(hints)) { 1419 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Failed to create ERP hints\n"); 1420 return ERR_CAST(hints); 1421 } 1422 err = mlxsw_sp_acl_erp_hints_check(mlxsw_sp, aregion, hints, 1423 &rehash_needed); 1424 if (err) 1425 goto errout; 1426 1427 if (!rehash_needed) { 1428 err = -EAGAIN; 1429 goto errout; 1430 } 1431 return hints; 1432 1433 errout: 1434 objagg_hints_put(hints); 1435 return ERR_PTR(err); 1436 } 1437 1438 void mlxsw_sp_acl_erp_rehash_hints_put(void *hints_priv) 1439 { 1440 struct objagg_hints *hints = hints_priv; 1441 1442 objagg_hints_put(hints); 1443 } 1444 1445 int mlxsw_sp_acl_erp_region_init(struct mlxsw_sp_acl_atcam_region *aregion, 1446 void *hints_priv) 1447 { 1448 struct mlxsw_sp_acl_erp_table *erp_table; 1449 struct objagg_hints *hints = hints_priv; 1450 int err; 1451 1452 erp_table = mlxsw_sp_acl_erp_table_create(aregion, hints); 1453 if (IS_ERR(erp_table)) 1454 return PTR_ERR(erp_table); 1455 aregion->erp_table = erp_table; 1456 1457 /* Initialize the region's master mask to all zeroes */ 1458 err = mlxsw_sp_acl_erp_master_mask_init(aregion); 1459 if (err) 1460 goto err_erp_master_mask_init; 1461 1462 /* Initialize the region to not use the eRP table */ 1463 err = mlxsw_sp_acl_erp_region_param_init(aregion); 1464 if (err) 1465 goto err_erp_region_param_init; 1466 1467 return 0; 1468 1469 err_erp_region_param_init: 1470 err_erp_master_mask_init: 1471 mlxsw_sp_acl_erp_table_destroy(erp_table); 1472 return err; 1473 } 1474 1475 void mlxsw_sp_acl_erp_region_fini(struct mlxsw_sp_acl_atcam_region *aregion) 1476 { 1477 mlxsw_sp_acl_erp_table_destroy(aregion->erp_table); 1478 } 1479 1480 static int 1481 mlxsw_sp_acl_erp_tables_sizes_query(struct mlxsw_sp *mlxsw_sp, 1482 struct mlxsw_sp_acl_erp_core *erp_core) 1483 { 1484 unsigned int size; 1485 1486 if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, ACL_ERPT_ENTRIES_2KB) || 1487 !MLXSW_CORE_RES_VALID(mlxsw_sp->core, ACL_ERPT_ENTRIES_4KB) || 1488 !MLXSW_CORE_RES_VALID(mlxsw_sp->core, ACL_ERPT_ENTRIES_8KB) || 1489 !MLXSW_CORE_RES_VALID(mlxsw_sp->core, ACL_ERPT_ENTRIES_12KB)) 1490 return -EIO; 1491 1492 size = MLXSW_CORE_RES_GET(mlxsw_sp->core, ACL_ERPT_ENTRIES_2KB); 1493 erp_core->erpt_entries_size[MLXSW_SP_ACL_ATCAM_REGION_TYPE_2KB] = size; 1494 1495 size = MLXSW_CORE_RES_GET(mlxsw_sp->core, ACL_ERPT_ENTRIES_4KB); 1496 erp_core->erpt_entries_size[MLXSW_SP_ACL_ATCAM_REGION_TYPE_4KB] = size; 1497 1498 size = MLXSW_CORE_RES_GET(mlxsw_sp->core, ACL_ERPT_ENTRIES_8KB); 1499 erp_core->erpt_entries_size[MLXSW_SP_ACL_ATCAM_REGION_TYPE_8KB] = size; 1500 1501 size = MLXSW_CORE_RES_GET(mlxsw_sp->core, ACL_ERPT_ENTRIES_12KB); 1502 erp_core->erpt_entries_size[MLXSW_SP_ACL_ATCAM_REGION_TYPE_12KB] = size; 1503 1504 return 0; 1505 } 1506 1507 static int mlxsw_sp_acl_erp_tables_init(struct mlxsw_sp *mlxsw_sp, 1508 struct mlxsw_sp_acl_erp_core *erp_core) 1509 { 1510 unsigned int erpt_bank_size; 1511 int err; 1512 1513 if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, ACL_MAX_ERPT_BANK_SIZE) || 1514 !MLXSW_CORE_RES_VALID(mlxsw_sp->core, ACL_MAX_ERPT_BANKS)) 1515 return -EIO; 1516 erpt_bank_size = MLXSW_CORE_RES_GET(mlxsw_sp->core, 1517 ACL_MAX_ERPT_BANK_SIZE); 1518 erp_core->num_erp_banks = MLXSW_CORE_RES_GET(mlxsw_sp->core, 1519 ACL_MAX_ERPT_BANKS); 1520 1521 erp_core->erp_tables = gen_pool_create(0, -1); 1522 if (!erp_core->erp_tables) 1523 return -ENOMEM; 1524 gen_pool_set_algo(erp_core->erp_tables, gen_pool_best_fit, NULL); 1525 1526 err = gen_pool_add(erp_core->erp_tables, 1527 MLXSW_SP_ACL_ERP_GENALLOC_OFFSET, erpt_bank_size, 1528 -1); 1529 if (err) 1530 goto err_gen_pool_add; 1531 1532 erp_core->bf = mlxsw_sp_acl_bf_init(mlxsw_sp, erp_core->num_erp_banks); 1533 if (IS_ERR(erp_core->bf)) { 1534 err = PTR_ERR(erp_core->bf); 1535 goto err_bf_init; 1536 } 1537 1538 /* Different regions require masks of different sizes */ 1539 err = mlxsw_sp_acl_erp_tables_sizes_query(mlxsw_sp, erp_core); 1540 if (err) 1541 goto err_erp_tables_sizes_query; 1542 1543 return 0; 1544 1545 err_erp_tables_sizes_query: 1546 mlxsw_sp_acl_bf_fini(erp_core->bf); 1547 err_bf_init: 1548 err_gen_pool_add: 1549 gen_pool_destroy(erp_core->erp_tables); 1550 return err; 1551 } 1552 1553 static void mlxsw_sp_acl_erp_tables_fini(struct mlxsw_sp *mlxsw_sp, 1554 struct mlxsw_sp_acl_erp_core *erp_core) 1555 { 1556 mlxsw_sp_acl_bf_fini(erp_core->bf); 1557 gen_pool_destroy(erp_core->erp_tables); 1558 } 1559 1560 int mlxsw_sp_acl_erps_init(struct mlxsw_sp *mlxsw_sp, 1561 struct mlxsw_sp_acl_atcam *atcam) 1562 { 1563 struct mlxsw_sp_acl_erp_core *erp_core; 1564 int err; 1565 1566 erp_core = kzalloc(sizeof(*erp_core), GFP_KERNEL); 1567 if (!erp_core) 1568 return -ENOMEM; 1569 erp_core->mlxsw_sp = mlxsw_sp; 1570 atcam->erp_core = erp_core; 1571 1572 err = mlxsw_sp_acl_erp_tables_init(mlxsw_sp, erp_core); 1573 if (err) 1574 goto err_erp_tables_init; 1575 1576 return 0; 1577 1578 err_erp_tables_init: 1579 kfree(erp_core); 1580 return err; 1581 } 1582 1583 void mlxsw_sp_acl_erps_fini(struct mlxsw_sp *mlxsw_sp, 1584 struct mlxsw_sp_acl_atcam *atcam) 1585 { 1586 mlxsw_sp_acl_erp_tables_fini(mlxsw_sp, atcam->erp_core); 1587 kfree(atcam->erp_core); 1588 } 1589