1 // SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0 2 /* Copyright (c) 2018 Mellanox Technologies. All rights reserved */ 3 4 #include <linux/bitmap.h> 5 #include <linux/errno.h> 6 #include <linux/genalloc.h> 7 #include <linux/gfp.h> 8 #include <linux/kernel.h> 9 #include <linux/list.h> 10 #include <linux/objagg.h> 11 #include <linux/rtnetlink.h> 12 #include <linux/slab.h> 13 14 #include "core.h" 15 #include "reg.h" 16 #include "spectrum.h" 17 #include "spectrum_acl_tcam.h" 18 19 /* gen_pool_alloc() returns 0 when allocation fails, so use an offset */ 20 #define MLXSW_SP_ACL_ERP_GENALLOC_OFFSET 0x100 21 #define MLXSW_SP_ACL_ERP_MAX_PER_REGION 16 22 23 struct mlxsw_sp_acl_erp_core { 24 unsigned int erpt_entries_size[MLXSW_SP_ACL_ATCAM_REGION_TYPE_MAX + 1]; 25 struct gen_pool *erp_tables; 26 struct mlxsw_sp *mlxsw_sp; 27 struct mlxsw_sp_acl_bf *bf; 28 unsigned int num_erp_banks; 29 }; 30 31 struct mlxsw_sp_acl_erp_key { 32 char mask[MLXSW_REG_PTCEX_FLEX_KEY_BLOCKS_LEN]; 33 #define __MASK_LEN 0x38 34 #define __MASK_IDX(i) (__MASK_LEN - (i) - 1) 35 bool ctcam; 36 }; 37 38 struct mlxsw_sp_acl_erp { 39 struct mlxsw_sp_acl_erp_key key; 40 u8 id; 41 u8 index; 42 DECLARE_BITMAP(mask_bitmap, MLXSW_SP_ACL_TCAM_MASK_LEN); 43 struct list_head list; 44 struct mlxsw_sp_acl_erp_table *erp_table; 45 }; 46 47 struct mlxsw_sp_acl_erp_master_mask { 48 DECLARE_BITMAP(bitmap, MLXSW_SP_ACL_TCAM_MASK_LEN); 49 unsigned int count[MLXSW_SP_ACL_TCAM_MASK_LEN]; 50 }; 51 52 struct mlxsw_sp_acl_erp_table { 53 struct mlxsw_sp_acl_erp_master_mask master_mask; 54 DECLARE_BITMAP(erp_id_bitmap, MLXSW_SP_ACL_ERP_MAX_PER_REGION); 55 DECLARE_BITMAP(erp_index_bitmap, MLXSW_SP_ACL_ERP_MAX_PER_REGION); 56 struct list_head atcam_erps_list; 57 struct mlxsw_sp_acl_erp_core *erp_core; 58 struct mlxsw_sp_acl_atcam_region *aregion; 59 const struct mlxsw_sp_acl_erp_table_ops *ops; 60 unsigned long base_index; 61 unsigned int num_atcam_erps; 62 unsigned int num_max_atcam_erps; 63 unsigned int num_ctcam_erps; 64 unsigned int num_deltas; 65 struct objagg *objagg; 66 }; 67 68 struct mlxsw_sp_acl_erp_table_ops { 69 struct mlxsw_sp_acl_erp * 70 (*erp_create)(struct mlxsw_sp_acl_erp_table *erp_table, 71 struct mlxsw_sp_acl_erp_key *key); 72 void (*erp_destroy)(struct mlxsw_sp_acl_erp_table *erp_table, 73 struct mlxsw_sp_acl_erp *erp); 74 }; 75 76 static struct mlxsw_sp_acl_erp * 77 mlxsw_sp_acl_erp_mask_create(struct mlxsw_sp_acl_erp_table *erp_table, 78 struct mlxsw_sp_acl_erp_key *key); 79 static void 80 mlxsw_sp_acl_erp_mask_destroy(struct mlxsw_sp_acl_erp_table *erp_table, 81 struct mlxsw_sp_acl_erp *erp); 82 static struct mlxsw_sp_acl_erp * 83 mlxsw_sp_acl_erp_second_mask_create(struct mlxsw_sp_acl_erp_table *erp_table, 84 struct mlxsw_sp_acl_erp_key *key); 85 static void 86 mlxsw_sp_acl_erp_second_mask_destroy(struct mlxsw_sp_acl_erp_table *erp_table, 87 struct mlxsw_sp_acl_erp *erp); 88 static struct mlxsw_sp_acl_erp * 89 mlxsw_sp_acl_erp_first_mask_create(struct mlxsw_sp_acl_erp_table *erp_table, 90 struct mlxsw_sp_acl_erp_key *key); 91 static void 92 mlxsw_sp_acl_erp_first_mask_destroy(struct mlxsw_sp_acl_erp_table *erp_table, 93 struct mlxsw_sp_acl_erp *erp); 94 static void 95 mlxsw_sp_acl_erp_no_mask_destroy(struct mlxsw_sp_acl_erp_table *erp_table, 96 struct mlxsw_sp_acl_erp *erp); 97 98 static const struct mlxsw_sp_acl_erp_table_ops erp_multiple_masks_ops = { 99 .erp_create = mlxsw_sp_acl_erp_mask_create, 100 .erp_destroy = mlxsw_sp_acl_erp_mask_destroy, 101 }; 102 103 static const struct mlxsw_sp_acl_erp_table_ops erp_two_masks_ops = { 104 .erp_create = mlxsw_sp_acl_erp_mask_create, 105 .erp_destroy = mlxsw_sp_acl_erp_second_mask_destroy, 106 }; 107 108 static const struct mlxsw_sp_acl_erp_table_ops erp_single_mask_ops = { 109 .erp_create = mlxsw_sp_acl_erp_second_mask_create, 110 .erp_destroy = mlxsw_sp_acl_erp_first_mask_destroy, 111 }; 112 113 static const struct mlxsw_sp_acl_erp_table_ops erp_no_mask_ops = { 114 .erp_create = mlxsw_sp_acl_erp_first_mask_create, 115 .erp_destroy = mlxsw_sp_acl_erp_no_mask_destroy, 116 }; 117 118 static bool 119 mlxsw_sp_acl_erp_table_is_used(const struct mlxsw_sp_acl_erp_table *erp_table) 120 { 121 return erp_table->ops != &erp_single_mask_ops && 122 erp_table->ops != &erp_no_mask_ops; 123 } 124 125 static unsigned int 126 mlxsw_sp_acl_erp_bank_get(const struct mlxsw_sp_acl_erp *erp) 127 { 128 return erp->index % erp->erp_table->erp_core->num_erp_banks; 129 } 130 131 static unsigned int 132 mlxsw_sp_acl_erp_table_entry_size(const struct mlxsw_sp_acl_erp_table *erp_table) 133 { 134 struct mlxsw_sp_acl_atcam_region *aregion = erp_table->aregion; 135 struct mlxsw_sp_acl_erp_core *erp_core = erp_table->erp_core; 136 137 return erp_core->erpt_entries_size[aregion->type]; 138 } 139 140 static int mlxsw_sp_acl_erp_id_get(struct mlxsw_sp_acl_erp_table *erp_table, 141 u8 *p_id) 142 { 143 u8 id; 144 145 id = find_first_zero_bit(erp_table->erp_id_bitmap, 146 MLXSW_SP_ACL_ERP_MAX_PER_REGION); 147 if (id < MLXSW_SP_ACL_ERP_MAX_PER_REGION) { 148 __set_bit(id, erp_table->erp_id_bitmap); 149 *p_id = id; 150 return 0; 151 } 152 153 return -ENOBUFS; 154 } 155 156 static void mlxsw_sp_acl_erp_id_put(struct mlxsw_sp_acl_erp_table *erp_table, 157 u8 id) 158 { 159 __clear_bit(id, erp_table->erp_id_bitmap); 160 } 161 162 static void 163 mlxsw_sp_acl_erp_master_mask_bit_set(unsigned long bit, 164 struct mlxsw_sp_acl_erp_master_mask *mask) 165 { 166 if (mask->count[bit]++ == 0) 167 __set_bit(bit, mask->bitmap); 168 } 169 170 static void 171 mlxsw_sp_acl_erp_master_mask_bit_clear(unsigned long bit, 172 struct mlxsw_sp_acl_erp_master_mask *mask) 173 { 174 if (--mask->count[bit] == 0) 175 __clear_bit(bit, mask->bitmap); 176 } 177 178 static int 179 mlxsw_sp_acl_erp_master_mask_update(struct mlxsw_sp_acl_erp_table *erp_table) 180 { 181 struct mlxsw_sp_acl_tcam_region *region = erp_table->aregion->region; 182 struct mlxsw_sp *mlxsw_sp = region->mlxsw_sp; 183 char percr_pl[MLXSW_REG_PERCR_LEN]; 184 char *master_mask; 185 186 mlxsw_reg_percr_pack(percr_pl, region->id); 187 master_mask = mlxsw_reg_percr_master_mask_data(percr_pl); 188 bitmap_to_arr32((u32 *) master_mask, erp_table->master_mask.bitmap, 189 MLXSW_SP_ACL_TCAM_MASK_LEN); 190 191 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(percr), percr_pl); 192 } 193 194 static int 195 mlxsw_sp_acl_erp_master_mask_set(struct mlxsw_sp_acl_erp_table *erp_table, 196 struct mlxsw_sp_acl_erp_key *key) 197 { 198 DECLARE_BITMAP(mask_bitmap, MLXSW_SP_ACL_TCAM_MASK_LEN); 199 unsigned long bit; 200 int err; 201 202 bitmap_from_arr32(mask_bitmap, (u32 *) key->mask, 203 MLXSW_SP_ACL_TCAM_MASK_LEN); 204 for_each_set_bit(bit, mask_bitmap, MLXSW_SP_ACL_TCAM_MASK_LEN) 205 mlxsw_sp_acl_erp_master_mask_bit_set(bit, 206 &erp_table->master_mask); 207 208 err = mlxsw_sp_acl_erp_master_mask_update(erp_table); 209 if (err) 210 goto err_master_mask_update; 211 212 return 0; 213 214 err_master_mask_update: 215 for_each_set_bit(bit, mask_bitmap, MLXSW_SP_ACL_TCAM_MASK_LEN) 216 mlxsw_sp_acl_erp_master_mask_bit_clear(bit, 217 &erp_table->master_mask); 218 return err; 219 } 220 221 static int 222 mlxsw_sp_acl_erp_master_mask_clear(struct mlxsw_sp_acl_erp_table *erp_table, 223 struct mlxsw_sp_acl_erp_key *key) 224 { 225 DECLARE_BITMAP(mask_bitmap, MLXSW_SP_ACL_TCAM_MASK_LEN); 226 unsigned long bit; 227 int err; 228 229 bitmap_from_arr32(mask_bitmap, (u32 *) key->mask, 230 MLXSW_SP_ACL_TCAM_MASK_LEN); 231 for_each_set_bit(bit, mask_bitmap, MLXSW_SP_ACL_TCAM_MASK_LEN) 232 mlxsw_sp_acl_erp_master_mask_bit_clear(bit, 233 &erp_table->master_mask); 234 235 err = mlxsw_sp_acl_erp_master_mask_update(erp_table); 236 if (err) 237 goto err_master_mask_update; 238 239 return 0; 240 241 err_master_mask_update: 242 for_each_set_bit(bit, mask_bitmap, MLXSW_SP_ACL_TCAM_MASK_LEN) 243 mlxsw_sp_acl_erp_master_mask_bit_set(bit, 244 &erp_table->master_mask); 245 return err; 246 } 247 248 static struct mlxsw_sp_acl_erp * 249 mlxsw_sp_acl_erp_generic_create(struct mlxsw_sp_acl_erp_table *erp_table, 250 struct mlxsw_sp_acl_erp_key *key) 251 { 252 struct mlxsw_sp_acl_erp *erp; 253 int err; 254 255 erp = kzalloc(sizeof(*erp), GFP_KERNEL); 256 if (!erp) 257 return ERR_PTR(-ENOMEM); 258 259 err = mlxsw_sp_acl_erp_id_get(erp_table, &erp->id); 260 if (err) 261 goto err_erp_id_get; 262 263 memcpy(&erp->key, key, sizeof(*key)); 264 list_add(&erp->list, &erp_table->atcam_erps_list); 265 erp_table->num_atcam_erps++; 266 erp->erp_table = erp_table; 267 268 err = mlxsw_sp_acl_erp_master_mask_set(erp_table, &erp->key); 269 if (err) 270 goto err_master_mask_set; 271 272 return erp; 273 274 err_master_mask_set: 275 erp_table->num_atcam_erps--; 276 list_del(&erp->list); 277 mlxsw_sp_acl_erp_id_put(erp_table, erp->id); 278 err_erp_id_get: 279 kfree(erp); 280 return ERR_PTR(err); 281 } 282 283 static void 284 mlxsw_sp_acl_erp_generic_destroy(struct mlxsw_sp_acl_erp *erp) 285 { 286 struct mlxsw_sp_acl_erp_table *erp_table = erp->erp_table; 287 288 mlxsw_sp_acl_erp_master_mask_clear(erp_table, &erp->key); 289 erp_table->num_atcam_erps--; 290 list_del(&erp->list); 291 mlxsw_sp_acl_erp_id_put(erp_table, erp->id); 292 kfree(erp); 293 } 294 295 static int 296 mlxsw_sp_acl_erp_table_alloc(struct mlxsw_sp_acl_erp_core *erp_core, 297 unsigned int num_erps, 298 enum mlxsw_sp_acl_atcam_region_type region_type, 299 unsigned long *p_index) 300 { 301 unsigned int num_rows, entry_size; 302 303 /* We only allow allocations of entire rows */ 304 if (num_erps % erp_core->num_erp_banks != 0) 305 return -EINVAL; 306 307 entry_size = erp_core->erpt_entries_size[region_type]; 308 num_rows = num_erps / erp_core->num_erp_banks; 309 310 *p_index = gen_pool_alloc(erp_core->erp_tables, num_rows * entry_size); 311 if (*p_index == 0) 312 return -ENOBUFS; 313 *p_index -= MLXSW_SP_ACL_ERP_GENALLOC_OFFSET; 314 315 return 0; 316 } 317 318 static void 319 mlxsw_sp_acl_erp_table_free(struct mlxsw_sp_acl_erp_core *erp_core, 320 unsigned int num_erps, 321 enum mlxsw_sp_acl_atcam_region_type region_type, 322 unsigned long index) 323 { 324 unsigned long base_index; 325 unsigned int entry_size; 326 size_t size; 327 328 entry_size = erp_core->erpt_entries_size[region_type]; 329 base_index = index + MLXSW_SP_ACL_ERP_GENALLOC_OFFSET; 330 size = num_erps / erp_core->num_erp_banks * entry_size; 331 gen_pool_free(erp_core->erp_tables, base_index, size); 332 } 333 334 static struct mlxsw_sp_acl_erp * 335 mlxsw_sp_acl_erp_table_master_rp(struct mlxsw_sp_acl_erp_table *erp_table) 336 { 337 if (!list_is_singular(&erp_table->atcam_erps_list)) 338 return NULL; 339 340 return list_first_entry(&erp_table->atcam_erps_list, 341 struct mlxsw_sp_acl_erp, list); 342 } 343 344 static int mlxsw_sp_acl_erp_index_get(struct mlxsw_sp_acl_erp_table *erp_table, 345 u8 *p_index) 346 { 347 u8 index; 348 349 index = find_first_zero_bit(erp_table->erp_index_bitmap, 350 erp_table->num_max_atcam_erps); 351 if (index < erp_table->num_max_atcam_erps) { 352 __set_bit(index, erp_table->erp_index_bitmap); 353 *p_index = index; 354 return 0; 355 } 356 357 return -ENOBUFS; 358 } 359 360 static void mlxsw_sp_acl_erp_index_put(struct mlxsw_sp_acl_erp_table *erp_table, 361 u8 index) 362 { 363 __clear_bit(index, erp_table->erp_index_bitmap); 364 } 365 366 static void 367 mlxsw_sp_acl_erp_table_locate(const struct mlxsw_sp_acl_erp_table *erp_table, 368 const struct mlxsw_sp_acl_erp *erp, 369 u8 *p_erpt_bank, u8 *p_erpt_index) 370 { 371 unsigned int entry_size = mlxsw_sp_acl_erp_table_entry_size(erp_table); 372 struct mlxsw_sp_acl_erp_core *erp_core = erp_table->erp_core; 373 unsigned int row; 374 375 *p_erpt_bank = erp->index % erp_core->num_erp_banks; 376 row = erp->index / erp_core->num_erp_banks; 377 *p_erpt_index = erp_table->base_index + row * entry_size; 378 } 379 380 static int 381 mlxsw_sp_acl_erp_table_erp_add(struct mlxsw_sp_acl_erp_table *erp_table, 382 struct mlxsw_sp_acl_erp *erp) 383 { 384 struct mlxsw_sp *mlxsw_sp = erp_table->erp_core->mlxsw_sp; 385 enum mlxsw_reg_perpt_key_size key_size; 386 char perpt_pl[MLXSW_REG_PERPT_LEN]; 387 u8 erpt_bank, erpt_index; 388 389 mlxsw_sp_acl_erp_table_locate(erp_table, erp, &erpt_bank, &erpt_index); 390 key_size = (enum mlxsw_reg_perpt_key_size) erp_table->aregion->type; 391 mlxsw_reg_perpt_pack(perpt_pl, erpt_bank, erpt_index, key_size, erp->id, 392 0, erp_table->base_index, erp->index, 393 erp->key.mask); 394 mlxsw_reg_perpt_erp_vector_pack(perpt_pl, erp_table->erp_index_bitmap, 395 MLXSW_SP_ACL_ERP_MAX_PER_REGION); 396 mlxsw_reg_perpt_erp_vector_set(perpt_pl, erp->index, true); 397 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(perpt), perpt_pl); 398 } 399 400 static void mlxsw_sp_acl_erp_table_erp_del(struct mlxsw_sp_acl_erp *erp) 401 { 402 char empty_mask[MLXSW_REG_PTCEX_FLEX_KEY_BLOCKS_LEN] = { 0 }; 403 struct mlxsw_sp_acl_erp_table *erp_table = erp->erp_table; 404 struct mlxsw_sp *mlxsw_sp = erp_table->erp_core->mlxsw_sp; 405 enum mlxsw_reg_perpt_key_size key_size; 406 char perpt_pl[MLXSW_REG_PERPT_LEN]; 407 u8 erpt_bank, erpt_index; 408 409 mlxsw_sp_acl_erp_table_locate(erp_table, erp, &erpt_bank, &erpt_index); 410 key_size = (enum mlxsw_reg_perpt_key_size) erp_table->aregion->type; 411 mlxsw_reg_perpt_pack(perpt_pl, erpt_bank, erpt_index, key_size, erp->id, 412 0, erp_table->base_index, erp->index, empty_mask); 413 mlxsw_reg_perpt_erp_vector_pack(perpt_pl, erp_table->erp_index_bitmap, 414 MLXSW_SP_ACL_ERP_MAX_PER_REGION); 415 mlxsw_reg_perpt_erp_vector_set(perpt_pl, erp->index, false); 416 mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(perpt), perpt_pl); 417 } 418 419 static int 420 mlxsw_sp_acl_erp_table_enable(struct mlxsw_sp_acl_erp_table *erp_table, 421 bool ctcam_le) 422 { 423 struct mlxsw_sp_acl_tcam_region *region = erp_table->aregion->region; 424 struct mlxsw_sp *mlxsw_sp = erp_table->erp_core->mlxsw_sp; 425 char pererp_pl[MLXSW_REG_PERERP_LEN]; 426 427 mlxsw_reg_pererp_pack(pererp_pl, region->id, ctcam_le, true, 0, 428 erp_table->base_index, 0); 429 mlxsw_reg_pererp_erp_vector_pack(pererp_pl, erp_table->erp_index_bitmap, 430 MLXSW_SP_ACL_ERP_MAX_PER_REGION); 431 432 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(pererp), pererp_pl); 433 } 434 435 static void 436 mlxsw_sp_acl_erp_table_disable(struct mlxsw_sp_acl_erp_table *erp_table) 437 { 438 struct mlxsw_sp_acl_tcam_region *region = erp_table->aregion->region; 439 struct mlxsw_sp *mlxsw_sp = erp_table->erp_core->mlxsw_sp; 440 char pererp_pl[MLXSW_REG_PERERP_LEN]; 441 struct mlxsw_sp_acl_erp *master_rp; 442 443 master_rp = mlxsw_sp_acl_erp_table_master_rp(erp_table); 444 /* It is possible we do not have a master RP when we disable the 445 * table when there are no rules in the A-TCAM and the last C-TCAM 446 * rule is deleted 447 */ 448 mlxsw_reg_pererp_pack(pererp_pl, region->id, false, false, 0, 0, 449 master_rp ? master_rp->id : 0); 450 mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(pererp), pererp_pl); 451 } 452 453 static int 454 mlxsw_sp_acl_erp_table_relocate(struct mlxsw_sp_acl_erp_table *erp_table) 455 { 456 struct mlxsw_sp_acl_erp *erp; 457 int err; 458 459 list_for_each_entry(erp, &erp_table->atcam_erps_list, list) { 460 err = mlxsw_sp_acl_erp_table_erp_add(erp_table, erp); 461 if (err) 462 goto err_table_erp_add; 463 } 464 465 return 0; 466 467 err_table_erp_add: 468 list_for_each_entry_continue_reverse(erp, &erp_table->atcam_erps_list, 469 list) 470 mlxsw_sp_acl_erp_table_erp_del(erp); 471 return err; 472 } 473 474 static int 475 mlxsw_sp_acl_erp_table_expand(struct mlxsw_sp_acl_erp_table *erp_table) 476 { 477 unsigned int num_erps, old_num_erps = erp_table->num_max_atcam_erps; 478 struct mlxsw_sp_acl_erp_core *erp_core = erp_table->erp_core; 479 unsigned long old_base_index = erp_table->base_index; 480 bool ctcam_le = erp_table->num_ctcam_erps > 0; 481 int err; 482 483 if (erp_table->num_atcam_erps < erp_table->num_max_atcam_erps) 484 return 0; 485 486 if (erp_table->num_max_atcam_erps == MLXSW_SP_ACL_ERP_MAX_PER_REGION) 487 return -ENOBUFS; 488 489 num_erps = old_num_erps + erp_core->num_erp_banks; 490 err = mlxsw_sp_acl_erp_table_alloc(erp_core, num_erps, 491 erp_table->aregion->type, 492 &erp_table->base_index); 493 if (err) 494 return err; 495 erp_table->num_max_atcam_erps = num_erps; 496 497 err = mlxsw_sp_acl_erp_table_relocate(erp_table); 498 if (err) 499 goto err_table_relocate; 500 501 err = mlxsw_sp_acl_erp_table_enable(erp_table, ctcam_le); 502 if (err) 503 goto err_table_enable; 504 505 mlxsw_sp_acl_erp_table_free(erp_core, old_num_erps, 506 erp_table->aregion->type, old_base_index); 507 508 return 0; 509 510 err_table_enable: 511 err_table_relocate: 512 erp_table->num_max_atcam_erps = old_num_erps; 513 mlxsw_sp_acl_erp_table_free(erp_core, num_erps, 514 erp_table->aregion->type, 515 erp_table->base_index); 516 erp_table->base_index = old_base_index; 517 return err; 518 } 519 520 static int 521 mlxsw_acl_erp_table_bf_add(struct mlxsw_sp_acl_erp_table *erp_table, 522 struct mlxsw_sp_acl_erp *erp) 523 { 524 struct mlxsw_sp_acl_atcam_region *aregion = erp_table->aregion; 525 unsigned int erp_bank = mlxsw_sp_acl_erp_bank_get(erp); 526 struct mlxsw_sp_acl_atcam_entry *aentry; 527 int err; 528 529 list_for_each_entry(aentry, &aregion->entries_list, list) { 530 err = mlxsw_sp_acl_bf_entry_add(aregion->region->mlxsw_sp, 531 erp_table->erp_core->bf, 532 aregion, erp_bank, aentry); 533 if (err) 534 goto bf_entry_add_err; 535 } 536 537 return 0; 538 539 bf_entry_add_err: 540 list_for_each_entry_continue_reverse(aentry, &aregion->entries_list, 541 list) 542 mlxsw_sp_acl_bf_entry_del(aregion->region->mlxsw_sp, 543 erp_table->erp_core->bf, 544 aregion, erp_bank, aentry); 545 return err; 546 } 547 548 static void 549 mlxsw_acl_erp_table_bf_del(struct mlxsw_sp_acl_erp_table *erp_table, 550 struct mlxsw_sp_acl_erp *erp) 551 { 552 struct mlxsw_sp_acl_atcam_region *aregion = erp_table->aregion; 553 unsigned int erp_bank = mlxsw_sp_acl_erp_bank_get(erp); 554 struct mlxsw_sp_acl_atcam_entry *aentry; 555 556 list_for_each_entry_reverse(aentry, &aregion->entries_list, list) 557 mlxsw_sp_acl_bf_entry_del(aregion->region->mlxsw_sp, 558 erp_table->erp_core->bf, 559 aregion, erp_bank, aentry); 560 } 561 562 static int 563 mlxsw_sp_acl_erp_region_table_trans(struct mlxsw_sp_acl_erp_table *erp_table) 564 { 565 struct mlxsw_sp_acl_erp_core *erp_core = erp_table->erp_core; 566 struct mlxsw_sp_acl_erp *master_rp; 567 int err; 568 569 /* Initially, allocate a single eRP row. Expand later as needed */ 570 err = mlxsw_sp_acl_erp_table_alloc(erp_core, erp_core->num_erp_banks, 571 erp_table->aregion->type, 572 &erp_table->base_index); 573 if (err) 574 return err; 575 erp_table->num_max_atcam_erps = erp_core->num_erp_banks; 576 577 /* Transition the sole RP currently configured (the master RP) 578 * to the eRP table 579 */ 580 master_rp = mlxsw_sp_acl_erp_table_master_rp(erp_table); 581 if (!master_rp) { 582 err = -EINVAL; 583 goto err_table_master_rp; 584 } 585 586 /* Make sure the master RP is using a valid index, as 587 * only a single eRP row is currently allocated. 588 */ 589 master_rp->index = 0; 590 __set_bit(master_rp->index, erp_table->erp_index_bitmap); 591 592 err = mlxsw_sp_acl_erp_table_erp_add(erp_table, master_rp); 593 if (err) 594 goto err_table_master_rp_add; 595 596 /* Update Bloom filter before enabling eRP table, as rules 597 * on the master RP were not set to Bloom filter up to this 598 * point. 599 */ 600 err = mlxsw_acl_erp_table_bf_add(erp_table, master_rp); 601 if (err) 602 goto err_table_bf_add; 603 604 err = mlxsw_sp_acl_erp_table_enable(erp_table, false); 605 if (err) 606 goto err_table_enable; 607 608 return 0; 609 610 err_table_enable: 611 mlxsw_acl_erp_table_bf_del(erp_table, master_rp); 612 err_table_bf_add: 613 mlxsw_sp_acl_erp_table_erp_del(master_rp); 614 err_table_master_rp_add: 615 __clear_bit(master_rp->index, erp_table->erp_index_bitmap); 616 err_table_master_rp: 617 mlxsw_sp_acl_erp_table_free(erp_core, erp_table->num_max_atcam_erps, 618 erp_table->aregion->type, 619 erp_table->base_index); 620 return err; 621 } 622 623 static void 624 mlxsw_sp_acl_erp_region_master_mask_trans(struct mlxsw_sp_acl_erp_table *erp_table) 625 { 626 struct mlxsw_sp_acl_erp_core *erp_core = erp_table->erp_core; 627 struct mlxsw_sp_acl_erp *master_rp; 628 629 mlxsw_sp_acl_erp_table_disable(erp_table); 630 master_rp = mlxsw_sp_acl_erp_table_master_rp(erp_table); 631 if (!master_rp) 632 return; 633 mlxsw_acl_erp_table_bf_del(erp_table, master_rp); 634 mlxsw_sp_acl_erp_table_erp_del(master_rp); 635 __clear_bit(master_rp->index, erp_table->erp_index_bitmap); 636 mlxsw_sp_acl_erp_table_free(erp_core, erp_table->num_max_atcam_erps, 637 erp_table->aregion->type, 638 erp_table->base_index); 639 } 640 641 static int 642 mlxsw_sp_acl_erp_region_erp_add(struct mlxsw_sp_acl_erp_table *erp_table, 643 struct mlxsw_sp_acl_erp *erp) 644 { 645 struct mlxsw_sp_acl_tcam_region *region = erp_table->aregion->region; 646 struct mlxsw_sp *mlxsw_sp = erp_table->erp_core->mlxsw_sp; 647 bool ctcam_le = erp_table->num_ctcam_erps > 0; 648 char pererp_pl[MLXSW_REG_PERERP_LEN]; 649 650 mlxsw_reg_pererp_pack(pererp_pl, region->id, ctcam_le, true, 0, 651 erp_table->base_index, 0); 652 mlxsw_reg_pererp_erp_vector_pack(pererp_pl, erp_table->erp_index_bitmap, 653 MLXSW_SP_ACL_ERP_MAX_PER_REGION); 654 mlxsw_reg_pererp_erpt_vector_set(pererp_pl, erp->index, true); 655 656 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(pererp), pererp_pl); 657 } 658 659 static void mlxsw_sp_acl_erp_region_erp_del(struct mlxsw_sp_acl_erp *erp) 660 { 661 struct mlxsw_sp_acl_erp_table *erp_table = erp->erp_table; 662 struct mlxsw_sp_acl_tcam_region *region = erp_table->aregion->region; 663 struct mlxsw_sp *mlxsw_sp = erp_table->erp_core->mlxsw_sp; 664 bool ctcam_le = erp_table->num_ctcam_erps > 0; 665 char pererp_pl[MLXSW_REG_PERERP_LEN]; 666 667 mlxsw_reg_pererp_pack(pererp_pl, region->id, ctcam_le, true, 0, 668 erp_table->base_index, 0); 669 mlxsw_reg_pererp_erp_vector_pack(pererp_pl, erp_table->erp_index_bitmap, 670 MLXSW_SP_ACL_ERP_MAX_PER_REGION); 671 mlxsw_reg_pererp_erpt_vector_set(pererp_pl, erp->index, false); 672 673 mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(pererp), pererp_pl); 674 } 675 676 static int 677 mlxsw_sp_acl_erp_region_ctcam_enable(struct mlxsw_sp_acl_erp_table *erp_table) 678 { 679 /* No need to re-enable lookup in the C-TCAM */ 680 if (erp_table->num_ctcam_erps > 1) 681 return 0; 682 683 return mlxsw_sp_acl_erp_table_enable(erp_table, true); 684 } 685 686 static void 687 mlxsw_sp_acl_erp_region_ctcam_disable(struct mlxsw_sp_acl_erp_table *erp_table) 688 { 689 /* Only disable C-TCAM lookup when last C-TCAM eRP is deleted */ 690 if (erp_table->num_ctcam_erps > 1) 691 return; 692 693 mlxsw_sp_acl_erp_table_enable(erp_table, false); 694 } 695 696 static int 697 __mlxsw_sp_acl_erp_table_other_inc(struct mlxsw_sp_acl_erp_table *erp_table, 698 unsigned int *inc_num) 699 { 700 int err; 701 702 /* If there are C-TCAM eRP or deltas in use we need to transition 703 * the region to use eRP table, if it is not already done 704 */ 705 if (!mlxsw_sp_acl_erp_table_is_used(erp_table)) { 706 err = mlxsw_sp_acl_erp_region_table_trans(erp_table); 707 if (err) 708 return err; 709 } 710 711 /* When C-TCAM or deltas are used, the eRP table must be used */ 712 if (erp_table->ops != &erp_multiple_masks_ops) 713 erp_table->ops = &erp_multiple_masks_ops; 714 715 (*inc_num)++; 716 717 return 0; 718 } 719 720 static int mlxsw_sp_acl_erp_ctcam_inc(struct mlxsw_sp_acl_erp_table *erp_table) 721 { 722 return __mlxsw_sp_acl_erp_table_other_inc(erp_table, 723 &erp_table->num_ctcam_erps); 724 } 725 726 static int mlxsw_sp_acl_erp_delta_inc(struct mlxsw_sp_acl_erp_table *erp_table) 727 { 728 return __mlxsw_sp_acl_erp_table_other_inc(erp_table, 729 &erp_table->num_deltas); 730 } 731 732 static void 733 __mlxsw_sp_acl_erp_table_other_dec(struct mlxsw_sp_acl_erp_table *erp_table, 734 unsigned int *dec_num) 735 { 736 (*dec_num)--; 737 738 /* If there are no C-TCAM eRP or deltas in use, the state we 739 * transition to depends on the number of A-TCAM eRPs currently 740 * in use. 741 */ 742 if (erp_table->num_ctcam_erps > 0 || erp_table->num_deltas > 0) 743 return; 744 745 switch (erp_table->num_atcam_erps) { 746 case 2: 747 /* Keep using the eRP table, but correctly set the 748 * operations pointer so that when an A-TCAM eRP is 749 * deleted we will transition to use the master mask 750 */ 751 erp_table->ops = &erp_two_masks_ops; 752 break; 753 case 1: 754 /* We only kept the eRP table because we had C-TCAM 755 * eRPs in use. Now that the last C-TCAM eRP is gone we 756 * can stop using the table and transition to use the 757 * master mask 758 */ 759 mlxsw_sp_acl_erp_region_master_mask_trans(erp_table); 760 erp_table->ops = &erp_single_mask_ops; 761 break; 762 case 0: 763 /* There are no more eRPs of any kind used by the region 764 * so free its eRP table and transition to initial state 765 */ 766 mlxsw_sp_acl_erp_table_disable(erp_table); 767 mlxsw_sp_acl_erp_table_free(erp_table->erp_core, 768 erp_table->num_max_atcam_erps, 769 erp_table->aregion->type, 770 erp_table->base_index); 771 erp_table->ops = &erp_no_mask_ops; 772 break; 773 default: 774 break; 775 } 776 } 777 778 static void mlxsw_sp_acl_erp_ctcam_dec(struct mlxsw_sp_acl_erp_table *erp_table) 779 { 780 __mlxsw_sp_acl_erp_table_other_dec(erp_table, 781 &erp_table->num_ctcam_erps); 782 } 783 784 static void mlxsw_sp_acl_erp_delta_dec(struct mlxsw_sp_acl_erp_table *erp_table) 785 { 786 __mlxsw_sp_acl_erp_table_other_dec(erp_table, 787 &erp_table->num_deltas); 788 } 789 790 static struct mlxsw_sp_acl_erp * 791 mlxsw_sp_acl_erp_ctcam_mask_create(struct mlxsw_sp_acl_erp_table *erp_table, 792 struct mlxsw_sp_acl_erp_key *key) 793 { 794 struct mlxsw_sp_acl_erp *erp; 795 int err; 796 797 erp = kzalloc(sizeof(*erp), GFP_KERNEL); 798 if (!erp) 799 return ERR_PTR(-ENOMEM); 800 801 memcpy(&erp->key, key, sizeof(*key)); 802 bitmap_from_arr32(erp->mask_bitmap, (u32 *) key->mask, 803 MLXSW_SP_ACL_TCAM_MASK_LEN); 804 805 err = mlxsw_sp_acl_erp_ctcam_inc(erp_table); 806 if (err) 807 goto err_erp_ctcam_inc; 808 809 erp->erp_table = erp_table; 810 811 err = mlxsw_sp_acl_erp_master_mask_set(erp_table, &erp->key); 812 if (err) 813 goto err_master_mask_set; 814 815 err = mlxsw_sp_acl_erp_region_ctcam_enable(erp_table); 816 if (err) 817 goto err_erp_region_ctcam_enable; 818 819 return erp; 820 821 err_erp_region_ctcam_enable: 822 mlxsw_sp_acl_erp_master_mask_clear(erp_table, &erp->key); 823 err_master_mask_set: 824 mlxsw_sp_acl_erp_ctcam_dec(erp_table); 825 err_erp_ctcam_inc: 826 kfree(erp); 827 return ERR_PTR(err); 828 } 829 830 static void 831 mlxsw_sp_acl_erp_ctcam_mask_destroy(struct mlxsw_sp_acl_erp *erp) 832 { 833 struct mlxsw_sp_acl_erp_table *erp_table = erp->erp_table; 834 835 mlxsw_sp_acl_erp_region_ctcam_disable(erp_table); 836 mlxsw_sp_acl_erp_master_mask_clear(erp_table, &erp->key); 837 mlxsw_sp_acl_erp_ctcam_dec(erp_table); 838 kfree(erp); 839 } 840 841 static struct mlxsw_sp_acl_erp * 842 mlxsw_sp_acl_erp_mask_create(struct mlxsw_sp_acl_erp_table *erp_table, 843 struct mlxsw_sp_acl_erp_key *key) 844 { 845 struct mlxsw_sp_acl_erp *erp; 846 int err; 847 848 if (key->ctcam) 849 return mlxsw_sp_acl_erp_ctcam_mask_create(erp_table, key); 850 851 /* Expand the eRP table for the new eRP, if needed */ 852 err = mlxsw_sp_acl_erp_table_expand(erp_table); 853 if (err) 854 return ERR_PTR(err); 855 856 erp = mlxsw_sp_acl_erp_generic_create(erp_table, key); 857 if (IS_ERR(erp)) 858 return erp; 859 860 err = mlxsw_sp_acl_erp_index_get(erp_table, &erp->index); 861 if (err) 862 goto err_erp_index_get; 863 864 err = mlxsw_sp_acl_erp_table_erp_add(erp_table, erp); 865 if (err) 866 goto err_table_erp_add; 867 868 err = mlxsw_sp_acl_erp_region_erp_add(erp_table, erp); 869 if (err) 870 goto err_region_erp_add; 871 872 erp_table->ops = &erp_multiple_masks_ops; 873 874 return erp; 875 876 err_region_erp_add: 877 mlxsw_sp_acl_erp_table_erp_del(erp); 878 err_table_erp_add: 879 mlxsw_sp_acl_erp_index_put(erp_table, erp->index); 880 err_erp_index_get: 881 mlxsw_sp_acl_erp_generic_destroy(erp); 882 return ERR_PTR(err); 883 } 884 885 static void 886 mlxsw_sp_acl_erp_mask_destroy(struct mlxsw_sp_acl_erp_table *erp_table, 887 struct mlxsw_sp_acl_erp *erp) 888 { 889 if (erp->key.ctcam) 890 return mlxsw_sp_acl_erp_ctcam_mask_destroy(erp); 891 892 mlxsw_sp_acl_erp_region_erp_del(erp); 893 mlxsw_sp_acl_erp_table_erp_del(erp); 894 mlxsw_sp_acl_erp_index_put(erp_table, erp->index); 895 mlxsw_sp_acl_erp_generic_destroy(erp); 896 897 if (erp_table->num_atcam_erps == 2 && erp_table->num_ctcam_erps == 0 && 898 erp_table->num_deltas == 0) 899 erp_table->ops = &erp_two_masks_ops; 900 } 901 902 static struct mlxsw_sp_acl_erp * 903 mlxsw_sp_acl_erp_second_mask_create(struct mlxsw_sp_acl_erp_table *erp_table, 904 struct mlxsw_sp_acl_erp_key *key) 905 { 906 struct mlxsw_sp_acl_erp *erp; 907 int err; 908 909 if (key->ctcam) 910 return mlxsw_sp_acl_erp_ctcam_mask_create(erp_table, key); 911 912 /* Transition to use eRP table instead of master mask */ 913 err = mlxsw_sp_acl_erp_region_table_trans(erp_table); 914 if (err) 915 return ERR_PTR(err); 916 917 erp = mlxsw_sp_acl_erp_generic_create(erp_table, key); 918 if (IS_ERR(erp)) { 919 err = PTR_ERR(erp); 920 goto err_erp_create; 921 } 922 923 err = mlxsw_sp_acl_erp_index_get(erp_table, &erp->index); 924 if (err) 925 goto err_erp_index_get; 926 927 err = mlxsw_sp_acl_erp_table_erp_add(erp_table, erp); 928 if (err) 929 goto err_table_erp_add; 930 931 err = mlxsw_sp_acl_erp_region_erp_add(erp_table, erp); 932 if (err) 933 goto err_region_erp_add; 934 935 erp_table->ops = &erp_two_masks_ops; 936 937 return erp; 938 939 err_region_erp_add: 940 mlxsw_sp_acl_erp_table_erp_del(erp); 941 err_table_erp_add: 942 mlxsw_sp_acl_erp_index_put(erp_table, erp->index); 943 err_erp_index_get: 944 mlxsw_sp_acl_erp_generic_destroy(erp); 945 err_erp_create: 946 mlxsw_sp_acl_erp_region_master_mask_trans(erp_table); 947 return ERR_PTR(err); 948 } 949 950 static void 951 mlxsw_sp_acl_erp_second_mask_destroy(struct mlxsw_sp_acl_erp_table *erp_table, 952 struct mlxsw_sp_acl_erp *erp) 953 { 954 if (erp->key.ctcam) 955 return mlxsw_sp_acl_erp_ctcam_mask_destroy(erp); 956 957 mlxsw_sp_acl_erp_region_erp_del(erp); 958 mlxsw_sp_acl_erp_table_erp_del(erp); 959 mlxsw_sp_acl_erp_index_put(erp_table, erp->index); 960 mlxsw_sp_acl_erp_generic_destroy(erp); 961 /* Transition to use master mask instead of eRP table */ 962 mlxsw_sp_acl_erp_region_master_mask_trans(erp_table); 963 964 erp_table->ops = &erp_single_mask_ops; 965 } 966 967 static struct mlxsw_sp_acl_erp * 968 mlxsw_sp_acl_erp_first_mask_create(struct mlxsw_sp_acl_erp_table *erp_table, 969 struct mlxsw_sp_acl_erp_key *key) 970 { 971 struct mlxsw_sp_acl_erp *erp; 972 973 if (key->ctcam) 974 return ERR_PTR(-EINVAL); 975 976 erp = mlxsw_sp_acl_erp_generic_create(erp_table, key); 977 if (IS_ERR(erp)) 978 return erp; 979 980 erp_table->ops = &erp_single_mask_ops; 981 982 return erp; 983 } 984 985 static void 986 mlxsw_sp_acl_erp_first_mask_destroy(struct mlxsw_sp_acl_erp_table *erp_table, 987 struct mlxsw_sp_acl_erp *erp) 988 { 989 mlxsw_sp_acl_erp_generic_destroy(erp); 990 erp_table->ops = &erp_no_mask_ops; 991 } 992 993 static void 994 mlxsw_sp_acl_erp_no_mask_destroy(struct mlxsw_sp_acl_erp_table *erp_table, 995 struct mlxsw_sp_acl_erp *erp) 996 { 997 WARN_ON(1); 998 } 999 1000 struct mlxsw_sp_acl_erp_mask * 1001 mlxsw_sp_acl_erp_mask_get(struct mlxsw_sp_acl_atcam_region *aregion, 1002 const char *mask, bool ctcam) 1003 { 1004 struct mlxsw_sp_acl_erp_key key; 1005 struct objagg_obj *objagg_obj; 1006 1007 /* eRPs are allocated from a shared resource, but currently all 1008 * allocations are done under RTNL. 1009 */ 1010 ASSERT_RTNL(); 1011 1012 memcpy(key.mask, mask, MLXSW_REG_PTCEX_FLEX_KEY_BLOCKS_LEN); 1013 key.ctcam = ctcam; 1014 objagg_obj = objagg_obj_get(aregion->erp_table->objagg, &key); 1015 if (IS_ERR(objagg_obj)) 1016 return ERR_CAST(objagg_obj); 1017 return (struct mlxsw_sp_acl_erp_mask *) objagg_obj; 1018 } 1019 1020 void mlxsw_sp_acl_erp_mask_put(struct mlxsw_sp_acl_atcam_region *aregion, 1021 struct mlxsw_sp_acl_erp_mask *erp_mask) 1022 { 1023 struct objagg_obj *objagg_obj = (struct objagg_obj *) erp_mask; 1024 1025 ASSERT_RTNL(); 1026 objagg_obj_put(aregion->erp_table->objagg, objagg_obj); 1027 } 1028 1029 int mlxsw_sp_acl_erp_bf_insert(struct mlxsw_sp *mlxsw_sp, 1030 struct mlxsw_sp_acl_atcam_region *aregion, 1031 struct mlxsw_sp_acl_erp_mask *erp_mask, 1032 struct mlxsw_sp_acl_atcam_entry *aentry) 1033 { 1034 struct objagg_obj *objagg_obj = (struct objagg_obj *) erp_mask; 1035 const struct mlxsw_sp_acl_erp *erp = objagg_obj_root_priv(objagg_obj); 1036 unsigned int erp_bank; 1037 1038 ASSERT_RTNL(); 1039 if (!mlxsw_sp_acl_erp_table_is_used(erp->erp_table)) 1040 return 0; 1041 1042 erp_bank = mlxsw_sp_acl_erp_bank_get(erp); 1043 return mlxsw_sp_acl_bf_entry_add(mlxsw_sp, 1044 erp->erp_table->erp_core->bf, 1045 aregion, erp_bank, aentry); 1046 } 1047 1048 void mlxsw_sp_acl_erp_bf_remove(struct mlxsw_sp *mlxsw_sp, 1049 struct mlxsw_sp_acl_atcam_region *aregion, 1050 struct mlxsw_sp_acl_erp_mask *erp_mask, 1051 struct mlxsw_sp_acl_atcam_entry *aentry) 1052 { 1053 struct objagg_obj *objagg_obj = (struct objagg_obj *) erp_mask; 1054 const struct mlxsw_sp_acl_erp *erp = objagg_obj_root_priv(objagg_obj); 1055 unsigned int erp_bank; 1056 1057 ASSERT_RTNL(); 1058 if (!mlxsw_sp_acl_erp_table_is_used(erp->erp_table)) 1059 return; 1060 1061 erp_bank = mlxsw_sp_acl_erp_bank_get(erp); 1062 mlxsw_sp_acl_bf_entry_del(mlxsw_sp, 1063 erp->erp_table->erp_core->bf, 1064 aregion, erp_bank, aentry); 1065 } 1066 1067 bool 1068 mlxsw_sp_acl_erp_mask_is_ctcam(const struct mlxsw_sp_acl_erp_mask *erp_mask) 1069 { 1070 struct objagg_obj *objagg_obj = (struct objagg_obj *) erp_mask; 1071 const struct mlxsw_sp_acl_erp_key *key = objagg_obj_raw(objagg_obj); 1072 1073 return key->ctcam; 1074 } 1075 1076 u8 mlxsw_sp_acl_erp_mask_erp_id(const struct mlxsw_sp_acl_erp_mask *erp_mask) 1077 { 1078 struct objagg_obj *objagg_obj = (struct objagg_obj *) erp_mask; 1079 const struct mlxsw_sp_acl_erp *erp = objagg_obj_root_priv(objagg_obj); 1080 1081 return erp->id; 1082 } 1083 1084 struct mlxsw_sp_acl_erp_delta { 1085 struct mlxsw_sp_acl_erp_key key; 1086 u16 start; 1087 u8 mask; 1088 }; 1089 1090 u16 mlxsw_sp_acl_erp_delta_start(const struct mlxsw_sp_acl_erp_delta *delta) 1091 { 1092 return delta->start; 1093 } 1094 1095 u8 mlxsw_sp_acl_erp_delta_mask(const struct mlxsw_sp_acl_erp_delta *delta) 1096 { 1097 return delta->mask; 1098 } 1099 1100 u8 mlxsw_sp_acl_erp_delta_value(const struct mlxsw_sp_acl_erp_delta *delta, 1101 const char *enc_key) 1102 { 1103 u16 start = delta->start; 1104 u8 mask = delta->mask; 1105 u16 tmp; 1106 1107 if (!mask) 1108 return 0; 1109 1110 tmp = (unsigned char) enc_key[__MASK_IDX(start / 8)]; 1111 if (start / 8 + 1 < __MASK_LEN) 1112 tmp |= (unsigned char) enc_key[__MASK_IDX(start / 8 + 1)] << 8; 1113 tmp >>= start % 8; 1114 tmp &= mask; 1115 return tmp; 1116 } 1117 1118 void mlxsw_sp_acl_erp_delta_clear(const struct mlxsw_sp_acl_erp_delta *delta, 1119 const char *enc_key) 1120 { 1121 u16 start = delta->start; 1122 u8 mask = delta->mask; 1123 unsigned char *byte; 1124 u16 tmp; 1125 1126 tmp = mask; 1127 tmp <<= start % 8; 1128 tmp = ~tmp; 1129 1130 byte = (unsigned char *) &enc_key[__MASK_IDX(start / 8)]; 1131 *byte &= tmp & 0xff; 1132 if (start / 8 + 1 < __MASK_LEN) { 1133 byte = (unsigned char *) &enc_key[__MASK_IDX(start / 8 + 1)]; 1134 *byte &= (tmp >> 8) & 0xff; 1135 } 1136 } 1137 1138 static const struct mlxsw_sp_acl_erp_delta 1139 mlxsw_sp_acl_erp_delta_default = {}; 1140 1141 const struct mlxsw_sp_acl_erp_delta * 1142 mlxsw_sp_acl_erp_delta(const struct mlxsw_sp_acl_erp_mask *erp_mask) 1143 { 1144 struct objagg_obj *objagg_obj = (struct objagg_obj *) erp_mask; 1145 const struct mlxsw_sp_acl_erp_delta *delta; 1146 1147 delta = objagg_obj_delta_priv(objagg_obj); 1148 if (!delta) 1149 delta = &mlxsw_sp_acl_erp_delta_default; 1150 return delta; 1151 } 1152 1153 static int 1154 mlxsw_sp_acl_erp_delta_fill(const struct mlxsw_sp_acl_erp_key *parent_key, 1155 const struct mlxsw_sp_acl_erp_key *key, 1156 u16 *delta_start, u8 *delta_mask) 1157 { 1158 int offset = 0; 1159 int si = -1; 1160 u16 pmask; 1161 u16 mask; 1162 int i; 1163 1164 /* The difference between 2 masks can be up to 8 consecutive bits. */ 1165 for (i = 0; i < __MASK_LEN; i++) { 1166 if (parent_key->mask[__MASK_IDX(i)] == key->mask[__MASK_IDX(i)]) 1167 continue; 1168 if (si == -1) 1169 si = i; 1170 else if (si != i - 1) 1171 return -EINVAL; 1172 } 1173 if (si == -1) { 1174 /* The masks are the same, this cannot happen. 1175 * That means the caller is broken. 1176 */ 1177 WARN_ON(1); 1178 *delta_start = 0; 1179 *delta_mask = 0; 1180 return 0; 1181 } 1182 pmask = (unsigned char) parent_key->mask[__MASK_IDX(si)]; 1183 mask = (unsigned char) key->mask[__MASK_IDX(si)]; 1184 if (si + 1 < __MASK_LEN) { 1185 pmask |= (unsigned char) parent_key->mask[__MASK_IDX(si + 1)] << 8; 1186 mask |= (unsigned char) key->mask[__MASK_IDX(si + 1)] << 8; 1187 } 1188 1189 if ((pmask ^ mask) & pmask) 1190 return -EINVAL; 1191 mask &= ~pmask; 1192 while (!(mask & (1 << offset))) 1193 offset++; 1194 while (!(mask & 1)) 1195 mask >>= 1; 1196 if (mask & 0xff00) 1197 return -EINVAL; 1198 1199 *delta_start = si * 8 + offset; 1200 *delta_mask = mask; 1201 1202 return 0; 1203 } 1204 1205 static void *mlxsw_sp_acl_erp_delta_create(void *priv, void *parent_obj, 1206 void *obj) 1207 { 1208 struct mlxsw_sp_acl_erp_key *parent_key = parent_obj; 1209 struct mlxsw_sp_acl_atcam_region *aregion = priv; 1210 struct mlxsw_sp_acl_erp_table *erp_table = aregion->erp_table; 1211 struct mlxsw_sp_acl_erp_key *key = obj; 1212 struct mlxsw_sp_acl_erp_delta *delta; 1213 u16 delta_start; 1214 u8 delta_mask; 1215 int err; 1216 1217 if (parent_key->ctcam || key->ctcam) 1218 return ERR_PTR(-EINVAL); 1219 err = mlxsw_sp_acl_erp_delta_fill(parent_key, key, 1220 &delta_start, &delta_mask); 1221 if (err) 1222 return ERR_PTR(-EINVAL); 1223 1224 delta = kzalloc(sizeof(*delta), GFP_KERNEL); 1225 if (!delta) 1226 return ERR_PTR(-ENOMEM); 1227 delta->start = delta_start; 1228 delta->mask = delta_mask; 1229 1230 err = mlxsw_sp_acl_erp_delta_inc(erp_table); 1231 if (err) 1232 goto err_erp_delta_inc; 1233 1234 memcpy(&delta->key, key, sizeof(*key)); 1235 err = mlxsw_sp_acl_erp_master_mask_set(erp_table, &delta->key); 1236 if (err) 1237 goto err_master_mask_set; 1238 1239 return delta; 1240 1241 err_master_mask_set: 1242 mlxsw_sp_acl_erp_delta_dec(erp_table); 1243 err_erp_delta_inc: 1244 kfree(delta); 1245 return ERR_PTR(err); 1246 } 1247 1248 static void mlxsw_sp_acl_erp_delta_destroy(void *priv, void *delta_priv) 1249 { 1250 struct mlxsw_sp_acl_erp_delta *delta = delta_priv; 1251 struct mlxsw_sp_acl_atcam_region *aregion = priv; 1252 struct mlxsw_sp_acl_erp_table *erp_table = aregion->erp_table; 1253 1254 mlxsw_sp_acl_erp_master_mask_clear(erp_table, &delta->key); 1255 mlxsw_sp_acl_erp_delta_dec(erp_table); 1256 kfree(delta); 1257 } 1258 1259 static void *mlxsw_sp_acl_erp_root_create(void *priv, void *obj) 1260 { 1261 struct mlxsw_sp_acl_atcam_region *aregion = priv; 1262 struct mlxsw_sp_acl_erp_table *erp_table = aregion->erp_table; 1263 struct mlxsw_sp_acl_erp_key *key = obj; 1264 1265 return erp_table->ops->erp_create(erp_table, key); 1266 } 1267 1268 static void mlxsw_sp_acl_erp_root_destroy(void *priv, void *root_priv) 1269 { 1270 struct mlxsw_sp_acl_atcam_region *aregion = priv; 1271 struct mlxsw_sp_acl_erp_table *erp_table = aregion->erp_table; 1272 1273 erp_table->ops->erp_destroy(erp_table, root_priv); 1274 } 1275 1276 static const struct objagg_ops mlxsw_sp_acl_erp_objagg_ops = { 1277 .obj_size = sizeof(struct mlxsw_sp_acl_erp_key), 1278 .delta_create = mlxsw_sp_acl_erp_delta_create, 1279 .delta_destroy = mlxsw_sp_acl_erp_delta_destroy, 1280 .root_create = mlxsw_sp_acl_erp_root_create, 1281 .root_destroy = mlxsw_sp_acl_erp_root_destroy, 1282 }; 1283 1284 static struct mlxsw_sp_acl_erp_table * 1285 mlxsw_sp_acl_erp_table_create(struct mlxsw_sp_acl_atcam_region *aregion) 1286 { 1287 struct mlxsw_sp_acl_erp_table *erp_table; 1288 int err; 1289 1290 erp_table = kzalloc(sizeof(*erp_table), GFP_KERNEL); 1291 if (!erp_table) 1292 return ERR_PTR(-ENOMEM); 1293 1294 erp_table->objagg = objagg_create(&mlxsw_sp_acl_erp_objagg_ops, 1295 aregion); 1296 if (IS_ERR(erp_table->objagg)) { 1297 err = PTR_ERR(erp_table->objagg); 1298 goto err_objagg_create; 1299 } 1300 1301 erp_table->erp_core = aregion->atcam->erp_core; 1302 erp_table->ops = &erp_no_mask_ops; 1303 INIT_LIST_HEAD(&erp_table->atcam_erps_list); 1304 erp_table->aregion = aregion; 1305 1306 return erp_table; 1307 1308 err_objagg_create: 1309 kfree(erp_table); 1310 return ERR_PTR(err); 1311 } 1312 1313 static void 1314 mlxsw_sp_acl_erp_table_destroy(struct mlxsw_sp_acl_erp_table *erp_table) 1315 { 1316 WARN_ON(!list_empty(&erp_table->atcam_erps_list)); 1317 objagg_destroy(erp_table->objagg); 1318 kfree(erp_table); 1319 } 1320 1321 static int 1322 mlxsw_sp_acl_erp_master_mask_init(struct mlxsw_sp_acl_atcam_region *aregion) 1323 { 1324 struct mlxsw_sp *mlxsw_sp = aregion->region->mlxsw_sp; 1325 char percr_pl[MLXSW_REG_PERCR_LEN]; 1326 1327 mlxsw_reg_percr_pack(percr_pl, aregion->region->id); 1328 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(percr), percr_pl); 1329 } 1330 1331 static int 1332 mlxsw_sp_acl_erp_region_param_init(struct mlxsw_sp_acl_atcam_region *aregion) 1333 { 1334 struct mlxsw_sp *mlxsw_sp = aregion->region->mlxsw_sp; 1335 char pererp_pl[MLXSW_REG_PERERP_LEN]; 1336 1337 mlxsw_reg_pererp_pack(pererp_pl, aregion->region->id, false, false, 0, 1338 0, 0); 1339 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(pererp), pererp_pl); 1340 } 1341 1342 int mlxsw_sp_acl_erp_region_init(struct mlxsw_sp_acl_atcam_region *aregion) 1343 { 1344 struct mlxsw_sp_acl_erp_table *erp_table; 1345 int err; 1346 1347 erp_table = mlxsw_sp_acl_erp_table_create(aregion); 1348 if (IS_ERR(erp_table)) 1349 return PTR_ERR(erp_table); 1350 aregion->erp_table = erp_table; 1351 1352 /* Initialize the region's master mask to all zeroes */ 1353 err = mlxsw_sp_acl_erp_master_mask_init(aregion); 1354 if (err) 1355 goto err_erp_master_mask_init; 1356 1357 /* Initialize the region to not use the eRP table */ 1358 err = mlxsw_sp_acl_erp_region_param_init(aregion); 1359 if (err) 1360 goto err_erp_region_param_init; 1361 1362 return 0; 1363 1364 err_erp_region_param_init: 1365 err_erp_master_mask_init: 1366 mlxsw_sp_acl_erp_table_destroy(erp_table); 1367 return err; 1368 } 1369 1370 void mlxsw_sp_acl_erp_region_fini(struct mlxsw_sp_acl_atcam_region *aregion) 1371 { 1372 mlxsw_sp_acl_erp_table_destroy(aregion->erp_table); 1373 } 1374 1375 static int 1376 mlxsw_sp_acl_erp_tables_sizes_query(struct mlxsw_sp *mlxsw_sp, 1377 struct mlxsw_sp_acl_erp_core *erp_core) 1378 { 1379 unsigned int size; 1380 1381 if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, ACL_ERPT_ENTRIES_2KB) || 1382 !MLXSW_CORE_RES_VALID(mlxsw_sp->core, ACL_ERPT_ENTRIES_4KB) || 1383 !MLXSW_CORE_RES_VALID(mlxsw_sp->core, ACL_ERPT_ENTRIES_8KB) || 1384 !MLXSW_CORE_RES_VALID(mlxsw_sp->core, ACL_ERPT_ENTRIES_12KB)) 1385 return -EIO; 1386 1387 size = MLXSW_CORE_RES_GET(mlxsw_sp->core, ACL_ERPT_ENTRIES_2KB); 1388 erp_core->erpt_entries_size[MLXSW_SP_ACL_ATCAM_REGION_TYPE_2KB] = size; 1389 1390 size = MLXSW_CORE_RES_GET(mlxsw_sp->core, ACL_ERPT_ENTRIES_4KB); 1391 erp_core->erpt_entries_size[MLXSW_SP_ACL_ATCAM_REGION_TYPE_4KB] = size; 1392 1393 size = MLXSW_CORE_RES_GET(mlxsw_sp->core, ACL_ERPT_ENTRIES_8KB); 1394 erp_core->erpt_entries_size[MLXSW_SP_ACL_ATCAM_REGION_TYPE_8KB] = size; 1395 1396 size = MLXSW_CORE_RES_GET(mlxsw_sp->core, ACL_ERPT_ENTRIES_12KB); 1397 erp_core->erpt_entries_size[MLXSW_SP_ACL_ATCAM_REGION_TYPE_12KB] = size; 1398 1399 return 0; 1400 } 1401 1402 static int mlxsw_sp_acl_erp_tables_init(struct mlxsw_sp *mlxsw_sp, 1403 struct mlxsw_sp_acl_erp_core *erp_core) 1404 { 1405 unsigned int erpt_bank_size; 1406 int err; 1407 1408 if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, ACL_MAX_ERPT_BANK_SIZE) || 1409 !MLXSW_CORE_RES_VALID(mlxsw_sp->core, ACL_MAX_ERPT_BANKS)) 1410 return -EIO; 1411 erpt_bank_size = MLXSW_CORE_RES_GET(mlxsw_sp->core, 1412 ACL_MAX_ERPT_BANK_SIZE); 1413 erp_core->num_erp_banks = MLXSW_CORE_RES_GET(mlxsw_sp->core, 1414 ACL_MAX_ERPT_BANKS); 1415 1416 erp_core->erp_tables = gen_pool_create(0, -1); 1417 if (!erp_core->erp_tables) 1418 return -ENOMEM; 1419 gen_pool_set_algo(erp_core->erp_tables, gen_pool_best_fit, NULL); 1420 1421 err = gen_pool_add(erp_core->erp_tables, 1422 MLXSW_SP_ACL_ERP_GENALLOC_OFFSET, erpt_bank_size, 1423 -1); 1424 if (err) 1425 goto err_gen_pool_add; 1426 1427 erp_core->bf = mlxsw_sp_acl_bf_init(mlxsw_sp, erp_core->num_erp_banks); 1428 if (IS_ERR(erp_core->bf)) { 1429 err = PTR_ERR(erp_core->bf); 1430 goto err_bf_init; 1431 } 1432 1433 /* Different regions require masks of different sizes */ 1434 err = mlxsw_sp_acl_erp_tables_sizes_query(mlxsw_sp, erp_core); 1435 if (err) 1436 goto err_erp_tables_sizes_query; 1437 1438 return 0; 1439 1440 err_erp_tables_sizes_query: 1441 mlxsw_sp_acl_bf_fini(erp_core->bf); 1442 err_bf_init: 1443 err_gen_pool_add: 1444 gen_pool_destroy(erp_core->erp_tables); 1445 return err; 1446 } 1447 1448 static void mlxsw_sp_acl_erp_tables_fini(struct mlxsw_sp *mlxsw_sp, 1449 struct mlxsw_sp_acl_erp_core *erp_core) 1450 { 1451 mlxsw_sp_acl_bf_fini(erp_core->bf); 1452 gen_pool_destroy(erp_core->erp_tables); 1453 } 1454 1455 int mlxsw_sp_acl_erps_init(struct mlxsw_sp *mlxsw_sp, 1456 struct mlxsw_sp_acl_atcam *atcam) 1457 { 1458 struct mlxsw_sp_acl_erp_core *erp_core; 1459 int err; 1460 1461 erp_core = kzalloc(sizeof(*erp_core), GFP_KERNEL); 1462 if (!erp_core) 1463 return -ENOMEM; 1464 erp_core->mlxsw_sp = mlxsw_sp; 1465 atcam->erp_core = erp_core; 1466 1467 err = mlxsw_sp_acl_erp_tables_init(mlxsw_sp, erp_core); 1468 if (err) 1469 goto err_erp_tables_init; 1470 1471 return 0; 1472 1473 err_erp_tables_init: 1474 kfree(erp_core); 1475 return err; 1476 } 1477 1478 void mlxsw_sp_acl_erps_fini(struct mlxsw_sp *mlxsw_sp, 1479 struct mlxsw_sp_acl_atcam *atcam) 1480 { 1481 mlxsw_sp_acl_erp_tables_fini(mlxsw_sp, atcam->erp_core); 1482 kfree(atcam->erp_core); 1483 } 1484