1 // SPDX-License-Identifier: GPL-2.0 2 /* Marvell RVU Ethernet driver 3 * 4 * Copyright (C) 2021 Marvell. 5 * 6 */ 7 8 #include <linux/netdevice.h> 9 #include <linux/etherdevice.h> 10 #include <linux/inetdevice.h> 11 #include <linux/rhashtable.h> 12 #include <linux/bitfield.h> 13 #include <net/flow_dissector.h> 14 #include <net/pkt_cls.h> 15 #include <net/tc_act/tc_gact.h> 16 #include <net/tc_act/tc_mirred.h> 17 #include <net/tc_act/tc_vlan.h> 18 #include <net/ipv6.h> 19 20 #include "cn10k.h" 21 #include "otx2_common.h" 22 #include "qos.h" 23 24 #define CN10K_MAX_BURST_MANTISSA 0x7FFFULL 25 #define CN10K_MAX_BURST_SIZE 8453888ULL 26 27 #define CN10K_TLX_BURST_MANTISSA GENMASK_ULL(43, 29) 28 #define CN10K_TLX_BURST_EXPONENT GENMASK_ULL(47, 44) 29 30 #define OTX2_UNSUPP_LSE_DEPTH GENMASK(6, 4) 31 32 #define MCAST_INVALID_GRP (-1U) 33 34 static void otx2_get_egress_burst_cfg(struct otx2_nic *nic, u32 burst, 35 u32 *burst_exp, u32 *burst_mantissa) 36 { 37 int max_burst, max_mantissa; 38 unsigned int tmp; 39 40 if (is_dev_otx2(nic->pdev)) { 41 max_burst = MAX_BURST_SIZE; 42 max_mantissa = MAX_BURST_MANTISSA; 43 } else { 44 max_burst = CN10K_MAX_BURST_SIZE; 45 max_mantissa = CN10K_MAX_BURST_MANTISSA; 46 } 47 48 /* Burst is calculated as 49 * ((256 + BURST_MANTISSA) << (1 + BURST_EXPONENT)) / 256 50 * Max supported burst size is 130,816 bytes. 51 */ 52 burst = min_t(u32, burst, max_burst); 53 if (burst) { 54 *burst_exp = ilog2(burst) ? ilog2(burst) - 1 : 0; 55 tmp = burst - rounddown_pow_of_two(burst); 56 if (burst < max_mantissa) 57 *burst_mantissa = tmp * 2; 58 else 59 *burst_mantissa = tmp / (1ULL << (*burst_exp - 7)); 60 } else { 61 *burst_exp = MAX_BURST_EXPONENT; 62 *burst_mantissa = max_mantissa; 63 } 64 } 65 66 static void otx2_get_egress_rate_cfg(u64 maxrate, u32 *exp, 67 u32 *mantissa, u32 *div_exp) 68 { 69 u64 tmp; 70 71 /* Rate calculation by hardware 72 * 73 * PIR_ADD = ((256 + mantissa) << exp) / 256 74 * rate = (2 * PIR_ADD) / ( 1 << div_exp) 75 * The resultant rate is in Mbps. 76 */ 77 78 /* 2Mbps to 100Gbps can be expressed with div_exp = 0. 79 * Setting this to '0' will ease the calculation of 80 * exponent and mantissa. 81 */ 82 *div_exp = 0; 83 84 if (maxrate) { 85 *exp = ilog2(maxrate) ? ilog2(maxrate) - 1 : 0; 86 tmp = maxrate - rounddown_pow_of_two(maxrate); 87 if (maxrate < MAX_RATE_MANTISSA) 88 *mantissa = tmp * 2; 89 else 90 *mantissa = tmp / (1ULL << (*exp - 7)); 91 } else { 92 /* Instead of disabling rate limiting, set all values to max */ 93 *exp = MAX_RATE_EXPONENT; 94 *mantissa = MAX_RATE_MANTISSA; 95 } 96 } 97 98 u64 otx2_get_txschq_rate_regval(struct otx2_nic *nic, 99 u64 maxrate, u32 burst) 100 { 101 u32 burst_exp, burst_mantissa; 102 u32 exp, mantissa, div_exp; 103 u64 regval = 0; 104 105 /* Get exponent and mantissa values from the desired rate */ 106 otx2_get_egress_burst_cfg(nic, burst, &burst_exp, &burst_mantissa); 107 otx2_get_egress_rate_cfg(maxrate, &exp, &mantissa, &div_exp); 108 109 if (is_dev_otx2(nic->pdev)) { 110 regval = FIELD_PREP(TLX_BURST_EXPONENT, (u64)burst_exp) | 111 FIELD_PREP(TLX_BURST_MANTISSA, (u64)burst_mantissa) | 112 FIELD_PREP(TLX_RATE_DIVIDER_EXPONENT, div_exp) | 113 FIELD_PREP(TLX_RATE_EXPONENT, exp) | 114 FIELD_PREP(TLX_RATE_MANTISSA, mantissa) | BIT_ULL(0); 115 } else { 116 regval = FIELD_PREP(CN10K_TLX_BURST_EXPONENT, (u64)burst_exp) | 117 FIELD_PREP(CN10K_TLX_BURST_MANTISSA, (u64)burst_mantissa) | 118 FIELD_PREP(TLX_RATE_DIVIDER_EXPONENT, div_exp) | 119 FIELD_PREP(TLX_RATE_EXPONENT, exp) | 120 FIELD_PREP(TLX_RATE_MANTISSA, mantissa) | BIT_ULL(0); 121 } 122 123 return regval; 124 } 125 126 static int otx2_set_matchall_egress_rate(struct otx2_nic *nic, 127 u32 burst, u64 maxrate) 128 { 129 struct otx2_hw *hw = &nic->hw; 130 struct nix_txschq_config *req; 131 int txschq, err; 132 133 /* All SQs share the same TL4, so pick the first scheduler */ 134 txschq = hw->txschq_list[NIX_TXSCH_LVL_TL4][0]; 135 136 mutex_lock(&nic->mbox.lock); 137 req = otx2_mbox_alloc_msg_nix_txschq_cfg(&nic->mbox); 138 if (!req) { 139 mutex_unlock(&nic->mbox.lock); 140 return -ENOMEM; 141 } 142 143 req->lvl = NIX_TXSCH_LVL_TL4; 144 req->num_regs = 1; 145 req->reg[0] = NIX_AF_TL4X_PIR(txschq); 146 req->regval[0] = otx2_get_txschq_rate_regval(nic, maxrate, burst); 147 148 err = otx2_sync_mbox_msg(&nic->mbox); 149 mutex_unlock(&nic->mbox.lock); 150 return err; 151 } 152 153 static int otx2_tc_validate_flow(struct otx2_nic *nic, 154 struct flow_action *actions, 155 struct netlink_ext_ack *extack) 156 { 157 if (nic->flags & OTX2_FLAG_INTF_DOWN) { 158 NL_SET_ERR_MSG_MOD(extack, "Interface not initialized"); 159 return -EINVAL; 160 } 161 162 if (!flow_action_has_entries(actions)) { 163 NL_SET_ERR_MSG_MOD(extack, "MATCHALL offload called with no action"); 164 return -EINVAL; 165 } 166 167 if (!flow_offload_has_one_action(actions)) { 168 NL_SET_ERR_MSG_MOD(extack, 169 "Egress MATCHALL offload supports only 1 policing action"); 170 return -EINVAL; 171 } 172 return 0; 173 } 174 175 static int otx2_policer_validate(const struct flow_action *action, 176 const struct flow_action_entry *act, 177 struct netlink_ext_ack *extack) 178 { 179 if (act->police.exceed.act_id != FLOW_ACTION_DROP) { 180 NL_SET_ERR_MSG_MOD(extack, 181 "Offload not supported when exceed action is not drop"); 182 return -EOPNOTSUPP; 183 } 184 185 if (act->police.notexceed.act_id != FLOW_ACTION_PIPE && 186 act->police.notexceed.act_id != FLOW_ACTION_ACCEPT) { 187 NL_SET_ERR_MSG_MOD(extack, 188 "Offload not supported when conform action is not pipe or ok"); 189 return -EOPNOTSUPP; 190 } 191 192 if (act->police.notexceed.act_id == FLOW_ACTION_ACCEPT && 193 !flow_action_is_last_entry(action, act)) { 194 NL_SET_ERR_MSG_MOD(extack, 195 "Offload not supported when conform action is ok, but action is not last"); 196 return -EOPNOTSUPP; 197 } 198 199 if (act->police.peakrate_bytes_ps || 200 act->police.avrate || act->police.overhead) { 201 NL_SET_ERR_MSG_MOD(extack, 202 "Offload not supported when peakrate/avrate/overhead is configured"); 203 return -EOPNOTSUPP; 204 } 205 206 return 0; 207 } 208 209 static int otx2_tc_egress_matchall_install(struct otx2_nic *nic, 210 struct tc_cls_matchall_offload *cls) 211 { 212 struct netlink_ext_ack *extack = cls->common.extack; 213 struct flow_action *actions = &cls->rule->action; 214 struct flow_action_entry *entry; 215 int err; 216 217 err = otx2_tc_validate_flow(nic, actions, extack); 218 if (err) 219 return err; 220 221 if (nic->flags & OTX2_FLAG_TC_MATCHALL_EGRESS_ENABLED) { 222 NL_SET_ERR_MSG_MOD(extack, 223 "Only one Egress MATCHALL ratelimiter can be offloaded"); 224 return -ENOMEM; 225 } 226 227 entry = &cls->rule->action.entries[0]; 228 switch (entry->id) { 229 case FLOW_ACTION_POLICE: 230 err = otx2_policer_validate(&cls->rule->action, entry, extack); 231 if (err) 232 return err; 233 234 if (entry->police.rate_pkt_ps) { 235 NL_SET_ERR_MSG_MOD(extack, "QoS offload not support packets per second"); 236 return -EOPNOTSUPP; 237 } 238 err = otx2_set_matchall_egress_rate(nic, entry->police.burst, 239 otx2_convert_rate(entry->police.rate_bytes_ps)); 240 if (err) 241 return err; 242 nic->flags |= OTX2_FLAG_TC_MATCHALL_EGRESS_ENABLED; 243 break; 244 default: 245 NL_SET_ERR_MSG_MOD(extack, 246 "Only police action is supported with Egress MATCHALL offload"); 247 return -EOPNOTSUPP; 248 } 249 250 return 0; 251 } 252 253 static int otx2_tc_egress_matchall_delete(struct otx2_nic *nic, 254 struct tc_cls_matchall_offload *cls) 255 { 256 struct netlink_ext_ack *extack = cls->common.extack; 257 int err; 258 259 if (nic->flags & OTX2_FLAG_INTF_DOWN) { 260 NL_SET_ERR_MSG_MOD(extack, "Interface not initialized"); 261 return -EINVAL; 262 } 263 264 err = otx2_set_matchall_egress_rate(nic, 0, 0); 265 nic->flags &= ~OTX2_FLAG_TC_MATCHALL_EGRESS_ENABLED; 266 return err; 267 } 268 269 static int otx2_tc_act_set_hw_police(struct otx2_nic *nic, 270 struct otx2_tc_flow *node) 271 { 272 int rc; 273 274 mutex_lock(&nic->mbox.lock); 275 276 rc = cn10k_alloc_leaf_profile(nic, &node->leaf_profile); 277 if (rc) { 278 mutex_unlock(&nic->mbox.lock); 279 return rc; 280 } 281 282 rc = cn10k_set_ipolicer_rate(nic, node->leaf_profile, 283 node->burst, node->rate, node->is_pps); 284 if (rc) 285 goto free_leaf; 286 287 rc = cn10k_map_unmap_rq_policer(nic, node->rq, node->leaf_profile, true); 288 if (rc) 289 goto free_leaf; 290 291 mutex_unlock(&nic->mbox.lock); 292 293 return 0; 294 295 free_leaf: 296 if (cn10k_free_leaf_profile(nic, node->leaf_profile)) 297 netdev_err(nic->netdev, 298 "Unable to free leaf bandwidth profile(%d)\n", 299 node->leaf_profile); 300 mutex_unlock(&nic->mbox.lock); 301 return rc; 302 } 303 304 static int otx2_tc_act_set_police(struct otx2_nic *nic, 305 struct otx2_tc_flow *node, 306 struct flow_cls_offload *f, 307 u64 rate, u32 burst, u32 mark, 308 struct npc_install_flow_req *req, bool pps) 309 { 310 struct netlink_ext_ack *extack = f->common.extack; 311 struct otx2_hw *hw = &nic->hw; 312 int rq_idx, rc; 313 314 rq_idx = find_first_zero_bit(&nic->rq_bmap, hw->rx_queues); 315 if (rq_idx >= hw->rx_queues) { 316 NL_SET_ERR_MSG_MOD(extack, "Police action rules exceeded"); 317 return -EINVAL; 318 } 319 320 req->match_id = mark & 0xFFFFULL; 321 req->index = rq_idx; 322 req->op = NIX_RX_ACTIONOP_UCAST; 323 324 node->is_act_police = true; 325 node->rq = rq_idx; 326 node->burst = burst; 327 node->rate = rate; 328 node->is_pps = pps; 329 330 rc = otx2_tc_act_set_hw_police(nic, node); 331 if (!rc) 332 set_bit(rq_idx, &nic->rq_bmap); 333 334 return rc; 335 } 336 337 static int otx2_tc_update_mcast(struct otx2_nic *nic, 338 struct npc_install_flow_req *req, 339 struct netlink_ext_ack *extack, 340 struct otx2_tc_flow *node, 341 struct nix_mcast_grp_update_req *ureq, 342 u8 num_intf) 343 { 344 struct nix_mcast_grp_update_req *grp_update_req; 345 struct nix_mcast_grp_create_req *creq; 346 struct nix_mcast_grp_create_rsp *crsp; 347 u32 grp_index; 348 int rc; 349 350 mutex_lock(&nic->mbox.lock); 351 creq = otx2_mbox_alloc_msg_nix_mcast_grp_create(&nic->mbox); 352 if (!creq) { 353 rc = -ENOMEM; 354 goto error; 355 } 356 357 creq->dir = NIX_MCAST_INGRESS; 358 /* Send message to AF */ 359 rc = otx2_sync_mbox_msg(&nic->mbox); 360 if (rc) { 361 NL_SET_ERR_MSG_MOD(extack, "Failed to create multicast group"); 362 goto error; 363 } 364 365 crsp = (struct nix_mcast_grp_create_rsp *)otx2_mbox_get_rsp(&nic->mbox.mbox, 366 0, 367 &creq->hdr); 368 if (IS_ERR(crsp)) { 369 rc = PTR_ERR(crsp); 370 goto error; 371 } 372 373 grp_index = crsp->mcast_grp_idx; 374 grp_update_req = otx2_mbox_alloc_msg_nix_mcast_grp_update(&nic->mbox); 375 if (!grp_update_req) { 376 NL_SET_ERR_MSG_MOD(extack, "Failed to update multicast group"); 377 rc = -ENOMEM; 378 goto error; 379 } 380 381 ureq->op = NIX_MCAST_OP_ADD_ENTRY; 382 ureq->mcast_grp_idx = grp_index; 383 ureq->num_mce_entry = num_intf; 384 ureq->pcifunc[0] = nic->pcifunc; 385 ureq->channel[0] = nic->hw.tx_chan_base; 386 387 ureq->dest_type[0] = NIX_RX_RSS; 388 ureq->rq_rss_index[0] = 0; 389 memcpy(&ureq->hdr, &grp_update_req->hdr, sizeof(struct mbox_msghdr)); 390 memcpy(grp_update_req, ureq, sizeof(struct nix_mcast_grp_update_req)); 391 392 /* Send message to AF */ 393 rc = otx2_sync_mbox_msg(&nic->mbox); 394 if (rc) { 395 NL_SET_ERR_MSG_MOD(extack, "Failed to update multicast group"); 396 goto error; 397 } 398 399 mutex_unlock(&nic->mbox.lock); 400 req->op = NIX_RX_ACTIONOP_MCAST; 401 req->index = grp_index; 402 node->mcast_grp_idx = grp_index; 403 return 0; 404 405 error: 406 mutex_unlock(&nic->mbox.lock); 407 return rc; 408 } 409 410 static int otx2_tc_parse_actions(struct otx2_nic *nic, 411 struct flow_action *flow_action, 412 struct npc_install_flow_req *req, 413 struct flow_cls_offload *f, 414 struct otx2_tc_flow *node) 415 { 416 struct nix_mcast_grp_update_req dummy_grp_update_req = { 0 }; 417 struct netlink_ext_ack *extack = f->common.extack; 418 bool pps = false, mcast = false; 419 struct flow_action_entry *act; 420 struct net_device *target; 421 struct otx2_nic *priv; 422 struct rep_dev *rdev; 423 u32 burst, mark = 0; 424 u8 nr_police = 0; 425 u8 num_intf = 1; 426 int err, i; 427 u64 rate; 428 429 if (!flow_action_has_entries(flow_action)) { 430 NL_SET_ERR_MSG_MOD(extack, "no tc actions specified"); 431 return -EINVAL; 432 } 433 434 flow_action_for_each(i, act, flow_action) { 435 switch (act->id) { 436 case FLOW_ACTION_DROP: 437 req->op = NIX_RX_ACTIONOP_DROP; 438 return 0; 439 case FLOW_ACTION_ACCEPT: 440 req->op = NIX_RX_ACTION_DEFAULT; 441 return 0; 442 case FLOW_ACTION_REDIRECT_INGRESS: 443 target = act->dev; 444 if (target->dev.parent) { 445 priv = netdev_priv(target); 446 if (rvu_get_pf(nic->pdev, nic->pcifunc) != 447 rvu_get_pf(nic->pdev, priv->pcifunc)) { 448 NL_SET_ERR_MSG_MOD(extack, 449 "can't redirect to other pf/vf"); 450 return -EOPNOTSUPP; 451 } 452 req->vf = priv->pcifunc & RVU_PFVF_FUNC_MASK; 453 } else { 454 rdev = netdev_priv(target); 455 req->vf = rdev->pcifunc & RVU_PFVF_FUNC_MASK; 456 } 457 458 /* if op is already set; avoid overwriting the same */ 459 if (!req->op) 460 req->op = NIX_RX_ACTION_DEFAULT; 461 break; 462 463 case FLOW_ACTION_VLAN_POP: 464 req->vtag0_valid = true; 465 /* use RX_VTAG_TYPE7 which is initialized to strip vlan tag */ 466 req->vtag0_type = NIX_AF_LFX_RX_VTAG_TYPE7; 467 break; 468 case FLOW_ACTION_POLICE: 469 /* Ingress ratelimiting is not supported on OcteonTx2 */ 470 if (is_dev_otx2(nic->pdev)) { 471 NL_SET_ERR_MSG_MOD(extack, 472 "Ingress policing not supported on this platform"); 473 return -EOPNOTSUPP; 474 } 475 476 err = otx2_policer_validate(flow_action, act, extack); 477 if (err) 478 return err; 479 480 if (act->police.rate_bytes_ps > 0) { 481 rate = act->police.rate_bytes_ps * 8; 482 burst = act->police.burst; 483 } else if (act->police.rate_pkt_ps > 0) { 484 /* The algorithm used to calculate rate 485 * mantissa, exponent values for a given token 486 * rate (token can be byte or packet) requires 487 * token rate to be mutiplied by 8. 488 */ 489 rate = act->police.rate_pkt_ps * 8; 490 burst = act->police.burst_pkt; 491 pps = true; 492 } 493 nr_police++; 494 break; 495 case FLOW_ACTION_MARK: 496 if (act->mark & ~OTX2_RX_MATCH_ID_MASK) { 497 NL_SET_ERR_MSG_MOD(extack, "Bad flow mark, only 16 bit supported"); 498 return -EOPNOTSUPP; 499 } 500 mark = act->mark; 501 req->match_id = mark & OTX2_RX_MATCH_ID_MASK; 502 req->op = NIX_RX_ACTION_DEFAULT; 503 nic->flags |= OTX2_FLAG_TC_MARK_ENABLED; 504 refcount_inc(&nic->flow_cfg->mark_flows); 505 break; 506 507 case FLOW_ACTION_RX_QUEUE_MAPPING: 508 req->op = NIX_RX_ACTIONOP_UCAST; 509 req->index = act->rx_queue; 510 break; 511 512 case FLOW_ACTION_MIRRED_INGRESS: 513 target = act->dev; 514 priv = netdev_priv(target); 515 dummy_grp_update_req.pcifunc[num_intf] = priv->pcifunc; 516 dummy_grp_update_req.channel[num_intf] = priv->hw.tx_chan_base; 517 dummy_grp_update_req.dest_type[num_intf] = NIX_RX_RSS; 518 dummy_grp_update_req.rq_rss_index[num_intf] = 0; 519 mcast = true; 520 num_intf++; 521 break; 522 523 default: 524 return -EOPNOTSUPP; 525 } 526 } 527 528 if (mcast) { 529 err = otx2_tc_update_mcast(nic, req, extack, node, 530 &dummy_grp_update_req, 531 num_intf); 532 if (err) 533 return err; 534 } 535 536 if (nr_police > 1) { 537 NL_SET_ERR_MSG_MOD(extack, 538 "rate limit police offload requires a single action"); 539 return -EOPNOTSUPP; 540 } 541 542 if (nr_police) 543 return otx2_tc_act_set_police(nic, node, f, rate, burst, 544 mark, req, pps); 545 546 return 0; 547 } 548 549 static int otx2_tc_process_vlan(struct otx2_nic *nic, struct flow_msg *flow_spec, 550 struct flow_msg *flow_mask, struct flow_rule *rule, 551 struct npc_install_flow_req *req, bool is_inner) 552 { 553 struct flow_match_vlan match; 554 u16 vlan_tci, vlan_tci_mask; 555 556 if (is_inner) 557 flow_rule_match_cvlan(rule, &match); 558 else 559 flow_rule_match_vlan(rule, &match); 560 561 if (!eth_type_vlan(match.key->vlan_tpid)) { 562 netdev_err(nic->netdev, "vlan tpid 0x%x not supported\n", 563 ntohs(match.key->vlan_tpid)); 564 return -EOPNOTSUPP; 565 } 566 567 if (!match.mask->vlan_id) { 568 struct flow_action_entry *act; 569 int i; 570 571 flow_action_for_each(i, act, &rule->action) { 572 if (act->id == FLOW_ACTION_DROP) { 573 netdev_err(nic->netdev, 574 "vlan tpid 0x%x with vlan_id %d is not supported for DROP rule.\n", 575 ntohs(match.key->vlan_tpid), match.key->vlan_id); 576 return -EOPNOTSUPP; 577 } 578 } 579 } 580 581 if (match.mask->vlan_id || 582 match.mask->vlan_dei || 583 match.mask->vlan_priority) { 584 vlan_tci = match.key->vlan_id | 585 match.key->vlan_dei << 12 | 586 match.key->vlan_priority << 13; 587 588 vlan_tci_mask = match.mask->vlan_id | 589 match.mask->vlan_dei << 12 | 590 match.mask->vlan_priority << 13; 591 if (is_inner) { 592 flow_spec->vlan_itci = htons(vlan_tci); 593 flow_mask->vlan_itci = htons(vlan_tci_mask); 594 req->features |= BIT_ULL(NPC_INNER_VID); 595 } else { 596 flow_spec->vlan_tci = htons(vlan_tci); 597 flow_mask->vlan_tci = htons(vlan_tci_mask); 598 req->features |= BIT_ULL(NPC_OUTER_VID); 599 } 600 } 601 602 return 0; 603 } 604 605 static int otx2_tc_prepare_flow(struct otx2_nic *nic, struct otx2_tc_flow *node, 606 struct flow_cls_offload *f, 607 struct npc_install_flow_req *req) 608 { 609 struct netlink_ext_ack *extack = f->common.extack; 610 struct flow_msg *flow_spec = &req->packet; 611 struct flow_msg *flow_mask = &req->mask; 612 struct flow_dissector *dissector; 613 struct flow_rule *rule; 614 u8 ip_proto = 0; 615 616 rule = flow_cls_offload_flow_rule(f); 617 dissector = rule->match.dissector; 618 619 if ((dissector->used_keys & 620 ~(BIT_ULL(FLOW_DISSECTOR_KEY_CONTROL) | 621 BIT_ULL(FLOW_DISSECTOR_KEY_BASIC) | 622 BIT_ULL(FLOW_DISSECTOR_KEY_ETH_ADDRS) | 623 BIT_ULL(FLOW_DISSECTOR_KEY_VLAN) | 624 BIT(FLOW_DISSECTOR_KEY_CVLAN) | 625 BIT_ULL(FLOW_DISSECTOR_KEY_IPV4_ADDRS) | 626 BIT_ULL(FLOW_DISSECTOR_KEY_IPV6_ADDRS) | 627 BIT_ULL(FLOW_DISSECTOR_KEY_PORTS) | 628 BIT(FLOW_DISSECTOR_KEY_IPSEC) | 629 BIT_ULL(FLOW_DISSECTOR_KEY_MPLS) | 630 BIT_ULL(FLOW_DISSECTOR_KEY_ICMP) | 631 BIT_ULL(FLOW_DISSECTOR_KEY_TCP) | 632 BIT_ULL(FLOW_DISSECTOR_KEY_IP)))) { 633 netdev_info(nic->netdev, "unsupported flow used key 0x%llx", 634 dissector->used_keys); 635 return -EOPNOTSUPP; 636 } 637 638 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_BASIC)) { 639 struct flow_match_basic match; 640 641 flow_rule_match_basic(rule, &match); 642 643 /* All EtherTypes can be matched, no hw limitation */ 644 flow_spec->etype = match.key->n_proto; 645 flow_mask->etype = match.mask->n_proto; 646 req->features |= BIT_ULL(NPC_ETYPE); 647 648 if (match.mask->ip_proto && 649 (match.key->ip_proto != IPPROTO_TCP && 650 match.key->ip_proto != IPPROTO_UDP && 651 match.key->ip_proto != IPPROTO_SCTP && 652 match.key->ip_proto != IPPROTO_ICMP && 653 match.key->ip_proto != IPPROTO_ESP && 654 match.key->ip_proto != IPPROTO_AH && 655 match.key->ip_proto != IPPROTO_ICMPV6)) { 656 netdev_info(nic->netdev, 657 "ip_proto=0x%x not supported\n", 658 match.key->ip_proto); 659 return -EOPNOTSUPP; 660 } 661 if (match.mask->ip_proto) 662 ip_proto = match.key->ip_proto; 663 664 if (ip_proto == IPPROTO_UDP) 665 req->features |= BIT_ULL(NPC_IPPROTO_UDP); 666 else if (ip_proto == IPPROTO_TCP) 667 req->features |= BIT_ULL(NPC_IPPROTO_TCP); 668 else if (ip_proto == IPPROTO_SCTP) 669 req->features |= BIT_ULL(NPC_IPPROTO_SCTP); 670 else if (ip_proto == IPPROTO_ICMP) 671 req->features |= BIT_ULL(NPC_IPPROTO_ICMP); 672 else if (ip_proto == IPPROTO_ICMPV6) 673 req->features |= BIT_ULL(NPC_IPPROTO_ICMP6); 674 else if (ip_proto == IPPROTO_ESP) 675 req->features |= BIT_ULL(NPC_IPPROTO_ESP); 676 else if (ip_proto == IPPROTO_AH) 677 req->features |= BIT_ULL(NPC_IPPROTO_AH); 678 } 679 680 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_CONTROL)) { 681 struct flow_match_control match; 682 u32 val; 683 684 flow_rule_match_control(rule, &match); 685 686 if (match.mask->flags & FLOW_DIS_IS_FRAGMENT) { 687 val = match.key->flags & FLOW_DIS_IS_FRAGMENT; 688 if (ntohs(flow_spec->etype) == ETH_P_IP) { 689 flow_spec->ip_flag = val ? IPV4_FLAG_MORE : 0; 690 flow_mask->ip_flag = IPV4_FLAG_MORE; 691 req->features |= BIT_ULL(NPC_IPFRAG_IPV4); 692 } else if (ntohs(flow_spec->etype) == ETH_P_IPV6) { 693 flow_spec->next_header = val ? 694 IPPROTO_FRAGMENT : 0; 695 flow_mask->next_header = 0xff; 696 req->features |= BIT_ULL(NPC_IPFRAG_IPV6); 697 } else { 698 NL_SET_ERR_MSG_MOD(extack, "flow-type should be either IPv4 and IPv6"); 699 return -EOPNOTSUPP; 700 } 701 } 702 703 if (!flow_rule_is_supp_control_flags(FLOW_DIS_IS_FRAGMENT, 704 match.mask->flags, extack)) 705 return -EOPNOTSUPP; 706 } 707 708 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ETH_ADDRS)) { 709 struct flow_match_eth_addrs match; 710 711 flow_rule_match_eth_addrs(rule, &match); 712 if (!is_zero_ether_addr(match.mask->src)) { 713 NL_SET_ERR_MSG_MOD(extack, "src mac match not supported"); 714 return -EOPNOTSUPP; 715 } 716 717 if (!is_zero_ether_addr(match.mask->dst)) { 718 ether_addr_copy(flow_spec->dmac, (u8 *)&match.key->dst); 719 ether_addr_copy(flow_mask->dmac, 720 (u8 *)&match.mask->dst); 721 req->features |= BIT_ULL(NPC_DMAC); 722 } 723 } 724 725 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_IPSEC)) { 726 struct flow_match_ipsec match; 727 728 flow_rule_match_ipsec(rule, &match); 729 if (!match.mask->spi) { 730 NL_SET_ERR_MSG_MOD(extack, "spi index not specified"); 731 return -EOPNOTSUPP; 732 } 733 if (ip_proto != IPPROTO_ESP && 734 ip_proto != IPPROTO_AH) { 735 NL_SET_ERR_MSG_MOD(extack, 736 "SPI index is valid only for ESP/AH proto"); 737 return -EOPNOTSUPP; 738 } 739 740 flow_spec->spi = match.key->spi; 741 flow_mask->spi = match.mask->spi; 742 req->features |= BIT_ULL(NPC_IPSEC_SPI); 743 } 744 745 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_IP)) { 746 struct flow_match_ip match; 747 748 flow_rule_match_ip(rule, &match); 749 if ((ntohs(flow_spec->etype) != ETH_P_IP) && 750 match.mask->tos) { 751 NL_SET_ERR_MSG_MOD(extack, "tos not supported"); 752 return -EOPNOTSUPP; 753 } 754 if (match.mask->ttl) { 755 NL_SET_ERR_MSG_MOD(extack, "ttl not supported"); 756 return -EOPNOTSUPP; 757 } 758 flow_spec->tos = match.key->tos; 759 flow_mask->tos = match.mask->tos; 760 req->features |= BIT_ULL(NPC_TOS); 761 } 762 763 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_VLAN)) { 764 int ret; 765 766 ret = otx2_tc_process_vlan(nic, flow_spec, flow_mask, rule, req, false); 767 if (ret) 768 return ret; 769 } 770 771 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_CVLAN)) { 772 int ret; 773 774 ret = otx2_tc_process_vlan(nic, flow_spec, flow_mask, rule, req, true); 775 if (ret) 776 return ret; 777 } 778 779 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_IPV4_ADDRS)) { 780 struct flow_match_ipv4_addrs match; 781 782 flow_rule_match_ipv4_addrs(rule, &match); 783 784 flow_spec->ip4dst = match.key->dst; 785 flow_mask->ip4dst = match.mask->dst; 786 req->features |= BIT_ULL(NPC_DIP_IPV4); 787 788 flow_spec->ip4src = match.key->src; 789 flow_mask->ip4src = match.mask->src; 790 req->features |= BIT_ULL(NPC_SIP_IPV4); 791 } else if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_IPV6_ADDRS)) { 792 struct flow_match_ipv6_addrs match; 793 794 flow_rule_match_ipv6_addrs(rule, &match); 795 796 if (ipv6_addr_loopback(&match.key->dst) || 797 ipv6_addr_loopback(&match.key->src)) { 798 NL_SET_ERR_MSG_MOD(extack, 799 "Flow matching IPv6 loopback addr not supported"); 800 return -EOPNOTSUPP; 801 } 802 803 if (!ipv6_addr_any(&match.mask->dst)) { 804 memcpy(&flow_spec->ip6dst, 805 (struct in6_addr *)&match.key->dst, 806 sizeof(flow_spec->ip6dst)); 807 memcpy(&flow_mask->ip6dst, 808 (struct in6_addr *)&match.mask->dst, 809 sizeof(flow_spec->ip6dst)); 810 req->features |= BIT_ULL(NPC_DIP_IPV6); 811 } 812 813 if (!ipv6_addr_any(&match.mask->src)) { 814 memcpy(&flow_spec->ip6src, 815 (struct in6_addr *)&match.key->src, 816 sizeof(flow_spec->ip6src)); 817 memcpy(&flow_mask->ip6src, 818 (struct in6_addr *)&match.mask->src, 819 sizeof(flow_spec->ip6src)); 820 req->features |= BIT_ULL(NPC_SIP_IPV6); 821 } 822 } 823 824 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_PORTS)) { 825 struct flow_match_ports match; 826 827 flow_rule_match_ports(rule, &match); 828 829 flow_spec->dport = match.key->dst; 830 flow_mask->dport = match.mask->dst; 831 832 if (flow_mask->dport) { 833 if (ip_proto == IPPROTO_UDP) 834 req->features |= BIT_ULL(NPC_DPORT_UDP); 835 else if (ip_proto == IPPROTO_TCP) 836 req->features |= BIT_ULL(NPC_DPORT_TCP); 837 else if (ip_proto == IPPROTO_SCTP) 838 req->features |= BIT_ULL(NPC_DPORT_SCTP); 839 } 840 841 flow_spec->sport = match.key->src; 842 flow_mask->sport = match.mask->src; 843 844 if (flow_mask->sport) { 845 if (ip_proto == IPPROTO_UDP) 846 req->features |= BIT_ULL(NPC_SPORT_UDP); 847 else if (ip_proto == IPPROTO_TCP) 848 req->features |= BIT_ULL(NPC_SPORT_TCP); 849 else if (ip_proto == IPPROTO_SCTP) 850 req->features |= BIT_ULL(NPC_SPORT_SCTP); 851 } 852 } 853 854 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_TCP)) { 855 struct flow_match_tcp match; 856 857 flow_rule_match_tcp(rule, &match); 858 859 flow_spec->tcp_flags = match.key->flags; 860 flow_mask->tcp_flags = match.mask->flags; 861 req->features |= BIT_ULL(NPC_TCP_FLAGS); 862 } 863 864 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_MPLS)) { 865 struct flow_match_mpls match; 866 u8 bit; 867 868 flow_rule_match_mpls(rule, &match); 869 870 if (match.mask->used_lses & OTX2_UNSUPP_LSE_DEPTH) { 871 NL_SET_ERR_MSG_MOD(extack, 872 "unsupported LSE depth for MPLS match offload"); 873 return -EOPNOTSUPP; 874 } 875 876 for_each_set_bit(bit, (unsigned long *)&match.mask->used_lses, 877 FLOW_DIS_MPLS_MAX) { 878 /* check if any of the fields LABEL,TC,BOS are set */ 879 if (*((u32 *)&match.mask->ls[bit]) & 880 OTX2_FLOWER_MASK_MPLS_NON_TTL) { 881 /* Hardware will capture 4 byte MPLS header into 882 * two fields NPC_MPLSX_LBTCBOS and NPC_MPLSX_TTL. 883 * Derive the associated NPC key based on header 884 * index and offset. 885 */ 886 887 req->features |= BIT_ULL(NPC_MPLS1_LBTCBOS + 888 2 * bit); 889 flow_spec->mpls_lse[bit] = 890 FIELD_PREP(OTX2_FLOWER_MASK_MPLS_LB, 891 match.key->ls[bit].mpls_label) | 892 FIELD_PREP(OTX2_FLOWER_MASK_MPLS_TC, 893 match.key->ls[bit].mpls_tc) | 894 FIELD_PREP(OTX2_FLOWER_MASK_MPLS_BOS, 895 match.key->ls[bit].mpls_bos); 896 897 flow_mask->mpls_lse[bit] = 898 FIELD_PREP(OTX2_FLOWER_MASK_MPLS_LB, 899 match.mask->ls[bit].mpls_label) | 900 FIELD_PREP(OTX2_FLOWER_MASK_MPLS_TC, 901 match.mask->ls[bit].mpls_tc) | 902 FIELD_PREP(OTX2_FLOWER_MASK_MPLS_BOS, 903 match.mask->ls[bit].mpls_bos); 904 } 905 906 if (match.mask->ls[bit].mpls_ttl) { 907 req->features |= BIT_ULL(NPC_MPLS1_TTL + 908 2 * bit); 909 flow_spec->mpls_lse[bit] |= 910 FIELD_PREP(OTX2_FLOWER_MASK_MPLS_TTL, 911 match.key->ls[bit].mpls_ttl); 912 flow_mask->mpls_lse[bit] |= 913 FIELD_PREP(OTX2_FLOWER_MASK_MPLS_TTL, 914 match.mask->ls[bit].mpls_ttl); 915 } 916 } 917 } 918 919 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ICMP)) { 920 struct flow_match_icmp match; 921 922 flow_rule_match_icmp(rule, &match); 923 924 flow_spec->icmp_type = match.key->type; 925 flow_mask->icmp_type = match.mask->type; 926 req->features |= BIT_ULL(NPC_TYPE_ICMP); 927 928 flow_spec->icmp_code = match.key->code; 929 flow_mask->icmp_code = match.mask->code; 930 req->features |= BIT_ULL(NPC_CODE_ICMP); 931 } 932 return otx2_tc_parse_actions(nic, &rule->action, req, f, node); 933 } 934 935 static void otx2_destroy_tc_flow_list(struct otx2_nic *pfvf) 936 { 937 struct otx2_flow_config *flow_cfg = pfvf->flow_cfg; 938 struct otx2_tc_flow *iter, *tmp; 939 940 if (!(pfvf->flags & OTX2_FLAG_MCAM_ENTRIES_ALLOC)) 941 return; 942 943 list_for_each_entry_safe(iter, tmp, &flow_cfg->flow_list_tc, list) { 944 list_del(&iter->list); 945 kfree(iter); 946 flow_cfg->nr_flows--; 947 } 948 } 949 950 static struct otx2_tc_flow * 951 otx2_tc_get_entry_by_cookie(struct otx2_flow_config *flow_cfg, 952 unsigned long cookie) 953 { 954 struct otx2_tc_flow *tmp; 955 956 list_for_each_entry(tmp, &flow_cfg->flow_list_tc, list) { 957 if (tmp->cookie == cookie) 958 return tmp; 959 } 960 961 return NULL; 962 } 963 964 struct otx2_tc_flow * 965 otx2_tc_get_entry_by_index(struct otx2_flow_config *flow_cfg, int index) 966 { 967 struct otx2_tc_flow *tmp; 968 int i = 0; 969 970 list_for_each_entry(tmp, &flow_cfg->flow_list_tc, list) { 971 if (i == index) 972 return tmp; 973 i++; 974 } 975 976 return NULL; 977 } 978 979 static void otx2_tc_del_from_flow_list(struct otx2_flow_config *flow_cfg, 980 struct otx2_tc_flow *node) 981 { 982 struct list_head *pos, *n; 983 struct otx2_tc_flow *tmp; 984 985 list_for_each_safe(pos, n, &flow_cfg->flow_list_tc) { 986 tmp = list_entry(pos, struct otx2_tc_flow, list); 987 if (node == tmp) { 988 list_del(&node->list); 989 return; 990 } 991 } 992 } 993 994 int otx2_tc_add_to_flow_list(struct otx2_flow_config *flow_cfg, 995 struct otx2_tc_flow *node) 996 { 997 struct list_head *pos, *n; 998 struct otx2_tc_flow *tmp; 999 int index = 0; 1000 1001 /* If the flow list is empty then add the new node */ 1002 if (list_empty(&flow_cfg->flow_list_tc)) { 1003 list_add(&node->list, &flow_cfg->flow_list_tc); 1004 return index; 1005 } 1006 1007 list_for_each_safe(pos, n, &flow_cfg->flow_list_tc) { 1008 tmp = list_entry(pos, struct otx2_tc_flow, list); 1009 if (node->prio < tmp->prio) 1010 break; 1011 index++; 1012 } 1013 1014 list_add(&node->list, pos->prev); 1015 return index; 1016 } 1017 1018 int otx2_add_mcam_flow_entry(struct otx2_nic *nic, 1019 struct npc_install_flow_req *req) 1020 { 1021 struct npc_install_flow_req *tmp_req; 1022 int err; 1023 1024 mutex_lock(&nic->mbox.lock); 1025 tmp_req = otx2_mbox_alloc_msg_npc_install_flow(&nic->mbox); 1026 if (!tmp_req) { 1027 mutex_unlock(&nic->mbox.lock); 1028 return -ENOMEM; 1029 } 1030 1031 memcpy(tmp_req, req, sizeof(struct npc_install_flow_req)); 1032 /* Send message to AF */ 1033 err = otx2_sync_mbox_msg(&nic->mbox); 1034 if (err) { 1035 netdev_err(nic->netdev, "Failed to install MCAM flow entry %d\n", 1036 req->entry); 1037 mutex_unlock(&nic->mbox.lock); 1038 return -EFAULT; 1039 } 1040 1041 mutex_unlock(&nic->mbox.lock); 1042 return 0; 1043 } 1044 1045 int otx2_del_mcam_flow_entry(struct otx2_nic *nic, u16 entry, u16 *cntr_val) 1046 { 1047 struct npc_delete_flow_rsp *rsp; 1048 struct npc_delete_flow_req *req; 1049 int err; 1050 1051 mutex_lock(&nic->mbox.lock); 1052 req = otx2_mbox_alloc_msg_npc_delete_flow(&nic->mbox); 1053 if (!req) { 1054 mutex_unlock(&nic->mbox.lock); 1055 return -ENOMEM; 1056 } 1057 1058 req->entry = entry; 1059 1060 /* Send message to AF */ 1061 err = otx2_sync_mbox_msg(&nic->mbox); 1062 if (err) { 1063 netdev_err(nic->netdev, "Failed to delete MCAM flow entry %d\n", 1064 entry); 1065 mutex_unlock(&nic->mbox.lock); 1066 return -EFAULT; 1067 } 1068 1069 if (cntr_val) { 1070 rsp = (struct npc_delete_flow_rsp *)otx2_mbox_get_rsp(&nic->mbox.mbox, 1071 0, &req->hdr); 1072 if (IS_ERR(rsp)) { 1073 netdev_err(nic->netdev, "Failed to get MCAM delete response for entry %d\n", 1074 entry); 1075 mutex_unlock(&nic->mbox.lock); 1076 return -EFAULT; 1077 } 1078 1079 *cntr_val = rsp->cntr_val; 1080 } 1081 1082 mutex_unlock(&nic->mbox.lock); 1083 return 0; 1084 } 1085 1086 static int otx2_tc_update_mcam_table_del_req(struct otx2_nic *nic, 1087 struct otx2_flow_config *flow_cfg, 1088 struct otx2_tc_flow *node) 1089 { 1090 struct list_head *pos, *n; 1091 struct otx2_tc_flow *tmp; 1092 int i = 0, index = 0; 1093 u16 cntr_val = 0; 1094 1095 if (is_cn20k(nic->pdev)) { 1096 cn20k_tc_update_mcam_table_del_req(nic, flow_cfg, node); 1097 return 0; 1098 } 1099 1100 /* Find and delete the entry from the list and re-install 1101 * all the entries from beginning to the index of the 1102 * deleted entry to higher mcam indexes. 1103 */ 1104 list_for_each_safe(pos, n, &flow_cfg->flow_list_tc) { 1105 tmp = list_entry(pos, struct otx2_tc_flow, list); 1106 if (node == tmp) { 1107 list_del(&tmp->list); 1108 break; 1109 } 1110 1111 otx2_del_mcam_flow_entry(nic, tmp->entry, &cntr_val); 1112 tmp->entry++; 1113 tmp->req.entry = tmp->entry; 1114 tmp->req.cntr_val = cntr_val; 1115 index++; 1116 } 1117 1118 list_for_each_safe(pos, n, &flow_cfg->flow_list_tc) { 1119 if (i == index) 1120 break; 1121 1122 tmp = list_entry(pos, struct otx2_tc_flow, list); 1123 otx2_add_mcam_flow_entry(nic, &tmp->req); 1124 i++; 1125 } 1126 1127 return 0; 1128 } 1129 1130 static int otx2_tc_update_mcam_table_add_req(struct otx2_nic *nic, 1131 struct otx2_flow_config *flow_cfg, 1132 struct otx2_tc_flow *node) 1133 { 1134 int mcam_idx = flow_cfg->max_flows - flow_cfg->nr_flows - 1; 1135 struct otx2_tc_flow *tmp; 1136 int list_idx, i; 1137 u16 cntr_val = 0; 1138 1139 if (is_cn20k(nic->pdev)) 1140 return cn20k_tc_update_mcam_table_add_req(nic, flow_cfg, node); 1141 1142 /* Find the index of the entry(list_idx) whose priority 1143 * is greater than the new entry and re-install all 1144 * the entries from beginning to list_idx to higher 1145 * mcam indexes. 1146 */ 1147 list_idx = otx2_tc_add_to_flow_list(flow_cfg, node); 1148 for (i = 0; i < list_idx; i++) { 1149 tmp = otx2_tc_get_entry_by_index(flow_cfg, i); 1150 if (!tmp) 1151 return -ENOMEM; 1152 1153 otx2_del_mcam_flow_entry(nic, tmp->entry, &cntr_val); 1154 tmp->entry = flow_cfg->flow_ent[mcam_idx]; 1155 tmp->req.entry = tmp->entry; 1156 tmp->req.cntr_val = cntr_val; 1157 otx2_add_mcam_flow_entry(nic, &tmp->req); 1158 mcam_idx++; 1159 } 1160 1161 return flow_cfg->flow_ent[mcam_idx]; 1162 } 1163 1164 static int otx2_tc_update_mcam_table(struct otx2_nic *nic, 1165 struct otx2_flow_config *flow_cfg, 1166 struct otx2_tc_flow *node, 1167 bool add_req) 1168 { 1169 if (add_req) 1170 return otx2_tc_update_mcam_table_add_req(nic, flow_cfg, node); 1171 1172 return otx2_tc_update_mcam_table_del_req(nic, flow_cfg, node); 1173 } 1174 1175 static int otx2_tc_del_flow(struct otx2_nic *nic, 1176 struct flow_cls_offload *tc_flow_cmd) 1177 { 1178 struct otx2_flow_config *flow_cfg = nic->flow_cfg; 1179 struct nix_mcast_grp_destroy_req *grp_destroy_req; 1180 struct otx2_tc_flow *flow_node; 1181 int err; 1182 1183 flow_node = otx2_tc_get_entry_by_cookie(flow_cfg, tc_flow_cmd->cookie); 1184 if (!flow_node) { 1185 netdev_err(nic->netdev, "tc flow not found for cookie 0x%lx\n", 1186 tc_flow_cmd->cookie); 1187 return -EINVAL; 1188 } 1189 1190 /* Disable TC MARK flag if they are no rules with skbedit mark action */ 1191 if (flow_node->req.match_id) 1192 if (!refcount_dec_and_test(&flow_cfg->mark_flows)) 1193 nic->flags &= ~OTX2_FLAG_TC_MARK_ENABLED; 1194 1195 if (flow_node->is_act_police) { 1196 __clear_bit(flow_node->rq, &nic->rq_bmap); 1197 1198 if (nic->flags & OTX2_FLAG_INTF_DOWN) 1199 goto free_mcam_flow; 1200 1201 mutex_lock(&nic->mbox.lock); 1202 1203 err = cn10k_map_unmap_rq_policer(nic, flow_node->rq, 1204 flow_node->leaf_profile, false); 1205 if (err) 1206 netdev_err(nic->netdev, 1207 "Unmapping RQ %d & profile %d failed\n", 1208 flow_node->rq, flow_node->leaf_profile); 1209 1210 err = cn10k_free_leaf_profile(nic, flow_node->leaf_profile); 1211 if (err) 1212 netdev_err(nic->netdev, 1213 "Unable to free leaf bandwidth profile(%d)\n", 1214 flow_node->leaf_profile); 1215 1216 mutex_unlock(&nic->mbox.lock); 1217 } 1218 /* Remove the multicast/mirror related nodes */ 1219 if (flow_node->mcast_grp_idx != MCAST_INVALID_GRP) { 1220 mutex_lock(&nic->mbox.lock); 1221 grp_destroy_req = otx2_mbox_alloc_msg_nix_mcast_grp_destroy(&nic->mbox); 1222 grp_destroy_req->mcast_grp_idx = flow_node->mcast_grp_idx; 1223 otx2_sync_mbox_msg(&nic->mbox); 1224 mutex_unlock(&nic->mbox.lock); 1225 } 1226 1227 free_mcam_flow: 1228 otx2_del_mcam_flow_entry(nic, flow_node->entry, NULL); 1229 otx2_tc_update_mcam_table(nic, flow_cfg, flow_node, false); 1230 kfree_rcu(flow_node, rcu); 1231 flow_cfg->nr_flows--; 1232 return 0; 1233 } 1234 1235 static int otx2_tc_add_flow(struct otx2_nic *nic, 1236 struct flow_cls_offload *tc_flow_cmd) 1237 { 1238 struct netlink_ext_ack *extack = tc_flow_cmd->common.extack; 1239 struct otx2_flow_config *flow_cfg = nic->flow_cfg; 1240 struct otx2_tc_flow *new_node, *old_node; 1241 struct npc_install_flow_req *req, dummy; 1242 int rc, err, entry; 1243 1244 if (!(nic->flags & OTX2_FLAG_TC_FLOWER_SUPPORT)) 1245 return -ENOMEM; 1246 1247 if (nic->flags & OTX2_FLAG_INTF_DOWN) { 1248 NL_SET_ERR_MSG_MOD(extack, "Interface not initialized"); 1249 return -EINVAL; 1250 } 1251 1252 if (!is_cn20k(nic->pdev) && flow_cfg->nr_flows == flow_cfg->max_flows) { 1253 NL_SET_ERR_MSG_MOD(extack, 1254 "Free MCAM entry not available to add the flow"); 1255 return -ENOMEM; 1256 } 1257 1258 /* allocate memory for the new flow and it's node */ 1259 new_node = kzalloc_obj(*new_node); 1260 if (!new_node) 1261 return -ENOMEM; 1262 spin_lock_init(&new_node->lock); 1263 new_node->cookie = tc_flow_cmd->cookie; 1264 new_node->prio = tc_flow_cmd->common.prio; 1265 new_node->mcast_grp_idx = MCAST_INVALID_GRP; 1266 1267 memset(&dummy, 0, sizeof(struct npc_install_flow_req)); 1268 1269 rc = otx2_tc_prepare_flow(nic, new_node, tc_flow_cmd, &dummy); 1270 if (rc) { 1271 kfree_rcu(new_node, rcu); 1272 return rc; 1273 } 1274 1275 /* If a flow exists with the same cookie, delete it */ 1276 old_node = otx2_tc_get_entry_by_cookie(flow_cfg, tc_flow_cmd->cookie); 1277 if (old_node) 1278 otx2_tc_del_flow(nic, tc_flow_cmd); 1279 1280 if (is_cn20k(nic->pdev)) { 1281 rc = cn20k_tc_alloc_entry(nic, tc_flow_cmd, new_node, &dummy); 1282 if (rc) { 1283 NL_SET_ERR_MSG_MOD(extack, 1284 "MCAM rule allocation failed"); 1285 kfree_rcu(new_node, rcu); 1286 return rc; 1287 } 1288 } 1289 1290 entry = otx2_tc_update_mcam_table(nic, flow_cfg, new_node, true); 1291 if (entry < 0) { 1292 NL_SET_ERR_MSG_MOD(extack, "Adding rule failed"); 1293 rc = entry; 1294 goto free_leaf; 1295 } 1296 1297 mutex_lock(&nic->mbox.lock); 1298 req = otx2_mbox_alloc_msg_npc_install_flow(&nic->mbox); 1299 if (!req) { 1300 mutex_unlock(&nic->mbox.lock); 1301 rc = -ENOMEM; 1302 goto free_leaf; 1303 } 1304 1305 memcpy(&dummy.hdr, &req->hdr, sizeof(struct mbox_msghdr)); 1306 memcpy(req, &dummy, sizeof(struct npc_install_flow_req)); 1307 req->channel = nic->hw.rx_chan_base; 1308 req->entry = (u16)entry; 1309 req->intf = NIX_INTF_RX; 1310 req->vf = nic->pcifunc; 1311 req->set_cntr = 1; 1312 new_node->entry = req->entry; 1313 1314 /* Send message to AF */ 1315 rc = otx2_sync_mbox_msg(&nic->mbox); 1316 if (rc) { 1317 NL_SET_ERR_MSG_MOD(extack, "Failed to install MCAM flow entry"); 1318 mutex_unlock(&nic->mbox.lock); 1319 goto free_leaf; 1320 } 1321 1322 mutex_unlock(&nic->mbox.lock); 1323 memcpy(&new_node->req, req, sizeof(struct npc_install_flow_req)); 1324 1325 flow_cfg->nr_flows++; 1326 return 0; 1327 1328 free_leaf: 1329 if (is_cn20k(nic->pdev)) 1330 cn20k_tc_free_mcam_entry(nic, new_node->entry); 1331 otx2_tc_del_from_flow_list(flow_cfg, new_node); 1332 if (new_node->is_act_police) { 1333 mutex_lock(&nic->mbox.lock); 1334 1335 err = cn10k_map_unmap_rq_policer(nic, new_node->rq, 1336 new_node->leaf_profile, false); 1337 if (err) 1338 netdev_err(nic->netdev, 1339 "Unmapping RQ %d & profile %d failed\n", 1340 new_node->rq, new_node->leaf_profile); 1341 err = cn10k_free_leaf_profile(nic, new_node->leaf_profile); 1342 if (err) 1343 netdev_err(nic->netdev, 1344 "Unable to free leaf bandwidth profile(%d)\n", 1345 new_node->leaf_profile); 1346 1347 __clear_bit(new_node->rq, &nic->rq_bmap); 1348 1349 mutex_unlock(&nic->mbox.lock); 1350 } 1351 kfree_rcu(new_node, rcu); 1352 1353 return rc; 1354 } 1355 1356 static int otx2_tc_get_flow_stats(struct otx2_nic *nic, 1357 struct flow_cls_offload *tc_flow_cmd) 1358 { 1359 struct npc_mcam_get_stats_req *req; 1360 struct npc_mcam_get_stats_rsp *rsp; 1361 struct otx2_tc_flow_stats *stats; 1362 struct otx2_tc_flow *flow_node; 1363 int err; 1364 1365 flow_node = otx2_tc_get_entry_by_cookie(nic->flow_cfg, tc_flow_cmd->cookie); 1366 if (!flow_node) { 1367 netdev_info(nic->netdev, "tc flow not found for cookie %lx", 1368 tc_flow_cmd->cookie); 1369 return -EINVAL; 1370 } 1371 1372 mutex_lock(&nic->mbox.lock); 1373 1374 req = otx2_mbox_alloc_msg_npc_mcam_entry_stats(&nic->mbox); 1375 if (!req) { 1376 mutex_unlock(&nic->mbox.lock); 1377 return -ENOMEM; 1378 } 1379 1380 req->entry = flow_node->entry; 1381 1382 err = otx2_sync_mbox_msg(&nic->mbox); 1383 if (err) { 1384 netdev_err(nic->netdev, "Failed to get stats for MCAM flow entry %d\n", 1385 req->entry); 1386 mutex_unlock(&nic->mbox.lock); 1387 return -EFAULT; 1388 } 1389 1390 rsp = (struct npc_mcam_get_stats_rsp *)otx2_mbox_get_rsp 1391 (&nic->mbox.mbox, 0, &req->hdr); 1392 if (IS_ERR(rsp)) { 1393 mutex_unlock(&nic->mbox.lock); 1394 return PTR_ERR(rsp); 1395 } 1396 1397 mutex_unlock(&nic->mbox.lock); 1398 1399 if (!rsp->stat_ena) 1400 return -EINVAL; 1401 1402 stats = &flow_node->stats; 1403 1404 spin_lock(&flow_node->lock); 1405 flow_stats_update(&tc_flow_cmd->stats, 0x0, rsp->stat - stats->pkts, 0x0, 0x0, 1406 FLOW_ACTION_HW_STATS_IMMEDIATE); 1407 stats->pkts = rsp->stat; 1408 spin_unlock(&flow_node->lock); 1409 1410 return 0; 1411 } 1412 1413 int otx2_setup_tc_cls_flower(struct otx2_nic *nic, 1414 struct flow_cls_offload *cls_flower) 1415 { 1416 switch (cls_flower->command) { 1417 case FLOW_CLS_REPLACE: 1418 return otx2_tc_add_flow(nic, cls_flower); 1419 case FLOW_CLS_DESTROY: 1420 return otx2_tc_del_flow(nic, cls_flower); 1421 case FLOW_CLS_STATS: 1422 return otx2_tc_get_flow_stats(nic, cls_flower); 1423 default: 1424 return -EOPNOTSUPP; 1425 } 1426 } 1427 EXPORT_SYMBOL(otx2_setup_tc_cls_flower); 1428 1429 static int otx2_tc_ingress_matchall_install(struct otx2_nic *nic, 1430 struct tc_cls_matchall_offload *cls) 1431 { 1432 struct netlink_ext_ack *extack = cls->common.extack; 1433 struct flow_action *actions = &cls->rule->action; 1434 struct flow_action_entry *entry; 1435 u64 rate; 1436 int err; 1437 1438 err = otx2_tc_validate_flow(nic, actions, extack); 1439 if (err) 1440 return err; 1441 1442 if (nic->flags & OTX2_FLAG_TC_MATCHALL_INGRESS_ENABLED) { 1443 NL_SET_ERR_MSG_MOD(extack, 1444 "Only one ingress MATCHALL ratelimitter can be offloaded"); 1445 return -ENOMEM; 1446 } 1447 1448 entry = &cls->rule->action.entries[0]; 1449 switch (entry->id) { 1450 case FLOW_ACTION_POLICE: 1451 /* Ingress ratelimiting is not supported on OcteonTx2 */ 1452 if (is_dev_otx2(nic->pdev)) { 1453 NL_SET_ERR_MSG_MOD(extack, 1454 "Ingress policing not supported on this platform"); 1455 return -EOPNOTSUPP; 1456 } 1457 1458 err = cn10k_alloc_matchall_ipolicer(nic); 1459 if (err) 1460 return err; 1461 1462 /* Convert to bits per second */ 1463 rate = entry->police.rate_bytes_ps * 8; 1464 err = cn10k_set_matchall_ipolicer_rate(nic, entry->police.burst, rate); 1465 if (err) 1466 return err; 1467 nic->flags |= OTX2_FLAG_TC_MATCHALL_INGRESS_ENABLED; 1468 break; 1469 default: 1470 NL_SET_ERR_MSG_MOD(extack, 1471 "Only police action supported with Ingress MATCHALL offload"); 1472 return -EOPNOTSUPP; 1473 } 1474 1475 return 0; 1476 } 1477 1478 static int otx2_tc_ingress_matchall_delete(struct otx2_nic *nic, 1479 struct tc_cls_matchall_offload *cls) 1480 { 1481 struct netlink_ext_ack *extack = cls->common.extack; 1482 int err; 1483 1484 if (nic->flags & OTX2_FLAG_INTF_DOWN) { 1485 NL_SET_ERR_MSG_MOD(extack, "Interface not initialized"); 1486 return -EINVAL; 1487 } 1488 1489 err = cn10k_free_matchall_ipolicer(nic); 1490 nic->flags &= ~OTX2_FLAG_TC_MATCHALL_INGRESS_ENABLED; 1491 return err; 1492 } 1493 1494 static int otx2_setup_tc_ingress_matchall(struct otx2_nic *nic, 1495 struct tc_cls_matchall_offload *cls_matchall) 1496 { 1497 switch (cls_matchall->command) { 1498 case TC_CLSMATCHALL_REPLACE: 1499 return otx2_tc_ingress_matchall_install(nic, cls_matchall); 1500 case TC_CLSMATCHALL_DESTROY: 1501 return otx2_tc_ingress_matchall_delete(nic, cls_matchall); 1502 case TC_CLSMATCHALL_STATS: 1503 default: 1504 break; 1505 } 1506 1507 return -EOPNOTSUPP; 1508 } 1509 1510 static int otx2_setup_tc_block_ingress_cb(enum tc_setup_type type, 1511 void *type_data, void *cb_priv) 1512 { 1513 struct otx2_nic *nic = cb_priv; 1514 bool ntuple; 1515 1516 if (!tc_cls_can_offload_and_chain0(nic->netdev, type_data)) 1517 return -EOPNOTSUPP; 1518 1519 ntuple = nic->netdev->features & NETIF_F_NTUPLE; 1520 switch (type) { 1521 case TC_SETUP_CLSFLOWER: 1522 if (ntuple) { 1523 netdev_warn(nic->netdev, 1524 "Can't install TC flower offload rule when NTUPLE is active"); 1525 return -EOPNOTSUPP; 1526 } 1527 1528 return otx2_setup_tc_cls_flower(nic, type_data); 1529 case TC_SETUP_CLSMATCHALL: 1530 return otx2_setup_tc_ingress_matchall(nic, type_data); 1531 default: 1532 break; 1533 } 1534 1535 return -EOPNOTSUPP; 1536 } 1537 1538 static int otx2_setup_tc_egress_matchall(struct otx2_nic *nic, 1539 struct tc_cls_matchall_offload *cls_matchall) 1540 { 1541 switch (cls_matchall->command) { 1542 case TC_CLSMATCHALL_REPLACE: 1543 return otx2_tc_egress_matchall_install(nic, cls_matchall); 1544 case TC_CLSMATCHALL_DESTROY: 1545 return otx2_tc_egress_matchall_delete(nic, cls_matchall); 1546 case TC_CLSMATCHALL_STATS: 1547 default: 1548 break; 1549 } 1550 1551 return -EOPNOTSUPP; 1552 } 1553 1554 static int otx2_setup_tc_block_egress_cb(enum tc_setup_type type, 1555 void *type_data, void *cb_priv) 1556 { 1557 struct otx2_nic *nic = cb_priv; 1558 1559 if (!tc_cls_can_offload_and_chain0(nic->netdev, type_data)) 1560 return -EOPNOTSUPP; 1561 1562 switch (type) { 1563 case TC_SETUP_CLSMATCHALL: 1564 return otx2_setup_tc_egress_matchall(nic, type_data); 1565 default: 1566 break; 1567 } 1568 1569 return -EOPNOTSUPP; 1570 } 1571 1572 static LIST_HEAD(otx2_block_cb_list); 1573 1574 static int otx2_setup_tc_block(struct net_device *netdev, 1575 struct flow_block_offload *f) 1576 { 1577 struct otx2_nic *nic = netdev_priv(netdev); 1578 flow_setup_cb_t *cb; 1579 bool ingress; 1580 1581 if (f->block_shared) 1582 return -EOPNOTSUPP; 1583 1584 if (f->binder_type == FLOW_BLOCK_BINDER_TYPE_CLSACT_INGRESS) { 1585 cb = otx2_setup_tc_block_ingress_cb; 1586 ingress = true; 1587 } else if (f->binder_type == FLOW_BLOCK_BINDER_TYPE_CLSACT_EGRESS) { 1588 cb = otx2_setup_tc_block_egress_cb; 1589 ingress = false; 1590 } else { 1591 return -EOPNOTSUPP; 1592 } 1593 1594 return flow_block_cb_setup_simple(f, &otx2_block_cb_list, cb, 1595 nic, nic, ingress); 1596 } 1597 1598 int otx2_setup_tc(struct net_device *netdev, enum tc_setup_type type, 1599 void *type_data) 1600 { 1601 switch (type) { 1602 case TC_SETUP_BLOCK: 1603 return otx2_setup_tc_block(netdev, type_data); 1604 case TC_SETUP_QDISC_HTB: 1605 return otx2_setup_tc_htb(netdev, type_data); 1606 default: 1607 return -EOPNOTSUPP; 1608 } 1609 } 1610 EXPORT_SYMBOL(otx2_setup_tc); 1611 1612 int otx2_init_tc(struct otx2_nic *nic) 1613 { 1614 /* Exclude receive queue 0 being used for police action */ 1615 set_bit(0, &nic->rq_bmap); 1616 1617 if (!nic->flow_cfg) { 1618 netdev_err(nic->netdev, 1619 "Can't init TC, nic->flow_cfg is not setup\n"); 1620 return -EINVAL; 1621 } 1622 1623 return 0; 1624 } 1625 EXPORT_SYMBOL(otx2_init_tc); 1626 1627 void otx2_shutdown_tc(struct otx2_nic *nic) 1628 { 1629 otx2_destroy_tc_flow_list(nic); 1630 } 1631 EXPORT_SYMBOL(otx2_shutdown_tc); 1632 1633 static void otx2_tc_config_ingress_rule(struct otx2_nic *nic, 1634 struct otx2_tc_flow *node) 1635 { 1636 struct npc_install_flow_req *req; 1637 1638 if (otx2_tc_act_set_hw_police(nic, node)) 1639 return; 1640 1641 mutex_lock(&nic->mbox.lock); 1642 1643 req = otx2_mbox_alloc_msg_npc_install_flow(&nic->mbox); 1644 if (!req) 1645 goto err; 1646 1647 memcpy(req, &node->req, sizeof(struct npc_install_flow_req)); 1648 1649 if (otx2_sync_mbox_msg(&nic->mbox)) 1650 netdev_err(nic->netdev, 1651 "Failed to install MCAM flow entry for ingress rule"); 1652 err: 1653 mutex_unlock(&nic->mbox.lock); 1654 } 1655 1656 void otx2_tc_apply_ingress_police_rules(struct otx2_nic *nic) 1657 { 1658 struct otx2_flow_config *flow_cfg = nic->flow_cfg; 1659 struct otx2_tc_flow *node; 1660 1661 /* If any ingress policer rules exist for the interface then 1662 * apply those rules. Ingress policer rules depend on bandwidth 1663 * profiles linked to the receive queues. Since no receive queues 1664 * exist when interface is down, ingress policer rules are stored 1665 * and configured in hardware after all receive queues are allocated 1666 * in otx2_open. 1667 */ 1668 list_for_each_entry(node, &flow_cfg->flow_list_tc, list) { 1669 if (node->is_act_police) 1670 otx2_tc_config_ingress_rule(nic, node); 1671 } 1672 } 1673 EXPORT_SYMBOL(otx2_tc_apply_ingress_police_rules); 1674