1 // SPDX-License-Identifier: GPL-2.0
2 /* Marvell RVU Ethernet driver
3 *
4 * Copyright (C) 2021 Marvell.
5 *
6 */
7
8 #include <linux/netdevice.h>
9 #include <linux/etherdevice.h>
10 #include <linux/inetdevice.h>
11 #include <linux/rhashtable.h>
12 #include <linux/bitfield.h>
13 #include <net/flow_dissector.h>
14 #include <net/pkt_cls.h>
15 #include <net/tc_act/tc_gact.h>
16 #include <net/tc_act/tc_mirred.h>
17 #include <net/tc_act/tc_vlan.h>
18 #include <net/ipv6.h>
19
20 #include "cn10k.h"
21 #include "otx2_common.h"
22 #include "qos.h"
23
24 #define CN10K_MAX_BURST_MANTISSA 0x7FFFULL
25 #define CN10K_MAX_BURST_SIZE 8453888ULL
26
27 #define CN10K_TLX_BURST_MANTISSA GENMASK_ULL(43, 29)
28 #define CN10K_TLX_BURST_EXPONENT GENMASK_ULL(47, 44)
29
30 #define OTX2_UNSUPP_LSE_DEPTH GENMASK(6, 4)
31
32 #define MCAST_INVALID_GRP (-1U)
33 #define RATE_MANTISSA_BITS 8
34
otx2_get_egress_burst_cfg(struct otx2_nic * nic,u32 burst,u32 * burst_exp,u32 * burst_mantissa)35 static void otx2_get_egress_burst_cfg(struct otx2_nic *nic, u32 burst,
36 u32 *burst_exp, u32 *burst_mantissa)
37 {
38 int max_burst, max_mantissa;
39 unsigned int tmp;
40
41 if (is_dev_otx2(nic->pdev)) {
42 max_burst = MAX_BURST_SIZE;
43 max_mantissa = MAX_BURST_MANTISSA;
44 } else {
45 max_burst = CN10K_MAX_BURST_SIZE;
46 max_mantissa = CN10K_MAX_BURST_MANTISSA;
47 }
48
49 /* Burst is calculated as
50 * ((256 + BURST_MANTISSA) << (1 + BURST_EXPONENT)) / 256
51 * Max supported burst size is 130,816 bytes.
52 */
53 burst = min_t(u32, burst, max_burst);
54 if (burst) {
55 *burst_exp = ilog2(burst) ? ilog2(burst) - 1 : 0;
56 tmp = burst - rounddown_pow_of_two(burst);
57 if (burst <= max_mantissa) {
58 *burst_mantissa = tmp * 2;
59 } else {
60 WARN_ON(*burst_exp < 7);
61 *burst_mantissa = tmp / (1ULL << (*burst_exp - 7));
62 }
63 } else {
64 *burst_exp = MAX_BURST_EXPONENT;
65 *burst_mantissa = max_mantissa;
66 }
67 }
68
otx2_get_egress_rate_cfg(u64 maxrate,u32 * exp,u32 * mantissa,u32 * div_exp)69 static void otx2_get_egress_rate_cfg(u64 maxrate, u32 *exp,
70 u32 *mantissa, u32 *div_exp)
71 {
72 /* Rate calculation by hardware
73 *
74 * PIR_ADD = ((256 + mantissa) << exp) / 256
75 * rate = (2 * PIR_ADD) / ( 1 << div_exp)
76 * The resultant rate is in Mbps.
77 *
78 * Use div_exp = 0 and compute exp/mantissa for maxrate / 2; the
79 * leading factor of two yields the full rate. Rates below 2 Mbps
80 * are floored to the smallest step (exp = 0, mantissa = 0).
81 */
82
83 *div_exp = 0;
84 if (maxrate) {
85 maxrate = maxrate / 2;
86 if (!maxrate) {
87 /* Rates below 2 Mbps map to the smallest step */
88 *exp = 0;
89 *mantissa = 0;
90 } else {
91 *exp = ilog2(maxrate);
92 /* Clear MSB and derive fractional bits */
93 maxrate &= ~BIT(*exp);
94 *mantissa = (maxrate << RATE_MANTISSA_BITS) >> *exp;
95 }
96 } else {
97 /* Instead of disabling rate limiting, set all values to max */
98 *exp = MAX_RATE_EXPONENT;
99 *mantissa = MAX_RATE_MANTISSA;
100 }
101 }
102
otx2_get_txschq_rate_regval(struct otx2_nic * nic,u64 maxrate,u32 burst)103 u64 otx2_get_txschq_rate_regval(struct otx2_nic *nic,
104 u64 maxrate, u32 burst)
105 {
106 u32 burst_exp, burst_mantissa;
107 u32 exp, mantissa, div_exp;
108 u64 regval = 0;
109
110 /* Get exponent and mantissa values from the desired rate */
111 otx2_get_egress_burst_cfg(nic, burst, &burst_exp, &burst_mantissa);
112 otx2_get_egress_rate_cfg(maxrate, &exp, &mantissa, &div_exp);
113
114 if (is_dev_otx2(nic->pdev)) {
115 regval = FIELD_PREP(TLX_BURST_EXPONENT, (u64)burst_exp) |
116 FIELD_PREP(TLX_BURST_MANTISSA, (u64)burst_mantissa) |
117 FIELD_PREP(TLX_RATE_DIVIDER_EXPONENT, div_exp) |
118 FIELD_PREP(TLX_RATE_EXPONENT, exp) |
119 FIELD_PREP(TLX_RATE_MANTISSA, mantissa) | BIT_ULL(0);
120 } else {
121 regval = FIELD_PREP(CN10K_TLX_BURST_EXPONENT, (u64)burst_exp) |
122 FIELD_PREP(CN10K_TLX_BURST_MANTISSA, (u64)burst_mantissa) |
123 FIELD_PREP(TLX_RATE_DIVIDER_EXPONENT, div_exp) |
124 FIELD_PREP(TLX_RATE_EXPONENT, exp) |
125 FIELD_PREP(TLX_RATE_MANTISSA, mantissa) | BIT_ULL(0);
126 }
127
128 return regval;
129 }
130
otx2_set_matchall_egress_rate(struct otx2_nic * nic,u32 burst,u64 maxrate)131 static int otx2_set_matchall_egress_rate(struct otx2_nic *nic,
132 u32 burst, u64 maxrate)
133 {
134 struct otx2_hw *hw = &nic->hw;
135 struct nix_txschq_config *req;
136 int txschq, err;
137
138 /* All SQs share the same TL4, so pick the first scheduler */
139 txschq = hw->txschq_list[NIX_TXSCH_LVL_TL4][0];
140
141 mutex_lock(&nic->mbox.lock);
142 req = otx2_mbox_alloc_msg_nix_txschq_cfg(&nic->mbox);
143 if (!req) {
144 mutex_unlock(&nic->mbox.lock);
145 return -ENOMEM;
146 }
147
148 req->lvl = NIX_TXSCH_LVL_TL4;
149 req->num_regs = 1;
150 req->reg[0] = NIX_AF_TL4X_PIR(txschq);
151 req->regval[0] = otx2_get_txschq_rate_regval(nic, maxrate, burst);
152
153 err = otx2_sync_mbox_msg(&nic->mbox);
154 mutex_unlock(&nic->mbox.lock);
155 return err;
156 }
157
otx2_tc_validate_flow(struct otx2_nic * nic,struct flow_action * actions,struct netlink_ext_ack * extack)158 static int otx2_tc_validate_flow(struct otx2_nic *nic,
159 struct flow_action *actions,
160 struct netlink_ext_ack *extack)
161 {
162 if (nic->flags & OTX2_FLAG_INTF_DOWN) {
163 NL_SET_ERR_MSG_MOD(extack, "Interface not initialized");
164 return -EINVAL;
165 }
166
167 if (!flow_action_has_entries(actions)) {
168 NL_SET_ERR_MSG_MOD(extack, "MATCHALL offload called with no action");
169 return -EINVAL;
170 }
171
172 if (!flow_offload_has_one_action(actions)) {
173 NL_SET_ERR_MSG_MOD(extack,
174 "Egress MATCHALL offload supports only 1 policing action");
175 return -EINVAL;
176 }
177 return 0;
178 }
179
otx2_policer_validate(const struct flow_action * action,const struct flow_action_entry * act,struct netlink_ext_ack * extack)180 static int otx2_policer_validate(const struct flow_action *action,
181 const struct flow_action_entry *act,
182 struct netlink_ext_ack *extack)
183 {
184 if (act->police.exceed.act_id != FLOW_ACTION_DROP) {
185 NL_SET_ERR_MSG_MOD(extack,
186 "Offload not supported when exceed action is not drop");
187 return -EOPNOTSUPP;
188 }
189
190 if (act->police.notexceed.act_id != FLOW_ACTION_PIPE &&
191 act->police.notexceed.act_id != FLOW_ACTION_ACCEPT) {
192 NL_SET_ERR_MSG_MOD(extack,
193 "Offload not supported when conform action is not pipe or ok");
194 return -EOPNOTSUPP;
195 }
196
197 if (act->police.notexceed.act_id == FLOW_ACTION_ACCEPT &&
198 !flow_action_is_last_entry(action, act)) {
199 NL_SET_ERR_MSG_MOD(extack,
200 "Offload not supported when conform action is ok, but action is not last");
201 return -EOPNOTSUPP;
202 }
203
204 if (act->police.peakrate_bytes_ps ||
205 act->police.avrate || act->police.overhead) {
206 NL_SET_ERR_MSG_MOD(extack,
207 "Offload not supported when peakrate/avrate/overhead is configured");
208 return -EOPNOTSUPP;
209 }
210
211 return 0;
212 }
213
otx2_tc_egress_matchall_install(struct otx2_nic * nic,struct tc_cls_matchall_offload * cls)214 static int otx2_tc_egress_matchall_install(struct otx2_nic *nic,
215 struct tc_cls_matchall_offload *cls)
216 {
217 struct netlink_ext_ack *extack = cls->common.extack;
218 struct flow_action *actions = &cls->rule->action;
219 struct flow_action_entry *entry;
220 int err;
221
222 err = otx2_tc_validate_flow(nic, actions, extack);
223 if (err)
224 return err;
225
226 if (nic->flags & OTX2_FLAG_TC_MATCHALL_EGRESS_ENABLED) {
227 NL_SET_ERR_MSG_MOD(extack,
228 "Only one Egress MATCHALL ratelimiter can be offloaded");
229 return -ENOMEM;
230 }
231
232 entry = &cls->rule->action.entries[0];
233 switch (entry->id) {
234 case FLOW_ACTION_POLICE:
235 err = otx2_policer_validate(&cls->rule->action, entry, extack);
236 if (err)
237 return err;
238
239 if (entry->police.rate_pkt_ps) {
240 NL_SET_ERR_MSG_MOD(extack, "QoS offload not support packets per second");
241 return -EOPNOTSUPP;
242 }
243 err = otx2_set_matchall_egress_rate(nic, entry->police.burst,
244 otx2_convert_rate(entry->police.rate_bytes_ps));
245 if (err)
246 return err;
247 nic->flags |= OTX2_FLAG_TC_MATCHALL_EGRESS_ENABLED;
248 break;
249 default:
250 NL_SET_ERR_MSG_MOD(extack,
251 "Only police action is supported with Egress MATCHALL offload");
252 return -EOPNOTSUPP;
253 }
254
255 return 0;
256 }
257
otx2_tc_egress_matchall_delete(struct otx2_nic * nic,struct tc_cls_matchall_offload * cls)258 static int otx2_tc_egress_matchall_delete(struct otx2_nic *nic,
259 struct tc_cls_matchall_offload *cls)
260 {
261 struct netlink_ext_ack *extack = cls->common.extack;
262 int err;
263
264 if (nic->flags & OTX2_FLAG_INTF_DOWN) {
265 NL_SET_ERR_MSG_MOD(extack, "Interface not initialized");
266 return -EINVAL;
267 }
268
269 err = otx2_set_matchall_egress_rate(nic, 0, 0);
270 nic->flags &= ~OTX2_FLAG_TC_MATCHALL_EGRESS_ENABLED;
271 return err;
272 }
273
otx2_tc_act_set_hw_police(struct otx2_nic * nic,struct otx2_tc_flow * node)274 static int otx2_tc_act_set_hw_police(struct otx2_nic *nic,
275 struct otx2_tc_flow *node)
276 {
277 int rc;
278
279 mutex_lock(&nic->mbox.lock);
280
281 rc = cn10k_alloc_leaf_profile(nic, &node->leaf_profile);
282 if (rc) {
283 mutex_unlock(&nic->mbox.lock);
284 return rc;
285 }
286
287 rc = cn10k_set_ipolicer_rate(nic, node->leaf_profile,
288 node->burst, node->rate, node->is_pps);
289 if (rc)
290 goto free_leaf;
291
292 rc = cn10k_map_unmap_rq_policer(nic, node->rq, node->leaf_profile, true);
293 if (rc)
294 goto free_leaf;
295
296 mutex_unlock(&nic->mbox.lock);
297
298 return 0;
299
300 free_leaf:
301 if (cn10k_free_leaf_profile(nic, node->leaf_profile))
302 netdev_err(nic->netdev,
303 "Unable to free leaf bandwidth profile(%d)\n",
304 node->leaf_profile);
305 mutex_unlock(&nic->mbox.lock);
306 return rc;
307 }
308
otx2_tc_act_set_police(struct otx2_nic * nic,struct otx2_tc_flow * node,struct flow_cls_offload * f,u64 rate,u32 burst,u32 mark,struct npc_install_flow_req * req,bool pps)309 static int otx2_tc_act_set_police(struct otx2_nic *nic,
310 struct otx2_tc_flow *node,
311 struct flow_cls_offload *f,
312 u64 rate, u32 burst, u32 mark,
313 struct npc_install_flow_req *req, bool pps)
314 {
315 struct netlink_ext_ack *extack = f->common.extack;
316 struct otx2_hw *hw = &nic->hw;
317 int rq_idx, rc;
318
319 rq_idx = find_first_zero_bit(&nic->rq_bmap, hw->rx_queues);
320 if (rq_idx >= hw->rx_queues) {
321 NL_SET_ERR_MSG_MOD(extack, "Police action rules exceeded");
322 return -EINVAL;
323 }
324
325 req->match_id = mark & 0xFFFFULL;
326 req->index = rq_idx;
327 req->op = NIX_RX_ACTIONOP_UCAST;
328
329 node->is_act_police = true;
330 node->rq = rq_idx;
331 node->burst = burst;
332 node->rate = rate;
333 node->is_pps = pps;
334
335 rc = otx2_tc_act_set_hw_police(nic, node);
336 if (!rc)
337 set_bit(rq_idx, &nic->rq_bmap);
338
339 return rc;
340 }
341
otx2_tc_update_mcast(struct otx2_nic * nic,struct npc_install_flow_req * req,struct netlink_ext_ack * extack,struct otx2_tc_flow * node,struct nix_mcast_grp_update_req * ureq,u8 num_intf)342 static int otx2_tc_update_mcast(struct otx2_nic *nic,
343 struct npc_install_flow_req *req,
344 struct netlink_ext_ack *extack,
345 struct otx2_tc_flow *node,
346 struct nix_mcast_grp_update_req *ureq,
347 u8 num_intf)
348 {
349 struct nix_mcast_grp_update_req *grp_update_req;
350 struct nix_mcast_grp_create_req *creq;
351 struct nix_mcast_grp_create_rsp *crsp;
352 u32 grp_index;
353 int rc;
354
355 mutex_lock(&nic->mbox.lock);
356 creq = otx2_mbox_alloc_msg_nix_mcast_grp_create(&nic->mbox);
357 if (!creq) {
358 rc = -ENOMEM;
359 goto error;
360 }
361
362 creq->dir = NIX_MCAST_INGRESS;
363 /* Send message to AF */
364 rc = otx2_sync_mbox_msg(&nic->mbox);
365 if (rc) {
366 NL_SET_ERR_MSG_MOD(extack, "Failed to create multicast group");
367 goto error;
368 }
369
370 crsp = (struct nix_mcast_grp_create_rsp *)otx2_mbox_get_rsp(&nic->mbox.mbox,
371 0,
372 &creq->hdr);
373 if (IS_ERR(crsp)) {
374 rc = PTR_ERR(crsp);
375 goto error;
376 }
377
378 grp_index = crsp->mcast_grp_idx;
379 grp_update_req = otx2_mbox_alloc_msg_nix_mcast_grp_update(&nic->mbox);
380 if (!grp_update_req) {
381 NL_SET_ERR_MSG_MOD(extack, "Failed to update multicast group");
382 rc = -ENOMEM;
383 goto error;
384 }
385
386 ureq->op = NIX_MCAST_OP_ADD_ENTRY;
387 ureq->mcast_grp_idx = grp_index;
388 ureq->num_mce_entry = num_intf;
389 ureq->pcifunc[0] = nic->pcifunc;
390 ureq->channel[0] = nic->hw.tx_chan_base;
391
392 ureq->dest_type[0] = NIX_RX_RSS;
393 ureq->rq_rss_index[0] = 0;
394 memcpy(&ureq->hdr, &grp_update_req->hdr, sizeof(struct mbox_msghdr));
395 memcpy(grp_update_req, ureq, sizeof(struct nix_mcast_grp_update_req));
396
397 /* Send message to AF */
398 rc = otx2_sync_mbox_msg(&nic->mbox);
399 if (rc) {
400 NL_SET_ERR_MSG_MOD(extack, "Failed to update multicast group");
401 goto error;
402 }
403
404 mutex_unlock(&nic->mbox.lock);
405 req->op = NIX_RX_ACTIONOP_MCAST;
406 req->index = grp_index;
407 node->mcast_grp_idx = grp_index;
408 return 0;
409
410 error:
411 mutex_unlock(&nic->mbox.lock);
412 return rc;
413 }
414
otx2_tc_parse_actions(struct otx2_nic * nic,struct flow_action * flow_action,struct npc_install_flow_req * req,struct flow_cls_offload * f,struct otx2_tc_flow * node)415 static int otx2_tc_parse_actions(struct otx2_nic *nic,
416 struct flow_action *flow_action,
417 struct npc_install_flow_req *req,
418 struct flow_cls_offload *f,
419 struct otx2_tc_flow *node)
420 {
421 struct nix_mcast_grp_update_req dummy_grp_update_req = { 0 };
422 struct netlink_ext_ack *extack = f->common.extack;
423 bool pps = false, mcast = false;
424 struct flow_action_entry *act;
425 struct net_device *target;
426 struct otx2_nic *priv;
427 struct rep_dev *rdev;
428 u32 burst, mark = 0;
429 u8 nr_police = 0;
430 u8 num_intf = 1;
431 int err, i;
432 u64 rate;
433
434 if (!flow_action_has_entries(flow_action)) {
435 NL_SET_ERR_MSG_MOD(extack, "no tc actions specified");
436 return -EINVAL;
437 }
438
439 flow_action_for_each(i, act, flow_action) {
440 switch (act->id) {
441 case FLOW_ACTION_DROP:
442 req->op = NIX_RX_ACTIONOP_DROP;
443 return 0;
444 case FLOW_ACTION_ACCEPT:
445 req->op = NIX_RX_ACTION_DEFAULT;
446 return 0;
447 case FLOW_ACTION_REDIRECT_INGRESS:
448 target = act->dev;
449 if (target->dev.parent) {
450 priv = netdev_priv(target);
451 if (rvu_get_pf(nic->pdev, nic->pcifunc) !=
452 rvu_get_pf(nic->pdev, priv->pcifunc)) {
453 NL_SET_ERR_MSG_MOD(extack,
454 "can't redirect to other pf/vf");
455 return -EOPNOTSUPP;
456 }
457 req->vf = priv->pcifunc & RVU_PFVF_FUNC_MASK;
458 } else {
459 rdev = netdev_priv(target);
460 req->vf = rdev->pcifunc & RVU_PFVF_FUNC_MASK;
461 }
462
463 /* if op is already set; avoid overwriting the same */
464 if (!req->op)
465 req->op = NIX_RX_ACTION_DEFAULT;
466 break;
467
468 case FLOW_ACTION_VLAN_POP:
469 req->vtag0_valid = true;
470 /* use RX_VTAG_TYPE7 which is initialized to strip vlan tag */
471 req->vtag0_type = NIX_AF_LFX_RX_VTAG_TYPE7;
472 break;
473 case FLOW_ACTION_POLICE:
474 /* Ingress ratelimiting is not supported on OcteonTx2 */
475 if (is_dev_otx2(nic->pdev)) {
476 NL_SET_ERR_MSG_MOD(extack,
477 "Ingress policing not supported on this platform");
478 return -EOPNOTSUPP;
479 }
480
481 err = otx2_policer_validate(flow_action, act, extack);
482 if (err)
483 return err;
484
485 if (act->police.rate_bytes_ps > 0) {
486 rate = act->police.rate_bytes_ps * 8;
487 burst = act->police.burst;
488 } else if (act->police.rate_pkt_ps > 0) {
489 /* The algorithm used to calculate rate
490 * mantissa, exponent values for a given token
491 * rate (token can be byte or packet) requires
492 * token rate to be mutiplied by 8.
493 */
494 rate = act->police.rate_pkt_ps * 8;
495 burst = act->police.burst_pkt;
496 pps = true;
497 }
498 nr_police++;
499 break;
500 case FLOW_ACTION_MARK:
501 if (act->mark & ~OTX2_RX_MATCH_ID_MASK) {
502 NL_SET_ERR_MSG_MOD(extack, "Bad flow mark, only 16 bit supported");
503 return -EOPNOTSUPP;
504 }
505 mark = act->mark;
506 req->match_id = mark & OTX2_RX_MATCH_ID_MASK;
507 req->op = NIX_RX_ACTION_DEFAULT;
508 nic->flags |= OTX2_FLAG_TC_MARK_ENABLED;
509 refcount_inc(&nic->flow_cfg->mark_flows);
510 break;
511
512 case FLOW_ACTION_RX_QUEUE_MAPPING:
513 req->op = NIX_RX_ACTIONOP_UCAST;
514 req->index = act->rx_queue;
515 break;
516
517 case FLOW_ACTION_MIRRED_INGRESS:
518 target = act->dev;
519 priv = netdev_priv(target);
520 dummy_grp_update_req.pcifunc[num_intf] = priv->pcifunc;
521 dummy_grp_update_req.channel[num_intf] = priv->hw.tx_chan_base;
522 dummy_grp_update_req.dest_type[num_intf] = NIX_RX_RSS;
523 dummy_grp_update_req.rq_rss_index[num_intf] = 0;
524 mcast = true;
525 num_intf++;
526 break;
527
528 default:
529 return -EOPNOTSUPP;
530 }
531 }
532
533 if (mcast) {
534 err = otx2_tc_update_mcast(nic, req, extack, node,
535 &dummy_grp_update_req,
536 num_intf);
537 if (err)
538 return err;
539 }
540
541 if (nr_police > 1) {
542 NL_SET_ERR_MSG_MOD(extack,
543 "rate limit police offload requires a single action");
544 return -EOPNOTSUPP;
545 }
546
547 if (nr_police)
548 return otx2_tc_act_set_police(nic, node, f, rate, burst,
549 mark, req, pps);
550
551 return 0;
552 }
553
otx2_tc_process_vlan(struct otx2_nic * nic,struct flow_msg * flow_spec,struct flow_msg * flow_mask,struct flow_rule * rule,struct npc_install_flow_req * req,bool is_inner)554 static int otx2_tc_process_vlan(struct otx2_nic *nic, struct flow_msg *flow_spec,
555 struct flow_msg *flow_mask, struct flow_rule *rule,
556 struct npc_install_flow_req *req, bool is_inner)
557 {
558 struct flow_match_vlan match;
559 u16 vlan_tci, vlan_tci_mask;
560
561 if (is_inner)
562 flow_rule_match_cvlan(rule, &match);
563 else
564 flow_rule_match_vlan(rule, &match);
565
566 if (!eth_type_vlan(match.key->vlan_tpid)) {
567 netdev_err(nic->netdev, "vlan tpid 0x%x not supported\n",
568 ntohs(match.key->vlan_tpid));
569 return -EOPNOTSUPP;
570 }
571
572 if (!match.mask->vlan_id) {
573 struct flow_action_entry *act;
574 int i;
575
576 flow_action_for_each(i, act, &rule->action) {
577 if (act->id == FLOW_ACTION_DROP) {
578 netdev_err(nic->netdev,
579 "vlan tpid 0x%x with vlan_id %d is not supported for DROP rule.\n",
580 ntohs(match.key->vlan_tpid), match.key->vlan_id);
581 return -EOPNOTSUPP;
582 }
583 }
584 }
585
586 if (match.mask->vlan_id ||
587 match.mask->vlan_dei ||
588 match.mask->vlan_priority) {
589 vlan_tci = match.key->vlan_id |
590 match.key->vlan_dei << 12 |
591 match.key->vlan_priority << 13;
592
593 vlan_tci_mask = match.mask->vlan_id |
594 match.mask->vlan_dei << 12 |
595 match.mask->vlan_priority << 13;
596 if (is_inner) {
597 flow_spec->vlan_itci = htons(vlan_tci);
598 flow_mask->vlan_itci = htons(vlan_tci_mask);
599 req->features |= BIT_ULL(NPC_INNER_VID);
600 } else {
601 flow_spec->vlan_tci = htons(vlan_tci);
602 flow_mask->vlan_tci = htons(vlan_tci_mask);
603 req->features |= BIT_ULL(NPC_OUTER_VID);
604 }
605 }
606
607 return 0;
608 }
609
otx2_tc_prepare_flow(struct otx2_nic * nic,struct otx2_tc_flow * node,struct flow_cls_offload * f,struct npc_install_flow_req * req)610 static int otx2_tc_prepare_flow(struct otx2_nic *nic, struct otx2_tc_flow *node,
611 struct flow_cls_offload *f,
612 struct npc_install_flow_req *req)
613 {
614 struct netlink_ext_ack *extack = f->common.extack;
615 struct flow_msg *flow_spec = &req->packet;
616 struct flow_msg *flow_mask = &req->mask;
617 struct flow_dissector *dissector;
618 struct flow_rule *rule;
619 u8 ip_proto = 0;
620
621 rule = flow_cls_offload_flow_rule(f);
622 dissector = rule->match.dissector;
623
624 if ((dissector->used_keys &
625 ~(BIT_ULL(FLOW_DISSECTOR_KEY_CONTROL) |
626 BIT_ULL(FLOW_DISSECTOR_KEY_BASIC) |
627 BIT_ULL(FLOW_DISSECTOR_KEY_ETH_ADDRS) |
628 BIT_ULL(FLOW_DISSECTOR_KEY_VLAN) |
629 BIT(FLOW_DISSECTOR_KEY_CVLAN) |
630 BIT_ULL(FLOW_DISSECTOR_KEY_IPV4_ADDRS) |
631 BIT_ULL(FLOW_DISSECTOR_KEY_IPV6_ADDRS) |
632 BIT_ULL(FLOW_DISSECTOR_KEY_PORTS) |
633 BIT(FLOW_DISSECTOR_KEY_IPSEC) |
634 BIT_ULL(FLOW_DISSECTOR_KEY_MPLS) |
635 BIT_ULL(FLOW_DISSECTOR_KEY_ICMP) |
636 BIT_ULL(FLOW_DISSECTOR_KEY_TCP) |
637 BIT_ULL(FLOW_DISSECTOR_KEY_IP)))) {
638 netdev_info(nic->netdev, "unsupported flow used key 0x%llx",
639 dissector->used_keys);
640 return -EOPNOTSUPP;
641 }
642
643 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_BASIC)) {
644 struct flow_match_basic match;
645
646 flow_rule_match_basic(rule, &match);
647
648 /* All EtherTypes can be matched, no hw limitation */
649 flow_spec->etype = match.key->n_proto;
650 flow_mask->etype = match.mask->n_proto;
651 req->features |= BIT_ULL(NPC_ETYPE);
652
653 if (match.mask->ip_proto &&
654 (match.key->ip_proto != IPPROTO_TCP &&
655 match.key->ip_proto != IPPROTO_UDP &&
656 match.key->ip_proto != IPPROTO_SCTP &&
657 match.key->ip_proto != IPPROTO_ICMP &&
658 match.key->ip_proto != IPPROTO_ESP &&
659 match.key->ip_proto != IPPROTO_AH &&
660 match.key->ip_proto != IPPROTO_ICMPV6)) {
661 netdev_info(nic->netdev,
662 "ip_proto=0x%x not supported\n",
663 match.key->ip_proto);
664 return -EOPNOTSUPP;
665 }
666 if (match.mask->ip_proto)
667 ip_proto = match.key->ip_proto;
668
669 if (ip_proto == IPPROTO_UDP)
670 req->features |= BIT_ULL(NPC_IPPROTO_UDP);
671 else if (ip_proto == IPPROTO_TCP)
672 req->features |= BIT_ULL(NPC_IPPROTO_TCP);
673 else if (ip_proto == IPPROTO_SCTP)
674 req->features |= BIT_ULL(NPC_IPPROTO_SCTP);
675 else if (ip_proto == IPPROTO_ICMP)
676 req->features |= BIT_ULL(NPC_IPPROTO_ICMP);
677 else if (ip_proto == IPPROTO_ICMPV6)
678 req->features |= BIT_ULL(NPC_IPPROTO_ICMP6);
679 else if (ip_proto == IPPROTO_ESP)
680 req->features |= BIT_ULL(NPC_IPPROTO_ESP);
681 else if (ip_proto == IPPROTO_AH)
682 req->features |= BIT_ULL(NPC_IPPROTO_AH);
683 }
684
685 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_CONTROL)) {
686 struct flow_match_control match;
687 u32 val;
688
689 flow_rule_match_control(rule, &match);
690
691 if (match.mask->flags & FLOW_DIS_IS_FRAGMENT) {
692 val = match.key->flags & FLOW_DIS_IS_FRAGMENT;
693 if (ntohs(flow_spec->etype) == ETH_P_IP) {
694 flow_spec->ip_flag = val ? IPV4_FLAG_MORE : 0;
695 flow_mask->ip_flag = IPV4_FLAG_MORE;
696 req->features |= BIT_ULL(NPC_IPFRAG_IPV4);
697 } else if (ntohs(flow_spec->etype) == ETH_P_IPV6) {
698 flow_spec->next_header = val ?
699 IPPROTO_FRAGMENT : 0;
700 flow_mask->next_header = 0xff;
701 req->features |= BIT_ULL(NPC_IPFRAG_IPV6);
702 } else {
703 NL_SET_ERR_MSG_MOD(extack, "flow-type should be either IPv4 and IPv6");
704 return -EOPNOTSUPP;
705 }
706 }
707
708 if (!flow_rule_is_supp_control_flags(FLOW_DIS_IS_FRAGMENT,
709 match.mask->flags, extack))
710 return -EOPNOTSUPP;
711 }
712
713 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ETH_ADDRS)) {
714 struct flow_match_eth_addrs match;
715
716 flow_rule_match_eth_addrs(rule, &match);
717 if (!is_zero_ether_addr(match.mask->src)) {
718 NL_SET_ERR_MSG_MOD(extack, "src mac match not supported");
719 return -EOPNOTSUPP;
720 }
721
722 if (!is_zero_ether_addr(match.mask->dst)) {
723 ether_addr_copy(flow_spec->dmac, (u8 *)&match.key->dst);
724 ether_addr_copy(flow_mask->dmac,
725 (u8 *)&match.mask->dst);
726 req->features |= BIT_ULL(NPC_DMAC);
727 }
728 }
729
730 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_IPSEC)) {
731 struct flow_match_ipsec match;
732
733 flow_rule_match_ipsec(rule, &match);
734 if (!match.mask->spi) {
735 NL_SET_ERR_MSG_MOD(extack, "spi index not specified");
736 return -EOPNOTSUPP;
737 }
738 if (ip_proto != IPPROTO_ESP &&
739 ip_proto != IPPROTO_AH) {
740 NL_SET_ERR_MSG_MOD(extack,
741 "SPI index is valid only for ESP/AH proto");
742 return -EOPNOTSUPP;
743 }
744
745 flow_spec->spi = match.key->spi;
746 flow_mask->spi = match.mask->spi;
747 req->features |= BIT_ULL(NPC_IPSEC_SPI);
748 }
749
750 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_IP)) {
751 struct flow_match_ip match;
752
753 flow_rule_match_ip(rule, &match);
754 if ((ntohs(flow_spec->etype) != ETH_P_IP) &&
755 match.mask->tos) {
756 NL_SET_ERR_MSG_MOD(extack, "tos not supported");
757 return -EOPNOTSUPP;
758 }
759 if (match.mask->ttl) {
760 NL_SET_ERR_MSG_MOD(extack, "ttl not supported");
761 return -EOPNOTSUPP;
762 }
763 flow_spec->tos = match.key->tos;
764 flow_mask->tos = match.mask->tos;
765 req->features |= BIT_ULL(NPC_TOS);
766 }
767
768 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_VLAN)) {
769 int ret;
770
771 ret = otx2_tc_process_vlan(nic, flow_spec, flow_mask, rule, req, false);
772 if (ret)
773 return ret;
774 }
775
776 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_CVLAN)) {
777 int ret;
778
779 ret = otx2_tc_process_vlan(nic, flow_spec, flow_mask, rule, req, true);
780 if (ret)
781 return ret;
782 }
783
784 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_IPV4_ADDRS)) {
785 struct flow_match_ipv4_addrs match;
786
787 flow_rule_match_ipv4_addrs(rule, &match);
788
789 flow_spec->ip4dst = match.key->dst;
790 flow_mask->ip4dst = match.mask->dst;
791 req->features |= BIT_ULL(NPC_DIP_IPV4);
792
793 flow_spec->ip4src = match.key->src;
794 flow_mask->ip4src = match.mask->src;
795 req->features |= BIT_ULL(NPC_SIP_IPV4);
796 } else if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_IPV6_ADDRS)) {
797 struct flow_match_ipv6_addrs match;
798
799 flow_rule_match_ipv6_addrs(rule, &match);
800
801 if (ipv6_addr_loopback(&match.key->dst) ||
802 ipv6_addr_loopback(&match.key->src)) {
803 NL_SET_ERR_MSG_MOD(extack,
804 "Flow matching IPv6 loopback addr not supported");
805 return -EOPNOTSUPP;
806 }
807
808 if (!ipv6_addr_any(&match.mask->dst)) {
809 memcpy(&flow_spec->ip6dst,
810 (struct in6_addr *)&match.key->dst,
811 sizeof(flow_spec->ip6dst));
812 memcpy(&flow_mask->ip6dst,
813 (struct in6_addr *)&match.mask->dst,
814 sizeof(flow_spec->ip6dst));
815 req->features |= BIT_ULL(NPC_DIP_IPV6);
816 }
817
818 if (!ipv6_addr_any(&match.mask->src)) {
819 memcpy(&flow_spec->ip6src,
820 (struct in6_addr *)&match.key->src,
821 sizeof(flow_spec->ip6src));
822 memcpy(&flow_mask->ip6src,
823 (struct in6_addr *)&match.mask->src,
824 sizeof(flow_spec->ip6src));
825 req->features |= BIT_ULL(NPC_SIP_IPV6);
826 }
827 }
828
829 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_PORTS)) {
830 struct flow_match_ports match;
831
832 flow_rule_match_ports(rule, &match);
833
834 flow_spec->dport = match.key->dst;
835 flow_mask->dport = match.mask->dst;
836
837 if (flow_mask->dport) {
838 if (ip_proto == IPPROTO_UDP)
839 req->features |= BIT_ULL(NPC_DPORT_UDP);
840 else if (ip_proto == IPPROTO_TCP)
841 req->features |= BIT_ULL(NPC_DPORT_TCP);
842 else if (ip_proto == IPPROTO_SCTP)
843 req->features |= BIT_ULL(NPC_DPORT_SCTP);
844 }
845
846 flow_spec->sport = match.key->src;
847 flow_mask->sport = match.mask->src;
848
849 if (flow_mask->sport) {
850 if (ip_proto == IPPROTO_UDP)
851 req->features |= BIT_ULL(NPC_SPORT_UDP);
852 else if (ip_proto == IPPROTO_TCP)
853 req->features |= BIT_ULL(NPC_SPORT_TCP);
854 else if (ip_proto == IPPROTO_SCTP)
855 req->features |= BIT_ULL(NPC_SPORT_SCTP);
856 }
857 }
858
859 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_TCP)) {
860 struct flow_match_tcp match;
861
862 flow_rule_match_tcp(rule, &match);
863
864 flow_spec->tcp_flags = match.key->flags;
865 flow_mask->tcp_flags = match.mask->flags;
866 req->features |= BIT_ULL(NPC_TCP_FLAGS);
867 }
868
869 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_MPLS)) {
870 struct flow_match_mpls match;
871 u8 bit;
872
873 flow_rule_match_mpls(rule, &match);
874
875 if (match.mask->used_lses & OTX2_UNSUPP_LSE_DEPTH) {
876 NL_SET_ERR_MSG_MOD(extack,
877 "unsupported LSE depth for MPLS match offload");
878 return -EOPNOTSUPP;
879 }
880
881 for_each_set_bit(bit, (unsigned long *)&match.mask->used_lses,
882 FLOW_DIS_MPLS_MAX) {
883 /* check if any of the fields LABEL,TC,BOS are set */
884 if (*((u32 *)&match.mask->ls[bit]) &
885 OTX2_FLOWER_MASK_MPLS_NON_TTL) {
886 /* Hardware will capture 4 byte MPLS header into
887 * two fields NPC_MPLSX_LBTCBOS and NPC_MPLSX_TTL.
888 * Derive the associated NPC key based on header
889 * index and offset.
890 */
891
892 req->features |= BIT_ULL(NPC_MPLS1_LBTCBOS +
893 2 * bit);
894 flow_spec->mpls_lse[bit] =
895 FIELD_PREP(OTX2_FLOWER_MASK_MPLS_LB,
896 match.key->ls[bit].mpls_label) |
897 FIELD_PREP(OTX2_FLOWER_MASK_MPLS_TC,
898 match.key->ls[bit].mpls_tc) |
899 FIELD_PREP(OTX2_FLOWER_MASK_MPLS_BOS,
900 match.key->ls[bit].mpls_bos);
901
902 flow_mask->mpls_lse[bit] =
903 FIELD_PREP(OTX2_FLOWER_MASK_MPLS_LB,
904 match.mask->ls[bit].mpls_label) |
905 FIELD_PREP(OTX2_FLOWER_MASK_MPLS_TC,
906 match.mask->ls[bit].mpls_tc) |
907 FIELD_PREP(OTX2_FLOWER_MASK_MPLS_BOS,
908 match.mask->ls[bit].mpls_bos);
909 }
910
911 if (match.mask->ls[bit].mpls_ttl) {
912 req->features |= BIT_ULL(NPC_MPLS1_TTL +
913 2 * bit);
914 flow_spec->mpls_lse[bit] |=
915 FIELD_PREP(OTX2_FLOWER_MASK_MPLS_TTL,
916 match.key->ls[bit].mpls_ttl);
917 flow_mask->mpls_lse[bit] |=
918 FIELD_PREP(OTX2_FLOWER_MASK_MPLS_TTL,
919 match.mask->ls[bit].mpls_ttl);
920 }
921 }
922 }
923
924 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ICMP)) {
925 struct flow_match_icmp match;
926
927 flow_rule_match_icmp(rule, &match);
928
929 flow_spec->icmp_type = match.key->type;
930 flow_mask->icmp_type = match.mask->type;
931 req->features |= BIT_ULL(NPC_TYPE_ICMP);
932
933 flow_spec->icmp_code = match.key->code;
934 flow_mask->icmp_code = match.mask->code;
935 req->features |= BIT_ULL(NPC_CODE_ICMP);
936 }
937 return otx2_tc_parse_actions(nic, &rule->action, req, f, node);
938 }
939
otx2_destroy_tc_flow_list(struct otx2_nic * pfvf)940 static void otx2_destroy_tc_flow_list(struct otx2_nic *pfvf)
941 {
942 struct otx2_flow_config *flow_cfg = pfvf->flow_cfg;
943 struct otx2_tc_flow *iter, *tmp;
944
945 if (!(pfvf->flags & OTX2_FLAG_MCAM_ENTRIES_ALLOC))
946 return;
947
948 list_for_each_entry_safe(iter, tmp, &flow_cfg->flow_list_tc, list) {
949 list_del(&iter->list);
950 kfree(iter);
951 flow_cfg->nr_flows--;
952 }
953 }
954
955 static struct otx2_tc_flow *
otx2_tc_get_entry_by_cookie(struct otx2_flow_config * flow_cfg,unsigned long cookie)956 otx2_tc_get_entry_by_cookie(struct otx2_flow_config *flow_cfg,
957 unsigned long cookie)
958 {
959 struct otx2_tc_flow *tmp;
960
961 list_for_each_entry(tmp, &flow_cfg->flow_list_tc, list) {
962 if (tmp->cookie == cookie)
963 return tmp;
964 }
965
966 return NULL;
967 }
968
969 struct otx2_tc_flow *
otx2_tc_get_entry_by_index(struct otx2_flow_config * flow_cfg,int index)970 otx2_tc_get_entry_by_index(struct otx2_flow_config *flow_cfg, int index)
971 {
972 struct otx2_tc_flow *tmp;
973 int i = 0;
974
975 list_for_each_entry(tmp, &flow_cfg->flow_list_tc, list) {
976 if (i == index)
977 return tmp;
978 i++;
979 }
980
981 return NULL;
982 }
983
otx2_tc_del_from_flow_list(struct otx2_flow_config * flow_cfg,struct otx2_tc_flow * node)984 static void otx2_tc_del_from_flow_list(struct otx2_flow_config *flow_cfg,
985 struct otx2_tc_flow *node)
986 {
987 struct list_head *pos, *n;
988 struct otx2_tc_flow *tmp;
989
990 list_for_each_safe(pos, n, &flow_cfg->flow_list_tc) {
991 tmp = list_entry(pos, struct otx2_tc_flow, list);
992 if (node == tmp) {
993 list_del(&node->list);
994 return;
995 }
996 }
997 }
998
otx2_tc_add_to_flow_list(struct otx2_flow_config * flow_cfg,struct otx2_tc_flow * node)999 int otx2_tc_add_to_flow_list(struct otx2_flow_config *flow_cfg,
1000 struct otx2_tc_flow *node)
1001 {
1002 struct list_head *pos, *n;
1003 struct otx2_tc_flow *tmp;
1004 int index = 0;
1005
1006 /* If the flow list is empty then add the new node */
1007 if (list_empty(&flow_cfg->flow_list_tc)) {
1008 list_add(&node->list, &flow_cfg->flow_list_tc);
1009 return index;
1010 }
1011
1012 list_for_each_safe(pos, n, &flow_cfg->flow_list_tc) {
1013 tmp = list_entry(pos, struct otx2_tc_flow, list);
1014 if (node->prio < tmp->prio)
1015 break;
1016 index++;
1017 }
1018
1019 list_add(&node->list, pos->prev);
1020 return index;
1021 }
1022
otx2_add_mcam_flow_entry(struct otx2_nic * nic,struct npc_install_flow_req * req)1023 int otx2_add_mcam_flow_entry(struct otx2_nic *nic,
1024 struct npc_install_flow_req *req)
1025 {
1026 struct npc_install_flow_req *tmp_req;
1027 int err;
1028
1029 mutex_lock(&nic->mbox.lock);
1030 tmp_req = otx2_mbox_alloc_msg_npc_install_flow(&nic->mbox);
1031 if (!tmp_req) {
1032 mutex_unlock(&nic->mbox.lock);
1033 return -ENOMEM;
1034 }
1035
1036 memcpy(tmp_req, req, sizeof(struct npc_install_flow_req));
1037 /* Send message to AF */
1038 err = otx2_sync_mbox_msg(&nic->mbox);
1039 if (err) {
1040 netdev_err(nic->netdev, "Failed to install MCAM flow entry %d\n",
1041 req->entry);
1042 mutex_unlock(&nic->mbox.lock);
1043 return -EFAULT;
1044 }
1045
1046 mutex_unlock(&nic->mbox.lock);
1047 return 0;
1048 }
1049
otx2_del_mcam_flow_entry(struct otx2_nic * nic,u16 entry,u16 * cntr_val)1050 int otx2_del_mcam_flow_entry(struct otx2_nic *nic, u16 entry, u16 *cntr_val)
1051 {
1052 struct npc_delete_flow_rsp *rsp;
1053 struct npc_delete_flow_req *req;
1054 int err;
1055
1056 mutex_lock(&nic->mbox.lock);
1057 req = otx2_mbox_alloc_msg_npc_delete_flow(&nic->mbox);
1058 if (!req) {
1059 mutex_unlock(&nic->mbox.lock);
1060 return -ENOMEM;
1061 }
1062
1063 req->entry = entry;
1064
1065 /* Send message to AF */
1066 err = otx2_sync_mbox_msg(&nic->mbox);
1067 if (err) {
1068 netdev_err(nic->netdev, "Failed to delete MCAM flow entry %d\n",
1069 entry);
1070 mutex_unlock(&nic->mbox.lock);
1071 return -EFAULT;
1072 }
1073
1074 if (cntr_val) {
1075 rsp = (struct npc_delete_flow_rsp *)otx2_mbox_get_rsp(&nic->mbox.mbox,
1076 0, &req->hdr);
1077 if (IS_ERR(rsp)) {
1078 netdev_err(nic->netdev, "Failed to get MCAM delete response for entry %d\n",
1079 entry);
1080 mutex_unlock(&nic->mbox.lock);
1081 return -EFAULT;
1082 }
1083
1084 *cntr_val = rsp->cntr_val;
1085 }
1086
1087 mutex_unlock(&nic->mbox.lock);
1088 return 0;
1089 }
1090
otx2_tc_update_mcam_table_del_req(struct otx2_nic * nic,struct otx2_flow_config * flow_cfg,struct otx2_tc_flow * node)1091 static int otx2_tc_update_mcam_table_del_req(struct otx2_nic *nic,
1092 struct otx2_flow_config *flow_cfg,
1093 struct otx2_tc_flow *node)
1094 {
1095 struct list_head *pos, *n;
1096 struct otx2_tc_flow *tmp;
1097 int i = 0, index = 0;
1098 u16 cntr_val = 0;
1099
1100 if (is_cn20k(nic->pdev)) {
1101 cn20k_tc_update_mcam_table_del_req(nic, flow_cfg, node);
1102 return 0;
1103 }
1104
1105 /* Find and delete the entry from the list and re-install
1106 * all the entries from beginning to the index of the
1107 * deleted entry to higher mcam indexes.
1108 */
1109 list_for_each_safe(pos, n, &flow_cfg->flow_list_tc) {
1110 tmp = list_entry(pos, struct otx2_tc_flow, list);
1111 if (node == tmp) {
1112 list_del(&tmp->list);
1113 break;
1114 }
1115
1116 otx2_del_mcam_flow_entry(nic, tmp->entry, &cntr_val);
1117 tmp->entry++;
1118 tmp->req.entry = tmp->entry;
1119 tmp->req.cntr_val = cntr_val;
1120 index++;
1121 }
1122
1123 list_for_each_safe(pos, n, &flow_cfg->flow_list_tc) {
1124 if (i == index)
1125 break;
1126
1127 tmp = list_entry(pos, struct otx2_tc_flow, list);
1128 otx2_add_mcam_flow_entry(nic, &tmp->req);
1129 i++;
1130 }
1131
1132 return 0;
1133 }
1134
otx2_tc_update_mcam_table_add_req(struct otx2_nic * nic,struct otx2_flow_config * flow_cfg,struct otx2_tc_flow * node)1135 static int otx2_tc_update_mcam_table_add_req(struct otx2_nic *nic,
1136 struct otx2_flow_config *flow_cfg,
1137 struct otx2_tc_flow *node)
1138 {
1139 int mcam_idx = flow_cfg->max_flows - flow_cfg->nr_flows - 1;
1140 struct otx2_tc_flow *tmp;
1141 int list_idx, i;
1142 u16 cntr_val = 0;
1143
1144 if (is_cn20k(nic->pdev))
1145 return cn20k_tc_update_mcam_table_add_req(nic, flow_cfg, node);
1146
1147 /* Find the index of the entry(list_idx) whose priority
1148 * is greater than the new entry and re-install all
1149 * the entries from beginning to list_idx to higher
1150 * mcam indexes.
1151 */
1152 list_idx = otx2_tc_add_to_flow_list(flow_cfg, node);
1153 for (i = 0; i < list_idx; i++) {
1154 tmp = otx2_tc_get_entry_by_index(flow_cfg, i);
1155 if (!tmp)
1156 return -ENOMEM;
1157
1158 otx2_del_mcam_flow_entry(nic, tmp->entry, &cntr_val);
1159 tmp->entry = flow_cfg->flow_ent[mcam_idx];
1160 tmp->req.entry = tmp->entry;
1161 tmp->req.cntr_val = cntr_val;
1162 otx2_add_mcam_flow_entry(nic, &tmp->req);
1163 mcam_idx++;
1164 }
1165
1166 return flow_cfg->flow_ent[mcam_idx];
1167 }
1168
otx2_tc_update_mcam_table(struct otx2_nic * nic,struct otx2_flow_config * flow_cfg,struct otx2_tc_flow * node,bool add_req)1169 static int otx2_tc_update_mcam_table(struct otx2_nic *nic,
1170 struct otx2_flow_config *flow_cfg,
1171 struct otx2_tc_flow *node,
1172 bool add_req)
1173 {
1174 if (add_req)
1175 return otx2_tc_update_mcam_table_add_req(nic, flow_cfg, node);
1176
1177 return otx2_tc_update_mcam_table_del_req(nic, flow_cfg, node);
1178 }
1179
otx2_tc_del_flow(struct otx2_nic * nic,struct flow_cls_offload * tc_flow_cmd)1180 static int otx2_tc_del_flow(struct otx2_nic *nic,
1181 struct flow_cls_offload *tc_flow_cmd)
1182 {
1183 struct otx2_flow_config *flow_cfg = nic->flow_cfg;
1184 struct nix_mcast_grp_destroy_req *grp_destroy_req;
1185 struct otx2_tc_flow *flow_node;
1186 int err;
1187
1188 flow_node = otx2_tc_get_entry_by_cookie(flow_cfg, tc_flow_cmd->cookie);
1189 if (!flow_node) {
1190 netdev_err(nic->netdev, "tc flow not found for cookie 0x%lx\n",
1191 tc_flow_cmd->cookie);
1192 return -EINVAL;
1193 }
1194
1195 /* Disable TC MARK flag if they are no rules with skbedit mark action */
1196 if (flow_node->req.match_id)
1197 if (!refcount_dec_and_test(&flow_cfg->mark_flows))
1198 nic->flags &= ~OTX2_FLAG_TC_MARK_ENABLED;
1199
1200 if (flow_node->is_act_police) {
1201 __clear_bit(flow_node->rq, &nic->rq_bmap);
1202
1203 if (nic->flags & OTX2_FLAG_INTF_DOWN)
1204 goto free_mcam_flow;
1205
1206 mutex_lock(&nic->mbox.lock);
1207
1208 err = cn10k_map_unmap_rq_policer(nic, flow_node->rq,
1209 flow_node->leaf_profile, false);
1210 if (err)
1211 netdev_err(nic->netdev,
1212 "Unmapping RQ %d & profile %d failed\n",
1213 flow_node->rq, flow_node->leaf_profile);
1214
1215 err = cn10k_free_leaf_profile(nic, flow_node->leaf_profile);
1216 if (err)
1217 netdev_err(nic->netdev,
1218 "Unable to free leaf bandwidth profile(%d)\n",
1219 flow_node->leaf_profile);
1220
1221 mutex_unlock(&nic->mbox.lock);
1222 }
1223 /* Remove the multicast/mirror related nodes */
1224 if (flow_node->mcast_grp_idx != MCAST_INVALID_GRP) {
1225 mutex_lock(&nic->mbox.lock);
1226 grp_destroy_req = otx2_mbox_alloc_msg_nix_mcast_grp_destroy(&nic->mbox);
1227 grp_destroy_req->mcast_grp_idx = flow_node->mcast_grp_idx;
1228 otx2_sync_mbox_msg(&nic->mbox);
1229 mutex_unlock(&nic->mbox.lock);
1230 }
1231
1232 free_mcam_flow:
1233 otx2_del_mcam_flow_entry(nic, flow_node->entry, NULL);
1234 otx2_tc_update_mcam_table(nic, flow_cfg, flow_node, false);
1235 kfree_rcu(flow_node, rcu);
1236 flow_cfg->nr_flows--;
1237 return 0;
1238 }
1239
otx2_tc_add_flow(struct otx2_nic * nic,struct flow_cls_offload * tc_flow_cmd)1240 static int otx2_tc_add_flow(struct otx2_nic *nic,
1241 struct flow_cls_offload *tc_flow_cmd)
1242 {
1243 struct netlink_ext_ack *extack = tc_flow_cmd->common.extack;
1244 struct otx2_flow_config *flow_cfg = nic->flow_cfg;
1245 struct otx2_tc_flow *new_node, *old_node;
1246 struct npc_install_flow_req *req, dummy;
1247 int rc, err, entry;
1248
1249 if (!(nic->flags & OTX2_FLAG_TC_FLOWER_SUPPORT))
1250 return -ENOMEM;
1251
1252 if (nic->flags & OTX2_FLAG_INTF_DOWN) {
1253 NL_SET_ERR_MSG_MOD(extack, "Interface not initialized");
1254 return -EINVAL;
1255 }
1256
1257 if (!is_cn20k(nic->pdev) && flow_cfg->nr_flows == flow_cfg->max_flows) {
1258 NL_SET_ERR_MSG_MOD(extack,
1259 "Free MCAM entry not available to add the flow");
1260 return -ENOMEM;
1261 }
1262
1263 /* allocate memory for the new flow and it's node */
1264 new_node = kzalloc_obj(*new_node);
1265 if (!new_node)
1266 return -ENOMEM;
1267 spin_lock_init(&new_node->lock);
1268 new_node->cookie = tc_flow_cmd->cookie;
1269 new_node->prio = tc_flow_cmd->common.prio;
1270 new_node->mcast_grp_idx = MCAST_INVALID_GRP;
1271
1272 memset(&dummy, 0, sizeof(struct npc_install_flow_req));
1273
1274 rc = otx2_tc_prepare_flow(nic, new_node, tc_flow_cmd, &dummy);
1275 if (rc) {
1276 kfree_rcu(new_node, rcu);
1277 return rc;
1278 }
1279
1280 /* If a flow exists with the same cookie, delete it */
1281 old_node = otx2_tc_get_entry_by_cookie(flow_cfg, tc_flow_cmd->cookie);
1282 if (old_node)
1283 otx2_tc_del_flow(nic, tc_flow_cmd);
1284
1285 if (is_cn20k(nic->pdev)) {
1286 rc = cn20k_tc_alloc_entry(nic, tc_flow_cmd, new_node, &dummy);
1287 if (rc) {
1288 NL_SET_ERR_MSG_MOD(extack,
1289 "MCAM rule allocation failed");
1290 kfree_rcu(new_node, rcu);
1291 return rc;
1292 }
1293 }
1294
1295 entry = otx2_tc_update_mcam_table(nic, flow_cfg, new_node, true);
1296 if (entry < 0) {
1297 NL_SET_ERR_MSG_MOD(extack, "Adding rule failed");
1298 rc = entry;
1299 goto free_leaf;
1300 }
1301
1302 mutex_lock(&nic->mbox.lock);
1303 req = otx2_mbox_alloc_msg_npc_install_flow(&nic->mbox);
1304 if (!req) {
1305 mutex_unlock(&nic->mbox.lock);
1306 rc = -ENOMEM;
1307 goto free_leaf;
1308 }
1309
1310 memcpy(&dummy.hdr, &req->hdr, sizeof(struct mbox_msghdr));
1311 memcpy(req, &dummy, sizeof(struct npc_install_flow_req));
1312 req->channel = nic->hw.rx_chan_base;
1313 req->entry = (u16)entry;
1314 req->intf = NIX_INTF_RX;
1315 req->vf = nic->pcifunc;
1316 req->set_cntr = 1;
1317 new_node->entry = req->entry;
1318
1319 /* Send message to AF */
1320 rc = otx2_sync_mbox_msg(&nic->mbox);
1321 if (rc) {
1322 NL_SET_ERR_MSG_MOD(extack, "Failed to install MCAM flow entry");
1323 mutex_unlock(&nic->mbox.lock);
1324 goto free_leaf;
1325 }
1326
1327 mutex_unlock(&nic->mbox.lock);
1328 memcpy(&new_node->req, req, sizeof(struct npc_install_flow_req));
1329
1330 flow_cfg->nr_flows++;
1331 return 0;
1332
1333 free_leaf:
1334 if (is_cn20k(nic->pdev))
1335 cn20k_tc_free_mcam_entry(nic, new_node->entry);
1336 otx2_tc_del_from_flow_list(flow_cfg, new_node);
1337 if (new_node->is_act_police) {
1338 mutex_lock(&nic->mbox.lock);
1339
1340 err = cn10k_map_unmap_rq_policer(nic, new_node->rq,
1341 new_node->leaf_profile, false);
1342 if (err)
1343 netdev_err(nic->netdev,
1344 "Unmapping RQ %d & profile %d failed\n",
1345 new_node->rq, new_node->leaf_profile);
1346 err = cn10k_free_leaf_profile(nic, new_node->leaf_profile);
1347 if (err)
1348 netdev_err(nic->netdev,
1349 "Unable to free leaf bandwidth profile(%d)\n",
1350 new_node->leaf_profile);
1351
1352 __clear_bit(new_node->rq, &nic->rq_bmap);
1353
1354 mutex_unlock(&nic->mbox.lock);
1355 }
1356 kfree_rcu(new_node, rcu);
1357
1358 return rc;
1359 }
1360
otx2_tc_get_flow_stats(struct otx2_nic * nic,struct flow_cls_offload * tc_flow_cmd)1361 static int otx2_tc_get_flow_stats(struct otx2_nic *nic,
1362 struct flow_cls_offload *tc_flow_cmd)
1363 {
1364 struct npc_mcam_get_stats_req *req;
1365 struct npc_mcam_get_stats_rsp *rsp;
1366 struct otx2_tc_flow_stats *stats;
1367 struct otx2_tc_flow *flow_node;
1368 int err;
1369
1370 flow_node = otx2_tc_get_entry_by_cookie(nic->flow_cfg, tc_flow_cmd->cookie);
1371 if (!flow_node) {
1372 netdev_info(nic->netdev, "tc flow not found for cookie %lx",
1373 tc_flow_cmd->cookie);
1374 return -EINVAL;
1375 }
1376
1377 mutex_lock(&nic->mbox.lock);
1378
1379 req = otx2_mbox_alloc_msg_npc_mcam_entry_stats(&nic->mbox);
1380 if (!req) {
1381 mutex_unlock(&nic->mbox.lock);
1382 return -ENOMEM;
1383 }
1384
1385 req->entry = flow_node->entry;
1386
1387 err = otx2_sync_mbox_msg(&nic->mbox);
1388 if (err) {
1389 netdev_err(nic->netdev, "Failed to get stats for MCAM flow entry %d\n",
1390 req->entry);
1391 mutex_unlock(&nic->mbox.lock);
1392 return -EFAULT;
1393 }
1394
1395 rsp = (struct npc_mcam_get_stats_rsp *)otx2_mbox_get_rsp
1396 (&nic->mbox.mbox, 0, &req->hdr);
1397 if (IS_ERR(rsp)) {
1398 mutex_unlock(&nic->mbox.lock);
1399 return PTR_ERR(rsp);
1400 }
1401
1402 mutex_unlock(&nic->mbox.lock);
1403
1404 if (!rsp->stat_ena)
1405 return -EINVAL;
1406
1407 stats = &flow_node->stats;
1408
1409 spin_lock(&flow_node->lock);
1410 flow_stats_update(&tc_flow_cmd->stats, 0x0, rsp->stat - stats->pkts, 0x0, 0x0,
1411 FLOW_ACTION_HW_STATS_IMMEDIATE);
1412 stats->pkts = rsp->stat;
1413 spin_unlock(&flow_node->lock);
1414
1415 return 0;
1416 }
1417
otx2_setup_tc_cls_flower(struct otx2_nic * nic,struct flow_cls_offload * cls_flower)1418 int otx2_setup_tc_cls_flower(struct otx2_nic *nic,
1419 struct flow_cls_offload *cls_flower)
1420 {
1421 switch (cls_flower->command) {
1422 case FLOW_CLS_REPLACE:
1423 return otx2_tc_add_flow(nic, cls_flower);
1424 case FLOW_CLS_DESTROY:
1425 return otx2_tc_del_flow(nic, cls_flower);
1426 case FLOW_CLS_STATS:
1427 return otx2_tc_get_flow_stats(nic, cls_flower);
1428 default:
1429 return -EOPNOTSUPP;
1430 }
1431 }
1432 EXPORT_SYMBOL(otx2_setup_tc_cls_flower);
1433
otx2_tc_ingress_matchall_install(struct otx2_nic * nic,struct tc_cls_matchall_offload * cls)1434 static int otx2_tc_ingress_matchall_install(struct otx2_nic *nic,
1435 struct tc_cls_matchall_offload *cls)
1436 {
1437 struct netlink_ext_ack *extack = cls->common.extack;
1438 struct flow_action *actions = &cls->rule->action;
1439 struct flow_action_entry *entry;
1440 u64 rate;
1441 int err;
1442
1443 err = otx2_tc_validate_flow(nic, actions, extack);
1444 if (err)
1445 return err;
1446
1447 if (nic->flags & OTX2_FLAG_TC_MATCHALL_INGRESS_ENABLED) {
1448 NL_SET_ERR_MSG_MOD(extack,
1449 "Only one ingress MATCHALL ratelimitter can be offloaded");
1450 return -ENOMEM;
1451 }
1452
1453 entry = &cls->rule->action.entries[0];
1454 switch (entry->id) {
1455 case FLOW_ACTION_POLICE:
1456 /* Ingress ratelimiting is not supported on OcteonTx2 */
1457 if (is_dev_otx2(nic->pdev)) {
1458 NL_SET_ERR_MSG_MOD(extack,
1459 "Ingress policing not supported on this platform");
1460 return -EOPNOTSUPP;
1461 }
1462
1463 err = cn10k_alloc_matchall_ipolicer(nic);
1464 if (err)
1465 return err;
1466
1467 /* Convert to bits per second */
1468 rate = entry->police.rate_bytes_ps * 8;
1469 err = cn10k_set_matchall_ipolicer_rate(nic, entry->police.burst, rate);
1470 if (err)
1471 return err;
1472 nic->flags |= OTX2_FLAG_TC_MATCHALL_INGRESS_ENABLED;
1473 break;
1474 default:
1475 NL_SET_ERR_MSG_MOD(extack,
1476 "Only police action supported with Ingress MATCHALL offload");
1477 return -EOPNOTSUPP;
1478 }
1479
1480 return 0;
1481 }
1482
otx2_tc_ingress_matchall_delete(struct otx2_nic * nic,struct tc_cls_matchall_offload * cls)1483 static int otx2_tc_ingress_matchall_delete(struct otx2_nic *nic,
1484 struct tc_cls_matchall_offload *cls)
1485 {
1486 struct netlink_ext_ack *extack = cls->common.extack;
1487 int err;
1488
1489 if (nic->flags & OTX2_FLAG_INTF_DOWN) {
1490 NL_SET_ERR_MSG_MOD(extack, "Interface not initialized");
1491 return -EINVAL;
1492 }
1493
1494 err = cn10k_free_matchall_ipolicer(nic);
1495 nic->flags &= ~OTX2_FLAG_TC_MATCHALL_INGRESS_ENABLED;
1496 return err;
1497 }
1498
otx2_setup_tc_ingress_matchall(struct otx2_nic * nic,struct tc_cls_matchall_offload * cls_matchall)1499 static int otx2_setup_tc_ingress_matchall(struct otx2_nic *nic,
1500 struct tc_cls_matchall_offload *cls_matchall)
1501 {
1502 switch (cls_matchall->command) {
1503 case TC_CLSMATCHALL_REPLACE:
1504 return otx2_tc_ingress_matchall_install(nic, cls_matchall);
1505 case TC_CLSMATCHALL_DESTROY:
1506 return otx2_tc_ingress_matchall_delete(nic, cls_matchall);
1507 case TC_CLSMATCHALL_STATS:
1508 default:
1509 break;
1510 }
1511
1512 return -EOPNOTSUPP;
1513 }
1514
otx2_setup_tc_block_ingress_cb(enum tc_setup_type type,void * type_data,void * cb_priv)1515 static int otx2_setup_tc_block_ingress_cb(enum tc_setup_type type,
1516 void *type_data, void *cb_priv)
1517 {
1518 struct otx2_nic *nic = cb_priv;
1519 bool ntuple;
1520
1521 if (!tc_cls_can_offload_and_chain0(nic->netdev, type_data))
1522 return -EOPNOTSUPP;
1523
1524 ntuple = nic->netdev->features & NETIF_F_NTUPLE;
1525 switch (type) {
1526 case TC_SETUP_CLSFLOWER:
1527 if (ntuple) {
1528 netdev_warn(nic->netdev,
1529 "Can't install TC flower offload rule when NTUPLE is active");
1530 return -EOPNOTSUPP;
1531 }
1532
1533 return otx2_setup_tc_cls_flower(nic, type_data);
1534 case TC_SETUP_CLSMATCHALL:
1535 return otx2_setup_tc_ingress_matchall(nic, type_data);
1536 default:
1537 break;
1538 }
1539
1540 return -EOPNOTSUPP;
1541 }
1542
otx2_setup_tc_egress_matchall(struct otx2_nic * nic,struct tc_cls_matchall_offload * cls_matchall)1543 static int otx2_setup_tc_egress_matchall(struct otx2_nic *nic,
1544 struct tc_cls_matchall_offload *cls_matchall)
1545 {
1546 switch (cls_matchall->command) {
1547 case TC_CLSMATCHALL_REPLACE:
1548 return otx2_tc_egress_matchall_install(nic, cls_matchall);
1549 case TC_CLSMATCHALL_DESTROY:
1550 return otx2_tc_egress_matchall_delete(nic, cls_matchall);
1551 case TC_CLSMATCHALL_STATS:
1552 default:
1553 break;
1554 }
1555
1556 return -EOPNOTSUPP;
1557 }
1558
otx2_setup_tc_block_egress_cb(enum tc_setup_type type,void * type_data,void * cb_priv)1559 static int otx2_setup_tc_block_egress_cb(enum tc_setup_type type,
1560 void *type_data, void *cb_priv)
1561 {
1562 struct otx2_nic *nic = cb_priv;
1563
1564 if (!tc_cls_can_offload_and_chain0(nic->netdev, type_data))
1565 return -EOPNOTSUPP;
1566
1567 switch (type) {
1568 case TC_SETUP_CLSMATCHALL:
1569 return otx2_setup_tc_egress_matchall(nic, type_data);
1570 default:
1571 break;
1572 }
1573
1574 return -EOPNOTSUPP;
1575 }
1576
1577 static LIST_HEAD(otx2_block_cb_list);
1578
otx2_setup_tc_block(struct net_device * netdev,struct flow_block_offload * f)1579 static int otx2_setup_tc_block(struct net_device *netdev,
1580 struct flow_block_offload *f)
1581 {
1582 struct otx2_nic *nic = netdev_priv(netdev);
1583 flow_setup_cb_t *cb;
1584 bool ingress;
1585
1586 if (f->block_shared)
1587 return -EOPNOTSUPP;
1588
1589 if (f->binder_type == FLOW_BLOCK_BINDER_TYPE_CLSACT_INGRESS) {
1590 cb = otx2_setup_tc_block_ingress_cb;
1591 ingress = true;
1592 } else if (f->binder_type == FLOW_BLOCK_BINDER_TYPE_CLSACT_EGRESS) {
1593 cb = otx2_setup_tc_block_egress_cb;
1594 ingress = false;
1595 } else {
1596 return -EOPNOTSUPP;
1597 }
1598
1599 return flow_block_cb_setup_simple(f, &otx2_block_cb_list, cb,
1600 nic, nic, ingress);
1601 }
1602
otx2_setup_tc(struct net_device * netdev,enum tc_setup_type type,void * type_data)1603 int otx2_setup_tc(struct net_device *netdev, enum tc_setup_type type,
1604 void *type_data)
1605 {
1606 switch (type) {
1607 case TC_SETUP_BLOCK:
1608 return otx2_setup_tc_block(netdev, type_data);
1609 case TC_SETUP_QDISC_HTB:
1610 return otx2_setup_tc_htb(netdev, type_data);
1611 default:
1612 return -EOPNOTSUPP;
1613 }
1614 }
1615 EXPORT_SYMBOL(otx2_setup_tc);
1616
otx2_init_tc(struct otx2_nic * nic)1617 int otx2_init_tc(struct otx2_nic *nic)
1618 {
1619 /* Exclude receive queue 0 being used for police action */
1620 set_bit(0, &nic->rq_bmap);
1621
1622 if (!nic->flow_cfg) {
1623 netdev_err(nic->netdev,
1624 "Can't init TC, nic->flow_cfg is not setup\n");
1625 return -EINVAL;
1626 }
1627
1628 return 0;
1629 }
1630 EXPORT_SYMBOL(otx2_init_tc);
1631
otx2_shutdown_tc(struct otx2_nic * nic)1632 void otx2_shutdown_tc(struct otx2_nic *nic)
1633 {
1634 otx2_destroy_tc_flow_list(nic);
1635 }
1636 EXPORT_SYMBOL(otx2_shutdown_tc);
1637
otx2_tc_config_ingress_rule(struct otx2_nic * nic,struct otx2_tc_flow * node)1638 static void otx2_tc_config_ingress_rule(struct otx2_nic *nic,
1639 struct otx2_tc_flow *node)
1640 {
1641 struct npc_install_flow_req *req;
1642
1643 if (otx2_tc_act_set_hw_police(nic, node))
1644 return;
1645
1646 mutex_lock(&nic->mbox.lock);
1647
1648 req = otx2_mbox_alloc_msg_npc_install_flow(&nic->mbox);
1649 if (!req)
1650 goto err;
1651
1652 memcpy(req, &node->req, sizeof(struct npc_install_flow_req));
1653
1654 if (otx2_sync_mbox_msg(&nic->mbox))
1655 netdev_err(nic->netdev,
1656 "Failed to install MCAM flow entry for ingress rule");
1657 err:
1658 mutex_unlock(&nic->mbox.lock);
1659 }
1660
otx2_tc_apply_ingress_police_rules(struct otx2_nic * nic)1661 void otx2_tc_apply_ingress_police_rules(struct otx2_nic *nic)
1662 {
1663 struct otx2_flow_config *flow_cfg = nic->flow_cfg;
1664 struct otx2_tc_flow *node;
1665
1666 /* If any ingress policer rules exist for the interface then
1667 * apply those rules. Ingress policer rules depend on bandwidth
1668 * profiles linked to the receive queues. Since no receive queues
1669 * exist when interface is down, ingress policer rules are stored
1670 * and configured in hardware after all receive queues are allocated
1671 * in otx2_open.
1672 */
1673 list_for_each_entry(node, &flow_cfg->flow_list_tc, list) {
1674 if (node->is_act_police)
1675 otx2_tc_config_ingress_rule(nic, node);
1676 }
1677 }
1678 EXPORT_SYMBOL(otx2_tc_apply_ingress_police_rules);
1679