1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) 2 /* Copyright (C) 2019 Netronome Systems, Inc. */ 3 4 #include <linux/hash.h> 5 #include <linux/hashtable.h> 6 #include <linux/jhash.h> 7 #include <linux/math64.h> 8 #include <linux/vmalloc.h> 9 #include <net/pkt_cls.h> 10 #include <net/pkt_sched.h> 11 12 #include "cmsg.h" 13 #include "main.h" 14 #include "../nfp_port.h" 15 16 #define NFP_FL_QOS_UPDATE msecs_to_jiffies(1000) 17 #define NFP_FL_QOS_PPS BIT(15) 18 #define NFP_FL_QOS_METER BIT(10) 19 20 struct nfp_police_cfg_head { 21 __be32 flags_opts; 22 union { 23 __be32 meter_id; 24 __be32 port; 25 }; 26 }; 27 28 enum NFP_FL_QOS_TYPES { 29 NFP_FL_QOS_TYPE_BPS, 30 NFP_FL_QOS_TYPE_PPS, 31 NFP_FL_QOS_TYPE_MAX, 32 }; 33 34 /* Police cmsg for configuring a trTCM traffic conditioner (8W/32B) 35 * See RFC 2698 for more details. 36 * ---------------------------------------------------------------- 37 * 3 2 1 38 * 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0 39 * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ 40 * | Reserved |p| Reserved | 41 * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ 42 * | Port Ingress | 43 * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ 44 * | Token Bucket Peak | 45 * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ 46 * | Token Bucket Committed | 47 * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ 48 * | Peak Burst Size | 49 * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ 50 * | Committed Burst Size | 51 * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ 52 * | Peak Information Rate | 53 * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ 54 * | Committed Information Rate | 55 * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ 56 * Word[0](FLag options): 57 * [15] p(pps) 1 for pps, 0 for bps 58 * 59 * Meter control message 60 * 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0 61 * +-------------------------------+-+---+-----+-+---------+-+---+-+ 62 * | Reserved |p| Y |TYPE |E|TSHFV |P| PC|R| 63 * +-------------------------------+-+---+-----+-+---------+-+---+-+ 64 * | meter ID | 65 * +-------------------------------+-------------------------------+ 66 * 67 */ 68 struct nfp_police_config { 69 struct nfp_police_cfg_head head; 70 __be32 bkt_tkn_p; 71 __be32 bkt_tkn_c; 72 __be32 pbs; 73 __be32 cbs; 74 __be32 pir; 75 __be32 cir; 76 }; 77 78 struct nfp_police_stats_reply { 79 struct nfp_police_cfg_head head; 80 __be64 pass_bytes; 81 __be64 pass_pkts; 82 __be64 drop_bytes; 83 __be64 drop_pkts; 84 }; 85 86 int nfp_flower_offload_one_police(struct nfp_app *app, bool ingress, 87 bool pps, u32 id, u32 rate, u32 burst) 88 { 89 struct nfp_police_config *config; 90 struct sk_buff *skb; 91 92 skb = nfp_flower_cmsg_alloc(app, sizeof(struct nfp_police_config), 93 NFP_FLOWER_CMSG_TYPE_QOS_MOD, GFP_KERNEL); 94 if (!skb) 95 return -ENOMEM; 96 97 config = nfp_flower_cmsg_get_data(skb); 98 memset(config, 0, sizeof(struct nfp_police_config)); 99 if (pps) 100 config->head.flags_opts |= cpu_to_be32(NFP_FL_QOS_PPS); 101 if (!ingress) 102 config->head.flags_opts |= cpu_to_be32(NFP_FL_QOS_METER); 103 104 if (ingress) 105 config->head.port = cpu_to_be32(id); 106 else 107 config->head.meter_id = cpu_to_be32(id); 108 109 config->bkt_tkn_p = cpu_to_be32(burst); 110 config->bkt_tkn_c = cpu_to_be32(burst); 111 config->pbs = cpu_to_be32(burst); 112 config->cbs = cpu_to_be32(burst); 113 config->pir = cpu_to_be32(rate); 114 config->cir = cpu_to_be32(rate); 115 nfp_ctrl_tx(app->ctrl, skb); 116 117 return 0; 118 } 119 120 static int nfp_policer_validate(const struct flow_action *action, 121 const struct flow_action_entry *act, 122 struct netlink_ext_ack *extack, 123 bool ingress) 124 { 125 if (act->police.exceed.act_id != FLOW_ACTION_DROP) { 126 NL_SET_ERR_MSG_MOD(extack, 127 "Offload not supported when exceed action is not drop"); 128 return -EOPNOTSUPP; 129 } 130 131 if (ingress) { 132 if (act->police.notexceed.act_id != FLOW_ACTION_CONTINUE && 133 act->police.notexceed.act_id != FLOW_ACTION_ACCEPT) { 134 NL_SET_ERR_MSG_MOD(extack, 135 "Offload not supported when conform action is not continue or ok"); 136 return -EOPNOTSUPP; 137 } 138 } else { 139 if (act->police.notexceed.act_id != FLOW_ACTION_PIPE && 140 act->police.notexceed.act_id != FLOW_ACTION_ACCEPT) { 141 NL_SET_ERR_MSG_MOD(extack, 142 "Offload not supported when conform action is not pipe or ok"); 143 return -EOPNOTSUPP; 144 } 145 } 146 147 if (act->police.notexceed.act_id == FLOW_ACTION_ACCEPT && 148 !flow_action_is_last_entry(action, act)) { 149 NL_SET_ERR_MSG_MOD(extack, 150 "Offload not supported when conform action is ok, but action is not last"); 151 return -EOPNOTSUPP; 152 } 153 154 if (act->police.peakrate_bytes_ps || 155 act->police.avrate || act->police.overhead) { 156 NL_SET_ERR_MSG_MOD(extack, 157 "Offload not supported when peakrate/avrate/overhead is configured"); 158 return -EOPNOTSUPP; 159 } 160 161 return 0; 162 } 163 164 static int 165 nfp_flower_install_rate_limiter(struct nfp_app *app, struct net_device *netdev, 166 struct tc_cls_matchall_offload *flow, 167 struct netlink_ext_ack *extack) 168 { 169 struct flow_action_entry *paction = &flow->rule->action.entries[0]; 170 u32 action_num = flow->rule->action.num_entries; 171 struct nfp_flower_priv *fl_priv = app->priv; 172 struct flow_action_entry *action = NULL; 173 struct nfp_flower_repr_priv *repr_priv; 174 u32 netdev_port_id, i; 175 struct nfp_repr *repr; 176 bool pps_support; 177 u32 bps_num = 0; 178 u32 pps_num = 0; 179 u32 burst; 180 bool pps; 181 u64 rate; 182 int err; 183 184 if (!nfp_netdev_is_nfp_repr(netdev)) { 185 NL_SET_ERR_MSG_MOD(extack, "unsupported offload: qos rate limit offload not supported on higher level port"); 186 return -EOPNOTSUPP; 187 } 188 repr = netdev_priv(netdev); 189 repr_priv = repr->app_priv; 190 netdev_port_id = nfp_repr_get_port_id(netdev); 191 pps_support = !!(fl_priv->flower_ext_feats & NFP_FL_FEATS_QOS_PPS); 192 193 if (repr_priv->block_shared) { 194 NL_SET_ERR_MSG_MOD(extack, "unsupported offload: qos rate limit offload not supported on shared blocks"); 195 return -EOPNOTSUPP; 196 } 197 198 if (repr->port->type != NFP_PORT_VF_PORT) { 199 NL_SET_ERR_MSG_MOD(extack, "unsupported offload: qos rate limit offload not supported on non-VF ports"); 200 return -EOPNOTSUPP; 201 } 202 203 if (pps_support) { 204 if (action_num > 2 || action_num == 0) { 205 NL_SET_ERR_MSG_MOD(extack, 206 "unsupported offload: qos rate limit offload only support action number 1 or 2"); 207 return -EOPNOTSUPP; 208 } 209 } else { 210 if (!flow_offload_has_one_action(&flow->rule->action)) { 211 NL_SET_ERR_MSG_MOD(extack, 212 "unsupported offload: qos rate limit offload requires a single action"); 213 return -EOPNOTSUPP; 214 } 215 } 216 217 if (flow->common.prio != 1) { 218 NL_SET_ERR_MSG_MOD(extack, "unsupported offload: qos rate limit offload requires highest priority"); 219 return -EOPNOTSUPP; 220 } 221 222 for (i = 0 ; i < action_num; i++) { 223 action = paction + i; 224 if (action->id != FLOW_ACTION_POLICE) { 225 NL_SET_ERR_MSG_MOD(extack, 226 "unsupported offload: qos rate limit offload requires police action"); 227 return -EOPNOTSUPP; 228 } 229 230 err = nfp_policer_validate(&flow->rule->action, action, extack, true); 231 if (err) 232 return err; 233 234 if (action->police.rate_bytes_ps > 0) { 235 if (bps_num++) { 236 NL_SET_ERR_MSG_MOD(extack, 237 "unsupported offload: qos rate limit offload only support one BPS action"); 238 return -EOPNOTSUPP; 239 } 240 } 241 if (action->police.rate_pkt_ps > 0) { 242 if (!pps_support) { 243 NL_SET_ERR_MSG_MOD(extack, 244 "unsupported offload: FW does not support PPS action"); 245 return -EOPNOTSUPP; 246 } 247 if (pps_num++) { 248 NL_SET_ERR_MSG_MOD(extack, 249 "unsupported offload: qos rate limit offload only support one PPS action"); 250 return -EOPNOTSUPP; 251 } 252 } 253 } 254 255 for (i = 0 ; i < action_num; i++) { 256 /* Set QoS data for this interface */ 257 action = paction + i; 258 if (action->police.rate_bytes_ps > 0) { 259 rate = action->police.rate_bytes_ps; 260 burst = action->police.burst; 261 } else if (action->police.rate_pkt_ps > 0) { 262 rate = action->police.rate_pkt_ps; 263 burst = action->police.burst_pkt; 264 } else { 265 NL_SET_ERR_MSG_MOD(extack, 266 "unsupported offload: qos rate limit is not BPS or PPS"); 267 continue; 268 } 269 270 if (rate != 0) { 271 pps = false; 272 if (action->police.rate_pkt_ps > 0) 273 pps = true; 274 nfp_flower_offload_one_police(repr->app, true, 275 pps, netdev_port_id, 276 rate, burst); 277 } 278 } 279 repr_priv->qos_table.netdev_port_id = netdev_port_id; 280 fl_priv->qos_rate_limiters++; 281 if (fl_priv->qos_rate_limiters == 1) 282 schedule_delayed_work(&fl_priv->qos_stats_work, 283 NFP_FL_QOS_UPDATE); 284 285 return 0; 286 } 287 288 static int 289 nfp_flower_remove_rate_limiter(struct nfp_app *app, struct net_device *netdev, 290 struct tc_cls_matchall_offload *flow, 291 struct netlink_ext_ack *extack) 292 { 293 struct nfp_flower_priv *fl_priv = app->priv; 294 struct nfp_flower_repr_priv *repr_priv; 295 struct nfp_police_config *config; 296 u32 netdev_port_id, i; 297 struct nfp_repr *repr; 298 struct sk_buff *skb; 299 bool pps_support; 300 301 if (!nfp_netdev_is_nfp_repr(netdev)) { 302 NL_SET_ERR_MSG_MOD(extack, "unsupported offload: qos rate limit offload not supported on higher level port"); 303 return -EOPNOTSUPP; 304 } 305 repr = netdev_priv(netdev); 306 307 netdev_port_id = nfp_repr_get_port_id(netdev); 308 repr_priv = repr->app_priv; 309 pps_support = !!(fl_priv->flower_ext_feats & NFP_FL_FEATS_QOS_PPS); 310 311 if (!repr_priv->qos_table.netdev_port_id) { 312 NL_SET_ERR_MSG_MOD(extack, "unsupported offload: cannot remove qos entry that does not exist"); 313 return -EOPNOTSUPP; 314 } 315 316 memset(&repr_priv->qos_table, 0, sizeof(struct nfp_fl_qos)); 317 fl_priv->qos_rate_limiters--; 318 if (!fl_priv->qos_rate_limiters) 319 cancel_delayed_work_sync(&fl_priv->qos_stats_work); 320 for (i = 0 ; i < NFP_FL_QOS_TYPE_MAX; i++) { 321 if (i == NFP_FL_QOS_TYPE_PPS && !pps_support) 322 break; 323 /* 0:bps 1:pps 324 * Clear QoS data for this interface. 325 * There is no need to check if a specific QOS_TYPE was 326 * configured as the firmware handles clearing a QoS entry 327 * safely, even if it wasn't explicitly added. 328 */ 329 skb = nfp_flower_cmsg_alloc(repr->app, sizeof(struct nfp_police_config), 330 NFP_FLOWER_CMSG_TYPE_QOS_DEL, GFP_KERNEL); 331 if (!skb) 332 return -ENOMEM; 333 334 config = nfp_flower_cmsg_get_data(skb); 335 memset(config, 0, sizeof(struct nfp_police_config)); 336 if (i == NFP_FL_QOS_TYPE_PPS) 337 config->head.flags_opts = cpu_to_be32(NFP_FL_QOS_PPS); 338 config->head.port = cpu_to_be32(netdev_port_id); 339 nfp_ctrl_tx(repr->app->ctrl, skb); 340 } 341 342 return 0; 343 } 344 345 void nfp_flower_stats_rlim_reply(struct nfp_app *app, struct sk_buff *skb) 346 { 347 struct nfp_flower_priv *fl_priv = app->priv; 348 struct nfp_flower_repr_priv *repr_priv; 349 struct nfp_police_stats_reply *msg; 350 struct nfp_stat_pair *curr_stats; 351 struct nfp_stat_pair *prev_stats; 352 struct net_device *netdev; 353 struct nfp_repr *repr; 354 u32 netdev_port_id; 355 356 msg = nfp_flower_cmsg_get_data(skb); 357 if (be32_to_cpu(msg->head.flags_opts) & NFP_FL_QOS_METER) 358 return nfp_act_stats_reply(app, msg); 359 360 netdev_port_id = be32_to_cpu(msg->head.port); 361 rcu_read_lock(); 362 netdev = nfp_app_dev_get(app, netdev_port_id, NULL); 363 if (!netdev) 364 goto exit_unlock_rcu; 365 366 repr = netdev_priv(netdev); 367 repr_priv = repr->app_priv; 368 curr_stats = &repr_priv->qos_table.curr_stats; 369 prev_stats = &repr_priv->qos_table.prev_stats; 370 371 spin_lock_bh(&fl_priv->qos_stats_lock); 372 curr_stats->pkts = be64_to_cpu(msg->pass_pkts) + 373 be64_to_cpu(msg->drop_pkts); 374 curr_stats->bytes = be64_to_cpu(msg->pass_bytes) + 375 be64_to_cpu(msg->drop_bytes); 376 377 if (!repr_priv->qos_table.last_update) { 378 prev_stats->pkts = curr_stats->pkts; 379 prev_stats->bytes = curr_stats->bytes; 380 } 381 382 repr_priv->qos_table.last_update = jiffies; 383 spin_unlock_bh(&fl_priv->qos_stats_lock); 384 385 exit_unlock_rcu: 386 rcu_read_unlock(); 387 } 388 389 static void 390 nfp_flower_stats_rlim_request(struct nfp_flower_priv *fl_priv, 391 u32 id, bool ingress) 392 { 393 struct nfp_police_cfg_head *head; 394 struct sk_buff *skb; 395 396 skb = nfp_flower_cmsg_alloc(fl_priv->app, 397 sizeof(struct nfp_police_cfg_head), 398 NFP_FLOWER_CMSG_TYPE_QOS_STATS, 399 GFP_ATOMIC); 400 if (!skb) 401 return; 402 head = nfp_flower_cmsg_get_data(skb); 403 404 memset(head, 0, sizeof(struct nfp_police_cfg_head)); 405 if (ingress) { 406 head->port = cpu_to_be32(id); 407 } else { 408 head->flags_opts = cpu_to_be32(NFP_FL_QOS_METER); 409 head->meter_id = cpu_to_be32(id); 410 } 411 412 nfp_ctrl_tx(fl_priv->app->ctrl, skb); 413 } 414 415 static void 416 nfp_flower_stats_rlim_request_all(struct nfp_flower_priv *fl_priv) 417 { 418 struct nfp_reprs *repr_set; 419 int i; 420 421 rcu_read_lock(); 422 repr_set = rcu_dereference(fl_priv->app->reprs[NFP_REPR_TYPE_VF]); 423 if (!repr_set) 424 goto exit_unlock_rcu; 425 426 for (i = 0; i < repr_set->num_reprs; i++) { 427 struct net_device *netdev; 428 429 netdev = rcu_dereference(repr_set->reprs[i]); 430 if (netdev) { 431 struct nfp_repr *priv = netdev_priv(netdev); 432 struct nfp_flower_repr_priv *repr_priv; 433 u32 netdev_port_id; 434 435 repr_priv = priv->app_priv; 436 netdev_port_id = repr_priv->qos_table.netdev_port_id; 437 if (!netdev_port_id) 438 continue; 439 440 nfp_flower_stats_rlim_request(fl_priv, 441 netdev_port_id, true); 442 } 443 } 444 445 exit_unlock_rcu: 446 rcu_read_unlock(); 447 } 448 449 static void update_stats_cache(struct work_struct *work) 450 { 451 struct delayed_work *delayed_work; 452 struct nfp_flower_priv *fl_priv; 453 454 delayed_work = to_delayed_work(work); 455 fl_priv = container_of(delayed_work, struct nfp_flower_priv, 456 qos_stats_work); 457 458 nfp_flower_stats_rlim_request_all(fl_priv); 459 nfp_flower_stats_meter_request_all(fl_priv); 460 461 schedule_delayed_work(&fl_priv->qos_stats_work, NFP_FL_QOS_UPDATE); 462 } 463 464 static int 465 nfp_flower_stats_rate_limiter(struct nfp_app *app, struct net_device *netdev, 466 struct tc_cls_matchall_offload *flow, 467 struct netlink_ext_ack *extack) 468 { 469 struct nfp_flower_priv *fl_priv = app->priv; 470 struct nfp_flower_repr_priv *repr_priv; 471 struct nfp_stat_pair *curr_stats; 472 struct nfp_stat_pair *prev_stats; 473 u64 diff_bytes, diff_pkts; 474 struct nfp_repr *repr; 475 476 if (!nfp_netdev_is_nfp_repr(netdev)) { 477 NL_SET_ERR_MSG_MOD(extack, "unsupported offload: qos rate limit offload not supported on higher level port"); 478 return -EOPNOTSUPP; 479 } 480 repr = netdev_priv(netdev); 481 482 repr_priv = repr->app_priv; 483 if (!repr_priv->qos_table.netdev_port_id) { 484 NL_SET_ERR_MSG_MOD(extack, "unsupported offload: cannot find qos entry for stats update"); 485 return -EOPNOTSUPP; 486 } 487 488 spin_lock_bh(&fl_priv->qos_stats_lock); 489 curr_stats = &repr_priv->qos_table.curr_stats; 490 prev_stats = &repr_priv->qos_table.prev_stats; 491 diff_pkts = curr_stats->pkts - prev_stats->pkts; 492 diff_bytes = curr_stats->bytes - prev_stats->bytes; 493 prev_stats->pkts = curr_stats->pkts; 494 prev_stats->bytes = curr_stats->bytes; 495 spin_unlock_bh(&fl_priv->qos_stats_lock); 496 497 flow_stats_update(&flow->stats, diff_bytes, diff_pkts, 0, 498 repr_priv->qos_table.last_update, 499 FLOW_ACTION_HW_STATS_DELAYED); 500 return 0; 501 } 502 503 void nfp_flower_qos_init(struct nfp_app *app) 504 { 505 struct nfp_flower_priv *fl_priv = app->priv; 506 507 spin_lock_init(&fl_priv->qos_stats_lock); 508 mutex_init(&fl_priv->meter_stats_lock); 509 nfp_init_meter_table(app); 510 511 INIT_DELAYED_WORK(&fl_priv->qos_stats_work, &update_stats_cache); 512 } 513 514 void nfp_flower_qos_cleanup(struct nfp_app *app) 515 { 516 struct nfp_flower_priv *fl_priv = app->priv; 517 518 cancel_delayed_work_sync(&fl_priv->qos_stats_work); 519 } 520 521 int nfp_flower_setup_qos_offload(struct nfp_app *app, struct net_device *netdev, 522 struct tc_cls_matchall_offload *flow) 523 { 524 struct netlink_ext_ack *extack = flow->common.extack; 525 struct nfp_flower_priv *fl_priv = app->priv; 526 int ret; 527 528 if (!(fl_priv->flower_ext_feats & NFP_FL_FEATS_VF_RLIM)) { 529 NL_SET_ERR_MSG_MOD(extack, "unsupported offload: loaded firmware does not support qos rate limit offload"); 530 return -EOPNOTSUPP; 531 } 532 533 mutex_lock(&fl_priv->nfp_fl_lock); 534 switch (flow->command) { 535 case TC_CLSMATCHALL_REPLACE: 536 ret = nfp_flower_install_rate_limiter(app, netdev, flow, extack); 537 break; 538 case TC_CLSMATCHALL_DESTROY: 539 ret = nfp_flower_remove_rate_limiter(app, netdev, flow, extack); 540 break; 541 case TC_CLSMATCHALL_STATS: 542 ret = nfp_flower_stats_rate_limiter(app, netdev, flow, extack); 543 break; 544 default: 545 ret = -EOPNOTSUPP; 546 break; 547 } 548 mutex_unlock(&fl_priv->nfp_fl_lock); 549 550 return ret; 551 } 552 553 /* Offload tc action, currently only for tc police */ 554 555 static const struct rhashtable_params stats_meter_table_params = { 556 .key_offset = offsetof(struct nfp_meter_entry, meter_id), 557 .head_offset = offsetof(struct nfp_meter_entry, ht_node), 558 .key_len = sizeof(u32), 559 }; 560 561 struct nfp_meter_entry * 562 nfp_flower_search_meter_entry(struct nfp_app *app, u32 meter_id) 563 { 564 struct nfp_flower_priv *priv = app->priv; 565 566 return rhashtable_lookup_fast(&priv->meter_table, &meter_id, 567 stats_meter_table_params); 568 } 569 570 static struct nfp_meter_entry * 571 nfp_flower_add_meter_entry(struct nfp_app *app, u32 meter_id) 572 { 573 struct nfp_meter_entry *meter_entry = NULL; 574 struct nfp_flower_priv *priv = app->priv; 575 576 meter_entry = rhashtable_lookup_fast(&priv->meter_table, 577 &meter_id, 578 stats_meter_table_params); 579 if (meter_entry) 580 return meter_entry; 581 582 meter_entry = kzalloc(sizeof(*meter_entry), GFP_KERNEL); 583 if (!meter_entry) 584 return NULL; 585 586 meter_entry->meter_id = meter_id; 587 meter_entry->used = jiffies; 588 if (rhashtable_insert_fast(&priv->meter_table, &meter_entry->ht_node, 589 stats_meter_table_params)) { 590 kfree(meter_entry); 591 return NULL; 592 } 593 594 priv->qos_rate_limiters++; 595 if (priv->qos_rate_limiters == 1) 596 schedule_delayed_work(&priv->qos_stats_work, 597 NFP_FL_QOS_UPDATE); 598 599 return meter_entry; 600 } 601 602 static void nfp_flower_del_meter_entry(struct nfp_app *app, u32 meter_id) 603 { 604 struct nfp_meter_entry *meter_entry = NULL; 605 struct nfp_flower_priv *priv = app->priv; 606 607 meter_entry = rhashtable_lookup_fast(&priv->meter_table, &meter_id, 608 stats_meter_table_params); 609 if (!meter_entry) 610 return; 611 612 rhashtable_remove_fast(&priv->meter_table, 613 &meter_entry->ht_node, 614 stats_meter_table_params); 615 kfree(meter_entry); 616 priv->qos_rate_limiters--; 617 if (!priv->qos_rate_limiters) 618 cancel_delayed_work_sync(&priv->qos_stats_work); 619 } 620 621 int nfp_flower_setup_meter_entry(struct nfp_app *app, 622 const struct flow_action_entry *action, 623 enum nfp_meter_op op, 624 u32 meter_id) 625 { 626 struct nfp_flower_priv *fl_priv = app->priv; 627 struct nfp_meter_entry *meter_entry = NULL; 628 int err = 0; 629 630 mutex_lock(&fl_priv->meter_stats_lock); 631 632 switch (op) { 633 case NFP_METER_DEL: 634 nfp_flower_del_meter_entry(app, meter_id); 635 goto exit_unlock; 636 case NFP_METER_ADD: 637 meter_entry = nfp_flower_add_meter_entry(app, meter_id); 638 break; 639 default: 640 err = -EOPNOTSUPP; 641 goto exit_unlock; 642 } 643 644 if (!meter_entry) { 645 err = -ENOMEM; 646 goto exit_unlock; 647 } 648 649 if (action->police.rate_bytes_ps > 0) { 650 meter_entry->bps = true; 651 meter_entry->rate = action->police.rate_bytes_ps; 652 meter_entry->burst = action->police.burst; 653 } else { 654 meter_entry->bps = false; 655 meter_entry->rate = action->police.rate_pkt_ps; 656 meter_entry->burst = action->police.burst_pkt; 657 } 658 659 exit_unlock: 660 mutex_unlock(&fl_priv->meter_stats_lock); 661 return err; 662 } 663 664 int nfp_init_meter_table(struct nfp_app *app) 665 { 666 struct nfp_flower_priv *priv = app->priv; 667 668 return rhashtable_init(&priv->meter_table, &stats_meter_table_params); 669 } 670 671 void 672 nfp_flower_stats_meter_request_all(struct nfp_flower_priv *fl_priv) 673 { 674 struct nfp_meter_entry *meter_entry = NULL; 675 struct rhashtable_iter iter; 676 677 mutex_lock(&fl_priv->meter_stats_lock); 678 rhashtable_walk_enter(&fl_priv->meter_table, &iter); 679 rhashtable_walk_start(&iter); 680 681 while ((meter_entry = rhashtable_walk_next(&iter)) != NULL) { 682 if (IS_ERR(meter_entry)) 683 continue; 684 nfp_flower_stats_rlim_request(fl_priv, 685 meter_entry->meter_id, false); 686 } 687 688 rhashtable_walk_stop(&iter); 689 rhashtable_walk_exit(&iter); 690 mutex_unlock(&fl_priv->meter_stats_lock); 691 } 692 693 static int 694 nfp_act_install_actions(struct nfp_app *app, struct flow_offload_action *fl_act, 695 struct netlink_ext_ack *extack) 696 { 697 struct flow_action_entry *paction = &fl_act->action.entries[0]; 698 u32 action_num = fl_act->action.num_entries; 699 struct nfp_flower_priv *fl_priv = app->priv; 700 struct flow_action_entry *action = NULL; 701 u32 burst, i, meter_id; 702 bool pps_support, pps; 703 bool add = false; 704 u64 rate; 705 int err; 706 707 pps_support = !!(fl_priv->flower_ext_feats & NFP_FL_FEATS_QOS_PPS); 708 709 for (i = 0 ; i < action_num; i++) { 710 /* Set qos associate data for this interface */ 711 action = paction + i; 712 if (action->id != FLOW_ACTION_POLICE) { 713 NL_SET_ERR_MSG_MOD(extack, 714 "unsupported offload: qos rate limit offload requires police action"); 715 continue; 716 } 717 718 err = nfp_policer_validate(&fl_act->action, action, extack, false); 719 if (err) 720 return err; 721 722 if (action->police.rate_bytes_ps > 0) { 723 rate = action->police.rate_bytes_ps; 724 burst = action->police.burst; 725 } else if (action->police.rate_pkt_ps > 0 && pps_support) { 726 rate = action->police.rate_pkt_ps; 727 burst = action->police.burst_pkt; 728 } else { 729 NL_SET_ERR_MSG_MOD(extack, 730 "unsupported offload: unsupported qos rate limit"); 731 continue; 732 } 733 734 if (rate != 0) { 735 meter_id = action->hw_index; 736 if (nfp_flower_setup_meter_entry(app, action, NFP_METER_ADD, meter_id)) 737 continue; 738 739 pps = false; 740 if (action->police.rate_pkt_ps > 0) 741 pps = true; 742 nfp_flower_offload_one_police(app, false, pps, meter_id, 743 rate, burst); 744 add = true; 745 } 746 } 747 748 return add ? 0 : -EOPNOTSUPP; 749 } 750 751 static int 752 nfp_act_remove_actions(struct nfp_app *app, struct flow_offload_action *fl_act, 753 struct netlink_ext_ack *extack) 754 { 755 struct nfp_meter_entry *meter_entry = NULL; 756 struct nfp_police_config *config; 757 struct sk_buff *skb; 758 u32 meter_id; 759 bool pps; 760 761 /* Delete qos associate data for this interface */ 762 if (fl_act->id != FLOW_ACTION_POLICE) { 763 NL_SET_ERR_MSG_MOD(extack, 764 "unsupported offload: qos rate limit offload requires police action"); 765 return -EOPNOTSUPP; 766 } 767 768 meter_id = fl_act->index; 769 meter_entry = nfp_flower_search_meter_entry(app, meter_id); 770 if (!meter_entry) { 771 NL_SET_ERR_MSG_MOD(extack, 772 "no meter entry when delete the action index."); 773 return -ENOENT; 774 } 775 pps = !meter_entry->bps; 776 777 skb = nfp_flower_cmsg_alloc(app, sizeof(struct nfp_police_config), 778 NFP_FLOWER_CMSG_TYPE_QOS_DEL, GFP_KERNEL); 779 if (!skb) 780 return -ENOMEM; 781 782 config = nfp_flower_cmsg_get_data(skb); 783 memset(config, 0, sizeof(struct nfp_police_config)); 784 config->head.flags_opts = cpu_to_be32(NFP_FL_QOS_METER); 785 config->head.meter_id = cpu_to_be32(meter_id); 786 if (pps) 787 config->head.flags_opts |= cpu_to_be32(NFP_FL_QOS_PPS); 788 789 nfp_ctrl_tx(app->ctrl, skb); 790 nfp_flower_setup_meter_entry(app, NULL, NFP_METER_DEL, meter_id); 791 792 return 0; 793 } 794 795 void 796 nfp_act_stats_reply(struct nfp_app *app, void *pmsg) 797 { 798 struct nfp_flower_priv *fl_priv = app->priv; 799 struct nfp_meter_entry *meter_entry = NULL; 800 struct nfp_police_stats_reply *msg = pmsg; 801 u32 meter_id; 802 803 meter_id = be32_to_cpu(msg->head.meter_id); 804 mutex_lock(&fl_priv->meter_stats_lock); 805 806 meter_entry = nfp_flower_search_meter_entry(app, meter_id); 807 if (!meter_entry) 808 goto exit_unlock; 809 810 meter_entry->stats.curr.pkts = be64_to_cpu(msg->pass_pkts) + 811 be64_to_cpu(msg->drop_pkts); 812 meter_entry->stats.curr.bytes = be64_to_cpu(msg->pass_bytes) + 813 be64_to_cpu(msg->drop_bytes); 814 meter_entry->stats.curr.drops = be64_to_cpu(msg->drop_pkts); 815 if (!meter_entry->stats.update) { 816 meter_entry->stats.prev.pkts = meter_entry->stats.curr.pkts; 817 meter_entry->stats.prev.bytes = meter_entry->stats.curr.bytes; 818 meter_entry->stats.prev.drops = meter_entry->stats.curr.drops; 819 } 820 821 meter_entry->stats.update = jiffies; 822 823 exit_unlock: 824 mutex_unlock(&fl_priv->meter_stats_lock); 825 } 826 827 static int 828 nfp_act_stats_actions(struct nfp_app *app, struct flow_offload_action *fl_act, 829 struct netlink_ext_ack *extack) 830 { 831 struct nfp_flower_priv *fl_priv = app->priv; 832 struct nfp_meter_entry *meter_entry = NULL; 833 u64 diff_bytes, diff_pkts, diff_drops; 834 int err = 0; 835 836 if (fl_act->id != FLOW_ACTION_POLICE) { 837 NL_SET_ERR_MSG_MOD(extack, 838 "unsupported offload: qos rate limit offload requires police action"); 839 return -EOPNOTSUPP; 840 } 841 842 mutex_lock(&fl_priv->meter_stats_lock); 843 meter_entry = nfp_flower_search_meter_entry(app, fl_act->index); 844 if (!meter_entry) { 845 err = -ENOENT; 846 goto exit_unlock; 847 } 848 diff_pkts = meter_entry->stats.curr.pkts > meter_entry->stats.prev.pkts ? 849 meter_entry->stats.curr.pkts - meter_entry->stats.prev.pkts : 0; 850 diff_bytes = meter_entry->stats.curr.bytes > meter_entry->stats.prev.bytes ? 851 meter_entry->stats.curr.bytes - meter_entry->stats.prev.bytes : 0; 852 diff_drops = meter_entry->stats.curr.drops > meter_entry->stats.prev.drops ? 853 meter_entry->stats.curr.drops - meter_entry->stats.prev.drops : 0; 854 855 flow_stats_update(&fl_act->stats, diff_bytes, diff_pkts, diff_drops, 856 meter_entry->stats.update, 857 FLOW_ACTION_HW_STATS_DELAYED); 858 859 meter_entry->stats.prev.pkts = meter_entry->stats.curr.pkts; 860 meter_entry->stats.prev.bytes = meter_entry->stats.curr.bytes; 861 meter_entry->stats.prev.drops = meter_entry->stats.curr.drops; 862 863 exit_unlock: 864 mutex_unlock(&fl_priv->meter_stats_lock); 865 return err; 866 } 867 868 int nfp_setup_tc_act_offload(struct nfp_app *app, 869 struct flow_offload_action *fl_act) 870 { 871 struct netlink_ext_ack *extack = fl_act->extack; 872 struct nfp_flower_priv *fl_priv = app->priv; 873 874 if (!(fl_priv->flower_ext_feats & NFP_FL_FEATS_QOS_METER)) 875 return -EOPNOTSUPP; 876 877 switch (fl_act->command) { 878 case FLOW_ACT_REPLACE: 879 return nfp_act_install_actions(app, fl_act, extack); 880 case FLOW_ACT_DESTROY: 881 return nfp_act_remove_actions(app, fl_act, extack); 882 case FLOW_ACT_STATS: 883 return nfp_act_stats_actions(app, fl_act, extack); 884 default: 885 return -EOPNOTSUPP; 886 } 887 } 888