1 /* Broadcom NetXtreme-C/E network driver. 2 * 3 * Copyright (c) 2017 Broadcom Limited 4 * 5 * This program is free software; you can redistribute it and/or modify 6 * it under the terms of the GNU General Public License as published by 7 * the Free Software Foundation. 8 */ 9 10 #include <linux/netdevice.h> 11 #include <linux/inetdevice.h> 12 #include <linux/if_vlan.h> 13 #include <net/flow_dissector.h> 14 #include <net/pkt_cls.h> 15 #include <net/tc_act/tc_gact.h> 16 #include <net/tc_act/tc_skbedit.h> 17 #include <net/tc_act/tc_mirred.h> 18 #include <net/tc_act/tc_vlan.h> 19 20 #include "bnxt_hsi.h" 21 #include "bnxt.h" 22 #include "bnxt_sriov.h" 23 #include "bnxt_tc.h" 24 #include "bnxt_vfr.h" 25 26 #define BNXT_FID_INVALID 0xffff 27 #define VLAN_TCI(vid, prio) ((vid) | ((prio) << VLAN_PRIO_SHIFT)) 28 29 /* Return the dst fid of the func for flow forwarding 30 * For PFs: src_fid is the fid of the PF 31 * For VF-reps: src_fid the fid of the VF 32 */ 33 static u16 bnxt_flow_get_dst_fid(struct bnxt *pf_bp, struct net_device *dev) 34 { 35 struct bnxt *bp; 36 37 /* check if dev belongs to the same switch */ 38 if (!switchdev_port_same_parent_id(pf_bp->dev, dev)) { 39 netdev_info(pf_bp->dev, "dev(ifindex=%d) not on same switch", 40 dev->ifindex); 41 return BNXT_FID_INVALID; 42 } 43 44 /* Is dev a VF-rep? */ 45 if (dev != pf_bp->dev) 46 return bnxt_vf_rep_get_fid(dev); 47 48 bp = netdev_priv(dev); 49 return bp->pf.fw_fid; 50 } 51 52 static int bnxt_tc_parse_redir(struct bnxt *bp, 53 struct bnxt_tc_actions *actions, 54 const struct tc_action *tc_act) 55 { 56 int ifindex = tcf_mirred_ifindex(tc_act); 57 struct net_device *dev; 58 u16 dst_fid; 59 60 dev = __dev_get_by_index(dev_net(bp->dev), ifindex); 61 if (!dev) { 62 netdev_info(bp->dev, "no dev for ifindex=%d", ifindex); 63 return -EINVAL; 64 } 65 66 /* find the FID from dev */ 67 dst_fid = bnxt_flow_get_dst_fid(bp, dev); 68 if (dst_fid == BNXT_FID_INVALID) { 69 netdev_info(bp->dev, "can't get fid for ifindex=%d", ifindex); 70 return -EINVAL; 71 } 72 73 actions->flags |= BNXT_TC_ACTION_FLAG_FWD; 74 actions->dst_fid = dst_fid; 75 actions->dst_dev = dev; 76 return 0; 77 } 78 79 static void bnxt_tc_parse_vlan(struct bnxt *bp, 80 struct bnxt_tc_actions *actions, 81 const struct tc_action *tc_act) 82 { 83 if (tcf_vlan_action(tc_act) == TCA_VLAN_ACT_POP) { 84 actions->flags |= BNXT_TC_ACTION_FLAG_POP_VLAN; 85 } else if (tcf_vlan_action(tc_act) == TCA_VLAN_ACT_PUSH) { 86 actions->flags |= BNXT_TC_ACTION_FLAG_PUSH_VLAN; 87 actions->push_vlan_tci = htons(tcf_vlan_push_vid(tc_act)); 88 actions->push_vlan_tpid = tcf_vlan_push_proto(tc_act); 89 } 90 } 91 92 static int bnxt_tc_parse_actions(struct bnxt *bp, 93 struct bnxt_tc_actions *actions, 94 struct tcf_exts *tc_exts) 95 { 96 const struct tc_action *tc_act; 97 LIST_HEAD(tc_actions); 98 int rc; 99 100 if (!tcf_exts_has_actions(tc_exts)) { 101 netdev_info(bp->dev, "no actions"); 102 return -EINVAL; 103 } 104 105 tcf_exts_to_list(tc_exts, &tc_actions); 106 list_for_each_entry(tc_act, &tc_actions, list) { 107 /* Drop action */ 108 if (is_tcf_gact_shot(tc_act)) { 109 actions->flags |= BNXT_TC_ACTION_FLAG_DROP; 110 return 0; /* don't bother with other actions */ 111 } 112 113 /* Redirect action */ 114 if (is_tcf_mirred_egress_redirect(tc_act)) { 115 rc = bnxt_tc_parse_redir(bp, actions, tc_act); 116 if (rc) 117 return rc; 118 continue; 119 } 120 121 /* Push/pop VLAN */ 122 if (is_tcf_vlan(tc_act)) { 123 bnxt_tc_parse_vlan(bp, actions, tc_act); 124 continue; 125 } 126 } 127 128 return 0; 129 } 130 131 #define GET_KEY(flow_cmd, key_type) \ 132 skb_flow_dissector_target((flow_cmd)->dissector, key_type,\ 133 (flow_cmd)->key) 134 #define GET_MASK(flow_cmd, key_type) \ 135 skb_flow_dissector_target((flow_cmd)->dissector, key_type,\ 136 (flow_cmd)->mask) 137 138 static int bnxt_tc_parse_flow(struct bnxt *bp, 139 struct tc_cls_flower_offload *tc_flow_cmd, 140 struct bnxt_tc_flow *flow) 141 { 142 struct flow_dissector *dissector = tc_flow_cmd->dissector; 143 u16 addr_type = 0; 144 145 /* KEY_CONTROL and KEY_BASIC are needed for forming a meaningful key */ 146 if ((dissector->used_keys & BIT(FLOW_DISSECTOR_KEY_CONTROL)) == 0 || 147 (dissector->used_keys & BIT(FLOW_DISSECTOR_KEY_BASIC)) == 0) { 148 netdev_info(bp->dev, "cannot form TC key: used_keys = 0x%x", 149 dissector->used_keys); 150 return -EOPNOTSUPP; 151 } 152 153 if (dissector_uses_key(dissector, FLOW_DISSECTOR_KEY_CONTROL)) { 154 struct flow_dissector_key_control *key = 155 GET_KEY(tc_flow_cmd, FLOW_DISSECTOR_KEY_CONTROL); 156 157 addr_type = key->addr_type; 158 } 159 160 if (dissector_uses_key(dissector, FLOW_DISSECTOR_KEY_BASIC)) { 161 struct flow_dissector_key_basic *key = 162 GET_KEY(tc_flow_cmd, FLOW_DISSECTOR_KEY_BASIC); 163 struct flow_dissector_key_basic *mask = 164 GET_MASK(tc_flow_cmd, FLOW_DISSECTOR_KEY_BASIC); 165 166 flow->l2_key.ether_type = key->n_proto; 167 flow->l2_mask.ether_type = mask->n_proto; 168 169 if (key->n_proto == htons(ETH_P_IP) || 170 key->n_proto == htons(ETH_P_IPV6)) { 171 flow->l4_key.ip_proto = key->ip_proto; 172 flow->l4_mask.ip_proto = mask->ip_proto; 173 } 174 } 175 176 if (dissector_uses_key(dissector, FLOW_DISSECTOR_KEY_ETH_ADDRS)) { 177 struct flow_dissector_key_eth_addrs *key = 178 GET_KEY(tc_flow_cmd, FLOW_DISSECTOR_KEY_ETH_ADDRS); 179 struct flow_dissector_key_eth_addrs *mask = 180 GET_MASK(tc_flow_cmd, FLOW_DISSECTOR_KEY_ETH_ADDRS); 181 182 flow->flags |= BNXT_TC_FLOW_FLAGS_ETH_ADDRS; 183 ether_addr_copy(flow->l2_key.dmac, key->dst); 184 ether_addr_copy(flow->l2_mask.dmac, mask->dst); 185 ether_addr_copy(flow->l2_key.smac, key->src); 186 ether_addr_copy(flow->l2_mask.smac, mask->src); 187 } 188 189 if (dissector_uses_key(dissector, FLOW_DISSECTOR_KEY_VLAN)) { 190 struct flow_dissector_key_vlan *key = 191 GET_KEY(tc_flow_cmd, FLOW_DISSECTOR_KEY_VLAN); 192 struct flow_dissector_key_vlan *mask = 193 GET_MASK(tc_flow_cmd, FLOW_DISSECTOR_KEY_VLAN); 194 195 flow->l2_key.inner_vlan_tci = 196 cpu_to_be16(VLAN_TCI(key->vlan_id, key->vlan_priority)); 197 flow->l2_mask.inner_vlan_tci = 198 cpu_to_be16((VLAN_TCI(mask->vlan_id, mask->vlan_priority))); 199 flow->l2_key.inner_vlan_tpid = htons(ETH_P_8021Q); 200 flow->l2_mask.inner_vlan_tpid = htons(0xffff); 201 flow->l2_key.num_vlans = 1; 202 } 203 204 if (dissector_uses_key(dissector, FLOW_DISSECTOR_KEY_IPV4_ADDRS)) { 205 struct flow_dissector_key_ipv4_addrs *key = 206 GET_KEY(tc_flow_cmd, FLOW_DISSECTOR_KEY_IPV4_ADDRS); 207 struct flow_dissector_key_ipv4_addrs *mask = 208 GET_MASK(tc_flow_cmd, FLOW_DISSECTOR_KEY_IPV4_ADDRS); 209 210 flow->flags |= BNXT_TC_FLOW_FLAGS_IPV4_ADDRS; 211 flow->l3_key.ipv4.daddr.s_addr = key->dst; 212 flow->l3_mask.ipv4.daddr.s_addr = mask->dst; 213 flow->l3_key.ipv4.saddr.s_addr = key->src; 214 flow->l3_mask.ipv4.saddr.s_addr = mask->src; 215 } else if (dissector_uses_key(dissector, 216 FLOW_DISSECTOR_KEY_IPV6_ADDRS)) { 217 struct flow_dissector_key_ipv6_addrs *key = 218 GET_KEY(tc_flow_cmd, FLOW_DISSECTOR_KEY_IPV6_ADDRS); 219 struct flow_dissector_key_ipv6_addrs *mask = 220 GET_MASK(tc_flow_cmd, FLOW_DISSECTOR_KEY_IPV6_ADDRS); 221 222 flow->flags |= BNXT_TC_FLOW_FLAGS_IPV6_ADDRS; 223 flow->l3_key.ipv6.daddr = key->dst; 224 flow->l3_mask.ipv6.daddr = mask->dst; 225 flow->l3_key.ipv6.saddr = key->src; 226 flow->l3_mask.ipv6.saddr = mask->src; 227 } 228 229 if (dissector_uses_key(dissector, FLOW_DISSECTOR_KEY_PORTS)) { 230 struct flow_dissector_key_ports *key = 231 GET_KEY(tc_flow_cmd, FLOW_DISSECTOR_KEY_PORTS); 232 struct flow_dissector_key_ports *mask = 233 GET_MASK(tc_flow_cmd, FLOW_DISSECTOR_KEY_PORTS); 234 235 flow->flags |= BNXT_TC_FLOW_FLAGS_PORTS; 236 flow->l4_key.ports.dport = key->dst; 237 flow->l4_mask.ports.dport = mask->dst; 238 flow->l4_key.ports.sport = key->src; 239 flow->l4_mask.ports.sport = mask->src; 240 } 241 242 if (dissector_uses_key(dissector, FLOW_DISSECTOR_KEY_ICMP)) { 243 struct flow_dissector_key_icmp *key = 244 GET_KEY(tc_flow_cmd, FLOW_DISSECTOR_KEY_ICMP); 245 struct flow_dissector_key_icmp *mask = 246 GET_MASK(tc_flow_cmd, FLOW_DISSECTOR_KEY_ICMP); 247 248 flow->flags |= BNXT_TC_FLOW_FLAGS_ICMP; 249 flow->l4_key.icmp.type = key->type; 250 flow->l4_key.icmp.code = key->code; 251 flow->l4_mask.icmp.type = mask->type; 252 flow->l4_mask.icmp.code = mask->code; 253 } 254 255 return bnxt_tc_parse_actions(bp, &flow->actions, tc_flow_cmd->exts); 256 } 257 258 static int bnxt_hwrm_cfa_flow_free(struct bnxt *bp, __le16 flow_handle) 259 { 260 struct hwrm_cfa_flow_free_input req = { 0 }; 261 int rc; 262 263 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_FLOW_FREE, -1, -1); 264 req.flow_handle = flow_handle; 265 266 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 267 if (rc) 268 netdev_info(bp->dev, "Error: %s: flow_handle=0x%x rc=%d", 269 __func__, flow_handle, rc); 270 return rc; 271 } 272 273 static int ipv6_mask_len(struct in6_addr *mask) 274 { 275 int mask_len = 0, i; 276 277 for (i = 0; i < 4; i++) 278 mask_len += inet_mask_len(mask->s6_addr32[i]); 279 280 return mask_len; 281 } 282 283 static bool is_wildcard(void *mask, int len) 284 { 285 const u8 *p = mask; 286 int i; 287 288 for (i = 0; i < len; i++) { 289 if (p[i] != 0) 290 return false; 291 } 292 return true; 293 } 294 295 static int bnxt_hwrm_cfa_flow_alloc(struct bnxt *bp, struct bnxt_tc_flow *flow, 296 __le16 ref_flow_handle, __le16 *flow_handle) 297 { 298 struct hwrm_cfa_flow_alloc_output *resp = bp->hwrm_cmd_resp_addr; 299 struct bnxt_tc_actions *actions = &flow->actions; 300 struct bnxt_tc_l3_key *l3_mask = &flow->l3_mask; 301 struct bnxt_tc_l3_key *l3_key = &flow->l3_key; 302 struct hwrm_cfa_flow_alloc_input req = { 0 }; 303 u16 flow_flags = 0, action_flags = 0; 304 int rc; 305 306 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_FLOW_ALLOC, -1, -1); 307 308 req.src_fid = cpu_to_le16(flow->src_fid); 309 req.ref_flow_handle = ref_flow_handle; 310 req.ethertype = flow->l2_key.ether_type; 311 req.ip_proto = flow->l4_key.ip_proto; 312 313 if (flow->flags & BNXT_TC_FLOW_FLAGS_ETH_ADDRS) { 314 memcpy(req.dmac, flow->l2_key.dmac, ETH_ALEN); 315 memcpy(req.smac, flow->l2_key.smac, ETH_ALEN); 316 } 317 318 if (flow->l2_key.num_vlans > 0) { 319 flow_flags |= CFA_FLOW_ALLOC_REQ_FLAGS_NUM_VLAN_ONE; 320 /* FW expects the inner_vlan_tci value to be set 321 * in outer_vlan_tci when num_vlans is 1 (which is 322 * always the case in TC.) 323 */ 324 req.outer_vlan_tci = flow->l2_key.inner_vlan_tci; 325 } 326 327 /* If all IP and L4 fields are wildcarded then this is an L2 flow */ 328 if (is_wildcard(&l3_mask, sizeof(l3_mask)) && 329 is_wildcard(&flow->l4_mask, sizeof(flow->l4_mask))) { 330 flow_flags |= CFA_FLOW_ALLOC_REQ_FLAGS_FLOWTYPE_L2; 331 } else { 332 flow_flags |= flow->l2_key.ether_type == htons(ETH_P_IP) ? 333 CFA_FLOW_ALLOC_REQ_FLAGS_FLOWTYPE_IPV4 : 334 CFA_FLOW_ALLOC_REQ_FLAGS_FLOWTYPE_IPV6; 335 336 if (flow->flags & BNXT_TC_FLOW_FLAGS_IPV4_ADDRS) { 337 req.ip_dst[0] = l3_key->ipv4.daddr.s_addr; 338 req.ip_dst_mask_len = 339 inet_mask_len(l3_mask->ipv4.daddr.s_addr); 340 req.ip_src[0] = l3_key->ipv4.saddr.s_addr; 341 req.ip_src_mask_len = 342 inet_mask_len(l3_mask->ipv4.saddr.s_addr); 343 } else if (flow->flags & BNXT_TC_FLOW_FLAGS_IPV6_ADDRS) { 344 memcpy(req.ip_dst, l3_key->ipv6.daddr.s6_addr32, 345 sizeof(req.ip_dst)); 346 req.ip_dst_mask_len = 347 ipv6_mask_len(&l3_mask->ipv6.daddr); 348 memcpy(req.ip_src, l3_key->ipv6.saddr.s6_addr32, 349 sizeof(req.ip_src)); 350 req.ip_src_mask_len = 351 ipv6_mask_len(&l3_mask->ipv6.saddr); 352 } 353 } 354 355 if (flow->flags & BNXT_TC_FLOW_FLAGS_PORTS) { 356 req.l4_src_port = flow->l4_key.ports.sport; 357 req.l4_src_port_mask = flow->l4_mask.ports.sport; 358 req.l4_dst_port = flow->l4_key.ports.dport; 359 req.l4_dst_port_mask = flow->l4_mask.ports.dport; 360 } else if (flow->flags & BNXT_TC_FLOW_FLAGS_ICMP) { 361 /* l4 ports serve as type/code when ip_proto is ICMP */ 362 req.l4_src_port = htons(flow->l4_key.icmp.type); 363 req.l4_src_port_mask = htons(flow->l4_mask.icmp.type); 364 req.l4_dst_port = htons(flow->l4_key.icmp.code); 365 req.l4_dst_port_mask = htons(flow->l4_mask.icmp.code); 366 } 367 req.flags = cpu_to_le16(flow_flags); 368 369 if (actions->flags & BNXT_TC_ACTION_FLAG_DROP) { 370 action_flags |= CFA_FLOW_ALLOC_REQ_ACTION_FLAGS_DROP; 371 } else { 372 if (actions->flags & BNXT_TC_ACTION_FLAG_FWD) { 373 action_flags |= CFA_FLOW_ALLOC_REQ_ACTION_FLAGS_FWD; 374 req.dst_fid = cpu_to_le16(actions->dst_fid); 375 } 376 if (actions->flags & BNXT_TC_ACTION_FLAG_PUSH_VLAN) { 377 action_flags |= 378 CFA_FLOW_ALLOC_REQ_ACTION_FLAGS_L2_HEADER_REWRITE; 379 req.l2_rewrite_vlan_tpid = actions->push_vlan_tpid; 380 req.l2_rewrite_vlan_tci = actions->push_vlan_tci; 381 memcpy(&req.l2_rewrite_dmac, &req.dmac, ETH_ALEN); 382 memcpy(&req.l2_rewrite_smac, &req.smac, ETH_ALEN); 383 } 384 if (actions->flags & BNXT_TC_ACTION_FLAG_POP_VLAN) { 385 action_flags |= 386 CFA_FLOW_ALLOC_REQ_ACTION_FLAGS_L2_HEADER_REWRITE; 387 /* Rewrite config with tpid = 0 implies vlan pop */ 388 req.l2_rewrite_vlan_tpid = 0; 389 memcpy(&req.l2_rewrite_dmac, &req.dmac, ETH_ALEN); 390 memcpy(&req.l2_rewrite_smac, &req.smac, ETH_ALEN); 391 } 392 } 393 req.action_flags = cpu_to_le16(action_flags); 394 395 mutex_lock(&bp->hwrm_cmd_lock); 396 397 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 398 if (!rc) 399 *flow_handle = resp->flow_handle; 400 401 mutex_unlock(&bp->hwrm_cmd_lock); 402 403 return rc; 404 } 405 406 /* Add val to accum while handling a possible wraparound 407 * of val. Eventhough val is of type u64, its actual width 408 * is denoted by mask and will wrap-around beyond that width. 409 */ 410 static void accumulate_val(u64 *accum, u64 val, u64 mask) 411 { 412 #define low_bits(x, mask) ((x) & (mask)) 413 #define high_bits(x, mask) ((x) & ~(mask)) 414 bool wrapped = val < low_bits(*accum, mask); 415 416 *accum = high_bits(*accum, mask) + val; 417 if (wrapped) 418 *accum += (mask + 1); 419 } 420 421 /* The HW counters' width is much less than 64bits. 422 * Handle possible wrap-around while updating the stat counters 423 */ 424 static void bnxt_flow_stats_fix_wraparound(struct bnxt_tc_info *tc_info, 425 struct bnxt_tc_flow_stats *stats, 426 struct bnxt_tc_flow_stats *hw_stats) 427 { 428 accumulate_val(&stats->bytes, hw_stats->bytes, tc_info->bytes_mask); 429 accumulate_val(&stats->packets, hw_stats->packets, 430 tc_info->packets_mask); 431 } 432 433 /* Fix possible wraparound of the stats queried from HW, calculate 434 * the delta from prev_stats, and also update the prev_stats. 435 * The HW flow stats are fetched under the hwrm_cmd_lock mutex. 436 * This routine is best called while under the mutex so that the 437 * stats processing happens atomically. 438 */ 439 static void bnxt_flow_stats_calc(struct bnxt_tc_info *tc_info, 440 struct bnxt_tc_flow *flow, 441 struct bnxt_tc_flow_stats *stats) 442 { 443 struct bnxt_tc_flow_stats *acc_stats, *prev_stats; 444 445 acc_stats = &flow->stats; 446 bnxt_flow_stats_fix_wraparound(tc_info, acc_stats, stats); 447 448 prev_stats = &flow->prev_stats; 449 stats->bytes = acc_stats->bytes - prev_stats->bytes; 450 stats->packets = acc_stats->packets - prev_stats->packets; 451 *prev_stats = *acc_stats; 452 } 453 454 static int bnxt_hwrm_cfa_flow_stats_get(struct bnxt *bp, 455 __le16 flow_handle, 456 struct bnxt_tc_flow *flow, 457 struct bnxt_tc_flow_stats *stats) 458 { 459 struct hwrm_cfa_flow_stats_output *resp = bp->hwrm_cmd_resp_addr; 460 struct hwrm_cfa_flow_stats_input req = { 0 }; 461 int rc; 462 463 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_FLOW_STATS, -1, -1); 464 req.num_flows = cpu_to_le16(1); 465 req.flow_handle_0 = flow_handle; 466 467 mutex_lock(&bp->hwrm_cmd_lock); 468 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 469 if (!rc) { 470 stats->packets = le64_to_cpu(resp->packet_0); 471 stats->bytes = le64_to_cpu(resp->byte_0); 472 bnxt_flow_stats_calc(&bp->tc_info, flow, stats); 473 } else { 474 netdev_info(bp->dev, "error rc=%d", rc); 475 } 476 477 mutex_unlock(&bp->hwrm_cmd_lock); 478 return rc; 479 } 480 481 static int bnxt_tc_put_l2_node(struct bnxt *bp, 482 struct bnxt_tc_flow_node *flow_node) 483 { 484 struct bnxt_tc_l2_node *l2_node = flow_node->l2_node; 485 struct bnxt_tc_info *tc_info = &bp->tc_info; 486 int rc; 487 488 /* remove flow_node from the L2 shared flow list */ 489 list_del(&flow_node->l2_list_node); 490 if (--l2_node->refcount == 0) { 491 rc = rhashtable_remove_fast(&tc_info->l2_table, &l2_node->node, 492 tc_info->l2_ht_params); 493 if (rc) 494 netdev_err(bp->dev, 495 "Error: %s: rhashtable_remove_fast: %d", 496 __func__, rc); 497 kfree_rcu(l2_node, rcu); 498 } 499 return 0; 500 } 501 502 static struct bnxt_tc_l2_node * 503 bnxt_tc_get_l2_node(struct bnxt *bp, struct rhashtable *l2_table, 504 struct rhashtable_params ht_params, 505 struct bnxt_tc_l2_key *l2_key) 506 { 507 struct bnxt_tc_l2_node *l2_node; 508 int rc; 509 510 l2_node = rhashtable_lookup_fast(l2_table, l2_key, ht_params); 511 if (!l2_node) { 512 l2_node = kzalloc(sizeof(*l2_node), GFP_KERNEL); 513 if (!l2_node) { 514 rc = -ENOMEM; 515 return NULL; 516 } 517 518 l2_node->key = *l2_key; 519 rc = rhashtable_insert_fast(l2_table, &l2_node->node, 520 ht_params); 521 if (rc) { 522 kfree(l2_node); 523 netdev_err(bp->dev, 524 "Error: %s: rhashtable_insert_fast: %d", 525 __func__, rc); 526 return NULL; 527 } 528 INIT_LIST_HEAD(&l2_node->common_l2_flows); 529 } 530 return l2_node; 531 } 532 533 /* Get the ref_flow_handle for a flow by checking if there are any other 534 * flows that share the same L2 key as this flow. 535 */ 536 static int 537 bnxt_tc_get_ref_flow_handle(struct bnxt *bp, struct bnxt_tc_flow *flow, 538 struct bnxt_tc_flow_node *flow_node, 539 __le16 *ref_flow_handle) 540 { 541 struct bnxt_tc_info *tc_info = &bp->tc_info; 542 struct bnxt_tc_flow_node *ref_flow_node; 543 struct bnxt_tc_l2_node *l2_node; 544 545 l2_node = bnxt_tc_get_l2_node(bp, &tc_info->l2_table, 546 tc_info->l2_ht_params, 547 &flow->l2_key); 548 if (!l2_node) 549 return -1; 550 551 /* If any other flow is using this l2_node, use it's flow_handle 552 * as the ref_flow_handle 553 */ 554 if (l2_node->refcount > 0) { 555 ref_flow_node = list_first_entry(&l2_node->common_l2_flows, 556 struct bnxt_tc_flow_node, 557 l2_list_node); 558 *ref_flow_handle = ref_flow_node->flow_handle; 559 } else { 560 *ref_flow_handle = cpu_to_le16(0xffff); 561 } 562 563 /* Insert the l2_node into the flow_node so that subsequent flows 564 * with a matching l2 key can use the flow_handle of this flow 565 * as their ref_flow_handle 566 */ 567 flow_node->l2_node = l2_node; 568 list_add(&flow_node->l2_list_node, &l2_node->common_l2_flows); 569 l2_node->refcount++; 570 return 0; 571 } 572 573 /* After the flow parsing is done, this routine is used for checking 574 * if there are any aspects of the flow that prevent it from being 575 * offloaded. 576 */ 577 static bool bnxt_tc_can_offload(struct bnxt *bp, struct bnxt_tc_flow *flow) 578 { 579 /* If L4 ports are specified then ip_proto must be TCP or UDP */ 580 if ((flow->flags & BNXT_TC_FLOW_FLAGS_PORTS) && 581 (flow->l4_key.ip_proto != IPPROTO_TCP && 582 flow->l4_key.ip_proto != IPPROTO_UDP)) { 583 netdev_info(bp->dev, "Cannot offload non-TCP/UDP (%d) ports", 584 flow->l4_key.ip_proto); 585 return false; 586 } 587 588 return true; 589 } 590 591 static int __bnxt_tc_del_flow(struct bnxt *bp, 592 struct bnxt_tc_flow_node *flow_node) 593 { 594 struct bnxt_tc_info *tc_info = &bp->tc_info; 595 int rc; 596 597 /* send HWRM cmd to free the flow-id */ 598 bnxt_hwrm_cfa_flow_free(bp, flow_node->flow_handle); 599 600 mutex_lock(&tc_info->lock); 601 602 /* release reference to l2 node */ 603 bnxt_tc_put_l2_node(bp, flow_node); 604 605 mutex_unlock(&tc_info->lock); 606 607 rc = rhashtable_remove_fast(&tc_info->flow_table, &flow_node->node, 608 tc_info->flow_ht_params); 609 if (rc) 610 netdev_err(bp->dev, "Error: %s: rhashtable_remove_fast rc=%d", 611 __func__, rc); 612 613 kfree_rcu(flow_node, rcu); 614 return 0; 615 } 616 617 /* Add a new flow or replace an existing flow. 618 * Notes on locking: 619 * There are essentially two critical sections here. 620 * 1. while adding a new flow 621 * a) lookup l2-key 622 * b) issue HWRM cmd and get flow_handle 623 * c) link l2-key with flow 624 * 2. while deleting a flow 625 * a) unlinking l2-key from flow 626 * A lock is needed to protect these two critical sections. 627 * 628 * The hash-tables are already protected by the rhashtable API. 629 */ 630 static int bnxt_tc_add_flow(struct bnxt *bp, u16 src_fid, 631 struct tc_cls_flower_offload *tc_flow_cmd) 632 { 633 struct bnxt_tc_flow_node *new_node, *old_node; 634 struct bnxt_tc_info *tc_info = &bp->tc_info; 635 struct bnxt_tc_flow *flow; 636 __le16 ref_flow_handle; 637 int rc; 638 639 /* allocate memory for the new flow and it's node */ 640 new_node = kzalloc(sizeof(*new_node), GFP_KERNEL); 641 if (!new_node) { 642 rc = -ENOMEM; 643 goto done; 644 } 645 new_node->cookie = tc_flow_cmd->cookie; 646 flow = &new_node->flow; 647 648 rc = bnxt_tc_parse_flow(bp, tc_flow_cmd, flow); 649 if (rc) 650 goto free_node; 651 flow->src_fid = src_fid; 652 653 if (!bnxt_tc_can_offload(bp, flow)) { 654 rc = -ENOSPC; 655 goto free_node; 656 } 657 658 /* If a flow exists with the same cookie, delete it */ 659 old_node = rhashtable_lookup_fast(&tc_info->flow_table, 660 &tc_flow_cmd->cookie, 661 tc_info->flow_ht_params); 662 if (old_node) 663 __bnxt_tc_del_flow(bp, old_node); 664 665 /* Check if the L2 part of the flow has been offloaded already. 666 * If so, bump up it's refcnt and get it's reference handle. 667 */ 668 mutex_lock(&tc_info->lock); 669 rc = bnxt_tc_get_ref_flow_handle(bp, flow, new_node, &ref_flow_handle); 670 if (rc) 671 goto unlock; 672 673 /* send HWRM cmd to alloc the flow */ 674 rc = bnxt_hwrm_cfa_flow_alloc(bp, flow, ref_flow_handle, 675 &new_node->flow_handle); 676 if (rc) 677 goto put_l2; 678 679 /* add new flow to flow-table */ 680 rc = rhashtable_insert_fast(&tc_info->flow_table, &new_node->node, 681 tc_info->flow_ht_params); 682 if (rc) 683 goto hwrm_flow_free; 684 685 mutex_unlock(&tc_info->lock); 686 return 0; 687 688 hwrm_flow_free: 689 bnxt_hwrm_cfa_flow_free(bp, new_node->flow_handle); 690 put_l2: 691 bnxt_tc_put_l2_node(bp, new_node); 692 unlock: 693 mutex_unlock(&tc_info->lock); 694 free_node: 695 kfree(new_node); 696 done: 697 netdev_err(bp->dev, "Error: %s: cookie=0x%lx error=%d", 698 __func__, tc_flow_cmd->cookie, rc); 699 return rc; 700 } 701 702 static int bnxt_tc_del_flow(struct bnxt *bp, 703 struct tc_cls_flower_offload *tc_flow_cmd) 704 { 705 struct bnxt_tc_info *tc_info = &bp->tc_info; 706 struct bnxt_tc_flow_node *flow_node; 707 708 flow_node = rhashtable_lookup_fast(&tc_info->flow_table, 709 &tc_flow_cmd->cookie, 710 tc_info->flow_ht_params); 711 if (!flow_node) { 712 netdev_info(bp->dev, "ERROR: no flow_node for cookie %lx", 713 tc_flow_cmd->cookie); 714 return -EINVAL; 715 } 716 717 return __bnxt_tc_del_flow(bp, flow_node); 718 } 719 720 static int bnxt_tc_get_flow_stats(struct bnxt *bp, 721 struct tc_cls_flower_offload *tc_flow_cmd) 722 { 723 struct bnxt_tc_info *tc_info = &bp->tc_info; 724 struct bnxt_tc_flow_node *flow_node; 725 struct bnxt_tc_flow_stats stats; 726 int rc; 727 728 flow_node = rhashtable_lookup_fast(&tc_info->flow_table, 729 &tc_flow_cmd->cookie, 730 tc_info->flow_ht_params); 731 if (!flow_node) { 732 netdev_info(bp->dev, "Error: no flow_node for cookie %lx", 733 tc_flow_cmd->cookie); 734 return -1; 735 } 736 737 rc = bnxt_hwrm_cfa_flow_stats_get(bp, flow_node->flow_handle, 738 &flow_node->flow, &stats); 739 if (rc) 740 return rc; 741 742 tcf_exts_stats_update(tc_flow_cmd->exts, stats.bytes, stats.packets, 0); 743 return 0; 744 } 745 746 int bnxt_tc_setup_flower(struct bnxt *bp, u16 src_fid, 747 struct tc_cls_flower_offload *cls_flower) 748 { 749 int rc = 0; 750 751 if (!is_classid_clsact_ingress(cls_flower->common.classid) || 752 cls_flower->common.chain_index) 753 return -EOPNOTSUPP; 754 755 switch (cls_flower->command) { 756 case TC_CLSFLOWER_REPLACE: 757 rc = bnxt_tc_add_flow(bp, src_fid, cls_flower); 758 break; 759 760 case TC_CLSFLOWER_DESTROY: 761 rc = bnxt_tc_del_flow(bp, cls_flower); 762 break; 763 764 case TC_CLSFLOWER_STATS: 765 rc = bnxt_tc_get_flow_stats(bp, cls_flower); 766 break; 767 } 768 return rc; 769 } 770 771 static const struct rhashtable_params bnxt_tc_flow_ht_params = { 772 .head_offset = offsetof(struct bnxt_tc_flow_node, node), 773 .key_offset = offsetof(struct bnxt_tc_flow_node, cookie), 774 .key_len = sizeof(((struct bnxt_tc_flow_node *)0)->cookie), 775 .automatic_shrinking = true 776 }; 777 778 static const struct rhashtable_params bnxt_tc_l2_ht_params = { 779 .head_offset = offsetof(struct bnxt_tc_l2_node, node), 780 .key_offset = offsetof(struct bnxt_tc_l2_node, key), 781 .key_len = BNXT_TC_L2_KEY_LEN, 782 .automatic_shrinking = true 783 }; 784 785 /* convert counter width in bits to a mask */ 786 #define mask(width) ((u64)~0 >> (64 - (width))) 787 788 int bnxt_init_tc(struct bnxt *bp) 789 { 790 struct bnxt_tc_info *tc_info = &bp->tc_info; 791 int rc; 792 793 if (bp->hwrm_spec_code < 0x10800) { 794 netdev_warn(bp->dev, 795 "Firmware does not support TC flower offload.\n"); 796 return -ENOTSUPP; 797 } 798 mutex_init(&tc_info->lock); 799 800 /* Counter widths are programmed by FW */ 801 tc_info->bytes_mask = mask(36); 802 tc_info->packets_mask = mask(28); 803 804 tc_info->flow_ht_params = bnxt_tc_flow_ht_params; 805 rc = rhashtable_init(&tc_info->flow_table, &tc_info->flow_ht_params); 806 if (rc) 807 return rc; 808 809 tc_info->l2_ht_params = bnxt_tc_l2_ht_params; 810 rc = rhashtable_init(&tc_info->l2_table, &tc_info->l2_ht_params); 811 if (rc) 812 goto destroy_flow_table; 813 814 tc_info->enabled = true; 815 bp->dev->hw_features |= NETIF_F_HW_TC; 816 bp->dev->features |= NETIF_F_HW_TC; 817 return 0; 818 819 destroy_flow_table: 820 rhashtable_destroy(&tc_info->flow_table); 821 return rc; 822 } 823 824 void bnxt_shutdown_tc(struct bnxt *bp) 825 { 826 struct bnxt_tc_info *tc_info = &bp->tc_info; 827 828 if (!tc_info->enabled) 829 return; 830 831 rhashtable_destroy(&tc_info->flow_table); 832 rhashtable_destroy(&tc_info->l2_table); 833 } 834