1 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause 2 /* Copyright (c) 2021, Microsoft Corporation. */ 3 4 #include <uapi/linux/bpf.h> 5 6 #include <linux/debugfs.h> 7 #include <linux/inetdevice.h> 8 #include <linux/etherdevice.h> 9 #include <linux/ethtool.h> 10 #include <linux/filter.h> 11 #include <linux/mm.h> 12 #include <linux/pci.h> 13 #include <linux/export.h> 14 #include <linux/skbuff.h> 15 16 #include <net/checksum.h> 17 #include <net/ip6_checksum.h> 18 #include <net/netdev_lock.h> 19 #include <net/page_pool/helpers.h> 20 #include <net/xdp.h> 21 22 #include <net/mana/mana.h> 23 #include <net/mana/mana_auxiliary.h> 24 #include <net/mana/hw_channel.h> 25 26 static DEFINE_IDA(mana_adev_ida); 27 28 static int mana_adev_idx_alloc(void) 29 { 30 return ida_alloc(&mana_adev_ida, GFP_KERNEL); 31 } 32 33 static void mana_adev_idx_free(int idx) 34 { 35 ida_free(&mana_adev_ida, idx); 36 } 37 38 static ssize_t mana_dbg_q_read(struct file *filp, char __user *buf, size_t count, 39 loff_t *pos) 40 { 41 struct gdma_queue *gdma_q = filp->private_data; 42 43 return simple_read_from_buffer(buf, count, pos, gdma_q->queue_mem_ptr, 44 gdma_q->queue_size); 45 } 46 47 static const struct file_operations mana_dbg_q_fops = { 48 .owner = THIS_MODULE, 49 .open = simple_open, 50 .read = mana_dbg_q_read, 51 }; 52 53 static bool mana_en_need_log(struct mana_port_context *apc, int err) 54 { 55 if (apc && apc->ac && apc->ac->gdma_dev && 56 apc->ac->gdma_dev->gdma_context) 57 return mana_need_log(apc->ac->gdma_dev->gdma_context, err); 58 else 59 return true; 60 } 61 62 static void mana_put_rx_page(struct mana_rxq *rxq, struct page *page, 63 bool from_pool) 64 { 65 if (from_pool) 66 page_pool_put_full_page(rxq->page_pool, page, false); 67 else 68 put_page(page); 69 } 70 71 /* Microsoft Azure Network Adapter (MANA) functions */ 72 73 static int mana_open(struct net_device *ndev) 74 { 75 struct mana_port_context *apc = netdev_priv(ndev); 76 int err; 77 err = mana_alloc_queues(ndev); 78 79 if (err) { 80 netdev_err(ndev, "%s failed to allocate queues: %d\n", __func__, err); 81 return err; 82 } 83 84 apc->port_is_up = true; 85 86 /* Ensure port state updated before txq state */ 87 smp_wmb(); 88 89 netif_tx_wake_all_queues(ndev); 90 netdev_dbg(ndev, "%s successful\n", __func__); 91 return 0; 92 } 93 94 static int mana_close(struct net_device *ndev) 95 { 96 struct mana_port_context *apc = netdev_priv(ndev); 97 98 if (!apc->port_is_up) 99 return 0; 100 101 return mana_detach(ndev, true); 102 } 103 104 static void mana_link_state_handle(struct work_struct *w) 105 { 106 struct mana_context *ac; 107 struct net_device *ndev; 108 u32 link_event; 109 bool link_up; 110 int i; 111 112 ac = container_of(w, struct mana_context, link_change_work); 113 114 rtnl_lock(); 115 116 link_event = READ_ONCE(ac->link_event); 117 118 if (link_event == HWC_DATA_HW_LINK_CONNECT) 119 link_up = true; 120 else if (link_event == HWC_DATA_HW_LINK_DISCONNECT) 121 link_up = false; 122 else 123 goto out; 124 125 /* Process all ports */ 126 for (i = 0; i < ac->num_ports; i++) { 127 ndev = ac->ports[i]; 128 if (!ndev) 129 continue; 130 131 if (link_up) { 132 netif_carrier_on(ndev); 133 134 __netdev_notify_peers(ndev); 135 } else { 136 netif_carrier_off(ndev); 137 } 138 } 139 140 out: 141 rtnl_unlock(); 142 } 143 144 static bool mana_can_tx(struct gdma_queue *wq) 145 { 146 return mana_gd_wq_avail_space(wq) >= MAX_TX_WQE_SIZE; 147 } 148 149 static unsigned int mana_checksum_info(struct sk_buff *skb) 150 { 151 if (skb->protocol == htons(ETH_P_IP)) { 152 struct iphdr *ip = ip_hdr(skb); 153 154 if (ip->protocol == IPPROTO_TCP) 155 return IPPROTO_TCP; 156 157 if (ip->protocol == IPPROTO_UDP) 158 return IPPROTO_UDP; 159 } else if (skb->protocol == htons(ETH_P_IPV6)) { 160 struct ipv6hdr *ip6 = ipv6_hdr(skb); 161 162 if (ip6->nexthdr == IPPROTO_TCP) 163 return IPPROTO_TCP; 164 165 if (ip6->nexthdr == IPPROTO_UDP) 166 return IPPROTO_UDP; 167 } 168 169 /* No csum offloading */ 170 return 0; 171 } 172 173 static void mana_add_sge(struct mana_tx_package *tp, struct mana_skb_head *ash, 174 int sg_i, dma_addr_t da, int sge_len, u32 gpa_mkey) 175 { 176 ash->dma_handle[sg_i] = da; 177 ash->size[sg_i] = sge_len; 178 179 tp->wqe_req.sgl[sg_i].address = da; 180 tp->wqe_req.sgl[sg_i].mem_key = gpa_mkey; 181 tp->wqe_req.sgl[sg_i].size = sge_len; 182 } 183 184 static int mana_map_skb(struct sk_buff *skb, struct mana_port_context *apc, 185 struct mana_tx_package *tp, int gso_hs) 186 { 187 struct mana_skb_head *ash = (struct mana_skb_head *)skb->head; 188 int hsg = 1; /* num of SGEs of linear part */ 189 struct gdma_dev *gd = apc->ac->gdma_dev; 190 int skb_hlen = skb_headlen(skb); 191 int sge0_len, sge1_len = 0; 192 struct gdma_context *gc; 193 struct device *dev; 194 skb_frag_t *frag; 195 dma_addr_t da; 196 int sg_i; 197 int i; 198 199 gc = gd->gdma_context; 200 dev = gc->dev; 201 202 if (gso_hs && gso_hs < skb_hlen) { 203 sge0_len = gso_hs; 204 sge1_len = skb_hlen - gso_hs; 205 } else { 206 sge0_len = skb_hlen; 207 } 208 209 da = dma_map_single(dev, skb->data, sge0_len, DMA_TO_DEVICE); 210 if (dma_mapping_error(dev, da)) 211 return -ENOMEM; 212 213 mana_add_sge(tp, ash, 0, da, sge0_len, gd->gpa_mkey); 214 215 if (sge1_len) { 216 sg_i = 1; 217 da = dma_map_single(dev, skb->data + sge0_len, sge1_len, 218 DMA_TO_DEVICE); 219 if (dma_mapping_error(dev, da)) 220 goto frag_err; 221 222 mana_add_sge(tp, ash, sg_i, da, sge1_len, gd->gpa_mkey); 223 hsg = 2; 224 } 225 226 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 227 sg_i = hsg + i; 228 229 frag = &skb_shinfo(skb)->frags[i]; 230 da = skb_frag_dma_map(dev, frag, 0, skb_frag_size(frag), 231 DMA_TO_DEVICE); 232 if (dma_mapping_error(dev, da)) 233 goto frag_err; 234 235 mana_add_sge(tp, ash, sg_i, da, skb_frag_size(frag), 236 gd->gpa_mkey); 237 } 238 239 return 0; 240 241 frag_err: 242 if (net_ratelimit()) 243 netdev_err(apc->ndev, "Failed to map skb of size %u to DMA\n", 244 skb->len); 245 for (i = sg_i - 1; i >= hsg; i--) 246 dma_unmap_page(dev, ash->dma_handle[i], ash->size[i], 247 DMA_TO_DEVICE); 248 249 for (i = hsg - 1; i >= 0; i--) 250 dma_unmap_single(dev, ash->dma_handle[i], ash->size[i], 251 DMA_TO_DEVICE); 252 253 return -ENOMEM; 254 } 255 256 /* Handle the case when GSO SKB linear length is too large. 257 * MANA NIC requires GSO packets to put only the packet header to SGE0. 258 * So, we need 2 SGEs for the skb linear part which contains more than the 259 * header. 260 * Return a positive value for the number of SGEs, or a negative value 261 * for an error. 262 */ 263 static int mana_fix_skb_head(struct net_device *ndev, struct sk_buff *skb, 264 int gso_hs) 265 { 266 int num_sge = 1 + skb_shinfo(skb)->nr_frags; 267 int skb_hlen = skb_headlen(skb); 268 269 if (gso_hs < skb_hlen) { 270 num_sge++; 271 } else if (gso_hs > skb_hlen) { 272 if (net_ratelimit()) 273 netdev_err(ndev, 274 "TX nonlinear head: hs:%d, skb_hlen:%d\n", 275 gso_hs, skb_hlen); 276 277 return -EINVAL; 278 } 279 280 return num_sge; 281 } 282 283 /* Get the GSO packet's header size */ 284 static int mana_get_gso_hs(struct sk_buff *skb) 285 { 286 int gso_hs; 287 288 if (skb->encapsulation) { 289 gso_hs = skb_inner_tcp_all_headers(skb); 290 } else { 291 if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4) { 292 gso_hs = skb_transport_offset(skb) + 293 sizeof(struct udphdr); 294 } else { 295 gso_hs = skb_tcp_all_headers(skb); 296 } 297 } 298 299 return gso_hs; 300 } 301 302 static void mana_per_port_queue_reset_work_handler(struct work_struct *work) 303 { 304 struct mana_port_context *apc = container_of(work, 305 struct mana_port_context, 306 queue_reset_work); 307 struct net_device *ndev = apc->ndev; 308 int err; 309 310 rtnl_lock(); 311 312 /* Block RDMA from grabbing the vport during the detach/attach 313 * window, same as mana_set_channels(). 314 */ 315 mutex_lock(&apc->vport_mutex); 316 apc->channel_changing = true; 317 mutex_unlock(&apc->vport_mutex); 318 319 /* Pre-allocate buffers to prevent failure in mana_attach later */ 320 err = mana_pre_alloc_rxbufs(apc, ndev->mtu, apc->num_queues); 321 if (err) { 322 netdev_err(ndev, "Insufficient memory for reset post tx stall detection\n"); 323 goto clear_flag; 324 } 325 326 err = mana_detach(ndev, false); 327 if (err) { 328 netdev_err(ndev, "mana_detach failed: %d\n", err); 329 goto dealloc_pre_rxbufs; 330 } 331 332 err = mana_attach(ndev); 333 if (err) 334 netdev_err(ndev, "mana_attach failed: %d\n", err); 335 336 dealloc_pre_rxbufs: 337 mana_pre_dealloc_rxbufs(apc); 338 clear_flag: 339 mutex_lock(&apc->vport_mutex); 340 apc->channel_changing = false; 341 mutex_unlock(&apc->vport_mutex); 342 343 rtnl_unlock(); 344 } 345 346 netdev_tx_t mana_start_xmit(struct sk_buff *skb, struct net_device *ndev) 347 { 348 enum mana_tx_pkt_format pkt_fmt = MANA_SHORT_PKT_FMT; 349 struct mana_port_context *apc = netdev_priv(ndev); 350 int gso_hs = 0; /* zero for non-GSO pkts */ 351 u16 txq_idx = skb_get_queue_mapping(skb); 352 struct gdma_dev *gd = apc->ac->gdma_dev; 353 bool ipv4 = false, ipv6 = false; 354 struct mana_tx_package pkg = {}; 355 struct netdev_queue *net_txq; 356 struct mana_stats_tx *tx_stats; 357 struct gdma_queue *gdma_sq; 358 int err, len, num_gso_seg; 359 unsigned int csum_type; 360 struct mana_txq *txq; 361 struct mana_cq *cq; 362 363 if (unlikely(!apc->port_is_up)) 364 goto tx_drop; 365 366 if (skb_cow_head(skb, MANA_HEADROOM)) 367 goto tx_drop_count; 368 369 txq = &apc->tx_qp[txq_idx]->txq; 370 gdma_sq = txq->gdma_sq; 371 cq = &apc->tx_qp[txq_idx]->tx_cq; 372 tx_stats = &txq->stats; 373 374 BUILD_BUG_ON(MAX_TX_WQE_SGL_ENTRIES != MANA_MAX_TX_WQE_SGL_ENTRIES); 375 if (MAX_SKB_FRAGS + 2 > MAX_TX_WQE_SGL_ENTRIES && 376 skb_shinfo(skb)->nr_frags + 2 > MAX_TX_WQE_SGL_ENTRIES) { 377 /* GSO skb with Hardware SGE limit exceeded is not expected here 378 * as they are handled in mana_features_check() callback 379 */ 380 if (skb_linearize(skb)) { 381 netdev_warn_once(ndev, "Failed to linearize skb with nr_frags=%d and is_gso=%d\n", 382 skb_shinfo(skb)->nr_frags, 383 skb_is_gso(skb)); 384 goto tx_drop_count; 385 } 386 apc->eth_stats.tx_linear_pkt_cnt++; 387 } 388 389 pkg.tx_oob.s_oob.vcq_num = cq->gdma_id; 390 pkg.tx_oob.s_oob.vsq_frame = txq->vsq_frame; 391 392 if (txq->vp_offset > MANA_SHORT_VPORT_OFFSET_MAX) { 393 pkg.tx_oob.l_oob.long_vp_offset = txq->vp_offset; 394 pkt_fmt = MANA_LONG_PKT_FMT; 395 } else { 396 pkg.tx_oob.s_oob.short_vp_offset = txq->vp_offset; 397 } 398 399 if (skb_vlan_tag_present(skb)) { 400 pkt_fmt = MANA_LONG_PKT_FMT; 401 pkg.tx_oob.l_oob.inject_vlan_pri_tag = 1; 402 pkg.tx_oob.l_oob.pcp = skb_vlan_tag_get_prio(skb); 403 pkg.tx_oob.l_oob.dei = skb_vlan_tag_get_cfi(skb); 404 pkg.tx_oob.l_oob.vlan_id = skb_vlan_tag_get_id(skb); 405 } 406 407 pkg.tx_oob.s_oob.pkt_fmt = pkt_fmt; 408 409 if (pkt_fmt == MANA_SHORT_PKT_FMT) { 410 pkg.wqe_req.inline_oob_size = sizeof(struct mana_tx_short_oob); 411 u64_stats_update_begin(&tx_stats->syncp); 412 tx_stats->short_pkt_fmt++; 413 u64_stats_update_end(&tx_stats->syncp); 414 } else { 415 pkg.wqe_req.inline_oob_size = sizeof(struct mana_tx_oob); 416 u64_stats_update_begin(&tx_stats->syncp); 417 tx_stats->long_pkt_fmt++; 418 u64_stats_update_end(&tx_stats->syncp); 419 } 420 421 pkg.wqe_req.inline_oob_data = &pkg.tx_oob; 422 pkg.wqe_req.flags = 0; 423 pkg.wqe_req.client_data_unit = 0; 424 425 pkg.wqe_req.num_sge = 1 + skb_shinfo(skb)->nr_frags; 426 427 if (skb->protocol == htons(ETH_P_IP)) 428 ipv4 = true; 429 else if (skb->protocol == htons(ETH_P_IPV6)) 430 ipv6 = true; 431 432 if (skb_is_gso(skb)) { 433 int num_sge; 434 435 gso_hs = mana_get_gso_hs(skb); 436 437 num_sge = mana_fix_skb_head(ndev, skb, gso_hs); 438 if (num_sge > 0) 439 pkg.wqe_req.num_sge = num_sge; 440 else 441 goto tx_drop_count; 442 443 u64_stats_update_begin(&tx_stats->syncp); 444 if (skb->encapsulation) { 445 tx_stats->tso_inner_packets++; 446 tx_stats->tso_inner_bytes += skb->len - gso_hs; 447 } else { 448 tx_stats->tso_packets++; 449 tx_stats->tso_bytes += skb->len - gso_hs; 450 } 451 u64_stats_update_end(&tx_stats->syncp); 452 453 pkg.tx_oob.s_oob.is_outer_ipv4 = ipv4; 454 pkg.tx_oob.s_oob.is_outer_ipv6 = ipv6; 455 456 pkg.tx_oob.s_oob.comp_iphdr_csum = 1; 457 pkg.tx_oob.s_oob.comp_tcp_csum = 1; 458 pkg.tx_oob.s_oob.trans_off = skb_transport_offset(skb); 459 460 pkg.wqe_req.client_data_unit = skb_shinfo(skb)->gso_size; 461 pkg.wqe_req.flags = GDMA_WR_OOB_IN_SGL | GDMA_WR_PAD_BY_SGE0; 462 if (ipv4) { 463 ip_hdr(skb)->tot_len = 0; 464 ip_hdr(skb)->check = 0; 465 tcp_hdr(skb)->check = 466 ~csum_tcpudp_magic(ip_hdr(skb)->saddr, 467 ip_hdr(skb)->daddr, 0, 468 IPPROTO_TCP, 0); 469 } else { 470 ipv6_hdr(skb)->payload_len = 0; 471 tcp_hdr(skb)->check = 472 ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr, 473 &ipv6_hdr(skb)->daddr, 0, 474 IPPROTO_TCP, 0); 475 } 476 } else if (skb->ip_summed == CHECKSUM_PARTIAL) { 477 csum_type = mana_checksum_info(skb); 478 479 u64_stats_update_begin(&tx_stats->syncp); 480 tx_stats->csum_partial++; 481 u64_stats_update_end(&tx_stats->syncp); 482 483 if (csum_type == IPPROTO_TCP) { 484 pkg.tx_oob.s_oob.is_outer_ipv4 = ipv4; 485 pkg.tx_oob.s_oob.is_outer_ipv6 = ipv6; 486 487 pkg.tx_oob.s_oob.comp_tcp_csum = 1; 488 pkg.tx_oob.s_oob.trans_off = skb_transport_offset(skb); 489 490 } else if (csum_type == IPPROTO_UDP) { 491 pkg.tx_oob.s_oob.is_outer_ipv4 = ipv4; 492 pkg.tx_oob.s_oob.is_outer_ipv6 = ipv6; 493 494 pkg.tx_oob.s_oob.comp_udp_csum = 1; 495 } else { 496 /* Can't do offload of this type of checksum */ 497 if (skb_checksum_help(skb)) 498 goto tx_drop_count; 499 } 500 } 501 502 if (pkg.wqe_req.num_sge <= ARRAY_SIZE(pkg.sgl_array)) { 503 pkg.wqe_req.sgl = pkg.sgl_array; 504 } else { 505 pkg.sgl_ptr = kmalloc_objs(struct gdma_sge, pkg.wqe_req.num_sge, 506 GFP_ATOMIC); 507 if (!pkg.sgl_ptr) 508 goto tx_drop_count; 509 510 pkg.wqe_req.sgl = pkg.sgl_ptr; 511 } 512 513 if (mana_map_skb(skb, apc, &pkg, gso_hs)) { 514 u64_stats_update_begin(&tx_stats->syncp); 515 tx_stats->mana_map_err++; 516 u64_stats_update_end(&tx_stats->syncp); 517 goto free_sgl_ptr; 518 } 519 520 skb_queue_tail(&txq->pending_skbs, skb); 521 522 len = skb->len; 523 num_gso_seg = skb_is_gso(skb) ? skb_shinfo(skb)->gso_segs : 1; 524 net_txq = netdev_get_tx_queue(ndev, txq_idx); 525 526 err = mana_gd_post_work_request(gdma_sq, &pkg.wqe_req, 527 (struct gdma_posted_wqe_info *)skb->cb); 528 if (!mana_can_tx(gdma_sq)) { 529 netif_tx_stop_queue(net_txq); 530 apc->eth_stats.stop_queue++; 531 } 532 533 if (err) { 534 (void)skb_dequeue_tail(&txq->pending_skbs); 535 mana_unmap_skb(skb, apc); 536 netdev_warn(ndev, "Failed to post TX OOB: %d\n", err); 537 goto free_sgl_ptr; 538 } 539 540 err = NETDEV_TX_OK; 541 atomic_inc(&txq->pending_sends); 542 543 mana_gd_wq_ring_doorbell(gd->gdma_context, gdma_sq); 544 545 /* skb may be freed after mana_gd_post_work_request. Do not use it. */ 546 skb = NULL; 547 548 /* Populated the packet and bytes counters based on post GSO packet 549 * calculations 550 */ 551 tx_stats = &txq->stats; 552 u64_stats_update_begin(&tx_stats->syncp); 553 tx_stats->packets += num_gso_seg; 554 tx_stats->bytes += len + ((num_gso_seg - 1) * gso_hs); 555 u64_stats_update_end(&tx_stats->syncp); 556 557 if (netif_tx_queue_stopped(net_txq) && mana_can_tx(gdma_sq)) { 558 netif_tx_wake_queue(net_txq); 559 apc->eth_stats.wake_queue++; 560 } 561 562 kfree(pkg.sgl_ptr); 563 return err; 564 565 free_sgl_ptr: 566 kfree(pkg.sgl_ptr); 567 tx_drop_count: 568 ndev->stats.tx_dropped++; 569 tx_drop: 570 dev_kfree_skb_any(skb); 571 return NETDEV_TX_OK; 572 } 573 574 #if (MAX_SKB_FRAGS + 2 > MANA_MAX_TX_WQE_SGL_ENTRIES) 575 static netdev_features_t mana_features_check(struct sk_buff *skb, 576 struct net_device *ndev, 577 netdev_features_t features) 578 { 579 if (skb_shinfo(skb)->nr_frags + 2 > MAX_TX_WQE_SGL_ENTRIES) { 580 /* Exceeds HW SGE limit. 581 * GSO case: 582 * Disable GSO so the stack will software-segment the skb 583 * into smaller skbs that fit the SGE budget. 584 * Non-GSO case: 585 * The xmit path will attempt skb_linearize() as a fallback. 586 */ 587 features &= ~NETIF_F_GSO_MASK; 588 } 589 return features; 590 } 591 #endif 592 593 static void mana_get_stats64(struct net_device *ndev, 594 struct rtnl_link_stats64 *st) 595 { 596 struct mana_port_context *apc = netdev_priv(ndev); 597 unsigned int num_queues = apc->num_queues; 598 struct mana_stats_rx *rx_stats; 599 struct mana_stats_tx *tx_stats; 600 unsigned int start; 601 u64 packets, bytes; 602 int q; 603 604 if (!apc->port_is_up) 605 return; 606 607 netdev_stats_to_stats64(st, &ndev->stats); 608 609 if (apc->ac->hwc_timeout_occurred) 610 netdev_warn_once(ndev, "HWC timeout occurred\n"); 611 612 st->rx_missed_errors = apc->ac->hc_stats.hc_rx_discards_no_wqe; 613 614 for (q = 0; q < num_queues; q++) { 615 rx_stats = &apc->rxqs[q]->stats; 616 617 do { 618 start = u64_stats_fetch_begin(&rx_stats->syncp); 619 packets = rx_stats->packets; 620 bytes = rx_stats->bytes; 621 } while (u64_stats_fetch_retry(&rx_stats->syncp, start)); 622 623 st->rx_packets += packets; 624 st->rx_bytes += bytes; 625 } 626 627 for (q = 0; q < num_queues; q++) { 628 tx_stats = &apc->tx_qp[q]->txq.stats; 629 630 do { 631 start = u64_stats_fetch_begin(&tx_stats->syncp); 632 packets = tx_stats->packets; 633 bytes = tx_stats->bytes; 634 } while (u64_stats_fetch_retry(&tx_stats->syncp, start)); 635 636 st->tx_packets += packets; 637 st->tx_bytes += bytes; 638 } 639 } 640 641 static int mana_get_tx_queue(struct net_device *ndev, struct sk_buff *skb, 642 int old_q) 643 { 644 struct mana_port_context *apc = netdev_priv(ndev); 645 u32 hash = skb_get_hash(skb); 646 struct sock *sk = skb->sk; 647 int txq; 648 649 txq = apc->indir_table[hash & (apc->indir_table_sz - 1)]; 650 651 if (txq != old_q && sk && sk_fullsock(sk) && 652 rcu_access_pointer(sk->sk_dst_cache)) 653 sk_tx_queue_set(sk, txq); 654 655 return txq; 656 } 657 658 static u16 mana_select_queue(struct net_device *ndev, struct sk_buff *skb, 659 struct net_device *sb_dev) 660 { 661 int txq; 662 663 if (ndev->real_num_tx_queues == 1) 664 return 0; 665 666 txq = sk_tx_queue_get(skb->sk); 667 668 if (txq < 0 || skb->ooo_okay || txq >= ndev->real_num_tx_queues) { 669 if (skb_rx_queue_recorded(skb)) 670 txq = skb_get_rx_queue(skb); 671 else 672 txq = mana_get_tx_queue(ndev, skb, txq); 673 } 674 675 return txq; 676 } 677 678 /* Release pre-allocated RX buffers */ 679 void mana_pre_dealloc_rxbufs(struct mana_port_context *mpc) 680 { 681 struct device *dev; 682 int i; 683 684 dev = mpc->ac->gdma_dev->gdma_context->dev; 685 686 if (!mpc->rxbufs_pre) 687 goto out1; 688 689 if (!mpc->das_pre) 690 goto out2; 691 692 while (mpc->rxbpre_total) { 693 i = --mpc->rxbpre_total; 694 dma_unmap_single(dev, mpc->das_pre[i], mpc->rxbpre_datasize, 695 DMA_FROM_DEVICE); 696 put_page(virt_to_head_page(mpc->rxbufs_pre[i])); 697 } 698 699 kvfree(mpc->das_pre); 700 mpc->das_pre = NULL; 701 702 out2: 703 kvfree(mpc->rxbufs_pre); 704 mpc->rxbufs_pre = NULL; 705 706 out1: 707 mpc->rxbpre_datasize = 0; 708 mpc->rxbpre_alloc_size = 0; 709 mpc->rxbpre_headroom = 0; 710 } 711 712 /* Get a buffer from the pre-allocated RX buffers */ 713 static void *mana_get_rxbuf_pre(struct mana_rxq *rxq, dma_addr_t *da) 714 { 715 struct net_device *ndev = rxq->ndev; 716 struct mana_port_context *mpc; 717 void *va; 718 719 mpc = netdev_priv(ndev); 720 721 if (!mpc->rxbufs_pre || !mpc->das_pre || !mpc->rxbpre_total) { 722 netdev_err(ndev, "No RX pre-allocated bufs\n"); 723 return NULL; 724 } 725 726 /* Check sizes to catch unexpected coding error */ 727 if (mpc->rxbpre_datasize != rxq->datasize) { 728 netdev_err(ndev, "rxbpre_datasize mismatch: %u: %u\n", 729 mpc->rxbpre_datasize, rxq->datasize); 730 return NULL; 731 } 732 733 if (mpc->rxbpre_alloc_size != rxq->alloc_size) { 734 netdev_err(ndev, "rxbpre_alloc_size mismatch: %u: %u\n", 735 mpc->rxbpre_alloc_size, rxq->alloc_size); 736 return NULL; 737 } 738 739 if (mpc->rxbpre_headroom != rxq->headroom) { 740 netdev_err(ndev, "rxbpre_headroom mismatch: %u: %u\n", 741 mpc->rxbpre_headroom, rxq->headroom); 742 return NULL; 743 } 744 745 mpc->rxbpre_total--; 746 747 *da = mpc->das_pre[mpc->rxbpre_total]; 748 va = mpc->rxbufs_pre[mpc->rxbpre_total]; 749 mpc->rxbufs_pre[mpc->rxbpre_total] = NULL; 750 751 /* Deallocate the array after all buffers are gone */ 752 if (!mpc->rxbpre_total) 753 mana_pre_dealloc_rxbufs(mpc); 754 755 return va; 756 } 757 758 /* Get RX buffer's data size, alloc size, XDP headroom based on MTU */ 759 static void mana_get_rxbuf_cfg(struct mana_port_context *apc, 760 int mtu, u32 *datasize, u32 *alloc_size, 761 u32 *headroom, u32 *frag_count) 762 { 763 u32 len, buf_size; 764 765 /* Calculate datasize first (consistent across all cases) */ 766 *datasize = mtu + ETH_HLEN; 767 768 /* For xdp and jumbo frames make sure only one packet fits per page */ 769 if (mtu + MANA_RXBUF_PAD > PAGE_SIZE / 2 || mana_xdp_get(apc)) { 770 if (mana_xdp_get(apc)) { 771 *headroom = XDP_PACKET_HEADROOM; 772 *alloc_size = PAGE_SIZE; 773 } else { 774 *headroom = 0; /* no support for XDP */ 775 *alloc_size = SKB_DATA_ALIGN(mtu + MANA_RXBUF_PAD + 776 *headroom); 777 } 778 779 *frag_count = 1; 780 781 /* In the single-buffer path, napi_build_skb() must see the 782 * actual backing allocation size so skb->truesize reflects 783 * the full page (or higher-order page), not just the usable 784 * packet area. 785 */ 786 *alloc_size = PAGE_SIZE << get_order(*alloc_size); 787 return; 788 } 789 790 /* Standard MTU case - optimize for multiple packets per page */ 791 *headroom = 0; 792 793 /* Calculate base buffer size needed */ 794 len = SKB_DATA_ALIGN(mtu + MANA_RXBUF_PAD + *headroom); 795 buf_size = ALIGN(len, MANA_RX_FRAG_ALIGNMENT); 796 797 /* Calculate how many packets can fit in a page */ 798 *frag_count = PAGE_SIZE / buf_size; 799 *alloc_size = buf_size; 800 } 801 802 int mana_pre_alloc_rxbufs(struct mana_port_context *mpc, int new_mtu, int num_queues) 803 { 804 struct device *dev; 805 struct page *page; 806 dma_addr_t da; 807 int num_rxb; 808 void *va; 809 int i; 810 811 mana_get_rxbuf_cfg(mpc, new_mtu, &mpc->rxbpre_datasize, 812 &mpc->rxbpre_alloc_size, &mpc->rxbpre_headroom, 813 &mpc->rxbpre_frag_count); 814 815 dev = mpc->ac->gdma_dev->gdma_context->dev; 816 817 num_rxb = num_queues * mpc->rx_queue_size; 818 819 WARN(mpc->rxbufs_pre, "mana rxbufs_pre exists\n"); 820 mpc->rxbufs_pre = kvmalloc_array(num_rxb, sizeof(void *), GFP_KERNEL); 821 if (!mpc->rxbufs_pre) 822 goto error; 823 824 mpc->das_pre = kvmalloc_objs(dma_addr_t, num_rxb); 825 if (!mpc->das_pre) 826 goto error; 827 828 mpc->rxbpre_total = 0; 829 830 for (i = 0; i < num_rxb; i++) { 831 page = dev_alloc_pages(get_order(mpc->rxbpre_alloc_size)); 832 if (!page) 833 goto error; 834 835 va = page_to_virt(page); 836 837 da = dma_map_single(dev, va + mpc->rxbpre_headroom, 838 mpc->rxbpre_datasize, DMA_FROM_DEVICE); 839 if (dma_mapping_error(dev, da)) { 840 put_page(page); 841 goto error; 842 } 843 844 mpc->rxbufs_pre[i] = va; 845 mpc->das_pre[i] = da; 846 mpc->rxbpre_total = i + 1; 847 } 848 849 return 0; 850 851 error: 852 netdev_err(mpc->ndev, "Failed to pre-allocate RX buffers for %d queues\n", num_queues); 853 mana_pre_dealloc_rxbufs(mpc); 854 return -ENOMEM; 855 } 856 857 static int mana_change_mtu(struct net_device *ndev, int new_mtu) 858 { 859 struct mana_port_context *mpc = netdev_priv(ndev); 860 unsigned int old_mtu = ndev->mtu; 861 int err; 862 863 /* Pre-allocate buffers to prevent failure in mana_attach later */ 864 err = mana_pre_alloc_rxbufs(mpc, new_mtu, mpc->num_queues); 865 if (err) { 866 netdev_err(ndev, "Insufficient memory for new MTU\n"); 867 return err; 868 } 869 870 err = mana_detach(ndev, false); 871 if (err) { 872 netdev_err(ndev, "mana_detach failed: %d\n", err); 873 goto out; 874 } 875 876 WRITE_ONCE(ndev->mtu, new_mtu); 877 878 err = mana_attach(ndev); 879 if (err) { 880 netdev_err(ndev, "mana_attach failed: %d\n", err); 881 WRITE_ONCE(ndev->mtu, old_mtu); 882 } 883 884 out: 885 mana_pre_dealloc_rxbufs(mpc); 886 return err; 887 } 888 889 static void mana_tx_timeout(struct net_device *netdev, unsigned int txqueue) 890 { 891 struct mana_port_context *apc = netdev_priv(netdev); 892 struct mana_context *ac = apc->ac; 893 struct gdma_context *gc = ac->gdma_dev->gdma_context; 894 895 /* Already in service, hence tx queue reset is not required.*/ 896 if (test_bit(GC_IN_SERVICE, &gc->flags)) 897 return; 898 899 /* Note: If there are pending queue reset work for this port(apc), 900 * subsequent request queued up from here are ignored. This is because 901 * we are using the same work instance per port(apc). 902 */ 903 queue_work(ac->per_port_queue_reset_wq, &apc->queue_reset_work); 904 } 905 906 static int mana_shaper_set(struct net_shaper_binding *binding, 907 const struct net_shaper *shaper, 908 struct netlink_ext_ack *extack) 909 { 910 struct mana_port_context *apc = netdev_priv(binding->netdev); 911 u32 old_speed, rate; 912 int err; 913 914 if (shaper->handle.scope != NET_SHAPER_SCOPE_NETDEV) { 915 NL_SET_ERR_MSG_MOD(extack, "net shaper scope should be netdev"); 916 return -EINVAL; 917 } 918 919 if (apc->handle.id && shaper->handle.id != apc->handle.id) { 920 NL_SET_ERR_MSG_MOD(extack, "Cannot create multiple shapers"); 921 return -EOPNOTSUPP; 922 } 923 924 if (!shaper->bw_max || (shaper->bw_max % 100000000)) { 925 NL_SET_ERR_MSG_MOD(extack, "Please use multiples of 100Mbps for bandwidth"); 926 return -EINVAL; 927 } 928 929 rate = div_u64(shaper->bw_max, 1000); /* Convert bps to Kbps */ 930 rate = div_u64(rate, 1000); /* Convert Kbps to Mbps */ 931 932 /* Get current speed */ 933 err = mana_query_link_cfg(apc); 934 old_speed = (err) ? SPEED_UNKNOWN : apc->speed; 935 936 if (!err) { 937 err = mana_set_bw_clamp(apc, rate, TRI_STATE_TRUE); 938 apc->speed = (err) ? old_speed : rate; 939 apc->handle = (err) ? apc->handle : shaper->handle; 940 } 941 942 return err; 943 } 944 945 static int mana_shaper_del(struct net_shaper_binding *binding, 946 const struct net_shaper_handle *handle, 947 struct netlink_ext_ack *extack) 948 { 949 struct mana_port_context *apc = netdev_priv(binding->netdev); 950 int err; 951 952 err = mana_set_bw_clamp(apc, 0, TRI_STATE_FALSE); 953 954 if (!err) { 955 /* Reset mana port context parameters */ 956 apc->handle.id = 0; 957 apc->handle.scope = NET_SHAPER_SCOPE_UNSPEC; 958 apc->speed = apc->max_speed; 959 } 960 961 return err; 962 } 963 964 static void mana_shaper_cap(struct net_shaper_binding *binding, 965 enum net_shaper_scope scope, 966 unsigned long *flags) 967 { 968 *flags = BIT(NET_SHAPER_A_CAPS_SUPPORT_BW_MAX) | 969 BIT(NET_SHAPER_A_CAPS_SUPPORT_METRIC_BPS); 970 } 971 972 static const struct net_shaper_ops mana_shaper_ops = { 973 .set = mana_shaper_set, 974 .delete = mana_shaper_del, 975 .capabilities = mana_shaper_cap, 976 }; 977 978 static const struct net_device_ops mana_devops = { 979 .ndo_open = mana_open, 980 .ndo_stop = mana_close, 981 .ndo_select_queue = mana_select_queue, 982 #if (MAX_SKB_FRAGS + 2 > MANA_MAX_TX_WQE_SGL_ENTRIES) 983 .ndo_features_check = mana_features_check, 984 #endif 985 .ndo_start_xmit = mana_start_xmit, 986 .ndo_validate_addr = eth_validate_addr, 987 .ndo_get_stats64 = mana_get_stats64, 988 .ndo_bpf = mana_bpf, 989 .ndo_xdp_xmit = mana_xdp_xmit, 990 .ndo_change_mtu = mana_change_mtu, 991 .ndo_tx_timeout = mana_tx_timeout, 992 .net_shaper_ops = &mana_shaper_ops, 993 }; 994 995 static void mana_cleanup_port_context(struct mana_port_context *apc) 996 { 997 /* 998 * make sure subsequent cleanup attempts don't end up removing already 999 * cleaned dentry pointer 1000 */ 1001 debugfs_remove(apc->mana_port_debugfs); 1002 apc->mana_port_debugfs = NULL; 1003 kfree(apc->rxqs); 1004 apc->rxqs = NULL; 1005 } 1006 1007 static void mana_cleanup_indir_table(struct mana_port_context *apc) 1008 { 1009 apc->indir_table_sz = 0; 1010 kfree(apc->indir_table); 1011 kfree(apc->rxobj_table); 1012 } 1013 1014 static int mana_init_port_context(struct mana_port_context *apc) 1015 { 1016 apc->rxqs = kzalloc_objs(struct mana_rxq *, apc->num_queues); 1017 1018 return !apc->rxqs ? -ENOMEM : 0; 1019 } 1020 1021 static int gdma_mana_send_request(struct gdma_context *gc, void *in_buf, 1022 u32 in_len, void *out_buf, u32 out_len) 1023 { 1024 struct gdma_resp_hdr *resp = out_buf; 1025 struct gdma_req_hdr *req = in_buf; 1026 struct device *dev = gc->dev; 1027 static atomic_t activity_id; 1028 int err; 1029 1030 req->dev_id = gc->mana.dev_id; 1031 req->activity_id = atomic_inc_return(&activity_id); 1032 1033 err = mana_gd_send_request(gc, in_len, in_buf, out_len, 1034 out_buf); 1035 if (err || resp->status) { 1036 if (err == -EOPNOTSUPP) 1037 return err; 1038 1039 if (req->req.msg_type != MANA_QUERY_PHY_STAT && 1040 mana_need_log(gc, err)) 1041 dev_err(dev, "Command 0x%x failed with status: 0x%x, err: %d\n", 1042 req->req.msg_type, resp->status, err); 1043 return err ? err : -EPROTO; 1044 } 1045 1046 if (req->dev_id.as_uint32 != resp->dev_id.as_uint32 || 1047 req->activity_id != resp->activity_id) { 1048 dev_err(dev, "Unexpected mana message response: %x,%x,%x,%x\n", 1049 req->dev_id.as_uint32, resp->dev_id.as_uint32, 1050 req->activity_id, resp->activity_id); 1051 return -EPROTO; 1052 } 1053 1054 return 0; 1055 } 1056 1057 static int mana_send_request(struct mana_context *ac, void *in_buf, 1058 u32 in_len, void *out_buf, u32 out_len) 1059 { 1060 struct gdma_context *gc = ac->gdma_dev->gdma_context; 1061 1062 return gdma_mana_send_request(gc, in_buf, in_len, out_buf, out_len); 1063 } 1064 1065 static int mana_verify_resp_hdr(const struct gdma_resp_hdr *resp_hdr, 1066 const enum mana_command_code expected_code, 1067 const u32 min_size) 1068 { 1069 if (resp_hdr->response.msg_type != expected_code) 1070 return -EPROTO; 1071 1072 if (resp_hdr->response.msg_version < GDMA_MESSAGE_V1) 1073 return -EPROTO; 1074 1075 if (resp_hdr->response.msg_size < min_size) 1076 return -EPROTO; 1077 1078 return 0; 1079 } 1080 1081 static int mana_pf_register_hw_vport(struct mana_port_context *apc) 1082 { 1083 struct mana_register_hw_vport_resp resp = {}; 1084 struct mana_register_hw_vport_req req = {}; 1085 int err; 1086 1087 mana_gd_init_req_hdr(&req.hdr, MANA_REGISTER_HW_PORT, 1088 sizeof(req), sizeof(resp)); 1089 req.attached_gfid = 1; 1090 req.is_pf_default_vport = 1; 1091 req.allow_all_ether_types = 1; 1092 1093 err = mana_send_request(apc->ac, &req, sizeof(req), &resp, 1094 sizeof(resp)); 1095 if (err) { 1096 netdev_err(apc->ndev, "Failed to register hw vPort: %d\n", err); 1097 return err; 1098 } 1099 1100 err = mana_verify_resp_hdr(&resp.hdr, MANA_REGISTER_HW_PORT, 1101 sizeof(resp)); 1102 if (err || resp.hdr.status) { 1103 netdev_err(apc->ndev, "Failed to register hw vPort: %d, 0x%x\n", 1104 err, resp.hdr.status); 1105 return err ? err : -EPROTO; 1106 } 1107 1108 apc->port_handle = resp.hw_vport_handle; 1109 return 0; 1110 } 1111 1112 static void mana_pf_deregister_hw_vport(struct mana_port_context *apc) 1113 { 1114 struct mana_deregister_hw_vport_resp resp = {}; 1115 struct mana_deregister_hw_vport_req req = {}; 1116 int err; 1117 1118 mana_gd_init_req_hdr(&req.hdr, MANA_DEREGISTER_HW_PORT, 1119 sizeof(req), sizeof(resp)); 1120 req.hw_vport_handle = apc->port_handle; 1121 1122 err = mana_send_request(apc->ac, &req, sizeof(req), &resp, 1123 sizeof(resp)); 1124 if (err) { 1125 if (mana_en_need_log(apc, err)) 1126 netdev_err(apc->ndev, "Failed to unregister hw vPort: %d\n", 1127 err); 1128 1129 return; 1130 } 1131 1132 err = mana_verify_resp_hdr(&resp.hdr, MANA_DEREGISTER_HW_PORT, 1133 sizeof(resp)); 1134 if (err || resp.hdr.status) 1135 netdev_err(apc->ndev, 1136 "Failed to deregister hw vPort: %d, 0x%x\n", 1137 err, resp.hdr.status); 1138 } 1139 1140 static int mana_pf_register_filter(struct mana_port_context *apc) 1141 { 1142 struct mana_register_filter_resp resp = {}; 1143 struct mana_register_filter_req req = {}; 1144 int err; 1145 1146 mana_gd_init_req_hdr(&req.hdr, MANA_REGISTER_FILTER, 1147 sizeof(req), sizeof(resp)); 1148 req.vport = apc->port_handle; 1149 memcpy(req.mac_addr, apc->mac_addr, ETH_ALEN); 1150 1151 err = mana_send_request(apc->ac, &req, sizeof(req), &resp, 1152 sizeof(resp)); 1153 if (err) { 1154 netdev_err(apc->ndev, "Failed to register filter: %d\n", err); 1155 return err; 1156 } 1157 1158 err = mana_verify_resp_hdr(&resp.hdr, MANA_REGISTER_FILTER, 1159 sizeof(resp)); 1160 if (err || resp.hdr.status) { 1161 netdev_err(apc->ndev, "Failed to register filter: %d, 0x%x\n", 1162 err, resp.hdr.status); 1163 return err ? err : -EPROTO; 1164 } 1165 1166 apc->pf_filter_handle = resp.filter_handle; 1167 return 0; 1168 } 1169 1170 static void mana_pf_deregister_filter(struct mana_port_context *apc) 1171 { 1172 struct mana_deregister_filter_resp resp = {}; 1173 struct mana_deregister_filter_req req = {}; 1174 int err; 1175 1176 mana_gd_init_req_hdr(&req.hdr, MANA_DEREGISTER_FILTER, 1177 sizeof(req), sizeof(resp)); 1178 req.filter_handle = apc->pf_filter_handle; 1179 1180 err = mana_send_request(apc->ac, &req, sizeof(req), &resp, 1181 sizeof(resp)); 1182 if (err) { 1183 if (mana_en_need_log(apc, err)) 1184 netdev_err(apc->ndev, "Failed to unregister filter: %d\n", 1185 err); 1186 1187 return; 1188 } 1189 1190 err = mana_verify_resp_hdr(&resp.hdr, MANA_DEREGISTER_FILTER, 1191 sizeof(resp)); 1192 if (err || resp.hdr.status) 1193 netdev_err(apc->ndev, 1194 "Failed to deregister filter: %d, 0x%x\n", 1195 err, resp.hdr.status); 1196 } 1197 1198 int mana_gd_query_device_cfg(struct gdma_context *gc, u32 proto_major_ver, 1199 u32 proto_minor_ver, u32 proto_micro_ver, 1200 u16 *max_num_vports, u8 *bm_hostmode) 1201 { 1202 struct mana_query_device_cfg_resp resp = {}; 1203 struct mana_query_device_cfg_req req = {}; 1204 struct device *dev = gc->dev; 1205 int err = 0; 1206 1207 mana_gd_init_req_hdr(&req.hdr, MANA_QUERY_DEV_CONFIG, 1208 sizeof(req), sizeof(resp)); 1209 1210 req.hdr.resp.msg_version = GDMA_MESSAGE_V3; 1211 1212 req.proto_major_ver = proto_major_ver; 1213 req.proto_minor_ver = proto_minor_ver; 1214 req.proto_micro_ver = proto_micro_ver; 1215 1216 err = gdma_mana_send_request(gc, &req, sizeof(req), 1217 &resp, sizeof(resp)); 1218 if (err) { 1219 dev_err(dev, "Failed to query config: %d", err); 1220 return err; 1221 } 1222 1223 err = mana_verify_resp_hdr(&resp.hdr, MANA_QUERY_DEV_CONFIG, 1224 sizeof(resp)); 1225 if (err || resp.hdr.status) { 1226 dev_err(dev, "Invalid query result: %d, 0x%x\n", err, 1227 resp.hdr.status); 1228 if (!err) 1229 err = -EPROTO; 1230 return err; 1231 } 1232 1233 *max_num_vports = resp.max_num_vports; 1234 1235 if (resp.hdr.response.msg_version >= GDMA_MESSAGE_V2) { 1236 if (resp.adapter_mtu == 0) { 1237 /* 1238 * Some older PF firmware versions report an 1239 * adapter_mtu of 0. MANA hardware always supports the 1240 * standard Ethernet MTU, so fall back to ETH_FRAME_LEN. 1241 * Jumbo frames will not be available in this case. 1242 */ 1243 dev_info(dev, 1244 "PF reported adapter_mtu of 0, falling back to %u (jumbo frames disabled)\n", 1245 ETH_FRAME_LEN); 1246 gc->adapter_mtu = ETH_FRAME_LEN; 1247 } else if (resp.adapter_mtu < ETH_MIN_MTU + ETH_HLEN) { 1248 dev_err(dev, "Adapter MTU too small: %u\n", 1249 resp.adapter_mtu); 1250 return -EPROTO; 1251 } else { 1252 gc->adapter_mtu = resp.adapter_mtu; 1253 } 1254 } else { 1255 gc->adapter_mtu = ETH_FRAME_LEN; 1256 } 1257 1258 if (resp.hdr.response.msg_version >= GDMA_MESSAGE_V3) 1259 *bm_hostmode = resp.bm_hostmode; 1260 else 1261 *bm_hostmode = 0; 1262 1263 return 0; 1264 } 1265 1266 static int mana_query_vport_cfg(struct mana_port_context *apc, u32 vport_index, 1267 u32 *max_sq, u32 *max_rq, u32 *num_indir_entry) 1268 { 1269 struct mana_query_vport_cfg_resp resp = {}; 1270 struct mana_query_vport_cfg_req req = {}; 1271 int err; 1272 1273 mana_gd_init_req_hdr(&req.hdr, MANA_QUERY_VPORT_CONFIG, 1274 sizeof(req), sizeof(resp)); 1275 1276 req.vport_index = vport_index; 1277 1278 err = mana_send_request(apc->ac, &req, sizeof(req), &resp, 1279 sizeof(resp)); 1280 if (err) 1281 return err; 1282 1283 err = mana_verify_resp_hdr(&resp.hdr, MANA_QUERY_VPORT_CONFIG, 1284 sizeof(resp)); 1285 if (err) 1286 return err; 1287 1288 if (resp.hdr.status) 1289 return -EPROTO; 1290 1291 *max_sq = resp.max_num_sq; 1292 *max_rq = resp.max_num_rq; 1293 1294 if (*max_sq == 0 || *max_rq == 0) { 1295 netdev_err(apc->ndev, "Invalid max queues from vPort config\n"); 1296 return -EPROTO; 1297 } 1298 1299 if (resp.num_indirection_ent > 0 && 1300 resp.num_indirection_ent <= MANA_INDIRECT_TABLE_MAX_SIZE && 1301 is_power_of_2(resp.num_indirection_ent)) { 1302 *num_indir_entry = resp.num_indirection_ent; 1303 } else { 1304 netdev_warn(apc->ndev, 1305 "Setting indirection table size to default %d for vPort %d\n", 1306 MANA_INDIRECT_TABLE_DEF_SIZE, apc->port_idx); 1307 *num_indir_entry = MANA_INDIRECT_TABLE_DEF_SIZE; 1308 } 1309 1310 apc->port_handle = resp.vport; 1311 ether_addr_copy(apc->mac_addr, resp.mac_addr); 1312 1313 apc->vport_max_sq = *max_sq; 1314 apc->vport_max_rq = *max_rq; 1315 1316 return 0; 1317 } 1318 1319 void mana_uncfg_vport(struct mana_port_context *apc) 1320 { 1321 mutex_lock(&apc->vport_mutex); 1322 apc->vport_use_count--; 1323 WARN_ON(apc->vport_use_count < 0); 1324 mutex_unlock(&apc->vport_mutex); 1325 } 1326 EXPORT_SYMBOL_NS(mana_uncfg_vport, "NET_MANA"); 1327 1328 int mana_cfg_vport(struct mana_port_context *apc, u32 protection_dom_id, 1329 u32 doorbell_pg_id, bool check_channel_changing) 1330 { 1331 struct mana_config_vport_resp resp = {}; 1332 struct mana_config_vport_req req = {}; 1333 int err; 1334 1335 /* This function is used to program the Ethernet port in the hardware 1336 * table. It can be called from the Ethernet driver or the RDMA driver. 1337 * 1338 * For Ethernet usage, the hardware supports only one active user on a 1339 * physical port. The driver checks on the port usage before programming 1340 * the hardware when creating the RAW QP (RDMA driver) or exposing the 1341 * device to kernel NET layer (Ethernet driver). 1342 * 1343 * Because the RDMA driver doesn't know in advance which QP type the 1344 * user will create, it exposes the device with all its ports. The user 1345 * may not be able to create RAW QP on a port if this port is already 1346 * in used by the Ethernet driver from the kernel. 1347 * 1348 * This physical port limitation only applies to the RAW QP. For RC QP, 1349 * the hardware doesn't have this limitation. The user can create RC 1350 * QPs on a physical port up to the hardware limits independent of the 1351 * Ethernet usage on the same port. 1352 */ 1353 mutex_lock(&apc->vport_mutex); 1354 if (apc->vport_use_count > 0 || 1355 (check_channel_changing && apc->channel_changing)) { 1356 mutex_unlock(&apc->vport_mutex); 1357 return -EBUSY; 1358 } 1359 apc->vport_use_count++; 1360 mutex_unlock(&apc->vport_mutex); 1361 1362 mana_gd_init_req_hdr(&req.hdr, MANA_CONFIG_VPORT_TX, 1363 sizeof(req), sizeof(resp)); 1364 req.vport = apc->port_handle; 1365 req.pdid = protection_dom_id; 1366 req.doorbell_pageid = doorbell_pg_id; 1367 1368 err = mana_send_request(apc->ac, &req, sizeof(req), &resp, 1369 sizeof(resp)); 1370 if (err) { 1371 netdev_err(apc->ndev, "Failed to configure vPort: %d\n", err); 1372 goto out; 1373 } 1374 1375 err = mana_verify_resp_hdr(&resp.hdr, MANA_CONFIG_VPORT_TX, 1376 sizeof(resp)); 1377 if (err || resp.hdr.status) { 1378 netdev_err(apc->ndev, "Failed to configure vPort: %d, 0x%x\n", 1379 err, resp.hdr.status); 1380 if (!err) 1381 err = -EPROTO; 1382 1383 goto out; 1384 } 1385 1386 apc->tx_shortform_allowed = resp.short_form_allowed; 1387 apc->tx_vp_offset = resp.tx_vport_offset; 1388 1389 netdev_info(apc->ndev, "Enabled vPort %llu PD %u DB %u MAC %pM\n", 1390 apc->port_handle, protection_dom_id, doorbell_pg_id, apc->mac_addr); 1391 out: 1392 if (err) 1393 mana_uncfg_vport(apc); 1394 1395 return err; 1396 } 1397 EXPORT_SYMBOL_NS(mana_cfg_vport, "NET_MANA"); 1398 1399 static int mana_cfg_vport_steering(struct mana_port_context *apc, 1400 enum TRI_STATE rx, 1401 bool update_default_rxobj, bool update_key, 1402 bool update_tab) 1403 { 1404 struct mana_cfg_rx_steer_req_v2 *req; 1405 struct mana_cfg_rx_steer_resp resp = {}; 1406 struct net_device *ndev = apc->ndev; 1407 u32 req_buf_size; 1408 int err; 1409 1410 req_buf_size = struct_size(req, indir_tab, apc->indir_table_sz); 1411 req = kzalloc(req_buf_size, GFP_KERNEL); 1412 if (!req) 1413 return -ENOMEM; 1414 1415 mana_gd_init_req_hdr(&req->hdr, MANA_CONFIG_VPORT_RX, req_buf_size, 1416 sizeof(resp)); 1417 1418 req->hdr.req.msg_version = GDMA_MESSAGE_V2; 1419 req->hdr.resp.msg_version = GDMA_MESSAGE_V2; 1420 1421 req->vport = apc->port_handle; 1422 req->num_indir_entries = apc->indir_table_sz; 1423 req->indir_tab_offset = offsetof(struct mana_cfg_rx_steer_req_v2, 1424 indir_tab); 1425 req->rx_enable = rx; 1426 req->rss_enable = apc->rss_state; 1427 req->update_default_rxobj = update_default_rxobj; 1428 req->update_hashkey = update_key; 1429 req->update_indir_tab = update_tab; 1430 req->default_rxobj = apc->default_rxobj; 1431 1432 if (rx != TRI_STATE_FALSE) 1433 req->cqe_coalescing_enable = apc->cqe_coalescing_enable; 1434 1435 if (update_key) 1436 memcpy(&req->hashkey, apc->hashkey, MANA_HASH_KEY_SIZE); 1437 1438 if (update_tab) 1439 memcpy(req->indir_tab, apc->rxobj_table, 1440 flex_array_size(req, indir_tab, req->num_indir_entries)); 1441 1442 err = mana_send_request(apc->ac, req, req_buf_size, &resp, 1443 sizeof(resp)); 1444 if (err) { 1445 if (mana_en_need_log(apc, err)) 1446 netdev_err(ndev, "Failed to configure vPort RX: %d\n", err); 1447 1448 goto out; 1449 } 1450 1451 err = mana_verify_resp_hdr(&resp.hdr, MANA_CONFIG_VPORT_RX, 1452 sizeof(resp)); 1453 if (err) { 1454 netdev_err(ndev, "vPort RX configuration failed: %d\n", err); 1455 goto out; 1456 } 1457 1458 if (resp.hdr.status) { 1459 netdev_err(ndev, "vPort RX configuration failed: 0x%x\n", 1460 resp.hdr.status); 1461 err = -EPROTO; 1462 goto out; 1463 } 1464 1465 if (resp.hdr.response.msg_version >= GDMA_MESSAGE_V2) 1466 apc->cqe_coalescing_timeout_ns = 1467 resp.cqe_coalescing_timeout_ns; 1468 1469 netdev_info(ndev, "Configured steering vPort %llu entries %u\n", 1470 apc->port_handle, apc->indir_table_sz); 1471 1472 apc->steer_rx = rx; 1473 apc->steer_rss = apc->rss_state; 1474 apc->steer_update_tab = update_tab; 1475 apc->steer_cqe_coalescing = req->cqe_coalescing_enable; 1476 out: 1477 kfree(req); 1478 return err; 1479 } 1480 1481 int mana_query_link_cfg(struct mana_port_context *apc) 1482 { 1483 struct net_device *ndev = apc->ndev; 1484 struct mana_query_link_config_resp resp = {}; 1485 struct mana_query_link_config_req req = {}; 1486 int err; 1487 1488 netdev_assert_locked(ndev); 1489 1490 err = apc->link_cfg_error; 1491 if (err <= 0) 1492 return err; 1493 1494 mana_gd_init_req_hdr(&req.hdr, MANA_QUERY_LINK_CONFIG, 1495 sizeof(req), sizeof(resp)); 1496 1497 req.vport = apc->port_handle; 1498 req.hdr.resp.msg_version = GDMA_MESSAGE_V2; 1499 1500 err = mana_send_request(apc->ac, &req, sizeof(req), &resp, 1501 sizeof(resp)); 1502 1503 if (err) { 1504 if (err == -EOPNOTSUPP) { 1505 netdev_info_once(ndev, "MANA_QUERY_LINK_CONFIG not supported\n"); 1506 apc->link_cfg_error = err; 1507 return err; 1508 } 1509 netdev_err(ndev, "Failed to query link config: %d\n", err); 1510 return err; 1511 } 1512 1513 err = mana_verify_resp_hdr(&resp.hdr, MANA_QUERY_LINK_CONFIG, 1514 sizeof(resp)); 1515 1516 if (err || resp.hdr.status) { 1517 netdev_err(ndev, "Failed to query link config: %d, 0x%x\n", err, 1518 resp.hdr.status); 1519 if (!err) 1520 err = -EOPNOTSUPP; 1521 return err; 1522 } 1523 1524 if (resp.qos_unconfigured) 1525 return -EINVAL; 1526 1527 apc->speed = resp.link_speed_mbps; 1528 apc->max_speed = resp.qos_speed_mbps; 1529 apc->link_cfg_error = 0; 1530 return 0; 1531 } 1532 1533 int mana_set_bw_clamp(struct mana_port_context *apc, u32 speed, 1534 int enable_clamping) 1535 { 1536 struct mana_set_bw_clamp_resp resp = {}; 1537 struct mana_set_bw_clamp_req req = {}; 1538 struct net_device *ndev = apc->ndev; 1539 int err; 1540 1541 netdev_assert_locked(ndev); 1542 1543 mana_gd_init_req_hdr(&req.hdr, MANA_SET_BW_CLAMP, 1544 sizeof(req), sizeof(resp)); 1545 req.vport = apc->port_handle; 1546 req.link_speed_mbps = speed; 1547 req.enable_clamping = enable_clamping; 1548 1549 err = mana_send_request(apc->ac, &req, sizeof(req), &resp, 1550 sizeof(resp)); 1551 1552 if (err) { 1553 if (err == -EOPNOTSUPP) { 1554 netdev_info_once(ndev, "MANA_SET_BW_CLAMP not supported\n"); 1555 return err; 1556 } 1557 netdev_err(ndev, "Failed to set bandwidth clamp for speed %u, err = %d", 1558 speed, err); 1559 return err; 1560 } 1561 1562 err = mana_verify_resp_hdr(&resp.hdr, MANA_SET_BW_CLAMP, 1563 sizeof(resp)); 1564 1565 if (err || resp.hdr.status) { 1566 netdev_err(ndev, "Failed to set bandwidth clamp: %d, 0x%x\n", err, 1567 resp.hdr.status); 1568 if (!err) 1569 err = -EOPNOTSUPP; 1570 return err; 1571 } 1572 1573 if (resp.qos_unconfigured) 1574 netdev_info(ndev, "QoS is unconfigured\n"); 1575 1576 /* Invalidate the cache; next query will re-fetch from firmware. */ 1577 apc->link_cfg_error = 1; 1578 return 0; 1579 } 1580 1581 int mana_create_wq_obj(struct mana_port_context *apc, 1582 mana_handle_t vport, 1583 u32 wq_type, struct mana_obj_spec *wq_spec, 1584 struct mana_obj_spec *cq_spec, 1585 mana_handle_t *wq_obj) 1586 { 1587 struct mana_create_wqobj_resp resp = {}; 1588 struct mana_create_wqobj_req req = {}; 1589 struct net_device *ndev = apc->ndev; 1590 int err; 1591 1592 mana_gd_init_req_hdr(&req.hdr, MANA_CREATE_WQ_OBJ, 1593 sizeof(req), sizeof(resp)); 1594 req.vport = vport; 1595 req.wq_type = wq_type; 1596 req.wq_gdma_region = wq_spec->gdma_region; 1597 req.cq_gdma_region = cq_spec->gdma_region; 1598 req.wq_size = wq_spec->queue_size; 1599 req.cq_size = cq_spec->queue_size; 1600 req.cq_moderation_ctx_id = cq_spec->modr_ctx_id; 1601 req.cq_parent_qid = cq_spec->attached_eq; 1602 1603 err = mana_send_request(apc->ac, &req, sizeof(req), &resp, 1604 sizeof(resp)); 1605 if (err) { 1606 netdev_err(ndev, "Failed to create WQ object: %d\n", err); 1607 goto out; 1608 } 1609 1610 err = mana_verify_resp_hdr(&resp.hdr, MANA_CREATE_WQ_OBJ, 1611 sizeof(resp)); 1612 if (err || resp.hdr.status) { 1613 netdev_err(ndev, "Failed to create WQ object: %d, 0x%x\n", err, 1614 resp.hdr.status); 1615 if (!err) 1616 err = -EPROTO; 1617 goto out; 1618 } 1619 1620 if (resp.wq_obj == INVALID_MANA_HANDLE) { 1621 netdev_err(ndev, "Got an invalid WQ object handle\n"); 1622 err = -EPROTO; 1623 goto out; 1624 } 1625 1626 *wq_obj = resp.wq_obj; 1627 wq_spec->queue_index = resp.wq_id; 1628 cq_spec->queue_index = resp.cq_id; 1629 1630 return 0; 1631 out: 1632 return err; 1633 } 1634 EXPORT_SYMBOL_NS(mana_create_wq_obj, "NET_MANA"); 1635 1636 void mana_destroy_wq_obj(struct mana_port_context *apc, u32 wq_type, 1637 mana_handle_t wq_obj) 1638 { 1639 struct mana_destroy_wqobj_resp resp = {}; 1640 struct mana_destroy_wqobj_req req = {}; 1641 struct net_device *ndev = apc->ndev; 1642 int err; 1643 1644 mana_gd_init_req_hdr(&req.hdr, MANA_DESTROY_WQ_OBJ, 1645 sizeof(req), sizeof(resp)); 1646 req.wq_type = wq_type; 1647 req.wq_obj_handle = wq_obj; 1648 1649 err = mana_send_request(apc->ac, &req, sizeof(req), &resp, 1650 sizeof(resp)); 1651 if (err) { 1652 if (mana_en_need_log(apc, err)) 1653 netdev_err(ndev, "Failed to destroy WQ object: %d\n", err); 1654 1655 return; 1656 } 1657 1658 err = mana_verify_resp_hdr(&resp.hdr, MANA_DESTROY_WQ_OBJ, 1659 sizeof(resp)); 1660 if (err || resp.hdr.status) 1661 netdev_err(ndev, "Failed to destroy WQ object: %d, 0x%x\n", err, 1662 resp.hdr.status); 1663 } 1664 EXPORT_SYMBOL_NS(mana_destroy_wq_obj, "NET_MANA"); 1665 1666 void mana_destroy_eq(struct mana_port_context *apc) 1667 { 1668 struct mana_context *ac = apc->ac; 1669 struct gdma_context *gc = ac->gdma_dev->gdma_context; 1670 struct gdma_queue *eq; 1671 unsigned int msi; 1672 int i; 1673 1674 if (!apc->eqs) 1675 return; 1676 1677 debugfs_remove_recursive(apc->mana_eqs_debugfs); 1678 apc->mana_eqs_debugfs = NULL; 1679 1680 for (i = 0; i < apc->num_queues; i++) { 1681 eq = apc->eqs[i].eq; 1682 if (!eq) 1683 continue; 1684 1685 msi = eq->eq.msix_index; 1686 mana_gd_destroy_queue(gc, eq); 1687 mana_gd_put_gic(gc, !gc->msi_sharing, msi); 1688 } 1689 1690 kfree(apc->eqs); 1691 apc->eqs = NULL; 1692 } 1693 EXPORT_SYMBOL_NS(mana_destroy_eq, "NET_MANA"); 1694 1695 static void mana_create_eq_debugfs(struct mana_port_context *apc, int i) 1696 { 1697 struct mana_eq eq = apc->eqs[i]; 1698 char eqnum[32]; 1699 1700 sprintf(eqnum, "eq%d", i); 1701 eq.mana_eq_debugfs = debugfs_create_dir(eqnum, apc->mana_eqs_debugfs); 1702 debugfs_create_u32("head", 0400, eq.mana_eq_debugfs, &eq.eq->head); 1703 debugfs_create_u32("tail", 0400, eq.mana_eq_debugfs, &eq.eq->tail); 1704 debugfs_create_u32("irq", 0400, eq.mana_eq_debugfs, &eq.eq->eq.irq); 1705 debugfs_create_file("eq_dump", 0400, eq.mana_eq_debugfs, eq.eq, &mana_dbg_q_fops); 1706 } 1707 1708 int mana_create_eq(struct mana_port_context *apc) 1709 { 1710 struct gdma_dev *gd = apc->ac->gdma_dev; 1711 struct gdma_context *gc = gd->gdma_context; 1712 struct gdma_queue_spec spec = {}; 1713 struct gdma_irq_context *gic; 1714 int err; 1715 int msi; 1716 int i; 1717 1718 if (WARN_ON(apc->eqs)) 1719 return -EEXIST; 1720 apc->eqs = kzalloc_objs(struct mana_eq, apc->num_queues); 1721 if (!apc->eqs) 1722 return -ENOMEM; 1723 1724 spec.type = GDMA_EQ; 1725 spec.monitor_avl_buf = false; 1726 spec.queue_size = EQ_SIZE; 1727 spec.eq.callback = NULL; 1728 spec.eq.context = apc->eqs; 1729 spec.eq.log2_throttle_limit = LOG2_EQ_THROTTLE; 1730 1731 apc->mana_eqs_debugfs = 1732 debugfs_create_dir("EQs", apc->mana_port_debugfs); 1733 1734 for (i = 0; i < apc->num_queues; i++) { 1735 msi = (i + 1) % gc->num_msix_usable; 1736 1737 gic = mana_gd_get_gic(gc, !gc->msi_sharing, &msi); 1738 if (IS_ERR(gic)) { 1739 err = PTR_ERR(gic); 1740 goto out; 1741 } 1742 spec.eq.msix_index = msi; 1743 1744 err = mana_gd_create_mana_eq(gd, &spec, &apc->eqs[i].eq); 1745 if (err) { 1746 dev_err(gc->dev, "Failed to create EQ %d : %d\n", i, err); 1747 mana_gd_put_gic(gc, !gc->msi_sharing, msi); 1748 goto out; 1749 } 1750 apc->eqs[i].eq->eq.irq = gic->irq; 1751 mana_create_eq_debugfs(apc, i); 1752 } 1753 1754 return 0; 1755 out: 1756 mana_destroy_eq(apc); 1757 return err; 1758 } 1759 EXPORT_SYMBOL_NS(mana_create_eq, "NET_MANA"); 1760 1761 static int mana_fence_rq(struct mana_port_context *apc, struct mana_rxq *rxq) 1762 { 1763 struct mana_fence_rq_resp resp = {}; 1764 struct mana_fence_rq_req req = {}; 1765 int err; 1766 1767 init_completion(&rxq->fence_event); 1768 1769 mana_gd_init_req_hdr(&req.hdr, MANA_FENCE_RQ, 1770 sizeof(req), sizeof(resp)); 1771 req.wq_obj_handle = rxq->rxobj; 1772 1773 err = mana_send_request(apc->ac, &req, sizeof(req), &resp, 1774 sizeof(resp)); 1775 if (err) { 1776 netdev_err(apc->ndev, "Failed to fence RQ %u: %d\n", 1777 rxq->rxq_idx, err); 1778 return err; 1779 } 1780 1781 err = mana_verify_resp_hdr(&resp.hdr, MANA_FENCE_RQ, sizeof(resp)); 1782 if (err || resp.hdr.status) { 1783 netdev_err(apc->ndev, "Failed to fence RQ %u: %d, 0x%x\n", 1784 rxq->rxq_idx, err, resp.hdr.status); 1785 if (!err) 1786 err = -EPROTO; 1787 1788 return err; 1789 } 1790 1791 if (wait_for_completion_timeout(&rxq->fence_event, 10 * HZ) == 0) { 1792 netdev_err(apc->ndev, "Failed to fence RQ %u: timed out\n", 1793 rxq->rxq_idx); 1794 return -ETIMEDOUT; 1795 } 1796 1797 return 0; 1798 } 1799 1800 static void mana_fence_rqs(struct mana_port_context *apc) 1801 { 1802 unsigned int rxq_idx; 1803 struct mana_rxq *rxq; 1804 int err; 1805 1806 if (!apc->rxqs) 1807 return; 1808 1809 for (rxq_idx = 0; rxq_idx < apc->num_queues; rxq_idx++) { 1810 rxq = apc->rxqs[rxq_idx]; 1811 err = mana_fence_rq(apc, rxq); 1812 1813 /* In case of any error, use sleep instead. */ 1814 if (err) 1815 msleep(100); 1816 } 1817 } 1818 1819 static int mana_move_wq_tail(struct gdma_queue *wq, u32 num_units) 1820 { 1821 u32 used_space_old; 1822 u32 used_space_new; 1823 1824 used_space_old = wq->head - wq->tail; 1825 used_space_new = wq->head - (wq->tail + num_units); 1826 1827 if (WARN_ON_ONCE(used_space_new > used_space_old)) 1828 return -ERANGE; 1829 1830 wq->tail += num_units; 1831 return 0; 1832 } 1833 1834 void mana_unmap_skb(struct sk_buff *skb, struct mana_port_context *apc) 1835 { 1836 struct mana_skb_head *ash = (struct mana_skb_head *)skb->head; 1837 struct gdma_context *gc = apc->ac->gdma_dev->gdma_context; 1838 struct device *dev = gc->dev; 1839 int hsg, i; 1840 1841 /* Number of SGEs of linear part */ 1842 hsg = (skb_is_gso(skb) && skb_headlen(skb) > ash->size[0]) ? 2 : 1; 1843 1844 for (i = 0; i < hsg; i++) 1845 dma_unmap_single(dev, ash->dma_handle[i], ash->size[i], 1846 DMA_TO_DEVICE); 1847 1848 for (i = hsg; i < skb_shinfo(skb)->nr_frags + hsg; i++) 1849 dma_unmap_page(dev, ash->dma_handle[i], ash->size[i], 1850 DMA_TO_DEVICE); 1851 } 1852 1853 static void mana_poll_tx_cq(struct mana_cq *cq) 1854 { 1855 struct gdma_comp *completions = cq->gdma_comp_buf; 1856 struct gdma_posted_wqe_info *wqe_info; 1857 unsigned int pkt_transmitted = 0; 1858 unsigned int wqe_unit_cnt = 0; 1859 struct mana_txq *txq = cq->txq; 1860 struct mana_port_context *apc; 1861 struct netdev_queue *net_txq; 1862 struct gdma_queue *gdma_wq; 1863 unsigned int avail_space; 1864 struct net_device *ndev; 1865 struct sk_buff *skb; 1866 bool txq_stopped; 1867 int comp_read; 1868 int i; 1869 1870 ndev = txq->ndev; 1871 apc = netdev_priv(ndev); 1872 1873 /* Limit CQEs polled to 4 wraparounds of the CQ to ensure the 1874 * doorbell can be rung in time for the hardware's requirement 1875 * of at least one doorbell ring every 8 wraparounds. 1876 */ 1877 comp_read = mana_gd_poll_cq(cq->gdma_cq, completions, 1878 min((cq->gdma_cq->queue_size / 1879 COMP_ENTRY_SIZE) * 4, 1880 CQE_POLLING_BUFFER)); 1881 1882 if (comp_read < 1) 1883 return; 1884 1885 for (i = 0; i < comp_read; i++) { 1886 struct mana_tx_comp_oob *cqe_oob; 1887 1888 if (WARN_ON_ONCE(!completions[i].is_sq)) 1889 return; 1890 1891 cqe_oob = (struct mana_tx_comp_oob *)completions[i].cqe_data; 1892 if (WARN_ON_ONCE(cqe_oob->cqe_hdr.client_type != 1893 MANA_CQE_COMPLETION)) 1894 return; 1895 1896 switch (cqe_oob->cqe_hdr.cqe_type) { 1897 case CQE_TX_OKAY: 1898 break; 1899 1900 case CQE_TX_SA_DROP: 1901 case CQE_TX_MTU_DROP: 1902 case CQE_TX_INVALID_OOB: 1903 case CQE_TX_INVALID_ETH_TYPE: 1904 case CQE_TX_HDR_PROCESSING_ERROR: 1905 case CQE_TX_VF_DISABLED: 1906 case CQE_TX_VPORT_IDX_OUT_OF_RANGE: 1907 case CQE_TX_VPORT_DISABLED: 1908 case CQE_TX_VLAN_TAGGING_VIOLATION: 1909 if (net_ratelimit()) 1910 netdev_err(ndev, "TX: CQE error %d\n", 1911 cqe_oob->cqe_hdr.cqe_type); 1912 1913 apc->eth_stats.tx_cqe_err++; 1914 break; 1915 1916 default: 1917 /* If the CQE type is unknown, log an error, 1918 * and still free the SKB, update tail, etc. 1919 */ 1920 if (net_ratelimit()) 1921 netdev_err(ndev, "TX: unknown CQE type %d\n", 1922 cqe_oob->cqe_hdr.cqe_type); 1923 1924 apc->eth_stats.tx_cqe_unknown_type++; 1925 break; 1926 } 1927 1928 if (WARN_ON_ONCE(txq->gdma_txq_id != completions[i].wq_num)) 1929 return; 1930 1931 skb = skb_dequeue(&txq->pending_skbs); 1932 if (WARN_ON_ONCE(!skb)) 1933 return; 1934 1935 wqe_info = (struct gdma_posted_wqe_info *)skb->cb; 1936 wqe_unit_cnt += wqe_info->wqe_size_in_bu; 1937 1938 mana_unmap_skb(skb, apc); 1939 1940 napi_consume_skb(skb, cq->budget); 1941 1942 pkt_transmitted++; 1943 } 1944 1945 if (WARN_ON_ONCE(wqe_unit_cnt == 0)) 1946 return; 1947 1948 mana_move_wq_tail(txq->gdma_sq, wqe_unit_cnt); 1949 1950 gdma_wq = txq->gdma_sq; 1951 avail_space = mana_gd_wq_avail_space(gdma_wq); 1952 1953 /* Ensure tail updated before checking q stop */ 1954 smp_mb(); 1955 1956 net_txq = txq->net_txq; 1957 txq_stopped = netif_tx_queue_stopped(net_txq); 1958 1959 /* Ensure checking txq_stopped before apc->port_is_up. */ 1960 smp_rmb(); 1961 1962 if (txq_stopped && apc->port_is_up && avail_space >= MAX_TX_WQE_SIZE) { 1963 netif_tx_wake_queue(net_txq); 1964 apc->eth_stats.wake_queue++; 1965 } 1966 1967 if (atomic_sub_return(pkt_transmitted, &txq->pending_sends) < 0) 1968 WARN_ON_ONCE(1); 1969 1970 cq->work_done = pkt_transmitted; 1971 } 1972 1973 static void mana_post_pkt_rxq(struct mana_rxq *rxq) 1974 { 1975 struct mana_recv_buf_oob *recv_buf_oob; 1976 u32 curr_index; 1977 int err; 1978 1979 curr_index = rxq->buf_index++; 1980 if (rxq->buf_index == rxq->num_rx_buf) 1981 rxq->buf_index = 0; 1982 1983 recv_buf_oob = &rxq->rx_oobs[curr_index]; 1984 1985 err = mana_gd_post_work_request(rxq->gdma_rq, &recv_buf_oob->wqe_req, 1986 &recv_buf_oob->wqe_inf); 1987 if (WARN_ON_ONCE(err)) 1988 return; 1989 1990 WARN_ON_ONCE(recv_buf_oob->wqe_inf.wqe_size_in_bu != 1); 1991 } 1992 1993 static struct sk_buff *mana_build_skb(struct mana_rxq *rxq, void *buf_va, 1994 uint pkt_len, struct xdp_buff *xdp) 1995 { 1996 struct sk_buff *skb = napi_build_skb(buf_va, rxq->alloc_size); 1997 1998 if (!skb) 1999 return NULL; 2000 2001 if (xdp->data_hard_start) { 2002 u32 metasize = xdp->data - xdp->data_meta; 2003 2004 skb_reserve(skb, xdp->data - xdp->data_hard_start); 2005 skb_put(skb, xdp->data_end - xdp->data); 2006 if (metasize) 2007 skb_metadata_set(skb, metasize); 2008 return skb; 2009 } 2010 2011 skb_reserve(skb, rxq->headroom); 2012 skb_put(skb, pkt_len); 2013 2014 return skb; 2015 } 2016 2017 static void mana_rx_skb(void *buf_va, bool from_pool, 2018 struct mana_rxcomp_oob *cqe, struct mana_rxq *rxq, 2019 int i) 2020 { 2021 struct mana_stats_rx *rx_stats = &rxq->stats; 2022 struct net_device *ndev = rxq->ndev; 2023 uint pkt_len = cqe->ppi[i].pkt_len; 2024 u16 rxq_idx = rxq->rxq_idx; 2025 struct napi_struct *napi; 2026 struct xdp_buff xdp = {}; 2027 struct sk_buff *skb; 2028 u32 hash_value; 2029 u32 act; 2030 2031 rxq->rx_cq.work_done++; 2032 napi = &rxq->rx_cq.napi; 2033 2034 if (!buf_va) { 2035 ++ndev->stats.rx_dropped; 2036 return; 2037 } 2038 2039 act = mana_run_xdp(ndev, rxq, &xdp, buf_va, pkt_len); 2040 2041 if (act == XDP_REDIRECT && !rxq->xdp_rc) 2042 return; 2043 2044 if (act != XDP_PASS && act != XDP_TX) 2045 goto drop_xdp; 2046 2047 skb = mana_build_skb(rxq, buf_va, pkt_len, &xdp); 2048 2049 if (!skb) 2050 goto drop; 2051 2052 if (from_pool) 2053 skb_mark_for_recycle(skb); 2054 2055 skb->dev = napi->dev; 2056 2057 skb->protocol = eth_type_trans(skb, ndev); 2058 skb_checksum_none_assert(skb); 2059 skb_record_rx_queue(skb, rxq_idx); 2060 2061 if ((ndev->features & NETIF_F_RXCSUM) && cqe->rx_iphdr_csum_succeed) { 2062 if (cqe->rx_tcp_csum_succeed || cqe->rx_udp_csum_succeed) 2063 skb->ip_summed = CHECKSUM_UNNECESSARY; 2064 } 2065 2066 if (cqe->rx_hashtype != 0 && (ndev->features & NETIF_F_RXHASH)) { 2067 hash_value = cqe->ppi[i].pkt_hash; 2068 2069 if (cqe->rx_hashtype & MANA_HASH_L4) 2070 skb_set_hash(skb, hash_value, PKT_HASH_TYPE_L4); 2071 else 2072 skb_set_hash(skb, hash_value, PKT_HASH_TYPE_L3); 2073 } 2074 2075 if (cqe->rx_vlantag_present) { 2076 u16 vlan_tci = cqe->rx_vlan_id; 2077 2078 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vlan_tci); 2079 } 2080 2081 u64_stats_update_begin(&rx_stats->syncp); 2082 rx_stats->packets++; 2083 rx_stats->bytes += pkt_len; 2084 2085 if (act == XDP_TX) 2086 rx_stats->xdp_tx++; 2087 u64_stats_update_end(&rx_stats->syncp); 2088 2089 if (act == XDP_TX) { 2090 skb_set_queue_mapping(skb, rxq_idx); 2091 mana_xdp_tx(skb, ndev); 2092 return; 2093 } 2094 2095 napi_gro_receive(napi, skb); 2096 2097 return; 2098 2099 drop_xdp: 2100 u64_stats_update_begin(&rx_stats->syncp); 2101 rx_stats->xdp_drop++; 2102 u64_stats_update_end(&rx_stats->syncp); 2103 2104 drop: 2105 if (from_pool) { 2106 if (rxq->frag_count == 1) 2107 page_pool_recycle_direct(rxq->page_pool, 2108 virt_to_head_page(buf_va)); 2109 else 2110 page_pool_free_va(rxq->page_pool, buf_va, true); 2111 } else { 2112 WARN_ON_ONCE(rxq->xdp_save_va); 2113 /* Save for reuse */ 2114 rxq->xdp_save_va = buf_va; 2115 } 2116 2117 ++ndev->stats.rx_dropped; 2118 2119 return; 2120 } 2121 2122 static void *mana_get_rxfrag(struct mana_rxq *rxq, struct device *dev, 2123 dma_addr_t *da, bool *from_pool, 2124 struct page **pp_page, u32 *dma_sync_offset) 2125 { 2126 struct page *page; 2127 u32 offset; 2128 void *va; 2129 2130 *from_pool = false; 2131 *pp_page = NULL; 2132 *dma_sync_offset = 0; 2133 2134 /* Don't use fragments for jumbo frames or XDP where it's 1 fragment 2135 * per page. 2136 */ 2137 if (rxq->frag_count == 1) { 2138 /* Reuse XDP dropped page if available */ 2139 if (rxq->xdp_save_va) { 2140 va = rxq->xdp_save_va; 2141 page = virt_to_head_page(va); 2142 rxq->xdp_save_va = NULL; 2143 } else { 2144 page = page_pool_dev_alloc_pages(rxq->page_pool); 2145 if (!page) 2146 return NULL; 2147 2148 *from_pool = true; 2149 va = page_to_virt(page); 2150 } 2151 2152 *da = dma_map_single(dev, va + rxq->headroom, rxq->datasize, 2153 DMA_FROM_DEVICE); 2154 if (dma_mapping_error(dev, *da)) { 2155 mana_put_rx_page(rxq, page, *from_pool); 2156 return NULL; 2157 } 2158 2159 return va; 2160 } 2161 2162 page = page_pool_dev_alloc_frag(rxq->page_pool, &offset, 2163 rxq->alloc_size); 2164 if (!page) 2165 return NULL; 2166 2167 va = page_to_virt(page) + offset; 2168 *da = page_pool_get_dma_addr(page) + offset + rxq->headroom; 2169 *from_pool = true; 2170 *pp_page = page; 2171 *dma_sync_offset = offset + rxq->headroom; 2172 2173 return va; 2174 } 2175 2176 /* Allocate frag for rx buffer, and save the old buf */ 2177 static void mana_refill_rx_oob(struct device *dev, struct mana_rxq *rxq, 2178 struct mana_recv_buf_oob *rxoob, u32 pktlen, 2179 void **old_buf, bool *old_fp) 2180 { 2181 struct page *pp_page; 2182 u32 dma_sync_offset; 2183 bool from_pool; 2184 dma_addr_t da; 2185 void *va; 2186 2187 va = mana_get_rxfrag(rxq, dev, &da, &from_pool, &pp_page, 2188 &dma_sync_offset); 2189 if (!va) 2190 return; 2191 if (!rxoob->from_pool || rxq->frag_count == 1) { 2192 dma_unmap_single(dev, rxoob->sgl[0].address, rxq->datasize, 2193 DMA_FROM_DEVICE); 2194 } else { 2195 /* The page pool maps the whole page and only syncs for device 2196 * automatically (PP_FLAG_DMA_SYNC_DEV). Sync the received bytes 2197 * for the CPU before they are read: this is required if DMA 2198 * is incoherent or bounce buffers are used. 2199 */ 2200 page_pool_dma_sync_for_cpu(rxq->page_pool, rxoob->pp_page, 2201 rxoob->dma_sync_offset, pktlen); 2202 } 2203 *old_buf = rxoob->buf_va; 2204 *old_fp = rxoob->from_pool; 2205 2206 rxoob->buf_va = va; 2207 rxoob->sgl[0].address = da; 2208 rxoob->from_pool = from_pool; 2209 rxoob->pp_page = pp_page; 2210 rxoob->dma_sync_offset = dma_sync_offset; 2211 } 2212 2213 static void mana_process_rx_cqe(struct mana_rxq *rxq, struct mana_cq *cq, 2214 struct gdma_comp *cqe) 2215 { 2216 struct mana_rxcomp_oob *oob = (struct mana_rxcomp_oob *)cqe->cqe_data; 2217 struct gdma_context *gc = rxq->gdma_rq->gdma_dev->gdma_context; 2218 struct net_device *ndev = rxq->ndev; 2219 struct mana_recv_buf_oob *rxbuf_oob; 2220 struct mana_port_context *apc; 2221 struct device *dev = gc->dev; 2222 bool coalesced = false; 2223 void *old_buf = NULL; 2224 u32 curr, pktlen; 2225 bool old_fp; 2226 int i; 2227 2228 apc = netdev_priv(ndev); 2229 2230 switch (oob->cqe_hdr.cqe_type) { 2231 case CQE_RX_OKAY: 2232 break; 2233 2234 case CQE_RX_TRUNCATED: 2235 ++ndev->stats.rx_dropped; 2236 rxbuf_oob = &rxq->rx_oobs[rxq->buf_index]; 2237 netdev_warn_once(ndev, "Dropped a truncated packet\n"); 2238 2239 mana_move_wq_tail(rxq->gdma_rq, 2240 rxbuf_oob->wqe_inf.wqe_size_in_bu); 2241 mana_post_pkt_rxq(rxq); 2242 return; 2243 2244 case CQE_RX_COALESCED_4: 2245 coalesced = true; 2246 break; 2247 2248 case CQE_RX_OBJECT_FENCE: 2249 complete(&rxq->fence_event); 2250 return; 2251 2252 default: 2253 netdev_err(ndev, "Unknown RX CQE type = %d\n", 2254 oob->cqe_hdr.cqe_type); 2255 apc->eth_stats.rx_cqe_unknown_type++; 2256 return; 2257 } 2258 2259 for (i = 0; i < MANA_RXCOMP_OOB_NUM_PPI; i++) { 2260 old_buf = NULL; 2261 pktlen = oob->ppi[i].pkt_len; 2262 if (pktlen == 0) 2263 break; 2264 2265 curr = rxq->buf_index; 2266 rxbuf_oob = &rxq->rx_oobs[curr]; 2267 WARN_ON_ONCE(rxbuf_oob->wqe_inf.wqe_size_in_bu != 1); 2268 2269 if (unlikely(pktlen > rxq->datasize)) { 2270 /* Increase it even if mana_rx_skb() isn't called. */ 2271 rxq->rx_cq.work_done++; 2272 2273 ++ndev->stats.rx_dropped; 2274 netdev_warn_once(ndev, 2275 "Dropped oversized RX packet: len=%u, datasize=%u\n", 2276 pktlen, rxq->datasize); 2277 2278 /* Reuse the RX buffer since rxbuf_oob is unchanged. */ 2279 } else { 2280 2281 mana_refill_rx_oob(dev, rxq, rxbuf_oob, pktlen, 2282 &old_buf, &old_fp); 2283 2284 /* Unsuccessful refill will have old_buf == NULL. 2285 * In this case, mana_rx_skb() will drop the packet. 2286 */ 2287 mana_rx_skb(old_buf, old_fp, oob, rxq, i); 2288 } 2289 2290 mana_move_wq_tail(rxq->gdma_rq, 2291 rxbuf_oob->wqe_inf.wqe_size_in_bu); 2292 2293 mana_post_pkt_rxq(rxq); 2294 2295 if (!coalesced) 2296 break; 2297 } 2298 2299 /* Collect coalesced CQE count based on packets processed. 2300 * Coalesced CQEs have at least 2 packets, so index is i - 2. 2301 */ 2302 if (i > 1) { 2303 u64_stats_update_begin(&rxq->stats.syncp); 2304 rxq->stats.coalesced_cqe[i - 2]++; 2305 u64_stats_update_end(&rxq->stats.syncp); 2306 } else if (!i && !pktlen) { 2307 u64_stats_update_begin(&rxq->stats.syncp); 2308 rxq->stats.pkt_len0_err++; 2309 u64_stats_update_end(&rxq->stats.syncp); 2310 netdev_err_once(ndev, 2311 "RX pkt len=0, rq=%u, cq=%u, rxobj=0x%llx\n", 2312 rxq->gdma_id, cq->gdma_id, rxq->rxobj); 2313 } 2314 } 2315 2316 static void mana_poll_rx_cq(struct mana_cq *cq) 2317 { 2318 struct gdma_comp *comp = cq->gdma_comp_buf; 2319 struct mana_rxq *rxq = cq->rxq; 2320 int comp_read, i; 2321 2322 /* Limit CQEs polled to 4 wraparounds of the CQ to ensure the 2323 * doorbell can be rung in time for the hardware's requirement 2324 * of at least one doorbell ring every 8 wraparounds. 2325 */ 2326 comp_read = mana_gd_poll_cq(cq->gdma_cq, comp, 2327 min((cq->gdma_cq->queue_size / 2328 COMP_ENTRY_SIZE) * 4, 2329 CQE_POLLING_BUFFER)); 2330 WARN_ON_ONCE(comp_read > CQE_POLLING_BUFFER); 2331 2332 rxq->xdp_flush = false; 2333 2334 for (i = 0; i < comp_read; i++) { 2335 if (WARN_ON_ONCE(comp[i].is_sq)) 2336 return; 2337 2338 /* verify recv cqe references the right rxq */ 2339 if (WARN_ON_ONCE(comp[i].wq_num != cq->rxq->gdma_id)) 2340 return; 2341 2342 mana_process_rx_cqe(rxq, cq, &comp[i]); 2343 } 2344 2345 if (comp_read > 0) { 2346 struct gdma_context *gc = rxq->gdma_rq->gdma_dev->gdma_context; 2347 2348 mana_gd_wq_ring_doorbell(gc, rxq->gdma_rq); 2349 } 2350 2351 if (rxq->xdp_flush) 2352 xdp_do_flush(); 2353 } 2354 2355 static int mana_cq_handler(void *context, struct gdma_queue *gdma_queue) 2356 { 2357 struct mana_cq *cq = context; 2358 int w; 2359 2360 WARN_ON_ONCE(cq->gdma_cq != gdma_queue); 2361 2362 if (cq->type == MANA_CQ_TYPE_RX) 2363 mana_poll_rx_cq(cq); 2364 else 2365 mana_poll_tx_cq(cq); 2366 2367 w = cq->work_done; 2368 cq->work_done_since_doorbell += w; 2369 2370 if (w < cq->budget) { 2371 mana_gd_ring_cq(gdma_queue, SET_ARM_BIT); 2372 cq->work_done_since_doorbell = 0; 2373 napi_complete_done(&cq->napi, w); 2374 } else if (cq->work_done_since_doorbell >= 2375 (cq->gdma_cq->queue_size / COMP_ENTRY_SIZE) * 4) { 2376 /* MANA hardware requires at least one doorbell ring every 8 2377 * wraparounds of CQ even if there is no need to arm the CQ. 2378 * This driver rings the doorbell as soon as it has processed 2379 * 4 wraparounds. 2380 */ 2381 mana_gd_ring_cq(gdma_queue, 0); 2382 cq->work_done_since_doorbell = 0; 2383 } 2384 2385 return w; 2386 } 2387 2388 static int mana_poll(struct napi_struct *napi, int budget) 2389 { 2390 struct mana_cq *cq = container_of(napi, struct mana_cq, napi); 2391 int w; 2392 2393 cq->work_done = 0; 2394 cq->budget = budget; 2395 2396 w = mana_cq_handler(cq, cq->gdma_cq); 2397 2398 return min(w, budget); 2399 } 2400 2401 static void mana_schedule_napi(void *context, struct gdma_queue *gdma_queue) 2402 { 2403 struct mana_cq *cq = context; 2404 2405 napi_schedule_irqoff(&cq->napi); 2406 } 2407 2408 static void mana_deinit_cq(struct mana_port_context *apc, struct mana_cq *cq) 2409 { 2410 struct gdma_dev *gd = apc->ac->gdma_dev; 2411 2412 if (!cq->gdma_cq) 2413 return; 2414 2415 mana_gd_destroy_queue(gd->gdma_context, cq->gdma_cq); 2416 } 2417 2418 static void mana_deinit_txq(struct mana_port_context *apc, struct mana_txq *txq) 2419 { 2420 struct gdma_dev *gd = apc->ac->gdma_dev; 2421 2422 if (!txq->gdma_sq) 2423 return; 2424 2425 mana_gd_destroy_queue(gd->gdma_context, txq->gdma_sq); 2426 } 2427 2428 static void mana_destroy_txq(struct mana_port_context *apc) 2429 { 2430 struct napi_struct *napi; 2431 int i; 2432 2433 if (!apc->tx_qp) 2434 return; 2435 2436 for (i = 0; i < apc->num_queues; i++) { 2437 if (!apc->tx_qp[i]) 2438 continue; 2439 2440 debugfs_remove_recursive(apc->tx_qp[i]->mana_tx_debugfs); 2441 apc->tx_qp[i]->mana_tx_debugfs = NULL; 2442 2443 napi = &apc->tx_qp[i]->tx_cq.napi; 2444 if (apc->tx_qp[i]->txq.napi_initialized) { 2445 napi_synchronize(napi); 2446 napi_disable_locked(napi); 2447 netif_napi_del_locked(napi); 2448 apc->tx_qp[i]->txq.napi_initialized = false; 2449 } 2450 2451 if (apc->tx_qp[i]->tx_object != INVALID_MANA_HANDLE) 2452 mana_destroy_wq_obj(apc, GDMA_SQ, apc->tx_qp[i]->tx_object); 2453 2454 mana_deinit_cq(apc, &apc->tx_qp[i]->tx_cq); 2455 2456 mana_deinit_txq(apc, &apc->tx_qp[i]->txq); 2457 2458 kvfree(apc->tx_qp[i]); 2459 } 2460 2461 kfree(apc->tx_qp); 2462 apc->tx_qp = NULL; 2463 } 2464 2465 static void mana_create_txq_debugfs(struct mana_port_context *apc, int idx) 2466 { 2467 struct mana_tx_qp *tx_qp = apc->tx_qp[idx]; 2468 char qnum[32]; 2469 2470 sprintf(qnum, "TX-%d", idx); 2471 tx_qp->mana_tx_debugfs = debugfs_create_dir(qnum, apc->mana_port_debugfs); 2472 debugfs_create_u32("sq_head", 0400, tx_qp->mana_tx_debugfs, 2473 &tx_qp->txq.gdma_sq->head); 2474 debugfs_create_u32("sq_tail", 0400, tx_qp->mana_tx_debugfs, 2475 &tx_qp->txq.gdma_sq->tail); 2476 debugfs_create_u32("sq_pend_skb_qlen", 0400, tx_qp->mana_tx_debugfs, 2477 &tx_qp->txq.pending_skbs.qlen); 2478 debugfs_create_u32("cq_head", 0400, tx_qp->mana_tx_debugfs, 2479 &tx_qp->tx_cq.gdma_cq->head); 2480 debugfs_create_u32("cq_tail", 0400, tx_qp->mana_tx_debugfs, 2481 &tx_qp->tx_cq.gdma_cq->tail); 2482 debugfs_create_u32("cq_budget", 0400, tx_qp->mana_tx_debugfs, 2483 &tx_qp->tx_cq.budget); 2484 debugfs_create_file("txq_dump", 0400, tx_qp->mana_tx_debugfs, 2485 tx_qp->txq.gdma_sq, &mana_dbg_q_fops); 2486 debugfs_create_file("cq_dump", 0400, tx_qp->mana_tx_debugfs, 2487 tx_qp->tx_cq.gdma_cq, &mana_dbg_q_fops); 2488 } 2489 2490 static int mana_create_txq(struct mana_port_context *apc, 2491 struct net_device *net) 2492 { 2493 struct mana_context *ac = apc->ac; 2494 struct gdma_dev *gd = ac->gdma_dev; 2495 struct mana_obj_spec wq_spec; 2496 struct mana_obj_spec cq_spec; 2497 struct gdma_queue_spec spec; 2498 struct gdma_context *gc; 2499 struct mana_txq *txq; 2500 struct mana_cq *cq; 2501 u32 txq_size; 2502 u32 cq_size; 2503 int err; 2504 int i; 2505 2506 apc->tx_qp = kzalloc_objs(struct mana_tx_qp *, apc->num_queues); 2507 if (!apc->tx_qp) 2508 return -ENOMEM; 2509 2510 /* The minimum size of the WQE is 32 bytes, hence 2511 * apc->tx_queue_size represents the maximum number of WQEs 2512 * the SQ can store. This value is then used to size other queues 2513 * to prevent overflow. 2514 * Also note that the txq_size is always going to be MANA_PAGE_ALIGNED, 2515 * as min val of apc->tx_queue_size is 128 and that would make 2516 * txq_size 128*32 = 4096 and the other higher values of apc->tx_queue_size 2517 * are always power of two 2518 */ 2519 txq_size = apc->tx_queue_size * 32; 2520 2521 cq_size = apc->tx_queue_size * COMP_ENTRY_SIZE; 2522 2523 gc = gd->gdma_context; 2524 2525 for (i = 0; i < apc->num_queues; i++) { 2526 apc->tx_qp[i] = kvzalloc_obj(*apc->tx_qp[i]); 2527 if (!apc->tx_qp[i]) { 2528 err = -ENOMEM; 2529 goto out; 2530 } 2531 2532 apc->tx_qp[i]->tx_object = INVALID_MANA_HANDLE; 2533 2534 /* Create SQ */ 2535 txq = &apc->tx_qp[i]->txq; 2536 2537 u64_stats_init(&txq->stats.syncp); 2538 txq->ndev = net; 2539 txq->net_txq = netdev_get_tx_queue(net, i); 2540 txq->vp_offset = apc->tx_vp_offset; 2541 txq->napi_initialized = false; 2542 skb_queue_head_init(&txq->pending_skbs); 2543 2544 memset(&spec, 0, sizeof(spec)); 2545 spec.type = GDMA_SQ; 2546 spec.monitor_avl_buf = true; 2547 spec.queue_size = txq_size; 2548 err = mana_gd_create_mana_wq_cq(gd, &spec, &txq->gdma_sq); 2549 if (err) 2550 goto out; 2551 2552 /* Create SQ's CQ */ 2553 cq = &apc->tx_qp[i]->tx_cq; 2554 cq->type = MANA_CQ_TYPE_TX; 2555 2556 cq->txq = txq; 2557 2558 memset(&spec, 0, sizeof(spec)); 2559 spec.type = GDMA_CQ; 2560 spec.monitor_avl_buf = false; 2561 spec.queue_size = cq_size; 2562 spec.cq.callback = mana_schedule_napi; 2563 spec.cq.parent_eq = apc->eqs[i].eq; 2564 spec.cq.context = cq; 2565 err = mana_gd_create_mana_wq_cq(gd, &spec, &cq->gdma_cq); 2566 if (err) 2567 goto out; 2568 2569 memset(&wq_spec, 0, sizeof(wq_spec)); 2570 memset(&cq_spec, 0, sizeof(cq_spec)); 2571 2572 wq_spec.gdma_region = txq->gdma_sq->mem_info.dma_region_handle; 2573 wq_spec.queue_size = txq->gdma_sq->queue_size; 2574 2575 cq_spec.gdma_region = cq->gdma_cq->mem_info.dma_region_handle; 2576 cq_spec.queue_size = cq->gdma_cq->queue_size; 2577 cq_spec.modr_ctx_id = 0; 2578 cq_spec.attached_eq = cq->gdma_cq->cq.parent->id; 2579 2580 err = mana_create_wq_obj(apc, apc->port_handle, GDMA_SQ, 2581 &wq_spec, &cq_spec, 2582 &apc->tx_qp[i]->tx_object); 2583 2584 if (err) 2585 goto out; 2586 2587 txq->gdma_sq->id = wq_spec.queue_index; 2588 cq->gdma_cq->id = cq_spec.queue_index; 2589 2590 txq->gdma_sq->mem_info.dma_region_handle = 2591 GDMA_INVALID_DMA_REGION; 2592 cq->gdma_cq->mem_info.dma_region_handle = 2593 GDMA_INVALID_DMA_REGION; 2594 2595 txq->gdma_txq_id = txq->gdma_sq->id; 2596 2597 cq->gdma_id = cq->gdma_cq->id; 2598 2599 if (WARN_ON(cq->gdma_id >= gc->max_num_cqs)) { 2600 err = -EINVAL; 2601 goto out; 2602 } 2603 2604 gc->cq_table[cq->gdma_id] = cq->gdma_cq; 2605 2606 mana_create_txq_debugfs(apc, i); 2607 2608 set_bit(NAPI_STATE_NO_BUSY_POLL, &cq->napi.state); 2609 netif_napi_add_locked(net, &cq->napi, mana_poll); 2610 napi_enable_locked(&cq->napi); 2611 txq->napi_initialized = true; 2612 2613 mana_gd_ring_cq(cq->gdma_cq, SET_ARM_BIT); 2614 } 2615 2616 return 0; 2617 out: 2618 netdev_err(net, "Failed to create %d TX queues, %d\n", 2619 apc->num_queues, err); 2620 mana_destroy_txq(apc); 2621 return err; 2622 } 2623 2624 static void mana_destroy_rxq(struct mana_port_context *apc, 2625 struct mana_rxq *rxq, bool napi_initialized) 2626 2627 { 2628 struct gdma_context *gc = apc->ac->gdma_dev->gdma_context; 2629 struct mana_recv_buf_oob *rx_oob; 2630 struct device *dev = gc->dev; 2631 struct napi_struct *napi; 2632 struct page *page; 2633 int i; 2634 2635 if (!rxq) 2636 return; 2637 2638 debugfs_remove_recursive(rxq->mana_rx_debugfs); 2639 rxq->mana_rx_debugfs = NULL; 2640 2641 napi = &rxq->rx_cq.napi; 2642 2643 if (napi_initialized) { 2644 napi_synchronize(napi); 2645 2646 napi_disable_locked(napi); 2647 netif_napi_del_locked(napi); 2648 } 2649 2650 if (xdp_rxq_info_is_reg(&rxq->xdp_rxq)) 2651 xdp_rxq_info_unreg(&rxq->xdp_rxq); 2652 2653 if (rxq->rxobj != INVALID_MANA_HANDLE) 2654 mana_destroy_wq_obj(apc, GDMA_RQ, rxq->rxobj); 2655 2656 mana_deinit_cq(apc, &rxq->rx_cq); 2657 2658 if (rxq->xdp_save_va) 2659 put_page(virt_to_head_page(rxq->xdp_save_va)); 2660 2661 for (i = 0; i < rxq->num_rx_buf; i++) { 2662 rx_oob = &rxq->rx_oobs[i]; 2663 2664 if (!rx_oob->buf_va) 2665 continue; 2666 2667 page = virt_to_head_page(rx_oob->buf_va); 2668 2669 if (rxq->frag_count == 1 || !rx_oob->from_pool) { 2670 dma_unmap_single(dev, rx_oob->sgl[0].address, 2671 rx_oob->sgl[0].size, DMA_FROM_DEVICE); 2672 mana_put_rx_page(rxq, page, rx_oob->from_pool); 2673 } else { 2674 page_pool_free_va(rxq->page_pool, rx_oob->buf_va, true); 2675 } 2676 2677 rx_oob->buf_va = NULL; 2678 } 2679 2680 page_pool_destroy(rxq->page_pool); 2681 2682 if (rxq->gdma_rq) 2683 mana_gd_destroy_queue(gc, rxq->gdma_rq); 2684 2685 kvfree(rxq); 2686 } 2687 2688 static int mana_fill_rx_oob(struct mana_recv_buf_oob *rx_oob, u32 mem_key, 2689 struct mana_rxq *rxq, struct device *dev) 2690 { 2691 struct mana_port_context *mpc = netdev_priv(rxq->ndev); 2692 struct page *pp_page = NULL; 2693 u32 dma_sync_offset = 0; 2694 bool from_pool = false; 2695 dma_addr_t da; 2696 void *va; 2697 2698 if (mpc->rxbufs_pre) 2699 va = mana_get_rxbuf_pre(rxq, &da); 2700 else 2701 va = mana_get_rxfrag(rxq, dev, &da, &from_pool, &pp_page, 2702 &dma_sync_offset); 2703 2704 if (!va) 2705 return -ENOMEM; 2706 2707 rx_oob->buf_va = va; 2708 rx_oob->from_pool = from_pool; 2709 rx_oob->pp_page = pp_page; 2710 rx_oob->dma_sync_offset = dma_sync_offset; 2711 2712 rx_oob->sgl[0].address = da; 2713 rx_oob->sgl[0].size = rxq->datasize; 2714 rx_oob->sgl[0].mem_key = mem_key; 2715 2716 return 0; 2717 } 2718 2719 #define MANA_WQE_HEADER_SIZE 16 2720 #define MANA_WQE_SGE_SIZE 16 2721 2722 static int mana_alloc_rx_wqe(struct mana_port_context *apc, 2723 struct mana_rxq *rxq, u32 *rxq_size, u32 *cq_size) 2724 { 2725 struct gdma_context *gc = apc->ac->gdma_dev->gdma_context; 2726 struct mana_recv_buf_oob *rx_oob; 2727 struct device *dev = gc->dev; 2728 u32 buf_idx; 2729 int ret; 2730 2731 WARN_ON(rxq->datasize == 0); 2732 2733 *rxq_size = 0; 2734 *cq_size = 0; 2735 2736 for (buf_idx = 0; buf_idx < rxq->num_rx_buf; buf_idx++) { 2737 rx_oob = &rxq->rx_oobs[buf_idx]; 2738 memset(rx_oob, 0, sizeof(*rx_oob)); 2739 2740 rx_oob->num_sge = 1; 2741 2742 ret = mana_fill_rx_oob(rx_oob, apc->ac->gdma_dev->gpa_mkey, rxq, 2743 dev); 2744 if (ret) 2745 return ret; 2746 2747 rx_oob->wqe_req.sgl = rx_oob->sgl; 2748 rx_oob->wqe_req.num_sge = rx_oob->num_sge; 2749 rx_oob->wqe_req.inline_oob_size = 0; 2750 rx_oob->wqe_req.inline_oob_data = NULL; 2751 rx_oob->wqe_req.flags = 0; 2752 rx_oob->wqe_req.client_data_unit = 0; 2753 2754 *rxq_size += ALIGN(MANA_WQE_HEADER_SIZE + 2755 MANA_WQE_SGE_SIZE * rx_oob->num_sge, 32); 2756 *cq_size += COMP_ENTRY_SIZE; 2757 } 2758 2759 return 0; 2760 } 2761 2762 static int mana_push_wqe(struct mana_rxq *rxq) 2763 { 2764 struct mana_recv_buf_oob *rx_oob; 2765 u32 buf_idx; 2766 int err; 2767 2768 for (buf_idx = 0; buf_idx < rxq->num_rx_buf; buf_idx++) { 2769 rx_oob = &rxq->rx_oobs[buf_idx]; 2770 2771 err = mana_gd_post_and_ring(rxq->gdma_rq, &rx_oob->wqe_req, 2772 &rx_oob->wqe_inf); 2773 if (err) 2774 return -ENOSPC; 2775 } 2776 2777 return 0; 2778 } 2779 2780 static int mana_create_page_pool(struct mana_rxq *rxq, struct gdma_context *gc) 2781 { 2782 struct mana_port_context *mpc = netdev_priv(rxq->ndev); 2783 struct page_pool_params pprm = {}; 2784 int ret; 2785 2786 pprm.pool_size = mpc->rx_queue_size / rxq->frag_count + 1; 2787 pprm.nid = gc->numa_node; 2788 pprm.napi = &rxq->rx_cq.napi; 2789 pprm.netdev = rxq->ndev; 2790 pprm.order = get_order(rxq->alloc_size); 2791 pprm.queue_idx = rxq->rxq_idx; 2792 pprm.dev = gc->dev; 2793 2794 /* Let the page pool do the dma map when page sharing with multiple 2795 * fragments enabled for rx buffers. 2796 */ 2797 if (rxq->frag_count > 1) { 2798 pprm.flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV; 2799 pprm.max_len = PAGE_SIZE; 2800 pprm.dma_dir = DMA_FROM_DEVICE; 2801 } 2802 2803 rxq->page_pool = page_pool_create(&pprm); 2804 2805 if (IS_ERR(rxq->page_pool)) { 2806 ret = PTR_ERR(rxq->page_pool); 2807 rxq->page_pool = NULL; 2808 return ret; 2809 } 2810 2811 return 0; 2812 } 2813 2814 static struct mana_rxq *mana_create_rxq(struct mana_port_context *apc, 2815 u32 rxq_idx, struct mana_eq *eq, 2816 struct net_device *ndev) 2817 { 2818 struct gdma_dev *gd = apc->ac->gdma_dev; 2819 struct mana_obj_spec wq_spec; 2820 struct mana_obj_spec cq_spec; 2821 struct gdma_queue_spec spec; 2822 struct mana_cq *cq = NULL; 2823 struct gdma_context *gc; 2824 u32 cq_size, rq_size; 2825 struct mana_rxq *rxq; 2826 int err; 2827 2828 gc = gd->gdma_context; 2829 2830 rxq = kvzalloc_flex(*rxq, rx_oobs, apc->rx_queue_size); 2831 if (!rxq) 2832 return NULL; 2833 2834 rxq->ndev = ndev; 2835 rxq->num_rx_buf = apc->rx_queue_size; 2836 rxq->rxq_idx = rxq_idx; 2837 rxq->rxobj = INVALID_MANA_HANDLE; 2838 2839 mana_get_rxbuf_cfg(apc, ndev->mtu, &rxq->datasize, &rxq->alloc_size, 2840 &rxq->headroom, &rxq->frag_count); 2841 /* Create page pool for RX queue */ 2842 err = mana_create_page_pool(rxq, gc); 2843 if (err) { 2844 netdev_err(ndev, "Create page pool err:%d\n", err); 2845 goto out; 2846 } 2847 2848 err = mana_alloc_rx_wqe(apc, rxq, &rq_size, &cq_size); 2849 if (err) 2850 goto out; 2851 2852 rq_size = MANA_PAGE_ALIGN(rq_size); 2853 cq_size = MANA_PAGE_ALIGN(cq_size); 2854 2855 /* Create RQ */ 2856 memset(&spec, 0, sizeof(spec)); 2857 spec.type = GDMA_RQ; 2858 spec.monitor_avl_buf = true; 2859 spec.queue_size = rq_size; 2860 err = mana_gd_create_mana_wq_cq(gd, &spec, &rxq->gdma_rq); 2861 if (err) 2862 goto out; 2863 2864 /* Create RQ's CQ */ 2865 cq = &rxq->rx_cq; 2866 cq->type = MANA_CQ_TYPE_RX; 2867 cq->rxq = rxq; 2868 2869 memset(&spec, 0, sizeof(spec)); 2870 spec.type = GDMA_CQ; 2871 spec.monitor_avl_buf = false; 2872 spec.queue_size = cq_size; 2873 spec.cq.callback = mana_schedule_napi; 2874 spec.cq.parent_eq = eq->eq; 2875 spec.cq.context = cq; 2876 err = mana_gd_create_mana_wq_cq(gd, &spec, &cq->gdma_cq); 2877 if (err) 2878 goto out; 2879 2880 memset(&wq_spec, 0, sizeof(wq_spec)); 2881 memset(&cq_spec, 0, sizeof(cq_spec)); 2882 wq_spec.gdma_region = rxq->gdma_rq->mem_info.dma_region_handle; 2883 wq_spec.queue_size = rxq->gdma_rq->queue_size; 2884 2885 cq_spec.gdma_region = cq->gdma_cq->mem_info.dma_region_handle; 2886 cq_spec.queue_size = cq->gdma_cq->queue_size; 2887 cq_spec.modr_ctx_id = 0; 2888 cq_spec.attached_eq = cq->gdma_cq->cq.parent->id; 2889 2890 err = mana_create_wq_obj(apc, apc->port_handle, GDMA_RQ, 2891 &wq_spec, &cq_spec, &rxq->rxobj); 2892 if (err) 2893 goto out; 2894 2895 rxq->gdma_rq->id = wq_spec.queue_index; 2896 cq->gdma_cq->id = cq_spec.queue_index; 2897 2898 rxq->gdma_rq->mem_info.dma_region_handle = GDMA_INVALID_DMA_REGION; 2899 cq->gdma_cq->mem_info.dma_region_handle = GDMA_INVALID_DMA_REGION; 2900 2901 rxq->gdma_id = rxq->gdma_rq->id; 2902 cq->gdma_id = cq->gdma_cq->id; 2903 2904 err = mana_push_wqe(rxq); 2905 if (err) 2906 goto out; 2907 2908 if (WARN_ON(cq->gdma_id >= gc->max_num_cqs)) { 2909 err = -EINVAL; 2910 goto out; 2911 } 2912 2913 gc->cq_table[cq->gdma_id] = cq->gdma_cq; 2914 2915 netif_napi_add_weight_locked(ndev, &cq->napi, mana_poll, 1); 2916 2917 WARN_ON(xdp_rxq_info_reg(&rxq->xdp_rxq, ndev, rxq_idx, 2918 cq->napi.napi_id)); 2919 WARN_ON(xdp_rxq_info_reg_mem_model(&rxq->xdp_rxq, MEM_TYPE_PAGE_POOL, 2920 rxq->page_pool)); 2921 2922 napi_enable_locked(&cq->napi); 2923 2924 mana_gd_ring_cq(cq->gdma_cq, SET_ARM_BIT); 2925 out: 2926 if (!err) 2927 return rxq; 2928 2929 netdev_err(ndev, "Failed to create RXQ: err = %d\n", err); 2930 2931 mana_destroy_rxq(apc, rxq, false); 2932 2933 return NULL; 2934 } 2935 2936 static void mana_create_rxq_debugfs(struct mana_port_context *apc, int idx) 2937 { 2938 struct mana_rxq *rxq; 2939 char qnum[32]; 2940 2941 rxq = apc->rxqs[idx]; 2942 2943 sprintf(qnum, "RX-%d", idx); 2944 rxq->mana_rx_debugfs = debugfs_create_dir(qnum, apc->mana_port_debugfs); 2945 debugfs_create_u32("rq_head", 0400, rxq->mana_rx_debugfs, &rxq->gdma_rq->head); 2946 debugfs_create_u32("rq_tail", 0400, rxq->mana_rx_debugfs, &rxq->gdma_rq->tail); 2947 debugfs_create_u32("rq_nbuf", 0400, rxq->mana_rx_debugfs, &rxq->num_rx_buf); 2948 debugfs_create_u32("cq_head", 0400, rxq->mana_rx_debugfs, 2949 &rxq->rx_cq.gdma_cq->head); 2950 debugfs_create_u32("cq_tail", 0400, rxq->mana_rx_debugfs, 2951 &rxq->rx_cq.gdma_cq->tail); 2952 debugfs_create_u32("cq_budget", 0400, rxq->mana_rx_debugfs, &rxq->rx_cq.budget); 2953 debugfs_create_file("rxq_dump", 0400, rxq->mana_rx_debugfs, rxq->gdma_rq, &mana_dbg_q_fops); 2954 debugfs_create_file("cq_dump", 0400, rxq->mana_rx_debugfs, rxq->rx_cq.gdma_cq, 2955 &mana_dbg_q_fops); 2956 } 2957 2958 static int mana_add_rx_queues(struct mana_port_context *apc, 2959 struct net_device *ndev) 2960 { 2961 struct mana_rxq *rxq; 2962 int err = 0; 2963 int i; 2964 2965 for (i = 0; i < apc->num_queues; i++) { 2966 rxq = mana_create_rxq(apc, i, &apc->eqs[i], ndev); 2967 if (!rxq) { 2968 err = -ENOMEM; 2969 netdev_err(ndev, "Failed to create rxq %d : %d\n", i, err); 2970 goto out; 2971 } 2972 2973 u64_stats_init(&rxq->stats.syncp); 2974 2975 apc->rxqs[i] = rxq; 2976 2977 mana_create_rxq_debugfs(apc, i); 2978 } 2979 2980 apc->default_rxobj = apc->rxqs[0]->rxobj; 2981 out: 2982 return err; 2983 } 2984 2985 static void mana_destroy_rxqs(struct mana_port_context *apc) 2986 { 2987 struct mana_rxq *rxq; 2988 u32 rxq_idx; 2989 2990 if (apc->rxqs) { 2991 2992 for (rxq_idx = 0; rxq_idx < apc->num_queues; rxq_idx++) { 2993 rxq = apc->rxqs[rxq_idx]; 2994 if (!rxq) 2995 continue; 2996 2997 mana_destroy_rxq(apc, rxq, true); 2998 apc->rxqs[rxq_idx] = NULL; 2999 } 3000 } 3001 } 3002 3003 static void mana_destroy_vport(struct mana_port_context *apc) 3004 { 3005 struct gdma_dev *gd = apc->ac->gdma_dev; 3006 3007 mana_uncfg_vport(apc); 3008 3009 if (gd->gdma_context->is_pf && !apc->ac->bm_hostmode) 3010 mana_pf_deregister_hw_vport(apc); 3011 } 3012 3013 static int mana_create_vport(struct mana_port_context *apc, 3014 struct net_device *net) 3015 { 3016 struct gdma_dev *gd = apc->ac->gdma_dev; 3017 int err; 3018 3019 apc->default_rxobj = INVALID_MANA_HANDLE; 3020 3021 if (gd->gdma_context->is_pf && !apc->ac->bm_hostmode) { 3022 err = mana_pf_register_hw_vport(apc); 3023 if (err) 3024 return err; 3025 } 3026 3027 err = mana_cfg_vport(apc, gd->pdid, gd->doorbell, false); 3028 if (err) { 3029 if (gd->gdma_context->is_pf && !apc->ac->bm_hostmode) 3030 mana_pf_deregister_hw_vport(apc); 3031 return err; 3032 } 3033 3034 return 0; 3035 } 3036 3037 static int mana_rss_table_alloc(struct mana_port_context *apc) 3038 { 3039 if (!apc->indir_table_sz) { 3040 netdev_err(apc->ndev, 3041 "Indirection table size not set for vPort %d\n", 3042 apc->port_idx); 3043 return -EINVAL; 3044 } 3045 3046 apc->indir_table = kcalloc(apc->indir_table_sz, sizeof(u32), GFP_KERNEL); 3047 if (!apc->indir_table) 3048 return -ENOMEM; 3049 3050 apc->rxobj_table = kzalloc_objs(mana_handle_t, apc->indir_table_sz); 3051 if (!apc->rxobj_table) { 3052 kfree(apc->indir_table); 3053 return -ENOMEM; 3054 } 3055 3056 return 0; 3057 } 3058 3059 static void mana_rss_table_init(struct mana_port_context *apc) 3060 { 3061 int i; 3062 3063 for (i = 0; i < apc->indir_table_sz; i++) 3064 apc->indir_table[i] = 3065 ethtool_rxfh_indir_default(i, apc->num_queues); 3066 } 3067 3068 int mana_disable_vport_rx(struct mana_port_context *apc) 3069 { 3070 return mana_cfg_vport_steering(apc, TRI_STATE_FALSE, false, false, 3071 false); 3072 } 3073 EXPORT_SYMBOL_NS(mana_disable_vport_rx, "NET_MANA"); 3074 3075 int mana_config_rss(struct mana_port_context *apc, enum TRI_STATE rx, 3076 bool update_hash, bool update_tab) 3077 { 3078 u32 queue_idx; 3079 int err; 3080 int i; 3081 3082 if (update_tab) { 3083 for (i = 0; i < apc->indir_table_sz; i++) { 3084 queue_idx = apc->indir_table[i]; 3085 apc->rxobj_table[i] = apc->rxqs[queue_idx]->rxobj; 3086 } 3087 } 3088 3089 err = mana_cfg_vport_steering(apc, rx, true, update_hash, update_tab); 3090 if (err) 3091 return err; 3092 3093 mana_fence_rqs(apc); 3094 3095 return 0; 3096 } 3097 3098 int mana_query_gf_stats(struct mana_context *ac) 3099 { 3100 struct gdma_context *gc = ac->gdma_dev->gdma_context; 3101 struct mana_query_gf_stat_resp resp = {}; 3102 struct mana_query_gf_stat_req req = {}; 3103 struct device *dev = gc->dev; 3104 int err; 3105 3106 mana_gd_init_req_hdr(&req.hdr, MANA_QUERY_GF_STAT, 3107 sizeof(req), sizeof(resp)); 3108 req.hdr.resp.msg_version = GDMA_MESSAGE_V2; 3109 req.req_stats = STATISTICS_FLAGS_RX_DISCARDS_NO_WQE | 3110 STATISTICS_FLAGS_RX_ERRORS_VPORT_DISABLED | 3111 STATISTICS_FLAGS_HC_RX_BYTES | 3112 STATISTICS_FLAGS_HC_RX_UCAST_PACKETS | 3113 STATISTICS_FLAGS_HC_RX_UCAST_BYTES | 3114 STATISTICS_FLAGS_HC_RX_MCAST_PACKETS | 3115 STATISTICS_FLAGS_HC_RX_MCAST_BYTES | 3116 STATISTICS_FLAGS_HC_RX_BCAST_PACKETS | 3117 STATISTICS_FLAGS_HC_RX_BCAST_BYTES | 3118 STATISTICS_FLAGS_TX_ERRORS_GF_DISABLED | 3119 STATISTICS_FLAGS_TX_ERRORS_VPORT_DISABLED | 3120 STATISTICS_FLAGS_TX_ERRORS_INVAL_VPORT_OFFSET_PACKETS | 3121 STATISTICS_FLAGS_TX_ERRORS_VLAN_ENFORCEMENT | 3122 STATISTICS_FLAGS_TX_ERRORS_ETH_TYPE_ENFORCEMENT | 3123 STATISTICS_FLAGS_TX_ERRORS_SA_ENFORCEMENT | 3124 STATISTICS_FLAGS_TX_ERRORS_SQPDID_ENFORCEMENT | 3125 STATISTICS_FLAGS_TX_ERRORS_CQPDID_ENFORCEMENT | 3126 STATISTICS_FLAGS_TX_ERRORS_MTU_VIOLATION | 3127 STATISTICS_FLAGS_TX_ERRORS_INVALID_OOB | 3128 STATISTICS_FLAGS_HC_TX_BYTES | 3129 STATISTICS_FLAGS_HC_TX_UCAST_PACKETS | 3130 STATISTICS_FLAGS_HC_TX_UCAST_BYTES | 3131 STATISTICS_FLAGS_HC_TX_MCAST_PACKETS | 3132 STATISTICS_FLAGS_HC_TX_MCAST_BYTES | 3133 STATISTICS_FLAGS_HC_TX_BCAST_PACKETS | 3134 STATISTICS_FLAGS_HC_TX_BCAST_BYTES | 3135 STATISTICS_FLAGS_TX_ERRORS_GDMA_ERROR; 3136 3137 err = mana_send_request(ac, &req, sizeof(req), &resp, 3138 sizeof(resp)); 3139 if (err) { 3140 dev_err(dev, "Failed to query GF stats: %d\n", err); 3141 return err; 3142 } 3143 err = mana_verify_resp_hdr(&resp.hdr, MANA_QUERY_GF_STAT, 3144 sizeof(resp)); 3145 if (err || resp.hdr.status) { 3146 dev_err(dev, "Failed to query GF stats: %d, 0x%x\n", err, 3147 resp.hdr.status); 3148 return err; 3149 } 3150 3151 ac->hc_stats.hc_rx_discards_no_wqe = resp.rx_discards_nowqe; 3152 ac->hc_stats.hc_rx_err_vport_disabled = resp.rx_err_vport_disabled; 3153 ac->hc_stats.hc_rx_bytes = resp.hc_rx_bytes; 3154 ac->hc_stats.hc_rx_ucast_pkts = resp.hc_rx_ucast_pkts; 3155 ac->hc_stats.hc_rx_ucast_bytes = resp.hc_rx_ucast_bytes; 3156 ac->hc_stats.hc_rx_bcast_pkts = resp.hc_rx_bcast_pkts; 3157 ac->hc_stats.hc_rx_bcast_bytes = resp.hc_rx_bcast_bytes; 3158 ac->hc_stats.hc_rx_mcast_pkts = resp.hc_rx_mcast_pkts; 3159 ac->hc_stats.hc_rx_mcast_bytes = resp.hc_rx_mcast_bytes; 3160 ac->hc_stats.hc_tx_err_gf_disabled = resp.tx_err_gf_disabled; 3161 ac->hc_stats.hc_tx_err_vport_disabled = resp.tx_err_vport_disabled; 3162 ac->hc_stats.hc_tx_err_inval_vportoffset_pkt = 3163 resp.tx_err_inval_vport_offset_pkt; 3164 ac->hc_stats.hc_tx_err_vlan_enforcement = 3165 resp.tx_err_vlan_enforcement; 3166 ac->hc_stats.hc_tx_err_eth_type_enforcement = 3167 resp.tx_err_ethtype_enforcement; 3168 ac->hc_stats.hc_tx_err_sa_enforcement = resp.tx_err_SA_enforcement; 3169 ac->hc_stats.hc_tx_err_sqpdid_enforcement = 3170 resp.tx_err_SQPDID_enforcement; 3171 ac->hc_stats.hc_tx_err_cqpdid_enforcement = 3172 resp.tx_err_CQPDID_enforcement; 3173 ac->hc_stats.hc_tx_err_mtu_violation = resp.tx_err_mtu_violation; 3174 ac->hc_stats.hc_tx_err_inval_oob = resp.tx_err_inval_oob; 3175 ac->hc_stats.hc_tx_bytes = resp.hc_tx_bytes; 3176 ac->hc_stats.hc_tx_ucast_pkts = resp.hc_tx_ucast_pkts; 3177 ac->hc_stats.hc_tx_ucast_bytes = resp.hc_tx_ucast_bytes; 3178 ac->hc_stats.hc_tx_bcast_pkts = resp.hc_tx_bcast_pkts; 3179 ac->hc_stats.hc_tx_bcast_bytes = resp.hc_tx_bcast_bytes; 3180 ac->hc_stats.hc_tx_mcast_pkts = resp.hc_tx_mcast_pkts; 3181 ac->hc_stats.hc_tx_mcast_bytes = resp.hc_tx_mcast_bytes; 3182 ac->hc_stats.hc_tx_err_gdma = resp.tx_err_gdma; 3183 3184 return 0; 3185 } 3186 3187 void mana_query_phy_stats(struct mana_port_context *apc) 3188 { 3189 struct mana_query_phy_stat_resp resp = {}; 3190 struct mana_query_phy_stat_req req = {}; 3191 struct net_device *ndev = apc->ndev; 3192 int err; 3193 3194 mana_gd_init_req_hdr(&req.hdr, MANA_QUERY_PHY_STAT, 3195 sizeof(req), sizeof(resp)); 3196 err = mana_send_request(apc->ac, &req, sizeof(req), &resp, 3197 sizeof(resp)); 3198 if (err) 3199 return; 3200 3201 err = mana_verify_resp_hdr(&resp.hdr, MANA_QUERY_PHY_STAT, 3202 sizeof(resp)); 3203 if (err || resp.hdr.status) { 3204 netdev_err(ndev, 3205 "Failed to query PHY stats: %d, resp:0x%x\n", 3206 err, resp.hdr.status); 3207 return; 3208 } 3209 3210 /* Aggregate drop counters */ 3211 apc->phy_stats.rx_pkt_drop_phy = resp.rx_pkt_drop_phy; 3212 apc->phy_stats.tx_pkt_drop_phy = resp.tx_pkt_drop_phy; 3213 3214 /* Per TC traffic Counters */ 3215 apc->phy_stats.rx_pkt_tc0_phy = resp.rx_pkt_tc0_phy; 3216 apc->phy_stats.tx_pkt_tc0_phy = resp.tx_pkt_tc0_phy; 3217 apc->phy_stats.rx_pkt_tc1_phy = resp.rx_pkt_tc1_phy; 3218 apc->phy_stats.tx_pkt_tc1_phy = resp.tx_pkt_tc1_phy; 3219 apc->phy_stats.rx_pkt_tc2_phy = resp.rx_pkt_tc2_phy; 3220 apc->phy_stats.tx_pkt_tc2_phy = resp.tx_pkt_tc2_phy; 3221 apc->phy_stats.rx_pkt_tc3_phy = resp.rx_pkt_tc3_phy; 3222 apc->phy_stats.tx_pkt_tc3_phy = resp.tx_pkt_tc3_phy; 3223 apc->phy_stats.rx_pkt_tc4_phy = resp.rx_pkt_tc4_phy; 3224 apc->phy_stats.tx_pkt_tc4_phy = resp.tx_pkt_tc4_phy; 3225 apc->phy_stats.rx_pkt_tc5_phy = resp.rx_pkt_tc5_phy; 3226 apc->phy_stats.tx_pkt_tc5_phy = resp.tx_pkt_tc5_phy; 3227 apc->phy_stats.rx_pkt_tc6_phy = resp.rx_pkt_tc6_phy; 3228 apc->phy_stats.tx_pkt_tc6_phy = resp.tx_pkt_tc6_phy; 3229 apc->phy_stats.rx_pkt_tc7_phy = resp.rx_pkt_tc7_phy; 3230 apc->phy_stats.tx_pkt_tc7_phy = resp.tx_pkt_tc7_phy; 3231 3232 /* Per TC byte Counters */ 3233 apc->phy_stats.rx_byte_tc0_phy = resp.rx_byte_tc0_phy; 3234 apc->phy_stats.tx_byte_tc0_phy = resp.tx_byte_tc0_phy; 3235 apc->phy_stats.rx_byte_tc1_phy = resp.rx_byte_tc1_phy; 3236 apc->phy_stats.tx_byte_tc1_phy = resp.tx_byte_tc1_phy; 3237 apc->phy_stats.rx_byte_tc2_phy = resp.rx_byte_tc2_phy; 3238 apc->phy_stats.tx_byte_tc2_phy = resp.tx_byte_tc2_phy; 3239 apc->phy_stats.rx_byte_tc3_phy = resp.rx_byte_tc3_phy; 3240 apc->phy_stats.tx_byte_tc3_phy = resp.tx_byte_tc3_phy; 3241 apc->phy_stats.rx_byte_tc4_phy = resp.rx_byte_tc4_phy; 3242 apc->phy_stats.tx_byte_tc4_phy = resp.tx_byte_tc4_phy; 3243 apc->phy_stats.rx_byte_tc5_phy = resp.rx_byte_tc5_phy; 3244 apc->phy_stats.tx_byte_tc5_phy = resp.tx_byte_tc5_phy; 3245 apc->phy_stats.rx_byte_tc6_phy = resp.rx_byte_tc6_phy; 3246 apc->phy_stats.tx_byte_tc6_phy = resp.tx_byte_tc6_phy; 3247 apc->phy_stats.rx_byte_tc7_phy = resp.rx_byte_tc7_phy; 3248 apc->phy_stats.tx_byte_tc7_phy = resp.tx_byte_tc7_phy; 3249 3250 /* Per TC pause Counters */ 3251 apc->phy_stats.rx_pause_tc0_phy = resp.rx_pause_tc0_phy; 3252 apc->phy_stats.tx_pause_tc0_phy = resp.tx_pause_tc0_phy; 3253 apc->phy_stats.rx_pause_tc1_phy = resp.rx_pause_tc1_phy; 3254 apc->phy_stats.tx_pause_tc1_phy = resp.tx_pause_tc1_phy; 3255 apc->phy_stats.rx_pause_tc2_phy = resp.rx_pause_tc2_phy; 3256 apc->phy_stats.tx_pause_tc2_phy = resp.tx_pause_tc2_phy; 3257 apc->phy_stats.rx_pause_tc3_phy = resp.rx_pause_tc3_phy; 3258 apc->phy_stats.tx_pause_tc3_phy = resp.tx_pause_tc3_phy; 3259 apc->phy_stats.rx_pause_tc4_phy = resp.rx_pause_tc4_phy; 3260 apc->phy_stats.tx_pause_tc4_phy = resp.tx_pause_tc4_phy; 3261 apc->phy_stats.rx_pause_tc5_phy = resp.rx_pause_tc5_phy; 3262 apc->phy_stats.tx_pause_tc5_phy = resp.tx_pause_tc5_phy; 3263 apc->phy_stats.rx_pause_tc6_phy = resp.rx_pause_tc6_phy; 3264 apc->phy_stats.tx_pause_tc6_phy = resp.tx_pause_tc6_phy; 3265 apc->phy_stats.rx_pause_tc7_phy = resp.rx_pause_tc7_phy; 3266 apc->phy_stats.tx_pause_tc7_phy = resp.tx_pause_tc7_phy; 3267 } 3268 3269 static int mana_init_port(struct net_device *ndev) 3270 { 3271 struct mana_port_context *apc = netdev_priv(ndev); 3272 struct gdma_dev *gd = apc->ac->gdma_dev; 3273 u32 max_txq, max_rxq, max_queues; 3274 int port_idx = apc->port_idx; 3275 struct gdma_context *gc; 3276 char vport[32]; 3277 int err; 3278 3279 err = mana_init_port_context(apc); 3280 if (err) 3281 return err; 3282 3283 gc = gd->gdma_context; 3284 3285 err = mana_query_vport_cfg(apc, port_idx, &max_txq, &max_rxq, 3286 &apc->indir_table_sz); 3287 if (err) { 3288 netdev_err(ndev, "Failed to query info for vPort %d\n", 3289 port_idx); 3290 goto reset_apc; 3291 } 3292 3293 max_queues = min_t(u32, max_txq, max_rxq); 3294 if (apc->max_queues > max_queues) 3295 apc->max_queues = max_queues; 3296 if (apc->max_queues > gc->max_num_queues_vport) 3297 apc->max_queues = gc->max_num_queues_vport; 3298 3299 if (apc->num_queues > apc->max_queues) 3300 apc->num_queues = apc->max_queues; 3301 3302 eth_hw_addr_set(ndev, apc->mac_addr); 3303 sprintf(vport, "vport%d", port_idx); 3304 apc->mana_port_debugfs = debugfs_create_dir(vport, gc->mana_pci_debugfs); 3305 3306 debugfs_create_u64("port_handle", 0400, apc->mana_port_debugfs, 3307 &apc->port_handle); 3308 debugfs_create_u32("max_sq", 0400, apc->mana_port_debugfs, 3309 &apc->vport_max_sq); 3310 debugfs_create_u32("max_rq", 0400, apc->mana_port_debugfs, 3311 &apc->vport_max_rq); 3312 debugfs_create_u32("indir_table_sz", 0400, apc->mana_port_debugfs, 3313 &apc->indir_table_sz); 3314 debugfs_create_u32("steer_rx", 0400, apc->mana_port_debugfs, 3315 &apc->steer_rx); 3316 debugfs_create_u32("steer_rss", 0400, apc->mana_port_debugfs, 3317 &apc->steer_rss); 3318 debugfs_create_bool("steer_update_tab", 0400, apc->mana_port_debugfs, 3319 &apc->steer_update_tab); 3320 debugfs_create_u32("steer_cqe_coalescing", 0400, apc->mana_port_debugfs, 3321 &apc->steer_cqe_coalescing); 3322 debugfs_create_u32("current_speed", 0400, apc->mana_port_debugfs, 3323 &apc->speed); 3324 return 0; 3325 3326 reset_apc: 3327 mana_cleanup_port_context(apc); 3328 return err; 3329 } 3330 3331 int mana_alloc_queues(struct net_device *ndev) 3332 { 3333 struct mana_port_context *apc = netdev_priv(ndev); 3334 struct gdma_dev *gd = apc->ac->gdma_dev; 3335 int err; 3336 3337 err = mana_create_vport(apc, ndev); 3338 if (err) { 3339 netdev_err(ndev, "Failed to create vPort %u : %d\n", 3340 apc->port_idx, err); 3341 return err; 3342 } 3343 3344 err = mana_create_eq(apc); 3345 if (err) { 3346 netdev_err(ndev, "Failed to create EQ on vPort %u: %d\n", 3347 apc->port_idx, err); 3348 goto destroy_vport; 3349 } 3350 3351 err = mana_create_txq(apc, ndev); 3352 if (err) { 3353 netdev_err(ndev, "Failed to create TXQ on vPort %u: %d\n", 3354 apc->port_idx, err); 3355 goto destroy_eq; 3356 } 3357 3358 err = netif_set_real_num_tx_queues(ndev, apc->num_queues); 3359 if (err) { 3360 netdev_err(ndev, 3361 "netif_set_real_num_tx_queues () failed for ndev with num_queues %u : %d\n", 3362 apc->num_queues, err); 3363 goto destroy_txq; 3364 } 3365 3366 err = mana_add_rx_queues(apc, ndev); 3367 if (err) 3368 goto destroy_rxq; 3369 3370 apc->rss_state = apc->num_queues > 1 ? TRI_STATE_TRUE : TRI_STATE_FALSE; 3371 3372 err = netif_set_real_num_rx_queues(ndev, apc->num_queues); 3373 if (err) { 3374 netdev_err(ndev, 3375 "netif_set_real_num_rx_queues () failed for ndev with num_queues %u : %d\n", 3376 apc->num_queues, err); 3377 goto destroy_rxq; 3378 } 3379 3380 mana_rss_table_init(apc); 3381 3382 err = mana_config_rss(apc, TRI_STATE_TRUE, true, true); 3383 if (err) { 3384 netdev_err(ndev, "Failed to configure RSS table: %d\n", err); 3385 goto destroy_rxq; 3386 } 3387 3388 if (gd->gdma_context->is_pf && !apc->ac->bm_hostmode) { 3389 err = mana_pf_register_filter(apc); 3390 if (err) 3391 goto destroy_rxq; 3392 } 3393 3394 mana_chn_setxdp(apc, mana_xdp_get(apc)); 3395 3396 return 0; 3397 3398 destroy_rxq: 3399 mana_destroy_rxqs(apc); 3400 destroy_txq: 3401 mana_destroy_txq(apc); 3402 destroy_eq: 3403 mana_destroy_eq(apc); 3404 destroy_vport: 3405 mana_destroy_vport(apc); 3406 return err; 3407 } 3408 3409 int mana_attach(struct net_device *ndev) 3410 { 3411 struct mana_port_context *apc = netdev_priv(ndev); 3412 int err; 3413 3414 ASSERT_RTNL(); 3415 3416 err = mana_init_port(ndev); 3417 if (err) 3418 return err; 3419 3420 if (apc->port_st_save) { 3421 err = mana_alloc_queues(ndev); 3422 if (err) { 3423 mana_cleanup_port_context(apc); 3424 return err; 3425 } 3426 } 3427 3428 apc->port_is_up = apc->port_st_save; 3429 3430 /* Ensure port state updated before txq state */ 3431 smp_wmb(); 3432 3433 netif_device_attach(ndev); 3434 3435 return 0; 3436 } 3437 3438 static int mana_dealloc_queues(struct net_device *ndev) 3439 { 3440 struct mana_port_context *apc = netdev_priv(ndev); 3441 unsigned long timeout = jiffies + 120 * HZ; 3442 struct gdma_dev *gd = apc->ac->gdma_dev; 3443 struct mana_txq *txq; 3444 struct sk_buff *skb; 3445 int i, err; 3446 u32 tsleep; 3447 3448 if (apc->port_is_up) 3449 return -EINVAL; 3450 3451 if (apc->rxqs) 3452 mana_chn_setxdp(apc, NULL); 3453 3454 if (gd->gdma_context->is_pf && !apc->ac->bm_hostmode) 3455 mana_pf_deregister_filter(apc); 3456 3457 /* No packet can be transmitted now since apc->port_is_up is false. 3458 * There is still a tiny chance that mana_poll_tx_cq() can re-enable 3459 * a txq because it may not timely see apc->port_is_up being cleared 3460 * to false, but it doesn't matter since mana_start_xmit() drops any 3461 * new packets due to apc->port_is_up being false. 3462 * 3463 * Drain all the in-flight TX packets. 3464 * A timeout of 120 seconds for all the queues is used. 3465 * This will break the while loop when h/w is not responding. 3466 * This value of 120 has been decided here considering max 3467 * number of queues. 3468 */ 3469 3470 if (apc->tx_qp) { 3471 for (i = 0; i < apc->num_queues; i++) { 3472 txq = &apc->tx_qp[i]->txq; 3473 tsleep = 1000; 3474 while (atomic_read(&txq->pending_sends) > 0 && 3475 time_before(jiffies, timeout)) { 3476 usleep_range(tsleep, tsleep + 1000); 3477 tsleep <<= 1; 3478 } 3479 if (atomic_read(&txq->pending_sends)) { 3480 err = 3481 pcie_flr(to_pci_dev(gd->gdma_context->dev)); 3482 if (err) { 3483 netdev_err(ndev, "flr failed %d with %d pkts pending in txq %u\n", 3484 err, 3485 atomic_read(&txq->pending_sends), 3486 txq->gdma_txq_id); 3487 } 3488 break; 3489 } 3490 } 3491 3492 for (i = 0; i < apc->num_queues; i++) { 3493 txq = &apc->tx_qp[i]->txq; 3494 while ((skb = skb_dequeue(&txq->pending_skbs))) { 3495 mana_unmap_skb(skb, apc); 3496 dev_kfree_skb_any(skb); 3497 } 3498 atomic_set(&txq->pending_sends, 0); 3499 } 3500 } 3501 3502 /* We're 100% sure the queues can no longer be woken up, because 3503 * we're sure now mana_poll_tx_cq() can't be running. 3504 */ 3505 3506 apc->rss_state = TRI_STATE_FALSE; 3507 err = mana_disable_vport_rx(apc); 3508 if (err && mana_en_need_log(apc, err)) 3509 netdev_err(ndev, "Failed to disable vPort: %d\n", err); 3510 3511 mana_fence_rqs(apc); 3512 3513 /* Even in err case, still need to cleanup the vPort */ 3514 mana_destroy_rxqs(apc); 3515 mana_destroy_txq(apc); 3516 mana_destroy_eq(apc); 3517 mana_destroy_vport(apc); 3518 3519 return 0; 3520 } 3521 3522 int mana_detach(struct net_device *ndev, bool from_close) 3523 { 3524 struct mana_port_context *apc = netdev_priv(ndev); 3525 int err; 3526 3527 ASSERT_RTNL(); 3528 3529 /* If already detached (indicates detach succeeded but attach failed 3530 * previously). Now skip mana detach and just retry mana_attach. 3531 */ 3532 if (!from_close && !netif_device_present(ndev)) 3533 return 0; 3534 3535 apc->port_st_save = apc->port_is_up; 3536 apc->port_is_up = false; 3537 3538 /* Ensure port state updated before txq state */ 3539 smp_wmb(); 3540 3541 netif_tx_disable(ndev); 3542 3543 if (apc->port_st_save) { 3544 err = mana_dealloc_queues(ndev); 3545 if (err) { 3546 netdev_err(ndev, "%s failed to deallocate queues: %d\n", __func__, err); 3547 return err; 3548 } 3549 } 3550 3551 if (!from_close) { 3552 netif_device_detach(ndev); 3553 mana_cleanup_port_context(apc); 3554 } 3555 3556 return 0; 3557 } 3558 3559 static int mana_probe_port(struct mana_context *ac, int port_idx, 3560 struct net_device **ndev_storage) 3561 { 3562 struct gdma_context *gc = ac->gdma_dev->gdma_context; 3563 struct mana_port_context *apc; 3564 struct net_device *ndev; 3565 int err; 3566 3567 ndev = alloc_etherdev_mq(sizeof(struct mana_port_context), 3568 gc->max_num_queues_vport); 3569 if (!ndev) 3570 return -ENOMEM; 3571 3572 *ndev_storage = ndev; 3573 3574 apc = netdev_priv(ndev); 3575 apc->ac = ac; 3576 apc->ndev = ndev; 3577 apc->max_queues = gc->max_num_queues_vport; 3578 /* Use MANA_DEF_NUM_QUEUES as default, still honoring the HW limit */ 3579 apc->num_queues = min(gc->max_num_queues_vport, MANA_DEF_NUM_QUEUES); 3580 apc->tx_queue_size = DEF_TX_BUFFERS_PER_QUEUE; 3581 apc->rx_queue_size = DEF_RX_BUFFERS_PER_QUEUE; 3582 apc->port_handle = INVALID_MANA_HANDLE; 3583 apc->pf_filter_handle = INVALID_MANA_HANDLE; 3584 apc->port_idx = port_idx; 3585 apc->link_cfg_error = 1; 3586 apc->cqe_coalescing_enable = 0; 3587 3588 mutex_init(&apc->vport_mutex); 3589 apc->vport_use_count = 0; 3590 3591 ndev->netdev_ops = &mana_devops; 3592 ndev->ethtool_ops = &mana_ethtool_ops; 3593 ndev->mtu = ETH_DATA_LEN; 3594 ndev->max_mtu = gc->adapter_mtu - ETH_HLEN; 3595 ndev->min_mtu = ETH_MIN_MTU; 3596 ndev->needed_headroom = MANA_HEADROOM; 3597 ndev->dev_port = port_idx; 3598 /* Recommended timeout based on HW FPGA re-config scenario. */ 3599 ndev->watchdog_timeo = 15 * HZ; 3600 SET_NETDEV_DEV(ndev, gc->dev); 3601 3602 netif_set_tso_max_size(ndev, GSO_MAX_SIZE); 3603 3604 netif_carrier_off(ndev); 3605 3606 netdev_rss_key_fill(apc->hashkey, MANA_HASH_KEY_SIZE); 3607 3608 err = mana_init_port(ndev); 3609 if (err) 3610 goto free_net; 3611 3612 err = mana_rss_table_alloc(apc); 3613 if (err) 3614 goto reset_apc; 3615 3616 /* Initialize the per port queue reset work.*/ 3617 INIT_WORK(&apc->queue_reset_work, 3618 mana_per_port_queue_reset_work_handler); 3619 3620 netdev_lockdep_set_classes(ndev); 3621 3622 ndev->hw_features = NETIF_F_SG | NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM; 3623 ndev->hw_features |= NETIF_F_RXCSUM; 3624 ndev->hw_features |= NETIF_F_TSO | NETIF_F_TSO6; 3625 ndev->hw_features |= NETIF_F_RXHASH; 3626 ndev->features = ndev->hw_features | NETIF_F_HW_VLAN_CTAG_TX | 3627 NETIF_F_HW_VLAN_CTAG_RX; 3628 ndev->vlan_features = ndev->features; 3629 xdp_set_features_flag(ndev, NETDEV_XDP_ACT_BASIC | 3630 NETDEV_XDP_ACT_REDIRECT | 3631 NETDEV_XDP_ACT_NDO_XMIT); 3632 3633 err = register_netdev(ndev); 3634 if (err) { 3635 netdev_err(ndev, "Unable to register netdev.\n"); 3636 goto free_indir; 3637 } 3638 3639 netif_carrier_on(ndev); 3640 3641 return 0; 3642 3643 free_indir: 3644 mana_cleanup_indir_table(apc); 3645 reset_apc: 3646 mana_cleanup_port_context(apc); 3647 free_net: 3648 *ndev_storage = NULL; 3649 netdev_err(ndev, "Failed to probe vPort %d: %d\n", port_idx, err); 3650 free_netdev(ndev); 3651 return err; 3652 } 3653 3654 static void adev_release(struct device *dev) 3655 { 3656 struct mana_adev *madev = container_of(dev, struct mana_adev, adev.dev); 3657 3658 kfree(madev); 3659 } 3660 3661 static void remove_adev(struct gdma_dev *gd) 3662 { 3663 struct auxiliary_device *adev = gd->adev; 3664 int id = adev->id; 3665 3666 auxiliary_device_delete(adev); 3667 auxiliary_device_uninit(adev); 3668 3669 mana_adev_idx_free(id); 3670 gd->adev = NULL; 3671 } 3672 3673 static int add_adev(struct gdma_dev *gd, const char *name) 3674 { 3675 struct auxiliary_device *adev; 3676 struct mana_adev *madev; 3677 int ret; 3678 int id; 3679 3680 madev = kzalloc_obj(*madev); 3681 if (!madev) 3682 return -ENOMEM; 3683 3684 adev = &madev->adev; 3685 ret = mana_adev_idx_alloc(); 3686 if (ret < 0) 3687 goto idx_fail; 3688 id = ret; 3689 adev->id = id; 3690 3691 adev->name = name; 3692 adev->dev.parent = gd->gdma_context->dev; 3693 adev->dev.release = adev_release; 3694 madev->mdev = gd; 3695 3696 ret = auxiliary_device_init(adev); 3697 if (ret) 3698 goto init_fail; 3699 3700 /* madev is owned by the auxiliary device */ 3701 madev = NULL; 3702 ret = auxiliary_device_add(adev); 3703 if (ret) 3704 goto add_fail; 3705 3706 gd->adev = adev; 3707 dev_dbg(gd->gdma_context->dev, 3708 "Auxiliary device added successfully\n"); 3709 return 0; 3710 3711 add_fail: 3712 auxiliary_device_uninit(adev); 3713 3714 init_fail: 3715 mana_adev_idx_free(id); 3716 3717 idx_fail: 3718 kfree(madev); 3719 3720 return ret; 3721 } 3722 3723 static void mana_rdma_service_handle(struct work_struct *work) 3724 { 3725 struct mana_service_work *serv_work = 3726 container_of(work, struct mana_service_work, work); 3727 struct gdma_dev *gd = serv_work->gdma_dev; 3728 struct device *dev = gd->gdma_context->dev; 3729 int ret; 3730 3731 if (READ_ONCE(gd->rdma_teardown)) 3732 goto out; 3733 3734 switch (serv_work->event) { 3735 case GDMA_SERVICE_TYPE_RDMA_SUSPEND: 3736 if (!gd->adev || gd->is_suspended) 3737 break; 3738 3739 remove_adev(gd); 3740 gd->is_suspended = true; 3741 break; 3742 3743 case GDMA_SERVICE_TYPE_RDMA_RESUME: 3744 if (!gd->is_suspended) 3745 break; 3746 3747 ret = add_adev(gd, "rdma"); 3748 if (ret) 3749 dev_err(dev, "Failed to add adev on resume: %d\n", ret); 3750 else 3751 gd->is_suspended = false; 3752 break; 3753 3754 default: 3755 dev_warn(dev, "unknown adev service event %u\n", 3756 serv_work->event); 3757 break; 3758 } 3759 3760 out: 3761 kfree(serv_work); 3762 } 3763 3764 int mana_rdma_service_event(struct gdma_context *gc, enum gdma_service_type event) 3765 { 3766 struct gdma_dev *gd = &gc->mana_ib; 3767 struct mana_service_work *serv_work; 3768 3769 if (gd->dev_id.type != GDMA_DEVICE_MANA_IB) { 3770 /* RDMA device is not detected on pci */ 3771 return 0; 3772 } 3773 3774 serv_work = kzalloc_obj(*serv_work, GFP_ATOMIC); 3775 if (!serv_work) 3776 return -ENOMEM; 3777 3778 serv_work->event = event; 3779 serv_work->gdma_dev = gd; 3780 3781 INIT_WORK(&serv_work->work, mana_rdma_service_handle); 3782 queue_work(gc->service_wq, &serv_work->work); 3783 3784 return 0; 3785 } 3786 3787 #define MANA_GF_STATS_PERIOD (2 * HZ) 3788 3789 static void mana_gf_stats_work_handler(struct work_struct *work) 3790 { 3791 struct mana_context *ac = 3792 container_of(to_delayed_work(work), struct mana_context, gf_stats_work); 3793 struct gdma_context *gc = ac->gdma_dev->gdma_context; 3794 int err; 3795 3796 err = mana_query_gf_stats(ac); 3797 if (err == -ETIMEDOUT) { 3798 /* HWC timeout detected - reset stats and stop rescheduling */ 3799 ac->hwc_timeout_occurred = true; 3800 memset(&ac->hc_stats, 0, sizeof(ac->hc_stats)); 3801 dev_warn(gc->dev, 3802 "Gf stats wk handler: gf stats query timed out.\n"); 3803 /* As HWC timed out, indicating a faulty HW state and needs a 3804 * reset. 3805 */ 3806 mana_schedule_serv_work(gc, GDMA_EQE_HWC_RESET_REQUEST); 3807 return; 3808 } 3809 schedule_delayed_work(&ac->gf_stats_work, MANA_GF_STATS_PERIOD); 3810 } 3811 3812 int mana_probe(struct gdma_dev *gd, bool resuming) 3813 { 3814 struct gdma_context *gc = gd->gdma_context; 3815 struct mana_context *ac = gd->driver_data; 3816 struct mana_port_context *apc = NULL; 3817 struct device *dev = gc->dev; 3818 u8 bm_hostmode = 0; 3819 u16 num_ports = 0; 3820 int err; 3821 int i; 3822 3823 dev_info(dev, 3824 "Microsoft Azure Network Adapter protocol version: %d.%d.%d\n", 3825 MANA_MAJOR_VERSION, MANA_MINOR_VERSION, MANA_MICRO_VERSION); 3826 3827 err = mana_gd_register_device(gd); 3828 if (err) 3829 return err; 3830 3831 if (!resuming) { 3832 ac = kzalloc_obj(*ac); 3833 if (!ac) 3834 return -ENOMEM; 3835 3836 ac->gdma_dev = gd; 3837 gd->driver_data = ac; 3838 3839 INIT_WORK(&ac->link_change_work, mana_link_state_handle); 3840 } 3841 3842 INIT_DELAYED_WORK(&ac->gf_stats_work, mana_gf_stats_work_handler); 3843 3844 err = mana_gd_query_device_cfg(gc, MANA_MAJOR_VERSION, 3845 MANA_MINOR_VERSION, 3846 MANA_MICRO_VERSION, 3847 &num_ports, &bm_hostmode); 3848 if (err) 3849 goto out; 3850 3851 ac->bm_hostmode = bm_hostmode; 3852 3853 debugfs_create_u16("adapter-MTU", 0400, 3854 gc->mana_pci_debugfs, &gc->adapter_mtu); 3855 3856 if (!resuming) { 3857 ac->num_ports = num_ports; 3858 } else { 3859 if (ac->num_ports != num_ports) { 3860 dev_err(dev, "The number of vPorts changed: %d->%d\n", 3861 ac->num_ports, num_ports); 3862 err = -EPROTO; 3863 goto out; 3864 } 3865 3866 enable_work(&ac->link_change_work); 3867 } 3868 3869 if (ac->num_ports > MAX_PORTS_IN_MANA_DEV) 3870 ac->num_ports = MAX_PORTS_IN_MANA_DEV; 3871 3872 debugfs_create_u16("num_vports", 0400, gc->mana_pci_debugfs, 3873 &ac->num_ports); 3874 debugfs_create_u8("bm_hostmode", 0400, gc->mana_pci_debugfs, 3875 &ac->bm_hostmode); 3876 3877 ac->per_port_queue_reset_wq = 3878 create_singlethread_workqueue("mana_per_port_queue_reset_wq"); 3879 if (!ac->per_port_queue_reset_wq) { 3880 dev_err(dev, "Failed to allocate per port queue reset workqueue\n"); 3881 err = -ENOMEM; 3882 goto out; 3883 } 3884 3885 if (!resuming) { 3886 for (i = 0; i < ac->num_ports; i++) { 3887 err = mana_probe_port(ac, i, &ac->ports[i]); 3888 /* Log the port for which the probe failed, stop probing 3889 * subsequent ports, and skip add_adev. 3890 * mana_remove() will clean up already-probed ports. 3891 */ 3892 if (err) { 3893 dev_err(dev, "Probe Failed for port %d\n", i); 3894 break; 3895 } 3896 } 3897 } else { 3898 for (i = 0; i < ac->num_ports; i++) { 3899 rtnl_lock(); 3900 apc = netdev_priv(ac->ports[i]); 3901 enable_work(&apc->queue_reset_work); 3902 netdev_lock(ac->ports[i]); 3903 apc->link_cfg_error = 1; 3904 netdev_unlock(ac->ports[i]); 3905 err = mana_attach(ac->ports[i]); 3906 rtnl_unlock(); 3907 /* Log the port for which the attach failed, stop 3908 * attaching subsequent ports, and skip add_adev. 3909 * mana_remove() will clean up already-attached ports. 3910 */ 3911 if (err) { 3912 dev_err(dev, "Attach Failed for port %d\n", i); 3913 break; 3914 } 3915 } 3916 } 3917 3918 if (!err) 3919 err = add_adev(gd, "eth"); 3920 3921 schedule_delayed_work(&ac->gf_stats_work, MANA_GF_STATS_PERIOD); 3922 3923 out: 3924 if (err) { 3925 mana_remove(gd, false); 3926 } else { 3927 dev_dbg(dev, "gd=%p, id=%u, num_ports=%d, type=%u, instance=%u\n", 3928 gd, gd->dev_id.as_uint32, ac->num_ports, 3929 gd->dev_id.type, gd->dev_id.instance); 3930 dev_dbg(dev, "%s succeeded\n", __func__); 3931 } 3932 3933 return err; 3934 } 3935 3936 void mana_remove(struct gdma_dev *gd, bool suspending) 3937 { 3938 struct gdma_context *gc = gd->gdma_context; 3939 struct mana_context *ac = gd->driver_data; 3940 struct mana_port_context *apc; 3941 struct device *dev; 3942 struct net_device *ndev; 3943 int err; 3944 int i; 3945 3946 if (!gc || !ac) 3947 return; 3948 3949 dev = gc->dev; 3950 3951 disable_work_sync(&ac->link_change_work); 3952 cancel_delayed_work_sync(&ac->gf_stats_work); 3953 3954 /* adev currently doesn't support suspending, always remove it */ 3955 if (gd->adev) 3956 remove_adev(gd); 3957 3958 for (i = 0; i < ac->num_ports; i++) { 3959 ndev = ac->ports[i]; 3960 if (!ndev) { 3961 if (i == 0) 3962 dev_err(dev, "No net device to remove\n"); 3963 break; 3964 } 3965 3966 apc = netdev_priv(ndev); 3967 disable_work_sync(&apc->queue_reset_work); 3968 3969 /* All cleanup actions should stay after rtnl_lock(), otherwise 3970 * other functions may access partially cleaned up data. 3971 */ 3972 rtnl_lock(); 3973 3974 err = mana_detach(ndev, false); 3975 if (err) 3976 netdev_err(ndev, "Failed to detach vPort %d: %d\n", 3977 i, err); 3978 3979 if (suspending) { 3980 /* No need to unregister the ndev. */ 3981 rtnl_unlock(); 3982 continue; 3983 } 3984 3985 unregister_netdevice(ndev); 3986 mana_cleanup_indir_table(apc); 3987 3988 rtnl_unlock(); 3989 3990 free_netdev(ndev); 3991 } 3992 3993 if (ac->per_port_queue_reset_wq) { 3994 destroy_workqueue(ac->per_port_queue_reset_wq); 3995 ac->per_port_queue_reset_wq = NULL; 3996 } 3997 3998 mana_gd_deregister_device(gd); 3999 4000 if (gc->mana_pci_debugfs) { 4001 debugfs_lookup_and_remove("bm_hostmode", gc->mana_pci_debugfs); 4002 debugfs_lookup_and_remove("num_vports", gc->mana_pci_debugfs); 4003 } 4004 4005 if (suspending) 4006 return; 4007 4008 gd->driver_data = NULL; 4009 gd->gdma_context = NULL; 4010 kfree(ac); 4011 dev_dbg(dev, "%s succeeded\n", __func__); 4012 } 4013 4014 int mana_rdma_probe(struct gdma_dev *gd) 4015 { 4016 int err = 0; 4017 4018 if (gd->dev_id.type != GDMA_DEVICE_MANA_IB) { 4019 /* RDMA device is not detected on pci */ 4020 return err; 4021 } 4022 4023 err = mana_gd_register_device(gd); 4024 if (err) 4025 return err; 4026 4027 err = add_adev(gd, "rdma"); 4028 if (err) 4029 mana_gd_deregister_device(gd); 4030 4031 return err; 4032 } 4033 4034 void mana_rdma_remove(struct gdma_dev *gd) 4035 { 4036 struct gdma_context *gc = gd->gdma_context; 4037 4038 if (gd->dev_id.type != GDMA_DEVICE_MANA_IB) { 4039 /* RDMA device is not detected on pci */ 4040 return; 4041 } 4042 4043 WRITE_ONCE(gd->rdma_teardown, true); 4044 4045 if (gc->service_wq) 4046 flush_workqueue(gc->service_wq); 4047 4048 if (gd->adev) 4049 remove_adev(gd); 4050 4051 mana_gd_deregister_device(gd); 4052 } 4053 4054 struct net_device *mana_get_primary_netdev(struct mana_context *ac, 4055 u32 port_index, 4056 netdevice_tracker *tracker) 4057 { 4058 struct net_device *ndev; 4059 4060 if (port_index >= ac->num_ports) 4061 return NULL; 4062 4063 rcu_read_lock(); 4064 4065 /* If mana is used in netvsc, the upper netdevice should be returned. */ 4066 ndev = netdev_master_upper_dev_get_rcu(ac->ports[port_index]); 4067 4068 /* If there is no upper device, use the parent Ethernet device */ 4069 if (!ndev) 4070 ndev = ac->ports[port_index]; 4071 4072 netdev_hold(ndev, tracker, GFP_ATOMIC); 4073 rcu_read_unlock(); 4074 4075 return ndev; 4076 } 4077 EXPORT_SYMBOL_NS(mana_get_primary_netdev, "NET_MANA"); 4078