1 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause 2 /* Copyright (c) 2021, Microsoft Corporation. */ 3 4 #include <uapi/linux/bpf.h> 5 6 #include <linux/debugfs.h> 7 #include <linux/inetdevice.h> 8 #include <linux/etherdevice.h> 9 #include <linux/ethtool.h> 10 #include <linux/filter.h> 11 #include <linux/mm.h> 12 #include <linux/export.h> 13 #include <linux/skbuff.h> 14 15 #include <net/checksum.h> 16 #include <net/ip6_checksum.h> 17 #include <net/netdev_lock.h> 18 #include <net/page_pool/helpers.h> 19 #include <net/xdp.h> 20 21 #include <net/mana/mana.h> 22 23 #include <net/mana/mana_auxiliary.h> 24 #include <net/mana/hw_channel.h> 25 26 static DEFINE_IDA(mana_adev_ida); 27 28 static int mana_adev_idx_alloc(void) 29 { 30 return ida_alloc(&mana_adev_ida, GFP_KERNEL); 31 } 32 33 static void mana_adev_idx_free(int idx) 34 { 35 ida_free(&mana_adev_ida, idx); 36 } 37 38 static ssize_t mana_dbg_q_read(struct file *filp, char __user *buf, size_t count, 39 loff_t *pos) 40 { 41 struct gdma_queue *gdma_q = filp->private_data; 42 43 if (gdma_q->mem_info.nr_pages) 44 return mana_gd_read_ring(gdma_q, buf, count, pos); 45 46 return simple_read_from_buffer(buf, count, pos, gdma_q->queue_mem_ptr, 47 gdma_q->queue_size); 48 } 49 50 static const struct file_operations mana_dbg_q_fops = { 51 .owner = THIS_MODULE, 52 .open = simple_open, 53 .read = mana_dbg_q_read, 54 }; 55 56 static bool mana_en_need_log(struct mana_port_context *apc, int err) 57 { 58 if (apc && apc->ac && apc->ac->gdma_dev && 59 apc->ac->gdma_dev->gdma_context) 60 return mana_need_log(apc->ac->gdma_dev->gdma_context, err); 61 else 62 return true; 63 } 64 65 static void mana_put_rx_page(struct mana_rxq *rxq, struct page *page, 66 bool from_pool) 67 { 68 if (from_pool) 69 page_pool_put_full_page(rxq->page_pool, page, false); 70 else 71 put_page(page); 72 } 73 74 /* Microsoft Azure Network Adapter (MANA) functions */ 75 76 static int mana_open(struct net_device *ndev) 77 { 78 struct mana_port_context *apc = netdev_priv(ndev); 79 int err; 80 err = mana_alloc_queues(ndev); 81 82 if (err) { 83 netdev_err(ndev, "%s failed to allocate queues: %d\n", __func__, err); 84 return err; 85 } 86 87 apc->port_is_up = true; 88 89 /* Ensure port state updated before txq state */ 90 smp_wmb(); 91 92 netif_tx_wake_all_queues(ndev); 93 netdev_dbg(ndev, "%s successful\n", __func__); 94 return 0; 95 } 96 97 static int mana_close(struct net_device *ndev) 98 { 99 struct mana_port_context *apc = netdev_priv(ndev); 100 101 if (!apc->port_is_up) 102 return 0; 103 104 return mana_detach(ndev, true); 105 } 106 107 static void mana_link_state_handle(struct work_struct *w) 108 { 109 struct mana_context *ac; 110 struct net_device *ndev; 111 u32 link_event; 112 bool link_up; 113 int i; 114 115 ac = container_of(w, struct mana_context, link_change_work); 116 117 rtnl_lock(); 118 119 link_event = READ_ONCE(ac->link_event); 120 121 if (link_event == HWC_DATA_HW_LINK_CONNECT) 122 link_up = true; 123 else if (link_event == HWC_DATA_HW_LINK_DISCONNECT) 124 link_up = false; 125 else 126 goto out; 127 128 /* Process all ports */ 129 for (i = 0; i < ac->num_ports; i++) { 130 ndev = ac->ports[i]; 131 if (!ndev) 132 continue; 133 134 if (link_up) { 135 netif_carrier_on(ndev); 136 137 __netdev_notify_peers(ndev); 138 } else { 139 netif_carrier_off(ndev); 140 } 141 } 142 143 out: 144 rtnl_unlock(); 145 } 146 147 static bool mana_can_tx(struct gdma_queue *wq) 148 { 149 return mana_gd_wq_avail_space(wq) >= MAX_TX_WQE_SIZE; 150 } 151 152 static unsigned int mana_checksum_info(struct sk_buff *skb) 153 { 154 if (skb->protocol == htons(ETH_P_IP)) { 155 struct iphdr *ip = ip_hdr(skb); 156 157 if (ip->protocol == IPPROTO_TCP) 158 return IPPROTO_TCP; 159 160 if (ip->protocol == IPPROTO_UDP) 161 return IPPROTO_UDP; 162 } else if (skb->protocol == htons(ETH_P_IPV6)) { 163 struct ipv6hdr *ip6 = ipv6_hdr(skb); 164 165 if (ip6->nexthdr == IPPROTO_TCP) 166 return IPPROTO_TCP; 167 168 if (ip6->nexthdr == IPPROTO_UDP) 169 return IPPROTO_UDP; 170 } 171 172 /* No csum offloading */ 173 return 0; 174 } 175 176 static void mana_add_sge(struct mana_tx_package *tp, struct mana_skb_head *ash, 177 int sg_i, dma_addr_t da, int sge_len, u32 gpa_mkey) 178 { 179 ash->dma_handle[sg_i] = da; 180 ash->size[sg_i] = sge_len; 181 182 tp->wqe_req.sgl[sg_i].address = da; 183 tp->wqe_req.sgl[sg_i].mem_key = gpa_mkey; 184 tp->wqe_req.sgl[sg_i].size = sge_len; 185 } 186 187 static int mana_map_skb(struct sk_buff *skb, struct mana_port_context *apc, 188 struct mana_tx_package *tp, int gso_hs) 189 { 190 struct mana_skb_head *ash = (struct mana_skb_head *)skb->head; 191 int hsg = 1; /* num of SGEs of linear part */ 192 struct gdma_dev *gd = apc->ac->gdma_dev; 193 int skb_hlen = skb_headlen(skb); 194 int sge0_len, sge1_len = 0; 195 struct gdma_context *gc; 196 struct device *dev; 197 skb_frag_t *frag; 198 dma_addr_t da; 199 int sg_i; 200 int i; 201 202 gc = gd->gdma_context; 203 dev = gc->dev; 204 205 if (gso_hs && gso_hs < skb_hlen) { 206 sge0_len = gso_hs; 207 sge1_len = skb_hlen - gso_hs; 208 } else { 209 sge0_len = skb_hlen; 210 } 211 212 da = dma_map_single(dev, skb->data, sge0_len, DMA_TO_DEVICE); 213 if (dma_mapping_error(dev, da)) 214 return -ENOMEM; 215 216 mana_add_sge(tp, ash, 0, da, sge0_len, gd->gpa_mkey); 217 218 if (sge1_len) { 219 sg_i = 1; 220 da = dma_map_single(dev, skb->data + sge0_len, sge1_len, 221 DMA_TO_DEVICE); 222 if (dma_mapping_error(dev, da)) 223 goto frag_err; 224 225 mana_add_sge(tp, ash, sg_i, da, sge1_len, gd->gpa_mkey); 226 hsg = 2; 227 } 228 229 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 230 sg_i = hsg + i; 231 232 frag = &skb_shinfo(skb)->frags[i]; 233 da = skb_frag_dma_map(dev, frag, 0, skb_frag_size(frag), 234 DMA_TO_DEVICE); 235 if (dma_mapping_error(dev, da)) 236 goto frag_err; 237 238 mana_add_sge(tp, ash, sg_i, da, skb_frag_size(frag), 239 gd->gpa_mkey); 240 } 241 242 return 0; 243 244 frag_err: 245 if (net_ratelimit()) 246 netdev_err(apc->ndev, "Failed to map skb of size %u to DMA\n", 247 skb->len); 248 for (i = sg_i - 1; i >= hsg; i--) 249 dma_unmap_page(dev, ash->dma_handle[i], ash->size[i], 250 DMA_TO_DEVICE); 251 252 for (i = hsg - 1; i >= 0; i--) 253 dma_unmap_single(dev, ash->dma_handle[i], ash->size[i], 254 DMA_TO_DEVICE); 255 256 return -ENOMEM; 257 } 258 259 /* Handle the case when GSO SKB linear length is too large. 260 * MANA NIC requires GSO packets to put only the packet header to SGE0. 261 * So, we need 2 SGEs for the skb linear part which contains more than the 262 * header. 263 * Return a positive value for the number of SGEs, or a negative value 264 * for an error. 265 */ 266 static int mana_fix_skb_head(struct net_device *ndev, struct sk_buff *skb, 267 int gso_hs) 268 { 269 int num_sge = 1 + skb_shinfo(skb)->nr_frags; 270 int skb_hlen = skb_headlen(skb); 271 272 if (gso_hs < skb_hlen) { 273 num_sge++; 274 } else if (gso_hs > skb_hlen) { 275 if (net_ratelimit()) 276 netdev_err(ndev, 277 "TX nonlinear head: hs:%d, skb_hlen:%d\n", 278 gso_hs, skb_hlen); 279 280 return -EINVAL; 281 } 282 283 return num_sge; 284 } 285 286 /* Get the GSO packet's header size */ 287 static int mana_get_gso_hs(struct sk_buff *skb) 288 { 289 int gso_hs; 290 291 if (skb->encapsulation) { 292 gso_hs = skb_inner_tcp_all_headers(skb); 293 } else { 294 if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4) { 295 gso_hs = skb_transport_offset(skb) + 296 sizeof(struct udphdr); 297 } else { 298 gso_hs = skb_tcp_all_headers(skb); 299 } 300 } 301 302 return gso_hs; 303 } 304 305 static void mana_per_port_queue_reset_work_handler(struct work_struct *work) 306 { 307 struct mana_port_context *apc = container_of(work, 308 struct mana_port_context, 309 queue_reset_work); 310 struct net_device *ndev = apc->ndev; 311 int err; 312 313 rtnl_lock(); 314 315 /* Block RDMA from grabbing the vport during the detach/attach 316 * window, same as mana_set_channels(). 317 */ 318 mutex_lock(&apc->vport_mutex); 319 apc->channel_changing = true; 320 mutex_unlock(&apc->vport_mutex); 321 322 /* Pre-allocate buffers to prevent failure in mana_attach later */ 323 err = mana_pre_alloc_rxbufs(apc, ndev->mtu, apc->num_queues); 324 if (err) { 325 netdev_err(ndev, "Insufficient memory for reset post tx stall detection\n"); 326 goto clear_flag; 327 } 328 329 err = mana_detach(ndev, false); 330 if (err) { 331 netdev_err(ndev, "mana_detach failed: %d\n", err); 332 goto dealloc_pre_rxbufs; 333 } 334 335 err = mana_attach(ndev); 336 if (err) 337 netdev_err(ndev, "mana_attach failed: %d\n", err); 338 339 dealloc_pre_rxbufs: 340 mana_pre_dealloc_rxbufs(apc); 341 clear_flag: 342 mutex_lock(&apc->vport_mutex); 343 apc->channel_changing = false; 344 mutex_unlock(&apc->vport_mutex); 345 346 rtnl_unlock(); 347 } 348 349 /* skb->cb of the SKBs pending a TX completion */ 350 struct mana_skb_cb { 351 struct gdma_posted_wqe_info wqe_info; 352 bool xdp; 353 }; 354 355 netdev_tx_t __mana_start_xmit(struct sk_buff *skb, struct net_device *ndev, 356 bool xdp) 357 { 358 enum mana_tx_pkt_format pkt_fmt = MANA_SHORT_PKT_FMT; 359 struct mana_port_context *apc = netdev_priv(ndev); 360 int gso_hs = 0; /* zero for non-GSO pkts */ 361 u16 txq_idx = skb_get_queue_mapping(skb); 362 struct gdma_dev *gd = apc->ac->gdma_dev; 363 bool ipv4 = false, ipv6 = false; 364 struct mana_tx_package pkg = {}; 365 struct netdev_queue *net_txq; 366 struct mana_stats_tx *tx_stats; 367 struct gdma_queue *gdma_sq; 368 int err, len, num_gso_seg; 369 unsigned int csum_type; 370 struct mana_skb_cb *cb; 371 struct mana_txq *txq; 372 struct mana_cq *cq; 373 374 if (unlikely(!apc->port_is_up)) 375 goto tx_drop; 376 377 if (skb_cow_head(skb, MANA_HEADROOM)) 378 goto tx_drop_count; 379 380 txq = &apc->tx_qp[txq_idx]->txq; 381 gdma_sq = txq->gdma_sq; 382 cq = &apc->tx_qp[txq_idx]->tx_cq; 383 tx_stats = &txq->stats; 384 385 BUILD_BUG_ON(MAX_TX_WQE_SGL_ENTRIES != MANA_MAX_TX_WQE_SGL_ENTRIES); 386 if (MAX_SKB_FRAGS + 2 > MAX_TX_WQE_SGL_ENTRIES && 387 skb_shinfo(skb)->nr_frags + 2 > MAX_TX_WQE_SGL_ENTRIES) { 388 /* GSO skb with Hardware SGE limit exceeded is not expected here 389 * as they are handled in mana_features_check() callback 390 */ 391 if (skb_linearize(skb)) { 392 netdev_warn_once(ndev, "Failed to linearize skb with nr_frags=%d and is_gso=%d\n", 393 skb_shinfo(skb)->nr_frags, 394 skb_is_gso(skb)); 395 goto tx_drop_count; 396 } 397 apc->eth_stats.tx_linear_pkt_cnt++; 398 } 399 400 pkg.tx_oob.s_oob.vcq_num = cq->gdma_id; 401 pkg.tx_oob.s_oob.vsq_frame = txq->vsq_frame; 402 403 if (txq->vp_offset > MANA_SHORT_VPORT_OFFSET_MAX) { 404 pkg.tx_oob.l_oob.long_vp_offset = txq->vp_offset; 405 pkt_fmt = MANA_LONG_PKT_FMT; 406 } else { 407 pkg.tx_oob.s_oob.short_vp_offset = txq->vp_offset; 408 } 409 410 if (skb_vlan_tag_present(skb)) { 411 pkt_fmt = MANA_LONG_PKT_FMT; 412 pkg.tx_oob.l_oob.inject_vlan_pri_tag = 1; 413 pkg.tx_oob.l_oob.pcp = skb_vlan_tag_get_prio(skb); 414 pkg.tx_oob.l_oob.dei = skb_vlan_tag_get_cfi(skb); 415 pkg.tx_oob.l_oob.vlan_id = skb_vlan_tag_get_id(skb); 416 } 417 418 pkg.tx_oob.s_oob.pkt_fmt = pkt_fmt; 419 420 if (pkt_fmt == MANA_SHORT_PKT_FMT) { 421 pkg.wqe_req.inline_oob_size = sizeof(struct mana_tx_short_oob); 422 u64_stats_update_begin(&tx_stats->syncp); 423 tx_stats->short_pkt_fmt++; 424 u64_stats_update_end(&tx_stats->syncp); 425 } else { 426 pkg.wqe_req.inline_oob_size = sizeof(struct mana_tx_oob); 427 u64_stats_update_begin(&tx_stats->syncp); 428 tx_stats->long_pkt_fmt++; 429 u64_stats_update_end(&tx_stats->syncp); 430 } 431 432 pkg.wqe_req.inline_oob_data = &pkg.tx_oob; 433 pkg.wqe_req.flags = 0; 434 pkg.wqe_req.client_data_unit = 0; 435 436 pkg.wqe_req.num_sge = 1 + skb_shinfo(skb)->nr_frags; 437 438 if (skb->protocol == htons(ETH_P_IP)) 439 ipv4 = true; 440 else if (skb->protocol == htons(ETH_P_IPV6)) 441 ipv6 = true; 442 443 if (skb_is_gso(skb)) { 444 int num_sge; 445 446 gso_hs = mana_get_gso_hs(skb); 447 448 num_sge = mana_fix_skb_head(ndev, skb, gso_hs); 449 if (num_sge > 0) 450 pkg.wqe_req.num_sge = num_sge; 451 else 452 goto tx_drop_count; 453 454 u64_stats_update_begin(&tx_stats->syncp); 455 if (skb->encapsulation) { 456 tx_stats->tso_inner_packets++; 457 tx_stats->tso_inner_bytes += skb->len - gso_hs; 458 } else { 459 tx_stats->tso_packets++; 460 tx_stats->tso_bytes += skb->len - gso_hs; 461 } 462 u64_stats_update_end(&tx_stats->syncp); 463 464 pkg.tx_oob.s_oob.is_outer_ipv4 = ipv4; 465 pkg.tx_oob.s_oob.is_outer_ipv6 = ipv6; 466 467 pkg.tx_oob.s_oob.comp_iphdr_csum = 1; 468 pkg.tx_oob.s_oob.comp_tcp_csum = 1; 469 pkg.tx_oob.s_oob.trans_off = skb_transport_offset(skb); 470 471 pkg.wqe_req.client_data_unit = skb_shinfo(skb)->gso_size; 472 pkg.wqe_req.flags = GDMA_WR_OOB_IN_SGL | GDMA_WR_PAD_BY_SGE0; 473 if (ipv4) { 474 ip_hdr(skb)->tot_len = 0; 475 ip_hdr(skb)->check = 0; 476 tcp_hdr(skb)->check = 477 ~csum_tcpudp_magic(ip_hdr(skb)->saddr, 478 ip_hdr(skb)->daddr, 0, 479 IPPROTO_TCP, 0); 480 } else { 481 ipv6_hdr(skb)->payload_len = 0; 482 tcp_hdr(skb)->check = 483 ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr, 484 &ipv6_hdr(skb)->daddr, 0, 485 IPPROTO_TCP, 0); 486 } 487 } else if (skb->ip_summed == CHECKSUM_PARTIAL) { 488 csum_type = mana_checksum_info(skb); 489 490 u64_stats_update_begin(&tx_stats->syncp); 491 tx_stats->csum_partial++; 492 u64_stats_update_end(&tx_stats->syncp); 493 494 if (csum_type == IPPROTO_TCP) { 495 pkg.tx_oob.s_oob.is_outer_ipv4 = ipv4; 496 pkg.tx_oob.s_oob.is_outer_ipv6 = ipv6; 497 498 pkg.tx_oob.s_oob.comp_tcp_csum = 1; 499 pkg.tx_oob.s_oob.trans_off = skb_transport_offset(skb); 500 501 } else if (csum_type == IPPROTO_UDP) { 502 pkg.tx_oob.s_oob.is_outer_ipv4 = ipv4; 503 pkg.tx_oob.s_oob.is_outer_ipv6 = ipv6; 504 505 pkg.tx_oob.s_oob.comp_udp_csum = 1; 506 } else { 507 /* Can't do offload of this type of checksum */ 508 if (skb_checksum_help(skb)) 509 goto tx_drop_count; 510 } 511 } 512 513 if (pkg.wqe_req.num_sge <= ARRAY_SIZE(pkg.sgl_array)) { 514 pkg.wqe_req.sgl = pkg.sgl_array; 515 } else { 516 pkg.sgl_ptr = kmalloc_objs(struct gdma_sge, pkg.wqe_req.num_sge, 517 GFP_ATOMIC); 518 if (!pkg.sgl_ptr) 519 goto tx_drop_count; 520 521 pkg.wqe_req.sgl = pkg.sgl_ptr; 522 } 523 524 if (mana_map_skb(skb, apc, &pkg, gso_hs)) { 525 u64_stats_update_begin(&tx_stats->syncp); 526 tx_stats->mana_map_err++; 527 u64_stats_update_end(&tx_stats->syncp); 528 goto free_sgl_ptr; 529 } 530 531 cb = (struct mana_skb_cb *)skb->cb; 532 cb->xdp = xdp; 533 534 skb_queue_tail(&txq->pending_skbs, skb); 535 536 len = skb->len; 537 num_gso_seg = skb_is_gso(skb) ? skb_shinfo(skb)->gso_segs : 1; 538 net_txq = netdev_get_tx_queue(ndev, txq_idx); 539 540 err = mana_gd_post_work_request(gdma_sq, &pkg.wqe_req, &cb->wqe_info); 541 if (!mana_can_tx(gdma_sq)) { 542 netif_tx_stop_queue(net_txq); 543 apc->eth_stats.stop_queue++; 544 } 545 546 if (err) { 547 (void)skb_dequeue_tail(&txq->pending_skbs); 548 mana_unmap_skb(skb, apc); 549 netdev_warn(ndev, "Failed to post TX OOB: %d\n", err); 550 goto free_sgl_ptr; 551 } 552 553 err = NETDEV_TX_OK; 554 atomic_inc(&txq->pending_sends); 555 556 /* Account the skb in BQL before the doorbell lets the hardware 557 * complete it. XDP frames bypass the qdisc and BQL can't hold them 558 * back, so leave them out: XDP traffic could otherwise keep the 559 * queue stopped, starving the stack and tripping the TX watchdog. 560 */ 561 if (!xdp) 562 netdev_tx_sent_queue(net_txq, len); 563 564 mana_gd_wq_ring_doorbell(gd->gdma_context, gdma_sq); 565 566 /* skb may be freed after mana_gd_post_work_request. Do not use it. */ 567 skb = NULL; 568 569 /* Populated the packet and bytes counters based on post GSO packet 570 * calculations 571 */ 572 tx_stats = &txq->stats; 573 u64_stats_update_begin(&tx_stats->syncp); 574 tx_stats->packets += num_gso_seg; 575 tx_stats->bytes += len + ((num_gso_seg - 1) * gso_hs); 576 u64_stats_update_end(&tx_stats->syncp); 577 578 if (netif_tx_queue_stopped(net_txq) && mana_can_tx(gdma_sq)) { 579 netif_tx_wake_queue(net_txq); 580 apc->eth_stats.wake_queue++; 581 } 582 583 kfree(pkg.sgl_ptr); 584 return err; 585 586 free_sgl_ptr: 587 kfree(pkg.sgl_ptr); 588 tx_drop_count: 589 ndev->stats.tx_dropped++; 590 tx_drop: 591 dev_kfree_skb_any(skb); 592 return NETDEV_TX_OK; 593 } 594 595 netdev_tx_t mana_start_xmit(struct sk_buff *skb, struct net_device *ndev) 596 { 597 return __mana_start_xmit(skb, ndev, false); 598 } 599 600 #if (MAX_SKB_FRAGS + 2 > MANA_MAX_TX_WQE_SGL_ENTRIES) 601 static netdev_features_t mana_features_check(struct sk_buff *skb, 602 struct net_device *ndev, 603 netdev_features_t features) 604 { 605 if (skb_shinfo(skb)->nr_frags + 2 > MAX_TX_WQE_SGL_ENTRIES) { 606 /* Exceeds HW SGE limit. 607 * GSO case: 608 * Disable GSO so the stack will software-segment the skb 609 * into smaller skbs that fit the SGE budget. 610 * Non-GSO case: 611 * The xmit path will attempt skb_linearize() as a fallback. 612 */ 613 features &= ~NETIF_F_GSO_MASK; 614 } 615 return features; 616 } 617 #endif 618 619 static void mana_get_stats64(struct net_device *ndev, 620 struct rtnl_link_stats64 *st) 621 { 622 struct mana_port_context *apc = netdev_priv(ndev); 623 unsigned int num_queues = apc->num_queues; 624 struct mana_stats_rx *rx_stats; 625 struct mana_stats_tx *tx_stats; 626 unsigned int start; 627 u64 packets, bytes; 628 int q; 629 630 if (!apc->port_is_up) 631 return; 632 633 netdev_stats_to_stats64(st, &ndev->stats); 634 635 if (apc->ac->hwc_timeout_occurred) 636 netdev_warn_once(ndev, "HWC timeout occurred\n"); 637 638 st->rx_missed_errors = apc->ac->hc_stats.hc_rx_discards_no_wqe; 639 640 for (q = 0; q < num_queues; q++) { 641 rx_stats = &apc->rxqs[q]->stats; 642 643 do { 644 start = u64_stats_fetch_begin(&rx_stats->syncp); 645 packets = rx_stats->packets; 646 bytes = rx_stats->bytes; 647 } while (u64_stats_fetch_retry(&rx_stats->syncp, start)); 648 649 st->rx_packets += packets; 650 st->rx_bytes += bytes; 651 } 652 653 for (q = 0; q < num_queues; q++) { 654 tx_stats = &apc->tx_qp[q]->txq.stats; 655 656 do { 657 start = u64_stats_fetch_begin(&tx_stats->syncp); 658 packets = tx_stats->packets; 659 bytes = tx_stats->bytes; 660 } while (u64_stats_fetch_retry(&tx_stats->syncp, start)); 661 662 st->tx_packets += packets; 663 st->tx_bytes += bytes; 664 } 665 } 666 667 static int mana_get_tx_queue(struct net_device *ndev, struct sk_buff *skb, 668 int old_q) 669 { 670 struct mana_port_context *apc = netdev_priv(ndev); 671 u32 hash = skb_get_hash(skb); 672 struct sock *sk = skb->sk; 673 int txq; 674 675 txq = apc->indir_table[hash & (apc->indir_table_sz - 1)]; 676 677 if (txq != old_q && sk && sk_fullsock(sk) && 678 rcu_access_pointer(sk->sk_dst_cache)) 679 sk_tx_queue_set(sk, txq); 680 681 return txq; 682 } 683 684 static u16 mana_select_queue(struct net_device *ndev, struct sk_buff *skb, 685 struct net_device *sb_dev) 686 { 687 int txq; 688 689 if (ndev->real_num_tx_queues == 1) 690 return 0; 691 692 txq = sk_tx_queue_get(skb->sk); 693 694 if (txq < 0 || skb->ooo_okay || txq >= ndev->real_num_tx_queues) { 695 if (skb_rx_queue_recorded(skb)) 696 txq = skb_get_rx_queue(skb); 697 else 698 txq = mana_get_tx_queue(ndev, skb, txq); 699 } 700 701 return txq; 702 } 703 704 /* Release pre-allocated RX buffers */ 705 void mana_pre_dealloc_rxbufs(struct mana_port_context *mpc) 706 { 707 struct device *dev; 708 int i; 709 710 dev = mpc->ac->gdma_dev->gdma_context->dev; 711 712 if (!mpc->rxbufs_pre) 713 goto out1; 714 715 if (!mpc->das_pre) 716 goto out2; 717 718 while (mpc->rxbpre_total) { 719 i = --mpc->rxbpre_total; 720 dma_unmap_single(dev, mpc->das_pre[i], mpc->rxbpre_datasize, 721 DMA_FROM_DEVICE); 722 put_page(virt_to_head_page(mpc->rxbufs_pre[i])); 723 } 724 725 kvfree(mpc->das_pre); 726 mpc->das_pre = NULL; 727 728 out2: 729 kvfree(mpc->rxbufs_pre); 730 mpc->rxbufs_pre = NULL; 731 732 out1: 733 mpc->rxbpre_datasize = 0; 734 mpc->rxbpre_alloc_size = 0; 735 mpc->rxbpre_headroom = 0; 736 } 737 738 /* Get a buffer from the pre-allocated RX buffers */ 739 static void *mana_get_rxbuf_pre(struct mana_rxq *rxq, dma_addr_t *da) 740 { 741 struct net_device *ndev = rxq->ndev; 742 struct mana_port_context *mpc; 743 void *va; 744 745 mpc = netdev_priv(ndev); 746 747 if (!mpc->rxbufs_pre || !mpc->das_pre || !mpc->rxbpre_total) { 748 netdev_err(ndev, "No RX pre-allocated bufs\n"); 749 return NULL; 750 } 751 752 /* Check sizes to catch unexpected coding error */ 753 if (mpc->rxbpre_datasize != rxq->datasize) { 754 netdev_err(ndev, "rxbpre_datasize mismatch: %u: %u\n", 755 mpc->rxbpre_datasize, rxq->datasize); 756 return NULL; 757 } 758 759 if (mpc->rxbpre_alloc_size != rxq->alloc_size) { 760 netdev_err(ndev, "rxbpre_alloc_size mismatch: %u: %u\n", 761 mpc->rxbpre_alloc_size, rxq->alloc_size); 762 return NULL; 763 } 764 765 if (mpc->rxbpre_headroom != rxq->headroom) { 766 netdev_err(ndev, "rxbpre_headroom mismatch: %u: %u\n", 767 mpc->rxbpre_headroom, rxq->headroom); 768 return NULL; 769 } 770 771 mpc->rxbpre_total--; 772 773 *da = mpc->das_pre[mpc->rxbpre_total]; 774 va = mpc->rxbufs_pre[mpc->rxbpre_total]; 775 mpc->rxbufs_pre[mpc->rxbpre_total] = NULL; 776 777 /* Deallocate the array after all buffers are gone */ 778 if (!mpc->rxbpre_total) 779 mana_pre_dealloc_rxbufs(mpc); 780 781 return va; 782 } 783 784 /* Reserve enough headroom to satisfy the skb_cow() in ip_forward() and avoid 785 * reallocation: the TX path keeps the SGE DMA mappings in struct mana_skb_head 786 * at skb->head, so the port advertises ndev->needed_headroom = MANA_HEADROOM. 787 */ 788 static u32 mana_get_rxbuf_headroom(struct mana_port_context *apc) 789 { 790 if (mana_xdp_get(apc)) 791 return mana_xdp_headroom(apc->ndev); 792 793 return LL_RESERVED_SPACE(apc->ndev); 794 } 795 796 static u32 mana_get_rxbuf_size(struct mana_port_context *apc, u32 mtu) 797 { 798 u32 len = SKB_DATA_ALIGN(mtu + MANA_RXBUF_PAD + 799 mana_get_rxbuf_headroom(apc)); 800 801 return ALIGN(len, MANA_RX_FRAG_ALIGNMENT); 802 } 803 804 /* Returns true when one RX buffer per page is already required by XDP or by 805 * the buffer size implied by the MTU, i.e. regardless of the 806 * MANA_PRIV_FLAG_USE_FULL_PAGE_RXBUF private flag. 807 */ 808 bool mana_single_rxbuf_per_page_forced(struct mana_port_context *apc, u32 mtu) 809 { 810 /* For xdp make sure only one packet fits per page. */ 811 if (mana_xdp_get(apc)) 812 return true; 813 814 /* Only use the page_pool fragment path when at least two buffers, 815 * including the headroom each of them has to reserve, actually fit 816 * into one page. Otherwise the fragment path degenerates into one 817 * buffer per page while still paying the fragment accounting cost. 818 */ 819 return PAGE_SIZE / mana_get_rxbuf_size(apc, mtu) < 2; 820 } 821 822 static bool 823 mana_use_single_rxbuf_per_page(struct mana_port_context *apc, u32 mtu) 824 { 825 /* On some platforms with 4K PAGE_SIZE, page_pool fragment allocation 826 * in the RX refill path (~2kB buffer) can cause significant throughput 827 * regression under high connection counts. Allow user to force one RX 828 * buffer per page via ethtool private flag to bypass the fragment 829 * path. 830 */ 831 if (apc->priv_flags & BIT(MANA_PRIV_FLAG_USE_FULL_PAGE_RXBUF)) 832 return true; 833 834 return mana_single_rxbuf_per_page_forced(apc, mtu); 835 } 836 837 /* Get RX buffer's data size, alloc size, headroom and frag count based on MTU */ 838 static void mana_get_rxbuf_cfg(struct mana_port_context *apc, 839 int mtu, u32 *datasize, u32 *alloc_size, 840 u32 *headroom, u32 *frag_count) 841 { 842 u32 buf_size; 843 844 /* Calculate datasize first (consistent across all cases) */ 845 *datasize = mtu + ETH_HLEN; 846 847 *headroom = mana_get_rxbuf_headroom(apc); 848 849 if (mana_use_single_rxbuf_per_page(apc, mtu)) { 850 if (mana_xdp_get(apc)) 851 *alloc_size = PAGE_SIZE; 852 else 853 *alloc_size = SKB_DATA_ALIGN(mtu + MANA_RXBUF_PAD + 854 *headroom); 855 856 *frag_count = 1; 857 858 /* In the single-buffer path, napi_build_skb() must see the 859 * actual backing allocation size so skb->truesize reflects 860 * the full page (or higher-order page), not just the usable 861 * packet area. 862 */ 863 *alloc_size = PAGE_SIZE << get_order(*alloc_size); 864 return; 865 } 866 867 /* Standard MTU case - optimize for multiple packets per page */ 868 buf_size = mana_get_rxbuf_size(apc, mtu); 869 870 /* Calculate how many packets can fit in a page */ 871 *frag_count = PAGE_SIZE / buf_size; 872 *alloc_size = buf_size; 873 } 874 875 int mana_pre_alloc_rxbufs(struct mana_port_context *mpc, int new_mtu, int num_queues) 876 { 877 struct device *dev; 878 struct page *page; 879 dma_addr_t da; 880 int num_rxb; 881 void *va; 882 int i; 883 884 mana_get_rxbuf_cfg(mpc, new_mtu, &mpc->rxbpre_datasize, 885 &mpc->rxbpre_alloc_size, &mpc->rxbpre_headroom, 886 &mpc->rxbpre_frag_count); 887 888 dev = mpc->ac->gdma_dev->gdma_context->dev; 889 890 num_rxb = num_queues * mpc->rx_queue_size; 891 892 WARN(mpc->rxbufs_pre, "mana rxbufs_pre exists\n"); 893 mpc->rxbufs_pre = kvmalloc_array(num_rxb, sizeof(void *), GFP_KERNEL); 894 if (!mpc->rxbufs_pre) 895 goto error; 896 897 mpc->das_pre = kvmalloc_objs(dma_addr_t, num_rxb); 898 if (!mpc->das_pre) 899 goto error; 900 901 mpc->rxbpre_total = 0; 902 903 for (i = 0; i < num_rxb; i++) { 904 page = dev_alloc_pages(get_order(mpc->rxbpre_alloc_size)); 905 if (!page) 906 goto error; 907 908 va = page_to_virt(page); 909 910 da = dma_map_single(dev, va + mpc->rxbpre_headroom, 911 mpc->rxbpre_datasize, DMA_FROM_DEVICE); 912 if (dma_mapping_error(dev, da)) { 913 put_page(page); 914 goto error; 915 } 916 917 mpc->rxbufs_pre[i] = va; 918 mpc->das_pre[i] = da; 919 mpc->rxbpre_total = i + 1; 920 } 921 922 return 0; 923 924 error: 925 netdev_err(mpc->ndev, "Failed to pre-allocate RX buffers for %d queues\n", num_queues); 926 mana_pre_dealloc_rxbufs(mpc); 927 return -ENOMEM; 928 } 929 930 static int mana_change_mtu(struct net_device *ndev, int new_mtu) 931 { 932 struct mana_port_context *mpc = netdev_priv(ndev); 933 unsigned int old_mtu = ndev->mtu; 934 int err; 935 936 /* Pre-allocate buffers to prevent failure in mana_attach later */ 937 err = mana_pre_alloc_rxbufs(mpc, new_mtu, mpc->num_queues); 938 if (err) { 939 netdev_err(ndev, "Insufficient memory for new MTU\n"); 940 return err; 941 } 942 943 err = mana_detach(ndev, false); 944 if (err) { 945 netdev_err(ndev, "mana_detach failed: %d\n", err); 946 goto out; 947 } 948 949 WRITE_ONCE(ndev->mtu, new_mtu); 950 951 err = mana_attach(ndev); 952 if (err) { 953 netdev_err(ndev, "mana_attach failed: %d\n", err); 954 WRITE_ONCE(ndev->mtu, old_mtu); 955 } 956 957 out: 958 mana_pre_dealloc_rxbufs(mpc); 959 return err; 960 } 961 962 static void mana_tx_timeout(struct net_device *netdev, unsigned int txqueue) 963 { 964 struct mana_port_context *apc = netdev_priv(netdev); 965 struct mana_context *ac = apc->ac; 966 struct gdma_context *gc = ac->gdma_dev->gdma_context; 967 968 /* Debug knob for bringup/qualification: when set, log the timeout and 969 * skip the reset so the failing state is preserved for telemetry. 970 * Disabled by default; production behaviour is unchanged. 971 */ 972 if (READ_ONCE(apc->tx_timeout_skip_reset)) { 973 netdev_warn(netdev, 974 "TX timeout on queue %u: reset skipped (tx_timeout_skip_reset enabled)\n", 975 txqueue); 976 return; 977 } 978 979 /* Already in service, hence tx queue reset is not required.*/ 980 if (test_bit(GC_IN_SERVICE, &gc->flags)) 981 return; 982 983 /* Note: If there are pending queue reset work for this port(apc), 984 * subsequent request queued up from here are ignored. This is because 985 * we are using the same work instance per port(apc). 986 */ 987 queue_work(ac->per_port_queue_reset_wq, &apc->queue_reset_work); 988 } 989 990 static int mana_shaper_set(struct net_shaper_binding *binding, 991 const struct net_shaper *shaper, 992 struct netlink_ext_ack *extack) 993 { 994 struct mana_port_context *apc = netdev_priv(binding->netdev); 995 u32 old_speed, rate; 996 int err; 997 998 if (shaper->handle.scope != NET_SHAPER_SCOPE_NETDEV) { 999 NL_SET_ERR_MSG_MOD(extack, "net shaper scope should be netdev"); 1000 return -EINVAL; 1001 } 1002 1003 if (apc->handle.id && shaper->handle.id != apc->handle.id) { 1004 NL_SET_ERR_MSG_MOD(extack, "Cannot create multiple shapers"); 1005 return -EOPNOTSUPP; 1006 } 1007 1008 if (!shaper->bw_max || (shaper->bw_max % 100000000)) { 1009 NL_SET_ERR_MSG_MOD(extack, "Please use multiples of 100Mbps for bandwidth"); 1010 return -EINVAL; 1011 } 1012 1013 rate = div_u64(shaper->bw_max, 1000); /* Convert bps to Kbps */ 1014 rate = div_u64(rate, 1000); /* Convert Kbps to Mbps */ 1015 1016 /* Get current speed */ 1017 err = mana_query_link_cfg(apc); 1018 old_speed = (err) ? SPEED_UNKNOWN : apc->speed; 1019 1020 if (!err) { 1021 err = mana_set_bw_clamp(apc, rate, TRI_STATE_TRUE); 1022 apc->speed = (err) ? old_speed : rate; 1023 apc->handle = (err) ? apc->handle : shaper->handle; 1024 } 1025 1026 return err; 1027 } 1028 1029 static int mana_shaper_del(struct net_shaper_binding *binding, 1030 const struct net_shaper_handle *handle, 1031 struct netlink_ext_ack *extack) 1032 { 1033 struct mana_port_context *apc = netdev_priv(binding->netdev); 1034 int err; 1035 1036 err = mana_set_bw_clamp(apc, 0, TRI_STATE_FALSE); 1037 1038 if (!err) { 1039 /* Reset mana port context parameters */ 1040 apc->handle.id = 0; 1041 apc->handle.scope = NET_SHAPER_SCOPE_UNSPEC; 1042 apc->speed = apc->max_speed; 1043 } 1044 1045 return err; 1046 } 1047 1048 static void mana_shaper_cap(struct net_shaper_binding *binding, 1049 enum net_shaper_scope scope, 1050 unsigned long *flags) 1051 { 1052 *flags = BIT(NET_SHAPER_A_CAPS_SUPPORT_BW_MAX) | 1053 BIT(NET_SHAPER_A_CAPS_SUPPORT_METRIC_BPS); 1054 } 1055 1056 static const struct net_shaper_ops mana_shaper_ops = { 1057 .set = mana_shaper_set, 1058 .delete = mana_shaper_del, 1059 .capabilities = mana_shaper_cap, 1060 }; 1061 1062 static const struct net_device_ops mana_devops = { 1063 .ndo_open = mana_open, 1064 .ndo_stop = mana_close, 1065 .ndo_select_queue = mana_select_queue, 1066 #if (MAX_SKB_FRAGS + 2 > MANA_MAX_TX_WQE_SGL_ENTRIES) 1067 .ndo_features_check = mana_features_check, 1068 #endif 1069 .ndo_start_xmit = mana_start_xmit, 1070 .ndo_validate_addr = eth_validate_addr, 1071 .ndo_get_stats64 = mana_get_stats64, 1072 .ndo_bpf = mana_bpf, 1073 .ndo_xdp_xmit = mana_xdp_xmit, 1074 .ndo_change_mtu = mana_change_mtu, 1075 .ndo_tx_timeout = mana_tx_timeout, 1076 .net_shaper_ops = &mana_shaper_ops, 1077 }; 1078 1079 static void mana_cleanup_port_context(struct mana_port_context *apc) 1080 { 1081 /* 1082 * make sure subsequent cleanup attempts don't end up removing already 1083 * cleaned dentry pointer 1084 */ 1085 debugfs_remove(apc->mana_port_debugfs); 1086 apc->mana_port_debugfs = NULL; 1087 kfree(apc->rxqs); 1088 apc->rxqs = NULL; 1089 } 1090 1091 static void mana_cleanup_indir_table(struct mana_port_context *apc) 1092 { 1093 apc->indir_table_sz = 0; 1094 kfree(apc->indir_table); 1095 kfree(apc->rxobj_table); 1096 } 1097 1098 static int mana_init_port_context(struct mana_port_context *apc) 1099 { 1100 apc->rxqs = kzalloc_objs(struct mana_rxq *, apc->num_queues); 1101 1102 return !apc->rxqs ? -ENOMEM : 0; 1103 } 1104 1105 static int gdma_mana_send_request(struct gdma_context *gc, void *in_buf, 1106 u32 in_len, void *out_buf, u32 out_len) 1107 { 1108 struct gdma_resp_hdr *resp = out_buf; 1109 struct gdma_req_hdr *req = in_buf; 1110 struct device *dev = gc->dev; 1111 static atomic_t activity_id; 1112 int err; 1113 1114 req->dev_id = gc->mana.dev_id; 1115 req->activity_id = atomic_inc_return(&activity_id); 1116 1117 err = mana_gd_send_request(gc, in_len, in_buf, out_len, 1118 out_buf); 1119 if (err || resp->status) { 1120 if (err == -EOPNOTSUPP) 1121 return err; 1122 1123 if (req->req.msg_type != MANA_QUERY_PHY_STAT && 1124 mana_need_log(gc, err)) 1125 dev_err(dev, "Command 0x%x failed with status: 0x%x, err: %d\n", 1126 req->req.msg_type, resp->status, err); 1127 return err ? err : -EPROTO; 1128 } 1129 1130 if (req->dev_id.as_uint32 != resp->dev_id.as_uint32 || 1131 req->activity_id != resp->activity_id) { 1132 dev_err(dev, "Unexpected mana message response: %x,%x,%x,%x\n", 1133 req->dev_id.as_uint32, resp->dev_id.as_uint32, 1134 req->activity_id, resp->activity_id); 1135 return -EPROTO; 1136 } 1137 1138 return 0; 1139 } 1140 1141 static int mana_send_request(struct mana_context *ac, void *in_buf, 1142 u32 in_len, void *out_buf, u32 out_len) 1143 { 1144 struct gdma_context *gc = ac->gdma_dev->gdma_context; 1145 1146 return gdma_mana_send_request(gc, in_buf, in_len, out_buf, out_len); 1147 } 1148 1149 static int mana_verify_resp_hdr(const struct gdma_resp_hdr *resp_hdr, 1150 const enum mana_command_code expected_code, 1151 const u32 min_size) 1152 { 1153 if (resp_hdr->response.msg_type != expected_code) 1154 return -EPROTO; 1155 1156 if (resp_hdr->response.msg_version < GDMA_MESSAGE_V1) 1157 return -EPROTO; 1158 1159 if (resp_hdr->response.msg_size < min_size) 1160 return -EPROTO; 1161 1162 return 0; 1163 } 1164 1165 static int mana_pf_register_hw_vport(struct mana_port_context *apc) 1166 { 1167 struct mana_register_hw_vport_resp resp = {}; 1168 struct mana_register_hw_vport_req req = {}; 1169 int err; 1170 1171 mana_gd_init_req_hdr(&req.hdr, MANA_REGISTER_HW_PORT, 1172 sizeof(req), sizeof(resp)); 1173 req.attached_gfid = 1; 1174 req.is_pf_default_vport = 1; 1175 req.allow_all_ether_types = 1; 1176 1177 err = mana_send_request(apc->ac, &req, sizeof(req), &resp, 1178 sizeof(resp)); 1179 if (err) { 1180 netdev_err(apc->ndev, "Failed to register hw vPort: %d\n", err); 1181 return err; 1182 } 1183 1184 err = mana_verify_resp_hdr(&resp.hdr, MANA_REGISTER_HW_PORT, 1185 sizeof(resp)); 1186 if (err || resp.hdr.status) { 1187 netdev_err(apc->ndev, "Failed to register hw vPort: %d, 0x%x\n", 1188 err, resp.hdr.status); 1189 return err ? err : -EPROTO; 1190 } 1191 1192 apc->port_handle = resp.hw_vport_handle; 1193 return 0; 1194 } 1195 1196 static void mana_pf_deregister_hw_vport(struct mana_port_context *apc) 1197 { 1198 struct mana_deregister_hw_vport_resp resp = {}; 1199 struct mana_deregister_hw_vport_req req = {}; 1200 int err; 1201 1202 mana_gd_init_req_hdr(&req.hdr, MANA_DEREGISTER_HW_PORT, 1203 sizeof(req), sizeof(resp)); 1204 req.hw_vport_handle = apc->port_handle; 1205 1206 err = mana_send_request(apc->ac, &req, sizeof(req), &resp, 1207 sizeof(resp)); 1208 if (err) { 1209 if (mana_en_need_log(apc, err)) 1210 netdev_err(apc->ndev, "Failed to unregister hw vPort: %d\n", 1211 err); 1212 1213 return; 1214 } 1215 1216 err = mana_verify_resp_hdr(&resp.hdr, MANA_DEREGISTER_HW_PORT, 1217 sizeof(resp)); 1218 if (err || resp.hdr.status) 1219 netdev_err(apc->ndev, 1220 "Failed to deregister hw vPort: %d, 0x%x\n", 1221 err, resp.hdr.status); 1222 } 1223 1224 static int mana_pf_register_filter(struct mana_port_context *apc) 1225 { 1226 struct mana_register_filter_resp resp = {}; 1227 struct mana_register_filter_req req = {}; 1228 int err; 1229 1230 mana_gd_init_req_hdr(&req.hdr, MANA_REGISTER_FILTER, 1231 sizeof(req), sizeof(resp)); 1232 req.vport = apc->port_handle; 1233 memcpy(req.mac_addr, apc->mac_addr, ETH_ALEN); 1234 1235 err = mana_send_request(apc->ac, &req, sizeof(req), &resp, 1236 sizeof(resp)); 1237 if (err) { 1238 netdev_err(apc->ndev, "Failed to register filter: %d\n", err); 1239 return err; 1240 } 1241 1242 err = mana_verify_resp_hdr(&resp.hdr, MANA_REGISTER_FILTER, 1243 sizeof(resp)); 1244 if (err || resp.hdr.status) { 1245 netdev_err(apc->ndev, "Failed to register filter: %d, 0x%x\n", 1246 err, resp.hdr.status); 1247 return err ? err : -EPROTO; 1248 } 1249 1250 apc->pf_filter_handle = resp.filter_handle; 1251 return 0; 1252 } 1253 1254 static void mana_pf_deregister_filter(struct mana_port_context *apc) 1255 { 1256 struct mana_deregister_filter_resp resp = {}; 1257 struct mana_deregister_filter_req req = {}; 1258 int err; 1259 1260 mana_gd_init_req_hdr(&req.hdr, MANA_DEREGISTER_FILTER, 1261 sizeof(req), sizeof(resp)); 1262 req.filter_handle = apc->pf_filter_handle; 1263 1264 err = mana_send_request(apc->ac, &req, sizeof(req), &resp, 1265 sizeof(resp)); 1266 if (err) { 1267 if (mana_en_need_log(apc, err)) 1268 netdev_err(apc->ndev, "Failed to unregister filter: %d\n", 1269 err); 1270 1271 return; 1272 } 1273 1274 err = mana_verify_resp_hdr(&resp.hdr, MANA_DEREGISTER_FILTER, 1275 sizeof(resp)); 1276 if (err || resp.hdr.status) 1277 netdev_err(apc->ndev, 1278 "Failed to deregister filter: %d, 0x%x\n", 1279 err, resp.hdr.status); 1280 } 1281 1282 int mana_gd_query_device_cfg(struct gdma_context *gc, u32 proto_major_ver, 1283 u32 proto_minor_ver, u32 proto_micro_ver, 1284 u16 *max_num_vports, u8 *bm_hostmode) 1285 { 1286 struct mana_query_device_cfg_resp resp = {}; 1287 struct mana_query_device_cfg_req req = {}; 1288 struct device *dev = gc->dev; 1289 int err = 0; 1290 1291 mana_gd_init_req_hdr(&req.hdr, MANA_QUERY_DEV_CONFIG, 1292 sizeof(req), sizeof(resp)); 1293 1294 req.hdr.resp.msg_version = GDMA_MESSAGE_V3; 1295 1296 req.proto_major_ver = proto_major_ver; 1297 req.proto_minor_ver = proto_minor_ver; 1298 req.proto_micro_ver = proto_micro_ver; 1299 1300 err = gdma_mana_send_request(gc, &req, sizeof(req), 1301 &resp, sizeof(resp)); 1302 if (err) { 1303 dev_err(dev, "Failed to query config: %d", err); 1304 return err; 1305 } 1306 1307 err = mana_verify_resp_hdr(&resp.hdr, MANA_QUERY_DEV_CONFIG, 1308 sizeof(resp)); 1309 if (err || resp.hdr.status) { 1310 dev_err(dev, "Invalid query result: %d, 0x%x\n", err, 1311 resp.hdr.status); 1312 if (!err) 1313 err = -EPROTO; 1314 return err; 1315 } 1316 1317 gc->cqe8_coalescing_sup = !!(resp.pf_cap_flags1 & 1318 MANA_PF_FLAG_1_CQE_8_COALESCING_SUPPORTED); 1319 1320 *max_num_vports = resp.max_num_vports; 1321 1322 if (resp.hdr.response.msg_version >= GDMA_MESSAGE_V2) { 1323 if (resp.adapter_mtu == 0) { 1324 /* 1325 * Some older PF firmware versions report an 1326 * adapter_mtu of 0. MANA hardware always supports the 1327 * standard Ethernet MTU, so fall back to ETH_FRAME_LEN. 1328 * Jumbo frames will not be available in this case. 1329 */ 1330 dev_info(dev, 1331 "PF reported adapter_mtu of 0, falling back to %u (jumbo frames disabled)\n", 1332 ETH_FRAME_LEN); 1333 gc->adapter_mtu = ETH_FRAME_LEN; 1334 } else if (resp.adapter_mtu < ETH_MIN_MTU + ETH_HLEN) { 1335 dev_err(dev, "Adapter MTU too small: %u\n", 1336 resp.adapter_mtu); 1337 return -EPROTO; 1338 } else { 1339 gc->adapter_mtu = resp.adapter_mtu; 1340 } 1341 } else { 1342 gc->adapter_mtu = ETH_FRAME_LEN; 1343 } 1344 1345 if (resp.hdr.response.msg_version >= GDMA_MESSAGE_V3) 1346 *bm_hostmode = resp.bm_hostmode; 1347 else 1348 *bm_hostmode = 0; 1349 1350 return 0; 1351 } 1352 1353 static int mana_query_vport_cfg(struct mana_port_context *apc, u32 vport_index, 1354 u32 *max_sq, u32 *max_rq, u32 *num_indir_entry) 1355 { 1356 struct mana_query_vport_cfg_resp resp = {}; 1357 struct mana_query_vport_cfg_req req = {}; 1358 int err; 1359 1360 mana_gd_init_req_hdr(&req.hdr, MANA_QUERY_VPORT_CONFIG, 1361 sizeof(req), sizeof(resp)); 1362 1363 req.vport_index = vport_index; 1364 1365 err = mana_send_request(apc->ac, &req, sizeof(req), &resp, 1366 sizeof(resp)); 1367 if (err) 1368 return err; 1369 1370 err = mana_verify_resp_hdr(&resp.hdr, MANA_QUERY_VPORT_CONFIG, 1371 sizeof(resp)); 1372 if (err) 1373 return err; 1374 1375 if (resp.hdr.status) 1376 return -EPROTO; 1377 1378 *max_sq = resp.max_num_sq; 1379 *max_rq = resp.max_num_rq; 1380 1381 if (*max_sq == 0 || *max_rq == 0) { 1382 netdev_err(apc->ndev, "Invalid max queues from vPort config\n"); 1383 return -EPROTO; 1384 } 1385 1386 if (resp.num_indirection_ent > 0 && 1387 resp.num_indirection_ent <= MANA_INDIRECT_TABLE_MAX_SIZE && 1388 is_power_of_2(resp.num_indirection_ent)) { 1389 *num_indir_entry = resp.num_indirection_ent; 1390 } else { 1391 netdev_warn(apc->ndev, 1392 "Setting indirection table size to default %d for vPort %d\n", 1393 MANA_INDIRECT_TABLE_DEF_SIZE, apc->port_idx); 1394 *num_indir_entry = MANA_INDIRECT_TABLE_DEF_SIZE; 1395 } 1396 1397 apc->port_handle = resp.vport; 1398 ether_addr_copy(apc->mac_addr, resp.mac_addr); 1399 1400 apc->vport_max_sq = *max_sq; 1401 apc->vport_max_rq = *max_rq; 1402 1403 return 0; 1404 } 1405 1406 void mana_uncfg_vport(struct mana_port_context *apc) 1407 { 1408 mutex_lock(&apc->vport_mutex); 1409 apc->vport_use_count--; 1410 WARN_ON(apc->vport_use_count < 0); 1411 mutex_unlock(&apc->vport_mutex); 1412 } 1413 EXPORT_SYMBOL_NS(mana_uncfg_vport, "NET_MANA"); 1414 1415 int mana_cfg_vport(struct mana_port_context *apc, u32 protection_dom_id, 1416 u32 doorbell_pg_id, bool check_channel_changing) 1417 { 1418 struct mana_config_vport_resp resp = {}; 1419 struct mana_config_vport_req req = {}; 1420 int err; 1421 1422 /* This function is used to program the Ethernet port in the hardware 1423 * table. It can be called from the Ethernet driver or the RDMA driver. 1424 * 1425 * For Ethernet usage, the hardware supports only one active user on a 1426 * physical port. The driver checks on the port usage before programming 1427 * the hardware when creating the RAW QP (RDMA driver) or exposing the 1428 * device to kernel NET layer (Ethernet driver). 1429 * 1430 * Because the RDMA driver doesn't know in advance which QP type the 1431 * user will create, it exposes the device with all its ports. The user 1432 * may not be able to create RAW QP on a port if this port is already 1433 * in used by the Ethernet driver from the kernel. 1434 * 1435 * This physical port limitation only applies to the RAW QP. For RC QP, 1436 * the hardware doesn't have this limitation. The user can create RC 1437 * QPs on a physical port up to the hardware limits independent of the 1438 * Ethernet usage on the same port. 1439 */ 1440 mutex_lock(&apc->vport_mutex); 1441 if (apc->vport_use_count > 0 || 1442 (check_channel_changing && apc->channel_changing)) { 1443 mutex_unlock(&apc->vport_mutex); 1444 return -EBUSY; 1445 } 1446 apc->vport_use_count++; 1447 mutex_unlock(&apc->vport_mutex); 1448 1449 mana_gd_init_req_hdr(&req.hdr, MANA_CONFIG_VPORT_TX, 1450 sizeof(req), sizeof(resp)); 1451 req.vport = apc->port_handle; 1452 req.pdid = protection_dom_id; 1453 req.doorbell_pageid = doorbell_pg_id; 1454 1455 err = mana_send_request(apc->ac, &req, sizeof(req), &resp, 1456 sizeof(resp)); 1457 if (err) { 1458 netdev_err(apc->ndev, "Failed to configure vPort: %d\n", err); 1459 goto out; 1460 } 1461 1462 err = mana_verify_resp_hdr(&resp.hdr, MANA_CONFIG_VPORT_TX, 1463 sizeof(resp)); 1464 if (err || resp.hdr.status) { 1465 netdev_err(apc->ndev, "Failed to configure vPort: %d, 0x%x\n", 1466 err, resp.hdr.status); 1467 if (!err) 1468 err = -EPROTO; 1469 1470 goto out; 1471 } 1472 1473 apc->tx_shortform_allowed = resp.short_form_allowed; 1474 apc->tx_vp_offset = resp.tx_vport_offset; 1475 1476 netdev_info(apc->ndev, "Enabled vPort %llu PD %u DB %u MAC %pM\n", 1477 apc->port_handle, protection_dom_id, doorbell_pg_id, apc->mac_addr); 1478 out: 1479 if (err) 1480 mana_uncfg_vport(apc); 1481 1482 return err; 1483 } 1484 EXPORT_SYMBOL_NS(mana_cfg_vport, "NET_MANA"); 1485 1486 static int mana_cfg_vport_steering(struct mana_port_context *apc, 1487 enum TRI_STATE rx, 1488 bool update_default_rxobj, bool update_key, 1489 bool update_tab) 1490 { 1491 struct mana_cfg_rx_steer_req_v2 *req; 1492 struct mana_cfg_rx_steer_resp resp = {}; 1493 struct net_device *ndev = apc->ndev; 1494 u32 req_buf_size; 1495 int err; 1496 1497 req_buf_size = struct_size(req, indir_tab, apc->indir_table_sz); 1498 req = kzalloc(req_buf_size, GFP_KERNEL); 1499 if (!req) 1500 return -ENOMEM; 1501 1502 mana_gd_init_req_hdr(&req->hdr, MANA_CONFIG_VPORT_RX, req_buf_size, 1503 sizeof(resp)); 1504 1505 /* Request & response versions can be different. 1506 * HW can handle newer msg versions, by skipping 1507 * new fields. 1508 */ 1509 req->hdr.req.msg_version = GDMA_MESSAGE_V5; 1510 req->hdr.resp.msg_version = GDMA_MESSAGE_V2; 1511 1512 req->vport = apc->port_handle; 1513 req->num_indir_entries = apc->indir_table_sz; 1514 req->indir_tab_offset = offsetof(struct mana_cfg_rx_steer_req_v2, 1515 indir_tab); 1516 req->rx_enable = rx; 1517 req->rss_enable = apc->rss_state; 1518 req->update_default_rxobj = update_default_rxobj; 1519 req->update_hashkey = update_key; 1520 req->update_indir_tab = update_tab; 1521 req->default_rxobj = apc->default_rxobj; 1522 1523 /* Request-msg v5 requires this field */ 1524 req->rss_hash_types = MANA_HASH_ENABLE_SUPPORTED; 1525 1526 if (rx != TRI_STATE_FALSE) { 1527 req->cqe_coalescing_enable = apc->cqe_coalescing_enable; 1528 req->cqe8_coalescing_enable = apc->cqe8_coalescing_enable; 1529 } 1530 1531 if (update_key) 1532 memcpy(&req->hashkey, apc->hashkey, MANA_HASH_KEY_SIZE); 1533 1534 if (update_tab) 1535 memcpy(req->indir_tab, apc->rxobj_table, 1536 flex_array_size(req, indir_tab, req->num_indir_entries)); 1537 1538 err = mana_send_request(apc->ac, req, req_buf_size, &resp, 1539 sizeof(resp)); 1540 if (err) { 1541 if (mana_en_need_log(apc, err)) 1542 netdev_err(ndev, "Failed to configure vPort RX: %d\n", err); 1543 1544 goto out; 1545 } 1546 1547 err = mana_verify_resp_hdr(&resp.hdr, MANA_CONFIG_VPORT_RX, 1548 sizeof(resp)); 1549 if (err) { 1550 netdev_err(ndev, "vPort RX configuration failed: %d\n", err); 1551 goto out; 1552 } 1553 1554 if (resp.hdr.status) { 1555 netdev_err(ndev, "vPort RX configuration failed: 0x%x\n", 1556 resp.hdr.status); 1557 err = -EPROTO; 1558 goto out; 1559 } 1560 1561 if (resp.hdr.response.msg_version >= GDMA_MESSAGE_V2) 1562 apc->cqe_coalescing_timeout_ns = 1563 resp.cqe_coalescing_timeout_ns; 1564 1565 netdev_info(ndev, "Configured steering vPort %llu entries %u\n", 1566 apc->port_handle, apc->indir_table_sz); 1567 1568 apc->steer_rx = rx; 1569 apc->steer_rss = apc->rss_state; 1570 apc->steer_update_tab = update_tab; 1571 apc->steer_cqe_coalescing = req->cqe_coalescing_enable; 1572 out: 1573 kfree(req); 1574 return err; 1575 } 1576 1577 int mana_query_link_cfg(struct mana_port_context *apc) 1578 { 1579 struct net_device *ndev = apc->ndev; 1580 struct mana_query_link_config_resp resp = {}; 1581 struct mana_query_link_config_req req = {}; 1582 int err; 1583 1584 netdev_assert_locked(ndev); 1585 1586 err = apc->link_cfg_error; 1587 if (err <= 0) 1588 return err; 1589 1590 mana_gd_init_req_hdr(&req.hdr, MANA_QUERY_LINK_CONFIG, 1591 sizeof(req), sizeof(resp)); 1592 1593 req.vport = apc->port_handle; 1594 req.hdr.resp.msg_version = GDMA_MESSAGE_V2; 1595 1596 err = mana_send_request(apc->ac, &req, sizeof(req), &resp, 1597 sizeof(resp)); 1598 1599 if (err) { 1600 if (err == -EOPNOTSUPP) { 1601 netdev_info_once(ndev, "MANA_QUERY_LINK_CONFIG not supported\n"); 1602 apc->link_cfg_error = err; 1603 return err; 1604 } 1605 netdev_err(ndev, "Failed to query link config: %d\n", err); 1606 return err; 1607 } 1608 1609 err = mana_verify_resp_hdr(&resp.hdr, MANA_QUERY_LINK_CONFIG, 1610 sizeof(resp)); 1611 1612 if (err || resp.hdr.status) { 1613 netdev_err(ndev, "Failed to query link config: %d, 0x%x\n", err, 1614 resp.hdr.status); 1615 if (!err) 1616 err = -EOPNOTSUPP; 1617 return err; 1618 } 1619 1620 if (resp.qos_unconfigured) 1621 return -EINVAL; 1622 1623 apc->speed = resp.link_speed_mbps; 1624 apc->max_speed = resp.qos_speed_mbps; 1625 apc->link_cfg_error = 0; 1626 return 0; 1627 } 1628 1629 int mana_set_bw_clamp(struct mana_port_context *apc, u32 speed, 1630 int enable_clamping) 1631 { 1632 struct mana_set_bw_clamp_resp resp = {}; 1633 struct mana_set_bw_clamp_req req = {}; 1634 struct net_device *ndev = apc->ndev; 1635 int err; 1636 1637 netdev_assert_locked(ndev); 1638 1639 mana_gd_init_req_hdr(&req.hdr, MANA_SET_BW_CLAMP, 1640 sizeof(req), sizeof(resp)); 1641 req.vport = apc->port_handle; 1642 req.link_speed_mbps = speed; 1643 req.enable_clamping = enable_clamping; 1644 1645 err = mana_send_request(apc->ac, &req, sizeof(req), &resp, 1646 sizeof(resp)); 1647 1648 if (err) { 1649 if (err == -EOPNOTSUPP) { 1650 netdev_info_once(ndev, "MANA_SET_BW_CLAMP not supported\n"); 1651 return err; 1652 } 1653 netdev_err(ndev, "Failed to set bandwidth clamp for speed %u, err = %d", 1654 speed, err); 1655 return err; 1656 } 1657 1658 err = mana_verify_resp_hdr(&resp.hdr, MANA_SET_BW_CLAMP, 1659 sizeof(resp)); 1660 1661 if (err || resp.hdr.status) { 1662 netdev_err(ndev, "Failed to set bandwidth clamp: %d, 0x%x\n", err, 1663 resp.hdr.status); 1664 if (!err) 1665 err = -EOPNOTSUPP; 1666 return err; 1667 } 1668 1669 if (resp.qos_unconfigured) 1670 netdev_info(ndev, "QoS is unconfigured\n"); 1671 1672 /* Invalidate the cache; next query will re-fetch from firmware. */ 1673 apc->link_cfg_error = 1; 1674 return 0; 1675 } 1676 1677 int mana_create_wq_obj(struct mana_port_context *apc, 1678 mana_handle_t vport, 1679 u32 wq_type, struct mana_obj_spec *wq_spec, 1680 struct mana_obj_spec *cq_spec, 1681 mana_handle_t *wq_obj) 1682 { 1683 struct mana_create_wqobj_resp resp = {}; 1684 struct mana_create_wqobj_req req = {}; 1685 struct net_device *ndev = apc->ndev; 1686 int err; 1687 1688 mana_gd_init_req_hdr(&req.hdr, MANA_CREATE_WQ_OBJ, 1689 sizeof(req), sizeof(resp)); 1690 1691 /* Our driver uses different message versions for request and 1692 * response in this case. 1693 * Our firmware is forward compatible with newer message versions, so 1694 * the old firmware still properly handles this message, just the new 1695 * feature fields are ignored, and queue creation will be successful. 1696 */ 1697 req.hdr.req.msg_version = GDMA_MESSAGE_V3; 1698 req.hdr.resp.msg_version = GDMA_MESSAGE_V2; 1699 req.vport = vport; 1700 req.wq_type = wq_type; 1701 req.wq_gdma_region = wq_spec->gdma_region; 1702 req.cq_gdma_region = cq_spec->gdma_region; 1703 req.wq_size = wq_spec->queue_size; 1704 req.cq_size = cq_spec->queue_size; 1705 req.cq_moderation_ctx_id = cq_spec->modr_ctx_id; 1706 req.cq_parent_qid = cq_spec->attached_eq; 1707 req.req_cq_moderation = cq_spec->req_cq_moderation; 1708 req.cq_moderation_comp = cq_spec->cq_moderation_comp; 1709 req.cq_moderation_usec = cq_spec->cq_moderation_usec; 1710 1711 err = mana_send_request(apc->ac, &req, sizeof(req), &resp, 1712 sizeof(resp)); 1713 if (err) { 1714 netdev_err(ndev, "Failed to create WQ object: %d\n", err); 1715 goto out; 1716 } 1717 1718 err = mana_verify_resp_hdr(&resp.hdr, MANA_CREATE_WQ_OBJ, 1719 sizeof(resp)); 1720 if (err || resp.hdr.status) { 1721 netdev_err(ndev, "Failed to create WQ object: %d, 0x%x\n", err, 1722 resp.hdr.status); 1723 if (!err) 1724 err = -EPROTO; 1725 goto out; 1726 } 1727 1728 if (resp.wq_obj == INVALID_MANA_HANDLE) { 1729 netdev_err(ndev, "Got an invalid WQ object handle\n"); 1730 err = -EPROTO; 1731 goto out; 1732 } 1733 1734 *wq_obj = resp.wq_obj; 1735 wq_spec->queue_index = resp.wq_id; 1736 cq_spec->queue_index = resp.cq_id; 1737 1738 return 0; 1739 out: 1740 return err; 1741 } 1742 EXPORT_SYMBOL_NS(mana_create_wq_obj, "NET_MANA"); 1743 1744 void mana_destroy_wq_obj(struct mana_port_context *apc, u32 wq_type, 1745 mana_handle_t wq_obj) 1746 { 1747 struct mana_destroy_wqobj_resp resp = {}; 1748 struct mana_destroy_wqobj_req req = {}; 1749 struct net_device *ndev = apc->ndev; 1750 int err; 1751 1752 mana_gd_init_req_hdr(&req.hdr, MANA_DESTROY_WQ_OBJ, 1753 sizeof(req), sizeof(resp)); 1754 req.wq_type = wq_type; 1755 req.wq_obj_handle = wq_obj; 1756 1757 err = mana_send_request(apc->ac, &req, sizeof(req), &resp, 1758 sizeof(resp)); 1759 if (err) { 1760 if (mana_en_need_log(apc, err)) 1761 netdev_err(ndev, "Failed to destroy WQ object: %d\n", err); 1762 1763 return; 1764 } 1765 1766 err = mana_verify_resp_hdr(&resp.hdr, MANA_DESTROY_WQ_OBJ, 1767 sizeof(resp)); 1768 if (err || resp.hdr.status) 1769 netdev_err(ndev, "Failed to destroy WQ object: %d, 0x%x\n", err, 1770 resp.hdr.status); 1771 } 1772 EXPORT_SYMBOL_NS(mana_destroy_wq_obj, "NET_MANA"); 1773 1774 void mana_destroy_eq(struct mana_port_context *apc) 1775 { 1776 struct mana_context *ac = apc->ac; 1777 struct gdma_context *gc = ac->gdma_dev->gdma_context; 1778 struct gdma_queue *eq; 1779 unsigned int msi; 1780 int i; 1781 1782 if (!apc->eqs) 1783 return; 1784 1785 debugfs_remove_recursive(apc->mana_eqs_debugfs); 1786 apc->mana_eqs_debugfs = NULL; 1787 1788 for (i = 0; i < apc->num_queues; i++) { 1789 eq = apc->eqs[i].eq; 1790 if (!eq) 1791 continue; 1792 1793 msi = eq->eq.msix_index; 1794 mana_gd_destroy_queue(gc, eq); 1795 mana_gd_put_gic(gc, !gc->msi_sharing, msi); 1796 } 1797 1798 kfree(apc->eqs); 1799 apc->eqs = NULL; 1800 } 1801 EXPORT_SYMBOL_NS(mana_destroy_eq, "NET_MANA"); 1802 1803 static void mana_create_eq_debugfs(struct mana_port_context *apc, int i) 1804 { 1805 struct mana_eq eq = apc->eqs[i]; 1806 char eqnum[32]; 1807 1808 sprintf(eqnum, "eq%d", i); 1809 eq.mana_eq_debugfs = debugfs_create_dir(eqnum, apc->mana_eqs_debugfs); 1810 debugfs_create_u32("head", 0400, eq.mana_eq_debugfs, &eq.eq->head); 1811 debugfs_create_u32("tail", 0400, eq.mana_eq_debugfs, &eq.eq->tail); 1812 debugfs_create_u32("irq", 0400, eq.mana_eq_debugfs, &eq.eq->eq.irq); 1813 debugfs_create_file("eq_dump", 0400, eq.mana_eq_debugfs, eq.eq, &mana_dbg_q_fops); 1814 } 1815 1816 int mana_create_eq(struct mana_port_context *apc) 1817 { 1818 struct gdma_dev *gd = apc->ac->gdma_dev; 1819 struct gdma_context *gc = gd->gdma_context; 1820 struct gdma_queue_spec spec = {}; 1821 struct gdma_irq_context *gic; 1822 int err; 1823 int msi; 1824 int i; 1825 1826 if (WARN_ON(apc->eqs)) 1827 return -EEXIST; 1828 apc->eqs = kzalloc_objs(struct mana_eq, apc->num_queues); 1829 if (!apc->eqs) 1830 return -ENOMEM; 1831 1832 spec.type = GDMA_EQ; 1833 spec.monitor_avl_buf = false; 1834 spec.queue_size = EQ_SIZE; 1835 spec.eq.callback = NULL; 1836 spec.eq.context = apc->eqs; 1837 spec.eq.log2_throttle_limit = LOG2_EQ_THROTTLE; 1838 1839 apc->mana_eqs_debugfs = 1840 debugfs_create_dir("EQs", apc->mana_port_debugfs); 1841 1842 for (i = 0; i < apc->num_queues; i++) { 1843 msi = (i + 1) % gc->num_msix_usable; 1844 1845 gic = mana_gd_get_gic(gc, !gc->msi_sharing, &msi); 1846 if (IS_ERR(gic)) { 1847 err = PTR_ERR(gic); 1848 goto out; 1849 } 1850 spec.eq.msix_index = msi; 1851 1852 err = mana_gd_create_mana_eq(gd, &spec, &apc->eqs[i].eq); 1853 if (err) { 1854 dev_err(gc->dev, "Failed to create EQ %d : %d\n", i, err); 1855 mana_gd_put_gic(gc, !gc->msi_sharing, msi); 1856 goto out; 1857 } 1858 apc->eqs[i].eq->eq.irq = gic->irq; 1859 mana_create_eq_debugfs(apc, i); 1860 } 1861 1862 return 0; 1863 out: 1864 mana_destroy_eq(apc); 1865 return err; 1866 } 1867 EXPORT_SYMBOL_NS(mana_create_eq, "NET_MANA"); 1868 1869 static int mana_fence_rq(struct mana_port_context *apc, struct mana_rxq *rxq) 1870 { 1871 struct mana_fence_rq_resp resp = {}; 1872 struct mana_fence_rq_req req = {}; 1873 int err; 1874 1875 init_completion(&rxq->fence_event); 1876 1877 mana_gd_init_req_hdr(&req.hdr, MANA_FENCE_RQ, 1878 sizeof(req), sizeof(resp)); 1879 req.wq_obj_handle = rxq->rxobj; 1880 1881 err = mana_send_request(apc->ac, &req, sizeof(req), &resp, 1882 sizeof(resp)); 1883 if (err) { 1884 netdev_err(apc->ndev, "Failed to fence RQ %u: %d\n", 1885 rxq->rxq_idx, err); 1886 return err; 1887 } 1888 1889 err = mana_verify_resp_hdr(&resp.hdr, MANA_FENCE_RQ, sizeof(resp)); 1890 if (err || resp.hdr.status) { 1891 netdev_err(apc->ndev, "Failed to fence RQ %u: %d, 0x%x\n", 1892 rxq->rxq_idx, err, resp.hdr.status); 1893 if (!err) 1894 err = -EPROTO; 1895 1896 return err; 1897 } 1898 1899 if (wait_for_completion_timeout(&rxq->fence_event, 10 * HZ) == 0) { 1900 netdev_err(apc->ndev, "Failed to fence RQ %u: timed out\n", 1901 rxq->rxq_idx); 1902 return -ETIMEDOUT; 1903 } 1904 1905 return 0; 1906 } 1907 1908 static void mana_fence_rqs(struct mana_port_context *apc) 1909 { 1910 unsigned int rxq_idx; 1911 struct mana_rxq *rxq; 1912 int err; 1913 1914 if (!apc->rxqs) 1915 return; 1916 1917 for (rxq_idx = 0; rxq_idx < apc->num_queues; rxq_idx++) { 1918 rxq = apc->rxqs[rxq_idx]; 1919 err = mana_fence_rq(apc, rxq); 1920 1921 /* In case of any error, use sleep instead. */ 1922 if (err) 1923 msleep(100); 1924 } 1925 } 1926 1927 static int mana_move_wq_tail(struct gdma_queue *wq, u32 num_units) 1928 { 1929 u32 used_space_old; 1930 u32 used_space_new; 1931 1932 used_space_old = wq->head - wq->tail; 1933 used_space_new = wq->head - (wq->tail + num_units); 1934 1935 if (WARN_ON_ONCE(used_space_new > used_space_old)) 1936 return -ERANGE; 1937 1938 wq->tail += num_units; 1939 return 0; 1940 } 1941 1942 void mana_unmap_skb(struct sk_buff *skb, struct mana_port_context *apc) 1943 { 1944 struct mana_skb_head *ash = (struct mana_skb_head *)skb->head; 1945 struct gdma_context *gc = apc->ac->gdma_dev->gdma_context; 1946 struct device *dev = gc->dev; 1947 int hsg, i; 1948 1949 /* Number of SGEs of linear part */ 1950 hsg = (skb_is_gso(skb) && skb_headlen(skb) > ash->size[0]) ? 2 : 1; 1951 1952 for (i = 0; i < hsg; i++) 1953 dma_unmap_single(dev, ash->dma_handle[i], ash->size[i], 1954 DMA_TO_DEVICE); 1955 1956 for (i = hsg; i < skb_shinfo(skb)->nr_frags + hsg; i++) 1957 dma_unmap_page(dev, ash->dma_handle[i], ash->size[i], 1958 DMA_TO_DEVICE); 1959 } 1960 1961 static void mana_poll_tx_cq(struct mana_cq *cq) 1962 { 1963 struct gdma_comp *completions = cq->gdma_comp_buf; 1964 unsigned int pkt_transmitted = 0; 1965 unsigned int wqe_unit_cnt = 0; 1966 unsigned int bql_bytes = 0; 1967 unsigned int bql_pkts = 0; 1968 unsigned int tx_bytes = 0; 1969 struct mana_txq *txq = cq->txq; 1970 struct mana_port_context *apc; 1971 struct netdev_queue *net_txq; 1972 struct gdma_queue *gdma_wq; 1973 unsigned int avail_space; 1974 struct net_device *ndev; 1975 struct mana_skb_cb *cb; 1976 struct sk_buff *skb; 1977 bool txq_stopped; 1978 int comp_read; 1979 int i; 1980 1981 ndev = txq->ndev; 1982 apc = netdev_priv(ndev); 1983 1984 /* Limit CQEs polled to 4 wraparounds of the CQ to ensure the 1985 * doorbell can be rung in time for the hardware's requirement 1986 * of at least one doorbell ring every 8 wraparounds. 1987 */ 1988 comp_read = mana_gd_poll_cq(cq->gdma_cq, completions, 1989 min((cq->gdma_cq->queue_size / 1990 COMP_ENTRY_SIZE) * 4, 1991 CQE_POLLING_BUFFER)); 1992 1993 if (comp_read < 1) 1994 return; 1995 1996 for (i = 0; i < comp_read; i++) { 1997 struct mana_tx_comp_oob *cqe_oob; 1998 1999 if (WARN_ON_ONCE(!completions[i].is_sq)) 2000 return; 2001 2002 cqe_oob = (struct mana_tx_comp_oob *)completions[i].cqe_data; 2003 if (WARN_ON_ONCE(cqe_oob->cqe_hdr.client_type != 2004 MANA_CQE_COMPLETION)) 2005 return; 2006 2007 switch (cqe_oob->cqe_hdr.cqe_type) { 2008 case CQE_TX_OKAY: 2009 break; 2010 2011 case CQE_TX_SA_DROP: 2012 case CQE_TX_MTU_DROP: 2013 case CQE_TX_INVALID_OOB: 2014 case CQE_TX_INVALID_ETH_TYPE: 2015 case CQE_TX_HDR_PROCESSING_ERROR: 2016 case CQE_TX_VF_DISABLED: 2017 case CQE_TX_VPORT_IDX_OUT_OF_RANGE: 2018 case CQE_TX_VPORT_DISABLED: 2019 case CQE_TX_VLAN_TAGGING_VIOLATION: 2020 if (net_ratelimit()) 2021 netdev_err(ndev, "TX: CQE error %d\n", 2022 cqe_oob->cqe_hdr.cqe_type); 2023 2024 apc->eth_stats.tx_cqe_err++; 2025 break; 2026 2027 default: 2028 /* If the CQE type is unknown, log an error, 2029 * and still free the SKB, update tail, etc. 2030 */ 2031 if (net_ratelimit()) 2032 netdev_err(ndev, "TX: unknown CQE type %d\n", 2033 cqe_oob->cqe_hdr.cqe_type); 2034 2035 apc->eth_stats.tx_cqe_unknown_type++; 2036 break; 2037 } 2038 2039 if (WARN_ON_ONCE(txq->gdma_txq_id != completions[i].wq_num)) 2040 return; 2041 2042 skb = skb_dequeue(&txq->pending_skbs); 2043 if (WARN_ON_ONCE(!skb)) 2044 return; 2045 2046 cb = (struct mana_skb_cb *)skb->cb; 2047 wqe_unit_cnt += cb->wqe_info.wqe_size_in_bu; 2048 2049 mana_unmap_skb(skb, apc); 2050 2051 tx_bytes += skb->len; 2052 if (!cb->xdp) { 2053 bql_pkts++; 2054 bql_bytes += skb->len; 2055 } 2056 2057 napi_consume_skb(skb, cq->budget); 2058 2059 pkt_transmitted++; 2060 } 2061 2062 if (WARN_ON_ONCE(wqe_unit_cnt == 0)) 2063 return; 2064 2065 mana_move_wq_tail(txq->gdma_sq, wqe_unit_cnt); 2066 2067 gdma_wq = txq->gdma_sq; 2068 avail_space = mana_gd_wq_avail_space(gdma_wq); 2069 2070 net_txq = txq->net_txq; 2071 netdev_tx_completed_queue(net_txq, bql_pkts, bql_bytes); 2072 2073 /* Ensure tail updated before checking q stop */ 2074 smp_mb(); 2075 2076 txq_stopped = netif_tx_queue_stopped(net_txq); 2077 2078 /* Ensure checking txq_stopped before apc->port_is_up. */ 2079 smp_rmb(); 2080 2081 if (txq_stopped && apc->port_is_up && avail_space >= MAX_TX_WQE_SIZE) { 2082 netif_tx_wake_queue(net_txq); 2083 apc->eth_stats.wake_queue++; 2084 } 2085 2086 if (atomic_sub_return(pkt_transmitted, &txq->pending_sends) < 0) 2087 WARN_ON_ONCE(1); 2088 2089 /* Feed DIM with the completion rate observed here, in NAPI context. */ 2090 cq->tx_dim_pkts += pkt_transmitted; 2091 cq->tx_dim_bytes += tx_bytes; 2092 2093 cq->work_done = pkt_transmitted; 2094 } 2095 2096 static void mana_post_pkt_rxq(struct mana_rxq *rxq) 2097 { 2098 struct mana_recv_buf_oob *recv_buf_oob; 2099 u32 curr_index; 2100 int err; 2101 2102 curr_index = rxq->buf_index++; 2103 if (rxq->buf_index == rxq->num_rx_buf) 2104 rxq->buf_index = 0; 2105 2106 recv_buf_oob = &rxq->rx_oobs[curr_index]; 2107 2108 err = mana_gd_post_work_request(rxq->gdma_rq, &recv_buf_oob->wqe_req, 2109 &recv_buf_oob->wqe_inf); 2110 if (WARN_ON_ONCE(err)) 2111 return; 2112 2113 WARN_ON_ONCE(recv_buf_oob->wqe_inf.wqe_size_in_bu != 1); 2114 } 2115 2116 static struct sk_buff *mana_build_skb(struct mana_rxq *rxq, void *buf_va, 2117 uint pkt_len, struct xdp_buff *xdp) 2118 { 2119 struct sk_buff *skb = napi_build_skb(buf_va, rxq->alloc_size); 2120 2121 if (!skb) 2122 return NULL; 2123 2124 if (xdp->data_hard_start) { 2125 u32 metasize = xdp->data - xdp->data_meta; 2126 2127 skb_reserve(skb, xdp->data - xdp->data_hard_start); 2128 skb_put(skb, xdp->data_end - xdp->data); 2129 if (metasize) 2130 skb_metadata_set(skb, metasize); 2131 return skb; 2132 } 2133 2134 skb_reserve(skb, rxq->headroom); 2135 skb_put(skb, pkt_len); 2136 2137 return skb; 2138 } 2139 2140 static void mana_rx_skb(void *buf_va, bool from_pool, 2141 struct mana_rxcomp_oob *cqe, struct mana_rxq *rxq, 2142 u32 pkt_len, u32 pkt_hash) 2143 { 2144 struct mana_stats_rx *rx_stats = &rxq->stats; 2145 struct net_device *ndev = rxq->ndev; 2146 u16 rxq_idx = rxq->rxq_idx; 2147 struct napi_struct *napi; 2148 struct xdp_buff xdp = {}; 2149 struct sk_buff *skb; 2150 u32 act; 2151 2152 rxq->rx_cq.work_done++; 2153 napi = &rxq->rx_cq.napi; 2154 2155 if (!buf_va) { 2156 ++ndev->stats.rx_dropped; 2157 return; 2158 } 2159 2160 act = mana_run_xdp(ndev, rxq, &xdp, buf_va, pkt_len); 2161 2162 if (act == XDP_REDIRECT && !rxq->xdp_rc) 2163 return; 2164 2165 if (act != XDP_PASS && act != XDP_TX) 2166 goto drop_xdp; 2167 2168 skb = mana_build_skb(rxq, buf_va, pkt_len, &xdp); 2169 2170 if (!skb) 2171 goto drop; 2172 2173 if (from_pool) 2174 skb_mark_for_recycle(skb); 2175 2176 skb->dev = napi->dev; 2177 2178 skb->protocol = eth_type_trans(skb, ndev); 2179 skb_checksum_none_assert(skb); 2180 skb_record_rx_queue(skb, rxq_idx); 2181 2182 if ((ndev->features & NETIF_F_RXCSUM) && cqe->rx_iphdr_csum_succeed) { 2183 if (cqe->rx_tcp_csum_succeed || cqe->rx_udp_csum_succeed) 2184 skb->ip_summed = CHECKSUM_UNNECESSARY; 2185 } 2186 2187 if (cqe->rx_hashtype != 0 && (ndev->features & NETIF_F_RXHASH)) { 2188 if (cqe->rx_hashtype & MANA_HASH_L4) 2189 skb_set_hash(skb, pkt_hash, PKT_HASH_TYPE_L4); 2190 else 2191 skb_set_hash(skb, pkt_hash, PKT_HASH_TYPE_L3); 2192 } 2193 2194 if (cqe->rx_vlantag_present) { 2195 u16 vlan_tci = cqe->rx_vlan_id; 2196 2197 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vlan_tci); 2198 } 2199 2200 u64_stats_update_begin(&rx_stats->syncp); 2201 rx_stats->packets++; 2202 rx_stats->bytes += pkt_len; 2203 2204 if (act == XDP_TX) 2205 rx_stats->xdp_tx++; 2206 u64_stats_update_end(&rx_stats->syncp); 2207 2208 if (act == XDP_TX) { 2209 skb_set_queue_mapping(skb, rxq_idx); 2210 mana_xdp_tx(skb, ndev); 2211 return; 2212 } 2213 2214 napi_gro_receive(napi, skb); 2215 2216 return; 2217 2218 drop_xdp: 2219 u64_stats_update_begin(&rx_stats->syncp); 2220 rx_stats->xdp_drop++; 2221 u64_stats_update_end(&rx_stats->syncp); 2222 2223 drop: 2224 if (from_pool) { 2225 if (rxq->frag_count == 1) 2226 page_pool_recycle_direct(rxq->page_pool, 2227 virt_to_head_page(buf_va)); 2228 else 2229 page_pool_free_va(rxq->page_pool, buf_va, true); 2230 } else { 2231 WARN_ON_ONCE(rxq->xdp_save_va); 2232 /* Save for reuse */ 2233 rxq->xdp_save_va = buf_va; 2234 } 2235 2236 ++ndev->stats.rx_dropped; 2237 2238 return; 2239 } 2240 2241 static void *mana_get_rxfrag(struct mana_rxq *rxq, struct device *dev, 2242 dma_addr_t *da, bool *from_pool, 2243 struct page **pp_page, u32 *dma_sync_offset) 2244 { 2245 struct page *page; 2246 u32 offset; 2247 void *va; 2248 2249 *from_pool = false; 2250 *pp_page = NULL; 2251 *dma_sync_offset = 0; 2252 2253 /* Don't use fragments for jumbo frames or XDP where it's 1 fragment 2254 * per page. 2255 */ 2256 if (rxq->frag_count == 1) { 2257 /* Reuse XDP dropped page if available */ 2258 if (rxq->xdp_save_va) { 2259 va = rxq->xdp_save_va; 2260 page = virt_to_head_page(va); 2261 rxq->xdp_save_va = NULL; 2262 } else { 2263 page = page_pool_dev_alloc_pages(rxq->page_pool); 2264 if (!page) 2265 return NULL; 2266 2267 *from_pool = true; 2268 va = page_to_virt(page); 2269 } 2270 2271 *da = dma_map_single(dev, va + rxq->headroom, rxq->datasize, 2272 DMA_FROM_DEVICE); 2273 if (dma_mapping_error(dev, *da)) { 2274 mana_put_rx_page(rxq, page, *from_pool); 2275 return NULL; 2276 } 2277 2278 return va; 2279 } 2280 2281 page = page_pool_dev_alloc_frag(rxq->page_pool, &offset, 2282 rxq->alloc_size); 2283 if (!page) 2284 return NULL; 2285 2286 va = page_to_virt(page) + offset; 2287 *da = page_pool_get_dma_addr(page) + offset + rxq->headroom; 2288 *from_pool = true; 2289 *pp_page = page; 2290 *dma_sync_offset = offset + rxq->headroom; 2291 2292 return va; 2293 } 2294 2295 /* Allocate frag for rx buffer, and save the old buf */ 2296 static void mana_refill_rx_oob(struct device *dev, struct mana_rxq *rxq, 2297 struct mana_recv_buf_oob *rxoob, u32 pktlen, 2298 void **old_buf, bool *old_fp) 2299 { 2300 struct page *pp_page; 2301 u32 dma_sync_offset; 2302 bool from_pool; 2303 dma_addr_t da; 2304 void *va; 2305 2306 va = mana_get_rxfrag(rxq, dev, &da, &from_pool, &pp_page, 2307 &dma_sync_offset); 2308 if (!va) 2309 return; 2310 if (!rxoob->from_pool || rxq->frag_count == 1) { 2311 dma_unmap_single(dev, rxoob->sgl[0].address, rxq->datasize, 2312 DMA_FROM_DEVICE); 2313 } else { 2314 /* The page pool maps the whole page and only syncs for device 2315 * automatically (PP_FLAG_DMA_SYNC_DEV). Sync the received bytes 2316 * for the CPU before they are read: this is required if DMA 2317 * is incoherent or bounce buffers are used. 2318 */ 2319 page_pool_dma_sync_for_cpu(rxq->page_pool, rxoob->pp_page, 2320 rxoob->dma_sync_offset, pktlen); 2321 } 2322 *old_buf = rxoob->buf_va; 2323 *old_fp = rxoob->from_pool; 2324 2325 rxoob->buf_va = va; 2326 rxoob->sgl[0].address = da; 2327 rxoob->from_pool = from_pool; 2328 rxoob->pp_page = pp_page; 2329 rxoob->dma_sync_offset = dma_sync_offset; 2330 } 2331 2332 static void mana_process_one_rx_pkt(struct device *dev, struct mana_rxq *rxq, 2333 struct mana_rxcomp_oob *oob, 2334 u32 pktlen, u32 pkt_hash) 2335 { 2336 struct mana_recv_buf_oob *rxbuf_oob; 2337 struct net_device *ndev = rxq->ndev; 2338 void *old_buf = NULL; 2339 bool old_fp; 2340 2341 rxbuf_oob = &rxq->rx_oobs[rxq->buf_index]; 2342 WARN_ON_ONCE(rxbuf_oob->wqe_inf.wqe_size_in_bu != 1); 2343 2344 if (unlikely(pktlen > rxq->datasize)) { 2345 /* Increase it even if mana_rx_skb() isn't called. */ 2346 rxq->rx_cq.work_done++; 2347 2348 ++ndev->stats.rx_dropped; 2349 netdev_warn_once(ndev, 2350 "Dropped oversized RX packet: len=%u, datasize=%u\n", 2351 pktlen, rxq->datasize); 2352 2353 /* Reuse the RX buffer since rxbuf_oob is unchanged. */ 2354 } else { 2355 mana_refill_rx_oob(dev, rxq, rxbuf_oob, pktlen, 2356 &old_buf, &old_fp); 2357 2358 /* Unsuccessful refill will have old_buf == NULL. 2359 * In this case, mana_rx_skb() will drop the packet. 2360 */ 2361 mana_rx_skb(old_buf, old_fp, oob, rxq, pktlen, pkt_hash); 2362 } 2363 2364 mana_move_wq_tail(rxq->gdma_rq, rxbuf_oob->wqe_inf.wqe_size_in_bu); 2365 2366 mana_post_pkt_rxq(rxq); 2367 } 2368 2369 static void mana_process_rx_cqe(struct mana_rxq *rxq, struct mana_cq *cq, 2370 struct gdma_comp *cqe) 2371 { 2372 struct mana_rxcomp_oob *oob = (struct mana_rxcomp_oob *)cqe->cqe_data; 2373 struct gdma_context *gc = rxq->gdma_rq->gdma_dev->gdma_context; 2374 struct net_device *ndev = rxq->ndev; 2375 struct mana_recv_buf_oob *rxbuf_oob; 2376 struct mana_port_context *apc; 2377 struct device *dev = gc->dev; 2378 bool coalesced_8 = false; 2379 bool coalesced = false; 2380 u32 pktlen; 2381 int pkt_i; 2382 int i; 2383 2384 apc = netdev_priv(ndev); 2385 2386 switch (oob->cqe_hdr.cqe_type) { 2387 case CQE_RX_OKAY: 2388 break; 2389 2390 case CQE_RX_TRUNCATED: 2391 ++ndev->stats.rx_dropped; 2392 rxbuf_oob = &rxq->rx_oobs[rxq->buf_index]; 2393 netdev_warn_once(ndev, "Dropped a truncated packet\n"); 2394 2395 mana_move_wq_tail(rxq->gdma_rq, 2396 rxbuf_oob->wqe_inf.wqe_size_in_bu); 2397 mana_post_pkt_rxq(rxq); 2398 return; 2399 2400 case CQE_RX_COALESCED_4: 2401 coalesced = true; 2402 break; 2403 2404 case CQE_RX_COALESCED_8: 2405 coalesced = true; 2406 coalesced_8 = true; 2407 break; 2408 2409 case CQE_RX_OBJECT_FENCE: 2410 complete(&rxq->fence_event); 2411 return; 2412 2413 default: 2414 netdev_err(ndev, "Unknown RX CQE type = %d\n", 2415 oob->cqe_hdr.cqe_type); 2416 apc->eth_stats.rx_cqe_unknown_type++; 2417 return; 2418 } 2419 2420 pkt_i = 0; 2421 for (i = 0; i < MANA_RXCOMP_OOB_NUM_PPI; i++) { 2422 u32 pkt_hash; 2423 2424 if (coalesced_8) { 2425 /* 8-pkt mode: 2 packets per PPI entry */ 2426 pktlen = oob->ppi[i].pkt_len0; 2427 pkt_hash = oob->ppi[i].pkt_hash0; 2428 } else { 2429 pktlen = oob->ppi[i].pkt_len; 2430 pkt_hash = oob->ppi[i].pkt_hash; 2431 } 2432 if (pktlen == 0) 2433 break; 2434 2435 mana_process_one_rx_pkt(dev, rxq, oob, pktlen, pkt_hash); 2436 pkt_i++; 2437 2438 if (!coalesced) 2439 break; 2440 2441 /* Process 2nd packet from the same PPI in 8-pkt mode */ 2442 if (coalesced_8) { 2443 pktlen = oob->ppi[i].pkt_len1; 2444 pkt_hash = oob->ppi[i].pkt_hash1; 2445 if (pktlen == 0) 2446 break; 2447 2448 mana_process_one_rx_pkt(dev, rxq, oob, pktlen, 2449 pkt_hash); 2450 pkt_i++; 2451 } 2452 } 2453 2454 /* Collect coalesced CQE count based on packets processed. 2455 * Coalesced CQEs have at least 2 packets, so index is pkt_i - 2. 2456 */ 2457 if (pkt_i > 1) { 2458 u64_stats_update_begin(&rxq->stats.syncp); 2459 rxq->stats.coalesced_cqe[pkt_i - 2]++; 2460 u64_stats_update_end(&rxq->stats.syncp); 2461 } else if (!pkt_i && !pktlen) { 2462 u64_stats_update_begin(&rxq->stats.syncp); 2463 rxq->stats.pkt_len0_err++; 2464 u64_stats_update_end(&rxq->stats.syncp); 2465 netdev_err_once(ndev, 2466 "RX pkt len=0, rq=%u, cq=%u, rxobj=0x%llx\n", 2467 rxq->gdma_id, cq->gdma_id, rxq->rxobj); 2468 } 2469 } 2470 2471 static void mana_poll_rx_cq(struct mana_cq *cq) 2472 { 2473 struct gdma_comp *comp = cq->gdma_comp_buf; 2474 struct mana_rxq *rxq = cq->rxq; 2475 int comp_read, i; 2476 2477 /* Limit CQEs polled to 4 wraparounds of the CQ to ensure the 2478 * doorbell can be rung in time for the hardware's requirement 2479 * of at least one doorbell ring every 8 wraparounds. 2480 */ 2481 comp_read = mana_gd_poll_cq(cq->gdma_cq, comp, 2482 min((cq->gdma_cq->queue_size / 2483 COMP_ENTRY_SIZE) * 4, 2484 CQE_POLLING_BUFFER)); 2485 WARN_ON_ONCE(comp_read > CQE_POLLING_BUFFER); 2486 2487 rxq->xdp_flush = false; 2488 2489 for (i = 0; i < comp_read; i++) { 2490 if (WARN_ON_ONCE(comp[i].is_sq)) 2491 return; 2492 2493 /* verify recv cqe references the right rxq */ 2494 if (WARN_ON_ONCE(comp[i].wq_num != cq->rxq->gdma_id)) 2495 return; 2496 2497 mana_process_rx_cqe(rxq, cq, &comp[i]); 2498 } 2499 2500 if (comp_read > 0) { 2501 struct gdma_context *gc = rxq->gdma_rq->gdma_dev->gdma_context; 2502 2503 mana_gd_wq_ring_doorbell(gc, rxq->gdma_rq); 2504 } 2505 2506 if (rxq->xdp_flush) 2507 xdp_do_flush(); 2508 } 2509 2510 static void mana_rx_dim_work(struct work_struct *work) 2511 { 2512 struct dim *dim = container_of(work, struct dim, work); 2513 struct dim_cq_moder cur_moder; 2514 struct mana_cq *cq; 2515 2516 cur_moder = net_dim_get_rx_moderation(dim->mode, dim->profile_ix); 2517 cq = container_of(dim, struct mana_cq, dim); 2518 2519 cur_moder.usec = min_t(u16, cur_moder.usec, MANA_INTR_MODR_USEC_MAX); 2520 cur_moder.pkts = min_t(u16, cur_moder.pkts, MANA_INTR_MODR_COMP_MAX); 2521 2522 mana_gd_ring_dim(cq->gdma_cq, cur_moder.usec, true, 2523 cur_moder.pkts, true); 2524 2525 dim->state = DIM_START_MEASURE; 2526 } 2527 2528 static void mana_tx_dim_work(struct work_struct *work) 2529 { 2530 struct dim *dim = container_of(work, struct dim, work); 2531 struct dim_cq_moder cur_moder; 2532 struct mana_cq *cq; 2533 2534 cur_moder = net_dim_get_tx_moderation(dim->mode, dim->profile_ix); 2535 cq = container_of(dim, struct mana_cq, dim); 2536 2537 cur_moder.usec = min_t(u16, cur_moder.usec, MANA_INTR_MODR_USEC_MAX); 2538 cur_moder.pkts = min_t(u16, cur_moder.pkts, MANA_INTR_MODR_COMP_MAX); 2539 2540 mana_gd_ring_dim(cq->gdma_cq, cur_moder.usec, true, 2541 cur_moder.pkts, true); 2542 2543 dim->state = DIM_START_MEASURE; 2544 } 2545 2546 /* The caller must update apc->rx/tx_dim_enabled before disabling and 2547 * after enabling. And synchronize_net() before draining the DIM work, 2548 * so that NAPI cannot observe a stale flag. 2549 */ 2550 void mana_dim_change(struct mana_cq *cq, bool enable) 2551 { 2552 bool is_rx = cq->type == MANA_CQ_TYPE_RX; 2553 struct mana_port_context *apc; 2554 work_func_t work_func; 2555 u32 usec, comp; 2556 2557 if (is_rx) { 2558 apc = netdev_priv(cq->rxq->ndev); 2559 usec = apc->intr_modr_rx_usec; 2560 comp = apc->intr_modr_rx_comp; 2561 work_func = mana_rx_dim_work; 2562 } else { 2563 apc = netdev_priv(cq->txq->ndev); 2564 usec = apc->intr_modr_tx_usec; 2565 comp = apc->intr_modr_tx_comp; 2566 work_func = mana_tx_dim_work; 2567 } 2568 2569 /* On enable, zero the DIM state so net_dim() starts measuring from 2570 * scratch. 2571 * On disable, drain any pending DIM work and restore the static 2572 * moderation values. 2573 */ 2574 if (enable) { 2575 memset(&cq->dim, 0, sizeof(cq->dim)); 2576 cq->dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE; 2577 INIT_WORK(&cq->dim.work, work_func); 2578 } else { 2579 cancel_work_sync(&cq->dim.work); 2580 mana_gd_ring_dim(cq->gdma_cq, usec, true, comp, true); 2581 } 2582 } 2583 2584 static void mana_update_rx_dim(struct mana_cq *cq) 2585 { 2586 struct mana_port_context *apc = netdev_priv(cq->rxq->ndev); 2587 struct dim_sample dim_sample = {}; 2588 struct mana_rxq *rxq = cq->rxq; 2589 2590 /* Pairs with smp_store_release() in mana_set_coalesce(): observing the 2591 * enable flag set guarantees the DIM (re)initialization is visible. 2592 */ 2593 if (!smp_load_acquire(&apc->rx_dim_enabled)) 2594 return; 2595 2596 dim_update_sample(READ_ONCE(cq->dim_event_ctr), rxq->stats.packets, 2597 rxq->stats.bytes, &dim_sample); 2598 net_dim(&cq->dim, &dim_sample); 2599 } 2600 2601 static void mana_update_tx_dim(struct mana_cq *cq) 2602 { 2603 struct mana_port_context *apc = netdev_priv(cq->txq->ndev); 2604 struct dim_sample dim_sample = {}; 2605 2606 /* Pairs with smp_store_release() in mana_set_coalesce(): observing the 2607 * enable flag set guarantees the DIM (re)initialization is visible. 2608 */ 2609 if (!smp_load_acquire(&apc->tx_dim_enabled)) 2610 return; 2611 2612 /* cq->tx_dim_pkts/bytes are accumulated in mana_poll_tx_cq(), in the 2613 * same NAPI context as this read, so they track the hardware 2614 * completion rate and need no u64_stats_sync protection. 2615 */ 2616 dim_update_sample(READ_ONCE(cq->dim_event_ctr), cq->tx_dim_pkts, 2617 cq->tx_dim_bytes, &dim_sample); 2618 net_dim(&cq->dim, &dim_sample); 2619 } 2620 2621 static int mana_cq_handler(void *context, struct gdma_queue *gdma_queue) 2622 { 2623 struct mana_cq *cq = context; 2624 int w; 2625 2626 WARN_ON_ONCE(cq->gdma_cq != gdma_queue); 2627 2628 if (cq->type == MANA_CQ_TYPE_RX) 2629 mana_poll_rx_cq(cq); 2630 else 2631 mana_poll_tx_cq(cq); 2632 2633 w = cq->work_done; 2634 cq->work_done_since_doorbell += w; 2635 2636 if (w < cq->budget) { 2637 mana_gd_ring_cq(gdma_queue, SET_ARM_BIT); 2638 cq->work_done_since_doorbell = 0; 2639 2640 /* Update DIM before napi_complete_done() to prevent running 2641 * net_dim() concurrently. 2642 */ 2643 if (cq->type == MANA_CQ_TYPE_RX) 2644 mana_update_rx_dim(cq); 2645 else 2646 mana_update_tx_dim(cq); 2647 2648 napi_complete_done(&cq->napi, w); 2649 } else if (cq->work_done_since_doorbell >= 2650 (cq->gdma_cq->queue_size / COMP_ENTRY_SIZE) * 4) { 2651 /* MANA hardware requires at least one doorbell ring every 8 2652 * wraparounds of CQ even if there is no need to arm the CQ. 2653 * This driver rings the doorbell as soon as it has processed 2654 * 4 wraparounds. 2655 */ 2656 mana_gd_ring_cq(gdma_queue, 0); 2657 cq->work_done_since_doorbell = 0; 2658 } 2659 2660 return w; 2661 } 2662 2663 static int mana_poll(struct napi_struct *napi, int budget) 2664 { 2665 struct mana_cq *cq = container_of(napi, struct mana_cq, napi); 2666 int w; 2667 2668 cq->work_done = 0; 2669 cq->budget = budget; 2670 2671 w = mana_cq_handler(cq, cq->gdma_cq); 2672 2673 return min(w, budget); 2674 } 2675 2676 static void mana_schedule_napi(void *context, struct gdma_queue *gdma_queue) 2677 { 2678 struct mana_cq *cq = context; 2679 2680 WRITE_ONCE(cq->dim_event_ctr, cq->dim_event_ctr + 1); 2681 napi_schedule_irqoff(&cq->napi); 2682 } 2683 2684 static void mana_deinit_cq(struct mana_port_context *apc, struct mana_cq *cq) 2685 { 2686 struct gdma_dev *gd = apc->ac->gdma_dev; 2687 2688 if (!cq->gdma_cq) 2689 return; 2690 2691 mana_gd_destroy_queue(gd->gdma_context, cq->gdma_cq); 2692 } 2693 2694 static void mana_deinit_txq(struct mana_port_context *apc, struct mana_txq *txq) 2695 { 2696 struct gdma_dev *gd = apc->ac->gdma_dev; 2697 2698 if (!txq->gdma_sq) 2699 return; 2700 2701 mana_gd_destroy_queue(gd->gdma_context, txq->gdma_sq); 2702 } 2703 2704 static void mana_destroy_txq(struct mana_port_context *apc) 2705 { 2706 struct napi_struct *napi; 2707 int i; 2708 2709 if (!apc->tx_qp) 2710 return; 2711 2712 for (i = 0; i < apc->num_queues; i++) { 2713 if (!apc->tx_qp[i]) 2714 continue; 2715 2716 debugfs_remove_recursive(apc->tx_qp[i]->mana_tx_debugfs); 2717 apc->tx_qp[i]->mana_tx_debugfs = NULL; 2718 2719 napi = &apc->tx_qp[i]->tx_cq.napi; 2720 if (apc->tx_qp[i]->txq.napi_initialized) { 2721 napi_synchronize(napi); 2722 napi_disable_locked(napi); 2723 cancel_work_sync(&apc->tx_qp[i]->tx_cq.dim.work); 2724 netif_napi_del_locked(napi); 2725 apc->tx_qp[i]->txq.napi_initialized = false; 2726 } 2727 2728 /* Forget the SKBs freed without being completed after a drain 2729 * timeout, now that NAPI can't complete any. Doing it here also 2730 * covers the queues that a lower channel count won't recreate, 2731 * which could otherwise stay stopped. 2732 */ 2733 netdev_tx_reset_queue(apc->tx_qp[i]->txq.net_txq); 2734 2735 if (apc->tx_qp[i]->tx_object != INVALID_MANA_HANDLE) 2736 mana_destroy_wq_obj(apc, GDMA_SQ, apc->tx_qp[i]->tx_object); 2737 2738 mana_deinit_cq(apc, &apc->tx_qp[i]->tx_cq); 2739 2740 mana_deinit_txq(apc, &apc->tx_qp[i]->txq); 2741 2742 kvfree(apc->tx_qp[i]); 2743 } 2744 2745 kfree(apc->tx_qp); 2746 apc->tx_qp = NULL; 2747 } 2748 2749 static void mana_create_txq_debugfs(struct mana_port_context *apc, int idx) 2750 { 2751 struct mana_tx_qp *tx_qp = apc->tx_qp[idx]; 2752 char qnum[32]; 2753 2754 sprintf(qnum, "TX-%d", idx); 2755 tx_qp->mana_tx_debugfs = debugfs_create_dir(qnum, apc->mana_port_debugfs); 2756 debugfs_create_u32("sq_head", 0400, tx_qp->mana_tx_debugfs, 2757 &tx_qp->txq.gdma_sq->head); 2758 debugfs_create_u32("sq_tail", 0400, tx_qp->mana_tx_debugfs, 2759 &tx_qp->txq.gdma_sq->tail); 2760 debugfs_create_u32("sq_pend_skb_qlen", 0400, tx_qp->mana_tx_debugfs, 2761 &tx_qp->txq.pending_skbs.qlen); 2762 debugfs_create_u32("cq_head", 0400, tx_qp->mana_tx_debugfs, 2763 &tx_qp->tx_cq.gdma_cq->head); 2764 debugfs_create_u32("cq_tail", 0400, tx_qp->mana_tx_debugfs, 2765 &tx_qp->tx_cq.gdma_cq->tail); 2766 debugfs_create_u32("cq_budget", 0400, tx_qp->mana_tx_debugfs, 2767 &tx_qp->tx_cq.budget); 2768 debugfs_create_file("txq_dump", 0400, tx_qp->mana_tx_debugfs, 2769 tx_qp->txq.gdma_sq, &mana_dbg_q_fops); 2770 debugfs_create_file("cq_dump", 0400, tx_qp->mana_tx_debugfs, 2771 tx_qp->tx_cq.gdma_cq, &mana_dbg_q_fops); 2772 } 2773 2774 static int mana_create_txq(struct mana_port_context *apc, 2775 struct net_device *net) 2776 { 2777 struct mana_context *ac = apc->ac; 2778 struct gdma_dev *gd = ac->gdma_dev; 2779 struct mana_obj_spec wq_spec; 2780 struct mana_obj_spec cq_spec; 2781 struct gdma_queue_spec spec; 2782 struct gdma_context *gc; 2783 struct mana_txq *txq; 2784 struct mana_cq *cq; 2785 u32 txq_size; 2786 u32 cq_size; 2787 int err; 2788 int i; 2789 2790 apc->tx_qp = kzalloc_objs(struct mana_tx_qp *, apc->num_queues); 2791 if (!apc->tx_qp) 2792 return -ENOMEM; 2793 2794 /* The minimum size of the WQE is 32 bytes, hence 2795 * apc->tx_queue_size represents the maximum number of WQEs 2796 * the SQ can store. This value is then used to size other queues 2797 * to prevent overflow. 2798 * Also note that the txq_size is always going to be MANA_PAGE_ALIGNED, 2799 * as min val of apc->tx_queue_size is 128 and that would make 2800 * txq_size 128*32 = 4096 and the other higher values of apc->tx_queue_size 2801 * are always power of two 2802 */ 2803 txq_size = apc->tx_queue_size * 32; 2804 2805 cq_size = apc->tx_queue_size * COMP_ENTRY_SIZE; 2806 2807 gc = gd->gdma_context; 2808 2809 for (i = 0; i < apc->num_queues; i++) { 2810 apc->tx_qp[i] = kvzalloc_obj(*apc->tx_qp[i]); 2811 if (!apc->tx_qp[i]) { 2812 err = -ENOMEM; 2813 goto out; 2814 } 2815 2816 apc->tx_qp[i]->tx_object = INVALID_MANA_HANDLE; 2817 2818 /* Create SQ */ 2819 txq = &apc->tx_qp[i]->txq; 2820 2821 u64_stats_init(&txq->stats.syncp); 2822 txq->ndev = net; 2823 txq->net_txq = netdev_get_tx_queue(net, i); 2824 txq->vp_offset = apc->tx_vp_offset; 2825 txq->napi_initialized = false; 2826 skb_queue_head_init(&txq->pending_skbs); 2827 2828 memset(&spec, 0, sizeof(spec)); 2829 spec.type = GDMA_SQ; 2830 spec.monitor_avl_buf = true; 2831 spec.queue_size = txq_size; 2832 err = mana_gd_create_mana_wq_cq(gd, &spec, &txq->gdma_sq); 2833 if (err) 2834 goto out; 2835 2836 /* Create SQ's CQ */ 2837 cq = &apc->tx_qp[i]->tx_cq; 2838 cq->type = MANA_CQ_TYPE_TX; 2839 2840 cq->txq = txq; 2841 2842 memset(&spec, 0, sizeof(spec)); 2843 spec.type = GDMA_CQ; 2844 spec.monitor_avl_buf = false; 2845 spec.queue_size = cq_size; 2846 spec.cq.callback = mana_schedule_napi; 2847 spec.cq.parent_eq = apc->eqs[i].eq; 2848 spec.cq.context = cq; 2849 err = mana_gd_create_mana_wq_cq(gd, &spec, &cq->gdma_cq); 2850 if (err) 2851 goto out; 2852 2853 memset(&wq_spec, 0, sizeof(wq_spec)); 2854 memset(&cq_spec, 0, sizeof(cq_spec)); 2855 2856 wq_spec.gdma_region = txq->gdma_sq->mem_info.dma_region_handle; 2857 wq_spec.queue_size = txq->gdma_sq->queue_size; 2858 2859 cq_spec.gdma_region = cq->gdma_cq->mem_info.dma_region_handle; 2860 cq_spec.queue_size = cq->gdma_cq->queue_size; 2861 cq_spec.modr_ctx_id = 0; 2862 cq_spec.attached_eq = cq->gdma_cq->cq.parent->id; 2863 2864 /* DIM setting can be changed at runtime */ 2865 cq_spec.req_cq_moderation = true; 2866 cq_spec.cq_moderation_usec = apc->intr_modr_tx_usec; 2867 cq_spec.cq_moderation_comp = apc->intr_modr_tx_comp; 2868 2869 err = mana_create_wq_obj(apc, apc->port_handle, GDMA_SQ, 2870 &wq_spec, &cq_spec, 2871 &apc->tx_qp[i]->tx_object); 2872 2873 if (err) 2874 goto out; 2875 2876 txq->gdma_sq->id = wq_spec.queue_index; 2877 cq->gdma_cq->id = cq_spec.queue_index; 2878 2879 txq->gdma_sq->mem_info.dma_region_handle = 2880 GDMA_INVALID_DMA_REGION; 2881 cq->gdma_cq->mem_info.dma_region_handle = 2882 GDMA_INVALID_DMA_REGION; 2883 2884 txq->gdma_txq_id = txq->gdma_sq->id; 2885 2886 cq->gdma_id = cq->gdma_cq->id; 2887 2888 if (WARN_ON(cq->gdma_id >= gc->max_num_cqs)) { 2889 err = -EINVAL; 2890 goto out; 2891 } 2892 2893 gc->cq_table[cq->gdma_id] = cq->gdma_cq; 2894 2895 mana_create_txq_debugfs(apc, i); 2896 2897 set_bit(NAPI_STATE_NO_BUSY_POLL, &cq->napi.state); 2898 netif_napi_add_locked(net, &cq->napi, mana_poll); 2899 2900 /* Initialize the DIM work before enabling NAPI, so that a poll 2901 * cannot reach net_dim() with an uninitialized cq->dim.work. 2902 */ 2903 INIT_WORK(&cq->dim.work, mana_tx_dim_work); 2904 cq->dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE; 2905 2906 napi_enable_locked(&cq->napi); 2907 txq->napi_initialized = true; 2908 2909 mana_gd_ring_cq(cq->gdma_cq, SET_ARM_BIT); 2910 } 2911 2912 return 0; 2913 out: 2914 netdev_err(net, "Failed to create %d TX queues, %d\n", 2915 apc->num_queues, err); 2916 mana_destroy_txq(apc); 2917 return err; 2918 } 2919 2920 static void mana_destroy_rxq(struct mana_port_context *apc, 2921 struct mana_rxq *rxq, bool napi_initialized) 2922 2923 { 2924 struct gdma_context *gc = apc->ac->gdma_dev->gdma_context; 2925 struct mana_recv_buf_oob *rx_oob; 2926 struct device *dev = gc->dev; 2927 struct napi_struct *napi; 2928 struct page *page; 2929 int i; 2930 2931 if (!rxq) 2932 return; 2933 2934 debugfs_remove_recursive(rxq->mana_rx_debugfs); 2935 rxq->mana_rx_debugfs = NULL; 2936 2937 napi = &rxq->rx_cq.napi; 2938 2939 if (napi_initialized) { 2940 napi_synchronize(napi); 2941 2942 napi_disable_locked(napi); 2943 cancel_work_sync(&rxq->rx_cq.dim.work); 2944 netif_napi_del_locked(napi); 2945 } 2946 2947 if (xdp_rxq_info_is_reg(&rxq->xdp_rxq)) 2948 xdp_rxq_info_unreg(&rxq->xdp_rxq); 2949 2950 if (rxq->rxobj != INVALID_MANA_HANDLE) 2951 mana_destroy_wq_obj(apc, GDMA_RQ, rxq->rxobj); 2952 2953 mana_deinit_cq(apc, &rxq->rx_cq); 2954 2955 if (rxq->xdp_save_va) 2956 put_page(virt_to_head_page(rxq->xdp_save_va)); 2957 2958 for (i = 0; i < rxq->num_rx_buf; i++) { 2959 rx_oob = &rxq->rx_oobs[i]; 2960 2961 if (!rx_oob->buf_va) 2962 continue; 2963 2964 page = virt_to_head_page(rx_oob->buf_va); 2965 2966 if (rxq->frag_count == 1 || !rx_oob->from_pool) { 2967 dma_unmap_single(dev, rx_oob->sgl[0].address, 2968 rx_oob->sgl[0].size, DMA_FROM_DEVICE); 2969 mana_put_rx_page(rxq, page, rx_oob->from_pool); 2970 } else { 2971 page_pool_free_va(rxq->page_pool, rx_oob->buf_va, true); 2972 } 2973 2974 rx_oob->buf_va = NULL; 2975 } 2976 2977 page_pool_destroy(rxq->page_pool); 2978 2979 if (rxq->gdma_rq) 2980 mana_gd_destroy_queue(gc, rxq->gdma_rq); 2981 2982 kvfree(rxq); 2983 } 2984 2985 static int mana_fill_rx_oob(struct mana_recv_buf_oob *rx_oob, u32 mem_key, 2986 struct mana_rxq *rxq, struct device *dev) 2987 { 2988 struct mana_port_context *mpc = netdev_priv(rxq->ndev); 2989 struct page *pp_page = NULL; 2990 u32 dma_sync_offset = 0; 2991 bool from_pool = false; 2992 dma_addr_t da; 2993 void *va; 2994 2995 if (mpc->rxbufs_pre) 2996 va = mana_get_rxbuf_pre(rxq, &da); 2997 else 2998 va = mana_get_rxfrag(rxq, dev, &da, &from_pool, &pp_page, 2999 &dma_sync_offset); 3000 3001 if (!va) 3002 return -ENOMEM; 3003 3004 rx_oob->buf_va = va; 3005 rx_oob->from_pool = from_pool; 3006 rx_oob->pp_page = pp_page; 3007 rx_oob->dma_sync_offset = dma_sync_offset; 3008 3009 rx_oob->sgl[0].address = da; 3010 rx_oob->sgl[0].size = rxq->datasize; 3011 rx_oob->sgl[0].mem_key = mem_key; 3012 3013 return 0; 3014 } 3015 3016 #define MANA_WQE_HEADER_SIZE 16 3017 #define MANA_WQE_SGE_SIZE 16 3018 3019 static int mana_alloc_rx_wqe(struct mana_port_context *apc, 3020 struct mana_rxq *rxq, u32 *rxq_size, u32 *cq_size) 3021 { 3022 struct gdma_context *gc = apc->ac->gdma_dev->gdma_context; 3023 struct mana_recv_buf_oob *rx_oob; 3024 struct device *dev = gc->dev; 3025 u32 buf_idx; 3026 int ret; 3027 3028 WARN_ON(rxq->datasize == 0); 3029 3030 *rxq_size = 0; 3031 *cq_size = 0; 3032 3033 for (buf_idx = 0; buf_idx < rxq->num_rx_buf; buf_idx++) { 3034 rx_oob = &rxq->rx_oobs[buf_idx]; 3035 memset(rx_oob, 0, sizeof(*rx_oob)); 3036 3037 rx_oob->num_sge = 1; 3038 3039 ret = mana_fill_rx_oob(rx_oob, apc->ac->gdma_dev->gpa_mkey, rxq, 3040 dev); 3041 if (ret) 3042 return ret; 3043 3044 rx_oob->wqe_req.sgl = rx_oob->sgl; 3045 rx_oob->wqe_req.num_sge = rx_oob->num_sge; 3046 rx_oob->wqe_req.inline_oob_size = 0; 3047 rx_oob->wqe_req.inline_oob_data = NULL; 3048 rx_oob->wqe_req.flags = 0; 3049 rx_oob->wqe_req.client_data_unit = 0; 3050 3051 *rxq_size += ALIGN(MANA_WQE_HEADER_SIZE + 3052 MANA_WQE_SGE_SIZE * rx_oob->num_sge, 32); 3053 *cq_size += COMP_ENTRY_SIZE; 3054 } 3055 3056 /* Reserve an extra slot for Fence completion 3057 * event (CQE_RX_OBJECT_FENCE) in case RX CQ is full. 3058 */ 3059 *cq_size += COMP_ENTRY_SIZE; 3060 return 0; 3061 } 3062 3063 static int mana_push_wqe(struct mana_rxq *rxq) 3064 { 3065 struct mana_recv_buf_oob *rx_oob; 3066 u32 buf_idx; 3067 int err; 3068 3069 for (buf_idx = 0; buf_idx < rxq->num_rx_buf; buf_idx++) { 3070 rx_oob = &rxq->rx_oobs[buf_idx]; 3071 3072 err = mana_gd_post_and_ring(rxq->gdma_rq, &rx_oob->wqe_req, 3073 &rx_oob->wqe_inf); 3074 if (err) 3075 return -ENOSPC; 3076 } 3077 3078 return 0; 3079 } 3080 3081 static int mana_create_page_pool(struct mana_rxq *rxq, struct gdma_context *gc) 3082 { 3083 struct mana_port_context *mpc = netdev_priv(rxq->ndev); 3084 struct page_pool_params pprm = {}; 3085 int ret; 3086 3087 pprm.pool_size = mpc->rx_queue_size / rxq->frag_count + 1; 3088 pprm.nid = gc->numa_node; 3089 pprm.napi = &rxq->rx_cq.napi; 3090 pprm.netdev = rxq->ndev; 3091 pprm.order = get_order(rxq->alloc_size); 3092 pprm.queue_idx = rxq->rxq_idx; 3093 pprm.dev = gc->dev; 3094 3095 /* Let the page pool do the dma map when page sharing with multiple 3096 * fragments enabled for rx buffers. 3097 */ 3098 if (rxq->frag_count > 1) { 3099 pprm.flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV; 3100 pprm.max_len = PAGE_SIZE; 3101 pprm.dma_dir = DMA_FROM_DEVICE; 3102 } 3103 3104 rxq->page_pool = page_pool_create(&pprm); 3105 3106 if (IS_ERR(rxq->page_pool)) { 3107 ret = PTR_ERR(rxq->page_pool); 3108 rxq->page_pool = NULL; 3109 return ret; 3110 } 3111 3112 return 0; 3113 } 3114 3115 static struct mana_rxq *mana_create_rxq(struct mana_port_context *apc, 3116 u32 rxq_idx, struct mana_eq *eq, 3117 struct net_device *ndev) 3118 { 3119 struct gdma_dev *gd = apc->ac->gdma_dev; 3120 struct mana_obj_spec wq_spec; 3121 struct mana_obj_spec cq_spec; 3122 struct gdma_queue_spec spec; 3123 struct mana_cq *cq = NULL; 3124 struct gdma_context *gc; 3125 u32 cq_size, rq_size; 3126 struct mana_rxq *rxq; 3127 int err; 3128 3129 gc = gd->gdma_context; 3130 3131 rxq = kvzalloc_flex(*rxq, rx_oobs, apc->rx_queue_size); 3132 if (!rxq) 3133 return ERR_PTR(-ENOMEM); 3134 3135 rxq->ndev = ndev; 3136 rxq->num_rx_buf = apc->rx_queue_size; 3137 rxq->rxq_idx = rxq_idx; 3138 rxq->rxobj = INVALID_MANA_HANDLE; 3139 3140 mana_get_rxbuf_cfg(apc, ndev->mtu, &rxq->datasize, &rxq->alloc_size, 3141 &rxq->headroom, &rxq->frag_count); 3142 /* Create page pool for RX queue */ 3143 err = mana_create_page_pool(rxq, gc); 3144 if (err) { 3145 netdev_err(ndev, "Create page pool err:%d\n", err); 3146 goto out; 3147 } 3148 3149 err = mana_alloc_rx_wqe(apc, rxq, &rq_size, &cq_size); 3150 if (err) 3151 goto out; 3152 3153 rq_size = MANA_PAGE_ALIGN(rq_size); 3154 cq_size = MANA_PAGE_ALIGN(roundup_pow_of_two(cq_size)); 3155 3156 /* Create RQ */ 3157 memset(&spec, 0, sizeof(spec)); 3158 spec.type = GDMA_RQ; 3159 spec.monitor_avl_buf = true; 3160 spec.queue_size = rq_size; 3161 err = mana_gd_create_mana_wq_cq(gd, &spec, &rxq->gdma_rq); 3162 if (err) 3163 goto out; 3164 3165 /* Create RQ's CQ */ 3166 cq = &rxq->rx_cq; 3167 cq->type = MANA_CQ_TYPE_RX; 3168 cq->rxq = rxq; 3169 3170 memset(&spec, 0, sizeof(spec)); 3171 spec.type = GDMA_CQ; 3172 spec.monitor_avl_buf = false; 3173 spec.queue_size = cq_size; 3174 spec.cq.callback = mana_schedule_napi; 3175 spec.cq.parent_eq = eq->eq; 3176 spec.cq.context = cq; 3177 err = mana_gd_create_mana_wq_cq(gd, &spec, &cq->gdma_cq); 3178 if (err) 3179 goto out; 3180 3181 memset(&wq_spec, 0, sizeof(wq_spec)); 3182 memset(&cq_spec, 0, sizeof(cq_spec)); 3183 wq_spec.gdma_region = rxq->gdma_rq->mem_info.dma_region_handle; 3184 wq_spec.queue_size = rxq->gdma_rq->queue_size; 3185 3186 cq_spec.gdma_region = cq->gdma_cq->mem_info.dma_region_handle; 3187 cq_spec.queue_size = cq->gdma_cq->queue_size; 3188 cq_spec.modr_ctx_id = 0; 3189 cq_spec.attached_eq = cq->gdma_cq->cq.parent->id; 3190 3191 /* DIM setting can be changed at runtime */ 3192 cq_spec.req_cq_moderation = true; 3193 cq_spec.cq_moderation_usec = apc->intr_modr_rx_usec; 3194 cq_spec.cq_moderation_comp = apc->intr_modr_rx_comp; 3195 3196 err = mana_create_wq_obj(apc, apc->port_handle, GDMA_RQ, 3197 &wq_spec, &cq_spec, &rxq->rxobj); 3198 if (err) 3199 goto out; 3200 3201 rxq->gdma_rq->id = wq_spec.queue_index; 3202 cq->gdma_cq->id = cq_spec.queue_index; 3203 3204 rxq->gdma_rq->mem_info.dma_region_handle = GDMA_INVALID_DMA_REGION; 3205 cq->gdma_cq->mem_info.dma_region_handle = GDMA_INVALID_DMA_REGION; 3206 3207 rxq->gdma_id = rxq->gdma_rq->id; 3208 cq->gdma_id = cq->gdma_cq->id; 3209 3210 err = mana_push_wqe(rxq); 3211 if (err) 3212 goto out; 3213 3214 if (WARN_ON(cq->gdma_id >= gc->max_num_cqs)) { 3215 err = -EINVAL; 3216 goto out; 3217 } 3218 3219 gc->cq_table[cq->gdma_id] = cq->gdma_cq; 3220 3221 netif_napi_add_weight_locked(ndev, &cq->napi, mana_poll, 1); 3222 3223 WARN_ON(xdp_rxq_info_reg(&rxq->xdp_rxq, ndev, rxq_idx, 3224 cq->napi.napi_id)); 3225 WARN_ON(xdp_rxq_info_reg_mem_model(&rxq->xdp_rxq, MEM_TYPE_PAGE_POOL, 3226 rxq->page_pool)); 3227 3228 /* Initialize the DIM work before enabling NAPI, so that a poll 3229 * cannot reach net_dim() with an uninitialized cq->dim.work. 3230 */ 3231 INIT_WORK(&cq->dim.work, mana_rx_dim_work); 3232 cq->dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE; 3233 3234 napi_enable_locked(&cq->napi); 3235 3236 mana_gd_ring_cq(cq->gdma_cq, SET_ARM_BIT); 3237 out: 3238 if (!err) 3239 return rxq; 3240 3241 netdev_err(ndev, "Failed to create RXQ: err = %d\n", err); 3242 3243 mana_destroy_rxq(apc, rxq, false); 3244 3245 return ERR_PTR(err); 3246 } 3247 3248 static void mana_create_rxq_debugfs(struct mana_port_context *apc, int idx) 3249 { 3250 struct mana_rxq *rxq; 3251 char qnum[32]; 3252 3253 rxq = apc->rxqs[idx]; 3254 3255 sprintf(qnum, "RX-%d", idx); 3256 rxq->mana_rx_debugfs = debugfs_create_dir(qnum, apc->mana_port_debugfs); 3257 debugfs_create_u32("rq_head", 0400, rxq->mana_rx_debugfs, &rxq->gdma_rq->head); 3258 debugfs_create_u32("rq_tail", 0400, rxq->mana_rx_debugfs, &rxq->gdma_rq->tail); 3259 debugfs_create_u32("rq_nbuf", 0400, rxq->mana_rx_debugfs, &rxq->num_rx_buf); 3260 debugfs_create_u32("cq_head", 0400, rxq->mana_rx_debugfs, 3261 &rxq->rx_cq.gdma_cq->head); 3262 debugfs_create_u32("cq_tail", 0400, rxq->mana_rx_debugfs, 3263 &rxq->rx_cq.gdma_cq->tail); 3264 debugfs_create_u32("cq_budget", 0400, rxq->mana_rx_debugfs, &rxq->rx_cq.budget); 3265 debugfs_create_file("rxq_dump", 0400, rxq->mana_rx_debugfs, rxq->gdma_rq, &mana_dbg_q_fops); 3266 debugfs_create_file("cq_dump", 0400, rxq->mana_rx_debugfs, rxq->rx_cq.gdma_cq, 3267 &mana_dbg_q_fops); 3268 } 3269 3270 static int mana_add_rx_queues(struct mana_port_context *apc, 3271 struct net_device *ndev) 3272 { 3273 struct mana_rxq *rxq; 3274 int err = 0; 3275 int i; 3276 3277 for (i = 0; i < apc->num_queues; i++) { 3278 rxq = mana_create_rxq(apc, i, &apc->eqs[i], ndev); 3279 if (IS_ERR(rxq)) { 3280 err = PTR_ERR(rxq); 3281 netdev_err(ndev, "Failed to create rxq %d : %d\n", i, err); 3282 goto out; 3283 } 3284 3285 u64_stats_init(&rxq->stats.syncp); 3286 3287 apc->rxqs[i] = rxq; 3288 3289 mana_create_rxq_debugfs(apc, i); 3290 } 3291 3292 apc->default_rxobj = apc->rxqs[0]->rxobj; 3293 out: 3294 return err; 3295 } 3296 3297 static void mana_destroy_rxqs(struct mana_port_context *apc) 3298 { 3299 struct mana_rxq *rxq; 3300 u32 rxq_idx; 3301 3302 if (apc->rxqs) { 3303 3304 for (rxq_idx = 0; rxq_idx < apc->num_queues; rxq_idx++) { 3305 rxq = apc->rxqs[rxq_idx]; 3306 if (!rxq) 3307 continue; 3308 3309 mana_destroy_rxq(apc, rxq, true); 3310 apc->rxqs[rxq_idx] = NULL; 3311 } 3312 } 3313 } 3314 3315 static void mana_destroy_vport(struct mana_port_context *apc) 3316 { 3317 struct gdma_dev *gd = apc->ac->gdma_dev; 3318 3319 mana_uncfg_vport(apc); 3320 3321 if (gd->gdma_context->is_pf && !apc->ac->bm_hostmode) 3322 mana_pf_deregister_hw_vport(apc); 3323 } 3324 3325 static int mana_create_vport(struct mana_port_context *apc, 3326 struct net_device *net) 3327 { 3328 struct gdma_dev *gd = apc->ac->gdma_dev; 3329 int err; 3330 3331 apc->default_rxobj = INVALID_MANA_HANDLE; 3332 3333 if (gd->gdma_context->is_pf && !apc->ac->bm_hostmode) { 3334 err = mana_pf_register_hw_vport(apc); 3335 if (err) 3336 return err; 3337 } 3338 3339 err = mana_cfg_vport(apc, gd->pdid, gd->doorbell, false); 3340 if (err) { 3341 if (gd->gdma_context->is_pf && !apc->ac->bm_hostmode) 3342 mana_pf_deregister_hw_vport(apc); 3343 return err; 3344 } 3345 3346 return 0; 3347 } 3348 3349 static int mana_rss_table_alloc(struct mana_port_context *apc) 3350 { 3351 if (!apc->indir_table_sz) { 3352 netdev_err(apc->ndev, 3353 "Indirection table size not set for vPort %d\n", 3354 apc->port_idx); 3355 return -EINVAL; 3356 } 3357 3358 apc->indir_table = kcalloc(apc->indir_table_sz, sizeof(u32), GFP_KERNEL); 3359 if (!apc->indir_table) 3360 return -ENOMEM; 3361 3362 apc->rxobj_table = kzalloc_objs(mana_handle_t, apc->indir_table_sz); 3363 if (!apc->rxobj_table) { 3364 kfree(apc->indir_table); 3365 return -ENOMEM; 3366 } 3367 3368 return 0; 3369 } 3370 3371 static void mana_rss_table_init(struct mana_port_context *apc) 3372 { 3373 int i; 3374 3375 for (i = 0; i < apc->indir_table_sz; i++) 3376 apc->indir_table[i] = 3377 ethtool_rxfh_indir_default(i, apc->num_queues); 3378 } 3379 3380 int mana_disable_vport_rx(struct mana_port_context *apc) 3381 { 3382 return mana_cfg_vport_steering(apc, TRI_STATE_FALSE, false, false, 3383 false); 3384 } 3385 EXPORT_SYMBOL_NS(mana_disable_vport_rx, "NET_MANA"); 3386 3387 int mana_config_rss(struct mana_port_context *apc, enum TRI_STATE rx, 3388 bool update_hash, bool update_tab) 3389 { 3390 u32 queue_idx; 3391 int err; 3392 int i; 3393 3394 if (update_tab) { 3395 for (i = 0; i < apc->indir_table_sz; i++) { 3396 queue_idx = apc->indir_table[i]; 3397 apc->rxobj_table[i] = apc->rxqs[queue_idx]->rxobj; 3398 } 3399 } 3400 3401 err = mana_cfg_vport_steering(apc, rx, true, update_hash, update_tab); 3402 if (err) 3403 return err; 3404 3405 mana_fence_rqs(apc); 3406 3407 return 0; 3408 } 3409 3410 int mana_query_gf_stats(struct mana_context *ac) 3411 { 3412 struct gdma_context *gc = ac->gdma_dev->gdma_context; 3413 struct mana_query_gf_stat_resp resp = {}; 3414 struct mana_query_gf_stat_req req = {}; 3415 struct device *dev = gc->dev; 3416 int err; 3417 3418 mana_gd_init_req_hdr(&req.hdr, MANA_QUERY_GF_STAT, 3419 sizeof(req), sizeof(resp)); 3420 req.hdr.resp.msg_version = GDMA_MESSAGE_V2; 3421 req.req_stats = STATISTICS_FLAGS_RX_DISCARDS_NO_WQE | 3422 STATISTICS_FLAGS_RX_ERRORS_VPORT_DISABLED | 3423 STATISTICS_FLAGS_HC_RX_BYTES | 3424 STATISTICS_FLAGS_HC_RX_UCAST_PACKETS | 3425 STATISTICS_FLAGS_HC_RX_UCAST_BYTES | 3426 STATISTICS_FLAGS_HC_RX_MCAST_PACKETS | 3427 STATISTICS_FLAGS_HC_RX_MCAST_BYTES | 3428 STATISTICS_FLAGS_HC_RX_BCAST_PACKETS | 3429 STATISTICS_FLAGS_HC_RX_BCAST_BYTES | 3430 STATISTICS_FLAGS_TX_ERRORS_GF_DISABLED | 3431 STATISTICS_FLAGS_TX_ERRORS_VPORT_DISABLED | 3432 STATISTICS_FLAGS_TX_ERRORS_INVAL_VPORT_OFFSET_PACKETS | 3433 STATISTICS_FLAGS_TX_ERRORS_VLAN_ENFORCEMENT | 3434 STATISTICS_FLAGS_TX_ERRORS_ETH_TYPE_ENFORCEMENT | 3435 STATISTICS_FLAGS_TX_ERRORS_SA_ENFORCEMENT | 3436 STATISTICS_FLAGS_TX_ERRORS_SQPDID_ENFORCEMENT | 3437 STATISTICS_FLAGS_TX_ERRORS_CQPDID_ENFORCEMENT | 3438 STATISTICS_FLAGS_TX_ERRORS_MTU_VIOLATION | 3439 STATISTICS_FLAGS_TX_ERRORS_INVALID_OOB | 3440 STATISTICS_FLAGS_HC_TX_BYTES | 3441 STATISTICS_FLAGS_HC_TX_UCAST_PACKETS | 3442 STATISTICS_FLAGS_HC_TX_UCAST_BYTES | 3443 STATISTICS_FLAGS_HC_TX_MCAST_PACKETS | 3444 STATISTICS_FLAGS_HC_TX_MCAST_BYTES | 3445 STATISTICS_FLAGS_HC_TX_BCAST_PACKETS | 3446 STATISTICS_FLAGS_HC_TX_BCAST_BYTES | 3447 STATISTICS_FLAGS_TX_ERRORS_GDMA_ERROR; 3448 3449 err = mana_send_request(ac, &req, sizeof(req), &resp, 3450 sizeof(resp)); 3451 if (err) { 3452 dev_err(dev, "Failed to query GF stats: %d\n", err); 3453 return err; 3454 } 3455 err = mana_verify_resp_hdr(&resp.hdr, MANA_QUERY_GF_STAT, 3456 sizeof(resp)); 3457 if (err || resp.hdr.status) { 3458 dev_err(dev, "Failed to query GF stats: %d, 0x%x\n", err, 3459 resp.hdr.status); 3460 return err; 3461 } 3462 3463 ac->hc_stats.hc_rx_discards_no_wqe = resp.rx_discards_nowqe; 3464 ac->hc_stats.hc_rx_err_vport_disabled = resp.rx_err_vport_disabled; 3465 ac->hc_stats.hc_rx_bytes = resp.hc_rx_bytes; 3466 ac->hc_stats.hc_rx_ucast_pkts = resp.hc_rx_ucast_pkts; 3467 ac->hc_stats.hc_rx_ucast_bytes = resp.hc_rx_ucast_bytes; 3468 ac->hc_stats.hc_rx_bcast_pkts = resp.hc_rx_bcast_pkts; 3469 ac->hc_stats.hc_rx_bcast_bytes = resp.hc_rx_bcast_bytes; 3470 ac->hc_stats.hc_rx_mcast_pkts = resp.hc_rx_mcast_pkts; 3471 ac->hc_stats.hc_rx_mcast_bytes = resp.hc_rx_mcast_bytes; 3472 ac->hc_stats.hc_tx_err_gf_disabled = resp.tx_err_gf_disabled; 3473 ac->hc_stats.hc_tx_err_vport_disabled = resp.tx_err_vport_disabled; 3474 ac->hc_stats.hc_tx_err_inval_vportoffset_pkt = 3475 resp.tx_err_inval_vport_offset_pkt; 3476 ac->hc_stats.hc_tx_err_vlan_enforcement = 3477 resp.tx_err_vlan_enforcement; 3478 ac->hc_stats.hc_tx_err_eth_type_enforcement = 3479 resp.tx_err_ethtype_enforcement; 3480 ac->hc_stats.hc_tx_err_sa_enforcement = resp.tx_err_SA_enforcement; 3481 ac->hc_stats.hc_tx_err_sqpdid_enforcement = 3482 resp.tx_err_SQPDID_enforcement; 3483 ac->hc_stats.hc_tx_err_cqpdid_enforcement = 3484 resp.tx_err_CQPDID_enforcement; 3485 ac->hc_stats.hc_tx_err_mtu_violation = resp.tx_err_mtu_violation; 3486 ac->hc_stats.hc_tx_err_inval_oob = resp.tx_err_inval_oob; 3487 ac->hc_stats.hc_tx_bytes = resp.hc_tx_bytes; 3488 ac->hc_stats.hc_tx_ucast_pkts = resp.hc_tx_ucast_pkts; 3489 ac->hc_stats.hc_tx_ucast_bytes = resp.hc_tx_ucast_bytes; 3490 ac->hc_stats.hc_tx_bcast_pkts = resp.hc_tx_bcast_pkts; 3491 ac->hc_stats.hc_tx_bcast_bytes = resp.hc_tx_bcast_bytes; 3492 ac->hc_stats.hc_tx_mcast_pkts = resp.hc_tx_mcast_pkts; 3493 ac->hc_stats.hc_tx_mcast_bytes = resp.hc_tx_mcast_bytes; 3494 ac->hc_stats.hc_tx_err_gdma = resp.tx_err_gdma; 3495 3496 return 0; 3497 } 3498 3499 void mana_query_phy_stats(struct mana_port_context *apc) 3500 { 3501 struct mana_query_phy_stat_resp resp = {}; 3502 struct mana_query_phy_stat_req req = {}; 3503 struct net_device *ndev = apc->ndev; 3504 int err; 3505 3506 mana_gd_init_req_hdr(&req.hdr, MANA_QUERY_PHY_STAT, 3507 sizeof(req), sizeof(resp)); 3508 err = mana_send_request(apc->ac, &req, sizeof(req), &resp, 3509 sizeof(resp)); 3510 if (err) 3511 return; 3512 3513 err = mana_verify_resp_hdr(&resp.hdr, MANA_QUERY_PHY_STAT, 3514 sizeof(resp)); 3515 if (err || resp.hdr.status) { 3516 netdev_err(ndev, 3517 "Failed to query PHY stats: %d, resp:0x%x\n", 3518 err, resp.hdr.status); 3519 return; 3520 } 3521 3522 /* Aggregate drop counters */ 3523 apc->phy_stats.rx_pkt_drop_phy = resp.rx_pkt_drop_phy; 3524 apc->phy_stats.tx_pkt_drop_phy = resp.tx_pkt_drop_phy; 3525 3526 /* Per TC traffic Counters */ 3527 apc->phy_stats.rx_pkt_tc0_phy = resp.rx_pkt_tc0_phy; 3528 apc->phy_stats.tx_pkt_tc0_phy = resp.tx_pkt_tc0_phy; 3529 apc->phy_stats.rx_pkt_tc1_phy = resp.rx_pkt_tc1_phy; 3530 apc->phy_stats.tx_pkt_tc1_phy = resp.tx_pkt_tc1_phy; 3531 apc->phy_stats.rx_pkt_tc2_phy = resp.rx_pkt_tc2_phy; 3532 apc->phy_stats.tx_pkt_tc2_phy = resp.tx_pkt_tc2_phy; 3533 apc->phy_stats.rx_pkt_tc3_phy = resp.rx_pkt_tc3_phy; 3534 apc->phy_stats.tx_pkt_tc3_phy = resp.tx_pkt_tc3_phy; 3535 apc->phy_stats.rx_pkt_tc4_phy = resp.rx_pkt_tc4_phy; 3536 apc->phy_stats.tx_pkt_tc4_phy = resp.tx_pkt_tc4_phy; 3537 apc->phy_stats.rx_pkt_tc5_phy = resp.rx_pkt_tc5_phy; 3538 apc->phy_stats.tx_pkt_tc5_phy = resp.tx_pkt_tc5_phy; 3539 apc->phy_stats.rx_pkt_tc6_phy = resp.rx_pkt_tc6_phy; 3540 apc->phy_stats.tx_pkt_tc6_phy = resp.tx_pkt_tc6_phy; 3541 apc->phy_stats.rx_pkt_tc7_phy = resp.rx_pkt_tc7_phy; 3542 apc->phy_stats.tx_pkt_tc7_phy = resp.tx_pkt_tc7_phy; 3543 3544 /* Per TC byte Counters */ 3545 apc->phy_stats.rx_byte_tc0_phy = resp.rx_byte_tc0_phy; 3546 apc->phy_stats.tx_byte_tc0_phy = resp.tx_byte_tc0_phy; 3547 apc->phy_stats.rx_byte_tc1_phy = resp.rx_byte_tc1_phy; 3548 apc->phy_stats.tx_byte_tc1_phy = resp.tx_byte_tc1_phy; 3549 apc->phy_stats.rx_byte_tc2_phy = resp.rx_byte_tc2_phy; 3550 apc->phy_stats.tx_byte_tc2_phy = resp.tx_byte_tc2_phy; 3551 apc->phy_stats.rx_byte_tc3_phy = resp.rx_byte_tc3_phy; 3552 apc->phy_stats.tx_byte_tc3_phy = resp.tx_byte_tc3_phy; 3553 apc->phy_stats.rx_byte_tc4_phy = resp.rx_byte_tc4_phy; 3554 apc->phy_stats.tx_byte_tc4_phy = resp.tx_byte_tc4_phy; 3555 apc->phy_stats.rx_byte_tc5_phy = resp.rx_byte_tc5_phy; 3556 apc->phy_stats.tx_byte_tc5_phy = resp.tx_byte_tc5_phy; 3557 apc->phy_stats.rx_byte_tc6_phy = resp.rx_byte_tc6_phy; 3558 apc->phy_stats.tx_byte_tc6_phy = resp.tx_byte_tc6_phy; 3559 apc->phy_stats.rx_byte_tc7_phy = resp.rx_byte_tc7_phy; 3560 apc->phy_stats.tx_byte_tc7_phy = resp.tx_byte_tc7_phy; 3561 3562 /* Per TC pause Counters */ 3563 apc->phy_stats.rx_pause_tc0_phy = resp.rx_pause_tc0_phy; 3564 apc->phy_stats.tx_pause_tc0_phy = resp.tx_pause_tc0_phy; 3565 apc->phy_stats.rx_pause_tc1_phy = resp.rx_pause_tc1_phy; 3566 apc->phy_stats.tx_pause_tc1_phy = resp.tx_pause_tc1_phy; 3567 apc->phy_stats.rx_pause_tc2_phy = resp.rx_pause_tc2_phy; 3568 apc->phy_stats.tx_pause_tc2_phy = resp.tx_pause_tc2_phy; 3569 apc->phy_stats.rx_pause_tc3_phy = resp.rx_pause_tc3_phy; 3570 apc->phy_stats.tx_pause_tc3_phy = resp.tx_pause_tc3_phy; 3571 apc->phy_stats.rx_pause_tc4_phy = resp.rx_pause_tc4_phy; 3572 apc->phy_stats.tx_pause_tc4_phy = resp.tx_pause_tc4_phy; 3573 apc->phy_stats.rx_pause_tc5_phy = resp.rx_pause_tc5_phy; 3574 apc->phy_stats.tx_pause_tc5_phy = resp.tx_pause_tc5_phy; 3575 apc->phy_stats.rx_pause_tc6_phy = resp.rx_pause_tc6_phy; 3576 apc->phy_stats.tx_pause_tc6_phy = resp.tx_pause_tc6_phy; 3577 apc->phy_stats.rx_pause_tc7_phy = resp.rx_pause_tc7_phy; 3578 apc->phy_stats.tx_pause_tc7_phy = resp.tx_pause_tc7_phy; 3579 } 3580 3581 static int mana_init_port(struct net_device *ndev) 3582 { 3583 struct mana_port_context *apc = netdev_priv(ndev); 3584 struct gdma_dev *gd = apc->ac->gdma_dev; 3585 u32 max_txq, max_rxq, max_queues; 3586 int port_idx = apc->port_idx; 3587 struct gdma_context *gc; 3588 char vport[32]; 3589 int err; 3590 3591 err = mana_init_port_context(apc); 3592 if (err) 3593 return err; 3594 3595 gc = gd->gdma_context; 3596 3597 err = mana_query_vport_cfg(apc, port_idx, &max_txq, &max_rxq, 3598 &apc->indir_table_sz); 3599 if (err) { 3600 netdev_err(ndev, "Failed to query info for vPort %d\n", 3601 port_idx); 3602 goto reset_apc; 3603 } 3604 3605 max_queues = min_t(u32, max_txq, max_rxq); 3606 if (apc->max_queues > max_queues) 3607 apc->max_queues = max_queues; 3608 if (apc->max_queues > gc->max_num_queues_vport) 3609 apc->max_queues = gc->max_num_queues_vport; 3610 3611 if (apc->num_queues > apc->max_queues) 3612 apc->num_queues = apc->max_queues; 3613 3614 eth_hw_addr_set(ndev, apc->mac_addr); 3615 sprintf(vport, "vport%d", port_idx); 3616 apc->mana_port_debugfs = debugfs_create_dir(vport, gc->mana_pci_debugfs); 3617 3618 debugfs_create_u64("port_handle", 0400, apc->mana_port_debugfs, 3619 &apc->port_handle); 3620 debugfs_create_u32("max_sq", 0400, apc->mana_port_debugfs, 3621 &apc->vport_max_sq); 3622 debugfs_create_u32("max_rq", 0400, apc->mana_port_debugfs, 3623 &apc->vport_max_rq); 3624 debugfs_create_u32("indir_table_sz", 0400, apc->mana_port_debugfs, 3625 &apc->indir_table_sz); 3626 debugfs_create_u32("steer_rx", 0400, apc->mana_port_debugfs, 3627 &apc->steer_rx); 3628 debugfs_create_u32("steer_rss", 0400, apc->mana_port_debugfs, 3629 &apc->steer_rss); 3630 debugfs_create_bool("steer_update_tab", 0400, apc->mana_port_debugfs, 3631 &apc->steer_update_tab); 3632 debugfs_create_u32("steer_cqe_coalescing", 0400, apc->mana_port_debugfs, 3633 &apc->steer_cqe_coalescing); 3634 debugfs_create_u32("current_speed", 0400, apc->mana_port_debugfs, 3635 &apc->speed); 3636 debugfs_create_bool("tx_timeout_skip_reset", 0600, 3637 apc->mana_port_debugfs, 3638 &apc->tx_timeout_skip_reset); 3639 return 0; 3640 3641 reset_apc: 3642 mana_cleanup_port_context(apc); 3643 return err; 3644 } 3645 3646 int mana_alloc_queues(struct net_device *ndev) 3647 { 3648 struct mana_port_context *apc = netdev_priv(ndev); 3649 struct gdma_dev *gd = apc->ac->gdma_dev; 3650 int err; 3651 3652 err = mana_create_vport(apc, ndev); 3653 if (err) { 3654 netdev_err(ndev, "Failed to create vPort %u : %d\n", 3655 apc->port_idx, err); 3656 return err; 3657 } 3658 3659 err = mana_create_eq(apc); 3660 if (err) { 3661 netdev_err(ndev, "Failed to create EQ on vPort %u: %d\n", 3662 apc->port_idx, err); 3663 goto destroy_vport; 3664 } 3665 3666 err = mana_create_txq(apc, ndev); 3667 if (err) { 3668 netdev_err(ndev, "Failed to create TXQ on vPort %u: %d\n", 3669 apc->port_idx, err); 3670 goto destroy_eq; 3671 } 3672 3673 err = netif_set_real_num_tx_queues(ndev, apc->num_queues); 3674 if (err) { 3675 netdev_err(ndev, 3676 "netif_set_real_num_tx_queues () failed for ndev with num_queues %u : %d\n", 3677 apc->num_queues, err); 3678 goto destroy_txq; 3679 } 3680 3681 err = mana_add_rx_queues(apc, ndev); 3682 if (err) 3683 goto destroy_rxq; 3684 3685 apc->rss_state = apc->num_queues > 1 ? TRI_STATE_TRUE : TRI_STATE_FALSE; 3686 3687 err = netif_set_real_num_rx_queues(ndev, apc->num_queues); 3688 if (err) { 3689 netdev_err(ndev, 3690 "netif_set_real_num_rx_queues () failed for ndev with num_queues %u : %d\n", 3691 apc->num_queues, err); 3692 goto destroy_rxq; 3693 } 3694 3695 mana_rss_table_init(apc); 3696 3697 err = mana_config_rss(apc, TRI_STATE_TRUE, true, true); 3698 if (err) { 3699 netdev_err(ndev, "Failed to configure RSS table: %d\n", err); 3700 goto destroy_rxq; 3701 } 3702 3703 if (gd->gdma_context->is_pf && !apc->ac->bm_hostmode) { 3704 err = mana_pf_register_filter(apc); 3705 if (err) 3706 goto destroy_rxq; 3707 } 3708 3709 mana_chn_setxdp(apc, mana_xdp_get(apc)); 3710 3711 return 0; 3712 3713 destroy_rxq: 3714 mana_destroy_rxqs(apc); 3715 destroy_txq: 3716 mana_destroy_txq(apc); 3717 destroy_eq: 3718 mana_destroy_eq(apc); 3719 destroy_vport: 3720 mana_destroy_vport(apc); 3721 return err; 3722 } 3723 3724 int mana_attach(struct net_device *ndev) 3725 { 3726 struct mana_port_context *apc = netdev_priv(ndev); 3727 int err; 3728 3729 ASSERT_RTNL(); 3730 3731 err = mana_init_port(ndev); 3732 if (err) 3733 return err; 3734 3735 if (apc->port_st_save) { 3736 err = mana_alloc_queues(ndev); 3737 if (err) { 3738 mana_cleanup_port_context(apc); 3739 return err; 3740 } 3741 } 3742 3743 apc->port_is_up = apc->port_st_save; 3744 3745 /* Ensure port state updated before txq state */ 3746 smp_wmb(); 3747 3748 netif_device_attach(ndev); 3749 3750 return 0; 3751 } 3752 3753 static int mana_dealloc_queues(struct net_device *ndev) 3754 { 3755 struct mana_port_context *apc = netdev_priv(ndev); 3756 unsigned long timeout = jiffies + 120 * HZ; 3757 struct gdma_dev *gd = apc->ac->gdma_dev; 3758 struct mana_txq *txq; 3759 struct sk_buff *skb; 3760 int i, err; 3761 u32 tsleep; 3762 3763 if (apc->port_is_up) 3764 return -EINVAL; 3765 3766 if (apc->rxqs) 3767 mana_chn_setxdp(apc, NULL); 3768 3769 if (gd->gdma_context->is_pf && !apc->ac->bm_hostmode) 3770 mana_pf_deregister_filter(apc); 3771 3772 /* No packet can be transmitted now since apc->port_is_up is false. 3773 * There is still a tiny chance that mana_poll_tx_cq() can re-enable 3774 * a txq because it may not timely see apc->port_is_up being cleared 3775 * to false, but it doesn't matter since mana_start_xmit() drops any 3776 * new packets due to apc->port_is_up being false. 3777 * 3778 * Drain all the in-flight TX packets. 3779 * A timeout of 120 seconds for all the queues is used. 3780 * This will break the while loop when h/w is not responding. 3781 * This value of 120 has been decided here considering max 3782 * number of queues. 3783 */ 3784 3785 if (apc->tx_qp) { 3786 for (i = 0; i < apc->num_queues; i++) { 3787 txq = &apc->tx_qp[i]->txq; 3788 tsleep = 1000; 3789 while (atomic_read(&txq->pending_sends) > 0 && 3790 time_before(jiffies, timeout)) { 3791 usleep_range(tsleep, tsleep + 1000); 3792 tsleep <<= 1; 3793 } 3794 if (atomic_read(&txq->pending_sends)) { 3795 err = mana_gd_dev_reset(gd->gdma_context); 3796 if (err) { 3797 netdev_err(ndev, "device reset failed %d with %d pkts pending in txq %u\n", 3798 err, 3799 atomic_read(&txq->pending_sends), 3800 txq->gdma_txq_id); 3801 } 3802 break; 3803 } 3804 } 3805 3806 for (i = 0; i < apc->num_queues; i++) { 3807 txq = &apc->tx_qp[i]->txq; 3808 while ((skb = skb_dequeue(&txq->pending_skbs))) { 3809 mana_unmap_skb(skb, apc); 3810 dev_kfree_skb_any(skb); 3811 } 3812 atomic_set(&txq->pending_sends, 0); 3813 } 3814 } 3815 3816 /* We're 100% sure the queues can no longer be woken up, because 3817 * we're sure now mana_poll_tx_cq() can't be running. 3818 */ 3819 3820 apc->rss_state = TRI_STATE_FALSE; 3821 err = mana_disable_vport_rx(apc); 3822 if (err && mana_en_need_log(apc, err)) 3823 netdev_err(ndev, "Failed to disable vPort: %d\n", err); 3824 3825 mana_fence_rqs(apc); 3826 3827 /* Even in err case, still need to cleanup the vPort */ 3828 mana_destroy_rxqs(apc); 3829 mana_destroy_txq(apc); 3830 mana_destroy_eq(apc); 3831 mana_destroy_vport(apc); 3832 3833 return 0; 3834 } 3835 3836 int mana_detach(struct net_device *ndev, bool from_close) 3837 { 3838 struct mana_port_context *apc = netdev_priv(ndev); 3839 int err; 3840 3841 ASSERT_RTNL(); 3842 3843 /* If already detached (indicates detach succeeded but attach failed 3844 * previously). Now skip mana detach and just retry mana_attach. 3845 */ 3846 if (!from_close && !netif_device_present(ndev)) 3847 return 0; 3848 3849 apc->port_st_save = apc->port_is_up; 3850 apc->port_is_up = false; 3851 3852 /* Ensure port state updated before txq state */ 3853 smp_wmb(); 3854 3855 netif_tx_disable(ndev); 3856 3857 if (apc->port_st_save) { 3858 err = mana_dealloc_queues(ndev); 3859 if (err) { 3860 netdev_err(ndev, "%s failed to deallocate queues: %d\n", __func__, err); 3861 return err; 3862 } 3863 } 3864 3865 if (!from_close) { 3866 netif_device_detach(ndev); 3867 mana_cleanup_port_context(apc); 3868 } 3869 3870 return 0; 3871 } 3872 3873 static int mana_probe_port(struct mana_context *ac, int port_idx, 3874 struct net_device **ndev_storage) 3875 { 3876 struct gdma_context *gc = ac->gdma_dev->gdma_context; 3877 struct mana_port_context *apc; 3878 struct net_device *ndev; 3879 int err; 3880 3881 ndev = alloc_etherdev_mq(sizeof(struct mana_port_context), 3882 gc->max_num_queues_vport); 3883 if (!ndev) 3884 return -ENOMEM; 3885 3886 *ndev_storage = ndev; 3887 3888 apc = netdev_priv(ndev); 3889 apc->ac = ac; 3890 apc->ndev = ndev; 3891 apc->max_queues = gc->max_num_queues_vport; 3892 /* Use MANA_DEF_NUM_QUEUES as default, still honoring the HW limit */ 3893 apc->num_queues = min(gc->max_num_queues_vport, MANA_DEF_NUM_QUEUES); 3894 apc->tx_queue_size = DEF_TX_BUFFERS_PER_QUEUE; 3895 apc->rx_queue_size = DEF_RX_BUFFERS_PER_QUEUE; 3896 apc->port_handle = INVALID_MANA_HANDLE; 3897 apc->pf_filter_handle = INVALID_MANA_HANDLE; 3898 apc->port_idx = port_idx; 3899 apc->link_cfg_error = 1; 3900 apc->cqe_coalescing_enable = 0; 3901 apc->cqe8_coalescing_enable = 0; 3902 3903 /* Initialize interrupt moderation settings if supported by HW */ 3904 if (gc->pf_cap_flags1 & GDMA_PF_CAP_FLAG_1_DYN_INTERRUPT_MODERATION) { 3905 apc->intr_modr_rx_usec = MANA_INTR_MODR_USEC_DEF; 3906 apc->intr_modr_rx_comp = MANA_INTR_MODR_COMP_DEF; 3907 apc->intr_modr_tx_usec = MANA_INTR_MODR_USEC_DEF; 3908 apc->intr_modr_tx_comp = MANA_INTR_MODR_COMP_DEF; 3909 apc->rx_dim_enabled = MANA_ADAPTIVE_RX_DEF; 3910 apc->tx_dim_enabled = MANA_ADAPTIVE_TX_DEF; 3911 } 3912 3913 mutex_init(&apc->vport_mutex); 3914 apc->vport_use_count = 0; 3915 3916 ndev->netdev_ops = &mana_devops; 3917 ndev->ethtool_ops = &mana_ethtool_ops; 3918 ndev->mtu = ETH_DATA_LEN; 3919 ndev->max_mtu = gc->adapter_mtu - ETH_HLEN; 3920 ndev->min_mtu = ETH_MIN_MTU; 3921 ndev->needed_headroom = MANA_HEADROOM; 3922 ndev->dev_port = port_idx; 3923 /* Recommended timeout based on HW FPGA re-config scenario. */ 3924 ndev->watchdog_timeo = 15 * HZ; 3925 SET_NETDEV_DEV(ndev, gc->dev); 3926 3927 netif_set_tso_max_size(ndev, GSO_MAX_SIZE); 3928 3929 netif_carrier_off(ndev); 3930 3931 netdev_rss_key_fill(apc->hashkey, MANA_HASH_KEY_SIZE); 3932 3933 err = mana_init_port(ndev); 3934 if (err) 3935 goto free_net; 3936 3937 err = mana_rss_table_alloc(apc); 3938 if (err) 3939 goto reset_apc; 3940 3941 /* Initialize the per port queue reset work.*/ 3942 INIT_WORK(&apc->queue_reset_work, 3943 mana_per_port_queue_reset_work_handler); 3944 3945 netdev_lockdep_set_classes(ndev); 3946 3947 ndev->hw_features = NETIF_F_SG | NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM; 3948 ndev->hw_features |= NETIF_F_RXCSUM; 3949 ndev->hw_features |= NETIF_F_TSO | NETIF_F_TSO6; 3950 ndev->hw_features |= NETIF_F_RXHASH; 3951 ndev->features = ndev->hw_features | NETIF_F_HW_VLAN_CTAG_TX | 3952 NETIF_F_HW_VLAN_CTAG_RX; 3953 ndev->vlan_features = ndev->features; 3954 xdp_set_features_flag(ndev, NETDEV_XDP_ACT_BASIC | 3955 NETDEV_XDP_ACT_REDIRECT | 3956 NETDEV_XDP_ACT_NDO_XMIT); 3957 3958 err = register_netdev(ndev); 3959 if (err) { 3960 netdev_err(ndev, "Unable to register netdev.\n"); 3961 goto free_indir; 3962 } 3963 3964 netif_carrier_on(ndev); 3965 3966 return 0; 3967 3968 free_indir: 3969 mana_cleanup_indir_table(apc); 3970 reset_apc: 3971 mana_cleanup_port_context(apc); 3972 free_net: 3973 *ndev_storage = NULL; 3974 netdev_err(ndev, "Failed to probe vPort %d: %d\n", port_idx, err); 3975 free_netdev(ndev); 3976 return err; 3977 } 3978 3979 static void adev_release(struct device *dev) 3980 { 3981 struct mana_adev *madev = container_of(dev, struct mana_adev, adev.dev); 3982 3983 kfree(madev); 3984 } 3985 3986 static void remove_adev(struct gdma_dev *gd) 3987 { 3988 struct auxiliary_device *adev = gd->adev; 3989 int id = adev->id; 3990 3991 auxiliary_device_delete(adev); 3992 auxiliary_device_uninit(adev); 3993 3994 mana_adev_idx_free(id); 3995 gd->adev = NULL; 3996 } 3997 3998 static int add_adev(struct gdma_dev *gd, const char *name) 3999 { 4000 struct auxiliary_device *adev; 4001 struct mana_adev *madev; 4002 int ret; 4003 int id; 4004 4005 madev = kzalloc_obj(*madev); 4006 if (!madev) 4007 return -ENOMEM; 4008 4009 adev = &madev->adev; 4010 ret = mana_adev_idx_alloc(); 4011 if (ret < 0) 4012 goto idx_fail; 4013 id = ret; 4014 adev->id = id; 4015 4016 adev->name = name; 4017 adev->dev.parent = gd->gdma_context->dev; 4018 adev->dev.release = adev_release; 4019 madev->mdev = gd; 4020 4021 ret = auxiliary_device_init(adev); 4022 if (ret) 4023 goto init_fail; 4024 4025 /* madev is owned by the auxiliary device */ 4026 madev = NULL; 4027 /* Keep match names independent of the common module's name. */ 4028 ret = __auxiliary_device_add(adev, 4029 gd->gdma_context->bus_ops->adev_prefix); 4030 if (ret) 4031 goto add_fail; 4032 4033 gd->adev = adev; 4034 dev_dbg(gd->gdma_context->dev, 4035 "Auxiliary device added successfully\n"); 4036 return 0; 4037 4038 add_fail: 4039 auxiliary_device_uninit(adev); 4040 4041 init_fail: 4042 mana_adev_idx_free(id); 4043 4044 idx_fail: 4045 kfree(madev); 4046 4047 return ret; 4048 } 4049 4050 static void mana_rdma_service_handle(struct work_struct *work) 4051 { 4052 struct mana_service_work *serv_work = 4053 container_of(work, struct mana_service_work, work); 4054 struct gdma_dev *gd = serv_work->gdma_dev; 4055 struct device *dev = gd->gdma_context->dev; 4056 int ret; 4057 4058 /* Pairs with the smp_store_release() in mana_rdma_probe(). */ 4059 if (smp_load_acquire(&gd->rdma_teardown)) 4060 goto out; 4061 4062 switch (serv_work->event) { 4063 case GDMA_SERVICE_TYPE_RDMA_SUSPEND: 4064 if (!gd->adev || gd->is_suspended) 4065 break; 4066 4067 remove_adev(gd); 4068 gd->is_suspended = true; 4069 break; 4070 4071 case GDMA_SERVICE_TYPE_RDMA_RESUME: 4072 if (!gd->is_suspended) 4073 break; 4074 4075 ret = add_adev(gd, "rdma"); 4076 if (ret) 4077 dev_err(dev, "Failed to add adev on resume: %d\n", ret); 4078 else 4079 gd->is_suspended = false; 4080 break; 4081 4082 default: 4083 dev_warn(dev, "unknown adev service event %u\n", 4084 serv_work->event); 4085 break; 4086 } 4087 4088 out: 4089 kfree(serv_work); 4090 } 4091 4092 int mana_rdma_service_event(struct gdma_context *gc, enum gdma_service_type event) 4093 { 4094 struct gdma_dev *gd = &gc->mana_ib; 4095 struct mana_service_work *serv_work; 4096 4097 if (gd->dev_id.type != GDMA_DEVICE_MANA_IB) { 4098 /* RDMA device is not detected on pci */ 4099 return 0; 4100 } 4101 4102 if (!gc->service_wq) 4103 return -EOPNOTSUPP; 4104 4105 serv_work = kzalloc_obj(*serv_work, GFP_ATOMIC); 4106 if (!serv_work) 4107 return -ENOMEM; 4108 4109 serv_work->event = event; 4110 serv_work->gdma_dev = gd; 4111 4112 INIT_WORK(&serv_work->work, mana_rdma_service_handle); 4113 queue_work(gc->service_wq, &serv_work->work); 4114 4115 return 0; 4116 } 4117 4118 #define MANA_GF_STATS_PERIOD (2 * HZ) 4119 4120 static void mana_gf_stats_work_handler(struct work_struct *work) 4121 { 4122 struct mana_context *ac = 4123 container_of(to_delayed_work(work), struct mana_context, gf_stats_work); 4124 struct gdma_context *gc = ac->gdma_dev->gdma_context; 4125 int err; 4126 4127 err = mana_query_gf_stats(ac); 4128 if (err == -ETIMEDOUT) { 4129 /* HWC timeout detected - reset stats and stop rescheduling */ 4130 ac->hwc_timeout_occurred = true; 4131 memset(&ac->hc_stats, 0, sizeof(ac->hc_stats)); 4132 dev_warn(gc->dev, 4133 "Gf stats wk handler: gf stats query timed out.\n"); 4134 /* As HWC timed out, indicating a faulty HW state and needs a 4135 * reset. 4136 */ 4137 mana_schedule_serv_work(gc, GDMA_EQE_HWC_RESET_REQUEST); 4138 return; 4139 } 4140 schedule_delayed_work(&ac->gf_stats_work, MANA_GF_STATS_PERIOD); 4141 } 4142 4143 int mana_probe(struct gdma_dev *gd, bool resuming) 4144 { 4145 struct gdma_context *gc = gd->gdma_context; 4146 struct mana_context *ac = gd->driver_data; 4147 struct mana_port_context *apc = NULL; 4148 struct device *dev = gc->dev; 4149 u8 bm_hostmode = 0; 4150 u16 num_ports = 0; 4151 int err; 4152 int i; 4153 4154 dev_info(dev, 4155 "Microsoft Azure Network Adapter protocol version: %d.%d.%d\n", 4156 MANA_MAJOR_VERSION, MANA_MINOR_VERSION, MANA_MICRO_VERSION); 4157 4158 err = mana_gd_register_device(gd); 4159 if (err) 4160 return err; 4161 4162 if (!resuming) { 4163 ac = kzalloc_obj(*ac); 4164 if (!ac) 4165 return -ENOMEM; 4166 4167 ac->gdma_dev = gd; 4168 gd->driver_data = ac; 4169 4170 INIT_WORK(&ac->link_change_work, mana_link_state_handle); 4171 } 4172 4173 INIT_DELAYED_WORK(&ac->gf_stats_work, mana_gf_stats_work_handler); 4174 4175 err = mana_gd_query_device_cfg(gc, MANA_MAJOR_VERSION, 4176 MANA_MINOR_VERSION, 4177 MANA_MICRO_VERSION, 4178 &num_ports, &bm_hostmode); 4179 if (err) 4180 goto out; 4181 4182 ac->bm_hostmode = bm_hostmode; 4183 4184 debugfs_create_u16("adapter-MTU", 0400, 4185 gc->mana_pci_debugfs, &gc->adapter_mtu); 4186 4187 if (!resuming) { 4188 ac->num_ports = num_ports; 4189 } else { 4190 if (ac->num_ports != num_ports) { 4191 dev_err(dev, "The number of vPorts changed: %d->%d\n", 4192 ac->num_ports, num_ports); 4193 err = -EPROTO; 4194 goto out; 4195 } 4196 4197 enable_work(&ac->link_change_work); 4198 } 4199 4200 if (ac->num_ports > MAX_PORTS_IN_MANA_DEV) 4201 ac->num_ports = MAX_PORTS_IN_MANA_DEV; 4202 4203 debugfs_create_u16("num_vports", 0400, gc->mana_pci_debugfs, 4204 &ac->num_ports); 4205 debugfs_create_u8("bm_hostmode", 0400, gc->mana_pci_debugfs, 4206 &ac->bm_hostmode); 4207 4208 ac->per_port_queue_reset_wq = 4209 create_singlethread_workqueue("mana_per_port_queue_reset_wq"); 4210 if (!ac->per_port_queue_reset_wq) { 4211 dev_err(dev, "Failed to allocate per port queue reset workqueue\n"); 4212 err = -ENOMEM; 4213 goto out; 4214 } 4215 4216 if (!resuming) { 4217 for (i = 0; i < ac->num_ports; i++) { 4218 err = mana_probe_port(ac, i, &ac->ports[i]); 4219 /* Log the port for which the probe failed, stop probing 4220 * subsequent ports, and skip add_adev. 4221 * mana_remove() will clean up already-probed ports. 4222 */ 4223 if (err) { 4224 dev_err(dev, "Probe Failed for port %d\n", i); 4225 break; 4226 } 4227 } 4228 } else { 4229 for (i = 0; i < ac->num_ports; i++) { 4230 rtnl_lock(); 4231 apc = netdev_priv(ac->ports[i]); 4232 enable_work(&apc->queue_reset_work); 4233 netdev_lock(ac->ports[i]); 4234 apc->link_cfg_error = 1; 4235 netdev_unlock(ac->ports[i]); 4236 err = mana_attach(ac->ports[i]); 4237 rtnl_unlock(); 4238 /* Log the port for which the attach failed, stop 4239 * attaching subsequent ports, and skip add_adev. 4240 * mana_remove() will clean up already-attached ports. 4241 */ 4242 if (err) { 4243 dev_err(dev, "Attach Failed for port %d\n", i); 4244 break; 4245 } 4246 } 4247 } 4248 4249 if (!err) 4250 err = add_adev(gd, "eth"); 4251 4252 schedule_delayed_work(&ac->gf_stats_work, MANA_GF_STATS_PERIOD); 4253 4254 out: 4255 if (err) { 4256 mana_remove(gd, false); 4257 } else { 4258 dev_dbg(dev, "gd=%p, id=%u, num_ports=%d, type=%u, instance=%u\n", 4259 gd, gd->dev_id.as_uint32, ac->num_ports, 4260 gd->dev_id.type, gd->dev_id.instance); 4261 dev_dbg(dev, "%s succeeded\n", __func__); 4262 } 4263 4264 return err; 4265 } 4266 EXPORT_SYMBOL_NS(mana_probe, "NET_MANA"); 4267 4268 void mana_remove(struct gdma_dev *gd, bool suspending) 4269 { 4270 struct gdma_context *gc = gd->gdma_context; 4271 struct mana_context *ac = gd->driver_data; 4272 struct mana_port_context *apc; 4273 struct device *dev; 4274 struct net_device *ndev; 4275 int err; 4276 int i; 4277 4278 if (!gc || !ac) 4279 return; 4280 4281 dev = gc->dev; 4282 4283 disable_work_sync(&ac->link_change_work); 4284 cancel_delayed_work_sync(&ac->gf_stats_work); 4285 4286 /* adev currently doesn't support suspending, always remove it */ 4287 if (gd->adev) 4288 remove_adev(gd); 4289 4290 for (i = 0; i < ac->num_ports; i++) { 4291 ndev = ac->ports[i]; 4292 if (!ndev) { 4293 if (i == 0) 4294 dev_err(dev, "No net device to remove\n"); 4295 break; 4296 } 4297 4298 apc = netdev_priv(ndev); 4299 disable_work_sync(&apc->queue_reset_work); 4300 4301 /* All cleanup actions should stay after rtnl_lock(), otherwise 4302 * other functions may access partially cleaned up data. 4303 */ 4304 rtnl_lock(); 4305 4306 err = mana_detach(ndev, false); 4307 if (err) 4308 netdev_err(ndev, "Failed to detach vPort %d: %d\n", 4309 i, err); 4310 4311 if (suspending) { 4312 /* No need to unregister the ndev. */ 4313 rtnl_unlock(); 4314 continue; 4315 } 4316 4317 unregister_netdevice(ndev); 4318 mana_cleanup_indir_table(apc); 4319 4320 rtnl_unlock(); 4321 4322 free_netdev(ndev); 4323 } 4324 4325 if (ac->per_port_queue_reset_wq) { 4326 destroy_workqueue(ac->per_port_queue_reset_wq); 4327 ac->per_port_queue_reset_wq = NULL; 4328 } 4329 4330 mana_gd_deregister_device(gd); 4331 4332 if (gc->mana_pci_debugfs) { 4333 debugfs_lookup_and_remove("bm_hostmode", gc->mana_pci_debugfs); 4334 debugfs_lookup_and_remove("num_vports", gc->mana_pci_debugfs); 4335 } 4336 4337 if (suspending) 4338 return; 4339 4340 gd->driver_data = NULL; 4341 gd->gdma_context = NULL; 4342 kfree(ac); 4343 dev_dbg(dev, "%s succeeded\n", __func__); 4344 } 4345 EXPORT_SYMBOL_NS(mana_remove, "NET_MANA"); 4346 4347 int mana_rdma_probe(struct gdma_dev *gd) 4348 { 4349 int err = 0; 4350 4351 if (gd->dev_id.type != GDMA_DEVICE_MANA_IB) { 4352 /* RDMA device is not detected on pci */ 4353 return err; 4354 } 4355 4356 err = mana_gd_register_device(gd); 4357 if (err) 4358 return err; 4359 4360 /* Clear the state left by a previous mana_rdma_remove() so servicing 4361 * events are handled again after a reset cycle. 4362 */ 4363 gd->is_suspended = false; 4364 4365 /* Publish is_suspended before re-opening the gate, so the handler 4366 * cannot observe an open gate with a stale is_suspended. Pairs 4367 * with the smp_load_acquire() in mana_rdma_service_handle(). This 4368 * matters on the reset path, where mana_rdma_remove() closed the 4369 * gate and drained the workqueue; on the initial probe path the 4370 * gate was never closed and both flags are already clear. It does 4371 * not order gd->adev, which add_adev() publishes below. 4372 */ 4373 smp_store_release(&gd->rdma_teardown, false); 4374 4375 err = add_adev(gd, "rdma"); 4376 if (err) 4377 mana_gd_deregister_device(gd); 4378 4379 return err; 4380 } 4381 EXPORT_SYMBOL_NS(mana_rdma_probe, "NET_MANA"); 4382 4383 void mana_rdma_remove(struct gdma_dev *gd) 4384 { 4385 struct gdma_context *gc = gd->gdma_context; 4386 4387 if (gd->dev_id.type != GDMA_DEVICE_MANA_IB) { 4388 /* RDMA device is not detected on pci */ 4389 return; 4390 } 4391 4392 WRITE_ONCE(gd->rdma_teardown, true); 4393 4394 if (gc->service_wq) 4395 flush_workqueue(gc->service_wq); 4396 4397 if (gd->adev) 4398 remove_adev(gd); 4399 4400 mana_gd_deregister_device(gd); 4401 } 4402 EXPORT_SYMBOL_NS(mana_rdma_remove, "NET_MANA"); 4403 4404 struct net_device *mana_get_primary_netdev(struct mana_context *ac, 4405 u32 port_index, 4406 netdevice_tracker *tracker) 4407 { 4408 struct net_device *ndev; 4409 4410 if (port_index >= ac->num_ports) 4411 return NULL; 4412 4413 rcu_read_lock(); 4414 4415 /* If mana is used in netvsc, the upper netdevice should be returned. */ 4416 ndev = netdev_master_upper_dev_get_rcu(ac->ports[port_index]); 4417 4418 /* If there is no upper device, use the parent Ethernet device */ 4419 if (!ndev) 4420 ndev = ac->ports[port_index]; 4421 4422 netdev_hold(ndev, tracker, GFP_ATOMIC); 4423 rcu_read_unlock(); 4424 4425 return ndev; 4426 } 4427 EXPORT_SYMBOL_NS(mana_get_primary_netdev, "NET_MANA"); 4428