1 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause 2 /* Copyright (c) 2021, Microsoft Corporation. */ 3 4 #include <uapi/linux/bpf.h> 5 6 #include <linux/debugfs.h> 7 #include <linux/inetdevice.h> 8 #include <linux/etherdevice.h> 9 #include <linux/ethtool.h> 10 #include <linux/filter.h> 11 #include <linux/mm.h> 12 #include <linux/pci.h> 13 #include <linux/export.h> 14 15 #include <net/checksum.h> 16 #include <net/ip6_checksum.h> 17 #include <net/netdev_lock.h> 18 #include <net/page_pool/helpers.h> 19 #include <net/xdp.h> 20 21 #include <net/mana/mana.h> 22 #include <net/mana/mana_auxiliary.h> 23 24 static DEFINE_IDA(mana_adev_ida); 25 26 static int mana_adev_idx_alloc(void) 27 { 28 return ida_alloc(&mana_adev_ida, GFP_KERNEL); 29 } 30 31 static void mana_adev_idx_free(int idx) 32 { 33 ida_free(&mana_adev_ida, idx); 34 } 35 36 static ssize_t mana_dbg_q_read(struct file *filp, char __user *buf, size_t count, 37 loff_t *pos) 38 { 39 struct gdma_queue *gdma_q = filp->private_data; 40 41 return simple_read_from_buffer(buf, count, pos, gdma_q->queue_mem_ptr, 42 gdma_q->queue_size); 43 } 44 45 static const struct file_operations mana_dbg_q_fops = { 46 .owner = THIS_MODULE, 47 .open = simple_open, 48 .read = mana_dbg_q_read, 49 }; 50 51 static bool mana_en_need_log(struct mana_port_context *apc, int err) 52 { 53 if (apc && apc->ac && apc->ac->gdma_dev && 54 apc->ac->gdma_dev->gdma_context) 55 return mana_need_log(apc->ac->gdma_dev->gdma_context, err); 56 else 57 return true; 58 } 59 60 static void mana_put_rx_page(struct mana_rxq *rxq, struct page *page, 61 bool from_pool) 62 { 63 if (from_pool) 64 page_pool_put_full_page(rxq->page_pool, page, false); 65 else 66 put_page(page); 67 } 68 69 /* Microsoft Azure Network Adapter (MANA) functions */ 70 71 static int mana_open(struct net_device *ndev) 72 { 73 struct mana_port_context *apc = netdev_priv(ndev); 74 int err; 75 err = mana_alloc_queues(ndev); 76 77 if (err) { 78 netdev_err(ndev, "%s failed to allocate queues: %d\n", __func__, err); 79 return err; 80 } 81 82 apc->port_is_up = true; 83 84 /* Ensure port state updated before txq state */ 85 smp_wmb(); 86 87 netif_carrier_on(ndev); 88 netif_tx_wake_all_queues(ndev); 89 netdev_dbg(ndev, "%s successful\n", __func__); 90 return 0; 91 } 92 93 static int mana_close(struct net_device *ndev) 94 { 95 struct mana_port_context *apc = netdev_priv(ndev); 96 97 if (!apc->port_is_up) 98 return 0; 99 100 return mana_detach(ndev, true); 101 } 102 103 static bool mana_can_tx(struct gdma_queue *wq) 104 { 105 return mana_gd_wq_avail_space(wq) >= MAX_TX_WQE_SIZE; 106 } 107 108 static unsigned int mana_checksum_info(struct sk_buff *skb) 109 { 110 if (skb->protocol == htons(ETH_P_IP)) { 111 struct iphdr *ip = ip_hdr(skb); 112 113 if (ip->protocol == IPPROTO_TCP) 114 return IPPROTO_TCP; 115 116 if (ip->protocol == IPPROTO_UDP) 117 return IPPROTO_UDP; 118 } else if (skb->protocol == htons(ETH_P_IPV6)) { 119 struct ipv6hdr *ip6 = ipv6_hdr(skb); 120 121 if (ip6->nexthdr == IPPROTO_TCP) 122 return IPPROTO_TCP; 123 124 if (ip6->nexthdr == IPPROTO_UDP) 125 return IPPROTO_UDP; 126 } 127 128 /* No csum offloading */ 129 return 0; 130 } 131 132 static void mana_add_sge(struct mana_tx_package *tp, struct mana_skb_head *ash, 133 int sg_i, dma_addr_t da, int sge_len, u32 gpa_mkey) 134 { 135 ash->dma_handle[sg_i] = da; 136 ash->size[sg_i] = sge_len; 137 138 tp->wqe_req.sgl[sg_i].address = da; 139 tp->wqe_req.sgl[sg_i].mem_key = gpa_mkey; 140 tp->wqe_req.sgl[sg_i].size = sge_len; 141 } 142 143 static int mana_map_skb(struct sk_buff *skb, struct mana_port_context *apc, 144 struct mana_tx_package *tp, int gso_hs) 145 { 146 struct mana_skb_head *ash = (struct mana_skb_head *)skb->head; 147 int hsg = 1; /* num of SGEs of linear part */ 148 struct gdma_dev *gd = apc->ac->gdma_dev; 149 int skb_hlen = skb_headlen(skb); 150 int sge0_len, sge1_len = 0; 151 struct gdma_context *gc; 152 struct device *dev; 153 skb_frag_t *frag; 154 dma_addr_t da; 155 int sg_i; 156 int i; 157 158 gc = gd->gdma_context; 159 dev = gc->dev; 160 161 if (gso_hs && gso_hs < skb_hlen) { 162 sge0_len = gso_hs; 163 sge1_len = skb_hlen - gso_hs; 164 } else { 165 sge0_len = skb_hlen; 166 } 167 168 da = dma_map_single(dev, skb->data, sge0_len, DMA_TO_DEVICE); 169 if (dma_mapping_error(dev, da)) 170 return -ENOMEM; 171 172 mana_add_sge(tp, ash, 0, da, sge0_len, gd->gpa_mkey); 173 174 if (sge1_len) { 175 sg_i = 1; 176 da = dma_map_single(dev, skb->data + sge0_len, sge1_len, 177 DMA_TO_DEVICE); 178 if (dma_mapping_error(dev, da)) 179 goto frag_err; 180 181 mana_add_sge(tp, ash, sg_i, da, sge1_len, gd->gpa_mkey); 182 hsg = 2; 183 } 184 185 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 186 sg_i = hsg + i; 187 188 frag = &skb_shinfo(skb)->frags[i]; 189 da = skb_frag_dma_map(dev, frag, 0, skb_frag_size(frag), 190 DMA_TO_DEVICE); 191 if (dma_mapping_error(dev, da)) 192 goto frag_err; 193 194 mana_add_sge(tp, ash, sg_i, da, skb_frag_size(frag), 195 gd->gpa_mkey); 196 } 197 198 return 0; 199 200 frag_err: 201 if (net_ratelimit()) 202 netdev_err(apc->ndev, "Failed to map skb of size %u to DMA\n", 203 skb->len); 204 for (i = sg_i - 1; i >= hsg; i--) 205 dma_unmap_page(dev, ash->dma_handle[i], ash->size[i], 206 DMA_TO_DEVICE); 207 208 for (i = hsg - 1; i >= 0; i--) 209 dma_unmap_single(dev, ash->dma_handle[i], ash->size[i], 210 DMA_TO_DEVICE); 211 212 return -ENOMEM; 213 } 214 215 /* Handle the case when GSO SKB linear length is too large. 216 * MANA NIC requires GSO packets to put only the packet header to SGE0. 217 * So, we need 2 SGEs for the skb linear part which contains more than the 218 * header. 219 * Return a positive value for the number of SGEs, or a negative value 220 * for an error. 221 */ 222 static int mana_fix_skb_head(struct net_device *ndev, struct sk_buff *skb, 223 int gso_hs) 224 { 225 int num_sge = 1 + skb_shinfo(skb)->nr_frags; 226 int skb_hlen = skb_headlen(skb); 227 228 if (gso_hs < skb_hlen) { 229 num_sge++; 230 } else if (gso_hs > skb_hlen) { 231 if (net_ratelimit()) 232 netdev_err(ndev, 233 "TX nonlinear head: hs:%d, skb_hlen:%d\n", 234 gso_hs, skb_hlen); 235 236 return -EINVAL; 237 } 238 239 return num_sge; 240 } 241 242 /* Get the GSO packet's header size */ 243 static int mana_get_gso_hs(struct sk_buff *skb) 244 { 245 int gso_hs; 246 247 if (skb->encapsulation) { 248 gso_hs = skb_inner_tcp_all_headers(skb); 249 } else { 250 if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4) { 251 gso_hs = skb_transport_offset(skb) + 252 sizeof(struct udphdr); 253 } else { 254 gso_hs = skb_tcp_all_headers(skb); 255 } 256 } 257 258 return gso_hs; 259 } 260 261 netdev_tx_t mana_start_xmit(struct sk_buff *skb, struct net_device *ndev) 262 { 263 enum mana_tx_pkt_format pkt_fmt = MANA_SHORT_PKT_FMT; 264 struct mana_port_context *apc = netdev_priv(ndev); 265 int gso_hs = 0; /* zero for non-GSO pkts */ 266 u16 txq_idx = skb_get_queue_mapping(skb); 267 struct gdma_dev *gd = apc->ac->gdma_dev; 268 bool ipv4 = false, ipv6 = false; 269 struct mana_tx_package pkg = {}; 270 struct netdev_queue *net_txq; 271 struct mana_stats_tx *tx_stats; 272 struct gdma_queue *gdma_sq; 273 int err, len, num_gso_seg; 274 unsigned int csum_type; 275 struct mana_txq *txq; 276 struct mana_cq *cq; 277 278 if (unlikely(!apc->port_is_up)) 279 goto tx_drop; 280 281 if (skb_cow_head(skb, MANA_HEADROOM)) 282 goto tx_drop_count; 283 284 if (unlikely(ipv6_hopopt_jumbo_remove(skb))) 285 goto tx_drop_count; 286 287 txq = &apc->tx_qp[txq_idx].txq; 288 gdma_sq = txq->gdma_sq; 289 cq = &apc->tx_qp[txq_idx].tx_cq; 290 tx_stats = &txq->stats; 291 292 pkg.tx_oob.s_oob.vcq_num = cq->gdma_id; 293 pkg.tx_oob.s_oob.vsq_frame = txq->vsq_frame; 294 295 if (txq->vp_offset > MANA_SHORT_VPORT_OFFSET_MAX) { 296 pkg.tx_oob.l_oob.long_vp_offset = txq->vp_offset; 297 pkt_fmt = MANA_LONG_PKT_FMT; 298 } else { 299 pkg.tx_oob.s_oob.short_vp_offset = txq->vp_offset; 300 } 301 302 if (skb_vlan_tag_present(skb)) { 303 pkt_fmt = MANA_LONG_PKT_FMT; 304 pkg.tx_oob.l_oob.inject_vlan_pri_tag = 1; 305 pkg.tx_oob.l_oob.pcp = skb_vlan_tag_get_prio(skb); 306 pkg.tx_oob.l_oob.dei = skb_vlan_tag_get_cfi(skb); 307 pkg.tx_oob.l_oob.vlan_id = skb_vlan_tag_get_id(skb); 308 } 309 310 pkg.tx_oob.s_oob.pkt_fmt = pkt_fmt; 311 312 if (pkt_fmt == MANA_SHORT_PKT_FMT) { 313 pkg.wqe_req.inline_oob_size = sizeof(struct mana_tx_short_oob); 314 u64_stats_update_begin(&tx_stats->syncp); 315 tx_stats->short_pkt_fmt++; 316 u64_stats_update_end(&tx_stats->syncp); 317 } else { 318 pkg.wqe_req.inline_oob_size = sizeof(struct mana_tx_oob); 319 u64_stats_update_begin(&tx_stats->syncp); 320 tx_stats->long_pkt_fmt++; 321 u64_stats_update_end(&tx_stats->syncp); 322 } 323 324 pkg.wqe_req.inline_oob_data = &pkg.tx_oob; 325 pkg.wqe_req.flags = 0; 326 pkg.wqe_req.client_data_unit = 0; 327 328 pkg.wqe_req.num_sge = 1 + skb_shinfo(skb)->nr_frags; 329 330 if (skb->protocol == htons(ETH_P_IP)) 331 ipv4 = true; 332 else if (skb->protocol == htons(ETH_P_IPV6)) 333 ipv6 = true; 334 335 if (skb_is_gso(skb)) { 336 int num_sge; 337 338 gso_hs = mana_get_gso_hs(skb); 339 340 num_sge = mana_fix_skb_head(ndev, skb, gso_hs); 341 if (num_sge > 0) 342 pkg.wqe_req.num_sge = num_sge; 343 else 344 goto tx_drop_count; 345 346 u64_stats_update_begin(&tx_stats->syncp); 347 if (skb->encapsulation) { 348 tx_stats->tso_inner_packets++; 349 tx_stats->tso_inner_bytes += skb->len - gso_hs; 350 } else { 351 tx_stats->tso_packets++; 352 tx_stats->tso_bytes += skb->len - gso_hs; 353 } 354 u64_stats_update_end(&tx_stats->syncp); 355 356 pkg.tx_oob.s_oob.is_outer_ipv4 = ipv4; 357 pkg.tx_oob.s_oob.is_outer_ipv6 = ipv6; 358 359 pkg.tx_oob.s_oob.comp_iphdr_csum = 1; 360 pkg.tx_oob.s_oob.comp_tcp_csum = 1; 361 pkg.tx_oob.s_oob.trans_off = skb_transport_offset(skb); 362 363 pkg.wqe_req.client_data_unit = skb_shinfo(skb)->gso_size; 364 pkg.wqe_req.flags = GDMA_WR_OOB_IN_SGL | GDMA_WR_PAD_BY_SGE0; 365 if (ipv4) { 366 ip_hdr(skb)->tot_len = 0; 367 ip_hdr(skb)->check = 0; 368 tcp_hdr(skb)->check = 369 ~csum_tcpudp_magic(ip_hdr(skb)->saddr, 370 ip_hdr(skb)->daddr, 0, 371 IPPROTO_TCP, 0); 372 } else { 373 ipv6_hdr(skb)->payload_len = 0; 374 tcp_hdr(skb)->check = 375 ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr, 376 &ipv6_hdr(skb)->daddr, 0, 377 IPPROTO_TCP, 0); 378 } 379 } else if (skb->ip_summed == CHECKSUM_PARTIAL) { 380 csum_type = mana_checksum_info(skb); 381 382 u64_stats_update_begin(&tx_stats->syncp); 383 tx_stats->csum_partial++; 384 u64_stats_update_end(&tx_stats->syncp); 385 386 if (csum_type == IPPROTO_TCP) { 387 pkg.tx_oob.s_oob.is_outer_ipv4 = ipv4; 388 pkg.tx_oob.s_oob.is_outer_ipv6 = ipv6; 389 390 pkg.tx_oob.s_oob.comp_tcp_csum = 1; 391 pkg.tx_oob.s_oob.trans_off = skb_transport_offset(skb); 392 393 } else if (csum_type == IPPROTO_UDP) { 394 pkg.tx_oob.s_oob.is_outer_ipv4 = ipv4; 395 pkg.tx_oob.s_oob.is_outer_ipv6 = ipv6; 396 397 pkg.tx_oob.s_oob.comp_udp_csum = 1; 398 } else { 399 /* Can't do offload of this type of checksum */ 400 if (skb_checksum_help(skb)) 401 goto tx_drop_count; 402 } 403 } 404 405 WARN_ON_ONCE(pkg.wqe_req.num_sge > MAX_TX_WQE_SGL_ENTRIES); 406 407 if (pkg.wqe_req.num_sge <= ARRAY_SIZE(pkg.sgl_array)) { 408 pkg.wqe_req.sgl = pkg.sgl_array; 409 } else { 410 pkg.sgl_ptr = kmalloc_array(pkg.wqe_req.num_sge, 411 sizeof(struct gdma_sge), 412 GFP_ATOMIC); 413 if (!pkg.sgl_ptr) 414 goto tx_drop_count; 415 416 pkg.wqe_req.sgl = pkg.sgl_ptr; 417 } 418 419 if (mana_map_skb(skb, apc, &pkg, gso_hs)) { 420 u64_stats_update_begin(&tx_stats->syncp); 421 tx_stats->mana_map_err++; 422 u64_stats_update_end(&tx_stats->syncp); 423 goto free_sgl_ptr; 424 } 425 426 skb_queue_tail(&txq->pending_skbs, skb); 427 428 len = skb->len; 429 num_gso_seg = skb_is_gso(skb) ? skb_shinfo(skb)->gso_segs : 1; 430 net_txq = netdev_get_tx_queue(ndev, txq_idx); 431 432 err = mana_gd_post_work_request(gdma_sq, &pkg.wqe_req, 433 (struct gdma_posted_wqe_info *)skb->cb); 434 if (!mana_can_tx(gdma_sq)) { 435 netif_tx_stop_queue(net_txq); 436 apc->eth_stats.stop_queue++; 437 } 438 439 if (err) { 440 (void)skb_dequeue_tail(&txq->pending_skbs); 441 netdev_warn(ndev, "Failed to post TX OOB: %d\n", err); 442 err = NETDEV_TX_BUSY; 443 goto tx_busy; 444 } 445 446 err = NETDEV_TX_OK; 447 atomic_inc(&txq->pending_sends); 448 449 mana_gd_wq_ring_doorbell(gd->gdma_context, gdma_sq); 450 451 /* skb may be freed after mana_gd_post_work_request. Do not use it. */ 452 skb = NULL; 453 454 /* Populated the packet and bytes counters based on post GSO packet 455 * calculations 456 */ 457 tx_stats = &txq->stats; 458 u64_stats_update_begin(&tx_stats->syncp); 459 tx_stats->packets += num_gso_seg; 460 tx_stats->bytes += len + ((num_gso_seg - 1) * gso_hs); 461 u64_stats_update_end(&tx_stats->syncp); 462 463 tx_busy: 464 if (netif_tx_queue_stopped(net_txq) && mana_can_tx(gdma_sq)) { 465 netif_tx_wake_queue(net_txq); 466 apc->eth_stats.wake_queue++; 467 } 468 469 kfree(pkg.sgl_ptr); 470 return err; 471 472 free_sgl_ptr: 473 kfree(pkg.sgl_ptr); 474 tx_drop_count: 475 ndev->stats.tx_dropped++; 476 tx_drop: 477 dev_kfree_skb_any(skb); 478 return NETDEV_TX_OK; 479 } 480 481 static void mana_get_stats64(struct net_device *ndev, 482 struct rtnl_link_stats64 *st) 483 { 484 struct mana_port_context *apc = netdev_priv(ndev); 485 unsigned int num_queues = apc->num_queues; 486 struct mana_stats_rx *rx_stats; 487 struct mana_stats_tx *tx_stats; 488 unsigned int start; 489 u64 packets, bytes; 490 int q; 491 492 if (!apc->port_is_up) 493 return; 494 495 netdev_stats_to_stats64(st, &ndev->stats); 496 497 for (q = 0; q < num_queues; q++) { 498 rx_stats = &apc->rxqs[q]->stats; 499 500 do { 501 start = u64_stats_fetch_begin(&rx_stats->syncp); 502 packets = rx_stats->packets; 503 bytes = rx_stats->bytes; 504 } while (u64_stats_fetch_retry(&rx_stats->syncp, start)); 505 506 st->rx_packets += packets; 507 st->rx_bytes += bytes; 508 } 509 510 for (q = 0; q < num_queues; q++) { 511 tx_stats = &apc->tx_qp[q].txq.stats; 512 513 do { 514 start = u64_stats_fetch_begin(&tx_stats->syncp); 515 packets = tx_stats->packets; 516 bytes = tx_stats->bytes; 517 } while (u64_stats_fetch_retry(&tx_stats->syncp, start)); 518 519 st->tx_packets += packets; 520 st->tx_bytes += bytes; 521 } 522 } 523 524 static int mana_get_tx_queue(struct net_device *ndev, struct sk_buff *skb, 525 int old_q) 526 { 527 struct mana_port_context *apc = netdev_priv(ndev); 528 u32 hash = skb_get_hash(skb); 529 struct sock *sk = skb->sk; 530 int txq; 531 532 txq = apc->indir_table[hash & (apc->indir_table_sz - 1)]; 533 534 if (txq != old_q && sk && sk_fullsock(sk) && 535 rcu_access_pointer(sk->sk_dst_cache)) 536 sk_tx_queue_set(sk, txq); 537 538 return txq; 539 } 540 541 static u16 mana_select_queue(struct net_device *ndev, struct sk_buff *skb, 542 struct net_device *sb_dev) 543 { 544 int txq; 545 546 if (ndev->real_num_tx_queues == 1) 547 return 0; 548 549 txq = sk_tx_queue_get(skb->sk); 550 551 if (txq < 0 || skb->ooo_okay || txq >= ndev->real_num_tx_queues) { 552 if (skb_rx_queue_recorded(skb)) 553 txq = skb_get_rx_queue(skb); 554 else 555 txq = mana_get_tx_queue(ndev, skb, txq); 556 } 557 558 return txq; 559 } 560 561 /* Release pre-allocated RX buffers */ 562 void mana_pre_dealloc_rxbufs(struct mana_port_context *mpc) 563 { 564 struct device *dev; 565 int i; 566 567 dev = mpc->ac->gdma_dev->gdma_context->dev; 568 569 if (!mpc->rxbufs_pre) 570 goto out1; 571 572 if (!mpc->das_pre) 573 goto out2; 574 575 while (mpc->rxbpre_total) { 576 i = --mpc->rxbpre_total; 577 dma_unmap_single(dev, mpc->das_pre[i], mpc->rxbpre_datasize, 578 DMA_FROM_DEVICE); 579 put_page(virt_to_head_page(mpc->rxbufs_pre[i])); 580 } 581 582 kfree(mpc->das_pre); 583 mpc->das_pre = NULL; 584 585 out2: 586 kfree(mpc->rxbufs_pre); 587 mpc->rxbufs_pre = NULL; 588 589 out1: 590 mpc->rxbpre_datasize = 0; 591 mpc->rxbpre_alloc_size = 0; 592 mpc->rxbpre_headroom = 0; 593 } 594 595 /* Get a buffer from the pre-allocated RX buffers */ 596 static void *mana_get_rxbuf_pre(struct mana_rxq *rxq, dma_addr_t *da) 597 { 598 struct net_device *ndev = rxq->ndev; 599 struct mana_port_context *mpc; 600 void *va; 601 602 mpc = netdev_priv(ndev); 603 604 if (!mpc->rxbufs_pre || !mpc->das_pre || !mpc->rxbpre_total) { 605 netdev_err(ndev, "No RX pre-allocated bufs\n"); 606 return NULL; 607 } 608 609 /* Check sizes to catch unexpected coding error */ 610 if (mpc->rxbpre_datasize != rxq->datasize) { 611 netdev_err(ndev, "rxbpre_datasize mismatch: %u: %u\n", 612 mpc->rxbpre_datasize, rxq->datasize); 613 return NULL; 614 } 615 616 if (mpc->rxbpre_alloc_size != rxq->alloc_size) { 617 netdev_err(ndev, "rxbpre_alloc_size mismatch: %u: %u\n", 618 mpc->rxbpre_alloc_size, rxq->alloc_size); 619 return NULL; 620 } 621 622 if (mpc->rxbpre_headroom != rxq->headroom) { 623 netdev_err(ndev, "rxbpre_headroom mismatch: %u: %u\n", 624 mpc->rxbpre_headroom, rxq->headroom); 625 return NULL; 626 } 627 628 mpc->rxbpre_total--; 629 630 *da = mpc->das_pre[mpc->rxbpre_total]; 631 va = mpc->rxbufs_pre[mpc->rxbpre_total]; 632 mpc->rxbufs_pre[mpc->rxbpre_total] = NULL; 633 634 /* Deallocate the array after all buffers are gone */ 635 if (!mpc->rxbpre_total) 636 mana_pre_dealloc_rxbufs(mpc); 637 638 return va; 639 } 640 641 /* Get RX buffer's data size, alloc size, XDP headroom based on MTU */ 642 static void mana_get_rxbuf_cfg(struct mana_port_context *apc, 643 int mtu, u32 *datasize, u32 *alloc_size, 644 u32 *headroom, u32 *frag_count) 645 { 646 u32 len, buf_size; 647 648 /* Calculate datasize first (consistent across all cases) */ 649 *datasize = mtu + ETH_HLEN; 650 651 /* For xdp and jumbo frames make sure only one packet fits per page */ 652 if (mtu + MANA_RXBUF_PAD > PAGE_SIZE / 2 || mana_xdp_get(apc)) { 653 if (mana_xdp_get(apc)) { 654 *headroom = XDP_PACKET_HEADROOM; 655 *alloc_size = PAGE_SIZE; 656 } else { 657 *headroom = 0; /* no support for XDP */ 658 *alloc_size = SKB_DATA_ALIGN(mtu + MANA_RXBUF_PAD + 659 *headroom); 660 } 661 662 *frag_count = 1; 663 return; 664 } 665 666 /* Standard MTU case - optimize for multiple packets per page */ 667 *headroom = 0; 668 669 /* Calculate base buffer size needed */ 670 len = SKB_DATA_ALIGN(mtu + MANA_RXBUF_PAD + *headroom); 671 buf_size = ALIGN(len, MANA_RX_FRAG_ALIGNMENT); 672 673 /* Calculate how many packets can fit in a page */ 674 *frag_count = PAGE_SIZE / buf_size; 675 *alloc_size = buf_size; 676 } 677 678 int mana_pre_alloc_rxbufs(struct mana_port_context *mpc, int new_mtu, int num_queues) 679 { 680 struct device *dev; 681 struct page *page; 682 dma_addr_t da; 683 int num_rxb; 684 void *va; 685 int i; 686 687 mana_get_rxbuf_cfg(mpc, new_mtu, &mpc->rxbpre_datasize, 688 &mpc->rxbpre_alloc_size, &mpc->rxbpre_headroom, 689 &mpc->rxbpre_frag_count); 690 691 dev = mpc->ac->gdma_dev->gdma_context->dev; 692 693 num_rxb = num_queues * mpc->rx_queue_size; 694 695 WARN(mpc->rxbufs_pre, "mana rxbufs_pre exists\n"); 696 mpc->rxbufs_pre = kmalloc_array(num_rxb, sizeof(void *), GFP_KERNEL); 697 if (!mpc->rxbufs_pre) 698 goto error; 699 700 mpc->das_pre = kmalloc_array(num_rxb, sizeof(dma_addr_t), GFP_KERNEL); 701 if (!mpc->das_pre) 702 goto error; 703 704 mpc->rxbpre_total = 0; 705 706 for (i = 0; i < num_rxb; i++) { 707 page = dev_alloc_pages(get_order(mpc->rxbpre_alloc_size)); 708 if (!page) 709 goto error; 710 711 va = page_to_virt(page); 712 713 da = dma_map_single(dev, va + mpc->rxbpre_headroom, 714 mpc->rxbpre_datasize, DMA_FROM_DEVICE); 715 if (dma_mapping_error(dev, da)) { 716 put_page(page); 717 goto error; 718 } 719 720 mpc->rxbufs_pre[i] = va; 721 mpc->das_pre[i] = da; 722 mpc->rxbpre_total = i + 1; 723 } 724 725 return 0; 726 727 error: 728 netdev_err(mpc->ndev, "Failed to pre-allocate RX buffers for %d queues\n", num_queues); 729 mana_pre_dealloc_rxbufs(mpc); 730 return -ENOMEM; 731 } 732 733 static int mana_change_mtu(struct net_device *ndev, int new_mtu) 734 { 735 struct mana_port_context *mpc = netdev_priv(ndev); 736 unsigned int old_mtu = ndev->mtu; 737 int err; 738 739 /* Pre-allocate buffers to prevent failure in mana_attach later */ 740 err = mana_pre_alloc_rxbufs(mpc, new_mtu, mpc->num_queues); 741 if (err) { 742 netdev_err(ndev, "Insufficient memory for new MTU\n"); 743 return err; 744 } 745 746 err = mana_detach(ndev, false); 747 if (err) { 748 netdev_err(ndev, "mana_detach failed: %d\n", err); 749 goto out; 750 } 751 752 WRITE_ONCE(ndev->mtu, new_mtu); 753 754 err = mana_attach(ndev); 755 if (err) { 756 netdev_err(ndev, "mana_attach failed: %d\n", err); 757 WRITE_ONCE(ndev->mtu, old_mtu); 758 } 759 760 out: 761 mana_pre_dealloc_rxbufs(mpc); 762 return err; 763 } 764 765 static int mana_shaper_set(struct net_shaper_binding *binding, 766 const struct net_shaper *shaper, 767 struct netlink_ext_ack *extack) 768 { 769 struct mana_port_context *apc = netdev_priv(binding->netdev); 770 u32 old_speed, rate; 771 int err; 772 773 if (shaper->handle.scope != NET_SHAPER_SCOPE_NETDEV) { 774 NL_SET_ERR_MSG_MOD(extack, "net shaper scope should be netdev"); 775 return -EINVAL; 776 } 777 778 if (apc->handle.id && shaper->handle.id != apc->handle.id) { 779 NL_SET_ERR_MSG_MOD(extack, "Cannot create multiple shapers"); 780 return -EOPNOTSUPP; 781 } 782 783 if (!shaper->bw_max || (shaper->bw_max % 100000000)) { 784 NL_SET_ERR_MSG_MOD(extack, "Please use multiples of 100Mbps for bandwidth"); 785 return -EINVAL; 786 } 787 788 rate = div_u64(shaper->bw_max, 1000); /* Convert bps to Kbps */ 789 rate = div_u64(rate, 1000); /* Convert Kbps to Mbps */ 790 791 /* Get current speed */ 792 err = mana_query_link_cfg(apc); 793 old_speed = (err) ? SPEED_UNKNOWN : apc->speed; 794 795 if (!err) { 796 err = mana_set_bw_clamp(apc, rate, TRI_STATE_TRUE); 797 apc->speed = (err) ? old_speed : rate; 798 apc->handle = (err) ? apc->handle : shaper->handle; 799 } 800 801 return err; 802 } 803 804 static int mana_shaper_del(struct net_shaper_binding *binding, 805 const struct net_shaper_handle *handle, 806 struct netlink_ext_ack *extack) 807 { 808 struct mana_port_context *apc = netdev_priv(binding->netdev); 809 int err; 810 811 err = mana_set_bw_clamp(apc, 0, TRI_STATE_FALSE); 812 813 if (!err) { 814 /* Reset mana port context parameters */ 815 apc->handle.id = 0; 816 apc->handle.scope = NET_SHAPER_SCOPE_UNSPEC; 817 apc->speed = 0; 818 } 819 820 return err; 821 } 822 823 static void mana_shaper_cap(struct net_shaper_binding *binding, 824 enum net_shaper_scope scope, 825 unsigned long *flags) 826 { 827 *flags = BIT(NET_SHAPER_A_CAPS_SUPPORT_BW_MAX) | 828 BIT(NET_SHAPER_A_CAPS_SUPPORT_METRIC_BPS); 829 } 830 831 static const struct net_shaper_ops mana_shaper_ops = { 832 .set = mana_shaper_set, 833 .delete = mana_shaper_del, 834 .capabilities = mana_shaper_cap, 835 }; 836 837 static const struct net_device_ops mana_devops = { 838 .ndo_open = mana_open, 839 .ndo_stop = mana_close, 840 .ndo_select_queue = mana_select_queue, 841 .ndo_start_xmit = mana_start_xmit, 842 .ndo_validate_addr = eth_validate_addr, 843 .ndo_get_stats64 = mana_get_stats64, 844 .ndo_bpf = mana_bpf, 845 .ndo_xdp_xmit = mana_xdp_xmit, 846 .ndo_change_mtu = mana_change_mtu, 847 .net_shaper_ops = &mana_shaper_ops, 848 }; 849 850 static void mana_cleanup_port_context(struct mana_port_context *apc) 851 { 852 /* 853 * make sure subsequent cleanup attempts don't end up removing already 854 * cleaned dentry pointer 855 */ 856 debugfs_remove(apc->mana_port_debugfs); 857 apc->mana_port_debugfs = NULL; 858 kfree(apc->rxqs); 859 apc->rxqs = NULL; 860 } 861 862 static void mana_cleanup_indir_table(struct mana_port_context *apc) 863 { 864 apc->indir_table_sz = 0; 865 kfree(apc->indir_table); 866 kfree(apc->rxobj_table); 867 } 868 869 static int mana_init_port_context(struct mana_port_context *apc) 870 { 871 apc->rxqs = kcalloc(apc->num_queues, sizeof(struct mana_rxq *), 872 GFP_KERNEL); 873 874 return !apc->rxqs ? -ENOMEM : 0; 875 } 876 877 static int mana_send_request(struct mana_context *ac, void *in_buf, 878 u32 in_len, void *out_buf, u32 out_len) 879 { 880 struct gdma_context *gc = ac->gdma_dev->gdma_context; 881 struct gdma_resp_hdr *resp = out_buf; 882 struct gdma_req_hdr *req = in_buf; 883 struct device *dev = gc->dev; 884 static atomic_t activity_id; 885 int err; 886 887 req->dev_id = gc->mana.dev_id; 888 req->activity_id = atomic_inc_return(&activity_id); 889 890 err = mana_gd_send_request(gc, in_len, in_buf, out_len, 891 out_buf); 892 if (err || resp->status) { 893 if (err == -EOPNOTSUPP) 894 return err; 895 896 if (req->req.msg_type != MANA_QUERY_PHY_STAT && 897 mana_need_log(gc, err)) 898 dev_err(dev, "Failed to send mana message: %d, 0x%x\n", 899 err, resp->status); 900 return err ? err : -EPROTO; 901 } 902 903 if (req->dev_id.as_uint32 != resp->dev_id.as_uint32 || 904 req->activity_id != resp->activity_id) { 905 dev_err(dev, "Unexpected mana message response: %x,%x,%x,%x\n", 906 req->dev_id.as_uint32, resp->dev_id.as_uint32, 907 req->activity_id, resp->activity_id); 908 return -EPROTO; 909 } 910 911 return 0; 912 } 913 914 static int mana_verify_resp_hdr(const struct gdma_resp_hdr *resp_hdr, 915 const enum mana_command_code expected_code, 916 const u32 min_size) 917 { 918 if (resp_hdr->response.msg_type != expected_code) 919 return -EPROTO; 920 921 if (resp_hdr->response.msg_version < GDMA_MESSAGE_V1) 922 return -EPROTO; 923 924 if (resp_hdr->response.msg_size < min_size) 925 return -EPROTO; 926 927 return 0; 928 } 929 930 static int mana_pf_register_hw_vport(struct mana_port_context *apc) 931 { 932 struct mana_register_hw_vport_resp resp = {}; 933 struct mana_register_hw_vport_req req = {}; 934 int err; 935 936 mana_gd_init_req_hdr(&req.hdr, MANA_REGISTER_HW_PORT, 937 sizeof(req), sizeof(resp)); 938 req.attached_gfid = 1; 939 req.is_pf_default_vport = 1; 940 req.allow_all_ether_types = 1; 941 942 err = mana_send_request(apc->ac, &req, sizeof(req), &resp, 943 sizeof(resp)); 944 if (err) { 945 netdev_err(apc->ndev, "Failed to register hw vPort: %d\n", err); 946 return err; 947 } 948 949 err = mana_verify_resp_hdr(&resp.hdr, MANA_REGISTER_HW_PORT, 950 sizeof(resp)); 951 if (err || resp.hdr.status) { 952 netdev_err(apc->ndev, "Failed to register hw vPort: %d, 0x%x\n", 953 err, resp.hdr.status); 954 return err ? err : -EPROTO; 955 } 956 957 apc->port_handle = resp.hw_vport_handle; 958 return 0; 959 } 960 961 static void mana_pf_deregister_hw_vport(struct mana_port_context *apc) 962 { 963 struct mana_deregister_hw_vport_resp resp = {}; 964 struct mana_deregister_hw_vport_req req = {}; 965 int err; 966 967 mana_gd_init_req_hdr(&req.hdr, MANA_DEREGISTER_HW_PORT, 968 sizeof(req), sizeof(resp)); 969 req.hw_vport_handle = apc->port_handle; 970 971 err = mana_send_request(apc->ac, &req, sizeof(req), &resp, 972 sizeof(resp)); 973 if (err) { 974 if (mana_en_need_log(apc, err)) 975 netdev_err(apc->ndev, "Failed to unregister hw vPort: %d\n", 976 err); 977 978 return; 979 } 980 981 err = mana_verify_resp_hdr(&resp.hdr, MANA_DEREGISTER_HW_PORT, 982 sizeof(resp)); 983 if (err || resp.hdr.status) 984 netdev_err(apc->ndev, 985 "Failed to deregister hw vPort: %d, 0x%x\n", 986 err, resp.hdr.status); 987 } 988 989 static int mana_pf_register_filter(struct mana_port_context *apc) 990 { 991 struct mana_register_filter_resp resp = {}; 992 struct mana_register_filter_req req = {}; 993 int err; 994 995 mana_gd_init_req_hdr(&req.hdr, MANA_REGISTER_FILTER, 996 sizeof(req), sizeof(resp)); 997 req.vport = apc->port_handle; 998 memcpy(req.mac_addr, apc->mac_addr, ETH_ALEN); 999 1000 err = mana_send_request(apc->ac, &req, sizeof(req), &resp, 1001 sizeof(resp)); 1002 if (err) { 1003 netdev_err(apc->ndev, "Failed to register filter: %d\n", err); 1004 return err; 1005 } 1006 1007 err = mana_verify_resp_hdr(&resp.hdr, MANA_REGISTER_FILTER, 1008 sizeof(resp)); 1009 if (err || resp.hdr.status) { 1010 netdev_err(apc->ndev, "Failed to register filter: %d, 0x%x\n", 1011 err, resp.hdr.status); 1012 return err ? err : -EPROTO; 1013 } 1014 1015 apc->pf_filter_handle = resp.filter_handle; 1016 return 0; 1017 } 1018 1019 static void mana_pf_deregister_filter(struct mana_port_context *apc) 1020 { 1021 struct mana_deregister_filter_resp resp = {}; 1022 struct mana_deregister_filter_req req = {}; 1023 int err; 1024 1025 mana_gd_init_req_hdr(&req.hdr, MANA_DEREGISTER_FILTER, 1026 sizeof(req), sizeof(resp)); 1027 req.filter_handle = apc->pf_filter_handle; 1028 1029 err = mana_send_request(apc->ac, &req, sizeof(req), &resp, 1030 sizeof(resp)); 1031 if (err) { 1032 if (mana_en_need_log(apc, err)) 1033 netdev_err(apc->ndev, "Failed to unregister filter: %d\n", 1034 err); 1035 1036 return; 1037 } 1038 1039 err = mana_verify_resp_hdr(&resp.hdr, MANA_DEREGISTER_FILTER, 1040 sizeof(resp)); 1041 if (err || resp.hdr.status) 1042 netdev_err(apc->ndev, 1043 "Failed to deregister filter: %d, 0x%x\n", 1044 err, resp.hdr.status); 1045 } 1046 1047 static int mana_query_device_cfg(struct mana_context *ac, u32 proto_major_ver, 1048 u32 proto_minor_ver, u32 proto_micro_ver, 1049 u16 *max_num_vports, u8 *bm_hostmode) 1050 { 1051 struct gdma_context *gc = ac->gdma_dev->gdma_context; 1052 struct mana_query_device_cfg_resp resp = {}; 1053 struct mana_query_device_cfg_req req = {}; 1054 struct device *dev = gc->dev; 1055 int err = 0; 1056 1057 mana_gd_init_req_hdr(&req.hdr, MANA_QUERY_DEV_CONFIG, 1058 sizeof(req), sizeof(resp)); 1059 1060 req.hdr.resp.msg_version = GDMA_MESSAGE_V3; 1061 1062 req.proto_major_ver = proto_major_ver; 1063 req.proto_minor_ver = proto_minor_ver; 1064 req.proto_micro_ver = proto_micro_ver; 1065 1066 err = mana_send_request(ac, &req, sizeof(req), &resp, sizeof(resp)); 1067 if (err) { 1068 dev_err(dev, "Failed to query config: %d", err); 1069 return err; 1070 } 1071 1072 err = mana_verify_resp_hdr(&resp.hdr, MANA_QUERY_DEV_CONFIG, 1073 sizeof(resp)); 1074 if (err || resp.hdr.status) { 1075 dev_err(dev, "Invalid query result: %d, 0x%x\n", err, 1076 resp.hdr.status); 1077 if (!err) 1078 err = -EPROTO; 1079 return err; 1080 } 1081 1082 *max_num_vports = resp.max_num_vports; 1083 1084 if (resp.hdr.response.msg_version >= GDMA_MESSAGE_V2) 1085 gc->adapter_mtu = resp.adapter_mtu; 1086 else 1087 gc->adapter_mtu = ETH_FRAME_LEN; 1088 1089 if (resp.hdr.response.msg_version >= GDMA_MESSAGE_V3) 1090 *bm_hostmode = resp.bm_hostmode; 1091 else 1092 *bm_hostmode = 0; 1093 1094 debugfs_create_u16("adapter-MTU", 0400, gc->mana_pci_debugfs, &gc->adapter_mtu); 1095 1096 return 0; 1097 } 1098 1099 static int mana_query_vport_cfg(struct mana_port_context *apc, u32 vport_index, 1100 u32 *max_sq, u32 *max_rq, u32 *num_indir_entry) 1101 { 1102 struct mana_query_vport_cfg_resp resp = {}; 1103 struct mana_query_vport_cfg_req req = {}; 1104 int err; 1105 1106 mana_gd_init_req_hdr(&req.hdr, MANA_QUERY_VPORT_CONFIG, 1107 sizeof(req), sizeof(resp)); 1108 1109 req.vport_index = vport_index; 1110 1111 err = mana_send_request(apc->ac, &req, sizeof(req), &resp, 1112 sizeof(resp)); 1113 if (err) 1114 return err; 1115 1116 err = mana_verify_resp_hdr(&resp.hdr, MANA_QUERY_VPORT_CONFIG, 1117 sizeof(resp)); 1118 if (err) 1119 return err; 1120 1121 if (resp.hdr.status) 1122 return -EPROTO; 1123 1124 *max_sq = resp.max_num_sq; 1125 *max_rq = resp.max_num_rq; 1126 if (resp.num_indirection_ent > 0 && 1127 resp.num_indirection_ent <= MANA_INDIRECT_TABLE_MAX_SIZE && 1128 is_power_of_2(resp.num_indirection_ent)) { 1129 *num_indir_entry = resp.num_indirection_ent; 1130 } else { 1131 netdev_warn(apc->ndev, 1132 "Setting indirection table size to default %d for vPort %d\n", 1133 MANA_INDIRECT_TABLE_DEF_SIZE, apc->port_idx); 1134 *num_indir_entry = MANA_INDIRECT_TABLE_DEF_SIZE; 1135 } 1136 1137 apc->port_handle = resp.vport; 1138 ether_addr_copy(apc->mac_addr, resp.mac_addr); 1139 1140 return 0; 1141 } 1142 1143 void mana_uncfg_vport(struct mana_port_context *apc) 1144 { 1145 mutex_lock(&apc->vport_mutex); 1146 apc->vport_use_count--; 1147 WARN_ON(apc->vport_use_count < 0); 1148 mutex_unlock(&apc->vport_mutex); 1149 } 1150 EXPORT_SYMBOL_NS(mana_uncfg_vport, "NET_MANA"); 1151 1152 int mana_cfg_vport(struct mana_port_context *apc, u32 protection_dom_id, 1153 u32 doorbell_pg_id) 1154 { 1155 struct mana_config_vport_resp resp = {}; 1156 struct mana_config_vport_req req = {}; 1157 int err; 1158 1159 /* This function is used to program the Ethernet port in the hardware 1160 * table. It can be called from the Ethernet driver or the RDMA driver. 1161 * 1162 * For Ethernet usage, the hardware supports only one active user on a 1163 * physical port. The driver checks on the port usage before programming 1164 * the hardware when creating the RAW QP (RDMA driver) or exposing the 1165 * device to kernel NET layer (Ethernet driver). 1166 * 1167 * Because the RDMA driver doesn't know in advance which QP type the 1168 * user will create, it exposes the device with all its ports. The user 1169 * may not be able to create RAW QP on a port if this port is already 1170 * in used by the Ethernet driver from the kernel. 1171 * 1172 * This physical port limitation only applies to the RAW QP. For RC QP, 1173 * the hardware doesn't have this limitation. The user can create RC 1174 * QPs on a physical port up to the hardware limits independent of the 1175 * Ethernet usage on the same port. 1176 */ 1177 mutex_lock(&apc->vport_mutex); 1178 if (apc->vport_use_count > 0) { 1179 mutex_unlock(&apc->vport_mutex); 1180 return -EBUSY; 1181 } 1182 apc->vport_use_count++; 1183 mutex_unlock(&apc->vport_mutex); 1184 1185 mana_gd_init_req_hdr(&req.hdr, MANA_CONFIG_VPORT_TX, 1186 sizeof(req), sizeof(resp)); 1187 req.vport = apc->port_handle; 1188 req.pdid = protection_dom_id; 1189 req.doorbell_pageid = doorbell_pg_id; 1190 1191 err = mana_send_request(apc->ac, &req, sizeof(req), &resp, 1192 sizeof(resp)); 1193 if (err) { 1194 netdev_err(apc->ndev, "Failed to configure vPort: %d\n", err); 1195 goto out; 1196 } 1197 1198 err = mana_verify_resp_hdr(&resp.hdr, MANA_CONFIG_VPORT_TX, 1199 sizeof(resp)); 1200 if (err || resp.hdr.status) { 1201 netdev_err(apc->ndev, "Failed to configure vPort: %d, 0x%x\n", 1202 err, resp.hdr.status); 1203 if (!err) 1204 err = -EPROTO; 1205 1206 goto out; 1207 } 1208 1209 apc->tx_shortform_allowed = resp.short_form_allowed; 1210 apc->tx_vp_offset = resp.tx_vport_offset; 1211 1212 netdev_info(apc->ndev, "Configured vPort %llu PD %u DB %u\n", 1213 apc->port_handle, protection_dom_id, doorbell_pg_id); 1214 out: 1215 if (err) 1216 mana_uncfg_vport(apc); 1217 1218 return err; 1219 } 1220 EXPORT_SYMBOL_NS(mana_cfg_vport, "NET_MANA"); 1221 1222 static int mana_cfg_vport_steering(struct mana_port_context *apc, 1223 enum TRI_STATE rx, 1224 bool update_default_rxobj, bool update_key, 1225 bool update_tab) 1226 { 1227 struct mana_cfg_rx_steer_req_v2 *req; 1228 struct mana_cfg_rx_steer_resp resp = {}; 1229 struct net_device *ndev = apc->ndev; 1230 u32 req_buf_size; 1231 int err; 1232 1233 req_buf_size = struct_size(req, indir_tab, apc->indir_table_sz); 1234 req = kzalloc(req_buf_size, GFP_KERNEL); 1235 if (!req) 1236 return -ENOMEM; 1237 1238 mana_gd_init_req_hdr(&req->hdr, MANA_CONFIG_VPORT_RX, req_buf_size, 1239 sizeof(resp)); 1240 1241 req->hdr.req.msg_version = GDMA_MESSAGE_V2; 1242 1243 req->vport = apc->port_handle; 1244 req->num_indir_entries = apc->indir_table_sz; 1245 req->indir_tab_offset = offsetof(struct mana_cfg_rx_steer_req_v2, 1246 indir_tab); 1247 req->rx_enable = rx; 1248 req->rss_enable = apc->rss_state; 1249 req->update_default_rxobj = update_default_rxobj; 1250 req->update_hashkey = update_key; 1251 req->update_indir_tab = update_tab; 1252 req->default_rxobj = apc->default_rxobj; 1253 req->cqe_coalescing_enable = 0; 1254 1255 if (update_key) 1256 memcpy(&req->hashkey, apc->hashkey, MANA_HASH_KEY_SIZE); 1257 1258 if (update_tab) 1259 memcpy(req->indir_tab, apc->rxobj_table, 1260 flex_array_size(req, indir_tab, req->num_indir_entries)); 1261 1262 err = mana_send_request(apc->ac, req, req_buf_size, &resp, 1263 sizeof(resp)); 1264 if (err) { 1265 if (mana_en_need_log(apc, err)) 1266 netdev_err(ndev, "Failed to configure vPort RX: %d\n", err); 1267 1268 goto out; 1269 } 1270 1271 err = mana_verify_resp_hdr(&resp.hdr, MANA_CONFIG_VPORT_RX, 1272 sizeof(resp)); 1273 if (err) { 1274 netdev_err(ndev, "vPort RX configuration failed: %d\n", err); 1275 goto out; 1276 } 1277 1278 if (resp.hdr.status) { 1279 netdev_err(ndev, "vPort RX configuration failed: 0x%x\n", 1280 resp.hdr.status); 1281 err = -EPROTO; 1282 } 1283 1284 netdev_info(ndev, "Configured steering vPort %llu entries %u\n", 1285 apc->port_handle, apc->indir_table_sz); 1286 out: 1287 kfree(req); 1288 return err; 1289 } 1290 1291 int mana_query_link_cfg(struct mana_port_context *apc) 1292 { 1293 struct net_device *ndev = apc->ndev; 1294 struct mana_query_link_config_resp resp = {}; 1295 struct mana_query_link_config_req req = {}; 1296 int err; 1297 1298 mana_gd_init_req_hdr(&req.hdr, MANA_QUERY_LINK_CONFIG, 1299 sizeof(req), sizeof(resp)); 1300 1301 req.vport = apc->port_handle; 1302 req.hdr.resp.msg_version = GDMA_MESSAGE_V2; 1303 1304 err = mana_send_request(apc->ac, &req, sizeof(req), &resp, 1305 sizeof(resp)); 1306 1307 if (err) { 1308 if (err == -EOPNOTSUPP) { 1309 netdev_info_once(ndev, "MANA_QUERY_LINK_CONFIG not supported\n"); 1310 return err; 1311 } 1312 netdev_err(ndev, "Failed to query link config: %d\n", err); 1313 return err; 1314 } 1315 1316 err = mana_verify_resp_hdr(&resp.hdr, MANA_QUERY_LINK_CONFIG, 1317 sizeof(resp)); 1318 1319 if (err || resp.hdr.status) { 1320 netdev_err(ndev, "Failed to query link config: %d, 0x%x\n", err, 1321 resp.hdr.status); 1322 if (!err) 1323 err = -EOPNOTSUPP; 1324 return err; 1325 } 1326 1327 if (resp.qos_unconfigured) { 1328 err = -EINVAL; 1329 return err; 1330 } 1331 apc->speed = resp.link_speed_mbps; 1332 apc->max_speed = resp.qos_speed_mbps; 1333 return 0; 1334 } 1335 1336 int mana_set_bw_clamp(struct mana_port_context *apc, u32 speed, 1337 int enable_clamping) 1338 { 1339 struct mana_set_bw_clamp_resp resp = {}; 1340 struct mana_set_bw_clamp_req req = {}; 1341 struct net_device *ndev = apc->ndev; 1342 int err; 1343 1344 mana_gd_init_req_hdr(&req.hdr, MANA_SET_BW_CLAMP, 1345 sizeof(req), sizeof(resp)); 1346 req.vport = apc->port_handle; 1347 req.link_speed_mbps = speed; 1348 req.enable_clamping = enable_clamping; 1349 1350 err = mana_send_request(apc->ac, &req, sizeof(req), &resp, 1351 sizeof(resp)); 1352 1353 if (err) { 1354 if (err == -EOPNOTSUPP) { 1355 netdev_info_once(ndev, "MANA_SET_BW_CLAMP not supported\n"); 1356 return err; 1357 } 1358 netdev_err(ndev, "Failed to set bandwidth clamp for speed %u, err = %d", 1359 speed, err); 1360 return err; 1361 } 1362 1363 err = mana_verify_resp_hdr(&resp.hdr, MANA_SET_BW_CLAMP, 1364 sizeof(resp)); 1365 1366 if (err || resp.hdr.status) { 1367 netdev_err(ndev, "Failed to set bandwidth clamp: %d, 0x%x\n", err, 1368 resp.hdr.status); 1369 if (!err) 1370 err = -EOPNOTSUPP; 1371 return err; 1372 } 1373 1374 if (resp.qos_unconfigured) 1375 netdev_info(ndev, "QoS is unconfigured\n"); 1376 1377 return 0; 1378 } 1379 1380 int mana_create_wq_obj(struct mana_port_context *apc, 1381 mana_handle_t vport, 1382 u32 wq_type, struct mana_obj_spec *wq_spec, 1383 struct mana_obj_spec *cq_spec, 1384 mana_handle_t *wq_obj) 1385 { 1386 struct mana_create_wqobj_resp resp = {}; 1387 struct mana_create_wqobj_req req = {}; 1388 struct net_device *ndev = apc->ndev; 1389 int err; 1390 1391 mana_gd_init_req_hdr(&req.hdr, MANA_CREATE_WQ_OBJ, 1392 sizeof(req), sizeof(resp)); 1393 req.vport = vport; 1394 req.wq_type = wq_type; 1395 req.wq_gdma_region = wq_spec->gdma_region; 1396 req.cq_gdma_region = cq_spec->gdma_region; 1397 req.wq_size = wq_spec->queue_size; 1398 req.cq_size = cq_spec->queue_size; 1399 req.cq_moderation_ctx_id = cq_spec->modr_ctx_id; 1400 req.cq_parent_qid = cq_spec->attached_eq; 1401 1402 err = mana_send_request(apc->ac, &req, sizeof(req), &resp, 1403 sizeof(resp)); 1404 if (err) { 1405 netdev_err(ndev, "Failed to create WQ object: %d\n", err); 1406 goto out; 1407 } 1408 1409 err = mana_verify_resp_hdr(&resp.hdr, MANA_CREATE_WQ_OBJ, 1410 sizeof(resp)); 1411 if (err || resp.hdr.status) { 1412 netdev_err(ndev, "Failed to create WQ object: %d, 0x%x\n", err, 1413 resp.hdr.status); 1414 if (!err) 1415 err = -EPROTO; 1416 goto out; 1417 } 1418 1419 if (resp.wq_obj == INVALID_MANA_HANDLE) { 1420 netdev_err(ndev, "Got an invalid WQ object handle\n"); 1421 err = -EPROTO; 1422 goto out; 1423 } 1424 1425 *wq_obj = resp.wq_obj; 1426 wq_spec->queue_index = resp.wq_id; 1427 cq_spec->queue_index = resp.cq_id; 1428 1429 return 0; 1430 out: 1431 return err; 1432 } 1433 EXPORT_SYMBOL_NS(mana_create_wq_obj, "NET_MANA"); 1434 1435 void mana_destroy_wq_obj(struct mana_port_context *apc, u32 wq_type, 1436 mana_handle_t wq_obj) 1437 { 1438 struct mana_destroy_wqobj_resp resp = {}; 1439 struct mana_destroy_wqobj_req req = {}; 1440 struct net_device *ndev = apc->ndev; 1441 int err; 1442 1443 mana_gd_init_req_hdr(&req.hdr, MANA_DESTROY_WQ_OBJ, 1444 sizeof(req), sizeof(resp)); 1445 req.wq_type = wq_type; 1446 req.wq_obj_handle = wq_obj; 1447 1448 err = mana_send_request(apc->ac, &req, sizeof(req), &resp, 1449 sizeof(resp)); 1450 if (err) { 1451 if (mana_en_need_log(apc, err)) 1452 netdev_err(ndev, "Failed to destroy WQ object: %d\n", err); 1453 1454 return; 1455 } 1456 1457 err = mana_verify_resp_hdr(&resp.hdr, MANA_DESTROY_WQ_OBJ, 1458 sizeof(resp)); 1459 if (err || resp.hdr.status) 1460 netdev_err(ndev, "Failed to destroy WQ object: %d, 0x%x\n", err, 1461 resp.hdr.status); 1462 } 1463 EXPORT_SYMBOL_NS(mana_destroy_wq_obj, "NET_MANA"); 1464 1465 static void mana_destroy_eq(struct mana_context *ac) 1466 { 1467 struct gdma_context *gc = ac->gdma_dev->gdma_context; 1468 struct gdma_queue *eq; 1469 int i; 1470 1471 if (!ac->eqs) 1472 return; 1473 1474 debugfs_remove_recursive(ac->mana_eqs_debugfs); 1475 ac->mana_eqs_debugfs = NULL; 1476 1477 for (i = 0; i < gc->max_num_queues; i++) { 1478 eq = ac->eqs[i].eq; 1479 if (!eq) 1480 continue; 1481 1482 mana_gd_destroy_queue(gc, eq); 1483 } 1484 1485 kfree(ac->eqs); 1486 ac->eqs = NULL; 1487 } 1488 1489 static void mana_create_eq_debugfs(struct mana_context *ac, int i) 1490 { 1491 struct mana_eq eq = ac->eqs[i]; 1492 char eqnum[32]; 1493 1494 sprintf(eqnum, "eq%d", i); 1495 eq.mana_eq_debugfs = debugfs_create_dir(eqnum, ac->mana_eqs_debugfs); 1496 debugfs_create_u32("head", 0400, eq.mana_eq_debugfs, &eq.eq->head); 1497 debugfs_create_u32("tail", 0400, eq.mana_eq_debugfs, &eq.eq->tail); 1498 debugfs_create_file("eq_dump", 0400, eq.mana_eq_debugfs, eq.eq, &mana_dbg_q_fops); 1499 } 1500 1501 static int mana_create_eq(struct mana_context *ac) 1502 { 1503 struct gdma_dev *gd = ac->gdma_dev; 1504 struct gdma_context *gc = gd->gdma_context; 1505 struct gdma_queue_spec spec = {}; 1506 int err; 1507 int i; 1508 1509 ac->eqs = kcalloc(gc->max_num_queues, sizeof(struct mana_eq), 1510 GFP_KERNEL); 1511 if (!ac->eqs) 1512 return -ENOMEM; 1513 1514 spec.type = GDMA_EQ; 1515 spec.monitor_avl_buf = false; 1516 spec.queue_size = EQ_SIZE; 1517 spec.eq.callback = NULL; 1518 spec.eq.context = ac->eqs; 1519 spec.eq.log2_throttle_limit = LOG2_EQ_THROTTLE; 1520 1521 ac->mana_eqs_debugfs = debugfs_create_dir("EQs", gc->mana_pci_debugfs); 1522 1523 for (i = 0; i < gc->max_num_queues; i++) { 1524 spec.eq.msix_index = (i + 1) % gc->num_msix_usable; 1525 err = mana_gd_create_mana_eq(gd, &spec, &ac->eqs[i].eq); 1526 if (err) { 1527 dev_err(gc->dev, "Failed to create EQ %d : %d\n", i, err); 1528 goto out; 1529 } 1530 mana_create_eq_debugfs(ac, i); 1531 } 1532 1533 return 0; 1534 out: 1535 mana_destroy_eq(ac); 1536 return err; 1537 } 1538 1539 static int mana_fence_rq(struct mana_port_context *apc, struct mana_rxq *rxq) 1540 { 1541 struct mana_fence_rq_resp resp = {}; 1542 struct mana_fence_rq_req req = {}; 1543 int err; 1544 1545 init_completion(&rxq->fence_event); 1546 1547 mana_gd_init_req_hdr(&req.hdr, MANA_FENCE_RQ, 1548 sizeof(req), sizeof(resp)); 1549 req.wq_obj_handle = rxq->rxobj; 1550 1551 err = mana_send_request(apc->ac, &req, sizeof(req), &resp, 1552 sizeof(resp)); 1553 if (err) { 1554 netdev_err(apc->ndev, "Failed to fence RQ %u: %d\n", 1555 rxq->rxq_idx, err); 1556 return err; 1557 } 1558 1559 err = mana_verify_resp_hdr(&resp.hdr, MANA_FENCE_RQ, sizeof(resp)); 1560 if (err || resp.hdr.status) { 1561 netdev_err(apc->ndev, "Failed to fence RQ %u: %d, 0x%x\n", 1562 rxq->rxq_idx, err, resp.hdr.status); 1563 if (!err) 1564 err = -EPROTO; 1565 1566 return err; 1567 } 1568 1569 if (wait_for_completion_timeout(&rxq->fence_event, 10 * HZ) == 0) { 1570 netdev_err(apc->ndev, "Failed to fence RQ %u: timed out\n", 1571 rxq->rxq_idx); 1572 return -ETIMEDOUT; 1573 } 1574 1575 return 0; 1576 } 1577 1578 static void mana_fence_rqs(struct mana_port_context *apc) 1579 { 1580 unsigned int rxq_idx; 1581 struct mana_rxq *rxq; 1582 int err; 1583 1584 for (rxq_idx = 0; rxq_idx < apc->num_queues; rxq_idx++) { 1585 rxq = apc->rxqs[rxq_idx]; 1586 err = mana_fence_rq(apc, rxq); 1587 1588 /* In case of any error, use sleep instead. */ 1589 if (err) 1590 msleep(100); 1591 } 1592 } 1593 1594 static int mana_move_wq_tail(struct gdma_queue *wq, u32 num_units) 1595 { 1596 u32 used_space_old; 1597 u32 used_space_new; 1598 1599 used_space_old = wq->head - wq->tail; 1600 used_space_new = wq->head - (wq->tail + num_units); 1601 1602 if (WARN_ON_ONCE(used_space_new > used_space_old)) 1603 return -ERANGE; 1604 1605 wq->tail += num_units; 1606 return 0; 1607 } 1608 1609 static void mana_unmap_skb(struct sk_buff *skb, struct mana_port_context *apc) 1610 { 1611 struct mana_skb_head *ash = (struct mana_skb_head *)skb->head; 1612 struct gdma_context *gc = apc->ac->gdma_dev->gdma_context; 1613 struct device *dev = gc->dev; 1614 int hsg, i; 1615 1616 /* Number of SGEs of linear part */ 1617 hsg = (skb_is_gso(skb) && skb_headlen(skb) > ash->size[0]) ? 2 : 1; 1618 1619 for (i = 0; i < hsg; i++) 1620 dma_unmap_single(dev, ash->dma_handle[i], ash->size[i], 1621 DMA_TO_DEVICE); 1622 1623 for (i = hsg; i < skb_shinfo(skb)->nr_frags + hsg; i++) 1624 dma_unmap_page(dev, ash->dma_handle[i], ash->size[i], 1625 DMA_TO_DEVICE); 1626 } 1627 1628 static void mana_poll_tx_cq(struct mana_cq *cq) 1629 { 1630 struct gdma_comp *completions = cq->gdma_comp_buf; 1631 struct gdma_posted_wqe_info *wqe_info; 1632 unsigned int pkt_transmitted = 0; 1633 unsigned int wqe_unit_cnt = 0; 1634 struct mana_txq *txq = cq->txq; 1635 struct mana_port_context *apc; 1636 struct netdev_queue *net_txq; 1637 struct gdma_queue *gdma_wq; 1638 unsigned int avail_space; 1639 struct net_device *ndev; 1640 struct sk_buff *skb; 1641 bool txq_stopped; 1642 int comp_read; 1643 int i; 1644 1645 ndev = txq->ndev; 1646 apc = netdev_priv(ndev); 1647 1648 comp_read = mana_gd_poll_cq(cq->gdma_cq, completions, 1649 CQE_POLLING_BUFFER); 1650 1651 if (comp_read < 1) 1652 return; 1653 1654 for (i = 0; i < comp_read; i++) { 1655 struct mana_tx_comp_oob *cqe_oob; 1656 1657 if (WARN_ON_ONCE(!completions[i].is_sq)) 1658 return; 1659 1660 cqe_oob = (struct mana_tx_comp_oob *)completions[i].cqe_data; 1661 if (WARN_ON_ONCE(cqe_oob->cqe_hdr.client_type != 1662 MANA_CQE_COMPLETION)) 1663 return; 1664 1665 switch (cqe_oob->cqe_hdr.cqe_type) { 1666 case CQE_TX_OKAY: 1667 break; 1668 1669 case CQE_TX_SA_DROP: 1670 case CQE_TX_MTU_DROP: 1671 case CQE_TX_INVALID_OOB: 1672 case CQE_TX_INVALID_ETH_TYPE: 1673 case CQE_TX_HDR_PROCESSING_ERROR: 1674 case CQE_TX_VF_DISABLED: 1675 case CQE_TX_VPORT_IDX_OUT_OF_RANGE: 1676 case CQE_TX_VPORT_DISABLED: 1677 case CQE_TX_VLAN_TAGGING_VIOLATION: 1678 if (net_ratelimit()) 1679 netdev_err(ndev, "TX: CQE error %d\n", 1680 cqe_oob->cqe_hdr.cqe_type); 1681 1682 apc->eth_stats.tx_cqe_err++; 1683 break; 1684 1685 default: 1686 /* If the CQE type is unknown, log an error, 1687 * and still free the SKB, update tail, etc. 1688 */ 1689 if (net_ratelimit()) 1690 netdev_err(ndev, "TX: unknown CQE type %d\n", 1691 cqe_oob->cqe_hdr.cqe_type); 1692 1693 apc->eth_stats.tx_cqe_unknown_type++; 1694 break; 1695 } 1696 1697 if (WARN_ON_ONCE(txq->gdma_txq_id != completions[i].wq_num)) 1698 return; 1699 1700 skb = skb_dequeue(&txq->pending_skbs); 1701 if (WARN_ON_ONCE(!skb)) 1702 return; 1703 1704 wqe_info = (struct gdma_posted_wqe_info *)skb->cb; 1705 wqe_unit_cnt += wqe_info->wqe_size_in_bu; 1706 1707 mana_unmap_skb(skb, apc); 1708 1709 napi_consume_skb(skb, cq->budget); 1710 1711 pkt_transmitted++; 1712 } 1713 1714 if (WARN_ON_ONCE(wqe_unit_cnt == 0)) 1715 return; 1716 1717 mana_move_wq_tail(txq->gdma_sq, wqe_unit_cnt); 1718 1719 gdma_wq = txq->gdma_sq; 1720 avail_space = mana_gd_wq_avail_space(gdma_wq); 1721 1722 /* Ensure tail updated before checking q stop */ 1723 smp_mb(); 1724 1725 net_txq = txq->net_txq; 1726 txq_stopped = netif_tx_queue_stopped(net_txq); 1727 1728 /* Ensure checking txq_stopped before apc->port_is_up. */ 1729 smp_rmb(); 1730 1731 if (txq_stopped && apc->port_is_up && avail_space >= MAX_TX_WQE_SIZE) { 1732 netif_tx_wake_queue(net_txq); 1733 apc->eth_stats.wake_queue++; 1734 } 1735 1736 if (atomic_sub_return(pkt_transmitted, &txq->pending_sends) < 0) 1737 WARN_ON_ONCE(1); 1738 1739 cq->work_done = pkt_transmitted; 1740 } 1741 1742 static void mana_post_pkt_rxq(struct mana_rxq *rxq) 1743 { 1744 struct mana_recv_buf_oob *recv_buf_oob; 1745 u32 curr_index; 1746 int err; 1747 1748 curr_index = rxq->buf_index++; 1749 if (rxq->buf_index == rxq->num_rx_buf) 1750 rxq->buf_index = 0; 1751 1752 recv_buf_oob = &rxq->rx_oobs[curr_index]; 1753 1754 err = mana_gd_post_work_request(rxq->gdma_rq, &recv_buf_oob->wqe_req, 1755 &recv_buf_oob->wqe_inf); 1756 if (WARN_ON_ONCE(err)) 1757 return; 1758 1759 WARN_ON_ONCE(recv_buf_oob->wqe_inf.wqe_size_in_bu != 1); 1760 } 1761 1762 static struct sk_buff *mana_build_skb(struct mana_rxq *rxq, void *buf_va, 1763 uint pkt_len, struct xdp_buff *xdp) 1764 { 1765 struct sk_buff *skb = napi_build_skb(buf_va, rxq->alloc_size); 1766 1767 if (!skb) 1768 return NULL; 1769 1770 if (xdp->data_hard_start) { 1771 u32 metasize = xdp->data - xdp->data_meta; 1772 1773 skb_reserve(skb, xdp->data - xdp->data_hard_start); 1774 skb_put(skb, xdp->data_end - xdp->data); 1775 if (metasize) 1776 skb_metadata_set(skb, metasize); 1777 return skb; 1778 } 1779 1780 skb_reserve(skb, rxq->headroom); 1781 skb_put(skb, pkt_len); 1782 1783 return skb; 1784 } 1785 1786 static void mana_rx_skb(void *buf_va, bool from_pool, 1787 struct mana_rxcomp_oob *cqe, struct mana_rxq *rxq) 1788 { 1789 struct mana_stats_rx *rx_stats = &rxq->stats; 1790 struct net_device *ndev = rxq->ndev; 1791 uint pkt_len = cqe->ppi[0].pkt_len; 1792 u16 rxq_idx = rxq->rxq_idx; 1793 struct napi_struct *napi; 1794 struct xdp_buff xdp = {}; 1795 struct sk_buff *skb; 1796 u32 hash_value; 1797 u32 act; 1798 1799 rxq->rx_cq.work_done++; 1800 napi = &rxq->rx_cq.napi; 1801 1802 if (!buf_va) { 1803 ++ndev->stats.rx_dropped; 1804 return; 1805 } 1806 1807 act = mana_run_xdp(ndev, rxq, &xdp, buf_va, pkt_len); 1808 1809 if (act == XDP_REDIRECT && !rxq->xdp_rc) 1810 return; 1811 1812 if (act != XDP_PASS && act != XDP_TX) 1813 goto drop_xdp; 1814 1815 skb = mana_build_skb(rxq, buf_va, pkt_len, &xdp); 1816 1817 if (!skb) 1818 goto drop; 1819 1820 if (from_pool) 1821 skb_mark_for_recycle(skb); 1822 1823 skb->dev = napi->dev; 1824 1825 skb->protocol = eth_type_trans(skb, ndev); 1826 skb_checksum_none_assert(skb); 1827 skb_record_rx_queue(skb, rxq_idx); 1828 1829 if ((ndev->features & NETIF_F_RXCSUM) && cqe->rx_iphdr_csum_succeed) { 1830 if (cqe->rx_tcp_csum_succeed || cqe->rx_udp_csum_succeed) 1831 skb->ip_summed = CHECKSUM_UNNECESSARY; 1832 } 1833 1834 if (cqe->rx_hashtype != 0 && (ndev->features & NETIF_F_RXHASH)) { 1835 hash_value = cqe->ppi[0].pkt_hash; 1836 1837 if (cqe->rx_hashtype & MANA_HASH_L4) 1838 skb_set_hash(skb, hash_value, PKT_HASH_TYPE_L4); 1839 else 1840 skb_set_hash(skb, hash_value, PKT_HASH_TYPE_L3); 1841 } 1842 1843 if (cqe->rx_vlantag_present) { 1844 u16 vlan_tci = cqe->rx_vlan_id; 1845 1846 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vlan_tci); 1847 } 1848 1849 u64_stats_update_begin(&rx_stats->syncp); 1850 rx_stats->packets++; 1851 rx_stats->bytes += pkt_len; 1852 1853 if (act == XDP_TX) 1854 rx_stats->xdp_tx++; 1855 u64_stats_update_end(&rx_stats->syncp); 1856 1857 if (act == XDP_TX) { 1858 skb_set_queue_mapping(skb, rxq_idx); 1859 mana_xdp_tx(skb, ndev); 1860 return; 1861 } 1862 1863 napi_gro_receive(napi, skb); 1864 1865 return; 1866 1867 drop_xdp: 1868 u64_stats_update_begin(&rx_stats->syncp); 1869 rx_stats->xdp_drop++; 1870 u64_stats_update_end(&rx_stats->syncp); 1871 1872 drop: 1873 if (from_pool) { 1874 if (rxq->frag_count == 1) 1875 page_pool_recycle_direct(rxq->page_pool, 1876 virt_to_head_page(buf_va)); 1877 else 1878 page_pool_free_va(rxq->page_pool, buf_va, true); 1879 } else { 1880 WARN_ON_ONCE(rxq->xdp_save_va); 1881 /* Save for reuse */ 1882 rxq->xdp_save_va = buf_va; 1883 } 1884 1885 ++ndev->stats.rx_dropped; 1886 1887 return; 1888 } 1889 1890 static void *mana_get_rxfrag(struct mana_rxq *rxq, struct device *dev, 1891 dma_addr_t *da, bool *from_pool) 1892 { 1893 struct page *page; 1894 u32 offset; 1895 void *va; 1896 *from_pool = false; 1897 1898 /* Don't use fragments for jumbo frames or XDP where it's 1 fragment 1899 * per page. 1900 */ 1901 if (rxq->frag_count == 1) { 1902 /* Reuse XDP dropped page if available */ 1903 if (rxq->xdp_save_va) { 1904 va = rxq->xdp_save_va; 1905 page = virt_to_head_page(va); 1906 rxq->xdp_save_va = NULL; 1907 } else { 1908 page = page_pool_dev_alloc_pages(rxq->page_pool); 1909 if (!page) 1910 return NULL; 1911 1912 *from_pool = true; 1913 va = page_to_virt(page); 1914 } 1915 1916 *da = dma_map_single(dev, va + rxq->headroom, rxq->datasize, 1917 DMA_FROM_DEVICE); 1918 if (dma_mapping_error(dev, *da)) { 1919 mana_put_rx_page(rxq, page, *from_pool); 1920 return NULL; 1921 } 1922 1923 return va; 1924 } 1925 1926 page = page_pool_dev_alloc_frag(rxq->page_pool, &offset, 1927 rxq->alloc_size); 1928 if (!page) 1929 return NULL; 1930 1931 va = page_to_virt(page) + offset; 1932 *da = page_pool_get_dma_addr(page) + offset + rxq->headroom; 1933 *from_pool = true; 1934 1935 return va; 1936 } 1937 1938 /* Allocate frag for rx buffer, and save the old buf */ 1939 static void mana_refill_rx_oob(struct device *dev, struct mana_rxq *rxq, 1940 struct mana_recv_buf_oob *rxoob, void **old_buf, 1941 bool *old_fp) 1942 { 1943 bool from_pool; 1944 dma_addr_t da; 1945 void *va; 1946 1947 va = mana_get_rxfrag(rxq, dev, &da, &from_pool); 1948 if (!va) 1949 return; 1950 if (!rxoob->from_pool || rxq->frag_count == 1) 1951 dma_unmap_single(dev, rxoob->sgl[0].address, rxq->datasize, 1952 DMA_FROM_DEVICE); 1953 *old_buf = rxoob->buf_va; 1954 *old_fp = rxoob->from_pool; 1955 1956 rxoob->buf_va = va; 1957 rxoob->sgl[0].address = da; 1958 rxoob->from_pool = from_pool; 1959 } 1960 1961 static void mana_process_rx_cqe(struct mana_rxq *rxq, struct mana_cq *cq, 1962 struct gdma_comp *cqe) 1963 { 1964 struct mana_rxcomp_oob *oob = (struct mana_rxcomp_oob *)cqe->cqe_data; 1965 struct gdma_context *gc = rxq->gdma_rq->gdma_dev->gdma_context; 1966 struct net_device *ndev = rxq->ndev; 1967 struct mana_recv_buf_oob *rxbuf_oob; 1968 struct mana_port_context *apc; 1969 struct device *dev = gc->dev; 1970 void *old_buf = NULL; 1971 u32 curr, pktlen; 1972 bool old_fp; 1973 1974 apc = netdev_priv(ndev); 1975 1976 switch (oob->cqe_hdr.cqe_type) { 1977 case CQE_RX_OKAY: 1978 break; 1979 1980 case CQE_RX_TRUNCATED: 1981 ++ndev->stats.rx_dropped; 1982 rxbuf_oob = &rxq->rx_oobs[rxq->buf_index]; 1983 netdev_warn_once(ndev, "Dropped a truncated packet\n"); 1984 goto drop; 1985 1986 case CQE_RX_COALESCED_4: 1987 netdev_err(ndev, "RX coalescing is unsupported\n"); 1988 apc->eth_stats.rx_coalesced_err++; 1989 return; 1990 1991 case CQE_RX_OBJECT_FENCE: 1992 complete(&rxq->fence_event); 1993 return; 1994 1995 default: 1996 netdev_err(ndev, "Unknown RX CQE type = %d\n", 1997 oob->cqe_hdr.cqe_type); 1998 apc->eth_stats.rx_cqe_unknown_type++; 1999 return; 2000 } 2001 2002 pktlen = oob->ppi[0].pkt_len; 2003 2004 if (pktlen == 0) { 2005 /* data packets should never have packetlength of zero */ 2006 netdev_err(ndev, "RX pkt len=0, rq=%u, cq=%u, rxobj=0x%llx\n", 2007 rxq->gdma_id, cq->gdma_id, rxq->rxobj); 2008 return; 2009 } 2010 2011 curr = rxq->buf_index; 2012 rxbuf_oob = &rxq->rx_oobs[curr]; 2013 WARN_ON_ONCE(rxbuf_oob->wqe_inf.wqe_size_in_bu != 1); 2014 2015 mana_refill_rx_oob(dev, rxq, rxbuf_oob, &old_buf, &old_fp); 2016 2017 /* Unsuccessful refill will have old_buf == NULL. 2018 * In this case, mana_rx_skb() will drop the packet. 2019 */ 2020 mana_rx_skb(old_buf, old_fp, oob, rxq); 2021 2022 drop: 2023 mana_move_wq_tail(rxq->gdma_rq, rxbuf_oob->wqe_inf.wqe_size_in_bu); 2024 2025 mana_post_pkt_rxq(rxq); 2026 } 2027 2028 static void mana_poll_rx_cq(struct mana_cq *cq) 2029 { 2030 struct gdma_comp *comp = cq->gdma_comp_buf; 2031 struct mana_rxq *rxq = cq->rxq; 2032 int comp_read, i; 2033 2034 comp_read = mana_gd_poll_cq(cq->gdma_cq, comp, CQE_POLLING_BUFFER); 2035 WARN_ON_ONCE(comp_read > CQE_POLLING_BUFFER); 2036 2037 rxq->xdp_flush = false; 2038 2039 for (i = 0; i < comp_read; i++) { 2040 if (WARN_ON_ONCE(comp[i].is_sq)) 2041 return; 2042 2043 /* verify recv cqe references the right rxq */ 2044 if (WARN_ON_ONCE(comp[i].wq_num != cq->rxq->gdma_id)) 2045 return; 2046 2047 mana_process_rx_cqe(rxq, cq, &comp[i]); 2048 } 2049 2050 if (comp_read > 0) { 2051 struct gdma_context *gc = rxq->gdma_rq->gdma_dev->gdma_context; 2052 2053 mana_gd_wq_ring_doorbell(gc, rxq->gdma_rq); 2054 } 2055 2056 if (rxq->xdp_flush) 2057 xdp_do_flush(); 2058 } 2059 2060 static int mana_cq_handler(void *context, struct gdma_queue *gdma_queue) 2061 { 2062 struct mana_cq *cq = context; 2063 int w; 2064 2065 WARN_ON_ONCE(cq->gdma_cq != gdma_queue); 2066 2067 if (cq->type == MANA_CQ_TYPE_RX) 2068 mana_poll_rx_cq(cq); 2069 else 2070 mana_poll_tx_cq(cq); 2071 2072 w = cq->work_done; 2073 cq->work_done_since_doorbell += w; 2074 2075 if (w < cq->budget) { 2076 mana_gd_ring_cq(gdma_queue, SET_ARM_BIT); 2077 cq->work_done_since_doorbell = 0; 2078 napi_complete_done(&cq->napi, w); 2079 } else if (cq->work_done_since_doorbell > 2080 cq->gdma_cq->queue_size / COMP_ENTRY_SIZE * 4) { 2081 /* MANA hardware requires at least one doorbell ring every 8 2082 * wraparounds of CQ even if there is no need to arm the CQ. 2083 * This driver rings the doorbell as soon as we have exceeded 2084 * 4 wraparounds. 2085 */ 2086 mana_gd_ring_cq(gdma_queue, 0); 2087 cq->work_done_since_doorbell = 0; 2088 } 2089 2090 return w; 2091 } 2092 2093 static int mana_poll(struct napi_struct *napi, int budget) 2094 { 2095 struct mana_cq *cq = container_of(napi, struct mana_cq, napi); 2096 int w; 2097 2098 cq->work_done = 0; 2099 cq->budget = budget; 2100 2101 w = mana_cq_handler(cq, cq->gdma_cq); 2102 2103 return min(w, budget); 2104 } 2105 2106 static void mana_schedule_napi(void *context, struct gdma_queue *gdma_queue) 2107 { 2108 struct mana_cq *cq = context; 2109 2110 napi_schedule_irqoff(&cq->napi); 2111 } 2112 2113 static void mana_deinit_cq(struct mana_port_context *apc, struct mana_cq *cq) 2114 { 2115 struct gdma_dev *gd = apc->ac->gdma_dev; 2116 2117 if (!cq->gdma_cq) 2118 return; 2119 2120 mana_gd_destroy_queue(gd->gdma_context, cq->gdma_cq); 2121 } 2122 2123 static void mana_deinit_txq(struct mana_port_context *apc, struct mana_txq *txq) 2124 { 2125 struct gdma_dev *gd = apc->ac->gdma_dev; 2126 2127 if (!txq->gdma_sq) 2128 return; 2129 2130 mana_gd_destroy_queue(gd->gdma_context, txq->gdma_sq); 2131 } 2132 2133 static void mana_destroy_txq(struct mana_port_context *apc) 2134 { 2135 struct napi_struct *napi; 2136 int i; 2137 2138 if (!apc->tx_qp) 2139 return; 2140 2141 for (i = 0; i < apc->num_queues; i++) { 2142 debugfs_remove_recursive(apc->tx_qp[i].mana_tx_debugfs); 2143 apc->tx_qp[i].mana_tx_debugfs = NULL; 2144 2145 napi = &apc->tx_qp[i].tx_cq.napi; 2146 if (apc->tx_qp[i].txq.napi_initialized) { 2147 napi_synchronize(napi); 2148 netdev_lock_ops_to_full(napi->dev); 2149 napi_disable_locked(napi); 2150 netif_napi_del_locked(napi); 2151 netdev_unlock_full_to_ops(napi->dev); 2152 apc->tx_qp[i].txq.napi_initialized = false; 2153 } 2154 mana_destroy_wq_obj(apc, GDMA_SQ, apc->tx_qp[i].tx_object); 2155 2156 mana_deinit_cq(apc, &apc->tx_qp[i].tx_cq); 2157 2158 mana_deinit_txq(apc, &apc->tx_qp[i].txq); 2159 } 2160 2161 kfree(apc->tx_qp); 2162 apc->tx_qp = NULL; 2163 } 2164 2165 static void mana_create_txq_debugfs(struct mana_port_context *apc, int idx) 2166 { 2167 struct mana_tx_qp *tx_qp = &apc->tx_qp[idx]; 2168 char qnum[32]; 2169 2170 sprintf(qnum, "TX-%d", idx); 2171 tx_qp->mana_tx_debugfs = debugfs_create_dir(qnum, apc->mana_port_debugfs); 2172 debugfs_create_u32("sq_head", 0400, tx_qp->mana_tx_debugfs, 2173 &tx_qp->txq.gdma_sq->head); 2174 debugfs_create_u32("sq_tail", 0400, tx_qp->mana_tx_debugfs, 2175 &tx_qp->txq.gdma_sq->tail); 2176 debugfs_create_u32("sq_pend_skb_qlen", 0400, tx_qp->mana_tx_debugfs, 2177 &tx_qp->txq.pending_skbs.qlen); 2178 debugfs_create_u32("cq_head", 0400, tx_qp->mana_tx_debugfs, 2179 &tx_qp->tx_cq.gdma_cq->head); 2180 debugfs_create_u32("cq_tail", 0400, tx_qp->mana_tx_debugfs, 2181 &tx_qp->tx_cq.gdma_cq->tail); 2182 debugfs_create_u32("cq_budget", 0400, tx_qp->mana_tx_debugfs, 2183 &tx_qp->tx_cq.budget); 2184 debugfs_create_file("txq_dump", 0400, tx_qp->mana_tx_debugfs, 2185 tx_qp->txq.gdma_sq, &mana_dbg_q_fops); 2186 debugfs_create_file("cq_dump", 0400, tx_qp->mana_tx_debugfs, 2187 tx_qp->tx_cq.gdma_cq, &mana_dbg_q_fops); 2188 } 2189 2190 static int mana_create_txq(struct mana_port_context *apc, 2191 struct net_device *net) 2192 { 2193 struct mana_context *ac = apc->ac; 2194 struct gdma_dev *gd = ac->gdma_dev; 2195 struct mana_obj_spec wq_spec; 2196 struct mana_obj_spec cq_spec; 2197 struct gdma_queue_spec spec; 2198 struct gdma_context *gc; 2199 struct mana_txq *txq; 2200 struct mana_cq *cq; 2201 u32 txq_size; 2202 u32 cq_size; 2203 int err; 2204 int i; 2205 2206 apc->tx_qp = kcalloc(apc->num_queues, sizeof(struct mana_tx_qp), 2207 GFP_KERNEL); 2208 if (!apc->tx_qp) 2209 return -ENOMEM; 2210 2211 /* The minimum size of the WQE is 32 bytes, hence 2212 * apc->tx_queue_size represents the maximum number of WQEs 2213 * the SQ can store. This value is then used to size other queues 2214 * to prevent overflow. 2215 * Also note that the txq_size is always going to be MANA_PAGE_ALIGNED, 2216 * as min val of apc->tx_queue_size is 128 and that would make 2217 * txq_size 128*32 = 4096 and the other higher values of apc->tx_queue_size 2218 * are always power of two 2219 */ 2220 txq_size = apc->tx_queue_size * 32; 2221 2222 cq_size = apc->tx_queue_size * COMP_ENTRY_SIZE; 2223 2224 gc = gd->gdma_context; 2225 2226 for (i = 0; i < apc->num_queues; i++) { 2227 apc->tx_qp[i].tx_object = INVALID_MANA_HANDLE; 2228 2229 /* Create SQ */ 2230 txq = &apc->tx_qp[i].txq; 2231 2232 u64_stats_init(&txq->stats.syncp); 2233 txq->ndev = net; 2234 txq->net_txq = netdev_get_tx_queue(net, i); 2235 txq->vp_offset = apc->tx_vp_offset; 2236 txq->napi_initialized = false; 2237 skb_queue_head_init(&txq->pending_skbs); 2238 2239 memset(&spec, 0, sizeof(spec)); 2240 spec.type = GDMA_SQ; 2241 spec.monitor_avl_buf = true; 2242 spec.queue_size = txq_size; 2243 err = mana_gd_create_mana_wq_cq(gd, &spec, &txq->gdma_sq); 2244 if (err) 2245 goto out; 2246 2247 /* Create SQ's CQ */ 2248 cq = &apc->tx_qp[i].tx_cq; 2249 cq->type = MANA_CQ_TYPE_TX; 2250 2251 cq->txq = txq; 2252 2253 memset(&spec, 0, sizeof(spec)); 2254 spec.type = GDMA_CQ; 2255 spec.monitor_avl_buf = false; 2256 spec.queue_size = cq_size; 2257 spec.cq.callback = mana_schedule_napi; 2258 spec.cq.parent_eq = ac->eqs[i].eq; 2259 spec.cq.context = cq; 2260 err = mana_gd_create_mana_wq_cq(gd, &spec, &cq->gdma_cq); 2261 if (err) 2262 goto out; 2263 2264 memset(&wq_spec, 0, sizeof(wq_spec)); 2265 memset(&cq_spec, 0, sizeof(cq_spec)); 2266 2267 wq_spec.gdma_region = txq->gdma_sq->mem_info.dma_region_handle; 2268 wq_spec.queue_size = txq->gdma_sq->queue_size; 2269 2270 cq_spec.gdma_region = cq->gdma_cq->mem_info.dma_region_handle; 2271 cq_spec.queue_size = cq->gdma_cq->queue_size; 2272 cq_spec.modr_ctx_id = 0; 2273 cq_spec.attached_eq = cq->gdma_cq->cq.parent->id; 2274 2275 err = mana_create_wq_obj(apc, apc->port_handle, GDMA_SQ, 2276 &wq_spec, &cq_spec, 2277 &apc->tx_qp[i].tx_object); 2278 2279 if (err) 2280 goto out; 2281 2282 txq->gdma_sq->id = wq_spec.queue_index; 2283 cq->gdma_cq->id = cq_spec.queue_index; 2284 2285 txq->gdma_sq->mem_info.dma_region_handle = 2286 GDMA_INVALID_DMA_REGION; 2287 cq->gdma_cq->mem_info.dma_region_handle = 2288 GDMA_INVALID_DMA_REGION; 2289 2290 txq->gdma_txq_id = txq->gdma_sq->id; 2291 2292 cq->gdma_id = cq->gdma_cq->id; 2293 2294 if (WARN_ON(cq->gdma_id >= gc->max_num_cqs)) { 2295 err = -EINVAL; 2296 goto out; 2297 } 2298 2299 gc->cq_table[cq->gdma_id] = cq->gdma_cq; 2300 2301 mana_create_txq_debugfs(apc, i); 2302 2303 set_bit(NAPI_STATE_NO_BUSY_POLL, &cq->napi.state); 2304 netdev_lock_ops_to_full(net); 2305 netif_napi_add_locked(net, &cq->napi, mana_poll); 2306 napi_enable_locked(&cq->napi); 2307 netdev_unlock_full_to_ops(net); 2308 txq->napi_initialized = true; 2309 2310 mana_gd_ring_cq(cq->gdma_cq, SET_ARM_BIT); 2311 } 2312 2313 return 0; 2314 out: 2315 netdev_err(net, "Failed to create %d TX queues, %d\n", 2316 apc->num_queues, err); 2317 mana_destroy_txq(apc); 2318 return err; 2319 } 2320 2321 static void mana_destroy_rxq(struct mana_port_context *apc, 2322 struct mana_rxq *rxq, bool napi_initialized) 2323 2324 { 2325 struct gdma_context *gc = apc->ac->gdma_dev->gdma_context; 2326 struct mana_recv_buf_oob *rx_oob; 2327 struct device *dev = gc->dev; 2328 struct napi_struct *napi; 2329 struct page *page; 2330 int i; 2331 2332 if (!rxq) 2333 return; 2334 2335 debugfs_remove_recursive(rxq->mana_rx_debugfs); 2336 rxq->mana_rx_debugfs = NULL; 2337 2338 napi = &rxq->rx_cq.napi; 2339 2340 if (napi_initialized) { 2341 napi_synchronize(napi); 2342 2343 netdev_lock_ops_to_full(napi->dev); 2344 napi_disable_locked(napi); 2345 netif_napi_del_locked(napi); 2346 netdev_unlock_full_to_ops(napi->dev); 2347 } 2348 xdp_rxq_info_unreg(&rxq->xdp_rxq); 2349 2350 mana_destroy_wq_obj(apc, GDMA_RQ, rxq->rxobj); 2351 2352 mana_deinit_cq(apc, &rxq->rx_cq); 2353 2354 if (rxq->xdp_save_va) 2355 put_page(virt_to_head_page(rxq->xdp_save_va)); 2356 2357 for (i = 0; i < rxq->num_rx_buf; i++) { 2358 rx_oob = &rxq->rx_oobs[i]; 2359 2360 if (!rx_oob->buf_va) 2361 continue; 2362 2363 page = virt_to_head_page(rx_oob->buf_va); 2364 2365 if (rxq->frag_count == 1 || !rx_oob->from_pool) { 2366 dma_unmap_single(dev, rx_oob->sgl[0].address, 2367 rx_oob->sgl[0].size, DMA_FROM_DEVICE); 2368 mana_put_rx_page(rxq, page, rx_oob->from_pool); 2369 } else { 2370 page_pool_free_va(rxq->page_pool, rx_oob->buf_va, true); 2371 } 2372 2373 rx_oob->buf_va = NULL; 2374 } 2375 2376 page_pool_destroy(rxq->page_pool); 2377 2378 if (rxq->gdma_rq) 2379 mana_gd_destroy_queue(gc, rxq->gdma_rq); 2380 2381 kfree(rxq); 2382 } 2383 2384 static int mana_fill_rx_oob(struct mana_recv_buf_oob *rx_oob, u32 mem_key, 2385 struct mana_rxq *rxq, struct device *dev) 2386 { 2387 struct mana_port_context *mpc = netdev_priv(rxq->ndev); 2388 bool from_pool = false; 2389 dma_addr_t da; 2390 void *va; 2391 2392 if (mpc->rxbufs_pre) 2393 va = mana_get_rxbuf_pre(rxq, &da); 2394 else 2395 va = mana_get_rxfrag(rxq, dev, &da, &from_pool); 2396 2397 if (!va) 2398 return -ENOMEM; 2399 2400 rx_oob->buf_va = va; 2401 rx_oob->from_pool = from_pool; 2402 2403 rx_oob->sgl[0].address = da; 2404 rx_oob->sgl[0].size = rxq->datasize; 2405 rx_oob->sgl[0].mem_key = mem_key; 2406 2407 return 0; 2408 } 2409 2410 #define MANA_WQE_HEADER_SIZE 16 2411 #define MANA_WQE_SGE_SIZE 16 2412 2413 static int mana_alloc_rx_wqe(struct mana_port_context *apc, 2414 struct mana_rxq *rxq, u32 *rxq_size, u32 *cq_size) 2415 { 2416 struct gdma_context *gc = apc->ac->gdma_dev->gdma_context; 2417 struct mana_recv_buf_oob *rx_oob; 2418 struct device *dev = gc->dev; 2419 u32 buf_idx; 2420 int ret; 2421 2422 WARN_ON(rxq->datasize == 0); 2423 2424 *rxq_size = 0; 2425 *cq_size = 0; 2426 2427 for (buf_idx = 0; buf_idx < rxq->num_rx_buf; buf_idx++) { 2428 rx_oob = &rxq->rx_oobs[buf_idx]; 2429 memset(rx_oob, 0, sizeof(*rx_oob)); 2430 2431 rx_oob->num_sge = 1; 2432 2433 ret = mana_fill_rx_oob(rx_oob, apc->ac->gdma_dev->gpa_mkey, rxq, 2434 dev); 2435 if (ret) 2436 return ret; 2437 2438 rx_oob->wqe_req.sgl = rx_oob->sgl; 2439 rx_oob->wqe_req.num_sge = rx_oob->num_sge; 2440 rx_oob->wqe_req.inline_oob_size = 0; 2441 rx_oob->wqe_req.inline_oob_data = NULL; 2442 rx_oob->wqe_req.flags = 0; 2443 rx_oob->wqe_req.client_data_unit = 0; 2444 2445 *rxq_size += ALIGN(MANA_WQE_HEADER_SIZE + 2446 MANA_WQE_SGE_SIZE * rx_oob->num_sge, 32); 2447 *cq_size += COMP_ENTRY_SIZE; 2448 } 2449 2450 return 0; 2451 } 2452 2453 static int mana_push_wqe(struct mana_rxq *rxq) 2454 { 2455 struct mana_recv_buf_oob *rx_oob; 2456 u32 buf_idx; 2457 int err; 2458 2459 for (buf_idx = 0; buf_idx < rxq->num_rx_buf; buf_idx++) { 2460 rx_oob = &rxq->rx_oobs[buf_idx]; 2461 2462 err = mana_gd_post_and_ring(rxq->gdma_rq, &rx_oob->wqe_req, 2463 &rx_oob->wqe_inf); 2464 if (err) 2465 return -ENOSPC; 2466 } 2467 2468 return 0; 2469 } 2470 2471 static int mana_create_page_pool(struct mana_rxq *rxq, struct gdma_context *gc) 2472 { 2473 struct mana_port_context *mpc = netdev_priv(rxq->ndev); 2474 struct page_pool_params pprm = {}; 2475 int ret; 2476 2477 pprm.pool_size = mpc->rx_queue_size / rxq->frag_count + 1; 2478 pprm.nid = gc->numa_node; 2479 pprm.napi = &rxq->rx_cq.napi; 2480 pprm.netdev = rxq->ndev; 2481 pprm.order = get_order(rxq->alloc_size); 2482 pprm.queue_idx = rxq->rxq_idx; 2483 pprm.dev = gc->dev; 2484 2485 /* Let the page pool do the dma map when page sharing with multiple 2486 * fragments enabled for rx buffers. 2487 */ 2488 if (rxq->frag_count > 1) { 2489 pprm.flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV; 2490 pprm.max_len = PAGE_SIZE; 2491 pprm.dma_dir = DMA_FROM_DEVICE; 2492 } 2493 2494 rxq->page_pool = page_pool_create(&pprm); 2495 2496 if (IS_ERR(rxq->page_pool)) { 2497 ret = PTR_ERR(rxq->page_pool); 2498 rxq->page_pool = NULL; 2499 return ret; 2500 } 2501 2502 return 0; 2503 } 2504 2505 static struct mana_rxq *mana_create_rxq(struct mana_port_context *apc, 2506 u32 rxq_idx, struct mana_eq *eq, 2507 struct net_device *ndev) 2508 { 2509 struct gdma_dev *gd = apc->ac->gdma_dev; 2510 struct mana_obj_spec wq_spec; 2511 struct mana_obj_spec cq_spec; 2512 struct gdma_queue_spec spec; 2513 struct mana_cq *cq = NULL; 2514 struct gdma_context *gc; 2515 u32 cq_size, rq_size; 2516 struct mana_rxq *rxq; 2517 int err; 2518 2519 gc = gd->gdma_context; 2520 2521 rxq = kzalloc(struct_size(rxq, rx_oobs, apc->rx_queue_size), 2522 GFP_KERNEL); 2523 if (!rxq) 2524 return NULL; 2525 2526 rxq->ndev = ndev; 2527 rxq->num_rx_buf = apc->rx_queue_size; 2528 rxq->rxq_idx = rxq_idx; 2529 rxq->rxobj = INVALID_MANA_HANDLE; 2530 2531 mana_get_rxbuf_cfg(apc, ndev->mtu, &rxq->datasize, &rxq->alloc_size, 2532 &rxq->headroom, &rxq->frag_count); 2533 /* Create page pool for RX queue */ 2534 err = mana_create_page_pool(rxq, gc); 2535 if (err) { 2536 netdev_err(ndev, "Create page pool err:%d\n", err); 2537 goto out; 2538 } 2539 2540 err = mana_alloc_rx_wqe(apc, rxq, &rq_size, &cq_size); 2541 if (err) 2542 goto out; 2543 2544 rq_size = MANA_PAGE_ALIGN(rq_size); 2545 cq_size = MANA_PAGE_ALIGN(cq_size); 2546 2547 /* Create RQ */ 2548 memset(&spec, 0, sizeof(spec)); 2549 spec.type = GDMA_RQ; 2550 spec.monitor_avl_buf = true; 2551 spec.queue_size = rq_size; 2552 err = mana_gd_create_mana_wq_cq(gd, &spec, &rxq->gdma_rq); 2553 if (err) 2554 goto out; 2555 2556 /* Create RQ's CQ */ 2557 cq = &rxq->rx_cq; 2558 cq->type = MANA_CQ_TYPE_RX; 2559 cq->rxq = rxq; 2560 2561 memset(&spec, 0, sizeof(spec)); 2562 spec.type = GDMA_CQ; 2563 spec.monitor_avl_buf = false; 2564 spec.queue_size = cq_size; 2565 spec.cq.callback = mana_schedule_napi; 2566 spec.cq.parent_eq = eq->eq; 2567 spec.cq.context = cq; 2568 err = mana_gd_create_mana_wq_cq(gd, &spec, &cq->gdma_cq); 2569 if (err) 2570 goto out; 2571 2572 memset(&wq_spec, 0, sizeof(wq_spec)); 2573 memset(&cq_spec, 0, sizeof(cq_spec)); 2574 wq_spec.gdma_region = rxq->gdma_rq->mem_info.dma_region_handle; 2575 wq_spec.queue_size = rxq->gdma_rq->queue_size; 2576 2577 cq_spec.gdma_region = cq->gdma_cq->mem_info.dma_region_handle; 2578 cq_spec.queue_size = cq->gdma_cq->queue_size; 2579 cq_spec.modr_ctx_id = 0; 2580 cq_spec.attached_eq = cq->gdma_cq->cq.parent->id; 2581 2582 err = mana_create_wq_obj(apc, apc->port_handle, GDMA_RQ, 2583 &wq_spec, &cq_spec, &rxq->rxobj); 2584 if (err) 2585 goto out; 2586 2587 rxq->gdma_rq->id = wq_spec.queue_index; 2588 cq->gdma_cq->id = cq_spec.queue_index; 2589 2590 rxq->gdma_rq->mem_info.dma_region_handle = GDMA_INVALID_DMA_REGION; 2591 cq->gdma_cq->mem_info.dma_region_handle = GDMA_INVALID_DMA_REGION; 2592 2593 rxq->gdma_id = rxq->gdma_rq->id; 2594 cq->gdma_id = cq->gdma_cq->id; 2595 2596 err = mana_push_wqe(rxq); 2597 if (err) 2598 goto out; 2599 2600 if (WARN_ON(cq->gdma_id >= gc->max_num_cqs)) { 2601 err = -EINVAL; 2602 goto out; 2603 } 2604 2605 gc->cq_table[cq->gdma_id] = cq->gdma_cq; 2606 2607 netdev_lock_ops_to_full(ndev); 2608 netif_napi_add_weight_locked(ndev, &cq->napi, mana_poll, 1); 2609 netdev_unlock_full_to_ops(ndev); 2610 2611 WARN_ON(xdp_rxq_info_reg(&rxq->xdp_rxq, ndev, rxq_idx, 2612 cq->napi.napi_id)); 2613 WARN_ON(xdp_rxq_info_reg_mem_model(&rxq->xdp_rxq, MEM_TYPE_PAGE_POOL, 2614 rxq->page_pool)); 2615 2616 netdev_lock_ops_to_full(ndev); 2617 napi_enable_locked(&cq->napi); 2618 netdev_unlock_full_to_ops(ndev); 2619 2620 mana_gd_ring_cq(cq->gdma_cq, SET_ARM_BIT); 2621 out: 2622 if (!err) 2623 return rxq; 2624 2625 netdev_err(ndev, "Failed to create RXQ: err = %d\n", err); 2626 2627 mana_destroy_rxq(apc, rxq, false); 2628 2629 if (cq) 2630 mana_deinit_cq(apc, cq); 2631 2632 return NULL; 2633 } 2634 2635 static void mana_create_rxq_debugfs(struct mana_port_context *apc, int idx) 2636 { 2637 struct mana_rxq *rxq; 2638 char qnum[32]; 2639 2640 rxq = apc->rxqs[idx]; 2641 2642 sprintf(qnum, "RX-%d", idx); 2643 rxq->mana_rx_debugfs = debugfs_create_dir(qnum, apc->mana_port_debugfs); 2644 debugfs_create_u32("rq_head", 0400, rxq->mana_rx_debugfs, &rxq->gdma_rq->head); 2645 debugfs_create_u32("rq_tail", 0400, rxq->mana_rx_debugfs, &rxq->gdma_rq->tail); 2646 debugfs_create_u32("rq_nbuf", 0400, rxq->mana_rx_debugfs, &rxq->num_rx_buf); 2647 debugfs_create_u32("cq_head", 0400, rxq->mana_rx_debugfs, 2648 &rxq->rx_cq.gdma_cq->head); 2649 debugfs_create_u32("cq_tail", 0400, rxq->mana_rx_debugfs, 2650 &rxq->rx_cq.gdma_cq->tail); 2651 debugfs_create_u32("cq_budget", 0400, rxq->mana_rx_debugfs, &rxq->rx_cq.budget); 2652 debugfs_create_file("rxq_dump", 0400, rxq->mana_rx_debugfs, rxq->gdma_rq, &mana_dbg_q_fops); 2653 debugfs_create_file("cq_dump", 0400, rxq->mana_rx_debugfs, rxq->rx_cq.gdma_cq, 2654 &mana_dbg_q_fops); 2655 } 2656 2657 static int mana_add_rx_queues(struct mana_port_context *apc, 2658 struct net_device *ndev) 2659 { 2660 struct mana_context *ac = apc->ac; 2661 struct mana_rxq *rxq; 2662 int err = 0; 2663 int i; 2664 2665 for (i = 0; i < apc->num_queues; i++) { 2666 rxq = mana_create_rxq(apc, i, &ac->eqs[i], ndev); 2667 if (!rxq) { 2668 err = -ENOMEM; 2669 netdev_err(ndev, "Failed to create rxq %d : %d\n", i, err); 2670 goto out; 2671 } 2672 2673 u64_stats_init(&rxq->stats.syncp); 2674 2675 apc->rxqs[i] = rxq; 2676 2677 mana_create_rxq_debugfs(apc, i); 2678 } 2679 2680 apc->default_rxobj = apc->rxqs[0]->rxobj; 2681 out: 2682 return err; 2683 } 2684 2685 static void mana_destroy_vport(struct mana_port_context *apc) 2686 { 2687 struct gdma_dev *gd = apc->ac->gdma_dev; 2688 struct mana_rxq *rxq; 2689 u32 rxq_idx; 2690 2691 for (rxq_idx = 0; rxq_idx < apc->num_queues; rxq_idx++) { 2692 rxq = apc->rxqs[rxq_idx]; 2693 if (!rxq) 2694 continue; 2695 2696 mana_destroy_rxq(apc, rxq, true); 2697 apc->rxqs[rxq_idx] = NULL; 2698 } 2699 2700 mana_destroy_txq(apc); 2701 mana_uncfg_vport(apc); 2702 2703 if (gd->gdma_context->is_pf && !apc->ac->bm_hostmode) 2704 mana_pf_deregister_hw_vport(apc); 2705 } 2706 2707 static int mana_create_vport(struct mana_port_context *apc, 2708 struct net_device *net) 2709 { 2710 struct gdma_dev *gd = apc->ac->gdma_dev; 2711 int err; 2712 2713 apc->default_rxobj = INVALID_MANA_HANDLE; 2714 2715 if (gd->gdma_context->is_pf && !apc->ac->bm_hostmode) { 2716 err = mana_pf_register_hw_vport(apc); 2717 if (err) 2718 return err; 2719 } 2720 2721 err = mana_cfg_vport(apc, gd->pdid, gd->doorbell); 2722 if (err) 2723 return err; 2724 2725 return mana_create_txq(apc, net); 2726 } 2727 2728 static int mana_rss_table_alloc(struct mana_port_context *apc) 2729 { 2730 if (!apc->indir_table_sz) { 2731 netdev_err(apc->ndev, 2732 "Indirection table size not set for vPort %d\n", 2733 apc->port_idx); 2734 return -EINVAL; 2735 } 2736 2737 apc->indir_table = kcalloc(apc->indir_table_sz, sizeof(u32), GFP_KERNEL); 2738 if (!apc->indir_table) 2739 return -ENOMEM; 2740 2741 apc->rxobj_table = kcalloc(apc->indir_table_sz, sizeof(mana_handle_t), GFP_KERNEL); 2742 if (!apc->rxobj_table) { 2743 kfree(apc->indir_table); 2744 return -ENOMEM; 2745 } 2746 2747 return 0; 2748 } 2749 2750 static void mana_rss_table_init(struct mana_port_context *apc) 2751 { 2752 int i; 2753 2754 for (i = 0; i < apc->indir_table_sz; i++) 2755 apc->indir_table[i] = 2756 ethtool_rxfh_indir_default(i, apc->num_queues); 2757 } 2758 2759 int mana_config_rss(struct mana_port_context *apc, enum TRI_STATE rx, 2760 bool update_hash, bool update_tab) 2761 { 2762 u32 queue_idx; 2763 int err; 2764 int i; 2765 2766 if (update_tab) { 2767 for (i = 0; i < apc->indir_table_sz; i++) { 2768 queue_idx = apc->indir_table[i]; 2769 apc->rxobj_table[i] = apc->rxqs[queue_idx]->rxobj; 2770 } 2771 } 2772 2773 err = mana_cfg_vport_steering(apc, rx, true, update_hash, update_tab); 2774 if (err) 2775 return err; 2776 2777 mana_fence_rqs(apc); 2778 2779 return 0; 2780 } 2781 2782 void mana_query_gf_stats(struct mana_port_context *apc) 2783 { 2784 struct mana_query_gf_stat_resp resp = {}; 2785 struct mana_query_gf_stat_req req = {}; 2786 struct net_device *ndev = apc->ndev; 2787 int err; 2788 2789 mana_gd_init_req_hdr(&req.hdr, MANA_QUERY_GF_STAT, 2790 sizeof(req), sizeof(resp)); 2791 req.hdr.resp.msg_version = GDMA_MESSAGE_V2; 2792 req.req_stats = STATISTICS_FLAGS_RX_DISCARDS_NO_WQE | 2793 STATISTICS_FLAGS_RX_ERRORS_VPORT_DISABLED | 2794 STATISTICS_FLAGS_HC_RX_BYTES | 2795 STATISTICS_FLAGS_HC_RX_UCAST_PACKETS | 2796 STATISTICS_FLAGS_HC_RX_UCAST_BYTES | 2797 STATISTICS_FLAGS_HC_RX_MCAST_PACKETS | 2798 STATISTICS_FLAGS_HC_RX_MCAST_BYTES | 2799 STATISTICS_FLAGS_HC_RX_BCAST_PACKETS | 2800 STATISTICS_FLAGS_HC_RX_BCAST_BYTES | 2801 STATISTICS_FLAGS_TX_ERRORS_GF_DISABLED | 2802 STATISTICS_FLAGS_TX_ERRORS_VPORT_DISABLED | 2803 STATISTICS_FLAGS_TX_ERRORS_INVAL_VPORT_OFFSET_PACKETS | 2804 STATISTICS_FLAGS_TX_ERRORS_VLAN_ENFORCEMENT | 2805 STATISTICS_FLAGS_TX_ERRORS_ETH_TYPE_ENFORCEMENT | 2806 STATISTICS_FLAGS_TX_ERRORS_SA_ENFORCEMENT | 2807 STATISTICS_FLAGS_TX_ERRORS_SQPDID_ENFORCEMENT | 2808 STATISTICS_FLAGS_TX_ERRORS_CQPDID_ENFORCEMENT | 2809 STATISTICS_FLAGS_TX_ERRORS_MTU_VIOLATION | 2810 STATISTICS_FLAGS_TX_ERRORS_INVALID_OOB | 2811 STATISTICS_FLAGS_HC_TX_BYTES | 2812 STATISTICS_FLAGS_HC_TX_UCAST_PACKETS | 2813 STATISTICS_FLAGS_HC_TX_UCAST_BYTES | 2814 STATISTICS_FLAGS_HC_TX_MCAST_PACKETS | 2815 STATISTICS_FLAGS_HC_TX_MCAST_BYTES | 2816 STATISTICS_FLAGS_HC_TX_BCAST_PACKETS | 2817 STATISTICS_FLAGS_HC_TX_BCAST_BYTES | 2818 STATISTICS_FLAGS_TX_ERRORS_GDMA_ERROR; 2819 2820 err = mana_send_request(apc->ac, &req, sizeof(req), &resp, 2821 sizeof(resp)); 2822 if (err) { 2823 netdev_err(ndev, "Failed to query GF stats: %d\n", err); 2824 return; 2825 } 2826 err = mana_verify_resp_hdr(&resp.hdr, MANA_QUERY_GF_STAT, 2827 sizeof(resp)); 2828 if (err || resp.hdr.status) { 2829 netdev_err(ndev, "Failed to query GF stats: %d, 0x%x\n", err, 2830 resp.hdr.status); 2831 return; 2832 } 2833 2834 apc->eth_stats.hc_rx_discards_no_wqe = resp.rx_discards_nowqe; 2835 apc->eth_stats.hc_rx_err_vport_disabled = resp.rx_err_vport_disabled; 2836 apc->eth_stats.hc_rx_bytes = resp.hc_rx_bytes; 2837 apc->eth_stats.hc_rx_ucast_pkts = resp.hc_rx_ucast_pkts; 2838 apc->eth_stats.hc_rx_ucast_bytes = resp.hc_rx_ucast_bytes; 2839 apc->eth_stats.hc_rx_bcast_pkts = resp.hc_rx_bcast_pkts; 2840 apc->eth_stats.hc_rx_bcast_bytes = resp.hc_rx_bcast_bytes; 2841 apc->eth_stats.hc_rx_mcast_pkts = resp.hc_rx_mcast_pkts; 2842 apc->eth_stats.hc_rx_mcast_bytes = resp.hc_rx_mcast_bytes; 2843 apc->eth_stats.hc_tx_err_gf_disabled = resp.tx_err_gf_disabled; 2844 apc->eth_stats.hc_tx_err_vport_disabled = resp.tx_err_vport_disabled; 2845 apc->eth_stats.hc_tx_err_inval_vportoffset_pkt = 2846 resp.tx_err_inval_vport_offset_pkt; 2847 apc->eth_stats.hc_tx_err_vlan_enforcement = 2848 resp.tx_err_vlan_enforcement; 2849 apc->eth_stats.hc_tx_err_eth_type_enforcement = 2850 resp.tx_err_ethtype_enforcement; 2851 apc->eth_stats.hc_tx_err_sa_enforcement = resp.tx_err_SA_enforcement; 2852 apc->eth_stats.hc_tx_err_sqpdid_enforcement = 2853 resp.tx_err_SQPDID_enforcement; 2854 apc->eth_stats.hc_tx_err_cqpdid_enforcement = 2855 resp.tx_err_CQPDID_enforcement; 2856 apc->eth_stats.hc_tx_err_mtu_violation = resp.tx_err_mtu_violation; 2857 apc->eth_stats.hc_tx_err_inval_oob = resp.tx_err_inval_oob; 2858 apc->eth_stats.hc_tx_bytes = resp.hc_tx_bytes; 2859 apc->eth_stats.hc_tx_ucast_pkts = resp.hc_tx_ucast_pkts; 2860 apc->eth_stats.hc_tx_ucast_bytes = resp.hc_tx_ucast_bytes; 2861 apc->eth_stats.hc_tx_bcast_pkts = resp.hc_tx_bcast_pkts; 2862 apc->eth_stats.hc_tx_bcast_bytes = resp.hc_tx_bcast_bytes; 2863 apc->eth_stats.hc_tx_mcast_pkts = resp.hc_tx_mcast_pkts; 2864 apc->eth_stats.hc_tx_mcast_bytes = resp.hc_tx_mcast_bytes; 2865 apc->eth_stats.hc_tx_err_gdma = resp.tx_err_gdma; 2866 } 2867 2868 void mana_query_phy_stats(struct mana_port_context *apc) 2869 { 2870 struct mana_query_phy_stat_resp resp = {}; 2871 struct mana_query_phy_stat_req req = {}; 2872 struct net_device *ndev = apc->ndev; 2873 int err; 2874 2875 mana_gd_init_req_hdr(&req.hdr, MANA_QUERY_PHY_STAT, 2876 sizeof(req), sizeof(resp)); 2877 err = mana_send_request(apc->ac, &req, sizeof(req), &resp, 2878 sizeof(resp)); 2879 if (err) 2880 return; 2881 2882 err = mana_verify_resp_hdr(&resp.hdr, MANA_QUERY_PHY_STAT, 2883 sizeof(resp)); 2884 if (err || resp.hdr.status) { 2885 netdev_err(ndev, 2886 "Failed to query PHY stats: %d, resp:0x%x\n", 2887 err, resp.hdr.status); 2888 return; 2889 } 2890 2891 /* Aggregate drop counters */ 2892 apc->phy_stats.rx_pkt_drop_phy = resp.rx_pkt_drop_phy; 2893 apc->phy_stats.tx_pkt_drop_phy = resp.tx_pkt_drop_phy; 2894 2895 /* Per TC traffic Counters */ 2896 apc->phy_stats.rx_pkt_tc0_phy = resp.rx_pkt_tc0_phy; 2897 apc->phy_stats.tx_pkt_tc0_phy = resp.tx_pkt_tc0_phy; 2898 apc->phy_stats.rx_pkt_tc1_phy = resp.rx_pkt_tc1_phy; 2899 apc->phy_stats.tx_pkt_tc1_phy = resp.tx_pkt_tc1_phy; 2900 apc->phy_stats.rx_pkt_tc2_phy = resp.rx_pkt_tc2_phy; 2901 apc->phy_stats.tx_pkt_tc2_phy = resp.tx_pkt_tc2_phy; 2902 apc->phy_stats.rx_pkt_tc3_phy = resp.rx_pkt_tc3_phy; 2903 apc->phy_stats.tx_pkt_tc3_phy = resp.tx_pkt_tc3_phy; 2904 apc->phy_stats.rx_pkt_tc4_phy = resp.rx_pkt_tc4_phy; 2905 apc->phy_stats.tx_pkt_tc4_phy = resp.tx_pkt_tc4_phy; 2906 apc->phy_stats.rx_pkt_tc5_phy = resp.rx_pkt_tc5_phy; 2907 apc->phy_stats.tx_pkt_tc5_phy = resp.tx_pkt_tc5_phy; 2908 apc->phy_stats.rx_pkt_tc6_phy = resp.rx_pkt_tc6_phy; 2909 apc->phy_stats.tx_pkt_tc6_phy = resp.tx_pkt_tc6_phy; 2910 apc->phy_stats.rx_pkt_tc7_phy = resp.rx_pkt_tc7_phy; 2911 apc->phy_stats.tx_pkt_tc7_phy = resp.tx_pkt_tc7_phy; 2912 2913 /* Per TC byte Counters */ 2914 apc->phy_stats.rx_byte_tc0_phy = resp.rx_byte_tc0_phy; 2915 apc->phy_stats.tx_byte_tc0_phy = resp.tx_byte_tc0_phy; 2916 apc->phy_stats.rx_byte_tc1_phy = resp.rx_byte_tc1_phy; 2917 apc->phy_stats.tx_byte_tc1_phy = resp.tx_byte_tc1_phy; 2918 apc->phy_stats.rx_byte_tc2_phy = resp.rx_byte_tc2_phy; 2919 apc->phy_stats.tx_byte_tc2_phy = resp.tx_byte_tc2_phy; 2920 apc->phy_stats.rx_byte_tc3_phy = resp.rx_byte_tc3_phy; 2921 apc->phy_stats.tx_byte_tc3_phy = resp.tx_byte_tc3_phy; 2922 apc->phy_stats.rx_byte_tc4_phy = resp.rx_byte_tc4_phy; 2923 apc->phy_stats.tx_byte_tc4_phy = resp.tx_byte_tc4_phy; 2924 apc->phy_stats.rx_byte_tc5_phy = resp.rx_byte_tc5_phy; 2925 apc->phy_stats.tx_byte_tc5_phy = resp.tx_byte_tc5_phy; 2926 apc->phy_stats.rx_byte_tc6_phy = resp.rx_byte_tc6_phy; 2927 apc->phy_stats.tx_byte_tc6_phy = resp.tx_byte_tc6_phy; 2928 apc->phy_stats.rx_byte_tc7_phy = resp.rx_byte_tc7_phy; 2929 apc->phy_stats.tx_byte_tc7_phy = resp.tx_byte_tc7_phy; 2930 2931 /* Per TC pause Counters */ 2932 apc->phy_stats.rx_pause_tc0_phy = resp.rx_pause_tc0_phy; 2933 apc->phy_stats.tx_pause_tc0_phy = resp.tx_pause_tc0_phy; 2934 apc->phy_stats.rx_pause_tc1_phy = resp.rx_pause_tc1_phy; 2935 apc->phy_stats.tx_pause_tc1_phy = resp.tx_pause_tc1_phy; 2936 apc->phy_stats.rx_pause_tc2_phy = resp.rx_pause_tc2_phy; 2937 apc->phy_stats.tx_pause_tc2_phy = resp.tx_pause_tc2_phy; 2938 apc->phy_stats.rx_pause_tc3_phy = resp.rx_pause_tc3_phy; 2939 apc->phy_stats.tx_pause_tc3_phy = resp.tx_pause_tc3_phy; 2940 apc->phy_stats.rx_pause_tc4_phy = resp.rx_pause_tc4_phy; 2941 apc->phy_stats.tx_pause_tc4_phy = resp.tx_pause_tc4_phy; 2942 apc->phy_stats.rx_pause_tc5_phy = resp.rx_pause_tc5_phy; 2943 apc->phy_stats.tx_pause_tc5_phy = resp.tx_pause_tc5_phy; 2944 apc->phy_stats.rx_pause_tc6_phy = resp.rx_pause_tc6_phy; 2945 apc->phy_stats.tx_pause_tc6_phy = resp.tx_pause_tc6_phy; 2946 apc->phy_stats.rx_pause_tc7_phy = resp.rx_pause_tc7_phy; 2947 apc->phy_stats.tx_pause_tc7_phy = resp.tx_pause_tc7_phy; 2948 } 2949 2950 static int mana_init_port(struct net_device *ndev) 2951 { 2952 struct mana_port_context *apc = netdev_priv(ndev); 2953 struct gdma_dev *gd = apc->ac->gdma_dev; 2954 u32 max_txq, max_rxq, max_queues; 2955 int port_idx = apc->port_idx; 2956 struct gdma_context *gc; 2957 char vport[32]; 2958 int err; 2959 2960 err = mana_init_port_context(apc); 2961 if (err) 2962 return err; 2963 2964 gc = gd->gdma_context; 2965 2966 err = mana_query_vport_cfg(apc, port_idx, &max_txq, &max_rxq, 2967 &apc->indir_table_sz); 2968 if (err) { 2969 netdev_err(ndev, "Failed to query info for vPort %d\n", 2970 port_idx); 2971 goto reset_apc; 2972 } 2973 2974 max_queues = min_t(u32, max_txq, max_rxq); 2975 if (apc->max_queues > max_queues) 2976 apc->max_queues = max_queues; 2977 2978 if (apc->num_queues > apc->max_queues) 2979 apc->num_queues = apc->max_queues; 2980 2981 eth_hw_addr_set(ndev, apc->mac_addr); 2982 sprintf(vport, "vport%d", port_idx); 2983 apc->mana_port_debugfs = debugfs_create_dir(vport, gc->mana_pci_debugfs); 2984 return 0; 2985 2986 reset_apc: 2987 mana_cleanup_port_context(apc); 2988 return err; 2989 } 2990 2991 int mana_alloc_queues(struct net_device *ndev) 2992 { 2993 struct mana_port_context *apc = netdev_priv(ndev); 2994 struct gdma_dev *gd = apc->ac->gdma_dev; 2995 int err; 2996 2997 err = mana_create_vport(apc, ndev); 2998 if (err) { 2999 netdev_err(ndev, "Failed to create vPort %u : %d\n", apc->port_idx, err); 3000 return err; 3001 } 3002 3003 err = netif_set_real_num_tx_queues(ndev, apc->num_queues); 3004 if (err) { 3005 netdev_err(ndev, 3006 "netif_set_real_num_tx_queues () failed for ndev with num_queues %u : %d\n", 3007 apc->num_queues, err); 3008 goto destroy_vport; 3009 } 3010 3011 err = mana_add_rx_queues(apc, ndev); 3012 if (err) 3013 goto destroy_vport; 3014 3015 apc->rss_state = apc->num_queues > 1 ? TRI_STATE_TRUE : TRI_STATE_FALSE; 3016 3017 err = netif_set_real_num_rx_queues(ndev, apc->num_queues); 3018 if (err) { 3019 netdev_err(ndev, 3020 "netif_set_real_num_rx_queues () failed for ndev with num_queues %u : %d\n", 3021 apc->num_queues, err); 3022 goto destroy_vport; 3023 } 3024 3025 mana_rss_table_init(apc); 3026 3027 err = mana_config_rss(apc, TRI_STATE_TRUE, true, true); 3028 if (err) { 3029 netdev_err(ndev, "Failed to configure RSS table: %d\n", err); 3030 goto destroy_vport; 3031 } 3032 3033 if (gd->gdma_context->is_pf && !apc->ac->bm_hostmode) { 3034 err = mana_pf_register_filter(apc); 3035 if (err) 3036 goto destroy_vport; 3037 } 3038 3039 mana_chn_setxdp(apc, mana_xdp_get(apc)); 3040 3041 return 0; 3042 3043 destroy_vport: 3044 mana_destroy_vport(apc); 3045 return err; 3046 } 3047 3048 int mana_attach(struct net_device *ndev) 3049 { 3050 struct mana_port_context *apc = netdev_priv(ndev); 3051 int err; 3052 3053 ASSERT_RTNL(); 3054 3055 err = mana_init_port(ndev); 3056 if (err) 3057 return err; 3058 3059 if (apc->port_st_save) { 3060 err = mana_alloc_queues(ndev); 3061 if (err) { 3062 mana_cleanup_port_context(apc); 3063 return err; 3064 } 3065 } 3066 3067 apc->port_is_up = apc->port_st_save; 3068 3069 /* Ensure port state updated before txq state */ 3070 smp_wmb(); 3071 3072 if (apc->port_is_up) 3073 netif_carrier_on(ndev); 3074 3075 netif_device_attach(ndev); 3076 3077 return 0; 3078 } 3079 3080 static int mana_dealloc_queues(struct net_device *ndev) 3081 { 3082 struct mana_port_context *apc = netdev_priv(ndev); 3083 unsigned long timeout = jiffies + 120 * HZ; 3084 struct gdma_dev *gd = apc->ac->gdma_dev; 3085 struct mana_txq *txq; 3086 struct sk_buff *skb; 3087 int i, err; 3088 u32 tsleep; 3089 3090 if (apc->port_is_up) 3091 return -EINVAL; 3092 3093 mana_chn_setxdp(apc, NULL); 3094 3095 if (gd->gdma_context->is_pf && !apc->ac->bm_hostmode) 3096 mana_pf_deregister_filter(apc); 3097 3098 /* No packet can be transmitted now since apc->port_is_up is false. 3099 * There is still a tiny chance that mana_poll_tx_cq() can re-enable 3100 * a txq because it may not timely see apc->port_is_up being cleared 3101 * to false, but it doesn't matter since mana_start_xmit() drops any 3102 * new packets due to apc->port_is_up being false. 3103 * 3104 * Drain all the in-flight TX packets. 3105 * A timeout of 120 seconds for all the queues is used. 3106 * This will break the while loop when h/w is not responding. 3107 * This value of 120 has been decided here considering max 3108 * number of queues. 3109 */ 3110 3111 for (i = 0; i < apc->num_queues; i++) { 3112 txq = &apc->tx_qp[i].txq; 3113 tsleep = 1000; 3114 while (atomic_read(&txq->pending_sends) > 0 && 3115 time_before(jiffies, timeout)) { 3116 usleep_range(tsleep, tsleep + 1000); 3117 tsleep <<= 1; 3118 } 3119 if (atomic_read(&txq->pending_sends)) { 3120 err = pcie_flr(to_pci_dev(gd->gdma_context->dev)); 3121 if (err) { 3122 netdev_err(ndev, "flr failed %d with %d pkts pending in txq %u\n", 3123 err, atomic_read(&txq->pending_sends), 3124 txq->gdma_txq_id); 3125 } 3126 break; 3127 } 3128 } 3129 3130 for (i = 0; i < apc->num_queues; i++) { 3131 txq = &apc->tx_qp[i].txq; 3132 while ((skb = skb_dequeue(&txq->pending_skbs))) { 3133 mana_unmap_skb(skb, apc); 3134 dev_kfree_skb_any(skb); 3135 } 3136 atomic_set(&txq->pending_sends, 0); 3137 } 3138 /* We're 100% sure the queues can no longer be woken up, because 3139 * we're sure now mana_poll_tx_cq() can't be running. 3140 */ 3141 3142 apc->rss_state = TRI_STATE_FALSE; 3143 err = mana_config_rss(apc, TRI_STATE_FALSE, false, false); 3144 if (err && mana_en_need_log(apc, err)) 3145 netdev_err(ndev, "Failed to disable vPort: %d\n", err); 3146 3147 /* Even in err case, still need to cleanup the vPort */ 3148 mana_destroy_vport(apc); 3149 3150 return 0; 3151 } 3152 3153 int mana_detach(struct net_device *ndev, bool from_close) 3154 { 3155 struct mana_port_context *apc = netdev_priv(ndev); 3156 int err; 3157 3158 ASSERT_RTNL(); 3159 3160 apc->port_st_save = apc->port_is_up; 3161 apc->port_is_up = false; 3162 3163 /* Ensure port state updated before txq state */ 3164 smp_wmb(); 3165 3166 netif_tx_disable(ndev); 3167 netif_carrier_off(ndev); 3168 3169 if (apc->port_st_save) { 3170 err = mana_dealloc_queues(ndev); 3171 if (err) { 3172 netdev_err(ndev, "%s failed to deallocate queues: %d\n", __func__, err); 3173 return err; 3174 } 3175 } 3176 3177 if (!from_close) { 3178 netif_device_detach(ndev); 3179 mana_cleanup_port_context(apc); 3180 } 3181 3182 return 0; 3183 } 3184 3185 static int mana_probe_port(struct mana_context *ac, int port_idx, 3186 struct net_device **ndev_storage) 3187 { 3188 struct gdma_context *gc = ac->gdma_dev->gdma_context; 3189 struct mana_port_context *apc; 3190 struct net_device *ndev; 3191 int err; 3192 3193 ndev = alloc_etherdev_mq(sizeof(struct mana_port_context), 3194 gc->max_num_queues); 3195 if (!ndev) 3196 return -ENOMEM; 3197 3198 *ndev_storage = ndev; 3199 3200 apc = netdev_priv(ndev); 3201 apc->ac = ac; 3202 apc->ndev = ndev; 3203 apc->max_queues = gc->max_num_queues; 3204 apc->num_queues = gc->max_num_queues; 3205 apc->tx_queue_size = DEF_TX_BUFFERS_PER_QUEUE; 3206 apc->rx_queue_size = DEF_RX_BUFFERS_PER_QUEUE; 3207 apc->port_handle = INVALID_MANA_HANDLE; 3208 apc->pf_filter_handle = INVALID_MANA_HANDLE; 3209 apc->port_idx = port_idx; 3210 3211 mutex_init(&apc->vport_mutex); 3212 apc->vport_use_count = 0; 3213 3214 ndev->netdev_ops = &mana_devops; 3215 ndev->ethtool_ops = &mana_ethtool_ops; 3216 ndev->mtu = ETH_DATA_LEN; 3217 ndev->max_mtu = gc->adapter_mtu - ETH_HLEN; 3218 ndev->min_mtu = ETH_MIN_MTU; 3219 ndev->needed_headroom = MANA_HEADROOM; 3220 ndev->dev_port = port_idx; 3221 SET_NETDEV_DEV(ndev, gc->dev); 3222 3223 netif_set_tso_max_size(ndev, GSO_MAX_SIZE); 3224 3225 netif_carrier_off(ndev); 3226 3227 netdev_rss_key_fill(apc->hashkey, MANA_HASH_KEY_SIZE); 3228 3229 err = mana_init_port(ndev); 3230 if (err) 3231 goto free_net; 3232 3233 err = mana_rss_table_alloc(apc); 3234 if (err) 3235 goto reset_apc; 3236 3237 netdev_lockdep_set_classes(ndev); 3238 3239 ndev->hw_features = NETIF_F_SG | NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM; 3240 ndev->hw_features |= NETIF_F_RXCSUM; 3241 ndev->hw_features |= NETIF_F_TSO | NETIF_F_TSO6; 3242 ndev->hw_features |= NETIF_F_RXHASH; 3243 ndev->features = ndev->hw_features | NETIF_F_HW_VLAN_CTAG_TX | 3244 NETIF_F_HW_VLAN_CTAG_RX; 3245 ndev->vlan_features = ndev->features; 3246 xdp_set_features_flag(ndev, NETDEV_XDP_ACT_BASIC | 3247 NETDEV_XDP_ACT_REDIRECT | 3248 NETDEV_XDP_ACT_NDO_XMIT); 3249 3250 err = register_netdev(ndev); 3251 if (err) { 3252 netdev_err(ndev, "Unable to register netdev.\n"); 3253 goto free_indir; 3254 } 3255 3256 debugfs_create_u32("current_speed", 0400, apc->mana_port_debugfs, &apc->speed); 3257 3258 return 0; 3259 3260 free_indir: 3261 mana_cleanup_indir_table(apc); 3262 reset_apc: 3263 mana_cleanup_port_context(apc); 3264 free_net: 3265 *ndev_storage = NULL; 3266 netdev_err(ndev, "Failed to probe vPort %d: %d\n", port_idx, err); 3267 free_netdev(ndev); 3268 return err; 3269 } 3270 3271 static void adev_release(struct device *dev) 3272 { 3273 struct mana_adev *madev = container_of(dev, struct mana_adev, adev.dev); 3274 3275 kfree(madev); 3276 } 3277 3278 static void remove_adev(struct gdma_dev *gd) 3279 { 3280 struct auxiliary_device *adev = gd->adev; 3281 int id = adev->id; 3282 3283 auxiliary_device_delete(adev); 3284 auxiliary_device_uninit(adev); 3285 3286 mana_adev_idx_free(id); 3287 gd->adev = NULL; 3288 } 3289 3290 static int add_adev(struct gdma_dev *gd, const char *name) 3291 { 3292 struct auxiliary_device *adev; 3293 struct mana_adev *madev; 3294 int ret; 3295 3296 madev = kzalloc(sizeof(*madev), GFP_KERNEL); 3297 if (!madev) 3298 return -ENOMEM; 3299 3300 adev = &madev->adev; 3301 ret = mana_adev_idx_alloc(); 3302 if (ret < 0) 3303 goto idx_fail; 3304 adev->id = ret; 3305 3306 adev->name = name; 3307 adev->dev.parent = gd->gdma_context->dev; 3308 adev->dev.release = adev_release; 3309 madev->mdev = gd; 3310 3311 ret = auxiliary_device_init(adev); 3312 if (ret) 3313 goto init_fail; 3314 3315 /* madev is owned by the auxiliary device */ 3316 madev = NULL; 3317 ret = auxiliary_device_add(adev); 3318 if (ret) 3319 goto add_fail; 3320 3321 gd->adev = adev; 3322 dev_dbg(gd->gdma_context->dev, 3323 "Auxiliary device added successfully\n"); 3324 return 0; 3325 3326 add_fail: 3327 auxiliary_device_uninit(adev); 3328 3329 init_fail: 3330 mana_adev_idx_free(adev->id); 3331 3332 idx_fail: 3333 kfree(madev); 3334 3335 return ret; 3336 } 3337 3338 static void mana_rdma_service_handle(struct work_struct *work) 3339 { 3340 struct mana_service_work *serv_work = 3341 container_of(work, struct mana_service_work, work); 3342 struct gdma_dev *gd = serv_work->gdma_dev; 3343 struct device *dev = gd->gdma_context->dev; 3344 int ret; 3345 3346 if (READ_ONCE(gd->rdma_teardown)) 3347 goto out; 3348 3349 switch (serv_work->event) { 3350 case GDMA_SERVICE_TYPE_RDMA_SUSPEND: 3351 if (!gd->adev || gd->is_suspended) 3352 break; 3353 3354 remove_adev(gd); 3355 gd->is_suspended = true; 3356 break; 3357 3358 case GDMA_SERVICE_TYPE_RDMA_RESUME: 3359 if (!gd->is_suspended) 3360 break; 3361 3362 ret = add_adev(gd, "rdma"); 3363 if (ret) 3364 dev_err(dev, "Failed to add adev on resume: %d\n", ret); 3365 else 3366 gd->is_suspended = false; 3367 break; 3368 3369 default: 3370 dev_warn(dev, "unknown adev service event %u\n", 3371 serv_work->event); 3372 break; 3373 } 3374 3375 out: 3376 kfree(serv_work); 3377 } 3378 3379 int mana_rdma_service_event(struct gdma_context *gc, enum gdma_service_type event) 3380 { 3381 struct gdma_dev *gd = &gc->mana_ib; 3382 struct mana_service_work *serv_work; 3383 3384 if (gd->dev_id.type != GDMA_DEVICE_MANA_IB) { 3385 /* RDMA device is not detected on pci */ 3386 return 0; 3387 } 3388 3389 serv_work = kzalloc(sizeof(*serv_work), GFP_ATOMIC); 3390 if (!serv_work) 3391 return -ENOMEM; 3392 3393 serv_work->event = event; 3394 serv_work->gdma_dev = gd; 3395 3396 INIT_WORK(&serv_work->work, mana_rdma_service_handle); 3397 queue_work(gc->service_wq, &serv_work->work); 3398 3399 return 0; 3400 } 3401 3402 int mana_probe(struct gdma_dev *gd, bool resuming) 3403 { 3404 struct gdma_context *gc = gd->gdma_context; 3405 struct mana_context *ac = gd->driver_data; 3406 struct device *dev = gc->dev; 3407 u8 bm_hostmode = 0; 3408 u16 num_ports = 0; 3409 int err; 3410 int i; 3411 3412 dev_info(dev, 3413 "Microsoft Azure Network Adapter protocol version: %d.%d.%d\n", 3414 MANA_MAJOR_VERSION, MANA_MINOR_VERSION, MANA_MICRO_VERSION); 3415 3416 err = mana_gd_register_device(gd); 3417 if (err) 3418 return err; 3419 3420 if (!resuming) { 3421 ac = kzalloc(sizeof(*ac), GFP_KERNEL); 3422 if (!ac) 3423 return -ENOMEM; 3424 3425 ac->gdma_dev = gd; 3426 gd->driver_data = ac; 3427 } 3428 3429 err = mana_create_eq(ac); 3430 if (err) { 3431 dev_err(dev, "Failed to create EQs: %d\n", err); 3432 goto out; 3433 } 3434 3435 err = mana_query_device_cfg(ac, MANA_MAJOR_VERSION, MANA_MINOR_VERSION, 3436 MANA_MICRO_VERSION, &num_ports, &bm_hostmode); 3437 if (err) 3438 goto out; 3439 3440 ac->bm_hostmode = bm_hostmode; 3441 3442 if (!resuming) { 3443 ac->num_ports = num_ports; 3444 } else { 3445 if (ac->num_ports != num_ports) { 3446 dev_err(dev, "The number of vPorts changed: %d->%d\n", 3447 ac->num_ports, num_ports); 3448 err = -EPROTO; 3449 goto out; 3450 } 3451 } 3452 3453 if (ac->num_ports == 0) 3454 dev_err(dev, "Failed to detect any vPort\n"); 3455 3456 if (ac->num_ports > MAX_PORTS_IN_MANA_DEV) 3457 ac->num_ports = MAX_PORTS_IN_MANA_DEV; 3458 3459 if (!resuming) { 3460 for (i = 0; i < ac->num_ports; i++) { 3461 err = mana_probe_port(ac, i, &ac->ports[i]); 3462 /* we log the port for which the probe failed and stop 3463 * probes for subsequent ports. 3464 * Note that we keep running ports, for which the probes 3465 * were successful, unless add_adev fails too 3466 */ 3467 if (err) { 3468 dev_err(dev, "Probe Failed for port %d\n", i); 3469 break; 3470 } 3471 } 3472 } else { 3473 for (i = 0; i < ac->num_ports; i++) { 3474 rtnl_lock(); 3475 err = mana_attach(ac->ports[i]); 3476 rtnl_unlock(); 3477 /* we log the port for which the attach failed and stop 3478 * attach for subsequent ports 3479 * Note that we keep running ports, for which the attach 3480 * were successful, unless add_adev fails too 3481 */ 3482 if (err) { 3483 dev_err(dev, "Attach Failed for port %d\n", i); 3484 break; 3485 } 3486 } 3487 } 3488 3489 err = add_adev(gd, "eth"); 3490 out: 3491 if (err) { 3492 mana_remove(gd, false); 3493 } else { 3494 dev_dbg(dev, "gd=%p, id=%u, num_ports=%d, type=%u, instance=%u\n", 3495 gd, gd->dev_id.as_uint32, ac->num_ports, 3496 gd->dev_id.type, gd->dev_id.instance); 3497 dev_dbg(dev, "%s succeeded\n", __func__); 3498 } 3499 3500 return err; 3501 } 3502 3503 void mana_remove(struct gdma_dev *gd, bool suspending) 3504 { 3505 struct gdma_context *gc = gd->gdma_context; 3506 struct mana_context *ac = gd->driver_data; 3507 struct mana_port_context *apc; 3508 struct device *dev = gc->dev; 3509 struct net_device *ndev; 3510 int err; 3511 int i; 3512 3513 /* adev currently doesn't support suspending, always remove it */ 3514 if (gd->adev) 3515 remove_adev(gd); 3516 3517 for (i = 0; i < ac->num_ports; i++) { 3518 ndev = ac->ports[i]; 3519 apc = netdev_priv(ndev); 3520 if (!ndev) { 3521 if (i == 0) 3522 dev_err(dev, "No net device to remove\n"); 3523 goto out; 3524 } 3525 3526 /* All cleanup actions should stay after rtnl_lock(), otherwise 3527 * other functions may access partially cleaned up data. 3528 */ 3529 rtnl_lock(); 3530 3531 err = mana_detach(ndev, false); 3532 if (err) 3533 netdev_err(ndev, "Failed to detach vPort %d: %d\n", 3534 i, err); 3535 3536 if (suspending) { 3537 /* No need to unregister the ndev. */ 3538 rtnl_unlock(); 3539 continue; 3540 } 3541 3542 unregister_netdevice(ndev); 3543 mana_cleanup_indir_table(apc); 3544 3545 rtnl_unlock(); 3546 3547 free_netdev(ndev); 3548 } 3549 3550 mana_destroy_eq(ac); 3551 out: 3552 mana_gd_deregister_device(gd); 3553 3554 if (suspending) 3555 return; 3556 3557 gd->driver_data = NULL; 3558 gd->gdma_context = NULL; 3559 kfree(ac); 3560 dev_dbg(dev, "%s succeeded\n", __func__); 3561 } 3562 3563 int mana_rdma_probe(struct gdma_dev *gd) 3564 { 3565 int err = 0; 3566 3567 if (gd->dev_id.type != GDMA_DEVICE_MANA_IB) { 3568 /* RDMA device is not detected on pci */ 3569 return err; 3570 } 3571 3572 err = mana_gd_register_device(gd); 3573 if (err) 3574 return err; 3575 3576 err = add_adev(gd, "rdma"); 3577 if (err) 3578 mana_gd_deregister_device(gd); 3579 3580 return err; 3581 } 3582 3583 void mana_rdma_remove(struct gdma_dev *gd) 3584 { 3585 struct gdma_context *gc = gd->gdma_context; 3586 3587 if (gd->dev_id.type != GDMA_DEVICE_MANA_IB) { 3588 /* RDMA device is not detected on pci */ 3589 return; 3590 } 3591 3592 WRITE_ONCE(gd->rdma_teardown, true); 3593 flush_workqueue(gc->service_wq); 3594 3595 if (gd->adev) 3596 remove_adev(gd); 3597 3598 mana_gd_deregister_device(gd); 3599 } 3600 3601 struct net_device *mana_get_primary_netdev(struct mana_context *ac, 3602 u32 port_index, 3603 netdevice_tracker *tracker) 3604 { 3605 struct net_device *ndev; 3606 3607 if (port_index >= ac->num_ports) 3608 return NULL; 3609 3610 rcu_read_lock(); 3611 3612 /* If mana is used in netvsc, the upper netdevice should be returned. */ 3613 ndev = netdev_master_upper_dev_get_rcu(ac->ports[port_index]); 3614 3615 /* If there is no upper device, use the parent Ethernet device */ 3616 if (!ndev) 3617 ndev = ac->ports[port_index]; 3618 3619 netdev_hold(ndev, tracker, GFP_ATOMIC); 3620 rcu_read_unlock(); 3621 3622 return ndev; 3623 } 3624 EXPORT_SYMBOL_NS(mana_get_primary_netdev, "NET_MANA"); 3625