1 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause 2 /* Copyright (c) 2021, Microsoft Corporation. */ 3 4 #include <uapi/linux/bpf.h> 5 6 #include <linux/debugfs.h> 7 #include <linux/inetdevice.h> 8 #include <linux/etherdevice.h> 9 #include <linux/ethtool.h> 10 #include <linux/filter.h> 11 #include <linux/mm.h> 12 #include <linux/pci.h> 13 #include <linux/export.h> 14 15 #include <net/checksum.h> 16 #include <net/ip6_checksum.h> 17 #include <net/netdev_lock.h> 18 #include <net/page_pool/helpers.h> 19 #include <net/xdp.h> 20 21 #include <net/mana/mana.h> 22 #include <net/mana/mana_auxiliary.h> 23 24 static DEFINE_IDA(mana_adev_ida); 25 26 static int mana_adev_idx_alloc(void) 27 { 28 return ida_alloc(&mana_adev_ida, GFP_KERNEL); 29 } 30 31 static void mana_adev_idx_free(int idx) 32 { 33 ida_free(&mana_adev_ida, idx); 34 } 35 36 static ssize_t mana_dbg_q_read(struct file *filp, char __user *buf, size_t count, 37 loff_t *pos) 38 { 39 struct gdma_queue *gdma_q = filp->private_data; 40 41 return simple_read_from_buffer(buf, count, pos, gdma_q->queue_mem_ptr, 42 gdma_q->queue_size); 43 } 44 45 static const struct file_operations mana_dbg_q_fops = { 46 .owner = THIS_MODULE, 47 .open = simple_open, 48 .read = mana_dbg_q_read, 49 }; 50 51 static bool mana_en_need_log(struct mana_port_context *apc, int err) 52 { 53 if (apc && apc->ac && apc->ac->gdma_dev && 54 apc->ac->gdma_dev->gdma_context) 55 return mana_need_log(apc->ac->gdma_dev->gdma_context, err); 56 else 57 return true; 58 } 59 60 static void mana_put_rx_page(struct mana_rxq *rxq, struct page *page, 61 bool from_pool) 62 { 63 if (from_pool) 64 page_pool_put_full_page(rxq->page_pool, page, false); 65 else 66 put_page(page); 67 } 68 69 /* Microsoft Azure Network Adapter (MANA) functions */ 70 71 static int mana_open(struct net_device *ndev) 72 { 73 struct mana_port_context *apc = netdev_priv(ndev); 74 int err; 75 err = mana_alloc_queues(ndev); 76 77 if (err) { 78 netdev_err(ndev, "%s failed to allocate queues: %d\n", __func__, err); 79 return err; 80 } 81 82 apc->port_is_up = true; 83 84 /* Ensure port state updated before txq state */ 85 smp_wmb(); 86 87 netif_carrier_on(ndev); 88 netif_tx_wake_all_queues(ndev); 89 netdev_dbg(ndev, "%s successful\n", __func__); 90 return 0; 91 } 92 93 static int mana_close(struct net_device *ndev) 94 { 95 struct mana_port_context *apc = netdev_priv(ndev); 96 97 if (!apc->port_is_up) 98 return 0; 99 100 return mana_detach(ndev, true); 101 } 102 103 static bool mana_can_tx(struct gdma_queue *wq) 104 { 105 return mana_gd_wq_avail_space(wq) >= MAX_TX_WQE_SIZE; 106 } 107 108 static unsigned int mana_checksum_info(struct sk_buff *skb) 109 { 110 if (skb->protocol == htons(ETH_P_IP)) { 111 struct iphdr *ip = ip_hdr(skb); 112 113 if (ip->protocol == IPPROTO_TCP) 114 return IPPROTO_TCP; 115 116 if (ip->protocol == IPPROTO_UDP) 117 return IPPROTO_UDP; 118 } else if (skb->protocol == htons(ETH_P_IPV6)) { 119 struct ipv6hdr *ip6 = ipv6_hdr(skb); 120 121 if (ip6->nexthdr == IPPROTO_TCP) 122 return IPPROTO_TCP; 123 124 if (ip6->nexthdr == IPPROTO_UDP) 125 return IPPROTO_UDP; 126 } 127 128 /* No csum offloading */ 129 return 0; 130 } 131 132 static void mana_add_sge(struct mana_tx_package *tp, struct mana_skb_head *ash, 133 int sg_i, dma_addr_t da, int sge_len, u32 gpa_mkey) 134 { 135 ash->dma_handle[sg_i] = da; 136 ash->size[sg_i] = sge_len; 137 138 tp->wqe_req.sgl[sg_i].address = da; 139 tp->wqe_req.sgl[sg_i].mem_key = gpa_mkey; 140 tp->wqe_req.sgl[sg_i].size = sge_len; 141 } 142 143 static int mana_map_skb(struct sk_buff *skb, struct mana_port_context *apc, 144 struct mana_tx_package *tp, int gso_hs) 145 { 146 struct mana_skb_head *ash = (struct mana_skb_head *)skb->head; 147 int hsg = 1; /* num of SGEs of linear part */ 148 struct gdma_dev *gd = apc->ac->gdma_dev; 149 int skb_hlen = skb_headlen(skb); 150 int sge0_len, sge1_len = 0; 151 struct gdma_context *gc; 152 struct device *dev; 153 skb_frag_t *frag; 154 dma_addr_t da; 155 int sg_i; 156 int i; 157 158 gc = gd->gdma_context; 159 dev = gc->dev; 160 161 if (gso_hs && gso_hs < skb_hlen) { 162 sge0_len = gso_hs; 163 sge1_len = skb_hlen - gso_hs; 164 } else { 165 sge0_len = skb_hlen; 166 } 167 168 da = dma_map_single(dev, skb->data, sge0_len, DMA_TO_DEVICE); 169 if (dma_mapping_error(dev, da)) 170 return -ENOMEM; 171 172 mana_add_sge(tp, ash, 0, da, sge0_len, gd->gpa_mkey); 173 174 if (sge1_len) { 175 sg_i = 1; 176 da = dma_map_single(dev, skb->data + sge0_len, sge1_len, 177 DMA_TO_DEVICE); 178 if (dma_mapping_error(dev, da)) 179 goto frag_err; 180 181 mana_add_sge(tp, ash, sg_i, da, sge1_len, gd->gpa_mkey); 182 hsg = 2; 183 } 184 185 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 186 sg_i = hsg + i; 187 188 frag = &skb_shinfo(skb)->frags[i]; 189 da = skb_frag_dma_map(dev, frag, 0, skb_frag_size(frag), 190 DMA_TO_DEVICE); 191 if (dma_mapping_error(dev, da)) 192 goto frag_err; 193 194 mana_add_sge(tp, ash, sg_i, da, skb_frag_size(frag), 195 gd->gpa_mkey); 196 } 197 198 return 0; 199 200 frag_err: 201 if (net_ratelimit()) 202 netdev_err(apc->ndev, "Failed to map skb of size %u to DMA\n", 203 skb->len); 204 for (i = sg_i - 1; i >= hsg; i--) 205 dma_unmap_page(dev, ash->dma_handle[i], ash->size[i], 206 DMA_TO_DEVICE); 207 208 for (i = hsg - 1; i >= 0; i--) 209 dma_unmap_single(dev, ash->dma_handle[i], ash->size[i], 210 DMA_TO_DEVICE); 211 212 return -ENOMEM; 213 } 214 215 /* Handle the case when GSO SKB linear length is too large. 216 * MANA NIC requires GSO packets to put only the packet header to SGE0. 217 * So, we need 2 SGEs for the skb linear part which contains more than the 218 * header. 219 * Return a positive value for the number of SGEs, or a negative value 220 * for an error. 221 */ 222 static int mana_fix_skb_head(struct net_device *ndev, struct sk_buff *skb, 223 int gso_hs) 224 { 225 int num_sge = 1 + skb_shinfo(skb)->nr_frags; 226 int skb_hlen = skb_headlen(skb); 227 228 if (gso_hs < skb_hlen) { 229 num_sge++; 230 } else if (gso_hs > skb_hlen) { 231 if (net_ratelimit()) 232 netdev_err(ndev, 233 "TX nonlinear head: hs:%d, skb_hlen:%d\n", 234 gso_hs, skb_hlen); 235 236 return -EINVAL; 237 } 238 239 return num_sge; 240 } 241 242 /* Get the GSO packet's header size */ 243 static int mana_get_gso_hs(struct sk_buff *skb) 244 { 245 int gso_hs; 246 247 if (skb->encapsulation) { 248 gso_hs = skb_inner_tcp_all_headers(skb); 249 } else { 250 if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4) { 251 gso_hs = skb_transport_offset(skb) + 252 sizeof(struct udphdr); 253 } else { 254 gso_hs = skb_tcp_all_headers(skb); 255 } 256 } 257 258 return gso_hs; 259 } 260 261 netdev_tx_t mana_start_xmit(struct sk_buff *skb, struct net_device *ndev) 262 { 263 enum mana_tx_pkt_format pkt_fmt = MANA_SHORT_PKT_FMT; 264 struct mana_port_context *apc = netdev_priv(ndev); 265 int gso_hs = 0; /* zero for non-GSO pkts */ 266 u16 txq_idx = skb_get_queue_mapping(skb); 267 struct gdma_dev *gd = apc->ac->gdma_dev; 268 bool ipv4 = false, ipv6 = false; 269 struct mana_tx_package pkg = {}; 270 struct netdev_queue *net_txq; 271 struct mana_stats_tx *tx_stats; 272 struct gdma_queue *gdma_sq; 273 int err, len, num_gso_seg; 274 unsigned int csum_type; 275 struct mana_txq *txq; 276 struct mana_cq *cq; 277 278 if (unlikely(!apc->port_is_up)) 279 goto tx_drop; 280 281 if (skb_cow_head(skb, MANA_HEADROOM)) 282 goto tx_drop_count; 283 284 if (unlikely(ipv6_hopopt_jumbo_remove(skb))) 285 goto tx_drop_count; 286 287 txq = &apc->tx_qp[txq_idx].txq; 288 gdma_sq = txq->gdma_sq; 289 cq = &apc->tx_qp[txq_idx].tx_cq; 290 tx_stats = &txq->stats; 291 292 pkg.tx_oob.s_oob.vcq_num = cq->gdma_id; 293 pkg.tx_oob.s_oob.vsq_frame = txq->vsq_frame; 294 295 if (txq->vp_offset > MANA_SHORT_VPORT_OFFSET_MAX) { 296 pkg.tx_oob.l_oob.long_vp_offset = txq->vp_offset; 297 pkt_fmt = MANA_LONG_PKT_FMT; 298 } else { 299 pkg.tx_oob.s_oob.short_vp_offset = txq->vp_offset; 300 } 301 302 if (skb_vlan_tag_present(skb)) { 303 pkt_fmt = MANA_LONG_PKT_FMT; 304 pkg.tx_oob.l_oob.inject_vlan_pri_tag = 1; 305 pkg.tx_oob.l_oob.pcp = skb_vlan_tag_get_prio(skb); 306 pkg.tx_oob.l_oob.dei = skb_vlan_tag_get_cfi(skb); 307 pkg.tx_oob.l_oob.vlan_id = skb_vlan_tag_get_id(skb); 308 } 309 310 pkg.tx_oob.s_oob.pkt_fmt = pkt_fmt; 311 312 if (pkt_fmt == MANA_SHORT_PKT_FMT) { 313 pkg.wqe_req.inline_oob_size = sizeof(struct mana_tx_short_oob); 314 u64_stats_update_begin(&tx_stats->syncp); 315 tx_stats->short_pkt_fmt++; 316 u64_stats_update_end(&tx_stats->syncp); 317 } else { 318 pkg.wqe_req.inline_oob_size = sizeof(struct mana_tx_oob); 319 u64_stats_update_begin(&tx_stats->syncp); 320 tx_stats->long_pkt_fmt++; 321 u64_stats_update_end(&tx_stats->syncp); 322 } 323 324 pkg.wqe_req.inline_oob_data = &pkg.tx_oob; 325 pkg.wqe_req.flags = 0; 326 pkg.wqe_req.client_data_unit = 0; 327 328 pkg.wqe_req.num_sge = 1 + skb_shinfo(skb)->nr_frags; 329 330 if (skb->protocol == htons(ETH_P_IP)) 331 ipv4 = true; 332 else if (skb->protocol == htons(ETH_P_IPV6)) 333 ipv6 = true; 334 335 if (skb_is_gso(skb)) { 336 int num_sge; 337 338 gso_hs = mana_get_gso_hs(skb); 339 340 num_sge = mana_fix_skb_head(ndev, skb, gso_hs); 341 if (num_sge > 0) 342 pkg.wqe_req.num_sge = num_sge; 343 else 344 goto tx_drop_count; 345 346 u64_stats_update_begin(&tx_stats->syncp); 347 if (skb->encapsulation) { 348 tx_stats->tso_inner_packets++; 349 tx_stats->tso_inner_bytes += skb->len - gso_hs; 350 } else { 351 tx_stats->tso_packets++; 352 tx_stats->tso_bytes += skb->len - gso_hs; 353 } 354 u64_stats_update_end(&tx_stats->syncp); 355 356 pkg.tx_oob.s_oob.is_outer_ipv4 = ipv4; 357 pkg.tx_oob.s_oob.is_outer_ipv6 = ipv6; 358 359 pkg.tx_oob.s_oob.comp_iphdr_csum = 1; 360 pkg.tx_oob.s_oob.comp_tcp_csum = 1; 361 pkg.tx_oob.s_oob.trans_off = skb_transport_offset(skb); 362 363 pkg.wqe_req.client_data_unit = skb_shinfo(skb)->gso_size; 364 pkg.wqe_req.flags = GDMA_WR_OOB_IN_SGL | GDMA_WR_PAD_BY_SGE0; 365 if (ipv4) { 366 ip_hdr(skb)->tot_len = 0; 367 ip_hdr(skb)->check = 0; 368 tcp_hdr(skb)->check = 369 ~csum_tcpudp_magic(ip_hdr(skb)->saddr, 370 ip_hdr(skb)->daddr, 0, 371 IPPROTO_TCP, 0); 372 } else { 373 ipv6_hdr(skb)->payload_len = 0; 374 tcp_hdr(skb)->check = 375 ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr, 376 &ipv6_hdr(skb)->daddr, 0, 377 IPPROTO_TCP, 0); 378 } 379 } else if (skb->ip_summed == CHECKSUM_PARTIAL) { 380 csum_type = mana_checksum_info(skb); 381 382 u64_stats_update_begin(&tx_stats->syncp); 383 tx_stats->csum_partial++; 384 u64_stats_update_end(&tx_stats->syncp); 385 386 if (csum_type == IPPROTO_TCP) { 387 pkg.tx_oob.s_oob.is_outer_ipv4 = ipv4; 388 pkg.tx_oob.s_oob.is_outer_ipv6 = ipv6; 389 390 pkg.tx_oob.s_oob.comp_tcp_csum = 1; 391 pkg.tx_oob.s_oob.trans_off = skb_transport_offset(skb); 392 393 } else if (csum_type == IPPROTO_UDP) { 394 pkg.tx_oob.s_oob.is_outer_ipv4 = ipv4; 395 pkg.tx_oob.s_oob.is_outer_ipv6 = ipv6; 396 397 pkg.tx_oob.s_oob.comp_udp_csum = 1; 398 } else { 399 /* Can't do offload of this type of checksum */ 400 if (skb_checksum_help(skb)) 401 goto tx_drop_count; 402 } 403 } 404 405 WARN_ON_ONCE(pkg.wqe_req.num_sge > MAX_TX_WQE_SGL_ENTRIES); 406 407 if (pkg.wqe_req.num_sge <= ARRAY_SIZE(pkg.sgl_array)) { 408 pkg.wqe_req.sgl = pkg.sgl_array; 409 } else { 410 pkg.sgl_ptr = kmalloc_array(pkg.wqe_req.num_sge, 411 sizeof(struct gdma_sge), 412 GFP_ATOMIC); 413 if (!pkg.sgl_ptr) 414 goto tx_drop_count; 415 416 pkg.wqe_req.sgl = pkg.sgl_ptr; 417 } 418 419 if (mana_map_skb(skb, apc, &pkg, gso_hs)) { 420 u64_stats_update_begin(&tx_stats->syncp); 421 tx_stats->mana_map_err++; 422 u64_stats_update_end(&tx_stats->syncp); 423 goto free_sgl_ptr; 424 } 425 426 skb_queue_tail(&txq->pending_skbs, skb); 427 428 len = skb->len; 429 num_gso_seg = skb_is_gso(skb) ? skb_shinfo(skb)->gso_segs : 1; 430 net_txq = netdev_get_tx_queue(ndev, txq_idx); 431 432 err = mana_gd_post_work_request(gdma_sq, &pkg.wqe_req, 433 (struct gdma_posted_wqe_info *)skb->cb); 434 if (!mana_can_tx(gdma_sq)) { 435 netif_tx_stop_queue(net_txq); 436 apc->eth_stats.stop_queue++; 437 } 438 439 if (err) { 440 (void)skb_dequeue_tail(&txq->pending_skbs); 441 netdev_warn(ndev, "Failed to post TX OOB: %d\n", err); 442 err = NETDEV_TX_BUSY; 443 goto tx_busy; 444 } 445 446 err = NETDEV_TX_OK; 447 atomic_inc(&txq->pending_sends); 448 449 mana_gd_wq_ring_doorbell(gd->gdma_context, gdma_sq); 450 451 /* skb may be freed after mana_gd_post_work_request. Do not use it. */ 452 skb = NULL; 453 454 /* Populated the packet and bytes counters based on post GSO packet 455 * calculations 456 */ 457 tx_stats = &txq->stats; 458 u64_stats_update_begin(&tx_stats->syncp); 459 tx_stats->packets += num_gso_seg; 460 tx_stats->bytes += len + ((num_gso_seg - 1) * gso_hs); 461 u64_stats_update_end(&tx_stats->syncp); 462 463 tx_busy: 464 if (netif_tx_queue_stopped(net_txq) && mana_can_tx(gdma_sq)) { 465 netif_tx_wake_queue(net_txq); 466 apc->eth_stats.wake_queue++; 467 } 468 469 kfree(pkg.sgl_ptr); 470 return err; 471 472 free_sgl_ptr: 473 kfree(pkg.sgl_ptr); 474 tx_drop_count: 475 ndev->stats.tx_dropped++; 476 tx_drop: 477 dev_kfree_skb_any(skb); 478 return NETDEV_TX_OK; 479 } 480 481 static void mana_get_stats64(struct net_device *ndev, 482 struct rtnl_link_stats64 *st) 483 { 484 struct mana_port_context *apc = netdev_priv(ndev); 485 unsigned int num_queues = apc->num_queues; 486 struct mana_stats_rx *rx_stats; 487 struct mana_stats_tx *tx_stats; 488 unsigned int start; 489 u64 packets, bytes; 490 int q; 491 492 if (!apc->port_is_up) 493 return; 494 495 netdev_stats_to_stats64(st, &ndev->stats); 496 497 for (q = 0; q < num_queues; q++) { 498 rx_stats = &apc->rxqs[q]->stats; 499 500 do { 501 start = u64_stats_fetch_begin(&rx_stats->syncp); 502 packets = rx_stats->packets; 503 bytes = rx_stats->bytes; 504 } while (u64_stats_fetch_retry(&rx_stats->syncp, start)); 505 506 st->rx_packets += packets; 507 st->rx_bytes += bytes; 508 } 509 510 for (q = 0; q < num_queues; q++) { 511 tx_stats = &apc->tx_qp[q].txq.stats; 512 513 do { 514 start = u64_stats_fetch_begin(&tx_stats->syncp); 515 packets = tx_stats->packets; 516 bytes = tx_stats->bytes; 517 } while (u64_stats_fetch_retry(&tx_stats->syncp, start)); 518 519 st->tx_packets += packets; 520 st->tx_bytes += bytes; 521 } 522 } 523 524 static int mana_get_tx_queue(struct net_device *ndev, struct sk_buff *skb, 525 int old_q) 526 { 527 struct mana_port_context *apc = netdev_priv(ndev); 528 u32 hash = skb_get_hash(skb); 529 struct sock *sk = skb->sk; 530 int txq; 531 532 txq = apc->indir_table[hash & (apc->indir_table_sz - 1)]; 533 534 if (txq != old_q && sk && sk_fullsock(sk) && 535 rcu_access_pointer(sk->sk_dst_cache)) 536 sk_tx_queue_set(sk, txq); 537 538 return txq; 539 } 540 541 static u16 mana_select_queue(struct net_device *ndev, struct sk_buff *skb, 542 struct net_device *sb_dev) 543 { 544 int txq; 545 546 if (ndev->real_num_tx_queues == 1) 547 return 0; 548 549 txq = sk_tx_queue_get(skb->sk); 550 551 if (txq < 0 || skb->ooo_okay || txq >= ndev->real_num_tx_queues) { 552 if (skb_rx_queue_recorded(skb)) 553 txq = skb_get_rx_queue(skb); 554 else 555 txq = mana_get_tx_queue(ndev, skb, txq); 556 } 557 558 return txq; 559 } 560 561 /* Release pre-allocated RX buffers */ 562 void mana_pre_dealloc_rxbufs(struct mana_port_context *mpc) 563 { 564 struct device *dev; 565 int i; 566 567 dev = mpc->ac->gdma_dev->gdma_context->dev; 568 569 if (!mpc->rxbufs_pre) 570 goto out1; 571 572 if (!mpc->das_pre) 573 goto out2; 574 575 while (mpc->rxbpre_total) { 576 i = --mpc->rxbpre_total; 577 dma_unmap_single(dev, mpc->das_pre[i], mpc->rxbpre_datasize, 578 DMA_FROM_DEVICE); 579 put_page(virt_to_head_page(mpc->rxbufs_pre[i])); 580 } 581 582 kfree(mpc->das_pre); 583 mpc->das_pre = NULL; 584 585 out2: 586 kfree(mpc->rxbufs_pre); 587 mpc->rxbufs_pre = NULL; 588 589 out1: 590 mpc->rxbpre_datasize = 0; 591 mpc->rxbpre_alloc_size = 0; 592 mpc->rxbpre_headroom = 0; 593 } 594 595 /* Get a buffer from the pre-allocated RX buffers */ 596 static void *mana_get_rxbuf_pre(struct mana_rxq *rxq, dma_addr_t *da) 597 { 598 struct net_device *ndev = rxq->ndev; 599 struct mana_port_context *mpc; 600 void *va; 601 602 mpc = netdev_priv(ndev); 603 604 if (!mpc->rxbufs_pre || !mpc->das_pre || !mpc->rxbpre_total) { 605 netdev_err(ndev, "No RX pre-allocated bufs\n"); 606 return NULL; 607 } 608 609 /* Check sizes to catch unexpected coding error */ 610 if (mpc->rxbpre_datasize != rxq->datasize) { 611 netdev_err(ndev, "rxbpre_datasize mismatch: %u: %u\n", 612 mpc->rxbpre_datasize, rxq->datasize); 613 return NULL; 614 } 615 616 if (mpc->rxbpre_alloc_size != rxq->alloc_size) { 617 netdev_err(ndev, "rxbpre_alloc_size mismatch: %u: %u\n", 618 mpc->rxbpre_alloc_size, rxq->alloc_size); 619 return NULL; 620 } 621 622 if (mpc->rxbpre_headroom != rxq->headroom) { 623 netdev_err(ndev, "rxbpre_headroom mismatch: %u: %u\n", 624 mpc->rxbpre_headroom, rxq->headroom); 625 return NULL; 626 } 627 628 mpc->rxbpre_total--; 629 630 *da = mpc->das_pre[mpc->rxbpre_total]; 631 va = mpc->rxbufs_pre[mpc->rxbpre_total]; 632 mpc->rxbufs_pre[mpc->rxbpre_total] = NULL; 633 634 /* Deallocate the array after all buffers are gone */ 635 if (!mpc->rxbpre_total) 636 mana_pre_dealloc_rxbufs(mpc); 637 638 return va; 639 } 640 641 /* Get RX buffer's data size, alloc size, XDP headroom based on MTU */ 642 static void mana_get_rxbuf_cfg(struct mana_port_context *apc, 643 int mtu, u32 *datasize, u32 *alloc_size, 644 u32 *headroom, u32 *frag_count) 645 { 646 u32 len, buf_size; 647 648 /* Calculate datasize first (consistent across all cases) */ 649 *datasize = mtu + ETH_HLEN; 650 651 /* For xdp and jumbo frames make sure only one packet fits per page */ 652 if (mtu + MANA_RXBUF_PAD > PAGE_SIZE / 2 || mana_xdp_get(apc)) { 653 if (mana_xdp_get(apc)) { 654 *headroom = XDP_PACKET_HEADROOM; 655 *alloc_size = PAGE_SIZE; 656 } else { 657 *headroom = 0; /* no support for XDP */ 658 *alloc_size = SKB_DATA_ALIGN(mtu + MANA_RXBUF_PAD + 659 *headroom); 660 } 661 662 *frag_count = 1; 663 return; 664 } 665 666 /* Standard MTU case - optimize for multiple packets per page */ 667 *headroom = 0; 668 669 /* Calculate base buffer size needed */ 670 len = SKB_DATA_ALIGN(mtu + MANA_RXBUF_PAD + *headroom); 671 buf_size = ALIGN(len, MANA_RX_FRAG_ALIGNMENT); 672 673 /* Calculate how many packets can fit in a page */ 674 *frag_count = PAGE_SIZE / buf_size; 675 *alloc_size = buf_size; 676 } 677 678 int mana_pre_alloc_rxbufs(struct mana_port_context *mpc, int new_mtu, int num_queues) 679 { 680 struct device *dev; 681 struct page *page; 682 dma_addr_t da; 683 int num_rxb; 684 void *va; 685 int i; 686 687 mana_get_rxbuf_cfg(mpc, new_mtu, &mpc->rxbpre_datasize, 688 &mpc->rxbpre_alloc_size, &mpc->rxbpre_headroom, 689 &mpc->rxbpre_frag_count); 690 691 dev = mpc->ac->gdma_dev->gdma_context->dev; 692 693 num_rxb = num_queues * mpc->rx_queue_size; 694 695 WARN(mpc->rxbufs_pre, "mana rxbufs_pre exists\n"); 696 mpc->rxbufs_pre = kmalloc_array(num_rxb, sizeof(void *), GFP_KERNEL); 697 if (!mpc->rxbufs_pre) 698 goto error; 699 700 mpc->das_pre = kmalloc_array(num_rxb, sizeof(dma_addr_t), GFP_KERNEL); 701 if (!mpc->das_pre) 702 goto error; 703 704 mpc->rxbpre_total = 0; 705 706 for (i = 0; i < num_rxb; i++) { 707 page = dev_alloc_pages(get_order(mpc->rxbpre_alloc_size)); 708 if (!page) 709 goto error; 710 711 va = page_to_virt(page); 712 713 da = dma_map_single(dev, va + mpc->rxbpre_headroom, 714 mpc->rxbpre_datasize, DMA_FROM_DEVICE); 715 if (dma_mapping_error(dev, da)) { 716 put_page(page); 717 goto error; 718 } 719 720 mpc->rxbufs_pre[i] = va; 721 mpc->das_pre[i] = da; 722 mpc->rxbpre_total = i + 1; 723 } 724 725 return 0; 726 727 error: 728 netdev_err(mpc->ndev, "Failed to pre-allocate RX buffers for %d queues\n", num_queues); 729 mana_pre_dealloc_rxbufs(mpc); 730 return -ENOMEM; 731 } 732 733 static int mana_change_mtu(struct net_device *ndev, int new_mtu) 734 { 735 struct mana_port_context *mpc = netdev_priv(ndev); 736 unsigned int old_mtu = ndev->mtu; 737 int err; 738 739 /* Pre-allocate buffers to prevent failure in mana_attach later */ 740 err = mana_pre_alloc_rxbufs(mpc, new_mtu, mpc->num_queues); 741 if (err) { 742 netdev_err(ndev, "Insufficient memory for new MTU\n"); 743 return err; 744 } 745 746 err = mana_detach(ndev, false); 747 if (err) { 748 netdev_err(ndev, "mana_detach failed: %d\n", err); 749 goto out; 750 } 751 752 WRITE_ONCE(ndev->mtu, new_mtu); 753 754 err = mana_attach(ndev); 755 if (err) { 756 netdev_err(ndev, "mana_attach failed: %d\n", err); 757 WRITE_ONCE(ndev->mtu, old_mtu); 758 } 759 760 out: 761 mana_pre_dealloc_rxbufs(mpc); 762 return err; 763 } 764 765 static int mana_shaper_set(struct net_shaper_binding *binding, 766 const struct net_shaper *shaper, 767 struct netlink_ext_ack *extack) 768 { 769 struct mana_port_context *apc = netdev_priv(binding->netdev); 770 u32 old_speed, rate; 771 int err; 772 773 if (shaper->handle.scope != NET_SHAPER_SCOPE_NETDEV) { 774 NL_SET_ERR_MSG_MOD(extack, "net shaper scope should be netdev"); 775 return -EINVAL; 776 } 777 778 if (apc->handle.id && shaper->handle.id != apc->handle.id) { 779 NL_SET_ERR_MSG_MOD(extack, "Cannot create multiple shapers"); 780 return -EOPNOTSUPP; 781 } 782 783 if (!shaper->bw_max || (shaper->bw_max % 100000000)) { 784 NL_SET_ERR_MSG_MOD(extack, "Please use multiples of 100Mbps for bandwidth"); 785 return -EINVAL; 786 } 787 788 rate = div_u64(shaper->bw_max, 1000); /* Convert bps to Kbps */ 789 rate = div_u64(rate, 1000); /* Convert Kbps to Mbps */ 790 791 /* Get current speed */ 792 err = mana_query_link_cfg(apc); 793 old_speed = (err) ? SPEED_UNKNOWN : apc->speed; 794 795 if (!err) { 796 err = mana_set_bw_clamp(apc, rate, TRI_STATE_TRUE); 797 apc->speed = (err) ? old_speed : rate; 798 apc->handle = (err) ? apc->handle : shaper->handle; 799 } 800 801 return err; 802 } 803 804 static int mana_shaper_del(struct net_shaper_binding *binding, 805 const struct net_shaper_handle *handle, 806 struct netlink_ext_ack *extack) 807 { 808 struct mana_port_context *apc = netdev_priv(binding->netdev); 809 int err; 810 811 err = mana_set_bw_clamp(apc, 0, TRI_STATE_FALSE); 812 813 if (!err) { 814 /* Reset mana port context parameters */ 815 apc->handle.id = 0; 816 apc->handle.scope = NET_SHAPER_SCOPE_UNSPEC; 817 apc->speed = 0; 818 } 819 820 return err; 821 } 822 823 static void mana_shaper_cap(struct net_shaper_binding *binding, 824 enum net_shaper_scope scope, 825 unsigned long *flags) 826 { 827 *flags = BIT(NET_SHAPER_A_CAPS_SUPPORT_BW_MAX) | 828 BIT(NET_SHAPER_A_CAPS_SUPPORT_METRIC_BPS); 829 } 830 831 static const struct net_shaper_ops mana_shaper_ops = { 832 .set = mana_shaper_set, 833 .delete = mana_shaper_del, 834 .capabilities = mana_shaper_cap, 835 }; 836 837 static const struct net_device_ops mana_devops = { 838 .ndo_open = mana_open, 839 .ndo_stop = mana_close, 840 .ndo_select_queue = mana_select_queue, 841 .ndo_start_xmit = mana_start_xmit, 842 .ndo_validate_addr = eth_validate_addr, 843 .ndo_get_stats64 = mana_get_stats64, 844 .ndo_bpf = mana_bpf, 845 .ndo_xdp_xmit = mana_xdp_xmit, 846 .ndo_change_mtu = mana_change_mtu, 847 .net_shaper_ops = &mana_shaper_ops, 848 }; 849 850 static void mana_cleanup_port_context(struct mana_port_context *apc) 851 { 852 /* 853 * make sure subsequent cleanup attempts don't end up removing already 854 * cleaned dentry pointer 855 */ 856 debugfs_remove(apc->mana_port_debugfs); 857 apc->mana_port_debugfs = NULL; 858 kfree(apc->rxqs); 859 apc->rxqs = NULL; 860 } 861 862 static void mana_cleanup_indir_table(struct mana_port_context *apc) 863 { 864 apc->indir_table_sz = 0; 865 kfree(apc->indir_table); 866 kfree(apc->rxobj_table); 867 } 868 869 static int mana_init_port_context(struct mana_port_context *apc) 870 { 871 apc->rxqs = kcalloc(apc->num_queues, sizeof(struct mana_rxq *), 872 GFP_KERNEL); 873 874 return !apc->rxqs ? -ENOMEM : 0; 875 } 876 877 static int mana_send_request(struct mana_context *ac, void *in_buf, 878 u32 in_len, void *out_buf, u32 out_len) 879 { 880 struct gdma_context *gc = ac->gdma_dev->gdma_context; 881 struct gdma_resp_hdr *resp = out_buf; 882 struct gdma_req_hdr *req = in_buf; 883 struct device *dev = gc->dev; 884 static atomic_t activity_id; 885 int err; 886 887 req->dev_id = gc->mana.dev_id; 888 req->activity_id = atomic_inc_return(&activity_id); 889 890 err = mana_gd_send_request(gc, in_len, in_buf, out_len, 891 out_buf); 892 if (err || resp->status) { 893 if (err == -EOPNOTSUPP) 894 return err; 895 896 if (req->req.msg_type != MANA_QUERY_PHY_STAT && 897 mana_need_log(gc, err)) 898 dev_err(dev, "Failed to send mana message: %d, 0x%x\n", 899 err, resp->status); 900 return err ? err : -EPROTO; 901 } 902 903 if (req->dev_id.as_uint32 != resp->dev_id.as_uint32 || 904 req->activity_id != resp->activity_id) { 905 dev_err(dev, "Unexpected mana message response: %x,%x,%x,%x\n", 906 req->dev_id.as_uint32, resp->dev_id.as_uint32, 907 req->activity_id, resp->activity_id); 908 return -EPROTO; 909 } 910 911 return 0; 912 } 913 914 static int mana_verify_resp_hdr(const struct gdma_resp_hdr *resp_hdr, 915 const enum mana_command_code expected_code, 916 const u32 min_size) 917 { 918 if (resp_hdr->response.msg_type != expected_code) 919 return -EPROTO; 920 921 if (resp_hdr->response.msg_version < GDMA_MESSAGE_V1) 922 return -EPROTO; 923 924 if (resp_hdr->response.msg_size < min_size) 925 return -EPROTO; 926 927 return 0; 928 } 929 930 static int mana_pf_register_hw_vport(struct mana_port_context *apc) 931 { 932 struct mana_register_hw_vport_resp resp = {}; 933 struct mana_register_hw_vport_req req = {}; 934 int err; 935 936 mana_gd_init_req_hdr(&req.hdr, MANA_REGISTER_HW_PORT, 937 sizeof(req), sizeof(resp)); 938 req.attached_gfid = 1; 939 req.is_pf_default_vport = 1; 940 req.allow_all_ether_types = 1; 941 942 err = mana_send_request(apc->ac, &req, sizeof(req), &resp, 943 sizeof(resp)); 944 if (err) { 945 netdev_err(apc->ndev, "Failed to register hw vPort: %d\n", err); 946 return err; 947 } 948 949 err = mana_verify_resp_hdr(&resp.hdr, MANA_REGISTER_HW_PORT, 950 sizeof(resp)); 951 if (err || resp.hdr.status) { 952 netdev_err(apc->ndev, "Failed to register hw vPort: %d, 0x%x\n", 953 err, resp.hdr.status); 954 return err ? err : -EPROTO; 955 } 956 957 apc->port_handle = resp.hw_vport_handle; 958 return 0; 959 } 960 961 static void mana_pf_deregister_hw_vport(struct mana_port_context *apc) 962 { 963 struct mana_deregister_hw_vport_resp resp = {}; 964 struct mana_deregister_hw_vport_req req = {}; 965 int err; 966 967 mana_gd_init_req_hdr(&req.hdr, MANA_DEREGISTER_HW_PORT, 968 sizeof(req), sizeof(resp)); 969 req.hw_vport_handle = apc->port_handle; 970 971 err = mana_send_request(apc->ac, &req, sizeof(req), &resp, 972 sizeof(resp)); 973 if (err) { 974 if (mana_en_need_log(apc, err)) 975 netdev_err(apc->ndev, "Failed to unregister hw vPort: %d\n", 976 err); 977 978 return; 979 } 980 981 err = mana_verify_resp_hdr(&resp.hdr, MANA_DEREGISTER_HW_PORT, 982 sizeof(resp)); 983 if (err || resp.hdr.status) 984 netdev_err(apc->ndev, 985 "Failed to deregister hw vPort: %d, 0x%x\n", 986 err, resp.hdr.status); 987 } 988 989 static int mana_pf_register_filter(struct mana_port_context *apc) 990 { 991 struct mana_register_filter_resp resp = {}; 992 struct mana_register_filter_req req = {}; 993 int err; 994 995 mana_gd_init_req_hdr(&req.hdr, MANA_REGISTER_FILTER, 996 sizeof(req), sizeof(resp)); 997 req.vport = apc->port_handle; 998 memcpy(req.mac_addr, apc->mac_addr, ETH_ALEN); 999 1000 err = mana_send_request(apc->ac, &req, sizeof(req), &resp, 1001 sizeof(resp)); 1002 if (err) { 1003 netdev_err(apc->ndev, "Failed to register filter: %d\n", err); 1004 return err; 1005 } 1006 1007 err = mana_verify_resp_hdr(&resp.hdr, MANA_REGISTER_FILTER, 1008 sizeof(resp)); 1009 if (err || resp.hdr.status) { 1010 netdev_err(apc->ndev, "Failed to register filter: %d, 0x%x\n", 1011 err, resp.hdr.status); 1012 return err ? err : -EPROTO; 1013 } 1014 1015 apc->pf_filter_handle = resp.filter_handle; 1016 return 0; 1017 } 1018 1019 static void mana_pf_deregister_filter(struct mana_port_context *apc) 1020 { 1021 struct mana_deregister_filter_resp resp = {}; 1022 struct mana_deregister_filter_req req = {}; 1023 int err; 1024 1025 mana_gd_init_req_hdr(&req.hdr, MANA_DEREGISTER_FILTER, 1026 sizeof(req), sizeof(resp)); 1027 req.filter_handle = apc->pf_filter_handle; 1028 1029 err = mana_send_request(apc->ac, &req, sizeof(req), &resp, 1030 sizeof(resp)); 1031 if (err) { 1032 if (mana_en_need_log(apc, err)) 1033 netdev_err(apc->ndev, "Failed to unregister filter: %d\n", 1034 err); 1035 1036 return; 1037 } 1038 1039 err = mana_verify_resp_hdr(&resp.hdr, MANA_DEREGISTER_FILTER, 1040 sizeof(resp)); 1041 if (err || resp.hdr.status) 1042 netdev_err(apc->ndev, 1043 "Failed to deregister filter: %d, 0x%x\n", 1044 err, resp.hdr.status); 1045 } 1046 1047 static int mana_query_device_cfg(struct mana_context *ac, u32 proto_major_ver, 1048 u32 proto_minor_ver, u32 proto_micro_ver, 1049 u16 *max_num_vports, u8 *bm_hostmode) 1050 { 1051 struct gdma_context *gc = ac->gdma_dev->gdma_context; 1052 struct mana_query_device_cfg_resp resp = {}; 1053 struct mana_query_device_cfg_req req = {}; 1054 struct device *dev = gc->dev; 1055 int err = 0; 1056 1057 mana_gd_init_req_hdr(&req.hdr, MANA_QUERY_DEV_CONFIG, 1058 sizeof(req), sizeof(resp)); 1059 1060 req.hdr.resp.msg_version = GDMA_MESSAGE_V3; 1061 1062 req.proto_major_ver = proto_major_ver; 1063 req.proto_minor_ver = proto_minor_ver; 1064 req.proto_micro_ver = proto_micro_ver; 1065 1066 err = mana_send_request(ac, &req, sizeof(req), &resp, sizeof(resp)); 1067 if (err) { 1068 dev_err(dev, "Failed to query config: %d", err); 1069 return err; 1070 } 1071 1072 err = mana_verify_resp_hdr(&resp.hdr, MANA_QUERY_DEV_CONFIG, 1073 sizeof(resp)); 1074 if (err || resp.hdr.status) { 1075 dev_err(dev, "Invalid query result: %d, 0x%x\n", err, 1076 resp.hdr.status); 1077 if (!err) 1078 err = -EPROTO; 1079 return err; 1080 } 1081 1082 *max_num_vports = resp.max_num_vports; 1083 1084 if (resp.hdr.response.msg_version >= GDMA_MESSAGE_V2) 1085 gc->adapter_mtu = resp.adapter_mtu; 1086 else 1087 gc->adapter_mtu = ETH_FRAME_LEN; 1088 1089 if (resp.hdr.response.msg_version >= GDMA_MESSAGE_V3) 1090 *bm_hostmode = resp.bm_hostmode; 1091 else 1092 *bm_hostmode = 0; 1093 1094 debugfs_create_u16("adapter-MTU", 0400, gc->mana_pci_debugfs, &gc->adapter_mtu); 1095 1096 return 0; 1097 } 1098 1099 static int mana_query_vport_cfg(struct mana_port_context *apc, u32 vport_index, 1100 u32 *max_sq, u32 *max_rq, u32 *num_indir_entry) 1101 { 1102 struct mana_query_vport_cfg_resp resp = {}; 1103 struct mana_query_vport_cfg_req req = {}; 1104 int err; 1105 1106 mana_gd_init_req_hdr(&req.hdr, MANA_QUERY_VPORT_CONFIG, 1107 sizeof(req), sizeof(resp)); 1108 1109 req.vport_index = vport_index; 1110 1111 err = mana_send_request(apc->ac, &req, sizeof(req), &resp, 1112 sizeof(resp)); 1113 if (err) 1114 return err; 1115 1116 err = mana_verify_resp_hdr(&resp.hdr, MANA_QUERY_VPORT_CONFIG, 1117 sizeof(resp)); 1118 if (err) 1119 return err; 1120 1121 if (resp.hdr.status) 1122 return -EPROTO; 1123 1124 *max_sq = resp.max_num_sq; 1125 *max_rq = resp.max_num_rq; 1126 if (resp.num_indirection_ent > 0 && 1127 resp.num_indirection_ent <= MANA_INDIRECT_TABLE_MAX_SIZE && 1128 is_power_of_2(resp.num_indirection_ent)) { 1129 *num_indir_entry = resp.num_indirection_ent; 1130 } else { 1131 netdev_warn(apc->ndev, 1132 "Setting indirection table size to default %d for vPort %d\n", 1133 MANA_INDIRECT_TABLE_DEF_SIZE, apc->port_idx); 1134 *num_indir_entry = MANA_INDIRECT_TABLE_DEF_SIZE; 1135 } 1136 1137 apc->port_handle = resp.vport; 1138 ether_addr_copy(apc->mac_addr, resp.mac_addr); 1139 1140 return 0; 1141 } 1142 1143 void mana_uncfg_vport(struct mana_port_context *apc) 1144 { 1145 mutex_lock(&apc->vport_mutex); 1146 apc->vport_use_count--; 1147 WARN_ON(apc->vport_use_count < 0); 1148 mutex_unlock(&apc->vport_mutex); 1149 } 1150 EXPORT_SYMBOL_NS(mana_uncfg_vport, "NET_MANA"); 1151 1152 int mana_cfg_vport(struct mana_port_context *apc, u32 protection_dom_id, 1153 u32 doorbell_pg_id) 1154 { 1155 struct mana_config_vport_resp resp = {}; 1156 struct mana_config_vport_req req = {}; 1157 int err; 1158 1159 /* This function is used to program the Ethernet port in the hardware 1160 * table. It can be called from the Ethernet driver or the RDMA driver. 1161 * 1162 * For Ethernet usage, the hardware supports only one active user on a 1163 * physical port. The driver checks on the port usage before programming 1164 * the hardware when creating the RAW QP (RDMA driver) or exposing the 1165 * device to kernel NET layer (Ethernet driver). 1166 * 1167 * Because the RDMA driver doesn't know in advance which QP type the 1168 * user will create, it exposes the device with all its ports. The user 1169 * may not be able to create RAW QP on a port if this port is already 1170 * in used by the Ethernet driver from the kernel. 1171 * 1172 * This physical port limitation only applies to the RAW QP. For RC QP, 1173 * the hardware doesn't have this limitation. The user can create RC 1174 * QPs on a physical port up to the hardware limits independent of the 1175 * Ethernet usage on the same port. 1176 */ 1177 mutex_lock(&apc->vport_mutex); 1178 if (apc->vport_use_count > 0) { 1179 mutex_unlock(&apc->vport_mutex); 1180 return -EBUSY; 1181 } 1182 apc->vport_use_count++; 1183 mutex_unlock(&apc->vport_mutex); 1184 1185 mana_gd_init_req_hdr(&req.hdr, MANA_CONFIG_VPORT_TX, 1186 sizeof(req), sizeof(resp)); 1187 req.vport = apc->port_handle; 1188 req.pdid = protection_dom_id; 1189 req.doorbell_pageid = doorbell_pg_id; 1190 1191 err = mana_send_request(apc->ac, &req, sizeof(req), &resp, 1192 sizeof(resp)); 1193 if (err) { 1194 netdev_err(apc->ndev, "Failed to configure vPort: %d\n", err); 1195 goto out; 1196 } 1197 1198 err = mana_verify_resp_hdr(&resp.hdr, MANA_CONFIG_VPORT_TX, 1199 sizeof(resp)); 1200 if (err || resp.hdr.status) { 1201 netdev_err(apc->ndev, "Failed to configure vPort: %d, 0x%x\n", 1202 err, resp.hdr.status); 1203 if (!err) 1204 err = -EPROTO; 1205 1206 goto out; 1207 } 1208 1209 apc->tx_shortform_allowed = resp.short_form_allowed; 1210 apc->tx_vp_offset = resp.tx_vport_offset; 1211 1212 netdev_info(apc->ndev, "Configured vPort %llu PD %u DB %u\n", 1213 apc->port_handle, protection_dom_id, doorbell_pg_id); 1214 out: 1215 if (err) 1216 mana_uncfg_vport(apc); 1217 1218 return err; 1219 } 1220 EXPORT_SYMBOL_NS(mana_cfg_vport, "NET_MANA"); 1221 1222 static int mana_cfg_vport_steering(struct mana_port_context *apc, 1223 enum TRI_STATE rx, 1224 bool update_default_rxobj, bool update_key, 1225 bool update_tab) 1226 { 1227 struct mana_cfg_rx_steer_req_v2 *req; 1228 struct mana_cfg_rx_steer_resp resp = {}; 1229 struct net_device *ndev = apc->ndev; 1230 u32 req_buf_size; 1231 int err; 1232 1233 req_buf_size = struct_size(req, indir_tab, apc->indir_table_sz); 1234 req = kzalloc(req_buf_size, GFP_KERNEL); 1235 if (!req) 1236 return -ENOMEM; 1237 1238 mana_gd_init_req_hdr(&req->hdr, MANA_CONFIG_VPORT_RX, req_buf_size, 1239 sizeof(resp)); 1240 1241 req->hdr.req.msg_version = GDMA_MESSAGE_V2; 1242 1243 req->vport = apc->port_handle; 1244 req->num_indir_entries = apc->indir_table_sz; 1245 req->indir_tab_offset = offsetof(struct mana_cfg_rx_steer_req_v2, 1246 indir_tab); 1247 req->rx_enable = rx; 1248 req->rss_enable = apc->rss_state; 1249 req->update_default_rxobj = update_default_rxobj; 1250 req->update_hashkey = update_key; 1251 req->update_indir_tab = update_tab; 1252 req->default_rxobj = apc->default_rxobj; 1253 req->cqe_coalescing_enable = 0; 1254 1255 if (update_key) 1256 memcpy(&req->hashkey, apc->hashkey, MANA_HASH_KEY_SIZE); 1257 1258 if (update_tab) 1259 memcpy(req->indir_tab, apc->rxobj_table, 1260 flex_array_size(req, indir_tab, req->num_indir_entries)); 1261 1262 err = mana_send_request(apc->ac, req, req_buf_size, &resp, 1263 sizeof(resp)); 1264 if (err) { 1265 if (mana_en_need_log(apc, err)) 1266 netdev_err(ndev, "Failed to configure vPort RX: %d\n", err); 1267 1268 goto out; 1269 } 1270 1271 err = mana_verify_resp_hdr(&resp.hdr, MANA_CONFIG_VPORT_RX, 1272 sizeof(resp)); 1273 if (err) { 1274 netdev_err(ndev, "vPort RX configuration failed: %d\n", err); 1275 goto out; 1276 } 1277 1278 if (resp.hdr.status) { 1279 netdev_err(ndev, "vPort RX configuration failed: 0x%x\n", 1280 resp.hdr.status); 1281 err = -EPROTO; 1282 } 1283 1284 netdev_info(ndev, "Configured steering vPort %llu entries %u\n", 1285 apc->port_handle, apc->indir_table_sz); 1286 out: 1287 kfree(req); 1288 return err; 1289 } 1290 1291 int mana_query_link_cfg(struct mana_port_context *apc) 1292 { 1293 struct net_device *ndev = apc->ndev; 1294 struct mana_query_link_config_resp resp = {}; 1295 struct mana_query_link_config_req req = {}; 1296 int err; 1297 1298 mana_gd_init_req_hdr(&req.hdr, MANA_QUERY_LINK_CONFIG, 1299 sizeof(req), sizeof(resp)); 1300 1301 req.vport = apc->port_handle; 1302 req.hdr.resp.msg_version = GDMA_MESSAGE_V2; 1303 1304 err = mana_send_request(apc->ac, &req, sizeof(req), &resp, 1305 sizeof(resp)); 1306 1307 if (err) { 1308 if (err == -EOPNOTSUPP) { 1309 netdev_info_once(ndev, "MANA_QUERY_LINK_CONFIG not supported\n"); 1310 return err; 1311 } 1312 netdev_err(ndev, "Failed to query link config: %d\n", err); 1313 return err; 1314 } 1315 1316 err = mana_verify_resp_hdr(&resp.hdr, MANA_QUERY_LINK_CONFIG, 1317 sizeof(resp)); 1318 1319 if (err || resp.hdr.status) { 1320 netdev_err(ndev, "Failed to query link config: %d, 0x%x\n", err, 1321 resp.hdr.status); 1322 if (!err) 1323 err = -EOPNOTSUPP; 1324 return err; 1325 } 1326 1327 if (resp.qos_unconfigured) { 1328 err = -EINVAL; 1329 return err; 1330 } 1331 apc->speed = resp.link_speed_mbps; 1332 apc->max_speed = resp.qos_speed_mbps; 1333 return 0; 1334 } 1335 1336 int mana_set_bw_clamp(struct mana_port_context *apc, u32 speed, 1337 int enable_clamping) 1338 { 1339 struct mana_set_bw_clamp_resp resp = {}; 1340 struct mana_set_bw_clamp_req req = {}; 1341 struct net_device *ndev = apc->ndev; 1342 int err; 1343 1344 mana_gd_init_req_hdr(&req.hdr, MANA_SET_BW_CLAMP, 1345 sizeof(req), sizeof(resp)); 1346 req.vport = apc->port_handle; 1347 req.link_speed_mbps = speed; 1348 req.enable_clamping = enable_clamping; 1349 1350 err = mana_send_request(apc->ac, &req, sizeof(req), &resp, 1351 sizeof(resp)); 1352 1353 if (err) { 1354 if (err == -EOPNOTSUPP) { 1355 netdev_info_once(ndev, "MANA_SET_BW_CLAMP not supported\n"); 1356 return err; 1357 } 1358 netdev_err(ndev, "Failed to set bandwidth clamp for speed %u, err = %d", 1359 speed, err); 1360 return err; 1361 } 1362 1363 err = mana_verify_resp_hdr(&resp.hdr, MANA_SET_BW_CLAMP, 1364 sizeof(resp)); 1365 1366 if (err || resp.hdr.status) { 1367 netdev_err(ndev, "Failed to set bandwidth clamp: %d, 0x%x\n", err, 1368 resp.hdr.status); 1369 if (!err) 1370 err = -EOPNOTSUPP; 1371 return err; 1372 } 1373 1374 if (resp.qos_unconfigured) 1375 netdev_info(ndev, "QoS is unconfigured\n"); 1376 1377 return 0; 1378 } 1379 1380 int mana_create_wq_obj(struct mana_port_context *apc, 1381 mana_handle_t vport, 1382 u32 wq_type, struct mana_obj_spec *wq_spec, 1383 struct mana_obj_spec *cq_spec, 1384 mana_handle_t *wq_obj) 1385 { 1386 struct mana_create_wqobj_resp resp = {}; 1387 struct mana_create_wqobj_req req = {}; 1388 struct net_device *ndev = apc->ndev; 1389 int err; 1390 1391 mana_gd_init_req_hdr(&req.hdr, MANA_CREATE_WQ_OBJ, 1392 sizeof(req), sizeof(resp)); 1393 req.vport = vport; 1394 req.wq_type = wq_type; 1395 req.wq_gdma_region = wq_spec->gdma_region; 1396 req.cq_gdma_region = cq_spec->gdma_region; 1397 req.wq_size = wq_spec->queue_size; 1398 req.cq_size = cq_spec->queue_size; 1399 req.cq_moderation_ctx_id = cq_spec->modr_ctx_id; 1400 req.cq_parent_qid = cq_spec->attached_eq; 1401 1402 err = mana_send_request(apc->ac, &req, sizeof(req), &resp, 1403 sizeof(resp)); 1404 if (err) { 1405 netdev_err(ndev, "Failed to create WQ object: %d\n", err); 1406 goto out; 1407 } 1408 1409 err = mana_verify_resp_hdr(&resp.hdr, MANA_CREATE_WQ_OBJ, 1410 sizeof(resp)); 1411 if (err || resp.hdr.status) { 1412 netdev_err(ndev, "Failed to create WQ object: %d, 0x%x\n", err, 1413 resp.hdr.status); 1414 if (!err) 1415 err = -EPROTO; 1416 goto out; 1417 } 1418 1419 if (resp.wq_obj == INVALID_MANA_HANDLE) { 1420 netdev_err(ndev, "Got an invalid WQ object handle\n"); 1421 err = -EPROTO; 1422 goto out; 1423 } 1424 1425 *wq_obj = resp.wq_obj; 1426 wq_spec->queue_index = resp.wq_id; 1427 cq_spec->queue_index = resp.cq_id; 1428 1429 return 0; 1430 out: 1431 return err; 1432 } 1433 EXPORT_SYMBOL_NS(mana_create_wq_obj, "NET_MANA"); 1434 1435 void mana_destroy_wq_obj(struct mana_port_context *apc, u32 wq_type, 1436 mana_handle_t wq_obj) 1437 { 1438 struct mana_destroy_wqobj_resp resp = {}; 1439 struct mana_destroy_wqobj_req req = {}; 1440 struct net_device *ndev = apc->ndev; 1441 int err; 1442 1443 mana_gd_init_req_hdr(&req.hdr, MANA_DESTROY_WQ_OBJ, 1444 sizeof(req), sizeof(resp)); 1445 req.wq_type = wq_type; 1446 req.wq_obj_handle = wq_obj; 1447 1448 err = mana_send_request(apc->ac, &req, sizeof(req), &resp, 1449 sizeof(resp)); 1450 if (err) { 1451 if (mana_en_need_log(apc, err)) 1452 netdev_err(ndev, "Failed to destroy WQ object: %d\n", err); 1453 1454 return; 1455 } 1456 1457 err = mana_verify_resp_hdr(&resp.hdr, MANA_DESTROY_WQ_OBJ, 1458 sizeof(resp)); 1459 if (err || resp.hdr.status) 1460 netdev_err(ndev, "Failed to destroy WQ object: %d, 0x%x\n", err, 1461 resp.hdr.status); 1462 } 1463 EXPORT_SYMBOL_NS(mana_destroy_wq_obj, "NET_MANA"); 1464 1465 static void mana_destroy_eq(struct mana_context *ac) 1466 { 1467 struct gdma_context *gc = ac->gdma_dev->gdma_context; 1468 struct gdma_queue *eq; 1469 int i; 1470 1471 if (!ac->eqs) 1472 return; 1473 1474 debugfs_remove_recursive(ac->mana_eqs_debugfs); 1475 ac->mana_eqs_debugfs = NULL; 1476 1477 for (i = 0; i < gc->max_num_queues; i++) { 1478 eq = ac->eqs[i].eq; 1479 if (!eq) 1480 continue; 1481 1482 mana_gd_destroy_queue(gc, eq); 1483 } 1484 1485 kfree(ac->eqs); 1486 ac->eqs = NULL; 1487 } 1488 1489 static void mana_create_eq_debugfs(struct mana_context *ac, int i) 1490 { 1491 struct mana_eq eq = ac->eqs[i]; 1492 char eqnum[32]; 1493 1494 sprintf(eqnum, "eq%d", i); 1495 eq.mana_eq_debugfs = debugfs_create_dir(eqnum, ac->mana_eqs_debugfs); 1496 debugfs_create_u32("head", 0400, eq.mana_eq_debugfs, &eq.eq->head); 1497 debugfs_create_u32("tail", 0400, eq.mana_eq_debugfs, &eq.eq->tail); 1498 debugfs_create_file("eq_dump", 0400, eq.mana_eq_debugfs, eq.eq, &mana_dbg_q_fops); 1499 } 1500 1501 static int mana_create_eq(struct mana_context *ac) 1502 { 1503 struct gdma_dev *gd = ac->gdma_dev; 1504 struct gdma_context *gc = gd->gdma_context; 1505 struct gdma_queue_spec spec = {}; 1506 int err; 1507 int i; 1508 1509 ac->eqs = kcalloc(gc->max_num_queues, sizeof(struct mana_eq), 1510 GFP_KERNEL); 1511 if (!ac->eqs) 1512 return -ENOMEM; 1513 1514 spec.type = GDMA_EQ; 1515 spec.monitor_avl_buf = false; 1516 spec.queue_size = EQ_SIZE; 1517 spec.eq.callback = NULL; 1518 spec.eq.context = ac->eqs; 1519 spec.eq.log2_throttle_limit = LOG2_EQ_THROTTLE; 1520 1521 ac->mana_eqs_debugfs = debugfs_create_dir("EQs", gc->mana_pci_debugfs); 1522 1523 for (i = 0; i < gc->max_num_queues; i++) { 1524 spec.eq.msix_index = (i + 1) % gc->num_msix_usable; 1525 err = mana_gd_create_mana_eq(gd, &spec, &ac->eqs[i].eq); 1526 if (err) { 1527 dev_err(gc->dev, "Failed to create EQ %d : %d\n", i, err); 1528 goto out; 1529 } 1530 mana_create_eq_debugfs(ac, i); 1531 } 1532 1533 return 0; 1534 out: 1535 mana_destroy_eq(ac); 1536 return err; 1537 } 1538 1539 static int mana_fence_rq(struct mana_port_context *apc, struct mana_rxq *rxq) 1540 { 1541 struct mana_fence_rq_resp resp = {}; 1542 struct mana_fence_rq_req req = {}; 1543 int err; 1544 1545 init_completion(&rxq->fence_event); 1546 1547 mana_gd_init_req_hdr(&req.hdr, MANA_FENCE_RQ, 1548 sizeof(req), sizeof(resp)); 1549 req.wq_obj_handle = rxq->rxobj; 1550 1551 err = mana_send_request(apc->ac, &req, sizeof(req), &resp, 1552 sizeof(resp)); 1553 if (err) { 1554 netdev_err(apc->ndev, "Failed to fence RQ %u: %d\n", 1555 rxq->rxq_idx, err); 1556 return err; 1557 } 1558 1559 err = mana_verify_resp_hdr(&resp.hdr, MANA_FENCE_RQ, sizeof(resp)); 1560 if (err || resp.hdr.status) { 1561 netdev_err(apc->ndev, "Failed to fence RQ %u: %d, 0x%x\n", 1562 rxq->rxq_idx, err, resp.hdr.status); 1563 if (!err) 1564 err = -EPROTO; 1565 1566 return err; 1567 } 1568 1569 if (wait_for_completion_timeout(&rxq->fence_event, 10 * HZ) == 0) { 1570 netdev_err(apc->ndev, "Failed to fence RQ %u: timed out\n", 1571 rxq->rxq_idx); 1572 return -ETIMEDOUT; 1573 } 1574 1575 return 0; 1576 } 1577 1578 static void mana_fence_rqs(struct mana_port_context *apc) 1579 { 1580 unsigned int rxq_idx; 1581 struct mana_rxq *rxq; 1582 int err; 1583 1584 for (rxq_idx = 0; rxq_idx < apc->num_queues; rxq_idx++) { 1585 rxq = apc->rxqs[rxq_idx]; 1586 err = mana_fence_rq(apc, rxq); 1587 1588 /* In case of any error, use sleep instead. */ 1589 if (err) 1590 msleep(100); 1591 } 1592 } 1593 1594 static int mana_move_wq_tail(struct gdma_queue *wq, u32 num_units) 1595 { 1596 u32 used_space_old; 1597 u32 used_space_new; 1598 1599 used_space_old = wq->head - wq->tail; 1600 used_space_new = wq->head - (wq->tail + num_units); 1601 1602 if (WARN_ON_ONCE(used_space_new > used_space_old)) 1603 return -ERANGE; 1604 1605 wq->tail += num_units; 1606 return 0; 1607 } 1608 1609 static void mana_unmap_skb(struct sk_buff *skb, struct mana_port_context *apc) 1610 { 1611 struct mana_skb_head *ash = (struct mana_skb_head *)skb->head; 1612 struct gdma_context *gc = apc->ac->gdma_dev->gdma_context; 1613 struct device *dev = gc->dev; 1614 int hsg, i; 1615 1616 /* Number of SGEs of linear part */ 1617 hsg = (skb_is_gso(skb) && skb_headlen(skb) > ash->size[0]) ? 2 : 1; 1618 1619 for (i = 0; i < hsg; i++) 1620 dma_unmap_single(dev, ash->dma_handle[i], ash->size[i], 1621 DMA_TO_DEVICE); 1622 1623 for (i = hsg; i < skb_shinfo(skb)->nr_frags + hsg; i++) 1624 dma_unmap_page(dev, ash->dma_handle[i], ash->size[i], 1625 DMA_TO_DEVICE); 1626 } 1627 1628 static void mana_poll_tx_cq(struct mana_cq *cq) 1629 { 1630 struct gdma_comp *completions = cq->gdma_comp_buf; 1631 struct gdma_posted_wqe_info *wqe_info; 1632 unsigned int pkt_transmitted = 0; 1633 unsigned int wqe_unit_cnt = 0; 1634 struct mana_txq *txq = cq->txq; 1635 struct mana_port_context *apc; 1636 struct netdev_queue *net_txq; 1637 struct gdma_queue *gdma_wq; 1638 unsigned int avail_space; 1639 struct net_device *ndev; 1640 struct sk_buff *skb; 1641 bool txq_stopped; 1642 int comp_read; 1643 int i; 1644 1645 ndev = txq->ndev; 1646 apc = netdev_priv(ndev); 1647 1648 comp_read = mana_gd_poll_cq(cq->gdma_cq, completions, 1649 CQE_POLLING_BUFFER); 1650 1651 if (comp_read < 1) 1652 return; 1653 1654 for (i = 0; i < comp_read; i++) { 1655 struct mana_tx_comp_oob *cqe_oob; 1656 1657 if (WARN_ON_ONCE(!completions[i].is_sq)) 1658 return; 1659 1660 cqe_oob = (struct mana_tx_comp_oob *)completions[i].cqe_data; 1661 if (WARN_ON_ONCE(cqe_oob->cqe_hdr.client_type != 1662 MANA_CQE_COMPLETION)) 1663 return; 1664 1665 switch (cqe_oob->cqe_hdr.cqe_type) { 1666 case CQE_TX_OKAY: 1667 break; 1668 1669 case CQE_TX_SA_DROP: 1670 case CQE_TX_MTU_DROP: 1671 case CQE_TX_INVALID_OOB: 1672 case CQE_TX_INVALID_ETH_TYPE: 1673 case CQE_TX_HDR_PROCESSING_ERROR: 1674 case CQE_TX_VF_DISABLED: 1675 case CQE_TX_VPORT_IDX_OUT_OF_RANGE: 1676 case CQE_TX_VPORT_DISABLED: 1677 case CQE_TX_VLAN_TAGGING_VIOLATION: 1678 if (net_ratelimit()) 1679 netdev_err(ndev, "TX: CQE error %d\n", 1680 cqe_oob->cqe_hdr.cqe_type); 1681 1682 apc->eth_stats.tx_cqe_err++; 1683 break; 1684 1685 default: 1686 /* If the CQE type is unknown, log an error, 1687 * and still free the SKB, update tail, etc. 1688 */ 1689 if (net_ratelimit()) 1690 netdev_err(ndev, "TX: unknown CQE type %d\n", 1691 cqe_oob->cqe_hdr.cqe_type); 1692 1693 apc->eth_stats.tx_cqe_unknown_type++; 1694 break; 1695 } 1696 1697 if (WARN_ON_ONCE(txq->gdma_txq_id != completions[i].wq_num)) 1698 return; 1699 1700 skb = skb_dequeue(&txq->pending_skbs); 1701 if (WARN_ON_ONCE(!skb)) 1702 return; 1703 1704 wqe_info = (struct gdma_posted_wqe_info *)skb->cb; 1705 wqe_unit_cnt += wqe_info->wqe_size_in_bu; 1706 1707 mana_unmap_skb(skb, apc); 1708 1709 napi_consume_skb(skb, cq->budget); 1710 1711 pkt_transmitted++; 1712 } 1713 1714 if (WARN_ON_ONCE(wqe_unit_cnt == 0)) 1715 return; 1716 1717 mana_move_wq_tail(txq->gdma_sq, wqe_unit_cnt); 1718 1719 gdma_wq = txq->gdma_sq; 1720 avail_space = mana_gd_wq_avail_space(gdma_wq); 1721 1722 /* Ensure tail updated before checking q stop */ 1723 smp_mb(); 1724 1725 net_txq = txq->net_txq; 1726 txq_stopped = netif_tx_queue_stopped(net_txq); 1727 1728 /* Ensure checking txq_stopped before apc->port_is_up. */ 1729 smp_rmb(); 1730 1731 if (txq_stopped && apc->port_is_up && avail_space >= MAX_TX_WQE_SIZE) { 1732 netif_tx_wake_queue(net_txq); 1733 apc->eth_stats.wake_queue++; 1734 } 1735 1736 if (atomic_sub_return(pkt_transmitted, &txq->pending_sends) < 0) 1737 WARN_ON_ONCE(1); 1738 1739 cq->work_done = pkt_transmitted; 1740 } 1741 1742 static void mana_post_pkt_rxq(struct mana_rxq *rxq) 1743 { 1744 struct mana_recv_buf_oob *recv_buf_oob; 1745 u32 curr_index; 1746 int err; 1747 1748 curr_index = rxq->buf_index++; 1749 if (rxq->buf_index == rxq->num_rx_buf) 1750 rxq->buf_index = 0; 1751 1752 recv_buf_oob = &rxq->rx_oobs[curr_index]; 1753 1754 err = mana_gd_post_work_request(rxq->gdma_rq, &recv_buf_oob->wqe_req, 1755 &recv_buf_oob->wqe_inf); 1756 if (WARN_ON_ONCE(err)) 1757 return; 1758 1759 WARN_ON_ONCE(recv_buf_oob->wqe_inf.wqe_size_in_bu != 1); 1760 } 1761 1762 static struct sk_buff *mana_build_skb(struct mana_rxq *rxq, void *buf_va, 1763 uint pkt_len, struct xdp_buff *xdp) 1764 { 1765 struct sk_buff *skb = napi_build_skb(buf_va, rxq->alloc_size); 1766 1767 if (!skb) 1768 return NULL; 1769 1770 if (xdp->data_hard_start) { 1771 u32 metasize = xdp->data - xdp->data_meta; 1772 1773 skb_reserve(skb, xdp->data - xdp->data_hard_start); 1774 skb_put(skb, xdp->data_end - xdp->data); 1775 if (metasize) 1776 skb_metadata_set(skb, metasize); 1777 return skb; 1778 } 1779 1780 skb_reserve(skb, rxq->headroom); 1781 skb_put(skb, pkt_len); 1782 1783 return skb; 1784 } 1785 1786 static void mana_rx_skb(void *buf_va, bool from_pool, 1787 struct mana_rxcomp_oob *cqe, struct mana_rxq *rxq) 1788 { 1789 struct mana_stats_rx *rx_stats = &rxq->stats; 1790 struct net_device *ndev = rxq->ndev; 1791 uint pkt_len = cqe->ppi[0].pkt_len; 1792 u16 rxq_idx = rxq->rxq_idx; 1793 struct napi_struct *napi; 1794 struct xdp_buff xdp = {}; 1795 struct sk_buff *skb; 1796 u32 hash_value; 1797 u32 act; 1798 1799 rxq->rx_cq.work_done++; 1800 napi = &rxq->rx_cq.napi; 1801 1802 if (!buf_va) { 1803 ++ndev->stats.rx_dropped; 1804 return; 1805 } 1806 1807 act = mana_run_xdp(ndev, rxq, &xdp, buf_va, pkt_len); 1808 1809 if (act == XDP_REDIRECT && !rxq->xdp_rc) 1810 return; 1811 1812 if (act != XDP_PASS && act != XDP_TX) 1813 goto drop_xdp; 1814 1815 skb = mana_build_skb(rxq, buf_va, pkt_len, &xdp); 1816 1817 if (!skb) 1818 goto drop; 1819 1820 if (from_pool) 1821 skb_mark_for_recycle(skb); 1822 1823 skb->dev = napi->dev; 1824 1825 skb->protocol = eth_type_trans(skb, ndev); 1826 skb_checksum_none_assert(skb); 1827 skb_record_rx_queue(skb, rxq_idx); 1828 1829 if ((ndev->features & NETIF_F_RXCSUM) && cqe->rx_iphdr_csum_succeed) { 1830 if (cqe->rx_tcp_csum_succeed || cqe->rx_udp_csum_succeed) 1831 skb->ip_summed = CHECKSUM_UNNECESSARY; 1832 } 1833 1834 if (cqe->rx_hashtype != 0 && (ndev->features & NETIF_F_RXHASH)) { 1835 hash_value = cqe->ppi[0].pkt_hash; 1836 1837 if (cqe->rx_hashtype & MANA_HASH_L4) 1838 skb_set_hash(skb, hash_value, PKT_HASH_TYPE_L4); 1839 else 1840 skb_set_hash(skb, hash_value, PKT_HASH_TYPE_L3); 1841 } 1842 1843 if (cqe->rx_vlantag_present) { 1844 u16 vlan_tci = cqe->rx_vlan_id; 1845 1846 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vlan_tci); 1847 } 1848 1849 u64_stats_update_begin(&rx_stats->syncp); 1850 rx_stats->packets++; 1851 rx_stats->bytes += pkt_len; 1852 1853 if (act == XDP_TX) 1854 rx_stats->xdp_tx++; 1855 u64_stats_update_end(&rx_stats->syncp); 1856 1857 if (act == XDP_TX) { 1858 skb_set_queue_mapping(skb, rxq_idx); 1859 mana_xdp_tx(skb, ndev); 1860 return; 1861 } 1862 1863 napi_gro_receive(napi, skb); 1864 1865 return; 1866 1867 drop_xdp: 1868 u64_stats_update_begin(&rx_stats->syncp); 1869 rx_stats->xdp_drop++; 1870 u64_stats_update_end(&rx_stats->syncp); 1871 1872 drop: 1873 if (from_pool) { 1874 if (rxq->frag_count == 1) 1875 page_pool_recycle_direct(rxq->page_pool, 1876 virt_to_head_page(buf_va)); 1877 else 1878 page_pool_free_va(rxq->page_pool, buf_va, true); 1879 } else { 1880 WARN_ON_ONCE(rxq->xdp_save_va); 1881 /* Save for reuse */ 1882 rxq->xdp_save_va = buf_va; 1883 } 1884 1885 ++ndev->stats.rx_dropped; 1886 1887 return; 1888 } 1889 1890 static void *mana_get_rxfrag(struct mana_rxq *rxq, struct device *dev, 1891 dma_addr_t *da, bool *from_pool) 1892 { 1893 struct page *page; 1894 u32 offset; 1895 void *va; 1896 *from_pool = false; 1897 1898 /* Don't use fragments for jumbo frames or XDP where it's 1 fragment 1899 * per page. 1900 */ 1901 if (rxq->frag_count == 1) { 1902 /* Reuse XDP dropped page if available */ 1903 if (rxq->xdp_save_va) { 1904 va = rxq->xdp_save_va; 1905 page = virt_to_head_page(va); 1906 rxq->xdp_save_va = NULL; 1907 } else { 1908 page = page_pool_dev_alloc_pages(rxq->page_pool); 1909 if (!page) 1910 return NULL; 1911 1912 *from_pool = true; 1913 va = page_to_virt(page); 1914 } 1915 1916 *da = dma_map_single(dev, va + rxq->headroom, rxq->datasize, 1917 DMA_FROM_DEVICE); 1918 if (dma_mapping_error(dev, *da)) { 1919 mana_put_rx_page(rxq, page, *from_pool); 1920 return NULL; 1921 } 1922 1923 return va; 1924 } 1925 1926 page = page_pool_dev_alloc_frag(rxq->page_pool, &offset, 1927 rxq->alloc_size); 1928 if (!page) 1929 return NULL; 1930 1931 va = page_to_virt(page) + offset; 1932 *da = page_pool_get_dma_addr(page) + offset + rxq->headroom; 1933 *from_pool = true; 1934 1935 return va; 1936 } 1937 1938 /* Allocate frag for rx buffer, and save the old buf */ 1939 static void mana_refill_rx_oob(struct device *dev, struct mana_rxq *rxq, 1940 struct mana_recv_buf_oob *rxoob, void **old_buf, 1941 bool *old_fp) 1942 { 1943 bool from_pool; 1944 dma_addr_t da; 1945 void *va; 1946 1947 va = mana_get_rxfrag(rxq, dev, &da, &from_pool); 1948 if (!va) 1949 return; 1950 if (!rxoob->from_pool || rxq->frag_count == 1) 1951 dma_unmap_single(dev, rxoob->sgl[0].address, rxq->datasize, 1952 DMA_FROM_DEVICE); 1953 *old_buf = rxoob->buf_va; 1954 *old_fp = rxoob->from_pool; 1955 1956 rxoob->buf_va = va; 1957 rxoob->sgl[0].address = da; 1958 rxoob->from_pool = from_pool; 1959 } 1960 1961 static void mana_process_rx_cqe(struct mana_rxq *rxq, struct mana_cq *cq, 1962 struct gdma_comp *cqe) 1963 { 1964 struct mana_rxcomp_oob *oob = (struct mana_rxcomp_oob *)cqe->cqe_data; 1965 struct gdma_context *gc = rxq->gdma_rq->gdma_dev->gdma_context; 1966 struct net_device *ndev = rxq->ndev; 1967 struct mana_recv_buf_oob *rxbuf_oob; 1968 struct mana_port_context *apc; 1969 struct device *dev = gc->dev; 1970 void *old_buf = NULL; 1971 u32 curr, pktlen; 1972 bool old_fp; 1973 1974 apc = netdev_priv(ndev); 1975 1976 switch (oob->cqe_hdr.cqe_type) { 1977 case CQE_RX_OKAY: 1978 break; 1979 1980 case CQE_RX_TRUNCATED: 1981 ++ndev->stats.rx_dropped; 1982 rxbuf_oob = &rxq->rx_oobs[rxq->buf_index]; 1983 netdev_warn_once(ndev, "Dropped a truncated packet\n"); 1984 goto drop; 1985 1986 case CQE_RX_COALESCED_4: 1987 netdev_err(ndev, "RX coalescing is unsupported\n"); 1988 apc->eth_stats.rx_coalesced_err++; 1989 return; 1990 1991 case CQE_RX_OBJECT_FENCE: 1992 complete(&rxq->fence_event); 1993 return; 1994 1995 default: 1996 netdev_err(ndev, "Unknown RX CQE type = %d\n", 1997 oob->cqe_hdr.cqe_type); 1998 apc->eth_stats.rx_cqe_unknown_type++; 1999 return; 2000 } 2001 2002 pktlen = oob->ppi[0].pkt_len; 2003 2004 if (pktlen == 0) { 2005 /* data packets should never have packetlength of zero */ 2006 netdev_err(ndev, "RX pkt len=0, rq=%u, cq=%u, rxobj=0x%llx\n", 2007 rxq->gdma_id, cq->gdma_id, rxq->rxobj); 2008 return; 2009 } 2010 2011 curr = rxq->buf_index; 2012 rxbuf_oob = &rxq->rx_oobs[curr]; 2013 WARN_ON_ONCE(rxbuf_oob->wqe_inf.wqe_size_in_bu != 1); 2014 2015 mana_refill_rx_oob(dev, rxq, rxbuf_oob, &old_buf, &old_fp); 2016 2017 /* Unsuccessful refill will have old_buf == NULL. 2018 * In this case, mana_rx_skb() will drop the packet. 2019 */ 2020 mana_rx_skb(old_buf, old_fp, oob, rxq); 2021 2022 drop: 2023 mana_move_wq_tail(rxq->gdma_rq, rxbuf_oob->wqe_inf.wqe_size_in_bu); 2024 2025 mana_post_pkt_rxq(rxq); 2026 } 2027 2028 static void mana_poll_rx_cq(struct mana_cq *cq) 2029 { 2030 struct gdma_comp *comp = cq->gdma_comp_buf; 2031 struct mana_rxq *rxq = cq->rxq; 2032 int comp_read, i; 2033 2034 comp_read = mana_gd_poll_cq(cq->gdma_cq, comp, CQE_POLLING_BUFFER); 2035 WARN_ON_ONCE(comp_read > CQE_POLLING_BUFFER); 2036 2037 rxq->xdp_flush = false; 2038 2039 for (i = 0; i < comp_read; i++) { 2040 if (WARN_ON_ONCE(comp[i].is_sq)) 2041 return; 2042 2043 /* verify recv cqe references the right rxq */ 2044 if (WARN_ON_ONCE(comp[i].wq_num != cq->rxq->gdma_id)) 2045 return; 2046 2047 mana_process_rx_cqe(rxq, cq, &comp[i]); 2048 } 2049 2050 if (comp_read > 0) { 2051 struct gdma_context *gc = rxq->gdma_rq->gdma_dev->gdma_context; 2052 2053 mana_gd_wq_ring_doorbell(gc, rxq->gdma_rq); 2054 } 2055 2056 if (rxq->xdp_flush) 2057 xdp_do_flush(); 2058 } 2059 2060 static int mana_cq_handler(void *context, struct gdma_queue *gdma_queue) 2061 { 2062 struct mana_cq *cq = context; 2063 int w; 2064 2065 WARN_ON_ONCE(cq->gdma_cq != gdma_queue); 2066 2067 if (cq->type == MANA_CQ_TYPE_RX) 2068 mana_poll_rx_cq(cq); 2069 else 2070 mana_poll_tx_cq(cq); 2071 2072 w = cq->work_done; 2073 cq->work_done_since_doorbell += w; 2074 2075 if (w < cq->budget) { 2076 mana_gd_ring_cq(gdma_queue, SET_ARM_BIT); 2077 cq->work_done_since_doorbell = 0; 2078 napi_complete_done(&cq->napi, w); 2079 } else if (cq->work_done_since_doorbell > 2080 cq->gdma_cq->queue_size / COMP_ENTRY_SIZE * 4) { 2081 /* MANA hardware requires at least one doorbell ring every 8 2082 * wraparounds of CQ even if there is no need to arm the CQ. 2083 * This driver rings the doorbell as soon as we have exceeded 2084 * 4 wraparounds. 2085 */ 2086 mana_gd_ring_cq(gdma_queue, 0); 2087 cq->work_done_since_doorbell = 0; 2088 } 2089 2090 return w; 2091 } 2092 2093 static int mana_poll(struct napi_struct *napi, int budget) 2094 { 2095 struct mana_cq *cq = container_of(napi, struct mana_cq, napi); 2096 int w; 2097 2098 cq->work_done = 0; 2099 cq->budget = budget; 2100 2101 w = mana_cq_handler(cq, cq->gdma_cq); 2102 2103 return min(w, budget); 2104 } 2105 2106 static void mana_schedule_napi(void *context, struct gdma_queue *gdma_queue) 2107 { 2108 struct mana_cq *cq = context; 2109 2110 napi_schedule_irqoff(&cq->napi); 2111 } 2112 2113 static void mana_deinit_cq(struct mana_port_context *apc, struct mana_cq *cq) 2114 { 2115 struct gdma_dev *gd = apc->ac->gdma_dev; 2116 2117 if (!cq->gdma_cq) 2118 return; 2119 2120 mana_gd_destroy_queue(gd->gdma_context, cq->gdma_cq); 2121 } 2122 2123 static void mana_deinit_txq(struct mana_port_context *apc, struct mana_txq *txq) 2124 { 2125 struct gdma_dev *gd = apc->ac->gdma_dev; 2126 2127 if (!txq->gdma_sq) 2128 return; 2129 2130 mana_gd_destroy_queue(gd->gdma_context, txq->gdma_sq); 2131 } 2132 2133 static void mana_destroy_txq(struct mana_port_context *apc) 2134 { 2135 struct napi_struct *napi; 2136 int i; 2137 2138 if (!apc->tx_qp) 2139 return; 2140 2141 for (i = 0; i < apc->num_queues; i++) { 2142 debugfs_remove_recursive(apc->tx_qp[i].mana_tx_debugfs); 2143 apc->tx_qp[i].mana_tx_debugfs = NULL; 2144 2145 napi = &apc->tx_qp[i].tx_cq.napi; 2146 if (apc->tx_qp[i].txq.napi_initialized) { 2147 napi_synchronize(napi); 2148 napi_disable_locked(napi); 2149 netif_napi_del_locked(napi); 2150 apc->tx_qp[i].txq.napi_initialized = false; 2151 } 2152 mana_destroy_wq_obj(apc, GDMA_SQ, apc->tx_qp[i].tx_object); 2153 2154 mana_deinit_cq(apc, &apc->tx_qp[i].tx_cq); 2155 2156 mana_deinit_txq(apc, &apc->tx_qp[i].txq); 2157 } 2158 2159 kfree(apc->tx_qp); 2160 apc->tx_qp = NULL; 2161 } 2162 2163 static void mana_create_txq_debugfs(struct mana_port_context *apc, int idx) 2164 { 2165 struct mana_tx_qp *tx_qp = &apc->tx_qp[idx]; 2166 char qnum[32]; 2167 2168 sprintf(qnum, "TX-%d", idx); 2169 tx_qp->mana_tx_debugfs = debugfs_create_dir(qnum, apc->mana_port_debugfs); 2170 debugfs_create_u32("sq_head", 0400, tx_qp->mana_tx_debugfs, 2171 &tx_qp->txq.gdma_sq->head); 2172 debugfs_create_u32("sq_tail", 0400, tx_qp->mana_tx_debugfs, 2173 &tx_qp->txq.gdma_sq->tail); 2174 debugfs_create_u32("sq_pend_skb_qlen", 0400, tx_qp->mana_tx_debugfs, 2175 &tx_qp->txq.pending_skbs.qlen); 2176 debugfs_create_u32("cq_head", 0400, tx_qp->mana_tx_debugfs, 2177 &tx_qp->tx_cq.gdma_cq->head); 2178 debugfs_create_u32("cq_tail", 0400, tx_qp->mana_tx_debugfs, 2179 &tx_qp->tx_cq.gdma_cq->tail); 2180 debugfs_create_u32("cq_budget", 0400, tx_qp->mana_tx_debugfs, 2181 &tx_qp->tx_cq.budget); 2182 debugfs_create_file("txq_dump", 0400, tx_qp->mana_tx_debugfs, 2183 tx_qp->txq.gdma_sq, &mana_dbg_q_fops); 2184 debugfs_create_file("cq_dump", 0400, tx_qp->mana_tx_debugfs, 2185 tx_qp->tx_cq.gdma_cq, &mana_dbg_q_fops); 2186 } 2187 2188 static int mana_create_txq(struct mana_port_context *apc, 2189 struct net_device *net) 2190 { 2191 struct mana_context *ac = apc->ac; 2192 struct gdma_dev *gd = ac->gdma_dev; 2193 struct mana_obj_spec wq_spec; 2194 struct mana_obj_spec cq_spec; 2195 struct gdma_queue_spec spec; 2196 struct gdma_context *gc; 2197 struct mana_txq *txq; 2198 struct mana_cq *cq; 2199 u32 txq_size; 2200 u32 cq_size; 2201 int err; 2202 int i; 2203 2204 apc->tx_qp = kcalloc(apc->num_queues, sizeof(struct mana_tx_qp), 2205 GFP_KERNEL); 2206 if (!apc->tx_qp) 2207 return -ENOMEM; 2208 2209 /* The minimum size of the WQE is 32 bytes, hence 2210 * apc->tx_queue_size represents the maximum number of WQEs 2211 * the SQ can store. This value is then used to size other queues 2212 * to prevent overflow. 2213 * Also note that the txq_size is always going to be MANA_PAGE_ALIGNED, 2214 * as min val of apc->tx_queue_size is 128 and that would make 2215 * txq_size 128*32 = 4096 and the other higher values of apc->tx_queue_size 2216 * are always power of two 2217 */ 2218 txq_size = apc->tx_queue_size * 32; 2219 2220 cq_size = apc->tx_queue_size * COMP_ENTRY_SIZE; 2221 2222 gc = gd->gdma_context; 2223 2224 for (i = 0; i < apc->num_queues; i++) { 2225 apc->tx_qp[i].tx_object = INVALID_MANA_HANDLE; 2226 2227 /* Create SQ */ 2228 txq = &apc->tx_qp[i].txq; 2229 2230 u64_stats_init(&txq->stats.syncp); 2231 txq->ndev = net; 2232 txq->net_txq = netdev_get_tx_queue(net, i); 2233 txq->vp_offset = apc->tx_vp_offset; 2234 txq->napi_initialized = false; 2235 skb_queue_head_init(&txq->pending_skbs); 2236 2237 memset(&spec, 0, sizeof(spec)); 2238 spec.type = GDMA_SQ; 2239 spec.monitor_avl_buf = true; 2240 spec.queue_size = txq_size; 2241 err = mana_gd_create_mana_wq_cq(gd, &spec, &txq->gdma_sq); 2242 if (err) 2243 goto out; 2244 2245 /* Create SQ's CQ */ 2246 cq = &apc->tx_qp[i].tx_cq; 2247 cq->type = MANA_CQ_TYPE_TX; 2248 2249 cq->txq = txq; 2250 2251 memset(&spec, 0, sizeof(spec)); 2252 spec.type = GDMA_CQ; 2253 spec.monitor_avl_buf = false; 2254 spec.queue_size = cq_size; 2255 spec.cq.callback = mana_schedule_napi; 2256 spec.cq.parent_eq = ac->eqs[i].eq; 2257 spec.cq.context = cq; 2258 err = mana_gd_create_mana_wq_cq(gd, &spec, &cq->gdma_cq); 2259 if (err) 2260 goto out; 2261 2262 memset(&wq_spec, 0, sizeof(wq_spec)); 2263 memset(&cq_spec, 0, sizeof(cq_spec)); 2264 2265 wq_spec.gdma_region = txq->gdma_sq->mem_info.dma_region_handle; 2266 wq_spec.queue_size = txq->gdma_sq->queue_size; 2267 2268 cq_spec.gdma_region = cq->gdma_cq->mem_info.dma_region_handle; 2269 cq_spec.queue_size = cq->gdma_cq->queue_size; 2270 cq_spec.modr_ctx_id = 0; 2271 cq_spec.attached_eq = cq->gdma_cq->cq.parent->id; 2272 2273 err = mana_create_wq_obj(apc, apc->port_handle, GDMA_SQ, 2274 &wq_spec, &cq_spec, 2275 &apc->tx_qp[i].tx_object); 2276 2277 if (err) 2278 goto out; 2279 2280 txq->gdma_sq->id = wq_spec.queue_index; 2281 cq->gdma_cq->id = cq_spec.queue_index; 2282 2283 txq->gdma_sq->mem_info.dma_region_handle = 2284 GDMA_INVALID_DMA_REGION; 2285 cq->gdma_cq->mem_info.dma_region_handle = 2286 GDMA_INVALID_DMA_REGION; 2287 2288 txq->gdma_txq_id = txq->gdma_sq->id; 2289 2290 cq->gdma_id = cq->gdma_cq->id; 2291 2292 if (WARN_ON(cq->gdma_id >= gc->max_num_cqs)) { 2293 err = -EINVAL; 2294 goto out; 2295 } 2296 2297 gc->cq_table[cq->gdma_id] = cq->gdma_cq; 2298 2299 mana_create_txq_debugfs(apc, i); 2300 2301 set_bit(NAPI_STATE_NO_BUSY_POLL, &cq->napi.state); 2302 netif_napi_add_locked(net, &cq->napi, mana_poll); 2303 napi_enable_locked(&cq->napi); 2304 txq->napi_initialized = true; 2305 2306 mana_gd_ring_cq(cq->gdma_cq, SET_ARM_BIT); 2307 } 2308 2309 return 0; 2310 out: 2311 netdev_err(net, "Failed to create %d TX queues, %d\n", 2312 apc->num_queues, err); 2313 mana_destroy_txq(apc); 2314 return err; 2315 } 2316 2317 static void mana_destroy_rxq(struct mana_port_context *apc, 2318 struct mana_rxq *rxq, bool napi_initialized) 2319 2320 { 2321 struct gdma_context *gc = apc->ac->gdma_dev->gdma_context; 2322 struct mana_recv_buf_oob *rx_oob; 2323 struct device *dev = gc->dev; 2324 struct napi_struct *napi; 2325 struct page *page; 2326 int i; 2327 2328 if (!rxq) 2329 return; 2330 2331 debugfs_remove_recursive(rxq->mana_rx_debugfs); 2332 rxq->mana_rx_debugfs = NULL; 2333 2334 napi = &rxq->rx_cq.napi; 2335 2336 if (napi_initialized) { 2337 napi_synchronize(napi); 2338 2339 napi_disable_locked(napi); 2340 netif_napi_del_locked(napi); 2341 } 2342 xdp_rxq_info_unreg(&rxq->xdp_rxq); 2343 2344 mana_destroy_wq_obj(apc, GDMA_RQ, rxq->rxobj); 2345 2346 mana_deinit_cq(apc, &rxq->rx_cq); 2347 2348 if (rxq->xdp_save_va) 2349 put_page(virt_to_head_page(rxq->xdp_save_va)); 2350 2351 for (i = 0; i < rxq->num_rx_buf; i++) { 2352 rx_oob = &rxq->rx_oobs[i]; 2353 2354 if (!rx_oob->buf_va) 2355 continue; 2356 2357 page = virt_to_head_page(rx_oob->buf_va); 2358 2359 if (rxq->frag_count == 1 || !rx_oob->from_pool) { 2360 dma_unmap_single(dev, rx_oob->sgl[0].address, 2361 rx_oob->sgl[0].size, DMA_FROM_DEVICE); 2362 mana_put_rx_page(rxq, page, rx_oob->from_pool); 2363 } else { 2364 page_pool_free_va(rxq->page_pool, rx_oob->buf_va, true); 2365 } 2366 2367 rx_oob->buf_va = NULL; 2368 } 2369 2370 page_pool_destroy(rxq->page_pool); 2371 2372 if (rxq->gdma_rq) 2373 mana_gd_destroy_queue(gc, rxq->gdma_rq); 2374 2375 kfree(rxq); 2376 } 2377 2378 static int mana_fill_rx_oob(struct mana_recv_buf_oob *rx_oob, u32 mem_key, 2379 struct mana_rxq *rxq, struct device *dev) 2380 { 2381 struct mana_port_context *mpc = netdev_priv(rxq->ndev); 2382 bool from_pool = false; 2383 dma_addr_t da; 2384 void *va; 2385 2386 if (mpc->rxbufs_pre) 2387 va = mana_get_rxbuf_pre(rxq, &da); 2388 else 2389 va = mana_get_rxfrag(rxq, dev, &da, &from_pool); 2390 2391 if (!va) 2392 return -ENOMEM; 2393 2394 rx_oob->buf_va = va; 2395 rx_oob->from_pool = from_pool; 2396 2397 rx_oob->sgl[0].address = da; 2398 rx_oob->sgl[0].size = rxq->datasize; 2399 rx_oob->sgl[0].mem_key = mem_key; 2400 2401 return 0; 2402 } 2403 2404 #define MANA_WQE_HEADER_SIZE 16 2405 #define MANA_WQE_SGE_SIZE 16 2406 2407 static int mana_alloc_rx_wqe(struct mana_port_context *apc, 2408 struct mana_rxq *rxq, u32 *rxq_size, u32 *cq_size) 2409 { 2410 struct gdma_context *gc = apc->ac->gdma_dev->gdma_context; 2411 struct mana_recv_buf_oob *rx_oob; 2412 struct device *dev = gc->dev; 2413 u32 buf_idx; 2414 int ret; 2415 2416 WARN_ON(rxq->datasize == 0); 2417 2418 *rxq_size = 0; 2419 *cq_size = 0; 2420 2421 for (buf_idx = 0; buf_idx < rxq->num_rx_buf; buf_idx++) { 2422 rx_oob = &rxq->rx_oobs[buf_idx]; 2423 memset(rx_oob, 0, sizeof(*rx_oob)); 2424 2425 rx_oob->num_sge = 1; 2426 2427 ret = mana_fill_rx_oob(rx_oob, apc->ac->gdma_dev->gpa_mkey, rxq, 2428 dev); 2429 if (ret) 2430 return ret; 2431 2432 rx_oob->wqe_req.sgl = rx_oob->sgl; 2433 rx_oob->wqe_req.num_sge = rx_oob->num_sge; 2434 rx_oob->wqe_req.inline_oob_size = 0; 2435 rx_oob->wqe_req.inline_oob_data = NULL; 2436 rx_oob->wqe_req.flags = 0; 2437 rx_oob->wqe_req.client_data_unit = 0; 2438 2439 *rxq_size += ALIGN(MANA_WQE_HEADER_SIZE + 2440 MANA_WQE_SGE_SIZE * rx_oob->num_sge, 32); 2441 *cq_size += COMP_ENTRY_SIZE; 2442 } 2443 2444 return 0; 2445 } 2446 2447 static int mana_push_wqe(struct mana_rxq *rxq) 2448 { 2449 struct mana_recv_buf_oob *rx_oob; 2450 u32 buf_idx; 2451 int err; 2452 2453 for (buf_idx = 0; buf_idx < rxq->num_rx_buf; buf_idx++) { 2454 rx_oob = &rxq->rx_oobs[buf_idx]; 2455 2456 err = mana_gd_post_and_ring(rxq->gdma_rq, &rx_oob->wqe_req, 2457 &rx_oob->wqe_inf); 2458 if (err) 2459 return -ENOSPC; 2460 } 2461 2462 return 0; 2463 } 2464 2465 static int mana_create_page_pool(struct mana_rxq *rxq, struct gdma_context *gc) 2466 { 2467 struct mana_port_context *mpc = netdev_priv(rxq->ndev); 2468 struct page_pool_params pprm = {}; 2469 int ret; 2470 2471 pprm.pool_size = mpc->rx_queue_size / rxq->frag_count + 1; 2472 pprm.nid = gc->numa_node; 2473 pprm.napi = &rxq->rx_cq.napi; 2474 pprm.netdev = rxq->ndev; 2475 pprm.order = get_order(rxq->alloc_size); 2476 pprm.queue_idx = rxq->rxq_idx; 2477 pprm.dev = gc->dev; 2478 2479 /* Let the page pool do the dma map when page sharing with multiple 2480 * fragments enabled for rx buffers. 2481 */ 2482 if (rxq->frag_count > 1) { 2483 pprm.flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV; 2484 pprm.max_len = PAGE_SIZE; 2485 pprm.dma_dir = DMA_FROM_DEVICE; 2486 } 2487 2488 rxq->page_pool = page_pool_create(&pprm); 2489 2490 if (IS_ERR(rxq->page_pool)) { 2491 ret = PTR_ERR(rxq->page_pool); 2492 rxq->page_pool = NULL; 2493 return ret; 2494 } 2495 2496 return 0; 2497 } 2498 2499 static struct mana_rxq *mana_create_rxq(struct mana_port_context *apc, 2500 u32 rxq_idx, struct mana_eq *eq, 2501 struct net_device *ndev) 2502 { 2503 struct gdma_dev *gd = apc->ac->gdma_dev; 2504 struct mana_obj_spec wq_spec; 2505 struct mana_obj_spec cq_spec; 2506 struct gdma_queue_spec spec; 2507 struct mana_cq *cq = NULL; 2508 struct gdma_context *gc; 2509 u32 cq_size, rq_size; 2510 struct mana_rxq *rxq; 2511 int err; 2512 2513 gc = gd->gdma_context; 2514 2515 rxq = kzalloc(struct_size(rxq, rx_oobs, apc->rx_queue_size), 2516 GFP_KERNEL); 2517 if (!rxq) 2518 return NULL; 2519 2520 rxq->ndev = ndev; 2521 rxq->num_rx_buf = apc->rx_queue_size; 2522 rxq->rxq_idx = rxq_idx; 2523 rxq->rxobj = INVALID_MANA_HANDLE; 2524 2525 mana_get_rxbuf_cfg(apc, ndev->mtu, &rxq->datasize, &rxq->alloc_size, 2526 &rxq->headroom, &rxq->frag_count); 2527 /* Create page pool for RX queue */ 2528 err = mana_create_page_pool(rxq, gc); 2529 if (err) { 2530 netdev_err(ndev, "Create page pool err:%d\n", err); 2531 goto out; 2532 } 2533 2534 err = mana_alloc_rx_wqe(apc, rxq, &rq_size, &cq_size); 2535 if (err) 2536 goto out; 2537 2538 rq_size = MANA_PAGE_ALIGN(rq_size); 2539 cq_size = MANA_PAGE_ALIGN(cq_size); 2540 2541 /* Create RQ */ 2542 memset(&spec, 0, sizeof(spec)); 2543 spec.type = GDMA_RQ; 2544 spec.monitor_avl_buf = true; 2545 spec.queue_size = rq_size; 2546 err = mana_gd_create_mana_wq_cq(gd, &spec, &rxq->gdma_rq); 2547 if (err) 2548 goto out; 2549 2550 /* Create RQ's CQ */ 2551 cq = &rxq->rx_cq; 2552 cq->type = MANA_CQ_TYPE_RX; 2553 cq->rxq = rxq; 2554 2555 memset(&spec, 0, sizeof(spec)); 2556 spec.type = GDMA_CQ; 2557 spec.monitor_avl_buf = false; 2558 spec.queue_size = cq_size; 2559 spec.cq.callback = mana_schedule_napi; 2560 spec.cq.parent_eq = eq->eq; 2561 spec.cq.context = cq; 2562 err = mana_gd_create_mana_wq_cq(gd, &spec, &cq->gdma_cq); 2563 if (err) 2564 goto out; 2565 2566 memset(&wq_spec, 0, sizeof(wq_spec)); 2567 memset(&cq_spec, 0, sizeof(cq_spec)); 2568 wq_spec.gdma_region = rxq->gdma_rq->mem_info.dma_region_handle; 2569 wq_spec.queue_size = rxq->gdma_rq->queue_size; 2570 2571 cq_spec.gdma_region = cq->gdma_cq->mem_info.dma_region_handle; 2572 cq_spec.queue_size = cq->gdma_cq->queue_size; 2573 cq_spec.modr_ctx_id = 0; 2574 cq_spec.attached_eq = cq->gdma_cq->cq.parent->id; 2575 2576 err = mana_create_wq_obj(apc, apc->port_handle, GDMA_RQ, 2577 &wq_spec, &cq_spec, &rxq->rxobj); 2578 if (err) 2579 goto out; 2580 2581 rxq->gdma_rq->id = wq_spec.queue_index; 2582 cq->gdma_cq->id = cq_spec.queue_index; 2583 2584 rxq->gdma_rq->mem_info.dma_region_handle = GDMA_INVALID_DMA_REGION; 2585 cq->gdma_cq->mem_info.dma_region_handle = GDMA_INVALID_DMA_REGION; 2586 2587 rxq->gdma_id = rxq->gdma_rq->id; 2588 cq->gdma_id = cq->gdma_cq->id; 2589 2590 err = mana_push_wqe(rxq); 2591 if (err) 2592 goto out; 2593 2594 if (WARN_ON(cq->gdma_id >= gc->max_num_cqs)) { 2595 err = -EINVAL; 2596 goto out; 2597 } 2598 2599 gc->cq_table[cq->gdma_id] = cq->gdma_cq; 2600 2601 netif_napi_add_weight_locked(ndev, &cq->napi, mana_poll, 1); 2602 2603 WARN_ON(xdp_rxq_info_reg(&rxq->xdp_rxq, ndev, rxq_idx, 2604 cq->napi.napi_id)); 2605 WARN_ON(xdp_rxq_info_reg_mem_model(&rxq->xdp_rxq, MEM_TYPE_PAGE_POOL, 2606 rxq->page_pool)); 2607 2608 napi_enable_locked(&cq->napi); 2609 2610 mana_gd_ring_cq(cq->gdma_cq, SET_ARM_BIT); 2611 out: 2612 if (!err) 2613 return rxq; 2614 2615 netdev_err(ndev, "Failed to create RXQ: err = %d\n", err); 2616 2617 mana_destroy_rxq(apc, rxq, false); 2618 2619 if (cq) 2620 mana_deinit_cq(apc, cq); 2621 2622 return NULL; 2623 } 2624 2625 static void mana_create_rxq_debugfs(struct mana_port_context *apc, int idx) 2626 { 2627 struct mana_rxq *rxq; 2628 char qnum[32]; 2629 2630 rxq = apc->rxqs[idx]; 2631 2632 sprintf(qnum, "RX-%d", idx); 2633 rxq->mana_rx_debugfs = debugfs_create_dir(qnum, apc->mana_port_debugfs); 2634 debugfs_create_u32("rq_head", 0400, rxq->mana_rx_debugfs, &rxq->gdma_rq->head); 2635 debugfs_create_u32("rq_tail", 0400, rxq->mana_rx_debugfs, &rxq->gdma_rq->tail); 2636 debugfs_create_u32("rq_nbuf", 0400, rxq->mana_rx_debugfs, &rxq->num_rx_buf); 2637 debugfs_create_u32("cq_head", 0400, rxq->mana_rx_debugfs, 2638 &rxq->rx_cq.gdma_cq->head); 2639 debugfs_create_u32("cq_tail", 0400, rxq->mana_rx_debugfs, 2640 &rxq->rx_cq.gdma_cq->tail); 2641 debugfs_create_u32("cq_budget", 0400, rxq->mana_rx_debugfs, &rxq->rx_cq.budget); 2642 debugfs_create_file("rxq_dump", 0400, rxq->mana_rx_debugfs, rxq->gdma_rq, &mana_dbg_q_fops); 2643 debugfs_create_file("cq_dump", 0400, rxq->mana_rx_debugfs, rxq->rx_cq.gdma_cq, 2644 &mana_dbg_q_fops); 2645 } 2646 2647 static int mana_add_rx_queues(struct mana_port_context *apc, 2648 struct net_device *ndev) 2649 { 2650 struct mana_context *ac = apc->ac; 2651 struct mana_rxq *rxq; 2652 int err = 0; 2653 int i; 2654 2655 for (i = 0; i < apc->num_queues; i++) { 2656 rxq = mana_create_rxq(apc, i, &ac->eqs[i], ndev); 2657 if (!rxq) { 2658 err = -ENOMEM; 2659 netdev_err(ndev, "Failed to create rxq %d : %d\n", i, err); 2660 goto out; 2661 } 2662 2663 u64_stats_init(&rxq->stats.syncp); 2664 2665 apc->rxqs[i] = rxq; 2666 2667 mana_create_rxq_debugfs(apc, i); 2668 } 2669 2670 apc->default_rxobj = apc->rxqs[0]->rxobj; 2671 out: 2672 return err; 2673 } 2674 2675 static void mana_destroy_vport(struct mana_port_context *apc) 2676 { 2677 struct gdma_dev *gd = apc->ac->gdma_dev; 2678 struct mana_rxq *rxq; 2679 u32 rxq_idx; 2680 2681 for (rxq_idx = 0; rxq_idx < apc->num_queues; rxq_idx++) { 2682 rxq = apc->rxqs[rxq_idx]; 2683 if (!rxq) 2684 continue; 2685 2686 mana_destroy_rxq(apc, rxq, true); 2687 apc->rxqs[rxq_idx] = NULL; 2688 } 2689 2690 mana_destroy_txq(apc); 2691 mana_uncfg_vport(apc); 2692 2693 if (gd->gdma_context->is_pf && !apc->ac->bm_hostmode) 2694 mana_pf_deregister_hw_vport(apc); 2695 } 2696 2697 static int mana_create_vport(struct mana_port_context *apc, 2698 struct net_device *net) 2699 { 2700 struct gdma_dev *gd = apc->ac->gdma_dev; 2701 int err; 2702 2703 apc->default_rxobj = INVALID_MANA_HANDLE; 2704 2705 if (gd->gdma_context->is_pf && !apc->ac->bm_hostmode) { 2706 err = mana_pf_register_hw_vport(apc); 2707 if (err) 2708 return err; 2709 } 2710 2711 err = mana_cfg_vport(apc, gd->pdid, gd->doorbell); 2712 if (err) 2713 return err; 2714 2715 return mana_create_txq(apc, net); 2716 } 2717 2718 static int mana_rss_table_alloc(struct mana_port_context *apc) 2719 { 2720 if (!apc->indir_table_sz) { 2721 netdev_err(apc->ndev, 2722 "Indirection table size not set for vPort %d\n", 2723 apc->port_idx); 2724 return -EINVAL; 2725 } 2726 2727 apc->indir_table = kcalloc(apc->indir_table_sz, sizeof(u32), GFP_KERNEL); 2728 if (!apc->indir_table) 2729 return -ENOMEM; 2730 2731 apc->rxobj_table = kcalloc(apc->indir_table_sz, sizeof(mana_handle_t), GFP_KERNEL); 2732 if (!apc->rxobj_table) { 2733 kfree(apc->indir_table); 2734 return -ENOMEM; 2735 } 2736 2737 return 0; 2738 } 2739 2740 static void mana_rss_table_init(struct mana_port_context *apc) 2741 { 2742 int i; 2743 2744 for (i = 0; i < apc->indir_table_sz; i++) 2745 apc->indir_table[i] = 2746 ethtool_rxfh_indir_default(i, apc->num_queues); 2747 } 2748 2749 int mana_config_rss(struct mana_port_context *apc, enum TRI_STATE rx, 2750 bool update_hash, bool update_tab) 2751 { 2752 u32 queue_idx; 2753 int err; 2754 int i; 2755 2756 if (update_tab) { 2757 for (i = 0; i < apc->indir_table_sz; i++) { 2758 queue_idx = apc->indir_table[i]; 2759 apc->rxobj_table[i] = apc->rxqs[queue_idx]->rxobj; 2760 } 2761 } 2762 2763 err = mana_cfg_vport_steering(apc, rx, true, update_hash, update_tab); 2764 if (err) 2765 return err; 2766 2767 mana_fence_rqs(apc); 2768 2769 return 0; 2770 } 2771 2772 void mana_query_gf_stats(struct mana_port_context *apc) 2773 { 2774 struct mana_query_gf_stat_resp resp = {}; 2775 struct mana_query_gf_stat_req req = {}; 2776 struct net_device *ndev = apc->ndev; 2777 int err; 2778 2779 mana_gd_init_req_hdr(&req.hdr, MANA_QUERY_GF_STAT, 2780 sizeof(req), sizeof(resp)); 2781 req.hdr.resp.msg_version = GDMA_MESSAGE_V2; 2782 req.req_stats = STATISTICS_FLAGS_RX_DISCARDS_NO_WQE | 2783 STATISTICS_FLAGS_RX_ERRORS_VPORT_DISABLED | 2784 STATISTICS_FLAGS_HC_RX_BYTES | 2785 STATISTICS_FLAGS_HC_RX_UCAST_PACKETS | 2786 STATISTICS_FLAGS_HC_RX_UCAST_BYTES | 2787 STATISTICS_FLAGS_HC_RX_MCAST_PACKETS | 2788 STATISTICS_FLAGS_HC_RX_MCAST_BYTES | 2789 STATISTICS_FLAGS_HC_RX_BCAST_PACKETS | 2790 STATISTICS_FLAGS_HC_RX_BCAST_BYTES | 2791 STATISTICS_FLAGS_TX_ERRORS_GF_DISABLED | 2792 STATISTICS_FLAGS_TX_ERRORS_VPORT_DISABLED | 2793 STATISTICS_FLAGS_TX_ERRORS_INVAL_VPORT_OFFSET_PACKETS | 2794 STATISTICS_FLAGS_TX_ERRORS_VLAN_ENFORCEMENT | 2795 STATISTICS_FLAGS_TX_ERRORS_ETH_TYPE_ENFORCEMENT | 2796 STATISTICS_FLAGS_TX_ERRORS_SA_ENFORCEMENT | 2797 STATISTICS_FLAGS_TX_ERRORS_SQPDID_ENFORCEMENT | 2798 STATISTICS_FLAGS_TX_ERRORS_CQPDID_ENFORCEMENT | 2799 STATISTICS_FLAGS_TX_ERRORS_MTU_VIOLATION | 2800 STATISTICS_FLAGS_TX_ERRORS_INVALID_OOB | 2801 STATISTICS_FLAGS_HC_TX_BYTES | 2802 STATISTICS_FLAGS_HC_TX_UCAST_PACKETS | 2803 STATISTICS_FLAGS_HC_TX_UCAST_BYTES | 2804 STATISTICS_FLAGS_HC_TX_MCAST_PACKETS | 2805 STATISTICS_FLAGS_HC_TX_MCAST_BYTES | 2806 STATISTICS_FLAGS_HC_TX_BCAST_PACKETS | 2807 STATISTICS_FLAGS_HC_TX_BCAST_BYTES | 2808 STATISTICS_FLAGS_TX_ERRORS_GDMA_ERROR; 2809 2810 err = mana_send_request(apc->ac, &req, sizeof(req), &resp, 2811 sizeof(resp)); 2812 if (err) { 2813 netdev_err(ndev, "Failed to query GF stats: %d\n", err); 2814 return; 2815 } 2816 err = mana_verify_resp_hdr(&resp.hdr, MANA_QUERY_GF_STAT, 2817 sizeof(resp)); 2818 if (err || resp.hdr.status) { 2819 netdev_err(ndev, "Failed to query GF stats: %d, 0x%x\n", err, 2820 resp.hdr.status); 2821 return; 2822 } 2823 2824 apc->eth_stats.hc_rx_discards_no_wqe = resp.rx_discards_nowqe; 2825 apc->eth_stats.hc_rx_err_vport_disabled = resp.rx_err_vport_disabled; 2826 apc->eth_stats.hc_rx_bytes = resp.hc_rx_bytes; 2827 apc->eth_stats.hc_rx_ucast_pkts = resp.hc_rx_ucast_pkts; 2828 apc->eth_stats.hc_rx_ucast_bytes = resp.hc_rx_ucast_bytes; 2829 apc->eth_stats.hc_rx_bcast_pkts = resp.hc_rx_bcast_pkts; 2830 apc->eth_stats.hc_rx_bcast_bytes = resp.hc_rx_bcast_bytes; 2831 apc->eth_stats.hc_rx_mcast_pkts = resp.hc_rx_mcast_pkts; 2832 apc->eth_stats.hc_rx_mcast_bytes = resp.hc_rx_mcast_bytes; 2833 apc->eth_stats.hc_tx_err_gf_disabled = resp.tx_err_gf_disabled; 2834 apc->eth_stats.hc_tx_err_vport_disabled = resp.tx_err_vport_disabled; 2835 apc->eth_stats.hc_tx_err_inval_vportoffset_pkt = 2836 resp.tx_err_inval_vport_offset_pkt; 2837 apc->eth_stats.hc_tx_err_vlan_enforcement = 2838 resp.tx_err_vlan_enforcement; 2839 apc->eth_stats.hc_tx_err_eth_type_enforcement = 2840 resp.tx_err_ethtype_enforcement; 2841 apc->eth_stats.hc_tx_err_sa_enforcement = resp.tx_err_SA_enforcement; 2842 apc->eth_stats.hc_tx_err_sqpdid_enforcement = 2843 resp.tx_err_SQPDID_enforcement; 2844 apc->eth_stats.hc_tx_err_cqpdid_enforcement = 2845 resp.tx_err_CQPDID_enforcement; 2846 apc->eth_stats.hc_tx_err_mtu_violation = resp.tx_err_mtu_violation; 2847 apc->eth_stats.hc_tx_err_inval_oob = resp.tx_err_inval_oob; 2848 apc->eth_stats.hc_tx_bytes = resp.hc_tx_bytes; 2849 apc->eth_stats.hc_tx_ucast_pkts = resp.hc_tx_ucast_pkts; 2850 apc->eth_stats.hc_tx_ucast_bytes = resp.hc_tx_ucast_bytes; 2851 apc->eth_stats.hc_tx_bcast_pkts = resp.hc_tx_bcast_pkts; 2852 apc->eth_stats.hc_tx_bcast_bytes = resp.hc_tx_bcast_bytes; 2853 apc->eth_stats.hc_tx_mcast_pkts = resp.hc_tx_mcast_pkts; 2854 apc->eth_stats.hc_tx_mcast_bytes = resp.hc_tx_mcast_bytes; 2855 apc->eth_stats.hc_tx_err_gdma = resp.tx_err_gdma; 2856 } 2857 2858 void mana_query_phy_stats(struct mana_port_context *apc) 2859 { 2860 struct mana_query_phy_stat_resp resp = {}; 2861 struct mana_query_phy_stat_req req = {}; 2862 struct net_device *ndev = apc->ndev; 2863 int err; 2864 2865 mana_gd_init_req_hdr(&req.hdr, MANA_QUERY_PHY_STAT, 2866 sizeof(req), sizeof(resp)); 2867 err = mana_send_request(apc->ac, &req, sizeof(req), &resp, 2868 sizeof(resp)); 2869 if (err) 2870 return; 2871 2872 err = mana_verify_resp_hdr(&resp.hdr, MANA_QUERY_PHY_STAT, 2873 sizeof(resp)); 2874 if (err || resp.hdr.status) { 2875 netdev_err(ndev, 2876 "Failed to query PHY stats: %d, resp:0x%x\n", 2877 err, resp.hdr.status); 2878 return; 2879 } 2880 2881 /* Aggregate drop counters */ 2882 apc->phy_stats.rx_pkt_drop_phy = resp.rx_pkt_drop_phy; 2883 apc->phy_stats.tx_pkt_drop_phy = resp.tx_pkt_drop_phy; 2884 2885 /* Per TC traffic Counters */ 2886 apc->phy_stats.rx_pkt_tc0_phy = resp.rx_pkt_tc0_phy; 2887 apc->phy_stats.tx_pkt_tc0_phy = resp.tx_pkt_tc0_phy; 2888 apc->phy_stats.rx_pkt_tc1_phy = resp.rx_pkt_tc1_phy; 2889 apc->phy_stats.tx_pkt_tc1_phy = resp.tx_pkt_tc1_phy; 2890 apc->phy_stats.rx_pkt_tc2_phy = resp.rx_pkt_tc2_phy; 2891 apc->phy_stats.tx_pkt_tc2_phy = resp.tx_pkt_tc2_phy; 2892 apc->phy_stats.rx_pkt_tc3_phy = resp.rx_pkt_tc3_phy; 2893 apc->phy_stats.tx_pkt_tc3_phy = resp.tx_pkt_tc3_phy; 2894 apc->phy_stats.rx_pkt_tc4_phy = resp.rx_pkt_tc4_phy; 2895 apc->phy_stats.tx_pkt_tc4_phy = resp.tx_pkt_tc4_phy; 2896 apc->phy_stats.rx_pkt_tc5_phy = resp.rx_pkt_tc5_phy; 2897 apc->phy_stats.tx_pkt_tc5_phy = resp.tx_pkt_tc5_phy; 2898 apc->phy_stats.rx_pkt_tc6_phy = resp.rx_pkt_tc6_phy; 2899 apc->phy_stats.tx_pkt_tc6_phy = resp.tx_pkt_tc6_phy; 2900 apc->phy_stats.rx_pkt_tc7_phy = resp.rx_pkt_tc7_phy; 2901 apc->phy_stats.tx_pkt_tc7_phy = resp.tx_pkt_tc7_phy; 2902 2903 /* Per TC byte Counters */ 2904 apc->phy_stats.rx_byte_tc0_phy = resp.rx_byte_tc0_phy; 2905 apc->phy_stats.tx_byte_tc0_phy = resp.tx_byte_tc0_phy; 2906 apc->phy_stats.rx_byte_tc1_phy = resp.rx_byte_tc1_phy; 2907 apc->phy_stats.tx_byte_tc1_phy = resp.tx_byte_tc1_phy; 2908 apc->phy_stats.rx_byte_tc2_phy = resp.rx_byte_tc2_phy; 2909 apc->phy_stats.tx_byte_tc2_phy = resp.tx_byte_tc2_phy; 2910 apc->phy_stats.rx_byte_tc3_phy = resp.rx_byte_tc3_phy; 2911 apc->phy_stats.tx_byte_tc3_phy = resp.tx_byte_tc3_phy; 2912 apc->phy_stats.rx_byte_tc4_phy = resp.rx_byte_tc4_phy; 2913 apc->phy_stats.tx_byte_tc4_phy = resp.tx_byte_tc4_phy; 2914 apc->phy_stats.rx_byte_tc5_phy = resp.rx_byte_tc5_phy; 2915 apc->phy_stats.tx_byte_tc5_phy = resp.tx_byte_tc5_phy; 2916 apc->phy_stats.rx_byte_tc6_phy = resp.rx_byte_tc6_phy; 2917 apc->phy_stats.tx_byte_tc6_phy = resp.tx_byte_tc6_phy; 2918 apc->phy_stats.rx_byte_tc7_phy = resp.rx_byte_tc7_phy; 2919 apc->phy_stats.tx_byte_tc7_phy = resp.tx_byte_tc7_phy; 2920 2921 /* Per TC pause Counters */ 2922 apc->phy_stats.rx_pause_tc0_phy = resp.rx_pause_tc0_phy; 2923 apc->phy_stats.tx_pause_tc0_phy = resp.tx_pause_tc0_phy; 2924 apc->phy_stats.rx_pause_tc1_phy = resp.rx_pause_tc1_phy; 2925 apc->phy_stats.tx_pause_tc1_phy = resp.tx_pause_tc1_phy; 2926 apc->phy_stats.rx_pause_tc2_phy = resp.rx_pause_tc2_phy; 2927 apc->phy_stats.tx_pause_tc2_phy = resp.tx_pause_tc2_phy; 2928 apc->phy_stats.rx_pause_tc3_phy = resp.rx_pause_tc3_phy; 2929 apc->phy_stats.tx_pause_tc3_phy = resp.tx_pause_tc3_phy; 2930 apc->phy_stats.rx_pause_tc4_phy = resp.rx_pause_tc4_phy; 2931 apc->phy_stats.tx_pause_tc4_phy = resp.tx_pause_tc4_phy; 2932 apc->phy_stats.rx_pause_tc5_phy = resp.rx_pause_tc5_phy; 2933 apc->phy_stats.tx_pause_tc5_phy = resp.tx_pause_tc5_phy; 2934 apc->phy_stats.rx_pause_tc6_phy = resp.rx_pause_tc6_phy; 2935 apc->phy_stats.tx_pause_tc6_phy = resp.tx_pause_tc6_phy; 2936 apc->phy_stats.rx_pause_tc7_phy = resp.rx_pause_tc7_phy; 2937 apc->phy_stats.tx_pause_tc7_phy = resp.tx_pause_tc7_phy; 2938 } 2939 2940 static int mana_init_port(struct net_device *ndev) 2941 { 2942 struct mana_port_context *apc = netdev_priv(ndev); 2943 struct gdma_dev *gd = apc->ac->gdma_dev; 2944 u32 max_txq, max_rxq, max_queues; 2945 int port_idx = apc->port_idx; 2946 struct gdma_context *gc; 2947 char vport[32]; 2948 int err; 2949 2950 err = mana_init_port_context(apc); 2951 if (err) 2952 return err; 2953 2954 gc = gd->gdma_context; 2955 2956 err = mana_query_vport_cfg(apc, port_idx, &max_txq, &max_rxq, 2957 &apc->indir_table_sz); 2958 if (err) { 2959 netdev_err(ndev, "Failed to query info for vPort %d\n", 2960 port_idx); 2961 goto reset_apc; 2962 } 2963 2964 max_queues = min_t(u32, max_txq, max_rxq); 2965 if (apc->max_queues > max_queues) 2966 apc->max_queues = max_queues; 2967 2968 if (apc->num_queues > apc->max_queues) 2969 apc->num_queues = apc->max_queues; 2970 2971 eth_hw_addr_set(ndev, apc->mac_addr); 2972 sprintf(vport, "vport%d", port_idx); 2973 apc->mana_port_debugfs = debugfs_create_dir(vport, gc->mana_pci_debugfs); 2974 return 0; 2975 2976 reset_apc: 2977 mana_cleanup_port_context(apc); 2978 return err; 2979 } 2980 2981 int mana_alloc_queues(struct net_device *ndev) 2982 { 2983 struct mana_port_context *apc = netdev_priv(ndev); 2984 struct gdma_dev *gd = apc->ac->gdma_dev; 2985 int err; 2986 2987 err = mana_create_vport(apc, ndev); 2988 if (err) { 2989 netdev_err(ndev, "Failed to create vPort %u : %d\n", apc->port_idx, err); 2990 return err; 2991 } 2992 2993 err = netif_set_real_num_tx_queues(ndev, apc->num_queues); 2994 if (err) { 2995 netdev_err(ndev, 2996 "netif_set_real_num_tx_queues () failed for ndev with num_queues %u : %d\n", 2997 apc->num_queues, err); 2998 goto destroy_vport; 2999 } 3000 3001 err = mana_add_rx_queues(apc, ndev); 3002 if (err) 3003 goto destroy_vport; 3004 3005 apc->rss_state = apc->num_queues > 1 ? TRI_STATE_TRUE : TRI_STATE_FALSE; 3006 3007 err = netif_set_real_num_rx_queues(ndev, apc->num_queues); 3008 if (err) { 3009 netdev_err(ndev, 3010 "netif_set_real_num_rx_queues () failed for ndev with num_queues %u : %d\n", 3011 apc->num_queues, err); 3012 goto destroy_vport; 3013 } 3014 3015 mana_rss_table_init(apc); 3016 3017 err = mana_config_rss(apc, TRI_STATE_TRUE, true, true); 3018 if (err) { 3019 netdev_err(ndev, "Failed to configure RSS table: %d\n", err); 3020 goto destroy_vport; 3021 } 3022 3023 if (gd->gdma_context->is_pf && !apc->ac->bm_hostmode) { 3024 err = mana_pf_register_filter(apc); 3025 if (err) 3026 goto destroy_vport; 3027 } 3028 3029 mana_chn_setxdp(apc, mana_xdp_get(apc)); 3030 3031 return 0; 3032 3033 destroy_vport: 3034 mana_destroy_vport(apc); 3035 return err; 3036 } 3037 3038 int mana_attach(struct net_device *ndev) 3039 { 3040 struct mana_port_context *apc = netdev_priv(ndev); 3041 int err; 3042 3043 ASSERT_RTNL(); 3044 3045 err = mana_init_port(ndev); 3046 if (err) 3047 return err; 3048 3049 if (apc->port_st_save) { 3050 err = mana_alloc_queues(ndev); 3051 if (err) { 3052 mana_cleanup_port_context(apc); 3053 return err; 3054 } 3055 } 3056 3057 apc->port_is_up = apc->port_st_save; 3058 3059 /* Ensure port state updated before txq state */ 3060 smp_wmb(); 3061 3062 if (apc->port_is_up) 3063 netif_carrier_on(ndev); 3064 3065 netif_device_attach(ndev); 3066 3067 return 0; 3068 } 3069 3070 static int mana_dealloc_queues(struct net_device *ndev) 3071 { 3072 struct mana_port_context *apc = netdev_priv(ndev); 3073 unsigned long timeout = jiffies + 120 * HZ; 3074 struct gdma_dev *gd = apc->ac->gdma_dev; 3075 struct mana_txq *txq; 3076 struct sk_buff *skb; 3077 int i, err; 3078 u32 tsleep; 3079 3080 if (apc->port_is_up) 3081 return -EINVAL; 3082 3083 mana_chn_setxdp(apc, NULL); 3084 3085 if (gd->gdma_context->is_pf && !apc->ac->bm_hostmode) 3086 mana_pf_deregister_filter(apc); 3087 3088 /* No packet can be transmitted now since apc->port_is_up is false. 3089 * There is still a tiny chance that mana_poll_tx_cq() can re-enable 3090 * a txq because it may not timely see apc->port_is_up being cleared 3091 * to false, but it doesn't matter since mana_start_xmit() drops any 3092 * new packets due to apc->port_is_up being false. 3093 * 3094 * Drain all the in-flight TX packets. 3095 * A timeout of 120 seconds for all the queues is used. 3096 * This will break the while loop when h/w is not responding. 3097 * This value of 120 has been decided here considering max 3098 * number of queues. 3099 */ 3100 3101 for (i = 0; i < apc->num_queues; i++) { 3102 txq = &apc->tx_qp[i].txq; 3103 tsleep = 1000; 3104 while (atomic_read(&txq->pending_sends) > 0 && 3105 time_before(jiffies, timeout)) { 3106 usleep_range(tsleep, tsleep + 1000); 3107 tsleep <<= 1; 3108 } 3109 if (atomic_read(&txq->pending_sends)) { 3110 err = pcie_flr(to_pci_dev(gd->gdma_context->dev)); 3111 if (err) { 3112 netdev_err(ndev, "flr failed %d with %d pkts pending in txq %u\n", 3113 err, atomic_read(&txq->pending_sends), 3114 txq->gdma_txq_id); 3115 } 3116 break; 3117 } 3118 } 3119 3120 for (i = 0; i < apc->num_queues; i++) { 3121 txq = &apc->tx_qp[i].txq; 3122 while ((skb = skb_dequeue(&txq->pending_skbs))) { 3123 mana_unmap_skb(skb, apc); 3124 dev_kfree_skb_any(skb); 3125 } 3126 atomic_set(&txq->pending_sends, 0); 3127 } 3128 /* We're 100% sure the queues can no longer be woken up, because 3129 * we're sure now mana_poll_tx_cq() can't be running. 3130 */ 3131 3132 apc->rss_state = TRI_STATE_FALSE; 3133 err = mana_config_rss(apc, TRI_STATE_FALSE, false, false); 3134 if (err && mana_en_need_log(apc, err)) 3135 netdev_err(ndev, "Failed to disable vPort: %d\n", err); 3136 3137 /* Even in err case, still need to cleanup the vPort */ 3138 mana_destroy_vport(apc); 3139 3140 return 0; 3141 } 3142 3143 int mana_detach(struct net_device *ndev, bool from_close) 3144 { 3145 struct mana_port_context *apc = netdev_priv(ndev); 3146 int err; 3147 3148 ASSERT_RTNL(); 3149 3150 apc->port_st_save = apc->port_is_up; 3151 apc->port_is_up = false; 3152 3153 /* Ensure port state updated before txq state */ 3154 smp_wmb(); 3155 3156 netif_tx_disable(ndev); 3157 netif_carrier_off(ndev); 3158 3159 if (apc->port_st_save) { 3160 err = mana_dealloc_queues(ndev); 3161 if (err) { 3162 netdev_err(ndev, "%s failed to deallocate queues: %d\n", __func__, err); 3163 return err; 3164 } 3165 } 3166 3167 if (!from_close) { 3168 netif_device_detach(ndev); 3169 mana_cleanup_port_context(apc); 3170 } 3171 3172 return 0; 3173 } 3174 3175 static int mana_probe_port(struct mana_context *ac, int port_idx, 3176 struct net_device **ndev_storage) 3177 { 3178 struct gdma_context *gc = ac->gdma_dev->gdma_context; 3179 struct mana_port_context *apc; 3180 struct net_device *ndev; 3181 int err; 3182 3183 ndev = alloc_etherdev_mq(sizeof(struct mana_port_context), 3184 gc->max_num_queues); 3185 if (!ndev) 3186 return -ENOMEM; 3187 3188 *ndev_storage = ndev; 3189 3190 apc = netdev_priv(ndev); 3191 apc->ac = ac; 3192 apc->ndev = ndev; 3193 apc->max_queues = gc->max_num_queues; 3194 apc->num_queues = gc->max_num_queues; 3195 apc->tx_queue_size = DEF_TX_BUFFERS_PER_QUEUE; 3196 apc->rx_queue_size = DEF_RX_BUFFERS_PER_QUEUE; 3197 apc->port_handle = INVALID_MANA_HANDLE; 3198 apc->pf_filter_handle = INVALID_MANA_HANDLE; 3199 apc->port_idx = port_idx; 3200 3201 mutex_init(&apc->vport_mutex); 3202 apc->vport_use_count = 0; 3203 3204 ndev->netdev_ops = &mana_devops; 3205 ndev->ethtool_ops = &mana_ethtool_ops; 3206 ndev->mtu = ETH_DATA_LEN; 3207 ndev->max_mtu = gc->adapter_mtu - ETH_HLEN; 3208 ndev->min_mtu = ETH_MIN_MTU; 3209 ndev->needed_headroom = MANA_HEADROOM; 3210 ndev->dev_port = port_idx; 3211 SET_NETDEV_DEV(ndev, gc->dev); 3212 3213 netif_set_tso_max_size(ndev, GSO_MAX_SIZE); 3214 3215 netif_carrier_off(ndev); 3216 3217 netdev_rss_key_fill(apc->hashkey, MANA_HASH_KEY_SIZE); 3218 3219 err = mana_init_port(ndev); 3220 if (err) 3221 goto free_net; 3222 3223 err = mana_rss_table_alloc(apc); 3224 if (err) 3225 goto reset_apc; 3226 3227 netdev_lockdep_set_classes(ndev); 3228 3229 ndev->hw_features = NETIF_F_SG | NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM; 3230 ndev->hw_features |= NETIF_F_RXCSUM; 3231 ndev->hw_features |= NETIF_F_TSO | NETIF_F_TSO6; 3232 ndev->hw_features |= NETIF_F_RXHASH; 3233 ndev->features = ndev->hw_features | NETIF_F_HW_VLAN_CTAG_TX | 3234 NETIF_F_HW_VLAN_CTAG_RX; 3235 ndev->vlan_features = ndev->features; 3236 xdp_set_features_flag(ndev, NETDEV_XDP_ACT_BASIC | 3237 NETDEV_XDP_ACT_REDIRECT | 3238 NETDEV_XDP_ACT_NDO_XMIT); 3239 3240 err = register_netdev(ndev); 3241 if (err) { 3242 netdev_err(ndev, "Unable to register netdev.\n"); 3243 goto free_indir; 3244 } 3245 3246 debugfs_create_u32("current_speed", 0400, apc->mana_port_debugfs, &apc->speed); 3247 3248 return 0; 3249 3250 free_indir: 3251 mana_cleanup_indir_table(apc); 3252 reset_apc: 3253 mana_cleanup_port_context(apc); 3254 free_net: 3255 *ndev_storage = NULL; 3256 netdev_err(ndev, "Failed to probe vPort %d: %d\n", port_idx, err); 3257 free_netdev(ndev); 3258 return err; 3259 } 3260 3261 static void adev_release(struct device *dev) 3262 { 3263 struct mana_adev *madev = container_of(dev, struct mana_adev, adev.dev); 3264 3265 kfree(madev); 3266 } 3267 3268 static void remove_adev(struct gdma_dev *gd) 3269 { 3270 struct auxiliary_device *adev = gd->adev; 3271 int id = adev->id; 3272 3273 auxiliary_device_delete(adev); 3274 auxiliary_device_uninit(adev); 3275 3276 mana_adev_idx_free(id); 3277 gd->adev = NULL; 3278 } 3279 3280 static int add_adev(struct gdma_dev *gd, const char *name) 3281 { 3282 struct auxiliary_device *adev; 3283 struct mana_adev *madev; 3284 int ret; 3285 3286 madev = kzalloc(sizeof(*madev), GFP_KERNEL); 3287 if (!madev) 3288 return -ENOMEM; 3289 3290 adev = &madev->adev; 3291 ret = mana_adev_idx_alloc(); 3292 if (ret < 0) 3293 goto idx_fail; 3294 adev->id = ret; 3295 3296 adev->name = name; 3297 adev->dev.parent = gd->gdma_context->dev; 3298 adev->dev.release = adev_release; 3299 madev->mdev = gd; 3300 3301 ret = auxiliary_device_init(adev); 3302 if (ret) 3303 goto init_fail; 3304 3305 /* madev is owned by the auxiliary device */ 3306 madev = NULL; 3307 ret = auxiliary_device_add(adev); 3308 if (ret) 3309 goto add_fail; 3310 3311 gd->adev = adev; 3312 dev_dbg(gd->gdma_context->dev, 3313 "Auxiliary device added successfully\n"); 3314 return 0; 3315 3316 add_fail: 3317 auxiliary_device_uninit(adev); 3318 3319 init_fail: 3320 mana_adev_idx_free(adev->id); 3321 3322 idx_fail: 3323 kfree(madev); 3324 3325 return ret; 3326 } 3327 3328 static void mana_rdma_service_handle(struct work_struct *work) 3329 { 3330 struct mana_service_work *serv_work = 3331 container_of(work, struct mana_service_work, work); 3332 struct gdma_dev *gd = serv_work->gdma_dev; 3333 struct device *dev = gd->gdma_context->dev; 3334 int ret; 3335 3336 if (READ_ONCE(gd->rdma_teardown)) 3337 goto out; 3338 3339 switch (serv_work->event) { 3340 case GDMA_SERVICE_TYPE_RDMA_SUSPEND: 3341 if (!gd->adev || gd->is_suspended) 3342 break; 3343 3344 remove_adev(gd); 3345 gd->is_suspended = true; 3346 break; 3347 3348 case GDMA_SERVICE_TYPE_RDMA_RESUME: 3349 if (!gd->is_suspended) 3350 break; 3351 3352 ret = add_adev(gd, "rdma"); 3353 if (ret) 3354 dev_err(dev, "Failed to add adev on resume: %d\n", ret); 3355 else 3356 gd->is_suspended = false; 3357 break; 3358 3359 default: 3360 dev_warn(dev, "unknown adev service event %u\n", 3361 serv_work->event); 3362 break; 3363 } 3364 3365 out: 3366 kfree(serv_work); 3367 } 3368 3369 int mana_rdma_service_event(struct gdma_context *gc, enum gdma_service_type event) 3370 { 3371 struct gdma_dev *gd = &gc->mana_ib; 3372 struct mana_service_work *serv_work; 3373 3374 if (gd->dev_id.type != GDMA_DEVICE_MANA_IB) { 3375 /* RDMA device is not detected on pci */ 3376 return 0; 3377 } 3378 3379 serv_work = kzalloc(sizeof(*serv_work), GFP_ATOMIC); 3380 if (!serv_work) 3381 return -ENOMEM; 3382 3383 serv_work->event = event; 3384 serv_work->gdma_dev = gd; 3385 3386 INIT_WORK(&serv_work->work, mana_rdma_service_handle); 3387 queue_work(gc->service_wq, &serv_work->work); 3388 3389 return 0; 3390 } 3391 3392 int mana_probe(struct gdma_dev *gd, bool resuming) 3393 { 3394 struct gdma_context *gc = gd->gdma_context; 3395 struct mana_context *ac = gd->driver_data; 3396 struct device *dev = gc->dev; 3397 u8 bm_hostmode = 0; 3398 u16 num_ports = 0; 3399 int err; 3400 int i; 3401 3402 dev_info(dev, 3403 "Microsoft Azure Network Adapter protocol version: %d.%d.%d\n", 3404 MANA_MAJOR_VERSION, MANA_MINOR_VERSION, MANA_MICRO_VERSION); 3405 3406 err = mana_gd_register_device(gd); 3407 if (err) 3408 return err; 3409 3410 if (!resuming) { 3411 ac = kzalloc(sizeof(*ac), GFP_KERNEL); 3412 if (!ac) 3413 return -ENOMEM; 3414 3415 ac->gdma_dev = gd; 3416 gd->driver_data = ac; 3417 } 3418 3419 err = mana_create_eq(ac); 3420 if (err) { 3421 dev_err(dev, "Failed to create EQs: %d\n", err); 3422 goto out; 3423 } 3424 3425 err = mana_query_device_cfg(ac, MANA_MAJOR_VERSION, MANA_MINOR_VERSION, 3426 MANA_MICRO_VERSION, &num_ports, &bm_hostmode); 3427 if (err) 3428 goto out; 3429 3430 ac->bm_hostmode = bm_hostmode; 3431 3432 if (!resuming) { 3433 ac->num_ports = num_ports; 3434 } else { 3435 if (ac->num_ports != num_ports) { 3436 dev_err(dev, "The number of vPorts changed: %d->%d\n", 3437 ac->num_ports, num_ports); 3438 err = -EPROTO; 3439 goto out; 3440 } 3441 } 3442 3443 if (ac->num_ports == 0) 3444 dev_err(dev, "Failed to detect any vPort\n"); 3445 3446 if (ac->num_ports > MAX_PORTS_IN_MANA_DEV) 3447 ac->num_ports = MAX_PORTS_IN_MANA_DEV; 3448 3449 if (!resuming) { 3450 for (i = 0; i < ac->num_ports; i++) { 3451 err = mana_probe_port(ac, i, &ac->ports[i]); 3452 /* we log the port for which the probe failed and stop 3453 * probes for subsequent ports. 3454 * Note that we keep running ports, for which the probes 3455 * were successful, unless add_adev fails too 3456 */ 3457 if (err) { 3458 dev_err(dev, "Probe Failed for port %d\n", i); 3459 break; 3460 } 3461 } 3462 } else { 3463 for (i = 0; i < ac->num_ports; i++) { 3464 rtnl_lock(); 3465 err = mana_attach(ac->ports[i]); 3466 rtnl_unlock(); 3467 /* we log the port for which the attach failed and stop 3468 * attach for subsequent ports 3469 * Note that we keep running ports, for which the attach 3470 * were successful, unless add_adev fails too 3471 */ 3472 if (err) { 3473 dev_err(dev, "Attach Failed for port %d\n", i); 3474 break; 3475 } 3476 } 3477 } 3478 3479 err = add_adev(gd, "eth"); 3480 out: 3481 if (err) { 3482 mana_remove(gd, false); 3483 } else { 3484 dev_dbg(dev, "gd=%p, id=%u, num_ports=%d, type=%u, instance=%u\n", 3485 gd, gd->dev_id.as_uint32, ac->num_ports, 3486 gd->dev_id.type, gd->dev_id.instance); 3487 dev_dbg(dev, "%s succeeded\n", __func__); 3488 } 3489 3490 return err; 3491 } 3492 3493 void mana_remove(struct gdma_dev *gd, bool suspending) 3494 { 3495 struct gdma_context *gc = gd->gdma_context; 3496 struct mana_context *ac = gd->driver_data; 3497 struct mana_port_context *apc; 3498 struct device *dev = gc->dev; 3499 struct net_device *ndev; 3500 int err; 3501 int i; 3502 3503 /* adev currently doesn't support suspending, always remove it */ 3504 if (gd->adev) 3505 remove_adev(gd); 3506 3507 for (i = 0; i < ac->num_ports; i++) { 3508 ndev = ac->ports[i]; 3509 apc = netdev_priv(ndev); 3510 if (!ndev) { 3511 if (i == 0) 3512 dev_err(dev, "No net device to remove\n"); 3513 goto out; 3514 } 3515 3516 /* All cleanup actions should stay after rtnl_lock(), otherwise 3517 * other functions may access partially cleaned up data. 3518 */ 3519 rtnl_lock(); 3520 3521 err = mana_detach(ndev, false); 3522 if (err) 3523 netdev_err(ndev, "Failed to detach vPort %d: %d\n", 3524 i, err); 3525 3526 if (suspending) { 3527 /* No need to unregister the ndev. */ 3528 rtnl_unlock(); 3529 continue; 3530 } 3531 3532 unregister_netdevice(ndev); 3533 mana_cleanup_indir_table(apc); 3534 3535 rtnl_unlock(); 3536 3537 free_netdev(ndev); 3538 } 3539 3540 mana_destroy_eq(ac); 3541 out: 3542 mana_gd_deregister_device(gd); 3543 3544 if (suspending) 3545 return; 3546 3547 gd->driver_data = NULL; 3548 gd->gdma_context = NULL; 3549 kfree(ac); 3550 dev_dbg(dev, "%s succeeded\n", __func__); 3551 } 3552 3553 int mana_rdma_probe(struct gdma_dev *gd) 3554 { 3555 int err = 0; 3556 3557 if (gd->dev_id.type != GDMA_DEVICE_MANA_IB) { 3558 /* RDMA device is not detected on pci */ 3559 return err; 3560 } 3561 3562 err = mana_gd_register_device(gd); 3563 if (err) 3564 return err; 3565 3566 err = add_adev(gd, "rdma"); 3567 if (err) 3568 mana_gd_deregister_device(gd); 3569 3570 return err; 3571 } 3572 3573 void mana_rdma_remove(struct gdma_dev *gd) 3574 { 3575 struct gdma_context *gc = gd->gdma_context; 3576 3577 if (gd->dev_id.type != GDMA_DEVICE_MANA_IB) { 3578 /* RDMA device is not detected on pci */ 3579 return; 3580 } 3581 3582 WRITE_ONCE(gd->rdma_teardown, true); 3583 flush_workqueue(gc->service_wq); 3584 3585 if (gd->adev) 3586 remove_adev(gd); 3587 3588 mana_gd_deregister_device(gd); 3589 } 3590 3591 struct net_device *mana_get_primary_netdev(struct mana_context *ac, 3592 u32 port_index, 3593 netdevice_tracker *tracker) 3594 { 3595 struct net_device *ndev; 3596 3597 if (port_index >= ac->num_ports) 3598 return NULL; 3599 3600 rcu_read_lock(); 3601 3602 /* If mana is used in netvsc, the upper netdevice should be returned. */ 3603 ndev = netdev_master_upper_dev_get_rcu(ac->ports[port_index]); 3604 3605 /* If there is no upper device, use the parent Ethernet device */ 3606 if (!ndev) 3607 ndev = ac->ports[port_index]; 3608 3609 netdev_hold(ndev, tracker, GFP_ATOMIC); 3610 rcu_read_unlock(); 3611 3612 return ndev; 3613 } 3614 EXPORT_SYMBOL_NS(mana_get_primary_netdev, "NET_MANA"); 3615