1 // SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) 2 /* Copyright 2017-2019 NXP */ 3 4 #include "enetc.h" 5 #include <linux/bpf_trace.h> 6 #include <linux/tcp.h> 7 #include <linux/udp.h> 8 #include <linux/vmalloc.h> 9 #include <linux/ptp_classify.h> 10 #include <net/ip6_checksum.h> 11 #include <net/pkt_sched.h> 12 #include <net/tso.h> 13 14 u32 enetc_port_mac_rd(struct enetc_si *si, u32 reg) 15 { 16 return enetc_port_rd(&si->hw, reg); 17 } 18 EXPORT_SYMBOL_GPL(enetc_port_mac_rd); 19 20 void enetc_port_mac_wr(struct enetc_si *si, u32 reg, u32 val) 21 { 22 enetc_port_wr(&si->hw, reg, val); 23 if (si->hw_features & ENETC_SI_F_QBU) 24 enetc_port_wr(&si->hw, reg + ENETC_PMAC_OFFSET, val); 25 } 26 EXPORT_SYMBOL_GPL(enetc_port_mac_wr); 27 28 static void enetc_change_preemptible_tcs(struct enetc_ndev_priv *priv, 29 u8 preemptible_tcs) 30 { 31 priv->preemptible_tcs = preemptible_tcs; 32 enetc_mm_commit_preemptible_tcs(priv); 33 } 34 35 static int enetc_num_stack_tx_queues(struct enetc_ndev_priv *priv) 36 { 37 int num_tx_rings = priv->num_tx_rings; 38 39 if (priv->xdp_prog) 40 return num_tx_rings - num_possible_cpus(); 41 42 return num_tx_rings; 43 } 44 45 static struct enetc_bdr *enetc_rx_ring_from_xdp_tx_ring(struct enetc_ndev_priv *priv, 46 struct enetc_bdr *tx_ring) 47 { 48 int index = &priv->tx_ring[tx_ring->index] - priv->xdp_tx_ring; 49 50 return priv->rx_ring[index]; 51 } 52 53 static struct sk_buff *enetc_tx_swbd_get_skb(struct enetc_tx_swbd *tx_swbd) 54 { 55 if (tx_swbd->is_xdp_tx || tx_swbd->is_xdp_redirect) 56 return NULL; 57 58 return tx_swbd->skb; 59 } 60 61 static struct xdp_frame * 62 enetc_tx_swbd_get_xdp_frame(struct enetc_tx_swbd *tx_swbd) 63 { 64 if (tx_swbd->is_xdp_redirect) 65 return tx_swbd->xdp_frame; 66 67 return NULL; 68 } 69 70 static void enetc_unmap_tx_buff(struct enetc_bdr *tx_ring, 71 struct enetc_tx_swbd *tx_swbd) 72 { 73 /* For XDP_TX, pages come from RX, whereas for the other contexts where 74 * we have is_dma_page_set, those come from skb_frag_dma_map. We need 75 * to match the DMA mapping length, so we need to differentiate those. 76 */ 77 if (tx_swbd->is_dma_page) 78 dma_unmap_page(tx_ring->dev, tx_swbd->dma, 79 tx_swbd->is_xdp_tx ? PAGE_SIZE : tx_swbd->len, 80 tx_swbd->dir); 81 else 82 dma_unmap_single(tx_ring->dev, tx_swbd->dma, 83 tx_swbd->len, tx_swbd->dir); 84 tx_swbd->dma = 0; 85 } 86 87 static void enetc_free_tx_frame(struct enetc_bdr *tx_ring, 88 struct enetc_tx_swbd *tx_swbd) 89 { 90 struct xdp_frame *xdp_frame = enetc_tx_swbd_get_xdp_frame(tx_swbd); 91 struct sk_buff *skb = enetc_tx_swbd_get_skb(tx_swbd); 92 93 if (tx_swbd->dma) 94 enetc_unmap_tx_buff(tx_ring, tx_swbd); 95 96 if (xdp_frame) { 97 xdp_return_frame(tx_swbd->xdp_frame); 98 tx_swbd->xdp_frame = NULL; 99 } else if (skb) { 100 dev_kfree_skb_any(skb); 101 tx_swbd->skb = NULL; 102 } 103 } 104 105 /* Let H/W know BD ring has been updated */ 106 static void enetc_update_tx_ring_tail(struct enetc_bdr *tx_ring) 107 { 108 /* includes wmb() */ 109 enetc_wr_reg_hot(tx_ring->tpir, tx_ring->next_to_use); 110 } 111 112 static int enetc_ptp_parse(struct sk_buff *skb, u8 *udp, 113 u8 *msgtype, u8 *twostep, 114 u16 *correction_offset, u16 *body_offset) 115 { 116 unsigned int ptp_class; 117 struct ptp_header *hdr; 118 unsigned int type; 119 u8 *base; 120 121 ptp_class = ptp_classify_raw(skb); 122 if (ptp_class == PTP_CLASS_NONE) 123 return -EINVAL; 124 125 hdr = ptp_parse_header(skb, ptp_class); 126 if (!hdr) 127 return -EINVAL; 128 129 type = ptp_class & PTP_CLASS_PMASK; 130 if (type == PTP_CLASS_IPV4 || type == PTP_CLASS_IPV6) 131 *udp = 1; 132 else 133 *udp = 0; 134 135 *msgtype = ptp_get_msgtype(hdr, ptp_class); 136 *twostep = hdr->flag_field[0] & 0x2; 137 138 base = skb_mac_header(skb); 139 *correction_offset = (u8 *)&hdr->correction - base; 140 *body_offset = (u8 *)hdr + sizeof(struct ptp_header) - base; 141 142 return 0; 143 } 144 145 static int enetc_map_tx_buffs(struct enetc_bdr *tx_ring, struct sk_buff *skb) 146 { 147 bool do_vlan, do_onestep_tstamp = false, do_twostep_tstamp = false; 148 struct enetc_ndev_priv *priv = netdev_priv(tx_ring->ndev); 149 struct enetc_hw *hw = &priv->si->hw; 150 struct enetc_tx_swbd *tx_swbd; 151 int len = skb_headlen(skb); 152 union enetc_tx_bd temp_bd; 153 u8 msgtype, twostep, udp; 154 union enetc_tx_bd *txbd; 155 u16 offset1, offset2; 156 int i, count = 0; 157 skb_frag_t *frag; 158 unsigned int f; 159 dma_addr_t dma; 160 u8 flags = 0; 161 162 i = tx_ring->next_to_use; 163 txbd = ENETC_TXBD(*tx_ring, i); 164 prefetchw(txbd); 165 166 dma = dma_map_single(tx_ring->dev, skb->data, len, DMA_TO_DEVICE); 167 if (unlikely(dma_mapping_error(tx_ring->dev, dma))) 168 goto dma_err; 169 170 temp_bd.addr = cpu_to_le64(dma); 171 temp_bd.buf_len = cpu_to_le16(len); 172 temp_bd.lstatus = 0; 173 174 tx_swbd = &tx_ring->tx_swbd[i]; 175 tx_swbd->dma = dma; 176 tx_swbd->len = len; 177 tx_swbd->is_dma_page = 0; 178 tx_swbd->dir = DMA_TO_DEVICE; 179 count++; 180 181 do_vlan = skb_vlan_tag_present(skb); 182 if (skb->cb[0] & ENETC_F_TX_ONESTEP_SYNC_TSTAMP) { 183 if (enetc_ptp_parse(skb, &udp, &msgtype, &twostep, &offset1, 184 &offset2) || 185 msgtype != PTP_MSGTYPE_SYNC || twostep) 186 WARN_ONCE(1, "Bad packet for one-step timestamping\n"); 187 else 188 do_onestep_tstamp = true; 189 } else if (skb->cb[0] & ENETC_F_TX_TSTAMP) { 190 do_twostep_tstamp = true; 191 } 192 193 tx_swbd->do_twostep_tstamp = do_twostep_tstamp; 194 tx_swbd->qbv_en = !!(priv->active_offloads & ENETC_F_QBV); 195 tx_swbd->check_wb = tx_swbd->do_twostep_tstamp || tx_swbd->qbv_en; 196 197 if (do_vlan || do_onestep_tstamp || do_twostep_tstamp) 198 flags |= ENETC_TXBD_FLAGS_EX; 199 200 if (tx_ring->tsd_enable) 201 flags |= ENETC_TXBD_FLAGS_TSE | ENETC_TXBD_FLAGS_TXSTART; 202 203 /* first BD needs frm_len and offload flags set */ 204 temp_bd.frm_len = cpu_to_le16(skb->len); 205 temp_bd.flags = flags; 206 207 if (flags & ENETC_TXBD_FLAGS_TSE) 208 temp_bd.txstart = enetc_txbd_set_tx_start(skb->skb_mstamp_ns, 209 flags); 210 211 if (flags & ENETC_TXBD_FLAGS_EX) { 212 u8 e_flags = 0; 213 *txbd = temp_bd; 214 enetc_clear_tx_bd(&temp_bd); 215 216 /* add extension BD for VLAN and/or timestamping */ 217 flags = 0; 218 tx_swbd++; 219 txbd++; 220 i++; 221 if (unlikely(i == tx_ring->bd_count)) { 222 i = 0; 223 tx_swbd = tx_ring->tx_swbd; 224 txbd = ENETC_TXBD(*tx_ring, 0); 225 } 226 prefetchw(txbd); 227 228 if (do_vlan) { 229 temp_bd.ext.vid = cpu_to_le16(skb_vlan_tag_get(skb)); 230 temp_bd.ext.tpid = 0; /* < C-TAG */ 231 e_flags |= ENETC_TXBD_E_FLAGS_VLAN_INS; 232 } 233 234 if (do_onestep_tstamp) { 235 u32 lo, hi, val; 236 u64 sec, nsec; 237 u8 *data; 238 239 lo = enetc_rd_hot(hw, ENETC_SICTR0); 240 hi = enetc_rd_hot(hw, ENETC_SICTR1); 241 sec = (u64)hi << 32 | lo; 242 nsec = do_div(sec, 1000000000); 243 244 /* Configure extension BD */ 245 temp_bd.ext.tstamp = cpu_to_le32(lo & 0x3fffffff); 246 e_flags |= ENETC_TXBD_E_FLAGS_ONE_STEP_PTP; 247 248 /* Update originTimestamp field of Sync packet 249 * - 48 bits seconds field 250 * - 32 bits nanseconds field 251 */ 252 data = skb_mac_header(skb); 253 *(__be16 *)(data + offset2) = 254 htons((sec >> 32) & 0xffff); 255 *(__be32 *)(data + offset2 + 2) = 256 htonl(sec & 0xffffffff); 257 *(__be32 *)(data + offset2 + 6) = htonl(nsec); 258 259 /* Configure single-step register */ 260 val = ENETC_PM0_SINGLE_STEP_EN; 261 val |= ENETC_SET_SINGLE_STEP_OFFSET(offset1); 262 if (udp) 263 val |= ENETC_PM0_SINGLE_STEP_CH; 264 265 enetc_port_mac_wr(priv->si, ENETC_PM0_SINGLE_STEP, 266 val); 267 } else if (do_twostep_tstamp) { 268 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; 269 e_flags |= ENETC_TXBD_E_FLAGS_TWO_STEP_PTP; 270 } 271 272 temp_bd.ext.e_flags = e_flags; 273 count++; 274 } 275 276 frag = &skb_shinfo(skb)->frags[0]; 277 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++, frag++) { 278 len = skb_frag_size(frag); 279 dma = skb_frag_dma_map(tx_ring->dev, frag, 0, len, 280 DMA_TO_DEVICE); 281 if (dma_mapping_error(tx_ring->dev, dma)) 282 goto dma_err; 283 284 *txbd = temp_bd; 285 enetc_clear_tx_bd(&temp_bd); 286 287 flags = 0; 288 tx_swbd++; 289 txbd++; 290 i++; 291 if (unlikely(i == tx_ring->bd_count)) { 292 i = 0; 293 tx_swbd = tx_ring->tx_swbd; 294 txbd = ENETC_TXBD(*tx_ring, 0); 295 } 296 prefetchw(txbd); 297 298 temp_bd.addr = cpu_to_le64(dma); 299 temp_bd.buf_len = cpu_to_le16(len); 300 301 tx_swbd->dma = dma; 302 tx_swbd->len = len; 303 tx_swbd->is_dma_page = 1; 304 tx_swbd->dir = DMA_TO_DEVICE; 305 count++; 306 } 307 308 /* last BD needs 'F' bit set */ 309 flags |= ENETC_TXBD_FLAGS_F; 310 temp_bd.flags = flags; 311 *txbd = temp_bd; 312 313 tx_ring->tx_swbd[i].is_eof = true; 314 tx_ring->tx_swbd[i].skb = skb; 315 316 enetc_bdr_idx_inc(tx_ring, &i); 317 tx_ring->next_to_use = i; 318 319 skb_tx_timestamp(skb); 320 321 enetc_update_tx_ring_tail(tx_ring); 322 323 return count; 324 325 dma_err: 326 dev_err(tx_ring->dev, "DMA map error"); 327 328 do { 329 tx_swbd = &tx_ring->tx_swbd[i]; 330 enetc_free_tx_frame(tx_ring, tx_swbd); 331 if (i == 0) 332 i = tx_ring->bd_count; 333 i--; 334 } while (count--); 335 336 return 0; 337 } 338 339 static void enetc_map_tx_tso_hdr(struct enetc_bdr *tx_ring, struct sk_buff *skb, 340 struct enetc_tx_swbd *tx_swbd, 341 union enetc_tx_bd *txbd, int *i, int hdr_len, 342 int data_len) 343 { 344 union enetc_tx_bd txbd_tmp; 345 u8 flags = 0, e_flags = 0; 346 dma_addr_t addr; 347 348 enetc_clear_tx_bd(&txbd_tmp); 349 addr = tx_ring->tso_headers_dma + *i * TSO_HEADER_SIZE; 350 351 if (skb_vlan_tag_present(skb)) 352 flags |= ENETC_TXBD_FLAGS_EX; 353 354 txbd_tmp.addr = cpu_to_le64(addr); 355 txbd_tmp.buf_len = cpu_to_le16(hdr_len); 356 357 /* first BD needs frm_len and offload flags set */ 358 txbd_tmp.frm_len = cpu_to_le16(hdr_len + data_len); 359 txbd_tmp.flags = flags; 360 361 /* For the TSO header we do not set the dma address since we do not 362 * want it unmapped when we do cleanup. We still set len so that we 363 * count the bytes sent. 364 */ 365 tx_swbd->len = hdr_len; 366 tx_swbd->do_twostep_tstamp = false; 367 tx_swbd->check_wb = false; 368 369 /* Actually write the header in the BD */ 370 *txbd = txbd_tmp; 371 372 /* Add extension BD for VLAN */ 373 if (flags & ENETC_TXBD_FLAGS_EX) { 374 /* Get the next BD */ 375 enetc_bdr_idx_inc(tx_ring, i); 376 txbd = ENETC_TXBD(*tx_ring, *i); 377 tx_swbd = &tx_ring->tx_swbd[*i]; 378 prefetchw(txbd); 379 380 /* Setup the VLAN fields */ 381 enetc_clear_tx_bd(&txbd_tmp); 382 txbd_tmp.ext.vid = cpu_to_le16(skb_vlan_tag_get(skb)); 383 txbd_tmp.ext.tpid = 0; /* < C-TAG */ 384 e_flags |= ENETC_TXBD_E_FLAGS_VLAN_INS; 385 386 /* Write the BD */ 387 txbd_tmp.ext.e_flags = e_flags; 388 *txbd = txbd_tmp; 389 } 390 } 391 392 static int enetc_map_tx_tso_data(struct enetc_bdr *tx_ring, struct sk_buff *skb, 393 struct enetc_tx_swbd *tx_swbd, 394 union enetc_tx_bd *txbd, char *data, 395 int size, bool last_bd) 396 { 397 union enetc_tx_bd txbd_tmp; 398 dma_addr_t addr; 399 u8 flags = 0; 400 401 enetc_clear_tx_bd(&txbd_tmp); 402 403 addr = dma_map_single(tx_ring->dev, data, size, DMA_TO_DEVICE); 404 if (unlikely(dma_mapping_error(tx_ring->dev, addr))) { 405 netdev_err(tx_ring->ndev, "DMA map error\n"); 406 return -ENOMEM; 407 } 408 409 if (last_bd) { 410 flags |= ENETC_TXBD_FLAGS_F; 411 tx_swbd->is_eof = 1; 412 } 413 414 txbd_tmp.addr = cpu_to_le64(addr); 415 txbd_tmp.buf_len = cpu_to_le16(size); 416 txbd_tmp.flags = flags; 417 418 tx_swbd->dma = addr; 419 tx_swbd->len = size; 420 tx_swbd->dir = DMA_TO_DEVICE; 421 422 *txbd = txbd_tmp; 423 424 return 0; 425 } 426 427 static __wsum enetc_tso_hdr_csum(struct tso_t *tso, struct sk_buff *skb, 428 char *hdr, int hdr_len, int *l4_hdr_len) 429 { 430 char *l4_hdr = hdr + skb_transport_offset(skb); 431 int mac_hdr_len = skb_network_offset(skb); 432 433 if (tso->tlen != sizeof(struct udphdr)) { 434 struct tcphdr *tcph = (struct tcphdr *)(l4_hdr); 435 436 tcph->check = 0; 437 } else { 438 struct udphdr *udph = (struct udphdr *)(l4_hdr); 439 440 udph->check = 0; 441 } 442 443 /* Compute the IP checksum. This is necessary since tso_build_hdr() 444 * already incremented the IP ID field. 445 */ 446 if (!tso->ipv6) { 447 struct iphdr *iph = (void *)(hdr + mac_hdr_len); 448 449 iph->check = 0; 450 iph->check = ip_fast_csum((unsigned char *)iph, iph->ihl); 451 } 452 453 /* Compute the checksum over the L4 header. */ 454 *l4_hdr_len = hdr_len - skb_transport_offset(skb); 455 return csum_partial(l4_hdr, *l4_hdr_len, 0); 456 } 457 458 static void enetc_tso_complete_csum(struct enetc_bdr *tx_ring, struct tso_t *tso, 459 struct sk_buff *skb, char *hdr, int len, 460 __wsum sum) 461 { 462 char *l4_hdr = hdr + skb_transport_offset(skb); 463 __sum16 csum_final; 464 465 /* Complete the L4 checksum by appending the pseudo-header to the 466 * already computed checksum. 467 */ 468 if (!tso->ipv6) 469 csum_final = csum_tcpudp_magic(ip_hdr(skb)->saddr, 470 ip_hdr(skb)->daddr, 471 len, ip_hdr(skb)->protocol, sum); 472 else 473 csum_final = csum_ipv6_magic(&ipv6_hdr(skb)->saddr, 474 &ipv6_hdr(skb)->daddr, 475 len, ipv6_hdr(skb)->nexthdr, sum); 476 477 if (tso->tlen != sizeof(struct udphdr)) { 478 struct tcphdr *tcph = (struct tcphdr *)(l4_hdr); 479 480 tcph->check = csum_final; 481 } else { 482 struct udphdr *udph = (struct udphdr *)(l4_hdr); 483 484 udph->check = csum_final; 485 } 486 } 487 488 static int enetc_map_tx_tso_buffs(struct enetc_bdr *tx_ring, struct sk_buff *skb) 489 { 490 int hdr_len, total_len, data_len; 491 struct enetc_tx_swbd *tx_swbd; 492 union enetc_tx_bd *txbd; 493 struct tso_t tso; 494 __wsum csum, csum2; 495 int count = 0, pos; 496 int err, i, bd_data_num; 497 498 /* Initialize the TSO handler, and prepare the first payload */ 499 hdr_len = tso_start(skb, &tso); 500 total_len = skb->len - hdr_len; 501 i = tx_ring->next_to_use; 502 503 while (total_len > 0) { 504 char *hdr; 505 506 /* Get the BD */ 507 txbd = ENETC_TXBD(*tx_ring, i); 508 tx_swbd = &tx_ring->tx_swbd[i]; 509 prefetchw(txbd); 510 511 /* Determine the length of this packet */ 512 data_len = min_t(int, skb_shinfo(skb)->gso_size, total_len); 513 total_len -= data_len; 514 515 /* prepare packet headers: MAC + IP + TCP */ 516 hdr = tx_ring->tso_headers + i * TSO_HEADER_SIZE; 517 tso_build_hdr(skb, hdr, &tso, data_len, total_len == 0); 518 519 /* compute the csum over the L4 header */ 520 csum = enetc_tso_hdr_csum(&tso, skb, hdr, hdr_len, &pos); 521 enetc_map_tx_tso_hdr(tx_ring, skb, tx_swbd, txbd, &i, hdr_len, data_len); 522 bd_data_num = 0; 523 count++; 524 525 while (data_len > 0) { 526 int size; 527 528 size = min_t(int, tso.size, data_len); 529 530 /* Advance the index in the BDR */ 531 enetc_bdr_idx_inc(tx_ring, &i); 532 txbd = ENETC_TXBD(*tx_ring, i); 533 tx_swbd = &tx_ring->tx_swbd[i]; 534 prefetchw(txbd); 535 536 /* Compute the checksum over this segment of data and 537 * add it to the csum already computed (over the L4 538 * header and possible other data segments). 539 */ 540 csum2 = csum_partial(tso.data, size, 0); 541 csum = csum_block_add(csum, csum2, pos); 542 pos += size; 543 544 err = enetc_map_tx_tso_data(tx_ring, skb, tx_swbd, txbd, 545 tso.data, size, 546 size == data_len); 547 if (err) 548 goto err_map_data; 549 550 data_len -= size; 551 count++; 552 bd_data_num++; 553 tso_build_data(skb, &tso, size); 554 555 if (unlikely(bd_data_num >= ENETC_MAX_SKB_FRAGS && data_len)) 556 goto err_chained_bd; 557 } 558 559 enetc_tso_complete_csum(tx_ring, &tso, skb, hdr, pos, csum); 560 561 if (total_len == 0) 562 tx_swbd->skb = skb; 563 564 /* Go to the next BD */ 565 enetc_bdr_idx_inc(tx_ring, &i); 566 } 567 568 tx_ring->next_to_use = i; 569 enetc_update_tx_ring_tail(tx_ring); 570 571 return count; 572 573 err_map_data: 574 dev_err(tx_ring->dev, "DMA map error"); 575 576 err_chained_bd: 577 do { 578 tx_swbd = &tx_ring->tx_swbd[i]; 579 enetc_free_tx_frame(tx_ring, tx_swbd); 580 if (i == 0) 581 i = tx_ring->bd_count; 582 i--; 583 } while (count--); 584 585 return 0; 586 } 587 588 static netdev_tx_t enetc_start_xmit(struct sk_buff *skb, 589 struct net_device *ndev) 590 { 591 struct enetc_ndev_priv *priv = netdev_priv(ndev); 592 struct enetc_bdr *tx_ring; 593 int count, err; 594 595 /* Queue one-step Sync packet if already locked */ 596 if (skb->cb[0] & ENETC_F_TX_ONESTEP_SYNC_TSTAMP) { 597 if (test_and_set_bit_lock(ENETC_TX_ONESTEP_TSTAMP_IN_PROGRESS, 598 &priv->flags)) { 599 skb_queue_tail(&priv->tx_skbs, skb); 600 return NETDEV_TX_OK; 601 } 602 } 603 604 tx_ring = priv->tx_ring[skb->queue_mapping]; 605 606 if (skb_is_gso(skb)) { 607 if (enetc_bd_unused(tx_ring) < tso_count_descs(skb)) { 608 netif_stop_subqueue(ndev, tx_ring->index); 609 return NETDEV_TX_BUSY; 610 } 611 612 enetc_lock_mdio(); 613 count = enetc_map_tx_tso_buffs(tx_ring, skb); 614 enetc_unlock_mdio(); 615 } else { 616 if (unlikely(skb_shinfo(skb)->nr_frags > ENETC_MAX_SKB_FRAGS)) 617 if (unlikely(skb_linearize(skb))) 618 goto drop_packet_err; 619 620 count = skb_shinfo(skb)->nr_frags + 1; /* fragments + head */ 621 if (enetc_bd_unused(tx_ring) < ENETC_TXBDS_NEEDED(count)) { 622 netif_stop_subqueue(ndev, tx_ring->index); 623 return NETDEV_TX_BUSY; 624 } 625 626 if (skb->ip_summed == CHECKSUM_PARTIAL) { 627 err = skb_checksum_help(skb); 628 if (err) 629 goto drop_packet_err; 630 } 631 enetc_lock_mdio(); 632 count = enetc_map_tx_buffs(tx_ring, skb); 633 enetc_unlock_mdio(); 634 } 635 636 if (unlikely(!count)) 637 goto drop_packet_err; 638 639 if (enetc_bd_unused(tx_ring) < ENETC_TXBDS_MAX_NEEDED) 640 netif_stop_subqueue(ndev, tx_ring->index); 641 642 return NETDEV_TX_OK; 643 644 drop_packet_err: 645 dev_kfree_skb_any(skb); 646 return NETDEV_TX_OK; 647 } 648 649 netdev_tx_t enetc_xmit(struct sk_buff *skb, struct net_device *ndev) 650 { 651 struct enetc_ndev_priv *priv = netdev_priv(ndev); 652 u8 udp, msgtype, twostep; 653 u16 offset1, offset2; 654 655 /* Mark tx timestamp type on skb->cb[0] if requires */ 656 if ((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && 657 (priv->active_offloads & ENETC_F_TX_TSTAMP_MASK)) { 658 skb->cb[0] = priv->active_offloads & ENETC_F_TX_TSTAMP_MASK; 659 } else { 660 skb->cb[0] = 0; 661 } 662 663 /* Fall back to two-step timestamp if not one-step Sync packet */ 664 if (skb->cb[0] & ENETC_F_TX_ONESTEP_SYNC_TSTAMP) { 665 if (enetc_ptp_parse(skb, &udp, &msgtype, &twostep, 666 &offset1, &offset2) || 667 msgtype != PTP_MSGTYPE_SYNC || twostep != 0) 668 skb->cb[0] = ENETC_F_TX_TSTAMP; 669 } 670 671 return enetc_start_xmit(skb, ndev); 672 } 673 EXPORT_SYMBOL_GPL(enetc_xmit); 674 675 static irqreturn_t enetc_msix(int irq, void *data) 676 { 677 struct enetc_int_vector *v = data; 678 int i; 679 680 enetc_lock_mdio(); 681 682 /* disable interrupts */ 683 enetc_wr_reg_hot(v->rbier, 0); 684 enetc_wr_reg_hot(v->ricr1, v->rx_ictt); 685 686 for_each_set_bit(i, &v->tx_rings_map, ENETC_MAX_NUM_TXQS) 687 enetc_wr_reg_hot(v->tbier_base + ENETC_BDR_OFF(i), 0); 688 689 enetc_unlock_mdio(); 690 691 napi_schedule(&v->napi); 692 693 return IRQ_HANDLED; 694 } 695 696 static void enetc_rx_dim_work(struct work_struct *w) 697 { 698 struct dim *dim = container_of(w, struct dim, work); 699 struct dim_cq_moder moder = 700 net_dim_get_rx_moderation(dim->mode, dim->profile_ix); 701 struct enetc_int_vector *v = 702 container_of(dim, struct enetc_int_vector, rx_dim); 703 704 v->rx_ictt = enetc_usecs_to_cycles(moder.usec); 705 dim->state = DIM_START_MEASURE; 706 } 707 708 static void enetc_rx_net_dim(struct enetc_int_vector *v) 709 { 710 struct dim_sample dim_sample = {}; 711 712 v->comp_cnt++; 713 714 if (!v->rx_napi_work) 715 return; 716 717 dim_update_sample(v->comp_cnt, 718 v->rx_ring.stats.packets, 719 v->rx_ring.stats.bytes, 720 &dim_sample); 721 net_dim(&v->rx_dim, dim_sample); 722 } 723 724 static int enetc_bd_ready_count(struct enetc_bdr *tx_ring, int ci) 725 { 726 int pi = enetc_rd_reg_hot(tx_ring->tcir) & ENETC_TBCIR_IDX_MASK; 727 728 return pi >= ci ? pi - ci : tx_ring->bd_count - ci + pi; 729 } 730 731 static bool enetc_page_reusable(struct page *page) 732 { 733 return (!page_is_pfmemalloc(page) && page_ref_count(page) == 1); 734 } 735 736 static void enetc_reuse_page(struct enetc_bdr *rx_ring, 737 struct enetc_rx_swbd *old) 738 { 739 struct enetc_rx_swbd *new; 740 741 new = &rx_ring->rx_swbd[rx_ring->next_to_alloc]; 742 743 /* next buf that may reuse a page */ 744 enetc_bdr_idx_inc(rx_ring, &rx_ring->next_to_alloc); 745 746 /* copy page reference */ 747 *new = *old; 748 } 749 750 static void enetc_get_tx_tstamp(struct enetc_hw *hw, union enetc_tx_bd *txbd, 751 u64 *tstamp) 752 { 753 u32 lo, hi, tstamp_lo; 754 755 lo = enetc_rd_hot(hw, ENETC_SICTR0); 756 hi = enetc_rd_hot(hw, ENETC_SICTR1); 757 tstamp_lo = le32_to_cpu(txbd->wb.tstamp); 758 if (lo <= tstamp_lo) 759 hi -= 1; 760 *tstamp = (u64)hi << 32 | tstamp_lo; 761 } 762 763 static void enetc_tstamp_tx(struct sk_buff *skb, u64 tstamp) 764 { 765 struct skb_shared_hwtstamps shhwtstamps; 766 767 if (skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS) { 768 memset(&shhwtstamps, 0, sizeof(shhwtstamps)); 769 shhwtstamps.hwtstamp = ns_to_ktime(tstamp); 770 skb_txtime_consumed(skb); 771 skb_tstamp_tx(skb, &shhwtstamps); 772 } 773 } 774 775 static void enetc_recycle_xdp_tx_buff(struct enetc_bdr *tx_ring, 776 struct enetc_tx_swbd *tx_swbd) 777 { 778 struct enetc_ndev_priv *priv = netdev_priv(tx_ring->ndev); 779 struct enetc_rx_swbd rx_swbd = { 780 .dma = tx_swbd->dma, 781 .page = tx_swbd->page, 782 .page_offset = tx_swbd->page_offset, 783 .dir = tx_swbd->dir, 784 .len = tx_swbd->len, 785 }; 786 struct enetc_bdr *rx_ring; 787 788 rx_ring = enetc_rx_ring_from_xdp_tx_ring(priv, tx_ring); 789 790 if (likely(enetc_swbd_unused(rx_ring))) { 791 enetc_reuse_page(rx_ring, &rx_swbd); 792 793 /* sync for use by the device */ 794 dma_sync_single_range_for_device(rx_ring->dev, rx_swbd.dma, 795 rx_swbd.page_offset, 796 ENETC_RXB_DMA_SIZE_XDP, 797 rx_swbd.dir); 798 799 rx_ring->stats.recycles++; 800 } else { 801 /* RX ring is already full, we need to unmap and free the 802 * page, since there's nothing useful we can do with it. 803 */ 804 rx_ring->stats.recycle_failures++; 805 806 dma_unmap_page(rx_ring->dev, rx_swbd.dma, PAGE_SIZE, 807 rx_swbd.dir); 808 __free_page(rx_swbd.page); 809 } 810 811 rx_ring->xdp.xdp_tx_in_flight--; 812 } 813 814 static bool enetc_clean_tx_ring(struct enetc_bdr *tx_ring, int napi_budget) 815 { 816 int tx_frm_cnt = 0, tx_byte_cnt = 0, tx_win_drop = 0; 817 struct net_device *ndev = tx_ring->ndev; 818 struct enetc_ndev_priv *priv = netdev_priv(ndev); 819 struct enetc_tx_swbd *tx_swbd; 820 int i, bds_to_clean; 821 bool do_twostep_tstamp; 822 u64 tstamp = 0; 823 824 i = tx_ring->next_to_clean; 825 tx_swbd = &tx_ring->tx_swbd[i]; 826 827 bds_to_clean = enetc_bd_ready_count(tx_ring, i); 828 829 do_twostep_tstamp = false; 830 831 while (bds_to_clean && tx_frm_cnt < ENETC_DEFAULT_TX_WORK) { 832 struct xdp_frame *xdp_frame = enetc_tx_swbd_get_xdp_frame(tx_swbd); 833 struct sk_buff *skb = enetc_tx_swbd_get_skb(tx_swbd); 834 bool is_eof = tx_swbd->is_eof; 835 836 if (unlikely(tx_swbd->check_wb)) { 837 union enetc_tx_bd *txbd = ENETC_TXBD(*tx_ring, i); 838 839 if (txbd->flags & ENETC_TXBD_FLAGS_W && 840 tx_swbd->do_twostep_tstamp) { 841 enetc_get_tx_tstamp(&priv->si->hw, txbd, 842 &tstamp); 843 do_twostep_tstamp = true; 844 } 845 846 if (tx_swbd->qbv_en && 847 txbd->wb.status & ENETC_TXBD_STATS_WIN) 848 tx_win_drop++; 849 } 850 851 if (tx_swbd->is_xdp_tx) 852 enetc_recycle_xdp_tx_buff(tx_ring, tx_swbd); 853 else if (likely(tx_swbd->dma)) 854 enetc_unmap_tx_buff(tx_ring, tx_swbd); 855 856 if (xdp_frame) { 857 xdp_return_frame(xdp_frame); 858 } else if (skb) { 859 if (unlikely(skb->cb[0] & ENETC_F_TX_ONESTEP_SYNC_TSTAMP)) { 860 /* Start work to release lock for next one-step 861 * timestamping packet. And send one skb in 862 * tx_skbs queue if has. 863 */ 864 schedule_work(&priv->tx_onestep_tstamp); 865 } else if (unlikely(do_twostep_tstamp)) { 866 enetc_tstamp_tx(skb, tstamp); 867 do_twostep_tstamp = false; 868 } 869 napi_consume_skb(skb, napi_budget); 870 } 871 872 tx_byte_cnt += tx_swbd->len; 873 /* Scrub the swbd here so we don't have to do that 874 * when we reuse it during xmit 875 */ 876 memset(tx_swbd, 0, sizeof(*tx_swbd)); 877 878 bds_to_clean--; 879 tx_swbd++; 880 i++; 881 if (unlikely(i == tx_ring->bd_count)) { 882 i = 0; 883 tx_swbd = tx_ring->tx_swbd; 884 } 885 886 /* BD iteration loop end */ 887 if (is_eof) { 888 tx_frm_cnt++; 889 /* re-arm interrupt source */ 890 enetc_wr_reg_hot(tx_ring->idr, BIT(tx_ring->index) | 891 BIT(16 + tx_ring->index)); 892 } 893 894 if (unlikely(!bds_to_clean)) 895 bds_to_clean = enetc_bd_ready_count(tx_ring, i); 896 } 897 898 tx_ring->next_to_clean = i; 899 tx_ring->stats.packets += tx_frm_cnt; 900 tx_ring->stats.bytes += tx_byte_cnt; 901 tx_ring->stats.win_drop += tx_win_drop; 902 903 if (unlikely(tx_frm_cnt && netif_carrier_ok(ndev) && 904 __netif_subqueue_stopped(ndev, tx_ring->index) && 905 !test_bit(ENETC_TX_DOWN, &priv->flags) && 906 (enetc_bd_unused(tx_ring) >= ENETC_TXBDS_MAX_NEEDED))) { 907 netif_wake_subqueue(ndev, tx_ring->index); 908 } 909 910 return tx_frm_cnt != ENETC_DEFAULT_TX_WORK; 911 } 912 913 static bool enetc_new_page(struct enetc_bdr *rx_ring, 914 struct enetc_rx_swbd *rx_swbd) 915 { 916 bool xdp = !!(rx_ring->xdp.prog); 917 struct page *page; 918 dma_addr_t addr; 919 920 page = dev_alloc_page(); 921 if (unlikely(!page)) 922 return false; 923 924 /* For XDP_TX, we forgo dma_unmap -> dma_map */ 925 rx_swbd->dir = xdp ? DMA_BIDIRECTIONAL : DMA_FROM_DEVICE; 926 927 addr = dma_map_page(rx_ring->dev, page, 0, PAGE_SIZE, rx_swbd->dir); 928 if (unlikely(dma_mapping_error(rx_ring->dev, addr))) { 929 __free_page(page); 930 931 return false; 932 } 933 934 rx_swbd->dma = addr; 935 rx_swbd->page = page; 936 rx_swbd->page_offset = rx_ring->buffer_offset; 937 938 return true; 939 } 940 941 static int enetc_refill_rx_ring(struct enetc_bdr *rx_ring, const int buff_cnt) 942 { 943 struct enetc_rx_swbd *rx_swbd; 944 union enetc_rx_bd *rxbd; 945 int i, j; 946 947 i = rx_ring->next_to_use; 948 rx_swbd = &rx_ring->rx_swbd[i]; 949 rxbd = enetc_rxbd(rx_ring, i); 950 951 for (j = 0; j < buff_cnt; j++) { 952 /* try reuse page */ 953 if (unlikely(!rx_swbd->page)) { 954 if (unlikely(!enetc_new_page(rx_ring, rx_swbd))) { 955 rx_ring->stats.rx_alloc_errs++; 956 break; 957 } 958 } 959 960 /* update RxBD */ 961 rxbd->w.addr = cpu_to_le64(rx_swbd->dma + 962 rx_swbd->page_offset); 963 /* clear 'R" as well */ 964 rxbd->r.lstatus = 0; 965 966 enetc_rxbd_next(rx_ring, &rxbd, &i); 967 rx_swbd = &rx_ring->rx_swbd[i]; 968 } 969 970 if (likely(j)) { 971 rx_ring->next_to_alloc = i; /* keep track from page reuse */ 972 rx_ring->next_to_use = i; 973 974 /* update ENETC's consumer index */ 975 enetc_wr_reg_hot(rx_ring->rcir, rx_ring->next_to_use); 976 } 977 978 return j; 979 } 980 981 static void enetc_get_rx_tstamp(struct net_device *ndev, 982 union enetc_rx_bd *rxbd, 983 struct sk_buff *skb) 984 { 985 struct skb_shared_hwtstamps *shhwtstamps = skb_hwtstamps(skb); 986 struct enetc_ndev_priv *priv = netdev_priv(ndev); 987 struct enetc_hw *hw = &priv->si->hw; 988 u32 lo, hi, tstamp_lo; 989 u64 tstamp; 990 991 if (le16_to_cpu(rxbd->r.flags) & ENETC_RXBD_FLAG_TSTMP) { 992 lo = enetc_rd_reg_hot(hw->reg + ENETC_SICTR0); 993 hi = enetc_rd_reg_hot(hw->reg + ENETC_SICTR1); 994 rxbd = enetc_rxbd_ext(rxbd); 995 tstamp_lo = le32_to_cpu(rxbd->ext.tstamp); 996 if (lo <= tstamp_lo) 997 hi -= 1; 998 999 tstamp = (u64)hi << 32 | tstamp_lo; 1000 memset(shhwtstamps, 0, sizeof(*shhwtstamps)); 1001 shhwtstamps->hwtstamp = ns_to_ktime(tstamp); 1002 } 1003 } 1004 1005 static void enetc_get_offloads(struct enetc_bdr *rx_ring, 1006 union enetc_rx_bd *rxbd, struct sk_buff *skb) 1007 { 1008 struct enetc_ndev_priv *priv = netdev_priv(rx_ring->ndev); 1009 1010 /* TODO: hashing */ 1011 if (rx_ring->ndev->features & NETIF_F_RXCSUM) { 1012 u16 inet_csum = le16_to_cpu(rxbd->r.inet_csum); 1013 1014 skb->csum = csum_unfold((__force __sum16)~htons(inet_csum)); 1015 skb->ip_summed = CHECKSUM_COMPLETE; 1016 } 1017 1018 if (le16_to_cpu(rxbd->r.flags) & ENETC_RXBD_FLAG_VLAN) { 1019 __be16 tpid = 0; 1020 1021 switch (le16_to_cpu(rxbd->r.flags) & ENETC_RXBD_FLAG_TPID) { 1022 case 0: 1023 tpid = htons(ETH_P_8021Q); 1024 break; 1025 case 1: 1026 tpid = htons(ETH_P_8021AD); 1027 break; 1028 case 2: 1029 tpid = htons(enetc_port_rd(&priv->si->hw, 1030 ENETC_PCVLANR1)); 1031 break; 1032 case 3: 1033 tpid = htons(enetc_port_rd(&priv->si->hw, 1034 ENETC_PCVLANR2)); 1035 break; 1036 default: 1037 break; 1038 } 1039 1040 __vlan_hwaccel_put_tag(skb, tpid, le16_to_cpu(rxbd->r.vlan_opt)); 1041 } 1042 1043 if (IS_ENABLED(CONFIG_FSL_ENETC_PTP_CLOCK) && 1044 (priv->active_offloads & ENETC_F_RX_TSTAMP)) 1045 enetc_get_rx_tstamp(rx_ring->ndev, rxbd, skb); 1046 } 1047 1048 /* This gets called during the non-XDP NAPI poll cycle as well as on XDP_PASS, 1049 * so it needs to work with both DMA_FROM_DEVICE as well as DMA_BIDIRECTIONAL 1050 * mapped buffers. 1051 */ 1052 static struct enetc_rx_swbd *enetc_get_rx_buff(struct enetc_bdr *rx_ring, 1053 int i, u16 size) 1054 { 1055 struct enetc_rx_swbd *rx_swbd = &rx_ring->rx_swbd[i]; 1056 1057 dma_sync_single_range_for_cpu(rx_ring->dev, rx_swbd->dma, 1058 rx_swbd->page_offset, 1059 size, rx_swbd->dir); 1060 return rx_swbd; 1061 } 1062 1063 /* Reuse the current page without performing half-page buffer flipping */ 1064 static void enetc_put_rx_buff(struct enetc_bdr *rx_ring, 1065 struct enetc_rx_swbd *rx_swbd) 1066 { 1067 size_t buffer_size = ENETC_RXB_TRUESIZE - rx_ring->buffer_offset; 1068 1069 enetc_reuse_page(rx_ring, rx_swbd); 1070 1071 dma_sync_single_range_for_device(rx_ring->dev, rx_swbd->dma, 1072 rx_swbd->page_offset, 1073 buffer_size, rx_swbd->dir); 1074 1075 rx_swbd->page = NULL; 1076 } 1077 1078 /* Reuse the current page by performing half-page buffer flipping */ 1079 static void enetc_flip_rx_buff(struct enetc_bdr *rx_ring, 1080 struct enetc_rx_swbd *rx_swbd) 1081 { 1082 if (likely(enetc_page_reusable(rx_swbd->page))) { 1083 rx_swbd->page_offset ^= ENETC_RXB_TRUESIZE; 1084 page_ref_inc(rx_swbd->page); 1085 1086 enetc_put_rx_buff(rx_ring, rx_swbd); 1087 } else { 1088 dma_unmap_page(rx_ring->dev, rx_swbd->dma, PAGE_SIZE, 1089 rx_swbd->dir); 1090 rx_swbd->page = NULL; 1091 } 1092 } 1093 1094 static struct sk_buff *enetc_map_rx_buff_to_skb(struct enetc_bdr *rx_ring, 1095 int i, u16 size) 1096 { 1097 struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size); 1098 struct sk_buff *skb; 1099 void *ba; 1100 1101 ba = page_address(rx_swbd->page) + rx_swbd->page_offset; 1102 skb = build_skb(ba - rx_ring->buffer_offset, ENETC_RXB_TRUESIZE); 1103 if (unlikely(!skb)) { 1104 rx_ring->stats.rx_alloc_errs++; 1105 return NULL; 1106 } 1107 1108 skb_reserve(skb, rx_ring->buffer_offset); 1109 __skb_put(skb, size); 1110 1111 enetc_flip_rx_buff(rx_ring, rx_swbd); 1112 1113 return skb; 1114 } 1115 1116 static void enetc_add_rx_buff_to_skb(struct enetc_bdr *rx_ring, int i, 1117 u16 size, struct sk_buff *skb) 1118 { 1119 struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size); 1120 1121 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_swbd->page, 1122 rx_swbd->page_offset, size, ENETC_RXB_TRUESIZE); 1123 1124 enetc_flip_rx_buff(rx_ring, rx_swbd); 1125 } 1126 1127 static bool enetc_check_bd_errors_and_consume(struct enetc_bdr *rx_ring, 1128 u32 bd_status, 1129 union enetc_rx_bd **rxbd, int *i) 1130 { 1131 if (likely(!(bd_status & ENETC_RXBD_LSTATUS(ENETC_RXBD_ERR_MASK)))) 1132 return false; 1133 1134 enetc_put_rx_buff(rx_ring, &rx_ring->rx_swbd[*i]); 1135 enetc_rxbd_next(rx_ring, rxbd, i); 1136 1137 while (!(bd_status & ENETC_RXBD_LSTATUS_F)) { 1138 dma_rmb(); 1139 bd_status = le32_to_cpu((*rxbd)->r.lstatus); 1140 1141 enetc_put_rx_buff(rx_ring, &rx_ring->rx_swbd[*i]); 1142 enetc_rxbd_next(rx_ring, rxbd, i); 1143 } 1144 1145 rx_ring->ndev->stats.rx_dropped++; 1146 rx_ring->ndev->stats.rx_errors++; 1147 1148 return true; 1149 } 1150 1151 static struct sk_buff *enetc_build_skb(struct enetc_bdr *rx_ring, 1152 u32 bd_status, union enetc_rx_bd **rxbd, 1153 int *i, int *cleaned_cnt, int buffer_size) 1154 { 1155 struct sk_buff *skb; 1156 u16 size; 1157 1158 size = le16_to_cpu((*rxbd)->r.buf_len); 1159 skb = enetc_map_rx_buff_to_skb(rx_ring, *i, size); 1160 if (!skb) 1161 return NULL; 1162 1163 enetc_get_offloads(rx_ring, *rxbd, skb); 1164 1165 (*cleaned_cnt)++; 1166 1167 enetc_rxbd_next(rx_ring, rxbd, i); 1168 1169 /* not last BD in frame? */ 1170 while (!(bd_status & ENETC_RXBD_LSTATUS_F)) { 1171 bd_status = le32_to_cpu((*rxbd)->r.lstatus); 1172 size = buffer_size; 1173 1174 if (bd_status & ENETC_RXBD_LSTATUS_F) { 1175 dma_rmb(); 1176 size = le16_to_cpu((*rxbd)->r.buf_len); 1177 } 1178 1179 enetc_add_rx_buff_to_skb(rx_ring, *i, size, skb); 1180 1181 (*cleaned_cnt)++; 1182 1183 enetc_rxbd_next(rx_ring, rxbd, i); 1184 } 1185 1186 skb_record_rx_queue(skb, rx_ring->index); 1187 skb->protocol = eth_type_trans(skb, rx_ring->ndev); 1188 1189 return skb; 1190 } 1191 1192 #define ENETC_RXBD_BUNDLE 16 /* # of BDs to update at once */ 1193 1194 static int enetc_clean_rx_ring(struct enetc_bdr *rx_ring, 1195 struct napi_struct *napi, int work_limit) 1196 { 1197 int rx_frm_cnt = 0, rx_byte_cnt = 0; 1198 int cleaned_cnt, i; 1199 1200 cleaned_cnt = enetc_bd_unused(rx_ring); 1201 /* next descriptor to process */ 1202 i = rx_ring->next_to_clean; 1203 1204 while (likely(rx_frm_cnt < work_limit)) { 1205 union enetc_rx_bd *rxbd; 1206 struct sk_buff *skb; 1207 u32 bd_status; 1208 1209 if (cleaned_cnt >= ENETC_RXBD_BUNDLE) 1210 cleaned_cnt -= enetc_refill_rx_ring(rx_ring, 1211 cleaned_cnt); 1212 1213 rxbd = enetc_rxbd(rx_ring, i); 1214 bd_status = le32_to_cpu(rxbd->r.lstatus); 1215 if (!bd_status) 1216 break; 1217 1218 enetc_wr_reg_hot(rx_ring->idr, BIT(rx_ring->index)); 1219 dma_rmb(); /* for reading other rxbd fields */ 1220 1221 if (enetc_check_bd_errors_and_consume(rx_ring, bd_status, 1222 &rxbd, &i)) 1223 break; 1224 1225 skb = enetc_build_skb(rx_ring, bd_status, &rxbd, &i, 1226 &cleaned_cnt, ENETC_RXB_DMA_SIZE); 1227 if (!skb) 1228 break; 1229 1230 /* When set, the outer VLAN header is extracted and reported 1231 * in the receive buffer descriptor. So rx_byte_cnt should 1232 * add the length of the extracted VLAN header. 1233 */ 1234 if (bd_status & ENETC_RXBD_FLAG_VLAN) 1235 rx_byte_cnt += VLAN_HLEN; 1236 rx_byte_cnt += skb->len + ETH_HLEN; 1237 rx_frm_cnt++; 1238 1239 napi_gro_receive(napi, skb); 1240 } 1241 1242 rx_ring->next_to_clean = i; 1243 1244 rx_ring->stats.packets += rx_frm_cnt; 1245 rx_ring->stats.bytes += rx_byte_cnt; 1246 1247 return rx_frm_cnt; 1248 } 1249 1250 static void enetc_xdp_map_tx_buff(struct enetc_bdr *tx_ring, int i, 1251 struct enetc_tx_swbd *tx_swbd, 1252 int frm_len) 1253 { 1254 union enetc_tx_bd *txbd = ENETC_TXBD(*tx_ring, i); 1255 1256 prefetchw(txbd); 1257 1258 enetc_clear_tx_bd(txbd); 1259 txbd->addr = cpu_to_le64(tx_swbd->dma + tx_swbd->page_offset); 1260 txbd->buf_len = cpu_to_le16(tx_swbd->len); 1261 txbd->frm_len = cpu_to_le16(frm_len); 1262 1263 memcpy(&tx_ring->tx_swbd[i], tx_swbd, sizeof(*tx_swbd)); 1264 } 1265 1266 /* Puts in the TX ring one XDP frame, mapped as an array of TX software buffer 1267 * descriptors. 1268 */ 1269 static bool enetc_xdp_tx(struct enetc_bdr *tx_ring, 1270 struct enetc_tx_swbd *xdp_tx_arr, int num_tx_swbd) 1271 { 1272 struct enetc_tx_swbd *tmp_tx_swbd = xdp_tx_arr; 1273 int i, k, frm_len = tmp_tx_swbd->len; 1274 1275 if (unlikely(enetc_bd_unused(tx_ring) < ENETC_TXBDS_NEEDED(num_tx_swbd))) 1276 return false; 1277 1278 while (unlikely(!tmp_tx_swbd->is_eof)) { 1279 tmp_tx_swbd++; 1280 frm_len += tmp_tx_swbd->len; 1281 } 1282 1283 i = tx_ring->next_to_use; 1284 1285 for (k = 0; k < num_tx_swbd; k++) { 1286 struct enetc_tx_swbd *xdp_tx_swbd = &xdp_tx_arr[k]; 1287 1288 enetc_xdp_map_tx_buff(tx_ring, i, xdp_tx_swbd, frm_len); 1289 1290 /* last BD needs 'F' bit set */ 1291 if (xdp_tx_swbd->is_eof) { 1292 union enetc_tx_bd *txbd = ENETC_TXBD(*tx_ring, i); 1293 1294 txbd->flags = ENETC_TXBD_FLAGS_F; 1295 } 1296 1297 enetc_bdr_idx_inc(tx_ring, &i); 1298 } 1299 1300 tx_ring->next_to_use = i; 1301 1302 return true; 1303 } 1304 1305 static int enetc_xdp_frame_to_xdp_tx_swbd(struct enetc_bdr *tx_ring, 1306 struct enetc_tx_swbd *xdp_tx_arr, 1307 struct xdp_frame *xdp_frame) 1308 { 1309 struct enetc_tx_swbd *xdp_tx_swbd = &xdp_tx_arr[0]; 1310 struct skb_shared_info *shinfo; 1311 void *data = xdp_frame->data; 1312 int len = xdp_frame->len; 1313 skb_frag_t *frag; 1314 dma_addr_t dma; 1315 unsigned int f; 1316 int n = 0; 1317 1318 dma = dma_map_single(tx_ring->dev, data, len, DMA_TO_DEVICE); 1319 if (unlikely(dma_mapping_error(tx_ring->dev, dma))) { 1320 netdev_err(tx_ring->ndev, "DMA map error\n"); 1321 return -1; 1322 } 1323 1324 xdp_tx_swbd->dma = dma; 1325 xdp_tx_swbd->dir = DMA_TO_DEVICE; 1326 xdp_tx_swbd->len = len; 1327 xdp_tx_swbd->is_xdp_redirect = true; 1328 xdp_tx_swbd->is_eof = false; 1329 xdp_tx_swbd->xdp_frame = NULL; 1330 1331 n++; 1332 1333 if (!xdp_frame_has_frags(xdp_frame)) 1334 goto out; 1335 1336 xdp_tx_swbd = &xdp_tx_arr[n]; 1337 1338 shinfo = xdp_get_shared_info_from_frame(xdp_frame); 1339 1340 for (f = 0, frag = &shinfo->frags[0]; f < shinfo->nr_frags; 1341 f++, frag++) { 1342 data = skb_frag_address(frag); 1343 len = skb_frag_size(frag); 1344 1345 dma = dma_map_single(tx_ring->dev, data, len, DMA_TO_DEVICE); 1346 if (unlikely(dma_mapping_error(tx_ring->dev, dma))) { 1347 /* Undo the DMA mapping for all fragments */ 1348 while (--n >= 0) 1349 enetc_unmap_tx_buff(tx_ring, &xdp_tx_arr[n]); 1350 1351 netdev_err(tx_ring->ndev, "DMA map error\n"); 1352 return -1; 1353 } 1354 1355 xdp_tx_swbd->dma = dma; 1356 xdp_tx_swbd->dir = DMA_TO_DEVICE; 1357 xdp_tx_swbd->len = len; 1358 xdp_tx_swbd->is_xdp_redirect = true; 1359 xdp_tx_swbd->is_eof = false; 1360 xdp_tx_swbd->xdp_frame = NULL; 1361 1362 n++; 1363 xdp_tx_swbd = &xdp_tx_arr[n]; 1364 } 1365 out: 1366 xdp_tx_arr[n - 1].is_eof = true; 1367 xdp_tx_arr[n - 1].xdp_frame = xdp_frame; 1368 1369 return n; 1370 } 1371 1372 int enetc_xdp_xmit(struct net_device *ndev, int num_frames, 1373 struct xdp_frame **frames, u32 flags) 1374 { 1375 struct enetc_tx_swbd xdp_redirect_arr[ENETC_MAX_SKB_FRAGS] = {0}; 1376 struct enetc_ndev_priv *priv = netdev_priv(ndev); 1377 struct enetc_bdr *tx_ring; 1378 int xdp_tx_bd_cnt, i, k; 1379 int xdp_tx_frm_cnt = 0; 1380 1381 if (unlikely(test_bit(ENETC_TX_DOWN, &priv->flags))) 1382 return -ENETDOWN; 1383 1384 enetc_lock_mdio(); 1385 1386 tx_ring = priv->xdp_tx_ring[smp_processor_id()]; 1387 1388 prefetchw(ENETC_TXBD(*tx_ring, tx_ring->next_to_use)); 1389 1390 for (k = 0; k < num_frames; k++) { 1391 xdp_tx_bd_cnt = enetc_xdp_frame_to_xdp_tx_swbd(tx_ring, 1392 xdp_redirect_arr, 1393 frames[k]); 1394 if (unlikely(xdp_tx_bd_cnt < 0)) 1395 break; 1396 1397 if (unlikely(!enetc_xdp_tx(tx_ring, xdp_redirect_arr, 1398 xdp_tx_bd_cnt))) { 1399 for (i = 0; i < xdp_tx_bd_cnt; i++) 1400 enetc_unmap_tx_buff(tx_ring, 1401 &xdp_redirect_arr[i]); 1402 tx_ring->stats.xdp_tx_drops++; 1403 break; 1404 } 1405 1406 xdp_tx_frm_cnt++; 1407 } 1408 1409 if (unlikely((flags & XDP_XMIT_FLUSH) || k != xdp_tx_frm_cnt)) 1410 enetc_update_tx_ring_tail(tx_ring); 1411 1412 tx_ring->stats.xdp_tx += xdp_tx_frm_cnt; 1413 1414 enetc_unlock_mdio(); 1415 1416 return xdp_tx_frm_cnt; 1417 } 1418 EXPORT_SYMBOL_GPL(enetc_xdp_xmit); 1419 1420 static void enetc_map_rx_buff_to_xdp(struct enetc_bdr *rx_ring, int i, 1421 struct xdp_buff *xdp_buff, u16 size) 1422 { 1423 struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size); 1424 void *hard_start = page_address(rx_swbd->page) + rx_swbd->page_offset; 1425 1426 /* To be used for XDP_TX */ 1427 rx_swbd->len = size; 1428 1429 xdp_prepare_buff(xdp_buff, hard_start - rx_ring->buffer_offset, 1430 rx_ring->buffer_offset, size, false); 1431 } 1432 1433 static void enetc_add_rx_buff_to_xdp(struct enetc_bdr *rx_ring, int i, 1434 u16 size, struct xdp_buff *xdp_buff) 1435 { 1436 struct skb_shared_info *shinfo = xdp_get_shared_info_from_buff(xdp_buff); 1437 struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size); 1438 skb_frag_t *frag; 1439 1440 /* To be used for XDP_TX */ 1441 rx_swbd->len = size; 1442 1443 if (!xdp_buff_has_frags(xdp_buff)) { 1444 xdp_buff_set_frags_flag(xdp_buff); 1445 shinfo->xdp_frags_size = size; 1446 shinfo->nr_frags = 0; 1447 } else { 1448 shinfo->xdp_frags_size += size; 1449 } 1450 1451 if (page_is_pfmemalloc(rx_swbd->page)) 1452 xdp_buff_set_frag_pfmemalloc(xdp_buff); 1453 1454 frag = &shinfo->frags[shinfo->nr_frags]; 1455 skb_frag_fill_page_desc(frag, rx_swbd->page, rx_swbd->page_offset, 1456 size); 1457 1458 shinfo->nr_frags++; 1459 } 1460 1461 static void enetc_build_xdp_buff(struct enetc_bdr *rx_ring, u32 bd_status, 1462 union enetc_rx_bd **rxbd, int *i, 1463 int *cleaned_cnt, struct xdp_buff *xdp_buff) 1464 { 1465 u16 size = le16_to_cpu((*rxbd)->r.buf_len); 1466 1467 xdp_init_buff(xdp_buff, ENETC_RXB_TRUESIZE, &rx_ring->xdp.rxq); 1468 1469 enetc_map_rx_buff_to_xdp(rx_ring, *i, xdp_buff, size); 1470 (*cleaned_cnt)++; 1471 enetc_rxbd_next(rx_ring, rxbd, i); 1472 1473 /* not last BD in frame? */ 1474 while (!(bd_status & ENETC_RXBD_LSTATUS_F)) { 1475 bd_status = le32_to_cpu((*rxbd)->r.lstatus); 1476 size = ENETC_RXB_DMA_SIZE_XDP; 1477 1478 if (bd_status & ENETC_RXBD_LSTATUS_F) { 1479 dma_rmb(); 1480 size = le16_to_cpu((*rxbd)->r.buf_len); 1481 } 1482 1483 enetc_add_rx_buff_to_xdp(rx_ring, *i, size, xdp_buff); 1484 (*cleaned_cnt)++; 1485 enetc_rxbd_next(rx_ring, rxbd, i); 1486 } 1487 } 1488 1489 /* Convert RX buffer descriptors to TX buffer descriptors. These will be 1490 * recycled back into the RX ring in enetc_clean_tx_ring. 1491 */ 1492 static int enetc_rx_swbd_to_xdp_tx_swbd(struct enetc_tx_swbd *xdp_tx_arr, 1493 struct enetc_bdr *rx_ring, 1494 int rx_ring_first, int rx_ring_last) 1495 { 1496 int n = 0; 1497 1498 for (; rx_ring_first != rx_ring_last; 1499 n++, enetc_bdr_idx_inc(rx_ring, &rx_ring_first)) { 1500 struct enetc_rx_swbd *rx_swbd = &rx_ring->rx_swbd[rx_ring_first]; 1501 struct enetc_tx_swbd *tx_swbd = &xdp_tx_arr[n]; 1502 1503 /* No need to dma_map, we already have DMA_BIDIRECTIONAL */ 1504 tx_swbd->dma = rx_swbd->dma; 1505 tx_swbd->dir = rx_swbd->dir; 1506 tx_swbd->page = rx_swbd->page; 1507 tx_swbd->page_offset = rx_swbd->page_offset; 1508 tx_swbd->len = rx_swbd->len; 1509 tx_swbd->is_dma_page = true; 1510 tx_swbd->is_xdp_tx = true; 1511 tx_swbd->is_eof = false; 1512 } 1513 1514 /* We rely on caller providing an rx_ring_last > rx_ring_first */ 1515 xdp_tx_arr[n - 1].is_eof = true; 1516 1517 return n; 1518 } 1519 1520 static void enetc_xdp_drop(struct enetc_bdr *rx_ring, int rx_ring_first, 1521 int rx_ring_last) 1522 { 1523 while (rx_ring_first != rx_ring_last) { 1524 enetc_put_rx_buff(rx_ring, 1525 &rx_ring->rx_swbd[rx_ring_first]); 1526 enetc_bdr_idx_inc(rx_ring, &rx_ring_first); 1527 } 1528 } 1529 1530 static int enetc_clean_rx_ring_xdp(struct enetc_bdr *rx_ring, 1531 struct napi_struct *napi, int work_limit, 1532 struct bpf_prog *prog) 1533 { 1534 int xdp_tx_bd_cnt, xdp_tx_frm_cnt = 0, xdp_redirect_frm_cnt = 0; 1535 struct enetc_tx_swbd xdp_tx_arr[ENETC_MAX_SKB_FRAGS] = {0}; 1536 struct enetc_ndev_priv *priv = netdev_priv(rx_ring->ndev); 1537 int rx_frm_cnt = 0, rx_byte_cnt = 0; 1538 struct enetc_bdr *tx_ring; 1539 int cleaned_cnt, i; 1540 u32 xdp_act; 1541 1542 cleaned_cnt = enetc_bd_unused(rx_ring); 1543 /* next descriptor to process */ 1544 i = rx_ring->next_to_clean; 1545 1546 while (likely(rx_frm_cnt < work_limit)) { 1547 union enetc_rx_bd *rxbd, *orig_rxbd; 1548 int orig_i, orig_cleaned_cnt; 1549 struct xdp_buff xdp_buff; 1550 struct sk_buff *skb; 1551 u32 bd_status; 1552 int err; 1553 1554 rxbd = enetc_rxbd(rx_ring, i); 1555 bd_status = le32_to_cpu(rxbd->r.lstatus); 1556 if (!bd_status) 1557 break; 1558 1559 enetc_wr_reg_hot(rx_ring->idr, BIT(rx_ring->index)); 1560 dma_rmb(); /* for reading other rxbd fields */ 1561 1562 if (enetc_check_bd_errors_and_consume(rx_ring, bd_status, 1563 &rxbd, &i)) 1564 break; 1565 1566 orig_rxbd = rxbd; 1567 orig_cleaned_cnt = cleaned_cnt; 1568 orig_i = i; 1569 1570 enetc_build_xdp_buff(rx_ring, bd_status, &rxbd, &i, 1571 &cleaned_cnt, &xdp_buff); 1572 1573 /* When set, the outer VLAN header is extracted and reported 1574 * in the receive buffer descriptor. So rx_byte_cnt should 1575 * add the length of the extracted VLAN header. 1576 */ 1577 if (bd_status & ENETC_RXBD_FLAG_VLAN) 1578 rx_byte_cnt += VLAN_HLEN; 1579 rx_byte_cnt += xdp_get_buff_len(&xdp_buff); 1580 1581 xdp_act = bpf_prog_run_xdp(prog, &xdp_buff); 1582 1583 switch (xdp_act) { 1584 default: 1585 bpf_warn_invalid_xdp_action(rx_ring->ndev, prog, xdp_act); 1586 fallthrough; 1587 case XDP_ABORTED: 1588 trace_xdp_exception(rx_ring->ndev, prog, xdp_act); 1589 fallthrough; 1590 case XDP_DROP: 1591 enetc_xdp_drop(rx_ring, orig_i, i); 1592 rx_ring->stats.xdp_drops++; 1593 break; 1594 case XDP_PASS: 1595 rxbd = orig_rxbd; 1596 cleaned_cnt = orig_cleaned_cnt; 1597 i = orig_i; 1598 1599 skb = enetc_build_skb(rx_ring, bd_status, &rxbd, 1600 &i, &cleaned_cnt, 1601 ENETC_RXB_DMA_SIZE_XDP); 1602 if (unlikely(!skb)) 1603 goto out; 1604 1605 napi_gro_receive(napi, skb); 1606 break; 1607 case XDP_TX: 1608 tx_ring = priv->xdp_tx_ring[rx_ring->index]; 1609 if (unlikely(test_bit(ENETC_TX_DOWN, &priv->flags))) { 1610 enetc_xdp_drop(rx_ring, orig_i, i); 1611 tx_ring->stats.xdp_tx_drops++; 1612 break; 1613 } 1614 1615 xdp_tx_bd_cnt = enetc_rx_swbd_to_xdp_tx_swbd(xdp_tx_arr, 1616 rx_ring, 1617 orig_i, i); 1618 1619 if (!enetc_xdp_tx(tx_ring, xdp_tx_arr, xdp_tx_bd_cnt)) { 1620 enetc_xdp_drop(rx_ring, orig_i, i); 1621 tx_ring->stats.xdp_tx_drops++; 1622 } else { 1623 tx_ring->stats.xdp_tx += xdp_tx_bd_cnt; 1624 rx_ring->xdp.xdp_tx_in_flight += xdp_tx_bd_cnt; 1625 xdp_tx_frm_cnt++; 1626 /* The XDP_TX enqueue was successful, so we 1627 * need to scrub the RX software BDs because 1628 * the ownership of the buffers no longer 1629 * belongs to the RX ring, and we must prevent 1630 * enetc_refill_rx_ring() from reusing 1631 * rx_swbd->page. 1632 */ 1633 while (orig_i != i) { 1634 rx_ring->rx_swbd[orig_i].page = NULL; 1635 enetc_bdr_idx_inc(rx_ring, &orig_i); 1636 } 1637 } 1638 break; 1639 case XDP_REDIRECT: 1640 err = xdp_do_redirect(rx_ring->ndev, &xdp_buff, prog); 1641 if (unlikely(err)) { 1642 enetc_xdp_drop(rx_ring, orig_i, i); 1643 rx_ring->stats.xdp_redirect_failures++; 1644 } else { 1645 while (orig_i != i) { 1646 enetc_flip_rx_buff(rx_ring, 1647 &rx_ring->rx_swbd[orig_i]); 1648 enetc_bdr_idx_inc(rx_ring, &orig_i); 1649 } 1650 xdp_redirect_frm_cnt++; 1651 rx_ring->stats.xdp_redirect++; 1652 } 1653 } 1654 1655 rx_frm_cnt++; 1656 } 1657 1658 out: 1659 rx_ring->next_to_clean = i; 1660 1661 rx_ring->stats.packets += rx_frm_cnt; 1662 rx_ring->stats.bytes += rx_byte_cnt; 1663 1664 if (xdp_redirect_frm_cnt) 1665 xdp_do_flush(); 1666 1667 if (xdp_tx_frm_cnt) 1668 enetc_update_tx_ring_tail(tx_ring); 1669 1670 if (cleaned_cnt > rx_ring->xdp.xdp_tx_in_flight) 1671 enetc_refill_rx_ring(rx_ring, enetc_bd_unused(rx_ring) - 1672 rx_ring->xdp.xdp_tx_in_flight); 1673 1674 return rx_frm_cnt; 1675 } 1676 1677 static int enetc_poll(struct napi_struct *napi, int budget) 1678 { 1679 struct enetc_int_vector 1680 *v = container_of(napi, struct enetc_int_vector, napi); 1681 struct enetc_bdr *rx_ring = &v->rx_ring; 1682 struct bpf_prog *prog; 1683 bool complete = true; 1684 int work_done; 1685 int i; 1686 1687 enetc_lock_mdio(); 1688 1689 for (i = 0; i < v->count_tx_rings; i++) 1690 if (!enetc_clean_tx_ring(&v->tx_ring[i], budget)) 1691 complete = false; 1692 1693 prog = rx_ring->xdp.prog; 1694 if (prog) 1695 work_done = enetc_clean_rx_ring_xdp(rx_ring, napi, budget, prog); 1696 else 1697 work_done = enetc_clean_rx_ring(rx_ring, napi, budget); 1698 if (work_done == budget) 1699 complete = false; 1700 if (work_done) 1701 v->rx_napi_work = true; 1702 1703 if (!complete) { 1704 enetc_unlock_mdio(); 1705 return budget; 1706 } 1707 1708 napi_complete_done(napi, work_done); 1709 1710 if (likely(v->rx_dim_en)) 1711 enetc_rx_net_dim(v); 1712 1713 v->rx_napi_work = false; 1714 1715 /* enable interrupts */ 1716 enetc_wr_reg_hot(v->rbier, ENETC_RBIER_RXTIE); 1717 1718 for_each_set_bit(i, &v->tx_rings_map, ENETC_MAX_NUM_TXQS) 1719 enetc_wr_reg_hot(v->tbier_base + ENETC_BDR_OFF(i), 1720 ENETC_TBIER_TXTIE); 1721 1722 enetc_unlock_mdio(); 1723 1724 return work_done; 1725 } 1726 1727 /* Probing and Init */ 1728 #define ENETC_MAX_RFS_SIZE 64 1729 void enetc_get_si_caps(struct enetc_si *si) 1730 { 1731 struct enetc_hw *hw = &si->hw; 1732 u32 val; 1733 1734 /* find out how many of various resources we have to work with */ 1735 val = enetc_rd(hw, ENETC_SICAPR0); 1736 si->num_rx_rings = (val >> 16) & 0xff; 1737 si->num_tx_rings = val & 0xff; 1738 1739 val = enetc_rd(hw, ENETC_SIRFSCAPR); 1740 si->num_fs_entries = ENETC_SIRFSCAPR_GET_NUM_RFS(val); 1741 si->num_fs_entries = min(si->num_fs_entries, ENETC_MAX_RFS_SIZE); 1742 1743 si->num_rss = 0; 1744 val = enetc_rd(hw, ENETC_SIPCAPR0); 1745 if (val & ENETC_SIPCAPR0_RSS) { 1746 u32 rss; 1747 1748 rss = enetc_rd(hw, ENETC_SIRSSCAPR); 1749 si->num_rss = ENETC_SIRSSCAPR_GET_NUM_RSS(rss); 1750 } 1751 1752 if (val & ENETC_SIPCAPR0_QBV) 1753 si->hw_features |= ENETC_SI_F_QBV; 1754 1755 if (val & ENETC_SIPCAPR0_QBU) 1756 si->hw_features |= ENETC_SI_F_QBU; 1757 1758 if (val & ENETC_SIPCAPR0_PSFP) 1759 si->hw_features |= ENETC_SI_F_PSFP; 1760 } 1761 EXPORT_SYMBOL_GPL(enetc_get_si_caps); 1762 1763 static int enetc_dma_alloc_bdr(struct enetc_bdr_resource *res) 1764 { 1765 size_t bd_base_size = res->bd_count * res->bd_size; 1766 1767 res->bd_base = dma_alloc_coherent(res->dev, bd_base_size, 1768 &res->bd_dma_base, GFP_KERNEL); 1769 if (!res->bd_base) 1770 return -ENOMEM; 1771 1772 /* h/w requires 128B alignment */ 1773 if (!IS_ALIGNED(res->bd_dma_base, 128)) { 1774 dma_free_coherent(res->dev, bd_base_size, res->bd_base, 1775 res->bd_dma_base); 1776 return -EINVAL; 1777 } 1778 1779 return 0; 1780 } 1781 1782 static void enetc_dma_free_bdr(const struct enetc_bdr_resource *res) 1783 { 1784 size_t bd_base_size = res->bd_count * res->bd_size; 1785 1786 dma_free_coherent(res->dev, bd_base_size, res->bd_base, 1787 res->bd_dma_base); 1788 } 1789 1790 static int enetc_alloc_tx_resource(struct enetc_bdr_resource *res, 1791 struct device *dev, size_t bd_count) 1792 { 1793 int err; 1794 1795 res->dev = dev; 1796 res->bd_count = bd_count; 1797 res->bd_size = sizeof(union enetc_tx_bd); 1798 1799 res->tx_swbd = vcalloc(bd_count, sizeof(*res->tx_swbd)); 1800 if (!res->tx_swbd) 1801 return -ENOMEM; 1802 1803 err = enetc_dma_alloc_bdr(res); 1804 if (err) 1805 goto err_alloc_bdr; 1806 1807 res->tso_headers = dma_alloc_coherent(dev, bd_count * TSO_HEADER_SIZE, 1808 &res->tso_headers_dma, 1809 GFP_KERNEL); 1810 if (!res->tso_headers) { 1811 err = -ENOMEM; 1812 goto err_alloc_tso; 1813 } 1814 1815 return 0; 1816 1817 err_alloc_tso: 1818 enetc_dma_free_bdr(res); 1819 err_alloc_bdr: 1820 vfree(res->tx_swbd); 1821 res->tx_swbd = NULL; 1822 1823 return err; 1824 } 1825 1826 static void enetc_free_tx_resource(const struct enetc_bdr_resource *res) 1827 { 1828 dma_free_coherent(res->dev, res->bd_count * TSO_HEADER_SIZE, 1829 res->tso_headers, res->tso_headers_dma); 1830 enetc_dma_free_bdr(res); 1831 vfree(res->tx_swbd); 1832 } 1833 1834 static struct enetc_bdr_resource * 1835 enetc_alloc_tx_resources(struct enetc_ndev_priv *priv) 1836 { 1837 struct enetc_bdr_resource *tx_res; 1838 int i, err; 1839 1840 tx_res = kcalloc(priv->num_tx_rings, sizeof(*tx_res), GFP_KERNEL); 1841 if (!tx_res) 1842 return ERR_PTR(-ENOMEM); 1843 1844 for (i = 0; i < priv->num_tx_rings; i++) { 1845 struct enetc_bdr *tx_ring = priv->tx_ring[i]; 1846 1847 err = enetc_alloc_tx_resource(&tx_res[i], tx_ring->dev, 1848 tx_ring->bd_count); 1849 if (err) 1850 goto fail; 1851 } 1852 1853 return tx_res; 1854 1855 fail: 1856 while (i-- > 0) 1857 enetc_free_tx_resource(&tx_res[i]); 1858 1859 kfree(tx_res); 1860 1861 return ERR_PTR(err); 1862 } 1863 1864 static void enetc_free_tx_resources(const struct enetc_bdr_resource *tx_res, 1865 size_t num_resources) 1866 { 1867 size_t i; 1868 1869 for (i = 0; i < num_resources; i++) 1870 enetc_free_tx_resource(&tx_res[i]); 1871 1872 kfree(tx_res); 1873 } 1874 1875 static int enetc_alloc_rx_resource(struct enetc_bdr_resource *res, 1876 struct device *dev, size_t bd_count, 1877 bool extended) 1878 { 1879 int err; 1880 1881 res->dev = dev; 1882 res->bd_count = bd_count; 1883 res->bd_size = sizeof(union enetc_rx_bd); 1884 if (extended) 1885 res->bd_size *= 2; 1886 1887 res->rx_swbd = vcalloc(bd_count, sizeof(struct enetc_rx_swbd)); 1888 if (!res->rx_swbd) 1889 return -ENOMEM; 1890 1891 err = enetc_dma_alloc_bdr(res); 1892 if (err) { 1893 vfree(res->rx_swbd); 1894 return err; 1895 } 1896 1897 return 0; 1898 } 1899 1900 static void enetc_free_rx_resource(const struct enetc_bdr_resource *res) 1901 { 1902 enetc_dma_free_bdr(res); 1903 vfree(res->rx_swbd); 1904 } 1905 1906 static struct enetc_bdr_resource * 1907 enetc_alloc_rx_resources(struct enetc_ndev_priv *priv, bool extended) 1908 { 1909 struct enetc_bdr_resource *rx_res; 1910 int i, err; 1911 1912 rx_res = kcalloc(priv->num_rx_rings, sizeof(*rx_res), GFP_KERNEL); 1913 if (!rx_res) 1914 return ERR_PTR(-ENOMEM); 1915 1916 for (i = 0; i < priv->num_rx_rings; i++) { 1917 struct enetc_bdr *rx_ring = priv->rx_ring[i]; 1918 1919 err = enetc_alloc_rx_resource(&rx_res[i], rx_ring->dev, 1920 rx_ring->bd_count, extended); 1921 if (err) 1922 goto fail; 1923 } 1924 1925 return rx_res; 1926 1927 fail: 1928 while (i-- > 0) 1929 enetc_free_rx_resource(&rx_res[i]); 1930 1931 kfree(rx_res); 1932 1933 return ERR_PTR(err); 1934 } 1935 1936 static void enetc_free_rx_resources(const struct enetc_bdr_resource *rx_res, 1937 size_t num_resources) 1938 { 1939 size_t i; 1940 1941 for (i = 0; i < num_resources; i++) 1942 enetc_free_rx_resource(&rx_res[i]); 1943 1944 kfree(rx_res); 1945 } 1946 1947 static void enetc_assign_tx_resource(struct enetc_bdr *tx_ring, 1948 const struct enetc_bdr_resource *res) 1949 { 1950 tx_ring->bd_base = res ? res->bd_base : NULL; 1951 tx_ring->bd_dma_base = res ? res->bd_dma_base : 0; 1952 tx_ring->tx_swbd = res ? res->tx_swbd : NULL; 1953 tx_ring->tso_headers = res ? res->tso_headers : NULL; 1954 tx_ring->tso_headers_dma = res ? res->tso_headers_dma : 0; 1955 } 1956 1957 static void enetc_assign_rx_resource(struct enetc_bdr *rx_ring, 1958 const struct enetc_bdr_resource *res) 1959 { 1960 rx_ring->bd_base = res ? res->bd_base : NULL; 1961 rx_ring->bd_dma_base = res ? res->bd_dma_base : 0; 1962 rx_ring->rx_swbd = res ? res->rx_swbd : NULL; 1963 } 1964 1965 static void enetc_assign_tx_resources(struct enetc_ndev_priv *priv, 1966 const struct enetc_bdr_resource *res) 1967 { 1968 int i; 1969 1970 if (priv->tx_res) 1971 enetc_free_tx_resources(priv->tx_res, priv->num_tx_rings); 1972 1973 for (i = 0; i < priv->num_tx_rings; i++) { 1974 enetc_assign_tx_resource(priv->tx_ring[i], 1975 res ? &res[i] : NULL); 1976 } 1977 1978 priv->tx_res = res; 1979 } 1980 1981 static void enetc_assign_rx_resources(struct enetc_ndev_priv *priv, 1982 const struct enetc_bdr_resource *res) 1983 { 1984 int i; 1985 1986 if (priv->rx_res) 1987 enetc_free_rx_resources(priv->rx_res, priv->num_rx_rings); 1988 1989 for (i = 0; i < priv->num_rx_rings; i++) { 1990 enetc_assign_rx_resource(priv->rx_ring[i], 1991 res ? &res[i] : NULL); 1992 } 1993 1994 priv->rx_res = res; 1995 } 1996 1997 static void enetc_free_tx_ring(struct enetc_bdr *tx_ring) 1998 { 1999 int i; 2000 2001 for (i = 0; i < tx_ring->bd_count; i++) { 2002 struct enetc_tx_swbd *tx_swbd = &tx_ring->tx_swbd[i]; 2003 2004 enetc_free_tx_frame(tx_ring, tx_swbd); 2005 } 2006 } 2007 2008 static void enetc_free_rx_ring(struct enetc_bdr *rx_ring) 2009 { 2010 int i; 2011 2012 for (i = 0; i < rx_ring->bd_count; i++) { 2013 struct enetc_rx_swbd *rx_swbd = &rx_ring->rx_swbd[i]; 2014 2015 if (!rx_swbd->page) 2016 continue; 2017 2018 dma_unmap_page(rx_ring->dev, rx_swbd->dma, PAGE_SIZE, 2019 rx_swbd->dir); 2020 __free_page(rx_swbd->page); 2021 rx_swbd->page = NULL; 2022 } 2023 } 2024 2025 static void enetc_free_rxtx_rings(struct enetc_ndev_priv *priv) 2026 { 2027 int i; 2028 2029 for (i = 0; i < priv->num_rx_rings; i++) 2030 enetc_free_rx_ring(priv->rx_ring[i]); 2031 2032 for (i = 0; i < priv->num_tx_rings; i++) 2033 enetc_free_tx_ring(priv->tx_ring[i]); 2034 } 2035 2036 static int enetc_setup_default_rss_table(struct enetc_si *si, int num_groups) 2037 { 2038 int *rss_table; 2039 int i; 2040 2041 rss_table = kmalloc_array(si->num_rss, sizeof(*rss_table), GFP_KERNEL); 2042 if (!rss_table) 2043 return -ENOMEM; 2044 2045 /* Set up RSS table defaults */ 2046 for (i = 0; i < si->num_rss; i++) 2047 rss_table[i] = i % num_groups; 2048 2049 enetc_set_rss_table(si, rss_table, si->num_rss); 2050 2051 kfree(rss_table); 2052 2053 return 0; 2054 } 2055 2056 int enetc_configure_si(struct enetc_ndev_priv *priv) 2057 { 2058 struct enetc_si *si = priv->si; 2059 struct enetc_hw *hw = &si->hw; 2060 int err; 2061 2062 /* set SI cache attributes */ 2063 enetc_wr(hw, ENETC_SICAR0, 2064 ENETC_SICAR_RD_COHERENT | ENETC_SICAR_WR_COHERENT); 2065 enetc_wr(hw, ENETC_SICAR1, ENETC_SICAR_MSI); 2066 /* enable SI */ 2067 enetc_wr(hw, ENETC_SIMR, ENETC_SIMR_EN); 2068 2069 if (si->num_rss) { 2070 err = enetc_setup_default_rss_table(si, priv->num_rx_rings); 2071 if (err) 2072 return err; 2073 } 2074 2075 return 0; 2076 } 2077 EXPORT_SYMBOL_GPL(enetc_configure_si); 2078 2079 void enetc_init_si_rings_params(struct enetc_ndev_priv *priv) 2080 { 2081 struct enetc_si *si = priv->si; 2082 int cpus = num_online_cpus(); 2083 2084 priv->tx_bd_count = ENETC_TX_RING_DEFAULT_SIZE; 2085 priv->rx_bd_count = ENETC_RX_RING_DEFAULT_SIZE; 2086 2087 /* Enable all available TX rings in order to configure as many 2088 * priorities as possible, when needed. 2089 * TODO: Make # of TX rings run-time configurable 2090 */ 2091 priv->num_rx_rings = min_t(int, cpus, si->num_rx_rings); 2092 priv->num_tx_rings = si->num_tx_rings; 2093 priv->bdr_int_num = cpus; 2094 priv->ic_mode = ENETC_IC_RX_ADAPTIVE | ENETC_IC_TX_MANUAL; 2095 priv->tx_ictt = ENETC_TXIC_TIMETHR; 2096 } 2097 EXPORT_SYMBOL_GPL(enetc_init_si_rings_params); 2098 2099 int enetc_alloc_si_resources(struct enetc_ndev_priv *priv) 2100 { 2101 struct enetc_si *si = priv->si; 2102 2103 priv->cls_rules = kcalloc(si->num_fs_entries, sizeof(*priv->cls_rules), 2104 GFP_KERNEL); 2105 if (!priv->cls_rules) 2106 return -ENOMEM; 2107 2108 return 0; 2109 } 2110 EXPORT_SYMBOL_GPL(enetc_alloc_si_resources); 2111 2112 void enetc_free_si_resources(struct enetc_ndev_priv *priv) 2113 { 2114 kfree(priv->cls_rules); 2115 } 2116 EXPORT_SYMBOL_GPL(enetc_free_si_resources); 2117 2118 static void enetc_setup_txbdr(struct enetc_hw *hw, struct enetc_bdr *tx_ring) 2119 { 2120 int idx = tx_ring->index; 2121 u32 tbmr; 2122 2123 enetc_txbdr_wr(hw, idx, ENETC_TBBAR0, 2124 lower_32_bits(tx_ring->bd_dma_base)); 2125 2126 enetc_txbdr_wr(hw, idx, ENETC_TBBAR1, 2127 upper_32_bits(tx_ring->bd_dma_base)); 2128 2129 WARN_ON(!IS_ALIGNED(tx_ring->bd_count, 64)); /* multiple of 64 */ 2130 enetc_txbdr_wr(hw, idx, ENETC_TBLENR, 2131 ENETC_RTBLENR_LEN(tx_ring->bd_count)); 2132 2133 /* clearing PI/CI registers for Tx not supported, adjust sw indexes */ 2134 tx_ring->next_to_use = enetc_txbdr_rd(hw, idx, ENETC_TBPIR); 2135 tx_ring->next_to_clean = enetc_txbdr_rd(hw, idx, ENETC_TBCIR); 2136 2137 /* enable Tx ints by setting pkt thr to 1 */ 2138 enetc_txbdr_wr(hw, idx, ENETC_TBICR0, ENETC_TBICR0_ICEN | 0x1); 2139 2140 tbmr = ENETC_TBMR_SET_PRIO(tx_ring->prio); 2141 if (tx_ring->ndev->features & NETIF_F_HW_VLAN_CTAG_TX) 2142 tbmr |= ENETC_TBMR_VIH; 2143 2144 /* enable ring */ 2145 enetc_txbdr_wr(hw, idx, ENETC_TBMR, tbmr); 2146 2147 tx_ring->tpir = hw->reg + ENETC_BDR(TX, idx, ENETC_TBPIR); 2148 tx_ring->tcir = hw->reg + ENETC_BDR(TX, idx, ENETC_TBCIR); 2149 tx_ring->idr = hw->reg + ENETC_SITXIDR; 2150 } 2151 2152 static void enetc_setup_rxbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring, 2153 bool extended) 2154 { 2155 int idx = rx_ring->index; 2156 u32 rbmr = 0; 2157 2158 enetc_rxbdr_wr(hw, idx, ENETC_RBBAR0, 2159 lower_32_bits(rx_ring->bd_dma_base)); 2160 2161 enetc_rxbdr_wr(hw, idx, ENETC_RBBAR1, 2162 upper_32_bits(rx_ring->bd_dma_base)); 2163 2164 WARN_ON(!IS_ALIGNED(rx_ring->bd_count, 64)); /* multiple of 64 */ 2165 enetc_rxbdr_wr(hw, idx, ENETC_RBLENR, 2166 ENETC_RTBLENR_LEN(rx_ring->bd_count)); 2167 2168 if (rx_ring->xdp.prog) 2169 enetc_rxbdr_wr(hw, idx, ENETC_RBBSR, ENETC_RXB_DMA_SIZE_XDP); 2170 else 2171 enetc_rxbdr_wr(hw, idx, ENETC_RBBSR, ENETC_RXB_DMA_SIZE); 2172 2173 /* Also prepare the consumer index in case page allocation never 2174 * succeeds. In that case, hardware will never advance producer index 2175 * to match consumer index, and will drop all frames. 2176 */ 2177 enetc_rxbdr_wr(hw, idx, ENETC_RBPIR, 0); 2178 enetc_rxbdr_wr(hw, idx, ENETC_RBCIR, 1); 2179 2180 /* enable Rx ints by setting pkt thr to 1 */ 2181 enetc_rxbdr_wr(hw, idx, ENETC_RBICR0, ENETC_RBICR0_ICEN | 0x1); 2182 2183 rx_ring->ext_en = extended; 2184 if (rx_ring->ext_en) 2185 rbmr |= ENETC_RBMR_BDS; 2186 2187 if (rx_ring->ndev->features & NETIF_F_HW_VLAN_CTAG_RX) 2188 rbmr |= ENETC_RBMR_VTE; 2189 2190 rx_ring->rcir = hw->reg + ENETC_BDR(RX, idx, ENETC_RBCIR); 2191 rx_ring->idr = hw->reg + ENETC_SIRXIDR; 2192 2193 rx_ring->next_to_clean = 0; 2194 rx_ring->next_to_use = 0; 2195 rx_ring->next_to_alloc = 0; 2196 2197 enetc_lock_mdio(); 2198 enetc_refill_rx_ring(rx_ring, enetc_bd_unused(rx_ring)); 2199 enetc_unlock_mdio(); 2200 2201 enetc_rxbdr_wr(hw, idx, ENETC_RBMR, rbmr); 2202 } 2203 2204 static void enetc_setup_bdrs(struct enetc_ndev_priv *priv, bool extended) 2205 { 2206 struct enetc_hw *hw = &priv->si->hw; 2207 int i; 2208 2209 for (i = 0; i < priv->num_tx_rings; i++) 2210 enetc_setup_txbdr(hw, priv->tx_ring[i]); 2211 2212 for (i = 0; i < priv->num_rx_rings; i++) 2213 enetc_setup_rxbdr(hw, priv->rx_ring[i], extended); 2214 } 2215 2216 static void enetc_enable_txbdr(struct enetc_hw *hw, struct enetc_bdr *tx_ring) 2217 { 2218 int idx = tx_ring->index; 2219 u32 tbmr; 2220 2221 tbmr = enetc_txbdr_rd(hw, idx, ENETC_TBMR); 2222 tbmr |= ENETC_TBMR_EN; 2223 enetc_txbdr_wr(hw, idx, ENETC_TBMR, tbmr); 2224 } 2225 2226 static void enetc_enable_rxbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring) 2227 { 2228 int idx = rx_ring->index; 2229 u32 rbmr; 2230 2231 rbmr = enetc_rxbdr_rd(hw, idx, ENETC_RBMR); 2232 rbmr |= ENETC_RBMR_EN; 2233 enetc_rxbdr_wr(hw, idx, ENETC_RBMR, rbmr); 2234 } 2235 2236 static void enetc_enable_rx_bdrs(struct enetc_ndev_priv *priv) 2237 { 2238 struct enetc_hw *hw = &priv->si->hw; 2239 int i; 2240 2241 for (i = 0; i < priv->num_rx_rings; i++) 2242 enetc_enable_rxbdr(hw, priv->rx_ring[i]); 2243 } 2244 2245 static void enetc_enable_tx_bdrs(struct enetc_ndev_priv *priv) 2246 { 2247 struct enetc_hw *hw = &priv->si->hw; 2248 int i; 2249 2250 for (i = 0; i < priv->num_tx_rings; i++) 2251 enetc_enable_txbdr(hw, priv->tx_ring[i]); 2252 } 2253 2254 static void enetc_disable_rxbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring) 2255 { 2256 int idx = rx_ring->index; 2257 2258 /* disable EN bit on ring */ 2259 enetc_rxbdr_wr(hw, idx, ENETC_RBMR, 0); 2260 } 2261 2262 static void enetc_disable_txbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring) 2263 { 2264 int idx = rx_ring->index; 2265 2266 /* disable EN bit on ring */ 2267 enetc_txbdr_wr(hw, idx, ENETC_TBMR, 0); 2268 } 2269 2270 static void enetc_disable_rx_bdrs(struct enetc_ndev_priv *priv) 2271 { 2272 struct enetc_hw *hw = &priv->si->hw; 2273 int i; 2274 2275 for (i = 0; i < priv->num_rx_rings; i++) 2276 enetc_disable_rxbdr(hw, priv->rx_ring[i]); 2277 } 2278 2279 static void enetc_disable_tx_bdrs(struct enetc_ndev_priv *priv) 2280 { 2281 struct enetc_hw *hw = &priv->si->hw; 2282 int i; 2283 2284 for (i = 0; i < priv->num_tx_rings; i++) 2285 enetc_disable_txbdr(hw, priv->tx_ring[i]); 2286 } 2287 2288 static void enetc_wait_txbdr(struct enetc_hw *hw, struct enetc_bdr *tx_ring) 2289 { 2290 int delay = 8, timeout = 100; 2291 int idx = tx_ring->index; 2292 2293 /* wait for busy to clear */ 2294 while (delay < timeout && 2295 enetc_txbdr_rd(hw, idx, ENETC_TBSR) & ENETC_TBSR_BUSY) { 2296 msleep(delay); 2297 delay *= 2; 2298 } 2299 2300 if (delay >= timeout) 2301 netdev_warn(tx_ring->ndev, "timeout for tx ring #%d clear\n", 2302 idx); 2303 } 2304 2305 static void enetc_wait_bdrs(struct enetc_ndev_priv *priv) 2306 { 2307 struct enetc_hw *hw = &priv->si->hw; 2308 int i; 2309 2310 for (i = 0; i < priv->num_tx_rings; i++) 2311 enetc_wait_txbdr(hw, priv->tx_ring[i]); 2312 } 2313 2314 static int enetc_setup_irqs(struct enetc_ndev_priv *priv) 2315 { 2316 struct pci_dev *pdev = priv->si->pdev; 2317 struct enetc_hw *hw = &priv->si->hw; 2318 int i, j, err; 2319 2320 for (i = 0; i < priv->bdr_int_num; i++) { 2321 int irq = pci_irq_vector(pdev, ENETC_BDR_INT_BASE_IDX + i); 2322 struct enetc_int_vector *v = priv->int_vector[i]; 2323 int entry = ENETC_BDR_INT_BASE_IDX + i; 2324 2325 snprintf(v->name, sizeof(v->name), "%s-rxtx%d", 2326 priv->ndev->name, i); 2327 err = request_irq(irq, enetc_msix, IRQF_NO_AUTOEN, v->name, v); 2328 if (err) { 2329 dev_err(priv->dev, "request_irq() failed!\n"); 2330 goto irq_err; 2331 } 2332 2333 v->tbier_base = hw->reg + ENETC_BDR(TX, 0, ENETC_TBIER); 2334 v->rbier = hw->reg + ENETC_BDR(RX, i, ENETC_RBIER); 2335 v->ricr1 = hw->reg + ENETC_BDR(RX, i, ENETC_RBICR1); 2336 2337 enetc_wr(hw, ENETC_SIMSIRRV(i), entry); 2338 2339 for (j = 0; j < v->count_tx_rings; j++) { 2340 int idx = v->tx_ring[j].index; 2341 2342 enetc_wr(hw, ENETC_SIMSITRV(idx), entry); 2343 } 2344 irq_set_affinity_hint(irq, get_cpu_mask(i % num_online_cpus())); 2345 } 2346 2347 return 0; 2348 2349 irq_err: 2350 while (i--) { 2351 int irq = pci_irq_vector(pdev, ENETC_BDR_INT_BASE_IDX + i); 2352 2353 irq_set_affinity_hint(irq, NULL); 2354 free_irq(irq, priv->int_vector[i]); 2355 } 2356 2357 return err; 2358 } 2359 2360 static void enetc_free_irqs(struct enetc_ndev_priv *priv) 2361 { 2362 struct pci_dev *pdev = priv->si->pdev; 2363 int i; 2364 2365 for (i = 0; i < priv->bdr_int_num; i++) { 2366 int irq = pci_irq_vector(pdev, ENETC_BDR_INT_BASE_IDX + i); 2367 2368 irq_set_affinity_hint(irq, NULL); 2369 free_irq(irq, priv->int_vector[i]); 2370 } 2371 } 2372 2373 static void enetc_setup_interrupts(struct enetc_ndev_priv *priv) 2374 { 2375 struct enetc_hw *hw = &priv->si->hw; 2376 u32 icpt, ictt; 2377 int i; 2378 2379 /* enable Tx & Rx event indication */ 2380 if (priv->ic_mode & 2381 (ENETC_IC_RX_MANUAL | ENETC_IC_RX_ADAPTIVE)) { 2382 icpt = ENETC_RBICR0_SET_ICPT(ENETC_RXIC_PKTTHR); 2383 /* init to non-0 minimum, will be adjusted later */ 2384 ictt = 0x1; 2385 } else { 2386 icpt = 0x1; /* enable Rx ints by setting pkt thr to 1 */ 2387 ictt = 0; 2388 } 2389 2390 for (i = 0; i < priv->num_rx_rings; i++) { 2391 enetc_rxbdr_wr(hw, i, ENETC_RBICR1, ictt); 2392 enetc_rxbdr_wr(hw, i, ENETC_RBICR0, ENETC_RBICR0_ICEN | icpt); 2393 enetc_rxbdr_wr(hw, i, ENETC_RBIER, ENETC_RBIER_RXTIE); 2394 } 2395 2396 if (priv->ic_mode & ENETC_IC_TX_MANUAL) 2397 icpt = ENETC_TBICR0_SET_ICPT(ENETC_TXIC_PKTTHR); 2398 else 2399 icpt = 0x1; /* enable Tx ints by setting pkt thr to 1 */ 2400 2401 for (i = 0; i < priv->num_tx_rings; i++) { 2402 enetc_txbdr_wr(hw, i, ENETC_TBICR1, priv->tx_ictt); 2403 enetc_txbdr_wr(hw, i, ENETC_TBICR0, ENETC_TBICR0_ICEN | icpt); 2404 enetc_txbdr_wr(hw, i, ENETC_TBIER, ENETC_TBIER_TXTIE); 2405 } 2406 } 2407 2408 static void enetc_clear_interrupts(struct enetc_ndev_priv *priv) 2409 { 2410 struct enetc_hw *hw = &priv->si->hw; 2411 int i; 2412 2413 for (i = 0; i < priv->num_tx_rings; i++) 2414 enetc_txbdr_wr(hw, i, ENETC_TBIER, 0); 2415 2416 for (i = 0; i < priv->num_rx_rings; i++) 2417 enetc_rxbdr_wr(hw, i, ENETC_RBIER, 0); 2418 } 2419 2420 static int enetc_phylink_connect(struct net_device *ndev) 2421 { 2422 struct enetc_ndev_priv *priv = netdev_priv(ndev); 2423 struct ethtool_keee edata; 2424 int err; 2425 2426 if (!priv->phylink) { 2427 /* phy-less mode */ 2428 netif_carrier_on(ndev); 2429 return 0; 2430 } 2431 2432 err = phylink_of_phy_connect(priv->phylink, priv->dev->of_node, 0); 2433 if (err) { 2434 dev_err(&ndev->dev, "could not attach to PHY\n"); 2435 return err; 2436 } 2437 2438 /* disable EEE autoneg, until ENETC driver supports it */ 2439 memset(&edata, 0, sizeof(struct ethtool_keee)); 2440 phylink_ethtool_set_eee(priv->phylink, &edata); 2441 2442 phylink_start(priv->phylink); 2443 2444 return 0; 2445 } 2446 2447 static void enetc_tx_onestep_tstamp(struct work_struct *work) 2448 { 2449 struct enetc_ndev_priv *priv; 2450 struct sk_buff *skb; 2451 2452 priv = container_of(work, struct enetc_ndev_priv, tx_onestep_tstamp); 2453 2454 netif_tx_lock_bh(priv->ndev); 2455 2456 clear_bit_unlock(ENETC_TX_ONESTEP_TSTAMP_IN_PROGRESS, &priv->flags); 2457 skb = skb_dequeue(&priv->tx_skbs); 2458 if (skb) 2459 enetc_start_xmit(skb, priv->ndev); 2460 2461 netif_tx_unlock_bh(priv->ndev); 2462 } 2463 2464 static void enetc_tx_onestep_tstamp_init(struct enetc_ndev_priv *priv) 2465 { 2466 INIT_WORK(&priv->tx_onestep_tstamp, enetc_tx_onestep_tstamp); 2467 skb_queue_head_init(&priv->tx_skbs); 2468 } 2469 2470 void enetc_start(struct net_device *ndev) 2471 { 2472 struct enetc_ndev_priv *priv = netdev_priv(ndev); 2473 int i; 2474 2475 enetc_setup_interrupts(priv); 2476 2477 for (i = 0; i < priv->bdr_int_num; i++) { 2478 int irq = pci_irq_vector(priv->si->pdev, 2479 ENETC_BDR_INT_BASE_IDX + i); 2480 2481 napi_enable(&priv->int_vector[i]->napi); 2482 enable_irq(irq); 2483 } 2484 2485 enetc_enable_tx_bdrs(priv); 2486 2487 enetc_enable_rx_bdrs(priv); 2488 2489 netif_tx_start_all_queues(ndev); 2490 2491 clear_bit(ENETC_TX_DOWN, &priv->flags); 2492 } 2493 EXPORT_SYMBOL_GPL(enetc_start); 2494 2495 int enetc_open(struct net_device *ndev) 2496 { 2497 struct enetc_ndev_priv *priv = netdev_priv(ndev); 2498 struct enetc_bdr_resource *tx_res, *rx_res; 2499 bool extended; 2500 int err; 2501 2502 extended = !!(priv->active_offloads & ENETC_F_RX_TSTAMP); 2503 2504 err = enetc_setup_irqs(priv); 2505 if (err) 2506 return err; 2507 2508 err = enetc_phylink_connect(ndev); 2509 if (err) 2510 goto err_phy_connect; 2511 2512 tx_res = enetc_alloc_tx_resources(priv); 2513 if (IS_ERR(tx_res)) { 2514 err = PTR_ERR(tx_res); 2515 goto err_alloc_tx; 2516 } 2517 2518 rx_res = enetc_alloc_rx_resources(priv, extended); 2519 if (IS_ERR(rx_res)) { 2520 err = PTR_ERR(rx_res); 2521 goto err_alloc_rx; 2522 } 2523 2524 enetc_tx_onestep_tstamp_init(priv); 2525 enetc_assign_tx_resources(priv, tx_res); 2526 enetc_assign_rx_resources(priv, rx_res); 2527 enetc_setup_bdrs(priv, extended); 2528 enetc_start(ndev); 2529 2530 return 0; 2531 2532 err_alloc_rx: 2533 enetc_free_tx_resources(tx_res, priv->num_tx_rings); 2534 err_alloc_tx: 2535 if (priv->phylink) 2536 phylink_disconnect_phy(priv->phylink); 2537 err_phy_connect: 2538 enetc_free_irqs(priv); 2539 2540 return err; 2541 } 2542 EXPORT_SYMBOL_GPL(enetc_open); 2543 2544 void enetc_stop(struct net_device *ndev) 2545 { 2546 struct enetc_ndev_priv *priv = netdev_priv(ndev); 2547 int i; 2548 2549 set_bit(ENETC_TX_DOWN, &priv->flags); 2550 2551 netif_tx_stop_all_queues(ndev); 2552 2553 enetc_disable_rx_bdrs(priv); 2554 2555 enetc_wait_bdrs(priv); 2556 2557 enetc_disable_tx_bdrs(priv); 2558 2559 for (i = 0; i < priv->bdr_int_num; i++) { 2560 int irq = pci_irq_vector(priv->si->pdev, 2561 ENETC_BDR_INT_BASE_IDX + i); 2562 2563 disable_irq(irq); 2564 napi_synchronize(&priv->int_vector[i]->napi); 2565 napi_disable(&priv->int_vector[i]->napi); 2566 } 2567 2568 enetc_clear_interrupts(priv); 2569 } 2570 EXPORT_SYMBOL_GPL(enetc_stop); 2571 2572 int enetc_close(struct net_device *ndev) 2573 { 2574 struct enetc_ndev_priv *priv = netdev_priv(ndev); 2575 2576 enetc_stop(ndev); 2577 2578 if (priv->phylink) { 2579 phylink_stop(priv->phylink); 2580 phylink_disconnect_phy(priv->phylink); 2581 } else { 2582 netif_carrier_off(ndev); 2583 } 2584 2585 enetc_free_rxtx_rings(priv); 2586 2587 /* Avoids dangling pointers and also frees old resources */ 2588 enetc_assign_rx_resources(priv, NULL); 2589 enetc_assign_tx_resources(priv, NULL); 2590 2591 enetc_free_irqs(priv); 2592 2593 return 0; 2594 } 2595 EXPORT_SYMBOL_GPL(enetc_close); 2596 2597 static int enetc_reconfigure(struct enetc_ndev_priv *priv, bool extended, 2598 int (*cb)(struct enetc_ndev_priv *priv, void *ctx), 2599 void *ctx) 2600 { 2601 struct enetc_bdr_resource *tx_res, *rx_res; 2602 int err; 2603 2604 ASSERT_RTNL(); 2605 2606 /* If the interface is down, run the callback right away, 2607 * without reconfiguration. 2608 */ 2609 if (!netif_running(priv->ndev)) { 2610 if (cb) { 2611 err = cb(priv, ctx); 2612 if (err) 2613 return err; 2614 } 2615 2616 return 0; 2617 } 2618 2619 tx_res = enetc_alloc_tx_resources(priv); 2620 if (IS_ERR(tx_res)) { 2621 err = PTR_ERR(tx_res); 2622 goto out; 2623 } 2624 2625 rx_res = enetc_alloc_rx_resources(priv, extended); 2626 if (IS_ERR(rx_res)) { 2627 err = PTR_ERR(rx_res); 2628 goto out_free_tx_res; 2629 } 2630 2631 enetc_stop(priv->ndev); 2632 enetc_free_rxtx_rings(priv); 2633 2634 /* Interface is down, run optional callback now */ 2635 if (cb) { 2636 err = cb(priv, ctx); 2637 if (err) 2638 goto out_restart; 2639 } 2640 2641 enetc_assign_tx_resources(priv, tx_res); 2642 enetc_assign_rx_resources(priv, rx_res); 2643 enetc_setup_bdrs(priv, extended); 2644 enetc_start(priv->ndev); 2645 2646 return 0; 2647 2648 out_restart: 2649 enetc_setup_bdrs(priv, extended); 2650 enetc_start(priv->ndev); 2651 enetc_free_rx_resources(rx_res, priv->num_rx_rings); 2652 out_free_tx_res: 2653 enetc_free_tx_resources(tx_res, priv->num_tx_rings); 2654 out: 2655 return err; 2656 } 2657 2658 static void enetc_debug_tx_ring_prios(struct enetc_ndev_priv *priv) 2659 { 2660 int i; 2661 2662 for (i = 0; i < priv->num_tx_rings; i++) 2663 netdev_dbg(priv->ndev, "TX ring %d prio %d\n", i, 2664 priv->tx_ring[i]->prio); 2665 } 2666 2667 void enetc_reset_tc_mqprio(struct net_device *ndev) 2668 { 2669 struct enetc_ndev_priv *priv = netdev_priv(ndev); 2670 struct enetc_hw *hw = &priv->si->hw; 2671 struct enetc_bdr *tx_ring; 2672 int num_stack_tx_queues; 2673 int i; 2674 2675 num_stack_tx_queues = enetc_num_stack_tx_queues(priv); 2676 2677 netdev_reset_tc(ndev); 2678 netif_set_real_num_tx_queues(ndev, num_stack_tx_queues); 2679 priv->min_num_stack_tx_queues = num_possible_cpus(); 2680 2681 /* Reset all ring priorities to 0 */ 2682 for (i = 0; i < priv->num_tx_rings; i++) { 2683 tx_ring = priv->tx_ring[i]; 2684 tx_ring->prio = 0; 2685 enetc_set_bdr_prio(hw, tx_ring->index, tx_ring->prio); 2686 } 2687 2688 enetc_debug_tx_ring_prios(priv); 2689 2690 enetc_change_preemptible_tcs(priv, 0); 2691 } 2692 EXPORT_SYMBOL_GPL(enetc_reset_tc_mqprio); 2693 2694 int enetc_setup_tc_mqprio(struct net_device *ndev, void *type_data) 2695 { 2696 struct tc_mqprio_qopt_offload *mqprio = type_data; 2697 struct enetc_ndev_priv *priv = netdev_priv(ndev); 2698 struct tc_mqprio_qopt *qopt = &mqprio->qopt; 2699 struct enetc_hw *hw = &priv->si->hw; 2700 int num_stack_tx_queues = 0; 2701 struct enetc_bdr *tx_ring; 2702 u8 num_tc = qopt->num_tc; 2703 int offset, count; 2704 int err, tc, q; 2705 2706 if (!num_tc) { 2707 enetc_reset_tc_mqprio(ndev); 2708 return 0; 2709 } 2710 2711 err = netdev_set_num_tc(ndev, num_tc); 2712 if (err) 2713 return err; 2714 2715 for (tc = 0; tc < num_tc; tc++) { 2716 offset = qopt->offset[tc]; 2717 count = qopt->count[tc]; 2718 num_stack_tx_queues += count; 2719 2720 err = netdev_set_tc_queue(ndev, tc, count, offset); 2721 if (err) 2722 goto err_reset_tc; 2723 2724 for (q = offset; q < offset + count; q++) { 2725 tx_ring = priv->tx_ring[q]; 2726 /* The prio_tc_map is skb_tx_hash()'s way of selecting 2727 * between TX queues based on skb->priority. As such, 2728 * there's nothing to offload based on it. 2729 * Make the mqprio "traffic class" be the priority of 2730 * this ring group, and leave the Tx IPV to traffic 2731 * class mapping as its default mapping value of 1:1. 2732 */ 2733 tx_ring->prio = tc; 2734 enetc_set_bdr_prio(hw, tx_ring->index, tx_ring->prio); 2735 } 2736 } 2737 2738 err = netif_set_real_num_tx_queues(ndev, num_stack_tx_queues); 2739 if (err) 2740 goto err_reset_tc; 2741 2742 priv->min_num_stack_tx_queues = num_stack_tx_queues; 2743 2744 enetc_debug_tx_ring_prios(priv); 2745 2746 enetc_change_preemptible_tcs(priv, mqprio->preemptible_tcs); 2747 2748 return 0; 2749 2750 err_reset_tc: 2751 enetc_reset_tc_mqprio(ndev); 2752 return err; 2753 } 2754 EXPORT_SYMBOL_GPL(enetc_setup_tc_mqprio); 2755 2756 static int enetc_reconfigure_xdp_cb(struct enetc_ndev_priv *priv, void *ctx) 2757 { 2758 struct bpf_prog *old_prog, *prog = ctx; 2759 int num_stack_tx_queues; 2760 int err, i; 2761 2762 old_prog = xchg(&priv->xdp_prog, prog); 2763 2764 num_stack_tx_queues = enetc_num_stack_tx_queues(priv); 2765 err = netif_set_real_num_tx_queues(priv->ndev, num_stack_tx_queues); 2766 if (err) { 2767 xchg(&priv->xdp_prog, old_prog); 2768 return err; 2769 } 2770 2771 if (old_prog) 2772 bpf_prog_put(old_prog); 2773 2774 for (i = 0; i < priv->num_rx_rings; i++) { 2775 struct enetc_bdr *rx_ring = priv->rx_ring[i]; 2776 2777 rx_ring->xdp.prog = prog; 2778 2779 if (prog) 2780 rx_ring->buffer_offset = XDP_PACKET_HEADROOM; 2781 else 2782 rx_ring->buffer_offset = ENETC_RXB_PAD; 2783 } 2784 2785 return 0; 2786 } 2787 2788 static int enetc_setup_xdp_prog(struct net_device *ndev, struct bpf_prog *prog, 2789 struct netlink_ext_ack *extack) 2790 { 2791 int num_xdp_tx_queues = prog ? num_possible_cpus() : 0; 2792 struct enetc_ndev_priv *priv = netdev_priv(ndev); 2793 bool extended; 2794 2795 if (priv->min_num_stack_tx_queues + num_xdp_tx_queues > 2796 priv->num_tx_rings) { 2797 NL_SET_ERR_MSG_FMT_MOD(extack, 2798 "Reserving %d XDP TXQs leaves under %d for stack (total %d)", 2799 num_xdp_tx_queues, 2800 priv->min_num_stack_tx_queues, 2801 priv->num_tx_rings); 2802 return -EBUSY; 2803 } 2804 2805 extended = !!(priv->active_offloads & ENETC_F_RX_TSTAMP); 2806 2807 /* The buffer layout is changing, so we need to drain the old 2808 * RX buffers and seed new ones. 2809 */ 2810 return enetc_reconfigure(priv, extended, enetc_reconfigure_xdp_cb, prog); 2811 } 2812 2813 int enetc_setup_bpf(struct net_device *ndev, struct netdev_bpf *bpf) 2814 { 2815 switch (bpf->command) { 2816 case XDP_SETUP_PROG: 2817 return enetc_setup_xdp_prog(ndev, bpf->prog, bpf->extack); 2818 default: 2819 return -EINVAL; 2820 } 2821 2822 return 0; 2823 } 2824 EXPORT_SYMBOL_GPL(enetc_setup_bpf); 2825 2826 struct net_device_stats *enetc_get_stats(struct net_device *ndev) 2827 { 2828 struct enetc_ndev_priv *priv = netdev_priv(ndev); 2829 struct net_device_stats *stats = &ndev->stats; 2830 unsigned long packets = 0, bytes = 0; 2831 unsigned long tx_dropped = 0; 2832 int i; 2833 2834 for (i = 0; i < priv->num_rx_rings; i++) { 2835 packets += priv->rx_ring[i]->stats.packets; 2836 bytes += priv->rx_ring[i]->stats.bytes; 2837 } 2838 2839 stats->rx_packets = packets; 2840 stats->rx_bytes = bytes; 2841 bytes = 0; 2842 packets = 0; 2843 2844 for (i = 0; i < priv->num_tx_rings; i++) { 2845 packets += priv->tx_ring[i]->stats.packets; 2846 bytes += priv->tx_ring[i]->stats.bytes; 2847 tx_dropped += priv->tx_ring[i]->stats.win_drop; 2848 } 2849 2850 stats->tx_packets = packets; 2851 stats->tx_bytes = bytes; 2852 stats->tx_dropped = tx_dropped; 2853 2854 return stats; 2855 } 2856 EXPORT_SYMBOL_GPL(enetc_get_stats); 2857 2858 static int enetc_set_rss(struct net_device *ndev, int en) 2859 { 2860 struct enetc_ndev_priv *priv = netdev_priv(ndev); 2861 struct enetc_hw *hw = &priv->si->hw; 2862 u32 reg; 2863 2864 enetc_wr(hw, ENETC_SIRBGCR, priv->num_rx_rings); 2865 2866 reg = enetc_rd(hw, ENETC_SIMR); 2867 reg &= ~ENETC_SIMR_RSSE; 2868 reg |= (en) ? ENETC_SIMR_RSSE : 0; 2869 enetc_wr(hw, ENETC_SIMR, reg); 2870 2871 return 0; 2872 } 2873 2874 static void enetc_enable_rxvlan(struct net_device *ndev, bool en) 2875 { 2876 struct enetc_ndev_priv *priv = netdev_priv(ndev); 2877 struct enetc_hw *hw = &priv->si->hw; 2878 int i; 2879 2880 for (i = 0; i < priv->num_rx_rings; i++) 2881 enetc_bdr_enable_rxvlan(hw, i, en); 2882 } 2883 2884 static void enetc_enable_txvlan(struct net_device *ndev, bool en) 2885 { 2886 struct enetc_ndev_priv *priv = netdev_priv(ndev); 2887 struct enetc_hw *hw = &priv->si->hw; 2888 int i; 2889 2890 for (i = 0; i < priv->num_tx_rings; i++) 2891 enetc_bdr_enable_txvlan(hw, i, en); 2892 } 2893 2894 void enetc_set_features(struct net_device *ndev, netdev_features_t features) 2895 { 2896 netdev_features_t changed = ndev->features ^ features; 2897 2898 if (changed & NETIF_F_RXHASH) 2899 enetc_set_rss(ndev, !!(features & NETIF_F_RXHASH)); 2900 2901 if (changed & NETIF_F_HW_VLAN_CTAG_RX) 2902 enetc_enable_rxvlan(ndev, 2903 !!(features & NETIF_F_HW_VLAN_CTAG_RX)); 2904 2905 if (changed & NETIF_F_HW_VLAN_CTAG_TX) 2906 enetc_enable_txvlan(ndev, 2907 !!(features & NETIF_F_HW_VLAN_CTAG_TX)); 2908 } 2909 EXPORT_SYMBOL_GPL(enetc_set_features); 2910 2911 static int enetc_hwtstamp_set(struct net_device *ndev, struct ifreq *ifr) 2912 { 2913 struct enetc_ndev_priv *priv = netdev_priv(ndev); 2914 int err, new_offloads = priv->active_offloads; 2915 struct hwtstamp_config config; 2916 2917 if (copy_from_user(&config, ifr->ifr_data, sizeof(config))) 2918 return -EFAULT; 2919 2920 switch (config.tx_type) { 2921 case HWTSTAMP_TX_OFF: 2922 new_offloads &= ~ENETC_F_TX_TSTAMP_MASK; 2923 break; 2924 case HWTSTAMP_TX_ON: 2925 new_offloads &= ~ENETC_F_TX_TSTAMP_MASK; 2926 new_offloads |= ENETC_F_TX_TSTAMP; 2927 break; 2928 case HWTSTAMP_TX_ONESTEP_SYNC: 2929 new_offloads &= ~ENETC_F_TX_TSTAMP_MASK; 2930 new_offloads |= ENETC_F_TX_ONESTEP_SYNC_TSTAMP; 2931 break; 2932 default: 2933 return -ERANGE; 2934 } 2935 2936 switch (config.rx_filter) { 2937 case HWTSTAMP_FILTER_NONE: 2938 new_offloads &= ~ENETC_F_RX_TSTAMP; 2939 break; 2940 default: 2941 new_offloads |= ENETC_F_RX_TSTAMP; 2942 config.rx_filter = HWTSTAMP_FILTER_ALL; 2943 } 2944 2945 if ((new_offloads ^ priv->active_offloads) & ENETC_F_RX_TSTAMP) { 2946 bool extended = !!(new_offloads & ENETC_F_RX_TSTAMP); 2947 2948 err = enetc_reconfigure(priv, extended, NULL, NULL); 2949 if (err) 2950 return err; 2951 } 2952 2953 priv->active_offloads = new_offloads; 2954 2955 return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ? 2956 -EFAULT : 0; 2957 } 2958 2959 static int enetc_hwtstamp_get(struct net_device *ndev, struct ifreq *ifr) 2960 { 2961 struct enetc_ndev_priv *priv = netdev_priv(ndev); 2962 struct hwtstamp_config config; 2963 2964 config.flags = 0; 2965 2966 if (priv->active_offloads & ENETC_F_TX_ONESTEP_SYNC_TSTAMP) 2967 config.tx_type = HWTSTAMP_TX_ONESTEP_SYNC; 2968 else if (priv->active_offloads & ENETC_F_TX_TSTAMP) 2969 config.tx_type = HWTSTAMP_TX_ON; 2970 else 2971 config.tx_type = HWTSTAMP_TX_OFF; 2972 2973 config.rx_filter = (priv->active_offloads & ENETC_F_RX_TSTAMP) ? 2974 HWTSTAMP_FILTER_ALL : HWTSTAMP_FILTER_NONE; 2975 2976 return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ? 2977 -EFAULT : 0; 2978 } 2979 2980 int enetc_ioctl(struct net_device *ndev, struct ifreq *rq, int cmd) 2981 { 2982 struct enetc_ndev_priv *priv = netdev_priv(ndev); 2983 2984 if (IS_ENABLED(CONFIG_FSL_ENETC_PTP_CLOCK)) { 2985 if (cmd == SIOCSHWTSTAMP) 2986 return enetc_hwtstamp_set(ndev, rq); 2987 if (cmd == SIOCGHWTSTAMP) 2988 return enetc_hwtstamp_get(ndev, rq); 2989 } 2990 2991 if (!priv->phylink) 2992 return -EOPNOTSUPP; 2993 2994 return phylink_mii_ioctl(priv->phylink, rq, cmd); 2995 } 2996 EXPORT_SYMBOL_GPL(enetc_ioctl); 2997 2998 int enetc_alloc_msix(struct enetc_ndev_priv *priv) 2999 { 3000 struct pci_dev *pdev = priv->si->pdev; 3001 int num_stack_tx_queues; 3002 int first_xdp_tx_ring; 3003 int i, n, err, nvec; 3004 int v_tx_rings; 3005 3006 nvec = ENETC_BDR_INT_BASE_IDX + priv->bdr_int_num; 3007 /* allocate MSIX for both messaging and Rx/Tx interrupts */ 3008 n = pci_alloc_irq_vectors(pdev, nvec, nvec, PCI_IRQ_MSIX); 3009 3010 if (n < 0) 3011 return n; 3012 3013 if (n != nvec) 3014 return -EPERM; 3015 3016 /* # of tx rings per int vector */ 3017 v_tx_rings = priv->num_tx_rings / priv->bdr_int_num; 3018 3019 for (i = 0; i < priv->bdr_int_num; i++) { 3020 struct enetc_int_vector *v; 3021 struct enetc_bdr *bdr; 3022 int j; 3023 3024 v = kzalloc(struct_size(v, tx_ring, v_tx_rings), GFP_KERNEL); 3025 if (!v) { 3026 err = -ENOMEM; 3027 goto fail; 3028 } 3029 3030 priv->int_vector[i] = v; 3031 3032 bdr = &v->rx_ring; 3033 bdr->index = i; 3034 bdr->ndev = priv->ndev; 3035 bdr->dev = priv->dev; 3036 bdr->bd_count = priv->rx_bd_count; 3037 bdr->buffer_offset = ENETC_RXB_PAD; 3038 priv->rx_ring[i] = bdr; 3039 3040 err = xdp_rxq_info_reg(&bdr->xdp.rxq, priv->ndev, i, 0); 3041 if (err) { 3042 kfree(v); 3043 goto fail; 3044 } 3045 3046 err = xdp_rxq_info_reg_mem_model(&bdr->xdp.rxq, 3047 MEM_TYPE_PAGE_SHARED, NULL); 3048 if (err) { 3049 xdp_rxq_info_unreg(&bdr->xdp.rxq); 3050 kfree(v); 3051 goto fail; 3052 } 3053 3054 /* init defaults for adaptive IC */ 3055 if (priv->ic_mode & ENETC_IC_RX_ADAPTIVE) { 3056 v->rx_ictt = 0x1; 3057 v->rx_dim_en = true; 3058 } 3059 INIT_WORK(&v->rx_dim.work, enetc_rx_dim_work); 3060 netif_napi_add(priv->ndev, &v->napi, enetc_poll); 3061 v->count_tx_rings = v_tx_rings; 3062 3063 for (j = 0; j < v_tx_rings; j++) { 3064 int idx; 3065 3066 /* default tx ring mapping policy */ 3067 idx = priv->bdr_int_num * j + i; 3068 __set_bit(idx, &v->tx_rings_map); 3069 bdr = &v->tx_ring[j]; 3070 bdr->index = idx; 3071 bdr->ndev = priv->ndev; 3072 bdr->dev = priv->dev; 3073 bdr->bd_count = priv->tx_bd_count; 3074 priv->tx_ring[idx] = bdr; 3075 } 3076 } 3077 3078 num_stack_tx_queues = enetc_num_stack_tx_queues(priv); 3079 3080 err = netif_set_real_num_tx_queues(priv->ndev, num_stack_tx_queues); 3081 if (err) 3082 goto fail; 3083 3084 err = netif_set_real_num_rx_queues(priv->ndev, priv->num_rx_rings); 3085 if (err) 3086 goto fail; 3087 3088 priv->min_num_stack_tx_queues = num_possible_cpus(); 3089 first_xdp_tx_ring = priv->num_tx_rings - num_possible_cpus(); 3090 priv->xdp_tx_ring = &priv->tx_ring[first_xdp_tx_ring]; 3091 3092 return 0; 3093 3094 fail: 3095 while (i--) { 3096 struct enetc_int_vector *v = priv->int_vector[i]; 3097 struct enetc_bdr *rx_ring = &v->rx_ring; 3098 3099 xdp_rxq_info_unreg_mem_model(&rx_ring->xdp.rxq); 3100 xdp_rxq_info_unreg(&rx_ring->xdp.rxq); 3101 netif_napi_del(&v->napi); 3102 cancel_work_sync(&v->rx_dim.work); 3103 kfree(v); 3104 } 3105 3106 pci_free_irq_vectors(pdev); 3107 3108 return err; 3109 } 3110 EXPORT_SYMBOL_GPL(enetc_alloc_msix); 3111 3112 void enetc_free_msix(struct enetc_ndev_priv *priv) 3113 { 3114 int i; 3115 3116 for (i = 0; i < priv->bdr_int_num; i++) { 3117 struct enetc_int_vector *v = priv->int_vector[i]; 3118 struct enetc_bdr *rx_ring = &v->rx_ring; 3119 3120 xdp_rxq_info_unreg_mem_model(&rx_ring->xdp.rxq); 3121 xdp_rxq_info_unreg(&rx_ring->xdp.rxq); 3122 netif_napi_del(&v->napi); 3123 cancel_work_sync(&v->rx_dim.work); 3124 } 3125 3126 for (i = 0; i < priv->num_rx_rings; i++) 3127 priv->rx_ring[i] = NULL; 3128 3129 for (i = 0; i < priv->num_tx_rings; i++) 3130 priv->tx_ring[i] = NULL; 3131 3132 for (i = 0; i < priv->bdr_int_num; i++) { 3133 kfree(priv->int_vector[i]); 3134 priv->int_vector[i] = NULL; 3135 } 3136 3137 /* disable all MSIX for this device */ 3138 pci_free_irq_vectors(priv->si->pdev); 3139 } 3140 EXPORT_SYMBOL_GPL(enetc_free_msix); 3141 3142 static void enetc_kfree_si(struct enetc_si *si) 3143 { 3144 char *p = (char *)si - si->pad; 3145 3146 kfree(p); 3147 } 3148 3149 static void enetc_detect_errata(struct enetc_si *si) 3150 { 3151 if (si->pdev->revision == ENETC_REV1) 3152 si->errata = ENETC_ERR_VLAN_ISOL | ENETC_ERR_UCMCSWP; 3153 } 3154 3155 int enetc_pci_probe(struct pci_dev *pdev, const char *name, int sizeof_priv) 3156 { 3157 struct enetc_si *si, *p; 3158 struct enetc_hw *hw; 3159 size_t alloc_size; 3160 int err, len; 3161 3162 pcie_flr(pdev); 3163 err = pci_enable_device_mem(pdev); 3164 if (err) 3165 return dev_err_probe(&pdev->dev, err, "device enable failed\n"); 3166 3167 /* set up for high or low dma */ 3168 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 3169 if (err) { 3170 dev_err(&pdev->dev, "DMA configuration failed: 0x%x\n", err); 3171 goto err_dma; 3172 } 3173 3174 err = pci_request_mem_regions(pdev, name); 3175 if (err) { 3176 dev_err(&pdev->dev, "pci_request_regions failed err=%d\n", err); 3177 goto err_pci_mem_reg; 3178 } 3179 3180 pci_set_master(pdev); 3181 3182 alloc_size = sizeof(struct enetc_si); 3183 if (sizeof_priv) { 3184 /* align priv to 32B */ 3185 alloc_size = ALIGN(alloc_size, ENETC_SI_ALIGN); 3186 alloc_size += sizeof_priv; 3187 } 3188 /* force 32B alignment for enetc_si */ 3189 alloc_size += ENETC_SI_ALIGN - 1; 3190 3191 p = kzalloc(alloc_size, GFP_KERNEL); 3192 if (!p) { 3193 err = -ENOMEM; 3194 goto err_alloc_si; 3195 } 3196 3197 si = PTR_ALIGN(p, ENETC_SI_ALIGN); 3198 si->pad = (char *)si - (char *)p; 3199 3200 pci_set_drvdata(pdev, si); 3201 si->pdev = pdev; 3202 hw = &si->hw; 3203 3204 len = pci_resource_len(pdev, ENETC_BAR_REGS); 3205 hw->reg = ioremap(pci_resource_start(pdev, ENETC_BAR_REGS), len); 3206 if (!hw->reg) { 3207 err = -ENXIO; 3208 dev_err(&pdev->dev, "ioremap() failed\n"); 3209 goto err_ioremap; 3210 } 3211 if (len > ENETC_PORT_BASE) 3212 hw->port = hw->reg + ENETC_PORT_BASE; 3213 if (len > ENETC_GLOBAL_BASE) 3214 hw->global = hw->reg + ENETC_GLOBAL_BASE; 3215 3216 enetc_detect_errata(si); 3217 3218 return 0; 3219 3220 err_ioremap: 3221 enetc_kfree_si(si); 3222 err_alloc_si: 3223 pci_release_mem_regions(pdev); 3224 err_pci_mem_reg: 3225 err_dma: 3226 pci_disable_device(pdev); 3227 3228 return err; 3229 } 3230 EXPORT_SYMBOL_GPL(enetc_pci_probe); 3231 3232 void enetc_pci_remove(struct pci_dev *pdev) 3233 { 3234 struct enetc_si *si = pci_get_drvdata(pdev); 3235 struct enetc_hw *hw = &si->hw; 3236 3237 iounmap(hw->reg); 3238 enetc_kfree_si(si); 3239 pci_release_mem_regions(pdev); 3240 pci_disable_device(pdev); 3241 } 3242 EXPORT_SYMBOL_GPL(enetc_pci_remove); 3243 3244 MODULE_DESCRIPTION("NXP ENETC Ethernet driver"); 3245 MODULE_LICENSE("Dual BSD/GPL"); 3246