1 // SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) 2 /* Copyright 2017-2019 NXP */ 3 4 #include "enetc.h" 5 #include <linux/bpf_trace.h> 6 #include <linux/clk.h> 7 #include <linux/tcp.h> 8 #include <linux/udp.h> 9 #include <linux/vmalloc.h> 10 #include <linux/ptp_classify.h> 11 #include <net/ip6_checksum.h> 12 #include <net/pkt_sched.h> 13 #include <net/tso.h> 14 15 u32 enetc_port_mac_rd(struct enetc_si *si, u32 reg) 16 { 17 return enetc_port_rd(&si->hw, reg); 18 } 19 EXPORT_SYMBOL_GPL(enetc_port_mac_rd); 20 21 void enetc_port_mac_wr(struct enetc_si *si, u32 reg, u32 val) 22 { 23 enetc_port_wr(&si->hw, reg, val); 24 if (si->hw_features & ENETC_SI_F_QBU) 25 enetc_port_wr(&si->hw, reg + si->drvdata->pmac_offset, val); 26 } 27 EXPORT_SYMBOL_GPL(enetc_port_mac_wr); 28 29 static void enetc_change_preemptible_tcs(struct enetc_ndev_priv *priv, 30 u8 preemptible_tcs) 31 { 32 priv->preemptible_tcs = preemptible_tcs; 33 enetc_mm_commit_preemptible_tcs(priv); 34 } 35 36 static int enetc_num_stack_tx_queues(struct enetc_ndev_priv *priv) 37 { 38 int num_tx_rings = priv->num_tx_rings; 39 40 if (priv->xdp_prog) 41 return num_tx_rings - num_possible_cpus(); 42 43 return num_tx_rings; 44 } 45 46 static struct enetc_bdr *enetc_rx_ring_from_xdp_tx_ring(struct enetc_ndev_priv *priv, 47 struct enetc_bdr *tx_ring) 48 { 49 int index = &priv->tx_ring[tx_ring->index] - priv->xdp_tx_ring; 50 51 return priv->rx_ring[index]; 52 } 53 54 static struct sk_buff *enetc_tx_swbd_get_skb(struct enetc_tx_swbd *tx_swbd) 55 { 56 if (tx_swbd->is_xdp_tx || tx_swbd->is_xdp_redirect) 57 return NULL; 58 59 return tx_swbd->skb; 60 } 61 62 static struct xdp_frame * 63 enetc_tx_swbd_get_xdp_frame(struct enetc_tx_swbd *tx_swbd) 64 { 65 if (tx_swbd->is_xdp_redirect) 66 return tx_swbd->xdp_frame; 67 68 return NULL; 69 } 70 71 static void enetc_unmap_tx_buff(struct enetc_bdr *tx_ring, 72 struct enetc_tx_swbd *tx_swbd) 73 { 74 /* For XDP_TX, pages come from RX, whereas for the other contexts where 75 * we have is_dma_page_set, those come from skb_frag_dma_map. We need 76 * to match the DMA mapping length, so we need to differentiate those. 77 */ 78 if (tx_swbd->is_dma_page) 79 dma_unmap_page(tx_ring->dev, tx_swbd->dma, 80 tx_swbd->is_xdp_tx ? PAGE_SIZE : tx_swbd->len, 81 tx_swbd->dir); 82 else 83 dma_unmap_single(tx_ring->dev, tx_swbd->dma, 84 tx_swbd->len, tx_swbd->dir); 85 tx_swbd->dma = 0; 86 } 87 88 static void enetc_free_tx_frame(struct enetc_bdr *tx_ring, 89 struct enetc_tx_swbd *tx_swbd) 90 { 91 struct xdp_frame *xdp_frame = enetc_tx_swbd_get_xdp_frame(tx_swbd); 92 struct sk_buff *skb = enetc_tx_swbd_get_skb(tx_swbd); 93 94 if (tx_swbd->dma) 95 enetc_unmap_tx_buff(tx_ring, tx_swbd); 96 97 if (xdp_frame) { 98 xdp_return_frame(tx_swbd->xdp_frame); 99 tx_swbd->xdp_frame = NULL; 100 } else if (skb) { 101 dev_kfree_skb_any(skb); 102 tx_swbd->skb = NULL; 103 } 104 } 105 106 /* Let H/W know BD ring has been updated */ 107 static void enetc_update_tx_ring_tail(struct enetc_bdr *tx_ring) 108 { 109 /* includes wmb() */ 110 enetc_wr_reg_hot(tx_ring->tpir, tx_ring->next_to_use); 111 } 112 113 static int enetc_ptp_parse(struct sk_buff *skb, u8 *udp, 114 u8 *msgtype, u8 *twostep, 115 u16 *correction_offset, u16 *body_offset) 116 { 117 unsigned int ptp_class; 118 struct ptp_header *hdr; 119 unsigned int type; 120 u8 *base; 121 122 ptp_class = ptp_classify_raw(skb); 123 if (ptp_class == PTP_CLASS_NONE) 124 return -EINVAL; 125 126 hdr = ptp_parse_header(skb, ptp_class); 127 if (!hdr) 128 return -EINVAL; 129 130 type = ptp_class & PTP_CLASS_PMASK; 131 if (type == PTP_CLASS_IPV4 || type == PTP_CLASS_IPV6) 132 *udp = 1; 133 else 134 *udp = 0; 135 136 *msgtype = ptp_get_msgtype(hdr, ptp_class); 137 *twostep = hdr->flag_field[0] & 0x2; 138 139 base = skb_mac_header(skb); 140 *correction_offset = (u8 *)&hdr->correction - base; 141 *body_offset = (u8 *)hdr + sizeof(struct ptp_header) - base; 142 143 return 0; 144 } 145 146 static int enetc_map_tx_buffs(struct enetc_bdr *tx_ring, struct sk_buff *skb) 147 { 148 bool do_vlan, do_onestep_tstamp = false, do_twostep_tstamp = false; 149 struct enetc_ndev_priv *priv = netdev_priv(tx_ring->ndev); 150 struct enetc_hw *hw = &priv->si->hw; 151 struct enetc_tx_swbd *tx_swbd; 152 int len = skb_headlen(skb); 153 union enetc_tx_bd temp_bd; 154 u8 msgtype, twostep, udp; 155 union enetc_tx_bd *txbd; 156 u16 offset1, offset2; 157 int i, count = 0; 158 skb_frag_t *frag; 159 unsigned int f; 160 dma_addr_t dma; 161 u8 flags = 0; 162 163 i = tx_ring->next_to_use; 164 txbd = ENETC_TXBD(*tx_ring, i); 165 prefetchw(txbd); 166 167 dma = dma_map_single(tx_ring->dev, skb->data, len, DMA_TO_DEVICE); 168 if (unlikely(dma_mapping_error(tx_ring->dev, dma))) 169 goto dma_err; 170 171 temp_bd.addr = cpu_to_le64(dma); 172 temp_bd.buf_len = cpu_to_le16(len); 173 temp_bd.lstatus = 0; 174 175 tx_swbd = &tx_ring->tx_swbd[i]; 176 tx_swbd->dma = dma; 177 tx_swbd->len = len; 178 tx_swbd->is_dma_page = 0; 179 tx_swbd->dir = DMA_TO_DEVICE; 180 count++; 181 182 do_vlan = skb_vlan_tag_present(skb); 183 if (skb->cb[0] & ENETC_F_TX_ONESTEP_SYNC_TSTAMP) { 184 if (enetc_ptp_parse(skb, &udp, &msgtype, &twostep, &offset1, 185 &offset2) || 186 msgtype != PTP_MSGTYPE_SYNC || twostep) 187 WARN_ONCE(1, "Bad packet for one-step timestamping\n"); 188 else 189 do_onestep_tstamp = true; 190 } else if (skb->cb[0] & ENETC_F_TX_TSTAMP) { 191 do_twostep_tstamp = true; 192 } 193 194 tx_swbd->do_twostep_tstamp = do_twostep_tstamp; 195 tx_swbd->qbv_en = !!(priv->active_offloads & ENETC_F_QBV); 196 tx_swbd->check_wb = tx_swbd->do_twostep_tstamp || tx_swbd->qbv_en; 197 198 if (do_vlan || do_onestep_tstamp || do_twostep_tstamp) 199 flags |= ENETC_TXBD_FLAGS_EX; 200 201 if (tx_ring->tsd_enable) 202 flags |= ENETC_TXBD_FLAGS_TSE | ENETC_TXBD_FLAGS_TXSTART; 203 204 /* first BD needs frm_len and offload flags set */ 205 temp_bd.frm_len = cpu_to_le16(skb->len); 206 temp_bd.flags = flags; 207 208 if (flags & ENETC_TXBD_FLAGS_TSE) 209 temp_bd.txstart = enetc_txbd_set_tx_start(skb->skb_mstamp_ns, 210 flags); 211 212 if (flags & ENETC_TXBD_FLAGS_EX) { 213 u8 e_flags = 0; 214 *txbd = temp_bd; 215 enetc_clear_tx_bd(&temp_bd); 216 217 /* add extension BD for VLAN and/or timestamping */ 218 flags = 0; 219 tx_swbd++; 220 txbd++; 221 i++; 222 if (unlikely(i == tx_ring->bd_count)) { 223 i = 0; 224 tx_swbd = tx_ring->tx_swbd; 225 txbd = ENETC_TXBD(*tx_ring, 0); 226 } 227 prefetchw(txbd); 228 229 if (do_vlan) { 230 temp_bd.ext.vid = cpu_to_le16(skb_vlan_tag_get(skb)); 231 temp_bd.ext.tpid = 0; /* < C-TAG */ 232 e_flags |= ENETC_TXBD_E_FLAGS_VLAN_INS; 233 } 234 235 if (do_onestep_tstamp) { 236 u32 lo, hi, val; 237 u64 sec, nsec; 238 u8 *data; 239 240 lo = enetc_rd_hot(hw, ENETC_SICTR0); 241 hi = enetc_rd_hot(hw, ENETC_SICTR1); 242 sec = (u64)hi << 32 | lo; 243 nsec = do_div(sec, 1000000000); 244 245 /* Configure extension BD */ 246 temp_bd.ext.tstamp = cpu_to_le32(lo & 0x3fffffff); 247 e_flags |= ENETC_TXBD_E_FLAGS_ONE_STEP_PTP; 248 249 /* Update originTimestamp field of Sync packet 250 * - 48 bits seconds field 251 * - 32 bits nanseconds field 252 */ 253 data = skb_mac_header(skb); 254 *(__be16 *)(data + offset2) = 255 htons((sec >> 32) & 0xffff); 256 *(__be32 *)(data + offset2 + 2) = 257 htonl(sec & 0xffffffff); 258 *(__be32 *)(data + offset2 + 6) = htonl(nsec); 259 260 /* Configure single-step register */ 261 val = ENETC_PM0_SINGLE_STEP_EN; 262 val |= ENETC_SET_SINGLE_STEP_OFFSET(offset1); 263 if (udp) 264 val |= ENETC_PM0_SINGLE_STEP_CH; 265 266 enetc_port_mac_wr(priv->si, ENETC_PM0_SINGLE_STEP, 267 val); 268 } else if (do_twostep_tstamp) { 269 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; 270 e_flags |= ENETC_TXBD_E_FLAGS_TWO_STEP_PTP; 271 } 272 273 temp_bd.ext.e_flags = e_flags; 274 count++; 275 } 276 277 frag = &skb_shinfo(skb)->frags[0]; 278 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++, frag++) { 279 len = skb_frag_size(frag); 280 dma = skb_frag_dma_map(tx_ring->dev, frag, 0, len, 281 DMA_TO_DEVICE); 282 if (dma_mapping_error(tx_ring->dev, dma)) 283 goto dma_err; 284 285 *txbd = temp_bd; 286 enetc_clear_tx_bd(&temp_bd); 287 288 flags = 0; 289 tx_swbd++; 290 txbd++; 291 i++; 292 if (unlikely(i == tx_ring->bd_count)) { 293 i = 0; 294 tx_swbd = tx_ring->tx_swbd; 295 txbd = ENETC_TXBD(*tx_ring, 0); 296 } 297 prefetchw(txbd); 298 299 temp_bd.addr = cpu_to_le64(dma); 300 temp_bd.buf_len = cpu_to_le16(len); 301 302 tx_swbd->dma = dma; 303 tx_swbd->len = len; 304 tx_swbd->is_dma_page = 1; 305 tx_swbd->dir = DMA_TO_DEVICE; 306 count++; 307 } 308 309 /* last BD needs 'F' bit set */ 310 flags |= ENETC_TXBD_FLAGS_F; 311 temp_bd.flags = flags; 312 *txbd = temp_bd; 313 314 tx_ring->tx_swbd[i].is_eof = true; 315 tx_ring->tx_swbd[i].skb = skb; 316 317 enetc_bdr_idx_inc(tx_ring, &i); 318 tx_ring->next_to_use = i; 319 320 skb_tx_timestamp(skb); 321 322 enetc_update_tx_ring_tail(tx_ring); 323 324 return count; 325 326 dma_err: 327 dev_err(tx_ring->dev, "DMA map error"); 328 329 do { 330 tx_swbd = &tx_ring->tx_swbd[i]; 331 enetc_free_tx_frame(tx_ring, tx_swbd); 332 if (i == 0) 333 i = tx_ring->bd_count; 334 i--; 335 } while (count--); 336 337 return 0; 338 } 339 340 static void enetc_map_tx_tso_hdr(struct enetc_bdr *tx_ring, struct sk_buff *skb, 341 struct enetc_tx_swbd *tx_swbd, 342 union enetc_tx_bd *txbd, int *i, int hdr_len, 343 int data_len) 344 { 345 union enetc_tx_bd txbd_tmp; 346 u8 flags = 0, e_flags = 0; 347 dma_addr_t addr; 348 349 enetc_clear_tx_bd(&txbd_tmp); 350 addr = tx_ring->tso_headers_dma + *i * TSO_HEADER_SIZE; 351 352 if (skb_vlan_tag_present(skb)) 353 flags |= ENETC_TXBD_FLAGS_EX; 354 355 txbd_tmp.addr = cpu_to_le64(addr); 356 txbd_tmp.buf_len = cpu_to_le16(hdr_len); 357 358 /* first BD needs frm_len and offload flags set */ 359 txbd_tmp.frm_len = cpu_to_le16(hdr_len + data_len); 360 txbd_tmp.flags = flags; 361 362 /* For the TSO header we do not set the dma address since we do not 363 * want it unmapped when we do cleanup. We still set len so that we 364 * count the bytes sent. 365 */ 366 tx_swbd->len = hdr_len; 367 tx_swbd->do_twostep_tstamp = false; 368 tx_swbd->check_wb = false; 369 370 /* Actually write the header in the BD */ 371 *txbd = txbd_tmp; 372 373 /* Add extension BD for VLAN */ 374 if (flags & ENETC_TXBD_FLAGS_EX) { 375 /* Get the next BD */ 376 enetc_bdr_idx_inc(tx_ring, i); 377 txbd = ENETC_TXBD(*tx_ring, *i); 378 tx_swbd = &tx_ring->tx_swbd[*i]; 379 prefetchw(txbd); 380 381 /* Setup the VLAN fields */ 382 enetc_clear_tx_bd(&txbd_tmp); 383 txbd_tmp.ext.vid = cpu_to_le16(skb_vlan_tag_get(skb)); 384 txbd_tmp.ext.tpid = 0; /* < C-TAG */ 385 e_flags |= ENETC_TXBD_E_FLAGS_VLAN_INS; 386 387 /* Write the BD */ 388 txbd_tmp.ext.e_flags = e_flags; 389 *txbd = txbd_tmp; 390 } 391 } 392 393 static int enetc_map_tx_tso_data(struct enetc_bdr *tx_ring, struct sk_buff *skb, 394 struct enetc_tx_swbd *tx_swbd, 395 union enetc_tx_bd *txbd, char *data, 396 int size, bool last_bd) 397 { 398 union enetc_tx_bd txbd_tmp; 399 dma_addr_t addr; 400 u8 flags = 0; 401 402 enetc_clear_tx_bd(&txbd_tmp); 403 404 addr = dma_map_single(tx_ring->dev, data, size, DMA_TO_DEVICE); 405 if (unlikely(dma_mapping_error(tx_ring->dev, addr))) { 406 netdev_err(tx_ring->ndev, "DMA map error\n"); 407 return -ENOMEM; 408 } 409 410 if (last_bd) { 411 flags |= ENETC_TXBD_FLAGS_F; 412 tx_swbd->is_eof = 1; 413 } 414 415 txbd_tmp.addr = cpu_to_le64(addr); 416 txbd_tmp.buf_len = cpu_to_le16(size); 417 txbd_tmp.flags = flags; 418 419 tx_swbd->dma = addr; 420 tx_swbd->len = size; 421 tx_swbd->dir = DMA_TO_DEVICE; 422 423 *txbd = txbd_tmp; 424 425 return 0; 426 } 427 428 static __wsum enetc_tso_hdr_csum(struct tso_t *tso, struct sk_buff *skb, 429 char *hdr, int hdr_len, int *l4_hdr_len) 430 { 431 char *l4_hdr = hdr + skb_transport_offset(skb); 432 int mac_hdr_len = skb_network_offset(skb); 433 434 if (tso->tlen != sizeof(struct udphdr)) { 435 struct tcphdr *tcph = (struct tcphdr *)(l4_hdr); 436 437 tcph->check = 0; 438 } else { 439 struct udphdr *udph = (struct udphdr *)(l4_hdr); 440 441 udph->check = 0; 442 } 443 444 /* Compute the IP checksum. This is necessary since tso_build_hdr() 445 * already incremented the IP ID field. 446 */ 447 if (!tso->ipv6) { 448 struct iphdr *iph = (void *)(hdr + mac_hdr_len); 449 450 iph->check = 0; 451 iph->check = ip_fast_csum((unsigned char *)iph, iph->ihl); 452 } 453 454 /* Compute the checksum over the L4 header. */ 455 *l4_hdr_len = hdr_len - skb_transport_offset(skb); 456 return csum_partial(l4_hdr, *l4_hdr_len, 0); 457 } 458 459 static void enetc_tso_complete_csum(struct enetc_bdr *tx_ring, struct tso_t *tso, 460 struct sk_buff *skb, char *hdr, int len, 461 __wsum sum) 462 { 463 char *l4_hdr = hdr + skb_transport_offset(skb); 464 __sum16 csum_final; 465 466 /* Complete the L4 checksum by appending the pseudo-header to the 467 * already computed checksum. 468 */ 469 if (!tso->ipv6) 470 csum_final = csum_tcpudp_magic(ip_hdr(skb)->saddr, 471 ip_hdr(skb)->daddr, 472 len, ip_hdr(skb)->protocol, sum); 473 else 474 csum_final = csum_ipv6_magic(&ipv6_hdr(skb)->saddr, 475 &ipv6_hdr(skb)->daddr, 476 len, ipv6_hdr(skb)->nexthdr, sum); 477 478 if (tso->tlen != sizeof(struct udphdr)) { 479 struct tcphdr *tcph = (struct tcphdr *)(l4_hdr); 480 481 tcph->check = csum_final; 482 } else { 483 struct udphdr *udph = (struct udphdr *)(l4_hdr); 484 485 udph->check = csum_final; 486 } 487 } 488 489 static int enetc_map_tx_tso_buffs(struct enetc_bdr *tx_ring, struct sk_buff *skb) 490 { 491 int hdr_len, total_len, data_len; 492 struct enetc_tx_swbd *tx_swbd; 493 union enetc_tx_bd *txbd; 494 struct tso_t tso; 495 __wsum csum, csum2; 496 int count = 0, pos; 497 int err, i, bd_data_num; 498 499 /* Initialize the TSO handler, and prepare the first payload */ 500 hdr_len = tso_start(skb, &tso); 501 total_len = skb->len - hdr_len; 502 i = tx_ring->next_to_use; 503 504 while (total_len > 0) { 505 char *hdr; 506 507 /* Get the BD */ 508 txbd = ENETC_TXBD(*tx_ring, i); 509 tx_swbd = &tx_ring->tx_swbd[i]; 510 prefetchw(txbd); 511 512 /* Determine the length of this packet */ 513 data_len = min_t(int, skb_shinfo(skb)->gso_size, total_len); 514 total_len -= data_len; 515 516 /* prepare packet headers: MAC + IP + TCP */ 517 hdr = tx_ring->tso_headers + i * TSO_HEADER_SIZE; 518 tso_build_hdr(skb, hdr, &tso, data_len, total_len == 0); 519 520 /* compute the csum over the L4 header */ 521 csum = enetc_tso_hdr_csum(&tso, skb, hdr, hdr_len, &pos); 522 enetc_map_tx_tso_hdr(tx_ring, skb, tx_swbd, txbd, &i, hdr_len, data_len); 523 bd_data_num = 0; 524 count++; 525 526 while (data_len > 0) { 527 int size; 528 529 size = min_t(int, tso.size, data_len); 530 531 /* Advance the index in the BDR */ 532 enetc_bdr_idx_inc(tx_ring, &i); 533 txbd = ENETC_TXBD(*tx_ring, i); 534 tx_swbd = &tx_ring->tx_swbd[i]; 535 prefetchw(txbd); 536 537 /* Compute the checksum over this segment of data and 538 * add it to the csum already computed (over the L4 539 * header and possible other data segments). 540 */ 541 csum2 = csum_partial(tso.data, size, 0); 542 csum = csum_block_add(csum, csum2, pos); 543 pos += size; 544 545 err = enetc_map_tx_tso_data(tx_ring, skb, tx_swbd, txbd, 546 tso.data, size, 547 size == data_len); 548 if (err) 549 goto err_map_data; 550 551 data_len -= size; 552 count++; 553 bd_data_num++; 554 tso_build_data(skb, &tso, size); 555 556 if (unlikely(bd_data_num >= ENETC_MAX_SKB_FRAGS && data_len)) 557 goto err_chained_bd; 558 } 559 560 enetc_tso_complete_csum(tx_ring, &tso, skb, hdr, pos, csum); 561 562 if (total_len == 0) 563 tx_swbd->skb = skb; 564 565 /* Go to the next BD */ 566 enetc_bdr_idx_inc(tx_ring, &i); 567 } 568 569 tx_ring->next_to_use = i; 570 enetc_update_tx_ring_tail(tx_ring); 571 572 return count; 573 574 err_map_data: 575 dev_err(tx_ring->dev, "DMA map error"); 576 577 err_chained_bd: 578 do { 579 tx_swbd = &tx_ring->tx_swbd[i]; 580 enetc_free_tx_frame(tx_ring, tx_swbd); 581 if (i == 0) 582 i = tx_ring->bd_count; 583 i--; 584 } while (count--); 585 586 return 0; 587 } 588 589 static netdev_tx_t enetc_start_xmit(struct sk_buff *skb, 590 struct net_device *ndev) 591 { 592 struct enetc_ndev_priv *priv = netdev_priv(ndev); 593 struct enetc_bdr *tx_ring; 594 int count, err; 595 596 /* Queue one-step Sync packet if already locked */ 597 if (skb->cb[0] & ENETC_F_TX_ONESTEP_SYNC_TSTAMP) { 598 if (test_and_set_bit_lock(ENETC_TX_ONESTEP_TSTAMP_IN_PROGRESS, 599 &priv->flags)) { 600 skb_queue_tail(&priv->tx_skbs, skb); 601 return NETDEV_TX_OK; 602 } 603 } 604 605 tx_ring = priv->tx_ring[skb->queue_mapping]; 606 607 if (skb_is_gso(skb)) { 608 if (enetc_bd_unused(tx_ring) < tso_count_descs(skb)) { 609 netif_stop_subqueue(ndev, tx_ring->index); 610 return NETDEV_TX_BUSY; 611 } 612 613 enetc_lock_mdio(); 614 count = enetc_map_tx_tso_buffs(tx_ring, skb); 615 enetc_unlock_mdio(); 616 } else { 617 if (unlikely(skb_shinfo(skb)->nr_frags > ENETC_MAX_SKB_FRAGS)) 618 if (unlikely(skb_linearize(skb))) 619 goto drop_packet_err; 620 621 count = skb_shinfo(skb)->nr_frags + 1; /* fragments + head */ 622 if (enetc_bd_unused(tx_ring) < ENETC_TXBDS_NEEDED(count)) { 623 netif_stop_subqueue(ndev, tx_ring->index); 624 return NETDEV_TX_BUSY; 625 } 626 627 if (skb->ip_summed == CHECKSUM_PARTIAL) { 628 err = skb_checksum_help(skb); 629 if (err) 630 goto drop_packet_err; 631 } 632 enetc_lock_mdio(); 633 count = enetc_map_tx_buffs(tx_ring, skb); 634 enetc_unlock_mdio(); 635 } 636 637 if (unlikely(!count)) 638 goto drop_packet_err; 639 640 if (enetc_bd_unused(tx_ring) < ENETC_TXBDS_MAX_NEEDED) 641 netif_stop_subqueue(ndev, tx_ring->index); 642 643 return NETDEV_TX_OK; 644 645 drop_packet_err: 646 dev_kfree_skb_any(skb); 647 return NETDEV_TX_OK; 648 } 649 650 netdev_tx_t enetc_xmit(struct sk_buff *skb, struct net_device *ndev) 651 { 652 struct enetc_ndev_priv *priv = netdev_priv(ndev); 653 u8 udp, msgtype, twostep; 654 u16 offset1, offset2; 655 656 /* Mark tx timestamp type on skb->cb[0] if requires */ 657 if ((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && 658 (priv->active_offloads & ENETC_F_TX_TSTAMP_MASK)) { 659 skb->cb[0] = priv->active_offloads & ENETC_F_TX_TSTAMP_MASK; 660 } else { 661 skb->cb[0] = 0; 662 } 663 664 /* Fall back to two-step timestamp if not one-step Sync packet */ 665 if (skb->cb[0] & ENETC_F_TX_ONESTEP_SYNC_TSTAMP) { 666 if (enetc_ptp_parse(skb, &udp, &msgtype, &twostep, 667 &offset1, &offset2) || 668 msgtype != PTP_MSGTYPE_SYNC || twostep != 0) 669 skb->cb[0] = ENETC_F_TX_TSTAMP; 670 } 671 672 return enetc_start_xmit(skb, ndev); 673 } 674 EXPORT_SYMBOL_GPL(enetc_xmit); 675 676 static irqreturn_t enetc_msix(int irq, void *data) 677 { 678 struct enetc_int_vector *v = data; 679 int i; 680 681 enetc_lock_mdio(); 682 683 /* disable interrupts */ 684 enetc_wr_reg_hot(v->rbier, 0); 685 enetc_wr_reg_hot(v->ricr1, v->rx_ictt); 686 687 for_each_set_bit(i, &v->tx_rings_map, ENETC_MAX_NUM_TXQS) 688 enetc_wr_reg_hot(v->tbier_base + ENETC_BDR_OFF(i), 0); 689 690 enetc_unlock_mdio(); 691 692 napi_schedule(&v->napi); 693 694 return IRQ_HANDLED; 695 } 696 697 static void enetc_rx_dim_work(struct work_struct *w) 698 { 699 struct dim *dim = container_of(w, struct dim, work); 700 struct dim_cq_moder moder = 701 net_dim_get_rx_moderation(dim->mode, dim->profile_ix); 702 struct enetc_int_vector *v = 703 container_of(dim, struct enetc_int_vector, rx_dim); 704 struct enetc_ndev_priv *priv = netdev_priv(v->rx_ring.ndev); 705 706 v->rx_ictt = enetc_usecs_to_cycles(moder.usec, priv->sysclk_freq); 707 dim->state = DIM_START_MEASURE; 708 } 709 710 static void enetc_rx_net_dim(struct enetc_int_vector *v) 711 { 712 struct dim_sample dim_sample = {}; 713 714 v->comp_cnt++; 715 716 if (!v->rx_napi_work) 717 return; 718 719 dim_update_sample(v->comp_cnt, 720 v->rx_ring.stats.packets, 721 v->rx_ring.stats.bytes, 722 &dim_sample); 723 net_dim(&v->rx_dim, &dim_sample); 724 } 725 726 static int enetc_bd_ready_count(struct enetc_bdr *tx_ring, int ci) 727 { 728 int pi = enetc_rd_reg_hot(tx_ring->tcir) & ENETC_TBCIR_IDX_MASK; 729 730 return pi >= ci ? pi - ci : tx_ring->bd_count - ci + pi; 731 } 732 733 static bool enetc_page_reusable(struct page *page) 734 { 735 return (!page_is_pfmemalloc(page) && page_ref_count(page) == 1); 736 } 737 738 static void enetc_reuse_page(struct enetc_bdr *rx_ring, 739 struct enetc_rx_swbd *old) 740 { 741 struct enetc_rx_swbd *new; 742 743 new = &rx_ring->rx_swbd[rx_ring->next_to_alloc]; 744 745 /* next buf that may reuse a page */ 746 enetc_bdr_idx_inc(rx_ring, &rx_ring->next_to_alloc); 747 748 /* copy page reference */ 749 *new = *old; 750 } 751 752 static void enetc_get_tx_tstamp(struct enetc_hw *hw, union enetc_tx_bd *txbd, 753 u64 *tstamp) 754 { 755 u32 lo, hi, tstamp_lo; 756 757 lo = enetc_rd_hot(hw, ENETC_SICTR0); 758 hi = enetc_rd_hot(hw, ENETC_SICTR1); 759 tstamp_lo = le32_to_cpu(txbd->wb.tstamp); 760 if (lo <= tstamp_lo) 761 hi -= 1; 762 *tstamp = (u64)hi << 32 | tstamp_lo; 763 } 764 765 static void enetc_tstamp_tx(struct sk_buff *skb, u64 tstamp) 766 { 767 struct skb_shared_hwtstamps shhwtstamps; 768 769 if (skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS) { 770 memset(&shhwtstamps, 0, sizeof(shhwtstamps)); 771 shhwtstamps.hwtstamp = ns_to_ktime(tstamp); 772 skb_txtime_consumed(skb); 773 skb_tstamp_tx(skb, &shhwtstamps); 774 } 775 } 776 777 static void enetc_recycle_xdp_tx_buff(struct enetc_bdr *tx_ring, 778 struct enetc_tx_swbd *tx_swbd) 779 { 780 struct enetc_ndev_priv *priv = netdev_priv(tx_ring->ndev); 781 struct enetc_rx_swbd rx_swbd = { 782 .dma = tx_swbd->dma, 783 .page = tx_swbd->page, 784 .page_offset = tx_swbd->page_offset, 785 .dir = tx_swbd->dir, 786 .len = tx_swbd->len, 787 }; 788 struct enetc_bdr *rx_ring; 789 790 rx_ring = enetc_rx_ring_from_xdp_tx_ring(priv, tx_ring); 791 792 if (likely(enetc_swbd_unused(rx_ring))) { 793 enetc_reuse_page(rx_ring, &rx_swbd); 794 795 /* sync for use by the device */ 796 dma_sync_single_range_for_device(rx_ring->dev, rx_swbd.dma, 797 rx_swbd.page_offset, 798 ENETC_RXB_DMA_SIZE_XDP, 799 rx_swbd.dir); 800 801 rx_ring->stats.recycles++; 802 } else { 803 /* RX ring is already full, we need to unmap and free the 804 * page, since there's nothing useful we can do with it. 805 */ 806 rx_ring->stats.recycle_failures++; 807 808 dma_unmap_page(rx_ring->dev, rx_swbd.dma, PAGE_SIZE, 809 rx_swbd.dir); 810 __free_page(rx_swbd.page); 811 } 812 813 rx_ring->xdp.xdp_tx_in_flight--; 814 } 815 816 static bool enetc_clean_tx_ring(struct enetc_bdr *tx_ring, int napi_budget) 817 { 818 int tx_frm_cnt = 0, tx_byte_cnt = 0, tx_win_drop = 0; 819 struct net_device *ndev = tx_ring->ndev; 820 struct enetc_ndev_priv *priv = netdev_priv(ndev); 821 struct enetc_tx_swbd *tx_swbd; 822 int i, bds_to_clean; 823 bool do_twostep_tstamp; 824 u64 tstamp = 0; 825 826 i = tx_ring->next_to_clean; 827 tx_swbd = &tx_ring->tx_swbd[i]; 828 829 bds_to_clean = enetc_bd_ready_count(tx_ring, i); 830 831 do_twostep_tstamp = false; 832 833 while (bds_to_clean && tx_frm_cnt < ENETC_DEFAULT_TX_WORK) { 834 struct xdp_frame *xdp_frame = enetc_tx_swbd_get_xdp_frame(tx_swbd); 835 struct sk_buff *skb = enetc_tx_swbd_get_skb(tx_swbd); 836 bool is_eof = tx_swbd->is_eof; 837 838 if (unlikely(tx_swbd->check_wb)) { 839 union enetc_tx_bd *txbd = ENETC_TXBD(*tx_ring, i); 840 841 if (txbd->flags & ENETC_TXBD_FLAGS_W && 842 tx_swbd->do_twostep_tstamp) { 843 enetc_get_tx_tstamp(&priv->si->hw, txbd, 844 &tstamp); 845 do_twostep_tstamp = true; 846 } 847 848 if (tx_swbd->qbv_en && 849 txbd->wb.status & ENETC_TXBD_STATS_WIN) 850 tx_win_drop++; 851 } 852 853 if (tx_swbd->is_xdp_tx) 854 enetc_recycle_xdp_tx_buff(tx_ring, tx_swbd); 855 else if (likely(tx_swbd->dma)) 856 enetc_unmap_tx_buff(tx_ring, tx_swbd); 857 858 if (xdp_frame) { 859 xdp_return_frame(xdp_frame); 860 } else if (skb) { 861 if (unlikely(skb->cb[0] & ENETC_F_TX_ONESTEP_SYNC_TSTAMP)) { 862 /* Start work to release lock for next one-step 863 * timestamping packet. And send one skb in 864 * tx_skbs queue if has. 865 */ 866 schedule_work(&priv->tx_onestep_tstamp); 867 } else if (unlikely(do_twostep_tstamp)) { 868 enetc_tstamp_tx(skb, tstamp); 869 do_twostep_tstamp = false; 870 } 871 napi_consume_skb(skb, napi_budget); 872 } 873 874 tx_byte_cnt += tx_swbd->len; 875 /* Scrub the swbd here so we don't have to do that 876 * when we reuse it during xmit 877 */ 878 memset(tx_swbd, 0, sizeof(*tx_swbd)); 879 880 bds_to_clean--; 881 tx_swbd++; 882 i++; 883 if (unlikely(i == tx_ring->bd_count)) { 884 i = 0; 885 tx_swbd = tx_ring->tx_swbd; 886 } 887 888 /* BD iteration loop end */ 889 if (is_eof) { 890 tx_frm_cnt++; 891 /* re-arm interrupt source */ 892 enetc_wr_reg_hot(tx_ring->idr, BIT(tx_ring->index) | 893 BIT(16 + tx_ring->index)); 894 } 895 896 if (unlikely(!bds_to_clean)) 897 bds_to_clean = enetc_bd_ready_count(tx_ring, i); 898 } 899 900 tx_ring->next_to_clean = i; 901 tx_ring->stats.packets += tx_frm_cnt; 902 tx_ring->stats.bytes += tx_byte_cnt; 903 tx_ring->stats.win_drop += tx_win_drop; 904 905 if (unlikely(tx_frm_cnt && netif_carrier_ok(ndev) && 906 __netif_subqueue_stopped(ndev, tx_ring->index) && 907 !test_bit(ENETC_TX_DOWN, &priv->flags) && 908 (enetc_bd_unused(tx_ring) >= ENETC_TXBDS_MAX_NEEDED))) { 909 netif_wake_subqueue(ndev, tx_ring->index); 910 } 911 912 return tx_frm_cnt != ENETC_DEFAULT_TX_WORK; 913 } 914 915 static bool enetc_new_page(struct enetc_bdr *rx_ring, 916 struct enetc_rx_swbd *rx_swbd) 917 { 918 bool xdp = !!(rx_ring->xdp.prog); 919 struct page *page; 920 dma_addr_t addr; 921 922 page = dev_alloc_page(); 923 if (unlikely(!page)) 924 return false; 925 926 /* For XDP_TX, we forgo dma_unmap -> dma_map */ 927 rx_swbd->dir = xdp ? DMA_BIDIRECTIONAL : DMA_FROM_DEVICE; 928 929 addr = dma_map_page(rx_ring->dev, page, 0, PAGE_SIZE, rx_swbd->dir); 930 if (unlikely(dma_mapping_error(rx_ring->dev, addr))) { 931 __free_page(page); 932 933 return false; 934 } 935 936 rx_swbd->dma = addr; 937 rx_swbd->page = page; 938 rx_swbd->page_offset = rx_ring->buffer_offset; 939 940 return true; 941 } 942 943 static int enetc_refill_rx_ring(struct enetc_bdr *rx_ring, const int buff_cnt) 944 { 945 struct enetc_rx_swbd *rx_swbd; 946 union enetc_rx_bd *rxbd; 947 int i, j; 948 949 i = rx_ring->next_to_use; 950 rx_swbd = &rx_ring->rx_swbd[i]; 951 rxbd = enetc_rxbd(rx_ring, i); 952 953 for (j = 0; j < buff_cnt; j++) { 954 /* try reuse page */ 955 if (unlikely(!rx_swbd->page)) { 956 if (unlikely(!enetc_new_page(rx_ring, rx_swbd))) { 957 rx_ring->stats.rx_alloc_errs++; 958 break; 959 } 960 } 961 962 /* update RxBD */ 963 rxbd->w.addr = cpu_to_le64(rx_swbd->dma + 964 rx_swbd->page_offset); 965 /* clear 'R" as well */ 966 rxbd->r.lstatus = 0; 967 968 enetc_rxbd_next(rx_ring, &rxbd, &i); 969 rx_swbd = &rx_ring->rx_swbd[i]; 970 } 971 972 if (likely(j)) { 973 rx_ring->next_to_alloc = i; /* keep track from page reuse */ 974 rx_ring->next_to_use = i; 975 976 /* update ENETC's consumer index */ 977 enetc_wr_reg_hot(rx_ring->rcir, rx_ring->next_to_use); 978 } 979 980 return j; 981 } 982 983 static void enetc_get_rx_tstamp(struct net_device *ndev, 984 union enetc_rx_bd *rxbd, 985 struct sk_buff *skb) 986 { 987 struct skb_shared_hwtstamps *shhwtstamps = skb_hwtstamps(skb); 988 struct enetc_ndev_priv *priv = netdev_priv(ndev); 989 struct enetc_hw *hw = &priv->si->hw; 990 u32 lo, hi, tstamp_lo; 991 u64 tstamp; 992 993 if (le16_to_cpu(rxbd->r.flags) & ENETC_RXBD_FLAG_TSTMP) { 994 lo = enetc_rd_reg_hot(hw->reg + ENETC_SICTR0); 995 hi = enetc_rd_reg_hot(hw->reg + ENETC_SICTR1); 996 rxbd = enetc_rxbd_ext(rxbd); 997 tstamp_lo = le32_to_cpu(rxbd->ext.tstamp); 998 if (lo <= tstamp_lo) 999 hi -= 1; 1000 1001 tstamp = (u64)hi << 32 | tstamp_lo; 1002 memset(shhwtstamps, 0, sizeof(*shhwtstamps)); 1003 shhwtstamps->hwtstamp = ns_to_ktime(tstamp); 1004 } 1005 } 1006 1007 static void enetc_get_offloads(struct enetc_bdr *rx_ring, 1008 union enetc_rx_bd *rxbd, struct sk_buff *skb) 1009 { 1010 struct enetc_ndev_priv *priv = netdev_priv(rx_ring->ndev); 1011 1012 /* TODO: hashing */ 1013 if (rx_ring->ndev->features & NETIF_F_RXCSUM) { 1014 u16 inet_csum = le16_to_cpu(rxbd->r.inet_csum); 1015 1016 skb->csum = csum_unfold((__force __sum16)~htons(inet_csum)); 1017 skb->ip_summed = CHECKSUM_COMPLETE; 1018 } 1019 1020 if (le16_to_cpu(rxbd->r.flags) & ENETC_RXBD_FLAG_VLAN) { 1021 __be16 tpid = 0; 1022 1023 switch (le16_to_cpu(rxbd->r.flags) & ENETC_RXBD_FLAG_TPID) { 1024 case 0: 1025 tpid = htons(ETH_P_8021Q); 1026 break; 1027 case 1: 1028 tpid = htons(ETH_P_8021AD); 1029 break; 1030 case 2: 1031 tpid = htons(enetc_port_rd(&priv->si->hw, 1032 ENETC_PCVLANR1)); 1033 break; 1034 case 3: 1035 tpid = htons(enetc_port_rd(&priv->si->hw, 1036 ENETC_PCVLANR2)); 1037 break; 1038 default: 1039 break; 1040 } 1041 1042 __vlan_hwaccel_put_tag(skb, tpid, le16_to_cpu(rxbd->r.vlan_opt)); 1043 } 1044 1045 if (IS_ENABLED(CONFIG_FSL_ENETC_PTP_CLOCK) && 1046 (priv->active_offloads & ENETC_F_RX_TSTAMP)) 1047 enetc_get_rx_tstamp(rx_ring->ndev, rxbd, skb); 1048 } 1049 1050 /* This gets called during the non-XDP NAPI poll cycle as well as on XDP_PASS, 1051 * so it needs to work with both DMA_FROM_DEVICE as well as DMA_BIDIRECTIONAL 1052 * mapped buffers. 1053 */ 1054 static struct enetc_rx_swbd *enetc_get_rx_buff(struct enetc_bdr *rx_ring, 1055 int i, u16 size) 1056 { 1057 struct enetc_rx_swbd *rx_swbd = &rx_ring->rx_swbd[i]; 1058 1059 dma_sync_single_range_for_cpu(rx_ring->dev, rx_swbd->dma, 1060 rx_swbd->page_offset, 1061 size, rx_swbd->dir); 1062 return rx_swbd; 1063 } 1064 1065 /* Reuse the current page without performing half-page buffer flipping */ 1066 static void enetc_put_rx_buff(struct enetc_bdr *rx_ring, 1067 struct enetc_rx_swbd *rx_swbd) 1068 { 1069 size_t buffer_size = ENETC_RXB_TRUESIZE - rx_ring->buffer_offset; 1070 1071 enetc_reuse_page(rx_ring, rx_swbd); 1072 1073 dma_sync_single_range_for_device(rx_ring->dev, rx_swbd->dma, 1074 rx_swbd->page_offset, 1075 buffer_size, rx_swbd->dir); 1076 1077 rx_swbd->page = NULL; 1078 } 1079 1080 /* Reuse the current page by performing half-page buffer flipping */ 1081 static void enetc_flip_rx_buff(struct enetc_bdr *rx_ring, 1082 struct enetc_rx_swbd *rx_swbd) 1083 { 1084 if (likely(enetc_page_reusable(rx_swbd->page))) { 1085 rx_swbd->page_offset ^= ENETC_RXB_TRUESIZE; 1086 page_ref_inc(rx_swbd->page); 1087 1088 enetc_put_rx_buff(rx_ring, rx_swbd); 1089 } else { 1090 dma_unmap_page(rx_ring->dev, rx_swbd->dma, PAGE_SIZE, 1091 rx_swbd->dir); 1092 rx_swbd->page = NULL; 1093 } 1094 } 1095 1096 static struct sk_buff *enetc_map_rx_buff_to_skb(struct enetc_bdr *rx_ring, 1097 int i, u16 size) 1098 { 1099 struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size); 1100 struct sk_buff *skb; 1101 void *ba; 1102 1103 ba = page_address(rx_swbd->page) + rx_swbd->page_offset; 1104 skb = build_skb(ba - rx_ring->buffer_offset, ENETC_RXB_TRUESIZE); 1105 if (unlikely(!skb)) { 1106 rx_ring->stats.rx_alloc_errs++; 1107 return NULL; 1108 } 1109 1110 skb_reserve(skb, rx_ring->buffer_offset); 1111 __skb_put(skb, size); 1112 1113 enetc_flip_rx_buff(rx_ring, rx_swbd); 1114 1115 return skb; 1116 } 1117 1118 static void enetc_add_rx_buff_to_skb(struct enetc_bdr *rx_ring, int i, 1119 u16 size, struct sk_buff *skb) 1120 { 1121 struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size); 1122 1123 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_swbd->page, 1124 rx_swbd->page_offset, size, ENETC_RXB_TRUESIZE); 1125 1126 enetc_flip_rx_buff(rx_ring, rx_swbd); 1127 } 1128 1129 static bool enetc_check_bd_errors_and_consume(struct enetc_bdr *rx_ring, 1130 u32 bd_status, 1131 union enetc_rx_bd **rxbd, int *i) 1132 { 1133 if (likely(!(bd_status & ENETC_RXBD_LSTATUS(ENETC_RXBD_ERR_MASK)))) 1134 return false; 1135 1136 enetc_put_rx_buff(rx_ring, &rx_ring->rx_swbd[*i]); 1137 enetc_rxbd_next(rx_ring, rxbd, i); 1138 1139 while (!(bd_status & ENETC_RXBD_LSTATUS_F)) { 1140 dma_rmb(); 1141 bd_status = le32_to_cpu((*rxbd)->r.lstatus); 1142 1143 enetc_put_rx_buff(rx_ring, &rx_ring->rx_swbd[*i]); 1144 enetc_rxbd_next(rx_ring, rxbd, i); 1145 } 1146 1147 rx_ring->ndev->stats.rx_dropped++; 1148 rx_ring->ndev->stats.rx_errors++; 1149 1150 return true; 1151 } 1152 1153 static struct sk_buff *enetc_build_skb(struct enetc_bdr *rx_ring, 1154 u32 bd_status, union enetc_rx_bd **rxbd, 1155 int *i, int *cleaned_cnt, int buffer_size) 1156 { 1157 struct sk_buff *skb; 1158 u16 size; 1159 1160 size = le16_to_cpu((*rxbd)->r.buf_len); 1161 skb = enetc_map_rx_buff_to_skb(rx_ring, *i, size); 1162 if (!skb) 1163 return NULL; 1164 1165 enetc_get_offloads(rx_ring, *rxbd, skb); 1166 1167 (*cleaned_cnt)++; 1168 1169 enetc_rxbd_next(rx_ring, rxbd, i); 1170 1171 /* not last BD in frame? */ 1172 while (!(bd_status & ENETC_RXBD_LSTATUS_F)) { 1173 bd_status = le32_to_cpu((*rxbd)->r.lstatus); 1174 size = buffer_size; 1175 1176 if (bd_status & ENETC_RXBD_LSTATUS_F) { 1177 dma_rmb(); 1178 size = le16_to_cpu((*rxbd)->r.buf_len); 1179 } 1180 1181 enetc_add_rx_buff_to_skb(rx_ring, *i, size, skb); 1182 1183 (*cleaned_cnt)++; 1184 1185 enetc_rxbd_next(rx_ring, rxbd, i); 1186 } 1187 1188 skb_record_rx_queue(skb, rx_ring->index); 1189 skb->protocol = eth_type_trans(skb, rx_ring->ndev); 1190 1191 return skb; 1192 } 1193 1194 #define ENETC_RXBD_BUNDLE 16 /* # of BDs to update at once */ 1195 1196 static int enetc_clean_rx_ring(struct enetc_bdr *rx_ring, 1197 struct napi_struct *napi, int work_limit) 1198 { 1199 int rx_frm_cnt = 0, rx_byte_cnt = 0; 1200 int cleaned_cnt, i; 1201 1202 cleaned_cnt = enetc_bd_unused(rx_ring); 1203 /* next descriptor to process */ 1204 i = rx_ring->next_to_clean; 1205 1206 while (likely(rx_frm_cnt < work_limit)) { 1207 union enetc_rx_bd *rxbd; 1208 struct sk_buff *skb; 1209 u32 bd_status; 1210 1211 if (cleaned_cnt >= ENETC_RXBD_BUNDLE) 1212 cleaned_cnt -= enetc_refill_rx_ring(rx_ring, 1213 cleaned_cnt); 1214 1215 rxbd = enetc_rxbd(rx_ring, i); 1216 bd_status = le32_to_cpu(rxbd->r.lstatus); 1217 if (!bd_status) 1218 break; 1219 1220 enetc_wr_reg_hot(rx_ring->idr, BIT(rx_ring->index)); 1221 dma_rmb(); /* for reading other rxbd fields */ 1222 1223 if (enetc_check_bd_errors_and_consume(rx_ring, bd_status, 1224 &rxbd, &i)) 1225 break; 1226 1227 skb = enetc_build_skb(rx_ring, bd_status, &rxbd, &i, 1228 &cleaned_cnt, ENETC_RXB_DMA_SIZE); 1229 if (!skb) 1230 break; 1231 1232 /* When set, the outer VLAN header is extracted and reported 1233 * in the receive buffer descriptor. So rx_byte_cnt should 1234 * add the length of the extracted VLAN header. 1235 */ 1236 if (bd_status & ENETC_RXBD_FLAG_VLAN) 1237 rx_byte_cnt += VLAN_HLEN; 1238 rx_byte_cnt += skb->len + ETH_HLEN; 1239 rx_frm_cnt++; 1240 1241 napi_gro_receive(napi, skb); 1242 } 1243 1244 rx_ring->next_to_clean = i; 1245 1246 rx_ring->stats.packets += rx_frm_cnt; 1247 rx_ring->stats.bytes += rx_byte_cnt; 1248 1249 return rx_frm_cnt; 1250 } 1251 1252 static void enetc_xdp_map_tx_buff(struct enetc_bdr *tx_ring, int i, 1253 struct enetc_tx_swbd *tx_swbd, 1254 int frm_len) 1255 { 1256 union enetc_tx_bd *txbd = ENETC_TXBD(*tx_ring, i); 1257 1258 prefetchw(txbd); 1259 1260 enetc_clear_tx_bd(txbd); 1261 txbd->addr = cpu_to_le64(tx_swbd->dma + tx_swbd->page_offset); 1262 txbd->buf_len = cpu_to_le16(tx_swbd->len); 1263 txbd->frm_len = cpu_to_le16(frm_len); 1264 1265 memcpy(&tx_ring->tx_swbd[i], tx_swbd, sizeof(*tx_swbd)); 1266 } 1267 1268 /* Puts in the TX ring one XDP frame, mapped as an array of TX software buffer 1269 * descriptors. 1270 */ 1271 static bool enetc_xdp_tx(struct enetc_bdr *tx_ring, 1272 struct enetc_tx_swbd *xdp_tx_arr, int num_tx_swbd) 1273 { 1274 struct enetc_tx_swbd *tmp_tx_swbd = xdp_tx_arr; 1275 int i, k, frm_len = tmp_tx_swbd->len; 1276 1277 if (unlikely(enetc_bd_unused(tx_ring) < ENETC_TXBDS_NEEDED(num_tx_swbd))) 1278 return false; 1279 1280 while (unlikely(!tmp_tx_swbd->is_eof)) { 1281 tmp_tx_swbd++; 1282 frm_len += tmp_tx_swbd->len; 1283 } 1284 1285 i = tx_ring->next_to_use; 1286 1287 for (k = 0; k < num_tx_swbd; k++) { 1288 struct enetc_tx_swbd *xdp_tx_swbd = &xdp_tx_arr[k]; 1289 1290 enetc_xdp_map_tx_buff(tx_ring, i, xdp_tx_swbd, frm_len); 1291 1292 /* last BD needs 'F' bit set */ 1293 if (xdp_tx_swbd->is_eof) { 1294 union enetc_tx_bd *txbd = ENETC_TXBD(*tx_ring, i); 1295 1296 txbd->flags = ENETC_TXBD_FLAGS_F; 1297 } 1298 1299 enetc_bdr_idx_inc(tx_ring, &i); 1300 } 1301 1302 tx_ring->next_to_use = i; 1303 1304 return true; 1305 } 1306 1307 static int enetc_xdp_frame_to_xdp_tx_swbd(struct enetc_bdr *tx_ring, 1308 struct enetc_tx_swbd *xdp_tx_arr, 1309 struct xdp_frame *xdp_frame) 1310 { 1311 struct enetc_tx_swbd *xdp_tx_swbd = &xdp_tx_arr[0]; 1312 struct skb_shared_info *shinfo; 1313 void *data = xdp_frame->data; 1314 int len = xdp_frame->len; 1315 skb_frag_t *frag; 1316 dma_addr_t dma; 1317 unsigned int f; 1318 int n = 0; 1319 1320 dma = dma_map_single(tx_ring->dev, data, len, DMA_TO_DEVICE); 1321 if (unlikely(dma_mapping_error(tx_ring->dev, dma))) { 1322 netdev_err(tx_ring->ndev, "DMA map error\n"); 1323 return -1; 1324 } 1325 1326 xdp_tx_swbd->dma = dma; 1327 xdp_tx_swbd->dir = DMA_TO_DEVICE; 1328 xdp_tx_swbd->len = len; 1329 xdp_tx_swbd->is_xdp_redirect = true; 1330 xdp_tx_swbd->is_eof = false; 1331 xdp_tx_swbd->xdp_frame = NULL; 1332 1333 n++; 1334 1335 if (!xdp_frame_has_frags(xdp_frame)) 1336 goto out; 1337 1338 xdp_tx_swbd = &xdp_tx_arr[n]; 1339 1340 shinfo = xdp_get_shared_info_from_frame(xdp_frame); 1341 1342 for (f = 0, frag = &shinfo->frags[0]; f < shinfo->nr_frags; 1343 f++, frag++) { 1344 data = skb_frag_address(frag); 1345 len = skb_frag_size(frag); 1346 1347 dma = dma_map_single(tx_ring->dev, data, len, DMA_TO_DEVICE); 1348 if (unlikely(dma_mapping_error(tx_ring->dev, dma))) { 1349 /* Undo the DMA mapping for all fragments */ 1350 while (--n >= 0) 1351 enetc_unmap_tx_buff(tx_ring, &xdp_tx_arr[n]); 1352 1353 netdev_err(tx_ring->ndev, "DMA map error\n"); 1354 return -1; 1355 } 1356 1357 xdp_tx_swbd->dma = dma; 1358 xdp_tx_swbd->dir = DMA_TO_DEVICE; 1359 xdp_tx_swbd->len = len; 1360 xdp_tx_swbd->is_xdp_redirect = true; 1361 xdp_tx_swbd->is_eof = false; 1362 xdp_tx_swbd->xdp_frame = NULL; 1363 1364 n++; 1365 xdp_tx_swbd = &xdp_tx_arr[n]; 1366 } 1367 out: 1368 xdp_tx_arr[n - 1].is_eof = true; 1369 xdp_tx_arr[n - 1].xdp_frame = xdp_frame; 1370 1371 return n; 1372 } 1373 1374 int enetc_xdp_xmit(struct net_device *ndev, int num_frames, 1375 struct xdp_frame **frames, u32 flags) 1376 { 1377 struct enetc_tx_swbd xdp_redirect_arr[ENETC_MAX_SKB_FRAGS] = {0}; 1378 struct enetc_ndev_priv *priv = netdev_priv(ndev); 1379 struct enetc_bdr *tx_ring; 1380 int xdp_tx_bd_cnt, i, k; 1381 int xdp_tx_frm_cnt = 0; 1382 1383 if (unlikely(test_bit(ENETC_TX_DOWN, &priv->flags))) 1384 return -ENETDOWN; 1385 1386 enetc_lock_mdio(); 1387 1388 tx_ring = priv->xdp_tx_ring[smp_processor_id()]; 1389 1390 prefetchw(ENETC_TXBD(*tx_ring, tx_ring->next_to_use)); 1391 1392 for (k = 0; k < num_frames; k++) { 1393 xdp_tx_bd_cnt = enetc_xdp_frame_to_xdp_tx_swbd(tx_ring, 1394 xdp_redirect_arr, 1395 frames[k]); 1396 if (unlikely(xdp_tx_bd_cnt < 0)) 1397 break; 1398 1399 if (unlikely(!enetc_xdp_tx(tx_ring, xdp_redirect_arr, 1400 xdp_tx_bd_cnt))) { 1401 for (i = 0; i < xdp_tx_bd_cnt; i++) 1402 enetc_unmap_tx_buff(tx_ring, 1403 &xdp_redirect_arr[i]); 1404 tx_ring->stats.xdp_tx_drops++; 1405 break; 1406 } 1407 1408 xdp_tx_frm_cnt++; 1409 } 1410 1411 if (unlikely((flags & XDP_XMIT_FLUSH) || k != xdp_tx_frm_cnt)) 1412 enetc_update_tx_ring_tail(tx_ring); 1413 1414 tx_ring->stats.xdp_tx += xdp_tx_frm_cnt; 1415 1416 enetc_unlock_mdio(); 1417 1418 return xdp_tx_frm_cnt; 1419 } 1420 EXPORT_SYMBOL_GPL(enetc_xdp_xmit); 1421 1422 static void enetc_map_rx_buff_to_xdp(struct enetc_bdr *rx_ring, int i, 1423 struct xdp_buff *xdp_buff, u16 size) 1424 { 1425 struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size); 1426 void *hard_start = page_address(rx_swbd->page) + rx_swbd->page_offset; 1427 1428 /* To be used for XDP_TX */ 1429 rx_swbd->len = size; 1430 1431 xdp_prepare_buff(xdp_buff, hard_start - rx_ring->buffer_offset, 1432 rx_ring->buffer_offset, size, false); 1433 } 1434 1435 static void enetc_add_rx_buff_to_xdp(struct enetc_bdr *rx_ring, int i, 1436 u16 size, struct xdp_buff *xdp_buff) 1437 { 1438 struct skb_shared_info *shinfo = xdp_get_shared_info_from_buff(xdp_buff); 1439 struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size); 1440 skb_frag_t *frag; 1441 1442 /* To be used for XDP_TX */ 1443 rx_swbd->len = size; 1444 1445 if (!xdp_buff_has_frags(xdp_buff)) { 1446 xdp_buff_set_frags_flag(xdp_buff); 1447 shinfo->xdp_frags_size = size; 1448 shinfo->nr_frags = 0; 1449 } else { 1450 shinfo->xdp_frags_size += size; 1451 } 1452 1453 if (page_is_pfmemalloc(rx_swbd->page)) 1454 xdp_buff_set_frag_pfmemalloc(xdp_buff); 1455 1456 frag = &shinfo->frags[shinfo->nr_frags]; 1457 skb_frag_fill_page_desc(frag, rx_swbd->page, rx_swbd->page_offset, 1458 size); 1459 1460 shinfo->nr_frags++; 1461 } 1462 1463 static void enetc_build_xdp_buff(struct enetc_bdr *rx_ring, u32 bd_status, 1464 union enetc_rx_bd **rxbd, int *i, 1465 int *cleaned_cnt, struct xdp_buff *xdp_buff) 1466 { 1467 u16 size = le16_to_cpu((*rxbd)->r.buf_len); 1468 1469 xdp_init_buff(xdp_buff, ENETC_RXB_TRUESIZE, &rx_ring->xdp.rxq); 1470 1471 enetc_map_rx_buff_to_xdp(rx_ring, *i, xdp_buff, size); 1472 (*cleaned_cnt)++; 1473 enetc_rxbd_next(rx_ring, rxbd, i); 1474 1475 /* not last BD in frame? */ 1476 while (!(bd_status & ENETC_RXBD_LSTATUS_F)) { 1477 bd_status = le32_to_cpu((*rxbd)->r.lstatus); 1478 size = ENETC_RXB_DMA_SIZE_XDP; 1479 1480 if (bd_status & ENETC_RXBD_LSTATUS_F) { 1481 dma_rmb(); 1482 size = le16_to_cpu((*rxbd)->r.buf_len); 1483 } 1484 1485 enetc_add_rx_buff_to_xdp(rx_ring, *i, size, xdp_buff); 1486 (*cleaned_cnt)++; 1487 enetc_rxbd_next(rx_ring, rxbd, i); 1488 } 1489 } 1490 1491 /* Convert RX buffer descriptors to TX buffer descriptors. These will be 1492 * recycled back into the RX ring in enetc_clean_tx_ring. 1493 */ 1494 static int enetc_rx_swbd_to_xdp_tx_swbd(struct enetc_tx_swbd *xdp_tx_arr, 1495 struct enetc_bdr *rx_ring, 1496 int rx_ring_first, int rx_ring_last) 1497 { 1498 int n = 0; 1499 1500 for (; rx_ring_first != rx_ring_last; 1501 n++, enetc_bdr_idx_inc(rx_ring, &rx_ring_first)) { 1502 struct enetc_rx_swbd *rx_swbd = &rx_ring->rx_swbd[rx_ring_first]; 1503 struct enetc_tx_swbd *tx_swbd = &xdp_tx_arr[n]; 1504 1505 /* No need to dma_map, we already have DMA_BIDIRECTIONAL */ 1506 tx_swbd->dma = rx_swbd->dma; 1507 tx_swbd->dir = rx_swbd->dir; 1508 tx_swbd->page = rx_swbd->page; 1509 tx_swbd->page_offset = rx_swbd->page_offset; 1510 tx_swbd->len = rx_swbd->len; 1511 tx_swbd->is_dma_page = true; 1512 tx_swbd->is_xdp_tx = true; 1513 tx_swbd->is_eof = false; 1514 } 1515 1516 /* We rely on caller providing an rx_ring_last > rx_ring_first */ 1517 xdp_tx_arr[n - 1].is_eof = true; 1518 1519 return n; 1520 } 1521 1522 static void enetc_xdp_drop(struct enetc_bdr *rx_ring, int rx_ring_first, 1523 int rx_ring_last) 1524 { 1525 while (rx_ring_first != rx_ring_last) { 1526 enetc_put_rx_buff(rx_ring, 1527 &rx_ring->rx_swbd[rx_ring_first]); 1528 enetc_bdr_idx_inc(rx_ring, &rx_ring_first); 1529 } 1530 } 1531 1532 static int enetc_clean_rx_ring_xdp(struct enetc_bdr *rx_ring, 1533 struct napi_struct *napi, int work_limit, 1534 struct bpf_prog *prog) 1535 { 1536 int xdp_tx_bd_cnt, xdp_tx_frm_cnt = 0, xdp_redirect_frm_cnt = 0; 1537 struct enetc_tx_swbd xdp_tx_arr[ENETC_MAX_SKB_FRAGS] = {0}; 1538 struct enetc_ndev_priv *priv = netdev_priv(rx_ring->ndev); 1539 int rx_frm_cnt = 0, rx_byte_cnt = 0; 1540 struct enetc_bdr *tx_ring; 1541 int cleaned_cnt, i; 1542 u32 xdp_act; 1543 1544 cleaned_cnt = enetc_bd_unused(rx_ring); 1545 /* next descriptor to process */ 1546 i = rx_ring->next_to_clean; 1547 1548 while (likely(rx_frm_cnt < work_limit)) { 1549 union enetc_rx_bd *rxbd, *orig_rxbd; 1550 int orig_i, orig_cleaned_cnt; 1551 struct xdp_buff xdp_buff; 1552 struct sk_buff *skb; 1553 u32 bd_status; 1554 int err; 1555 1556 rxbd = enetc_rxbd(rx_ring, i); 1557 bd_status = le32_to_cpu(rxbd->r.lstatus); 1558 if (!bd_status) 1559 break; 1560 1561 enetc_wr_reg_hot(rx_ring->idr, BIT(rx_ring->index)); 1562 dma_rmb(); /* for reading other rxbd fields */ 1563 1564 if (enetc_check_bd_errors_and_consume(rx_ring, bd_status, 1565 &rxbd, &i)) 1566 break; 1567 1568 orig_rxbd = rxbd; 1569 orig_cleaned_cnt = cleaned_cnt; 1570 orig_i = i; 1571 1572 enetc_build_xdp_buff(rx_ring, bd_status, &rxbd, &i, 1573 &cleaned_cnt, &xdp_buff); 1574 1575 /* When set, the outer VLAN header is extracted and reported 1576 * in the receive buffer descriptor. So rx_byte_cnt should 1577 * add the length of the extracted VLAN header. 1578 */ 1579 if (bd_status & ENETC_RXBD_FLAG_VLAN) 1580 rx_byte_cnt += VLAN_HLEN; 1581 rx_byte_cnt += xdp_get_buff_len(&xdp_buff); 1582 1583 xdp_act = bpf_prog_run_xdp(prog, &xdp_buff); 1584 1585 switch (xdp_act) { 1586 default: 1587 bpf_warn_invalid_xdp_action(rx_ring->ndev, prog, xdp_act); 1588 fallthrough; 1589 case XDP_ABORTED: 1590 trace_xdp_exception(rx_ring->ndev, prog, xdp_act); 1591 fallthrough; 1592 case XDP_DROP: 1593 enetc_xdp_drop(rx_ring, orig_i, i); 1594 rx_ring->stats.xdp_drops++; 1595 break; 1596 case XDP_PASS: 1597 rxbd = orig_rxbd; 1598 cleaned_cnt = orig_cleaned_cnt; 1599 i = orig_i; 1600 1601 skb = enetc_build_skb(rx_ring, bd_status, &rxbd, 1602 &i, &cleaned_cnt, 1603 ENETC_RXB_DMA_SIZE_XDP); 1604 if (unlikely(!skb)) 1605 goto out; 1606 1607 napi_gro_receive(napi, skb); 1608 break; 1609 case XDP_TX: 1610 tx_ring = priv->xdp_tx_ring[rx_ring->index]; 1611 if (unlikely(test_bit(ENETC_TX_DOWN, &priv->flags))) { 1612 enetc_xdp_drop(rx_ring, orig_i, i); 1613 tx_ring->stats.xdp_tx_drops++; 1614 break; 1615 } 1616 1617 xdp_tx_bd_cnt = enetc_rx_swbd_to_xdp_tx_swbd(xdp_tx_arr, 1618 rx_ring, 1619 orig_i, i); 1620 1621 if (!enetc_xdp_tx(tx_ring, xdp_tx_arr, xdp_tx_bd_cnt)) { 1622 enetc_xdp_drop(rx_ring, orig_i, i); 1623 tx_ring->stats.xdp_tx_drops++; 1624 } else { 1625 tx_ring->stats.xdp_tx += xdp_tx_bd_cnt; 1626 rx_ring->xdp.xdp_tx_in_flight += xdp_tx_bd_cnt; 1627 xdp_tx_frm_cnt++; 1628 /* The XDP_TX enqueue was successful, so we 1629 * need to scrub the RX software BDs because 1630 * the ownership of the buffers no longer 1631 * belongs to the RX ring, and we must prevent 1632 * enetc_refill_rx_ring() from reusing 1633 * rx_swbd->page. 1634 */ 1635 while (orig_i != i) { 1636 rx_ring->rx_swbd[orig_i].page = NULL; 1637 enetc_bdr_idx_inc(rx_ring, &orig_i); 1638 } 1639 } 1640 break; 1641 case XDP_REDIRECT: 1642 err = xdp_do_redirect(rx_ring->ndev, &xdp_buff, prog); 1643 if (unlikely(err)) { 1644 enetc_xdp_drop(rx_ring, orig_i, i); 1645 rx_ring->stats.xdp_redirect_failures++; 1646 } else { 1647 while (orig_i != i) { 1648 enetc_flip_rx_buff(rx_ring, 1649 &rx_ring->rx_swbd[orig_i]); 1650 enetc_bdr_idx_inc(rx_ring, &orig_i); 1651 } 1652 xdp_redirect_frm_cnt++; 1653 rx_ring->stats.xdp_redirect++; 1654 } 1655 } 1656 1657 rx_frm_cnt++; 1658 } 1659 1660 out: 1661 rx_ring->next_to_clean = i; 1662 1663 rx_ring->stats.packets += rx_frm_cnt; 1664 rx_ring->stats.bytes += rx_byte_cnt; 1665 1666 if (xdp_redirect_frm_cnt) 1667 xdp_do_flush(); 1668 1669 if (xdp_tx_frm_cnt) 1670 enetc_update_tx_ring_tail(tx_ring); 1671 1672 if (cleaned_cnt > rx_ring->xdp.xdp_tx_in_flight) 1673 enetc_refill_rx_ring(rx_ring, enetc_bd_unused(rx_ring) - 1674 rx_ring->xdp.xdp_tx_in_flight); 1675 1676 return rx_frm_cnt; 1677 } 1678 1679 static int enetc_poll(struct napi_struct *napi, int budget) 1680 { 1681 struct enetc_int_vector 1682 *v = container_of(napi, struct enetc_int_vector, napi); 1683 struct enetc_bdr *rx_ring = &v->rx_ring; 1684 struct bpf_prog *prog; 1685 bool complete = true; 1686 int work_done; 1687 int i; 1688 1689 enetc_lock_mdio(); 1690 1691 for (i = 0; i < v->count_tx_rings; i++) 1692 if (!enetc_clean_tx_ring(&v->tx_ring[i], budget)) 1693 complete = false; 1694 1695 prog = rx_ring->xdp.prog; 1696 if (prog) 1697 work_done = enetc_clean_rx_ring_xdp(rx_ring, napi, budget, prog); 1698 else 1699 work_done = enetc_clean_rx_ring(rx_ring, napi, budget); 1700 if (work_done == budget) 1701 complete = false; 1702 if (work_done) 1703 v->rx_napi_work = true; 1704 1705 if (!complete) { 1706 enetc_unlock_mdio(); 1707 return budget; 1708 } 1709 1710 napi_complete_done(napi, work_done); 1711 1712 if (likely(v->rx_dim_en)) 1713 enetc_rx_net_dim(v); 1714 1715 v->rx_napi_work = false; 1716 1717 /* enable interrupts */ 1718 enetc_wr_reg_hot(v->rbier, ENETC_RBIER_RXTIE); 1719 1720 for_each_set_bit(i, &v->tx_rings_map, ENETC_MAX_NUM_TXQS) 1721 enetc_wr_reg_hot(v->tbier_base + ENETC_BDR_OFF(i), 1722 ENETC_TBIER_TXTIE); 1723 1724 enetc_unlock_mdio(); 1725 1726 return work_done; 1727 } 1728 1729 /* Probing and Init */ 1730 #define ENETC_MAX_RFS_SIZE 64 1731 void enetc_get_si_caps(struct enetc_si *si) 1732 { 1733 struct enetc_hw *hw = &si->hw; 1734 u32 val; 1735 1736 /* find out how many of various resources we have to work with */ 1737 val = enetc_rd(hw, ENETC_SICAPR0); 1738 si->num_rx_rings = (val >> 16) & 0xff; 1739 si->num_tx_rings = val & 0xff; 1740 1741 val = enetc_rd(hw, ENETC_SIPCAPR0); 1742 if (val & ENETC_SIPCAPR0_RFS) { 1743 val = enetc_rd(hw, ENETC_SIRFSCAPR); 1744 si->num_fs_entries = ENETC_SIRFSCAPR_GET_NUM_RFS(val); 1745 si->num_fs_entries = min(si->num_fs_entries, ENETC_MAX_RFS_SIZE); 1746 } else { 1747 /* ENETC which not supports RFS */ 1748 si->num_fs_entries = 0; 1749 } 1750 1751 si->num_rss = 0; 1752 val = enetc_rd(hw, ENETC_SIPCAPR0); 1753 if (val & ENETC_SIPCAPR0_RSS) { 1754 u32 rss; 1755 1756 rss = enetc_rd(hw, ENETC_SIRSSCAPR); 1757 si->num_rss = ENETC_SIRSSCAPR_GET_NUM_RSS(rss); 1758 } 1759 1760 if (val & ENETC_SIPCAPR0_QBV) 1761 si->hw_features |= ENETC_SI_F_QBV; 1762 1763 if (val & ENETC_SIPCAPR0_QBU) 1764 si->hw_features |= ENETC_SI_F_QBU; 1765 1766 if (val & ENETC_SIPCAPR0_PSFP) 1767 si->hw_features |= ENETC_SI_F_PSFP; 1768 } 1769 EXPORT_SYMBOL_GPL(enetc_get_si_caps); 1770 1771 static int enetc_dma_alloc_bdr(struct enetc_bdr_resource *res) 1772 { 1773 size_t bd_base_size = res->bd_count * res->bd_size; 1774 1775 res->bd_base = dma_alloc_coherent(res->dev, bd_base_size, 1776 &res->bd_dma_base, GFP_KERNEL); 1777 if (!res->bd_base) 1778 return -ENOMEM; 1779 1780 /* h/w requires 128B alignment */ 1781 if (!IS_ALIGNED(res->bd_dma_base, 128)) { 1782 dma_free_coherent(res->dev, bd_base_size, res->bd_base, 1783 res->bd_dma_base); 1784 return -EINVAL; 1785 } 1786 1787 return 0; 1788 } 1789 1790 static void enetc_dma_free_bdr(const struct enetc_bdr_resource *res) 1791 { 1792 size_t bd_base_size = res->bd_count * res->bd_size; 1793 1794 dma_free_coherent(res->dev, bd_base_size, res->bd_base, 1795 res->bd_dma_base); 1796 } 1797 1798 static int enetc_alloc_tx_resource(struct enetc_bdr_resource *res, 1799 struct device *dev, size_t bd_count) 1800 { 1801 int err; 1802 1803 res->dev = dev; 1804 res->bd_count = bd_count; 1805 res->bd_size = sizeof(union enetc_tx_bd); 1806 1807 res->tx_swbd = vcalloc(bd_count, sizeof(*res->tx_swbd)); 1808 if (!res->tx_swbd) 1809 return -ENOMEM; 1810 1811 err = enetc_dma_alloc_bdr(res); 1812 if (err) 1813 goto err_alloc_bdr; 1814 1815 res->tso_headers = dma_alloc_coherent(dev, bd_count * TSO_HEADER_SIZE, 1816 &res->tso_headers_dma, 1817 GFP_KERNEL); 1818 if (!res->tso_headers) { 1819 err = -ENOMEM; 1820 goto err_alloc_tso; 1821 } 1822 1823 return 0; 1824 1825 err_alloc_tso: 1826 enetc_dma_free_bdr(res); 1827 err_alloc_bdr: 1828 vfree(res->tx_swbd); 1829 res->tx_swbd = NULL; 1830 1831 return err; 1832 } 1833 1834 static void enetc_free_tx_resource(const struct enetc_bdr_resource *res) 1835 { 1836 dma_free_coherent(res->dev, res->bd_count * TSO_HEADER_SIZE, 1837 res->tso_headers, res->tso_headers_dma); 1838 enetc_dma_free_bdr(res); 1839 vfree(res->tx_swbd); 1840 } 1841 1842 static struct enetc_bdr_resource * 1843 enetc_alloc_tx_resources(struct enetc_ndev_priv *priv) 1844 { 1845 struct enetc_bdr_resource *tx_res; 1846 int i, err; 1847 1848 tx_res = kcalloc(priv->num_tx_rings, sizeof(*tx_res), GFP_KERNEL); 1849 if (!tx_res) 1850 return ERR_PTR(-ENOMEM); 1851 1852 for (i = 0; i < priv->num_tx_rings; i++) { 1853 struct enetc_bdr *tx_ring = priv->tx_ring[i]; 1854 1855 err = enetc_alloc_tx_resource(&tx_res[i], tx_ring->dev, 1856 tx_ring->bd_count); 1857 if (err) 1858 goto fail; 1859 } 1860 1861 return tx_res; 1862 1863 fail: 1864 while (i-- > 0) 1865 enetc_free_tx_resource(&tx_res[i]); 1866 1867 kfree(tx_res); 1868 1869 return ERR_PTR(err); 1870 } 1871 1872 static void enetc_free_tx_resources(const struct enetc_bdr_resource *tx_res, 1873 size_t num_resources) 1874 { 1875 size_t i; 1876 1877 for (i = 0; i < num_resources; i++) 1878 enetc_free_tx_resource(&tx_res[i]); 1879 1880 kfree(tx_res); 1881 } 1882 1883 static int enetc_alloc_rx_resource(struct enetc_bdr_resource *res, 1884 struct device *dev, size_t bd_count, 1885 bool extended) 1886 { 1887 int err; 1888 1889 res->dev = dev; 1890 res->bd_count = bd_count; 1891 res->bd_size = sizeof(union enetc_rx_bd); 1892 if (extended) 1893 res->bd_size *= 2; 1894 1895 res->rx_swbd = vcalloc(bd_count, sizeof(struct enetc_rx_swbd)); 1896 if (!res->rx_swbd) 1897 return -ENOMEM; 1898 1899 err = enetc_dma_alloc_bdr(res); 1900 if (err) { 1901 vfree(res->rx_swbd); 1902 return err; 1903 } 1904 1905 return 0; 1906 } 1907 1908 static void enetc_free_rx_resource(const struct enetc_bdr_resource *res) 1909 { 1910 enetc_dma_free_bdr(res); 1911 vfree(res->rx_swbd); 1912 } 1913 1914 static struct enetc_bdr_resource * 1915 enetc_alloc_rx_resources(struct enetc_ndev_priv *priv, bool extended) 1916 { 1917 struct enetc_bdr_resource *rx_res; 1918 int i, err; 1919 1920 rx_res = kcalloc(priv->num_rx_rings, sizeof(*rx_res), GFP_KERNEL); 1921 if (!rx_res) 1922 return ERR_PTR(-ENOMEM); 1923 1924 for (i = 0; i < priv->num_rx_rings; i++) { 1925 struct enetc_bdr *rx_ring = priv->rx_ring[i]; 1926 1927 err = enetc_alloc_rx_resource(&rx_res[i], rx_ring->dev, 1928 rx_ring->bd_count, extended); 1929 if (err) 1930 goto fail; 1931 } 1932 1933 return rx_res; 1934 1935 fail: 1936 while (i-- > 0) 1937 enetc_free_rx_resource(&rx_res[i]); 1938 1939 kfree(rx_res); 1940 1941 return ERR_PTR(err); 1942 } 1943 1944 static void enetc_free_rx_resources(const struct enetc_bdr_resource *rx_res, 1945 size_t num_resources) 1946 { 1947 size_t i; 1948 1949 for (i = 0; i < num_resources; i++) 1950 enetc_free_rx_resource(&rx_res[i]); 1951 1952 kfree(rx_res); 1953 } 1954 1955 static void enetc_assign_tx_resource(struct enetc_bdr *tx_ring, 1956 const struct enetc_bdr_resource *res) 1957 { 1958 tx_ring->bd_base = res ? res->bd_base : NULL; 1959 tx_ring->bd_dma_base = res ? res->bd_dma_base : 0; 1960 tx_ring->tx_swbd = res ? res->tx_swbd : NULL; 1961 tx_ring->tso_headers = res ? res->tso_headers : NULL; 1962 tx_ring->tso_headers_dma = res ? res->tso_headers_dma : 0; 1963 } 1964 1965 static void enetc_assign_rx_resource(struct enetc_bdr *rx_ring, 1966 const struct enetc_bdr_resource *res) 1967 { 1968 rx_ring->bd_base = res ? res->bd_base : NULL; 1969 rx_ring->bd_dma_base = res ? res->bd_dma_base : 0; 1970 rx_ring->rx_swbd = res ? res->rx_swbd : NULL; 1971 } 1972 1973 static void enetc_assign_tx_resources(struct enetc_ndev_priv *priv, 1974 const struct enetc_bdr_resource *res) 1975 { 1976 int i; 1977 1978 if (priv->tx_res) 1979 enetc_free_tx_resources(priv->tx_res, priv->num_tx_rings); 1980 1981 for (i = 0; i < priv->num_tx_rings; i++) { 1982 enetc_assign_tx_resource(priv->tx_ring[i], 1983 res ? &res[i] : NULL); 1984 } 1985 1986 priv->tx_res = res; 1987 } 1988 1989 static void enetc_assign_rx_resources(struct enetc_ndev_priv *priv, 1990 const struct enetc_bdr_resource *res) 1991 { 1992 int i; 1993 1994 if (priv->rx_res) 1995 enetc_free_rx_resources(priv->rx_res, priv->num_rx_rings); 1996 1997 for (i = 0; i < priv->num_rx_rings; i++) { 1998 enetc_assign_rx_resource(priv->rx_ring[i], 1999 res ? &res[i] : NULL); 2000 } 2001 2002 priv->rx_res = res; 2003 } 2004 2005 static void enetc_free_tx_ring(struct enetc_bdr *tx_ring) 2006 { 2007 int i; 2008 2009 for (i = 0; i < tx_ring->bd_count; i++) { 2010 struct enetc_tx_swbd *tx_swbd = &tx_ring->tx_swbd[i]; 2011 2012 enetc_free_tx_frame(tx_ring, tx_swbd); 2013 } 2014 } 2015 2016 static void enetc_free_rx_ring(struct enetc_bdr *rx_ring) 2017 { 2018 int i; 2019 2020 for (i = 0; i < rx_ring->bd_count; i++) { 2021 struct enetc_rx_swbd *rx_swbd = &rx_ring->rx_swbd[i]; 2022 2023 if (!rx_swbd->page) 2024 continue; 2025 2026 dma_unmap_page(rx_ring->dev, rx_swbd->dma, PAGE_SIZE, 2027 rx_swbd->dir); 2028 __free_page(rx_swbd->page); 2029 rx_swbd->page = NULL; 2030 } 2031 } 2032 2033 static void enetc_free_rxtx_rings(struct enetc_ndev_priv *priv) 2034 { 2035 int i; 2036 2037 for (i = 0; i < priv->num_rx_rings; i++) 2038 enetc_free_rx_ring(priv->rx_ring[i]); 2039 2040 for (i = 0; i < priv->num_tx_rings; i++) 2041 enetc_free_tx_ring(priv->tx_ring[i]); 2042 } 2043 2044 static int enetc_setup_default_rss_table(struct enetc_si *si, int num_groups) 2045 { 2046 int *rss_table; 2047 int i; 2048 2049 rss_table = kmalloc_array(si->num_rss, sizeof(*rss_table), GFP_KERNEL); 2050 if (!rss_table) 2051 return -ENOMEM; 2052 2053 /* Set up RSS table defaults */ 2054 for (i = 0; i < si->num_rss; i++) 2055 rss_table[i] = i % num_groups; 2056 2057 enetc_set_rss_table(si, rss_table, si->num_rss); 2058 2059 kfree(rss_table); 2060 2061 return 0; 2062 } 2063 2064 int enetc_configure_si(struct enetc_ndev_priv *priv) 2065 { 2066 struct enetc_si *si = priv->si; 2067 struct enetc_hw *hw = &si->hw; 2068 int err; 2069 2070 /* set SI cache attributes */ 2071 enetc_wr(hw, ENETC_SICAR0, 2072 ENETC_SICAR_RD_COHERENT | ENETC_SICAR_WR_COHERENT); 2073 enetc_wr(hw, ENETC_SICAR1, ENETC_SICAR_MSI); 2074 /* enable SI */ 2075 enetc_wr(hw, ENETC_SIMR, ENETC_SIMR_EN); 2076 2077 /* TODO: RSS support for i.MX95 will be supported later, and the 2078 * is_enetc_rev1() condition will be removed 2079 */ 2080 if (si->num_rss && is_enetc_rev1(si)) { 2081 err = enetc_setup_default_rss_table(si, priv->num_rx_rings); 2082 if (err) 2083 return err; 2084 } 2085 2086 return 0; 2087 } 2088 EXPORT_SYMBOL_GPL(enetc_configure_si); 2089 2090 void enetc_init_si_rings_params(struct enetc_ndev_priv *priv) 2091 { 2092 struct enetc_si *si = priv->si; 2093 int cpus = num_online_cpus(); 2094 2095 priv->tx_bd_count = ENETC_TX_RING_DEFAULT_SIZE; 2096 priv->rx_bd_count = ENETC_RX_RING_DEFAULT_SIZE; 2097 2098 /* Enable all available TX rings in order to configure as many 2099 * priorities as possible, when needed. 2100 * TODO: Make # of TX rings run-time configurable 2101 */ 2102 priv->num_rx_rings = min_t(int, cpus, si->num_rx_rings); 2103 priv->num_tx_rings = si->num_tx_rings; 2104 priv->bdr_int_num = priv->num_rx_rings; 2105 priv->ic_mode = ENETC_IC_RX_ADAPTIVE | ENETC_IC_TX_MANUAL; 2106 priv->tx_ictt = enetc_usecs_to_cycles(600, priv->sysclk_freq); 2107 } 2108 EXPORT_SYMBOL_GPL(enetc_init_si_rings_params); 2109 2110 int enetc_alloc_si_resources(struct enetc_ndev_priv *priv) 2111 { 2112 struct enetc_si *si = priv->si; 2113 2114 priv->cls_rules = kcalloc(si->num_fs_entries, sizeof(*priv->cls_rules), 2115 GFP_KERNEL); 2116 if (!priv->cls_rules) 2117 return -ENOMEM; 2118 2119 return 0; 2120 } 2121 EXPORT_SYMBOL_GPL(enetc_alloc_si_resources); 2122 2123 void enetc_free_si_resources(struct enetc_ndev_priv *priv) 2124 { 2125 kfree(priv->cls_rules); 2126 } 2127 EXPORT_SYMBOL_GPL(enetc_free_si_resources); 2128 2129 static void enetc_setup_txbdr(struct enetc_hw *hw, struct enetc_bdr *tx_ring) 2130 { 2131 int idx = tx_ring->index; 2132 u32 tbmr; 2133 2134 enetc_txbdr_wr(hw, idx, ENETC_TBBAR0, 2135 lower_32_bits(tx_ring->bd_dma_base)); 2136 2137 enetc_txbdr_wr(hw, idx, ENETC_TBBAR1, 2138 upper_32_bits(tx_ring->bd_dma_base)); 2139 2140 WARN_ON(!IS_ALIGNED(tx_ring->bd_count, 64)); /* multiple of 64 */ 2141 enetc_txbdr_wr(hw, idx, ENETC_TBLENR, 2142 ENETC_RTBLENR_LEN(tx_ring->bd_count)); 2143 2144 /* clearing PI/CI registers for Tx not supported, adjust sw indexes */ 2145 tx_ring->next_to_use = enetc_txbdr_rd(hw, idx, ENETC_TBPIR); 2146 tx_ring->next_to_clean = enetc_txbdr_rd(hw, idx, ENETC_TBCIR); 2147 2148 /* enable Tx ints by setting pkt thr to 1 */ 2149 enetc_txbdr_wr(hw, idx, ENETC_TBICR0, ENETC_TBICR0_ICEN | 0x1); 2150 2151 tbmr = ENETC_TBMR_SET_PRIO(tx_ring->prio); 2152 if (tx_ring->ndev->features & NETIF_F_HW_VLAN_CTAG_TX) 2153 tbmr |= ENETC_TBMR_VIH; 2154 2155 /* enable ring */ 2156 enetc_txbdr_wr(hw, idx, ENETC_TBMR, tbmr); 2157 2158 tx_ring->tpir = hw->reg + ENETC_BDR(TX, idx, ENETC_TBPIR); 2159 tx_ring->tcir = hw->reg + ENETC_BDR(TX, idx, ENETC_TBCIR); 2160 tx_ring->idr = hw->reg + ENETC_SITXIDR; 2161 } 2162 2163 static void enetc_setup_rxbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring, 2164 bool extended) 2165 { 2166 int idx = rx_ring->index; 2167 u32 rbmr = 0; 2168 2169 enetc_rxbdr_wr(hw, idx, ENETC_RBBAR0, 2170 lower_32_bits(rx_ring->bd_dma_base)); 2171 2172 enetc_rxbdr_wr(hw, idx, ENETC_RBBAR1, 2173 upper_32_bits(rx_ring->bd_dma_base)); 2174 2175 WARN_ON(!IS_ALIGNED(rx_ring->bd_count, 64)); /* multiple of 64 */ 2176 enetc_rxbdr_wr(hw, idx, ENETC_RBLENR, 2177 ENETC_RTBLENR_LEN(rx_ring->bd_count)); 2178 2179 if (rx_ring->xdp.prog) 2180 enetc_rxbdr_wr(hw, idx, ENETC_RBBSR, ENETC_RXB_DMA_SIZE_XDP); 2181 else 2182 enetc_rxbdr_wr(hw, idx, ENETC_RBBSR, ENETC_RXB_DMA_SIZE); 2183 2184 /* Also prepare the consumer index in case page allocation never 2185 * succeeds. In that case, hardware will never advance producer index 2186 * to match consumer index, and will drop all frames. 2187 */ 2188 enetc_rxbdr_wr(hw, idx, ENETC_RBPIR, 0); 2189 enetc_rxbdr_wr(hw, idx, ENETC_RBCIR, 1); 2190 2191 /* enable Rx ints by setting pkt thr to 1 */ 2192 enetc_rxbdr_wr(hw, idx, ENETC_RBICR0, ENETC_RBICR0_ICEN | 0x1); 2193 2194 rx_ring->ext_en = extended; 2195 if (rx_ring->ext_en) 2196 rbmr |= ENETC_RBMR_BDS; 2197 2198 if (rx_ring->ndev->features & NETIF_F_HW_VLAN_CTAG_RX) 2199 rbmr |= ENETC_RBMR_VTE; 2200 2201 rx_ring->rcir = hw->reg + ENETC_BDR(RX, idx, ENETC_RBCIR); 2202 rx_ring->idr = hw->reg + ENETC_SIRXIDR; 2203 2204 rx_ring->next_to_clean = 0; 2205 rx_ring->next_to_use = 0; 2206 rx_ring->next_to_alloc = 0; 2207 2208 enetc_lock_mdio(); 2209 enetc_refill_rx_ring(rx_ring, enetc_bd_unused(rx_ring)); 2210 enetc_unlock_mdio(); 2211 2212 enetc_rxbdr_wr(hw, idx, ENETC_RBMR, rbmr); 2213 } 2214 2215 static void enetc_setup_bdrs(struct enetc_ndev_priv *priv, bool extended) 2216 { 2217 struct enetc_hw *hw = &priv->si->hw; 2218 int i; 2219 2220 for (i = 0; i < priv->num_tx_rings; i++) 2221 enetc_setup_txbdr(hw, priv->tx_ring[i]); 2222 2223 for (i = 0; i < priv->num_rx_rings; i++) 2224 enetc_setup_rxbdr(hw, priv->rx_ring[i], extended); 2225 } 2226 2227 static void enetc_enable_txbdr(struct enetc_hw *hw, struct enetc_bdr *tx_ring) 2228 { 2229 int idx = tx_ring->index; 2230 u32 tbmr; 2231 2232 tbmr = enetc_txbdr_rd(hw, idx, ENETC_TBMR); 2233 tbmr |= ENETC_TBMR_EN; 2234 enetc_txbdr_wr(hw, idx, ENETC_TBMR, tbmr); 2235 } 2236 2237 static void enetc_enable_rxbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring) 2238 { 2239 int idx = rx_ring->index; 2240 u32 rbmr; 2241 2242 rbmr = enetc_rxbdr_rd(hw, idx, ENETC_RBMR); 2243 rbmr |= ENETC_RBMR_EN; 2244 enetc_rxbdr_wr(hw, idx, ENETC_RBMR, rbmr); 2245 } 2246 2247 static void enetc_enable_rx_bdrs(struct enetc_ndev_priv *priv) 2248 { 2249 struct enetc_hw *hw = &priv->si->hw; 2250 int i; 2251 2252 for (i = 0; i < priv->num_rx_rings; i++) 2253 enetc_enable_rxbdr(hw, priv->rx_ring[i]); 2254 } 2255 2256 static void enetc_enable_tx_bdrs(struct enetc_ndev_priv *priv) 2257 { 2258 struct enetc_hw *hw = &priv->si->hw; 2259 int i; 2260 2261 for (i = 0; i < priv->num_tx_rings; i++) 2262 enetc_enable_txbdr(hw, priv->tx_ring[i]); 2263 } 2264 2265 static void enetc_disable_rxbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring) 2266 { 2267 int idx = rx_ring->index; 2268 2269 /* disable EN bit on ring */ 2270 enetc_rxbdr_wr(hw, idx, ENETC_RBMR, 0); 2271 } 2272 2273 static void enetc_disable_txbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring) 2274 { 2275 int idx = rx_ring->index; 2276 2277 /* disable EN bit on ring */ 2278 enetc_txbdr_wr(hw, idx, ENETC_TBMR, 0); 2279 } 2280 2281 static void enetc_disable_rx_bdrs(struct enetc_ndev_priv *priv) 2282 { 2283 struct enetc_hw *hw = &priv->si->hw; 2284 int i; 2285 2286 for (i = 0; i < priv->num_rx_rings; i++) 2287 enetc_disable_rxbdr(hw, priv->rx_ring[i]); 2288 } 2289 2290 static void enetc_disable_tx_bdrs(struct enetc_ndev_priv *priv) 2291 { 2292 struct enetc_hw *hw = &priv->si->hw; 2293 int i; 2294 2295 for (i = 0; i < priv->num_tx_rings; i++) 2296 enetc_disable_txbdr(hw, priv->tx_ring[i]); 2297 } 2298 2299 static void enetc_wait_txbdr(struct enetc_hw *hw, struct enetc_bdr *tx_ring) 2300 { 2301 int delay = 8, timeout = 100; 2302 int idx = tx_ring->index; 2303 2304 /* wait for busy to clear */ 2305 while (delay < timeout && 2306 enetc_txbdr_rd(hw, idx, ENETC_TBSR) & ENETC_TBSR_BUSY) { 2307 msleep(delay); 2308 delay *= 2; 2309 } 2310 2311 if (delay >= timeout) 2312 netdev_warn(tx_ring->ndev, "timeout for tx ring #%d clear\n", 2313 idx); 2314 } 2315 2316 static void enetc_wait_bdrs(struct enetc_ndev_priv *priv) 2317 { 2318 struct enetc_hw *hw = &priv->si->hw; 2319 int i; 2320 2321 for (i = 0; i < priv->num_tx_rings; i++) 2322 enetc_wait_txbdr(hw, priv->tx_ring[i]); 2323 } 2324 2325 static int enetc_setup_irqs(struct enetc_ndev_priv *priv) 2326 { 2327 struct pci_dev *pdev = priv->si->pdev; 2328 struct enetc_hw *hw = &priv->si->hw; 2329 int i, j, err; 2330 2331 for (i = 0; i < priv->bdr_int_num; i++) { 2332 int irq = pci_irq_vector(pdev, ENETC_BDR_INT_BASE_IDX + i); 2333 struct enetc_int_vector *v = priv->int_vector[i]; 2334 int entry = ENETC_BDR_INT_BASE_IDX + i; 2335 2336 snprintf(v->name, sizeof(v->name), "%s-rxtx%d", 2337 priv->ndev->name, i); 2338 err = request_irq(irq, enetc_msix, IRQF_NO_AUTOEN, v->name, v); 2339 if (err) { 2340 dev_err(priv->dev, "request_irq() failed!\n"); 2341 goto irq_err; 2342 } 2343 2344 v->tbier_base = hw->reg + ENETC_BDR(TX, 0, ENETC_TBIER); 2345 v->rbier = hw->reg + ENETC_BDR(RX, i, ENETC_RBIER); 2346 v->ricr1 = hw->reg + ENETC_BDR(RX, i, ENETC_RBICR1); 2347 2348 enetc_wr(hw, ENETC_SIMSIRRV(i), entry); 2349 2350 for (j = 0; j < v->count_tx_rings; j++) { 2351 int idx = v->tx_ring[j].index; 2352 2353 enetc_wr(hw, ENETC_SIMSITRV(idx), entry); 2354 } 2355 irq_set_affinity_hint(irq, get_cpu_mask(i % num_online_cpus())); 2356 } 2357 2358 return 0; 2359 2360 irq_err: 2361 while (i--) { 2362 int irq = pci_irq_vector(pdev, ENETC_BDR_INT_BASE_IDX + i); 2363 2364 irq_set_affinity_hint(irq, NULL); 2365 free_irq(irq, priv->int_vector[i]); 2366 } 2367 2368 return err; 2369 } 2370 2371 static void enetc_free_irqs(struct enetc_ndev_priv *priv) 2372 { 2373 struct pci_dev *pdev = priv->si->pdev; 2374 int i; 2375 2376 for (i = 0; i < priv->bdr_int_num; i++) { 2377 int irq = pci_irq_vector(pdev, ENETC_BDR_INT_BASE_IDX + i); 2378 2379 irq_set_affinity_hint(irq, NULL); 2380 free_irq(irq, priv->int_vector[i]); 2381 } 2382 } 2383 2384 static void enetc_setup_interrupts(struct enetc_ndev_priv *priv) 2385 { 2386 struct enetc_hw *hw = &priv->si->hw; 2387 u32 icpt, ictt; 2388 int i; 2389 2390 /* enable Tx & Rx event indication */ 2391 if (priv->ic_mode & 2392 (ENETC_IC_RX_MANUAL | ENETC_IC_RX_ADAPTIVE)) { 2393 icpt = ENETC_RBICR0_SET_ICPT(ENETC_RXIC_PKTTHR); 2394 /* init to non-0 minimum, will be adjusted later */ 2395 ictt = 0x1; 2396 } else { 2397 icpt = 0x1; /* enable Rx ints by setting pkt thr to 1 */ 2398 ictt = 0; 2399 } 2400 2401 for (i = 0; i < priv->num_rx_rings; i++) { 2402 enetc_rxbdr_wr(hw, i, ENETC_RBICR1, ictt); 2403 enetc_rxbdr_wr(hw, i, ENETC_RBICR0, ENETC_RBICR0_ICEN | icpt); 2404 enetc_rxbdr_wr(hw, i, ENETC_RBIER, ENETC_RBIER_RXTIE); 2405 } 2406 2407 if (priv->ic_mode & ENETC_IC_TX_MANUAL) 2408 icpt = ENETC_TBICR0_SET_ICPT(ENETC_TXIC_PKTTHR); 2409 else 2410 icpt = 0x1; /* enable Tx ints by setting pkt thr to 1 */ 2411 2412 for (i = 0; i < priv->num_tx_rings; i++) { 2413 enetc_txbdr_wr(hw, i, ENETC_TBICR1, priv->tx_ictt); 2414 enetc_txbdr_wr(hw, i, ENETC_TBICR0, ENETC_TBICR0_ICEN | icpt); 2415 enetc_txbdr_wr(hw, i, ENETC_TBIER, ENETC_TBIER_TXTIE); 2416 } 2417 } 2418 2419 static void enetc_clear_interrupts(struct enetc_ndev_priv *priv) 2420 { 2421 struct enetc_hw *hw = &priv->si->hw; 2422 int i; 2423 2424 for (i = 0; i < priv->num_tx_rings; i++) 2425 enetc_txbdr_wr(hw, i, ENETC_TBIER, 0); 2426 2427 for (i = 0; i < priv->num_rx_rings; i++) 2428 enetc_rxbdr_wr(hw, i, ENETC_RBIER, 0); 2429 } 2430 2431 static int enetc_phylink_connect(struct net_device *ndev) 2432 { 2433 struct enetc_ndev_priv *priv = netdev_priv(ndev); 2434 struct ethtool_keee edata; 2435 int err; 2436 2437 if (!priv->phylink) { 2438 /* phy-less mode */ 2439 netif_carrier_on(ndev); 2440 return 0; 2441 } 2442 2443 err = phylink_of_phy_connect(priv->phylink, priv->dev->of_node, 0); 2444 if (err) { 2445 dev_err(&ndev->dev, "could not attach to PHY\n"); 2446 return err; 2447 } 2448 2449 /* disable EEE autoneg, until ENETC driver supports it */ 2450 memset(&edata, 0, sizeof(struct ethtool_keee)); 2451 phylink_ethtool_set_eee(priv->phylink, &edata); 2452 2453 phylink_start(priv->phylink); 2454 2455 return 0; 2456 } 2457 2458 static void enetc_tx_onestep_tstamp(struct work_struct *work) 2459 { 2460 struct enetc_ndev_priv *priv; 2461 struct sk_buff *skb; 2462 2463 priv = container_of(work, struct enetc_ndev_priv, tx_onestep_tstamp); 2464 2465 netif_tx_lock_bh(priv->ndev); 2466 2467 clear_bit_unlock(ENETC_TX_ONESTEP_TSTAMP_IN_PROGRESS, &priv->flags); 2468 skb = skb_dequeue(&priv->tx_skbs); 2469 if (skb) 2470 enetc_start_xmit(skb, priv->ndev); 2471 2472 netif_tx_unlock_bh(priv->ndev); 2473 } 2474 2475 static void enetc_tx_onestep_tstamp_init(struct enetc_ndev_priv *priv) 2476 { 2477 INIT_WORK(&priv->tx_onestep_tstamp, enetc_tx_onestep_tstamp); 2478 skb_queue_head_init(&priv->tx_skbs); 2479 } 2480 2481 void enetc_start(struct net_device *ndev) 2482 { 2483 struct enetc_ndev_priv *priv = netdev_priv(ndev); 2484 int i; 2485 2486 enetc_setup_interrupts(priv); 2487 2488 for (i = 0; i < priv->bdr_int_num; i++) { 2489 int irq = pci_irq_vector(priv->si->pdev, 2490 ENETC_BDR_INT_BASE_IDX + i); 2491 2492 napi_enable(&priv->int_vector[i]->napi); 2493 enable_irq(irq); 2494 } 2495 2496 enetc_enable_tx_bdrs(priv); 2497 2498 enetc_enable_rx_bdrs(priv); 2499 2500 netif_tx_start_all_queues(ndev); 2501 2502 clear_bit(ENETC_TX_DOWN, &priv->flags); 2503 } 2504 EXPORT_SYMBOL_GPL(enetc_start); 2505 2506 int enetc_open(struct net_device *ndev) 2507 { 2508 struct enetc_ndev_priv *priv = netdev_priv(ndev); 2509 struct enetc_bdr_resource *tx_res, *rx_res; 2510 bool extended; 2511 int err; 2512 2513 extended = !!(priv->active_offloads & ENETC_F_RX_TSTAMP); 2514 2515 err = clk_prepare_enable(priv->ref_clk); 2516 if (err) 2517 return err; 2518 2519 err = enetc_setup_irqs(priv); 2520 if (err) 2521 goto err_setup_irqs; 2522 2523 err = enetc_phylink_connect(ndev); 2524 if (err) 2525 goto err_phy_connect; 2526 2527 tx_res = enetc_alloc_tx_resources(priv); 2528 if (IS_ERR(tx_res)) { 2529 err = PTR_ERR(tx_res); 2530 goto err_alloc_tx; 2531 } 2532 2533 rx_res = enetc_alloc_rx_resources(priv, extended); 2534 if (IS_ERR(rx_res)) { 2535 err = PTR_ERR(rx_res); 2536 goto err_alloc_rx; 2537 } 2538 2539 enetc_tx_onestep_tstamp_init(priv); 2540 enetc_assign_tx_resources(priv, tx_res); 2541 enetc_assign_rx_resources(priv, rx_res); 2542 enetc_setup_bdrs(priv, extended); 2543 enetc_start(ndev); 2544 2545 return 0; 2546 2547 err_alloc_rx: 2548 enetc_free_tx_resources(tx_res, priv->num_tx_rings); 2549 err_alloc_tx: 2550 if (priv->phylink) 2551 phylink_disconnect_phy(priv->phylink); 2552 err_phy_connect: 2553 enetc_free_irqs(priv); 2554 err_setup_irqs: 2555 clk_disable_unprepare(priv->ref_clk); 2556 2557 return err; 2558 } 2559 EXPORT_SYMBOL_GPL(enetc_open); 2560 2561 void enetc_stop(struct net_device *ndev) 2562 { 2563 struct enetc_ndev_priv *priv = netdev_priv(ndev); 2564 int i; 2565 2566 set_bit(ENETC_TX_DOWN, &priv->flags); 2567 2568 netif_tx_stop_all_queues(ndev); 2569 2570 enetc_disable_rx_bdrs(priv); 2571 2572 enetc_wait_bdrs(priv); 2573 2574 enetc_disable_tx_bdrs(priv); 2575 2576 for (i = 0; i < priv->bdr_int_num; i++) { 2577 int irq = pci_irq_vector(priv->si->pdev, 2578 ENETC_BDR_INT_BASE_IDX + i); 2579 2580 disable_irq(irq); 2581 napi_synchronize(&priv->int_vector[i]->napi); 2582 napi_disable(&priv->int_vector[i]->napi); 2583 } 2584 2585 enetc_clear_interrupts(priv); 2586 } 2587 EXPORT_SYMBOL_GPL(enetc_stop); 2588 2589 int enetc_close(struct net_device *ndev) 2590 { 2591 struct enetc_ndev_priv *priv = netdev_priv(ndev); 2592 2593 enetc_stop(ndev); 2594 2595 if (priv->phylink) { 2596 phylink_stop(priv->phylink); 2597 phylink_disconnect_phy(priv->phylink); 2598 } else { 2599 netif_carrier_off(ndev); 2600 } 2601 2602 enetc_free_rxtx_rings(priv); 2603 2604 /* Avoids dangling pointers and also frees old resources */ 2605 enetc_assign_rx_resources(priv, NULL); 2606 enetc_assign_tx_resources(priv, NULL); 2607 2608 enetc_free_irqs(priv); 2609 clk_disable_unprepare(priv->ref_clk); 2610 2611 return 0; 2612 } 2613 EXPORT_SYMBOL_GPL(enetc_close); 2614 2615 static int enetc_reconfigure(struct enetc_ndev_priv *priv, bool extended, 2616 int (*cb)(struct enetc_ndev_priv *priv, void *ctx), 2617 void *ctx) 2618 { 2619 struct enetc_bdr_resource *tx_res, *rx_res; 2620 int err; 2621 2622 ASSERT_RTNL(); 2623 2624 /* If the interface is down, run the callback right away, 2625 * without reconfiguration. 2626 */ 2627 if (!netif_running(priv->ndev)) { 2628 if (cb) { 2629 err = cb(priv, ctx); 2630 if (err) 2631 return err; 2632 } 2633 2634 return 0; 2635 } 2636 2637 tx_res = enetc_alloc_tx_resources(priv); 2638 if (IS_ERR(tx_res)) { 2639 err = PTR_ERR(tx_res); 2640 goto out; 2641 } 2642 2643 rx_res = enetc_alloc_rx_resources(priv, extended); 2644 if (IS_ERR(rx_res)) { 2645 err = PTR_ERR(rx_res); 2646 goto out_free_tx_res; 2647 } 2648 2649 enetc_stop(priv->ndev); 2650 enetc_free_rxtx_rings(priv); 2651 2652 /* Interface is down, run optional callback now */ 2653 if (cb) { 2654 err = cb(priv, ctx); 2655 if (err) 2656 goto out_restart; 2657 } 2658 2659 enetc_assign_tx_resources(priv, tx_res); 2660 enetc_assign_rx_resources(priv, rx_res); 2661 enetc_setup_bdrs(priv, extended); 2662 enetc_start(priv->ndev); 2663 2664 return 0; 2665 2666 out_restart: 2667 enetc_setup_bdrs(priv, extended); 2668 enetc_start(priv->ndev); 2669 enetc_free_rx_resources(rx_res, priv->num_rx_rings); 2670 out_free_tx_res: 2671 enetc_free_tx_resources(tx_res, priv->num_tx_rings); 2672 out: 2673 return err; 2674 } 2675 2676 static void enetc_debug_tx_ring_prios(struct enetc_ndev_priv *priv) 2677 { 2678 int i; 2679 2680 for (i = 0; i < priv->num_tx_rings; i++) 2681 netdev_dbg(priv->ndev, "TX ring %d prio %d\n", i, 2682 priv->tx_ring[i]->prio); 2683 } 2684 2685 void enetc_reset_tc_mqprio(struct net_device *ndev) 2686 { 2687 struct enetc_ndev_priv *priv = netdev_priv(ndev); 2688 struct enetc_hw *hw = &priv->si->hw; 2689 struct enetc_bdr *tx_ring; 2690 int num_stack_tx_queues; 2691 int i; 2692 2693 num_stack_tx_queues = enetc_num_stack_tx_queues(priv); 2694 2695 netdev_reset_tc(ndev); 2696 netif_set_real_num_tx_queues(ndev, num_stack_tx_queues); 2697 priv->min_num_stack_tx_queues = num_possible_cpus(); 2698 2699 /* Reset all ring priorities to 0 */ 2700 for (i = 0; i < priv->num_tx_rings; i++) { 2701 tx_ring = priv->tx_ring[i]; 2702 tx_ring->prio = 0; 2703 enetc_set_bdr_prio(hw, tx_ring->index, tx_ring->prio); 2704 } 2705 2706 enetc_debug_tx_ring_prios(priv); 2707 2708 enetc_change_preemptible_tcs(priv, 0); 2709 } 2710 EXPORT_SYMBOL_GPL(enetc_reset_tc_mqprio); 2711 2712 int enetc_setup_tc_mqprio(struct net_device *ndev, void *type_data) 2713 { 2714 struct tc_mqprio_qopt_offload *mqprio = type_data; 2715 struct enetc_ndev_priv *priv = netdev_priv(ndev); 2716 struct tc_mqprio_qopt *qopt = &mqprio->qopt; 2717 struct enetc_hw *hw = &priv->si->hw; 2718 int num_stack_tx_queues = 0; 2719 struct enetc_bdr *tx_ring; 2720 u8 num_tc = qopt->num_tc; 2721 int offset, count; 2722 int err, tc, q; 2723 2724 if (!num_tc) { 2725 enetc_reset_tc_mqprio(ndev); 2726 return 0; 2727 } 2728 2729 err = netdev_set_num_tc(ndev, num_tc); 2730 if (err) 2731 return err; 2732 2733 for (tc = 0; tc < num_tc; tc++) { 2734 offset = qopt->offset[tc]; 2735 count = qopt->count[tc]; 2736 num_stack_tx_queues += count; 2737 2738 err = netdev_set_tc_queue(ndev, tc, count, offset); 2739 if (err) 2740 goto err_reset_tc; 2741 2742 for (q = offset; q < offset + count; q++) { 2743 tx_ring = priv->tx_ring[q]; 2744 /* The prio_tc_map is skb_tx_hash()'s way of selecting 2745 * between TX queues based on skb->priority. As such, 2746 * there's nothing to offload based on it. 2747 * Make the mqprio "traffic class" be the priority of 2748 * this ring group, and leave the Tx IPV to traffic 2749 * class mapping as its default mapping value of 1:1. 2750 */ 2751 tx_ring->prio = tc; 2752 enetc_set_bdr_prio(hw, tx_ring->index, tx_ring->prio); 2753 } 2754 } 2755 2756 err = netif_set_real_num_tx_queues(ndev, num_stack_tx_queues); 2757 if (err) 2758 goto err_reset_tc; 2759 2760 priv->min_num_stack_tx_queues = num_stack_tx_queues; 2761 2762 enetc_debug_tx_ring_prios(priv); 2763 2764 enetc_change_preemptible_tcs(priv, mqprio->preemptible_tcs); 2765 2766 return 0; 2767 2768 err_reset_tc: 2769 enetc_reset_tc_mqprio(ndev); 2770 return err; 2771 } 2772 EXPORT_SYMBOL_GPL(enetc_setup_tc_mqprio); 2773 2774 static int enetc_reconfigure_xdp_cb(struct enetc_ndev_priv *priv, void *ctx) 2775 { 2776 struct bpf_prog *old_prog, *prog = ctx; 2777 int num_stack_tx_queues; 2778 int err, i; 2779 2780 old_prog = xchg(&priv->xdp_prog, prog); 2781 2782 num_stack_tx_queues = enetc_num_stack_tx_queues(priv); 2783 err = netif_set_real_num_tx_queues(priv->ndev, num_stack_tx_queues); 2784 if (err) { 2785 xchg(&priv->xdp_prog, old_prog); 2786 return err; 2787 } 2788 2789 if (old_prog) 2790 bpf_prog_put(old_prog); 2791 2792 for (i = 0; i < priv->num_rx_rings; i++) { 2793 struct enetc_bdr *rx_ring = priv->rx_ring[i]; 2794 2795 rx_ring->xdp.prog = prog; 2796 2797 if (prog) 2798 rx_ring->buffer_offset = XDP_PACKET_HEADROOM; 2799 else 2800 rx_ring->buffer_offset = ENETC_RXB_PAD; 2801 } 2802 2803 return 0; 2804 } 2805 2806 static int enetc_setup_xdp_prog(struct net_device *ndev, struct bpf_prog *prog, 2807 struct netlink_ext_ack *extack) 2808 { 2809 int num_xdp_tx_queues = prog ? num_possible_cpus() : 0; 2810 struct enetc_ndev_priv *priv = netdev_priv(ndev); 2811 bool extended; 2812 2813 if (priv->min_num_stack_tx_queues + num_xdp_tx_queues > 2814 priv->num_tx_rings) { 2815 NL_SET_ERR_MSG_FMT_MOD(extack, 2816 "Reserving %d XDP TXQs leaves under %d for stack (total %d)", 2817 num_xdp_tx_queues, 2818 priv->min_num_stack_tx_queues, 2819 priv->num_tx_rings); 2820 return -EBUSY; 2821 } 2822 2823 extended = !!(priv->active_offloads & ENETC_F_RX_TSTAMP); 2824 2825 /* The buffer layout is changing, so we need to drain the old 2826 * RX buffers and seed new ones. 2827 */ 2828 return enetc_reconfigure(priv, extended, enetc_reconfigure_xdp_cb, prog); 2829 } 2830 2831 int enetc_setup_bpf(struct net_device *ndev, struct netdev_bpf *bpf) 2832 { 2833 switch (bpf->command) { 2834 case XDP_SETUP_PROG: 2835 return enetc_setup_xdp_prog(ndev, bpf->prog, bpf->extack); 2836 default: 2837 return -EINVAL; 2838 } 2839 2840 return 0; 2841 } 2842 EXPORT_SYMBOL_GPL(enetc_setup_bpf); 2843 2844 struct net_device_stats *enetc_get_stats(struct net_device *ndev) 2845 { 2846 struct enetc_ndev_priv *priv = netdev_priv(ndev); 2847 struct net_device_stats *stats = &ndev->stats; 2848 unsigned long packets = 0, bytes = 0; 2849 unsigned long tx_dropped = 0; 2850 int i; 2851 2852 for (i = 0; i < priv->num_rx_rings; i++) { 2853 packets += priv->rx_ring[i]->stats.packets; 2854 bytes += priv->rx_ring[i]->stats.bytes; 2855 } 2856 2857 stats->rx_packets = packets; 2858 stats->rx_bytes = bytes; 2859 bytes = 0; 2860 packets = 0; 2861 2862 for (i = 0; i < priv->num_tx_rings; i++) { 2863 packets += priv->tx_ring[i]->stats.packets; 2864 bytes += priv->tx_ring[i]->stats.bytes; 2865 tx_dropped += priv->tx_ring[i]->stats.win_drop; 2866 } 2867 2868 stats->tx_packets = packets; 2869 stats->tx_bytes = bytes; 2870 stats->tx_dropped = tx_dropped; 2871 2872 return stats; 2873 } 2874 EXPORT_SYMBOL_GPL(enetc_get_stats); 2875 2876 static int enetc_set_rss(struct net_device *ndev, int en) 2877 { 2878 struct enetc_ndev_priv *priv = netdev_priv(ndev); 2879 struct enetc_hw *hw = &priv->si->hw; 2880 u32 reg; 2881 2882 enetc_wr(hw, ENETC_SIRBGCR, priv->num_rx_rings); 2883 2884 reg = enetc_rd(hw, ENETC_SIMR); 2885 reg &= ~ENETC_SIMR_RSSE; 2886 reg |= (en) ? ENETC_SIMR_RSSE : 0; 2887 enetc_wr(hw, ENETC_SIMR, reg); 2888 2889 return 0; 2890 } 2891 2892 static void enetc_enable_rxvlan(struct net_device *ndev, bool en) 2893 { 2894 struct enetc_ndev_priv *priv = netdev_priv(ndev); 2895 struct enetc_hw *hw = &priv->si->hw; 2896 int i; 2897 2898 for (i = 0; i < priv->num_rx_rings; i++) 2899 enetc_bdr_enable_rxvlan(hw, i, en); 2900 } 2901 2902 static void enetc_enable_txvlan(struct net_device *ndev, bool en) 2903 { 2904 struct enetc_ndev_priv *priv = netdev_priv(ndev); 2905 struct enetc_hw *hw = &priv->si->hw; 2906 int i; 2907 2908 for (i = 0; i < priv->num_tx_rings; i++) 2909 enetc_bdr_enable_txvlan(hw, i, en); 2910 } 2911 2912 void enetc_set_features(struct net_device *ndev, netdev_features_t features) 2913 { 2914 netdev_features_t changed = ndev->features ^ features; 2915 2916 if (changed & NETIF_F_RXHASH) 2917 enetc_set_rss(ndev, !!(features & NETIF_F_RXHASH)); 2918 2919 if (changed & NETIF_F_HW_VLAN_CTAG_RX) 2920 enetc_enable_rxvlan(ndev, 2921 !!(features & NETIF_F_HW_VLAN_CTAG_RX)); 2922 2923 if (changed & NETIF_F_HW_VLAN_CTAG_TX) 2924 enetc_enable_txvlan(ndev, 2925 !!(features & NETIF_F_HW_VLAN_CTAG_TX)); 2926 } 2927 EXPORT_SYMBOL_GPL(enetc_set_features); 2928 2929 static int enetc_hwtstamp_set(struct net_device *ndev, struct ifreq *ifr) 2930 { 2931 struct enetc_ndev_priv *priv = netdev_priv(ndev); 2932 int err, new_offloads = priv->active_offloads; 2933 struct hwtstamp_config config; 2934 2935 if (copy_from_user(&config, ifr->ifr_data, sizeof(config))) 2936 return -EFAULT; 2937 2938 switch (config.tx_type) { 2939 case HWTSTAMP_TX_OFF: 2940 new_offloads &= ~ENETC_F_TX_TSTAMP_MASK; 2941 break; 2942 case HWTSTAMP_TX_ON: 2943 new_offloads &= ~ENETC_F_TX_TSTAMP_MASK; 2944 new_offloads |= ENETC_F_TX_TSTAMP; 2945 break; 2946 case HWTSTAMP_TX_ONESTEP_SYNC: 2947 new_offloads &= ~ENETC_F_TX_TSTAMP_MASK; 2948 new_offloads |= ENETC_F_TX_ONESTEP_SYNC_TSTAMP; 2949 break; 2950 default: 2951 return -ERANGE; 2952 } 2953 2954 switch (config.rx_filter) { 2955 case HWTSTAMP_FILTER_NONE: 2956 new_offloads &= ~ENETC_F_RX_TSTAMP; 2957 break; 2958 default: 2959 new_offloads |= ENETC_F_RX_TSTAMP; 2960 config.rx_filter = HWTSTAMP_FILTER_ALL; 2961 } 2962 2963 if ((new_offloads ^ priv->active_offloads) & ENETC_F_RX_TSTAMP) { 2964 bool extended = !!(new_offloads & ENETC_F_RX_TSTAMP); 2965 2966 err = enetc_reconfigure(priv, extended, NULL, NULL); 2967 if (err) 2968 return err; 2969 } 2970 2971 priv->active_offloads = new_offloads; 2972 2973 return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ? 2974 -EFAULT : 0; 2975 } 2976 2977 static int enetc_hwtstamp_get(struct net_device *ndev, struct ifreq *ifr) 2978 { 2979 struct enetc_ndev_priv *priv = netdev_priv(ndev); 2980 struct hwtstamp_config config; 2981 2982 config.flags = 0; 2983 2984 if (priv->active_offloads & ENETC_F_TX_ONESTEP_SYNC_TSTAMP) 2985 config.tx_type = HWTSTAMP_TX_ONESTEP_SYNC; 2986 else if (priv->active_offloads & ENETC_F_TX_TSTAMP) 2987 config.tx_type = HWTSTAMP_TX_ON; 2988 else 2989 config.tx_type = HWTSTAMP_TX_OFF; 2990 2991 config.rx_filter = (priv->active_offloads & ENETC_F_RX_TSTAMP) ? 2992 HWTSTAMP_FILTER_ALL : HWTSTAMP_FILTER_NONE; 2993 2994 return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ? 2995 -EFAULT : 0; 2996 } 2997 2998 int enetc_ioctl(struct net_device *ndev, struct ifreq *rq, int cmd) 2999 { 3000 struct enetc_ndev_priv *priv = netdev_priv(ndev); 3001 3002 if (IS_ENABLED(CONFIG_FSL_ENETC_PTP_CLOCK)) { 3003 if (cmd == SIOCSHWTSTAMP) 3004 return enetc_hwtstamp_set(ndev, rq); 3005 if (cmd == SIOCGHWTSTAMP) 3006 return enetc_hwtstamp_get(ndev, rq); 3007 } 3008 3009 if (!priv->phylink) 3010 return -EOPNOTSUPP; 3011 3012 return phylink_mii_ioctl(priv->phylink, rq, cmd); 3013 } 3014 EXPORT_SYMBOL_GPL(enetc_ioctl); 3015 3016 static int enetc_int_vector_init(struct enetc_ndev_priv *priv, int i, 3017 int v_tx_rings) 3018 { 3019 struct enetc_int_vector *v; 3020 struct enetc_bdr *bdr; 3021 int j, err; 3022 3023 v = kzalloc(struct_size(v, tx_ring, v_tx_rings), GFP_KERNEL); 3024 if (!v) 3025 return -ENOMEM; 3026 3027 priv->int_vector[i] = v; 3028 bdr = &v->rx_ring; 3029 bdr->index = i; 3030 bdr->ndev = priv->ndev; 3031 bdr->dev = priv->dev; 3032 bdr->bd_count = priv->rx_bd_count; 3033 bdr->buffer_offset = ENETC_RXB_PAD; 3034 priv->rx_ring[i] = bdr; 3035 3036 err = xdp_rxq_info_reg(&bdr->xdp.rxq, priv->ndev, i, 0); 3037 if (err) 3038 goto free_vector; 3039 3040 err = xdp_rxq_info_reg_mem_model(&bdr->xdp.rxq, MEM_TYPE_PAGE_SHARED, 3041 NULL); 3042 if (err) { 3043 xdp_rxq_info_unreg(&bdr->xdp.rxq); 3044 goto free_vector; 3045 } 3046 3047 /* init defaults for adaptive IC */ 3048 if (priv->ic_mode & ENETC_IC_RX_ADAPTIVE) { 3049 v->rx_ictt = 0x1; 3050 v->rx_dim_en = true; 3051 } 3052 3053 INIT_WORK(&v->rx_dim.work, enetc_rx_dim_work); 3054 netif_napi_add(priv->ndev, &v->napi, enetc_poll); 3055 v->count_tx_rings = v_tx_rings; 3056 3057 for (j = 0; j < v_tx_rings; j++) { 3058 int idx; 3059 3060 /* default tx ring mapping policy */ 3061 idx = priv->bdr_int_num * j + i; 3062 __set_bit(idx, &v->tx_rings_map); 3063 bdr = &v->tx_ring[j]; 3064 bdr->index = idx; 3065 bdr->ndev = priv->ndev; 3066 bdr->dev = priv->dev; 3067 bdr->bd_count = priv->tx_bd_count; 3068 priv->tx_ring[idx] = bdr; 3069 } 3070 3071 return 0; 3072 3073 free_vector: 3074 priv->rx_ring[i] = NULL; 3075 priv->int_vector[i] = NULL; 3076 kfree(v); 3077 3078 return err; 3079 } 3080 3081 static void enetc_int_vector_destroy(struct enetc_ndev_priv *priv, int i) 3082 { 3083 struct enetc_int_vector *v = priv->int_vector[i]; 3084 struct enetc_bdr *rx_ring = &v->rx_ring; 3085 int j, tx_ring_index; 3086 3087 xdp_rxq_info_unreg_mem_model(&rx_ring->xdp.rxq); 3088 xdp_rxq_info_unreg(&rx_ring->xdp.rxq); 3089 netif_napi_del(&v->napi); 3090 cancel_work_sync(&v->rx_dim.work); 3091 3092 for (j = 0; j < v->count_tx_rings; j++) { 3093 tx_ring_index = priv->bdr_int_num * j + i; 3094 priv->tx_ring[tx_ring_index] = NULL; 3095 } 3096 3097 priv->rx_ring[i] = NULL; 3098 priv->int_vector[i] = NULL; 3099 kfree(v); 3100 } 3101 3102 int enetc_alloc_msix(struct enetc_ndev_priv *priv) 3103 { 3104 struct pci_dev *pdev = priv->si->pdev; 3105 int v_tx_rings, v_remainder; 3106 int num_stack_tx_queues; 3107 int first_xdp_tx_ring; 3108 int i, n, err, nvec; 3109 3110 nvec = ENETC_BDR_INT_BASE_IDX + priv->bdr_int_num; 3111 /* allocate MSIX for both messaging and Rx/Tx interrupts */ 3112 n = pci_alloc_irq_vectors(pdev, nvec, nvec, PCI_IRQ_MSIX); 3113 3114 if (n < 0) 3115 return n; 3116 3117 if (n != nvec) 3118 return -EPERM; 3119 3120 /* # of tx rings per int vector */ 3121 v_tx_rings = priv->num_tx_rings / priv->bdr_int_num; 3122 v_remainder = priv->num_tx_rings % priv->bdr_int_num; 3123 3124 for (i = 0; i < priv->bdr_int_num; i++) { 3125 /* Distribute the remaining TX rings to the first v_remainder 3126 * interrupt vectors 3127 */ 3128 int num_tx_rings = i < v_remainder ? v_tx_rings + 1 : v_tx_rings; 3129 3130 err = enetc_int_vector_init(priv, i, num_tx_rings); 3131 if (err) 3132 goto fail; 3133 } 3134 3135 num_stack_tx_queues = enetc_num_stack_tx_queues(priv); 3136 3137 err = netif_set_real_num_tx_queues(priv->ndev, num_stack_tx_queues); 3138 if (err) 3139 goto fail; 3140 3141 err = netif_set_real_num_rx_queues(priv->ndev, priv->num_rx_rings); 3142 if (err) 3143 goto fail; 3144 3145 priv->min_num_stack_tx_queues = num_possible_cpus(); 3146 first_xdp_tx_ring = priv->num_tx_rings - num_possible_cpus(); 3147 priv->xdp_tx_ring = &priv->tx_ring[first_xdp_tx_ring]; 3148 3149 return 0; 3150 3151 fail: 3152 while (i--) 3153 enetc_int_vector_destroy(priv, i); 3154 3155 pci_free_irq_vectors(pdev); 3156 3157 return err; 3158 } 3159 EXPORT_SYMBOL_GPL(enetc_alloc_msix); 3160 3161 void enetc_free_msix(struct enetc_ndev_priv *priv) 3162 { 3163 int i; 3164 3165 for (i = 0; i < priv->bdr_int_num; i++) 3166 enetc_int_vector_destroy(priv, i); 3167 3168 /* disable all MSIX for this device */ 3169 pci_free_irq_vectors(priv->si->pdev); 3170 } 3171 EXPORT_SYMBOL_GPL(enetc_free_msix); 3172 3173 static void enetc_kfree_si(struct enetc_si *si) 3174 { 3175 char *p = (char *)si - si->pad; 3176 3177 kfree(p); 3178 } 3179 3180 static void enetc_detect_errata(struct enetc_si *si) 3181 { 3182 if (si->pdev->revision == ENETC_REV1) 3183 si->errata = ENETC_ERR_VLAN_ISOL | ENETC_ERR_UCMCSWP; 3184 } 3185 3186 int enetc_pci_probe(struct pci_dev *pdev, const char *name, int sizeof_priv) 3187 { 3188 struct enetc_si *si, *p; 3189 struct enetc_hw *hw; 3190 size_t alloc_size; 3191 int err, len; 3192 3193 pcie_flr(pdev); 3194 err = pci_enable_device_mem(pdev); 3195 if (err) 3196 return dev_err_probe(&pdev->dev, err, "device enable failed\n"); 3197 3198 /* set up for high or low dma */ 3199 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 3200 if (err) { 3201 dev_err(&pdev->dev, "DMA configuration failed: 0x%x\n", err); 3202 goto err_dma; 3203 } 3204 3205 err = pci_request_mem_regions(pdev, name); 3206 if (err) { 3207 dev_err(&pdev->dev, "pci_request_regions failed err=%d\n", err); 3208 goto err_pci_mem_reg; 3209 } 3210 3211 pci_set_master(pdev); 3212 3213 alloc_size = sizeof(struct enetc_si); 3214 if (sizeof_priv) { 3215 /* align priv to 32B */ 3216 alloc_size = ALIGN(alloc_size, ENETC_SI_ALIGN); 3217 alloc_size += sizeof_priv; 3218 } 3219 /* force 32B alignment for enetc_si */ 3220 alloc_size += ENETC_SI_ALIGN - 1; 3221 3222 p = kzalloc(alloc_size, GFP_KERNEL); 3223 if (!p) { 3224 err = -ENOMEM; 3225 goto err_alloc_si; 3226 } 3227 3228 si = PTR_ALIGN(p, ENETC_SI_ALIGN); 3229 si->pad = (char *)si - (char *)p; 3230 3231 pci_set_drvdata(pdev, si); 3232 si->pdev = pdev; 3233 hw = &si->hw; 3234 3235 len = pci_resource_len(pdev, ENETC_BAR_REGS); 3236 hw->reg = ioremap(pci_resource_start(pdev, ENETC_BAR_REGS), len); 3237 if (!hw->reg) { 3238 err = -ENXIO; 3239 dev_err(&pdev->dev, "ioremap() failed\n"); 3240 goto err_ioremap; 3241 } 3242 if (len > ENETC_PORT_BASE) 3243 hw->port = hw->reg + ENETC_PORT_BASE; 3244 if (len > ENETC_GLOBAL_BASE) 3245 hw->global = hw->reg + ENETC_GLOBAL_BASE; 3246 3247 enetc_detect_errata(si); 3248 3249 return 0; 3250 3251 err_ioremap: 3252 enetc_kfree_si(si); 3253 err_alloc_si: 3254 pci_release_mem_regions(pdev); 3255 err_pci_mem_reg: 3256 err_dma: 3257 pci_disable_device(pdev); 3258 3259 return err; 3260 } 3261 EXPORT_SYMBOL_GPL(enetc_pci_probe); 3262 3263 void enetc_pci_remove(struct pci_dev *pdev) 3264 { 3265 struct enetc_si *si = pci_get_drvdata(pdev); 3266 struct enetc_hw *hw = &si->hw; 3267 3268 iounmap(hw->reg); 3269 enetc_kfree_si(si); 3270 pci_release_mem_regions(pdev); 3271 pci_disable_device(pdev); 3272 } 3273 EXPORT_SYMBOL_GPL(enetc_pci_remove); 3274 3275 static const struct enetc_drvdata enetc_pf_data = { 3276 .sysclk_freq = ENETC_CLK_400M, 3277 .pmac_offset = ENETC_PMAC_OFFSET, 3278 .eth_ops = &enetc_pf_ethtool_ops, 3279 }; 3280 3281 static const struct enetc_drvdata enetc4_pf_data = { 3282 .sysclk_freq = ENETC_CLK_333M, 3283 .pmac_offset = ENETC4_PMAC_OFFSET, 3284 .eth_ops = &enetc4_pf_ethtool_ops, 3285 }; 3286 3287 static const struct enetc_drvdata enetc_vf_data = { 3288 .sysclk_freq = ENETC_CLK_400M, 3289 .eth_ops = &enetc_vf_ethtool_ops, 3290 }; 3291 3292 static const struct enetc_platform_info enetc_info[] = { 3293 { .revision = ENETC_REV_1_0, 3294 .dev_id = ENETC_DEV_ID_PF, 3295 .data = &enetc_pf_data, 3296 }, 3297 { .revision = ENETC_REV_4_1, 3298 .dev_id = NXP_ENETC_PF_DEV_ID, 3299 .data = &enetc4_pf_data, 3300 }, 3301 { .revision = ENETC_REV_1_0, 3302 .dev_id = ENETC_DEV_ID_VF, 3303 .data = &enetc_vf_data, 3304 }, 3305 }; 3306 3307 int enetc_get_driver_data(struct enetc_si *si) 3308 { 3309 u16 dev_id = si->pdev->device; 3310 int i; 3311 3312 for (i = 0; i < ARRAY_SIZE(enetc_info); i++) { 3313 if (si->revision == enetc_info[i].revision && 3314 dev_id == enetc_info[i].dev_id) { 3315 si->drvdata = enetc_info[i].data; 3316 3317 return 0; 3318 } 3319 } 3320 3321 return -ERANGE; 3322 } 3323 EXPORT_SYMBOL_GPL(enetc_get_driver_data); 3324 3325 MODULE_DESCRIPTION("NXP ENETC Ethernet driver"); 3326 MODULE_LICENSE("Dual BSD/GPL"); 3327