1 // SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) 2 /* Copyright 2017-2019 NXP */ 3 4 #include "enetc.h" 5 #include <linux/bpf_trace.h> 6 #include <linux/clk.h> 7 #include <linux/tcp.h> 8 #include <linux/udp.h> 9 #include <linux/vmalloc.h> 10 #include <linux/ptp_classify.h> 11 #include <net/ip6_checksum.h> 12 #include <net/pkt_sched.h> 13 #include <net/tso.h> 14 15 u32 enetc_port_mac_rd(struct enetc_si *si, u32 reg) 16 { 17 return enetc_port_rd(&si->hw, reg); 18 } 19 EXPORT_SYMBOL_GPL(enetc_port_mac_rd); 20 21 void enetc_port_mac_wr(struct enetc_si *si, u32 reg, u32 val) 22 { 23 enetc_port_wr(&si->hw, reg, val); 24 if (si->hw_features & ENETC_SI_F_QBU) 25 enetc_port_wr(&si->hw, reg + si->drvdata->pmac_offset, val); 26 } 27 EXPORT_SYMBOL_GPL(enetc_port_mac_wr); 28 29 static void enetc_change_preemptible_tcs(struct enetc_ndev_priv *priv, 30 u8 preemptible_tcs) 31 { 32 if (!(priv->si->hw_features & ENETC_SI_F_QBU)) 33 return; 34 35 priv->preemptible_tcs = preemptible_tcs; 36 enetc_mm_commit_preemptible_tcs(priv); 37 } 38 39 static int enetc_mac_addr_hash_idx(const u8 *addr) 40 { 41 u64 fold = __swab64(ether_addr_to_u64(addr)) >> 16; 42 u64 mask = 0; 43 int res = 0; 44 int i; 45 46 for (i = 0; i < 8; i++) 47 mask |= BIT_ULL(i * 6); 48 49 for (i = 0; i < 6; i++) 50 res |= (hweight64(fold & (mask << i)) & 0x1) << i; 51 52 return res; 53 } 54 55 void enetc_add_mac_addr_ht_filter(struct enetc_mac_filter *filter, 56 const unsigned char *addr) 57 { 58 int idx = enetc_mac_addr_hash_idx(addr); 59 60 /* add hash table entry */ 61 __set_bit(idx, filter->mac_hash_table); 62 filter->mac_addr_cnt++; 63 } 64 EXPORT_SYMBOL_GPL(enetc_add_mac_addr_ht_filter); 65 66 void enetc_reset_mac_addr_filter(struct enetc_mac_filter *filter) 67 { 68 filter->mac_addr_cnt = 0; 69 70 bitmap_zero(filter->mac_hash_table, 71 ENETC_MADDR_HASH_TBL_SZ); 72 } 73 EXPORT_SYMBOL_GPL(enetc_reset_mac_addr_filter); 74 75 static int enetc_num_stack_tx_queues(struct enetc_ndev_priv *priv) 76 { 77 int num_tx_rings = priv->num_tx_rings; 78 79 if (priv->xdp_prog) 80 return num_tx_rings - num_possible_cpus(); 81 82 return num_tx_rings; 83 } 84 85 static struct enetc_bdr *enetc_rx_ring_from_xdp_tx_ring(struct enetc_ndev_priv *priv, 86 struct enetc_bdr *tx_ring) 87 { 88 int index = &priv->tx_ring[tx_ring->index] - priv->xdp_tx_ring; 89 90 return priv->rx_ring[index]; 91 } 92 93 static struct sk_buff *enetc_tx_swbd_get_skb(struct enetc_tx_swbd *tx_swbd) 94 { 95 if (tx_swbd->is_xdp_tx || tx_swbd->is_xdp_redirect) 96 return NULL; 97 98 return tx_swbd->skb; 99 } 100 101 static struct xdp_frame * 102 enetc_tx_swbd_get_xdp_frame(struct enetc_tx_swbd *tx_swbd) 103 { 104 if (tx_swbd->is_xdp_redirect) 105 return tx_swbd->xdp_frame; 106 107 return NULL; 108 } 109 110 static void enetc_unmap_tx_buff(struct enetc_bdr *tx_ring, 111 struct enetc_tx_swbd *tx_swbd) 112 { 113 /* For XDP_TX, pages come from RX, whereas for the other contexts where 114 * we have is_dma_page_set, those come from skb_frag_dma_map. We need 115 * to match the DMA mapping length, so we need to differentiate those. 116 */ 117 if (tx_swbd->is_dma_page) 118 dma_unmap_page(tx_ring->dev, tx_swbd->dma, 119 tx_swbd->is_xdp_tx ? PAGE_SIZE : tx_swbd->len, 120 tx_swbd->dir); 121 else 122 dma_unmap_single(tx_ring->dev, tx_swbd->dma, 123 tx_swbd->len, tx_swbd->dir); 124 tx_swbd->dma = 0; 125 } 126 127 static void enetc_free_tx_frame(struct enetc_bdr *tx_ring, 128 struct enetc_tx_swbd *tx_swbd) 129 { 130 struct xdp_frame *xdp_frame = enetc_tx_swbd_get_xdp_frame(tx_swbd); 131 struct sk_buff *skb = enetc_tx_swbd_get_skb(tx_swbd); 132 133 if (tx_swbd->dma) 134 enetc_unmap_tx_buff(tx_ring, tx_swbd); 135 136 if (xdp_frame) { 137 xdp_return_frame(tx_swbd->xdp_frame); 138 tx_swbd->xdp_frame = NULL; 139 } else if (skb) { 140 dev_kfree_skb_any(skb); 141 tx_swbd->skb = NULL; 142 } 143 } 144 145 /* Let H/W know BD ring has been updated */ 146 static void enetc_update_tx_ring_tail(struct enetc_bdr *tx_ring) 147 { 148 /* includes wmb() */ 149 enetc_wr_reg_hot(tx_ring->tpir, tx_ring->next_to_use); 150 } 151 152 static int enetc_ptp_parse(struct sk_buff *skb, u8 *udp, 153 u8 *msgtype, u8 *twostep, 154 u16 *correction_offset, u16 *body_offset) 155 { 156 unsigned int ptp_class; 157 struct ptp_header *hdr; 158 unsigned int type; 159 u8 *base; 160 161 ptp_class = ptp_classify_raw(skb); 162 if (ptp_class == PTP_CLASS_NONE) 163 return -EINVAL; 164 165 hdr = ptp_parse_header(skb, ptp_class); 166 if (!hdr) 167 return -EINVAL; 168 169 type = ptp_class & PTP_CLASS_PMASK; 170 if (type == PTP_CLASS_IPV4 || type == PTP_CLASS_IPV6) 171 *udp = 1; 172 else 173 *udp = 0; 174 175 *msgtype = ptp_get_msgtype(hdr, ptp_class); 176 *twostep = hdr->flag_field[0] & 0x2; 177 178 base = skb_mac_header(skb); 179 *correction_offset = (u8 *)&hdr->correction - base; 180 *body_offset = (u8 *)hdr + sizeof(struct ptp_header) - base; 181 182 return 0; 183 } 184 185 static bool enetc_tx_csum_offload_check(struct sk_buff *skb) 186 { 187 switch (skb->csum_offset) { 188 case offsetof(struct tcphdr, check): 189 case offsetof(struct udphdr, check): 190 return true; 191 default: 192 return false; 193 } 194 } 195 196 static bool enetc_skb_is_ipv6(struct sk_buff *skb) 197 { 198 return vlan_get_protocol(skb) == htons(ETH_P_IPV6); 199 } 200 201 static bool enetc_skb_is_tcp(struct sk_buff *skb) 202 { 203 return skb->csum_offset == offsetof(struct tcphdr, check); 204 } 205 206 /** 207 * enetc_unwind_tx_frame() - Unwind the DMA mappings of a multi-buffer Tx frame 208 * @tx_ring: Pointer to the Tx ring on which the buffer descriptors are located 209 * @count: Number of Tx buffer descriptors which need to be unmapped 210 * @i: Index of the last successfully mapped Tx buffer descriptor 211 */ 212 static void enetc_unwind_tx_frame(struct enetc_bdr *tx_ring, int count, int i) 213 { 214 while (count--) { 215 struct enetc_tx_swbd *tx_swbd = &tx_ring->tx_swbd[i]; 216 217 enetc_free_tx_frame(tx_ring, tx_swbd); 218 if (i == 0) 219 i = tx_ring->bd_count; 220 i--; 221 } 222 } 223 224 static int enetc_map_tx_buffs(struct enetc_bdr *tx_ring, struct sk_buff *skb) 225 { 226 bool do_vlan, do_onestep_tstamp = false, do_twostep_tstamp = false; 227 struct enetc_ndev_priv *priv = netdev_priv(tx_ring->ndev); 228 struct enetc_hw *hw = &priv->si->hw; 229 struct enetc_tx_swbd *tx_swbd; 230 int len = skb_headlen(skb); 231 union enetc_tx_bd temp_bd; 232 u8 msgtype, twostep, udp; 233 union enetc_tx_bd *txbd; 234 u16 offset1, offset2; 235 int i, count = 0; 236 skb_frag_t *frag; 237 unsigned int f; 238 dma_addr_t dma; 239 u8 flags = 0; 240 241 enetc_clear_tx_bd(&temp_bd); 242 if (skb->ip_summed == CHECKSUM_PARTIAL) { 243 /* Can not support TSD and checksum offload at the same time */ 244 if (priv->active_offloads & ENETC_F_TXCSUM && 245 enetc_tx_csum_offload_check(skb) && !tx_ring->tsd_enable) { 246 temp_bd.l3_aux0 = FIELD_PREP(ENETC_TX_BD_L3_START, 247 skb_network_offset(skb)); 248 temp_bd.l3_aux1 = FIELD_PREP(ENETC_TX_BD_L3_HDR_LEN, 249 skb_network_header_len(skb) / 4); 250 temp_bd.l3_aux1 |= FIELD_PREP(ENETC_TX_BD_L3T, 251 enetc_skb_is_ipv6(skb)); 252 if (enetc_skb_is_tcp(skb)) 253 temp_bd.l4_aux = FIELD_PREP(ENETC_TX_BD_L4T, 254 ENETC_TXBD_L4T_TCP); 255 else 256 temp_bd.l4_aux = FIELD_PREP(ENETC_TX_BD_L4T, 257 ENETC_TXBD_L4T_UDP); 258 flags |= ENETC_TXBD_FLAGS_CSUM_LSO | ENETC_TXBD_FLAGS_L4CS; 259 } else if (skb_checksum_help(skb)) { 260 return 0; 261 } 262 } 263 264 i = tx_ring->next_to_use; 265 txbd = ENETC_TXBD(*tx_ring, i); 266 prefetchw(txbd); 267 268 dma = dma_map_single(tx_ring->dev, skb->data, len, DMA_TO_DEVICE); 269 if (unlikely(dma_mapping_error(tx_ring->dev, dma))) 270 goto dma_err; 271 272 temp_bd.addr = cpu_to_le64(dma); 273 temp_bd.buf_len = cpu_to_le16(len); 274 275 tx_swbd = &tx_ring->tx_swbd[i]; 276 tx_swbd->dma = dma; 277 tx_swbd->len = len; 278 tx_swbd->is_dma_page = 0; 279 tx_swbd->dir = DMA_TO_DEVICE; 280 count++; 281 282 do_vlan = skb_vlan_tag_present(skb); 283 if (skb->cb[0] & ENETC_F_TX_ONESTEP_SYNC_TSTAMP) { 284 if (enetc_ptp_parse(skb, &udp, &msgtype, &twostep, &offset1, 285 &offset2) || 286 msgtype != PTP_MSGTYPE_SYNC || twostep) 287 WARN_ONCE(1, "Bad packet for one-step timestamping\n"); 288 else 289 do_onestep_tstamp = true; 290 } else if (skb->cb[0] & ENETC_F_TX_TSTAMP) { 291 do_twostep_tstamp = true; 292 } 293 294 tx_swbd->do_twostep_tstamp = do_twostep_tstamp; 295 tx_swbd->qbv_en = !!(priv->active_offloads & ENETC_F_QBV); 296 tx_swbd->check_wb = tx_swbd->do_twostep_tstamp || tx_swbd->qbv_en; 297 298 if (do_vlan || do_onestep_tstamp || do_twostep_tstamp) 299 flags |= ENETC_TXBD_FLAGS_EX; 300 301 if (tx_ring->tsd_enable) 302 flags |= ENETC_TXBD_FLAGS_TSE | ENETC_TXBD_FLAGS_TXSTART; 303 304 /* first BD needs frm_len and offload flags set */ 305 temp_bd.frm_len = cpu_to_le16(skb->len); 306 temp_bd.flags = flags; 307 308 if (flags & ENETC_TXBD_FLAGS_TSE) 309 temp_bd.txstart = enetc_txbd_set_tx_start(skb->skb_mstamp_ns, 310 flags); 311 312 if (flags & ENETC_TXBD_FLAGS_EX) { 313 u8 e_flags = 0; 314 *txbd = temp_bd; 315 enetc_clear_tx_bd(&temp_bd); 316 317 /* add extension BD for VLAN and/or timestamping */ 318 flags = 0; 319 tx_swbd++; 320 txbd++; 321 i++; 322 if (unlikely(i == tx_ring->bd_count)) { 323 i = 0; 324 tx_swbd = tx_ring->tx_swbd; 325 txbd = ENETC_TXBD(*tx_ring, 0); 326 } 327 prefetchw(txbd); 328 329 if (do_vlan) { 330 temp_bd.ext.vid = cpu_to_le16(skb_vlan_tag_get(skb)); 331 temp_bd.ext.tpid = 0; /* < C-TAG */ 332 e_flags |= ENETC_TXBD_E_FLAGS_VLAN_INS; 333 } 334 335 if (do_onestep_tstamp) { 336 __be32 new_sec_l, new_nsec; 337 u32 lo, hi, nsec, val; 338 __be16 new_sec_h; 339 u8 *data; 340 u64 sec; 341 342 lo = enetc_rd_hot(hw, ENETC_SICTR0); 343 hi = enetc_rd_hot(hw, ENETC_SICTR1); 344 sec = (u64)hi << 32 | lo; 345 nsec = do_div(sec, 1000000000); 346 347 /* Configure extension BD */ 348 temp_bd.ext.tstamp = cpu_to_le32(lo & 0x3fffffff); 349 e_flags |= ENETC_TXBD_E_FLAGS_ONE_STEP_PTP; 350 351 /* Update originTimestamp field of Sync packet 352 * - 48 bits seconds field 353 * - 32 bits nanseconds field 354 * 355 * In addition, the UDP checksum needs to be updated 356 * by software after updating originTimestamp field, 357 * otherwise the hardware will calculate the wrong 358 * checksum when updating the correction field and 359 * update it to the packet. 360 */ 361 data = skb_mac_header(skb); 362 new_sec_h = htons((sec >> 32) & 0xffff); 363 new_sec_l = htonl(sec & 0xffffffff); 364 new_nsec = htonl(nsec); 365 if (udp) { 366 struct udphdr *uh = udp_hdr(skb); 367 __be32 old_sec_l, old_nsec; 368 __be16 old_sec_h; 369 370 old_sec_h = *(__be16 *)(data + offset2); 371 inet_proto_csum_replace2(&uh->check, skb, old_sec_h, 372 new_sec_h, false); 373 374 old_sec_l = *(__be32 *)(data + offset2 + 2); 375 inet_proto_csum_replace4(&uh->check, skb, old_sec_l, 376 new_sec_l, false); 377 378 old_nsec = *(__be32 *)(data + offset2 + 6); 379 inet_proto_csum_replace4(&uh->check, skb, old_nsec, 380 new_nsec, false); 381 } 382 383 *(__be16 *)(data + offset2) = new_sec_h; 384 *(__be32 *)(data + offset2 + 2) = new_sec_l; 385 *(__be32 *)(data + offset2 + 6) = new_nsec; 386 387 /* Configure single-step register */ 388 val = ENETC_PM0_SINGLE_STEP_EN; 389 val |= ENETC_SET_SINGLE_STEP_OFFSET(offset1); 390 if (udp) 391 val |= ENETC_PM0_SINGLE_STEP_CH; 392 393 enetc_port_mac_wr(priv->si, ENETC_PM0_SINGLE_STEP, 394 val); 395 } else if (do_twostep_tstamp) { 396 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; 397 e_flags |= ENETC_TXBD_E_FLAGS_TWO_STEP_PTP; 398 } 399 400 temp_bd.ext.e_flags = e_flags; 401 count++; 402 } 403 404 frag = &skb_shinfo(skb)->frags[0]; 405 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++, frag++) { 406 len = skb_frag_size(frag); 407 dma = skb_frag_dma_map(tx_ring->dev, frag, 0, len, 408 DMA_TO_DEVICE); 409 if (dma_mapping_error(tx_ring->dev, dma)) 410 goto dma_err; 411 412 *txbd = temp_bd; 413 enetc_clear_tx_bd(&temp_bd); 414 415 flags = 0; 416 tx_swbd++; 417 txbd++; 418 i++; 419 if (unlikely(i == tx_ring->bd_count)) { 420 i = 0; 421 tx_swbd = tx_ring->tx_swbd; 422 txbd = ENETC_TXBD(*tx_ring, 0); 423 } 424 prefetchw(txbd); 425 426 temp_bd.addr = cpu_to_le64(dma); 427 temp_bd.buf_len = cpu_to_le16(len); 428 429 tx_swbd->dma = dma; 430 tx_swbd->len = len; 431 tx_swbd->is_dma_page = 1; 432 tx_swbd->dir = DMA_TO_DEVICE; 433 count++; 434 } 435 436 /* last BD needs 'F' bit set */ 437 flags |= ENETC_TXBD_FLAGS_F; 438 temp_bd.flags = flags; 439 *txbd = temp_bd; 440 441 tx_ring->tx_swbd[i].is_eof = true; 442 tx_ring->tx_swbd[i].skb = skb; 443 444 enetc_bdr_idx_inc(tx_ring, &i); 445 tx_ring->next_to_use = i; 446 447 skb_tx_timestamp(skb); 448 449 enetc_update_tx_ring_tail(tx_ring); 450 451 return count; 452 453 dma_err: 454 dev_err(tx_ring->dev, "DMA map error"); 455 456 enetc_unwind_tx_frame(tx_ring, count, i); 457 458 return 0; 459 } 460 461 static int enetc_map_tx_tso_hdr(struct enetc_bdr *tx_ring, struct sk_buff *skb, 462 struct enetc_tx_swbd *tx_swbd, 463 union enetc_tx_bd *txbd, int *i, int hdr_len, 464 int data_len) 465 { 466 union enetc_tx_bd txbd_tmp; 467 u8 flags = 0, e_flags = 0; 468 dma_addr_t addr; 469 int count = 1; 470 471 enetc_clear_tx_bd(&txbd_tmp); 472 addr = tx_ring->tso_headers_dma + *i * TSO_HEADER_SIZE; 473 474 if (skb_vlan_tag_present(skb)) 475 flags |= ENETC_TXBD_FLAGS_EX; 476 477 txbd_tmp.addr = cpu_to_le64(addr); 478 txbd_tmp.buf_len = cpu_to_le16(hdr_len); 479 480 /* first BD needs frm_len and offload flags set */ 481 txbd_tmp.frm_len = cpu_to_le16(hdr_len + data_len); 482 txbd_tmp.flags = flags; 483 484 /* For the TSO header we do not set the dma address since we do not 485 * want it unmapped when we do cleanup. We still set len so that we 486 * count the bytes sent. 487 */ 488 tx_swbd->len = hdr_len; 489 tx_swbd->do_twostep_tstamp = false; 490 tx_swbd->check_wb = false; 491 492 /* Actually write the header in the BD */ 493 *txbd = txbd_tmp; 494 495 /* Add extension BD for VLAN */ 496 if (flags & ENETC_TXBD_FLAGS_EX) { 497 /* Get the next BD */ 498 enetc_bdr_idx_inc(tx_ring, i); 499 txbd = ENETC_TXBD(*tx_ring, *i); 500 tx_swbd = &tx_ring->tx_swbd[*i]; 501 prefetchw(txbd); 502 503 /* Setup the VLAN fields */ 504 enetc_clear_tx_bd(&txbd_tmp); 505 txbd_tmp.ext.vid = cpu_to_le16(skb_vlan_tag_get(skb)); 506 txbd_tmp.ext.tpid = 0; /* < C-TAG */ 507 e_flags |= ENETC_TXBD_E_FLAGS_VLAN_INS; 508 509 /* Write the BD */ 510 txbd_tmp.ext.e_flags = e_flags; 511 *txbd = txbd_tmp; 512 count++; 513 } 514 515 return count; 516 } 517 518 static int enetc_map_tx_tso_data(struct enetc_bdr *tx_ring, struct sk_buff *skb, 519 struct enetc_tx_swbd *tx_swbd, 520 union enetc_tx_bd *txbd, char *data, 521 int size, bool last_bd) 522 { 523 union enetc_tx_bd txbd_tmp; 524 dma_addr_t addr; 525 u8 flags = 0; 526 527 enetc_clear_tx_bd(&txbd_tmp); 528 529 addr = dma_map_single(tx_ring->dev, data, size, DMA_TO_DEVICE); 530 if (unlikely(dma_mapping_error(tx_ring->dev, addr))) { 531 netdev_err(tx_ring->ndev, "DMA map error\n"); 532 return -ENOMEM; 533 } 534 535 if (last_bd) { 536 flags |= ENETC_TXBD_FLAGS_F; 537 tx_swbd->is_eof = 1; 538 } 539 540 txbd_tmp.addr = cpu_to_le64(addr); 541 txbd_tmp.buf_len = cpu_to_le16(size); 542 txbd_tmp.flags = flags; 543 544 tx_swbd->dma = addr; 545 tx_swbd->len = size; 546 tx_swbd->dir = DMA_TO_DEVICE; 547 548 *txbd = txbd_tmp; 549 550 return 0; 551 } 552 553 static __wsum enetc_tso_hdr_csum(struct tso_t *tso, struct sk_buff *skb, 554 char *hdr, int hdr_len, int *l4_hdr_len) 555 { 556 char *l4_hdr = hdr + skb_transport_offset(skb); 557 int mac_hdr_len = skb_network_offset(skb); 558 559 if (tso->tlen != sizeof(struct udphdr)) { 560 struct tcphdr *tcph = (struct tcphdr *)(l4_hdr); 561 562 tcph->check = 0; 563 } else { 564 struct udphdr *udph = (struct udphdr *)(l4_hdr); 565 566 udph->check = 0; 567 } 568 569 /* Compute the IP checksum. This is necessary since tso_build_hdr() 570 * already incremented the IP ID field. 571 */ 572 if (!tso->ipv6) { 573 struct iphdr *iph = (void *)(hdr + mac_hdr_len); 574 575 iph->check = 0; 576 iph->check = ip_fast_csum((unsigned char *)iph, iph->ihl); 577 } 578 579 /* Compute the checksum over the L4 header. */ 580 *l4_hdr_len = hdr_len - skb_transport_offset(skb); 581 return csum_partial(l4_hdr, *l4_hdr_len, 0); 582 } 583 584 static void enetc_tso_complete_csum(struct enetc_bdr *tx_ring, struct tso_t *tso, 585 struct sk_buff *skb, char *hdr, int len, 586 __wsum sum) 587 { 588 char *l4_hdr = hdr + skb_transport_offset(skb); 589 __sum16 csum_final; 590 591 /* Complete the L4 checksum by appending the pseudo-header to the 592 * already computed checksum. 593 */ 594 if (!tso->ipv6) 595 csum_final = csum_tcpudp_magic(ip_hdr(skb)->saddr, 596 ip_hdr(skb)->daddr, 597 len, ip_hdr(skb)->protocol, sum); 598 else 599 csum_final = csum_ipv6_magic(&ipv6_hdr(skb)->saddr, 600 &ipv6_hdr(skb)->daddr, 601 len, ipv6_hdr(skb)->nexthdr, sum); 602 603 if (tso->tlen != sizeof(struct udphdr)) { 604 struct tcphdr *tcph = (struct tcphdr *)(l4_hdr); 605 606 tcph->check = csum_final; 607 } else { 608 struct udphdr *udph = (struct udphdr *)(l4_hdr); 609 610 udph->check = csum_final; 611 } 612 } 613 614 static int enetc_lso_count_descs(const struct sk_buff *skb) 615 { 616 /* 4 BDs: 1 BD for LSO header + 1 BD for extended BD + 1 BD 617 * for linear area data but not include LSO header, namely 618 * skb_headlen(skb) - lso_hdr_len (it may be 0, but that's 619 * okay, we only need to consider the worst case). And 1 BD 620 * for gap. 621 */ 622 return skb_shinfo(skb)->nr_frags + 4; 623 } 624 625 static int enetc_lso_get_hdr_len(const struct sk_buff *skb) 626 { 627 int hdr_len, tlen; 628 629 tlen = skb_is_gso_tcp(skb) ? tcp_hdrlen(skb) : sizeof(struct udphdr); 630 hdr_len = skb_transport_offset(skb) + tlen; 631 632 return hdr_len; 633 } 634 635 static void enetc_lso_start(struct sk_buff *skb, struct enetc_lso_t *lso) 636 { 637 lso->lso_seg_size = skb_shinfo(skb)->gso_size; 638 lso->ipv6 = enetc_skb_is_ipv6(skb); 639 lso->tcp = skb_is_gso_tcp(skb); 640 lso->l3_hdr_len = skb_network_header_len(skb); 641 lso->l3_start = skb_network_offset(skb); 642 lso->hdr_len = enetc_lso_get_hdr_len(skb); 643 lso->total_len = skb->len - lso->hdr_len; 644 } 645 646 static void enetc_lso_map_hdr(struct enetc_bdr *tx_ring, struct sk_buff *skb, 647 int *i, struct enetc_lso_t *lso) 648 { 649 union enetc_tx_bd txbd_tmp, *txbd; 650 struct enetc_tx_swbd *tx_swbd; 651 u16 frm_len, frm_len_ext; 652 u8 flags, e_flags = 0; 653 dma_addr_t addr; 654 char *hdr; 655 656 /* Get the first BD of the LSO BDs chain */ 657 txbd = ENETC_TXBD(*tx_ring, *i); 658 tx_swbd = &tx_ring->tx_swbd[*i]; 659 prefetchw(txbd); 660 661 /* Prepare LSO header: MAC + IP + TCP/UDP */ 662 hdr = tx_ring->tso_headers + *i * TSO_HEADER_SIZE; 663 memcpy(hdr, skb->data, lso->hdr_len); 664 addr = tx_ring->tso_headers_dma + *i * TSO_HEADER_SIZE; 665 666 /* {frm_len_ext, frm_len} indicates the total length of 667 * large transmit data unit. frm_len contains the 16 least 668 * significant bits and frm_len_ext contains the 4 most 669 * significant bits. 670 */ 671 frm_len = lso->total_len & 0xffff; 672 frm_len_ext = (lso->total_len >> 16) & 0xf; 673 674 /* Set the flags of the first BD */ 675 flags = ENETC_TXBD_FLAGS_EX | ENETC_TXBD_FLAGS_CSUM_LSO | 676 ENETC_TXBD_FLAGS_LSO | ENETC_TXBD_FLAGS_L4CS; 677 678 enetc_clear_tx_bd(&txbd_tmp); 679 txbd_tmp.addr = cpu_to_le64(addr); 680 txbd_tmp.hdr_len = cpu_to_le16(lso->hdr_len); 681 682 /* first BD needs frm_len and offload flags set */ 683 txbd_tmp.frm_len = cpu_to_le16(frm_len); 684 txbd_tmp.flags = flags; 685 686 txbd_tmp.l3_aux0 = FIELD_PREP(ENETC_TX_BD_L3_START, lso->l3_start); 687 /* l3_hdr_size in 32-bits (4 bytes) */ 688 txbd_tmp.l3_aux1 = FIELD_PREP(ENETC_TX_BD_L3_HDR_LEN, 689 lso->l3_hdr_len / 4); 690 if (lso->ipv6) 691 txbd_tmp.l3_aux1 |= ENETC_TX_BD_L3T; 692 else 693 txbd_tmp.l3_aux0 |= ENETC_TX_BD_IPCS; 694 695 txbd_tmp.l4_aux = FIELD_PREP(ENETC_TX_BD_L4T, lso->tcp ? 696 ENETC_TXBD_L4T_TCP : ENETC_TXBD_L4T_UDP); 697 698 /* For the LSO header we do not set the dma address since 699 * we do not want it unmapped when we do cleanup. We still 700 * set len so that we count the bytes sent. 701 */ 702 tx_swbd->len = lso->hdr_len; 703 tx_swbd->do_twostep_tstamp = false; 704 tx_swbd->check_wb = false; 705 706 /* Actually write the header in the BD */ 707 *txbd = txbd_tmp; 708 709 /* Get the next BD, and the next BD is extended BD */ 710 enetc_bdr_idx_inc(tx_ring, i); 711 txbd = ENETC_TXBD(*tx_ring, *i); 712 tx_swbd = &tx_ring->tx_swbd[*i]; 713 prefetchw(txbd); 714 715 enetc_clear_tx_bd(&txbd_tmp); 716 if (skb_vlan_tag_present(skb)) { 717 /* Setup the VLAN fields */ 718 txbd_tmp.ext.vid = cpu_to_le16(skb_vlan_tag_get(skb)); 719 txbd_tmp.ext.tpid = ENETC_TPID_8021Q; 720 e_flags = ENETC_TXBD_E_FLAGS_VLAN_INS; 721 } 722 723 /* Write the BD */ 724 txbd_tmp.ext.e_flags = e_flags; 725 txbd_tmp.ext.lso_sg_size = cpu_to_le16(lso->lso_seg_size); 726 txbd_tmp.ext.frm_len_ext = cpu_to_le16(frm_len_ext); 727 *txbd = txbd_tmp; 728 } 729 730 static int enetc_lso_map_data(struct enetc_bdr *tx_ring, struct sk_buff *skb, 731 int *i, struct enetc_lso_t *lso, int *count) 732 { 733 union enetc_tx_bd txbd_tmp, *txbd = NULL; 734 struct enetc_tx_swbd *tx_swbd; 735 skb_frag_t *frag; 736 dma_addr_t dma; 737 u8 flags = 0; 738 int len, f; 739 740 len = skb_headlen(skb) - lso->hdr_len; 741 if (len > 0) { 742 dma = dma_map_single(tx_ring->dev, skb->data + lso->hdr_len, 743 len, DMA_TO_DEVICE); 744 if (dma_mapping_error(tx_ring->dev, dma)) 745 return -ENOMEM; 746 747 enetc_bdr_idx_inc(tx_ring, i); 748 txbd = ENETC_TXBD(*tx_ring, *i); 749 tx_swbd = &tx_ring->tx_swbd[*i]; 750 prefetchw(txbd); 751 *count += 1; 752 753 enetc_clear_tx_bd(&txbd_tmp); 754 txbd_tmp.addr = cpu_to_le64(dma); 755 txbd_tmp.buf_len = cpu_to_le16(len); 756 757 tx_swbd->dma = dma; 758 tx_swbd->len = len; 759 tx_swbd->is_dma_page = 0; 760 tx_swbd->dir = DMA_TO_DEVICE; 761 } 762 763 frag = &skb_shinfo(skb)->frags[0]; 764 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++, frag++) { 765 if (txbd) 766 *txbd = txbd_tmp; 767 768 len = skb_frag_size(frag); 769 dma = skb_frag_dma_map(tx_ring->dev, frag); 770 if (dma_mapping_error(tx_ring->dev, dma)) 771 return -ENOMEM; 772 773 /* Get the next BD */ 774 enetc_bdr_idx_inc(tx_ring, i); 775 txbd = ENETC_TXBD(*tx_ring, *i); 776 tx_swbd = &tx_ring->tx_swbd[*i]; 777 prefetchw(txbd); 778 *count += 1; 779 780 enetc_clear_tx_bd(&txbd_tmp); 781 txbd_tmp.addr = cpu_to_le64(dma); 782 txbd_tmp.buf_len = cpu_to_le16(len); 783 784 tx_swbd->dma = dma; 785 tx_swbd->len = len; 786 tx_swbd->is_dma_page = 1; 787 tx_swbd->dir = DMA_TO_DEVICE; 788 } 789 790 /* Last BD needs 'F' bit set */ 791 flags |= ENETC_TXBD_FLAGS_F; 792 txbd_tmp.flags = flags; 793 *txbd = txbd_tmp; 794 795 tx_swbd->is_eof = 1; 796 tx_swbd->skb = skb; 797 798 return 0; 799 } 800 801 static int enetc_lso_hw_offload(struct enetc_bdr *tx_ring, struct sk_buff *skb) 802 { 803 struct enetc_tx_swbd *tx_swbd; 804 struct enetc_lso_t lso = {0}; 805 int err, i, count = 0; 806 807 /* Initialize the LSO handler */ 808 enetc_lso_start(skb, &lso); 809 i = tx_ring->next_to_use; 810 811 enetc_lso_map_hdr(tx_ring, skb, &i, &lso); 812 /* First BD and an extend BD */ 813 count += 2; 814 815 err = enetc_lso_map_data(tx_ring, skb, &i, &lso, &count); 816 if (err) 817 goto dma_err; 818 819 /* Go to the next BD */ 820 enetc_bdr_idx_inc(tx_ring, &i); 821 tx_ring->next_to_use = i; 822 enetc_update_tx_ring_tail(tx_ring); 823 824 return count; 825 826 dma_err: 827 do { 828 tx_swbd = &tx_ring->tx_swbd[i]; 829 enetc_free_tx_frame(tx_ring, tx_swbd); 830 if (i == 0) 831 i = tx_ring->bd_count; 832 i--; 833 } while (--count); 834 835 return 0; 836 } 837 838 static int enetc_map_tx_tso_buffs(struct enetc_bdr *tx_ring, struct sk_buff *skb) 839 { 840 struct enetc_ndev_priv *priv = netdev_priv(tx_ring->ndev); 841 int hdr_len, total_len, data_len; 842 struct enetc_tx_swbd *tx_swbd; 843 union enetc_tx_bd *txbd; 844 struct tso_t tso; 845 __wsum csum, csum2; 846 int count = 0, pos; 847 int err, i, bd_data_num; 848 849 /* Initialize the TSO handler, and prepare the first payload */ 850 hdr_len = tso_start(skb, &tso); 851 total_len = skb->len - hdr_len; 852 i = tx_ring->next_to_use; 853 854 while (total_len > 0) { 855 char *hdr; 856 857 /* Get the BD */ 858 txbd = ENETC_TXBD(*tx_ring, i); 859 tx_swbd = &tx_ring->tx_swbd[i]; 860 prefetchw(txbd); 861 862 /* Determine the length of this packet */ 863 data_len = min_t(int, skb_shinfo(skb)->gso_size, total_len); 864 total_len -= data_len; 865 866 /* prepare packet headers: MAC + IP + TCP */ 867 hdr = tx_ring->tso_headers + i * TSO_HEADER_SIZE; 868 tso_build_hdr(skb, hdr, &tso, data_len, total_len == 0); 869 870 /* compute the csum over the L4 header */ 871 csum = enetc_tso_hdr_csum(&tso, skb, hdr, hdr_len, &pos); 872 count += enetc_map_tx_tso_hdr(tx_ring, skb, tx_swbd, txbd, 873 &i, hdr_len, data_len); 874 bd_data_num = 0; 875 876 while (data_len > 0) { 877 int size; 878 879 size = min_t(int, tso.size, data_len); 880 881 /* Advance the index in the BDR */ 882 enetc_bdr_idx_inc(tx_ring, &i); 883 txbd = ENETC_TXBD(*tx_ring, i); 884 tx_swbd = &tx_ring->tx_swbd[i]; 885 prefetchw(txbd); 886 887 /* Compute the checksum over this segment of data and 888 * add it to the csum already computed (over the L4 889 * header and possible other data segments). 890 */ 891 csum2 = csum_partial(tso.data, size, 0); 892 csum = csum_block_add(csum, csum2, pos); 893 pos += size; 894 895 err = enetc_map_tx_tso_data(tx_ring, skb, tx_swbd, txbd, 896 tso.data, size, 897 size == data_len); 898 if (err) { 899 if (i == 0) 900 i = tx_ring->bd_count; 901 i--; 902 903 goto err_map_data; 904 } 905 906 data_len -= size; 907 count++; 908 bd_data_num++; 909 tso_build_data(skb, &tso, size); 910 911 if (unlikely(bd_data_num >= priv->max_frags && data_len)) 912 goto err_chained_bd; 913 } 914 915 enetc_tso_complete_csum(tx_ring, &tso, skb, hdr, pos, csum); 916 917 if (total_len == 0) 918 tx_swbd->skb = skb; 919 920 /* Go to the next BD */ 921 enetc_bdr_idx_inc(tx_ring, &i); 922 } 923 924 tx_ring->next_to_use = i; 925 enetc_update_tx_ring_tail(tx_ring); 926 927 return count; 928 929 err_map_data: 930 dev_err(tx_ring->dev, "DMA map error"); 931 932 err_chained_bd: 933 enetc_unwind_tx_frame(tx_ring, count, i); 934 935 return 0; 936 } 937 938 static netdev_tx_t enetc_start_xmit(struct sk_buff *skb, 939 struct net_device *ndev) 940 { 941 struct enetc_ndev_priv *priv = netdev_priv(ndev); 942 struct enetc_bdr *tx_ring; 943 int count; 944 945 /* Queue one-step Sync packet if already locked */ 946 if (skb->cb[0] & ENETC_F_TX_ONESTEP_SYNC_TSTAMP) { 947 if (test_and_set_bit_lock(ENETC_TX_ONESTEP_TSTAMP_IN_PROGRESS, 948 &priv->flags)) { 949 skb_queue_tail(&priv->tx_skbs, skb); 950 return NETDEV_TX_OK; 951 } 952 } 953 954 tx_ring = priv->tx_ring[skb->queue_mapping]; 955 956 if (skb_is_gso(skb)) { 957 /* LSO data unit lengths of up to 256KB are supported */ 958 if (priv->active_offloads & ENETC_F_LSO && 959 (skb->len - enetc_lso_get_hdr_len(skb)) <= 960 ENETC_LSO_MAX_DATA_LEN) { 961 if (enetc_bd_unused(tx_ring) < enetc_lso_count_descs(skb)) { 962 netif_stop_subqueue(ndev, tx_ring->index); 963 return NETDEV_TX_BUSY; 964 } 965 966 count = enetc_lso_hw_offload(tx_ring, skb); 967 } else { 968 if (enetc_bd_unused(tx_ring) < tso_count_descs(skb)) { 969 netif_stop_subqueue(ndev, tx_ring->index); 970 return NETDEV_TX_BUSY; 971 } 972 973 enetc_lock_mdio(); 974 count = enetc_map_tx_tso_buffs(tx_ring, skb); 975 enetc_unlock_mdio(); 976 } 977 } else { 978 if (unlikely(skb_shinfo(skb)->nr_frags > priv->max_frags)) 979 if (unlikely(skb_linearize(skb))) 980 goto drop_packet_err; 981 982 count = skb_shinfo(skb)->nr_frags + 1; /* fragments + head */ 983 if (enetc_bd_unused(tx_ring) < ENETC_TXBDS_NEEDED(count)) { 984 netif_stop_subqueue(ndev, tx_ring->index); 985 return NETDEV_TX_BUSY; 986 } 987 988 enetc_lock_mdio(); 989 count = enetc_map_tx_buffs(tx_ring, skb); 990 enetc_unlock_mdio(); 991 } 992 993 if (unlikely(!count)) 994 goto drop_packet_err; 995 996 if (enetc_bd_unused(tx_ring) < ENETC_TXBDS_MAX_NEEDED(priv->max_frags)) 997 netif_stop_subqueue(ndev, tx_ring->index); 998 999 return NETDEV_TX_OK; 1000 1001 drop_packet_err: 1002 dev_kfree_skb_any(skb); 1003 return NETDEV_TX_OK; 1004 } 1005 1006 netdev_tx_t enetc_xmit(struct sk_buff *skb, struct net_device *ndev) 1007 { 1008 struct enetc_ndev_priv *priv = netdev_priv(ndev); 1009 u8 udp, msgtype, twostep; 1010 u16 offset1, offset2; 1011 1012 /* Mark tx timestamp type on skb->cb[0] if requires */ 1013 if ((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && 1014 (priv->active_offloads & ENETC_F_TX_TSTAMP_MASK)) { 1015 skb->cb[0] = priv->active_offloads & ENETC_F_TX_TSTAMP_MASK; 1016 } else { 1017 skb->cb[0] = 0; 1018 } 1019 1020 /* Fall back to two-step timestamp if not one-step Sync packet */ 1021 if (skb->cb[0] & ENETC_F_TX_ONESTEP_SYNC_TSTAMP) { 1022 if (enetc_ptp_parse(skb, &udp, &msgtype, &twostep, 1023 &offset1, &offset2) || 1024 msgtype != PTP_MSGTYPE_SYNC || twostep != 0) 1025 skb->cb[0] = ENETC_F_TX_TSTAMP; 1026 } 1027 1028 return enetc_start_xmit(skb, ndev); 1029 } 1030 EXPORT_SYMBOL_GPL(enetc_xmit); 1031 1032 static irqreturn_t enetc_msix(int irq, void *data) 1033 { 1034 struct enetc_int_vector *v = data; 1035 int i; 1036 1037 enetc_lock_mdio(); 1038 1039 /* disable interrupts */ 1040 enetc_wr_reg_hot(v->rbier, 0); 1041 enetc_wr_reg_hot(v->ricr1, v->rx_ictt); 1042 1043 for_each_set_bit(i, &v->tx_rings_map, ENETC_MAX_NUM_TXQS) 1044 enetc_wr_reg_hot(v->tbier_base + ENETC_BDR_OFF(i), 0); 1045 1046 enetc_unlock_mdio(); 1047 1048 napi_schedule(&v->napi); 1049 1050 return IRQ_HANDLED; 1051 } 1052 1053 static void enetc_rx_dim_work(struct work_struct *w) 1054 { 1055 struct dim *dim = container_of(w, struct dim, work); 1056 struct dim_cq_moder moder = 1057 net_dim_get_rx_moderation(dim->mode, dim->profile_ix); 1058 struct enetc_int_vector *v = 1059 container_of(dim, struct enetc_int_vector, rx_dim); 1060 struct enetc_ndev_priv *priv = netdev_priv(v->rx_ring.ndev); 1061 1062 v->rx_ictt = enetc_usecs_to_cycles(moder.usec, priv->sysclk_freq); 1063 dim->state = DIM_START_MEASURE; 1064 } 1065 1066 static void enetc_rx_net_dim(struct enetc_int_vector *v) 1067 { 1068 struct dim_sample dim_sample = {}; 1069 1070 v->comp_cnt++; 1071 1072 if (!v->rx_napi_work) 1073 return; 1074 1075 dim_update_sample(v->comp_cnt, 1076 v->rx_ring.stats.packets, 1077 v->rx_ring.stats.bytes, 1078 &dim_sample); 1079 net_dim(&v->rx_dim, &dim_sample); 1080 } 1081 1082 static int enetc_bd_ready_count(struct enetc_bdr *tx_ring, int ci) 1083 { 1084 int pi = enetc_rd_reg_hot(tx_ring->tcir) & ENETC_TBCIR_IDX_MASK; 1085 1086 return pi >= ci ? pi - ci : tx_ring->bd_count - ci + pi; 1087 } 1088 1089 static bool enetc_page_reusable(struct page *page) 1090 { 1091 return (!page_is_pfmemalloc(page) && page_ref_count(page) == 1); 1092 } 1093 1094 static void enetc_reuse_page(struct enetc_bdr *rx_ring, 1095 struct enetc_rx_swbd *old) 1096 { 1097 struct enetc_rx_swbd *new; 1098 1099 new = &rx_ring->rx_swbd[rx_ring->next_to_alloc]; 1100 1101 /* next buf that may reuse a page */ 1102 enetc_bdr_idx_inc(rx_ring, &rx_ring->next_to_alloc); 1103 1104 /* copy page reference */ 1105 *new = *old; 1106 } 1107 1108 static void enetc_get_tx_tstamp(struct enetc_hw *hw, union enetc_tx_bd *txbd, 1109 u64 *tstamp) 1110 { 1111 u32 lo, hi, tstamp_lo; 1112 1113 lo = enetc_rd_hot(hw, ENETC_SICTR0); 1114 hi = enetc_rd_hot(hw, ENETC_SICTR1); 1115 tstamp_lo = le32_to_cpu(txbd->wb.tstamp); 1116 if (lo <= tstamp_lo) 1117 hi -= 1; 1118 *tstamp = (u64)hi << 32 | tstamp_lo; 1119 } 1120 1121 static void enetc_tstamp_tx(struct sk_buff *skb, u64 tstamp) 1122 { 1123 struct skb_shared_hwtstamps shhwtstamps; 1124 1125 if (skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS) { 1126 memset(&shhwtstamps, 0, sizeof(shhwtstamps)); 1127 shhwtstamps.hwtstamp = ns_to_ktime(tstamp); 1128 skb_txtime_consumed(skb); 1129 skb_tstamp_tx(skb, &shhwtstamps); 1130 } 1131 } 1132 1133 static void enetc_recycle_xdp_tx_buff(struct enetc_bdr *tx_ring, 1134 struct enetc_tx_swbd *tx_swbd) 1135 { 1136 struct enetc_ndev_priv *priv = netdev_priv(tx_ring->ndev); 1137 struct enetc_rx_swbd rx_swbd = { 1138 .dma = tx_swbd->dma, 1139 .page = tx_swbd->page, 1140 .page_offset = tx_swbd->page_offset, 1141 .dir = tx_swbd->dir, 1142 .len = tx_swbd->len, 1143 }; 1144 struct enetc_bdr *rx_ring; 1145 1146 rx_ring = enetc_rx_ring_from_xdp_tx_ring(priv, tx_ring); 1147 1148 if (likely(enetc_swbd_unused(rx_ring))) { 1149 enetc_reuse_page(rx_ring, &rx_swbd); 1150 1151 /* sync for use by the device */ 1152 dma_sync_single_range_for_device(rx_ring->dev, rx_swbd.dma, 1153 rx_swbd.page_offset, 1154 ENETC_RXB_DMA_SIZE_XDP, 1155 rx_swbd.dir); 1156 1157 rx_ring->stats.recycles++; 1158 } else { 1159 /* RX ring is already full, we need to unmap and free the 1160 * page, since there's nothing useful we can do with it. 1161 */ 1162 rx_ring->stats.recycle_failures++; 1163 1164 dma_unmap_page(rx_ring->dev, rx_swbd.dma, PAGE_SIZE, 1165 rx_swbd.dir); 1166 __free_page(rx_swbd.page); 1167 } 1168 1169 rx_ring->xdp.xdp_tx_in_flight--; 1170 } 1171 1172 static bool enetc_clean_tx_ring(struct enetc_bdr *tx_ring, int napi_budget) 1173 { 1174 int tx_frm_cnt = 0, tx_byte_cnt = 0, tx_win_drop = 0; 1175 struct net_device *ndev = tx_ring->ndev; 1176 struct enetc_ndev_priv *priv = netdev_priv(ndev); 1177 struct enetc_tx_swbd *tx_swbd; 1178 int i, bds_to_clean; 1179 bool do_twostep_tstamp; 1180 u64 tstamp = 0; 1181 1182 i = tx_ring->next_to_clean; 1183 tx_swbd = &tx_ring->tx_swbd[i]; 1184 1185 bds_to_clean = enetc_bd_ready_count(tx_ring, i); 1186 1187 do_twostep_tstamp = false; 1188 1189 while (bds_to_clean && tx_frm_cnt < ENETC_DEFAULT_TX_WORK) { 1190 struct xdp_frame *xdp_frame = enetc_tx_swbd_get_xdp_frame(tx_swbd); 1191 struct sk_buff *skb = enetc_tx_swbd_get_skb(tx_swbd); 1192 bool is_eof = tx_swbd->is_eof; 1193 1194 if (unlikely(tx_swbd->check_wb)) { 1195 union enetc_tx_bd *txbd = ENETC_TXBD(*tx_ring, i); 1196 1197 if (txbd->flags & ENETC_TXBD_FLAGS_W && 1198 tx_swbd->do_twostep_tstamp) { 1199 enetc_get_tx_tstamp(&priv->si->hw, txbd, 1200 &tstamp); 1201 do_twostep_tstamp = true; 1202 } 1203 1204 if (tx_swbd->qbv_en && 1205 txbd->wb.status & ENETC_TXBD_STATS_WIN) 1206 tx_win_drop++; 1207 } 1208 1209 if (tx_swbd->is_xdp_tx) 1210 enetc_recycle_xdp_tx_buff(tx_ring, tx_swbd); 1211 else if (likely(tx_swbd->dma)) 1212 enetc_unmap_tx_buff(tx_ring, tx_swbd); 1213 1214 if (xdp_frame) { 1215 xdp_return_frame(xdp_frame); 1216 } else if (skb) { 1217 if (unlikely(skb->cb[0] & ENETC_F_TX_ONESTEP_SYNC_TSTAMP)) { 1218 /* Start work to release lock for next one-step 1219 * timestamping packet. And send one skb in 1220 * tx_skbs queue if has. 1221 */ 1222 schedule_work(&priv->tx_onestep_tstamp); 1223 } else if (unlikely(do_twostep_tstamp)) { 1224 enetc_tstamp_tx(skb, tstamp); 1225 do_twostep_tstamp = false; 1226 } 1227 napi_consume_skb(skb, napi_budget); 1228 } 1229 1230 tx_byte_cnt += tx_swbd->len; 1231 /* Scrub the swbd here so we don't have to do that 1232 * when we reuse it during xmit 1233 */ 1234 memset(tx_swbd, 0, sizeof(*tx_swbd)); 1235 1236 bds_to_clean--; 1237 tx_swbd++; 1238 i++; 1239 if (unlikely(i == tx_ring->bd_count)) { 1240 i = 0; 1241 tx_swbd = tx_ring->tx_swbd; 1242 } 1243 1244 /* BD iteration loop end */ 1245 if (is_eof) { 1246 tx_frm_cnt++; 1247 /* re-arm interrupt source */ 1248 enetc_wr_reg_hot(tx_ring->idr, BIT(tx_ring->index) | 1249 BIT(16 + tx_ring->index)); 1250 } 1251 1252 if (unlikely(!bds_to_clean)) 1253 bds_to_clean = enetc_bd_ready_count(tx_ring, i); 1254 } 1255 1256 tx_ring->next_to_clean = i; 1257 tx_ring->stats.packets += tx_frm_cnt; 1258 tx_ring->stats.bytes += tx_byte_cnt; 1259 tx_ring->stats.win_drop += tx_win_drop; 1260 1261 if (unlikely(tx_frm_cnt && netif_carrier_ok(ndev) && 1262 __netif_subqueue_stopped(ndev, tx_ring->index) && 1263 !test_bit(ENETC_TX_DOWN, &priv->flags) && 1264 (enetc_bd_unused(tx_ring) >= 1265 ENETC_TXBDS_MAX_NEEDED(priv->max_frags)))) { 1266 netif_wake_subqueue(ndev, tx_ring->index); 1267 } 1268 1269 return tx_frm_cnt != ENETC_DEFAULT_TX_WORK; 1270 } 1271 1272 static bool enetc_new_page(struct enetc_bdr *rx_ring, 1273 struct enetc_rx_swbd *rx_swbd) 1274 { 1275 bool xdp = !!(rx_ring->xdp.prog); 1276 struct page *page; 1277 dma_addr_t addr; 1278 1279 page = dev_alloc_page(); 1280 if (unlikely(!page)) 1281 return false; 1282 1283 /* For XDP_TX, we forgo dma_unmap -> dma_map */ 1284 rx_swbd->dir = xdp ? DMA_BIDIRECTIONAL : DMA_FROM_DEVICE; 1285 1286 addr = dma_map_page(rx_ring->dev, page, 0, PAGE_SIZE, rx_swbd->dir); 1287 if (unlikely(dma_mapping_error(rx_ring->dev, addr))) { 1288 __free_page(page); 1289 1290 return false; 1291 } 1292 1293 rx_swbd->dma = addr; 1294 rx_swbd->page = page; 1295 rx_swbd->page_offset = rx_ring->buffer_offset; 1296 1297 return true; 1298 } 1299 1300 static int enetc_refill_rx_ring(struct enetc_bdr *rx_ring, const int buff_cnt) 1301 { 1302 struct enetc_rx_swbd *rx_swbd; 1303 union enetc_rx_bd *rxbd; 1304 int i, j; 1305 1306 i = rx_ring->next_to_use; 1307 rx_swbd = &rx_ring->rx_swbd[i]; 1308 rxbd = enetc_rxbd(rx_ring, i); 1309 1310 for (j = 0; j < buff_cnt; j++) { 1311 /* try reuse page */ 1312 if (unlikely(!rx_swbd->page)) { 1313 if (unlikely(!enetc_new_page(rx_ring, rx_swbd))) { 1314 rx_ring->stats.rx_alloc_errs++; 1315 break; 1316 } 1317 } 1318 1319 /* update RxBD */ 1320 rxbd->w.addr = cpu_to_le64(rx_swbd->dma + 1321 rx_swbd->page_offset); 1322 /* clear 'R" as well */ 1323 rxbd->r.lstatus = 0; 1324 1325 enetc_rxbd_next(rx_ring, &rxbd, &i); 1326 rx_swbd = &rx_ring->rx_swbd[i]; 1327 } 1328 1329 if (likely(j)) { 1330 rx_ring->next_to_alloc = i; /* keep track from page reuse */ 1331 rx_ring->next_to_use = i; 1332 1333 /* update ENETC's consumer index */ 1334 enetc_wr_reg_hot(rx_ring->rcir, rx_ring->next_to_use); 1335 } 1336 1337 return j; 1338 } 1339 1340 static void enetc_get_rx_tstamp(struct net_device *ndev, 1341 union enetc_rx_bd *rxbd, 1342 struct sk_buff *skb) 1343 { 1344 struct skb_shared_hwtstamps *shhwtstamps = skb_hwtstamps(skb); 1345 struct enetc_ndev_priv *priv = netdev_priv(ndev); 1346 struct enetc_hw *hw = &priv->si->hw; 1347 u32 lo, hi, tstamp_lo; 1348 u64 tstamp; 1349 1350 if (le16_to_cpu(rxbd->r.flags) & ENETC_RXBD_FLAG_TSTMP) { 1351 lo = enetc_rd_reg_hot(hw->reg + ENETC_SICTR0); 1352 hi = enetc_rd_reg_hot(hw->reg + ENETC_SICTR1); 1353 rxbd = enetc_rxbd_ext(rxbd); 1354 tstamp_lo = le32_to_cpu(rxbd->ext.tstamp); 1355 if (lo <= tstamp_lo) 1356 hi -= 1; 1357 1358 tstamp = (u64)hi << 32 | tstamp_lo; 1359 memset(shhwtstamps, 0, sizeof(*shhwtstamps)); 1360 shhwtstamps->hwtstamp = ns_to_ktime(tstamp); 1361 } 1362 } 1363 1364 static void enetc_get_offloads(struct enetc_bdr *rx_ring, 1365 union enetc_rx_bd *rxbd, struct sk_buff *skb) 1366 { 1367 struct enetc_ndev_priv *priv = netdev_priv(rx_ring->ndev); 1368 1369 /* TODO: hashing */ 1370 if (rx_ring->ndev->features & NETIF_F_RXCSUM) { 1371 u16 inet_csum = le16_to_cpu(rxbd->r.inet_csum); 1372 1373 skb->csum = csum_unfold((__force __sum16)~htons(inet_csum)); 1374 skb->ip_summed = CHECKSUM_COMPLETE; 1375 } 1376 1377 if (le16_to_cpu(rxbd->r.flags) & ENETC_RXBD_FLAG_VLAN) { 1378 struct enetc_hw *hw = &priv->si->hw; 1379 __be16 tpid = 0; 1380 1381 switch (le16_to_cpu(rxbd->r.flags) & ENETC_RXBD_FLAG_TPID) { 1382 case 0: 1383 tpid = htons(ETH_P_8021Q); 1384 break; 1385 case 1: 1386 tpid = htons(ETH_P_8021AD); 1387 break; 1388 case 2: 1389 tpid = htons(enetc_rd_hot(hw, ENETC_SICVLANR1) & 1390 SICVLANR_ETYPE); 1391 break; 1392 case 3: 1393 tpid = htons(enetc_rd_hot(hw, ENETC_SICVLANR2) & 1394 SICVLANR_ETYPE); 1395 } 1396 1397 __vlan_hwaccel_put_tag(skb, tpid, le16_to_cpu(rxbd->r.vlan_opt)); 1398 } 1399 1400 if (IS_ENABLED(CONFIG_FSL_ENETC_PTP_CLOCK) && 1401 (priv->active_offloads & ENETC_F_RX_TSTAMP)) 1402 enetc_get_rx_tstamp(rx_ring->ndev, rxbd, skb); 1403 } 1404 1405 /* This gets called during the non-XDP NAPI poll cycle as well as on XDP_PASS, 1406 * so it needs to work with both DMA_FROM_DEVICE as well as DMA_BIDIRECTIONAL 1407 * mapped buffers. 1408 */ 1409 static struct enetc_rx_swbd *enetc_get_rx_buff(struct enetc_bdr *rx_ring, 1410 int i, u16 size) 1411 { 1412 struct enetc_rx_swbd *rx_swbd = &rx_ring->rx_swbd[i]; 1413 1414 dma_sync_single_range_for_cpu(rx_ring->dev, rx_swbd->dma, 1415 rx_swbd->page_offset, 1416 size, rx_swbd->dir); 1417 return rx_swbd; 1418 } 1419 1420 /* Reuse the current page without performing half-page buffer flipping */ 1421 static void enetc_put_rx_buff(struct enetc_bdr *rx_ring, 1422 struct enetc_rx_swbd *rx_swbd) 1423 { 1424 size_t buffer_size = ENETC_RXB_TRUESIZE - rx_ring->buffer_offset; 1425 1426 enetc_reuse_page(rx_ring, rx_swbd); 1427 1428 dma_sync_single_range_for_device(rx_ring->dev, rx_swbd->dma, 1429 rx_swbd->page_offset, 1430 buffer_size, rx_swbd->dir); 1431 1432 rx_swbd->page = NULL; 1433 } 1434 1435 /* Reuse the current page by performing half-page buffer flipping */ 1436 static void enetc_flip_rx_buff(struct enetc_bdr *rx_ring, 1437 struct enetc_rx_swbd *rx_swbd) 1438 { 1439 if (likely(enetc_page_reusable(rx_swbd->page))) { 1440 rx_swbd->page_offset ^= ENETC_RXB_TRUESIZE; 1441 page_ref_inc(rx_swbd->page); 1442 1443 enetc_put_rx_buff(rx_ring, rx_swbd); 1444 } else { 1445 dma_unmap_page(rx_ring->dev, rx_swbd->dma, PAGE_SIZE, 1446 rx_swbd->dir); 1447 rx_swbd->page = NULL; 1448 } 1449 } 1450 1451 static struct sk_buff *enetc_map_rx_buff_to_skb(struct enetc_bdr *rx_ring, 1452 int i, u16 size) 1453 { 1454 struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size); 1455 struct sk_buff *skb; 1456 void *ba; 1457 1458 ba = page_address(rx_swbd->page) + rx_swbd->page_offset; 1459 skb = build_skb(ba - rx_ring->buffer_offset, ENETC_RXB_TRUESIZE); 1460 if (unlikely(!skb)) { 1461 rx_ring->stats.rx_alloc_errs++; 1462 return NULL; 1463 } 1464 1465 skb_reserve(skb, rx_ring->buffer_offset); 1466 __skb_put(skb, size); 1467 1468 enetc_flip_rx_buff(rx_ring, rx_swbd); 1469 1470 return skb; 1471 } 1472 1473 static void enetc_add_rx_buff_to_skb(struct enetc_bdr *rx_ring, int i, 1474 u16 size, struct sk_buff *skb) 1475 { 1476 struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size); 1477 1478 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_swbd->page, 1479 rx_swbd->page_offset, size, ENETC_RXB_TRUESIZE); 1480 1481 enetc_flip_rx_buff(rx_ring, rx_swbd); 1482 } 1483 1484 static bool enetc_check_bd_errors_and_consume(struct enetc_bdr *rx_ring, 1485 u32 bd_status, 1486 union enetc_rx_bd **rxbd, int *i) 1487 { 1488 if (likely(!(bd_status & ENETC_RXBD_LSTATUS(ENETC_RXBD_ERR_MASK)))) 1489 return false; 1490 1491 enetc_put_rx_buff(rx_ring, &rx_ring->rx_swbd[*i]); 1492 enetc_rxbd_next(rx_ring, rxbd, i); 1493 1494 while (!(bd_status & ENETC_RXBD_LSTATUS_F)) { 1495 dma_rmb(); 1496 bd_status = le32_to_cpu((*rxbd)->r.lstatus); 1497 1498 enetc_put_rx_buff(rx_ring, &rx_ring->rx_swbd[*i]); 1499 enetc_rxbd_next(rx_ring, rxbd, i); 1500 } 1501 1502 rx_ring->ndev->stats.rx_dropped++; 1503 rx_ring->ndev->stats.rx_errors++; 1504 1505 return true; 1506 } 1507 1508 static struct sk_buff *enetc_build_skb(struct enetc_bdr *rx_ring, 1509 u32 bd_status, union enetc_rx_bd **rxbd, 1510 int *i, int *cleaned_cnt, int buffer_size) 1511 { 1512 struct sk_buff *skb; 1513 u16 size; 1514 1515 size = le16_to_cpu((*rxbd)->r.buf_len); 1516 skb = enetc_map_rx_buff_to_skb(rx_ring, *i, size); 1517 if (!skb) 1518 return NULL; 1519 1520 enetc_get_offloads(rx_ring, *rxbd, skb); 1521 1522 (*cleaned_cnt)++; 1523 1524 enetc_rxbd_next(rx_ring, rxbd, i); 1525 1526 /* not last BD in frame? */ 1527 while (!(bd_status & ENETC_RXBD_LSTATUS_F)) { 1528 bd_status = le32_to_cpu((*rxbd)->r.lstatus); 1529 size = buffer_size; 1530 1531 if (bd_status & ENETC_RXBD_LSTATUS_F) { 1532 dma_rmb(); 1533 size = le16_to_cpu((*rxbd)->r.buf_len); 1534 } 1535 1536 enetc_add_rx_buff_to_skb(rx_ring, *i, size, skb); 1537 1538 (*cleaned_cnt)++; 1539 1540 enetc_rxbd_next(rx_ring, rxbd, i); 1541 } 1542 1543 skb_record_rx_queue(skb, rx_ring->index); 1544 skb->protocol = eth_type_trans(skb, rx_ring->ndev); 1545 1546 return skb; 1547 } 1548 1549 #define ENETC_RXBD_BUNDLE 16 /* # of BDs to update at once */ 1550 1551 static int enetc_clean_rx_ring(struct enetc_bdr *rx_ring, 1552 struct napi_struct *napi, int work_limit) 1553 { 1554 int rx_frm_cnt = 0, rx_byte_cnt = 0; 1555 int cleaned_cnt, i; 1556 1557 cleaned_cnt = enetc_bd_unused(rx_ring); 1558 /* next descriptor to process */ 1559 i = rx_ring->next_to_clean; 1560 1561 while (likely(rx_frm_cnt < work_limit)) { 1562 union enetc_rx_bd *rxbd; 1563 struct sk_buff *skb; 1564 u32 bd_status; 1565 1566 if (cleaned_cnt >= ENETC_RXBD_BUNDLE) 1567 cleaned_cnt -= enetc_refill_rx_ring(rx_ring, 1568 cleaned_cnt); 1569 1570 rxbd = enetc_rxbd(rx_ring, i); 1571 bd_status = le32_to_cpu(rxbd->r.lstatus); 1572 if (!bd_status) 1573 break; 1574 1575 enetc_wr_reg_hot(rx_ring->idr, BIT(rx_ring->index)); 1576 dma_rmb(); /* for reading other rxbd fields */ 1577 1578 if (enetc_check_bd_errors_and_consume(rx_ring, bd_status, 1579 &rxbd, &i)) 1580 break; 1581 1582 skb = enetc_build_skb(rx_ring, bd_status, &rxbd, &i, 1583 &cleaned_cnt, ENETC_RXB_DMA_SIZE); 1584 if (!skb) 1585 break; 1586 1587 /* When set, the outer VLAN header is extracted and reported 1588 * in the receive buffer descriptor. So rx_byte_cnt should 1589 * add the length of the extracted VLAN header. 1590 */ 1591 if (bd_status & ENETC_RXBD_FLAG_VLAN) 1592 rx_byte_cnt += VLAN_HLEN; 1593 rx_byte_cnt += skb->len + ETH_HLEN; 1594 rx_frm_cnt++; 1595 1596 napi_gro_receive(napi, skb); 1597 } 1598 1599 rx_ring->next_to_clean = i; 1600 1601 rx_ring->stats.packets += rx_frm_cnt; 1602 rx_ring->stats.bytes += rx_byte_cnt; 1603 1604 return rx_frm_cnt; 1605 } 1606 1607 static void enetc_xdp_map_tx_buff(struct enetc_bdr *tx_ring, int i, 1608 struct enetc_tx_swbd *tx_swbd, 1609 int frm_len) 1610 { 1611 union enetc_tx_bd *txbd = ENETC_TXBD(*tx_ring, i); 1612 1613 prefetchw(txbd); 1614 1615 enetc_clear_tx_bd(txbd); 1616 txbd->addr = cpu_to_le64(tx_swbd->dma + tx_swbd->page_offset); 1617 txbd->buf_len = cpu_to_le16(tx_swbd->len); 1618 txbd->frm_len = cpu_to_le16(frm_len); 1619 1620 memcpy(&tx_ring->tx_swbd[i], tx_swbd, sizeof(*tx_swbd)); 1621 } 1622 1623 /* Puts in the TX ring one XDP frame, mapped as an array of TX software buffer 1624 * descriptors. 1625 */ 1626 static bool enetc_xdp_tx(struct enetc_bdr *tx_ring, 1627 struct enetc_tx_swbd *xdp_tx_arr, int num_tx_swbd) 1628 { 1629 struct enetc_tx_swbd *tmp_tx_swbd = xdp_tx_arr; 1630 int i, k, frm_len = tmp_tx_swbd->len; 1631 1632 if (unlikely(enetc_bd_unused(tx_ring) < ENETC_TXBDS_NEEDED(num_tx_swbd))) 1633 return false; 1634 1635 while (unlikely(!tmp_tx_swbd->is_eof)) { 1636 tmp_tx_swbd++; 1637 frm_len += tmp_tx_swbd->len; 1638 } 1639 1640 i = tx_ring->next_to_use; 1641 1642 for (k = 0; k < num_tx_swbd; k++) { 1643 struct enetc_tx_swbd *xdp_tx_swbd = &xdp_tx_arr[k]; 1644 1645 enetc_xdp_map_tx_buff(tx_ring, i, xdp_tx_swbd, frm_len); 1646 1647 /* last BD needs 'F' bit set */ 1648 if (xdp_tx_swbd->is_eof) { 1649 union enetc_tx_bd *txbd = ENETC_TXBD(*tx_ring, i); 1650 1651 txbd->flags = ENETC_TXBD_FLAGS_F; 1652 } 1653 1654 enetc_bdr_idx_inc(tx_ring, &i); 1655 } 1656 1657 tx_ring->next_to_use = i; 1658 1659 return true; 1660 } 1661 1662 static int enetc_xdp_frame_to_xdp_tx_swbd(struct enetc_bdr *tx_ring, 1663 struct enetc_tx_swbd *xdp_tx_arr, 1664 struct xdp_frame *xdp_frame) 1665 { 1666 struct enetc_tx_swbd *xdp_tx_swbd = &xdp_tx_arr[0]; 1667 struct skb_shared_info *shinfo; 1668 void *data = xdp_frame->data; 1669 int len = xdp_frame->len; 1670 skb_frag_t *frag; 1671 dma_addr_t dma; 1672 unsigned int f; 1673 int n = 0; 1674 1675 dma = dma_map_single(tx_ring->dev, data, len, DMA_TO_DEVICE); 1676 if (unlikely(dma_mapping_error(tx_ring->dev, dma))) { 1677 netdev_err(tx_ring->ndev, "DMA map error\n"); 1678 return -1; 1679 } 1680 1681 xdp_tx_swbd->dma = dma; 1682 xdp_tx_swbd->dir = DMA_TO_DEVICE; 1683 xdp_tx_swbd->len = len; 1684 xdp_tx_swbd->is_xdp_redirect = true; 1685 xdp_tx_swbd->is_eof = false; 1686 xdp_tx_swbd->xdp_frame = NULL; 1687 1688 n++; 1689 1690 if (!xdp_frame_has_frags(xdp_frame)) 1691 goto out; 1692 1693 xdp_tx_swbd = &xdp_tx_arr[n]; 1694 1695 shinfo = xdp_get_shared_info_from_frame(xdp_frame); 1696 1697 for (f = 0, frag = &shinfo->frags[0]; f < shinfo->nr_frags; 1698 f++, frag++) { 1699 data = skb_frag_address(frag); 1700 len = skb_frag_size(frag); 1701 1702 dma = dma_map_single(tx_ring->dev, data, len, DMA_TO_DEVICE); 1703 if (unlikely(dma_mapping_error(tx_ring->dev, dma))) { 1704 /* Undo the DMA mapping for all fragments */ 1705 while (--n >= 0) 1706 enetc_unmap_tx_buff(tx_ring, &xdp_tx_arr[n]); 1707 1708 netdev_err(tx_ring->ndev, "DMA map error\n"); 1709 return -1; 1710 } 1711 1712 xdp_tx_swbd->dma = dma; 1713 xdp_tx_swbd->dir = DMA_TO_DEVICE; 1714 xdp_tx_swbd->len = len; 1715 xdp_tx_swbd->is_xdp_redirect = true; 1716 xdp_tx_swbd->is_eof = false; 1717 xdp_tx_swbd->xdp_frame = NULL; 1718 1719 n++; 1720 xdp_tx_swbd = &xdp_tx_arr[n]; 1721 } 1722 out: 1723 xdp_tx_arr[n - 1].is_eof = true; 1724 xdp_tx_arr[n - 1].xdp_frame = xdp_frame; 1725 1726 return n; 1727 } 1728 1729 int enetc_xdp_xmit(struct net_device *ndev, int num_frames, 1730 struct xdp_frame **frames, u32 flags) 1731 { 1732 struct enetc_tx_swbd xdp_redirect_arr[ENETC_MAX_SKB_FRAGS] = {0}; 1733 struct enetc_ndev_priv *priv = netdev_priv(ndev); 1734 struct enetc_bdr *tx_ring; 1735 int xdp_tx_bd_cnt, i, k; 1736 int xdp_tx_frm_cnt = 0; 1737 1738 if (unlikely(test_bit(ENETC_TX_DOWN, &priv->flags))) 1739 return -ENETDOWN; 1740 1741 enetc_lock_mdio(); 1742 1743 tx_ring = priv->xdp_tx_ring[smp_processor_id()]; 1744 1745 prefetchw(ENETC_TXBD(*tx_ring, tx_ring->next_to_use)); 1746 1747 for (k = 0; k < num_frames; k++) { 1748 xdp_tx_bd_cnt = enetc_xdp_frame_to_xdp_tx_swbd(tx_ring, 1749 xdp_redirect_arr, 1750 frames[k]); 1751 if (unlikely(xdp_tx_bd_cnt < 0)) 1752 break; 1753 1754 if (unlikely(!enetc_xdp_tx(tx_ring, xdp_redirect_arr, 1755 xdp_tx_bd_cnt))) { 1756 for (i = 0; i < xdp_tx_bd_cnt; i++) 1757 enetc_unmap_tx_buff(tx_ring, 1758 &xdp_redirect_arr[i]); 1759 tx_ring->stats.xdp_tx_drops++; 1760 break; 1761 } 1762 1763 xdp_tx_frm_cnt++; 1764 } 1765 1766 if (unlikely((flags & XDP_XMIT_FLUSH) || k != xdp_tx_frm_cnt)) 1767 enetc_update_tx_ring_tail(tx_ring); 1768 1769 tx_ring->stats.xdp_tx += xdp_tx_frm_cnt; 1770 1771 enetc_unlock_mdio(); 1772 1773 return xdp_tx_frm_cnt; 1774 } 1775 EXPORT_SYMBOL_GPL(enetc_xdp_xmit); 1776 1777 static void enetc_map_rx_buff_to_xdp(struct enetc_bdr *rx_ring, int i, 1778 struct xdp_buff *xdp_buff, u16 size) 1779 { 1780 struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size); 1781 void *hard_start = page_address(rx_swbd->page) + rx_swbd->page_offset; 1782 1783 /* To be used for XDP_TX */ 1784 rx_swbd->len = size; 1785 1786 xdp_prepare_buff(xdp_buff, hard_start - rx_ring->buffer_offset, 1787 rx_ring->buffer_offset, size, false); 1788 } 1789 1790 static void enetc_add_rx_buff_to_xdp(struct enetc_bdr *rx_ring, int i, 1791 u16 size, struct xdp_buff *xdp_buff) 1792 { 1793 struct skb_shared_info *shinfo = xdp_get_shared_info_from_buff(xdp_buff); 1794 struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size); 1795 skb_frag_t *frag; 1796 1797 /* To be used for XDP_TX */ 1798 rx_swbd->len = size; 1799 1800 if (!xdp_buff_has_frags(xdp_buff)) { 1801 xdp_buff_set_frags_flag(xdp_buff); 1802 shinfo->xdp_frags_size = size; 1803 shinfo->nr_frags = 0; 1804 } else { 1805 shinfo->xdp_frags_size += size; 1806 } 1807 1808 if (page_is_pfmemalloc(rx_swbd->page)) 1809 xdp_buff_set_frag_pfmemalloc(xdp_buff); 1810 1811 frag = &shinfo->frags[shinfo->nr_frags]; 1812 skb_frag_fill_page_desc(frag, rx_swbd->page, rx_swbd->page_offset, 1813 size); 1814 1815 shinfo->nr_frags++; 1816 } 1817 1818 static void enetc_build_xdp_buff(struct enetc_bdr *rx_ring, u32 bd_status, 1819 union enetc_rx_bd **rxbd, int *i, 1820 int *cleaned_cnt, struct xdp_buff *xdp_buff) 1821 { 1822 u16 size = le16_to_cpu((*rxbd)->r.buf_len); 1823 1824 xdp_init_buff(xdp_buff, ENETC_RXB_TRUESIZE, &rx_ring->xdp.rxq); 1825 1826 enetc_map_rx_buff_to_xdp(rx_ring, *i, xdp_buff, size); 1827 (*cleaned_cnt)++; 1828 enetc_rxbd_next(rx_ring, rxbd, i); 1829 1830 /* not last BD in frame? */ 1831 while (!(bd_status & ENETC_RXBD_LSTATUS_F)) { 1832 bd_status = le32_to_cpu((*rxbd)->r.lstatus); 1833 size = ENETC_RXB_DMA_SIZE_XDP; 1834 1835 if (bd_status & ENETC_RXBD_LSTATUS_F) { 1836 dma_rmb(); 1837 size = le16_to_cpu((*rxbd)->r.buf_len); 1838 } 1839 1840 enetc_add_rx_buff_to_xdp(rx_ring, *i, size, xdp_buff); 1841 (*cleaned_cnt)++; 1842 enetc_rxbd_next(rx_ring, rxbd, i); 1843 } 1844 } 1845 1846 /* Convert RX buffer descriptors to TX buffer descriptors. These will be 1847 * recycled back into the RX ring in enetc_clean_tx_ring. 1848 */ 1849 static int enetc_rx_swbd_to_xdp_tx_swbd(struct enetc_tx_swbd *xdp_tx_arr, 1850 struct enetc_bdr *rx_ring, 1851 int rx_ring_first, int rx_ring_last) 1852 { 1853 int n = 0; 1854 1855 for (; rx_ring_first != rx_ring_last; 1856 n++, enetc_bdr_idx_inc(rx_ring, &rx_ring_first)) { 1857 struct enetc_rx_swbd *rx_swbd = &rx_ring->rx_swbd[rx_ring_first]; 1858 struct enetc_tx_swbd *tx_swbd = &xdp_tx_arr[n]; 1859 1860 /* No need to dma_map, we already have DMA_BIDIRECTIONAL */ 1861 tx_swbd->dma = rx_swbd->dma; 1862 tx_swbd->dir = rx_swbd->dir; 1863 tx_swbd->page = rx_swbd->page; 1864 tx_swbd->page_offset = rx_swbd->page_offset; 1865 tx_swbd->len = rx_swbd->len; 1866 tx_swbd->is_dma_page = true; 1867 tx_swbd->is_xdp_tx = true; 1868 tx_swbd->is_eof = false; 1869 } 1870 1871 /* We rely on caller providing an rx_ring_last > rx_ring_first */ 1872 xdp_tx_arr[n - 1].is_eof = true; 1873 1874 return n; 1875 } 1876 1877 static void enetc_xdp_drop(struct enetc_bdr *rx_ring, int rx_ring_first, 1878 int rx_ring_last) 1879 { 1880 while (rx_ring_first != rx_ring_last) { 1881 enetc_put_rx_buff(rx_ring, 1882 &rx_ring->rx_swbd[rx_ring_first]); 1883 enetc_bdr_idx_inc(rx_ring, &rx_ring_first); 1884 } 1885 } 1886 1887 static void enetc_bulk_flip_buff(struct enetc_bdr *rx_ring, int rx_ring_first, 1888 int rx_ring_last) 1889 { 1890 while (rx_ring_first != rx_ring_last) { 1891 enetc_flip_rx_buff(rx_ring, 1892 &rx_ring->rx_swbd[rx_ring_first]); 1893 enetc_bdr_idx_inc(rx_ring, &rx_ring_first); 1894 } 1895 } 1896 1897 static int enetc_clean_rx_ring_xdp(struct enetc_bdr *rx_ring, 1898 struct napi_struct *napi, int work_limit, 1899 struct bpf_prog *prog) 1900 { 1901 int xdp_tx_bd_cnt, xdp_tx_frm_cnt = 0, xdp_redirect_frm_cnt = 0; 1902 struct enetc_tx_swbd xdp_tx_arr[ENETC_MAX_SKB_FRAGS] = {0}; 1903 struct enetc_ndev_priv *priv = netdev_priv(rx_ring->ndev); 1904 int rx_frm_cnt = 0, rx_byte_cnt = 0; 1905 struct enetc_bdr *tx_ring; 1906 int cleaned_cnt, i; 1907 u32 xdp_act; 1908 1909 cleaned_cnt = enetc_bd_unused(rx_ring); 1910 /* next descriptor to process */ 1911 i = rx_ring->next_to_clean; 1912 1913 while (likely(rx_frm_cnt < work_limit)) { 1914 union enetc_rx_bd *rxbd, *orig_rxbd; 1915 struct xdp_buff xdp_buff; 1916 struct sk_buff *skb; 1917 int orig_i, err; 1918 u32 bd_status; 1919 1920 rxbd = enetc_rxbd(rx_ring, i); 1921 bd_status = le32_to_cpu(rxbd->r.lstatus); 1922 if (!bd_status) 1923 break; 1924 1925 enetc_wr_reg_hot(rx_ring->idr, BIT(rx_ring->index)); 1926 dma_rmb(); /* for reading other rxbd fields */ 1927 1928 if (enetc_check_bd_errors_and_consume(rx_ring, bd_status, 1929 &rxbd, &i)) 1930 break; 1931 1932 orig_rxbd = rxbd; 1933 orig_i = i; 1934 1935 enetc_build_xdp_buff(rx_ring, bd_status, &rxbd, &i, 1936 &cleaned_cnt, &xdp_buff); 1937 1938 /* When set, the outer VLAN header is extracted and reported 1939 * in the receive buffer descriptor. So rx_byte_cnt should 1940 * add the length of the extracted VLAN header. 1941 */ 1942 if (bd_status & ENETC_RXBD_FLAG_VLAN) 1943 rx_byte_cnt += VLAN_HLEN; 1944 rx_byte_cnt += xdp_get_buff_len(&xdp_buff); 1945 1946 xdp_act = bpf_prog_run_xdp(prog, &xdp_buff); 1947 1948 switch (xdp_act) { 1949 default: 1950 bpf_warn_invalid_xdp_action(rx_ring->ndev, prog, xdp_act); 1951 fallthrough; 1952 case XDP_ABORTED: 1953 trace_xdp_exception(rx_ring->ndev, prog, xdp_act); 1954 fallthrough; 1955 case XDP_DROP: 1956 enetc_xdp_drop(rx_ring, orig_i, i); 1957 rx_ring->stats.xdp_drops++; 1958 break; 1959 case XDP_PASS: 1960 skb = xdp_build_skb_from_buff(&xdp_buff); 1961 /* Probably under memory pressure, stop NAPI */ 1962 if (unlikely(!skb)) { 1963 enetc_xdp_drop(rx_ring, orig_i, i); 1964 rx_ring->stats.xdp_drops++; 1965 goto out; 1966 } 1967 1968 enetc_get_offloads(rx_ring, orig_rxbd, skb); 1969 1970 /* These buffers are about to be owned by the stack. 1971 * Update our buffer cache (the rx_swbd array elements) 1972 * with their other page halves. 1973 */ 1974 enetc_bulk_flip_buff(rx_ring, orig_i, i); 1975 1976 napi_gro_receive(napi, skb); 1977 break; 1978 case XDP_TX: 1979 tx_ring = priv->xdp_tx_ring[rx_ring->index]; 1980 if (unlikely(test_bit(ENETC_TX_DOWN, &priv->flags))) { 1981 enetc_xdp_drop(rx_ring, orig_i, i); 1982 tx_ring->stats.xdp_tx_drops++; 1983 break; 1984 } 1985 1986 xdp_tx_bd_cnt = enetc_rx_swbd_to_xdp_tx_swbd(xdp_tx_arr, 1987 rx_ring, 1988 orig_i, i); 1989 1990 if (!enetc_xdp_tx(tx_ring, xdp_tx_arr, xdp_tx_bd_cnt)) { 1991 enetc_xdp_drop(rx_ring, orig_i, i); 1992 tx_ring->stats.xdp_tx_drops++; 1993 } else { 1994 tx_ring->stats.xdp_tx++; 1995 rx_ring->xdp.xdp_tx_in_flight += xdp_tx_bd_cnt; 1996 xdp_tx_frm_cnt++; 1997 /* The XDP_TX enqueue was successful, so we 1998 * need to scrub the RX software BDs because 1999 * the ownership of the buffers no longer 2000 * belongs to the RX ring, and we must prevent 2001 * enetc_refill_rx_ring() from reusing 2002 * rx_swbd->page. 2003 */ 2004 while (orig_i != i) { 2005 rx_ring->rx_swbd[orig_i].page = NULL; 2006 enetc_bdr_idx_inc(rx_ring, &orig_i); 2007 } 2008 } 2009 break; 2010 case XDP_REDIRECT: 2011 err = xdp_do_redirect(rx_ring->ndev, &xdp_buff, prog); 2012 if (unlikely(err)) { 2013 enetc_xdp_drop(rx_ring, orig_i, i); 2014 rx_ring->stats.xdp_redirect_failures++; 2015 } else { 2016 enetc_bulk_flip_buff(rx_ring, orig_i, i); 2017 xdp_redirect_frm_cnt++; 2018 rx_ring->stats.xdp_redirect++; 2019 } 2020 } 2021 2022 rx_frm_cnt++; 2023 } 2024 2025 out: 2026 rx_ring->next_to_clean = i; 2027 2028 rx_ring->stats.packets += rx_frm_cnt; 2029 rx_ring->stats.bytes += rx_byte_cnt; 2030 2031 if (xdp_redirect_frm_cnt) 2032 xdp_do_flush(); 2033 2034 if (xdp_tx_frm_cnt) 2035 enetc_update_tx_ring_tail(tx_ring); 2036 2037 if (cleaned_cnt > rx_ring->xdp.xdp_tx_in_flight) 2038 enetc_refill_rx_ring(rx_ring, enetc_bd_unused(rx_ring) - 2039 rx_ring->xdp.xdp_tx_in_flight); 2040 2041 return rx_frm_cnt; 2042 } 2043 2044 static int enetc_poll(struct napi_struct *napi, int budget) 2045 { 2046 struct enetc_int_vector 2047 *v = container_of(napi, struct enetc_int_vector, napi); 2048 struct enetc_bdr *rx_ring = &v->rx_ring; 2049 struct bpf_prog *prog; 2050 bool complete = true; 2051 int work_done; 2052 int i; 2053 2054 enetc_lock_mdio(); 2055 2056 for (i = 0; i < v->count_tx_rings; i++) 2057 if (!enetc_clean_tx_ring(&v->tx_ring[i], budget)) 2058 complete = false; 2059 2060 prog = rx_ring->xdp.prog; 2061 if (prog) 2062 work_done = enetc_clean_rx_ring_xdp(rx_ring, napi, budget, prog); 2063 else 2064 work_done = enetc_clean_rx_ring(rx_ring, napi, budget); 2065 if (work_done == budget) 2066 complete = false; 2067 if (work_done) 2068 v->rx_napi_work = true; 2069 2070 if (!complete) { 2071 enetc_unlock_mdio(); 2072 return budget; 2073 } 2074 2075 napi_complete_done(napi, work_done); 2076 2077 if (likely(v->rx_dim_en)) 2078 enetc_rx_net_dim(v); 2079 2080 v->rx_napi_work = false; 2081 2082 /* enable interrupts */ 2083 enetc_wr_reg_hot(v->rbier, ENETC_RBIER_RXTIE); 2084 2085 for_each_set_bit(i, &v->tx_rings_map, ENETC_MAX_NUM_TXQS) 2086 enetc_wr_reg_hot(v->tbier_base + ENETC_BDR_OFF(i), 2087 ENETC_TBIER_TXTIE); 2088 2089 enetc_unlock_mdio(); 2090 2091 return work_done; 2092 } 2093 2094 /* Probing and Init */ 2095 #define ENETC_MAX_RFS_SIZE 64 2096 void enetc_get_si_caps(struct enetc_si *si) 2097 { 2098 struct enetc_hw *hw = &si->hw; 2099 u32 val; 2100 2101 /* find out how many of various resources we have to work with */ 2102 val = enetc_rd(hw, ENETC_SICAPR0); 2103 si->num_rx_rings = (val >> 16) & 0xff; 2104 si->num_tx_rings = val & 0xff; 2105 2106 val = enetc_rd(hw, ENETC_SIPCAPR0); 2107 if (val & ENETC_SIPCAPR0_RFS) { 2108 val = enetc_rd(hw, ENETC_SIRFSCAPR); 2109 si->num_fs_entries = ENETC_SIRFSCAPR_GET_NUM_RFS(val); 2110 si->num_fs_entries = min(si->num_fs_entries, ENETC_MAX_RFS_SIZE); 2111 } else { 2112 /* ENETC which not supports RFS */ 2113 si->num_fs_entries = 0; 2114 } 2115 2116 si->num_rss = 0; 2117 val = enetc_rd(hw, ENETC_SIPCAPR0); 2118 if (val & ENETC_SIPCAPR0_RSS) { 2119 u32 rss; 2120 2121 rss = enetc_rd(hw, ENETC_SIRSSCAPR); 2122 si->num_rss = ENETC_SIRSSCAPR_GET_NUM_RSS(rss); 2123 } 2124 2125 if (val & ENETC_SIPCAPR0_LSO) 2126 si->hw_features |= ENETC_SI_F_LSO; 2127 } 2128 EXPORT_SYMBOL_GPL(enetc_get_si_caps); 2129 2130 static int enetc_dma_alloc_bdr(struct enetc_bdr_resource *res) 2131 { 2132 size_t bd_base_size = res->bd_count * res->bd_size; 2133 2134 res->bd_base = dma_alloc_coherent(res->dev, bd_base_size, 2135 &res->bd_dma_base, GFP_KERNEL); 2136 if (!res->bd_base) 2137 return -ENOMEM; 2138 2139 /* h/w requires 128B alignment */ 2140 if (!IS_ALIGNED(res->bd_dma_base, 128)) { 2141 dma_free_coherent(res->dev, bd_base_size, res->bd_base, 2142 res->bd_dma_base); 2143 return -EINVAL; 2144 } 2145 2146 return 0; 2147 } 2148 2149 static void enetc_dma_free_bdr(const struct enetc_bdr_resource *res) 2150 { 2151 size_t bd_base_size = res->bd_count * res->bd_size; 2152 2153 dma_free_coherent(res->dev, bd_base_size, res->bd_base, 2154 res->bd_dma_base); 2155 } 2156 2157 static int enetc_alloc_tx_resource(struct enetc_bdr_resource *res, 2158 struct device *dev, size_t bd_count) 2159 { 2160 int err; 2161 2162 res->dev = dev; 2163 res->bd_count = bd_count; 2164 res->bd_size = sizeof(union enetc_tx_bd); 2165 2166 res->tx_swbd = vcalloc(bd_count, sizeof(*res->tx_swbd)); 2167 if (!res->tx_swbd) 2168 return -ENOMEM; 2169 2170 err = enetc_dma_alloc_bdr(res); 2171 if (err) 2172 goto err_alloc_bdr; 2173 2174 res->tso_headers = dma_alloc_coherent(dev, bd_count * TSO_HEADER_SIZE, 2175 &res->tso_headers_dma, 2176 GFP_KERNEL); 2177 if (!res->tso_headers) { 2178 err = -ENOMEM; 2179 goto err_alloc_tso; 2180 } 2181 2182 return 0; 2183 2184 err_alloc_tso: 2185 enetc_dma_free_bdr(res); 2186 err_alloc_bdr: 2187 vfree(res->tx_swbd); 2188 res->tx_swbd = NULL; 2189 2190 return err; 2191 } 2192 2193 static void enetc_free_tx_resource(const struct enetc_bdr_resource *res) 2194 { 2195 dma_free_coherent(res->dev, res->bd_count * TSO_HEADER_SIZE, 2196 res->tso_headers, res->tso_headers_dma); 2197 enetc_dma_free_bdr(res); 2198 vfree(res->tx_swbd); 2199 } 2200 2201 static struct enetc_bdr_resource * 2202 enetc_alloc_tx_resources(struct enetc_ndev_priv *priv) 2203 { 2204 struct enetc_bdr_resource *tx_res; 2205 int i, err; 2206 2207 tx_res = kcalloc(priv->num_tx_rings, sizeof(*tx_res), GFP_KERNEL); 2208 if (!tx_res) 2209 return ERR_PTR(-ENOMEM); 2210 2211 for (i = 0; i < priv->num_tx_rings; i++) { 2212 struct enetc_bdr *tx_ring = priv->tx_ring[i]; 2213 2214 err = enetc_alloc_tx_resource(&tx_res[i], tx_ring->dev, 2215 tx_ring->bd_count); 2216 if (err) 2217 goto fail; 2218 } 2219 2220 return tx_res; 2221 2222 fail: 2223 while (i-- > 0) 2224 enetc_free_tx_resource(&tx_res[i]); 2225 2226 kfree(tx_res); 2227 2228 return ERR_PTR(err); 2229 } 2230 2231 static void enetc_free_tx_resources(const struct enetc_bdr_resource *tx_res, 2232 size_t num_resources) 2233 { 2234 size_t i; 2235 2236 for (i = 0; i < num_resources; i++) 2237 enetc_free_tx_resource(&tx_res[i]); 2238 2239 kfree(tx_res); 2240 } 2241 2242 static int enetc_alloc_rx_resource(struct enetc_bdr_resource *res, 2243 struct device *dev, size_t bd_count, 2244 bool extended) 2245 { 2246 int err; 2247 2248 res->dev = dev; 2249 res->bd_count = bd_count; 2250 res->bd_size = sizeof(union enetc_rx_bd); 2251 if (extended) 2252 res->bd_size *= 2; 2253 2254 res->rx_swbd = vcalloc(bd_count, sizeof(struct enetc_rx_swbd)); 2255 if (!res->rx_swbd) 2256 return -ENOMEM; 2257 2258 err = enetc_dma_alloc_bdr(res); 2259 if (err) { 2260 vfree(res->rx_swbd); 2261 return err; 2262 } 2263 2264 return 0; 2265 } 2266 2267 static void enetc_free_rx_resource(const struct enetc_bdr_resource *res) 2268 { 2269 enetc_dma_free_bdr(res); 2270 vfree(res->rx_swbd); 2271 } 2272 2273 static struct enetc_bdr_resource * 2274 enetc_alloc_rx_resources(struct enetc_ndev_priv *priv, bool extended) 2275 { 2276 struct enetc_bdr_resource *rx_res; 2277 int i, err; 2278 2279 rx_res = kcalloc(priv->num_rx_rings, sizeof(*rx_res), GFP_KERNEL); 2280 if (!rx_res) 2281 return ERR_PTR(-ENOMEM); 2282 2283 for (i = 0; i < priv->num_rx_rings; i++) { 2284 struct enetc_bdr *rx_ring = priv->rx_ring[i]; 2285 2286 err = enetc_alloc_rx_resource(&rx_res[i], rx_ring->dev, 2287 rx_ring->bd_count, extended); 2288 if (err) 2289 goto fail; 2290 } 2291 2292 return rx_res; 2293 2294 fail: 2295 while (i-- > 0) 2296 enetc_free_rx_resource(&rx_res[i]); 2297 2298 kfree(rx_res); 2299 2300 return ERR_PTR(err); 2301 } 2302 2303 static void enetc_free_rx_resources(const struct enetc_bdr_resource *rx_res, 2304 size_t num_resources) 2305 { 2306 size_t i; 2307 2308 for (i = 0; i < num_resources; i++) 2309 enetc_free_rx_resource(&rx_res[i]); 2310 2311 kfree(rx_res); 2312 } 2313 2314 static void enetc_assign_tx_resource(struct enetc_bdr *tx_ring, 2315 const struct enetc_bdr_resource *res) 2316 { 2317 tx_ring->bd_base = res ? res->bd_base : NULL; 2318 tx_ring->bd_dma_base = res ? res->bd_dma_base : 0; 2319 tx_ring->tx_swbd = res ? res->tx_swbd : NULL; 2320 tx_ring->tso_headers = res ? res->tso_headers : NULL; 2321 tx_ring->tso_headers_dma = res ? res->tso_headers_dma : 0; 2322 } 2323 2324 static void enetc_assign_rx_resource(struct enetc_bdr *rx_ring, 2325 const struct enetc_bdr_resource *res) 2326 { 2327 rx_ring->bd_base = res ? res->bd_base : NULL; 2328 rx_ring->bd_dma_base = res ? res->bd_dma_base : 0; 2329 rx_ring->rx_swbd = res ? res->rx_swbd : NULL; 2330 } 2331 2332 static void enetc_assign_tx_resources(struct enetc_ndev_priv *priv, 2333 const struct enetc_bdr_resource *res) 2334 { 2335 int i; 2336 2337 if (priv->tx_res) 2338 enetc_free_tx_resources(priv->tx_res, priv->num_tx_rings); 2339 2340 for (i = 0; i < priv->num_tx_rings; i++) { 2341 enetc_assign_tx_resource(priv->tx_ring[i], 2342 res ? &res[i] : NULL); 2343 } 2344 2345 priv->tx_res = res; 2346 } 2347 2348 static void enetc_assign_rx_resources(struct enetc_ndev_priv *priv, 2349 const struct enetc_bdr_resource *res) 2350 { 2351 int i; 2352 2353 if (priv->rx_res) 2354 enetc_free_rx_resources(priv->rx_res, priv->num_rx_rings); 2355 2356 for (i = 0; i < priv->num_rx_rings; i++) { 2357 enetc_assign_rx_resource(priv->rx_ring[i], 2358 res ? &res[i] : NULL); 2359 } 2360 2361 priv->rx_res = res; 2362 } 2363 2364 static void enetc_free_tx_ring(struct enetc_bdr *tx_ring) 2365 { 2366 int i; 2367 2368 for (i = 0; i < tx_ring->bd_count; i++) { 2369 struct enetc_tx_swbd *tx_swbd = &tx_ring->tx_swbd[i]; 2370 2371 enetc_free_tx_frame(tx_ring, tx_swbd); 2372 } 2373 } 2374 2375 static void enetc_free_rx_ring(struct enetc_bdr *rx_ring) 2376 { 2377 int i; 2378 2379 for (i = 0; i < rx_ring->bd_count; i++) { 2380 struct enetc_rx_swbd *rx_swbd = &rx_ring->rx_swbd[i]; 2381 2382 if (!rx_swbd->page) 2383 continue; 2384 2385 dma_unmap_page(rx_ring->dev, rx_swbd->dma, PAGE_SIZE, 2386 rx_swbd->dir); 2387 __free_page(rx_swbd->page); 2388 rx_swbd->page = NULL; 2389 } 2390 } 2391 2392 static void enetc_free_rxtx_rings(struct enetc_ndev_priv *priv) 2393 { 2394 int i; 2395 2396 for (i = 0; i < priv->num_rx_rings; i++) 2397 enetc_free_rx_ring(priv->rx_ring[i]); 2398 2399 for (i = 0; i < priv->num_tx_rings; i++) 2400 enetc_free_tx_ring(priv->tx_ring[i]); 2401 } 2402 2403 static int enetc_setup_default_rss_table(struct enetc_si *si, int num_groups) 2404 { 2405 int *rss_table; 2406 int i; 2407 2408 rss_table = kmalloc_array(si->num_rss, sizeof(*rss_table), GFP_KERNEL); 2409 if (!rss_table) 2410 return -ENOMEM; 2411 2412 /* Set up RSS table defaults */ 2413 for (i = 0; i < si->num_rss; i++) 2414 rss_table[i] = i % num_groups; 2415 2416 si->ops->set_rss_table(si, rss_table, si->num_rss); 2417 2418 kfree(rss_table); 2419 2420 return 0; 2421 } 2422 2423 static void enetc_set_lso_flags_mask(struct enetc_hw *hw) 2424 { 2425 enetc_wr(hw, ENETC4_SILSOSFMR0, 2426 SILSOSFMR0_VAL_SET(ENETC4_TCP_NL_SEG_FLAGS_DMASK, 2427 ENETC4_TCP_NL_SEG_FLAGS_DMASK)); 2428 enetc_wr(hw, ENETC4_SILSOSFMR1, 0); 2429 } 2430 2431 static void enetc_set_rss(struct net_device *ndev, int en) 2432 { 2433 struct enetc_ndev_priv *priv = netdev_priv(ndev); 2434 struct enetc_hw *hw = &priv->si->hw; 2435 u32 reg; 2436 2437 enetc_wr(hw, ENETC_SIRBGCR, priv->num_rx_rings); 2438 2439 reg = enetc_rd(hw, ENETC_SIMR); 2440 reg &= ~ENETC_SIMR_RSSE; 2441 reg |= (en) ? ENETC_SIMR_RSSE : 0; 2442 enetc_wr(hw, ENETC_SIMR, reg); 2443 } 2444 2445 int enetc_configure_si(struct enetc_ndev_priv *priv) 2446 { 2447 struct enetc_si *si = priv->si; 2448 struct enetc_hw *hw = &si->hw; 2449 int err; 2450 2451 /* set SI cache attributes */ 2452 enetc_wr(hw, ENETC_SICAR0, 2453 ENETC_SICAR_RD_COHERENT | ENETC_SICAR_WR_COHERENT); 2454 enetc_wr(hw, ENETC_SICAR1, ENETC_SICAR_MSI); 2455 /* enable SI */ 2456 enetc_wr(hw, ENETC_SIMR, ENETC_SIMR_EN); 2457 2458 if (si->hw_features & ENETC_SI_F_LSO) 2459 enetc_set_lso_flags_mask(hw); 2460 2461 if (si->num_rss) { 2462 err = enetc_setup_default_rss_table(si, priv->num_rx_rings); 2463 if (err) 2464 return err; 2465 2466 if (priv->ndev->features & NETIF_F_RXHASH) 2467 enetc_set_rss(priv->ndev, true); 2468 } 2469 2470 return 0; 2471 } 2472 EXPORT_SYMBOL_GPL(enetc_configure_si); 2473 2474 void enetc_init_si_rings_params(struct enetc_ndev_priv *priv) 2475 { 2476 struct enetc_si *si = priv->si; 2477 int cpus = num_online_cpus(); 2478 2479 priv->tx_bd_count = ENETC_TX_RING_DEFAULT_SIZE; 2480 priv->rx_bd_count = ENETC_RX_RING_DEFAULT_SIZE; 2481 2482 /* Enable all available TX rings in order to configure as many 2483 * priorities as possible, when needed. 2484 * TODO: Make # of TX rings run-time configurable 2485 */ 2486 priv->num_rx_rings = min_t(int, cpus, si->num_rx_rings); 2487 priv->num_tx_rings = si->num_tx_rings; 2488 priv->bdr_int_num = priv->num_rx_rings; 2489 priv->ic_mode = ENETC_IC_RX_ADAPTIVE | ENETC_IC_TX_MANUAL; 2490 priv->tx_ictt = enetc_usecs_to_cycles(600, priv->sysclk_freq); 2491 } 2492 EXPORT_SYMBOL_GPL(enetc_init_si_rings_params); 2493 2494 int enetc_alloc_si_resources(struct enetc_ndev_priv *priv) 2495 { 2496 struct enetc_si *si = priv->si; 2497 2498 priv->cls_rules = kcalloc(si->num_fs_entries, sizeof(*priv->cls_rules), 2499 GFP_KERNEL); 2500 if (!priv->cls_rules) 2501 return -ENOMEM; 2502 2503 return 0; 2504 } 2505 EXPORT_SYMBOL_GPL(enetc_alloc_si_resources); 2506 2507 void enetc_free_si_resources(struct enetc_ndev_priv *priv) 2508 { 2509 kfree(priv->cls_rules); 2510 } 2511 EXPORT_SYMBOL_GPL(enetc_free_si_resources); 2512 2513 static void enetc_setup_txbdr(struct enetc_hw *hw, struct enetc_bdr *tx_ring) 2514 { 2515 int idx = tx_ring->index; 2516 u32 tbmr; 2517 2518 enetc_txbdr_wr(hw, idx, ENETC_TBBAR0, 2519 lower_32_bits(tx_ring->bd_dma_base)); 2520 2521 enetc_txbdr_wr(hw, idx, ENETC_TBBAR1, 2522 upper_32_bits(tx_ring->bd_dma_base)); 2523 2524 WARN_ON(!IS_ALIGNED(tx_ring->bd_count, 64)); /* multiple of 64 */ 2525 enetc_txbdr_wr(hw, idx, ENETC_TBLENR, 2526 ENETC_RTBLENR_LEN(tx_ring->bd_count)); 2527 2528 /* clearing PI/CI registers for Tx not supported, adjust sw indexes */ 2529 tx_ring->next_to_use = enetc_txbdr_rd(hw, idx, ENETC_TBPIR); 2530 tx_ring->next_to_clean = enetc_txbdr_rd(hw, idx, ENETC_TBCIR); 2531 2532 /* enable Tx ints by setting pkt thr to 1 */ 2533 enetc_txbdr_wr(hw, idx, ENETC_TBICR0, ENETC_TBICR0_ICEN | 0x1); 2534 2535 tbmr = ENETC_TBMR_SET_PRIO(tx_ring->prio); 2536 if (tx_ring->ndev->features & NETIF_F_HW_VLAN_CTAG_TX) 2537 tbmr |= ENETC_TBMR_VIH; 2538 2539 /* enable ring */ 2540 enetc_txbdr_wr(hw, idx, ENETC_TBMR, tbmr); 2541 2542 tx_ring->tpir = hw->reg + ENETC_BDR(TX, idx, ENETC_TBPIR); 2543 tx_ring->tcir = hw->reg + ENETC_BDR(TX, idx, ENETC_TBCIR); 2544 tx_ring->idr = hw->reg + ENETC_SITXIDR; 2545 } 2546 2547 static void enetc_setup_rxbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring, 2548 bool extended) 2549 { 2550 int idx = rx_ring->index; 2551 u32 rbmr = 0; 2552 2553 enetc_rxbdr_wr(hw, idx, ENETC_RBBAR0, 2554 lower_32_bits(rx_ring->bd_dma_base)); 2555 2556 enetc_rxbdr_wr(hw, idx, ENETC_RBBAR1, 2557 upper_32_bits(rx_ring->bd_dma_base)); 2558 2559 WARN_ON(!IS_ALIGNED(rx_ring->bd_count, 64)); /* multiple of 64 */ 2560 enetc_rxbdr_wr(hw, idx, ENETC_RBLENR, 2561 ENETC_RTBLENR_LEN(rx_ring->bd_count)); 2562 2563 if (rx_ring->xdp.prog) 2564 enetc_rxbdr_wr(hw, idx, ENETC_RBBSR, ENETC_RXB_DMA_SIZE_XDP); 2565 else 2566 enetc_rxbdr_wr(hw, idx, ENETC_RBBSR, ENETC_RXB_DMA_SIZE); 2567 2568 /* Also prepare the consumer index in case page allocation never 2569 * succeeds. In that case, hardware will never advance producer index 2570 * to match consumer index, and will drop all frames. 2571 */ 2572 enetc_rxbdr_wr(hw, idx, ENETC_RBPIR, 0); 2573 enetc_rxbdr_wr(hw, idx, ENETC_RBCIR, 1); 2574 2575 /* enable Rx ints by setting pkt thr to 1 */ 2576 enetc_rxbdr_wr(hw, idx, ENETC_RBICR0, ENETC_RBICR0_ICEN | 0x1); 2577 2578 rx_ring->ext_en = extended; 2579 if (rx_ring->ext_en) 2580 rbmr |= ENETC_RBMR_BDS; 2581 2582 if (rx_ring->ndev->features & NETIF_F_HW_VLAN_CTAG_RX) 2583 rbmr |= ENETC_RBMR_VTE; 2584 2585 rx_ring->rcir = hw->reg + ENETC_BDR(RX, idx, ENETC_RBCIR); 2586 rx_ring->idr = hw->reg + ENETC_SIRXIDR; 2587 2588 rx_ring->next_to_clean = 0; 2589 rx_ring->next_to_use = 0; 2590 rx_ring->next_to_alloc = 0; 2591 2592 enetc_lock_mdio(); 2593 enetc_refill_rx_ring(rx_ring, enetc_bd_unused(rx_ring)); 2594 enetc_unlock_mdio(); 2595 2596 enetc_rxbdr_wr(hw, idx, ENETC_RBMR, rbmr); 2597 } 2598 2599 static void enetc_setup_bdrs(struct enetc_ndev_priv *priv, bool extended) 2600 { 2601 struct enetc_hw *hw = &priv->si->hw; 2602 int i; 2603 2604 for (i = 0; i < priv->num_tx_rings; i++) 2605 enetc_setup_txbdr(hw, priv->tx_ring[i]); 2606 2607 for (i = 0; i < priv->num_rx_rings; i++) 2608 enetc_setup_rxbdr(hw, priv->rx_ring[i], extended); 2609 } 2610 2611 static void enetc_enable_txbdr(struct enetc_hw *hw, struct enetc_bdr *tx_ring) 2612 { 2613 int idx = tx_ring->index; 2614 u32 tbmr; 2615 2616 tbmr = enetc_txbdr_rd(hw, idx, ENETC_TBMR); 2617 tbmr |= ENETC_TBMR_EN; 2618 enetc_txbdr_wr(hw, idx, ENETC_TBMR, tbmr); 2619 } 2620 2621 static void enetc_enable_rxbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring) 2622 { 2623 int idx = rx_ring->index; 2624 u32 rbmr; 2625 2626 rbmr = enetc_rxbdr_rd(hw, idx, ENETC_RBMR); 2627 rbmr |= ENETC_RBMR_EN; 2628 enetc_rxbdr_wr(hw, idx, ENETC_RBMR, rbmr); 2629 } 2630 2631 static void enetc_enable_rx_bdrs(struct enetc_ndev_priv *priv) 2632 { 2633 struct enetc_hw *hw = &priv->si->hw; 2634 int i; 2635 2636 for (i = 0; i < priv->num_rx_rings; i++) 2637 enetc_enable_rxbdr(hw, priv->rx_ring[i]); 2638 } 2639 2640 static void enetc_enable_tx_bdrs(struct enetc_ndev_priv *priv) 2641 { 2642 struct enetc_hw *hw = &priv->si->hw; 2643 int i; 2644 2645 for (i = 0; i < priv->num_tx_rings; i++) 2646 enetc_enable_txbdr(hw, priv->tx_ring[i]); 2647 } 2648 2649 static void enetc_disable_rxbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring) 2650 { 2651 int idx = rx_ring->index; 2652 2653 /* disable EN bit on ring */ 2654 enetc_rxbdr_wr(hw, idx, ENETC_RBMR, 0); 2655 } 2656 2657 static void enetc_disable_txbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring) 2658 { 2659 int idx = rx_ring->index; 2660 2661 /* disable EN bit on ring */ 2662 enetc_txbdr_wr(hw, idx, ENETC_TBMR, 0); 2663 } 2664 2665 static void enetc_disable_rx_bdrs(struct enetc_ndev_priv *priv) 2666 { 2667 struct enetc_hw *hw = &priv->si->hw; 2668 int i; 2669 2670 for (i = 0; i < priv->num_rx_rings; i++) 2671 enetc_disable_rxbdr(hw, priv->rx_ring[i]); 2672 } 2673 2674 static void enetc_disable_tx_bdrs(struct enetc_ndev_priv *priv) 2675 { 2676 struct enetc_hw *hw = &priv->si->hw; 2677 int i; 2678 2679 for (i = 0; i < priv->num_tx_rings; i++) 2680 enetc_disable_txbdr(hw, priv->tx_ring[i]); 2681 } 2682 2683 static void enetc_wait_txbdr(struct enetc_hw *hw, struct enetc_bdr *tx_ring) 2684 { 2685 int delay = 8, timeout = 100; 2686 int idx = tx_ring->index; 2687 2688 /* wait for busy to clear */ 2689 while (delay < timeout && 2690 enetc_txbdr_rd(hw, idx, ENETC_TBSR) & ENETC_TBSR_BUSY) { 2691 msleep(delay); 2692 delay *= 2; 2693 } 2694 2695 if (delay >= timeout) 2696 netdev_warn(tx_ring->ndev, "timeout for tx ring #%d clear\n", 2697 idx); 2698 } 2699 2700 static void enetc_wait_bdrs(struct enetc_ndev_priv *priv) 2701 { 2702 struct enetc_hw *hw = &priv->si->hw; 2703 int i; 2704 2705 for (i = 0; i < priv->num_tx_rings; i++) 2706 enetc_wait_txbdr(hw, priv->tx_ring[i]); 2707 } 2708 2709 static int enetc_setup_irqs(struct enetc_ndev_priv *priv) 2710 { 2711 struct pci_dev *pdev = priv->si->pdev; 2712 struct enetc_hw *hw = &priv->si->hw; 2713 int i, j, err; 2714 2715 for (i = 0; i < priv->bdr_int_num; i++) { 2716 int irq = pci_irq_vector(pdev, ENETC_BDR_INT_BASE_IDX + i); 2717 struct enetc_int_vector *v = priv->int_vector[i]; 2718 int entry = ENETC_BDR_INT_BASE_IDX + i; 2719 2720 snprintf(v->name, sizeof(v->name), "%s-rxtx%d", 2721 priv->ndev->name, i); 2722 err = request_irq(irq, enetc_msix, IRQF_NO_AUTOEN, v->name, v); 2723 if (err) { 2724 dev_err(priv->dev, "request_irq() failed!\n"); 2725 goto irq_err; 2726 } 2727 2728 v->tbier_base = hw->reg + ENETC_BDR(TX, 0, ENETC_TBIER); 2729 v->rbier = hw->reg + ENETC_BDR(RX, i, ENETC_RBIER); 2730 v->ricr1 = hw->reg + ENETC_BDR(RX, i, ENETC_RBICR1); 2731 2732 enetc_wr(hw, ENETC_SIMSIRRV(i), entry); 2733 2734 for (j = 0; j < v->count_tx_rings; j++) { 2735 int idx = v->tx_ring[j].index; 2736 2737 enetc_wr(hw, ENETC_SIMSITRV(idx), entry); 2738 } 2739 irq_set_affinity_hint(irq, get_cpu_mask(i % num_online_cpus())); 2740 } 2741 2742 return 0; 2743 2744 irq_err: 2745 while (i--) { 2746 int irq = pci_irq_vector(pdev, ENETC_BDR_INT_BASE_IDX + i); 2747 2748 irq_set_affinity_hint(irq, NULL); 2749 free_irq(irq, priv->int_vector[i]); 2750 } 2751 2752 return err; 2753 } 2754 2755 static void enetc_free_irqs(struct enetc_ndev_priv *priv) 2756 { 2757 struct pci_dev *pdev = priv->si->pdev; 2758 int i; 2759 2760 for (i = 0; i < priv->bdr_int_num; i++) { 2761 int irq = pci_irq_vector(pdev, ENETC_BDR_INT_BASE_IDX + i); 2762 2763 irq_set_affinity_hint(irq, NULL); 2764 free_irq(irq, priv->int_vector[i]); 2765 } 2766 } 2767 2768 static void enetc_setup_interrupts(struct enetc_ndev_priv *priv) 2769 { 2770 struct enetc_hw *hw = &priv->si->hw; 2771 u32 icpt, ictt; 2772 int i; 2773 2774 /* enable Tx & Rx event indication */ 2775 if (priv->ic_mode & 2776 (ENETC_IC_RX_MANUAL | ENETC_IC_RX_ADAPTIVE)) { 2777 icpt = ENETC_RBICR0_SET_ICPT(ENETC_RXIC_PKTTHR); 2778 /* init to non-0 minimum, will be adjusted later */ 2779 ictt = 0x1; 2780 } else { 2781 icpt = 0x1; /* enable Rx ints by setting pkt thr to 1 */ 2782 ictt = 0; 2783 } 2784 2785 for (i = 0; i < priv->num_rx_rings; i++) { 2786 enetc_rxbdr_wr(hw, i, ENETC_RBICR1, ictt); 2787 enetc_rxbdr_wr(hw, i, ENETC_RBICR0, ENETC_RBICR0_ICEN | icpt); 2788 enetc_rxbdr_wr(hw, i, ENETC_RBIER, ENETC_RBIER_RXTIE); 2789 } 2790 2791 if (priv->ic_mode & ENETC_IC_TX_MANUAL) 2792 icpt = ENETC_TBICR0_SET_ICPT(ENETC_TXIC_PKTTHR); 2793 else 2794 icpt = 0x1; /* enable Tx ints by setting pkt thr to 1 */ 2795 2796 for (i = 0; i < priv->num_tx_rings; i++) { 2797 enetc_txbdr_wr(hw, i, ENETC_TBICR1, priv->tx_ictt); 2798 enetc_txbdr_wr(hw, i, ENETC_TBICR0, ENETC_TBICR0_ICEN | icpt); 2799 enetc_txbdr_wr(hw, i, ENETC_TBIER, ENETC_TBIER_TXTIE); 2800 } 2801 } 2802 2803 static void enetc_clear_interrupts(struct enetc_ndev_priv *priv) 2804 { 2805 struct enetc_hw *hw = &priv->si->hw; 2806 int i; 2807 2808 for (i = 0; i < priv->num_tx_rings; i++) 2809 enetc_txbdr_wr(hw, i, ENETC_TBIER, 0); 2810 2811 for (i = 0; i < priv->num_rx_rings; i++) 2812 enetc_rxbdr_wr(hw, i, ENETC_RBIER, 0); 2813 } 2814 2815 static int enetc_phylink_connect(struct net_device *ndev) 2816 { 2817 struct enetc_ndev_priv *priv = netdev_priv(ndev); 2818 struct ethtool_keee edata; 2819 int err; 2820 2821 if (!priv->phylink) { 2822 /* phy-less mode */ 2823 netif_carrier_on(ndev); 2824 return 0; 2825 } 2826 2827 err = phylink_of_phy_connect(priv->phylink, priv->dev->of_node, 0); 2828 if (err) { 2829 dev_err(&ndev->dev, "could not attach to PHY\n"); 2830 return err; 2831 } 2832 2833 /* disable EEE autoneg, until ENETC driver supports it */ 2834 memset(&edata, 0, sizeof(struct ethtool_keee)); 2835 phylink_ethtool_set_eee(priv->phylink, &edata); 2836 2837 phylink_start(priv->phylink); 2838 2839 return 0; 2840 } 2841 2842 static void enetc_tx_onestep_tstamp(struct work_struct *work) 2843 { 2844 struct enetc_ndev_priv *priv; 2845 struct sk_buff *skb; 2846 2847 priv = container_of(work, struct enetc_ndev_priv, tx_onestep_tstamp); 2848 2849 netif_tx_lock_bh(priv->ndev); 2850 2851 clear_bit_unlock(ENETC_TX_ONESTEP_TSTAMP_IN_PROGRESS, &priv->flags); 2852 skb = skb_dequeue(&priv->tx_skbs); 2853 if (skb) 2854 enetc_start_xmit(skb, priv->ndev); 2855 2856 netif_tx_unlock_bh(priv->ndev); 2857 } 2858 2859 static void enetc_tx_onestep_tstamp_init(struct enetc_ndev_priv *priv) 2860 { 2861 INIT_WORK(&priv->tx_onestep_tstamp, enetc_tx_onestep_tstamp); 2862 skb_queue_head_init(&priv->tx_skbs); 2863 } 2864 2865 void enetc_start(struct net_device *ndev) 2866 { 2867 struct enetc_ndev_priv *priv = netdev_priv(ndev); 2868 int i; 2869 2870 enetc_setup_interrupts(priv); 2871 2872 for (i = 0; i < priv->bdr_int_num; i++) { 2873 int irq = pci_irq_vector(priv->si->pdev, 2874 ENETC_BDR_INT_BASE_IDX + i); 2875 2876 napi_enable(&priv->int_vector[i]->napi); 2877 enable_irq(irq); 2878 } 2879 2880 enetc_enable_tx_bdrs(priv); 2881 2882 enetc_enable_rx_bdrs(priv); 2883 2884 netif_tx_start_all_queues(ndev); 2885 2886 clear_bit(ENETC_TX_DOWN, &priv->flags); 2887 } 2888 EXPORT_SYMBOL_GPL(enetc_start); 2889 2890 int enetc_open(struct net_device *ndev) 2891 { 2892 struct enetc_ndev_priv *priv = netdev_priv(ndev); 2893 struct enetc_bdr_resource *tx_res, *rx_res; 2894 bool extended; 2895 int err; 2896 2897 extended = !!(priv->active_offloads & ENETC_F_RX_TSTAMP); 2898 2899 err = clk_prepare_enable(priv->ref_clk); 2900 if (err) 2901 return err; 2902 2903 err = enetc_setup_irqs(priv); 2904 if (err) 2905 goto err_setup_irqs; 2906 2907 err = enetc_phylink_connect(ndev); 2908 if (err) 2909 goto err_phy_connect; 2910 2911 tx_res = enetc_alloc_tx_resources(priv); 2912 if (IS_ERR(tx_res)) { 2913 err = PTR_ERR(tx_res); 2914 goto err_alloc_tx; 2915 } 2916 2917 rx_res = enetc_alloc_rx_resources(priv, extended); 2918 if (IS_ERR(rx_res)) { 2919 err = PTR_ERR(rx_res); 2920 goto err_alloc_rx; 2921 } 2922 2923 enetc_tx_onestep_tstamp_init(priv); 2924 enetc_assign_tx_resources(priv, tx_res); 2925 enetc_assign_rx_resources(priv, rx_res); 2926 enetc_setup_bdrs(priv, extended); 2927 enetc_start(ndev); 2928 2929 return 0; 2930 2931 err_alloc_rx: 2932 enetc_free_tx_resources(tx_res, priv->num_tx_rings); 2933 err_alloc_tx: 2934 if (priv->phylink) 2935 phylink_disconnect_phy(priv->phylink); 2936 err_phy_connect: 2937 enetc_free_irqs(priv); 2938 err_setup_irqs: 2939 clk_disable_unprepare(priv->ref_clk); 2940 2941 return err; 2942 } 2943 EXPORT_SYMBOL_GPL(enetc_open); 2944 2945 void enetc_stop(struct net_device *ndev) 2946 { 2947 struct enetc_ndev_priv *priv = netdev_priv(ndev); 2948 int i; 2949 2950 set_bit(ENETC_TX_DOWN, &priv->flags); 2951 2952 netif_tx_stop_all_queues(ndev); 2953 2954 enetc_disable_rx_bdrs(priv); 2955 2956 enetc_wait_bdrs(priv); 2957 2958 enetc_disable_tx_bdrs(priv); 2959 2960 for (i = 0; i < priv->bdr_int_num; i++) { 2961 int irq = pci_irq_vector(priv->si->pdev, 2962 ENETC_BDR_INT_BASE_IDX + i); 2963 2964 disable_irq(irq); 2965 napi_synchronize(&priv->int_vector[i]->napi); 2966 napi_disable(&priv->int_vector[i]->napi); 2967 } 2968 2969 enetc_clear_interrupts(priv); 2970 } 2971 EXPORT_SYMBOL_GPL(enetc_stop); 2972 2973 int enetc_close(struct net_device *ndev) 2974 { 2975 struct enetc_ndev_priv *priv = netdev_priv(ndev); 2976 2977 enetc_stop(ndev); 2978 2979 if (priv->phylink) { 2980 phylink_stop(priv->phylink); 2981 phylink_disconnect_phy(priv->phylink); 2982 } else { 2983 netif_carrier_off(ndev); 2984 } 2985 2986 enetc_free_rxtx_rings(priv); 2987 2988 /* Avoids dangling pointers and also frees old resources */ 2989 enetc_assign_rx_resources(priv, NULL); 2990 enetc_assign_tx_resources(priv, NULL); 2991 2992 enetc_free_irqs(priv); 2993 clk_disable_unprepare(priv->ref_clk); 2994 2995 return 0; 2996 } 2997 EXPORT_SYMBOL_GPL(enetc_close); 2998 2999 static int enetc_reconfigure(struct enetc_ndev_priv *priv, bool extended, 3000 int (*cb)(struct enetc_ndev_priv *priv, void *ctx), 3001 void *ctx) 3002 { 3003 struct enetc_bdr_resource *tx_res, *rx_res; 3004 int err; 3005 3006 ASSERT_RTNL(); 3007 3008 /* If the interface is down, run the callback right away, 3009 * without reconfiguration. 3010 */ 3011 if (!netif_running(priv->ndev)) { 3012 if (cb) { 3013 err = cb(priv, ctx); 3014 if (err) 3015 return err; 3016 } 3017 3018 return 0; 3019 } 3020 3021 tx_res = enetc_alloc_tx_resources(priv); 3022 if (IS_ERR(tx_res)) { 3023 err = PTR_ERR(tx_res); 3024 goto out; 3025 } 3026 3027 rx_res = enetc_alloc_rx_resources(priv, extended); 3028 if (IS_ERR(rx_res)) { 3029 err = PTR_ERR(rx_res); 3030 goto out_free_tx_res; 3031 } 3032 3033 enetc_stop(priv->ndev); 3034 enetc_free_rxtx_rings(priv); 3035 3036 /* Interface is down, run optional callback now */ 3037 if (cb) { 3038 err = cb(priv, ctx); 3039 if (err) 3040 goto out_restart; 3041 } 3042 3043 enetc_assign_tx_resources(priv, tx_res); 3044 enetc_assign_rx_resources(priv, rx_res); 3045 enetc_setup_bdrs(priv, extended); 3046 enetc_start(priv->ndev); 3047 3048 return 0; 3049 3050 out_restart: 3051 enetc_setup_bdrs(priv, extended); 3052 enetc_start(priv->ndev); 3053 enetc_free_rx_resources(rx_res, priv->num_rx_rings); 3054 out_free_tx_res: 3055 enetc_free_tx_resources(tx_res, priv->num_tx_rings); 3056 out: 3057 return err; 3058 } 3059 3060 static void enetc_debug_tx_ring_prios(struct enetc_ndev_priv *priv) 3061 { 3062 int i; 3063 3064 for (i = 0; i < priv->num_tx_rings; i++) 3065 netdev_dbg(priv->ndev, "TX ring %d prio %d\n", i, 3066 priv->tx_ring[i]->prio); 3067 } 3068 3069 void enetc_reset_tc_mqprio(struct net_device *ndev) 3070 { 3071 struct enetc_ndev_priv *priv = netdev_priv(ndev); 3072 struct enetc_hw *hw = &priv->si->hw; 3073 struct enetc_bdr *tx_ring; 3074 int num_stack_tx_queues; 3075 int i; 3076 3077 num_stack_tx_queues = enetc_num_stack_tx_queues(priv); 3078 3079 netdev_reset_tc(ndev); 3080 netif_set_real_num_tx_queues(ndev, num_stack_tx_queues); 3081 priv->min_num_stack_tx_queues = num_possible_cpus(); 3082 3083 /* Reset all ring priorities to 0 */ 3084 for (i = 0; i < priv->num_tx_rings; i++) { 3085 tx_ring = priv->tx_ring[i]; 3086 tx_ring->prio = 0; 3087 enetc_set_bdr_prio(hw, tx_ring->index, tx_ring->prio); 3088 } 3089 3090 enetc_debug_tx_ring_prios(priv); 3091 3092 enetc_change_preemptible_tcs(priv, 0); 3093 } 3094 EXPORT_SYMBOL_GPL(enetc_reset_tc_mqprio); 3095 3096 int enetc_setup_tc_mqprio(struct net_device *ndev, void *type_data) 3097 { 3098 struct tc_mqprio_qopt_offload *mqprio = type_data; 3099 struct enetc_ndev_priv *priv = netdev_priv(ndev); 3100 struct tc_mqprio_qopt *qopt = &mqprio->qopt; 3101 struct enetc_hw *hw = &priv->si->hw; 3102 int num_stack_tx_queues = 0; 3103 struct enetc_bdr *tx_ring; 3104 u8 num_tc = qopt->num_tc; 3105 int offset, count; 3106 int err, tc, q; 3107 3108 if (!num_tc) { 3109 enetc_reset_tc_mqprio(ndev); 3110 return 0; 3111 } 3112 3113 err = netdev_set_num_tc(ndev, num_tc); 3114 if (err) 3115 return err; 3116 3117 for (tc = 0; tc < num_tc; tc++) { 3118 offset = qopt->offset[tc]; 3119 count = qopt->count[tc]; 3120 num_stack_tx_queues += count; 3121 3122 err = netdev_set_tc_queue(ndev, tc, count, offset); 3123 if (err) 3124 goto err_reset_tc; 3125 3126 for (q = offset; q < offset + count; q++) { 3127 tx_ring = priv->tx_ring[q]; 3128 /* The prio_tc_map is skb_tx_hash()'s way of selecting 3129 * between TX queues based on skb->priority. As such, 3130 * there's nothing to offload based on it. 3131 * Make the mqprio "traffic class" be the priority of 3132 * this ring group, and leave the Tx IPV to traffic 3133 * class mapping as its default mapping value of 1:1. 3134 */ 3135 tx_ring->prio = tc; 3136 enetc_set_bdr_prio(hw, tx_ring->index, tx_ring->prio); 3137 } 3138 } 3139 3140 err = netif_set_real_num_tx_queues(ndev, num_stack_tx_queues); 3141 if (err) 3142 goto err_reset_tc; 3143 3144 priv->min_num_stack_tx_queues = num_stack_tx_queues; 3145 3146 enetc_debug_tx_ring_prios(priv); 3147 3148 enetc_change_preemptible_tcs(priv, mqprio->preemptible_tcs); 3149 3150 return 0; 3151 3152 err_reset_tc: 3153 enetc_reset_tc_mqprio(ndev); 3154 return err; 3155 } 3156 EXPORT_SYMBOL_GPL(enetc_setup_tc_mqprio); 3157 3158 static int enetc_reconfigure_xdp_cb(struct enetc_ndev_priv *priv, void *ctx) 3159 { 3160 struct bpf_prog *old_prog, *prog = ctx; 3161 int num_stack_tx_queues; 3162 int err, i; 3163 3164 old_prog = xchg(&priv->xdp_prog, prog); 3165 3166 num_stack_tx_queues = enetc_num_stack_tx_queues(priv); 3167 err = netif_set_real_num_tx_queues(priv->ndev, num_stack_tx_queues); 3168 if (err) { 3169 xchg(&priv->xdp_prog, old_prog); 3170 return err; 3171 } 3172 3173 if (old_prog) 3174 bpf_prog_put(old_prog); 3175 3176 for (i = 0; i < priv->num_rx_rings; i++) { 3177 struct enetc_bdr *rx_ring = priv->rx_ring[i]; 3178 3179 rx_ring->xdp.prog = prog; 3180 3181 if (prog) 3182 rx_ring->buffer_offset = XDP_PACKET_HEADROOM; 3183 else 3184 rx_ring->buffer_offset = ENETC_RXB_PAD; 3185 } 3186 3187 return 0; 3188 } 3189 3190 static int enetc_setup_xdp_prog(struct net_device *ndev, struct bpf_prog *prog, 3191 struct netlink_ext_ack *extack) 3192 { 3193 int num_xdp_tx_queues = prog ? num_possible_cpus() : 0; 3194 struct enetc_ndev_priv *priv = netdev_priv(ndev); 3195 bool extended; 3196 3197 if (priv->min_num_stack_tx_queues + num_xdp_tx_queues > 3198 priv->num_tx_rings) { 3199 NL_SET_ERR_MSG_FMT_MOD(extack, 3200 "Reserving %d XDP TXQs leaves under %d for stack (total %d)", 3201 num_xdp_tx_queues, 3202 priv->min_num_stack_tx_queues, 3203 priv->num_tx_rings); 3204 return -EBUSY; 3205 } 3206 3207 extended = !!(priv->active_offloads & ENETC_F_RX_TSTAMP); 3208 3209 /* The buffer layout is changing, so we need to drain the old 3210 * RX buffers and seed new ones. 3211 */ 3212 return enetc_reconfigure(priv, extended, enetc_reconfigure_xdp_cb, prog); 3213 } 3214 3215 int enetc_setup_bpf(struct net_device *ndev, struct netdev_bpf *bpf) 3216 { 3217 switch (bpf->command) { 3218 case XDP_SETUP_PROG: 3219 return enetc_setup_xdp_prog(ndev, bpf->prog, bpf->extack); 3220 default: 3221 return -EINVAL; 3222 } 3223 3224 return 0; 3225 } 3226 EXPORT_SYMBOL_GPL(enetc_setup_bpf); 3227 3228 struct net_device_stats *enetc_get_stats(struct net_device *ndev) 3229 { 3230 struct enetc_ndev_priv *priv = netdev_priv(ndev); 3231 struct net_device_stats *stats = &ndev->stats; 3232 unsigned long packets = 0, bytes = 0; 3233 unsigned long tx_dropped = 0; 3234 int i; 3235 3236 for (i = 0; i < priv->num_rx_rings; i++) { 3237 packets += priv->rx_ring[i]->stats.packets; 3238 bytes += priv->rx_ring[i]->stats.bytes; 3239 } 3240 3241 stats->rx_packets = packets; 3242 stats->rx_bytes = bytes; 3243 bytes = 0; 3244 packets = 0; 3245 3246 for (i = 0; i < priv->num_tx_rings; i++) { 3247 packets += priv->tx_ring[i]->stats.packets; 3248 bytes += priv->tx_ring[i]->stats.bytes; 3249 tx_dropped += priv->tx_ring[i]->stats.win_drop; 3250 } 3251 3252 stats->tx_packets = packets; 3253 stats->tx_bytes = bytes; 3254 stats->tx_dropped = tx_dropped; 3255 3256 return stats; 3257 } 3258 EXPORT_SYMBOL_GPL(enetc_get_stats); 3259 3260 static void enetc_enable_rxvlan(struct net_device *ndev, bool en) 3261 { 3262 struct enetc_ndev_priv *priv = netdev_priv(ndev); 3263 struct enetc_hw *hw = &priv->si->hw; 3264 int i; 3265 3266 for (i = 0; i < priv->num_rx_rings; i++) 3267 enetc_bdr_enable_rxvlan(hw, i, en); 3268 } 3269 3270 static void enetc_enable_txvlan(struct net_device *ndev, bool en) 3271 { 3272 struct enetc_ndev_priv *priv = netdev_priv(ndev); 3273 struct enetc_hw *hw = &priv->si->hw; 3274 int i; 3275 3276 for (i = 0; i < priv->num_tx_rings; i++) 3277 enetc_bdr_enable_txvlan(hw, i, en); 3278 } 3279 3280 void enetc_set_features(struct net_device *ndev, netdev_features_t features) 3281 { 3282 netdev_features_t changed = ndev->features ^ features; 3283 3284 if (changed & NETIF_F_RXHASH) 3285 enetc_set_rss(ndev, !!(features & NETIF_F_RXHASH)); 3286 3287 if (changed & NETIF_F_HW_VLAN_CTAG_RX) 3288 enetc_enable_rxvlan(ndev, 3289 !!(features & NETIF_F_HW_VLAN_CTAG_RX)); 3290 3291 if (changed & NETIF_F_HW_VLAN_CTAG_TX) 3292 enetc_enable_txvlan(ndev, 3293 !!(features & NETIF_F_HW_VLAN_CTAG_TX)); 3294 } 3295 EXPORT_SYMBOL_GPL(enetc_set_features); 3296 3297 int enetc_hwtstamp_set(struct net_device *ndev, 3298 struct kernel_hwtstamp_config *config, 3299 struct netlink_ext_ack *extack) 3300 { 3301 struct enetc_ndev_priv *priv = netdev_priv(ndev); 3302 int err, new_offloads = priv->active_offloads; 3303 3304 if (!IS_ENABLED(CONFIG_FSL_ENETC_PTP_CLOCK)) 3305 return -EOPNOTSUPP; 3306 3307 switch (config->tx_type) { 3308 case HWTSTAMP_TX_OFF: 3309 new_offloads &= ~ENETC_F_TX_TSTAMP_MASK; 3310 break; 3311 case HWTSTAMP_TX_ON: 3312 new_offloads &= ~ENETC_F_TX_TSTAMP_MASK; 3313 new_offloads |= ENETC_F_TX_TSTAMP; 3314 break; 3315 case HWTSTAMP_TX_ONESTEP_SYNC: 3316 if (!enetc_si_is_pf(priv->si)) 3317 return -EOPNOTSUPP; 3318 3319 new_offloads &= ~ENETC_F_TX_TSTAMP_MASK; 3320 new_offloads |= ENETC_F_TX_ONESTEP_SYNC_TSTAMP; 3321 break; 3322 default: 3323 return -ERANGE; 3324 } 3325 3326 switch (config->rx_filter) { 3327 case HWTSTAMP_FILTER_NONE: 3328 new_offloads &= ~ENETC_F_RX_TSTAMP; 3329 break; 3330 default: 3331 new_offloads |= ENETC_F_RX_TSTAMP; 3332 config->rx_filter = HWTSTAMP_FILTER_ALL; 3333 } 3334 3335 if ((new_offloads ^ priv->active_offloads) & ENETC_F_RX_TSTAMP) { 3336 bool extended = !!(new_offloads & ENETC_F_RX_TSTAMP); 3337 3338 err = enetc_reconfigure(priv, extended, NULL, NULL); 3339 if (err) 3340 return err; 3341 } 3342 3343 priv->active_offloads = new_offloads; 3344 3345 return 0; 3346 } 3347 EXPORT_SYMBOL_GPL(enetc_hwtstamp_set); 3348 3349 int enetc_hwtstamp_get(struct net_device *ndev, 3350 struct kernel_hwtstamp_config *config) 3351 { 3352 struct enetc_ndev_priv *priv = netdev_priv(ndev); 3353 3354 if (!IS_ENABLED(CONFIG_FSL_ENETC_PTP_CLOCK)) 3355 return -EOPNOTSUPP; 3356 3357 if (priv->active_offloads & ENETC_F_TX_ONESTEP_SYNC_TSTAMP) 3358 config->tx_type = HWTSTAMP_TX_ONESTEP_SYNC; 3359 else if (priv->active_offloads & ENETC_F_TX_TSTAMP) 3360 config->tx_type = HWTSTAMP_TX_ON; 3361 else 3362 config->tx_type = HWTSTAMP_TX_OFF; 3363 3364 config->rx_filter = (priv->active_offloads & ENETC_F_RX_TSTAMP) ? 3365 HWTSTAMP_FILTER_ALL : HWTSTAMP_FILTER_NONE; 3366 3367 return 0; 3368 } 3369 EXPORT_SYMBOL_GPL(enetc_hwtstamp_get); 3370 3371 int enetc_ioctl(struct net_device *ndev, struct ifreq *rq, int cmd) 3372 { 3373 struct enetc_ndev_priv *priv = netdev_priv(ndev); 3374 3375 if (!priv->phylink) 3376 return -EOPNOTSUPP; 3377 3378 return phylink_mii_ioctl(priv->phylink, rq, cmd); 3379 } 3380 EXPORT_SYMBOL_GPL(enetc_ioctl); 3381 3382 static int enetc_int_vector_init(struct enetc_ndev_priv *priv, int i, 3383 int v_tx_rings) 3384 { 3385 struct enetc_int_vector *v; 3386 struct enetc_bdr *bdr; 3387 int j, err; 3388 3389 v = kzalloc(struct_size(v, tx_ring, v_tx_rings), GFP_KERNEL); 3390 if (!v) 3391 return -ENOMEM; 3392 3393 priv->int_vector[i] = v; 3394 bdr = &v->rx_ring; 3395 bdr->index = i; 3396 bdr->ndev = priv->ndev; 3397 bdr->dev = priv->dev; 3398 bdr->bd_count = priv->rx_bd_count; 3399 bdr->buffer_offset = ENETC_RXB_PAD; 3400 priv->rx_ring[i] = bdr; 3401 3402 err = __xdp_rxq_info_reg(&bdr->xdp.rxq, priv->ndev, i, 0, 3403 ENETC_RXB_DMA_SIZE_XDP); 3404 if (err) 3405 goto free_vector; 3406 3407 err = xdp_rxq_info_reg_mem_model(&bdr->xdp.rxq, MEM_TYPE_PAGE_SHARED, 3408 NULL); 3409 if (err) { 3410 xdp_rxq_info_unreg(&bdr->xdp.rxq); 3411 goto free_vector; 3412 } 3413 3414 /* init defaults for adaptive IC */ 3415 if (priv->ic_mode & ENETC_IC_RX_ADAPTIVE) { 3416 v->rx_ictt = 0x1; 3417 v->rx_dim_en = true; 3418 } 3419 3420 INIT_WORK(&v->rx_dim.work, enetc_rx_dim_work); 3421 netif_napi_add(priv->ndev, &v->napi, enetc_poll); 3422 v->count_tx_rings = v_tx_rings; 3423 3424 for (j = 0; j < v_tx_rings; j++) { 3425 int idx; 3426 3427 /* default tx ring mapping policy */ 3428 idx = priv->bdr_int_num * j + i; 3429 __set_bit(idx, &v->tx_rings_map); 3430 bdr = &v->tx_ring[j]; 3431 bdr->index = idx; 3432 bdr->ndev = priv->ndev; 3433 bdr->dev = priv->dev; 3434 bdr->bd_count = priv->tx_bd_count; 3435 priv->tx_ring[idx] = bdr; 3436 } 3437 3438 return 0; 3439 3440 free_vector: 3441 priv->rx_ring[i] = NULL; 3442 priv->int_vector[i] = NULL; 3443 kfree(v); 3444 3445 return err; 3446 } 3447 3448 static void enetc_int_vector_destroy(struct enetc_ndev_priv *priv, int i) 3449 { 3450 struct enetc_int_vector *v = priv->int_vector[i]; 3451 struct enetc_bdr *rx_ring = &v->rx_ring; 3452 int j, tx_ring_index; 3453 3454 xdp_rxq_info_unreg_mem_model(&rx_ring->xdp.rxq); 3455 xdp_rxq_info_unreg(&rx_ring->xdp.rxq); 3456 netif_napi_del(&v->napi); 3457 cancel_work_sync(&v->rx_dim.work); 3458 3459 for (j = 0; j < v->count_tx_rings; j++) { 3460 tx_ring_index = priv->bdr_int_num * j + i; 3461 priv->tx_ring[tx_ring_index] = NULL; 3462 } 3463 3464 priv->rx_ring[i] = NULL; 3465 priv->int_vector[i] = NULL; 3466 kfree(v); 3467 } 3468 3469 int enetc_alloc_msix(struct enetc_ndev_priv *priv) 3470 { 3471 struct pci_dev *pdev = priv->si->pdev; 3472 int v_tx_rings, v_remainder; 3473 int num_stack_tx_queues; 3474 int first_xdp_tx_ring; 3475 int i, n, err, nvec; 3476 3477 nvec = ENETC_BDR_INT_BASE_IDX + priv->bdr_int_num; 3478 /* allocate MSIX for both messaging and Rx/Tx interrupts */ 3479 n = pci_alloc_irq_vectors(pdev, nvec, nvec, PCI_IRQ_MSIX); 3480 3481 if (n < 0) 3482 return n; 3483 3484 if (n != nvec) 3485 return -EPERM; 3486 3487 /* # of tx rings per int vector */ 3488 v_tx_rings = priv->num_tx_rings / priv->bdr_int_num; 3489 v_remainder = priv->num_tx_rings % priv->bdr_int_num; 3490 3491 for (i = 0; i < priv->bdr_int_num; i++) { 3492 /* Distribute the remaining TX rings to the first v_remainder 3493 * interrupt vectors 3494 */ 3495 int num_tx_rings = i < v_remainder ? v_tx_rings + 1 : v_tx_rings; 3496 3497 err = enetc_int_vector_init(priv, i, num_tx_rings); 3498 if (err) 3499 goto fail; 3500 } 3501 3502 num_stack_tx_queues = enetc_num_stack_tx_queues(priv); 3503 3504 err = netif_set_real_num_tx_queues(priv->ndev, num_stack_tx_queues); 3505 if (err) 3506 goto fail; 3507 3508 err = netif_set_real_num_rx_queues(priv->ndev, priv->num_rx_rings); 3509 if (err) 3510 goto fail; 3511 3512 priv->min_num_stack_tx_queues = num_possible_cpus(); 3513 first_xdp_tx_ring = priv->num_tx_rings - num_possible_cpus(); 3514 priv->xdp_tx_ring = &priv->tx_ring[first_xdp_tx_ring]; 3515 3516 return 0; 3517 3518 fail: 3519 while (i--) 3520 enetc_int_vector_destroy(priv, i); 3521 3522 pci_free_irq_vectors(pdev); 3523 3524 return err; 3525 } 3526 EXPORT_SYMBOL_GPL(enetc_alloc_msix); 3527 3528 void enetc_free_msix(struct enetc_ndev_priv *priv) 3529 { 3530 int i; 3531 3532 for (i = 0; i < priv->bdr_int_num; i++) 3533 enetc_int_vector_destroy(priv, i); 3534 3535 /* disable all MSIX for this device */ 3536 pci_free_irq_vectors(priv->si->pdev); 3537 } 3538 EXPORT_SYMBOL_GPL(enetc_free_msix); 3539 3540 static void enetc_kfree_si(struct enetc_si *si) 3541 { 3542 char *p = (char *)si - si->pad; 3543 3544 kfree(p); 3545 } 3546 3547 static void enetc_detect_errata(struct enetc_si *si) 3548 { 3549 if (si->pdev->revision == ENETC_REV1) 3550 si->errata = ENETC_ERR_VLAN_ISOL | ENETC_ERR_UCMCSWP; 3551 } 3552 3553 int enetc_pci_probe(struct pci_dev *pdev, const char *name, int sizeof_priv) 3554 { 3555 struct enetc_si *si, *p; 3556 struct enetc_hw *hw; 3557 size_t alloc_size; 3558 int err, len; 3559 3560 pcie_flr(pdev); 3561 err = pci_enable_device_mem(pdev); 3562 if (err) 3563 return dev_err_probe(&pdev->dev, err, "device enable failed\n"); 3564 3565 /* set up for high or low dma */ 3566 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 3567 if (err) { 3568 dev_err(&pdev->dev, "DMA configuration failed: 0x%x\n", err); 3569 goto err_dma; 3570 } 3571 3572 err = pci_request_mem_regions(pdev, name); 3573 if (err) { 3574 dev_err(&pdev->dev, "pci_request_regions failed err=%d\n", err); 3575 goto err_pci_mem_reg; 3576 } 3577 3578 pci_set_master(pdev); 3579 3580 alloc_size = sizeof(struct enetc_si); 3581 if (sizeof_priv) { 3582 /* align priv to 32B */ 3583 alloc_size = ALIGN(alloc_size, ENETC_SI_ALIGN); 3584 alloc_size += sizeof_priv; 3585 } 3586 /* force 32B alignment for enetc_si */ 3587 alloc_size += ENETC_SI_ALIGN - 1; 3588 3589 p = kzalloc(alloc_size, GFP_KERNEL); 3590 if (!p) { 3591 err = -ENOMEM; 3592 goto err_alloc_si; 3593 } 3594 3595 si = PTR_ALIGN(p, ENETC_SI_ALIGN); 3596 si->pad = (char *)si - (char *)p; 3597 3598 pci_set_drvdata(pdev, si); 3599 si->pdev = pdev; 3600 hw = &si->hw; 3601 3602 len = pci_resource_len(pdev, ENETC_BAR_REGS); 3603 hw->reg = ioremap(pci_resource_start(pdev, ENETC_BAR_REGS), len); 3604 if (!hw->reg) { 3605 err = -ENXIO; 3606 dev_err(&pdev->dev, "ioremap() failed\n"); 3607 goto err_ioremap; 3608 } 3609 if (len > ENETC_PORT_BASE) 3610 hw->port = hw->reg + ENETC_PORT_BASE; 3611 if (len > ENETC_GLOBAL_BASE) 3612 hw->global = hw->reg + ENETC_GLOBAL_BASE; 3613 3614 enetc_detect_errata(si); 3615 3616 return 0; 3617 3618 err_ioremap: 3619 enetc_kfree_si(si); 3620 err_alloc_si: 3621 pci_release_mem_regions(pdev); 3622 err_pci_mem_reg: 3623 err_dma: 3624 pci_disable_device(pdev); 3625 3626 return err; 3627 } 3628 EXPORT_SYMBOL_GPL(enetc_pci_probe); 3629 3630 void enetc_pci_remove(struct pci_dev *pdev) 3631 { 3632 struct enetc_si *si = pci_get_drvdata(pdev); 3633 struct enetc_hw *hw = &si->hw; 3634 3635 iounmap(hw->reg); 3636 enetc_kfree_si(si); 3637 pci_release_mem_regions(pdev); 3638 pci_disable_device(pdev); 3639 } 3640 EXPORT_SYMBOL_GPL(enetc_pci_remove); 3641 3642 static const struct enetc_drvdata enetc_pf_data = { 3643 .sysclk_freq = ENETC_CLK_400M, 3644 .pmac_offset = ENETC_PMAC_OFFSET, 3645 .max_frags = ENETC_MAX_SKB_FRAGS, 3646 .eth_ops = &enetc_pf_ethtool_ops, 3647 }; 3648 3649 static const struct enetc_drvdata enetc4_pf_data = { 3650 .sysclk_freq = ENETC_CLK_333M, 3651 .tx_csum = true, 3652 .max_frags = ENETC4_MAX_SKB_FRAGS, 3653 .pmac_offset = ENETC4_PMAC_OFFSET, 3654 .eth_ops = &enetc4_pf_ethtool_ops, 3655 }; 3656 3657 static const struct enetc_drvdata enetc_vf_data = { 3658 .sysclk_freq = ENETC_CLK_400M, 3659 .max_frags = ENETC_MAX_SKB_FRAGS, 3660 .eth_ops = &enetc_vf_ethtool_ops, 3661 }; 3662 3663 static const struct enetc_platform_info enetc_info[] = { 3664 { .revision = ENETC_REV_1_0, 3665 .dev_id = ENETC_DEV_ID_PF, 3666 .data = &enetc_pf_data, 3667 }, 3668 { .revision = ENETC_REV_4_1, 3669 .dev_id = NXP_ENETC_PF_DEV_ID, 3670 .data = &enetc4_pf_data, 3671 }, 3672 { .revision = ENETC_REV_1_0, 3673 .dev_id = ENETC_DEV_ID_VF, 3674 .data = &enetc_vf_data, 3675 }, 3676 }; 3677 3678 int enetc_get_driver_data(struct enetc_si *si) 3679 { 3680 u16 dev_id = si->pdev->device; 3681 int i; 3682 3683 for (i = 0; i < ARRAY_SIZE(enetc_info); i++) { 3684 if (si->revision == enetc_info[i].revision && 3685 dev_id == enetc_info[i].dev_id) { 3686 si->drvdata = enetc_info[i].data; 3687 3688 return 0; 3689 } 3690 } 3691 3692 return -ERANGE; 3693 } 3694 EXPORT_SYMBOL_GPL(enetc_get_driver_data); 3695 3696 MODULE_DESCRIPTION("NXP ENETC Ethernet driver"); 3697 MODULE_LICENSE("Dual BSD/GPL"); 3698