1 // SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) 2 /* Copyright 2017-2019 NXP */ 3 4 #include "enetc.h" 5 #include <linux/bpf_trace.h> 6 #include <linux/clk.h> 7 #include <linux/tcp.h> 8 #include <linux/udp.h> 9 #include <linux/vmalloc.h> 10 #include <linux/ptp_classify.h> 11 #include <net/ip6_checksum.h> 12 #include <net/pkt_sched.h> 13 #include <net/tso.h> 14 15 u32 enetc_port_mac_rd(struct enetc_si *si, u32 reg) 16 { 17 return enetc_port_rd(&si->hw, reg); 18 } 19 EXPORT_SYMBOL_GPL(enetc_port_mac_rd); 20 21 void enetc_port_mac_wr(struct enetc_si *si, u32 reg, u32 val) 22 { 23 enetc_port_wr(&si->hw, reg, val); 24 if (si->hw_features & ENETC_SI_F_QBU) 25 enetc_port_wr(&si->hw, reg + si->drvdata->pmac_offset, val); 26 } 27 EXPORT_SYMBOL_GPL(enetc_port_mac_wr); 28 29 static void enetc_change_preemptible_tcs(struct enetc_ndev_priv *priv, 30 u8 preemptible_tcs) 31 { 32 if (!(priv->si->hw_features & ENETC_SI_F_QBU)) 33 return; 34 35 priv->preemptible_tcs = preemptible_tcs; 36 enetc_mm_commit_preemptible_tcs(priv); 37 } 38 39 static int enetc_mac_addr_hash_idx(const u8 *addr) 40 { 41 u64 fold = __swab64(ether_addr_to_u64(addr)) >> 16; 42 u64 mask = 0; 43 int res = 0; 44 int i; 45 46 for (i = 0; i < 8; i++) 47 mask |= BIT_ULL(i * 6); 48 49 for (i = 0; i < 6; i++) 50 res |= (hweight64(fold & (mask << i)) & 0x1) << i; 51 52 return res; 53 } 54 55 void enetc_add_mac_addr_ht_filter(struct enetc_mac_filter *filter, 56 const unsigned char *addr) 57 { 58 int idx = enetc_mac_addr_hash_idx(addr); 59 60 /* add hash table entry */ 61 __set_bit(idx, filter->mac_hash_table); 62 filter->mac_addr_cnt++; 63 } 64 EXPORT_SYMBOL_GPL(enetc_add_mac_addr_ht_filter); 65 66 void enetc_reset_mac_addr_filter(struct enetc_mac_filter *filter) 67 { 68 filter->mac_addr_cnt = 0; 69 70 bitmap_zero(filter->mac_hash_table, 71 ENETC_MADDR_HASH_TBL_SZ); 72 } 73 EXPORT_SYMBOL_GPL(enetc_reset_mac_addr_filter); 74 75 static int enetc_num_stack_tx_queues(struct enetc_ndev_priv *priv) 76 { 77 int num_tx_rings = priv->num_tx_rings; 78 79 if (priv->xdp_prog) 80 return num_tx_rings - num_possible_cpus(); 81 82 return num_tx_rings; 83 } 84 85 static struct enetc_bdr *enetc_rx_ring_from_xdp_tx_ring(struct enetc_ndev_priv *priv, 86 struct enetc_bdr *tx_ring) 87 { 88 int index = &priv->tx_ring[tx_ring->index] - priv->xdp_tx_ring; 89 90 return priv->rx_ring[index]; 91 } 92 93 static struct sk_buff *enetc_tx_swbd_get_skb(struct enetc_tx_swbd *tx_swbd) 94 { 95 if (tx_swbd->is_xdp_tx || tx_swbd->is_xdp_redirect) 96 return NULL; 97 98 return tx_swbd->skb; 99 } 100 101 static struct xdp_frame * 102 enetc_tx_swbd_get_xdp_frame(struct enetc_tx_swbd *tx_swbd) 103 { 104 if (tx_swbd->is_xdp_redirect) 105 return tx_swbd->xdp_frame; 106 107 return NULL; 108 } 109 110 static void enetc_unmap_tx_buff(struct enetc_bdr *tx_ring, 111 struct enetc_tx_swbd *tx_swbd) 112 { 113 /* For XDP_TX, pages come from RX, whereas for the other contexts where 114 * we have is_dma_page_set, those come from skb_frag_dma_map. We need 115 * to match the DMA mapping length, so we need to differentiate those. 116 */ 117 if (tx_swbd->is_dma_page) 118 dma_unmap_page(tx_ring->dev, tx_swbd->dma, 119 tx_swbd->is_xdp_tx ? PAGE_SIZE : tx_swbd->len, 120 tx_swbd->dir); 121 else 122 dma_unmap_single(tx_ring->dev, tx_swbd->dma, 123 tx_swbd->len, tx_swbd->dir); 124 tx_swbd->dma = 0; 125 } 126 127 static void enetc_free_tx_frame(struct enetc_bdr *tx_ring, 128 struct enetc_tx_swbd *tx_swbd) 129 { 130 struct xdp_frame *xdp_frame = enetc_tx_swbd_get_xdp_frame(tx_swbd); 131 struct sk_buff *skb = enetc_tx_swbd_get_skb(tx_swbd); 132 133 if (tx_swbd->dma) 134 enetc_unmap_tx_buff(tx_ring, tx_swbd); 135 136 if (xdp_frame) { 137 xdp_return_frame(tx_swbd->xdp_frame); 138 tx_swbd->xdp_frame = NULL; 139 } else if (skb) { 140 dev_kfree_skb_any(skb); 141 tx_swbd->skb = NULL; 142 } 143 } 144 145 /* Let H/W know BD ring has been updated */ 146 static void enetc_update_tx_ring_tail(struct enetc_bdr *tx_ring) 147 { 148 /* includes wmb() */ 149 enetc_wr_reg_hot(tx_ring->tpir, tx_ring->next_to_use); 150 } 151 152 static int enetc_ptp_parse(struct sk_buff *skb, u8 *udp, 153 u8 *msgtype, u8 *twostep, 154 u16 *correction_offset, u16 *body_offset) 155 { 156 unsigned int ptp_class; 157 struct ptp_header *hdr; 158 unsigned int type; 159 u8 *base; 160 161 ptp_class = ptp_classify_raw(skb); 162 if (ptp_class == PTP_CLASS_NONE) 163 return -EINVAL; 164 165 hdr = ptp_parse_header(skb, ptp_class); 166 if (!hdr) 167 return -EINVAL; 168 169 type = ptp_class & PTP_CLASS_PMASK; 170 if (type == PTP_CLASS_IPV4 || type == PTP_CLASS_IPV6) 171 *udp = 1; 172 else 173 *udp = 0; 174 175 *msgtype = ptp_get_msgtype(hdr, ptp_class); 176 *twostep = hdr->flag_field[0] & 0x2; 177 178 base = skb_mac_header(skb); 179 *correction_offset = (u8 *)&hdr->correction - base; 180 *body_offset = (u8 *)hdr + sizeof(struct ptp_header) - base; 181 182 return 0; 183 } 184 185 static bool enetc_tx_csum_offload_check(struct sk_buff *skb) 186 { 187 switch (skb->csum_offset) { 188 case offsetof(struct tcphdr, check): 189 case offsetof(struct udphdr, check): 190 return true; 191 default: 192 return false; 193 } 194 } 195 196 static bool enetc_skb_is_ipv6(struct sk_buff *skb) 197 { 198 return vlan_get_protocol(skb) == htons(ETH_P_IPV6); 199 } 200 201 static bool enetc_skb_is_tcp(struct sk_buff *skb) 202 { 203 return skb->csum_offset == offsetof(struct tcphdr, check); 204 } 205 206 /** 207 * enetc_unwind_tx_frame() - Unwind the DMA mappings of a multi-buffer Tx frame 208 * @tx_ring: Pointer to the Tx ring on which the buffer descriptors are located 209 * @count: Number of Tx buffer descriptors which need to be unmapped 210 * @i: Index of the last successfully mapped Tx buffer descriptor 211 */ 212 static void enetc_unwind_tx_frame(struct enetc_bdr *tx_ring, int count, int i) 213 { 214 while (count--) { 215 struct enetc_tx_swbd *tx_swbd = &tx_ring->tx_swbd[i]; 216 217 enetc_free_tx_frame(tx_ring, tx_swbd); 218 if (i == 0) 219 i = tx_ring->bd_count; 220 i--; 221 } 222 } 223 224 static void enetc_set_one_step_ts(struct enetc_si *si, bool udp, int offset) 225 { 226 u32 val = ENETC_PM0_SINGLE_STEP_EN; 227 228 val |= ENETC_SET_SINGLE_STEP_OFFSET(offset); 229 if (udp) 230 val |= ENETC_PM0_SINGLE_STEP_CH; 231 232 /* The "Correction" field of a packet is updated based on the 233 * current time and the timestamp provided 234 */ 235 enetc_port_mac_wr(si, ENETC_PM0_SINGLE_STEP, val); 236 } 237 238 static void enetc4_set_one_step_ts(struct enetc_si *si, bool udp, int offset) 239 { 240 u32 val = PM_SINGLE_STEP_EN; 241 242 val |= PM_SINGLE_STEP_OFFSET_SET(offset); 243 if (udp) 244 val |= PM_SINGLE_STEP_CH; 245 246 enetc_port_mac_wr(si, ENETC4_PM_SINGLE_STEP(0), val); 247 } 248 249 static u32 enetc_update_ptp_sync_msg(struct enetc_ndev_priv *priv, 250 struct sk_buff *skb, bool csum_offload) 251 { 252 struct enetc_skb_cb *enetc_cb = ENETC_SKB_CB(skb); 253 u16 tstamp_off = enetc_cb->origin_tstamp_off; 254 u16 corr_off = enetc_cb->correction_off; 255 struct enetc_si *si = priv->si; 256 struct enetc_hw *hw = &si->hw; 257 __be32 new_sec_l, new_nsec; 258 __be16 new_sec_h; 259 u32 lo, hi, nsec; 260 u8 *data; 261 u64 sec; 262 263 lo = enetc_rd_hot(hw, ENETC_SICTR0); 264 hi = enetc_rd_hot(hw, ENETC_SICTR1); 265 sec = (u64)hi << 32 | lo; 266 nsec = do_div(sec, 1000000000); 267 268 /* Update originTimestamp field of Sync packet 269 * - 48 bits seconds field 270 * - 32 bits nanseconds field 271 * 272 * In addition, if csum_offload is false, the UDP checksum needs 273 * to be updated by software after updating originTimestamp field, 274 * otherwise the hardware will calculate the wrong checksum when 275 * updating the correction field and update it to the packet. 276 */ 277 278 data = skb_mac_header(skb); 279 new_sec_h = htons((sec >> 32) & 0xffff); 280 new_sec_l = htonl(sec & 0xffffffff); 281 new_nsec = htonl(nsec); 282 if (enetc_cb->udp && !csum_offload) { 283 struct udphdr *uh = udp_hdr(skb); 284 __be32 old_sec_l, old_nsec; 285 __be16 old_sec_h; 286 287 old_sec_h = *(__be16 *)(data + tstamp_off); 288 inet_proto_csum_replace2(&uh->check, skb, old_sec_h, 289 new_sec_h, false); 290 291 old_sec_l = *(__be32 *)(data + tstamp_off + 2); 292 inet_proto_csum_replace4(&uh->check, skb, old_sec_l, 293 new_sec_l, false); 294 295 old_nsec = *(__be32 *)(data + tstamp_off + 6); 296 inet_proto_csum_replace4(&uh->check, skb, old_nsec, 297 new_nsec, false); 298 } 299 300 *(__be16 *)(data + tstamp_off) = new_sec_h; 301 *(__be32 *)(data + tstamp_off + 2) = new_sec_l; 302 *(__be32 *)(data + tstamp_off + 6) = new_nsec; 303 304 /* Configure single-step register */ 305 if (is_enetc_rev1(si)) 306 enetc_set_one_step_ts(si, enetc_cb->udp, corr_off); 307 else 308 enetc4_set_one_step_ts(si, enetc_cb->udp, corr_off); 309 310 return lo & ENETC_TXBD_TSTAMP; 311 } 312 313 static int enetc_map_tx_buffs(struct enetc_bdr *tx_ring, struct sk_buff *skb) 314 { 315 bool do_vlan, do_onestep_tstamp = false, do_twostep_tstamp = false; 316 struct enetc_ndev_priv *priv = netdev_priv(tx_ring->ndev); 317 struct enetc_skb_cb *enetc_cb = ENETC_SKB_CB(skb); 318 struct enetc_tx_swbd *tx_swbd; 319 int len = skb_headlen(skb); 320 union enetc_tx_bd temp_bd; 321 bool csum_offload = false; 322 union enetc_tx_bd *txbd; 323 int i, count = 0; 324 skb_frag_t *frag; 325 unsigned int f; 326 dma_addr_t dma; 327 u8 flags = 0; 328 u32 tstamp; 329 330 enetc_clear_tx_bd(&temp_bd); 331 if (skb->ip_summed == CHECKSUM_PARTIAL) { 332 /* Can not support TSD and checksum offload at the same time */ 333 if (priv->active_offloads & ENETC_F_TXCSUM && 334 enetc_tx_csum_offload_check(skb) && !tx_ring->tsd_enable) { 335 temp_bd.l3_aux0 = FIELD_PREP(ENETC_TX_BD_L3_START, 336 skb_network_offset(skb)); 337 temp_bd.l3_aux1 = FIELD_PREP(ENETC_TX_BD_L3_HDR_LEN, 338 skb_network_header_len(skb) / 4); 339 temp_bd.l3_aux1 |= FIELD_PREP(ENETC_TX_BD_L3T, 340 enetc_skb_is_ipv6(skb)); 341 if (enetc_skb_is_tcp(skb)) 342 temp_bd.l4_aux = FIELD_PREP(ENETC_TX_BD_L4T, 343 ENETC_TXBD_L4T_TCP); 344 else 345 temp_bd.l4_aux = FIELD_PREP(ENETC_TX_BD_L4T, 346 ENETC_TXBD_L4T_UDP); 347 flags |= ENETC_TXBD_FLAGS_CSUM_LSO | ENETC_TXBD_FLAGS_L4CS; 348 csum_offload = true; 349 } else if (skb_checksum_help(skb)) { 350 return 0; 351 } 352 } 353 354 if (enetc_cb->flag & ENETC_F_TX_ONESTEP_SYNC_TSTAMP) { 355 do_onestep_tstamp = true; 356 tstamp = enetc_update_ptp_sync_msg(priv, skb, csum_offload); 357 } else if (enetc_cb->flag & ENETC_F_TX_TSTAMP) { 358 do_twostep_tstamp = true; 359 } 360 361 i = tx_ring->next_to_use; 362 txbd = ENETC_TXBD(*tx_ring, i); 363 prefetchw(txbd); 364 365 dma = dma_map_single(tx_ring->dev, skb->data, len, DMA_TO_DEVICE); 366 if (unlikely(dma_mapping_error(tx_ring->dev, dma))) 367 goto dma_err; 368 369 temp_bd.addr = cpu_to_le64(dma); 370 temp_bd.buf_len = cpu_to_le16(len); 371 372 tx_swbd = &tx_ring->tx_swbd[i]; 373 tx_swbd->dma = dma; 374 tx_swbd->len = len; 375 tx_swbd->is_dma_page = 0; 376 tx_swbd->dir = DMA_TO_DEVICE; 377 count++; 378 379 do_vlan = skb_vlan_tag_present(skb); 380 tx_swbd->do_twostep_tstamp = do_twostep_tstamp; 381 tx_swbd->qbv_en = !!(priv->active_offloads & ENETC_F_QBV); 382 tx_swbd->check_wb = tx_swbd->do_twostep_tstamp || tx_swbd->qbv_en; 383 384 if (do_vlan || do_onestep_tstamp || do_twostep_tstamp) 385 flags |= ENETC_TXBD_FLAGS_EX; 386 387 if (tx_ring->tsd_enable) 388 flags |= ENETC_TXBD_FLAGS_TSE | ENETC_TXBD_FLAGS_TXSTART; 389 390 /* first BD needs frm_len and offload flags set */ 391 temp_bd.frm_len = cpu_to_le16(skb->len); 392 temp_bd.flags = flags; 393 394 if (flags & ENETC_TXBD_FLAGS_TSE) 395 temp_bd.txstart = enetc_txbd_set_tx_start(skb->skb_mstamp_ns, 396 flags); 397 398 if (flags & ENETC_TXBD_FLAGS_EX) { 399 u8 e_flags = 0; 400 *txbd = temp_bd; 401 enetc_clear_tx_bd(&temp_bd); 402 403 /* add extension BD for VLAN and/or timestamping */ 404 flags = 0; 405 tx_swbd++; 406 txbd++; 407 i++; 408 if (unlikely(i == tx_ring->bd_count)) { 409 i = 0; 410 tx_swbd = tx_ring->tx_swbd; 411 txbd = ENETC_TXBD(*tx_ring, 0); 412 } 413 prefetchw(txbd); 414 415 if (do_vlan) { 416 temp_bd.ext.vid = cpu_to_le16(skb_vlan_tag_get(skb)); 417 temp_bd.ext.tpid = 0; /* < C-TAG */ 418 e_flags |= ENETC_TXBD_E_FLAGS_VLAN_INS; 419 } 420 421 if (do_onestep_tstamp) { 422 /* Configure extension BD */ 423 temp_bd.ext.tstamp = cpu_to_le32(tstamp); 424 e_flags |= ENETC_TXBD_E_FLAGS_ONE_STEP_PTP; 425 } else if (do_twostep_tstamp) { 426 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; 427 e_flags |= ENETC_TXBD_E_FLAGS_TWO_STEP_PTP; 428 } 429 430 temp_bd.ext.e_flags = e_flags; 431 count++; 432 } 433 434 frag = &skb_shinfo(skb)->frags[0]; 435 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++, frag++) { 436 len = skb_frag_size(frag); 437 dma = skb_frag_dma_map(tx_ring->dev, frag, 0, len, 438 DMA_TO_DEVICE); 439 if (dma_mapping_error(tx_ring->dev, dma)) 440 goto dma_err; 441 442 *txbd = temp_bd; 443 enetc_clear_tx_bd(&temp_bd); 444 445 flags = 0; 446 tx_swbd++; 447 txbd++; 448 i++; 449 if (unlikely(i == tx_ring->bd_count)) { 450 i = 0; 451 tx_swbd = tx_ring->tx_swbd; 452 txbd = ENETC_TXBD(*tx_ring, 0); 453 } 454 prefetchw(txbd); 455 456 temp_bd.addr = cpu_to_le64(dma); 457 temp_bd.buf_len = cpu_to_le16(len); 458 459 tx_swbd->dma = dma; 460 tx_swbd->len = len; 461 tx_swbd->is_dma_page = 1; 462 tx_swbd->dir = DMA_TO_DEVICE; 463 count++; 464 } 465 466 /* last BD needs 'F' bit set */ 467 flags |= ENETC_TXBD_FLAGS_F; 468 temp_bd.flags = flags; 469 *txbd = temp_bd; 470 471 tx_ring->tx_swbd[i].is_eof = true; 472 tx_ring->tx_swbd[i].skb = skb; 473 474 enetc_bdr_idx_inc(tx_ring, &i); 475 tx_ring->next_to_use = i; 476 477 skb_tx_timestamp(skb); 478 479 enetc_update_tx_ring_tail(tx_ring); 480 481 return count; 482 483 dma_err: 484 dev_err(tx_ring->dev, "DMA map error"); 485 486 enetc_unwind_tx_frame(tx_ring, count, i); 487 488 return 0; 489 } 490 491 static int enetc_map_tx_tso_hdr(struct enetc_bdr *tx_ring, struct sk_buff *skb, 492 struct enetc_tx_swbd *tx_swbd, 493 union enetc_tx_bd *txbd, int *i, int hdr_len, 494 int data_len) 495 { 496 union enetc_tx_bd txbd_tmp; 497 u8 flags = 0, e_flags = 0; 498 dma_addr_t addr; 499 int count = 1; 500 501 enetc_clear_tx_bd(&txbd_tmp); 502 addr = tx_ring->tso_headers_dma + *i * TSO_HEADER_SIZE; 503 504 if (skb_vlan_tag_present(skb)) 505 flags |= ENETC_TXBD_FLAGS_EX; 506 507 txbd_tmp.addr = cpu_to_le64(addr); 508 txbd_tmp.buf_len = cpu_to_le16(hdr_len); 509 510 /* first BD needs frm_len and offload flags set */ 511 txbd_tmp.frm_len = cpu_to_le16(hdr_len + data_len); 512 txbd_tmp.flags = flags; 513 514 /* For the TSO header we do not set the dma address since we do not 515 * want it unmapped when we do cleanup. We still set len so that we 516 * count the bytes sent. 517 */ 518 tx_swbd->len = hdr_len; 519 tx_swbd->do_twostep_tstamp = false; 520 tx_swbd->check_wb = false; 521 522 /* Actually write the header in the BD */ 523 *txbd = txbd_tmp; 524 525 /* Add extension BD for VLAN */ 526 if (flags & ENETC_TXBD_FLAGS_EX) { 527 /* Get the next BD */ 528 enetc_bdr_idx_inc(tx_ring, i); 529 txbd = ENETC_TXBD(*tx_ring, *i); 530 tx_swbd = &tx_ring->tx_swbd[*i]; 531 prefetchw(txbd); 532 533 /* Setup the VLAN fields */ 534 enetc_clear_tx_bd(&txbd_tmp); 535 txbd_tmp.ext.vid = cpu_to_le16(skb_vlan_tag_get(skb)); 536 txbd_tmp.ext.tpid = 0; /* < C-TAG */ 537 e_flags |= ENETC_TXBD_E_FLAGS_VLAN_INS; 538 539 /* Write the BD */ 540 txbd_tmp.ext.e_flags = e_flags; 541 *txbd = txbd_tmp; 542 count++; 543 } 544 545 return count; 546 } 547 548 static int enetc_map_tx_tso_data(struct enetc_bdr *tx_ring, struct sk_buff *skb, 549 struct enetc_tx_swbd *tx_swbd, 550 union enetc_tx_bd *txbd, char *data, 551 int size, bool last_bd) 552 { 553 union enetc_tx_bd txbd_tmp; 554 dma_addr_t addr; 555 u8 flags = 0; 556 557 enetc_clear_tx_bd(&txbd_tmp); 558 559 addr = dma_map_single(tx_ring->dev, data, size, DMA_TO_DEVICE); 560 if (unlikely(dma_mapping_error(tx_ring->dev, addr))) { 561 netdev_err(tx_ring->ndev, "DMA map error\n"); 562 return -ENOMEM; 563 } 564 565 if (last_bd) { 566 flags |= ENETC_TXBD_FLAGS_F; 567 tx_swbd->is_eof = 1; 568 } 569 570 txbd_tmp.addr = cpu_to_le64(addr); 571 txbd_tmp.buf_len = cpu_to_le16(size); 572 txbd_tmp.flags = flags; 573 574 tx_swbd->dma = addr; 575 tx_swbd->len = size; 576 tx_swbd->dir = DMA_TO_DEVICE; 577 578 *txbd = txbd_tmp; 579 580 return 0; 581 } 582 583 static __wsum enetc_tso_hdr_csum(struct tso_t *tso, struct sk_buff *skb, 584 char *hdr, int hdr_len, int *l4_hdr_len) 585 { 586 char *l4_hdr = hdr + skb_transport_offset(skb); 587 int mac_hdr_len = skb_network_offset(skb); 588 589 if (tso->tlen != sizeof(struct udphdr)) { 590 struct tcphdr *tcph = (struct tcphdr *)(l4_hdr); 591 592 tcph->check = 0; 593 } else { 594 struct udphdr *udph = (struct udphdr *)(l4_hdr); 595 596 udph->check = 0; 597 } 598 599 /* Compute the IP checksum. This is necessary since tso_build_hdr() 600 * already incremented the IP ID field. 601 */ 602 if (!tso->ipv6) { 603 struct iphdr *iph = (void *)(hdr + mac_hdr_len); 604 605 iph->check = 0; 606 iph->check = ip_fast_csum((unsigned char *)iph, iph->ihl); 607 } 608 609 /* Compute the checksum over the L4 header. */ 610 *l4_hdr_len = hdr_len - skb_transport_offset(skb); 611 return csum_partial(l4_hdr, *l4_hdr_len, 0); 612 } 613 614 static void enetc_tso_complete_csum(struct enetc_bdr *tx_ring, struct tso_t *tso, 615 struct sk_buff *skb, char *hdr, int len, 616 __wsum sum) 617 { 618 char *l4_hdr = hdr + skb_transport_offset(skb); 619 __sum16 csum_final; 620 621 /* Complete the L4 checksum by appending the pseudo-header to the 622 * already computed checksum. 623 */ 624 if (!tso->ipv6) 625 csum_final = csum_tcpudp_magic(ip_hdr(skb)->saddr, 626 ip_hdr(skb)->daddr, 627 len, ip_hdr(skb)->protocol, sum); 628 else 629 csum_final = csum_ipv6_magic(&ipv6_hdr(skb)->saddr, 630 &ipv6_hdr(skb)->daddr, 631 len, ipv6_hdr(skb)->nexthdr, sum); 632 633 if (tso->tlen != sizeof(struct udphdr)) { 634 struct tcphdr *tcph = (struct tcphdr *)(l4_hdr); 635 636 tcph->check = csum_final; 637 } else { 638 struct udphdr *udph = (struct udphdr *)(l4_hdr); 639 640 udph->check = csum_final; 641 } 642 } 643 644 static int enetc_lso_count_descs(const struct sk_buff *skb) 645 { 646 /* 4 BDs: 1 BD for LSO header + 1 BD for extended BD + 1 BD 647 * for linear area data but not include LSO header, namely 648 * skb_headlen(skb) - lso_hdr_len (it may be 0, but that's 649 * okay, we only need to consider the worst case). And 1 BD 650 * for gap. 651 */ 652 return skb_shinfo(skb)->nr_frags + 4; 653 } 654 655 static int enetc_lso_get_hdr_len(const struct sk_buff *skb) 656 { 657 int hdr_len, tlen; 658 659 tlen = skb_is_gso_tcp(skb) ? tcp_hdrlen(skb) : sizeof(struct udphdr); 660 hdr_len = skb_transport_offset(skb) + tlen; 661 662 return hdr_len; 663 } 664 665 static void enetc_lso_start(struct sk_buff *skb, struct enetc_lso_t *lso) 666 { 667 lso->lso_seg_size = skb_shinfo(skb)->gso_size; 668 lso->ipv6 = enetc_skb_is_ipv6(skb); 669 lso->tcp = skb_is_gso_tcp(skb); 670 lso->l3_hdr_len = skb_network_header_len(skb); 671 lso->l3_start = skb_network_offset(skb); 672 lso->hdr_len = enetc_lso_get_hdr_len(skb); 673 lso->total_len = skb->len - lso->hdr_len; 674 } 675 676 static void enetc_lso_map_hdr(struct enetc_bdr *tx_ring, struct sk_buff *skb, 677 int *i, struct enetc_lso_t *lso) 678 { 679 union enetc_tx_bd txbd_tmp, *txbd; 680 struct enetc_tx_swbd *tx_swbd; 681 u16 frm_len, frm_len_ext; 682 u8 flags, e_flags = 0; 683 dma_addr_t addr; 684 char *hdr; 685 686 /* Get the first BD of the LSO BDs chain */ 687 txbd = ENETC_TXBD(*tx_ring, *i); 688 tx_swbd = &tx_ring->tx_swbd[*i]; 689 prefetchw(txbd); 690 691 /* Prepare LSO header: MAC + IP + TCP/UDP */ 692 hdr = tx_ring->tso_headers + *i * TSO_HEADER_SIZE; 693 memcpy(hdr, skb->data, lso->hdr_len); 694 addr = tx_ring->tso_headers_dma + *i * TSO_HEADER_SIZE; 695 696 /* {frm_len_ext, frm_len} indicates the total length of 697 * large transmit data unit. frm_len contains the 16 least 698 * significant bits and frm_len_ext contains the 4 most 699 * significant bits. 700 */ 701 frm_len = lso->total_len & 0xffff; 702 frm_len_ext = (lso->total_len >> 16) & 0xf; 703 704 /* Set the flags of the first BD */ 705 flags = ENETC_TXBD_FLAGS_EX | ENETC_TXBD_FLAGS_CSUM_LSO | 706 ENETC_TXBD_FLAGS_LSO | ENETC_TXBD_FLAGS_L4CS; 707 708 enetc_clear_tx_bd(&txbd_tmp); 709 txbd_tmp.addr = cpu_to_le64(addr); 710 txbd_tmp.hdr_len = cpu_to_le16(lso->hdr_len); 711 712 /* first BD needs frm_len and offload flags set */ 713 txbd_tmp.frm_len = cpu_to_le16(frm_len); 714 txbd_tmp.flags = flags; 715 716 txbd_tmp.l3_aux0 = FIELD_PREP(ENETC_TX_BD_L3_START, lso->l3_start); 717 /* l3_hdr_size in 32-bits (4 bytes) */ 718 txbd_tmp.l3_aux1 = FIELD_PREP(ENETC_TX_BD_L3_HDR_LEN, 719 lso->l3_hdr_len / 4); 720 if (lso->ipv6) 721 txbd_tmp.l3_aux1 |= ENETC_TX_BD_L3T; 722 else 723 txbd_tmp.l3_aux0 |= ENETC_TX_BD_IPCS; 724 725 txbd_tmp.l4_aux = FIELD_PREP(ENETC_TX_BD_L4T, lso->tcp ? 726 ENETC_TXBD_L4T_TCP : ENETC_TXBD_L4T_UDP); 727 728 /* For the LSO header we do not set the dma address since 729 * we do not want it unmapped when we do cleanup. We still 730 * set len so that we count the bytes sent. 731 */ 732 tx_swbd->len = lso->hdr_len; 733 tx_swbd->do_twostep_tstamp = false; 734 tx_swbd->check_wb = false; 735 736 /* Actually write the header in the BD */ 737 *txbd = txbd_tmp; 738 739 /* Get the next BD, and the next BD is extended BD */ 740 enetc_bdr_idx_inc(tx_ring, i); 741 txbd = ENETC_TXBD(*tx_ring, *i); 742 tx_swbd = &tx_ring->tx_swbd[*i]; 743 prefetchw(txbd); 744 745 enetc_clear_tx_bd(&txbd_tmp); 746 if (skb_vlan_tag_present(skb)) { 747 /* Setup the VLAN fields */ 748 txbd_tmp.ext.vid = cpu_to_le16(skb_vlan_tag_get(skb)); 749 txbd_tmp.ext.tpid = ENETC_TPID_8021Q; 750 e_flags = ENETC_TXBD_E_FLAGS_VLAN_INS; 751 } 752 753 /* Write the BD */ 754 txbd_tmp.ext.e_flags = e_flags; 755 txbd_tmp.ext.lso_sg_size = cpu_to_le16(lso->lso_seg_size); 756 txbd_tmp.ext.frm_len_ext = cpu_to_le16(frm_len_ext); 757 *txbd = txbd_tmp; 758 } 759 760 static int enetc_lso_map_data(struct enetc_bdr *tx_ring, struct sk_buff *skb, 761 int *i, struct enetc_lso_t *lso, int *count) 762 { 763 union enetc_tx_bd txbd_tmp, *txbd = NULL; 764 struct enetc_tx_swbd *tx_swbd; 765 skb_frag_t *frag; 766 dma_addr_t dma; 767 u8 flags = 0; 768 int len, f; 769 770 len = skb_headlen(skb) - lso->hdr_len; 771 if (len > 0) { 772 dma = dma_map_single(tx_ring->dev, skb->data + lso->hdr_len, 773 len, DMA_TO_DEVICE); 774 if (dma_mapping_error(tx_ring->dev, dma)) 775 return -ENOMEM; 776 777 enetc_bdr_idx_inc(tx_ring, i); 778 txbd = ENETC_TXBD(*tx_ring, *i); 779 tx_swbd = &tx_ring->tx_swbd[*i]; 780 prefetchw(txbd); 781 *count += 1; 782 783 enetc_clear_tx_bd(&txbd_tmp); 784 txbd_tmp.addr = cpu_to_le64(dma); 785 txbd_tmp.buf_len = cpu_to_le16(len); 786 787 tx_swbd->dma = dma; 788 tx_swbd->len = len; 789 tx_swbd->is_dma_page = 0; 790 tx_swbd->dir = DMA_TO_DEVICE; 791 } 792 793 frag = &skb_shinfo(skb)->frags[0]; 794 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++, frag++) { 795 if (txbd) 796 *txbd = txbd_tmp; 797 798 len = skb_frag_size(frag); 799 dma = skb_frag_dma_map(tx_ring->dev, frag); 800 if (dma_mapping_error(tx_ring->dev, dma)) 801 return -ENOMEM; 802 803 /* Get the next BD */ 804 enetc_bdr_idx_inc(tx_ring, i); 805 txbd = ENETC_TXBD(*tx_ring, *i); 806 tx_swbd = &tx_ring->tx_swbd[*i]; 807 prefetchw(txbd); 808 *count += 1; 809 810 enetc_clear_tx_bd(&txbd_tmp); 811 txbd_tmp.addr = cpu_to_le64(dma); 812 txbd_tmp.buf_len = cpu_to_le16(len); 813 814 tx_swbd->dma = dma; 815 tx_swbd->len = len; 816 tx_swbd->is_dma_page = 1; 817 tx_swbd->dir = DMA_TO_DEVICE; 818 } 819 820 /* Last BD needs 'F' bit set */ 821 flags |= ENETC_TXBD_FLAGS_F; 822 txbd_tmp.flags = flags; 823 *txbd = txbd_tmp; 824 825 tx_swbd->is_eof = 1; 826 tx_swbd->skb = skb; 827 828 return 0; 829 } 830 831 static int enetc_lso_hw_offload(struct enetc_bdr *tx_ring, struct sk_buff *skb) 832 { 833 struct enetc_tx_swbd *tx_swbd; 834 struct enetc_lso_t lso = {0}; 835 int err, i, count = 0; 836 837 /* Initialize the LSO handler */ 838 enetc_lso_start(skb, &lso); 839 i = tx_ring->next_to_use; 840 841 enetc_lso_map_hdr(tx_ring, skb, &i, &lso); 842 /* First BD and an extend BD */ 843 count += 2; 844 845 err = enetc_lso_map_data(tx_ring, skb, &i, &lso, &count); 846 if (err) 847 goto dma_err; 848 849 /* Go to the next BD */ 850 enetc_bdr_idx_inc(tx_ring, &i); 851 tx_ring->next_to_use = i; 852 enetc_update_tx_ring_tail(tx_ring); 853 854 return count; 855 856 dma_err: 857 do { 858 tx_swbd = &tx_ring->tx_swbd[i]; 859 enetc_free_tx_frame(tx_ring, tx_swbd); 860 if (i == 0) 861 i = tx_ring->bd_count; 862 i--; 863 } while (--count); 864 865 return 0; 866 } 867 868 static int enetc_map_tx_tso_buffs(struct enetc_bdr *tx_ring, struct sk_buff *skb) 869 { 870 struct enetc_ndev_priv *priv = netdev_priv(tx_ring->ndev); 871 int hdr_len, total_len, data_len; 872 struct enetc_tx_swbd *tx_swbd; 873 union enetc_tx_bd *txbd; 874 struct tso_t tso; 875 __wsum csum, csum2; 876 int count = 0, pos; 877 int err, i, bd_data_num; 878 879 /* Initialize the TSO handler, and prepare the first payload */ 880 hdr_len = tso_start(skb, &tso); 881 total_len = skb->len - hdr_len; 882 i = tx_ring->next_to_use; 883 884 while (total_len > 0) { 885 char *hdr; 886 887 /* Get the BD */ 888 txbd = ENETC_TXBD(*tx_ring, i); 889 tx_swbd = &tx_ring->tx_swbd[i]; 890 prefetchw(txbd); 891 892 /* Determine the length of this packet */ 893 data_len = min_t(int, skb_shinfo(skb)->gso_size, total_len); 894 total_len -= data_len; 895 896 /* prepare packet headers: MAC + IP + TCP */ 897 hdr = tx_ring->tso_headers + i * TSO_HEADER_SIZE; 898 tso_build_hdr(skb, hdr, &tso, data_len, total_len == 0); 899 900 /* compute the csum over the L4 header */ 901 csum = enetc_tso_hdr_csum(&tso, skb, hdr, hdr_len, &pos); 902 count += enetc_map_tx_tso_hdr(tx_ring, skb, tx_swbd, txbd, 903 &i, hdr_len, data_len); 904 bd_data_num = 0; 905 906 while (data_len > 0) { 907 int size; 908 909 size = min_t(int, tso.size, data_len); 910 911 /* Advance the index in the BDR */ 912 enetc_bdr_idx_inc(tx_ring, &i); 913 txbd = ENETC_TXBD(*tx_ring, i); 914 tx_swbd = &tx_ring->tx_swbd[i]; 915 prefetchw(txbd); 916 917 /* Compute the checksum over this segment of data and 918 * add it to the csum already computed (over the L4 919 * header and possible other data segments). 920 */ 921 csum2 = csum_partial(tso.data, size, 0); 922 csum = csum_block_add(csum, csum2, pos); 923 pos += size; 924 925 err = enetc_map_tx_tso_data(tx_ring, skb, tx_swbd, txbd, 926 tso.data, size, 927 size == data_len); 928 if (err) { 929 if (i == 0) 930 i = tx_ring->bd_count; 931 i--; 932 933 goto err_map_data; 934 } 935 936 data_len -= size; 937 count++; 938 bd_data_num++; 939 tso_build_data(skb, &tso, size); 940 941 if (unlikely(bd_data_num >= priv->max_frags && data_len)) 942 goto err_chained_bd; 943 } 944 945 enetc_tso_complete_csum(tx_ring, &tso, skb, hdr, pos, csum); 946 947 if (total_len == 0) 948 tx_swbd->skb = skb; 949 950 /* Go to the next BD */ 951 enetc_bdr_idx_inc(tx_ring, &i); 952 } 953 954 tx_ring->next_to_use = i; 955 enetc_update_tx_ring_tail(tx_ring); 956 957 return count; 958 959 err_map_data: 960 dev_err(tx_ring->dev, "DMA map error"); 961 962 err_chained_bd: 963 enetc_unwind_tx_frame(tx_ring, count, i); 964 965 return 0; 966 } 967 968 static netdev_tx_t enetc_start_xmit(struct sk_buff *skb, 969 struct net_device *ndev) 970 { 971 struct enetc_skb_cb *enetc_cb = ENETC_SKB_CB(skb); 972 struct enetc_ndev_priv *priv = netdev_priv(ndev); 973 struct enetc_bdr *tx_ring; 974 int count; 975 976 /* Queue one-step Sync packet if already locked */ 977 if (enetc_cb->flag & ENETC_F_TX_ONESTEP_SYNC_TSTAMP) { 978 if (test_and_set_bit_lock(ENETC_TX_ONESTEP_TSTAMP_IN_PROGRESS, 979 &priv->flags)) { 980 skb_queue_tail(&priv->tx_skbs, skb); 981 return NETDEV_TX_OK; 982 } 983 } 984 985 tx_ring = priv->tx_ring[skb->queue_mapping]; 986 987 if (skb_is_gso(skb)) { 988 /* LSO data unit lengths of up to 256KB are supported */ 989 if (priv->active_offloads & ENETC_F_LSO && 990 (skb->len - enetc_lso_get_hdr_len(skb)) <= 991 ENETC_LSO_MAX_DATA_LEN) { 992 if (enetc_bd_unused(tx_ring) < enetc_lso_count_descs(skb)) { 993 netif_stop_subqueue(ndev, tx_ring->index); 994 return NETDEV_TX_BUSY; 995 } 996 997 count = enetc_lso_hw_offload(tx_ring, skb); 998 } else { 999 if (enetc_bd_unused(tx_ring) < tso_count_descs(skb)) { 1000 netif_stop_subqueue(ndev, tx_ring->index); 1001 return NETDEV_TX_BUSY; 1002 } 1003 1004 enetc_lock_mdio(); 1005 count = enetc_map_tx_tso_buffs(tx_ring, skb); 1006 enetc_unlock_mdio(); 1007 } 1008 } else { 1009 if (unlikely(skb_shinfo(skb)->nr_frags > priv->max_frags)) 1010 if (unlikely(skb_linearize(skb))) 1011 goto drop_packet_err; 1012 1013 count = skb_shinfo(skb)->nr_frags + 1; /* fragments + head */ 1014 if (enetc_bd_unused(tx_ring) < ENETC_TXBDS_NEEDED(count)) { 1015 netif_stop_subqueue(ndev, tx_ring->index); 1016 return NETDEV_TX_BUSY; 1017 } 1018 1019 enetc_lock_mdio(); 1020 count = enetc_map_tx_buffs(tx_ring, skb); 1021 enetc_unlock_mdio(); 1022 } 1023 1024 if (unlikely(!count)) 1025 goto drop_packet_err; 1026 1027 if (enetc_bd_unused(tx_ring) < ENETC_TXBDS_MAX_NEEDED(priv->max_frags)) 1028 netif_stop_subqueue(ndev, tx_ring->index); 1029 1030 return NETDEV_TX_OK; 1031 1032 drop_packet_err: 1033 dev_kfree_skb_any(skb); 1034 return NETDEV_TX_OK; 1035 } 1036 1037 netdev_tx_t enetc_xmit(struct sk_buff *skb, struct net_device *ndev) 1038 { 1039 struct enetc_skb_cb *enetc_cb = ENETC_SKB_CB(skb); 1040 struct enetc_ndev_priv *priv = netdev_priv(ndev); 1041 u8 udp, msgtype, twostep; 1042 u16 offset1, offset2; 1043 1044 /* Mark tx timestamp type on enetc_cb->flag if requires */ 1045 if ((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && 1046 (priv->active_offloads & ENETC_F_TX_TSTAMP_MASK)) 1047 enetc_cb->flag = priv->active_offloads & ENETC_F_TX_TSTAMP_MASK; 1048 else 1049 enetc_cb->flag = 0; 1050 1051 /* Fall back to two-step timestamp if not one-step Sync packet */ 1052 if (enetc_cb->flag & ENETC_F_TX_ONESTEP_SYNC_TSTAMP) { 1053 if (enetc_ptp_parse(skb, &udp, &msgtype, &twostep, 1054 &offset1, &offset2) || 1055 msgtype != PTP_MSGTYPE_SYNC || twostep != 0) { 1056 enetc_cb->flag = ENETC_F_TX_TSTAMP; 1057 } else { 1058 enetc_cb->udp = !!udp; 1059 enetc_cb->correction_off = offset1; 1060 enetc_cb->origin_tstamp_off = offset2; 1061 } 1062 } 1063 1064 return enetc_start_xmit(skb, ndev); 1065 } 1066 EXPORT_SYMBOL_GPL(enetc_xmit); 1067 1068 static irqreturn_t enetc_msix(int irq, void *data) 1069 { 1070 struct enetc_int_vector *v = data; 1071 int i; 1072 1073 enetc_lock_mdio(); 1074 1075 /* disable interrupts */ 1076 enetc_wr_reg_hot(v->rbier, 0); 1077 enetc_wr_reg_hot(v->ricr1, v->rx_ictt); 1078 1079 for_each_set_bit(i, &v->tx_rings_map, ENETC_MAX_NUM_TXQS) 1080 enetc_wr_reg_hot(v->tbier_base + ENETC_BDR_OFF(i), 0); 1081 1082 enetc_unlock_mdio(); 1083 1084 napi_schedule(&v->napi); 1085 1086 return IRQ_HANDLED; 1087 } 1088 1089 static void enetc_rx_dim_work(struct work_struct *w) 1090 { 1091 struct dim *dim = container_of(w, struct dim, work); 1092 struct dim_cq_moder moder = 1093 net_dim_get_rx_moderation(dim->mode, dim->profile_ix); 1094 struct enetc_int_vector *v = 1095 container_of(dim, struct enetc_int_vector, rx_dim); 1096 struct enetc_ndev_priv *priv = netdev_priv(v->rx_ring.ndev); 1097 1098 v->rx_ictt = enetc_usecs_to_cycles(moder.usec, priv->sysclk_freq); 1099 dim->state = DIM_START_MEASURE; 1100 } 1101 1102 static void enetc_rx_net_dim(struct enetc_int_vector *v) 1103 { 1104 struct dim_sample dim_sample = {}; 1105 1106 v->comp_cnt++; 1107 1108 if (!v->rx_napi_work) 1109 return; 1110 1111 dim_update_sample(v->comp_cnt, 1112 v->rx_ring.stats.packets, 1113 v->rx_ring.stats.bytes, 1114 &dim_sample); 1115 net_dim(&v->rx_dim, &dim_sample); 1116 } 1117 1118 static int enetc_bd_ready_count(struct enetc_bdr *tx_ring, int ci) 1119 { 1120 int pi = enetc_rd_reg_hot(tx_ring->tcir) & ENETC_TBCIR_IDX_MASK; 1121 1122 return pi >= ci ? pi - ci : tx_ring->bd_count - ci + pi; 1123 } 1124 1125 static bool enetc_page_reusable(struct page *page) 1126 { 1127 return (!page_is_pfmemalloc(page) && page_ref_count(page) == 1); 1128 } 1129 1130 static void enetc_reuse_page(struct enetc_bdr *rx_ring, 1131 struct enetc_rx_swbd *old) 1132 { 1133 struct enetc_rx_swbd *new; 1134 1135 new = &rx_ring->rx_swbd[rx_ring->next_to_alloc]; 1136 1137 /* next buf that may reuse a page */ 1138 enetc_bdr_idx_inc(rx_ring, &rx_ring->next_to_alloc); 1139 1140 /* copy page reference */ 1141 *new = *old; 1142 } 1143 1144 static void enetc_get_tx_tstamp(struct enetc_hw *hw, union enetc_tx_bd *txbd, 1145 u64 *tstamp) 1146 { 1147 u32 lo, hi, tstamp_lo; 1148 1149 lo = enetc_rd_hot(hw, ENETC_SICTR0); 1150 hi = enetc_rd_hot(hw, ENETC_SICTR1); 1151 tstamp_lo = le32_to_cpu(txbd->wb.tstamp); 1152 if (lo <= tstamp_lo) 1153 hi -= 1; 1154 *tstamp = (u64)hi << 32 | tstamp_lo; 1155 } 1156 1157 static void enetc_tstamp_tx(struct sk_buff *skb, u64 tstamp) 1158 { 1159 struct skb_shared_hwtstamps shhwtstamps; 1160 1161 if (skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS) { 1162 memset(&shhwtstamps, 0, sizeof(shhwtstamps)); 1163 shhwtstamps.hwtstamp = ns_to_ktime(tstamp); 1164 skb_txtime_consumed(skb); 1165 skb_tstamp_tx(skb, &shhwtstamps); 1166 } 1167 } 1168 1169 static void enetc_recycle_xdp_tx_buff(struct enetc_bdr *tx_ring, 1170 struct enetc_tx_swbd *tx_swbd) 1171 { 1172 struct enetc_ndev_priv *priv = netdev_priv(tx_ring->ndev); 1173 struct enetc_rx_swbd rx_swbd = { 1174 .dma = tx_swbd->dma, 1175 .page = tx_swbd->page, 1176 .page_offset = tx_swbd->page_offset, 1177 .dir = tx_swbd->dir, 1178 .len = tx_swbd->len, 1179 }; 1180 struct enetc_bdr *rx_ring; 1181 1182 rx_ring = enetc_rx_ring_from_xdp_tx_ring(priv, tx_ring); 1183 1184 if (likely(enetc_swbd_unused(rx_ring))) { 1185 enetc_reuse_page(rx_ring, &rx_swbd); 1186 1187 /* sync for use by the device */ 1188 dma_sync_single_range_for_device(rx_ring->dev, rx_swbd.dma, 1189 rx_swbd.page_offset, 1190 ENETC_RXB_DMA_SIZE_XDP, 1191 rx_swbd.dir); 1192 1193 rx_ring->stats.recycles++; 1194 } else { 1195 /* RX ring is already full, we need to unmap and free the 1196 * page, since there's nothing useful we can do with it. 1197 */ 1198 rx_ring->stats.recycle_failures++; 1199 1200 dma_unmap_page(rx_ring->dev, rx_swbd.dma, PAGE_SIZE, 1201 rx_swbd.dir); 1202 __free_page(rx_swbd.page); 1203 } 1204 1205 rx_ring->xdp.xdp_tx_in_flight--; 1206 } 1207 1208 static bool enetc_clean_tx_ring(struct enetc_bdr *tx_ring, int napi_budget) 1209 { 1210 int tx_frm_cnt = 0, tx_byte_cnt = 0, tx_win_drop = 0; 1211 struct net_device *ndev = tx_ring->ndev; 1212 struct enetc_ndev_priv *priv = netdev_priv(ndev); 1213 struct enetc_tx_swbd *tx_swbd; 1214 int i, bds_to_clean; 1215 bool do_twostep_tstamp; 1216 u64 tstamp = 0; 1217 1218 i = tx_ring->next_to_clean; 1219 tx_swbd = &tx_ring->tx_swbd[i]; 1220 1221 bds_to_clean = enetc_bd_ready_count(tx_ring, i); 1222 1223 do_twostep_tstamp = false; 1224 1225 while (bds_to_clean && tx_frm_cnt < ENETC_DEFAULT_TX_WORK) { 1226 struct xdp_frame *xdp_frame = enetc_tx_swbd_get_xdp_frame(tx_swbd); 1227 struct sk_buff *skb = enetc_tx_swbd_get_skb(tx_swbd); 1228 bool is_eof = tx_swbd->is_eof; 1229 1230 if (unlikely(tx_swbd->check_wb)) { 1231 union enetc_tx_bd *txbd = ENETC_TXBD(*tx_ring, i); 1232 1233 if (txbd->flags & ENETC_TXBD_FLAGS_W && 1234 tx_swbd->do_twostep_tstamp) { 1235 enetc_get_tx_tstamp(&priv->si->hw, txbd, 1236 &tstamp); 1237 do_twostep_tstamp = true; 1238 } 1239 1240 if (tx_swbd->qbv_en && 1241 txbd->wb.status & ENETC_TXBD_STATS_WIN) 1242 tx_win_drop++; 1243 } 1244 1245 if (tx_swbd->is_xdp_tx) 1246 enetc_recycle_xdp_tx_buff(tx_ring, tx_swbd); 1247 else if (likely(tx_swbd->dma)) 1248 enetc_unmap_tx_buff(tx_ring, tx_swbd); 1249 1250 if (xdp_frame) { 1251 xdp_return_frame(xdp_frame); 1252 } else if (skb) { 1253 struct enetc_skb_cb *enetc_cb = ENETC_SKB_CB(skb); 1254 1255 if (unlikely(enetc_cb->flag & ENETC_F_TX_ONESTEP_SYNC_TSTAMP)) { 1256 /* Start work to release lock for next one-step 1257 * timestamping packet. And send one skb in 1258 * tx_skbs queue if has. 1259 */ 1260 schedule_work(&priv->tx_onestep_tstamp); 1261 } else if (unlikely(do_twostep_tstamp)) { 1262 enetc_tstamp_tx(skb, tstamp); 1263 do_twostep_tstamp = false; 1264 } 1265 napi_consume_skb(skb, napi_budget); 1266 } 1267 1268 tx_byte_cnt += tx_swbd->len; 1269 /* Scrub the swbd here so we don't have to do that 1270 * when we reuse it during xmit 1271 */ 1272 memset(tx_swbd, 0, sizeof(*tx_swbd)); 1273 1274 bds_to_clean--; 1275 tx_swbd++; 1276 i++; 1277 if (unlikely(i == tx_ring->bd_count)) { 1278 i = 0; 1279 tx_swbd = tx_ring->tx_swbd; 1280 } 1281 1282 /* BD iteration loop end */ 1283 if (is_eof) { 1284 tx_frm_cnt++; 1285 /* re-arm interrupt source */ 1286 enetc_wr_reg_hot(tx_ring->idr, BIT(tx_ring->index) | 1287 BIT(16 + tx_ring->index)); 1288 } 1289 1290 if (unlikely(!bds_to_clean)) 1291 bds_to_clean = enetc_bd_ready_count(tx_ring, i); 1292 } 1293 1294 tx_ring->next_to_clean = i; 1295 tx_ring->stats.packets += tx_frm_cnt; 1296 tx_ring->stats.bytes += tx_byte_cnt; 1297 tx_ring->stats.win_drop += tx_win_drop; 1298 1299 if (unlikely(tx_frm_cnt && netif_carrier_ok(ndev) && 1300 __netif_subqueue_stopped(ndev, tx_ring->index) && 1301 !test_bit(ENETC_TX_DOWN, &priv->flags) && 1302 (enetc_bd_unused(tx_ring) >= 1303 ENETC_TXBDS_MAX_NEEDED(priv->max_frags)))) { 1304 netif_wake_subqueue(ndev, tx_ring->index); 1305 } 1306 1307 return tx_frm_cnt != ENETC_DEFAULT_TX_WORK; 1308 } 1309 1310 static bool enetc_new_page(struct enetc_bdr *rx_ring, 1311 struct enetc_rx_swbd *rx_swbd) 1312 { 1313 bool xdp = !!(rx_ring->xdp.prog); 1314 struct page *page; 1315 dma_addr_t addr; 1316 1317 page = dev_alloc_page(); 1318 if (unlikely(!page)) 1319 return false; 1320 1321 /* For XDP_TX, we forgo dma_unmap -> dma_map */ 1322 rx_swbd->dir = xdp ? DMA_BIDIRECTIONAL : DMA_FROM_DEVICE; 1323 1324 addr = dma_map_page(rx_ring->dev, page, 0, PAGE_SIZE, rx_swbd->dir); 1325 if (unlikely(dma_mapping_error(rx_ring->dev, addr))) { 1326 __free_page(page); 1327 1328 return false; 1329 } 1330 1331 rx_swbd->dma = addr; 1332 rx_swbd->page = page; 1333 rx_swbd->page_offset = rx_ring->buffer_offset; 1334 1335 return true; 1336 } 1337 1338 static int enetc_refill_rx_ring(struct enetc_bdr *rx_ring, const int buff_cnt) 1339 { 1340 struct enetc_rx_swbd *rx_swbd; 1341 union enetc_rx_bd *rxbd; 1342 int i, j; 1343 1344 i = rx_ring->next_to_use; 1345 rx_swbd = &rx_ring->rx_swbd[i]; 1346 rxbd = enetc_rxbd(rx_ring, i); 1347 1348 for (j = 0; j < buff_cnt; j++) { 1349 /* try reuse page */ 1350 if (unlikely(!rx_swbd->page)) { 1351 if (unlikely(!enetc_new_page(rx_ring, rx_swbd))) { 1352 rx_ring->stats.rx_alloc_errs++; 1353 break; 1354 } 1355 } 1356 1357 /* update RxBD */ 1358 rxbd->w.addr = cpu_to_le64(rx_swbd->dma + 1359 rx_swbd->page_offset); 1360 /* clear 'R" as well */ 1361 rxbd->r.lstatus = 0; 1362 1363 enetc_rxbd_next(rx_ring, &rxbd, &i); 1364 rx_swbd = &rx_ring->rx_swbd[i]; 1365 } 1366 1367 if (likely(j)) { 1368 rx_ring->next_to_alloc = i; /* keep track from page reuse */ 1369 rx_ring->next_to_use = i; 1370 1371 /* update ENETC's consumer index */ 1372 enetc_wr_reg_hot(rx_ring->rcir, rx_ring->next_to_use); 1373 } 1374 1375 return j; 1376 } 1377 1378 static void enetc_get_rx_tstamp(struct net_device *ndev, 1379 union enetc_rx_bd *rxbd, 1380 struct sk_buff *skb) 1381 { 1382 struct skb_shared_hwtstamps *shhwtstamps = skb_hwtstamps(skb); 1383 struct enetc_ndev_priv *priv = netdev_priv(ndev); 1384 struct enetc_hw *hw = &priv->si->hw; 1385 u32 lo, hi, tstamp_lo; 1386 u64 tstamp; 1387 1388 if (le16_to_cpu(rxbd->r.flags) & ENETC_RXBD_FLAG_TSTMP) { 1389 lo = enetc_rd_reg_hot(hw->reg + ENETC_SICTR0); 1390 hi = enetc_rd_reg_hot(hw->reg + ENETC_SICTR1); 1391 rxbd = enetc_rxbd_ext(rxbd); 1392 tstamp_lo = le32_to_cpu(rxbd->ext.tstamp); 1393 if (lo <= tstamp_lo) 1394 hi -= 1; 1395 1396 tstamp = (u64)hi << 32 | tstamp_lo; 1397 memset(shhwtstamps, 0, sizeof(*shhwtstamps)); 1398 shhwtstamps->hwtstamp = ns_to_ktime(tstamp); 1399 } 1400 } 1401 1402 static void enetc_get_offloads(struct enetc_bdr *rx_ring, 1403 union enetc_rx_bd *rxbd, struct sk_buff *skb) 1404 { 1405 struct enetc_ndev_priv *priv = netdev_priv(rx_ring->ndev); 1406 1407 /* TODO: hashing */ 1408 if (rx_ring->ndev->features & NETIF_F_RXCSUM) { 1409 u16 inet_csum = le16_to_cpu(rxbd->r.inet_csum); 1410 1411 skb->csum = csum_unfold((__force __sum16)~htons(inet_csum)); 1412 skb->ip_summed = CHECKSUM_COMPLETE; 1413 } 1414 1415 if (le16_to_cpu(rxbd->r.flags) & ENETC_RXBD_FLAG_VLAN) { 1416 struct enetc_hw *hw = &priv->si->hw; 1417 __be16 tpid = 0; 1418 1419 switch (le16_to_cpu(rxbd->r.flags) & ENETC_RXBD_FLAG_TPID) { 1420 case 0: 1421 tpid = htons(ETH_P_8021Q); 1422 break; 1423 case 1: 1424 tpid = htons(ETH_P_8021AD); 1425 break; 1426 case 2: 1427 tpid = htons(enetc_rd_hot(hw, ENETC_SICVLANR1) & 1428 SICVLANR_ETYPE); 1429 break; 1430 case 3: 1431 tpid = htons(enetc_rd_hot(hw, ENETC_SICVLANR2) & 1432 SICVLANR_ETYPE); 1433 } 1434 1435 __vlan_hwaccel_put_tag(skb, tpid, le16_to_cpu(rxbd->r.vlan_opt)); 1436 } 1437 1438 if (priv->active_offloads & ENETC_F_RX_TSTAMP) 1439 enetc_get_rx_tstamp(rx_ring->ndev, rxbd, skb); 1440 } 1441 1442 /* This gets called during the non-XDP NAPI poll cycle as well as on XDP_PASS, 1443 * so it needs to work with both DMA_FROM_DEVICE as well as DMA_BIDIRECTIONAL 1444 * mapped buffers. 1445 */ 1446 static struct enetc_rx_swbd *enetc_get_rx_buff(struct enetc_bdr *rx_ring, 1447 int i, u16 size) 1448 { 1449 struct enetc_rx_swbd *rx_swbd = &rx_ring->rx_swbd[i]; 1450 1451 dma_sync_single_range_for_cpu(rx_ring->dev, rx_swbd->dma, 1452 rx_swbd->page_offset, 1453 size, rx_swbd->dir); 1454 return rx_swbd; 1455 } 1456 1457 /* Reuse the current page without performing half-page buffer flipping */ 1458 static void enetc_put_rx_buff(struct enetc_bdr *rx_ring, 1459 struct enetc_rx_swbd *rx_swbd) 1460 { 1461 size_t buffer_size = ENETC_RXB_TRUESIZE - rx_ring->buffer_offset; 1462 1463 enetc_reuse_page(rx_ring, rx_swbd); 1464 1465 dma_sync_single_range_for_device(rx_ring->dev, rx_swbd->dma, 1466 rx_swbd->page_offset, 1467 buffer_size, rx_swbd->dir); 1468 1469 rx_swbd->page = NULL; 1470 } 1471 1472 /* Reuse the current page by performing half-page buffer flipping */ 1473 static void enetc_flip_rx_buff(struct enetc_bdr *rx_ring, 1474 struct enetc_rx_swbd *rx_swbd) 1475 { 1476 if (likely(enetc_page_reusable(rx_swbd->page))) { 1477 rx_swbd->page_offset ^= ENETC_RXB_TRUESIZE; 1478 page_ref_inc(rx_swbd->page); 1479 1480 enetc_put_rx_buff(rx_ring, rx_swbd); 1481 } else { 1482 dma_unmap_page(rx_ring->dev, rx_swbd->dma, PAGE_SIZE, 1483 rx_swbd->dir); 1484 rx_swbd->page = NULL; 1485 } 1486 } 1487 1488 static struct sk_buff *enetc_map_rx_buff_to_skb(struct enetc_bdr *rx_ring, 1489 int i, u16 size) 1490 { 1491 struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size); 1492 struct sk_buff *skb; 1493 void *ba; 1494 1495 ba = page_address(rx_swbd->page) + rx_swbd->page_offset; 1496 skb = build_skb(ba - rx_ring->buffer_offset, ENETC_RXB_TRUESIZE); 1497 if (unlikely(!skb)) { 1498 rx_ring->stats.rx_alloc_errs++; 1499 return NULL; 1500 } 1501 1502 skb_reserve(skb, rx_ring->buffer_offset); 1503 __skb_put(skb, size); 1504 1505 enetc_flip_rx_buff(rx_ring, rx_swbd); 1506 1507 return skb; 1508 } 1509 1510 static void enetc_add_rx_buff_to_skb(struct enetc_bdr *rx_ring, int i, 1511 u16 size, struct sk_buff *skb) 1512 { 1513 struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size); 1514 1515 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_swbd->page, 1516 rx_swbd->page_offset, size, ENETC_RXB_TRUESIZE); 1517 1518 enetc_flip_rx_buff(rx_ring, rx_swbd); 1519 } 1520 1521 static bool enetc_check_bd_errors_and_consume(struct enetc_bdr *rx_ring, 1522 u32 bd_status, 1523 union enetc_rx_bd **rxbd, int *i) 1524 { 1525 if (likely(!(bd_status & ENETC_RXBD_LSTATUS(ENETC_RXBD_ERR_MASK)))) 1526 return false; 1527 1528 enetc_put_rx_buff(rx_ring, &rx_ring->rx_swbd[*i]); 1529 enetc_rxbd_next(rx_ring, rxbd, i); 1530 1531 while (!(bd_status & ENETC_RXBD_LSTATUS_F)) { 1532 dma_rmb(); 1533 bd_status = le32_to_cpu((*rxbd)->r.lstatus); 1534 1535 enetc_put_rx_buff(rx_ring, &rx_ring->rx_swbd[*i]); 1536 enetc_rxbd_next(rx_ring, rxbd, i); 1537 } 1538 1539 rx_ring->ndev->stats.rx_dropped++; 1540 rx_ring->ndev->stats.rx_errors++; 1541 1542 return true; 1543 } 1544 1545 static struct sk_buff *enetc_build_skb(struct enetc_bdr *rx_ring, 1546 u32 bd_status, union enetc_rx_bd **rxbd, 1547 int *i, int *cleaned_cnt, int buffer_size) 1548 { 1549 struct sk_buff *skb; 1550 u16 size; 1551 1552 size = le16_to_cpu((*rxbd)->r.buf_len); 1553 skb = enetc_map_rx_buff_to_skb(rx_ring, *i, size); 1554 if (!skb) 1555 return NULL; 1556 1557 enetc_get_offloads(rx_ring, *rxbd, skb); 1558 1559 (*cleaned_cnt)++; 1560 1561 enetc_rxbd_next(rx_ring, rxbd, i); 1562 1563 /* not last BD in frame? */ 1564 while (!(bd_status & ENETC_RXBD_LSTATUS_F)) { 1565 bd_status = le32_to_cpu((*rxbd)->r.lstatus); 1566 size = buffer_size; 1567 1568 if (bd_status & ENETC_RXBD_LSTATUS_F) { 1569 dma_rmb(); 1570 size = le16_to_cpu((*rxbd)->r.buf_len); 1571 } 1572 1573 enetc_add_rx_buff_to_skb(rx_ring, *i, size, skb); 1574 1575 (*cleaned_cnt)++; 1576 1577 enetc_rxbd_next(rx_ring, rxbd, i); 1578 } 1579 1580 skb_record_rx_queue(skb, rx_ring->index); 1581 skb->protocol = eth_type_trans(skb, rx_ring->ndev); 1582 1583 return skb; 1584 } 1585 1586 #define ENETC_RXBD_BUNDLE 16 /* # of BDs to update at once */ 1587 1588 static int enetc_clean_rx_ring(struct enetc_bdr *rx_ring, 1589 struct napi_struct *napi, int work_limit) 1590 { 1591 int rx_frm_cnt = 0, rx_byte_cnt = 0; 1592 int cleaned_cnt, i; 1593 1594 cleaned_cnt = enetc_bd_unused(rx_ring); 1595 /* next descriptor to process */ 1596 i = rx_ring->next_to_clean; 1597 1598 enetc_lock_mdio(); 1599 1600 while (likely(rx_frm_cnt < work_limit)) { 1601 union enetc_rx_bd *rxbd; 1602 struct sk_buff *skb; 1603 u32 bd_status; 1604 1605 if (cleaned_cnt >= ENETC_RXBD_BUNDLE) 1606 cleaned_cnt -= enetc_refill_rx_ring(rx_ring, 1607 cleaned_cnt); 1608 1609 rxbd = enetc_rxbd(rx_ring, i); 1610 bd_status = le32_to_cpu(rxbd->r.lstatus); 1611 if (!bd_status) 1612 break; 1613 1614 enetc_wr_reg_hot(rx_ring->idr, BIT(rx_ring->index)); 1615 dma_rmb(); /* for reading other rxbd fields */ 1616 1617 if (enetc_check_bd_errors_and_consume(rx_ring, bd_status, 1618 &rxbd, &i)) 1619 break; 1620 1621 skb = enetc_build_skb(rx_ring, bd_status, &rxbd, &i, 1622 &cleaned_cnt, ENETC_RXB_DMA_SIZE); 1623 if (!skb) 1624 break; 1625 1626 /* When set, the outer VLAN header is extracted and reported 1627 * in the receive buffer descriptor. So rx_byte_cnt should 1628 * add the length of the extracted VLAN header. 1629 */ 1630 if (bd_status & ENETC_RXBD_FLAG_VLAN) 1631 rx_byte_cnt += VLAN_HLEN; 1632 rx_byte_cnt += skb->len + ETH_HLEN; 1633 rx_frm_cnt++; 1634 1635 enetc_unlock_mdio(); 1636 napi_gro_receive(napi, skb); 1637 enetc_lock_mdio(); 1638 } 1639 1640 rx_ring->next_to_clean = i; 1641 1642 rx_ring->stats.packets += rx_frm_cnt; 1643 rx_ring->stats.bytes += rx_byte_cnt; 1644 1645 enetc_unlock_mdio(); 1646 1647 return rx_frm_cnt; 1648 } 1649 1650 static void enetc_xdp_map_tx_buff(struct enetc_bdr *tx_ring, int i, 1651 struct enetc_tx_swbd *tx_swbd, 1652 int frm_len) 1653 { 1654 union enetc_tx_bd *txbd = ENETC_TXBD(*tx_ring, i); 1655 1656 prefetchw(txbd); 1657 1658 enetc_clear_tx_bd(txbd); 1659 txbd->addr = cpu_to_le64(tx_swbd->dma + tx_swbd->page_offset); 1660 txbd->buf_len = cpu_to_le16(tx_swbd->len); 1661 txbd->frm_len = cpu_to_le16(frm_len); 1662 1663 memcpy(&tx_ring->tx_swbd[i], tx_swbd, sizeof(*tx_swbd)); 1664 } 1665 1666 /* Puts in the TX ring one XDP frame, mapped as an array of TX software buffer 1667 * descriptors. 1668 */ 1669 static bool enetc_xdp_tx(struct enetc_bdr *tx_ring, 1670 struct enetc_tx_swbd *xdp_tx_arr, int num_tx_swbd) 1671 { 1672 struct enetc_tx_swbd *tmp_tx_swbd = xdp_tx_arr; 1673 int i, k, frm_len = tmp_tx_swbd->len; 1674 1675 if (unlikely(enetc_bd_unused(tx_ring) < ENETC_TXBDS_NEEDED(num_tx_swbd))) 1676 return false; 1677 1678 while (unlikely(!tmp_tx_swbd->is_eof)) { 1679 tmp_tx_swbd++; 1680 frm_len += tmp_tx_swbd->len; 1681 } 1682 1683 i = tx_ring->next_to_use; 1684 1685 for (k = 0; k < num_tx_swbd; k++) { 1686 struct enetc_tx_swbd *xdp_tx_swbd = &xdp_tx_arr[k]; 1687 1688 enetc_xdp_map_tx_buff(tx_ring, i, xdp_tx_swbd, frm_len); 1689 1690 /* last BD needs 'F' bit set */ 1691 if (xdp_tx_swbd->is_eof) { 1692 union enetc_tx_bd *txbd = ENETC_TXBD(*tx_ring, i); 1693 1694 txbd->flags = ENETC_TXBD_FLAGS_F; 1695 } 1696 1697 enetc_bdr_idx_inc(tx_ring, &i); 1698 } 1699 1700 tx_ring->next_to_use = i; 1701 1702 return true; 1703 } 1704 1705 static int enetc_xdp_frame_to_xdp_tx_swbd(struct enetc_bdr *tx_ring, 1706 struct enetc_tx_swbd *xdp_tx_arr, 1707 struct xdp_frame *xdp_frame) 1708 { 1709 struct enetc_tx_swbd *xdp_tx_swbd = &xdp_tx_arr[0]; 1710 struct skb_shared_info *shinfo; 1711 void *data = xdp_frame->data; 1712 int len = xdp_frame->len; 1713 skb_frag_t *frag; 1714 dma_addr_t dma; 1715 unsigned int f; 1716 int n = 0; 1717 1718 dma = dma_map_single(tx_ring->dev, data, len, DMA_TO_DEVICE); 1719 if (unlikely(dma_mapping_error(tx_ring->dev, dma))) { 1720 netdev_err(tx_ring->ndev, "DMA map error\n"); 1721 return -1; 1722 } 1723 1724 xdp_tx_swbd->dma = dma; 1725 xdp_tx_swbd->dir = DMA_TO_DEVICE; 1726 xdp_tx_swbd->len = len; 1727 xdp_tx_swbd->is_xdp_redirect = true; 1728 xdp_tx_swbd->is_eof = false; 1729 xdp_tx_swbd->xdp_frame = NULL; 1730 1731 n++; 1732 1733 if (!xdp_frame_has_frags(xdp_frame)) 1734 goto out; 1735 1736 xdp_tx_swbd = &xdp_tx_arr[n]; 1737 1738 shinfo = xdp_get_shared_info_from_frame(xdp_frame); 1739 1740 for (f = 0, frag = &shinfo->frags[0]; f < shinfo->nr_frags; 1741 f++, frag++) { 1742 data = skb_frag_address(frag); 1743 len = skb_frag_size(frag); 1744 1745 dma = dma_map_single(tx_ring->dev, data, len, DMA_TO_DEVICE); 1746 if (unlikely(dma_mapping_error(tx_ring->dev, dma))) { 1747 /* Undo the DMA mapping for all fragments */ 1748 while (--n >= 0) 1749 enetc_unmap_tx_buff(tx_ring, &xdp_tx_arr[n]); 1750 1751 netdev_err(tx_ring->ndev, "DMA map error\n"); 1752 return -1; 1753 } 1754 1755 xdp_tx_swbd->dma = dma; 1756 xdp_tx_swbd->dir = DMA_TO_DEVICE; 1757 xdp_tx_swbd->len = len; 1758 xdp_tx_swbd->is_xdp_redirect = true; 1759 xdp_tx_swbd->is_eof = false; 1760 xdp_tx_swbd->xdp_frame = NULL; 1761 1762 n++; 1763 xdp_tx_swbd = &xdp_tx_arr[n]; 1764 } 1765 out: 1766 xdp_tx_arr[n - 1].is_eof = true; 1767 xdp_tx_arr[n - 1].xdp_frame = xdp_frame; 1768 1769 return n; 1770 } 1771 1772 int enetc_xdp_xmit(struct net_device *ndev, int num_frames, 1773 struct xdp_frame **frames, u32 flags) 1774 { 1775 struct enetc_tx_swbd xdp_redirect_arr[ENETC_MAX_SKB_FRAGS] = {0}; 1776 struct enetc_ndev_priv *priv = netdev_priv(ndev); 1777 struct enetc_bdr *tx_ring; 1778 int xdp_tx_bd_cnt, i, k; 1779 int xdp_tx_frm_cnt = 0; 1780 1781 if (unlikely(test_bit(ENETC_TX_DOWN, &priv->flags))) 1782 return -ENETDOWN; 1783 1784 enetc_lock_mdio(); 1785 1786 tx_ring = priv->xdp_tx_ring[smp_processor_id()]; 1787 1788 prefetchw(ENETC_TXBD(*tx_ring, tx_ring->next_to_use)); 1789 1790 for (k = 0; k < num_frames; k++) { 1791 xdp_tx_bd_cnt = enetc_xdp_frame_to_xdp_tx_swbd(tx_ring, 1792 xdp_redirect_arr, 1793 frames[k]); 1794 if (unlikely(xdp_tx_bd_cnt < 0)) 1795 break; 1796 1797 if (unlikely(!enetc_xdp_tx(tx_ring, xdp_redirect_arr, 1798 xdp_tx_bd_cnt))) { 1799 for (i = 0; i < xdp_tx_bd_cnt; i++) 1800 enetc_unmap_tx_buff(tx_ring, 1801 &xdp_redirect_arr[i]); 1802 tx_ring->stats.xdp_tx_drops++; 1803 break; 1804 } 1805 1806 xdp_tx_frm_cnt++; 1807 } 1808 1809 if (unlikely((flags & XDP_XMIT_FLUSH) || k != xdp_tx_frm_cnt)) 1810 enetc_update_tx_ring_tail(tx_ring); 1811 1812 tx_ring->stats.xdp_tx += xdp_tx_frm_cnt; 1813 1814 enetc_unlock_mdio(); 1815 1816 return xdp_tx_frm_cnt; 1817 } 1818 EXPORT_SYMBOL_GPL(enetc_xdp_xmit); 1819 1820 static void enetc_map_rx_buff_to_xdp(struct enetc_bdr *rx_ring, int i, 1821 struct xdp_buff *xdp_buff, u16 size) 1822 { 1823 struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size); 1824 void *hard_start = page_address(rx_swbd->page) + rx_swbd->page_offset; 1825 1826 /* To be used for XDP_TX */ 1827 rx_swbd->len = size; 1828 1829 xdp_prepare_buff(xdp_buff, hard_start - rx_ring->buffer_offset, 1830 rx_ring->buffer_offset, size, false); 1831 } 1832 1833 static void enetc_add_rx_buff_to_xdp(struct enetc_bdr *rx_ring, int i, 1834 u16 size, struct xdp_buff *xdp_buff) 1835 { 1836 struct skb_shared_info *shinfo = xdp_get_shared_info_from_buff(xdp_buff); 1837 struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size); 1838 skb_frag_t *frag; 1839 1840 /* To be used for XDP_TX */ 1841 rx_swbd->len = size; 1842 1843 if (!xdp_buff_has_frags(xdp_buff)) { 1844 xdp_buff_set_frags_flag(xdp_buff); 1845 shinfo->xdp_frags_size = size; 1846 shinfo->nr_frags = 0; 1847 } else { 1848 shinfo->xdp_frags_size += size; 1849 } 1850 1851 if (page_is_pfmemalloc(rx_swbd->page)) 1852 xdp_buff_set_frag_pfmemalloc(xdp_buff); 1853 1854 frag = &shinfo->frags[shinfo->nr_frags]; 1855 skb_frag_fill_page_desc(frag, rx_swbd->page, rx_swbd->page_offset, 1856 size); 1857 1858 shinfo->nr_frags++; 1859 } 1860 1861 static void enetc_build_xdp_buff(struct enetc_bdr *rx_ring, u32 bd_status, 1862 union enetc_rx_bd **rxbd, int *i, 1863 int *cleaned_cnt, struct xdp_buff *xdp_buff) 1864 { 1865 u16 size = le16_to_cpu((*rxbd)->r.buf_len); 1866 1867 xdp_init_buff(xdp_buff, ENETC_RXB_TRUESIZE, &rx_ring->xdp.rxq); 1868 1869 enetc_map_rx_buff_to_xdp(rx_ring, *i, xdp_buff, size); 1870 (*cleaned_cnt)++; 1871 enetc_rxbd_next(rx_ring, rxbd, i); 1872 1873 /* not last BD in frame? */ 1874 while (!(bd_status & ENETC_RXBD_LSTATUS_F)) { 1875 bd_status = le32_to_cpu((*rxbd)->r.lstatus); 1876 size = ENETC_RXB_DMA_SIZE_XDP; 1877 1878 if (bd_status & ENETC_RXBD_LSTATUS_F) { 1879 dma_rmb(); 1880 size = le16_to_cpu((*rxbd)->r.buf_len); 1881 } 1882 1883 enetc_add_rx_buff_to_xdp(rx_ring, *i, size, xdp_buff); 1884 (*cleaned_cnt)++; 1885 enetc_rxbd_next(rx_ring, rxbd, i); 1886 } 1887 } 1888 1889 /* Convert RX buffer descriptors to TX buffer descriptors. These will be 1890 * recycled back into the RX ring in enetc_clean_tx_ring. 1891 */ 1892 static int enetc_rx_swbd_to_xdp_tx_swbd(struct enetc_tx_swbd *xdp_tx_arr, 1893 struct enetc_bdr *rx_ring, 1894 int rx_ring_first, int rx_ring_last) 1895 { 1896 int n = 0; 1897 1898 for (; rx_ring_first != rx_ring_last; 1899 n++, enetc_bdr_idx_inc(rx_ring, &rx_ring_first)) { 1900 struct enetc_rx_swbd *rx_swbd = &rx_ring->rx_swbd[rx_ring_first]; 1901 struct enetc_tx_swbd *tx_swbd = &xdp_tx_arr[n]; 1902 1903 /* No need to dma_map, we already have DMA_BIDIRECTIONAL */ 1904 tx_swbd->dma = rx_swbd->dma; 1905 tx_swbd->dir = rx_swbd->dir; 1906 tx_swbd->page = rx_swbd->page; 1907 tx_swbd->page_offset = rx_swbd->page_offset; 1908 tx_swbd->len = rx_swbd->len; 1909 tx_swbd->is_dma_page = true; 1910 tx_swbd->is_xdp_tx = true; 1911 tx_swbd->is_eof = false; 1912 } 1913 1914 /* We rely on caller providing an rx_ring_last > rx_ring_first */ 1915 xdp_tx_arr[n - 1].is_eof = true; 1916 1917 return n; 1918 } 1919 1920 static void enetc_xdp_drop(struct enetc_bdr *rx_ring, int rx_ring_first, 1921 int rx_ring_last) 1922 { 1923 while (rx_ring_first != rx_ring_last) { 1924 enetc_put_rx_buff(rx_ring, 1925 &rx_ring->rx_swbd[rx_ring_first]); 1926 enetc_bdr_idx_inc(rx_ring, &rx_ring_first); 1927 } 1928 } 1929 1930 static void enetc_bulk_flip_buff(struct enetc_bdr *rx_ring, int rx_ring_first, 1931 int rx_ring_last) 1932 { 1933 while (rx_ring_first != rx_ring_last) { 1934 enetc_flip_rx_buff(rx_ring, 1935 &rx_ring->rx_swbd[rx_ring_first]); 1936 enetc_bdr_idx_inc(rx_ring, &rx_ring_first); 1937 } 1938 } 1939 1940 static int enetc_clean_rx_ring_xdp(struct enetc_bdr *rx_ring, 1941 struct napi_struct *napi, int work_limit, 1942 struct bpf_prog *prog) 1943 { 1944 int xdp_tx_bd_cnt, xdp_tx_frm_cnt = 0, xdp_redirect_frm_cnt = 0; 1945 struct enetc_tx_swbd xdp_tx_arr[ENETC_MAX_SKB_FRAGS] = {0}; 1946 struct enetc_ndev_priv *priv = netdev_priv(rx_ring->ndev); 1947 int rx_frm_cnt = 0, rx_byte_cnt = 0; 1948 struct enetc_bdr *tx_ring; 1949 int cleaned_cnt, i; 1950 u32 xdp_act; 1951 1952 cleaned_cnt = enetc_bd_unused(rx_ring); 1953 /* next descriptor to process */ 1954 i = rx_ring->next_to_clean; 1955 1956 enetc_lock_mdio(); 1957 1958 while (likely(rx_frm_cnt < work_limit)) { 1959 union enetc_rx_bd *rxbd, *orig_rxbd; 1960 struct xdp_buff xdp_buff; 1961 struct sk_buff *skb; 1962 int orig_i, err; 1963 u32 bd_status; 1964 1965 rxbd = enetc_rxbd(rx_ring, i); 1966 bd_status = le32_to_cpu(rxbd->r.lstatus); 1967 if (!bd_status) 1968 break; 1969 1970 enetc_wr_reg_hot(rx_ring->idr, BIT(rx_ring->index)); 1971 dma_rmb(); /* for reading other rxbd fields */ 1972 1973 if (enetc_check_bd_errors_and_consume(rx_ring, bd_status, 1974 &rxbd, &i)) 1975 break; 1976 1977 orig_rxbd = rxbd; 1978 orig_i = i; 1979 1980 enetc_build_xdp_buff(rx_ring, bd_status, &rxbd, &i, 1981 &cleaned_cnt, &xdp_buff); 1982 1983 /* When set, the outer VLAN header is extracted and reported 1984 * in the receive buffer descriptor. So rx_byte_cnt should 1985 * add the length of the extracted VLAN header. 1986 */ 1987 if (bd_status & ENETC_RXBD_FLAG_VLAN) 1988 rx_byte_cnt += VLAN_HLEN; 1989 rx_byte_cnt += xdp_get_buff_len(&xdp_buff); 1990 1991 xdp_act = bpf_prog_run_xdp(prog, &xdp_buff); 1992 1993 switch (xdp_act) { 1994 default: 1995 bpf_warn_invalid_xdp_action(rx_ring->ndev, prog, xdp_act); 1996 fallthrough; 1997 case XDP_ABORTED: 1998 trace_xdp_exception(rx_ring->ndev, prog, xdp_act); 1999 fallthrough; 2000 case XDP_DROP: 2001 enetc_xdp_drop(rx_ring, orig_i, i); 2002 rx_ring->stats.xdp_drops++; 2003 break; 2004 case XDP_PASS: 2005 skb = xdp_build_skb_from_buff(&xdp_buff); 2006 /* Probably under memory pressure, stop NAPI */ 2007 if (unlikely(!skb)) { 2008 enetc_xdp_drop(rx_ring, orig_i, i); 2009 rx_ring->stats.xdp_drops++; 2010 goto out; 2011 } 2012 2013 enetc_get_offloads(rx_ring, orig_rxbd, skb); 2014 2015 /* These buffers are about to be owned by the stack. 2016 * Update our buffer cache (the rx_swbd array elements) 2017 * with their other page halves. 2018 */ 2019 enetc_bulk_flip_buff(rx_ring, orig_i, i); 2020 2021 enetc_unlock_mdio(); 2022 napi_gro_receive(napi, skb); 2023 enetc_lock_mdio(); 2024 break; 2025 case XDP_TX: 2026 tx_ring = priv->xdp_tx_ring[rx_ring->index]; 2027 if (unlikely(test_bit(ENETC_TX_DOWN, &priv->flags))) { 2028 enetc_xdp_drop(rx_ring, orig_i, i); 2029 tx_ring->stats.xdp_tx_drops++; 2030 break; 2031 } 2032 2033 xdp_tx_bd_cnt = enetc_rx_swbd_to_xdp_tx_swbd(xdp_tx_arr, 2034 rx_ring, 2035 orig_i, i); 2036 2037 if (!enetc_xdp_tx(tx_ring, xdp_tx_arr, xdp_tx_bd_cnt)) { 2038 enetc_xdp_drop(rx_ring, orig_i, i); 2039 tx_ring->stats.xdp_tx_drops++; 2040 } else { 2041 tx_ring->stats.xdp_tx++; 2042 rx_ring->xdp.xdp_tx_in_flight += xdp_tx_bd_cnt; 2043 xdp_tx_frm_cnt++; 2044 /* The XDP_TX enqueue was successful, so we 2045 * need to scrub the RX software BDs because 2046 * the ownership of the buffers no longer 2047 * belongs to the RX ring, and we must prevent 2048 * enetc_refill_rx_ring() from reusing 2049 * rx_swbd->page. 2050 */ 2051 while (orig_i != i) { 2052 rx_ring->rx_swbd[orig_i].page = NULL; 2053 enetc_bdr_idx_inc(rx_ring, &orig_i); 2054 } 2055 } 2056 break; 2057 case XDP_REDIRECT: 2058 enetc_unlock_mdio(); 2059 err = xdp_do_redirect(rx_ring->ndev, &xdp_buff, prog); 2060 enetc_lock_mdio(); 2061 if (unlikely(err)) { 2062 enetc_xdp_drop(rx_ring, orig_i, i); 2063 rx_ring->stats.xdp_redirect_failures++; 2064 } else { 2065 enetc_bulk_flip_buff(rx_ring, orig_i, i); 2066 xdp_redirect_frm_cnt++; 2067 rx_ring->stats.xdp_redirect++; 2068 } 2069 } 2070 2071 rx_frm_cnt++; 2072 } 2073 2074 out: 2075 rx_ring->next_to_clean = i; 2076 2077 rx_ring->stats.packets += rx_frm_cnt; 2078 rx_ring->stats.bytes += rx_byte_cnt; 2079 2080 if (xdp_redirect_frm_cnt) { 2081 enetc_unlock_mdio(); 2082 xdp_do_flush(); 2083 enetc_lock_mdio(); 2084 } 2085 2086 if (xdp_tx_frm_cnt) 2087 enetc_update_tx_ring_tail(tx_ring); 2088 2089 if (cleaned_cnt > rx_ring->xdp.xdp_tx_in_flight) 2090 enetc_refill_rx_ring(rx_ring, enetc_bd_unused(rx_ring) - 2091 rx_ring->xdp.xdp_tx_in_flight); 2092 2093 enetc_unlock_mdio(); 2094 2095 return rx_frm_cnt; 2096 } 2097 2098 static int enetc_poll(struct napi_struct *napi, int budget) 2099 { 2100 struct enetc_int_vector 2101 *v = container_of(napi, struct enetc_int_vector, napi); 2102 struct enetc_bdr *rx_ring = &v->rx_ring; 2103 struct bpf_prog *prog; 2104 bool complete = true; 2105 int work_done; 2106 int i; 2107 2108 enetc_lock_mdio(); 2109 2110 for (i = 0; i < v->count_tx_rings; i++) 2111 if (!enetc_clean_tx_ring(&v->tx_ring[i], budget)) 2112 complete = false; 2113 enetc_unlock_mdio(); 2114 2115 prog = rx_ring->xdp.prog; 2116 if (prog) 2117 work_done = enetc_clean_rx_ring_xdp(rx_ring, napi, budget, prog); 2118 else 2119 work_done = enetc_clean_rx_ring(rx_ring, napi, budget); 2120 if (work_done == budget) 2121 complete = false; 2122 if (work_done) 2123 v->rx_napi_work = true; 2124 2125 if (!complete) 2126 return budget; 2127 2128 napi_complete_done(napi, work_done); 2129 2130 if (likely(v->rx_dim_en)) 2131 enetc_rx_net_dim(v); 2132 2133 v->rx_napi_work = false; 2134 2135 enetc_lock_mdio(); 2136 /* enable interrupts */ 2137 enetc_wr_reg_hot(v->rbier, ENETC_RBIER_RXTIE); 2138 2139 for_each_set_bit(i, &v->tx_rings_map, ENETC_MAX_NUM_TXQS) 2140 enetc_wr_reg_hot(v->tbier_base + ENETC_BDR_OFF(i), 2141 ENETC_TBIER_TXTIE); 2142 2143 enetc_unlock_mdio(); 2144 2145 return work_done; 2146 } 2147 2148 /* Probing and Init */ 2149 #define ENETC_MAX_RFS_SIZE 64 2150 void enetc_get_si_caps(struct enetc_si *si) 2151 { 2152 struct enetc_hw *hw = &si->hw; 2153 u32 val; 2154 2155 /* find out how many of various resources we have to work with */ 2156 val = enetc_rd(hw, ENETC_SICAPR0); 2157 si->num_rx_rings = (val >> 16) & 0xff; 2158 si->num_tx_rings = val & 0xff; 2159 2160 val = enetc_rd(hw, ENETC_SIPCAPR0); 2161 if (val & ENETC_SIPCAPR0_RFS) { 2162 val = enetc_rd(hw, ENETC_SIRFSCAPR); 2163 si->num_fs_entries = ENETC_SIRFSCAPR_GET_NUM_RFS(val); 2164 si->num_fs_entries = min(si->num_fs_entries, ENETC_MAX_RFS_SIZE); 2165 } else { 2166 /* ENETC which not supports RFS */ 2167 si->num_fs_entries = 0; 2168 } 2169 2170 si->num_rss = 0; 2171 val = enetc_rd(hw, ENETC_SIPCAPR0); 2172 if (val & ENETC_SIPCAPR0_RSS) { 2173 u32 rss; 2174 2175 rss = enetc_rd(hw, ENETC_SIRSSCAPR); 2176 si->num_rss = ENETC_SIRSSCAPR_GET_NUM_RSS(rss); 2177 } 2178 2179 if (val & ENETC_SIPCAPR0_LSO) 2180 si->hw_features |= ENETC_SI_F_LSO; 2181 } 2182 EXPORT_SYMBOL_GPL(enetc_get_si_caps); 2183 2184 static int enetc_dma_alloc_bdr(struct enetc_bdr_resource *res) 2185 { 2186 size_t bd_base_size = res->bd_count * res->bd_size; 2187 2188 res->bd_base = dma_alloc_coherent(res->dev, bd_base_size, 2189 &res->bd_dma_base, GFP_KERNEL); 2190 if (!res->bd_base) 2191 return -ENOMEM; 2192 2193 /* h/w requires 128B alignment */ 2194 if (!IS_ALIGNED(res->bd_dma_base, 128)) { 2195 dma_free_coherent(res->dev, bd_base_size, res->bd_base, 2196 res->bd_dma_base); 2197 return -EINVAL; 2198 } 2199 2200 return 0; 2201 } 2202 2203 static void enetc_dma_free_bdr(const struct enetc_bdr_resource *res) 2204 { 2205 size_t bd_base_size = res->bd_count * res->bd_size; 2206 2207 dma_free_coherent(res->dev, bd_base_size, res->bd_base, 2208 res->bd_dma_base); 2209 } 2210 2211 static int enetc_alloc_tx_resource(struct enetc_bdr_resource *res, 2212 struct device *dev, size_t bd_count) 2213 { 2214 int err; 2215 2216 res->dev = dev; 2217 res->bd_count = bd_count; 2218 res->bd_size = sizeof(union enetc_tx_bd); 2219 2220 res->tx_swbd = vcalloc(bd_count, sizeof(*res->tx_swbd)); 2221 if (!res->tx_swbd) 2222 return -ENOMEM; 2223 2224 err = enetc_dma_alloc_bdr(res); 2225 if (err) 2226 goto err_alloc_bdr; 2227 2228 res->tso_headers = dma_alloc_coherent(dev, bd_count * TSO_HEADER_SIZE, 2229 &res->tso_headers_dma, 2230 GFP_KERNEL); 2231 if (!res->tso_headers) { 2232 err = -ENOMEM; 2233 goto err_alloc_tso; 2234 } 2235 2236 return 0; 2237 2238 err_alloc_tso: 2239 enetc_dma_free_bdr(res); 2240 err_alloc_bdr: 2241 vfree(res->tx_swbd); 2242 res->tx_swbd = NULL; 2243 2244 return err; 2245 } 2246 2247 static void enetc_free_tx_resource(const struct enetc_bdr_resource *res) 2248 { 2249 dma_free_coherent(res->dev, res->bd_count * TSO_HEADER_SIZE, 2250 res->tso_headers, res->tso_headers_dma); 2251 enetc_dma_free_bdr(res); 2252 vfree(res->tx_swbd); 2253 } 2254 2255 static struct enetc_bdr_resource * 2256 enetc_alloc_tx_resources(struct enetc_ndev_priv *priv) 2257 { 2258 struct enetc_bdr_resource *tx_res; 2259 int i, err; 2260 2261 tx_res = kcalloc(priv->num_tx_rings, sizeof(*tx_res), GFP_KERNEL); 2262 if (!tx_res) 2263 return ERR_PTR(-ENOMEM); 2264 2265 for (i = 0; i < priv->num_tx_rings; i++) { 2266 struct enetc_bdr *tx_ring = priv->tx_ring[i]; 2267 2268 err = enetc_alloc_tx_resource(&tx_res[i], tx_ring->dev, 2269 tx_ring->bd_count); 2270 if (err) 2271 goto fail; 2272 } 2273 2274 return tx_res; 2275 2276 fail: 2277 while (i-- > 0) 2278 enetc_free_tx_resource(&tx_res[i]); 2279 2280 kfree(tx_res); 2281 2282 return ERR_PTR(err); 2283 } 2284 2285 static void enetc_free_tx_resources(const struct enetc_bdr_resource *tx_res, 2286 size_t num_resources) 2287 { 2288 size_t i; 2289 2290 for (i = 0; i < num_resources; i++) 2291 enetc_free_tx_resource(&tx_res[i]); 2292 2293 kfree(tx_res); 2294 } 2295 2296 static int enetc_alloc_rx_resource(struct enetc_bdr_resource *res, 2297 struct device *dev, size_t bd_count, 2298 bool extended) 2299 { 2300 int err; 2301 2302 res->dev = dev; 2303 res->bd_count = bd_count; 2304 res->bd_size = sizeof(union enetc_rx_bd); 2305 if (extended) 2306 res->bd_size *= 2; 2307 2308 res->rx_swbd = vcalloc(bd_count, sizeof(struct enetc_rx_swbd)); 2309 if (!res->rx_swbd) 2310 return -ENOMEM; 2311 2312 err = enetc_dma_alloc_bdr(res); 2313 if (err) { 2314 vfree(res->rx_swbd); 2315 return err; 2316 } 2317 2318 return 0; 2319 } 2320 2321 static void enetc_free_rx_resource(const struct enetc_bdr_resource *res) 2322 { 2323 enetc_dma_free_bdr(res); 2324 vfree(res->rx_swbd); 2325 } 2326 2327 static struct enetc_bdr_resource * 2328 enetc_alloc_rx_resources(struct enetc_ndev_priv *priv, bool extended) 2329 { 2330 struct enetc_bdr_resource *rx_res; 2331 int i, err; 2332 2333 rx_res = kcalloc(priv->num_rx_rings, sizeof(*rx_res), GFP_KERNEL); 2334 if (!rx_res) 2335 return ERR_PTR(-ENOMEM); 2336 2337 for (i = 0; i < priv->num_rx_rings; i++) { 2338 struct enetc_bdr *rx_ring = priv->rx_ring[i]; 2339 2340 err = enetc_alloc_rx_resource(&rx_res[i], rx_ring->dev, 2341 rx_ring->bd_count, extended); 2342 if (err) 2343 goto fail; 2344 } 2345 2346 return rx_res; 2347 2348 fail: 2349 while (i-- > 0) 2350 enetc_free_rx_resource(&rx_res[i]); 2351 2352 kfree(rx_res); 2353 2354 return ERR_PTR(err); 2355 } 2356 2357 static void enetc_free_rx_resources(const struct enetc_bdr_resource *rx_res, 2358 size_t num_resources) 2359 { 2360 size_t i; 2361 2362 for (i = 0; i < num_resources; i++) 2363 enetc_free_rx_resource(&rx_res[i]); 2364 2365 kfree(rx_res); 2366 } 2367 2368 static void enetc_assign_tx_resource(struct enetc_bdr *tx_ring, 2369 const struct enetc_bdr_resource *res) 2370 { 2371 tx_ring->bd_base = res ? res->bd_base : NULL; 2372 tx_ring->bd_dma_base = res ? res->bd_dma_base : 0; 2373 tx_ring->tx_swbd = res ? res->tx_swbd : NULL; 2374 tx_ring->tso_headers = res ? res->tso_headers : NULL; 2375 tx_ring->tso_headers_dma = res ? res->tso_headers_dma : 0; 2376 } 2377 2378 static void enetc_assign_rx_resource(struct enetc_bdr *rx_ring, 2379 const struct enetc_bdr_resource *res) 2380 { 2381 rx_ring->bd_base = res ? res->bd_base : NULL; 2382 rx_ring->bd_dma_base = res ? res->bd_dma_base : 0; 2383 rx_ring->rx_swbd = res ? res->rx_swbd : NULL; 2384 } 2385 2386 static void enetc_assign_tx_resources(struct enetc_ndev_priv *priv, 2387 const struct enetc_bdr_resource *res) 2388 { 2389 int i; 2390 2391 if (priv->tx_res) 2392 enetc_free_tx_resources(priv->tx_res, priv->num_tx_rings); 2393 2394 for (i = 0; i < priv->num_tx_rings; i++) { 2395 enetc_assign_tx_resource(priv->tx_ring[i], 2396 res ? &res[i] : NULL); 2397 } 2398 2399 priv->tx_res = res; 2400 } 2401 2402 static void enetc_assign_rx_resources(struct enetc_ndev_priv *priv, 2403 const struct enetc_bdr_resource *res) 2404 { 2405 int i; 2406 2407 if (priv->rx_res) 2408 enetc_free_rx_resources(priv->rx_res, priv->num_rx_rings); 2409 2410 for (i = 0; i < priv->num_rx_rings; i++) { 2411 enetc_assign_rx_resource(priv->rx_ring[i], 2412 res ? &res[i] : NULL); 2413 } 2414 2415 priv->rx_res = res; 2416 } 2417 2418 static void enetc_free_tx_ring(struct enetc_bdr *tx_ring) 2419 { 2420 int i; 2421 2422 for (i = 0; i < tx_ring->bd_count; i++) { 2423 struct enetc_tx_swbd *tx_swbd = &tx_ring->tx_swbd[i]; 2424 2425 enetc_free_tx_frame(tx_ring, tx_swbd); 2426 } 2427 } 2428 2429 static void enetc_free_rx_ring(struct enetc_bdr *rx_ring) 2430 { 2431 int i; 2432 2433 for (i = 0; i < rx_ring->bd_count; i++) { 2434 struct enetc_rx_swbd *rx_swbd = &rx_ring->rx_swbd[i]; 2435 2436 if (!rx_swbd->page) 2437 continue; 2438 2439 dma_unmap_page(rx_ring->dev, rx_swbd->dma, PAGE_SIZE, 2440 rx_swbd->dir); 2441 __free_page(rx_swbd->page); 2442 rx_swbd->page = NULL; 2443 } 2444 } 2445 2446 static void enetc_free_rxtx_rings(struct enetc_ndev_priv *priv) 2447 { 2448 int i; 2449 2450 for (i = 0; i < priv->num_rx_rings; i++) 2451 enetc_free_rx_ring(priv->rx_ring[i]); 2452 2453 for (i = 0; i < priv->num_tx_rings; i++) 2454 enetc_free_tx_ring(priv->tx_ring[i]); 2455 } 2456 2457 static int enetc_setup_default_rss_table(struct enetc_si *si, int num_groups) 2458 { 2459 int *rss_table; 2460 int i; 2461 2462 rss_table = kmalloc_array(si->num_rss, sizeof(*rss_table), GFP_KERNEL); 2463 if (!rss_table) 2464 return -ENOMEM; 2465 2466 /* Set up RSS table defaults */ 2467 for (i = 0; i < si->num_rss; i++) 2468 rss_table[i] = i % num_groups; 2469 2470 si->ops->set_rss_table(si, rss_table, si->num_rss); 2471 2472 kfree(rss_table); 2473 2474 return 0; 2475 } 2476 2477 static void enetc_set_lso_flags_mask(struct enetc_hw *hw) 2478 { 2479 enetc_wr(hw, ENETC4_SILSOSFMR0, 2480 SILSOSFMR0_VAL_SET(ENETC4_TCP_NL_SEG_FLAGS_DMASK, 2481 ENETC4_TCP_NL_SEG_FLAGS_DMASK)); 2482 enetc_wr(hw, ENETC4_SILSOSFMR1, 0); 2483 } 2484 2485 static void enetc_set_rss(struct net_device *ndev, int en) 2486 { 2487 struct enetc_ndev_priv *priv = netdev_priv(ndev); 2488 struct enetc_hw *hw = &priv->si->hw; 2489 u32 reg; 2490 2491 enetc_wr(hw, ENETC_SIRBGCR, priv->num_rx_rings); 2492 2493 reg = enetc_rd(hw, ENETC_SIMR); 2494 reg &= ~ENETC_SIMR_RSSE; 2495 reg |= (en) ? ENETC_SIMR_RSSE : 0; 2496 enetc_wr(hw, ENETC_SIMR, reg); 2497 } 2498 2499 int enetc_configure_si(struct enetc_ndev_priv *priv) 2500 { 2501 struct enetc_si *si = priv->si; 2502 struct enetc_hw *hw = &si->hw; 2503 int err; 2504 2505 /* set SI cache attributes */ 2506 enetc_wr(hw, ENETC_SICAR0, 2507 ENETC_SICAR_RD_COHERENT | ENETC_SICAR_WR_COHERENT); 2508 enetc_wr(hw, ENETC_SICAR1, ENETC_SICAR_MSI); 2509 /* enable SI */ 2510 enetc_wr(hw, ENETC_SIMR, ENETC_SIMR_EN); 2511 2512 if (si->hw_features & ENETC_SI_F_LSO) 2513 enetc_set_lso_flags_mask(hw); 2514 2515 if (si->num_rss) { 2516 err = enetc_setup_default_rss_table(si, priv->num_rx_rings); 2517 if (err) 2518 return err; 2519 2520 if (priv->ndev->features & NETIF_F_RXHASH) 2521 enetc_set_rss(priv->ndev, true); 2522 } 2523 2524 return 0; 2525 } 2526 EXPORT_SYMBOL_GPL(enetc_configure_si); 2527 2528 void enetc_init_si_rings_params(struct enetc_ndev_priv *priv) 2529 { 2530 struct enetc_si *si = priv->si; 2531 int cpus = num_online_cpus(); 2532 2533 priv->tx_bd_count = ENETC_TX_RING_DEFAULT_SIZE; 2534 priv->rx_bd_count = ENETC_RX_RING_DEFAULT_SIZE; 2535 2536 /* Enable all available TX rings in order to configure as many 2537 * priorities as possible, when needed. 2538 * TODO: Make # of TX rings run-time configurable 2539 */ 2540 priv->num_rx_rings = min_t(int, cpus, si->num_rx_rings); 2541 priv->num_tx_rings = si->num_tx_rings; 2542 priv->bdr_int_num = priv->num_rx_rings; 2543 priv->ic_mode = ENETC_IC_RX_ADAPTIVE | ENETC_IC_TX_MANUAL; 2544 priv->tx_ictt = enetc_usecs_to_cycles(600, priv->sysclk_freq); 2545 } 2546 EXPORT_SYMBOL_GPL(enetc_init_si_rings_params); 2547 2548 int enetc_alloc_si_resources(struct enetc_ndev_priv *priv) 2549 { 2550 struct enetc_si *si = priv->si; 2551 2552 priv->cls_rules = kcalloc(si->num_fs_entries, sizeof(*priv->cls_rules), 2553 GFP_KERNEL); 2554 if (!priv->cls_rules) 2555 return -ENOMEM; 2556 2557 return 0; 2558 } 2559 EXPORT_SYMBOL_GPL(enetc_alloc_si_resources); 2560 2561 void enetc_free_si_resources(struct enetc_ndev_priv *priv) 2562 { 2563 kfree(priv->cls_rules); 2564 } 2565 EXPORT_SYMBOL_GPL(enetc_free_si_resources); 2566 2567 static void enetc_setup_txbdr(struct enetc_hw *hw, struct enetc_bdr *tx_ring) 2568 { 2569 int idx = tx_ring->index; 2570 u32 tbmr; 2571 2572 enetc_txbdr_wr(hw, idx, ENETC_TBBAR0, 2573 lower_32_bits(tx_ring->bd_dma_base)); 2574 2575 enetc_txbdr_wr(hw, idx, ENETC_TBBAR1, 2576 upper_32_bits(tx_ring->bd_dma_base)); 2577 2578 WARN_ON(!IS_ALIGNED(tx_ring->bd_count, 64)); /* multiple of 64 */ 2579 enetc_txbdr_wr(hw, idx, ENETC_TBLENR, 2580 ENETC_RTBLENR_LEN(tx_ring->bd_count)); 2581 2582 /* clearing PI/CI registers for Tx not supported, adjust sw indexes */ 2583 tx_ring->next_to_use = enetc_txbdr_rd(hw, idx, ENETC_TBPIR); 2584 tx_ring->next_to_clean = enetc_txbdr_rd(hw, idx, ENETC_TBCIR); 2585 2586 /* enable Tx ints by setting pkt thr to 1 */ 2587 enetc_txbdr_wr(hw, idx, ENETC_TBICR0, ENETC_TBICR0_ICEN | 0x1); 2588 2589 tbmr = ENETC_TBMR_SET_PRIO(tx_ring->prio); 2590 if (tx_ring->ndev->features & NETIF_F_HW_VLAN_CTAG_TX) 2591 tbmr |= ENETC_TBMR_VIH; 2592 2593 /* enable ring */ 2594 enetc_txbdr_wr(hw, idx, ENETC_TBMR, tbmr); 2595 2596 tx_ring->tpir = hw->reg + ENETC_BDR(TX, idx, ENETC_TBPIR); 2597 tx_ring->tcir = hw->reg + ENETC_BDR(TX, idx, ENETC_TBCIR); 2598 tx_ring->idr = hw->reg + ENETC_SITXIDR; 2599 } 2600 2601 static void enetc_setup_rxbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring, 2602 bool extended) 2603 { 2604 int idx = rx_ring->index; 2605 u32 rbmr = 0; 2606 2607 enetc_rxbdr_wr(hw, idx, ENETC_RBBAR0, 2608 lower_32_bits(rx_ring->bd_dma_base)); 2609 2610 enetc_rxbdr_wr(hw, idx, ENETC_RBBAR1, 2611 upper_32_bits(rx_ring->bd_dma_base)); 2612 2613 WARN_ON(!IS_ALIGNED(rx_ring->bd_count, 64)); /* multiple of 64 */ 2614 enetc_rxbdr_wr(hw, idx, ENETC_RBLENR, 2615 ENETC_RTBLENR_LEN(rx_ring->bd_count)); 2616 2617 if (rx_ring->xdp.prog) 2618 enetc_rxbdr_wr(hw, idx, ENETC_RBBSR, ENETC_RXB_DMA_SIZE_XDP); 2619 else 2620 enetc_rxbdr_wr(hw, idx, ENETC_RBBSR, ENETC_RXB_DMA_SIZE); 2621 2622 /* Also prepare the consumer index in case page allocation never 2623 * succeeds. In that case, hardware will never advance producer index 2624 * to match consumer index, and will drop all frames. 2625 */ 2626 enetc_rxbdr_wr(hw, idx, ENETC_RBPIR, 0); 2627 enetc_rxbdr_wr(hw, idx, ENETC_RBCIR, 1); 2628 2629 /* enable Rx ints by setting pkt thr to 1 */ 2630 enetc_rxbdr_wr(hw, idx, ENETC_RBICR0, ENETC_RBICR0_ICEN | 0x1); 2631 2632 rx_ring->ext_en = extended; 2633 if (rx_ring->ext_en) 2634 rbmr |= ENETC_RBMR_BDS; 2635 2636 if (rx_ring->ndev->features & NETIF_F_HW_VLAN_CTAG_RX) 2637 rbmr |= ENETC_RBMR_VTE; 2638 2639 rx_ring->rcir = hw->reg + ENETC_BDR(RX, idx, ENETC_RBCIR); 2640 rx_ring->idr = hw->reg + ENETC_SIRXIDR; 2641 2642 rx_ring->next_to_clean = 0; 2643 rx_ring->next_to_use = 0; 2644 rx_ring->next_to_alloc = 0; 2645 2646 enetc_lock_mdio(); 2647 enetc_refill_rx_ring(rx_ring, enetc_bd_unused(rx_ring)); 2648 enetc_unlock_mdio(); 2649 2650 enetc_rxbdr_wr(hw, idx, ENETC_RBMR, rbmr); 2651 } 2652 2653 static void enetc_setup_bdrs(struct enetc_ndev_priv *priv, bool extended) 2654 { 2655 struct enetc_hw *hw = &priv->si->hw; 2656 int i; 2657 2658 for (i = 0; i < priv->num_tx_rings; i++) 2659 enetc_setup_txbdr(hw, priv->tx_ring[i]); 2660 2661 for (i = 0; i < priv->num_rx_rings; i++) 2662 enetc_setup_rxbdr(hw, priv->rx_ring[i], extended); 2663 } 2664 2665 static void enetc_enable_txbdr(struct enetc_hw *hw, struct enetc_bdr *tx_ring) 2666 { 2667 int idx = tx_ring->index; 2668 u32 tbmr; 2669 2670 tbmr = enetc_txbdr_rd(hw, idx, ENETC_TBMR); 2671 tbmr |= ENETC_TBMR_EN; 2672 enetc_txbdr_wr(hw, idx, ENETC_TBMR, tbmr); 2673 } 2674 2675 static void enetc_enable_rxbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring) 2676 { 2677 int idx = rx_ring->index; 2678 u32 rbmr; 2679 2680 rbmr = enetc_rxbdr_rd(hw, idx, ENETC_RBMR); 2681 rbmr |= ENETC_RBMR_EN; 2682 enetc_rxbdr_wr(hw, idx, ENETC_RBMR, rbmr); 2683 } 2684 2685 static void enetc_enable_rx_bdrs(struct enetc_ndev_priv *priv) 2686 { 2687 struct enetc_hw *hw = &priv->si->hw; 2688 int i; 2689 2690 for (i = 0; i < priv->num_rx_rings; i++) 2691 enetc_enable_rxbdr(hw, priv->rx_ring[i]); 2692 } 2693 2694 static void enetc_enable_tx_bdrs(struct enetc_ndev_priv *priv) 2695 { 2696 struct enetc_hw *hw = &priv->si->hw; 2697 int i; 2698 2699 for (i = 0; i < priv->num_tx_rings; i++) 2700 enetc_enable_txbdr(hw, priv->tx_ring[i]); 2701 } 2702 2703 static void enetc_disable_rxbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring) 2704 { 2705 int idx = rx_ring->index; 2706 2707 /* disable EN bit on ring */ 2708 enetc_rxbdr_wr(hw, idx, ENETC_RBMR, 0); 2709 } 2710 2711 static void enetc_disable_txbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring) 2712 { 2713 int idx = rx_ring->index; 2714 2715 /* disable EN bit on ring */ 2716 enetc_txbdr_wr(hw, idx, ENETC_TBMR, 0); 2717 } 2718 2719 static void enetc_disable_rx_bdrs(struct enetc_ndev_priv *priv) 2720 { 2721 struct enetc_hw *hw = &priv->si->hw; 2722 int i; 2723 2724 for (i = 0; i < priv->num_rx_rings; i++) 2725 enetc_disable_rxbdr(hw, priv->rx_ring[i]); 2726 } 2727 2728 static void enetc_disable_tx_bdrs(struct enetc_ndev_priv *priv) 2729 { 2730 struct enetc_hw *hw = &priv->si->hw; 2731 int i; 2732 2733 for (i = 0; i < priv->num_tx_rings; i++) 2734 enetc_disable_txbdr(hw, priv->tx_ring[i]); 2735 } 2736 2737 static void enetc_wait_txbdr(struct enetc_hw *hw, struct enetc_bdr *tx_ring) 2738 { 2739 int delay = 8, timeout = 100; 2740 int idx = tx_ring->index; 2741 2742 /* wait for busy to clear */ 2743 while (delay < timeout && 2744 enetc_txbdr_rd(hw, idx, ENETC_TBSR) & ENETC_TBSR_BUSY) { 2745 msleep(delay); 2746 delay *= 2; 2747 } 2748 2749 if (delay >= timeout) 2750 netdev_warn(tx_ring->ndev, "timeout for tx ring #%d clear\n", 2751 idx); 2752 } 2753 2754 static void enetc_wait_bdrs(struct enetc_ndev_priv *priv) 2755 { 2756 struct enetc_hw *hw = &priv->si->hw; 2757 int i; 2758 2759 for (i = 0; i < priv->num_tx_rings; i++) 2760 enetc_wait_txbdr(hw, priv->tx_ring[i]); 2761 } 2762 2763 static int enetc_setup_irqs(struct enetc_ndev_priv *priv) 2764 { 2765 struct pci_dev *pdev = priv->si->pdev; 2766 struct enetc_hw *hw = &priv->si->hw; 2767 int i, j, err; 2768 2769 for (i = 0; i < priv->bdr_int_num; i++) { 2770 int irq = pci_irq_vector(pdev, ENETC_BDR_INT_BASE_IDX + i); 2771 struct enetc_int_vector *v = priv->int_vector[i]; 2772 int entry = ENETC_BDR_INT_BASE_IDX + i; 2773 2774 snprintf(v->name, sizeof(v->name), "%s-rxtx%d", 2775 priv->ndev->name, i); 2776 err = request_irq(irq, enetc_msix, IRQF_NO_AUTOEN, v->name, v); 2777 if (err) { 2778 dev_err(priv->dev, "request_irq() failed!\n"); 2779 goto irq_err; 2780 } 2781 2782 v->tbier_base = hw->reg + ENETC_BDR(TX, 0, ENETC_TBIER); 2783 v->rbier = hw->reg + ENETC_BDR(RX, i, ENETC_RBIER); 2784 v->ricr1 = hw->reg + ENETC_BDR(RX, i, ENETC_RBICR1); 2785 2786 enetc_wr(hw, ENETC_SIMSIRRV(i), entry); 2787 2788 for (j = 0; j < v->count_tx_rings; j++) { 2789 int idx = v->tx_ring[j].index; 2790 2791 enetc_wr(hw, ENETC_SIMSITRV(idx), entry); 2792 } 2793 irq_set_affinity_hint(irq, get_cpu_mask(i % num_online_cpus())); 2794 } 2795 2796 return 0; 2797 2798 irq_err: 2799 while (i--) { 2800 int irq = pci_irq_vector(pdev, ENETC_BDR_INT_BASE_IDX + i); 2801 2802 irq_set_affinity_hint(irq, NULL); 2803 free_irq(irq, priv->int_vector[i]); 2804 } 2805 2806 return err; 2807 } 2808 2809 static void enetc_free_irqs(struct enetc_ndev_priv *priv) 2810 { 2811 struct pci_dev *pdev = priv->si->pdev; 2812 int i; 2813 2814 for (i = 0; i < priv->bdr_int_num; i++) { 2815 int irq = pci_irq_vector(pdev, ENETC_BDR_INT_BASE_IDX + i); 2816 2817 irq_set_affinity_hint(irq, NULL); 2818 free_irq(irq, priv->int_vector[i]); 2819 } 2820 } 2821 2822 static void enetc_setup_interrupts(struct enetc_ndev_priv *priv) 2823 { 2824 struct enetc_hw *hw = &priv->si->hw; 2825 u32 icpt, ictt; 2826 int i; 2827 2828 /* enable Tx & Rx event indication */ 2829 if (priv->ic_mode & 2830 (ENETC_IC_RX_MANUAL | ENETC_IC_RX_ADAPTIVE)) { 2831 icpt = ENETC_RBICR0_SET_ICPT(ENETC_RXIC_PKTTHR); 2832 /* init to non-0 minimum, will be adjusted later */ 2833 ictt = 0x1; 2834 } else { 2835 icpt = 0x1; /* enable Rx ints by setting pkt thr to 1 */ 2836 ictt = 0; 2837 } 2838 2839 for (i = 0; i < priv->num_rx_rings; i++) { 2840 enetc_rxbdr_wr(hw, i, ENETC_RBICR1, ictt); 2841 enetc_rxbdr_wr(hw, i, ENETC_RBICR0, ENETC_RBICR0_ICEN | icpt); 2842 enetc_rxbdr_wr(hw, i, ENETC_RBIER, ENETC_RBIER_RXTIE); 2843 } 2844 2845 if (priv->ic_mode & ENETC_IC_TX_MANUAL) 2846 icpt = ENETC_TBICR0_SET_ICPT(ENETC_TXIC_PKTTHR); 2847 else 2848 icpt = 0x1; /* enable Tx ints by setting pkt thr to 1 */ 2849 2850 for (i = 0; i < priv->num_tx_rings; i++) { 2851 enetc_txbdr_wr(hw, i, ENETC_TBICR1, priv->tx_ictt); 2852 enetc_txbdr_wr(hw, i, ENETC_TBICR0, ENETC_TBICR0_ICEN | icpt); 2853 enetc_txbdr_wr(hw, i, ENETC_TBIER, ENETC_TBIER_TXTIE); 2854 } 2855 } 2856 2857 static void enetc_clear_interrupts(struct enetc_ndev_priv *priv) 2858 { 2859 struct enetc_hw *hw = &priv->si->hw; 2860 int i; 2861 2862 for (i = 0; i < priv->num_tx_rings; i++) 2863 enetc_txbdr_wr(hw, i, ENETC_TBIER, 0); 2864 2865 for (i = 0; i < priv->num_rx_rings; i++) 2866 enetc_rxbdr_wr(hw, i, ENETC_RBIER, 0); 2867 } 2868 2869 static int enetc_phylink_connect(struct net_device *ndev) 2870 { 2871 struct enetc_ndev_priv *priv = netdev_priv(ndev); 2872 struct ethtool_keee edata; 2873 int err; 2874 2875 if (!priv->phylink) { 2876 /* phy-less mode */ 2877 netif_carrier_on(ndev); 2878 return 0; 2879 } 2880 2881 err = phylink_of_phy_connect(priv->phylink, priv->dev->of_node, 0); 2882 if (err) { 2883 dev_err(&ndev->dev, "could not attach to PHY\n"); 2884 return err; 2885 } 2886 2887 /* disable EEE autoneg, until ENETC driver supports it */ 2888 memset(&edata, 0, sizeof(struct ethtool_keee)); 2889 phylink_ethtool_set_eee(priv->phylink, &edata); 2890 2891 phylink_start(priv->phylink); 2892 2893 return 0; 2894 } 2895 2896 static void enetc_tx_onestep_tstamp(struct work_struct *work) 2897 { 2898 struct enetc_ndev_priv *priv; 2899 struct sk_buff *skb; 2900 2901 priv = container_of(work, struct enetc_ndev_priv, tx_onestep_tstamp); 2902 2903 netif_tx_lock_bh(priv->ndev); 2904 2905 clear_bit_unlock(ENETC_TX_ONESTEP_TSTAMP_IN_PROGRESS, &priv->flags); 2906 skb = skb_dequeue(&priv->tx_skbs); 2907 if (skb) 2908 enetc_start_xmit(skb, priv->ndev); 2909 2910 netif_tx_unlock_bh(priv->ndev); 2911 } 2912 2913 static void enetc_tx_onestep_tstamp_init(struct enetc_ndev_priv *priv) 2914 { 2915 INIT_WORK(&priv->tx_onestep_tstamp, enetc_tx_onestep_tstamp); 2916 skb_queue_head_init(&priv->tx_skbs); 2917 } 2918 2919 void enetc_start(struct net_device *ndev) 2920 { 2921 struct enetc_ndev_priv *priv = netdev_priv(ndev); 2922 int i; 2923 2924 enetc_setup_interrupts(priv); 2925 2926 for (i = 0; i < priv->bdr_int_num; i++) { 2927 int irq = pci_irq_vector(priv->si->pdev, 2928 ENETC_BDR_INT_BASE_IDX + i); 2929 2930 napi_enable(&priv->int_vector[i]->napi); 2931 enable_irq(irq); 2932 } 2933 2934 enetc_enable_tx_bdrs(priv); 2935 2936 enetc_enable_rx_bdrs(priv); 2937 2938 netif_tx_start_all_queues(ndev); 2939 2940 clear_bit(ENETC_TX_DOWN, &priv->flags); 2941 } 2942 EXPORT_SYMBOL_GPL(enetc_start); 2943 2944 int enetc_open(struct net_device *ndev) 2945 { 2946 struct enetc_ndev_priv *priv = netdev_priv(ndev); 2947 struct enetc_bdr_resource *tx_res, *rx_res; 2948 bool extended; 2949 int err; 2950 2951 extended = !!(priv->active_offloads & ENETC_F_RX_TSTAMP); 2952 2953 err = clk_prepare_enable(priv->ref_clk); 2954 if (err) 2955 return err; 2956 2957 err = enetc_setup_irqs(priv); 2958 if (err) 2959 goto err_setup_irqs; 2960 2961 err = enetc_phylink_connect(ndev); 2962 if (err) 2963 goto err_phy_connect; 2964 2965 tx_res = enetc_alloc_tx_resources(priv); 2966 if (IS_ERR(tx_res)) { 2967 err = PTR_ERR(tx_res); 2968 goto err_alloc_tx; 2969 } 2970 2971 rx_res = enetc_alloc_rx_resources(priv, extended); 2972 if (IS_ERR(rx_res)) { 2973 err = PTR_ERR(rx_res); 2974 goto err_alloc_rx; 2975 } 2976 2977 enetc_tx_onestep_tstamp_init(priv); 2978 enetc_assign_tx_resources(priv, tx_res); 2979 enetc_assign_rx_resources(priv, rx_res); 2980 enetc_setup_bdrs(priv, extended); 2981 enetc_start(ndev); 2982 2983 return 0; 2984 2985 err_alloc_rx: 2986 enetc_free_tx_resources(tx_res, priv->num_tx_rings); 2987 err_alloc_tx: 2988 if (priv->phylink) 2989 phylink_disconnect_phy(priv->phylink); 2990 err_phy_connect: 2991 enetc_free_irqs(priv); 2992 err_setup_irqs: 2993 clk_disable_unprepare(priv->ref_clk); 2994 2995 return err; 2996 } 2997 EXPORT_SYMBOL_GPL(enetc_open); 2998 2999 void enetc_stop(struct net_device *ndev) 3000 { 3001 struct enetc_ndev_priv *priv = netdev_priv(ndev); 3002 int i; 3003 3004 set_bit(ENETC_TX_DOWN, &priv->flags); 3005 3006 netif_tx_stop_all_queues(ndev); 3007 3008 enetc_disable_rx_bdrs(priv); 3009 3010 enetc_wait_bdrs(priv); 3011 3012 enetc_disable_tx_bdrs(priv); 3013 3014 for (i = 0; i < priv->bdr_int_num; i++) { 3015 int irq = pci_irq_vector(priv->si->pdev, 3016 ENETC_BDR_INT_BASE_IDX + i); 3017 3018 disable_irq(irq); 3019 napi_synchronize(&priv->int_vector[i]->napi); 3020 napi_disable(&priv->int_vector[i]->napi); 3021 } 3022 3023 enetc_clear_interrupts(priv); 3024 } 3025 EXPORT_SYMBOL_GPL(enetc_stop); 3026 3027 int enetc_close(struct net_device *ndev) 3028 { 3029 struct enetc_ndev_priv *priv = netdev_priv(ndev); 3030 3031 enetc_stop(ndev); 3032 3033 if (priv->phylink) { 3034 phylink_stop(priv->phylink); 3035 phylink_disconnect_phy(priv->phylink); 3036 } else { 3037 netif_carrier_off(ndev); 3038 } 3039 3040 enetc_free_rxtx_rings(priv); 3041 3042 /* Avoids dangling pointers and also frees old resources */ 3043 enetc_assign_rx_resources(priv, NULL); 3044 enetc_assign_tx_resources(priv, NULL); 3045 3046 enetc_free_irqs(priv); 3047 clk_disable_unprepare(priv->ref_clk); 3048 3049 return 0; 3050 } 3051 EXPORT_SYMBOL_GPL(enetc_close); 3052 3053 static int enetc_reconfigure(struct enetc_ndev_priv *priv, bool extended, 3054 int (*cb)(struct enetc_ndev_priv *priv, void *ctx), 3055 void *ctx) 3056 { 3057 struct enetc_bdr_resource *tx_res, *rx_res; 3058 int err; 3059 3060 ASSERT_RTNL(); 3061 3062 /* If the interface is down, run the callback right away, 3063 * without reconfiguration. 3064 */ 3065 if (!netif_running(priv->ndev)) { 3066 if (cb) { 3067 err = cb(priv, ctx); 3068 if (err) 3069 return err; 3070 } 3071 3072 return 0; 3073 } 3074 3075 tx_res = enetc_alloc_tx_resources(priv); 3076 if (IS_ERR(tx_res)) { 3077 err = PTR_ERR(tx_res); 3078 goto out; 3079 } 3080 3081 rx_res = enetc_alloc_rx_resources(priv, extended); 3082 if (IS_ERR(rx_res)) { 3083 err = PTR_ERR(rx_res); 3084 goto out_free_tx_res; 3085 } 3086 3087 enetc_stop(priv->ndev); 3088 enetc_free_rxtx_rings(priv); 3089 3090 /* Interface is down, run optional callback now */ 3091 if (cb) { 3092 err = cb(priv, ctx); 3093 if (err) 3094 goto out_restart; 3095 } 3096 3097 enetc_assign_tx_resources(priv, tx_res); 3098 enetc_assign_rx_resources(priv, rx_res); 3099 enetc_setup_bdrs(priv, extended); 3100 enetc_start(priv->ndev); 3101 3102 return 0; 3103 3104 out_restart: 3105 enetc_setup_bdrs(priv, extended); 3106 enetc_start(priv->ndev); 3107 enetc_free_rx_resources(rx_res, priv->num_rx_rings); 3108 out_free_tx_res: 3109 enetc_free_tx_resources(tx_res, priv->num_tx_rings); 3110 out: 3111 return err; 3112 } 3113 3114 static void enetc_debug_tx_ring_prios(struct enetc_ndev_priv *priv) 3115 { 3116 int i; 3117 3118 for (i = 0; i < priv->num_tx_rings; i++) 3119 netdev_dbg(priv->ndev, "TX ring %d prio %d\n", i, 3120 priv->tx_ring[i]->prio); 3121 } 3122 3123 void enetc_reset_tc_mqprio(struct net_device *ndev) 3124 { 3125 struct enetc_ndev_priv *priv = netdev_priv(ndev); 3126 struct enetc_hw *hw = &priv->si->hw; 3127 struct enetc_bdr *tx_ring; 3128 int num_stack_tx_queues; 3129 int i; 3130 3131 num_stack_tx_queues = enetc_num_stack_tx_queues(priv); 3132 3133 netdev_reset_tc(ndev); 3134 netif_set_real_num_tx_queues(ndev, num_stack_tx_queues); 3135 priv->min_num_stack_tx_queues = num_possible_cpus(); 3136 3137 /* Reset all ring priorities to 0 */ 3138 for (i = 0; i < priv->num_tx_rings; i++) { 3139 tx_ring = priv->tx_ring[i]; 3140 tx_ring->prio = 0; 3141 enetc_set_bdr_prio(hw, tx_ring->index, tx_ring->prio); 3142 } 3143 3144 enetc_debug_tx_ring_prios(priv); 3145 3146 enetc_change_preemptible_tcs(priv, 0); 3147 } 3148 EXPORT_SYMBOL_GPL(enetc_reset_tc_mqprio); 3149 3150 int enetc_setup_tc_mqprio(struct net_device *ndev, void *type_data) 3151 { 3152 struct tc_mqprio_qopt_offload *mqprio = type_data; 3153 struct enetc_ndev_priv *priv = netdev_priv(ndev); 3154 struct tc_mqprio_qopt *qopt = &mqprio->qopt; 3155 struct enetc_hw *hw = &priv->si->hw; 3156 int num_stack_tx_queues = 0; 3157 struct enetc_bdr *tx_ring; 3158 u8 num_tc = qopt->num_tc; 3159 int offset, count; 3160 int err, tc, q; 3161 3162 if (!num_tc) { 3163 enetc_reset_tc_mqprio(ndev); 3164 return 0; 3165 } 3166 3167 err = netdev_set_num_tc(ndev, num_tc); 3168 if (err) 3169 return err; 3170 3171 for (tc = 0; tc < num_tc; tc++) { 3172 offset = qopt->offset[tc]; 3173 count = qopt->count[tc]; 3174 num_stack_tx_queues += count; 3175 3176 err = netdev_set_tc_queue(ndev, tc, count, offset); 3177 if (err) 3178 goto err_reset_tc; 3179 3180 for (q = offset; q < offset + count; q++) { 3181 tx_ring = priv->tx_ring[q]; 3182 /* The prio_tc_map is skb_tx_hash()'s way of selecting 3183 * between TX queues based on skb->priority. As such, 3184 * there's nothing to offload based on it. 3185 * Make the mqprio "traffic class" be the priority of 3186 * this ring group, and leave the Tx IPV to traffic 3187 * class mapping as its default mapping value of 1:1. 3188 */ 3189 tx_ring->prio = tc; 3190 enetc_set_bdr_prio(hw, tx_ring->index, tx_ring->prio); 3191 } 3192 } 3193 3194 err = netif_set_real_num_tx_queues(ndev, num_stack_tx_queues); 3195 if (err) 3196 goto err_reset_tc; 3197 3198 priv->min_num_stack_tx_queues = num_stack_tx_queues; 3199 3200 enetc_debug_tx_ring_prios(priv); 3201 3202 enetc_change_preemptible_tcs(priv, mqprio->preemptible_tcs); 3203 3204 return 0; 3205 3206 err_reset_tc: 3207 enetc_reset_tc_mqprio(ndev); 3208 return err; 3209 } 3210 EXPORT_SYMBOL_GPL(enetc_setup_tc_mqprio); 3211 3212 static int enetc_reconfigure_xdp_cb(struct enetc_ndev_priv *priv, void *ctx) 3213 { 3214 struct bpf_prog *old_prog, *prog = ctx; 3215 int num_stack_tx_queues; 3216 int err, i; 3217 3218 old_prog = xchg(&priv->xdp_prog, prog); 3219 3220 num_stack_tx_queues = enetc_num_stack_tx_queues(priv); 3221 err = netif_set_real_num_tx_queues(priv->ndev, num_stack_tx_queues); 3222 if (err) { 3223 xchg(&priv->xdp_prog, old_prog); 3224 return err; 3225 } 3226 3227 if (old_prog) 3228 bpf_prog_put(old_prog); 3229 3230 for (i = 0; i < priv->num_rx_rings; i++) { 3231 struct enetc_bdr *rx_ring = priv->rx_ring[i]; 3232 3233 rx_ring->xdp.prog = prog; 3234 3235 if (prog) 3236 rx_ring->buffer_offset = XDP_PACKET_HEADROOM; 3237 else 3238 rx_ring->buffer_offset = ENETC_RXB_PAD; 3239 } 3240 3241 return 0; 3242 } 3243 3244 static int enetc_setup_xdp_prog(struct net_device *ndev, struct bpf_prog *prog, 3245 struct netlink_ext_ack *extack) 3246 { 3247 int num_xdp_tx_queues = prog ? num_possible_cpus() : 0; 3248 struct enetc_ndev_priv *priv = netdev_priv(ndev); 3249 bool extended; 3250 3251 if (priv->min_num_stack_tx_queues + num_xdp_tx_queues > 3252 priv->num_tx_rings) { 3253 NL_SET_ERR_MSG_FMT_MOD(extack, 3254 "Reserving %d XDP TXQs leaves under %d for stack (total %d)", 3255 num_xdp_tx_queues, 3256 priv->min_num_stack_tx_queues, 3257 priv->num_tx_rings); 3258 return -EBUSY; 3259 } 3260 3261 extended = !!(priv->active_offloads & ENETC_F_RX_TSTAMP); 3262 3263 /* The buffer layout is changing, so we need to drain the old 3264 * RX buffers and seed new ones. 3265 */ 3266 return enetc_reconfigure(priv, extended, enetc_reconfigure_xdp_cb, prog); 3267 } 3268 3269 int enetc_setup_bpf(struct net_device *ndev, struct netdev_bpf *bpf) 3270 { 3271 switch (bpf->command) { 3272 case XDP_SETUP_PROG: 3273 return enetc_setup_xdp_prog(ndev, bpf->prog, bpf->extack); 3274 default: 3275 return -EINVAL; 3276 } 3277 3278 return 0; 3279 } 3280 EXPORT_SYMBOL_GPL(enetc_setup_bpf); 3281 3282 struct net_device_stats *enetc_get_stats(struct net_device *ndev) 3283 { 3284 struct enetc_ndev_priv *priv = netdev_priv(ndev); 3285 struct net_device_stats *stats = &ndev->stats; 3286 unsigned long packets = 0, bytes = 0; 3287 unsigned long tx_dropped = 0; 3288 int i; 3289 3290 for (i = 0; i < priv->num_rx_rings; i++) { 3291 packets += priv->rx_ring[i]->stats.packets; 3292 bytes += priv->rx_ring[i]->stats.bytes; 3293 } 3294 3295 stats->rx_packets = packets; 3296 stats->rx_bytes = bytes; 3297 bytes = 0; 3298 packets = 0; 3299 3300 for (i = 0; i < priv->num_tx_rings; i++) { 3301 packets += priv->tx_ring[i]->stats.packets; 3302 bytes += priv->tx_ring[i]->stats.bytes; 3303 tx_dropped += priv->tx_ring[i]->stats.win_drop; 3304 } 3305 3306 stats->tx_packets = packets; 3307 stats->tx_bytes = bytes; 3308 stats->tx_dropped = tx_dropped; 3309 3310 return stats; 3311 } 3312 EXPORT_SYMBOL_GPL(enetc_get_stats); 3313 3314 static void enetc_enable_rxvlan(struct net_device *ndev, bool en) 3315 { 3316 struct enetc_ndev_priv *priv = netdev_priv(ndev); 3317 struct enetc_hw *hw = &priv->si->hw; 3318 int i; 3319 3320 for (i = 0; i < priv->num_rx_rings; i++) 3321 enetc_bdr_enable_rxvlan(hw, i, en); 3322 } 3323 3324 static void enetc_enable_txvlan(struct net_device *ndev, bool en) 3325 { 3326 struct enetc_ndev_priv *priv = netdev_priv(ndev); 3327 struct enetc_hw *hw = &priv->si->hw; 3328 int i; 3329 3330 for (i = 0; i < priv->num_tx_rings; i++) 3331 enetc_bdr_enable_txvlan(hw, i, en); 3332 } 3333 3334 void enetc_set_features(struct net_device *ndev, netdev_features_t features) 3335 { 3336 netdev_features_t changed = ndev->features ^ features; 3337 3338 if (changed & NETIF_F_RXHASH) 3339 enetc_set_rss(ndev, !!(features & NETIF_F_RXHASH)); 3340 3341 if (changed & NETIF_F_HW_VLAN_CTAG_RX) 3342 enetc_enable_rxvlan(ndev, 3343 !!(features & NETIF_F_HW_VLAN_CTAG_RX)); 3344 3345 if (changed & NETIF_F_HW_VLAN_CTAG_TX) 3346 enetc_enable_txvlan(ndev, 3347 !!(features & NETIF_F_HW_VLAN_CTAG_TX)); 3348 } 3349 EXPORT_SYMBOL_GPL(enetc_set_features); 3350 3351 int enetc_hwtstamp_set(struct net_device *ndev, 3352 struct kernel_hwtstamp_config *config, 3353 struct netlink_ext_ack *extack) 3354 { 3355 struct enetc_ndev_priv *priv = netdev_priv(ndev); 3356 int err, new_offloads = priv->active_offloads; 3357 3358 if (!enetc_ptp_clock_is_enabled(priv->si)) 3359 return -EOPNOTSUPP; 3360 3361 switch (config->tx_type) { 3362 case HWTSTAMP_TX_OFF: 3363 new_offloads &= ~ENETC_F_TX_TSTAMP_MASK; 3364 break; 3365 case HWTSTAMP_TX_ON: 3366 new_offloads &= ~ENETC_F_TX_TSTAMP_MASK; 3367 new_offloads |= ENETC_F_TX_TSTAMP; 3368 break; 3369 case HWTSTAMP_TX_ONESTEP_SYNC: 3370 if (!enetc_si_is_pf(priv->si)) 3371 return -EOPNOTSUPP; 3372 3373 new_offloads &= ~ENETC_F_TX_TSTAMP_MASK; 3374 new_offloads |= ENETC_F_TX_ONESTEP_SYNC_TSTAMP; 3375 break; 3376 default: 3377 return -ERANGE; 3378 } 3379 3380 switch (config->rx_filter) { 3381 case HWTSTAMP_FILTER_NONE: 3382 new_offloads &= ~ENETC_F_RX_TSTAMP; 3383 break; 3384 default: 3385 new_offloads |= ENETC_F_RX_TSTAMP; 3386 config->rx_filter = HWTSTAMP_FILTER_ALL; 3387 } 3388 3389 if ((new_offloads ^ priv->active_offloads) & ENETC_F_RX_TSTAMP) { 3390 bool extended = !!(new_offloads & ENETC_F_RX_TSTAMP); 3391 3392 err = enetc_reconfigure(priv, extended, NULL, NULL); 3393 if (err) 3394 return err; 3395 } 3396 3397 priv->active_offloads = new_offloads; 3398 3399 return 0; 3400 } 3401 EXPORT_SYMBOL_GPL(enetc_hwtstamp_set); 3402 3403 int enetc_hwtstamp_get(struct net_device *ndev, 3404 struct kernel_hwtstamp_config *config) 3405 { 3406 struct enetc_ndev_priv *priv = netdev_priv(ndev); 3407 3408 if (!enetc_ptp_clock_is_enabled(priv->si)) 3409 return -EOPNOTSUPP; 3410 3411 if (priv->active_offloads & ENETC_F_TX_ONESTEP_SYNC_TSTAMP) 3412 config->tx_type = HWTSTAMP_TX_ONESTEP_SYNC; 3413 else if (priv->active_offloads & ENETC_F_TX_TSTAMP) 3414 config->tx_type = HWTSTAMP_TX_ON; 3415 else 3416 config->tx_type = HWTSTAMP_TX_OFF; 3417 3418 config->rx_filter = (priv->active_offloads & ENETC_F_RX_TSTAMP) ? 3419 HWTSTAMP_FILTER_ALL : HWTSTAMP_FILTER_NONE; 3420 3421 return 0; 3422 } 3423 EXPORT_SYMBOL_GPL(enetc_hwtstamp_get); 3424 3425 int enetc_ioctl(struct net_device *ndev, struct ifreq *rq, int cmd) 3426 { 3427 struct enetc_ndev_priv *priv = netdev_priv(ndev); 3428 3429 if (!priv->phylink) 3430 return -EOPNOTSUPP; 3431 3432 return phylink_mii_ioctl(priv->phylink, rq, cmd); 3433 } 3434 EXPORT_SYMBOL_GPL(enetc_ioctl); 3435 3436 static int enetc_int_vector_init(struct enetc_ndev_priv *priv, int i, 3437 int v_tx_rings) 3438 { 3439 struct enetc_int_vector *v; 3440 struct enetc_bdr *bdr; 3441 int j, err; 3442 3443 v = kzalloc(struct_size(v, tx_ring, v_tx_rings), GFP_KERNEL); 3444 if (!v) 3445 return -ENOMEM; 3446 3447 priv->int_vector[i] = v; 3448 bdr = &v->rx_ring; 3449 bdr->index = i; 3450 bdr->ndev = priv->ndev; 3451 bdr->dev = priv->dev; 3452 bdr->bd_count = priv->rx_bd_count; 3453 bdr->buffer_offset = ENETC_RXB_PAD; 3454 priv->rx_ring[i] = bdr; 3455 3456 err = __xdp_rxq_info_reg(&bdr->xdp.rxq, priv->ndev, i, 0, 3457 ENETC_RXB_DMA_SIZE_XDP); 3458 if (err) 3459 goto free_vector; 3460 3461 err = xdp_rxq_info_reg_mem_model(&bdr->xdp.rxq, MEM_TYPE_PAGE_SHARED, 3462 NULL); 3463 if (err) { 3464 xdp_rxq_info_unreg(&bdr->xdp.rxq); 3465 goto free_vector; 3466 } 3467 3468 /* init defaults for adaptive IC */ 3469 if (priv->ic_mode & ENETC_IC_RX_ADAPTIVE) { 3470 v->rx_ictt = 0x1; 3471 v->rx_dim_en = true; 3472 } 3473 3474 INIT_WORK(&v->rx_dim.work, enetc_rx_dim_work); 3475 netif_napi_add(priv->ndev, &v->napi, enetc_poll); 3476 v->count_tx_rings = v_tx_rings; 3477 3478 for (j = 0; j < v_tx_rings; j++) { 3479 int idx; 3480 3481 /* default tx ring mapping policy */ 3482 idx = priv->bdr_int_num * j + i; 3483 __set_bit(idx, &v->tx_rings_map); 3484 bdr = &v->tx_ring[j]; 3485 bdr->index = idx; 3486 bdr->ndev = priv->ndev; 3487 bdr->dev = priv->dev; 3488 bdr->bd_count = priv->tx_bd_count; 3489 priv->tx_ring[idx] = bdr; 3490 } 3491 3492 return 0; 3493 3494 free_vector: 3495 priv->rx_ring[i] = NULL; 3496 priv->int_vector[i] = NULL; 3497 kfree(v); 3498 3499 return err; 3500 } 3501 3502 static void enetc_int_vector_destroy(struct enetc_ndev_priv *priv, int i) 3503 { 3504 struct enetc_int_vector *v = priv->int_vector[i]; 3505 struct enetc_bdr *rx_ring = &v->rx_ring; 3506 int j, tx_ring_index; 3507 3508 xdp_rxq_info_unreg_mem_model(&rx_ring->xdp.rxq); 3509 xdp_rxq_info_unreg(&rx_ring->xdp.rxq); 3510 netif_napi_del(&v->napi); 3511 cancel_work_sync(&v->rx_dim.work); 3512 3513 for (j = 0; j < v->count_tx_rings; j++) { 3514 tx_ring_index = priv->bdr_int_num * j + i; 3515 priv->tx_ring[tx_ring_index] = NULL; 3516 } 3517 3518 priv->rx_ring[i] = NULL; 3519 priv->int_vector[i] = NULL; 3520 kfree(v); 3521 } 3522 3523 int enetc_alloc_msix(struct enetc_ndev_priv *priv) 3524 { 3525 struct pci_dev *pdev = priv->si->pdev; 3526 int v_tx_rings, v_remainder; 3527 int num_stack_tx_queues; 3528 int first_xdp_tx_ring; 3529 int i, n, err, nvec; 3530 3531 nvec = ENETC_BDR_INT_BASE_IDX + priv->bdr_int_num; 3532 /* allocate MSIX for both messaging and Rx/Tx interrupts */ 3533 n = pci_alloc_irq_vectors(pdev, nvec, nvec, PCI_IRQ_MSIX); 3534 3535 if (n < 0) 3536 return n; 3537 3538 if (n != nvec) 3539 return -EPERM; 3540 3541 /* # of tx rings per int vector */ 3542 v_tx_rings = priv->num_tx_rings / priv->bdr_int_num; 3543 v_remainder = priv->num_tx_rings % priv->bdr_int_num; 3544 3545 for (i = 0; i < priv->bdr_int_num; i++) { 3546 /* Distribute the remaining TX rings to the first v_remainder 3547 * interrupt vectors 3548 */ 3549 int num_tx_rings = i < v_remainder ? v_tx_rings + 1 : v_tx_rings; 3550 3551 err = enetc_int_vector_init(priv, i, num_tx_rings); 3552 if (err) 3553 goto fail; 3554 } 3555 3556 num_stack_tx_queues = enetc_num_stack_tx_queues(priv); 3557 3558 err = netif_set_real_num_tx_queues(priv->ndev, num_stack_tx_queues); 3559 if (err) 3560 goto fail; 3561 3562 err = netif_set_real_num_rx_queues(priv->ndev, priv->num_rx_rings); 3563 if (err) 3564 goto fail; 3565 3566 priv->min_num_stack_tx_queues = num_possible_cpus(); 3567 first_xdp_tx_ring = priv->num_tx_rings - num_possible_cpus(); 3568 priv->xdp_tx_ring = &priv->tx_ring[first_xdp_tx_ring]; 3569 3570 return 0; 3571 3572 fail: 3573 while (i--) 3574 enetc_int_vector_destroy(priv, i); 3575 3576 pci_free_irq_vectors(pdev); 3577 3578 return err; 3579 } 3580 EXPORT_SYMBOL_GPL(enetc_alloc_msix); 3581 3582 void enetc_free_msix(struct enetc_ndev_priv *priv) 3583 { 3584 int i; 3585 3586 for (i = 0; i < priv->bdr_int_num; i++) 3587 enetc_int_vector_destroy(priv, i); 3588 3589 /* disable all MSIX for this device */ 3590 pci_free_irq_vectors(priv->si->pdev); 3591 } 3592 EXPORT_SYMBOL_GPL(enetc_free_msix); 3593 3594 static void enetc_kfree_si(struct enetc_si *si) 3595 { 3596 char *p = (char *)si - si->pad; 3597 3598 kfree(p); 3599 } 3600 3601 static void enetc_detect_errata(struct enetc_si *si) 3602 { 3603 if (si->pdev->revision == ENETC_REV1) 3604 si->errata = ENETC_ERR_VLAN_ISOL | ENETC_ERR_UCMCSWP; 3605 } 3606 3607 int enetc_pci_probe(struct pci_dev *pdev, const char *name, int sizeof_priv) 3608 { 3609 struct enetc_si *si, *p; 3610 struct enetc_hw *hw; 3611 size_t alloc_size; 3612 int err, len; 3613 3614 pcie_flr(pdev); 3615 err = pci_enable_device_mem(pdev); 3616 if (err) 3617 return dev_err_probe(&pdev->dev, err, "device enable failed\n"); 3618 3619 /* set up for high or low dma */ 3620 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 3621 if (err) { 3622 dev_err(&pdev->dev, "DMA configuration failed: 0x%x\n", err); 3623 goto err_dma; 3624 } 3625 3626 err = pci_request_mem_regions(pdev, name); 3627 if (err) { 3628 dev_err(&pdev->dev, "pci_request_regions failed err=%d\n", err); 3629 goto err_pci_mem_reg; 3630 } 3631 3632 pci_set_master(pdev); 3633 3634 alloc_size = sizeof(struct enetc_si); 3635 if (sizeof_priv) { 3636 /* align priv to 32B */ 3637 alloc_size = ALIGN(alloc_size, ENETC_SI_ALIGN); 3638 alloc_size += sizeof_priv; 3639 } 3640 /* force 32B alignment for enetc_si */ 3641 alloc_size += ENETC_SI_ALIGN - 1; 3642 3643 p = kzalloc(alloc_size, GFP_KERNEL); 3644 if (!p) { 3645 err = -ENOMEM; 3646 goto err_alloc_si; 3647 } 3648 3649 si = PTR_ALIGN(p, ENETC_SI_ALIGN); 3650 si->pad = (char *)si - (char *)p; 3651 3652 pci_set_drvdata(pdev, si); 3653 si->pdev = pdev; 3654 hw = &si->hw; 3655 3656 len = pci_resource_len(pdev, ENETC_BAR_REGS); 3657 hw->reg = ioremap(pci_resource_start(pdev, ENETC_BAR_REGS), len); 3658 if (!hw->reg) { 3659 err = -ENXIO; 3660 dev_err(&pdev->dev, "ioremap() failed\n"); 3661 goto err_ioremap; 3662 } 3663 if (len > ENETC_PORT_BASE) 3664 hw->port = hw->reg + ENETC_PORT_BASE; 3665 if (len > ENETC_GLOBAL_BASE) 3666 hw->global = hw->reg + ENETC_GLOBAL_BASE; 3667 3668 enetc_detect_errata(si); 3669 3670 return 0; 3671 3672 err_ioremap: 3673 enetc_kfree_si(si); 3674 err_alloc_si: 3675 pci_release_mem_regions(pdev); 3676 err_pci_mem_reg: 3677 err_dma: 3678 pci_disable_device(pdev); 3679 3680 return err; 3681 } 3682 EXPORT_SYMBOL_GPL(enetc_pci_probe); 3683 3684 void enetc_pci_remove(struct pci_dev *pdev) 3685 { 3686 struct enetc_si *si = pci_get_drvdata(pdev); 3687 struct enetc_hw *hw = &si->hw; 3688 3689 iounmap(hw->reg); 3690 enetc_kfree_si(si); 3691 pci_release_mem_regions(pdev); 3692 pci_disable_device(pdev); 3693 } 3694 EXPORT_SYMBOL_GPL(enetc_pci_remove); 3695 3696 static const struct enetc_drvdata enetc_pf_data = { 3697 .sysclk_freq = ENETC_CLK_400M, 3698 .pmac_offset = ENETC_PMAC_OFFSET, 3699 .max_frags = ENETC_MAX_SKB_FRAGS, 3700 .eth_ops = &enetc_pf_ethtool_ops, 3701 }; 3702 3703 static const struct enetc_drvdata enetc4_pf_data = { 3704 .sysclk_freq = ENETC_CLK_333M, 3705 .tx_csum = true, 3706 .max_frags = ENETC4_MAX_SKB_FRAGS, 3707 .pmac_offset = ENETC4_PMAC_OFFSET, 3708 .eth_ops = &enetc4_pf_ethtool_ops, 3709 }; 3710 3711 static const struct enetc_drvdata enetc_vf_data = { 3712 .sysclk_freq = ENETC_CLK_400M, 3713 .max_frags = ENETC_MAX_SKB_FRAGS, 3714 .eth_ops = &enetc_vf_ethtool_ops, 3715 }; 3716 3717 static const struct enetc_platform_info enetc_info[] = { 3718 { .revision = ENETC_REV_1_0, 3719 .dev_id = ENETC_DEV_ID_PF, 3720 .data = &enetc_pf_data, 3721 }, 3722 { .revision = ENETC_REV_4_1, 3723 .dev_id = NXP_ENETC_PF_DEV_ID, 3724 .data = &enetc4_pf_data, 3725 }, 3726 { .revision = ENETC_REV_1_0, 3727 .dev_id = ENETC_DEV_ID_VF, 3728 .data = &enetc_vf_data, 3729 }, 3730 }; 3731 3732 int enetc_get_driver_data(struct enetc_si *si) 3733 { 3734 u16 dev_id = si->pdev->device; 3735 int i; 3736 3737 for (i = 0; i < ARRAY_SIZE(enetc_info); i++) { 3738 if (si->revision == enetc_info[i].revision && 3739 dev_id == enetc_info[i].dev_id) { 3740 si->drvdata = enetc_info[i].data; 3741 3742 return 0; 3743 } 3744 } 3745 3746 return -ERANGE; 3747 } 3748 EXPORT_SYMBOL_GPL(enetc_get_driver_data); 3749 3750 MODULE_DESCRIPTION("NXP ENETC Ethernet driver"); 3751 MODULE_LICENSE("Dual BSD/GPL"); 3752