1 // SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) 2 /* Copyright 2017-2019 NXP */ 3 4 #include "enetc.h" 5 #include <linux/bpf_trace.h> 6 #include <linux/clk.h> 7 #include <linux/tcp.h> 8 #include <linux/udp.h> 9 #include <linux/vmalloc.h> 10 #include <linux/ptp_classify.h> 11 #include <net/ip6_checksum.h> 12 #include <net/pkt_sched.h> 13 #include <net/tso.h> 14 15 u32 enetc_port_mac_rd(struct enetc_si *si, u32 reg) 16 { 17 /* ENETC with pseudo MAC does not have Ethernet MAC 18 * port registers. 19 */ 20 if (enetc_is_pseudo_mac(si)) 21 return 0; 22 23 return enetc_port_rd(&si->hw, reg); 24 } 25 EXPORT_SYMBOL_GPL(enetc_port_mac_rd); 26 27 void enetc_port_mac_wr(struct enetc_si *si, u32 reg, u32 val) 28 { 29 if (enetc_is_pseudo_mac(si)) 30 return; 31 32 enetc_port_wr(&si->hw, reg, val); 33 if (si->hw_features & ENETC_SI_F_QBU) 34 enetc_port_wr(&si->hw, reg + si->drvdata->pmac_offset, val); 35 } 36 EXPORT_SYMBOL_GPL(enetc_port_mac_wr); 37 38 static void enetc_change_preemptible_tcs(struct enetc_ndev_priv *priv, 39 u8 preemptible_tcs) 40 { 41 if (!(priv->si->hw_features & ENETC_SI_F_QBU)) 42 return; 43 44 priv->preemptible_tcs = preemptible_tcs; 45 enetc_mm_commit_preemptible_tcs(priv); 46 } 47 48 static int enetc_mac_addr_hash_idx(const u8 *addr) 49 { 50 u64 fold = __swab64(ether_addr_to_u64(addr)) >> 16; 51 u64 mask = 0; 52 int res = 0; 53 int i; 54 55 for (i = 0; i < 8; i++) 56 mask |= BIT_ULL(i * 6); 57 58 for (i = 0; i < 6; i++) 59 res |= (hweight64(fold & (mask << i)) & 0x1) << i; 60 61 return res; 62 } 63 64 void enetc_add_mac_addr_ht_filter(struct enetc_mac_filter *filter, 65 const unsigned char *addr) 66 { 67 int idx = enetc_mac_addr_hash_idx(addr); 68 69 /* add hash table entry */ 70 __set_bit(idx, filter->mac_hash_table); 71 filter->mac_addr_cnt++; 72 } 73 EXPORT_SYMBOL_GPL(enetc_add_mac_addr_ht_filter); 74 75 void enetc_reset_mac_addr_filter(struct enetc_mac_filter *filter) 76 { 77 filter->mac_addr_cnt = 0; 78 79 bitmap_zero(filter->mac_hash_table, 80 ENETC_MADDR_HASH_TBL_SZ); 81 } 82 EXPORT_SYMBOL_GPL(enetc_reset_mac_addr_filter); 83 84 void enetc_set_congestion_mode(struct enetc_ndev_priv *priv, bool enable) 85 { 86 struct enetc_si *si = priv->si; 87 struct enetc_hw *hw = &si->hw; 88 89 spin_lock(&si->gen_lock); 90 91 assign_bit(ENETC_RXBDR_CM, &priv->flags, enable); 92 93 for (int i = 0; i < priv->num_rx_rings; i++) { 94 u32 old_rbmr = enetc_rxbdr_rd(hw, i, ENETC_RBMR); 95 u32 rbmr; 96 97 rbmr = u32_replace_bits(old_rbmr, enable, ENETC_RBMR_CM); 98 if (rbmr == old_rbmr) 99 continue; 100 101 enetc_rxbdr_wr(hw, i, ENETC_RBMR, rbmr); 102 } 103 104 spin_unlock(&si->gen_lock); 105 } 106 EXPORT_SYMBOL_GPL(enetc_set_congestion_mode); 107 108 static int enetc_num_stack_tx_queues(struct enetc_ndev_priv *priv) 109 { 110 int num_tx_rings = priv->num_tx_rings; 111 112 if (priv->xdp_prog) 113 return num_tx_rings - num_possible_cpus(); 114 115 return num_tx_rings; 116 } 117 118 static struct enetc_bdr *enetc_rx_ring_from_xdp_tx_ring(struct enetc_ndev_priv *priv, 119 struct enetc_bdr *tx_ring) 120 { 121 int index = &priv->tx_ring[tx_ring->index] - priv->xdp_tx_ring; 122 123 return priv->rx_ring[index]; 124 } 125 126 static struct sk_buff *enetc_tx_swbd_get_skb(struct enetc_tx_swbd *tx_swbd) 127 { 128 if (tx_swbd->is_xdp_tx || tx_swbd->is_xdp_redirect) 129 return NULL; 130 131 return tx_swbd->skb; 132 } 133 134 static struct xdp_frame * 135 enetc_tx_swbd_get_xdp_frame(struct enetc_tx_swbd *tx_swbd) 136 { 137 if (tx_swbd->is_xdp_redirect) 138 return tx_swbd->xdp_frame; 139 140 return NULL; 141 } 142 143 static void enetc_unmap_tx_buff(struct enetc_bdr *tx_ring, 144 struct enetc_tx_swbd *tx_swbd) 145 { 146 /* For XDP_TX, pages come from RX, whereas for the other contexts where 147 * we have is_dma_page_set, those come from skb_frag_dma_map. We need 148 * to match the DMA mapping length, so we need to differentiate those. 149 */ 150 if (tx_swbd->is_dma_page) 151 dma_unmap_page(tx_ring->dev, tx_swbd->dma, 152 tx_swbd->is_xdp_tx ? PAGE_SIZE : tx_swbd->len, 153 tx_swbd->dir); 154 else 155 dma_unmap_single(tx_ring->dev, tx_swbd->dma, 156 tx_swbd->len, tx_swbd->dir); 157 tx_swbd->dma = 0; 158 } 159 160 static void enetc_free_tx_frame(struct enetc_bdr *tx_ring, 161 struct enetc_tx_swbd *tx_swbd) 162 { 163 struct xdp_frame *xdp_frame = enetc_tx_swbd_get_xdp_frame(tx_swbd); 164 struct sk_buff *skb = enetc_tx_swbd_get_skb(tx_swbd); 165 166 if (tx_swbd->dma) 167 enetc_unmap_tx_buff(tx_ring, tx_swbd); 168 169 if (xdp_frame) { 170 xdp_return_frame(tx_swbd->xdp_frame); 171 tx_swbd->xdp_frame = NULL; 172 } else if (skb) { 173 dev_kfree_skb_any(skb); 174 tx_swbd->skb = NULL; 175 } 176 } 177 178 /* Let H/W know BD ring has been updated */ 179 static void enetc_update_tx_ring_tail(struct enetc_bdr *tx_ring) 180 { 181 /* includes wmb() */ 182 enetc_wr_reg_hot(tx_ring->tpir, tx_ring->next_to_use); 183 } 184 185 static int enetc_ptp_parse(struct sk_buff *skb, u8 *udp, 186 u8 *msgtype, u8 *twostep, 187 u16 *correction_offset, u16 *body_offset) 188 { 189 unsigned int ptp_class; 190 struct ptp_header *hdr; 191 unsigned int type; 192 u8 *base; 193 194 ptp_class = ptp_classify_raw(skb); 195 if (ptp_class == PTP_CLASS_NONE) 196 return -EINVAL; 197 198 hdr = ptp_parse_header(skb, ptp_class); 199 if (!hdr) 200 return -EINVAL; 201 202 type = ptp_class & PTP_CLASS_PMASK; 203 if (type == PTP_CLASS_IPV4 || type == PTP_CLASS_IPV6) 204 *udp = 1; 205 else 206 *udp = 0; 207 208 *msgtype = ptp_get_msgtype(hdr, ptp_class); 209 *twostep = hdr->flag_field[0] & 0x2; 210 211 base = skb_mac_header(skb); 212 *correction_offset = (u8 *)&hdr->correction - base; 213 *body_offset = (u8 *)hdr + sizeof(struct ptp_header) - base; 214 215 return 0; 216 } 217 218 static bool enetc_tx_csum_offload_check(struct sk_buff *skb) 219 { 220 switch (skb->csum_offset) { 221 case offsetof(struct tcphdr, check): 222 case offsetof(struct udphdr, check): 223 return true; 224 default: 225 return false; 226 } 227 } 228 229 static bool enetc_skb_is_ipv6(struct sk_buff *skb) 230 { 231 return vlan_get_protocol(skb) == htons(ETH_P_IPV6); 232 } 233 234 static bool enetc_skb_is_tcp(struct sk_buff *skb) 235 { 236 return skb->csum_offset == offsetof(struct tcphdr, check); 237 } 238 239 /** 240 * enetc_unwind_tx_frame() - Unwind the DMA mappings of a multi-buffer Tx frame 241 * @tx_ring: Pointer to the Tx ring on which the buffer descriptors are located 242 * @count: Number of Tx buffer descriptors which need to be unmapped 243 * @i: Index of the last successfully mapped Tx buffer descriptor 244 */ 245 static void enetc_unwind_tx_frame(struct enetc_bdr *tx_ring, int count, int i) 246 { 247 while (count--) { 248 struct enetc_tx_swbd *tx_swbd = &tx_ring->tx_swbd[i]; 249 250 enetc_free_tx_frame(tx_ring, tx_swbd); 251 if (i == 0) 252 i = tx_ring->bd_count; 253 i--; 254 } 255 } 256 257 static void enetc_set_one_step_ts(struct enetc_si *si, bool udp, int offset) 258 { 259 u32 val = ENETC_PM0_SINGLE_STEP_EN; 260 261 val |= ENETC_SET_SINGLE_STEP_OFFSET(offset); 262 if (udp) 263 val |= ENETC_PM0_SINGLE_STEP_CH; 264 265 /* The "Correction" field of a packet is updated based on the 266 * current time and the timestamp provided 267 */ 268 enetc_port_mac_wr(si, ENETC_PM0_SINGLE_STEP, val); 269 } 270 271 static void enetc4_set_one_step_ts(struct enetc_si *si, bool udp, int offset) 272 { 273 u32 val = PM_SINGLE_STEP_EN; 274 275 val |= PM_SINGLE_STEP_OFFSET_SET(offset); 276 if (udp) 277 val |= PM_SINGLE_STEP_CH; 278 279 enetc_port_mac_wr(si, ENETC4_PM_SINGLE_STEP(0), val); 280 } 281 282 static u32 enetc_update_ptp_sync_msg(struct enetc_ndev_priv *priv, 283 struct sk_buff *skb, bool csum_offload) 284 { 285 struct enetc_skb_cb *enetc_cb = ENETC_SKB_CB(skb); 286 u16 tstamp_off = enetc_cb->origin_tstamp_off; 287 u16 corr_off = enetc_cb->correction_off; 288 struct enetc_si *si = priv->si; 289 struct enetc_hw *hw = &si->hw; 290 __be32 new_sec_l, new_nsec; 291 __be16 new_sec_h; 292 u32 lo, hi, nsec; 293 u8 *data; 294 u64 sec; 295 296 lo = enetc_rd_hot(hw, ENETC_SICTR0); 297 hi = enetc_rd_hot(hw, ENETC_SICTR1); 298 sec = (u64)hi << 32 | lo; 299 nsec = do_div(sec, 1000000000); 300 301 /* Update originTimestamp field of Sync packet 302 * - 48 bits seconds field 303 * - 32 bits nanseconds field 304 * 305 * In addition, if csum_offload is false, the UDP checksum needs 306 * to be updated by software after updating originTimestamp field, 307 * otherwise the hardware will calculate the wrong checksum when 308 * updating the correction field and update it to the packet. 309 */ 310 311 data = skb_mac_header(skb); 312 new_sec_h = htons((sec >> 32) & 0xffff); 313 new_sec_l = htonl(sec & 0xffffffff); 314 new_nsec = htonl(nsec); 315 if (enetc_cb->udp && !csum_offload) { 316 struct udphdr *uh = udp_hdr(skb); 317 __be32 old_sec_l, old_nsec; 318 __be16 old_sec_h; 319 320 old_sec_h = *(__be16 *)(data + tstamp_off); 321 inet_proto_csum_replace2(&uh->check, skb, old_sec_h, 322 new_sec_h, false); 323 324 old_sec_l = *(__be32 *)(data + tstamp_off + 2); 325 inet_proto_csum_replace4(&uh->check, skb, old_sec_l, 326 new_sec_l, false); 327 328 old_nsec = *(__be32 *)(data + tstamp_off + 6); 329 inet_proto_csum_replace4(&uh->check, skb, old_nsec, 330 new_nsec, false); 331 } 332 333 *(__be16 *)(data + tstamp_off) = new_sec_h; 334 *(__be32 *)(data + tstamp_off + 2) = new_sec_l; 335 *(__be32 *)(data + tstamp_off + 6) = new_nsec; 336 337 /* Configure single-step register */ 338 if (is_enetc_rev1(si)) 339 enetc_set_one_step_ts(si, enetc_cb->udp, corr_off); 340 else 341 enetc4_set_one_step_ts(si, enetc_cb->udp, corr_off); 342 343 return lo & ENETC_TXBD_TSTAMP; 344 } 345 346 static int enetc_map_tx_buffs(struct enetc_bdr *tx_ring, struct sk_buff *skb) 347 { 348 bool do_vlan, do_onestep_tstamp = false, do_twostep_tstamp = false; 349 struct enetc_ndev_priv *priv = netdev_priv(tx_ring->ndev); 350 struct enetc_skb_cb *enetc_cb = ENETC_SKB_CB(skb); 351 struct enetc_tx_swbd *tx_swbd; 352 int len = skb_headlen(skb); 353 union enetc_tx_bd temp_bd; 354 bool csum_offload = false; 355 union enetc_tx_bd *txbd; 356 int i, count = 0; 357 skb_frag_t *frag; 358 unsigned int f; 359 dma_addr_t dma; 360 u8 flags = 0; 361 u32 tstamp; 362 363 enetc_clear_tx_bd(&temp_bd); 364 if (skb->ip_summed == CHECKSUM_PARTIAL) { 365 /* Can not support TSD and checksum offload at the same time */ 366 if (priv->active_offloads & ENETC_F_TXCSUM && 367 enetc_tx_csum_offload_check(skb) && !tx_ring->tsd_enable) { 368 temp_bd.l3_aux0 = FIELD_PREP(ENETC_TX_BD_L3_START, 369 skb_network_offset(skb)); 370 temp_bd.l3_aux1 = FIELD_PREP(ENETC_TX_BD_L3_HDR_LEN, 371 skb_network_header_len(skb) / 4); 372 temp_bd.l3_aux1 |= FIELD_PREP(ENETC_TX_BD_L3T, 373 enetc_skb_is_ipv6(skb)); 374 if (enetc_skb_is_tcp(skb)) 375 temp_bd.l4_aux = FIELD_PREP(ENETC_TX_BD_L4T, 376 ENETC_TXBD_L4T_TCP); 377 else 378 temp_bd.l4_aux = FIELD_PREP(ENETC_TX_BD_L4T, 379 ENETC_TXBD_L4T_UDP); 380 flags |= ENETC_TXBD_FLAGS_CSUM_LSO | ENETC_TXBD_FLAGS_L4CS; 381 csum_offload = true; 382 } else if (skb_checksum_help(skb)) { 383 return 0; 384 } 385 } 386 387 if (enetc_cb->flag & ENETC_F_TX_ONESTEP_SYNC_TSTAMP) { 388 do_onestep_tstamp = true; 389 tstamp = enetc_update_ptp_sync_msg(priv, skb, csum_offload); 390 } else if (enetc_cb->flag & ENETC_F_TX_TSTAMP) { 391 do_twostep_tstamp = true; 392 } 393 394 i = tx_ring->next_to_use; 395 txbd = ENETC_TXBD(*tx_ring, i); 396 prefetchw(txbd); 397 398 dma = dma_map_single(tx_ring->dev, skb->data, len, DMA_TO_DEVICE); 399 if (unlikely(dma_mapping_error(tx_ring->dev, dma))) 400 goto dma_err; 401 402 temp_bd.addr = cpu_to_le64(dma); 403 temp_bd.buf_len = cpu_to_le16(len); 404 405 tx_swbd = &tx_ring->tx_swbd[i]; 406 tx_swbd->dma = dma; 407 tx_swbd->len = len; 408 tx_swbd->is_dma_page = 0; 409 tx_swbd->dir = DMA_TO_DEVICE; 410 count++; 411 412 do_vlan = skb_vlan_tag_present(skb); 413 tx_swbd->do_twostep_tstamp = do_twostep_tstamp; 414 tx_swbd->qbv_en = !!(priv->active_offloads & ENETC_F_QBV); 415 tx_swbd->check_wb = tx_swbd->do_twostep_tstamp || tx_swbd->qbv_en; 416 417 if (do_vlan || do_onestep_tstamp || do_twostep_tstamp) 418 flags |= ENETC_TXBD_FLAGS_EX; 419 420 if (tx_ring->tsd_enable) 421 flags |= ENETC_TXBD_FLAGS_TSE | ENETC_TXBD_FLAGS_TXSTART; 422 423 /* first BD needs frm_len and offload flags set */ 424 temp_bd.frm_len = cpu_to_le16(skb->len); 425 temp_bd.flags = flags; 426 427 if (flags & ENETC_TXBD_FLAGS_TSE) 428 temp_bd.txstart = enetc_txbd_set_tx_start(skb->skb_mstamp_ns, 429 flags); 430 431 if (flags & ENETC_TXBD_FLAGS_EX) { 432 u8 e_flags = 0; 433 *txbd = temp_bd; 434 enetc_clear_tx_bd(&temp_bd); 435 436 /* add extension BD for VLAN and/or timestamping */ 437 flags = 0; 438 tx_swbd++; 439 txbd++; 440 i++; 441 if (unlikely(i == tx_ring->bd_count)) { 442 i = 0; 443 tx_swbd = tx_ring->tx_swbd; 444 txbd = ENETC_TXBD(*tx_ring, 0); 445 } 446 prefetchw(txbd); 447 448 if (do_vlan) { 449 temp_bd.ext.vid = cpu_to_le16(skb_vlan_tag_get(skb)); 450 temp_bd.ext.tpid = 0; /* < C-TAG */ 451 e_flags |= ENETC_TXBD_E_FLAGS_VLAN_INS; 452 } 453 454 if (do_onestep_tstamp) { 455 /* Configure extension BD */ 456 temp_bd.ext.tstamp = cpu_to_le32(tstamp); 457 e_flags |= ENETC_TXBD_E_FLAGS_ONE_STEP_PTP; 458 } else if (do_twostep_tstamp) { 459 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; 460 e_flags |= ENETC_TXBD_E_FLAGS_TWO_STEP_PTP; 461 } 462 463 temp_bd.ext.e_flags = e_flags; 464 count++; 465 } 466 467 frag = &skb_shinfo(skb)->frags[0]; 468 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++, frag++) { 469 len = skb_frag_size(frag); 470 dma = skb_frag_dma_map(tx_ring->dev, frag, 0, len, 471 DMA_TO_DEVICE); 472 if (dma_mapping_error(tx_ring->dev, dma)) 473 goto dma_err; 474 475 *txbd = temp_bd; 476 enetc_clear_tx_bd(&temp_bd); 477 478 flags = 0; 479 tx_swbd++; 480 txbd++; 481 i++; 482 if (unlikely(i == tx_ring->bd_count)) { 483 i = 0; 484 tx_swbd = tx_ring->tx_swbd; 485 txbd = ENETC_TXBD(*tx_ring, 0); 486 } 487 prefetchw(txbd); 488 489 temp_bd.addr = cpu_to_le64(dma); 490 temp_bd.buf_len = cpu_to_le16(len); 491 492 tx_swbd->dma = dma; 493 tx_swbd->len = len; 494 tx_swbd->is_dma_page = 1; 495 tx_swbd->dir = DMA_TO_DEVICE; 496 count++; 497 } 498 499 /* last BD needs 'F' bit set */ 500 flags |= ENETC_TXBD_FLAGS_F; 501 temp_bd.flags = flags; 502 *txbd = temp_bd; 503 504 tx_ring->tx_swbd[i].is_eof = true; 505 tx_ring->tx_swbd[i].skb = skb; 506 507 enetc_bdr_idx_inc(tx_ring, &i); 508 tx_ring->next_to_use = i; 509 510 skb_tx_timestamp(skb); 511 512 enetc_update_tx_ring_tail(tx_ring); 513 514 return count; 515 516 dma_err: 517 dev_err(tx_ring->dev, "DMA map error"); 518 519 enetc_unwind_tx_frame(tx_ring, count, i); 520 521 return 0; 522 } 523 524 static int enetc_map_tx_tso_hdr(struct enetc_bdr *tx_ring, struct sk_buff *skb, 525 struct enetc_tx_swbd *tx_swbd, 526 union enetc_tx_bd *txbd, int *i, int hdr_len, 527 int data_len) 528 { 529 union enetc_tx_bd txbd_tmp; 530 u8 flags = 0, e_flags = 0; 531 dma_addr_t addr; 532 int count = 1; 533 534 enetc_clear_tx_bd(&txbd_tmp); 535 addr = tx_ring->tso_headers_dma + *i * TSO_HEADER_SIZE; 536 537 if (skb_vlan_tag_present(skb)) 538 flags |= ENETC_TXBD_FLAGS_EX; 539 540 txbd_tmp.addr = cpu_to_le64(addr); 541 txbd_tmp.buf_len = cpu_to_le16(hdr_len); 542 543 /* first BD needs frm_len and offload flags set */ 544 txbd_tmp.frm_len = cpu_to_le16(hdr_len + data_len); 545 txbd_tmp.flags = flags; 546 547 /* For the TSO header we do not set the dma address since we do not 548 * want it unmapped when we do cleanup. We still set len so that we 549 * count the bytes sent. 550 */ 551 tx_swbd->len = hdr_len; 552 tx_swbd->do_twostep_tstamp = false; 553 tx_swbd->check_wb = false; 554 555 /* Actually write the header in the BD */ 556 *txbd = txbd_tmp; 557 558 /* Add extension BD for VLAN */ 559 if (flags & ENETC_TXBD_FLAGS_EX) { 560 /* Get the next BD */ 561 enetc_bdr_idx_inc(tx_ring, i); 562 txbd = ENETC_TXBD(*tx_ring, *i); 563 tx_swbd = &tx_ring->tx_swbd[*i]; 564 prefetchw(txbd); 565 566 /* Setup the VLAN fields */ 567 enetc_clear_tx_bd(&txbd_tmp); 568 txbd_tmp.ext.vid = cpu_to_le16(skb_vlan_tag_get(skb)); 569 txbd_tmp.ext.tpid = 0; /* < C-TAG */ 570 e_flags |= ENETC_TXBD_E_FLAGS_VLAN_INS; 571 572 /* Write the BD */ 573 txbd_tmp.ext.e_flags = e_flags; 574 *txbd = txbd_tmp; 575 count++; 576 } 577 578 return count; 579 } 580 581 static int enetc_map_tx_tso_data(struct enetc_bdr *tx_ring, struct sk_buff *skb, 582 struct enetc_tx_swbd *tx_swbd, 583 union enetc_tx_bd *txbd, char *data, 584 int size, bool last_bd) 585 { 586 union enetc_tx_bd txbd_tmp; 587 dma_addr_t addr; 588 u8 flags = 0; 589 590 enetc_clear_tx_bd(&txbd_tmp); 591 592 addr = dma_map_single(tx_ring->dev, data, size, DMA_TO_DEVICE); 593 if (unlikely(dma_mapping_error(tx_ring->dev, addr))) { 594 netdev_err(tx_ring->ndev, "DMA map error\n"); 595 return -ENOMEM; 596 } 597 598 if (last_bd) { 599 flags |= ENETC_TXBD_FLAGS_F; 600 tx_swbd->is_eof = 1; 601 } 602 603 txbd_tmp.addr = cpu_to_le64(addr); 604 txbd_tmp.buf_len = cpu_to_le16(size); 605 txbd_tmp.flags = flags; 606 607 tx_swbd->dma = addr; 608 tx_swbd->len = size; 609 tx_swbd->dir = DMA_TO_DEVICE; 610 611 *txbd = txbd_tmp; 612 613 return 0; 614 } 615 616 static __wsum enetc_tso_hdr_csum(struct tso_t *tso, struct sk_buff *skb, 617 char *hdr, int hdr_len, int *l4_hdr_len) 618 { 619 char *l4_hdr = hdr + skb_transport_offset(skb); 620 int mac_hdr_len = skb_network_offset(skb); 621 622 if (tso->tlen != sizeof(struct udphdr)) { 623 struct tcphdr *tcph = (struct tcphdr *)(l4_hdr); 624 625 tcph->check = 0; 626 } else { 627 struct udphdr *udph = (struct udphdr *)(l4_hdr); 628 629 udph->check = 0; 630 } 631 632 /* Compute the IP checksum. This is necessary since tso_build_hdr() 633 * already incremented the IP ID field. 634 */ 635 if (!tso->ipv6) { 636 struct iphdr *iph = (void *)(hdr + mac_hdr_len); 637 638 iph->check = 0; 639 iph->check = ip_fast_csum((unsigned char *)iph, iph->ihl); 640 } 641 642 /* Compute the checksum over the L4 header. */ 643 *l4_hdr_len = hdr_len - skb_transport_offset(skb); 644 return csum_partial(l4_hdr, *l4_hdr_len, 0); 645 } 646 647 static void enetc_tso_complete_csum(struct enetc_bdr *tx_ring, struct tso_t *tso, 648 struct sk_buff *skb, char *hdr, int len, 649 __wsum sum) 650 { 651 char *l4_hdr = hdr + skb_transport_offset(skb); 652 __sum16 csum_final; 653 654 /* Complete the L4 checksum by appending the pseudo-header to the 655 * already computed checksum. 656 */ 657 if (!tso->ipv6) 658 csum_final = csum_tcpudp_magic(ip_hdr(skb)->saddr, 659 ip_hdr(skb)->daddr, 660 len, ip_hdr(skb)->protocol, sum); 661 else 662 csum_final = csum_ipv6_magic(&ipv6_hdr(skb)->saddr, 663 &ipv6_hdr(skb)->daddr, 664 len, ipv6_hdr(skb)->nexthdr, sum); 665 666 if (tso->tlen != sizeof(struct udphdr)) { 667 struct tcphdr *tcph = (struct tcphdr *)(l4_hdr); 668 669 tcph->check = csum_final; 670 } else { 671 struct udphdr *udph = (struct udphdr *)(l4_hdr); 672 673 udph->check = csum_final; 674 } 675 } 676 677 static int enetc_lso_count_descs(const struct sk_buff *skb) 678 { 679 /* 4 BDs: 1 BD for LSO header + 1 BD for extended BD + 1 BD 680 * for linear area data but not include LSO header, namely 681 * skb_headlen(skb) - lso_hdr_len (it may be 0, but that's 682 * okay, we only need to consider the worst case). And 1 BD 683 * for gap. 684 */ 685 return skb_shinfo(skb)->nr_frags + 4; 686 } 687 688 static int enetc_lso_get_hdr_len(const struct sk_buff *skb) 689 { 690 int hdr_len, tlen; 691 692 tlen = skb_is_gso_tcp(skb) ? tcp_hdrlen(skb) : sizeof(struct udphdr); 693 hdr_len = skb_transport_offset(skb) + tlen; 694 695 return hdr_len; 696 } 697 698 static void enetc_lso_start(struct sk_buff *skb, struct enetc_lso_t *lso) 699 { 700 lso->lso_seg_size = skb_shinfo(skb)->gso_size; 701 lso->ipv6 = enetc_skb_is_ipv6(skb); 702 lso->tcp = skb_is_gso_tcp(skb); 703 lso->l3_hdr_len = skb_network_header_len(skb); 704 lso->l3_start = skb_network_offset(skb); 705 lso->hdr_len = enetc_lso_get_hdr_len(skb); 706 lso->total_len = skb->len - lso->hdr_len; 707 } 708 709 static void enetc_lso_map_hdr(struct enetc_bdr *tx_ring, struct sk_buff *skb, 710 int *i, struct enetc_lso_t *lso) 711 { 712 union enetc_tx_bd txbd_tmp, *txbd; 713 struct enetc_tx_swbd *tx_swbd; 714 u16 frm_len, frm_len_ext; 715 u8 flags, e_flags = 0; 716 dma_addr_t addr; 717 char *hdr; 718 719 /* Get the first BD of the LSO BDs chain */ 720 txbd = ENETC_TXBD(*tx_ring, *i); 721 tx_swbd = &tx_ring->tx_swbd[*i]; 722 prefetchw(txbd); 723 724 /* Prepare LSO header: MAC + IP + TCP/UDP */ 725 hdr = tx_ring->tso_headers + *i * TSO_HEADER_SIZE; 726 memcpy(hdr, skb->data, lso->hdr_len); 727 addr = tx_ring->tso_headers_dma + *i * TSO_HEADER_SIZE; 728 729 /* {frm_len_ext, frm_len} indicates the total length of 730 * large transmit data unit. frm_len contains the 16 least 731 * significant bits and frm_len_ext contains the 4 most 732 * significant bits. 733 */ 734 frm_len = lso->total_len & 0xffff; 735 frm_len_ext = (lso->total_len >> 16) & 0xf; 736 737 /* Set the flags of the first BD */ 738 flags = ENETC_TXBD_FLAGS_EX | ENETC_TXBD_FLAGS_CSUM_LSO | 739 ENETC_TXBD_FLAGS_LSO | ENETC_TXBD_FLAGS_L4CS; 740 741 enetc_clear_tx_bd(&txbd_tmp); 742 txbd_tmp.addr = cpu_to_le64(addr); 743 txbd_tmp.hdr_len = cpu_to_le16(lso->hdr_len); 744 745 /* first BD needs frm_len and offload flags set */ 746 txbd_tmp.frm_len = cpu_to_le16(frm_len); 747 txbd_tmp.flags = flags; 748 749 txbd_tmp.l3_aux0 = FIELD_PREP(ENETC_TX_BD_L3_START, lso->l3_start); 750 /* l3_hdr_size in 32-bits (4 bytes) */ 751 txbd_tmp.l3_aux1 = FIELD_PREP(ENETC_TX_BD_L3_HDR_LEN, 752 lso->l3_hdr_len / 4); 753 if (lso->ipv6) 754 txbd_tmp.l3_aux1 |= ENETC_TX_BD_L3T; 755 else 756 txbd_tmp.l3_aux0 |= ENETC_TX_BD_IPCS; 757 758 txbd_tmp.l4_aux = FIELD_PREP(ENETC_TX_BD_L4T, lso->tcp ? 759 ENETC_TXBD_L4T_TCP : ENETC_TXBD_L4T_UDP); 760 761 /* For the LSO header we do not set the dma address since 762 * we do not want it unmapped when we do cleanup. We still 763 * set len so that we count the bytes sent. 764 */ 765 tx_swbd->len = lso->hdr_len; 766 tx_swbd->do_twostep_tstamp = false; 767 tx_swbd->check_wb = false; 768 769 /* Actually write the header in the BD */ 770 *txbd = txbd_tmp; 771 772 /* Get the next BD, and the next BD is extended BD */ 773 enetc_bdr_idx_inc(tx_ring, i); 774 txbd = ENETC_TXBD(*tx_ring, *i); 775 tx_swbd = &tx_ring->tx_swbd[*i]; 776 prefetchw(txbd); 777 778 enetc_clear_tx_bd(&txbd_tmp); 779 if (skb_vlan_tag_present(skb)) { 780 /* Setup the VLAN fields */ 781 txbd_tmp.ext.vid = cpu_to_le16(skb_vlan_tag_get(skb)); 782 txbd_tmp.ext.tpid = ENETC_TPID_8021Q; 783 e_flags = ENETC_TXBD_E_FLAGS_VLAN_INS; 784 } 785 786 /* Write the BD */ 787 txbd_tmp.ext.e_flags = e_flags; 788 txbd_tmp.ext.lso_sg_size = cpu_to_le16(lso->lso_seg_size); 789 txbd_tmp.ext.frm_len_ext = cpu_to_le16(frm_len_ext); 790 *txbd = txbd_tmp; 791 } 792 793 static int enetc_lso_map_data(struct enetc_bdr *tx_ring, struct sk_buff *skb, 794 int *i, struct enetc_lso_t *lso, int *count) 795 { 796 union enetc_tx_bd txbd_tmp, *txbd = NULL; 797 struct enetc_tx_swbd *tx_swbd; 798 skb_frag_t *frag; 799 dma_addr_t dma; 800 u8 flags = 0; 801 int len, f; 802 803 len = skb_headlen(skb) - lso->hdr_len; 804 if (len > 0) { 805 dma = dma_map_single(tx_ring->dev, skb->data + lso->hdr_len, 806 len, DMA_TO_DEVICE); 807 if (dma_mapping_error(tx_ring->dev, dma)) 808 return -ENOMEM; 809 810 enetc_bdr_idx_inc(tx_ring, i); 811 txbd = ENETC_TXBD(*tx_ring, *i); 812 tx_swbd = &tx_ring->tx_swbd[*i]; 813 prefetchw(txbd); 814 *count += 1; 815 816 enetc_clear_tx_bd(&txbd_tmp); 817 txbd_tmp.addr = cpu_to_le64(dma); 818 txbd_tmp.buf_len = cpu_to_le16(len); 819 820 tx_swbd->dma = dma; 821 tx_swbd->len = len; 822 tx_swbd->is_dma_page = 0; 823 tx_swbd->dir = DMA_TO_DEVICE; 824 } 825 826 frag = &skb_shinfo(skb)->frags[0]; 827 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++, frag++) { 828 if (txbd) 829 *txbd = txbd_tmp; 830 831 len = skb_frag_size(frag); 832 dma = skb_frag_dma_map(tx_ring->dev, frag); 833 if (dma_mapping_error(tx_ring->dev, dma)) 834 return -ENOMEM; 835 836 /* Get the next BD */ 837 enetc_bdr_idx_inc(tx_ring, i); 838 txbd = ENETC_TXBD(*tx_ring, *i); 839 tx_swbd = &tx_ring->tx_swbd[*i]; 840 prefetchw(txbd); 841 *count += 1; 842 843 enetc_clear_tx_bd(&txbd_tmp); 844 txbd_tmp.addr = cpu_to_le64(dma); 845 txbd_tmp.buf_len = cpu_to_le16(len); 846 847 tx_swbd->dma = dma; 848 tx_swbd->len = len; 849 tx_swbd->is_dma_page = 1; 850 tx_swbd->dir = DMA_TO_DEVICE; 851 } 852 853 /* Last BD needs 'F' bit set */ 854 flags |= ENETC_TXBD_FLAGS_F; 855 txbd_tmp.flags = flags; 856 *txbd = txbd_tmp; 857 858 tx_swbd->is_eof = 1; 859 tx_swbd->skb = skb; 860 861 return 0; 862 } 863 864 static int enetc_lso_hw_offload(struct enetc_bdr *tx_ring, struct sk_buff *skb) 865 { 866 struct enetc_tx_swbd *tx_swbd; 867 struct enetc_lso_t lso = {0}; 868 int err, i, count = 0; 869 870 /* Initialize the LSO handler */ 871 enetc_lso_start(skb, &lso); 872 i = tx_ring->next_to_use; 873 874 enetc_lso_map_hdr(tx_ring, skb, &i, &lso); 875 /* First BD and an extend BD */ 876 count += 2; 877 878 err = enetc_lso_map_data(tx_ring, skb, &i, &lso, &count); 879 if (err) 880 goto dma_err; 881 882 /* Go to the next BD */ 883 enetc_bdr_idx_inc(tx_ring, &i); 884 tx_ring->next_to_use = i; 885 enetc_update_tx_ring_tail(tx_ring); 886 887 return count; 888 889 dma_err: 890 do { 891 tx_swbd = &tx_ring->tx_swbd[i]; 892 enetc_free_tx_frame(tx_ring, tx_swbd); 893 if (i == 0) 894 i = tx_ring->bd_count; 895 i--; 896 } while (--count); 897 898 return 0; 899 } 900 901 static int enetc_map_tx_tso_buffs(struct enetc_bdr *tx_ring, struct sk_buff *skb) 902 { 903 struct enetc_ndev_priv *priv = netdev_priv(tx_ring->ndev); 904 int hdr_len, total_len, data_len; 905 struct enetc_tx_swbd *tx_swbd; 906 union enetc_tx_bd *txbd; 907 struct tso_t tso; 908 __wsum csum, csum2; 909 int count = 0, pos; 910 int err, i, bd_data_num; 911 912 /* Initialize the TSO handler, and prepare the first payload */ 913 hdr_len = tso_start(skb, &tso); 914 total_len = skb->len - hdr_len; 915 i = tx_ring->next_to_use; 916 917 while (total_len > 0) { 918 char *hdr; 919 920 /* Get the BD */ 921 txbd = ENETC_TXBD(*tx_ring, i); 922 tx_swbd = &tx_ring->tx_swbd[i]; 923 prefetchw(txbd); 924 925 /* Determine the length of this packet */ 926 data_len = min_t(int, skb_shinfo(skb)->gso_size, total_len); 927 total_len -= data_len; 928 929 /* prepare packet headers: MAC + IP + TCP */ 930 hdr = tx_ring->tso_headers + i * TSO_HEADER_SIZE; 931 tso_build_hdr(skb, hdr, &tso, data_len, total_len == 0); 932 933 /* compute the csum over the L4 header */ 934 csum = enetc_tso_hdr_csum(&tso, skb, hdr, hdr_len, &pos); 935 count += enetc_map_tx_tso_hdr(tx_ring, skb, tx_swbd, txbd, 936 &i, hdr_len, data_len); 937 bd_data_num = 0; 938 939 while (data_len > 0) { 940 int size; 941 942 size = min_t(int, tso.size, data_len); 943 944 /* Advance the index in the BDR */ 945 enetc_bdr_idx_inc(tx_ring, &i); 946 txbd = ENETC_TXBD(*tx_ring, i); 947 tx_swbd = &tx_ring->tx_swbd[i]; 948 prefetchw(txbd); 949 950 /* Compute the checksum over this segment of data and 951 * add it to the csum already computed (over the L4 952 * header and possible other data segments). 953 */ 954 csum2 = csum_partial(tso.data, size, 0); 955 csum = csum_block_add(csum, csum2, pos); 956 pos += size; 957 958 err = enetc_map_tx_tso_data(tx_ring, skb, tx_swbd, txbd, 959 tso.data, size, 960 size == data_len); 961 if (err) { 962 if (i == 0) 963 i = tx_ring->bd_count; 964 i--; 965 966 goto err_map_data; 967 } 968 969 data_len -= size; 970 count++; 971 bd_data_num++; 972 tso_build_data(skb, &tso, size); 973 974 if (unlikely(bd_data_num >= priv->max_frags && data_len)) 975 goto err_chained_bd; 976 } 977 978 enetc_tso_complete_csum(tx_ring, &tso, skb, hdr, pos, csum); 979 980 if (total_len == 0) 981 tx_swbd->skb = skb; 982 983 /* Go to the next BD */ 984 enetc_bdr_idx_inc(tx_ring, &i); 985 } 986 987 tx_ring->next_to_use = i; 988 enetc_update_tx_ring_tail(tx_ring); 989 990 return count; 991 992 err_map_data: 993 dev_err(tx_ring->dev, "DMA map error"); 994 995 err_chained_bd: 996 enetc_unwind_tx_frame(tx_ring, count, i); 997 998 return 0; 999 } 1000 1001 static netdev_tx_t enetc_start_xmit(struct sk_buff *skb, 1002 struct net_device *ndev) 1003 { 1004 struct enetc_skb_cb *enetc_cb = ENETC_SKB_CB(skb); 1005 struct enetc_ndev_priv *priv = netdev_priv(ndev); 1006 struct enetc_bdr *tx_ring; 1007 int count; 1008 1009 /* Queue one-step Sync packet if already locked */ 1010 if (enetc_cb->flag & ENETC_F_TX_ONESTEP_SYNC_TSTAMP) { 1011 if (test_and_set_bit_lock(ENETC_TX_ONESTEP_TSTAMP_IN_PROGRESS, 1012 &priv->flags)) { 1013 skb_queue_tail(&priv->tx_skbs, skb); 1014 return NETDEV_TX_OK; 1015 } 1016 } 1017 1018 tx_ring = priv->tx_ring[skb->queue_mapping]; 1019 1020 if (skb_is_gso(skb)) { 1021 /* LSO data unit lengths of up to 256KB are supported */ 1022 if (priv->active_offloads & ENETC_F_LSO && 1023 (skb->len - enetc_lso_get_hdr_len(skb)) <= 1024 ENETC_LSO_MAX_DATA_LEN) { 1025 if (enetc_bd_unused(tx_ring) < enetc_lso_count_descs(skb)) { 1026 netif_stop_subqueue(ndev, tx_ring->index); 1027 return NETDEV_TX_BUSY; 1028 } 1029 1030 count = enetc_lso_hw_offload(tx_ring, skb); 1031 } else { 1032 if (enetc_bd_unused(tx_ring) < tso_count_descs(skb)) { 1033 netif_stop_subqueue(ndev, tx_ring->index); 1034 return NETDEV_TX_BUSY; 1035 } 1036 1037 enetc_lock_mdio(); 1038 count = enetc_map_tx_tso_buffs(tx_ring, skb); 1039 enetc_unlock_mdio(); 1040 } 1041 } else { 1042 if (unlikely(skb_shinfo(skb)->nr_frags > priv->max_frags)) 1043 if (unlikely(skb_linearize(skb))) 1044 goto drop_packet_err; 1045 1046 count = skb_shinfo(skb)->nr_frags + 1; /* fragments + head */ 1047 if (enetc_bd_unused(tx_ring) < ENETC_TXBDS_NEEDED(count)) { 1048 netif_stop_subqueue(ndev, tx_ring->index); 1049 return NETDEV_TX_BUSY; 1050 } 1051 1052 enetc_lock_mdio(); 1053 count = enetc_map_tx_buffs(tx_ring, skb); 1054 enetc_unlock_mdio(); 1055 } 1056 1057 if (unlikely(!count)) 1058 goto drop_packet_err; 1059 1060 if (enetc_bd_unused(tx_ring) < ENETC_TXBDS_MAX_NEEDED(priv->max_frags)) 1061 netif_stop_subqueue(ndev, tx_ring->index); 1062 1063 return NETDEV_TX_OK; 1064 1065 drop_packet_err: 1066 dev_kfree_skb_any(skb); 1067 return NETDEV_TX_OK; 1068 } 1069 1070 netdev_tx_t enetc_xmit(struct sk_buff *skb, struct net_device *ndev) 1071 { 1072 struct enetc_skb_cb *enetc_cb = ENETC_SKB_CB(skb); 1073 struct enetc_ndev_priv *priv = netdev_priv(ndev); 1074 u8 udp, msgtype, twostep; 1075 u16 offset1, offset2; 1076 1077 /* Mark tx timestamp type on enetc_cb->flag if requires */ 1078 if ((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && 1079 (priv->active_offloads & ENETC_F_TX_TSTAMP_MASK)) 1080 enetc_cb->flag = priv->active_offloads & ENETC_F_TX_TSTAMP_MASK; 1081 else 1082 enetc_cb->flag = 0; 1083 1084 /* Fall back to two-step timestamp if not one-step Sync packet */ 1085 if (enetc_cb->flag & ENETC_F_TX_ONESTEP_SYNC_TSTAMP) { 1086 if (enetc_ptp_parse(skb, &udp, &msgtype, &twostep, 1087 &offset1, &offset2) || 1088 msgtype != PTP_MSGTYPE_SYNC || twostep != 0) { 1089 enetc_cb->flag = ENETC_F_TX_TSTAMP; 1090 } else { 1091 enetc_cb->udp = !!udp; 1092 enetc_cb->correction_off = offset1; 1093 enetc_cb->origin_tstamp_off = offset2; 1094 } 1095 } 1096 1097 return enetc_start_xmit(skb, ndev); 1098 } 1099 EXPORT_SYMBOL_GPL(enetc_xmit); 1100 1101 static irqreturn_t enetc_msix(int irq, void *data) 1102 { 1103 struct enetc_int_vector *v = data; 1104 int i; 1105 1106 enetc_lock_mdio(); 1107 1108 /* disable interrupts */ 1109 enetc_wr_reg_hot(v->rbier, 0); 1110 enetc_wr_reg_hot(v->ricr1, v->rx_ictt); 1111 1112 for_each_set_bit(i, &v->tx_rings_map, ENETC_MAX_NUM_TXQS) 1113 enetc_wr_reg_hot(v->tbier_base + ENETC_BDR_OFF(i), 0); 1114 1115 enetc_unlock_mdio(); 1116 1117 napi_schedule(&v->napi); 1118 1119 return IRQ_HANDLED; 1120 } 1121 1122 static void enetc_rx_dim_work(struct work_struct *w) 1123 { 1124 struct dim *dim = container_of(w, struct dim, work); 1125 struct dim_cq_moder moder = 1126 net_dim_get_rx_moderation(dim->mode, dim->profile_ix); 1127 struct enetc_int_vector *v = 1128 container_of(dim, struct enetc_int_vector, rx_dim); 1129 struct enetc_ndev_priv *priv = netdev_priv(v->rx_ring.ndev); 1130 1131 v->rx_ictt = enetc_usecs_to_cycles(moder.usec, priv->sysclk_freq); 1132 dim->state = DIM_START_MEASURE; 1133 } 1134 1135 static void enetc_rx_net_dim(struct enetc_int_vector *v) 1136 { 1137 struct dim_sample dim_sample = {}; 1138 1139 v->comp_cnt++; 1140 1141 if (!v->rx_napi_work) 1142 return; 1143 1144 dim_update_sample(v->comp_cnt, 1145 v->rx_ring.stats.packets, 1146 v->rx_ring.stats.bytes, 1147 &dim_sample); 1148 net_dim(&v->rx_dim, &dim_sample); 1149 } 1150 1151 static int enetc_bd_ready_count(struct enetc_bdr *tx_ring, int ci) 1152 { 1153 int pi = enetc_rd_reg_hot(tx_ring->tcir) & ENETC_TBCIR_IDX_MASK; 1154 1155 return pi >= ci ? pi - ci : tx_ring->bd_count - ci + pi; 1156 } 1157 1158 static bool enetc_page_reusable(struct page *page) 1159 { 1160 return (!page_is_pfmemalloc(page) && page_ref_count(page) == 1); 1161 } 1162 1163 static void enetc_reuse_page(struct enetc_bdr *rx_ring, 1164 struct enetc_rx_swbd *old) 1165 { 1166 struct enetc_rx_swbd *new; 1167 1168 new = &rx_ring->rx_swbd[rx_ring->next_to_alloc]; 1169 1170 /* next buf that may reuse a page */ 1171 enetc_bdr_idx_inc(rx_ring, &rx_ring->next_to_alloc); 1172 1173 /* copy page reference */ 1174 *new = *old; 1175 } 1176 1177 static void enetc_get_tx_tstamp(struct enetc_hw *hw, union enetc_tx_bd *txbd, 1178 u64 *tstamp) 1179 { 1180 u32 lo, hi, tstamp_lo; 1181 1182 lo = enetc_rd_hot(hw, ENETC_SICTR0); 1183 hi = enetc_rd_hot(hw, ENETC_SICTR1); 1184 tstamp_lo = le32_to_cpu(txbd->wb.tstamp); 1185 if (lo <= tstamp_lo) 1186 hi -= 1; 1187 *tstamp = (u64)hi << 32 | tstamp_lo; 1188 } 1189 1190 static void enetc_tstamp_tx(struct sk_buff *skb, u64 tstamp) 1191 { 1192 struct skb_shared_hwtstamps shhwtstamps; 1193 1194 if (skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS) { 1195 memset(&shhwtstamps, 0, sizeof(shhwtstamps)); 1196 shhwtstamps.hwtstamp = ns_to_ktime(tstamp); 1197 skb_txtime_consumed(skb); 1198 skb_tstamp_tx(skb, &shhwtstamps); 1199 } 1200 } 1201 1202 static void enetc_recycle_xdp_tx_buff(struct enetc_bdr *tx_ring, 1203 struct enetc_tx_swbd *tx_swbd) 1204 { 1205 struct enetc_ndev_priv *priv = netdev_priv(tx_ring->ndev); 1206 struct enetc_rx_swbd rx_swbd = { 1207 .dma = tx_swbd->dma, 1208 .page = tx_swbd->page, 1209 .page_offset = tx_swbd->page_offset, 1210 .dir = tx_swbd->dir, 1211 .len = tx_swbd->len, 1212 }; 1213 struct enetc_bdr *rx_ring; 1214 1215 rx_ring = enetc_rx_ring_from_xdp_tx_ring(priv, tx_ring); 1216 1217 if (likely(enetc_swbd_unused(rx_ring))) { 1218 enetc_reuse_page(rx_ring, &rx_swbd); 1219 1220 /* sync for use by the device */ 1221 dma_sync_single_range_for_device(rx_ring->dev, rx_swbd.dma, 1222 rx_swbd.page_offset, 1223 ENETC_RXB_DMA_SIZE_XDP, 1224 rx_swbd.dir); 1225 1226 rx_ring->stats.recycles++; 1227 } else { 1228 /* RX ring is already full, we need to unmap and free the 1229 * page, since there's nothing useful we can do with it. 1230 */ 1231 rx_ring->stats.recycle_failures++; 1232 1233 dma_unmap_page(rx_ring->dev, rx_swbd.dma, PAGE_SIZE, 1234 rx_swbd.dir); 1235 __free_page(rx_swbd.page); 1236 } 1237 1238 rx_ring->xdp.xdp_tx_in_flight--; 1239 } 1240 1241 static bool enetc_clean_tx_ring(struct enetc_bdr *tx_ring, int napi_budget) 1242 { 1243 int tx_frm_cnt = 0, tx_byte_cnt = 0, tx_win_drop = 0; 1244 struct net_device *ndev = tx_ring->ndev; 1245 struct enetc_ndev_priv *priv = netdev_priv(ndev); 1246 struct enetc_tx_swbd *tx_swbd; 1247 int i, bds_to_clean; 1248 bool do_twostep_tstamp; 1249 u64 tstamp = 0; 1250 1251 i = tx_ring->next_to_clean; 1252 tx_swbd = &tx_ring->tx_swbd[i]; 1253 1254 bds_to_clean = enetc_bd_ready_count(tx_ring, i); 1255 1256 do_twostep_tstamp = false; 1257 1258 while (bds_to_clean && tx_frm_cnt < ENETC_DEFAULT_TX_WORK) { 1259 struct xdp_frame *xdp_frame = enetc_tx_swbd_get_xdp_frame(tx_swbd); 1260 struct sk_buff *skb = enetc_tx_swbd_get_skb(tx_swbd); 1261 bool is_eof = tx_swbd->is_eof; 1262 1263 if (unlikely(tx_swbd->check_wb)) { 1264 union enetc_tx_bd *txbd = ENETC_TXBD(*tx_ring, i); 1265 1266 if (txbd->flags & ENETC_TXBD_FLAGS_W && 1267 tx_swbd->do_twostep_tstamp) { 1268 enetc_get_tx_tstamp(&priv->si->hw, txbd, 1269 &tstamp); 1270 do_twostep_tstamp = true; 1271 } 1272 1273 if (tx_swbd->qbv_en && 1274 txbd->wb.status & ENETC_TXBD_STATS_WIN) 1275 tx_win_drop++; 1276 } 1277 1278 if (tx_swbd->is_xdp_tx) 1279 enetc_recycle_xdp_tx_buff(tx_ring, tx_swbd); 1280 else if (likely(tx_swbd->dma)) 1281 enetc_unmap_tx_buff(tx_ring, tx_swbd); 1282 1283 if (xdp_frame) { 1284 xdp_return_frame(xdp_frame); 1285 } else if (skb) { 1286 struct enetc_skb_cb *enetc_cb = ENETC_SKB_CB(skb); 1287 1288 if (unlikely(enetc_cb->flag & ENETC_F_TX_ONESTEP_SYNC_TSTAMP)) { 1289 /* Start work to release lock for next one-step 1290 * timestamping packet. And send one skb in 1291 * tx_skbs queue if has. 1292 */ 1293 schedule_work(&priv->tx_onestep_tstamp); 1294 } else if (unlikely(do_twostep_tstamp)) { 1295 enetc_tstamp_tx(skb, tstamp); 1296 do_twostep_tstamp = false; 1297 } 1298 napi_consume_skb(skb, napi_budget); 1299 } 1300 1301 tx_byte_cnt += tx_swbd->len; 1302 /* Scrub the swbd here so we don't have to do that 1303 * when we reuse it during xmit 1304 */ 1305 memset(tx_swbd, 0, sizeof(*tx_swbd)); 1306 1307 bds_to_clean--; 1308 tx_swbd++; 1309 i++; 1310 if (unlikely(i == tx_ring->bd_count)) { 1311 i = 0; 1312 tx_swbd = tx_ring->tx_swbd; 1313 } 1314 1315 /* BD iteration loop end */ 1316 if (is_eof) { 1317 tx_frm_cnt++; 1318 /* re-arm interrupt source */ 1319 enetc_wr_reg_hot(tx_ring->idr, BIT(tx_ring->index) | 1320 BIT(16 + tx_ring->index)); 1321 } 1322 1323 if (unlikely(!bds_to_clean)) 1324 bds_to_clean = enetc_bd_ready_count(tx_ring, i); 1325 } 1326 1327 tx_ring->next_to_clean = i; 1328 tx_ring->stats.packets += tx_frm_cnt; 1329 tx_ring->stats.bytes += tx_byte_cnt; 1330 tx_ring->stats.win_drop += tx_win_drop; 1331 1332 if (unlikely(tx_frm_cnt && netif_carrier_ok(ndev) && 1333 __netif_subqueue_stopped(ndev, tx_ring->index) && 1334 !test_bit(ENETC_TX_DOWN, &priv->flags) && 1335 (enetc_bd_unused(tx_ring) >= 1336 ENETC_TXBDS_MAX_NEEDED(priv->max_frags)))) { 1337 netif_wake_subqueue(ndev, tx_ring->index); 1338 } 1339 1340 return tx_frm_cnt != ENETC_DEFAULT_TX_WORK; 1341 } 1342 1343 static bool enetc_new_page(struct enetc_bdr *rx_ring, 1344 struct enetc_rx_swbd *rx_swbd) 1345 { 1346 bool xdp = !!(rx_ring->xdp.prog); 1347 struct page *page; 1348 dma_addr_t addr; 1349 1350 page = dev_alloc_page(); 1351 if (unlikely(!page)) 1352 return false; 1353 1354 /* For XDP_TX, we forgo dma_unmap -> dma_map */ 1355 rx_swbd->dir = xdp ? DMA_BIDIRECTIONAL : DMA_FROM_DEVICE; 1356 1357 addr = dma_map_page(rx_ring->dev, page, 0, PAGE_SIZE, rx_swbd->dir); 1358 if (unlikely(dma_mapping_error(rx_ring->dev, addr))) { 1359 __free_page(page); 1360 1361 return false; 1362 } 1363 1364 rx_swbd->dma = addr; 1365 rx_swbd->page = page; 1366 rx_swbd->page_offset = rx_ring->buffer_offset; 1367 1368 return true; 1369 } 1370 1371 static int enetc_refill_rx_ring(struct enetc_bdr *rx_ring, const int buff_cnt) 1372 { 1373 struct enetc_rx_swbd *rx_swbd; 1374 union enetc_rx_bd *rxbd; 1375 int i, j; 1376 1377 i = rx_ring->next_to_use; 1378 rx_swbd = &rx_ring->rx_swbd[i]; 1379 rxbd = enetc_rxbd(rx_ring, i); 1380 1381 for (j = 0; j < buff_cnt; j++) { 1382 /* try reuse page */ 1383 if (unlikely(!rx_swbd->page)) { 1384 if (unlikely(!enetc_new_page(rx_ring, rx_swbd))) { 1385 rx_ring->stats.rx_alloc_errs++; 1386 break; 1387 } 1388 } 1389 1390 /* update RxBD */ 1391 rxbd->w.addr = cpu_to_le64(rx_swbd->dma + 1392 rx_swbd->page_offset); 1393 /* clear 'R" as well */ 1394 rxbd->r.lstatus = 0; 1395 1396 enetc_rxbd_next(rx_ring, &rxbd, &i); 1397 rx_swbd = &rx_ring->rx_swbd[i]; 1398 } 1399 1400 if (likely(j)) { 1401 rx_ring->next_to_alloc = i; /* keep track from page reuse */ 1402 rx_ring->next_to_use = i; 1403 1404 /* update ENETC's consumer index */ 1405 enetc_wr_reg_hot(rx_ring->rcir, rx_ring->next_to_use); 1406 } 1407 1408 return j; 1409 } 1410 1411 static void enetc_get_rx_tstamp(struct net_device *ndev, 1412 union enetc_rx_bd *rxbd, 1413 struct sk_buff *skb) 1414 { 1415 struct skb_shared_hwtstamps *shhwtstamps = skb_hwtstamps(skb); 1416 struct enetc_ndev_priv *priv = netdev_priv(ndev); 1417 struct enetc_hw *hw = &priv->si->hw; 1418 u32 lo, hi, tstamp_lo; 1419 u64 tstamp; 1420 1421 if (le16_to_cpu(rxbd->r.flags) & ENETC_RXBD_FLAG_TSTMP) { 1422 lo = enetc_rd_reg_hot(hw->reg + ENETC_SICTR0); 1423 hi = enetc_rd_reg_hot(hw->reg + ENETC_SICTR1); 1424 rxbd = enetc_rxbd_ext(rxbd); 1425 tstamp_lo = le32_to_cpu(rxbd->ext.tstamp); 1426 if (lo <= tstamp_lo) 1427 hi -= 1; 1428 1429 tstamp = (u64)hi << 32 | tstamp_lo; 1430 memset(shhwtstamps, 0, sizeof(*shhwtstamps)); 1431 shhwtstamps->hwtstamp = ns_to_ktime(tstamp); 1432 } 1433 } 1434 1435 static void enetc_get_offloads(struct enetc_bdr *rx_ring, 1436 union enetc_rx_bd *rxbd, struct sk_buff *skb) 1437 { 1438 struct enetc_ndev_priv *priv = netdev_priv(rx_ring->ndev); 1439 1440 /* TODO: hashing */ 1441 if (rx_ring->ndev->features & NETIF_F_RXCSUM) { 1442 u16 inet_csum = le16_to_cpu(rxbd->r.inet_csum); 1443 1444 skb->csum = csum_unfold((__force __sum16)~htons(inet_csum)); 1445 skb->ip_summed = CHECKSUM_COMPLETE; 1446 } 1447 1448 if (le16_to_cpu(rxbd->r.flags) & ENETC_RXBD_FLAG_VLAN) { 1449 struct enetc_hw *hw = &priv->si->hw; 1450 __be16 tpid = 0; 1451 1452 switch (le16_to_cpu(rxbd->r.flags) & ENETC_RXBD_FLAG_TPID) { 1453 case 0: 1454 tpid = htons(ETH_P_8021Q); 1455 break; 1456 case 1: 1457 tpid = htons(ETH_P_8021AD); 1458 break; 1459 case 2: 1460 tpid = htons(enetc_rd_hot(hw, ENETC_SICVLANR1) & 1461 SICVLANR_ETYPE); 1462 break; 1463 case 3: 1464 tpid = htons(enetc_rd_hot(hw, ENETC_SICVLANR2) & 1465 SICVLANR_ETYPE); 1466 } 1467 1468 __vlan_hwaccel_put_tag(skb, tpid, le16_to_cpu(rxbd->r.vlan_opt)); 1469 } 1470 1471 if (priv->active_offloads & ENETC_F_RX_TSTAMP) 1472 enetc_get_rx_tstamp(rx_ring->ndev, rxbd, skb); 1473 } 1474 1475 /* This gets called during the non-XDP NAPI poll cycle as well as on XDP_PASS, 1476 * so it needs to work with both DMA_FROM_DEVICE as well as DMA_BIDIRECTIONAL 1477 * mapped buffers. 1478 */ 1479 static struct enetc_rx_swbd *enetc_get_rx_buff(struct enetc_bdr *rx_ring, 1480 int i, u16 size) 1481 { 1482 struct enetc_rx_swbd *rx_swbd = &rx_ring->rx_swbd[i]; 1483 1484 dma_sync_single_range_for_cpu(rx_ring->dev, rx_swbd->dma, 1485 rx_swbd->page_offset, 1486 size, rx_swbd->dir); 1487 return rx_swbd; 1488 } 1489 1490 /* Reuse the current page without performing half-page buffer flipping */ 1491 static void enetc_put_rx_buff(struct enetc_bdr *rx_ring, 1492 struct enetc_rx_swbd *rx_swbd) 1493 { 1494 size_t buffer_size = ENETC_RXB_TRUESIZE - rx_ring->buffer_offset; 1495 1496 enetc_reuse_page(rx_ring, rx_swbd); 1497 1498 dma_sync_single_range_for_device(rx_ring->dev, rx_swbd->dma, 1499 rx_swbd->page_offset, 1500 buffer_size, rx_swbd->dir); 1501 1502 rx_swbd->page = NULL; 1503 } 1504 1505 /* Reuse the current page by performing half-page buffer flipping */ 1506 static void enetc_flip_rx_buff(struct enetc_bdr *rx_ring, 1507 struct enetc_rx_swbd *rx_swbd) 1508 { 1509 if (likely(enetc_page_reusable(rx_swbd->page))) { 1510 rx_swbd->page_offset ^= ENETC_RXB_TRUESIZE; 1511 page_ref_inc(rx_swbd->page); 1512 1513 enetc_put_rx_buff(rx_ring, rx_swbd); 1514 } else { 1515 dma_unmap_page(rx_ring->dev, rx_swbd->dma, PAGE_SIZE, 1516 rx_swbd->dir); 1517 rx_swbd->page = NULL; 1518 } 1519 } 1520 1521 static struct sk_buff *enetc_map_rx_buff_to_skb(struct enetc_bdr *rx_ring, 1522 int i, u16 size) 1523 { 1524 struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size); 1525 struct sk_buff *skb; 1526 void *ba; 1527 1528 ba = page_address(rx_swbd->page) + rx_swbd->page_offset; 1529 skb = build_skb(ba - rx_ring->buffer_offset, ENETC_RXB_TRUESIZE); 1530 if (unlikely(!skb)) { 1531 rx_ring->stats.rx_alloc_errs++; 1532 return NULL; 1533 } 1534 1535 skb_reserve(skb, rx_ring->buffer_offset); 1536 __skb_put(skb, size); 1537 1538 enetc_flip_rx_buff(rx_ring, rx_swbd); 1539 1540 return skb; 1541 } 1542 1543 static void enetc_add_rx_buff_to_skb(struct enetc_bdr *rx_ring, int i, 1544 u16 size, struct sk_buff *skb) 1545 { 1546 struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size); 1547 1548 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_swbd->page, 1549 rx_swbd->page_offset, size, ENETC_RXB_TRUESIZE); 1550 1551 enetc_flip_rx_buff(rx_ring, rx_swbd); 1552 } 1553 1554 static bool enetc_check_bd_errors_and_consume(struct enetc_bdr *rx_ring, 1555 u32 bd_status, 1556 union enetc_rx_bd **rxbd, int *i) 1557 { 1558 if (likely(!(bd_status & ENETC_RXBD_LSTATUS(ENETC_RXBD_ERR_MASK)))) 1559 return false; 1560 1561 enetc_put_rx_buff(rx_ring, &rx_ring->rx_swbd[*i]); 1562 enetc_rxbd_next(rx_ring, rxbd, i); 1563 1564 while (!(bd_status & ENETC_RXBD_LSTATUS_F)) { 1565 dma_rmb(); 1566 bd_status = le32_to_cpu((*rxbd)->r.lstatus); 1567 1568 enetc_put_rx_buff(rx_ring, &rx_ring->rx_swbd[*i]); 1569 enetc_rxbd_next(rx_ring, rxbd, i); 1570 } 1571 1572 rx_ring->ndev->stats.rx_dropped++; 1573 rx_ring->ndev->stats.rx_errors++; 1574 1575 return true; 1576 } 1577 1578 static struct sk_buff *enetc_build_skb(struct enetc_bdr *rx_ring, 1579 u32 bd_status, union enetc_rx_bd **rxbd, 1580 int *i, int *cleaned_cnt, int buffer_size) 1581 { 1582 struct sk_buff *skb; 1583 u16 size; 1584 1585 size = le16_to_cpu((*rxbd)->r.buf_len); 1586 skb = enetc_map_rx_buff_to_skb(rx_ring, *i, size); 1587 if (!skb) 1588 return NULL; 1589 1590 enetc_get_offloads(rx_ring, *rxbd, skb); 1591 1592 (*cleaned_cnt)++; 1593 1594 enetc_rxbd_next(rx_ring, rxbd, i); 1595 1596 /* not last BD in frame? */ 1597 while (!(bd_status & ENETC_RXBD_LSTATUS_F)) { 1598 bd_status = le32_to_cpu((*rxbd)->r.lstatus); 1599 size = buffer_size; 1600 1601 if (bd_status & ENETC_RXBD_LSTATUS_F) { 1602 dma_rmb(); 1603 size = le16_to_cpu((*rxbd)->r.buf_len); 1604 } 1605 1606 enetc_add_rx_buff_to_skb(rx_ring, *i, size, skb); 1607 1608 (*cleaned_cnt)++; 1609 1610 enetc_rxbd_next(rx_ring, rxbd, i); 1611 } 1612 1613 skb_record_rx_queue(skb, rx_ring->index); 1614 skb->protocol = eth_type_trans(skb, rx_ring->ndev); 1615 1616 return skb; 1617 } 1618 1619 #define ENETC_RXBD_BUNDLE 16 /* # of BDs to update at once */ 1620 1621 static int enetc_clean_rx_ring(struct enetc_bdr *rx_ring, 1622 struct napi_struct *napi, int work_limit) 1623 { 1624 int rx_frm_cnt = 0, rx_byte_cnt = 0; 1625 int cleaned_cnt, i; 1626 1627 cleaned_cnt = enetc_bd_unused(rx_ring); 1628 /* next descriptor to process */ 1629 i = rx_ring->next_to_clean; 1630 1631 enetc_lock_mdio(); 1632 1633 while (likely(rx_frm_cnt < work_limit)) { 1634 union enetc_rx_bd *rxbd; 1635 struct sk_buff *skb; 1636 u32 bd_status; 1637 1638 if (cleaned_cnt >= ENETC_RXBD_BUNDLE) 1639 cleaned_cnt -= enetc_refill_rx_ring(rx_ring, 1640 cleaned_cnt); 1641 1642 rxbd = enetc_rxbd(rx_ring, i); 1643 bd_status = le32_to_cpu(rxbd->r.lstatus); 1644 if (!bd_status) 1645 break; 1646 1647 enetc_wr_reg_hot(rx_ring->idr, BIT(rx_ring->index)); 1648 dma_rmb(); /* for reading other rxbd fields */ 1649 1650 if (enetc_check_bd_errors_and_consume(rx_ring, bd_status, 1651 &rxbd, &i)) 1652 break; 1653 1654 skb = enetc_build_skb(rx_ring, bd_status, &rxbd, &i, 1655 &cleaned_cnt, ENETC_RXB_DMA_SIZE); 1656 if (!skb) 1657 break; 1658 1659 /* When set, the outer VLAN header is extracted and reported 1660 * in the receive buffer descriptor. So rx_byte_cnt should 1661 * add the length of the extracted VLAN header. 1662 */ 1663 if (bd_status & ENETC_RXBD_FLAG_VLAN) 1664 rx_byte_cnt += VLAN_HLEN; 1665 rx_byte_cnt += skb->len + ETH_HLEN; 1666 rx_frm_cnt++; 1667 1668 enetc_unlock_mdio(); 1669 napi_gro_receive(napi, skb); 1670 enetc_lock_mdio(); 1671 } 1672 1673 rx_ring->next_to_clean = i; 1674 1675 rx_ring->stats.packets += rx_frm_cnt; 1676 rx_ring->stats.bytes += rx_byte_cnt; 1677 1678 enetc_unlock_mdio(); 1679 1680 return rx_frm_cnt; 1681 } 1682 1683 static void enetc_xdp_map_tx_buff(struct enetc_bdr *tx_ring, int i, 1684 struct enetc_tx_swbd *tx_swbd, 1685 int frm_len) 1686 { 1687 union enetc_tx_bd *txbd = ENETC_TXBD(*tx_ring, i); 1688 1689 prefetchw(txbd); 1690 1691 enetc_clear_tx_bd(txbd); 1692 txbd->addr = cpu_to_le64(tx_swbd->dma + tx_swbd->page_offset); 1693 txbd->buf_len = cpu_to_le16(tx_swbd->len); 1694 txbd->frm_len = cpu_to_le16(frm_len); 1695 1696 memcpy(&tx_ring->tx_swbd[i], tx_swbd, sizeof(*tx_swbd)); 1697 } 1698 1699 /* Puts in the TX ring one XDP frame, mapped as an array of TX software buffer 1700 * descriptors. 1701 */ 1702 static bool enetc_xdp_tx(struct enetc_bdr *tx_ring, 1703 struct enetc_tx_swbd *xdp_tx_arr, int num_tx_swbd) 1704 { 1705 struct enetc_tx_swbd *tmp_tx_swbd = xdp_tx_arr; 1706 int i, k, frm_len = tmp_tx_swbd->len; 1707 1708 if (unlikely(enetc_bd_unused(tx_ring) < ENETC_TXBDS_NEEDED(num_tx_swbd))) 1709 return false; 1710 1711 while (unlikely(!tmp_tx_swbd->is_eof)) { 1712 tmp_tx_swbd++; 1713 frm_len += tmp_tx_swbd->len; 1714 } 1715 1716 i = tx_ring->next_to_use; 1717 1718 for (k = 0; k < num_tx_swbd; k++) { 1719 struct enetc_tx_swbd *xdp_tx_swbd = &xdp_tx_arr[k]; 1720 1721 enetc_xdp_map_tx_buff(tx_ring, i, xdp_tx_swbd, frm_len); 1722 1723 /* last BD needs 'F' bit set */ 1724 if (xdp_tx_swbd->is_eof) { 1725 union enetc_tx_bd *txbd = ENETC_TXBD(*tx_ring, i); 1726 1727 txbd->flags = ENETC_TXBD_FLAGS_F; 1728 } 1729 1730 enetc_bdr_idx_inc(tx_ring, &i); 1731 } 1732 1733 tx_ring->next_to_use = i; 1734 1735 return true; 1736 } 1737 1738 static int enetc_xdp_frame_to_xdp_tx_swbd(struct enetc_bdr *tx_ring, 1739 struct enetc_tx_swbd *xdp_tx_arr, 1740 struct xdp_frame *xdp_frame) 1741 { 1742 struct enetc_tx_swbd *xdp_tx_swbd = &xdp_tx_arr[0]; 1743 struct skb_shared_info *shinfo; 1744 void *data = xdp_frame->data; 1745 int len = xdp_frame->len; 1746 skb_frag_t *frag; 1747 dma_addr_t dma; 1748 unsigned int f; 1749 int n = 0; 1750 1751 dma = dma_map_single(tx_ring->dev, data, len, DMA_TO_DEVICE); 1752 if (unlikely(dma_mapping_error(tx_ring->dev, dma))) { 1753 netdev_err(tx_ring->ndev, "DMA map error\n"); 1754 return -1; 1755 } 1756 1757 xdp_tx_swbd->dma = dma; 1758 xdp_tx_swbd->dir = DMA_TO_DEVICE; 1759 xdp_tx_swbd->len = len; 1760 xdp_tx_swbd->is_xdp_redirect = true; 1761 xdp_tx_swbd->is_eof = false; 1762 xdp_tx_swbd->xdp_frame = NULL; 1763 1764 n++; 1765 1766 if (!xdp_frame_has_frags(xdp_frame)) 1767 goto out; 1768 1769 xdp_tx_swbd = &xdp_tx_arr[n]; 1770 1771 shinfo = xdp_get_shared_info_from_frame(xdp_frame); 1772 1773 for (f = 0, frag = &shinfo->frags[0]; f < shinfo->nr_frags; 1774 f++, frag++) { 1775 data = skb_frag_address(frag); 1776 len = skb_frag_size(frag); 1777 1778 dma = dma_map_single(tx_ring->dev, data, len, DMA_TO_DEVICE); 1779 if (unlikely(dma_mapping_error(tx_ring->dev, dma))) { 1780 /* Undo the DMA mapping for all fragments */ 1781 while (--n >= 0) 1782 enetc_unmap_tx_buff(tx_ring, &xdp_tx_arr[n]); 1783 1784 netdev_err(tx_ring->ndev, "DMA map error\n"); 1785 return -1; 1786 } 1787 1788 xdp_tx_swbd->dma = dma; 1789 xdp_tx_swbd->dir = DMA_TO_DEVICE; 1790 xdp_tx_swbd->len = len; 1791 xdp_tx_swbd->is_xdp_redirect = true; 1792 xdp_tx_swbd->is_eof = false; 1793 xdp_tx_swbd->xdp_frame = NULL; 1794 1795 n++; 1796 xdp_tx_swbd = &xdp_tx_arr[n]; 1797 } 1798 out: 1799 xdp_tx_arr[n - 1].is_eof = true; 1800 xdp_tx_arr[n - 1].xdp_frame = xdp_frame; 1801 1802 return n; 1803 } 1804 1805 int enetc_xdp_xmit(struct net_device *ndev, int num_frames, 1806 struct xdp_frame **frames, u32 flags) 1807 { 1808 struct enetc_tx_swbd xdp_redirect_arr[ENETC_MAX_SKB_FRAGS] = {0}; 1809 struct enetc_ndev_priv *priv = netdev_priv(ndev); 1810 struct skb_shared_info *shinfo; 1811 struct enetc_bdr *tx_ring; 1812 int xdp_tx_bd_cnt, i, k; 1813 int xdp_tx_frm_cnt = 0; 1814 1815 if (unlikely(test_bit(ENETC_TX_DOWN, &priv->flags) || 1816 !netif_carrier_ok(ndev))) 1817 return -ENETDOWN; 1818 1819 enetc_lock_mdio(); 1820 1821 tx_ring = priv->xdp_tx_ring[smp_processor_id()]; 1822 1823 prefetchw(ENETC_TXBD(*tx_ring, tx_ring->next_to_use)); 1824 1825 for (k = 0; k < num_frames; k++) { 1826 if (xdp_frame_has_frags(frames[k])) { 1827 shinfo = xdp_get_shared_info_from_frame(frames[k]); 1828 if (unlikely((shinfo->nr_frags + 1) > ENETC_MAX_SKB_FRAGS)) 1829 break; 1830 } 1831 1832 xdp_tx_bd_cnt = enetc_xdp_frame_to_xdp_tx_swbd(tx_ring, 1833 xdp_redirect_arr, 1834 frames[k]); 1835 if (unlikely(xdp_tx_bd_cnt < 0)) 1836 break; 1837 1838 if (unlikely(!enetc_xdp_tx(tx_ring, xdp_redirect_arr, 1839 xdp_tx_bd_cnt))) { 1840 for (i = 0; i < xdp_tx_bd_cnt; i++) 1841 enetc_unmap_tx_buff(tx_ring, 1842 &xdp_redirect_arr[i]); 1843 tx_ring->stats.xdp_tx_drops++; 1844 break; 1845 } 1846 1847 xdp_tx_frm_cnt++; 1848 } 1849 1850 if (unlikely((flags & XDP_XMIT_FLUSH) || k != xdp_tx_frm_cnt)) 1851 enetc_update_tx_ring_tail(tx_ring); 1852 1853 tx_ring->stats.xdp_tx += xdp_tx_frm_cnt; 1854 1855 enetc_unlock_mdio(); 1856 1857 return xdp_tx_frm_cnt; 1858 } 1859 EXPORT_SYMBOL_GPL(enetc_xdp_xmit); 1860 1861 static void enetc_map_rx_buff_to_xdp(struct enetc_bdr *rx_ring, int i, 1862 struct xdp_buff *xdp_buff, u16 size) 1863 { 1864 struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size); 1865 void *hard_start = page_address(rx_swbd->page) + rx_swbd->page_offset; 1866 1867 /* To be used for XDP_TX */ 1868 rx_swbd->len = size; 1869 1870 xdp_prepare_buff(xdp_buff, hard_start - rx_ring->buffer_offset, 1871 rx_ring->buffer_offset, size, false); 1872 } 1873 1874 static void enetc_add_rx_buff_to_xdp(struct enetc_bdr *rx_ring, int i, 1875 u16 size, struct xdp_buff *xdp_buff) 1876 { 1877 struct skb_shared_info *shinfo = xdp_get_shared_info_from_buff(xdp_buff); 1878 struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size); 1879 skb_frag_t *frag; 1880 1881 /* To be used for XDP_TX */ 1882 rx_swbd->len = size; 1883 1884 if (!xdp_buff_has_frags(xdp_buff)) { 1885 xdp_buff_set_frags_flag(xdp_buff); 1886 shinfo->xdp_frags_size = size; 1887 shinfo->nr_frags = 0; 1888 } else { 1889 shinfo->xdp_frags_size += size; 1890 } 1891 1892 if (page_is_pfmemalloc(rx_swbd->page)) 1893 xdp_buff_set_frag_pfmemalloc(xdp_buff); 1894 1895 frag = &shinfo->frags[shinfo->nr_frags]; 1896 skb_frag_fill_page_desc(frag, rx_swbd->page, rx_swbd->page_offset, 1897 size); 1898 1899 shinfo->nr_frags++; 1900 } 1901 1902 static void enetc_build_xdp_buff(struct enetc_bdr *rx_ring, u32 bd_status, 1903 union enetc_rx_bd **rxbd, int *i, 1904 int *cleaned_cnt, struct xdp_buff *xdp_buff) 1905 { 1906 u16 size = le16_to_cpu((*rxbd)->r.buf_len); 1907 1908 xdp_init_buff(xdp_buff, ENETC_RXB_TRUESIZE, &rx_ring->xdp.rxq); 1909 1910 enetc_map_rx_buff_to_xdp(rx_ring, *i, xdp_buff, size); 1911 (*cleaned_cnt)++; 1912 enetc_rxbd_next(rx_ring, rxbd, i); 1913 1914 /* not last BD in frame? */ 1915 while (!(bd_status & ENETC_RXBD_LSTATUS_F)) { 1916 bd_status = le32_to_cpu((*rxbd)->r.lstatus); 1917 size = ENETC_RXB_DMA_SIZE_XDP; 1918 1919 if (bd_status & ENETC_RXBD_LSTATUS_F) { 1920 dma_rmb(); 1921 size = le16_to_cpu((*rxbd)->r.buf_len); 1922 } 1923 1924 enetc_add_rx_buff_to_xdp(rx_ring, *i, size, xdp_buff); 1925 (*cleaned_cnt)++; 1926 enetc_rxbd_next(rx_ring, rxbd, i); 1927 } 1928 } 1929 1930 /* Convert RX buffer descriptors to TX buffer descriptors. These will be 1931 * recycled back into the RX ring in enetc_clean_tx_ring. 1932 */ 1933 static int enetc_rx_swbd_to_xdp_tx_swbd(struct enetc_tx_swbd *xdp_tx_arr, 1934 struct enetc_bdr *rx_ring, 1935 int rx_ring_first, int rx_ring_last) 1936 { 1937 int n = 0; 1938 1939 for (; rx_ring_first != rx_ring_last; 1940 n++, enetc_bdr_idx_inc(rx_ring, &rx_ring_first)) { 1941 struct enetc_rx_swbd *rx_swbd = &rx_ring->rx_swbd[rx_ring_first]; 1942 struct enetc_tx_swbd *tx_swbd = &xdp_tx_arr[n]; 1943 1944 /* No need to dma_map, we already have DMA_BIDIRECTIONAL */ 1945 tx_swbd->dma = rx_swbd->dma; 1946 tx_swbd->dir = rx_swbd->dir; 1947 tx_swbd->page = rx_swbd->page; 1948 tx_swbd->page_offset = rx_swbd->page_offset; 1949 tx_swbd->len = rx_swbd->len; 1950 tx_swbd->is_dma_page = true; 1951 tx_swbd->is_xdp_tx = true; 1952 tx_swbd->is_eof = false; 1953 } 1954 1955 /* We rely on caller providing an rx_ring_last > rx_ring_first */ 1956 xdp_tx_arr[n - 1].is_eof = true; 1957 1958 return n; 1959 } 1960 1961 static void enetc_xdp_drop(struct enetc_bdr *rx_ring, int rx_ring_first, 1962 int rx_ring_last) 1963 { 1964 while (rx_ring_first != rx_ring_last) { 1965 enetc_put_rx_buff(rx_ring, 1966 &rx_ring->rx_swbd[rx_ring_first]); 1967 enetc_bdr_idx_inc(rx_ring, &rx_ring_first); 1968 } 1969 } 1970 1971 static void enetc_bulk_flip_buff(struct enetc_bdr *rx_ring, int rx_ring_first, 1972 int rx_ring_last) 1973 { 1974 while (rx_ring_first != rx_ring_last) { 1975 enetc_flip_rx_buff(rx_ring, 1976 &rx_ring->rx_swbd[rx_ring_first]); 1977 enetc_bdr_idx_inc(rx_ring, &rx_ring_first); 1978 } 1979 } 1980 1981 static int enetc_clean_rx_ring_xdp(struct enetc_bdr *rx_ring, 1982 struct napi_struct *napi, int work_limit, 1983 struct bpf_prog *prog) 1984 { 1985 int xdp_tx_bd_cnt, xdp_tx_frm_cnt = 0, xdp_redirect_frm_cnt = 0; 1986 struct enetc_tx_swbd xdp_tx_arr[ENETC_MAX_SKB_FRAGS] = {0}; 1987 struct enetc_ndev_priv *priv = netdev_priv(rx_ring->ndev); 1988 int rx_frm_cnt = 0, rx_byte_cnt = 0; 1989 struct enetc_bdr *tx_ring; 1990 int cleaned_cnt, i; 1991 u32 xdp_act; 1992 1993 cleaned_cnt = enetc_bd_unused(rx_ring); 1994 /* next descriptor to process */ 1995 i = rx_ring->next_to_clean; 1996 1997 enetc_lock_mdio(); 1998 1999 while (likely(rx_frm_cnt < work_limit)) { 2000 union enetc_rx_bd *rxbd, *orig_rxbd; 2001 struct xdp_buff xdp_buff; 2002 struct sk_buff *skb; 2003 int orig_i, err; 2004 u32 bd_status; 2005 2006 rxbd = enetc_rxbd(rx_ring, i); 2007 bd_status = le32_to_cpu(rxbd->r.lstatus); 2008 if (!bd_status) 2009 break; 2010 2011 enetc_wr_reg_hot(rx_ring->idr, BIT(rx_ring->index)); 2012 dma_rmb(); /* for reading other rxbd fields */ 2013 2014 if (enetc_check_bd_errors_and_consume(rx_ring, bd_status, 2015 &rxbd, &i)) 2016 break; 2017 2018 orig_rxbd = rxbd; 2019 orig_i = i; 2020 2021 enetc_build_xdp_buff(rx_ring, bd_status, &rxbd, &i, 2022 &cleaned_cnt, &xdp_buff); 2023 2024 /* When set, the outer VLAN header is extracted and reported 2025 * in the receive buffer descriptor. So rx_byte_cnt should 2026 * add the length of the extracted VLAN header. 2027 */ 2028 if (bd_status & ENETC_RXBD_FLAG_VLAN) 2029 rx_byte_cnt += VLAN_HLEN; 2030 rx_byte_cnt += xdp_get_buff_len(&xdp_buff); 2031 2032 xdp_act = bpf_prog_run_xdp(prog, &xdp_buff); 2033 2034 switch (xdp_act) { 2035 default: 2036 bpf_warn_invalid_xdp_action(rx_ring->ndev, prog, xdp_act); 2037 fallthrough; 2038 case XDP_ABORTED: 2039 trace_xdp_exception(rx_ring->ndev, prog, xdp_act); 2040 fallthrough; 2041 case XDP_DROP: 2042 enetc_xdp_drop(rx_ring, orig_i, i); 2043 rx_ring->stats.xdp_drops++; 2044 break; 2045 case XDP_PASS: 2046 skb = xdp_build_skb_from_buff(&xdp_buff); 2047 /* Probably under memory pressure, stop NAPI */ 2048 if (unlikely(!skb)) { 2049 enetc_xdp_drop(rx_ring, orig_i, i); 2050 rx_ring->stats.xdp_drops++; 2051 goto out; 2052 } 2053 2054 enetc_get_offloads(rx_ring, orig_rxbd, skb); 2055 2056 /* These buffers are about to be owned by the stack. 2057 * Update our buffer cache (the rx_swbd array elements) 2058 * with their other page halves. 2059 */ 2060 enetc_bulk_flip_buff(rx_ring, orig_i, i); 2061 2062 enetc_unlock_mdio(); 2063 napi_gro_receive(napi, skb); 2064 enetc_lock_mdio(); 2065 break; 2066 case XDP_TX: 2067 tx_ring = priv->xdp_tx_ring[rx_ring->index]; 2068 if (unlikely(test_bit(ENETC_TX_DOWN, &priv->flags))) { 2069 enetc_xdp_drop(rx_ring, orig_i, i); 2070 tx_ring->stats.xdp_tx_drops++; 2071 break; 2072 } 2073 2074 xdp_tx_bd_cnt = enetc_rx_swbd_to_xdp_tx_swbd(xdp_tx_arr, 2075 rx_ring, 2076 orig_i, i); 2077 2078 if (!enetc_xdp_tx(tx_ring, xdp_tx_arr, xdp_tx_bd_cnt)) { 2079 enetc_xdp_drop(rx_ring, orig_i, i); 2080 tx_ring->stats.xdp_tx_drops++; 2081 } else { 2082 tx_ring->stats.xdp_tx++; 2083 rx_ring->xdp.xdp_tx_in_flight += xdp_tx_bd_cnt; 2084 xdp_tx_frm_cnt++; 2085 /* The XDP_TX enqueue was successful, so we 2086 * need to scrub the RX software BDs because 2087 * the ownership of the buffers no longer 2088 * belongs to the RX ring, and we must prevent 2089 * enetc_refill_rx_ring() from reusing 2090 * rx_swbd->page. 2091 */ 2092 while (orig_i != i) { 2093 rx_ring->rx_swbd[orig_i].page = NULL; 2094 enetc_bdr_idx_inc(rx_ring, &orig_i); 2095 } 2096 } 2097 break; 2098 case XDP_REDIRECT: 2099 enetc_unlock_mdio(); 2100 err = xdp_do_redirect(rx_ring->ndev, &xdp_buff, prog); 2101 enetc_lock_mdio(); 2102 if (unlikely(err)) { 2103 enetc_xdp_drop(rx_ring, orig_i, i); 2104 rx_ring->stats.xdp_redirect_failures++; 2105 } else { 2106 enetc_bulk_flip_buff(rx_ring, orig_i, i); 2107 xdp_redirect_frm_cnt++; 2108 rx_ring->stats.xdp_redirect++; 2109 } 2110 } 2111 2112 rx_frm_cnt++; 2113 } 2114 2115 out: 2116 rx_ring->next_to_clean = i; 2117 2118 rx_ring->stats.packets += rx_frm_cnt; 2119 rx_ring->stats.bytes += rx_byte_cnt; 2120 2121 if (xdp_redirect_frm_cnt) { 2122 enetc_unlock_mdio(); 2123 xdp_do_flush(); 2124 enetc_lock_mdio(); 2125 } 2126 2127 if (xdp_tx_frm_cnt) 2128 enetc_update_tx_ring_tail(tx_ring); 2129 2130 if (cleaned_cnt > rx_ring->xdp.xdp_tx_in_flight) 2131 enetc_refill_rx_ring(rx_ring, enetc_bd_unused(rx_ring) - 2132 rx_ring->xdp.xdp_tx_in_flight); 2133 2134 enetc_unlock_mdio(); 2135 2136 return rx_frm_cnt; 2137 } 2138 2139 static int enetc_poll(struct napi_struct *napi, int budget) 2140 { 2141 struct enetc_int_vector 2142 *v = container_of(napi, struct enetc_int_vector, napi); 2143 struct enetc_bdr *rx_ring = &v->rx_ring; 2144 struct bpf_prog *prog; 2145 bool complete = true; 2146 int work_done; 2147 int i; 2148 2149 enetc_lock_mdio(); 2150 2151 for (i = 0; i < v->count_tx_rings; i++) 2152 if (!enetc_clean_tx_ring(&v->tx_ring[i], budget)) 2153 complete = false; 2154 enetc_unlock_mdio(); 2155 2156 prog = rx_ring->xdp.prog; 2157 if (prog) 2158 work_done = enetc_clean_rx_ring_xdp(rx_ring, napi, budget, prog); 2159 else 2160 work_done = enetc_clean_rx_ring(rx_ring, napi, budget); 2161 if (work_done == budget) 2162 complete = false; 2163 if (work_done) 2164 v->rx_napi_work = true; 2165 2166 if (!complete) 2167 return budget; 2168 2169 napi_complete_done(napi, work_done); 2170 2171 if (likely(v->rx_dim_en)) 2172 enetc_rx_net_dim(v); 2173 2174 v->rx_napi_work = false; 2175 2176 enetc_lock_mdio(); 2177 /* enable interrupts */ 2178 enetc_wr_reg_hot(v->rbier, ENETC_RBIER_RXTIE); 2179 2180 for_each_set_bit(i, &v->tx_rings_map, ENETC_MAX_NUM_TXQS) 2181 enetc_wr_reg_hot(v->tbier_base + ENETC_BDR_OFF(i), 2182 ENETC_TBIER_TXTIE); 2183 2184 enetc_unlock_mdio(); 2185 2186 return work_done; 2187 } 2188 2189 /* Probing and Init */ 2190 #define ENETC_MAX_RFS_SIZE 64 2191 void enetc_get_si_caps(struct enetc_si *si) 2192 { 2193 struct enetc_hw *hw = &si->hw; 2194 u32 val; 2195 2196 /* find out how many of various resources we have to work with */ 2197 val = enetc_rd(hw, ENETC_SICAPR0); 2198 si->num_rx_rings = (val >> 16) & 0xff; 2199 si->num_tx_rings = val & 0xff; 2200 2201 val = enetc_rd(hw, ENETC_SIPCAPR0); 2202 if (val & ENETC_SIPCAPR0_RFS) { 2203 val = enetc_rd(hw, ENETC_SIRFSCAPR); 2204 si->num_fs_entries = ENETC_SIRFSCAPR_GET_NUM_RFS(val); 2205 si->num_fs_entries = min(si->num_fs_entries, ENETC_MAX_RFS_SIZE); 2206 } else { 2207 /* ENETC which not supports RFS */ 2208 si->num_fs_entries = 0; 2209 } 2210 2211 si->num_rss = 0; 2212 val = enetc_rd(hw, ENETC_SIPCAPR0); 2213 if (val & ENETC_SIPCAPR0_RSS) { 2214 u32 rss; 2215 2216 rss = enetc_rd(hw, ENETC_SIRSSCAPR); 2217 si->num_rss = ENETC_SIRSSCAPR_GET_NUM_RSS(rss); 2218 } 2219 2220 if (val & ENETC_SIPCAPR0_LSO) 2221 si->hw_features |= ENETC_SI_F_LSO; 2222 } 2223 EXPORT_SYMBOL_GPL(enetc_get_si_caps); 2224 2225 static int enetc_dma_alloc_bdr(struct enetc_bdr_resource *res) 2226 { 2227 size_t bd_base_size = res->bd_count * res->bd_size; 2228 2229 res->bd_base = dma_alloc_coherent(res->dev, bd_base_size, 2230 &res->bd_dma_base, GFP_KERNEL); 2231 if (!res->bd_base) 2232 return -ENOMEM; 2233 2234 /* h/w requires 128B alignment */ 2235 if (!IS_ALIGNED(res->bd_dma_base, 128)) { 2236 dma_free_coherent(res->dev, bd_base_size, res->bd_base, 2237 res->bd_dma_base); 2238 return -EINVAL; 2239 } 2240 2241 return 0; 2242 } 2243 2244 static void enetc_dma_free_bdr(const struct enetc_bdr_resource *res) 2245 { 2246 size_t bd_base_size = res->bd_count * res->bd_size; 2247 2248 dma_free_coherent(res->dev, bd_base_size, res->bd_base, 2249 res->bd_dma_base); 2250 } 2251 2252 static int enetc_alloc_tx_resource(struct enetc_bdr_resource *res, 2253 struct device *dev, size_t bd_count) 2254 { 2255 int err; 2256 2257 res->dev = dev; 2258 res->bd_count = bd_count; 2259 res->bd_size = sizeof(union enetc_tx_bd); 2260 2261 res->tx_swbd = vcalloc(bd_count, sizeof(*res->tx_swbd)); 2262 if (!res->tx_swbd) 2263 return -ENOMEM; 2264 2265 err = enetc_dma_alloc_bdr(res); 2266 if (err) 2267 goto err_alloc_bdr; 2268 2269 res->tso_headers = dma_alloc_coherent(dev, bd_count * TSO_HEADER_SIZE, 2270 &res->tso_headers_dma, 2271 GFP_KERNEL); 2272 if (!res->tso_headers) { 2273 err = -ENOMEM; 2274 goto err_alloc_tso; 2275 } 2276 2277 return 0; 2278 2279 err_alloc_tso: 2280 enetc_dma_free_bdr(res); 2281 err_alloc_bdr: 2282 vfree(res->tx_swbd); 2283 res->tx_swbd = NULL; 2284 2285 return err; 2286 } 2287 2288 static void enetc_free_tx_resource(const struct enetc_bdr_resource *res) 2289 { 2290 dma_free_coherent(res->dev, res->bd_count * TSO_HEADER_SIZE, 2291 res->tso_headers, res->tso_headers_dma); 2292 enetc_dma_free_bdr(res); 2293 vfree(res->tx_swbd); 2294 } 2295 2296 static struct enetc_bdr_resource * 2297 enetc_alloc_tx_resources(struct enetc_ndev_priv *priv) 2298 { 2299 struct enetc_bdr_resource *tx_res; 2300 int i, err; 2301 2302 tx_res = kzalloc_objs(*tx_res, priv->num_tx_rings); 2303 if (!tx_res) 2304 return ERR_PTR(-ENOMEM); 2305 2306 for (i = 0; i < priv->num_tx_rings; i++) { 2307 struct enetc_bdr *tx_ring = priv->tx_ring[i]; 2308 2309 err = enetc_alloc_tx_resource(&tx_res[i], tx_ring->dev, 2310 tx_ring->bd_count); 2311 if (err) 2312 goto fail; 2313 } 2314 2315 return tx_res; 2316 2317 fail: 2318 while (i-- > 0) 2319 enetc_free_tx_resource(&tx_res[i]); 2320 2321 kfree(tx_res); 2322 2323 return ERR_PTR(err); 2324 } 2325 2326 static void enetc_free_tx_resources(const struct enetc_bdr_resource *tx_res, 2327 size_t num_resources) 2328 { 2329 size_t i; 2330 2331 for (i = 0; i < num_resources; i++) 2332 enetc_free_tx_resource(&tx_res[i]); 2333 2334 kfree(tx_res); 2335 } 2336 2337 static int enetc_alloc_rx_resource(struct enetc_bdr_resource *res, 2338 struct device *dev, size_t bd_count, 2339 bool extended) 2340 { 2341 int err; 2342 2343 res->dev = dev; 2344 res->bd_count = bd_count; 2345 res->bd_size = sizeof(union enetc_rx_bd); 2346 if (extended) 2347 res->bd_size *= 2; 2348 2349 res->rx_swbd = vcalloc(bd_count, sizeof(struct enetc_rx_swbd)); 2350 if (!res->rx_swbd) 2351 return -ENOMEM; 2352 2353 err = enetc_dma_alloc_bdr(res); 2354 if (err) { 2355 vfree(res->rx_swbd); 2356 return err; 2357 } 2358 2359 return 0; 2360 } 2361 2362 static void enetc_free_rx_resource(const struct enetc_bdr_resource *res) 2363 { 2364 enetc_dma_free_bdr(res); 2365 vfree(res->rx_swbd); 2366 } 2367 2368 static struct enetc_bdr_resource * 2369 enetc_alloc_rx_resources(struct enetc_ndev_priv *priv, bool extended) 2370 { 2371 struct enetc_bdr_resource *rx_res; 2372 int i, err; 2373 2374 rx_res = kzalloc_objs(*rx_res, priv->num_rx_rings); 2375 if (!rx_res) 2376 return ERR_PTR(-ENOMEM); 2377 2378 for (i = 0; i < priv->num_rx_rings; i++) { 2379 struct enetc_bdr *rx_ring = priv->rx_ring[i]; 2380 2381 err = enetc_alloc_rx_resource(&rx_res[i], rx_ring->dev, 2382 rx_ring->bd_count, extended); 2383 if (err) 2384 goto fail; 2385 } 2386 2387 return rx_res; 2388 2389 fail: 2390 while (i-- > 0) 2391 enetc_free_rx_resource(&rx_res[i]); 2392 2393 kfree(rx_res); 2394 2395 return ERR_PTR(err); 2396 } 2397 2398 static void enetc_free_rx_resources(const struct enetc_bdr_resource *rx_res, 2399 size_t num_resources) 2400 { 2401 size_t i; 2402 2403 for (i = 0; i < num_resources; i++) 2404 enetc_free_rx_resource(&rx_res[i]); 2405 2406 kfree(rx_res); 2407 } 2408 2409 static void enetc_assign_tx_resource(struct enetc_bdr *tx_ring, 2410 const struct enetc_bdr_resource *res) 2411 { 2412 tx_ring->bd_base = res ? res->bd_base : NULL; 2413 tx_ring->bd_dma_base = res ? res->bd_dma_base : 0; 2414 tx_ring->tx_swbd = res ? res->tx_swbd : NULL; 2415 tx_ring->tso_headers = res ? res->tso_headers : NULL; 2416 tx_ring->tso_headers_dma = res ? res->tso_headers_dma : 0; 2417 } 2418 2419 static void enetc_assign_rx_resource(struct enetc_bdr *rx_ring, 2420 const struct enetc_bdr_resource *res) 2421 { 2422 rx_ring->bd_base = res ? res->bd_base : NULL; 2423 rx_ring->bd_dma_base = res ? res->bd_dma_base : 0; 2424 rx_ring->rx_swbd = res ? res->rx_swbd : NULL; 2425 } 2426 2427 static void enetc_assign_tx_resources(struct enetc_ndev_priv *priv, 2428 const struct enetc_bdr_resource *res) 2429 { 2430 int i; 2431 2432 if (priv->tx_res) 2433 enetc_free_tx_resources(priv->tx_res, priv->num_tx_rings); 2434 2435 for (i = 0; i < priv->num_tx_rings; i++) { 2436 enetc_assign_tx_resource(priv->tx_ring[i], 2437 res ? &res[i] : NULL); 2438 } 2439 2440 priv->tx_res = res; 2441 } 2442 2443 static void enetc_assign_rx_resources(struct enetc_ndev_priv *priv, 2444 const struct enetc_bdr_resource *res) 2445 { 2446 int i; 2447 2448 if (priv->rx_res) 2449 enetc_free_rx_resources(priv->rx_res, priv->num_rx_rings); 2450 2451 for (i = 0; i < priv->num_rx_rings; i++) { 2452 enetc_assign_rx_resource(priv->rx_ring[i], 2453 res ? &res[i] : NULL); 2454 } 2455 2456 priv->rx_res = res; 2457 } 2458 2459 static void enetc_free_tx_ring(struct enetc_bdr *tx_ring) 2460 { 2461 int i; 2462 2463 for (i = 0; i < tx_ring->bd_count; i++) { 2464 struct enetc_tx_swbd *tx_swbd = &tx_ring->tx_swbd[i]; 2465 2466 enetc_free_tx_frame(tx_ring, tx_swbd); 2467 } 2468 } 2469 2470 static void enetc_free_rx_ring(struct enetc_bdr *rx_ring) 2471 { 2472 int i; 2473 2474 for (i = 0; i < rx_ring->bd_count; i++) { 2475 struct enetc_rx_swbd *rx_swbd = &rx_ring->rx_swbd[i]; 2476 2477 if (!rx_swbd->page) 2478 continue; 2479 2480 dma_unmap_page(rx_ring->dev, rx_swbd->dma, PAGE_SIZE, 2481 rx_swbd->dir); 2482 __free_page(rx_swbd->page); 2483 rx_swbd->page = NULL; 2484 } 2485 } 2486 2487 static void enetc_free_rxtx_rings(struct enetc_ndev_priv *priv) 2488 { 2489 int i; 2490 2491 for (i = 0; i < priv->num_rx_rings; i++) 2492 enetc_free_rx_ring(priv->rx_ring[i]); 2493 2494 for (i = 0; i < priv->num_tx_rings; i++) 2495 enetc_free_tx_ring(priv->tx_ring[i]); 2496 } 2497 2498 static int enetc_setup_default_rss_table(struct enetc_si *si, int num_groups) 2499 { 2500 int *rss_table; 2501 int i; 2502 2503 rss_table = kmalloc_objs(*rss_table, si->num_rss); 2504 if (!rss_table) 2505 return -ENOMEM; 2506 2507 /* Set up RSS table defaults */ 2508 for (i = 0; i < si->num_rss; i++) 2509 rss_table[i] = i % num_groups; 2510 2511 si->ops->set_rss_table(si, rss_table, si->num_rss); 2512 2513 kfree(rss_table); 2514 2515 return 0; 2516 } 2517 2518 static void enetc_set_lso_flags_mask(struct enetc_hw *hw) 2519 { 2520 enetc_wr(hw, ENETC4_SILSOSFMR0, 2521 SILSOSFMR0_VAL_SET(ENETC4_TCP_NL_SEG_FLAGS_DMASK, 2522 ENETC4_TCP_NL_SEG_FLAGS_DMASK)); 2523 enetc_wr(hw, ENETC4_SILSOSFMR1, 0); 2524 } 2525 2526 static void enetc_set_rss(struct net_device *ndev, int en) 2527 { 2528 struct enetc_ndev_priv *priv = netdev_priv(ndev); 2529 struct enetc_hw *hw = &priv->si->hw; 2530 u32 reg; 2531 2532 enetc_wr(hw, ENETC_SIRBGCR, priv->num_rx_rings); 2533 2534 reg = enetc_rd(hw, ENETC_SIMR); 2535 reg &= ~ENETC_SIMR_RSSE; 2536 reg |= (en) ? ENETC_SIMR_RSSE : 0; 2537 enetc_wr(hw, ENETC_SIMR, reg); 2538 } 2539 2540 int enetc_configure_si(struct enetc_ndev_priv *priv) 2541 { 2542 struct enetc_si *si = priv->si; 2543 struct enetc_hw *hw = &si->hw; 2544 int err; 2545 2546 if (is_enetc_rev1(si)) { 2547 /* set SI cache attributes */ 2548 enetc_wr(hw, ENETC_SICAR0, 2549 ENETC_SICAR_RD_COHERENT | ENETC_SICAR_WR_COHERENT); 2550 enetc_wr(hw, ENETC_SICAR1, ENETC_SICAR_MSI); 2551 } 2552 2553 /* enable SI */ 2554 enetc_wr(hw, ENETC_SIMR, ENETC_SIMR_EN); 2555 2556 if (si->hw_features & ENETC_SI_F_LSO) 2557 enetc_set_lso_flags_mask(hw); 2558 2559 if (si->num_rss) { 2560 err = enetc_setup_default_rss_table(si, priv->num_rx_rings); 2561 if (err) 2562 return err; 2563 2564 if (priv->ndev->features & NETIF_F_RXHASH) 2565 enetc_set_rss(priv->ndev, true); 2566 } 2567 2568 return 0; 2569 } 2570 EXPORT_SYMBOL_GPL(enetc_configure_si); 2571 2572 void enetc_init_si_rings_params(struct enetc_ndev_priv *priv) 2573 { 2574 struct enetc_si *si = priv->si; 2575 int cpus = num_online_cpus(); 2576 2577 priv->tx_bd_count = ENETC_TX_RING_DEFAULT_SIZE; 2578 priv->rx_bd_count = ENETC_RX_RING_DEFAULT_SIZE; 2579 2580 /* Enable all available TX rings in order to configure as many 2581 * priorities as possible, when needed. 2582 * TODO: Make # of TX rings run-time configurable 2583 */ 2584 priv->num_rx_rings = min_t(int, cpus, si->num_rx_rings); 2585 priv->num_tx_rings = si->num_tx_rings; 2586 priv->bdr_int_num = priv->num_rx_rings; 2587 priv->ic_mode = ENETC_IC_RX_ADAPTIVE | ENETC_IC_TX_MANUAL; 2588 priv->tx_ictt = enetc_usecs_to_cycles(600, priv->sysclk_freq); 2589 } 2590 EXPORT_SYMBOL_GPL(enetc_init_si_rings_params); 2591 2592 int enetc_alloc_si_resources(struct enetc_ndev_priv *priv) 2593 { 2594 struct enetc_si *si = priv->si; 2595 2596 priv->cls_rules = kzalloc_objs(*priv->cls_rules, si->num_fs_entries); 2597 if (!priv->cls_rules) 2598 return -ENOMEM; 2599 2600 return 0; 2601 } 2602 EXPORT_SYMBOL_GPL(enetc_alloc_si_resources); 2603 2604 void enetc_free_si_resources(struct enetc_ndev_priv *priv) 2605 { 2606 kfree(priv->cls_rules); 2607 } 2608 EXPORT_SYMBOL_GPL(enetc_free_si_resources); 2609 2610 static void enetc_setup_txbdr(struct enetc_hw *hw, struct enetc_bdr *tx_ring) 2611 { 2612 struct enetc_si *si = container_of(hw, struct enetc_si, hw); 2613 int idx = tx_ring->index; 2614 u32 tbmr; 2615 2616 enetc_txbdr_wr(hw, idx, ENETC_TBBAR0, 2617 lower_32_bits(tx_ring->bd_dma_base)); 2618 2619 enetc_txbdr_wr(hw, idx, ENETC_TBBAR1, 2620 upper_32_bits(tx_ring->bd_dma_base)); 2621 2622 WARN_ON(!IS_ALIGNED(tx_ring->bd_count, 64)); /* multiple of 64 */ 2623 enetc_txbdr_wr(hw, idx, ENETC_TBLENR, 2624 ENETC_RTBLENR_LEN(tx_ring->bd_count)); 2625 2626 /* For ENETC v1, clearing PI/CI registers for Tx not supported, 2627 * adjust sw indexes 2628 */ 2629 tx_ring->next_to_use = enetc_txbdr_rd(hw, idx, ENETC_TBPIR); 2630 tx_ring->next_to_clean = enetc_txbdr_rd(hw, idx, ENETC_TBCIR); 2631 2632 if (tx_ring->next_to_use != tx_ring->next_to_clean && 2633 !is_enetc_rev1(si)) { 2634 tx_ring->next_to_use = 0; 2635 tx_ring->next_to_clean = 0; 2636 enetc_txbdr_wr(hw, idx, ENETC_TBPIR, 0); 2637 enetc_txbdr_wr(hw, idx, ENETC_TBCIR, 0); 2638 } 2639 2640 /* enable Tx ints by setting pkt thr to 1 */ 2641 enetc_txbdr_wr(hw, idx, ENETC_TBICR0, ENETC_TBICR0_ICEN | 0x1); 2642 2643 tbmr = ENETC_TBMR_SET_PRIO(tx_ring->prio); 2644 if (tx_ring->ndev->features & NETIF_F_HW_VLAN_CTAG_TX) 2645 tbmr |= ENETC_TBMR_VIH; 2646 2647 /* enable ring */ 2648 enetc_txbdr_wr(hw, idx, ENETC_TBMR, tbmr); 2649 2650 tx_ring->tpir = hw->reg + ENETC_BDR(TX, idx, ENETC_TBPIR); 2651 tx_ring->tcir = hw->reg + ENETC_BDR(TX, idx, ENETC_TBCIR); 2652 tx_ring->idr = hw->reg + ENETC_SITXIDR; 2653 } 2654 2655 static void enetc_setup_rxbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring, 2656 bool extended) 2657 { 2658 int idx = rx_ring->index; 2659 2660 enetc_rxbdr_wr(hw, idx, ENETC_RBBAR0, 2661 lower_32_bits(rx_ring->bd_dma_base)); 2662 2663 enetc_rxbdr_wr(hw, idx, ENETC_RBBAR1, 2664 upper_32_bits(rx_ring->bd_dma_base)); 2665 2666 WARN_ON(!IS_ALIGNED(rx_ring->bd_count, 64)); /* multiple of 64 */ 2667 enetc_rxbdr_wr(hw, idx, ENETC_RBLENR, 2668 ENETC_RTBLENR_LEN(rx_ring->bd_count)); 2669 2670 if (rx_ring->xdp.prog) 2671 enetc_rxbdr_wr(hw, idx, ENETC_RBBSR, ENETC_RXB_DMA_SIZE_XDP); 2672 else 2673 enetc_rxbdr_wr(hw, idx, ENETC_RBBSR, ENETC_RXB_DMA_SIZE); 2674 2675 /* Also prepare the consumer index in case page allocation never 2676 * succeeds. In that case, hardware will never advance producer index 2677 * to match consumer index, and will drop all frames. 2678 */ 2679 enetc_rxbdr_wr(hw, idx, ENETC_RBPIR, 0); 2680 enetc_rxbdr_wr(hw, idx, ENETC_RBCIR, 1); 2681 2682 /* enable Rx ints by setting pkt thr to 1 */ 2683 enetc_rxbdr_wr(hw, idx, ENETC_RBICR0, ENETC_RBICR0_ICEN | 0x1); 2684 2685 rx_ring->ext_en = extended; 2686 rx_ring->rcir = hw->reg + ENETC_BDR(RX, idx, ENETC_RBCIR); 2687 rx_ring->idr = hw->reg + ENETC_SIRXIDR; 2688 2689 rx_ring->next_to_clean = 0; 2690 rx_ring->next_to_use = 0; 2691 rx_ring->next_to_alloc = 0; 2692 2693 enetc_lock_mdio(); 2694 enetc_refill_rx_ring(rx_ring, enetc_bd_unused(rx_ring)); 2695 enetc_unlock_mdio(); 2696 } 2697 2698 static void enetc_setup_bdrs(struct enetc_ndev_priv *priv, bool extended) 2699 { 2700 struct enetc_hw *hw = &priv->si->hw; 2701 int i; 2702 2703 for (i = 0; i < priv->num_tx_rings; i++) 2704 enetc_setup_txbdr(hw, priv->tx_ring[i]); 2705 2706 for (i = 0; i < priv->num_rx_rings; i++) 2707 enetc_setup_rxbdr(hw, priv->rx_ring[i], extended); 2708 } 2709 2710 static void enetc_enable_txbdr(struct enetc_hw *hw, struct enetc_bdr *tx_ring) 2711 { 2712 int idx = tx_ring->index; 2713 u32 tbmr; 2714 2715 tbmr = enetc_txbdr_rd(hw, idx, ENETC_TBMR); 2716 tbmr |= ENETC_TBMR_EN; 2717 enetc_txbdr_wr(hw, idx, ENETC_TBMR, tbmr); 2718 } 2719 2720 static void enetc_enable_rxbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring) 2721 { 2722 struct enetc_ndev_priv *priv = netdev_priv(rx_ring->ndev); 2723 int idx = rx_ring->index; 2724 u32 rbmr = ENETC_RBMR_EN; 2725 2726 if (rx_ring->ext_en) 2727 rbmr |= ENETC_RBMR_BDS; 2728 2729 if (rx_ring->ndev->features & NETIF_F_HW_VLAN_CTAG_RX) 2730 rbmr |= ENETC_RBMR_VTE; 2731 2732 if (test_bit(ENETC_RXBDR_CM, &priv->flags)) 2733 rbmr |= ENETC_RBMR_CM; 2734 2735 enetc_rxbdr_wr(hw, idx, ENETC_RBMR, rbmr); 2736 } 2737 2738 static void enetc_enable_rx_bdrs(struct enetc_ndev_priv *priv) 2739 { 2740 struct enetc_si *si = priv->si; 2741 struct enetc_hw *hw = &si->hw; 2742 int i; 2743 2744 spin_lock(&si->gen_lock); 2745 2746 for (i = 0; i < priv->num_rx_rings; i++) 2747 enetc_enable_rxbdr(hw, priv->rx_ring[i]); 2748 2749 spin_unlock(&si->gen_lock); 2750 } 2751 2752 static void enetc_enable_tx_bdrs(struct enetc_ndev_priv *priv) 2753 { 2754 struct enetc_hw *hw = &priv->si->hw; 2755 int i; 2756 2757 for (i = 0; i < priv->num_tx_rings; i++) 2758 enetc_enable_txbdr(hw, priv->tx_ring[i]); 2759 } 2760 2761 static void enetc_disable_rxbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring) 2762 { 2763 int idx = rx_ring->index; 2764 2765 /* disable EN bit on ring */ 2766 enetc_rxbdr_wr(hw, idx, ENETC_RBMR, 0); 2767 } 2768 2769 static void enetc_disable_txbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring) 2770 { 2771 int idx = rx_ring->index; 2772 2773 /* disable EN bit on ring */ 2774 enetc_txbdr_wr(hw, idx, ENETC_TBMR, 0); 2775 } 2776 2777 static void enetc_disable_rx_bdrs(struct enetc_ndev_priv *priv) 2778 { 2779 struct enetc_si *si = priv->si; 2780 struct enetc_hw *hw = &si->hw; 2781 int i; 2782 2783 spin_lock(&si->gen_lock); 2784 2785 for (i = 0; i < priv->num_rx_rings; i++) 2786 enetc_disable_rxbdr(hw, priv->rx_ring[i]); 2787 2788 spin_unlock(&si->gen_lock); 2789 } 2790 2791 static void enetc_disable_tx_bdrs(struct enetc_ndev_priv *priv) 2792 { 2793 struct enetc_hw *hw = &priv->si->hw; 2794 int i; 2795 2796 for (i = 0; i < priv->num_tx_rings; i++) 2797 enetc_disable_txbdr(hw, priv->tx_ring[i]); 2798 } 2799 2800 static void enetc_wait_txbdr(struct enetc_hw *hw, struct enetc_bdr *tx_ring) 2801 { 2802 int delay = 8, timeout = 100; 2803 int idx = tx_ring->index; 2804 2805 /* wait for busy to clear */ 2806 while (delay < timeout && 2807 enetc_txbdr_rd(hw, idx, ENETC_TBSR) & ENETC_TBSR_BUSY) { 2808 msleep(delay); 2809 delay *= 2; 2810 } 2811 2812 if (delay >= timeout) 2813 netdev_warn(tx_ring->ndev, "timeout for tx ring #%d clear\n", 2814 idx); 2815 } 2816 2817 static void enetc_wait_bdrs(struct enetc_ndev_priv *priv) 2818 { 2819 struct enetc_hw *hw = &priv->si->hw; 2820 int i; 2821 2822 for (i = 0; i < priv->num_tx_rings; i++) 2823 enetc_wait_txbdr(hw, priv->tx_ring[i]); 2824 } 2825 2826 static int enetc_setup_irqs(struct enetc_ndev_priv *priv) 2827 { 2828 struct pci_dev *pdev = priv->si->pdev; 2829 struct enetc_hw *hw = &priv->si->hw; 2830 int i, j, err; 2831 2832 for (i = 0; i < priv->bdr_int_num; i++) { 2833 int irq = pci_irq_vector(pdev, ENETC_BDR_INT_BASE_IDX + i); 2834 struct enetc_int_vector *v = priv->int_vector[i]; 2835 int entry = ENETC_BDR_INT_BASE_IDX + i; 2836 2837 snprintf(v->name, sizeof(v->name), "%s-rxtx%d", 2838 priv->ndev->name, i); 2839 err = request_irq(irq, enetc_msix, IRQF_NO_AUTOEN, v->name, v); 2840 if (err) { 2841 dev_err(priv->dev, "request_irq() failed!\n"); 2842 goto irq_err; 2843 } 2844 2845 v->tbier_base = hw->reg + ENETC_BDR(TX, 0, ENETC_TBIER); 2846 v->rbier = hw->reg + ENETC_BDR(RX, i, ENETC_RBIER); 2847 v->ricr1 = hw->reg + ENETC_BDR(RX, i, ENETC_RBICR1); 2848 2849 enetc_wr(hw, ENETC_SIMSIRRV(i), entry); 2850 2851 for (j = 0; j < v->count_tx_rings; j++) { 2852 int idx = v->tx_ring[j].index; 2853 2854 enetc_wr(hw, ENETC_SIMSITRV(idx), entry); 2855 } 2856 irq_set_affinity_hint(irq, get_cpu_mask(i % num_online_cpus())); 2857 } 2858 2859 return 0; 2860 2861 irq_err: 2862 while (i--) { 2863 int irq = pci_irq_vector(pdev, ENETC_BDR_INT_BASE_IDX + i); 2864 2865 irq_set_affinity_hint(irq, NULL); 2866 free_irq(irq, priv->int_vector[i]); 2867 } 2868 2869 return err; 2870 } 2871 2872 static void enetc_free_irqs(struct enetc_ndev_priv *priv) 2873 { 2874 struct pci_dev *pdev = priv->si->pdev; 2875 int i; 2876 2877 for (i = 0; i < priv->bdr_int_num; i++) { 2878 int irq = pci_irq_vector(pdev, ENETC_BDR_INT_BASE_IDX + i); 2879 2880 irq_set_affinity_hint(irq, NULL); 2881 free_irq(irq, priv->int_vector[i]); 2882 } 2883 } 2884 2885 static void enetc_setup_interrupts(struct enetc_ndev_priv *priv) 2886 { 2887 struct enetc_hw *hw = &priv->si->hw; 2888 u32 icpt, ictt; 2889 int i; 2890 2891 /* enable Tx & Rx event indication */ 2892 if (priv->ic_mode & 2893 (ENETC_IC_RX_MANUAL | ENETC_IC_RX_ADAPTIVE)) { 2894 icpt = ENETC_RBICR0_SET_ICPT(ENETC_RXIC_PKTTHR); 2895 /* init to non-0 minimum, will be adjusted later */ 2896 ictt = 0x1; 2897 } else { 2898 icpt = 0x1; /* enable Rx ints by setting pkt thr to 1 */ 2899 ictt = 0; 2900 } 2901 2902 for (i = 0; i < priv->num_rx_rings; i++) { 2903 enetc_rxbdr_wr(hw, i, ENETC_RBICR1, ictt); 2904 enetc_rxbdr_wr(hw, i, ENETC_RBICR0, ENETC_RBICR0_ICEN | icpt); 2905 enetc_rxbdr_wr(hw, i, ENETC_RBIER, ENETC_RBIER_RXTIE); 2906 } 2907 2908 if (priv->ic_mode & ENETC_IC_TX_MANUAL) 2909 icpt = ENETC_TBICR0_SET_ICPT(ENETC_TXIC_PKTTHR); 2910 else 2911 icpt = 0x1; /* enable Tx ints by setting pkt thr to 1 */ 2912 2913 for (i = 0; i < priv->num_tx_rings; i++) { 2914 enetc_txbdr_wr(hw, i, ENETC_TBICR1, priv->tx_ictt); 2915 enetc_txbdr_wr(hw, i, ENETC_TBICR0, ENETC_TBICR0_ICEN | icpt); 2916 enetc_txbdr_wr(hw, i, ENETC_TBIER, ENETC_TBIER_TXTIE); 2917 } 2918 } 2919 2920 static void enetc_clear_interrupts(struct enetc_ndev_priv *priv) 2921 { 2922 struct enetc_hw *hw = &priv->si->hw; 2923 int i; 2924 2925 for (i = 0; i < priv->num_tx_rings; i++) 2926 enetc_txbdr_wr(hw, i, ENETC_TBIER, 0); 2927 2928 for (i = 0; i < priv->num_rx_rings; i++) 2929 enetc_rxbdr_wr(hw, i, ENETC_RBIER, 0); 2930 } 2931 2932 static int enetc_phylink_connect(struct net_device *ndev) 2933 { 2934 struct enetc_ndev_priv *priv = netdev_priv(ndev); 2935 struct enetc_si *si = priv->si; 2936 struct ethtool_keee edata; 2937 int err; 2938 2939 if (!priv->phylink) { 2940 /* phy-less mode */ 2941 if (!si->ops->vf_reg_link_status_notifier) 2942 goto carrier_on; 2943 2944 /* For phy-less VFs on ENETC v4, attempt to register a link 2945 * status notifier with the PF via the VSI-to-PSI messaging 2946 * channel. If registration succeeds, the PF will immediately 2947 * send the current link status and broadcast future link 2948 * transitions; carrier state is then managed in 2949 * enetc_vf_msg_handle_link_status(). If registration fails, 2950 * fall back to the LS1028A behaviour and assert carrier 2951 * unconditionally via netif_carrier_on(). 2952 */ 2953 if (!si->ops->vf_reg_link_status_notifier(si)) 2954 return 0; 2955 2956 dev_warn(&ndev->dev, 2957 "Link status notifier registration failed\n"); 2958 2959 carrier_on: 2960 netif_carrier_on(ndev); 2961 return 0; 2962 } 2963 2964 err = phylink_of_phy_connect(priv->phylink, priv->dev->of_node, 0); 2965 if (err) { 2966 dev_err(&ndev->dev, "could not attach to PHY\n"); 2967 return err; 2968 } 2969 2970 /* disable EEE autoneg, until ENETC driver supports it */ 2971 memset(&edata, 0, sizeof(struct ethtool_keee)); 2972 phylink_ethtool_set_eee(priv->phylink, &edata); 2973 2974 phylink_start(priv->phylink); 2975 2976 return 0; 2977 } 2978 2979 static void enetc_tx_onestep_tstamp(struct work_struct *work) 2980 { 2981 struct enetc_ndev_priv *priv; 2982 struct sk_buff *skb; 2983 2984 priv = container_of(work, struct enetc_ndev_priv, tx_onestep_tstamp); 2985 2986 netif_tx_lock_bh(priv->ndev); 2987 2988 clear_bit_unlock(ENETC_TX_ONESTEP_TSTAMP_IN_PROGRESS, &priv->flags); 2989 skb = skb_dequeue(&priv->tx_skbs); 2990 if (skb) 2991 enetc_start_xmit(skb, priv->ndev); 2992 2993 netif_tx_unlock_bh(priv->ndev); 2994 } 2995 2996 static void enetc_tx_onestep_tstamp_init(struct enetc_ndev_priv *priv) 2997 { 2998 INIT_WORK(&priv->tx_onestep_tstamp, enetc_tx_onestep_tstamp); 2999 skb_queue_head_init(&priv->tx_skbs); 3000 } 3001 3002 void enetc_start(struct net_device *ndev) 3003 { 3004 struct enetc_ndev_priv *priv = netdev_priv(ndev); 3005 int i; 3006 3007 enetc_setup_interrupts(priv); 3008 3009 for (i = 0; i < priv->bdr_int_num; i++) { 3010 int irq = pci_irq_vector(priv->si->pdev, 3011 ENETC_BDR_INT_BASE_IDX + i); 3012 3013 napi_enable(&priv->int_vector[i]->napi); 3014 enable_irq(irq); 3015 } 3016 3017 enetc_enable_tx_bdrs(priv); 3018 3019 enetc_enable_rx_bdrs(priv); 3020 3021 netif_tx_start_all_queues(ndev); 3022 3023 clear_bit(ENETC_TX_DOWN, &priv->flags); 3024 } 3025 EXPORT_SYMBOL_GPL(enetc_start); 3026 3027 int enetc_open(struct net_device *ndev) 3028 { 3029 struct enetc_ndev_priv *priv = netdev_priv(ndev); 3030 struct enetc_bdr_resource *tx_res, *rx_res; 3031 bool extended; 3032 int err; 3033 3034 extended = !!(priv->active_offloads & ENETC_F_RX_TSTAMP); 3035 3036 err = clk_prepare_enable(priv->ref_clk); 3037 if (err) 3038 return err; 3039 3040 err = enetc_setup_irqs(priv); 3041 if (err) 3042 goto err_setup_irqs; 3043 3044 tx_res = enetc_alloc_tx_resources(priv); 3045 if (IS_ERR(tx_res)) { 3046 err = PTR_ERR(tx_res); 3047 goto err_alloc_tx; 3048 } 3049 3050 rx_res = enetc_alloc_rx_resources(priv, extended); 3051 if (IS_ERR(rx_res)) { 3052 err = PTR_ERR(rx_res); 3053 goto err_alloc_rx; 3054 } 3055 3056 err = enetc_phylink_connect(ndev); 3057 if (err) 3058 goto err_phy_connect; 3059 3060 enetc_tx_onestep_tstamp_init(priv); 3061 enetc_assign_tx_resources(priv, tx_res); 3062 enetc_assign_rx_resources(priv, rx_res); 3063 enetc_setup_bdrs(priv, extended); 3064 enetc_start(ndev); 3065 3066 return 0; 3067 3068 err_phy_connect: 3069 enetc_free_rx_resources(rx_res, priv->num_rx_rings); 3070 err_alloc_rx: 3071 enetc_free_tx_resources(tx_res, priv->num_tx_rings); 3072 err_alloc_tx: 3073 enetc_free_irqs(priv); 3074 err_setup_irqs: 3075 clk_disable_unprepare(priv->ref_clk); 3076 3077 return err; 3078 } 3079 EXPORT_SYMBOL_GPL(enetc_open); 3080 3081 void enetc_stop(struct net_device *ndev) 3082 { 3083 struct enetc_ndev_priv *priv = netdev_priv(ndev); 3084 int i; 3085 3086 set_bit(ENETC_TX_DOWN, &priv->flags); 3087 3088 netif_tx_stop_all_queues(ndev); 3089 3090 enetc_disable_rx_bdrs(priv); 3091 3092 enetc_wait_bdrs(priv); 3093 3094 enetc_disable_tx_bdrs(priv); 3095 3096 for (i = 0; i < priv->bdr_int_num; i++) { 3097 int irq = pci_irq_vector(priv->si->pdev, 3098 ENETC_BDR_INT_BASE_IDX + i); 3099 3100 disable_irq(irq); 3101 napi_synchronize(&priv->int_vector[i]->napi); 3102 napi_disable(&priv->int_vector[i]->napi); 3103 } 3104 3105 enetc_clear_interrupts(priv); 3106 } 3107 EXPORT_SYMBOL_GPL(enetc_stop); 3108 3109 int enetc_close(struct net_device *ndev) 3110 { 3111 struct enetc_ndev_priv *priv = netdev_priv(ndev); 3112 struct enetc_si *si = priv->si; 3113 3114 enetc_stop(ndev); 3115 3116 if (priv->phylink) { 3117 phylink_stop(priv->phylink); 3118 phylink_disconnect_phy(priv->phylink); 3119 } else { 3120 if (!si->ops->vf_unreg_link_status_notifier) 3121 goto carrier_off; 3122 3123 /* No need to check whether the previous registration was 3124 * successful. Sending the deregistration message has no 3125 * impact; the PF side simply clears the corresponding bit 3126 * in link_status_ms_mask for the VF. 3127 */ 3128 if (!si->ops->vf_unreg_link_status_notifier(si)) 3129 goto carrier_off; 3130 3131 dev_warn(&ndev->dev, 3132 "Link status notifier unregistration failed\n"); 3133 carrier_off: 3134 netif_carrier_off(ndev); 3135 } 3136 3137 enetc_free_rxtx_rings(priv); 3138 3139 /* Avoids dangling pointers and also frees old resources */ 3140 enetc_assign_rx_resources(priv, NULL); 3141 enetc_assign_tx_resources(priv, NULL); 3142 3143 enetc_free_irqs(priv); 3144 clk_disable_unprepare(priv->ref_clk); 3145 3146 return 0; 3147 } 3148 EXPORT_SYMBOL_GPL(enetc_close); 3149 3150 static int enetc_reconfigure(struct enetc_ndev_priv *priv, bool extended, 3151 int (*cb)(struct enetc_ndev_priv *priv, void *ctx), 3152 void *ctx) 3153 { 3154 struct enetc_bdr_resource *tx_res, *rx_res; 3155 int err; 3156 3157 ASSERT_RTNL(); 3158 3159 /* If the interface is down, run the callback right away, 3160 * without reconfiguration. 3161 */ 3162 if (!netif_running(priv->ndev)) { 3163 if (cb) { 3164 err = cb(priv, ctx); 3165 if (err) 3166 return err; 3167 } 3168 3169 return 0; 3170 } 3171 3172 tx_res = enetc_alloc_tx_resources(priv); 3173 if (IS_ERR(tx_res)) { 3174 err = PTR_ERR(tx_res); 3175 goto out; 3176 } 3177 3178 rx_res = enetc_alloc_rx_resources(priv, extended); 3179 if (IS_ERR(rx_res)) { 3180 err = PTR_ERR(rx_res); 3181 goto out_free_tx_res; 3182 } 3183 3184 enetc_stop(priv->ndev); 3185 enetc_free_rxtx_rings(priv); 3186 3187 /* Interface is down, run optional callback now */ 3188 if (cb) { 3189 err = cb(priv, ctx); 3190 if (err) 3191 goto out_restart; 3192 } 3193 3194 enetc_assign_tx_resources(priv, tx_res); 3195 enetc_assign_rx_resources(priv, rx_res); 3196 enetc_setup_bdrs(priv, extended); 3197 enetc_start(priv->ndev); 3198 3199 return 0; 3200 3201 out_restart: 3202 enetc_setup_bdrs(priv, extended); 3203 enetc_start(priv->ndev); 3204 enetc_free_rx_resources(rx_res, priv->num_rx_rings); 3205 out_free_tx_res: 3206 enetc_free_tx_resources(tx_res, priv->num_tx_rings); 3207 out: 3208 return err; 3209 } 3210 3211 static void enetc_debug_tx_ring_prios(struct enetc_ndev_priv *priv) 3212 { 3213 int i; 3214 3215 for (i = 0; i < priv->num_tx_rings; i++) 3216 netdev_dbg(priv->ndev, "TX ring %d prio %d\n", i, 3217 priv->tx_ring[i]->prio); 3218 } 3219 3220 void enetc_reset_tc_mqprio(struct net_device *ndev) 3221 { 3222 struct enetc_ndev_priv *priv = netdev_priv(ndev); 3223 struct enetc_hw *hw = &priv->si->hw; 3224 struct enetc_bdr *tx_ring; 3225 int num_stack_tx_queues; 3226 int i; 3227 3228 num_stack_tx_queues = enetc_num_stack_tx_queues(priv); 3229 3230 netdev_reset_tc(ndev); 3231 netif_set_real_num_tx_queues(ndev, num_stack_tx_queues); 3232 priv->min_num_stack_tx_queues = num_possible_cpus(); 3233 3234 /* Reset all ring priorities to 0 */ 3235 for (i = 0; i < priv->num_tx_rings; i++) { 3236 tx_ring = priv->tx_ring[i]; 3237 tx_ring->prio = 0; 3238 enetc_set_bdr_prio(hw, tx_ring->index, tx_ring->prio); 3239 } 3240 3241 enetc_debug_tx_ring_prios(priv); 3242 3243 enetc_change_preemptible_tcs(priv, 0); 3244 } 3245 EXPORT_SYMBOL_GPL(enetc_reset_tc_mqprio); 3246 3247 int enetc_setup_tc_mqprio(struct net_device *ndev, void *type_data) 3248 { 3249 struct tc_mqprio_qopt_offload *mqprio = type_data; 3250 struct enetc_ndev_priv *priv = netdev_priv(ndev); 3251 struct tc_mqprio_qopt *qopt = &mqprio->qopt; 3252 struct enetc_hw *hw = &priv->si->hw; 3253 int num_stack_tx_queues = 0; 3254 struct enetc_bdr *tx_ring; 3255 u8 num_tc = qopt->num_tc; 3256 int offset, count; 3257 int err, tc, q; 3258 3259 if (!num_tc) { 3260 enetc_reset_tc_mqprio(ndev); 3261 return 0; 3262 } 3263 3264 err = netdev_set_num_tc(ndev, num_tc); 3265 if (err) 3266 return err; 3267 3268 for (tc = 0; tc < num_tc; tc++) { 3269 offset = qopt->offset[tc]; 3270 count = qopt->count[tc]; 3271 num_stack_tx_queues += count; 3272 3273 err = netdev_set_tc_queue(ndev, tc, count, offset); 3274 if (err) 3275 goto err_reset_tc; 3276 3277 for (q = offset; q < offset + count; q++) { 3278 tx_ring = priv->tx_ring[q]; 3279 /* The prio_tc_map is skb_tx_hash()'s way of selecting 3280 * between TX queues based on skb->priority. As such, 3281 * there's nothing to offload based on it. 3282 * Make the mqprio "traffic class" be the priority of 3283 * this ring group, and leave the Tx IPV to traffic 3284 * class mapping as its default mapping value of 1:1. 3285 */ 3286 tx_ring->prio = tc; 3287 enetc_set_bdr_prio(hw, tx_ring->index, tx_ring->prio); 3288 } 3289 } 3290 3291 err = netif_set_real_num_tx_queues(ndev, num_stack_tx_queues); 3292 if (err) 3293 goto err_reset_tc; 3294 3295 priv->min_num_stack_tx_queues = num_stack_tx_queues; 3296 3297 enetc_debug_tx_ring_prios(priv); 3298 3299 enetc_change_preemptible_tcs(priv, mqprio->preemptible_tcs); 3300 3301 return 0; 3302 3303 err_reset_tc: 3304 enetc_reset_tc_mqprio(ndev); 3305 return err; 3306 } 3307 EXPORT_SYMBOL_GPL(enetc_setup_tc_mqprio); 3308 3309 static int enetc_reconfigure_xdp_cb(struct enetc_ndev_priv *priv, void *ctx) 3310 { 3311 struct bpf_prog *old_prog, *prog = ctx; 3312 int num_stack_tx_queues; 3313 int err, i; 3314 3315 old_prog = xchg(&priv->xdp_prog, prog); 3316 3317 num_stack_tx_queues = enetc_num_stack_tx_queues(priv); 3318 err = netif_set_real_num_tx_queues(priv->ndev, num_stack_tx_queues); 3319 if (err) { 3320 xchg(&priv->xdp_prog, old_prog); 3321 return err; 3322 } 3323 3324 if (old_prog) 3325 bpf_prog_put(old_prog); 3326 3327 for (i = 0; i < priv->num_rx_rings; i++) { 3328 struct enetc_bdr *rx_ring = priv->rx_ring[i]; 3329 3330 rx_ring->xdp.prog = prog; 3331 3332 if (prog) 3333 rx_ring->buffer_offset = XDP_PACKET_HEADROOM; 3334 else 3335 rx_ring->buffer_offset = ENETC_RXB_PAD; 3336 } 3337 3338 return 0; 3339 } 3340 3341 static int enetc_setup_xdp_prog(struct net_device *ndev, struct bpf_prog *prog, 3342 struct netlink_ext_ack *extack) 3343 { 3344 int num_xdp_tx_queues = prog ? num_possible_cpus() : 0; 3345 struct enetc_ndev_priv *priv = netdev_priv(ndev); 3346 bool extended; 3347 3348 if (priv->min_num_stack_tx_queues + num_xdp_tx_queues > 3349 priv->num_tx_rings) { 3350 NL_SET_ERR_MSG_FMT_MOD(extack, 3351 "Reserving %d XDP TXQs leaves under %d for stack (total %d)", 3352 num_xdp_tx_queues, 3353 priv->min_num_stack_tx_queues, 3354 priv->num_tx_rings); 3355 return -EBUSY; 3356 } 3357 3358 extended = !!(priv->active_offloads & ENETC_F_RX_TSTAMP); 3359 3360 /* The buffer layout is changing, so we need to drain the old 3361 * RX buffers and seed new ones. 3362 */ 3363 return enetc_reconfigure(priv, extended, enetc_reconfigure_xdp_cb, prog); 3364 } 3365 3366 int enetc_setup_bpf(struct net_device *ndev, struct netdev_bpf *bpf) 3367 { 3368 switch (bpf->command) { 3369 case XDP_SETUP_PROG: 3370 return enetc_setup_xdp_prog(ndev, bpf->prog, bpf->extack); 3371 default: 3372 return -EINVAL; 3373 } 3374 3375 return 0; 3376 } 3377 EXPORT_SYMBOL_GPL(enetc_setup_bpf); 3378 3379 struct net_device_stats *enetc_get_stats(struct net_device *ndev) 3380 { 3381 struct enetc_ndev_priv *priv = netdev_priv(ndev); 3382 struct net_device_stats *stats = &ndev->stats; 3383 unsigned long packets = 0, bytes = 0; 3384 unsigned long tx_dropped = 0; 3385 int i; 3386 3387 for (i = 0; i < priv->num_rx_rings; i++) { 3388 packets += priv->rx_ring[i]->stats.packets; 3389 bytes += priv->rx_ring[i]->stats.bytes; 3390 } 3391 3392 stats->rx_packets = packets; 3393 stats->rx_bytes = bytes; 3394 bytes = 0; 3395 packets = 0; 3396 3397 for (i = 0; i < priv->num_tx_rings; i++) { 3398 packets += priv->tx_ring[i]->stats.packets; 3399 bytes += priv->tx_ring[i]->stats.bytes; 3400 tx_dropped += priv->tx_ring[i]->stats.win_drop; 3401 } 3402 3403 stats->tx_packets = packets; 3404 stats->tx_bytes = bytes; 3405 stats->tx_dropped = tx_dropped; 3406 3407 return stats; 3408 } 3409 EXPORT_SYMBOL_GPL(enetc_get_stats); 3410 3411 static void enetc_enable_rxvlan(struct net_device *ndev, bool en) 3412 { 3413 struct enetc_ndev_priv *priv = netdev_priv(ndev); 3414 struct enetc_si *si = priv->si; 3415 struct enetc_hw *hw = &si->hw; 3416 int i; 3417 3418 spin_lock(&si->gen_lock); 3419 3420 for (i = 0; i < priv->num_rx_rings; i++) 3421 enetc_bdr_enable_rxvlan(hw, i, en); 3422 3423 spin_unlock(&si->gen_lock); 3424 } 3425 3426 static void enetc_enable_txvlan(struct net_device *ndev, bool en) 3427 { 3428 struct enetc_ndev_priv *priv = netdev_priv(ndev); 3429 struct enetc_hw *hw = &priv->si->hw; 3430 int i; 3431 3432 for (i = 0; i < priv->num_tx_rings; i++) 3433 enetc_bdr_enable_txvlan(hw, i, en); 3434 } 3435 3436 void enetc_set_features(struct net_device *ndev, netdev_features_t features) 3437 { 3438 netdev_features_t changed = ndev->features ^ features; 3439 3440 if (changed & NETIF_F_RXHASH) 3441 enetc_set_rss(ndev, !!(features & NETIF_F_RXHASH)); 3442 3443 if (changed & NETIF_F_HW_VLAN_CTAG_RX) 3444 enetc_enable_rxvlan(ndev, 3445 !!(features & NETIF_F_HW_VLAN_CTAG_RX)); 3446 3447 if (changed & NETIF_F_HW_VLAN_CTAG_TX) 3448 enetc_enable_txvlan(ndev, 3449 !!(features & NETIF_F_HW_VLAN_CTAG_TX)); 3450 } 3451 EXPORT_SYMBOL_GPL(enetc_set_features); 3452 3453 int enetc_hwtstamp_set(struct net_device *ndev, 3454 struct kernel_hwtstamp_config *config, 3455 struct netlink_ext_ack *extack) 3456 { 3457 struct enetc_ndev_priv *priv = netdev_priv(ndev); 3458 int err, new_offloads = priv->active_offloads; 3459 3460 if (!enetc_ptp_clock_is_enabled(priv->si)) 3461 return -EOPNOTSUPP; 3462 3463 switch (config->tx_type) { 3464 case HWTSTAMP_TX_OFF: 3465 new_offloads &= ~ENETC_F_TX_TSTAMP_MASK; 3466 break; 3467 case HWTSTAMP_TX_ON: 3468 new_offloads &= ~ENETC_F_TX_TSTAMP_MASK; 3469 new_offloads |= ENETC_F_TX_TSTAMP; 3470 break; 3471 case HWTSTAMP_TX_ONESTEP_SYNC: 3472 if (!enetc_si_is_pf(priv->si) || 3473 enetc_is_pseudo_mac(priv->si)) 3474 return -EOPNOTSUPP; 3475 3476 new_offloads &= ~ENETC_F_TX_TSTAMP_MASK; 3477 new_offloads |= ENETC_F_TX_ONESTEP_SYNC_TSTAMP; 3478 break; 3479 default: 3480 return -ERANGE; 3481 } 3482 3483 switch (config->rx_filter) { 3484 case HWTSTAMP_FILTER_NONE: 3485 new_offloads &= ~ENETC_F_RX_TSTAMP; 3486 break; 3487 default: 3488 new_offloads |= ENETC_F_RX_TSTAMP; 3489 config->rx_filter = HWTSTAMP_FILTER_ALL; 3490 } 3491 3492 if ((new_offloads ^ priv->active_offloads) & ENETC_F_RX_TSTAMP) { 3493 bool extended = !!(new_offloads & ENETC_F_RX_TSTAMP); 3494 3495 err = enetc_reconfigure(priv, extended, NULL, NULL); 3496 if (err) 3497 return err; 3498 } 3499 3500 priv->active_offloads = new_offloads; 3501 3502 return 0; 3503 } 3504 EXPORT_SYMBOL_GPL(enetc_hwtstamp_set); 3505 3506 int enetc_hwtstamp_get(struct net_device *ndev, 3507 struct kernel_hwtstamp_config *config) 3508 { 3509 struct enetc_ndev_priv *priv = netdev_priv(ndev); 3510 3511 if (!enetc_ptp_clock_is_enabled(priv->si)) 3512 return -EOPNOTSUPP; 3513 3514 if (priv->active_offloads & ENETC_F_TX_ONESTEP_SYNC_TSTAMP) 3515 config->tx_type = HWTSTAMP_TX_ONESTEP_SYNC; 3516 else if (priv->active_offloads & ENETC_F_TX_TSTAMP) 3517 config->tx_type = HWTSTAMP_TX_ON; 3518 else 3519 config->tx_type = HWTSTAMP_TX_OFF; 3520 3521 config->rx_filter = (priv->active_offloads & ENETC_F_RX_TSTAMP) ? 3522 HWTSTAMP_FILTER_ALL : HWTSTAMP_FILTER_NONE; 3523 3524 return 0; 3525 } 3526 EXPORT_SYMBOL_GPL(enetc_hwtstamp_get); 3527 3528 int enetc_ioctl(struct net_device *ndev, struct ifreq *rq, int cmd) 3529 { 3530 struct enetc_ndev_priv *priv = netdev_priv(ndev); 3531 3532 if (!priv->phylink) 3533 return -EOPNOTSUPP; 3534 3535 return phylink_mii_ioctl(priv->phylink, rq, cmd); 3536 } 3537 EXPORT_SYMBOL_GPL(enetc_ioctl); 3538 3539 static int enetc_int_vector_init(struct enetc_ndev_priv *priv, int i, 3540 int v_tx_rings) 3541 { 3542 struct enetc_int_vector *v; 3543 struct enetc_bdr *bdr; 3544 int j, err; 3545 3546 v = kzalloc_flex(*v, tx_ring, v_tx_rings); 3547 if (!v) 3548 return -ENOMEM; 3549 3550 priv->int_vector[i] = v; 3551 bdr = &v->rx_ring; 3552 bdr->index = i; 3553 bdr->ndev = priv->ndev; 3554 bdr->dev = priv->dev; 3555 bdr->bd_count = priv->rx_bd_count; 3556 bdr->buffer_offset = ENETC_RXB_PAD; 3557 priv->rx_ring[i] = bdr; 3558 3559 err = __xdp_rxq_info_reg(&bdr->xdp.rxq, priv->ndev, i, 0, 3560 ENETC_RXB_TRUESIZE); 3561 if (err) 3562 goto free_vector; 3563 3564 err = xdp_rxq_info_reg_mem_model(&bdr->xdp.rxq, MEM_TYPE_PAGE_SHARED, 3565 NULL); 3566 if (err) { 3567 xdp_rxq_info_unreg(&bdr->xdp.rxq); 3568 goto free_vector; 3569 } 3570 3571 /* init defaults for adaptive IC */ 3572 if (priv->ic_mode & ENETC_IC_RX_ADAPTIVE) { 3573 v->rx_ictt = 0x1; 3574 v->rx_dim_en = true; 3575 } 3576 3577 INIT_WORK(&v->rx_dim.work, enetc_rx_dim_work); 3578 netif_napi_add(priv->ndev, &v->napi, enetc_poll); 3579 v->count_tx_rings = v_tx_rings; 3580 3581 for (j = 0; j < v_tx_rings; j++) { 3582 int idx; 3583 3584 /* default tx ring mapping policy */ 3585 idx = priv->bdr_int_num * j + i; 3586 __set_bit(idx, &v->tx_rings_map); 3587 bdr = &v->tx_ring[j]; 3588 bdr->index = idx; 3589 bdr->ndev = priv->ndev; 3590 bdr->dev = priv->dev; 3591 bdr->bd_count = priv->tx_bd_count; 3592 priv->tx_ring[idx] = bdr; 3593 } 3594 3595 return 0; 3596 3597 free_vector: 3598 priv->rx_ring[i] = NULL; 3599 priv->int_vector[i] = NULL; 3600 kfree(v); 3601 3602 return err; 3603 } 3604 3605 static void enetc_int_vector_destroy(struct enetc_ndev_priv *priv, int i) 3606 { 3607 struct enetc_int_vector *v = priv->int_vector[i]; 3608 struct enetc_bdr *rx_ring = &v->rx_ring; 3609 int j, tx_ring_index; 3610 3611 xdp_rxq_info_unreg_mem_model(&rx_ring->xdp.rxq); 3612 xdp_rxq_info_unreg(&rx_ring->xdp.rxq); 3613 netif_napi_del(&v->napi); 3614 cancel_work_sync(&v->rx_dim.work); 3615 3616 for (j = 0; j < v->count_tx_rings; j++) { 3617 tx_ring_index = priv->bdr_int_num * j + i; 3618 priv->tx_ring[tx_ring_index] = NULL; 3619 } 3620 3621 priv->rx_ring[i] = NULL; 3622 priv->int_vector[i] = NULL; 3623 kfree(v); 3624 } 3625 3626 int enetc_alloc_msix(struct enetc_ndev_priv *priv) 3627 { 3628 struct pci_dev *pdev = priv->si->pdev; 3629 int v_tx_rings, v_remainder; 3630 int num_stack_tx_queues; 3631 int first_xdp_tx_ring; 3632 int i, n, err, nvec; 3633 3634 nvec = ENETC_BDR_INT_BASE_IDX + priv->bdr_int_num; 3635 /* allocate MSIX for both messaging and Rx/Tx interrupts */ 3636 n = pci_alloc_irq_vectors(pdev, nvec, nvec, PCI_IRQ_MSIX); 3637 3638 if (n < 0) 3639 return n; 3640 3641 if (n != nvec) 3642 return -EPERM; 3643 3644 /* # of tx rings per int vector */ 3645 v_tx_rings = priv->num_tx_rings / priv->bdr_int_num; 3646 v_remainder = priv->num_tx_rings % priv->bdr_int_num; 3647 3648 for (i = 0; i < priv->bdr_int_num; i++) { 3649 /* Distribute the remaining TX rings to the first v_remainder 3650 * interrupt vectors 3651 */ 3652 int num_tx_rings = i < v_remainder ? v_tx_rings + 1 : v_tx_rings; 3653 3654 err = enetc_int_vector_init(priv, i, num_tx_rings); 3655 if (err) 3656 goto fail; 3657 } 3658 3659 num_stack_tx_queues = enetc_num_stack_tx_queues(priv); 3660 3661 err = netif_set_real_num_tx_queues(priv->ndev, num_stack_tx_queues); 3662 if (err) 3663 goto fail; 3664 3665 err = netif_set_real_num_rx_queues(priv->ndev, priv->num_rx_rings); 3666 if (err) 3667 goto fail; 3668 3669 priv->min_num_stack_tx_queues = num_possible_cpus(); 3670 first_xdp_tx_ring = priv->num_tx_rings - num_possible_cpus(); 3671 priv->xdp_tx_ring = &priv->tx_ring[first_xdp_tx_ring]; 3672 3673 return 0; 3674 3675 fail: 3676 while (i--) 3677 enetc_int_vector_destroy(priv, i); 3678 3679 pci_free_irq_vectors(pdev); 3680 3681 return err; 3682 } 3683 EXPORT_SYMBOL_GPL(enetc_alloc_msix); 3684 3685 void enetc_free_msix(struct enetc_ndev_priv *priv) 3686 { 3687 int i; 3688 3689 for (i = 0; i < priv->bdr_int_num; i++) 3690 enetc_int_vector_destroy(priv, i); 3691 3692 /* disable all MSIX for this device */ 3693 pci_free_irq_vectors(priv->si->pdev); 3694 } 3695 EXPORT_SYMBOL_GPL(enetc_free_msix); 3696 3697 static void enetc_kfree_si(struct enetc_si *si) 3698 { 3699 char *p = (char *)si - si->pad; 3700 3701 kfree(p); 3702 } 3703 3704 static void enetc_detect_errata(struct enetc_si *si) 3705 { 3706 if (si->pdev->revision == ENETC_REV1) 3707 si->errata = ENETC_ERR_VLAN_ISOL | ENETC_ERR_UCMCSWP; 3708 } 3709 3710 int enetc_pci_probe(struct pci_dev *pdev, const char *name, int sizeof_priv) 3711 { 3712 struct enetc_si *si, *p; 3713 struct enetc_hw *hw; 3714 size_t alloc_size; 3715 int err, len; 3716 3717 pcie_flr(pdev); 3718 err = pci_enable_device_mem(pdev); 3719 if (err) 3720 return dev_err_probe(&pdev->dev, err, "device enable failed\n"); 3721 3722 /* set up for high or low dma */ 3723 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 3724 if (err) { 3725 dev_err(&pdev->dev, "DMA configuration failed: 0x%x\n", err); 3726 goto err_dma; 3727 } 3728 3729 err = pci_request_mem_regions(pdev, name); 3730 if (err) { 3731 dev_err(&pdev->dev, "pci_request_regions failed err=%d\n", err); 3732 goto err_pci_mem_reg; 3733 } 3734 3735 pci_set_master(pdev); 3736 3737 alloc_size = sizeof(struct enetc_si); 3738 if (sizeof_priv) { 3739 /* align priv to 32B */ 3740 alloc_size = ALIGN(alloc_size, ENETC_SI_ALIGN); 3741 alloc_size += sizeof_priv; 3742 } 3743 /* force 32B alignment for enetc_si */ 3744 alloc_size += ENETC_SI_ALIGN - 1; 3745 3746 p = kzalloc(alloc_size, GFP_KERNEL); 3747 if (!p) { 3748 err = -ENOMEM; 3749 goto err_alloc_si; 3750 } 3751 3752 si = PTR_ALIGN(p, ENETC_SI_ALIGN); 3753 si->pad = (char *)si - (char *)p; 3754 spin_lock_init(&si->gen_lock); 3755 3756 pci_set_drvdata(pdev, si); 3757 si->pdev = pdev; 3758 hw = &si->hw; 3759 3760 len = pci_resource_len(pdev, ENETC_BAR_REGS); 3761 hw->reg = ioremap(pci_resource_start(pdev, ENETC_BAR_REGS), len); 3762 if (!hw->reg) { 3763 err = -ENXIO; 3764 dev_err(&pdev->dev, "ioremap() failed\n"); 3765 goto err_ioremap; 3766 } 3767 if (len > ENETC_PORT_BASE) 3768 hw->port = hw->reg + ENETC_PORT_BASE; 3769 if (len > ENETC_GLOBAL_BASE) 3770 hw->global = hw->reg + ENETC_GLOBAL_BASE; 3771 3772 enetc_detect_errata(si); 3773 3774 return 0; 3775 3776 err_ioremap: 3777 enetc_kfree_si(si); 3778 err_alloc_si: 3779 pci_release_mem_regions(pdev); 3780 err_pci_mem_reg: 3781 err_dma: 3782 pci_disable_device(pdev); 3783 3784 return err; 3785 } 3786 EXPORT_SYMBOL_GPL(enetc_pci_probe); 3787 3788 void enetc_pci_remove(struct pci_dev *pdev) 3789 { 3790 struct enetc_si *si = pci_get_drvdata(pdev); 3791 struct enetc_hw *hw = &si->hw; 3792 3793 iounmap(hw->reg); 3794 enetc_kfree_si(si); 3795 pci_release_mem_regions(pdev); 3796 pci_disable_device(pdev); 3797 } 3798 EXPORT_SYMBOL_GPL(enetc_pci_remove); 3799 3800 static const struct enetc_drvdata enetc_pf_data = { 3801 .sysclk_freq = ENETC_CLK_400M, 3802 .pmac_offset = ENETC_PMAC_OFFSET, 3803 .max_frags = ENETC_MAX_SKB_FRAGS, 3804 .eth_ops = &enetc_pf_ethtool_ops, 3805 }; 3806 3807 static const struct enetc_drvdata enetc4_pf_data = { 3808 .sysclk_freq = ENETC_CLK_333M, 3809 .tx_csum = true, 3810 .max_frags = ENETC4_MAX_SKB_FRAGS, 3811 .pmac_offset = ENETC4_PMAC_OFFSET, 3812 .eth_ops = &enetc4_pf_ethtool_ops, 3813 }; 3814 3815 static const struct enetc_drvdata enetc4_ppm_data = { 3816 .sysclk_freq = ENETC_CLK_333M, 3817 .tx_csum = true, 3818 .max_frags = ENETC4_MAX_SKB_FRAGS, 3819 .eth_ops = &enetc4_ppm_ethtool_ops, 3820 }; 3821 3822 static const struct enetc_drvdata enetc_vf_data = { 3823 .sysclk_freq = ENETC_CLK_400M, 3824 .max_frags = ENETC_MAX_SKB_FRAGS, 3825 .eth_ops = &enetc_vf_ethtool_ops, 3826 }; 3827 3828 static const struct enetc_drvdata enetc4_vf_data = { 3829 .sysclk_freq = ENETC_CLK_333M, 3830 .tx_csum = true, 3831 .max_frags = ENETC4_MAX_SKB_FRAGS, 3832 .eth_ops = &enetc_vf_ethtool_ops, 3833 }; 3834 3835 static const struct enetc_platform_info enetc_info[] = { 3836 { .revision = ENETC_REV_1_0, 3837 .dev_id = ENETC_DEV_ID_PF, 3838 .data = &enetc_pf_data, 3839 }, 3840 { .revision = ENETC_REV_4_1, 3841 .dev_id = NXP_ENETC_PF_DEV_ID, 3842 .data = &enetc4_pf_data, 3843 }, 3844 { .revision = ENETC_REV_1_0, 3845 .dev_id = ENETC_DEV_ID_VF, 3846 .data = &enetc_vf_data, 3847 }, 3848 { .revision = ENETC_REV_4_1, 3849 .dev_id = NXP_ENETC_VF_DEV_ID, 3850 .data = &enetc4_vf_data, 3851 }, 3852 { 3853 .revision = ENETC_REV_4_3, 3854 .dev_id = NXP_ENETC_PPM_DEV_ID, 3855 .data = &enetc4_ppm_data, 3856 }, 3857 { .revision = ENETC_REV_4_3, 3858 .dev_id = NXP_ENETC_PF_DEV_ID, 3859 .data = &enetc4_pf_data, 3860 }, 3861 { .revision = ENETC_REV_4_3, 3862 .dev_id = NXP_ENETC_VF_DEV_ID, 3863 .data = &enetc4_vf_data, 3864 }, 3865 }; 3866 3867 int enetc_get_driver_data(struct enetc_si *si) 3868 { 3869 u16 dev_id = si->pdev->device; 3870 int i; 3871 3872 for (i = 0; i < ARRAY_SIZE(enetc_info); i++) { 3873 if (si->revision == enetc_info[i].revision && 3874 dev_id == enetc_info[i].dev_id) { 3875 si->drvdata = enetc_info[i].data; 3876 3877 return 0; 3878 } 3879 } 3880 3881 return -ERANGE; 3882 } 3883 EXPORT_SYMBOL_GPL(enetc_get_driver_data); 3884 3885 MODULE_DESCRIPTION("NXP ENETC Ethernet driver"); 3886 MODULE_LICENSE("Dual BSD/GPL"); 3887