1 // SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) 2 /* Copyright 2017-2019 NXP */ 3 4 #include "enetc.h" 5 #include <linux/bpf_trace.h> 6 #include <linux/clk.h> 7 #include <linux/tcp.h> 8 #include <linux/udp.h> 9 #include <linux/vmalloc.h> 10 #include <linux/ptp_classify.h> 11 #include <net/ip6_checksum.h> 12 #include <net/pkt_sched.h> 13 #include <net/tso.h> 14 15 u32 enetc_port_mac_rd(struct enetc_si *si, u32 reg) 16 { 17 /* ENETC with pseudo MAC does not have Ethernet MAC 18 * port registers. 19 */ 20 if (enetc_is_pseudo_mac(si)) 21 return 0; 22 23 return enetc_port_rd(&si->hw, reg); 24 } 25 EXPORT_SYMBOL_GPL(enetc_port_mac_rd); 26 27 void enetc_port_mac_wr(struct enetc_si *si, u32 reg, u32 val) 28 { 29 if (enetc_is_pseudo_mac(si)) 30 return; 31 32 enetc_port_wr(&si->hw, reg, val); 33 if (si->hw_features & ENETC_SI_F_QBU) 34 enetc_port_wr(&si->hw, reg + si->drvdata->pmac_offset, val); 35 } 36 EXPORT_SYMBOL_GPL(enetc_port_mac_wr); 37 38 static void enetc_change_preemptible_tcs(struct enetc_ndev_priv *priv, 39 u8 preemptible_tcs) 40 { 41 if (!(priv->si->hw_features & ENETC_SI_F_QBU)) 42 return; 43 44 priv->preemptible_tcs = preemptible_tcs; 45 enetc_mm_commit_preemptible_tcs(priv); 46 } 47 48 static int enetc_mac_addr_hash_idx(const u8 *addr) 49 { 50 u64 fold = __swab64(ether_addr_to_u64(addr)) >> 16; 51 u64 mask = 0; 52 int res = 0; 53 int i; 54 55 for (i = 0; i < 8; i++) 56 mask |= BIT_ULL(i * 6); 57 58 for (i = 0; i < 6; i++) 59 res |= (hweight64(fold & (mask << i)) & 0x1) << i; 60 61 return res; 62 } 63 64 void enetc_add_mac_addr_ht_filter(struct enetc_mac_filter *filter, 65 const unsigned char *addr) 66 { 67 int idx = enetc_mac_addr_hash_idx(addr); 68 69 /* add hash table entry */ 70 __set_bit(idx, filter->mac_hash_table); 71 filter->mac_addr_cnt++; 72 } 73 EXPORT_SYMBOL_GPL(enetc_add_mac_addr_ht_filter); 74 75 void enetc_reset_mac_addr_filter(struct enetc_mac_filter *filter) 76 { 77 filter->mac_addr_cnt = 0; 78 79 bitmap_zero(filter->mac_hash_table, 80 ENETC_MADDR_HASH_TBL_SZ); 81 } 82 EXPORT_SYMBOL_GPL(enetc_reset_mac_addr_filter); 83 84 static int enetc_num_stack_tx_queues(struct enetc_ndev_priv *priv) 85 { 86 int num_tx_rings = priv->num_tx_rings; 87 88 if (priv->xdp_prog) 89 return num_tx_rings - num_possible_cpus(); 90 91 return num_tx_rings; 92 } 93 94 static struct enetc_bdr *enetc_rx_ring_from_xdp_tx_ring(struct enetc_ndev_priv *priv, 95 struct enetc_bdr *tx_ring) 96 { 97 int index = &priv->tx_ring[tx_ring->index] - priv->xdp_tx_ring; 98 99 return priv->rx_ring[index]; 100 } 101 102 static struct sk_buff *enetc_tx_swbd_get_skb(struct enetc_tx_swbd *tx_swbd) 103 { 104 if (tx_swbd->is_xdp_tx || tx_swbd->is_xdp_redirect) 105 return NULL; 106 107 return tx_swbd->skb; 108 } 109 110 static struct xdp_frame * 111 enetc_tx_swbd_get_xdp_frame(struct enetc_tx_swbd *tx_swbd) 112 { 113 if (tx_swbd->is_xdp_redirect) 114 return tx_swbd->xdp_frame; 115 116 return NULL; 117 } 118 119 static void enetc_unmap_tx_buff(struct enetc_bdr *tx_ring, 120 struct enetc_tx_swbd *tx_swbd) 121 { 122 /* For XDP_TX, pages come from RX, whereas for the other contexts where 123 * we have is_dma_page_set, those come from skb_frag_dma_map. We need 124 * to match the DMA mapping length, so we need to differentiate those. 125 */ 126 if (tx_swbd->is_dma_page) 127 dma_unmap_page(tx_ring->dev, tx_swbd->dma, 128 tx_swbd->is_xdp_tx ? PAGE_SIZE : tx_swbd->len, 129 tx_swbd->dir); 130 else 131 dma_unmap_single(tx_ring->dev, tx_swbd->dma, 132 tx_swbd->len, tx_swbd->dir); 133 tx_swbd->dma = 0; 134 } 135 136 static void enetc_free_tx_frame(struct enetc_bdr *tx_ring, 137 struct enetc_tx_swbd *tx_swbd) 138 { 139 struct xdp_frame *xdp_frame = enetc_tx_swbd_get_xdp_frame(tx_swbd); 140 struct sk_buff *skb = enetc_tx_swbd_get_skb(tx_swbd); 141 142 if (tx_swbd->dma) 143 enetc_unmap_tx_buff(tx_ring, tx_swbd); 144 145 if (xdp_frame) { 146 xdp_return_frame(tx_swbd->xdp_frame); 147 tx_swbd->xdp_frame = NULL; 148 } else if (skb) { 149 dev_kfree_skb_any(skb); 150 tx_swbd->skb = NULL; 151 } 152 } 153 154 /* Let H/W know BD ring has been updated */ 155 static void enetc_update_tx_ring_tail(struct enetc_bdr *tx_ring) 156 { 157 /* includes wmb() */ 158 enetc_wr_reg_hot(tx_ring->tpir, tx_ring->next_to_use); 159 } 160 161 static int enetc_ptp_parse(struct sk_buff *skb, u8 *udp, 162 u8 *msgtype, u8 *twostep, 163 u16 *correction_offset, u16 *body_offset) 164 { 165 unsigned int ptp_class; 166 struct ptp_header *hdr; 167 unsigned int type; 168 u8 *base; 169 170 ptp_class = ptp_classify_raw(skb); 171 if (ptp_class == PTP_CLASS_NONE) 172 return -EINVAL; 173 174 hdr = ptp_parse_header(skb, ptp_class); 175 if (!hdr) 176 return -EINVAL; 177 178 type = ptp_class & PTP_CLASS_PMASK; 179 if (type == PTP_CLASS_IPV4 || type == PTP_CLASS_IPV6) 180 *udp = 1; 181 else 182 *udp = 0; 183 184 *msgtype = ptp_get_msgtype(hdr, ptp_class); 185 *twostep = hdr->flag_field[0] & 0x2; 186 187 base = skb_mac_header(skb); 188 *correction_offset = (u8 *)&hdr->correction - base; 189 *body_offset = (u8 *)hdr + sizeof(struct ptp_header) - base; 190 191 return 0; 192 } 193 194 static bool enetc_tx_csum_offload_check(struct sk_buff *skb) 195 { 196 switch (skb->csum_offset) { 197 case offsetof(struct tcphdr, check): 198 case offsetof(struct udphdr, check): 199 return true; 200 default: 201 return false; 202 } 203 } 204 205 static bool enetc_skb_is_ipv6(struct sk_buff *skb) 206 { 207 return vlan_get_protocol(skb) == htons(ETH_P_IPV6); 208 } 209 210 static bool enetc_skb_is_tcp(struct sk_buff *skb) 211 { 212 return skb->csum_offset == offsetof(struct tcphdr, check); 213 } 214 215 /** 216 * enetc_unwind_tx_frame() - Unwind the DMA mappings of a multi-buffer Tx frame 217 * @tx_ring: Pointer to the Tx ring on which the buffer descriptors are located 218 * @count: Number of Tx buffer descriptors which need to be unmapped 219 * @i: Index of the last successfully mapped Tx buffer descriptor 220 */ 221 static void enetc_unwind_tx_frame(struct enetc_bdr *tx_ring, int count, int i) 222 { 223 while (count--) { 224 struct enetc_tx_swbd *tx_swbd = &tx_ring->tx_swbd[i]; 225 226 enetc_free_tx_frame(tx_ring, tx_swbd); 227 if (i == 0) 228 i = tx_ring->bd_count; 229 i--; 230 } 231 } 232 233 static void enetc_set_one_step_ts(struct enetc_si *si, bool udp, int offset) 234 { 235 u32 val = ENETC_PM0_SINGLE_STEP_EN; 236 237 val |= ENETC_SET_SINGLE_STEP_OFFSET(offset); 238 if (udp) 239 val |= ENETC_PM0_SINGLE_STEP_CH; 240 241 /* The "Correction" field of a packet is updated based on the 242 * current time and the timestamp provided 243 */ 244 enetc_port_mac_wr(si, ENETC_PM0_SINGLE_STEP, val); 245 } 246 247 static void enetc4_set_one_step_ts(struct enetc_si *si, bool udp, int offset) 248 { 249 u32 val = PM_SINGLE_STEP_EN; 250 251 val |= PM_SINGLE_STEP_OFFSET_SET(offset); 252 if (udp) 253 val |= PM_SINGLE_STEP_CH; 254 255 enetc_port_mac_wr(si, ENETC4_PM_SINGLE_STEP(0), val); 256 } 257 258 static u32 enetc_update_ptp_sync_msg(struct enetc_ndev_priv *priv, 259 struct sk_buff *skb, bool csum_offload) 260 { 261 struct enetc_skb_cb *enetc_cb = ENETC_SKB_CB(skb); 262 u16 tstamp_off = enetc_cb->origin_tstamp_off; 263 u16 corr_off = enetc_cb->correction_off; 264 struct enetc_si *si = priv->si; 265 struct enetc_hw *hw = &si->hw; 266 __be32 new_sec_l, new_nsec; 267 __be16 new_sec_h; 268 u32 lo, hi, nsec; 269 u8 *data; 270 u64 sec; 271 272 lo = enetc_rd_hot(hw, ENETC_SICTR0); 273 hi = enetc_rd_hot(hw, ENETC_SICTR1); 274 sec = (u64)hi << 32 | lo; 275 nsec = do_div(sec, 1000000000); 276 277 /* Update originTimestamp field of Sync packet 278 * - 48 bits seconds field 279 * - 32 bits nanseconds field 280 * 281 * In addition, if csum_offload is false, the UDP checksum needs 282 * to be updated by software after updating originTimestamp field, 283 * otherwise the hardware will calculate the wrong checksum when 284 * updating the correction field and update it to the packet. 285 */ 286 287 data = skb_mac_header(skb); 288 new_sec_h = htons((sec >> 32) & 0xffff); 289 new_sec_l = htonl(sec & 0xffffffff); 290 new_nsec = htonl(nsec); 291 if (enetc_cb->udp && !csum_offload) { 292 struct udphdr *uh = udp_hdr(skb); 293 __be32 old_sec_l, old_nsec; 294 __be16 old_sec_h; 295 296 old_sec_h = *(__be16 *)(data + tstamp_off); 297 inet_proto_csum_replace2(&uh->check, skb, old_sec_h, 298 new_sec_h, false); 299 300 old_sec_l = *(__be32 *)(data + tstamp_off + 2); 301 inet_proto_csum_replace4(&uh->check, skb, old_sec_l, 302 new_sec_l, false); 303 304 old_nsec = *(__be32 *)(data + tstamp_off + 6); 305 inet_proto_csum_replace4(&uh->check, skb, old_nsec, 306 new_nsec, false); 307 } 308 309 *(__be16 *)(data + tstamp_off) = new_sec_h; 310 *(__be32 *)(data + tstamp_off + 2) = new_sec_l; 311 *(__be32 *)(data + tstamp_off + 6) = new_nsec; 312 313 /* Configure single-step register */ 314 if (is_enetc_rev1(si)) 315 enetc_set_one_step_ts(si, enetc_cb->udp, corr_off); 316 else 317 enetc4_set_one_step_ts(si, enetc_cb->udp, corr_off); 318 319 return lo & ENETC_TXBD_TSTAMP; 320 } 321 322 static int enetc_map_tx_buffs(struct enetc_bdr *tx_ring, struct sk_buff *skb) 323 { 324 bool do_vlan, do_onestep_tstamp = false, do_twostep_tstamp = false; 325 struct enetc_ndev_priv *priv = netdev_priv(tx_ring->ndev); 326 struct enetc_skb_cb *enetc_cb = ENETC_SKB_CB(skb); 327 struct enetc_tx_swbd *tx_swbd; 328 int len = skb_headlen(skb); 329 union enetc_tx_bd temp_bd; 330 bool csum_offload = false; 331 union enetc_tx_bd *txbd; 332 int i, count = 0; 333 skb_frag_t *frag; 334 unsigned int f; 335 dma_addr_t dma; 336 u8 flags = 0; 337 u32 tstamp; 338 339 enetc_clear_tx_bd(&temp_bd); 340 if (skb->ip_summed == CHECKSUM_PARTIAL) { 341 /* Can not support TSD and checksum offload at the same time */ 342 if (priv->active_offloads & ENETC_F_TXCSUM && 343 enetc_tx_csum_offload_check(skb) && !tx_ring->tsd_enable) { 344 temp_bd.l3_aux0 = FIELD_PREP(ENETC_TX_BD_L3_START, 345 skb_network_offset(skb)); 346 temp_bd.l3_aux1 = FIELD_PREP(ENETC_TX_BD_L3_HDR_LEN, 347 skb_network_header_len(skb) / 4); 348 temp_bd.l3_aux1 |= FIELD_PREP(ENETC_TX_BD_L3T, 349 enetc_skb_is_ipv6(skb)); 350 if (enetc_skb_is_tcp(skb)) 351 temp_bd.l4_aux = FIELD_PREP(ENETC_TX_BD_L4T, 352 ENETC_TXBD_L4T_TCP); 353 else 354 temp_bd.l4_aux = FIELD_PREP(ENETC_TX_BD_L4T, 355 ENETC_TXBD_L4T_UDP); 356 flags |= ENETC_TXBD_FLAGS_CSUM_LSO | ENETC_TXBD_FLAGS_L4CS; 357 csum_offload = true; 358 } else if (skb_checksum_help(skb)) { 359 return 0; 360 } 361 } 362 363 if (enetc_cb->flag & ENETC_F_TX_ONESTEP_SYNC_TSTAMP) { 364 do_onestep_tstamp = true; 365 tstamp = enetc_update_ptp_sync_msg(priv, skb, csum_offload); 366 } else if (enetc_cb->flag & ENETC_F_TX_TSTAMP) { 367 do_twostep_tstamp = true; 368 } 369 370 i = tx_ring->next_to_use; 371 txbd = ENETC_TXBD(*tx_ring, i); 372 prefetchw(txbd); 373 374 dma = dma_map_single(tx_ring->dev, skb->data, len, DMA_TO_DEVICE); 375 if (unlikely(dma_mapping_error(tx_ring->dev, dma))) 376 goto dma_err; 377 378 temp_bd.addr = cpu_to_le64(dma); 379 temp_bd.buf_len = cpu_to_le16(len); 380 381 tx_swbd = &tx_ring->tx_swbd[i]; 382 tx_swbd->dma = dma; 383 tx_swbd->len = len; 384 tx_swbd->is_dma_page = 0; 385 tx_swbd->dir = DMA_TO_DEVICE; 386 count++; 387 388 do_vlan = skb_vlan_tag_present(skb); 389 tx_swbd->do_twostep_tstamp = do_twostep_tstamp; 390 tx_swbd->qbv_en = !!(priv->active_offloads & ENETC_F_QBV); 391 tx_swbd->check_wb = tx_swbd->do_twostep_tstamp || tx_swbd->qbv_en; 392 393 if (do_vlan || do_onestep_tstamp || do_twostep_tstamp) 394 flags |= ENETC_TXBD_FLAGS_EX; 395 396 if (tx_ring->tsd_enable) 397 flags |= ENETC_TXBD_FLAGS_TSE | ENETC_TXBD_FLAGS_TXSTART; 398 399 /* first BD needs frm_len and offload flags set */ 400 temp_bd.frm_len = cpu_to_le16(skb->len); 401 temp_bd.flags = flags; 402 403 if (flags & ENETC_TXBD_FLAGS_TSE) 404 temp_bd.txstart = enetc_txbd_set_tx_start(skb->skb_mstamp_ns, 405 flags); 406 407 if (flags & ENETC_TXBD_FLAGS_EX) { 408 u8 e_flags = 0; 409 *txbd = temp_bd; 410 enetc_clear_tx_bd(&temp_bd); 411 412 /* add extension BD for VLAN and/or timestamping */ 413 flags = 0; 414 tx_swbd++; 415 txbd++; 416 i++; 417 if (unlikely(i == tx_ring->bd_count)) { 418 i = 0; 419 tx_swbd = tx_ring->tx_swbd; 420 txbd = ENETC_TXBD(*tx_ring, 0); 421 } 422 prefetchw(txbd); 423 424 if (do_vlan) { 425 temp_bd.ext.vid = cpu_to_le16(skb_vlan_tag_get(skb)); 426 temp_bd.ext.tpid = 0; /* < C-TAG */ 427 e_flags |= ENETC_TXBD_E_FLAGS_VLAN_INS; 428 } 429 430 if (do_onestep_tstamp) { 431 /* Configure extension BD */ 432 temp_bd.ext.tstamp = cpu_to_le32(tstamp); 433 e_flags |= ENETC_TXBD_E_FLAGS_ONE_STEP_PTP; 434 } else if (do_twostep_tstamp) { 435 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; 436 e_flags |= ENETC_TXBD_E_FLAGS_TWO_STEP_PTP; 437 } 438 439 temp_bd.ext.e_flags = e_flags; 440 count++; 441 } 442 443 frag = &skb_shinfo(skb)->frags[0]; 444 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++, frag++) { 445 len = skb_frag_size(frag); 446 dma = skb_frag_dma_map(tx_ring->dev, frag, 0, len, 447 DMA_TO_DEVICE); 448 if (dma_mapping_error(tx_ring->dev, dma)) 449 goto dma_err; 450 451 *txbd = temp_bd; 452 enetc_clear_tx_bd(&temp_bd); 453 454 flags = 0; 455 tx_swbd++; 456 txbd++; 457 i++; 458 if (unlikely(i == tx_ring->bd_count)) { 459 i = 0; 460 tx_swbd = tx_ring->tx_swbd; 461 txbd = ENETC_TXBD(*tx_ring, 0); 462 } 463 prefetchw(txbd); 464 465 temp_bd.addr = cpu_to_le64(dma); 466 temp_bd.buf_len = cpu_to_le16(len); 467 468 tx_swbd->dma = dma; 469 tx_swbd->len = len; 470 tx_swbd->is_dma_page = 1; 471 tx_swbd->dir = DMA_TO_DEVICE; 472 count++; 473 } 474 475 /* last BD needs 'F' bit set */ 476 flags |= ENETC_TXBD_FLAGS_F; 477 temp_bd.flags = flags; 478 *txbd = temp_bd; 479 480 tx_ring->tx_swbd[i].is_eof = true; 481 tx_ring->tx_swbd[i].skb = skb; 482 483 enetc_bdr_idx_inc(tx_ring, &i); 484 tx_ring->next_to_use = i; 485 486 skb_tx_timestamp(skb); 487 488 enetc_update_tx_ring_tail(tx_ring); 489 490 return count; 491 492 dma_err: 493 dev_err(tx_ring->dev, "DMA map error"); 494 495 enetc_unwind_tx_frame(tx_ring, count, i); 496 497 return 0; 498 } 499 500 static int enetc_map_tx_tso_hdr(struct enetc_bdr *tx_ring, struct sk_buff *skb, 501 struct enetc_tx_swbd *tx_swbd, 502 union enetc_tx_bd *txbd, int *i, int hdr_len, 503 int data_len) 504 { 505 union enetc_tx_bd txbd_tmp; 506 u8 flags = 0, e_flags = 0; 507 dma_addr_t addr; 508 int count = 1; 509 510 enetc_clear_tx_bd(&txbd_tmp); 511 addr = tx_ring->tso_headers_dma + *i * TSO_HEADER_SIZE; 512 513 if (skb_vlan_tag_present(skb)) 514 flags |= ENETC_TXBD_FLAGS_EX; 515 516 txbd_tmp.addr = cpu_to_le64(addr); 517 txbd_tmp.buf_len = cpu_to_le16(hdr_len); 518 519 /* first BD needs frm_len and offload flags set */ 520 txbd_tmp.frm_len = cpu_to_le16(hdr_len + data_len); 521 txbd_tmp.flags = flags; 522 523 /* For the TSO header we do not set the dma address since we do not 524 * want it unmapped when we do cleanup. We still set len so that we 525 * count the bytes sent. 526 */ 527 tx_swbd->len = hdr_len; 528 tx_swbd->do_twostep_tstamp = false; 529 tx_swbd->check_wb = false; 530 531 /* Actually write the header in the BD */ 532 *txbd = txbd_tmp; 533 534 /* Add extension BD for VLAN */ 535 if (flags & ENETC_TXBD_FLAGS_EX) { 536 /* Get the next BD */ 537 enetc_bdr_idx_inc(tx_ring, i); 538 txbd = ENETC_TXBD(*tx_ring, *i); 539 tx_swbd = &tx_ring->tx_swbd[*i]; 540 prefetchw(txbd); 541 542 /* Setup the VLAN fields */ 543 enetc_clear_tx_bd(&txbd_tmp); 544 txbd_tmp.ext.vid = cpu_to_le16(skb_vlan_tag_get(skb)); 545 txbd_tmp.ext.tpid = 0; /* < C-TAG */ 546 e_flags |= ENETC_TXBD_E_FLAGS_VLAN_INS; 547 548 /* Write the BD */ 549 txbd_tmp.ext.e_flags = e_flags; 550 *txbd = txbd_tmp; 551 count++; 552 } 553 554 return count; 555 } 556 557 static int enetc_map_tx_tso_data(struct enetc_bdr *tx_ring, struct sk_buff *skb, 558 struct enetc_tx_swbd *tx_swbd, 559 union enetc_tx_bd *txbd, char *data, 560 int size, bool last_bd) 561 { 562 union enetc_tx_bd txbd_tmp; 563 dma_addr_t addr; 564 u8 flags = 0; 565 566 enetc_clear_tx_bd(&txbd_tmp); 567 568 addr = dma_map_single(tx_ring->dev, data, size, DMA_TO_DEVICE); 569 if (unlikely(dma_mapping_error(tx_ring->dev, addr))) { 570 netdev_err(tx_ring->ndev, "DMA map error\n"); 571 return -ENOMEM; 572 } 573 574 if (last_bd) { 575 flags |= ENETC_TXBD_FLAGS_F; 576 tx_swbd->is_eof = 1; 577 } 578 579 txbd_tmp.addr = cpu_to_le64(addr); 580 txbd_tmp.buf_len = cpu_to_le16(size); 581 txbd_tmp.flags = flags; 582 583 tx_swbd->dma = addr; 584 tx_swbd->len = size; 585 tx_swbd->dir = DMA_TO_DEVICE; 586 587 *txbd = txbd_tmp; 588 589 return 0; 590 } 591 592 static __wsum enetc_tso_hdr_csum(struct tso_t *tso, struct sk_buff *skb, 593 char *hdr, int hdr_len, int *l4_hdr_len) 594 { 595 char *l4_hdr = hdr + skb_transport_offset(skb); 596 int mac_hdr_len = skb_network_offset(skb); 597 598 if (tso->tlen != sizeof(struct udphdr)) { 599 struct tcphdr *tcph = (struct tcphdr *)(l4_hdr); 600 601 tcph->check = 0; 602 } else { 603 struct udphdr *udph = (struct udphdr *)(l4_hdr); 604 605 udph->check = 0; 606 } 607 608 /* Compute the IP checksum. This is necessary since tso_build_hdr() 609 * already incremented the IP ID field. 610 */ 611 if (!tso->ipv6) { 612 struct iphdr *iph = (void *)(hdr + mac_hdr_len); 613 614 iph->check = 0; 615 iph->check = ip_fast_csum((unsigned char *)iph, iph->ihl); 616 } 617 618 /* Compute the checksum over the L4 header. */ 619 *l4_hdr_len = hdr_len - skb_transport_offset(skb); 620 return csum_partial(l4_hdr, *l4_hdr_len, 0); 621 } 622 623 static void enetc_tso_complete_csum(struct enetc_bdr *tx_ring, struct tso_t *tso, 624 struct sk_buff *skb, char *hdr, int len, 625 __wsum sum) 626 { 627 char *l4_hdr = hdr + skb_transport_offset(skb); 628 __sum16 csum_final; 629 630 /* Complete the L4 checksum by appending the pseudo-header to the 631 * already computed checksum. 632 */ 633 if (!tso->ipv6) 634 csum_final = csum_tcpudp_magic(ip_hdr(skb)->saddr, 635 ip_hdr(skb)->daddr, 636 len, ip_hdr(skb)->protocol, sum); 637 else 638 csum_final = csum_ipv6_magic(&ipv6_hdr(skb)->saddr, 639 &ipv6_hdr(skb)->daddr, 640 len, ipv6_hdr(skb)->nexthdr, sum); 641 642 if (tso->tlen != sizeof(struct udphdr)) { 643 struct tcphdr *tcph = (struct tcphdr *)(l4_hdr); 644 645 tcph->check = csum_final; 646 } else { 647 struct udphdr *udph = (struct udphdr *)(l4_hdr); 648 649 udph->check = csum_final; 650 } 651 } 652 653 static int enetc_lso_count_descs(const struct sk_buff *skb) 654 { 655 /* 4 BDs: 1 BD for LSO header + 1 BD for extended BD + 1 BD 656 * for linear area data but not include LSO header, namely 657 * skb_headlen(skb) - lso_hdr_len (it may be 0, but that's 658 * okay, we only need to consider the worst case). And 1 BD 659 * for gap. 660 */ 661 return skb_shinfo(skb)->nr_frags + 4; 662 } 663 664 static int enetc_lso_get_hdr_len(const struct sk_buff *skb) 665 { 666 int hdr_len, tlen; 667 668 tlen = skb_is_gso_tcp(skb) ? tcp_hdrlen(skb) : sizeof(struct udphdr); 669 hdr_len = skb_transport_offset(skb) + tlen; 670 671 return hdr_len; 672 } 673 674 static void enetc_lso_start(struct sk_buff *skb, struct enetc_lso_t *lso) 675 { 676 lso->lso_seg_size = skb_shinfo(skb)->gso_size; 677 lso->ipv6 = enetc_skb_is_ipv6(skb); 678 lso->tcp = skb_is_gso_tcp(skb); 679 lso->l3_hdr_len = skb_network_header_len(skb); 680 lso->l3_start = skb_network_offset(skb); 681 lso->hdr_len = enetc_lso_get_hdr_len(skb); 682 lso->total_len = skb->len - lso->hdr_len; 683 } 684 685 static void enetc_lso_map_hdr(struct enetc_bdr *tx_ring, struct sk_buff *skb, 686 int *i, struct enetc_lso_t *lso) 687 { 688 union enetc_tx_bd txbd_tmp, *txbd; 689 struct enetc_tx_swbd *tx_swbd; 690 u16 frm_len, frm_len_ext; 691 u8 flags, e_flags = 0; 692 dma_addr_t addr; 693 char *hdr; 694 695 /* Get the first BD of the LSO BDs chain */ 696 txbd = ENETC_TXBD(*tx_ring, *i); 697 tx_swbd = &tx_ring->tx_swbd[*i]; 698 prefetchw(txbd); 699 700 /* Prepare LSO header: MAC + IP + TCP/UDP */ 701 hdr = tx_ring->tso_headers + *i * TSO_HEADER_SIZE; 702 memcpy(hdr, skb->data, lso->hdr_len); 703 addr = tx_ring->tso_headers_dma + *i * TSO_HEADER_SIZE; 704 705 /* {frm_len_ext, frm_len} indicates the total length of 706 * large transmit data unit. frm_len contains the 16 least 707 * significant bits and frm_len_ext contains the 4 most 708 * significant bits. 709 */ 710 frm_len = lso->total_len & 0xffff; 711 frm_len_ext = (lso->total_len >> 16) & 0xf; 712 713 /* Set the flags of the first BD */ 714 flags = ENETC_TXBD_FLAGS_EX | ENETC_TXBD_FLAGS_CSUM_LSO | 715 ENETC_TXBD_FLAGS_LSO | ENETC_TXBD_FLAGS_L4CS; 716 717 enetc_clear_tx_bd(&txbd_tmp); 718 txbd_tmp.addr = cpu_to_le64(addr); 719 txbd_tmp.hdr_len = cpu_to_le16(lso->hdr_len); 720 721 /* first BD needs frm_len and offload flags set */ 722 txbd_tmp.frm_len = cpu_to_le16(frm_len); 723 txbd_tmp.flags = flags; 724 725 txbd_tmp.l3_aux0 = FIELD_PREP(ENETC_TX_BD_L3_START, lso->l3_start); 726 /* l3_hdr_size in 32-bits (4 bytes) */ 727 txbd_tmp.l3_aux1 = FIELD_PREP(ENETC_TX_BD_L3_HDR_LEN, 728 lso->l3_hdr_len / 4); 729 if (lso->ipv6) 730 txbd_tmp.l3_aux1 |= ENETC_TX_BD_L3T; 731 else 732 txbd_tmp.l3_aux0 |= ENETC_TX_BD_IPCS; 733 734 txbd_tmp.l4_aux = FIELD_PREP(ENETC_TX_BD_L4T, lso->tcp ? 735 ENETC_TXBD_L4T_TCP : ENETC_TXBD_L4T_UDP); 736 737 /* For the LSO header we do not set the dma address since 738 * we do not want it unmapped when we do cleanup. We still 739 * set len so that we count the bytes sent. 740 */ 741 tx_swbd->len = lso->hdr_len; 742 tx_swbd->do_twostep_tstamp = false; 743 tx_swbd->check_wb = false; 744 745 /* Actually write the header in the BD */ 746 *txbd = txbd_tmp; 747 748 /* Get the next BD, and the next BD is extended BD */ 749 enetc_bdr_idx_inc(tx_ring, i); 750 txbd = ENETC_TXBD(*tx_ring, *i); 751 tx_swbd = &tx_ring->tx_swbd[*i]; 752 prefetchw(txbd); 753 754 enetc_clear_tx_bd(&txbd_tmp); 755 if (skb_vlan_tag_present(skb)) { 756 /* Setup the VLAN fields */ 757 txbd_tmp.ext.vid = cpu_to_le16(skb_vlan_tag_get(skb)); 758 txbd_tmp.ext.tpid = ENETC_TPID_8021Q; 759 e_flags = ENETC_TXBD_E_FLAGS_VLAN_INS; 760 } 761 762 /* Write the BD */ 763 txbd_tmp.ext.e_flags = e_flags; 764 txbd_tmp.ext.lso_sg_size = cpu_to_le16(lso->lso_seg_size); 765 txbd_tmp.ext.frm_len_ext = cpu_to_le16(frm_len_ext); 766 *txbd = txbd_tmp; 767 } 768 769 static int enetc_lso_map_data(struct enetc_bdr *tx_ring, struct sk_buff *skb, 770 int *i, struct enetc_lso_t *lso, int *count) 771 { 772 union enetc_tx_bd txbd_tmp, *txbd = NULL; 773 struct enetc_tx_swbd *tx_swbd; 774 skb_frag_t *frag; 775 dma_addr_t dma; 776 u8 flags = 0; 777 int len, f; 778 779 len = skb_headlen(skb) - lso->hdr_len; 780 if (len > 0) { 781 dma = dma_map_single(tx_ring->dev, skb->data + lso->hdr_len, 782 len, DMA_TO_DEVICE); 783 if (dma_mapping_error(tx_ring->dev, dma)) 784 return -ENOMEM; 785 786 enetc_bdr_idx_inc(tx_ring, i); 787 txbd = ENETC_TXBD(*tx_ring, *i); 788 tx_swbd = &tx_ring->tx_swbd[*i]; 789 prefetchw(txbd); 790 *count += 1; 791 792 enetc_clear_tx_bd(&txbd_tmp); 793 txbd_tmp.addr = cpu_to_le64(dma); 794 txbd_tmp.buf_len = cpu_to_le16(len); 795 796 tx_swbd->dma = dma; 797 tx_swbd->len = len; 798 tx_swbd->is_dma_page = 0; 799 tx_swbd->dir = DMA_TO_DEVICE; 800 } 801 802 frag = &skb_shinfo(skb)->frags[0]; 803 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++, frag++) { 804 if (txbd) 805 *txbd = txbd_tmp; 806 807 len = skb_frag_size(frag); 808 dma = skb_frag_dma_map(tx_ring->dev, frag); 809 if (dma_mapping_error(tx_ring->dev, dma)) 810 return -ENOMEM; 811 812 /* Get the next BD */ 813 enetc_bdr_idx_inc(tx_ring, i); 814 txbd = ENETC_TXBD(*tx_ring, *i); 815 tx_swbd = &tx_ring->tx_swbd[*i]; 816 prefetchw(txbd); 817 *count += 1; 818 819 enetc_clear_tx_bd(&txbd_tmp); 820 txbd_tmp.addr = cpu_to_le64(dma); 821 txbd_tmp.buf_len = cpu_to_le16(len); 822 823 tx_swbd->dma = dma; 824 tx_swbd->len = len; 825 tx_swbd->is_dma_page = 1; 826 tx_swbd->dir = DMA_TO_DEVICE; 827 } 828 829 /* Last BD needs 'F' bit set */ 830 flags |= ENETC_TXBD_FLAGS_F; 831 txbd_tmp.flags = flags; 832 *txbd = txbd_tmp; 833 834 tx_swbd->is_eof = 1; 835 tx_swbd->skb = skb; 836 837 return 0; 838 } 839 840 static int enetc_lso_hw_offload(struct enetc_bdr *tx_ring, struct sk_buff *skb) 841 { 842 struct enetc_tx_swbd *tx_swbd; 843 struct enetc_lso_t lso = {0}; 844 int err, i, count = 0; 845 846 /* Initialize the LSO handler */ 847 enetc_lso_start(skb, &lso); 848 i = tx_ring->next_to_use; 849 850 enetc_lso_map_hdr(tx_ring, skb, &i, &lso); 851 /* First BD and an extend BD */ 852 count += 2; 853 854 err = enetc_lso_map_data(tx_ring, skb, &i, &lso, &count); 855 if (err) 856 goto dma_err; 857 858 /* Go to the next BD */ 859 enetc_bdr_idx_inc(tx_ring, &i); 860 tx_ring->next_to_use = i; 861 enetc_update_tx_ring_tail(tx_ring); 862 863 return count; 864 865 dma_err: 866 do { 867 tx_swbd = &tx_ring->tx_swbd[i]; 868 enetc_free_tx_frame(tx_ring, tx_swbd); 869 if (i == 0) 870 i = tx_ring->bd_count; 871 i--; 872 } while (--count); 873 874 return 0; 875 } 876 877 static int enetc_map_tx_tso_buffs(struct enetc_bdr *tx_ring, struct sk_buff *skb) 878 { 879 struct enetc_ndev_priv *priv = netdev_priv(tx_ring->ndev); 880 int hdr_len, total_len, data_len; 881 struct enetc_tx_swbd *tx_swbd; 882 union enetc_tx_bd *txbd; 883 struct tso_t tso; 884 __wsum csum, csum2; 885 int count = 0, pos; 886 int err, i, bd_data_num; 887 888 /* Initialize the TSO handler, and prepare the first payload */ 889 hdr_len = tso_start(skb, &tso); 890 total_len = skb->len - hdr_len; 891 i = tx_ring->next_to_use; 892 893 while (total_len > 0) { 894 char *hdr; 895 896 /* Get the BD */ 897 txbd = ENETC_TXBD(*tx_ring, i); 898 tx_swbd = &tx_ring->tx_swbd[i]; 899 prefetchw(txbd); 900 901 /* Determine the length of this packet */ 902 data_len = min_t(int, skb_shinfo(skb)->gso_size, total_len); 903 total_len -= data_len; 904 905 /* prepare packet headers: MAC + IP + TCP */ 906 hdr = tx_ring->tso_headers + i * TSO_HEADER_SIZE; 907 tso_build_hdr(skb, hdr, &tso, data_len, total_len == 0); 908 909 /* compute the csum over the L4 header */ 910 csum = enetc_tso_hdr_csum(&tso, skb, hdr, hdr_len, &pos); 911 count += enetc_map_tx_tso_hdr(tx_ring, skb, tx_swbd, txbd, 912 &i, hdr_len, data_len); 913 bd_data_num = 0; 914 915 while (data_len > 0) { 916 int size; 917 918 size = min_t(int, tso.size, data_len); 919 920 /* Advance the index in the BDR */ 921 enetc_bdr_idx_inc(tx_ring, &i); 922 txbd = ENETC_TXBD(*tx_ring, i); 923 tx_swbd = &tx_ring->tx_swbd[i]; 924 prefetchw(txbd); 925 926 /* Compute the checksum over this segment of data and 927 * add it to the csum already computed (over the L4 928 * header and possible other data segments). 929 */ 930 csum2 = csum_partial(tso.data, size, 0); 931 csum = csum_block_add(csum, csum2, pos); 932 pos += size; 933 934 err = enetc_map_tx_tso_data(tx_ring, skb, tx_swbd, txbd, 935 tso.data, size, 936 size == data_len); 937 if (err) { 938 if (i == 0) 939 i = tx_ring->bd_count; 940 i--; 941 942 goto err_map_data; 943 } 944 945 data_len -= size; 946 count++; 947 bd_data_num++; 948 tso_build_data(skb, &tso, size); 949 950 if (unlikely(bd_data_num >= priv->max_frags && data_len)) 951 goto err_chained_bd; 952 } 953 954 enetc_tso_complete_csum(tx_ring, &tso, skb, hdr, pos, csum); 955 956 if (total_len == 0) 957 tx_swbd->skb = skb; 958 959 /* Go to the next BD */ 960 enetc_bdr_idx_inc(tx_ring, &i); 961 } 962 963 tx_ring->next_to_use = i; 964 enetc_update_tx_ring_tail(tx_ring); 965 966 return count; 967 968 err_map_data: 969 dev_err(tx_ring->dev, "DMA map error"); 970 971 err_chained_bd: 972 enetc_unwind_tx_frame(tx_ring, count, i); 973 974 return 0; 975 } 976 977 static netdev_tx_t enetc_start_xmit(struct sk_buff *skb, 978 struct net_device *ndev) 979 { 980 struct enetc_skb_cb *enetc_cb = ENETC_SKB_CB(skb); 981 struct enetc_ndev_priv *priv = netdev_priv(ndev); 982 struct enetc_bdr *tx_ring; 983 int count; 984 985 /* Queue one-step Sync packet if already locked */ 986 if (enetc_cb->flag & ENETC_F_TX_ONESTEP_SYNC_TSTAMP) { 987 if (test_and_set_bit_lock(ENETC_TX_ONESTEP_TSTAMP_IN_PROGRESS, 988 &priv->flags)) { 989 skb_queue_tail(&priv->tx_skbs, skb); 990 return NETDEV_TX_OK; 991 } 992 } 993 994 tx_ring = priv->tx_ring[skb->queue_mapping]; 995 996 if (skb_is_gso(skb)) { 997 /* LSO data unit lengths of up to 256KB are supported */ 998 if (priv->active_offloads & ENETC_F_LSO && 999 (skb->len - enetc_lso_get_hdr_len(skb)) <= 1000 ENETC_LSO_MAX_DATA_LEN) { 1001 if (enetc_bd_unused(tx_ring) < enetc_lso_count_descs(skb)) { 1002 netif_stop_subqueue(ndev, tx_ring->index); 1003 return NETDEV_TX_BUSY; 1004 } 1005 1006 count = enetc_lso_hw_offload(tx_ring, skb); 1007 } else { 1008 if (enetc_bd_unused(tx_ring) < tso_count_descs(skb)) { 1009 netif_stop_subqueue(ndev, tx_ring->index); 1010 return NETDEV_TX_BUSY; 1011 } 1012 1013 enetc_lock_mdio(); 1014 count = enetc_map_tx_tso_buffs(tx_ring, skb); 1015 enetc_unlock_mdio(); 1016 } 1017 } else { 1018 if (unlikely(skb_shinfo(skb)->nr_frags > priv->max_frags)) 1019 if (unlikely(skb_linearize(skb))) 1020 goto drop_packet_err; 1021 1022 count = skb_shinfo(skb)->nr_frags + 1; /* fragments + head */ 1023 if (enetc_bd_unused(tx_ring) < ENETC_TXBDS_NEEDED(count)) { 1024 netif_stop_subqueue(ndev, tx_ring->index); 1025 return NETDEV_TX_BUSY; 1026 } 1027 1028 enetc_lock_mdio(); 1029 count = enetc_map_tx_buffs(tx_ring, skb); 1030 enetc_unlock_mdio(); 1031 } 1032 1033 if (unlikely(!count)) 1034 goto drop_packet_err; 1035 1036 if (enetc_bd_unused(tx_ring) < ENETC_TXBDS_MAX_NEEDED(priv->max_frags)) 1037 netif_stop_subqueue(ndev, tx_ring->index); 1038 1039 return NETDEV_TX_OK; 1040 1041 drop_packet_err: 1042 dev_kfree_skb_any(skb); 1043 return NETDEV_TX_OK; 1044 } 1045 1046 netdev_tx_t enetc_xmit(struct sk_buff *skb, struct net_device *ndev) 1047 { 1048 struct enetc_skb_cb *enetc_cb = ENETC_SKB_CB(skb); 1049 struct enetc_ndev_priv *priv = netdev_priv(ndev); 1050 u8 udp, msgtype, twostep; 1051 u16 offset1, offset2; 1052 1053 /* Mark tx timestamp type on enetc_cb->flag if requires */ 1054 if ((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && 1055 (priv->active_offloads & ENETC_F_TX_TSTAMP_MASK)) 1056 enetc_cb->flag = priv->active_offloads & ENETC_F_TX_TSTAMP_MASK; 1057 else 1058 enetc_cb->flag = 0; 1059 1060 /* Fall back to two-step timestamp if not one-step Sync packet */ 1061 if (enetc_cb->flag & ENETC_F_TX_ONESTEP_SYNC_TSTAMP) { 1062 if (enetc_ptp_parse(skb, &udp, &msgtype, &twostep, 1063 &offset1, &offset2) || 1064 msgtype != PTP_MSGTYPE_SYNC || twostep != 0) { 1065 enetc_cb->flag = ENETC_F_TX_TSTAMP; 1066 } else { 1067 enetc_cb->udp = !!udp; 1068 enetc_cb->correction_off = offset1; 1069 enetc_cb->origin_tstamp_off = offset2; 1070 } 1071 } 1072 1073 return enetc_start_xmit(skb, ndev); 1074 } 1075 EXPORT_SYMBOL_GPL(enetc_xmit); 1076 1077 static irqreturn_t enetc_msix(int irq, void *data) 1078 { 1079 struct enetc_int_vector *v = data; 1080 int i; 1081 1082 enetc_lock_mdio(); 1083 1084 /* disable interrupts */ 1085 enetc_wr_reg_hot(v->rbier, 0); 1086 enetc_wr_reg_hot(v->ricr1, v->rx_ictt); 1087 1088 for_each_set_bit(i, &v->tx_rings_map, ENETC_MAX_NUM_TXQS) 1089 enetc_wr_reg_hot(v->tbier_base + ENETC_BDR_OFF(i), 0); 1090 1091 enetc_unlock_mdio(); 1092 1093 napi_schedule(&v->napi); 1094 1095 return IRQ_HANDLED; 1096 } 1097 1098 static void enetc_rx_dim_work(struct work_struct *w) 1099 { 1100 struct dim *dim = container_of(w, struct dim, work); 1101 struct dim_cq_moder moder = 1102 net_dim_get_rx_moderation(dim->mode, dim->profile_ix); 1103 struct enetc_int_vector *v = 1104 container_of(dim, struct enetc_int_vector, rx_dim); 1105 struct enetc_ndev_priv *priv = netdev_priv(v->rx_ring.ndev); 1106 1107 v->rx_ictt = enetc_usecs_to_cycles(moder.usec, priv->sysclk_freq); 1108 dim->state = DIM_START_MEASURE; 1109 } 1110 1111 static void enetc_rx_net_dim(struct enetc_int_vector *v) 1112 { 1113 struct dim_sample dim_sample = {}; 1114 1115 v->comp_cnt++; 1116 1117 if (!v->rx_napi_work) 1118 return; 1119 1120 dim_update_sample(v->comp_cnt, 1121 v->rx_ring.stats.packets, 1122 v->rx_ring.stats.bytes, 1123 &dim_sample); 1124 net_dim(&v->rx_dim, &dim_sample); 1125 } 1126 1127 static int enetc_bd_ready_count(struct enetc_bdr *tx_ring, int ci) 1128 { 1129 int pi = enetc_rd_reg_hot(tx_ring->tcir) & ENETC_TBCIR_IDX_MASK; 1130 1131 return pi >= ci ? pi - ci : tx_ring->bd_count - ci + pi; 1132 } 1133 1134 static bool enetc_page_reusable(struct page *page) 1135 { 1136 return (!page_is_pfmemalloc(page) && page_ref_count(page) == 1); 1137 } 1138 1139 static void enetc_reuse_page(struct enetc_bdr *rx_ring, 1140 struct enetc_rx_swbd *old) 1141 { 1142 struct enetc_rx_swbd *new; 1143 1144 new = &rx_ring->rx_swbd[rx_ring->next_to_alloc]; 1145 1146 /* next buf that may reuse a page */ 1147 enetc_bdr_idx_inc(rx_ring, &rx_ring->next_to_alloc); 1148 1149 /* copy page reference */ 1150 *new = *old; 1151 } 1152 1153 static void enetc_get_tx_tstamp(struct enetc_hw *hw, union enetc_tx_bd *txbd, 1154 u64 *tstamp) 1155 { 1156 u32 lo, hi, tstamp_lo; 1157 1158 lo = enetc_rd_hot(hw, ENETC_SICTR0); 1159 hi = enetc_rd_hot(hw, ENETC_SICTR1); 1160 tstamp_lo = le32_to_cpu(txbd->wb.tstamp); 1161 if (lo <= tstamp_lo) 1162 hi -= 1; 1163 *tstamp = (u64)hi << 32 | tstamp_lo; 1164 } 1165 1166 static void enetc_tstamp_tx(struct sk_buff *skb, u64 tstamp) 1167 { 1168 struct skb_shared_hwtstamps shhwtstamps; 1169 1170 if (skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS) { 1171 memset(&shhwtstamps, 0, sizeof(shhwtstamps)); 1172 shhwtstamps.hwtstamp = ns_to_ktime(tstamp); 1173 skb_txtime_consumed(skb); 1174 skb_tstamp_tx(skb, &shhwtstamps); 1175 } 1176 } 1177 1178 static void enetc_recycle_xdp_tx_buff(struct enetc_bdr *tx_ring, 1179 struct enetc_tx_swbd *tx_swbd) 1180 { 1181 struct enetc_ndev_priv *priv = netdev_priv(tx_ring->ndev); 1182 struct enetc_rx_swbd rx_swbd = { 1183 .dma = tx_swbd->dma, 1184 .page = tx_swbd->page, 1185 .page_offset = tx_swbd->page_offset, 1186 .dir = tx_swbd->dir, 1187 .len = tx_swbd->len, 1188 }; 1189 struct enetc_bdr *rx_ring; 1190 1191 rx_ring = enetc_rx_ring_from_xdp_tx_ring(priv, tx_ring); 1192 1193 if (likely(enetc_swbd_unused(rx_ring))) { 1194 enetc_reuse_page(rx_ring, &rx_swbd); 1195 1196 /* sync for use by the device */ 1197 dma_sync_single_range_for_device(rx_ring->dev, rx_swbd.dma, 1198 rx_swbd.page_offset, 1199 ENETC_RXB_DMA_SIZE_XDP, 1200 rx_swbd.dir); 1201 1202 rx_ring->stats.recycles++; 1203 } else { 1204 /* RX ring is already full, we need to unmap and free the 1205 * page, since there's nothing useful we can do with it. 1206 */ 1207 rx_ring->stats.recycle_failures++; 1208 1209 dma_unmap_page(rx_ring->dev, rx_swbd.dma, PAGE_SIZE, 1210 rx_swbd.dir); 1211 __free_page(rx_swbd.page); 1212 } 1213 1214 rx_ring->xdp.xdp_tx_in_flight--; 1215 } 1216 1217 static bool enetc_clean_tx_ring(struct enetc_bdr *tx_ring, int napi_budget) 1218 { 1219 int tx_frm_cnt = 0, tx_byte_cnt = 0, tx_win_drop = 0; 1220 struct net_device *ndev = tx_ring->ndev; 1221 struct enetc_ndev_priv *priv = netdev_priv(ndev); 1222 struct enetc_tx_swbd *tx_swbd; 1223 int i, bds_to_clean; 1224 bool do_twostep_tstamp; 1225 u64 tstamp = 0; 1226 1227 i = tx_ring->next_to_clean; 1228 tx_swbd = &tx_ring->tx_swbd[i]; 1229 1230 bds_to_clean = enetc_bd_ready_count(tx_ring, i); 1231 1232 do_twostep_tstamp = false; 1233 1234 while (bds_to_clean && tx_frm_cnt < ENETC_DEFAULT_TX_WORK) { 1235 struct xdp_frame *xdp_frame = enetc_tx_swbd_get_xdp_frame(tx_swbd); 1236 struct sk_buff *skb = enetc_tx_swbd_get_skb(tx_swbd); 1237 bool is_eof = tx_swbd->is_eof; 1238 1239 if (unlikely(tx_swbd->check_wb)) { 1240 union enetc_tx_bd *txbd = ENETC_TXBD(*tx_ring, i); 1241 1242 if (txbd->flags & ENETC_TXBD_FLAGS_W && 1243 tx_swbd->do_twostep_tstamp) { 1244 enetc_get_tx_tstamp(&priv->si->hw, txbd, 1245 &tstamp); 1246 do_twostep_tstamp = true; 1247 } 1248 1249 if (tx_swbd->qbv_en && 1250 txbd->wb.status & ENETC_TXBD_STATS_WIN) 1251 tx_win_drop++; 1252 } 1253 1254 if (tx_swbd->is_xdp_tx) 1255 enetc_recycle_xdp_tx_buff(tx_ring, tx_swbd); 1256 else if (likely(tx_swbd->dma)) 1257 enetc_unmap_tx_buff(tx_ring, tx_swbd); 1258 1259 if (xdp_frame) { 1260 xdp_return_frame(xdp_frame); 1261 } else if (skb) { 1262 struct enetc_skb_cb *enetc_cb = ENETC_SKB_CB(skb); 1263 1264 if (unlikely(enetc_cb->flag & ENETC_F_TX_ONESTEP_SYNC_TSTAMP)) { 1265 /* Start work to release lock for next one-step 1266 * timestamping packet. And send one skb in 1267 * tx_skbs queue if has. 1268 */ 1269 schedule_work(&priv->tx_onestep_tstamp); 1270 } else if (unlikely(do_twostep_tstamp)) { 1271 enetc_tstamp_tx(skb, tstamp); 1272 do_twostep_tstamp = false; 1273 } 1274 napi_consume_skb(skb, napi_budget); 1275 } 1276 1277 tx_byte_cnt += tx_swbd->len; 1278 /* Scrub the swbd here so we don't have to do that 1279 * when we reuse it during xmit 1280 */ 1281 memset(tx_swbd, 0, sizeof(*tx_swbd)); 1282 1283 bds_to_clean--; 1284 tx_swbd++; 1285 i++; 1286 if (unlikely(i == tx_ring->bd_count)) { 1287 i = 0; 1288 tx_swbd = tx_ring->tx_swbd; 1289 } 1290 1291 /* BD iteration loop end */ 1292 if (is_eof) { 1293 tx_frm_cnt++; 1294 /* re-arm interrupt source */ 1295 enetc_wr_reg_hot(tx_ring->idr, BIT(tx_ring->index) | 1296 BIT(16 + tx_ring->index)); 1297 } 1298 1299 if (unlikely(!bds_to_clean)) 1300 bds_to_clean = enetc_bd_ready_count(tx_ring, i); 1301 } 1302 1303 tx_ring->next_to_clean = i; 1304 tx_ring->stats.packets += tx_frm_cnt; 1305 tx_ring->stats.bytes += tx_byte_cnt; 1306 tx_ring->stats.win_drop += tx_win_drop; 1307 1308 if (unlikely(tx_frm_cnt && netif_carrier_ok(ndev) && 1309 __netif_subqueue_stopped(ndev, tx_ring->index) && 1310 !test_bit(ENETC_TX_DOWN, &priv->flags) && 1311 (enetc_bd_unused(tx_ring) >= 1312 ENETC_TXBDS_MAX_NEEDED(priv->max_frags)))) { 1313 netif_wake_subqueue(ndev, tx_ring->index); 1314 } 1315 1316 return tx_frm_cnt != ENETC_DEFAULT_TX_WORK; 1317 } 1318 1319 static bool enetc_new_page(struct enetc_bdr *rx_ring, 1320 struct enetc_rx_swbd *rx_swbd) 1321 { 1322 bool xdp = !!(rx_ring->xdp.prog); 1323 struct page *page; 1324 dma_addr_t addr; 1325 1326 page = dev_alloc_page(); 1327 if (unlikely(!page)) 1328 return false; 1329 1330 /* For XDP_TX, we forgo dma_unmap -> dma_map */ 1331 rx_swbd->dir = xdp ? DMA_BIDIRECTIONAL : DMA_FROM_DEVICE; 1332 1333 addr = dma_map_page(rx_ring->dev, page, 0, PAGE_SIZE, rx_swbd->dir); 1334 if (unlikely(dma_mapping_error(rx_ring->dev, addr))) { 1335 __free_page(page); 1336 1337 return false; 1338 } 1339 1340 rx_swbd->dma = addr; 1341 rx_swbd->page = page; 1342 rx_swbd->page_offset = rx_ring->buffer_offset; 1343 1344 return true; 1345 } 1346 1347 static int enetc_refill_rx_ring(struct enetc_bdr *rx_ring, const int buff_cnt) 1348 { 1349 struct enetc_rx_swbd *rx_swbd; 1350 union enetc_rx_bd *rxbd; 1351 int i, j; 1352 1353 i = rx_ring->next_to_use; 1354 rx_swbd = &rx_ring->rx_swbd[i]; 1355 rxbd = enetc_rxbd(rx_ring, i); 1356 1357 for (j = 0; j < buff_cnt; j++) { 1358 /* try reuse page */ 1359 if (unlikely(!rx_swbd->page)) { 1360 if (unlikely(!enetc_new_page(rx_ring, rx_swbd))) { 1361 rx_ring->stats.rx_alloc_errs++; 1362 break; 1363 } 1364 } 1365 1366 /* update RxBD */ 1367 rxbd->w.addr = cpu_to_le64(rx_swbd->dma + 1368 rx_swbd->page_offset); 1369 /* clear 'R" as well */ 1370 rxbd->r.lstatus = 0; 1371 1372 enetc_rxbd_next(rx_ring, &rxbd, &i); 1373 rx_swbd = &rx_ring->rx_swbd[i]; 1374 } 1375 1376 if (likely(j)) { 1377 rx_ring->next_to_alloc = i; /* keep track from page reuse */ 1378 rx_ring->next_to_use = i; 1379 1380 /* update ENETC's consumer index */ 1381 enetc_wr_reg_hot(rx_ring->rcir, rx_ring->next_to_use); 1382 } 1383 1384 return j; 1385 } 1386 1387 static void enetc_get_rx_tstamp(struct net_device *ndev, 1388 union enetc_rx_bd *rxbd, 1389 struct sk_buff *skb) 1390 { 1391 struct skb_shared_hwtstamps *shhwtstamps = skb_hwtstamps(skb); 1392 struct enetc_ndev_priv *priv = netdev_priv(ndev); 1393 struct enetc_hw *hw = &priv->si->hw; 1394 u32 lo, hi, tstamp_lo; 1395 u64 tstamp; 1396 1397 if (le16_to_cpu(rxbd->r.flags) & ENETC_RXBD_FLAG_TSTMP) { 1398 lo = enetc_rd_reg_hot(hw->reg + ENETC_SICTR0); 1399 hi = enetc_rd_reg_hot(hw->reg + ENETC_SICTR1); 1400 rxbd = enetc_rxbd_ext(rxbd); 1401 tstamp_lo = le32_to_cpu(rxbd->ext.tstamp); 1402 if (lo <= tstamp_lo) 1403 hi -= 1; 1404 1405 tstamp = (u64)hi << 32 | tstamp_lo; 1406 memset(shhwtstamps, 0, sizeof(*shhwtstamps)); 1407 shhwtstamps->hwtstamp = ns_to_ktime(tstamp); 1408 } 1409 } 1410 1411 static void enetc_get_offloads(struct enetc_bdr *rx_ring, 1412 union enetc_rx_bd *rxbd, struct sk_buff *skb) 1413 { 1414 struct enetc_ndev_priv *priv = netdev_priv(rx_ring->ndev); 1415 1416 /* TODO: hashing */ 1417 if (rx_ring->ndev->features & NETIF_F_RXCSUM) { 1418 u16 inet_csum = le16_to_cpu(rxbd->r.inet_csum); 1419 1420 skb->csum = csum_unfold((__force __sum16)~htons(inet_csum)); 1421 skb->ip_summed = CHECKSUM_COMPLETE; 1422 } 1423 1424 if (le16_to_cpu(rxbd->r.flags) & ENETC_RXBD_FLAG_VLAN) { 1425 struct enetc_hw *hw = &priv->si->hw; 1426 __be16 tpid = 0; 1427 1428 switch (le16_to_cpu(rxbd->r.flags) & ENETC_RXBD_FLAG_TPID) { 1429 case 0: 1430 tpid = htons(ETH_P_8021Q); 1431 break; 1432 case 1: 1433 tpid = htons(ETH_P_8021AD); 1434 break; 1435 case 2: 1436 tpid = htons(enetc_rd_hot(hw, ENETC_SICVLANR1) & 1437 SICVLANR_ETYPE); 1438 break; 1439 case 3: 1440 tpid = htons(enetc_rd_hot(hw, ENETC_SICVLANR2) & 1441 SICVLANR_ETYPE); 1442 } 1443 1444 __vlan_hwaccel_put_tag(skb, tpid, le16_to_cpu(rxbd->r.vlan_opt)); 1445 } 1446 1447 if (priv->active_offloads & ENETC_F_RX_TSTAMP) 1448 enetc_get_rx_tstamp(rx_ring->ndev, rxbd, skb); 1449 } 1450 1451 /* This gets called during the non-XDP NAPI poll cycle as well as on XDP_PASS, 1452 * so it needs to work with both DMA_FROM_DEVICE as well as DMA_BIDIRECTIONAL 1453 * mapped buffers. 1454 */ 1455 static struct enetc_rx_swbd *enetc_get_rx_buff(struct enetc_bdr *rx_ring, 1456 int i, u16 size) 1457 { 1458 struct enetc_rx_swbd *rx_swbd = &rx_ring->rx_swbd[i]; 1459 1460 dma_sync_single_range_for_cpu(rx_ring->dev, rx_swbd->dma, 1461 rx_swbd->page_offset, 1462 size, rx_swbd->dir); 1463 return rx_swbd; 1464 } 1465 1466 /* Reuse the current page without performing half-page buffer flipping */ 1467 static void enetc_put_rx_buff(struct enetc_bdr *rx_ring, 1468 struct enetc_rx_swbd *rx_swbd) 1469 { 1470 size_t buffer_size = ENETC_RXB_TRUESIZE - rx_ring->buffer_offset; 1471 1472 enetc_reuse_page(rx_ring, rx_swbd); 1473 1474 dma_sync_single_range_for_device(rx_ring->dev, rx_swbd->dma, 1475 rx_swbd->page_offset, 1476 buffer_size, rx_swbd->dir); 1477 1478 rx_swbd->page = NULL; 1479 } 1480 1481 /* Reuse the current page by performing half-page buffer flipping */ 1482 static void enetc_flip_rx_buff(struct enetc_bdr *rx_ring, 1483 struct enetc_rx_swbd *rx_swbd) 1484 { 1485 if (likely(enetc_page_reusable(rx_swbd->page))) { 1486 rx_swbd->page_offset ^= ENETC_RXB_TRUESIZE; 1487 page_ref_inc(rx_swbd->page); 1488 1489 enetc_put_rx_buff(rx_ring, rx_swbd); 1490 } else { 1491 dma_unmap_page(rx_ring->dev, rx_swbd->dma, PAGE_SIZE, 1492 rx_swbd->dir); 1493 rx_swbd->page = NULL; 1494 } 1495 } 1496 1497 static struct sk_buff *enetc_map_rx_buff_to_skb(struct enetc_bdr *rx_ring, 1498 int i, u16 size) 1499 { 1500 struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size); 1501 struct sk_buff *skb; 1502 void *ba; 1503 1504 ba = page_address(rx_swbd->page) + rx_swbd->page_offset; 1505 skb = build_skb(ba - rx_ring->buffer_offset, ENETC_RXB_TRUESIZE); 1506 if (unlikely(!skb)) { 1507 rx_ring->stats.rx_alloc_errs++; 1508 return NULL; 1509 } 1510 1511 skb_reserve(skb, rx_ring->buffer_offset); 1512 __skb_put(skb, size); 1513 1514 enetc_flip_rx_buff(rx_ring, rx_swbd); 1515 1516 return skb; 1517 } 1518 1519 static void enetc_add_rx_buff_to_skb(struct enetc_bdr *rx_ring, int i, 1520 u16 size, struct sk_buff *skb) 1521 { 1522 struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size); 1523 1524 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_swbd->page, 1525 rx_swbd->page_offset, size, ENETC_RXB_TRUESIZE); 1526 1527 enetc_flip_rx_buff(rx_ring, rx_swbd); 1528 } 1529 1530 static bool enetc_check_bd_errors_and_consume(struct enetc_bdr *rx_ring, 1531 u32 bd_status, 1532 union enetc_rx_bd **rxbd, int *i) 1533 { 1534 if (likely(!(bd_status & ENETC_RXBD_LSTATUS(ENETC_RXBD_ERR_MASK)))) 1535 return false; 1536 1537 enetc_put_rx_buff(rx_ring, &rx_ring->rx_swbd[*i]); 1538 enetc_rxbd_next(rx_ring, rxbd, i); 1539 1540 while (!(bd_status & ENETC_RXBD_LSTATUS_F)) { 1541 dma_rmb(); 1542 bd_status = le32_to_cpu((*rxbd)->r.lstatus); 1543 1544 enetc_put_rx_buff(rx_ring, &rx_ring->rx_swbd[*i]); 1545 enetc_rxbd_next(rx_ring, rxbd, i); 1546 } 1547 1548 rx_ring->ndev->stats.rx_dropped++; 1549 rx_ring->ndev->stats.rx_errors++; 1550 1551 return true; 1552 } 1553 1554 static struct sk_buff *enetc_build_skb(struct enetc_bdr *rx_ring, 1555 u32 bd_status, union enetc_rx_bd **rxbd, 1556 int *i, int *cleaned_cnt, int buffer_size) 1557 { 1558 struct sk_buff *skb; 1559 u16 size; 1560 1561 size = le16_to_cpu((*rxbd)->r.buf_len); 1562 skb = enetc_map_rx_buff_to_skb(rx_ring, *i, size); 1563 if (!skb) 1564 return NULL; 1565 1566 enetc_get_offloads(rx_ring, *rxbd, skb); 1567 1568 (*cleaned_cnt)++; 1569 1570 enetc_rxbd_next(rx_ring, rxbd, i); 1571 1572 /* not last BD in frame? */ 1573 while (!(bd_status & ENETC_RXBD_LSTATUS_F)) { 1574 bd_status = le32_to_cpu((*rxbd)->r.lstatus); 1575 size = buffer_size; 1576 1577 if (bd_status & ENETC_RXBD_LSTATUS_F) { 1578 dma_rmb(); 1579 size = le16_to_cpu((*rxbd)->r.buf_len); 1580 } 1581 1582 enetc_add_rx_buff_to_skb(rx_ring, *i, size, skb); 1583 1584 (*cleaned_cnt)++; 1585 1586 enetc_rxbd_next(rx_ring, rxbd, i); 1587 } 1588 1589 skb_record_rx_queue(skb, rx_ring->index); 1590 skb->protocol = eth_type_trans(skb, rx_ring->ndev); 1591 1592 return skb; 1593 } 1594 1595 #define ENETC_RXBD_BUNDLE 16 /* # of BDs to update at once */ 1596 1597 static int enetc_clean_rx_ring(struct enetc_bdr *rx_ring, 1598 struct napi_struct *napi, int work_limit) 1599 { 1600 int rx_frm_cnt = 0, rx_byte_cnt = 0; 1601 int cleaned_cnt, i; 1602 1603 cleaned_cnt = enetc_bd_unused(rx_ring); 1604 /* next descriptor to process */ 1605 i = rx_ring->next_to_clean; 1606 1607 enetc_lock_mdio(); 1608 1609 while (likely(rx_frm_cnt < work_limit)) { 1610 union enetc_rx_bd *rxbd; 1611 struct sk_buff *skb; 1612 u32 bd_status; 1613 1614 if (cleaned_cnt >= ENETC_RXBD_BUNDLE) 1615 cleaned_cnt -= enetc_refill_rx_ring(rx_ring, 1616 cleaned_cnt); 1617 1618 rxbd = enetc_rxbd(rx_ring, i); 1619 bd_status = le32_to_cpu(rxbd->r.lstatus); 1620 if (!bd_status) 1621 break; 1622 1623 enetc_wr_reg_hot(rx_ring->idr, BIT(rx_ring->index)); 1624 dma_rmb(); /* for reading other rxbd fields */ 1625 1626 if (enetc_check_bd_errors_and_consume(rx_ring, bd_status, 1627 &rxbd, &i)) 1628 break; 1629 1630 skb = enetc_build_skb(rx_ring, bd_status, &rxbd, &i, 1631 &cleaned_cnt, ENETC_RXB_DMA_SIZE); 1632 if (!skb) 1633 break; 1634 1635 /* When set, the outer VLAN header is extracted and reported 1636 * in the receive buffer descriptor. So rx_byte_cnt should 1637 * add the length of the extracted VLAN header. 1638 */ 1639 if (bd_status & ENETC_RXBD_FLAG_VLAN) 1640 rx_byte_cnt += VLAN_HLEN; 1641 rx_byte_cnt += skb->len + ETH_HLEN; 1642 rx_frm_cnt++; 1643 1644 enetc_unlock_mdio(); 1645 napi_gro_receive(napi, skb); 1646 enetc_lock_mdio(); 1647 } 1648 1649 rx_ring->next_to_clean = i; 1650 1651 rx_ring->stats.packets += rx_frm_cnt; 1652 rx_ring->stats.bytes += rx_byte_cnt; 1653 1654 enetc_unlock_mdio(); 1655 1656 return rx_frm_cnt; 1657 } 1658 1659 static void enetc_xdp_map_tx_buff(struct enetc_bdr *tx_ring, int i, 1660 struct enetc_tx_swbd *tx_swbd, 1661 int frm_len) 1662 { 1663 union enetc_tx_bd *txbd = ENETC_TXBD(*tx_ring, i); 1664 1665 prefetchw(txbd); 1666 1667 enetc_clear_tx_bd(txbd); 1668 txbd->addr = cpu_to_le64(tx_swbd->dma + tx_swbd->page_offset); 1669 txbd->buf_len = cpu_to_le16(tx_swbd->len); 1670 txbd->frm_len = cpu_to_le16(frm_len); 1671 1672 memcpy(&tx_ring->tx_swbd[i], tx_swbd, sizeof(*tx_swbd)); 1673 } 1674 1675 /* Puts in the TX ring one XDP frame, mapped as an array of TX software buffer 1676 * descriptors. 1677 */ 1678 static bool enetc_xdp_tx(struct enetc_bdr *tx_ring, 1679 struct enetc_tx_swbd *xdp_tx_arr, int num_tx_swbd) 1680 { 1681 struct enetc_tx_swbd *tmp_tx_swbd = xdp_tx_arr; 1682 int i, k, frm_len = tmp_tx_swbd->len; 1683 1684 if (unlikely(enetc_bd_unused(tx_ring) < ENETC_TXBDS_NEEDED(num_tx_swbd))) 1685 return false; 1686 1687 while (unlikely(!tmp_tx_swbd->is_eof)) { 1688 tmp_tx_swbd++; 1689 frm_len += tmp_tx_swbd->len; 1690 } 1691 1692 i = tx_ring->next_to_use; 1693 1694 for (k = 0; k < num_tx_swbd; k++) { 1695 struct enetc_tx_swbd *xdp_tx_swbd = &xdp_tx_arr[k]; 1696 1697 enetc_xdp_map_tx_buff(tx_ring, i, xdp_tx_swbd, frm_len); 1698 1699 /* last BD needs 'F' bit set */ 1700 if (xdp_tx_swbd->is_eof) { 1701 union enetc_tx_bd *txbd = ENETC_TXBD(*tx_ring, i); 1702 1703 txbd->flags = ENETC_TXBD_FLAGS_F; 1704 } 1705 1706 enetc_bdr_idx_inc(tx_ring, &i); 1707 } 1708 1709 tx_ring->next_to_use = i; 1710 1711 return true; 1712 } 1713 1714 static int enetc_xdp_frame_to_xdp_tx_swbd(struct enetc_bdr *tx_ring, 1715 struct enetc_tx_swbd *xdp_tx_arr, 1716 struct xdp_frame *xdp_frame) 1717 { 1718 struct enetc_tx_swbd *xdp_tx_swbd = &xdp_tx_arr[0]; 1719 struct skb_shared_info *shinfo; 1720 void *data = xdp_frame->data; 1721 int len = xdp_frame->len; 1722 skb_frag_t *frag; 1723 dma_addr_t dma; 1724 unsigned int f; 1725 int n = 0; 1726 1727 dma = dma_map_single(tx_ring->dev, data, len, DMA_TO_DEVICE); 1728 if (unlikely(dma_mapping_error(tx_ring->dev, dma))) { 1729 netdev_err(tx_ring->ndev, "DMA map error\n"); 1730 return -1; 1731 } 1732 1733 xdp_tx_swbd->dma = dma; 1734 xdp_tx_swbd->dir = DMA_TO_DEVICE; 1735 xdp_tx_swbd->len = len; 1736 xdp_tx_swbd->is_xdp_redirect = true; 1737 xdp_tx_swbd->is_eof = false; 1738 xdp_tx_swbd->xdp_frame = NULL; 1739 1740 n++; 1741 1742 if (!xdp_frame_has_frags(xdp_frame)) 1743 goto out; 1744 1745 xdp_tx_swbd = &xdp_tx_arr[n]; 1746 1747 shinfo = xdp_get_shared_info_from_frame(xdp_frame); 1748 1749 for (f = 0, frag = &shinfo->frags[0]; f < shinfo->nr_frags; 1750 f++, frag++) { 1751 data = skb_frag_address(frag); 1752 len = skb_frag_size(frag); 1753 1754 dma = dma_map_single(tx_ring->dev, data, len, DMA_TO_DEVICE); 1755 if (unlikely(dma_mapping_error(tx_ring->dev, dma))) { 1756 /* Undo the DMA mapping for all fragments */ 1757 while (--n >= 0) 1758 enetc_unmap_tx_buff(tx_ring, &xdp_tx_arr[n]); 1759 1760 netdev_err(tx_ring->ndev, "DMA map error\n"); 1761 return -1; 1762 } 1763 1764 xdp_tx_swbd->dma = dma; 1765 xdp_tx_swbd->dir = DMA_TO_DEVICE; 1766 xdp_tx_swbd->len = len; 1767 xdp_tx_swbd->is_xdp_redirect = true; 1768 xdp_tx_swbd->is_eof = false; 1769 xdp_tx_swbd->xdp_frame = NULL; 1770 1771 n++; 1772 xdp_tx_swbd = &xdp_tx_arr[n]; 1773 } 1774 out: 1775 xdp_tx_arr[n - 1].is_eof = true; 1776 xdp_tx_arr[n - 1].xdp_frame = xdp_frame; 1777 1778 return n; 1779 } 1780 1781 int enetc_xdp_xmit(struct net_device *ndev, int num_frames, 1782 struct xdp_frame **frames, u32 flags) 1783 { 1784 struct enetc_tx_swbd xdp_redirect_arr[ENETC_MAX_SKB_FRAGS] = {0}; 1785 struct enetc_ndev_priv *priv = netdev_priv(ndev); 1786 struct enetc_bdr *tx_ring; 1787 int xdp_tx_bd_cnt, i, k; 1788 int xdp_tx_frm_cnt = 0; 1789 1790 if (unlikely(test_bit(ENETC_TX_DOWN, &priv->flags) || 1791 !netif_carrier_ok(ndev))) 1792 return -ENETDOWN; 1793 1794 enetc_lock_mdio(); 1795 1796 tx_ring = priv->xdp_tx_ring[smp_processor_id()]; 1797 1798 prefetchw(ENETC_TXBD(*tx_ring, tx_ring->next_to_use)); 1799 1800 for (k = 0; k < num_frames; k++) { 1801 xdp_tx_bd_cnt = enetc_xdp_frame_to_xdp_tx_swbd(tx_ring, 1802 xdp_redirect_arr, 1803 frames[k]); 1804 if (unlikely(xdp_tx_bd_cnt < 0)) 1805 break; 1806 1807 if (unlikely(!enetc_xdp_tx(tx_ring, xdp_redirect_arr, 1808 xdp_tx_bd_cnt))) { 1809 for (i = 0; i < xdp_tx_bd_cnt; i++) 1810 enetc_unmap_tx_buff(tx_ring, 1811 &xdp_redirect_arr[i]); 1812 tx_ring->stats.xdp_tx_drops++; 1813 break; 1814 } 1815 1816 xdp_tx_frm_cnt++; 1817 } 1818 1819 if (unlikely((flags & XDP_XMIT_FLUSH) || k != xdp_tx_frm_cnt)) 1820 enetc_update_tx_ring_tail(tx_ring); 1821 1822 tx_ring->stats.xdp_tx += xdp_tx_frm_cnt; 1823 1824 enetc_unlock_mdio(); 1825 1826 return xdp_tx_frm_cnt; 1827 } 1828 EXPORT_SYMBOL_GPL(enetc_xdp_xmit); 1829 1830 static void enetc_map_rx_buff_to_xdp(struct enetc_bdr *rx_ring, int i, 1831 struct xdp_buff *xdp_buff, u16 size) 1832 { 1833 struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size); 1834 void *hard_start = page_address(rx_swbd->page) + rx_swbd->page_offset; 1835 1836 /* To be used for XDP_TX */ 1837 rx_swbd->len = size; 1838 1839 xdp_prepare_buff(xdp_buff, hard_start - rx_ring->buffer_offset, 1840 rx_ring->buffer_offset, size, false); 1841 } 1842 1843 static void enetc_add_rx_buff_to_xdp(struct enetc_bdr *rx_ring, int i, 1844 u16 size, struct xdp_buff *xdp_buff) 1845 { 1846 struct skb_shared_info *shinfo = xdp_get_shared_info_from_buff(xdp_buff); 1847 struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size); 1848 skb_frag_t *frag; 1849 1850 /* To be used for XDP_TX */ 1851 rx_swbd->len = size; 1852 1853 if (!xdp_buff_has_frags(xdp_buff)) { 1854 xdp_buff_set_frags_flag(xdp_buff); 1855 shinfo->xdp_frags_size = size; 1856 shinfo->nr_frags = 0; 1857 } else { 1858 shinfo->xdp_frags_size += size; 1859 } 1860 1861 if (page_is_pfmemalloc(rx_swbd->page)) 1862 xdp_buff_set_frag_pfmemalloc(xdp_buff); 1863 1864 frag = &shinfo->frags[shinfo->nr_frags]; 1865 skb_frag_fill_page_desc(frag, rx_swbd->page, rx_swbd->page_offset, 1866 size); 1867 1868 shinfo->nr_frags++; 1869 } 1870 1871 static void enetc_build_xdp_buff(struct enetc_bdr *rx_ring, u32 bd_status, 1872 union enetc_rx_bd **rxbd, int *i, 1873 int *cleaned_cnt, struct xdp_buff *xdp_buff) 1874 { 1875 u16 size = le16_to_cpu((*rxbd)->r.buf_len); 1876 1877 xdp_init_buff(xdp_buff, ENETC_RXB_TRUESIZE, &rx_ring->xdp.rxq); 1878 1879 enetc_map_rx_buff_to_xdp(rx_ring, *i, xdp_buff, size); 1880 (*cleaned_cnt)++; 1881 enetc_rxbd_next(rx_ring, rxbd, i); 1882 1883 /* not last BD in frame? */ 1884 while (!(bd_status & ENETC_RXBD_LSTATUS_F)) { 1885 bd_status = le32_to_cpu((*rxbd)->r.lstatus); 1886 size = ENETC_RXB_DMA_SIZE_XDP; 1887 1888 if (bd_status & ENETC_RXBD_LSTATUS_F) { 1889 dma_rmb(); 1890 size = le16_to_cpu((*rxbd)->r.buf_len); 1891 } 1892 1893 enetc_add_rx_buff_to_xdp(rx_ring, *i, size, xdp_buff); 1894 (*cleaned_cnt)++; 1895 enetc_rxbd_next(rx_ring, rxbd, i); 1896 } 1897 } 1898 1899 /* Convert RX buffer descriptors to TX buffer descriptors. These will be 1900 * recycled back into the RX ring in enetc_clean_tx_ring. 1901 */ 1902 static int enetc_rx_swbd_to_xdp_tx_swbd(struct enetc_tx_swbd *xdp_tx_arr, 1903 struct enetc_bdr *rx_ring, 1904 int rx_ring_first, int rx_ring_last) 1905 { 1906 int n = 0; 1907 1908 for (; rx_ring_first != rx_ring_last; 1909 n++, enetc_bdr_idx_inc(rx_ring, &rx_ring_first)) { 1910 struct enetc_rx_swbd *rx_swbd = &rx_ring->rx_swbd[rx_ring_first]; 1911 struct enetc_tx_swbd *tx_swbd = &xdp_tx_arr[n]; 1912 1913 /* No need to dma_map, we already have DMA_BIDIRECTIONAL */ 1914 tx_swbd->dma = rx_swbd->dma; 1915 tx_swbd->dir = rx_swbd->dir; 1916 tx_swbd->page = rx_swbd->page; 1917 tx_swbd->page_offset = rx_swbd->page_offset; 1918 tx_swbd->len = rx_swbd->len; 1919 tx_swbd->is_dma_page = true; 1920 tx_swbd->is_xdp_tx = true; 1921 tx_swbd->is_eof = false; 1922 } 1923 1924 /* We rely on caller providing an rx_ring_last > rx_ring_first */ 1925 xdp_tx_arr[n - 1].is_eof = true; 1926 1927 return n; 1928 } 1929 1930 static void enetc_xdp_drop(struct enetc_bdr *rx_ring, int rx_ring_first, 1931 int rx_ring_last) 1932 { 1933 while (rx_ring_first != rx_ring_last) { 1934 enetc_put_rx_buff(rx_ring, 1935 &rx_ring->rx_swbd[rx_ring_first]); 1936 enetc_bdr_idx_inc(rx_ring, &rx_ring_first); 1937 } 1938 } 1939 1940 static void enetc_bulk_flip_buff(struct enetc_bdr *rx_ring, int rx_ring_first, 1941 int rx_ring_last) 1942 { 1943 while (rx_ring_first != rx_ring_last) { 1944 enetc_flip_rx_buff(rx_ring, 1945 &rx_ring->rx_swbd[rx_ring_first]); 1946 enetc_bdr_idx_inc(rx_ring, &rx_ring_first); 1947 } 1948 } 1949 1950 static int enetc_clean_rx_ring_xdp(struct enetc_bdr *rx_ring, 1951 struct napi_struct *napi, int work_limit, 1952 struct bpf_prog *prog) 1953 { 1954 int xdp_tx_bd_cnt, xdp_tx_frm_cnt = 0, xdp_redirect_frm_cnt = 0; 1955 struct enetc_tx_swbd xdp_tx_arr[ENETC_MAX_SKB_FRAGS] = {0}; 1956 struct enetc_ndev_priv *priv = netdev_priv(rx_ring->ndev); 1957 int rx_frm_cnt = 0, rx_byte_cnt = 0; 1958 struct enetc_bdr *tx_ring; 1959 int cleaned_cnt, i; 1960 u32 xdp_act; 1961 1962 cleaned_cnt = enetc_bd_unused(rx_ring); 1963 /* next descriptor to process */ 1964 i = rx_ring->next_to_clean; 1965 1966 enetc_lock_mdio(); 1967 1968 while (likely(rx_frm_cnt < work_limit)) { 1969 union enetc_rx_bd *rxbd, *orig_rxbd; 1970 struct xdp_buff xdp_buff; 1971 struct sk_buff *skb; 1972 int orig_i, err; 1973 u32 bd_status; 1974 1975 rxbd = enetc_rxbd(rx_ring, i); 1976 bd_status = le32_to_cpu(rxbd->r.lstatus); 1977 if (!bd_status) 1978 break; 1979 1980 enetc_wr_reg_hot(rx_ring->idr, BIT(rx_ring->index)); 1981 dma_rmb(); /* for reading other rxbd fields */ 1982 1983 if (enetc_check_bd_errors_and_consume(rx_ring, bd_status, 1984 &rxbd, &i)) 1985 break; 1986 1987 orig_rxbd = rxbd; 1988 orig_i = i; 1989 1990 enetc_build_xdp_buff(rx_ring, bd_status, &rxbd, &i, 1991 &cleaned_cnt, &xdp_buff); 1992 1993 /* When set, the outer VLAN header is extracted and reported 1994 * in the receive buffer descriptor. So rx_byte_cnt should 1995 * add the length of the extracted VLAN header. 1996 */ 1997 if (bd_status & ENETC_RXBD_FLAG_VLAN) 1998 rx_byte_cnt += VLAN_HLEN; 1999 rx_byte_cnt += xdp_get_buff_len(&xdp_buff); 2000 2001 xdp_act = bpf_prog_run_xdp(prog, &xdp_buff); 2002 2003 switch (xdp_act) { 2004 default: 2005 bpf_warn_invalid_xdp_action(rx_ring->ndev, prog, xdp_act); 2006 fallthrough; 2007 case XDP_ABORTED: 2008 trace_xdp_exception(rx_ring->ndev, prog, xdp_act); 2009 fallthrough; 2010 case XDP_DROP: 2011 enetc_xdp_drop(rx_ring, orig_i, i); 2012 rx_ring->stats.xdp_drops++; 2013 break; 2014 case XDP_PASS: 2015 skb = xdp_build_skb_from_buff(&xdp_buff); 2016 /* Probably under memory pressure, stop NAPI */ 2017 if (unlikely(!skb)) { 2018 enetc_xdp_drop(rx_ring, orig_i, i); 2019 rx_ring->stats.xdp_drops++; 2020 goto out; 2021 } 2022 2023 enetc_get_offloads(rx_ring, orig_rxbd, skb); 2024 2025 /* These buffers are about to be owned by the stack. 2026 * Update our buffer cache (the rx_swbd array elements) 2027 * with their other page halves. 2028 */ 2029 enetc_bulk_flip_buff(rx_ring, orig_i, i); 2030 2031 enetc_unlock_mdio(); 2032 napi_gro_receive(napi, skb); 2033 enetc_lock_mdio(); 2034 break; 2035 case XDP_TX: 2036 tx_ring = priv->xdp_tx_ring[rx_ring->index]; 2037 if (unlikely(test_bit(ENETC_TX_DOWN, &priv->flags))) { 2038 enetc_xdp_drop(rx_ring, orig_i, i); 2039 tx_ring->stats.xdp_tx_drops++; 2040 break; 2041 } 2042 2043 xdp_tx_bd_cnt = enetc_rx_swbd_to_xdp_tx_swbd(xdp_tx_arr, 2044 rx_ring, 2045 orig_i, i); 2046 2047 if (!enetc_xdp_tx(tx_ring, xdp_tx_arr, xdp_tx_bd_cnt)) { 2048 enetc_xdp_drop(rx_ring, orig_i, i); 2049 tx_ring->stats.xdp_tx_drops++; 2050 } else { 2051 tx_ring->stats.xdp_tx++; 2052 rx_ring->xdp.xdp_tx_in_flight += xdp_tx_bd_cnt; 2053 xdp_tx_frm_cnt++; 2054 /* The XDP_TX enqueue was successful, so we 2055 * need to scrub the RX software BDs because 2056 * the ownership of the buffers no longer 2057 * belongs to the RX ring, and we must prevent 2058 * enetc_refill_rx_ring() from reusing 2059 * rx_swbd->page. 2060 */ 2061 while (orig_i != i) { 2062 rx_ring->rx_swbd[orig_i].page = NULL; 2063 enetc_bdr_idx_inc(rx_ring, &orig_i); 2064 } 2065 } 2066 break; 2067 case XDP_REDIRECT: 2068 enetc_unlock_mdio(); 2069 err = xdp_do_redirect(rx_ring->ndev, &xdp_buff, prog); 2070 enetc_lock_mdio(); 2071 if (unlikely(err)) { 2072 enetc_xdp_drop(rx_ring, orig_i, i); 2073 rx_ring->stats.xdp_redirect_failures++; 2074 } else { 2075 enetc_bulk_flip_buff(rx_ring, orig_i, i); 2076 xdp_redirect_frm_cnt++; 2077 rx_ring->stats.xdp_redirect++; 2078 } 2079 } 2080 2081 rx_frm_cnt++; 2082 } 2083 2084 out: 2085 rx_ring->next_to_clean = i; 2086 2087 rx_ring->stats.packets += rx_frm_cnt; 2088 rx_ring->stats.bytes += rx_byte_cnt; 2089 2090 if (xdp_redirect_frm_cnt) { 2091 enetc_unlock_mdio(); 2092 xdp_do_flush(); 2093 enetc_lock_mdio(); 2094 } 2095 2096 if (xdp_tx_frm_cnt) 2097 enetc_update_tx_ring_tail(tx_ring); 2098 2099 if (cleaned_cnt > rx_ring->xdp.xdp_tx_in_flight) 2100 enetc_refill_rx_ring(rx_ring, enetc_bd_unused(rx_ring) - 2101 rx_ring->xdp.xdp_tx_in_flight); 2102 2103 enetc_unlock_mdio(); 2104 2105 return rx_frm_cnt; 2106 } 2107 2108 static int enetc_poll(struct napi_struct *napi, int budget) 2109 { 2110 struct enetc_int_vector 2111 *v = container_of(napi, struct enetc_int_vector, napi); 2112 struct enetc_bdr *rx_ring = &v->rx_ring; 2113 struct bpf_prog *prog; 2114 bool complete = true; 2115 int work_done; 2116 int i; 2117 2118 enetc_lock_mdio(); 2119 2120 for (i = 0; i < v->count_tx_rings; i++) 2121 if (!enetc_clean_tx_ring(&v->tx_ring[i], budget)) 2122 complete = false; 2123 enetc_unlock_mdio(); 2124 2125 prog = rx_ring->xdp.prog; 2126 if (prog) 2127 work_done = enetc_clean_rx_ring_xdp(rx_ring, napi, budget, prog); 2128 else 2129 work_done = enetc_clean_rx_ring(rx_ring, napi, budget); 2130 if (work_done == budget) 2131 complete = false; 2132 if (work_done) 2133 v->rx_napi_work = true; 2134 2135 if (!complete) 2136 return budget; 2137 2138 napi_complete_done(napi, work_done); 2139 2140 if (likely(v->rx_dim_en)) 2141 enetc_rx_net_dim(v); 2142 2143 v->rx_napi_work = false; 2144 2145 enetc_lock_mdio(); 2146 /* enable interrupts */ 2147 enetc_wr_reg_hot(v->rbier, ENETC_RBIER_RXTIE); 2148 2149 for_each_set_bit(i, &v->tx_rings_map, ENETC_MAX_NUM_TXQS) 2150 enetc_wr_reg_hot(v->tbier_base + ENETC_BDR_OFF(i), 2151 ENETC_TBIER_TXTIE); 2152 2153 enetc_unlock_mdio(); 2154 2155 return work_done; 2156 } 2157 2158 /* Probing and Init */ 2159 #define ENETC_MAX_RFS_SIZE 64 2160 void enetc_get_si_caps(struct enetc_si *si) 2161 { 2162 struct enetc_hw *hw = &si->hw; 2163 u32 val; 2164 2165 /* find out how many of various resources we have to work with */ 2166 val = enetc_rd(hw, ENETC_SICAPR0); 2167 si->num_rx_rings = (val >> 16) & 0xff; 2168 si->num_tx_rings = val & 0xff; 2169 2170 val = enetc_rd(hw, ENETC_SIPCAPR0); 2171 if (val & ENETC_SIPCAPR0_RFS) { 2172 val = enetc_rd(hw, ENETC_SIRFSCAPR); 2173 si->num_fs_entries = ENETC_SIRFSCAPR_GET_NUM_RFS(val); 2174 si->num_fs_entries = min(si->num_fs_entries, ENETC_MAX_RFS_SIZE); 2175 } else { 2176 /* ENETC which not supports RFS */ 2177 si->num_fs_entries = 0; 2178 } 2179 2180 si->num_rss = 0; 2181 val = enetc_rd(hw, ENETC_SIPCAPR0); 2182 if (val & ENETC_SIPCAPR0_RSS) { 2183 u32 rss; 2184 2185 rss = enetc_rd(hw, ENETC_SIRSSCAPR); 2186 si->num_rss = ENETC_SIRSSCAPR_GET_NUM_RSS(rss); 2187 } 2188 2189 if (val & ENETC_SIPCAPR0_LSO) 2190 si->hw_features |= ENETC_SI_F_LSO; 2191 } 2192 EXPORT_SYMBOL_GPL(enetc_get_si_caps); 2193 2194 static int enetc_dma_alloc_bdr(struct enetc_bdr_resource *res) 2195 { 2196 size_t bd_base_size = res->bd_count * res->bd_size; 2197 2198 res->bd_base = dma_alloc_coherent(res->dev, bd_base_size, 2199 &res->bd_dma_base, GFP_KERNEL); 2200 if (!res->bd_base) 2201 return -ENOMEM; 2202 2203 /* h/w requires 128B alignment */ 2204 if (!IS_ALIGNED(res->bd_dma_base, 128)) { 2205 dma_free_coherent(res->dev, bd_base_size, res->bd_base, 2206 res->bd_dma_base); 2207 return -EINVAL; 2208 } 2209 2210 return 0; 2211 } 2212 2213 static void enetc_dma_free_bdr(const struct enetc_bdr_resource *res) 2214 { 2215 size_t bd_base_size = res->bd_count * res->bd_size; 2216 2217 dma_free_coherent(res->dev, bd_base_size, res->bd_base, 2218 res->bd_dma_base); 2219 } 2220 2221 static int enetc_alloc_tx_resource(struct enetc_bdr_resource *res, 2222 struct device *dev, size_t bd_count) 2223 { 2224 int err; 2225 2226 res->dev = dev; 2227 res->bd_count = bd_count; 2228 res->bd_size = sizeof(union enetc_tx_bd); 2229 2230 res->tx_swbd = vcalloc(bd_count, sizeof(*res->tx_swbd)); 2231 if (!res->tx_swbd) 2232 return -ENOMEM; 2233 2234 err = enetc_dma_alloc_bdr(res); 2235 if (err) 2236 goto err_alloc_bdr; 2237 2238 res->tso_headers = dma_alloc_coherent(dev, bd_count * TSO_HEADER_SIZE, 2239 &res->tso_headers_dma, 2240 GFP_KERNEL); 2241 if (!res->tso_headers) { 2242 err = -ENOMEM; 2243 goto err_alloc_tso; 2244 } 2245 2246 return 0; 2247 2248 err_alloc_tso: 2249 enetc_dma_free_bdr(res); 2250 err_alloc_bdr: 2251 vfree(res->tx_swbd); 2252 res->tx_swbd = NULL; 2253 2254 return err; 2255 } 2256 2257 static void enetc_free_tx_resource(const struct enetc_bdr_resource *res) 2258 { 2259 dma_free_coherent(res->dev, res->bd_count * TSO_HEADER_SIZE, 2260 res->tso_headers, res->tso_headers_dma); 2261 enetc_dma_free_bdr(res); 2262 vfree(res->tx_swbd); 2263 } 2264 2265 static struct enetc_bdr_resource * 2266 enetc_alloc_tx_resources(struct enetc_ndev_priv *priv) 2267 { 2268 struct enetc_bdr_resource *tx_res; 2269 int i, err; 2270 2271 tx_res = kcalloc(priv->num_tx_rings, sizeof(*tx_res), GFP_KERNEL); 2272 if (!tx_res) 2273 return ERR_PTR(-ENOMEM); 2274 2275 for (i = 0; i < priv->num_tx_rings; i++) { 2276 struct enetc_bdr *tx_ring = priv->tx_ring[i]; 2277 2278 err = enetc_alloc_tx_resource(&tx_res[i], tx_ring->dev, 2279 tx_ring->bd_count); 2280 if (err) 2281 goto fail; 2282 } 2283 2284 return tx_res; 2285 2286 fail: 2287 while (i-- > 0) 2288 enetc_free_tx_resource(&tx_res[i]); 2289 2290 kfree(tx_res); 2291 2292 return ERR_PTR(err); 2293 } 2294 2295 static void enetc_free_tx_resources(const struct enetc_bdr_resource *tx_res, 2296 size_t num_resources) 2297 { 2298 size_t i; 2299 2300 for (i = 0; i < num_resources; i++) 2301 enetc_free_tx_resource(&tx_res[i]); 2302 2303 kfree(tx_res); 2304 } 2305 2306 static int enetc_alloc_rx_resource(struct enetc_bdr_resource *res, 2307 struct device *dev, size_t bd_count, 2308 bool extended) 2309 { 2310 int err; 2311 2312 res->dev = dev; 2313 res->bd_count = bd_count; 2314 res->bd_size = sizeof(union enetc_rx_bd); 2315 if (extended) 2316 res->bd_size *= 2; 2317 2318 res->rx_swbd = vcalloc(bd_count, sizeof(struct enetc_rx_swbd)); 2319 if (!res->rx_swbd) 2320 return -ENOMEM; 2321 2322 err = enetc_dma_alloc_bdr(res); 2323 if (err) { 2324 vfree(res->rx_swbd); 2325 return err; 2326 } 2327 2328 return 0; 2329 } 2330 2331 static void enetc_free_rx_resource(const struct enetc_bdr_resource *res) 2332 { 2333 enetc_dma_free_bdr(res); 2334 vfree(res->rx_swbd); 2335 } 2336 2337 static struct enetc_bdr_resource * 2338 enetc_alloc_rx_resources(struct enetc_ndev_priv *priv, bool extended) 2339 { 2340 struct enetc_bdr_resource *rx_res; 2341 int i, err; 2342 2343 rx_res = kcalloc(priv->num_rx_rings, sizeof(*rx_res), GFP_KERNEL); 2344 if (!rx_res) 2345 return ERR_PTR(-ENOMEM); 2346 2347 for (i = 0; i < priv->num_rx_rings; i++) { 2348 struct enetc_bdr *rx_ring = priv->rx_ring[i]; 2349 2350 err = enetc_alloc_rx_resource(&rx_res[i], rx_ring->dev, 2351 rx_ring->bd_count, extended); 2352 if (err) 2353 goto fail; 2354 } 2355 2356 return rx_res; 2357 2358 fail: 2359 while (i-- > 0) 2360 enetc_free_rx_resource(&rx_res[i]); 2361 2362 kfree(rx_res); 2363 2364 return ERR_PTR(err); 2365 } 2366 2367 static void enetc_free_rx_resources(const struct enetc_bdr_resource *rx_res, 2368 size_t num_resources) 2369 { 2370 size_t i; 2371 2372 for (i = 0; i < num_resources; i++) 2373 enetc_free_rx_resource(&rx_res[i]); 2374 2375 kfree(rx_res); 2376 } 2377 2378 static void enetc_assign_tx_resource(struct enetc_bdr *tx_ring, 2379 const struct enetc_bdr_resource *res) 2380 { 2381 tx_ring->bd_base = res ? res->bd_base : NULL; 2382 tx_ring->bd_dma_base = res ? res->bd_dma_base : 0; 2383 tx_ring->tx_swbd = res ? res->tx_swbd : NULL; 2384 tx_ring->tso_headers = res ? res->tso_headers : NULL; 2385 tx_ring->tso_headers_dma = res ? res->tso_headers_dma : 0; 2386 } 2387 2388 static void enetc_assign_rx_resource(struct enetc_bdr *rx_ring, 2389 const struct enetc_bdr_resource *res) 2390 { 2391 rx_ring->bd_base = res ? res->bd_base : NULL; 2392 rx_ring->bd_dma_base = res ? res->bd_dma_base : 0; 2393 rx_ring->rx_swbd = res ? res->rx_swbd : NULL; 2394 } 2395 2396 static void enetc_assign_tx_resources(struct enetc_ndev_priv *priv, 2397 const struct enetc_bdr_resource *res) 2398 { 2399 int i; 2400 2401 if (priv->tx_res) 2402 enetc_free_tx_resources(priv->tx_res, priv->num_tx_rings); 2403 2404 for (i = 0; i < priv->num_tx_rings; i++) { 2405 enetc_assign_tx_resource(priv->tx_ring[i], 2406 res ? &res[i] : NULL); 2407 } 2408 2409 priv->tx_res = res; 2410 } 2411 2412 static void enetc_assign_rx_resources(struct enetc_ndev_priv *priv, 2413 const struct enetc_bdr_resource *res) 2414 { 2415 int i; 2416 2417 if (priv->rx_res) 2418 enetc_free_rx_resources(priv->rx_res, priv->num_rx_rings); 2419 2420 for (i = 0; i < priv->num_rx_rings; i++) { 2421 enetc_assign_rx_resource(priv->rx_ring[i], 2422 res ? &res[i] : NULL); 2423 } 2424 2425 priv->rx_res = res; 2426 } 2427 2428 static void enetc_free_tx_ring(struct enetc_bdr *tx_ring) 2429 { 2430 int i; 2431 2432 for (i = 0; i < tx_ring->bd_count; i++) { 2433 struct enetc_tx_swbd *tx_swbd = &tx_ring->tx_swbd[i]; 2434 2435 enetc_free_tx_frame(tx_ring, tx_swbd); 2436 } 2437 } 2438 2439 static void enetc_free_rx_ring(struct enetc_bdr *rx_ring) 2440 { 2441 int i; 2442 2443 for (i = 0; i < rx_ring->bd_count; i++) { 2444 struct enetc_rx_swbd *rx_swbd = &rx_ring->rx_swbd[i]; 2445 2446 if (!rx_swbd->page) 2447 continue; 2448 2449 dma_unmap_page(rx_ring->dev, rx_swbd->dma, PAGE_SIZE, 2450 rx_swbd->dir); 2451 __free_page(rx_swbd->page); 2452 rx_swbd->page = NULL; 2453 } 2454 } 2455 2456 static void enetc_free_rxtx_rings(struct enetc_ndev_priv *priv) 2457 { 2458 int i; 2459 2460 for (i = 0; i < priv->num_rx_rings; i++) 2461 enetc_free_rx_ring(priv->rx_ring[i]); 2462 2463 for (i = 0; i < priv->num_tx_rings; i++) 2464 enetc_free_tx_ring(priv->tx_ring[i]); 2465 } 2466 2467 static int enetc_setup_default_rss_table(struct enetc_si *si, int num_groups) 2468 { 2469 int *rss_table; 2470 int i; 2471 2472 rss_table = kmalloc_array(si->num_rss, sizeof(*rss_table), GFP_KERNEL); 2473 if (!rss_table) 2474 return -ENOMEM; 2475 2476 /* Set up RSS table defaults */ 2477 for (i = 0; i < si->num_rss; i++) 2478 rss_table[i] = i % num_groups; 2479 2480 si->ops->set_rss_table(si, rss_table, si->num_rss); 2481 2482 kfree(rss_table); 2483 2484 return 0; 2485 } 2486 2487 static void enetc_set_lso_flags_mask(struct enetc_hw *hw) 2488 { 2489 enetc_wr(hw, ENETC4_SILSOSFMR0, 2490 SILSOSFMR0_VAL_SET(ENETC4_TCP_NL_SEG_FLAGS_DMASK, 2491 ENETC4_TCP_NL_SEG_FLAGS_DMASK)); 2492 enetc_wr(hw, ENETC4_SILSOSFMR1, 0); 2493 } 2494 2495 static void enetc_set_rss(struct net_device *ndev, int en) 2496 { 2497 struct enetc_ndev_priv *priv = netdev_priv(ndev); 2498 struct enetc_hw *hw = &priv->si->hw; 2499 u32 reg; 2500 2501 enetc_wr(hw, ENETC_SIRBGCR, priv->num_rx_rings); 2502 2503 reg = enetc_rd(hw, ENETC_SIMR); 2504 reg &= ~ENETC_SIMR_RSSE; 2505 reg |= (en) ? ENETC_SIMR_RSSE : 0; 2506 enetc_wr(hw, ENETC_SIMR, reg); 2507 } 2508 2509 int enetc_configure_si(struct enetc_ndev_priv *priv) 2510 { 2511 struct enetc_si *si = priv->si; 2512 struct enetc_hw *hw = &si->hw; 2513 int err; 2514 2515 if (is_enetc_rev1(si)) { 2516 /* set SI cache attributes */ 2517 enetc_wr(hw, ENETC_SICAR0, 2518 ENETC_SICAR_RD_COHERENT | ENETC_SICAR_WR_COHERENT); 2519 enetc_wr(hw, ENETC_SICAR1, ENETC_SICAR_MSI); 2520 } 2521 2522 /* enable SI */ 2523 enetc_wr(hw, ENETC_SIMR, ENETC_SIMR_EN); 2524 2525 if (si->hw_features & ENETC_SI_F_LSO) 2526 enetc_set_lso_flags_mask(hw); 2527 2528 if (si->num_rss) { 2529 err = enetc_setup_default_rss_table(si, priv->num_rx_rings); 2530 if (err) 2531 return err; 2532 2533 if (priv->ndev->features & NETIF_F_RXHASH) 2534 enetc_set_rss(priv->ndev, true); 2535 } 2536 2537 return 0; 2538 } 2539 EXPORT_SYMBOL_GPL(enetc_configure_si); 2540 2541 void enetc_init_si_rings_params(struct enetc_ndev_priv *priv) 2542 { 2543 struct enetc_si *si = priv->si; 2544 int cpus = num_online_cpus(); 2545 2546 priv->tx_bd_count = ENETC_TX_RING_DEFAULT_SIZE; 2547 priv->rx_bd_count = ENETC_RX_RING_DEFAULT_SIZE; 2548 2549 /* Enable all available TX rings in order to configure as many 2550 * priorities as possible, when needed. 2551 * TODO: Make # of TX rings run-time configurable 2552 */ 2553 priv->num_rx_rings = min_t(int, cpus, si->num_rx_rings); 2554 priv->num_tx_rings = si->num_tx_rings; 2555 priv->bdr_int_num = priv->num_rx_rings; 2556 priv->ic_mode = ENETC_IC_RX_ADAPTIVE | ENETC_IC_TX_MANUAL; 2557 priv->tx_ictt = enetc_usecs_to_cycles(600, priv->sysclk_freq); 2558 } 2559 EXPORT_SYMBOL_GPL(enetc_init_si_rings_params); 2560 2561 int enetc_alloc_si_resources(struct enetc_ndev_priv *priv) 2562 { 2563 struct enetc_si *si = priv->si; 2564 2565 priv->cls_rules = kcalloc(si->num_fs_entries, sizeof(*priv->cls_rules), 2566 GFP_KERNEL); 2567 if (!priv->cls_rules) 2568 return -ENOMEM; 2569 2570 return 0; 2571 } 2572 EXPORT_SYMBOL_GPL(enetc_alloc_si_resources); 2573 2574 void enetc_free_si_resources(struct enetc_ndev_priv *priv) 2575 { 2576 kfree(priv->cls_rules); 2577 } 2578 EXPORT_SYMBOL_GPL(enetc_free_si_resources); 2579 2580 static void enetc_setup_txbdr(struct enetc_hw *hw, struct enetc_bdr *tx_ring) 2581 { 2582 int idx = tx_ring->index; 2583 u32 tbmr; 2584 2585 enetc_txbdr_wr(hw, idx, ENETC_TBBAR0, 2586 lower_32_bits(tx_ring->bd_dma_base)); 2587 2588 enetc_txbdr_wr(hw, idx, ENETC_TBBAR1, 2589 upper_32_bits(tx_ring->bd_dma_base)); 2590 2591 WARN_ON(!IS_ALIGNED(tx_ring->bd_count, 64)); /* multiple of 64 */ 2592 enetc_txbdr_wr(hw, idx, ENETC_TBLENR, 2593 ENETC_RTBLENR_LEN(tx_ring->bd_count)); 2594 2595 /* clearing PI/CI registers for Tx not supported, adjust sw indexes */ 2596 tx_ring->next_to_use = enetc_txbdr_rd(hw, idx, ENETC_TBPIR); 2597 tx_ring->next_to_clean = enetc_txbdr_rd(hw, idx, ENETC_TBCIR); 2598 2599 /* enable Tx ints by setting pkt thr to 1 */ 2600 enetc_txbdr_wr(hw, idx, ENETC_TBICR0, ENETC_TBICR0_ICEN | 0x1); 2601 2602 tbmr = ENETC_TBMR_SET_PRIO(tx_ring->prio); 2603 if (tx_ring->ndev->features & NETIF_F_HW_VLAN_CTAG_TX) 2604 tbmr |= ENETC_TBMR_VIH; 2605 2606 /* enable ring */ 2607 enetc_txbdr_wr(hw, idx, ENETC_TBMR, tbmr); 2608 2609 tx_ring->tpir = hw->reg + ENETC_BDR(TX, idx, ENETC_TBPIR); 2610 tx_ring->tcir = hw->reg + ENETC_BDR(TX, idx, ENETC_TBCIR); 2611 tx_ring->idr = hw->reg + ENETC_SITXIDR; 2612 } 2613 2614 static void enetc_setup_rxbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring, 2615 bool extended) 2616 { 2617 int idx = rx_ring->index; 2618 u32 rbmr = 0; 2619 2620 enetc_rxbdr_wr(hw, idx, ENETC_RBBAR0, 2621 lower_32_bits(rx_ring->bd_dma_base)); 2622 2623 enetc_rxbdr_wr(hw, idx, ENETC_RBBAR1, 2624 upper_32_bits(rx_ring->bd_dma_base)); 2625 2626 WARN_ON(!IS_ALIGNED(rx_ring->bd_count, 64)); /* multiple of 64 */ 2627 enetc_rxbdr_wr(hw, idx, ENETC_RBLENR, 2628 ENETC_RTBLENR_LEN(rx_ring->bd_count)); 2629 2630 if (rx_ring->xdp.prog) 2631 enetc_rxbdr_wr(hw, idx, ENETC_RBBSR, ENETC_RXB_DMA_SIZE_XDP); 2632 else 2633 enetc_rxbdr_wr(hw, idx, ENETC_RBBSR, ENETC_RXB_DMA_SIZE); 2634 2635 /* Also prepare the consumer index in case page allocation never 2636 * succeeds. In that case, hardware will never advance producer index 2637 * to match consumer index, and will drop all frames. 2638 */ 2639 enetc_rxbdr_wr(hw, idx, ENETC_RBPIR, 0); 2640 enetc_rxbdr_wr(hw, idx, ENETC_RBCIR, 1); 2641 2642 /* enable Rx ints by setting pkt thr to 1 */ 2643 enetc_rxbdr_wr(hw, idx, ENETC_RBICR0, ENETC_RBICR0_ICEN | 0x1); 2644 2645 rx_ring->ext_en = extended; 2646 if (rx_ring->ext_en) 2647 rbmr |= ENETC_RBMR_BDS; 2648 2649 if (rx_ring->ndev->features & NETIF_F_HW_VLAN_CTAG_RX) 2650 rbmr |= ENETC_RBMR_VTE; 2651 2652 rx_ring->rcir = hw->reg + ENETC_BDR(RX, idx, ENETC_RBCIR); 2653 rx_ring->idr = hw->reg + ENETC_SIRXIDR; 2654 2655 rx_ring->next_to_clean = 0; 2656 rx_ring->next_to_use = 0; 2657 rx_ring->next_to_alloc = 0; 2658 2659 enetc_lock_mdio(); 2660 enetc_refill_rx_ring(rx_ring, enetc_bd_unused(rx_ring)); 2661 enetc_unlock_mdio(); 2662 2663 enetc_rxbdr_wr(hw, idx, ENETC_RBMR, rbmr); 2664 } 2665 2666 static void enetc_setup_bdrs(struct enetc_ndev_priv *priv, bool extended) 2667 { 2668 struct enetc_hw *hw = &priv->si->hw; 2669 int i; 2670 2671 for (i = 0; i < priv->num_tx_rings; i++) 2672 enetc_setup_txbdr(hw, priv->tx_ring[i]); 2673 2674 for (i = 0; i < priv->num_rx_rings; i++) 2675 enetc_setup_rxbdr(hw, priv->rx_ring[i], extended); 2676 } 2677 2678 static void enetc_enable_txbdr(struct enetc_hw *hw, struct enetc_bdr *tx_ring) 2679 { 2680 int idx = tx_ring->index; 2681 u32 tbmr; 2682 2683 tbmr = enetc_txbdr_rd(hw, idx, ENETC_TBMR); 2684 tbmr |= ENETC_TBMR_EN; 2685 enetc_txbdr_wr(hw, idx, ENETC_TBMR, tbmr); 2686 } 2687 2688 static void enetc_enable_rxbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring) 2689 { 2690 int idx = rx_ring->index; 2691 u32 rbmr; 2692 2693 rbmr = enetc_rxbdr_rd(hw, idx, ENETC_RBMR); 2694 rbmr |= ENETC_RBMR_EN; 2695 enetc_rxbdr_wr(hw, idx, ENETC_RBMR, rbmr); 2696 } 2697 2698 static void enetc_enable_rx_bdrs(struct enetc_ndev_priv *priv) 2699 { 2700 struct enetc_hw *hw = &priv->si->hw; 2701 int i; 2702 2703 for (i = 0; i < priv->num_rx_rings; i++) 2704 enetc_enable_rxbdr(hw, priv->rx_ring[i]); 2705 } 2706 2707 static void enetc_enable_tx_bdrs(struct enetc_ndev_priv *priv) 2708 { 2709 struct enetc_hw *hw = &priv->si->hw; 2710 int i; 2711 2712 for (i = 0; i < priv->num_tx_rings; i++) 2713 enetc_enable_txbdr(hw, priv->tx_ring[i]); 2714 } 2715 2716 static void enetc_disable_rxbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring) 2717 { 2718 int idx = rx_ring->index; 2719 2720 /* disable EN bit on ring */ 2721 enetc_rxbdr_wr(hw, idx, ENETC_RBMR, 0); 2722 } 2723 2724 static void enetc_disable_txbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring) 2725 { 2726 int idx = rx_ring->index; 2727 2728 /* disable EN bit on ring */ 2729 enetc_txbdr_wr(hw, idx, ENETC_TBMR, 0); 2730 } 2731 2732 static void enetc_disable_rx_bdrs(struct enetc_ndev_priv *priv) 2733 { 2734 struct enetc_hw *hw = &priv->si->hw; 2735 int i; 2736 2737 for (i = 0; i < priv->num_rx_rings; i++) 2738 enetc_disable_rxbdr(hw, priv->rx_ring[i]); 2739 } 2740 2741 static void enetc_disable_tx_bdrs(struct enetc_ndev_priv *priv) 2742 { 2743 struct enetc_hw *hw = &priv->si->hw; 2744 int i; 2745 2746 for (i = 0; i < priv->num_tx_rings; i++) 2747 enetc_disable_txbdr(hw, priv->tx_ring[i]); 2748 } 2749 2750 static void enetc_wait_txbdr(struct enetc_hw *hw, struct enetc_bdr *tx_ring) 2751 { 2752 int delay = 8, timeout = 100; 2753 int idx = tx_ring->index; 2754 2755 /* wait for busy to clear */ 2756 while (delay < timeout && 2757 enetc_txbdr_rd(hw, idx, ENETC_TBSR) & ENETC_TBSR_BUSY) { 2758 msleep(delay); 2759 delay *= 2; 2760 } 2761 2762 if (delay >= timeout) 2763 netdev_warn(tx_ring->ndev, "timeout for tx ring #%d clear\n", 2764 idx); 2765 } 2766 2767 static void enetc_wait_bdrs(struct enetc_ndev_priv *priv) 2768 { 2769 struct enetc_hw *hw = &priv->si->hw; 2770 int i; 2771 2772 for (i = 0; i < priv->num_tx_rings; i++) 2773 enetc_wait_txbdr(hw, priv->tx_ring[i]); 2774 } 2775 2776 static int enetc_setup_irqs(struct enetc_ndev_priv *priv) 2777 { 2778 struct pci_dev *pdev = priv->si->pdev; 2779 struct enetc_hw *hw = &priv->si->hw; 2780 int i, j, err; 2781 2782 for (i = 0; i < priv->bdr_int_num; i++) { 2783 int irq = pci_irq_vector(pdev, ENETC_BDR_INT_BASE_IDX + i); 2784 struct enetc_int_vector *v = priv->int_vector[i]; 2785 int entry = ENETC_BDR_INT_BASE_IDX + i; 2786 2787 snprintf(v->name, sizeof(v->name), "%s-rxtx%d", 2788 priv->ndev->name, i); 2789 err = request_irq(irq, enetc_msix, IRQF_NO_AUTOEN, v->name, v); 2790 if (err) { 2791 dev_err(priv->dev, "request_irq() failed!\n"); 2792 goto irq_err; 2793 } 2794 2795 v->tbier_base = hw->reg + ENETC_BDR(TX, 0, ENETC_TBIER); 2796 v->rbier = hw->reg + ENETC_BDR(RX, i, ENETC_RBIER); 2797 v->ricr1 = hw->reg + ENETC_BDR(RX, i, ENETC_RBICR1); 2798 2799 enetc_wr(hw, ENETC_SIMSIRRV(i), entry); 2800 2801 for (j = 0; j < v->count_tx_rings; j++) { 2802 int idx = v->tx_ring[j].index; 2803 2804 enetc_wr(hw, ENETC_SIMSITRV(idx), entry); 2805 } 2806 irq_set_affinity_hint(irq, get_cpu_mask(i % num_online_cpus())); 2807 } 2808 2809 return 0; 2810 2811 irq_err: 2812 while (i--) { 2813 int irq = pci_irq_vector(pdev, ENETC_BDR_INT_BASE_IDX + i); 2814 2815 irq_set_affinity_hint(irq, NULL); 2816 free_irq(irq, priv->int_vector[i]); 2817 } 2818 2819 return err; 2820 } 2821 2822 static void enetc_free_irqs(struct enetc_ndev_priv *priv) 2823 { 2824 struct pci_dev *pdev = priv->si->pdev; 2825 int i; 2826 2827 for (i = 0; i < priv->bdr_int_num; i++) { 2828 int irq = pci_irq_vector(pdev, ENETC_BDR_INT_BASE_IDX + i); 2829 2830 irq_set_affinity_hint(irq, NULL); 2831 free_irq(irq, priv->int_vector[i]); 2832 } 2833 } 2834 2835 static void enetc_setup_interrupts(struct enetc_ndev_priv *priv) 2836 { 2837 struct enetc_hw *hw = &priv->si->hw; 2838 u32 icpt, ictt; 2839 int i; 2840 2841 /* enable Tx & Rx event indication */ 2842 if (priv->ic_mode & 2843 (ENETC_IC_RX_MANUAL | ENETC_IC_RX_ADAPTIVE)) { 2844 icpt = ENETC_RBICR0_SET_ICPT(ENETC_RXIC_PKTTHR); 2845 /* init to non-0 minimum, will be adjusted later */ 2846 ictt = 0x1; 2847 } else { 2848 icpt = 0x1; /* enable Rx ints by setting pkt thr to 1 */ 2849 ictt = 0; 2850 } 2851 2852 for (i = 0; i < priv->num_rx_rings; i++) { 2853 enetc_rxbdr_wr(hw, i, ENETC_RBICR1, ictt); 2854 enetc_rxbdr_wr(hw, i, ENETC_RBICR0, ENETC_RBICR0_ICEN | icpt); 2855 enetc_rxbdr_wr(hw, i, ENETC_RBIER, ENETC_RBIER_RXTIE); 2856 } 2857 2858 if (priv->ic_mode & ENETC_IC_TX_MANUAL) 2859 icpt = ENETC_TBICR0_SET_ICPT(ENETC_TXIC_PKTTHR); 2860 else 2861 icpt = 0x1; /* enable Tx ints by setting pkt thr to 1 */ 2862 2863 for (i = 0; i < priv->num_tx_rings; i++) { 2864 enetc_txbdr_wr(hw, i, ENETC_TBICR1, priv->tx_ictt); 2865 enetc_txbdr_wr(hw, i, ENETC_TBICR0, ENETC_TBICR0_ICEN | icpt); 2866 enetc_txbdr_wr(hw, i, ENETC_TBIER, ENETC_TBIER_TXTIE); 2867 } 2868 } 2869 2870 static void enetc_clear_interrupts(struct enetc_ndev_priv *priv) 2871 { 2872 struct enetc_hw *hw = &priv->si->hw; 2873 int i; 2874 2875 for (i = 0; i < priv->num_tx_rings; i++) 2876 enetc_txbdr_wr(hw, i, ENETC_TBIER, 0); 2877 2878 for (i = 0; i < priv->num_rx_rings; i++) 2879 enetc_rxbdr_wr(hw, i, ENETC_RBIER, 0); 2880 } 2881 2882 static int enetc_phylink_connect(struct net_device *ndev) 2883 { 2884 struct enetc_ndev_priv *priv = netdev_priv(ndev); 2885 struct ethtool_keee edata; 2886 int err; 2887 2888 if (!priv->phylink) { 2889 /* phy-less mode */ 2890 netif_carrier_on(ndev); 2891 return 0; 2892 } 2893 2894 err = phylink_of_phy_connect(priv->phylink, priv->dev->of_node, 0); 2895 if (err) { 2896 dev_err(&ndev->dev, "could not attach to PHY\n"); 2897 return err; 2898 } 2899 2900 /* disable EEE autoneg, until ENETC driver supports it */ 2901 memset(&edata, 0, sizeof(struct ethtool_keee)); 2902 phylink_ethtool_set_eee(priv->phylink, &edata); 2903 2904 phylink_start(priv->phylink); 2905 2906 return 0; 2907 } 2908 2909 static void enetc_tx_onestep_tstamp(struct work_struct *work) 2910 { 2911 struct enetc_ndev_priv *priv; 2912 struct sk_buff *skb; 2913 2914 priv = container_of(work, struct enetc_ndev_priv, tx_onestep_tstamp); 2915 2916 netif_tx_lock_bh(priv->ndev); 2917 2918 clear_bit_unlock(ENETC_TX_ONESTEP_TSTAMP_IN_PROGRESS, &priv->flags); 2919 skb = skb_dequeue(&priv->tx_skbs); 2920 if (skb) 2921 enetc_start_xmit(skb, priv->ndev); 2922 2923 netif_tx_unlock_bh(priv->ndev); 2924 } 2925 2926 static void enetc_tx_onestep_tstamp_init(struct enetc_ndev_priv *priv) 2927 { 2928 INIT_WORK(&priv->tx_onestep_tstamp, enetc_tx_onestep_tstamp); 2929 skb_queue_head_init(&priv->tx_skbs); 2930 } 2931 2932 void enetc_start(struct net_device *ndev) 2933 { 2934 struct enetc_ndev_priv *priv = netdev_priv(ndev); 2935 int i; 2936 2937 enetc_setup_interrupts(priv); 2938 2939 for (i = 0; i < priv->bdr_int_num; i++) { 2940 int irq = pci_irq_vector(priv->si->pdev, 2941 ENETC_BDR_INT_BASE_IDX + i); 2942 2943 napi_enable(&priv->int_vector[i]->napi); 2944 enable_irq(irq); 2945 } 2946 2947 enetc_enable_tx_bdrs(priv); 2948 2949 enetc_enable_rx_bdrs(priv); 2950 2951 netif_tx_start_all_queues(ndev); 2952 2953 clear_bit(ENETC_TX_DOWN, &priv->flags); 2954 } 2955 EXPORT_SYMBOL_GPL(enetc_start); 2956 2957 int enetc_open(struct net_device *ndev) 2958 { 2959 struct enetc_ndev_priv *priv = netdev_priv(ndev); 2960 struct enetc_bdr_resource *tx_res, *rx_res; 2961 bool extended; 2962 int err; 2963 2964 extended = !!(priv->active_offloads & ENETC_F_RX_TSTAMP); 2965 2966 err = clk_prepare_enable(priv->ref_clk); 2967 if (err) 2968 return err; 2969 2970 err = enetc_setup_irqs(priv); 2971 if (err) 2972 goto err_setup_irqs; 2973 2974 err = enetc_phylink_connect(ndev); 2975 if (err) 2976 goto err_phy_connect; 2977 2978 tx_res = enetc_alloc_tx_resources(priv); 2979 if (IS_ERR(tx_res)) { 2980 err = PTR_ERR(tx_res); 2981 goto err_alloc_tx; 2982 } 2983 2984 rx_res = enetc_alloc_rx_resources(priv, extended); 2985 if (IS_ERR(rx_res)) { 2986 err = PTR_ERR(rx_res); 2987 goto err_alloc_rx; 2988 } 2989 2990 enetc_tx_onestep_tstamp_init(priv); 2991 enetc_assign_tx_resources(priv, tx_res); 2992 enetc_assign_rx_resources(priv, rx_res); 2993 enetc_setup_bdrs(priv, extended); 2994 enetc_start(ndev); 2995 2996 return 0; 2997 2998 err_alloc_rx: 2999 enetc_free_tx_resources(tx_res, priv->num_tx_rings); 3000 err_alloc_tx: 3001 if (priv->phylink) 3002 phylink_disconnect_phy(priv->phylink); 3003 err_phy_connect: 3004 enetc_free_irqs(priv); 3005 err_setup_irqs: 3006 clk_disable_unprepare(priv->ref_clk); 3007 3008 return err; 3009 } 3010 EXPORT_SYMBOL_GPL(enetc_open); 3011 3012 void enetc_stop(struct net_device *ndev) 3013 { 3014 struct enetc_ndev_priv *priv = netdev_priv(ndev); 3015 int i; 3016 3017 set_bit(ENETC_TX_DOWN, &priv->flags); 3018 3019 netif_tx_stop_all_queues(ndev); 3020 3021 enetc_disable_rx_bdrs(priv); 3022 3023 enetc_wait_bdrs(priv); 3024 3025 enetc_disable_tx_bdrs(priv); 3026 3027 for (i = 0; i < priv->bdr_int_num; i++) { 3028 int irq = pci_irq_vector(priv->si->pdev, 3029 ENETC_BDR_INT_BASE_IDX + i); 3030 3031 disable_irq(irq); 3032 napi_synchronize(&priv->int_vector[i]->napi); 3033 napi_disable(&priv->int_vector[i]->napi); 3034 } 3035 3036 enetc_clear_interrupts(priv); 3037 } 3038 EXPORT_SYMBOL_GPL(enetc_stop); 3039 3040 int enetc_close(struct net_device *ndev) 3041 { 3042 struct enetc_ndev_priv *priv = netdev_priv(ndev); 3043 3044 enetc_stop(ndev); 3045 3046 if (priv->phylink) { 3047 phylink_stop(priv->phylink); 3048 phylink_disconnect_phy(priv->phylink); 3049 } else { 3050 netif_carrier_off(ndev); 3051 } 3052 3053 enetc_free_rxtx_rings(priv); 3054 3055 /* Avoids dangling pointers and also frees old resources */ 3056 enetc_assign_rx_resources(priv, NULL); 3057 enetc_assign_tx_resources(priv, NULL); 3058 3059 enetc_free_irqs(priv); 3060 clk_disable_unprepare(priv->ref_clk); 3061 3062 return 0; 3063 } 3064 EXPORT_SYMBOL_GPL(enetc_close); 3065 3066 static int enetc_reconfigure(struct enetc_ndev_priv *priv, bool extended, 3067 int (*cb)(struct enetc_ndev_priv *priv, void *ctx), 3068 void *ctx) 3069 { 3070 struct enetc_bdr_resource *tx_res, *rx_res; 3071 int err; 3072 3073 ASSERT_RTNL(); 3074 3075 /* If the interface is down, run the callback right away, 3076 * without reconfiguration. 3077 */ 3078 if (!netif_running(priv->ndev)) { 3079 if (cb) { 3080 err = cb(priv, ctx); 3081 if (err) 3082 return err; 3083 } 3084 3085 return 0; 3086 } 3087 3088 tx_res = enetc_alloc_tx_resources(priv); 3089 if (IS_ERR(tx_res)) { 3090 err = PTR_ERR(tx_res); 3091 goto out; 3092 } 3093 3094 rx_res = enetc_alloc_rx_resources(priv, extended); 3095 if (IS_ERR(rx_res)) { 3096 err = PTR_ERR(rx_res); 3097 goto out_free_tx_res; 3098 } 3099 3100 enetc_stop(priv->ndev); 3101 enetc_free_rxtx_rings(priv); 3102 3103 /* Interface is down, run optional callback now */ 3104 if (cb) { 3105 err = cb(priv, ctx); 3106 if (err) 3107 goto out_restart; 3108 } 3109 3110 enetc_assign_tx_resources(priv, tx_res); 3111 enetc_assign_rx_resources(priv, rx_res); 3112 enetc_setup_bdrs(priv, extended); 3113 enetc_start(priv->ndev); 3114 3115 return 0; 3116 3117 out_restart: 3118 enetc_setup_bdrs(priv, extended); 3119 enetc_start(priv->ndev); 3120 enetc_free_rx_resources(rx_res, priv->num_rx_rings); 3121 out_free_tx_res: 3122 enetc_free_tx_resources(tx_res, priv->num_tx_rings); 3123 out: 3124 return err; 3125 } 3126 3127 static void enetc_debug_tx_ring_prios(struct enetc_ndev_priv *priv) 3128 { 3129 int i; 3130 3131 for (i = 0; i < priv->num_tx_rings; i++) 3132 netdev_dbg(priv->ndev, "TX ring %d prio %d\n", i, 3133 priv->tx_ring[i]->prio); 3134 } 3135 3136 void enetc_reset_tc_mqprio(struct net_device *ndev) 3137 { 3138 struct enetc_ndev_priv *priv = netdev_priv(ndev); 3139 struct enetc_hw *hw = &priv->si->hw; 3140 struct enetc_bdr *tx_ring; 3141 int num_stack_tx_queues; 3142 int i; 3143 3144 num_stack_tx_queues = enetc_num_stack_tx_queues(priv); 3145 3146 netdev_reset_tc(ndev); 3147 netif_set_real_num_tx_queues(ndev, num_stack_tx_queues); 3148 priv->min_num_stack_tx_queues = num_possible_cpus(); 3149 3150 /* Reset all ring priorities to 0 */ 3151 for (i = 0; i < priv->num_tx_rings; i++) { 3152 tx_ring = priv->tx_ring[i]; 3153 tx_ring->prio = 0; 3154 enetc_set_bdr_prio(hw, tx_ring->index, tx_ring->prio); 3155 } 3156 3157 enetc_debug_tx_ring_prios(priv); 3158 3159 enetc_change_preemptible_tcs(priv, 0); 3160 } 3161 EXPORT_SYMBOL_GPL(enetc_reset_tc_mqprio); 3162 3163 int enetc_setup_tc_mqprio(struct net_device *ndev, void *type_data) 3164 { 3165 struct tc_mqprio_qopt_offload *mqprio = type_data; 3166 struct enetc_ndev_priv *priv = netdev_priv(ndev); 3167 struct tc_mqprio_qopt *qopt = &mqprio->qopt; 3168 struct enetc_hw *hw = &priv->si->hw; 3169 int num_stack_tx_queues = 0; 3170 struct enetc_bdr *tx_ring; 3171 u8 num_tc = qopt->num_tc; 3172 int offset, count; 3173 int err, tc, q; 3174 3175 if (!num_tc) { 3176 enetc_reset_tc_mqprio(ndev); 3177 return 0; 3178 } 3179 3180 err = netdev_set_num_tc(ndev, num_tc); 3181 if (err) 3182 return err; 3183 3184 for (tc = 0; tc < num_tc; tc++) { 3185 offset = qopt->offset[tc]; 3186 count = qopt->count[tc]; 3187 num_stack_tx_queues += count; 3188 3189 err = netdev_set_tc_queue(ndev, tc, count, offset); 3190 if (err) 3191 goto err_reset_tc; 3192 3193 for (q = offset; q < offset + count; q++) { 3194 tx_ring = priv->tx_ring[q]; 3195 /* The prio_tc_map is skb_tx_hash()'s way of selecting 3196 * between TX queues based on skb->priority. As such, 3197 * there's nothing to offload based on it. 3198 * Make the mqprio "traffic class" be the priority of 3199 * this ring group, and leave the Tx IPV to traffic 3200 * class mapping as its default mapping value of 1:1. 3201 */ 3202 tx_ring->prio = tc; 3203 enetc_set_bdr_prio(hw, tx_ring->index, tx_ring->prio); 3204 } 3205 } 3206 3207 err = netif_set_real_num_tx_queues(ndev, num_stack_tx_queues); 3208 if (err) 3209 goto err_reset_tc; 3210 3211 priv->min_num_stack_tx_queues = num_stack_tx_queues; 3212 3213 enetc_debug_tx_ring_prios(priv); 3214 3215 enetc_change_preemptible_tcs(priv, mqprio->preemptible_tcs); 3216 3217 return 0; 3218 3219 err_reset_tc: 3220 enetc_reset_tc_mqprio(ndev); 3221 return err; 3222 } 3223 EXPORT_SYMBOL_GPL(enetc_setup_tc_mqprio); 3224 3225 static int enetc_reconfigure_xdp_cb(struct enetc_ndev_priv *priv, void *ctx) 3226 { 3227 struct bpf_prog *old_prog, *prog = ctx; 3228 int num_stack_tx_queues; 3229 int err, i; 3230 3231 old_prog = xchg(&priv->xdp_prog, prog); 3232 3233 num_stack_tx_queues = enetc_num_stack_tx_queues(priv); 3234 err = netif_set_real_num_tx_queues(priv->ndev, num_stack_tx_queues); 3235 if (err) { 3236 xchg(&priv->xdp_prog, old_prog); 3237 return err; 3238 } 3239 3240 if (old_prog) 3241 bpf_prog_put(old_prog); 3242 3243 for (i = 0; i < priv->num_rx_rings; i++) { 3244 struct enetc_bdr *rx_ring = priv->rx_ring[i]; 3245 3246 rx_ring->xdp.prog = prog; 3247 3248 if (prog) 3249 rx_ring->buffer_offset = XDP_PACKET_HEADROOM; 3250 else 3251 rx_ring->buffer_offset = ENETC_RXB_PAD; 3252 } 3253 3254 return 0; 3255 } 3256 3257 static int enetc_setup_xdp_prog(struct net_device *ndev, struct bpf_prog *prog, 3258 struct netlink_ext_ack *extack) 3259 { 3260 int num_xdp_tx_queues = prog ? num_possible_cpus() : 0; 3261 struct enetc_ndev_priv *priv = netdev_priv(ndev); 3262 bool extended; 3263 3264 if (priv->min_num_stack_tx_queues + num_xdp_tx_queues > 3265 priv->num_tx_rings) { 3266 NL_SET_ERR_MSG_FMT_MOD(extack, 3267 "Reserving %d XDP TXQs leaves under %d for stack (total %d)", 3268 num_xdp_tx_queues, 3269 priv->min_num_stack_tx_queues, 3270 priv->num_tx_rings); 3271 return -EBUSY; 3272 } 3273 3274 extended = !!(priv->active_offloads & ENETC_F_RX_TSTAMP); 3275 3276 /* The buffer layout is changing, so we need to drain the old 3277 * RX buffers and seed new ones. 3278 */ 3279 return enetc_reconfigure(priv, extended, enetc_reconfigure_xdp_cb, prog); 3280 } 3281 3282 int enetc_setup_bpf(struct net_device *ndev, struct netdev_bpf *bpf) 3283 { 3284 switch (bpf->command) { 3285 case XDP_SETUP_PROG: 3286 return enetc_setup_xdp_prog(ndev, bpf->prog, bpf->extack); 3287 default: 3288 return -EINVAL; 3289 } 3290 3291 return 0; 3292 } 3293 EXPORT_SYMBOL_GPL(enetc_setup_bpf); 3294 3295 struct net_device_stats *enetc_get_stats(struct net_device *ndev) 3296 { 3297 struct enetc_ndev_priv *priv = netdev_priv(ndev); 3298 struct net_device_stats *stats = &ndev->stats; 3299 unsigned long packets = 0, bytes = 0; 3300 unsigned long tx_dropped = 0; 3301 int i; 3302 3303 for (i = 0; i < priv->num_rx_rings; i++) { 3304 packets += priv->rx_ring[i]->stats.packets; 3305 bytes += priv->rx_ring[i]->stats.bytes; 3306 } 3307 3308 stats->rx_packets = packets; 3309 stats->rx_bytes = bytes; 3310 bytes = 0; 3311 packets = 0; 3312 3313 for (i = 0; i < priv->num_tx_rings; i++) { 3314 packets += priv->tx_ring[i]->stats.packets; 3315 bytes += priv->tx_ring[i]->stats.bytes; 3316 tx_dropped += priv->tx_ring[i]->stats.win_drop; 3317 } 3318 3319 stats->tx_packets = packets; 3320 stats->tx_bytes = bytes; 3321 stats->tx_dropped = tx_dropped; 3322 3323 return stats; 3324 } 3325 EXPORT_SYMBOL_GPL(enetc_get_stats); 3326 3327 static void enetc_enable_rxvlan(struct net_device *ndev, bool en) 3328 { 3329 struct enetc_ndev_priv *priv = netdev_priv(ndev); 3330 struct enetc_hw *hw = &priv->si->hw; 3331 int i; 3332 3333 for (i = 0; i < priv->num_rx_rings; i++) 3334 enetc_bdr_enable_rxvlan(hw, i, en); 3335 } 3336 3337 static void enetc_enable_txvlan(struct net_device *ndev, bool en) 3338 { 3339 struct enetc_ndev_priv *priv = netdev_priv(ndev); 3340 struct enetc_hw *hw = &priv->si->hw; 3341 int i; 3342 3343 for (i = 0; i < priv->num_tx_rings; i++) 3344 enetc_bdr_enable_txvlan(hw, i, en); 3345 } 3346 3347 void enetc_set_features(struct net_device *ndev, netdev_features_t features) 3348 { 3349 netdev_features_t changed = ndev->features ^ features; 3350 3351 if (changed & NETIF_F_RXHASH) 3352 enetc_set_rss(ndev, !!(features & NETIF_F_RXHASH)); 3353 3354 if (changed & NETIF_F_HW_VLAN_CTAG_RX) 3355 enetc_enable_rxvlan(ndev, 3356 !!(features & NETIF_F_HW_VLAN_CTAG_RX)); 3357 3358 if (changed & NETIF_F_HW_VLAN_CTAG_TX) 3359 enetc_enable_txvlan(ndev, 3360 !!(features & NETIF_F_HW_VLAN_CTAG_TX)); 3361 } 3362 EXPORT_SYMBOL_GPL(enetc_set_features); 3363 3364 int enetc_hwtstamp_set(struct net_device *ndev, 3365 struct kernel_hwtstamp_config *config, 3366 struct netlink_ext_ack *extack) 3367 { 3368 struct enetc_ndev_priv *priv = netdev_priv(ndev); 3369 int err, new_offloads = priv->active_offloads; 3370 3371 if (!enetc_ptp_clock_is_enabled(priv->si)) 3372 return -EOPNOTSUPP; 3373 3374 switch (config->tx_type) { 3375 case HWTSTAMP_TX_OFF: 3376 new_offloads &= ~ENETC_F_TX_TSTAMP_MASK; 3377 break; 3378 case HWTSTAMP_TX_ON: 3379 new_offloads &= ~ENETC_F_TX_TSTAMP_MASK; 3380 new_offloads |= ENETC_F_TX_TSTAMP; 3381 break; 3382 case HWTSTAMP_TX_ONESTEP_SYNC: 3383 if (!enetc_si_is_pf(priv->si) || 3384 enetc_is_pseudo_mac(priv->si)) 3385 return -EOPNOTSUPP; 3386 3387 new_offloads &= ~ENETC_F_TX_TSTAMP_MASK; 3388 new_offloads |= ENETC_F_TX_ONESTEP_SYNC_TSTAMP; 3389 break; 3390 default: 3391 return -ERANGE; 3392 } 3393 3394 switch (config->rx_filter) { 3395 case HWTSTAMP_FILTER_NONE: 3396 new_offloads &= ~ENETC_F_RX_TSTAMP; 3397 break; 3398 default: 3399 new_offloads |= ENETC_F_RX_TSTAMP; 3400 config->rx_filter = HWTSTAMP_FILTER_ALL; 3401 } 3402 3403 if ((new_offloads ^ priv->active_offloads) & ENETC_F_RX_TSTAMP) { 3404 bool extended = !!(new_offloads & ENETC_F_RX_TSTAMP); 3405 3406 err = enetc_reconfigure(priv, extended, NULL, NULL); 3407 if (err) 3408 return err; 3409 } 3410 3411 priv->active_offloads = new_offloads; 3412 3413 return 0; 3414 } 3415 EXPORT_SYMBOL_GPL(enetc_hwtstamp_set); 3416 3417 int enetc_hwtstamp_get(struct net_device *ndev, 3418 struct kernel_hwtstamp_config *config) 3419 { 3420 struct enetc_ndev_priv *priv = netdev_priv(ndev); 3421 3422 if (!enetc_ptp_clock_is_enabled(priv->si)) 3423 return -EOPNOTSUPP; 3424 3425 if (priv->active_offloads & ENETC_F_TX_ONESTEP_SYNC_TSTAMP) 3426 config->tx_type = HWTSTAMP_TX_ONESTEP_SYNC; 3427 else if (priv->active_offloads & ENETC_F_TX_TSTAMP) 3428 config->tx_type = HWTSTAMP_TX_ON; 3429 else 3430 config->tx_type = HWTSTAMP_TX_OFF; 3431 3432 config->rx_filter = (priv->active_offloads & ENETC_F_RX_TSTAMP) ? 3433 HWTSTAMP_FILTER_ALL : HWTSTAMP_FILTER_NONE; 3434 3435 return 0; 3436 } 3437 EXPORT_SYMBOL_GPL(enetc_hwtstamp_get); 3438 3439 int enetc_ioctl(struct net_device *ndev, struct ifreq *rq, int cmd) 3440 { 3441 struct enetc_ndev_priv *priv = netdev_priv(ndev); 3442 3443 if (!priv->phylink) 3444 return -EOPNOTSUPP; 3445 3446 return phylink_mii_ioctl(priv->phylink, rq, cmd); 3447 } 3448 EXPORT_SYMBOL_GPL(enetc_ioctl); 3449 3450 static int enetc_int_vector_init(struct enetc_ndev_priv *priv, int i, 3451 int v_tx_rings) 3452 { 3453 struct enetc_int_vector *v; 3454 struct enetc_bdr *bdr; 3455 int j, err; 3456 3457 v = kzalloc(struct_size(v, tx_ring, v_tx_rings), GFP_KERNEL); 3458 if (!v) 3459 return -ENOMEM; 3460 3461 priv->int_vector[i] = v; 3462 bdr = &v->rx_ring; 3463 bdr->index = i; 3464 bdr->ndev = priv->ndev; 3465 bdr->dev = priv->dev; 3466 bdr->bd_count = priv->rx_bd_count; 3467 bdr->buffer_offset = ENETC_RXB_PAD; 3468 priv->rx_ring[i] = bdr; 3469 3470 err = __xdp_rxq_info_reg(&bdr->xdp.rxq, priv->ndev, i, 0, 3471 ENETC_RXB_DMA_SIZE_XDP); 3472 if (err) 3473 goto free_vector; 3474 3475 err = xdp_rxq_info_reg_mem_model(&bdr->xdp.rxq, MEM_TYPE_PAGE_SHARED, 3476 NULL); 3477 if (err) { 3478 xdp_rxq_info_unreg(&bdr->xdp.rxq); 3479 goto free_vector; 3480 } 3481 3482 /* init defaults for adaptive IC */ 3483 if (priv->ic_mode & ENETC_IC_RX_ADAPTIVE) { 3484 v->rx_ictt = 0x1; 3485 v->rx_dim_en = true; 3486 } 3487 3488 INIT_WORK(&v->rx_dim.work, enetc_rx_dim_work); 3489 netif_napi_add(priv->ndev, &v->napi, enetc_poll); 3490 v->count_tx_rings = v_tx_rings; 3491 3492 for (j = 0; j < v_tx_rings; j++) { 3493 int idx; 3494 3495 /* default tx ring mapping policy */ 3496 idx = priv->bdr_int_num * j + i; 3497 __set_bit(idx, &v->tx_rings_map); 3498 bdr = &v->tx_ring[j]; 3499 bdr->index = idx; 3500 bdr->ndev = priv->ndev; 3501 bdr->dev = priv->dev; 3502 bdr->bd_count = priv->tx_bd_count; 3503 priv->tx_ring[idx] = bdr; 3504 } 3505 3506 return 0; 3507 3508 free_vector: 3509 priv->rx_ring[i] = NULL; 3510 priv->int_vector[i] = NULL; 3511 kfree(v); 3512 3513 return err; 3514 } 3515 3516 static void enetc_int_vector_destroy(struct enetc_ndev_priv *priv, int i) 3517 { 3518 struct enetc_int_vector *v = priv->int_vector[i]; 3519 struct enetc_bdr *rx_ring = &v->rx_ring; 3520 int j, tx_ring_index; 3521 3522 xdp_rxq_info_unreg_mem_model(&rx_ring->xdp.rxq); 3523 xdp_rxq_info_unreg(&rx_ring->xdp.rxq); 3524 netif_napi_del(&v->napi); 3525 cancel_work_sync(&v->rx_dim.work); 3526 3527 for (j = 0; j < v->count_tx_rings; j++) { 3528 tx_ring_index = priv->bdr_int_num * j + i; 3529 priv->tx_ring[tx_ring_index] = NULL; 3530 } 3531 3532 priv->rx_ring[i] = NULL; 3533 priv->int_vector[i] = NULL; 3534 kfree(v); 3535 } 3536 3537 int enetc_alloc_msix(struct enetc_ndev_priv *priv) 3538 { 3539 struct pci_dev *pdev = priv->si->pdev; 3540 int v_tx_rings, v_remainder; 3541 int num_stack_tx_queues; 3542 int first_xdp_tx_ring; 3543 int i, n, err, nvec; 3544 3545 nvec = ENETC_BDR_INT_BASE_IDX + priv->bdr_int_num; 3546 /* allocate MSIX for both messaging and Rx/Tx interrupts */ 3547 n = pci_alloc_irq_vectors(pdev, nvec, nvec, PCI_IRQ_MSIX); 3548 3549 if (n < 0) 3550 return n; 3551 3552 if (n != nvec) 3553 return -EPERM; 3554 3555 /* # of tx rings per int vector */ 3556 v_tx_rings = priv->num_tx_rings / priv->bdr_int_num; 3557 v_remainder = priv->num_tx_rings % priv->bdr_int_num; 3558 3559 for (i = 0; i < priv->bdr_int_num; i++) { 3560 /* Distribute the remaining TX rings to the first v_remainder 3561 * interrupt vectors 3562 */ 3563 int num_tx_rings = i < v_remainder ? v_tx_rings + 1 : v_tx_rings; 3564 3565 err = enetc_int_vector_init(priv, i, num_tx_rings); 3566 if (err) 3567 goto fail; 3568 } 3569 3570 num_stack_tx_queues = enetc_num_stack_tx_queues(priv); 3571 3572 err = netif_set_real_num_tx_queues(priv->ndev, num_stack_tx_queues); 3573 if (err) 3574 goto fail; 3575 3576 err = netif_set_real_num_rx_queues(priv->ndev, priv->num_rx_rings); 3577 if (err) 3578 goto fail; 3579 3580 priv->min_num_stack_tx_queues = num_possible_cpus(); 3581 first_xdp_tx_ring = priv->num_tx_rings - num_possible_cpus(); 3582 priv->xdp_tx_ring = &priv->tx_ring[first_xdp_tx_ring]; 3583 3584 return 0; 3585 3586 fail: 3587 while (i--) 3588 enetc_int_vector_destroy(priv, i); 3589 3590 pci_free_irq_vectors(pdev); 3591 3592 return err; 3593 } 3594 EXPORT_SYMBOL_GPL(enetc_alloc_msix); 3595 3596 void enetc_free_msix(struct enetc_ndev_priv *priv) 3597 { 3598 int i; 3599 3600 for (i = 0; i < priv->bdr_int_num; i++) 3601 enetc_int_vector_destroy(priv, i); 3602 3603 /* disable all MSIX for this device */ 3604 pci_free_irq_vectors(priv->si->pdev); 3605 } 3606 EXPORT_SYMBOL_GPL(enetc_free_msix); 3607 3608 static void enetc_kfree_si(struct enetc_si *si) 3609 { 3610 char *p = (char *)si - si->pad; 3611 3612 kfree(p); 3613 } 3614 3615 static void enetc_detect_errata(struct enetc_si *si) 3616 { 3617 if (si->pdev->revision == ENETC_REV1) 3618 si->errata = ENETC_ERR_VLAN_ISOL | ENETC_ERR_UCMCSWP; 3619 } 3620 3621 int enetc_pci_probe(struct pci_dev *pdev, const char *name, int sizeof_priv) 3622 { 3623 struct enetc_si *si, *p; 3624 struct enetc_hw *hw; 3625 size_t alloc_size; 3626 int err, len; 3627 3628 pcie_flr(pdev); 3629 err = pci_enable_device_mem(pdev); 3630 if (err) 3631 return dev_err_probe(&pdev->dev, err, "device enable failed\n"); 3632 3633 /* set up for high or low dma */ 3634 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 3635 if (err) { 3636 dev_err(&pdev->dev, "DMA configuration failed: 0x%x\n", err); 3637 goto err_dma; 3638 } 3639 3640 err = pci_request_mem_regions(pdev, name); 3641 if (err) { 3642 dev_err(&pdev->dev, "pci_request_regions failed err=%d\n", err); 3643 goto err_pci_mem_reg; 3644 } 3645 3646 pci_set_master(pdev); 3647 3648 alloc_size = sizeof(struct enetc_si); 3649 if (sizeof_priv) { 3650 /* align priv to 32B */ 3651 alloc_size = ALIGN(alloc_size, ENETC_SI_ALIGN); 3652 alloc_size += sizeof_priv; 3653 } 3654 /* force 32B alignment for enetc_si */ 3655 alloc_size += ENETC_SI_ALIGN - 1; 3656 3657 p = kzalloc(alloc_size, GFP_KERNEL); 3658 if (!p) { 3659 err = -ENOMEM; 3660 goto err_alloc_si; 3661 } 3662 3663 si = PTR_ALIGN(p, ENETC_SI_ALIGN); 3664 si->pad = (char *)si - (char *)p; 3665 3666 pci_set_drvdata(pdev, si); 3667 si->pdev = pdev; 3668 hw = &si->hw; 3669 3670 len = pci_resource_len(pdev, ENETC_BAR_REGS); 3671 hw->reg = ioremap(pci_resource_start(pdev, ENETC_BAR_REGS), len); 3672 if (!hw->reg) { 3673 err = -ENXIO; 3674 dev_err(&pdev->dev, "ioremap() failed\n"); 3675 goto err_ioremap; 3676 } 3677 if (len > ENETC_PORT_BASE) 3678 hw->port = hw->reg + ENETC_PORT_BASE; 3679 if (len > ENETC_GLOBAL_BASE) 3680 hw->global = hw->reg + ENETC_GLOBAL_BASE; 3681 3682 enetc_detect_errata(si); 3683 3684 return 0; 3685 3686 err_ioremap: 3687 enetc_kfree_si(si); 3688 err_alloc_si: 3689 pci_release_mem_regions(pdev); 3690 err_pci_mem_reg: 3691 err_dma: 3692 pci_disable_device(pdev); 3693 3694 return err; 3695 } 3696 EXPORT_SYMBOL_GPL(enetc_pci_probe); 3697 3698 void enetc_pci_remove(struct pci_dev *pdev) 3699 { 3700 struct enetc_si *si = pci_get_drvdata(pdev); 3701 struct enetc_hw *hw = &si->hw; 3702 3703 iounmap(hw->reg); 3704 enetc_kfree_si(si); 3705 pci_release_mem_regions(pdev); 3706 pci_disable_device(pdev); 3707 } 3708 EXPORT_SYMBOL_GPL(enetc_pci_remove); 3709 3710 static const struct enetc_drvdata enetc_pf_data = { 3711 .sysclk_freq = ENETC_CLK_400M, 3712 .pmac_offset = ENETC_PMAC_OFFSET, 3713 .max_frags = ENETC_MAX_SKB_FRAGS, 3714 .eth_ops = &enetc_pf_ethtool_ops, 3715 }; 3716 3717 static const struct enetc_drvdata enetc4_pf_data = { 3718 .sysclk_freq = ENETC_CLK_333M, 3719 .tx_csum = true, 3720 .max_frags = ENETC4_MAX_SKB_FRAGS, 3721 .pmac_offset = ENETC4_PMAC_OFFSET, 3722 .eth_ops = &enetc4_pf_ethtool_ops, 3723 }; 3724 3725 static const struct enetc_drvdata enetc4_ppm_data = { 3726 .sysclk_freq = ENETC_CLK_333M, 3727 .tx_csum = true, 3728 .max_frags = ENETC4_MAX_SKB_FRAGS, 3729 .eth_ops = &enetc4_ppm_ethtool_ops, 3730 }; 3731 3732 static const struct enetc_drvdata enetc_vf_data = { 3733 .sysclk_freq = ENETC_CLK_400M, 3734 .max_frags = ENETC_MAX_SKB_FRAGS, 3735 .eth_ops = &enetc_vf_ethtool_ops, 3736 }; 3737 3738 static const struct enetc_platform_info enetc_info[] = { 3739 { .revision = ENETC_REV_1_0, 3740 .dev_id = ENETC_DEV_ID_PF, 3741 .data = &enetc_pf_data, 3742 }, 3743 { .revision = ENETC_REV_4_1, 3744 .dev_id = NXP_ENETC_PF_DEV_ID, 3745 .data = &enetc4_pf_data, 3746 }, 3747 { .revision = ENETC_REV_1_0, 3748 .dev_id = ENETC_DEV_ID_VF, 3749 .data = &enetc_vf_data, 3750 }, 3751 { 3752 .revision = ENETC_REV_4_3, 3753 .dev_id = NXP_ENETC_PPM_DEV_ID, 3754 .data = &enetc4_ppm_data, 3755 }, 3756 { .revision = ENETC_REV_4_3, 3757 .dev_id = NXP_ENETC_PF_DEV_ID, 3758 .data = &enetc4_pf_data, 3759 }, 3760 }; 3761 3762 int enetc_get_driver_data(struct enetc_si *si) 3763 { 3764 u16 dev_id = si->pdev->device; 3765 int i; 3766 3767 for (i = 0; i < ARRAY_SIZE(enetc_info); i++) { 3768 if (si->revision == enetc_info[i].revision && 3769 dev_id == enetc_info[i].dev_id) { 3770 si->drvdata = enetc_info[i].data; 3771 3772 return 0; 3773 } 3774 } 3775 3776 return -ERANGE; 3777 } 3778 EXPORT_SYMBOL_GPL(enetc_get_driver_data); 3779 3780 MODULE_DESCRIPTION("NXP ENETC Ethernet driver"); 3781 MODULE_LICENSE("Dual BSD/GPL"); 3782