1 // SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) 2 /* Copyright 2017-2019 NXP */ 3 4 #include "enetc.h" 5 #include <linux/bpf_trace.h> 6 #include <linux/clk.h> 7 #include <linux/tcp.h> 8 #include <linux/udp.h> 9 #include <linux/vmalloc.h> 10 #include <linux/ptp_classify.h> 11 #include <net/ip6_checksum.h> 12 #include <net/pkt_sched.h> 13 #include <net/tso.h> 14 15 u32 enetc_port_mac_rd(struct enetc_si *si, u32 reg) 16 { 17 return enetc_port_rd(&si->hw, reg); 18 } 19 EXPORT_SYMBOL_GPL(enetc_port_mac_rd); 20 21 void enetc_port_mac_wr(struct enetc_si *si, u32 reg, u32 val) 22 { 23 enetc_port_wr(&si->hw, reg, val); 24 if (si->hw_features & ENETC_SI_F_QBU) 25 enetc_port_wr(&si->hw, reg + si->drvdata->pmac_offset, val); 26 } 27 EXPORT_SYMBOL_GPL(enetc_port_mac_wr); 28 29 static void enetc_change_preemptible_tcs(struct enetc_ndev_priv *priv, 30 u8 preemptible_tcs) 31 { 32 if (!(priv->si->hw_features & ENETC_SI_F_QBU)) 33 return; 34 35 priv->preemptible_tcs = preemptible_tcs; 36 enetc_mm_commit_preemptible_tcs(priv); 37 } 38 39 static int enetc_num_stack_tx_queues(struct enetc_ndev_priv *priv) 40 { 41 int num_tx_rings = priv->num_tx_rings; 42 43 if (priv->xdp_prog) 44 return num_tx_rings - num_possible_cpus(); 45 46 return num_tx_rings; 47 } 48 49 static struct enetc_bdr *enetc_rx_ring_from_xdp_tx_ring(struct enetc_ndev_priv *priv, 50 struct enetc_bdr *tx_ring) 51 { 52 int index = &priv->tx_ring[tx_ring->index] - priv->xdp_tx_ring; 53 54 return priv->rx_ring[index]; 55 } 56 57 static struct sk_buff *enetc_tx_swbd_get_skb(struct enetc_tx_swbd *tx_swbd) 58 { 59 if (tx_swbd->is_xdp_tx || tx_swbd->is_xdp_redirect) 60 return NULL; 61 62 return tx_swbd->skb; 63 } 64 65 static struct xdp_frame * 66 enetc_tx_swbd_get_xdp_frame(struct enetc_tx_swbd *tx_swbd) 67 { 68 if (tx_swbd->is_xdp_redirect) 69 return tx_swbd->xdp_frame; 70 71 return NULL; 72 } 73 74 static void enetc_unmap_tx_buff(struct enetc_bdr *tx_ring, 75 struct enetc_tx_swbd *tx_swbd) 76 { 77 /* For XDP_TX, pages come from RX, whereas for the other contexts where 78 * we have is_dma_page_set, those come from skb_frag_dma_map. We need 79 * to match the DMA mapping length, so we need to differentiate those. 80 */ 81 if (tx_swbd->is_dma_page) 82 dma_unmap_page(tx_ring->dev, tx_swbd->dma, 83 tx_swbd->is_xdp_tx ? PAGE_SIZE : tx_swbd->len, 84 tx_swbd->dir); 85 else 86 dma_unmap_single(tx_ring->dev, tx_swbd->dma, 87 tx_swbd->len, tx_swbd->dir); 88 tx_swbd->dma = 0; 89 } 90 91 static void enetc_free_tx_frame(struct enetc_bdr *tx_ring, 92 struct enetc_tx_swbd *tx_swbd) 93 { 94 struct xdp_frame *xdp_frame = enetc_tx_swbd_get_xdp_frame(tx_swbd); 95 struct sk_buff *skb = enetc_tx_swbd_get_skb(tx_swbd); 96 97 if (tx_swbd->dma) 98 enetc_unmap_tx_buff(tx_ring, tx_swbd); 99 100 if (xdp_frame) { 101 xdp_return_frame(tx_swbd->xdp_frame); 102 tx_swbd->xdp_frame = NULL; 103 } else if (skb) { 104 dev_kfree_skb_any(skb); 105 tx_swbd->skb = NULL; 106 } 107 } 108 109 /* Let H/W know BD ring has been updated */ 110 static void enetc_update_tx_ring_tail(struct enetc_bdr *tx_ring) 111 { 112 /* includes wmb() */ 113 enetc_wr_reg_hot(tx_ring->tpir, tx_ring->next_to_use); 114 } 115 116 static int enetc_ptp_parse(struct sk_buff *skb, u8 *udp, 117 u8 *msgtype, u8 *twostep, 118 u16 *correction_offset, u16 *body_offset) 119 { 120 unsigned int ptp_class; 121 struct ptp_header *hdr; 122 unsigned int type; 123 u8 *base; 124 125 ptp_class = ptp_classify_raw(skb); 126 if (ptp_class == PTP_CLASS_NONE) 127 return -EINVAL; 128 129 hdr = ptp_parse_header(skb, ptp_class); 130 if (!hdr) 131 return -EINVAL; 132 133 type = ptp_class & PTP_CLASS_PMASK; 134 if (type == PTP_CLASS_IPV4 || type == PTP_CLASS_IPV6) 135 *udp = 1; 136 else 137 *udp = 0; 138 139 *msgtype = ptp_get_msgtype(hdr, ptp_class); 140 *twostep = hdr->flag_field[0] & 0x2; 141 142 base = skb_mac_header(skb); 143 *correction_offset = (u8 *)&hdr->correction - base; 144 *body_offset = (u8 *)hdr + sizeof(struct ptp_header) - base; 145 146 return 0; 147 } 148 149 static bool enetc_tx_csum_offload_check(struct sk_buff *skb) 150 { 151 switch (skb->csum_offset) { 152 case offsetof(struct tcphdr, check): 153 case offsetof(struct udphdr, check): 154 return true; 155 default: 156 return false; 157 } 158 } 159 160 static bool enetc_skb_is_ipv6(struct sk_buff *skb) 161 { 162 return vlan_get_protocol(skb) == htons(ETH_P_IPV6); 163 } 164 165 static bool enetc_skb_is_tcp(struct sk_buff *skb) 166 { 167 return skb->csum_offset == offsetof(struct tcphdr, check); 168 } 169 170 static int enetc_map_tx_buffs(struct enetc_bdr *tx_ring, struct sk_buff *skb) 171 { 172 bool do_vlan, do_onestep_tstamp = false, do_twostep_tstamp = false; 173 struct enetc_ndev_priv *priv = netdev_priv(tx_ring->ndev); 174 struct enetc_hw *hw = &priv->si->hw; 175 struct enetc_tx_swbd *tx_swbd; 176 int len = skb_headlen(skb); 177 union enetc_tx_bd temp_bd; 178 u8 msgtype, twostep, udp; 179 union enetc_tx_bd *txbd; 180 u16 offset1, offset2; 181 int i, count = 0; 182 skb_frag_t *frag; 183 unsigned int f; 184 dma_addr_t dma; 185 u8 flags = 0; 186 187 enetc_clear_tx_bd(&temp_bd); 188 if (skb->ip_summed == CHECKSUM_PARTIAL) { 189 /* Can not support TSD and checksum offload at the same time */ 190 if (priv->active_offloads & ENETC_F_TXCSUM && 191 enetc_tx_csum_offload_check(skb) && !tx_ring->tsd_enable) { 192 temp_bd.l3_aux0 = FIELD_PREP(ENETC_TX_BD_L3_START, 193 skb_network_offset(skb)); 194 temp_bd.l3_aux1 = FIELD_PREP(ENETC_TX_BD_L3_HDR_LEN, 195 skb_network_header_len(skb) / 4); 196 temp_bd.l3_aux1 |= FIELD_PREP(ENETC_TX_BD_L3T, 197 enetc_skb_is_ipv6(skb)); 198 if (enetc_skb_is_tcp(skb)) 199 temp_bd.l4_aux = FIELD_PREP(ENETC_TX_BD_L4T, 200 ENETC_TXBD_L4T_TCP); 201 else 202 temp_bd.l4_aux = FIELD_PREP(ENETC_TX_BD_L4T, 203 ENETC_TXBD_L4T_UDP); 204 flags |= ENETC_TXBD_FLAGS_CSUM_LSO | ENETC_TXBD_FLAGS_L4CS; 205 } else if (skb_checksum_help(skb)) { 206 return 0; 207 } 208 } 209 210 i = tx_ring->next_to_use; 211 txbd = ENETC_TXBD(*tx_ring, i); 212 prefetchw(txbd); 213 214 dma = dma_map_single(tx_ring->dev, skb->data, len, DMA_TO_DEVICE); 215 if (unlikely(dma_mapping_error(tx_ring->dev, dma))) 216 goto dma_err; 217 218 temp_bd.addr = cpu_to_le64(dma); 219 temp_bd.buf_len = cpu_to_le16(len); 220 221 tx_swbd = &tx_ring->tx_swbd[i]; 222 tx_swbd->dma = dma; 223 tx_swbd->len = len; 224 tx_swbd->is_dma_page = 0; 225 tx_swbd->dir = DMA_TO_DEVICE; 226 count++; 227 228 do_vlan = skb_vlan_tag_present(skb); 229 if (skb->cb[0] & ENETC_F_TX_ONESTEP_SYNC_TSTAMP) { 230 if (enetc_ptp_parse(skb, &udp, &msgtype, &twostep, &offset1, 231 &offset2) || 232 msgtype != PTP_MSGTYPE_SYNC || twostep) 233 WARN_ONCE(1, "Bad packet for one-step timestamping\n"); 234 else 235 do_onestep_tstamp = true; 236 } else if (skb->cb[0] & ENETC_F_TX_TSTAMP) { 237 do_twostep_tstamp = true; 238 } 239 240 tx_swbd->do_twostep_tstamp = do_twostep_tstamp; 241 tx_swbd->qbv_en = !!(priv->active_offloads & ENETC_F_QBV); 242 tx_swbd->check_wb = tx_swbd->do_twostep_tstamp || tx_swbd->qbv_en; 243 244 if (do_vlan || do_onestep_tstamp || do_twostep_tstamp) 245 flags |= ENETC_TXBD_FLAGS_EX; 246 247 if (tx_ring->tsd_enable) 248 flags |= ENETC_TXBD_FLAGS_TSE | ENETC_TXBD_FLAGS_TXSTART; 249 250 /* first BD needs frm_len and offload flags set */ 251 temp_bd.frm_len = cpu_to_le16(skb->len); 252 temp_bd.flags = flags; 253 254 if (flags & ENETC_TXBD_FLAGS_TSE) 255 temp_bd.txstart = enetc_txbd_set_tx_start(skb->skb_mstamp_ns, 256 flags); 257 258 if (flags & ENETC_TXBD_FLAGS_EX) { 259 u8 e_flags = 0; 260 *txbd = temp_bd; 261 enetc_clear_tx_bd(&temp_bd); 262 263 /* add extension BD for VLAN and/or timestamping */ 264 flags = 0; 265 tx_swbd++; 266 txbd++; 267 i++; 268 if (unlikely(i == tx_ring->bd_count)) { 269 i = 0; 270 tx_swbd = tx_ring->tx_swbd; 271 txbd = ENETC_TXBD(*tx_ring, 0); 272 } 273 prefetchw(txbd); 274 275 if (do_vlan) { 276 temp_bd.ext.vid = cpu_to_le16(skb_vlan_tag_get(skb)); 277 temp_bd.ext.tpid = 0; /* < C-TAG */ 278 e_flags |= ENETC_TXBD_E_FLAGS_VLAN_INS; 279 } 280 281 if (do_onestep_tstamp) { 282 u32 lo, hi, val; 283 u64 sec, nsec; 284 u8 *data; 285 286 lo = enetc_rd_hot(hw, ENETC_SICTR0); 287 hi = enetc_rd_hot(hw, ENETC_SICTR1); 288 sec = (u64)hi << 32 | lo; 289 nsec = do_div(sec, 1000000000); 290 291 /* Configure extension BD */ 292 temp_bd.ext.tstamp = cpu_to_le32(lo & 0x3fffffff); 293 e_flags |= ENETC_TXBD_E_FLAGS_ONE_STEP_PTP; 294 295 /* Update originTimestamp field of Sync packet 296 * - 48 bits seconds field 297 * - 32 bits nanseconds field 298 */ 299 data = skb_mac_header(skb); 300 *(__be16 *)(data + offset2) = 301 htons((sec >> 32) & 0xffff); 302 *(__be32 *)(data + offset2 + 2) = 303 htonl(sec & 0xffffffff); 304 *(__be32 *)(data + offset2 + 6) = htonl(nsec); 305 306 /* Configure single-step register */ 307 val = ENETC_PM0_SINGLE_STEP_EN; 308 val |= ENETC_SET_SINGLE_STEP_OFFSET(offset1); 309 if (udp) 310 val |= ENETC_PM0_SINGLE_STEP_CH; 311 312 enetc_port_mac_wr(priv->si, ENETC_PM0_SINGLE_STEP, 313 val); 314 } else if (do_twostep_tstamp) { 315 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; 316 e_flags |= ENETC_TXBD_E_FLAGS_TWO_STEP_PTP; 317 } 318 319 temp_bd.ext.e_flags = e_flags; 320 count++; 321 } 322 323 frag = &skb_shinfo(skb)->frags[0]; 324 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++, frag++) { 325 len = skb_frag_size(frag); 326 dma = skb_frag_dma_map(tx_ring->dev, frag, 0, len, 327 DMA_TO_DEVICE); 328 if (dma_mapping_error(tx_ring->dev, dma)) 329 goto dma_err; 330 331 *txbd = temp_bd; 332 enetc_clear_tx_bd(&temp_bd); 333 334 flags = 0; 335 tx_swbd++; 336 txbd++; 337 i++; 338 if (unlikely(i == tx_ring->bd_count)) { 339 i = 0; 340 tx_swbd = tx_ring->tx_swbd; 341 txbd = ENETC_TXBD(*tx_ring, 0); 342 } 343 prefetchw(txbd); 344 345 temp_bd.addr = cpu_to_le64(dma); 346 temp_bd.buf_len = cpu_to_le16(len); 347 348 tx_swbd->dma = dma; 349 tx_swbd->len = len; 350 tx_swbd->is_dma_page = 1; 351 tx_swbd->dir = DMA_TO_DEVICE; 352 count++; 353 } 354 355 /* last BD needs 'F' bit set */ 356 flags |= ENETC_TXBD_FLAGS_F; 357 temp_bd.flags = flags; 358 *txbd = temp_bd; 359 360 tx_ring->tx_swbd[i].is_eof = true; 361 tx_ring->tx_swbd[i].skb = skb; 362 363 enetc_bdr_idx_inc(tx_ring, &i); 364 tx_ring->next_to_use = i; 365 366 skb_tx_timestamp(skb); 367 368 enetc_update_tx_ring_tail(tx_ring); 369 370 return count; 371 372 dma_err: 373 dev_err(tx_ring->dev, "DMA map error"); 374 375 do { 376 tx_swbd = &tx_ring->tx_swbd[i]; 377 enetc_free_tx_frame(tx_ring, tx_swbd); 378 if (i == 0) 379 i = tx_ring->bd_count; 380 i--; 381 } while (count--); 382 383 return 0; 384 } 385 386 static void enetc_map_tx_tso_hdr(struct enetc_bdr *tx_ring, struct sk_buff *skb, 387 struct enetc_tx_swbd *tx_swbd, 388 union enetc_tx_bd *txbd, int *i, int hdr_len, 389 int data_len) 390 { 391 union enetc_tx_bd txbd_tmp; 392 u8 flags = 0, e_flags = 0; 393 dma_addr_t addr; 394 395 enetc_clear_tx_bd(&txbd_tmp); 396 addr = tx_ring->tso_headers_dma + *i * TSO_HEADER_SIZE; 397 398 if (skb_vlan_tag_present(skb)) 399 flags |= ENETC_TXBD_FLAGS_EX; 400 401 txbd_tmp.addr = cpu_to_le64(addr); 402 txbd_tmp.buf_len = cpu_to_le16(hdr_len); 403 404 /* first BD needs frm_len and offload flags set */ 405 txbd_tmp.frm_len = cpu_to_le16(hdr_len + data_len); 406 txbd_tmp.flags = flags; 407 408 /* For the TSO header we do not set the dma address since we do not 409 * want it unmapped when we do cleanup. We still set len so that we 410 * count the bytes sent. 411 */ 412 tx_swbd->len = hdr_len; 413 tx_swbd->do_twostep_tstamp = false; 414 tx_swbd->check_wb = false; 415 416 /* Actually write the header in the BD */ 417 *txbd = txbd_tmp; 418 419 /* Add extension BD for VLAN */ 420 if (flags & ENETC_TXBD_FLAGS_EX) { 421 /* Get the next BD */ 422 enetc_bdr_idx_inc(tx_ring, i); 423 txbd = ENETC_TXBD(*tx_ring, *i); 424 tx_swbd = &tx_ring->tx_swbd[*i]; 425 prefetchw(txbd); 426 427 /* Setup the VLAN fields */ 428 enetc_clear_tx_bd(&txbd_tmp); 429 txbd_tmp.ext.vid = cpu_to_le16(skb_vlan_tag_get(skb)); 430 txbd_tmp.ext.tpid = 0; /* < C-TAG */ 431 e_flags |= ENETC_TXBD_E_FLAGS_VLAN_INS; 432 433 /* Write the BD */ 434 txbd_tmp.ext.e_flags = e_flags; 435 *txbd = txbd_tmp; 436 } 437 } 438 439 static int enetc_map_tx_tso_data(struct enetc_bdr *tx_ring, struct sk_buff *skb, 440 struct enetc_tx_swbd *tx_swbd, 441 union enetc_tx_bd *txbd, char *data, 442 int size, bool last_bd) 443 { 444 union enetc_tx_bd txbd_tmp; 445 dma_addr_t addr; 446 u8 flags = 0; 447 448 enetc_clear_tx_bd(&txbd_tmp); 449 450 addr = dma_map_single(tx_ring->dev, data, size, DMA_TO_DEVICE); 451 if (unlikely(dma_mapping_error(tx_ring->dev, addr))) { 452 netdev_err(tx_ring->ndev, "DMA map error\n"); 453 return -ENOMEM; 454 } 455 456 if (last_bd) { 457 flags |= ENETC_TXBD_FLAGS_F; 458 tx_swbd->is_eof = 1; 459 } 460 461 txbd_tmp.addr = cpu_to_le64(addr); 462 txbd_tmp.buf_len = cpu_to_le16(size); 463 txbd_tmp.flags = flags; 464 465 tx_swbd->dma = addr; 466 tx_swbd->len = size; 467 tx_swbd->dir = DMA_TO_DEVICE; 468 469 *txbd = txbd_tmp; 470 471 return 0; 472 } 473 474 static __wsum enetc_tso_hdr_csum(struct tso_t *tso, struct sk_buff *skb, 475 char *hdr, int hdr_len, int *l4_hdr_len) 476 { 477 char *l4_hdr = hdr + skb_transport_offset(skb); 478 int mac_hdr_len = skb_network_offset(skb); 479 480 if (tso->tlen != sizeof(struct udphdr)) { 481 struct tcphdr *tcph = (struct tcphdr *)(l4_hdr); 482 483 tcph->check = 0; 484 } else { 485 struct udphdr *udph = (struct udphdr *)(l4_hdr); 486 487 udph->check = 0; 488 } 489 490 /* Compute the IP checksum. This is necessary since tso_build_hdr() 491 * already incremented the IP ID field. 492 */ 493 if (!tso->ipv6) { 494 struct iphdr *iph = (void *)(hdr + mac_hdr_len); 495 496 iph->check = 0; 497 iph->check = ip_fast_csum((unsigned char *)iph, iph->ihl); 498 } 499 500 /* Compute the checksum over the L4 header. */ 501 *l4_hdr_len = hdr_len - skb_transport_offset(skb); 502 return csum_partial(l4_hdr, *l4_hdr_len, 0); 503 } 504 505 static void enetc_tso_complete_csum(struct enetc_bdr *tx_ring, struct tso_t *tso, 506 struct sk_buff *skb, char *hdr, int len, 507 __wsum sum) 508 { 509 char *l4_hdr = hdr + skb_transport_offset(skb); 510 __sum16 csum_final; 511 512 /* Complete the L4 checksum by appending the pseudo-header to the 513 * already computed checksum. 514 */ 515 if (!tso->ipv6) 516 csum_final = csum_tcpudp_magic(ip_hdr(skb)->saddr, 517 ip_hdr(skb)->daddr, 518 len, ip_hdr(skb)->protocol, sum); 519 else 520 csum_final = csum_ipv6_magic(&ipv6_hdr(skb)->saddr, 521 &ipv6_hdr(skb)->daddr, 522 len, ipv6_hdr(skb)->nexthdr, sum); 523 524 if (tso->tlen != sizeof(struct udphdr)) { 525 struct tcphdr *tcph = (struct tcphdr *)(l4_hdr); 526 527 tcph->check = csum_final; 528 } else { 529 struct udphdr *udph = (struct udphdr *)(l4_hdr); 530 531 udph->check = csum_final; 532 } 533 } 534 535 static int enetc_lso_count_descs(const struct sk_buff *skb) 536 { 537 /* 4 BDs: 1 BD for LSO header + 1 BD for extended BD + 1 BD 538 * for linear area data but not include LSO header, namely 539 * skb_headlen(skb) - lso_hdr_len (it may be 0, but that's 540 * okay, we only need to consider the worst case). And 1 BD 541 * for gap. 542 */ 543 return skb_shinfo(skb)->nr_frags + 4; 544 } 545 546 static int enetc_lso_get_hdr_len(const struct sk_buff *skb) 547 { 548 int hdr_len, tlen; 549 550 tlen = skb_is_gso_tcp(skb) ? tcp_hdrlen(skb) : sizeof(struct udphdr); 551 hdr_len = skb_transport_offset(skb) + tlen; 552 553 return hdr_len; 554 } 555 556 static void enetc_lso_start(struct sk_buff *skb, struct enetc_lso_t *lso) 557 { 558 lso->lso_seg_size = skb_shinfo(skb)->gso_size; 559 lso->ipv6 = enetc_skb_is_ipv6(skb); 560 lso->tcp = skb_is_gso_tcp(skb); 561 lso->l3_hdr_len = skb_network_header_len(skb); 562 lso->l3_start = skb_network_offset(skb); 563 lso->hdr_len = enetc_lso_get_hdr_len(skb); 564 lso->total_len = skb->len - lso->hdr_len; 565 } 566 567 static void enetc_lso_map_hdr(struct enetc_bdr *tx_ring, struct sk_buff *skb, 568 int *i, struct enetc_lso_t *lso) 569 { 570 union enetc_tx_bd txbd_tmp, *txbd; 571 struct enetc_tx_swbd *tx_swbd; 572 u16 frm_len, frm_len_ext; 573 u8 flags, e_flags = 0; 574 dma_addr_t addr; 575 char *hdr; 576 577 /* Get the first BD of the LSO BDs chain */ 578 txbd = ENETC_TXBD(*tx_ring, *i); 579 tx_swbd = &tx_ring->tx_swbd[*i]; 580 prefetchw(txbd); 581 582 /* Prepare LSO header: MAC + IP + TCP/UDP */ 583 hdr = tx_ring->tso_headers + *i * TSO_HEADER_SIZE; 584 memcpy(hdr, skb->data, lso->hdr_len); 585 addr = tx_ring->tso_headers_dma + *i * TSO_HEADER_SIZE; 586 587 /* {frm_len_ext, frm_len} indicates the total length of 588 * large transmit data unit. frm_len contains the 16 least 589 * significant bits and frm_len_ext contains the 4 most 590 * significant bits. 591 */ 592 frm_len = lso->total_len & 0xffff; 593 frm_len_ext = (lso->total_len >> 16) & 0xf; 594 595 /* Set the flags of the first BD */ 596 flags = ENETC_TXBD_FLAGS_EX | ENETC_TXBD_FLAGS_CSUM_LSO | 597 ENETC_TXBD_FLAGS_LSO | ENETC_TXBD_FLAGS_L4CS; 598 599 enetc_clear_tx_bd(&txbd_tmp); 600 txbd_tmp.addr = cpu_to_le64(addr); 601 txbd_tmp.hdr_len = cpu_to_le16(lso->hdr_len); 602 603 /* first BD needs frm_len and offload flags set */ 604 txbd_tmp.frm_len = cpu_to_le16(frm_len); 605 txbd_tmp.flags = flags; 606 607 txbd_tmp.l3_aux0 = FIELD_PREP(ENETC_TX_BD_L3_START, lso->l3_start); 608 /* l3_hdr_size in 32-bits (4 bytes) */ 609 txbd_tmp.l3_aux1 = FIELD_PREP(ENETC_TX_BD_L3_HDR_LEN, 610 lso->l3_hdr_len / 4); 611 if (lso->ipv6) 612 txbd_tmp.l3_aux1 |= ENETC_TX_BD_L3T; 613 else 614 txbd_tmp.l3_aux0 |= ENETC_TX_BD_IPCS; 615 616 txbd_tmp.l4_aux = FIELD_PREP(ENETC_TX_BD_L4T, lso->tcp ? 617 ENETC_TXBD_L4T_TCP : ENETC_TXBD_L4T_UDP); 618 619 /* For the LSO header we do not set the dma address since 620 * we do not want it unmapped when we do cleanup. We still 621 * set len so that we count the bytes sent. 622 */ 623 tx_swbd->len = lso->hdr_len; 624 tx_swbd->do_twostep_tstamp = false; 625 tx_swbd->check_wb = false; 626 627 /* Actually write the header in the BD */ 628 *txbd = txbd_tmp; 629 630 /* Get the next BD, and the next BD is extended BD */ 631 enetc_bdr_idx_inc(tx_ring, i); 632 txbd = ENETC_TXBD(*tx_ring, *i); 633 tx_swbd = &tx_ring->tx_swbd[*i]; 634 prefetchw(txbd); 635 636 enetc_clear_tx_bd(&txbd_tmp); 637 if (skb_vlan_tag_present(skb)) { 638 /* Setup the VLAN fields */ 639 txbd_tmp.ext.vid = cpu_to_le16(skb_vlan_tag_get(skb)); 640 txbd_tmp.ext.tpid = ENETC_TPID_8021Q; 641 e_flags = ENETC_TXBD_E_FLAGS_VLAN_INS; 642 } 643 644 /* Write the BD */ 645 txbd_tmp.ext.e_flags = e_flags; 646 txbd_tmp.ext.lso_sg_size = cpu_to_le16(lso->lso_seg_size); 647 txbd_tmp.ext.frm_len_ext = cpu_to_le16(frm_len_ext); 648 *txbd = txbd_tmp; 649 } 650 651 static int enetc_lso_map_data(struct enetc_bdr *tx_ring, struct sk_buff *skb, 652 int *i, struct enetc_lso_t *lso, int *count) 653 { 654 union enetc_tx_bd txbd_tmp, *txbd = NULL; 655 struct enetc_tx_swbd *tx_swbd; 656 skb_frag_t *frag; 657 dma_addr_t dma; 658 u8 flags = 0; 659 int len, f; 660 661 len = skb_headlen(skb) - lso->hdr_len; 662 if (len > 0) { 663 dma = dma_map_single(tx_ring->dev, skb->data + lso->hdr_len, 664 len, DMA_TO_DEVICE); 665 if (dma_mapping_error(tx_ring->dev, dma)) 666 return -ENOMEM; 667 668 enetc_bdr_idx_inc(tx_ring, i); 669 txbd = ENETC_TXBD(*tx_ring, *i); 670 tx_swbd = &tx_ring->tx_swbd[*i]; 671 prefetchw(txbd); 672 *count += 1; 673 674 enetc_clear_tx_bd(&txbd_tmp); 675 txbd_tmp.addr = cpu_to_le64(dma); 676 txbd_tmp.buf_len = cpu_to_le16(len); 677 678 tx_swbd->dma = dma; 679 tx_swbd->len = len; 680 tx_swbd->is_dma_page = 0; 681 tx_swbd->dir = DMA_TO_DEVICE; 682 } 683 684 frag = &skb_shinfo(skb)->frags[0]; 685 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++, frag++) { 686 if (txbd) 687 *txbd = txbd_tmp; 688 689 len = skb_frag_size(frag); 690 dma = skb_frag_dma_map(tx_ring->dev, frag); 691 if (dma_mapping_error(tx_ring->dev, dma)) 692 return -ENOMEM; 693 694 /* Get the next BD */ 695 enetc_bdr_idx_inc(tx_ring, i); 696 txbd = ENETC_TXBD(*tx_ring, *i); 697 tx_swbd = &tx_ring->tx_swbd[*i]; 698 prefetchw(txbd); 699 *count += 1; 700 701 enetc_clear_tx_bd(&txbd_tmp); 702 txbd_tmp.addr = cpu_to_le64(dma); 703 txbd_tmp.buf_len = cpu_to_le16(len); 704 705 tx_swbd->dma = dma; 706 tx_swbd->len = len; 707 tx_swbd->is_dma_page = 1; 708 tx_swbd->dir = DMA_TO_DEVICE; 709 } 710 711 /* Last BD needs 'F' bit set */ 712 flags |= ENETC_TXBD_FLAGS_F; 713 txbd_tmp.flags = flags; 714 *txbd = txbd_tmp; 715 716 tx_swbd->is_eof = 1; 717 tx_swbd->skb = skb; 718 719 return 0; 720 } 721 722 static int enetc_lso_hw_offload(struct enetc_bdr *tx_ring, struct sk_buff *skb) 723 { 724 struct enetc_tx_swbd *tx_swbd; 725 struct enetc_lso_t lso = {0}; 726 int err, i, count = 0; 727 728 /* Initialize the LSO handler */ 729 enetc_lso_start(skb, &lso); 730 i = tx_ring->next_to_use; 731 732 enetc_lso_map_hdr(tx_ring, skb, &i, &lso); 733 /* First BD and an extend BD */ 734 count += 2; 735 736 err = enetc_lso_map_data(tx_ring, skb, &i, &lso, &count); 737 if (err) 738 goto dma_err; 739 740 /* Go to the next BD */ 741 enetc_bdr_idx_inc(tx_ring, &i); 742 tx_ring->next_to_use = i; 743 enetc_update_tx_ring_tail(tx_ring); 744 745 return count; 746 747 dma_err: 748 do { 749 tx_swbd = &tx_ring->tx_swbd[i]; 750 enetc_free_tx_frame(tx_ring, tx_swbd); 751 if (i == 0) 752 i = tx_ring->bd_count; 753 i--; 754 } while (--count); 755 756 return 0; 757 } 758 759 static int enetc_map_tx_tso_buffs(struct enetc_bdr *tx_ring, struct sk_buff *skb) 760 { 761 struct enetc_ndev_priv *priv = netdev_priv(tx_ring->ndev); 762 int hdr_len, total_len, data_len; 763 struct enetc_tx_swbd *tx_swbd; 764 union enetc_tx_bd *txbd; 765 struct tso_t tso; 766 __wsum csum, csum2; 767 int count = 0, pos; 768 int err, i, bd_data_num; 769 770 /* Initialize the TSO handler, and prepare the first payload */ 771 hdr_len = tso_start(skb, &tso); 772 total_len = skb->len - hdr_len; 773 i = tx_ring->next_to_use; 774 775 while (total_len > 0) { 776 char *hdr; 777 778 /* Get the BD */ 779 txbd = ENETC_TXBD(*tx_ring, i); 780 tx_swbd = &tx_ring->tx_swbd[i]; 781 prefetchw(txbd); 782 783 /* Determine the length of this packet */ 784 data_len = min_t(int, skb_shinfo(skb)->gso_size, total_len); 785 total_len -= data_len; 786 787 /* prepare packet headers: MAC + IP + TCP */ 788 hdr = tx_ring->tso_headers + i * TSO_HEADER_SIZE; 789 tso_build_hdr(skb, hdr, &tso, data_len, total_len == 0); 790 791 /* compute the csum over the L4 header */ 792 csum = enetc_tso_hdr_csum(&tso, skb, hdr, hdr_len, &pos); 793 enetc_map_tx_tso_hdr(tx_ring, skb, tx_swbd, txbd, &i, hdr_len, data_len); 794 bd_data_num = 0; 795 count++; 796 797 while (data_len > 0) { 798 int size; 799 800 size = min_t(int, tso.size, data_len); 801 802 /* Advance the index in the BDR */ 803 enetc_bdr_idx_inc(tx_ring, &i); 804 txbd = ENETC_TXBD(*tx_ring, i); 805 tx_swbd = &tx_ring->tx_swbd[i]; 806 prefetchw(txbd); 807 808 /* Compute the checksum over this segment of data and 809 * add it to the csum already computed (over the L4 810 * header and possible other data segments). 811 */ 812 csum2 = csum_partial(tso.data, size, 0); 813 csum = csum_block_add(csum, csum2, pos); 814 pos += size; 815 816 err = enetc_map_tx_tso_data(tx_ring, skb, tx_swbd, txbd, 817 tso.data, size, 818 size == data_len); 819 if (err) 820 goto err_map_data; 821 822 data_len -= size; 823 count++; 824 bd_data_num++; 825 tso_build_data(skb, &tso, size); 826 827 if (unlikely(bd_data_num >= priv->max_frags && data_len)) 828 goto err_chained_bd; 829 } 830 831 enetc_tso_complete_csum(tx_ring, &tso, skb, hdr, pos, csum); 832 833 if (total_len == 0) 834 tx_swbd->skb = skb; 835 836 /* Go to the next BD */ 837 enetc_bdr_idx_inc(tx_ring, &i); 838 } 839 840 tx_ring->next_to_use = i; 841 enetc_update_tx_ring_tail(tx_ring); 842 843 return count; 844 845 err_map_data: 846 dev_err(tx_ring->dev, "DMA map error"); 847 848 err_chained_bd: 849 do { 850 tx_swbd = &tx_ring->tx_swbd[i]; 851 enetc_free_tx_frame(tx_ring, tx_swbd); 852 if (i == 0) 853 i = tx_ring->bd_count; 854 i--; 855 } while (count--); 856 857 return 0; 858 } 859 860 static netdev_tx_t enetc_start_xmit(struct sk_buff *skb, 861 struct net_device *ndev) 862 { 863 struct enetc_ndev_priv *priv = netdev_priv(ndev); 864 struct enetc_bdr *tx_ring; 865 int count; 866 867 /* Queue one-step Sync packet if already locked */ 868 if (skb->cb[0] & ENETC_F_TX_ONESTEP_SYNC_TSTAMP) { 869 if (test_and_set_bit_lock(ENETC_TX_ONESTEP_TSTAMP_IN_PROGRESS, 870 &priv->flags)) { 871 skb_queue_tail(&priv->tx_skbs, skb); 872 return NETDEV_TX_OK; 873 } 874 } 875 876 tx_ring = priv->tx_ring[skb->queue_mapping]; 877 878 if (skb_is_gso(skb)) { 879 /* LSO data unit lengths of up to 256KB are supported */ 880 if (priv->active_offloads & ENETC_F_LSO && 881 (skb->len - enetc_lso_get_hdr_len(skb)) <= 882 ENETC_LSO_MAX_DATA_LEN) { 883 if (enetc_bd_unused(tx_ring) < enetc_lso_count_descs(skb)) { 884 netif_stop_subqueue(ndev, tx_ring->index); 885 return NETDEV_TX_BUSY; 886 } 887 888 count = enetc_lso_hw_offload(tx_ring, skb); 889 } else { 890 if (enetc_bd_unused(tx_ring) < tso_count_descs(skb)) { 891 netif_stop_subqueue(ndev, tx_ring->index); 892 return NETDEV_TX_BUSY; 893 } 894 895 enetc_lock_mdio(); 896 count = enetc_map_tx_tso_buffs(tx_ring, skb); 897 enetc_unlock_mdio(); 898 } 899 } else { 900 if (unlikely(skb_shinfo(skb)->nr_frags > priv->max_frags)) 901 if (unlikely(skb_linearize(skb))) 902 goto drop_packet_err; 903 904 count = skb_shinfo(skb)->nr_frags + 1; /* fragments + head */ 905 if (enetc_bd_unused(tx_ring) < ENETC_TXBDS_NEEDED(count)) { 906 netif_stop_subqueue(ndev, tx_ring->index); 907 return NETDEV_TX_BUSY; 908 } 909 910 enetc_lock_mdio(); 911 count = enetc_map_tx_buffs(tx_ring, skb); 912 enetc_unlock_mdio(); 913 } 914 915 if (unlikely(!count)) 916 goto drop_packet_err; 917 918 if (enetc_bd_unused(tx_ring) < ENETC_TXBDS_MAX_NEEDED(priv->max_frags)) 919 netif_stop_subqueue(ndev, tx_ring->index); 920 921 return NETDEV_TX_OK; 922 923 drop_packet_err: 924 dev_kfree_skb_any(skb); 925 return NETDEV_TX_OK; 926 } 927 928 netdev_tx_t enetc_xmit(struct sk_buff *skb, struct net_device *ndev) 929 { 930 struct enetc_ndev_priv *priv = netdev_priv(ndev); 931 u8 udp, msgtype, twostep; 932 u16 offset1, offset2; 933 934 /* Mark tx timestamp type on skb->cb[0] if requires */ 935 if ((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && 936 (priv->active_offloads & ENETC_F_TX_TSTAMP_MASK)) { 937 skb->cb[0] = priv->active_offloads & ENETC_F_TX_TSTAMP_MASK; 938 } else { 939 skb->cb[0] = 0; 940 } 941 942 /* Fall back to two-step timestamp if not one-step Sync packet */ 943 if (skb->cb[0] & ENETC_F_TX_ONESTEP_SYNC_TSTAMP) { 944 if (enetc_ptp_parse(skb, &udp, &msgtype, &twostep, 945 &offset1, &offset2) || 946 msgtype != PTP_MSGTYPE_SYNC || twostep != 0) 947 skb->cb[0] = ENETC_F_TX_TSTAMP; 948 } 949 950 return enetc_start_xmit(skb, ndev); 951 } 952 EXPORT_SYMBOL_GPL(enetc_xmit); 953 954 static irqreturn_t enetc_msix(int irq, void *data) 955 { 956 struct enetc_int_vector *v = data; 957 int i; 958 959 enetc_lock_mdio(); 960 961 /* disable interrupts */ 962 enetc_wr_reg_hot(v->rbier, 0); 963 enetc_wr_reg_hot(v->ricr1, v->rx_ictt); 964 965 for_each_set_bit(i, &v->tx_rings_map, ENETC_MAX_NUM_TXQS) 966 enetc_wr_reg_hot(v->tbier_base + ENETC_BDR_OFF(i), 0); 967 968 enetc_unlock_mdio(); 969 970 napi_schedule(&v->napi); 971 972 return IRQ_HANDLED; 973 } 974 975 static void enetc_rx_dim_work(struct work_struct *w) 976 { 977 struct dim *dim = container_of(w, struct dim, work); 978 struct dim_cq_moder moder = 979 net_dim_get_rx_moderation(dim->mode, dim->profile_ix); 980 struct enetc_int_vector *v = 981 container_of(dim, struct enetc_int_vector, rx_dim); 982 struct enetc_ndev_priv *priv = netdev_priv(v->rx_ring.ndev); 983 984 v->rx_ictt = enetc_usecs_to_cycles(moder.usec, priv->sysclk_freq); 985 dim->state = DIM_START_MEASURE; 986 } 987 988 static void enetc_rx_net_dim(struct enetc_int_vector *v) 989 { 990 struct dim_sample dim_sample = {}; 991 992 v->comp_cnt++; 993 994 if (!v->rx_napi_work) 995 return; 996 997 dim_update_sample(v->comp_cnt, 998 v->rx_ring.stats.packets, 999 v->rx_ring.stats.bytes, 1000 &dim_sample); 1001 net_dim(&v->rx_dim, &dim_sample); 1002 } 1003 1004 static int enetc_bd_ready_count(struct enetc_bdr *tx_ring, int ci) 1005 { 1006 int pi = enetc_rd_reg_hot(tx_ring->tcir) & ENETC_TBCIR_IDX_MASK; 1007 1008 return pi >= ci ? pi - ci : tx_ring->bd_count - ci + pi; 1009 } 1010 1011 static bool enetc_page_reusable(struct page *page) 1012 { 1013 return (!page_is_pfmemalloc(page) && page_ref_count(page) == 1); 1014 } 1015 1016 static void enetc_reuse_page(struct enetc_bdr *rx_ring, 1017 struct enetc_rx_swbd *old) 1018 { 1019 struct enetc_rx_swbd *new; 1020 1021 new = &rx_ring->rx_swbd[rx_ring->next_to_alloc]; 1022 1023 /* next buf that may reuse a page */ 1024 enetc_bdr_idx_inc(rx_ring, &rx_ring->next_to_alloc); 1025 1026 /* copy page reference */ 1027 *new = *old; 1028 } 1029 1030 static void enetc_get_tx_tstamp(struct enetc_hw *hw, union enetc_tx_bd *txbd, 1031 u64 *tstamp) 1032 { 1033 u32 lo, hi, tstamp_lo; 1034 1035 lo = enetc_rd_hot(hw, ENETC_SICTR0); 1036 hi = enetc_rd_hot(hw, ENETC_SICTR1); 1037 tstamp_lo = le32_to_cpu(txbd->wb.tstamp); 1038 if (lo <= tstamp_lo) 1039 hi -= 1; 1040 *tstamp = (u64)hi << 32 | tstamp_lo; 1041 } 1042 1043 static void enetc_tstamp_tx(struct sk_buff *skb, u64 tstamp) 1044 { 1045 struct skb_shared_hwtstamps shhwtstamps; 1046 1047 if (skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS) { 1048 memset(&shhwtstamps, 0, sizeof(shhwtstamps)); 1049 shhwtstamps.hwtstamp = ns_to_ktime(tstamp); 1050 skb_txtime_consumed(skb); 1051 skb_tstamp_tx(skb, &shhwtstamps); 1052 } 1053 } 1054 1055 static void enetc_recycle_xdp_tx_buff(struct enetc_bdr *tx_ring, 1056 struct enetc_tx_swbd *tx_swbd) 1057 { 1058 struct enetc_ndev_priv *priv = netdev_priv(tx_ring->ndev); 1059 struct enetc_rx_swbd rx_swbd = { 1060 .dma = tx_swbd->dma, 1061 .page = tx_swbd->page, 1062 .page_offset = tx_swbd->page_offset, 1063 .dir = tx_swbd->dir, 1064 .len = tx_swbd->len, 1065 }; 1066 struct enetc_bdr *rx_ring; 1067 1068 rx_ring = enetc_rx_ring_from_xdp_tx_ring(priv, tx_ring); 1069 1070 if (likely(enetc_swbd_unused(rx_ring))) { 1071 enetc_reuse_page(rx_ring, &rx_swbd); 1072 1073 /* sync for use by the device */ 1074 dma_sync_single_range_for_device(rx_ring->dev, rx_swbd.dma, 1075 rx_swbd.page_offset, 1076 ENETC_RXB_DMA_SIZE_XDP, 1077 rx_swbd.dir); 1078 1079 rx_ring->stats.recycles++; 1080 } else { 1081 /* RX ring is already full, we need to unmap and free the 1082 * page, since there's nothing useful we can do with it. 1083 */ 1084 rx_ring->stats.recycle_failures++; 1085 1086 dma_unmap_page(rx_ring->dev, rx_swbd.dma, PAGE_SIZE, 1087 rx_swbd.dir); 1088 __free_page(rx_swbd.page); 1089 } 1090 1091 rx_ring->xdp.xdp_tx_in_flight--; 1092 } 1093 1094 static bool enetc_clean_tx_ring(struct enetc_bdr *tx_ring, int napi_budget) 1095 { 1096 int tx_frm_cnt = 0, tx_byte_cnt = 0, tx_win_drop = 0; 1097 struct net_device *ndev = tx_ring->ndev; 1098 struct enetc_ndev_priv *priv = netdev_priv(ndev); 1099 struct enetc_tx_swbd *tx_swbd; 1100 int i, bds_to_clean; 1101 bool do_twostep_tstamp; 1102 u64 tstamp = 0; 1103 1104 i = tx_ring->next_to_clean; 1105 tx_swbd = &tx_ring->tx_swbd[i]; 1106 1107 bds_to_clean = enetc_bd_ready_count(tx_ring, i); 1108 1109 do_twostep_tstamp = false; 1110 1111 while (bds_to_clean && tx_frm_cnt < ENETC_DEFAULT_TX_WORK) { 1112 struct xdp_frame *xdp_frame = enetc_tx_swbd_get_xdp_frame(tx_swbd); 1113 struct sk_buff *skb = enetc_tx_swbd_get_skb(tx_swbd); 1114 bool is_eof = tx_swbd->is_eof; 1115 1116 if (unlikely(tx_swbd->check_wb)) { 1117 union enetc_tx_bd *txbd = ENETC_TXBD(*tx_ring, i); 1118 1119 if (txbd->flags & ENETC_TXBD_FLAGS_W && 1120 tx_swbd->do_twostep_tstamp) { 1121 enetc_get_tx_tstamp(&priv->si->hw, txbd, 1122 &tstamp); 1123 do_twostep_tstamp = true; 1124 } 1125 1126 if (tx_swbd->qbv_en && 1127 txbd->wb.status & ENETC_TXBD_STATS_WIN) 1128 tx_win_drop++; 1129 } 1130 1131 if (tx_swbd->is_xdp_tx) 1132 enetc_recycle_xdp_tx_buff(tx_ring, tx_swbd); 1133 else if (likely(tx_swbd->dma)) 1134 enetc_unmap_tx_buff(tx_ring, tx_swbd); 1135 1136 if (xdp_frame) { 1137 xdp_return_frame(xdp_frame); 1138 } else if (skb) { 1139 if (unlikely(skb->cb[0] & ENETC_F_TX_ONESTEP_SYNC_TSTAMP)) { 1140 /* Start work to release lock for next one-step 1141 * timestamping packet. And send one skb in 1142 * tx_skbs queue if has. 1143 */ 1144 schedule_work(&priv->tx_onestep_tstamp); 1145 } else if (unlikely(do_twostep_tstamp)) { 1146 enetc_tstamp_tx(skb, tstamp); 1147 do_twostep_tstamp = false; 1148 } 1149 napi_consume_skb(skb, napi_budget); 1150 } 1151 1152 tx_byte_cnt += tx_swbd->len; 1153 /* Scrub the swbd here so we don't have to do that 1154 * when we reuse it during xmit 1155 */ 1156 memset(tx_swbd, 0, sizeof(*tx_swbd)); 1157 1158 bds_to_clean--; 1159 tx_swbd++; 1160 i++; 1161 if (unlikely(i == tx_ring->bd_count)) { 1162 i = 0; 1163 tx_swbd = tx_ring->tx_swbd; 1164 } 1165 1166 /* BD iteration loop end */ 1167 if (is_eof) { 1168 tx_frm_cnt++; 1169 /* re-arm interrupt source */ 1170 enetc_wr_reg_hot(tx_ring->idr, BIT(tx_ring->index) | 1171 BIT(16 + tx_ring->index)); 1172 } 1173 1174 if (unlikely(!bds_to_clean)) 1175 bds_to_clean = enetc_bd_ready_count(tx_ring, i); 1176 } 1177 1178 tx_ring->next_to_clean = i; 1179 tx_ring->stats.packets += tx_frm_cnt; 1180 tx_ring->stats.bytes += tx_byte_cnt; 1181 tx_ring->stats.win_drop += tx_win_drop; 1182 1183 if (unlikely(tx_frm_cnt && netif_carrier_ok(ndev) && 1184 __netif_subqueue_stopped(ndev, tx_ring->index) && 1185 !test_bit(ENETC_TX_DOWN, &priv->flags) && 1186 (enetc_bd_unused(tx_ring) >= 1187 ENETC_TXBDS_MAX_NEEDED(priv->max_frags)))) { 1188 netif_wake_subqueue(ndev, tx_ring->index); 1189 } 1190 1191 return tx_frm_cnt != ENETC_DEFAULT_TX_WORK; 1192 } 1193 1194 static bool enetc_new_page(struct enetc_bdr *rx_ring, 1195 struct enetc_rx_swbd *rx_swbd) 1196 { 1197 bool xdp = !!(rx_ring->xdp.prog); 1198 struct page *page; 1199 dma_addr_t addr; 1200 1201 page = dev_alloc_page(); 1202 if (unlikely(!page)) 1203 return false; 1204 1205 /* For XDP_TX, we forgo dma_unmap -> dma_map */ 1206 rx_swbd->dir = xdp ? DMA_BIDIRECTIONAL : DMA_FROM_DEVICE; 1207 1208 addr = dma_map_page(rx_ring->dev, page, 0, PAGE_SIZE, rx_swbd->dir); 1209 if (unlikely(dma_mapping_error(rx_ring->dev, addr))) { 1210 __free_page(page); 1211 1212 return false; 1213 } 1214 1215 rx_swbd->dma = addr; 1216 rx_swbd->page = page; 1217 rx_swbd->page_offset = rx_ring->buffer_offset; 1218 1219 return true; 1220 } 1221 1222 static int enetc_refill_rx_ring(struct enetc_bdr *rx_ring, const int buff_cnt) 1223 { 1224 struct enetc_rx_swbd *rx_swbd; 1225 union enetc_rx_bd *rxbd; 1226 int i, j; 1227 1228 i = rx_ring->next_to_use; 1229 rx_swbd = &rx_ring->rx_swbd[i]; 1230 rxbd = enetc_rxbd(rx_ring, i); 1231 1232 for (j = 0; j < buff_cnt; j++) { 1233 /* try reuse page */ 1234 if (unlikely(!rx_swbd->page)) { 1235 if (unlikely(!enetc_new_page(rx_ring, rx_swbd))) { 1236 rx_ring->stats.rx_alloc_errs++; 1237 break; 1238 } 1239 } 1240 1241 /* update RxBD */ 1242 rxbd->w.addr = cpu_to_le64(rx_swbd->dma + 1243 rx_swbd->page_offset); 1244 /* clear 'R" as well */ 1245 rxbd->r.lstatus = 0; 1246 1247 enetc_rxbd_next(rx_ring, &rxbd, &i); 1248 rx_swbd = &rx_ring->rx_swbd[i]; 1249 } 1250 1251 if (likely(j)) { 1252 rx_ring->next_to_alloc = i; /* keep track from page reuse */ 1253 rx_ring->next_to_use = i; 1254 1255 /* update ENETC's consumer index */ 1256 enetc_wr_reg_hot(rx_ring->rcir, rx_ring->next_to_use); 1257 } 1258 1259 return j; 1260 } 1261 1262 static void enetc_get_rx_tstamp(struct net_device *ndev, 1263 union enetc_rx_bd *rxbd, 1264 struct sk_buff *skb) 1265 { 1266 struct skb_shared_hwtstamps *shhwtstamps = skb_hwtstamps(skb); 1267 struct enetc_ndev_priv *priv = netdev_priv(ndev); 1268 struct enetc_hw *hw = &priv->si->hw; 1269 u32 lo, hi, tstamp_lo; 1270 u64 tstamp; 1271 1272 if (le16_to_cpu(rxbd->r.flags) & ENETC_RXBD_FLAG_TSTMP) { 1273 lo = enetc_rd_reg_hot(hw->reg + ENETC_SICTR0); 1274 hi = enetc_rd_reg_hot(hw->reg + ENETC_SICTR1); 1275 rxbd = enetc_rxbd_ext(rxbd); 1276 tstamp_lo = le32_to_cpu(rxbd->ext.tstamp); 1277 if (lo <= tstamp_lo) 1278 hi -= 1; 1279 1280 tstamp = (u64)hi << 32 | tstamp_lo; 1281 memset(shhwtstamps, 0, sizeof(*shhwtstamps)); 1282 shhwtstamps->hwtstamp = ns_to_ktime(tstamp); 1283 } 1284 } 1285 1286 static void enetc_get_offloads(struct enetc_bdr *rx_ring, 1287 union enetc_rx_bd *rxbd, struct sk_buff *skb) 1288 { 1289 struct enetc_ndev_priv *priv = netdev_priv(rx_ring->ndev); 1290 1291 /* TODO: hashing */ 1292 if (rx_ring->ndev->features & NETIF_F_RXCSUM) { 1293 u16 inet_csum = le16_to_cpu(rxbd->r.inet_csum); 1294 1295 skb->csum = csum_unfold((__force __sum16)~htons(inet_csum)); 1296 skb->ip_summed = CHECKSUM_COMPLETE; 1297 } 1298 1299 if (le16_to_cpu(rxbd->r.flags) & ENETC_RXBD_FLAG_VLAN) { 1300 __be16 tpid = 0; 1301 1302 switch (le16_to_cpu(rxbd->r.flags) & ENETC_RXBD_FLAG_TPID) { 1303 case 0: 1304 tpid = htons(ETH_P_8021Q); 1305 break; 1306 case 1: 1307 tpid = htons(ETH_P_8021AD); 1308 break; 1309 case 2: 1310 tpid = htons(enetc_port_rd(&priv->si->hw, 1311 ENETC_PCVLANR1)); 1312 break; 1313 case 3: 1314 tpid = htons(enetc_port_rd(&priv->si->hw, 1315 ENETC_PCVLANR2)); 1316 break; 1317 default: 1318 break; 1319 } 1320 1321 __vlan_hwaccel_put_tag(skb, tpid, le16_to_cpu(rxbd->r.vlan_opt)); 1322 } 1323 1324 if (IS_ENABLED(CONFIG_FSL_ENETC_PTP_CLOCK) && 1325 (priv->active_offloads & ENETC_F_RX_TSTAMP)) 1326 enetc_get_rx_tstamp(rx_ring->ndev, rxbd, skb); 1327 } 1328 1329 /* This gets called during the non-XDP NAPI poll cycle as well as on XDP_PASS, 1330 * so it needs to work with both DMA_FROM_DEVICE as well as DMA_BIDIRECTIONAL 1331 * mapped buffers. 1332 */ 1333 static struct enetc_rx_swbd *enetc_get_rx_buff(struct enetc_bdr *rx_ring, 1334 int i, u16 size) 1335 { 1336 struct enetc_rx_swbd *rx_swbd = &rx_ring->rx_swbd[i]; 1337 1338 dma_sync_single_range_for_cpu(rx_ring->dev, rx_swbd->dma, 1339 rx_swbd->page_offset, 1340 size, rx_swbd->dir); 1341 return rx_swbd; 1342 } 1343 1344 /* Reuse the current page without performing half-page buffer flipping */ 1345 static void enetc_put_rx_buff(struct enetc_bdr *rx_ring, 1346 struct enetc_rx_swbd *rx_swbd) 1347 { 1348 size_t buffer_size = ENETC_RXB_TRUESIZE - rx_ring->buffer_offset; 1349 1350 enetc_reuse_page(rx_ring, rx_swbd); 1351 1352 dma_sync_single_range_for_device(rx_ring->dev, rx_swbd->dma, 1353 rx_swbd->page_offset, 1354 buffer_size, rx_swbd->dir); 1355 1356 rx_swbd->page = NULL; 1357 } 1358 1359 /* Reuse the current page by performing half-page buffer flipping */ 1360 static void enetc_flip_rx_buff(struct enetc_bdr *rx_ring, 1361 struct enetc_rx_swbd *rx_swbd) 1362 { 1363 if (likely(enetc_page_reusable(rx_swbd->page))) { 1364 rx_swbd->page_offset ^= ENETC_RXB_TRUESIZE; 1365 page_ref_inc(rx_swbd->page); 1366 1367 enetc_put_rx_buff(rx_ring, rx_swbd); 1368 } else { 1369 dma_unmap_page(rx_ring->dev, rx_swbd->dma, PAGE_SIZE, 1370 rx_swbd->dir); 1371 rx_swbd->page = NULL; 1372 } 1373 } 1374 1375 static struct sk_buff *enetc_map_rx_buff_to_skb(struct enetc_bdr *rx_ring, 1376 int i, u16 size) 1377 { 1378 struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size); 1379 struct sk_buff *skb; 1380 void *ba; 1381 1382 ba = page_address(rx_swbd->page) + rx_swbd->page_offset; 1383 skb = build_skb(ba - rx_ring->buffer_offset, ENETC_RXB_TRUESIZE); 1384 if (unlikely(!skb)) { 1385 rx_ring->stats.rx_alloc_errs++; 1386 return NULL; 1387 } 1388 1389 skb_reserve(skb, rx_ring->buffer_offset); 1390 __skb_put(skb, size); 1391 1392 enetc_flip_rx_buff(rx_ring, rx_swbd); 1393 1394 return skb; 1395 } 1396 1397 static void enetc_add_rx_buff_to_skb(struct enetc_bdr *rx_ring, int i, 1398 u16 size, struct sk_buff *skb) 1399 { 1400 struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size); 1401 1402 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_swbd->page, 1403 rx_swbd->page_offset, size, ENETC_RXB_TRUESIZE); 1404 1405 enetc_flip_rx_buff(rx_ring, rx_swbd); 1406 } 1407 1408 static bool enetc_check_bd_errors_and_consume(struct enetc_bdr *rx_ring, 1409 u32 bd_status, 1410 union enetc_rx_bd **rxbd, int *i) 1411 { 1412 if (likely(!(bd_status & ENETC_RXBD_LSTATUS(ENETC_RXBD_ERR_MASK)))) 1413 return false; 1414 1415 enetc_put_rx_buff(rx_ring, &rx_ring->rx_swbd[*i]); 1416 enetc_rxbd_next(rx_ring, rxbd, i); 1417 1418 while (!(bd_status & ENETC_RXBD_LSTATUS_F)) { 1419 dma_rmb(); 1420 bd_status = le32_to_cpu((*rxbd)->r.lstatus); 1421 1422 enetc_put_rx_buff(rx_ring, &rx_ring->rx_swbd[*i]); 1423 enetc_rxbd_next(rx_ring, rxbd, i); 1424 } 1425 1426 rx_ring->ndev->stats.rx_dropped++; 1427 rx_ring->ndev->stats.rx_errors++; 1428 1429 return true; 1430 } 1431 1432 static struct sk_buff *enetc_build_skb(struct enetc_bdr *rx_ring, 1433 u32 bd_status, union enetc_rx_bd **rxbd, 1434 int *i, int *cleaned_cnt, int buffer_size) 1435 { 1436 struct sk_buff *skb; 1437 u16 size; 1438 1439 size = le16_to_cpu((*rxbd)->r.buf_len); 1440 skb = enetc_map_rx_buff_to_skb(rx_ring, *i, size); 1441 if (!skb) 1442 return NULL; 1443 1444 enetc_get_offloads(rx_ring, *rxbd, skb); 1445 1446 (*cleaned_cnt)++; 1447 1448 enetc_rxbd_next(rx_ring, rxbd, i); 1449 1450 /* not last BD in frame? */ 1451 while (!(bd_status & ENETC_RXBD_LSTATUS_F)) { 1452 bd_status = le32_to_cpu((*rxbd)->r.lstatus); 1453 size = buffer_size; 1454 1455 if (bd_status & ENETC_RXBD_LSTATUS_F) { 1456 dma_rmb(); 1457 size = le16_to_cpu((*rxbd)->r.buf_len); 1458 } 1459 1460 enetc_add_rx_buff_to_skb(rx_ring, *i, size, skb); 1461 1462 (*cleaned_cnt)++; 1463 1464 enetc_rxbd_next(rx_ring, rxbd, i); 1465 } 1466 1467 skb_record_rx_queue(skb, rx_ring->index); 1468 skb->protocol = eth_type_trans(skb, rx_ring->ndev); 1469 1470 return skb; 1471 } 1472 1473 #define ENETC_RXBD_BUNDLE 16 /* # of BDs to update at once */ 1474 1475 static int enetc_clean_rx_ring(struct enetc_bdr *rx_ring, 1476 struct napi_struct *napi, int work_limit) 1477 { 1478 int rx_frm_cnt = 0, rx_byte_cnt = 0; 1479 int cleaned_cnt, i; 1480 1481 cleaned_cnt = enetc_bd_unused(rx_ring); 1482 /* next descriptor to process */ 1483 i = rx_ring->next_to_clean; 1484 1485 while (likely(rx_frm_cnt < work_limit)) { 1486 union enetc_rx_bd *rxbd; 1487 struct sk_buff *skb; 1488 u32 bd_status; 1489 1490 if (cleaned_cnt >= ENETC_RXBD_BUNDLE) 1491 cleaned_cnt -= enetc_refill_rx_ring(rx_ring, 1492 cleaned_cnt); 1493 1494 rxbd = enetc_rxbd(rx_ring, i); 1495 bd_status = le32_to_cpu(rxbd->r.lstatus); 1496 if (!bd_status) 1497 break; 1498 1499 enetc_wr_reg_hot(rx_ring->idr, BIT(rx_ring->index)); 1500 dma_rmb(); /* for reading other rxbd fields */ 1501 1502 if (enetc_check_bd_errors_and_consume(rx_ring, bd_status, 1503 &rxbd, &i)) 1504 break; 1505 1506 skb = enetc_build_skb(rx_ring, bd_status, &rxbd, &i, 1507 &cleaned_cnt, ENETC_RXB_DMA_SIZE); 1508 if (!skb) 1509 break; 1510 1511 /* When set, the outer VLAN header is extracted and reported 1512 * in the receive buffer descriptor. So rx_byte_cnt should 1513 * add the length of the extracted VLAN header. 1514 */ 1515 if (bd_status & ENETC_RXBD_FLAG_VLAN) 1516 rx_byte_cnt += VLAN_HLEN; 1517 rx_byte_cnt += skb->len + ETH_HLEN; 1518 rx_frm_cnt++; 1519 1520 napi_gro_receive(napi, skb); 1521 } 1522 1523 rx_ring->next_to_clean = i; 1524 1525 rx_ring->stats.packets += rx_frm_cnt; 1526 rx_ring->stats.bytes += rx_byte_cnt; 1527 1528 return rx_frm_cnt; 1529 } 1530 1531 static void enetc_xdp_map_tx_buff(struct enetc_bdr *tx_ring, int i, 1532 struct enetc_tx_swbd *tx_swbd, 1533 int frm_len) 1534 { 1535 union enetc_tx_bd *txbd = ENETC_TXBD(*tx_ring, i); 1536 1537 prefetchw(txbd); 1538 1539 enetc_clear_tx_bd(txbd); 1540 txbd->addr = cpu_to_le64(tx_swbd->dma + tx_swbd->page_offset); 1541 txbd->buf_len = cpu_to_le16(tx_swbd->len); 1542 txbd->frm_len = cpu_to_le16(frm_len); 1543 1544 memcpy(&tx_ring->tx_swbd[i], tx_swbd, sizeof(*tx_swbd)); 1545 } 1546 1547 /* Puts in the TX ring one XDP frame, mapped as an array of TX software buffer 1548 * descriptors. 1549 */ 1550 static bool enetc_xdp_tx(struct enetc_bdr *tx_ring, 1551 struct enetc_tx_swbd *xdp_tx_arr, int num_tx_swbd) 1552 { 1553 struct enetc_tx_swbd *tmp_tx_swbd = xdp_tx_arr; 1554 int i, k, frm_len = tmp_tx_swbd->len; 1555 1556 if (unlikely(enetc_bd_unused(tx_ring) < ENETC_TXBDS_NEEDED(num_tx_swbd))) 1557 return false; 1558 1559 while (unlikely(!tmp_tx_swbd->is_eof)) { 1560 tmp_tx_swbd++; 1561 frm_len += tmp_tx_swbd->len; 1562 } 1563 1564 i = tx_ring->next_to_use; 1565 1566 for (k = 0; k < num_tx_swbd; k++) { 1567 struct enetc_tx_swbd *xdp_tx_swbd = &xdp_tx_arr[k]; 1568 1569 enetc_xdp_map_tx_buff(tx_ring, i, xdp_tx_swbd, frm_len); 1570 1571 /* last BD needs 'F' bit set */ 1572 if (xdp_tx_swbd->is_eof) { 1573 union enetc_tx_bd *txbd = ENETC_TXBD(*tx_ring, i); 1574 1575 txbd->flags = ENETC_TXBD_FLAGS_F; 1576 } 1577 1578 enetc_bdr_idx_inc(tx_ring, &i); 1579 } 1580 1581 tx_ring->next_to_use = i; 1582 1583 return true; 1584 } 1585 1586 static int enetc_xdp_frame_to_xdp_tx_swbd(struct enetc_bdr *tx_ring, 1587 struct enetc_tx_swbd *xdp_tx_arr, 1588 struct xdp_frame *xdp_frame) 1589 { 1590 struct enetc_tx_swbd *xdp_tx_swbd = &xdp_tx_arr[0]; 1591 struct skb_shared_info *shinfo; 1592 void *data = xdp_frame->data; 1593 int len = xdp_frame->len; 1594 skb_frag_t *frag; 1595 dma_addr_t dma; 1596 unsigned int f; 1597 int n = 0; 1598 1599 dma = dma_map_single(tx_ring->dev, data, len, DMA_TO_DEVICE); 1600 if (unlikely(dma_mapping_error(tx_ring->dev, dma))) { 1601 netdev_err(tx_ring->ndev, "DMA map error\n"); 1602 return -1; 1603 } 1604 1605 xdp_tx_swbd->dma = dma; 1606 xdp_tx_swbd->dir = DMA_TO_DEVICE; 1607 xdp_tx_swbd->len = len; 1608 xdp_tx_swbd->is_xdp_redirect = true; 1609 xdp_tx_swbd->is_eof = false; 1610 xdp_tx_swbd->xdp_frame = NULL; 1611 1612 n++; 1613 1614 if (!xdp_frame_has_frags(xdp_frame)) 1615 goto out; 1616 1617 xdp_tx_swbd = &xdp_tx_arr[n]; 1618 1619 shinfo = xdp_get_shared_info_from_frame(xdp_frame); 1620 1621 for (f = 0, frag = &shinfo->frags[0]; f < shinfo->nr_frags; 1622 f++, frag++) { 1623 data = skb_frag_address(frag); 1624 len = skb_frag_size(frag); 1625 1626 dma = dma_map_single(tx_ring->dev, data, len, DMA_TO_DEVICE); 1627 if (unlikely(dma_mapping_error(tx_ring->dev, dma))) { 1628 /* Undo the DMA mapping for all fragments */ 1629 while (--n >= 0) 1630 enetc_unmap_tx_buff(tx_ring, &xdp_tx_arr[n]); 1631 1632 netdev_err(tx_ring->ndev, "DMA map error\n"); 1633 return -1; 1634 } 1635 1636 xdp_tx_swbd->dma = dma; 1637 xdp_tx_swbd->dir = DMA_TO_DEVICE; 1638 xdp_tx_swbd->len = len; 1639 xdp_tx_swbd->is_xdp_redirect = true; 1640 xdp_tx_swbd->is_eof = false; 1641 xdp_tx_swbd->xdp_frame = NULL; 1642 1643 n++; 1644 xdp_tx_swbd = &xdp_tx_arr[n]; 1645 } 1646 out: 1647 xdp_tx_arr[n - 1].is_eof = true; 1648 xdp_tx_arr[n - 1].xdp_frame = xdp_frame; 1649 1650 return n; 1651 } 1652 1653 int enetc_xdp_xmit(struct net_device *ndev, int num_frames, 1654 struct xdp_frame **frames, u32 flags) 1655 { 1656 struct enetc_tx_swbd xdp_redirect_arr[ENETC_MAX_SKB_FRAGS] = {0}; 1657 struct enetc_ndev_priv *priv = netdev_priv(ndev); 1658 struct enetc_bdr *tx_ring; 1659 int xdp_tx_bd_cnt, i, k; 1660 int xdp_tx_frm_cnt = 0; 1661 1662 if (unlikely(test_bit(ENETC_TX_DOWN, &priv->flags))) 1663 return -ENETDOWN; 1664 1665 enetc_lock_mdio(); 1666 1667 tx_ring = priv->xdp_tx_ring[smp_processor_id()]; 1668 1669 prefetchw(ENETC_TXBD(*tx_ring, tx_ring->next_to_use)); 1670 1671 for (k = 0; k < num_frames; k++) { 1672 xdp_tx_bd_cnt = enetc_xdp_frame_to_xdp_tx_swbd(tx_ring, 1673 xdp_redirect_arr, 1674 frames[k]); 1675 if (unlikely(xdp_tx_bd_cnt < 0)) 1676 break; 1677 1678 if (unlikely(!enetc_xdp_tx(tx_ring, xdp_redirect_arr, 1679 xdp_tx_bd_cnt))) { 1680 for (i = 0; i < xdp_tx_bd_cnt; i++) 1681 enetc_unmap_tx_buff(tx_ring, 1682 &xdp_redirect_arr[i]); 1683 tx_ring->stats.xdp_tx_drops++; 1684 break; 1685 } 1686 1687 xdp_tx_frm_cnt++; 1688 } 1689 1690 if (unlikely((flags & XDP_XMIT_FLUSH) || k != xdp_tx_frm_cnt)) 1691 enetc_update_tx_ring_tail(tx_ring); 1692 1693 tx_ring->stats.xdp_tx += xdp_tx_frm_cnt; 1694 1695 enetc_unlock_mdio(); 1696 1697 return xdp_tx_frm_cnt; 1698 } 1699 EXPORT_SYMBOL_GPL(enetc_xdp_xmit); 1700 1701 static void enetc_map_rx_buff_to_xdp(struct enetc_bdr *rx_ring, int i, 1702 struct xdp_buff *xdp_buff, u16 size) 1703 { 1704 struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size); 1705 void *hard_start = page_address(rx_swbd->page) + rx_swbd->page_offset; 1706 1707 /* To be used for XDP_TX */ 1708 rx_swbd->len = size; 1709 1710 xdp_prepare_buff(xdp_buff, hard_start - rx_ring->buffer_offset, 1711 rx_ring->buffer_offset, size, false); 1712 } 1713 1714 static void enetc_add_rx_buff_to_xdp(struct enetc_bdr *rx_ring, int i, 1715 u16 size, struct xdp_buff *xdp_buff) 1716 { 1717 struct skb_shared_info *shinfo = xdp_get_shared_info_from_buff(xdp_buff); 1718 struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size); 1719 skb_frag_t *frag; 1720 1721 /* To be used for XDP_TX */ 1722 rx_swbd->len = size; 1723 1724 if (!xdp_buff_has_frags(xdp_buff)) { 1725 xdp_buff_set_frags_flag(xdp_buff); 1726 shinfo->xdp_frags_size = size; 1727 shinfo->nr_frags = 0; 1728 } else { 1729 shinfo->xdp_frags_size += size; 1730 } 1731 1732 if (page_is_pfmemalloc(rx_swbd->page)) 1733 xdp_buff_set_frag_pfmemalloc(xdp_buff); 1734 1735 frag = &shinfo->frags[shinfo->nr_frags]; 1736 skb_frag_fill_page_desc(frag, rx_swbd->page, rx_swbd->page_offset, 1737 size); 1738 1739 shinfo->nr_frags++; 1740 } 1741 1742 static void enetc_build_xdp_buff(struct enetc_bdr *rx_ring, u32 bd_status, 1743 union enetc_rx_bd **rxbd, int *i, 1744 int *cleaned_cnt, struct xdp_buff *xdp_buff) 1745 { 1746 u16 size = le16_to_cpu((*rxbd)->r.buf_len); 1747 1748 xdp_init_buff(xdp_buff, ENETC_RXB_TRUESIZE, &rx_ring->xdp.rxq); 1749 1750 enetc_map_rx_buff_to_xdp(rx_ring, *i, xdp_buff, size); 1751 (*cleaned_cnt)++; 1752 enetc_rxbd_next(rx_ring, rxbd, i); 1753 1754 /* not last BD in frame? */ 1755 while (!(bd_status & ENETC_RXBD_LSTATUS_F)) { 1756 bd_status = le32_to_cpu((*rxbd)->r.lstatus); 1757 size = ENETC_RXB_DMA_SIZE_XDP; 1758 1759 if (bd_status & ENETC_RXBD_LSTATUS_F) { 1760 dma_rmb(); 1761 size = le16_to_cpu((*rxbd)->r.buf_len); 1762 } 1763 1764 enetc_add_rx_buff_to_xdp(rx_ring, *i, size, xdp_buff); 1765 (*cleaned_cnt)++; 1766 enetc_rxbd_next(rx_ring, rxbd, i); 1767 } 1768 } 1769 1770 /* Convert RX buffer descriptors to TX buffer descriptors. These will be 1771 * recycled back into the RX ring in enetc_clean_tx_ring. 1772 */ 1773 static int enetc_rx_swbd_to_xdp_tx_swbd(struct enetc_tx_swbd *xdp_tx_arr, 1774 struct enetc_bdr *rx_ring, 1775 int rx_ring_first, int rx_ring_last) 1776 { 1777 int n = 0; 1778 1779 for (; rx_ring_first != rx_ring_last; 1780 n++, enetc_bdr_idx_inc(rx_ring, &rx_ring_first)) { 1781 struct enetc_rx_swbd *rx_swbd = &rx_ring->rx_swbd[rx_ring_first]; 1782 struct enetc_tx_swbd *tx_swbd = &xdp_tx_arr[n]; 1783 1784 /* No need to dma_map, we already have DMA_BIDIRECTIONAL */ 1785 tx_swbd->dma = rx_swbd->dma; 1786 tx_swbd->dir = rx_swbd->dir; 1787 tx_swbd->page = rx_swbd->page; 1788 tx_swbd->page_offset = rx_swbd->page_offset; 1789 tx_swbd->len = rx_swbd->len; 1790 tx_swbd->is_dma_page = true; 1791 tx_swbd->is_xdp_tx = true; 1792 tx_swbd->is_eof = false; 1793 } 1794 1795 /* We rely on caller providing an rx_ring_last > rx_ring_first */ 1796 xdp_tx_arr[n - 1].is_eof = true; 1797 1798 return n; 1799 } 1800 1801 static void enetc_xdp_drop(struct enetc_bdr *rx_ring, int rx_ring_first, 1802 int rx_ring_last) 1803 { 1804 while (rx_ring_first != rx_ring_last) { 1805 enetc_put_rx_buff(rx_ring, 1806 &rx_ring->rx_swbd[rx_ring_first]); 1807 enetc_bdr_idx_inc(rx_ring, &rx_ring_first); 1808 } 1809 } 1810 1811 static int enetc_clean_rx_ring_xdp(struct enetc_bdr *rx_ring, 1812 struct napi_struct *napi, int work_limit, 1813 struct bpf_prog *prog) 1814 { 1815 int xdp_tx_bd_cnt, xdp_tx_frm_cnt = 0, xdp_redirect_frm_cnt = 0; 1816 struct enetc_tx_swbd xdp_tx_arr[ENETC_MAX_SKB_FRAGS] = {0}; 1817 struct enetc_ndev_priv *priv = netdev_priv(rx_ring->ndev); 1818 int rx_frm_cnt = 0, rx_byte_cnt = 0; 1819 struct enetc_bdr *tx_ring; 1820 int cleaned_cnt, i; 1821 u32 xdp_act; 1822 1823 cleaned_cnt = enetc_bd_unused(rx_ring); 1824 /* next descriptor to process */ 1825 i = rx_ring->next_to_clean; 1826 1827 while (likely(rx_frm_cnt < work_limit)) { 1828 union enetc_rx_bd *rxbd, *orig_rxbd; 1829 int orig_i, orig_cleaned_cnt; 1830 struct xdp_buff xdp_buff; 1831 struct sk_buff *skb; 1832 u32 bd_status; 1833 int err; 1834 1835 rxbd = enetc_rxbd(rx_ring, i); 1836 bd_status = le32_to_cpu(rxbd->r.lstatus); 1837 if (!bd_status) 1838 break; 1839 1840 enetc_wr_reg_hot(rx_ring->idr, BIT(rx_ring->index)); 1841 dma_rmb(); /* for reading other rxbd fields */ 1842 1843 if (enetc_check_bd_errors_and_consume(rx_ring, bd_status, 1844 &rxbd, &i)) 1845 break; 1846 1847 orig_rxbd = rxbd; 1848 orig_cleaned_cnt = cleaned_cnt; 1849 orig_i = i; 1850 1851 enetc_build_xdp_buff(rx_ring, bd_status, &rxbd, &i, 1852 &cleaned_cnt, &xdp_buff); 1853 1854 /* When set, the outer VLAN header is extracted and reported 1855 * in the receive buffer descriptor. So rx_byte_cnt should 1856 * add the length of the extracted VLAN header. 1857 */ 1858 if (bd_status & ENETC_RXBD_FLAG_VLAN) 1859 rx_byte_cnt += VLAN_HLEN; 1860 rx_byte_cnt += xdp_get_buff_len(&xdp_buff); 1861 1862 xdp_act = bpf_prog_run_xdp(prog, &xdp_buff); 1863 1864 switch (xdp_act) { 1865 default: 1866 bpf_warn_invalid_xdp_action(rx_ring->ndev, prog, xdp_act); 1867 fallthrough; 1868 case XDP_ABORTED: 1869 trace_xdp_exception(rx_ring->ndev, prog, xdp_act); 1870 fallthrough; 1871 case XDP_DROP: 1872 enetc_xdp_drop(rx_ring, orig_i, i); 1873 rx_ring->stats.xdp_drops++; 1874 break; 1875 case XDP_PASS: 1876 rxbd = orig_rxbd; 1877 cleaned_cnt = orig_cleaned_cnt; 1878 i = orig_i; 1879 1880 skb = enetc_build_skb(rx_ring, bd_status, &rxbd, 1881 &i, &cleaned_cnt, 1882 ENETC_RXB_DMA_SIZE_XDP); 1883 if (unlikely(!skb)) 1884 goto out; 1885 1886 napi_gro_receive(napi, skb); 1887 break; 1888 case XDP_TX: 1889 tx_ring = priv->xdp_tx_ring[rx_ring->index]; 1890 if (unlikely(test_bit(ENETC_TX_DOWN, &priv->flags))) { 1891 enetc_xdp_drop(rx_ring, orig_i, i); 1892 tx_ring->stats.xdp_tx_drops++; 1893 break; 1894 } 1895 1896 xdp_tx_bd_cnt = enetc_rx_swbd_to_xdp_tx_swbd(xdp_tx_arr, 1897 rx_ring, 1898 orig_i, i); 1899 1900 if (!enetc_xdp_tx(tx_ring, xdp_tx_arr, xdp_tx_bd_cnt)) { 1901 enetc_xdp_drop(rx_ring, orig_i, i); 1902 tx_ring->stats.xdp_tx_drops++; 1903 } else { 1904 tx_ring->stats.xdp_tx += xdp_tx_bd_cnt; 1905 rx_ring->xdp.xdp_tx_in_flight += xdp_tx_bd_cnt; 1906 xdp_tx_frm_cnt++; 1907 /* The XDP_TX enqueue was successful, so we 1908 * need to scrub the RX software BDs because 1909 * the ownership of the buffers no longer 1910 * belongs to the RX ring, and we must prevent 1911 * enetc_refill_rx_ring() from reusing 1912 * rx_swbd->page. 1913 */ 1914 while (orig_i != i) { 1915 rx_ring->rx_swbd[orig_i].page = NULL; 1916 enetc_bdr_idx_inc(rx_ring, &orig_i); 1917 } 1918 } 1919 break; 1920 case XDP_REDIRECT: 1921 err = xdp_do_redirect(rx_ring->ndev, &xdp_buff, prog); 1922 if (unlikely(err)) { 1923 enetc_xdp_drop(rx_ring, orig_i, i); 1924 rx_ring->stats.xdp_redirect_failures++; 1925 } else { 1926 while (orig_i != i) { 1927 enetc_flip_rx_buff(rx_ring, 1928 &rx_ring->rx_swbd[orig_i]); 1929 enetc_bdr_idx_inc(rx_ring, &orig_i); 1930 } 1931 xdp_redirect_frm_cnt++; 1932 rx_ring->stats.xdp_redirect++; 1933 } 1934 } 1935 1936 rx_frm_cnt++; 1937 } 1938 1939 out: 1940 rx_ring->next_to_clean = i; 1941 1942 rx_ring->stats.packets += rx_frm_cnt; 1943 rx_ring->stats.bytes += rx_byte_cnt; 1944 1945 if (xdp_redirect_frm_cnt) 1946 xdp_do_flush(); 1947 1948 if (xdp_tx_frm_cnt) 1949 enetc_update_tx_ring_tail(tx_ring); 1950 1951 if (cleaned_cnt > rx_ring->xdp.xdp_tx_in_flight) 1952 enetc_refill_rx_ring(rx_ring, enetc_bd_unused(rx_ring) - 1953 rx_ring->xdp.xdp_tx_in_flight); 1954 1955 return rx_frm_cnt; 1956 } 1957 1958 static int enetc_poll(struct napi_struct *napi, int budget) 1959 { 1960 struct enetc_int_vector 1961 *v = container_of(napi, struct enetc_int_vector, napi); 1962 struct enetc_bdr *rx_ring = &v->rx_ring; 1963 struct bpf_prog *prog; 1964 bool complete = true; 1965 int work_done; 1966 int i; 1967 1968 enetc_lock_mdio(); 1969 1970 for (i = 0; i < v->count_tx_rings; i++) 1971 if (!enetc_clean_tx_ring(&v->tx_ring[i], budget)) 1972 complete = false; 1973 1974 prog = rx_ring->xdp.prog; 1975 if (prog) 1976 work_done = enetc_clean_rx_ring_xdp(rx_ring, napi, budget, prog); 1977 else 1978 work_done = enetc_clean_rx_ring(rx_ring, napi, budget); 1979 if (work_done == budget) 1980 complete = false; 1981 if (work_done) 1982 v->rx_napi_work = true; 1983 1984 if (!complete) { 1985 enetc_unlock_mdio(); 1986 return budget; 1987 } 1988 1989 napi_complete_done(napi, work_done); 1990 1991 if (likely(v->rx_dim_en)) 1992 enetc_rx_net_dim(v); 1993 1994 v->rx_napi_work = false; 1995 1996 /* enable interrupts */ 1997 enetc_wr_reg_hot(v->rbier, ENETC_RBIER_RXTIE); 1998 1999 for_each_set_bit(i, &v->tx_rings_map, ENETC_MAX_NUM_TXQS) 2000 enetc_wr_reg_hot(v->tbier_base + ENETC_BDR_OFF(i), 2001 ENETC_TBIER_TXTIE); 2002 2003 enetc_unlock_mdio(); 2004 2005 return work_done; 2006 } 2007 2008 /* Probing and Init */ 2009 #define ENETC_MAX_RFS_SIZE 64 2010 void enetc_get_si_caps(struct enetc_si *si) 2011 { 2012 struct enetc_hw *hw = &si->hw; 2013 u32 val; 2014 2015 /* find out how many of various resources we have to work with */ 2016 val = enetc_rd(hw, ENETC_SICAPR0); 2017 si->num_rx_rings = (val >> 16) & 0xff; 2018 si->num_tx_rings = val & 0xff; 2019 2020 val = enetc_rd(hw, ENETC_SIPCAPR0); 2021 if (val & ENETC_SIPCAPR0_RFS) { 2022 val = enetc_rd(hw, ENETC_SIRFSCAPR); 2023 si->num_fs_entries = ENETC_SIRFSCAPR_GET_NUM_RFS(val); 2024 si->num_fs_entries = min(si->num_fs_entries, ENETC_MAX_RFS_SIZE); 2025 } else { 2026 /* ENETC which not supports RFS */ 2027 si->num_fs_entries = 0; 2028 } 2029 2030 si->num_rss = 0; 2031 val = enetc_rd(hw, ENETC_SIPCAPR0); 2032 if (val & ENETC_SIPCAPR0_RSS) { 2033 u32 rss; 2034 2035 rss = enetc_rd(hw, ENETC_SIRSSCAPR); 2036 si->num_rss = ENETC_SIRSSCAPR_GET_NUM_RSS(rss); 2037 } 2038 2039 if (val & ENETC_SIPCAPR0_LSO) 2040 si->hw_features |= ENETC_SI_F_LSO; 2041 } 2042 EXPORT_SYMBOL_GPL(enetc_get_si_caps); 2043 2044 static int enetc_dma_alloc_bdr(struct enetc_bdr_resource *res) 2045 { 2046 size_t bd_base_size = res->bd_count * res->bd_size; 2047 2048 res->bd_base = dma_alloc_coherent(res->dev, bd_base_size, 2049 &res->bd_dma_base, GFP_KERNEL); 2050 if (!res->bd_base) 2051 return -ENOMEM; 2052 2053 /* h/w requires 128B alignment */ 2054 if (!IS_ALIGNED(res->bd_dma_base, 128)) { 2055 dma_free_coherent(res->dev, bd_base_size, res->bd_base, 2056 res->bd_dma_base); 2057 return -EINVAL; 2058 } 2059 2060 return 0; 2061 } 2062 2063 static void enetc_dma_free_bdr(const struct enetc_bdr_resource *res) 2064 { 2065 size_t bd_base_size = res->bd_count * res->bd_size; 2066 2067 dma_free_coherent(res->dev, bd_base_size, res->bd_base, 2068 res->bd_dma_base); 2069 } 2070 2071 static int enetc_alloc_tx_resource(struct enetc_bdr_resource *res, 2072 struct device *dev, size_t bd_count) 2073 { 2074 int err; 2075 2076 res->dev = dev; 2077 res->bd_count = bd_count; 2078 res->bd_size = sizeof(union enetc_tx_bd); 2079 2080 res->tx_swbd = vcalloc(bd_count, sizeof(*res->tx_swbd)); 2081 if (!res->tx_swbd) 2082 return -ENOMEM; 2083 2084 err = enetc_dma_alloc_bdr(res); 2085 if (err) 2086 goto err_alloc_bdr; 2087 2088 res->tso_headers = dma_alloc_coherent(dev, bd_count * TSO_HEADER_SIZE, 2089 &res->tso_headers_dma, 2090 GFP_KERNEL); 2091 if (!res->tso_headers) { 2092 err = -ENOMEM; 2093 goto err_alloc_tso; 2094 } 2095 2096 return 0; 2097 2098 err_alloc_tso: 2099 enetc_dma_free_bdr(res); 2100 err_alloc_bdr: 2101 vfree(res->tx_swbd); 2102 res->tx_swbd = NULL; 2103 2104 return err; 2105 } 2106 2107 static void enetc_free_tx_resource(const struct enetc_bdr_resource *res) 2108 { 2109 dma_free_coherent(res->dev, res->bd_count * TSO_HEADER_SIZE, 2110 res->tso_headers, res->tso_headers_dma); 2111 enetc_dma_free_bdr(res); 2112 vfree(res->tx_swbd); 2113 } 2114 2115 static struct enetc_bdr_resource * 2116 enetc_alloc_tx_resources(struct enetc_ndev_priv *priv) 2117 { 2118 struct enetc_bdr_resource *tx_res; 2119 int i, err; 2120 2121 tx_res = kcalloc(priv->num_tx_rings, sizeof(*tx_res), GFP_KERNEL); 2122 if (!tx_res) 2123 return ERR_PTR(-ENOMEM); 2124 2125 for (i = 0; i < priv->num_tx_rings; i++) { 2126 struct enetc_bdr *tx_ring = priv->tx_ring[i]; 2127 2128 err = enetc_alloc_tx_resource(&tx_res[i], tx_ring->dev, 2129 tx_ring->bd_count); 2130 if (err) 2131 goto fail; 2132 } 2133 2134 return tx_res; 2135 2136 fail: 2137 while (i-- > 0) 2138 enetc_free_tx_resource(&tx_res[i]); 2139 2140 kfree(tx_res); 2141 2142 return ERR_PTR(err); 2143 } 2144 2145 static void enetc_free_tx_resources(const struct enetc_bdr_resource *tx_res, 2146 size_t num_resources) 2147 { 2148 size_t i; 2149 2150 for (i = 0; i < num_resources; i++) 2151 enetc_free_tx_resource(&tx_res[i]); 2152 2153 kfree(tx_res); 2154 } 2155 2156 static int enetc_alloc_rx_resource(struct enetc_bdr_resource *res, 2157 struct device *dev, size_t bd_count, 2158 bool extended) 2159 { 2160 int err; 2161 2162 res->dev = dev; 2163 res->bd_count = bd_count; 2164 res->bd_size = sizeof(union enetc_rx_bd); 2165 if (extended) 2166 res->bd_size *= 2; 2167 2168 res->rx_swbd = vcalloc(bd_count, sizeof(struct enetc_rx_swbd)); 2169 if (!res->rx_swbd) 2170 return -ENOMEM; 2171 2172 err = enetc_dma_alloc_bdr(res); 2173 if (err) { 2174 vfree(res->rx_swbd); 2175 return err; 2176 } 2177 2178 return 0; 2179 } 2180 2181 static void enetc_free_rx_resource(const struct enetc_bdr_resource *res) 2182 { 2183 enetc_dma_free_bdr(res); 2184 vfree(res->rx_swbd); 2185 } 2186 2187 static struct enetc_bdr_resource * 2188 enetc_alloc_rx_resources(struct enetc_ndev_priv *priv, bool extended) 2189 { 2190 struct enetc_bdr_resource *rx_res; 2191 int i, err; 2192 2193 rx_res = kcalloc(priv->num_rx_rings, sizeof(*rx_res), GFP_KERNEL); 2194 if (!rx_res) 2195 return ERR_PTR(-ENOMEM); 2196 2197 for (i = 0; i < priv->num_rx_rings; i++) { 2198 struct enetc_bdr *rx_ring = priv->rx_ring[i]; 2199 2200 err = enetc_alloc_rx_resource(&rx_res[i], rx_ring->dev, 2201 rx_ring->bd_count, extended); 2202 if (err) 2203 goto fail; 2204 } 2205 2206 return rx_res; 2207 2208 fail: 2209 while (i-- > 0) 2210 enetc_free_rx_resource(&rx_res[i]); 2211 2212 kfree(rx_res); 2213 2214 return ERR_PTR(err); 2215 } 2216 2217 static void enetc_free_rx_resources(const struct enetc_bdr_resource *rx_res, 2218 size_t num_resources) 2219 { 2220 size_t i; 2221 2222 for (i = 0; i < num_resources; i++) 2223 enetc_free_rx_resource(&rx_res[i]); 2224 2225 kfree(rx_res); 2226 } 2227 2228 static void enetc_assign_tx_resource(struct enetc_bdr *tx_ring, 2229 const struct enetc_bdr_resource *res) 2230 { 2231 tx_ring->bd_base = res ? res->bd_base : NULL; 2232 tx_ring->bd_dma_base = res ? res->bd_dma_base : 0; 2233 tx_ring->tx_swbd = res ? res->tx_swbd : NULL; 2234 tx_ring->tso_headers = res ? res->tso_headers : NULL; 2235 tx_ring->tso_headers_dma = res ? res->tso_headers_dma : 0; 2236 } 2237 2238 static void enetc_assign_rx_resource(struct enetc_bdr *rx_ring, 2239 const struct enetc_bdr_resource *res) 2240 { 2241 rx_ring->bd_base = res ? res->bd_base : NULL; 2242 rx_ring->bd_dma_base = res ? res->bd_dma_base : 0; 2243 rx_ring->rx_swbd = res ? res->rx_swbd : NULL; 2244 } 2245 2246 static void enetc_assign_tx_resources(struct enetc_ndev_priv *priv, 2247 const struct enetc_bdr_resource *res) 2248 { 2249 int i; 2250 2251 if (priv->tx_res) 2252 enetc_free_tx_resources(priv->tx_res, priv->num_tx_rings); 2253 2254 for (i = 0; i < priv->num_tx_rings; i++) { 2255 enetc_assign_tx_resource(priv->tx_ring[i], 2256 res ? &res[i] : NULL); 2257 } 2258 2259 priv->tx_res = res; 2260 } 2261 2262 static void enetc_assign_rx_resources(struct enetc_ndev_priv *priv, 2263 const struct enetc_bdr_resource *res) 2264 { 2265 int i; 2266 2267 if (priv->rx_res) 2268 enetc_free_rx_resources(priv->rx_res, priv->num_rx_rings); 2269 2270 for (i = 0; i < priv->num_rx_rings; i++) { 2271 enetc_assign_rx_resource(priv->rx_ring[i], 2272 res ? &res[i] : NULL); 2273 } 2274 2275 priv->rx_res = res; 2276 } 2277 2278 static void enetc_free_tx_ring(struct enetc_bdr *tx_ring) 2279 { 2280 int i; 2281 2282 for (i = 0; i < tx_ring->bd_count; i++) { 2283 struct enetc_tx_swbd *tx_swbd = &tx_ring->tx_swbd[i]; 2284 2285 enetc_free_tx_frame(tx_ring, tx_swbd); 2286 } 2287 } 2288 2289 static void enetc_free_rx_ring(struct enetc_bdr *rx_ring) 2290 { 2291 int i; 2292 2293 for (i = 0; i < rx_ring->bd_count; i++) { 2294 struct enetc_rx_swbd *rx_swbd = &rx_ring->rx_swbd[i]; 2295 2296 if (!rx_swbd->page) 2297 continue; 2298 2299 dma_unmap_page(rx_ring->dev, rx_swbd->dma, PAGE_SIZE, 2300 rx_swbd->dir); 2301 __free_page(rx_swbd->page); 2302 rx_swbd->page = NULL; 2303 } 2304 } 2305 2306 static void enetc_free_rxtx_rings(struct enetc_ndev_priv *priv) 2307 { 2308 int i; 2309 2310 for (i = 0; i < priv->num_rx_rings; i++) 2311 enetc_free_rx_ring(priv->rx_ring[i]); 2312 2313 for (i = 0; i < priv->num_tx_rings; i++) 2314 enetc_free_tx_ring(priv->tx_ring[i]); 2315 } 2316 2317 static int enetc_setup_default_rss_table(struct enetc_si *si, int num_groups) 2318 { 2319 int *rss_table; 2320 int i; 2321 2322 rss_table = kmalloc_array(si->num_rss, sizeof(*rss_table), GFP_KERNEL); 2323 if (!rss_table) 2324 return -ENOMEM; 2325 2326 /* Set up RSS table defaults */ 2327 for (i = 0; i < si->num_rss; i++) 2328 rss_table[i] = i % num_groups; 2329 2330 enetc_set_rss_table(si, rss_table, si->num_rss); 2331 2332 kfree(rss_table); 2333 2334 return 0; 2335 } 2336 2337 static void enetc_set_lso_flags_mask(struct enetc_hw *hw) 2338 { 2339 enetc_wr(hw, ENETC4_SILSOSFMR0, 2340 SILSOSFMR0_VAL_SET(ENETC4_TCP_NL_SEG_FLAGS_DMASK, 2341 ENETC4_TCP_NL_SEG_FLAGS_DMASK)); 2342 enetc_wr(hw, ENETC4_SILSOSFMR1, 0); 2343 } 2344 2345 int enetc_configure_si(struct enetc_ndev_priv *priv) 2346 { 2347 struct enetc_si *si = priv->si; 2348 struct enetc_hw *hw = &si->hw; 2349 int err; 2350 2351 /* set SI cache attributes */ 2352 enetc_wr(hw, ENETC_SICAR0, 2353 ENETC_SICAR_RD_COHERENT | ENETC_SICAR_WR_COHERENT); 2354 enetc_wr(hw, ENETC_SICAR1, ENETC_SICAR_MSI); 2355 /* enable SI */ 2356 enetc_wr(hw, ENETC_SIMR, ENETC_SIMR_EN); 2357 2358 if (si->hw_features & ENETC_SI_F_LSO) 2359 enetc_set_lso_flags_mask(hw); 2360 2361 /* TODO: RSS support for i.MX95 will be supported later, and the 2362 * is_enetc_rev1() condition will be removed 2363 */ 2364 if (si->num_rss && is_enetc_rev1(si)) { 2365 err = enetc_setup_default_rss_table(si, priv->num_rx_rings); 2366 if (err) 2367 return err; 2368 } 2369 2370 return 0; 2371 } 2372 EXPORT_SYMBOL_GPL(enetc_configure_si); 2373 2374 void enetc_init_si_rings_params(struct enetc_ndev_priv *priv) 2375 { 2376 struct enetc_si *si = priv->si; 2377 int cpus = num_online_cpus(); 2378 2379 priv->tx_bd_count = ENETC_TX_RING_DEFAULT_SIZE; 2380 priv->rx_bd_count = ENETC_RX_RING_DEFAULT_SIZE; 2381 2382 /* Enable all available TX rings in order to configure as many 2383 * priorities as possible, when needed. 2384 * TODO: Make # of TX rings run-time configurable 2385 */ 2386 priv->num_rx_rings = min_t(int, cpus, si->num_rx_rings); 2387 priv->num_tx_rings = si->num_tx_rings; 2388 priv->bdr_int_num = priv->num_rx_rings; 2389 priv->ic_mode = ENETC_IC_RX_ADAPTIVE | ENETC_IC_TX_MANUAL; 2390 priv->tx_ictt = enetc_usecs_to_cycles(600, priv->sysclk_freq); 2391 } 2392 EXPORT_SYMBOL_GPL(enetc_init_si_rings_params); 2393 2394 int enetc_alloc_si_resources(struct enetc_ndev_priv *priv) 2395 { 2396 struct enetc_si *si = priv->si; 2397 2398 priv->cls_rules = kcalloc(si->num_fs_entries, sizeof(*priv->cls_rules), 2399 GFP_KERNEL); 2400 if (!priv->cls_rules) 2401 return -ENOMEM; 2402 2403 return 0; 2404 } 2405 EXPORT_SYMBOL_GPL(enetc_alloc_si_resources); 2406 2407 void enetc_free_si_resources(struct enetc_ndev_priv *priv) 2408 { 2409 kfree(priv->cls_rules); 2410 } 2411 EXPORT_SYMBOL_GPL(enetc_free_si_resources); 2412 2413 static void enetc_setup_txbdr(struct enetc_hw *hw, struct enetc_bdr *tx_ring) 2414 { 2415 int idx = tx_ring->index; 2416 u32 tbmr; 2417 2418 enetc_txbdr_wr(hw, idx, ENETC_TBBAR0, 2419 lower_32_bits(tx_ring->bd_dma_base)); 2420 2421 enetc_txbdr_wr(hw, idx, ENETC_TBBAR1, 2422 upper_32_bits(tx_ring->bd_dma_base)); 2423 2424 WARN_ON(!IS_ALIGNED(tx_ring->bd_count, 64)); /* multiple of 64 */ 2425 enetc_txbdr_wr(hw, idx, ENETC_TBLENR, 2426 ENETC_RTBLENR_LEN(tx_ring->bd_count)); 2427 2428 /* clearing PI/CI registers for Tx not supported, adjust sw indexes */ 2429 tx_ring->next_to_use = enetc_txbdr_rd(hw, idx, ENETC_TBPIR); 2430 tx_ring->next_to_clean = enetc_txbdr_rd(hw, idx, ENETC_TBCIR); 2431 2432 /* enable Tx ints by setting pkt thr to 1 */ 2433 enetc_txbdr_wr(hw, idx, ENETC_TBICR0, ENETC_TBICR0_ICEN | 0x1); 2434 2435 tbmr = ENETC_TBMR_SET_PRIO(tx_ring->prio); 2436 if (tx_ring->ndev->features & NETIF_F_HW_VLAN_CTAG_TX) 2437 tbmr |= ENETC_TBMR_VIH; 2438 2439 /* enable ring */ 2440 enetc_txbdr_wr(hw, idx, ENETC_TBMR, tbmr); 2441 2442 tx_ring->tpir = hw->reg + ENETC_BDR(TX, idx, ENETC_TBPIR); 2443 tx_ring->tcir = hw->reg + ENETC_BDR(TX, idx, ENETC_TBCIR); 2444 tx_ring->idr = hw->reg + ENETC_SITXIDR; 2445 } 2446 2447 static void enetc_setup_rxbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring, 2448 bool extended) 2449 { 2450 int idx = rx_ring->index; 2451 u32 rbmr = 0; 2452 2453 enetc_rxbdr_wr(hw, idx, ENETC_RBBAR0, 2454 lower_32_bits(rx_ring->bd_dma_base)); 2455 2456 enetc_rxbdr_wr(hw, idx, ENETC_RBBAR1, 2457 upper_32_bits(rx_ring->bd_dma_base)); 2458 2459 WARN_ON(!IS_ALIGNED(rx_ring->bd_count, 64)); /* multiple of 64 */ 2460 enetc_rxbdr_wr(hw, idx, ENETC_RBLENR, 2461 ENETC_RTBLENR_LEN(rx_ring->bd_count)); 2462 2463 if (rx_ring->xdp.prog) 2464 enetc_rxbdr_wr(hw, idx, ENETC_RBBSR, ENETC_RXB_DMA_SIZE_XDP); 2465 else 2466 enetc_rxbdr_wr(hw, idx, ENETC_RBBSR, ENETC_RXB_DMA_SIZE); 2467 2468 /* Also prepare the consumer index in case page allocation never 2469 * succeeds. In that case, hardware will never advance producer index 2470 * to match consumer index, and will drop all frames. 2471 */ 2472 enetc_rxbdr_wr(hw, idx, ENETC_RBPIR, 0); 2473 enetc_rxbdr_wr(hw, idx, ENETC_RBCIR, 1); 2474 2475 /* enable Rx ints by setting pkt thr to 1 */ 2476 enetc_rxbdr_wr(hw, idx, ENETC_RBICR0, ENETC_RBICR0_ICEN | 0x1); 2477 2478 rx_ring->ext_en = extended; 2479 if (rx_ring->ext_en) 2480 rbmr |= ENETC_RBMR_BDS; 2481 2482 if (rx_ring->ndev->features & NETIF_F_HW_VLAN_CTAG_RX) 2483 rbmr |= ENETC_RBMR_VTE; 2484 2485 rx_ring->rcir = hw->reg + ENETC_BDR(RX, idx, ENETC_RBCIR); 2486 rx_ring->idr = hw->reg + ENETC_SIRXIDR; 2487 2488 rx_ring->next_to_clean = 0; 2489 rx_ring->next_to_use = 0; 2490 rx_ring->next_to_alloc = 0; 2491 2492 enetc_lock_mdio(); 2493 enetc_refill_rx_ring(rx_ring, enetc_bd_unused(rx_ring)); 2494 enetc_unlock_mdio(); 2495 2496 enetc_rxbdr_wr(hw, idx, ENETC_RBMR, rbmr); 2497 } 2498 2499 static void enetc_setup_bdrs(struct enetc_ndev_priv *priv, bool extended) 2500 { 2501 struct enetc_hw *hw = &priv->si->hw; 2502 int i; 2503 2504 for (i = 0; i < priv->num_tx_rings; i++) 2505 enetc_setup_txbdr(hw, priv->tx_ring[i]); 2506 2507 for (i = 0; i < priv->num_rx_rings; i++) 2508 enetc_setup_rxbdr(hw, priv->rx_ring[i], extended); 2509 } 2510 2511 static void enetc_enable_txbdr(struct enetc_hw *hw, struct enetc_bdr *tx_ring) 2512 { 2513 int idx = tx_ring->index; 2514 u32 tbmr; 2515 2516 tbmr = enetc_txbdr_rd(hw, idx, ENETC_TBMR); 2517 tbmr |= ENETC_TBMR_EN; 2518 enetc_txbdr_wr(hw, idx, ENETC_TBMR, tbmr); 2519 } 2520 2521 static void enetc_enable_rxbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring) 2522 { 2523 int idx = rx_ring->index; 2524 u32 rbmr; 2525 2526 rbmr = enetc_rxbdr_rd(hw, idx, ENETC_RBMR); 2527 rbmr |= ENETC_RBMR_EN; 2528 enetc_rxbdr_wr(hw, idx, ENETC_RBMR, rbmr); 2529 } 2530 2531 static void enetc_enable_rx_bdrs(struct enetc_ndev_priv *priv) 2532 { 2533 struct enetc_hw *hw = &priv->si->hw; 2534 int i; 2535 2536 for (i = 0; i < priv->num_rx_rings; i++) 2537 enetc_enable_rxbdr(hw, priv->rx_ring[i]); 2538 } 2539 2540 static void enetc_enable_tx_bdrs(struct enetc_ndev_priv *priv) 2541 { 2542 struct enetc_hw *hw = &priv->si->hw; 2543 int i; 2544 2545 for (i = 0; i < priv->num_tx_rings; i++) 2546 enetc_enable_txbdr(hw, priv->tx_ring[i]); 2547 } 2548 2549 static void enetc_disable_rxbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring) 2550 { 2551 int idx = rx_ring->index; 2552 2553 /* disable EN bit on ring */ 2554 enetc_rxbdr_wr(hw, idx, ENETC_RBMR, 0); 2555 } 2556 2557 static void enetc_disable_txbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring) 2558 { 2559 int idx = rx_ring->index; 2560 2561 /* disable EN bit on ring */ 2562 enetc_txbdr_wr(hw, idx, ENETC_TBMR, 0); 2563 } 2564 2565 static void enetc_disable_rx_bdrs(struct enetc_ndev_priv *priv) 2566 { 2567 struct enetc_hw *hw = &priv->si->hw; 2568 int i; 2569 2570 for (i = 0; i < priv->num_rx_rings; i++) 2571 enetc_disable_rxbdr(hw, priv->rx_ring[i]); 2572 } 2573 2574 static void enetc_disable_tx_bdrs(struct enetc_ndev_priv *priv) 2575 { 2576 struct enetc_hw *hw = &priv->si->hw; 2577 int i; 2578 2579 for (i = 0; i < priv->num_tx_rings; i++) 2580 enetc_disable_txbdr(hw, priv->tx_ring[i]); 2581 } 2582 2583 static void enetc_wait_txbdr(struct enetc_hw *hw, struct enetc_bdr *tx_ring) 2584 { 2585 int delay = 8, timeout = 100; 2586 int idx = tx_ring->index; 2587 2588 /* wait for busy to clear */ 2589 while (delay < timeout && 2590 enetc_txbdr_rd(hw, idx, ENETC_TBSR) & ENETC_TBSR_BUSY) { 2591 msleep(delay); 2592 delay *= 2; 2593 } 2594 2595 if (delay >= timeout) 2596 netdev_warn(tx_ring->ndev, "timeout for tx ring #%d clear\n", 2597 idx); 2598 } 2599 2600 static void enetc_wait_bdrs(struct enetc_ndev_priv *priv) 2601 { 2602 struct enetc_hw *hw = &priv->si->hw; 2603 int i; 2604 2605 for (i = 0; i < priv->num_tx_rings; i++) 2606 enetc_wait_txbdr(hw, priv->tx_ring[i]); 2607 } 2608 2609 static int enetc_setup_irqs(struct enetc_ndev_priv *priv) 2610 { 2611 struct pci_dev *pdev = priv->si->pdev; 2612 struct enetc_hw *hw = &priv->si->hw; 2613 int i, j, err; 2614 2615 for (i = 0; i < priv->bdr_int_num; i++) { 2616 int irq = pci_irq_vector(pdev, ENETC_BDR_INT_BASE_IDX + i); 2617 struct enetc_int_vector *v = priv->int_vector[i]; 2618 int entry = ENETC_BDR_INT_BASE_IDX + i; 2619 2620 snprintf(v->name, sizeof(v->name), "%s-rxtx%d", 2621 priv->ndev->name, i); 2622 err = request_irq(irq, enetc_msix, IRQF_NO_AUTOEN, v->name, v); 2623 if (err) { 2624 dev_err(priv->dev, "request_irq() failed!\n"); 2625 goto irq_err; 2626 } 2627 2628 v->tbier_base = hw->reg + ENETC_BDR(TX, 0, ENETC_TBIER); 2629 v->rbier = hw->reg + ENETC_BDR(RX, i, ENETC_RBIER); 2630 v->ricr1 = hw->reg + ENETC_BDR(RX, i, ENETC_RBICR1); 2631 2632 enetc_wr(hw, ENETC_SIMSIRRV(i), entry); 2633 2634 for (j = 0; j < v->count_tx_rings; j++) { 2635 int idx = v->tx_ring[j].index; 2636 2637 enetc_wr(hw, ENETC_SIMSITRV(idx), entry); 2638 } 2639 irq_set_affinity_hint(irq, get_cpu_mask(i % num_online_cpus())); 2640 } 2641 2642 return 0; 2643 2644 irq_err: 2645 while (i--) { 2646 int irq = pci_irq_vector(pdev, ENETC_BDR_INT_BASE_IDX + i); 2647 2648 irq_set_affinity_hint(irq, NULL); 2649 free_irq(irq, priv->int_vector[i]); 2650 } 2651 2652 return err; 2653 } 2654 2655 static void enetc_free_irqs(struct enetc_ndev_priv *priv) 2656 { 2657 struct pci_dev *pdev = priv->si->pdev; 2658 int i; 2659 2660 for (i = 0; i < priv->bdr_int_num; i++) { 2661 int irq = pci_irq_vector(pdev, ENETC_BDR_INT_BASE_IDX + i); 2662 2663 irq_set_affinity_hint(irq, NULL); 2664 free_irq(irq, priv->int_vector[i]); 2665 } 2666 } 2667 2668 static void enetc_setup_interrupts(struct enetc_ndev_priv *priv) 2669 { 2670 struct enetc_hw *hw = &priv->si->hw; 2671 u32 icpt, ictt; 2672 int i; 2673 2674 /* enable Tx & Rx event indication */ 2675 if (priv->ic_mode & 2676 (ENETC_IC_RX_MANUAL | ENETC_IC_RX_ADAPTIVE)) { 2677 icpt = ENETC_RBICR0_SET_ICPT(ENETC_RXIC_PKTTHR); 2678 /* init to non-0 minimum, will be adjusted later */ 2679 ictt = 0x1; 2680 } else { 2681 icpt = 0x1; /* enable Rx ints by setting pkt thr to 1 */ 2682 ictt = 0; 2683 } 2684 2685 for (i = 0; i < priv->num_rx_rings; i++) { 2686 enetc_rxbdr_wr(hw, i, ENETC_RBICR1, ictt); 2687 enetc_rxbdr_wr(hw, i, ENETC_RBICR0, ENETC_RBICR0_ICEN | icpt); 2688 enetc_rxbdr_wr(hw, i, ENETC_RBIER, ENETC_RBIER_RXTIE); 2689 } 2690 2691 if (priv->ic_mode & ENETC_IC_TX_MANUAL) 2692 icpt = ENETC_TBICR0_SET_ICPT(ENETC_TXIC_PKTTHR); 2693 else 2694 icpt = 0x1; /* enable Tx ints by setting pkt thr to 1 */ 2695 2696 for (i = 0; i < priv->num_tx_rings; i++) { 2697 enetc_txbdr_wr(hw, i, ENETC_TBICR1, priv->tx_ictt); 2698 enetc_txbdr_wr(hw, i, ENETC_TBICR0, ENETC_TBICR0_ICEN | icpt); 2699 enetc_txbdr_wr(hw, i, ENETC_TBIER, ENETC_TBIER_TXTIE); 2700 } 2701 } 2702 2703 static void enetc_clear_interrupts(struct enetc_ndev_priv *priv) 2704 { 2705 struct enetc_hw *hw = &priv->si->hw; 2706 int i; 2707 2708 for (i = 0; i < priv->num_tx_rings; i++) 2709 enetc_txbdr_wr(hw, i, ENETC_TBIER, 0); 2710 2711 for (i = 0; i < priv->num_rx_rings; i++) 2712 enetc_rxbdr_wr(hw, i, ENETC_RBIER, 0); 2713 } 2714 2715 static int enetc_phylink_connect(struct net_device *ndev) 2716 { 2717 struct enetc_ndev_priv *priv = netdev_priv(ndev); 2718 struct ethtool_keee edata; 2719 int err; 2720 2721 if (!priv->phylink) { 2722 /* phy-less mode */ 2723 netif_carrier_on(ndev); 2724 return 0; 2725 } 2726 2727 err = phylink_of_phy_connect(priv->phylink, priv->dev->of_node, 0); 2728 if (err) { 2729 dev_err(&ndev->dev, "could not attach to PHY\n"); 2730 return err; 2731 } 2732 2733 /* disable EEE autoneg, until ENETC driver supports it */ 2734 memset(&edata, 0, sizeof(struct ethtool_keee)); 2735 phylink_ethtool_set_eee(priv->phylink, &edata); 2736 2737 phylink_start(priv->phylink); 2738 2739 return 0; 2740 } 2741 2742 static void enetc_tx_onestep_tstamp(struct work_struct *work) 2743 { 2744 struct enetc_ndev_priv *priv; 2745 struct sk_buff *skb; 2746 2747 priv = container_of(work, struct enetc_ndev_priv, tx_onestep_tstamp); 2748 2749 netif_tx_lock_bh(priv->ndev); 2750 2751 clear_bit_unlock(ENETC_TX_ONESTEP_TSTAMP_IN_PROGRESS, &priv->flags); 2752 skb = skb_dequeue(&priv->tx_skbs); 2753 if (skb) 2754 enetc_start_xmit(skb, priv->ndev); 2755 2756 netif_tx_unlock_bh(priv->ndev); 2757 } 2758 2759 static void enetc_tx_onestep_tstamp_init(struct enetc_ndev_priv *priv) 2760 { 2761 INIT_WORK(&priv->tx_onestep_tstamp, enetc_tx_onestep_tstamp); 2762 skb_queue_head_init(&priv->tx_skbs); 2763 } 2764 2765 void enetc_start(struct net_device *ndev) 2766 { 2767 struct enetc_ndev_priv *priv = netdev_priv(ndev); 2768 int i; 2769 2770 enetc_setup_interrupts(priv); 2771 2772 for (i = 0; i < priv->bdr_int_num; i++) { 2773 int irq = pci_irq_vector(priv->si->pdev, 2774 ENETC_BDR_INT_BASE_IDX + i); 2775 2776 napi_enable(&priv->int_vector[i]->napi); 2777 enable_irq(irq); 2778 } 2779 2780 enetc_enable_tx_bdrs(priv); 2781 2782 enetc_enable_rx_bdrs(priv); 2783 2784 netif_tx_start_all_queues(ndev); 2785 2786 clear_bit(ENETC_TX_DOWN, &priv->flags); 2787 } 2788 EXPORT_SYMBOL_GPL(enetc_start); 2789 2790 int enetc_open(struct net_device *ndev) 2791 { 2792 struct enetc_ndev_priv *priv = netdev_priv(ndev); 2793 struct enetc_bdr_resource *tx_res, *rx_res; 2794 bool extended; 2795 int err; 2796 2797 extended = !!(priv->active_offloads & ENETC_F_RX_TSTAMP); 2798 2799 err = clk_prepare_enable(priv->ref_clk); 2800 if (err) 2801 return err; 2802 2803 err = enetc_setup_irqs(priv); 2804 if (err) 2805 goto err_setup_irqs; 2806 2807 err = enetc_phylink_connect(ndev); 2808 if (err) 2809 goto err_phy_connect; 2810 2811 tx_res = enetc_alloc_tx_resources(priv); 2812 if (IS_ERR(tx_res)) { 2813 err = PTR_ERR(tx_res); 2814 goto err_alloc_tx; 2815 } 2816 2817 rx_res = enetc_alloc_rx_resources(priv, extended); 2818 if (IS_ERR(rx_res)) { 2819 err = PTR_ERR(rx_res); 2820 goto err_alloc_rx; 2821 } 2822 2823 enetc_tx_onestep_tstamp_init(priv); 2824 enetc_assign_tx_resources(priv, tx_res); 2825 enetc_assign_rx_resources(priv, rx_res); 2826 enetc_setup_bdrs(priv, extended); 2827 enetc_start(ndev); 2828 2829 return 0; 2830 2831 err_alloc_rx: 2832 enetc_free_tx_resources(tx_res, priv->num_tx_rings); 2833 err_alloc_tx: 2834 if (priv->phylink) 2835 phylink_disconnect_phy(priv->phylink); 2836 err_phy_connect: 2837 enetc_free_irqs(priv); 2838 err_setup_irqs: 2839 clk_disable_unprepare(priv->ref_clk); 2840 2841 return err; 2842 } 2843 EXPORT_SYMBOL_GPL(enetc_open); 2844 2845 void enetc_stop(struct net_device *ndev) 2846 { 2847 struct enetc_ndev_priv *priv = netdev_priv(ndev); 2848 int i; 2849 2850 set_bit(ENETC_TX_DOWN, &priv->flags); 2851 2852 netif_tx_stop_all_queues(ndev); 2853 2854 enetc_disable_rx_bdrs(priv); 2855 2856 enetc_wait_bdrs(priv); 2857 2858 enetc_disable_tx_bdrs(priv); 2859 2860 for (i = 0; i < priv->bdr_int_num; i++) { 2861 int irq = pci_irq_vector(priv->si->pdev, 2862 ENETC_BDR_INT_BASE_IDX + i); 2863 2864 disable_irq(irq); 2865 napi_synchronize(&priv->int_vector[i]->napi); 2866 napi_disable(&priv->int_vector[i]->napi); 2867 } 2868 2869 enetc_clear_interrupts(priv); 2870 } 2871 EXPORT_SYMBOL_GPL(enetc_stop); 2872 2873 int enetc_close(struct net_device *ndev) 2874 { 2875 struct enetc_ndev_priv *priv = netdev_priv(ndev); 2876 2877 enetc_stop(ndev); 2878 2879 if (priv->phylink) { 2880 phylink_stop(priv->phylink); 2881 phylink_disconnect_phy(priv->phylink); 2882 } else { 2883 netif_carrier_off(ndev); 2884 } 2885 2886 enetc_free_rxtx_rings(priv); 2887 2888 /* Avoids dangling pointers and also frees old resources */ 2889 enetc_assign_rx_resources(priv, NULL); 2890 enetc_assign_tx_resources(priv, NULL); 2891 2892 enetc_free_irqs(priv); 2893 clk_disable_unprepare(priv->ref_clk); 2894 2895 return 0; 2896 } 2897 EXPORT_SYMBOL_GPL(enetc_close); 2898 2899 static int enetc_reconfigure(struct enetc_ndev_priv *priv, bool extended, 2900 int (*cb)(struct enetc_ndev_priv *priv, void *ctx), 2901 void *ctx) 2902 { 2903 struct enetc_bdr_resource *tx_res, *rx_res; 2904 int err; 2905 2906 ASSERT_RTNL(); 2907 2908 /* If the interface is down, run the callback right away, 2909 * without reconfiguration. 2910 */ 2911 if (!netif_running(priv->ndev)) { 2912 if (cb) { 2913 err = cb(priv, ctx); 2914 if (err) 2915 return err; 2916 } 2917 2918 return 0; 2919 } 2920 2921 tx_res = enetc_alloc_tx_resources(priv); 2922 if (IS_ERR(tx_res)) { 2923 err = PTR_ERR(tx_res); 2924 goto out; 2925 } 2926 2927 rx_res = enetc_alloc_rx_resources(priv, extended); 2928 if (IS_ERR(rx_res)) { 2929 err = PTR_ERR(rx_res); 2930 goto out_free_tx_res; 2931 } 2932 2933 enetc_stop(priv->ndev); 2934 enetc_free_rxtx_rings(priv); 2935 2936 /* Interface is down, run optional callback now */ 2937 if (cb) { 2938 err = cb(priv, ctx); 2939 if (err) 2940 goto out_restart; 2941 } 2942 2943 enetc_assign_tx_resources(priv, tx_res); 2944 enetc_assign_rx_resources(priv, rx_res); 2945 enetc_setup_bdrs(priv, extended); 2946 enetc_start(priv->ndev); 2947 2948 return 0; 2949 2950 out_restart: 2951 enetc_setup_bdrs(priv, extended); 2952 enetc_start(priv->ndev); 2953 enetc_free_rx_resources(rx_res, priv->num_rx_rings); 2954 out_free_tx_res: 2955 enetc_free_tx_resources(tx_res, priv->num_tx_rings); 2956 out: 2957 return err; 2958 } 2959 2960 static void enetc_debug_tx_ring_prios(struct enetc_ndev_priv *priv) 2961 { 2962 int i; 2963 2964 for (i = 0; i < priv->num_tx_rings; i++) 2965 netdev_dbg(priv->ndev, "TX ring %d prio %d\n", i, 2966 priv->tx_ring[i]->prio); 2967 } 2968 2969 void enetc_reset_tc_mqprio(struct net_device *ndev) 2970 { 2971 struct enetc_ndev_priv *priv = netdev_priv(ndev); 2972 struct enetc_hw *hw = &priv->si->hw; 2973 struct enetc_bdr *tx_ring; 2974 int num_stack_tx_queues; 2975 int i; 2976 2977 num_stack_tx_queues = enetc_num_stack_tx_queues(priv); 2978 2979 netdev_reset_tc(ndev); 2980 netif_set_real_num_tx_queues(ndev, num_stack_tx_queues); 2981 priv->min_num_stack_tx_queues = num_possible_cpus(); 2982 2983 /* Reset all ring priorities to 0 */ 2984 for (i = 0; i < priv->num_tx_rings; i++) { 2985 tx_ring = priv->tx_ring[i]; 2986 tx_ring->prio = 0; 2987 enetc_set_bdr_prio(hw, tx_ring->index, tx_ring->prio); 2988 } 2989 2990 enetc_debug_tx_ring_prios(priv); 2991 2992 enetc_change_preemptible_tcs(priv, 0); 2993 } 2994 EXPORT_SYMBOL_GPL(enetc_reset_tc_mqprio); 2995 2996 int enetc_setup_tc_mqprio(struct net_device *ndev, void *type_data) 2997 { 2998 struct tc_mqprio_qopt_offload *mqprio = type_data; 2999 struct enetc_ndev_priv *priv = netdev_priv(ndev); 3000 struct tc_mqprio_qopt *qopt = &mqprio->qopt; 3001 struct enetc_hw *hw = &priv->si->hw; 3002 int num_stack_tx_queues = 0; 3003 struct enetc_bdr *tx_ring; 3004 u8 num_tc = qopt->num_tc; 3005 int offset, count; 3006 int err, tc, q; 3007 3008 if (!num_tc) { 3009 enetc_reset_tc_mqprio(ndev); 3010 return 0; 3011 } 3012 3013 err = netdev_set_num_tc(ndev, num_tc); 3014 if (err) 3015 return err; 3016 3017 for (tc = 0; tc < num_tc; tc++) { 3018 offset = qopt->offset[tc]; 3019 count = qopt->count[tc]; 3020 num_stack_tx_queues += count; 3021 3022 err = netdev_set_tc_queue(ndev, tc, count, offset); 3023 if (err) 3024 goto err_reset_tc; 3025 3026 for (q = offset; q < offset + count; q++) { 3027 tx_ring = priv->tx_ring[q]; 3028 /* The prio_tc_map is skb_tx_hash()'s way of selecting 3029 * between TX queues based on skb->priority. As such, 3030 * there's nothing to offload based on it. 3031 * Make the mqprio "traffic class" be the priority of 3032 * this ring group, and leave the Tx IPV to traffic 3033 * class mapping as its default mapping value of 1:1. 3034 */ 3035 tx_ring->prio = tc; 3036 enetc_set_bdr_prio(hw, tx_ring->index, tx_ring->prio); 3037 } 3038 } 3039 3040 err = netif_set_real_num_tx_queues(ndev, num_stack_tx_queues); 3041 if (err) 3042 goto err_reset_tc; 3043 3044 priv->min_num_stack_tx_queues = num_stack_tx_queues; 3045 3046 enetc_debug_tx_ring_prios(priv); 3047 3048 enetc_change_preemptible_tcs(priv, mqprio->preemptible_tcs); 3049 3050 return 0; 3051 3052 err_reset_tc: 3053 enetc_reset_tc_mqprio(ndev); 3054 return err; 3055 } 3056 EXPORT_SYMBOL_GPL(enetc_setup_tc_mqprio); 3057 3058 static int enetc_reconfigure_xdp_cb(struct enetc_ndev_priv *priv, void *ctx) 3059 { 3060 struct bpf_prog *old_prog, *prog = ctx; 3061 int num_stack_tx_queues; 3062 int err, i; 3063 3064 old_prog = xchg(&priv->xdp_prog, prog); 3065 3066 num_stack_tx_queues = enetc_num_stack_tx_queues(priv); 3067 err = netif_set_real_num_tx_queues(priv->ndev, num_stack_tx_queues); 3068 if (err) { 3069 xchg(&priv->xdp_prog, old_prog); 3070 return err; 3071 } 3072 3073 if (old_prog) 3074 bpf_prog_put(old_prog); 3075 3076 for (i = 0; i < priv->num_rx_rings; i++) { 3077 struct enetc_bdr *rx_ring = priv->rx_ring[i]; 3078 3079 rx_ring->xdp.prog = prog; 3080 3081 if (prog) 3082 rx_ring->buffer_offset = XDP_PACKET_HEADROOM; 3083 else 3084 rx_ring->buffer_offset = ENETC_RXB_PAD; 3085 } 3086 3087 return 0; 3088 } 3089 3090 static int enetc_setup_xdp_prog(struct net_device *ndev, struct bpf_prog *prog, 3091 struct netlink_ext_ack *extack) 3092 { 3093 int num_xdp_tx_queues = prog ? num_possible_cpus() : 0; 3094 struct enetc_ndev_priv *priv = netdev_priv(ndev); 3095 bool extended; 3096 3097 if (priv->min_num_stack_tx_queues + num_xdp_tx_queues > 3098 priv->num_tx_rings) { 3099 NL_SET_ERR_MSG_FMT_MOD(extack, 3100 "Reserving %d XDP TXQs leaves under %d for stack (total %d)", 3101 num_xdp_tx_queues, 3102 priv->min_num_stack_tx_queues, 3103 priv->num_tx_rings); 3104 return -EBUSY; 3105 } 3106 3107 extended = !!(priv->active_offloads & ENETC_F_RX_TSTAMP); 3108 3109 /* The buffer layout is changing, so we need to drain the old 3110 * RX buffers and seed new ones. 3111 */ 3112 return enetc_reconfigure(priv, extended, enetc_reconfigure_xdp_cb, prog); 3113 } 3114 3115 int enetc_setup_bpf(struct net_device *ndev, struct netdev_bpf *bpf) 3116 { 3117 switch (bpf->command) { 3118 case XDP_SETUP_PROG: 3119 return enetc_setup_xdp_prog(ndev, bpf->prog, bpf->extack); 3120 default: 3121 return -EINVAL; 3122 } 3123 3124 return 0; 3125 } 3126 EXPORT_SYMBOL_GPL(enetc_setup_bpf); 3127 3128 struct net_device_stats *enetc_get_stats(struct net_device *ndev) 3129 { 3130 struct enetc_ndev_priv *priv = netdev_priv(ndev); 3131 struct net_device_stats *stats = &ndev->stats; 3132 unsigned long packets = 0, bytes = 0; 3133 unsigned long tx_dropped = 0; 3134 int i; 3135 3136 for (i = 0; i < priv->num_rx_rings; i++) { 3137 packets += priv->rx_ring[i]->stats.packets; 3138 bytes += priv->rx_ring[i]->stats.bytes; 3139 } 3140 3141 stats->rx_packets = packets; 3142 stats->rx_bytes = bytes; 3143 bytes = 0; 3144 packets = 0; 3145 3146 for (i = 0; i < priv->num_tx_rings; i++) { 3147 packets += priv->tx_ring[i]->stats.packets; 3148 bytes += priv->tx_ring[i]->stats.bytes; 3149 tx_dropped += priv->tx_ring[i]->stats.win_drop; 3150 } 3151 3152 stats->tx_packets = packets; 3153 stats->tx_bytes = bytes; 3154 stats->tx_dropped = tx_dropped; 3155 3156 return stats; 3157 } 3158 EXPORT_SYMBOL_GPL(enetc_get_stats); 3159 3160 static int enetc_set_rss(struct net_device *ndev, int en) 3161 { 3162 struct enetc_ndev_priv *priv = netdev_priv(ndev); 3163 struct enetc_hw *hw = &priv->si->hw; 3164 u32 reg; 3165 3166 enetc_wr(hw, ENETC_SIRBGCR, priv->num_rx_rings); 3167 3168 reg = enetc_rd(hw, ENETC_SIMR); 3169 reg &= ~ENETC_SIMR_RSSE; 3170 reg |= (en) ? ENETC_SIMR_RSSE : 0; 3171 enetc_wr(hw, ENETC_SIMR, reg); 3172 3173 return 0; 3174 } 3175 3176 static void enetc_enable_rxvlan(struct net_device *ndev, bool en) 3177 { 3178 struct enetc_ndev_priv *priv = netdev_priv(ndev); 3179 struct enetc_hw *hw = &priv->si->hw; 3180 int i; 3181 3182 for (i = 0; i < priv->num_rx_rings; i++) 3183 enetc_bdr_enable_rxvlan(hw, i, en); 3184 } 3185 3186 static void enetc_enable_txvlan(struct net_device *ndev, bool en) 3187 { 3188 struct enetc_ndev_priv *priv = netdev_priv(ndev); 3189 struct enetc_hw *hw = &priv->si->hw; 3190 int i; 3191 3192 for (i = 0; i < priv->num_tx_rings; i++) 3193 enetc_bdr_enable_txvlan(hw, i, en); 3194 } 3195 3196 void enetc_set_features(struct net_device *ndev, netdev_features_t features) 3197 { 3198 netdev_features_t changed = ndev->features ^ features; 3199 3200 if (changed & NETIF_F_RXHASH) 3201 enetc_set_rss(ndev, !!(features & NETIF_F_RXHASH)); 3202 3203 if (changed & NETIF_F_HW_VLAN_CTAG_RX) 3204 enetc_enable_rxvlan(ndev, 3205 !!(features & NETIF_F_HW_VLAN_CTAG_RX)); 3206 3207 if (changed & NETIF_F_HW_VLAN_CTAG_TX) 3208 enetc_enable_txvlan(ndev, 3209 !!(features & NETIF_F_HW_VLAN_CTAG_TX)); 3210 } 3211 EXPORT_SYMBOL_GPL(enetc_set_features); 3212 3213 static int enetc_hwtstamp_set(struct net_device *ndev, struct ifreq *ifr) 3214 { 3215 struct enetc_ndev_priv *priv = netdev_priv(ndev); 3216 int err, new_offloads = priv->active_offloads; 3217 struct hwtstamp_config config; 3218 3219 if (copy_from_user(&config, ifr->ifr_data, sizeof(config))) 3220 return -EFAULT; 3221 3222 switch (config.tx_type) { 3223 case HWTSTAMP_TX_OFF: 3224 new_offloads &= ~ENETC_F_TX_TSTAMP_MASK; 3225 break; 3226 case HWTSTAMP_TX_ON: 3227 new_offloads &= ~ENETC_F_TX_TSTAMP_MASK; 3228 new_offloads |= ENETC_F_TX_TSTAMP; 3229 break; 3230 case HWTSTAMP_TX_ONESTEP_SYNC: 3231 new_offloads &= ~ENETC_F_TX_TSTAMP_MASK; 3232 new_offloads |= ENETC_F_TX_ONESTEP_SYNC_TSTAMP; 3233 break; 3234 default: 3235 return -ERANGE; 3236 } 3237 3238 switch (config.rx_filter) { 3239 case HWTSTAMP_FILTER_NONE: 3240 new_offloads &= ~ENETC_F_RX_TSTAMP; 3241 break; 3242 default: 3243 new_offloads |= ENETC_F_RX_TSTAMP; 3244 config.rx_filter = HWTSTAMP_FILTER_ALL; 3245 } 3246 3247 if ((new_offloads ^ priv->active_offloads) & ENETC_F_RX_TSTAMP) { 3248 bool extended = !!(new_offloads & ENETC_F_RX_TSTAMP); 3249 3250 err = enetc_reconfigure(priv, extended, NULL, NULL); 3251 if (err) 3252 return err; 3253 } 3254 3255 priv->active_offloads = new_offloads; 3256 3257 return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ? 3258 -EFAULT : 0; 3259 } 3260 3261 static int enetc_hwtstamp_get(struct net_device *ndev, struct ifreq *ifr) 3262 { 3263 struct enetc_ndev_priv *priv = netdev_priv(ndev); 3264 struct hwtstamp_config config; 3265 3266 config.flags = 0; 3267 3268 if (priv->active_offloads & ENETC_F_TX_ONESTEP_SYNC_TSTAMP) 3269 config.tx_type = HWTSTAMP_TX_ONESTEP_SYNC; 3270 else if (priv->active_offloads & ENETC_F_TX_TSTAMP) 3271 config.tx_type = HWTSTAMP_TX_ON; 3272 else 3273 config.tx_type = HWTSTAMP_TX_OFF; 3274 3275 config.rx_filter = (priv->active_offloads & ENETC_F_RX_TSTAMP) ? 3276 HWTSTAMP_FILTER_ALL : HWTSTAMP_FILTER_NONE; 3277 3278 return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ? 3279 -EFAULT : 0; 3280 } 3281 3282 int enetc_ioctl(struct net_device *ndev, struct ifreq *rq, int cmd) 3283 { 3284 struct enetc_ndev_priv *priv = netdev_priv(ndev); 3285 3286 if (IS_ENABLED(CONFIG_FSL_ENETC_PTP_CLOCK)) { 3287 if (cmd == SIOCSHWTSTAMP) 3288 return enetc_hwtstamp_set(ndev, rq); 3289 if (cmd == SIOCGHWTSTAMP) 3290 return enetc_hwtstamp_get(ndev, rq); 3291 } 3292 3293 if (!priv->phylink) 3294 return -EOPNOTSUPP; 3295 3296 return phylink_mii_ioctl(priv->phylink, rq, cmd); 3297 } 3298 EXPORT_SYMBOL_GPL(enetc_ioctl); 3299 3300 static int enetc_int_vector_init(struct enetc_ndev_priv *priv, int i, 3301 int v_tx_rings) 3302 { 3303 struct enetc_int_vector *v; 3304 struct enetc_bdr *bdr; 3305 int j, err; 3306 3307 v = kzalloc(struct_size(v, tx_ring, v_tx_rings), GFP_KERNEL); 3308 if (!v) 3309 return -ENOMEM; 3310 3311 priv->int_vector[i] = v; 3312 bdr = &v->rx_ring; 3313 bdr->index = i; 3314 bdr->ndev = priv->ndev; 3315 bdr->dev = priv->dev; 3316 bdr->bd_count = priv->rx_bd_count; 3317 bdr->buffer_offset = ENETC_RXB_PAD; 3318 priv->rx_ring[i] = bdr; 3319 3320 err = xdp_rxq_info_reg(&bdr->xdp.rxq, priv->ndev, i, 0); 3321 if (err) 3322 goto free_vector; 3323 3324 err = xdp_rxq_info_reg_mem_model(&bdr->xdp.rxq, MEM_TYPE_PAGE_SHARED, 3325 NULL); 3326 if (err) { 3327 xdp_rxq_info_unreg(&bdr->xdp.rxq); 3328 goto free_vector; 3329 } 3330 3331 /* init defaults for adaptive IC */ 3332 if (priv->ic_mode & ENETC_IC_RX_ADAPTIVE) { 3333 v->rx_ictt = 0x1; 3334 v->rx_dim_en = true; 3335 } 3336 3337 INIT_WORK(&v->rx_dim.work, enetc_rx_dim_work); 3338 netif_napi_add(priv->ndev, &v->napi, enetc_poll); 3339 v->count_tx_rings = v_tx_rings; 3340 3341 for (j = 0; j < v_tx_rings; j++) { 3342 int idx; 3343 3344 /* default tx ring mapping policy */ 3345 idx = priv->bdr_int_num * j + i; 3346 __set_bit(idx, &v->tx_rings_map); 3347 bdr = &v->tx_ring[j]; 3348 bdr->index = idx; 3349 bdr->ndev = priv->ndev; 3350 bdr->dev = priv->dev; 3351 bdr->bd_count = priv->tx_bd_count; 3352 priv->tx_ring[idx] = bdr; 3353 } 3354 3355 return 0; 3356 3357 free_vector: 3358 priv->rx_ring[i] = NULL; 3359 priv->int_vector[i] = NULL; 3360 kfree(v); 3361 3362 return err; 3363 } 3364 3365 static void enetc_int_vector_destroy(struct enetc_ndev_priv *priv, int i) 3366 { 3367 struct enetc_int_vector *v = priv->int_vector[i]; 3368 struct enetc_bdr *rx_ring = &v->rx_ring; 3369 int j, tx_ring_index; 3370 3371 xdp_rxq_info_unreg_mem_model(&rx_ring->xdp.rxq); 3372 xdp_rxq_info_unreg(&rx_ring->xdp.rxq); 3373 netif_napi_del(&v->napi); 3374 cancel_work_sync(&v->rx_dim.work); 3375 3376 for (j = 0; j < v->count_tx_rings; j++) { 3377 tx_ring_index = priv->bdr_int_num * j + i; 3378 priv->tx_ring[tx_ring_index] = NULL; 3379 } 3380 3381 priv->rx_ring[i] = NULL; 3382 priv->int_vector[i] = NULL; 3383 kfree(v); 3384 } 3385 3386 int enetc_alloc_msix(struct enetc_ndev_priv *priv) 3387 { 3388 struct pci_dev *pdev = priv->si->pdev; 3389 int v_tx_rings, v_remainder; 3390 int num_stack_tx_queues; 3391 int first_xdp_tx_ring; 3392 int i, n, err, nvec; 3393 3394 nvec = ENETC_BDR_INT_BASE_IDX + priv->bdr_int_num; 3395 /* allocate MSIX for both messaging and Rx/Tx interrupts */ 3396 n = pci_alloc_irq_vectors(pdev, nvec, nvec, PCI_IRQ_MSIX); 3397 3398 if (n < 0) 3399 return n; 3400 3401 if (n != nvec) 3402 return -EPERM; 3403 3404 /* # of tx rings per int vector */ 3405 v_tx_rings = priv->num_tx_rings / priv->bdr_int_num; 3406 v_remainder = priv->num_tx_rings % priv->bdr_int_num; 3407 3408 for (i = 0; i < priv->bdr_int_num; i++) { 3409 /* Distribute the remaining TX rings to the first v_remainder 3410 * interrupt vectors 3411 */ 3412 int num_tx_rings = i < v_remainder ? v_tx_rings + 1 : v_tx_rings; 3413 3414 err = enetc_int_vector_init(priv, i, num_tx_rings); 3415 if (err) 3416 goto fail; 3417 } 3418 3419 num_stack_tx_queues = enetc_num_stack_tx_queues(priv); 3420 3421 err = netif_set_real_num_tx_queues(priv->ndev, num_stack_tx_queues); 3422 if (err) 3423 goto fail; 3424 3425 err = netif_set_real_num_rx_queues(priv->ndev, priv->num_rx_rings); 3426 if (err) 3427 goto fail; 3428 3429 priv->min_num_stack_tx_queues = num_possible_cpus(); 3430 first_xdp_tx_ring = priv->num_tx_rings - num_possible_cpus(); 3431 priv->xdp_tx_ring = &priv->tx_ring[first_xdp_tx_ring]; 3432 3433 return 0; 3434 3435 fail: 3436 while (i--) 3437 enetc_int_vector_destroy(priv, i); 3438 3439 pci_free_irq_vectors(pdev); 3440 3441 return err; 3442 } 3443 EXPORT_SYMBOL_GPL(enetc_alloc_msix); 3444 3445 void enetc_free_msix(struct enetc_ndev_priv *priv) 3446 { 3447 int i; 3448 3449 for (i = 0; i < priv->bdr_int_num; i++) 3450 enetc_int_vector_destroy(priv, i); 3451 3452 /* disable all MSIX for this device */ 3453 pci_free_irq_vectors(priv->si->pdev); 3454 } 3455 EXPORT_SYMBOL_GPL(enetc_free_msix); 3456 3457 static void enetc_kfree_si(struct enetc_si *si) 3458 { 3459 char *p = (char *)si - si->pad; 3460 3461 kfree(p); 3462 } 3463 3464 static void enetc_detect_errata(struct enetc_si *si) 3465 { 3466 if (si->pdev->revision == ENETC_REV1) 3467 si->errata = ENETC_ERR_VLAN_ISOL | ENETC_ERR_UCMCSWP; 3468 } 3469 3470 int enetc_pci_probe(struct pci_dev *pdev, const char *name, int sizeof_priv) 3471 { 3472 struct enetc_si *si, *p; 3473 struct enetc_hw *hw; 3474 size_t alloc_size; 3475 int err, len; 3476 3477 pcie_flr(pdev); 3478 err = pci_enable_device_mem(pdev); 3479 if (err) 3480 return dev_err_probe(&pdev->dev, err, "device enable failed\n"); 3481 3482 /* set up for high or low dma */ 3483 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 3484 if (err) { 3485 dev_err(&pdev->dev, "DMA configuration failed: 0x%x\n", err); 3486 goto err_dma; 3487 } 3488 3489 err = pci_request_mem_regions(pdev, name); 3490 if (err) { 3491 dev_err(&pdev->dev, "pci_request_regions failed err=%d\n", err); 3492 goto err_pci_mem_reg; 3493 } 3494 3495 pci_set_master(pdev); 3496 3497 alloc_size = sizeof(struct enetc_si); 3498 if (sizeof_priv) { 3499 /* align priv to 32B */ 3500 alloc_size = ALIGN(alloc_size, ENETC_SI_ALIGN); 3501 alloc_size += sizeof_priv; 3502 } 3503 /* force 32B alignment for enetc_si */ 3504 alloc_size += ENETC_SI_ALIGN - 1; 3505 3506 p = kzalloc(alloc_size, GFP_KERNEL); 3507 if (!p) { 3508 err = -ENOMEM; 3509 goto err_alloc_si; 3510 } 3511 3512 si = PTR_ALIGN(p, ENETC_SI_ALIGN); 3513 si->pad = (char *)si - (char *)p; 3514 3515 pci_set_drvdata(pdev, si); 3516 si->pdev = pdev; 3517 hw = &si->hw; 3518 3519 len = pci_resource_len(pdev, ENETC_BAR_REGS); 3520 hw->reg = ioremap(pci_resource_start(pdev, ENETC_BAR_REGS), len); 3521 if (!hw->reg) { 3522 err = -ENXIO; 3523 dev_err(&pdev->dev, "ioremap() failed\n"); 3524 goto err_ioremap; 3525 } 3526 if (len > ENETC_PORT_BASE) 3527 hw->port = hw->reg + ENETC_PORT_BASE; 3528 if (len > ENETC_GLOBAL_BASE) 3529 hw->global = hw->reg + ENETC_GLOBAL_BASE; 3530 3531 enetc_detect_errata(si); 3532 3533 return 0; 3534 3535 err_ioremap: 3536 enetc_kfree_si(si); 3537 err_alloc_si: 3538 pci_release_mem_regions(pdev); 3539 err_pci_mem_reg: 3540 err_dma: 3541 pci_disable_device(pdev); 3542 3543 return err; 3544 } 3545 EXPORT_SYMBOL_GPL(enetc_pci_probe); 3546 3547 void enetc_pci_remove(struct pci_dev *pdev) 3548 { 3549 struct enetc_si *si = pci_get_drvdata(pdev); 3550 struct enetc_hw *hw = &si->hw; 3551 3552 iounmap(hw->reg); 3553 enetc_kfree_si(si); 3554 pci_release_mem_regions(pdev); 3555 pci_disable_device(pdev); 3556 } 3557 EXPORT_SYMBOL_GPL(enetc_pci_remove); 3558 3559 static const struct enetc_drvdata enetc_pf_data = { 3560 .sysclk_freq = ENETC_CLK_400M, 3561 .pmac_offset = ENETC_PMAC_OFFSET, 3562 .max_frags = ENETC_MAX_SKB_FRAGS, 3563 .eth_ops = &enetc_pf_ethtool_ops, 3564 }; 3565 3566 static const struct enetc_drvdata enetc4_pf_data = { 3567 .sysclk_freq = ENETC_CLK_333M, 3568 .tx_csum = true, 3569 .max_frags = ENETC4_MAX_SKB_FRAGS, 3570 .pmac_offset = ENETC4_PMAC_OFFSET, 3571 .eth_ops = &enetc4_pf_ethtool_ops, 3572 }; 3573 3574 static const struct enetc_drvdata enetc_vf_data = { 3575 .sysclk_freq = ENETC_CLK_400M, 3576 .max_frags = ENETC_MAX_SKB_FRAGS, 3577 .eth_ops = &enetc_vf_ethtool_ops, 3578 }; 3579 3580 static const struct enetc_platform_info enetc_info[] = { 3581 { .revision = ENETC_REV_1_0, 3582 .dev_id = ENETC_DEV_ID_PF, 3583 .data = &enetc_pf_data, 3584 }, 3585 { .revision = ENETC_REV_4_1, 3586 .dev_id = NXP_ENETC_PF_DEV_ID, 3587 .data = &enetc4_pf_data, 3588 }, 3589 { .revision = ENETC_REV_1_0, 3590 .dev_id = ENETC_DEV_ID_VF, 3591 .data = &enetc_vf_data, 3592 }, 3593 }; 3594 3595 int enetc_get_driver_data(struct enetc_si *si) 3596 { 3597 u16 dev_id = si->pdev->device; 3598 int i; 3599 3600 for (i = 0; i < ARRAY_SIZE(enetc_info); i++) { 3601 if (si->revision == enetc_info[i].revision && 3602 dev_id == enetc_info[i].dev_id) { 3603 si->drvdata = enetc_info[i].data; 3604 3605 return 0; 3606 } 3607 } 3608 3609 return -ERANGE; 3610 } 3611 EXPORT_SYMBOL_GPL(enetc_get_driver_data); 3612 3613 MODULE_DESCRIPTION("NXP ENETC Ethernet driver"); 3614 MODULE_LICENSE("Dual BSD/GPL"); 3615