1 // SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) 2 /* Copyright 2017-2019 NXP */ 3 4 #include "enetc.h" 5 #include <linux/tcp.h> 6 #include <linux/udp.h> 7 #include <linux/of_mdio.h> 8 #include <linux/vmalloc.h> 9 10 /* ENETC overhead: optional extension BD + 1 BD gap */ 11 #define ENETC_TXBDS_NEEDED(val) ((val) + 2) 12 /* max # of chained Tx BDs is 15, including head and extension BD */ 13 #define ENETC_MAX_SKB_FRAGS 13 14 #define ENETC_TXBDS_MAX_NEEDED ENETC_TXBDS_NEEDED(ENETC_MAX_SKB_FRAGS + 1) 15 16 static int enetc_map_tx_buffs(struct enetc_bdr *tx_ring, struct sk_buff *skb, 17 int active_offloads); 18 19 netdev_tx_t enetc_xmit(struct sk_buff *skb, struct net_device *ndev) 20 { 21 struct enetc_ndev_priv *priv = netdev_priv(ndev); 22 struct enetc_bdr *tx_ring; 23 int count; 24 25 tx_ring = priv->tx_ring[skb->queue_mapping]; 26 27 if (unlikely(skb_shinfo(skb)->nr_frags > ENETC_MAX_SKB_FRAGS)) 28 if (unlikely(skb_linearize(skb))) 29 goto drop_packet_err; 30 31 count = skb_shinfo(skb)->nr_frags + 1; /* fragments + head */ 32 if (enetc_bd_unused(tx_ring) < ENETC_TXBDS_NEEDED(count)) { 33 netif_stop_subqueue(ndev, tx_ring->index); 34 return NETDEV_TX_BUSY; 35 } 36 37 count = enetc_map_tx_buffs(tx_ring, skb, priv->active_offloads); 38 if (unlikely(!count)) 39 goto drop_packet_err; 40 41 if (enetc_bd_unused(tx_ring) < ENETC_TXBDS_MAX_NEEDED) 42 netif_stop_subqueue(ndev, tx_ring->index); 43 44 return NETDEV_TX_OK; 45 46 drop_packet_err: 47 dev_kfree_skb_any(skb); 48 return NETDEV_TX_OK; 49 } 50 51 static bool enetc_tx_csum(struct sk_buff *skb, union enetc_tx_bd *txbd) 52 { 53 int l3_start, l3_hsize; 54 u16 l3_flags, l4_flags; 55 56 if (skb->ip_summed != CHECKSUM_PARTIAL) 57 return false; 58 59 switch (skb->csum_offset) { 60 case offsetof(struct tcphdr, check): 61 l4_flags = ENETC_TXBD_L4_TCP; 62 break; 63 case offsetof(struct udphdr, check): 64 l4_flags = ENETC_TXBD_L4_UDP; 65 break; 66 default: 67 skb_checksum_help(skb); 68 return false; 69 } 70 71 l3_start = skb_network_offset(skb); 72 l3_hsize = skb_network_header_len(skb); 73 74 l3_flags = 0; 75 if (skb->protocol == htons(ETH_P_IPV6)) 76 l3_flags = ENETC_TXBD_L3_IPV6; 77 78 /* write BD fields */ 79 txbd->l3_csoff = enetc_txbd_l3_csoff(l3_start, l3_hsize, l3_flags); 80 txbd->l4_csoff = l4_flags; 81 82 return true; 83 } 84 85 static void enetc_unmap_tx_buff(struct enetc_bdr *tx_ring, 86 struct enetc_tx_swbd *tx_swbd) 87 { 88 if (tx_swbd->is_dma_page) 89 dma_unmap_page(tx_ring->dev, tx_swbd->dma, 90 tx_swbd->len, DMA_TO_DEVICE); 91 else 92 dma_unmap_single(tx_ring->dev, tx_swbd->dma, 93 tx_swbd->len, DMA_TO_DEVICE); 94 tx_swbd->dma = 0; 95 } 96 97 static void enetc_free_tx_skb(struct enetc_bdr *tx_ring, 98 struct enetc_tx_swbd *tx_swbd) 99 { 100 if (tx_swbd->dma) 101 enetc_unmap_tx_buff(tx_ring, tx_swbd); 102 103 if (tx_swbd->skb) { 104 dev_kfree_skb_any(tx_swbd->skb); 105 tx_swbd->skb = NULL; 106 } 107 } 108 109 static int enetc_map_tx_buffs(struct enetc_bdr *tx_ring, struct sk_buff *skb, 110 int active_offloads) 111 { 112 struct enetc_tx_swbd *tx_swbd; 113 skb_frag_t *frag; 114 int len = skb_headlen(skb); 115 union enetc_tx_bd temp_bd; 116 union enetc_tx_bd *txbd; 117 bool do_vlan, do_tstamp; 118 int i, count = 0; 119 unsigned int f; 120 dma_addr_t dma; 121 u8 flags = 0; 122 123 i = tx_ring->next_to_use; 124 txbd = ENETC_TXBD(*tx_ring, i); 125 prefetchw(txbd); 126 127 dma = dma_map_single(tx_ring->dev, skb->data, len, DMA_TO_DEVICE); 128 if (unlikely(dma_mapping_error(tx_ring->dev, dma))) 129 goto dma_err; 130 131 temp_bd.addr = cpu_to_le64(dma); 132 temp_bd.buf_len = cpu_to_le16(len); 133 temp_bd.lstatus = 0; 134 135 tx_swbd = &tx_ring->tx_swbd[i]; 136 tx_swbd->dma = dma; 137 tx_swbd->len = len; 138 tx_swbd->is_dma_page = 0; 139 count++; 140 141 do_vlan = skb_vlan_tag_present(skb); 142 do_tstamp = (active_offloads & ENETC_F_TX_TSTAMP) && 143 (skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP); 144 tx_swbd->do_tstamp = do_tstamp; 145 tx_swbd->check_wb = tx_swbd->do_tstamp; 146 147 if (do_vlan || do_tstamp) 148 flags |= ENETC_TXBD_FLAGS_EX; 149 150 if (enetc_tx_csum(skb, &temp_bd)) 151 flags |= ENETC_TXBD_FLAGS_CSUM | ENETC_TXBD_FLAGS_L4CS; 152 153 /* first BD needs frm_len and offload flags set */ 154 temp_bd.frm_len = cpu_to_le16(skb->len); 155 temp_bd.flags = flags; 156 157 if (flags & ENETC_TXBD_FLAGS_EX) { 158 u8 e_flags = 0; 159 *txbd = temp_bd; 160 enetc_clear_tx_bd(&temp_bd); 161 162 /* add extension BD for VLAN and/or timestamping */ 163 flags = 0; 164 tx_swbd++; 165 txbd++; 166 i++; 167 if (unlikely(i == tx_ring->bd_count)) { 168 i = 0; 169 tx_swbd = tx_ring->tx_swbd; 170 txbd = ENETC_TXBD(*tx_ring, 0); 171 } 172 prefetchw(txbd); 173 174 if (do_vlan) { 175 temp_bd.ext.vid = cpu_to_le16(skb_vlan_tag_get(skb)); 176 temp_bd.ext.tpid = 0; /* < C-TAG */ 177 e_flags |= ENETC_TXBD_E_FLAGS_VLAN_INS; 178 } 179 180 if (do_tstamp) { 181 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; 182 e_flags |= ENETC_TXBD_E_FLAGS_TWO_STEP_PTP; 183 } 184 185 temp_bd.ext.e_flags = e_flags; 186 count++; 187 } 188 189 frag = &skb_shinfo(skb)->frags[0]; 190 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++, frag++) { 191 len = skb_frag_size(frag); 192 dma = skb_frag_dma_map(tx_ring->dev, frag, 0, len, 193 DMA_TO_DEVICE); 194 if (dma_mapping_error(tx_ring->dev, dma)) 195 goto dma_err; 196 197 *txbd = temp_bd; 198 enetc_clear_tx_bd(&temp_bd); 199 200 flags = 0; 201 tx_swbd++; 202 txbd++; 203 i++; 204 if (unlikely(i == tx_ring->bd_count)) { 205 i = 0; 206 tx_swbd = tx_ring->tx_swbd; 207 txbd = ENETC_TXBD(*tx_ring, 0); 208 } 209 prefetchw(txbd); 210 211 temp_bd.addr = cpu_to_le64(dma); 212 temp_bd.buf_len = cpu_to_le16(len); 213 214 tx_swbd->dma = dma; 215 tx_swbd->len = len; 216 tx_swbd->is_dma_page = 1; 217 count++; 218 } 219 220 /* last BD needs 'F' bit set */ 221 flags |= ENETC_TXBD_FLAGS_F; 222 temp_bd.flags = flags; 223 *txbd = temp_bd; 224 225 tx_ring->tx_swbd[i].skb = skb; 226 227 enetc_bdr_idx_inc(tx_ring, &i); 228 tx_ring->next_to_use = i; 229 230 skb_tx_timestamp(skb); 231 232 /* let H/W know BD ring has been updated */ 233 enetc_wr_reg(tx_ring->tpir, i); /* includes wmb() */ 234 235 return count; 236 237 dma_err: 238 dev_err(tx_ring->dev, "DMA map error"); 239 240 do { 241 tx_swbd = &tx_ring->tx_swbd[i]; 242 enetc_free_tx_skb(tx_ring, tx_swbd); 243 if (i == 0) 244 i = tx_ring->bd_count; 245 i--; 246 } while (count--); 247 248 return 0; 249 } 250 251 static irqreturn_t enetc_msix(int irq, void *data) 252 { 253 struct enetc_int_vector *v = data; 254 int i; 255 256 /* disable interrupts */ 257 enetc_wr_reg(v->rbier, 0); 258 259 for_each_set_bit(i, &v->tx_rings_map, v->count_tx_rings) 260 enetc_wr_reg(v->tbier_base + ENETC_BDR_OFF(i), 0); 261 262 napi_schedule_irqoff(&v->napi); 263 264 return IRQ_HANDLED; 265 } 266 267 static bool enetc_clean_tx_ring(struct enetc_bdr *tx_ring, int napi_budget); 268 static int enetc_clean_rx_ring(struct enetc_bdr *rx_ring, 269 struct napi_struct *napi, int work_limit); 270 271 static int enetc_poll(struct napi_struct *napi, int budget) 272 { 273 struct enetc_int_vector 274 *v = container_of(napi, struct enetc_int_vector, napi); 275 bool complete = true; 276 int work_done; 277 int i; 278 279 for (i = 0; i < v->count_tx_rings; i++) 280 if (!enetc_clean_tx_ring(&v->tx_ring[i], budget)) 281 complete = false; 282 283 work_done = enetc_clean_rx_ring(&v->rx_ring, napi, budget); 284 if (work_done == budget) 285 complete = false; 286 287 if (!complete) 288 return budget; 289 290 napi_complete_done(napi, work_done); 291 292 /* enable interrupts */ 293 enetc_wr_reg(v->rbier, ENETC_RBIER_RXTIE); 294 295 for_each_set_bit(i, &v->tx_rings_map, v->count_tx_rings) 296 enetc_wr_reg(v->tbier_base + ENETC_BDR_OFF(i), 297 ENETC_TBIER_TXTIE); 298 299 return work_done; 300 } 301 302 static int enetc_bd_ready_count(struct enetc_bdr *tx_ring, int ci) 303 { 304 int pi = enetc_rd_reg(tx_ring->tcir) & ENETC_TBCIR_IDX_MASK; 305 306 return pi >= ci ? pi - ci : tx_ring->bd_count - ci + pi; 307 } 308 309 static void enetc_get_tx_tstamp(struct enetc_hw *hw, union enetc_tx_bd *txbd, 310 u64 *tstamp) 311 { 312 u32 lo, hi, tstamp_lo; 313 314 lo = enetc_rd(hw, ENETC_SICTR0); 315 hi = enetc_rd(hw, ENETC_SICTR1); 316 tstamp_lo = le32_to_cpu(txbd->wb.tstamp); 317 if (lo <= tstamp_lo) 318 hi -= 1; 319 *tstamp = (u64)hi << 32 | tstamp_lo; 320 } 321 322 static void enetc_tstamp_tx(struct sk_buff *skb, u64 tstamp) 323 { 324 struct skb_shared_hwtstamps shhwtstamps; 325 326 if (skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS) { 327 memset(&shhwtstamps, 0, sizeof(shhwtstamps)); 328 shhwtstamps.hwtstamp = ns_to_ktime(tstamp); 329 skb_tstamp_tx(skb, &shhwtstamps); 330 } 331 } 332 333 static bool enetc_clean_tx_ring(struct enetc_bdr *tx_ring, int napi_budget) 334 { 335 struct net_device *ndev = tx_ring->ndev; 336 int tx_frm_cnt = 0, tx_byte_cnt = 0; 337 struct enetc_tx_swbd *tx_swbd; 338 int i, bds_to_clean; 339 bool do_tstamp; 340 u64 tstamp = 0; 341 342 i = tx_ring->next_to_clean; 343 tx_swbd = &tx_ring->tx_swbd[i]; 344 bds_to_clean = enetc_bd_ready_count(tx_ring, i); 345 346 do_tstamp = false; 347 348 while (bds_to_clean && tx_frm_cnt < ENETC_DEFAULT_TX_WORK) { 349 bool is_eof = !!tx_swbd->skb; 350 351 if (unlikely(tx_swbd->check_wb)) { 352 struct enetc_ndev_priv *priv = netdev_priv(ndev); 353 union enetc_tx_bd *txbd; 354 355 txbd = ENETC_TXBD(*tx_ring, i); 356 357 if (txbd->flags & ENETC_TXBD_FLAGS_W && 358 tx_swbd->do_tstamp) { 359 enetc_get_tx_tstamp(&priv->si->hw, txbd, 360 &tstamp); 361 do_tstamp = true; 362 } 363 } 364 365 if (likely(tx_swbd->dma)) 366 enetc_unmap_tx_buff(tx_ring, tx_swbd); 367 368 if (is_eof) { 369 if (unlikely(do_tstamp)) { 370 enetc_tstamp_tx(tx_swbd->skb, tstamp); 371 do_tstamp = false; 372 } 373 napi_consume_skb(tx_swbd->skb, napi_budget); 374 tx_swbd->skb = NULL; 375 } 376 377 tx_byte_cnt += tx_swbd->len; 378 379 bds_to_clean--; 380 tx_swbd++; 381 i++; 382 if (unlikely(i == tx_ring->bd_count)) { 383 i = 0; 384 tx_swbd = tx_ring->tx_swbd; 385 } 386 387 /* BD iteration loop end */ 388 if (is_eof) { 389 tx_frm_cnt++; 390 /* re-arm interrupt source */ 391 enetc_wr_reg(tx_ring->idr, BIT(tx_ring->index) | 392 BIT(16 + tx_ring->index)); 393 } 394 395 if (unlikely(!bds_to_clean)) 396 bds_to_clean = enetc_bd_ready_count(tx_ring, i); 397 } 398 399 tx_ring->next_to_clean = i; 400 tx_ring->stats.packets += tx_frm_cnt; 401 tx_ring->stats.bytes += tx_byte_cnt; 402 403 if (unlikely(tx_frm_cnt && netif_carrier_ok(ndev) && 404 __netif_subqueue_stopped(ndev, tx_ring->index) && 405 (enetc_bd_unused(tx_ring) >= ENETC_TXBDS_MAX_NEEDED))) { 406 netif_wake_subqueue(ndev, tx_ring->index); 407 } 408 409 return tx_frm_cnt != ENETC_DEFAULT_TX_WORK; 410 } 411 412 static bool enetc_new_page(struct enetc_bdr *rx_ring, 413 struct enetc_rx_swbd *rx_swbd) 414 { 415 struct page *page; 416 dma_addr_t addr; 417 418 page = dev_alloc_page(); 419 if (unlikely(!page)) 420 return false; 421 422 addr = dma_map_page(rx_ring->dev, page, 0, PAGE_SIZE, DMA_FROM_DEVICE); 423 if (unlikely(dma_mapping_error(rx_ring->dev, addr))) { 424 __free_page(page); 425 426 return false; 427 } 428 429 rx_swbd->dma = addr; 430 rx_swbd->page = page; 431 rx_swbd->page_offset = ENETC_RXB_PAD; 432 433 return true; 434 } 435 436 static int enetc_refill_rx_ring(struct enetc_bdr *rx_ring, const int buff_cnt) 437 { 438 struct enetc_rx_swbd *rx_swbd; 439 union enetc_rx_bd *rxbd; 440 int i, j; 441 442 i = rx_ring->next_to_use; 443 rx_swbd = &rx_ring->rx_swbd[i]; 444 rxbd = ENETC_RXBD(*rx_ring, i); 445 446 for (j = 0; j < buff_cnt; j++) { 447 /* try reuse page */ 448 if (unlikely(!rx_swbd->page)) { 449 if (unlikely(!enetc_new_page(rx_ring, rx_swbd))) { 450 rx_ring->stats.rx_alloc_errs++; 451 break; 452 } 453 } 454 455 /* update RxBD */ 456 rxbd->w.addr = cpu_to_le64(rx_swbd->dma + 457 rx_swbd->page_offset); 458 /* clear 'R" as well */ 459 rxbd->r.lstatus = 0; 460 461 rx_swbd++; 462 rxbd++; 463 i++; 464 if (unlikely(i == rx_ring->bd_count)) { 465 i = 0; 466 rx_swbd = rx_ring->rx_swbd; 467 rxbd = ENETC_RXBD(*rx_ring, 0); 468 } 469 } 470 471 if (likely(j)) { 472 rx_ring->next_to_alloc = i; /* keep track from page reuse */ 473 rx_ring->next_to_use = i; 474 /* update ENETC's consumer index */ 475 enetc_wr_reg(rx_ring->rcir, i); 476 } 477 478 return j; 479 } 480 481 #ifdef CONFIG_FSL_ENETC_HW_TIMESTAMPING 482 static void enetc_get_rx_tstamp(struct net_device *ndev, 483 union enetc_rx_bd *rxbd, 484 struct sk_buff *skb) 485 { 486 struct skb_shared_hwtstamps *shhwtstamps = skb_hwtstamps(skb); 487 struct enetc_ndev_priv *priv = netdev_priv(ndev); 488 struct enetc_hw *hw = &priv->si->hw; 489 u32 lo, hi, tstamp_lo; 490 u64 tstamp; 491 492 if (le16_to_cpu(rxbd->r.flags) & ENETC_RXBD_FLAG_TSTMP) { 493 lo = enetc_rd(hw, ENETC_SICTR0); 494 hi = enetc_rd(hw, ENETC_SICTR1); 495 tstamp_lo = le32_to_cpu(rxbd->r.tstamp); 496 if (lo <= tstamp_lo) 497 hi -= 1; 498 499 tstamp = (u64)hi << 32 | tstamp_lo; 500 memset(shhwtstamps, 0, sizeof(*shhwtstamps)); 501 shhwtstamps->hwtstamp = ns_to_ktime(tstamp); 502 } 503 } 504 #endif 505 506 static void enetc_get_offloads(struct enetc_bdr *rx_ring, 507 union enetc_rx_bd *rxbd, struct sk_buff *skb) 508 { 509 #ifdef CONFIG_FSL_ENETC_HW_TIMESTAMPING 510 struct enetc_ndev_priv *priv = netdev_priv(rx_ring->ndev); 511 #endif 512 /* TODO: hashing */ 513 if (rx_ring->ndev->features & NETIF_F_RXCSUM) { 514 u16 inet_csum = le16_to_cpu(rxbd->r.inet_csum); 515 516 skb->csum = csum_unfold((__force __sum16)~htons(inet_csum)); 517 skb->ip_summed = CHECKSUM_COMPLETE; 518 } 519 520 /* copy VLAN to skb, if one is extracted, for now we assume it's a 521 * standard TPID, but HW also supports custom values 522 */ 523 if (le16_to_cpu(rxbd->r.flags) & ENETC_RXBD_FLAG_VLAN) 524 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), 525 le16_to_cpu(rxbd->r.vlan_opt)); 526 #ifdef CONFIG_FSL_ENETC_HW_TIMESTAMPING 527 if (priv->active_offloads & ENETC_F_RX_TSTAMP) 528 enetc_get_rx_tstamp(rx_ring->ndev, rxbd, skb); 529 #endif 530 } 531 532 static void enetc_process_skb(struct enetc_bdr *rx_ring, 533 struct sk_buff *skb) 534 { 535 skb_record_rx_queue(skb, rx_ring->index); 536 skb->protocol = eth_type_trans(skb, rx_ring->ndev); 537 } 538 539 static bool enetc_page_reusable(struct page *page) 540 { 541 return (!page_is_pfmemalloc(page) && page_ref_count(page) == 1); 542 } 543 544 static void enetc_reuse_page(struct enetc_bdr *rx_ring, 545 struct enetc_rx_swbd *old) 546 { 547 struct enetc_rx_swbd *new; 548 549 new = &rx_ring->rx_swbd[rx_ring->next_to_alloc]; 550 551 /* next buf that may reuse a page */ 552 enetc_bdr_idx_inc(rx_ring, &rx_ring->next_to_alloc); 553 554 /* copy page reference */ 555 *new = *old; 556 } 557 558 static struct enetc_rx_swbd *enetc_get_rx_buff(struct enetc_bdr *rx_ring, 559 int i, u16 size) 560 { 561 struct enetc_rx_swbd *rx_swbd = &rx_ring->rx_swbd[i]; 562 563 dma_sync_single_range_for_cpu(rx_ring->dev, rx_swbd->dma, 564 rx_swbd->page_offset, 565 size, DMA_FROM_DEVICE); 566 return rx_swbd; 567 } 568 569 static void enetc_put_rx_buff(struct enetc_bdr *rx_ring, 570 struct enetc_rx_swbd *rx_swbd) 571 { 572 if (likely(enetc_page_reusable(rx_swbd->page))) { 573 rx_swbd->page_offset ^= ENETC_RXB_TRUESIZE; 574 page_ref_inc(rx_swbd->page); 575 576 enetc_reuse_page(rx_ring, rx_swbd); 577 578 /* sync for use by the device */ 579 dma_sync_single_range_for_device(rx_ring->dev, rx_swbd->dma, 580 rx_swbd->page_offset, 581 ENETC_RXB_DMA_SIZE, 582 DMA_FROM_DEVICE); 583 } else { 584 dma_unmap_page(rx_ring->dev, rx_swbd->dma, 585 PAGE_SIZE, DMA_FROM_DEVICE); 586 } 587 588 rx_swbd->page = NULL; 589 } 590 591 static struct sk_buff *enetc_map_rx_buff_to_skb(struct enetc_bdr *rx_ring, 592 int i, u16 size) 593 { 594 struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size); 595 struct sk_buff *skb; 596 void *ba; 597 598 ba = page_address(rx_swbd->page) + rx_swbd->page_offset; 599 skb = build_skb(ba - ENETC_RXB_PAD, ENETC_RXB_TRUESIZE); 600 if (unlikely(!skb)) { 601 rx_ring->stats.rx_alloc_errs++; 602 return NULL; 603 } 604 605 skb_reserve(skb, ENETC_RXB_PAD); 606 __skb_put(skb, size); 607 608 enetc_put_rx_buff(rx_ring, rx_swbd); 609 610 return skb; 611 } 612 613 static void enetc_add_rx_buff_to_skb(struct enetc_bdr *rx_ring, int i, 614 u16 size, struct sk_buff *skb) 615 { 616 struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size); 617 618 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_swbd->page, 619 rx_swbd->page_offset, size, ENETC_RXB_TRUESIZE); 620 621 enetc_put_rx_buff(rx_ring, rx_swbd); 622 } 623 624 #define ENETC_RXBD_BUNDLE 16 /* # of BDs to update at once */ 625 626 static int enetc_clean_rx_ring(struct enetc_bdr *rx_ring, 627 struct napi_struct *napi, int work_limit) 628 { 629 int rx_frm_cnt = 0, rx_byte_cnt = 0; 630 int cleaned_cnt, i; 631 632 cleaned_cnt = enetc_bd_unused(rx_ring); 633 /* next descriptor to process */ 634 i = rx_ring->next_to_clean; 635 636 while (likely(rx_frm_cnt < work_limit)) { 637 union enetc_rx_bd *rxbd; 638 struct sk_buff *skb; 639 u32 bd_status; 640 u16 size; 641 642 if (cleaned_cnt >= ENETC_RXBD_BUNDLE) { 643 int count = enetc_refill_rx_ring(rx_ring, cleaned_cnt); 644 645 cleaned_cnt -= count; 646 } 647 648 rxbd = ENETC_RXBD(*rx_ring, i); 649 bd_status = le32_to_cpu(rxbd->r.lstatus); 650 if (!bd_status) 651 break; 652 653 enetc_wr_reg(rx_ring->idr, BIT(rx_ring->index)); 654 dma_rmb(); /* for reading other rxbd fields */ 655 size = le16_to_cpu(rxbd->r.buf_len); 656 skb = enetc_map_rx_buff_to_skb(rx_ring, i, size); 657 if (!skb) 658 break; 659 660 enetc_get_offloads(rx_ring, rxbd, skb); 661 662 cleaned_cnt++; 663 rxbd++; 664 i++; 665 if (unlikely(i == rx_ring->bd_count)) { 666 i = 0; 667 rxbd = ENETC_RXBD(*rx_ring, 0); 668 } 669 670 if (unlikely(bd_status & 671 ENETC_RXBD_LSTATUS(ENETC_RXBD_ERR_MASK))) { 672 dev_kfree_skb(skb); 673 while (!(bd_status & ENETC_RXBD_LSTATUS_F)) { 674 dma_rmb(); 675 bd_status = le32_to_cpu(rxbd->r.lstatus); 676 rxbd++; 677 i++; 678 if (unlikely(i == rx_ring->bd_count)) { 679 i = 0; 680 rxbd = ENETC_RXBD(*rx_ring, 0); 681 } 682 } 683 684 rx_ring->ndev->stats.rx_dropped++; 685 rx_ring->ndev->stats.rx_errors++; 686 687 break; 688 } 689 690 /* not last BD in frame? */ 691 while (!(bd_status & ENETC_RXBD_LSTATUS_F)) { 692 bd_status = le32_to_cpu(rxbd->r.lstatus); 693 size = ENETC_RXB_DMA_SIZE; 694 695 if (bd_status & ENETC_RXBD_LSTATUS_F) { 696 dma_rmb(); 697 size = le16_to_cpu(rxbd->r.buf_len); 698 } 699 700 enetc_add_rx_buff_to_skb(rx_ring, i, size, skb); 701 702 cleaned_cnt++; 703 rxbd++; 704 i++; 705 if (unlikely(i == rx_ring->bd_count)) { 706 i = 0; 707 rxbd = ENETC_RXBD(*rx_ring, 0); 708 } 709 } 710 711 rx_byte_cnt += skb->len; 712 713 enetc_process_skb(rx_ring, skb); 714 715 napi_gro_receive(napi, skb); 716 717 rx_frm_cnt++; 718 } 719 720 rx_ring->next_to_clean = i; 721 722 rx_ring->stats.packets += rx_frm_cnt; 723 rx_ring->stats.bytes += rx_byte_cnt; 724 725 return rx_frm_cnt; 726 } 727 728 /* Probing and Init */ 729 #define ENETC_MAX_RFS_SIZE 64 730 void enetc_get_si_caps(struct enetc_si *si) 731 { 732 struct enetc_hw *hw = &si->hw; 733 u32 val; 734 735 /* find out how many of various resources we have to work with */ 736 val = enetc_rd(hw, ENETC_SICAPR0); 737 si->num_rx_rings = (val >> 16) & 0xff; 738 si->num_tx_rings = val & 0xff; 739 740 val = enetc_rd(hw, ENETC_SIRFSCAPR); 741 si->num_fs_entries = ENETC_SIRFSCAPR_GET_NUM_RFS(val); 742 si->num_fs_entries = min(si->num_fs_entries, ENETC_MAX_RFS_SIZE); 743 744 si->num_rss = 0; 745 val = enetc_rd(hw, ENETC_SIPCAPR0); 746 if (val & ENETC_SIPCAPR0_RSS) { 747 u32 rss; 748 749 rss = enetc_rd(hw, ENETC_SIRSSCAPR); 750 si->num_rss = ENETC_SIRSSCAPR_GET_NUM_RSS(rss); 751 } 752 753 if (val & ENETC_SIPCAPR0_QBV) 754 si->hw_features |= ENETC_SI_F_QBV; 755 } 756 757 static int enetc_dma_alloc_bdr(struct enetc_bdr *r, size_t bd_size) 758 { 759 r->bd_base = dma_alloc_coherent(r->dev, r->bd_count * bd_size, 760 &r->bd_dma_base, GFP_KERNEL); 761 if (!r->bd_base) 762 return -ENOMEM; 763 764 /* h/w requires 128B alignment */ 765 if (!IS_ALIGNED(r->bd_dma_base, 128)) { 766 dma_free_coherent(r->dev, r->bd_count * bd_size, r->bd_base, 767 r->bd_dma_base); 768 return -EINVAL; 769 } 770 771 return 0; 772 } 773 774 static int enetc_alloc_txbdr(struct enetc_bdr *txr) 775 { 776 int err; 777 778 txr->tx_swbd = vzalloc(txr->bd_count * sizeof(struct enetc_tx_swbd)); 779 if (!txr->tx_swbd) 780 return -ENOMEM; 781 782 err = enetc_dma_alloc_bdr(txr, sizeof(union enetc_tx_bd)); 783 if (err) { 784 vfree(txr->tx_swbd); 785 return err; 786 } 787 788 txr->next_to_clean = 0; 789 txr->next_to_use = 0; 790 791 return 0; 792 } 793 794 static void enetc_free_txbdr(struct enetc_bdr *txr) 795 { 796 int size, i; 797 798 for (i = 0; i < txr->bd_count; i++) 799 enetc_free_tx_skb(txr, &txr->tx_swbd[i]); 800 801 size = txr->bd_count * sizeof(union enetc_tx_bd); 802 803 dma_free_coherent(txr->dev, size, txr->bd_base, txr->bd_dma_base); 804 txr->bd_base = NULL; 805 806 vfree(txr->tx_swbd); 807 txr->tx_swbd = NULL; 808 } 809 810 static int enetc_alloc_tx_resources(struct enetc_ndev_priv *priv) 811 { 812 int i, err; 813 814 for (i = 0; i < priv->num_tx_rings; i++) { 815 err = enetc_alloc_txbdr(priv->tx_ring[i]); 816 817 if (err) 818 goto fail; 819 } 820 821 return 0; 822 823 fail: 824 while (i-- > 0) 825 enetc_free_txbdr(priv->tx_ring[i]); 826 827 return err; 828 } 829 830 static void enetc_free_tx_resources(struct enetc_ndev_priv *priv) 831 { 832 int i; 833 834 for (i = 0; i < priv->num_tx_rings; i++) 835 enetc_free_txbdr(priv->tx_ring[i]); 836 } 837 838 static int enetc_alloc_rxbdr(struct enetc_bdr *rxr) 839 { 840 int err; 841 842 rxr->rx_swbd = vzalloc(rxr->bd_count * sizeof(struct enetc_rx_swbd)); 843 if (!rxr->rx_swbd) 844 return -ENOMEM; 845 846 err = enetc_dma_alloc_bdr(rxr, sizeof(union enetc_rx_bd)); 847 if (err) { 848 vfree(rxr->rx_swbd); 849 return err; 850 } 851 852 rxr->next_to_clean = 0; 853 rxr->next_to_use = 0; 854 rxr->next_to_alloc = 0; 855 856 return 0; 857 } 858 859 static void enetc_free_rxbdr(struct enetc_bdr *rxr) 860 { 861 int size; 862 863 size = rxr->bd_count * sizeof(union enetc_rx_bd); 864 865 dma_free_coherent(rxr->dev, size, rxr->bd_base, rxr->bd_dma_base); 866 rxr->bd_base = NULL; 867 868 vfree(rxr->rx_swbd); 869 rxr->rx_swbd = NULL; 870 } 871 872 static int enetc_alloc_rx_resources(struct enetc_ndev_priv *priv) 873 { 874 int i, err; 875 876 for (i = 0; i < priv->num_rx_rings; i++) { 877 err = enetc_alloc_rxbdr(priv->rx_ring[i]); 878 879 if (err) 880 goto fail; 881 } 882 883 return 0; 884 885 fail: 886 while (i-- > 0) 887 enetc_free_rxbdr(priv->rx_ring[i]); 888 889 return err; 890 } 891 892 static void enetc_free_rx_resources(struct enetc_ndev_priv *priv) 893 { 894 int i; 895 896 for (i = 0; i < priv->num_rx_rings; i++) 897 enetc_free_rxbdr(priv->rx_ring[i]); 898 } 899 900 static void enetc_free_tx_ring(struct enetc_bdr *tx_ring) 901 { 902 int i; 903 904 if (!tx_ring->tx_swbd) 905 return; 906 907 for (i = 0; i < tx_ring->bd_count; i++) { 908 struct enetc_tx_swbd *tx_swbd = &tx_ring->tx_swbd[i]; 909 910 enetc_free_tx_skb(tx_ring, tx_swbd); 911 } 912 913 tx_ring->next_to_clean = 0; 914 tx_ring->next_to_use = 0; 915 } 916 917 static void enetc_free_rx_ring(struct enetc_bdr *rx_ring) 918 { 919 int i; 920 921 if (!rx_ring->rx_swbd) 922 return; 923 924 for (i = 0; i < rx_ring->bd_count; i++) { 925 struct enetc_rx_swbd *rx_swbd = &rx_ring->rx_swbd[i]; 926 927 if (!rx_swbd->page) 928 continue; 929 930 dma_unmap_page(rx_ring->dev, rx_swbd->dma, 931 PAGE_SIZE, DMA_FROM_DEVICE); 932 __free_page(rx_swbd->page); 933 rx_swbd->page = NULL; 934 } 935 936 rx_ring->next_to_clean = 0; 937 rx_ring->next_to_use = 0; 938 rx_ring->next_to_alloc = 0; 939 } 940 941 static void enetc_free_rxtx_rings(struct enetc_ndev_priv *priv) 942 { 943 int i; 944 945 for (i = 0; i < priv->num_rx_rings; i++) 946 enetc_free_rx_ring(priv->rx_ring[i]); 947 948 for (i = 0; i < priv->num_tx_rings; i++) 949 enetc_free_tx_ring(priv->tx_ring[i]); 950 } 951 952 static int enetc_alloc_cbdr(struct device *dev, struct enetc_cbdr *cbdr) 953 { 954 int size = cbdr->bd_count * sizeof(struct enetc_cbd); 955 956 cbdr->bd_base = dma_alloc_coherent(dev, size, &cbdr->bd_dma_base, 957 GFP_KERNEL); 958 if (!cbdr->bd_base) 959 return -ENOMEM; 960 961 /* h/w requires 128B alignment */ 962 if (!IS_ALIGNED(cbdr->bd_dma_base, 128)) { 963 dma_free_coherent(dev, size, cbdr->bd_base, cbdr->bd_dma_base); 964 return -EINVAL; 965 } 966 967 cbdr->next_to_clean = 0; 968 cbdr->next_to_use = 0; 969 970 return 0; 971 } 972 973 static void enetc_free_cbdr(struct device *dev, struct enetc_cbdr *cbdr) 974 { 975 int size = cbdr->bd_count * sizeof(struct enetc_cbd); 976 977 dma_free_coherent(dev, size, cbdr->bd_base, cbdr->bd_dma_base); 978 cbdr->bd_base = NULL; 979 } 980 981 static void enetc_setup_cbdr(struct enetc_hw *hw, struct enetc_cbdr *cbdr) 982 { 983 /* set CBDR cache attributes */ 984 enetc_wr(hw, ENETC_SICAR2, 985 ENETC_SICAR_RD_COHERENT | ENETC_SICAR_WR_COHERENT); 986 987 enetc_wr(hw, ENETC_SICBDRBAR0, lower_32_bits(cbdr->bd_dma_base)); 988 enetc_wr(hw, ENETC_SICBDRBAR1, upper_32_bits(cbdr->bd_dma_base)); 989 enetc_wr(hw, ENETC_SICBDRLENR, ENETC_RTBLENR_LEN(cbdr->bd_count)); 990 991 enetc_wr(hw, ENETC_SICBDRPIR, 0); 992 enetc_wr(hw, ENETC_SICBDRCIR, 0); 993 994 /* enable ring */ 995 enetc_wr(hw, ENETC_SICBDRMR, BIT(31)); 996 997 cbdr->pir = hw->reg + ENETC_SICBDRPIR; 998 cbdr->cir = hw->reg + ENETC_SICBDRCIR; 999 } 1000 1001 static void enetc_clear_cbdr(struct enetc_hw *hw) 1002 { 1003 enetc_wr(hw, ENETC_SICBDRMR, 0); 1004 } 1005 1006 static int enetc_setup_default_rss_table(struct enetc_si *si, int num_groups) 1007 { 1008 int *rss_table; 1009 int i; 1010 1011 rss_table = kmalloc_array(si->num_rss, sizeof(*rss_table), GFP_KERNEL); 1012 if (!rss_table) 1013 return -ENOMEM; 1014 1015 /* Set up RSS table defaults */ 1016 for (i = 0; i < si->num_rss; i++) 1017 rss_table[i] = i % num_groups; 1018 1019 enetc_set_rss_table(si, rss_table, si->num_rss); 1020 1021 kfree(rss_table); 1022 1023 return 0; 1024 } 1025 1026 static int enetc_configure_si(struct enetc_ndev_priv *priv) 1027 { 1028 struct enetc_si *si = priv->si; 1029 struct enetc_hw *hw = &si->hw; 1030 int err; 1031 1032 enetc_setup_cbdr(hw, &si->cbd_ring); 1033 /* set SI cache attributes */ 1034 enetc_wr(hw, ENETC_SICAR0, 1035 ENETC_SICAR_RD_COHERENT | ENETC_SICAR_WR_COHERENT); 1036 enetc_wr(hw, ENETC_SICAR1, ENETC_SICAR_MSI); 1037 /* enable SI */ 1038 enetc_wr(hw, ENETC_SIMR, ENETC_SIMR_EN); 1039 1040 if (si->num_rss) { 1041 err = enetc_setup_default_rss_table(si, priv->num_rx_rings); 1042 if (err) 1043 return err; 1044 } 1045 1046 return 0; 1047 } 1048 1049 void enetc_init_si_rings_params(struct enetc_ndev_priv *priv) 1050 { 1051 struct enetc_si *si = priv->si; 1052 int cpus = num_online_cpus(); 1053 1054 priv->tx_bd_count = ENETC_BDR_DEFAULT_SIZE; 1055 priv->rx_bd_count = ENETC_BDR_DEFAULT_SIZE; 1056 1057 /* Enable all available TX rings in order to configure as many 1058 * priorities as possible, when needed. 1059 * TODO: Make # of TX rings run-time configurable 1060 */ 1061 priv->num_rx_rings = min_t(int, cpus, si->num_rx_rings); 1062 priv->num_tx_rings = si->num_tx_rings; 1063 priv->bdr_int_num = cpus; 1064 1065 /* SI specific */ 1066 si->cbd_ring.bd_count = ENETC_CBDR_DEFAULT_SIZE; 1067 } 1068 1069 int enetc_alloc_si_resources(struct enetc_ndev_priv *priv) 1070 { 1071 struct enetc_si *si = priv->si; 1072 int err; 1073 1074 err = enetc_alloc_cbdr(priv->dev, &si->cbd_ring); 1075 if (err) 1076 return err; 1077 1078 priv->cls_rules = kcalloc(si->num_fs_entries, sizeof(*priv->cls_rules), 1079 GFP_KERNEL); 1080 if (!priv->cls_rules) { 1081 err = -ENOMEM; 1082 goto err_alloc_cls; 1083 } 1084 1085 err = enetc_configure_si(priv); 1086 if (err) 1087 goto err_config_si; 1088 1089 return 0; 1090 1091 err_config_si: 1092 kfree(priv->cls_rules); 1093 err_alloc_cls: 1094 enetc_clear_cbdr(&si->hw); 1095 enetc_free_cbdr(priv->dev, &si->cbd_ring); 1096 1097 return err; 1098 } 1099 1100 void enetc_free_si_resources(struct enetc_ndev_priv *priv) 1101 { 1102 struct enetc_si *si = priv->si; 1103 1104 enetc_clear_cbdr(&si->hw); 1105 enetc_free_cbdr(priv->dev, &si->cbd_ring); 1106 1107 kfree(priv->cls_rules); 1108 } 1109 1110 static void enetc_setup_txbdr(struct enetc_hw *hw, struct enetc_bdr *tx_ring) 1111 { 1112 int idx = tx_ring->index; 1113 u32 tbmr; 1114 1115 enetc_txbdr_wr(hw, idx, ENETC_TBBAR0, 1116 lower_32_bits(tx_ring->bd_dma_base)); 1117 1118 enetc_txbdr_wr(hw, idx, ENETC_TBBAR1, 1119 upper_32_bits(tx_ring->bd_dma_base)); 1120 1121 WARN_ON(!IS_ALIGNED(tx_ring->bd_count, 64)); /* multiple of 64 */ 1122 enetc_txbdr_wr(hw, idx, ENETC_TBLENR, 1123 ENETC_RTBLENR_LEN(tx_ring->bd_count)); 1124 1125 /* clearing PI/CI registers for Tx not supported, adjust sw indexes */ 1126 tx_ring->next_to_use = enetc_txbdr_rd(hw, idx, ENETC_TBPIR); 1127 tx_ring->next_to_clean = enetc_txbdr_rd(hw, idx, ENETC_TBCIR); 1128 1129 /* enable Tx ints by setting pkt thr to 1 */ 1130 enetc_txbdr_wr(hw, idx, ENETC_TBICIR0, ENETC_TBICIR0_ICEN | 0x1); 1131 1132 tbmr = ENETC_TBMR_EN; 1133 if (tx_ring->ndev->features & NETIF_F_HW_VLAN_CTAG_TX) 1134 tbmr |= ENETC_TBMR_VIH; 1135 1136 /* enable ring */ 1137 enetc_txbdr_wr(hw, idx, ENETC_TBMR, tbmr); 1138 1139 tx_ring->tpir = hw->reg + ENETC_BDR(TX, idx, ENETC_TBPIR); 1140 tx_ring->tcir = hw->reg + ENETC_BDR(TX, idx, ENETC_TBCIR); 1141 tx_ring->idr = hw->reg + ENETC_SITXIDR; 1142 } 1143 1144 static void enetc_setup_rxbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring) 1145 { 1146 int idx = rx_ring->index; 1147 u32 rbmr; 1148 1149 enetc_rxbdr_wr(hw, idx, ENETC_RBBAR0, 1150 lower_32_bits(rx_ring->bd_dma_base)); 1151 1152 enetc_rxbdr_wr(hw, idx, ENETC_RBBAR1, 1153 upper_32_bits(rx_ring->bd_dma_base)); 1154 1155 WARN_ON(!IS_ALIGNED(rx_ring->bd_count, 64)); /* multiple of 64 */ 1156 enetc_rxbdr_wr(hw, idx, ENETC_RBLENR, 1157 ENETC_RTBLENR_LEN(rx_ring->bd_count)); 1158 1159 enetc_rxbdr_wr(hw, idx, ENETC_RBBSR, ENETC_RXB_DMA_SIZE); 1160 1161 enetc_rxbdr_wr(hw, idx, ENETC_RBPIR, 0); 1162 1163 /* enable Rx ints by setting pkt thr to 1 */ 1164 enetc_rxbdr_wr(hw, idx, ENETC_RBICIR0, ENETC_RBICIR0_ICEN | 0x1); 1165 1166 rbmr = ENETC_RBMR_EN; 1167 #ifdef CONFIG_FSL_ENETC_HW_TIMESTAMPING 1168 rbmr |= ENETC_RBMR_BDS; 1169 #endif 1170 if (rx_ring->ndev->features & NETIF_F_HW_VLAN_CTAG_RX) 1171 rbmr |= ENETC_RBMR_VTE; 1172 1173 rx_ring->rcir = hw->reg + ENETC_BDR(RX, idx, ENETC_RBCIR); 1174 rx_ring->idr = hw->reg + ENETC_SIRXIDR; 1175 1176 enetc_refill_rx_ring(rx_ring, enetc_bd_unused(rx_ring)); 1177 1178 /* enable ring */ 1179 enetc_rxbdr_wr(hw, idx, ENETC_RBMR, rbmr); 1180 } 1181 1182 static void enetc_setup_bdrs(struct enetc_ndev_priv *priv) 1183 { 1184 int i; 1185 1186 for (i = 0; i < priv->num_tx_rings; i++) 1187 enetc_setup_txbdr(&priv->si->hw, priv->tx_ring[i]); 1188 1189 for (i = 0; i < priv->num_rx_rings; i++) 1190 enetc_setup_rxbdr(&priv->si->hw, priv->rx_ring[i]); 1191 } 1192 1193 static void enetc_clear_rxbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring) 1194 { 1195 int idx = rx_ring->index; 1196 1197 /* disable EN bit on ring */ 1198 enetc_rxbdr_wr(hw, idx, ENETC_RBMR, 0); 1199 } 1200 1201 static void enetc_clear_txbdr(struct enetc_hw *hw, struct enetc_bdr *tx_ring) 1202 { 1203 int delay = 8, timeout = 100; 1204 int idx = tx_ring->index; 1205 1206 /* disable EN bit on ring */ 1207 enetc_txbdr_wr(hw, idx, ENETC_TBMR, 0); 1208 1209 /* wait for busy to clear */ 1210 while (delay < timeout && 1211 enetc_txbdr_rd(hw, idx, ENETC_TBSR) & ENETC_TBSR_BUSY) { 1212 msleep(delay); 1213 delay *= 2; 1214 } 1215 1216 if (delay >= timeout) 1217 netdev_warn(tx_ring->ndev, "timeout for tx ring #%d clear\n", 1218 idx); 1219 } 1220 1221 static void enetc_clear_bdrs(struct enetc_ndev_priv *priv) 1222 { 1223 int i; 1224 1225 for (i = 0; i < priv->num_tx_rings; i++) 1226 enetc_clear_txbdr(&priv->si->hw, priv->tx_ring[i]); 1227 1228 for (i = 0; i < priv->num_rx_rings; i++) 1229 enetc_clear_rxbdr(&priv->si->hw, priv->rx_ring[i]); 1230 1231 udelay(1); 1232 } 1233 1234 static int enetc_setup_irqs(struct enetc_ndev_priv *priv) 1235 { 1236 struct pci_dev *pdev = priv->si->pdev; 1237 cpumask_t cpu_mask; 1238 int i, j, err; 1239 1240 for (i = 0; i < priv->bdr_int_num; i++) { 1241 int irq = pci_irq_vector(pdev, ENETC_BDR_INT_BASE_IDX + i); 1242 struct enetc_int_vector *v = priv->int_vector[i]; 1243 int entry = ENETC_BDR_INT_BASE_IDX + i; 1244 struct enetc_hw *hw = &priv->si->hw; 1245 1246 snprintf(v->name, sizeof(v->name), "%s-rxtx%d", 1247 priv->ndev->name, i); 1248 err = request_irq(irq, enetc_msix, 0, v->name, v); 1249 if (err) { 1250 dev_err(priv->dev, "request_irq() failed!\n"); 1251 goto irq_err; 1252 } 1253 1254 v->tbier_base = hw->reg + ENETC_BDR(TX, 0, ENETC_TBIER); 1255 v->rbier = hw->reg + ENETC_BDR(RX, i, ENETC_RBIER); 1256 1257 enetc_wr(hw, ENETC_SIMSIRRV(i), entry); 1258 1259 for (j = 0; j < v->count_tx_rings; j++) { 1260 int idx = v->tx_ring[j].index; 1261 1262 enetc_wr(hw, ENETC_SIMSITRV(idx), entry); 1263 } 1264 cpumask_clear(&cpu_mask); 1265 cpumask_set_cpu(i % num_online_cpus(), &cpu_mask); 1266 irq_set_affinity_hint(irq, &cpu_mask); 1267 } 1268 1269 return 0; 1270 1271 irq_err: 1272 while (i--) { 1273 int irq = pci_irq_vector(pdev, ENETC_BDR_INT_BASE_IDX + i); 1274 1275 irq_set_affinity_hint(irq, NULL); 1276 free_irq(irq, priv->int_vector[i]); 1277 } 1278 1279 return err; 1280 } 1281 1282 static void enetc_free_irqs(struct enetc_ndev_priv *priv) 1283 { 1284 struct pci_dev *pdev = priv->si->pdev; 1285 int i; 1286 1287 for (i = 0; i < priv->bdr_int_num; i++) { 1288 int irq = pci_irq_vector(pdev, ENETC_BDR_INT_BASE_IDX + i); 1289 1290 irq_set_affinity_hint(irq, NULL); 1291 free_irq(irq, priv->int_vector[i]); 1292 } 1293 } 1294 1295 static void enetc_enable_interrupts(struct enetc_ndev_priv *priv) 1296 { 1297 int i; 1298 1299 /* enable Tx & Rx event indication */ 1300 for (i = 0; i < priv->num_rx_rings; i++) { 1301 enetc_rxbdr_wr(&priv->si->hw, i, 1302 ENETC_RBIER, ENETC_RBIER_RXTIE); 1303 } 1304 1305 for (i = 0; i < priv->num_tx_rings; i++) { 1306 enetc_txbdr_wr(&priv->si->hw, i, 1307 ENETC_TBIER, ENETC_TBIER_TXTIE); 1308 } 1309 } 1310 1311 static void enetc_disable_interrupts(struct enetc_ndev_priv *priv) 1312 { 1313 int i; 1314 1315 for (i = 0; i < priv->num_tx_rings; i++) 1316 enetc_txbdr_wr(&priv->si->hw, i, ENETC_TBIER, 0); 1317 1318 for (i = 0; i < priv->num_rx_rings; i++) 1319 enetc_rxbdr_wr(&priv->si->hw, i, ENETC_RBIER, 0); 1320 } 1321 1322 static void adjust_link(struct net_device *ndev) 1323 { 1324 struct enetc_ndev_priv *priv = netdev_priv(ndev); 1325 struct phy_device *phydev = ndev->phydev; 1326 1327 if (priv->active_offloads & ENETC_F_QBV) 1328 enetc_sched_speed_set(ndev); 1329 1330 phy_print_status(phydev); 1331 } 1332 1333 static int enetc_phy_connect(struct net_device *ndev) 1334 { 1335 struct enetc_ndev_priv *priv = netdev_priv(ndev); 1336 struct phy_device *phydev; 1337 struct ethtool_eee edata; 1338 1339 if (!priv->phy_node) 1340 return 0; /* phy-less mode */ 1341 1342 phydev = of_phy_connect(ndev, priv->phy_node, &adjust_link, 1343 0, priv->if_mode); 1344 if (!phydev) { 1345 dev_err(&ndev->dev, "could not attach to PHY\n"); 1346 return -ENODEV; 1347 } 1348 1349 phy_attached_info(phydev); 1350 1351 /* disable EEE autoneg, until ENETC driver supports it */ 1352 memset(&edata, 0, sizeof(struct ethtool_eee)); 1353 phy_ethtool_set_eee(phydev, &edata); 1354 1355 return 0; 1356 } 1357 1358 int enetc_open(struct net_device *ndev) 1359 { 1360 struct enetc_ndev_priv *priv = netdev_priv(ndev); 1361 int i, err; 1362 1363 err = enetc_setup_irqs(priv); 1364 if (err) 1365 return err; 1366 1367 err = enetc_phy_connect(ndev); 1368 if (err) 1369 goto err_phy_connect; 1370 1371 err = enetc_alloc_tx_resources(priv); 1372 if (err) 1373 goto err_alloc_tx; 1374 1375 err = enetc_alloc_rx_resources(priv); 1376 if (err) 1377 goto err_alloc_rx; 1378 1379 enetc_setup_bdrs(priv); 1380 1381 err = netif_set_real_num_tx_queues(ndev, priv->num_tx_rings); 1382 if (err) 1383 goto err_set_queues; 1384 1385 err = netif_set_real_num_rx_queues(ndev, priv->num_rx_rings); 1386 if (err) 1387 goto err_set_queues; 1388 1389 for (i = 0; i < priv->bdr_int_num; i++) 1390 napi_enable(&priv->int_vector[i]->napi); 1391 1392 enetc_enable_interrupts(priv); 1393 1394 if (ndev->phydev) 1395 phy_start(ndev->phydev); 1396 else 1397 netif_carrier_on(ndev); 1398 1399 netif_tx_start_all_queues(ndev); 1400 1401 return 0; 1402 1403 err_set_queues: 1404 enetc_free_rx_resources(priv); 1405 err_alloc_rx: 1406 enetc_free_tx_resources(priv); 1407 err_alloc_tx: 1408 if (ndev->phydev) 1409 phy_disconnect(ndev->phydev); 1410 err_phy_connect: 1411 enetc_free_irqs(priv); 1412 1413 return err; 1414 } 1415 1416 int enetc_close(struct net_device *ndev) 1417 { 1418 struct enetc_ndev_priv *priv = netdev_priv(ndev); 1419 int i; 1420 1421 netif_tx_stop_all_queues(ndev); 1422 1423 if (ndev->phydev) { 1424 phy_stop(ndev->phydev); 1425 phy_disconnect(ndev->phydev); 1426 } else { 1427 netif_carrier_off(ndev); 1428 } 1429 1430 for (i = 0; i < priv->bdr_int_num; i++) { 1431 napi_synchronize(&priv->int_vector[i]->napi); 1432 napi_disable(&priv->int_vector[i]->napi); 1433 } 1434 1435 enetc_disable_interrupts(priv); 1436 enetc_clear_bdrs(priv); 1437 1438 enetc_free_rxtx_rings(priv); 1439 enetc_free_rx_resources(priv); 1440 enetc_free_tx_resources(priv); 1441 enetc_free_irqs(priv); 1442 1443 return 0; 1444 } 1445 1446 static int enetc_setup_tc_mqprio(struct net_device *ndev, void *type_data) 1447 { 1448 struct enetc_ndev_priv *priv = netdev_priv(ndev); 1449 struct tc_mqprio_qopt *mqprio = type_data; 1450 struct enetc_bdr *tx_ring; 1451 u8 num_tc; 1452 int i; 1453 1454 mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS; 1455 num_tc = mqprio->num_tc; 1456 1457 if (!num_tc) { 1458 netdev_reset_tc(ndev); 1459 netif_set_real_num_tx_queues(ndev, priv->num_tx_rings); 1460 1461 /* Reset all ring priorities to 0 */ 1462 for (i = 0; i < priv->num_tx_rings; i++) { 1463 tx_ring = priv->tx_ring[i]; 1464 enetc_set_bdr_prio(&priv->si->hw, tx_ring->index, 0); 1465 } 1466 1467 return 0; 1468 } 1469 1470 /* Check if we have enough BD rings available to accommodate all TCs */ 1471 if (num_tc > priv->num_tx_rings) { 1472 netdev_err(ndev, "Max %d traffic classes supported\n", 1473 priv->num_tx_rings); 1474 return -EINVAL; 1475 } 1476 1477 /* For the moment, we use only one BD ring per TC. 1478 * 1479 * Configure num_tc BD rings with increasing priorities. 1480 */ 1481 for (i = 0; i < num_tc; i++) { 1482 tx_ring = priv->tx_ring[i]; 1483 enetc_set_bdr_prio(&priv->si->hw, tx_ring->index, i); 1484 } 1485 1486 /* Reset the number of netdev queues based on the TC count */ 1487 netif_set_real_num_tx_queues(ndev, num_tc); 1488 1489 netdev_set_num_tc(ndev, num_tc); 1490 1491 /* Each TC is associated with one netdev queue */ 1492 for (i = 0; i < num_tc; i++) 1493 netdev_set_tc_queue(ndev, i, 1, i); 1494 1495 return 0; 1496 } 1497 1498 int enetc_setup_tc(struct net_device *ndev, enum tc_setup_type type, 1499 void *type_data) 1500 { 1501 switch (type) { 1502 case TC_SETUP_QDISC_MQPRIO: 1503 return enetc_setup_tc_mqprio(ndev, type_data); 1504 case TC_SETUP_QDISC_TAPRIO: 1505 return enetc_setup_tc_taprio(ndev, type_data); 1506 case TC_SETUP_QDISC_CBS: 1507 return enetc_setup_tc_cbs(ndev, type_data); 1508 default: 1509 return -EOPNOTSUPP; 1510 } 1511 } 1512 1513 struct net_device_stats *enetc_get_stats(struct net_device *ndev) 1514 { 1515 struct enetc_ndev_priv *priv = netdev_priv(ndev); 1516 struct net_device_stats *stats = &ndev->stats; 1517 unsigned long packets = 0, bytes = 0; 1518 int i; 1519 1520 for (i = 0; i < priv->num_rx_rings; i++) { 1521 packets += priv->rx_ring[i]->stats.packets; 1522 bytes += priv->rx_ring[i]->stats.bytes; 1523 } 1524 1525 stats->rx_packets = packets; 1526 stats->rx_bytes = bytes; 1527 bytes = 0; 1528 packets = 0; 1529 1530 for (i = 0; i < priv->num_tx_rings; i++) { 1531 packets += priv->tx_ring[i]->stats.packets; 1532 bytes += priv->tx_ring[i]->stats.bytes; 1533 } 1534 1535 stats->tx_packets = packets; 1536 stats->tx_bytes = bytes; 1537 1538 return stats; 1539 } 1540 1541 static int enetc_set_rss(struct net_device *ndev, int en) 1542 { 1543 struct enetc_ndev_priv *priv = netdev_priv(ndev); 1544 struct enetc_hw *hw = &priv->si->hw; 1545 u32 reg; 1546 1547 enetc_wr(hw, ENETC_SIRBGCR, priv->num_rx_rings); 1548 1549 reg = enetc_rd(hw, ENETC_SIMR); 1550 reg &= ~ENETC_SIMR_RSSE; 1551 reg |= (en) ? ENETC_SIMR_RSSE : 0; 1552 enetc_wr(hw, ENETC_SIMR, reg); 1553 1554 return 0; 1555 } 1556 1557 int enetc_set_features(struct net_device *ndev, 1558 netdev_features_t features) 1559 { 1560 netdev_features_t changed = ndev->features ^ features; 1561 1562 if (changed & NETIF_F_RXHASH) 1563 enetc_set_rss(ndev, !!(features & NETIF_F_RXHASH)); 1564 1565 return 0; 1566 } 1567 1568 #ifdef CONFIG_FSL_ENETC_HW_TIMESTAMPING 1569 static int enetc_hwtstamp_set(struct net_device *ndev, struct ifreq *ifr) 1570 { 1571 struct enetc_ndev_priv *priv = netdev_priv(ndev); 1572 struct hwtstamp_config config; 1573 1574 if (copy_from_user(&config, ifr->ifr_data, sizeof(config))) 1575 return -EFAULT; 1576 1577 switch (config.tx_type) { 1578 case HWTSTAMP_TX_OFF: 1579 priv->active_offloads &= ~ENETC_F_TX_TSTAMP; 1580 break; 1581 case HWTSTAMP_TX_ON: 1582 priv->active_offloads |= ENETC_F_TX_TSTAMP; 1583 break; 1584 default: 1585 return -ERANGE; 1586 } 1587 1588 switch (config.rx_filter) { 1589 case HWTSTAMP_FILTER_NONE: 1590 priv->active_offloads &= ~ENETC_F_RX_TSTAMP; 1591 break; 1592 default: 1593 priv->active_offloads |= ENETC_F_RX_TSTAMP; 1594 config.rx_filter = HWTSTAMP_FILTER_ALL; 1595 } 1596 1597 return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ? 1598 -EFAULT : 0; 1599 } 1600 1601 static int enetc_hwtstamp_get(struct net_device *ndev, struct ifreq *ifr) 1602 { 1603 struct enetc_ndev_priv *priv = netdev_priv(ndev); 1604 struct hwtstamp_config config; 1605 1606 config.flags = 0; 1607 1608 if (priv->active_offloads & ENETC_F_TX_TSTAMP) 1609 config.tx_type = HWTSTAMP_TX_ON; 1610 else 1611 config.tx_type = HWTSTAMP_TX_OFF; 1612 1613 config.rx_filter = (priv->active_offloads & ENETC_F_RX_TSTAMP) ? 1614 HWTSTAMP_FILTER_ALL : HWTSTAMP_FILTER_NONE; 1615 1616 return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ? 1617 -EFAULT : 0; 1618 } 1619 #endif 1620 1621 int enetc_ioctl(struct net_device *ndev, struct ifreq *rq, int cmd) 1622 { 1623 #ifdef CONFIG_FSL_ENETC_HW_TIMESTAMPING 1624 if (cmd == SIOCSHWTSTAMP) 1625 return enetc_hwtstamp_set(ndev, rq); 1626 if (cmd == SIOCGHWTSTAMP) 1627 return enetc_hwtstamp_get(ndev, rq); 1628 #endif 1629 1630 if (!ndev->phydev) 1631 return -EOPNOTSUPP; 1632 return phy_mii_ioctl(ndev->phydev, rq, cmd); 1633 } 1634 1635 int enetc_alloc_msix(struct enetc_ndev_priv *priv) 1636 { 1637 struct pci_dev *pdev = priv->si->pdev; 1638 int size, v_tx_rings; 1639 int i, n, err, nvec; 1640 1641 nvec = ENETC_BDR_INT_BASE_IDX + priv->bdr_int_num; 1642 /* allocate MSIX for both messaging and Rx/Tx interrupts */ 1643 n = pci_alloc_irq_vectors(pdev, nvec, nvec, PCI_IRQ_MSIX); 1644 1645 if (n < 0) 1646 return n; 1647 1648 if (n != nvec) 1649 return -EPERM; 1650 1651 /* # of tx rings per int vector */ 1652 v_tx_rings = priv->num_tx_rings / priv->bdr_int_num; 1653 size = sizeof(struct enetc_int_vector) + 1654 sizeof(struct enetc_bdr) * v_tx_rings; 1655 1656 for (i = 0; i < priv->bdr_int_num; i++) { 1657 struct enetc_int_vector *v; 1658 struct enetc_bdr *bdr; 1659 int j; 1660 1661 v = kzalloc(size, GFP_KERNEL); 1662 if (!v) { 1663 err = -ENOMEM; 1664 goto fail; 1665 } 1666 1667 priv->int_vector[i] = v; 1668 1669 netif_napi_add(priv->ndev, &v->napi, enetc_poll, 1670 NAPI_POLL_WEIGHT); 1671 v->count_tx_rings = v_tx_rings; 1672 1673 for (j = 0; j < v_tx_rings; j++) { 1674 int idx; 1675 1676 /* default tx ring mapping policy */ 1677 if (priv->bdr_int_num == ENETC_MAX_BDR_INT) 1678 idx = 2 * j + i; /* 2 CPUs */ 1679 else 1680 idx = j + i * v_tx_rings; /* default */ 1681 1682 __set_bit(idx, &v->tx_rings_map); 1683 bdr = &v->tx_ring[j]; 1684 bdr->index = idx; 1685 bdr->ndev = priv->ndev; 1686 bdr->dev = priv->dev; 1687 bdr->bd_count = priv->tx_bd_count; 1688 priv->tx_ring[idx] = bdr; 1689 } 1690 1691 bdr = &v->rx_ring; 1692 bdr->index = i; 1693 bdr->ndev = priv->ndev; 1694 bdr->dev = priv->dev; 1695 bdr->bd_count = priv->rx_bd_count; 1696 priv->rx_ring[i] = bdr; 1697 } 1698 1699 return 0; 1700 1701 fail: 1702 while (i--) { 1703 netif_napi_del(&priv->int_vector[i]->napi); 1704 kfree(priv->int_vector[i]); 1705 } 1706 1707 pci_free_irq_vectors(pdev); 1708 1709 return err; 1710 } 1711 1712 void enetc_free_msix(struct enetc_ndev_priv *priv) 1713 { 1714 int i; 1715 1716 for (i = 0; i < priv->bdr_int_num; i++) { 1717 struct enetc_int_vector *v = priv->int_vector[i]; 1718 1719 netif_napi_del(&v->napi); 1720 } 1721 1722 for (i = 0; i < priv->num_rx_rings; i++) 1723 priv->rx_ring[i] = NULL; 1724 1725 for (i = 0; i < priv->num_tx_rings; i++) 1726 priv->tx_ring[i] = NULL; 1727 1728 for (i = 0; i < priv->bdr_int_num; i++) { 1729 kfree(priv->int_vector[i]); 1730 priv->int_vector[i] = NULL; 1731 } 1732 1733 /* disable all MSIX for this device */ 1734 pci_free_irq_vectors(priv->si->pdev); 1735 } 1736 1737 static void enetc_kfree_si(struct enetc_si *si) 1738 { 1739 char *p = (char *)si - si->pad; 1740 1741 kfree(p); 1742 } 1743 1744 static void enetc_detect_errata(struct enetc_si *si) 1745 { 1746 if (si->pdev->revision == ENETC_REV1) 1747 si->errata = ENETC_ERR_TXCSUM | ENETC_ERR_VLAN_ISOL | 1748 ENETC_ERR_UCMCSWP; 1749 } 1750 1751 int enetc_pci_probe(struct pci_dev *pdev, const char *name, int sizeof_priv) 1752 { 1753 struct enetc_si *si, *p; 1754 struct enetc_hw *hw; 1755 size_t alloc_size; 1756 int err, len; 1757 1758 pcie_flr(pdev); 1759 err = pci_enable_device_mem(pdev); 1760 if (err) { 1761 dev_err(&pdev->dev, "device enable failed\n"); 1762 return err; 1763 } 1764 1765 /* set up for high or low dma */ 1766 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 1767 if (err) { 1768 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); 1769 if (err) { 1770 dev_err(&pdev->dev, 1771 "DMA configuration failed: 0x%x\n", err); 1772 goto err_dma; 1773 } 1774 } 1775 1776 err = pci_request_mem_regions(pdev, name); 1777 if (err) { 1778 dev_err(&pdev->dev, "pci_request_regions failed err=%d\n", err); 1779 goto err_pci_mem_reg; 1780 } 1781 1782 pci_set_master(pdev); 1783 1784 alloc_size = sizeof(struct enetc_si); 1785 if (sizeof_priv) { 1786 /* align priv to 32B */ 1787 alloc_size = ALIGN(alloc_size, ENETC_SI_ALIGN); 1788 alloc_size += sizeof_priv; 1789 } 1790 /* force 32B alignment for enetc_si */ 1791 alloc_size += ENETC_SI_ALIGN - 1; 1792 1793 p = kzalloc(alloc_size, GFP_KERNEL); 1794 if (!p) { 1795 err = -ENOMEM; 1796 goto err_alloc_si; 1797 } 1798 1799 si = PTR_ALIGN(p, ENETC_SI_ALIGN); 1800 si->pad = (char *)si - (char *)p; 1801 1802 pci_set_drvdata(pdev, si); 1803 si->pdev = pdev; 1804 hw = &si->hw; 1805 1806 len = pci_resource_len(pdev, ENETC_BAR_REGS); 1807 hw->reg = ioremap(pci_resource_start(pdev, ENETC_BAR_REGS), len); 1808 if (!hw->reg) { 1809 err = -ENXIO; 1810 dev_err(&pdev->dev, "ioremap() failed\n"); 1811 goto err_ioremap; 1812 } 1813 if (len > ENETC_PORT_BASE) 1814 hw->port = hw->reg + ENETC_PORT_BASE; 1815 if (len > ENETC_GLOBAL_BASE) 1816 hw->global = hw->reg + ENETC_GLOBAL_BASE; 1817 1818 enetc_detect_errata(si); 1819 1820 return 0; 1821 1822 err_ioremap: 1823 enetc_kfree_si(si); 1824 err_alloc_si: 1825 pci_release_mem_regions(pdev); 1826 err_pci_mem_reg: 1827 err_dma: 1828 pci_disable_device(pdev); 1829 1830 return err; 1831 } 1832 1833 void enetc_pci_remove(struct pci_dev *pdev) 1834 { 1835 struct enetc_si *si = pci_get_drvdata(pdev); 1836 struct enetc_hw *hw = &si->hw; 1837 1838 iounmap(hw->reg); 1839 enetc_kfree_si(si); 1840 pci_release_mem_regions(pdev); 1841 pci_disable_device(pdev); 1842 } 1843