1 // SPDX-License-Identifier: GPL-2.0 2 3 /* Texas Instruments ICSSG Ethernet Driver 4 * 5 * Copyright (C) 2018-2022 Texas Instruments Incorporated - https://www.ti.com/ 6 * Copyright (C) Siemens AG, 2024 7 * 8 */ 9 10 #include <linux/dma-mapping.h> 11 #include <linux/dma/ti-cppi5.h> 12 #include <linux/etherdevice.h> 13 #include <linux/interrupt.h> 14 #include <linux/kernel.h> 15 #include <linux/of.h> 16 #include <linux/of_mdio.h> 17 #include <linux/phy.h> 18 #include <linux/remoteproc/pruss.h> 19 #include <linux/regmap.h> 20 #include <linux/remoteproc.h> 21 22 #include "icssg_prueth.h" 23 #include "../k3-cppi-desc-pool.h" 24 25 /* Netif debug messages possible */ 26 #define PRUETH_EMAC_DEBUG (NETIF_MSG_DRV | \ 27 NETIF_MSG_PROBE | \ 28 NETIF_MSG_LINK | \ 29 NETIF_MSG_TIMER | \ 30 NETIF_MSG_IFDOWN | \ 31 NETIF_MSG_IFUP | \ 32 NETIF_MSG_RX_ERR | \ 33 NETIF_MSG_TX_ERR | \ 34 NETIF_MSG_TX_QUEUED | \ 35 NETIF_MSG_INTR | \ 36 NETIF_MSG_TX_DONE | \ 37 NETIF_MSG_RX_STATUS | \ 38 NETIF_MSG_PKTDATA | \ 39 NETIF_MSG_HW | \ 40 NETIF_MSG_WOL) 41 42 #define prueth_napi_to_emac(napi) container_of(napi, struct prueth_emac, napi_rx) 43 44 void prueth_cleanup_rx_chns(struct prueth_emac *emac, 45 struct prueth_rx_chn *rx_chn, 46 int max_rflows) 47 { 48 if (rx_chn->desc_pool) 49 k3_cppi_desc_pool_destroy(rx_chn->desc_pool); 50 51 if (rx_chn->rx_chn) 52 k3_udma_glue_release_rx_chn(rx_chn->rx_chn); 53 } 54 55 void prueth_cleanup_tx_chns(struct prueth_emac *emac) 56 { 57 int i; 58 59 for (i = 0; i < emac->tx_ch_num; i++) { 60 struct prueth_tx_chn *tx_chn = &emac->tx_chns[i]; 61 62 if (tx_chn->desc_pool) 63 k3_cppi_desc_pool_destroy(tx_chn->desc_pool); 64 65 if (tx_chn->tx_chn) 66 k3_udma_glue_release_tx_chn(tx_chn->tx_chn); 67 68 /* Assume prueth_cleanup_tx_chns() is called at the 69 * end after all channel resources are freed 70 */ 71 memset(tx_chn, 0, sizeof(*tx_chn)); 72 } 73 } 74 75 void prueth_ndev_del_tx_napi(struct prueth_emac *emac, int num) 76 { 77 int i; 78 79 for (i = 0; i < num; i++) { 80 struct prueth_tx_chn *tx_chn = &emac->tx_chns[i]; 81 82 if (tx_chn->irq) 83 free_irq(tx_chn->irq, tx_chn); 84 netif_napi_del(&tx_chn->napi_tx); 85 } 86 } 87 88 void prueth_xmit_free(struct prueth_tx_chn *tx_chn, 89 struct cppi5_host_desc_t *desc) 90 { 91 struct cppi5_host_desc_t *first_desc, *next_desc; 92 dma_addr_t buf_dma, next_desc_dma; 93 u32 buf_dma_len; 94 95 first_desc = desc; 96 next_desc = first_desc; 97 98 cppi5_hdesc_get_obuf(first_desc, &buf_dma, &buf_dma_len); 99 k3_udma_glue_tx_cppi5_to_dma_addr(tx_chn->tx_chn, &buf_dma); 100 101 dma_unmap_single(tx_chn->dma_dev, buf_dma, buf_dma_len, 102 DMA_TO_DEVICE); 103 104 next_desc_dma = cppi5_hdesc_get_next_hbdesc(first_desc); 105 k3_udma_glue_tx_cppi5_to_dma_addr(tx_chn->tx_chn, &next_desc_dma); 106 while (next_desc_dma) { 107 next_desc = k3_cppi_desc_pool_dma2virt(tx_chn->desc_pool, 108 next_desc_dma); 109 cppi5_hdesc_get_obuf(next_desc, &buf_dma, &buf_dma_len); 110 k3_udma_glue_tx_cppi5_to_dma_addr(tx_chn->tx_chn, &buf_dma); 111 112 dma_unmap_page(tx_chn->dma_dev, buf_dma, buf_dma_len, 113 DMA_TO_DEVICE); 114 115 next_desc_dma = cppi5_hdesc_get_next_hbdesc(next_desc); 116 k3_udma_glue_tx_cppi5_to_dma_addr(tx_chn->tx_chn, &next_desc_dma); 117 118 k3_cppi_desc_pool_free(tx_chn->desc_pool, next_desc); 119 } 120 121 k3_cppi_desc_pool_free(tx_chn->desc_pool, first_desc); 122 } 123 124 int emac_tx_complete_packets(struct prueth_emac *emac, int chn, 125 int budget) 126 { 127 struct net_device *ndev = emac->ndev; 128 struct cppi5_host_desc_t *desc_tx; 129 struct netdev_queue *netif_txq; 130 struct prueth_tx_chn *tx_chn; 131 unsigned int total_bytes = 0; 132 struct sk_buff *skb; 133 dma_addr_t desc_dma; 134 int res, num_tx = 0; 135 void **swdata; 136 137 tx_chn = &emac->tx_chns[chn]; 138 139 while (true) { 140 res = k3_udma_glue_pop_tx_chn(tx_chn->tx_chn, &desc_dma); 141 if (res == -ENODATA) 142 break; 143 144 /* teardown completion */ 145 if (cppi5_desc_is_tdcm(desc_dma)) { 146 if (atomic_dec_and_test(&emac->tdown_cnt)) 147 complete(&emac->tdown_complete); 148 break; 149 } 150 151 desc_tx = k3_cppi_desc_pool_dma2virt(tx_chn->desc_pool, 152 desc_dma); 153 swdata = cppi5_hdesc_get_swdata(desc_tx); 154 155 /* was this command's TX complete? */ 156 if (emac->is_sr1 && *(swdata) == emac->cmd_data) { 157 prueth_xmit_free(tx_chn, desc_tx); 158 continue; 159 } 160 161 skb = *(swdata); 162 prueth_xmit_free(tx_chn, desc_tx); 163 164 ndev = skb->dev; 165 ndev->stats.tx_packets++; 166 ndev->stats.tx_bytes += skb->len; 167 total_bytes += skb->len; 168 napi_consume_skb(skb, budget); 169 num_tx++; 170 } 171 172 if (!num_tx) 173 return 0; 174 175 netif_txq = netdev_get_tx_queue(ndev, chn); 176 netdev_tx_completed_queue(netif_txq, num_tx, total_bytes); 177 178 if (netif_tx_queue_stopped(netif_txq)) { 179 /* If the TX queue was stopped, wake it now 180 * if we have enough room. 181 */ 182 __netif_tx_lock(netif_txq, smp_processor_id()); 183 if (netif_running(ndev) && 184 (k3_cppi_desc_pool_avail(tx_chn->desc_pool) >= 185 MAX_SKB_FRAGS)) 186 netif_tx_wake_queue(netif_txq); 187 __netif_tx_unlock(netif_txq); 188 } 189 190 return num_tx; 191 } 192 193 static int emac_napi_tx_poll(struct napi_struct *napi_tx, int budget) 194 { 195 struct prueth_tx_chn *tx_chn = prueth_napi_to_tx_chn(napi_tx); 196 struct prueth_emac *emac = tx_chn->emac; 197 int num_tx_packets; 198 199 num_tx_packets = emac_tx_complete_packets(emac, tx_chn->id, budget); 200 201 if (num_tx_packets >= budget) 202 return budget; 203 204 if (napi_complete_done(napi_tx, num_tx_packets)) 205 enable_irq(tx_chn->irq); 206 207 return num_tx_packets; 208 } 209 210 static irqreturn_t prueth_tx_irq(int irq, void *dev_id) 211 { 212 struct prueth_tx_chn *tx_chn = dev_id; 213 214 disable_irq_nosync(irq); 215 napi_schedule(&tx_chn->napi_tx); 216 217 return IRQ_HANDLED; 218 } 219 220 int prueth_ndev_add_tx_napi(struct prueth_emac *emac) 221 { 222 struct prueth *prueth = emac->prueth; 223 int i, ret; 224 225 for (i = 0; i < emac->tx_ch_num; i++) { 226 struct prueth_tx_chn *tx_chn = &emac->tx_chns[i]; 227 228 netif_napi_add_tx(emac->ndev, &tx_chn->napi_tx, emac_napi_tx_poll); 229 ret = request_irq(tx_chn->irq, prueth_tx_irq, 230 IRQF_TRIGGER_HIGH, tx_chn->name, 231 tx_chn); 232 if (ret) { 233 netif_napi_del(&tx_chn->napi_tx); 234 dev_err(prueth->dev, "unable to request TX IRQ %d\n", 235 tx_chn->irq); 236 goto fail; 237 } 238 } 239 240 return 0; 241 fail: 242 prueth_ndev_del_tx_napi(emac, i); 243 return ret; 244 } 245 246 int prueth_init_tx_chns(struct prueth_emac *emac) 247 { 248 static const struct k3_ring_cfg ring_cfg = { 249 .elm_size = K3_RINGACC_RING_ELSIZE_8, 250 .mode = K3_RINGACC_RING_MODE_RING, 251 .flags = 0, 252 .size = PRUETH_MAX_TX_DESC, 253 }; 254 struct k3_udma_glue_tx_channel_cfg tx_cfg; 255 struct device *dev = emac->prueth->dev; 256 struct net_device *ndev = emac->ndev; 257 int ret, slice, i; 258 u32 hdesc_size; 259 260 slice = prueth_emac_slice(emac); 261 if (slice < 0) 262 return slice; 263 264 init_completion(&emac->tdown_complete); 265 266 hdesc_size = cppi5_hdesc_calc_size(true, PRUETH_NAV_PS_DATA_SIZE, 267 PRUETH_NAV_SW_DATA_SIZE); 268 memset(&tx_cfg, 0, sizeof(tx_cfg)); 269 tx_cfg.swdata_size = PRUETH_NAV_SW_DATA_SIZE; 270 tx_cfg.tx_cfg = ring_cfg; 271 tx_cfg.txcq_cfg = ring_cfg; 272 273 for (i = 0; i < emac->tx_ch_num; i++) { 274 struct prueth_tx_chn *tx_chn = &emac->tx_chns[i]; 275 276 /* To differentiate channels for SLICE0 vs SLICE1 */ 277 snprintf(tx_chn->name, sizeof(tx_chn->name), 278 "tx%d-%d", slice, i); 279 280 tx_chn->emac = emac; 281 tx_chn->id = i; 282 tx_chn->descs_num = PRUETH_MAX_TX_DESC; 283 284 tx_chn->tx_chn = 285 k3_udma_glue_request_tx_chn(dev, tx_chn->name, 286 &tx_cfg); 287 if (IS_ERR(tx_chn->tx_chn)) { 288 ret = PTR_ERR(tx_chn->tx_chn); 289 tx_chn->tx_chn = NULL; 290 netdev_err(ndev, 291 "Failed to request tx dma ch: %d\n", ret); 292 goto fail; 293 } 294 295 tx_chn->dma_dev = k3_udma_glue_tx_get_dma_device(tx_chn->tx_chn); 296 tx_chn->desc_pool = 297 k3_cppi_desc_pool_create_name(tx_chn->dma_dev, 298 tx_chn->descs_num, 299 hdesc_size, 300 tx_chn->name); 301 if (IS_ERR(tx_chn->desc_pool)) { 302 ret = PTR_ERR(tx_chn->desc_pool); 303 tx_chn->desc_pool = NULL; 304 netdev_err(ndev, "Failed to create tx pool: %d\n", ret); 305 goto fail; 306 } 307 308 ret = k3_udma_glue_tx_get_irq(tx_chn->tx_chn); 309 if (ret < 0) { 310 netdev_err(ndev, "failed to get tx irq\n"); 311 goto fail; 312 } 313 tx_chn->irq = ret; 314 315 snprintf(tx_chn->name, sizeof(tx_chn->name), "%s-tx%d", 316 dev_name(dev), tx_chn->id); 317 } 318 319 return 0; 320 321 fail: 322 prueth_cleanup_tx_chns(emac); 323 return ret; 324 } 325 326 int prueth_init_rx_chns(struct prueth_emac *emac, 327 struct prueth_rx_chn *rx_chn, 328 char *name, u32 max_rflows, 329 u32 max_desc_num) 330 { 331 struct k3_udma_glue_rx_channel_cfg rx_cfg; 332 struct device *dev = emac->prueth->dev; 333 struct net_device *ndev = emac->ndev; 334 u32 fdqring_id, hdesc_size; 335 int i, ret = 0, slice; 336 int flow_id_base; 337 338 slice = prueth_emac_slice(emac); 339 if (slice < 0) 340 return slice; 341 342 /* To differentiate channels for SLICE0 vs SLICE1 */ 343 snprintf(rx_chn->name, sizeof(rx_chn->name), "%s%d", name, slice); 344 345 hdesc_size = cppi5_hdesc_calc_size(true, PRUETH_NAV_PS_DATA_SIZE, 346 PRUETH_NAV_SW_DATA_SIZE); 347 memset(&rx_cfg, 0, sizeof(rx_cfg)); 348 rx_cfg.swdata_size = PRUETH_NAV_SW_DATA_SIZE; 349 rx_cfg.flow_id_num = max_rflows; 350 rx_cfg.flow_id_base = -1; /* udmax will auto select flow id base */ 351 352 /* init all flows */ 353 rx_chn->dev = dev; 354 rx_chn->descs_num = max_desc_num; 355 356 rx_chn->rx_chn = k3_udma_glue_request_rx_chn(dev, rx_chn->name, 357 &rx_cfg); 358 if (IS_ERR(rx_chn->rx_chn)) { 359 ret = PTR_ERR(rx_chn->rx_chn); 360 rx_chn->rx_chn = NULL; 361 netdev_err(ndev, "Failed to request rx dma ch: %d\n", ret); 362 goto fail; 363 } 364 365 rx_chn->dma_dev = k3_udma_glue_rx_get_dma_device(rx_chn->rx_chn); 366 rx_chn->desc_pool = k3_cppi_desc_pool_create_name(rx_chn->dma_dev, 367 rx_chn->descs_num, 368 hdesc_size, 369 rx_chn->name); 370 if (IS_ERR(rx_chn->desc_pool)) { 371 ret = PTR_ERR(rx_chn->desc_pool); 372 rx_chn->desc_pool = NULL; 373 netdev_err(ndev, "Failed to create rx pool: %d\n", ret); 374 goto fail; 375 } 376 377 flow_id_base = k3_udma_glue_rx_get_flow_id_base(rx_chn->rx_chn); 378 if (emac->is_sr1 && !strcmp(name, "rxmgm")) { 379 emac->rx_mgm_flow_id_base = flow_id_base; 380 netdev_dbg(ndev, "mgm flow id base = %d\n", flow_id_base); 381 } else { 382 emac->rx_flow_id_base = flow_id_base; 383 netdev_dbg(ndev, "flow id base = %d\n", flow_id_base); 384 } 385 386 fdqring_id = K3_RINGACC_RING_ID_ANY; 387 for (i = 0; i < rx_cfg.flow_id_num; i++) { 388 struct k3_ring_cfg rxring_cfg = { 389 .elm_size = K3_RINGACC_RING_ELSIZE_8, 390 .mode = K3_RINGACC_RING_MODE_RING, 391 .flags = 0, 392 }; 393 struct k3_ring_cfg fdqring_cfg = { 394 .elm_size = K3_RINGACC_RING_ELSIZE_8, 395 .flags = K3_RINGACC_RING_SHARED, 396 }; 397 struct k3_udma_glue_rx_flow_cfg rx_flow_cfg = { 398 .rx_cfg = rxring_cfg, 399 .rxfdq_cfg = fdqring_cfg, 400 .ring_rxq_id = K3_RINGACC_RING_ID_ANY, 401 .src_tag_lo_sel = 402 K3_UDMA_GLUE_SRC_TAG_LO_USE_REMOTE_SRC_TAG, 403 }; 404 405 rx_flow_cfg.ring_rxfdq0_id = fdqring_id; 406 rx_flow_cfg.rx_cfg.size = max_desc_num; 407 rx_flow_cfg.rxfdq_cfg.size = max_desc_num; 408 rx_flow_cfg.rxfdq_cfg.mode = emac->prueth->pdata.fdqring_mode; 409 410 ret = k3_udma_glue_rx_flow_init(rx_chn->rx_chn, 411 i, &rx_flow_cfg); 412 if (ret) { 413 netdev_err(ndev, "Failed to init rx flow%d %d\n", 414 i, ret); 415 goto fail; 416 } 417 if (!i) 418 fdqring_id = k3_udma_glue_rx_flow_get_fdq_id(rx_chn->rx_chn, 419 i); 420 rx_chn->irq[i] = k3_udma_glue_rx_get_irq(rx_chn->rx_chn, i); 421 if (rx_chn->irq[i] <= 0) { 422 ret = rx_chn->irq[i]; 423 netdev_err(ndev, "Failed to get rx dma irq"); 424 goto fail; 425 } 426 } 427 428 return 0; 429 430 fail: 431 prueth_cleanup_rx_chns(emac, rx_chn, max_rflows); 432 return ret; 433 } 434 435 int prueth_dma_rx_push(struct prueth_emac *emac, 436 struct sk_buff *skb, 437 struct prueth_rx_chn *rx_chn) 438 { 439 struct net_device *ndev = emac->ndev; 440 struct cppi5_host_desc_t *desc_rx; 441 u32 pkt_len = skb_tailroom(skb); 442 dma_addr_t desc_dma; 443 dma_addr_t buf_dma; 444 void **swdata; 445 446 desc_rx = k3_cppi_desc_pool_alloc(rx_chn->desc_pool); 447 if (!desc_rx) { 448 netdev_err(ndev, "rx push: failed to allocate descriptor\n"); 449 return -ENOMEM; 450 } 451 desc_dma = k3_cppi_desc_pool_virt2dma(rx_chn->desc_pool, desc_rx); 452 453 buf_dma = dma_map_single(rx_chn->dma_dev, skb->data, pkt_len, DMA_FROM_DEVICE); 454 if (unlikely(dma_mapping_error(rx_chn->dma_dev, buf_dma))) { 455 k3_cppi_desc_pool_free(rx_chn->desc_pool, desc_rx); 456 netdev_err(ndev, "rx push: failed to map rx pkt buffer\n"); 457 return -EINVAL; 458 } 459 460 cppi5_hdesc_init(desc_rx, CPPI5_INFO0_HDESC_EPIB_PRESENT, 461 PRUETH_NAV_PS_DATA_SIZE); 462 k3_udma_glue_rx_dma_to_cppi5_addr(rx_chn->rx_chn, &buf_dma); 463 cppi5_hdesc_attach_buf(desc_rx, buf_dma, skb_tailroom(skb), buf_dma, skb_tailroom(skb)); 464 465 swdata = cppi5_hdesc_get_swdata(desc_rx); 466 *swdata = skb; 467 468 return k3_udma_glue_push_rx_chn(rx_chn->rx_chn, 0, 469 desc_rx, desc_dma); 470 } 471 472 u64 icssg_ts_to_ns(u32 hi_sw, u32 hi, u32 lo, u32 cycle_time_ns) 473 { 474 u32 iepcount_lo, iepcount_hi, hi_rollover_count; 475 u64 ns; 476 477 iepcount_lo = lo & GENMASK(19, 0); 478 iepcount_hi = (hi & GENMASK(11, 0)) << 12 | lo >> 20; 479 hi_rollover_count = hi >> 11; 480 481 ns = ((u64)hi_rollover_count) << 23 | (iepcount_hi + hi_sw); 482 ns = ns * cycle_time_ns + iepcount_lo; 483 484 return ns; 485 } 486 487 void emac_rx_timestamp(struct prueth_emac *emac, 488 struct sk_buff *skb, u32 *psdata) 489 { 490 struct skb_shared_hwtstamps *ssh; 491 u64 ns; 492 493 if (emac->is_sr1) { 494 ns = (u64)psdata[1] << 32 | psdata[0]; 495 } else { 496 u32 hi_sw = readl(emac->prueth->shram.va + 497 TIMESYNC_FW_WC_COUNT_HI_SW_OFFSET_OFFSET); 498 ns = icssg_ts_to_ns(hi_sw, psdata[1], psdata[0], 499 IEP_DEFAULT_CYCLE_TIME_NS); 500 } 501 502 ssh = skb_hwtstamps(skb); 503 memset(ssh, 0, sizeof(*ssh)); 504 ssh->hwtstamp = ns_to_ktime(ns); 505 } 506 507 static int emac_rx_packet(struct prueth_emac *emac, u32 flow_id) 508 { 509 struct prueth_rx_chn *rx_chn = &emac->rx_chns; 510 u32 buf_dma_len, pkt_len, port_id = 0; 511 struct net_device *ndev = emac->ndev; 512 struct cppi5_host_desc_t *desc_rx; 513 struct sk_buff *skb, *new_skb; 514 dma_addr_t desc_dma, buf_dma; 515 void **swdata; 516 u32 *psdata; 517 int ret; 518 519 ret = k3_udma_glue_pop_rx_chn(rx_chn->rx_chn, flow_id, &desc_dma); 520 if (ret) { 521 if (ret != -ENODATA) 522 netdev_err(ndev, "rx pop: failed: %d\n", ret); 523 return ret; 524 } 525 526 if (cppi5_desc_is_tdcm(desc_dma)) /* Teardown ? */ 527 return 0; 528 529 desc_rx = k3_cppi_desc_pool_dma2virt(rx_chn->desc_pool, desc_dma); 530 531 swdata = cppi5_hdesc_get_swdata(desc_rx); 532 skb = *swdata; 533 534 psdata = cppi5_hdesc_get_psdata(desc_rx); 535 /* RX HW timestamp */ 536 if (emac->rx_ts_enabled) 537 emac_rx_timestamp(emac, skb, psdata); 538 539 cppi5_hdesc_get_obuf(desc_rx, &buf_dma, &buf_dma_len); 540 k3_udma_glue_rx_cppi5_to_dma_addr(rx_chn->rx_chn, &buf_dma); 541 pkt_len = cppi5_hdesc_get_pktlen(desc_rx); 542 /* firmware adds 4 CRC bytes, strip them */ 543 pkt_len -= 4; 544 cppi5_desc_get_tags_ids(&desc_rx->hdr, &port_id, NULL); 545 546 dma_unmap_single(rx_chn->dma_dev, buf_dma, buf_dma_len, DMA_FROM_DEVICE); 547 k3_cppi_desc_pool_free(rx_chn->desc_pool, desc_rx); 548 549 skb->dev = ndev; 550 new_skb = netdev_alloc_skb_ip_align(ndev, PRUETH_MAX_PKT_SIZE); 551 /* if allocation fails we drop the packet but push the 552 * descriptor back to the ring with old skb to prevent a stall 553 */ 554 if (!new_skb) { 555 ndev->stats.rx_dropped++; 556 new_skb = skb; 557 } else { 558 /* send the filled skb up the n/w stack */ 559 skb_put(skb, pkt_len); 560 skb->protocol = eth_type_trans(skb, ndev); 561 napi_gro_receive(&emac->napi_rx, skb); 562 ndev->stats.rx_bytes += pkt_len; 563 ndev->stats.rx_packets++; 564 } 565 566 /* queue another RX DMA */ 567 ret = prueth_dma_rx_push(emac, new_skb, &emac->rx_chns); 568 if (WARN_ON(ret < 0)) { 569 dev_kfree_skb_any(new_skb); 570 ndev->stats.rx_errors++; 571 ndev->stats.rx_dropped++; 572 } 573 574 return ret; 575 } 576 577 static void prueth_rx_cleanup(void *data, dma_addr_t desc_dma) 578 { 579 struct prueth_rx_chn *rx_chn = data; 580 struct cppi5_host_desc_t *desc_rx; 581 struct sk_buff *skb; 582 dma_addr_t buf_dma; 583 u32 buf_dma_len; 584 void **swdata; 585 586 desc_rx = k3_cppi_desc_pool_dma2virt(rx_chn->desc_pool, desc_dma); 587 swdata = cppi5_hdesc_get_swdata(desc_rx); 588 skb = *swdata; 589 cppi5_hdesc_get_obuf(desc_rx, &buf_dma, &buf_dma_len); 590 k3_udma_glue_rx_cppi5_to_dma_addr(rx_chn->rx_chn, &buf_dma); 591 592 dma_unmap_single(rx_chn->dma_dev, buf_dma, buf_dma_len, 593 DMA_FROM_DEVICE); 594 k3_cppi_desc_pool_free(rx_chn->desc_pool, desc_rx); 595 596 dev_kfree_skb_any(skb); 597 } 598 599 static int prueth_tx_ts_cookie_get(struct prueth_emac *emac) 600 { 601 int i; 602 603 /* search and get the next free slot */ 604 for (i = 0; i < PRUETH_MAX_TX_TS_REQUESTS; i++) { 605 if (!emac->tx_ts_skb[i]) { 606 emac->tx_ts_skb[i] = ERR_PTR(-EBUSY); /* reserve slot */ 607 return i; 608 } 609 } 610 611 return -EBUSY; 612 } 613 614 /** 615 * emac_ndo_start_xmit - EMAC Transmit function 616 * @skb: SKB pointer 617 * @ndev: EMAC network adapter 618 * 619 * Called by the system to transmit a packet - we queue the packet in 620 * EMAC hardware transmit queue 621 * Doesn't wait for completion we'll check for TX completion in 622 * emac_tx_complete_packets(). 623 * 624 * Return: enum netdev_tx 625 */ 626 enum netdev_tx emac_ndo_start_xmit(struct sk_buff *skb, struct net_device *ndev) 627 { 628 struct cppi5_host_desc_t *first_desc, *next_desc, *cur_desc; 629 struct prueth_emac *emac = netdev_priv(ndev); 630 struct netdev_queue *netif_txq; 631 struct prueth_tx_chn *tx_chn; 632 dma_addr_t desc_dma, buf_dma; 633 int i, ret = 0, q_idx; 634 bool in_tx_ts = 0; 635 int tx_ts_cookie; 636 void **swdata; 637 u32 pkt_len; 638 u32 *epib; 639 640 pkt_len = skb_headlen(skb); 641 q_idx = skb_get_queue_mapping(skb); 642 643 tx_chn = &emac->tx_chns[q_idx]; 644 netif_txq = netdev_get_tx_queue(ndev, q_idx); 645 646 /* Map the linear buffer */ 647 buf_dma = dma_map_single(tx_chn->dma_dev, skb->data, pkt_len, DMA_TO_DEVICE); 648 if (dma_mapping_error(tx_chn->dma_dev, buf_dma)) { 649 netdev_err(ndev, "tx: failed to map skb buffer\n"); 650 ret = NETDEV_TX_OK; 651 goto drop_free_skb; 652 } 653 654 first_desc = k3_cppi_desc_pool_alloc(tx_chn->desc_pool); 655 if (!first_desc) { 656 netdev_dbg(ndev, "tx: failed to allocate descriptor\n"); 657 dma_unmap_single(tx_chn->dma_dev, buf_dma, pkt_len, DMA_TO_DEVICE); 658 goto drop_stop_q_busy; 659 } 660 661 cppi5_hdesc_init(first_desc, CPPI5_INFO0_HDESC_EPIB_PRESENT, 662 PRUETH_NAV_PS_DATA_SIZE); 663 cppi5_hdesc_set_pkttype(first_desc, 0); 664 epib = first_desc->epib; 665 epib[0] = 0; 666 epib[1] = 0; 667 if (skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP && 668 emac->tx_ts_enabled) { 669 tx_ts_cookie = prueth_tx_ts_cookie_get(emac); 670 if (tx_ts_cookie >= 0) { 671 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; 672 /* Request TX timestamp */ 673 epib[0] = (u32)tx_ts_cookie; 674 epib[1] = 0x80000000; /* TX TS request */ 675 emac->tx_ts_skb[tx_ts_cookie] = skb_get(skb); 676 in_tx_ts = 1; 677 } 678 } 679 680 /* set dst tag to indicate internal qid at the firmware which is at 681 * bit8..bit15. bit0..bit7 indicates port num for directed 682 * packets in case of switch mode operation 683 */ 684 cppi5_desc_set_tags_ids(&first_desc->hdr, 0, (emac->port_id | (q_idx << 8))); 685 k3_udma_glue_tx_dma_to_cppi5_addr(tx_chn->tx_chn, &buf_dma); 686 cppi5_hdesc_attach_buf(first_desc, buf_dma, pkt_len, buf_dma, pkt_len); 687 swdata = cppi5_hdesc_get_swdata(first_desc); 688 *swdata = skb; 689 690 /* Handle the case where skb is fragmented in pages */ 691 cur_desc = first_desc; 692 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 693 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 694 u32 frag_size = skb_frag_size(frag); 695 696 next_desc = k3_cppi_desc_pool_alloc(tx_chn->desc_pool); 697 if (!next_desc) { 698 netdev_err(ndev, 699 "tx: failed to allocate frag. descriptor\n"); 700 goto free_desc_stop_q_busy_cleanup_tx_ts; 701 } 702 703 buf_dma = skb_frag_dma_map(tx_chn->dma_dev, frag, 0, frag_size, 704 DMA_TO_DEVICE); 705 if (dma_mapping_error(tx_chn->dma_dev, buf_dma)) { 706 netdev_err(ndev, "tx: Failed to map skb page\n"); 707 k3_cppi_desc_pool_free(tx_chn->desc_pool, next_desc); 708 ret = NETDEV_TX_OK; 709 goto cleanup_tx_ts; 710 } 711 712 cppi5_hdesc_reset_hbdesc(next_desc); 713 k3_udma_glue_tx_dma_to_cppi5_addr(tx_chn->tx_chn, &buf_dma); 714 cppi5_hdesc_attach_buf(next_desc, 715 buf_dma, frag_size, buf_dma, frag_size); 716 717 desc_dma = k3_cppi_desc_pool_virt2dma(tx_chn->desc_pool, 718 next_desc); 719 k3_udma_glue_tx_dma_to_cppi5_addr(tx_chn->tx_chn, &desc_dma); 720 cppi5_hdesc_link_hbdesc(cur_desc, desc_dma); 721 722 pkt_len += frag_size; 723 cur_desc = next_desc; 724 } 725 WARN_ON_ONCE(pkt_len != skb->len); 726 727 /* report bql before sending packet */ 728 netdev_tx_sent_queue(netif_txq, pkt_len); 729 730 cppi5_hdesc_set_pktlen(first_desc, pkt_len); 731 desc_dma = k3_cppi_desc_pool_virt2dma(tx_chn->desc_pool, first_desc); 732 /* cppi5_desc_dump(first_desc, 64); */ 733 734 skb_tx_timestamp(skb); /* SW timestamp if SKBTX_IN_PROGRESS not set */ 735 ret = k3_udma_glue_push_tx_chn(tx_chn->tx_chn, first_desc, desc_dma); 736 if (ret) { 737 netdev_err(ndev, "tx: push failed: %d\n", ret); 738 goto drop_free_descs; 739 } 740 741 if (in_tx_ts) 742 atomic_inc(&emac->tx_ts_pending); 743 744 if (k3_cppi_desc_pool_avail(tx_chn->desc_pool) < MAX_SKB_FRAGS) { 745 netif_tx_stop_queue(netif_txq); 746 /* Barrier, so that stop_queue visible to other cpus */ 747 smp_mb__after_atomic(); 748 749 if (k3_cppi_desc_pool_avail(tx_chn->desc_pool) >= 750 MAX_SKB_FRAGS) 751 netif_tx_wake_queue(netif_txq); 752 } 753 754 return NETDEV_TX_OK; 755 756 cleanup_tx_ts: 757 if (in_tx_ts) { 758 dev_kfree_skb_any(emac->tx_ts_skb[tx_ts_cookie]); 759 emac->tx_ts_skb[tx_ts_cookie] = NULL; 760 } 761 762 drop_free_descs: 763 prueth_xmit_free(tx_chn, first_desc); 764 765 drop_free_skb: 766 dev_kfree_skb_any(skb); 767 768 /* error */ 769 ndev->stats.tx_dropped++; 770 netdev_err(ndev, "tx: error: %d\n", ret); 771 772 return ret; 773 774 free_desc_stop_q_busy_cleanup_tx_ts: 775 if (in_tx_ts) { 776 dev_kfree_skb_any(emac->tx_ts_skb[tx_ts_cookie]); 777 emac->tx_ts_skb[tx_ts_cookie] = NULL; 778 } 779 prueth_xmit_free(tx_chn, first_desc); 780 781 drop_stop_q_busy: 782 netif_tx_stop_queue(netif_txq); 783 return NETDEV_TX_BUSY; 784 } 785 786 static void prueth_tx_cleanup(void *data, dma_addr_t desc_dma) 787 { 788 struct prueth_tx_chn *tx_chn = data; 789 struct cppi5_host_desc_t *desc_tx; 790 struct sk_buff *skb; 791 void **swdata; 792 793 desc_tx = k3_cppi_desc_pool_dma2virt(tx_chn->desc_pool, desc_dma); 794 swdata = cppi5_hdesc_get_swdata(desc_tx); 795 skb = *(swdata); 796 prueth_xmit_free(tx_chn, desc_tx); 797 798 dev_kfree_skb_any(skb); 799 } 800 801 irqreturn_t prueth_rx_irq(int irq, void *dev_id) 802 { 803 struct prueth_emac *emac = dev_id; 804 805 disable_irq_nosync(irq); 806 napi_schedule(&emac->napi_rx); 807 808 return IRQ_HANDLED; 809 } 810 811 void prueth_emac_stop(struct prueth_emac *emac) 812 { 813 struct prueth *prueth = emac->prueth; 814 int slice; 815 816 switch (emac->port_id) { 817 case PRUETH_PORT_MII0: 818 slice = ICSS_SLICE0; 819 break; 820 case PRUETH_PORT_MII1: 821 slice = ICSS_SLICE1; 822 break; 823 default: 824 netdev_err(emac->ndev, "invalid port\n"); 825 return; 826 } 827 828 emac->fw_running = 0; 829 if (!emac->is_sr1) 830 rproc_shutdown(prueth->txpru[slice]); 831 rproc_shutdown(prueth->rtu[slice]); 832 rproc_shutdown(prueth->pru[slice]); 833 } 834 835 void prueth_cleanup_tx_ts(struct prueth_emac *emac) 836 { 837 int i; 838 839 for (i = 0; i < PRUETH_MAX_TX_TS_REQUESTS; i++) { 840 if (emac->tx_ts_skb[i]) { 841 dev_kfree_skb_any(emac->tx_ts_skb[i]); 842 emac->tx_ts_skb[i] = NULL; 843 } 844 } 845 } 846 847 int emac_napi_rx_poll(struct napi_struct *napi_rx, int budget) 848 { 849 struct prueth_emac *emac = prueth_napi_to_emac(napi_rx); 850 int rx_flow = emac->is_sr1 ? 851 PRUETH_RX_FLOW_DATA_SR1 : PRUETH_RX_FLOW_DATA; 852 int flow = emac->is_sr1 ? 853 PRUETH_MAX_RX_FLOWS_SR1 : PRUETH_MAX_RX_FLOWS; 854 int num_rx = 0; 855 int cur_budget; 856 int ret; 857 858 while (flow--) { 859 cur_budget = budget - num_rx; 860 861 while (cur_budget--) { 862 ret = emac_rx_packet(emac, flow); 863 if (ret) 864 break; 865 num_rx++; 866 } 867 868 if (num_rx >= budget) 869 break; 870 } 871 872 if (num_rx < budget && napi_complete_done(napi_rx, num_rx)) 873 enable_irq(emac->rx_chns.irq[rx_flow]); 874 875 return num_rx; 876 } 877 878 int prueth_prepare_rx_chan(struct prueth_emac *emac, 879 struct prueth_rx_chn *chn, 880 int buf_size) 881 { 882 struct sk_buff *skb; 883 int i, ret; 884 885 for (i = 0; i < chn->descs_num; i++) { 886 skb = __netdev_alloc_skb_ip_align(NULL, buf_size, GFP_KERNEL); 887 if (!skb) 888 return -ENOMEM; 889 890 ret = prueth_dma_rx_push(emac, skb, chn); 891 if (ret < 0) { 892 netdev_err(emac->ndev, 893 "cannot submit skb for rx chan %s ret %d\n", 894 chn->name, ret); 895 kfree_skb(skb); 896 return ret; 897 } 898 } 899 900 return 0; 901 } 902 903 void prueth_reset_tx_chan(struct prueth_emac *emac, int ch_num, 904 bool free_skb) 905 { 906 int i; 907 908 for (i = 0; i < ch_num; i++) { 909 if (free_skb) 910 k3_udma_glue_reset_tx_chn(emac->tx_chns[i].tx_chn, 911 &emac->tx_chns[i], 912 prueth_tx_cleanup); 913 k3_udma_glue_disable_tx_chn(emac->tx_chns[i].tx_chn); 914 } 915 } 916 917 void prueth_reset_rx_chan(struct prueth_rx_chn *chn, 918 int num_flows, bool disable) 919 { 920 int i; 921 922 for (i = 0; i < num_flows; i++) 923 k3_udma_glue_reset_rx_chn(chn->rx_chn, i, chn, 924 prueth_rx_cleanup, !!i); 925 if (disable) 926 k3_udma_glue_disable_rx_chn(chn->rx_chn); 927 } 928 929 void emac_ndo_tx_timeout(struct net_device *ndev, unsigned int txqueue) 930 { 931 ndev->stats.tx_errors++; 932 } 933 934 static int emac_set_ts_config(struct net_device *ndev, struct ifreq *ifr) 935 { 936 struct prueth_emac *emac = netdev_priv(ndev); 937 struct hwtstamp_config config; 938 939 if (copy_from_user(&config, ifr->ifr_data, sizeof(config))) 940 return -EFAULT; 941 942 switch (config.tx_type) { 943 case HWTSTAMP_TX_OFF: 944 emac->tx_ts_enabled = 0; 945 break; 946 case HWTSTAMP_TX_ON: 947 emac->tx_ts_enabled = 1; 948 break; 949 default: 950 return -ERANGE; 951 } 952 953 switch (config.rx_filter) { 954 case HWTSTAMP_FILTER_NONE: 955 emac->rx_ts_enabled = 0; 956 break; 957 case HWTSTAMP_FILTER_ALL: 958 case HWTSTAMP_FILTER_SOME: 959 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT: 960 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC: 961 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ: 962 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT: 963 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC: 964 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ: 965 case HWTSTAMP_FILTER_PTP_V2_L2_EVENT: 966 case HWTSTAMP_FILTER_PTP_V2_L2_SYNC: 967 case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ: 968 case HWTSTAMP_FILTER_PTP_V2_EVENT: 969 case HWTSTAMP_FILTER_PTP_V2_SYNC: 970 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ: 971 case HWTSTAMP_FILTER_NTP_ALL: 972 emac->rx_ts_enabled = 1; 973 config.rx_filter = HWTSTAMP_FILTER_ALL; 974 break; 975 default: 976 return -ERANGE; 977 } 978 979 return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ? 980 -EFAULT : 0; 981 } 982 983 static int emac_get_ts_config(struct net_device *ndev, struct ifreq *ifr) 984 { 985 struct prueth_emac *emac = netdev_priv(ndev); 986 struct hwtstamp_config config; 987 988 config.flags = 0; 989 config.tx_type = emac->tx_ts_enabled ? HWTSTAMP_TX_ON : HWTSTAMP_TX_OFF; 990 config.rx_filter = emac->rx_ts_enabled ? HWTSTAMP_FILTER_ALL : HWTSTAMP_FILTER_NONE; 991 992 return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ? 993 -EFAULT : 0; 994 } 995 996 int emac_ndo_ioctl(struct net_device *ndev, struct ifreq *ifr, int cmd) 997 { 998 switch (cmd) { 999 case SIOCGHWTSTAMP: 1000 return emac_get_ts_config(ndev, ifr); 1001 case SIOCSHWTSTAMP: 1002 return emac_set_ts_config(ndev, ifr); 1003 default: 1004 break; 1005 } 1006 1007 return phy_do_ioctl(ndev, ifr, cmd); 1008 } 1009 1010 void emac_ndo_get_stats64(struct net_device *ndev, 1011 struct rtnl_link_stats64 *stats) 1012 { 1013 struct prueth_emac *emac = netdev_priv(ndev); 1014 1015 emac_update_hardware_stats(emac); 1016 1017 stats->rx_packets = emac_get_stat_by_name(emac, "rx_packets"); 1018 stats->rx_bytes = emac_get_stat_by_name(emac, "rx_bytes"); 1019 stats->tx_packets = emac_get_stat_by_name(emac, "tx_packets"); 1020 stats->tx_bytes = emac_get_stat_by_name(emac, "tx_bytes"); 1021 stats->rx_crc_errors = emac_get_stat_by_name(emac, "rx_crc_errors"); 1022 stats->rx_over_errors = emac_get_stat_by_name(emac, "rx_over_errors"); 1023 stats->multicast = emac_get_stat_by_name(emac, "rx_multicast_frames"); 1024 1025 stats->rx_errors = ndev->stats.rx_errors; 1026 stats->rx_dropped = ndev->stats.rx_dropped; 1027 stats->tx_errors = ndev->stats.tx_errors; 1028 stats->tx_dropped = ndev->stats.tx_dropped; 1029 } 1030 1031 int emac_ndo_get_phys_port_name(struct net_device *ndev, char *name, 1032 size_t len) 1033 { 1034 struct prueth_emac *emac = netdev_priv(ndev); 1035 int ret; 1036 1037 ret = snprintf(name, len, "p%d", emac->port_id); 1038 if (ret >= len) 1039 return -EINVAL; 1040 1041 return 0; 1042 } 1043 1044 /* get emac_port corresponding to eth_node name */ 1045 int prueth_node_port(struct device_node *eth_node) 1046 { 1047 u32 port_id; 1048 int ret; 1049 1050 ret = of_property_read_u32(eth_node, "reg", &port_id); 1051 if (ret) 1052 return ret; 1053 1054 if (port_id == 0) 1055 return PRUETH_PORT_MII0; 1056 else if (port_id == 1) 1057 return PRUETH_PORT_MII1; 1058 else 1059 return PRUETH_PORT_INVALID; 1060 } 1061 1062 /* get MAC instance corresponding to eth_node name */ 1063 int prueth_node_mac(struct device_node *eth_node) 1064 { 1065 u32 port_id; 1066 int ret; 1067 1068 ret = of_property_read_u32(eth_node, "reg", &port_id); 1069 if (ret) 1070 return ret; 1071 1072 if (port_id == 0) 1073 return PRUETH_MAC0; 1074 else if (port_id == 1) 1075 return PRUETH_MAC1; 1076 else 1077 return PRUETH_MAC_INVALID; 1078 } 1079 1080 void prueth_netdev_exit(struct prueth *prueth, 1081 struct device_node *eth_node) 1082 { 1083 struct prueth_emac *emac; 1084 enum prueth_mac mac; 1085 1086 mac = prueth_node_mac(eth_node); 1087 if (mac == PRUETH_MAC_INVALID) 1088 return; 1089 1090 emac = prueth->emac[mac]; 1091 if (!emac) 1092 return; 1093 1094 if (of_phy_is_fixed_link(emac->phy_node)) 1095 of_phy_deregister_fixed_link(emac->phy_node); 1096 1097 netif_napi_del(&emac->napi_rx); 1098 1099 pruss_release_mem_region(prueth->pruss, &emac->dram); 1100 destroy_workqueue(emac->cmd_wq); 1101 free_netdev(emac->ndev); 1102 prueth->emac[mac] = NULL; 1103 } 1104 1105 int prueth_get_cores(struct prueth *prueth, int slice, bool is_sr1) 1106 { 1107 struct device *dev = prueth->dev; 1108 enum pruss_pru_id pruss_id; 1109 struct device_node *np; 1110 int idx = -1, ret; 1111 1112 np = dev->of_node; 1113 1114 switch (slice) { 1115 case ICSS_SLICE0: 1116 idx = 0; 1117 break; 1118 case ICSS_SLICE1: 1119 idx = is_sr1 ? 2 : 3; 1120 break; 1121 default: 1122 return -EINVAL; 1123 } 1124 1125 prueth->pru[slice] = pru_rproc_get(np, idx, &pruss_id); 1126 if (IS_ERR(prueth->pru[slice])) { 1127 ret = PTR_ERR(prueth->pru[slice]); 1128 prueth->pru[slice] = NULL; 1129 return dev_err_probe(dev, ret, "unable to get PRU%d\n", slice); 1130 } 1131 prueth->pru_id[slice] = pruss_id; 1132 1133 idx++; 1134 prueth->rtu[slice] = pru_rproc_get(np, idx, NULL); 1135 if (IS_ERR(prueth->rtu[slice])) { 1136 ret = PTR_ERR(prueth->rtu[slice]); 1137 prueth->rtu[slice] = NULL; 1138 return dev_err_probe(dev, ret, "unable to get RTU%d\n", slice); 1139 } 1140 1141 if (is_sr1) 1142 return 0; 1143 1144 idx++; 1145 prueth->txpru[slice] = pru_rproc_get(np, idx, NULL); 1146 if (IS_ERR(prueth->txpru[slice])) { 1147 ret = PTR_ERR(prueth->txpru[slice]); 1148 prueth->txpru[slice] = NULL; 1149 return dev_err_probe(dev, ret, "unable to get TX_PRU%d\n", slice); 1150 } 1151 1152 return 0; 1153 } 1154 1155 void prueth_put_cores(struct prueth *prueth, int slice) 1156 { 1157 if (prueth->txpru[slice]) 1158 pru_rproc_put(prueth->txpru[slice]); 1159 1160 if (prueth->rtu[slice]) 1161 pru_rproc_put(prueth->rtu[slice]); 1162 1163 if (prueth->pru[slice]) 1164 pru_rproc_put(prueth->pru[slice]); 1165 } 1166 1167 #ifdef CONFIG_PM_SLEEP 1168 static int prueth_suspend(struct device *dev) 1169 { 1170 struct prueth *prueth = dev_get_drvdata(dev); 1171 struct net_device *ndev; 1172 int i, ret; 1173 1174 for (i = 0; i < PRUETH_NUM_MACS; i++) { 1175 ndev = prueth->registered_netdevs[i]; 1176 1177 if (!ndev) 1178 continue; 1179 1180 if (netif_running(ndev)) { 1181 netif_device_detach(ndev); 1182 ret = ndev->netdev_ops->ndo_stop(ndev); 1183 if (ret < 0) { 1184 netdev_err(ndev, "failed to stop: %d", ret); 1185 return ret; 1186 } 1187 } 1188 } 1189 1190 return 0; 1191 } 1192 1193 static int prueth_resume(struct device *dev) 1194 { 1195 struct prueth *prueth = dev_get_drvdata(dev); 1196 struct net_device *ndev; 1197 int i, ret; 1198 1199 for (i = 0; i < PRUETH_NUM_MACS; i++) { 1200 ndev = prueth->registered_netdevs[i]; 1201 1202 if (!ndev) 1203 continue; 1204 1205 if (netif_running(ndev)) { 1206 ret = ndev->netdev_ops->ndo_open(ndev); 1207 if (ret < 0) { 1208 netdev_err(ndev, "failed to start: %d", ret); 1209 return ret; 1210 } 1211 netif_device_attach(ndev); 1212 } 1213 } 1214 1215 return 0; 1216 } 1217 #endif /* CONFIG_PM_SLEEP */ 1218 1219 const struct dev_pm_ops prueth_dev_pm_ops = { 1220 SET_SYSTEM_SLEEP_PM_OPS(prueth_suspend, prueth_resume) 1221 }; 1222