1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * K3 NAVSS DMA glue interface 4 * 5 * Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com 6 * 7 */ 8 9 #include <linux/atomic.h> 10 #include <linux/delay.h> 11 #include <linux/dma-mapping.h> 12 #include <linux/io.h> 13 #include <linux/init.h> 14 #include <linux/of.h> 15 #include <linux/platform_device.h> 16 #include <linux/soc/ti/k3-ringacc.h> 17 #include <linux/dma/ti-cppi5.h> 18 #include <linux/dma/k3-udma-glue.h> 19 20 #include "k3-udma.h" 21 #include "k3-psil-priv.h" 22 23 struct k3_udma_glue_common { 24 struct device *dev; 25 struct udma_dev *udmax; 26 const struct udma_tisci_rm *tisci_rm; 27 struct k3_ringacc *ringacc; 28 u32 src_thread; 29 u32 dst_thread; 30 31 u32 hdesc_size; 32 bool epib; 33 u32 psdata_size; 34 u32 swdata_size; 35 }; 36 37 struct k3_udma_glue_tx_channel { 38 struct k3_udma_glue_common common; 39 40 struct udma_tchan *udma_tchanx; 41 int udma_tchan_id; 42 43 struct k3_ring *ringtx; 44 struct k3_ring *ringtxcq; 45 46 bool psil_paired; 47 48 int virq; 49 50 atomic_t free_pkts; 51 bool tx_pause_on_err; 52 bool tx_filt_einfo; 53 bool tx_filt_pswords; 54 bool tx_supr_tdpkt; 55 }; 56 57 struct k3_udma_glue_rx_flow { 58 struct udma_rflow *udma_rflow; 59 int udma_rflow_id; 60 struct k3_ring *ringrx; 61 struct k3_ring *ringrxfdq; 62 63 int virq; 64 }; 65 66 struct k3_udma_glue_rx_channel { 67 struct k3_udma_glue_common common; 68 69 struct udma_rchan *udma_rchanx; 70 int udma_rchan_id; 71 bool remote; 72 73 bool psil_paired; 74 75 u32 swdata_size; 76 int flow_id_base; 77 78 struct k3_udma_glue_rx_flow *flows; 79 u32 flow_num; 80 u32 flows_ready; 81 }; 82 83 #define K3_UDMAX_TDOWN_TIMEOUT_US 1000 84 85 static int of_k3_udma_glue_parse(struct device_node *udmax_np, 86 struct k3_udma_glue_common *common) 87 { 88 common->ringacc = of_k3_ringacc_get_by_phandle(udmax_np, 89 "ti,ringacc"); 90 if (IS_ERR(common->ringacc)) 91 return PTR_ERR(common->ringacc); 92 93 common->udmax = of_xudma_dev_get(udmax_np, NULL); 94 if (IS_ERR(common->udmax)) 95 return PTR_ERR(common->udmax); 96 97 common->tisci_rm = xudma_dev_get_tisci_rm(common->udmax); 98 99 return 0; 100 } 101 102 static int of_k3_udma_glue_parse_chn(struct device_node *chn_np, 103 const char *name, struct k3_udma_glue_common *common, 104 bool tx_chn) 105 { 106 struct psil_endpoint_config *ep_config; 107 struct of_phandle_args dma_spec; 108 u32 thread_id; 109 int ret = 0; 110 int index; 111 112 if (unlikely(!name)) 113 return -EINVAL; 114 115 index = of_property_match_string(chn_np, "dma-names", name); 116 if (index < 0) 117 return index; 118 119 if (of_parse_phandle_with_args(chn_np, "dmas", "#dma-cells", index, 120 &dma_spec)) 121 return -ENOENT; 122 123 thread_id = dma_spec.args[0]; 124 125 if (tx_chn && !(thread_id & K3_PSIL_DST_THREAD_ID_OFFSET)) { 126 ret = -EINVAL; 127 goto out_put_spec; 128 } 129 130 if (!tx_chn && (thread_id & K3_PSIL_DST_THREAD_ID_OFFSET)) { 131 ret = -EINVAL; 132 goto out_put_spec; 133 } 134 135 /* get psil endpoint config */ 136 ep_config = psil_get_ep_config(thread_id); 137 if (IS_ERR(ep_config)) { 138 dev_err(common->dev, 139 "No configuration for psi-l thread 0x%04x\n", 140 thread_id); 141 ret = PTR_ERR(ep_config); 142 goto out_put_spec; 143 } 144 145 common->epib = ep_config->needs_epib; 146 common->psdata_size = ep_config->psd_size; 147 148 if (tx_chn) 149 common->dst_thread = thread_id; 150 else 151 common->src_thread = thread_id; 152 153 ret = of_k3_udma_glue_parse(dma_spec.np, common); 154 155 out_put_spec: 156 of_node_put(dma_spec.np); 157 return ret; 158 }; 159 160 static void k3_udma_glue_dump_tx_chn(struct k3_udma_glue_tx_channel *tx_chn) 161 { 162 struct device *dev = tx_chn->common.dev; 163 164 dev_dbg(dev, "dump_tx_chn:\n" 165 "udma_tchan_id: %d\n" 166 "src_thread: %08x\n" 167 "dst_thread: %08x\n", 168 tx_chn->udma_tchan_id, 169 tx_chn->common.src_thread, 170 tx_chn->common.dst_thread); 171 } 172 173 static void k3_udma_glue_dump_tx_rt_chn(struct k3_udma_glue_tx_channel *chn, 174 char *mark) 175 { 176 struct device *dev = chn->common.dev; 177 178 dev_dbg(dev, "=== dump ===> %s\n", mark); 179 dev_dbg(dev, "0x%08X: %08X\n", UDMA_TCHAN_RT_CTL_REG, 180 xudma_tchanrt_read(chn->udma_tchanx, UDMA_TCHAN_RT_CTL_REG)); 181 dev_dbg(dev, "0x%08X: %08X\n", UDMA_TCHAN_RT_PEER_RT_EN_REG, 182 xudma_tchanrt_read(chn->udma_tchanx, 183 UDMA_TCHAN_RT_PEER_RT_EN_REG)); 184 dev_dbg(dev, "0x%08X: %08X\n", UDMA_TCHAN_RT_PCNT_REG, 185 xudma_tchanrt_read(chn->udma_tchanx, UDMA_TCHAN_RT_PCNT_REG)); 186 dev_dbg(dev, "0x%08X: %08X\n", UDMA_TCHAN_RT_BCNT_REG, 187 xudma_tchanrt_read(chn->udma_tchanx, UDMA_TCHAN_RT_BCNT_REG)); 188 dev_dbg(dev, "0x%08X: %08X\n", UDMA_TCHAN_RT_SBCNT_REG, 189 xudma_tchanrt_read(chn->udma_tchanx, UDMA_TCHAN_RT_SBCNT_REG)); 190 } 191 192 static int k3_udma_glue_cfg_tx_chn(struct k3_udma_glue_tx_channel *tx_chn) 193 { 194 const struct udma_tisci_rm *tisci_rm = tx_chn->common.tisci_rm; 195 struct ti_sci_msg_rm_udmap_tx_ch_cfg req; 196 197 memset(&req, 0, sizeof(req)); 198 199 req.valid_params = TI_SCI_MSG_VALUE_RM_UDMAP_CH_PAUSE_ON_ERR_VALID | 200 TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_FILT_EINFO_VALID | 201 TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_FILT_PSWORDS_VALID | 202 TI_SCI_MSG_VALUE_RM_UDMAP_CH_CHAN_TYPE_VALID | 203 TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_SUPR_TDPKT_VALID | 204 TI_SCI_MSG_VALUE_RM_UDMAP_CH_FETCH_SIZE_VALID | 205 TI_SCI_MSG_VALUE_RM_UDMAP_CH_CQ_QNUM_VALID; 206 req.nav_id = tisci_rm->tisci_dev_id; 207 req.index = tx_chn->udma_tchan_id; 208 if (tx_chn->tx_pause_on_err) 209 req.tx_pause_on_err = 1; 210 if (tx_chn->tx_filt_einfo) 211 req.tx_filt_einfo = 1; 212 if (tx_chn->tx_filt_pswords) 213 req.tx_filt_pswords = 1; 214 req.tx_chan_type = TI_SCI_RM_UDMAP_CHAN_TYPE_PKT_PBRR; 215 if (tx_chn->tx_supr_tdpkt) 216 req.tx_supr_tdpkt = 1; 217 req.tx_fetch_size = tx_chn->common.hdesc_size >> 2; 218 req.txcq_qnum = k3_ringacc_get_ring_id(tx_chn->ringtxcq); 219 220 return tisci_rm->tisci_udmap_ops->tx_ch_cfg(tisci_rm->tisci, &req); 221 } 222 223 struct k3_udma_glue_tx_channel *k3_udma_glue_request_tx_chn(struct device *dev, 224 const char *name, struct k3_udma_glue_tx_channel_cfg *cfg) 225 { 226 struct k3_udma_glue_tx_channel *tx_chn; 227 int ret; 228 229 tx_chn = devm_kzalloc(dev, sizeof(*tx_chn), GFP_KERNEL); 230 if (!tx_chn) 231 return ERR_PTR(-ENOMEM); 232 233 tx_chn->common.dev = dev; 234 tx_chn->common.swdata_size = cfg->swdata_size; 235 tx_chn->tx_pause_on_err = cfg->tx_pause_on_err; 236 tx_chn->tx_filt_einfo = cfg->tx_filt_einfo; 237 tx_chn->tx_filt_pswords = cfg->tx_filt_pswords; 238 tx_chn->tx_supr_tdpkt = cfg->tx_supr_tdpkt; 239 240 /* parse of udmap channel */ 241 ret = of_k3_udma_glue_parse_chn(dev->of_node, name, 242 &tx_chn->common, true); 243 if (ret) 244 goto err; 245 246 tx_chn->common.hdesc_size = cppi5_hdesc_calc_size(tx_chn->common.epib, 247 tx_chn->common.psdata_size, 248 tx_chn->common.swdata_size); 249 250 /* request and cfg UDMAP TX channel */ 251 tx_chn->udma_tchanx = xudma_tchan_get(tx_chn->common.udmax, -1); 252 if (IS_ERR(tx_chn->udma_tchanx)) { 253 ret = PTR_ERR(tx_chn->udma_tchanx); 254 dev_err(dev, "UDMAX tchanx get err %d\n", ret); 255 goto err; 256 } 257 tx_chn->udma_tchan_id = xudma_tchan_get_id(tx_chn->udma_tchanx); 258 259 atomic_set(&tx_chn->free_pkts, cfg->txcq_cfg.size); 260 261 /* request and cfg rings */ 262 tx_chn->ringtx = k3_ringacc_request_ring(tx_chn->common.ringacc, 263 tx_chn->udma_tchan_id, 0); 264 if (!tx_chn->ringtx) { 265 ret = -ENODEV; 266 dev_err(dev, "Failed to get TX ring %u\n", 267 tx_chn->udma_tchan_id); 268 goto err; 269 } 270 271 tx_chn->ringtxcq = k3_ringacc_request_ring(tx_chn->common.ringacc, 272 -1, 0); 273 if (!tx_chn->ringtxcq) { 274 ret = -ENODEV; 275 dev_err(dev, "Failed to get TXCQ ring\n"); 276 goto err; 277 } 278 279 ret = k3_ringacc_ring_cfg(tx_chn->ringtx, &cfg->tx_cfg); 280 if (ret) { 281 dev_err(dev, "Failed to cfg ringtx %d\n", ret); 282 goto err; 283 } 284 285 ret = k3_ringacc_ring_cfg(tx_chn->ringtxcq, &cfg->txcq_cfg); 286 if (ret) { 287 dev_err(dev, "Failed to cfg ringtx %d\n", ret); 288 goto err; 289 } 290 291 /* request and cfg psi-l */ 292 tx_chn->common.src_thread = 293 xudma_dev_get_psil_base(tx_chn->common.udmax) + 294 tx_chn->udma_tchan_id; 295 296 ret = k3_udma_glue_cfg_tx_chn(tx_chn); 297 if (ret) { 298 dev_err(dev, "Failed to cfg tchan %d\n", ret); 299 goto err; 300 } 301 302 ret = xudma_navss_psil_pair(tx_chn->common.udmax, 303 tx_chn->common.src_thread, 304 tx_chn->common.dst_thread); 305 if (ret) { 306 dev_err(dev, "PSI-L request err %d\n", ret); 307 goto err; 308 } 309 310 tx_chn->psil_paired = true; 311 312 /* reset TX RT registers */ 313 k3_udma_glue_disable_tx_chn(tx_chn); 314 315 k3_udma_glue_dump_tx_chn(tx_chn); 316 317 return tx_chn; 318 319 err: 320 k3_udma_glue_release_tx_chn(tx_chn); 321 return ERR_PTR(ret); 322 } 323 EXPORT_SYMBOL_GPL(k3_udma_glue_request_tx_chn); 324 325 void k3_udma_glue_release_tx_chn(struct k3_udma_glue_tx_channel *tx_chn) 326 { 327 if (tx_chn->psil_paired) { 328 xudma_navss_psil_unpair(tx_chn->common.udmax, 329 tx_chn->common.src_thread, 330 tx_chn->common.dst_thread); 331 tx_chn->psil_paired = false; 332 } 333 334 if (!IS_ERR_OR_NULL(tx_chn->udma_tchanx)) 335 xudma_tchan_put(tx_chn->common.udmax, 336 tx_chn->udma_tchanx); 337 338 if (tx_chn->ringtxcq) 339 k3_ringacc_ring_free(tx_chn->ringtxcq); 340 341 if (tx_chn->ringtx) 342 k3_ringacc_ring_free(tx_chn->ringtx); 343 } 344 EXPORT_SYMBOL_GPL(k3_udma_glue_release_tx_chn); 345 346 int k3_udma_glue_push_tx_chn(struct k3_udma_glue_tx_channel *tx_chn, 347 struct cppi5_host_desc_t *desc_tx, 348 dma_addr_t desc_dma) 349 { 350 u32 ringtxcq_id; 351 352 if (!atomic_add_unless(&tx_chn->free_pkts, -1, 0)) 353 return -ENOMEM; 354 355 ringtxcq_id = k3_ringacc_get_ring_id(tx_chn->ringtxcq); 356 cppi5_desc_set_retpolicy(&desc_tx->hdr, 0, ringtxcq_id); 357 358 return k3_ringacc_ring_push(tx_chn->ringtx, &desc_dma); 359 } 360 EXPORT_SYMBOL_GPL(k3_udma_glue_push_tx_chn); 361 362 int k3_udma_glue_pop_tx_chn(struct k3_udma_glue_tx_channel *tx_chn, 363 dma_addr_t *desc_dma) 364 { 365 int ret; 366 367 ret = k3_ringacc_ring_pop(tx_chn->ringtxcq, desc_dma); 368 if (!ret) 369 atomic_inc(&tx_chn->free_pkts); 370 371 return ret; 372 } 373 EXPORT_SYMBOL_GPL(k3_udma_glue_pop_tx_chn); 374 375 int k3_udma_glue_enable_tx_chn(struct k3_udma_glue_tx_channel *tx_chn) 376 { 377 u32 txrt_ctl; 378 379 txrt_ctl = UDMA_PEER_RT_EN_ENABLE; 380 xudma_tchanrt_write(tx_chn->udma_tchanx, 381 UDMA_TCHAN_RT_PEER_RT_EN_REG, 382 txrt_ctl); 383 384 txrt_ctl = xudma_tchanrt_read(tx_chn->udma_tchanx, 385 UDMA_TCHAN_RT_CTL_REG); 386 txrt_ctl |= UDMA_CHAN_RT_CTL_EN; 387 xudma_tchanrt_write(tx_chn->udma_tchanx, UDMA_TCHAN_RT_CTL_REG, 388 txrt_ctl); 389 390 k3_udma_glue_dump_tx_rt_chn(tx_chn, "txchn en"); 391 return 0; 392 } 393 EXPORT_SYMBOL_GPL(k3_udma_glue_enable_tx_chn); 394 395 void k3_udma_glue_disable_tx_chn(struct k3_udma_glue_tx_channel *tx_chn) 396 { 397 k3_udma_glue_dump_tx_rt_chn(tx_chn, "txchn dis1"); 398 399 xudma_tchanrt_write(tx_chn->udma_tchanx, UDMA_TCHAN_RT_CTL_REG, 0); 400 401 xudma_tchanrt_write(tx_chn->udma_tchanx, 402 UDMA_TCHAN_RT_PEER_RT_EN_REG, 0); 403 k3_udma_glue_dump_tx_rt_chn(tx_chn, "txchn dis2"); 404 } 405 EXPORT_SYMBOL_GPL(k3_udma_glue_disable_tx_chn); 406 407 void k3_udma_glue_tdown_tx_chn(struct k3_udma_glue_tx_channel *tx_chn, 408 bool sync) 409 { 410 int i = 0; 411 u32 val; 412 413 k3_udma_glue_dump_tx_rt_chn(tx_chn, "txchn tdown1"); 414 415 xudma_tchanrt_write(tx_chn->udma_tchanx, UDMA_TCHAN_RT_CTL_REG, 416 UDMA_CHAN_RT_CTL_EN | UDMA_CHAN_RT_CTL_TDOWN); 417 418 val = xudma_tchanrt_read(tx_chn->udma_tchanx, UDMA_TCHAN_RT_CTL_REG); 419 420 while (sync && (val & UDMA_CHAN_RT_CTL_EN)) { 421 val = xudma_tchanrt_read(tx_chn->udma_tchanx, 422 UDMA_TCHAN_RT_CTL_REG); 423 udelay(1); 424 if (i > K3_UDMAX_TDOWN_TIMEOUT_US) { 425 dev_err(tx_chn->common.dev, "TX tdown timeout\n"); 426 break; 427 } 428 i++; 429 } 430 431 val = xudma_tchanrt_read(tx_chn->udma_tchanx, 432 UDMA_TCHAN_RT_PEER_RT_EN_REG); 433 if (sync && (val & UDMA_PEER_RT_EN_ENABLE)) 434 dev_err(tx_chn->common.dev, "TX tdown peer not stopped\n"); 435 k3_udma_glue_dump_tx_rt_chn(tx_chn, "txchn tdown2"); 436 } 437 EXPORT_SYMBOL_GPL(k3_udma_glue_tdown_tx_chn); 438 439 void k3_udma_glue_reset_tx_chn(struct k3_udma_glue_tx_channel *tx_chn, 440 void *data, 441 void (*cleanup)(void *data, dma_addr_t desc_dma)) 442 { 443 dma_addr_t desc_dma; 444 int occ_tx, i, ret; 445 446 /* reset TXCQ as it is not input for udma - expected to be empty */ 447 if (tx_chn->ringtxcq) 448 k3_ringacc_ring_reset(tx_chn->ringtxcq); 449 450 /* 451 * TXQ reset need to be special way as it is input for udma and its 452 * state cached by udma, so: 453 * 1) save TXQ occ 454 * 2) clean up TXQ and call callback .cleanup() for each desc 455 * 3) reset TXQ in a special way 456 */ 457 occ_tx = k3_ringacc_ring_get_occ(tx_chn->ringtx); 458 dev_dbg(tx_chn->common.dev, "TX reset occ_tx %u\n", occ_tx); 459 460 for (i = 0; i < occ_tx; i++) { 461 ret = k3_ringacc_ring_pop(tx_chn->ringtx, &desc_dma); 462 if (ret) { 463 dev_err(tx_chn->common.dev, "TX reset pop %d\n", ret); 464 break; 465 } 466 cleanup(data, desc_dma); 467 } 468 469 k3_ringacc_ring_reset_dma(tx_chn->ringtx, occ_tx); 470 } 471 EXPORT_SYMBOL_GPL(k3_udma_glue_reset_tx_chn); 472 473 u32 k3_udma_glue_tx_get_hdesc_size(struct k3_udma_glue_tx_channel *tx_chn) 474 { 475 return tx_chn->common.hdesc_size; 476 } 477 EXPORT_SYMBOL_GPL(k3_udma_glue_tx_get_hdesc_size); 478 479 u32 k3_udma_glue_tx_get_txcq_id(struct k3_udma_glue_tx_channel *tx_chn) 480 { 481 return k3_ringacc_get_ring_id(tx_chn->ringtxcq); 482 } 483 EXPORT_SYMBOL_GPL(k3_udma_glue_tx_get_txcq_id); 484 485 int k3_udma_glue_tx_get_irq(struct k3_udma_glue_tx_channel *tx_chn) 486 { 487 tx_chn->virq = k3_ringacc_get_ring_irq_num(tx_chn->ringtxcq); 488 489 return tx_chn->virq; 490 } 491 EXPORT_SYMBOL_GPL(k3_udma_glue_tx_get_irq); 492 493 static int k3_udma_glue_cfg_rx_chn(struct k3_udma_glue_rx_channel *rx_chn) 494 { 495 const struct udma_tisci_rm *tisci_rm = rx_chn->common.tisci_rm; 496 struct ti_sci_msg_rm_udmap_rx_ch_cfg req; 497 int ret; 498 499 memset(&req, 0, sizeof(req)); 500 501 req.valid_params = TI_SCI_MSG_VALUE_RM_UDMAP_CH_FETCH_SIZE_VALID | 502 TI_SCI_MSG_VALUE_RM_UDMAP_CH_CQ_QNUM_VALID | 503 TI_SCI_MSG_VALUE_RM_UDMAP_CH_CHAN_TYPE_VALID | 504 TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_FLOWID_START_VALID | 505 TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_FLOWID_CNT_VALID; 506 507 req.nav_id = tisci_rm->tisci_dev_id; 508 req.index = rx_chn->udma_rchan_id; 509 req.rx_fetch_size = rx_chn->common.hdesc_size >> 2; 510 /* 511 * TODO: we can't support rxcq_qnum/RCHAN[a]_RCQ cfg with current sysfw 512 * and udmax impl, so just configure it to invalid value. 513 * req.rxcq_qnum = k3_ringacc_get_ring_id(rx_chn->flows[0].ringrx); 514 */ 515 req.rxcq_qnum = 0xFFFF; 516 if (rx_chn->flow_num && rx_chn->flow_id_base != rx_chn->udma_rchan_id) { 517 /* Default flow + extra ones */ 518 req.flowid_start = rx_chn->flow_id_base; 519 req.flowid_cnt = rx_chn->flow_num; 520 } 521 req.rx_chan_type = TI_SCI_RM_UDMAP_CHAN_TYPE_PKT_PBRR; 522 523 ret = tisci_rm->tisci_udmap_ops->rx_ch_cfg(tisci_rm->tisci, &req); 524 if (ret) 525 dev_err(rx_chn->common.dev, "rchan%d cfg failed %d\n", 526 rx_chn->udma_rchan_id, ret); 527 528 return ret; 529 } 530 531 static void k3_udma_glue_release_rx_flow(struct k3_udma_glue_rx_channel *rx_chn, 532 u32 flow_num) 533 { 534 struct k3_udma_glue_rx_flow *flow = &rx_chn->flows[flow_num]; 535 536 if (IS_ERR_OR_NULL(flow->udma_rflow)) 537 return; 538 539 if (flow->ringrxfdq) 540 k3_ringacc_ring_free(flow->ringrxfdq); 541 542 if (flow->ringrx) 543 k3_ringacc_ring_free(flow->ringrx); 544 545 xudma_rflow_put(rx_chn->common.udmax, flow->udma_rflow); 546 flow->udma_rflow = NULL; 547 rx_chn->flows_ready--; 548 } 549 550 static int k3_udma_glue_cfg_rx_flow(struct k3_udma_glue_rx_channel *rx_chn, 551 u32 flow_idx, 552 struct k3_udma_glue_rx_flow_cfg *flow_cfg) 553 { 554 struct k3_udma_glue_rx_flow *flow = &rx_chn->flows[flow_idx]; 555 const struct udma_tisci_rm *tisci_rm = rx_chn->common.tisci_rm; 556 struct device *dev = rx_chn->common.dev; 557 struct ti_sci_msg_rm_udmap_flow_cfg req; 558 int rx_ring_id; 559 int rx_ringfdq_id; 560 int ret = 0; 561 562 flow->udma_rflow = xudma_rflow_get(rx_chn->common.udmax, 563 flow->udma_rflow_id); 564 if (IS_ERR(flow->udma_rflow)) { 565 ret = PTR_ERR(flow->udma_rflow); 566 dev_err(dev, "UDMAX rflow get err %d\n", ret); 567 return ret; 568 } 569 570 if (flow->udma_rflow_id != xudma_rflow_get_id(flow->udma_rflow)) { 571 ret = -ENODEV; 572 goto err_rflow_put; 573 } 574 575 /* request and cfg rings */ 576 flow->ringrx = k3_ringacc_request_ring(rx_chn->common.ringacc, 577 flow_cfg->ring_rxq_id, 0); 578 if (!flow->ringrx) { 579 ret = -ENODEV; 580 dev_err(dev, "Failed to get RX ring\n"); 581 goto err_rflow_put; 582 } 583 584 flow->ringrxfdq = k3_ringacc_request_ring(rx_chn->common.ringacc, 585 flow_cfg->ring_rxfdq0_id, 0); 586 if (!flow->ringrxfdq) { 587 ret = -ENODEV; 588 dev_err(dev, "Failed to get RXFDQ ring\n"); 589 goto err_ringrx_free; 590 } 591 592 ret = k3_ringacc_ring_cfg(flow->ringrx, &flow_cfg->rx_cfg); 593 if (ret) { 594 dev_err(dev, "Failed to cfg ringrx %d\n", ret); 595 goto err_ringrxfdq_free; 596 } 597 598 ret = k3_ringacc_ring_cfg(flow->ringrxfdq, &flow_cfg->rxfdq_cfg); 599 if (ret) { 600 dev_err(dev, "Failed to cfg ringrxfdq %d\n", ret); 601 goto err_ringrxfdq_free; 602 } 603 604 if (rx_chn->remote) { 605 rx_ring_id = TI_SCI_RESOURCE_NULL; 606 rx_ringfdq_id = TI_SCI_RESOURCE_NULL; 607 } else { 608 rx_ring_id = k3_ringacc_get_ring_id(flow->ringrx); 609 rx_ringfdq_id = k3_ringacc_get_ring_id(flow->ringrxfdq); 610 } 611 612 memset(&req, 0, sizeof(req)); 613 614 req.valid_params = 615 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_EINFO_PRESENT_VALID | 616 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_PSINFO_PRESENT_VALID | 617 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_ERROR_HANDLING_VALID | 618 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DESC_TYPE_VALID | 619 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DEST_QNUM_VALID | 620 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_SRC_TAG_HI_SEL_VALID | 621 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_SRC_TAG_LO_SEL_VALID | 622 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DEST_TAG_HI_SEL_VALID | 623 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DEST_TAG_LO_SEL_VALID | 624 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ0_SZ0_QNUM_VALID | 625 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ1_QNUM_VALID | 626 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ2_QNUM_VALID | 627 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ3_QNUM_VALID; 628 req.nav_id = tisci_rm->tisci_dev_id; 629 req.flow_index = flow->udma_rflow_id; 630 if (rx_chn->common.epib) 631 req.rx_einfo_present = 1; 632 if (rx_chn->common.psdata_size) 633 req.rx_psinfo_present = 1; 634 if (flow_cfg->rx_error_handling) 635 req.rx_error_handling = 1; 636 req.rx_desc_type = 0; 637 req.rx_dest_qnum = rx_ring_id; 638 req.rx_src_tag_hi_sel = 0; 639 req.rx_src_tag_lo_sel = flow_cfg->src_tag_lo_sel; 640 req.rx_dest_tag_hi_sel = 0; 641 req.rx_dest_tag_lo_sel = 0; 642 req.rx_fdq0_sz0_qnum = rx_ringfdq_id; 643 req.rx_fdq1_qnum = rx_ringfdq_id; 644 req.rx_fdq2_qnum = rx_ringfdq_id; 645 req.rx_fdq3_qnum = rx_ringfdq_id; 646 647 ret = tisci_rm->tisci_udmap_ops->rx_flow_cfg(tisci_rm->tisci, &req); 648 if (ret) { 649 dev_err(dev, "flow%d config failed: %d\n", flow->udma_rflow_id, 650 ret); 651 goto err_ringrxfdq_free; 652 } 653 654 rx_chn->flows_ready++; 655 dev_dbg(dev, "flow%d config done. ready:%d\n", 656 flow->udma_rflow_id, rx_chn->flows_ready); 657 658 return 0; 659 660 err_ringrxfdq_free: 661 k3_ringacc_ring_free(flow->ringrxfdq); 662 663 err_ringrx_free: 664 k3_ringacc_ring_free(flow->ringrx); 665 666 err_rflow_put: 667 xudma_rflow_put(rx_chn->common.udmax, flow->udma_rflow); 668 flow->udma_rflow = NULL; 669 670 return ret; 671 } 672 673 static void k3_udma_glue_dump_rx_chn(struct k3_udma_glue_rx_channel *chn) 674 { 675 struct device *dev = chn->common.dev; 676 677 dev_dbg(dev, "dump_rx_chn:\n" 678 "udma_rchan_id: %d\n" 679 "src_thread: %08x\n" 680 "dst_thread: %08x\n" 681 "epib: %d\n" 682 "hdesc_size: %u\n" 683 "psdata_size: %u\n" 684 "swdata_size: %u\n" 685 "flow_id_base: %d\n" 686 "flow_num: %d\n", 687 chn->udma_rchan_id, 688 chn->common.src_thread, 689 chn->common.dst_thread, 690 chn->common.epib, 691 chn->common.hdesc_size, 692 chn->common.psdata_size, 693 chn->common.swdata_size, 694 chn->flow_id_base, 695 chn->flow_num); 696 } 697 698 static void k3_udma_glue_dump_rx_rt_chn(struct k3_udma_glue_rx_channel *chn, 699 char *mark) 700 { 701 struct device *dev = chn->common.dev; 702 703 dev_dbg(dev, "=== dump ===> %s\n", mark); 704 705 dev_dbg(dev, "0x%08X: %08X\n", UDMA_RCHAN_RT_CTL_REG, 706 xudma_rchanrt_read(chn->udma_rchanx, UDMA_RCHAN_RT_CTL_REG)); 707 dev_dbg(dev, "0x%08X: %08X\n", UDMA_RCHAN_RT_PEER_RT_EN_REG, 708 xudma_rchanrt_read(chn->udma_rchanx, 709 UDMA_RCHAN_RT_PEER_RT_EN_REG)); 710 dev_dbg(dev, "0x%08X: %08X\n", UDMA_RCHAN_RT_PCNT_REG, 711 xudma_rchanrt_read(chn->udma_rchanx, UDMA_RCHAN_RT_PCNT_REG)); 712 dev_dbg(dev, "0x%08X: %08X\n", UDMA_RCHAN_RT_BCNT_REG, 713 xudma_rchanrt_read(chn->udma_rchanx, UDMA_RCHAN_RT_BCNT_REG)); 714 dev_dbg(dev, "0x%08X: %08X\n", UDMA_RCHAN_RT_SBCNT_REG, 715 xudma_rchanrt_read(chn->udma_rchanx, UDMA_RCHAN_RT_SBCNT_REG)); 716 } 717 718 static int 719 k3_udma_glue_allocate_rx_flows(struct k3_udma_glue_rx_channel *rx_chn, 720 struct k3_udma_glue_rx_channel_cfg *cfg) 721 { 722 int ret; 723 724 /* default rflow */ 725 if (cfg->flow_id_use_rxchan_id) 726 return 0; 727 728 /* not a GP rflows */ 729 if (rx_chn->flow_id_base != -1 && 730 !xudma_rflow_is_gp(rx_chn->common.udmax, rx_chn->flow_id_base)) 731 return 0; 732 733 /* Allocate range of GP rflows */ 734 ret = xudma_alloc_gp_rflow_range(rx_chn->common.udmax, 735 rx_chn->flow_id_base, 736 rx_chn->flow_num); 737 if (ret < 0) { 738 dev_err(rx_chn->common.dev, "UDMAX reserve_rflow %d cnt:%d err: %d\n", 739 rx_chn->flow_id_base, rx_chn->flow_num, ret); 740 return ret; 741 } 742 rx_chn->flow_id_base = ret; 743 744 return 0; 745 } 746 747 static struct k3_udma_glue_rx_channel * 748 k3_udma_glue_request_rx_chn_priv(struct device *dev, const char *name, 749 struct k3_udma_glue_rx_channel_cfg *cfg) 750 { 751 struct k3_udma_glue_rx_channel *rx_chn; 752 int ret, i; 753 754 if (cfg->flow_id_num <= 0) 755 return ERR_PTR(-EINVAL); 756 757 if (cfg->flow_id_num != 1 && 758 (cfg->def_flow_cfg || cfg->flow_id_use_rxchan_id)) 759 return ERR_PTR(-EINVAL); 760 761 rx_chn = devm_kzalloc(dev, sizeof(*rx_chn), GFP_KERNEL); 762 if (!rx_chn) 763 return ERR_PTR(-ENOMEM); 764 765 rx_chn->common.dev = dev; 766 rx_chn->common.swdata_size = cfg->swdata_size; 767 rx_chn->remote = false; 768 769 /* parse of udmap channel */ 770 ret = of_k3_udma_glue_parse_chn(dev->of_node, name, 771 &rx_chn->common, false); 772 if (ret) 773 goto err; 774 775 rx_chn->common.hdesc_size = cppi5_hdesc_calc_size(rx_chn->common.epib, 776 rx_chn->common.psdata_size, 777 rx_chn->common.swdata_size); 778 779 /* request and cfg UDMAP RX channel */ 780 rx_chn->udma_rchanx = xudma_rchan_get(rx_chn->common.udmax, -1); 781 if (IS_ERR(rx_chn->udma_rchanx)) { 782 ret = PTR_ERR(rx_chn->udma_rchanx); 783 dev_err(dev, "UDMAX rchanx get err %d\n", ret); 784 goto err; 785 } 786 rx_chn->udma_rchan_id = xudma_rchan_get_id(rx_chn->udma_rchanx); 787 788 rx_chn->flow_num = cfg->flow_id_num; 789 rx_chn->flow_id_base = cfg->flow_id_base; 790 791 /* Use RX channel id as flow id: target dev can't generate flow_id */ 792 if (cfg->flow_id_use_rxchan_id) 793 rx_chn->flow_id_base = rx_chn->udma_rchan_id; 794 795 rx_chn->flows = devm_kcalloc(dev, rx_chn->flow_num, 796 sizeof(*rx_chn->flows), GFP_KERNEL); 797 if (!rx_chn->flows) { 798 ret = -ENOMEM; 799 goto err; 800 } 801 802 ret = k3_udma_glue_allocate_rx_flows(rx_chn, cfg); 803 if (ret) 804 goto err; 805 806 for (i = 0; i < rx_chn->flow_num; i++) 807 rx_chn->flows[i].udma_rflow_id = rx_chn->flow_id_base + i; 808 809 /* request and cfg psi-l */ 810 rx_chn->common.dst_thread = 811 xudma_dev_get_psil_base(rx_chn->common.udmax) + 812 rx_chn->udma_rchan_id; 813 814 ret = k3_udma_glue_cfg_rx_chn(rx_chn); 815 if (ret) { 816 dev_err(dev, "Failed to cfg rchan %d\n", ret); 817 goto err; 818 } 819 820 /* init default RX flow only if flow_num = 1 */ 821 if (cfg->def_flow_cfg) { 822 ret = k3_udma_glue_cfg_rx_flow(rx_chn, 0, cfg->def_flow_cfg); 823 if (ret) 824 goto err; 825 } 826 827 ret = xudma_navss_psil_pair(rx_chn->common.udmax, 828 rx_chn->common.src_thread, 829 rx_chn->common.dst_thread); 830 if (ret) { 831 dev_err(dev, "PSI-L request err %d\n", ret); 832 goto err; 833 } 834 835 rx_chn->psil_paired = true; 836 837 /* reset RX RT registers */ 838 k3_udma_glue_disable_rx_chn(rx_chn); 839 840 k3_udma_glue_dump_rx_chn(rx_chn); 841 842 return rx_chn; 843 844 err: 845 k3_udma_glue_release_rx_chn(rx_chn); 846 return ERR_PTR(ret); 847 } 848 849 static struct k3_udma_glue_rx_channel * 850 k3_udma_glue_request_remote_rx_chn(struct device *dev, const char *name, 851 struct k3_udma_glue_rx_channel_cfg *cfg) 852 { 853 struct k3_udma_glue_rx_channel *rx_chn; 854 int ret, i; 855 856 if (cfg->flow_id_num <= 0 || 857 cfg->flow_id_use_rxchan_id || 858 cfg->def_flow_cfg || 859 cfg->flow_id_base < 0) 860 return ERR_PTR(-EINVAL); 861 862 /* 863 * Remote RX channel is under control of Remote CPU core, so 864 * Linux can only request and manipulate by dedicated RX flows 865 */ 866 867 rx_chn = devm_kzalloc(dev, sizeof(*rx_chn), GFP_KERNEL); 868 if (!rx_chn) 869 return ERR_PTR(-ENOMEM); 870 871 rx_chn->common.dev = dev; 872 rx_chn->common.swdata_size = cfg->swdata_size; 873 rx_chn->remote = true; 874 rx_chn->udma_rchan_id = -1; 875 rx_chn->flow_num = cfg->flow_id_num; 876 rx_chn->flow_id_base = cfg->flow_id_base; 877 rx_chn->psil_paired = false; 878 879 /* parse of udmap channel */ 880 ret = of_k3_udma_glue_parse_chn(dev->of_node, name, 881 &rx_chn->common, false); 882 if (ret) 883 goto err; 884 885 rx_chn->common.hdesc_size = cppi5_hdesc_calc_size(rx_chn->common.epib, 886 rx_chn->common.psdata_size, 887 rx_chn->common.swdata_size); 888 889 rx_chn->flows = devm_kcalloc(dev, rx_chn->flow_num, 890 sizeof(*rx_chn->flows), GFP_KERNEL); 891 if (!rx_chn->flows) { 892 ret = -ENOMEM; 893 goto err; 894 } 895 896 ret = k3_udma_glue_allocate_rx_flows(rx_chn, cfg); 897 if (ret) 898 goto err; 899 900 for (i = 0; i < rx_chn->flow_num; i++) 901 rx_chn->flows[i].udma_rflow_id = rx_chn->flow_id_base + i; 902 903 k3_udma_glue_dump_rx_chn(rx_chn); 904 905 return rx_chn; 906 907 err: 908 k3_udma_glue_release_rx_chn(rx_chn); 909 return ERR_PTR(ret); 910 } 911 912 struct k3_udma_glue_rx_channel * 913 k3_udma_glue_request_rx_chn(struct device *dev, const char *name, 914 struct k3_udma_glue_rx_channel_cfg *cfg) 915 { 916 if (cfg->remote) 917 return k3_udma_glue_request_remote_rx_chn(dev, name, cfg); 918 else 919 return k3_udma_glue_request_rx_chn_priv(dev, name, cfg); 920 } 921 EXPORT_SYMBOL_GPL(k3_udma_glue_request_rx_chn); 922 923 void k3_udma_glue_release_rx_chn(struct k3_udma_glue_rx_channel *rx_chn) 924 { 925 int i; 926 927 if (IS_ERR_OR_NULL(rx_chn->common.udmax)) 928 return; 929 930 if (rx_chn->psil_paired) { 931 xudma_navss_psil_unpair(rx_chn->common.udmax, 932 rx_chn->common.src_thread, 933 rx_chn->common.dst_thread); 934 rx_chn->psil_paired = false; 935 } 936 937 for (i = 0; i < rx_chn->flow_num; i++) 938 k3_udma_glue_release_rx_flow(rx_chn, i); 939 940 if (xudma_rflow_is_gp(rx_chn->common.udmax, rx_chn->flow_id_base)) 941 xudma_free_gp_rflow_range(rx_chn->common.udmax, 942 rx_chn->flow_id_base, 943 rx_chn->flow_num); 944 945 if (!IS_ERR_OR_NULL(rx_chn->udma_rchanx)) 946 xudma_rchan_put(rx_chn->common.udmax, 947 rx_chn->udma_rchanx); 948 } 949 EXPORT_SYMBOL_GPL(k3_udma_glue_release_rx_chn); 950 951 int k3_udma_glue_rx_flow_init(struct k3_udma_glue_rx_channel *rx_chn, 952 u32 flow_idx, 953 struct k3_udma_glue_rx_flow_cfg *flow_cfg) 954 { 955 if (flow_idx >= rx_chn->flow_num) 956 return -EINVAL; 957 958 return k3_udma_glue_cfg_rx_flow(rx_chn, flow_idx, flow_cfg); 959 } 960 EXPORT_SYMBOL_GPL(k3_udma_glue_rx_flow_init); 961 962 u32 k3_udma_glue_rx_flow_get_fdq_id(struct k3_udma_glue_rx_channel *rx_chn, 963 u32 flow_idx) 964 { 965 struct k3_udma_glue_rx_flow *flow; 966 967 if (flow_idx >= rx_chn->flow_num) 968 return -EINVAL; 969 970 flow = &rx_chn->flows[flow_idx]; 971 972 return k3_ringacc_get_ring_id(flow->ringrxfdq); 973 } 974 EXPORT_SYMBOL_GPL(k3_udma_glue_rx_flow_get_fdq_id); 975 976 u32 k3_udma_glue_rx_get_flow_id_base(struct k3_udma_glue_rx_channel *rx_chn) 977 { 978 return rx_chn->flow_id_base; 979 } 980 EXPORT_SYMBOL_GPL(k3_udma_glue_rx_get_flow_id_base); 981 982 int k3_udma_glue_rx_flow_enable(struct k3_udma_glue_rx_channel *rx_chn, 983 u32 flow_idx) 984 { 985 struct k3_udma_glue_rx_flow *flow = &rx_chn->flows[flow_idx]; 986 const struct udma_tisci_rm *tisci_rm = rx_chn->common.tisci_rm; 987 struct device *dev = rx_chn->common.dev; 988 struct ti_sci_msg_rm_udmap_flow_cfg req; 989 int rx_ring_id; 990 int rx_ringfdq_id; 991 int ret = 0; 992 993 if (!rx_chn->remote) 994 return -EINVAL; 995 996 rx_ring_id = k3_ringacc_get_ring_id(flow->ringrx); 997 rx_ringfdq_id = k3_ringacc_get_ring_id(flow->ringrxfdq); 998 999 memset(&req, 0, sizeof(req)); 1000 1001 req.valid_params = 1002 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DEST_QNUM_VALID | 1003 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ0_SZ0_QNUM_VALID | 1004 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ1_QNUM_VALID | 1005 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ2_QNUM_VALID | 1006 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ3_QNUM_VALID; 1007 req.nav_id = tisci_rm->tisci_dev_id; 1008 req.flow_index = flow->udma_rflow_id; 1009 req.rx_dest_qnum = rx_ring_id; 1010 req.rx_fdq0_sz0_qnum = rx_ringfdq_id; 1011 req.rx_fdq1_qnum = rx_ringfdq_id; 1012 req.rx_fdq2_qnum = rx_ringfdq_id; 1013 req.rx_fdq3_qnum = rx_ringfdq_id; 1014 1015 ret = tisci_rm->tisci_udmap_ops->rx_flow_cfg(tisci_rm->tisci, &req); 1016 if (ret) { 1017 dev_err(dev, "flow%d enable failed: %d\n", flow->udma_rflow_id, 1018 ret); 1019 } 1020 1021 return ret; 1022 } 1023 EXPORT_SYMBOL_GPL(k3_udma_glue_rx_flow_enable); 1024 1025 int k3_udma_glue_rx_flow_disable(struct k3_udma_glue_rx_channel *rx_chn, 1026 u32 flow_idx) 1027 { 1028 struct k3_udma_glue_rx_flow *flow = &rx_chn->flows[flow_idx]; 1029 const struct udma_tisci_rm *tisci_rm = rx_chn->common.tisci_rm; 1030 struct device *dev = rx_chn->common.dev; 1031 struct ti_sci_msg_rm_udmap_flow_cfg req; 1032 int ret = 0; 1033 1034 if (!rx_chn->remote) 1035 return -EINVAL; 1036 1037 memset(&req, 0, sizeof(req)); 1038 req.valid_params = 1039 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DEST_QNUM_VALID | 1040 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ0_SZ0_QNUM_VALID | 1041 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ1_QNUM_VALID | 1042 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ2_QNUM_VALID | 1043 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ3_QNUM_VALID; 1044 req.nav_id = tisci_rm->tisci_dev_id; 1045 req.flow_index = flow->udma_rflow_id; 1046 req.rx_dest_qnum = TI_SCI_RESOURCE_NULL; 1047 req.rx_fdq0_sz0_qnum = TI_SCI_RESOURCE_NULL; 1048 req.rx_fdq1_qnum = TI_SCI_RESOURCE_NULL; 1049 req.rx_fdq2_qnum = TI_SCI_RESOURCE_NULL; 1050 req.rx_fdq3_qnum = TI_SCI_RESOURCE_NULL; 1051 1052 ret = tisci_rm->tisci_udmap_ops->rx_flow_cfg(tisci_rm->tisci, &req); 1053 if (ret) { 1054 dev_err(dev, "flow%d disable failed: %d\n", flow->udma_rflow_id, 1055 ret); 1056 } 1057 1058 return ret; 1059 } 1060 EXPORT_SYMBOL_GPL(k3_udma_glue_rx_flow_disable); 1061 1062 int k3_udma_glue_enable_rx_chn(struct k3_udma_glue_rx_channel *rx_chn) 1063 { 1064 u32 rxrt_ctl; 1065 1066 if (rx_chn->remote) 1067 return -EINVAL; 1068 1069 if (rx_chn->flows_ready < rx_chn->flow_num) 1070 return -EINVAL; 1071 1072 rxrt_ctl = xudma_rchanrt_read(rx_chn->udma_rchanx, 1073 UDMA_RCHAN_RT_CTL_REG); 1074 rxrt_ctl |= UDMA_CHAN_RT_CTL_EN; 1075 xudma_rchanrt_write(rx_chn->udma_rchanx, UDMA_RCHAN_RT_CTL_REG, 1076 rxrt_ctl); 1077 1078 xudma_rchanrt_write(rx_chn->udma_rchanx, 1079 UDMA_RCHAN_RT_PEER_RT_EN_REG, 1080 UDMA_PEER_RT_EN_ENABLE); 1081 1082 k3_udma_glue_dump_rx_rt_chn(rx_chn, "rxrt en"); 1083 return 0; 1084 } 1085 EXPORT_SYMBOL_GPL(k3_udma_glue_enable_rx_chn); 1086 1087 void k3_udma_glue_disable_rx_chn(struct k3_udma_glue_rx_channel *rx_chn) 1088 { 1089 k3_udma_glue_dump_rx_rt_chn(rx_chn, "rxrt dis1"); 1090 1091 xudma_rchanrt_write(rx_chn->udma_rchanx, 1092 UDMA_RCHAN_RT_PEER_RT_EN_REG, 1093 0); 1094 xudma_rchanrt_write(rx_chn->udma_rchanx, UDMA_RCHAN_RT_CTL_REG, 0); 1095 1096 k3_udma_glue_dump_rx_rt_chn(rx_chn, "rxrt dis2"); 1097 } 1098 EXPORT_SYMBOL_GPL(k3_udma_glue_disable_rx_chn); 1099 1100 void k3_udma_glue_tdown_rx_chn(struct k3_udma_glue_rx_channel *rx_chn, 1101 bool sync) 1102 { 1103 int i = 0; 1104 u32 val; 1105 1106 if (rx_chn->remote) 1107 return; 1108 1109 k3_udma_glue_dump_rx_rt_chn(rx_chn, "rxrt tdown1"); 1110 1111 xudma_rchanrt_write(rx_chn->udma_rchanx, UDMA_RCHAN_RT_PEER_RT_EN_REG, 1112 UDMA_PEER_RT_EN_ENABLE | UDMA_PEER_RT_EN_TEARDOWN); 1113 1114 val = xudma_rchanrt_read(rx_chn->udma_rchanx, UDMA_RCHAN_RT_CTL_REG); 1115 1116 while (sync && (val & UDMA_CHAN_RT_CTL_EN)) { 1117 val = xudma_rchanrt_read(rx_chn->udma_rchanx, 1118 UDMA_RCHAN_RT_CTL_REG); 1119 udelay(1); 1120 if (i > K3_UDMAX_TDOWN_TIMEOUT_US) { 1121 dev_err(rx_chn->common.dev, "RX tdown timeout\n"); 1122 break; 1123 } 1124 i++; 1125 } 1126 1127 val = xudma_rchanrt_read(rx_chn->udma_rchanx, 1128 UDMA_RCHAN_RT_PEER_RT_EN_REG); 1129 if (sync && (val & UDMA_PEER_RT_EN_ENABLE)) 1130 dev_err(rx_chn->common.dev, "TX tdown peer not stopped\n"); 1131 k3_udma_glue_dump_rx_rt_chn(rx_chn, "rxrt tdown2"); 1132 } 1133 EXPORT_SYMBOL_GPL(k3_udma_glue_tdown_rx_chn); 1134 1135 void k3_udma_glue_reset_rx_chn(struct k3_udma_glue_rx_channel *rx_chn, 1136 u32 flow_num, void *data, 1137 void (*cleanup)(void *data, dma_addr_t desc_dma), bool skip_fdq) 1138 { 1139 struct k3_udma_glue_rx_flow *flow = &rx_chn->flows[flow_num]; 1140 struct device *dev = rx_chn->common.dev; 1141 dma_addr_t desc_dma; 1142 int occ_rx, i, ret; 1143 1144 /* reset RXCQ as it is not input for udma - expected to be empty */ 1145 occ_rx = k3_ringacc_ring_get_occ(flow->ringrx); 1146 dev_dbg(dev, "RX reset flow %u occ_rx %u\n", flow_num, occ_rx); 1147 if (flow->ringrx) 1148 k3_ringacc_ring_reset(flow->ringrx); 1149 1150 /* Skip RX FDQ in case one FDQ is used for the set of flows */ 1151 if (skip_fdq) 1152 return; 1153 1154 /* 1155 * RX FDQ reset need to be special way as it is input for udma and its 1156 * state cached by udma, so: 1157 * 1) save RX FDQ occ 1158 * 2) clean up RX FDQ and call callback .cleanup() for each desc 1159 * 3) reset RX FDQ in a special way 1160 */ 1161 occ_rx = k3_ringacc_ring_get_occ(flow->ringrxfdq); 1162 dev_dbg(dev, "RX reset flow %u occ_rx_fdq %u\n", flow_num, occ_rx); 1163 1164 for (i = 0; i < occ_rx; i++) { 1165 ret = k3_ringacc_ring_pop(flow->ringrxfdq, &desc_dma); 1166 if (ret) { 1167 dev_err(dev, "RX reset pop %d\n", ret); 1168 break; 1169 } 1170 cleanup(data, desc_dma); 1171 } 1172 1173 k3_ringacc_ring_reset_dma(flow->ringrxfdq, occ_rx); 1174 } 1175 EXPORT_SYMBOL_GPL(k3_udma_glue_reset_rx_chn); 1176 1177 int k3_udma_glue_push_rx_chn(struct k3_udma_glue_rx_channel *rx_chn, 1178 u32 flow_num, struct cppi5_host_desc_t *desc_rx, 1179 dma_addr_t desc_dma) 1180 { 1181 struct k3_udma_glue_rx_flow *flow = &rx_chn->flows[flow_num]; 1182 1183 return k3_ringacc_ring_push(flow->ringrxfdq, &desc_dma); 1184 } 1185 EXPORT_SYMBOL_GPL(k3_udma_glue_push_rx_chn); 1186 1187 int k3_udma_glue_pop_rx_chn(struct k3_udma_glue_rx_channel *rx_chn, 1188 u32 flow_num, dma_addr_t *desc_dma) 1189 { 1190 struct k3_udma_glue_rx_flow *flow = &rx_chn->flows[flow_num]; 1191 1192 return k3_ringacc_ring_pop(flow->ringrx, desc_dma); 1193 } 1194 EXPORT_SYMBOL_GPL(k3_udma_glue_pop_rx_chn); 1195 1196 int k3_udma_glue_rx_get_irq(struct k3_udma_glue_rx_channel *rx_chn, 1197 u32 flow_num) 1198 { 1199 struct k3_udma_glue_rx_flow *flow; 1200 1201 flow = &rx_chn->flows[flow_num]; 1202 1203 flow->virq = k3_ringacc_get_ring_irq_num(flow->ringrx); 1204 1205 return flow->virq; 1206 } 1207 EXPORT_SYMBOL_GPL(k3_udma_glue_rx_get_irq); 1208