1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * K3 NAVSS DMA glue interface 4 * 5 * Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com 6 * 7 */ 8 9 #include <linux/atomic.h> 10 #include <linux/delay.h> 11 #include <linux/dma-mapping.h> 12 #include <linux/io.h> 13 #include <linux/init.h> 14 #include <linux/of.h> 15 #include <linux/platform_device.h> 16 #include <linux/soc/ti/k3-ringacc.h> 17 #include <linux/dma/ti-cppi5.h> 18 #include <linux/dma/k3-udma-glue.h> 19 20 #include "k3-udma.h" 21 #include "k3-psil-priv.h" 22 23 struct k3_udma_glue_common { 24 struct device *dev; 25 struct udma_dev *udmax; 26 const struct udma_tisci_rm *tisci_rm; 27 struct k3_ringacc *ringacc; 28 u32 src_thread; 29 u32 dst_thread; 30 31 u32 hdesc_size; 32 bool epib; 33 u32 psdata_size; 34 u32 swdata_size; 35 u32 atype; 36 }; 37 38 struct k3_udma_glue_tx_channel { 39 struct k3_udma_glue_common common; 40 41 struct udma_tchan *udma_tchanx; 42 int udma_tchan_id; 43 44 struct k3_ring *ringtx; 45 struct k3_ring *ringtxcq; 46 47 bool psil_paired; 48 49 int virq; 50 51 atomic_t free_pkts; 52 bool tx_pause_on_err; 53 bool tx_filt_einfo; 54 bool tx_filt_pswords; 55 bool tx_supr_tdpkt; 56 }; 57 58 struct k3_udma_glue_rx_flow { 59 struct udma_rflow *udma_rflow; 60 int udma_rflow_id; 61 struct k3_ring *ringrx; 62 struct k3_ring *ringrxfdq; 63 64 int virq; 65 }; 66 67 struct k3_udma_glue_rx_channel { 68 struct k3_udma_glue_common common; 69 70 struct udma_rchan *udma_rchanx; 71 int udma_rchan_id; 72 bool remote; 73 74 bool psil_paired; 75 76 u32 swdata_size; 77 int flow_id_base; 78 79 struct k3_udma_glue_rx_flow *flows; 80 u32 flow_num; 81 u32 flows_ready; 82 }; 83 84 #define K3_UDMAX_TDOWN_TIMEOUT_US 1000 85 86 static int of_k3_udma_glue_parse(struct device_node *udmax_np, 87 struct k3_udma_glue_common *common) 88 { 89 common->udmax = of_xudma_dev_get(udmax_np, NULL); 90 if (IS_ERR(common->udmax)) 91 return PTR_ERR(common->udmax); 92 93 common->ringacc = xudma_get_ringacc(common->udmax); 94 common->tisci_rm = xudma_dev_get_tisci_rm(common->udmax); 95 96 return 0; 97 } 98 99 static int of_k3_udma_glue_parse_chn(struct device_node *chn_np, 100 const char *name, struct k3_udma_glue_common *common, 101 bool tx_chn) 102 { 103 struct psil_endpoint_config *ep_config; 104 struct of_phandle_args dma_spec; 105 u32 thread_id; 106 int ret = 0; 107 int index; 108 109 if (unlikely(!name)) 110 return -EINVAL; 111 112 index = of_property_match_string(chn_np, "dma-names", name); 113 if (index < 0) 114 return index; 115 116 if (of_parse_phandle_with_args(chn_np, "dmas", "#dma-cells", index, 117 &dma_spec)) 118 return -ENOENT; 119 120 thread_id = dma_spec.args[0]; 121 if (dma_spec.args_count == 2) { 122 if (dma_spec.args[1] > 2) { 123 dev_err(common->dev, "Invalid channel atype: %u\n", 124 dma_spec.args[1]); 125 ret = -EINVAL; 126 goto out_put_spec; 127 } 128 common->atype = dma_spec.args[1]; 129 } 130 131 if (tx_chn && !(thread_id & K3_PSIL_DST_THREAD_ID_OFFSET)) { 132 ret = -EINVAL; 133 goto out_put_spec; 134 } 135 136 if (!tx_chn && (thread_id & K3_PSIL_DST_THREAD_ID_OFFSET)) { 137 ret = -EINVAL; 138 goto out_put_spec; 139 } 140 141 /* get psil endpoint config */ 142 ep_config = psil_get_ep_config(thread_id); 143 if (IS_ERR(ep_config)) { 144 dev_err(common->dev, 145 "No configuration for psi-l thread 0x%04x\n", 146 thread_id); 147 ret = PTR_ERR(ep_config); 148 goto out_put_spec; 149 } 150 151 common->epib = ep_config->needs_epib; 152 common->psdata_size = ep_config->psd_size; 153 154 if (tx_chn) 155 common->dst_thread = thread_id; 156 else 157 common->src_thread = thread_id; 158 159 ret = of_k3_udma_glue_parse(dma_spec.np, common); 160 161 out_put_spec: 162 of_node_put(dma_spec.np); 163 return ret; 164 }; 165 166 static void k3_udma_glue_dump_tx_chn(struct k3_udma_glue_tx_channel *tx_chn) 167 { 168 struct device *dev = tx_chn->common.dev; 169 170 dev_dbg(dev, "dump_tx_chn:\n" 171 "udma_tchan_id: %d\n" 172 "src_thread: %08x\n" 173 "dst_thread: %08x\n", 174 tx_chn->udma_tchan_id, 175 tx_chn->common.src_thread, 176 tx_chn->common.dst_thread); 177 } 178 179 static void k3_udma_glue_dump_tx_rt_chn(struct k3_udma_glue_tx_channel *chn, 180 char *mark) 181 { 182 struct device *dev = chn->common.dev; 183 184 dev_dbg(dev, "=== dump ===> %s\n", mark); 185 dev_dbg(dev, "0x%08X: %08X\n", UDMA_CHAN_RT_CTL_REG, 186 xudma_tchanrt_read(chn->udma_tchanx, UDMA_CHAN_RT_CTL_REG)); 187 dev_dbg(dev, "0x%08X: %08X\n", UDMA_CHAN_RT_PEER_RT_EN_REG, 188 xudma_tchanrt_read(chn->udma_tchanx, 189 UDMA_CHAN_RT_PEER_RT_EN_REG)); 190 dev_dbg(dev, "0x%08X: %08X\n", UDMA_CHAN_RT_PCNT_REG, 191 xudma_tchanrt_read(chn->udma_tchanx, UDMA_CHAN_RT_PCNT_REG)); 192 dev_dbg(dev, "0x%08X: %08X\n", UDMA_CHAN_RT_BCNT_REG, 193 xudma_tchanrt_read(chn->udma_tchanx, UDMA_CHAN_RT_BCNT_REG)); 194 dev_dbg(dev, "0x%08X: %08X\n", UDMA_CHAN_RT_SBCNT_REG, 195 xudma_tchanrt_read(chn->udma_tchanx, UDMA_CHAN_RT_SBCNT_REG)); 196 } 197 198 static int k3_udma_glue_cfg_tx_chn(struct k3_udma_glue_tx_channel *tx_chn) 199 { 200 const struct udma_tisci_rm *tisci_rm = tx_chn->common.tisci_rm; 201 struct ti_sci_msg_rm_udmap_tx_ch_cfg req; 202 203 memset(&req, 0, sizeof(req)); 204 205 req.valid_params = TI_SCI_MSG_VALUE_RM_UDMAP_CH_PAUSE_ON_ERR_VALID | 206 TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_FILT_EINFO_VALID | 207 TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_FILT_PSWORDS_VALID | 208 TI_SCI_MSG_VALUE_RM_UDMAP_CH_CHAN_TYPE_VALID | 209 TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_SUPR_TDPKT_VALID | 210 TI_SCI_MSG_VALUE_RM_UDMAP_CH_FETCH_SIZE_VALID | 211 TI_SCI_MSG_VALUE_RM_UDMAP_CH_CQ_QNUM_VALID | 212 TI_SCI_MSG_VALUE_RM_UDMAP_CH_ATYPE_VALID; 213 req.nav_id = tisci_rm->tisci_dev_id; 214 req.index = tx_chn->udma_tchan_id; 215 if (tx_chn->tx_pause_on_err) 216 req.tx_pause_on_err = 1; 217 if (tx_chn->tx_filt_einfo) 218 req.tx_filt_einfo = 1; 219 if (tx_chn->tx_filt_pswords) 220 req.tx_filt_pswords = 1; 221 req.tx_chan_type = TI_SCI_RM_UDMAP_CHAN_TYPE_PKT_PBRR; 222 if (tx_chn->tx_supr_tdpkt) 223 req.tx_supr_tdpkt = 1; 224 req.tx_fetch_size = tx_chn->common.hdesc_size >> 2; 225 req.txcq_qnum = k3_ringacc_get_ring_id(tx_chn->ringtxcq); 226 req.tx_atype = tx_chn->common.atype; 227 228 return tisci_rm->tisci_udmap_ops->tx_ch_cfg(tisci_rm->tisci, &req); 229 } 230 231 struct k3_udma_glue_tx_channel *k3_udma_glue_request_tx_chn(struct device *dev, 232 const char *name, struct k3_udma_glue_tx_channel_cfg *cfg) 233 { 234 struct k3_udma_glue_tx_channel *tx_chn; 235 int ret; 236 237 tx_chn = devm_kzalloc(dev, sizeof(*tx_chn), GFP_KERNEL); 238 if (!tx_chn) 239 return ERR_PTR(-ENOMEM); 240 241 tx_chn->common.dev = dev; 242 tx_chn->common.swdata_size = cfg->swdata_size; 243 tx_chn->tx_pause_on_err = cfg->tx_pause_on_err; 244 tx_chn->tx_filt_einfo = cfg->tx_filt_einfo; 245 tx_chn->tx_filt_pswords = cfg->tx_filt_pswords; 246 tx_chn->tx_supr_tdpkt = cfg->tx_supr_tdpkt; 247 248 /* parse of udmap channel */ 249 ret = of_k3_udma_glue_parse_chn(dev->of_node, name, 250 &tx_chn->common, true); 251 if (ret) 252 goto err; 253 254 tx_chn->common.hdesc_size = cppi5_hdesc_calc_size(tx_chn->common.epib, 255 tx_chn->common.psdata_size, 256 tx_chn->common.swdata_size); 257 258 /* request and cfg UDMAP TX channel */ 259 tx_chn->udma_tchanx = xudma_tchan_get(tx_chn->common.udmax, -1); 260 if (IS_ERR(tx_chn->udma_tchanx)) { 261 ret = PTR_ERR(tx_chn->udma_tchanx); 262 dev_err(dev, "UDMAX tchanx get err %d\n", ret); 263 goto err; 264 } 265 tx_chn->udma_tchan_id = xudma_tchan_get_id(tx_chn->udma_tchanx); 266 267 atomic_set(&tx_chn->free_pkts, cfg->txcq_cfg.size); 268 269 /* request and cfg rings */ 270 ret = k3_ringacc_request_rings_pair(tx_chn->common.ringacc, 271 tx_chn->udma_tchan_id, -1, 272 &tx_chn->ringtx, 273 &tx_chn->ringtxcq); 274 if (ret) { 275 dev_err(dev, "Failed to get TX/TXCQ rings %d\n", ret); 276 goto err; 277 } 278 279 /* Set the dma_dev for the rings to be configured */ 280 cfg->tx_cfg.dma_dev = k3_udma_glue_tx_get_dma_device(tx_chn); 281 cfg->txcq_cfg.dma_dev = cfg->tx_cfg.dma_dev; 282 283 ret = k3_ringacc_ring_cfg(tx_chn->ringtx, &cfg->tx_cfg); 284 if (ret) { 285 dev_err(dev, "Failed to cfg ringtx %d\n", ret); 286 goto err; 287 } 288 289 ret = k3_ringacc_ring_cfg(tx_chn->ringtxcq, &cfg->txcq_cfg); 290 if (ret) { 291 dev_err(dev, "Failed to cfg ringtx %d\n", ret); 292 goto err; 293 } 294 295 /* request and cfg psi-l */ 296 tx_chn->common.src_thread = 297 xudma_dev_get_psil_base(tx_chn->common.udmax) + 298 tx_chn->udma_tchan_id; 299 300 ret = k3_udma_glue_cfg_tx_chn(tx_chn); 301 if (ret) { 302 dev_err(dev, "Failed to cfg tchan %d\n", ret); 303 goto err; 304 } 305 306 k3_udma_glue_dump_tx_chn(tx_chn); 307 308 return tx_chn; 309 310 err: 311 k3_udma_glue_release_tx_chn(tx_chn); 312 return ERR_PTR(ret); 313 } 314 EXPORT_SYMBOL_GPL(k3_udma_glue_request_tx_chn); 315 316 void k3_udma_glue_release_tx_chn(struct k3_udma_glue_tx_channel *tx_chn) 317 { 318 if (tx_chn->psil_paired) { 319 xudma_navss_psil_unpair(tx_chn->common.udmax, 320 tx_chn->common.src_thread, 321 tx_chn->common.dst_thread); 322 tx_chn->psil_paired = false; 323 } 324 325 if (!IS_ERR_OR_NULL(tx_chn->udma_tchanx)) 326 xudma_tchan_put(tx_chn->common.udmax, 327 tx_chn->udma_tchanx); 328 329 if (tx_chn->ringtxcq) 330 k3_ringacc_ring_free(tx_chn->ringtxcq); 331 332 if (tx_chn->ringtx) 333 k3_ringacc_ring_free(tx_chn->ringtx); 334 } 335 EXPORT_SYMBOL_GPL(k3_udma_glue_release_tx_chn); 336 337 int k3_udma_glue_push_tx_chn(struct k3_udma_glue_tx_channel *tx_chn, 338 struct cppi5_host_desc_t *desc_tx, 339 dma_addr_t desc_dma) 340 { 341 u32 ringtxcq_id; 342 343 if (!atomic_add_unless(&tx_chn->free_pkts, -1, 0)) 344 return -ENOMEM; 345 346 ringtxcq_id = k3_ringacc_get_ring_id(tx_chn->ringtxcq); 347 cppi5_desc_set_retpolicy(&desc_tx->hdr, 0, ringtxcq_id); 348 349 return k3_ringacc_ring_push(tx_chn->ringtx, &desc_dma); 350 } 351 EXPORT_SYMBOL_GPL(k3_udma_glue_push_tx_chn); 352 353 int k3_udma_glue_pop_tx_chn(struct k3_udma_glue_tx_channel *tx_chn, 354 dma_addr_t *desc_dma) 355 { 356 int ret; 357 358 ret = k3_ringacc_ring_pop(tx_chn->ringtxcq, desc_dma); 359 if (!ret) 360 atomic_inc(&tx_chn->free_pkts); 361 362 return ret; 363 } 364 EXPORT_SYMBOL_GPL(k3_udma_glue_pop_tx_chn); 365 366 int k3_udma_glue_enable_tx_chn(struct k3_udma_glue_tx_channel *tx_chn) 367 { 368 int ret; 369 370 ret = xudma_navss_psil_pair(tx_chn->common.udmax, 371 tx_chn->common.src_thread, 372 tx_chn->common.dst_thread); 373 if (ret) { 374 dev_err(tx_chn->common.dev, "PSI-L request err %d\n", ret); 375 return ret; 376 } 377 378 tx_chn->psil_paired = true; 379 380 xudma_tchanrt_write(tx_chn->udma_tchanx, UDMA_CHAN_RT_PEER_RT_EN_REG, 381 UDMA_PEER_RT_EN_ENABLE); 382 383 xudma_tchanrt_write(tx_chn->udma_tchanx, UDMA_CHAN_RT_CTL_REG, 384 UDMA_CHAN_RT_CTL_EN); 385 386 k3_udma_glue_dump_tx_rt_chn(tx_chn, "txchn en"); 387 return 0; 388 } 389 EXPORT_SYMBOL_GPL(k3_udma_glue_enable_tx_chn); 390 391 void k3_udma_glue_disable_tx_chn(struct k3_udma_glue_tx_channel *tx_chn) 392 { 393 k3_udma_glue_dump_tx_rt_chn(tx_chn, "txchn dis1"); 394 395 xudma_tchanrt_write(tx_chn->udma_tchanx, UDMA_CHAN_RT_CTL_REG, 0); 396 397 xudma_tchanrt_write(tx_chn->udma_tchanx, 398 UDMA_CHAN_RT_PEER_RT_EN_REG, 0); 399 k3_udma_glue_dump_tx_rt_chn(tx_chn, "txchn dis2"); 400 401 if (tx_chn->psil_paired) { 402 xudma_navss_psil_unpair(tx_chn->common.udmax, 403 tx_chn->common.src_thread, 404 tx_chn->common.dst_thread); 405 tx_chn->psil_paired = false; 406 } 407 } 408 EXPORT_SYMBOL_GPL(k3_udma_glue_disable_tx_chn); 409 410 void k3_udma_glue_tdown_tx_chn(struct k3_udma_glue_tx_channel *tx_chn, 411 bool sync) 412 { 413 int i = 0; 414 u32 val; 415 416 k3_udma_glue_dump_tx_rt_chn(tx_chn, "txchn tdown1"); 417 418 xudma_tchanrt_write(tx_chn->udma_tchanx, UDMA_CHAN_RT_CTL_REG, 419 UDMA_CHAN_RT_CTL_EN | UDMA_CHAN_RT_CTL_TDOWN); 420 421 val = xudma_tchanrt_read(tx_chn->udma_tchanx, UDMA_CHAN_RT_CTL_REG); 422 423 while (sync && (val & UDMA_CHAN_RT_CTL_EN)) { 424 val = xudma_tchanrt_read(tx_chn->udma_tchanx, 425 UDMA_CHAN_RT_CTL_REG); 426 udelay(1); 427 if (i > K3_UDMAX_TDOWN_TIMEOUT_US) { 428 dev_err(tx_chn->common.dev, "TX tdown timeout\n"); 429 break; 430 } 431 i++; 432 } 433 434 val = xudma_tchanrt_read(tx_chn->udma_tchanx, 435 UDMA_CHAN_RT_PEER_RT_EN_REG); 436 if (sync && (val & UDMA_PEER_RT_EN_ENABLE)) 437 dev_err(tx_chn->common.dev, "TX tdown peer not stopped\n"); 438 k3_udma_glue_dump_tx_rt_chn(tx_chn, "txchn tdown2"); 439 } 440 EXPORT_SYMBOL_GPL(k3_udma_glue_tdown_tx_chn); 441 442 void k3_udma_glue_reset_tx_chn(struct k3_udma_glue_tx_channel *tx_chn, 443 void *data, 444 void (*cleanup)(void *data, dma_addr_t desc_dma)) 445 { 446 dma_addr_t desc_dma; 447 int occ_tx, i, ret; 448 449 /* reset TXCQ as it is not input for udma - expected to be empty */ 450 if (tx_chn->ringtxcq) 451 k3_ringacc_ring_reset(tx_chn->ringtxcq); 452 453 /* 454 * TXQ reset need to be special way as it is input for udma and its 455 * state cached by udma, so: 456 * 1) save TXQ occ 457 * 2) clean up TXQ and call callback .cleanup() for each desc 458 * 3) reset TXQ in a special way 459 */ 460 occ_tx = k3_ringacc_ring_get_occ(tx_chn->ringtx); 461 dev_dbg(tx_chn->common.dev, "TX reset occ_tx %u\n", occ_tx); 462 463 for (i = 0; i < occ_tx; i++) { 464 ret = k3_ringacc_ring_pop(tx_chn->ringtx, &desc_dma); 465 if (ret) { 466 dev_err(tx_chn->common.dev, "TX reset pop %d\n", ret); 467 break; 468 } 469 cleanup(data, desc_dma); 470 } 471 472 k3_ringacc_ring_reset_dma(tx_chn->ringtx, occ_tx); 473 } 474 EXPORT_SYMBOL_GPL(k3_udma_glue_reset_tx_chn); 475 476 u32 k3_udma_glue_tx_get_hdesc_size(struct k3_udma_glue_tx_channel *tx_chn) 477 { 478 return tx_chn->common.hdesc_size; 479 } 480 EXPORT_SYMBOL_GPL(k3_udma_glue_tx_get_hdesc_size); 481 482 u32 k3_udma_glue_tx_get_txcq_id(struct k3_udma_glue_tx_channel *tx_chn) 483 { 484 return k3_ringacc_get_ring_id(tx_chn->ringtxcq); 485 } 486 EXPORT_SYMBOL_GPL(k3_udma_glue_tx_get_txcq_id); 487 488 int k3_udma_glue_tx_get_irq(struct k3_udma_glue_tx_channel *tx_chn) 489 { 490 tx_chn->virq = k3_ringacc_get_ring_irq_num(tx_chn->ringtxcq); 491 492 return tx_chn->virq; 493 } 494 EXPORT_SYMBOL_GPL(k3_udma_glue_tx_get_irq); 495 496 struct device * 497 k3_udma_glue_tx_get_dma_device(struct k3_udma_glue_tx_channel *tx_chn) 498 { 499 return xudma_get_device(tx_chn->common.udmax); 500 } 501 EXPORT_SYMBOL_GPL(k3_udma_glue_tx_get_dma_device); 502 503 static int k3_udma_glue_cfg_rx_chn(struct k3_udma_glue_rx_channel *rx_chn) 504 { 505 const struct udma_tisci_rm *tisci_rm = rx_chn->common.tisci_rm; 506 struct ti_sci_msg_rm_udmap_rx_ch_cfg req; 507 int ret; 508 509 memset(&req, 0, sizeof(req)); 510 511 req.valid_params = TI_SCI_MSG_VALUE_RM_UDMAP_CH_FETCH_SIZE_VALID | 512 TI_SCI_MSG_VALUE_RM_UDMAP_CH_CQ_QNUM_VALID | 513 TI_SCI_MSG_VALUE_RM_UDMAP_CH_CHAN_TYPE_VALID | 514 TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_FLOWID_START_VALID | 515 TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_FLOWID_CNT_VALID | 516 TI_SCI_MSG_VALUE_RM_UDMAP_CH_ATYPE_VALID; 517 518 req.nav_id = tisci_rm->tisci_dev_id; 519 req.index = rx_chn->udma_rchan_id; 520 req.rx_fetch_size = rx_chn->common.hdesc_size >> 2; 521 /* 522 * TODO: we can't support rxcq_qnum/RCHAN[a]_RCQ cfg with current sysfw 523 * and udmax impl, so just configure it to invalid value. 524 * req.rxcq_qnum = k3_ringacc_get_ring_id(rx_chn->flows[0].ringrx); 525 */ 526 req.rxcq_qnum = 0xFFFF; 527 if (rx_chn->flow_num && rx_chn->flow_id_base != rx_chn->udma_rchan_id) { 528 /* Default flow + extra ones */ 529 req.flowid_start = rx_chn->flow_id_base; 530 req.flowid_cnt = rx_chn->flow_num; 531 } 532 req.rx_chan_type = TI_SCI_RM_UDMAP_CHAN_TYPE_PKT_PBRR; 533 req.rx_atype = rx_chn->common.atype; 534 535 ret = tisci_rm->tisci_udmap_ops->rx_ch_cfg(tisci_rm->tisci, &req); 536 if (ret) 537 dev_err(rx_chn->common.dev, "rchan%d cfg failed %d\n", 538 rx_chn->udma_rchan_id, ret); 539 540 return ret; 541 } 542 543 static void k3_udma_glue_release_rx_flow(struct k3_udma_glue_rx_channel *rx_chn, 544 u32 flow_num) 545 { 546 struct k3_udma_glue_rx_flow *flow = &rx_chn->flows[flow_num]; 547 548 if (IS_ERR_OR_NULL(flow->udma_rflow)) 549 return; 550 551 if (flow->ringrxfdq) 552 k3_ringacc_ring_free(flow->ringrxfdq); 553 554 if (flow->ringrx) 555 k3_ringacc_ring_free(flow->ringrx); 556 557 xudma_rflow_put(rx_chn->common.udmax, flow->udma_rflow); 558 flow->udma_rflow = NULL; 559 rx_chn->flows_ready--; 560 } 561 562 static int k3_udma_glue_cfg_rx_flow(struct k3_udma_glue_rx_channel *rx_chn, 563 u32 flow_idx, 564 struct k3_udma_glue_rx_flow_cfg *flow_cfg) 565 { 566 struct k3_udma_glue_rx_flow *flow = &rx_chn->flows[flow_idx]; 567 const struct udma_tisci_rm *tisci_rm = rx_chn->common.tisci_rm; 568 struct device *dev = rx_chn->common.dev; 569 struct ti_sci_msg_rm_udmap_flow_cfg req; 570 int rx_ring_id; 571 int rx_ringfdq_id; 572 int ret = 0; 573 574 flow->udma_rflow = xudma_rflow_get(rx_chn->common.udmax, 575 flow->udma_rflow_id); 576 if (IS_ERR(flow->udma_rflow)) { 577 ret = PTR_ERR(flow->udma_rflow); 578 dev_err(dev, "UDMAX rflow get err %d\n", ret); 579 return ret; 580 } 581 582 if (flow->udma_rflow_id != xudma_rflow_get_id(flow->udma_rflow)) { 583 ret = -ENODEV; 584 goto err_rflow_put; 585 } 586 587 /* request and cfg rings */ 588 ret = k3_ringacc_request_rings_pair(rx_chn->common.ringacc, 589 flow_cfg->ring_rxfdq0_id, 590 flow_cfg->ring_rxq_id, 591 &flow->ringrxfdq, 592 &flow->ringrx); 593 if (ret) { 594 dev_err(dev, "Failed to get RX/RXFDQ rings %d\n", ret); 595 goto err_rflow_put; 596 } 597 598 /* Set the dma_dev for the rings to be configured */ 599 flow_cfg->rx_cfg.dma_dev = k3_udma_glue_rx_get_dma_device(rx_chn); 600 flow_cfg->rxfdq_cfg.dma_dev = flow_cfg->rx_cfg.dma_dev; 601 602 ret = k3_ringacc_ring_cfg(flow->ringrx, &flow_cfg->rx_cfg); 603 if (ret) { 604 dev_err(dev, "Failed to cfg ringrx %d\n", ret); 605 goto err_ringrxfdq_free; 606 } 607 608 ret = k3_ringacc_ring_cfg(flow->ringrxfdq, &flow_cfg->rxfdq_cfg); 609 if (ret) { 610 dev_err(dev, "Failed to cfg ringrxfdq %d\n", ret); 611 goto err_ringrxfdq_free; 612 } 613 614 if (rx_chn->remote) { 615 rx_ring_id = TI_SCI_RESOURCE_NULL; 616 rx_ringfdq_id = TI_SCI_RESOURCE_NULL; 617 } else { 618 rx_ring_id = k3_ringacc_get_ring_id(flow->ringrx); 619 rx_ringfdq_id = k3_ringacc_get_ring_id(flow->ringrxfdq); 620 } 621 622 memset(&req, 0, sizeof(req)); 623 624 req.valid_params = 625 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_EINFO_PRESENT_VALID | 626 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_PSINFO_PRESENT_VALID | 627 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_ERROR_HANDLING_VALID | 628 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DESC_TYPE_VALID | 629 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DEST_QNUM_VALID | 630 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_SRC_TAG_HI_SEL_VALID | 631 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_SRC_TAG_LO_SEL_VALID | 632 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DEST_TAG_HI_SEL_VALID | 633 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DEST_TAG_LO_SEL_VALID | 634 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ0_SZ0_QNUM_VALID | 635 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ1_QNUM_VALID | 636 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ2_QNUM_VALID | 637 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ3_QNUM_VALID; 638 req.nav_id = tisci_rm->tisci_dev_id; 639 req.flow_index = flow->udma_rflow_id; 640 if (rx_chn->common.epib) 641 req.rx_einfo_present = 1; 642 if (rx_chn->common.psdata_size) 643 req.rx_psinfo_present = 1; 644 if (flow_cfg->rx_error_handling) 645 req.rx_error_handling = 1; 646 req.rx_desc_type = 0; 647 req.rx_dest_qnum = rx_ring_id; 648 req.rx_src_tag_hi_sel = 0; 649 req.rx_src_tag_lo_sel = flow_cfg->src_tag_lo_sel; 650 req.rx_dest_tag_hi_sel = 0; 651 req.rx_dest_tag_lo_sel = 0; 652 req.rx_fdq0_sz0_qnum = rx_ringfdq_id; 653 req.rx_fdq1_qnum = rx_ringfdq_id; 654 req.rx_fdq2_qnum = rx_ringfdq_id; 655 req.rx_fdq3_qnum = rx_ringfdq_id; 656 657 ret = tisci_rm->tisci_udmap_ops->rx_flow_cfg(tisci_rm->tisci, &req); 658 if (ret) { 659 dev_err(dev, "flow%d config failed: %d\n", flow->udma_rflow_id, 660 ret); 661 goto err_ringrxfdq_free; 662 } 663 664 rx_chn->flows_ready++; 665 dev_dbg(dev, "flow%d config done. ready:%d\n", 666 flow->udma_rflow_id, rx_chn->flows_ready); 667 668 return 0; 669 670 err_ringrxfdq_free: 671 k3_ringacc_ring_free(flow->ringrxfdq); 672 k3_ringacc_ring_free(flow->ringrx); 673 674 err_rflow_put: 675 xudma_rflow_put(rx_chn->common.udmax, flow->udma_rflow); 676 flow->udma_rflow = NULL; 677 678 return ret; 679 } 680 681 static void k3_udma_glue_dump_rx_chn(struct k3_udma_glue_rx_channel *chn) 682 { 683 struct device *dev = chn->common.dev; 684 685 dev_dbg(dev, "dump_rx_chn:\n" 686 "udma_rchan_id: %d\n" 687 "src_thread: %08x\n" 688 "dst_thread: %08x\n" 689 "epib: %d\n" 690 "hdesc_size: %u\n" 691 "psdata_size: %u\n" 692 "swdata_size: %u\n" 693 "flow_id_base: %d\n" 694 "flow_num: %d\n", 695 chn->udma_rchan_id, 696 chn->common.src_thread, 697 chn->common.dst_thread, 698 chn->common.epib, 699 chn->common.hdesc_size, 700 chn->common.psdata_size, 701 chn->common.swdata_size, 702 chn->flow_id_base, 703 chn->flow_num); 704 } 705 706 static void k3_udma_glue_dump_rx_rt_chn(struct k3_udma_glue_rx_channel *chn, 707 char *mark) 708 { 709 struct device *dev = chn->common.dev; 710 711 dev_dbg(dev, "=== dump ===> %s\n", mark); 712 713 dev_dbg(dev, "0x%08X: %08X\n", UDMA_CHAN_RT_CTL_REG, 714 xudma_rchanrt_read(chn->udma_rchanx, UDMA_CHAN_RT_CTL_REG)); 715 dev_dbg(dev, "0x%08X: %08X\n", UDMA_CHAN_RT_PEER_RT_EN_REG, 716 xudma_rchanrt_read(chn->udma_rchanx, 717 UDMA_CHAN_RT_PEER_RT_EN_REG)); 718 dev_dbg(dev, "0x%08X: %08X\n", UDMA_CHAN_RT_PCNT_REG, 719 xudma_rchanrt_read(chn->udma_rchanx, UDMA_CHAN_RT_PCNT_REG)); 720 dev_dbg(dev, "0x%08X: %08X\n", UDMA_CHAN_RT_BCNT_REG, 721 xudma_rchanrt_read(chn->udma_rchanx, UDMA_CHAN_RT_BCNT_REG)); 722 dev_dbg(dev, "0x%08X: %08X\n", UDMA_CHAN_RT_SBCNT_REG, 723 xudma_rchanrt_read(chn->udma_rchanx, UDMA_CHAN_RT_SBCNT_REG)); 724 } 725 726 static int 727 k3_udma_glue_allocate_rx_flows(struct k3_udma_glue_rx_channel *rx_chn, 728 struct k3_udma_glue_rx_channel_cfg *cfg) 729 { 730 int ret; 731 732 /* default rflow */ 733 if (cfg->flow_id_use_rxchan_id) 734 return 0; 735 736 /* not a GP rflows */ 737 if (rx_chn->flow_id_base != -1 && 738 !xudma_rflow_is_gp(rx_chn->common.udmax, rx_chn->flow_id_base)) 739 return 0; 740 741 /* Allocate range of GP rflows */ 742 ret = xudma_alloc_gp_rflow_range(rx_chn->common.udmax, 743 rx_chn->flow_id_base, 744 rx_chn->flow_num); 745 if (ret < 0) { 746 dev_err(rx_chn->common.dev, "UDMAX reserve_rflow %d cnt:%d err: %d\n", 747 rx_chn->flow_id_base, rx_chn->flow_num, ret); 748 return ret; 749 } 750 rx_chn->flow_id_base = ret; 751 752 return 0; 753 } 754 755 static struct k3_udma_glue_rx_channel * 756 k3_udma_glue_request_rx_chn_priv(struct device *dev, const char *name, 757 struct k3_udma_glue_rx_channel_cfg *cfg) 758 { 759 struct k3_udma_glue_rx_channel *rx_chn; 760 int ret, i; 761 762 if (cfg->flow_id_num <= 0) 763 return ERR_PTR(-EINVAL); 764 765 if (cfg->flow_id_num != 1 && 766 (cfg->def_flow_cfg || cfg->flow_id_use_rxchan_id)) 767 return ERR_PTR(-EINVAL); 768 769 rx_chn = devm_kzalloc(dev, sizeof(*rx_chn), GFP_KERNEL); 770 if (!rx_chn) 771 return ERR_PTR(-ENOMEM); 772 773 rx_chn->common.dev = dev; 774 rx_chn->common.swdata_size = cfg->swdata_size; 775 rx_chn->remote = false; 776 777 /* parse of udmap channel */ 778 ret = of_k3_udma_glue_parse_chn(dev->of_node, name, 779 &rx_chn->common, false); 780 if (ret) 781 goto err; 782 783 rx_chn->common.hdesc_size = cppi5_hdesc_calc_size(rx_chn->common.epib, 784 rx_chn->common.psdata_size, 785 rx_chn->common.swdata_size); 786 787 /* request and cfg UDMAP RX channel */ 788 rx_chn->udma_rchanx = xudma_rchan_get(rx_chn->common.udmax, -1); 789 if (IS_ERR(rx_chn->udma_rchanx)) { 790 ret = PTR_ERR(rx_chn->udma_rchanx); 791 dev_err(dev, "UDMAX rchanx get err %d\n", ret); 792 goto err; 793 } 794 rx_chn->udma_rchan_id = xudma_rchan_get_id(rx_chn->udma_rchanx); 795 796 rx_chn->flow_num = cfg->flow_id_num; 797 rx_chn->flow_id_base = cfg->flow_id_base; 798 799 /* Use RX channel id as flow id: target dev can't generate flow_id */ 800 if (cfg->flow_id_use_rxchan_id) 801 rx_chn->flow_id_base = rx_chn->udma_rchan_id; 802 803 rx_chn->flows = devm_kcalloc(dev, rx_chn->flow_num, 804 sizeof(*rx_chn->flows), GFP_KERNEL); 805 if (!rx_chn->flows) { 806 ret = -ENOMEM; 807 goto err; 808 } 809 810 ret = k3_udma_glue_allocate_rx_flows(rx_chn, cfg); 811 if (ret) 812 goto err; 813 814 for (i = 0; i < rx_chn->flow_num; i++) 815 rx_chn->flows[i].udma_rflow_id = rx_chn->flow_id_base + i; 816 817 /* request and cfg psi-l */ 818 rx_chn->common.dst_thread = 819 xudma_dev_get_psil_base(rx_chn->common.udmax) + 820 rx_chn->udma_rchan_id; 821 822 ret = k3_udma_glue_cfg_rx_chn(rx_chn); 823 if (ret) { 824 dev_err(dev, "Failed to cfg rchan %d\n", ret); 825 goto err; 826 } 827 828 /* init default RX flow only if flow_num = 1 */ 829 if (cfg->def_flow_cfg) { 830 ret = k3_udma_glue_cfg_rx_flow(rx_chn, 0, cfg->def_flow_cfg); 831 if (ret) 832 goto err; 833 } 834 835 k3_udma_glue_dump_rx_chn(rx_chn); 836 837 return rx_chn; 838 839 err: 840 k3_udma_glue_release_rx_chn(rx_chn); 841 return ERR_PTR(ret); 842 } 843 844 static struct k3_udma_glue_rx_channel * 845 k3_udma_glue_request_remote_rx_chn(struct device *dev, const char *name, 846 struct k3_udma_glue_rx_channel_cfg *cfg) 847 { 848 struct k3_udma_glue_rx_channel *rx_chn; 849 int ret, i; 850 851 if (cfg->flow_id_num <= 0 || 852 cfg->flow_id_use_rxchan_id || 853 cfg->def_flow_cfg || 854 cfg->flow_id_base < 0) 855 return ERR_PTR(-EINVAL); 856 857 /* 858 * Remote RX channel is under control of Remote CPU core, so 859 * Linux can only request and manipulate by dedicated RX flows 860 */ 861 862 rx_chn = devm_kzalloc(dev, sizeof(*rx_chn), GFP_KERNEL); 863 if (!rx_chn) 864 return ERR_PTR(-ENOMEM); 865 866 rx_chn->common.dev = dev; 867 rx_chn->common.swdata_size = cfg->swdata_size; 868 rx_chn->remote = true; 869 rx_chn->udma_rchan_id = -1; 870 rx_chn->flow_num = cfg->flow_id_num; 871 rx_chn->flow_id_base = cfg->flow_id_base; 872 rx_chn->psil_paired = false; 873 874 /* parse of udmap channel */ 875 ret = of_k3_udma_glue_parse_chn(dev->of_node, name, 876 &rx_chn->common, false); 877 if (ret) 878 goto err; 879 880 rx_chn->common.hdesc_size = cppi5_hdesc_calc_size(rx_chn->common.epib, 881 rx_chn->common.psdata_size, 882 rx_chn->common.swdata_size); 883 884 rx_chn->flows = devm_kcalloc(dev, rx_chn->flow_num, 885 sizeof(*rx_chn->flows), GFP_KERNEL); 886 if (!rx_chn->flows) { 887 ret = -ENOMEM; 888 goto err; 889 } 890 891 ret = k3_udma_glue_allocate_rx_flows(rx_chn, cfg); 892 if (ret) 893 goto err; 894 895 for (i = 0; i < rx_chn->flow_num; i++) 896 rx_chn->flows[i].udma_rflow_id = rx_chn->flow_id_base + i; 897 898 k3_udma_glue_dump_rx_chn(rx_chn); 899 900 return rx_chn; 901 902 err: 903 k3_udma_glue_release_rx_chn(rx_chn); 904 return ERR_PTR(ret); 905 } 906 907 struct k3_udma_glue_rx_channel * 908 k3_udma_glue_request_rx_chn(struct device *dev, const char *name, 909 struct k3_udma_glue_rx_channel_cfg *cfg) 910 { 911 if (cfg->remote) 912 return k3_udma_glue_request_remote_rx_chn(dev, name, cfg); 913 else 914 return k3_udma_glue_request_rx_chn_priv(dev, name, cfg); 915 } 916 EXPORT_SYMBOL_GPL(k3_udma_glue_request_rx_chn); 917 918 void k3_udma_glue_release_rx_chn(struct k3_udma_glue_rx_channel *rx_chn) 919 { 920 int i; 921 922 if (IS_ERR_OR_NULL(rx_chn->common.udmax)) 923 return; 924 925 if (rx_chn->psil_paired) { 926 xudma_navss_psil_unpair(rx_chn->common.udmax, 927 rx_chn->common.src_thread, 928 rx_chn->common.dst_thread); 929 rx_chn->psil_paired = false; 930 } 931 932 for (i = 0; i < rx_chn->flow_num; i++) 933 k3_udma_glue_release_rx_flow(rx_chn, i); 934 935 if (xudma_rflow_is_gp(rx_chn->common.udmax, rx_chn->flow_id_base)) 936 xudma_free_gp_rflow_range(rx_chn->common.udmax, 937 rx_chn->flow_id_base, 938 rx_chn->flow_num); 939 940 if (!IS_ERR_OR_NULL(rx_chn->udma_rchanx)) 941 xudma_rchan_put(rx_chn->common.udmax, 942 rx_chn->udma_rchanx); 943 } 944 EXPORT_SYMBOL_GPL(k3_udma_glue_release_rx_chn); 945 946 int k3_udma_glue_rx_flow_init(struct k3_udma_glue_rx_channel *rx_chn, 947 u32 flow_idx, 948 struct k3_udma_glue_rx_flow_cfg *flow_cfg) 949 { 950 if (flow_idx >= rx_chn->flow_num) 951 return -EINVAL; 952 953 return k3_udma_glue_cfg_rx_flow(rx_chn, flow_idx, flow_cfg); 954 } 955 EXPORT_SYMBOL_GPL(k3_udma_glue_rx_flow_init); 956 957 u32 k3_udma_glue_rx_flow_get_fdq_id(struct k3_udma_glue_rx_channel *rx_chn, 958 u32 flow_idx) 959 { 960 struct k3_udma_glue_rx_flow *flow; 961 962 if (flow_idx >= rx_chn->flow_num) 963 return -EINVAL; 964 965 flow = &rx_chn->flows[flow_idx]; 966 967 return k3_ringacc_get_ring_id(flow->ringrxfdq); 968 } 969 EXPORT_SYMBOL_GPL(k3_udma_glue_rx_flow_get_fdq_id); 970 971 u32 k3_udma_glue_rx_get_flow_id_base(struct k3_udma_glue_rx_channel *rx_chn) 972 { 973 return rx_chn->flow_id_base; 974 } 975 EXPORT_SYMBOL_GPL(k3_udma_glue_rx_get_flow_id_base); 976 977 int k3_udma_glue_rx_flow_enable(struct k3_udma_glue_rx_channel *rx_chn, 978 u32 flow_idx) 979 { 980 struct k3_udma_glue_rx_flow *flow = &rx_chn->flows[flow_idx]; 981 const struct udma_tisci_rm *tisci_rm = rx_chn->common.tisci_rm; 982 struct device *dev = rx_chn->common.dev; 983 struct ti_sci_msg_rm_udmap_flow_cfg req; 984 int rx_ring_id; 985 int rx_ringfdq_id; 986 int ret = 0; 987 988 if (!rx_chn->remote) 989 return -EINVAL; 990 991 rx_ring_id = k3_ringacc_get_ring_id(flow->ringrx); 992 rx_ringfdq_id = k3_ringacc_get_ring_id(flow->ringrxfdq); 993 994 memset(&req, 0, sizeof(req)); 995 996 req.valid_params = 997 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DEST_QNUM_VALID | 998 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ0_SZ0_QNUM_VALID | 999 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ1_QNUM_VALID | 1000 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ2_QNUM_VALID | 1001 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ3_QNUM_VALID; 1002 req.nav_id = tisci_rm->tisci_dev_id; 1003 req.flow_index = flow->udma_rflow_id; 1004 req.rx_dest_qnum = rx_ring_id; 1005 req.rx_fdq0_sz0_qnum = rx_ringfdq_id; 1006 req.rx_fdq1_qnum = rx_ringfdq_id; 1007 req.rx_fdq2_qnum = rx_ringfdq_id; 1008 req.rx_fdq3_qnum = rx_ringfdq_id; 1009 1010 ret = tisci_rm->tisci_udmap_ops->rx_flow_cfg(tisci_rm->tisci, &req); 1011 if (ret) { 1012 dev_err(dev, "flow%d enable failed: %d\n", flow->udma_rflow_id, 1013 ret); 1014 } 1015 1016 return ret; 1017 } 1018 EXPORT_SYMBOL_GPL(k3_udma_glue_rx_flow_enable); 1019 1020 int k3_udma_glue_rx_flow_disable(struct k3_udma_glue_rx_channel *rx_chn, 1021 u32 flow_idx) 1022 { 1023 struct k3_udma_glue_rx_flow *flow = &rx_chn->flows[flow_idx]; 1024 const struct udma_tisci_rm *tisci_rm = rx_chn->common.tisci_rm; 1025 struct device *dev = rx_chn->common.dev; 1026 struct ti_sci_msg_rm_udmap_flow_cfg req; 1027 int ret = 0; 1028 1029 if (!rx_chn->remote) 1030 return -EINVAL; 1031 1032 memset(&req, 0, sizeof(req)); 1033 req.valid_params = 1034 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DEST_QNUM_VALID | 1035 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ0_SZ0_QNUM_VALID | 1036 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ1_QNUM_VALID | 1037 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ2_QNUM_VALID | 1038 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ3_QNUM_VALID; 1039 req.nav_id = tisci_rm->tisci_dev_id; 1040 req.flow_index = flow->udma_rflow_id; 1041 req.rx_dest_qnum = TI_SCI_RESOURCE_NULL; 1042 req.rx_fdq0_sz0_qnum = TI_SCI_RESOURCE_NULL; 1043 req.rx_fdq1_qnum = TI_SCI_RESOURCE_NULL; 1044 req.rx_fdq2_qnum = TI_SCI_RESOURCE_NULL; 1045 req.rx_fdq3_qnum = TI_SCI_RESOURCE_NULL; 1046 1047 ret = tisci_rm->tisci_udmap_ops->rx_flow_cfg(tisci_rm->tisci, &req); 1048 if (ret) { 1049 dev_err(dev, "flow%d disable failed: %d\n", flow->udma_rflow_id, 1050 ret); 1051 } 1052 1053 return ret; 1054 } 1055 EXPORT_SYMBOL_GPL(k3_udma_glue_rx_flow_disable); 1056 1057 int k3_udma_glue_enable_rx_chn(struct k3_udma_glue_rx_channel *rx_chn) 1058 { 1059 int ret; 1060 1061 if (rx_chn->remote) 1062 return -EINVAL; 1063 1064 if (rx_chn->flows_ready < rx_chn->flow_num) 1065 return -EINVAL; 1066 1067 ret = xudma_navss_psil_pair(rx_chn->common.udmax, 1068 rx_chn->common.src_thread, 1069 rx_chn->common.dst_thread); 1070 if (ret) { 1071 dev_err(rx_chn->common.dev, "PSI-L request err %d\n", ret); 1072 return ret; 1073 } 1074 1075 rx_chn->psil_paired = true; 1076 1077 xudma_rchanrt_write(rx_chn->udma_rchanx, UDMA_CHAN_RT_CTL_REG, 1078 UDMA_CHAN_RT_CTL_EN); 1079 1080 xudma_rchanrt_write(rx_chn->udma_rchanx, UDMA_CHAN_RT_PEER_RT_EN_REG, 1081 UDMA_PEER_RT_EN_ENABLE); 1082 1083 k3_udma_glue_dump_rx_rt_chn(rx_chn, "rxrt en"); 1084 return 0; 1085 } 1086 EXPORT_SYMBOL_GPL(k3_udma_glue_enable_rx_chn); 1087 1088 void k3_udma_glue_disable_rx_chn(struct k3_udma_glue_rx_channel *rx_chn) 1089 { 1090 k3_udma_glue_dump_rx_rt_chn(rx_chn, "rxrt dis1"); 1091 1092 xudma_rchanrt_write(rx_chn->udma_rchanx, 1093 UDMA_CHAN_RT_PEER_RT_EN_REG, 0); 1094 xudma_rchanrt_write(rx_chn->udma_rchanx, UDMA_CHAN_RT_CTL_REG, 0); 1095 1096 k3_udma_glue_dump_rx_rt_chn(rx_chn, "rxrt dis2"); 1097 1098 if (rx_chn->psil_paired) { 1099 xudma_navss_psil_unpair(rx_chn->common.udmax, 1100 rx_chn->common.src_thread, 1101 rx_chn->common.dst_thread); 1102 rx_chn->psil_paired = false; 1103 } 1104 } 1105 EXPORT_SYMBOL_GPL(k3_udma_glue_disable_rx_chn); 1106 1107 void k3_udma_glue_tdown_rx_chn(struct k3_udma_glue_rx_channel *rx_chn, 1108 bool sync) 1109 { 1110 int i = 0; 1111 u32 val; 1112 1113 if (rx_chn->remote) 1114 return; 1115 1116 k3_udma_glue_dump_rx_rt_chn(rx_chn, "rxrt tdown1"); 1117 1118 xudma_rchanrt_write(rx_chn->udma_rchanx, UDMA_CHAN_RT_PEER_RT_EN_REG, 1119 UDMA_PEER_RT_EN_ENABLE | UDMA_PEER_RT_EN_TEARDOWN); 1120 1121 val = xudma_rchanrt_read(rx_chn->udma_rchanx, UDMA_CHAN_RT_CTL_REG); 1122 1123 while (sync && (val & UDMA_CHAN_RT_CTL_EN)) { 1124 val = xudma_rchanrt_read(rx_chn->udma_rchanx, 1125 UDMA_CHAN_RT_CTL_REG); 1126 udelay(1); 1127 if (i > K3_UDMAX_TDOWN_TIMEOUT_US) { 1128 dev_err(rx_chn->common.dev, "RX tdown timeout\n"); 1129 break; 1130 } 1131 i++; 1132 } 1133 1134 val = xudma_rchanrt_read(rx_chn->udma_rchanx, 1135 UDMA_CHAN_RT_PEER_RT_EN_REG); 1136 if (sync && (val & UDMA_PEER_RT_EN_ENABLE)) 1137 dev_err(rx_chn->common.dev, "TX tdown peer not stopped\n"); 1138 k3_udma_glue_dump_rx_rt_chn(rx_chn, "rxrt tdown2"); 1139 } 1140 EXPORT_SYMBOL_GPL(k3_udma_glue_tdown_rx_chn); 1141 1142 void k3_udma_glue_reset_rx_chn(struct k3_udma_glue_rx_channel *rx_chn, 1143 u32 flow_num, void *data, 1144 void (*cleanup)(void *data, dma_addr_t desc_dma), bool skip_fdq) 1145 { 1146 struct k3_udma_glue_rx_flow *flow = &rx_chn->flows[flow_num]; 1147 struct device *dev = rx_chn->common.dev; 1148 dma_addr_t desc_dma; 1149 int occ_rx, i, ret; 1150 1151 /* reset RXCQ as it is not input for udma - expected to be empty */ 1152 occ_rx = k3_ringacc_ring_get_occ(flow->ringrx); 1153 dev_dbg(dev, "RX reset flow %u occ_rx %u\n", flow_num, occ_rx); 1154 if (flow->ringrx) 1155 k3_ringacc_ring_reset(flow->ringrx); 1156 1157 /* Skip RX FDQ in case one FDQ is used for the set of flows */ 1158 if (skip_fdq) 1159 return; 1160 1161 /* 1162 * RX FDQ reset need to be special way as it is input for udma and its 1163 * state cached by udma, so: 1164 * 1) save RX FDQ occ 1165 * 2) clean up RX FDQ and call callback .cleanup() for each desc 1166 * 3) reset RX FDQ in a special way 1167 */ 1168 occ_rx = k3_ringacc_ring_get_occ(flow->ringrxfdq); 1169 dev_dbg(dev, "RX reset flow %u occ_rx_fdq %u\n", flow_num, occ_rx); 1170 1171 for (i = 0; i < occ_rx; i++) { 1172 ret = k3_ringacc_ring_pop(flow->ringrxfdq, &desc_dma); 1173 if (ret) { 1174 dev_err(dev, "RX reset pop %d\n", ret); 1175 break; 1176 } 1177 cleanup(data, desc_dma); 1178 } 1179 1180 k3_ringacc_ring_reset_dma(flow->ringrxfdq, occ_rx); 1181 } 1182 EXPORT_SYMBOL_GPL(k3_udma_glue_reset_rx_chn); 1183 1184 int k3_udma_glue_push_rx_chn(struct k3_udma_glue_rx_channel *rx_chn, 1185 u32 flow_num, struct cppi5_host_desc_t *desc_rx, 1186 dma_addr_t desc_dma) 1187 { 1188 struct k3_udma_glue_rx_flow *flow = &rx_chn->flows[flow_num]; 1189 1190 return k3_ringacc_ring_push(flow->ringrxfdq, &desc_dma); 1191 } 1192 EXPORT_SYMBOL_GPL(k3_udma_glue_push_rx_chn); 1193 1194 int k3_udma_glue_pop_rx_chn(struct k3_udma_glue_rx_channel *rx_chn, 1195 u32 flow_num, dma_addr_t *desc_dma) 1196 { 1197 struct k3_udma_glue_rx_flow *flow = &rx_chn->flows[flow_num]; 1198 1199 return k3_ringacc_ring_pop(flow->ringrx, desc_dma); 1200 } 1201 EXPORT_SYMBOL_GPL(k3_udma_glue_pop_rx_chn); 1202 1203 int k3_udma_glue_rx_get_irq(struct k3_udma_glue_rx_channel *rx_chn, 1204 u32 flow_num) 1205 { 1206 struct k3_udma_glue_rx_flow *flow; 1207 1208 flow = &rx_chn->flows[flow_num]; 1209 1210 flow->virq = k3_ringacc_get_ring_irq_num(flow->ringrx); 1211 1212 return flow->virq; 1213 } 1214 EXPORT_SYMBOL_GPL(k3_udma_glue_rx_get_irq); 1215 1216 struct device * 1217 k3_udma_glue_rx_get_dma_device(struct k3_udma_glue_rx_channel *rx_chn) 1218 { 1219 return xudma_get_device(rx_chn->common.udmax); 1220 } 1221 EXPORT_SYMBOL_GPL(k3_udma_glue_rx_get_dma_device); 1222