1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (c) 2021, MediaTek Inc. 4 * Copyright (c) 2021-2022, Intel Corporation. 5 * 6 * Authors: 7 * Amir Hanania <amir.hanania@intel.com> 8 * Haijun Liu <haijun.liu@mediatek.com> 9 * Eliot Lee <eliot.lee@intel.com> 10 * Moises Veleta <moises.veleta@intel.com> 11 * Ricardo Martinez <ricardo.martinez@linux.intel.com> 12 * 13 * Contributors: 14 * Chiranjeevi Rapolu <chiranjeevi.rapolu@intel.com> 15 * Sreehari Kancharla <sreehari.kancharla@intel.com> 16 */ 17 18 #include <linux/atomic.h> 19 #include <linux/bitfield.h> 20 #include <linux/delay.h> 21 #include <linux/device.h> 22 #include <linux/dma-direction.h> 23 #include <linux/dma-mapping.h> 24 #include <linux/err.h> 25 #include <linux/gfp.h> 26 #include <linux/kernel.h> 27 #include <linux/kthread.h> 28 #include <linux/list.h> 29 #include <linux/minmax.h> 30 #include <linux/netdevice.h> 31 #include <linux/pm_runtime.h> 32 #include <linux/sched.h> 33 #include <linux/spinlock.h> 34 #include <linux/skbuff.h> 35 #include <linux/types.h> 36 #include <linux/wait.h> 37 #include <linux/workqueue.h> 38 39 #include "t7xx_dpmaif.h" 40 #include "t7xx_hif_dpmaif.h" 41 #include "t7xx_hif_dpmaif_tx.h" 42 #include "t7xx_pci.h" 43 44 #define DPMAIF_SKB_TX_BURST_CNT 5 45 #define DPMAIF_DRB_LIST_LEN 6144 46 47 /* DRB dtype */ 48 #define DES_DTYP_PD 0 49 #define DES_DTYP_MSG 1 50 51 static unsigned int t7xx_dpmaif_update_drb_rd_idx(struct dpmaif_ctrl *dpmaif_ctrl, 52 unsigned int q_num) 53 { 54 struct dpmaif_tx_queue *txq = &dpmaif_ctrl->txq[q_num]; 55 unsigned int old_sw_rd_idx, new_hw_rd_idx, drb_cnt; 56 unsigned long flags; 57 58 if (!txq->que_started) 59 return 0; 60 61 old_sw_rd_idx = txq->drb_rd_idx; 62 new_hw_rd_idx = t7xx_dpmaif_ul_get_rd_idx(&dpmaif_ctrl->hw_info, q_num); 63 if (new_hw_rd_idx >= DPMAIF_DRB_LIST_LEN) { 64 dev_err(dpmaif_ctrl->dev, "Out of range read index: %u\n", new_hw_rd_idx); 65 return 0; 66 } 67 68 if (old_sw_rd_idx <= new_hw_rd_idx) 69 drb_cnt = new_hw_rd_idx - old_sw_rd_idx; 70 else 71 drb_cnt = txq->drb_size_cnt - old_sw_rd_idx + new_hw_rd_idx; 72 73 spin_lock_irqsave(&txq->tx_lock, flags); 74 txq->drb_rd_idx = new_hw_rd_idx; 75 spin_unlock_irqrestore(&txq->tx_lock, flags); 76 77 return drb_cnt; 78 } 79 80 static unsigned int t7xx_dpmaif_release_tx_buffer(struct dpmaif_ctrl *dpmaif_ctrl, 81 unsigned int q_num, unsigned int release_cnt) 82 { 83 struct dpmaif_tx_queue *txq = &dpmaif_ctrl->txq[q_num]; 84 struct dpmaif_callbacks *cb = dpmaif_ctrl->callbacks; 85 struct dpmaif_drb_skb *cur_drb_skb, *drb_skb_base; 86 struct dpmaif_drb *cur_drb, *drb_base; 87 unsigned int drb_cnt, i, cur_idx; 88 unsigned long flags; 89 90 drb_skb_base = txq->drb_skb_base; 91 drb_base = txq->drb_base; 92 93 spin_lock_irqsave(&txq->tx_lock, flags); 94 drb_cnt = txq->drb_size_cnt; 95 cur_idx = txq->drb_release_rd_idx; 96 spin_unlock_irqrestore(&txq->tx_lock, flags); 97 98 for (i = 0; i < release_cnt; i++) { 99 cur_drb = drb_base + cur_idx; 100 if (FIELD_GET(DRB_HDR_DTYP, le32_to_cpu(cur_drb->header)) == DES_DTYP_PD) { 101 cur_drb_skb = drb_skb_base + cur_idx; 102 if (!cur_drb_skb->is_msg) 103 dma_unmap_single(dpmaif_ctrl->dev, cur_drb_skb->bus_addr, 104 cur_drb_skb->data_len, DMA_TO_DEVICE); 105 106 if (!FIELD_GET(DRB_HDR_CONT, le32_to_cpu(cur_drb->header))) { 107 if (!cur_drb_skb->skb) { 108 dev_err(dpmaif_ctrl->dev, 109 "txq%u: DRB check fail, invalid skb\n", q_num); 110 continue; 111 } 112 113 dev_kfree_skb_any(cur_drb_skb->skb); 114 } 115 116 cur_drb_skb->skb = NULL; 117 } 118 119 spin_lock_irqsave(&txq->tx_lock, flags); 120 cur_idx = t7xx_ring_buf_get_next_wr_idx(drb_cnt, cur_idx); 121 txq->drb_release_rd_idx = cur_idx; 122 spin_unlock_irqrestore(&txq->tx_lock, flags); 123 124 if (atomic_inc_return(&txq->tx_budget) > txq->drb_size_cnt / 8) 125 cb->state_notify(dpmaif_ctrl->t7xx_dev, DMPAIF_TXQ_STATE_IRQ, txq->index); 126 } 127 128 if (FIELD_GET(DRB_HDR_CONT, le32_to_cpu(cur_drb->header))) 129 dev_err(dpmaif_ctrl->dev, "txq%u: DRB not marked as the last one\n", q_num); 130 131 return i; 132 } 133 134 static int t7xx_dpmaif_tx_release(struct dpmaif_ctrl *dpmaif_ctrl, 135 unsigned int q_num, unsigned int budget) 136 { 137 struct dpmaif_tx_queue *txq = &dpmaif_ctrl->txq[q_num]; 138 unsigned int rel_cnt, real_rel_cnt; 139 140 /* Update read index from HW */ 141 t7xx_dpmaif_update_drb_rd_idx(dpmaif_ctrl, q_num); 142 143 rel_cnt = t7xx_ring_buf_rd_wr_count(txq->drb_size_cnt, txq->drb_release_rd_idx, 144 txq->drb_rd_idx, DPMAIF_READ); 145 146 real_rel_cnt = min_not_zero(budget, rel_cnt); 147 if (real_rel_cnt) 148 real_rel_cnt = t7xx_dpmaif_release_tx_buffer(dpmaif_ctrl, q_num, real_rel_cnt); 149 150 return real_rel_cnt < rel_cnt ? -EAGAIN : 0; 151 } 152 153 static bool t7xx_dpmaif_drb_ring_not_empty(struct dpmaif_tx_queue *txq) 154 { 155 return !!t7xx_dpmaif_update_drb_rd_idx(txq->dpmaif_ctrl, txq->index); 156 } 157 158 static void t7xx_dpmaif_tx_done(struct work_struct *work) 159 { 160 struct dpmaif_tx_queue *txq = container_of(work, struct dpmaif_tx_queue, dpmaif_tx_work); 161 struct dpmaif_ctrl *dpmaif_ctrl = txq->dpmaif_ctrl; 162 struct dpmaif_hw_info *hw_info; 163 int ret; 164 165 ret = pm_runtime_resume_and_get(dpmaif_ctrl->dev); 166 if (ret < 0 && ret != -EACCES) 167 return; 168 169 /* The device may be in low power state. Disable sleep if needed */ 170 t7xx_pci_disable_sleep(dpmaif_ctrl->t7xx_dev); 171 if (t7xx_pci_sleep_disable_complete(dpmaif_ctrl->t7xx_dev)) { 172 hw_info = &dpmaif_ctrl->hw_info; 173 ret = t7xx_dpmaif_tx_release(dpmaif_ctrl, txq->index, txq->drb_size_cnt); 174 if (ret == -EAGAIN || 175 (t7xx_dpmaif_ul_clr_done(hw_info, txq->index) && 176 t7xx_dpmaif_drb_ring_not_empty(txq))) { 177 queue_work(dpmaif_ctrl->txq[txq->index].worker, 178 &dpmaif_ctrl->txq[txq->index].dpmaif_tx_work); 179 /* Give the device time to enter the low power state */ 180 t7xx_dpmaif_clr_ip_busy_sts(hw_info); 181 } else { 182 t7xx_dpmaif_clr_ip_busy_sts(hw_info); 183 t7xx_dpmaif_unmask_ulq_intr(hw_info, txq->index); 184 } 185 } 186 187 t7xx_pci_enable_sleep(dpmaif_ctrl->t7xx_dev); 188 pm_runtime_put_autosuspend(dpmaif_ctrl->dev); 189 } 190 191 static void t7xx_setup_msg_drb(struct dpmaif_ctrl *dpmaif_ctrl, unsigned int q_num, 192 unsigned int cur_idx, unsigned int pkt_len, unsigned int count_l, 193 unsigned int channel_id) 194 { 195 struct dpmaif_drb *drb_base = dpmaif_ctrl->txq[q_num].drb_base; 196 struct dpmaif_drb *drb = drb_base + cur_idx; 197 198 drb->header = cpu_to_le32(FIELD_PREP(DRB_HDR_DTYP, DES_DTYP_MSG) | 199 FIELD_PREP(DRB_HDR_CONT, 1) | 200 FIELD_PREP(DRB_HDR_DATA_LEN, pkt_len)); 201 202 drb->msg.msg_hdr = cpu_to_le32(FIELD_PREP(DRB_MSG_COUNT_L, count_l) | 203 FIELD_PREP(DRB_MSG_CHANNEL_ID, channel_id) | 204 FIELD_PREP(DRB_MSG_L4_CHK, 1)); 205 } 206 207 static void t7xx_setup_payload_drb(struct dpmaif_ctrl *dpmaif_ctrl, unsigned int q_num, 208 unsigned int cur_idx, dma_addr_t data_addr, 209 unsigned int pkt_size, bool last_one) 210 { 211 struct dpmaif_drb *drb_base = dpmaif_ctrl->txq[q_num].drb_base; 212 struct dpmaif_drb *drb = drb_base + cur_idx; 213 u32 header; 214 215 header = FIELD_PREP(DRB_HDR_DTYP, DES_DTYP_PD) | FIELD_PREP(DRB_HDR_DATA_LEN, pkt_size); 216 if (!last_one) 217 header |= FIELD_PREP(DRB_HDR_CONT, 1); 218 219 drb->header = cpu_to_le32(header); 220 drb->pd.data_addr_l = cpu_to_le32(lower_32_bits(data_addr)); 221 drb->pd.data_addr_h = cpu_to_le32(upper_32_bits(data_addr)); 222 } 223 224 static void t7xx_record_drb_skb(struct dpmaif_ctrl *dpmaif_ctrl, unsigned int q_num, 225 unsigned int cur_idx, struct sk_buff *skb, bool is_msg, 226 bool is_frag, bool is_last_one, dma_addr_t bus_addr, 227 unsigned int data_len) 228 { 229 struct dpmaif_drb_skb *drb_skb_base = dpmaif_ctrl->txq[q_num].drb_skb_base; 230 struct dpmaif_drb_skb *drb_skb = drb_skb_base + cur_idx; 231 232 drb_skb->skb = skb; 233 drb_skb->bus_addr = bus_addr; 234 drb_skb->data_len = data_len; 235 drb_skb->index = cur_idx; 236 drb_skb->is_msg = is_msg; 237 drb_skb->is_frag = is_frag; 238 drb_skb->is_last = is_last_one; 239 } 240 241 static int t7xx_dpmaif_add_skb_to_ring(struct dpmaif_ctrl *dpmaif_ctrl, struct sk_buff *skb) 242 { 243 struct dpmaif_callbacks *cb = dpmaif_ctrl->callbacks; 244 unsigned int wr_cnt, send_cnt, payload_cnt; 245 unsigned int cur_idx, drb_wr_idx_backup; 246 struct skb_shared_info *shinfo; 247 struct dpmaif_tx_queue *txq; 248 struct t7xx_skb_cb *skb_cb; 249 unsigned long flags; 250 251 skb_cb = T7XX_SKB_CB(skb); 252 txq = &dpmaif_ctrl->txq[skb_cb->txq_number]; 253 if (!txq->que_started || dpmaif_ctrl->state != DPMAIF_STATE_PWRON) 254 return -ENODEV; 255 256 atomic_set(&txq->tx_processing, 1); 257 /* Ensure tx_processing is changed to 1 before actually begin TX flow */ 258 smp_mb(); 259 260 shinfo = skb_shinfo(skb); 261 if (shinfo->frag_list) 262 dev_warn_ratelimited(dpmaif_ctrl->dev, "frag_list not supported\n"); 263 264 payload_cnt = shinfo->nr_frags + 1; 265 /* nr_frags: frag cnt, 1: skb->data, 1: msg DRB */ 266 send_cnt = payload_cnt + 1; 267 268 spin_lock_irqsave(&txq->tx_lock, flags); 269 cur_idx = txq->drb_wr_idx; 270 drb_wr_idx_backup = cur_idx; 271 txq->drb_wr_idx += send_cnt; 272 if (txq->drb_wr_idx >= txq->drb_size_cnt) 273 txq->drb_wr_idx -= txq->drb_size_cnt; 274 t7xx_setup_msg_drb(dpmaif_ctrl, txq->index, cur_idx, skb->len, 0, skb_cb->netif_idx); 275 t7xx_record_drb_skb(dpmaif_ctrl, txq->index, cur_idx, skb, true, 0, 0, 0, 0); 276 spin_unlock_irqrestore(&txq->tx_lock, flags); 277 278 for (wr_cnt = 0; wr_cnt < payload_cnt; wr_cnt++) { 279 bool is_frag, is_last_one = wr_cnt == payload_cnt - 1; 280 unsigned int data_len; 281 dma_addr_t bus_addr; 282 void *data_addr; 283 284 if (!wr_cnt) { 285 data_len = skb_headlen(skb); 286 data_addr = skb->data; 287 is_frag = false; 288 } else { 289 skb_frag_t *frag = shinfo->frags + wr_cnt - 1; 290 291 data_len = skb_frag_size(frag); 292 data_addr = skb_frag_address(frag); 293 is_frag = true; 294 } 295 296 bus_addr = dma_map_single(dpmaif_ctrl->dev, data_addr, data_len, DMA_TO_DEVICE); 297 if (dma_mapping_error(dpmaif_ctrl->dev, bus_addr)) 298 goto unmap_buffers; 299 300 cur_idx = t7xx_ring_buf_get_next_wr_idx(txq->drb_size_cnt, cur_idx); 301 302 spin_lock_irqsave(&txq->tx_lock, flags); 303 t7xx_setup_payload_drb(dpmaif_ctrl, txq->index, cur_idx, bus_addr, data_len, 304 is_last_one); 305 t7xx_record_drb_skb(dpmaif_ctrl, txq->index, cur_idx, skb, false, is_frag, 306 is_last_one, bus_addr, data_len); 307 spin_unlock_irqrestore(&txq->tx_lock, flags); 308 } 309 310 if (atomic_sub_return(send_cnt, &txq->tx_budget) <= (MAX_SKB_FRAGS + 2)) 311 cb->state_notify(dpmaif_ctrl->t7xx_dev, DMPAIF_TXQ_STATE_FULL, txq->index); 312 313 atomic_set(&txq->tx_processing, 0); 314 315 return 0; 316 317 unmap_buffers: 318 while (wr_cnt--) { 319 struct dpmaif_drb_skb *drb_skb = txq->drb_skb_base; 320 321 cur_idx = cur_idx ? cur_idx - 1 : txq->drb_size_cnt - 1; 322 drb_skb += cur_idx; 323 dma_unmap_single(dpmaif_ctrl->dev, drb_skb->bus_addr, 324 drb_skb->data_len, DMA_TO_DEVICE); 325 } 326 327 txq->drb_wr_idx = drb_wr_idx_backup; 328 atomic_set(&txq->tx_processing, 0); 329 330 return -ENOMEM; 331 } 332 333 static bool t7xx_tx_lists_are_all_empty(const struct dpmaif_ctrl *dpmaif_ctrl) 334 { 335 int i; 336 337 for (i = 0; i < DPMAIF_TXQ_NUM; i++) { 338 if (!skb_queue_empty(&dpmaif_ctrl->txq[i].tx_skb_head)) 339 return false; 340 } 341 342 return true; 343 } 344 345 /* Currently, only the default TX queue is used */ 346 static struct dpmaif_tx_queue *t7xx_select_tx_queue(struct dpmaif_ctrl *dpmaif_ctrl) 347 { 348 struct dpmaif_tx_queue *txq; 349 350 txq = &dpmaif_ctrl->txq[DPMAIF_TX_DEFAULT_QUEUE]; 351 if (!txq->que_started) 352 return NULL; 353 354 return txq; 355 } 356 357 static unsigned int t7xx_txq_drb_wr_available(struct dpmaif_tx_queue *txq) 358 { 359 return t7xx_ring_buf_rd_wr_count(txq->drb_size_cnt, txq->drb_release_rd_idx, 360 txq->drb_wr_idx, DPMAIF_WRITE); 361 } 362 363 static unsigned int t7xx_skb_drb_cnt(struct sk_buff *skb) 364 { 365 /* Normal DRB (frags data + skb linear data) + msg DRB */ 366 return skb_shinfo(skb)->nr_frags + 2; 367 } 368 369 static int t7xx_txq_burst_send_skb(struct dpmaif_tx_queue *txq) 370 { 371 unsigned int drb_remain_cnt, i; 372 unsigned int send_drb_cnt; 373 int drb_cnt = 0; 374 int ret = 0; 375 376 drb_remain_cnt = t7xx_txq_drb_wr_available(txq); 377 378 for (i = 0; i < DPMAIF_SKB_TX_BURST_CNT; i++) { 379 struct sk_buff *skb; 380 381 skb = skb_peek(&txq->tx_skb_head); 382 if (!skb) 383 break; 384 385 send_drb_cnt = t7xx_skb_drb_cnt(skb); 386 if (drb_remain_cnt < send_drb_cnt) { 387 drb_remain_cnt = t7xx_txq_drb_wr_available(txq); 388 continue; 389 } 390 391 drb_remain_cnt -= send_drb_cnt; 392 393 ret = t7xx_dpmaif_add_skb_to_ring(txq->dpmaif_ctrl, skb); 394 if (ret < 0) { 395 dev_err(txq->dpmaif_ctrl->dev, 396 "Failed to add skb to device's ring: %d\n", ret); 397 break; 398 } 399 400 drb_cnt += send_drb_cnt; 401 skb_unlink(skb, &txq->tx_skb_head); 402 } 403 404 if (drb_cnt > 0) 405 return drb_cnt; 406 407 return ret; 408 } 409 410 static void t7xx_do_tx_hw_push(struct dpmaif_ctrl *dpmaif_ctrl) 411 { 412 bool wait_disable_sleep = true; 413 414 do { 415 struct dpmaif_tx_queue *txq; 416 int drb_send_cnt; 417 418 txq = t7xx_select_tx_queue(dpmaif_ctrl); 419 if (!txq) 420 return; 421 422 drb_send_cnt = t7xx_txq_burst_send_skb(txq); 423 if (drb_send_cnt <= 0) { 424 usleep_range(10, 20); 425 cond_resched(); 426 continue; 427 } 428 429 /* Wait for the PCIe resource to unlock */ 430 if (wait_disable_sleep) { 431 if (!t7xx_pci_sleep_disable_complete(dpmaif_ctrl->t7xx_dev)) 432 return; 433 434 wait_disable_sleep = false; 435 } 436 437 t7xx_dpmaif_ul_update_hw_drb_cnt(&dpmaif_ctrl->hw_info, txq->index, 438 drb_send_cnt * DPMAIF_UL_DRB_SIZE_WORD); 439 440 cond_resched(); 441 } while (!t7xx_tx_lists_are_all_empty(dpmaif_ctrl) && !kthread_should_stop() && 442 (dpmaif_ctrl->state == DPMAIF_STATE_PWRON)); 443 } 444 445 static int t7xx_dpmaif_tx_hw_push_thread(void *arg) 446 { 447 struct dpmaif_ctrl *dpmaif_ctrl = arg; 448 int ret; 449 450 while (!kthread_should_stop()) { 451 if (t7xx_tx_lists_are_all_empty(dpmaif_ctrl) || 452 dpmaif_ctrl->state != DPMAIF_STATE_PWRON) { 453 if (wait_event_interruptible(dpmaif_ctrl->tx_wq, 454 (!t7xx_tx_lists_are_all_empty(dpmaif_ctrl) && 455 dpmaif_ctrl->state == DPMAIF_STATE_PWRON) || 456 kthread_should_stop())) 457 continue; 458 459 if (kthread_should_stop()) 460 break; 461 } 462 463 ret = pm_runtime_resume_and_get(dpmaif_ctrl->dev); 464 if (ret < 0 && ret != -EACCES) 465 return ret; 466 467 t7xx_pci_disable_sleep(dpmaif_ctrl->t7xx_dev); 468 t7xx_do_tx_hw_push(dpmaif_ctrl); 469 t7xx_pci_enable_sleep(dpmaif_ctrl->t7xx_dev); 470 pm_runtime_put_autosuspend(dpmaif_ctrl->dev); 471 } 472 473 return 0; 474 } 475 476 int t7xx_dpmaif_tx_thread_init(struct dpmaif_ctrl *dpmaif_ctrl) 477 { 478 init_waitqueue_head(&dpmaif_ctrl->tx_wq); 479 dpmaif_ctrl->tx_thread = kthread_run(t7xx_dpmaif_tx_hw_push_thread, 480 dpmaif_ctrl, "dpmaif_tx_hw_push"); 481 return PTR_ERR_OR_ZERO(dpmaif_ctrl->tx_thread); 482 } 483 484 void t7xx_dpmaif_tx_thread_rel(struct dpmaif_ctrl *dpmaif_ctrl) 485 { 486 if (dpmaif_ctrl->tx_thread) 487 kthread_stop(dpmaif_ctrl->tx_thread); 488 } 489 490 /** 491 * t7xx_dpmaif_tx_send_skb() - Add skb to the transmit queue. 492 * @dpmaif_ctrl: Pointer to struct dpmaif_ctrl. 493 * @txq_number: Queue number to xmit on. 494 * @skb: Pointer to the skb to transmit. 495 * 496 * Add the skb to the queue of the skbs to be transmit. 497 * Wake up the thread that push the skbs from the queue to the HW. 498 * 499 * Return: 500 * * 0 - Success. 501 * * -EBUSY - Tx budget exhausted. 502 * In normal circumstances t7xx_dpmaif_add_skb_to_ring() must report the txq full 503 * state to prevent this error condition. 504 */ 505 int t7xx_dpmaif_tx_send_skb(struct dpmaif_ctrl *dpmaif_ctrl, unsigned int txq_number, 506 struct sk_buff *skb) 507 { 508 struct dpmaif_tx_queue *txq = &dpmaif_ctrl->txq[txq_number]; 509 struct dpmaif_callbacks *cb = dpmaif_ctrl->callbacks; 510 struct t7xx_skb_cb *skb_cb; 511 512 if (atomic_read(&txq->tx_budget) <= t7xx_skb_drb_cnt(skb)) { 513 cb->state_notify(dpmaif_ctrl->t7xx_dev, DMPAIF_TXQ_STATE_FULL, txq_number); 514 return -EBUSY; 515 } 516 517 skb_cb = T7XX_SKB_CB(skb); 518 skb_cb->txq_number = txq_number; 519 skb_queue_tail(&txq->tx_skb_head, skb); 520 wake_up(&dpmaif_ctrl->tx_wq); 521 522 return 0; 523 } 524 525 void t7xx_dpmaif_irq_tx_done(struct dpmaif_ctrl *dpmaif_ctrl, unsigned int que_mask) 526 { 527 int i; 528 529 for (i = 0; i < DPMAIF_TXQ_NUM; i++) { 530 if (que_mask & BIT(i)) 531 queue_work(dpmaif_ctrl->txq[i].worker, &dpmaif_ctrl->txq[i].dpmaif_tx_work); 532 } 533 } 534 535 static int t7xx_dpmaif_tx_drb_buf_init(struct dpmaif_tx_queue *txq) 536 { 537 size_t brb_skb_size, brb_pd_size; 538 539 brb_pd_size = DPMAIF_DRB_LIST_LEN * sizeof(struct dpmaif_drb); 540 brb_skb_size = DPMAIF_DRB_LIST_LEN * sizeof(struct dpmaif_drb_skb); 541 542 txq->drb_size_cnt = DPMAIF_DRB_LIST_LEN; 543 544 /* For HW && AP SW */ 545 txq->drb_base = dma_alloc_coherent(txq->dpmaif_ctrl->dev, brb_pd_size, 546 &txq->drb_bus_addr, GFP_KERNEL | __GFP_ZERO); 547 if (!txq->drb_base) 548 return -ENOMEM; 549 550 /* For AP SW to record the skb information */ 551 txq->drb_skb_base = devm_kzalloc(txq->dpmaif_ctrl->dev, brb_skb_size, GFP_KERNEL); 552 if (!txq->drb_skb_base) { 553 dma_free_coherent(txq->dpmaif_ctrl->dev, brb_pd_size, 554 txq->drb_base, txq->drb_bus_addr); 555 return -ENOMEM; 556 } 557 558 return 0; 559 } 560 561 static void t7xx_dpmaif_tx_free_drb_skb(struct dpmaif_tx_queue *txq) 562 { 563 struct dpmaif_drb_skb *drb_skb, *drb_skb_base = txq->drb_skb_base; 564 unsigned int i; 565 566 if (!drb_skb_base) 567 return; 568 569 for (i = 0; i < txq->drb_size_cnt; i++) { 570 drb_skb = drb_skb_base + i; 571 if (!drb_skb->skb) 572 continue; 573 574 if (!drb_skb->is_msg) 575 dma_unmap_single(txq->dpmaif_ctrl->dev, drb_skb->bus_addr, 576 drb_skb->data_len, DMA_TO_DEVICE); 577 578 if (drb_skb->is_last) { 579 dev_kfree_skb(drb_skb->skb); 580 drb_skb->skb = NULL; 581 } 582 } 583 } 584 585 static void t7xx_dpmaif_tx_drb_buf_rel(struct dpmaif_tx_queue *txq) 586 { 587 if (txq->drb_base) 588 dma_free_coherent(txq->dpmaif_ctrl->dev, 589 txq->drb_size_cnt * sizeof(struct dpmaif_drb), 590 txq->drb_base, txq->drb_bus_addr); 591 592 t7xx_dpmaif_tx_free_drb_skb(txq); 593 } 594 595 /** 596 * t7xx_dpmaif_txq_init() - Initialize TX queue. 597 * @txq: Pointer to struct dpmaif_tx_queue. 598 * 599 * Initialize the TX queue data structure and allocate memory for it to use. 600 * 601 * Return: 602 * * 0 - Success. 603 * * -ERROR - Error code from failure sub-initializations. 604 */ 605 int t7xx_dpmaif_txq_init(struct dpmaif_tx_queue *txq) 606 { 607 int ret; 608 609 skb_queue_head_init(&txq->tx_skb_head); 610 init_waitqueue_head(&txq->req_wq); 611 atomic_set(&txq->tx_budget, DPMAIF_DRB_LIST_LEN); 612 613 ret = t7xx_dpmaif_tx_drb_buf_init(txq); 614 if (ret) { 615 dev_err(txq->dpmaif_ctrl->dev, "Failed to initialize DRB buffers: %d\n", ret); 616 return ret; 617 } 618 619 txq->worker = alloc_ordered_workqueue("md_dpmaif_tx%d_worker", 620 WQ_MEM_RECLAIM | (txq->index ? 0 : WQ_HIGHPRI), 621 txq->index); 622 if (!txq->worker) 623 return -ENOMEM; 624 625 INIT_WORK(&txq->dpmaif_tx_work, t7xx_dpmaif_tx_done); 626 spin_lock_init(&txq->tx_lock); 627 628 return 0; 629 } 630 631 void t7xx_dpmaif_txq_free(struct dpmaif_tx_queue *txq) 632 { 633 if (txq->worker) 634 destroy_workqueue(txq->worker); 635 636 skb_queue_purge(&txq->tx_skb_head); 637 t7xx_dpmaif_tx_drb_buf_rel(txq); 638 } 639 640 void t7xx_dpmaif_tx_stop(struct dpmaif_ctrl *dpmaif_ctrl) 641 { 642 int i; 643 644 for (i = 0; i < DPMAIF_TXQ_NUM; i++) { 645 struct dpmaif_tx_queue *txq; 646 int count = 0; 647 648 txq = &dpmaif_ctrl->txq[i]; 649 txq->que_started = false; 650 /* Make sure TXQ is disabled */ 651 smp_mb(); 652 653 /* Wait for active Tx to be done */ 654 while (atomic_read(&txq->tx_processing)) { 655 if (++count >= DPMAIF_MAX_CHECK_COUNT) { 656 dev_err(dpmaif_ctrl->dev, "TX queue stop failed\n"); 657 break; 658 } 659 } 660 } 661 } 662 663 static void t7xx_dpmaif_txq_flush_rel(struct dpmaif_tx_queue *txq) 664 { 665 txq->que_started = false; 666 667 cancel_work_sync(&txq->dpmaif_tx_work); 668 flush_work(&txq->dpmaif_tx_work); 669 t7xx_dpmaif_tx_free_drb_skb(txq); 670 671 txq->drb_rd_idx = 0; 672 txq->drb_wr_idx = 0; 673 txq->drb_release_rd_idx = 0; 674 } 675 676 void t7xx_dpmaif_tx_clear(struct dpmaif_ctrl *dpmaif_ctrl) 677 { 678 int i; 679 680 for (i = 0; i < DPMAIF_TXQ_NUM; i++) 681 t7xx_dpmaif_txq_flush_rel(&dpmaif_ctrl->txq[i]); 682 } 683