1 // SPDX-License-Identifier: BSD-3-Clause-Clear
2 /*
3 * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved.
4 * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
5 */
6
7 #include "core.h"
8 #include "dp_tx.h"
9 #include "hif.h"
10 #include "hal.h"
11 #include "debug.h"
12 #include "peer.h"
13 #include "dp_cmn.h"
14
15 enum ath12k_dp_desc_type {
16 ATH12K_DP_TX_DESC,
17 ATH12K_DP_RX_DESC,
18 };
19
ath12k_dp_peer_cleanup(struct ath12k * ar,int vdev_id,const u8 * addr)20 void ath12k_dp_peer_cleanup(struct ath12k *ar, int vdev_id, const u8 *addr)
21 {
22 struct ath12k_base *ab = ar->ab;
23 struct ath12k_dp_link_peer *peer;
24 struct ath12k_dp *dp = ath12k_ab_to_dp(ab);
25
26 /* TODO: Any other peer specific DP cleanup */
27
28 spin_lock_bh(&dp->dp_lock);
29 peer = ath12k_dp_link_peer_find_by_vdev_and_addr(dp, vdev_id, addr);
30 if (!peer || !peer->dp_peer) {
31 ath12k_warn(ab, "failed to lookup peer %pM on vdev %d\n",
32 addr, vdev_id);
33 spin_unlock_bh(&dp->dp_lock);
34 return;
35 }
36
37 if (!peer->primary_link) {
38 spin_unlock_bh(&dp->dp_lock);
39 return;
40 }
41
42 ath12k_dp_rx_peer_tid_cleanup(ar, peer);
43 peer->dp_peer->dp_setup_done = false;
44 spin_unlock_bh(&dp->dp_lock);
45 }
46
ath12k_dp_peer_setup(struct ath12k * ar,int vdev_id,const u8 * addr)47 int ath12k_dp_peer_setup(struct ath12k *ar, int vdev_id, const u8 *addr)
48 {
49 struct ath12k_base *ab = ar->ab;
50 struct ath12k_dp_link_peer *peer;
51 u32 reo_dest;
52 int ret = 0, tid;
53 struct ath12k_dp *dp = ath12k_ab_to_dp(ab);
54
55 /* NOTE: reo_dest ring id starts from 1 unlike mac_id which starts from 0 */
56 reo_dest = ar->dp.mac_id + 1;
57 ret = ath12k_wmi_set_peer_param(ar, addr, vdev_id,
58 WMI_PEER_SET_DEFAULT_ROUTING,
59 DP_RX_HASH_ENABLE | (reo_dest << 1));
60
61 if (ret) {
62 ath12k_warn(ab, "failed to set default routing %d peer :%pM vdev_id :%d\n",
63 ret, addr, vdev_id);
64 return ret;
65 }
66
67 for (tid = 0; tid <= IEEE80211_NUM_TIDS; tid++) {
68 ret = ath12k_dp_rx_peer_tid_setup(ar, addr, vdev_id, tid, 1, 0,
69 HAL_PN_TYPE_NONE);
70 if (ret) {
71 ath12k_warn(ab, "failed to setup rxd tid queue for tid %d: %d\n",
72 tid, ret);
73 goto peer_clean;
74 }
75 }
76
77 ret = ath12k_dp_rx_peer_frag_setup(ar, addr, vdev_id);
78 if (ret) {
79 ath12k_warn(ab, "failed to setup rx defrag context\n");
80 goto peer_clean;
81 }
82
83 /* TODO: Setup other peer specific resource used in data path */
84
85 return 0;
86
87 peer_clean:
88 spin_lock_bh(&dp->dp_lock);
89
90 peer = ath12k_dp_link_peer_find_by_vdev_and_addr(dp, vdev_id, addr);
91 if (!peer) {
92 ath12k_warn(ab, "failed to find the peer to del rx tid\n");
93 spin_unlock_bh(&dp->dp_lock);
94 return -ENOENT;
95 }
96
97 for (tid--; tid >= 0; tid--)
98 ath12k_dp_arch_rx_peer_tid_delete(dp, peer, tid);
99
100 spin_unlock_bh(&dp->dp_lock);
101
102 return ret;
103 }
104
ath12k_dp_srng_cleanup(struct ath12k_base * ab,struct dp_srng * ring)105 void ath12k_dp_srng_cleanup(struct ath12k_base *ab, struct dp_srng *ring)
106 {
107 if (!ring->vaddr_unaligned)
108 return;
109
110 dma_free_coherent(ab->dev, ring->size, ring->vaddr_unaligned,
111 ring->paddr_unaligned);
112
113 ring->vaddr_unaligned = NULL;
114 }
115
ath12k_dp_srng_find_ring_in_mask(int ring_num,const u8 * grp_mask)116 static int ath12k_dp_srng_find_ring_in_mask(int ring_num, const u8 *grp_mask)
117 {
118 int ext_group_num;
119 u8 mask = 1 << ring_num;
120
121 for (ext_group_num = 0; ext_group_num < ATH12K_EXT_IRQ_GRP_NUM_MAX;
122 ext_group_num++) {
123 if (mask & grp_mask[ext_group_num])
124 return ext_group_num;
125 }
126
127 return -ENOENT;
128 }
129
ath12k_dp_srng_calculate_msi_group(struct ath12k_base * ab,enum hal_ring_type type,int ring_num)130 static int ath12k_dp_srng_calculate_msi_group(struct ath12k_base *ab,
131 enum hal_ring_type type, int ring_num)
132 {
133 const struct ath12k_hal_tcl_to_wbm_rbm_map *map;
134 const u8 *grp_mask;
135 int i;
136
137 switch (type) {
138 case HAL_WBM2SW_RELEASE:
139 if (ring_num == HAL_WBM2SW_REL_ERR_RING_NUM) {
140 grp_mask = &ab->hw_params->ring_mask->rx_wbm_rel[0];
141 ring_num = 0;
142 } else {
143 map = ab->hal.tcl_to_wbm_rbm_map;
144 for (i = 0; i < ab->hw_params->max_tx_ring; i++) {
145 if (ring_num == map[i].wbm_ring_num) {
146 ring_num = i;
147 break;
148 }
149 }
150
151 grp_mask = &ab->hw_params->ring_mask->tx[0];
152 }
153 break;
154 case HAL_REO_EXCEPTION:
155 grp_mask = &ab->hw_params->ring_mask->rx_err[0];
156 break;
157 case HAL_REO_DST:
158 grp_mask = &ab->hw_params->ring_mask->rx[0];
159 break;
160 case HAL_REO_STATUS:
161 grp_mask = &ab->hw_params->ring_mask->reo_status[0];
162 break;
163 case HAL_RXDMA_MONITOR_STATUS:
164 grp_mask = &ab->hw_params->ring_mask->rx_mon_status[0];
165 break;
166 case HAL_RXDMA_MONITOR_DST:
167 grp_mask = &ab->hw_params->ring_mask->rx_mon_dest[0];
168 break;
169 case HAL_TX_MONITOR_DST:
170 grp_mask = &ab->hw_params->ring_mask->tx_mon_dest[0];
171 break;
172 case HAL_RXDMA_BUF:
173 grp_mask = &ab->hw_params->ring_mask->host2rxdma[0];
174 break;
175 case HAL_RXDMA_MONITOR_BUF:
176 case HAL_TCL_DATA:
177 case HAL_TCL_CMD:
178 case HAL_REO_CMD:
179 case HAL_SW2WBM_RELEASE:
180 case HAL_WBM_IDLE_LINK:
181 case HAL_TCL_STATUS:
182 case HAL_REO_REINJECT:
183 case HAL_CE_SRC:
184 case HAL_CE_DST:
185 case HAL_CE_DST_STATUS:
186 default:
187 return -ENOENT;
188 }
189
190 return ath12k_dp_srng_find_ring_in_mask(ring_num, grp_mask);
191 }
192
ath12k_dp_srng_msi_setup(struct ath12k_base * ab,struct hal_srng_params * ring_params,enum hal_ring_type type,int ring_num)193 static void ath12k_dp_srng_msi_setup(struct ath12k_base *ab,
194 struct hal_srng_params *ring_params,
195 enum hal_ring_type type, int ring_num)
196 {
197 int msi_group_number, msi_data_count;
198 u32 msi_data_start, msi_irq_start, addr_lo, addr_hi;
199 int ret;
200
201 ret = ath12k_hif_get_user_msi_vector(ab, "DP",
202 &msi_data_count, &msi_data_start,
203 &msi_irq_start);
204 if (ret)
205 return;
206
207 msi_group_number = ath12k_dp_srng_calculate_msi_group(ab, type,
208 ring_num);
209 if (msi_group_number < 0) {
210 ath12k_dbg(ab, ATH12K_DBG_PCI,
211 "ring not part of an ext_group; ring_type: %d,ring_num %d",
212 type, ring_num);
213 ring_params->msi_addr = 0;
214 ring_params->msi_data = 0;
215 return;
216 }
217
218 if (msi_group_number > msi_data_count) {
219 ath12k_dbg(ab, ATH12K_DBG_PCI,
220 "multiple msi_groups share one msi, msi_group_num %d",
221 msi_group_number);
222 }
223
224 ath12k_hif_get_msi_address(ab, &addr_lo, &addr_hi);
225
226 ring_params->msi_addr = addr_lo;
227 ring_params->msi_addr |= (dma_addr_t)(((uint64_t)addr_hi) << 32);
228 ring_params->msi_data = (msi_group_number % msi_data_count)
229 + msi_data_start;
230 ring_params->flags |= HAL_SRNG_FLAGS_MSI_INTR;
231 }
232
ath12k_dp_srng_setup(struct ath12k_base * ab,struct dp_srng * ring,enum hal_ring_type type,int ring_num,int mac_id,int num_entries)233 int ath12k_dp_srng_setup(struct ath12k_base *ab, struct dp_srng *ring,
234 enum hal_ring_type type, int ring_num,
235 int mac_id, int num_entries)
236 {
237 struct hal_srng_params params = {};
238 int entry_sz = ath12k_hal_srng_get_entrysize(ab, type);
239 int max_entries = ath12k_hal_srng_get_max_entries(ab, type);
240 int ret;
241
242 if (max_entries < 0 || entry_sz < 0)
243 return -EINVAL;
244
245 if (num_entries > max_entries)
246 num_entries = max_entries;
247
248 ring->size = (num_entries * entry_sz) + HAL_RING_BASE_ALIGN - 1;
249 ring->vaddr_unaligned = dma_alloc_coherent(ab->dev, ring->size,
250 &ring->paddr_unaligned,
251 GFP_KERNEL);
252 if (!ring->vaddr_unaligned)
253 return -ENOMEM;
254
255 ring->vaddr = PTR_ALIGN(ring->vaddr_unaligned, HAL_RING_BASE_ALIGN);
256 ring->paddr = ring->paddr_unaligned + ((unsigned long)ring->vaddr -
257 (unsigned long)ring->vaddr_unaligned);
258
259 params.ring_base_vaddr = ring->vaddr;
260 params.ring_base_paddr = ring->paddr;
261 params.num_entries = num_entries;
262 ath12k_dp_srng_msi_setup(ab, ¶ms, type, ring_num + mac_id);
263
264 switch (type) {
265 case HAL_REO_DST:
266 params.intr_batch_cntr_thres_entries =
267 HAL_SRNG_INT_BATCH_THRESHOLD_RX;
268 params.intr_timer_thres_us = HAL_SRNG_INT_TIMER_THRESHOLD_RX;
269 break;
270 case HAL_RXDMA_BUF:
271 case HAL_RXDMA_MONITOR_BUF:
272 params.low_threshold = num_entries >> 3;
273 params.flags |= HAL_SRNG_FLAGS_LOW_THRESH_INTR_EN;
274 params.intr_batch_cntr_thres_entries = 0;
275 params.intr_timer_thres_us = HAL_SRNG_INT_TIMER_THRESHOLD_RX;
276 break;
277 case HAL_RXDMA_MONITOR_STATUS:
278 params.low_threshold = num_entries >> 3;
279 params.flags |= HAL_SRNG_FLAGS_LOW_THRESH_INTR_EN;
280 params.intr_batch_cntr_thres_entries = 1;
281 params.intr_timer_thres_us = HAL_SRNG_INT_TIMER_THRESHOLD_RX;
282 break;
283 case HAL_TX_MONITOR_DST:
284 params.low_threshold = DP_TX_MONITOR_BUF_SIZE_MAX >> 3;
285 params.flags |= HAL_SRNG_FLAGS_LOW_THRESH_INTR_EN;
286 params.intr_batch_cntr_thres_entries = 0;
287 params.intr_timer_thres_us = HAL_SRNG_INT_TIMER_THRESHOLD_RX;
288 break;
289 case HAL_WBM2SW_RELEASE:
290 if (ab->hw_params->hw_ops->dp_srng_is_tx_comp_ring(ring_num)) {
291 params.intr_batch_cntr_thres_entries =
292 HAL_SRNG_INT_BATCH_THRESHOLD_TX;
293 params.intr_timer_thres_us =
294 HAL_SRNG_INT_TIMER_THRESHOLD_TX;
295 break;
296 }
297 /* follow through when ring_num != HAL_WBM2SW_REL_ERR_RING_NUM */
298 fallthrough;
299 case HAL_REO_EXCEPTION:
300 case HAL_REO_REINJECT:
301 case HAL_REO_CMD:
302 case HAL_REO_STATUS:
303 case HAL_TCL_DATA:
304 case HAL_TCL_CMD:
305 case HAL_TCL_STATUS:
306 case HAL_WBM_IDLE_LINK:
307 case HAL_SW2WBM_RELEASE:
308 case HAL_RXDMA_DST:
309 case HAL_RXDMA_MONITOR_DST:
310 case HAL_RXDMA_MONITOR_DESC:
311 params.intr_batch_cntr_thres_entries =
312 HAL_SRNG_INT_BATCH_THRESHOLD_OTHER;
313 params.intr_timer_thres_us = HAL_SRNG_INT_TIMER_THRESHOLD_OTHER;
314 break;
315 case HAL_RXDMA_DIR_BUF:
316 break;
317 default:
318 ath12k_warn(ab, "Not a valid ring type in dp :%d\n", type);
319 return -EINVAL;
320 }
321
322 ret = ath12k_hal_srng_setup(ab, type, ring_num, mac_id, ¶ms);
323 if (ret < 0) {
324 ath12k_warn(ab, "failed to setup srng: %d ring_id %d\n",
325 ret, ring_num);
326 return ret;
327 }
328
329 ring->ring_id = ret;
330
331 return 0;
332 }
333
ath12k_dp_tx_get_bank_profile(struct ath12k_base * ab,struct ath12k_link_vif * arvif,struct ath12k_dp * dp)334 static int ath12k_dp_tx_get_bank_profile(struct ath12k_base *ab,
335 struct ath12k_link_vif *arvif,
336 struct ath12k_dp *dp)
337 {
338 int bank_id = DP_INVALID_BANK_ID;
339 int i;
340 u32 bank_config;
341 bool configure_register = false;
342
343 /* convert vdev params into hal_tx_bank_config */
344 bank_config = ath12k_dp_arch_tx_get_vdev_bank_config(dp, arvif);
345
346 spin_lock_bh(&dp->tx_bank_lock);
347 /* TODO: implement using idr kernel framework*/
348 for (i = 0; i < dp->num_bank_profiles; i++) {
349 if (dp->bank_profiles[i].is_configured &&
350 (dp->bank_profiles[i].bank_config ^ bank_config) == 0) {
351 bank_id = i;
352 goto inc_ref_and_return;
353 }
354 if (!dp->bank_profiles[i].is_configured ||
355 !dp->bank_profiles[i].num_users) {
356 bank_id = i;
357 goto configure_and_return;
358 }
359 }
360
361 if (bank_id == DP_INVALID_BANK_ID) {
362 spin_unlock_bh(&dp->tx_bank_lock);
363 ath12k_err(ab, "unable to find TX bank!");
364 return bank_id;
365 }
366
367 configure_and_return:
368 dp->bank_profiles[bank_id].is_configured = true;
369 dp->bank_profiles[bank_id].bank_config = bank_config;
370 configure_register = true;
371 inc_ref_and_return:
372 dp->bank_profiles[bank_id].num_users++;
373 spin_unlock_bh(&dp->tx_bank_lock);
374
375 if (configure_register)
376 ath12k_hal_tx_configure_bank_register(ab,
377 bank_config, bank_id);
378
379 ath12k_dbg(ab, ATH12K_DBG_DP_HTT, "dp_htt tcl bank_id %d input 0x%x match 0x%x num_users %u",
380 bank_id, bank_config, dp->bank_profiles[bank_id].bank_config,
381 dp->bank_profiles[bank_id].num_users);
382
383 return bank_id;
384 }
385
ath12k_dp_tx_put_bank_profile(struct ath12k_dp * dp,u8 bank_id)386 void ath12k_dp_tx_put_bank_profile(struct ath12k_dp *dp, u8 bank_id)
387 {
388 spin_lock_bh(&dp->tx_bank_lock);
389 dp->bank_profiles[bank_id].num_users--;
390 spin_unlock_bh(&dp->tx_bank_lock);
391 }
392
ath12k_dp_deinit_bank_profiles(struct ath12k_base * ab)393 static void ath12k_dp_deinit_bank_profiles(struct ath12k_base *ab)
394 {
395 struct ath12k_dp *dp = ath12k_ab_to_dp(ab);
396
397 kfree(dp->bank_profiles);
398 dp->bank_profiles = NULL;
399 }
400
ath12k_dp_init_bank_profiles(struct ath12k_base * ab)401 static int ath12k_dp_init_bank_profiles(struct ath12k_base *ab)
402 {
403 struct ath12k_dp *dp = ath12k_ab_to_dp(ab);
404 u32 num_tcl_banks = ab->hw_params->num_tcl_banks;
405 int i;
406
407 dp->num_bank_profiles = num_tcl_banks;
408 dp->bank_profiles = kmalloc_objs(struct ath12k_dp_tx_bank_profile,
409 num_tcl_banks);
410 if (!dp->bank_profiles)
411 return -ENOMEM;
412
413 spin_lock_init(&dp->tx_bank_lock);
414
415 for (i = 0; i < num_tcl_banks; i++) {
416 dp->bank_profiles[i].is_configured = false;
417 dp->bank_profiles[i].num_users = 0;
418 }
419
420 return 0;
421 }
422
ath12k_dp_srng_common_cleanup(struct ath12k_base * ab)423 static void ath12k_dp_srng_common_cleanup(struct ath12k_base *ab)
424 {
425 struct ath12k_dp *dp = ath12k_ab_to_dp(ab);
426 int i;
427
428 ath12k_dp_srng_cleanup(ab, &dp->reo_status_ring);
429 ath12k_dp_srng_cleanup(ab, &dp->reo_cmd_ring);
430 ath12k_dp_srng_cleanup(ab, &dp->reo_except_ring);
431 ath12k_dp_srng_cleanup(ab, &dp->rx_rel_ring);
432 ath12k_dp_srng_cleanup(ab, &dp->reo_reinject_ring);
433 for (i = 0; i < ab->hw_params->max_tx_ring; i++) {
434 ath12k_dp_srng_cleanup(ab, &dp->tx_ring[i].tcl_comp_ring);
435 ath12k_dp_srng_cleanup(ab, &dp->tx_ring[i].tcl_data_ring);
436 }
437 ath12k_dp_srng_cleanup(ab, &dp->wbm_desc_rel_ring);
438 }
439
ath12k_dp_srng_common_setup(struct ath12k_base * ab)440 static int ath12k_dp_srng_common_setup(struct ath12k_base *ab)
441 {
442 struct ath12k_dp *dp = ath12k_ab_to_dp(ab);
443 const struct ath12k_hal_tcl_to_wbm_rbm_map *map;
444 struct hal_srng *srng;
445 int i, ret, tx_comp_ring_num;
446 u32 ring_hash_map;
447
448 ret = ath12k_dp_srng_setup(ab, &dp->wbm_desc_rel_ring,
449 HAL_SW2WBM_RELEASE, 0, 0,
450 DP_WBM_RELEASE_RING_SIZE);
451 if (ret) {
452 ath12k_warn(ab, "failed to set up wbm2sw_release ring :%d\n",
453 ret);
454 goto err;
455 }
456
457 for (i = 0; i < ab->hw_params->max_tx_ring; i++) {
458 map = ab->hal.tcl_to_wbm_rbm_map;
459 tx_comp_ring_num = map[i].wbm_ring_num;
460
461 ret = ath12k_dp_srng_setup(ab, &dp->tx_ring[i].tcl_data_ring,
462 HAL_TCL_DATA, i, 0,
463 DP_TCL_DATA_RING_SIZE);
464 if (ret) {
465 ath12k_warn(ab, "failed to set up tcl_data ring (%d) :%d\n",
466 i, ret);
467 goto err;
468 }
469
470 ret = ath12k_dp_srng_setup(ab, &dp->tx_ring[i].tcl_comp_ring,
471 HAL_WBM2SW_RELEASE, tx_comp_ring_num, 0,
472 DP_TX_COMP_RING_SIZE(ab));
473 if (ret) {
474 ath12k_warn(ab, "failed to set up tcl_comp ring (%d) :%d\n",
475 tx_comp_ring_num, ret);
476 goto err;
477 }
478 }
479
480 ret = ath12k_dp_srng_setup(ab, &dp->reo_reinject_ring, HAL_REO_REINJECT,
481 0, 0, DP_REO_REINJECT_RING_SIZE);
482 if (ret) {
483 ath12k_warn(ab, "failed to set up reo_reinject ring :%d\n",
484 ret);
485 goto err;
486 }
487
488 ret = ath12k_dp_srng_setup(ab, &dp->rx_rel_ring, HAL_WBM2SW_RELEASE,
489 HAL_WBM2SW_REL_ERR_RING_NUM, 0,
490 DP_RX_RELEASE_RING_SIZE(ab));
491 if (ret) {
492 ath12k_warn(ab, "failed to set up rx_rel ring :%d\n", ret);
493 goto err;
494 }
495
496 ret = ath12k_dp_srng_setup(ab, &dp->reo_except_ring, HAL_REO_EXCEPTION,
497 0, 0, DP_REO_EXCEPTION_RING_SIZE);
498 if (ret) {
499 ath12k_warn(ab, "failed to set up reo_exception ring :%d\n",
500 ret);
501 goto err;
502 }
503
504 ret = ath12k_dp_srng_setup(ab, &dp->reo_cmd_ring, HAL_REO_CMD,
505 0, 0, DP_REO_CMD_RING_SIZE);
506 if (ret) {
507 ath12k_warn(ab, "failed to set up reo_cmd ring :%d\n", ret);
508 goto err;
509 }
510
511 srng = &ab->hal.srng_list[dp->reo_cmd_ring.ring_id];
512 ath12k_hal_reo_init_cmd_ring(ab, srng);
513
514 ret = ath12k_dp_srng_setup(ab, &dp->reo_status_ring, HAL_REO_STATUS,
515 0, 0, DP_REO_STATUS_RING_SIZE);
516 if (ret) {
517 ath12k_warn(ab, "failed to set up reo_status ring :%d\n", ret);
518 goto err;
519 }
520
521 /* When hash based routing of rx packet is enabled, 32 entries to map
522 * the hash values to the ring will be configured. Each hash entry uses
523 * four bits to map to a particular ring. The ring mapping will be
524 * 0:TCL, 1:SW1, 2:SW2, 3:SW3, 4:SW4, 5:Release, 6:FW and 7:SW5
525 * 8:SW6, 9:SW7, 10:SW8, 11:Not used.
526 */
527 ring_hash_map = HAL_HASH_ROUTING_RING_SW1 |
528 HAL_HASH_ROUTING_RING_SW2 << 4 |
529 HAL_HASH_ROUTING_RING_SW3 << 8 |
530 HAL_HASH_ROUTING_RING_SW4 << 12 |
531 HAL_HASH_ROUTING_RING_SW1 << 16 |
532 HAL_HASH_ROUTING_RING_SW2 << 20 |
533 HAL_HASH_ROUTING_RING_SW3 << 24 |
534 HAL_HASH_ROUTING_RING_SW4 << 28;
535
536 ath12k_hal_reo_hw_setup(ab, ring_hash_map);
537
538 return 0;
539
540 err:
541 ath12k_dp_srng_common_cleanup(ab);
542
543 return ret;
544 }
545
ath12k_dp_scatter_idle_link_desc_cleanup(struct ath12k_base * ab)546 static void ath12k_dp_scatter_idle_link_desc_cleanup(struct ath12k_base *ab)
547 {
548 struct ath12k_dp *dp = ath12k_ab_to_dp(ab);
549 struct hal_wbm_idle_scatter_list *slist = dp->scatter_list;
550 int i;
551
552 for (i = 0; i < DP_IDLE_SCATTER_BUFS_MAX; i++) {
553 if (!slist[i].vaddr)
554 continue;
555
556 dma_free_coherent(ab->dev, HAL_WBM_IDLE_SCATTER_BUF_SIZE_MAX,
557 slist[i].vaddr, slist[i].paddr);
558 slist[i].vaddr = NULL;
559 }
560 }
561
ath12k_dp_scatter_idle_link_desc_setup(struct ath12k_base * ab,int size,u32 n_link_desc_bank,u32 n_link_desc,u32 last_bank_sz)562 static int ath12k_dp_scatter_idle_link_desc_setup(struct ath12k_base *ab,
563 int size,
564 u32 n_link_desc_bank,
565 u32 n_link_desc,
566 u32 last_bank_sz)
567 {
568 struct ath12k_dp *dp = ath12k_ab_to_dp(ab);
569 struct dp_link_desc_bank *link_desc_banks = dp->link_desc_banks;
570 struct hal_wbm_idle_scatter_list *slist = dp->scatter_list;
571 u32 n_entries_per_buf;
572 int num_scatter_buf, scatter_idx;
573 struct hal_wbm_link_desc *scatter_buf;
574 int align_bytes, n_entries;
575 dma_addr_t paddr;
576 int rem_entries;
577 int i;
578 int ret = 0;
579 u32 end_offset, cookie;
580 enum hal_rx_buf_return_buf_manager rbm = dp->idle_link_rbm;
581
582 n_entries_per_buf = HAL_WBM_IDLE_SCATTER_BUF_SIZE /
583 ath12k_hal_srng_get_entrysize(ab, HAL_WBM_IDLE_LINK);
584 num_scatter_buf = DIV_ROUND_UP(size, HAL_WBM_IDLE_SCATTER_BUF_SIZE);
585
586 if (num_scatter_buf > DP_IDLE_SCATTER_BUFS_MAX)
587 return -EINVAL;
588
589 for (i = 0; i < num_scatter_buf; i++) {
590 slist[i].vaddr = dma_alloc_coherent(ab->dev,
591 HAL_WBM_IDLE_SCATTER_BUF_SIZE_MAX,
592 &slist[i].paddr, GFP_KERNEL);
593 if (!slist[i].vaddr) {
594 ret = -ENOMEM;
595 goto err;
596 }
597 }
598
599 scatter_idx = 0;
600 scatter_buf = slist[scatter_idx].vaddr;
601 rem_entries = n_entries_per_buf;
602
603 for (i = 0; i < n_link_desc_bank; i++) {
604 align_bytes = link_desc_banks[i].vaddr -
605 link_desc_banks[i].vaddr_unaligned;
606 n_entries = (DP_LINK_DESC_ALLOC_SIZE_THRESH - align_bytes) /
607 HAL_LINK_DESC_SIZE;
608 paddr = link_desc_banks[i].paddr;
609 while (n_entries) {
610 cookie = DP_LINK_DESC_COOKIE_SET(n_entries, i);
611 ath12k_hal_set_link_desc_addr(dp->hal, scatter_buf, cookie,
612 paddr, rbm);
613 n_entries--;
614 paddr += HAL_LINK_DESC_SIZE;
615 if (rem_entries) {
616 rem_entries--;
617 scatter_buf++;
618 continue;
619 }
620
621 rem_entries = n_entries_per_buf;
622 scatter_idx++;
623 scatter_buf = slist[scatter_idx].vaddr;
624 }
625 }
626
627 end_offset = (scatter_buf - slist[scatter_idx].vaddr) *
628 sizeof(struct hal_wbm_link_desc);
629 ath12k_hal_setup_link_idle_list(ab, slist, num_scatter_buf,
630 n_link_desc, end_offset);
631
632 return 0;
633
634 err:
635 ath12k_dp_scatter_idle_link_desc_cleanup(ab);
636
637 return ret;
638 }
639
640 static void
ath12k_dp_link_desc_bank_free(struct ath12k_base * ab,struct dp_link_desc_bank * link_desc_banks)641 ath12k_dp_link_desc_bank_free(struct ath12k_base *ab,
642 struct dp_link_desc_bank *link_desc_banks)
643 {
644 int i;
645
646 for (i = 0; i < DP_LINK_DESC_BANKS_MAX; i++) {
647 if (link_desc_banks[i].vaddr_unaligned) {
648 dma_free_coherent(ab->dev,
649 link_desc_banks[i].size,
650 link_desc_banks[i].vaddr_unaligned,
651 link_desc_banks[i].paddr_unaligned);
652 link_desc_banks[i].vaddr_unaligned = NULL;
653 }
654 }
655 }
656
ath12k_dp_link_desc_bank_alloc(struct ath12k_base * ab,struct dp_link_desc_bank * desc_bank,int n_link_desc_bank,int last_bank_sz)657 static int ath12k_dp_link_desc_bank_alloc(struct ath12k_base *ab,
658 struct dp_link_desc_bank *desc_bank,
659 int n_link_desc_bank,
660 int last_bank_sz)
661 {
662 struct ath12k_dp *dp = ath12k_ab_to_dp(ab);
663 int i;
664 int ret = 0;
665 int desc_sz = DP_LINK_DESC_ALLOC_SIZE_THRESH;
666
667 for (i = 0; i < n_link_desc_bank; i++) {
668 if (i == (n_link_desc_bank - 1) && last_bank_sz)
669 desc_sz = last_bank_sz;
670
671 desc_bank[i].vaddr_unaligned =
672 dma_alloc_coherent(ab->dev, desc_sz,
673 &desc_bank[i].paddr_unaligned,
674 GFP_KERNEL);
675 if (!desc_bank[i].vaddr_unaligned) {
676 ret = -ENOMEM;
677 goto err;
678 }
679
680 desc_bank[i].vaddr = PTR_ALIGN(desc_bank[i].vaddr_unaligned,
681 HAL_LINK_DESC_ALIGN);
682 desc_bank[i].paddr = desc_bank[i].paddr_unaligned +
683 ((unsigned long)desc_bank[i].vaddr -
684 (unsigned long)desc_bank[i].vaddr_unaligned);
685 desc_bank[i].size = desc_sz;
686 }
687
688 return 0;
689
690 err:
691 ath12k_dp_link_desc_bank_free(ab, dp->link_desc_banks);
692
693 return ret;
694 }
695
ath12k_dp_link_desc_cleanup(struct ath12k_base * ab,struct dp_link_desc_bank * desc_bank,u32 ring_type,struct dp_srng * ring)696 void ath12k_dp_link_desc_cleanup(struct ath12k_base *ab,
697 struct dp_link_desc_bank *desc_bank,
698 u32 ring_type, struct dp_srng *ring)
699 {
700 ath12k_dp_link_desc_bank_free(ab, desc_bank);
701
702 if (ring_type != HAL_RXDMA_MONITOR_DESC) {
703 ath12k_dp_srng_cleanup(ab, ring);
704 ath12k_dp_scatter_idle_link_desc_cleanup(ab);
705 }
706 }
707
ath12k_wbm_idle_ring_setup(struct ath12k_base * ab,u32 * n_link_desc)708 static int ath12k_wbm_idle_ring_setup(struct ath12k_base *ab, u32 *n_link_desc)
709 {
710 struct ath12k_dp *dp = ath12k_ab_to_dp(ab);
711 u32 n_mpdu_link_desc, n_mpdu_queue_desc;
712 u32 n_tx_msdu_link_desc, n_rx_msdu_link_desc;
713 int ret = 0;
714
715 n_mpdu_link_desc = (DP_NUM_TIDS_MAX * DP_AVG_MPDUS_PER_TID_MAX) /
716 HAL_NUM_MPDUS_PER_LINK_DESC;
717
718 n_mpdu_queue_desc = n_mpdu_link_desc /
719 HAL_NUM_MPDU_LINKS_PER_QUEUE_DESC;
720
721 n_tx_msdu_link_desc = (DP_NUM_TIDS_MAX * DP_AVG_FLOWS_PER_TID *
722 DP_AVG_MSDUS_PER_FLOW) /
723 HAL_NUM_TX_MSDUS_PER_LINK_DESC;
724
725 n_rx_msdu_link_desc = (DP_NUM_TIDS_MAX * DP_AVG_MPDUS_PER_TID_MAX *
726 DP_AVG_MSDUS_PER_MPDU) /
727 HAL_NUM_RX_MSDUS_PER_LINK_DESC;
728
729 *n_link_desc = n_mpdu_link_desc + n_mpdu_queue_desc +
730 n_tx_msdu_link_desc + n_rx_msdu_link_desc;
731
732 if (*n_link_desc & (*n_link_desc - 1))
733 *n_link_desc = 1 << fls(*n_link_desc);
734
735 ret = ath12k_dp_srng_setup(ab, &dp->wbm_idle_ring,
736 HAL_WBM_IDLE_LINK, 0, 0, *n_link_desc);
737 if (ret) {
738 ath12k_warn(ab, "failed to setup wbm_idle_ring: %d\n", ret);
739 return ret;
740 }
741 return ret;
742 }
743
ath12k_dp_link_desc_setup(struct ath12k_base * ab,struct dp_link_desc_bank * link_desc_banks,u32 ring_type,struct hal_srng * srng,u32 n_link_desc)744 int ath12k_dp_link_desc_setup(struct ath12k_base *ab,
745 struct dp_link_desc_bank *link_desc_banks,
746 u32 ring_type, struct hal_srng *srng,
747 u32 n_link_desc)
748 {
749 struct ath12k_dp *dp = ath12k_ab_to_dp(ab);
750 u32 tot_mem_sz;
751 u32 n_link_desc_bank, last_bank_sz;
752 u32 entry_sz, align_bytes, n_entries;
753 struct hal_wbm_link_desc *desc;
754 u32 paddr;
755 int i, ret;
756 u32 cookie;
757 enum hal_rx_buf_return_buf_manager rbm = dp->idle_link_rbm;
758
759 tot_mem_sz = n_link_desc * HAL_LINK_DESC_SIZE;
760 tot_mem_sz += HAL_LINK_DESC_ALIGN;
761
762 if (tot_mem_sz <= DP_LINK_DESC_ALLOC_SIZE_THRESH) {
763 n_link_desc_bank = 1;
764 last_bank_sz = tot_mem_sz;
765 } else {
766 n_link_desc_bank = tot_mem_sz /
767 (DP_LINK_DESC_ALLOC_SIZE_THRESH -
768 HAL_LINK_DESC_ALIGN);
769 last_bank_sz = tot_mem_sz %
770 (DP_LINK_DESC_ALLOC_SIZE_THRESH -
771 HAL_LINK_DESC_ALIGN);
772
773 if (last_bank_sz)
774 n_link_desc_bank += 1;
775 }
776
777 if (n_link_desc_bank > DP_LINK_DESC_BANKS_MAX)
778 return -EINVAL;
779
780 ret = ath12k_dp_link_desc_bank_alloc(ab, link_desc_banks,
781 n_link_desc_bank, last_bank_sz);
782 if (ret)
783 return ret;
784
785 /* Setup link desc idle list for HW internal usage */
786 entry_sz = ath12k_hal_srng_get_entrysize(ab, ring_type);
787 tot_mem_sz = entry_sz * n_link_desc;
788
789 /* Setup scatter desc list when the total memory requirement is more */
790 if (tot_mem_sz > DP_LINK_DESC_ALLOC_SIZE_THRESH &&
791 ring_type != HAL_RXDMA_MONITOR_DESC) {
792 ret = ath12k_dp_scatter_idle_link_desc_setup(ab, tot_mem_sz,
793 n_link_desc_bank,
794 n_link_desc,
795 last_bank_sz);
796 if (ret) {
797 ath12k_warn(ab, "failed to setup scatting idle list descriptor :%d\n",
798 ret);
799 goto fail_desc_bank_free;
800 }
801
802 return 0;
803 }
804
805 spin_lock_bh(&srng->lock);
806
807 ath12k_hal_srng_access_begin(ab, srng);
808
809 for (i = 0; i < n_link_desc_bank; i++) {
810 align_bytes = link_desc_banks[i].vaddr -
811 link_desc_banks[i].vaddr_unaligned;
812 n_entries = (link_desc_banks[i].size - align_bytes) /
813 HAL_LINK_DESC_SIZE;
814 paddr = link_desc_banks[i].paddr;
815 while (n_entries &&
816 (desc = ath12k_hal_srng_src_get_next_entry(ab, srng))) {
817 cookie = DP_LINK_DESC_COOKIE_SET(n_entries, i);
818 ath12k_hal_set_link_desc_addr(dp->hal, desc, cookie, paddr,
819 rbm);
820 n_entries--;
821 paddr += HAL_LINK_DESC_SIZE;
822 }
823 }
824
825 ath12k_hal_srng_access_end(ab, srng);
826
827 spin_unlock_bh(&srng->lock);
828
829 return 0;
830
831 fail_desc_bank_free:
832 ath12k_dp_link_desc_bank_free(ab, link_desc_banks);
833
834 return ret;
835 }
836
ath12k_dp_pdev_free(struct ath12k_base * ab)837 void ath12k_dp_pdev_free(struct ath12k_base *ab)
838 {
839 struct ath12k_dp *dp = ath12k_ab_to_dp(ab);
840 struct ath12k *ar;
841 int i;
842
843 for (i = 0; i < ab->num_radios; i++) {
844 ar = ab->pdevs[i].ar;
845 rcu_assign_pointer(dp->dp_pdevs[ar->pdev_idx], NULL);
846 }
847
848 synchronize_rcu();
849
850 for (i = 0; i < ab->num_radios; i++)
851 ath12k_dp_rx_pdev_free(ab, i);
852 }
853
ath12k_dp_pdev_pre_alloc(struct ath12k * ar)854 void ath12k_dp_pdev_pre_alloc(struct ath12k *ar)
855 {
856 struct ath12k_pdev_dp *dp = &ar->dp;
857
858 dp->mac_id = ar->pdev_idx;
859 atomic_set(&dp->num_tx_pending, 0);
860 init_waitqueue_head(&dp->tx_empty_waitq);
861 /* TODO: Add any RXDMA setup required per pdev */
862 }
863
ath12k_dp_pdev_alloc(struct ath12k_base * ab)864 int ath12k_dp_pdev_alloc(struct ath12k_base *ab)
865 {
866 struct ath12k_dp *dp = ath12k_ab_to_dp(ab);
867 struct ath12k_pdev_dp *dp_pdev;
868 struct ath12k *ar;
869 int ret;
870 int i;
871
872 ret = ath12k_dp_rx_htt_setup(ab);
873 if (ret)
874 goto out;
875
876 /* TODO: Per-pdev rx ring unlike tx ring which is mapped to different AC's */
877 for (i = 0; i < ab->num_radios; i++) {
878 ar = ab->pdevs[i].ar;
879
880 dp_pdev = &ar->dp;
881
882 dp_pdev->hw = ar->ah->hw;
883 dp_pdev->dp = dp;
884 dp_pdev->hw_link_id = ar->hw_link_id;
885 dp_pdev->dp_hw = &ar->ah->dp_hw;
886
887 ret = ath12k_dp_rx_pdev_alloc(ab, i);
888 if (ret) {
889 ath12k_warn(ab, "failed to allocate pdev rx for pdev_id :%d\n",
890 i);
891 goto err;
892 }
893 ret = ath12k_dp_rx_pdev_mon_attach(ar);
894 if (ret) {
895 ath12k_warn(ab, "failed to initialize mon pdev %d\n", i);
896 goto err;
897 }
898 }
899
900 for (i = 0; i < ab->num_radios; i++) {
901 ar = ab->pdevs[i].ar;
902 rcu_assign_pointer(dp->dp_pdevs[ar->pdev_idx], &ar->dp);
903 }
904
905 return 0;
906 err:
907 ath12k_dp_pdev_free(ab);
908 out:
909 return ret;
910 }
911
ath12k_dp_update_vdev_search(struct ath12k_link_vif * arvif)912 static void ath12k_dp_update_vdev_search(struct ath12k_link_vif *arvif)
913 {
914 u8 link_id = arvif->link_id;
915 struct ath12k_vif *ahvif = arvif->ahvif;
916 struct ath12k_dp_link_vif *dp_link_vif;
917
918 dp_link_vif = ath12k_dp_vif_to_dp_link_vif(&ahvif->dp_vif, link_id);
919
920 switch (arvif->ahvif->vdev_type) {
921 case WMI_VDEV_TYPE_STA:
922 dp_link_vif->hal_addr_search_flags = HAL_TX_ADDRY_EN;
923 dp_link_vif->search_type = HAL_TX_ADDR_SEARCH_DEFAULT;
924 break;
925 case WMI_VDEV_TYPE_AP:
926 case WMI_VDEV_TYPE_IBSS:
927 dp_link_vif->hal_addr_search_flags = HAL_TX_ADDRX_EN;
928 dp_link_vif->search_type = HAL_TX_ADDR_SEARCH_DEFAULT;
929 break;
930 case WMI_VDEV_TYPE_MONITOR:
931 default:
932 return;
933 }
934 }
935
ath12k_dp_vdev_tx_attach(struct ath12k * ar,struct ath12k_link_vif * arvif)936 void ath12k_dp_vdev_tx_attach(struct ath12k *ar, struct ath12k_link_vif *arvif)
937 {
938 struct ath12k_base *ab = ar->ab;
939 struct ath12k_vif *ahvif = arvif->ahvif;
940 u8 link_id = arvif->link_id;
941 int bank_id;
942 struct ath12k_dp_link_vif *dp_link_vif;
943
944 dp_link_vif = ath12k_dp_vif_to_dp_link_vif(&ahvif->dp_vif, link_id);
945
946 dp_link_vif->tcl_metadata = u32_encode_bits(1, HTT_TCL_META_DATA_TYPE) |
947 u32_encode_bits(arvif->vdev_id,
948 HTT_TCL_META_DATA_VDEV_ID) |
949 u32_encode_bits(ar->pdev->pdev_id,
950 HTT_TCL_META_DATA_PDEV_ID);
951
952 /* set HTT extension valid bit to 0 by default */
953 dp_link_vif->tcl_metadata &= ~HTT_TCL_META_DATA_VALID_HTT;
954
955 ath12k_dp_update_vdev_search(arvif);
956 dp_link_vif->vdev_id_check_en = true;
957 bank_id = ath12k_dp_tx_get_bank_profile(ab, arvif, ath12k_ab_to_dp(ab));
958 dp_link_vif->bank_id = bank_id;
959
960 /* TODO: error path for bank id failure */
961 if (bank_id == DP_INVALID_BANK_ID) {
962 ath12k_err(ar->ab, "Failed to initialize DP TX Banks");
963 return;
964 }
965 }
966
ath12k_dp_cc_cleanup(struct ath12k_base * ab)967 static void ath12k_dp_cc_cleanup(struct ath12k_base *ab)
968 {
969 struct ath12k_rx_desc_info *desc_info;
970 struct ath12k_tx_desc_info *tx_desc_info, *tmp1;
971 struct ath12k_dp *dp = ath12k_ab_to_dp(ab);
972 struct ath12k_skb_cb *skb_cb;
973 struct sk_buff *skb;
974 struct ath12k *ar;
975 int i, j;
976 u32 pool_id, tx_spt_page;
977
978 if (!dp->spt_info)
979 return;
980
981 /* RX Descriptor cleanup */
982 spin_lock_bh(&dp->rx_desc_lock);
983
984 if (dp->rxbaddr) {
985 for (i = 0; i < ATH12K_NUM_RX_SPT_PAGES(ab); i++) {
986 if (!dp->rxbaddr[i])
987 continue;
988
989 desc_info = dp->rxbaddr[i];
990
991 for (j = 0; j < ATH12K_MAX_SPT_ENTRIES; j++) {
992 if (!desc_info[j].in_use) {
993 list_del(&desc_info[j].list);
994 continue;
995 }
996
997 skb = desc_info[j].skb;
998 if (!skb)
999 continue;
1000
1001 dma_unmap_single(ab->dev,
1002 ATH12K_SKB_RXCB(skb)->paddr,
1003 skb->len + skb_tailroom(skb),
1004 DMA_FROM_DEVICE);
1005 dev_kfree_skb_any(skb);
1006 }
1007
1008 kfree(dp->rxbaddr[i]);
1009 dp->rxbaddr[i] = NULL;
1010 }
1011
1012 kfree(dp->rxbaddr);
1013 dp->rxbaddr = NULL;
1014 }
1015
1016 spin_unlock_bh(&dp->rx_desc_lock);
1017
1018 /* TX Descriptor cleanup */
1019 for (i = 0; i < ATH12K_HW_MAX_QUEUES; i++) {
1020 spin_lock_bh(&dp->tx_desc_lock[i]);
1021
1022 list_for_each_entry_safe(tx_desc_info, tmp1,
1023 &dp->tx_desc_used_list[i], list) {
1024 list_del(&tx_desc_info->list);
1025 skb = tx_desc_info->skb;
1026
1027 if (!skb)
1028 continue;
1029
1030 skb_cb = ATH12K_SKB_CB(skb);
1031 if (skb_cb->paddr_ext_desc) {
1032 dma_unmap_single(ab->dev,
1033 skb_cb->paddr_ext_desc,
1034 tx_desc_info->skb_ext_desc->len,
1035 DMA_TO_DEVICE);
1036 dev_kfree_skb_any(tx_desc_info->skb_ext_desc);
1037 }
1038
1039 /* if we are unregistering, hw would've been destroyed and
1040 * ar is no longer valid.
1041 */
1042 if (!(test_bit(ATH12K_FLAG_UNREGISTERING, &ab->dev_flags))) {
1043 ar = skb_cb->ar;
1044
1045 if (atomic_dec_and_test(&ar->dp.num_tx_pending))
1046 wake_up(&ar->dp.tx_empty_waitq);
1047 }
1048
1049 dma_unmap_single(ab->dev, ATH12K_SKB_CB(skb)->paddr,
1050 skb->len, DMA_TO_DEVICE);
1051 dev_kfree_skb_any(skb);
1052 }
1053
1054 spin_unlock_bh(&dp->tx_desc_lock[i]);
1055 }
1056
1057 if (dp->txbaddr) {
1058 for (pool_id = 0; pool_id < ATH12K_HW_MAX_QUEUES; pool_id++) {
1059 spin_lock_bh(&dp->tx_desc_lock[pool_id]);
1060
1061 for (i = 0; i < ATH12K_TX_SPT_PAGES_PER_POOL(ab); i++) {
1062 tx_spt_page = i + pool_id *
1063 ATH12K_TX_SPT_PAGES_PER_POOL(ab);
1064 if (!dp->txbaddr[tx_spt_page])
1065 continue;
1066
1067 kfree(dp->txbaddr[tx_spt_page]);
1068 dp->txbaddr[tx_spt_page] = NULL;
1069 }
1070
1071 spin_unlock_bh(&dp->tx_desc_lock[pool_id]);
1072 }
1073
1074 kfree(dp->txbaddr);
1075 dp->txbaddr = NULL;
1076 }
1077
1078 /* unmap SPT pages */
1079 for (i = 0; i < dp->num_spt_pages; i++) {
1080 if (!dp->spt_info[i].vaddr)
1081 continue;
1082
1083 dma_free_coherent(ab->dev, ATH12K_PAGE_SIZE,
1084 dp->spt_info[i].vaddr, dp->spt_info[i].paddr);
1085 dp->spt_info[i].vaddr = NULL;
1086 }
1087
1088 kfree(dp->spt_info);
1089 dp->spt_info = NULL;
1090 }
1091
ath12k_dp_reoq_lut_cleanup(struct ath12k_base * ab)1092 static void ath12k_dp_reoq_lut_cleanup(struct ath12k_base *ab)
1093 {
1094 struct ath12k_dp *dp = ath12k_ab_to_dp(ab);
1095
1096 if (!ab->hw_params->reoq_lut_support)
1097 return;
1098
1099 if (dp->reoq_lut.vaddr_unaligned) {
1100 dma_free_coherent(ab->dev, dp->reoq_lut.size,
1101 dp->reoq_lut.vaddr_unaligned,
1102 dp->reoq_lut.paddr_unaligned);
1103 dp->reoq_lut.vaddr_unaligned = NULL;
1104 }
1105
1106 if (dp->ml_reoq_lut.vaddr_unaligned) {
1107 dma_free_coherent(ab->dev, dp->ml_reoq_lut.size,
1108 dp->ml_reoq_lut.vaddr_unaligned,
1109 dp->ml_reoq_lut.paddr_unaligned);
1110 dp->ml_reoq_lut.vaddr_unaligned = NULL;
1111 }
1112 }
1113
ath12k_dp_cleanup(struct ath12k_base * ab)1114 static void ath12k_dp_cleanup(struct ath12k_base *ab)
1115 {
1116 struct ath12k_dp *dp = ath12k_ab_to_dp(ab);
1117 int i;
1118
1119 ath12k_dp_link_peer_rhash_tbl_destroy(dp);
1120
1121 if (!dp->ab)
1122 return;
1123
1124 ath12k_dp_link_desc_cleanup(ab, dp->link_desc_banks,
1125 HAL_WBM_IDLE_LINK, &dp->wbm_idle_ring);
1126
1127 ath12k_dp_cc_cleanup(ab);
1128 ath12k_dp_reoq_lut_cleanup(ab);
1129 ath12k_dp_deinit_bank_profiles(ab);
1130 ath12k_dp_srng_common_cleanup(ab);
1131
1132 ath12k_dp_rx_reo_cmd_list_cleanup(ab);
1133
1134 for (i = 0; i < ab->hw_params->max_tx_ring; i++) {
1135 kfree(dp->tx_ring[i].tx_status);
1136 dp->tx_ring[i].tx_status = NULL;
1137 }
1138
1139 ath12k_dp_rx_free(ab);
1140 /* Deinit any SOC level resource */
1141 }
1142
ath12k_dp_cc_cookie_gen(u16 ppt_idx,u16 spt_idx)1143 static u32 ath12k_dp_cc_cookie_gen(u16 ppt_idx, u16 spt_idx)
1144 {
1145 return (u32)ppt_idx << ATH12K_CC_PPT_SHIFT | spt_idx;
1146 }
1147
ath12k_dp_cc_get_desc_addr_ptr(struct ath12k_dp * dp,u16 ppt_idx,u16 spt_idx)1148 static void *ath12k_dp_cc_get_desc_addr_ptr(struct ath12k_dp *dp,
1149 u16 ppt_idx, u16 spt_idx)
1150 {
1151 return dp->spt_info[ppt_idx].vaddr + spt_idx;
1152 }
1153
ath12k_dp_get_rx_desc(struct ath12k_dp * dp,u32 cookie)1154 struct ath12k_rx_desc_info *ath12k_dp_get_rx_desc(struct ath12k_dp *dp,
1155 u32 cookie)
1156 {
1157 struct ath12k_rx_desc_info **desc_addr_ptr;
1158 u16 start_ppt_idx, end_ppt_idx, ppt_idx, spt_idx;
1159
1160 ppt_idx = u32_get_bits(cookie, ATH12K_DP_CC_COOKIE_PPT);
1161 spt_idx = u32_get_bits(cookie, ATH12K_DP_CC_COOKIE_SPT);
1162
1163 start_ppt_idx = dp->rx_ppt_base + ATH12K_RX_SPT_PAGE_OFFSET(dp->ab);
1164 end_ppt_idx = start_ppt_idx + ATH12K_NUM_RX_SPT_PAGES(dp->ab);
1165
1166 if (ppt_idx < start_ppt_idx ||
1167 ppt_idx >= end_ppt_idx ||
1168 spt_idx > ATH12K_MAX_SPT_ENTRIES)
1169 return NULL;
1170
1171 ppt_idx = ppt_idx - dp->rx_ppt_base;
1172 desc_addr_ptr = ath12k_dp_cc_get_desc_addr_ptr(dp, ppt_idx, spt_idx);
1173
1174 return *desc_addr_ptr;
1175 }
1176 EXPORT_SYMBOL(ath12k_dp_get_rx_desc);
1177
ath12k_dp_get_tx_desc(struct ath12k_dp * dp,u32 cookie)1178 struct ath12k_tx_desc_info *ath12k_dp_get_tx_desc(struct ath12k_dp *dp,
1179 u32 cookie)
1180 {
1181 struct ath12k_tx_desc_info **desc_addr_ptr;
1182 u16 start_ppt_idx, end_ppt_idx, ppt_idx, spt_idx;
1183
1184 ppt_idx = u32_get_bits(cookie, ATH12K_DP_CC_COOKIE_PPT);
1185 spt_idx = u32_get_bits(cookie, ATH12K_DP_CC_COOKIE_SPT);
1186
1187 start_ppt_idx = ATH12K_TX_SPT_PAGE_OFFSET;
1188 end_ppt_idx = start_ppt_idx +
1189 (ATH12K_TX_SPT_PAGES_PER_POOL(dp->ab) * ATH12K_HW_MAX_QUEUES);
1190
1191 if (ppt_idx < start_ppt_idx ||
1192 ppt_idx >= end_ppt_idx ||
1193 spt_idx > ATH12K_MAX_SPT_ENTRIES)
1194 return NULL;
1195
1196 desc_addr_ptr = ath12k_dp_cc_get_desc_addr_ptr(dp, ppt_idx, spt_idx);
1197
1198 return *desc_addr_ptr;
1199 }
1200 EXPORT_SYMBOL(ath12k_dp_get_tx_desc);
1201
ath12k_dp_cc_desc_init(struct ath12k_base * ab)1202 static int ath12k_dp_cc_desc_init(struct ath12k_base *ab)
1203 {
1204 struct ath12k_dp *dp = ath12k_ab_to_dp(ab);
1205 struct ath12k_rx_desc_info *rx_descs, **rx_desc_addr;
1206 struct ath12k_tx_desc_info *tx_descs, **tx_desc_addr;
1207 u32 num_rx_spt_pages = ATH12K_NUM_RX_SPT_PAGES(ab);
1208 u32 i, j, pool_id, tx_spt_page;
1209 u32 ppt_idx, cookie_ppt_idx;
1210
1211 spin_lock_bh(&dp->rx_desc_lock);
1212
1213 dp->rxbaddr = kzalloc_objs(struct ath12k_rx_desc_info *,
1214 num_rx_spt_pages, GFP_ATOMIC);
1215
1216 if (!dp->rxbaddr) {
1217 spin_unlock_bh(&dp->rx_desc_lock);
1218 return -ENOMEM;
1219 }
1220
1221 /* First ATH12K_NUM_RX_SPT_PAGES(ab) of allocated SPT pages are used for
1222 * RX
1223 */
1224 for (i = 0; i < num_rx_spt_pages; i++) {
1225 rx_descs = kzalloc_objs(*rx_descs, ATH12K_MAX_SPT_ENTRIES,
1226 GFP_ATOMIC);
1227
1228 if (!rx_descs) {
1229 spin_unlock_bh(&dp->rx_desc_lock);
1230 return -ENOMEM;
1231 }
1232
1233 ppt_idx = ATH12K_RX_SPT_PAGE_OFFSET(ab) + i;
1234 cookie_ppt_idx = dp->rx_ppt_base + ppt_idx;
1235 dp->rxbaddr[i] = &rx_descs[0];
1236
1237 for (j = 0; j < ATH12K_MAX_SPT_ENTRIES; j++) {
1238 rx_descs[j].cookie = ath12k_dp_cc_cookie_gen(cookie_ppt_idx, j);
1239 rx_descs[j].magic = ATH12K_DP_RX_DESC_MAGIC;
1240 rx_descs[j].device_id = ab->device_id;
1241 list_add_tail(&rx_descs[j].list, &dp->rx_desc_free_list);
1242
1243 /* Update descriptor VA in SPT */
1244 rx_desc_addr = ath12k_dp_cc_get_desc_addr_ptr(dp, ppt_idx, j);
1245 *rx_desc_addr = &rx_descs[j];
1246 }
1247 }
1248
1249 spin_unlock_bh(&dp->rx_desc_lock);
1250
1251 dp->txbaddr = kzalloc_objs(struct ath12k_tx_desc_info *,
1252 ATH12K_NUM_TX_SPT_PAGES(ab), GFP_ATOMIC);
1253
1254 if (!dp->txbaddr)
1255 return -ENOMEM;
1256
1257 for (pool_id = 0; pool_id < ATH12K_HW_MAX_QUEUES; pool_id++) {
1258 spin_lock_bh(&dp->tx_desc_lock[pool_id]);
1259 for (i = 0; i < ATH12K_TX_SPT_PAGES_PER_POOL(ab); i++) {
1260 tx_descs = kzalloc_objs(*tx_descs,
1261 ATH12K_MAX_SPT_ENTRIES,
1262 GFP_ATOMIC);
1263
1264 if (!tx_descs) {
1265 spin_unlock_bh(&dp->tx_desc_lock[pool_id]);
1266 /* Caller takes care of TX pending and RX desc cleanup */
1267 return -ENOMEM;
1268 }
1269
1270 tx_spt_page = i + pool_id *
1271 ATH12K_TX_SPT_PAGES_PER_POOL(ab);
1272 ppt_idx = ATH12K_TX_SPT_PAGE_OFFSET + tx_spt_page;
1273
1274 dp->txbaddr[tx_spt_page] = &tx_descs[0];
1275
1276 for (j = 0; j < ATH12K_MAX_SPT_ENTRIES; j++) {
1277 tx_descs[j].desc_id = ath12k_dp_cc_cookie_gen(ppt_idx, j);
1278 tx_descs[j].pool_id = pool_id;
1279 list_add_tail(&tx_descs[j].list,
1280 &dp->tx_desc_free_list[pool_id]);
1281
1282 /* Update descriptor VA in SPT */
1283 tx_desc_addr =
1284 ath12k_dp_cc_get_desc_addr_ptr(dp, ppt_idx, j);
1285 *tx_desc_addr = &tx_descs[j];
1286 }
1287 }
1288 spin_unlock_bh(&dp->tx_desc_lock[pool_id]);
1289 }
1290 return 0;
1291 }
1292
ath12k_dp_cmem_init(struct ath12k_base * ab,struct ath12k_dp * dp,enum ath12k_dp_desc_type type)1293 static int ath12k_dp_cmem_init(struct ath12k_base *ab,
1294 struct ath12k_dp *dp,
1295 enum ath12k_dp_desc_type type)
1296 {
1297 u32 cmem_base;
1298 int i, start, end;
1299
1300 cmem_base = ab->qmi.dev_mem[ATH12K_QMI_DEVMEM_CMEM_INDEX].start;
1301
1302 switch (type) {
1303 case ATH12K_DP_TX_DESC:
1304 start = ATH12K_TX_SPT_PAGE_OFFSET;
1305 end = start + ATH12K_NUM_TX_SPT_PAGES(ab);
1306 break;
1307 case ATH12K_DP_RX_DESC:
1308 cmem_base += ATH12K_PPT_ADDR_OFFSET(dp->rx_ppt_base);
1309 start = ATH12K_RX_SPT_PAGE_OFFSET(ab);
1310 end = start + ATH12K_NUM_RX_SPT_PAGES(ab);
1311 break;
1312 default:
1313 ath12k_err(ab, "invalid descriptor type %d in cmem init\n", type);
1314 return -EINVAL;
1315 }
1316
1317 /* Write to PPT in CMEM */
1318 for (i = start; i < end; i++)
1319 ath12k_hif_write32(ab, cmem_base + ATH12K_PPT_ADDR_OFFSET(i),
1320 dp->spt_info[i].paddr >> ATH12K_SPT_4K_ALIGN_OFFSET);
1321
1322 return 0;
1323 }
1324
ath12k_dp_partner_cc_init(struct ath12k_base * ab)1325 void ath12k_dp_partner_cc_init(struct ath12k_base *ab)
1326 {
1327 struct ath12k_hw_group *ag = ab->ag;
1328 int i;
1329
1330 for (i = 0; i < ag->num_devices; i++) {
1331 if (ag->ab[i] == ab)
1332 continue;
1333
1334 ath12k_dp_cmem_init(ab, ath12k_ab_to_dp(ag->ab[i]), ATH12K_DP_RX_DESC);
1335 }
1336 }
1337
ath12k_dp_get_num_spt_pages(struct ath12k_base * ab)1338 static u32 ath12k_dp_get_num_spt_pages(struct ath12k_base *ab)
1339 {
1340 return ATH12K_NUM_RX_SPT_PAGES(ab) + ATH12K_NUM_TX_SPT_PAGES(ab);
1341 }
1342
ath12k_dp_cc_init(struct ath12k_base * ab)1343 static int ath12k_dp_cc_init(struct ath12k_base *ab)
1344 {
1345 struct ath12k_dp *dp = ath12k_ab_to_dp(ab);
1346 int i, ret = 0;
1347
1348 INIT_LIST_HEAD(&dp->rx_desc_free_list);
1349 spin_lock_init(&dp->rx_desc_lock);
1350
1351 for (i = 0; i < ATH12K_HW_MAX_QUEUES; i++) {
1352 INIT_LIST_HEAD(&dp->tx_desc_free_list[i]);
1353 INIT_LIST_HEAD(&dp->tx_desc_used_list[i]);
1354 spin_lock_init(&dp->tx_desc_lock[i]);
1355 }
1356
1357 dp->num_spt_pages = ath12k_dp_get_num_spt_pages(ab);
1358 if (dp->num_spt_pages > ATH12K_MAX_PPT_ENTRIES)
1359 dp->num_spt_pages = ATH12K_MAX_PPT_ENTRIES;
1360
1361 dp->spt_info = kzalloc_objs(struct ath12k_spt_info, dp->num_spt_pages);
1362
1363 if (!dp->spt_info) {
1364 ath12k_warn(ab, "SPT page allocation failure");
1365 return -ENOMEM;
1366 }
1367
1368 dp->rx_ppt_base = ab->device_id * ATH12K_NUM_RX_SPT_PAGES(ab);
1369
1370 for (i = 0; i < dp->num_spt_pages; i++) {
1371 dp->spt_info[i].vaddr = dma_alloc_coherent(ab->dev,
1372 ATH12K_PAGE_SIZE,
1373 &dp->spt_info[i].paddr,
1374 GFP_KERNEL);
1375
1376 if (!dp->spt_info[i].vaddr) {
1377 ret = -ENOMEM;
1378 goto free;
1379 }
1380
1381 if (dp->spt_info[i].paddr & ATH12K_SPT_4K_ALIGN_CHECK) {
1382 ath12k_warn(ab, "SPT allocated memory is not 4K aligned");
1383 ret = -EINVAL;
1384 goto free;
1385 }
1386 }
1387
1388 ret = ath12k_dp_cmem_init(ab, dp, ATH12K_DP_TX_DESC);
1389 if (ret) {
1390 ath12k_warn(ab, "HW CC Tx cmem init failed %d", ret);
1391 goto free;
1392 }
1393
1394 ret = ath12k_dp_cmem_init(ab, dp, ATH12K_DP_RX_DESC);
1395 if (ret) {
1396 ath12k_warn(ab, "HW CC Rx cmem init failed %d", ret);
1397 goto free;
1398 }
1399
1400 ret = ath12k_dp_cc_desc_init(ab);
1401 if (ret) {
1402 ath12k_warn(ab, "HW CC desc init failed %d", ret);
1403 goto free;
1404 }
1405
1406 return 0;
1407 free:
1408 ath12k_dp_cc_cleanup(ab);
1409 return ret;
1410 }
1411
ath12k_dp_alloc_reoq_lut(struct ath12k_base * ab,struct ath12k_reo_q_addr_lut * lut)1412 static int ath12k_dp_alloc_reoq_lut(struct ath12k_base *ab,
1413 struct ath12k_reo_q_addr_lut *lut)
1414 {
1415 lut->size = DP_REOQ_LUT_SIZE + HAL_REO_QLUT_ADDR_ALIGN - 1;
1416 lut->vaddr_unaligned = dma_alloc_coherent(ab->dev, lut->size,
1417 &lut->paddr_unaligned,
1418 GFP_KERNEL | __GFP_ZERO);
1419 if (!lut->vaddr_unaligned)
1420 return -ENOMEM;
1421
1422 lut->vaddr = PTR_ALIGN(lut->vaddr_unaligned, HAL_REO_QLUT_ADDR_ALIGN);
1423 lut->paddr = lut->paddr_unaligned +
1424 ((unsigned long)lut->vaddr - (unsigned long)lut->vaddr_unaligned);
1425 return 0;
1426 }
1427
ath12k_dp_reoq_lut_setup(struct ath12k_base * ab)1428 static int ath12k_dp_reoq_lut_setup(struct ath12k_base *ab)
1429 {
1430 struct ath12k_dp *dp = ath12k_ab_to_dp(ab);
1431 int ret;
1432
1433 if (!ab->hw_params->reoq_lut_support)
1434 return 0;
1435
1436 ret = ath12k_dp_alloc_reoq_lut(ab, &dp->reoq_lut);
1437 if (ret) {
1438 ath12k_warn(ab, "failed to allocate memory for reoq table");
1439 return ret;
1440 }
1441
1442 ret = ath12k_dp_alloc_reoq_lut(ab, &dp->ml_reoq_lut);
1443 if (ret) {
1444 ath12k_warn(ab, "failed to allocate memory for ML reoq table");
1445 dma_free_coherent(ab->dev, dp->reoq_lut.size,
1446 dp->reoq_lut.vaddr_unaligned,
1447 dp->reoq_lut.paddr_unaligned);
1448 dp->reoq_lut.vaddr_unaligned = NULL;
1449 return ret;
1450 }
1451
1452 /* Bits in the register have address [39:8] LUT base address to be
1453 * allocated such that LSBs are assumed to be zero. Also, current
1454 * design supports paddr up to 4 GB max hence it fits in 32 bit
1455 * register only
1456 */
1457
1458 ath12k_hal_write_reoq_lut_addr(ab, dp->reoq_lut.paddr >> 8);
1459 ath12k_hal_write_ml_reoq_lut_addr(ab, dp->ml_reoq_lut.paddr >> 8);
1460 ath12k_hal_reoq_lut_addr_read_enable(ab);
1461 ath12k_hal_reoq_lut_set_max_peerid(ab);
1462
1463 return 0;
1464 }
1465
ath12k_dp_setup(struct ath12k_base * ab)1466 static int ath12k_dp_setup(struct ath12k_base *ab)
1467 {
1468 struct ath12k_dp *dp;
1469 struct hal_srng *srng = NULL;
1470 size_t size = 0;
1471 u32 n_link_desc = 0;
1472 int ret;
1473 int i;
1474
1475 dp = ath12k_ab_to_dp(ab);
1476 dp->ab = ab;
1477
1478 INIT_LIST_HEAD(&dp->reo_cmd_list);
1479 INIT_LIST_HEAD(&dp->reo_cmd_cache_flush_list);
1480 INIT_LIST_HEAD(&dp->reo_cmd_update_rx_queue_list);
1481 spin_lock_init(&dp->reo_cmd_lock);
1482 spin_lock_init(&dp->reo_rxq_flush_lock);
1483
1484 spin_lock_init(&dp->dp_lock);
1485 INIT_LIST_HEAD(&dp->peers);
1486
1487 mutex_init(&dp->link_peer_rhash_tbl_lock);
1488
1489 dp->reo_cmd_cache_flush_count = 0;
1490 dp->idle_link_rbm =
1491 ath12k_hal_get_idle_link_rbm(&ab->hal, ab->device_id);
1492
1493 ret = ath12k_dp_link_peer_rhash_tbl_init(dp);
1494 if (ret) {
1495 ath12k_warn(ab, "failed to init link_peer rhash table: %d\n", ret);
1496 return ret;
1497 }
1498
1499 ret = ath12k_wbm_idle_ring_setup(ab, &n_link_desc);
1500 if (ret) {
1501 ath12k_warn(ab, "failed to setup wbm_idle_ring: %d\n", ret);
1502 goto rhash_destroy;
1503 }
1504
1505 srng = &ab->hal.srng_list[dp->wbm_idle_ring.ring_id];
1506
1507 ret = ath12k_dp_link_desc_setup(ab, dp->link_desc_banks,
1508 HAL_WBM_IDLE_LINK, srng, n_link_desc);
1509 if (ret) {
1510 ath12k_warn(ab, "failed to setup link desc: %d\n", ret);
1511 goto rhash_destroy;
1512 }
1513
1514 ret = ath12k_dp_cc_init(ab);
1515
1516 if (ret) {
1517 ath12k_warn(ab, "failed to setup cookie converter %d\n", ret);
1518 goto fail_link_desc_cleanup;
1519 }
1520 ret = ath12k_dp_init_bank_profiles(ab);
1521 if (ret) {
1522 ath12k_warn(ab, "failed to setup bank profiles %d\n", ret);
1523 goto fail_hw_cc_cleanup;
1524 }
1525
1526 ret = ath12k_dp_srng_common_setup(ab);
1527 if (ret)
1528 goto fail_dp_bank_profiles_cleanup;
1529
1530 size = ab->hal.hal_wbm_release_ring_tx_size *
1531 DP_TX_COMP_RING_SIZE(ab);
1532
1533 ret = ath12k_dp_reoq_lut_setup(ab);
1534 if (ret) {
1535 ath12k_warn(ab, "failed to setup reoq table %d\n", ret);
1536 goto fail_cmn_srng_cleanup;
1537 }
1538
1539 for (i = 0; i < ab->hw_params->max_tx_ring; i++) {
1540 dp->tx_ring[i].tcl_data_ring_id = i;
1541
1542 dp->tx_ring[i].tx_status_head = 0;
1543 dp->tx_ring[i].tx_status_tail = DP_TX_COMP_RING_SIZE(ab) - 1;
1544 dp->tx_ring[i].tx_status = kmalloc(size, GFP_KERNEL);
1545 if (!dp->tx_ring[i].tx_status) {
1546 ret = -ENOMEM;
1547 /* FIXME: The allocated tx status is not freed
1548 * properly here
1549 */
1550 goto fail_cmn_reoq_cleanup;
1551 }
1552 }
1553
1554 for (i = 0; i < HAL_DSCP_TID_MAP_TBL_NUM_ENTRIES_MAX; i++)
1555 ath12k_hal_tx_set_dscp_tid_map(ab, i);
1556
1557 ret = ath12k_dp_rx_alloc(ab);
1558 if (ret)
1559 goto fail_dp_rx_free;
1560
1561 /* Init any SOC level resource for DP */
1562
1563 return 0;
1564
1565 fail_dp_rx_free:
1566 ath12k_dp_rx_free(ab);
1567
1568 fail_cmn_reoq_cleanup:
1569 ath12k_dp_reoq_lut_addr_reset(dp);
1570 ath12k_dp_reoq_lut_cleanup(ab);
1571
1572 fail_cmn_srng_cleanup:
1573 ath12k_dp_srng_common_cleanup(ab);
1574
1575 fail_dp_bank_profiles_cleanup:
1576 ath12k_dp_deinit_bank_profiles(ab);
1577
1578 fail_hw_cc_cleanup:
1579 ath12k_dp_cc_cleanup(ab);
1580
1581 fail_link_desc_cleanup:
1582 ath12k_dp_link_desc_cleanup(ab, dp->link_desc_banks,
1583 HAL_WBM_IDLE_LINK, &dp->wbm_idle_ring);
1584 rhash_destroy:
1585 ath12k_dp_link_peer_rhash_tbl_destroy(dp);
1586
1587 return ret;
1588 }
1589
ath12k_dp_cmn_device_deinit(struct ath12k_dp * dp)1590 void ath12k_dp_cmn_device_deinit(struct ath12k_dp *dp)
1591 {
1592 ath12k_dp_cleanup(dp->ab);
1593 }
1594
ath12k_dp_cmn_device_init(struct ath12k_dp * dp)1595 int ath12k_dp_cmn_device_init(struct ath12k_dp *dp)
1596 {
1597 int ret;
1598
1599 ret = ath12k_dp_setup(dp->ab);
1600 if (ret)
1601 return ret;
1602
1603 return 0;
1604 }
1605
ath12k_dp_cmn_hw_group_unassign(struct ath12k_dp * dp,struct ath12k_hw_group * ag)1606 void ath12k_dp_cmn_hw_group_unassign(struct ath12k_dp *dp,
1607 struct ath12k_hw_group *ag)
1608 {
1609 struct ath12k_dp_hw_group *dp_hw_grp = &ag->dp_hw_grp;
1610
1611 lockdep_assert_held(&ag->mutex);
1612
1613 dp_hw_grp->dp[dp->device_id] = NULL;
1614
1615 dp->ag = NULL;
1616 dp->device_id = ATH12K_INVALID_DEVICE_ID;
1617 }
1618
ath12k_dp_cmn_hw_group_assign(struct ath12k_dp * dp,struct ath12k_hw_group * ag)1619 void ath12k_dp_cmn_hw_group_assign(struct ath12k_dp *dp,
1620 struct ath12k_hw_group *ag)
1621 {
1622 struct ath12k_base *ab = dp->ab;
1623 struct ath12k_dp_hw_group *dp_hw_grp = &ag->dp_hw_grp;
1624
1625 dp->ag = ag;
1626 dp->device_id = ab->device_id;
1627 dp_hw_grp->dp[dp->device_id] = dp;
1628 }
1629
ath12k_dp_reoq_lut_addr_reset(struct ath12k_dp * dp)1630 void ath12k_dp_reoq_lut_addr_reset(struct ath12k_dp *dp)
1631 {
1632 struct ath12k_base *ab = dp->ab;
1633
1634 if (dp->reoq_lut.vaddr_unaligned)
1635 ath12k_hal_write_reoq_lut_addr(ab, 0);
1636
1637 if (dp->ml_reoq_lut.vaddr_unaligned)
1638 ath12k_hal_write_ml_reoq_lut_addr(ab, 0);
1639 }
1640