1 // SPDX-License-Identifier: BSD-3-Clause-Clear
2 /*
3 * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved.
4 * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
5 */
6
7 #include <crypto/hash.h>
8 #include "core.h"
9 #include "dp_tx.h"
10 #include "hif.h"
11 #include "hal.h"
12 #include "debug.h"
13 #include "peer.h"
14 #include "dp_cmn.h"
15
16 enum ath12k_dp_desc_type {
17 ATH12K_DP_TX_DESC,
18 ATH12K_DP_RX_DESC,
19 };
20
ath12k_dp_peer_cleanup(struct ath12k * ar,int vdev_id,const u8 * addr)21 void ath12k_dp_peer_cleanup(struct ath12k *ar, int vdev_id, const u8 *addr)
22 {
23 struct ath12k_base *ab = ar->ab;
24 struct ath12k_dp_link_peer *peer;
25 struct ath12k_dp *dp = ath12k_ab_to_dp(ab);
26
27 /* TODO: Any other peer specific DP cleanup */
28
29 spin_lock_bh(&dp->dp_lock);
30 peer = ath12k_dp_link_peer_find_by_vdev_and_addr(dp, vdev_id, addr);
31 if (!peer || !peer->dp_peer) {
32 ath12k_warn(ab, "failed to lookup peer %pM on vdev %d\n",
33 addr, vdev_id);
34 spin_unlock_bh(&dp->dp_lock);
35 return;
36 }
37
38 if (!peer->primary_link) {
39 spin_unlock_bh(&dp->dp_lock);
40 return;
41 }
42
43 ath12k_dp_rx_peer_tid_cleanup(ar, peer);
44 crypto_free_shash(peer->dp_peer->tfm_mmic);
45 peer->dp_peer->dp_setup_done = false;
46 spin_unlock_bh(&dp->dp_lock);
47 }
48
ath12k_dp_peer_setup(struct ath12k * ar,int vdev_id,const u8 * addr)49 int ath12k_dp_peer_setup(struct ath12k *ar, int vdev_id, const u8 *addr)
50 {
51 struct ath12k_base *ab = ar->ab;
52 struct ath12k_dp_link_peer *peer;
53 u32 reo_dest;
54 int ret = 0, tid;
55 struct ath12k_dp *dp = ath12k_ab_to_dp(ab);
56
57 /* NOTE: reo_dest ring id starts from 1 unlike mac_id which starts from 0 */
58 reo_dest = ar->dp.mac_id + 1;
59 ret = ath12k_wmi_set_peer_param(ar, addr, vdev_id,
60 WMI_PEER_SET_DEFAULT_ROUTING,
61 DP_RX_HASH_ENABLE | (reo_dest << 1));
62
63 if (ret) {
64 ath12k_warn(ab, "failed to set default routing %d peer :%pM vdev_id :%d\n",
65 ret, addr, vdev_id);
66 return ret;
67 }
68
69 for (tid = 0; tid <= IEEE80211_NUM_TIDS; tid++) {
70 ret = ath12k_dp_rx_peer_tid_setup(ar, addr, vdev_id, tid, 1, 0,
71 HAL_PN_TYPE_NONE);
72 if (ret) {
73 ath12k_warn(ab, "failed to setup rxd tid queue for tid %d: %d\n",
74 tid, ret);
75 goto peer_clean;
76 }
77 }
78
79 ret = ath12k_dp_rx_peer_frag_setup(ar, addr, vdev_id);
80 if (ret) {
81 ath12k_warn(ab, "failed to setup rx defrag context\n");
82 goto peer_clean;
83 }
84
85 /* TODO: Setup other peer specific resource used in data path */
86
87 return 0;
88
89 peer_clean:
90 spin_lock_bh(&dp->dp_lock);
91
92 peer = ath12k_dp_link_peer_find_by_vdev_and_addr(dp, vdev_id, addr);
93 if (!peer) {
94 ath12k_warn(ab, "failed to find the peer to del rx tid\n");
95 spin_unlock_bh(&dp->dp_lock);
96 return -ENOENT;
97 }
98
99 for (tid--; tid >= 0; tid--)
100 ath12k_dp_arch_rx_peer_tid_delete(dp, peer, tid);
101
102 spin_unlock_bh(&dp->dp_lock);
103
104 return ret;
105 }
106
ath12k_dp_srng_cleanup(struct ath12k_base * ab,struct dp_srng * ring)107 void ath12k_dp_srng_cleanup(struct ath12k_base *ab, struct dp_srng *ring)
108 {
109 if (!ring->vaddr_unaligned)
110 return;
111
112 dma_free_coherent(ab->dev, ring->size, ring->vaddr_unaligned,
113 ring->paddr_unaligned);
114
115 ring->vaddr_unaligned = NULL;
116 }
117
ath12k_dp_srng_find_ring_in_mask(int ring_num,const u8 * grp_mask)118 static int ath12k_dp_srng_find_ring_in_mask(int ring_num, const u8 *grp_mask)
119 {
120 int ext_group_num;
121 u8 mask = 1 << ring_num;
122
123 for (ext_group_num = 0; ext_group_num < ATH12K_EXT_IRQ_GRP_NUM_MAX;
124 ext_group_num++) {
125 if (mask & grp_mask[ext_group_num])
126 return ext_group_num;
127 }
128
129 return -ENOENT;
130 }
131
ath12k_dp_srng_calculate_msi_group(struct ath12k_base * ab,enum hal_ring_type type,int ring_num)132 static int ath12k_dp_srng_calculate_msi_group(struct ath12k_base *ab,
133 enum hal_ring_type type, int ring_num)
134 {
135 const struct ath12k_hal_tcl_to_wbm_rbm_map *map;
136 const u8 *grp_mask;
137 int i;
138
139 switch (type) {
140 case HAL_WBM2SW_RELEASE:
141 if (ring_num == HAL_WBM2SW_REL_ERR_RING_NUM) {
142 grp_mask = &ab->hw_params->ring_mask->rx_wbm_rel[0];
143 ring_num = 0;
144 } else {
145 map = ab->hal.tcl_to_wbm_rbm_map;
146 for (i = 0; i < ab->hw_params->max_tx_ring; i++) {
147 if (ring_num == map[i].wbm_ring_num) {
148 ring_num = i;
149 break;
150 }
151 }
152
153 grp_mask = &ab->hw_params->ring_mask->tx[0];
154 }
155 break;
156 case HAL_REO_EXCEPTION:
157 grp_mask = &ab->hw_params->ring_mask->rx_err[0];
158 break;
159 case HAL_REO_DST:
160 grp_mask = &ab->hw_params->ring_mask->rx[0];
161 break;
162 case HAL_REO_STATUS:
163 grp_mask = &ab->hw_params->ring_mask->reo_status[0];
164 break;
165 case HAL_RXDMA_MONITOR_STATUS:
166 grp_mask = &ab->hw_params->ring_mask->rx_mon_status[0];
167 break;
168 case HAL_RXDMA_MONITOR_DST:
169 grp_mask = &ab->hw_params->ring_mask->rx_mon_dest[0];
170 break;
171 case HAL_TX_MONITOR_DST:
172 grp_mask = &ab->hw_params->ring_mask->tx_mon_dest[0];
173 break;
174 case HAL_RXDMA_BUF:
175 grp_mask = &ab->hw_params->ring_mask->host2rxdma[0];
176 break;
177 case HAL_RXDMA_MONITOR_BUF:
178 case HAL_TCL_DATA:
179 case HAL_TCL_CMD:
180 case HAL_REO_CMD:
181 case HAL_SW2WBM_RELEASE:
182 case HAL_WBM_IDLE_LINK:
183 case HAL_TCL_STATUS:
184 case HAL_REO_REINJECT:
185 case HAL_CE_SRC:
186 case HAL_CE_DST:
187 case HAL_CE_DST_STATUS:
188 default:
189 return -ENOENT;
190 }
191
192 return ath12k_dp_srng_find_ring_in_mask(ring_num, grp_mask);
193 }
194
ath12k_dp_srng_msi_setup(struct ath12k_base * ab,struct hal_srng_params * ring_params,enum hal_ring_type type,int ring_num)195 static void ath12k_dp_srng_msi_setup(struct ath12k_base *ab,
196 struct hal_srng_params *ring_params,
197 enum hal_ring_type type, int ring_num)
198 {
199 int msi_group_number, msi_data_count;
200 u32 msi_data_start, msi_irq_start, addr_lo, addr_hi;
201 int ret;
202
203 ret = ath12k_hif_get_user_msi_vector(ab, "DP",
204 &msi_data_count, &msi_data_start,
205 &msi_irq_start);
206 if (ret)
207 return;
208
209 msi_group_number = ath12k_dp_srng_calculate_msi_group(ab, type,
210 ring_num);
211 if (msi_group_number < 0) {
212 ath12k_dbg(ab, ATH12K_DBG_PCI,
213 "ring not part of an ext_group; ring_type: %d,ring_num %d",
214 type, ring_num);
215 ring_params->msi_addr = 0;
216 ring_params->msi_data = 0;
217 return;
218 }
219
220 if (msi_group_number > msi_data_count) {
221 ath12k_dbg(ab, ATH12K_DBG_PCI,
222 "multiple msi_groups share one msi, msi_group_num %d",
223 msi_group_number);
224 }
225
226 ath12k_hif_get_msi_address(ab, &addr_lo, &addr_hi);
227
228 ring_params->msi_addr = addr_lo;
229 ring_params->msi_addr |= (dma_addr_t)(((uint64_t)addr_hi) << 32);
230 ring_params->msi_data = (msi_group_number % msi_data_count)
231 + msi_data_start;
232 ring_params->flags |= HAL_SRNG_FLAGS_MSI_INTR;
233 }
234
ath12k_dp_srng_setup(struct ath12k_base * ab,struct dp_srng * ring,enum hal_ring_type type,int ring_num,int mac_id,int num_entries)235 int ath12k_dp_srng_setup(struct ath12k_base *ab, struct dp_srng *ring,
236 enum hal_ring_type type, int ring_num,
237 int mac_id, int num_entries)
238 {
239 struct hal_srng_params params = {};
240 int entry_sz = ath12k_hal_srng_get_entrysize(ab, type);
241 int max_entries = ath12k_hal_srng_get_max_entries(ab, type);
242 int ret;
243
244 if (max_entries < 0 || entry_sz < 0)
245 return -EINVAL;
246
247 if (num_entries > max_entries)
248 num_entries = max_entries;
249
250 ring->size = (num_entries * entry_sz) + HAL_RING_BASE_ALIGN - 1;
251 ring->vaddr_unaligned = dma_alloc_coherent(ab->dev, ring->size,
252 &ring->paddr_unaligned,
253 GFP_KERNEL);
254 if (!ring->vaddr_unaligned)
255 return -ENOMEM;
256
257 ring->vaddr = PTR_ALIGN(ring->vaddr_unaligned, HAL_RING_BASE_ALIGN);
258 ring->paddr = ring->paddr_unaligned + ((unsigned long)ring->vaddr -
259 (unsigned long)ring->vaddr_unaligned);
260
261 params.ring_base_vaddr = ring->vaddr;
262 params.ring_base_paddr = ring->paddr;
263 params.num_entries = num_entries;
264 ath12k_dp_srng_msi_setup(ab, ¶ms, type, ring_num + mac_id);
265
266 switch (type) {
267 case HAL_REO_DST:
268 params.intr_batch_cntr_thres_entries =
269 HAL_SRNG_INT_BATCH_THRESHOLD_RX;
270 params.intr_timer_thres_us = HAL_SRNG_INT_TIMER_THRESHOLD_RX;
271 break;
272 case HAL_RXDMA_BUF:
273 case HAL_RXDMA_MONITOR_BUF:
274 params.low_threshold = num_entries >> 3;
275 params.flags |= HAL_SRNG_FLAGS_LOW_THRESH_INTR_EN;
276 params.intr_batch_cntr_thres_entries = 0;
277 params.intr_timer_thres_us = HAL_SRNG_INT_TIMER_THRESHOLD_RX;
278 break;
279 case HAL_RXDMA_MONITOR_STATUS:
280 params.low_threshold = num_entries >> 3;
281 params.flags |= HAL_SRNG_FLAGS_LOW_THRESH_INTR_EN;
282 params.intr_batch_cntr_thres_entries = 1;
283 params.intr_timer_thres_us = HAL_SRNG_INT_TIMER_THRESHOLD_RX;
284 break;
285 case HAL_TX_MONITOR_DST:
286 params.low_threshold = DP_TX_MONITOR_BUF_SIZE_MAX >> 3;
287 params.flags |= HAL_SRNG_FLAGS_LOW_THRESH_INTR_EN;
288 params.intr_batch_cntr_thres_entries = 0;
289 params.intr_timer_thres_us = HAL_SRNG_INT_TIMER_THRESHOLD_RX;
290 break;
291 case HAL_WBM2SW_RELEASE:
292 if (ab->hw_params->hw_ops->dp_srng_is_tx_comp_ring(ring_num)) {
293 params.intr_batch_cntr_thres_entries =
294 HAL_SRNG_INT_BATCH_THRESHOLD_TX;
295 params.intr_timer_thres_us =
296 HAL_SRNG_INT_TIMER_THRESHOLD_TX;
297 break;
298 }
299 /* follow through when ring_num != HAL_WBM2SW_REL_ERR_RING_NUM */
300 fallthrough;
301 case HAL_REO_EXCEPTION:
302 case HAL_REO_REINJECT:
303 case HAL_REO_CMD:
304 case HAL_REO_STATUS:
305 case HAL_TCL_DATA:
306 case HAL_TCL_CMD:
307 case HAL_TCL_STATUS:
308 case HAL_WBM_IDLE_LINK:
309 case HAL_SW2WBM_RELEASE:
310 case HAL_RXDMA_DST:
311 case HAL_RXDMA_MONITOR_DST:
312 case HAL_RXDMA_MONITOR_DESC:
313 params.intr_batch_cntr_thres_entries =
314 HAL_SRNG_INT_BATCH_THRESHOLD_OTHER;
315 params.intr_timer_thres_us = HAL_SRNG_INT_TIMER_THRESHOLD_OTHER;
316 break;
317 case HAL_RXDMA_DIR_BUF:
318 break;
319 default:
320 ath12k_warn(ab, "Not a valid ring type in dp :%d\n", type);
321 return -EINVAL;
322 }
323
324 ret = ath12k_hal_srng_setup(ab, type, ring_num, mac_id, ¶ms);
325 if (ret < 0) {
326 ath12k_warn(ab, "failed to setup srng: %d ring_id %d\n",
327 ret, ring_num);
328 return ret;
329 }
330
331 ring->ring_id = ret;
332
333 return 0;
334 }
335
ath12k_dp_tx_get_bank_profile(struct ath12k_base * ab,struct ath12k_link_vif * arvif,struct ath12k_dp * dp)336 static int ath12k_dp_tx_get_bank_profile(struct ath12k_base *ab,
337 struct ath12k_link_vif *arvif,
338 struct ath12k_dp *dp)
339 {
340 int bank_id = DP_INVALID_BANK_ID;
341 int i;
342 u32 bank_config;
343 bool configure_register = false;
344
345 /* convert vdev params into hal_tx_bank_config */
346 bank_config = ath12k_dp_arch_tx_get_vdev_bank_config(dp, arvif);
347
348 spin_lock_bh(&dp->tx_bank_lock);
349 /* TODO: implement using idr kernel framework*/
350 for (i = 0; i < dp->num_bank_profiles; i++) {
351 if (dp->bank_profiles[i].is_configured &&
352 (dp->bank_profiles[i].bank_config ^ bank_config) == 0) {
353 bank_id = i;
354 goto inc_ref_and_return;
355 }
356 if (!dp->bank_profiles[i].is_configured ||
357 !dp->bank_profiles[i].num_users) {
358 bank_id = i;
359 goto configure_and_return;
360 }
361 }
362
363 if (bank_id == DP_INVALID_BANK_ID) {
364 spin_unlock_bh(&dp->tx_bank_lock);
365 ath12k_err(ab, "unable to find TX bank!");
366 return bank_id;
367 }
368
369 configure_and_return:
370 dp->bank_profiles[bank_id].is_configured = true;
371 dp->bank_profiles[bank_id].bank_config = bank_config;
372 configure_register = true;
373 inc_ref_and_return:
374 dp->bank_profiles[bank_id].num_users++;
375 spin_unlock_bh(&dp->tx_bank_lock);
376
377 if (configure_register)
378 ath12k_hal_tx_configure_bank_register(ab,
379 bank_config, bank_id);
380
381 ath12k_dbg(ab, ATH12K_DBG_DP_HTT, "dp_htt tcl bank_id %d input 0x%x match 0x%x num_users %u",
382 bank_id, bank_config, dp->bank_profiles[bank_id].bank_config,
383 dp->bank_profiles[bank_id].num_users);
384
385 return bank_id;
386 }
387
ath12k_dp_tx_put_bank_profile(struct ath12k_dp * dp,u8 bank_id)388 void ath12k_dp_tx_put_bank_profile(struct ath12k_dp *dp, u8 bank_id)
389 {
390 spin_lock_bh(&dp->tx_bank_lock);
391 dp->bank_profiles[bank_id].num_users--;
392 spin_unlock_bh(&dp->tx_bank_lock);
393 }
394
ath12k_dp_deinit_bank_profiles(struct ath12k_base * ab)395 static void ath12k_dp_deinit_bank_profiles(struct ath12k_base *ab)
396 {
397 struct ath12k_dp *dp = ath12k_ab_to_dp(ab);
398
399 kfree(dp->bank_profiles);
400 dp->bank_profiles = NULL;
401 }
402
ath12k_dp_init_bank_profiles(struct ath12k_base * ab)403 static int ath12k_dp_init_bank_profiles(struct ath12k_base *ab)
404 {
405 struct ath12k_dp *dp = ath12k_ab_to_dp(ab);
406 u32 num_tcl_banks = ab->hw_params->num_tcl_banks;
407 int i;
408
409 dp->num_bank_profiles = num_tcl_banks;
410 dp->bank_profiles = kmalloc_objs(struct ath12k_dp_tx_bank_profile,
411 num_tcl_banks);
412 if (!dp->bank_profiles)
413 return -ENOMEM;
414
415 spin_lock_init(&dp->tx_bank_lock);
416
417 for (i = 0; i < num_tcl_banks; i++) {
418 dp->bank_profiles[i].is_configured = false;
419 dp->bank_profiles[i].num_users = 0;
420 }
421
422 return 0;
423 }
424
ath12k_dp_srng_common_cleanup(struct ath12k_base * ab)425 static void ath12k_dp_srng_common_cleanup(struct ath12k_base *ab)
426 {
427 struct ath12k_dp *dp = ath12k_ab_to_dp(ab);
428 int i;
429
430 ath12k_dp_srng_cleanup(ab, &dp->reo_status_ring);
431 ath12k_dp_srng_cleanup(ab, &dp->reo_cmd_ring);
432 ath12k_dp_srng_cleanup(ab, &dp->reo_except_ring);
433 ath12k_dp_srng_cleanup(ab, &dp->rx_rel_ring);
434 ath12k_dp_srng_cleanup(ab, &dp->reo_reinject_ring);
435 for (i = 0; i < ab->hw_params->max_tx_ring; i++) {
436 ath12k_dp_srng_cleanup(ab, &dp->tx_ring[i].tcl_comp_ring);
437 ath12k_dp_srng_cleanup(ab, &dp->tx_ring[i].tcl_data_ring);
438 }
439 ath12k_dp_srng_cleanup(ab, &dp->wbm_desc_rel_ring);
440 }
441
ath12k_dp_srng_common_setup(struct ath12k_base * ab)442 static int ath12k_dp_srng_common_setup(struct ath12k_base *ab)
443 {
444 struct ath12k_dp *dp = ath12k_ab_to_dp(ab);
445 const struct ath12k_hal_tcl_to_wbm_rbm_map *map;
446 struct hal_srng *srng;
447 int i, ret, tx_comp_ring_num;
448 u32 ring_hash_map;
449
450 ret = ath12k_dp_srng_setup(ab, &dp->wbm_desc_rel_ring,
451 HAL_SW2WBM_RELEASE, 0, 0,
452 DP_WBM_RELEASE_RING_SIZE);
453 if (ret) {
454 ath12k_warn(ab, "failed to set up wbm2sw_release ring :%d\n",
455 ret);
456 goto err;
457 }
458
459 for (i = 0; i < ab->hw_params->max_tx_ring; i++) {
460 map = ab->hal.tcl_to_wbm_rbm_map;
461 tx_comp_ring_num = map[i].wbm_ring_num;
462
463 ret = ath12k_dp_srng_setup(ab, &dp->tx_ring[i].tcl_data_ring,
464 HAL_TCL_DATA, i, 0,
465 DP_TCL_DATA_RING_SIZE);
466 if (ret) {
467 ath12k_warn(ab, "failed to set up tcl_data ring (%d) :%d\n",
468 i, ret);
469 goto err;
470 }
471
472 ret = ath12k_dp_srng_setup(ab, &dp->tx_ring[i].tcl_comp_ring,
473 HAL_WBM2SW_RELEASE, tx_comp_ring_num, 0,
474 DP_TX_COMP_RING_SIZE(ab));
475 if (ret) {
476 ath12k_warn(ab, "failed to set up tcl_comp ring (%d) :%d\n",
477 tx_comp_ring_num, ret);
478 goto err;
479 }
480 }
481
482 ret = ath12k_dp_srng_setup(ab, &dp->reo_reinject_ring, HAL_REO_REINJECT,
483 0, 0, DP_REO_REINJECT_RING_SIZE);
484 if (ret) {
485 ath12k_warn(ab, "failed to set up reo_reinject ring :%d\n",
486 ret);
487 goto err;
488 }
489
490 ret = ath12k_dp_srng_setup(ab, &dp->rx_rel_ring, HAL_WBM2SW_RELEASE,
491 HAL_WBM2SW_REL_ERR_RING_NUM, 0,
492 DP_RX_RELEASE_RING_SIZE);
493 if (ret) {
494 ath12k_warn(ab, "failed to set up rx_rel ring :%d\n", ret);
495 goto err;
496 }
497
498 ret = ath12k_dp_srng_setup(ab, &dp->reo_except_ring, HAL_REO_EXCEPTION,
499 0, 0, DP_REO_EXCEPTION_RING_SIZE);
500 if (ret) {
501 ath12k_warn(ab, "failed to set up reo_exception ring :%d\n",
502 ret);
503 goto err;
504 }
505
506 ret = ath12k_dp_srng_setup(ab, &dp->reo_cmd_ring, HAL_REO_CMD,
507 0, 0, DP_REO_CMD_RING_SIZE);
508 if (ret) {
509 ath12k_warn(ab, "failed to set up reo_cmd ring :%d\n", ret);
510 goto err;
511 }
512
513 srng = &ab->hal.srng_list[dp->reo_cmd_ring.ring_id];
514 ath12k_hal_reo_init_cmd_ring(ab, srng);
515
516 ret = ath12k_dp_srng_setup(ab, &dp->reo_status_ring, HAL_REO_STATUS,
517 0, 0, DP_REO_STATUS_RING_SIZE);
518 if (ret) {
519 ath12k_warn(ab, "failed to set up reo_status ring :%d\n", ret);
520 goto err;
521 }
522
523 /* When hash based routing of rx packet is enabled, 32 entries to map
524 * the hash values to the ring will be configured. Each hash entry uses
525 * four bits to map to a particular ring. The ring mapping will be
526 * 0:TCL, 1:SW1, 2:SW2, 3:SW3, 4:SW4, 5:Release, 6:FW and 7:SW5
527 * 8:SW6, 9:SW7, 10:SW8, 11:Not used.
528 */
529 ring_hash_map = HAL_HASH_ROUTING_RING_SW1 |
530 HAL_HASH_ROUTING_RING_SW2 << 4 |
531 HAL_HASH_ROUTING_RING_SW3 << 8 |
532 HAL_HASH_ROUTING_RING_SW4 << 12 |
533 HAL_HASH_ROUTING_RING_SW1 << 16 |
534 HAL_HASH_ROUTING_RING_SW2 << 20 |
535 HAL_HASH_ROUTING_RING_SW3 << 24 |
536 HAL_HASH_ROUTING_RING_SW4 << 28;
537
538 ath12k_hal_reo_hw_setup(ab, ring_hash_map);
539
540 return 0;
541
542 err:
543 ath12k_dp_srng_common_cleanup(ab);
544
545 return ret;
546 }
547
ath12k_dp_scatter_idle_link_desc_cleanup(struct ath12k_base * ab)548 static void ath12k_dp_scatter_idle_link_desc_cleanup(struct ath12k_base *ab)
549 {
550 struct ath12k_dp *dp = ath12k_ab_to_dp(ab);
551 struct hal_wbm_idle_scatter_list *slist = dp->scatter_list;
552 int i;
553
554 for (i = 0; i < DP_IDLE_SCATTER_BUFS_MAX; i++) {
555 if (!slist[i].vaddr)
556 continue;
557
558 dma_free_coherent(ab->dev, HAL_WBM_IDLE_SCATTER_BUF_SIZE_MAX,
559 slist[i].vaddr, slist[i].paddr);
560 slist[i].vaddr = NULL;
561 }
562 }
563
ath12k_dp_scatter_idle_link_desc_setup(struct ath12k_base * ab,int size,u32 n_link_desc_bank,u32 n_link_desc,u32 last_bank_sz)564 static int ath12k_dp_scatter_idle_link_desc_setup(struct ath12k_base *ab,
565 int size,
566 u32 n_link_desc_bank,
567 u32 n_link_desc,
568 u32 last_bank_sz)
569 {
570 struct ath12k_dp *dp = ath12k_ab_to_dp(ab);
571 struct dp_link_desc_bank *link_desc_banks = dp->link_desc_banks;
572 struct hal_wbm_idle_scatter_list *slist = dp->scatter_list;
573 u32 n_entries_per_buf;
574 int num_scatter_buf, scatter_idx;
575 struct hal_wbm_link_desc *scatter_buf;
576 int align_bytes, n_entries;
577 dma_addr_t paddr;
578 int rem_entries;
579 int i;
580 int ret = 0;
581 u32 end_offset, cookie;
582 enum hal_rx_buf_return_buf_manager rbm = dp->idle_link_rbm;
583
584 n_entries_per_buf = HAL_WBM_IDLE_SCATTER_BUF_SIZE /
585 ath12k_hal_srng_get_entrysize(ab, HAL_WBM_IDLE_LINK);
586 num_scatter_buf = DIV_ROUND_UP(size, HAL_WBM_IDLE_SCATTER_BUF_SIZE);
587
588 if (num_scatter_buf > DP_IDLE_SCATTER_BUFS_MAX)
589 return -EINVAL;
590
591 for (i = 0; i < num_scatter_buf; i++) {
592 slist[i].vaddr = dma_alloc_coherent(ab->dev,
593 HAL_WBM_IDLE_SCATTER_BUF_SIZE_MAX,
594 &slist[i].paddr, GFP_KERNEL);
595 if (!slist[i].vaddr) {
596 ret = -ENOMEM;
597 goto err;
598 }
599 }
600
601 scatter_idx = 0;
602 scatter_buf = slist[scatter_idx].vaddr;
603 rem_entries = n_entries_per_buf;
604
605 for (i = 0; i < n_link_desc_bank; i++) {
606 #if defined(__linux__)
607 align_bytes = link_desc_banks[i].vaddr -
608 link_desc_banks[i].vaddr_unaligned;
609 #elif defined(__FreeBSD__)
610 align_bytes = (uintptr_t)link_desc_banks[i].vaddr -
611 (uintptr_t)link_desc_banks[i].vaddr_unaligned;
612 #endif
613 n_entries = (DP_LINK_DESC_ALLOC_SIZE_THRESH - align_bytes) /
614 HAL_LINK_DESC_SIZE;
615 paddr = link_desc_banks[i].paddr;
616 while (n_entries) {
617 cookie = DP_LINK_DESC_COOKIE_SET(n_entries, i);
618 ath12k_hal_set_link_desc_addr(dp->hal, scatter_buf, cookie,
619 paddr, rbm);
620 n_entries--;
621 paddr += HAL_LINK_DESC_SIZE;
622 if (rem_entries) {
623 rem_entries--;
624 scatter_buf++;
625 continue;
626 }
627
628 rem_entries = n_entries_per_buf;
629 scatter_idx++;
630 scatter_buf = slist[scatter_idx].vaddr;
631 }
632 }
633
634 end_offset = (scatter_buf - slist[scatter_idx].vaddr) *
635 sizeof(struct hal_wbm_link_desc);
636 ath12k_hal_setup_link_idle_list(ab, slist, num_scatter_buf,
637 n_link_desc, end_offset);
638
639 return 0;
640
641 err:
642 ath12k_dp_scatter_idle_link_desc_cleanup(ab);
643
644 return ret;
645 }
646
647 static void
ath12k_dp_link_desc_bank_free(struct ath12k_base * ab,struct dp_link_desc_bank * link_desc_banks)648 ath12k_dp_link_desc_bank_free(struct ath12k_base *ab,
649 struct dp_link_desc_bank *link_desc_banks)
650 {
651 int i;
652
653 for (i = 0; i < DP_LINK_DESC_BANKS_MAX; i++) {
654 if (link_desc_banks[i].vaddr_unaligned) {
655 dma_free_coherent(ab->dev,
656 link_desc_banks[i].size,
657 link_desc_banks[i].vaddr_unaligned,
658 link_desc_banks[i].paddr_unaligned);
659 link_desc_banks[i].vaddr_unaligned = NULL;
660 }
661 }
662 }
663
ath12k_dp_link_desc_bank_alloc(struct ath12k_base * ab,struct dp_link_desc_bank * desc_bank,int n_link_desc_bank,int last_bank_sz)664 static int ath12k_dp_link_desc_bank_alloc(struct ath12k_base *ab,
665 struct dp_link_desc_bank *desc_bank,
666 int n_link_desc_bank,
667 int last_bank_sz)
668 {
669 struct ath12k_dp *dp = ath12k_ab_to_dp(ab);
670 int i;
671 int ret = 0;
672 int desc_sz = DP_LINK_DESC_ALLOC_SIZE_THRESH;
673
674 for (i = 0; i < n_link_desc_bank; i++) {
675 if (i == (n_link_desc_bank - 1) && last_bank_sz)
676 desc_sz = last_bank_sz;
677
678 desc_bank[i].vaddr_unaligned =
679 dma_alloc_coherent(ab->dev, desc_sz,
680 &desc_bank[i].paddr_unaligned,
681 GFP_KERNEL);
682 if (!desc_bank[i].vaddr_unaligned) {
683 ret = -ENOMEM;
684 goto err;
685 }
686
687 desc_bank[i].vaddr = PTR_ALIGN(desc_bank[i].vaddr_unaligned,
688 HAL_LINK_DESC_ALIGN);
689 desc_bank[i].paddr = desc_bank[i].paddr_unaligned +
690 ((unsigned long)desc_bank[i].vaddr -
691 (unsigned long)desc_bank[i].vaddr_unaligned);
692 desc_bank[i].size = desc_sz;
693 }
694
695 return 0;
696
697 err:
698 ath12k_dp_link_desc_bank_free(ab, dp->link_desc_banks);
699
700 return ret;
701 }
702
ath12k_dp_link_desc_cleanup(struct ath12k_base * ab,struct dp_link_desc_bank * desc_bank,u32 ring_type,struct dp_srng * ring)703 void ath12k_dp_link_desc_cleanup(struct ath12k_base *ab,
704 struct dp_link_desc_bank *desc_bank,
705 u32 ring_type, struct dp_srng *ring)
706 {
707 ath12k_dp_link_desc_bank_free(ab, desc_bank);
708
709 if (ring_type != HAL_RXDMA_MONITOR_DESC) {
710 ath12k_dp_srng_cleanup(ab, ring);
711 ath12k_dp_scatter_idle_link_desc_cleanup(ab);
712 }
713 }
714
ath12k_wbm_idle_ring_setup(struct ath12k_base * ab,u32 * n_link_desc)715 static int ath12k_wbm_idle_ring_setup(struct ath12k_base *ab, u32 *n_link_desc)
716 {
717 struct ath12k_dp *dp = ath12k_ab_to_dp(ab);
718 u32 n_mpdu_link_desc, n_mpdu_queue_desc;
719 u32 n_tx_msdu_link_desc, n_rx_msdu_link_desc;
720 int ret = 0;
721
722 n_mpdu_link_desc = (DP_NUM_TIDS_MAX * DP_AVG_MPDUS_PER_TID_MAX) /
723 HAL_NUM_MPDUS_PER_LINK_DESC;
724
725 n_mpdu_queue_desc = n_mpdu_link_desc /
726 HAL_NUM_MPDU_LINKS_PER_QUEUE_DESC;
727
728 n_tx_msdu_link_desc = (DP_NUM_TIDS_MAX * DP_AVG_FLOWS_PER_TID *
729 DP_AVG_MSDUS_PER_FLOW) /
730 HAL_NUM_TX_MSDUS_PER_LINK_DESC;
731
732 n_rx_msdu_link_desc = (DP_NUM_TIDS_MAX * DP_AVG_MPDUS_PER_TID_MAX *
733 DP_AVG_MSDUS_PER_MPDU) /
734 HAL_NUM_RX_MSDUS_PER_LINK_DESC;
735
736 *n_link_desc = n_mpdu_link_desc + n_mpdu_queue_desc +
737 n_tx_msdu_link_desc + n_rx_msdu_link_desc;
738
739 if (*n_link_desc & (*n_link_desc - 1))
740 *n_link_desc = 1 << fls(*n_link_desc);
741
742 ret = ath12k_dp_srng_setup(ab, &dp->wbm_idle_ring,
743 HAL_WBM_IDLE_LINK, 0, 0, *n_link_desc);
744 if (ret) {
745 ath12k_warn(ab, "failed to setup wbm_idle_ring: %d\n", ret);
746 return ret;
747 }
748 return ret;
749 }
750
ath12k_dp_link_desc_setup(struct ath12k_base * ab,struct dp_link_desc_bank * link_desc_banks,u32 ring_type,struct hal_srng * srng,u32 n_link_desc)751 int ath12k_dp_link_desc_setup(struct ath12k_base *ab,
752 struct dp_link_desc_bank *link_desc_banks,
753 u32 ring_type, struct hal_srng *srng,
754 u32 n_link_desc)
755 {
756 struct ath12k_dp *dp = ath12k_ab_to_dp(ab);
757 u32 tot_mem_sz;
758 u32 n_link_desc_bank, last_bank_sz;
759 u32 entry_sz, align_bytes, n_entries;
760 struct hal_wbm_link_desc *desc;
761 u32 paddr;
762 int i, ret;
763 u32 cookie;
764 enum hal_rx_buf_return_buf_manager rbm = dp->idle_link_rbm;
765
766 tot_mem_sz = n_link_desc * HAL_LINK_DESC_SIZE;
767 tot_mem_sz += HAL_LINK_DESC_ALIGN;
768
769 if (tot_mem_sz <= DP_LINK_DESC_ALLOC_SIZE_THRESH) {
770 n_link_desc_bank = 1;
771 last_bank_sz = tot_mem_sz;
772 } else {
773 n_link_desc_bank = tot_mem_sz /
774 (DP_LINK_DESC_ALLOC_SIZE_THRESH -
775 HAL_LINK_DESC_ALIGN);
776 last_bank_sz = tot_mem_sz %
777 (DP_LINK_DESC_ALLOC_SIZE_THRESH -
778 HAL_LINK_DESC_ALIGN);
779
780 if (last_bank_sz)
781 n_link_desc_bank += 1;
782 }
783
784 if (n_link_desc_bank > DP_LINK_DESC_BANKS_MAX)
785 return -EINVAL;
786
787 ret = ath12k_dp_link_desc_bank_alloc(ab, link_desc_banks,
788 n_link_desc_bank, last_bank_sz);
789 if (ret)
790 return ret;
791
792 /* Setup link desc idle list for HW internal usage */
793 entry_sz = ath12k_hal_srng_get_entrysize(ab, ring_type);
794 tot_mem_sz = entry_sz * n_link_desc;
795
796 /* Setup scatter desc list when the total memory requirement is more */
797 if (tot_mem_sz > DP_LINK_DESC_ALLOC_SIZE_THRESH &&
798 ring_type != HAL_RXDMA_MONITOR_DESC) {
799 ret = ath12k_dp_scatter_idle_link_desc_setup(ab, tot_mem_sz,
800 n_link_desc_bank,
801 n_link_desc,
802 last_bank_sz);
803 if (ret) {
804 ath12k_warn(ab, "failed to setup scatting idle list descriptor :%d\n",
805 ret);
806 goto fail_desc_bank_free;
807 }
808
809 return 0;
810 }
811
812 spin_lock_bh(&srng->lock);
813
814 ath12k_hal_srng_access_begin(ab, srng);
815
816 for (i = 0; i < n_link_desc_bank; i++) {
817 #if defined(__linux__)
818 align_bytes = link_desc_banks[i].vaddr -
819 link_desc_banks[i].vaddr_unaligned;
820 #elif defined(__FreeBSD__)
821 align_bytes = (uintptr_t)link_desc_banks[i].vaddr -
822 (uintptr_t)link_desc_banks[i].vaddr_unaligned;
823 #endif
824 n_entries = (link_desc_banks[i].size - align_bytes) /
825 HAL_LINK_DESC_SIZE;
826 paddr = link_desc_banks[i].paddr;
827 while (n_entries &&
828 (desc = ath12k_hal_srng_src_get_next_entry(ab, srng))) {
829 cookie = DP_LINK_DESC_COOKIE_SET(n_entries, i);
830 ath12k_hal_set_link_desc_addr(dp->hal, desc, cookie, paddr,
831 rbm);
832 n_entries--;
833 paddr += HAL_LINK_DESC_SIZE;
834 }
835 }
836
837 ath12k_hal_srng_access_end(ab, srng);
838
839 spin_unlock_bh(&srng->lock);
840
841 return 0;
842
843 fail_desc_bank_free:
844 ath12k_dp_link_desc_bank_free(ab, link_desc_banks);
845
846 return ret;
847 }
848
ath12k_dp_pdev_free(struct ath12k_base * ab)849 void ath12k_dp_pdev_free(struct ath12k_base *ab)
850 {
851 struct ath12k_dp *dp = ath12k_ab_to_dp(ab);
852 struct ath12k *ar;
853 int i;
854
855 for (i = 0; i < ab->num_radios; i++) {
856 ar = ab->pdevs[i].ar;
857 rcu_assign_pointer(dp->dp_pdevs[ar->pdev_idx], NULL);
858 }
859
860 synchronize_rcu();
861
862 for (i = 0; i < ab->num_radios; i++)
863 ath12k_dp_rx_pdev_free(ab, i);
864 }
865
ath12k_dp_pdev_pre_alloc(struct ath12k * ar)866 void ath12k_dp_pdev_pre_alloc(struct ath12k *ar)
867 {
868 struct ath12k_pdev_dp *dp = &ar->dp;
869
870 dp->mac_id = ar->pdev_idx;
871 atomic_set(&dp->num_tx_pending, 0);
872 init_waitqueue_head(&dp->tx_empty_waitq);
873 /* TODO: Add any RXDMA setup required per pdev */
874 }
875
ath12k_dp_pdev_alloc(struct ath12k_base * ab)876 int ath12k_dp_pdev_alloc(struct ath12k_base *ab)
877 {
878 struct ath12k_dp *dp = ath12k_ab_to_dp(ab);
879 struct ath12k_pdev_dp *dp_pdev;
880 struct ath12k *ar;
881 int ret;
882 int i;
883
884 ret = ath12k_dp_rx_htt_setup(ab);
885 if (ret)
886 goto out;
887
888 /* TODO: Per-pdev rx ring unlike tx ring which is mapped to different AC's */
889 for (i = 0; i < ab->num_radios; i++) {
890 ar = ab->pdevs[i].ar;
891
892 dp_pdev = &ar->dp;
893
894 dp_pdev->hw = ar->ah->hw;
895 dp_pdev->dp = dp;
896 dp_pdev->hw_link_id = ar->hw_link_id;
897 dp_pdev->dp_hw = &ar->ah->dp_hw;
898
899 ret = ath12k_dp_rx_pdev_alloc(ab, i);
900 if (ret) {
901 ath12k_warn(ab, "failed to allocate pdev rx for pdev_id :%d\n",
902 i);
903 goto err;
904 }
905 ret = ath12k_dp_rx_pdev_mon_attach(ar);
906 if (ret) {
907 ath12k_warn(ab, "failed to initialize mon pdev %d\n", i);
908 goto err;
909 }
910 }
911
912 for (i = 0; i < ab->num_radios; i++) {
913 ar = ab->pdevs[i].ar;
914 rcu_assign_pointer(dp->dp_pdevs[ar->pdev_idx], &ar->dp);
915 }
916
917 return 0;
918 err:
919 ath12k_dp_pdev_free(ab);
920 out:
921 return ret;
922 }
923
ath12k_dp_update_vdev_search(struct ath12k_link_vif * arvif)924 static void ath12k_dp_update_vdev_search(struct ath12k_link_vif *arvif)
925 {
926 u8 link_id = arvif->link_id;
927 struct ath12k_vif *ahvif = arvif->ahvif;
928 struct ath12k_dp_link_vif *dp_link_vif;
929
930 dp_link_vif = ath12k_dp_vif_to_dp_link_vif(&ahvif->dp_vif, link_id);
931
932 switch (arvif->ahvif->vdev_type) {
933 case WMI_VDEV_TYPE_STA:
934 dp_link_vif->hal_addr_search_flags = HAL_TX_ADDRY_EN;
935 dp_link_vif->search_type = HAL_TX_ADDR_SEARCH_DEFAULT;
936 break;
937 case WMI_VDEV_TYPE_AP:
938 case WMI_VDEV_TYPE_IBSS:
939 dp_link_vif->hal_addr_search_flags = HAL_TX_ADDRX_EN;
940 dp_link_vif->search_type = HAL_TX_ADDR_SEARCH_DEFAULT;
941 break;
942 case WMI_VDEV_TYPE_MONITOR:
943 default:
944 return;
945 }
946 }
947
ath12k_dp_vdev_tx_attach(struct ath12k * ar,struct ath12k_link_vif * arvif)948 void ath12k_dp_vdev_tx_attach(struct ath12k *ar, struct ath12k_link_vif *arvif)
949 {
950 struct ath12k_base *ab = ar->ab;
951 struct ath12k_vif *ahvif = arvif->ahvif;
952 u8 link_id = arvif->link_id;
953 int bank_id;
954 struct ath12k_dp_link_vif *dp_link_vif;
955
956 dp_link_vif = ath12k_dp_vif_to_dp_link_vif(&ahvif->dp_vif, link_id);
957
958 dp_link_vif->tcl_metadata |= u32_encode_bits(1, HTT_TCL_META_DATA_TYPE) |
959 u32_encode_bits(arvif->vdev_id,
960 HTT_TCL_META_DATA_VDEV_ID) |
961 u32_encode_bits(ar->pdev->pdev_id,
962 HTT_TCL_META_DATA_PDEV_ID);
963
964 /* set HTT extension valid bit to 0 by default */
965 dp_link_vif->tcl_metadata &= ~HTT_TCL_META_DATA_VALID_HTT;
966
967 ath12k_dp_update_vdev_search(arvif);
968 dp_link_vif->vdev_id_check_en = true;
969 bank_id = ath12k_dp_tx_get_bank_profile(ab, arvif, ath12k_ab_to_dp(ab));
970 dp_link_vif->bank_id = bank_id;
971
972 /* TODO: error path for bank id failure */
973 if (bank_id == DP_INVALID_BANK_ID) {
974 ath12k_err(ar->ab, "Failed to initialize DP TX Banks");
975 return;
976 }
977 }
978
ath12k_dp_cc_cleanup(struct ath12k_base * ab)979 static void ath12k_dp_cc_cleanup(struct ath12k_base *ab)
980 {
981 struct ath12k_rx_desc_info *desc_info;
982 struct ath12k_tx_desc_info *tx_desc_info, *tmp1;
983 struct ath12k_dp *dp = ath12k_ab_to_dp(ab);
984 struct ath12k_skb_cb *skb_cb;
985 struct sk_buff *skb;
986 struct ath12k *ar;
987 int i, j;
988 u32 pool_id, tx_spt_page;
989
990 if (!dp->spt_info)
991 return;
992
993 /* RX Descriptor cleanup */
994 spin_lock_bh(&dp->rx_desc_lock);
995
996 if (dp->rxbaddr) {
997 for (i = 0; i < ATH12K_NUM_RX_SPT_PAGES(ab); i++) {
998 if (!dp->rxbaddr[i])
999 continue;
1000
1001 desc_info = dp->rxbaddr[i];
1002
1003 for (j = 0; j < ATH12K_MAX_SPT_ENTRIES; j++) {
1004 if (!desc_info[j].in_use) {
1005 list_del(&desc_info[j].list);
1006 continue;
1007 }
1008
1009 skb = desc_info[j].skb;
1010 if (!skb)
1011 continue;
1012
1013 dma_unmap_single(ab->dev,
1014 ATH12K_SKB_RXCB(skb)->paddr,
1015 skb->len + skb_tailroom(skb),
1016 DMA_FROM_DEVICE);
1017 dev_kfree_skb_any(skb);
1018 }
1019
1020 kfree(dp->rxbaddr[i]);
1021 dp->rxbaddr[i] = NULL;
1022 }
1023
1024 kfree(dp->rxbaddr);
1025 dp->rxbaddr = NULL;
1026 }
1027
1028 spin_unlock_bh(&dp->rx_desc_lock);
1029
1030 /* TX Descriptor cleanup */
1031 for (i = 0; i < ATH12K_HW_MAX_QUEUES; i++) {
1032 spin_lock_bh(&dp->tx_desc_lock[i]);
1033
1034 list_for_each_entry_safe(tx_desc_info, tmp1,
1035 &dp->tx_desc_used_list[i], list) {
1036 list_del(&tx_desc_info->list);
1037 skb = tx_desc_info->skb;
1038
1039 if (!skb)
1040 continue;
1041
1042 skb_cb = ATH12K_SKB_CB(skb);
1043 if (skb_cb->paddr_ext_desc) {
1044 dma_unmap_single(ab->dev,
1045 skb_cb->paddr_ext_desc,
1046 tx_desc_info->skb_ext_desc->len,
1047 DMA_TO_DEVICE);
1048 dev_kfree_skb_any(tx_desc_info->skb_ext_desc);
1049 }
1050
1051 /* if we are unregistering, hw would've been destroyed and
1052 * ar is no longer valid.
1053 */
1054 if (!(test_bit(ATH12K_FLAG_UNREGISTERING, &ab->dev_flags))) {
1055 ar = skb_cb->ar;
1056
1057 if (atomic_dec_and_test(&ar->dp.num_tx_pending))
1058 wake_up(&ar->dp.tx_empty_waitq);
1059 }
1060
1061 dma_unmap_single(ab->dev, ATH12K_SKB_CB(skb)->paddr,
1062 skb->len, DMA_TO_DEVICE);
1063 dev_kfree_skb_any(skb);
1064 }
1065
1066 spin_unlock_bh(&dp->tx_desc_lock[i]);
1067 }
1068
1069 if (dp->txbaddr) {
1070 for (pool_id = 0; pool_id < ATH12K_HW_MAX_QUEUES; pool_id++) {
1071 spin_lock_bh(&dp->tx_desc_lock[pool_id]);
1072
1073 for (i = 0; i < ATH12K_TX_SPT_PAGES_PER_POOL(ab); i++) {
1074 tx_spt_page = i + pool_id *
1075 ATH12K_TX_SPT_PAGES_PER_POOL(ab);
1076 if (!dp->txbaddr[tx_spt_page])
1077 continue;
1078
1079 kfree(dp->txbaddr[tx_spt_page]);
1080 dp->txbaddr[tx_spt_page] = NULL;
1081 }
1082
1083 spin_unlock_bh(&dp->tx_desc_lock[pool_id]);
1084 }
1085
1086 kfree(dp->txbaddr);
1087 dp->txbaddr = NULL;
1088 }
1089
1090 /* unmap SPT pages */
1091 for (i = 0; i < dp->num_spt_pages; i++) {
1092 if (!dp->spt_info[i].vaddr)
1093 continue;
1094
1095 dma_free_coherent(ab->dev, ATH12K_PAGE_SIZE,
1096 dp->spt_info[i].vaddr, dp->spt_info[i].paddr);
1097 dp->spt_info[i].vaddr = NULL;
1098 }
1099
1100 kfree(dp->spt_info);
1101 dp->spt_info = NULL;
1102 }
1103
ath12k_dp_reoq_lut_cleanup(struct ath12k_base * ab)1104 static void ath12k_dp_reoq_lut_cleanup(struct ath12k_base *ab)
1105 {
1106 struct ath12k_dp *dp = ath12k_ab_to_dp(ab);
1107
1108 if (!ab->hw_params->reoq_lut_support)
1109 return;
1110
1111 if (dp->reoq_lut.vaddr_unaligned) {
1112 ath12k_hal_write_reoq_lut_addr(ab, 0);
1113 dma_free_coherent(ab->dev, dp->reoq_lut.size,
1114 dp->reoq_lut.vaddr_unaligned,
1115 dp->reoq_lut.paddr_unaligned);
1116 dp->reoq_lut.vaddr_unaligned = NULL;
1117 }
1118
1119 if (dp->ml_reoq_lut.vaddr_unaligned) {
1120 ath12k_hal_write_ml_reoq_lut_addr(ab, 0);
1121 dma_free_coherent(ab->dev, dp->ml_reoq_lut.size,
1122 dp->ml_reoq_lut.vaddr_unaligned,
1123 dp->ml_reoq_lut.paddr_unaligned);
1124 dp->ml_reoq_lut.vaddr_unaligned = NULL;
1125 }
1126 }
1127
ath12k_dp_cleanup(struct ath12k_base * ab)1128 static void ath12k_dp_cleanup(struct ath12k_base *ab)
1129 {
1130 struct ath12k_dp *dp = ath12k_ab_to_dp(ab);
1131 int i;
1132
1133 ath12k_dp_link_peer_rhash_tbl_destroy(dp);
1134
1135 if (!dp->ab)
1136 return;
1137
1138 ath12k_dp_link_desc_cleanup(ab, dp->link_desc_banks,
1139 HAL_WBM_IDLE_LINK, &dp->wbm_idle_ring);
1140
1141 ath12k_dp_cc_cleanup(ab);
1142 ath12k_dp_reoq_lut_cleanup(ab);
1143 ath12k_dp_deinit_bank_profiles(ab);
1144 ath12k_dp_srng_common_cleanup(ab);
1145
1146 ath12k_dp_rx_reo_cmd_list_cleanup(ab);
1147
1148 for (i = 0; i < ab->hw_params->max_tx_ring; i++) {
1149 kfree(dp->tx_ring[i].tx_status);
1150 dp->tx_ring[i].tx_status = NULL;
1151 }
1152
1153 ath12k_dp_rx_free(ab);
1154 /* Deinit any SOC level resource */
1155 }
1156
ath12k_dp_cc_cookie_gen(u16 ppt_idx,u16 spt_idx)1157 static u32 ath12k_dp_cc_cookie_gen(u16 ppt_idx, u16 spt_idx)
1158 {
1159 return (u32)ppt_idx << ATH12K_CC_PPT_SHIFT | spt_idx;
1160 }
1161
ath12k_dp_cc_get_desc_addr_ptr(struct ath12k_dp * dp,u16 ppt_idx,u16 spt_idx)1162 static void *ath12k_dp_cc_get_desc_addr_ptr(struct ath12k_dp *dp,
1163 u16 ppt_idx, u16 spt_idx)
1164 {
1165 return dp->spt_info[ppt_idx].vaddr + spt_idx;
1166 }
1167
ath12k_dp_get_rx_desc(struct ath12k_dp * dp,u32 cookie)1168 struct ath12k_rx_desc_info *ath12k_dp_get_rx_desc(struct ath12k_dp *dp,
1169 u32 cookie)
1170 {
1171 struct ath12k_rx_desc_info **desc_addr_ptr;
1172 u16 start_ppt_idx, end_ppt_idx, ppt_idx, spt_idx;
1173
1174 ppt_idx = u32_get_bits(cookie, ATH12K_DP_CC_COOKIE_PPT);
1175 spt_idx = u32_get_bits(cookie, ATH12K_DP_CC_COOKIE_SPT);
1176
1177 start_ppt_idx = dp->rx_ppt_base + ATH12K_RX_SPT_PAGE_OFFSET(dp->ab);
1178 end_ppt_idx = start_ppt_idx + ATH12K_NUM_RX_SPT_PAGES(dp->ab);
1179
1180 if (ppt_idx < start_ppt_idx ||
1181 ppt_idx >= end_ppt_idx ||
1182 spt_idx > ATH12K_MAX_SPT_ENTRIES)
1183 return NULL;
1184
1185 ppt_idx = ppt_idx - dp->rx_ppt_base;
1186 desc_addr_ptr = ath12k_dp_cc_get_desc_addr_ptr(dp, ppt_idx, spt_idx);
1187
1188 return *desc_addr_ptr;
1189 }
1190 EXPORT_SYMBOL(ath12k_dp_get_rx_desc);
1191
ath12k_dp_get_tx_desc(struct ath12k_dp * dp,u32 cookie)1192 struct ath12k_tx_desc_info *ath12k_dp_get_tx_desc(struct ath12k_dp *dp,
1193 u32 cookie)
1194 {
1195 struct ath12k_tx_desc_info **desc_addr_ptr;
1196 u16 start_ppt_idx, end_ppt_idx, ppt_idx, spt_idx;
1197
1198 ppt_idx = u32_get_bits(cookie, ATH12K_DP_CC_COOKIE_PPT);
1199 spt_idx = u32_get_bits(cookie, ATH12K_DP_CC_COOKIE_SPT);
1200
1201 start_ppt_idx = ATH12K_TX_SPT_PAGE_OFFSET;
1202 end_ppt_idx = start_ppt_idx +
1203 (ATH12K_TX_SPT_PAGES_PER_POOL(dp->ab) * ATH12K_HW_MAX_QUEUES);
1204
1205 if (ppt_idx < start_ppt_idx ||
1206 ppt_idx >= end_ppt_idx ||
1207 spt_idx > ATH12K_MAX_SPT_ENTRIES)
1208 return NULL;
1209
1210 desc_addr_ptr = ath12k_dp_cc_get_desc_addr_ptr(dp, ppt_idx, spt_idx);
1211
1212 return *desc_addr_ptr;
1213 }
1214 EXPORT_SYMBOL(ath12k_dp_get_tx_desc);
1215
ath12k_dp_cc_desc_init(struct ath12k_base * ab)1216 static int ath12k_dp_cc_desc_init(struct ath12k_base *ab)
1217 {
1218 struct ath12k_dp *dp = ath12k_ab_to_dp(ab);
1219 struct ath12k_rx_desc_info *rx_descs, **rx_desc_addr;
1220 struct ath12k_tx_desc_info *tx_descs, **tx_desc_addr;
1221 u32 num_rx_spt_pages = ATH12K_NUM_RX_SPT_PAGES(ab);
1222 u32 i, j, pool_id, tx_spt_page;
1223 u32 ppt_idx, cookie_ppt_idx;
1224
1225 spin_lock_bh(&dp->rx_desc_lock);
1226
1227 dp->rxbaddr = kzalloc_objs(struct ath12k_rx_desc_info *,
1228 num_rx_spt_pages, GFP_ATOMIC);
1229
1230 if (!dp->rxbaddr) {
1231 spin_unlock_bh(&dp->rx_desc_lock);
1232 return -ENOMEM;
1233 }
1234
1235 /* First ATH12K_NUM_RX_SPT_PAGES(ab) of allocated SPT pages are used for
1236 * RX
1237 */
1238 for (i = 0; i < num_rx_spt_pages; i++) {
1239 rx_descs = kzalloc_objs(*rx_descs, ATH12K_MAX_SPT_ENTRIES,
1240 GFP_ATOMIC);
1241
1242 if (!rx_descs) {
1243 spin_unlock_bh(&dp->rx_desc_lock);
1244 return -ENOMEM;
1245 }
1246
1247 ppt_idx = ATH12K_RX_SPT_PAGE_OFFSET(ab) + i;
1248 cookie_ppt_idx = dp->rx_ppt_base + ppt_idx;
1249 dp->rxbaddr[i] = &rx_descs[0];
1250
1251 for (j = 0; j < ATH12K_MAX_SPT_ENTRIES; j++) {
1252 rx_descs[j].cookie = ath12k_dp_cc_cookie_gen(cookie_ppt_idx, j);
1253 rx_descs[j].magic = ATH12K_DP_RX_DESC_MAGIC;
1254 rx_descs[j].device_id = ab->device_id;
1255 list_add_tail(&rx_descs[j].list, &dp->rx_desc_free_list);
1256
1257 /* Update descriptor VA in SPT */
1258 rx_desc_addr = ath12k_dp_cc_get_desc_addr_ptr(dp, ppt_idx, j);
1259 *rx_desc_addr = &rx_descs[j];
1260 }
1261 }
1262
1263 spin_unlock_bh(&dp->rx_desc_lock);
1264
1265 dp->txbaddr = kzalloc_objs(struct ath12k_tx_desc_info *,
1266 ATH12K_NUM_TX_SPT_PAGES(ab), GFP_ATOMIC);
1267
1268 if (!dp->txbaddr)
1269 return -ENOMEM;
1270
1271 for (pool_id = 0; pool_id < ATH12K_HW_MAX_QUEUES; pool_id++) {
1272 spin_lock_bh(&dp->tx_desc_lock[pool_id]);
1273 for (i = 0; i < ATH12K_TX_SPT_PAGES_PER_POOL(ab); i++) {
1274 tx_descs = kzalloc_objs(*tx_descs,
1275 ATH12K_MAX_SPT_ENTRIES,
1276 GFP_ATOMIC);
1277
1278 if (!tx_descs) {
1279 spin_unlock_bh(&dp->tx_desc_lock[pool_id]);
1280 /* Caller takes care of TX pending and RX desc cleanup */
1281 return -ENOMEM;
1282 }
1283
1284 tx_spt_page = i + pool_id *
1285 ATH12K_TX_SPT_PAGES_PER_POOL(ab);
1286 ppt_idx = ATH12K_TX_SPT_PAGE_OFFSET + tx_spt_page;
1287
1288 dp->txbaddr[tx_spt_page] = &tx_descs[0];
1289
1290 for (j = 0; j < ATH12K_MAX_SPT_ENTRIES; j++) {
1291 tx_descs[j].desc_id = ath12k_dp_cc_cookie_gen(ppt_idx, j);
1292 tx_descs[j].pool_id = pool_id;
1293 list_add_tail(&tx_descs[j].list,
1294 &dp->tx_desc_free_list[pool_id]);
1295
1296 /* Update descriptor VA in SPT */
1297 tx_desc_addr =
1298 ath12k_dp_cc_get_desc_addr_ptr(dp, ppt_idx, j);
1299 *tx_desc_addr = &tx_descs[j];
1300 }
1301 }
1302 spin_unlock_bh(&dp->tx_desc_lock[pool_id]);
1303 }
1304 return 0;
1305 }
1306
ath12k_dp_cmem_init(struct ath12k_base * ab,struct ath12k_dp * dp,enum ath12k_dp_desc_type type)1307 static int ath12k_dp_cmem_init(struct ath12k_base *ab,
1308 struct ath12k_dp *dp,
1309 enum ath12k_dp_desc_type type)
1310 {
1311 u32 cmem_base;
1312 int i, start, end;
1313
1314 cmem_base = ab->qmi.dev_mem[ATH12K_QMI_DEVMEM_CMEM_INDEX].start;
1315
1316 switch (type) {
1317 case ATH12K_DP_TX_DESC:
1318 start = ATH12K_TX_SPT_PAGE_OFFSET;
1319 end = start + ATH12K_NUM_TX_SPT_PAGES(ab);
1320 break;
1321 case ATH12K_DP_RX_DESC:
1322 cmem_base += ATH12K_PPT_ADDR_OFFSET(dp->rx_ppt_base);
1323 start = ATH12K_RX_SPT_PAGE_OFFSET(ab);
1324 end = start + ATH12K_NUM_RX_SPT_PAGES(ab);
1325 break;
1326 default:
1327 ath12k_err(ab, "invalid descriptor type %d in cmem init\n", type);
1328 return -EINVAL;
1329 }
1330
1331 /* Write to PPT in CMEM */
1332 for (i = start; i < end; i++)
1333 ath12k_hif_write32(ab, cmem_base + ATH12K_PPT_ADDR_OFFSET(i),
1334 dp->spt_info[i].paddr >> ATH12K_SPT_4K_ALIGN_OFFSET);
1335
1336 return 0;
1337 }
1338
ath12k_dp_partner_cc_init(struct ath12k_base * ab)1339 void ath12k_dp_partner_cc_init(struct ath12k_base *ab)
1340 {
1341 struct ath12k_hw_group *ag = ab->ag;
1342 int i;
1343
1344 for (i = 0; i < ag->num_devices; i++) {
1345 if (ag->ab[i] == ab)
1346 continue;
1347
1348 ath12k_dp_cmem_init(ab, ath12k_ab_to_dp(ag->ab[i]), ATH12K_DP_RX_DESC);
1349 }
1350 }
1351
ath12k_dp_get_num_spt_pages(struct ath12k_base * ab)1352 static u32 ath12k_dp_get_num_spt_pages(struct ath12k_base *ab)
1353 {
1354 return ATH12K_NUM_RX_SPT_PAGES(ab) + ATH12K_NUM_TX_SPT_PAGES(ab);
1355 }
1356
ath12k_dp_cc_init(struct ath12k_base * ab)1357 static int ath12k_dp_cc_init(struct ath12k_base *ab)
1358 {
1359 struct ath12k_dp *dp = ath12k_ab_to_dp(ab);
1360 int i, ret = 0;
1361
1362 INIT_LIST_HEAD(&dp->rx_desc_free_list);
1363 spin_lock_init(&dp->rx_desc_lock);
1364
1365 for (i = 0; i < ATH12K_HW_MAX_QUEUES; i++) {
1366 INIT_LIST_HEAD(&dp->tx_desc_free_list[i]);
1367 INIT_LIST_HEAD(&dp->tx_desc_used_list[i]);
1368 spin_lock_init(&dp->tx_desc_lock[i]);
1369 }
1370
1371 dp->num_spt_pages = ath12k_dp_get_num_spt_pages(ab);
1372 if (dp->num_spt_pages > ATH12K_MAX_PPT_ENTRIES)
1373 dp->num_spt_pages = ATH12K_MAX_PPT_ENTRIES;
1374
1375 dp->spt_info = kzalloc_objs(struct ath12k_spt_info, dp->num_spt_pages);
1376
1377 if (!dp->spt_info) {
1378 ath12k_warn(ab, "SPT page allocation failure");
1379 return -ENOMEM;
1380 }
1381
1382 dp->rx_ppt_base = ab->device_id * ATH12K_NUM_RX_SPT_PAGES(ab);
1383
1384 for (i = 0; i < dp->num_spt_pages; i++) {
1385 dp->spt_info[i].vaddr = dma_alloc_coherent(ab->dev,
1386 ATH12K_PAGE_SIZE,
1387 &dp->spt_info[i].paddr,
1388 GFP_KERNEL);
1389
1390 if (!dp->spt_info[i].vaddr) {
1391 ret = -ENOMEM;
1392 goto free;
1393 }
1394
1395 if (dp->spt_info[i].paddr & ATH12K_SPT_4K_ALIGN_CHECK) {
1396 ath12k_warn(ab, "SPT allocated memory is not 4K aligned");
1397 ret = -EINVAL;
1398 goto free;
1399 }
1400 }
1401
1402 ret = ath12k_dp_cmem_init(ab, dp, ATH12K_DP_TX_DESC);
1403 if (ret) {
1404 ath12k_warn(ab, "HW CC Tx cmem init failed %d", ret);
1405 goto free;
1406 }
1407
1408 ret = ath12k_dp_cmem_init(ab, dp, ATH12K_DP_RX_DESC);
1409 if (ret) {
1410 ath12k_warn(ab, "HW CC Rx cmem init failed %d", ret);
1411 goto free;
1412 }
1413
1414 ret = ath12k_dp_cc_desc_init(ab);
1415 if (ret) {
1416 ath12k_warn(ab, "HW CC desc init failed %d", ret);
1417 goto free;
1418 }
1419
1420 return 0;
1421 free:
1422 ath12k_dp_cc_cleanup(ab);
1423 return ret;
1424 }
1425
ath12k_dp_alloc_reoq_lut(struct ath12k_base * ab,struct ath12k_reo_q_addr_lut * lut)1426 static int ath12k_dp_alloc_reoq_lut(struct ath12k_base *ab,
1427 struct ath12k_reo_q_addr_lut *lut)
1428 {
1429 lut->size = DP_REOQ_LUT_SIZE + HAL_REO_QLUT_ADDR_ALIGN - 1;
1430 lut->vaddr_unaligned = dma_alloc_coherent(ab->dev, lut->size,
1431 &lut->paddr_unaligned,
1432 GFP_KERNEL | __GFP_ZERO);
1433 if (!lut->vaddr_unaligned)
1434 return -ENOMEM;
1435
1436 lut->vaddr = PTR_ALIGN(lut->vaddr_unaligned, HAL_REO_QLUT_ADDR_ALIGN);
1437 lut->paddr = lut->paddr_unaligned +
1438 ((unsigned long)lut->vaddr - (unsigned long)lut->vaddr_unaligned);
1439 return 0;
1440 }
1441
ath12k_dp_reoq_lut_setup(struct ath12k_base * ab)1442 static int ath12k_dp_reoq_lut_setup(struct ath12k_base *ab)
1443 {
1444 struct ath12k_dp *dp = ath12k_ab_to_dp(ab);
1445 int ret;
1446
1447 if (!ab->hw_params->reoq_lut_support)
1448 return 0;
1449
1450 ret = ath12k_dp_alloc_reoq_lut(ab, &dp->reoq_lut);
1451 if (ret) {
1452 ath12k_warn(ab, "failed to allocate memory for reoq table");
1453 return ret;
1454 }
1455
1456 ret = ath12k_dp_alloc_reoq_lut(ab, &dp->ml_reoq_lut);
1457 if (ret) {
1458 ath12k_warn(ab, "failed to allocate memory for ML reoq table");
1459 dma_free_coherent(ab->dev, dp->reoq_lut.size,
1460 dp->reoq_lut.vaddr_unaligned,
1461 dp->reoq_lut.paddr_unaligned);
1462 dp->reoq_lut.vaddr_unaligned = NULL;
1463 return ret;
1464 }
1465
1466 /* Bits in the register have address [39:8] LUT base address to be
1467 * allocated such that LSBs are assumed to be zero. Also, current
1468 * design supports paddr up to 4 GB max hence it fits in 32 bit
1469 * register only
1470 */
1471
1472 ath12k_hal_write_reoq_lut_addr(ab, dp->reoq_lut.paddr >> 8);
1473 ath12k_hal_write_ml_reoq_lut_addr(ab, dp->ml_reoq_lut.paddr >> 8);
1474 ath12k_hal_reoq_lut_addr_read_enable(ab);
1475 ath12k_hal_reoq_lut_set_max_peerid(ab);
1476
1477 return 0;
1478 }
1479
ath12k_dp_setup(struct ath12k_base * ab)1480 static int ath12k_dp_setup(struct ath12k_base *ab)
1481 {
1482 struct ath12k_dp *dp;
1483 struct hal_srng *srng = NULL;
1484 size_t size = 0;
1485 u32 n_link_desc = 0;
1486 int ret;
1487 int i;
1488
1489 dp = ath12k_ab_to_dp(ab);
1490 dp->ab = ab;
1491
1492 INIT_LIST_HEAD(&dp->reo_cmd_list);
1493 INIT_LIST_HEAD(&dp->reo_cmd_cache_flush_list);
1494 INIT_LIST_HEAD(&dp->reo_cmd_update_rx_queue_list);
1495 spin_lock_init(&dp->reo_cmd_lock);
1496 spin_lock_init(&dp->reo_rxq_flush_lock);
1497
1498 spin_lock_init(&dp->dp_lock);
1499 INIT_LIST_HEAD(&dp->peers);
1500
1501 mutex_init(&dp->link_peer_rhash_tbl_lock);
1502
1503 dp->reo_cmd_cache_flush_count = 0;
1504 dp->idle_link_rbm =
1505 ath12k_hal_get_idle_link_rbm(&ab->hal, ab->device_id);
1506
1507 ret = ath12k_dp_link_peer_rhash_tbl_init(dp);
1508 if (ret) {
1509 ath12k_warn(ab, "failed to init link_peer rhash table: %d\n", ret);
1510 return ret;
1511 }
1512
1513 ret = ath12k_wbm_idle_ring_setup(ab, &n_link_desc);
1514 if (ret) {
1515 ath12k_warn(ab, "failed to setup wbm_idle_ring: %d\n", ret);
1516 goto rhash_destroy;
1517 }
1518
1519 srng = &ab->hal.srng_list[dp->wbm_idle_ring.ring_id];
1520
1521 ret = ath12k_dp_link_desc_setup(ab, dp->link_desc_banks,
1522 HAL_WBM_IDLE_LINK, srng, n_link_desc);
1523 if (ret) {
1524 ath12k_warn(ab, "failed to setup link desc: %d\n", ret);
1525 goto rhash_destroy;
1526 }
1527
1528 ret = ath12k_dp_cc_init(ab);
1529
1530 if (ret) {
1531 ath12k_warn(ab, "failed to setup cookie converter %d\n", ret);
1532 goto fail_link_desc_cleanup;
1533 }
1534 ret = ath12k_dp_init_bank_profiles(ab);
1535 if (ret) {
1536 ath12k_warn(ab, "failed to setup bank profiles %d\n", ret);
1537 goto fail_hw_cc_cleanup;
1538 }
1539
1540 ret = ath12k_dp_srng_common_setup(ab);
1541 if (ret)
1542 goto fail_dp_bank_profiles_cleanup;
1543
1544 size = ab->hal.hal_wbm_release_ring_tx_size *
1545 DP_TX_COMP_RING_SIZE(ab);
1546
1547 ret = ath12k_dp_reoq_lut_setup(ab);
1548 if (ret) {
1549 ath12k_warn(ab, "failed to setup reoq table %d\n", ret);
1550 goto fail_cmn_srng_cleanup;
1551 }
1552
1553 for (i = 0; i < ab->hw_params->max_tx_ring; i++) {
1554 dp->tx_ring[i].tcl_data_ring_id = i;
1555
1556 dp->tx_ring[i].tx_status_head = 0;
1557 dp->tx_ring[i].tx_status_tail = DP_TX_COMP_RING_SIZE(ab) - 1;
1558 dp->tx_ring[i].tx_status = kmalloc(size, GFP_KERNEL);
1559 if (!dp->tx_ring[i].tx_status) {
1560 ret = -ENOMEM;
1561 /* FIXME: The allocated tx status is not freed
1562 * properly here
1563 */
1564 goto fail_cmn_reoq_cleanup;
1565 }
1566 }
1567
1568 for (i = 0; i < HAL_DSCP_TID_MAP_TBL_NUM_ENTRIES_MAX; i++)
1569 ath12k_hal_tx_set_dscp_tid_map(ab, i);
1570
1571 ret = ath12k_dp_rx_alloc(ab);
1572 if (ret)
1573 goto fail_dp_rx_free;
1574
1575 /* Init any SOC level resource for DP */
1576
1577 return 0;
1578
1579 fail_dp_rx_free:
1580 ath12k_dp_rx_free(ab);
1581
1582 fail_cmn_reoq_cleanup:
1583 ath12k_dp_reoq_lut_cleanup(ab);
1584
1585 fail_cmn_srng_cleanup:
1586 ath12k_dp_srng_common_cleanup(ab);
1587
1588 fail_dp_bank_profiles_cleanup:
1589 ath12k_dp_deinit_bank_profiles(ab);
1590
1591 fail_hw_cc_cleanup:
1592 ath12k_dp_cc_cleanup(ab);
1593
1594 fail_link_desc_cleanup:
1595 ath12k_dp_link_desc_cleanup(ab, dp->link_desc_banks,
1596 HAL_WBM_IDLE_LINK, &dp->wbm_idle_ring);
1597 rhash_destroy:
1598 ath12k_dp_link_peer_rhash_tbl_destroy(dp);
1599
1600 return ret;
1601 }
1602
ath12k_dp_cmn_device_deinit(struct ath12k_dp * dp)1603 void ath12k_dp_cmn_device_deinit(struct ath12k_dp *dp)
1604 {
1605 ath12k_dp_cleanup(dp->ab);
1606 }
1607
ath12k_dp_cmn_device_init(struct ath12k_dp * dp)1608 int ath12k_dp_cmn_device_init(struct ath12k_dp *dp)
1609 {
1610 int ret;
1611
1612 ret = ath12k_dp_setup(dp->ab);
1613 if (ret)
1614 return ret;
1615
1616 return 0;
1617 }
1618
ath12k_dp_cmn_hw_group_unassign(struct ath12k_dp * dp,struct ath12k_hw_group * ag)1619 void ath12k_dp_cmn_hw_group_unassign(struct ath12k_dp *dp,
1620 struct ath12k_hw_group *ag)
1621 {
1622 struct ath12k_dp_hw_group *dp_hw_grp = &ag->dp_hw_grp;
1623
1624 lockdep_assert_held(&ag->mutex);
1625
1626 dp_hw_grp->dp[dp->device_id] = NULL;
1627
1628 dp->ag = NULL;
1629 dp->device_id = ATH12K_INVALID_DEVICE_ID;
1630 }
1631
ath12k_dp_cmn_hw_group_assign(struct ath12k_dp * dp,struct ath12k_hw_group * ag)1632 void ath12k_dp_cmn_hw_group_assign(struct ath12k_dp *dp,
1633 struct ath12k_hw_group *ag)
1634 {
1635 struct ath12k_base *ab = dp->ab;
1636 struct ath12k_dp_hw_group *dp_hw_grp = &ag->dp_hw_grp;
1637
1638 dp->ag = ag;
1639 dp->device_id = ab->device_id;
1640 dp_hw_grp->dp[dp->device_id] = dp;
1641 }
1642