1 // SPDX-License-Identifier: BSD-3-Clause-Clear 2 /* 3 * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved. 4 * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. 5 */ 6 7 #include <crypto/hash.h> 8 #include "core.h" 9 #include "dp_tx.h" 10 #include "hif.h" 11 #include "hal.h" 12 #include "debug.h" 13 #include "peer.h" 14 #include "dp_cmn.h" 15 16 enum ath12k_dp_desc_type { 17 ATH12K_DP_TX_DESC, 18 ATH12K_DP_RX_DESC, 19 }; 20 21 void ath12k_dp_peer_cleanup(struct ath12k *ar, int vdev_id, const u8 *addr) 22 { 23 struct ath12k_base *ab = ar->ab; 24 struct ath12k_dp_link_peer *peer; 25 struct ath12k_dp *dp = ath12k_ab_to_dp(ab); 26 27 /* TODO: Any other peer specific DP cleanup */ 28 29 spin_lock_bh(&dp->dp_lock); 30 peer = ath12k_dp_link_peer_find_by_vdev_and_addr(dp, vdev_id, addr); 31 if (!peer || !peer->dp_peer) { 32 ath12k_warn(ab, "failed to lookup peer %pM on vdev %d\n", 33 addr, vdev_id); 34 spin_unlock_bh(&dp->dp_lock); 35 return; 36 } 37 38 if (!peer->primary_link) { 39 spin_unlock_bh(&dp->dp_lock); 40 return; 41 } 42 43 ath12k_dp_rx_peer_tid_cleanup(ar, peer); 44 crypto_free_shash(peer->dp_peer->tfm_mmic); 45 peer->dp_peer->dp_setup_done = false; 46 spin_unlock_bh(&dp->dp_lock); 47 } 48 49 int ath12k_dp_peer_setup(struct ath12k *ar, int vdev_id, const u8 *addr) 50 { 51 struct ath12k_base *ab = ar->ab; 52 struct ath12k_dp_link_peer *peer; 53 u32 reo_dest; 54 int ret = 0, tid; 55 struct ath12k_dp *dp = ath12k_ab_to_dp(ab); 56 57 /* NOTE: reo_dest ring id starts from 1 unlike mac_id which starts from 0 */ 58 reo_dest = ar->dp.mac_id + 1; 59 ret = ath12k_wmi_set_peer_param(ar, addr, vdev_id, 60 WMI_PEER_SET_DEFAULT_ROUTING, 61 DP_RX_HASH_ENABLE | (reo_dest << 1)); 62 63 if (ret) { 64 ath12k_warn(ab, "failed to set default routing %d peer :%pM vdev_id :%d\n", 65 ret, addr, vdev_id); 66 return ret; 67 } 68 69 for (tid = 0; tid <= IEEE80211_NUM_TIDS; tid++) { 70 ret = ath12k_dp_rx_peer_tid_setup(ar, addr, vdev_id, tid, 1, 0, 71 HAL_PN_TYPE_NONE); 72 if (ret) { 73 ath12k_warn(ab, "failed to setup rxd tid queue for tid %d: %d\n", 74 tid, ret); 75 goto peer_clean; 76 } 77 } 78 79 ret = ath12k_dp_rx_peer_frag_setup(ar, addr, vdev_id); 80 if (ret) { 81 ath12k_warn(ab, "failed to setup rx defrag context\n"); 82 goto peer_clean; 83 } 84 85 /* TODO: Setup other peer specific resource used in data path */ 86 87 return 0; 88 89 peer_clean: 90 spin_lock_bh(&dp->dp_lock); 91 92 peer = ath12k_dp_link_peer_find_by_vdev_and_addr(dp, vdev_id, addr); 93 if (!peer) { 94 ath12k_warn(ab, "failed to find the peer to del rx tid\n"); 95 spin_unlock_bh(&dp->dp_lock); 96 return -ENOENT; 97 } 98 99 for (tid--; tid >= 0; tid--) 100 ath12k_dp_arch_rx_peer_tid_delete(dp, peer, tid); 101 102 spin_unlock_bh(&dp->dp_lock); 103 104 return ret; 105 } 106 107 void ath12k_dp_srng_cleanup(struct ath12k_base *ab, struct dp_srng *ring) 108 { 109 if (!ring->vaddr_unaligned) 110 return; 111 112 dma_free_coherent(ab->dev, ring->size, ring->vaddr_unaligned, 113 ring->paddr_unaligned); 114 115 ring->vaddr_unaligned = NULL; 116 } 117 118 static int ath12k_dp_srng_find_ring_in_mask(int ring_num, const u8 *grp_mask) 119 { 120 int ext_group_num; 121 u8 mask = 1 << ring_num; 122 123 for (ext_group_num = 0; ext_group_num < ATH12K_EXT_IRQ_GRP_NUM_MAX; 124 ext_group_num++) { 125 if (mask & grp_mask[ext_group_num]) 126 return ext_group_num; 127 } 128 129 return -ENOENT; 130 } 131 132 static int ath12k_dp_srng_calculate_msi_group(struct ath12k_base *ab, 133 enum hal_ring_type type, int ring_num) 134 { 135 const struct ath12k_hal_tcl_to_wbm_rbm_map *map; 136 const u8 *grp_mask; 137 int i; 138 139 switch (type) { 140 case HAL_WBM2SW_RELEASE: 141 if (ring_num == HAL_WBM2SW_REL_ERR_RING_NUM) { 142 grp_mask = &ab->hw_params->ring_mask->rx_wbm_rel[0]; 143 ring_num = 0; 144 } else { 145 map = ab->hal.tcl_to_wbm_rbm_map; 146 for (i = 0; i < ab->hw_params->max_tx_ring; i++) { 147 if (ring_num == map[i].wbm_ring_num) { 148 ring_num = i; 149 break; 150 } 151 } 152 153 grp_mask = &ab->hw_params->ring_mask->tx[0]; 154 } 155 break; 156 case HAL_REO_EXCEPTION: 157 grp_mask = &ab->hw_params->ring_mask->rx_err[0]; 158 break; 159 case HAL_REO_DST: 160 grp_mask = &ab->hw_params->ring_mask->rx[0]; 161 break; 162 case HAL_REO_STATUS: 163 grp_mask = &ab->hw_params->ring_mask->reo_status[0]; 164 break; 165 case HAL_RXDMA_MONITOR_STATUS: 166 grp_mask = &ab->hw_params->ring_mask->rx_mon_status[0]; 167 break; 168 case HAL_RXDMA_MONITOR_DST: 169 grp_mask = &ab->hw_params->ring_mask->rx_mon_dest[0]; 170 break; 171 case HAL_TX_MONITOR_DST: 172 grp_mask = &ab->hw_params->ring_mask->tx_mon_dest[0]; 173 break; 174 case HAL_RXDMA_BUF: 175 grp_mask = &ab->hw_params->ring_mask->host2rxdma[0]; 176 break; 177 case HAL_RXDMA_MONITOR_BUF: 178 case HAL_TCL_DATA: 179 case HAL_TCL_CMD: 180 case HAL_REO_CMD: 181 case HAL_SW2WBM_RELEASE: 182 case HAL_WBM_IDLE_LINK: 183 case HAL_TCL_STATUS: 184 case HAL_REO_REINJECT: 185 case HAL_CE_SRC: 186 case HAL_CE_DST: 187 case HAL_CE_DST_STATUS: 188 default: 189 return -ENOENT; 190 } 191 192 return ath12k_dp_srng_find_ring_in_mask(ring_num, grp_mask); 193 } 194 195 static void ath12k_dp_srng_msi_setup(struct ath12k_base *ab, 196 struct hal_srng_params *ring_params, 197 enum hal_ring_type type, int ring_num) 198 { 199 int msi_group_number, msi_data_count; 200 u32 msi_data_start, msi_irq_start, addr_lo, addr_hi; 201 int ret; 202 203 ret = ath12k_hif_get_user_msi_vector(ab, "DP", 204 &msi_data_count, &msi_data_start, 205 &msi_irq_start); 206 if (ret) 207 return; 208 209 msi_group_number = ath12k_dp_srng_calculate_msi_group(ab, type, 210 ring_num); 211 if (msi_group_number < 0) { 212 ath12k_dbg(ab, ATH12K_DBG_PCI, 213 "ring not part of an ext_group; ring_type: %d,ring_num %d", 214 type, ring_num); 215 ring_params->msi_addr = 0; 216 ring_params->msi_data = 0; 217 return; 218 } 219 220 if (msi_group_number > msi_data_count) { 221 ath12k_dbg(ab, ATH12K_DBG_PCI, 222 "multiple msi_groups share one msi, msi_group_num %d", 223 msi_group_number); 224 } 225 226 ath12k_hif_get_msi_address(ab, &addr_lo, &addr_hi); 227 228 ring_params->msi_addr = addr_lo; 229 ring_params->msi_addr |= (dma_addr_t)(((uint64_t)addr_hi) << 32); 230 ring_params->msi_data = (msi_group_number % msi_data_count) 231 + msi_data_start; 232 ring_params->flags |= HAL_SRNG_FLAGS_MSI_INTR; 233 } 234 235 int ath12k_dp_srng_setup(struct ath12k_base *ab, struct dp_srng *ring, 236 enum hal_ring_type type, int ring_num, 237 int mac_id, int num_entries) 238 { 239 struct hal_srng_params params = {}; 240 int entry_sz = ath12k_hal_srng_get_entrysize(ab, type); 241 int max_entries = ath12k_hal_srng_get_max_entries(ab, type); 242 int ret; 243 244 if (max_entries < 0 || entry_sz < 0) 245 return -EINVAL; 246 247 if (num_entries > max_entries) 248 num_entries = max_entries; 249 250 ring->size = (num_entries * entry_sz) + HAL_RING_BASE_ALIGN - 1; 251 ring->vaddr_unaligned = dma_alloc_coherent(ab->dev, ring->size, 252 &ring->paddr_unaligned, 253 GFP_KERNEL); 254 if (!ring->vaddr_unaligned) 255 return -ENOMEM; 256 257 ring->vaddr = PTR_ALIGN(ring->vaddr_unaligned, HAL_RING_BASE_ALIGN); 258 ring->paddr = ring->paddr_unaligned + ((unsigned long)ring->vaddr - 259 (unsigned long)ring->vaddr_unaligned); 260 261 params.ring_base_vaddr = ring->vaddr; 262 params.ring_base_paddr = ring->paddr; 263 params.num_entries = num_entries; 264 ath12k_dp_srng_msi_setup(ab, ¶ms, type, ring_num + mac_id); 265 266 switch (type) { 267 case HAL_REO_DST: 268 params.intr_batch_cntr_thres_entries = 269 HAL_SRNG_INT_BATCH_THRESHOLD_RX; 270 params.intr_timer_thres_us = HAL_SRNG_INT_TIMER_THRESHOLD_RX; 271 break; 272 case HAL_RXDMA_BUF: 273 case HAL_RXDMA_MONITOR_BUF: 274 params.low_threshold = num_entries >> 3; 275 params.flags |= HAL_SRNG_FLAGS_LOW_THRESH_INTR_EN; 276 params.intr_batch_cntr_thres_entries = 0; 277 params.intr_timer_thres_us = HAL_SRNG_INT_TIMER_THRESHOLD_RX; 278 break; 279 case HAL_RXDMA_MONITOR_STATUS: 280 params.low_threshold = num_entries >> 3; 281 params.flags |= HAL_SRNG_FLAGS_LOW_THRESH_INTR_EN; 282 params.intr_batch_cntr_thres_entries = 1; 283 params.intr_timer_thres_us = HAL_SRNG_INT_TIMER_THRESHOLD_RX; 284 break; 285 case HAL_TX_MONITOR_DST: 286 params.low_threshold = DP_TX_MONITOR_BUF_SIZE_MAX >> 3; 287 params.flags |= HAL_SRNG_FLAGS_LOW_THRESH_INTR_EN; 288 params.intr_batch_cntr_thres_entries = 0; 289 params.intr_timer_thres_us = HAL_SRNG_INT_TIMER_THRESHOLD_RX; 290 break; 291 case HAL_WBM2SW_RELEASE: 292 if (ab->hw_params->hw_ops->dp_srng_is_tx_comp_ring(ring_num)) { 293 params.intr_batch_cntr_thres_entries = 294 HAL_SRNG_INT_BATCH_THRESHOLD_TX; 295 params.intr_timer_thres_us = 296 HAL_SRNG_INT_TIMER_THRESHOLD_TX; 297 break; 298 } 299 /* follow through when ring_num != HAL_WBM2SW_REL_ERR_RING_NUM */ 300 fallthrough; 301 case HAL_REO_EXCEPTION: 302 case HAL_REO_REINJECT: 303 case HAL_REO_CMD: 304 case HAL_REO_STATUS: 305 case HAL_TCL_DATA: 306 case HAL_TCL_CMD: 307 case HAL_TCL_STATUS: 308 case HAL_WBM_IDLE_LINK: 309 case HAL_SW2WBM_RELEASE: 310 case HAL_RXDMA_DST: 311 case HAL_RXDMA_MONITOR_DST: 312 case HAL_RXDMA_MONITOR_DESC: 313 params.intr_batch_cntr_thres_entries = 314 HAL_SRNG_INT_BATCH_THRESHOLD_OTHER; 315 params.intr_timer_thres_us = HAL_SRNG_INT_TIMER_THRESHOLD_OTHER; 316 break; 317 case HAL_RXDMA_DIR_BUF: 318 break; 319 default: 320 ath12k_warn(ab, "Not a valid ring type in dp :%d\n", type); 321 return -EINVAL; 322 } 323 324 ret = ath12k_hal_srng_setup(ab, type, ring_num, mac_id, ¶ms); 325 if (ret < 0) { 326 ath12k_warn(ab, "failed to setup srng: %d ring_id %d\n", 327 ret, ring_num); 328 return ret; 329 } 330 331 ring->ring_id = ret; 332 333 return 0; 334 } 335 336 static int ath12k_dp_tx_get_bank_profile(struct ath12k_base *ab, 337 struct ath12k_link_vif *arvif, 338 struct ath12k_dp *dp) 339 { 340 int bank_id = DP_INVALID_BANK_ID; 341 int i; 342 u32 bank_config; 343 bool configure_register = false; 344 345 /* convert vdev params into hal_tx_bank_config */ 346 bank_config = ath12k_dp_arch_tx_get_vdev_bank_config(dp, arvif); 347 348 spin_lock_bh(&dp->tx_bank_lock); 349 /* TODO: implement using idr kernel framework*/ 350 for (i = 0; i < dp->num_bank_profiles; i++) { 351 if (dp->bank_profiles[i].is_configured && 352 (dp->bank_profiles[i].bank_config ^ bank_config) == 0) { 353 bank_id = i; 354 goto inc_ref_and_return; 355 } 356 if (!dp->bank_profiles[i].is_configured || 357 !dp->bank_profiles[i].num_users) { 358 bank_id = i; 359 goto configure_and_return; 360 } 361 } 362 363 if (bank_id == DP_INVALID_BANK_ID) { 364 spin_unlock_bh(&dp->tx_bank_lock); 365 ath12k_err(ab, "unable to find TX bank!"); 366 return bank_id; 367 } 368 369 configure_and_return: 370 dp->bank_profiles[bank_id].is_configured = true; 371 dp->bank_profiles[bank_id].bank_config = bank_config; 372 configure_register = true; 373 inc_ref_and_return: 374 dp->bank_profiles[bank_id].num_users++; 375 spin_unlock_bh(&dp->tx_bank_lock); 376 377 if (configure_register) 378 ath12k_hal_tx_configure_bank_register(ab, 379 bank_config, bank_id); 380 381 ath12k_dbg(ab, ATH12K_DBG_DP_HTT, "dp_htt tcl bank_id %d input 0x%x match 0x%x num_users %u", 382 bank_id, bank_config, dp->bank_profiles[bank_id].bank_config, 383 dp->bank_profiles[bank_id].num_users); 384 385 return bank_id; 386 } 387 388 void ath12k_dp_tx_put_bank_profile(struct ath12k_dp *dp, u8 bank_id) 389 { 390 spin_lock_bh(&dp->tx_bank_lock); 391 dp->bank_profiles[bank_id].num_users--; 392 spin_unlock_bh(&dp->tx_bank_lock); 393 } 394 395 static void ath12k_dp_deinit_bank_profiles(struct ath12k_base *ab) 396 { 397 struct ath12k_dp *dp = ath12k_ab_to_dp(ab); 398 399 kfree(dp->bank_profiles); 400 dp->bank_profiles = NULL; 401 } 402 403 static int ath12k_dp_init_bank_profiles(struct ath12k_base *ab) 404 { 405 struct ath12k_dp *dp = ath12k_ab_to_dp(ab); 406 u32 num_tcl_banks = ab->hw_params->num_tcl_banks; 407 int i; 408 409 dp->num_bank_profiles = num_tcl_banks; 410 dp->bank_profiles = kmalloc_array(num_tcl_banks, 411 sizeof(struct ath12k_dp_tx_bank_profile), 412 GFP_KERNEL); 413 if (!dp->bank_profiles) 414 return -ENOMEM; 415 416 spin_lock_init(&dp->tx_bank_lock); 417 418 for (i = 0; i < num_tcl_banks; i++) { 419 dp->bank_profiles[i].is_configured = false; 420 dp->bank_profiles[i].num_users = 0; 421 } 422 423 return 0; 424 } 425 426 static void ath12k_dp_srng_common_cleanup(struct ath12k_base *ab) 427 { 428 struct ath12k_dp *dp = ath12k_ab_to_dp(ab); 429 int i; 430 431 ath12k_dp_srng_cleanup(ab, &dp->reo_status_ring); 432 ath12k_dp_srng_cleanup(ab, &dp->reo_cmd_ring); 433 ath12k_dp_srng_cleanup(ab, &dp->reo_except_ring); 434 ath12k_dp_srng_cleanup(ab, &dp->rx_rel_ring); 435 ath12k_dp_srng_cleanup(ab, &dp->reo_reinject_ring); 436 for (i = 0; i < ab->hw_params->max_tx_ring; i++) { 437 ath12k_dp_srng_cleanup(ab, &dp->tx_ring[i].tcl_comp_ring); 438 ath12k_dp_srng_cleanup(ab, &dp->tx_ring[i].tcl_data_ring); 439 } 440 ath12k_dp_srng_cleanup(ab, &dp->wbm_desc_rel_ring); 441 } 442 443 static int ath12k_dp_srng_common_setup(struct ath12k_base *ab) 444 { 445 struct ath12k_dp *dp = ath12k_ab_to_dp(ab); 446 const struct ath12k_hal_tcl_to_wbm_rbm_map *map; 447 struct hal_srng *srng; 448 int i, ret, tx_comp_ring_num; 449 u32 ring_hash_map; 450 451 ret = ath12k_dp_srng_setup(ab, &dp->wbm_desc_rel_ring, 452 HAL_SW2WBM_RELEASE, 0, 0, 453 DP_WBM_RELEASE_RING_SIZE); 454 if (ret) { 455 ath12k_warn(ab, "failed to set up wbm2sw_release ring :%d\n", 456 ret); 457 goto err; 458 } 459 460 for (i = 0; i < ab->hw_params->max_tx_ring; i++) { 461 map = ab->hal.tcl_to_wbm_rbm_map; 462 tx_comp_ring_num = map[i].wbm_ring_num; 463 464 ret = ath12k_dp_srng_setup(ab, &dp->tx_ring[i].tcl_data_ring, 465 HAL_TCL_DATA, i, 0, 466 DP_TCL_DATA_RING_SIZE); 467 if (ret) { 468 ath12k_warn(ab, "failed to set up tcl_data ring (%d) :%d\n", 469 i, ret); 470 goto err; 471 } 472 473 ret = ath12k_dp_srng_setup(ab, &dp->tx_ring[i].tcl_comp_ring, 474 HAL_WBM2SW_RELEASE, tx_comp_ring_num, 0, 475 DP_TX_COMP_RING_SIZE(ab)); 476 if (ret) { 477 ath12k_warn(ab, "failed to set up tcl_comp ring (%d) :%d\n", 478 tx_comp_ring_num, ret); 479 goto err; 480 } 481 } 482 483 ret = ath12k_dp_srng_setup(ab, &dp->reo_reinject_ring, HAL_REO_REINJECT, 484 0, 0, DP_REO_REINJECT_RING_SIZE); 485 if (ret) { 486 ath12k_warn(ab, "failed to set up reo_reinject ring :%d\n", 487 ret); 488 goto err; 489 } 490 491 ret = ath12k_dp_srng_setup(ab, &dp->rx_rel_ring, HAL_WBM2SW_RELEASE, 492 HAL_WBM2SW_REL_ERR_RING_NUM, 0, 493 DP_RX_RELEASE_RING_SIZE); 494 if (ret) { 495 ath12k_warn(ab, "failed to set up rx_rel ring :%d\n", ret); 496 goto err; 497 } 498 499 ret = ath12k_dp_srng_setup(ab, &dp->reo_except_ring, HAL_REO_EXCEPTION, 500 0, 0, DP_REO_EXCEPTION_RING_SIZE); 501 if (ret) { 502 ath12k_warn(ab, "failed to set up reo_exception ring :%d\n", 503 ret); 504 goto err; 505 } 506 507 ret = ath12k_dp_srng_setup(ab, &dp->reo_cmd_ring, HAL_REO_CMD, 508 0, 0, DP_REO_CMD_RING_SIZE); 509 if (ret) { 510 ath12k_warn(ab, "failed to set up reo_cmd ring :%d\n", ret); 511 goto err; 512 } 513 514 srng = &ab->hal.srng_list[dp->reo_cmd_ring.ring_id]; 515 ath12k_hal_reo_init_cmd_ring(ab, srng); 516 517 ret = ath12k_dp_srng_setup(ab, &dp->reo_status_ring, HAL_REO_STATUS, 518 0, 0, DP_REO_STATUS_RING_SIZE); 519 if (ret) { 520 ath12k_warn(ab, "failed to set up reo_status ring :%d\n", ret); 521 goto err; 522 } 523 524 /* When hash based routing of rx packet is enabled, 32 entries to map 525 * the hash values to the ring will be configured. Each hash entry uses 526 * four bits to map to a particular ring. The ring mapping will be 527 * 0:TCL, 1:SW1, 2:SW2, 3:SW3, 4:SW4, 5:Release, 6:FW and 7:SW5 528 * 8:SW6, 9:SW7, 10:SW8, 11:Not used. 529 */ 530 ring_hash_map = HAL_HASH_ROUTING_RING_SW1 | 531 HAL_HASH_ROUTING_RING_SW2 << 4 | 532 HAL_HASH_ROUTING_RING_SW3 << 8 | 533 HAL_HASH_ROUTING_RING_SW4 << 12 | 534 HAL_HASH_ROUTING_RING_SW1 << 16 | 535 HAL_HASH_ROUTING_RING_SW2 << 20 | 536 HAL_HASH_ROUTING_RING_SW3 << 24 | 537 HAL_HASH_ROUTING_RING_SW4 << 28; 538 539 ath12k_hal_reo_hw_setup(ab, ring_hash_map); 540 541 return 0; 542 543 err: 544 ath12k_dp_srng_common_cleanup(ab); 545 546 return ret; 547 } 548 549 static void ath12k_dp_scatter_idle_link_desc_cleanup(struct ath12k_base *ab) 550 { 551 struct ath12k_dp *dp = ath12k_ab_to_dp(ab); 552 struct hal_wbm_idle_scatter_list *slist = dp->scatter_list; 553 int i; 554 555 for (i = 0; i < DP_IDLE_SCATTER_BUFS_MAX; i++) { 556 if (!slist[i].vaddr) 557 continue; 558 559 dma_free_coherent(ab->dev, HAL_WBM_IDLE_SCATTER_BUF_SIZE_MAX, 560 slist[i].vaddr, slist[i].paddr); 561 slist[i].vaddr = NULL; 562 } 563 } 564 565 static int ath12k_dp_scatter_idle_link_desc_setup(struct ath12k_base *ab, 566 int size, 567 u32 n_link_desc_bank, 568 u32 n_link_desc, 569 u32 last_bank_sz) 570 { 571 struct ath12k_dp *dp = ath12k_ab_to_dp(ab); 572 struct dp_link_desc_bank *link_desc_banks = dp->link_desc_banks; 573 struct hal_wbm_idle_scatter_list *slist = dp->scatter_list; 574 u32 n_entries_per_buf; 575 int num_scatter_buf, scatter_idx; 576 struct hal_wbm_link_desc *scatter_buf; 577 int align_bytes, n_entries; 578 dma_addr_t paddr; 579 int rem_entries; 580 int i; 581 int ret = 0; 582 u32 end_offset, cookie; 583 enum hal_rx_buf_return_buf_manager rbm = dp->idle_link_rbm; 584 585 n_entries_per_buf = HAL_WBM_IDLE_SCATTER_BUF_SIZE / 586 ath12k_hal_srng_get_entrysize(ab, HAL_WBM_IDLE_LINK); 587 num_scatter_buf = DIV_ROUND_UP(size, HAL_WBM_IDLE_SCATTER_BUF_SIZE); 588 589 if (num_scatter_buf > DP_IDLE_SCATTER_BUFS_MAX) 590 return -EINVAL; 591 592 for (i = 0; i < num_scatter_buf; i++) { 593 slist[i].vaddr = dma_alloc_coherent(ab->dev, 594 HAL_WBM_IDLE_SCATTER_BUF_SIZE_MAX, 595 &slist[i].paddr, GFP_KERNEL); 596 if (!slist[i].vaddr) { 597 ret = -ENOMEM; 598 goto err; 599 } 600 } 601 602 scatter_idx = 0; 603 scatter_buf = slist[scatter_idx].vaddr; 604 rem_entries = n_entries_per_buf; 605 606 for (i = 0; i < n_link_desc_bank; i++) { 607 align_bytes = link_desc_banks[i].vaddr - 608 link_desc_banks[i].vaddr_unaligned; 609 n_entries = (DP_LINK_DESC_ALLOC_SIZE_THRESH - align_bytes) / 610 HAL_LINK_DESC_SIZE; 611 paddr = link_desc_banks[i].paddr; 612 while (n_entries) { 613 cookie = DP_LINK_DESC_COOKIE_SET(n_entries, i); 614 ath12k_hal_set_link_desc_addr(dp->hal, scatter_buf, cookie, 615 paddr, rbm); 616 n_entries--; 617 paddr += HAL_LINK_DESC_SIZE; 618 if (rem_entries) { 619 rem_entries--; 620 scatter_buf++; 621 continue; 622 } 623 624 rem_entries = n_entries_per_buf; 625 scatter_idx++; 626 scatter_buf = slist[scatter_idx].vaddr; 627 } 628 } 629 630 end_offset = (scatter_buf - slist[scatter_idx].vaddr) * 631 sizeof(struct hal_wbm_link_desc); 632 ath12k_hal_setup_link_idle_list(ab, slist, num_scatter_buf, 633 n_link_desc, end_offset); 634 635 return 0; 636 637 err: 638 ath12k_dp_scatter_idle_link_desc_cleanup(ab); 639 640 return ret; 641 } 642 643 static void 644 ath12k_dp_link_desc_bank_free(struct ath12k_base *ab, 645 struct dp_link_desc_bank *link_desc_banks) 646 { 647 int i; 648 649 for (i = 0; i < DP_LINK_DESC_BANKS_MAX; i++) { 650 if (link_desc_banks[i].vaddr_unaligned) { 651 dma_free_coherent(ab->dev, 652 link_desc_banks[i].size, 653 link_desc_banks[i].vaddr_unaligned, 654 link_desc_banks[i].paddr_unaligned); 655 link_desc_banks[i].vaddr_unaligned = NULL; 656 } 657 } 658 } 659 660 static int ath12k_dp_link_desc_bank_alloc(struct ath12k_base *ab, 661 struct dp_link_desc_bank *desc_bank, 662 int n_link_desc_bank, 663 int last_bank_sz) 664 { 665 struct ath12k_dp *dp = ath12k_ab_to_dp(ab); 666 int i; 667 int ret = 0; 668 int desc_sz = DP_LINK_DESC_ALLOC_SIZE_THRESH; 669 670 for (i = 0; i < n_link_desc_bank; i++) { 671 if (i == (n_link_desc_bank - 1) && last_bank_sz) 672 desc_sz = last_bank_sz; 673 674 desc_bank[i].vaddr_unaligned = 675 dma_alloc_coherent(ab->dev, desc_sz, 676 &desc_bank[i].paddr_unaligned, 677 GFP_KERNEL); 678 if (!desc_bank[i].vaddr_unaligned) { 679 ret = -ENOMEM; 680 goto err; 681 } 682 683 desc_bank[i].vaddr = PTR_ALIGN(desc_bank[i].vaddr_unaligned, 684 HAL_LINK_DESC_ALIGN); 685 desc_bank[i].paddr = desc_bank[i].paddr_unaligned + 686 ((unsigned long)desc_bank[i].vaddr - 687 (unsigned long)desc_bank[i].vaddr_unaligned); 688 desc_bank[i].size = desc_sz; 689 } 690 691 return 0; 692 693 err: 694 ath12k_dp_link_desc_bank_free(ab, dp->link_desc_banks); 695 696 return ret; 697 } 698 699 void ath12k_dp_link_desc_cleanup(struct ath12k_base *ab, 700 struct dp_link_desc_bank *desc_bank, 701 u32 ring_type, struct dp_srng *ring) 702 { 703 ath12k_dp_link_desc_bank_free(ab, desc_bank); 704 705 if (ring_type != HAL_RXDMA_MONITOR_DESC) { 706 ath12k_dp_srng_cleanup(ab, ring); 707 ath12k_dp_scatter_idle_link_desc_cleanup(ab); 708 } 709 } 710 711 static int ath12k_wbm_idle_ring_setup(struct ath12k_base *ab, u32 *n_link_desc) 712 { 713 struct ath12k_dp *dp = ath12k_ab_to_dp(ab); 714 u32 n_mpdu_link_desc, n_mpdu_queue_desc; 715 u32 n_tx_msdu_link_desc, n_rx_msdu_link_desc; 716 int ret = 0; 717 718 n_mpdu_link_desc = (DP_NUM_TIDS_MAX * DP_AVG_MPDUS_PER_TID_MAX) / 719 HAL_NUM_MPDUS_PER_LINK_DESC; 720 721 n_mpdu_queue_desc = n_mpdu_link_desc / 722 HAL_NUM_MPDU_LINKS_PER_QUEUE_DESC; 723 724 n_tx_msdu_link_desc = (DP_NUM_TIDS_MAX * DP_AVG_FLOWS_PER_TID * 725 DP_AVG_MSDUS_PER_FLOW) / 726 HAL_NUM_TX_MSDUS_PER_LINK_DESC; 727 728 n_rx_msdu_link_desc = (DP_NUM_TIDS_MAX * DP_AVG_MPDUS_PER_TID_MAX * 729 DP_AVG_MSDUS_PER_MPDU) / 730 HAL_NUM_RX_MSDUS_PER_LINK_DESC; 731 732 *n_link_desc = n_mpdu_link_desc + n_mpdu_queue_desc + 733 n_tx_msdu_link_desc + n_rx_msdu_link_desc; 734 735 if (*n_link_desc & (*n_link_desc - 1)) 736 *n_link_desc = 1 << fls(*n_link_desc); 737 738 ret = ath12k_dp_srng_setup(ab, &dp->wbm_idle_ring, 739 HAL_WBM_IDLE_LINK, 0, 0, *n_link_desc); 740 if (ret) { 741 ath12k_warn(ab, "failed to setup wbm_idle_ring: %d\n", ret); 742 return ret; 743 } 744 return ret; 745 } 746 747 int ath12k_dp_link_desc_setup(struct ath12k_base *ab, 748 struct dp_link_desc_bank *link_desc_banks, 749 u32 ring_type, struct hal_srng *srng, 750 u32 n_link_desc) 751 { 752 struct ath12k_dp *dp = ath12k_ab_to_dp(ab); 753 u32 tot_mem_sz; 754 u32 n_link_desc_bank, last_bank_sz; 755 u32 entry_sz, align_bytes, n_entries; 756 struct hal_wbm_link_desc *desc; 757 u32 paddr; 758 int i, ret; 759 u32 cookie; 760 enum hal_rx_buf_return_buf_manager rbm = dp->idle_link_rbm; 761 762 tot_mem_sz = n_link_desc * HAL_LINK_DESC_SIZE; 763 tot_mem_sz += HAL_LINK_DESC_ALIGN; 764 765 if (tot_mem_sz <= DP_LINK_DESC_ALLOC_SIZE_THRESH) { 766 n_link_desc_bank = 1; 767 last_bank_sz = tot_mem_sz; 768 } else { 769 n_link_desc_bank = tot_mem_sz / 770 (DP_LINK_DESC_ALLOC_SIZE_THRESH - 771 HAL_LINK_DESC_ALIGN); 772 last_bank_sz = tot_mem_sz % 773 (DP_LINK_DESC_ALLOC_SIZE_THRESH - 774 HAL_LINK_DESC_ALIGN); 775 776 if (last_bank_sz) 777 n_link_desc_bank += 1; 778 } 779 780 if (n_link_desc_bank > DP_LINK_DESC_BANKS_MAX) 781 return -EINVAL; 782 783 ret = ath12k_dp_link_desc_bank_alloc(ab, link_desc_banks, 784 n_link_desc_bank, last_bank_sz); 785 if (ret) 786 return ret; 787 788 /* Setup link desc idle list for HW internal usage */ 789 entry_sz = ath12k_hal_srng_get_entrysize(ab, ring_type); 790 tot_mem_sz = entry_sz * n_link_desc; 791 792 /* Setup scatter desc list when the total memory requirement is more */ 793 if (tot_mem_sz > DP_LINK_DESC_ALLOC_SIZE_THRESH && 794 ring_type != HAL_RXDMA_MONITOR_DESC) { 795 ret = ath12k_dp_scatter_idle_link_desc_setup(ab, tot_mem_sz, 796 n_link_desc_bank, 797 n_link_desc, 798 last_bank_sz); 799 if (ret) { 800 ath12k_warn(ab, "failed to setup scatting idle list descriptor :%d\n", 801 ret); 802 goto fail_desc_bank_free; 803 } 804 805 return 0; 806 } 807 808 spin_lock_bh(&srng->lock); 809 810 ath12k_hal_srng_access_begin(ab, srng); 811 812 for (i = 0; i < n_link_desc_bank; i++) { 813 align_bytes = link_desc_banks[i].vaddr - 814 link_desc_banks[i].vaddr_unaligned; 815 n_entries = (link_desc_banks[i].size - align_bytes) / 816 HAL_LINK_DESC_SIZE; 817 paddr = link_desc_banks[i].paddr; 818 while (n_entries && 819 (desc = ath12k_hal_srng_src_get_next_entry(ab, srng))) { 820 cookie = DP_LINK_DESC_COOKIE_SET(n_entries, i); 821 ath12k_hal_set_link_desc_addr(dp->hal, desc, cookie, paddr, 822 rbm); 823 n_entries--; 824 paddr += HAL_LINK_DESC_SIZE; 825 } 826 } 827 828 ath12k_hal_srng_access_end(ab, srng); 829 830 spin_unlock_bh(&srng->lock); 831 832 return 0; 833 834 fail_desc_bank_free: 835 ath12k_dp_link_desc_bank_free(ab, link_desc_banks); 836 837 return ret; 838 } 839 840 void ath12k_dp_pdev_free(struct ath12k_base *ab) 841 { 842 struct ath12k_dp *dp = ath12k_ab_to_dp(ab); 843 struct ath12k *ar; 844 int i; 845 846 for (i = 0; i < ab->num_radios; i++) { 847 ar = ab->pdevs[i].ar; 848 rcu_assign_pointer(dp->dp_pdevs[ar->pdev_idx], NULL); 849 } 850 851 synchronize_rcu(); 852 853 for (i = 0; i < ab->num_radios; i++) 854 ath12k_dp_rx_pdev_free(ab, i); 855 } 856 857 void ath12k_dp_pdev_pre_alloc(struct ath12k *ar) 858 { 859 struct ath12k_pdev_dp *dp = &ar->dp; 860 861 dp->mac_id = ar->pdev_idx; 862 atomic_set(&dp->num_tx_pending, 0); 863 init_waitqueue_head(&dp->tx_empty_waitq); 864 /* TODO: Add any RXDMA setup required per pdev */ 865 } 866 867 int ath12k_dp_pdev_alloc(struct ath12k_base *ab) 868 { 869 struct ath12k_dp *dp = ath12k_ab_to_dp(ab); 870 struct ath12k_pdev_dp *dp_pdev; 871 struct ath12k *ar; 872 int ret; 873 int i; 874 875 ret = ath12k_dp_rx_htt_setup(ab); 876 if (ret) 877 goto out; 878 879 /* TODO: Per-pdev rx ring unlike tx ring which is mapped to different AC's */ 880 for (i = 0; i < ab->num_radios; i++) { 881 ar = ab->pdevs[i].ar; 882 883 dp_pdev = &ar->dp; 884 885 dp_pdev->hw = ar->ah->hw; 886 dp_pdev->dp = dp; 887 dp_pdev->hw_link_id = ar->hw_link_id; 888 dp_pdev->dp_hw = &ar->ah->dp_hw; 889 890 ret = ath12k_dp_rx_pdev_alloc(ab, i); 891 if (ret) { 892 ath12k_warn(ab, "failed to allocate pdev rx for pdev_id :%d\n", 893 i); 894 goto err; 895 } 896 ret = ath12k_dp_rx_pdev_mon_attach(ar); 897 if (ret) { 898 ath12k_warn(ab, "failed to initialize mon pdev %d\n", i); 899 goto err; 900 } 901 } 902 903 for (i = 0; i < ab->num_radios; i++) { 904 ar = ab->pdevs[i].ar; 905 rcu_assign_pointer(dp->dp_pdevs[ar->pdev_idx], &ar->dp); 906 } 907 908 return 0; 909 err: 910 ath12k_dp_pdev_free(ab); 911 out: 912 return ret; 913 } 914 915 static void ath12k_dp_update_vdev_search(struct ath12k_link_vif *arvif) 916 { 917 u8 link_id = arvif->link_id; 918 struct ath12k_vif *ahvif = arvif->ahvif; 919 struct ath12k_dp_link_vif *dp_link_vif; 920 921 dp_link_vif = ath12k_dp_vif_to_dp_link_vif(&ahvif->dp_vif, link_id); 922 923 switch (arvif->ahvif->vdev_type) { 924 case WMI_VDEV_TYPE_STA: 925 dp_link_vif->hal_addr_search_flags = HAL_TX_ADDRY_EN; 926 dp_link_vif->search_type = HAL_TX_ADDR_SEARCH_DEFAULT; 927 break; 928 case WMI_VDEV_TYPE_AP: 929 case WMI_VDEV_TYPE_IBSS: 930 dp_link_vif->hal_addr_search_flags = HAL_TX_ADDRX_EN; 931 dp_link_vif->search_type = HAL_TX_ADDR_SEARCH_DEFAULT; 932 break; 933 case WMI_VDEV_TYPE_MONITOR: 934 default: 935 return; 936 } 937 } 938 939 void ath12k_dp_vdev_tx_attach(struct ath12k *ar, struct ath12k_link_vif *arvif) 940 { 941 struct ath12k_base *ab = ar->ab; 942 struct ath12k_vif *ahvif = arvif->ahvif; 943 u8 link_id = arvif->link_id; 944 int bank_id; 945 struct ath12k_dp_link_vif *dp_link_vif; 946 947 dp_link_vif = ath12k_dp_vif_to_dp_link_vif(&ahvif->dp_vif, link_id); 948 949 dp_link_vif->tcl_metadata |= u32_encode_bits(1, HTT_TCL_META_DATA_TYPE) | 950 u32_encode_bits(arvif->vdev_id, 951 HTT_TCL_META_DATA_VDEV_ID) | 952 u32_encode_bits(ar->pdev->pdev_id, 953 HTT_TCL_META_DATA_PDEV_ID); 954 955 /* set HTT extension valid bit to 0 by default */ 956 dp_link_vif->tcl_metadata &= ~HTT_TCL_META_DATA_VALID_HTT; 957 958 ath12k_dp_update_vdev_search(arvif); 959 dp_link_vif->vdev_id_check_en = true; 960 bank_id = ath12k_dp_tx_get_bank_profile(ab, arvif, ath12k_ab_to_dp(ab)); 961 dp_link_vif->bank_id = bank_id; 962 963 /* TODO: error path for bank id failure */ 964 if (bank_id == DP_INVALID_BANK_ID) { 965 ath12k_err(ar->ab, "Failed to initialize DP TX Banks"); 966 return; 967 } 968 } 969 970 static void ath12k_dp_cc_cleanup(struct ath12k_base *ab) 971 { 972 struct ath12k_rx_desc_info *desc_info; 973 struct ath12k_tx_desc_info *tx_desc_info, *tmp1; 974 struct ath12k_dp *dp = ath12k_ab_to_dp(ab); 975 struct ath12k_skb_cb *skb_cb; 976 struct sk_buff *skb; 977 struct ath12k *ar; 978 int i, j; 979 u32 pool_id, tx_spt_page; 980 981 if (!dp->spt_info) 982 return; 983 984 /* RX Descriptor cleanup */ 985 spin_lock_bh(&dp->rx_desc_lock); 986 987 if (dp->rxbaddr) { 988 for (i = 0; i < ATH12K_NUM_RX_SPT_PAGES(ab); i++) { 989 if (!dp->rxbaddr[i]) 990 continue; 991 992 desc_info = dp->rxbaddr[i]; 993 994 for (j = 0; j < ATH12K_MAX_SPT_ENTRIES; j++) { 995 if (!desc_info[j].in_use) { 996 list_del(&desc_info[j].list); 997 continue; 998 } 999 1000 skb = desc_info[j].skb; 1001 if (!skb) 1002 continue; 1003 1004 dma_unmap_single(ab->dev, 1005 ATH12K_SKB_RXCB(skb)->paddr, 1006 skb->len + skb_tailroom(skb), 1007 DMA_FROM_DEVICE); 1008 dev_kfree_skb_any(skb); 1009 } 1010 1011 kfree(dp->rxbaddr[i]); 1012 dp->rxbaddr[i] = NULL; 1013 } 1014 1015 kfree(dp->rxbaddr); 1016 dp->rxbaddr = NULL; 1017 } 1018 1019 spin_unlock_bh(&dp->rx_desc_lock); 1020 1021 /* TX Descriptor cleanup */ 1022 for (i = 0; i < ATH12K_HW_MAX_QUEUES; i++) { 1023 spin_lock_bh(&dp->tx_desc_lock[i]); 1024 1025 list_for_each_entry_safe(tx_desc_info, tmp1, 1026 &dp->tx_desc_used_list[i], list) { 1027 list_del(&tx_desc_info->list); 1028 skb = tx_desc_info->skb; 1029 1030 if (!skb) 1031 continue; 1032 1033 skb_cb = ATH12K_SKB_CB(skb); 1034 if (skb_cb->paddr_ext_desc) { 1035 dma_unmap_single(ab->dev, 1036 skb_cb->paddr_ext_desc, 1037 tx_desc_info->skb_ext_desc->len, 1038 DMA_TO_DEVICE); 1039 dev_kfree_skb_any(tx_desc_info->skb_ext_desc); 1040 } 1041 1042 /* if we are unregistering, hw would've been destroyed and 1043 * ar is no longer valid. 1044 */ 1045 if (!(test_bit(ATH12K_FLAG_UNREGISTERING, &ab->dev_flags))) { 1046 ar = skb_cb->ar; 1047 1048 if (atomic_dec_and_test(&ar->dp.num_tx_pending)) 1049 wake_up(&ar->dp.tx_empty_waitq); 1050 } 1051 1052 dma_unmap_single(ab->dev, ATH12K_SKB_CB(skb)->paddr, 1053 skb->len, DMA_TO_DEVICE); 1054 dev_kfree_skb_any(skb); 1055 } 1056 1057 spin_unlock_bh(&dp->tx_desc_lock[i]); 1058 } 1059 1060 if (dp->txbaddr) { 1061 for (pool_id = 0; pool_id < ATH12K_HW_MAX_QUEUES; pool_id++) { 1062 spin_lock_bh(&dp->tx_desc_lock[pool_id]); 1063 1064 for (i = 0; i < ATH12K_TX_SPT_PAGES_PER_POOL(ab); i++) { 1065 tx_spt_page = i + pool_id * 1066 ATH12K_TX_SPT_PAGES_PER_POOL(ab); 1067 if (!dp->txbaddr[tx_spt_page]) 1068 continue; 1069 1070 kfree(dp->txbaddr[tx_spt_page]); 1071 dp->txbaddr[tx_spt_page] = NULL; 1072 } 1073 1074 spin_unlock_bh(&dp->tx_desc_lock[pool_id]); 1075 } 1076 1077 kfree(dp->txbaddr); 1078 dp->txbaddr = NULL; 1079 } 1080 1081 /* unmap SPT pages */ 1082 for (i = 0; i < dp->num_spt_pages; i++) { 1083 if (!dp->spt_info[i].vaddr) 1084 continue; 1085 1086 dma_free_coherent(ab->dev, ATH12K_PAGE_SIZE, 1087 dp->spt_info[i].vaddr, dp->spt_info[i].paddr); 1088 dp->spt_info[i].vaddr = NULL; 1089 } 1090 1091 kfree(dp->spt_info); 1092 dp->spt_info = NULL; 1093 } 1094 1095 static void ath12k_dp_reoq_lut_cleanup(struct ath12k_base *ab) 1096 { 1097 struct ath12k_dp *dp = ath12k_ab_to_dp(ab); 1098 1099 if (!ab->hw_params->reoq_lut_support) 1100 return; 1101 1102 if (dp->reoq_lut.vaddr_unaligned) { 1103 ath12k_hal_write_reoq_lut_addr(ab, 0); 1104 dma_free_coherent(ab->dev, dp->reoq_lut.size, 1105 dp->reoq_lut.vaddr_unaligned, 1106 dp->reoq_lut.paddr_unaligned); 1107 dp->reoq_lut.vaddr_unaligned = NULL; 1108 } 1109 1110 if (dp->ml_reoq_lut.vaddr_unaligned) { 1111 ath12k_hal_write_ml_reoq_lut_addr(ab, 0); 1112 dma_free_coherent(ab->dev, dp->ml_reoq_lut.size, 1113 dp->ml_reoq_lut.vaddr_unaligned, 1114 dp->ml_reoq_lut.paddr_unaligned); 1115 dp->ml_reoq_lut.vaddr_unaligned = NULL; 1116 } 1117 } 1118 1119 static void ath12k_dp_cleanup(struct ath12k_base *ab) 1120 { 1121 struct ath12k_dp *dp = ath12k_ab_to_dp(ab); 1122 int i; 1123 1124 ath12k_dp_link_peer_rhash_tbl_destroy(dp); 1125 1126 if (!dp->ab) 1127 return; 1128 1129 ath12k_dp_link_desc_cleanup(ab, dp->link_desc_banks, 1130 HAL_WBM_IDLE_LINK, &dp->wbm_idle_ring); 1131 1132 ath12k_dp_cc_cleanup(ab); 1133 ath12k_dp_reoq_lut_cleanup(ab); 1134 ath12k_dp_deinit_bank_profiles(ab); 1135 ath12k_dp_srng_common_cleanup(ab); 1136 1137 ath12k_dp_rx_reo_cmd_list_cleanup(ab); 1138 1139 for (i = 0; i < ab->hw_params->max_tx_ring; i++) { 1140 kfree(dp->tx_ring[i].tx_status); 1141 dp->tx_ring[i].tx_status = NULL; 1142 } 1143 1144 ath12k_dp_rx_free(ab); 1145 /* Deinit any SOC level resource */ 1146 } 1147 1148 static u32 ath12k_dp_cc_cookie_gen(u16 ppt_idx, u16 spt_idx) 1149 { 1150 return (u32)ppt_idx << ATH12K_CC_PPT_SHIFT | spt_idx; 1151 } 1152 1153 static void *ath12k_dp_cc_get_desc_addr_ptr(struct ath12k_dp *dp, 1154 u16 ppt_idx, u16 spt_idx) 1155 { 1156 return dp->spt_info[ppt_idx].vaddr + spt_idx; 1157 } 1158 1159 struct ath12k_rx_desc_info *ath12k_dp_get_rx_desc(struct ath12k_dp *dp, 1160 u32 cookie) 1161 { 1162 struct ath12k_rx_desc_info **desc_addr_ptr; 1163 u16 start_ppt_idx, end_ppt_idx, ppt_idx, spt_idx; 1164 1165 ppt_idx = u32_get_bits(cookie, ATH12K_DP_CC_COOKIE_PPT); 1166 spt_idx = u32_get_bits(cookie, ATH12K_DP_CC_COOKIE_SPT); 1167 1168 start_ppt_idx = dp->rx_ppt_base + ATH12K_RX_SPT_PAGE_OFFSET(dp->ab); 1169 end_ppt_idx = start_ppt_idx + ATH12K_NUM_RX_SPT_PAGES(dp->ab); 1170 1171 if (ppt_idx < start_ppt_idx || 1172 ppt_idx >= end_ppt_idx || 1173 spt_idx > ATH12K_MAX_SPT_ENTRIES) 1174 return NULL; 1175 1176 ppt_idx = ppt_idx - dp->rx_ppt_base; 1177 desc_addr_ptr = ath12k_dp_cc_get_desc_addr_ptr(dp, ppt_idx, spt_idx); 1178 1179 return *desc_addr_ptr; 1180 } 1181 EXPORT_SYMBOL(ath12k_dp_get_rx_desc); 1182 1183 struct ath12k_tx_desc_info *ath12k_dp_get_tx_desc(struct ath12k_dp *dp, 1184 u32 cookie) 1185 { 1186 struct ath12k_tx_desc_info **desc_addr_ptr; 1187 u16 start_ppt_idx, end_ppt_idx, ppt_idx, spt_idx; 1188 1189 ppt_idx = u32_get_bits(cookie, ATH12K_DP_CC_COOKIE_PPT); 1190 spt_idx = u32_get_bits(cookie, ATH12K_DP_CC_COOKIE_SPT); 1191 1192 start_ppt_idx = ATH12K_TX_SPT_PAGE_OFFSET; 1193 end_ppt_idx = start_ppt_idx + 1194 (ATH12K_TX_SPT_PAGES_PER_POOL(dp->ab) * ATH12K_HW_MAX_QUEUES); 1195 1196 if (ppt_idx < start_ppt_idx || 1197 ppt_idx >= end_ppt_idx || 1198 spt_idx > ATH12K_MAX_SPT_ENTRIES) 1199 return NULL; 1200 1201 desc_addr_ptr = ath12k_dp_cc_get_desc_addr_ptr(dp, ppt_idx, spt_idx); 1202 1203 return *desc_addr_ptr; 1204 } 1205 EXPORT_SYMBOL(ath12k_dp_get_tx_desc); 1206 1207 static int ath12k_dp_cc_desc_init(struct ath12k_base *ab) 1208 { 1209 struct ath12k_dp *dp = ath12k_ab_to_dp(ab); 1210 struct ath12k_rx_desc_info *rx_descs, **rx_desc_addr; 1211 struct ath12k_tx_desc_info *tx_descs, **tx_desc_addr; 1212 u32 num_rx_spt_pages = ATH12K_NUM_RX_SPT_PAGES(ab); 1213 u32 i, j, pool_id, tx_spt_page; 1214 u32 ppt_idx, cookie_ppt_idx; 1215 1216 spin_lock_bh(&dp->rx_desc_lock); 1217 1218 dp->rxbaddr = kcalloc(num_rx_spt_pages, 1219 sizeof(struct ath12k_rx_desc_info *), GFP_ATOMIC); 1220 1221 if (!dp->rxbaddr) { 1222 spin_unlock_bh(&dp->rx_desc_lock); 1223 return -ENOMEM; 1224 } 1225 1226 /* First ATH12K_NUM_RX_SPT_PAGES(ab) of allocated SPT pages are used for 1227 * RX 1228 */ 1229 for (i = 0; i < num_rx_spt_pages; i++) { 1230 rx_descs = kcalloc(ATH12K_MAX_SPT_ENTRIES, sizeof(*rx_descs), 1231 GFP_ATOMIC); 1232 1233 if (!rx_descs) { 1234 spin_unlock_bh(&dp->rx_desc_lock); 1235 return -ENOMEM; 1236 } 1237 1238 ppt_idx = ATH12K_RX_SPT_PAGE_OFFSET(ab) + i; 1239 cookie_ppt_idx = dp->rx_ppt_base + ppt_idx; 1240 dp->rxbaddr[i] = &rx_descs[0]; 1241 1242 for (j = 0; j < ATH12K_MAX_SPT_ENTRIES; j++) { 1243 rx_descs[j].cookie = ath12k_dp_cc_cookie_gen(cookie_ppt_idx, j); 1244 rx_descs[j].magic = ATH12K_DP_RX_DESC_MAGIC; 1245 rx_descs[j].device_id = ab->device_id; 1246 list_add_tail(&rx_descs[j].list, &dp->rx_desc_free_list); 1247 1248 /* Update descriptor VA in SPT */ 1249 rx_desc_addr = ath12k_dp_cc_get_desc_addr_ptr(dp, ppt_idx, j); 1250 *rx_desc_addr = &rx_descs[j]; 1251 } 1252 } 1253 1254 spin_unlock_bh(&dp->rx_desc_lock); 1255 1256 dp->txbaddr = kcalloc(ATH12K_NUM_TX_SPT_PAGES(ab), 1257 sizeof(struct ath12k_tx_desc_info *), GFP_ATOMIC); 1258 1259 if (!dp->txbaddr) 1260 return -ENOMEM; 1261 1262 for (pool_id = 0; pool_id < ATH12K_HW_MAX_QUEUES; pool_id++) { 1263 spin_lock_bh(&dp->tx_desc_lock[pool_id]); 1264 for (i = 0; i < ATH12K_TX_SPT_PAGES_PER_POOL(ab); i++) { 1265 tx_descs = kcalloc(ATH12K_MAX_SPT_ENTRIES, sizeof(*tx_descs), 1266 GFP_ATOMIC); 1267 1268 if (!tx_descs) { 1269 spin_unlock_bh(&dp->tx_desc_lock[pool_id]); 1270 /* Caller takes care of TX pending and RX desc cleanup */ 1271 return -ENOMEM; 1272 } 1273 1274 tx_spt_page = i + pool_id * 1275 ATH12K_TX_SPT_PAGES_PER_POOL(ab); 1276 ppt_idx = ATH12K_TX_SPT_PAGE_OFFSET + tx_spt_page; 1277 1278 dp->txbaddr[tx_spt_page] = &tx_descs[0]; 1279 1280 for (j = 0; j < ATH12K_MAX_SPT_ENTRIES; j++) { 1281 tx_descs[j].desc_id = ath12k_dp_cc_cookie_gen(ppt_idx, j); 1282 tx_descs[j].pool_id = pool_id; 1283 list_add_tail(&tx_descs[j].list, 1284 &dp->tx_desc_free_list[pool_id]); 1285 1286 /* Update descriptor VA in SPT */ 1287 tx_desc_addr = 1288 ath12k_dp_cc_get_desc_addr_ptr(dp, ppt_idx, j); 1289 *tx_desc_addr = &tx_descs[j]; 1290 } 1291 } 1292 spin_unlock_bh(&dp->tx_desc_lock[pool_id]); 1293 } 1294 return 0; 1295 } 1296 1297 static int ath12k_dp_cmem_init(struct ath12k_base *ab, 1298 struct ath12k_dp *dp, 1299 enum ath12k_dp_desc_type type) 1300 { 1301 u32 cmem_base; 1302 int i, start, end; 1303 1304 cmem_base = ab->qmi.dev_mem[ATH12K_QMI_DEVMEM_CMEM_INDEX].start; 1305 1306 switch (type) { 1307 case ATH12K_DP_TX_DESC: 1308 start = ATH12K_TX_SPT_PAGE_OFFSET; 1309 end = start + ATH12K_NUM_TX_SPT_PAGES(ab); 1310 break; 1311 case ATH12K_DP_RX_DESC: 1312 cmem_base += ATH12K_PPT_ADDR_OFFSET(dp->rx_ppt_base); 1313 start = ATH12K_RX_SPT_PAGE_OFFSET(ab); 1314 end = start + ATH12K_NUM_RX_SPT_PAGES(ab); 1315 break; 1316 default: 1317 ath12k_err(ab, "invalid descriptor type %d in cmem init\n", type); 1318 return -EINVAL; 1319 } 1320 1321 /* Write to PPT in CMEM */ 1322 for (i = start; i < end; i++) 1323 ath12k_hif_write32(ab, cmem_base + ATH12K_PPT_ADDR_OFFSET(i), 1324 dp->spt_info[i].paddr >> ATH12K_SPT_4K_ALIGN_OFFSET); 1325 1326 return 0; 1327 } 1328 1329 void ath12k_dp_partner_cc_init(struct ath12k_base *ab) 1330 { 1331 struct ath12k_hw_group *ag = ab->ag; 1332 int i; 1333 1334 for (i = 0; i < ag->num_devices; i++) { 1335 if (ag->ab[i] == ab) 1336 continue; 1337 1338 ath12k_dp_cmem_init(ab, ath12k_ab_to_dp(ag->ab[i]), ATH12K_DP_RX_DESC); 1339 } 1340 } 1341 1342 static u32 ath12k_dp_get_num_spt_pages(struct ath12k_base *ab) 1343 { 1344 return ATH12K_NUM_RX_SPT_PAGES(ab) + ATH12K_NUM_TX_SPT_PAGES(ab); 1345 } 1346 1347 static int ath12k_dp_cc_init(struct ath12k_base *ab) 1348 { 1349 struct ath12k_dp *dp = ath12k_ab_to_dp(ab); 1350 int i, ret = 0; 1351 1352 INIT_LIST_HEAD(&dp->rx_desc_free_list); 1353 spin_lock_init(&dp->rx_desc_lock); 1354 1355 for (i = 0; i < ATH12K_HW_MAX_QUEUES; i++) { 1356 INIT_LIST_HEAD(&dp->tx_desc_free_list[i]); 1357 INIT_LIST_HEAD(&dp->tx_desc_used_list[i]); 1358 spin_lock_init(&dp->tx_desc_lock[i]); 1359 } 1360 1361 dp->num_spt_pages = ath12k_dp_get_num_spt_pages(ab); 1362 if (dp->num_spt_pages > ATH12K_MAX_PPT_ENTRIES) 1363 dp->num_spt_pages = ATH12K_MAX_PPT_ENTRIES; 1364 1365 dp->spt_info = kcalloc(dp->num_spt_pages, sizeof(struct ath12k_spt_info), 1366 GFP_KERNEL); 1367 1368 if (!dp->spt_info) { 1369 ath12k_warn(ab, "SPT page allocation failure"); 1370 return -ENOMEM; 1371 } 1372 1373 dp->rx_ppt_base = ab->device_id * ATH12K_NUM_RX_SPT_PAGES(ab); 1374 1375 for (i = 0; i < dp->num_spt_pages; i++) { 1376 dp->spt_info[i].vaddr = dma_alloc_coherent(ab->dev, 1377 ATH12K_PAGE_SIZE, 1378 &dp->spt_info[i].paddr, 1379 GFP_KERNEL); 1380 1381 if (!dp->spt_info[i].vaddr) { 1382 ret = -ENOMEM; 1383 goto free; 1384 } 1385 1386 if (dp->spt_info[i].paddr & ATH12K_SPT_4K_ALIGN_CHECK) { 1387 ath12k_warn(ab, "SPT allocated memory is not 4K aligned"); 1388 ret = -EINVAL; 1389 goto free; 1390 } 1391 } 1392 1393 ret = ath12k_dp_cmem_init(ab, dp, ATH12K_DP_TX_DESC); 1394 if (ret) { 1395 ath12k_warn(ab, "HW CC Tx cmem init failed %d", ret); 1396 goto free; 1397 } 1398 1399 ret = ath12k_dp_cmem_init(ab, dp, ATH12K_DP_RX_DESC); 1400 if (ret) { 1401 ath12k_warn(ab, "HW CC Rx cmem init failed %d", ret); 1402 goto free; 1403 } 1404 1405 ret = ath12k_dp_cc_desc_init(ab); 1406 if (ret) { 1407 ath12k_warn(ab, "HW CC desc init failed %d", ret); 1408 goto free; 1409 } 1410 1411 return 0; 1412 free: 1413 ath12k_dp_cc_cleanup(ab); 1414 return ret; 1415 } 1416 1417 static int ath12k_dp_alloc_reoq_lut(struct ath12k_base *ab, 1418 struct ath12k_reo_q_addr_lut *lut) 1419 { 1420 lut->size = DP_REOQ_LUT_SIZE + HAL_REO_QLUT_ADDR_ALIGN - 1; 1421 lut->vaddr_unaligned = dma_alloc_coherent(ab->dev, lut->size, 1422 &lut->paddr_unaligned, 1423 GFP_KERNEL | __GFP_ZERO); 1424 if (!lut->vaddr_unaligned) 1425 return -ENOMEM; 1426 1427 lut->vaddr = PTR_ALIGN(lut->vaddr_unaligned, HAL_REO_QLUT_ADDR_ALIGN); 1428 lut->paddr = lut->paddr_unaligned + 1429 ((unsigned long)lut->vaddr - (unsigned long)lut->vaddr_unaligned); 1430 return 0; 1431 } 1432 1433 static int ath12k_dp_reoq_lut_setup(struct ath12k_base *ab) 1434 { 1435 struct ath12k_dp *dp = ath12k_ab_to_dp(ab); 1436 int ret; 1437 1438 if (!ab->hw_params->reoq_lut_support) 1439 return 0; 1440 1441 ret = ath12k_dp_alloc_reoq_lut(ab, &dp->reoq_lut); 1442 if (ret) { 1443 ath12k_warn(ab, "failed to allocate memory for reoq table"); 1444 return ret; 1445 } 1446 1447 ret = ath12k_dp_alloc_reoq_lut(ab, &dp->ml_reoq_lut); 1448 if (ret) { 1449 ath12k_warn(ab, "failed to allocate memory for ML reoq table"); 1450 dma_free_coherent(ab->dev, dp->reoq_lut.size, 1451 dp->reoq_lut.vaddr_unaligned, 1452 dp->reoq_lut.paddr_unaligned); 1453 dp->reoq_lut.vaddr_unaligned = NULL; 1454 return ret; 1455 } 1456 1457 /* Bits in the register have address [39:8] LUT base address to be 1458 * allocated such that LSBs are assumed to be zero. Also, current 1459 * design supports paddr up to 4 GB max hence it fits in 32 bit 1460 * register only 1461 */ 1462 1463 ath12k_hal_write_reoq_lut_addr(ab, dp->reoq_lut.paddr >> 8); 1464 ath12k_hal_write_ml_reoq_lut_addr(ab, dp->ml_reoq_lut.paddr >> 8); 1465 ath12k_hal_reoq_lut_addr_read_enable(ab); 1466 ath12k_hal_reoq_lut_set_max_peerid(ab); 1467 1468 return 0; 1469 } 1470 1471 static int ath12k_dp_setup(struct ath12k_base *ab) 1472 { 1473 struct ath12k_dp *dp; 1474 struct hal_srng *srng = NULL; 1475 size_t size = 0; 1476 u32 n_link_desc = 0; 1477 int ret; 1478 int i; 1479 1480 dp = ath12k_ab_to_dp(ab); 1481 dp->ab = ab; 1482 1483 INIT_LIST_HEAD(&dp->reo_cmd_list); 1484 INIT_LIST_HEAD(&dp->reo_cmd_cache_flush_list); 1485 INIT_LIST_HEAD(&dp->reo_cmd_update_rx_queue_list); 1486 spin_lock_init(&dp->reo_cmd_lock); 1487 spin_lock_init(&dp->reo_rxq_flush_lock); 1488 1489 spin_lock_init(&dp->dp_lock); 1490 INIT_LIST_HEAD(&dp->peers); 1491 1492 mutex_init(&dp->link_peer_rhash_tbl_lock); 1493 1494 dp->reo_cmd_cache_flush_count = 0; 1495 dp->idle_link_rbm = 1496 ath12k_hal_get_idle_link_rbm(&ab->hal, ab->device_id); 1497 1498 ret = ath12k_dp_link_peer_rhash_tbl_init(dp); 1499 if (ret) { 1500 ath12k_warn(ab, "failed to init link_peer rhash table: %d\n", ret); 1501 return ret; 1502 } 1503 1504 ret = ath12k_wbm_idle_ring_setup(ab, &n_link_desc); 1505 if (ret) { 1506 ath12k_warn(ab, "failed to setup wbm_idle_ring: %d\n", ret); 1507 goto rhash_destroy; 1508 } 1509 1510 srng = &ab->hal.srng_list[dp->wbm_idle_ring.ring_id]; 1511 1512 ret = ath12k_dp_link_desc_setup(ab, dp->link_desc_banks, 1513 HAL_WBM_IDLE_LINK, srng, n_link_desc); 1514 if (ret) { 1515 ath12k_warn(ab, "failed to setup link desc: %d\n", ret); 1516 goto rhash_destroy; 1517 } 1518 1519 ret = ath12k_dp_cc_init(ab); 1520 1521 if (ret) { 1522 ath12k_warn(ab, "failed to setup cookie converter %d\n", ret); 1523 goto fail_link_desc_cleanup; 1524 } 1525 ret = ath12k_dp_init_bank_profiles(ab); 1526 if (ret) { 1527 ath12k_warn(ab, "failed to setup bank profiles %d\n", ret); 1528 goto fail_hw_cc_cleanup; 1529 } 1530 1531 ret = ath12k_dp_srng_common_setup(ab); 1532 if (ret) 1533 goto fail_dp_bank_profiles_cleanup; 1534 1535 size = ab->hal.hal_wbm_release_ring_tx_size * 1536 DP_TX_COMP_RING_SIZE(ab); 1537 1538 ret = ath12k_dp_reoq_lut_setup(ab); 1539 if (ret) { 1540 ath12k_warn(ab, "failed to setup reoq table %d\n", ret); 1541 goto fail_cmn_srng_cleanup; 1542 } 1543 1544 for (i = 0; i < ab->hw_params->max_tx_ring; i++) { 1545 dp->tx_ring[i].tcl_data_ring_id = i; 1546 1547 dp->tx_ring[i].tx_status_head = 0; 1548 dp->tx_ring[i].tx_status_tail = DP_TX_COMP_RING_SIZE(ab) - 1; 1549 dp->tx_ring[i].tx_status = kmalloc(size, GFP_KERNEL); 1550 if (!dp->tx_ring[i].tx_status) { 1551 ret = -ENOMEM; 1552 /* FIXME: The allocated tx status is not freed 1553 * properly here 1554 */ 1555 goto fail_cmn_reoq_cleanup; 1556 } 1557 } 1558 1559 for (i = 0; i < HAL_DSCP_TID_MAP_TBL_NUM_ENTRIES_MAX; i++) 1560 ath12k_hal_tx_set_dscp_tid_map(ab, i); 1561 1562 ret = ath12k_dp_rx_alloc(ab); 1563 if (ret) 1564 goto fail_dp_rx_free; 1565 1566 /* Init any SOC level resource for DP */ 1567 1568 return 0; 1569 1570 fail_dp_rx_free: 1571 ath12k_dp_rx_free(ab); 1572 1573 fail_cmn_reoq_cleanup: 1574 ath12k_dp_reoq_lut_cleanup(ab); 1575 1576 fail_cmn_srng_cleanup: 1577 ath12k_dp_srng_common_cleanup(ab); 1578 1579 fail_dp_bank_profiles_cleanup: 1580 ath12k_dp_deinit_bank_profiles(ab); 1581 1582 fail_hw_cc_cleanup: 1583 ath12k_dp_cc_cleanup(ab); 1584 1585 fail_link_desc_cleanup: 1586 ath12k_dp_link_desc_cleanup(ab, dp->link_desc_banks, 1587 HAL_WBM_IDLE_LINK, &dp->wbm_idle_ring); 1588 rhash_destroy: 1589 ath12k_dp_link_peer_rhash_tbl_destroy(dp); 1590 1591 return ret; 1592 } 1593 1594 void ath12k_dp_cmn_device_deinit(struct ath12k_dp *dp) 1595 { 1596 ath12k_dp_cleanup(dp->ab); 1597 } 1598 1599 int ath12k_dp_cmn_device_init(struct ath12k_dp *dp) 1600 { 1601 int ret; 1602 1603 ret = ath12k_dp_setup(dp->ab); 1604 if (ret) 1605 return ret; 1606 1607 return 0; 1608 } 1609 1610 void ath12k_dp_cmn_hw_group_unassign(struct ath12k_dp *dp, 1611 struct ath12k_hw_group *ag) 1612 { 1613 struct ath12k_dp_hw_group *dp_hw_grp = &ag->dp_hw_grp; 1614 1615 lockdep_assert_held(&ag->mutex); 1616 1617 dp_hw_grp->dp[dp->device_id] = NULL; 1618 1619 dp->ag = NULL; 1620 dp->device_id = ATH12K_INVALID_DEVICE_ID; 1621 } 1622 1623 void ath12k_dp_cmn_hw_group_assign(struct ath12k_dp *dp, 1624 struct ath12k_hw_group *ag) 1625 { 1626 struct ath12k_base *ab = dp->ab; 1627 struct ath12k_dp_hw_group *dp_hw_grp = &ag->dp_hw_grp; 1628 1629 dp->ag = ag; 1630 dp->device_id = ab->device_id; 1631 dp_hw_grp->dp[dp->device_id] = dp; 1632 } 1633