1 // SPDX-License-Identifier: BSD-3-Clause-Clear 2 /* 3 * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved. 4 * Copyright (c) 2021-2024 Qualcomm Innovation Center, Inc. All rights reserved. 5 */ 6 7 #include <crypto/hash.h> 8 #include "core.h" 9 #include "dp_tx.h" 10 #include "hal_tx.h" 11 #include "hif.h" 12 #include "debug.h" 13 #include "dp_rx.h" 14 #include "peer.h" 15 #include "dp_mon.h" 16 17 static void ath12k_dp_htt_htc_tx_complete(struct ath12k_base *ab, 18 struct sk_buff *skb) 19 { 20 dev_kfree_skb_any(skb); 21 } 22 23 void ath12k_dp_peer_cleanup(struct ath12k *ar, int vdev_id, const u8 *addr) 24 { 25 struct ath12k_base *ab = ar->ab; 26 struct ath12k_peer *peer; 27 28 /* TODO: Any other peer specific DP cleanup */ 29 30 spin_lock_bh(&ab->base_lock); 31 peer = ath12k_peer_find(ab, vdev_id, addr); 32 if (!peer) { 33 ath12k_warn(ab, "failed to lookup peer %pM on vdev %d\n", 34 addr, vdev_id); 35 spin_unlock_bh(&ab->base_lock); 36 return; 37 } 38 39 ath12k_dp_rx_peer_tid_cleanup(ar, peer); 40 crypto_free_shash(peer->tfm_mmic); 41 peer->dp_setup_done = false; 42 spin_unlock_bh(&ab->base_lock); 43 } 44 45 int ath12k_dp_peer_setup(struct ath12k *ar, int vdev_id, const u8 *addr) 46 { 47 struct ath12k_base *ab = ar->ab; 48 struct ath12k_peer *peer; 49 u32 reo_dest; 50 int ret = 0, tid; 51 52 /* NOTE: reo_dest ring id starts from 1 unlike mac_id which starts from 0 */ 53 reo_dest = ar->dp.mac_id + 1; 54 ret = ath12k_wmi_set_peer_param(ar, addr, vdev_id, 55 WMI_PEER_SET_DEFAULT_ROUTING, 56 DP_RX_HASH_ENABLE | (reo_dest << 1)); 57 58 if (ret) { 59 ath12k_warn(ab, "failed to set default routing %d peer :%pM vdev_id :%d\n", 60 ret, addr, vdev_id); 61 return ret; 62 } 63 64 for (tid = 0; tid <= IEEE80211_NUM_TIDS; tid++) { 65 ret = ath12k_dp_rx_peer_tid_setup(ar, addr, vdev_id, tid, 1, 0, 66 HAL_PN_TYPE_NONE); 67 if (ret) { 68 ath12k_warn(ab, "failed to setup rxd tid queue for tid %d: %d\n", 69 tid, ret); 70 goto peer_clean; 71 } 72 } 73 74 ret = ath12k_dp_rx_peer_frag_setup(ar, addr, vdev_id); 75 if (ret) { 76 ath12k_warn(ab, "failed to setup rx defrag context\n"); 77 goto peer_clean; 78 } 79 80 /* TODO: Setup other peer specific resource used in data path */ 81 82 return 0; 83 84 peer_clean: 85 spin_lock_bh(&ab->base_lock); 86 87 peer = ath12k_peer_find(ab, vdev_id, addr); 88 if (!peer) { 89 ath12k_warn(ab, "failed to find the peer to del rx tid\n"); 90 spin_unlock_bh(&ab->base_lock); 91 return -ENOENT; 92 } 93 94 for (; tid >= 0; tid--) 95 ath12k_dp_rx_peer_tid_delete(ar, peer, tid); 96 97 spin_unlock_bh(&ab->base_lock); 98 99 return ret; 100 } 101 102 void ath12k_dp_srng_cleanup(struct ath12k_base *ab, struct dp_srng *ring) 103 { 104 if (!ring->vaddr_unaligned) 105 return; 106 107 dma_free_coherent(ab->dev, ring->size, ring->vaddr_unaligned, 108 ring->paddr_unaligned); 109 110 ring->vaddr_unaligned = NULL; 111 } 112 113 static int ath12k_dp_srng_find_ring_in_mask(int ring_num, const u8 *grp_mask) 114 { 115 int ext_group_num; 116 u8 mask = 1 << ring_num; 117 118 for (ext_group_num = 0; ext_group_num < ATH12K_EXT_IRQ_GRP_NUM_MAX; 119 ext_group_num++) { 120 if (mask & grp_mask[ext_group_num]) 121 return ext_group_num; 122 } 123 124 return -ENOENT; 125 } 126 127 static int ath12k_dp_srng_calculate_msi_group(struct ath12k_base *ab, 128 enum hal_ring_type type, int ring_num) 129 { 130 const u8 *grp_mask; 131 132 switch (type) { 133 case HAL_WBM2SW_RELEASE: 134 if (ring_num == HAL_WBM2SW_REL_ERR_RING_NUM) { 135 grp_mask = &ab->hw_params->ring_mask->rx_wbm_rel[0]; 136 ring_num = 0; 137 } else { 138 grp_mask = &ab->hw_params->ring_mask->tx[0]; 139 } 140 break; 141 case HAL_REO_EXCEPTION: 142 grp_mask = &ab->hw_params->ring_mask->rx_err[0]; 143 break; 144 case HAL_REO_DST: 145 grp_mask = &ab->hw_params->ring_mask->rx[0]; 146 break; 147 case HAL_REO_STATUS: 148 grp_mask = &ab->hw_params->ring_mask->reo_status[0]; 149 break; 150 case HAL_RXDMA_MONITOR_STATUS: 151 case HAL_RXDMA_MONITOR_DST: 152 grp_mask = &ab->hw_params->ring_mask->rx_mon_dest[0]; 153 break; 154 case HAL_TX_MONITOR_DST: 155 grp_mask = &ab->hw_params->ring_mask->tx_mon_dest[0]; 156 break; 157 case HAL_RXDMA_BUF: 158 grp_mask = &ab->hw_params->ring_mask->host2rxdma[0]; 159 break; 160 case HAL_RXDMA_MONITOR_BUF: 161 case HAL_TCL_DATA: 162 case HAL_TCL_CMD: 163 case HAL_REO_CMD: 164 case HAL_SW2WBM_RELEASE: 165 case HAL_WBM_IDLE_LINK: 166 case HAL_TCL_STATUS: 167 case HAL_REO_REINJECT: 168 case HAL_CE_SRC: 169 case HAL_CE_DST: 170 case HAL_CE_DST_STATUS: 171 default: 172 return -ENOENT; 173 } 174 175 return ath12k_dp_srng_find_ring_in_mask(ring_num, grp_mask); 176 } 177 178 static void ath12k_dp_srng_msi_setup(struct ath12k_base *ab, 179 struct hal_srng_params *ring_params, 180 enum hal_ring_type type, int ring_num) 181 { 182 int msi_group_number, msi_data_count; 183 u32 msi_data_start, msi_irq_start, addr_lo, addr_hi; 184 int ret; 185 186 ret = ath12k_hif_get_user_msi_vector(ab, "DP", 187 &msi_data_count, &msi_data_start, 188 &msi_irq_start); 189 if (ret) 190 return; 191 192 msi_group_number = ath12k_dp_srng_calculate_msi_group(ab, type, 193 ring_num); 194 if (msi_group_number < 0) { 195 ath12k_dbg(ab, ATH12K_DBG_PCI, 196 "ring not part of an ext_group; ring_type: %d,ring_num %d", 197 type, ring_num); 198 ring_params->msi_addr = 0; 199 ring_params->msi_data = 0; 200 return; 201 } 202 203 if (msi_group_number > msi_data_count) { 204 ath12k_dbg(ab, ATH12K_DBG_PCI, 205 "multiple msi_groups share one msi, msi_group_num %d", 206 msi_group_number); 207 } 208 209 ath12k_hif_get_msi_address(ab, &addr_lo, &addr_hi); 210 211 ring_params->msi_addr = addr_lo; 212 ring_params->msi_addr |= (dma_addr_t)(((uint64_t)addr_hi) << 32); 213 ring_params->msi_data = (msi_group_number % msi_data_count) 214 + msi_data_start; 215 ring_params->flags |= HAL_SRNG_FLAGS_MSI_INTR; 216 } 217 218 int ath12k_dp_srng_setup(struct ath12k_base *ab, struct dp_srng *ring, 219 enum hal_ring_type type, int ring_num, 220 int mac_id, int num_entries) 221 { 222 struct hal_srng_params params = { 0 }; 223 int entry_sz = ath12k_hal_srng_get_entrysize(ab, type); 224 int max_entries = ath12k_hal_srng_get_max_entries(ab, type); 225 int ret; 226 227 if (max_entries < 0 || entry_sz < 0) 228 return -EINVAL; 229 230 if (num_entries > max_entries) 231 num_entries = max_entries; 232 233 ring->size = (num_entries * entry_sz) + HAL_RING_BASE_ALIGN - 1; 234 ring->vaddr_unaligned = dma_alloc_coherent(ab->dev, ring->size, 235 &ring->paddr_unaligned, 236 GFP_KERNEL); 237 if (!ring->vaddr_unaligned) 238 return -ENOMEM; 239 240 ring->vaddr = PTR_ALIGN(ring->vaddr_unaligned, HAL_RING_BASE_ALIGN); 241 ring->paddr = ring->paddr_unaligned + ((unsigned long)ring->vaddr - 242 (unsigned long)ring->vaddr_unaligned); 243 244 params.ring_base_vaddr = ring->vaddr; 245 params.ring_base_paddr = ring->paddr; 246 params.num_entries = num_entries; 247 ath12k_dp_srng_msi_setup(ab, ¶ms, type, ring_num + mac_id); 248 249 switch (type) { 250 case HAL_REO_DST: 251 params.intr_batch_cntr_thres_entries = 252 HAL_SRNG_INT_BATCH_THRESHOLD_RX; 253 params.intr_timer_thres_us = HAL_SRNG_INT_TIMER_THRESHOLD_RX; 254 break; 255 case HAL_RXDMA_BUF: 256 case HAL_RXDMA_MONITOR_BUF: 257 case HAL_RXDMA_MONITOR_STATUS: 258 params.low_threshold = num_entries >> 3; 259 params.flags |= HAL_SRNG_FLAGS_LOW_THRESH_INTR_EN; 260 params.intr_batch_cntr_thres_entries = 0; 261 params.intr_timer_thres_us = HAL_SRNG_INT_TIMER_THRESHOLD_RX; 262 break; 263 case HAL_TX_MONITOR_DST: 264 params.low_threshold = DP_TX_MONITOR_BUF_SIZE_MAX >> 3; 265 params.flags |= HAL_SRNG_FLAGS_LOW_THRESH_INTR_EN; 266 params.intr_batch_cntr_thres_entries = 0; 267 params.intr_timer_thres_us = HAL_SRNG_INT_TIMER_THRESHOLD_RX; 268 break; 269 case HAL_WBM2SW_RELEASE: 270 if (ab->hw_params->hw_ops->dp_srng_is_tx_comp_ring(ring_num)) { 271 params.intr_batch_cntr_thres_entries = 272 HAL_SRNG_INT_BATCH_THRESHOLD_TX; 273 params.intr_timer_thres_us = 274 HAL_SRNG_INT_TIMER_THRESHOLD_TX; 275 break; 276 } 277 /* follow through when ring_num != HAL_WBM2SW_REL_ERR_RING_NUM */ 278 fallthrough; 279 case HAL_REO_EXCEPTION: 280 case HAL_REO_REINJECT: 281 case HAL_REO_CMD: 282 case HAL_REO_STATUS: 283 case HAL_TCL_DATA: 284 case HAL_TCL_CMD: 285 case HAL_TCL_STATUS: 286 case HAL_WBM_IDLE_LINK: 287 case HAL_SW2WBM_RELEASE: 288 case HAL_RXDMA_DST: 289 case HAL_RXDMA_MONITOR_DST: 290 case HAL_RXDMA_MONITOR_DESC: 291 params.intr_batch_cntr_thres_entries = 292 HAL_SRNG_INT_BATCH_THRESHOLD_OTHER; 293 params.intr_timer_thres_us = HAL_SRNG_INT_TIMER_THRESHOLD_OTHER; 294 break; 295 case HAL_RXDMA_DIR_BUF: 296 break; 297 default: 298 ath12k_warn(ab, "Not a valid ring type in dp :%d\n", type); 299 return -EINVAL; 300 } 301 302 ret = ath12k_hal_srng_setup(ab, type, ring_num, mac_id, ¶ms); 303 if (ret < 0) { 304 ath12k_warn(ab, "failed to setup srng: %d ring_id %d\n", 305 ret, ring_num); 306 return ret; 307 } 308 309 ring->ring_id = ret; 310 311 return 0; 312 } 313 314 static 315 u32 ath12k_dp_tx_get_vdev_bank_config(struct ath12k_base *ab, struct ath12k_vif *arvif) 316 { 317 u32 bank_config = 0; 318 319 /* Only valid for raw frames with HW crypto enabled. 320 * With SW crypto, mac80211 sets key per packet 321 */ 322 if (arvif->tx_encap_type == HAL_TCL_ENCAP_TYPE_RAW && 323 test_bit(ATH12K_FLAG_HW_CRYPTO_DISABLED, &ab->dev_flags)) 324 bank_config |= 325 u32_encode_bits(ath12k_dp_tx_get_encrypt_type(arvif->key_cipher), 326 HAL_TX_BANK_CONFIG_ENCRYPT_TYPE); 327 328 bank_config |= u32_encode_bits(arvif->tx_encap_type, 329 HAL_TX_BANK_CONFIG_ENCAP_TYPE); 330 bank_config |= u32_encode_bits(0, HAL_TX_BANK_CONFIG_SRC_BUFFER_SWAP) | 331 u32_encode_bits(0, HAL_TX_BANK_CONFIG_LINK_META_SWAP) | 332 u32_encode_bits(0, HAL_TX_BANK_CONFIG_EPD); 333 334 /* only valid if idx_lookup_override is not set in tcl_data_cmd */ 335 bank_config |= u32_encode_bits(0, HAL_TX_BANK_CONFIG_INDEX_LOOKUP_EN); 336 337 bank_config |= u32_encode_bits(arvif->hal_addr_search_flags & HAL_TX_ADDRX_EN, 338 HAL_TX_BANK_CONFIG_ADDRX_EN) | 339 u32_encode_bits(!!(arvif->hal_addr_search_flags & 340 HAL_TX_ADDRY_EN), 341 HAL_TX_BANK_CONFIG_ADDRY_EN); 342 343 bank_config |= u32_encode_bits(ieee80211_vif_is_mesh(arvif->vif) ? 3 : 0, 344 HAL_TX_BANK_CONFIG_MESH_EN) | 345 u32_encode_bits(arvif->vdev_id_check_en, 346 HAL_TX_BANK_CONFIG_VDEV_ID_CHECK_EN); 347 348 bank_config |= u32_encode_bits(0, HAL_TX_BANK_CONFIG_DSCP_TIP_MAP_ID); 349 350 return bank_config; 351 } 352 353 static int ath12k_dp_tx_get_bank_profile(struct ath12k_base *ab, struct ath12k_vif *arvif, 354 struct ath12k_dp *dp) 355 { 356 int bank_id = DP_INVALID_BANK_ID; 357 int i; 358 u32 bank_config; 359 bool configure_register = false; 360 361 /* convert vdev params into hal_tx_bank_config */ 362 bank_config = ath12k_dp_tx_get_vdev_bank_config(ab, arvif); 363 364 spin_lock_bh(&dp->tx_bank_lock); 365 /* TODO: implement using idr kernel framework*/ 366 for (i = 0; i < dp->num_bank_profiles; i++) { 367 if (dp->bank_profiles[i].is_configured && 368 (dp->bank_profiles[i].bank_config ^ bank_config) == 0) { 369 bank_id = i; 370 goto inc_ref_and_return; 371 } 372 if (!dp->bank_profiles[i].is_configured || 373 !dp->bank_profiles[i].num_users) { 374 bank_id = i; 375 goto configure_and_return; 376 } 377 } 378 379 if (bank_id == DP_INVALID_BANK_ID) { 380 spin_unlock_bh(&dp->tx_bank_lock); 381 ath12k_err(ab, "unable to find TX bank!"); 382 return bank_id; 383 } 384 385 configure_and_return: 386 dp->bank_profiles[bank_id].is_configured = true; 387 dp->bank_profiles[bank_id].bank_config = bank_config; 388 configure_register = true; 389 inc_ref_and_return: 390 dp->bank_profiles[bank_id].num_users++; 391 spin_unlock_bh(&dp->tx_bank_lock); 392 393 if (configure_register) 394 ath12k_hal_tx_configure_bank_register(ab, bank_config, bank_id); 395 396 ath12k_dbg(ab, ATH12K_DBG_DP_HTT, "dp_htt tcl bank_id %d input 0x%x match 0x%x num_users %u", 397 bank_id, bank_config, dp->bank_profiles[bank_id].bank_config, 398 dp->bank_profiles[bank_id].num_users); 399 400 return bank_id; 401 } 402 403 void ath12k_dp_tx_put_bank_profile(struct ath12k_dp *dp, u8 bank_id) 404 { 405 spin_lock_bh(&dp->tx_bank_lock); 406 dp->bank_profiles[bank_id].num_users--; 407 spin_unlock_bh(&dp->tx_bank_lock); 408 } 409 410 static void ath12k_dp_deinit_bank_profiles(struct ath12k_base *ab) 411 { 412 struct ath12k_dp *dp = &ab->dp; 413 414 kfree(dp->bank_profiles); 415 dp->bank_profiles = NULL; 416 } 417 418 static int ath12k_dp_init_bank_profiles(struct ath12k_base *ab) 419 { 420 struct ath12k_dp *dp = &ab->dp; 421 u32 num_tcl_banks = ab->hw_params->num_tcl_banks; 422 int i; 423 424 dp->num_bank_profiles = num_tcl_banks; 425 dp->bank_profiles = kmalloc_array(num_tcl_banks, 426 sizeof(struct ath12k_dp_tx_bank_profile), 427 GFP_KERNEL); 428 if (!dp->bank_profiles) 429 return -ENOMEM; 430 431 spin_lock_init(&dp->tx_bank_lock); 432 433 for (i = 0; i < num_tcl_banks; i++) { 434 dp->bank_profiles[i].is_configured = false; 435 dp->bank_profiles[i].num_users = 0; 436 } 437 438 return 0; 439 } 440 441 static void ath12k_dp_srng_common_cleanup(struct ath12k_base *ab) 442 { 443 struct ath12k_dp *dp = &ab->dp; 444 int i; 445 446 ath12k_dp_srng_cleanup(ab, &dp->reo_status_ring); 447 ath12k_dp_srng_cleanup(ab, &dp->reo_cmd_ring); 448 ath12k_dp_srng_cleanup(ab, &dp->reo_except_ring); 449 ath12k_dp_srng_cleanup(ab, &dp->rx_rel_ring); 450 ath12k_dp_srng_cleanup(ab, &dp->reo_reinject_ring); 451 for (i = 0; i < ab->hw_params->max_tx_ring; i++) { 452 ath12k_dp_srng_cleanup(ab, &dp->tx_ring[i].tcl_comp_ring); 453 ath12k_dp_srng_cleanup(ab, &dp->tx_ring[i].tcl_data_ring); 454 } 455 ath12k_dp_srng_cleanup(ab, &dp->tcl_status_ring); 456 ath12k_dp_srng_cleanup(ab, &dp->tcl_cmd_ring); 457 ath12k_dp_srng_cleanup(ab, &dp->wbm_desc_rel_ring); 458 } 459 460 static int ath12k_dp_srng_common_setup(struct ath12k_base *ab) 461 { 462 struct ath12k_dp *dp = &ab->dp; 463 const struct ath12k_hal_tcl_to_wbm_rbm_map *map; 464 struct hal_srng *srng; 465 int i, ret, tx_comp_ring_num; 466 u32 ring_hash_map; 467 468 ret = ath12k_dp_srng_setup(ab, &dp->wbm_desc_rel_ring, 469 HAL_SW2WBM_RELEASE, 0, 0, 470 DP_WBM_RELEASE_RING_SIZE); 471 if (ret) { 472 ath12k_warn(ab, "failed to set up wbm2sw_release ring :%d\n", 473 ret); 474 goto err; 475 } 476 477 ret = ath12k_dp_srng_setup(ab, &dp->tcl_cmd_ring, HAL_TCL_CMD, 0, 0, 478 DP_TCL_CMD_RING_SIZE); 479 if (ret) { 480 ath12k_warn(ab, "failed to set up tcl_cmd ring :%d\n", ret); 481 goto err; 482 } 483 484 ret = ath12k_dp_srng_setup(ab, &dp->tcl_status_ring, HAL_TCL_STATUS, 485 0, 0, DP_TCL_STATUS_RING_SIZE); 486 if (ret) { 487 ath12k_warn(ab, "failed to set up tcl_status ring :%d\n", ret); 488 goto err; 489 } 490 491 for (i = 0; i < ab->hw_params->max_tx_ring; i++) { 492 map = ab->hw_params->hal_ops->tcl_to_wbm_rbm_map; 493 tx_comp_ring_num = map[i].wbm_ring_num; 494 495 ret = ath12k_dp_srng_setup(ab, &dp->tx_ring[i].tcl_data_ring, 496 HAL_TCL_DATA, i, 0, 497 DP_TCL_DATA_RING_SIZE); 498 if (ret) { 499 ath12k_warn(ab, "failed to set up tcl_data ring (%d) :%d\n", 500 i, ret); 501 goto err; 502 } 503 504 ret = ath12k_dp_srng_setup(ab, &dp->tx_ring[i].tcl_comp_ring, 505 HAL_WBM2SW_RELEASE, tx_comp_ring_num, 0, 506 DP_TX_COMP_RING_SIZE); 507 if (ret) { 508 ath12k_warn(ab, "failed to set up tcl_comp ring (%d) :%d\n", 509 tx_comp_ring_num, ret); 510 goto err; 511 } 512 } 513 514 ret = ath12k_dp_srng_setup(ab, &dp->reo_reinject_ring, HAL_REO_REINJECT, 515 0, 0, DP_REO_REINJECT_RING_SIZE); 516 if (ret) { 517 ath12k_warn(ab, "failed to set up reo_reinject ring :%d\n", 518 ret); 519 goto err; 520 } 521 522 ret = ath12k_dp_srng_setup(ab, &dp->rx_rel_ring, HAL_WBM2SW_RELEASE, 523 HAL_WBM2SW_REL_ERR_RING_NUM, 0, 524 DP_RX_RELEASE_RING_SIZE); 525 if (ret) { 526 ath12k_warn(ab, "failed to set up rx_rel ring :%d\n", ret); 527 goto err; 528 } 529 530 ret = ath12k_dp_srng_setup(ab, &dp->reo_except_ring, HAL_REO_EXCEPTION, 531 0, 0, DP_REO_EXCEPTION_RING_SIZE); 532 if (ret) { 533 ath12k_warn(ab, "failed to set up reo_exception ring :%d\n", 534 ret); 535 goto err; 536 } 537 538 ret = ath12k_dp_srng_setup(ab, &dp->reo_cmd_ring, HAL_REO_CMD, 539 0, 0, DP_REO_CMD_RING_SIZE); 540 if (ret) { 541 ath12k_warn(ab, "failed to set up reo_cmd ring :%d\n", ret); 542 goto err; 543 } 544 545 srng = &ab->hal.srng_list[dp->reo_cmd_ring.ring_id]; 546 ath12k_hal_reo_init_cmd_ring(ab, srng); 547 548 ret = ath12k_dp_srng_setup(ab, &dp->reo_status_ring, HAL_REO_STATUS, 549 0, 0, DP_REO_STATUS_RING_SIZE); 550 if (ret) { 551 ath12k_warn(ab, "failed to set up reo_status ring :%d\n", ret); 552 goto err; 553 } 554 555 /* When hash based routing of rx packet is enabled, 32 entries to map 556 * the hash values to the ring will be configured. Each hash entry uses 557 * four bits to map to a particular ring. The ring mapping will be 558 * 0:TCL, 1:SW1, 2:SW2, 3:SW3, 4:SW4, 5:Release, 6:FW and 7:SW5 559 * 8:SW6, 9:SW7, 10:SW8, 11:Not used. 560 */ 561 ring_hash_map = HAL_HASH_ROUTING_RING_SW1 | 562 HAL_HASH_ROUTING_RING_SW2 << 4 | 563 HAL_HASH_ROUTING_RING_SW3 << 8 | 564 HAL_HASH_ROUTING_RING_SW4 << 12 | 565 HAL_HASH_ROUTING_RING_SW1 << 16 | 566 HAL_HASH_ROUTING_RING_SW2 << 20 | 567 HAL_HASH_ROUTING_RING_SW3 << 24 | 568 HAL_HASH_ROUTING_RING_SW4 << 28; 569 570 ath12k_hal_reo_hw_setup(ab, ring_hash_map); 571 572 return 0; 573 574 err: 575 ath12k_dp_srng_common_cleanup(ab); 576 577 return ret; 578 } 579 580 static void ath12k_dp_scatter_idle_link_desc_cleanup(struct ath12k_base *ab) 581 { 582 struct ath12k_dp *dp = &ab->dp; 583 struct hal_wbm_idle_scatter_list *slist = dp->scatter_list; 584 int i; 585 586 for (i = 0; i < DP_IDLE_SCATTER_BUFS_MAX; i++) { 587 if (!slist[i].vaddr) 588 continue; 589 590 dma_free_coherent(ab->dev, HAL_WBM_IDLE_SCATTER_BUF_SIZE_MAX, 591 slist[i].vaddr, slist[i].paddr); 592 slist[i].vaddr = NULL; 593 } 594 } 595 596 static int ath12k_dp_scatter_idle_link_desc_setup(struct ath12k_base *ab, 597 int size, 598 u32 n_link_desc_bank, 599 u32 n_link_desc, 600 u32 last_bank_sz) 601 { 602 struct ath12k_dp *dp = &ab->dp; 603 struct dp_link_desc_bank *link_desc_banks = dp->link_desc_banks; 604 struct hal_wbm_idle_scatter_list *slist = dp->scatter_list; 605 u32 n_entries_per_buf; 606 int num_scatter_buf, scatter_idx; 607 struct hal_wbm_link_desc *scatter_buf; 608 int align_bytes, n_entries; 609 dma_addr_t paddr; 610 int rem_entries; 611 int i; 612 int ret = 0; 613 u32 end_offset, cookie; 614 615 n_entries_per_buf = HAL_WBM_IDLE_SCATTER_BUF_SIZE / 616 ath12k_hal_srng_get_entrysize(ab, HAL_WBM_IDLE_LINK); 617 num_scatter_buf = DIV_ROUND_UP(size, HAL_WBM_IDLE_SCATTER_BUF_SIZE); 618 619 if (num_scatter_buf > DP_IDLE_SCATTER_BUFS_MAX) 620 return -EINVAL; 621 622 for (i = 0; i < num_scatter_buf; i++) { 623 slist[i].vaddr = dma_alloc_coherent(ab->dev, 624 HAL_WBM_IDLE_SCATTER_BUF_SIZE_MAX, 625 &slist[i].paddr, GFP_KERNEL); 626 if (!slist[i].vaddr) { 627 ret = -ENOMEM; 628 goto err; 629 } 630 } 631 632 scatter_idx = 0; 633 scatter_buf = slist[scatter_idx].vaddr; 634 rem_entries = n_entries_per_buf; 635 636 for (i = 0; i < n_link_desc_bank; i++) { 637 align_bytes = link_desc_banks[i].vaddr - 638 link_desc_banks[i].vaddr_unaligned; 639 n_entries = (DP_LINK_DESC_ALLOC_SIZE_THRESH - align_bytes) / 640 HAL_LINK_DESC_SIZE; 641 paddr = link_desc_banks[i].paddr; 642 while (n_entries) { 643 cookie = DP_LINK_DESC_COOKIE_SET(n_entries, i); 644 ath12k_hal_set_link_desc_addr(scatter_buf, cookie, paddr); 645 n_entries--; 646 paddr += HAL_LINK_DESC_SIZE; 647 if (rem_entries) { 648 rem_entries--; 649 scatter_buf++; 650 continue; 651 } 652 653 rem_entries = n_entries_per_buf; 654 scatter_idx++; 655 scatter_buf = slist[scatter_idx].vaddr; 656 } 657 } 658 659 end_offset = (scatter_buf - slist[scatter_idx].vaddr) * 660 sizeof(struct hal_wbm_link_desc); 661 ath12k_hal_setup_link_idle_list(ab, slist, num_scatter_buf, 662 n_link_desc, end_offset); 663 664 return 0; 665 666 err: 667 ath12k_dp_scatter_idle_link_desc_cleanup(ab); 668 669 return ret; 670 } 671 672 static void 673 ath12k_dp_link_desc_bank_free(struct ath12k_base *ab, 674 struct dp_link_desc_bank *link_desc_banks) 675 { 676 int i; 677 678 for (i = 0; i < DP_LINK_DESC_BANKS_MAX; i++) { 679 if (link_desc_banks[i].vaddr_unaligned) { 680 dma_free_coherent(ab->dev, 681 link_desc_banks[i].size, 682 link_desc_banks[i].vaddr_unaligned, 683 link_desc_banks[i].paddr_unaligned); 684 link_desc_banks[i].vaddr_unaligned = NULL; 685 } 686 } 687 } 688 689 static int ath12k_dp_link_desc_bank_alloc(struct ath12k_base *ab, 690 struct dp_link_desc_bank *desc_bank, 691 int n_link_desc_bank, 692 int last_bank_sz) 693 { 694 struct ath12k_dp *dp = &ab->dp; 695 int i; 696 int ret = 0; 697 int desc_sz = DP_LINK_DESC_ALLOC_SIZE_THRESH; 698 699 for (i = 0; i < n_link_desc_bank; i++) { 700 if (i == (n_link_desc_bank - 1) && last_bank_sz) 701 desc_sz = last_bank_sz; 702 703 desc_bank[i].vaddr_unaligned = 704 dma_alloc_coherent(ab->dev, desc_sz, 705 &desc_bank[i].paddr_unaligned, 706 GFP_KERNEL); 707 if (!desc_bank[i].vaddr_unaligned) { 708 ret = -ENOMEM; 709 goto err; 710 } 711 712 desc_bank[i].vaddr = PTR_ALIGN(desc_bank[i].vaddr_unaligned, 713 HAL_LINK_DESC_ALIGN); 714 desc_bank[i].paddr = desc_bank[i].paddr_unaligned + 715 ((unsigned long)desc_bank[i].vaddr - 716 (unsigned long)desc_bank[i].vaddr_unaligned); 717 desc_bank[i].size = desc_sz; 718 } 719 720 return 0; 721 722 err: 723 ath12k_dp_link_desc_bank_free(ab, dp->link_desc_banks); 724 725 return ret; 726 } 727 728 void ath12k_dp_link_desc_cleanup(struct ath12k_base *ab, 729 struct dp_link_desc_bank *desc_bank, 730 u32 ring_type, struct dp_srng *ring) 731 { 732 ath12k_dp_link_desc_bank_free(ab, desc_bank); 733 734 if (ring_type != HAL_RXDMA_MONITOR_DESC) { 735 ath12k_dp_srng_cleanup(ab, ring); 736 ath12k_dp_scatter_idle_link_desc_cleanup(ab); 737 } 738 } 739 740 static int ath12k_wbm_idle_ring_setup(struct ath12k_base *ab, u32 *n_link_desc) 741 { 742 struct ath12k_dp *dp = &ab->dp; 743 u32 n_mpdu_link_desc, n_mpdu_queue_desc; 744 u32 n_tx_msdu_link_desc, n_rx_msdu_link_desc; 745 int ret = 0; 746 747 n_mpdu_link_desc = (DP_NUM_TIDS_MAX * DP_AVG_MPDUS_PER_TID_MAX) / 748 HAL_NUM_MPDUS_PER_LINK_DESC; 749 750 n_mpdu_queue_desc = n_mpdu_link_desc / 751 HAL_NUM_MPDU_LINKS_PER_QUEUE_DESC; 752 753 n_tx_msdu_link_desc = (DP_NUM_TIDS_MAX * DP_AVG_FLOWS_PER_TID * 754 DP_AVG_MSDUS_PER_FLOW) / 755 HAL_NUM_TX_MSDUS_PER_LINK_DESC; 756 757 n_rx_msdu_link_desc = (DP_NUM_TIDS_MAX * DP_AVG_MPDUS_PER_TID_MAX * 758 DP_AVG_MSDUS_PER_MPDU) / 759 HAL_NUM_RX_MSDUS_PER_LINK_DESC; 760 761 *n_link_desc = n_mpdu_link_desc + n_mpdu_queue_desc + 762 n_tx_msdu_link_desc + n_rx_msdu_link_desc; 763 764 if (*n_link_desc & (*n_link_desc - 1)) 765 *n_link_desc = 1 << fls(*n_link_desc); 766 767 ret = ath12k_dp_srng_setup(ab, &dp->wbm_idle_ring, 768 HAL_WBM_IDLE_LINK, 0, 0, *n_link_desc); 769 if (ret) { 770 ath12k_warn(ab, "failed to setup wbm_idle_ring: %d\n", ret); 771 return ret; 772 } 773 return ret; 774 } 775 776 int ath12k_dp_link_desc_setup(struct ath12k_base *ab, 777 struct dp_link_desc_bank *link_desc_banks, 778 u32 ring_type, struct hal_srng *srng, 779 u32 n_link_desc) 780 { 781 u32 tot_mem_sz; 782 u32 n_link_desc_bank, last_bank_sz; 783 u32 entry_sz, align_bytes, n_entries; 784 struct hal_wbm_link_desc *desc; 785 u32 paddr; 786 int i, ret; 787 u32 cookie; 788 789 tot_mem_sz = n_link_desc * HAL_LINK_DESC_SIZE; 790 tot_mem_sz += HAL_LINK_DESC_ALIGN; 791 792 if (tot_mem_sz <= DP_LINK_DESC_ALLOC_SIZE_THRESH) { 793 n_link_desc_bank = 1; 794 last_bank_sz = tot_mem_sz; 795 } else { 796 n_link_desc_bank = tot_mem_sz / 797 (DP_LINK_DESC_ALLOC_SIZE_THRESH - 798 HAL_LINK_DESC_ALIGN); 799 last_bank_sz = tot_mem_sz % 800 (DP_LINK_DESC_ALLOC_SIZE_THRESH - 801 HAL_LINK_DESC_ALIGN); 802 803 if (last_bank_sz) 804 n_link_desc_bank += 1; 805 } 806 807 if (n_link_desc_bank > DP_LINK_DESC_BANKS_MAX) 808 return -EINVAL; 809 810 ret = ath12k_dp_link_desc_bank_alloc(ab, link_desc_banks, 811 n_link_desc_bank, last_bank_sz); 812 if (ret) 813 return ret; 814 815 /* Setup link desc idle list for HW internal usage */ 816 entry_sz = ath12k_hal_srng_get_entrysize(ab, ring_type); 817 tot_mem_sz = entry_sz * n_link_desc; 818 819 /* Setup scatter desc list when the total memory requirement is more */ 820 if (tot_mem_sz > DP_LINK_DESC_ALLOC_SIZE_THRESH && 821 ring_type != HAL_RXDMA_MONITOR_DESC) { 822 ret = ath12k_dp_scatter_idle_link_desc_setup(ab, tot_mem_sz, 823 n_link_desc_bank, 824 n_link_desc, 825 last_bank_sz); 826 if (ret) { 827 ath12k_warn(ab, "failed to setup scatting idle list descriptor :%d\n", 828 ret); 829 goto fail_desc_bank_free; 830 } 831 832 return 0; 833 } 834 835 spin_lock_bh(&srng->lock); 836 837 ath12k_hal_srng_access_begin(ab, srng); 838 839 for (i = 0; i < n_link_desc_bank; i++) { 840 align_bytes = link_desc_banks[i].vaddr - 841 link_desc_banks[i].vaddr_unaligned; 842 n_entries = (link_desc_banks[i].size - align_bytes) / 843 HAL_LINK_DESC_SIZE; 844 paddr = link_desc_banks[i].paddr; 845 while (n_entries && 846 (desc = ath12k_hal_srng_src_get_next_entry(ab, srng))) { 847 cookie = DP_LINK_DESC_COOKIE_SET(n_entries, i); 848 ath12k_hal_set_link_desc_addr(desc, 849 cookie, paddr); 850 n_entries--; 851 paddr += HAL_LINK_DESC_SIZE; 852 } 853 } 854 855 ath12k_hal_srng_access_end(ab, srng); 856 857 spin_unlock_bh(&srng->lock); 858 859 return 0; 860 861 fail_desc_bank_free: 862 ath12k_dp_link_desc_bank_free(ab, link_desc_banks); 863 864 return ret; 865 } 866 867 int ath12k_dp_service_srng(struct ath12k_base *ab, 868 struct ath12k_ext_irq_grp *irq_grp, 869 int budget) 870 { 871 struct napi_struct *napi = &irq_grp->napi; 872 int grp_id = irq_grp->grp_id; 873 int work_done = 0; 874 int i = 0, j; 875 int tot_work_done = 0; 876 enum dp_monitor_mode monitor_mode; 877 u8 ring_mask; 878 879 while (i < ab->hw_params->max_tx_ring) { 880 if (ab->hw_params->ring_mask->tx[grp_id] & 881 BIT(ab->hw_params->hal_ops->tcl_to_wbm_rbm_map[i].wbm_ring_num)) 882 ath12k_dp_tx_completion_handler(ab, i); 883 i++; 884 } 885 886 if (ab->hw_params->ring_mask->rx_err[grp_id]) { 887 work_done = ath12k_dp_rx_process_err(ab, napi, budget); 888 budget -= work_done; 889 tot_work_done += work_done; 890 if (budget <= 0) 891 goto done; 892 } 893 894 if (ab->hw_params->ring_mask->rx_wbm_rel[grp_id]) { 895 work_done = ath12k_dp_rx_process_wbm_err(ab, 896 napi, 897 budget); 898 budget -= work_done; 899 tot_work_done += work_done; 900 901 if (budget <= 0) 902 goto done; 903 } 904 905 if (ab->hw_params->ring_mask->rx[grp_id]) { 906 i = fls(ab->hw_params->ring_mask->rx[grp_id]) - 1; 907 work_done = ath12k_dp_rx_process(ab, i, napi, 908 budget); 909 budget -= work_done; 910 tot_work_done += work_done; 911 if (budget <= 0) 912 goto done; 913 } 914 915 if (ab->hw_params->ring_mask->rx_mon_dest[grp_id]) { 916 monitor_mode = ATH12K_DP_RX_MONITOR_MODE; 917 ring_mask = ab->hw_params->ring_mask->rx_mon_dest[grp_id]; 918 for (i = 0; i < ab->num_radios; i++) { 919 for (j = 0; j < ab->hw_params->num_rxmda_per_pdev; j++) { 920 int id = i * ab->hw_params->num_rxmda_per_pdev + j; 921 922 if (ring_mask & BIT(id)) { 923 work_done = 924 ath12k_dp_mon_process_ring(ab, id, napi, budget, 925 monitor_mode); 926 budget -= work_done; 927 tot_work_done += work_done; 928 929 if (budget <= 0) 930 goto done; 931 } 932 } 933 } 934 } 935 936 if (ab->hw_params->ring_mask->tx_mon_dest[grp_id]) { 937 monitor_mode = ATH12K_DP_TX_MONITOR_MODE; 938 ring_mask = ab->hw_params->ring_mask->tx_mon_dest[grp_id]; 939 for (i = 0; i < ab->num_radios; i++) { 940 for (j = 0; j < ab->hw_params->num_rxmda_per_pdev; j++) { 941 int id = i * ab->hw_params->num_rxmda_per_pdev + j; 942 943 if (ring_mask & BIT(id)) { 944 work_done = 945 ath12k_dp_mon_process_ring(ab, id, napi, budget, 946 monitor_mode); 947 budget -= work_done; 948 tot_work_done += work_done; 949 950 if (budget <= 0) 951 goto done; 952 } 953 } 954 } 955 } 956 957 if (ab->hw_params->ring_mask->reo_status[grp_id]) 958 ath12k_dp_rx_process_reo_status(ab); 959 960 if (ab->hw_params->ring_mask->host2rxdma[grp_id]) { 961 struct ath12k_dp *dp = &ab->dp; 962 struct dp_rxdma_ring *rx_ring = &dp->rx_refill_buf_ring; 963 964 ath12k_dp_rx_bufs_replenish(ab, rx_ring, 0); 965 } 966 967 /* TODO: Implement handler for other interrupts */ 968 969 done: 970 return tot_work_done; 971 } 972 973 void ath12k_dp_pdev_free(struct ath12k_base *ab) 974 { 975 int i; 976 977 del_timer_sync(&ab->mon_reap_timer); 978 979 for (i = 0; i < ab->num_radios; i++) 980 ath12k_dp_rx_pdev_free(ab, i); 981 } 982 983 void ath12k_dp_pdev_pre_alloc(struct ath12k_base *ab) 984 { 985 struct ath12k *ar; 986 struct ath12k_pdev_dp *dp; 987 int i; 988 989 for (i = 0; i < ab->num_radios; i++) { 990 ar = ab->pdevs[i].ar; 991 dp = &ar->dp; 992 dp->mac_id = i; 993 atomic_set(&dp->num_tx_pending, 0); 994 init_waitqueue_head(&dp->tx_empty_waitq); 995 996 /* TODO: Add any RXDMA setup required per pdev */ 997 } 998 } 999 1000 bool ath12k_dp_wmask_compaction_rx_tlv_supported(struct ath12k_base *ab) 1001 { 1002 if (test_bit(WMI_TLV_SERVICE_WMSK_COMPACTION_RX_TLVS, ab->wmi_ab.svc_map) && 1003 ab->hw_params->hal_ops->rxdma_ring_wmask_rx_mpdu_start && 1004 ab->hw_params->hal_ops->rxdma_ring_wmask_rx_msdu_end && 1005 ab->hw_params->hal_ops->get_hal_rx_compact_ops) { 1006 return true; 1007 } 1008 return false; 1009 } 1010 1011 void ath12k_dp_hal_rx_desc_init(struct ath12k_base *ab) 1012 { 1013 if (ath12k_dp_wmask_compaction_rx_tlv_supported(ab)) { 1014 /* RX TLVS compaction is supported, hence change the hal_rx_ops 1015 * to compact hal_rx_ops. 1016 */ 1017 ab->hal_rx_ops = ab->hw_params->hal_ops->get_hal_rx_compact_ops(); 1018 } 1019 ab->hal.hal_desc_sz = 1020 ab->hal_rx_ops->rx_desc_get_desc_size(); 1021 } 1022 1023 static void ath12k_dp_service_mon_ring(struct timer_list *t) 1024 { 1025 struct ath12k_base *ab = from_timer(ab, t, mon_reap_timer); 1026 int i; 1027 1028 for (i = 0; i < ab->hw_params->num_rxmda_per_pdev; i++) 1029 ath12k_dp_mon_process_ring(ab, i, NULL, DP_MON_SERVICE_BUDGET, 1030 ATH12K_DP_RX_MONITOR_MODE); 1031 1032 mod_timer(&ab->mon_reap_timer, jiffies + 1033 msecs_to_jiffies(ATH12K_MON_TIMER_INTERVAL)); 1034 } 1035 1036 static void ath12k_dp_mon_reap_timer_init(struct ath12k_base *ab) 1037 { 1038 if (ab->hw_params->rxdma1_enable) 1039 return; 1040 1041 timer_setup(&ab->mon_reap_timer, ath12k_dp_service_mon_ring, 0); 1042 } 1043 1044 int ath12k_dp_pdev_alloc(struct ath12k_base *ab) 1045 { 1046 struct ath12k *ar; 1047 int ret; 1048 int i; 1049 1050 ret = ath12k_dp_rx_htt_setup(ab); 1051 if (ret) 1052 goto out; 1053 1054 ath12k_dp_mon_reap_timer_init(ab); 1055 1056 /* TODO: Per-pdev rx ring unlike tx ring which is mapped to different AC's */ 1057 for (i = 0; i < ab->num_radios; i++) { 1058 ar = ab->pdevs[i].ar; 1059 ret = ath12k_dp_rx_pdev_alloc(ab, i); 1060 if (ret) { 1061 ath12k_warn(ab, "failed to allocate pdev rx for pdev_id :%d\n", 1062 i); 1063 goto err; 1064 } 1065 ret = ath12k_dp_rx_pdev_mon_attach(ar); 1066 if (ret) { 1067 ath12k_warn(ab, "failed to initialize mon pdev %d\n", i); 1068 goto err; 1069 } 1070 } 1071 1072 return 0; 1073 err: 1074 ath12k_dp_pdev_free(ab); 1075 out: 1076 return ret; 1077 } 1078 1079 int ath12k_dp_htt_connect(struct ath12k_dp *dp) 1080 { 1081 struct ath12k_htc_svc_conn_req conn_req = {0}; 1082 struct ath12k_htc_svc_conn_resp conn_resp = {0}; 1083 int status; 1084 1085 conn_req.ep_ops.ep_tx_complete = ath12k_dp_htt_htc_tx_complete; 1086 conn_req.ep_ops.ep_rx_complete = ath12k_dp_htt_htc_t2h_msg_handler; 1087 1088 /* connect to control service */ 1089 conn_req.service_id = ATH12K_HTC_SVC_ID_HTT_DATA_MSG; 1090 1091 status = ath12k_htc_connect_service(&dp->ab->htc, &conn_req, 1092 &conn_resp); 1093 1094 if (status) 1095 return status; 1096 1097 dp->eid = conn_resp.eid; 1098 1099 return 0; 1100 } 1101 1102 static void ath12k_dp_update_vdev_search(struct ath12k_vif *arvif) 1103 { 1104 switch (arvif->vdev_type) { 1105 case WMI_VDEV_TYPE_STA: 1106 /* TODO: Verify the search type and flags since ast hash 1107 * is not part of peer mapv3 1108 */ 1109 arvif->hal_addr_search_flags = HAL_TX_ADDRY_EN; 1110 arvif->search_type = HAL_TX_ADDR_SEARCH_DEFAULT; 1111 break; 1112 case WMI_VDEV_TYPE_AP: 1113 case WMI_VDEV_TYPE_IBSS: 1114 arvif->hal_addr_search_flags = HAL_TX_ADDRX_EN; 1115 arvif->search_type = HAL_TX_ADDR_SEARCH_DEFAULT; 1116 break; 1117 case WMI_VDEV_TYPE_MONITOR: 1118 default: 1119 return; 1120 } 1121 } 1122 1123 void ath12k_dp_vdev_tx_attach(struct ath12k *ar, struct ath12k_vif *arvif) 1124 { 1125 struct ath12k_base *ab = ar->ab; 1126 1127 arvif->tcl_metadata |= u32_encode_bits(1, HTT_TCL_META_DATA_TYPE) | 1128 u32_encode_bits(arvif->vdev_id, 1129 HTT_TCL_META_DATA_VDEV_ID) | 1130 u32_encode_bits(ar->pdev->pdev_id, 1131 HTT_TCL_META_DATA_PDEV_ID); 1132 1133 /* set HTT extension valid bit to 0 by default */ 1134 arvif->tcl_metadata &= ~HTT_TCL_META_DATA_VALID_HTT; 1135 1136 ath12k_dp_update_vdev_search(arvif); 1137 arvif->vdev_id_check_en = true; 1138 arvif->bank_id = ath12k_dp_tx_get_bank_profile(ab, arvif, &ab->dp); 1139 1140 /* TODO: error path for bank id failure */ 1141 if (arvif->bank_id == DP_INVALID_BANK_ID) { 1142 ath12k_err(ar->ab, "Failed to initialize DP TX Banks"); 1143 return; 1144 } 1145 } 1146 1147 static void ath12k_dp_cc_cleanup(struct ath12k_base *ab) 1148 { 1149 struct ath12k_rx_desc_info *desc_info, *tmp; 1150 struct ath12k_tx_desc_info *tx_desc_info, *tmp1; 1151 struct ath12k_dp *dp = &ab->dp; 1152 struct sk_buff *skb; 1153 int i; 1154 u32 pool_id, tx_spt_page; 1155 1156 if (!dp->spt_info) 1157 return; 1158 1159 /* RX Descriptor cleanup */ 1160 spin_lock_bh(&dp->rx_desc_lock); 1161 1162 list_for_each_entry_safe(desc_info, tmp, &dp->rx_desc_used_list, list) { 1163 list_del(&desc_info->list); 1164 skb = desc_info->skb; 1165 1166 if (!skb) 1167 continue; 1168 1169 dma_unmap_single(ab->dev, ATH12K_SKB_RXCB(skb)->paddr, 1170 skb->len + skb_tailroom(skb), DMA_FROM_DEVICE); 1171 dev_kfree_skb_any(skb); 1172 } 1173 1174 for (i = 0; i < ATH12K_NUM_RX_SPT_PAGES; i++) { 1175 if (!dp->spt_info->rxbaddr[i]) 1176 continue; 1177 1178 kfree(dp->spt_info->rxbaddr[i]); 1179 dp->spt_info->rxbaddr[i] = NULL; 1180 } 1181 1182 spin_unlock_bh(&dp->rx_desc_lock); 1183 1184 /* TX Descriptor cleanup */ 1185 for (i = 0; i < ATH12K_HW_MAX_QUEUES; i++) { 1186 spin_lock_bh(&dp->tx_desc_lock[i]); 1187 1188 list_for_each_entry_safe(tx_desc_info, tmp1, &dp->tx_desc_used_list[i], 1189 list) { 1190 list_del(&tx_desc_info->list); 1191 skb = tx_desc_info->skb; 1192 1193 if (!skb) 1194 continue; 1195 1196 dma_unmap_single(ab->dev, ATH12K_SKB_CB(skb)->paddr, 1197 skb->len, DMA_TO_DEVICE); 1198 dev_kfree_skb_any(skb); 1199 } 1200 1201 spin_unlock_bh(&dp->tx_desc_lock[i]); 1202 } 1203 1204 for (pool_id = 0; pool_id < ATH12K_HW_MAX_QUEUES; pool_id++) { 1205 spin_lock_bh(&dp->tx_desc_lock[pool_id]); 1206 1207 for (i = 0; i < ATH12K_TX_SPT_PAGES_PER_POOL; i++) { 1208 tx_spt_page = i + pool_id * ATH12K_TX_SPT_PAGES_PER_POOL; 1209 if (!dp->spt_info->txbaddr[tx_spt_page]) 1210 continue; 1211 1212 kfree(dp->spt_info->txbaddr[tx_spt_page]); 1213 dp->spt_info->txbaddr[tx_spt_page] = NULL; 1214 } 1215 1216 spin_unlock_bh(&dp->tx_desc_lock[pool_id]); 1217 } 1218 1219 /* unmap SPT pages */ 1220 for (i = 0; i < dp->num_spt_pages; i++) { 1221 if (!dp->spt_info[i].vaddr) 1222 continue; 1223 1224 dma_free_coherent(ab->dev, ATH12K_PAGE_SIZE, 1225 dp->spt_info[i].vaddr, dp->spt_info[i].paddr); 1226 dp->spt_info[i].vaddr = NULL; 1227 } 1228 1229 kfree(dp->spt_info); 1230 } 1231 1232 static void ath12k_dp_reoq_lut_cleanup(struct ath12k_base *ab) 1233 { 1234 struct ath12k_dp *dp = &ab->dp; 1235 1236 if (!ab->hw_params->reoq_lut_support) 1237 return; 1238 1239 if (!dp->reoq_lut.vaddr) 1240 return; 1241 1242 dma_free_coherent(ab->dev, DP_REOQ_LUT_SIZE, 1243 dp->reoq_lut.vaddr, dp->reoq_lut.paddr); 1244 dp->reoq_lut.vaddr = NULL; 1245 1246 ath12k_hif_write32(ab, 1247 HAL_SEQ_WCSS_UMAC_REO_REG + HAL_REO1_QDESC_LUT_BASE0(ab), 0); 1248 } 1249 1250 void ath12k_dp_free(struct ath12k_base *ab) 1251 { 1252 struct ath12k_dp *dp = &ab->dp; 1253 int i; 1254 1255 ath12k_dp_link_desc_cleanup(ab, dp->link_desc_banks, 1256 HAL_WBM_IDLE_LINK, &dp->wbm_idle_ring); 1257 1258 ath12k_dp_cc_cleanup(ab); 1259 ath12k_dp_reoq_lut_cleanup(ab); 1260 ath12k_dp_deinit_bank_profiles(ab); 1261 ath12k_dp_srng_common_cleanup(ab); 1262 1263 ath12k_dp_rx_reo_cmd_list_cleanup(ab); 1264 1265 for (i = 0; i < ab->hw_params->max_tx_ring; i++) 1266 kfree(dp->tx_ring[i].tx_status); 1267 1268 ath12k_dp_rx_free(ab); 1269 /* Deinit any SOC level resource */ 1270 } 1271 1272 void ath12k_dp_cc_config(struct ath12k_base *ab) 1273 { 1274 u32 cmem_base = ab->qmi.dev_mem[ATH12K_QMI_DEVMEM_CMEM_INDEX].start; 1275 u32 reo_base = HAL_SEQ_WCSS_UMAC_REO_REG; 1276 u32 wbm_base = HAL_SEQ_WCSS_UMAC_WBM_REG; 1277 u32 val = 0; 1278 1279 ath12k_hif_write32(ab, reo_base + HAL_REO1_SW_COOKIE_CFG0(ab), cmem_base); 1280 1281 val |= u32_encode_bits(ATH12K_CMEM_ADDR_MSB, 1282 HAL_REO1_SW_COOKIE_CFG_CMEM_BASE_ADDR_MSB) | 1283 u32_encode_bits(ATH12K_CC_PPT_MSB, 1284 HAL_REO1_SW_COOKIE_CFG_COOKIE_PPT_MSB) | 1285 u32_encode_bits(ATH12K_CC_SPT_MSB, 1286 HAL_REO1_SW_COOKIE_CFG_COOKIE_SPT_MSB) | 1287 u32_encode_bits(1, HAL_REO1_SW_COOKIE_CFG_ALIGN) | 1288 u32_encode_bits(1, HAL_REO1_SW_COOKIE_CFG_ENABLE) | 1289 u32_encode_bits(1, HAL_REO1_SW_COOKIE_CFG_GLOBAL_ENABLE); 1290 1291 ath12k_hif_write32(ab, reo_base + HAL_REO1_SW_COOKIE_CFG1(ab), val); 1292 1293 /* Enable HW CC for WBM */ 1294 ath12k_hif_write32(ab, wbm_base + HAL_WBM_SW_COOKIE_CFG0, cmem_base); 1295 1296 val = u32_encode_bits(ATH12K_CMEM_ADDR_MSB, 1297 HAL_WBM_SW_COOKIE_CFG_CMEM_BASE_ADDR_MSB) | 1298 u32_encode_bits(ATH12K_CC_PPT_MSB, 1299 HAL_WBM_SW_COOKIE_CFG_COOKIE_PPT_MSB) | 1300 u32_encode_bits(ATH12K_CC_SPT_MSB, 1301 HAL_WBM_SW_COOKIE_CFG_COOKIE_SPT_MSB) | 1302 u32_encode_bits(1, HAL_WBM_SW_COOKIE_CFG_ALIGN); 1303 1304 ath12k_hif_write32(ab, wbm_base + HAL_WBM_SW_COOKIE_CFG1, val); 1305 1306 /* Enable conversion complete indication */ 1307 val = ath12k_hif_read32(ab, wbm_base + HAL_WBM_SW_COOKIE_CFG2); 1308 val |= u32_encode_bits(1, HAL_WBM_SW_COOKIE_CFG_RELEASE_PATH_EN) | 1309 u32_encode_bits(1, HAL_WBM_SW_COOKIE_CFG_ERR_PATH_EN) | 1310 u32_encode_bits(1, HAL_WBM_SW_COOKIE_CFG_CONV_IND_EN); 1311 1312 ath12k_hif_write32(ab, wbm_base + HAL_WBM_SW_COOKIE_CFG2, val); 1313 1314 /* Enable Cookie conversion for WBM2SW Rings */ 1315 val = ath12k_hif_read32(ab, wbm_base + HAL_WBM_SW_COOKIE_CONVERT_CFG); 1316 val |= u32_encode_bits(1, HAL_WBM_SW_COOKIE_CONV_CFG_GLOBAL_EN) | 1317 ab->hw_params->hal_params->wbm2sw_cc_enable; 1318 1319 ath12k_hif_write32(ab, wbm_base + HAL_WBM_SW_COOKIE_CONVERT_CFG, val); 1320 } 1321 1322 static u32 ath12k_dp_cc_cookie_gen(u16 ppt_idx, u16 spt_idx) 1323 { 1324 return (u32)ppt_idx << ATH12K_CC_PPT_SHIFT | spt_idx; 1325 } 1326 1327 static inline void *ath12k_dp_cc_get_desc_addr_ptr(struct ath12k_base *ab, 1328 u16 ppt_idx, u16 spt_idx) 1329 { 1330 struct ath12k_dp *dp = &ab->dp; 1331 1332 return dp->spt_info[ppt_idx].vaddr + spt_idx; 1333 } 1334 1335 struct ath12k_rx_desc_info *ath12k_dp_get_rx_desc(struct ath12k_base *ab, 1336 u32 cookie) 1337 { 1338 struct ath12k_rx_desc_info **desc_addr_ptr; 1339 u16 ppt_idx, spt_idx; 1340 1341 ppt_idx = u32_get_bits(cookie, ATH12K_DP_CC_COOKIE_PPT); 1342 spt_idx = u32_get_bits(cookie, ATH12k_DP_CC_COOKIE_SPT); 1343 1344 if (ppt_idx > ATH12K_NUM_RX_SPT_PAGES || 1345 spt_idx > ATH12K_MAX_SPT_ENTRIES) 1346 return NULL; 1347 1348 desc_addr_ptr = ath12k_dp_cc_get_desc_addr_ptr(ab, ppt_idx, spt_idx); 1349 1350 return *desc_addr_ptr; 1351 } 1352 1353 struct ath12k_tx_desc_info *ath12k_dp_get_tx_desc(struct ath12k_base *ab, 1354 u32 cookie) 1355 { 1356 struct ath12k_tx_desc_info **desc_addr_ptr; 1357 u16 ppt_idx, spt_idx; 1358 1359 ppt_idx = u32_get_bits(cookie, ATH12K_DP_CC_COOKIE_PPT); 1360 spt_idx = u32_get_bits(cookie, ATH12k_DP_CC_COOKIE_SPT); 1361 1362 if (ppt_idx < ATH12K_NUM_RX_SPT_PAGES || 1363 ppt_idx > ab->dp.num_spt_pages || 1364 spt_idx > ATH12K_MAX_SPT_ENTRIES) 1365 return NULL; 1366 1367 desc_addr_ptr = ath12k_dp_cc_get_desc_addr_ptr(ab, ppt_idx, spt_idx); 1368 1369 return *desc_addr_ptr; 1370 } 1371 1372 static int ath12k_dp_cc_desc_init(struct ath12k_base *ab) 1373 { 1374 struct ath12k_dp *dp = &ab->dp; 1375 struct ath12k_rx_desc_info *rx_descs, **rx_desc_addr; 1376 struct ath12k_tx_desc_info *tx_descs, **tx_desc_addr; 1377 u32 i, j, pool_id, tx_spt_page; 1378 u32 ppt_idx; 1379 1380 spin_lock_bh(&dp->rx_desc_lock); 1381 1382 /* First ATH12K_NUM_RX_SPT_PAGES of allocated SPT pages are used for RX */ 1383 for (i = 0; i < ATH12K_NUM_RX_SPT_PAGES; i++) { 1384 rx_descs = kcalloc(ATH12K_MAX_SPT_ENTRIES, sizeof(*rx_descs), 1385 GFP_ATOMIC); 1386 1387 if (!rx_descs) { 1388 spin_unlock_bh(&dp->rx_desc_lock); 1389 return -ENOMEM; 1390 } 1391 1392 dp->spt_info->rxbaddr[i] = &rx_descs[0]; 1393 1394 for (j = 0; j < ATH12K_MAX_SPT_ENTRIES; j++) { 1395 rx_descs[j].cookie = ath12k_dp_cc_cookie_gen(i, j); 1396 rx_descs[j].magic = ATH12K_DP_RX_DESC_MAGIC; 1397 list_add_tail(&rx_descs[j].list, &dp->rx_desc_free_list); 1398 1399 /* Update descriptor VA in SPT */ 1400 rx_desc_addr = ath12k_dp_cc_get_desc_addr_ptr(ab, i, j); 1401 *rx_desc_addr = &rx_descs[j]; 1402 } 1403 } 1404 1405 spin_unlock_bh(&dp->rx_desc_lock); 1406 1407 for (pool_id = 0; pool_id < ATH12K_HW_MAX_QUEUES; pool_id++) { 1408 spin_lock_bh(&dp->tx_desc_lock[pool_id]); 1409 for (i = 0; i < ATH12K_TX_SPT_PAGES_PER_POOL; i++) { 1410 tx_descs = kcalloc(ATH12K_MAX_SPT_ENTRIES, sizeof(*tx_descs), 1411 GFP_ATOMIC); 1412 1413 if (!tx_descs) { 1414 spin_unlock_bh(&dp->tx_desc_lock[pool_id]); 1415 /* Caller takes care of TX pending and RX desc cleanup */ 1416 return -ENOMEM; 1417 } 1418 1419 tx_spt_page = i + pool_id * ATH12K_TX_SPT_PAGES_PER_POOL; 1420 dp->spt_info->txbaddr[tx_spt_page] = &tx_descs[0]; 1421 1422 for (j = 0; j < ATH12K_MAX_SPT_ENTRIES; j++) { 1423 ppt_idx = ATH12K_NUM_RX_SPT_PAGES + tx_spt_page; 1424 tx_descs[j].desc_id = ath12k_dp_cc_cookie_gen(ppt_idx, j); 1425 tx_descs[j].pool_id = pool_id; 1426 list_add_tail(&tx_descs[j].list, 1427 &dp->tx_desc_free_list[pool_id]); 1428 1429 /* Update descriptor VA in SPT */ 1430 tx_desc_addr = 1431 ath12k_dp_cc_get_desc_addr_ptr(ab, ppt_idx, j); 1432 *tx_desc_addr = &tx_descs[j]; 1433 } 1434 } 1435 spin_unlock_bh(&dp->tx_desc_lock[pool_id]); 1436 } 1437 return 0; 1438 } 1439 1440 static int ath12k_dp_cc_init(struct ath12k_base *ab) 1441 { 1442 struct ath12k_dp *dp = &ab->dp; 1443 int i, ret = 0; 1444 u32 cmem_base; 1445 1446 INIT_LIST_HEAD(&dp->rx_desc_free_list); 1447 INIT_LIST_HEAD(&dp->rx_desc_used_list); 1448 spin_lock_init(&dp->rx_desc_lock); 1449 1450 for (i = 0; i < ATH12K_HW_MAX_QUEUES; i++) { 1451 INIT_LIST_HEAD(&dp->tx_desc_free_list[i]); 1452 INIT_LIST_HEAD(&dp->tx_desc_used_list[i]); 1453 spin_lock_init(&dp->tx_desc_lock[i]); 1454 } 1455 1456 dp->num_spt_pages = ATH12K_NUM_SPT_PAGES; 1457 if (dp->num_spt_pages > ATH12K_MAX_PPT_ENTRIES) 1458 dp->num_spt_pages = ATH12K_MAX_PPT_ENTRIES; 1459 1460 dp->spt_info = kcalloc(dp->num_spt_pages, sizeof(struct ath12k_spt_info), 1461 GFP_KERNEL); 1462 1463 if (!dp->spt_info) { 1464 ath12k_warn(ab, "SPT page allocation failure"); 1465 return -ENOMEM; 1466 } 1467 1468 cmem_base = ab->qmi.dev_mem[ATH12K_QMI_DEVMEM_CMEM_INDEX].start; 1469 1470 for (i = 0; i < dp->num_spt_pages; i++) { 1471 dp->spt_info[i].vaddr = dma_alloc_coherent(ab->dev, 1472 ATH12K_PAGE_SIZE, 1473 &dp->spt_info[i].paddr, 1474 GFP_KERNEL); 1475 1476 if (!dp->spt_info[i].vaddr) { 1477 ret = -ENOMEM; 1478 goto free; 1479 } 1480 1481 if (dp->spt_info[i].paddr & ATH12K_SPT_4K_ALIGN_CHECK) { 1482 ath12k_warn(ab, "SPT allocated memory is not 4K aligned"); 1483 ret = -EINVAL; 1484 goto free; 1485 } 1486 1487 /* Write to PPT in CMEM */ 1488 ath12k_hif_write32(ab, cmem_base + ATH12K_PPT_ADDR_OFFSET(i), 1489 dp->spt_info[i].paddr >> ATH12K_SPT_4K_ALIGN_OFFSET); 1490 } 1491 1492 ret = ath12k_dp_cc_desc_init(ab); 1493 if (ret) { 1494 ath12k_warn(ab, "HW CC desc init failed %d", ret); 1495 goto free; 1496 } 1497 1498 return 0; 1499 free: 1500 ath12k_dp_cc_cleanup(ab); 1501 return ret; 1502 } 1503 1504 static int ath12k_dp_reoq_lut_setup(struct ath12k_base *ab) 1505 { 1506 struct ath12k_dp *dp = &ab->dp; 1507 1508 if (!ab->hw_params->reoq_lut_support) 1509 return 0; 1510 1511 dp->reoq_lut.vaddr = dma_alloc_coherent(ab->dev, 1512 DP_REOQ_LUT_SIZE, 1513 &dp->reoq_lut.paddr, 1514 GFP_KERNEL | __GFP_ZERO); 1515 if (!dp->reoq_lut.vaddr) { 1516 ath12k_warn(ab, "failed to allocate memory for reoq table"); 1517 return -ENOMEM; 1518 } 1519 1520 ath12k_hif_write32(ab, HAL_SEQ_WCSS_UMAC_REO_REG + HAL_REO1_QDESC_LUT_BASE0(ab), 1521 dp->reoq_lut.paddr); 1522 return 0; 1523 } 1524 1525 int ath12k_dp_alloc(struct ath12k_base *ab) 1526 { 1527 struct ath12k_dp *dp = &ab->dp; 1528 struct hal_srng *srng = NULL; 1529 size_t size = 0; 1530 u32 n_link_desc = 0; 1531 int ret; 1532 int i; 1533 1534 dp->ab = ab; 1535 1536 INIT_LIST_HEAD(&dp->reo_cmd_list); 1537 INIT_LIST_HEAD(&dp->reo_cmd_cache_flush_list); 1538 spin_lock_init(&dp->reo_cmd_lock); 1539 1540 dp->reo_cmd_cache_flush_count = 0; 1541 1542 ret = ath12k_wbm_idle_ring_setup(ab, &n_link_desc); 1543 if (ret) { 1544 ath12k_warn(ab, "failed to setup wbm_idle_ring: %d\n", ret); 1545 return ret; 1546 } 1547 1548 srng = &ab->hal.srng_list[dp->wbm_idle_ring.ring_id]; 1549 1550 ret = ath12k_dp_link_desc_setup(ab, dp->link_desc_banks, 1551 HAL_WBM_IDLE_LINK, srng, n_link_desc); 1552 if (ret) { 1553 ath12k_warn(ab, "failed to setup link desc: %d\n", ret); 1554 return ret; 1555 } 1556 1557 ret = ath12k_dp_cc_init(ab); 1558 1559 if (ret) { 1560 ath12k_warn(ab, "failed to setup cookie converter %d\n", ret); 1561 goto fail_link_desc_cleanup; 1562 } 1563 ret = ath12k_dp_init_bank_profiles(ab); 1564 if (ret) { 1565 ath12k_warn(ab, "failed to setup bank profiles %d\n", ret); 1566 goto fail_hw_cc_cleanup; 1567 } 1568 1569 ret = ath12k_dp_srng_common_setup(ab); 1570 if (ret) 1571 goto fail_dp_bank_profiles_cleanup; 1572 1573 size = sizeof(struct hal_wbm_release_ring_tx) * DP_TX_COMP_RING_SIZE; 1574 1575 ret = ath12k_dp_reoq_lut_setup(ab); 1576 if (ret) { 1577 ath12k_warn(ab, "failed to setup reoq table %d\n", ret); 1578 goto fail_cmn_srng_cleanup; 1579 } 1580 1581 for (i = 0; i < ab->hw_params->max_tx_ring; i++) { 1582 dp->tx_ring[i].tcl_data_ring_id = i; 1583 1584 dp->tx_ring[i].tx_status_head = 0; 1585 dp->tx_ring[i].tx_status_tail = DP_TX_COMP_RING_SIZE - 1; 1586 dp->tx_ring[i].tx_status = kmalloc(size, GFP_KERNEL); 1587 if (!dp->tx_ring[i].tx_status) { 1588 ret = -ENOMEM; 1589 /* FIXME: The allocated tx status is not freed 1590 * properly here 1591 */ 1592 goto fail_cmn_reoq_cleanup; 1593 } 1594 } 1595 1596 for (i = 0; i < HAL_DSCP_TID_MAP_TBL_NUM_ENTRIES_MAX; i++) 1597 ath12k_hal_tx_set_dscp_tid_map(ab, i); 1598 1599 ret = ath12k_dp_rx_alloc(ab); 1600 if (ret) 1601 goto fail_dp_rx_free; 1602 1603 /* Init any SOC level resource for DP */ 1604 1605 return 0; 1606 1607 fail_dp_rx_free: 1608 ath12k_dp_rx_free(ab); 1609 1610 fail_cmn_reoq_cleanup: 1611 ath12k_dp_reoq_lut_cleanup(ab); 1612 1613 fail_cmn_srng_cleanup: 1614 ath12k_dp_srng_common_cleanup(ab); 1615 1616 fail_dp_bank_profiles_cleanup: 1617 ath12k_dp_deinit_bank_profiles(ab); 1618 1619 fail_hw_cc_cleanup: 1620 ath12k_dp_cc_cleanup(ab); 1621 1622 fail_link_desc_cleanup: 1623 ath12k_dp_link_desc_cleanup(ab, dp->link_desc_banks, 1624 HAL_WBM_IDLE_LINK, &dp->wbm_idle_ring); 1625 1626 return ret; 1627 } 1628