1 // SPDX-License-Identifier: BSD-3-Clause-Clear 2 /* 3 * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved. 4 * Copyright (c) 2021-2024 Qualcomm Innovation Center, Inc. All rights reserved. 5 */ 6 7 #include <crypto/hash.h> 8 #include "core.h" 9 #include "dp_tx.h" 10 #include "hal_tx.h" 11 #include "hif.h" 12 #include "debug.h" 13 #include "dp_rx.h" 14 #include "peer.h" 15 #include "dp_mon.h" 16 17 static void ath12k_dp_htt_htc_tx_complete(struct ath12k_base *ab, 18 struct sk_buff *skb) 19 { 20 dev_kfree_skb_any(skb); 21 } 22 23 void ath12k_dp_peer_cleanup(struct ath12k *ar, int vdev_id, const u8 *addr) 24 { 25 struct ath12k_base *ab = ar->ab; 26 struct ath12k_peer *peer; 27 28 /* TODO: Any other peer specific DP cleanup */ 29 30 spin_lock_bh(&ab->base_lock); 31 peer = ath12k_peer_find(ab, vdev_id, addr); 32 if (!peer) { 33 ath12k_warn(ab, "failed to lookup peer %pM on vdev %d\n", 34 addr, vdev_id); 35 spin_unlock_bh(&ab->base_lock); 36 return; 37 } 38 39 ath12k_dp_rx_peer_tid_cleanup(ar, peer); 40 crypto_free_shash(peer->tfm_mmic); 41 peer->dp_setup_done = false; 42 spin_unlock_bh(&ab->base_lock); 43 } 44 45 int ath12k_dp_peer_setup(struct ath12k *ar, int vdev_id, const u8 *addr) 46 { 47 struct ath12k_base *ab = ar->ab; 48 struct ath12k_peer *peer; 49 u32 reo_dest; 50 int ret = 0, tid; 51 52 /* NOTE: reo_dest ring id starts from 1 unlike mac_id which starts from 0 */ 53 reo_dest = ar->dp.mac_id + 1; 54 ret = ath12k_wmi_set_peer_param(ar, addr, vdev_id, 55 WMI_PEER_SET_DEFAULT_ROUTING, 56 DP_RX_HASH_ENABLE | (reo_dest << 1)); 57 58 if (ret) { 59 ath12k_warn(ab, "failed to set default routing %d peer :%pM vdev_id :%d\n", 60 ret, addr, vdev_id); 61 return ret; 62 } 63 64 for (tid = 0; tid <= IEEE80211_NUM_TIDS; tid++) { 65 ret = ath12k_dp_rx_peer_tid_setup(ar, addr, vdev_id, tid, 1, 0, 66 HAL_PN_TYPE_NONE); 67 if (ret) { 68 ath12k_warn(ab, "failed to setup rxd tid queue for tid %d: %d\n", 69 tid, ret); 70 goto peer_clean; 71 } 72 } 73 74 ret = ath12k_dp_rx_peer_frag_setup(ar, addr, vdev_id); 75 if (ret) { 76 ath12k_warn(ab, "failed to setup rx defrag context\n"); 77 goto peer_clean; 78 } 79 80 /* TODO: Setup other peer specific resource used in data path */ 81 82 return 0; 83 84 peer_clean: 85 spin_lock_bh(&ab->base_lock); 86 87 peer = ath12k_peer_find(ab, vdev_id, addr); 88 if (!peer) { 89 ath12k_warn(ab, "failed to find the peer to del rx tid\n"); 90 spin_unlock_bh(&ab->base_lock); 91 return -ENOENT; 92 } 93 94 for (; tid >= 0; tid--) 95 ath12k_dp_rx_peer_tid_delete(ar, peer, tid); 96 97 spin_unlock_bh(&ab->base_lock); 98 99 return ret; 100 } 101 102 void ath12k_dp_srng_cleanup(struct ath12k_base *ab, struct dp_srng *ring) 103 { 104 if (!ring->vaddr_unaligned) 105 return; 106 107 dma_free_coherent(ab->dev, ring->size, ring->vaddr_unaligned, 108 ring->paddr_unaligned); 109 110 ring->vaddr_unaligned = NULL; 111 } 112 113 static int ath12k_dp_srng_find_ring_in_mask(int ring_num, const u8 *grp_mask) 114 { 115 int ext_group_num; 116 u8 mask = 1 << ring_num; 117 118 for (ext_group_num = 0; ext_group_num < ATH12K_EXT_IRQ_GRP_NUM_MAX; 119 ext_group_num++) { 120 if (mask & grp_mask[ext_group_num]) 121 return ext_group_num; 122 } 123 124 return -ENOENT; 125 } 126 127 static int ath12k_dp_srng_calculate_msi_group(struct ath12k_base *ab, 128 enum hal_ring_type type, int ring_num) 129 { 130 const u8 *grp_mask; 131 132 switch (type) { 133 case HAL_WBM2SW_RELEASE: 134 if (ring_num == HAL_WBM2SW_REL_ERR_RING_NUM) { 135 grp_mask = &ab->hw_params->ring_mask->rx_wbm_rel[0]; 136 ring_num = 0; 137 } else { 138 grp_mask = &ab->hw_params->ring_mask->tx[0]; 139 } 140 break; 141 case HAL_REO_EXCEPTION: 142 grp_mask = &ab->hw_params->ring_mask->rx_err[0]; 143 break; 144 case HAL_REO_DST: 145 grp_mask = &ab->hw_params->ring_mask->rx[0]; 146 break; 147 case HAL_REO_STATUS: 148 grp_mask = &ab->hw_params->ring_mask->reo_status[0]; 149 break; 150 case HAL_RXDMA_MONITOR_STATUS: 151 case HAL_RXDMA_MONITOR_DST: 152 grp_mask = &ab->hw_params->ring_mask->rx_mon_dest[0]; 153 break; 154 case HAL_TX_MONITOR_DST: 155 grp_mask = &ab->hw_params->ring_mask->tx_mon_dest[0]; 156 break; 157 case HAL_RXDMA_BUF: 158 grp_mask = &ab->hw_params->ring_mask->host2rxdma[0]; 159 break; 160 case HAL_RXDMA_MONITOR_BUF: 161 case HAL_TCL_DATA: 162 case HAL_TCL_CMD: 163 case HAL_REO_CMD: 164 case HAL_SW2WBM_RELEASE: 165 case HAL_WBM_IDLE_LINK: 166 case HAL_TCL_STATUS: 167 case HAL_REO_REINJECT: 168 case HAL_CE_SRC: 169 case HAL_CE_DST: 170 case HAL_CE_DST_STATUS: 171 default: 172 return -ENOENT; 173 } 174 175 return ath12k_dp_srng_find_ring_in_mask(ring_num, grp_mask); 176 } 177 178 static void ath12k_dp_srng_msi_setup(struct ath12k_base *ab, 179 struct hal_srng_params *ring_params, 180 enum hal_ring_type type, int ring_num) 181 { 182 int msi_group_number, msi_data_count; 183 u32 msi_data_start, msi_irq_start, addr_lo, addr_hi; 184 int ret; 185 186 ret = ath12k_hif_get_user_msi_vector(ab, "DP", 187 &msi_data_count, &msi_data_start, 188 &msi_irq_start); 189 if (ret) 190 return; 191 192 msi_group_number = ath12k_dp_srng_calculate_msi_group(ab, type, 193 ring_num); 194 if (msi_group_number < 0) { 195 ath12k_dbg(ab, ATH12K_DBG_PCI, 196 "ring not part of an ext_group; ring_type: %d,ring_num %d", 197 type, ring_num); 198 ring_params->msi_addr = 0; 199 ring_params->msi_data = 0; 200 return; 201 } 202 203 if (msi_group_number > msi_data_count) { 204 ath12k_dbg(ab, ATH12K_DBG_PCI, 205 "multiple msi_groups share one msi, msi_group_num %d", 206 msi_group_number); 207 } 208 209 ath12k_hif_get_msi_address(ab, &addr_lo, &addr_hi); 210 211 ring_params->msi_addr = addr_lo; 212 ring_params->msi_addr |= (dma_addr_t)(((uint64_t)addr_hi) << 32); 213 ring_params->msi_data = (msi_group_number % msi_data_count) 214 + msi_data_start; 215 ring_params->flags |= HAL_SRNG_FLAGS_MSI_INTR; 216 } 217 218 int ath12k_dp_srng_setup(struct ath12k_base *ab, struct dp_srng *ring, 219 enum hal_ring_type type, int ring_num, 220 int mac_id, int num_entries) 221 { 222 struct hal_srng_params params = { 0 }; 223 int entry_sz = ath12k_hal_srng_get_entrysize(ab, type); 224 int max_entries = ath12k_hal_srng_get_max_entries(ab, type); 225 int ret; 226 227 if (max_entries < 0 || entry_sz < 0) 228 return -EINVAL; 229 230 if (num_entries > max_entries) 231 num_entries = max_entries; 232 233 ring->size = (num_entries * entry_sz) + HAL_RING_BASE_ALIGN - 1; 234 ring->vaddr_unaligned = dma_alloc_coherent(ab->dev, ring->size, 235 &ring->paddr_unaligned, 236 GFP_KERNEL); 237 if (!ring->vaddr_unaligned) 238 return -ENOMEM; 239 240 ring->vaddr = PTR_ALIGN(ring->vaddr_unaligned, HAL_RING_BASE_ALIGN); 241 ring->paddr = ring->paddr_unaligned + ((unsigned long)ring->vaddr - 242 (unsigned long)ring->vaddr_unaligned); 243 244 params.ring_base_vaddr = ring->vaddr; 245 params.ring_base_paddr = ring->paddr; 246 params.num_entries = num_entries; 247 ath12k_dp_srng_msi_setup(ab, ¶ms, type, ring_num + mac_id); 248 249 switch (type) { 250 case HAL_REO_DST: 251 params.intr_batch_cntr_thres_entries = 252 HAL_SRNG_INT_BATCH_THRESHOLD_RX; 253 params.intr_timer_thres_us = HAL_SRNG_INT_TIMER_THRESHOLD_RX; 254 break; 255 case HAL_RXDMA_BUF: 256 case HAL_RXDMA_MONITOR_BUF: 257 case HAL_RXDMA_MONITOR_STATUS: 258 params.low_threshold = num_entries >> 3; 259 params.flags |= HAL_SRNG_FLAGS_LOW_THRESH_INTR_EN; 260 params.intr_batch_cntr_thres_entries = 0; 261 params.intr_timer_thres_us = HAL_SRNG_INT_TIMER_THRESHOLD_RX; 262 break; 263 case HAL_TX_MONITOR_DST: 264 params.low_threshold = DP_TX_MONITOR_BUF_SIZE_MAX >> 3; 265 params.flags |= HAL_SRNG_FLAGS_LOW_THRESH_INTR_EN; 266 params.intr_batch_cntr_thres_entries = 0; 267 params.intr_timer_thres_us = HAL_SRNG_INT_TIMER_THRESHOLD_RX; 268 break; 269 case HAL_WBM2SW_RELEASE: 270 if (ab->hw_params->hw_ops->dp_srng_is_tx_comp_ring(ring_num)) { 271 params.intr_batch_cntr_thres_entries = 272 HAL_SRNG_INT_BATCH_THRESHOLD_TX; 273 params.intr_timer_thres_us = 274 HAL_SRNG_INT_TIMER_THRESHOLD_TX; 275 break; 276 } 277 /* follow through when ring_num != HAL_WBM2SW_REL_ERR_RING_NUM */ 278 fallthrough; 279 case HAL_REO_EXCEPTION: 280 case HAL_REO_REINJECT: 281 case HAL_REO_CMD: 282 case HAL_REO_STATUS: 283 case HAL_TCL_DATA: 284 case HAL_TCL_CMD: 285 case HAL_TCL_STATUS: 286 case HAL_WBM_IDLE_LINK: 287 case HAL_SW2WBM_RELEASE: 288 case HAL_RXDMA_DST: 289 case HAL_RXDMA_MONITOR_DST: 290 case HAL_RXDMA_MONITOR_DESC: 291 params.intr_batch_cntr_thres_entries = 292 HAL_SRNG_INT_BATCH_THRESHOLD_OTHER; 293 params.intr_timer_thres_us = HAL_SRNG_INT_TIMER_THRESHOLD_OTHER; 294 break; 295 case HAL_RXDMA_DIR_BUF: 296 break; 297 default: 298 ath12k_warn(ab, "Not a valid ring type in dp :%d\n", type); 299 return -EINVAL; 300 } 301 302 ret = ath12k_hal_srng_setup(ab, type, ring_num, mac_id, ¶ms); 303 if (ret < 0) { 304 ath12k_warn(ab, "failed to setup srng: %d ring_id %d\n", 305 ret, ring_num); 306 return ret; 307 } 308 309 ring->ring_id = ret; 310 311 return 0; 312 } 313 314 static 315 u32 ath12k_dp_tx_get_vdev_bank_config(struct ath12k_base *ab, struct ath12k_vif *arvif) 316 { 317 u32 bank_config = 0; 318 319 /* Only valid for raw frames with HW crypto enabled. 320 * With SW crypto, mac80211 sets key per packet 321 */ 322 if (arvif->tx_encap_type == HAL_TCL_ENCAP_TYPE_RAW && 323 test_bit(ATH12K_FLAG_HW_CRYPTO_DISABLED, &ab->dev_flags)) 324 bank_config |= 325 u32_encode_bits(ath12k_dp_tx_get_encrypt_type(arvif->key_cipher), 326 HAL_TX_BANK_CONFIG_ENCRYPT_TYPE); 327 328 bank_config |= u32_encode_bits(arvif->tx_encap_type, 329 HAL_TX_BANK_CONFIG_ENCAP_TYPE); 330 bank_config |= u32_encode_bits(0, HAL_TX_BANK_CONFIG_SRC_BUFFER_SWAP) | 331 u32_encode_bits(0, HAL_TX_BANK_CONFIG_LINK_META_SWAP) | 332 u32_encode_bits(0, HAL_TX_BANK_CONFIG_EPD); 333 334 /* only valid if idx_lookup_override is not set in tcl_data_cmd */ 335 bank_config |= u32_encode_bits(0, HAL_TX_BANK_CONFIG_INDEX_LOOKUP_EN); 336 337 bank_config |= u32_encode_bits(arvif->hal_addr_search_flags & HAL_TX_ADDRX_EN, 338 HAL_TX_BANK_CONFIG_ADDRX_EN) | 339 u32_encode_bits(!!(arvif->hal_addr_search_flags & 340 HAL_TX_ADDRY_EN), 341 HAL_TX_BANK_CONFIG_ADDRY_EN); 342 343 bank_config |= u32_encode_bits(ieee80211_vif_is_mesh(arvif->vif) ? 3 : 0, 344 HAL_TX_BANK_CONFIG_MESH_EN) | 345 u32_encode_bits(arvif->vdev_id_check_en, 346 HAL_TX_BANK_CONFIG_VDEV_ID_CHECK_EN); 347 348 bank_config |= u32_encode_bits(0, HAL_TX_BANK_CONFIG_DSCP_TIP_MAP_ID); 349 350 return bank_config; 351 } 352 353 static int ath12k_dp_tx_get_bank_profile(struct ath12k_base *ab, struct ath12k_vif *arvif, 354 struct ath12k_dp *dp) 355 { 356 int bank_id = DP_INVALID_BANK_ID; 357 int i; 358 u32 bank_config; 359 bool configure_register = false; 360 361 /* convert vdev params into hal_tx_bank_config */ 362 bank_config = ath12k_dp_tx_get_vdev_bank_config(ab, arvif); 363 364 spin_lock_bh(&dp->tx_bank_lock); 365 /* TODO: implement using idr kernel framework*/ 366 for (i = 0; i < dp->num_bank_profiles; i++) { 367 if (dp->bank_profiles[i].is_configured && 368 (dp->bank_profiles[i].bank_config ^ bank_config) == 0) { 369 bank_id = i; 370 goto inc_ref_and_return; 371 } 372 if (!dp->bank_profiles[i].is_configured || 373 !dp->bank_profiles[i].num_users) { 374 bank_id = i; 375 goto configure_and_return; 376 } 377 } 378 379 if (bank_id == DP_INVALID_BANK_ID) { 380 spin_unlock_bh(&dp->tx_bank_lock); 381 ath12k_err(ab, "unable to find TX bank!"); 382 return bank_id; 383 } 384 385 configure_and_return: 386 dp->bank_profiles[bank_id].is_configured = true; 387 dp->bank_profiles[bank_id].bank_config = bank_config; 388 configure_register = true; 389 inc_ref_and_return: 390 dp->bank_profiles[bank_id].num_users++; 391 spin_unlock_bh(&dp->tx_bank_lock); 392 393 if (configure_register) 394 ath12k_hal_tx_configure_bank_register(ab, bank_config, bank_id); 395 396 ath12k_dbg(ab, ATH12K_DBG_DP_HTT, "dp_htt tcl bank_id %d input 0x%x match 0x%x num_users %u", 397 bank_id, bank_config, dp->bank_profiles[bank_id].bank_config, 398 dp->bank_profiles[bank_id].num_users); 399 400 return bank_id; 401 } 402 403 void ath12k_dp_tx_put_bank_profile(struct ath12k_dp *dp, u8 bank_id) 404 { 405 spin_lock_bh(&dp->tx_bank_lock); 406 dp->bank_profiles[bank_id].num_users--; 407 spin_unlock_bh(&dp->tx_bank_lock); 408 } 409 410 static void ath12k_dp_deinit_bank_profiles(struct ath12k_base *ab) 411 { 412 struct ath12k_dp *dp = &ab->dp; 413 414 kfree(dp->bank_profiles); 415 dp->bank_profiles = NULL; 416 } 417 418 static int ath12k_dp_init_bank_profiles(struct ath12k_base *ab) 419 { 420 struct ath12k_dp *dp = &ab->dp; 421 u32 num_tcl_banks = ab->hw_params->num_tcl_banks; 422 int i; 423 424 dp->num_bank_profiles = num_tcl_banks; 425 dp->bank_profiles = kmalloc_array(num_tcl_banks, 426 sizeof(struct ath12k_dp_tx_bank_profile), 427 GFP_KERNEL); 428 if (!dp->bank_profiles) 429 return -ENOMEM; 430 431 spin_lock_init(&dp->tx_bank_lock); 432 433 for (i = 0; i < num_tcl_banks; i++) { 434 dp->bank_profiles[i].is_configured = false; 435 dp->bank_profiles[i].num_users = 0; 436 } 437 438 return 0; 439 } 440 441 static void ath12k_dp_srng_common_cleanup(struct ath12k_base *ab) 442 { 443 struct ath12k_dp *dp = &ab->dp; 444 int i; 445 446 ath12k_dp_srng_cleanup(ab, &dp->reo_status_ring); 447 ath12k_dp_srng_cleanup(ab, &dp->reo_cmd_ring); 448 ath12k_dp_srng_cleanup(ab, &dp->reo_except_ring); 449 ath12k_dp_srng_cleanup(ab, &dp->rx_rel_ring); 450 ath12k_dp_srng_cleanup(ab, &dp->reo_reinject_ring); 451 for (i = 0; i < ab->hw_params->max_tx_ring; i++) { 452 ath12k_dp_srng_cleanup(ab, &dp->tx_ring[i].tcl_comp_ring); 453 ath12k_dp_srng_cleanup(ab, &dp->tx_ring[i].tcl_data_ring); 454 } 455 ath12k_dp_srng_cleanup(ab, &dp->tcl_status_ring); 456 ath12k_dp_srng_cleanup(ab, &dp->tcl_cmd_ring); 457 ath12k_dp_srng_cleanup(ab, &dp->wbm_desc_rel_ring); 458 } 459 460 static int ath12k_dp_srng_common_setup(struct ath12k_base *ab) 461 { 462 struct ath12k_dp *dp = &ab->dp; 463 const struct ath12k_hal_tcl_to_wbm_rbm_map *map; 464 struct hal_srng *srng; 465 int i, ret, tx_comp_ring_num; 466 u32 ring_hash_map; 467 468 ret = ath12k_dp_srng_setup(ab, &dp->wbm_desc_rel_ring, 469 HAL_SW2WBM_RELEASE, 0, 0, 470 DP_WBM_RELEASE_RING_SIZE); 471 if (ret) { 472 ath12k_warn(ab, "failed to set up wbm2sw_release ring :%d\n", 473 ret); 474 goto err; 475 } 476 477 ret = ath12k_dp_srng_setup(ab, &dp->tcl_cmd_ring, HAL_TCL_CMD, 0, 0, 478 DP_TCL_CMD_RING_SIZE); 479 if (ret) { 480 ath12k_warn(ab, "failed to set up tcl_cmd ring :%d\n", ret); 481 goto err; 482 } 483 484 ret = ath12k_dp_srng_setup(ab, &dp->tcl_status_ring, HAL_TCL_STATUS, 485 0, 0, DP_TCL_STATUS_RING_SIZE); 486 if (ret) { 487 ath12k_warn(ab, "failed to set up tcl_status ring :%d\n", ret); 488 goto err; 489 } 490 491 for (i = 0; i < ab->hw_params->max_tx_ring; i++) { 492 map = ab->hw_params->hal_ops->tcl_to_wbm_rbm_map; 493 tx_comp_ring_num = map[i].wbm_ring_num; 494 495 ret = ath12k_dp_srng_setup(ab, &dp->tx_ring[i].tcl_data_ring, 496 HAL_TCL_DATA, i, 0, 497 DP_TCL_DATA_RING_SIZE); 498 if (ret) { 499 ath12k_warn(ab, "failed to set up tcl_data ring (%d) :%d\n", 500 i, ret); 501 goto err; 502 } 503 504 ret = ath12k_dp_srng_setup(ab, &dp->tx_ring[i].tcl_comp_ring, 505 HAL_WBM2SW_RELEASE, tx_comp_ring_num, 0, 506 DP_TX_COMP_RING_SIZE); 507 if (ret) { 508 ath12k_warn(ab, "failed to set up tcl_comp ring (%d) :%d\n", 509 tx_comp_ring_num, ret); 510 goto err; 511 } 512 } 513 514 ret = ath12k_dp_srng_setup(ab, &dp->reo_reinject_ring, HAL_REO_REINJECT, 515 0, 0, DP_REO_REINJECT_RING_SIZE); 516 if (ret) { 517 ath12k_warn(ab, "failed to set up reo_reinject ring :%d\n", 518 ret); 519 goto err; 520 } 521 522 ret = ath12k_dp_srng_setup(ab, &dp->rx_rel_ring, HAL_WBM2SW_RELEASE, 523 HAL_WBM2SW_REL_ERR_RING_NUM, 0, 524 DP_RX_RELEASE_RING_SIZE); 525 if (ret) { 526 ath12k_warn(ab, "failed to set up rx_rel ring :%d\n", ret); 527 goto err; 528 } 529 530 ret = ath12k_dp_srng_setup(ab, &dp->reo_except_ring, HAL_REO_EXCEPTION, 531 0, 0, DP_REO_EXCEPTION_RING_SIZE); 532 if (ret) { 533 ath12k_warn(ab, "failed to set up reo_exception ring :%d\n", 534 ret); 535 goto err; 536 } 537 538 ret = ath12k_dp_srng_setup(ab, &dp->reo_cmd_ring, HAL_REO_CMD, 539 0, 0, DP_REO_CMD_RING_SIZE); 540 if (ret) { 541 ath12k_warn(ab, "failed to set up reo_cmd ring :%d\n", ret); 542 goto err; 543 } 544 545 srng = &ab->hal.srng_list[dp->reo_cmd_ring.ring_id]; 546 ath12k_hal_reo_init_cmd_ring(ab, srng); 547 548 ret = ath12k_dp_srng_setup(ab, &dp->reo_status_ring, HAL_REO_STATUS, 549 0, 0, DP_REO_STATUS_RING_SIZE); 550 if (ret) { 551 ath12k_warn(ab, "failed to set up reo_status ring :%d\n", ret); 552 goto err; 553 } 554 555 /* When hash based routing of rx packet is enabled, 32 entries to map 556 * the hash values to the ring will be configured. Each hash entry uses 557 * four bits to map to a particular ring. The ring mapping will be 558 * 0:TCL, 1:SW1, 2:SW2, 3:SW3, 4:SW4, 5:Release, 6:FW and 7:SW5 559 * 8:SW6, 9:SW7, 10:SW8, 11:Not used. 560 */ 561 ring_hash_map = HAL_HASH_ROUTING_RING_SW1 | 562 HAL_HASH_ROUTING_RING_SW2 << 4 | 563 HAL_HASH_ROUTING_RING_SW3 << 8 | 564 HAL_HASH_ROUTING_RING_SW4 << 12 | 565 HAL_HASH_ROUTING_RING_SW1 << 16 | 566 HAL_HASH_ROUTING_RING_SW2 << 20 | 567 HAL_HASH_ROUTING_RING_SW3 << 24 | 568 HAL_HASH_ROUTING_RING_SW4 << 28; 569 570 ath12k_hal_reo_hw_setup(ab, ring_hash_map); 571 572 return 0; 573 574 err: 575 ath12k_dp_srng_common_cleanup(ab); 576 577 return ret; 578 } 579 580 static void ath12k_dp_scatter_idle_link_desc_cleanup(struct ath12k_base *ab) 581 { 582 struct ath12k_dp *dp = &ab->dp; 583 struct hal_wbm_idle_scatter_list *slist = dp->scatter_list; 584 int i; 585 586 for (i = 0; i < DP_IDLE_SCATTER_BUFS_MAX; i++) { 587 if (!slist[i].vaddr) 588 continue; 589 590 dma_free_coherent(ab->dev, HAL_WBM_IDLE_SCATTER_BUF_SIZE_MAX, 591 slist[i].vaddr, slist[i].paddr); 592 slist[i].vaddr = NULL; 593 } 594 } 595 596 static int ath12k_dp_scatter_idle_link_desc_setup(struct ath12k_base *ab, 597 int size, 598 u32 n_link_desc_bank, 599 u32 n_link_desc, 600 u32 last_bank_sz) 601 { 602 struct ath12k_dp *dp = &ab->dp; 603 struct dp_link_desc_bank *link_desc_banks = dp->link_desc_banks; 604 struct hal_wbm_idle_scatter_list *slist = dp->scatter_list; 605 u32 n_entries_per_buf; 606 int num_scatter_buf, scatter_idx; 607 struct hal_wbm_link_desc *scatter_buf; 608 int align_bytes, n_entries; 609 dma_addr_t paddr; 610 int rem_entries; 611 int i; 612 int ret = 0; 613 u32 end_offset, cookie; 614 615 n_entries_per_buf = HAL_WBM_IDLE_SCATTER_BUF_SIZE / 616 ath12k_hal_srng_get_entrysize(ab, HAL_WBM_IDLE_LINK); 617 num_scatter_buf = DIV_ROUND_UP(size, HAL_WBM_IDLE_SCATTER_BUF_SIZE); 618 619 if (num_scatter_buf > DP_IDLE_SCATTER_BUFS_MAX) 620 return -EINVAL; 621 622 for (i = 0; i < num_scatter_buf; i++) { 623 slist[i].vaddr = dma_alloc_coherent(ab->dev, 624 HAL_WBM_IDLE_SCATTER_BUF_SIZE_MAX, 625 &slist[i].paddr, GFP_KERNEL); 626 if (!slist[i].vaddr) { 627 ret = -ENOMEM; 628 goto err; 629 } 630 } 631 632 scatter_idx = 0; 633 scatter_buf = slist[scatter_idx].vaddr; 634 rem_entries = n_entries_per_buf; 635 636 for (i = 0; i < n_link_desc_bank; i++) { 637 align_bytes = link_desc_banks[i].vaddr - 638 link_desc_banks[i].vaddr_unaligned; 639 n_entries = (DP_LINK_DESC_ALLOC_SIZE_THRESH - align_bytes) / 640 HAL_LINK_DESC_SIZE; 641 paddr = link_desc_banks[i].paddr; 642 while (n_entries) { 643 cookie = DP_LINK_DESC_COOKIE_SET(n_entries, i); 644 ath12k_hal_set_link_desc_addr(scatter_buf, cookie, paddr); 645 n_entries--; 646 paddr += HAL_LINK_DESC_SIZE; 647 if (rem_entries) { 648 rem_entries--; 649 scatter_buf++; 650 continue; 651 } 652 653 rem_entries = n_entries_per_buf; 654 scatter_idx++; 655 scatter_buf = slist[scatter_idx].vaddr; 656 } 657 } 658 659 end_offset = (scatter_buf - slist[scatter_idx].vaddr) * 660 sizeof(struct hal_wbm_link_desc); 661 ath12k_hal_setup_link_idle_list(ab, slist, num_scatter_buf, 662 n_link_desc, end_offset); 663 664 return 0; 665 666 err: 667 ath12k_dp_scatter_idle_link_desc_cleanup(ab); 668 669 return ret; 670 } 671 672 static void 673 ath12k_dp_link_desc_bank_free(struct ath12k_base *ab, 674 struct dp_link_desc_bank *link_desc_banks) 675 { 676 int i; 677 678 for (i = 0; i < DP_LINK_DESC_BANKS_MAX; i++) { 679 if (link_desc_banks[i].vaddr_unaligned) { 680 dma_free_coherent(ab->dev, 681 link_desc_banks[i].size, 682 link_desc_banks[i].vaddr_unaligned, 683 link_desc_banks[i].paddr_unaligned); 684 link_desc_banks[i].vaddr_unaligned = NULL; 685 } 686 } 687 } 688 689 static int ath12k_dp_link_desc_bank_alloc(struct ath12k_base *ab, 690 struct dp_link_desc_bank *desc_bank, 691 int n_link_desc_bank, 692 int last_bank_sz) 693 { 694 struct ath12k_dp *dp = &ab->dp; 695 int i; 696 int ret = 0; 697 int desc_sz = DP_LINK_DESC_ALLOC_SIZE_THRESH; 698 699 for (i = 0; i < n_link_desc_bank; i++) { 700 if (i == (n_link_desc_bank - 1) && last_bank_sz) 701 desc_sz = last_bank_sz; 702 703 desc_bank[i].vaddr_unaligned = 704 dma_alloc_coherent(ab->dev, desc_sz, 705 &desc_bank[i].paddr_unaligned, 706 GFP_KERNEL); 707 if (!desc_bank[i].vaddr_unaligned) { 708 ret = -ENOMEM; 709 goto err; 710 } 711 712 desc_bank[i].vaddr = PTR_ALIGN(desc_bank[i].vaddr_unaligned, 713 HAL_LINK_DESC_ALIGN); 714 desc_bank[i].paddr = desc_bank[i].paddr_unaligned + 715 ((unsigned long)desc_bank[i].vaddr - 716 (unsigned long)desc_bank[i].vaddr_unaligned); 717 desc_bank[i].size = desc_sz; 718 } 719 720 return 0; 721 722 err: 723 ath12k_dp_link_desc_bank_free(ab, dp->link_desc_banks); 724 725 return ret; 726 } 727 728 void ath12k_dp_link_desc_cleanup(struct ath12k_base *ab, 729 struct dp_link_desc_bank *desc_bank, 730 u32 ring_type, struct dp_srng *ring) 731 { 732 ath12k_dp_link_desc_bank_free(ab, desc_bank); 733 734 if (ring_type != HAL_RXDMA_MONITOR_DESC) { 735 ath12k_dp_srng_cleanup(ab, ring); 736 ath12k_dp_scatter_idle_link_desc_cleanup(ab); 737 } 738 } 739 740 static int ath12k_wbm_idle_ring_setup(struct ath12k_base *ab, u32 *n_link_desc) 741 { 742 struct ath12k_dp *dp = &ab->dp; 743 u32 n_mpdu_link_desc, n_mpdu_queue_desc; 744 u32 n_tx_msdu_link_desc, n_rx_msdu_link_desc; 745 int ret = 0; 746 747 n_mpdu_link_desc = (DP_NUM_TIDS_MAX * DP_AVG_MPDUS_PER_TID_MAX) / 748 HAL_NUM_MPDUS_PER_LINK_DESC; 749 750 n_mpdu_queue_desc = n_mpdu_link_desc / 751 HAL_NUM_MPDU_LINKS_PER_QUEUE_DESC; 752 753 n_tx_msdu_link_desc = (DP_NUM_TIDS_MAX * DP_AVG_FLOWS_PER_TID * 754 DP_AVG_MSDUS_PER_FLOW) / 755 HAL_NUM_TX_MSDUS_PER_LINK_DESC; 756 757 n_rx_msdu_link_desc = (DP_NUM_TIDS_MAX * DP_AVG_MPDUS_PER_TID_MAX * 758 DP_AVG_MSDUS_PER_MPDU) / 759 HAL_NUM_RX_MSDUS_PER_LINK_DESC; 760 761 *n_link_desc = n_mpdu_link_desc + n_mpdu_queue_desc + 762 n_tx_msdu_link_desc + n_rx_msdu_link_desc; 763 764 if (*n_link_desc & (*n_link_desc - 1)) 765 *n_link_desc = 1 << fls(*n_link_desc); 766 767 ret = ath12k_dp_srng_setup(ab, &dp->wbm_idle_ring, 768 HAL_WBM_IDLE_LINK, 0, 0, *n_link_desc); 769 if (ret) { 770 ath12k_warn(ab, "failed to setup wbm_idle_ring: %d\n", ret); 771 return ret; 772 } 773 return ret; 774 } 775 776 int ath12k_dp_link_desc_setup(struct ath12k_base *ab, 777 struct dp_link_desc_bank *link_desc_banks, 778 u32 ring_type, struct hal_srng *srng, 779 u32 n_link_desc) 780 { 781 u32 tot_mem_sz; 782 u32 n_link_desc_bank, last_bank_sz; 783 u32 entry_sz, align_bytes, n_entries; 784 struct hal_wbm_link_desc *desc; 785 u32 paddr; 786 int i, ret; 787 u32 cookie; 788 789 tot_mem_sz = n_link_desc * HAL_LINK_DESC_SIZE; 790 tot_mem_sz += HAL_LINK_DESC_ALIGN; 791 792 if (tot_mem_sz <= DP_LINK_DESC_ALLOC_SIZE_THRESH) { 793 n_link_desc_bank = 1; 794 last_bank_sz = tot_mem_sz; 795 } else { 796 n_link_desc_bank = tot_mem_sz / 797 (DP_LINK_DESC_ALLOC_SIZE_THRESH - 798 HAL_LINK_DESC_ALIGN); 799 last_bank_sz = tot_mem_sz % 800 (DP_LINK_DESC_ALLOC_SIZE_THRESH - 801 HAL_LINK_DESC_ALIGN); 802 803 if (last_bank_sz) 804 n_link_desc_bank += 1; 805 } 806 807 if (n_link_desc_bank > DP_LINK_DESC_BANKS_MAX) 808 return -EINVAL; 809 810 ret = ath12k_dp_link_desc_bank_alloc(ab, link_desc_banks, 811 n_link_desc_bank, last_bank_sz); 812 if (ret) 813 return ret; 814 815 /* Setup link desc idle list for HW internal usage */ 816 entry_sz = ath12k_hal_srng_get_entrysize(ab, ring_type); 817 tot_mem_sz = entry_sz * n_link_desc; 818 819 /* Setup scatter desc list when the total memory requirement is more */ 820 if (tot_mem_sz > DP_LINK_DESC_ALLOC_SIZE_THRESH && 821 ring_type != HAL_RXDMA_MONITOR_DESC) { 822 ret = ath12k_dp_scatter_idle_link_desc_setup(ab, tot_mem_sz, 823 n_link_desc_bank, 824 n_link_desc, 825 last_bank_sz); 826 if (ret) { 827 ath12k_warn(ab, "failed to setup scatting idle list descriptor :%d\n", 828 ret); 829 goto fail_desc_bank_free; 830 } 831 832 return 0; 833 } 834 835 spin_lock_bh(&srng->lock); 836 837 ath12k_hal_srng_access_begin(ab, srng); 838 839 for (i = 0; i < n_link_desc_bank; i++) { 840 align_bytes = link_desc_banks[i].vaddr - 841 link_desc_banks[i].vaddr_unaligned; 842 n_entries = (link_desc_banks[i].size - align_bytes) / 843 HAL_LINK_DESC_SIZE; 844 paddr = link_desc_banks[i].paddr; 845 while (n_entries && 846 (desc = ath12k_hal_srng_src_get_next_entry(ab, srng))) { 847 cookie = DP_LINK_DESC_COOKIE_SET(n_entries, i); 848 ath12k_hal_set_link_desc_addr(desc, 849 cookie, paddr); 850 n_entries--; 851 paddr += HAL_LINK_DESC_SIZE; 852 } 853 } 854 855 ath12k_hal_srng_access_end(ab, srng); 856 857 spin_unlock_bh(&srng->lock); 858 859 return 0; 860 861 fail_desc_bank_free: 862 ath12k_dp_link_desc_bank_free(ab, link_desc_banks); 863 864 return ret; 865 } 866 867 int ath12k_dp_service_srng(struct ath12k_base *ab, 868 struct ath12k_ext_irq_grp *irq_grp, 869 int budget) 870 { 871 struct napi_struct *napi = &irq_grp->napi; 872 int grp_id = irq_grp->grp_id; 873 int work_done = 0; 874 int i = 0, j; 875 int tot_work_done = 0; 876 enum dp_monitor_mode monitor_mode; 877 u8 ring_mask; 878 879 while (i < ab->hw_params->max_tx_ring) { 880 if (ab->hw_params->ring_mask->tx[grp_id] & 881 BIT(ab->hw_params->hal_ops->tcl_to_wbm_rbm_map[i].wbm_ring_num)) 882 ath12k_dp_tx_completion_handler(ab, i); 883 i++; 884 } 885 886 if (ab->hw_params->ring_mask->rx_err[grp_id]) { 887 work_done = ath12k_dp_rx_process_err(ab, napi, budget); 888 budget -= work_done; 889 tot_work_done += work_done; 890 if (budget <= 0) 891 goto done; 892 } 893 894 if (ab->hw_params->ring_mask->rx_wbm_rel[grp_id]) { 895 work_done = ath12k_dp_rx_process_wbm_err(ab, 896 napi, 897 budget); 898 budget -= work_done; 899 tot_work_done += work_done; 900 901 if (budget <= 0) 902 goto done; 903 } 904 905 if (ab->hw_params->ring_mask->rx[grp_id]) { 906 i = fls(ab->hw_params->ring_mask->rx[grp_id]) - 1; 907 work_done = ath12k_dp_rx_process(ab, i, napi, 908 budget); 909 budget -= work_done; 910 tot_work_done += work_done; 911 if (budget <= 0) 912 goto done; 913 } 914 915 if (ab->hw_params->ring_mask->rx_mon_dest[grp_id]) { 916 monitor_mode = ATH12K_DP_RX_MONITOR_MODE; 917 ring_mask = ab->hw_params->ring_mask->rx_mon_dest[grp_id]; 918 for (i = 0; i < ab->num_radios; i++) { 919 for (j = 0; j < ab->hw_params->num_rxmda_per_pdev; j++) { 920 int id = i * ab->hw_params->num_rxmda_per_pdev + j; 921 922 if (ring_mask & BIT(id)) { 923 work_done = 924 ath12k_dp_mon_process_ring(ab, id, napi, budget, 925 monitor_mode); 926 budget -= work_done; 927 tot_work_done += work_done; 928 929 if (budget <= 0) 930 goto done; 931 } 932 } 933 } 934 } 935 936 if (ab->hw_params->ring_mask->tx_mon_dest[grp_id]) { 937 monitor_mode = ATH12K_DP_TX_MONITOR_MODE; 938 ring_mask = ab->hw_params->ring_mask->tx_mon_dest[grp_id]; 939 for (i = 0; i < ab->num_radios; i++) { 940 for (j = 0; j < ab->hw_params->num_rxmda_per_pdev; j++) { 941 int id = i * ab->hw_params->num_rxmda_per_pdev + j; 942 943 if (ring_mask & BIT(id)) { 944 work_done = 945 ath12k_dp_mon_process_ring(ab, id, napi, budget, 946 monitor_mode); 947 budget -= work_done; 948 tot_work_done += work_done; 949 950 if (budget <= 0) 951 goto done; 952 } 953 } 954 } 955 } 956 957 if (ab->hw_params->ring_mask->reo_status[grp_id]) 958 ath12k_dp_rx_process_reo_status(ab); 959 960 if (ab->hw_params->ring_mask->host2rxdma[grp_id]) { 961 struct ath12k_dp *dp = &ab->dp; 962 struct dp_rxdma_ring *rx_ring = &dp->rx_refill_buf_ring; 963 LIST_HEAD(list); 964 965 ath12k_dp_rx_bufs_replenish(ab, rx_ring, &list, 0); 966 } 967 968 /* TODO: Implement handler for other interrupts */ 969 970 done: 971 return tot_work_done; 972 } 973 974 void ath12k_dp_pdev_free(struct ath12k_base *ab) 975 { 976 int i; 977 978 del_timer_sync(&ab->mon_reap_timer); 979 980 for (i = 0; i < ab->num_radios; i++) 981 ath12k_dp_rx_pdev_free(ab, i); 982 } 983 984 void ath12k_dp_pdev_pre_alloc(struct ath12k_base *ab) 985 { 986 struct ath12k *ar; 987 struct ath12k_pdev_dp *dp; 988 int i; 989 990 for (i = 0; i < ab->num_radios; i++) { 991 ar = ab->pdevs[i].ar; 992 dp = &ar->dp; 993 dp->mac_id = i; 994 atomic_set(&dp->num_tx_pending, 0); 995 init_waitqueue_head(&dp->tx_empty_waitq); 996 997 /* TODO: Add any RXDMA setup required per pdev */ 998 } 999 } 1000 1001 bool ath12k_dp_wmask_compaction_rx_tlv_supported(struct ath12k_base *ab) 1002 { 1003 if (test_bit(WMI_TLV_SERVICE_WMSK_COMPACTION_RX_TLVS, ab->wmi_ab.svc_map) && 1004 ab->hw_params->hal_ops->rxdma_ring_wmask_rx_mpdu_start && 1005 ab->hw_params->hal_ops->rxdma_ring_wmask_rx_msdu_end && 1006 ab->hw_params->hal_ops->get_hal_rx_compact_ops) { 1007 return true; 1008 } 1009 return false; 1010 } 1011 1012 void ath12k_dp_hal_rx_desc_init(struct ath12k_base *ab) 1013 { 1014 if (ath12k_dp_wmask_compaction_rx_tlv_supported(ab)) { 1015 /* RX TLVS compaction is supported, hence change the hal_rx_ops 1016 * to compact hal_rx_ops. 1017 */ 1018 ab->hal_rx_ops = ab->hw_params->hal_ops->get_hal_rx_compact_ops(); 1019 } 1020 ab->hal.hal_desc_sz = 1021 ab->hal_rx_ops->rx_desc_get_desc_size(); 1022 } 1023 1024 static void ath12k_dp_service_mon_ring(struct timer_list *t) 1025 { 1026 struct ath12k_base *ab = from_timer(ab, t, mon_reap_timer); 1027 int i; 1028 1029 for (i = 0; i < ab->hw_params->num_rxmda_per_pdev; i++) 1030 ath12k_dp_mon_process_ring(ab, i, NULL, DP_MON_SERVICE_BUDGET, 1031 ATH12K_DP_RX_MONITOR_MODE); 1032 1033 mod_timer(&ab->mon_reap_timer, jiffies + 1034 msecs_to_jiffies(ATH12K_MON_TIMER_INTERVAL)); 1035 } 1036 1037 static void ath12k_dp_mon_reap_timer_init(struct ath12k_base *ab) 1038 { 1039 if (ab->hw_params->rxdma1_enable) 1040 return; 1041 1042 timer_setup(&ab->mon_reap_timer, ath12k_dp_service_mon_ring, 0); 1043 } 1044 1045 int ath12k_dp_pdev_alloc(struct ath12k_base *ab) 1046 { 1047 struct ath12k *ar; 1048 int ret; 1049 int i; 1050 1051 ret = ath12k_dp_rx_htt_setup(ab); 1052 if (ret) 1053 goto out; 1054 1055 ath12k_dp_mon_reap_timer_init(ab); 1056 1057 /* TODO: Per-pdev rx ring unlike tx ring which is mapped to different AC's */ 1058 for (i = 0; i < ab->num_radios; i++) { 1059 ar = ab->pdevs[i].ar; 1060 ret = ath12k_dp_rx_pdev_alloc(ab, i); 1061 if (ret) { 1062 ath12k_warn(ab, "failed to allocate pdev rx for pdev_id :%d\n", 1063 i); 1064 goto err; 1065 } 1066 ret = ath12k_dp_rx_pdev_mon_attach(ar); 1067 if (ret) { 1068 ath12k_warn(ab, "failed to initialize mon pdev %d\n", i); 1069 goto err; 1070 } 1071 } 1072 1073 return 0; 1074 err: 1075 ath12k_dp_pdev_free(ab); 1076 out: 1077 return ret; 1078 } 1079 1080 int ath12k_dp_htt_connect(struct ath12k_dp *dp) 1081 { 1082 struct ath12k_htc_svc_conn_req conn_req = {0}; 1083 struct ath12k_htc_svc_conn_resp conn_resp = {0}; 1084 int status; 1085 1086 conn_req.ep_ops.ep_tx_complete = ath12k_dp_htt_htc_tx_complete; 1087 conn_req.ep_ops.ep_rx_complete = ath12k_dp_htt_htc_t2h_msg_handler; 1088 1089 /* connect to control service */ 1090 conn_req.service_id = ATH12K_HTC_SVC_ID_HTT_DATA_MSG; 1091 1092 status = ath12k_htc_connect_service(&dp->ab->htc, &conn_req, 1093 &conn_resp); 1094 1095 if (status) 1096 return status; 1097 1098 dp->eid = conn_resp.eid; 1099 1100 return 0; 1101 } 1102 1103 static void ath12k_dp_update_vdev_search(struct ath12k_vif *arvif) 1104 { 1105 switch (arvif->vdev_type) { 1106 case WMI_VDEV_TYPE_STA: 1107 /* TODO: Verify the search type and flags since ast hash 1108 * is not part of peer mapv3 1109 */ 1110 arvif->hal_addr_search_flags = HAL_TX_ADDRY_EN; 1111 arvif->search_type = HAL_TX_ADDR_SEARCH_DEFAULT; 1112 break; 1113 case WMI_VDEV_TYPE_AP: 1114 case WMI_VDEV_TYPE_IBSS: 1115 arvif->hal_addr_search_flags = HAL_TX_ADDRX_EN; 1116 arvif->search_type = HAL_TX_ADDR_SEARCH_DEFAULT; 1117 break; 1118 case WMI_VDEV_TYPE_MONITOR: 1119 default: 1120 return; 1121 } 1122 } 1123 1124 void ath12k_dp_vdev_tx_attach(struct ath12k *ar, struct ath12k_vif *arvif) 1125 { 1126 struct ath12k_base *ab = ar->ab; 1127 1128 arvif->tcl_metadata |= u32_encode_bits(1, HTT_TCL_META_DATA_TYPE) | 1129 u32_encode_bits(arvif->vdev_id, 1130 HTT_TCL_META_DATA_VDEV_ID) | 1131 u32_encode_bits(ar->pdev->pdev_id, 1132 HTT_TCL_META_DATA_PDEV_ID); 1133 1134 /* set HTT extension valid bit to 0 by default */ 1135 arvif->tcl_metadata &= ~HTT_TCL_META_DATA_VALID_HTT; 1136 1137 ath12k_dp_update_vdev_search(arvif); 1138 arvif->vdev_id_check_en = true; 1139 arvif->bank_id = ath12k_dp_tx_get_bank_profile(ab, arvif, &ab->dp); 1140 1141 /* TODO: error path for bank id failure */ 1142 if (arvif->bank_id == DP_INVALID_BANK_ID) { 1143 ath12k_err(ar->ab, "Failed to initialize DP TX Banks"); 1144 return; 1145 } 1146 } 1147 1148 static void ath12k_dp_cc_cleanup(struct ath12k_base *ab) 1149 { 1150 struct ath12k_rx_desc_info *desc_info; 1151 struct ath12k_tx_desc_info *tx_desc_info, *tmp1; 1152 struct ath12k_dp *dp = &ab->dp; 1153 struct sk_buff *skb; 1154 int i, j; 1155 u32 pool_id, tx_spt_page; 1156 1157 if (!dp->spt_info) 1158 return; 1159 1160 /* RX Descriptor cleanup */ 1161 spin_lock_bh(&dp->rx_desc_lock); 1162 1163 for (i = 0; i < ATH12K_NUM_RX_SPT_PAGES; i++) { 1164 desc_info = dp->spt_info->rxbaddr[i]; 1165 1166 for (j = 0; j < ATH12K_MAX_SPT_ENTRIES; j++) { 1167 if (!desc_info[j].in_use) { 1168 list_del(&desc_info[j].list); 1169 continue; 1170 } 1171 1172 skb = desc_info[j].skb; 1173 if (!skb) 1174 continue; 1175 1176 dma_unmap_single(ab->dev, ATH12K_SKB_RXCB(skb)->paddr, 1177 skb->len + skb_tailroom(skb), DMA_FROM_DEVICE); 1178 dev_kfree_skb_any(skb); 1179 } 1180 } 1181 1182 for (i = 0; i < ATH12K_NUM_RX_SPT_PAGES; i++) { 1183 if (!dp->spt_info->rxbaddr[i]) 1184 continue; 1185 1186 kfree(dp->spt_info->rxbaddr[i]); 1187 dp->spt_info->rxbaddr[i] = NULL; 1188 } 1189 1190 spin_unlock_bh(&dp->rx_desc_lock); 1191 1192 /* TX Descriptor cleanup */ 1193 for (i = 0; i < ATH12K_HW_MAX_QUEUES; i++) { 1194 spin_lock_bh(&dp->tx_desc_lock[i]); 1195 1196 list_for_each_entry_safe(tx_desc_info, tmp1, &dp->tx_desc_used_list[i], 1197 list) { 1198 list_del(&tx_desc_info->list); 1199 skb = tx_desc_info->skb; 1200 1201 if (!skb) 1202 continue; 1203 1204 dma_unmap_single(ab->dev, ATH12K_SKB_CB(skb)->paddr, 1205 skb->len, DMA_TO_DEVICE); 1206 dev_kfree_skb_any(skb); 1207 } 1208 1209 spin_unlock_bh(&dp->tx_desc_lock[i]); 1210 } 1211 1212 for (pool_id = 0; pool_id < ATH12K_HW_MAX_QUEUES; pool_id++) { 1213 spin_lock_bh(&dp->tx_desc_lock[pool_id]); 1214 1215 for (i = 0; i < ATH12K_TX_SPT_PAGES_PER_POOL; i++) { 1216 tx_spt_page = i + pool_id * ATH12K_TX_SPT_PAGES_PER_POOL; 1217 if (!dp->spt_info->txbaddr[tx_spt_page]) 1218 continue; 1219 1220 kfree(dp->spt_info->txbaddr[tx_spt_page]); 1221 dp->spt_info->txbaddr[tx_spt_page] = NULL; 1222 } 1223 1224 spin_unlock_bh(&dp->tx_desc_lock[pool_id]); 1225 } 1226 1227 /* unmap SPT pages */ 1228 for (i = 0; i < dp->num_spt_pages; i++) { 1229 if (!dp->spt_info[i].vaddr) 1230 continue; 1231 1232 dma_free_coherent(ab->dev, ATH12K_PAGE_SIZE, 1233 dp->spt_info[i].vaddr, dp->spt_info[i].paddr); 1234 dp->spt_info[i].vaddr = NULL; 1235 } 1236 1237 kfree(dp->spt_info); 1238 } 1239 1240 static void ath12k_dp_reoq_lut_cleanup(struct ath12k_base *ab) 1241 { 1242 struct ath12k_dp *dp = &ab->dp; 1243 1244 if (!ab->hw_params->reoq_lut_support) 1245 return; 1246 1247 if (!dp->reoq_lut.vaddr) 1248 return; 1249 1250 dma_free_coherent(ab->dev, DP_REOQ_LUT_SIZE, 1251 dp->reoq_lut.vaddr, dp->reoq_lut.paddr); 1252 dp->reoq_lut.vaddr = NULL; 1253 1254 ath12k_hif_write32(ab, 1255 HAL_SEQ_WCSS_UMAC_REO_REG + HAL_REO1_QDESC_LUT_BASE0(ab), 0); 1256 } 1257 1258 void ath12k_dp_free(struct ath12k_base *ab) 1259 { 1260 struct ath12k_dp *dp = &ab->dp; 1261 int i; 1262 1263 ath12k_dp_link_desc_cleanup(ab, dp->link_desc_banks, 1264 HAL_WBM_IDLE_LINK, &dp->wbm_idle_ring); 1265 1266 ath12k_dp_cc_cleanup(ab); 1267 ath12k_dp_reoq_lut_cleanup(ab); 1268 ath12k_dp_deinit_bank_profiles(ab); 1269 ath12k_dp_srng_common_cleanup(ab); 1270 1271 ath12k_dp_rx_reo_cmd_list_cleanup(ab); 1272 1273 for (i = 0; i < ab->hw_params->max_tx_ring; i++) 1274 kfree(dp->tx_ring[i].tx_status); 1275 1276 ath12k_dp_rx_free(ab); 1277 /* Deinit any SOC level resource */ 1278 } 1279 1280 void ath12k_dp_cc_config(struct ath12k_base *ab) 1281 { 1282 u32 cmem_base = ab->qmi.dev_mem[ATH12K_QMI_DEVMEM_CMEM_INDEX].start; 1283 u32 reo_base = HAL_SEQ_WCSS_UMAC_REO_REG; 1284 u32 wbm_base = HAL_SEQ_WCSS_UMAC_WBM_REG; 1285 u32 val = 0; 1286 1287 ath12k_hif_write32(ab, reo_base + HAL_REO1_SW_COOKIE_CFG0(ab), cmem_base); 1288 1289 val |= u32_encode_bits(ATH12K_CMEM_ADDR_MSB, 1290 HAL_REO1_SW_COOKIE_CFG_CMEM_BASE_ADDR_MSB) | 1291 u32_encode_bits(ATH12K_CC_PPT_MSB, 1292 HAL_REO1_SW_COOKIE_CFG_COOKIE_PPT_MSB) | 1293 u32_encode_bits(ATH12K_CC_SPT_MSB, 1294 HAL_REO1_SW_COOKIE_CFG_COOKIE_SPT_MSB) | 1295 u32_encode_bits(1, HAL_REO1_SW_COOKIE_CFG_ALIGN) | 1296 u32_encode_bits(1, HAL_REO1_SW_COOKIE_CFG_ENABLE) | 1297 u32_encode_bits(1, HAL_REO1_SW_COOKIE_CFG_GLOBAL_ENABLE); 1298 1299 ath12k_hif_write32(ab, reo_base + HAL_REO1_SW_COOKIE_CFG1(ab), val); 1300 1301 /* Enable HW CC for WBM */ 1302 ath12k_hif_write32(ab, wbm_base + HAL_WBM_SW_COOKIE_CFG0, cmem_base); 1303 1304 val = u32_encode_bits(ATH12K_CMEM_ADDR_MSB, 1305 HAL_WBM_SW_COOKIE_CFG_CMEM_BASE_ADDR_MSB) | 1306 u32_encode_bits(ATH12K_CC_PPT_MSB, 1307 HAL_WBM_SW_COOKIE_CFG_COOKIE_PPT_MSB) | 1308 u32_encode_bits(ATH12K_CC_SPT_MSB, 1309 HAL_WBM_SW_COOKIE_CFG_COOKIE_SPT_MSB) | 1310 u32_encode_bits(1, HAL_WBM_SW_COOKIE_CFG_ALIGN); 1311 1312 ath12k_hif_write32(ab, wbm_base + HAL_WBM_SW_COOKIE_CFG1, val); 1313 1314 /* Enable conversion complete indication */ 1315 val = ath12k_hif_read32(ab, wbm_base + HAL_WBM_SW_COOKIE_CFG2); 1316 val |= u32_encode_bits(1, HAL_WBM_SW_COOKIE_CFG_RELEASE_PATH_EN) | 1317 u32_encode_bits(1, HAL_WBM_SW_COOKIE_CFG_ERR_PATH_EN) | 1318 u32_encode_bits(1, HAL_WBM_SW_COOKIE_CFG_CONV_IND_EN); 1319 1320 ath12k_hif_write32(ab, wbm_base + HAL_WBM_SW_COOKIE_CFG2, val); 1321 1322 /* Enable Cookie conversion for WBM2SW Rings */ 1323 val = ath12k_hif_read32(ab, wbm_base + HAL_WBM_SW_COOKIE_CONVERT_CFG); 1324 val |= u32_encode_bits(1, HAL_WBM_SW_COOKIE_CONV_CFG_GLOBAL_EN) | 1325 ab->hw_params->hal_params->wbm2sw_cc_enable; 1326 1327 ath12k_hif_write32(ab, wbm_base + HAL_WBM_SW_COOKIE_CONVERT_CFG, val); 1328 } 1329 1330 static u32 ath12k_dp_cc_cookie_gen(u16 ppt_idx, u16 spt_idx) 1331 { 1332 return (u32)ppt_idx << ATH12K_CC_PPT_SHIFT | spt_idx; 1333 } 1334 1335 static inline void *ath12k_dp_cc_get_desc_addr_ptr(struct ath12k_base *ab, 1336 u16 ppt_idx, u16 spt_idx) 1337 { 1338 struct ath12k_dp *dp = &ab->dp; 1339 1340 return dp->spt_info[ppt_idx].vaddr + spt_idx; 1341 } 1342 1343 struct ath12k_rx_desc_info *ath12k_dp_get_rx_desc(struct ath12k_base *ab, 1344 u32 cookie) 1345 { 1346 struct ath12k_rx_desc_info **desc_addr_ptr; 1347 u16 ppt_idx, spt_idx; 1348 1349 ppt_idx = u32_get_bits(cookie, ATH12K_DP_CC_COOKIE_PPT); 1350 spt_idx = u32_get_bits(cookie, ATH12k_DP_CC_COOKIE_SPT); 1351 1352 if (ppt_idx > ATH12K_NUM_RX_SPT_PAGES || 1353 spt_idx > ATH12K_MAX_SPT_ENTRIES) 1354 return NULL; 1355 1356 desc_addr_ptr = ath12k_dp_cc_get_desc_addr_ptr(ab, ppt_idx, spt_idx); 1357 1358 return *desc_addr_ptr; 1359 } 1360 1361 struct ath12k_tx_desc_info *ath12k_dp_get_tx_desc(struct ath12k_base *ab, 1362 u32 cookie) 1363 { 1364 struct ath12k_tx_desc_info **desc_addr_ptr; 1365 u16 ppt_idx, spt_idx; 1366 1367 ppt_idx = u32_get_bits(cookie, ATH12K_DP_CC_COOKIE_PPT); 1368 spt_idx = u32_get_bits(cookie, ATH12k_DP_CC_COOKIE_SPT); 1369 1370 if (ppt_idx < ATH12K_NUM_RX_SPT_PAGES || 1371 ppt_idx > ab->dp.num_spt_pages || 1372 spt_idx > ATH12K_MAX_SPT_ENTRIES) 1373 return NULL; 1374 1375 desc_addr_ptr = ath12k_dp_cc_get_desc_addr_ptr(ab, ppt_idx, spt_idx); 1376 1377 return *desc_addr_ptr; 1378 } 1379 1380 static int ath12k_dp_cc_desc_init(struct ath12k_base *ab) 1381 { 1382 struct ath12k_dp *dp = &ab->dp; 1383 struct ath12k_rx_desc_info *rx_descs, **rx_desc_addr; 1384 struct ath12k_tx_desc_info *tx_descs, **tx_desc_addr; 1385 u32 i, j, pool_id, tx_spt_page; 1386 u32 ppt_idx; 1387 1388 spin_lock_bh(&dp->rx_desc_lock); 1389 1390 /* First ATH12K_NUM_RX_SPT_PAGES of allocated SPT pages are used for RX */ 1391 for (i = 0; i < ATH12K_NUM_RX_SPT_PAGES; i++) { 1392 rx_descs = kcalloc(ATH12K_MAX_SPT_ENTRIES, sizeof(*rx_descs), 1393 GFP_ATOMIC); 1394 1395 if (!rx_descs) { 1396 spin_unlock_bh(&dp->rx_desc_lock); 1397 return -ENOMEM; 1398 } 1399 1400 dp->spt_info->rxbaddr[i] = &rx_descs[0]; 1401 1402 for (j = 0; j < ATH12K_MAX_SPT_ENTRIES; j++) { 1403 rx_descs[j].cookie = ath12k_dp_cc_cookie_gen(i, j); 1404 rx_descs[j].magic = ATH12K_DP_RX_DESC_MAGIC; 1405 list_add_tail(&rx_descs[j].list, &dp->rx_desc_free_list); 1406 1407 /* Update descriptor VA in SPT */ 1408 rx_desc_addr = ath12k_dp_cc_get_desc_addr_ptr(ab, i, j); 1409 *rx_desc_addr = &rx_descs[j]; 1410 } 1411 } 1412 1413 spin_unlock_bh(&dp->rx_desc_lock); 1414 1415 for (pool_id = 0; pool_id < ATH12K_HW_MAX_QUEUES; pool_id++) { 1416 spin_lock_bh(&dp->tx_desc_lock[pool_id]); 1417 for (i = 0; i < ATH12K_TX_SPT_PAGES_PER_POOL; i++) { 1418 tx_descs = kcalloc(ATH12K_MAX_SPT_ENTRIES, sizeof(*tx_descs), 1419 GFP_ATOMIC); 1420 1421 if (!tx_descs) { 1422 spin_unlock_bh(&dp->tx_desc_lock[pool_id]); 1423 /* Caller takes care of TX pending and RX desc cleanup */ 1424 return -ENOMEM; 1425 } 1426 1427 tx_spt_page = i + pool_id * ATH12K_TX_SPT_PAGES_PER_POOL; 1428 dp->spt_info->txbaddr[tx_spt_page] = &tx_descs[0]; 1429 1430 for (j = 0; j < ATH12K_MAX_SPT_ENTRIES; j++) { 1431 ppt_idx = ATH12K_NUM_RX_SPT_PAGES + tx_spt_page; 1432 tx_descs[j].desc_id = ath12k_dp_cc_cookie_gen(ppt_idx, j); 1433 tx_descs[j].pool_id = pool_id; 1434 list_add_tail(&tx_descs[j].list, 1435 &dp->tx_desc_free_list[pool_id]); 1436 1437 /* Update descriptor VA in SPT */ 1438 tx_desc_addr = 1439 ath12k_dp_cc_get_desc_addr_ptr(ab, ppt_idx, j); 1440 *tx_desc_addr = &tx_descs[j]; 1441 } 1442 } 1443 spin_unlock_bh(&dp->tx_desc_lock[pool_id]); 1444 } 1445 return 0; 1446 } 1447 1448 static int ath12k_dp_cc_init(struct ath12k_base *ab) 1449 { 1450 struct ath12k_dp *dp = &ab->dp; 1451 int i, ret = 0; 1452 u32 cmem_base; 1453 1454 INIT_LIST_HEAD(&dp->rx_desc_free_list); 1455 spin_lock_init(&dp->rx_desc_lock); 1456 1457 for (i = 0; i < ATH12K_HW_MAX_QUEUES; i++) { 1458 INIT_LIST_HEAD(&dp->tx_desc_free_list[i]); 1459 INIT_LIST_HEAD(&dp->tx_desc_used_list[i]); 1460 spin_lock_init(&dp->tx_desc_lock[i]); 1461 } 1462 1463 dp->num_spt_pages = ATH12K_NUM_SPT_PAGES; 1464 if (dp->num_spt_pages > ATH12K_MAX_PPT_ENTRIES) 1465 dp->num_spt_pages = ATH12K_MAX_PPT_ENTRIES; 1466 1467 dp->spt_info = kcalloc(dp->num_spt_pages, sizeof(struct ath12k_spt_info), 1468 GFP_KERNEL); 1469 1470 if (!dp->spt_info) { 1471 ath12k_warn(ab, "SPT page allocation failure"); 1472 return -ENOMEM; 1473 } 1474 1475 cmem_base = ab->qmi.dev_mem[ATH12K_QMI_DEVMEM_CMEM_INDEX].start; 1476 1477 for (i = 0; i < dp->num_spt_pages; i++) { 1478 dp->spt_info[i].vaddr = dma_alloc_coherent(ab->dev, 1479 ATH12K_PAGE_SIZE, 1480 &dp->spt_info[i].paddr, 1481 GFP_KERNEL); 1482 1483 if (!dp->spt_info[i].vaddr) { 1484 ret = -ENOMEM; 1485 goto free; 1486 } 1487 1488 if (dp->spt_info[i].paddr & ATH12K_SPT_4K_ALIGN_CHECK) { 1489 ath12k_warn(ab, "SPT allocated memory is not 4K aligned"); 1490 ret = -EINVAL; 1491 goto free; 1492 } 1493 1494 /* Write to PPT in CMEM */ 1495 ath12k_hif_write32(ab, cmem_base + ATH12K_PPT_ADDR_OFFSET(i), 1496 dp->spt_info[i].paddr >> ATH12K_SPT_4K_ALIGN_OFFSET); 1497 } 1498 1499 ret = ath12k_dp_cc_desc_init(ab); 1500 if (ret) { 1501 ath12k_warn(ab, "HW CC desc init failed %d", ret); 1502 goto free; 1503 } 1504 1505 return 0; 1506 free: 1507 ath12k_dp_cc_cleanup(ab); 1508 return ret; 1509 } 1510 1511 static int ath12k_dp_reoq_lut_setup(struct ath12k_base *ab) 1512 { 1513 struct ath12k_dp *dp = &ab->dp; 1514 1515 if (!ab->hw_params->reoq_lut_support) 1516 return 0; 1517 1518 dp->reoq_lut.vaddr = dma_alloc_coherent(ab->dev, 1519 DP_REOQ_LUT_SIZE, 1520 &dp->reoq_lut.paddr, 1521 GFP_KERNEL | __GFP_ZERO); 1522 if (!dp->reoq_lut.vaddr) { 1523 ath12k_warn(ab, "failed to allocate memory for reoq table"); 1524 return -ENOMEM; 1525 } 1526 1527 ath12k_hif_write32(ab, HAL_SEQ_WCSS_UMAC_REO_REG + HAL_REO1_QDESC_LUT_BASE0(ab), 1528 dp->reoq_lut.paddr); 1529 return 0; 1530 } 1531 1532 int ath12k_dp_alloc(struct ath12k_base *ab) 1533 { 1534 struct ath12k_dp *dp = &ab->dp; 1535 struct hal_srng *srng = NULL; 1536 size_t size = 0; 1537 u32 n_link_desc = 0; 1538 int ret; 1539 int i; 1540 1541 dp->ab = ab; 1542 1543 INIT_LIST_HEAD(&dp->reo_cmd_list); 1544 INIT_LIST_HEAD(&dp->reo_cmd_cache_flush_list); 1545 spin_lock_init(&dp->reo_cmd_lock); 1546 1547 dp->reo_cmd_cache_flush_count = 0; 1548 1549 ret = ath12k_wbm_idle_ring_setup(ab, &n_link_desc); 1550 if (ret) { 1551 ath12k_warn(ab, "failed to setup wbm_idle_ring: %d\n", ret); 1552 return ret; 1553 } 1554 1555 srng = &ab->hal.srng_list[dp->wbm_idle_ring.ring_id]; 1556 1557 ret = ath12k_dp_link_desc_setup(ab, dp->link_desc_banks, 1558 HAL_WBM_IDLE_LINK, srng, n_link_desc); 1559 if (ret) { 1560 ath12k_warn(ab, "failed to setup link desc: %d\n", ret); 1561 return ret; 1562 } 1563 1564 ret = ath12k_dp_cc_init(ab); 1565 1566 if (ret) { 1567 ath12k_warn(ab, "failed to setup cookie converter %d\n", ret); 1568 goto fail_link_desc_cleanup; 1569 } 1570 ret = ath12k_dp_init_bank_profiles(ab); 1571 if (ret) { 1572 ath12k_warn(ab, "failed to setup bank profiles %d\n", ret); 1573 goto fail_hw_cc_cleanup; 1574 } 1575 1576 ret = ath12k_dp_srng_common_setup(ab); 1577 if (ret) 1578 goto fail_dp_bank_profiles_cleanup; 1579 1580 size = sizeof(struct hal_wbm_release_ring_tx) * DP_TX_COMP_RING_SIZE; 1581 1582 ret = ath12k_dp_reoq_lut_setup(ab); 1583 if (ret) { 1584 ath12k_warn(ab, "failed to setup reoq table %d\n", ret); 1585 goto fail_cmn_srng_cleanup; 1586 } 1587 1588 for (i = 0; i < ab->hw_params->max_tx_ring; i++) { 1589 dp->tx_ring[i].tcl_data_ring_id = i; 1590 1591 dp->tx_ring[i].tx_status_head = 0; 1592 dp->tx_ring[i].tx_status_tail = DP_TX_COMP_RING_SIZE - 1; 1593 dp->tx_ring[i].tx_status = kmalloc(size, GFP_KERNEL); 1594 if (!dp->tx_ring[i].tx_status) { 1595 ret = -ENOMEM; 1596 /* FIXME: The allocated tx status is not freed 1597 * properly here 1598 */ 1599 goto fail_cmn_reoq_cleanup; 1600 } 1601 } 1602 1603 for (i = 0; i < HAL_DSCP_TID_MAP_TBL_NUM_ENTRIES_MAX; i++) 1604 ath12k_hal_tx_set_dscp_tid_map(ab, i); 1605 1606 ret = ath12k_dp_rx_alloc(ab); 1607 if (ret) 1608 goto fail_dp_rx_free; 1609 1610 /* Init any SOC level resource for DP */ 1611 1612 return 0; 1613 1614 fail_dp_rx_free: 1615 ath12k_dp_rx_free(ab); 1616 1617 fail_cmn_reoq_cleanup: 1618 ath12k_dp_reoq_lut_cleanup(ab); 1619 1620 fail_cmn_srng_cleanup: 1621 ath12k_dp_srng_common_cleanup(ab); 1622 1623 fail_dp_bank_profiles_cleanup: 1624 ath12k_dp_deinit_bank_profiles(ab); 1625 1626 fail_hw_cc_cleanup: 1627 ath12k_dp_cc_cleanup(ab); 1628 1629 fail_link_desc_cleanup: 1630 ath12k_dp_link_desc_cleanup(ab, dp->link_desc_banks, 1631 HAL_WBM_IDLE_LINK, &dp->wbm_idle_ring); 1632 1633 return ret; 1634 } 1635