1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright (C) 2021 Felix Fietkau <nbd@nbd.name> */ 3 4 #include <linux/kernel.h> 5 #include <linux/platform_device.h> 6 #include <linux/slab.h> 7 #include <linux/module.h> 8 #include <linux/bitfield.h> 9 #include <linux/dma-mapping.h> 10 #include <linux/skbuff.h> 11 #include <linux/of_platform.h> 12 #include <linux/of_address.h> 13 #include <linux/of_reserved_mem.h> 14 #include <linux/mfd/syscon.h> 15 #include <linux/debugfs.h> 16 #include <linux/soc/mediatek/mtk_wed.h> 17 #include <net/flow_offload.h> 18 #include <net/pkt_cls.h> 19 #include "mtk_eth_soc.h" 20 #include "mtk_wed.h" 21 #include "mtk_ppe.h" 22 #include "mtk_wed_wo.h" 23 24 #define MTK_PCIE_BASE(n) (0x1a143000 + (n) * 0x2000) 25 26 #define MTK_WED_PKT_SIZE 1920 27 #define MTK_WED_BUF_SIZE 2048 28 #define MTK_WED_PAGE_BUF_SIZE 128 29 #define MTK_WED_BUF_PER_PAGE (PAGE_SIZE / 2048) 30 #define MTK_WED_RX_BUF_PER_PAGE (PAGE_SIZE / MTK_WED_PAGE_BUF_SIZE) 31 #define MTK_WED_RX_RING_SIZE 1536 32 #define MTK_WED_RX_PG_BM_CNT 8192 33 #define MTK_WED_AMSDU_BUF_SIZE (PAGE_SIZE << 4) 34 #define MTK_WED_AMSDU_NPAGES 32 35 36 #define MTK_WED_TX_RING_SIZE 2048 37 #define MTK_WED_WDMA_RING_SIZE 1024 38 #define MTK_WED_MAX_GROUP_SIZE 0x100 39 #define MTK_WED_VLD_GROUP_SIZE 0x40 40 #define MTK_WED_PER_GROUP_PKT 128 41 42 #define MTK_WED_FBUF_SIZE 128 43 #define MTK_WED_MIOD_CNT 16 44 #define MTK_WED_FB_CMD_CNT 1024 45 #define MTK_WED_RRO_QUE_CNT 8192 46 #define MTK_WED_MIOD_ENTRY_CNT 128 47 48 #define MTK_WED_TX_BM_DMA_SIZE 65536 49 #define MTK_WED_TX_BM_PKT_CNT 32768 50 51 static struct mtk_wed_hw *hw_list[3]; 52 static DEFINE_MUTEX(hw_lock); 53 54 struct mtk_wed_flow_block_priv { 55 struct mtk_wed_hw *hw; 56 struct net_device *dev; 57 }; 58 59 static const struct mtk_wed_soc_data mt7622_data = { 60 .regmap = { 61 .tx_bm_tkid = 0x088, 62 .wpdma_rx_ring0 = 0x770, 63 .reset_idx_tx_mask = GENMASK(3, 0), 64 .reset_idx_rx_mask = GENMASK(17, 16), 65 }, 66 .tx_ring_desc_size = sizeof(struct mtk_wdma_desc), 67 .wdma_desc_size = sizeof(struct mtk_wdma_desc), 68 }; 69 70 static const struct mtk_wed_soc_data mt7986_data = { 71 .regmap = { 72 .tx_bm_tkid = 0x0c8, 73 .wpdma_rx_ring0 = 0x770, 74 .reset_idx_tx_mask = GENMASK(1, 0), 75 .reset_idx_rx_mask = GENMASK(7, 6), 76 }, 77 .tx_ring_desc_size = sizeof(struct mtk_wdma_desc), 78 .wdma_desc_size = 2 * sizeof(struct mtk_wdma_desc), 79 }; 80 81 static const struct mtk_wed_soc_data mt7988_data = { 82 .regmap = { 83 .tx_bm_tkid = 0x0c8, 84 .wpdma_rx_ring0 = 0x7d0, 85 .reset_idx_tx_mask = GENMASK(1, 0), 86 .reset_idx_rx_mask = GENMASK(7, 6), 87 }, 88 .tx_ring_desc_size = sizeof(struct mtk_wed_bm_desc), 89 .wdma_desc_size = 2 * sizeof(struct mtk_wdma_desc), 90 }; 91 92 static void 93 wed_m32(struct mtk_wed_device *dev, u32 reg, u32 mask, u32 val) 94 { 95 regmap_update_bits(dev->hw->regs, reg, mask | val, val); 96 } 97 98 static void 99 wed_set(struct mtk_wed_device *dev, u32 reg, u32 mask) 100 { 101 return wed_m32(dev, reg, 0, mask); 102 } 103 104 static void 105 wed_clr(struct mtk_wed_device *dev, u32 reg, u32 mask) 106 { 107 return wed_m32(dev, reg, mask, 0); 108 } 109 110 static void 111 wdma_m32(struct mtk_wed_device *dev, u32 reg, u32 mask, u32 val) 112 { 113 wdma_w32(dev, reg, (wdma_r32(dev, reg) & ~mask) | val); 114 } 115 116 static void 117 wdma_set(struct mtk_wed_device *dev, u32 reg, u32 mask) 118 { 119 wdma_m32(dev, reg, 0, mask); 120 } 121 122 static void 123 wdma_clr(struct mtk_wed_device *dev, u32 reg, u32 mask) 124 { 125 wdma_m32(dev, reg, mask, 0); 126 } 127 128 static u32 129 wifi_r32(struct mtk_wed_device *dev, u32 reg) 130 { 131 return readl(dev->wlan.base + reg); 132 } 133 134 static void 135 wifi_w32(struct mtk_wed_device *dev, u32 reg, u32 val) 136 { 137 writel(val, dev->wlan.base + reg); 138 } 139 140 static u32 141 mtk_wed_read_reset(struct mtk_wed_device *dev) 142 { 143 return wed_r32(dev, MTK_WED_RESET); 144 } 145 146 static u32 147 mtk_wdma_read_reset(struct mtk_wed_device *dev) 148 { 149 return wdma_r32(dev, MTK_WDMA_GLO_CFG); 150 } 151 152 static void 153 mtk_wdma_v3_rx_reset(struct mtk_wed_device *dev) 154 { 155 u32 status; 156 157 if (!mtk_wed_is_v3_or_greater(dev->hw)) 158 return; 159 160 wdma_clr(dev, MTK_WDMA_PREF_TX_CFG, MTK_WDMA_PREF_TX_CFG_PREF_EN); 161 wdma_clr(dev, MTK_WDMA_PREF_RX_CFG, MTK_WDMA_PREF_RX_CFG_PREF_EN); 162 163 if (read_poll_timeout(wdma_r32, status, 164 !(status & MTK_WDMA_PREF_TX_CFG_PREF_BUSY), 165 0, 10000, false, dev, MTK_WDMA_PREF_TX_CFG)) 166 dev_err(dev->hw->dev, "rx reset failed\n"); 167 168 if (read_poll_timeout(wdma_r32, status, 169 !(status & MTK_WDMA_PREF_RX_CFG_PREF_BUSY), 170 0, 10000, false, dev, MTK_WDMA_PREF_RX_CFG)) 171 dev_err(dev->hw->dev, "rx reset failed\n"); 172 173 wdma_clr(dev, MTK_WDMA_WRBK_TX_CFG, MTK_WDMA_WRBK_TX_CFG_WRBK_EN); 174 wdma_clr(dev, MTK_WDMA_WRBK_RX_CFG, MTK_WDMA_WRBK_RX_CFG_WRBK_EN); 175 176 if (read_poll_timeout(wdma_r32, status, 177 !(status & MTK_WDMA_WRBK_TX_CFG_WRBK_BUSY), 178 0, 10000, false, dev, MTK_WDMA_WRBK_TX_CFG)) 179 dev_err(dev->hw->dev, "rx reset failed\n"); 180 181 if (read_poll_timeout(wdma_r32, status, 182 !(status & MTK_WDMA_WRBK_RX_CFG_WRBK_BUSY), 183 0, 10000, false, dev, MTK_WDMA_WRBK_RX_CFG)) 184 dev_err(dev->hw->dev, "rx reset failed\n"); 185 186 /* prefetch FIFO */ 187 wdma_w32(dev, MTK_WDMA_PREF_RX_FIFO_CFG, 188 MTK_WDMA_PREF_RX_FIFO_CFG_RING0_CLEAR | 189 MTK_WDMA_PREF_RX_FIFO_CFG_RING1_CLEAR); 190 wdma_clr(dev, MTK_WDMA_PREF_RX_FIFO_CFG, 191 MTK_WDMA_PREF_RX_FIFO_CFG_RING0_CLEAR | 192 MTK_WDMA_PREF_RX_FIFO_CFG_RING1_CLEAR); 193 194 /* core FIFO */ 195 wdma_w32(dev, MTK_WDMA_XDMA_RX_FIFO_CFG, 196 MTK_WDMA_XDMA_RX_FIFO_CFG_RX_PAR_FIFO_CLEAR | 197 MTK_WDMA_XDMA_RX_FIFO_CFG_RX_CMD_FIFO_CLEAR | 198 MTK_WDMA_XDMA_RX_FIFO_CFG_RX_DMAD_FIFO_CLEAR | 199 MTK_WDMA_XDMA_RX_FIFO_CFG_RX_ARR_FIFO_CLEAR | 200 MTK_WDMA_XDMA_RX_FIFO_CFG_RX_LEN_FIFO_CLEAR | 201 MTK_WDMA_XDMA_RX_FIFO_CFG_RX_WID_FIFO_CLEAR | 202 MTK_WDMA_XDMA_RX_FIFO_CFG_RX_BID_FIFO_CLEAR); 203 wdma_clr(dev, MTK_WDMA_XDMA_RX_FIFO_CFG, 204 MTK_WDMA_XDMA_RX_FIFO_CFG_RX_PAR_FIFO_CLEAR | 205 MTK_WDMA_XDMA_RX_FIFO_CFG_RX_CMD_FIFO_CLEAR | 206 MTK_WDMA_XDMA_RX_FIFO_CFG_RX_DMAD_FIFO_CLEAR | 207 MTK_WDMA_XDMA_RX_FIFO_CFG_RX_ARR_FIFO_CLEAR | 208 MTK_WDMA_XDMA_RX_FIFO_CFG_RX_LEN_FIFO_CLEAR | 209 MTK_WDMA_XDMA_RX_FIFO_CFG_RX_WID_FIFO_CLEAR | 210 MTK_WDMA_XDMA_RX_FIFO_CFG_RX_BID_FIFO_CLEAR); 211 212 /* writeback FIFO */ 213 wdma_w32(dev, MTK_WDMA_WRBK_RX_FIFO_CFG(0), 214 MTK_WDMA_WRBK_RX_FIFO_CFG_RING_CLEAR); 215 wdma_w32(dev, MTK_WDMA_WRBK_RX_FIFO_CFG(1), 216 MTK_WDMA_WRBK_RX_FIFO_CFG_RING_CLEAR); 217 218 wdma_clr(dev, MTK_WDMA_WRBK_RX_FIFO_CFG(0), 219 MTK_WDMA_WRBK_RX_FIFO_CFG_RING_CLEAR); 220 wdma_clr(dev, MTK_WDMA_WRBK_RX_FIFO_CFG(1), 221 MTK_WDMA_WRBK_RX_FIFO_CFG_RING_CLEAR); 222 223 /* prefetch ring status */ 224 wdma_w32(dev, MTK_WDMA_PREF_SIDX_CFG, 225 MTK_WDMA_PREF_SIDX_CFG_RX_RING_CLEAR); 226 wdma_clr(dev, MTK_WDMA_PREF_SIDX_CFG, 227 MTK_WDMA_PREF_SIDX_CFG_RX_RING_CLEAR); 228 229 /* writeback ring status */ 230 wdma_w32(dev, MTK_WDMA_WRBK_SIDX_CFG, 231 MTK_WDMA_WRBK_SIDX_CFG_RX_RING_CLEAR); 232 wdma_clr(dev, MTK_WDMA_WRBK_SIDX_CFG, 233 MTK_WDMA_WRBK_SIDX_CFG_RX_RING_CLEAR); 234 } 235 236 static int 237 mtk_wdma_rx_reset(struct mtk_wed_device *dev) 238 { 239 u32 status, mask = MTK_WDMA_GLO_CFG_RX_DMA_BUSY; 240 int i, ret; 241 242 wdma_clr(dev, MTK_WDMA_GLO_CFG, MTK_WDMA_GLO_CFG_RX_DMA_EN); 243 ret = readx_poll_timeout(mtk_wdma_read_reset, dev, status, 244 !(status & mask), 0, 10000); 245 if (ret) 246 dev_err(dev->hw->dev, "rx reset failed\n"); 247 248 mtk_wdma_v3_rx_reset(dev); 249 wdma_w32(dev, MTK_WDMA_RESET_IDX, MTK_WDMA_RESET_IDX_RX); 250 wdma_w32(dev, MTK_WDMA_RESET_IDX, 0); 251 252 for (i = 0; i < ARRAY_SIZE(dev->rx_wdma); i++) { 253 if (dev->rx_wdma[i].desc) 254 continue; 255 256 wdma_w32(dev, 257 MTK_WDMA_RING_RX(i) + MTK_WED_RING_OFS_CPU_IDX, 0); 258 } 259 260 return ret; 261 } 262 263 static u32 264 mtk_wed_check_busy(struct mtk_wed_device *dev, u32 reg, u32 mask) 265 { 266 return !!(wed_r32(dev, reg) & mask); 267 } 268 269 static int 270 mtk_wed_poll_busy(struct mtk_wed_device *dev, u32 reg, u32 mask) 271 { 272 int sleep = 15000; 273 int timeout = 100 * sleep; 274 u32 val; 275 276 return read_poll_timeout(mtk_wed_check_busy, val, !val, sleep, 277 timeout, false, dev, reg, mask); 278 } 279 280 static void 281 mtk_wdma_v3_tx_reset(struct mtk_wed_device *dev) 282 { 283 u32 status; 284 285 if (!mtk_wed_is_v3_or_greater(dev->hw)) 286 return; 287 288 wdma_clr(dev, MTK_WDMA_PREF_TX_CFG, MTK_WDMA_PREF_TX_CFG_PREF_EN); 289 wdma_clr(dev, MTK_WDMA_PREF_RX_CFG, MTK_WDMA_PREF_RX_CFG_PREF_EN); 290 291 if (read_poll_timeout(wdma_r32, status, 292 !(status & MTK_WDMA_PREF_TX_CFG_PREF_BUSY), 293 0, 10000, false, dev, MTK_WDMA_PREF_TX_CFG)) 294 dev_err(dev->hw->dev, "tx reset failed\n"); 295 296 if (read_poll_timeout(wdma_r32, status, 297 !(status & MTK_WDMA_PREF_RX_CFG_PREF_BUSY), 298 0, 10000, false, dev, MTK_WDMA_PREF_RX_CFG)) 299 dev_err(dev->hw->dev, "tx reset failed\n"); 300 301 wdma_clr(dev, MTK_WDMA_WRBK_TX_CFG, MTK_WDMA_WRBK_TX_CFG_WRBK_EN); 302 wdma_clr(dev, MTK_WDMA_WRBK_RX_CFG, MTK_WDMA_WRBK_RX_CFG_WRBK_EN); 303 304 if (read_poll_timeout(wdma_r32, status, 305 !(status & MTK_WDMA_WRBK_TX_CFG_WRBK_BUSY), 306 0, 10000, false, dev, MTK_WDMA_WRBK_TX_CFG)) 307 dev_err(dev->hw->dev, "tx reset failed\n"); 308 309 if (read_poll_timeout(wdma_r32, status, 310 !(status & MTK_WDMA_WRBK_RX_CFG_WRBK_BUSY), 311 0, 10000, false, dev, MTK_WDMA_WRBK_RX_CFG)) 312 dev_err(dev->hw->dev, "tx reset failed\n"); 313 314 /* prefetch FIFO */ 315 wdma_w32(dev, MTK_WDMA_PREF_TX_FIFO_CFG, 316 MTK_WDMA_PREF_TX_FIFO_CFG_RING0_CLEAR | 317 MTK_WDMA_PREF_TX_FIFO_CFG_RING1_CLEAR); 318 wdma_clr(dev, MTK_WDMA_PREF_TX_FIFO_CFG, 319 MTK_WDMA_PREF_TX_FIFO_CFG_RING0_CLEAR | 320 MTK_WDMA_PREF_TX_FIFO_CFG_RING1_CLEAR); 321 322 /* core FIFO */ 323 wdma_w32(dev, MTK_WDMA_XDMA_TX_FIFO_CFG, 324 MTK_WDMA_XDMA_TX_FIFO_CFG_TX_PAR_FIFO_CLEAR | 325 MTK_WDMA_XDMA_TX_FIFO_CFG_TX_CMD_FIFO_CLEAR | 326 MTK_WDMA_XDMA_TX_FIFO_CFG_TX_DMAD_FIFO_CLEAR | 327 MTK_WDMA_XDMA_TX_FIFO_CFG_TX_ARR_FIFO_CLEAR); 328 wdma_clr(dev, MTK_WDMA_XDMA_TX_FIFO_CFG, 329 MTK_WDMA_XDMA_TX_FIFO_CFG_TX_PAR_FIFO_CLEAR | 330 MTK_WDMA_XDMA_TX_FIFO_CFG_TX_CMD_FIFO_CLEAR | 331 MTK_WDMA_XDMA_TX_FIFO_CFG_TX_DMAD_FIFO_CLEAR | 332 MTK_WDMA_XDMA_TX_FIFO_CFG_TX_ARR_FIFO_CLEAR); 333 334 /* writeback FIFO */ 335 wdma_w32(dev, MTK_WDMA_WRBK_TX_FIFO_CFG(0), 336 MTK_WDMA_WRBK_TX_FIFO_CFG_RING_CLEAR); 337 wdma_w32(dev, MTK_WDMA_WRBK_TX_FIFO_CFG(1), 338 MTK_WDMA_WRBK_TX_FIFO_CFG_RING_CLEAR); 339 340 wdma_clr(dev, MTK_WDMA_WRBK_TX_FIFO_CFG(0), 341 MTK_WDMA_WRBK_TX_FIFO_CFG_RING_CLEAR); 342 wdma_clr(dev, MTK_WDMA_WRBK_TX_FIFO_CFG(1), 343 MTK_WDMA_WRBK_TX_FIFO_CFG_RING_CLEAR); 344 345 /* prefetch ring status */ 346 wdma_w32(dev, MTK_WDMA_PREF_SIDX_CFG, 347 MTK_WDMA_PREF_SIDX_CFG_TX_RING_CLEAR); 348 wdma_clr(dev, MTK_WDMA_PREF_SIDX_CFG, 349 MTK_WDMA_PREF_SIDX_CFG_TX_RING_CLEAR); 350 351 /* writeback ring status */ 352 wdma_w32(dev, MTK_WDMA_WRBK_SIDX_CFG, 353 MTK_WDMA_WRBK_SIDX_CFG_TX_RING_CLEAR); 354 wdma_clr(dev, MTK_WDMA_WRBK_SIDX_CFG, 355 MTK_WDMA_WRBK_SIDX_CFG_TX_RING_CLEAR); 356 } 357 358 static void 359 mtk_wdma_tx_reset(struct mtk_wed_device *dev) 360 { 361 u32 status, mask = MTK_WDMA_GLO_CFG_TX_DMA_BUSY; 362 int i; 363 364 wdma_clr(dev, MTK_WDMA_GLO_CFG, MTK_WDMA_GLO_CFG_TX_DMA_EN); 365 if (readx_poll_timeout(mtk_wdma_read_reset, dev, status, 366 !(status & mask), 0, 10000)) 367 dev_err(dev->hw->dev, "tx reset failed\n"); 368 369 mtk_wdma_v3_tx_reset(dev); 370 wdma_w32(dev, MTK_WDMA_RESET_IDX, MTK_WDMA_RESET_IDX_TX); 371 wdma_w32(dev, MTK_WDMA_RESET_IDX, 0); 372 373 for (i = 0; i < ARRAY_SIZE(dev->tx_wdma); i++) 374 wdma_w32(dev, 375 MTK_WDMA_RING_TX(i) + MTK_WED_RING_OFS_CPU_IDX, 0); 376 } 377 378 static void 379 mtk_wed_reset(struct mtk_wed_device *dev, u32 mask) 380 { 381 u32 status; 382 383 wed_w32(dev, MTK_WED_RESET, mask); 384 if (readx_poll_timeout(mtk_wed_read_reset, dev, status, 385 !(status & mask), 0, 1000)) 386 WARN_ON_ONCE(1); 387 } 388 389 static u32 390 mtk_wed_wo_read_status(struct mtk_wed_device *dev) 391 { 392 return wed_r32(dev, MTK_WED_SCR0 + 4 * MTK_WED_DUMMY_CR_WO_STATUS); 393 } 394 395 static void 396 mtk_wed_wo_reset(struct mtk_wed_device *dev) 397 { 398 struct mtk_wed_wo *wo = dev->hw->wed_wo; 399 u8 state = MTK_WED_WO_STATE_DISABLE; 400 void __iomem *reg; 401 u32 val; 402 403 mtk_wdma_tx_reset(dev); 404 mtk_wed_reset(dev, MTK_WED_RESET_WED); 405 406 if (mtk_wed_mcu_send_msg(wo, MTK_WED_MODULE_ID_WO, 407 MTK_WED_WO_CMD_CHANGE_STATE, &state, 408 sizeof(state), false)) 409 return; 410 411 if (readx_poll_timeout(mtk_wed_wo_read_status, dev, val, 412 val == MTK_WED_WOIF_DISABLE_DONE, 413 100, MTK_WOCPU_TIMEOUT)) 414 dev_err(dev->hw->dev, "failed to disable wed-wo\n"); 415 416 reg = ioremap(MTK_WED_WO_CPU_MCUSYS_RESET_ADDR, 4); 417 418 val = readl(reg); 419 switch (dev->hw->index) { 420 case 0: 421 val |= MTK_WED_WO_CPU_WO0_MCUSYS_RESET_MASK; 422 writel(val, reg); 423 val &= ~MTK_WED_WO_CPU_WO0_MCUSYS_RESET_MASK; 424 writel(val, reg); 425 break; 426 case 1: 427 val |= MTK_WED_WO_CPU_WO1_MCUSYS_RESET_MASK; 428 writel(val, reg); 429 val &= ~MTK_WED_WO_CPU_WO1_MCUSYS_RESET_MASK; 430 writel(val, reg); 431 break; 432 default: 433 break; 434 } 435 iounmap(reg); 436 } 437 438 void mtk_wed_fe_reset(void) 439 { 440 int i; 441 442 mutex_lock(&hw_lock); 443 444 for (i = 0; i < ARRAY_SIZE(hw_list); i++) { 445 struct mtk_wed_hw *hw = hw_list[i]; 446 struct mtk_wed_device *dev; 447 int err; 448 449 if (!hw) 450 break; 451 452 dev = hw->wed_dev; 453 if (!dev || !dev->wlan.reset) 454 continue; 455 456 /* reset callback blocks until WLAN reset is completed */ 457 err = dev->wlan.reset(dev); 458 if (err) 459 dev_err(dev->dev, "wlan reset failed: %d\n", err); 460 } 461 462 mutex_unlock(&hw_lock); 463 } 464 465 void mtk_wed_fe_reset_complete(void) 466 { 467 int i; 468 469 mutex_lock(&hw_lock); 470 471 for (i = 0; i < ARRAY_SIZE(hw_list); i++) { 472 struct mtk_wed_hw *hw = hw_list[i]; 473 struct mtk_wed_device *dev; 474 475 if (!hw) 476 break; 477 478 dev = hw->wed_dev; 479 if (!dev || !dev->wlan.reset_complete) 480 continue; 481 482 dev->wlan.reset_complete(dev); 483 } 484 485 mutex_unlock(&hw_lock); 486 } 487 488 static struct mtk_wed_hw * 489 mtk_wed_assign(struct mtk_wed_device *dev) 490 { 491 struct mtk_wed_hw *hw; 492 int i; 493 494 if (dev->wlan.bus_type == MTK_WED_BUS_PCIE) { 495 hw = hw_list[pci_domain_nr(dev->wlan.pci_dev->bus)]; 496 if (!hw) 497 return NULL; 498 499 if (!hw->wed_dev) 500 goto out; 501 502 if (mtk_wed_is_v1(hw)) 503 return NULL; 504 505 /* MT7986 WED devices do not have any pcie slot restrictions */ 506 } 507 /* MT7986 PCIE or AXI */ 508 for (i = 0; i < ARRAY_SIZE(hw_list); i++) { 509 hw = hw_list[i]; 510 if (hw && !hw->wed_dev) 511 goto out; 512 } 513 514 return NULL; 515 516 out: 517 hw->wed_dev = dev; 518 return hw; 519 } 520 521 static int 522 mtk_wed_amsdu_buffer_alloc(struct mtk_wed_device *dev) 523 { 524 struct mtk_wed_hw *hw = dev->hw; 525 struct mtk_wed_amsdu *wed_amsdu; 526 int i; 527 528 if (!mtk_wed_is_v3_or_greater(hw)) 529 return 0; 530 531 wed_amsdu = devm_kcalloc(hw->dev, MTK_WED_AMSDU_NPAGES, 532 sizeof(*wed_amsdu), GFP_KERNEL); 533 if (!wed_amsdu) 534 return -ENOMEM; 535 536 for (i = 0; i < MTK_WED_AMSDU_NPAGES; i++) { 537 void *ptr; 538 539 /* each segment is 64K */ 540 ptr = (void *)__get_free_pages(GFP_KERNEL | __GFP_NOWARN | 541 __GFP_ZERO | __GFP_COMP | 542 GFP_DMA32, 543 get_order(MTK_WED_AMSDU_BUF_SIZE)); 544 if (!ptr) 545 goto error; 546 547 wed_amsdu[i].txd = ptr; 548 wed_amsdu[i].txd_phy = dma_map_single(hw->dev, ptr, 549 MTK_WED_AMSDU_BUF_SIZE, 550 DMA_TO_DEVICE); 551 if (dma_mapping_error(hw->dev, wed_amsdu[i].txd_phy)) 552 goto error; 553 } 554 dev->hw->wed_amsdu = wed_amsdu; 555 556 return 0; 557 558 error: 559 for (i--; i >= 0; i--) 560 dma_unmap_single(hw->dev, wed_amsdu[i].txd_phy, 561 MTK_WED_AMSDU_BUF_SIZE, DMA_TO_DEVICE); 562 return -ENOMEM; 563 } 564 565 static void 566 mtk_wed_amsdu_free_buffer(struct mtk_wed_device *dev) 567 { 568 struct mtk_wed_amsdu *wed_amsdu = dev->hw->wed_amsdu; 569 int i; 570 571 if (!wed_amsdu) 572 return; 573 574 for (i = 0; i < MTK_WED_AMSDU_NPAGES; i++) { 575 dma_unmap_single(dev->hw->dev, wed_amsdu[i].txd_phy, 576 MTK_WED_AMSDU_BUF_SIZE, DMA_TO_DEVICE); 577 free_pages((unsigned long)wed_amsdu[i].txd, 578 get_order(MTK_WED_AMSDU_BUF_SIZE)); 579 } 580 } 581 582 static int 583 mtk_wed_amsdu_init(struct mtk_wed_device *dev) 584 { 585 struct mtk_wed_amsdu *wed_amsdu = dev->hw->wed_amsdu; 586 int i, ret; 587 588 if (!wed_amsdu) 589 return 0; 590 591 for (i = 0; i < MTK_WED_AMSDU_NPAGES; i++) 592 wed_w32(dev, MTK_WED_AMSDU_HIFTXD_BASE_L(i), 593 wed_amsdu[i].txd_phy); 594 595 /* init all sta parameter */ 596 wed_w32(dev, MTK_WED_AMSDU_STA_INFO_INIT, MTK_WED_AMSDU_STA_RMVL | 597 MTK_WED_AMSDU_STA_WTBL_HDRT_MODE | 598 FIELD_PREP(MTK_WED_AMSDU_STA_MAX_AMSDU_LEN, 599 dev->wlan.amsdu_max_len >> 8) | 600 FIELD_PREP(MTK_WED_AMSDU_STA_MAX_AMSDU_NUM, 601 dev->wlan.amsdu_max_subframes)); 602 603 wed_w32(dev, MTK_WED_AMSDU_STA_INFO, MTK_WED_AMSDU_STA_INFO_DO_INIT); 604 605 ret = mtk_wed_poll_busy(dev, MTK_WED_AMSDU_STA_INFO, 606 MTK_WED_AMSDU_STA_INFO_DO_INIT); 607 if (ret) { 608 dev_err(dev->hw->dev, "amsdu initialization failed\n"); 609 return ret; 610 } 611 612 /* init partial amsdu offload txd src */ 613 wed_set(dev, MTK_WED_AMSDU_HIFTXD_CFG, 614 FIELD_PREP(MTK_WED_AMSDU_HIFTXD_SRC, dev->hw->index)); 615 616 /* init qmem */ 617 wed_set(dev, MTK_WED_AMSDU_PSE, MTK_WED_AMSDU_PSE_RESET); 618 ret = mtk_wed_poll_busy(dev, MTK_WED_MON_AMSDU_QMEM_STS1, BIT(29)); 619 if (ret) { 620 pr_info("%s: amsdu qmem initialization failed\n", __func__); 621 return ret; 622 } 623 624 /* eagle E1 PCIE1 tx ring 22 flow control issue */ 625 if (dev->wlan.id == 0x7991) 626 wed_clr(dev, MTK_WED_AMSDU_FIFO, MTK_WED_AMSDU_IS_PRIOR0_RING); 627 628 wed_set(dev, MTK_WED_CTRL, MTK_WED_CTRL_TX_AMSDU_EN); 629 630 return 0; 631 } 632 633 static int 634 mtk_wed_tx_buffer_alloc(struct mtk_wed_device *dev) 635 { 636 u32 desc_size = dev->hw->soc->tx_ring_desc_size; 637 int i, page_idx = 0, n_pages, ring_size; 638 int token = dev->wlan.token_start; 639 struct mtk_wed_buf *page_list; 640 dma_addr_t desc_phys; 641 void *desc_ptr; 642 643 if (!mtk_wed_is_v3_or_greater(dev->hw)) { 644 ring_size = dev->wlan.nbuf & ~(MTK_WED_BUF_PER_PAGE - 1); 645 dev->tx_buf_ring.size = ring_size; 646 } else { 647 dev->tx_buf_ring.size = MTK_WED_TX_BM_DMA_SIZE; 648 ring_size = MTK_WED_TX_BM_PKT_CNT; 649 } 650 n_pages = dev->tx_buf_ring.size / MTK_WED_BUF_PER_PAGE; 651 652 page_list = kcalloc(n_pages, sizeof(*page_list), GFP_KERNEL); 653 if (!page_list) 654 return -ENOMEM; 655 656 dev->tx_buf_ring.pages = page_list; 657 658 desc_ptr = dma_alloc_coherent(dev->hw->dev, 659 dev->tx_buf_ring.size * desc_size, 660 &desc_phys, GFP_KERNEL); 661 if (!desc_ptr) 662 return -ENOMEM; 663 664 dev->tx_buf_ring.desc = desc_ptr; 665 dev->tx_buf_ring.desc_phys = desc_phys; 666 667 for (i = 0; i < ring_size; i += MTK_WED_BUF_PER_PAGE) { 668 dma_addr_t page_phys, buf_phys; 669 struct page *page; 670 void *buf; 671 int s; 672 673 page = __dev_alloc_page(GFP_KERNEL); 674 if (!page) 675 return -ENOMEM; 676 677 page_phys = dma_map_page(dev->hw->dev, page, 0, PAGE_SIZE, 678 DMA_BIDIRECTIONAL); 679 if (dma_mapping_error(dev->hw->dev, page_phys)) { 680 __free_page(page); 681 return -ENOMEM; 682 } 683 684 page_list[page_idx].p = page; 685 page_list[page_idx++].phy_addr = page_phys; 686 dma_sync_single_for_cpu(dev->hw->dev, page_phys, PAGE_SIZE, 687 DMA_BIDIRECTIONAL); 688 689 buf = page_to_virt(page); 690 buf_phys = page_phys; 691 692 for (s = 0; s < MTK_WED_BUF_PER_PAGE; s++) { 693 struct mtk_wdma_desc *desc = desc_ptr; 694 u32 ctrl; 695 696 desc->buf0 = cpu_to_le32(buf_phys); 697 if (!mtk_wed_is_v3_or_greater(dev->hw)) { 698 u32 txd_size; 699 700 txd_size = dev->wlan.init_buf(buf, buf_phys, 701 token++); 702 desc->buf1 = cpu_to_le32(buf_phys + txd_size); 703 ctrl = FIELD_PREP(MTK_WDMA_DESC_CTRL_LEN0, txd_size); 704 if (mtk_wed_is_v1(dev->hw)) 705 ctrl |= MTK_WDMA_DESC_CTRL_LAST_SEG1 | 706 FIELD_PREP(MTK_WDMA_DESC_CTRL_LEN1, 707 MTK_WED_BUF_SIZE - txd_size); 708 else 709 ctrl |= MTK_WDMA_DESC_CTRL_LAST_SEG0 | 710 FIELD_PREP(MTK_WDMA_DESC_CTRL_LEN1_V2, 711 MTK_WED_BUF_SIZE - txd_size); 712 desc->info = 0; 713 } else { 714 ctrl = token << 16 | TX_DMA_PREP_ADDR64(buf_phys); 715 } 716 desc->ctrl = cpu_to_le32(ctrl); 717 718 desc_ptr += desc_size; 719 buf += MTK_WED_BUF_SIZE; 720 buf_phys += MTK_WED_BUF_SIZE; 721 } 722 723 dma_sync_single_for_device(dev->hw->dev, page_phys, PAGE_SIZE, 724 DMA_BIDIRECTIONAL); 725 } 726 727 return 0; 728 } 729 730 static void 731 mtk_wed_free_tx_buffer(struct mtk_wed_device *dev) 732 { 733 struct mtk_wed_buf *page_list = dev->tx_buf_ring.pages; 734 struct mtk_wed_hw *hw = dev->hw; 735 int i, page_idx = 0; 736 737 if (!page_list) 738 return; 739 740 if (!dev->tx_buf_ring.desc) 741 goto free_pagelist; 742 743 for (i = 0; i < dev->tx_buf_ring.size; i += MTK_WED_BUF_PER_PAGE) { 744 dma_addr_t page_phy = page_list[page_idx].phy_addr; 745 void *page = page_list[page_idx++].p; 746 747 if (!page) 748 break; 749 750 dma_unmap_page(dev->hw->dev, page_phy, PAGE_SIZE, 751 DMA_BIDIRECTIONAL); 752 __free_page(page); 753 } 754 755 dma_free_coherent(dev->hw->dev, 756 dev->tx_buf_ring.size * hw->soc->tx_ring_desc_size, 757 dev->tx_buf_ring.desc, 758 dev->tx_buf_ring.desc_phys); 759 760 free_pagelist: 761 kfree(page_list); 762 } 763 764 static int 765 mtk_wed_hwrro_buffer_alloc(struct mtk_wed_device *dev) 766 { 767 int n_pages = MTK_WED_RX_PG_BM_CNT / MTK_WED_RX_BUF_PER_PAGE; 768 struct mtk_wed_buf *page_list; 769 struct mtk_wed_bm_desc *desc; 770 dma_addr_t desc_phys; 771 int i, page_idx = 0; 772 773 if (!dev->wlan.hw_rro) 774 return 0; 775 776 page_list = kcalloc(n_pages, sizeof(*page_list), GFP_KERNEL); 777 if (!page_list) 778 return -ENOMEM; 779 780 dev->hw_rro.size = dev->wlan.rx_nbuf & ~(MTK_WED_BUF_PER_PAGE - 1); 781 dev->hw_rro.pages = page_list; 782 desc = dma_alloc_coherent(dev->hw->dev, 783 dev->wlan.rx_nbuf * sizeof(*desc), 784 &desc_phys, GFP_KERNEL); 785 if (!desc) 786 return -ENOMEM; 787 788 dev->hw_rro.desc = desc; 789 dev->hw_rro.desc_phys = desc_phys; 790 791 for (i = 0; i < MTK_WED_RX_PG_BM_CNT; i += MTK_WED_RX_BUF_PER_PAGE) { 792 dma_addr_t page_phys, buf_phys; 793 struct page *page; 794 int s; 795 796 page = __dev_alloc_page(GFP_KERNEL); 797 if (!page) 798 return -ENOMEM; 799 800 page_phys = dma_map_page(dev->hw->dev, page, 0, PAGE_SIZE, 801 DMA_BIDIRECTIONAL); 802 if (dma_mapping_error(dev->hw->dev, page_phys)) { 803 __free_page(page); 804 return -ENOMEM; 805 } 806 807 page_list[page_idx].p = page; 808 page_list[page_idx++].phy_addr = page_phys; 809 dma_sync_single_for_cpu(dev->hw->dev, page_phys, PAGE_SIZE, 810 DMA_BIDIRECTIONAL); 811 812 buf_phys = page_phys; 813 for (s = 0; s < MTK_WED_RX_BUF_PER_PAGE; s++) { 814 desc->buf0 = cpu_to_le32(buf_phys); 815 desc->token = cpu_to_le32(RX_DMA_PREP_ADDR64(buf_phys)); 816 buf_phys += MTK_WED_PAGE_BUF_SIZE; 817 desc++; 818 } 819 820 dma_sync_single_for_device(dev->hw->dev, page_phys, PAGE_SIZE, 821 DMA_BIDIRECTIONAL); 822 } 823 824 return 0; 825 } 826 827 static int 828 mtk_wed_rx_buffer_alloc(struct mtk_wed_device *dev) 829 { 830 struct mtk_wed_bm_desc *desc; 831 dma_addr_t desc_phys; 832 833 dev->rx_buf_ring.size = dev->wlan.rx_nbuf; 834 desc = dma_alloc_coherent(dev->hw->dev, 835 dev->wlan.rx_nbuf * sizeof(*desc), 836 &desc_phys, GFP_KERNEL); 837 if (!desc) 838 return -ENOMEM; 839 840 dev->rx_buf_ring.desc = desc; 841 dev->rx_buf_ring.desc_phys = desc_phys; 842 dev->wlan.init_rx_buf(dev, dev->wlan.rx_npkt); 843 844 return mtk_wed_hwrro_buffer_alloc(dev); 845 } 846 847 static void 848 mtk_wed_hwrro_free_buffer(struct mtk_wed_device *dev) 849 { 850 struct mtk_wed_buf *page_list = dev->hw_rro.pages; 851 struct mtk_wed_bm_desc *desc = dev->hw_rro.desc; 852 int i, page_idx = 0; 853 854 if (!dev->wlan.hw_rro) 855 return; 856 857 if (!page_list) 858 return; 859 860 if (!desc) 861 goto free_pagelist; 862 863 for (i = 0; i < MTK_WED_RX_PG_BM_CNT; i += MTK_WED_RX_BUF_PER_PAGE) { 864 dma_addr_t buf_addr = page_list[page_idx].phy_addr; 865 void *page = page_list[page_idx++].p; 866 867 if (!page) 868 break; 869 870 dma_unmap_page(dev->hw->dev, buf_addr, PAGE_SIZE, 871 DMA_BIDIRECTIONAL); 872 __free_page(page); 873 } 874 875 dma_free_coherent(dev->hw->dev, dev->hw_rro.size * sizeof(*desc), 876 desc, dev->hw_rro.desc_phys); 877 878 free_pagelist: 879 kfree(page_list); 880 } 881 882 static void 883 mtk_wed_free_rx_buffer(struct mtk_wed_device *dev) 884 { 885 struct mtk_wed_bm_desc *desc = dev->rx_buf_ring.desc; 886 887 if (!desc) 888 return; 889 890 dev->wlan.release_rx_buf(dev); 891 dma_free_coherent(dev->hw->dev, dev->rx_buf_ring.size * sizeof(*desc), 892 desc, dev->rx_buf_ring.desc_phys); 893 894 mtk_wed_hwrro_free_buffer(dev); 895 } 896 897 static void 898 mtk_wed_hwrro_init(struct mtk_wed_device *dev) 899 { 900 if (!mtk_wed_get_rx_capa(dev) || !dev->wlan.hw_rro) 901 return; 902 903 wed_set(dev, MTK_WED_RRO_PG_BM_RX_DMAM, 904 FIELD_PREP(MTK_WED_RRO_PG_BM_RX_SDL0, 128)); 905 906 wed_w32(dev, MTK_WED_RRO_PG_BM_BASE, dev->hw_rro.desc_phys); 907 908 wed_w32(dev, MTK_WED_RRO_PG_BM_INIT_PTR, 909 MTK_WED_RRO_PG_BM_INIT_SW_TAIL_IDX | 910 FIELD_PREP(MTK_WED_RRO_PG_BM_SW_TAIL_IDX, 911 MTK_WED_RX_PG_BM_CNT)); 912 913 /* enable rx_page_bm to fetch dmad */ 914 wed_set(dev, MTK_WED_CTRL, MTK_WED_CTRL_WED_RX_PG_BM_EN); 915 } 916 917 static void 918 mtk_wed_rx_buffer_hw_init(struct mtk_wed_device *dev) 919 { 920 wed_w32(dev, MTK_WED_RX_BM_RX_DMAD, 921 FIELD_PREP(MTK_WED_RX_BM_RX_DMAD_SDL0, dev->wlan.rx_size)); 922 wed_w32(dev, MTK_WED_RX_BM_BASE, dev->rx_buf_ring.desc_phys); 923 wed_w32(dev, MTK_WED_RX_BM_INIT_PTR, MTK_WED_RX_BM_INIT_SW_TAIL | 924 FIELD_PREP(MTK_WED_RX_BM_SW_TAIL, dev->wlan.rx_npkt)); 925 wed_w32(dev, MTK_WED_RX_BM_DYN_ALLOC_TH, 926 FIELD_PREP(MTK_WED_RX_BM_DYN_ALLOC_TH_H, 0xffff)); 927 wed_set(dev, MTK_WED_CTRL, MTK_WED_CTRL_WED_RX_BM_EN); 928 929 mtk_wed_hwrro_init(dev); 930 } 931 932 static void 933 mtk_wed_free_ring(struct mtk_wed_device *dev, struct mtk_wed_ring *ring) 934 { 935 if (!ring->desc) 936 return; 937 938 dma_free_coherent(dev->hw->dev, ring->size * ring->desc_size, 939 ring->desc, ring->desc_phys); 940 } 941 942 static void 943 mtk_wed_free_rx_rings(struct mtk_wed_device *dev) 944 { 945 mtk_wed_free_rx_buffer(dev); 946 mtk_wed_free_ring(dev, &dev->rro.ring); 947 } 948 949 static void 950 mtk_wed_free_tx_rings(struct mtk_wed_device *dev) 951 { 952 int i; 953 954 for (i = 0; i < ARRAY_SIZE(dev->tx_ring); i++) 955 mtk_wed_free_ring(dev, &dev->tx_ring[i]); 956 for (i = 0; i < ARRAY_SIZE(dev->rx_wdma); i++) 957 mtk_wed_free_ring(dev, &dev->rx_wdma[i]); 958 } 959 960 static void 961 mtk_wed_set_ext_int(struct mtk_wed_device *dev, bool en) 962 { 963 u32 mask = MTK_WED_EXT_INT_STATUS_ERROR_MASK; 964 965 switch (dev->hw->version) { 966 case 1: 967 mask |= MTK_WED_EXT_INT_STATUS_TX_DRV_R_RESP_ERR; 968 break; 969 case 2: 970 mask |= MTK_WED_EXT_INT_STATUS_RX_FBUF_LO_TH | 971 MTK_WED_EXT_INT_STATUS_RX_FBUF_HI_TH | 972 MTK_WED_EXT_INT_STATUS_RX_DRV_COHERENT | 973 MTK_WED_EXT_INT_STATUS_TX_DMA_W_RESP_ERR; 974 break; 975 case 3: 976 mask = MTK_WED_EXT_INT_STATUS_RX_DRV_COHERENT | 977 MTK_WED_EXT_INT_STATUS_TKID_WO_PYLD; 978 break; 979 default: 980 break; 981 } 982 983 if (!dev->hw->num_flows) 984 mask &= ~MTK_WED_EXT_INT_STATUS_TKID_WO_PYLD; 985 986 wed_w32(dev, MTK_WED_EXT_INT_MASK, en ? mask : 0); 987 wed_r32(dev, MTK_WED_EXT_INT_MASK); 988 } 989 990 static void 991 mtk_wed_set_512_support(struct mtk_wed_device *dev, bool enable) 992 { 993 if (!mtk_wed_is_v2(dev->hw)) 994 return; 995 996 if (enable) { 997 wed_w32(dev, MTK_WED_TXDP_CTRL, MTK_WED_TXDP_DW9_OVERWR); 998 wed_w32(dev, MTK_WED_TXP_DW1, 999 FIELD_PREP(MTK_WED_WPDMA_WRITE_TXP, 0x0103)); 1000 } else { 1001 wed_w32(dev, MTK_WED_TXP_DW1, 1002 FIELD_PREP(MTK_WED_WPDMA_WRITE_TXP, 0x0100)); 1003 wed_clr(dev, MTK_WED_TXDP_CTRL, MTK_WED_TXDP_DW9_OVERWR); 1004 } 1005 } 1006 1007 static int 1008 mtk_wed_check_wfdma_rx_fill(struct mtk_wed_device *dev, 1009 struct mtk_wed_ring *ring) 1010 { 1011 int i; 1012 1013 for (i = 0; i < 3; i++) { 1014 u32 cur_idx = readl(ring->wpdma + MTK_WED_RING_OFS_CPU_IDX); 1015 1016 if (cur_idx == MTK_WED_RX_RING_SIZE - 1) 1017 break; 1018 1019 usleep_range(100000, 200000); 1020 } 1021 1022 if (i == 3) { 1023 dev_err(dev->hw->dev, "rx dma enable failed\n"); 1024 return -ETIMEDOUT; 1025 } 1026 1027 return 0; 1028 } 1029 1030 static void 1031 mtk_wed_dma_disable(struct mtk_wed_device *dev) 1032 { 1033 wed_clr(dev, MTK_WED_WPDMA_GLO_CFG, 1034 MTK_WED_WPDMA_GLO_CFG_TX_DRV_EN | 1035 MTK_WED_WPDMA_GLO_CFG_RX_DRV_EN); 1036 1037 wed_clr(dev, MTK_WED_WDMA_GLO_CFG, MTK_WED_WDMA_GLO_CFG_RX_DRV_EN); 1038 1039 wed_clr(dev, MTK_WED_GLO_CFG, 1040 MTK_WED_GLO_CFG_TX_DMA_EN | 1041 MTK_WED_GLO_CFG_RX_DMA_EN); 1042 1043 wdma_clr(dev, MTK_WDMA_GLO_CFG, 1044 MTK_WDMA_GLO_CFG_TX_DMA_EN | 1045 MTK_WDMA_GLO_CFG_RX_INFO1_PRERES | 1046 MTK_WDMA_GLO_CFG_RX_INFO2_PRERES); 1047 1048 if (mtk_wed_is_v1(dev->hw)) { 1049 regmap_write(dev->hw->mirror, dev->hw->index * 4, 0); 1050 wdma_clr(dev, MTK_WDMA_GLO_CFG, 1051 MTK_WDMA_GLO_CFG_RX_INFO3_PRERES); 1052 } else { 1053 wed_clr(dev, MTK_WED_WPDMA_GLO_CFG, 1054 MTK_WED_WPDMA_GLO_CFG_RX_DRV_R0_PKT_PROC | 1055 MTK_WED_WPDMA_GLO_CFG_RX_DRV_R0_CRX_SYNC); 1056 1057 wed_clr(dev, MTK_WED_WPDMA_RX_D_GLO_CFG, 1058 MTK_WED_WPDMA_RX_D_RX_DRV_EN); 1059 wed_clr(dev, MTK_WED_WDMA_GLO_CFG, 1060 MTK_WED_WDMA_GLO_CFG_TX_DDONE_CHK); 1061 1062 if (mtk_wed_is_v3_or_greater(dev->hw) && 1063 mtk_wed_get_rx_capa(dev)) { 1064 wdma_clr(dev, MTK_WDMA_PREF_TX_CFG, 1065 MTK_WDMA_PREF_TX_CFG_PREF_EN); 1066 wdma_clr(dev, MTK_WDMA_PREF_RX_CFG, 1067 MTK_WDMA_PREF_RX_CFG_PREF_EN); 1068 } 1069 } 1070 1071 mtk_wed_set_512_support(dev, false); 1072 } 1073 1074 static void 1075 mtk_wed_stop(struct mtk_wed_device *dev) 1076 { 1077 mtk_wed_dma_disable(dev); 1078 mtk_wed_set_ext_int(dev, false); 1079 1080 wed_w32(dev, MTK_WED_WPDMA_INT_TRIGGER, 0); 1081 wed_w32(dev, MTK_WED_WDMA_INT_TRIGGER, 0); 1082 wdma_w32(dev, MTK_WDMA_INT_MASK, 0); 1083 wdma_w32(dev, MTK_WDMA_INT_GRP2, 0); 1084 1085 if (!mtk_wed_get_rx_capa(dev)) 1086 return; 1087 1088 wed_w32(dev, MTK_WED_EXT_INT_MASK1, 0); 1089 wed_w32(dev, MTK_WED_EXT_INT_MASK2, 0); 1090 } 1091 1092 static void 1093 mtk_wed_deinit(struct mtk_wed_device *dev) 1094 { 1095 mtk_wed_stop(dev); 1096 1097 wed_clr(dev, MTK_WED_CTRL, 1098 MTK_WED_CTRL_WDMA_INT_AGENT_EN | 1099 MTK_WED_CTRL_WPDMA_INT_AGENT_EN | 1100 MTK_WED_CTRL_WED_TX_BM_EN | 1101 MTK_WED_CTRL_WED_TX_FREE_AGENT_EN); 1102 1103 if (mtk_wed_is_v1(dev->hw)) 1104 return; 1105 1106 wed_clr(dev, MTK_WED_CTRL, 1107 MTK_WED_CTRL_RX_ROUTE_QM_EN | 1108 MTK_WED_CTRL_WED_RX_BM_EN | 1109 MTK_WED_CTRL_RX_RRO_QM_EN); 1110 1111 if (mtk_wed_is_v3_or_greater(dev->hw)) { 1112 wed_clr(dev, MTK_WED_CTRL, MTK_WED_CTRL_TX_AMSDU_EN); 1113 wed_clr(dev, MTK_WED_RESET, MTK_WED_RESET_TX_AMSDU); 1114 wed_clr(dev, MTK_WED_PCIE_INT_CTRL, 1115 MTK_WED_PCIE_INT_CTRL_MSK_EN_POLA | 1116 MTK_WED_PCIE_INT_CTRL_MSK_IRQ_FILTER); 1117 } 1118 } 1119 1120 static void 1121 __mtk_wed_detach(struct mtk_wed_device *dev) 1122 { 1123 struct mtk_wed_hw *hw = dev->hw; 1124 1125 mtk_wed_deinit(dev); 1126 1127 mtk_wdma_rx_reset(dev); 1128 mtk_wed_reset(dev, MTK_WED_RESET_WED); 1129 mtk_wed_amsdu_free_buffer(dev); 1130 mtk_wed_free_tx_buffer(dev); 1131 mtk_wed_free_tx_rings(dev); 1132 1133 if (mtk_wed_get_rx_capa(dev)) { 1134 if (hw->wed_wo) 1135 mtk_wed_wo_reset(dev); 1136 mtk_wed_free_rx_rings(dev); 1137 if (hw->wed_wo) 1138 mtk_wed_wo_deinit(hw); 1139 } 1140 1141 if (dev->wlan.bus_type == MTK_WED_BUS_PCIE) { 1142 struct device_node *wlan_node; 1143 1144 wlan_node = dev->wlan.pci_dev->dev.of_node; 1145 if (of_dma_is_coherent(wlan_node) && hw->hifsys) 1146 regmap_update_bits(hw->hifsys, HIFSYS_DMA_AG_MAP, 1147 BIT(hw->index), BIT(hw->index)); 1148 } 1149 1150 if ((!hw_list[!hw->index] || !hw_list[!hw->index]->wed_dev) && 1151 hw->eth->dma_dev != hw->eth->dev) 1152 mtk_eth_set_dma_device(hw->eth, hw->eth->dev); 1153 1154 memset(dev, 0, sizeof(*dev)); 1155 module_put(THIS_MODULE); 1156 1157 hw->wed_dev = NULL; 1158 } 1159 1160 static void 1161 mtk_wed_detach(struct mtk_wed_device *dev) 1162 { 1163 mutex_lock(&hw_lock); 1164 __mtk_wed_detach(dev); 1165 mutex_unlock(&hw_lock); 1166 } 1167 1168 static void 1169 mtk_wed_bus_init(struct mtk_wed_device *dev) 1170 { 1171 switch (dev->wlan.bus_type) { 1172 case MTK_WED_BUS_PCIE: { 1173 struct device_node *np = dev->hw->eth->dev->of_node; 1174 1175 if (mtk_wed_is_v2(dev->hw)) { 1176 struct regmap *regs; 1177 1178 regs = syscon_regmap_lookup_by_phandle(np, 1179 "mediatek,wed-pcie"); 1180 if (IS_ERR(regs)) 1181 break; 1182 1183 regmap_update_bits(regs, 0, BIT(0), BIT(0)); 1184 } 1185 1186 if (dev->wlan.msi) { 1187 wed_w32(dev, MTK_WED_PCIE_CFG_INTM, 1188 dev->hw->pcie_base | 0xc08); 1189 wed_w32(dev, MTK_WED_PCIE_CFG_BASE, 1190 dev->hw->pcie_base | 0xc04); 1191 wed_w32(dev, MTK_WED_PCIE_INT_TRIGGER, BIT(8)); 1192 } else { 1193 wed_w32(dev, MTK_WED_PCIE_CFG_INTM, 1194 dev->hw->pcie_base | 0x180); 1195 wed_w32(dev, MTK_WED_PCIE_CFG_BASE, 1196 dev->hw->pcie_base | 0x184); 1197 wed_w32(dev, MTK_WED_PCIE_INT_TRIGGER, BIT(24)); 1198 } 1199 1200 wed_w32(dev, MTK_WED_PCIE_INT_CTRL, 1201 FIELD_PREP(MTK_WED_PCIE_INT_CTRL_POLL_EN, 2)); 1202 1203 /* pcie interrupt control: pola/source selection */ 1204 wed_set(dev, MTK_WED_PCIE_INT_CTRL, 1205 MTK_WED_PCIE_INT_CTRL_MSK_EN_POLA | 1206 MTK_WED_PCIE_INT_CTRL_MSK_IRQ_FILTER | 1207 FIELD_PREP(MTK_WED_PCIE_INT_CTRL_SRC_SEL, 1208 dev->hw->index)); 1209 break; 1210 } 1211 case MTK_WED_BUS_AXI: 1212 wed_set(dev, MTK_WED_WPDMA_INT_CTRL, 1213 MTK_WED_WPDMA_INT_CTRL_SIG_SRC | 1214 FIELD_PREP(MTK_WED_WPDMA_INT_CTRL_SRC_SEL, 0)); 1215 break; 1216 default: 1217 break; 1218 } 1219 } 1220 1221 static void 1222 mtk_wed_set_wpdma(struct mtk_wed_device *dev) 1223 { 1224 int i; 1225 1226 if (mtk_wed_is_v1(dev->hw)) { 1227 wed_w32(dev, MTK_WED_WPDMA_CFG_BASE, dev->wlan.wpdma_phys); 1228 return; 1229 } 1230 1231 mtk_wed_bus_init(dev); 1232 1233 wed_w32(dev, MTK_WED_WPDMA_CFG_BASE, dev->wlan.wpdma_int); 1234 wed_w32(dev, MTK_WED_WPDMA_CFG_INT_MASK, dev->wlan.wpdma_mask); 1235 wed_w32(dev, MTK_WED_WPDMA_CFG_TX, dev->wlan.wpdma_tx); 1236 wed_w32(dev, MTK_WED_WPDMA_CFG_TX_FREE, dev->wlan.wpdma_txfree); 1237 1238 if (!mtk_wed_get_rx_capa(dev)) 1239 return; 1240 1241 wed_w32(dev, MTK_WED_WPDMA_RX_GLO_CFG, dev->wlan.wpdma_rx_glo); 1242 wed_w32(dev, dev->hw->soc->regmap.wpdma_rx_ring0, dev->wlan.wpdma_rx); 1243 1244 if (!dev->wlan.hw_rro) 1245 return; 1246 1247 wed_w32(dev, MTK_WED_RRO_RX_D_CFG(0), dev->wlan.wpdma_rx_rro[0]); 1248 wed_w32(dev, MTK_WED_RRO_RX_D_CFG(1), dev->wlan.wpdma_rx_rro[1]); 1249 for (i = 0; i < MTK_WED_RX_PAGE_QUEUES; i++) 1250 wed_w32(dev, MTK_WED_RRO_MSDU_PG_RING_CFG(i), 1251 dev->wlan.wpdma_rx_pg + i * 0x10); 1252 } 1253 1254 static void 1255 mtk_wed_hw_init_early(struct mtk_wed_device *dev) 1256 { 1257 u32 set = FIELD_PREP(MTK_WED_WDMA_GLO_CFG_BT_SIZE, 2); 1258 u32 mask = MTK_WED_WDMA_GLO_CFG_BT_SIZE; 1259 1260 mtk_wed_deinit(dev); 1261 mtk_wed_reset(dev, MTK_WED_RESET_WED); 1262 mtk_wed_set_wpdma(dev); 1263 1264 if (!mtk_wed_is_v3_or_greater(dev->hw)) { 1265 mask |= MTK_WED_WDMA_GLO_CFG_DYNAMIC_DMAD_RECYCLE | 1266 MTK_WED_WDMA_GLO_CFG_RX_DIS_FSM_AUTO_IDLE; 1267 set |= MTK_WED_WDMA_GLO_CFG_DYNAMIC_SKIP_DMAD_PREP | 1268 MTK_WED_WDMA_GLO_CFG_IDLE_DMAD_SUPPLY; 1269 } 1270 wed_m32(dev, MTK_WED_WDMA_GLO_CFG, mask, set); 1271 1272 if (mtk_wed_is_v1(dev->hw)) { 1273 u32 offset = dev->hw->index ? 0x04000400 : 0; 1274 1275 wdma_set(dev, MTK_WDMA_GLO_CFG, 1276 MTK_WDMA_GLO_CFG_RX_INFO1_PRERES | 1277 MTK_WDMA_GLO_CFG_RX_INFO2_PRERES | 1278 MTK_WDMA_GLO_CFG_RX_INFO3_PRERES); 1279 1280 wed_w32(dev, MTK_WED_WDMA_OFFSET0, 0x2a042a20 + offset); 1281 wed_w32(dev, MTK_WED_WDMA_OFFSET1, 0x29002800 + offset); 1282 wed_w32(dev, MTK_WED_PCIE_CFG_BASE, 1283 MTK_PCIE_BASE(dev->hw->index)); 1284 } else { 1285 wed_w32(dev, MTK_WED_WDMA_CFG_BASE, dev->hw->wdma_phy); 1286 wed_set(dev, MTK_WED_CTRL, MTK_WED_CTRL_ETH_DMAD_FMT); 1287 wed_w32(dev, MTK_WED_WDMA_OFFSET0, 1288 FIELD_PREP(MTK_WED_WDMA_OFST0_GLO_INTS, 1289 MTK_WDMA_INT_STATUS) | 1290 FIELD_PREP(MTK_WED_WDMA_OFST0_GLO_CFG, 1291 MTK_WDMA_GLO_CFG)); 1292 1293 wed_w32(dev, MTK_WED_WDMA_OFFSET1, 1294 FIELD_PREP(MTK_WED_WDMA_OFST1_TX_CTRL, 1295 MTK_WDMA_RING_TX(0)) | 1296 FIELD_PREP(MTK_WED_WDMA_OFST1_RX_CTRL, 1297 MTK_WDMA_RING_RX(0))); 1298 } 1299 } 1300 1301 static int 1302 mtk_wed_rro_ring_alloc(struct mtk_wed_device *dev, struct mtk_wed_ring *ring, 1303 int size) 1304 { 1305 ring->desc = dma_alloc_coherent(dev->hw->dev, 1306 size * sizeof(*ring->desc), 1307 &ring->desc_phys, GFP_KERNEL); 1308 if (!ring->desc) 1309 return -ENOMEM; 1310 1311 ring->desc_size = sizeof(*ring->desc); 1312 ring->size = size; 1313 1314 return 0; 1315 } 1316 1317 #define MTK_WED_MIOD_COUNT (MTK_WED_MIOD_ENTRY_CNT * MTK_WED_MIOD_CNT) 1318 static int 1319 mtk_wed_rro_alloc(struct mtk_wed_device *dev) 1320 { 1321 struct reserved_mem *rmem; 1322 struct device_node *np; 1323 int index; 1324 1325 index = of_property_match_string(dev->hw->node, "memory-region-names", 1326 "wo-dlm"); 1327 if (index < 0) 1328 return index; 1329 1330 np = of_parse_phandle(dev->hw->node, "memory-region", index); 1331 if (!np) 1332 return -ENODEV; 1333 1334 rmem = of_reserved_mem_lookup(np); 1335 of_node_put(np); 1336 1337 if (!rmem) 1338 return -ENODEV; 1339 1340 dev->rro.miod_phys = rmem->base; 1341 dev->rro.fdbk_phys = MTK_WED_MIOD_COUNT + dev->rro.miod_phys; 1342 1343 return mtk_wed_rro_ring_alloc(dev, &dev->rro.ring, 1344 MTK_WED_RRO_QUE_CNT); 1345 } 1346 1347 static int 1348 mtk_wed_rro_cfg(struct mtk_wed_device *dev) 1349 { 1350 struct mtk_wed_wo *wo = dev->hw->wed_wo; 1351 struct { 1352 struct { 1353 __le32 base; 1354 __le32 cnt; 1355 __le32 unit; 1356 } ring[2]; 1357 __le32 wed; 1358 u8 version; 1359 } req = { 1360 .ring[0] = { 1361 .base = cpu_to_le32(MTK_WED_WOCPU_VIEW_MIOD_BASE), 1362 .cnt = cpu_to_le32(MTK_WED_MIOD_CNT), 1363 .unit = cpu_to_le32(MTK_WED_MIOD_ENTRY_CNT), 1364 }, 1365 .ring[1] = { 1366 .base = cpu_to_le32(MTK_WED_WOCPU_VIEW_MIOD_BASE + 1367 MTK_WED_MIOD_COUNT), 1368 .cnt = cpu_to_le32(MTK_WED_FB_CMD_CNT), 1369 .unit = cpu_to_le32(4), 1370 }, 1371 }; 1372 1373 return mtk_wed_mcu_send_msg(wo, MTK_WED_MODULE_ID_WO, 1374 MTK_WED_WO_CMD_WED_CFG, 1375 &req, sizeof(req), true); 1376 } 1377 1378 static void 1379 mtk_wed_rro_hw_init(struct mtk_wed_device *dev) 1380 { 1381 wed_w32(dev, MTK_WED_RROQM_MIOD_CFG, 1382 FIELD_PREP(MTK_WED_RROQM_MIOD_MID_DW, 0x70 >> 2) | 1383 FIELD_PREP(MTK_WED_RROQM_MIOD_MOD_DW, 0x10 >> 2) | 1384 FIELD_PREP(MTK_WED_RROQM_MIOD_ENTRY_DW, 1385 MTK_WED_MIOD_ENTRY_CNT >> 2)); 1386 1387 wed_w32(dev, MTK_WED_RROQM_MIOD_CTRL0, dev->rro.miod_phys); 1388 wed_w32(dev, MTK_WED_RROQM_MIOD_CTRL1, 1389 FIELD_PREP(MTK_WED_RROQM_MIOD_CNT, MTK_WED_MIOD_CNT)); 1390 wed_w32(dev, MTK_WED_RROQM_FDBK_CTRL0, dev->rro.fdbk_phys); 1391 wed_w32(dev, MTK_WED_RROQM_FDBK_CTRL1, 1392 FIELD_PREP(MTK_WED_RROQM_FDBK_CNT, MTK_WED_FB_CMD_CNT)); 1393 wed_w32(dev, MTK_WED_RROQM_FDBK_CTRL2, 0); 1394 wed_w32(dev, MTK_WED_RROQ_BASE_L, dev->rro.ring.desc_phys); 1395 1396 wed_set(dev, MTK_WED_RROQM_RST_IDX, 1397 MTK_WED_RROQM_RST_IDX_MIOD | 1398 MTK_WED_RROQM_RST_IDX_FDBK); 1399 1400 wed_w32(dev, MTK_WED_RROQM_RST_IDX, 0); 1401 wed_w32(dev, MTK_WED_RROQM_MIOD_CTRL2, MTK_WED_MIOD_CNT - 1); 1402 wed_set(dev, MTK_WED_CTRL, MTK_WED_CTRL_RX_RRO_QM_EN); 1403 } 1404 1405 static void 1406 mtk_wed_route_qm_hw_init(struct mtk_wed_device *dev) 1407 { 1408 wed_w32(dev, MTK_WED_RESET, MTK_WED_RESET_RX_ROUTE_QM); 1409 1410 for (;;) { 1411 usleep_range(100, 200); 1412 if (!(wed_r32(dev, MTK_WED_RESET) & MTK_WED_RESET_RX_ROUTE_QM)) 1413 break; 1414 } 1415 1416 /* configure RX_ROUTE_QM */ 1417 if (mtk_wed_is_v2(dev->hw)) { 1418 wed_clr(dev, MTK_WED_RTQM_GLO_CFG, MTK_WED_RTQM_Q_RST); 1419 wed_clr(dev, MTK_WED_RTQM_GLO_CFG, MTK_WED_RTQM_TXDMAD_FPORT); 1420 wed_set(dev, MTK_WED_RTQM_GLO_CFG, 1421 FIELD_PREP(MTK_WED_RTQM_TXDMAD_FPORT, 1422 0x3 + dev->hw->index)); 1423 wed_clr(dev, MTK_WED_RTQM_GLO_CFG, MTK_WED_RTQM_Q_RST); 1424 } else { 1425 wed_set(dev, MTK_WED_RTQM_ENQ_CFG0, 1426 FIELD_PREP(MTK_WED_RTQM_ENQ_CFG_TXDMAD_FPORT, 1427 0x3 + dev->hw->index)); 1428 } 1429 /* enable RX_ROUTE_QM */ 1430 wed_set(dev, MTK_WED_CTRL, MTK_WED_CTRL_RX_ROUTE_QM_EN); 1431 } 1432 1433 static void 1434 mtk_wed_hw_init(struct mtk_wed_device *dev) 1435 { 1436 if (dev->init_done) 1437 return; 1438 1439 dev->init_done = true; 1440 mtk_wed_set_ext_int(dev, false); 1441 1442 wed_w32(dev, MTK_WED_TX_BM_BASE, dev->tx_buf_ring.desc_phys); 1443 wed_w32(dev, MTK_WED_TX_BM_BUF_LEN, MTK_WED_PKT_SIZE); 1444 1445 if (mtk_wed_is_v1(dev->hw)) { 1446 wed_w32(dev, MTK_WED_TX_BM_CTRL, 1447 MTK_WED_TX_BM_CTRL_PAUSE | 1448 FIELD_PREP(MTK_WED_TX_BM_CTRL_VLD_GRP_NUM, 1449 dev->tx_buf_ring.size / 128) | 1450 FIELD_PREP(MTK_WED_TX_BM_CTRL_RSV_GRP_NUM, 1451 MTK_WED_TX_RING_SIZE / 256)); 1452 wed_w32(dev, MTK_WED_TX_BM_DYN_THR, 1453 FIELD_PREP(MTK_WED_TX_BM_DYN_THR_LO, 1) | 1454 MTK_WED_TX_BM_DYN_THR_HI); 1455 } else if (mtk_wed_is_v2(dev->hw)) { 1456 wed_w32(dev, MTK_WED_TX_BM_CTRL, 1457 MTK_WED_TX_BM_CTRL_PAUSE | 1458 FIELD_PREP(MTK_WED_TX_BM_CTRL_VLD_GRP_NUM, 1459 dev->tx_buf_ring.size / 128) | 1460 FIELD_PREP(MTK_WED_TX_BM_CTRL_RSV_GRP_NUM, 1461 MTK_WED_TX_RING_SIZE / 256)); 1462 wed_w32(dev, MTK_WED_TX_TKID_DYN_THR, 1463 FIELD_PREP(MTK_WED_TX_TKID_DYN_THR_LO, 0) | 1464 MTK_WED_TX_TKID_DYN_THR_HI); 1465 wed_w32(dev, MTK_WED_TX_BM_DYN_THR, 1466 FIELD_PREP(MTK_WED_TX_BM_DYN_THR_LO_V2, 0) | 1467 MTK_WED_TX_BM_DYN_THR_HI_V2); 1468 wed_w32(dev, MTK_WED_TX_TKID_CTRL, 1469 MTK_WED_TX_TKID_CTRL_PAUSE | 1470 FIELD_PREP(MTK_WED_TX_TKID_CTRL_VLD_GRP_NUM, 1471 dev->tx_buf_ring.size / 128) | 1472 FIELD_PREP(MTK_WED_TX_TKID_CTRL_RSV_GRP_NUM, 1473 dev->tx_buf_ring.size / 128)); 1474 } 1475 1476 wed_w32(dev, dev->hw->soc->regmap.tx_bm_tkid, 1477 FIELD_PREP(MTK_WED_TX_BM_TKID_START, dev->wlan.token_start) | 1478 FIELD_PREP(MTK_WED_TX_BM_TKID_END, 1479 dev->wlan.token_start + dev->wlan.nbuf - 1)); 1480 1481 mtk_wed_reset(dev, MTK_WED_RESET_TX_BM); 1482 1483 if (mtk_wed_is_v3_or_greater(dev->hw)) { 1484 /* switch to new bm architecture */ 1485 wed_clr(dev, MTK_WED_TX_BM_CTRL, 1486 MTK_WED_TX_BM_CTRL_LEGACY_EN); 1487 1488 wed_w32(dev, MTK_WED_TX_TKID_CTRL, 1489 MTK_WED_TX_TKID_CTRL_PAUSE | 1490 FIELD_PREP(MTK_WED_TX_TKID_CTRL_VLD_GRP_NUM_V3, 1491 dev->wlan.nbuf / 128) | 1492 FIELD_PREP(MTK_WED_TX_TKID_CTRL_RSV_GRP_NUM_V3, 1493 dev->wlan.nbuf / 128)); 1494 /* return SKBID + SDP back to bm */ 1495 wed_set(dev, MTK_WED_TX_TKID_CTRL, 1496 MTK_WED_TX_TKID_CTRL_FREE_FORMAT); 1497 1498 wed_w32(dev, MTK_WED_TX_BM_INIT_PTR, 1499 MTK_WED_TX_BM_PKT_CNT | 1500 MTK_WED_TX_BM_INIT_SW_TAIL_IDX); 1501 } 1502 1503 if (mtk_wed_is_v1(dev->hw)) { 1504 wed_set(dev, MTK_WED_CTRL, 1505 MTK_WED_CTRL_WED_TX_BM_EN | 1506 MTK_WED_CTRL_WED_TX_FREE_AGENT_EN); 1507 } else if (mtk_wed_get_rx_capa(dev)) { 1508 /* rx hw init */ 1509 wed_w32(dev, MTK_WED_WPDMA_RX_D_RST_IDX, 1510 MTK_WED_WPDMA_RX_D_RST_CRX_IDX | 1511 MTK_WED_WPDMA_RX_D_RST_DRV_IDX); 1512 wed_w32(dev, MTK_WED_WPDMA_RX_D_RST_IDX, 0); 1513 1514 /* reset prefetch index of ring */ 1515 wed_set(dev, MTK_WED_WPDMA_RX_D_PREF_RX0_SIDX, 1516 MTK_WED_WPDMA_RX_D_PREF_SIDX_IDX_CLR); 1517 wed_clr(dev, MTK_WED_WPDMA_RX_D_PREF_RX0_SIDX, 1518 MTK_WED_WPDMA_RX_D_PREF_SIDX_IDX_CLR); 1519 1520 wed_set(dev, MTK_WED_WPDMA_RX_D_PREF_RX1_SIDX, 1521 MTK_WED_WPDMA_RX_D_PREF_SIDX_IDX_CLR); 1522 wed_clr(dev, MTK_WED_WPDMA_RX_D_PREF_RX1_SIDX, 1523 MTK_WED_WPDMA_RX_D_PREF_SIDX_IDX_CLR); 1524 1525 /* reset prefetch FIFO of ring */ 1526 wed_set(dev, MTK_WED_WPDMA_RX_D_PREF_FIFO_CFG, 1527 MTK_WED_WPDMA_RX_D_PREF_FIFO_CFG_R0_CLR | 1528 MTK_WED_WPDMA_RX_D_PREF_FIFO_CFG_R1_CLR); 1529 wed_w32(dev, MTK_WED_WPDMA_RX_D_PREF_FIFO_CFG, 0); 1530 1531 mtk_wed_rx_buffer_hw_init(dev); 1532 mtk_wed_rro_hw_init(dev); 1533 mtk_wed_route_qm_hw_init(dev); 1534 } 1535 1536 wed_clr(dev, MTK_WED_TX_BM_CTRL, MTK_WED_TX_BM_CTRL_PAUSE); 1537 if (!mtk_wed_is_v1(dev->hw)) 1538 wed_clr(dev, MTK_WED_TX_TKID_CTRL, MTK_WED_TX_TKID_CTRL_PAUSE); 1539 } 1540 1541 static void 1542 mtk_wed_ring_reset(struct mtk_wed_ring *ring, int size, bool tx) 1543 { 1544 void *head = (void *)ring->desc; 1545 int i; 1546 1547 for (i = 0; i < size; i++) { 1548 struct mtk_wdma_desc *desc; 1549 1550 desc = (struct mtk_wdma_desc *)(head + i * ring->desc_size); 1551 desc->buf0 = 0; 1552 if (tx) 1553 desc->ctrl = cpu_to_le32(MTK_WDMA_DESC_CTRL_DMA_DONE); 1554 else 1555 desc->ctrl = cpu_to_le32(MTK_WFDMA_DESC_CTRL_TO_HOST); 1556 desc->buf1 = 0; 1557 desc->info = 0; 1558 } 1559 } 1560 1561 static int 1562 mtk_wed_rx_reset(struct mtk_wed_device *dev) 1563 { 1564 struct mtk_wed_wo *wo = dev->hw->wed_wo; 1565 u8 val = MTK_WED_WO_STATE_SER_RESET; 1566 int i, ret; 1567 1568 ret = mtk_wed_mcu_send_msg(wo, MTK_WED_MODULE_ID_WO, 1569 MTK_WED_WO_CMD_CHANGE_STATE, &val, 1570 sizeof(val), true); 1571 if (ret) 1572 return ret; 1573 1574 if (dev->wlan.hw_rro) { 1575 wed_clr(dev, MTK_WED_CTRL, MTK_WED_CTRL_WED_RX_IND_CMD_EN); 1576 mtk_wed_poll_busy(dev, MTK_WED_RRO_RX_HW_STS, 1577 MTK_WED_RX_IND_CMD_BUSY); 1578 mtk_wed_reset(dev, MTK_WED_RESET_RRO_RX_TO_PG); 1579 } 1580 1581 wed_clr(dev, MTK_WED_WPDMA_RX_D_GLO_CFG, MTK_WED_WPDMA_RX_D_RX_DRV_EN); 1582 ret = mtk_wed_poll_busy(dev, MTK_WED_WPDMA_RX_D_GLO_CFG, 1583 MTK_WED_WPDMA_RX_D_RX_DRV_BUSY); 1584 if (!ret && mtk_wed_is_v3_or_greater(dev->hw)) 1585 ret = mtk_wed_poll_busy(dev, MTK_WED_WPDMA_RX_D_PREF_CFG, 1586 MTK_WED_WPDMA_RX_D_PREF_BUSY); 1587 if (ret) { 1588 mtk_wed_reset(dev, MTK_WED_RESET_WPDMA_INT_AGENT); 1589 mtk_wed_reset(dev, MTK_WED_RESET_WPDMA_RX_D_DRV); 1590 } else { 1591 if (mtk_wed_is_v3_or_greater(dev->hw)) { 1592 /* 1.a. disable prefetch HW */ 1593 wed_clr(dev, MTK_WED_WPDMA_RX_D_PREF_CFG, 1594 MTK_WED_WPDMA_RX_D_PREF_EN); 1595 mtk_wed_poll_busy(dev, MTK_WED_WPDMA_RX_D_PREF_CFG, 1596 MTK_WED_WPDMA_RX_D_PREF_BUSY); 1597 wed_w32(dev, MTK_WED_WPDMA_RX_D_RST_IDX, 1598 MTK_WED_WPDMA_RX_D_RST_DRV_IDX_ALL); 1599 } 1600 1601 wed_w32(dev, MTK_WED_WPDMA_RX_D_RST_IDX, 1602 MTK_WED_WPDMA_RX_D_RST_CRX_IDX | 1603 MTK_WED_WPDMA_RX_D_RST_DRV_IDX); 1604 1605 wed_set(dev, MTK_WED_WPDMA_RX_D_GLO_CFG, 1606 MTK_WED_WPDMA_RX_D_RST_INIT_COMPLETE | 1607 MTK_WED_WPDMA_RX_D_FSM_RETURN_IDLE); 1608 wed_clr(dev, MTK_WED_WPDMA_RX_D_GLO_CFG, 1609 MTK_WED_WPDMA_RX_D_RST_INIT_COMPLETE | 1610 MTK_WED_WPDMA_RX_D_FSM_RETURN_IDLE); 1611 1612 wed_w32(dev, MTK_WED_WPDMA_RX_D_RST_IDX, 0); 1613 } 1614 1615 /* reset rro qm */ 1616 wed_clr(dev, MTK_WED_CTRL, MTK_WED_CTRL_RX_RRO_QM_EN); 1617 ret = mtk_wed_poll_busy(dev, MTK_WED_CTRL, 1618 MTK_WED_CTRL_RX_RRO_QM_BUSY); 1619 if (ret) { 1620 mtk_wed_reset(dev, MTK_WED_RESET_RX_RRO_QM); 1621 } else { 1622 wed_set(dev, MTK_WED_RROQM_RST_IDX, 1623 MTK_WED_RROQM_RST_IDX_MIOD | 1624 MTK_WED_RROQM_RST_IDX_FDBK); 1625 wed_w32(dev, MTK_WED_RROQM_RST_IDX, 0); 1626 } 1627 1628 if (dev->wlan.hw_rro) { 1629 /* disable rro msdu page drv */ 1630 wed_clr(dev, MTK_WED_RRO_MSDU_PG_RING2_CFG, 1631 MTK_WED_RRO_MSDU_PG_DRV_EN); 1632 1633 /* disable rro data drv */ 1634 wed_clr(dev, MTK_WED_RRO_RX_D_CFG(2), MTK_WED_RRO_RX_D_DRV_EN); 1635 1636 /* rro msdu page drv reset */ 1637 wed_w32(dev, MTK_WED_RRO_MSDU_PG_RING2_CFG, 1638 MTK_WED_RRO_MSDU_PG_DRV_CLR); 1639 mtk_wed_poll_busy(dev, MTK_WED_RRO_MSDU_PG_RING2_CFG, 1640 MTK_WED_RRO_MSDU_PG_DRV_CLR); 1641 1642 /* rro data drv reset */ 1643 wed_w32(dev, MTK_WED_RRO_RX_D_CFG(2), 1644 MTK_WED_RRO_RX_D_DRV_CLR); 1645 mtk_wed_poll_busy(dev, MTK_WED_RRO_RX_D_CFG(2), 1646 MTK_WED_RRO_RX_D_DRV_CLR); 1647 } 1648 1649 /* reset route qm */ 1650 wed_clr(dev, MTK_WED_CTRL, MTK_WED_CTRL_RX_ROUTE_QM_EN); 1651 ret = mtk_wed_poll_busy(dev, MTK_WED_CTRL, 1652 MTK_WED_CTRL_RX_ROUTE_QM_BUSY); 1653 if (ret) { 1654 mtk_wed_reset(dev, MTK_WED_RESET_RX_ROUTE_QM); 1655 } else if (mtk_wed_is_v3_or_greater(dev->hw)) { 1656 wed_set(dev, MTK_WED_RTQM_RST, BIT(0)); 1657 wed_clr(dev, MTK_WED_RTQM_RST, BIT(0)); 1658 mtk_wed_reset(dev, MTK_WED_RESET_RX_ROUTE_QM); 1659 } else { 1660 wed_set(dev, MTK_WED_RTQM_GLO_CFG, MTK_WED_RTQM_Q_RST); 1661 } 1662 1663 /* reset tx wdma */ 1664 mtk_wdma_tx_reset(dev); 1665 1666 /* reset tx wdma drv */ 1667 wed_clr(dev, MTK_WED_WDMA_GLO_CFG, MTK_WED_WDMA_GLO_CFG_TX_DRV_EN); 1668 if (mtk_wed_is_v3_or_greater(dev->hw)) 1669 mtk_wed_poll_busy(dev, MTK_WED_WPDMA_STATUS, 1670 MTK_WED_WPDMA_STATUS_TX_DRV); 1671 else 1672 mtk_wed_poll_busy(dev, MTK_WED_CTRL, 1673 MTK_WED_CTRL_WDMA_INT_AGENT_BUSY); 1674 mtk_wed_reset(dev, MTK_WED_RESET_WDMA_TX_DRV); 1675 1676 /* reset wed rx dma */ 1677 ret = mtk_wed_poll_busy(dev, MTK_WED_GLO_CFG, 1678 MTK_WED_GLO_CFG_RX_DMA_BUSY); 1679 wed_clr(dev, MTK_WED_GLO_CFG, MTK_WED_GLO_CFG_RX_DMA_EN); 1680 if (ret) { 1681 mtk_wed_reset(dev, MTK_WED_RESET_WED_RX_DMA); 1682 } else { 1683 wed_set(dev, MTK_WED_RESET_IDX, 1684 dev->hw->soc->regmap.reset_idx_rx_mask); 1685 wed_w32(dev, MTK_WED_RESET_IDX, 0); 1686 } 1687 1688 /* reset rx bm */ 1689 wed_clr(dev, MTK_WED_CTRL, MTK_WED_CTRL_WED_RX_BM_EN); 1690 mtk_wed_poll_busy(dev, MTK_WED_CTRL, 1691 MTK_WED_CTRL_WED_RX_BM_BUSY); 1692 mtk_wed_reset(dev, MTK_WED_RESET_RX_BM); 1693 1694 if (dev->wlan.hw_rro) { 1695 wed_clr(dev, MTK_WED_CTRL, MTK_WED_CTRL_WED_RX_PG_BM_EN); 1696 mtk_wed_poll_busy(dev, MTK_WED_CTRL, 1697 MTK_WED_CTRL_WED_RX_PG_BM_BUSY); 1698 wed_set(dev, MTK_WED_RESET, MTK_WED_RESET_RX_PG_BM); 1699 wed_clr(dev, MTK_WED_RESET, MTK_WED_RESET_RX_PG_BM); 1700 } 1701 1702 /* wo change to enable state */ 1703 val = MTK_WED_WO_STATE_ENABLE; 1704 ret = mtk_wed_mcu_send_msg(wo, MTK_WED_MODULE_ID_WO, 1705 MTK_WED_WO_CMD_CHANGE_STATE, &val, 1706 sizeof(val), true); 1707 if (ret) 1708 return ret; 1709 1710 /* wed_rx_ring_reset */ 1711 for (i = 0; i < ARRAY_SIZE(dev->rx_ring); i++) { 1712 if (!dev->rx_ring[i].desc) 1713 continue; 1714 1715 mtk_wed_ring_reset(&dev->rx_ring[i], MTK_WED_RX_RING_SIZE, 1716 false); 1717 } 1718 mtk_wed_free_rx_buffer(dev); 1719 mtk_wed_hwrro_free_buffer(dev); 1720 1721 return 0; 1722 } 1723 1724 static void 1725 mtk_wed_reset_dma(struct mtk_wed_device *dev) 1726 { 1727 bool busy = false; 1728 u32 val; 1729 int i; 1730 1731 for (i = 0; i < ARRAY_SIZE(dev->tx_ring); i++) { 1732 if (!dev->tx_ring[i].desc) 1733 continue; 1734 1735 mtk_wed_ring_reset(&dev->tx_ring[i], MTK_WED_TX_RING_SIZE, 1736 true); 1737 } 1738 1739 /* 1. reset WED tx DMA */ 1740 wed_clr(dev, MTK_WED_GLO_CFG, MTK_WED_GLO_CFG_TX_DMA_EN); 1741 busy = mtk_wed_poll_busy(dev, MTK_WED_GLO_CFG, 1742 MTK_WED_GLO_CFG_TX_DMA_BUSY); 1743 if (busy) { 1744 mtk_wed_reset(dev, MTK_WED_RESET_WED_TX_DMA); 1745 } else { 1746 wed_w32(dev, MTK_WED_RESET_IDX, 1747 dev->hw->soc->regmap.reset_idx_tx_mask); 1748 wed_w32(dev, MTK_WED_RESET_IDX, 0); 1749 } 1750 1751 /* 2. reset WDMA rx DMA */ 1752 busy = !!mtk_wdma_rx_reset(dev); 1753 if (mtk_wed_is_v3_or_greater(dev->hw)) { 1754 val = MTK_WED_WDMA_GLO_CFG_RX_DIS_FSM_AUTO_IDLE | 1755 wed_r32(dev, MTK_WED_WDMA_GLO_CFG); 1756 val &= ~MTK_WED_WDMA_GLO_CFG_RX_DRV_EN; 1757 wed_w32(dev, MTK_WED_WDMA_GLO_CFG, val); 1758 } else { 1759 wed_clr(dev, MTK_WED_WDMA_GLO_CFG, 1760 MTK_WED_WDMA_GLO_CFG_RX_DRV_EN); 1761 } 1762 1763 if (!busy) 1764 busy = mtk_wed_poll_busy(dev, MTK_WED_WDMA_GLO_CFG, 1765 MTK_WED_WDMA_GLO_CFG_RX_DRV_BUSY); 1766 if (!busy && mtk_wed_is_v3_or_greater(dev->hw)) 1767 busy = mtk_wed_poll_busy(dev, MTK_WED_WDMA_RX_PREF_CFG, 1768 MTK_WED_WDMA_RX_PREF_BUSY); 1769 1770 if (busy) { 1771 mtk_wed_reset(dev, MTK_WED_RESET_WDMA_INT_AGENT); 1772 mtk_wed_reset(dev, MTK_WED_RESET_WDMA_RX_DRV); 1773 } else { 1774 if (mtk_wed_is_v3_or_greater(dev->hw)) { 1775 /* 1.a. disable prefetch HW */ 1776 wed_clr(dev, MTK_WED_WDMA_RX_PREF_CFG, 1777 MTK_WED_WDMA_RX_PREF_EN); 1778 mtk_wed_poll_busy(dev, MTK_WED_WDMA_RX_PREF_CFG, 1779 MTK_WED_WDMA_RX_PREF_BUSY); 1780 wed_clr(dev, MTK_WED_WDMA_RX_PREF_CFG, 1781 MTK_WED_WDMA_RX_PREF_DDONE2_EN); 1782 1783 /* 2. Reset dma index */ 1784 wed_w32(dev, MTK_WED_WDMA_RESET_IDX, 1785 MTK_WED_WDMA_RESET_IDX_RX_ALL); 1786 } 1787 1788 wed_w32(dev, MTK_WED_WDMA_RESET_IDX, 1789 MTK_WED_WDMA_RESET_IDX_RX | MTK_WED_WDMA_RESET_IDX_DRV); 1790 wed_w32(dev, MTK_WED_WDMA_RESET_IDX, 0); 1791 1792 wed_set(dev, MTK_WED_WDMA_GLO_CFG, 1793 MTK_WED_WDMA_GLO_CFG_RST_INIT_COMPLETE); 1794 1795 wed_clr(dev, MTK_WED_WDMA_GLO_CFG, 1796 MTK_WED_WDMA_GLO_CFG_RST_INIT_COMPLETE); 1797 } 1798 1799 /* 3. reset WED WPDMA tx */ 1800 wed_clr(dev, MTK_WED_CTRL, MTK_WED_CTRL_WED_TX_FREE_AGENT_EN); 1801 1802 for (i = 0; i < 100; i++) { 1803 if (mtk_wed_is_v1(dev->hw)) 1804 val = FIELD_GET(MTK_WED_TX_BM_INTF_TKFIFO_FDEP, 1805 wed_r32(dev, MTK_WED_TX_BM_INTF)); 1806 else 1807 val = FIELD_GET(MTK_WED_TX_TKID_INTF_TKFIFO_FDEP, 1808 wed_r32(dev, MTK_WED_TX_TKID_INTF)); 1809 if (val == 0x40) 1810 break; 1811 } 1812 1813 mtk_wed_reset(dev, MTK_WED_RESET_TX_FREE_AGENT); 1814 wed_clr(dev, MTK_WED_CTRL, MTK_WED_CTRL_WED_TX_BM_EN); 1815 mtk_wed_reset(dev, MTK_WED_RESET_TX_BM); 1816 1817 /* 4. reset WED WPDMA tx */ 1818 busy = mtk_wed_poll_busy(dev, MTK_WED_WPDMA_GLO_CFG, 1819 MTK_WED_WPDMA_GLO_CFG_TX_DRV_BUSY); 1820 wed_clr(dev, MTK_WED_WPDMA_GLO_CFG, 1821 MTK_WED_WPDMA_GLO_CFG_TX_DRV_EN | 1822 MTK_WED_WPDMA_GLO_CFG_RX_DRV_EN); 1823 if (!busy) 1824 busy = mtk_wed_poll_busy(dev, MTK_WED_WPDMA_GLO_CFG, 1825 MTK_WED_WPDMA_GLO_CFG_RX_DRV_BUSY); 1826 1827 if (busy) { 1828 mtk_wed_reset(dev, MTK_WED_RESET_WPDMA_INT_AGENT); 1829 mtk_wed_reset(dev, MTK_WED_RESET_WPDMA_TX_DRV); 1830 mtk_wed_reset(dev, MTK_WED_RESET_WPDMA_RX_DRV); 1831 if (mtk_wed_is_v3_or_greater(dev->hw)) 1832 wed_w32(dev, MTK_WED_RX1_CTRL2, 0); 1833 } else { 1834 wed_w32(dev, MTK_WED_WPDMA_RESET_IDX, 1835 MTK_WED_WPDMA_RESET_IDX_TX | 1836 MTK_WED_WPDMA_RESET_IDX_RX); 1837 wed_w32(dev, MTK_WED_WPDMA_RESET_IDX, 0); 1838 } 1839 1840 dev->init_done = false; 1841 if (mtk_wed_is_v1(dev->hw)) 1842 return; 1843 1844 if (!busy) { 1845 wed_w32(dev, MTK_WED_RESET_IDX, MTK_WED_RESET_WPDMA_IDX_RX); 1846 wed_w32(dev, MTK_WED_RESET_IDX, 0); 1847 } 1848 1849 if (mtk_wed_is_v3_or_greater(dev->hw)) { 1850 /* reset amsdu engine */ 1851 wed_clr(dev, MTK_WED_CTRL, MTK_WED_CTRL_TX_AMSDU_EN); 1852 mtk_wed_reset(dev, MTK_WED_RESET_TX_AMSDU); 1853 } 1854 1855 if (mtk_wed_get_rx_capa(dev)) 1856 mtk_wed_rx_reset(dev); 1857 } 1858 1859 static int 1860 mtk_wed_ring_alloc(struct mtk_wed_device *dev, struct mtk_wed_ring *ring, 1861 int size, u32 desc_size, bool tx) 1862 { 1863 ring->desc = dma_alloc_coherent(dev->hw->dev, size * desc_size, 1864 &ring->desc_phys, GFP_KERNEL); 1865 if (!ring->desc) 1866 return -ENOMEM; 1867 1868 ring->desc_size = desc_size; 1869 ring->size = size; 1870 mtk_wed_ring_reset(ring, size, tx); 1871 1872 return 0; 1873 } 1874 1875 static int 1876 mtk_wed_wdma_rx_ring_setup(struct mtk_wed_device *dev, int idx, int size, 1877 bool reset) 1878 { 1879 struct mtk_wed_ring *wdma; 1880 1881 if (idx >= ARRAY_SIZE(dev->rx_wdma)) 1882 return -EINVAL; 1883 1884 wdma = &dev->rx_wdma[idx]; 1885 if (!reset && mtk_wed_ring_alloc(dev, wdma, MTK_WED_WDMA_RING_SIZE, 1886 dev->hw->soc->wdma_desc_size, true)) 1887 return -ENOMEM; 1888 1889 wdma_w32(dev, MTK_WDMA_RING_RX(idx) + MTK_WED_RING_OFS_BASE, 1890 wdma->desc_phys); 1891 wdma_w32(dev, MTK_WDMA_RING_RX(idx) + MTK_WED_RING_OFS_COUNT, 1892 size); 1893 wdma_w32(dev, MTK_WDMA_RING_RX(idx) + MTK_WED_RING_OFS_CPU_IDX, 0); 1894 1895 wed_w32(dev, MTK_WED_WDMA_RING_RX(idx) + MTK_WED_RING_OFS_BASE, 1896 wdma->desc_phys); 1897 wed_w32(dev, MTK_WED_WDMA_RING_RX(idx) + MTK_WED_RING_OFS_COUNT, 1898 size); 1899 1900 return 0; 1901 } 1902 1903 static int 1904 mtk_wed_wdma_tx_ring_setup(struct mtk_wed_device *dev, int idx, int size, 1905 bool reset) 1906 { 1907 struct mtk_wed_ring *wdma; 1908 1909 if (idx >= ARRAY_SIZE(dev->tx_wdma)) 1910 return -EINVAL; 1911 1912 wdma = &dev->tx_wdma[idx]; 1913 if (!reset && mtk_wed_ring_alloc(dev, wdma, MTK_WED_WDMA_RING_SIZE, 1914 dev->hw->soc->wdma_desc_size, true)) 1915 return -ENOMEM; 1916 1917 if (mtk_wed_is_v3_or_greater(dev->hw)) { 1918 struct mtk_wdma_desc *desc = wdma->desc; 1919 int i; 1920 1921 for (i = 0; i < MTK_WED_WDMA_RING_SIZE; i++) { 1922 desc->buf0 = 0; 1923 desc->ctrl = cpu_to_le32(MTK_WDMA_DESC_CTRL_DMA_DONE); 1924 desc->buf1 = 0; 1925 desc->info = cpu_to_le32(MTK_WDMA_TXD0_DESC_INFO_DMA_DONE); 1926 desc++; 1927 desc->buf0 = 0; 1928 desc->ctrl = cpu_to_le32(MTK_WDMA_DESC_CTRL_DMA_DONE); 1929 desc->buf1 = 0; 1930 desc->info = cpu_to_le32(MTK_WDMA_TXD1_DESC_INFO_DMA_DONE); 1931 desc++; 1932 } 1933 } 1934 1935 wdma_w32(dev, MTK_WDMA_RING_TX(idx) + MTK_WED_RING_OFS_BASE, 1936 wdma->desc_phys); 1937 wdma_w32(dev, MTK_WDMA_RING_TX(idx) + MTK_WED_RING_OFS_COUNT, 1938 size); 1939 wdma_w32(dev, MTK_WDMA_RING_TX(idx) + MTK_WED_RING_OFS_CPU_IDX, 0); 1940 wdma_w32(dev, MTK_WDMA_RING_TX(idx) + MTK_WED_RING_OFS_DMA_IDX, 0); 1941 1942 if (reset) 1943 mtk_wed_ring_reset(wdma, MTK_WED_WDMA_RING_SIZE, true); 1944 1945 if (!idx) { 1946 wed_w32(dev, MTK_WED_WDMA_RING_TX + MTK_WED_RING_OFS_BASE, 1947 wdma->desc_phys); 1948 wed_w32(dev, MTK_WED_WDMA_RING_TX + MTK_WED_RING_OFS_COUNT, 1949 size); 1950 wed_w32(dev, MTK_WED_WDMA_RING_TX + MTK_WED_RING_OFS_CPU_IDX, 1951 0); 1952 wed_w32(dev, MTK_WED_WDMA_RING_TX + MTK_WED_RING_OFS_DMA_IDX, 1953 0); 1954 } 1955 1956 return 0; 1957 } 1958 1959 static void 1960 mtk_wed_ppe_check(struct mtk_wed_device *dev, struct sk_buff *skb, 1961 u32 reason, u32 hash) 1962 { 1963 struct mtk_eth *eth = dev->hw->eth; 1964 struct ethhdr *eh; 1965 1966 if (!skb) 1967 return; 1968 1969 if (reason != MTK_PPE_CPU_REASON_HIT_UNBIND_RATE_REACHED) 1970 return; 1971 1972 skb_set_mac_header(skb, 0); 1973 eh = eth_hdr(skb); 1974 skb->protocol = eh->h_proto; 1975 mtk_ppe_check_skb(eth->ppe[dev->hw->index], skb, hash); 1976 } 1977 1978 static void 1979 mtk_wed_configure_irq(struct mtk_wed_device *dev, u32 irq_mask) 1980 { 1981 u32 wdma_mask = FIELD_PREP(MTK_WDMA_INT_MASK_RX_DONE, GENMASK(1, 0)); 1982 1983 /* wed control cr set */ 1984 wed_set(dev, MTK_WED_CTRL, 1985 MTK_WED_CTRL_WDMA_INT_AGENT_EN | 1986 MTK_WED_CTRL_WPDMA_INT_AGENT_EN | 1987 MTK_WED_CTRL_WED_TX_BM_EN | 1988 MTK_WED_CTRL_WED_TX_FREE_AGENT_EN); 1989 1990 if (mtk_wed_is_v1(dev->hw)) { 1991 wed_w32(dev, MTK_WED_PCIE_INT_TRIGGER, 1992 MTK_WED_PCIE_INT_TRIGGER_STATUS); 1993 1994 wed_w32(dev, MTK_WED_WPDMA_INT_TRIGGER, 1995 MTK_WED_WPDMA_INT_TRIGGER_RX_DONE | 1996 MTK_WED_WPDMA_INT_TRIGGER_TX_DONE); 1997 1998 wed_clr(dev, MTK_WED_WDMA_INT_CTRL, wdma_mask); 1999 } else { 2000 if (mtk_wed_is_v3_or_greater(dev->hw)) 2001 wed_set(dev, MTK_WED_CTRL, MTK_WED_CTRL_TX_TKID_ALI_EN); 2002 2003 /* initail tx interrupt trigger */ 2004 wed_w32(dev, MTK_WED_WPDMA_INT_CTRL_TX, 2005 MTK_WED_WPDMA_INT_CTRL_TX0_DONE_EN | 2006 MTK_WED_WPDMA_INT_CTRL_TX0_DONE_CLR | 2007 MTK_WED_WPDMA_INT_CTRL_TX1_DONE_EN | 2008 MTK_WED_WPDMA_INT_CTRL_TX1_DONE_CLR | 2009 FIELD_PREP(MTK_WED_WPDMA_INT_CTRL_TX0_DONE_TRIG, 2010 dev->wlan.tx_tbit[0]) | 2011 FIELD_PREP(MTK_WED_WPDMA_INT_CTRL_TX1_DONE_TRIG, 2012 dev->wlan.tx_tbit[1])); 2013 2014 /* initail txfree interrupt trigger */ 2015 wed_w32(dev, MTK_WED_WPDMA_INT_CTRL_TX_FREE, 2016 MTK_WED_WPDMA_INT_CTRL_TX_FREE_DONE_EN | 2017 MTK_WED_WPDMA_INT_CTRL_TX_FREE_DONE_CLR | 2018 FIELD_PREP(MTK_WED_WPDMA_INT_CTRL_TX_FREE_DONE_TRIG, 2019 dev->wlan.txfree_tbit)); 2020 2021 if (mtk_wed_get_rx_capa(dev)) { 2022 wed_w32(dev, MTK_WED_WPDMA_INT_CTRL_RX, 2023 MTK_WED_WPDMA_INT_CTRL_RX0_EN | 2024 MTK_WED_WPDMA_INT_CTRL_RX0_CLR | 2025 MTK_WED_WPDMA_INT_CTRL_RX1_EN | 2026 MTK_WED_WPDMA_INT_CTRL_RX1_CLR | 2027 FIELD_PREP(MTK_WED_WPDMA_INT_CTRL_RX0_DONE_TRIG, 2028 dev->wlan.rx_tbit[0]) | 2029 FIELD_PREP(MTK_WED_WPDMA_INT_CTRL_RX1_DONE_TRIG, 2030 dev->wlan.rx_tbit[1])); 2031 2032 wdma_mask |= FIELD_PREP(MTK_WDMA_INT_MASK_TX_DONE, 2033 GENMASK(1, 0)); 2034 } 2035 2036 wed_w32(dev, MTK_WED_WDMA_INT_CLR, wdma_mask); 2037 wed_set(dev, MTK_WED_WDMA_INT_CTRL, 2038 FIELD_PREP(MTK_WED_WDMA_INT_CTRL_POLL_SRC_SEL, 2039 dev->wdma_idx)); 2040 } 2041 2042 wed_w32(dev, MTK_WED_WDMA_INT_TRIGGER, wdma_mask); 2043 2044 wdma_w32(dev, MTK_WDMA_INT_MASK, wdma_mask); 2045 wdma_w32(dev, MTK_WDMA_INT_GRP2, wdma_mask); 2046 wed_w32(dev, MTK_WED_WPDMA_INT_MASK, irq_mask); 2047 wed_w32(dev, MTK_WED_INT_MASK, irq_mask); 2048 } 2049 2050 #define MTK_WFMDA_RX_DMA_EN BIT(2) 2051 static void 2052 mtk_wed_dma_enable(struct mtk_wed_device *dev) 2053 { 2054 int i; 2055 2056 if (!mtk_wed_is_v3_or_greater(dev->hw)) { 2057 wed_set(dev, MTK_WED_WPDMA_INT_CTRL, 2058 MTK_WED_WPDMA_INT_CTRL_SUBRT_ADV); 2059 wed_set(dev, MTK_WED_WPDMA_GLO_CFG, 2060 MTK_WED_WPDMA_GLO_CFG_TX_DRV_EN | 2061 MTK_WED_WPDMA_GLO_CFG_RX_DRV_EN); 2062 wdma_set(dev, MTK_WDMA_GLO_CFG, 2063 MTK_WDMA_GLO_CFG_TX_DMA_EN | 2064 MTK_WDMA_GLO_CFG_RX_INFO1_PRERES | 2065 MTK_WDMA_GLO_CFG_RX_INFO2_PRERES); 2066 wed_set(dev, MTK_WED_WPDMA_CTRL, MTK_WED_WPDMA_CTRL_SDL1_FIXED); 2067 } else { 2068 wed_set(dev, MTK_WED_WPDMA_GLO_CFG, 2069 MTK_WED_WPDMA_GLO_CFG_TX_DRV_EN | 2070 MTK_WED_WPDMA_GLO_CFG_RX_DRV_EN | 2071 MTK_WED_WPDMA_GLO_CFG_RX_DDONE2_WR); 2072 wdma_set(dev, MTK_WDMA_GLO_CFG, MTK_WDMA_GLO_CFG_TX_DMA_EN); 2073 } 2074 2075 wed_set(dev, MTK_WED_GLO_CFG, 2076 MTK_WED_GLO_CFG_TX_DMA_EN | 2077 MTK_WED_GLO_CFG_RX_DMA_EN); 2078 2079 wed_set(dev, MTK_WED_WDMA_GLO_CFG, 2080 MTK_WED_WDMA_GLO_CFG_RX_DRV_EN); 2081 2082 if (mtk_wed_is_v1(dev->hw)) { 2083 wdma_set(dev, MTK_WDMA_GLO_CFG, 2084 MTK_WDMA_GLO_CFG_RX_INFO3_PRERES); 2085 return; 2086 } 2087 2088 wed_set(dev, MTK_WED_WPDMA_GLO_CFG, 2089 MTK_WED_WPDMA_GLO_CFG_RX_DRV_R0_PKT_PROC | 2090 MTK_WED_WPDMA_GLO_CFG_RX_DRV_R0_CRX_SYNC); 2091 2092 if (mtk_wed_is_v3_or_greater(dev->hw)) { 2093 wed_set(dev, MTK_WED_WDMA_RX_PREF_CFG, 2094 FIELD_PREP(MTK_WED_WDMA_RX_PREF_BURST_SIZE, 0x10) | 2095 FIELD_PREP(MTK_WED_WDMA_RX_PREF_LOW_THRES, 0x8)); 2096 wed_clr(dev, MTK_WED_WDMA_RX_PREF_CFG, 2097 MTK_WED_WDMA_RX_PREF_DDONE2_EN); 2098 wed_set(dev, MTK_WED_WDMA_RX_PREF_CFG, MTK_WED_WDMA_RX_PREF_EN); 2099 2100 wed_clr(dev, MTK_WED_WPDMA_GLO_CFG, 2101 MTK_WED_WPDMA_GLO_CFG_TX_DDONE_CHK_LAST); 2102 wed_set(dev, MTK_WED_WPDMA_GLO_CFG, 2103 MTK_WED_WPDMA_GLO_CFG_TX_DDONE_CHK | 2104 MTK_WED_WPDMA_GLO_CFG_RX_DRV_EVENT_PKT_FMT_CHK | 2105 MTK_WED_WPDMA_GLO_CFG_RX_DRV_UNS_VER_FORCE_4); 2106 2107 wdma_set(dev, MTK_WDMA_PREF_RX_CFG, MTK_WDMA_PREF_RX_CFG_PREF_EN); 2108 wdma_set(dev, MTK_WDMA_WRBK_RX_CFG, MTK_WDMA_WRBK_RX_CFG_WRBK_EN); 2109 } 2110 2111 wed_clr(dev, MTK_WED_WPDMA_GLO_CFG, 2112 MTK_WED_WPDMA_GLO_CFG_TX_TKID_KEEP | 2113 MTK_WED_WPDMA_GLO_CFG_TX_DMAD_DW3_PREV); 2114 2115 if (!mtk_wed_get_rx_capa(dev)) 2116 return; 2117 2118 wed_set(dev, MTK_WED_WDMA_GLO_CFG, 2119 MTK_WED_WDMA_GLO_CFG_TX_DRV_EN | 2120 MTK_WED_WDMA_GLO_CFG_TX_DDONE_CHK); 2121 2122 wed_clr(dev, MTK_WED_WPDMA_RX_D_GLO_CFG, MTK_WED_WPDMA_RX_D_RXD_READ_LEN); 2123 wed_set(dev, MTK_WED_WPDMA_RX_D_GLO_CFG, 2124 MTK_WED_WPDMA_RX_D_RX_DRV_EN | 2125 FIELD_PREP(MTK_WED_WPDMA_RX_D_RXD_READ_LEN, 0x18) | 2126 FIELD_PREP(MTK_WED_WPDMA_RX_D_INIT_PHASE_RXEN_SEL, 0x2)); 2127 2128 if (mtk_wed_is_v3_or_greater(dev->hw)) { 2129 wed_set(dev, MTK_WED_WPDMA_RX_D_PREF_CFG, 2130 MTK_WED_WPDMA_RX_D_PREF_EN | 2131 FIELD_PREP(MTK_WED_WPDMA_RX_D_PREF_BURST_SIZE, 0x10) | 2132 FIELD_PREP(MTK_WED_WPDMA_RX_D_PREF_LOW_THRES, 0x8)); 2133 2134 wed_set(dev, MTK_WED_RRO_RX_D_CFG(2), MTK_WED_RRO_RX_D_DRV_EN); 2135 wdma_set(dev, MTK_WDMA_PREF_TX_CFG, MTK_WDMA_PREF_TX_CFG_PREF_EN); 2136 wdma_set(dev, MTK_WDMA_WRBK_TX_CFG, MTK_WDMA_WRBK_TX_CFG_WRBK_EN); 2137 } 2138 2139 for (i = 0; i < MTK_WED_RX_QUEUES; i++) { 2140 struct mtk_wed_ring *ring = &dev->rx_ring[i]; 2141 u32 val; 2142 2143 if (!(ring->flags & MTK_WED_RING_CONFIGURED)) 2144 continue; /* queue is not configured by mt76 */ 2145 2146 if (mtk_wed_check_wfdma_rx_fill(dev, ring)) { 2147 dev_err(dev->hw->dev, 2148 "rx_ring(%d) dma enable failed\n", i); 2149 continue; 2150 } 2151 2152 val = wifi_r32(dev, 2153 dev->wlan.wpdma_rx_glo - 2154 dev->wlan.phy_base) | MTK_WFMDA_RX_DMA_EN; 2155 wifi_w32(dev, 2156 dev->wlan.wpdma_rx_glo - dev->wlan.phy_base, 2157 val); 2158 } 2159 } 2160 2161 static void 2162 mtk_wed_start_hw_rro(struct mtk_wed_device *dev, u32 irq_mask, bool reset) 2163 { 2164 int i; 2165 2166 wed_w32(dev, MTK_WED_WPDMA_INT_MASK, irq_mask); 2167 wed_w32(dev, MTK_WED_INT_MASK, irq_mask); 2168 2169 if (!mtk_wed_get_rx_capa(dev) || !dev->wlan.hw_rro) 2170 return; 2171 2172 if (reset) { 2173 wed_set(dev, MTK_WED_RRO_MSDU_PG_RING2_CFG, 2174 MTK_WED_RRO_MSDU_PG_DRV_EN); 2175 return; 2176 } 2177 2178 wed_set(dev, MTK_WED_RRO_RX_D_CFG(2), MTK_WED_RRO_MSDU_PG_DRV_CLR); 2179 wed_w32(dev, MTK_WED_RRO_MSDU_PG_RING2_CFG, 2180 MTK_WED_RRO_MSDU_PG_DRV_CLR); 2181 2182 wed_w32(dev, MTK_WED_WPDMA_INT_CTRL_RRO_RX, 2183 MTK_WED_WPDMA_INT_CTRL_RRO_RX0_EN | 2184 MTK_WED_WPDMA_INT_CTRL_RRO_RX0_CLR | 2185 MTK_WED_WPDMA_INT_CTRL_RRO_RX1_EN | 2186 MTK_WED_WPDMA_INT_CTRL_RRO_RX1_CLR | 2187 FIELD_PREP(MTK_WED_WPDMA_INT_CTRL_RRO_RX0_DONE_TRIG, 2188 dev->wlan.rro_rx_tbit[0]) | 2189 FIELD_PREP(MTK_WED_WPDMA_INT_CTRL_RRO_RX1_DONE_TRIG, 2190 dev->wlan.rro_rx_tbit[1])); 2191 2192 wed_w32(dev, MTK_WED_WPDMA_INT_CTRL_RRO_MSDU_PG, 2193 MTK_WED_WPDMA_INT_CTRL_RRO_PG0_EN | 2194 MTK_WED_WPDMA_INT_CTRL_RRO_PG0_CLR | 2195 MTK_WED_WPDMA_INT_CTRL_RRO_PG1_EN | 2196 MTK_WED_WPDMA_INT_CTRL_RRO_PG1_CLR | 2197 MTK_WED_WPDMA_INT_CTRL_RRO_PG2_EN | 2198 MTK_WED_WPDMA_INT_CTRL_RRO_PG2_CLR | 2199 FIELD_PREP(MTK_WED_WPDMA_INT_CTRL_RRO_PG0_DONE_TRIG, 2200 dev->wlan.rx_pg_tbit[0]) | 2201 FIELD_PREP(MTK_WED_WPDMA_INT_CTRL_RRO_PG1_DONE_TRIG, 2202 dev->wlan.rx_pg_tbit[1]) | 2203 FIELD_PREP(MTK_WED_WPDMA_INT_CTRL_RRO_PG2_DONE_TRIG, 2204 dev->wlan.rx_pg_tbit[2])); 2205 2206 /* RRO_MSDU_PG_RING2_CFG1_FLD_DRV_EN should be enabled after 2207 * WM FWDL completed, otherwise RRO_MSDU_PG ring may broken 2208 */ 2209 wed_set(dev, MTK_WED_RRO_MSDU_PG_RING2_CFG, 2210 MTK_WED_RRO_MSDU_PG_DRV_EN); 2211 2212 for (i = 0; i < MTK_WED_RX_QUEUES; i++) { 2213 struct mtk_wed_ring *ring = &dev->rx_rro_ring[i]; 2214 2215 if (!(ring->flags & MTK_WED_RING_CONFIGURED)) 2216 continue; 2217 2218 if (mtk_wed_check_wfdma_rx_fill(dev, ring)) 2219 dev_err(dev->hw->dev, 2220 "rx_rro_ring(%d) initialization failed\n", i); 2221 } 2222 2223 for (i = 0; i < MTK_WED_RX_PAGE_QUEUES; i++) { 2224 struct mtk_wed_ring *ring = &dev->rx_page_ring[i]; 2225 2226 if (!(ring->flags & MTK_WED_RING_CONFIGURED)) 2227 continue; 2228 2229 if (mtk_wed_check_wfdma_rx_fill(dev, ring)) 2230 dev_err(dev->hw->dev, 2231 "rx_page_ring(%d) initialization failed\n", i); 2232 } 2233 } 2234 2235 static void 2236 mtk_wed_rro_rx_ring_setup(struct mtk_wed_device *dev, int idx, 2237 void __iomem *regs) 2238 { 2239 struct mtk_wed_ring *ring = &dev->rx_rro_ring[idx]; 2240 2241 ring->wpdma = regs; 2242 wed_w32(dev, MTK_WED_RRO_RX_D_RX(idx) + MTK_WED_RING_OFS_BASE, 2243 readl(regs)); 2244 wed_w32(dev, MTK_WED_RRO_RX_D_RX(idx) + MTK_WED_RING_OFS_COUNT, 2245 readl(regs + MTK_WED_RING_OFS_COUNT)); 2246 ring->flags |= MTK_WED_RING_CONFIGURED; 2247 } 2248 2249 static void 2250 mtk_wed_msdu_pg_rx_ring_setup(struct mtk_wed_device *dev, int idx, void __iomem *regs) 2251 { 2252 struct mtk_wed_ring *ring = &dev->rx_page_ring[idx]; 2253 2254 ring->wpdma = regs; 2255 wed_w32(dev, MTK_WED_RRO_MSDU_PG_CTRL0(idx) + MTK_WED_RING_OFS_BASE, 2256 readl(regs)); 2257 wed_w32(dev, MTK_WED_RRO_MSDU_PG_CTRL0(idx) + MTK_WED_RING_OFS_COUNT, 2258 readl(regs + MTK_WED_RING_OFS_COUNT)); 2259 ring->flags |= MTK_WED_RING_CONFIGURED; 2260 } 2261 2262 static int 2263 mtk_wed_ind_rx_ring_setup(struct mtk_wed_device *dev, void __iomem *regs) 2264 { 2265 struct mtk_wed_ring *ring = &dev->ind_cmd_ring; 2266 u32 val = readl(regs + MTK_WED_RING_OFS_COUNT); 2267 int i, count = 0; 2268 2269 ring->wpdma = regs; 2270 wed_w32(dev, MTK_WED_IND_CMD_RX_CTRL1 + MTK_WED_RING_OFS_BASE, 2271 readl(regs) & 0xfffffff0); 2272 2273 wed_w32(dev, MTK_WED_IND_CMD_RX_CTRL1 + MTK_WED_RING_OFS_COUNT, 2274 readl(regs + MTK_WED_RING_OFS_COUNT)); 2275 2276 /* ack sn cr */ 2277 wed_w32(dev, MTK_WED_RRO_CFG0, dev->wlan.phy_base + 2278 dev->wlan.ind_cmd.ack_sn_addr); 2279 wed_w32(dev, MTK_WED_RRO_CFG1, 2280 FIELD_PREP(MTK_WED_RRO_CFG1_MAX_WIN_SZ, 2281 dev->wlan.ind_cmd.win_size) | 2282 FIELD_PREP(MTK_WED_RRO_CFG1_PARTICL_SE_ID, 2283 dev->wlan.ind_cmd.particular_sid)); 2284 2285 /* particular session addr element */ 2286 wed_w32(dev, MTK_WED_ADDR_ELEM_CFG0, 2287 dev->wlan.ind_cmd.particular_se_phys); 2288 2289 for (i = 0; i < dev->wlan.ind_cmd.se_group_nums; i++) { 2290 wed_w32(dev, MTK_WED_RADDR_ELEM_TBL_WDATA, 2291 dev->wlan.ind_cmd.addr_elem_phys[i] >> 4); 2292 wed_w32(dev, MTK_WED_ADDR_ELEM_TBL_CFG, 2293 MTK_WED_ADDR_ELEM_TBL_WR | (i & 0x7f)); 2294 2295 val = wed_r32(dev, MTK_WED_ADDR_ELEM_TBL_CFG); 2296 while (!(val & MTK_WED_ADDR_ELEM_TBL_WR_RDY) && count++ < 100) 2297 val = wed_r32(dev, MTK_WED_ADDR_ELEM_TBL_CFG); 2298 if (count >= 100) 2299 dev_err(dev->hw->dev, 2300 "write ba session base failed\n"); 2301 } 2302 2303 /* pn check init */ 2304 for (i = 0; i < dev->wlan.ind_cmd.particular_sid; i++) { 2305 wed_w32(dev, MTK_WED_PN_CHECK_WDATA_M, 2306 MTK_WED_PN_CHECK_IS_FIRST); 2307 2308 wed_w32(dev, MTK_WED_PN_CHECK_CFG, MTK_WED_PN_CHECK_WR | 2309 FIELD_PREP(MTK_WED_PN_CHECK_SE_ID, i)); 2310 2311 count = 0; 2312 val = wed_r32(dev, MTK_WED_PN_CHECK_CFG); 2313 while (!(val & MTK_WED_PN_CHECK_WR_RDY) && count++ < 100) 2314 val = wed_r32(dev, MTK_WED_PN_CHECK_CFG); 2315 if (count >= 100) 2316 dev_err(dev->hw->dev, 2317 "session(%d) initialization failed\n", i); 2318 } 2319 2320 wed_w32(dev, MTK_WED_RX_IND_CMD_CNT0, MTK_WED_RX_IND_CMD_DBG_CNT_EN); 2321 wed_set(dev, MTK_WED_CTRL, MTK_WED_CTRL_WED_RX_IND_CMD_EN); 2322 2323 return 0; 2324 } 2325 2326 static void 2327 mtk_wed_start(struct mtk_wed_device *dev, u32 irq_mask) 2328 { 2329 int i; 2330 2331 if (mtk_wed_get_rx_capa(dev) && mtk_wed_rx_buffer_alloc(dev)) 2332 return; 2333 2334 for (i = 0; i < ARRAY_SIZE(dev->rx_wdma); i++) 2335 if (!dev->rx_wdma[i].desc) 2336 mtk_wed_wdma_rx_ring_setup(dev, i, 16, false); 2337 2338 mtk_wed_hw_init(dev); 2339 mtk_wed_configure_irq(dev, irq_mask); 2340 2341 mtk_wed_set_ext_int(dev, true); 2342 2343 if (mtk_wed_is_v1(dev->hw)) { 2344 u32 val = dev->wlan.wpdma_phys | MTK_PCIE_MIRROR_MAP_EN | 2345 FIELD_PREP(MTK_PCIE_MIRROR_MAP_WED_ID, 2346 dev->hw->index); 2347 2348 val |= BIT(0) | (BIT(1) * !!dev->hw->index); 2349 regmap_write(dev->hw->mirror, dev->hw->index * 4, val); 2350 } else if (mtk_wed_get_rx_capa(dev)) { 2351 /* driver set mid ready and only once */ 2352 wed_w32(dev, MTK_WED_EXT_INT_MASK1, 2353 MTK_WED_EXT_INT_STATUS_WPDMA_MID_RDY); 2354 wed_w32(dev, MTK_WED_EXT_INT_MASK2, 2355 MTK_WED_EXT_INT_STATUS_WPDMA_MID_RDY); 2356 2357 wed_r32(dev, MTK_WED_EXT_INT_MASK1); 2358 wed_r32(dev, MTK_WED_EXT_INT_MASK2); 2359 2360 if (mtk_wed_is_v3_or_greater(dev->hw)) { 2361 wed_w32(dev, MTK_WED_EXT_INT_MASK3, 2362 MTK_WED_EXT_INT_STATUS_WPDMA_MID_RDY); 2363 wed_r32(dev, MTK_WED_EXT_INT_MASK3); 2364 } 2365 2366 if (mtk_wed_rro_cfg(dev)) 2367 return; 2368 } 2369 2370 mtk_wed_set_512_support(dev, dev->wlan.wcid_512); 2371 mtk_wed_amsdu_init(dev); 2372 2373 mtk_wed_dma_enable(dev); 2374 dev->running = true; 2375 } 2376 2377 static int 2378 mtk_wed_attach(struct mtk_wed_device *dev) 2379 __releases(RCU) 2380 { 2381 struct mtk_wed_hw *hw; 2382 struct device *device; 2383 int ret = 0; 2384 2385 RCU_LOCKDEP_WARN(!rcu_read_lock_held(), 2386 "mtk_wed_attach without holding the RCU read lock"); 2387 2388 if ((dev->wlan.bus_type == MTK_WED_BUS_PCIE && 2389 pci_domain_nr(dev->wlan.pci_dev->bus) > 1) || 2390 !try_module_get(THIS_MODULE)) 2391 ret = -ENODEV; 2392 2393 rcu_read_unlock(); 2394 2395 if (ret) 2396 return ret; 2397 2398 mutex_lock(&hw_lock); 2399 2400 hw = mtk_wed_assign(dev); 2401 if (!hw) { 2402 module_put(THIS_MODULE); 2403 ret = -ENODEV; 2404 goto unlock; 2405 } 2406 2407 device = dev->wlan.bus_type == MTK_WED_BUS_PCIE 2408 ? &dev->wlan.pci_dev->dev 2409 : &dev->wlan.platform_dev->dev; 2410 dev_info(device, "attaching wed device %d version %d\n", 2411 hw->index, hw->version); 2412 2413 dev->hw = hw; 2414 dev->dev = hw->dev; 2415 dev->irq = hw->irq; 2416 dev->wdma_idx = hw->index; 2417 dev->version = hw->version; 2418 dev->hw->pcie_base = mtk_wed_get_pcie_base(dev); 2419 2420 if (hw->eth->dma_dev == hw->eth->dev && 2421 of_dma_is_coherent(hw->eth->dev->of_node)) 2422 mtk_eth_set_dma_device(hw->eth, hw->dev); 2423 2424 ret = mtk_wed_tx_buffer_alloc(dev); 2425 if (ret) 2426 goto out; 2427 2428 ret = mtk_wed_amsdu_buffer_alloc(dev); 2429 if (ret) 2430 goto out; 2431 2432 if (mtk_wed_get_rx_capa(dev)) { 2433 ret = mtk_wed_rro_alloc(dev); 2434 if (ret) 2435 goto out; 2436 } 2437 2438 mtk_wed_hw_init_early(dev); 2439 if (mtk_wed_is_v1(hw)) 2440 regmap_update_bits(hw->hifsys, HIFSYS_DMA_AG_MAP, 2441 BIT(hw->index), 0); 2442 else 2443 dev->rev_id = wed_r32(dev, MTK_WED_REV_ID); 2444 2445 if (mtk_wed_get_rx_capa(dev)) 2446 ret = mtk_wed_wo_init(hw); 2447 out: 2448 if (ret) { 2449 dev_err(dev->hw->dev, "failed to attach wed device\n"); 2450 __mtk_wed_detach(dev); 2451 } 2452 unlock: 2453 mutex_unlock(&hw_lock); 2454 2455 return ret; 2456 } 2457 2458 static int 2459 mtk_wed_tx_ring_setup(struct mtk_wed_device *dev, int idx, void __iomem *regs, 2460 bool reset) 2461 { 2462 struct mtk_wed_ring *ring = &dev->tx_ring[idx]; 2463 2464 /* 2465 * Tx ring redirection: 2466 * Instead of configuring the WLAN PDMA TX ring directly, the WLAN 2467 * driver allocated DMA ring gets configured into WED MTK_WED_RING_TX(n) 2468 * registers. 2469 * 2470 * WED driver posts its own DMA ring as WLAN PDMA TX and configures it 2471 * into MTK_WED_WPDMA_RING_TX(n) registers. 2472 * It gets filled with packets picked up from WED TX ring and from 2473 * WDMA RX. 2474 */ 2475 2476 if (WARN_ON(idx >= ARRAY_SIZE(dev->tx_ring))) 2477 return -EINVAL; 2478 2479 if (!reset && mtk_wed_ring_alloc(dev, ring, MTK_WED_TX_RING_SIZE, 2480 sizeof(*ring->desc), true)) 2481 return -ENOMEM; 2482 2483 if (mtk_wed_wdma_rx_ring_setup(dev, idx, MTK_WED_WDMA_RING_SIZE, 2484 reset)) 2485 return -ENOMEM; 2486 2487 ring->reg_base = MTK_WED_RING_TX(idx); 2488 ring->wpdma = regs; 2489 2490 if (mtk_wed_is_v3_or_greater(dev->hw) && idx == 1) { 2491 /* reset prefetch index */ 2492 wed_set(dev, MTK_WED_WDMA_RX_PREF_CFG, 2493 MTK_WED_WDMA_RX_PREF_RX0_SIDX_CLR | 2494 MTK_WED_WDMA_RX_PREF_RX1_SIDX_CLR); 2495 2496 wed_clr(dev, MTK_WED_WDMA_RX_PREF_CFG, 2497 MTK_WED_WDMA_RX_PREF_RX0_SIDX_CLR | 2498 MTK_WED_WDMA_RX_PREF_RX1_SIDX_CLR); 2499 2500 /* reset prefetch FIFO */ 2501 wed_w32(dev, MTK_WED_WDMA_RX_PREF_FIFO_CFG, 2502 MTK_WED_WDMA_RX_PREF_FIFO_RX0_CLR | 2503 MTK_WED_WDMA_RX_PREF_FIFO_RX1_CLR); 2504 wed_w32(dev, MTK_WED_WDMA_RX_PREF_FIFO_CFG, 0); 2505 } 2506 2507 /* WED -> WPDMA */ 2508 wpdma_tx_w32(dev, idx, MTK_WED_RING_OFS_BASE, ring->desc_phys); 2509 wpdma_tx_w32(dev, idx, MTK_WED_RING_OFS_COUNT, MTK_WED_TX_RING_SIZE); 2510 wpdma_tx_w32(dev, idx, MTK_WED_RING_OFS_CPU_IDX, 0); 2511 2512 wed_w32(dev, MTK_WED_WPDMA_RING_TX(idx) + MTK_WED_RING_OFS_BASE, 2513 ring->desc_phys); 2514 wed_w32(dev, MTK_WED_WPDMA_RING_TX(idx) + MTK_WED_RING_OFS_COUNT, 2515 MTK_WED_TX_RING_SIZE); 2516 wed_w32(dev, MTK_WED_WPDMA_RING_TX(idx) + MTK_WED_RING_OFS_CPU_IDX, 0); 2517 2518 return 0; 2519 } 2520 2521 static int 2522 mtk_wed_txfree_ring_setup(struct mtk_wed_device *dev, void __iomem *regs) 2523 { 2524 struct mtk_wed_ring *ring = &dev->txfree_ring; 2525 int i, index = mtk_wed_is_v1(dev->hw); 2526 2527 /* 2528 * For txfree event handling, the same DMA ring is shared between WED 2529 * and WLAN. The WLAN driver accesses the ring index registers through 2530 * WED 2531 */ 2532 ring->reg_base = MTK_WED_RING_RX(index); 2533 ring->wpdma = regs; 2534 2535 for (i = 0; i < 12; i += 4) { 2536 u32 val = readl(regs + i); 2537 2538 wed_w32(dev, MTK_WED_RING_RX(index) + i, val); 2539 wed_w32(dev, MTK_WED_WPDMA_RING_RX(index) + i, val); 2540 } 2541 2542 return 0; 2543 } 2544 2545 static int 2546 mtk_wed_rx_ring_setup(struct mtk_wed_device *dev, int idx, void __iomem *regs, 2547 bool reset) 2548 { 2549 struct mtk_wed_ring *ring = &dev->rx_ring[idx]; 2550 2551 if (WARN_ON(idx >= ARRAY_SIZE(dev->rx_ring))) 2552 return -EINVAL; 2553 2554 if (!reset && mtk_wed_ring_alloc(dev, ring, MTK_WED_RX_RING_SIZE, 2555 sizeof(*ring->desc), false)) 2556 return -ENOMEM; 2557 2558 if (mtk_wed_wdma_tx_ring_setup(dev, idx, MTK_WED_WDMA_RING_SIZE, 2559 reset)) 2560 return -ENOMEM; 2561 2562 ring->reg_base = MTK_WED_RING_RX_DATA(idx); 2563 ring->wpdma = regs; 2564 ring->flags |= MTK_WED_RING_CONFIGURED; 2565 2566 /* WPDMA -> WED */ 2567 wpdma_rx_w32(dev, idx, MTK_WED_RING_OFS_BASE, ring->desc_phys); 2568 wpdma_rx_w32(dev, idx, MTK_WED_RING_OFS_COUNT, MTK_WED_RX_RING_SIZE); 2569 2570 wed_w32(dev, MTK_WED_WPDMA_RING_RX_DATA(idx) + MTK_WED_RING_OFS_BASE, 2571 ring->desc_phys); 2572 wed_w32(dev, MTK_WED_WPDMA_RING_RX_DATA(idx) + MTK_WED_RING_OFS_COUNT, 2573 MTK_WED_RX_RING_SIZE); 2574 2575 return 0; 2576 } 2577 2578 static u32 2579 mtk_wed_irq_get(struct mtk_wed_device *dev, u32 mask) 2580 { 2581 u32 val, ext_mask; 2582 2583 if (mtk_wed_is_v3_or_greater(dev->hw)) 2584 ext_mask = MTK_WED_EXT_INT_STATUS_RX_DRV_COHERENT | 2585 MTK_WED_EXT_INT_STATUS_TKID_WO_PYLD; 2586 else 2587 ext_mask = MTK_WED_EXT_INT_STATUS_ERROR_MASK; 2588 2589 val = wed_r32(dev, MTK_WED_EXT_INT_STATUS); 2590 wed_w32(dev, MTK_WED_EXT_INT_STATUS, val); 2591 val &= ext_mask; 2592 if (!dev->hw->num_flows) 2593 val &= ~MTK_WED_EXT_INT_STATUS_TKID_WO_PYLD; 2594 if (val && net_ratelimit()) 2595 pr_err("mtk_wed%d: error status=%08x\n", dev->hw->index, val); 2596 2597 val = wed_r32(dev, MTK_WED_INT_STATUS); 2598 val &= mask; 2599 wed_w32(dev, MTK_WED_INT_STATUS, val); /* ACK */ 2600 2601 return val; 2602 } 2603 2604 static void 2605 mtk_wed_irq_set_mask(struct mtk_wed_device *dev, u32 mask) 2606 { 2607 mtk_wed_set_ext_int(dev, !!mask); 2608 wed_w32(dev, MTK_WED_INT_MASK, mask); 2609 } 2610 2611 int mtk_wed_flow_add(int index) 2612 { 2613 struct mtk_wed_hw *hw = hw_list[index]; 2614 int ret = 0; 2615 2616 mutex_lock(&hw_lock); 2617 2618 if (!hw || !hw->wed_dev) { 2619 ret = -ENODEV; 2620 goto out; 2621 } 2622 2623 if (!hw->wed_dev->wlan.offload_enable) 2624 goto out; 2625 2626 if (hw->num_flows) { 2627 hw->num_flows++; 2628 goto out; 2629 } 2630 2631 ret = hw->wed_dev->wlan.offload_enable(hw->wed_dev); 2632 if (!ret) 2633 hw->num_flows++; 2634 mtk_wed_set_ext_int(hw->wed_dev, true); 2635 2636 out: 2637 mutex_unlock(&hw_lock); 2638 2639 return ret; 2640 } 2641 2642 void mtk_wed_flow_remove(int index) 2643 { 2644 struct mtk_wed_hw *hw = hw_list[index]; 2645 2646 mutex_lock(&hw_lock); 2647 2648 if (!hw || !hw->wed_dev) 2649 goto out; 2650 2651 if (!hw->wed_dev->wlan.offload_disable) 2652 goto out; 2653 2654 if (--hw->num_flows) 2655 goto out; 2656 2657 hw->wed_dev->wlan.offload_disable(hw->wed_dev); 2658 mtk_wed_set_ext_int(hw->wed_dev, true); 2659 2660 out: 2661 mutex_unlock(&hw_lock); 2662 } 2663 2664 static int 2665 mtk_wed_setup_tc_block_cb(enum tc_setup_type type, void *type_data, void *cb_priv) 2666 { 2667 struct mtk_wed_flow_block_priv *priv = cb_priv; 2668 struct flow_cls_offload *cls = type_data; 2669 struct mtk_wed_hw *hw = NULL; 2670 2671 if (!priv || !tc_can_offload(priv->dev)) 2672 return -EOPNOTSUPP; 2673 2674 if (type != TC_SETUP_CLSFLOWER) 2675 return -EOPNOTSUPP; 2676 2677 hw = priv->hw; 2678 return mtk_flow_offload_cmd(hw->eth, cls, hw->index); 2679 } 2680 2681 static int 2682 mtk_wed_setup_tc_block(struct mtk_wed_hw *hw, struct net_device *dev, 2683 struct flow_block_offload *f) 2684 { 2685 struct mtk_wed_flow_block_priv *priv; 2686 static LIST_HEAD(block_cb_list); 2687 struct flow_block_cb *block_cb; 2688 struct mtk_eth *eth = hw->eth; 2689 flow_setup_cb_t *cb; 2690 2691 if (!eth->soc->offload_version) 2692 return -EOPNOTSUPP; 2693 2694 if (f->binder_type != FLOW_BLOCK_BINDER_TYPE_CLSACT_INGRESS) 2695 return -EOPNOTSUPP; 2696 2697 cb = mtk_wed_setup_tc_block_cb; 2698 f->driver_block_list = &block_cb_list; 2699 2700 switch (f->command) { 2701 case FLOW_BLOCK_BIND: 2702 block_cb = flow_block_cb_lookup(f->block, cb, dev); 2703 if (block_cb) { 2704 flow_block_cb_incref(block_cb); 2705 return 0; 2706 } 2707 2708 priv = kzalloc(sizeof(*priv), GFP_KERNEL); 2709 if (!priv) 2710 return -ENOMEM; 2711 2712 priv->hw = hw; 2713 priv->dev = dev; 2714 block_cb = flow_block_cb_alloc(cb, dev, priv, NULL); 2715 if (IS_ERR(block_cb)) { 2716 kfree(priv); 2717 return PTR_ERR(block_cb); 2718 } 2719 2720 flow_block_cb_incref(block_cb); 2721 flow_block_cb_add(block_cb, f); 2722 list_add_tail(&block_cb->driver_list, &block_cb_list); 2723 return 0; 2724 case FLOW_BLOCK_UNBIND: 2725 block_cb = flow_block_cb_lookup(f->block, cb, dev); 2726 if (!block_cb) 2727 return -ENOENT; 2728 2729 if (!flow_block_cb_decref(block_cb)) { 2730 flow_block_cb_remove(block_cb, f); 2731 list_del(&block_cb->driver_list); 2732 kfree(block_cb->cb_priv); 2733 block_cb->cb_priv = NULL; 2734 } 2735 return 0; 2736 default: 2737 return -EOPNOTSUPP; 2738 } 2739 } 2740 2741 static int 2742 mtk_wed_setup_tc(struct mtk_wed_device *wed, struct net_device *dev, 2743 enum tc_setup_type type, void *type_data) 2744 { 2745 struct mtk_wed_hw *hw = wed->hw; 2746 2747 if (mtk_wed_is_v1(hw)) 2748 return -EOPNOTSUPP; 2749 2750 switch (type) { 2751 case TC_SETUP_BLOCK: 2752 case TC_SETUP_FT: 2753 return mtk_wed_setup_tc_block(hw, dev, type_data); 2754 default: 2755 return -EOPNOTSUPP; 2756 } 2757 } 2758 2759 void mtk_wed_add_hw(struct device_node *np, struct mtk_eth *eth, 2760 void __iomem *wdma, phys_addr_t wdma_phy, 2761 int index) 2762 { 2763 static const struct mtk_wed_ops wed_ops = { 2764 .attach = mtk_wed_attach, 2765 .tx_ring_setup = mtk_wed_tx_ring_setup, 2766 .rx_ring_setup = mtk_wed_rx_ring_setup, 2767 .txfree_ring_setup = mtk_wed_txfree_ring_setup, 2768 .msg_update = mtk_wed_mcu_msg_update, 2769 .start = mtk_wed_start, 2770 .stop = mtk_wed_stop, 2771 .reset_dma = mtk_wed_reset_dma, 2772 .reg_read = wed_r32, 2773 .reg_write = wed_w32, 2774 .irq_get = mtk_wed_irq_get, 2775 .irq_set_mask = mtk_wed_irq_set_mask, 2776 .detach = mtk_wed_detach, 2777 .ppe_check = mtk_wed_ppe_check, 2778 .setup_tc = mtk_wed_setup_tc, 2779 .start_hw_rro = mtk_wed_start_hw_rro, 2780 .rro_rx_ring_setup = mtk_wed_rro_rx_ring_setup, 2781 .msdu_pg_rx_ring_setup = mtk_wed_msdu_pg_rx_ring_setup, 2782 .ind_rx_ring_setup = mtk_wed_ind_rx_ring_setup, 2783 }; 2784 struct device_node *eth_np = eth->dev->of_node; 2785 struct platform_device *pdev; 2786 struct mtk_wed_hw *hw; 2787 struct regmap *regs; 2788 int irq; 2789 2790 if (!np) 2791 return; 2792 2793 pdev = of_find_device_by_node(np); 2794 if (!pdev) 2795 goto err_of_node_put; 2796 2797 get_device(&pdev->dev); 2798 irq = platform_get_irq(pdev, 0); 2799 if (irq < 0) 2800 goto err_put_device; 2801 2802 regs = syscon_regmap_lookup_by_phandle(np, NULL); 2803 if (IS_ERR(regs)) 2804 goto err_put_device; 2805 2806 rcu_assign_pointer(mtk_soc_wed_ops, &wed_ops); 2807 2808 mutex_lock(&hw_lock); 2809 2810 if (WARN_ON(hw_list[index])) 2811 goto unlock; 2812 2813 hw = kzalloc(sizeof(*hw), GFP_KERNEL); 2814 if (!hw) 2815 goto unlock; 2816 2817 hw->node = np; 2818 hw->regs = regs; 2819 hw->eth = eth; 2820 hw->dev = &pdev->dev; 2821 hw->wdma_phy = wdma_phy; 2822 hw->wdma = wdma; 2823 hw->index = index; 2824 hw->irq = irq; 2825 hw->version = eth->soc->version; 2826 2827 switch (hw->version) { 2828 case 2: 2829 hw->soc = &mt7986_data; 2830 break; 2831 case 3: 2832 hw->soc = &mt7988_data; 2833 break; 2834 default: 2835 case 1: 2836 hw->mirror = syscon_regmap_lookup_by_phandle(eth_np, 2837 "mediatek,pcie-mirror"); 2838 hw->hifsys = syscon_regmap_lookup_by_phandle(eth_np, 2839 "mediatek,hifsys"); 2840 if (IS_ERR(hw->mirror) || IS_ERR(hw->hifsys)) { 2841 kfree(hw); 2842 goto unlock; 2843 } 2844 2845 if (!index) { 2846 regmap_write(hw->mirror, 0, 0); 2847 regmap_write(hw->mirror, 4, 0); 2848 } 2849 hw->soc = &mt7622_data; 2850 break; 2851 } 2852 2853 mtk_wed_hw_add_debugfs(hw); 2854 2855 hw_list[index] = hw; 2856 2857 mutex_unlock(&hw_lock); 2858 2859 return; 2860 2861 unlock: 2862 mutex_unlock(&hw_lock); 2863 err_put_device: 2864 put_device(&pdev->dev); 2865 err_of_node_put: 2866 of_node_put(np); 2867 } 2868 2869 void mtk_wed_exit(void) 2870 { 2871 int i; 2872 2873 rcu_assign_pointer(mtk_soc_wed_ops, NULL); 2874 2875 synchronize_rcu(); 2876 2877 for (i = 0; i < ARRAY_SIZE(hw_list); i++) { 2878 struct mtk_wed_hw *hw; 2879 2880 hw = hw_list[i]; 2881 if (!hw) 2882 continue; 2883 2884 hw_list[i] = NULL; 2885 debugfs_remove(hw->debugfs_dir); 2886 put_device(hw->dev); 2887 of_node_put(hw->node); 2888 kfree(hw); 2889 } 2890 } 2891