1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (c) 2025 AIROHA Inc 4 * Author: Lorenzo Bianconi <lorenzo@kernel.org> 5 */ 6 #include <linux/kernel.h> 7 #include <net/flow_offload.h> 8 #include <net/pkt_cls.h> 9 10 #include "mt76.h" 11 #include "dma.h" 12 #include "mt76_connac.h" 13 14 #define MT76_NPU_RX_BUF_SIZE (1800 + \ 15 SKB_DATA_ALIGN(sizeof(struct skb_shared_info))) 16 17 int mt76_npu_fill_rx_queue(struct mt76_dev *dev, struct mt76_queue *q) 18 { 19 int nframes = 0; 20 21 while (q->queued < q->ndesc - 1) { 22 struct airoha_npu_rx_dma_desc *desc = (void *)q->desc; 23 struct mt76_queue_entry *e = &q->entry[q->head]; 24 struct page *page; 25 int offset; 26 27 e->buf = mt76_get_page_pool_buf(q, &offset, q->buf_size); 28 if (!e->buf) 29 break; 30 31 e->dma_len[0] = SKB_WITH_OVERHEAD(q->buf_size); 32 page = virt_to_head_page(e->buf); 33 e->dma_addr[0] = page_pool_get_dma_addr(page) + offset; 34 35 memset(&desc[q->head], 0, sizeof(*desc)); 36 desc[q->head].addr = e->dma_addr[0]; 37 38 q->head = (q->head + 1) % q->ndesc; 39 q->queued++; 40 nframes++; 41 } 42 43 return nframes; 44 } 45 46 void mt76_npu_queue_cleanup(struct mt76_dev *dev, struct mt76_queue *q) 47 { 48 spin_lock_bh(&q->lock); 49 while (q->queued > 0) { 50 struct mt76_queue_entry *e = &q->entry[q->tail]; 51 52 dma_sync_single_for_cpu(dev->dma_dev, e->dma_addr[0], 53 e->dma_len[0], 54 page_pool_get_dma_dir(q->page_pool)); 55 mt76_put_page_pool_buf(e->buf, false); 56 q->tail = (q->tail + 1) % q->ndesc; 57 q->queued--; 58 } 59 spin_unlock_bh(&q->lock); 60 } 61 62 static struct sk_buff *mt76_npu_dequeue(struct mt76_dev *dev, 63 struct mt76_queue *q, 64 u32 *info) 65 { 66 struct airoha_npu_rx_dma_desc *desc = (void *)q->desc; 67 int i, nframes, index = q->tail; 68 struct sk_buff *skb = NULL; 69 70 nframes = FIELD_GET(NPU_RX_DMA_PKT_COUNT_MASK, desc[index].info); 71 nframes = max_t(int, nframes, 1); 72 73 for (i = 0; i < nframes; i++) { 74 struct mt76_queue_entry *e = &q->entry[index]; 75 int len = FIELD_GET(NPU_RX_DMA_DESC_CUR_LEN_MASK, 76 desc[index].ctrl); 77 78 if (!FIELD_GET(NPU_RX_DMA_DESC_DONE_MASK, desc[index].ctrl)) { 79 dev_kfree_skb(skb); 80 return NULL; 81 } 82 83 dma_sync_single_for_cpu(dev->dma_dev, e->dma_addr[0], 84 e->dma_len[0], 85 page_pool_get_dma_dir(q->page_pool)); 86 87 if (!skb) { 88 skb = napi_build_skb(e->buf, q->buf_size); 89 if (!skb) 90 return NULL; 91 92 __skb_put(skb, len); 93 skb_reset_mac_header(skb); 94 skb_mark_for_recycle(skb); 95 } else { 96 struct skb_shared_info *shinfo = skb_shinfo(skb); 97 struct page *page = virt_to_head_page(e->buf); 98 int nr_frags = shinfo->nr_frags; 99 100 if (nr_frags < ARRAY_SIZE(shinfo->frags)) 101 skb_add_rx_frag(skb, nr_frags, page, 102 e->buf - page_address(page), 103 len, q->buf_size); 104 } 105 106 *info = desc[index].info; 107 index = (index + 1) % q->ndesc; 108 } 109 q->tail = index; 110 q->queued -= i; 111 Q_WRITE(q, dma_idx, q->tail); 112 113 return skb; 114 } 115 116 void mt76_npu_check_ppe(struct mt76_dev *dev, struct sk_buff *skb, 117 u32 info) 118 { 119 struct airoha_ppe_dev *ppe_dev; 120 u16 reason, hash; 121 122 if (!mt76_npu_device_active(dev)) 123 return; 124 125 rcu_read_lock(); 126 127 ppe_dev = rcu_dereference(dev->mmio.ppe_dev); 128 if (!ppe_dev) 129 goto out; 130 131 hash = FIELD_GET(NPU_RX_DMA_FOE_ID_MASK, info); 132 skb_set_hash(skb, hash, PKT_HASH_TYPE_L4); 133 134 reason = FIELD_GET(NPU_RX_DMA_CRSN_MASK, info); 135 if (reason == PPE_CPU_REASON_HIT_UNBIND_RATE_REACHED) { 136 skb_set_mac_header(skb, 0); 137 airoha_ppe_dev_check_skb(ppe_dev, skb, hash, true); 138 } 139 out: 140 rcu_read_unlock(); 141 } 142 EXPORT_SYMBOL_GPL(mt76_npu_check_ppe); 143 144 static int mt76_npu_rx_poll(struct napi_struct *napi, int budget) 145 { 146 struct mt76_dev *dev = mt76_priv(napi->dev); 147 enum mt76_rxq_id qid = napi - dev->napi; 148 struct airoha_npu *npu; 149 int done = 0; 150 151 rcu_read_lock(); 152 153 npu = rcu_dereference(dev->mmio.npu); 154 if (!npu) 155 goto out; 156 157 while (done < budget) { 158 struct sk_buff *skb; 159 u32 info = 0; 160 161 skb = mt76_npu_dequeue(dev, &dev->q_rx[qid], &info); 162 if (!skb) 163 break; 164 165 dev->drv->rx_skb(dev, qid, skb, &info); 166 mt76_rx_poll_complete(dev, qid, napi); 167 done++; 168 } 169 170 mt76_npu_fill_rx_queue(dev, &dev->q_rx[qid]); 171 out: 172 if (done < budget && napi_complete(napi)) 173 dev->drv->rx_poll_complete(dev, qid); 174 175 rcu_read_unlock(); 176 177 return done; 178 } 179 180 static irqreturn_t mt76_npu_irq_handler(int irq, void *q_instance) 181 { 182 struct mt76_queue *q = q_instance; 183 struct mt76_dev *dev = q->dev; 184 int qid = q - &dev->q_rx[0]; 185 int index = qid - MT_RXQ_NPU0; 186 struct airoha_npu *npu; 187 u32 status; 188 189 rcu_read_lock(); 190 191 npu = rcu_dereference(dev->mmio.npu); 192 if (!npu) 193 goto out; 194 195 status = airoha_npu_wlan_get_irq_status(npu, index); 196 airoha_npu_wlan_set_irq_status(npu, status); 197 198 airoha_npu_wlan_disable_irq(npu, index); 199 napi_schedule(&dev->napi[qid]); 200 out: 201 rcu_read_unlock(); 202 203 return IRQ_HANDLED; 204 } 205 206 int mt76_npu_dma_add_buf(struct mt76_phy *phy, struct mt76_queue *q, 207 struct sk_buff *skb, struct mt76_queue_buf *buf, 208 void *txwi_ptr) 209 { 210 u16 txwi_len = min_t(u16, phy->dev->drv->txwi_size, NPU_TXWI_LEN); 211 struct airoha_npu_tx_dma_desc *desc = (void *)q->desc; 212 int ret; 213 214 /* TODO: Take into account unlinear skbs */ 215 memcpy(desc[q->head].txwi, txwi_ptr, txwi_len); 216 desc[q->head].addr = buf->addr; 217 desc[q->head].ctrl = FIELD_PREP(NPU_TX_DMA_DESC_VEND_LEN_MASK, txwi_len) | 218 FIELD_PREP(NPU_TX_DMA_DESC_LEN_MASK, skb->len) | 219 NPU_TX_DMA_DESC_DONE_MASK; 220 221 ret = q->head; 222 q->entry[q->head].skip_buf0 = true; 223 q->entry[q->head].skip_buf1 = true; 224 q->entry[q->head].txwi = NULL; 225 q->entry[q->head].skb = NULL; 226 q->entry[q->head].wcid = 0xffff; 227 228 q->head = (q->head + 1) % q->ndesc; 229 q->queued++; 230 231 return ret; 232 } 233 234 void mt76_npu_txdesc_cleanup(struct mt76_queue *q, int index) 235 { 236 struct airoha_npu_tx_dma_desc *desc = (void *)q->desc; 237 238 if (!mt76_queue_is_npu_tx(q)) 239 return; 240 241 desc[index].ctrl &= ~NPU_TX_DMA_DESC_DONE_MASK; 242 } 243 244 void mt76_npu_queue_setup(struct mt76_dev *dev, struct mt76_queue *q) 245 { 246 int qid = FIELD_GET(MT_QFLAG_WED_RING, q->flags); 247 bool xmit = mt76_queue_is_npu_tx(q); 248 struct airoha_npu *npu; 249 250 if (!mt76_queue_is_npu(q)) 251 return; 252 253 npu = rcu_dereference_protected(dev->mmio.npu, &dev->mutex); 254 if (npu) 255 q->wed_regs = airoha_npu_wlan_get_queue_addr(npu, qid, xmit); 256 } 257 258 int mt76_npu_rx_queue_init(struct mt76_dev *dev, struct mt76_queue *q) 259 { 260 int err, irq, qid = q - &dev->q_rx[0]; 261 int size, index = qid - MT_RXQ_NPU0; 262 struct airoha_npu *npu; 263 const char *name; 264 265 mutex_lock(&dev->mutex); 266 267 npu = rcu_dereference_protected(dev->mmio.npu, &dev->mutex); 268 irq = npu && index < ARRAY_SIZE(npu->irqs) ? npu->irqs[index] 269 : -EINVAL; 270 if (irq < 0) { 271 err = irq; 272 goto out; 273 } 274 275 q->flags = MT_NPU_Q_RX(index); 276 size = qid == MT_RXQ_NPU1 ? NPU_RX1_DESC_NUM : NPU_RX0_DESC_NUM; 277 err = dev->queue_ops->alloc(dev, q, 0, size, 278 MT76_NPU_RX_BUF_SIZE, 0); 279 if (err) 280 goto out; 281 282 name = devm_kasprintf(dev->dev, GFP_KERNEL, "mt76-npu.%d", index); 283 if (!name) { 284 err = -ENOMEM; 285 goto out; 286 } 287 288 err = devm_request_irq(dev->dev, irq, mt76_npu_irq_handler, 289 IRQF_SHARED, name, q); 290 if (err) 291 goto out; 292 293 netif_napi_add(dev->napi_dev, &dev->napi[qid], mt76_npu_rx_poll); 294 mt76_npu_fill_rx_queue(dev, q); 295 napi_enable(&dev->napi[qid]); 296 out: 297 mutex_unlock(&dev->mutex); 298 299 return err; 300 } 301 EXPORT_SYMBOL_GPL(mt76_npu_rx_queue_init); 302 303 static int mt76_npu_setup_tc_block_cb(enum tc_setup_type type, 304 void *type_data, void *cb_priv) 305 { 306 struct mt76_phy *phy = cb_priv; 307 struct mt76_dev *dev = phy->dev; 308 struct airoha_ppe_dev *ppe_dev; 309 int err = -EOPNOTSUPP; 310 311 if (type != TC_SETUP_CLSFLOWER) 312 return -EOPNOTSUPP; 313 314 mutex_lock(&dev->mutex); 315 316 ppe_dev = rcu_dereference_protected(dev->mmio.ppe_dev, &dev->mutex); 317 if (ppe_dev) 318 err = airoha_ppe_dev_setup_tc_block_cb(ppe_dev, type_data); 319 320 mutex_unlock(&dev->mutex); 321 322 return err; 323 } 324 325 static int mt76_npu_setup_tc_block(struct mt76_phy *phy, 326 struct net_device *dev, 327 struct flow_block_offload *f) 328 { 329 flow_setup_cb_t *cb = mt76_npu_setup_tc_block_cb; 330 static LIST_HEAD(block_cb_list); 331 struct flow_block_cb *block_cb; 332 333 if (f->binder_type != FLOW_BLOCK_BINDER_TYPE_CLSACT_INGRESS) 334 return -EOPNOTSUPP; 335 336 if (!tc_can_offload(dev)) 337 return -EOPNOTSUPP; 338 339 f->driver_block_list = &block_cb_list; 340 switch (f->command) { 341 case FLOW_BLOCK_BIND: 342 block_cb = flow_block_cb_lookup(f->block, cb, dev); 343 if (block_cb) { 344 flow_block_cb_incref(block_cb); 345 return 0; 346 } 347 348 block_cb = flow_block_cb_alloc(cb, dev, phy, NULL); 349 if (IS_ERR(block_cb)) 350 return PTR_ERR(block_cb); 351 352 flow_block_cb_incref(block_cb); 353 flow_block_cb_add(block_cb, f); 354 list_add_tail(&block_cb->driver_list, &block_cb_list); 355 return 0; 356 case FLOW_BLOCK_UNBIND: 357 block_cb = flow_block_cb_lookup(f->block, cb, dev); 358 if (!block_cb) 359 return -ENOENT; 360 361 if (!flow_block_cb_decref(block_cb)) { 362 flow_block_cb_remove(block_cb, f); 363 list_del(&block_cb->driver_list); 364 } 365 return 0; 366 default: 367 return -EOPNOTSUPP; 368 } 369 } 370 371 int mt76_npu_net_setup_tc(struct ieee80211_hw *hw, struct ieee80211_vif *vif, 372 struct net_device *dev, enum tc_setup_type type, 373 void *type_data) 374 { 375 struct mt76_phy *phy = hw->priv; 376 377 if (!tc_can_offload(dev)) 378 return -EOPNOTSUPP; 379 380 if (!mt76_npu_device_active(phy->dev)) 381 return -EOPNOTSUPP; 382 383 switch (type) { 384 case TC_SETUP_BLOCK: 385 case TC_SETUP_FT: 386 return mt76_npu_setup_tc_block(phy, dev, type_data); 387 default: 388 return -EOPNOTSUPP; 389 } 390 } 391 EXPORT_SYMBOL_GPL(mt76_npu_net_setup_tc); 392 393 void mt76_npu_disable_irqs(struct mt76_dev *dev) 394 { 395 struct airoha_npu *npu; 396 int i; 397 398 rcu_read_lock(); 399 400 npu = rcu_dereference(dev->mmio.npu); 401 if (!npu) 402 goto unlock; 403 404 for (i = MT_RXQ_NPU0; i <= MT_RXQ_NPU1; i++) { 405 int qid = i - MT_RXQ_NPU0; 406 u32 status; 407 408 status = airoha_npu_wlan_get_irq_status(npu, qid); 409 airoha_npu_wlan_set_irq_status(npu, status); 410 airoha_npu_wlan_disable_irq(npu, qid); 411 } 412 unlock: 413 rcu_read_unlock(); 414 } 415 EXPORT_SYMBOL_GPL(mt76_npu_disable_irqs); 416 417 int mt76_npu_init(struct mt76_dev *dev, phys_addr_t phy_addr, int type) 418 { 419 struct airoha_ppe_dev *ppe_dev; 420 struct airoha_npu *npu; 421 int err = 0; 422 423 /* NPU offloading is only supported by MT7992 */ 424 if (!is_mt7992(dev)) 425 return 0; 426 427 mutex_lock(&dev->mutex); 428 429 npu = airoha_npu_get(dev->dev); 430 if (IS_ERR(npu)) { 431 request_module("airoha-npu"); 432 npu = airoha_npu_get(dev->dev); 433 } 434 435 if (IS_ERR(npu)) { 436 err = PTR_ERR(npu); 437 goto error_unlock; 438 } 439 440 ppe_dev = airoha_ppe_get_dev(dev->dev); 441 if (IS_ERR(ppe_dev)) { 442 request_module("airoha-eth"); 443 ppe_dev = airoha_ppe_get_dev(dev->dev); 444 } 445 446 if (IS_ERR(ppe_dev)) { 447 err = PTR_ERR(ppe_dev); 448 goto error_npu_put; 449 } 450 451 err = airoha_npu_wlan_init_reserved_memory(npu); 452 if (err) 453 goto error_ppe_put; 454 455 dev->dma_dev = npu->dev; 456 dev->mmio.phy_addr = phy_addr; 457 dev->mmio.npu_type = type; 458 /* NPU offloading requires HW-RRO for RX packet reordering. */ 459 dev->hwrro_mode = MT76_HWRRO_V3_1; 460 461 rcu_assign_pointer(dev->mmio.npu, npu); 462 rcu_assign_pointer(dev->mmio.ppe_dev, ppe_dev); 463 synchronize_rcu(); 464 465 mutex_unlock(&dev->mutex); 466 467 return 0; 468 469 error_ppe_put: 470 airoha_ppe_put_dev(ppe_dev); 471 error_npu_put: 472 airoha_npu_put(npu); 473 error_unlock: 474 mutex_unlock(&dev->mutex); 475 476 return err; 477 } 478 EXPORT_SYMBOL_GPL(mt76_npu_init); 479 480 void mt76_npu_deinit(struct mt76_dev *dev) 481 { 482 struct airoha_ppe_dev *ppe_dev; 483 struct airoha_npu *npu; 484 485 mutex_lock(&dev->mutex); 486 487 npu = rcu_replace_pointer(dev->mmio.npu, NULL, 488 lockdep_is_held(&dev->mutex)); 489 if (npu) 490 airoha_npu_put(npu); 491 492 ppe_dev = rcu_replace_pointer(dev->mmio.ppe_dev, NULL, 493 lockdep_is_held(&dev->mutex)); 494 if (ppe_dev) 495 airoha_ppe_put_dev(ppe_dev); 496 497 mutex_unlock(&dev->mutex); 498 499 mt76_npu_queue_cleanup(dev, &dev->q_rx[MT_RXQ_NPU0]); 500 mt76_npu_queue_cleanup(dev, &dev->q_rx[MT_RXQ_NPU1]); 501 } 502