1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (c) 2025 AIROHA Inc 4 * Author: Lorenzo Bianconi <lorenzo@kernel.org> 5 */ 6 #include <linux/kernel.h> 7 #include <net/flow_offload.h> 8 #include <net/pkt_cls.h> 9 10 #include "mt76.h" 11 #include "dma.h" 12 #include "mt76_connac.h" 13 14 #define MT76_NPU_RX_BUF_SIZE (1800 + \ 15 SKB_DATA_ALIGN(sizeof(struct skb_shared_info))) 16 17 int mt76_npu_fill_rx_queue(struct mt76_dev *dev, struct mt76_queue *q) 18 { 19 int nframes = 0; 20 21 while (q->queued < q->ndesc - 1) { 22 struct airoha_npu_rx_dma_desc *desc = (void *)q->desc; 23 struct mt76_queue_entry *e = &q->entry[q->head]; 24 struct page *page; 25 int offset; 26 27 e->buf = mt76_get_page_pool_buf(q, &offset, q->buf_size); 28 if (!e->buf) 29 break; 30 31 e->dma_len[0] = SKB_WITH_OVERHEAD(q->buf_size); 32 page = virt_to_head_page(e->buf); 33 e->dma_addr[0] = page_pool_get_dma_addr(page) + offset; 34 35 memset(&desc[q->head], 0, sizeof(*desc)); 36 desc[q->head].addr = e->dma_addr[0]; 37 38 q->head = (q->head + 1) % q->ndesc; 39 q->queued++; 40 nframes++; 41 } 42 43 return nframes; 44 } 45 46 void mt76_npu_queue_cleanup(struct mt76_dev *dev, struct mt76_queue *q) 47 { 48 spin_lock_bh(&q->lock); 49 while (q->queued > 0) { 50 struct mt76_queue_entry *e = &q->entry[q->tail]; 51 52 dma_sync_single_for_cpu(dev->dma_dev, e->dma_addr[0], 53 e->dma_len[0], 54 page_pool_get_dma_dir(q->page_pool)); 55 mt76_put_page_pool_buf(e->buf, false); 56 q->tail = (q->tail + 1) % q->ndesc; 57 q->queued--; 58 } 59 spin_unlock_bh(&q->lock); 60 } 61 62 static struct sk_buff *mt76_npu_dequeue(struct mt76_dev *dev, 63 struct mt76_queue *q, 64 u32 *info) 65 { 66 struct airoha_npu_rx_dma_desc *desc = (void *)q->desc; 67 int i, nframes, index = q->tail; 68 struct sk_buff *skb = NULL; 69 70 nframes = FIELD_GET(NPU_RX_DMA_PKT_COUNT_MASK, desc[index].info); 71 nframes = max_t(int, nframes, 1); 72 73 for (i = 0; i < nframes; i++) { 74 struct mt76_queue_entry *e = &q->entry[index]; 75 int len = FIELD_GET(NPU_RX_DMA_DESC_CUR_LEN_MASK, 76 desc[index].ctrl); 77 78 if (!FIELD_GET(NPU_RX_DMA_DESC_DONE_MASK, desc[index].ctrl)) { 79 dev_kfree_skb(skb); 80 return NULL; 81 } 82 83 dma_sync_single_for_cpu(dev->dma_dev, e->dma_addr[0], 84 e->dma_len[0], 85 page_pool_get_dma_dir(q->page_pool)); 86 87 if (!skb) { 88 skb = napi_build_skb(e->buf, q->buf_size); 89 if (!skb) 90 return NULL; 91 92 __skb_put(skb, len); 93 skb_reset_mac_header(skb); 94 skb_mark_for_recycle(skb); 95 } else { 96 struct skb_shared_info *shinfo = skb_shinfo(skb); 97 struct page *page = virt_to_head_page(e->buf); 98 int nr_frags = shinfo->nr_frags; 99 100 if (nr_frags < ARRAY_SIZE(shinfo->frags)) 101 skb_add_rx_frag(skb, nr_frags, page, 102 e->buf - page_address(page), 103 len, q->buf_size); 104 } 105 106 *info = desc[index].info; 107 index = (index + 1) % q->ndesc; 108 } 109 q->tail = index; 110 q->queued -= i; 111 Q_WRITE(q, dma_idx, q->tail); 112 113 return skb; 114 } 115 116 void mt76_npu_check_ppe(struct mt76_dev *dev, struct sk_buff *skb, 117 u32 info) 118 { 119 struct airoha_ppe_dev *ppe_dev; 120 u16 reason, hash; 121 122 if (!mt76_npu_device_active(dev)) 123 return; 124 125 rcu_read_lock(); 126 127 ppe_dev = rcu_dereference(dev->mmio.ppe_dev); 128 if (!ppe_dev) 129 goto out; 130 131 hash = FIELD_GET(NPU_RX_DMA_FOE_ID_MASK, info); 132 skb_set_hash(skb, hash, PKT_HASH_TYPE_L4); 133 134 reason = FIELD_GET(NPU_RX_DMA_CRSN_MASK, info); 135 if (reason == PPE_CPU_REASON_HIT_UNBIND_RATE_REACHED) { 136 skb_set_mac_header(skb, 0); 137 airoha_ppe_dev_check_skb(ppe_dev, skb, hash, true); 138 } 139 out: 140 rcu_read_unlock(); 141 } 142 EXPORT_SYMBOL_GPL(mt76_npu_check_ppe); 143 144 static int mt76_npu_rx_poll(struct napi_struct *napi, int budget) 145 { 146 struct mt76_dev *dev = mt76_priv(napi->dev); 147 enum mt76_rxq_id qid = napi - dev->napi; 148 struct airoha_npu *npu; 149 int done = 0; 150 151 rcu_read_lock(); 152 153 npu = rcu_dereference(dev->mmio.npu); 154 if (!npu) 155 goto out; 156 157 while (done < budget) { 158 struct sk_buff *skb; 159 u32 info = 0; 160 161 skb = mt76_npu_dequeue(dev, &dev->q_rx[qid], &info); 162 if (!skb) 163 break; 164 165 dev->drv->rx_skb(dev, qid, skb, &info); 166 mt76_rx_poll_complete(dev, qid, napi); 167 done++; 168 } 169 170 mt76_npu_fill_rx_queue(dev, &dev->q_rx[qid]); 171 out: 172 if (done < budget && napi_complete(napi)) 173 dev->drv->rx_poll_complete(dev, qid); 174 175 rcu_read_unlock(); 176 177 return done; 178 } 179 180 static irqreturn_t mt76_npu_irq_handler(int irq, void *q_instance) 181 { 182 struct mt76_queue *q = q_instance; 183 struct mt76_dev *dev = q->dev; 184 int qid = q - &dev->q_rx[0]; 185 int index = qid - MT_RXQ_NPU0; 186 struct airoha_npu *npu; 187 u32 status; 188 189 rcu_read_lock(); 190 191 npu = rcu_dereference(dev->mmio.npu); 192 if (!npu) 193 goto out; 194 195 status = airoha_npu_wlan_get_irq_status(npu, index); 196 airoha_npu_wlan_set_irq_status(npu, status); 197 198 airoha_npu_wlan_disable_irq(npu, index); 199 napi_schedule(&dev->napi[qid]); 200 out: 201 rcu_read_unlock(); 202 203 return IRQ_HANDLED; 204 } 205 206 int mt76_npu_dma_add_buf(struct mt76_phy *phy, struct mt76_queue *q, 207 struct sk_buff *skb, struct mt76_queue_buf *buf, 208 void *txwi_ptr) 209 { 210 u16 txwi_len = min_t(u16, phy->dev->drv->txwi_size, NPU_TXWI_LEN); 211 struct airoha_npu_tx_dma_desc *desc = (void *)q->desc; 212 int ret; 213 214 /* TODO: Take into account unlinear skbs */ 215 memcpy(desc[q->head].txwi, txwi_ptr, txwi_len); 216 desc[q->head].addr = buf->addr; 217 desc[q->head].ctrl = FIELD_PREP(NPU_TX_DMA_DESC_VEND_LEN_MASK, txwi_len) | 218 FIELD_PREP(NPU_TX_DMA_DESC_LEN_MASK, skb->len) | 219 NPU_TX_DMA_DESC_DONE_MASK; 220 221 ret = q->head; 222 q->entry[q->head].skip_buf0 = true; 223 q->entry[q->head].skip_buf1 = true; 224 q->entry[q->head].txwi = NULL; 225 q->entry[q->head].skb = NULL; 226 q->entry[q->head].wcid = 0xffff; 227 228 q->head = (q->head + 1) % q->ndesc; 229 q->queued++; 230 231 return ret; 232 } 233 234 void mt76_npu_txdesc_cleanup(struct mt76_queue *q, int index) 235 { 236 struct airoha_npu_tx_dma_desc *desc = (void *)q->desc; 237 238 if (!mt76_queue_is_npu_tx(q)) 239 return; 240 241 desc[index].ctrl &= ~NPU_TX_DMA_DESC_DONE_MASK; 242 } 243 244 void mt76_npu_queue_setup(struct mt76_dev *dev, struct mt76_queue *q) 245 { 246 int qid = FIELD_GET(MT_QFLAG_WED_RING, q->flags); 247 bool xmit = mt76_queue_is_npu_tx(q); 248 struct airoha_npu *npu; 249 250 if (!mt76_queue_is_npu(q)) 251 return; 252 253 npu = rcu_dereference_protected(dev->mmio.npu, &dev->mutex); 254 if (npu) 255 q->wed_regs = airoha_npu_wlan_get_queue_addr(npu, qid, xmit); 256 } 257 258 int mt76_npu_rx_queue_init(struct mt76_dev *dev, struct mt76_queue *q) 259 { 260 int err, irq, qid = q - &dev->q_rx[0]; 261 int size, index = qid - MT_RXQ_NPU0; 262 struct airoha_npu *npu; 263 const char *name; 264 265 mutex_lock(&dev->mutex); 266 267 npu = rcu_dereference_protected(dev->mmio.npu, &dev->mutex); 268 irq = npu && index < ARRAY_SIZE(npu->irqs) ? npu->irqs[index] 269 : -EINVAL; 270 if (irq < 0) { 271 err = irq; 272 goto out; 273 } 274 275 q->flags = MT_NPU_Q_RX(index); 276 size = qid == MT_RXQ_NPU1 ? NPU_RX1_DESC_NUM : NPU_RX0_DESC_NUM; 277 err = dev->queue_ops->alloc(dev, q, 0, size, 278 MT76_NPU_RX_BUF_SIZE, 0); 279 if (err) 280 goto out; 281 282 name = devm_kasprintf(dev->dev, GFP_KERNEL, "mt76-npu.%d", index); 283 if (!name) { 284 err = -ENOMEM; 285 goto out; 286 } 287 288 err = devm_request_irq(dev->dev, irq, mt76_npu_irq_handler, 289 IRQF_SHARED, name, q); 290 if (err) 291 goto out; 292 293 netif_napi_add(dev->napi_dev, &dev->napi[qid], mt76_npu_rx_poll); 294 mt76_npu_fill_rx_queue(dev, q); 295 napi_enable(&dev->napi[qid]); 296 out: 297 mutex_unlock(&dev->mutex); 298 299 return err; 300 } 301 EXPORT_SYMBOL_GPL(mt76_npu_rx_queue_init); 302 303 static int mt76_npu_setup_tc_block_cb(enum tc_setup_type type, 304 void *type_data, void *cb_priv) 305 { 306 struct mt76_phy *phy = cb_priv; 307 struct mt76_dev *dev = phy->dev; 308 struct airoha_ppe_dev *ppe_dev; 309 int err = -EOPNOTSUPP; 310 311 if (type != TC_SETUP_CLSFLOWER) 312 return -EOPNOTSUPP; 313 314 mutex_lock(&dev->mutex); 315 316 ppe_dev = rcu_dereference_protected(dev->mmio.ppe_dev, &dev->mutex); 317 if (ppe_dev) 318 err = airoha_ppe_dev_setup_tc_block_cb(ppe_dev, type_data); 319 320 mutex_unlock(&dev->mutex); 321 322 return err; 323 } 324 325 static int mt76_npu_setup_tc_block(struct mt76_phy *phy, 326 struct net_device *dev, 327 struct flow_block_offload *f) 328 { 329 flow_setup_cb_t *cb = mt76_npu_setup_tc_block_cb; 330 static LIST_HEAD(block_cb_list); 331 struct flow_block_cb *block_cb; 332 333 if (f->binder_type != FLOW_BLOCK_BINDER_TYPE_CLSACT_INGRESS) 334 return -EOPNOTSUPP; 335 336 if (!tc_can_offload(dev)) 337 return -EOPNOTSUPP; 338 339 f->driver_block_list = &block_cb_list; 340 switch (f->command) { 341 case FLOW_BLOCK_BIND: 342 block_cb = flow_block_cb_lookup(f->block, cb, dev); 343 if (block_cb) { 344 flow_block_cb_incref(block_cb); 345 return 0; 346 } 347 348 block_cb = flow_block_cb_alloc(cb, dev, phy, NULL); 349 if (IS_ERR(block_cb)) 350 return PTR_ERR(block_cb); 351 352 flow_block_cb_incref(block_cb); 353 flow_block_cb_add(block_cb, f); 354 list_add_tail(&block_cb->driver_list, &block_cb_list); 355 return 0; 356 case FLOW_BLOCK_UNBIND: 357 block_cb = flow_block_cb_lookup(f->block, cb, dev); 358 if (!block_cb) 359 return -ENOENT; 360 361 if (!flow_block_cb_decref(block_cb)) { 362 flow_block_cb_remove(block_cb, f); 363 list_del(&block_cb->driver_list); 364 } 365 return 0; 366 default: 367 return -EOPNOTSUPP; 368 } 369 } 370 371 int mt76_npu_net_setup_tc(struct ieee80211_hw *hw, struct ieee80211_vif *vif, 372 struct net_device *dev, enum tc_setup_type type, 373 void *type_data) 374 { 375 struct mt76_phy *phy = hw->priv; 376 377 if (!tc_can_offload(dev)) 378 return -EOPNOTSUPP; 379 380 if (!mt76_npu_device_active(phy->dev)) 381 return -EOPNOTSUPP; 382 383 switch (type) { 384 case TC_SETUP_BLOCK: 385 case TC_SETUP_FT: 386 return mt76_npu_setup_tc_block(phy, dev, type_data); 387 default: 388 return -EOPNOTSUPP; 389 } 390 } 391 EXPORT_SYMBOL_GPL(mt76_npu_net_setup_tc); 392 393 int mt76_npu_send_txrx_addr(struct mt76_dev *dev, int ifindex, 394 u32 direction, u32 i_count_addr, 395 u32 o_status_addr, u32 o_count_addr) 396 { 397 struct { 398 __le32 dir; 399 __le32 in_count_addr; 400 __le32 out_status_addr; 401 __le32 out_count_addr; 402 } info = { 403 .dir = cpu_to_le32(direction), 404 .in_count_addr = cpu_to_le32(i_count_addr), 405 .out_status_addr = cpu_to_le32(o_status_addr), 406 .out_count_addr = cpu_to_le32(o_count_addr), 407 }; 408 struct airoha_npu *npu; 409 int err = -ENODEV; 410 411 rcu_read_lock(); 412 npu = rcu_dereference(dev->mmio.npu); 413 if (npu) 414 err = airoha_npu_wlan_send_msg(npu, ifindex, 415 WLAN_FUNC_SET_WAIT_INODE_TXRX_REG_ADDR, 416 &info, sizeof(info), GFP_ATOMIC); 417 rcu_read_unlock(); 418 419 return err; 420 } 421 EXPORT_SYMBOL_GPL(mt76_npu_send_txrx_addr); 422 423 void mt76_npu_disable_irqs(struct mt76_dev *dev) 424 { 425 struct airoha_npu *npu; 426 int i; 427 428 rcu_read_lock(); 429 430 npu = rcu_dereference(dev->mmio.npu); 431 if (!npu) 432 goto unlock; 433 434 for (i = MT_RXQ_NPU0; i <= MT_RXQ_NPU1; i++) { 435 int qid = i - MT_RXQ_NPU0; 436 u32 status; 437 438 status = airoha_npu_wlan_get_irq_status(npu, qid); 439 airoha_npu_wlan_set_irq_status(npu, status); 440 airoha_npu_wlan_disable_irq(npu, qid); 441 } 442 unlock: 443 rcu_read_unlock(); 444 } 445 EXPORT_SYMBOL_GPL(mt76_npu_disable_irqs); 446 447 int mt76_npu_init(struct mt76_dev *dev, phys_addr_t phy_addr, int type) 448 { 449 struct airoha_ppe_dev *ppe_dev; 450 struct airoha_npu *npu; 451 int err = 0; 452 453 mutex_lock(&dev->mutex); 454 455 npu = airoha_npu_get(dev->dev); 456 if (IS_ERR(npu)) { 457 request_module("airoha-npu"); 458 npu = airoha_npu_get(dev->dev); 459 } 460 461 if (IS_ERR(npu)) { 462 err = PTR_ERR(npu); 463 goto error_unlock; 464 } 465 466 ppe_dev = airoha_ppe_get_dev(dev->dev); 467 if (IS_ERR(ppe_dev)) { 468 request_module("airoha-eth"); 469 ppe_dev = airoha_ppe_get_dev(dev->dev); 470 } 471 472 if (IS_ERR(ppe_dev)) { 473 err = PTR_ERR(ppe_dev); 474 goto error_npu_put; 475 } 476 477 err = airoha_npu_wlan_init_reserved_memory(npu); 478 if (err) 479 goto error_ppe_put; 480 481 dev->dma_dev = npu->dev; 482 dev->mmio.phy_addr = phy_addr; 483 dev->mmio.npu_type = type; 484 /* NPU offloading requires HW-RRO for RX packet reordering. */ 485 dev->hwrro_mode = is_mt7996(dev) ? MT76_HWRRO_V3 : MT76_HWRRO_V3_1; 486 dev->rx_token_size = 32768; 487 488 rcu_assign_pointer(dev->mmio.npu, npu); 489 rcu_assign_pointer(dev->mmio.ppe_dev, ppe_dev); 490 synchronize_rcu(); 491 492 mutex_unlock(&dev->mutex); 493 494 return 0; 495 496 error_ppe_put: 497 airoha_ppe_put_dev(ppe_dev); 498 error_npu_put: 499 airoha_npu_put(npu); 500 error_unlock: 501 mutex_unlock(&dev->mutex); 502 503 return err; 504 } 505 EXPORT_SYMBOL_GPL(mt76_npu_init); 506 507 void mt76_npu_deinit(struct mt76_dev *dev) 508 { 509 struct airoha_ppe_dev *ppe_dev; 510 struct airoha_npu *npu; 511 512 mutex_lock(&dev->mutex); 513 514 npu = rcu_replace_pointer(dev->mmio.npu, NULL, 515 lockdep_is_held(&dev->mutex)); 516 if (npu) 517 airoha_npu_put(npu); 518 519 ppe_dev = rcu_replace_pointer(dev->mmio.ppe_dev, NULL, 520 lockdep_is_held(&dev->mutex)); 521 if (ppe_dev) 522 airoha_ppe_put_dev(ppe_dev); 523 524 mutex_unlock(&dev->mutex); 525 526 mt76_npu_queue_cleanup(dev, &dev->q_rx[MT_RXQ_NPU0]); 527 mt76_npu_queue_cleanup(dev, &dev->q_rx[MT_RXQ_NPU1]); 528 } 529