1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (c) 2025 AIROHA Inc
4 * Author: Lorenzo Bianconi <lorenzo@kernel.org>
5 */
6
7 #include <linux/devcoredump.h>
8 #include <linux/firmware.h>
9 #include <linux/platform_device.h>
10 #include <linux/of_net.h>
11 #include <linux/of_platform.h>
12 #include <linux/of_reserved_mem.h>
13 #include <linux/regmap.h>
14
15 #include "airoha_eth.h"
16 #include "airoha_npu.h"
17
18 #define NPU_EN7581_FIRMWARE_DATA "airoha/en7581_npu_data.bin"
19 #define NPU_EN7581_FIRMWARE_RV32 "airoha/en7581_npu_rv32.bin"
20 #define NPU_EN7581_FIRMWARE_RV32_MAX_SIZE 0x200000
21 #define NPU_EN7581_FIRMWARE_DATA_MAX_SIZE 0x10000
22 #define NPU_DUMP_SIZE 512
23
24 #define REG_NPU_LOCAL_SRAM 0x0
25
26 #define NPU_PC_BASE_ADDR 0x305000
27 #define REG_PC_DBG(_n) (0x305000 + ((_n) * 0x100))
28
29 #define NPU_CLUSTER_BASE_ADDR 0x306000
30
31 #define REG_CR_BOOT_TRIGGER (NPU_CLUSTER_BASE_ADDR + 0x000)
32 #define REG_CR_BOOT_CONFIG (NPU_CLUSTER_BASE_ADDR + 0x004)
33 #define REG_CR_BOOT_BASE(_n) (NPU_CLUSTER_BASE_ADDR + 0x020 + ((_n) << 2))
34
35 #define NPU_MBOX_BASE_ADDR 0x30c000
36
37 #define REG_CR_MBOX_INT_STATUS (NPU_MBOX_BASE_ADDR + 0x000)
38 #define MBOX_INT_STATUS_MASK BIT(8)
39
40 #define REG_CR_MBOX_INT_MASK(_n) (NPU_MBOX_BASE_ADDR + 0x004 + ((_n) << 2))
41 #define REG_CR_MBQ0_CTRL(_n) (NPU_MBOX_BASE_ADDR + 0x030 + ((_n) << 2))
42 #define REG_CR_MBQ8_CTRL(_n) (NPU_MBOX_BASE_ADDR + 0x0b0 + ((_n) << 2))
43 #define REG_CR_NPU_MIB(_n) (NPU_MBOX_BASE_ADDR + 0x140 + ((_n) << 2))
44
45 #define NPU_TIMER_BASE_ADDR 0x310100
46 #define REG_WDT_TIMER_CTRL(_n) (NPU_TIMER_BASE_ADDR + ((_n) * 0x100))
47 #define WDT_EN_MASK BIT(25)
48 #define WDT_INTR_MASK BIT(21)
49
50 enum {
51 NPU_OP_SET = 1,
52 NPU_OP_SET_NO_WAIT,
53 NPU_OP_GET,
54 NPU_OP_GET_NO_WAIT,
55 };
56
57 enum {
58 NPU_FUNC_WIFI,
59 NPU_FUNC_TUNNEL,
60 NPU_FUNC_NOTIFY,
61 NPU_FUNC_DBA,
62 NPU_FUNC_TR471,
63 NPU_FUNC_PPE,
64 };
65
66 enum {
67 NPU_MBOX_ERROR,
68 NPU_MBOX_SUCCESS,
69 };
70
71 enum {
72 PPE_FUNC_SET_WAIT,
73 PPE_FUNC_SET_WAIT_HWNAT_INIT,
74 PPE_FUNC_SET_WAIT_HWNAT_DEINIT,
75 PPE_FUNC_SET_WAIT_API,
76 PPE_FUNC_SET_WAIT_FLOW_STATS_SETUP,
77 };
78
79 enum {
80 PPE2_SRAM_SET_ENTRY,
81 PPE_SRAM_SET_ENTRY,
82 PPE_SRAM_SET_VAL,
83 PPE_SRAM_RESET_VAL,
84 };
85
86 enum {
87 QDMA_WAN_ETHER = 1,
88 QDMA_WAN_PON_XDSL,
89 };
90
91 #define MBOX_MSG_FUNC_ID GENMASK(14, 11)
92 #define MBOX_MSG_STATIC_BUF BIT(5)
93 #define MBOX_MSG_STATUS GENMASK(4, 2)
94 #define MBOX_MSG_DONE BIT(1)
95 #define MBOX_MSG_WAIT_RSP BIT(0)
96
97 #define PPE_TYPE_L2B_IPV4 2
98 #define PPE_TYPE_L2B_IPV4_IPV6 3
99
100 struct ppe_mbox_data {
101 u32 func_type;
102 u32 func_id;
103 union {
104 struct {
105 u8 cds;
106 u8 xpon_hal_api;
107 u8 wan_xsi;
108 u8 ct_joyme4;
109 u8 max_packet;
110 u8 rsv[3];
111 u32 ppe_type;
112 u32 wan_mode;
113 u32 wan_sel;
114 } init_info;
115 struct {
116 u32 func_id;
117 u32 size;
118 u32 data;
119 } set_info;
120 struct {
121 u32 npu_stats_addr;
122 u32 foe_stats_addr;
123 } stats_info;
124 };
125 };
126
airoha_npu_send_msg(struct airoha_npu * npu,int func_id,void * p,int size)127 static int airoha_npu_send_msg(struct airoha_npu *npu, int func_id,
128 void *p, int size)
129 {
130 u16 core = 0; /* FIXME */
131 u32 val, offset = core << 4;
132 dma_addr_t dma_addr;
133 int ret;
134
135 dma_addr = dma_map_single(npu->dev, p, size, DMA_TO_DEVICE);
136 ret = dma_mapping_error(npu->dev, dma_addr);
137 if (ret)
138 return ret;
139
140 spin_lock_bh(&npu->cores[core].lock);
141
142 regmap_write(npu->regmap, REG_CR_MBQ0_CTRL(0) + offset, dma_addr);
143 regmap_write(npu->regmap, REG_CR_MBQ0_CTRL(1) + offset, size);
144 regmap_read(npu->regmap, REG_CR_MBQ0_CTRL(2) + offset, &val);
145 regmap_write(npu->regmap, REG_CR_MBQ0_CTRL(2) + offset, val + 1);
146 val = FIELD_PREP(MBOX_MSG_FUNC_ID, func_id) | MBOX_MSG_WAIT_RSP;
147 regmap_write(npu->regmap, REG_CR_MBQ0_CTRL(3) + offset, val);
148
149 ret = regmap_read_poll_timeout_atomic(npu->regmap,
150 REG_CR_MBQ0_CTRL(3) + offset,
151 val, (val & MBOX_MSG_DONE),
152 100, 100 * MSEC_PER_SEC);
153 if (!ret && FIELD_GET(MBOX_MSG_STATUS, val) != NPU_MBOX_SUCCESS)
154 ret = -EINVAL;
155
156 spin_unlock_bh(&npu->cores[core].lock);
157
158 dma_unmap_single(npu->dev, dma_addr, size, DMA_TO_DEVICE);
159
160 return ret;
161 }
162
airoha_npu_run_firmware(struct device * dev,void __iomem * base,struct reserved_mem * rmem)163 static int airoha_npu_run_firmware(struct device *dev, void __iomem *base,
164 struct reserved_mem *rmem)
165 {
166 const struct firmware *fw;
167 void __iomem *addr;
168 int ret;
169
170 ret = request_firmware(&fw, NPU_EN7581_FIRMWARE_RV32, dev);
171 if (ret)
172 return ret == -ENOENT ? -EPROBE_DEFER : ret;
173
174 if (fw->size > NPU_EN7581_FIRMWARE_RV32_MAX_SIZE) {
175 dev_err(dev, "%s: fw size too overlimit (%zu)\n",
176 NPU_EN7581_FIRMWARE_RV32, fw->size);
177 ret = -E2BIG;
178 goto out;
179 }
180
181 addr = devm_ioremap(dev, rmem->base, rmem->size);
182 if (!addr) {
183 ret = -ENOMEM;
184 goto out;
185 }
186
187 memcpy_toio(addr, fw->data, fw->size);
188 release_firmware(fw);
189
190 ret = request_firmware(&fw, NPU_EN7581_FIRMWARE_DATA, dev);
191 if (ret)
192 return ret == -ENOENT ? -EPROBE_DEFER : ret;
193
194 if (fw->size > NPU_EN7581_FIRMWARE_DATA_MAX_SIZE) {
195 dev_err(dev, "%s: fw size too overlimit (%zu)\n",
196 NPU_EN7581_FIRMWARE_DATA, fw->size);
197 ret = -E2BIG;
198 goto out;
199 }
200
201 memcpy_toio(base + REG_NPU_LOCAL_SRAM, fw->data, fw->size);
202 out:
203 release_firmware(fw);
204
205 return ret;
206 }
207
airoha_npu_mbox_handler(int irq,void * npu_instance)208 static irqreturn_t airoha_npu_mbox_handler(int irq, void *npu_instance)
209 {
210 struct airoha_npu *npu = npu_instance;
211
212 /* clear mbox interrupt status */
213 regmap_write(npu->regmap, REG_CR_MBOX_INT_STATUS,
214 MBOX_INT_STATUS_MASK);
215
216 /* acknowledge npu */
217 regmap_update_bits(npu->regmap, REG_CR_MBQ8_CTRL(3),
218 MBOX_MSG_STATUS | MBOX_MSG_DONE, MBOX_MSG_DONE);
219
220 return IRQ_HANDLED;
221 }
222
airoha_npu_wdt_work(struct work_struct * work)223 static void airoha_npu_wdt_work(struct work_struct *work)
224 {
225 struct airoha_npu_core *core;
226 struct airoha_npu *npu;
227 void *dump;
228 u32 val[3];
229 int c;
230
231 core = container_of(work, struct airoha_npu_core, wdt_work);
232 npu = core->npu;
233
234 dump = vzalloc(NPU_DUMP_SIZE);
235 if (!dump)
236 return;
237
238 c = core - &npu->cores[0];
239 regmap_bulk_read(npu->regmap, REG_PC_DBG(c), val, ARRAY_SIZE(val));
240 snprintf(dump, NPU_DUMP_SIZE, "PC: %08x SP: %08x LR: %08x\n",
241 val[0], val[1], val[2]);
242
243 dev_coredumpv(npu->dev, dump, NPU_DUMP_SIZE, GFP_KERNEL);
244 }
245
airoha_npu_wdt_handler(int irq,void * core_instance)246 static irqreturn_t airoha_npu_wdt_handler(int irq, void *core_instance)
247 {
248 struct airoha_npu_core *core = core_instance;
249 struct airoha_npu *npu = core->npu;
250 int c = core - &npu->cores[0];
251 u32 val;
252
253 regmap_set_bits(npu->regmap, REG_WDT_TIMER_CTRL(c), WDT_INTR_MASK);
254 if (!regmap_read(npu->regmap, REG_WDT_TIMER_CTRL(c), &val) &&
255 FIELD_GET(WDT_EN_MASK, val))
256 schedule_work(&core->wdt_work);
257
258 return IRQ_HANDLED;
259 }
260
airoha_npu_ppe_init(struct airoha_npu * npu)261 static int airoha_npu_ppe_init(struct airoha_npu *npu)
262 {
263 struct ppe_mbox_data *ppe_data;
264 int err;
265
266 ppe_data = kzalloc(sizeof(*ppe_data), GFP_KERNEL);
267 if (!ppe_data)
268 return -ENOMEM;
269
270 ppe_data->func_type = NPU_OP_SET;
271 ppe_data->func_id = PPE_FUNC_SET_WAIT_HWNAT_INIT;
272 ppe_data->init_info.ppe_type = PPE_TYPE_L2B_IPV4_IPV6;
273 ppe_data->init_info.wan_mode = QDMA_WAN_ETHER;
274
275 err = airoha_npu_send_msg(npu, NPU_FUNC_PPE, ppe_data,
276 sizeof(*ppe_data));
277 kfree(ppe_data);
278
279 return err;
280 }
281
airoha_npu_ppe_deinit(struct airoha_npu * npu)282 static int airoha_npu_ppe_deinit(struct airoha_npu *npu)
283 {
284 struct ppe_mbox_data *ppe_data;
285 int err;
286
287 ppe_data = kzalloc(sizeof(*ppe_data), GFP_KERNEL);
288 if (!ppe_data)
289 return -ENOMEM;
290
291 ppe_data->func_type = NPU_OP_SET;
292 ppe_data->func_id = PPE_FUNC_SET_WAIT_HWNAT_DEINIT;
293
294 err = airoha_npu_send_msg(npu, NPU_FUNC_PPE, ppe_data,
295 sizeof(*ppe_data));
296 kfree(ppe_data);
297
298 return err;
299 }
300
airoha_npu_ppe_flush_sram_entries(struct airoha_npu * npu,dma_addr_t foe_addr,int sram_num_entries)301 static int airoha_npu_ppe_flush_sram_entries(struct airoha_npu *npu,
302 dma_addr_t foe_addr,
303 int sram_num_entries)
304 {
305 struct ppe_mbox_data *ppe_data;
306 int err;
307
308 ppe_data = kzalloc(sizeof(*ppe_data), GFP_KERNEL);
309 if (!ppe_data)
310 return -ENOMEM;
311
312 ppe_data->func_type = NPU_OP_SET;
313 ppe_data->func_id = PPE_FUNC_SET_WAIT_API;
314 ppe_data->set_info.func_id = PPE_SRAM_RESET_VAL;
315 ppe_data->set_info.data = foe_addr;
316 ppe_data->set_info.size = sram_num_entries;
317
318 err = airoha_npu_send_msg(npu, NPU_FUNC_PPE, ppe_data,
319 sizeof(*ppe_data));
320 kfree(ppe_data);
321
322 return err;
323 }
324
airoha_npu_foe_commit_entry(struct airoha_npu * npu,dma_addr_t foe_addr,u32 entry_size,u32 hash,bool ppe2)325 static int airoha_npu_foe_commit_entry(struct airoha_npu *npu,
326 dma_addr_t foe_addr,
327 u32 entry_size, u32 hash, bool ppe2)
328 {
329 struct ppe_mbox_data *ppe_data;
330 int err;
331
332 ppe_data = kzalloc(sizeof(*ppe_data), GFP_ATOMIC);
333 if (!ppe_data)
334 return -ENOMEM;
335
336 ppe_data->func_type = NPU_OP_SET;
337 ppe_data->func_id = PPE_FUNC_SET_WAIT_API;
338 ppe_data->set_info.data = foe_addr;
339 ppe_data->set_info.size = entry_size;
340 ppe_data->set_info.func_id = ppe2 ? PPE2_SRAM_SET_ENTRY
341 : PPE_SRAM_SET_ENTRY;
342
343 err = airoha_npu_send_msg(npu, NPU_FUNC_PPE, ppe_data,
344 sizeof(*ppe_data));
345 if (err)
346 goto out;
347
348 ppe_data->set_info.func_id = PPE_SRAM_SET_VAL;
349 ppe_data->set_info.data = hash;
350 ppe_data->set_info.size = sizeof(u32);
351
352 err = airoha_npu_send_msg(npu, NPU_FUNC_PPE, ppe_data,
353 sizeof(*ppe_data));
354 out:
355 kfree(ppe_data);
356
357 return err;
358 }
359
airoha_npu_stats_setup(struct airoha_npu * npu,dma_addr_t foe_stats_addr)360 static int airoha_npu_stats_setup(struct airoha_npu *npu,
361 dma_addr_t foe_stats_addr)
362 {
363 int err, size = PPE_STATS_NUM_ENTRIES * sizeof(*npu->stats);
364 struct ppe_mbox_data *ppe_data;
365
366 if (!size) /* flow stats are disabled */
367 return 0;
368
369 ppe_data = kzalloc(sizeof(*ppe_data), GFP_ATOMIC);
370 if (!ppe_data)
371 return -ENOMEM;
372
373 ppe_data->func_type = NPU_OP_SET;
374 ppe_data->func_id = PPE_FUNC_SET_WAIT_FLOW_STATS_SETUP;
375 ppe_data->stats_info.foe_stats_addr = foe_stats_addr;
376
377 err = airoha_npu_send_msg(npu, NPU_FUNC_PPE, ppe_data,
378 sizeof(*ppe_data));
379 if (err)
380 goto out;
381
382 npu->stats = devm_ioremap(npu->dev,
383 ppe_data->stats_info.npu_stats_addr,
384 size);
385 if (!npu->stats)
386 err = -ENOMEM;
387 out:
388 kfree(ppe_data);
389
390 return err;
391 }
392
airoha_npu_get(struct device * dev,dma_addr_t * stats_addr)393 struct airoha_npu *airoha_npu_get(struct device *dev, dma_addr_t *stats_addr)
394 {
395 struct platform_device *pdev;
396 struct device_node *np;
397 struct airoha_npu *npu;
398
399 np = of_parse_phandle(dev->of_node, "airoha,npu", 0);
400 if (!np)
401 return ERR_PTR(-ENODEV);
402
403 pdev = of_find_device_by_node(np);
404
405 if (!pdev) {
406 dev_err(dev, "cannot find device node %s\n", np->name);
407 of_node_put(np);
408 return ERR_PTR(-ENODEV);
409 }
410 of_node_put(np);
411
412 if (!try_module_get(THIS_MODULE)) {
413 dev_err(dev, "failed to get the device driver module\n");
414 npu = ERR_PTR(-ENODEV);
415 goto error_pdev_put;
416 }
417
418 npu = platform_get_drvdata(pdev);
419 if (!npu) {
420 npu = ERR_PTR(-ENODEV);
421 goto error_module_put;
422 }
423
424 if (!device_link_add(dev, &pdev->dev, DL_FLAG_AUTOREMOVE_SUPPLIER)) {
425 dev_err(&pdev->dev,
426 "failed to create device link to consumer %s\n",
427 dev_name(dev));
428 npu = ERR_PTR(-EINVAL);
429 goto error_module_put;
430 }
431
432 if (stats_addr) {
433 int err;
434
435 err = airoha_npu_stats_setup(npu, *stats_addr);
436 if (err) {
437 dev_err(dev, "failed to allocate npu stats buffer\n");
438 npu = ERR_PTR(err);
439 goto error_module_put;
440 }
441 }
442
443 return npu;
444
445 error_module_put:
446 module_put(THIS_MODULE);
447 error_pdev_put:
448 platform_device_put(pdev);
449
450 return npu;
451 }
452 EXPORT_SYMBOL_GPL(airoha_npu_get);
453
airoha_npu_put(struct airoha_npu * npu)454 void airoha_npu_put(struct airoha_npu *npu)
455 {
456 module_put(THIS_MODULE);
457 put_device(npu->dev);
458 }
459 EXPORT_SYMBOL_GPL(airoha_npu_put);
460
461 static const struct of_device_id of_airoha_npu_match[] = {
462 { .compatible = "airoha,en7581-npu" },
463 { /* sentinel */ }
464 };
465 MODULE_DEVICE_TABLE(of, of_airoha_npu_match);
466
467 static const struct regmap_config regmap_config = {
468 .name = "npu",
469 .reg_bits = 32,
470 .val_bits = 32,
471 .reg_stride = 4,
472 .disable_locking = true,
473 };
474
airoha_npu_probe(struct platform_device * pdev)475 static int airoha_npu_probe(struct platform_device *pdev)
476 {
477 struct device *dev = &pdev->dev;
478 struct reserved_mem *rmem;
479 struct airoha_npu *npu;
480 struct device_node *np;
481 void __iomem *base;
482 int i, irq, err;
483
484 base = devm_platform_ioremap_resource(pdev, 0);
485 if (IS_ERR(base))
486 return PTR_ERR(base);
487
488 npu = devm_kzalloc(dev, sizeof(*npu), GFP_KERNEL);
489 if (!npu)
490 return -ENOMEM;
491
492 npu->dev = dev;
493 npu->ops.ppe_init = airoha_npu_ppe_init;
494 npu->ops.ppe_deinit = airoha_npu_ppe_deinit;
495 npu->ops.ppe_flush_sram_entries = airoha_npu_ppe_flush_sram_entries;
496 npu->ops.ppe_foe_commit_entry = airoha_npu_foe_commit_entry;
497
498 npu->regmap = devm_regmap_init_mmio(dev, base, ®map_config);
499 if (IS_ERR(npu->regmap))
500 return PTR_ERR(npu->regmap);
501
502 np = of_parse_phandle(dev->of_node, "memory-region", 0);
503 if (!np)
504 return -ENODEV;
505
506 rmem = of_reserved_mem_lookup(np);
507 of_node_put(np);
508
509 if (!rmem)
510 return -ENODEV;
511
512 irq = platform_get_irq(pdev, 0);
513 if (irq < 0)
514 return irq;
515
516 err = devm_request_irq(dev, irq, airoha_npu_mbox_handler,
517 IRQF_SHARED, "airoha-npu-mbox", npu);
518 if (err)
519 return err;
520
521 for (i = 0; i < ARRAY_SIZE(npu->cores); i++) {
522 struct airoha_npu_core *core = &npu->cores[i];
523
524 spin_lock_init(&core->lock);
525 core->npu = npu;
526
527 irq = platform_get_irq(pdev, i + 1);
528 if (irq < 0)
529 return irq;
530
531 err = devm_request_irq(dev, irq, airoha_npu_wdt_handler,
532 IRQF_SHARED, "airoha-npu-wdt", core);
533 if (err)
534 return err;
535
536 INIT_WORK(&core->wdt_work, airoha_npu_wdt_work);
537 }
538
539 err = dma_set_coherent_mask(dev, DMA_BIT_MASK(32));
540 if (err)
541 return err;
542
543 err = airoha_npu_run_firmware(dev, base, rmem);
544 if (err)
545 return dev_err_probe(dev, err, "failed to run npu firmware\n");
546
547 regmap_write(npu->regmap, REG_CR_NPU_MIB(10),
548 rmem->base + NPU_EN7581_FIRMWARE_RV32_MAX_SIZE);
549 regmap_write(npu->regmap, REG_CR_NPU_MIB(11), 0x40000); /* SRAM 256K */
550 regmap_write(npu->regmap, REG_CR_NPU_MIB(12), 0);
551 regmap_write(npu->regmap, REG_CR_NPU_MIB(21), 1);
552 msleep(100);
553
554 /* setting booting address */
555 for (i = 0; i < NPU_NUM_CORES; i++)
556 regmap_write(npu->regmap, REG_CR_BOOT_BASE(i), rmem->base);
557 usleep_range(1000, 2000);
558
559 /* enable NPU cores */
560 /* do not start core3 since it is used for WiFi offloading */
561 regmap_write(npu->regmap, REG_CR_BOOT_CONFIG, 0xf7);
562 regmap_write(npu->regmap, REG_CR_BOOT_TRIGGER, 0x1);
563 msleep(100);
564
565 platform_set_drvdata(pdev, npu);
566
567 return 0;
568 }
569
airoha_npu_remove(struct platform_device * pdev)570 static void airoha_npu_remove(struct platform_device *pdev)
571 {
572 struct airoha_npu *npu = platform_get_drvdata(pdev);
573 int i;
574
575 for (i = 0; i < ARRAY_SIZE(npu->cores); i++)
576 cancel_work_sync(&npu->cores[i].wdt_work);
577 }
578
579 static struct platform_driver airoha_npu_driver = {
580 .probe = airoha_npu_probe,
581 .remove = airoha_npu_remove,
582 .driver = {
583 .name = "airoha-npu",
584 .of_match_table = of_airoha_npu_match,
585 },
586 };
587 module_platform_driver(airoha_npu_driver);
588
589 MODULE_LICENSE("GPL");
590 MODULE_AUTHOR("Lorenzo Bianconi <lorenzo@kernel.org>");
591 MODULE_DESCRIPTION("Airoha Network Processor Unit driver");
592