1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (c) 2025 AIROHA Inc 4 * Author: Lorenzo Bianconi <lorenzo@kernel.org> 5 */ 6 7 #include <linux/devcoredump.h> 8 #include <linux/firmware.h> 9 #include <linux/platform_device.h> 10 #include <linux/of_net.h> 11 #include <linux/of_platform.h> 12 #include <linux/of_reserved_mem.h> 13 #include <linux/regmap.h> 14 15 #include "airoha_npu.h" 16 17 #define NPU_EN7581_FIRMWARE_DATA "airoha/en7581_npu_data.bin" 18 #define NPU_EN7581_FIRMWARE_RV32 "airoha/en7581_npu_rv32.bin" 19 #define NPU_EN7581_FIRMWARE_RV32_MAX_SIZE 0x200000 20 #define NPU_EN7581_FIRMWARE_DATA_MAX_SIZE 0x10000 21 #define NPU_DUMP_SIZE 512 22 23 #define REG_NPU_LOCAL_SRAM 0x0 24 25 #define NPU_PC_BASE_ADDR 0x305000 26 #define REG_PC_DBG(_n) (0x305000 + ((_n) * 0x100)) 27 28 #define NPU_CLUSTER_BASE_ADDR 0x306000 29 30 #define REG_CR_BOOT_TRIGGER (NPU_CLUSTER_BASE_ADDR + 0x000) 31 #define REG_CR_BOOT_CONFIG (NPU_CLUSTER_BASE_ADDR + 0x004) 32 #define REG_CR_BOOT_BASE(_n) (NPU_CLUSTER_BASE_ADDR + 0x020 + ((_n) << 2)) 33 34 #define NPU_MBOX_BASE_ADDR 0x30c000 35 36 #define REG_CR_MBOX_INT_STATUS (NPU_MBOX_BASE_ADDR + 0x000) 37 #define MBOX_INT_STATUS_MASK BIT(8) 38 39 #define REG_CR_MBOX_INT_MASK(_n) (NPU_MBOX_BASE_ADDR + 0x004 + ((_n) << 2)) 40 #define REG_CR_MBQ0_CTRL(_n) (NPU_MBOX_BASE_ADDR + 0x030 + ((_n) << 2)) 41 #define REG_CR_MBQ8_CTRL(_n) (NPU_MBOX_BASE_ADDR + 0x0b0 + ((_n) << 2)) 42 #define REG_CR_NPU_MIB(_n) (NPU_MBOX_BASE_ADDR + 0x140 + ((_n) << 2)) 43 44 #define NPU_TIMER_BASE_ADDR 0x310100 45 #define REG_WDT_TIMER_CTRL(_n) (NPU_TIMER_BASE_ADDR + ((_n) * 0x100)) 46 #define WDT_EN_MASK BIT(25) 47 #define WDT_INTR_MASK BIT(21) 48 49 enum { 50 NPU_OP_SET = 1, 51 NPU_OP_SET_NO_WAIT, 52 NPU_OP_GET, 53 NPU_OP_GET_NO_WAIT, 54 }; 55 56 enum { 57 NPU_FUNC_WIFI, 58 NPU_FUNC_TUNNEL, 59 NPU_FUNC_NOTIFY, 60 NPU_FUNC_DBA, 61 NPU_FUNC_TR471, 62 NPU_FUNC_PPE, 63 }; 64 65 enum { 66 NPU_MBOX_ERROR, 67 NPU_MBOX_SUCCESS, 68 }; 69 70 enum { 71 PPE_FUNC_SET_WAIT, 72 PPE_FUNC_SET_WAIT_HWNAT_INIT, 73 PPE_FUNC_SET_WAIT_HWNAT_DEINIT, 74 PPE_FUNC_SET_WAIT_API, 75 }; 76 77 enum { 78 PPE2_SRAM_SET_ENTRY, 79 PPE_SRAM_SET_ENTRY, 80 PPE_SRAM_SET_VAL, 81 PPE_SRAM_RESET_VAL, 82 }; 83 84 enum { 85 QDMA_WAN_ETHER = 1, 86 QDMA_WAN_PON_XDSL, 87 }; 88 89 #define MBOX_MSG_FUNC_ID GENMASK(14, 11) 90 #define MBOX_MSG_STATIC_BUF BIT(5) 91 #define MBOX_MSG_STATUS GENMASK(4, 2) 92 #define MBOX_MSG_DONE BIT(1) 93 #define MBOX_MSG_WAIT_RSP BIT(0) 94 95 #define PPE_TYPE_L2B_IPV4 2 96 #define PPE_TYPE_L2B_IPV4_IPV6 3 97 98 struct ppe_mbox_data { 99 u32 func_type; 100 u32 func_id; 101 union { 102 struct { 103 u8 cds; 104 u8 xpon_hal_api; 105 u8 wan_xsi; 106 u8 ct_joyme4; 107 u8 max_packet; 108 u8 rsv[3]; 109 u32 ppe_type; 110 u32 wan_mode; 111 u32 wan_sel; 112 } init_info; 113 struct { 114 u32 func_id; 115 u32 size; 116 u32 data; 117 } set_info; 118 }; 119 }; 120 121 static int airoha_npu_send_msg(struct airoha_npu *npu, int func_id, 122 void *p, int size) 123 { 124 u16 core = 0; /* FIXME */ 125 u32 val, offset = core << 4; 126 dma_addr_t dma_addr; 127 void *addr; 128 int ret; 129 130 addr = kmemdup(p, size, GFP_ATOMIC); 131 if (!addr) 132 return -ENOMEM; 133 134 dma_addr = dma_map_single(npu->dev, addr, size, DMA_TO_DEVICE); 135 ret = dma_mapping_error(npu->dev, dma_addr); 136 if (ret) 137 goto out; 138 139 spin_lock_bh(&npu->cores[core].lock); 140 141 regmap_write(npu->regmap, REG_CR_MBQ0_CTRL(0) + offset, dma_addr); 142 regmap_write(npu->regmap, REG_CR_MBQ0_CTRL(1) + offset, size); 143 regmap_read(npu->regmap, REG_CR_MBQ0_CTRL(2) + offset, &val); 144 regmap_write(npu->regmap, REG_CR_MBQ0_CTRL(2) + offset, val + 1); 145 val = FIELD_PREP(MBOX_MSG_FUNC_ID, func_id) | MBOX_MSG_WAIT_RSP; 146 regmap_write(npu->regmap, REG_CR_MBQ0_CTRL(3) + offset, val); 147 148 ret = regmap_read_poll_timeout_atomic(npu->regmap, 149 REG_CR_MBQ0_CTRL(3) + offset, 150 val, (val & MBOX_MSG_DONE), 151 100, 100 * MSEC_PER_SEC); 152 if (!ret && FIELD_GET(MBOX_MSG_STATUS, val) != NPU_MBOX_SUCCESS) 153 ret = -EINVAL; 154 155 spin_unlock_bh(&npu->cores[core].lock); 156 157 dma_unmap_single(npu->dev, dma_addr, size, DMA_TO_DEVICE); 158 out: 159 kfree(addr); 160 161 return ret; 162 } 163 164 static int airoha_npu_run_firmware(struct device *dev, void __iomem *base, 165 struct reserved_mem *rmem) 166 { 167 const struct firmware *fw; 168 void __iomem *addr; 169 int ret; 170 171 ret = request_firmware(&fw, NPU_EN7581_FIRMWARE_RV32, dev); 172 if (ret) 173 return ret == -ENOENT ? -EPROBE_DEFER : ret; 174 175 if (fw->size > NPU_EN7581_FIRMWARE_RV32_MAX_SIZE) { 176 dev_err(dev, "%s: fw size too overlimit (%zu)\n", 177 NPU_EN7581_FIRMWARE_RV32, fw->size); 178 ret = -E2BIG; 179 goto out; 180 } 181 182 addr = devm_ioremap(dev, rmem->base, rmem->size); 183 if (!addr) { 184 ret = -ENOMEM; 185 goto out; 186 } 187 188 memcpy_toio(addr, fw->data, fw->size); 189 release_firmware(fw); 190 191 ret = request_firmware(&fw, NPU_EN7581_FIRMWARE_DATA, dev); 192 if (ret) 193 return ret == -ENOENT ? -EPROBE_DEFER : ret; 194 195 if (fw->size > NPU_EN7581_FIRMWARE_DATA_MAX_SIZE) { 196 dev_err(dev, "%s: fw size too overlimit (%zu)\n", 197 NPU_EN7581_FIRMWARE_DATA, fw->size); 198 ret = -E2BIG; 199 goto out; 200 } 201 202 memcpy_toio(base + REG_NPU_LOCAL_SRAM, fw->data, fw->size); 203 out: 204 release_firmware(fw); 205 206 return ret; 207 } 208 209 static irqreturn_t airoha_npu_mbox_handler(int irq, void *npu_instance) 210 { 211 struct airoha_npu *npu = npu_instance; 212 213 /* clear mbox interrupt status */ 214 regmap_write(npu->regmap, REG_CR_MBOX_INT_STATUS, 215 MBOX_INT_STATUS_MASK); 216 217 /* acknowledge npu */ 218 regmap_update_bits(npu->regmap, REG_CR_MBQ8_CTRL(3), 219 MBOX_MSG_STATUS | MBOX_MSG_DONE, MBOX_MSG_DONE); 220 221 return IRQ_HANDLED; 222 } 223 224 static void airoha_npu_wdt_work(struct work_struct *work) 225 { 226 struct airoha_npu_core *core; 227 struct airoha_npu *npu; 228 void *dump; 229 u32 val[3]; 230 int c; 231 232 core = container_of(work, struct airoha_npu_core, wdt_work); 233 npu = core->npu; 234 235 dump = vzalloc(NPU_DUMP_SIZE); 236 if (!dump) 237 return; 238 239 c = core - &npu->cores[0]; 240 regmap_bulk_read(npu->regmap, REG_PC_DBG(c), val, ARRAY_SIZE(val)); 241 snprintf(dump, NPU_DUMP_SIZE, "PC: %08x SP: %08x LR: %08x\n", 242 val[0], val[1], val[2]); 243 244 dev_coredumpv(npu->dev, dump, NPU_DUMP_SIZE, GFP_KERNEL); 245 } 246 247 static irqreturn_t airoha_npu_wdt_handler(int irq, void *core_instance) 248 { 249 struct airoha_npu_core *core = core_instance; 250 struct airoha_npu *npu = core->npu; 251 int c = core - &npu->cores[0]; 252 u32 val; 253 254 regmap_set_bits(npu->regmap, REG_WDT_TIMER_CTRL(c), WDT_INTR_MASK); 255 if (!regmap_read(npu->regmap, REG_WDT_TIMER_CTRL(c), &val) && 256 FIELD_GET(WDT_EN_MASK, val)) 257 schedule_work(&core->wdt_work); 258 259 return IRQ_HANDLED; 260 } 261 262 static int airoha_npu_ppe_init(struct airoha_npu *npu) 263 { 264 struct ppe_mbox_data ppe_data = { 265 .func_type = NPU_OP_SET, 266 .func_id = PPE_FUNC_SET_WAIT_HWNAT_INIT, 267 .init_info = { 268 .ppe_type = PPE_TYPE_L2B_IPV4_IPV6, 269 .wan_mode = QDMA_WAN_ETHER, 270 }, 271 }; 272 273 return airoha_npu_send_msg(npu, NPU_FUNC_PPE, &ppe_data, 274 sizeof(struct ppe_mbox_data)); 275 } 276 277 static int airoha_npu_ppe_deinit(struct airoha_npu *npu) 278 { 279 struct ppe_mbox_data ppe_data = { 280 .func_type = NPU_OP_SET, 281 .func_id = PPE_FUNC_SET_WAIT_HWNAT_DEINIT, 282 }; 283 284 return airoha_npu_send_msg(npu, NPU_FUNC_PPE, &ppe_data, 285 sizeof(struct ppe_mbox_data)); 286 } 287 288 static int airoha_npu_ppe_flush_sram_entries(struct airoha_npu *npu, 289 dma_addr_t foe_addr, 290 int sram_num_entries) 291 { 292 struct ppe_mbox_data ppe_data = { 293 .func_type = NPU_OP_SET, 294 .func_id = PPE_FUNC_SET_WAIT_API, 295 .set_info = { 296 .func_id = PPE_SRAM_RESET_VAL, 297 .data = foe_addr, 298 .size = sram_num_entries, 299 }, 300 }; 301 302 return airoha_npu_send_msg(npu, NPU_FUNC_PPE, &ppe_data, 303 sizeof(struct ppe_mbox_data)); 304 } 305 306 static int airoha_npu_foe_commit_entry(struct airoha_npu *npu, 307 dma_addr_t foe_addr, 308 u32 entry_size, u32 hash, bool ppe2) 309 { 310 struct ppe_mbox_data ppe_data = { 311 .func_type = NPU_OP_SET, 312 .func_id = PPE_FUNC_SET_WAIT_API, 313 .set_info = { 314 .data = foe_addr, 315 .size = entry_size, 316 }, 317 }; 318 int err; 319 320 ppe_data.set_info.func_id = ppe2 ? PPE2_SRAM_SET_ENTRY 321 : PPE_SRAM_SET_ENTRY; 322 323 err = airoha_npu_send_msg(npu, NPU_FUNC_PPE, &ppe_data, 324 sizeof(struct ppe_mbox_data)); 325 if (err) 326 return err; 327 328 ppe_data.set_info.func_id = PPE_SRAM_SET_VAL; 329 ppe_data.set_info.data = hash; 330 ppe_data.set_info.size = sizeof(u32); 331 332 return airoha_npu_send_msg(npu, NPU_FUNC_PPE, &ppe_data, 333 sizeof(struct ppe_mbox_data)); 334 } 335 336 struct airoha_npu *airoha_npu_get(struct device *dev) 337 { 338 struct platform_device *pdev; 339 struct device_node *np; 340 struct airoha_npu *npu; 341 342 np = of_parse_phandle(dev->of_node, "airoha,npu", 0); 343 if (!np) 344 return ERR_PTR(-ENODEV); 345 346 pdev = of_find_device_by_node(np); 347 of_node_put(np); 348 349 if (!pdev) { 350 dev_err(dev, "cannot find device node %s\n", np->name); 351 return ERR_PTR(-ENODEV); 352 } 353 354 if (!try_module_get(THIS_MODULE)) { 355 dev_err(dev, "failed to get the device driver module\n"); 356 npu = ERR_PTR(-ENODEV); 357 goto error_pdev_put; 358 } 359 360 npu = platform_get_drvdata(pdev); 361 if (!npu) { 362 npu = ERR_PTR(-ENODEV); 363 goto error_module_put; 364 } 365 366 if (!device_link_add(dev, &pdev->dev, DL_FLAG_AUTOREMOVE_SUPPLIER)) { 367 dev_err(&pdev->dev, 368 "failed to create device link to consumer %s\n", 369 dev_name(dev)); 370 npu = ERR_PTR(-EINVAL); 371 goto error_module_put; 372 } 373 374 return npu; 375 376 error_module_put: 377 module_put(THIS_MODULE); 378 error_pdev_put: 379 platform_device_put(pdev); 380 381 return npu; 382 } 383 EXPORT_SYMBOL_GPL(airoha_npu_get); 384 385 void airoha_npu_put(struct airoha_npu *npu) 386 { 387 module_put(THIS_MODULE); 388 put_device(npu->dev); 389 } 390 EXPORT_SYMBOL_GPL(airoha_npu_put); 391 392 static const struct of_device_id of_airoha_npu_match[] = { 393 { .compatible = "airoha,en7581-npu" }, 394 { /* sentinel */ } 395 }; 396 MODULE_DEVICE_TABLE(of, of_airoha_npu_match); 397 398 static const struct regmap_config regmap_config = { 399 .name = "npu", 400 .reg_bits = 32, 401 .val_bits = 32, 402 .reg_stride = 4, 403 .disable_locking = true, 404 }; 405 406 static int airoha_npu_probe(struct platform_device *pdev) 407 { 408 struct device *dev = &pdev->dev; 409 struct reserved_mem *rmem; 410 struct airoha_npu *npu; 411 struct device_node *np; 412 void __iomem *base; 413 int i, irq, err; 414 415 base = devm_platform_ioremap_resource(pdev, 0); 416 if (IS_ERR(base)) 417 return PTR_ERR(base); 418 419 npu = devm_kzalloc(dev, sizeof(*npu), GFP_KERNEL); 420 if (!npu) 421 return -ENOMEM; 422 423 npu->dev = dev; 424 npu->ops.ppe_init = airoha_npu_ppe_init; 425 npu->ops.ppe_deinit = airoha_npu_ppe_deinit; 426 npu->ops.ppe_flush_sram_entries = airoha_npu_ppe_flush_sram_entries; 427 npu->ops.ppe_foe_commit_entry = airoha_npu_foe_commit_entry; 428 429 npu->regmap = devm_regmap_init_mmio(dev, base, ®map_config); 430 if (IS_ERR(npu->regmap)) 431 return PTR_ERR(npu->regmap); 432 433 np = of_parse_phandle(dev->of_node, "memory-region", 0); 434 if (!np) 435 return -ENODEV; 436 437 rmem = of_reserved_mem_lookup(np); 438 of_node_put(np); 439 440 if (!rmem) 441 return -ENODEV; 442 443 irq = platform_get_irq(pdev, 0); 444 if (irq < 0) 445 return irq; 446 447 err = devm_request_irq(dev, irq, airoha_npu_mbox_handler, 448 IRQF_SHARED, "airoha-npu-mbox", npu); 449 if (err) 450 return err; 451 452 for (i = 0; i < ARRAY_SIZE(npu->cores); i++) { 453 struct airoha_npu_core *core = &npu->cores[i]; 454 455 spin_lock_init(&core->lock); 456 core->npu = npu; 457 458 irq = platform_get_irq(pdev, i + 1); 459 if (irq < 0) 460 return irq; 461 462 err = devm_request_irq(dev, irq, airoha_npu_wdt_handler, 463 IRQF_SHARED, "airoha-npu-wdt", core); 464 if (err) 465 return err; 466 467 INIT_WORK(&core->wdt_work, airoha_npu_wdt_work); 468 } 469 470 err = dma_set_coherent_mask(dev, DMA_BIT_MASK(32)); 471 if (err) 472 return err; 473 474 err = airoha_npu_run_firmware(dev, base, rmem); 475 if (err) 476 return dev_err_probe(dev, err, "failed to run npu firmware\n"); 477 478 regmap_write(npu->regmap, REG_CR_NPU_MIB(10), 479 rmem->base + NPU_EN7581_FIRMWARE_RV32_MAX_SIZE); 480 regmap_write(npu->regmap, REG_CR_NPU_MIB(11), 0x40000); /* SRAM 256K */ 481 regmap_write(npu->regmap, REG_CR_NPU_MIB(12), 0); 482 regmap_write(npu->regmap, REG_CR_NPU_MIB(21), 1); 483 msleep(100); 484 485 /* setting booting address */ 486 for (i = 0; i < NPU_NUM_CORES; i++) 487 regmap_write(npu->regmap, REG_CR_BOOT_BASE(i), rmem->base); 488 usleep_range(1000, 2000); 489 490 /* enable NPU cores */ 491 /* do not start core3 since it is used for WiFi offloading */ 492 regmap_write(npu->regmap, REG_CR_BOOT_CONFIG, 0xf7); 493 regmap_write(npu->regmap, REG_CR_BOOT_TRIGGER, 0x1); 494 msleep(100); 495 496 platform_set_drvdata(pdev, npu); 497 498 return 0; 499 } 500 501 static void airoha_npu_remove(struct platform_device *pdev) 502 { 503 struct airoha_npu *npu = platform_get_drvdata(pdev); 504 int i; 505 506 for (i = 0; i < ARRAY_SIZE(npu->cores); i++) 507 cancel_work_sync(&npu->cores[i].wdt_work); 508 } 509 510 static struct platform_driver airoha_npu_driver = { 511 .probe = airoha_npu_probe, 512 .remove = airoha_npu_remove, 513 .driver = { 514 .name = "airoha-npu", 515 .of_match_table = of_airoha_npu_match, 516 }, 517 }; 518 module_platform_driver(airoha_npu_driver); 519 520 MODULE_LICENSE("GPL"); 521 MODULE_AUTHOR("Lorenzo Bianconi <lorenzo@kernel.org>"); 522 MODULE_DESCRIPTION("Airoha Network Processor Unit driver"); 523