1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright 2021 NXP */ 3 4 #include <dt-bindings/firmware/imx/rsrc.h> 5 #include <linux/arm-smccc.h> 6 #include <linux/clk.h> 7 #include <linux/err.h> 8 #include <linux/firmware.h> 9 #include <linux/firmware/imx/sci.h> 10 #include <linux/interrupt.h> 11 #include <linux/kernel.h> 12 #include <linux/mailbox_client.h> 13 #include <linux/mfd/syscon.h> 14 #include <linux/module.h> 15 #include <linux/of_address.h> 16 #include <linux/of_device.h> 17 #include <linux/of_reserved_mem.h> 18 #include <linux/platform_device.h> 19 #include <linux/pm_domain.h> 20 #include <linux/pm_runtime.h> 21 #include <linux/regmap.h> 22 #include <linux/remoteproc.h> 23 #include <linux/slab.h> 24 25 #include "imx_rproc.h" 26 #include "remoteproc_elf_helpers.h" 27 #include "remoteproc_internal.h" 28 29 #define DSP_RPROC_CLK_MAX 5 30 31 #define REMOTE_IS_READY BIT(0) 32 #define REMOTE_READY_WAIT_MAX_RETRIES 500 33 34 /* att flags */ 35 /* DSP own area */ 36 #define ATT_OWN BIT(31) 37 /* DSP instruction area */ 38 #define ATT_IRAM BIT(30) 39 40 /* Definitions for i.MX8MP */ 41 /* DAP registers */ 42 #define IMX8M_DAP_DEBUG 0x28800000 43 #define IMX8M_DAP_DEBUG_SIZE (64 * 1024) 44 #define IMX8M_DAP_PWRCTL (0x4000 + 0x3020) 45 #define IMX8M_PWRCTL_CORERESET BIT(16) 46 47 /* DSP audio mix registers */ 48 #define IMX8M_AudioDSP_REG0 0x100 49 #define IMX8M_AudioDSP_REG1 0x104 50 #define IMX8M_AudioDSP_REG2 0x108 51 #define IMX8M_AudioDSP_REG3 0x10c 52 53 #define IMX8M_AudioDSP_REG2_RUNSTALL BIT(5) 54 #define IMX8M_AudioDSP_REG2_PWAITMODE BIT(1) 55 56 /* Definitions for i.MX8ULP */ 57 #define IMX8ULP_SIM_LPAV_REG_SYSCTRL0 0x8 58 #define IMX8ULP_SYSCTRL0_DSP_DBG_RST BIT(25) 59 #define IMX8ULP_SYSCTRL0_DSP_PLAT_CLK_EN BIT(19) 60 #define IMX8ULP_SYSCTRL0_DSP_PBCLK_EN BIT(18) 61 #define IMX8ULP_SYSCTRL0_DSP_CLK_EN BIT(17) 62 #define IMX8ULP_SYSCTRL0_DSP_RST BIT(16) 63 #define IMX8ULP_SYSCTRL0_DSP_OCD_HALT BIT(14) 64 #define IMX8ULP_SYSCTRL0_DSP_STALL BIT(13) 65 66 #define IMX8ULP_SIP_HIFI_XRDC 0xc200000e 67 68 /* 69 * enum - Predefined Mailbox Messages 70 * 71 * @RP_MBOX_SUSPEND_SYSTEM: system suspend request for the remote processor 72 * 73 * @RP_MBOX_SUSPEND_ACK: successful response from remote processor for a 74 * suspend request 75 * 76 * @RP_MBOX_RESUME_SYSTEM: system resume request for the remote processor 77 * 78 * @RP_MBOX_RESUME_ACK: successful response from remote processor for a 79 * resume request 80 */ 81 enum imx_dsp_rp_mbox_messages { 82 RP_MBOX_SUSPEND_SYSTEM = 0xFF11, 83 RP_MBOX_SUSPEND_ACK = 0xFF12, 84 RP_MBOX_RESUME_SYSTEM = 0xFF13, 85 RP_MBOX_RESUME_ACK = 0xFF14, 86 }; 87 88 /** 89 * struct imx_dsp_rproc - DSP remote processor state 90 * @regmap: regmap handler 91 * @rproc: rproc handler 92 * @dsp_dcfg: device configuration pointer 93 * @clks: clocks needed by this device 94 * @cl: mailbox client to request the mailbox channel 95 * @cl_rxdb: mailbox client to request the mailbox channel for doorbell 96 * @tx_ch: mailbox tx channel handle 97 * @rx_ch: mailbox rx channel handle 98 * @rxdb_ch: mailbox rx doorbell channel handle 99 * @pd_dev: power domain device 100 * @pd_dev_link: power domain device link 101 * @ipc_handle: System Control Unit ipc handle 102 * @rproc_work: work for processing virtio interrupts 103 * @pm_comp: completion primitive to sync for suspend response 104 * @num_domains: power domain number 105 * @flags: control flags 106 */ 107 struct imx_dsp_rproc { 108 struct regmap *regmap; 109 struct rproc *rproc; 110 const struct imx_dsp_rproc_dcfg *dsp_dcfg; 111 struct clk_bulk_data clks[DSP_RPROC_CLK_MAX]; 112 struct mbox_client cl; 113 struct mbox_client cl_rxdb; 114 struct mbox_chan *tx_ch; 115 struct mbox_chan *rx_ch; 116 struct mbox_chan *rxdb_ch; 117 struct device **pd_dev; 118 struct device_link **pd_dev_link; 119 struct imx_sc_ipc *ipc_handle; 120 struct work_struct rproc_work; 121 struct completion pm_comp; 122 int num_domains; 123 u32 flags; 124 }; 125 126 /** 127 * struct imx_dsp_rproc_dcfg - DSP remote processor configuration 128 * @dcfg: imx_rproc_dcfg handler 129 * @reset: reset callback function 130 */ 131 struct imx_dsp_rproc_dcfg { 132 const struct imx_rproc_dcfg *dcfg; 133 int (*reset)(struct imx_dsp_rproc *priv); 134 }; 135 136 static const struct imx_rproc_att imx_dsp_rproc_att_imx8qm[] = { 137 /* dev addr , sys addr , size , flags */ 138 { 0x596e8000, 0x556e8000, 0x00008000, ATT_OWN }, 139 { 0x596f0000, 0x556f0000, 0x00008000, ATT_OWN }, 140 { 0x596f8000, 0x556f8000, 0x00000800, ATT_OWN | ATT_IRAM}, 141 { 0x55700000, 0x55700000, 0x00070000, ATT_OWN }, 142 /* DDR (Data) */ 143 { 0x80000000, 0x80000000, 0x60000000, 0}, 144 }; 145 146 static const struct imx_rproc_att imx_dsp_rproc_att_imx8qxp[] = { 147 /* dev addr , sys addr , size , flags */ 148 { 0x596e8000, 0x596e8000, 0x00008000, ATT_OWN }, 149 { 0x596f0000, 0x596f0000, 0x00008000, ATT_OWN }, 150 { 0x596f8000, 0x596f8000, 0x00000800, ATT_OWN | ATT_IRAM}, 151 { 0x59700000, 0x59700000, 0x00070000, ATT_OWN }, 152 /* DDR (Data) */ 153 { 0x80000000, 0x80000000, 0x60000000, 0}, 154 }; 155 156 static const struct imx_rproc_att imx_dsp_rproc_att_imx8mp[] = { 157 /* dev addr , sys addr , size , flags */ 158 { 0x3b6e8000, 0x3b6e8000, 0x00008000, ATT_OWN }, 159 { 0x3b6f0000, 0x3b6f0000, 0x00008000, ATT_OWN }, 160 { 0x3b6f8000, 0x3b6f8000, 0x00000800, ATT_OWN | ATT_IRAM}, 161 { 0x3b700000, 0x3b700000, 0x00040000, ATT_OWN }, 162 /* DDR (Data) */ 163 { 0x40000000, 0x40000000, 0x80000000, 0}, 164 }; 165 166 static const struct imx_rproc_att imx_dsp_rproc_att_imx8ulp[] = { 167 /* dev addr , sys addr , size , flags */ 168 { 0x21170000, 0x21170000, 0x00010000, ATT_OWN | ATT_IRAM}, 169 { 0x21180000, 0x21180000, 0x00010000, ATT_OWN }, 170 /* DDR (Data) */ 171 { 0x0c000000, 0x80000000, 0x10000000, 0}, 172 { 0x30000000, 0x90000000, 0x10000000, 0}, 173 }; 174 175 /* Reset function for DSP on i.MX8MP */ 176 static int imx8mp_dsp_reset(struct imx_dsp_rproc *priv) 177 { 178 void __iomem *dap = ioremap_wc(IMX8M_DAP_DEBUG, IMX8M_DAP_DEBUG_SIZE); 179 int pwrctl; 180 181 /* Put DSP into reset and stall */ 182 pwrctl = readl(dap + IMX8M_DAP_PWRCTL); 183 pwrctl |= IMX8M_PWRCTL_CORERESET; 184 writel(pwrctl, dap + IMX8M_DAP_PWRCTL); 185 186 /* Keep reset asserted for 10 cycles */ 187 usleep_range(1, 2); 188 189 regmap_update_bits(priv->regmap, IMX8M_AudioDSP_REG2, 190 IMX8M_AudioDSP_REG2_RUNSTALL, 191 IMX8M_AudioDSP_REG2_RUNSTALL); 192 193 /* Take the DSP out of reset and keep stalled for FW loading */ 194 pwrctl = readl(dap + IMX8M_DAP_PWRCTL); 195 pwrctl &= ~IMX8M_PWRCTL_CORERESET; 196 writel(pwrctl, dap + IMX8M_DAP_PWRCTL); 197 198 iounmap(dap); 199 return 0; 200 } 201 202 /* Reset function for DSP on i.MX8ULP */ 203 static int imx8ulp_dsp_reset(struct imx_dsp_rproc *priv) 204 { 205 struct arm_smccc_res res; 206 207 /* Put DSP into reset and stall */ 208 regmap_update_bits(priv->regmap, IMX8ULP_SIM_LPAV_REG_SYSCTRL0, 209 IMX8ULP_SYSCTRL0_DSP_RST, IMX8ULP_SYSCTRL0_DSP_RST); 210 regmap_update_bits(priv->regmap, IMX8ULP_SIM_LPAV_REG_SYSCTRL0, 211 IMX8ULP_SYSCTRL0_DSP_STALL, 212 IMX8ULP_SYSCTRL0_DSP_STALL); 213 214 /* Configure resources of DSP through TFA */ 215 arm_smccc_smc(IMX8ULP_SIP_HIFI_XRDC, 0, 0, 0, 0, 0, 0, 0, &res); 216 217 /* Take the DSP out of reset and keep stalled for FW loading */ 218 regmap_update_bits(priv->regmap, IMX8ULP_SIM_LPAV_REG_SYSCTRL0, 219 IMX8ULP_SYSCTRL0_DSP_RST, 0); 220 regmap_update_bits(priv->regmap, IMX8ULP_SIM_LPAV_REG_SYSCTRL0, 221 IMX8ULP_SYSCTRL0_DSP_DBG_RST, 0); 222 223 return 0; 224 } 225 226 /* Specific configuration for i.MX8MP */ 227 static const struct imx_rproc_dcfg dsp_rproc_cfg_imx8mp = { 228 .src_reg = IMX8M_AudioDSP_REG2, 229 .src_mask = IMX8M_AudioDSP_REG2_RUNSTALL, 230 .src_start = 0, 231 .src_stop = IMX8M_AudioDSP_REG2_RUNSTALL, 232 .att = imx_dsp_rproc_att_imx8mp, 233 .att_size = ARRAY_SIZE(imx_dsp_rproc_att_imx8mp), 234 .method = IMX_RPROC_MMIO, 235 }; 236 237 static const struct imx_dsp_rproc_dcfg imx_dsp_rproc_cfg_imx8mp = { 238 .dcfg = &dsp_rproc_cfg_imx8mp, 239 .reset = imx8mp_dsp_reset, 240 }; 241 242 /* Specific configuration for i.MX8ULP */ 243 static const struct imx_rproc_dcfg dsp_rproc_cfg_imx8ulp = { 244 .src_reg = IMX8ULP_SIM_LPAV_REG_SYSCTRL0, 245 .src_mask = IMX8ULP_SYSCTRL0_DSP_STALL, 246 .src_start = 0, 247 .src_stop = IMX8ULP_SYSCTRL0_DSP_STALL, 248 .att = imx_dsp_rproc_att_imx8ulp, 249 .att_size = ARRAY_SIZE(imx_dsp_rproc_att_imx8ulp), 250 .method = IMX_RPROC_MMIO, 251 }; 252 253 static const struct imx_dsp_rproc_dcfg imx_dsp_rproc_cfg_imx8ulp = { 254 .dcfg = &dsp_rproc_cfg_imx8ulp, 255 .reset = imx8ulp_dsp_reset, 256 }; 257 258 /* Specific configuration for i.MX8QXP */ 259 static const struct imx_rproc_dcfg dsp_rproc_cfg_imx8qxp = { 260 .att = imx_dsp_rproc_att_imx8qxp, 261 .att_size = ARRAY_SIZE(imx_dsp_rproc_att_imx8qxp), 262 .method = IMX_RPROC_SCU_API, 263 }; 264 265 static const struct imx_dsp_rproc_dcfg imx_dsp_rproc_cfg_imx8qxp = { 266 .dcfg = &dsp_rproc_cfg_imx8qxp, 267 }; 268 269 /* Specific configuration for i.MX8QM */ 270 static const struct imx_rproc_dcfg dsp_rproc_cfg_imx8qm = { 271 .att = imx_dsp_rproc_att_imx8qm, 272 .att_size = ARRAY_SIZE(imx_dsp_rproc_att_imx8qm), 273 .method = IMX_RPROC_SCU_API, 274 }; 275 276 static const struct imx_dsp_rproc_dcfg imx_dsp_rproc_cfg_imx8qm = { 277 .dcfg = &dsp_rproc_cfg_imx8qm, 278 }; 279 280 static int imx_dsp_rproc_ready(struct rproc *rproc) 281 { 282 struct imx_dsp_rproc *priv = rproc->priv; 283 int i; 284 285 if (!priv->rxdb_ch) 286 return 0; 287 288 for (i = 0; i < REMOTE_READY_WAIT_MAX_RETRIES; i++) { 289 if (priv->flags & REMOTE_IS_READY) 290 return 0; 291 usleep_range(100, 200); 292 } 293 294 return -ETIMEDOUT; 295 } 296 297 /* 298 * Start function for rproc_ops 299 * 300 * There is a handshake for start procedure: when DSP starts, it 301 * will send a doorbell message to this driver, then the 302 * REMOTE_IS_READY flags is set, then driver will kick 303 * a message to DSP. 304 */ 305 static int imx_dsp_rproc_start(struct rproc *rproc) 306 { 307 struct imx_dsp_rproc *priv = rproc->priv; 308 const struct imx_dsp_rproc_dcfg *dsp_dcfg = priv->dsp_dcfg; 309 const struct imx_rproc_dcfg *dcfg = dsp_dcfg->dcfg; 310 struct device *dev = rproc->dev.parent; 311 int ret; 312 313 switch (dcfg->method) { 314 case IMX_RPROC_MMIO: 315 ret = regmap_update_bits(priv->regmap, 316 dcfg->src_reg, 317 dcfg->src_mask, 318 dcfg->src_start); 319 break; 320 case IMX_RPROC_SCU_API: 321 ret = imx_sc_pm_cpu_start(priv->ipc_handle, 322 IMX_SC_R_DSP, 323 true, 324 rproc->bootaddr); 325 break; 326 default: 327 return -EOPNOTSUPP; 328 } 329 330 if (ret) 331 dev_err(dev, "Failed to enable remote core!\n"); 332 else 333 ret = imx_dsp_rproc_ready(rproc); 334 335 return ret; 336 } 337 338 /* 339 * Stop function for rproc_ops 340 * It clears the REMOTE_IS_READY flags 341 */ 342 static int imx_dsp_rproc_stop(struct rproc *rproc) 343 { 344 struct imx_dsp_rproc *priv = rproc->priv; 345 const struct imx_dsp_rproc_dcfg *dsp_dcfg = priv->dsp_dcfg; 346 const struct imx_rproc_dcfg *dcfg = dsp_dcfg->dcfg; 347 struct device *dev = rproc->dev.parent; 348 int ret = 0; 349 350 /* Make sure work is finished */ 351 flush_work(&priv->rproc_work); 352 353 if (rproc->state == RPROC_CRASHED) { 354 priv->flags &= ~REMOTE_IS_READY; 355 return 0; 356 } 357 358 switch (dcfg->method) { 359 case IMX_RPROC_MMIO: 360 ret = regmap_update_bits(priv->regmap, dcfg->src_reg, dcfg->src_mask, 361 dcfg->src_stop); 362 break; 363 case IMX_RPROC_SCU_API: 364 ret = imx_sc_pm_cpu_start(priv->ipc_handle, 365 IMX_SC_R_DSP, 366 false, 367 rproc->bootaddr); 368 break; 369 default: 370 return -EOPNOTSUPP; 371 } 372 373 if (ret) 374 dev_err(dev, "Failed to stop remote core\n"); 375 else 376 priv->flags &= ~REMOTE_IS_READY; 377 378 return ret; 379 } 380 381 /** 382 * imx_dsp_rproc_sys_to_da() - internal memory translation helper 383 * @priv: private data pointer 384 * @sys: system address (DDR address) 385 * @len: length of the memory buffer 386 * @da: device address to translate 387 * 388 * Convert system address (DDR address) to device address (DSP) 389 * for there may be memory remap for device. 390 */ 391 static int imx_dsp_rproc_sys_to_da(struct imx_dsp_rproc *priv, u64 sys, 392 size_t len, u64 *da) 393 { 394 const struct imx_dsp_rproc_dcfg *dsp_dcfg = priv->dsp_dcfg; 395 const struct imx_rproc_dcfg *dcfg = dsp_dcfg->dcfg; 396 int i; 397 398 /* Parse address translation table */ 399 for (i = 0; i < dcfg->att_size; i++) { 400 const struct imx_rproc_att *att = &dcfg->att[i]; 401 402 if (sys >= att->sa && sys + len <= att->sa + att->size) { 403 unsigned int offset = sys - att->sa; 404 405 *da = att->da + offset; 406 return 0; 407 } 408 } 409 410 return -ENOENT; 411 } 412 413 /* Main virtqueue message work function 414 * 415 * This function is executed upon scheduling of the i.MX DSP remoteproc 416 * driver's workqueue. The workqueue is scheduled by the mailbox rx 417 * handler. 418 * 419 * This work function processes both the Tx and Rx virtqueue indices on 420 * every invocation. The rproc_vq_interrupt function can detect if there 421 * are new unprocessed messages or not (returns IRQ_NONE vs IRQ_HANDLED), 422 * but there is no need to check for these return values. The index 0 423 * triggering will process all pending Rx buffers, and the index 1 triggering 424 * will process all newly available Tx buffers and will wakeup any potentially 425 * blocked senders. 426 * 427 * NOTE: 428 * The current logic is based on an inherent design assumption of supporting 429 * only 2 vrings, but this can be changed if needed. 430 */ 431 static void imx_dsp_rproc_vq_work(struct work_struct *work) 432 { 433 struct imx_dsp_rproc *priv = container_of(work, struct imx_dsp_rproc, 434 rproc_work); 435 436 rproc_vq_interrupt(priv->rproc, 0); 437 rproc_vq_interrupt(priv->rproc, 1); 438 } 439 440 /** 441 * imx_dsp_rproc_rx_tx_callback() - inbound mailbox message handler 442 * @cl: mailbox client pointer used for requesting the mailbox channel 443 * @data: mailbox payload 444 * 445 * This handler is invoked by mailbox driver whenever a mailbox 446 * message is received. Usually, the SUSPEND and RESUME related messages 447 * are handled in this function, other messages are handled by remoteproc core 448 */ 449 static void imx_dsp_rproc_rx_tx_callback(struct mbox_client *cl, void *data) 450 { 451 struct rproc *rproc = dev_get_drvdata(cl->dev); 452 struct imx_dsp_rproc *priv = rproc->priv; 453 struct device *dev = rproc->dev.parent; 454 u32 message = (u32)(*(u32 *)data); 455 456 dev_dbg(dev, "mbox msg: 0x%x\n", message); 457 458 switch (message) { 459 case RP_MBOX_SUSPEND_ACK: 460 complete(&priv->pm_comp); 461 break; 462 case RP_MBOX_RESUME_ACK: 463 complete(&priv->pm_comp); 464 break; 465 default: 466 schedule_work(&priv->rproc_work); 467 break; 468 } 469 } 470 471 /** 472 * imx_dsp_rproc_rxdb_callback() - inbound mailbox message handler 473 * @cl: mailbox client pointer used for requesting the mailbox channel 474 * @data: mailbox payload 475 * 476 * For doorbell, there is no message specified, just set REMOTE_IS_READY 477 * flag. 478 */ 479 static void imx_dsp_rproc_rxdb_callback(struct mbox_client *cl, void *data) 480 { 481 struct rproc *rproc = dev_get_drvdata(cl->dev); 482 struct imx_dsp_rproc *priv = rproc->priv; 483 484 /* Remote is ready after firmware is loaded and running */ 485 priv->flags |= REMOTE_IS_READY; 486 } 487 488 /** 489 * imx_dsp_rproc_mbox_init() - request mailbox channels 490 * @priv: private data pointer 491 * 492 * Request three mailbox channels (tx, rx, rxdb). 493 */ 494 static int imx_dsp_rproc_mbox_init(struct imx_dsp_rproc *priv) 495 { 496 struct device *dev = priv->rproc->dev.parent; 497 struct mbox_client *cl; 498 int ret; 499 500 if (!of_get_property(dev->of_node, "mbox-names", NULL)) 501 return 0; 502 503 cl = &priv->cl; 504 cl->dev = dev; 505 cl->tx_block = true; 506 cl->tx_tout = 100; 507 cl->knows_txdone = false; 508 cl->rx_callback = imx_dsp_rproc_rx_tx_callback; 509 510 /* Channel for sending message */ 511 priv->tx_ch = mbox_request_channel_byname(cl, "tx"); 512 if (IS_ERR(priv->tx_ch)) { 513 ret = PTR_ERR(priv->tx_ch); 514 dev_dbg(cl->dev, "failed to request tx mailbox channel: %d\n", 515 ret); 516 goto err_out; 517 } 518 519 /* Channel for receiving message */ 520 priv->rx_ch = mbox_request_channel_byname(cl, "rx"); 521 if (IS_ERR(priv->rx_ch)) { 522 ret = PTR_ERR(priv->rx_ch); 523 dev_dbg(cl->dev, "failed to request rx mailbox channel: %d\n", 524 ret); 525 goto err_out; 526 } 527 528 cl = &priv->cl_rxdb; 529 cl->dev = dev; 530 cl->rx_callback = imx_dsp_rproc_rxdb_callback; 531 532 /* 533 * RX door bell is used to receive the ready signal from remote 534 * after firmware loaded. 535 */ 536 priv->rxdb_ch = mbox_request_channel_byname(cl, "rxdb"); 537 if (IS_ERR(priv->rxdb_ch)) { 538 ret = PTR_ERR(priv->rxdb_ch); 539 dev_dbg(cl->dev, "failed to request mbox chan rxdb, ret %d\n", 540 ret); 541 goto err_out; 542 } 543 544 return 0; 545 546 err_out: 547 if (!IS_ERR(priv->tx_ch)) 548 mbox_free_channel(priv->tx_ch); 549 if (!IS_ERR(priv->rx_ch)) 550 mbox_free_channel(priv->rx_ch); 551 if (!IS_ERR(priv->rxdb_ch)) 552 mbox_free_channel(priv->rxdb_ch); 553 554 return ret; 555 } 556 557 static void imx_dsp_rproc_free_mbox(struct imx_dsp_rproc *priv) 558 { 559 mbox_free_channel(priv->tx_ch); 560 mbox_free_channel(priv->rx_ch); 561 mbox_free_channel(priv->rxdb_ch); 562 } 563 564 /** 565 * imx_dsp_rproc_add_carveout() - request mailbox channels 566 * @priv: private data pointer 567 * 568 * This function registers specified memory entry in @rproc carveouts list 569 * The carveouts can help to mapping the memory address for DSP. 570 */ 571 static int imx_dsp_rproc_add_carveout(struct imx_dsp_rproc *priv) 572 { 573 const struct imx_dsp_rproc_dcfg *dsp_dcfg = priv->dsp_dcfg; 574 const struct imx_rproc_dcfg *dcfg = dsp_dcfg->dcfg; 575 struct rproc *rproc = priv->rproc; 576 struct device *dev = rproc->dev.parent; 577 struct device_node *np = dev->of_node; 578 struct of_phandle_iterator it; 579 struct rproc_mem_entry *mem; 580 struct reserved_mem *rmem; 581 void __iomem *cpu_addr; 582 int a; 583 u64 da; 584 585 /* Remap required addresses */ 586 for (a = 0; a < dcfg->att_size; a++) { 587 const struct imx_rproc_att *att = &dcfg->att[a]; 588 589 if (!(att->flags & ATT_OWN)) 590 continue; 591 592 if (imx_dsp_rproc_sys_to_da(priv, att->sa, att->size, &da)) 593 return -EINVAL; 594 595 cpu_addr = devm_ioremap_wc(dev, att->sa, att->size); 596 if (!cpu_addr) { 597 dev_err(dev, "failed to map memory %p\n", &att->sa); 598 return -ENOMEM; 599 } 600 601 /* Register memory region */ 602 mem = rproc_mem_entry_init(dev, cpu_addr, (dma_addr_t)att->sa, 603 att->size, da, NULL, NULL, "dsp_mem"); 604 605 if (mem) 606 rproc_coredump_add_segment(rproc, da, att->size); 607 else 608 return -ENOMEM; 609 610 rproc_add_carveout(rproc, mem); 611 } 612 613 of_phandle_iterator_init(&it, np, "memory-region", NULL, 0); 614 while (of_phandle_iterator_next(&it) == 0) { 615 /* 616 * Ignore the first memory region which will be used vdev buffer. 617 * No need to do extra handlings, rproc_add_virtio_dev will handle it. 618 */ 619 if (!strcmp(it.node->name, "vdev0buffer")) 620 continue; 621 622 rmem = of_reserved_mem_lookup(it.node); 623 if (!rmem) { 624 dev_err(dev, "unable to acquire memory-region\n"); 625 return -EINVAL; 626 } 627 628 if (imx_dsp_rproc_sys_to_da(priv, rmem->base, rmem->size, &da)) 629 return -EINVAL; 630 631 cpu_addr = devm_ioremap_wc(dev, rmem->base, rmem->size); 632 if (!cpu_addr) { 633 dev_err(dev, "failed to map memory %p\n", &rmem->base); 634 return -ENOMEM; 635 } 636 637 /* Register memory region */ 638 mem = rproc_mem_entry_init(dev, cpu_addr, (dma_addr_t)rmem->base, 639 rmem->size, da, NULL, NULL, it.node->name); 640 641 if (mem) 642 rproc_coredump_add_segment(rproc, da, rmem->size); 643 else 644 return -ENOMEM; 645 646 rproc_add_carveout(rproc, mem); 647 } 648 649 return 0; 650 } 651 652 /** 653 * imx_dsp_rproc_elf_load_segments() - load firmware segments to memory 654 * @rproc: remote processor which will be booted using these fw segments 655 * @fw: the ELF firmware image 656 * 657 * This function specially checks if memsz is zero or not, otherwise it 658 * is mostly same as rproc_elf_load_segments(). 659 */ 660 static int imx_dsp_rproc_elf_load_segments(struct rproc *rproc, 661 const struct firmware *fw) 662 { 663 struct device *dev = &rproc->dev; 664 u8 class = fw_elf_get_class(fw); 665 u32 elf_phdr_get_size = elf_size_of_phdr(class); 666 const u8 *elf_data = fw->data; 667 const void *ehdr, *phdr; 668 int i, ret = 0; 669 u16 phnum; 670 671 ehdr = elf_data; 672 phnum = elf_hdr_get_e_phnum(class, ehdr); 673 phdr = elf_data + elf_hdr_get_e_phoff(class, ehdr); 674 675 /* go through the available ELF segments */ 676 for (i = 0; i < phnum; i++, phdr += elf_phdr_get_size) { 677 u64 da = elf_phdr_get_p_paddr(class, phdr); 678 u64 memsz = elf_phdr_get_p_memsz(class, phdr); 679 u64 filesz = elf_phdr_get_p_filesz(class, phdr); 680 u64 offset = elf_phdr_get_p_offset(class, phdr); 681 u32 type = elf_phdr_get_p_type(class, phdr); 682 void *ptr; 683 684 /* 685 * There is a case that with PT_LOAD type, the 686 * filesz = memsz = 0. If memsz = 0, rproc_da_to_va 687 * should return NULL ptr, then error is returned. 688 * So this case should be skipped from the loop. 689 * Add !memsz checking here. 690 */ 691 if (type != PT_LOAD || !memsz) 692 continue; 693 694 dev_dbg(dev, "phdr: type %d da 0x%llx memsz 0x%llx filesz 0x%llx\n", 695 type, da, memsz, filesz); 696 697 if (filesz > memsz) { 698 dev_err(dev, "bad phdr filesz 0x%llx memsz 0x%llx\n", 699 filesz, memsz); 700 ret = -EINVAL; 701 break; 702 } 703 704 if (offset + filesz > fw->size) { 705 dev_err(dev, "truncated fw: need 0x%llx avail 0x%zx\n", 706 offset + filesz, fw->size); 707 ret = -EINVAL; 708 break; 709 } 710 711 if (!rproc_u64_fit_in_size_t(memsz)) { 712 dev_err(dev, "size (%llx) does not fit in size_t type\n", 713 memsz); 714 ret = -EOVERFLOW; 715 break; 716 } 717 718 /* grab the kernel address for this device address */ 719 ptr = rproc_da_to_va(rproc, da, memsz, NULL); 720 if (!ptr) { 721 dev_err(dev, "bad phdr da 0x%llx mem 0x%llx\n", da, 722 memsz); 723 ret = -EINVAL; 724 break; 725 } 726 727 /* put the segment where the remote processor expects it */ 728 if (filesz) 729 memcpy(ptr, elf_data + offset, filesz); 730 731 /* 732 * Zero out remaining memory for this segment. 733 * 734 * This isn't strictly required since dma_alloc_coherent already 735 * did this for us. albeit harmless, we may consider removing 736 * this. 737 */ 738 if (memsz > filesz) 739 memset(ptr + filesz, 0, memsz - filesz); 740 } 741 742 return ret; 743 } 744 745 /* Prepare function for rproc_ops */ 746 static int imx_dsp_rproc_prepare(struct rproc *rproc) 747 { 748 struct imx_dsp_rproc *priv = rproc->priv; 749 struct device *dev = rproc->dev.parent; 750 struct rproc_mem_entry *carveout; 751 int ret; 752 753 ret = imx_dsp_rproc_add_carveout(priv); 754 if (ret) { 755 dev_err(dev, "failed on imx_dsp_rproc_add_carveout\n"); 756 return ret; 757 } 758 759 pm_runtime_get_sync(dev); 760 761 /* 762 * Clear buffers after pm rumtime for internal ocram is not 763 * accessible if power and clock are not enabled. 764 */ 765 list_for_each_entry(carveout, &rproc->carveouts, node) { 766 if (carveout->va) 767 memset(carveout->va, 0, carveout->len); 768 } 769 770 return 0; 771 } 772 773 /* Unprepare function for rproc_ops */ 774 static int imx_dsp_rproc_unprepare(struct rproc *rproc) 775 { 776 pm_runtime_put_sync(rproc->dev.parent); 777 778 return 0; 779 } 780 781 /* Kick function for rproc_ops */ 782 static void imx_dsp_rproc_kick(struct rproc *rproc, int vqid) 783 { 784 struct imx_dsp_rproc *priv = rproc->priv; 785 struct device *dev = rproc->dev.parent; 786 int err; 787 __u32 mmsg; 788 789 if (!priv->tx_ch) { 790 dev_err(dev, "No initialized mbox tx channel\n"); 791 return; 792 } 793 794 /* 795 * Send the index of the triggered virtqueue as the mu payload. 796 * Let remote processor know which virtqueue is used. 797 */ 798 mmsg = vqid; 799 800 err = mbox_send_message(priv->tx_ch, (void *)&mmsg); 801 if (err < 0) 802 dev_err(dev, "%s: failed (%d, err:%d)\n", __func__, vqid, err); 803 } 804 805 static const struct rproc_ops imx_dsp_rproc_ops = { 806 .prepare = imx_dsp_rproc_prepare, 807 .unprepare = imx_dsp_rproc_unprepare, 808 .start = imx_dsp_rproc_start, 809 .stop = imx_dsp_rproc_stop, 810 .kick = imx_dsp_rproc_kick, 811 .load = imx_dsp_rproc_elf_load_segments, 812 .parse_fw = rproc_elf_load_rsc_table, 813 .sanity_check = rproc_elf_sanity_check, 814 .get_boot_addr = rproc_elf_get_boot_addr, 815 }; 816 817 /** 818 * imx_dsp_attach_pm_domains() - attach the power domains 819 * @priv: private data pointer 820 * 821 * On i.MX8QM and i.MX8QXP there is multiple power domains 822 * required, so need to link them. 823 */ 824 static int imx_dsp_attach_pm_domains(struct imx_dsp_rproc *priv) 825 { 826 struct device *dev = priv->rproc->dev.parent; 827 int ret, i; 828 829 priv->num_domains = of_count_phandle_with_args(dev->of_node, 830 "power-domains", 831 "#power-domain-cells"); 832 833 /* If only one domain, then no need to link the device */ 834 if (priv->num_domains <= 1) 835 return 0; 836 837 priv->pd_dev = devm_kmalloc_array(dev, priv->num_domains, 838 sizeof(*priv->pd_dev), 839 GFP_KERNEL); 840 if (!priv->pd_dev) 841 return -ENOMEM; 842 843 priv->pd_dev_link = devm_kmalloc_array(dev, priv->num_domains, 844 sizeof(*priv->pd_dev_link), 845 GFP_KERNEL); 846 if (!priv->pd_dev_link) 847 return -ENOMEM; 848 849 for (i = 0; i < priv->num_domains; i++) { 850 priv->pd_dev[i] = dev_pm_domain_attach_by_id(dev, i); 851 if (IS_ERR(priv->pd_dev[i])) { 852 ret = PTR_ERR(priv->pd_dev[i]); 853 goto detach_pm; 854 } 855 856 /* 857 * device_link_add will check priv->pd_dev[i], if it is 858 * NULL, then will break. 859 */ 860 priv->pd_dev_link[i] = device_link_add(dev, 861 priv->pd_dev[i], 862 DL_FLAG_STATELESS | 863 DL_FLAG_PM_RUNTIME); 864 if (!priv->pd_dev_link[i]) { 865 dev_pm_domain_detach(priv->pd_dev[i], false); 866 ret = -EINVAL; 867 goto detach_pm; 868 } 869 } 870 871 return 0; 872 873 detach_pm: 874 while (--i >= 0) { 875 device_link_del(priv->pd_dev_link[i]); 876 dev_pm_domain_detach(priv->pd_dev[i], false); 877 } 878 879 return ret; 880 } 881 882 static int imx_dsp_detach_pm_domains(struct imx_dsp_rproc *priv) 883 { 884 int i; 885 886 if (priv->num_domains <= 1) 887 return 0; 888 889 for (i = 0; i < priv->num_domains; i++) { 890 device_link_del(priv->pd_dev_link[i]); 891 dev_pm_domain_detach(priv->pd_dev[i], false); 892 } 893 894 return 0; 895 } 896 897 /** 898 * imx_dsp_rproc_detect_mode() - detect DSP control mode 899 * @priv: private data pointer 900 * 901 * Different platform has different control method for DSP, which depends 902 * on how the DSP is integrated in platform. 903 * 904 * For i.MX8QXP and i.MX8QM, DSP should be started and stopped by System 905 * Control Unit. 906 * For i.MX8MP and i.MX8ULP, DSP should be started and stopped by system 907 * integration module. 908 */ 909 static int imx_dsp_rproc_detect_mode(struct imx_dsp_rproc *priv) 910 { 911 const struct imx_dsp_rproc_dcfg *dsp_dcfg = priv->dsp_dcfg; 912 struct device *dev = priv->rproc->dev.parent; 913 struct regmap *regmap; 914 int ret = 0; 915 916 switch (dsp_dcfg->dcfg->method) { 917 case IMX_RPROC_SCU_API: 918 ret = imx_scu_get_handle(&priv->ipc_handle); 919 if (ret) 920 return ret; 921 break; 922 case IMX_RPROC_MMIO: 923 regmap = syscon_regmap_lookup_by_phandle(dev->of_node, "fsl,dsp-ctrl"); 924 if (IS_ERR(regmap)) { 925 dev_err(dev, "failed to find syscon\n"); 926 return PTR_ERR(regmap); 927 } 928 929 priv->regmap = regmap; 930 break; 931 default: 932 ret = -EOPNOTSUPP; 933 break; 934 } 935 936 return ret; 937 } 938 939 static const char *imx_dsp_clks_names[DSP_RPROC_CLK_MAX] = { 940 /* DSP clocks */ 941 "core", "ocram", "debug", "ipg", "mu", 942 }; 943 944 static int imx_dsp_rproc_clk_get(struct imx_dsp_rproc *priv) 945 { 946 struct device *dev = priv->rproc->dev.parent; 947 struct clk_bulk_data *clks = priv->clks; 948 int i; 949 950 for (i = 0; i < DSP_RPROC_CLK_MAX; i++) 951 clks[i].id = imx_dsp_clks_names[i]; 952 953 return devm_clk_bulk_get_optional(dev, DSP_RPROC_CLK_MAX, clks); 954 } 955 956 static int imx_dsp_rproc_probe(struct platform_device *pdev) 957 { 958 const struct imx_dsp_rproc_dcfg *dsp_dcfg; 959 struct device *dev = &pdev->dev; 960 struct imx_dsp_rproc *priv; 961 struct rproc *rproc; 962 const char *fw_name; 963 int ret; 964 965 dsp_dcfg = of_device_get_match_data(dev); 966 if (!dsp_dcfg) 967 return -ENODEV; 968 969 ret = rproc_of_parse_firmware(dev, 0, &fw_name); 970 if (ret) { 971 dev_err(dev, "failed to parse firmware-name property, ret = %d\n", 972 ret); 973 return ret; 974 } 975 976 rproc = rproc_alloc(dev, "imx-dsp-rproc", &imx_dsp_rproc_ops, fw_name, 977 sizeof(*priv)); 978 if (!rproc) 979 return -ENOMEM; 980 981 priv = rproc->priv; 982 priv->rproc = rproc; 983 priv->dsp_dcfg = dsp_dcfg; 984 985 dev_set_drvdata(dev, rproc); 986 987 INIT_WORK(&priv->rproc_work, imx_dsp_rproc_vq_work); 988 989 ret = imx_dsp_rproc_detect_mode(priv); 990 if (ret) { 991 dev_err(dev, "failed on imx_dsp_rproc_detect_mode\n"); 992 goto err_put_rproc; 993 } 994 995 /* There are multiple power domains required by DSP on some platform */ 996 ret = imx_dsp_attach_pm_domains(priv); 997 if (ret) { 998 dev_err(dev, "failed on imx_dsp_attach_pm_domains\n"); 999 goto err_put_rproc; 1000 } 1001 /* Get clocks */ 1002 ret = imx_dsp_rproc_clk_get(priv); 1003 if (ret) { 1004 dev_err(dev, "failed on imx_dsp_rproc_clk_get\n"); 1005 goto err_detach_domains; 1006 } 1007 1008 init_completion(&priv->pm_comp); 1009 rproc->auto_boot = false; 1010 ret = rproc_add(rproc); 1011 if (ret) { 1012 dev_err(dev, "rproc_add failed\n"); 1013 goto err_detach_domains; 1014 } 1015 1016 pm_runtime_enable(dev); 1017 1018 return 0; 1019 1020 err_detach_domains: 1021 imx_dsp_detach_pm_domains(priv); 1022 err_put_rproc: 1023 rproc_free(rproc); 1024 1025 return ret; 1026 } 1027 1028 static int imx_dsp_rproc_remove(struct platform_device *pdev) 1029 { 1030 struct rproc *rproc = platform_get_drvdata(pdev); 1031 struct imx_dsp_rproc *priv = rproc->priv; 1032 1033 pm_runtime_disable(&pdev->dev); 1034 rproc_del(rproc); 1035 imx_dsp_detach_pm_domains(priv); 1036 rproc_free(rproc); 1037 1038 return 0; 1039 } 1040 1041 /* pm runtime functions */ 1042 static int imx_dsp_runtime_resume(struct device *dev) 1043 { 1044 struct rproc *rproc = dev_get_drvdata(dev); 1045 struct imx_dsp_rproc *priv = rproc->priv; 1046 const struct imx_dsp_rproc_dcfg *dsp_dcfg = priv->dsp_dcfg; 1047 int ret; 1048 1049 /* 1050 * There is power domain attached with mailbox, if setup mailbox 1051 * in probe(), then the power of mailbox is always enabled, 1052 * the power can't be saved. 1053 * So move setup of mailbox to runtime resume. 1054 */ 1055 ret = imx_dsp_rproc_mbox_init(priv); 1056 if (ret) { 1057 dev_err(dev, "failed on imx_dsp_rproc_mbox_init\n"); 1058 return ret; 1059 } 1060 1061 ret = clk_bulk_prepare_enable(DSP_RPROC_CLK_MAX, priv->clks); 1062 if (ret) { 1063 dev_err(dev, "failed on clk_bulk_prepare_enable\n"); 1064 return ret; 1065 } 1066 1067 /* Reset DSP if needed */ 1068 if (dsp_dcfg->reset) 1069 dsp_dcfg->reset(priv); 1070 1071 return 0; 1072 } 1073 1074 static int imx_dsp_runtime_suspend(struct device *dev) 1075 { 1076 struct rproc *rproc = dev_get_drvdata(dev); 1077 struct imx_dsp_rproc *priv = rproc->priv; 1078 1079 clk_bulk_disable_unprepare(DSP_RPROC_CLK_MAX, priv->clks); 1080 1081 imx_dsp_rproc_free_mbox(priv); 1082 1083 return 0; 1084 } 1085 1086 static void imx_dsp_load_firmware(const struct firmware *fw, void *context) 1087 { 1088 struct rproc *rproc = context; 1089 int ret; 1090 1091 /* 1092 * Same flow as start procedure. 1093 * Load the ELF segments to memory firstly. 1094 */ 1095 ret = rproc_load_segments(rproc, fw); 1096 if (ret) 1097 goto out; 1098 1099 /* Start the remote processor */ 1100 ret = rproc->ops->start(rproc); 1101 if (ret) 1102 goto out; 1103 1104 rproc->ops->kick(rproc, 0); 1105 1106 out: 1107 release_firmware(fw); 1108 } 1109 1110 static __maybe_unused int imx_dsp_suspend(struct device *dev) 1111 { 1112 struct rproc *rproc = dev_get_drvdata(dev); 1113 struct imx_dsp_rproc *priv = rproc->priv; 1114 __u32 mmsg = RP_MBOX_SUSPEND_SYSTEM; 1115 int ret; 1116 1117 if (rproc->state != RPROC_RUNNING) 1118 goto out; 1119 1120 reinit_completion(&priv->pm_comp); 1121 1122 /* Tell DSP that suspend is happening */ 1123 ret = mbox_send_message(priv->tx_ch, (void *)&mmsg); 1124 if (ret < 0) { 1125 dev_err(dev, "PM mbox_send_message failed: %d\n", ret); 1126 return ret; 1127 } 1128 1129 /* 1130 * DSP need to save the context at suspend. 1131 * Here waiting the response for DSP, then power can be disabled. 1132 */ 1133 if (!wait_for_completion_timeout(&priv->pm_comp, msecs_to_jiffies(100))) 1134 return -EBUSY; 1135 1136 out: 1137 /* 1138 * The power of DSP is disabled in suspend, so force pm runtime 1139 * to be suspend, then we can reenable the power and clocks at 1140 * resume stage. 1141 */ 1142 return pm_runtime_force_suspend(dev); 1143 } 1144 1145 static __maybe_unused int imx_dsp_resume(struct device *dev) 1146 { 1147 struct rproc *rproc = dev_get_drvdata(dev); 1148 int ret = 0; 1149 1150 ret = pm_runtime_force_resume(dev); 1151 if (ret) 1152 return ret; 1153 1154 if (rproc->state != RPROC_RUNNING) 1155 return 0; 1156 1157 /* 1158 * The power of DSP is disabled at suspend, the memory of dsp 1159 * is reset, the image segments are lost. So need to reload 1160 * firmware and restart the DSP if it is in running state. 1161 */ 1162 ret = request_firmware_nowait(THIS_MODULE, FW_ACTION_UEVENT, 1163 rproc->firmware, dev, GFP_KERNEL, 1164 rproc, imx_dsp_load_firmware); 1165 if (ret < 0) { 1166 dev_err(dev, "load firmware failed: %d\n", ret); 1167 goto err; 1168 } 1169 1170 return 0; 1171 1172 err: 1173 pm_runtime_force_suspend(dev); 1174 1175 return ret; 1176 } 1177 1178 static const struct dev_pm_ops imx_dsp_rproc_pm_ops = { 1179 SET_SYSTEM_SLEEP_PM_OPS(imx_dsp_suspend, imx_dsp_resume) 1180 SET_RUNTIME_PM_OPS(imx_dsp_runtime_suspend, 1181 imx_dsp_runtime_resume, NULL) 1182 }; 1183 1184 static const struct of_device_id imx_dsp_rproc_of_match[] = { 1185 { .compatible = "fsl,imx8qxp-hifi4", .data = &imx_dsp_rproc_cfg_imx8qxp }, 1186 { .compatible = "fsl,imx8qm-hifi4", .data = &imx_dsp_rproc_cfg_imx8qm }, 1187 { .compatible = "fsl,imx8mp-hifi4", .data = &imx_dsp_rproc_cfg_imx8mp }, 1188 { .compatible = "fsl,imx8ulp-hifi4", .data = &imx_dsp_rproc_cfg_imx8ulp }, 1189 {}, 1190 }; 1191 MODULE_DEVICE_TABLE(of, imx_dsp_rproc_of_match); 1192 1193 static struct platform_driver imx_dsp_rproc_driver = { 1194 .probe = imx_dsp_rproc_probe, 1195 .remove = imx_dsp_rproc_remove, 1196 .driver = { 1197 .name = "imx-dsp-rproc", 1198 .of_match_table = imx_dsp_rproc_of_match, 1199 .pm = &imx_dsp_rproc_pm_ops, 1200 }, 1201 }; 1202 module_platform_driver(imx_dsp_rproc_driver); 1203 1204 MODULE_LICENSE("GPL v2"); 1205 MODULE_DESCRIPTION("i.MX HiFi Core Remote Processor Control Driver"); 1206 MODULE_AUTHOR("Shengjiu Wang <shengjiu.wang@nxp.com>"); 1207