1 // SPDX-License-Identifier: GPL-2.0-only 2 // 3 // Copyright(c) 2021-2022 Intel Corporation 4 // 5 // Authors: Cezary Rojewski <cezary.rojewski@intel.com> 6 // Amadeusz Slawinski <amadeuszx.slawinski@linux.intel.com> 7 // 8 9 #include <linux/io-64-nonatomic-lo-hi.h> 10 #include <linux/slab.h> 11 #include <sound/hdaudio_ext.h> 12 #include "avs.h" 13 #include "messages.h" 14 #include "registers.h" 15 #include "trace.h" 16 17 #define AVS_IPC_TIMEOUT_MS 300 18 #define AVS_D0IX_DELAY_MS 300 19 20 static int 21 avs_dsp_set_d0ix(struct avs_dev *adev, bool enable) 22 { 23 struct avs_ipc *ipc = adev->ipc; 24 int ret; 25 26 /* Is transition required? */ 27 if (ipc->in_d0ix == enable) 28 return 0; 29 30 ret = avs_dsp_op(adev, set_d0ix, enable); 31 if (ret) { 32 /* Prevent further d0ix attempts on conscious IPC failure. */ 33 if (ret == -AVS_EIPC) 34 atomic_inc(&ipc->d0ix_disable_depth); 35 36 ipc->in_d0ix = false; 37 return ret; 38 } 39 40 ipc->in_d0ix = enable; 41 return 0; 42 } 43 44 static void avs_dsp_schedule_d0ix(struct avs_dev *adev, struct avs_ipc_msg *tx) 45 { 46 if (atomic_read(&adev->ipc->d0ix_disable_depth)) 47 return; 48 49 mod_delayed_work(system_power_efficient_wq, &adev->ipc->d0ix_work, 50 msecs_to_jiffies(AVS_D0IX_DELAY_MS)); 51 } 52 53 static void avs_dsp_d0ix_work(struct work_struct *work) 54 { 55 struct avs_ipc *ipc = container_of(work, struct avs_ipc, d0ix_work.work); 56 57 avs_dsp_set_d0ix(to_avs_dev(ipc->dev), true); 58 } 59 60 static int avs_dsp_wake_d0i0(struct avs_dev *adev, struct avs_ipc_msg *tx) 61 { 62 struct avs_ipc *ipc = adev->ipc; 63 64 if (!atomic_read(&ipc->d0ix_disable_depth)) { 65 cancel_delayed_work_sync(&ipc->d0ix_work); 66 return avs_dsp_set_d0ix(adev, false); 67 } 68 69 return 0; 70 } 71 72 int avs_dsp_disable_d0ix(struct avs_dev *adev) 73 { 74 struct avs_ipc *ipc = adev->ipc; 75 76 /* Prevent PG only on the first disable. */ 77 if (atomic_inc_return(&ipc->d0ix_disable_depth) == 1) { 78 cancel_delayed_work_sync(&ipc->d0ix_work); 79 return avs_dsp_set_d0ix(adev, false); 80 } 81 82 return 0; 83 } 84 85 int avs_dsp_enable_d0ix(struct avs_dev *adev) 86 { 87 struct avs_ipc *ipc = adev->ipc; 88 89 if (atomic_dec_and_test(&ipc->d0ix_disable_depth)) 90 queue_delayed_work(system_power_efficient_wq, &ipc->d0ix_work, 91 msecs_to_jiffies(AVS_D0IX_DELAY_MS)); 92 return 0; 93 } 94 95 static void avs_dsp_recovery(struct avs_dev *adev) 96 { 97 struct avs_soc_component *acomp; 98 unsigned int core_mask; 99 int ret; 100 101 mutex_lock(&adev->comp_list_mutex); 102 /* disconnect all running streams */ 103 list_for_each_entry(acomp, &adev->comp_list, node) { 104 struct snd_soc_pcm_runtime *rtd; 105 struct snd_soc_card *card; 106 107 card = acomp->base.card; 108 if (!card) 109 continue; 110 111 for_each_card_rtds(card, rtd) { 112 struct snd_pcm *pcm; 113 int dir; 114 115 pcm = rtd->pcm; 116 if (!pcm || rtd->dai_link->no_pcm) 117 continue; 118 119 for_each_pcm_streams(dir) { 120 struct snd_pcm_substream *substream; 121 122 substream = pcm->streams[dir].substream; 123 if (!substream || !substream->runtime) 124 continue; 125 126 /* No need for _irq() as we are in nonatomic context. */ 127 snd_pcm_stream_lock(substream); 128 snd_pcm_stop(substream, SNDRV_PCM_STATE_DISCONNECTED); 129 snd_pcm_stream_unlock(substream); 130 } 131 } 132 } 133 mutex_unlock(&adev->comp_list_mutex); 134 135 /* forcibly shutdown all cores */ 136 core_mask = GENMASK(adev->hw_cfg.dsp_cores - 1, 0); 137 avs_dsp_core_disable(adev, core_mask); 138 139 /* attempt dsp reboot */ 140 ret = avs_dsp_boot_firmware(adev, true); 141 if (ret < 0) 142 dev_err(adev->dev, "dsp reboot failed: %d\n", ret); 143 144 pm_runtime_mark_last_busy(adev->dev); 145 pm_runtime_enable(adev->dev); 146 pm_request_autosuspend(adev->dev); 147 148 atomic_set(&adev->ipc->recovering, 0); 149 } 150 151 static void avs_dsp_recovery_work(struct work_struct *work) 152 { 153 struct avs_ipc *ipc = container_of(work, struct avs_ipc, recovery_work); 154 155 avs_dsp_recovery(to_avs_dev(ipc->dev)); 156 } 157 158 static void avs_dsp_exception_caught(struct avs_dev *adev, union avs_notify_msg *msg) 159 { 160 struct avs_ipc *ipc = adev->ipc; 161 162 /* Account for the double-exception case. */ 163 ipc->ready = false; 164 165 if (!atomic_add_unless(&ipc->recovering, 1, 1)) { 166 dev_err(adev->dev, "dsp recovery is already in progress\n"); 167 return; 168 } 169 170 dev_crit(adev->dev, "communication severed, rebooting dsp..\n"); 171 172 cancel_delayed_work_sync(&ipc->d0ix_work); 173 ipc->in_d0ix = false; 174 /* Re-enabled on recovery completion. */ 175 pm_runtime_disable(adev->dev); 176 177 /* Process received notification. */ 178 avs_dsp_op(adev, coredump, msg); 179 180 schedule_work(&ipc->recovery_work); 181 } 182 183 static void avs_dsp_receive_rx(struct avs_dev *adev, u64 header) 184 { 185 struct avs_ipc *ipc = adev->ipc; 186 union avs_reply_msg msg = AVS_MSG(header); 187 u32 sts, lec; 188 189 sts = snd_hdac_adsp_readl(adev, AVS_FW_REG_STATUS(adev)); 190 lec = snd_hdac_adsp_readl(adev, AVS_FW_REG_ERROR(adev)); 191 trace_avs_ipc_reply_msg(header, sts, lec); 192 193 ipc->rx.header = header; 194 /* Abort copying payload if request processing was unsuccessful. */ 195 if (!msg.status) { 196 /* update size in case of LARGE_CONFIG_GET */ 197 if (msg.msg_target == AVS_MOD_MSG && 198 msg.global_msg_type == AVS_MOD_LARGE_CONFIG_GET) 199 ipc->rx.size = min_t(u32, AVS_MAILBOX_SIZE, 200 msg.ext.large_config.data_off_size); 201 202 memcpy_fromio(ipc->rx.data, avs_uplink_addr(adev), ipc->rx.size); 203 trace_avs_msg_payload(ipc->rx.data, ipc->rx.size); 204 } 205 } 206 207 static void avs_dsp_process_notification(struct avs_dev *adev, u64 header) 208 { 209 struct avs_notify_mod_data mod_data; 210 union avs_notify_msg msg = AVS_MSG(header); 211 size_t data_size = 0; 212 void *data = NULL; 213 u32 sts, lec; 214 215 sts = snd_hdac_adsp_readl(adev, AVS_FW_REG_STATUS(adev)); 216 lec = snd_hdac_adsp_readl(adev, AVS_FW_REG_ERROR(adev)); 217 trace_avs_ipc_notify_msg(header, sts, lec); 218 219 /* Ignore spurious notifications until handshake is established. */ 220 if (!adev->ipc->ready && msg.notify_msg_type != AVS_NOTIFY_FW_READY) { 221 dev_dbg(adev->dev, "FW not ready, skip notification: 0x%08x\n", msg.primary); 222 return; 223 } 224 225 /* Calculate notification payload size. */ 226 switch (msg.notify_msg_type) { 227 case AVS_NOTIFY_FW_READY: 228 break; 229 230 case AVS_NOTIFY_PHRASE_DETECTED: 231 data_size = sizeof(struct avs_notify_voice_data); 232 break; 233 234 case AVS_NOTIFY_RESOURCE_EVENT: 235 data_size = sizeof(struct avs_notify_res_data); 236 break; 237 238 case AVS_NOTIFY_LOG_BUFFER_STATUS: 239 case AVS_NOTIFY_EXCEPTION_CAUGHT: 240 break; 241 242 case AVS_NOTIFY_MODULE_EVENT: 243 /* To know the total payload size, header needs to be read first. */ 244 memcpy_fromio(&mod_data, avs_uplink_addr(adev), sizeof(mod_data)); 245 data_size = sizeof(mod_data) + mod_data.data_size; 246 break; 247 248 default: 249 dev_info(adev->dev, "unknown notification: 0x%08x\n", msg.primary); 250 break; 251 } 252 253 if (data_size) { 254 data = kmalloc(data_size, GFP_KERNEL); 255 if (!data) 256 return; 257 258 memcpy_fromio(data, avs_uplink_addr(adev), data_size); 259 trace_avs_msg_payload(data, data_size); 260 } 261 262 /* Perform notification-specific operations. */ 263 switch (msg.notify_msg_type) { 264 case AVS_NOTIFY_FW_READY: 265 dev_dbg(adev->dev, "FW READY 0x%08x\n", msg.primary); 266 adev->ipc->ready = true; 267 complete(&adev->fw_ready); 268 break; 269 270 case AVS_NOTIFY_LOG_BUFFER_STATUS: 271 avs_log_buffer_status_locked(adev, &msg); 272 break; 273 274 case AVS_NOTIFY_EXCEPTION_CAUGHT: 275 avs_dsp_exception_caught(adev, &msg); 276 break; 277 278 default: 279 break; 280 } 281 282 kfree(data); 283 } 284 285 void avs_dsp_process_response(struct avs_dev *adev, u64 header) 286 { 287 struct avs_ipc *ipc = adev->ipc; 288 289 /* 290 * Response may either be solicited - a reply for a request that has 291 * been sent beforehand - or unsolicited (notification). 292 */ 293 if (avs_msg_is_reply(header)) { 294 /* Response processing is invoked from IRQ thread. */ 295 spin_lock_irq(&ipc->rx_lock); 296 avs_dsp_receive_rx(adev, header); 297 ipc->rx_completed = true; 298 spin_unlock_irq(&ipc->rx_lock); 299 } else { 300 avs_dsp_process_notification(adev, header); 301 } 302 303 complete(&ipc->busy_completion); 304 } 305 306 static bool avs_ipc_is_busy(struct avs_ipc *ipc) 307 { 308 struct avs_dev *adev = to_avs_dev(ipc->dev); 309 const struct avs_spec *const spec = adev->spec; 310 u32 hipc_rsp; 311 312 hipc_rsp = snd_hdac_adsp_readl(adev, spec->hipc->rsp_offset); 313 return hipc_rsp & spec->hipc->rsp_busy_mask; 314 } 315 316 static int avs_ipc_wait_busy_completion(struct avs_ipc *ipc, int timeout) 317 { 318 u32 repeats_left = 128; /* to avoid infinite looping */ 319 int ret; 320 321 again: 322 ret = wait_for_completion_timeout(&ipc->busy_completion, msecs_to_jiffies(timeout)); 323 324 /* DSP could be unresponsive at this point. */ 325 if (!ipc->ready) 326 return -EPERM; 327 328 if (!ret) { 329 if (!avs_ipc_is_busy(ipc)) 330 return -ETIMEDOUT; 331 /* 332 * Firmware did its job, either notification or reply 333 * has been received - now wait until it's processed. 334 */ 335 wait_for_completion_killable(&ipc->busy_completion); 336 } 337 338 /* Ongoing notification's bottom-half may cause early wakeup */ 339 spin_lock(&ipc->rx_lock); 340 if (!ipc->rx_completed) { 341 if (repeats_left) { 342 /* Reply delayed due to notification. */ 343 repeats_left--; 344 reinit_completion(&ipc->busy_completion); 345 spin_unlock(&ipc->rx_lock); 346 goto again; 347 } 348 349 spin_unlock(&ipc->rx_lock); 350 return -ETIMEDOUT; 351 } 352 353 spin_unlock(&ipc->rx_lock); 354 return 0; 355 } 356 357 static void avs_ipc_msg_init(struct avs_ipc *ipc, struct avs_ipc_msg *reply) 358 { 359 lockdep_assert_held(&ipc->rx_lock); 360 361 ipc->rx.header = 0; 362 ipc->rx.size = reply ? reply->size : 0; 363 ipc->rx_completed = false; 364 365 reinit_completion(&ipc->done_completion); 366 reinit_completion(&ipc->busy_completion); 367 } 368 369 static void avs_dsp_send_tx(struct avs_dev *adev, struct avs_ipc_msg *tx, bool read_fwregs) 370 { 371 const struct avs_spec *const spec = adev->spec; 372 u32 sts = UINT_MAX; 373 u32 lec = UINT_MAX; 374 375 tx->header |= spec->hipc->req_busy_mask; 376 if (read_fwregs) { 377 sts = snd_hdac_adsp_readl(adev, AVS_FW_REG_STATUS(adev)); 378 lec = snd_hdac_adsp_readl(adev, AVS_FW_REG_ERROR(adev)); 379 } 380 381 trace_avs_request(tx, sts, lec); 382 383 if (tx->size) 384 memcpy_toio(avs_downlink_addr(adev), tx->data, tx->size); 385 snd_hdac_adsp_writel(adev, spec->hipc->req_ext_offset, tx->header >> 32); 386 snd_hdac_adsp_writel(adev, spec->hipc->req_offset, tx->header & UINT_MAX); 387 } 388 389 static int avs_dsp_do_send_msg(struct avs_dev *adev, struct avs_ipc_msg *request, 390 struct avs_ipc_msg *reply, int timeout, const char *name) 391 { 392 struct avs_ipc *ipc = adev->ipc; 393 int ret; 394 395 if (!ipc->ready) 396 return -EPERM; 397 398 mutex_lock(&ipc->msg_mutex); 399 400 spin_lock(&ipc->rx_lock); 401 avs_ipc_msg_init(ipc, reply); 402 avs_dsp_send_tx(adev, request, true); 403 spin_unlock(&ipc->rx_lock); 404 405 ret = avs_ipc_wait_busy_completion(ipc, timeout); 406 if (ret) { 407 if (ret == -ETIMEDOUT) { 408 union avs_notify_msg msg = AVS_NOTIFICATION(EXCEPTION_CAUGHT); 409 410 /* Same treatment as on exception, just stack_dump=0. */ 411 avs_dsp_exception_caught(adev, &msg); 412 } 413 goto exit; 414 } 415 416 ret = ipc->rx.rsp.status; 417 /* 418 * If IPC channel is blocked e.g.: due to ongoing recovery, 419 * -EPERM error code is expected and thus it's not an actual error. 420 * 421 * Unsupported IPCs are of no harm either. 422 */ 423 if (ret == -EPERM || ret == AVS_IPC_NOT_SUPPORTED) 424 dev_dbg(adev->dev, "%s (0x%08x 0x%08x) failed: %d\n", 425 name, request->glb.primary, request->glb.ext.val, ret); 426 else if (ret) 427 dev_err(adev->dev, "%s (0x%08x 0x%08x) failed: %d\n", 428 name, request->glb.primary, request->glb.ext.val, ret); 429 430 if (reply) { 431 reply->header = ipc->rx.header; 432 reply->size = ipc->rx.size; 433 if (reply->data && ipc->rx.size) 434 memcpy(reply->data, ipc->rx.data, reply->size); 435 } 436 437 exit: 438 mutex_unlock(&ipc->msg_mutex); 439 return ret; 440 } 441 442 static int avs_dsp_send_msg_sequence(struct avs_dev *adev, struct avs_ipc_msg *request, 443 struct avs_ipc_msg *reply, int timeout, bool wake_d0i0, 444 bool schedule_d0ix, const char *name) 445 { 446 int ret; 447 448 trace_avs_d0ix("wake", wake_d0i0, request->header); 449 if (wake_d0i0) { 450 ret = avs_dsp_wake_d0i0(adev, request); 451 if (ret) 452 return ret; 453 } 454 455 ret = avs_dsp_do_send_msg(adev, request, reply, timeout, name); 456 if (ret) 457 return ret; 458 459 trace_avs_d0ix("schedule", schedule_d0ix, request->header); 460 if (schedule_d0ix) 461 avs_dsp_schedule_d0ix(adev, request); 462 463 return 0; 464 } 465 466 int avs_dsp_send_msg_timeout(struct avs_dev *adev, struct avs_ipc_msg *request, 467 struct avs_ipc_msg *reply, int timeout, const char *name) 468 { 469 bool wake_d0i0 = avs_dsp_op(adev, d0ix_toggle, request, true); 470 bool schedule_d0ix = avs_dsp_op(adev, d0ix_toggle, request, false); 471 472 return avs_dsp_send_msg_sequence(adev, request, reply, timeout, wake_d0i0, schedule_d0ix, 473 name); 474 } 475 476 int avs_dsp_send_msg(struct avs_dev *adev, struct avs_ipc_msg *request, 477 struct avs_ipc_msg *reply, const char *name) 478 { 479 return avs_dsp_send_msg_timeout(adev, request, reply, adev->ipc->default_timeout_ms, name); 480 } 481 482 int avs_dsp_send_pm_msg_timeout(struct avs_dev *adev, struct avs_ipc_msg *request, 483 struct avs_ipc_msg *reply, int timeout, bool wake_d0i0, 484 const char *name) 485 { 486 return avs_dsp_send_msg_sequence(adev, request, reply, timeout, wake_d0i0, false, name); 487 } 488 489 int avs_dsp_send_pm_msg(struct avs_dev *adev, struct avs_ipc_msg *request, 490 struct avs_ipc_msg *reply, bool wake_d0i0, const char *name) 491 { 492 return avs_dsp_send_pm_msg_timeout(adev, request, reply, adev->ipc->default_timeout_ms, 493 wake_d0i0, name); 494 } 495 496 static int avs_dsp_do_send_rom_msg(struct avs_dev *adev, struct avs_ipc_msg *request, int timeout, 497 const char *name) 498 { 499 struct avs_ipc *ipc = adev->ipc; 500 int ret; 501 502 mutex_lock(&ipc->msg_mutex); 503 504 spin_lock(&ipc->rx_lock); 505 avs_ipc_msg_init(ipc, NULL); 506 /* 507 * with hw still stalled, memory windows may not be 508 * configured properly so avoid accessing SRAM 509 */ 510 avs_dsp_send_tx(adev, request, false); 511 spin_unlock(&ipc->rx_lock); 512 513 /* ROM messages must be sent before main core is unstalled */ 514 ret = avs_dsp_op(adev, stall, AVS_MAIN_CORE_MASK, false); 515 if (!ret) { 516 ret = wait_for_completion_timeout(&ipc->done_completion, msecs_to_jiffies(timeout)); 517 ret = ret ? 0 : -ETIMEDOUT; 518 } 519 if (ret) 520 dev_err(adev->dev, "%s (0x%08x 0x%08x) failed: %d\n", 521 name, request->glb.primary, request->glb.ext.val, ret); 522 523 mutex_unlock(&ipc->msg_mutex); 524 525 return ret; 526 } 527 528 int avs_dsp_send_rom_msg_timeout(struct avs_dev *adev, struct avs_ipc_msg *request, int timeout, 529 const char *name) 530 { 531 return avs_dsp_do_send_rom_msg(adev, request, timeout, name); 532 } 533 534 int avs_dsp_send_rom_msg(struct avs_dev *adev, struct avs_ipc_msg *request, const char *name) 535 { 536 return avs_dsp_send_rom_msg_timeout(adev, request, adev->ipc->default_timeout_ms, name); 537 } 538 539 void avs_dsp_interrupt_control(struct avs_dev *adev, bool enable) 540 { 541 const struct avs_spec *const spec = adev->spec; 542 u32 value, mask; 543 544 /* 545 * No particular bit setting order. All of these are required 546 * to have a functional SW <-> FW communication. 547 */ 548 value = enable ? AVS_ADSP_ADSPIC_IPC : 0; 549 snd_hdac_adsp_updatel(adev, AVS_ADSP_REG_ADSPIC, AVS_ADSP_ADSPIC_IPC, value); 550 551 mask = AVS_ADSP_HIPCCTL_DONE | AVS_ADSP_HIPCCTL_BUSY; 552 value = enable ? mask : 0; 553 snd_hdac_adsp_updatel(adev, spec->hipc->ctl_offset, mask, value); 554 } 555 556 int avs_ipc_init(struct avs_ipc *ipc, struct device *dev) 557 { 558 ipc->rx.data = devm_kzalloc(dev, AVS_MAILBOX_SIZE, GFP_KERNEL); 559 if (!ipc->rx.data) 560 return -ENOMEM; 561 562 ipc->dev = dev; 563 ipc->ready = false; 564 ipc->default_timeout_ms = AVS_IPC_TIMEOUT_MS; 565 INIT_WORK(&ipc->recovery_work, avs_dsp_recovery_work); 566 INIT_DELAYED_WORK(&ipc->d0ix_work, avs_dsp_d0ix_work); 567 init_completion(&ipc->done_completion); 568 init_completion(&ipc->busy_completion); 569 spin_lock_init(&ipc->rx_lock); 570 mutex_init(&ipc->msg_mutex); 571 572 return 0; 573 } 574 575 void avs_ipc_block(struct avs_ipc *ipc) 576 { 577 ipc->ready = false; 578 cancel_work_sync(&ipc->recovery_work); 579 cancel_delayed_work_sync(&ipc->d0ix_work); 580 ipc->in_d0ix = false; 581 } 582