1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause)
14 #include "sof-priv.h"
15 #include "sof-audio.h"
16 #include "ipc4-fw-reg.h"
17 #include "ipc4-priv.h"
18 #include "ipc4-telemetry.h"
28 	{3, "Not enough space in the IPC reply buffer to complete the request"},
30 	{5, "Replaced ADSP IPC PENDING (unused)"},
95 			dev_err(sdev->dev, "FW reported error: %u - %s\n",  in sof_ipc4_check_reply_status()
102 		dev_err(sdev->dev, "FW reported error: %u - Unknown\n", status);  in sof_ipc4_check_reply_status()
108 		ret = -EOPNOTSUPP;  in sof_ipc4_check_reply_status()
116 		ret = -ENOENT;  in sof_ipc4_check_reply_status()
121 		ret = -EBUSY;  in sof_ipc4_check_reply_status()
124 		ret = -EINVAL;  in sof_ipc4_check_reply_status()
190 	val = msg->primary & SOF_IPC4_MSG_TARGET_MASK;  in sof_ipc4_log_header()
191 	type = SOF_IPC4_MSG_TYPE_GET(msg->primary);  in sof_ipc4_log_header()
208 			u32 notif = SOF_IPC4_NOTIFICATION_TYPE_GET(msg->primary);  in sof_ipc4_log_header()
223 		if (data_size_valid && msg->data_size)  in sof_ipc4_log_header()
225 				text, msg->primary, msg->extension, str, str2,  in sof_ipc4_log_header()
226 				msg->data_size);  in sof_ipc4_log_header()
228 			dev_dbg(dev, "%s: %#x|%#x: %s|%s\n", text, msg->primary,  in sof_ipc4_log_header()
229 				msg->extension, str, str2);  in sof_ipc4_log_header()
231 		if (data_size_valid && msg->data_size)  in sof_ipc4_log_header()
233 				text, msg->primary, msg->extension, str,  in sof_ipc4_log_header()
234 				msg->data_size);  in sof_ipc4_log_header()
236 			dev_dbg(dev, "%s: %#x|%#x: %s\n", text, msg->primary,  in sof_ipc4_log_header()
237 				msg->extension, str);  in sof_ipc4_log_header()
246 	    !SOF_IPC4_MSG_IS_MODULE_MSG(msg->primary) &&  in sof_ipc4_log_header()
247 	    SOF_IPC4_MSG_TYPE_GET(msg->primary) == SOF_IPC4_GLB_NOTIFICATION &&  in sof_ipc4_log_header()
248 	    SOF_IPC4_NOTIFICATION_TYPE_GET(msg->primary) == SOF_IPC4_NOTIFY_LOG_BUFFER_STATUS)  in sof_ipc4_log_header()
251 	if (data_size_valid && msg->data_size)  in sof_ipc4_log_header()
253 			msg->primary, msg->extension, msg->data_size);  in sof_ipc4_log_header()
255 		dev_dbg(dev, "%s: %#x|%#x\n", text, msg->primary, msg->extension);  in sof_ipc4_log_header()
268 	struct snd_sof_ipc_msg *msg = sdev->msg;  in sof_ipc4_get_reply()
273 	ipc4_reply = msg->reply_data;  in sof_ipc4_get_reply()
275 	sof_ipc4_log_header(sdev->dev, "ipc tx reply", ipc4_reply, false);  in sof_ipc4_get_reply()
277 	ret = sof_ipc4_check_reply_status(sdev, ipc4_reply->primary);  in sof_ipc4_get_reply()
282 	if (!msg->reply_size || !SOF_IPC4_MSG_IS_MODULE_MSG(ipc4_reply->primary) ||  in sof_ipc4_get_reply()
283 	    (SOF_IPC4_MSG_TYPE_GET(ipc4_reply->primary) != SOF_IPC4_MOD_LARGE_CONFIG_GET))  in sof_ipc4_get_reply()
287 	snd_sof_dsp_mailbox_read(sdev, sdev->dsp_box.offset, ipc4_reply->data_ptr,  in sof_ipc4_get_reply()
288 				 msg->reply_size);  in sof_ipc4_get_reply()
293 /* wait for IPC message reply */
294 static int ipc4_wait_tx_done(struct snd_sof_ipc *ipc, void *reply_data)  in ipc4_wait_tx_done()  argument
296 	struct snd_sof_ipc_msg *msg = &ipc->msg;  in ipc4_wait_tx_done()
297 	struct sof_ipc4_msg *ipc4_msg = msg->msg_data;  in ipc4_wait_tx_done()
298 	struct snd_sof_dev *sdev = ipc->sdev;  in ipc4_wait_tx_done()
301 	/* wait for DSP IPC completion */  in ipc4_wait_tx_done()
302 	ret = wait_event_timeout(msg->waitq, msg->ipc_complete,  in ipc4_wait_tx_done()
303 				 msecs_to_jiffies(sdev->ipc_timeout));  in ipc4_wait_tx_done()
305 		dev_err(sdev->dev, "ipc timed out for %#x|%#x\n",  in ipc4_wait_tx_done()
306 			ipc4_msg->primary, ipc4_msg->extension);  in ipc4_wait_tx_done()
307 		snd_sof_handle_fw_exception(ipc->sdev, "IPC timeout");  in ipc4_wait_tx_done()
308 		return -ETIMEDOUT;  in ipc4_wait_tx_done()
311 	if (msg->reply_error) {  in ipc4_wait_tx_done()
312 		dev_err(sdev->dev, "ipc error for msg %#x|%#x\n",  in ipc4_wait_tx_done()
313 			ipc4_msg->primary, ipc4_msg->extension);  in ipc4_wait_tx_done()
314 		ret =  msg->reply_error;  in ipc4_wait_tx_done()
317 			struct sof_ipc4_msg *ipc4_reply = msg->reply_data;  in ipc4_wait_tx_done()
321 			ipc4_reply_data->header_u64 = ipc4_reply->header_u64;  in ipc4_wait_tx_done()
322 			if (msg->reply_size && ipc4_reply_data->data_ptr) {  in ipc4_wait_tx_done()
324 				memcpy(ipc4_reply_data->data_ptr, ipc4_reply->data_ptr,  in ipc4_wait_tx_done()
325 				       msg->reply_size);  in ipc4_wait_tx_done()
326 				ipc4_reply_data->data_size = msg->reply_size;  in ipc4_wait_tx_done()
331 		sof_ipc4_log_header(sdev->dev, "ipc tx done ", ipc4_msg, true);  in ipc4_wait_tx_done()
334 	/* re-enable dumps after successful IPC tx */  in ipc4_wait_tx_done()
335 	if (sdev->ipc_dump_printed) {  in ipc4_wait_tx_done()
336 		sdev->dbg_dump_printed = false;  in ipc4_wait_tx_done()
337 		sdev->ipc_dump_printed = false;  in ipc4_wait_tx_done()
343 static int ipc4_tx_msg_unlocked(struct snd_sof_ipc *ipc,  in ipc4_tx_msg_unlocked()  argument
348 	struct snd_sof_dev *sdev = ipc->sdev;  in ipc4_tx_msg_unlocked()
351 	if (msg_bytes > ipc->max_payload_size || reply_bytes > ipc->max_payload_size)  in ipc4_tx_msg_unlocked()
352 		return -EINVAL;  in ipc4_tx_msg_unlocked()
354 	sof_ipc4_log_header(sdev->dev, "ipc tx      ", msg_data, true);  in ipc4_tx_msg_unlocked()
358 		dev_err_ratelimited(sdev->dev,  in ipc4_tx_msg_unlocked()
359 				    "%s: ipc message send for %#x|%#x failed: %d\n",  in ipc4_tx_msg_unlocked()
360 				    __func__, ipc4_msg->primary, ipc4_msg->extension, ret);  in ipc4_tx_msg_unlocked()
365 	return ipc4_wait_tx_done(ipc, reply_data);  in ipc4_tx_msg_unlocked()
371 	struct snd_sof_ipc *ipc = sdev->ipc;  in sof_ipc4_tx_msg()  local
375 		return -EINVAL;  in sof_ipc4_tx_msg()
382 		/* ensure the DSP is in D0i0 before sending a new IPC */  in sof_ipc4_tx_msg()
388 	/* Serialise IPC TX */  in sof_ipc4_tx_msg()
389 	mutex_lock(&ipc->tx_mutex);  in sof_ipc4_tx_msg()
391 	ret = ipc4_tx_msg_unlocked(ipc, msg_data, msg_bytes, reply_data, reply_bytes);  in sof_ipc4_tx_msg()
403 			sof_ipc4_dump_payload(sdev, msg->data_ptr, msg->data_size);  in sof_ipc4_tx_msg()
406 	mutex_unlock(&ipc->tx_mutex);  in sof_ipc4_tx_msg()
417 	size_t payload_limit = sdev->ipc->max_payload_size;  in sof_ipc4_set_get_data()
427 		return -EINVAL;  in sof_ipc4_set_get_data()
429 	if ((ipc4_msg->primary & SOF_IPC4_MSG_TARGET_MASK) !=  in sof_ipc4_set_get_data()
431 		return -EINVAL;  in sof_ipc4_set_get_data()
433 	ipc4_msg->primary &= ~SOF_IPC4_MSG_TYPE_MASK;  in sof_ipc4_set_get_data()
434 	tx.primary = ipc4_msg->primary;  in sof_ipc4_set_get_data()
435 	tx.extension = ipc4_msg->extension;  in sof_ipc4_set_get_data()
447 	/* ensure the DSP is in D0i0 before sending IPC */  in sof_ipc4_set_get_data()
452 	/* Serialise IPC TX */  in sof_ipc4_set_get_data()
453 	mutex_lock(&sdev->ipc->tx_mutex);  in sof_ipc4_set_get_data()
474 			tx.data_ptr = ipc4_msg->data_ptr + offset;  in sof_ipc4_set_get_data()
482 			rx.data_ptr = ipc4_msg->data_ptr + offset;  in sof_ipc4_set_get_data()
489 		ret = ipc4_tx_msg_unlocked(sdev->ipc, &tx, tx_size, &rx, rx_size);  in sof_ipc4_set_get_data()
491 			dev_err(sdev->dev,  in sof_ipc4_set_get_data()
502 				dev_err(sdev->dev,  in sof_ipc4_set_get_data()
505 				ret = -ENOMEM;  in sof_ipc4_set_get_data()
518 		remaining -= chunk_size;  in sof_ipc4_set_get_data()
523 		ipc4_msg->data_size = offset;  in sof_ipc4_set_get_data()
527 		sof_ipc4_dump_payload(sdev, ipc4_msg->data_ptr, ipc4_msg->data_size);  in sof_ipc4_set_get_data()
529 	mutex_unlock(&sdev->ipc->tx_mutex);  in sof_ipc4_set_get_data()
537 	struct snd_sof_ipc_msg *msg = &sdev->ipc->msg;  in sof_ipc4_init_msg_memory()
540 	sdev->ipc->max_payload_size = SOF_IPC4_MSG_MAX_SIZE;  in sof_ipc4_init_msg_memory()
543 	msg->reply_data = devm_kzalloc(sdev->dev, sdev->ipc->max_payload_size +  in sof_ipc4_init_msg_memory()
545 	if (!msg->reply_data)  in sof_ipc4_init_msg_memory()
546 		return -ENOMEM;  in sof_ipc4_init_msg_memory()
548 	ipc4_msg = msg->reply_data;  in sof_ipc4_init_msg_memory()
549 	ipc4_msg->data_ptr = msg->reply_data + sizeof(struct sof_ipc4_msg);  in sof_ipc4_init_msg_memory()
562 	slot_desc_type_offset = sdev->debug_box.offset + sizeof(u32);  in sof_ipc4_find_debug_slot_offset_by_type()
567 			return sdev->debug_box.offset + (i + 1) * SOF_IPC4_DEBUG_SLOT_SIZE;  in sof_ipc4_find_debug_slot_offset_by_type()
572 	dev_dbg(sdev->dev, "Slot type %#x is not available in debug window\n", slot_type);  in sof_ipc4_find_debug_slot_offset_by_type()
579 	/* no need to re-check version/ABI for subsequent boots */  in ipc4_fw_ready()
580 	if (!sdev->first_boot)  in ipc4_fw_ready()
591 	struct sof_ipc4_notify_module_data *data = ipc4_msg->data_ptr;  in sof_ipc4_module_notification_handler()
595 	 * we need to re-allocate the buffer and re-read the whole payload,  in sof_ipc4_module_notification_handler()
598 	if (data->event_data_size) {  in sof_ipc4_module_notification_handler()
602 		ipc4_msg->data_size += data->event_data_size;  in sof_ipc4_module_notification_handler()
604 		new = krealloc(ipc4_msg->data_ptr, ipc4_msg->data_size, GFP_KERNEL);  in sof_ipc4_module_notification_handler()
606 			ipc4_msg->data_size -= data->event_data_size;  in sof_ipc4_module_notification_handler()
610 		/* re-read the whole payload */  in sof_ipc4_module_notification_handler()
611 		ipc4_msg->data_ptr = new;  in sof_ipc4_module_notification_handler()
612 		ret = snd_sof_ipc_msg_data(sdev, NULL, ipc4_msg->data_ptr,  in sof_ipc4_module_notification_handler()
613 					   ipc4_msg->data_size);  in sof_ipc4_module_notification_handler()
615 			dev_err(sdev->dev,  in sof_ipc4_module_notification_handler()
620 		data = ipc4_msg->data_ptr;  in sof_ipc4_module_notification_handler()
624 	if ((data->event_id & SOF_IPC4_NOTIFY_MODULE_EVENTID_ALSA_MAGIC_MASK) ==  in sof_ipc4_module_notification_handler()
626 		const struct sof_ipc_tplg_ops *tplg_ops = sdev->ipc->ops->tplg;  in sof_ipc4_module_notification_handler()
628 		if (tplg_ops->control->update)  in sof_ipc4_module_notification_handler()
629 			tplg_ops->control->update(sdev, ipc4_msg);  in sof_ipc4_module_notification_handler()
635 	struct sof_ipc4_msg *ipc4_msg = sdev->ipc->msg.rx_data;  in sof_ipc4_rx_msg()
640 	if (!ipc4_msg || !SOF_IPC4_MSG_IS_NOTIFICATION(ipc4_msg->primary))  in sof_ipc4_rx_msg()
643 	ipc4_msg->data_ptr = NULL;  in sof_ipc4_rx_msg()
644 	ipc4_msg->data_size = 0;  in sof_ipc4_rx_msg()
646 	sof_ipc4_log_header(sdev->dev, "ipc rx      ", ipc4_msg, false);  in sof_ipc4_rx_msg()
648 	switch (SOF_IPC4_NOTIFICATION_TYPE_GET(ipc4_msg->primary)) {  in sof_ipc4_rx_msg()
651 		if (sdev->fw_state == SOF_FW_BOOT_IN_PROGRESS) {  in sof_ipc4_rx_msg()
659 			wake_up(&sdev->boot_wait);  in sof_ipc4_rx_msg()
667 		sof_ipc4_mtrace_update_pos(sdev, SOF_IPC4_LOG_CORE_GET(ipc4_msg->primary));  in sof_ipc4_rx_msg()
677 		dev_dbg(sdev->dev, "Unhandled DSP message: %#x|%#x\n",  in sof_ipc4_rx_msg()
678 			ipc4_msg->primary, ipc4_msg->extension);  in sof_ipc4_rx_msg()
683 		ipc4_msg->data_ptr = kmalloc(data_size, GFP_KERNEL);  in sof_ipc4_rx_msg()
684 		if (!ipc4_msg->data_ptr)  in sof_ipc4_rx_msg()
687 		ipc4_msg->data_size = data_size;  in sof_ipc4_rx_msg()
688 		err = snd_sof_ipc_msg_data(sdev, NULL, ipc4_msg->data_ptr, ipc4_msg->data_size);  in sof_ipc4_rx_msg()
690 			dev_err(sdev->dev, "failed to read IPC notification data: %d\n", err);  in sof_ipc4_rx_msg()
691 			kfree(ipc4_msg->data_ptr);  in sof_ipc4_rx_msg()
692 			ipc4_msg->data_ptr = NULL;  in sof_ipc4_rx_msg()
693 			ipc4_msg->data_size = 0;  in sof_ipc4_rx_msg()
702 	sof_ipc4_log_header(sdev->dev, "ipc rx done ", ipc4_msg, true);  in sof_ipc4_rx_msg()
706 			sof_ipc4_dump_payload(sdev, ipc4_msg->data_ptr,  in sof_ipc4_rx_msg()
707 					      ipc4_msg->data_size);  in sof_ipc4_rx_msg()
709 		kfree(ipc4_msg->data_ptr);  in sof_ipc4_rx_msg()
710 		ipc4_msg->data_ptr = NULL;  in sof_ipc4_rx_msg()
711 		ipc4_msg->data_size = 0;  in sof_ipc4_rx_msg()
770 	struct sof_ipc4_fw_data *ipc4_data = sdev->private;  in sof_ipc4_init()
773 	mutex_init(&ipc4_data->pipeline_state_mutex);  in sof_ipc4_init()
775 	xa_init_flags(&ipc4_data->fw_lib_xa, XA_FLAGS_ALLOC);  in sof_ipc4_init()
777 	/* Set up the windows for IPC communication */  in sof_ipc4_init()
780 		dev_err(sdev->dev, "%s: No mailbox offset\n", __func__);  in sof_ipc4_init()
784 	sdev->dsp_box.offset = inbox_offset;  in sof_ipc4_init()
785 	sdev->dsp_box.size = SOF_IPC4_MSG_MAX_SIZE;  in sof_ipc4_init()
786 	sdev->host_box.offset = snd_sof_dsp_get_window_offset(sdev,  in sof_ipc4_init()
788 	sdev->host_box.size = SOF_IPC4_MSG_MAX_SIZE;  in sof_ipc4_init()
790 	sdev->debug_box.offset = snd_sof_dsp_get_window_offset(sdev,  in sof_ipc4_init()
793 	sdev->fw_info_box.offset = snd_sof_dsp_get_window_offset(sdev,  in sof_ipc4_init()
795 	sdev->fw_info_box.size = sizeof(struct sof_ipc4_fw_registers);  in sof_ipc4_init()
797 	dev_dbg(sdev->dev, "mailbox upstream %#x - size %#x\n",  in sof_ipc4_init()
798 		sdev->dsp_box.offset, SOF_IPC4_MSG_MAX_SIZE);  in sof_ipc4_init()
799 	dev_dbg(sdev->dev, "mailbox downstream %#x - size %#x\n",  in sof_ipc4_init()
800 		sdev->host_box.offset, SOF_IPC4_MSG_MAX_SIZE);  in sof_ipc4_init()
801 	dev_dbg(sdev->dev, "debug box %#x\n", sdev->debug_box.offset);  in sof_ipc4_init()
808 	struct sof_ipc4_fw_data *ipc4_data = sdev->private;  in sof_ipc4_exit()
812 	xa_for_each(&ipc4_data->fw_lib_xa, lib_id, fw_lib) {  in sof_ipc4_exit()
818 			release_firmware(fw_lib->sof_fw.fw);  in sof_ipc4_exit()
820 		fw_lib->sof_fw.fw = NULL;  in sof_ipc4_exit()
823 	xa_destroy(&ipc4_data->fw_lib_xa);  in sof_ipc4_exit()
828 	if (sdev->first_boot)  in sof_ipc4_post_boot()