1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause) 2 // 3 // This file is provided under a dual BSD/GPLv2 license. When using or 4 // redistributing this file, you may do so under either license. 5 // 6 // Copyright(c) 2022 Intel Corporation 7 // 8 // Authors: Rander Wang <rander.wang@linux.intel.com> 9 // Peter Ujfalusi <peter.ujfalusi@linux.intel.com> 10 // 11 #include <linux/firmware.h> 12 #include <sound/sof/header.h> 13 #include <sound/sof/ipc4/header.h> 14 #include "sof-priv.h" 15 #include "sof-audio.h" 16 #include "ipc4-fw-reg.h" 17 #include "ipc4-priv.h" 18 #include "ipc4-topology.h" 19 #include "ipc4-telemetry.h" 20 #include "ops.h" 21 22 static const struct sof_ipc4_fw_status { 23 int status; 24 char *msg; 25 } ipc4_status[] = { 26 {0, "The operation was successful"}, 27 {1, "Invalid parameter specified"}, 28 {2, "Unknown message type specified"}, 29 {3, "Not enough space in the IPC reply buffer to complete the request"}, 30 {4, "The system or resource is busy"}, 31 {5, "Replaced ADSP IPC PENDING (unused)"}, 32 {6, "Unknown error while processing the request"}, 33 {7, "Unsupported operation requested"}, 34 {8, "Reserved (ADSP_STAGE_UNINITIALIZED removed)"}, 35 {9, "Specified resource not found"}, 36 {10, "A resource's ID requested to be created is already assigned"}, 37 {11, "Reserved (ADSP_IPC_OUT_OF_MIPS removed)"}, 38 {12, "Required resource is in invalid state"}, 39 {13, "Requested power transition failed to complete"}, 40 {14, "Manifest of the library being loaded is invalid"}, 41 {15, "Requested service or data is unavailable on the target platform"}, 42 {42, "Library target address is out of storage memory range"}, 43 {43, "Reserved"}, 44 {44, "Image verification by CSE failed"}, 45 {100, "General module management error"}, 46 {101, "Module loading failed"}, 47 {102, "Integrity check of the loaded module content failed"}, 48 {103, "Attempt to unload code of the module in use"}, 49 {104, "Other failure of module instance initialization request"}, 50 {105, "Reserved (ADSP_IPC_OUT_OF_MIPS removed)"}, 51 {106, "Reserved (ADSP_IPC_CONFIG_GET_ERROR removed)"}, 52 {107, "Reserved (ADSP_IPC_CONFIG_SET_ERROR removed)"}, 53 {108, "Reserved (ADSP_IPC_LARGE_CONFIG_GET_ERROR removed)"}, 54 {109, "Reserved (ADSP_IPC_LARGE_CONFIG_SET_ERROR removed)"}, 55 {110, "Invalid (out of range) module ID provided"}, 56 {111, "Invalid module instance ID provided"}, 57 {112, "Invalid queue (pin) ID provided"}, 58 {113, "Invalid destination queue (pin) ID provided"}, 59 {114, "Reserved (ADSP_IPC_BIND_UNBIND_DST_SINK_UNSUPPORTED removed)"}, 60 {115, "Reserved (ADSP_IPC_UNLOAD_INST_EXISTS removed)"}, 61 {116, "Invalid target code ID provided"}, 62 {117, "Injection DMA buffer is too small for probing the input pin"}, 63 {118, "Extraction DMA buffer is too small for probing the output pin"}, 64 {120, "Invalid ID of configuration item provided in TLV list"}, 65 {121, "Invalid length of configuration item provided in TLV list"}, 66 {122, "Invalid structure of configuration item provided"}, 67 {140, "Initialization of DMA Gateway failed"}, 68 {141, "Invalid ID of gateway provided"}, 69 {142, "Setting state of DMA Gateway failed"}, 70 {143, "DMA_CONTROL message targeting gateway not allocated yet"}, 71 {150, "Attempt to configure SCLK while I2S port is running"}, 72 {151, "Attempt to configure MCLK while I2S port is running"}, 73 {152, "Attempt to stop SCLK that is not running"}, 74 {153, "Attempt to stop MCLK that is not running"}, 75 {160, "Reserved (ADSP_IPC_PIPELINE_NOT_INITIALIZED removed)"}, 76 {161, "Reserved (ADSP_IPC_PIPELINE_NOT_EXIST removed)"}, 77 {162, "Reserved (ADSP_IPC_PIPELINE_SAVE_FAILED removed)"}, 78 {163, "Reserved (ADSP_IPC_PIPELINE_RESTORE_FAILED removed)"}, 79 {165, "Reserved (ADSP_IPC_PIPELINE_ALREADY_EXISTS removed)"}, 80 }; 81 82 typedef void (*ipc4_notification_handler)(struct snd_sof_dev *sdev, 83 struct sof_ipc4_msg *msg); 84 85 static int sof_ipc4_check_reply_status(struct snd_sof_dev *sdev, u32 status) 86 { 87 int i, ret; 88 89 status &= SOF_IPC4_REPLY_STATUS; 90 91 if (!status) 92 return 0; 93 94 for (i = 0; i < ARRAY_SIZE(ipc4_status); i++) { 95 if (ipc4_status[i].status == status) { 96 dev_err(sdev->dev, "FW reported error: %u - %s\n", 97 status, ipc4_status[i].msg); 98 goto to_errno; 99 } 100 } 101 102 if (i == ARRAY_SIZE(ipc4_status)) 103 dev_err(sdev->dev, "FW reported error: %u - Unknown\n", status); 104 105 to_errno: 106 switch (status) { 107 case 2: 108 case 15: 109 ret = -EOPNOTSUPP; 110 break; 111 case 8: 112 case 11: 113 case 105 ... 109: 114 case 114 ... 115: 115 case 160 ... 163: 116 case 165: 117 ret = -ENOENT; 118 break; 119 case 4: 120 case 150: 121 case 151: 122 ret = -EBUSY; 123 break; 124 default: 125 ret = -EINVAL; 126 break; 127 } 128 129 return ret; 130 } 131 132 #if IS_ENABLED(CONFIG_SND_SOC_SOF_DEBUG_VERBOSE_IPC) 133 #define DBG_IPC4_MSG_TYPE_ENTRY(type) [SOF_IPC4_##type] = #type 134 static const char * const ipc4_dbg_mod_msg_type[] = { 135 DBG_IPC4_MSG_TYPE_ENTRY(MOD_INIT_INSTANCE), 136 DBG_IPC4_MSG_TYPE_ENTRY(MOD_CONFIG_GET), 137 DBG_IPC4_MSG_TYPE_ENTRY(MOD_CONFIG_SET), 138 DBG_IPC4_MSG_TYPE_ENTRY(MOD_LARGE_CONFIG_GET), 139 DBG_IPC4_MSG_TYPE_ENTRY(MOD_LARGE_CONFIG_SET), 140 DBG_IPC4_MSG_TYPE_ENTRY(MOD_BIND), 141 DBG_IPC4_MSG_TYPE_ENTRY(MOD_UNBIND), 142 DBG_IPC4_MSG_TYPE_ENTRY(MOD_SET_DX), 143 DBG_IPC4_MSG_TYPE_ENTRY(MOD_SET_D0IX), 144 DBG_IPC4_MSG_TYPE_ENTRY(MOD_ENTER_MODULE_RESTORE), 145 DBG_IPC4_MSG_TYPE_ENTRY(MOD_EXIT_MODULE_RESTORE), 146 DBG_IPC4_MSG_TYPE_ENTRY(MOD_DELETE_INSTANCE), 147 }; 148 149 static const char * const ipc4_dbg_glb_msg_type[] = { 150 DBG_IPC4_MSG_TYPE_ENTRY(GLB_BOOT_CONFIG), 151 DBG_IPC4_MSG_TYPE_ENTRY(GLB_ROM_CONTROL), 152 DBG_IPC4_MSG_TYPE_ENTRY(GLB_IPCGATEWAY_CMD), 153 DBG_IPC4_MSG_TYPE_ENTRY(GLB_PERF_MEASUREMENTS_CMD), 154 DBG_IPC4_MSG_TYPE_ENTRY(GLB_CHAIN_DMA), 155 DBG_IPC4_MSG_TYPE_ENTRY(GLB_LOAD_MULTIPLE_MODULES), 156 DBG_IPC4_MSG_TYPE_ENTRY(GLB_UNLOAD_MULTIPLE_MODULES), 157 DBG_IPC4_MSG_TYPE_ENTRY(GLB_CREATE_PIPELINE), 158 DBG_IPC4_MSG_TYPE_ENTRY(GLB_DELETE_PIPELINE), 159 DBG_IPC4_MSG_TYPE_ENTRY(GLB_SET_PIPELINE_STATE), 160 DBG_IPC4_MSG_TYPE_ENTRY(GLB_GET_PIPELINE_STATE), 161 DBG_IPC4_MSG_TYPE_ENTRY(GLB_GET_PIPELINE_CONTEXT_SIZE), 162 DBG_IPC4_MSG_TYPE_ENTRY(GLB_SAVE_PIPELINE), 163 DBG_IPC4_MSG_TYPE_ENTRY(GLB_RESTORE_PIPELINE), 164 DBG_IPC4_MSG_TYPE_ENTRY(GLB_LOAD_LIBRARY), 165 DBG_IPC4_MSG_TYPE_ENTRY(GLB_LOAD_LIBRARY_PREPARE), 166 DBG_IPC4_MSG_TYPE_ENTRY(GLB_INTERNAL_MESSAGE), 167 DBG_IPC4_MSG_TYPE_ENTRY(GLB_NOTIFICATION), 168 }; 169 170 #define DBG_IPC4_NOTIFICATION_TYPE_ENTRY(type) [SOF_IPC4_NOTIFY_##type] = #type 171 static const char * const ipc4_dbg_notification_type[] = { 172 DBG_IPC4_NOTIFICATION_TYPE_ENTRY(PHRASE_DETECTED), 173 DBG_IPC4_NOTIFICATION_TYPE_ENTRY(RESOURCE_EVENT), 174 DBG_IPC4_NOTIFICATION_TYPE_ENTRY(LOG_BUFFER_STATUS), 175 DBG_IPC4_NOTIFICATION_TYPE_ENTRY(TIMESTAMP_CAPTURED), 176 DBG_IPC4_NOTIFICATION_TYPE_ENTRY(FW_READY), 177 DBG_IPC4_NOTIFICATION_TYPE_ENTRY(FW_AUD_CLASS_RESULT), 178 DBG_IPC4_NOTIFICATION_TYPE_ENTRY(EXCEPTION_CAUGHT), 179 DBG_IPC4_NOTIFICATION_TYPE_ENTRY(MODULE_NOTIFICATION), 180 DBG_IPC4_NOTIFICATION_TYPE_ENTRY(PROBE_DATA_AVAILABLE), 181 DBG_IPC4_NOTIFICATION_TYPE_ENTRY(ASYNC_MSG_SRVC_MESSAGE), 182 }; 183 184 static void sof_ipc4_log_header(struct device *dev, u8 *text, struct sof_ipc4_msg *msg, 185 bool data_size_valid) 186 { 187 u32 val, type; 188 const u8 *str2 = NULL; 189 const u8 *str = NULL; 190 191 val = msg->primary & SOF_IPC4_MSG_TARGET_MASK; 192 type = SOF_IPC4_MSG_TYPE_GET(msg->primary); 193 194 if (val == SOF_IPC4_MSG_TARGET(SOF_IPC4_MODULE_MSG)) { 195 /* Module message */ 196 if (type < SOF_IPC4_MOD_TYPE_LAST) 197 str = ipc4_dbg_mod_msg_type[type]; 198 if (!str) 199 str = "Unknown Module message type"; 200 } else { 201 /* Global FW message */ 202 if (type < SOF_IPC4_GLB_TYPE_LAST) 203 str = ipc4_dbg_glb_msg_type[type]; 204 if (!str) 205 str = "Unknown Global message type"; 206 207 if (type == SOF_IPC4_GLB_NOTIFICATION) { 208 /* Notification message */ 209 u32 notif = SOF_IPC4_NOTIFICATION_TYPE_GET(msg->primary); 210 211 /* Do not print log buffer notification if not desired */ 212 if (notif == SOF_IPC4_NOTIFY_LOG_BUFFER_STATUS && 213 !sof_debug_check_flag(SOF_DBG_PRINT_DMA_POSITION_UPDATE_LOGS)) 214 return; 215 216 if (notif < SOF_IPC4_NOTIFY_TYPE_LAST) 217 str2 = ipc4_dbg_notification_type[notif]; 218 if (!str2) 219 str2 = "Unknown Global notification"; 220 } 221 } 222 223 if (str2) { 224 if (data_size_valid && msg->data_size) 225 dev_dbg(dev, "%s: %#x|%#x: %s|%s [data size: %zu]\n", 226 text, msg->primary, msg->extension, str, str2, 227 msg->data_size); 228 else 229 dev_dbg(dev, "%s: %#x|%#x: %s|%s\n", text, msg->primary, 230 msg->extension, str, str2); 231 } else { 232 if (data_size_valid && msg->data_size) 233 dev_dbg(dev, "%s: %#x|%#x: %s [data size: %zu]\n", 234 text, msg->primary, msg->extension, str, 235 msg->data_size); 236 else 237 dev_dbg(dev, "%s: %#x|%#x: %s\n", text, msg->primary, 238 msg->extension, str); 239 } 240 } 241 242 const char *sof_ipc4_pipeline_state_str(enum sof_ipc4_pipeline_state state) 243 { 244 switch (state) { 245 case SOF_IPC4_PIPE_INVALID_STATE: 246 return " (INVALID_STATE)"; 247 case SOF_IPC4_PIPE_UNINITIALIZED: 248 return " (UNINITIALIZED)"; 249 case SOF_IPC4_PIPE_RESET: 250 return " (RESET)"; 251 case SOF_IPC4_PIPE_PAUSED: 252 return " (PAUSED)"; 253 case SOF_IPC4_PIPE_RUNNING: 254 return " (RUNNING)"; 255 case SOF_IPC4_PIPE_EOS: 256 return " (EOS)"; 257 default: 258 return " (<unknown>)"; 259 } 260 } 261 #else /* CONFIG_SND_SOC_SOF_DEBUG_VERBOSE_IPC */ 262 static void sof_ipc4_log_header(struct device *dev, u8 *text, struct sof_ipc4_msg *msg, 263 bool data_size_valid) 264 { 265 /* Do not print log buffer notification if not desired */ 266 if (!sof_debug_check_flag(SOF_DBG_PRINT_DMA_POSITION_UPDATE_LOGS) && 267 !SOF_IPC4_MSG_IS_MODULE_MSG(msg->primary) && 268 SOF_IPC4_MSG_TYPE_GET(msg->primary) == SOF_IPC4_GLB_NOTIFICATION && 269 SOF_IPC4_NOTIFICATION_TYPE_GET(msg->primary) == SOF_IPC4_NOTIFY_LOG_BUFFER_STATUS) 270 return; 271 272 if (data_size_valid && msg->data_size) 273 dev_dbg(dev, "%s: %#x|%#x [data size: %zu]\n", text, 274 msg->primary, msg->extension, msg->data_size); 275 else 276 dev_dbg(dev, "%s: %#x|%#x\n", text, msg->primary, msg->extension); 277 } 278 279 const char *sof_ipc4_pipeline_state_str(enum sof_ipc4_pipeline_state state) 280 { 281 return ""; 282 } 283 #endif 284 285 static void sof_ipc4_dump_payload(struct snd_sof_dev *sdev, 286 void *ipc_data, size_t size) 287 { 288 print_hex_dump_debug("Message payload: ", DUMP_PREFIX_OFFSET, 289 16, 4, ipc_data, size, false); 290 } 291 292 static const char *sof_ipc4_resource_type_str(u32 type) 293 { 294 switch (type) { 295 case SOF_IPC4_MODULE_INSTANCE: 296 return "resource: MODULE_INSTANCE"; 297 case SOF_IPC4_PIPELINE: 298 return "resource: PIPELINE"; 299 case SOF_IPC4_GATEWAY: 300 return "resource: GATEWAY"; 301 case SOF_IPC4_EDF_TASK: 302 return "resource: EDF_TASK"; 303 case SOF_IPC4_INVALID_RESOURCE_TYPE: 304 return "Resource is invalid"; 305 default: 306 return "Unknown resource type"; 307 } 308 } 309 310 static const char *sof_ipc4_resource_event_type_str(u32 event_type) 311 { 312 switch (event_type) { 313 case SOF_IPC4_MIXER_UNDERRUN_DETECTED: 314 return "event: MIXER_UNDERRUN_DETECTED"; 315 case SOF_IPC4_PROCESS_DATA_ERROR: 316 return "event: PROCESS_DATA_ERROR"; 317 case SOF_IPC4_GATEWAY_UNDERRUN_DETECTED: 318 return "event: GATEWAY_UNDERRUN_DETECTED"; 319 case SOF_IPC4_GATEWAY_OVERRUN_DETECTED: 320 return "event: GATEWAY_OVERRUN_DETECTED"; 321 default: 322 return "Unknown event type"; 323 } 324 } 325 326 static void sof_ipc4_resource_event_handler(struct snd_sof_dev *sdev, 327 struct sof_ipc4_msg *ipc4_msg) 328 { 329 struct sof_ipc4_notify_resource_data *data = ipc4_msg->data_ptr; 330 331 /* Print event details */ 332 switch (data->event_type) { 333 case SOF_IPC4_MIXER_UNDERRUN_DETECTED: 334 dev_dbg(sdev->dev, "%s (%u): eos %u, mixed %u, expected %u\n", 335 sof_ipc4_resource_event_type_str(data->event_type), 336 data->event_type, data->data.mixer_underrun.eos_flag, 337 data->data.mixer_underrun.data_mixed, 338 data->data.mixer_underrun.expected_data_mixed); 339 break; 340 case SOF_IPC4_PROCESS_DATA_ERROR: 341 dev_dbg(sdev->dev, "%s (%u): error_code %#x\n", 342 sof_ipc4_resource_event_type_str(data->event_type), 343 data->event_type, data->data.process_data_error.error_code); 344 break; 345 case SOF_IPC4_GATEWAY_UNDERRUN_DETECTED: 346 case SOF_IPC4_GATEWAY_OVERRUN_DETECTED: 347 dev_dbg(sdev->dev, "%s (%u)\n", 348 sof_ipc4_resource_event_type_str(data->event_type), 349 data->event_type); 350 break; 351 default: 352 dev_dbg(sdev->dev, "%s (%u): raw dws %#x %#x %#x %#x %#x %#x\n", 353 sof_ipc4_resource_event_type_str(data->event_type), 354 data->event_type, 355 data->data.dws[0], data->data.dws[1], data->data.dws[2], 356 data->data.dws[3], data->data.dws[4], data->data.dws[5]); 357 break; 358 } 359 360 /* Print resource details */ 361 if (data->resource_type == SOF_IPC4_MODULE_INSTANCE) { 362 u32 module_id = SOF_IPC4_MOD_ID_GET(data->resource_id); 363 u32 instance_id = SOF_IPC4_MOD_INSTANCE_GET(data->resource_id); 364 365 dev_dbg(sdev->dev, "%s (%u), module_id %u, instance_id %u\n", 366 sof_ipc4_resource_type_str(data->resource_type), 367 data->resource_type, module_id, instance_id); 368 } else if (data->resource_type != SOF_IPC4_INVALID_RESOURCE_TYPE) { 369 dev_dbg(sdev->dev, "%s (%u), id %u\n", 370 sof_ipc4_resource_type_str(data->resource_type), 371 data->resource_type, data->resource_id); 372 } 373 } 374 375 static int sof_ipc4_get_reply(struct snd_sof_dev *sdev) 376 { 377 struct snd_sof_ipc_msg *msg = sdev->msg; 378 struct sof_ipc4_msg *ipc4_reply; 379 int ret; 380 381 /* get the generic reply */ 382 ipc4_reply = msg->reply_data; 383 384 sof_ipc4_log_header(sdev->dev, "ipc tx reply", ipc4_reply, false); 385 386 ret = sof_ipc4_check_reply_status(sdev, ipc4_reply->primary); 387 if (ret) 388 return ret; 389 390 /* No other information is expected for non large config get replies */ 391 if (!msg->reply_size || !SOF_IPC4_MSG_IS_MODULE_MSG(ipc4_reply->primary) || 392 (SOF_IPC4_MSG_TYPE_GET(ipc4_reply->primary) != SOF_IPC4_MOD_LARGE_CONFIG_GET)) 393 return 0; 394 395 /* Read the requested payload */ 396 snd_sof_dsp_mailbox_read(sdev, sdev->dsp_box.offset, ipc4_reply->data_ptr, 397 msg->reply_size); 398 399 return 0; 400 } 401 402 /* wait for IPC message reply */ 403 static int ipc4_wait_tx_done(struct snd_sof_ipc *ipc, void *reply_data) 404 { 405 struct snd_sof_ipc_msg *msg = &ipc->msg; 406 struct sof_ipc4_msg *ipc4_msg = msg->msg_data; 407 struct snd_sof_dev *sdev = ipc->sdev; 408 int ret; 409 410 /* wait for DSP IPC completion */ 411 ret = wait_event_timeout(msg->waitq, msg->ipc_complete, 412 msecs_to_jiffies(sdev->ipc_timeout)); 413 if (ret == 0) { 414 dev_err(sdev->dev, "ipc timed out for %#x|%#x\n", 415 ipc4_msg->primary, ipc4_msg->extension); 416 snd_sof_handle_fw_exception(ipc->sdev, "IPC timeout"); 417 return -ETIMEDOUT; 418 } 419 420 if (msg->reply_error) { 421 dev_err(sdev->dev, "ipc error for msg %#x|%#x\n", 422 ipc4_msg->primary, ipc4_msg->extension); 423 ret = msg->reply_error; 424 } else { 425 if (reply_data) { 426 struct sof_ipc4_msg *ipc4_reply = msg->reply_data; 427 struct sof_ipc4_msg *ipc4_reply_data = reply_data; 428 429 /* Copy the header */ 430 ipc4_reply_data->header_u64 = ipc4_reply->header_u64; 431 if (msg->reply_size && ipc4_reply_data->data_ptr) { 432 /* copy the payload returned from DSP */ 433 memcpy(ipc4_reply_data->data_ptr, ipc4_reply->data_ptr, 434 msg->reply_size); 435 ipc4_reply_data->data_size = msg->reply_size; 436 } 437 } 438 439 ret = 0; 440 sof_ipc4_log_header(sdev->dev, "ipc tx done ", ipc4_msg, true); 441 } 442 443 /* re-enable dumps after successful IPC tx */ 444 if (sdev->ipc_dump_printed) { 445 sdev->dbg_dump_printed = false; 446 sdev->ipc_dump_printed = false; 447 } 448 449 return ret; 450 } 451 452 static int ipc4_tx_msg_unlocked(struct snd_sof_ipc *ipc, 453 void *msg_data, size_t msg_bytes, 454 void *reply_data, size_t reply_bytes) 455 { 456 struct sof_ipc4_msg *ipc4_msg = msg_data; 457 struct snd_sof_dev *sdev = ipc->sdev; 458 int ret; 459 460 if (msg_bytes > ipc->max_payload_size || reply_bytes > ipc->max_payload_size) 461 return -EINVAL; 462 463 sof_ipc4_log_header(sdev->dev, "ipc tx ", msg_data, true); 464 465 ret = sof_ipc_send_msg(sdev, msg_data, msg_bytes, reply_bytes); 466 if (ret) { 467 dev_err_ratelimited(sdev->dev, 468 "%s: ipc message send for %#x|%#x failed: %d\n", 469 __func__, ipc4_msg->primary, ipc4_msg->extension, ret); 470 return ret; 471 } 472 473 /* now wait for completion */ 474 return ipc4_wait_tx_done(ipc, reply_data); 475 } 476 477 static int sof_ipc4_tx_msg(struct snd_sof_dev *sdev, void *msg_data, size_t msg_bytes, 478 void *reply_data, size_t reply_bytes, bool no_pm) 479 { 480 struct snd_sof_ipc *ipc = sdev->ipc; 481 int ret; 482 483 if (!msg_data) 484 return -EINVAL; 485 486 if (!no_pm) { 487 const struct sof_dsp_power_state target_state = { 488 .state = SOF_DSP_PM_D0, 489 }; 490 491 /* ensure the DSP is in D0i0 before sending a new IPC */ 492 ret = snd_sof_dsp_set_power_state(sdev, &target_state); 493 if (ret < 0) 494 return ret; 495 } 496 497 /* Serialise IPC TX */ 498 guard(mutex)(&ipc->tx_mutex); 499 500 ret = ipc4_tx_msg_unlocked(ipc, msg_data, msg_bytes, reply_data, reply_bytes); 501 502 if (sof_debug_check_flag(SOF_DBG_DUMP_IPC_MESSAGE_PAYLOAD)) { 503 struct sof_ipc4_msg *msg = NULL; 504 505 /* payload is indicated by non zero msg/reply_bytes */ 506 if (msg_bytes) 507 msg = msg_data; 508 else if (reply_bytes) 509 msg = reply_data; 510 511 if (msg) 512 sof_ipc4_dump_payload(sdev, msg->data_ptr, msg->data_size); 513 } 514 515 return ret; 516 } 517 518 static bool sof_ipc4_tx_payload_for_get_data(struct sof_ipc4_msg *tx) 519 { 520 /* 521 * Messages that require TX payload with LARGE_CONFIG_GET. 522 * The TX payload is placed into the IPC message data section by caller, 523 * which needs to be copied to temporary buffer since the received data 524 * will overwrite it. 525 */ 526 switch (tx->extension & SOF_IPC4_MOD_EXT_MSG_PARAM_ID_MASK) { 527 case SOF_IPC4_MOD_EXT_MSG_PARAM_ID(SOF_IPC4_SWITCH_CONTROL_PARAM_ID): 528 case SOF_IPC4_MOD_EXT_MSG_PARAM_ID(SOF_IPC4_ENUM_CONTROL_PARAM_ID): 529 case SOF_IPC4_MOD_EXT_MSG_PARAM_ID(SOF_IPC4_BYTES_CONTROL_PARAM_ID): 530 return true; 531 default: 532 return false; 533 } 534 } 535 536 static int sof_ipc4_set_get_data(struct snd_sof_dev *sdev, void *data, 537 size_t payload_bytes, bool set) 538 { 539 const struct sof_dsp_power_state target_state = { 540 .state = SOF_DSP_PM_D0, 541 }; 542 size_t payload_limit = sdev->ipc->max_payload_size; 543 struct sof_ipc4_msg *ipc4_msg = data; 544 struct sof_ipc4_msg tx = {{ 0 }}; 545 struct sof_ipc4_msg rx = {{ 0 }}; 546 size_t remaining = payload_bytes; 547 void *tx_payload_for_get = NULL; 548 size_t tx_data_size = 0; 549 size_t offset = 0; 550 size_t chunk_size; 551 int ret; 552 553 if (!data) 554 return -EINVAL; 555 556 if ((ipc4_msg->primary & SOF_IPC4_MSG_TARGET_MASK) != 557 SOF_IPC4_MSG_TARGET(SOF_IPC4_MODULE_MSG)) 558 return -EINVAL; 559 560 ipc4_msg->primary &= ~SOF_IPC4_MSG_TYPE_MASK; 561 tx.primary = ipc4_msg->primary; 562 tx.extension = ipc4_msg->extension; 563 564 if (set) 565 tx.primary |= SOF_IPC4_MSG_TYPE_SET(SOF_IPC4_MOD_LARGE_CONFIG_SET); 566 else 567 tx.primary |= SOF_IPC4_MSG_TYPE_SET(SOF_IPC4_MOD_LARGE_CONFIG_GET); 568 569 tx.extension &= ~SOF_IPC4_MOD_EXT_MSG_SIZE_MASK; 570 tx.extension |= SOF_IPC4_MOD_EXT_MSG_SIZE(payload_bytes); 571 572 tx.extension |= SOF_IPC4_MOD_EXT_MSG_FIRST_BLOCK(1); 573 574 if (sof_ipc4_tx_payload_for_get_data(&tx)) { 575 tx_data_size = min(ipc4_msg->data_size, payload_limit); 576 tx_payload_for_get = kmemdup(ipc4_msg->data_ptr, tx_data_size, 577 GFP_KERNEL); 578 if (!tx_payload_for_get) 579 return -ENOMEM; 580 } 581 582 /* ensure the DSP is in D0i0 before sending IPC */ 583 ret = snd_sof_dsp_set_power_state(sdev, &target_state); 584 if (ret < 0) { 585 kfree(tx_payload_for_get); 586 return ret; 587 } 588 589 /* Serialise IPC TX */ 590 guard(mutex)(&sdev->ipc->tx_mutex); 591 592 do { 593 size_t tx_size, rx_size; 594 595 if (remaining > payload_limit) { 596 chunk_size = payload_limit; 597 } else { 598 chunk_size = remaining; 599 if (set) 600 tx.extension |= SOF_IPC4_MOD_EXT_MSG_LAST_BLOCK(1); 601 } 602 603 if (offset) { 604 tx.extension &= ~SOF_IPC4_MOD_EXT_MSG_FIRST_BLOCK_MASK; 605 tx.extension &= ~SOF_IPC4_MOD_EXT_MSG_SIZE_MASK; 606 tx.extension |= SOF_IPC4_MOD_EXT_MSG_SIZE(offset); 607 } 608 609 if (set) { 610 tx.data_size = chunk_size; 611 tx.data_ptr = ipc4_msg->data_ptr + offset; 612 613 tx_size = chunk_size; 614 rx_size = 0; 615 } else { 616 rx.primary = 0; 617 rx.extension = 0; 618 rx.data_size = chunk_size; 619 rx.data_ptr = ipc4_msg->data_ptr + offset; 620 621 if (tx_payload_for_get) { 622 tx_size = tx_data_size; 623 tx.data_size = tx_size; 624 tx.data_ptr = tx_payload_for_get; 625 } else { 626 tx_size = 0; 627 tx.data_size = 0; 628 tx.data_ptr = NULL; 629 } 630 rx_size = chunk_size; 631 } 632 633 /* Send the message for the current chunk */ 634 ret = ipc4_tx_msg_unlocked(sdev->ipc, &tx, tx_size, &rx, rx_size); 635 if (ret < 0) { 636 dev_err(sdev->dev, 637 "%s: large config %s failed at offset %zu: %d\n", 638 __func__, set ? "set" : "get", offset, ret); 639 goto out; 640 } 641 642 if (!set && rx.extension & SOF_IPC4_MOD_EXT_MSG_FIRST_BLOCK_MASK) { 643 /* Verify the firmware reported total payload size */ 644 rx_size = rx.extension & SOF_IPC4_MOD_EXT_MSG_SIZE_MASK; 645 646 if (rx_size > payload_bytes) { 647 dev_err(sdev->dev, 648 "%s: Receive buffer (%zu) is too small for %zu\n", 649 __func__, payload_bytes, rx_size); 650 ret = -ENOMEM; 651 goto out; 652 } 653 654 if (rx_size < chunk_size) { 655 chunk_size = rx_size; 656 remaining = rx_size; 657 } else if (rx_size < payload_bytes) { 658 remaining = rx_size; 659 } 660 } 661 662 offset += chunk_size; 663 remaining -= chunk_size; 664 } while (remaining); 665 666 /* Adjust the received data size if needed */ 667 if (!set && payload_bytes != offset) 668 ipc4_msg->data_size = offset; 669 670 out: 671 if (sof_debug_check_flag(SOF_DBG_DUMP_IPC_MESSAGE_PAYLOAD)) 672 sof_ipc4_dump_payload(sdev, ipc4_msg->data_ptr, ipc4_msg->data_size); 673 674 kfree(tx_payload_for_get); 675 676 return ret; 677 } 678 679 static int sof_ipc4_init_msg_memory(struct snd_sof_dev *sdev) 680 { 681 struct sof_ipc4_msg *ipc4_msg; 682 struct snd_sof_ipc_msg *msg = &sdev->ipc->msg; 683 684 /* TODO: get max_payload_size from firmware */ 685 sdev->ipc->max_payload_size = SOF_IPC4_MSG_MAX_SIZE; 686 687 /* Allocate memory for the ipc4 container and the maximum payload */ 688 msg->reply_data = devm_kzalloc(sdev->dev, sdev->ipc->max_payload_size + 689 sizeof(struct sof_ipc4_msg), GFP_KERNEL); 690 if (!msg->reply_data) 691 return -ENOMEM; 692 693 ipc4_msg = msg->reply_data; 694 ipc4_msg->data_ptr = msg->reply_data + sizeof(struct sof_ipc4_msg); 695 696 return 0; 697 } 698 699 size_t sof_ipc4_find_debug_slot_offset_by_type(struct snd_sof_dev *sdev, 700 u32 slot_type) 701 { 702 size_t slot_desc_type_offset; 703 u32 type; 704 int i; 705 706 /* The type is the second u32 in the slot descriptor */ 707 slot_desc_type_offset = sdev->debug_box.offset + sizeof(u32); 708 for (i = 0; i < SOF_IPC4_MAX_DEBUG_SLOTS; i++) { 709 sof_mailbox_read(sdev, slot_desc_type_offset, &type, sizeof(type)); 710 711 if (type == slot_type) 712 return sdev->debug_box.offset + (i + 1) * SOF_IPC4_DEBUG_SLOT_SIZE; 713 714 slot_desc_type_offset += SOF_IPC4_DEBUG_DESCRIPTOR_SIZE; 715 } 716 717 dev_dbg(sdev->dev, "Slot type %#x is not available in debug window\n", slot_type); 718 return 0; 719 } 720 EXPORT_SYMBOL(sof_ipc4_find_debug_slot_offset_by_type); 721 722 static int ipc4_fw_ready(struct snd_sof_dev *sdev, struct sof_ipc4_msg *ipc4_msg) 723 { 724 if (!sdev->first_boot) { 725 struct sof_ipc4_fw_data *ipc4_data = sdev->private; 726 727 /* 728 * After the initial boot only check if the libraries have been 729 * restored when full context save is not enabled 730 */ 731 if (!ipc4_data->fw_context_save) 732 ipc4_data->libraries_restored = !!(ipc4_msg->primary & 733 SOF_IPC4_FW_READY_LIB_RESTORED); 734 735 return 0; 736 } 737 738 sof_ipc4_create_exception_debugfs_node(sdev); 739 740 return sof_ipc4_init_msg_memory(sdev); 741 } 742 743 static void sof_ipc4_module_notification_handler(struct snd_sof_dev *sdev, 744 struct sof_ipc4_msg *ipc4_msg) 745 { 746 struct sof_ipc4_notify_module_data *data = ipc4_msg->data_ptr; 747 748 /* 749 * If the notification includes additional, module specific data, then 750 * we need to re-allocate the buffer and re-read the whole payload, 751 * including the event_data 752 */ 753 if (data->event_data_size) { 754 void *new; 755 int ret; 756 757 ipc4_msg->data_size += data->event_data_size; 758 759 new = krealloc(ipc4_msg->data_ptr, ipc4_msg->data_size, GFP_KERNEL); 760 if (!new) { 761 ipc4_msg->data_size -= data->event_data_size; 762 return; 763 } 764 765 /* re-read the whole payload */ 766 ipc4_msg->data_ptr = new; 767 ret = snd_sof_ipc_msg_data(sdev, NULL, ipc4_msg->data_ptr, 768 ipc4_msg->data_size); 769 if (ret < 0) { 770 dev_err(sdev->dev, 771 "Failed to read the full module notification: %d\n", 772 ret); 773 return; 774 } 775 data = ipc4_msg->data_ptr; 776 } 777 778 /* Handle ALSA kcontrol notification */ 779 if ((data->event_id & SOF_IPC4_NOTIFY_MODULE_EVENTID_ALSA_MAGIC_MASK) == 780 SOF_IPC4_NOTIFY_MODULE_EVENTID_ALSA_MAGIC_VAL) { 781 const struct sof_ipc_tplg_ops *tplg_ops = sdev->ipc->ops->tplg; 782 783 if (tplg_ops->control->update) 784 tplg_ops->control->update(sdev, ipc4_msg); 785 } 786 } 787 788 static void sof_ipc4_rx_msg(struct snd_sof_dev *sdev) 789 { 790 struct sof_ipc4_msg *ipc4_msg = sdev->ipc->msg.rx_data; 791 ipc4_notification_handler handler_func = NULL; 792 size_t data_size = 0; 793 int err; 794 795 if (!ipc4_msg || !SOF_IPC4_MSG_IS_NOTIFICATION(ipc4_msg->primary)) 796 return; 797 798 ipc4_msg->data_ptr = NULL; 799 ipc4_msg->data_size = 0; 800 801 sof_ipc4_log_header(sdev->dev, "ipc rx ", ipc4_msg, false); 802 803 switch (SOF_IPC4_NOTIFICATION_TYPE_GET(ipc4_msg->primary)) { 804 case SOF_IPC4_NOTIFY_FW_READY: 805 /* check for FW boot completion */ 806 if (sdev->fw_state == SOF_FW_BOOT_IN_PROGRESS) { 807 err = ipc4_fw_ready(sdev, ipc4_msg); 808 if (err < 0) 809 sof_set_fw_state(sdev, SOF_FW_BOOT_READY_FAILED); 810 else 811 sof_set_fw_state(sdev, SOF_FW_BOOT_READY_OK); 812 813 /* wake up firmware loader */ 814 wake_up(&sdev->boot_wait); 815 } 816 817 break; 818 case SOF_IPC4_NOTIFY_RESOURCE_EVENT: 819 data_size = sizeof(struct sof_ipc4_notify_resource_data); 820 handler_func = sof_ipc4_resource_event_handler; 821 break; 822 case SOF_IPC4_NOTIFY_LOG_BUFFER_STATUS: 823 sof_ipc4_mtrace_update_pos(sdev, SOF_IPC4_LOG_CORE_GET(ipc4_msg->primary)); 824 break; 825 case SOF_IPC4_NOTIFY_EXCEPTION_CAUGHT: 826 snd_sof_dsp_panic(sdev, 0, true); 827 break; 828 case SOF_IPC4_NOTIFY_MODULE_NOTIFICATION: 829 data_size = sizeof(struct sof_ipc4_notify_module_data); 830 handler_func = sof_ipc4_module_notification_handler; 831 break; 832 default: 833 dev_dbg(sdev->dev, "Unhandled DSP message: %#x|%#x\n", 834 ipc4_msg->primary, ipc4_msg->extension); 835 break; 836 } 837 838 if (data_size) { 839 ipc4_msg->data_ptr = kmalloc(data_size, GFP_KERNEL); 840 if (!ipc4_msg->data_ptr) 841 return; 842 843 ipc4_msg->data_size = data_size; 844 err = snd_sof_ipc_msg_data(sdev, NULL, ipc4_msg->data_ptr, ipc4_msg->data_size); 845 if (err < 0) { 846 dev_err(sdev->dev, "failed to read IPC notification data: %d\n", err); 847 kfree(ipc4_msg->data_ptr); 848 ipc4_msg->data_ptr = NULL; 849 ipc4_msg->data_size = 0; 850 return; 851 } 852 } 853 854 /* Handle notifications with payload */ 855 if (handler_func) 856 handler_func(sdev, ipc4_msg); 857 858 sof_ipc4_log_header(sdev->dev, "ipc rx done ", ipc4_msg, true); 859 860 if (data_size) { 861 if (sof_debug_check_flag(SOF_DBG_DUMP_IPC_MESSAGE_PAYLOAD)) 862 sof_ipc4_dump_payload(sdev, ipc4_msg->data_ptr, 863 ipc4_msg->data_size); 864 865 kfree(ipc4_msg->data_ptr); 866 ipc4_msg->data_ptr = NULL; 867 ipc4_msg->data_size = 0; 868 } 869 } 870 871 static int sof_ipc4_set_core_state(struct snd_sof_dev *sdev, int core_idx, bool on) 872 { 873 struct sof_ipc4_dx_state_info dx_state; 874 struct sof_ipc4_msg msg; 875 876 dx_state.core_mask = BIT(core_idx); 877 if (on) 878 dx_state.dx_mask = BIT(core_idx); 879 else 880 dx_state.dx_mask = 0; 881 882 msg.primary = SOF_IPC4_MSG_TYPE_SET(SOF_IPC4_MOD_SET_DX); 883 msg.primary |= SOF_IPC4_MSG_DIR(SOF_IPC4_MSG_REQUEST); 884 msg.primary |= SOF_IPC4_MSG_TARGET(SOF_IPC4_MODULE_MSG); 885 msg.extension = 0; 886 msg.data_ptr = &dx_state; 887 msg.data_size = sizeof(dx_state); 888 889 return sof_ipc4_tx_msg(sdev, &msg, msg.data_size, NULL, 0, false); 890 } 891 892 /* 893 * The context save callback is used to send a message to the firmware notifying 894 * it that the primary core is going to be turned off, which is used as an 895 * indication to prepare for a full power down, thus preparing for IMR boot 896 * (when supported) 897 * 898 * Note: in IPC4 there is no message used to restore context, thus no context 899 * restore callback is implemented 900 */ 901 static int sof_ipc4_ctx_save(struct snd_sof_dev *sdev) 902 { 903 return sof_ipc4_set_core_state(sdev, SOF_DSP_PRIMARY_CORE, false); 904 } 905 906 static int sof_ipc4_set_pm_gate(struct snd_sof_dev *sdev, u32 flags) 907 { 908 struct sof_ipc4_msg msg = {{0}}; 909 910 msg.primary = SOF_IPC4_MSG_TYPE_SET(SOF_IPC4_MOD_SET_D0IX); 911 msg.primary |= SOF_IPC4_MSG_DIR(SOF_IPC4_MSG_REQUEST); 912 msg.primary |= SOF_IPC4_MSG_TARGET(SOF_IPC4_MODULE_MSG); 913 msg.extension = flags; 914 915 return sof_ipc4_tx_msg(sdev, &msg, 0, NULL, 0, true); 916 } 917 918 static const struct sof_ipc_pm_ops ipc4_pm_ops = { 919 .ctx_save = sof_ipc4_ctx_save, 920 .set_core_state = sof_ipc4_set_core_state, 921 .set_pm_gate = sof_ipc4_set_pm_gate, 922 }; 923 924 static int sof_ipc4_init(struct snd_sof_dev *sdev) 925 { 926 struct sof_ipc4_fw_data *ipc4_data = sdev->private; 927 int inbox_offset; 928 929 mutex_init(&ipc4_data->pipeline_state_mutex); 930 931 xa_init_flags(&ipc4_data->fw_lib_xa, XA_FLAGS_ALLOC); 932 933 /* Set up the windows for IPC communication */ 934 inbox_offset = snd_sof_dsp_get_mailbox_offset(sdev); 935 if (inbox_offset < 0) { 936 dev_err(sdev->dev, "%s: No mailbox offset\n", __func__); 937 return inbox_offset; 938 } 939 940 sdev->dsp_box.offset = inbox_offset; 941 sdev->dsp_box.size = SOF_IPC4_MSG_MAX_SIZE; 942 sdev->host_box.offset = snd_sof_dsp_get_window_offset(sdev, 943 SOF_IPC4_OUTBOX_WINDOW_IDX); 944 sdev->host_box.size = SOF_IPC4_MSG_MAX_SIZE; 945 946 sdev->debug_box.offset = snd_sof_dsp_get_window_offset(sdev, 947 SOF_IPC4_DEBUG_WINDOW_IDX); 948 949 sdev->fw_info_box.offset = snd_sof_dsp_get_window_offset(sdev, 950 SOF_IPC4_INBOX_WINDOW_IDX); 951 sdev->fw_info_box.size = sizeof(struct sof_ipc4_fw_registers); 952 953 dev_dbg(sdev->dev, "mailbox upstream %#x - size %#x\n", 954 sdev->dsp_box.offset, SOF_IPC4_MSG_MAX_SIZE); 955 dev_dbg(sdev->dev, "mailbox downstream %#x - size %#x\n", 956 sdev->host_box.offset, SOF_IPC4_MSG_MAX_SIZE); 957 dev_dbg(sdev->dev, "debug box %#x\n", sdev->debug_box.offset); 958 959 return 0; 960 } 961 962 static void sof_ipc4_exit(struct snd_sof_dev *sdev) 963 { 964 struct sof_ipc4_fw_data *ipc4_data = sdev->private; 965 struct sof_ipc4_fw_library *fw_lib; 966 unsigned long lib_id; 967 968 xa_for_each(&ipc4_data->fw_lib_xa, lib_id, fw_lib) { 969 /* 970 * The basefw (ID == 0) is handled by generic code, it is not 971 * loaded by IPC4 code. 972 */ 973 if (lib_id != 0) 974 release_firmware(fw_lib->sof_fw.fw); 975 976 fw_lib->sof_fw.fw = NULL; 977 } 978 979 xa_destroy(&ipc4_data->fw_lib_xa); 980 } 981 982 static int sof_ipc4_post_boot(struct snd_sof_dev *sdev) 983 { 984 if (sdev->first_boot) { 985 int ret = sof_ipc4_complete_split_release(sdev); 986 987 if (ret) 988 return ret; 989 990 return sof_ipc4_query_fw_configuration(sdev); 991 } 992 993 return sof_ipc4_reload_fw_libraries(sdev); 994 } 995 996 const struct sof_ipc_ops ipc4_ops = { 997 .init = sof_ipc4_init, 998 .exit = sof_ipc4_exit, 999 .post_fw_boot = sof_ipc4_post_boot, 1000 .tx_msg = sof_ipc4_tx_msg, 1001 .rx_msg = sof_ipc4_rx_msg, 1002 .set_get_data = sof_ipc4_set_get_data, 1003 .get_reply = sof_ipc4_get_reply, 1004 .pm = &ipc4_pm_ops, 1005 .fw_loader = &ipc4_loader_ops, 1006 .tplg = &ipc4_tplg_ops, 1007 .pcm = &ipc4_pcm_ops, 1008 .fw_tracing = &ipc4_mtrace_ops, 1009 }; 1010 1011 void sof_ipc4_mic_privacy_state_change(struct snd_sof_dev *sdev, bool state) 1012 { 1013 struct sof_ipc4_msg msg; 1014 u32 data = state; 1015 1016 /* 1017 * The mic privacy change notification's role is to notify the running 1018 * firmware that there is a change in mic privacy state from whatever 1019 * the state was before - since the firmware booted up or since the 1020 * previous change during runtime. 1021 * 1022 * If the firmware has not been booted up, there is no need to send 1023 * change notification (the firmware is not booted up). 1024 * The firmware checks the current state during its boot. 1025 */ 1026 if (sdev->fw_state != SOF_FW_BOOT_COMPLETE) 1027 return; 1028 1029 msg.primary = SOF_IPC4_MSG_TARGET(SOF_IPC4_MODULE_MSG); 1030 msg.primary |= SOF_IPC4_MSG_DIR(SOF_IPC4_MSG_REQUEST); 1031 msg.primary |= SOF_IPC4_MOD_ID(SOF_IPC4_MOD_INIT_BASEFW_MOD_ID); 1032 msg.primary |= SOF_IPC4_MOD_INSTANCE(SOF_IPC4_MOD_INIT_BASEFW_INSTANCE_ID); 1033 msg.extension = SOF_IPC4_MOD_EXT_MSG_PARAM_ID(SOF_IPC4_FW_PARAM_MIC_PRIVACY_STATE_CHANGE); 1034 1035 msg.data_size = sizeof(data); 1036 msg.data_ptr = &data; 1037 1038 sof_ipc4_set_get_data(sdev, &msg, msg.data_size, true); 1039 } 1040 EXPORT_SYMBOL(sof_ipc4_mic_privacy_state_change); 1041