1 // SPDX-License-Identifier: GPL-2.0-only
3  * Copyright (C) 2020-21 Intel Corporation.
16 /* Check the wwan ips if it is valid with Channel as input. */
20 		return chnl->ctype == IPC_CTYPE_WWAN &&  in ipc_imem_check_wwan_ips()
21 		       chnl->if_id == IPC_MEM_MUX_IP_CH_IF_ID;  in ipc_imem_check_wwan_ips()
32 	ipc_imem->device_sleep = state;  in ipc_imem_msg_send_device_sleep()
34 	return ipc_protocol_tq_msg_send(ipc_imem->ipc_protocol,  in ipc_imem_msg_send_device_sleep()
42 	if (pipe->nr_of_queued_entries >= pipe->max_nr_of_queued_entries)  in ipc_imem_dl_skb_alloc()
45 	return ipc_protocol_dl_td_prepare(ipc_imem->ipc_protocol, pipe);  in ipc_imem_dl_skb_alloc()
59 		struct ipc_pipe *pipe = &ipc_imem->channels[i].dl_pipe;  in ipc_imem_tq_td_alloc_timer()
61 		if (!pipe->is_open || pipe->nr_of_queued_entries > 0)  in ipc_imem_tq_td_alloc_timer()
67 		if (pipe->nr_of_queued_entries == 0)  in ipc_imem_tq_td_alloc_timer()
72 		ipc_protocol_doorbell_trigger(ipc_imem->ipc_protocol,  in ipc_imem_tq_td_alloc_timer()
76 		ipc_imem->hrtimer_period =  in ipc_imem_tq_td_alloc_timer()
78 		if (!hrtimer_active(&ipc_imem->td_alloc_timer))  in ipc_imem_tq_td_alloc_timer()
79 			hrtimer_start(&ipc_imem->td_alloc_timer,  in ipc_imem_tq_td_alloc_timer()
80 				      ipc_imem->hrtimer_period,  in ipc_imem_tq_td_alloc_timer()
100 	ipc_protocol_doorbell_trigger(ipc_imem->ipc_protocol,  in ipc_imem_tq_fast_update_timer_cb()
120 	ipc_mux_ul_adb_finish(ipc_imem->mux);  in ipc_imem_tq_adb_timer_cb()
138 	ipc_mmio_update_cp_capability(ipc_imem->mmio);  in ipc_imem_setup_cp_mux_cap_init()
140 	if (ipc_imem->mmio->mux_protocol == MUX_UNKNOWN) {  in ipc_imem_setup_cp_mux_cap_init()
141 		dev_err(ipc_imem->dev, "Failed to get Mux capability.");  in ipc_imem_setup_cp_mux_cap_init()
142 		return -EINVAL;  in ipc_imem_setup_cp_mux_cap_init()
145 	cfg->protocol = ipc_imem->mmio->mux_protocol;  in ipc_imem_setup_cp_mux_cap_init()
147 	cfg->ul_flow = (ipc_imem->mmio->has_ul_flow_credit == 1) ?  in ipc_imem_setup_cp_mux_cap_init()
151 	/* The instance ID is same as channel ID because this is been reused  in ipc_imem_setup_cp_mux_cap_init()
152 	 * for channel alloc function.  in ipc_imem_setup_cp_mux_cap_init()
154 	cfg->instance_id = IPC_MEM_MUX_IP_CH_IF_ID;  in ipc_imem_setup_cp_mux_cap_init()
166 		ipc_protocol_tq_msg_send(ipc_imem->ipc_protocol,  in ipc_imem_msg_send_feature_set()
170 		ipc_protocol_msg_send(ipc_imem->ipc_protocol,  in ipc_imem_msg_send_feature_set()
175  * ipc_imem_td_update_timer_start - Starts the TD Update Timer if not started.
176  * @ipc_imem:                       Pointer to imem data-struct
180 	/* Use the TD update timer only in the runtime phase */  in ipc_imem_td_update_timer_start()
181 	if (!ipc_imem->enter_runtime || ipc_imem->td_update_timer_suspended) {  in ipc_imem_td_update_timer_start()
183 		ipc_protocol_doorbell_trigger(ipc_imem->ipc_protocol,  in ipc_imem_td_update_timer_start()
188 	if (!hrtimer_active(&ipc_imem->tdupdate_timer)) {  in ipc_imem_td_update_timer_start()
189 		ipc_imem->hrtimer_period =  in ipc_imem_td_update_timer_start()
191 		if (!hrtimer_active(&ipc_imem->tdupdate_timer))  in ipc_imem_td_update_timer_start()
192 			hrtimer_start(&ipc_imem->tdupdate_timer,  in ipc_imem_td_update_timer_start()
193 				      ipc_imem->hrtimer_period,  in ipc_imem_td_update_timer_start()
205  * ipc_imem_adb_timer_start -	Starts the adb Timer if not starting.
206  * @ipc_imem:			Pointer to imem data-struct
210 	if (!hrtimer_active(&ipc_imem->adb_timer)) {  in ipc_imem_adb_timer_start()
211 		ipc_imem->hrtimer_period =  in ipc_imem_adb_timer_start()
213 		hrtimer_start(&ipc_imem->adb_timer,  in ipc_imem_adb_timer_start()
214 			      ipc_imem->hrtimer_period,  in ipc_imem_adb_timer_start()
221 	struct ipc_mem_channel *channel;  in ipc_imem_ul_write_td()  local
229 	for (i = 0; i < ipc_imem->nr_of_channels; i++) {  in ipc_imem_ul_write_td()
230 		channel = &ipc_imem->channels[i];  in ipc_imem_ul_write_td()
232 		if (channel->state != IMEM_CHANNEL_ACTIVE)  in ipc_imem_ul_write_td()
235 		pipe = &channel->ul_pipe;  in ipc_imem_ul_write_td()
238 		ul_list = &channel->ul_list;  in ipc_imem_ul_write_td()
241 		if (!ipc_imem_check_wwan_ips(channel)) {  in ipc_imem_ul_write_td()
243 				ipc_protocol_ul_td_send(ipc_imem->ipc_protocol,  in ipc_imem_ul_write_td()
247 				ipc_protocol_ul_td_send(ipc_imem->ipc_protocol,  in ipc_imem_ul_write_td()
255 		ipc_protocol_doorbell_trigger(ipc_imem->ipc_protocol,  in ipc_imem_ul_write_td()
266 	ipc_imem->ipc_requested_state = IPC_MEM_DEVICE_IPC_INIT;  in ipc_imem_ipc_init_check()
269 	ipc_doorbell_fire(ipc_imem->pcie, IPC_DOORBELL_IRQ_IPC,  in ipc_imem_ipc_init_check()
273 		if (ipc_mmio_get_ipc_state(ipc_imem->mmio) ==  in ipc_imem_ipc_init_check()
274 		    ipc_imem->ipc_requested_state) {  in ipc_imem_ipc_init_check()
276 			ipc_mmio_config(ipc_imem->mmio);  in ipc_imem_ipc_init_check()
279 			ipc_imem->ipc_requested_state =  in ipc_imem_ipc_init_check()
281 			ipc_doorbell_fire(ipc_imem->pcie, IPC_DOORBELL_IRQ_IPC,  in ipc_imem_ipc_init_check()
287 	} while (--timeout);  in ipc_imem_ipc_init_check()
290 	dev_err(ipc_imem->dev, "%s: ipc_status(%d) ne. IPC_MEM_DEVICE_IPC_INIT",  in ipc_imem_ipc_init_check()
291 		ipc_imem_phase_get_string(ipc_imem->phase),  in ipc_imem_ipc_init_check()
292 		ipc_mmio_get_ipc_state(ipc_imem->mmio));  in ipc_imem_ipc_init_check()
294 	ipc_uevent_send(ipc_imem->dev, UEVENT_MDM_TIMEOUT);  in ipc_imem_ipc_init_check()
307 	switch (pipe->channel->ctype) {  in ipc_imem_dl_skb_process()
309 		port_id = pipe->channel->channel_id;  in ipc_imem_dl_skb_process()
310 		ipc_pcie_addr_unmap(ipc_imem->pcie, IPC_CB(skb)->len,  in ipc_imem_dl_skb_process()
311 				    IPC_CB(skb)->mapping,  in ipc_imem_dl_skb_process()
312 				    IPC_CB(skb)->direction);  in ipc_imem_dl_skb_process()
314 			ipc_imem_sys_devlink_notify_rx(ipc_imem->ipc_devlink,  in ipc_imem_dl_skb_process()
319 			wwan_port_rx(ipc_imem->ipc_port[port_id]->iosm_port,  in ipc_imem_dl_skb_process()
324 		if (pipe->channel->if_id == IPC_MEM_MUX_IP_CH_IF_ID)  in ipc_imem_dl_skb_process()
325 			ipc_mux_dl_decode(ipc_imem->mux, skb);  in ipc_imem_dl_skb_process()
328 		dev_err(ipc_imem->dev, "Invalid channel type");  in ipc_imem_dl_skb_process()
338 	struct ipc_mem_channel *channel;  in ipc_imem_dl_pipe_process()  local
343 	channel = pipe->channel;  in ipc_imem_dl_pipe_process()
345 	ipc_protocol_get_head_tail_index(ipc_imem->ipc_protocol, pipe, &head,  in ipc_imem_dl_pipe_process()
347 	if (pipe->old_tail != tail) {  in ipc_imem_dl_pipe_process()
348 		if (pipe->old_tail < tail)  in ipc_imem_dl_pipe_process()
349 			cnt = tail - pipe->old_tail;  in ipc_imem_dl_pipe_process()
351 			cnt = pipe->nr_of_entries - pipe->old_tail + tail;  in ipc_imem_dl_pipe_process()
357 	while (cnt--) {  in ipc_imem_dl_pipe_process()
358 		skb = ipc_protocol_dl_td_process(ipc_imem->ipc_protocol, pipe);  in ipc_imem_dl_pipe_process()
364 	/* try to allocate new empty DL SKbs from head..tail - 1*/  in ipc_imem_dl_pipe_process()
368 	if (processed && !ipc_imem_check_wwan_ips(channel)) {  in ipc_imem_dl_pipe_process()
370 		ipc_protocol_doorbell_trigger(ipc_imem->ipc_protocol,  in ipc_imem_dl_pipe_process()
375 		ipc_imem_hrtimer_stop(&ipc_imem->fast_update_timer);  in ipc_imem_dl_pipe_process()
378 	/* Any control channel process will get immediate HP update.  in ipc_imem_dl_pipe_process()
379 	 * Start Fast update timer only for IP channel if all the TDs were  in ipc_imem_dl_pipe_process()
382 	if (processed && (processed_td_cnt == pipe->nr_of_entries - 1)) {  in ipc_imem_dl_pipe_process()
383 		ipc_imem->hrtimer_period =  in ipc_imem_dl_pipe_process()
385 		hrtimer_start(&ipc_imem->fast_update_timer,  in ipc_imem_dl_pipe_process()
386 			      ipc_imem->hrtimer_period, HRTIMER_MODE_REL);  in ipc_imem_dl_pipe_process()
389 	if (ipc_imem->app_notify_dl_pend)  in ipc_imem_dl_pipe_process()
390 		complete(&ipc_imem->dl_pend_sem);  in ipc_imem_dl_pipe_process()
397 	struct ipc_mem_channel *channel;  in ipc_imem_ul_pipe_process()  local
402 	channel = pipe->channel;  in ipc_imem_ul_pipe_process()
404 	/* Get the internal phase. */  in ipc_imem_ul_pipe_process()
405 	ipc_protocol_get_head_tail_index(ipc_imem->ipc_protocol, pipe, &head,  in ipc_imem_ul_pipe_process()
408 	if (pipe->old_tail != tail) {  in ipc_imem_ul_pipe_process()
409 		if (pipe->old_tail < tail)  in ipc_imem_ul_pipe_process()
410 			cnt = tail - pipe->old_tail;  in ipc_imem_ul_pipe_process()
412 			cnt = pipe->nr_of_entries - pipe->old_tail + tail;  in ipc_imem_ul_pipe_process()
416 	while (cnt--) {  in ipc_imem_ul_pipe_process()
417 		skb = ipc_protocol_ul_td_process(ipc_imem->ipc_protocol, pipe);  in ipc_imem_ul_pipe_process()
422 		/* If the user app was suspended in uplink direction - blocking  in ipc_imem_ul_pipe_process()
425 		if (IPC_CB(skb)->op_type == UL_USR_OP_BLOCKED)  in ipc_imem_ul_pipe_process()
426 			complete(&channel->ul_sem);  in ipc_imem_ul_pipe_process()
429 		if (IPC_CB(skb)->op_type == UL_MUX_OP_ADB) {  in ipc_imem_ul_pipe_process()
430 			if (channel->if_id == IPC_MEM_MUX_IP_CH_IF_ID)  in ipc_imem_ul_pipe_process()
431 				ipc_mux_ul_encoded_process(ipc_imem->mux, skb);  in ipc_imem_ul_pipe_process()
433 				dev_err(ipc_imem->dev,  in ipc_imem_ul_pipe_process()
435 					channel->if_id);  in ipc_imem_ul_pipe_process()
437 			ipc_pcie_kfree_skb(ipc_imem->pcie, skb);  in ipc_imem_ul_pipe_process()
441 	/* Trace channel stats for IP UL pipe. */  in ipc_imem_ul_pipe_process()
442 	if (ipc_imem_check_wwan_ips(pipe->channel))  in ipc_imem_ul_pipe_process()
443 		ipc_mux_check_n_restart_tx(ipc_imem->mux);  in ipc_imem_ul_pipe_process()
445 	if (ipc_imem->app_notify_ul_pend)  in ipc_imem_ul_pipe_process()
446 		complete(&ipc_imem->ul_pend_sem);  in ipc_imem_ul_pipe_process()
452 	struct ipc_mem_channel *channel;  in ipc_imem_rom_irq_exec()  local
454 	channel = ipc_imem->ipc_devlink->devlink_sio.channel;  in ipc_imem_rom_irq_exec()
455 	ipc_imem->rom_exit_code = ipc_mmio_get_rom_exit_code(ipc_imem->mmio);  in ipc_imem_rom_irq_exec()
456 	complete(&channel->ul_sem);  in ipc_imem_rom_irq_exec()
463 	ipc_protocol_doorbell_trigger(ipc_imem->ipc_protocol,  in ipc_imem_tq_td_update_timer_cb()
468 /* Consider link power management in the runtime phase. */
472 	if (ipc_protocol_pm_dev_sleep_handle(ipc_imem->ipc_protocol) &&  in ipc_imem_slp_control_exec()
473 	    hrtimer_active(&ipc_imem->tdupdate_timer)) {  in ipc_imem_slp_control_exec()
477 		ipc_imem_hrtimer_stop(&ipc_imem->tdupdate_timer);  in ipc_imem_slp_control_exec()
479 		ipc_imem_hrtimer_stop(&ipc_imem->fast_update_timer);  in ipc_imem_slp_control_exec()
487 	/* Update & check the current operation phase. */  in ipc_imem_tq_startup_timer_cb()
489 		return -EIO;  in ipc_imem_tq_startup_timer_cb()
491 	if (ipc_mmio_get_ipc_state(ipc_imem->mmio) ==  in ipc_imem_tq_startup_timer_cb()
493 		ipc_imem->ipc_requested_state = IPC_MEM_DEVICE_IPC_INIT;  in ipc_imem_tq_startup_timer_cb()
495 		ipc_doorbell_fire(ipc_imem->pcie, IPC_DOORBELL_IRQ_IPC,  in ipc_imem_tq_startup_timer_cb()
498 		ipc_imem->hrtimer_period = ktime_set(0, 100 * 1000UL * 1000ULL);  in ipc_imem_tq_startup_timer_cb()
500 		if (!hrtimer_active(&ipc_imem->startup_timer))  in ipc_imem_tq_startup_timer_cb()
501 			hrtimer_start(&ipc_imem->startup_timer,  in ipc_imem_tq_startup_timer_cb()
502 				      ipc_imem->hrtimer_period,  in ipc_imem_tq_startup_timer_cb()
504 	} else if (ipc_mmio_get_ipc_state(ipc_imem->mmio) ==  in ipc_imem_tq_startup_timer_cb()
506 		/* Startup complete  - disable timer */  in ipc_imem_tq_startup_timer_cb()
507 		ipc_imem_hrtimer_stop(&ipc_imem->startup_timer);  in ipc_imem_tq_startup_timer_cb()
510 		ipc_mmio_config(ipc_imem->mmio);  in ipc_imem_tq_startup_timer_cb()
511 		ipc_imem->ipc_requested_state = IPC_MEM_DEVICE_IPC_RUNNING;  in ipc_imem_tq_startup_timer_cb()
512 		ipc_doorbell_fire(ipc_imem->pcie, IPC_DOORBELL_IRQ_IPC,  in ipc_imem_tq_startup_timer_cb()
525 	if (ktime_to_ns(ipc_imem->hrtimer_period)) {  in ipc_imem_startup_timer_cb()
526 		hrtimer_forward_now(&ipc_imem->startup_timer,  in ipc_imem_startup_timer_cb()
527 				    ipc_imem->hrtimer_period);  in ipc_imem_startup_timer_cb()
540 	return (ipc_imem->phase == IPC_P_RUN &&  in ipc_imem_get_exec_stage_buffered()
541 		ipc_imem->ipc_status == IPC_MEM_DEVICE_IPC_RUNNING) ?  in ipc_imem_get_exec_stage_buffered()
542 		       ipc_protocol_get_ap_exec_stage(ipc_imem->ipc_protocol) :  in ipc_imem_get_exec_stage_buffered()
543 		       ipc_mmio_get_exec_stage(ipc_imem->mmio);  in ipc_imem_get_exec_stage_buffered()
554 		ipc_uevent_send(ipc_imem->dev, UEVENT_MDM_READY);  in ipc_imem_send_mdm_rdy_cb()
572 	if (ipc_imem->phase != IPC_P_RUN) {  in ipc_imem_run_state_worker()
573 		dev_err(ipc_imem->dev,  in ipc_imem_run_state_worker()
578 	if (test_and_clear_bit(IOSM_DEVLINK_INIT, &ipc_imem->flag))  in ipc_imem_run_state_worker()
579 		ipc_devlink_deinit(ipc_imem->ipc_devlink);  in ipc_imem_run_state_worker()
585 	ipc_imem->mux = ipc_mux_init(&mux_cfg, ipc_imem);  in ipc_imem_run_state_worker()
586 	if (!ipc_imem->mux)  in ipc_imem_run_state_worker()
593 	ipc_imem->mux->wwan = ipc_imem->wwan;  in ipc_imem_run_state_worker()
597 			ipc_imem->ipc_port[ctrl_chl_idx] = NULL;  in ipc_imem_run_state_worker()
599 			if (ipc_imem->pcie->pci->device == INTEL_CP_DEVICE_7560_ID &&  in ipc_imem_run_state_worker()
605 			if (ipc_imem->pcie->pci->device == INTEL_CP_DEVICE_7360_ID &&  in ipc_imem_run_state_worker()
614 				ipc_imem->ipc_port[ctrl_chl_idx] =  in ipc_imem_run_state_worker()
629 	set_bit(FULLY_FUNCTIONAL, &ipc_imem->flag);  in ipc_imem_run_state_worker()
637 	ipc_mux_deinit(ipc_imem->mux);  in ipc_imem_run_state_worker()
639 	ipc_uevent_send(ipc_imem->dev, UEVENT_CD_READY_LINK_DOWN);  in ipc_imem_run_state_worker()
645 	enum ipc_phase old_phase, phase;  in ipc_imem_handle_irq()  local
651 		ipc_imem->ev_irq_pending[irq] = false;  in ipc_imem_handle_irq()
653 	/* Get the internal phase. */  in ipc_imem_handle_irq()
654 	old_phase = ipc_imem->phase;  in ipc_imem_handle_irq()
657 		dev_dbg(ipc_imem->dev,  in ipc_imem_handle_irq()
663 	/* Update the phase controlled by CP. */  in ipc_imem_handle_irq()
664 	phase = ipc_imem_phase_update(ipc_imem);  in ipc_imem_handle_irq()
666 	switch (phase) {  in ipc_imem_handle_irq()
668 		if (!ipc_imem->enter_runtime) {  in ipc_imem_handle_irq()
670 			ipc_imem->enter_runtime = 1;  in ipc_imem_handle_irq()
676 						       ipc_imem->device_sleep);  in ipc_imem_handle_irq()
684 			ipc_protocol_get_ipc_status(ipc_imem->ipc_protocol);  in ipc_imem_handle_irq()
687 		if (ipc_imem->ipc_status != curr_ipc_status) {  in ipc_imem_handle_irq()
688 			ipc_imem->ipc_status = curr_ipc_status;  in ipc_imem_handle_irq()
690 			if (ipc_imem->ipc_status ==  in ipc_imem_handle_irq()
692 				schedule_work(&ipc_imem->run_state_worker);  in ipc_imem_handle_irq()
696 		/* Consider power management in the runtime phase. */  in ipc_imem_handle_irq()
703 		dev_err(ipc_imem->dev, "confused phase %s",  in ipc_imem_handle_irq()
704 			ipc_imem_phase_get_string(phase));  in ipc_imem_handle_irq()
712 		/* On CP the PSI phase is already active. */  in ipc_imem_handle_irq()
731 		struct ipc_pipe *ul_pipe = &ipc_imem->channels[i].ul_pipe;  in ipc_imem_handle_irq()
732 		struct ipc_pipe *dl_pipe = &ipc_imem->channels[i].dl_pipe;  in ipc_imem_handle_irq()
734 		if (dl_pipe->is_open &&  in ipc_imem_handle_irq()
735 		    (irq == IMEM_IRQ_DONT_CARE || irq == dl_pipe->irq)) {  in ipc_imem_handle_irq()
738 			if (dl_pipe->nr_of_queued_entries == 0)  in ipc_imem_handle_irq()
742 		if (ul_pipe->is_open)  in ipc_imem_handle_irq()
747 	if (ipc_mux_ul_data_encode(ipc_imem->mux)) {  in ipc_imem_handle_irq()
749 		if (ipc_imem->mux->protocol == MUX_AGGREGATION)  in ipc_imem_handle_irq()
760 		ipc_imem->hrtimer_period =  in ipc_imem_handle_irq()
762 		if (!hrtimer_active(&ipc_imem->tdupdate_timer))  in ipc_imem_handle_irq()
763 			hrtimer_start(&ipc_imem->tdupdate_timer,  in ipc_imem_handle_irq()
764 				      ipc_imem->hrtimer_period,  in ipc_imem_handle_irq()
770 	 * phase, wake up the flash app to open the pipes.  in ipc_imem_handle_irq()
772 	if ((phase == IPC_P_PSI || phase == IPC_P_EBL) &&  in ipc_imem_handle_irq()
773 	    ipc_imem->ipc_requested_state == IPC_MEM_DEVICE_IPC_RUNNING &&  in ipc_imem_handle_irq()
774 	    ipc_mmio_get_ipc_state(ipc_imem->mmio) ==  in ipc_imem_handle_irq()
776 		complete(&ipc_imem->ipc_devlink->devlink_sio.channel->ul_sem);  in ipc_imem_handle_irq()
780 	ipc_imem->ipc_requested_state = IPC_MEM_DEVICE_IPC_DONT_CARE;  in ipc_imem_handle_irq()
783 		ipc_imem->hrtimer_period =  in ipc_imem_handle_irq()
785 		if (!hrtimer_active(&ipc_imem->td_alloc_timer))  in ipc_imem_handle_irq()
786 			hrtimer_start(&ipc_imem->td_alloc_timer,  in ipc_imem_handle_irq()
787 				      ipc_imem->hrtimer_period,  in ipc_imem_handle_irq()
808 /* Check the execution stage and update the AP phase */
814 		if (ipc_imem->phase != IPC_P_ROM) {  in ipc_imem_phase_update_check()
816 			ipc_uevent_send(ipc_imem->dev, UEVENT_ROM_READY);  in ipc_imem_phase_update_check()
819 		ipc_imem->phase = IPC_P_ROM;  in ipc_imem_phase_update_check()
823 		ipc_imem->phase = IPC_P_PSI;  in ipc_imem_phase_update_check()
827 		ipc_imem->phase = IPC_P_EBL;  in ipc_imem_phase_update_check()
831 		if (ipc_imem->phase != IPC_P_RUN &&  in ipc_imem_phase_update_check()
832 		    ipc_imem->ipc_status == IPC_MEM_DEVICE_IPC_RUNNING) {  in ipc_imem_phase_update_check()
833 			ipc_uevent_send(ipc_imem->dev, UEVENT_MDM_READY);  in ipc_imem_phase_update_check()
835 		ipc_imem->phase = IPC_P_RUN;  in ipc_imem_phase_update_check()
839 		if (ipc_imem->phase != IPC_P_CRASH)  in ipc_imem_phase_update_check()
840 			ipc_uevent_send(ipc_imem->dev, UEVENT_CRASH);  in ipc_imem_phase_update_check()
842 		ipc_imem->phase = IPC_P_CRASH;  in ipc_imem_phase_update_check()
846 		if (ipc_imem->phase != IPC_P_CD_READY)  in ipc_imem_phase_update_check()
847 			ipc_uevent_send(ipc_imem->dev, UEVENT_CD_READY);  in ipc_imem_phase_update_check()
848 		ipc_imem->phase = IPC_P_CD_READY;  in ipc_imem_phase_update_check()
855 		ipc_uevent_send(ipc_imem->dev, UEVENT_CD_READY_LINK_DOWN);  in ipc_imem_phase_update_check()
859 	return ipc_imem->phase;  in ipc_imem_phase_update_check()
870 	if (ipc_protocol_msg_send(ipc_imem->ipc_protocol,  in ipc_imem_pipe_open()
872 		pipe->is_open = true;  in ipc_imem_pipe_open()
874 	return pipe->is_open;  in ipc_imem_pipe_open()
885 	for (i = 0; i < dl_pipe->nr_of_entries - 1; i++)  in ipc_imem_tq_pipe_td_alloc()
892 		ipc_protocol_doorbell_trigger(ipc_imem->ipc_protocol, arg);  in ipc_imem_tq_pipe_td_alloc()
908 /* Get the CP execution state and map it to the AP phase. */
913 	/* If the CP stage is undef, return the internal precalculated phase. */  in ipc_imem_phase_update()
914 	return ipc_imem->phase == IPC_P_OFF_REQ ?  in ipc_imem_phase_update()
915 		       ipc_imem->phase :  in ipc_imem_phase_update()
919 const char *ipc_imem_phase_get_string(enum ipc_phase phase)  in ipc_imem_phase_get_string()  argument
921 	switch (phase) {  in ipc_imem_phase_get_string()
923 		return "A-RUN";  in ipc_imem_phase_get_string()
926 		return "A-OFF";  in ipc_imem_phase_get_string()
929 		return "A-ROM";  in ipc_imem_phase_get_string()
932 		return "A-PSI";  in ipc_imem_phase_get_string()
935 		return "A-EBL";  in ipc_imem_phase_get_string()
938 		return "A-CRASH";  in ipc_imem_phase_get_string()
941 		return "A-CD_READY";  in ipc_imem_phase_get_string()
944 		return "A-OFF_REQ";  in ipc_imem_phase_get_string()
947 		return "A-???";  in ipc_imem_phase_get_string()
955 	pipe->is_open = false;  in ipc_imem_pipe_close()
956 	ipc_protocol_msg_send(ipc_imem->ipc_protocol, IPC_MSG_PREP_PIPE_CLOSE,  in ipc_imem_pipe_close()
964 	struct ipc_mem_channel *channel;  in ipc_imem_channel_close()  local
966 	if (channel_id < 0 || channel_id >= ipc_imem->nr_of_channels) {  in ipc_imem_channel_close()
967 		dev_err(ipc_imem->dev, "invalid channel id %d", channel_id);  in ipc_imem_channel_close()
971 	channel = &ipc_imem->channels[channel_id];  in ipc_imem_channel_close()
973 	if (channel->state == IMEM_CHANNEL_FREE) {  in ipc_imem_channel_close()
974 		dev_err(ipc_imem->dev, "ch[%d]: invalid channel state %d",  in ipc_imem_channel_close()
975 			channel_id, channel->state);  in ipc_imem_channel_close()
979 	/* Free only the channel id in the CP power off mode. */  in ipc_imem_channel_close()
980 	if (channel->state == IMEM_CHANNEL_RESERVED)  in ipc_imem_channel_close()
981 		/* Release only the channel id. */  in ipc_imem_channel_close()
984 	if (ipc_imem->phase == IPC_P_RUN) {  in ipc_imem_channel_close()
985 		ipc_imem_pipe_close(ipc_imem, &channel->ul_pipe);  in ipc_imem_channel_close()
986 		ipc_imem_pipe_close(ipc_imem, &channel->dl_pipe);  in ipc_imem_channel_close()
989 	ipc_imem_pipe_cleanup(ipc_imem, &channel->ul_pipe);  in ipc_imem_channel_close()
990 	ipc_imem_pipe_cleanup(ipc_imem, &channel->dl_pipe);  in ipc_imem_channel_close()
993 	ipc_imem_channel_free(channel);  in ipc_imem_channel_close()
999 	struct ipc_mem_channel *channel;  in ipc_imem_channel_open()  local
1002 		dev_err(ipc_imem->dev, "invalid channel ID: %d", channel_id);  in ipc_imem_channel_open()
1006 	channel = &ipc_imem->channels[channel_id];  in ipc_imem_channel_open()
1008 	channel->state = IMEM_CHANNEL_ACTIVE;  in ipc_imem_channel_open()
1010 	if (!ipc_imem_pipe_open(ipc_imem, &channel->ul_pipe))  in ipc_imem_channel_open()
1013 	if (!ipc_imem_pipe_open(ipc_imem, &channel->dl_pipe))  in ipc_imem_channel_open()
1018 				     &channel->dl_pipe, 0, false)) {  in ipc_imem_channel_open()
1019 		dev_err(ipc_imem->dev, "td allocation failed : %d", channel_id);  in ipc_imem_channel_open()
1023 	/* Active channel. */  in ipc_imem_channel_open()
1024 	return channel;  in ipc_imem_channel_open()
1026 	ipc_imem_pipe_close(ipc_imem, &channel->dl_pipe);  in ipc_imem_channel_open()
1028 	ipc_imem_pipe_close(ipc_imem, &channel->ul_pipe);  in ipc_imem_channel_open()
1030 	ipc_imem_channel_free(channel);  in ipc_imem_channel_open()
1036 	ipc_protocol_suspend(ipc_imem->ipc_protocol);  in ipc_imem_pm_suspend()
1041 	ipc_protocol_s2idle_sleep(ipc_imem->ipc_protocol, sleep);  in ipc_imem_pm_s2idle_sleep()
1048 	if (ipc_protocol_resume(ipc_imem->ipc_protocol)) {  in ipc_imem_pm_resume()
1049 		stage = ipc_mmio_get_exec_stage(ipc_imem->mmio);  in ipc_imem_pm_resume()
1054 void ipc_imem_channel_free(struct ipc_mem_channel *channel)  in ipc_imem_channel_free()  argument
1056 	/* Reset dynamic channel elements. */  in ipc_imem_channel_free()
1057 	channel->state = IMEM_CHANNEL_FREE;  in ipc_imem_channel_free()
1063 	struct ipc_mem_channel *channel;  in ipc_imem_channel_alloc()  local
1066 	/* Find channel of given type/index */  in ipc_imem_channel_alloc()
1067 	for (i = 0; i < ipc_imem->nr_of_channels; i++) {  in ipc_imem_channel_alloc()
1068 		channel = &ipc_imem->channels[i];  in ipc_imem_channel_alloc()
1069 		if (channel->ctype == ctype && channel->index == index)  in ipc_imem_channel_alloc()
1073 	if (i >= ipc_imem->nr_of_channels) {  in ipc_imem_channel_alloc()
1074 		dev_dbg(ipc_imem->dev,  in ipc_imem_channel_alloc()
1075 			"no channel definition for index=%d ctype=%d", index,  in ipc_imem_channel_alloc()
1077 		return -ECHRNG;  in ipc_imem_channel_alloc()
1080 	if (ipc_imem->channels[i].state != IMEM_CHANNEL_FREE) {  in ipc_imem_channel_alloc()
1081 		dev_dbg(ipc_imem->dev, "channel is in use");  in ipc_imem_channel_alloc()
1082 		return -EBUSY;  in ipc_imem_channel_alloc()
1085 	if (channel->ctype == IPC_CTYPE_WWAN &&  in ipc_imem_channel_alloc()
1087 		channel->if_id = index;  in ipc_imem_channel_alloc()
1089 	channel->channel_id = index;  in ipc_imem_channel_alloc()
1090 	channel->state = IMEM_CHANNEL_RESERVED;  in ipc_imem_channel_alloc()
1098 	struct ipc_mem_channel *channel;  in ipc_imem_channel_init()  local
1102 		dev_err(ipc_imem->dev, "invalid pipe: ul_pipe=%d, dl_pipe=%d",  in ipc_imem_channel_init()
1107 	if (ipc_imem->nr_of_channels >= IPC_MEM_MAX_CHANNELS) {  in ipc_imem_channel_init()
1108 		dev_err(ipc_imem->dev, "too many channels");  in ipc_imem_channel_init()
1112 	channel = &ipc_imem->channels[ipc_imem->nr_of_channels];  in ipc_imem_channel_init()
1113 	channel->channel_id = ipc_imem->nr_of_channels;  in ipc_imem_channel_init()
1114 	channel->ctype = ctype;  in ipc_imem_channel_init()
1115 	channel->index = chnl_cfg.id;  in ipc_imem_channel_init()
1116 	channel->net_err_count = 0;  in ipc_imem_channel_init()
1117 	channel->state = IMEM_CHANNEL_FREE;  in ipc_imem_channel_init()
1118 	ipc_imem->nr_of_channels++;  in ipc_imem_channel_init()
1120 	ipc_imem_channel_update(ipc_imem, channel->channel_id, chnl_cfg,  in ipc_imem_channel_init()
1123 	skb_queue_head_init(&channel->ul_list);  in ipc_imem_channel_init()
1125 	init_completion(&channel->ul_sem);  in ipc_imem_channel_init()
1131 	struct ipc_mem_channel *channel;  in ipc_imem_channel_update()  local
1133 	if (id < 0 || id >= ipc_imem->nr_of_channels) {  in ipc_imem_channel_update()
1134 		dev_err(ipc_imem->dev, "invalid channel id %d", id);  in ipc_imem_channel_update()
1138 	channel = &ipc_imem->channels[id];  in ipc_imem_channel_update()
1140 	if (channel->state != IMEM_CHANNEL_FREE &&  in ipc_imem_channel_update()
1141 	    channel->state != IMEM_CHANNEL_RESERVED) {  in ipc_imem_channel_update()
1142 		dev_err(ipc_imem->dev, "invalid channel state %d",  in ipc_imem_channel_update()
1143 			channel->state);  in ipc_imem_channel_update()
1147 	channel->ul_pipe.nr_of_entries = chnl_cfg.ul_nr_of_entries;  in ipc_imem_channel_update()
1148 	channel->ul_pipe.pipe_nr = chnl_cfg.ul_pipe;  in ipc_imem_channel_update()
1149 	channel->ul_pipe.is_open = false;  in ipc_imem_channel_update()
1150 	channel->ul_pipe.irq = IPC_UL_PIPE_IRQ_VECTOR;  in ipc_imem_channel_update()
1151 	channel->ul_pipe.channel = channel;  in ipc_imem_channel_update()
1152 	channel->ul_pipe.dir = IPC_MEM_DIR_UL;  in ipc_imem_channel_update()
1153 	channel->ul_pipe.accumulation_backoff = chnl_cfg.accumulation_backoff;  in ipc_imem_channel_update()
1154 	channel->ul_pipe.irq_moderation = irq_moderation;  in ipc_imem_channel_update()
1155 	channel->ul_pipe.buf_size = 0;  in ipc_imem_channel_update()
1157 	channel->dl_pipe.nr_of_entries = chnl_cfg.dl_nr_of_entries;  in ipc_imem_channel_update()
1158 	channel->dl_pipe.pipe_nr = chnl_cfg.dl_pipe;  in ipc_imem_channel_update()
1159 	channel->dl_pipe.is_open = false;  in ipc_imem_channel_update()
1160 	channel->dl_pipe.irq = IPC_DL_PIPE_IRQ_VECTOR;  in ipc_imem_channel_update()
1161 	channel->dl_pipe.channel = channel;  in ipc_imem_channel_update()
1162 	channel->dl_pipe.dir = IPC_MEM_DIR_DL;  in ipc_imem_channel_update()
1163 	channel->dl_pipe.accumulation_backoff = chnl_cfg.accumulation_backoff;  in ipc_imem_channel_update()
1164 	channel->dl_pipe.irq_moderation = irq_moderation;  in ipc_imem_channel_update()
1165 	channel->dl_pipe.buf_size = chnl_cfg.dl_buf_size;  in ipc_imem_channel_update()
1172 	for (i = 0; i < ipc_imem->nr_of_channels; i++) {  in ipc_imem_channel_reset()
1173 		struct ipc_mem_channel *channel;  in ipc_imem_channel_reset()  local
1175 		channel = &ipc_imem->channels[i];  in ipc_imem_channel_reset()
1177 		ipc_imem_pipe_cleanup(ipc_imem, &channel->dl_pipe);  in ipc_imem_channel_reset()
1178 		ipc_imem_pipe_cleanup(ipc_imem, &channel->ul_pipe);  in ipc_imem_channel_reset()
1180 		ipc_imem_channel_free(channel);  in ipc_imem_channel_reset()
1191 	pipe->is_open = false;  in ipc_imem_pipe_cleanup()
1194 	while ((skb = skb_dequeue(&pipe->channel->ul_list)))  in ipc_imem_pipe_cleanup()
1195 		ipc_pcie_kfree_skb(ipc_imem->pcie, skb);  in ipc_imem_pipe_cleanup()
1197 	ipc_protocol_pipe_cleanup(ipc_imem->ipc_protocol, pipe);  in ipc_imem_pipe_cleanup()
1209 	if (ipc_pcie_check_data_link_active(ipc_imem->pcie)) {  in ipc_imem_device_ipc_uninit()
1214 		ipc_doorbell_fire(ipc_imem->pcie, IPC_DOORBELL_IRQ_IPC,  in ipc_imem_device_ipc_uninit()
1216 		ipc_state = ipc_mmio_get_ipc_state(ipc_imem->mmio);  in ipc_imem_device_ipc_uninit()
1225 			timeout--;  in ipc_imem_device_ipc_uninit()
1226 			ipc_state = ipc_mmio_get_ipc_state(ipc_imem->mmio);  in ipc_imem_device_ipc_uninit()
1233 	ipc_imem->phase = IPC_P_OFF_REQ;  in ipc_imem_cleanup()
1236 	ipc_uevent_send(ipc_imem->dev, UEVENT_MDM_NOT_READY);  in ipc_imem_cleanup()
1238 	hrtimer_cancel(&ipc_imem->td_alloc_timer);  in ipc_imem_cleanup()
1239 	hrtimer_cancel(&ipc_imem->tdupdate_timer);  in ipc_imem_cleanup()
1240 	hrtimer_cancel(&ipc_imem->fast_update_timer);  in ipc_imem_cleanup()
1241 	hrtimer_cancel(&ipc_imem->startup_timer);  in ipc_imem_cleanup()
1244 	cancel_work_sync(&ipc_imem->run_state_worker);  in ipc_imem_cleanup()
1246 	if (test_and_clear_bit(FULLY_FUNCTIONAL, &ipc_imem->flag)) {  in ipc_imem_cleanup()
1247 		ipc_mux_deinit(ipc_imem->mux);  in ipc_imem_cleanup()
1249 		ipc_wwan_deinit(ipc_imem->wwan);  in ipc_imem_cleanup()
1250 		ipc_port_deinit(ipc_imem->ipc_port);  in ipc_imem_cleanup()
1253 	if (test_and_clear_bit(IOSM_DEVLINK_INIT, &ipc_imem->flag))  in ipc_imem_cleanup()
1254 		ipc_devlink_deinit(ipc_imem->ipc_devlink);  in ipc_imem_cleanup()
1259 	ipc_protocol_deinit(ipc_imem->ipc_protocol);  in ipc_imem_cleanup()
1260 	ipc_task_deinit(ipc_imem->ipc_task);  in ipc_imem_cleanup()
1262 	kfree(ipc_imem->ipc_task);  in ipc_imem_cleanup()
1263 	kfree(ipc_imem->mmio);  in ipc_imem_cleanup()
1265 	ipc_imem->phase = IPC_P_OFF;  in ipc_imem_cleanup()
1275 	enum ipc_phase phase;  in ipc_imem_config()  local
1278 	init_completion(&ipc_imem->ul_pend_sem);  in ipc_imem_config()
1280 	init_completion(&ipc_imem->dl_pend_sem);  in ipc_imem_config()
1283 	ipc_imem->ipc_status = IPC_MEM_DEVICE_IPC_UNINIT;  in ipc_imem_config()
1284 	ipc_imem->enter_runtime = 0;  in ipc_imem_config()
1286 	phase = ipc_imem_phase_update(ipc_imem);  in ipc_imem_config()
1288 	/* Either CP shall be in the power off or power on phase. */  in ipc_imem_config()
1289 	switch (phase) {  in ipc_imem_config()
1291 		ipc_imem->hrtimer_period = ktime_set(0, 1000 * 1000 * 1000ULL);  in ipc_imem_config()
1293 		if (!hrtimer_active(&ipc_imem->startup_timer))  in ipc_imem_config()
1294 			hrtimer_start(&ipc_imem->startup_timer,  in ipc_imem_config()
1295 				      ipc_imem->hrtimer_period,  in ipc_imem_config()
1303 		ipc_imem->ipc_requested_state = IPC_MEM_DEVICE_IPC_UNINIT;  in ipc_imem_config()
1306 		if (ipc_imem->ipc_requested_state ==  in ipc_imem_config()
1307 		    ipc_mmio_get_ipc_state(ipc_imem->mmio)) {  in ipc_imem_config()
1312 		dev_err(ipc_imem->dev,  in ipc_imem_config()
1314 			ipc_mmio_get_ipc_state(ipc_imem->mmio));  in ipc_imem_config()
1318 		dev_dbg(ipc_imem->dev,  in ipc_imem_config()
1319 			"Modem is in phase %d, reset Modem to collect CD",  in ipc_imem_config()
1320 			phase);  in ipc_imem_config()
1323 		dev_err(ipc_imem->dev, "unexpected operation phase %d", phase);  in ipc_imem_config()
1327 	complete(&ipc_imem->dl_pend_sem);  in ipc_imem_config()
1328 	complete(&ipc_imem->ul_pend_sem);  in ipc_imem_config()
1329 	ipc_imem->phase = IPC_P_OFF;  in ipc_imem_config()
1330 	return -EIO;  in ipc_imem_config()
1337 	struct iosm_imem *ipc_imem = kzalloc(sizeof(*pcie->imem), GFP_KERNEL);  in ipc_imem_init()
1344 	ipc_imem->pcie = pcie;  in ipc_imem_init()
1345 	ipc_imem->dev = dev;  in ipc_imem_init()
1347 	ipc_imem->pci_device_id = device_id;  in ipc_imem_init()
1349 	ipc_imem->cp_version = 0;  in ipc_imem_init()
1350 	ipc_imem->device_sleep = IPC_HOST_SLEEP_ENTER_SLEEP;  in ipc_imem_init()
1353 	ipc_imem->nr_of_channels = 0;  in ipc_imem_init()
1356 	ipc_imem->mmio = ipc_mmio_init(mmio, ipc_imem->dev);  in ipc_imem_init()
1357 	if (!ipc_imem->mmio) {  in ipc_imem_init()
1358 		dev_err(ipc_imem->dev, "failed to initialize mmio region");  in ipc_imem_init()
1362 	ipc_imem->ipc_task = kzalloc(sizeof(*ipc_imem->ipc_task),  in ipc_imem_init()
1366 	if (!ipc_imem->ipc_task)  in ipc_imem_init()
1369 	if (ipc_task_init(ipc_imem->ipc_task))  in ipc_imem_init()
1372 	ipc_imem->ipc_task->dev = ipc_imem->dev;  in ipc_imem_init()
1374 	INIT_WORK(&ipc_imem->run_state_worker, ipc_imem_run_state_worker);  in ipc_imem_init()
1376 	ipc_imem->ipc_protocol = ipc_protocol_init(ipc_imem);  in ipc_imem_init()
1378 	if (!ipc_imem->ipc_protocol)  in ipc_imem_init()
1381 	/* The phase is set to power off. */  in ipc_imem_init()
1382 	ipc_imem->phase = IPC_P_OFF;  in ipc_imem_init()
1384 	hrtimer_init(&ipc_imem->startup_timer, CLOCK_MONOTONIC,  in ipc_imem_init()
1386 	ipc_imem->startup_timer.function = ipc_imem_startup_timer_cb;  in ipc_imem_init()
1388 	hrtimer_init(&ipc_imem->tdupdate_timer, CLOCK_MONOTONIC,  in ipc_imem_init()
1390 	ipc_imem->tdupdate_timer.function = ipc_imem_td_update_timer_cb;  in ipc_imem_init()
1392 	hrtimer_init(&ipc_imem->fast_update_timer, CLOCK_MONOTONIC,  in ipc_imem_init()
1394 	ipc_imem->fast_update_timer.function = ipc_imem_fast_update_timer_cb;  in ipc_imem_init()
1396 	hrtimer_init(&ipc_imem->td_alloc_timer, CLOCK_MONOTONIC,  in ipc_imem_init()
1398 	ipc_imem->td_alloc_timer.function = ipc_imem_td_alloc_timer_cb;  in ipc_imem_init()
1400 	hrtimer_init(&ipc_imem->adb_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);  in ipc_imem_init()
1401 	ipc_imem->adb_timer.function = ipc_imem_adb_timer_cb;  in ipc_imem_init()
1404 		dev_err(ipc_imem->dev, "failed to initialize the imem");  in ipc_imem_init()
1408 	stage = ipc_mmio_get_exec_stage(ipc_imem->mmio);  in ipc_imem_init()
1411 		ipc_imem->ipc_devlink = ipc_devlink_init(ipc_imem);  in ipc_imem_init()
1412 		if (!ipc_imem->ipc_devlink) {  in ipc_imem_init()
1413 			dev_err(ipc_imem->dev, "Devlink register failed");  in ipc_imem_init()
1420 		set_bit(IOSM_DEVLINK_INIT, &ipc_imem->flag);  in ipc_imem_init()
1424 	ipc_devlink_deinit(ipc_imem->ipc_devlink);  in ipc_imem_init()
1426 	hrtimer_cancel(&ipc_imem->td_alloc_timer);  in ipc_imem_init()
1427 	hrtimer_cancel(&ipc_imem->fast_update_timer);  in ipc_imem_init()
1428 	hrtimer_cancel(&ipc_imem->tdupdate_timer);  in ipc_imem_init()
1429 	hrtimer_cancel(&ipc_imem->startup_timer);  in ipc_imem_init()
1431 	cancel_work_sync(&ipc_imem->run_state_worker);  in ipc_imem_init()
1432 	ipc_task_deinit(ipc_imem->ipc_task);  in ipc_imem_init()
1434 	kfree(ipc_imem->ipc_task);  in ipc_imem_init()
1436 	kfree(ipc_imem->mmio);  in ipc_imem_init()
1445 	if (ipc_imem && !ipc_imem->ev_irq_pending[irq]) {  in ipc_imem_irq_process()
1446 		ipc_imem->ev_irq_pending[irq] = true;  in ipc_imem_irq_process()
1454 	ipc_imem->td_update_timer_suspended = suspend;  in ipc_imem_td_update_timer_suspend()
1458  * change the execution phase to ROM
1466 	int rc = -EINVAL;  in ipc_imem_devlink_trigger_chip_info_cb()
1470 	stage = ipc_mmio_get_exec_stage(ipc_imem->mmio);  in ipc_imem_devlink_trigger_chip_info_cb()
1472 		dev_err(ipc_imem->dev,  in ipc_imem_devlink_trigger_chip_info_cb()
1477 	size = ipc_imem->mmio->chip_info_size;  in ipc_imem_devlink_trigger_chip_info_cb()
1481 	skb = ipc_pcie_alloc_local_skb(ipc_imem->pcie, GFP_ATOMIC, size);  in ipc_imem_devlink_trigger_chip_info_cb()
1483 		dev_err(ipc_imem->dev, "exhausted skbuf kernel DL memory");  in ipc_imem_devlink_trigger_chip_info_cb()
1484 		rc = -ENOMEM;  in ipc_imem_devlink_trigger_chip_info_cb()
1488 	ipc_mmio_copy_chip_info(ipc_imem->mmio, skb_put(skb, size), size);  in ipc_imem_devlink_trigger_chip_info_cb()
1489 	/* First change to the ROM boot phase. */  in ipc_imem_devlink_trigger_chip_info_cb()
1490 	dev_dbg(ipc_imem->dev, "execution_stage[%X] eq. BOOT", stage);  in ipc_imem_devlink_trigger_chip_info_cb()
1491 	ipc_imem->phase = ipc_imem_phase_update(ipc_imem);  in ipc_imem_devlink_trigger_chip_info_cb()
1492 	ipc_imem_sys_devlink_notify_rx(ipc_imem->ipc_devlink, skb);  in ipc_imem_devlink_trigger_chip_info_cb()