28 channel_to_ops(struct tegra_bpmp_channel *channel)  in channel_to_ops()  argument
30 	struct tegra_bpmp *bpmp = channel->bpmp;  in channel_to_ops()
72 tegra_bpmp_channel_get_thread_index(struct tegra_bpmp_channel *channel)  in tegra_bpmp_channel_get_thread_index()  argument
74 	struct tegra_bpmp *bpmp = channel->bpmp;  in tegra_bpmp_channel_get_thread_index()
80 	index = channel - channel->bpmp->threaded_channels;  in tegra_bpmp_channel_get_thread_index()
95 static bool tegra_bpmp_is_response_ready(struct tegra_bpmp_channel *channel)  in tegra_bpmp_is_response_ready()  argument
97 	const struct tegra_bpmp_ops *ops = channel_to_ops(channel);  in tegra_bpmp_is_response_ready()
99 	return ops->is_response_ready(channel);  in tegra_bpmp_is_response_ready()
102 static bool tegra_bpmp_is_request_ready(struct tegra_bpmp_channel *channel)  in tegra_bpmp_is_request_ready()  argument
104 	const struct tegra_bpmp_ops *ops = channel_to_ops(channel);  in tegra_bpmp_is_request_ready()
106 	return ops->is_request_ready(channel);  in tegra_bpmp_is_request_ready()
109 static int tegra_bpmp_wait_response(struct tegra_bpmp_channel *channel)  in tegra_bpmp_wait_response()  argument
111 	unsigned long timeout = channel->bpmp->soc->channels.cpu_tx.timeout;  in tegra_bpmp_wait_response()
117 		if (tegra_bpmp_is_response_ready(channel))  in tegra_bpmp_wait_response()
124 static int tegra_bpmp_ack_response(struct tegra_bpmp_channel *channel)  in tegra_bpmp_ack_response()  argument
126 	const struct tegra_bpmp_ops *ops = channel_to_ops(channel);  in tegra_bpmp_ack_response()
128 	return ops->ack_response(channel);  in tegra_bpmp_ack_response()
131 static int tegra_bpmp_ack_request(struct tegra_bpmp_channel *channel)  in tegra_bpmp_ack_request()  argument
133 	const struct tegra_bpmp_ops *ops = channel_to_ops(channel);  in tegra_bpmp_ack_request()
135 	return ops->ack_request(channel);  in tegra_bpmp_ack_request()
139 tegra_bpmp_is_request_channel_free(struct tegra_bpmp_channel *channel)  in tegra_bpmp_is_request_channel_free()  argument
141 	const struct tegra_bpmp_ops *ops = channel_to_ops(channel);  in tegra_bpmp_is_request_channel_free()
143 	return ops->is_request_channel_free(channel);  in tegra_bpmp_is_request_channel_free()
147 tegra_bpmp_is_response_channel_free(struct tegra_bpmp_channel *channel)  in tegra_bpmp_is_response_channel_free()  argument
149 	const struct tegra_bpmp_ops *ops = channel_to_ops(channel);  in tegra_bpmp_is_response_channel_free()
151 	return ops->is_response_channel_free(channel);  in tegra_bpmp_is_response_channel_free()
155 tegra_bpmp_wait_request_channel_free(struct tegra_bpmp_channel *channel)  in tegra_bpmp_wait_request_channel_free()  argument
157 	unsigned long timeout = channel->bpmp->soc->channels.cpu_tx.timeout;  in tegra_bpmp_wait_request_channel_free()
163 		if (tegra_bpmp_is_request_channel_free(channel))  in tegra_bpmp_wait_request_channel_free()
172 static int tegra_bpmp_post_request(struct tegra_bpmp_channel *channel)  in tegra_bpmp_post_request()  argument
174 	const struct tegra_bpmp_ops *ops = channel_to_ops(channel);  in tegra_bpmp_post_request()
176 	return ops->post_request(channel);  in tegra_bpmp_post_request()
179 static int tegra_bpmp_post_response(struct tegra_bpmp_channel *channel)  in tegra_bpmp_post_response()  argument
181 	const struct tegra_bpmp_ops *ops = channel_to_ops(channel);  in tegra_bpmp_post_response()
183 	return ops->post_response(channel);  in tegra_bpmp_post_response()
191 static ssize_t __tegra_bpmp_channel_read(struct tegra_bpmp_channel *channel,  in __tegra_bpmp_channel_read()  argument
197 		tegra_bpmp_mb_read(data, &channel->ib, size);  in __tegra_bpmp_channel_read()
199 	err = tegra_bpmp_ack_response(channel);  in __tegra_bpmp_channel_read()
203 	*ret = tegra_bpmp_mb_read_field(&channel->ib, code);  in __tegra_bpmp_channel_read()
208 static ssize_t tegra_bpmp_channel_read(struct tegra_bpmp_channel *channel,  in tegra_bpmp_channel_read()  argument
211 	struct tegra_bpmp *bpmp = channel->bpmp;  in tegra_bpmp_channel_read()
216 	index = tegra_bpmp_channel_get_thread_index(channel);  in tegra_bpmp_channel_read()
223 	err = __tegra_bpmp_channel_read(channel, data, size, ret);  in tegra_bpmp_channel_read()
233 static ssize_t __tegra_bpmp_channel_write(struct tegra_bpmp_channel *channel,  in __tegra_bpmp_channel_write()  argument
237 	tegra_bpmp_mb_write_field(&channel->ob, code, mrq);  in __tegra_bpmp_channel_write()
238 	tegra_bpmp_mb_write_field(&channel->ob, flags, flags);  in __tegra_bpmp_channel_write()
241 		tegra_bpmp_mb_write(&channel->ob, data, size);  in __tegra_bpmp_channel_write()
243 	return tegra_bpmp_post_request(channel);  in __tegra_bpmp_channel_write()
252 	struct tegra_bpmp_channel *channel;  in tegra_bpmp_write_threaded()  local
269 	channel = &bpmp->threaded_channels[index];  in tegra_bpmp_write_threaded()
271 	if (!tegra_bpmp_is_request_channel_free(channel)) {  in tegra_bpmp_write_threaded()
278 	err = __tegra_bpmp_channel_write(channel, mrq, MSG_ACK | MSG_RING,  in tegra_bpmp_write_threaded()
286 	return channel;  in tegra_bpmp_write_threaded()
297 static ssize_t tegra_bpmp_channel_write(struct tegra_bpmp_channel *channel,  in tegra_bpmp_channel_write()  argument
303 	err = tegra_bpmp_wait_request_channel_free(channel);  in tegra_bpmp_channel_write()
307 	return __tegra_bpmp_channel_write(channel, mrq, flags, data, size);  in tegra_bpmp_channel_write()
315 	struct tegra_bpmp_channel *channel;  in tegra_bpmp_transfer_atomic()  local
332 	channel = bpmp->tx_channel;  in tegra_bpmp_transfer_atomic()
336 	err = tegra_bpmp_channel_write(channel, msg->mrq, MSG_ACK,  in tegra_bpmp_transfer_atomic()
349 	err = tegra_bpmp_wait_response(channel);  in tegra_bpmp_transfer_atomic()
353 	return __tegra_bpmp_channel_read(channel, msg->rx.data, msg->rx.size,  in tegra_bpmp_transfer_atomic()
361 	struct tegra_bpmp_channel *channel;  in tegra_bpmp_transfer()  local
379 	channel = tegra_bpmp_write_threaded(bpmp, msg->mrq, msg->tx.data,  in tegra_bpmp_transfer()
381 	if (IS_ERR(channel))  in tegra_bpmp_transfer()
382 		return PTR_ERR(channel);  in tegra_bpmp_transfer()
390 	err = wait_for_completion_timeout(&channel->completion, timeout);  in tegra_bpmp_transfer()
394 	return tegra_bpmp_channel_read(channel, msg->rx.data, msg->rx.size,  in tegra_bpmp_transfer()
411 void tegra_bpmp_mrq_return(struct tegra_bpmp_channel *channel, int code,  in tegra_bpmp_mrq_return()  argument
414 	unsigned long flags = tegra_bpmp_mb_read_field(&channel->ib, flags);  in tegra_bpmp_mrq_return()
415 	struct tegra_bpmp *bpmp = channel->bpmp;  in tegra_bpmp_mrq_return()
421 	err = tegra_bpmp_ack_request(channel);  in tegra_bpmp_mrq_return()
428 	if (WARN_ON(!tegra_bpmp_is_response_channel_free(channel)))  in tegra_bpmp_mrq_return()
431 	tegra_bpmp_mb_write_field(&channel->ob, code, code);  in tegra_bpmp_mrq_return()
434 		tegra_bpmp_mb_write(&channel->ob, data, size);  in tegra_bpmp_mrq_return()
436 	err = tegra_bpmp_post_response(channel);  in tegra_bpmp_mrq_return()
450 				  struct tegra_bpmp_channel *channel)  in tegra_bpmp_handle_mrq()  argument
460 		tegra_bpmp_mrq_return(channel, -EINVAL, &zero, sizeof(zero));  in tegra_bpmp_handle_mrq()
464 	entry->handler(mrq, channel, entry->data);  in tegra_bpmp_handle_mrq()
540 				       struct tegra_bpmp_channel *channel,  in tegra_bpmp_mrq_handle_ping()  argument
546 	tegra_bpmp_mb_read(&request, &channel->ib, sizeof(request));  in tegra_bpmp_mrq_handle_ping()
551 	tegra_bpmp_mrq_return(channel, 0, &response, sizeof(response));  in tegra_bpmp_mrq_handle_ping()
660 static void tegra_bpmp_channel_signal(struct tegra_bpmp_channel *channel)  in tegra_bpmp_channel_signal()  argument
662 	unsigned long flags = tegra_bpmp_mb_read_field(&channel->ob, flags);  in tegra_bpmp_channel_signal()
667 	complete(&channel->completion);  in tegra_bpmp_channel_signal()
672 	struct tegra_bpmp_channel *channel;  in tegra_bpmp_handle_rx()  local
676 	channel = bpmp->rx_channel;  in tegra_bpmp_handle_rx()
680 	if (tegra_bpmp_is_request_ready(channel)) {  in tegra_bpmp_handle_rx()
681 		unsigned int mrq = tegra_bpmp_mb_read_field(&channel->ib, code);  in tegra_bpmp_handle_rx()
683 		tegra_bpmp_handle_mrq(bpmp, mrq, channel);  in tegra_bpmp_handle_rx()
689 		struct tegra_bpmp_channel *channel;  in tegra_bpmp_handle_rx()  local
691 		channel = &bpmp->threaded_channels[i];  in tegra_bpmp_handle_rx()
693 		if (tegra_bpmp_is_response_ready(channel)) {  in tegra_bpmp_handle_rx()
694 			tegra_bpmp_channel_signal(channel);  in tegra_bpmp_handle_rx()