1 // SPDX-License-Identifier: GPL-2.0-only
18 #include <soc/tegra/bpmp-abi.h>
21 #include "bpmp-private.h"
28 channel_to_ops(struct tegra_bpmp_channel *channel)  in channel_to_ops()  argument
30 	struct tegra_bpmp *bpmp = channel->bpmp;  in channel_to_ops()
32 	return bpmp->soc->ops;  in channel_to_ops()
41 	np = of_parse_phandle(dev->of_node, "nvidia,bpmp", 0);  in tegra_bpmp_get()
43 		return ERR_PTR(-ENOENT);  in tegra_bpmp_get()
47 		bpmp = ERR_PTR(-ENODEV);  in tegra_bpmp_get()
53 		bpmp = ERR_PTR(-EPROBE_DEFER);  in tegra_bpmp_get()
54 		put_device(&pdev->dev);  in tegra_bpmp_get()
67 		put_device(bpmp->dev);  in tegra_bpmp_put()
72 tegra_bpmp_channel_get_thread_index(struct tegra_bpmp_channel *channel)  in tegra_bpmp_channel_get_thread_index()  argument
74 	struct tegra_bpmp *bpmp = channel->bpmp;  in tegra_bpmp_channel_get_thread_index()
78 	count = bpmp->soc->channels.thread.count;  in tegra_bpmp_channel_get_thread_index()
80 	index = channel - channel->bpmp->threaded_channels;  in tegra_bpmp_channel_get_thread_index()
82 		return -EINVAL;  in tegra_bpmp_channel_get_thread_index()
89 	return (msg->tx.size <= MSG_DATA_MIN_SZ) &&  in tegra_bpmp_message_valid()
90 	       (msg->rx.size <= MSG_DATA_MIN_SZ) &&  in tegra_bpmp_message_valid()
91 	       (msg->tx.size == 0 || msg->tx.data) &&  in tegra_bpmp_message_valid()
92 	       (msg->rx.size == 0 || msg->rx.data);  in tegra_bpmp_message_valid()
95 static bool tegra_bpmp_is_response_ready(struct tegra_bpmp_channel *channel)  in tegra_bpmp_is_response_ready()  argument
97 	const struct tegra_bpmp_ops *ops = channel_to_ops(channel);  in tegra_bpmp_is_response_ready()
99 	return ops->is_response_ready(channel);  in tegra_bpmp_is_response_ready()
102 static bool tegra_bpmp_is_request_ready(struct tegra_bpmp_channel *channel)  in tegra_bpmp_is_request_ready()  argument
104 	const struct tegra_bpmp_ops *ops = channel_to_ops(channel);  in tegra_bpmp_is_request_ready()
106 	return ops->is_request_ready(channel);  in tegra_bpmp_is_request_ready()
109 static int tegra_bpmp_wait_response(struct tegra_bpmp_channel *channel)  in tegra_bpmp_wait_response()  argument
111 	unsigned long timeout = channel->bpmp->soc->channels.cpu_tx.timeout;  in tegra_bpmp_wait_response()
117 		if (tegra_bpmp_is_response_ready(channel))  in tegra_bpmp_wait_response()
121 	return -ETIMEDOUT;  in tegra_bpmp_wait_response()
124 static int tegra_bpmp_ack_response(struct tegra_bpmp_channel *channel)  in tegra_bpmp_ack_response()  argument
126 	const struct tegra_bpmp_ops *ops = channel_to_ops(channel);  in tegra_bpmp_ack_response()
128 	return ops->ack_response(channel);  in tegra_bpmp_ack_response()
131 static int tegra_bpmp_ack_request(struct tegra_bpmp_channel *channel)  in tegra_bpmp_ack_request()  argument
133 	const struct tegra_bpmp_ops *ops = channel_to_ops(channel);  in tegra_bpmp_ack_request()
135 	return ops->ack_request(channel);  in tegra_bpmp_ack_request()
139 tegra_bpmp_is_request_channel_free(struct tegra_bpmp_channel *channel)  in tegra_bpmp_is_request_channel_free()  argument
141 	const struct tegra_bpmp_ops *ops = channel_to_ops(channel);  in tegra_bpmp_is_request_channel_free()
143 	return ops->is_request_channel_free(channel);  in tegra_bpmp_is_request_channel_free()
147 tegra_bpmp_is_response_channel_free(struct tegra_bpmp_channel *channel)  in tegra_bpmp_is_response_channel_free()  argument
149 	const struct tegra_bpmp_ops *ops = channel_to_ops(channel);  in tegra_bpmp_is_response_channel_free()
151 	return ops->is_response_channel_free(channel);  in tegra_bpmp_is_response_channel_free()
155 tegra_bpmp_wait_request_channel_free(struct tegra_bpmp_channel *channel)  in tegra_bpmp_wait_request_channel_free()  argument
157 	unsigned long timeout = channel->bpmp->soc->channels.cpu_tx.timeout;  in tegra_bpmp_wait_request_channel_free()
163 		if (tegra_bpmp_is_request_channel_free(channel))  in tegra_bpmp_wait_request_channel_free()
169 	return -ETIMEDOUT;  in tegra_bpmp_wait_request_channel_free()
172 static int tegra_bpmp_post_request(struct tegra_bpmp_channel *channel)  in tegra_bpmp_post_request()  argument
174 	const struct tegra_bpmp_ops *ops = channel_to_ops(channel);  in tegra_bpmp_post_request()
176 	return ops->post_request(channel);  in tegra_bpmp_post_request()
179 static int tegra_bpmp_post_response(struct tegra_bpmp_channel *channel)  in tegra_bpmp_post_response()  argument
181 	const struct tegra_bpmp_ops *ops = channel_to_ops(channel);  in tegra_bpmp_post_response()
183 	return ops->post_response(channel);  in tegra_bpmp_post_response()
188 	return bpmp->soc->ops->ring_doorbell(bpmp);  in tegra_bpmp_ring_doorbell()
191 static ssize_t __tegra_bpmp_channel_read(struct tegra_bpmp_channel *channel,  in __tegra_bpmp_channel_read()  argument
192 					 void *data, size_t size, int *ret)  in __tegra_bpmp_channel_read()  argument
196 	if (data && size > 0)  in __tegra_bpmp_channel_read()
197 		tegra_bpmp_mb_read(data, &channel->ib, size);  in __tegra_bpmp_channel_read()
199 	err = tegra_bpmp_ack_response(channel);  in __tegra_bpmp_channel_read()
203 	*ret = tegra_bpmp_mb_read_field(&channel->ib, code);  in __tegra_bpmp_channel_read()
208 static ssize_t tegra_bpmp_channel_read(struct tegra_bpmp_channel *channel,  in tegra_bpmp_channel_read()  argument
209 				       void *data, size_t size, int *ret)  in tegra_bpmp_channel_read()  argument
211 	struct tegra_bpmp *bpmp = channel->bpmp;  in tegra_bpmp_channel_read()
216 	index = tegra_bpmp_channel_get_thread_index(channel);  in tegra_bpmp_channel_read()
222 	spin_lock_irqsave(&bpmp->lock, flags);  in tegra_bpmp_channel_read()
223 	err = __tegra_bpmp_channel_read(channel, data, size, ret);  in tegra_bpmp_channel_read()
224 	clear_bit(index, bpmp->threaded.allocated);  in tegra_bpmp_channel_read()
225 	spin_unlock_irqrestore(&bpmp->lock, flags);  in tegra_bpmp_channel_read()
228 	up(&bpmp->threaded.lock);  in tegra_bpmp_channel_read()
233 static ssize_t __tegra_bpmp_channel_write(struct tegra_bpmp_channel *channel,  in __tegra_bpmp_channel_write()  argument
235 					  const void *data, size_t size)  in __tegra_bpmp_channel_write()  argument
237 	tegra_bpmp_mb_write_field(&channel->ob, code, mrq);  in __tegra_bpmp_channel_write()
238 	tegra_bpmp_mb_write_field(&channel->ob, flags, flags);  in __tegra_bpmp_channel_write()
240 	if (data && size > 0)  in __tegra_bpmp_channel_write()
241 		tegra_bpmp_mb_write(&channel->ob, data, size);  in __tegra_bpmp_channel_write()
243 	return tegra_bpmp_post_request(channel);  in __tegra_bpmp_channel_write()
248 			  const void *data, size_t size)  in tegra_bpmp_write_threaded()  argument
250 	unsigned long timeout = bpmp->soc->channels.thread.timeout;  in tegra_bpmp_write_threaded()
251 	unsigned int count = bpmp->soc->channels.thread.count;  in tegra_bpmp_write_threaded()
252 	struct tegra_bpmp_channel *channel;  in tegra_bpmp_write_threaded()  local
257 	err = down_timeout(&bpmp->threaded.lock, usecs_to_jiffies(timeout));  in tegra_bpmp_write_threaded()
261 	spin_lock_irqsave(&bpmp->lock, flags);  in tegra_bpmp_write_threaded()
263 	index = find_first_zero_bit(bpmp->threaded.allocated, count);  in tegra_bpmp_write_threaded()
265 		err = -EBUSY;  in tegra_bpmp_write_threaded()
269 	channel = &bpmp->threaded_channels[index];  in tegra_bpmp_write_threaded()
271 	if (!tegra_bpmp_is_request_channel_free(channel)) {  in tegra_bpmp_write_threaded()
272 		err = -EBUSY;  in tegra_bpmp_write_threaded()
276 	set_bit(index, bpmp->threaded.allocated);  in tegra_bpmp_write_threaded()
278 	err = __tegra_bpmp_channel_write(channel, mrq, MSG_ACK | MSG_RING,  in tegra_bpmp_write_threaded()
279 					 data, size);  in tegra_bpmp_write_threaded()
283 	set_bit(index, bpmp->threaded.busy);  in tegra_bpmp_write_threaded()
285 	spin_unlock_irqrestore(&bpmp->lock, flags);  in tegra_bpmp_write_threaded()
286 	return channel;  in tegra_bpmp_write_threaded()
289 	clear_bit(index, bpmp->threaded.allocated);  in tegra_bpmp_write_threaded()
291 	spin_unlock_irqrestore(&bpmp->lock, flags);  in tegra_bpmp_write_threaded()
292 	up(&bpmp->threaded.lock);  in tegra_bpmp_write_threaded()
297 static ssize_t tegra_bpmp_channel_write(struct tegra_bpmp_channel *channel,  in tegra_bpmp_channel_write()  argument
299 					const void *data, size_t size)  in tegra_bpmp_channel_write()  argument
303 	err = tegra_bpmp_wait_request_channel_free(channel);  in tegra_bpmp_channel_write()
307 	return __tegra_bpmp_channel_write(channel, mrq, flags, data, size);  in tegra_bpmp_channel_write()
315 	struct tegra_bpmp_channel *channel;  in tegra_bpmp_transfer_atomic()  local
319 		return -EPERM;  in tegra_bpmp_transfer_atomic()
322 		return -EINVAL;  in tegra_bpmp_transfer_atomic()
324 	if (bpmp->suspended) {  in tegra_bpmp_transfer_atomic()
326 		if (msg->flags & TEGRA_BPMP_MESSAGE_RESET)  in tegra_bpmp_transfer_atomic()
327 			tegra_bpmp_resume(bpmp->dev);  in tegra_bpmp_transfer_atomic()
329 			return -EAGAIN;  in tegra_bpmp_transfer_atomic()
332 	channel = bpmp->tx_channel;  in tegra_bpmp_transfer_atomic()
334 	spin_lock(&bpmp->atomic_tx_lock);  in tegra_bpmp_transfer_atomic()
336 	err = tegra_bpmp_channel_write(channel, msg->mrq, MSG_ACK,  in tegra_bpmp_transfer_atomic()
337 				       msg->tx.data, msg->tx.size);  in tegra_bpmp_transfer_atomic()
339 		spin_unlock(&bpmp->atomic_tx_lock);  in tegra_bpmp_transfer_atomic()
343 	spin_unlock(&bpmp->atomic_tx_lock);  in tegra_bpmp_transfer_atomic()
349 	err = tegra_bpmp_wait_response(channel);  in tegra_bpmp_transfer_atomic()
353 	return __tegra_bpmp_channel_read(channel, msg->rx.data, msg->rx.size,  in tegra_bpmp_transfer_atomic()
354 					 &msg->rx.ret);  in tegra_bpmp_transfer_atomic()
361 	struct tegra_bpmp_channel *channel;  in tegra_bpmp_transfer()  local
366 		return -EPERM;  in tegra_bpmp_transfer()
369 		return -EINVAL;  in tegra_bpmp_transfer()
371 	if (bpmp->suspended) {  in tegra_bpmp_transfer()
373 		if (msg->flags & TEGRA_BPMP_MESSAGE_RESET)  in tegra_bpmp_transfer()
374 			tegra_bpmp_resume(bpmp->dev);  in tegra_bpmp_transfer()
376 			return -EAGAIN;  in tegra_bpmp_transfer()
379 	channel = tegra_bpmp_write_threaded(bpmp, msg->mrq, msg->tx.data,  in tegra_bpmp_transfer()
380 					    msg->tx.size);  in tegra_bpmp_transfer()
381 	if (IS_ERR(channel))  in tegra_bpmp_transfer()
382 		return PTR_ERR(channel);  in tegra_bpmp_transfer()
388 	timeout = usecs_to_jiffies(bpmp->soc->channels.thread.timeout);  in tegra_bpmp_transfer()
390 	err = wait_for_completion_timeout(&channel->completion, timeout);  in tegra_bpmp_transfer()
392 		return -ETIMEDOUT;  in tegra_bpmp_transfer()
394 	return tegra_bpmp_channel_read(channel, msg->rx.data, msg->rx.size,  in tegra_bpmp_transfer()
395 				       &msg->rx.ret);  in tegra_bpmp_transfer()
404 	list_for_each_entry(entry, &bpmp->mrqs, list)  in tegra_bpmp_find_mrq()
405 		if (entry->mrq == mrq)  in tegra_bpmp_find_mrq()
411 void tegra_bpmp_mrq_return(struct tegra_bpmp_channel *channel, int code,  in tegra_bpmp_mrq_return()  argument
412 			   const void *data, size_t size)  in tegra_bpmp_mrq_return()  argument
414 	unsigned long flags = tegra_bpmp_mb_read_field(&channel->ib, flags);  in tegra_bpmp_mrq_return()
415 	struct tegra_bpmp *bpmp = channel->bpmp;  in tegra_bpmp_mrq_return()
421 	err = tegra_bpmp_ack_request(channel);  in tegra_bpmp_mrq_return()
428 	if (WARN_ON(!tegra_bpmp_is_response_channel_free(channel)))  in tegra_bpmp_mrq_return()
431 	tegra_bpmp_mb_write_field(&channel->ob, code, code);  in tegra_bpmp_mrq_return()
433 	if (data && size > 0)  in tegra_bpmp_mrq_return()
434 		tegra_bpmp_mb_write(&channel->ob, data, size);  in tegra_bpmp_mrq_return()
436 	err = tegra_bpmp_post_response(channel);  in tegra_bpmp_mrq_return()
450 				  struct tegra_bpmp_channel *channel)  in tegra_bpmp_handle_mrq()  argument
455 	spin_lock(&bpmp->lock);  in tegra_bpmp_handle_mrq()
459 		spin_unlock(&bpmp->lock);  in tegra_bpmp_handle_mrq()
460 		tegra_bpmp_mrq_return(channel, -EINVAL, &zero, sizeof(zero));  in tegra_bpmp_handle_mrq()
464 	entry->handler(mrq, channel, entry->data);  in tegra_bpmp_handle_mrq()
466 	spin_unlock(&bpmp->lock);  in tegra_bpmp_handle_mrq()
470 			   tegra_bpmp_mrq_handler_t handler, void *data)  in tegra_bpmp_request_mrq()  argument
476 		return -EINVAL;  in tegra_bpmp_request_mrq()
478 	entry = devm_kzalloc(bpmp->dev, sizeof(*entry), GFP_KERNEL);  in tegra_bpmp_request_mrq()
480 		return -ENOMEM;  in tegra_bpmp_request_mrq()
482 	spin_lock_irqsave(&bpmp->lock, flags);  in tegra_bpmp_request_mrq()
484 	entry->mrq = mrq;  in tegra_bpmp_request_mrq()
485 	entry->handler = handler;  in tegra_bpmp_request_mrq()
486 	entry->data = data;  in tegra_bpmp_request_mrq()
487 	list_add(&entry->list, &bpmp->mrqs);  in tegra_bpmp_request_mrq()
489 	spin_unlock_irqrestore(&bpmp->lock, flags);  in tegra_bpmp_request_mrq()
495 void tegra_bpmp_free_mrq(struct tegra_bpmp *bpmp, unsigned int mrq, void *data)  in tegra_bpmp_free_mrq()  argument
500 	spin_lock_irqsave(&bpmp->lock, flags);  in tegra_bpmp_free_mrq()
506 	list_del(&entry->list);  in tegra_bpmp_free_mrq()
507 	devm_kfree(bpmp->dev, entry);  in tegra_bpmp_free_mrq()
510 	spin_unlock_irqrestore(&bpmp->lock, flags);  in tegra_bpmp_free_mrq()
521 			.data = &req,  in tegra_bpmp_mrq_is_supported()
525 			.data = &resp,  in tegra_bpmp_mrq_is_supported()
540 				       struct tegra_bpmp_channel *channel,  in tegra_bpmp_mrq_handle_ping()  argument
541 				       void *data)  in tegra_bpmp_mrq_handle_ping()  argument
546 	tegra_bpmp_mb_read(&request, &channel->ib, sizeof(request));  in tegra_bpmp_mrq_handle_ping()
551 	tegra_bpmp_mrq_return(channel, 0, &response, sizeof(response));  in tegra_bpmp_mrq_handle_ping()
570 	msg.tx.data = &request;  in tegra_bpmp_ping()
572 	msg.rx.data = &response;  in tegra_bpmp_ping()
582 		dev_dbg(bpmp->dev,  in tegra_bpmp_ping()
602 		return -EINVAL;  in tegra_bpmp_get_firmware_tag_old()
604 	virt = dma_alloc_coherent(bpmp->dev, TAG_SZ, &phys,  in tegra_bpmp_get_firmware_tag_old()
607 		return -ENOMEM;  in tegra_bpmp_get_firmware_tag_old()
614 	msg.tx.data = &request;  in tegra_bpmp_get_firmware_tag_old()
624 	dma_free_coherent(bpmp->dev, TAG_SZ, virt, phys);  in tegra_bpmp_get_firmware_tag_old()
637 				.data = &resp,  in tegra_bpmp_get_firmware_tag()
644 			return -EINVAL;  in tegra_bpmp_get_firmware_tag()
651 			return -EINVAL;  in tegra_bpmp_get_firmware_tag()
660 static void tegra_bpmp_channel_signal(struct tegra_bpmp_channel *channel)  in tegra_bpmp_channel_signal()  argument
662 	unsigned long flags = tegra_bpmp_mb_read_field(&channel->ob, flags);  in tegra_bpmp_channel_signal()
667 	complete(&channel->completion);  in tegra_bpmp_channel_signal()
672 	struct tegra_bpmp_channel *channel;  in tegra_bpmp_handle_rx()  local
676 	channel = bpmp->rx_channel;  in tegra_bpmp_handle_rx()
677 	count = bpmp->soc->channels.thread.count;  in tegra_bpmp_handle_rx()
678 	busy = bpmp->threaded.busy;  in tegra_bpmp_handle_rx()
680 	if (tegra_bpmp_is_request_ready(channel)) {  in tegra_bpmp_handle_rx()
681 		unsigned int mrq = tegra_bpmp_mb_read_field(&channel->ib, code);  in tegra_bpmp_handle_rx()
683 		tegra_bpmp_handle_mrq(bpmp, mrq, channel);  in tegra_bpmp_handle_rx()
686 	spin_lock(&bpmp->lock);  in tegra_bpmp_handle_rx()
689 		struct tegra_bpmp_channel *channel;  in tegra_bpmp_handle_rx()  local
691 		channel = &bpmp->threaded_channels[i];  in tegra_bpmp_handle_rx()
693 		if (tegra_bpmp_is_response_ready(channel)) {  in tegra_bpmp_handle_rx()
694 			tegra_bpmp_channel_signal(channel);  in tegra_bpmp_handle_rx()
699 	spin_unlock(&bpmp->lock);  in tegra_bpmp_handle_rx()
709 	bpmp = devm_kzalloc(&pdev->dev, sizeof(*bpmp), GFP_KERNEL);  in tegra_bpmp_probe()
711 		return -ENOMEM;  in tegra_bpmp_probe()
713 	bpmp->soc = of_device_get_match_data(&pdev->dev);  in tegra_bpmp_probe()
714 	bpmp->dev = &pdev->dev;  in tegra_bpmp_probe()
716 	INIT_LIST_HEAD(&bpmp->mrqs);  in tegra_bpmp_probe()
717 	spin_lock_init(&bpmp->lock);  in tegra_bpmp_probe()
719 	bpmp->threaded.count = bpmp->soc->channels.thread.count;  in tegra_bpmp_probe()
720 	sema_init(&bpmp->threaded.lock, bpmp->threaded.count);  in tegra_bpmp_probe()
722 	size = BITS_TO_LONGS(bpmp->threaded.count) * sizeof(long);  in tegra_bpmp_probe()
724 	bpmp->threaded.allocated = devm_kzalloc(&pdev->dev, size, GFP_KERNEL);  in tegra_bpmp_probe()
725 	if (!bpmp->threaded.allocated)  in tegra_bpmp_probe()
726 		return -ENOMEM;  in tegra_bpmp_probe()
728 	bpmp->threaded.busy = devm_kzalloc(&pdev->dev, size, GFP_KERNEL);  in tegra_bpmp_probe()
729 	if (!bpmp->threaded.busy)  in tegra_bpmp_probe()
730 		return -ENOMEM;  in tegra_bpmp_probe()
732 	spin_lock_init(&bpmp->atomic_tx_lock);  in tegra_bpmp_probe()
733 	bpmp->tx_channel = devm_kzalloc(&pdev->dev, sizeof(*bpmp->tx_channel),  in tegra_bpmp_probe()
735 	if (!bpmp->tx_channel)  in tegra_bpmp_probe()
736 		return -ENOMEM;  in tegra_bpmp_probe()
738 	bpmp->rx_channel = devm_kzalloc(&pdev->dev, sizeof(*bpmp->rx_channel),  in tegra_bpmp_probe()
740 	if (!bpmp->rx_channel)  in tegra_bpmp_probe()
741 		return -ENOMEM;  in tegra_bpmp_probe()
743 	bpmp->threaded_channels = devm_kcalloc(&pdev->dev, bpmp->threaded.count,  in tegra_bpmp_probe()
744 					       sizeof(*bpmp->threaded_channels),  in tegra_bpmp_probe()
746 	if (!bpmp->threaded_channels)  in tegra_bpmp_probe()
747 		return -ENOMEM;  in tegra_bpmp_probe()
751 	err = bpmp->soc->ops->init(bpmp);  in tegra_bpmp_probe()
762 		dev_err(&pdev->dev, "failed to ping BPMP: %d\n", err);  in tegra_bpmp_probe()
768 		dev_err(&pdev->dev, "failed to get firmware tag: %d\n", err);  in tegra_bpmp_probe()
772 	dev_info(&pdev->dev, "firmware: %.*s\n", (int)sizeof(tag), tag);  in tegra_bpmp_probe()
774 	err = of_platform_default_populate(pdev->dev.of_node, NULL, &pdev->dev);  in tegra_bpmp_probe()
778 	if (of_property_present(pdev->dev.of_node, "#clock-cells")) {  in tegra_bpmp_probe()
784 	if (of_property_present(pdev->dev.of_node, "#reset-cells")) {  in tegra_bpmp_probe()
790 	if (of_property_present(pdev->dev.of_node, "#power-domain-cells")) {  in tegra_bpmp_probe()
798 		dev_err(&pdev->dev, "debugfs initialization failed: %d\n", err);  in tegra_bpmp_probe()
805 	if (bpmp->soc->ops->deinit)  in tegra_bpmp_probe()
806 		bpmp->soc->ops->deinit(bpmp);  in tegra_bpmp_probe()
815 	bpmp->suspended = true;  in tegra_bpmp_suspend()
824 	bpmp->suspended = false;  in tegra_bpmp_resume()
826 	if (bpmp->soc->ops->resume)  in tegra_bpmp_resume()
827 		return bpmp->soc->ops->resume(bpmp);  in tegra_bpmp_resume()
888 	{ .compatible = "nvidia,tegra186-bpmp", .data = &tegra186_soc },
891 	{ .compatible = "nvidia,tegra210-bpmp", .data = &tegra210_soc },
898 		.name = "tegra-bpmp",