xref: /linux/drivers/firmware/tegra/bpmp.c (revision ebf68996de0ab250c5d520eb2291ab65643e9a1e) 
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Copyright (c) 2016, NVIDIA CORPORATION.  All rights reserved.
4  */
5 
6 #include <linux/clk/tegra.h>
7 #include <linux/genalloc.h>
8 #include <linux/mailbox_client.h>
9 #include <linux/of.h>
10 #include <linux/of_address.h>
11 #include <linux/of_device.h>
12 #include <linux/platform_device.h>
13 #include <linux/pm.h>
14 #include <linux/semaphore.h>
15 #include <linux/sched/clock.h>
16 
17 #include <soc/tegra/bpmp.h>
18 #include <soc/tegra/bpmp-abi.h>
19 #include <soc/tegra/ivc.h>
20 
21 #include "bpmp-private.h"
22 
23 #define MSG_ACK		BIT(0)
24 #define MSG_RING	BIT(1)
25 #define TAG_SZ		32
26 
27 static inline struct tegra_bpmp *
28 mbox_client_to_bpmp(struct mbox_client *client)
29 {
30 	return container_of(client, struct tegra_bpmp, mbox.client);
31 }
32 
33 static inline const struct tegra_bpmp_ops *
34 channel_to_ops(struct tegra_bpmp_channel *channel)
35 {
36 	struct tegra_bpmp *bpmp = channel->bpmp;
37 
38 	return bpmp->soc->ops;
39 }
40 
41 struct tegra_bpmp *tegra_bpmp_get(struct device *dev)
42 {
43 	struct platform_device *pdev;
44 	struct tegra_bpmp *bpmp;
45 	struct device_node *np;
46 
47 	np = of_parse_phandle(dev->of_node, "nvidia,bpmp", 0);
48 	if (!np)
49 		return ERR_PTR(-ENOENT);
50 
51 	pdev = of_find_device_by_node(np);
52 	if (!pdev) {
53 		bpmp = ERR_PTR(-ENODEV);
54 		goto put;
55 	}
56 
57 	bpmp = platform_get_drvdata(pdev);
58 	if (!bpmp) {
59 		bpmp = ERR_PTR(-EPROBE_DEFER);
60 		put_device(&pdev->dev);
61 		goto put;
62 	}
63 
64 put:
65 	of_node_put(np);
66 	return bpmp;
67 }
68 EXPORT_SYMBOL_GPL(tegra_bpmp_get);
69 
70 void tegra_bpmp_put(struct tegra_bpmp *bpmp)
71 {
72 	if (bpmp)
73 		put_device(bpmp->dev);
74 }
75 EXPORT_SYMBOL_GPL(tegra_bpmp_put);
76 
77 static int
78 tegra_bpmp_channel_get_thread_index(struct tegra_bpmp_channel *channel)
79 {
80 	struct tegra_bpmp *bpmp = channel->bpmp;
81 	unsigned int count;
82 	int index;
83 
84 	count = bpmp->soc->channels.thread.count;
85 
86 	index = channel - channel->bpmp->threaded_channels;
87 	if (index < 0 || index >= count)
88 		return -EINVAL;
89 
90 	return index;
91 }
92 
93 static bool tegra_bpmp_message_valid(const struct tegra_bpmp_message *msg)
94 {
95 	return (msg->tx.size <= MSG_DATA_MIN_SZ) &&
96 	       (msg->rx.size <= MSG_DATA_MIN_SZ) &&
97 	       (msg->tx.size == 0 || msg->tx.data) &&
98 	       (msg->rx.size == 0 || msg->rx.data);
99 }
100 
101 static bool tegra_bpmp_is_response_ready(struct tegra_bpmp_channel *channel)
102 {
103 	const struct tegra_bpmp_ops *ops = channel_to_ops(channel);
104 
105 	return ops->is_response_ready(channel);
106 }
107 
108 static bool tegra_bpmp_is_request_ready(struct tegra_bpmp_channel *channel)
109 {
110 	const struct tegra_bpmp_ops *ops = channel_to_ops(channel);
111 
112 	return ops->is_request_ready(channel);
113 }
114 
115 static int tegra_bpmp_wait_response(struct tegra_bpmp_channel *channel)
116 {
117 	unsigned long timeout = channel->bpmp->soc->channels.cpu_tx.timeout;
118 	ktime_t end;
119 
120 	end = ktime_add_us(ktime_get(), timeout);
121 
122 	do {
123 		if (tegra_bpmp_is_response_ready(channel))
124 			return 0;
125 	} while (ktime_before(ktime_get(), end));
126 
127 	return -ETIMEDOUT;
128 }
129 
130 static int tegra_bpmp_ack_response(struct tegra_bpmp_channel *channel)
131 {
132 	const struct tegra_bpmp_ops *ops = channel_to_ops(channel);
133 
134 	return ops->ack_response(channel);
135 }
136 
137 static int tegra_bpmp_ack_request(struct tegra_bpmp_channel *channel)
138 {
139 	const struct tegra_bpmp_ops *ops = channel_to_ops(channel);
140 
141 	return ops->ack_request(channel);
142 }
143 
144 static bool
145 tegra_bpmp_is_request_channel_free(struct tegra_bpmp_channel *channel)
146 {
147 	const struct tegra_bpmp_ops *ops = channel_to_ops(channel);
148 
149 	return ops->is_request_channel_free(channel);
150 }
151 
152 static bool
153 tegra_bpmp_is_response_channel_free(struct tegra_bpmp_channel *channel)
154 {
155 	const struct tegra_bpmp_ops *ops = channel_to_ops(channel);
156 
157 	return ops->is_response_channel_free(channel);
158 }
159 
160 static int
161 tegra_bpmp_wait_request_channel_free(struct tegra_bpmp_channel *channel)
162 {
163 	unsigned long timeout = channel->bpmp->soc->channels.cpu_tx.timeout;
164 	ktime_t start, now;
165 
166 	start = ns_to_ktime(local_clock());
167 
168 	do {
169 		if (tegra_bpmp_is_request_channel_free(channel))
170 			return 0;
171 
172 		now = ns_to_ktime(local_clock());
173 	} while (ktime_us_delta(now, start) < timeout);
174 
175 	return -ETIMEDOUT;
176 }
177 
178 static int tegra_bpmp_post_request(struct tegra_bpmp_channel *channel)
179 {
180 	const struct tegra_bpmp_ops *ops = channel_to_ops(channel);
181 
182 	return ops->post_request(channel);
183 }
184 
185 static int tegra_bpmp_post_response(struct tegra_bpmp_channel *channel)
186 {
187 	const struct tegra_bpmp_ops *ops = channel_to_ops(channel);
188 
189 	return ops->post_response(channel);
190 }
191 
192 static int tegra_bpmp_ring_doorbell(struct tegra_bpmp *bpmp)
193 {
194 	return bpmp->soc->ops->ring_doorbell(bpmp);
195 }
196 
197 static ssize_t __tegra_bpmp_channel_read(struct tegra_bpmp_channel *channel,
198 					 void *data, size_t size, int *ret)
199 {
200 	int err;
201 
202 	if (data && size > 0)
203 		memcpy(data, channel->ib->data, size);
204 
205 	err = tegra_bpmp_ack_response(channel);
206 	if (err < 0)
207 		return err;
208 
209 	*ret = channel->ib->code;
210 
211 	return 0;
212 }
213 
214 static ssize_t tegra_bpmp_channel_read(struct tegra_bpmp_channel *channel,
215 				       void *data, size_t size, int *ret)
216 {
217 	struct tegra_bpmp *bpmp = channel->bpmp;
218 	unsigned long flags;
219 	ssize_t err;
220 	int index;
221 
222 	index = tegra_bpmp_channel_get_thread_index(channel);
223 	if (index < 0) {
224 		err = index;
225 		goto unlock;
226 	}
227 
228 	spin_lock_irqsave(&bpmp->lock, flags);
229 	err = __tegra_bpmp_channel_read(channel, data, size, ret);
230 	clear_bit(index, bpmp->threaded.allocated);
231 	spin_unlock_irqrestore(&bpmp->lock, flags);
232 
233 unlock:
234 	up(&bpmp->threaded.lock);
235 
236 	return err;
237 }
238 
239 static ssize_t __tegra_bpmp_channel_write(struct tegra_bpmp_channel *channel,
240 					  unsigned int mrq, unsigned long flags,
241 					  const void *data, size_t size)
242 {
243 	channel->ob->code = mrq;
244 	channel->ob->flags = flags;
245 
246 	if (data && size > 0)
247 		memcpy(channel->ob->data, data, size);
248 
249 	return tegra_bpmp_post_request(channel);
250 }
251 
252 static struct tegra_bpmp_channel *
253 tegra_bpmp_write_threaded(struct tegra_bpmp *bpmp, unsigned int mrq,
254 			  const void *data, size_t size)
255 {
256 	unsigned long timeout = bpmp->soc->channels.thread.timeout;
257 	unsigned int count = bpmp->soc->channels.thread.count;
258 	struct tegra_bpmp_channel *channel;
259 	unsigned long flags;
260 	unsigned int index;
261 	int err;
262 
263 	err = down_timeout(&bpmp->threaded.lock, usecs_to_jiffies(timeout));
264 	if (err < 0)
265 		return ERR_PTR(err);
266 
267 	spin_lock_irqsave(&bpmp->lock, flags);
268 
269 	index = find_first_zero_bit(bpmp->threaded.allocated, count);
270 	if (index == count) {
271 		err = -EBUSY;
272 		goto unlock;
273 	}
274 
275 	channel = &bpmp->threaded_channels[index];
276 
277 	if (!tegra_bpmp_is_request_channel_free(channel)) {
278 		err = -EBUSY;
279 		goto unlock;
280 	}
281 
282 	set_bit(index, bpmp->threaded.allocated);
283 
284 	err = __tegra_bpmp_channel_write(channel, mrq, MSG_ACK | MSG_RING,
285 					 data, size);
286 	if (err < 0)
287 		goto clear_allocated;
288 
289 	set_bit(index, bpmp->threaded.busy);
290 
291 	spin_unlock_irqrestore(&bpmp->lock, flags);
292 	return channel;
293 
294 clear_allocated:
295 	clear_bit(index, bpmp->threaded.allocated);
296 unlock:
297 	spin_unlock_irqrestore(&bpmp->lock, flags);
298 	up(&bpmp->threaded.lock);
299 
300 	return ERR_PTR(err);
301 }
302 
303 static ssize_t tegra_bpmp_channel_write(struct tegra_bpmp_channel *channel,
304 					unsigned int mrq, unsigned long flags,
305 					const void *data, size_t size)
306 {
307 	int err;
308 
309 	err = tegra_bpmp_wait_request_channel_free(channel);
310 	if (err < 0)
311 		return err;
312 
313 	return __tegra_bpmp_channel_write(channel, mrq, flags, data, size);
314 }
315 
316 int tegra_bpmp_transfer_atomic(struct tegra_bpmp *bpmp,
317 			       struct tegra_bpmp_message *msg)
318 {
319 	struct tegra_bpmp_channel *channel;
320 	int err;
321 
322 	if (WARN_ON(!irqs_disabled()))
323 		return -EPERM;
324 
325 	if (!tegra_bpmp_message_valid(msg))
326 		return -EINVAL;
327 
328 	channel = bpmp->tx_channel;
329 
330 	spin_lock(&bpmp->atomic_tx_lock);
331 
332 	err = tegra_bpmp_channel_write(channel, msg->mrq, MSG_ACK,
333 				       msg->tx.data, msg->tx.size);
334 	if (err < 0) {
335 		spin_unlock(&bpmp->atomic_tx_lock);
336 		return err;
337 	}
338 
339 	spin_unlock(&bpmp->atomic_tx_lock);
340 
341 	err = tegra_bpmp_ring_doorbell(bpmp);
342 	if (err < 0)
343 		return err;
344 
345 	err = tegra_bpmp_wait_response(channel);
346 	if (err < 0)
347 		return err;
348 
349 	return __tegra_bpmp_channel_read(channel, msg->rx.data, msg->rx.size,
350 					 &msg->rx.ret);
351 }
352 EXPORT_SYMBOL_GPL(tegra_bpmp_transfer_atomic);
353 
354 int tegra_bpmp_transfer(struct tegra_bpmp *bpmp,
355 			struct tegra_bpmp_message *msg)
356 {
357 	struct tegra_bpmp_channel *channel;
358 	unsigned long timeout;
359 	int err;
360 
361 	if (WARN_ON(irqs_disabled()))
362 		return -EPERM;
363 
364 	if (!tegra_bpmp_message_valid(msg))
365 		return -EINVAL;
366 
367 	channel = tegra_bpmp_write_threaded(bpmp, msg->mrq, msg->tx.data,
368 					    msg->tx.size);
369 	if (IS_ERR(channel))
370 		return PTR_ERR(channel);
371 
372 	err = tegra_bpmp_ring_doorbell(bpmp);
373 	if (err < 0)
374 		return err;
375 
376 	timeout = usecs_to_jiffies(bpmp->soc->channels.thread.timeout);
377 
378 	err = wait_for_completion_timeout(&channel->completion, timeout);
379 	if (err == 0)
380 		return -ETIMEDOUT;
381 
382 	return tegra_bpmp_channel_read(channel, msg->rx.data, msg->rx.size,
383 				       &msg->rx.ret);
384 }
385 EXPORT_SYMBOL_GPL(tegra_bpmp_transfer);
386 
387 static struct tegra_bpmp_mrq *tegra_bpmp_find_mrq(struct tegra_bpmp *bpmp,
388 						  unsigned int mrq)
389 {
390 	struct tegra_bpmp_mrq *entry;
391 
392 	list_for_each_entry(entry, &bpmp->mrqs, list)
393 		if (entry->mrq == mrq)
394 			return entry;
395 
396 	return NULL;
397 }
398 
399 void tegra_bpmp_mrq_return(struct tegra_bpmp_channel *channel, int code,
400 			   const void *data, size_t size)
401 {
402 	unsigned long flags = channel->ib->flags;
403 	struct tegra_bpmp *bpmp = channel->bpmp;
404 	int err;
405 
406 	if (WARN_ON(size > MSG_DATA_MIN_SZ))
407 		return;
408 
409 	err = tegra_bpmp_ack_request(channel);
410 	if (WARN_ON(err < 0))
411 		return;
412 
413 	if ((flags & MSG_ACK) == 0)
414 		return;
415 
416 	if (WARN_ON(!tegra_bpmp_is_response_channel_free(channel)))
417 		return;
418 
419 	channel->ob->code = code;
420 
421 	if (data && size > 0)
422 		memcpy(channel->ob->data, data, size);
423 
424 	err = tegra_bpmp_post_response(channel);
425 	if (WARN_ON(err < 0))
426 		return;
427 
428 	if (flags & MSG_RING) {
429 		err = tegra_bpmp_ring_doorbell(bpmp);
430 		if (WARN_ON(err < 0))
431 			return;
432 	}
433 }
434 EXPORT_SYMBOL_GPL(tegra_bpmp_mrq_return);
435 
436 static void tegra_bpmp_handle_mrq(struct tegra_bpmp *bpmp,
437 				  unsigned int mrq,
438 				  struct tegra_bpmp_channel *channel)
439 {
440 	struct tegra_bpmp_mrq *entry;
441 	u32 zero = 0;
442 
443 	spin_lock(&bpmp->lock);
444 
445 	entry = tegra_bpmp_find_mrq(bpmp, mrq);
446 	if (!entry) {
447 		spin_unlock(&bpmp->lock);
448 		tegra_bpmp_mrq_return(channel, -EINVAL, &zero, sizeof(zero));
449 		return;
450 	}
451 
452 	entry->handler(mrq, channel, entry->data);
453 
454 	spin_unlock(&bpmp->lock);
455 }
456 
457 int tegra_bpmp_request_mrq(struct tegra_bpmp *bpmp, unsigned int mrq,
458 			   tegra_bpmp_mrq_handler_t handler, void *data)
459 {
460 	struct tegra_bpmp_mrq *entry;
461 	unsigned long flags;
462 
463 	if (!handler)
464 		return -EINVAL;
465 
466 	entry = devm_kzalloc(bpmp->dev, sizeof(*entry), GFP_KERNEL);
467 	if (!entry)
468 		return -ENOMEM;
469 
470 	spin_lock_irqsave(&bpmp->lock, flags);
471 
472 	entry->mrq = mrq;
473 	entry->handler = handler;
474 	entry->data = data;
475 	list_add(&entry->list, &bpmp->mrqs);
476 
477 	spin_unlock_irqrestore(&bpmp->lock, flags);
478 
479 	return 0;
480 }
481 EXPORT_SYMBOL_GPL(tegra_bpmp_request_mrq);
482 
483 void tegra_bpmp_free_mrq(struct tegra_bpmp *bpmp, unsigned int mrq, void *data)
484 {
485 	struct tegra_bpmp_mrq *entry;
486 	unsigned long flags;
487 
488 	spin_lock_irqsave(&bpmp->lock, flags);
489 
490 	entry = tegra_bpmp_find_mrq(bpmp, mrq);
491 	if (!entry)
492 		goto unlock;
493 
494 	list_del(&entry->list);
495 	devm_kfree(bpmp->dev, entry);
496 
497 unlock:
498 	spin_unlock_irqrestore(&bpmp->lock, flags);
499 }
500 EXPORT_SYMBOL_GPL(tegra_bpmp_free_mrq);
501 
502 bool tegra_bpmp_mrq_is_supported(struct tegra_bpmp *bpmp, unsigned int mrq)
503 {
504 	struct mrq_query_abi_request req = { .mrq = cpu_to_le32(mrq) };
505 	struct mrq_query_abi_response resp;
506 	struct tegra_bpmp_message msg = {
507 		.mrq = MRQ_QUERY_ABI,
508 		.tx = {
509 			.data = &req,
510 			.size = sizeof(req),
511 		},
512 		.rx = {
513 			.data = &resp,
514 			.size = sizeof(resp),
515 		},
516 	};
517 	int ret;
518 
519 	ret = tegra_bpmp_transfer(bpmp, &msg);
520 	if (ret || msg.rx.ret)
521 		return false;
522 
523 	return resp.status == 0;
524 }
525 EXPORT_SYMBOL_GPL(tegra_bpmp_mrq_is_supported);
526 
527 static void tegra_bpmp_mrq_handle_ping(unsigned int mrq,
528 				       struct tegra_bpmp_channel *channel,
529 				       void *data)
530 {
531 	struct mrq_ping_request *request;
532 	struct mrq_ping_response response;
533 
534 	request = (struct mrq_ping_request *)channel->ib->data;
535 
536 	memset(&response, 0, sizeof(response));
537 	response.reply = request->challenge << 1;
538 
539 	tegra_bpmp_mrq_return(channel, 0, &response, sizeof(response));
540 }
541 
542 static int tegra_bpmp_ping(struct tegra_bpmp *bpmp)
543 {
544 	struct mrq_ping_response response;
545 	struct mrq_ping_request request;
546 	struct tegra_bpmp_message msg;
547 	unsigned long flags;
548 	ktime_t start, end;
549 	int err;
550 
551 	memset(&request, 0, sizeof(request));
552 	request.challenge = 1;
553 
554 	memset(&response, 0, sizeof(response));
555 
556 	memset(&msg, 0, sizeof(msg));
557 	msg.mrq = MRQ_PING;
558 	msg.tx.data = &request;
559 	msg.tx.size = sizeof(request);
560 	msg.rx.data = &response;
561 	msg.rx.size = sizeof(response);
562 
563 	local_irq_save(flags);
564 	start = ktime_get();
565 	err = tegra_bpmp_transfer_atomic(bpmp, &msg);
566 	end = ktime_get();
567 	local_irq_restore(flags);
568 
569 	if (!err)
570 		dev_dbg(bpmp->dev,
571 			"ping ok: challenge: %u, response: %u, time: %lld\n",
572 			request.challenge, response.reply,
573 			ktime_to_us(ktime_sub(end, start)));
574 
575 	return err;
576 }
577 
578 /* deprecated version of tag query */
579 static int tegra_bpmp_get_firmware_tag_old(struct tegra_bpmp *bpmp, char *tag,
580 					   size_t size)
581 {
582 	struct mrq_query_tag_request request;
583 	struct tegra_bpmp_message msg;
584 	unsigned long flags;
585 	dma_addr_t phys;
586 	void *virt;
587 	int err;
588 
589 	if (size != TAG_SZ)
590 		return -EINVAL;
591 
592 	virt = dma_alloc_coherent(bpmp->dev, TAG_SZ, &phys,
593 				  GFP_KERNEL | GFP_DMA32);
594 	if (!virt)
595 		return -ENOMEM;
596 
597 	memset(&request, 0, sizeof(request));
598 	request.addr = phys;
599 
600 	memset(&msg, 0, sizeof(msg));
601 	msg.mrq = MRQ_QUERY_TAG;
602 	msg.tx.data = &request;
603 	msg.tx.size = sizeof(request);
604 
605 	local_irq_save(flags);
606 	err = tegra_bpmp_transfer_atomic(bpmp, &msg);
607 	local_irq_restore(flags);
608 
609 	if (err == 0)
610 		memcpy(tag, virt, TAG_SZ);
611 
612 	dma_free_coherent(bpmp->dev, TAG_SZ, virt, phys);
613 
614 	return err;
615 }
616 
617 static int tegra_bpmp_get_firmware_tag(struct tegra_bpmp *bpmp, char *tag,
618 				       size_t size)
619 {
620 	if (tegra_bpmp_mrq_is_supported(bpmp, MRQ_QUERY_FW_TAG)) {
621 		struct mrq_query_fw_tag_response resp;
622 		struct tegra_bpmp_message msg = {
623 			.mrq = MRQ_QUERY_FW_TAG,
624 			.rx = {
625 				.data = &resp,
626 				.size = sizeof(resp),
627 			},
628 		};
629 		int err;
630 
631 		if (size != sizeof(resp.tag))
632 			return -EINVAL;
633 
634 		err = tegra_bpmp_transfer(bpmp, &msg);
635 
636 		if (err)
637 			return err;
638 		if (msg.rx.ret < 0)
639 			return -EINVAL;
640 
641 		memcpy(tag, resp.tag, sizeof(resp.tag));
642 		return 0;
643 	}
644 
645 	return tegra_bpmp_get_firmware_tag_old(bpmp, tag, size);
646 }
647 
648 static void tegra_bpmp_channel_signal(struct tegra_bpmp_channel *channel)
649 {
650 	unsigned long flags = channel->ob->flags;
651 
652 	if ((flags & MSG_RING) == 0)
653 		return;
654 
655 	complete(&channel->completion);
656 }
657 
658 void tegra_bpmp_handle_rx(struct tegra_bpmp *bpmp)
659 {
660 	struct tegra_bpmp_channel *channel;
661 	unsigned int i, count;
662 	unsigned long *busy;
663 
664 	channel = bpmp->rx_channel;
665 	count = bpmp->soc->channels.thread.count;
666 	busy = bpmp->threaded.busy;
667 
668 	if (tegra_bpmp_is_request_ready(channel))
669 		tegra_bpmp_handle_mrq(bpmp, channel->ib->code, channel);
670 
671 	spin_lock(&bpmp->lock);
672 
673 	for_each_set_bit(i, busy, count) {
674 		struct tegra_bpmp_channel *channel;
675 
676 		channel = &bpmp->threaded_channels[i];
677 
678 		if (tegra_bpmp_is_response_ready(channel)) {
679 			tegra_bpmp_channel_signal(channel);
680 			clear_bit(i, busy);
681 		}
682 	}
683 
684 	spin_unlock(&bpmp->lock);
685 }
686 
687 static int tegra_bpmp_probe(struct platform_device *pdev)
688 {
689 	struct tegra_bpmp *bpmp;
690 	char tag[TAG_SZ];
691 	size_t size;
692 	int err;
693 
694 	bpmp = devm_kzalloc(&pdev->dev, sizeof(*bpmp), GFP_KERNEL);
695 	if (!bpmp)
696 		return -ENOMEM;
697 
698 	bpmp->soc = of_device_get_match_data(&pdev->dev);
699 	bpmp->dev = &pdev->dev;
700 
701 	INIT_LIST_HEAD(&bpmp->mrqs);
702 	spin_lock_init(&bpmp->lock);
703 
704 	bpmp->threaded.count = bpmp->soc->channels.thread.count;
705 	sema_init(&bpmp->threaded.lock, bpmp->threaded.count);
706 
707 	size = BITS_TO_LONGS(bpmp->threaded.count) * sizeof(long);
708 
709 	bpmp->threaded.allocated = devm_kzalloc(&pdev->dev, size, GFP_KERNEL);
710 	if (!bpmp->threaded.allocated)
711 		return -ENOMEM;
712 
713 	bpmp->threaded.busy = devm_kzalloc(&pdev->dev, size, GFP_KERNEL);
714 	if (!bpmp->threaded.busy)
715 		return -ENOMEM;
716 
717 	spin_lock_init(&bpmp->atomic_tx_lock);
718 	bpmp->tx_channel = devm_kzalloc(&pdev->dev, sizeof(*bpmp->tx_channel),
719 					GFP_KERNEL);
720 	if (!bpmp->tx_channel)
721 		return -ENOMEM;
722 
723 	bpmp->rx_channel = devm_kzalloc(&pdev->dev, sizeof(*bpmp->rx_channel),
724 	                                GFP_KERNEL);
725 	if (!bpmp->rx_channel)
726 		return -ENOMEM;
727 
728 	bpmp->threaded_channels = devm_kcalloc(&pdev->dev, bpmp->threaded.count,
729 					       sizeof(*bpmp->threaded_channels),
730 					       GFP_KERNEL);
731 	if (!bpmp->threaded_channels)
732 		return -ENOMEM;
733 
734 	err = bpmp->soc->ops->init(bpmp);
735 	if (err < 0)
736 		return err;
737 
738 	err = tegra_bpmp_request_mrq(bpmp, MRQ_PING,
739 				     tegra_bpmp_mrq_handle_ping, bpmp);
740 	if (err < 0)
741 		goto deinit;
742 
743 	err = tegra_bpmp_ping(bpmp);
744 	if (err < 0) {
745 		dev_err(&pdev->dev, "failed to ping BPMP: %d\n", err);
746 		goto free_mrq;
747 	}
748 
749 	err = tegra_bpmp_get_firmware_tag(bpmp, tag, sizeof(tag));
750 	if (err < 0) {
751 		dev_err(&pdev->dev, "failed to get firmware tag: %d\n", err);
752 		goto free_mrq;
753 	}
754 
755 	dev_info(&pdev->dev, "firmware: %.*s\n", (int)sizeof(tag), tag);
756 
757 	platform_set_drvdata(pdev, bpmp);
758 
759 	err = of_platform_default_populate(pdev->dev.of_node, NULL, &pdev->dev);
760 	if (err < 0)
761 		goto free_mrq;
762 
763 	if (of_find_property(pdev->dev.of_node, "#clock-cells", NULL)) {
764 		err = tegra_bpmp_init_clocks(bpmp);
765 		if (err < 0)
766 			goto free_mrq;
767 	}
768 
769 	if (of_find_property(pdev->dev.of_node, "#reset-cells", NULL)) {
770 		err = tegra_bpmp_init_resets(bpmp);
771 		if (err < 0)
772 			goto free_mrq;
773 	}
774 
775 	if (of_find_property(pdev->dev.of_node, "#power-domain-cells", NULL)) {
776 		err = tegra_bpmp_init_powergates(bpmp);
777 		if (err < 0)
778 			goto free_mrq;
779 	}
780 
781 	err = tegra_bpmp_init_debugfs(bpmp);
782 	if (err < 0)
783 		dev_err(&pdev->dev, "debugfs initialization failed: %d\n", err);
784 
785 	return 0;
786 
787 free_mrq:
788 	tegra_bpmp_free_mrq(bpmp, MRQ_PING, bpmp);
789 deinit:
790 	if (bpmp->soc->ops->deinit)
791 		bpmp->soc->ops->deinit(bpmp);
792 
793 	return err;
794 }
795 
796 static int __maybe_unused tegra_bpmp_resume(struct device *dev)
797 {
798 	struct tegra_bpmp *bpmp = dev_get_drvdata(dev);
799 
800 	if (bpmp->soc->ops->resume)
801 		return bpmp->soc->ops->resume(bpmp);
802 	else
803 		return 0;
804 }
805 
806 static SIMPLE_DEV_PM_OPS(tegra_bpmp_pm_ops, NULL, tegra_bpmp_resume);
807 
808 #if IS_ENABLED(CONFIG_ARCH_TEGRA_186_SOC) || \
809     IS_ENABLED(CONFIG_ARCH_TEGRA_194_SOC)
810 static const struct tegra_bpmp_soc tegra186_soc = {
811 	.channels = {
812 		.cpu_tx = {
813 			.offset = 3,
814 			.timeout = 60 * USEC_PER_SEC,
815 		},
816 		.thread = {
817 			.offset = 0,
818 			.count = 3,
819 			.timeout = 600 * USEC_PER_SEC,
820 		},
821 		.cpu_rx = {
822 			.offset = 13,
823 			.timeout = 0,
824 		},
825 	},
826 	.ops = &tegra186_bpmp_ops,
827 	.num_resets = 193,
828 };
829 #endif
830 
831 #if IS_ENABLED(CONFIG_ARCH_TEGRA_210_SOC)
832 static const struct tegra_bpmp_soc tegra210_soc = {
833 	.channels = {
834 		.cpu_tx = {
835 			.offset = 0,
836 			.count = 1,
837 			.timeout = 60 * USEC_PER_SEC,
838 		},
839 		.thread = {
840 			.offset = 4,
841 			.count = 1,
842 			.timeout = 600 * USEC_PER_SEC,
843 		},
844 		.cpu_rx = {
845 			.offset = 8,
846 			.count = 1,
847 			.timeout = 0,
848 		},
849 	},
850 	.ops = &tegra210_bpmp_ops,
851 };
852 #endif
853 
854 static const struct of_device_id tegra_bpmp_match[] = {
855 #if IS_ENABLED(CONFIG_ARCH_TEGRA_186_SOC) || \
856     IS_ENABLED(CONFIG_ARCH_TEGRA_194_SOC)
857 	{ .compatible = "nvidia,tegra186-bpmp", .data = &tegra186_soc },
858 #endif
859 #if IS_ENABLED(CONFIG_ARCH_TEGRA_210_SOC)
860 	{ .compatible = "nvidia,tegra210-bpmp", .data = &tegra210_soc },
861 #endif
862 	{ }
863 };
864 
865 static struct platform_driver tegra_bpmp_driver = {
866 	.driver = {
867 		.name = "tegra-bpmp",
868 		.of_match_table = tegra_bpmp_match,
869 		.pm = &tegra_bpmp_pm_ops,
870 	},
871 	.probe = tegra_bpmp_probe,
872 };
873 
874 static int __init tegra_bpmp_init(void)
875 {
876 	return platform_driver_register(&tegra_bpmp_driver);
877 }
878 core_initcall(tegra_bpmp_init);
879