1 // SPDX-License-Identifier: GPL-2.0-only
3 * Copyright (c) 2016, NVIDIA CORPORATION. All rights reserved.
6 #include <linux/clk/tegra.h>
7 #include <linux/genalloc.h>
8 #include <linux/mailbox_client.h>
10 #include <linux/of_address.h>
11 #include <linux/of_device.h>
12 #include <linux/platform_device.h>
14 #include <linux/semaphore.h>
15 #include <linux/sched/clock.h>
17 #include <soc/tegra/bpmp.h>
18 #include <soc/tegra/bpmp-abi.h>
19 #include <soc/tegra/ivc.h>
21 #include "bpmp-private.h"
23 #define MSG_ACK BIT(0)
24 #define MSG_RING BIT(1)
27 static inline struct tegra_bpmp *
28 mbox_client_to_bpmp(struct mbox_client *client)
30 return container_of(client, struct tegra_bpmp, mbox.client);
33 static inline const struct tegra_bpmp_ops *
34 channel_to_ops(struct tegra_bpmp_channel *channel)
36 struct tegra_bpmp *bpmp = channel->bpmp;
38 return bpmp->soc->ops;
41 struct tegra_bpmp *tegra_bpmp_get(struct device *dev)
43 struct platform_device *pdev;
44 struct tegra_bpmp *bpmp;
45 struct device_node *np;
47 np = of_parse_phandle(dev->of_node, "nvidia,bpmp", 0);
49 return ERR_PTR(-ENOENT);
51 pdev = of_find_device_by_node(np);
53 bpmp = ERR_PTR(-ENODEV);
57 bpmp = platform_get_drvdata(pdev);
59 bpmp = ERR_PTR(-EPROBE_DEFER);
60 put_device(&pdev->dev);
68 EXPORT_SYMBOL_GPL(tegra_bpmp_get);
70 void tegra_bpmp_put(struct tegra_bpmp *bpmp)
73 put_device(bpmp->dev);
75 EXPORT_SYMBOL_GPL(tegra_bpmp_put);
78 tegra_bpmp_channel_get_thread_index(struct tegra_bpmp_channel *channel)
80 struct tegra_bpmp *bpmp = channel->bpmp;
84 count = bpmp->soc->channels.thread.count;
86 index = channel - channel->bpmp->threaded_channels;
87 if (index < 0 || index >= count)
93 static bool tegra_bpmp_message_valid(const struct tegra_bpmp_message *msg)
95 return (msg->tx.size <= MSG_DATA_MIN_SZ) &&
96 (msg->rx.size <= MSG_DATA_MIN_SZ) &&
97 (msg->tx.size == 0 || msg->tx.data) &&
98 (msg->rx.size == 0 || msg->rx.data);
101 static bool tegra_bpmp_is_response_ready(struct tegra_bpmp_channel *channel)
103 const struct tegra_bpmp_ops *ops = channel_to_ops(channel);
105 return ops->is_response_ready(channel);
108 static bool tegra_bpmp_is_request_ready(struct tegra_bpmp_channel *channel)
110 const struct tegra_bpmp_ops *ops = channel_to_ops(channel);
112 return ops->is_request_ready(channel);
115 static int tegra_bpmp_wait_response(struct tegra_bpmp_channel *channel)
117 unsigned long timeout = channel->bpmp->soc->channels.cpu_tx.timeout;
120 end = ktime_add_us(ktime_get(), timeout);
123 if (tegra_bpmp_is_response_ready(channel))
125 } while (ktime_before(ktime_get(), end));
130 static int tegra_bpmp_ack_response(struct tegra_bpmp_channel *channel)
132 const struct tegra_bpmp_ops *ops = channel_to_ops(channel);
134 return ops->ack_response(channel);
137 static int tegra_bpmp_ack_request(struct tegra_bpmp_channel *channel)
139 const struct tegra_bpmp_ops *ops = channel_to_ops(channel);
141 return ops->ack_request(channel);
145 tegra_bpmp_is_request_channel_free(struct tegra_bpmp_channel *channel)
147 const struct tegra_bpmp_ops *ops = channel_to_ops(channel);
149 return ops->is_request_channel_free(channel);
153 tegra_bpmp_is_response_channel_free(struct tegra_bpmp_channel *channel)
155 const struct tegra_bpmp_ops *ops = channel_to_ops(channel);
157 return ops->is_response_channel_free(channel);
161 tegra_bpmp_wait_request_channel_free(struct tegra_bpmp_channel *channel)
163 unsigned long timeout = channel->bpmp->soc->channels.cpu_tx.timeout;
166 start = ns_to_ktime(local_clock());
169 if (tegra_bpmp_is_request_channel_free(channel))
172 now = ns_to_ktime(local_clock());
173 } while (ktime_us_delta(now, start) < timeout);
178 static int tegra_bpmp_post_request(struct tegra_bpmp_channel *channel)
180 const struct tegra_bpmp_ops *ops = channel_to_ops(channel);
182 return ops->post_request(channel);
185 static int tegra_bpmp_post_response(struct tegra_bpmp_channel *channel)
187 const struct tegra_bpmp_ops *ops = channel_to_ops(channel);
189 return ops->post_response(channel);
192 static int tegra_bpmp_ring_doorbell(struct tegra_bpmp *bpmp)
194 return bpmp->soc->ops->ring_doorbell(bpmp);
197 static ssize_t __tegra_bpmp_channel_read(struct tegra_bpmp_channel *channel,
198 void *data, size_t size, int *ret)
202 if (data && size > 0)
203 memcpy(data, channel->ib->data, size);
205 err = tegra_bpmp_ack_response(channel);
209 *ret = channel->ib->code;
214 static ssize_t tegra_bpmp_channel_read(struct tegra_bpmp_channel *channel,
215 void *data, size_t size, int *ret)
217 struct tegra_bpmp *bpmp = channel->bpmp;
222 index = tegra_bpmp_channel_get_thread_index(channel);
228 spin_lock_irqsave(&bpmp->lock, flags);
229 err = __tegra_bpmp_channel_read(channel, data, size, ret);
230 clear_bit(index, bpmp->threaded.allocated);
231 spin_unlock_irqrestore(&bpmp->lock, flags);
234 up(&bpmp->threaded.lock);
239 static ssize_t __tegra_bpmp_channel_write(struct tegra_bpmp_channel *channel,
240 unsigned int mrq, unsigned long flags,
241 const void *data, size_t size)
243 channel->ob->code = mrq;
244 channel->ob->flags = flags;
246 if (data && size > 0)
247 memcpy(channel->ob->data, data, size);
249 return tegra_bpmp_post_request(channel);
252 static struct tegra_bpmp_channel *
253 tegra_bpmp_write_threaded(struct tegra_bpmp *bpmp, unsigned int mrq,
254 const void *data, size_t size)
256 unsigned long timeout = bpmp->soc->channels.thread.timeout;
257 unsigned int count = bpmp->soc->channels.thread.count;
258 struct tegra_bpmp_channel *channel;
263 err = down_timeout(&bpmp->threaded.lock, usecs_to_jiffies(timeout));
267 spin_lock_irqsave(&bpmp->lock, flags);
269 index = find_first_zero_bit(bpmp->threaded.allocated, count);
270 if (index == count) {
275 channel = &bpmp->threaded_channels[index];
277 if (!tegra_bpmp_is_request_channel_free(channel)) {
282 set_bit(index, bpmp->threaded.allocated);
284 err = __tegra_bpmp_channel_write(channel, mrq, MSG_ACK | MSG_RING,
287 goto clear_allocated;
289 set_bit(index, bpmp->threaded.busy);
291 spin_unlock_irqrestore(&bpmp->lock, flags);
295 clear_bit(index, bpmp->threaded.allocated);
297 spin_unlock_irqrestore(&bpmp->lock, flags);
298 up(&bpmp->threaded.lock);
303 static ssize_t tegra_bpmp_channel_write(struct tegra_bpmp_channel *channel,
304 unsigned int mrq, unsigned long flags,
305 const void *data, size_t size)
309 err = tegra_bpmp_wait_request_channel_free(channel);
313 return __tegra_bpmp_channel_write(channel, mrq, flags, data, size);
316 int tegra_bpmp_transfer_atomic(struct tegra_bpmp *bpmp,
317 struct tegra_bpmp_message *msg)
319 struct tegra_bpmp_channel *channel;
322 if (WARN_ON(!irqs_disabled()))
325 if (!tegra_bpmp_message_valid(msg))
328 channel = bpmp->tx_channel;
330 spin_lock(&bpmp->atomic_tx_lock);
332 err = tegra_bpmp_channel_write(channel, msg->mrq, MSG_ACK,
333 msg->tx.data, msg->tx.size);
335 spin_unlock(&bpmp->atomic_tx_lock);
339 spin_unlock(&bpmp->atomic_tx_lock);
341 err = tegra_bpmp_ring_doorbell(bpmp);
345 err = tegra_bpmp_wait_response(channel);
349 return __tegra_bpmp_channel_read(channel, msg->rx.data, msg->rx.size,
352 EXPORT_SYMBOL_GPL(tegra_bpmp_transfer_atomic);
354 int tegra_bpmp_transfer(struct tegra_bpmp *bpmp,
355 struct tegra_bpmp_message *msg)
357 struct tegra_bpmp_channel *channel;
358 unsigned long timeout;
361 if (WARN_ON(irqs_disabled()))
364 if (!tegra_bpmp_message_valid(msg))
367 channel = tegra_bpmp_write_threaded(bpmp, msg->mrq, msg->tx.data,
370 return PTR_ERR(channel);
372 err = tegra_bpmp_ring_doorbell(bpmp);
376 timeout = usecs_to_jiffies(bpmp->soc->channels.thread.timeout);
378 err = wait_for_completion_timeout(&channel->completion, timeout);
382 return tegra_bpmp_channel_read(channel, msg->rx.data, msg->rx.size,
385 EXPORT_SYMBOL_GPL(tegra_bpmp_transfer);
387 static struct tegra_bpmp_mrq *tegra_bpmp_find_mrq(struct tegra_bpmp *bpmp,
390 struct tegra_bpmp_mrq *entry;
392 list_for_each_entry(entry, &bpmp->mrqs, list)
393 if (entry->mrq == mrq)
399 void tegra_bpmp_mrq_return(struct tegra_bpmp_channel *channel, int code,
400 const void *data, size_t size)
402 unsigned long flags = channel->ib->flags;
403 struct tegra_bpmp *bpmp = channel->bpmp;
406 if (WARN_ON(size > MSG_DATA_MIN_SZ))
409 err = tegra_bpmp_ack_request(channel);
410 if (WARN_ON(err < 0))
413 if ((flags & MSG_ACK) == 0)
416 if (WARN_ON(!tegra_bpmp_is_response_channel_free(channel)))
419 channel->ob->code = code;
421 if (data && size > 0)
422 memcpy(channel->ob->data, data, size);
424 err = tegra_bpmp_post_response(channel);
425 if (WARN_ON(err < 0))
428 if (flags & MSG_RING) {
429 err = tegra_bpmp_ring_doorbell(bpmp);
430 if (WARN_ON(err < 0))
434 EXPORT_SYMBOL_GPL(tegra_bpmp_mrq_return);
436 static void tegra_bpmp_handle_mrq(struct tegra_bpmp *bpmp,
438 struct tegra_bpmp_channel *channel)
440 struct tegra_bpmp_mrq *entry;
443 spin_lock(&bpmp->lock);
445 entry = tegra_bpmp_find_mrq(bpmp, mrq);
447 spin_unlock(&bpmp->lock);
448 tegra_bpmp_mrq_return(channel, -EINVAL, &zero, sizeof(zero));
452 entry->handler(mrq, channel, entry->data);
454 spin_unlock(&bpmp->lock);
457 int tegra_bpmp_request_mrq(struct tegra_bpmp *bpmp, unsigned int mrq,
458 tegra_bpmp_mrq_handler_t handler, void *data)
460 struct tegra_bpmp_mrq *entry;
466 entry = devm_kzalloc(bpmp->dev, sizeof(*entry), GFP_KERNEL);
470 spin_lock_irqsave(&bpmp->lock, flags);
473 entry->handler = handler;
475 list_add(&entry->list, &bpmp->mrqs);
477 spin_unlock_irqrestore(&bpmp->lock, flags);
481 EXPORT_SYMBOL_GPL(tegra_bpmp_request_mrq);
483 void tegra_bpmp_free_mrq(struct tegra_bpmp *bpmp, unsigned int mrq, void *data)
485 struct tegra_bpmp_mrq *entry;
488 spin_lock_irqsave(&bpmp->lock, flags);
490 entry = tegra_bpmp_find_mrq(bpmp, mrq);
494 list_del(&entry->list);
495 devm_kfree(bpmp->dev, entry);
498 spin_unlock_irqrestore(&bpmp->lock, flags);
500 EXPORT_SYMBOL_GPL(tegra_bpmp_free_mrq);
502 bool tegra_bpmp_mrq_is_supported(struct tegra_bpmp *bpmp, unsigned int mrq)
504 struct mrq_query_abi_request req = { .mrq = cpu_to_le32(mrq) };
505 struct mrq_query_abi_response resp;
506 struct tegra_bpmp_message msg = {
507 .mrq = MRQ_QUERY_ABI,
514 .size = sizeof(resp),
519 ret = tegra_bpmp_transfer(bpmp, &msg);
520 if (ret || msg.rx.ret)
523 return resp.status == 0;
525 EXPORT_SYMBOL_GPL(tegra_bpmp_mrq_is_supported);
527 static void tegra_bpmp_mrq_handle_ping(unsigned int mrq,
528 struct tegra_bpmp_channel *channel,
531 struct mrq_ping_request *request;
532 struct mrq_ping_response response;
534 request = (struct mrq_ping_request *)channel->ib->data;
536 memset(&response, 0, sizeof(response));
537 response.reply = request->challenge << 1;
539 tegra_bpmp_mrq_return(channel, 0, &response, sizeof(response));
542 static int tegra_bpmp_ping(struct tegra_bpmp *bpmp)
544 struct mrq_ping_response response;
545 struct mrq_ping_request request;
546 struct tegra_bpmp_message msg;
551 memset(&request, 0, sizeof(request));
552 request.challenge = 1;
554 memset(&response, 0, sizeof(response));
556 memset(&msg, 0, sizeof(msg));
558 msg.tx.data = &request;
559 msg.tx.size = sizeof(request);
560 msg.rx.data = &response;
561 msg.rx.size = sizeof(response);
563 local_irq_save(flags);
565 err = tegra_bpmp_transfer_atomic(bpmp, &msg);
567 local_irq_restore(flags);
571 "ping ok: challenge: %u, response: %u, time: %lld\n",
572 request.challenge, response.reply,
573 ktime_to_us(ktime_sub(end, start)));
578 /* deprecated version of tag query */
579 static int tegra_bpmp_get_firmware_tag_old(struct tegra_bpmp *bpmp, char *tag,
582 struct mrq_query_tag_request request;
583 struct tegra_bpmp_message msg;
592 virt = dma_alloc_coherent(bpmp->dev, TAG_SZ, &phys,
593 GFP_KERNEL | GFP_DMA32);
597 memset(&request, 0, sizeof(request));
600 memset(&msg, 0, sizeof(msg));
601 msg.mrq = MRQ_QUERY_TAG;
602 msg.tx.data = &request;
603 msg.tx.size = sizeof(request);
605 local_irq_save(flags);
606 err = tegra_bpmp_transfer_atomic(bpmp, &msg);
607 local_irq_restore(flags);
610 memcpy(tag, virt, TAG_SZ);
612 dma_free_coherent(bpmp->dev, TAG_SZ, virt, phys);
617 static int tegra_bpmp_get_firmware_tag(struct tegra_bpmp *bpmp, char *tag,
620 if (tegra_bpmp_mrq_is_supported(bpmp, MRQ_QUERY_FW_TAG)) {
621 struct mrq_query_fw_tag_response resp;
622 struct tegra_bpmp_message msg = {
623 .mrq = MRQ_QUERY_FW_TAG,
626 .size = sizeof(resp),
631 if (size != sizeof(resp.tag))
634 err = tegra_bpmp_transfer(bpmp, &msg);
641 memcpy(tag, resp.tag, sizeof(resp.tag));
645 return tegra_bpmp_get_firmware_tag_old(bpmp, tag, size);
648 static void tegra_bpmp_channel_signal(struct tegra_bpmp_channel *channel)
650 unsigned long flags = channel->ob->flags;
652 if ((flags & MSG_RING) == 0)
655 complete(&channel->completion);
658 void tegra_bpmp_handle_rx(struct tegra_bpmp *bpmp)
660 struct tegra_bpmp_channel *channel;
661 unsigned int i, count;
664 channel = bpmp->rx_channel;
665 count = bpmp->soc->channels.thread.count;
666 busy = bpmp->threaded.busy;
668 if (tegra_bpmp_is_request_ready(channel))
669 tegra_bpmp_handle_mrq(bpmp, channel->ib->code, channel);
671 spin_lock(&bpmp->lock);
673 for_each_set_bit(i, busy, count) {
674 struct tegra_bpmp_channel *channel;
676 channel = &bpmp->threaded_channels[i];
678 if (tegra_bpmp_is_response_ready(channel)) {
679 tegra_bpmp_channel_signal(channel);
684 spin_unlock(&bpmp->lock);
687 static int tegra_bpmp_probe(struct platform_device *pdev)
689 struct tegra_bpmp *bpmp;
694 bpmp = devm_kzalloc(&pdev->dev, sizeof(*bpmp), GFP_KERNEL);
698 bpmp->soc = of_device_get_match_data(&pdev->dev);
699 bpmp->dev = &pdev->dev;
701 INIT_LIST_HEAD(&bpmp->mrqs);
702 spin_lock_init(&bpmp->lock);
704 bpmp->threaded.count = bpmp->soc->channels.thread.count;
705 sema_init(&bpmp->threaded.lock, bpmp->threaded.count);
707 size = BITS_TO_LONGS(bpmp->threaded.count) * sizeof(long);
709 bpmp->threaded.allocated = devm_kzalloc(&pdev->dev, size, GFP_KERNEL);
710 if (!bpmp->threaded.allocated)
713 bpmp->threaded.busy = devm_kzalloc(&pdev->dev, size, GFP_KERNEL);
714 if (!bpmp->threaded.busy)
717 spin_lock_init(&bpmp->atomic_tx_lock);
718 bpmp->tx_channel = devm_kzalloc(&pdev->dev, sizeof(*bpmp->tx_channel),
720 if (!bpmp->tx_channel)
723 bpmp->rx_channel = devm_kzalloc(&pdev->dev, sizeof(*bpmp->rx_channel),
725 if (!bpmp->rx_channel)
728 bpmp->threaded_channels = devm_kcalloc(&pdev->dev, bpmp->threaded.count,
729 sizeof(*bpmp->threaded_channels),
731 if (!bpmp->threaded_channels)
734 err = bpmp->soc->ops->init(bpmp);
738 err = tegra_bpmp_request_mrq(bpmp, MRQ_PING,
739 tegra_bpmp_mrq_handle_ping, bpmp);
743 err = tegra_bpmp_ping(bpmp);
745 dev_err(&pdev->dev, "failed to ping BPMP: %d\n", err);
749 err = tegra_bpmp_get_firmware_tag(bpmp, tag, sizeof(tag));
751 dev_err(&pdev->dev, "failed to get firmware tag: %d\n", err);
755 dev_info(&pdev->dev, "firmware: %.*s\n", (int)sizeof(tag), tag);
757 platform_set_drvdata(pdev, bpmp);
759 err = of_platform_default_populate(pdev->dev.of_node, NULL, &pdev->dev);
763 if (of_find_property(pdev->dev.of_node, "#clock-cells", NULL)) {
764 err = tegra_bpmp_init_clocks(bpmp);
769 if (of_find_property(pdev->dev.of_node, "#reset-cells", NULL)) {
770 err = tegra_bpmp_init_resets(bpmp);
775 if (of_find_property(pdev->dev.of_node, "#power-domain-cells", NULL)) {
776 err = tegra_bpmp_init_powergates(bpmp);
781 err = tegra_bpmp_init_debugfs(bpmp);
783 dev_err(&pdev->dev, "debugfs initialization failed: %d\n", err);
788 tegra_bpmp_free_mrq(bpmp, MRQ_PING, bpmp);
790 if (bpmp->soc->ops->deinit)
791 bpmp->soc->ops->deinit(bpmp);
796 static int __maybe_unused tegra_bpmp_resume(struct device *dev)
798 struct tegra_bpmp *bpmp = dev_get_drvdata(dev);
800 if (bpmp->soc->ops->resume)
801 return bpmp->soc->ops->resume(bpmp);
806 static const struct dev_pm_ops tegra_bpmp_pm_ops = {
807 .resume_early = tegra_bpmp_resume,
810 #if IS_ENABLED(CONFIG_ARCH_TEGRA_186_SOC) || \
811 IS_ENABLED(CONFIG_ARCH_TEGRA_194_SOC)
812 static const struct tegra_bpmp_soc tegra186_soc = {
816 .timeout = 60 * USEC_PER_SEC,
821 .timeout = 600 * USEC_PER_SEC,
828 .ops = &tegra186_bpmp_ops,
833 #if IS_ENABLED(CONFIG_ARCH_TEGRA_210_SOC)
834 static const struct tegra_bpmp_soc tegra210_soc = {
839 .timeout = 60 * USEC_PER_SEC,
844 .timeout = 600 * USEC_PER_SEC,
852 .ops = &tegra210_bpmp_ops,
856 static const struct of_device_id tegra_bpmp_match[] = {
857 #if IS_ENABLED(CONFIG_ARCH_TEGRA_186_SOC) || \
858 IS_ENABLED(CONFIG_ARCH_TEGRA_194_SOC)
859 { .compatible = "nvidia,tegra186-bpmp", .data = &tegra186_soc },
861 #if IS_ENABLED(CONFIG_ARCH_TEGRA_210_SOC)
862 { .compatible = "nvidia,tegra210-bpmp", .data = &tegra210_soc },
867 static struct platform_driver tegra_bpmp_driver = {
869 .name = "tegra-bpmp",
870 .of_match_table = tegra_bpmp_match,
871 .pm = &tegra_bpmp_pm_ops,
873 .probe = tegra_bpmp_probe,
876 static int __init tegra_bpmp_init(void)
878 return platform_driver_register(&tegra_bpmp_driver);
880 core_initcall(tegra_bpmp_init);