drivers/firmware/tegra/bpmp.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright (c) 2016, NVIDIA CORPORATION.  All rights reserved.
   4  */
   5
   6 #include <linux/clk/tegra.h>
   7 #include <linux/genalloc.h>
   8 #include <linux/mailbox_client.h>
   9 #include <linux/of.h>
  10 #include <linux/of_address.h>
  11 #include <linux/of_device.h>
  12 #include <linux/platform_device.h>
  13 #include <linux/pm.h>
  14 #include <linux/semaphore.h>
  15 #include <linux/sched/clock.h>
  16
  17 #include <soc/tegra/bpmp.h>
  18 #include <soc/tegra/bpmp-abi.h>
  19 #include <soc/tegra/ivc.h>
  20
  21 #include "bpmp-private.h"
  22
  23 #define MSG_ACK         BIT(0)
  24 #define MSG_RING        BIT(1)
  25 #define TAG_SZ          32
  26
  27 static inline struct tegra_bpmp *
  28 mbox_client_to_bpmp(struct mbox_client *client)
  29 {
  30         return container_of(client, struct tegra_bpmp, mbox.client);
  31 }
  32
  33 static inline const struct tegra_bpmp_ops *
  34 channel_to_ops(struct tegra_bpmp_channel *channel)
  35 {
  36         struct tegra_bpmp *bpmp = channel->bpmp;
  37
  38         return bpmp->soc->ops;
  39 }
  40
  41 struct tegra_bpmp *tegra_bpmp_get(struct device *dev)
  42 {
  43         struct platform_device *pdev;
  44         struct tegra_bpmp *bpmp;
  45         struct device_node *np;
  46
  47         np = of_parse_phandle(dev->of_node, "nvidia,bpmp", 0);
  48         if (!np)
  49                 return ERR_PTR(-ENOENT);
  50
  51         pdev = of_find_device_by_node(np);
  52         if (!pdev) {
  53                 bpmp = ERR_PTR(-ENODEV);
  54                 goto put;
  55         }
  56
  57         bpmp = platform_get_drvdata(pdev);
  58         if (!bpmp) {
  59                 bpmp = ERR_PTR(-EPROBE_DEFER);
  60                 put_device(&pdev->dev);
  61                 goto put;
  62         }
  63
  64 put:
  65         of_node_put(np);
  66         return bpmp;
  67 }
  68 EXPORT_SYMBOL_GPL(tegra_bpmp_get);
  69
  70 void tegra_bpmp_put(struct tegra_bpmp *bpmp)
  71 {
  72         if (bpmp)
  73                 put_device(bpmp->dev);
  74 }
  75 EXPORT_SYMBOL_GPL(tegra_bpmp_put);
  76
  77 static int
  78 tegra_bpmp_channel_get_thread_index(struct tegra_bpmp_channel *channel)
  79 {
  80         struct tegra_bpmp *bpmp = channel->bpmp;
  81         unsigned int count;
  82         int index;
  83
  84         count = bpmp->soc->channels.thread.count;
  85
  86         index = channel - channel->bpmp->threaded_channels;
  87         if (index < 0 || index >= count)
  88                 return -EINVAL;
  89
  90         return index;
  91 }
  92
  93 static bool tegra_bpmp_message_valid(const struct tegra_bpmp_message *msg)
  94 {
  95         return (msg->tx.size <= MSG_DATA_MIN_SZ) &&
  96                (msg->rx.size <= MSG_DATA_MIN_SZ) &&
  97                (msg->tx.size == 0 || msg->tx.data) &&
  98                (msg->rx.size == 0 || msg->rx.data);
  99 }
 100
 101 static bool tegra_bpmp_is_response_ready(struct tegra_bpmp_channel *channel)
 102 {
 103         const struct tegra_bpmp_ops *ops = channel_to_ops(channel);
 104
 105         return ops->is_response_ready(channel);
 106 }
 107
 108 static bool tegra_bpmp_is_request_ready(struct tegra_bpmp_channel *channel)
 109 {
 110         const struct tegra_bpmp_ops *ops = channel_to_ops(channel);
 111
 112         return ops->is_request_ready(channel);
 113 }
 114
 115 static int tegra_bpmp_wait_response(struct tegra_bpmp_channel *channel)
 116 {
 117         unsigned long timeout = channel->bpmp->soc->channels.cpu_tx.timeout;
 118         ktime_t end;
 119
 120         end = ktime_add_us(ktime_get(), timeout);
 121
 122         do {
 123                 if (tegra_bpmp_is_response_ready(channel))
 124                         return 0;
 125         } while (ktime_before(ktime_get(), end));
 126
 127         return -ETIMEDOUT;
 128 }
 129
 130 static int tegra_bpmp_ack_response(struct tegra_bpmp_channel *channel)
 131 {
 132         const struct tegra_bpmp_ops *ops = channel_to_ops(channel);
 133
 134         return ops->ack_response(channel);
 135 }
 136
 137 static int tegra_bpmp_ack_request(struct tegra_bpmp_channel *channel)
 138 {
 139         const struct tegra_bpmp_ops *ops = channel_to_ops(channel);
 140
 141         return ops->ack_request(channel);
 142 }
 143
 144 static bool
 145 tegra_bpmp_is_request_channel_free(struct tegra_bpmp_channel *channel)
 146 {
 147         const struct tegra_bpmp_ops *ops = channel_to_ops(channel);
 148
 149         return ops->is_request_channel_free(channel);
 150 }
 151
 152 static bool
 153 tegra_bpmp_is_response_channel_free(struct tegra_bpmp_channel *channel)
 154 {
 155         const struct tegra_bpmp_ops *ops = channel_to_ops(channel);
 156
 157         return ops->is_response_channel_free(channel);
 158 }
 159
 160 static int
 161 tegra_bpmp_wait_request_channel_free(struct tegra_bpmp_channel *channel)
 162 {
 163         unsigned long timeout = channel->bpmp->soc->channels.cpu_tx.timeout;
 164         ktime_t start, now;
 165
 166         start = ns_to_ktime(local_clock());
 167
 168         do {
 169                 if (tegra_bpmp_is_request_channel_free(channel))
 170                         return 0;
 171
 172                 now = ns_to_ktime(local_clock());
 173         } while (ktime_us_delta(now, start) < timeout);
 174
 175         return -ETIMEDOUT;
 176 }
 177
 178 static int tegra_bpmp_post_request(struct tegra_bpmp_channel *channel)
 179 {
 180         const struct tegra_bpmp_ops *ops = channel_to_ops(channel);
 181
 182         return ops->post_request(channel);
 183 }
 184
 185 static int tegra_bpmp_post_response(struct tegra_bpmp_channel *channel)
 186 {
 187         const struct tegra_bpmp_ops *ops = channel_to_ops(channel);
 188
 189         return ops->post_response(channel);
 190 }
 191
 192 static int tegra_bpmp_ring_doorbell(struct tegra_bpmp *bpmp)
 193 {
 194         return bpmp->soc->ops->ring_doorbell(bpmp);
 195 }
 196
 197 static ssize_t __tegra_bpmp_channel_read(struct tegra_bpmp_channel *channel,
 198                                          void *data, size_t size, int *ret)
 199 {
 200         int err;
 201
 202         if (data && size > 0)
 203                 memcpy(data, channel->ib->data, size);
 204
 205         err = tegra_bpmp_ack_response(channel);
 206         if (err < 0)
 207                 return err;
 208
 209         *ret = channel->ib->code;
 210
 211         return 0;
 212 }
 213
 214 static ssize_t tegra_bpmp_channel_read(struct tegra_bpmp_channel *channel,
 215                                        void *data, size_t size, int *ret)
 216 {
 217         struct tegra_bpmp *bpmp = channel->bpmp;
 218         unsigned long flags;
 219         ssize_t err;
 220         int index;
 221
 222         index = tegra_bpmp_channel_get_thread_index(channel);
 223         if (index < 0) {
 224                 err = index;
 225                 goto unlock;
 226         }
 227
 228         spin_lock_irqsave(&bpmp->lock, flags);
 229         err = __tegra_bpmp_channel_read(channel, data, size, ret);
 230         clear_bit(index, bpmp->threaded.allocated);
 231         spin_unlock_irqrestore(&bpmp->lock, flags);
 232
 233 unlock:
 234         up(&bpmp->threaded.lock);
 235
 236         return err;
 237 }
 238
 239 static ssize_t __tegra_bpmp_channel_write(struct tegra_bpmp_channel *channel,
 240                                           unsigned int mrq, unsigned long flags,
 241                                           const void *data, size_t size)
 242 {
 243         channel->ob->code = mrq;
 244         channel->ob->flags = flags;
 245
 246         if (data && size > 0)
 247                 memcpy(channel->ob->data, data, size);
 248
 249         return tegra_bpmp_post_request(channel);
 250 }
 251
 252 static struct tegra_bpmp_channel *
 253 tegra_bpmp_write_threaded(struct tegra_bpmp *bpmp, unsigned int mrq,
 254                           const void *data, size_t size)
 255 {
 256         unsigned long timeout = bpmp->soc->channels.thread.timeout;
 257         unsigned int count = bpmp->soc->channels.thread.count;
 258         struct tegra_bpmp_channel *channel;
 259         unsigned long flags;
 260         unsigned int index;
 261         int err;
 262
 263         err = down_timeout(&bpmp->threaded.lock, usecs_to_jiffies(timeout));
 264         if (err < 0)
 265                 return ERR_PTR(err);
 266
 267         spin_lock_irqsave(&bpmp->lock, flags);
 268
 269         index = find_first_zero_bit(bpmp->threaded.allocated, count);
 270         if (index == count) {
 271                 err = -EBUSY;
 272                 goto unlock;
 273         }
 274
 275         channel = &bpmp->threaded_channels[index];
 276
 277         if (!tegra_bpmp_is_request_channel_free(channel)) {
 278                 err = -EBUSY;
 279                 goto unlock;
 280         }
 281
 282         set_bit(index, bpmp->threaded.allocated);
 283
 284         err = __tegra_bpmp_channel_write(channel, mrq, MSG_ACK | MSG_RING,
 285                                          data, size);
 286         if (err < 0)
 287                 goto clear_allocated;
 288
 289         set_bit(index, bpmp->threaded.busy);
 290
 291         spin_unlock_irqrestore(&bpmp->lock, flags);
 292         return channel;
 293
 294 clear_allocated:
 295         clear_bit(index, bpmp->threaded.allocated);
 296 unlock:
 297         spin_unlock_irqrestore(&bpmp->lock, flags);
 298         up(&bpmp->threaded.lock);
 299
 300         return ERR_PTR(err);
 301 }
 302
 303 static ssize_t tegra_bpmp_channel_write(struct tegra_bpmp_channel *channel,
 304                                         unsigned int mrq, unsigned long flags,
 305                                         const void *data, size_t size)
 306 {
 307         int err;
 308
 309         err = tegra_bpmp_wait_request_channel_free(channel);
 310         if (err < 0)
 311                 return err;
 312
 313         return __tegra_bpmp_channel_write(channel, mrq, flags, data, size);
 314 }
 315
 316 int tegra_bpmp_transfer_atomic(struct tegra_bpmp *bpmp,
 317                                struct tegra_bpmp_message *msg)
 318 {
 319         struct tegra_bpmp_channel *channel;
 320         int err;
 321
 322         if (WARN_ON(!irqs_disabled()))
 323                 return -EPERM;
 324
 325         if (!tegra_bpmp_message_valid(msg))
 326                 return -EINVAL;
 327
 328         channel = bpmp->tx_channel;
 329
 330         spin_lock(&bpmp->atomic_tx_lock);
 331
 332         err = tegra_bpmp_channel_write(channel, msg->mrq, MSG_ACK,
 333                                        msg->tx.data, msg->tx.size);
 334         if (err < 0) {
 335                 spin_unlock(&bpmp->atomic_tx_lock);
 336                 return err;
 337         }
 338
 339         spin_unlock(&bpmp->atomic_tx_lock);
 340
 341         err = tegra_bpmp_ring_doorbell(bpmp);
 342         if (err < 0)
 343                 return err;
 344
 345         err = tegra_bpmp_wait_response(channel);
 346         if (err < 0)
 347                 return err;
 348
 349         return __tegra_bpmp_channel_read(channel, msg->rx.data, msg->rx.size,
 350                                          &msg->rx.ret);
 351 }
 352 EXPORT_SYMBOL_GPL(tegra_bpmp_transfer_atomic);
 353
 354 int tegra_bpmp_transfer(struct tegra_bpmp *bpmp,
 355                         struct tegra_bpmp_message *msg)
 356 {
 357         struct tegra_bpmp_channel *channel;
 358         unsigned long timeout;
 359         int err;
 360
 361         if (WARN_ON(irqs_disabled()))
 362                 return -EPERM;
 363
 364         if (!tegra_bpmp_message_valid(msg))
 365                 return -EINVAL;
 366
 367         channel = tegra_bpmp_write_threaded(bpmp, msg->mrq, msg->tx.data,
 368                                             msg->tx.size);
 369         if (IS_ERR(channel))
 370                 return PTR_ERR(channel);
 371
 372         err = tegra_bpmp_ring_doorbell(bpmp);
 373         if (err < 0)
 374                 return err;
 375
 376         timeout = usecs_to_jiffies(bpmp->soc->channels.thread.timeout);
 377
 378         err = wait_for_completion_timeout(&channel->completion, timeout);
 379         if (err == 0)
 380                 return -ETIMEDOUT;
 381
 382         return tegra_bpmp_channel_read(channel, msg->rx.data, msg->rx.size,
 383                                        &msg->rx.ret);
 384 }
 385 EXPORT_SYMBOL_GPL(tegra_bpmp_transfer);
 386
 387 static struct tegra_bpmp_mrq *tegra_bpmp_find_mrq(struct tegra_bpmp *bpmp,
 388                                                   unsigned int mrq)
 389 {
 390         struct tegra_bpmp_mrq *entry;
 391
 392         list_for_each_entry(entry, &bpmp->mrqs, list)
 393                 if (entry->mrq == mrq)
 394                         return entry;
 395
 396         return NULL;
 397 }
 398
 399 void tegra_bpmp_mrq_return(struct tegra_bpmp_channel *channel, int code,
 400                            const void *data, size_t size)
 401 {
 402         unsigned long flags = channel->ib->flags;
 403         struct tegra_bpmp *bpmp = channel->bpmp;
 404         int err;
 405
 406         if (WARN_ON(size > MSG_DATA_MIN_SZ))
 407                 return;
 408
 409         err = tegra_bpmp_ack_request(channel);
 410         if (WARN_ON(err < 0))
 411                 return;
 412
 413         if ((flags & MSG_ACK) == 0)
 414                 return;
 415
 416         if (WARN_ON(!tegra_bpmp_is_response_channel_free(channel)))
 417                 return;
 418
 419         channel->ob->code = code;
 420
 421         if (data && size > 0)
 422                 memcpy(channel->ob->data, data, size);
 423
 424         err = tegra_bpmp_post_response(channel);
 425         if (WARN_ON(err < 0))
 426                 return;
 427
 428         if (flags & MSG_RING) {
 429                 err = tegra_bpmp_ring_doorbell(bpmp);
 430                 if (WARN_ON(err < 0))
 431                         return;
 432         }
 433 }
 434 EXPORT_SYMBOL_GPL(tegra_bpmp_mrq_return);
 435
 436 static void tegra_bpmp_handle_mrq(struct tegra_bpmp *bpmp,
 437                                   unsigned int mrq,
 438                                   struct tegra_bpmp_channel *channel)
 439 {
 440         struct tegra_bpmp_mrq *entry;
 441         u32 zero = 0;
 442
 443         spin_lock(&bpmp->lock);
 444
 445         entry = tegra_bpmp_find_mrq(bpmp, mrq);
 446         if (!entry) {
 447                 spin_unlock(&bpmp->lock);
 448                 tegra_bpmp_mrq_return(channel, -EINVAL, &zero, sizeof(zero));
 449                 return;
 450         }
 451
 452         entry->handler(mrq, channel, entry->data);
 453
 454         spin_unlock(&bpmp->lock);
 455 }
 456
 457 int tegra_bpmp_request_mrq(struct tegra_bpmp *bpmp, unsigned int mrq,
 458                            tegra_bpmp_mrq_handler_t handler, void *data)
 459 {
 460         struct tegra_bpmp_mrq *entry;
 461         unsigned long flags;
 462
 463         if (!handler)
 464                 return -EINVAL;
 465
 466         entry = devm_kzalloc(bpmp->dev, sizeof(*entry), GFP_KERNEL);
 467         if (!entry)
 468                 return -ENOMEM;
 469
 470         spin_lock_irqsave(&bpmp->lock, flags);
 471
 472         entry->mrq = mrq;
 473         entry->handler = handler;
 474         entry->data = data;
 475         list_add(&entry->list, &bpmp->mrqs);
 476
 477         spin_unlock_irqrestore(&bpmp->lock, flags);
 478
 479         return 0;
 480 }
 481 EXPORT_SYMBOL_GPL(tegra_bpmp_request_mrq);
 482
 483 void tegra_bpmp_free_mrq(struct tegra_bpmp *bpmp, unsigned int mrq, void *data)
 484 {
 485         struct tegra_bpmp_mrq *entry;
 486         unsigned long flags;
 487
 488         spin_lock_irqsave(&bpmp->lock, flags);
 489
 490         entry = tegra_bpmp_find_mrq(bpmp, mrq);
 491         if (!entry)
 492                 goto unlock;
 493
 494         list_del(&entry->list);
 495         devm_kfree(bpmp->dev, entry);
 496
 497 unlock:
 498         spin_unlock_irqrestore(&bpmp->lock, flags);
 499 }
 500 EXPORT_SYMBOL_GPL(tegra_bpmp_free_mrq);
 501
 502 bool tegra_bpmp_mrq_is_supported(struct tegra_bpmp *bpmp, unsigned int mrq)
 503 {
 504         struct mrq_query_abi_request req = { .mrq = cpu_to_le32(mrq) };
 505         struct mrq_query_abi_response resp;
 506         struct tegra_bpmp_message msg = {
 507                 .mrq = MRQ_QUERY_ABI,
 508                 .tx = {
 509                         .data = &req,
 510                         .size = sizeof(req),
 511                 },
 512                 .rx = {
 513                         .data = &resp,
 514                         .size = sizeof(resp),
 515                 },
 516         };
 517         int ret;
 518
 519         ret = tegra_bpmp_transfer(bpmp, &msg);
 520         if (ret || msg.rx.ret)
 521                 return false;
 522
 523         return resp.status == 0;
 524 }
 525 EXPORT_SYMBOL_GPL(tegra_bpmp_mrq_is_supported);
 526
 527 static void tegra_bpmp_mrq_handle_ping(unsigned int mrq,
 528                                        struct tegra_bpmp_channel *channel,
 529                                        void *data)
 530 {
 531         struct mrq_ping_request *request;
 532         struct mrq_ping_response response;
 533
 534         request = (struct mrq_ping_request *)channel->ib->data;
 535
 536         memset(&response, 0, sizeof(response));
 537         response.reply = request->challenge << 1;
 538
 539         tegra_bpmp_mrq_return(channel, 0, &response, sizeof(response));
 540 }
 541
 542 static int tegra_bpmp_ping(struct tegra_bpmp *bpmp)
 543 {
 544         struct mrq_ping_response response;
 545         struct mrq_ping_request request;
 546         struct tegra_bpmp_message msg;
 547         unsigned long flags;
 548         ktime_t start, end;
 549         int err;
 550
 551         memset(&request, 0, sizeof(request));
 552         request.challenge = 1;
 553
 554         memset(&response, 0, sizeof(response));
 555
 556         memset(&msg, 0, sizeof(msg));
 557         msg.mrq = MRQ_PING;
 558         msg.tx.data = &request;
 559         msg.tx.size = sizeof(request);
 560         msg.rx.data = &response;
 561         msg.rx.size = sizeof(response);
 562
 563         local_irq_save(flags);
 564         start = ktime_get();
 565         err = tegra_bpmp_transfer_atomic(bpmp, &msg);
 566         end = ktime_get();
 567         local_irq_restore(flags);
 568
 569         if (!err)
 570                 dev_dbg(bpmp->dev,
 571                         "ping ok: challenge: %u, response: %u, time: %lld\n",
 572                         request.challenge, response.reply,
 573                         ktime_to_us(ktime_sub(end, start)));
 574
 575         return err;
 576 }
 577
 578 /* deprecated version of tag query */
 579 static int tegra_bpmp_get_firmware_tag_old(struct tegra_bpmp *bpmp, char *tag,
 580                                            size_t size)
 581 {
 582         struct mrq_query_tag_request request;
 583         struct tegra_bpmp_message msg;
 584         unsigned long flags;
 585         dma_addr_t phys;
 586         void *virt;
 587         int err;
 588
 589         if (size != TAG_SZ)
 590                 return -EINVAL;
 591
 592         virt = dma_alloc_coherent(bpmp->dev, TAG_SZ, &phys,
 593                                   GFP_KERNEL | GFP_DMA32);
 594         if (!virt)
 595                 return -ENOMEM;
 596
 597         memset(&request, 0, sizeof(request));
 598         request.addr = phys;
 599
 600         memset(&msg, 0, sizeof(msg));
 601         msg.mrq = MRQ_QUERY_TAG;
 602         msg.tx.data = &request;
 603         msg.tx.size = sizeof(request);
 604
 605         local_irq_save(flags);
 606         err = tegra_bpmp_transfer_atomic(bpmp, &msg);
 607         local_irq_restore(flags);
 608
 609         if (err == 0)
 610                 memcpy(tag, virt, TAG_SZ);
 611
 612         dma_free_coherent(bpmp->dev, TAG_SZ, virt, phys);
 613
 614         return err;
 615 }
 616
 617 static int tegra_bpmp_get_firmware_tag(struct tegra_bpmp *bpmp, char *tag,
 618                                        size_t size)
 619 {
 620         if (tegra_bpmp_mrq_is_supported(bpmp, MRQ_QUERY_FW_TAG)) {
 621                 struct mrq_query_fw_tag_response resp;
 622                 struct tegra_bpmp_message msg = {
 623                         .mrq = MRQ_QUERY_FW_TAG,
 624                         .rx = {
 625                                 .data = &resp,
 626                                 .size = sizeof(resp),
 627                         },
 628                 };
 629                 int err;
 630
 631                 if (size != sizeof(resp.tag))
 632                         return -EINVAL;
 633
 634                 err = tegra_bpmp_transfer(bpmp, &msg);
 635
 636                 if (err)
 637                         return err;
 638                 if (msg.rx.ret < 0)
 639                         return -EINVAL;
 640
 641                 memcpy(tag, resp.tag, sizeof(resp.tag));
 642                 return 0;
 643         }
 644
 645         return tegra_bpmp_get_firmware_tag_old(bpmp, tag, size);
 646 }
 647
 648 static void tegra_bpmp_channel_signal(struct tegra_bpmp_channel *channel)
 649 {
 650         unsigned long flags = channel->ob->flags;
 651
 652         if ((flags & MSG_RING) == 0)
 653                 return;
 654
 655         complete(&channel->completion);
 656 }
 657
 658 void tegra_bpmp_handle_rx(struct tegra_bpmp *bpmp)
 659 {
 660         struct tegra_bpmp_channel *channel;
 661         unsigned int i, count;
 662         unsigned long *busy;
 663
 664         channel = bpmp->rx_channel;
 665         count = bpmp->soc->channels.thread.count;
 666         busy = bpmp->threaded.busy;
 667
 668         if (tegra_bpmp_is_request_ready(channel))
 669                 tegra_bpmp_handle_mrq(bpmp, channel->ib->code, channel);
 670
 671         spin_lock(&bpmp->lock);
 672
 673         for_each_set_bit(i, busy, count) {
 674                 struct tegra_bpmp_channel *channel;
 675
 676                 channel = &bpmp->threaded_channels[i];
 677
 678                 if (tegra_bpmp_is_response_ready(channel)) {
 679                         tegra_bpmp_channel_signal(channel);
 680                         clear_bit(i, busy);
 681                 }
 682         }
 683
 684         spin_unlock(&bpmp->lock);
 685 }
 686
 687 static int tegra_bpmp_probe(struct platform_device *pdev)
 688 {
 689         struct tegra_bpmp *bpmp;
 690         char tag[TAG_SZ];
 691         size_t size;
 692         int err;
 693
 694         bpmp = devm_kzalloc(&pdev->dev, sizeof(*bpmp), GFP_KERNEL);
 695         if (!bpmp)
 696                 return -ENOMEM;
 697
 698         bpmp->soc = of_device_get_match_data(&pdev->dev);
 699         bpmp->dev = &pdev->dev;
 700
 701         INIT_LIST_HEAD(&bpmp->mrqs);
 702         spin_lock_init(&bpmp->lock);
 703
 704         bpmp->threaded.count = bpmp->soc->channels.thread.count;
 705         sema_init(&bpmp->threaded.lock, bpmp->threaded.count);
 706
 707         size = BITS_TO_LONGS(bpmp->threaded.count) * sizeof(long);
 708
 709         bpmp->threaded.allocated = devm_kzalloc(&pdev->dev, size, GFP_KERNEL);
 710         if (!bpmp->threaded.allocated)
 711                 return -ENOMEM;
 712
 713         bpmp->threaded.busy = devm_kzalloc(&pdev->dev, size, GFP_KERNEL);
 714         if (!bpmp->threaded.busy)
 715                 return -ENOMEM;
 716
 717         spin_lock_init(&bpmp->atomic_tx_lock);
 718         bpmp->tx_channel = devm_kzalloc(&pdev->dev, sizeof(*bpmp->tx_channel),
 719                                         GFP_KERNEL);
 720         if (!bpmp->tx_channel)
 721                 return -ENOMEM;
 722
 723         bpmp->rx_channel = devm_kzalloc(&pdev->dev, sizeof(*bpmp->rx_channel),
 724                                         GFP_KERNEL);
 725         if (!bpmp->rx_channel)
 726                 return -ENOMEM;
 727
 728         bpmp->threaded_channels = devm_kcalloc(&pdev->dev, bpmp->threaded.count,
 729                                                sizeof(*bpmp->threaded_channels),
 730                                                GFP_KERNEL);
 731         if (!bpmp->threaded_channels)
 732                 return -ENOMEM;
 733
 734         err = bpmp->soc->ops->init(bpmp);
 735         if (err < 0)
 736                 return err;
 737
 738         err = tegra_bpmp_request_mrq(bpmp, MRQ_PING,
 739                                      tegra_bpmp_mrq_handle_ping, bpmp);
 740         if (err < 0)
 741                 goto deinit;
 742
 743         err = tegra_bpmp_ping(bpmp);
 744         if (err < 0) {
 745                 dev_err(&pdev->dev, "failed to ping BPMP: %d\n", err);
 746                 goto free_mrq;
 747         }
 748
 749         err = tegra_bpmp_get_firmware_tag(bpmp, tag, sizeof(tag));
 750         if (err < 0) {
 751                 dev_err(&pdev->dev, "failed to get firmware tag: %d\n", err);
 752                 goto free_mrq;
 753         }
 754
 755         dev_info(&pdev->dev, "firmware: %.*s\n", (int)sizeof(tag), tag);
 756
 757         platform_set_drvdata(pdev, bpmp);
 758
 759         err = of_platform_default_populate(pdev->dev.of_node, NULL, &pdev->dev);
 760         if (err < 0)
 761                 goto free_mrq;
 762
 763         if (of_find_property(pdev->dev.of_node, "#clock-cells", NULL)) {
 764                 err = tegra_bpmp_init_clocks(bpmp);
 765                 if (err < 0)
 766                         goto free_mrq;
 767         }
 768
 769         if (of_find_property(pdev->dev.of_node, "#reset-cells", NULL)) {
 770                 err = tegra_bpmp_init_resets(bpmp);
 771                 if (err < 0)
 772                         goto free_mrq;
 773         }
 774
 775         if (of_find_property(pdev->dev.of_node, "#power-domain-cells", NULL)) {
 776                 err = tegra_bpmp_init_powergates(bpmp);
 777                 if (err < 0)
 778                         goto free_mrq;
 779         }
 780
 781         err = tegra_bpmp_init_debugfs(bpmp);
 782         if (err < 0)
 783                 dev_err(&pdev->dev, "debugfs initialization failed: %d\n", err);
 784
 785         return 0;
 786
 787 free_mrq:
 788         tegra_bpmp_free_mrq(bpmp, MRQ_PING, bpmp);
 789 deinit:
 790         if (bpmp->soc->ops->deinit)
 791                 bpmp->soc->ops->deinit(bpmp);
 792
 793         return err;
 794 }
 795
 796 static int __maybe_unused tegra_bpmp_resume(struct device *dev)
 797 {
 798         struct tegra_bpmp *bpmp = dev_get_drvdata(dev);
 799
 800         if (bpmp->soc->ops->resume)
 801                 return bpmp->soc->ops->resume(bpmp);
 802         else
 803                 return 0;
 804 }
 805
 806 static SIMPLE_DEV_PM_OPS(tegra_bpmp_pm_ops, NULL, tegra_bpmp_resume);
 807
 808 #if IS_ENABLED(CONFIG_ARCH_TEGRA_186_SOC) || \
 809     IS_ENABLED(CONFIG_ARCH_TEGRA_194_SOC)
 810 static const struct tegra_bpmp_soc tegra186_soc = {
 811         .channels = {
 812                 .cpu_tx = {
 813                         .offset = 3,
 814                         .timeout = 60 * USEC_PER_SEC,
 815                 },
 816                 .thread = {
 817                         .offset = 0,
 818                         .count = 3,
 819                         .timeout = 600 * USEC_PER_SEC,
 820                 },
 821                 .cpu_rx = {
 822                         .offset = 13,
 823                         .timeout = 0,
 824                 },
 825         },
 826         .ops = &tegra186_bpmp_ops,
 827         .num_resets = 193,
 828 };
 829 #endif
 830
 831 #if IS_ENABLED(CONFIG_ARCH_TEGRA_210_SOC)
 832 static const struct tegra_bpmp_soc tegra210_soc = {
 833         .channels = {
 834                 .cpu_tx = {
 835                         .offset = 0,
 836                         .count = 1,
 837                         .timeout = 60 * USEC_PER_SEC,
 838                 },
 839                 .thread = {
 840                         .offset = 4,
 841                         .count = 1,
 842                         .timeout = 600 * USEC_PER_SEC,
 843                 },
 844                 .cpu_rx = {
 845                         .offset = 8,
 846                         .count = 1,
 847                         .timeout = 0,
 848                 },
 849         },
 850         .ops = &tegra210_bpmp_ops,
 851 };
 852 #endif
 853
 854 static const struct of_device_id tegra_bpmp_match[] = {
 855 #if IS_ENABLED(CONFIG_ARCH_TEGRA_186_SOC) || \
 856     IS_ENABLED(CONFIG_ARCH_TEGRA_194_SOC)
 857         { .compatible = "nvidia,tegra186-bpmp", .data = &tegra186_soc },
 858 #endif
 859 #if IS_ENABLED(CONFIG_ARCH_TEGRA_210_SOC)
 860         { .compatible = "nvidia,tegra210-bpmp", .data = &tegra210_soc },
 861 #endif
 862         { }
 863 };
 864
 865 static struct platform_driver tegra_bpmp_driver = {
 866         .driver = {
 867                 .name = "tegra-bpmp",
 868                 .of_match_table = tegra_bpmp_match,
 869                 .pm = &tegra_bpmp_pm_ops,
 870         },
 871         .probe = tegra_bpmp_probe,
 872 };
 873
 874 static int __init tegra_bpmp_init(void)
 875 {
 876         return platform_driver_register(&tegra_bpmp_driver);
 877 }
 878 core_initcall(tegra_bpmp_init);