drivers/rpmsg/qcom_glink_native.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Copyright (c) 2016-2017, Linaro Ltd
   4  */
   5
   6 #include <linux/idr.h>
   7 #include <linux/interrupt.h>
   8 #include <linux/io.h>
   9 #include <linux/list.h>
  10 #include <linux/mfd/syscon.h>
  11 #include <linux/module.h>
  12 #include <linux/of.h>
  13 #include <linux/of_address.h>
  14 #include <linux/of_irq.h>
  15 #include <linux/platform_device.h>
  16 #include <linux/regmap.h>
  17 #include <linux/rpmsg.h>
  18 #include <linux/sizes.h>
  19 #include <linux/slab.h>
  20 #include <linux/workqueue.h>
  21 #include <linux/mailbox_client.h>
  22
  23 #include "rpmsg_internal.h"
  24 #include "qcom_glink_native.h"
  25
  26 #define GLINK_NAME_SIZE         32
  27 #define GLINK_VERSION_1         1
  28
  29 #define RPM_GLINK_CID_MIN       1
  30 #define RPM_GLINK_CID_MAX       65536
  31
  32 struct glink_msg {
  33         __le16 cmd;
  34         __le16 param1;
  35         __le32 param2;
  36         u8 data[];
  37 } __packed;
  38
  39 /**
  40  * struct glink_defer_cmd - deferred incoming control message
  41  * @node:       list node
  42  * @msg:        message header
  43  * data:        payload of the message
  44  *
  45  * Copy of a received control message, to be added to @rx_queue and processed
  46  * by @rx_work of @qcom_glink.
  47  */
  48 struct glink_defer_cmd {
  49         struct list_head node;
  50
  51         struct glink_msg msg;
  52         u8 data[];
  53 };
  54
  55 /**
  56  * struct glink_core_rx_intent - RX intent
  57  * RX intent
  58  *
  59  * data: pointer to the data (may be NULL for zero-copy)
  60  * id: remote or local intent ID
  61  * size: size of the original intent (do not modify)
  62  * reuse: To mark if the intent can be reused after first use
  63  * in_use: To mark if intent is already in use for the channel
  64  * offset: next write offset (initially 0)
  65  */
  66 struct glink_core_rx_intent {
  67         void *data;
  68         u32 id;
  69         size_t size;
  70         bool reuse;
  71         bool in_use;
  72         u32 offset;
  73
  74         struct list_head node;
  75 };
  76
  77 /**
  78  * struct qcom_glink - driver context, relates to one remote subsystem
  79  * @dev:        reference to the associated struct device
  80  * @mbox_client: mailbox client
  81  * @mbox_chan:  mailbox channel
  82  * @rx_pipe:    pipe object for receive FIFO
  83  * @tx_pipe:    pipe object for transmit FIFO
  84  * @irq:        IRQ for signaling incoming events
  85  * @rx_work:    worker for handling received control messages
  86  * @rx_lock:    protects the @rx_queue
  87  * @rx_queue:   queue of received control messages to be processed in @rx_work
  88  * @tx_lock:    synchronizes operations on the tx fifo
  89  * @idr_lock:   synchronizes @lcids and @rcids modifications
  90  * @lcids:      idr of all channels with a known local channel id
  91  * @rcids:      idr of all channels with a known remote channel id
  92  */
  93 struct qcom_glink {
  94         struct device *dev;
  95
  96         struct mbox_client mbox_client;
  97         struct mbox_chan *mbox_chan;
  98
  99         struct qcom_glink_pipe *rx_pipe;
 100         struct qcom_glink_pipe *tx_pipe;
 101
 102         int irq;
 103
 104         struct work_struct rx_work;
 105         spinlock_t rx_lock;
 106         struct list_head rx_queue;
 107
 108         spinlock_t tx_lock;
 109
 110         spinlock_t idr_lock;
 111         struct idr lcids;
 112         struct idr rcids;
 113         unsigned long features;
 114
 115         bool intentless;
 116 };
 117
 118 enum {
 119         GLINK_STATE_CLOSED,
 120         GLINK_STATE_OPENING,
 121         GLINK_STATE_OPEN,
 122         GLINK_STATE_CLOSING,
 123 };
 124
 125 /**
 126  * struct glink_channel - internal representation of a channel
 127  * @rpdev:      rpdev reference, only used for primary endpoints
 128  * @ept:        rpmsg endpoint this channel is associated with
 129  * @glink:      qcom_glink context handle
 130  * @refcount:   refcount for the channel object
 131  * @recv_lock:  guard for @ept.cb
 132  * @name:       unique channel name/identifier
 133  * @lcid:       channel id, in local space
 134  * @rcid:       channel id, in remote space
 135  * @intent_lock: lock for protection of @liids, @riids
 136  * @liids:      idr of all local intents
 137  * @riids:      idr of all remote intents
 138  * @intent_work: worker responsible for transmitting rx_done packets
 139  * @done_intents: list of intents that needs to be announced rx_done
 140  * @buf:        receive buffer, for gathering fragments
 141  * @buf_offset: write offset in @buf
 142  * @buf_size:   size of current @buf
 143  * @open_ack:   completed once remote has acked the open-request
 144  * @open_req:   completed once open-request has been received
 145  * @intent_req_lock: Synchronises multiple intent requests
 146  * @intent_req_result: Result of intent request
 147  * @intent_req_comp: Completion for intent_req signalling
 148  */
 149 struct glink_channel {
 150         struct rpmsg_endpoint ept;
 151
 152         struct rpmsg_device *rpdev;
 153         struct qcom_glink *glink;
 154
 155         struct kref refcount;
 156
 157         spinlock_t recv_lock;
 158
 159         char *name;
 160         unsigned int lcid;
 161         unsigned int rcid;
 162
 163         spinlock_t intent_lock;
 164         struct idr liids;
 165         struct idr riids;
 166         struct work_struct intent_work;
 167         struct list_head done_intents;
 168
 169         struct glink_core_rx_intent *buf;
 170         int buf_offset;
 171         int buf_size;
 172
 173         struct completion open_ack;
 174         struct completion open_req;
 175
 176         struct mutex intent_req_lock;
 177         bool intent_req_result;
 178         struct completion intent_req_comp;
 179 };
 180
 181 #define to_glink_channel(_ept) container_of(_ept, struct glink_channel, ept)
 182
 183 static const struct rpmsg_endpoint_ops glink_endpoint_ops;
 184
 185 #define RPM_CMD_VERSION                 0
 186 #define RPM_CMD_VERSION_ACK             1
 187 #define RPM_CMD_OPEN                    2
 188 #define RPM_CMD_CLOSE                   3
 189 #define RPM_CMD_OPEN_ACK                4
 190 #define RPM_CMD_INTENT                  5
 191 #define RPM_CMD_RX_DONE                 6
 192 #define RPM_CMD_RX_INTENT_REQ           7
 193 #define RPM_CMD_RX_INTENT_REQ_ACK       8
 194 #define RPM_CMD_TX_DATA                 9
 195 #define RPM_CMD_CLOSE_ACK               11
 196 #define RPM_CMD_TX_DATA_CONT            12
 197 #define RPM_CMD_READ_NOTIF              13
 198 #define RPM_CMD_RX_DONE_W_REUSE         14
 199
 200 #define GLINK_FEATURE_INTENTLESS        BIT(1)
 201
 202 static void qcom_glink_rx_done_work(struct work_struct *work);
 203
 204 static struct glink_channel *qcom_glink_alloc_channel(struct qcom_glink *glink,
 205                                                       const char *name)
 206 {
 207         struct glink_channel *channel;
 208
 209         channel = kzalloc(sizeof(*channel), GFP_KERNEL);
 210         if (!channel)
 211                 return ERR_PTR(-ENOMEM);
 212
 213         /* Setup glink internal glink_channel data */
 214         spin_lock_init(&channel->recv_lock);
 215         spin_lock_init(&channel->intent_lock);
 216         mutex_init(&channel->intent_req_lock);
 217
 218         channel->glink = glink;
 219         channel->name = kstrdup(name, GFP_KERNEL);
 220
 221         init_completion(&channel->open_req);
 222         init_completion(&channel->open_ack);
 223         init_completion(&channel->intent_req_comp);
 224
 225         INIT_LIST_HEAD(&channel->done_intents);
 226         INIT_WORK(&channel->intent_work, qcom_glink_rx_done_work);
 227
 228         idr_init(&channel->liids);
 229         idr_init(&channel->riids);
 230         kref_init(&channel->refcount);
 231
 232         return channel;
 233 }
 234
 235 static void qcom_glink_channel_release(struct kref *ref)
 236 {
 237         struct glink_channel *channel = container_of(ref, struct glink_channel,
 238                                                      refcount);
 239         unsigned long flags;
 240
 241         spin_lock_irqsave(&channel->intent_lock, flags);
 242         idr_destroy(&channel->liids);
 243         idr_destroy(&channel->riids);
 244         spin_unlock_irqrestore(&channel->intent_lock, flags);
 245
 246         kfree(channel->name);
 247         kfree(channel);
 248 }
 249
 250 static size_t qcom_glink_rx_avail(struct qcom_glink *glink)
 251 {
 252         return glink->rx_pipe->avail(glink->rx_pipe);
 253 }
 254
 255 static void qcom_glink_rx_peak(struct qcom_glink *glink,
 256                                void *data, unsigned int offset, size_t count)
 257 {
 258         glink->rx_pipe->peak(glink->rx_pipe, data, offset, count);
 259 }
 260
 261 static void qcom_glink_rx_advance(struct qcom_glink *glink, size_t count)
 262 {
 263         glink->rx_pipe->advance(glink->rx_pipe, count);
 264 }
 265
 266 static size_t qcom_glink_tx_avail(struct qcom_glink *glink)
 267 {
 268         return glink->tx_pipe->avail(glink->tx_pipe);
 269 }
 270
 271 static void qcom_glink_tx_write(struct qcom_glink *glink,
 272                                 const void *hdr, size_t hlen,
 273                                 const void *data, size_t dlen)
 274 {
 275         glink->tx_pipe->write(glink->tx_pipe, hdr, hlen, data, dlen);
 276 }
 277
 278 static int qcom_glink_tx(struct qcom_glink *glink,
 279                          const void *hdr, size_t hlen,
 280                          const void *data, size_t dlen, bool wait)
 281 {
 282         unsigned int tlen = hlen + dlen;
 283         unsigned long flags;
 284         int ret = 0;
 285
 286         /* Reject packets that are too big */
 287         if (tlen >= glink->tx_pipe->length)
 288                 return -EINVAL;
 289
 290         spin_lock_irqsave(&glink->tx_lock, flags);
 291
 292         while (qcom_glink_tx_avail(glink) < tlen) {
 293                 if (!wait) {
 294                         ret = -EAGAIN;
 295                         goto out;
 296                 }
 297
 298                 /* Wait without holding the tx_lock */
 299                 spin_unlock_irqrestore(&glink->tx_lock, flags);
 300
 301                 usleep_range(10000, 15000);
 302
 303                 spin_lock_irqsave(&glink->tx_lock, flags);
 304         }
 305
 306         qcom_glink_tx_write(glink, hdr, hlen, data, dlen);
 307
 308         mbox_send_message(glink->mbox_chan, NULL);
 309         mbox_client_txdone(glink->mbox_chan, 0);
 310
 311 out:
 312         spin_unlock_irqrestore(&glink->tx_lock, flags);
 313
 314         return ret;
 315 }
 316
 317 static int qcom_glink_send_version(struct qcom_glink *glink)
 318 {
 319         struct glink_msg msg;
 320
 321         msg.cmd = cpu_to_le16(RPM_CMD_VERSION);
 322         msg.param1 = cpu_to_le16(GLINK_VERSION_1);
 323         msg.param2 = cpu_to_le32(glink->features);
 324
 325         return qcom_glink_tx(glink, &msg, sizeof(msg), NULL, 0, true);
 326 }
 327
 328 static void qcom_glink_send_version_ack(struct qcom_glink *glink)
 329 {
 330         struct glink_msg msg;
 331
 332         msg.cmd = cpu_to_le16(RPM_CMD_VERSION_ACK);
 333         msg.param1 = cpu_to_le16(GLINK_VERSION_1);
 334         msg.param2 = cpu_to_le32(glink->features);
 335
 336         qcom_glink_tx(glink, &msg, sizeof(msg), NULL, 0, true);
 337 }
 338
 339 static void qcom_glink_send_open_ack(struct qcom_glink *glink,
 340                                      struct glink_channel *channel)
 341 {
 342         struct glink_msg msg;
 343
 344         msg.cmd = cpu_to_le16(RPM_CMD_OPEN_ACK);
 345         msg.param1 = cpu_to_le16(channel->rcid);
 346         msg.param2 = cpu_to_le32(0);
 347
 348         qcom_glink_tx(glink, &msg, sizeof(msg), NULL, 0, true);
 349 }
 350
 351 static void qcom_glink_handle_intent_req_ack(struct qcom_glink *glink,
 352                                              unsigned int cid, bool granted)
 353 {
 354         struct glink_channel *channel;
 355         unsigned long flags;
 356
 357         spin_lock_irqsave(&glink->idr_lock, flags);
 358         channel = idr_find(&glink->rcids, cid);
 359         spin_unlock_irqrestore(&glink->idr_lock, flags);
 360         if (!channel) {
 361                 dev_err(glink->dev, "unable to find channel\n");
 362                 return;
 363         }
 364
 365         channel->intent_req_result = granted;
 366         complete(&channel->intent_req_comp);
 367 }
 368
 369 /**
 370  * qcom_glink_send_open_req() - send a RPM_CMD_OPEN request to the remote
 371  * @glink: Ptr to the glink edge
 372  * @channel: Ptr to the channel that the open req is sent
 373  *
 374  * Allocates a local channel id and sends a RPM_CMD_OPEN message to the remote.
 375  * Will return with refcount held, regardless of outcome.
 376  *
 377  * Returns 0 on success, negative errno otherwise.
 378  */
 379 static int qcom_glink_send_open_req(struct qcom_glink *glink,
 380                                     struct glink_channel *channel)
 381 {
 382         struct {
 383                 struct glink_msg msg;
 384                 u8 name[GLINK_NAME_SIZE];
 385         } __packed req;
 386         int name_len = strlen(channel->name) + 1;
 387         int req_len = ALIGN(sizeof(req.msg) + name_len, 8);
 388         int ret;
 389         unsigned long flags;
 390
 391         kref_get(&channel->refcount);
 392
 393         spin_lock_irqsave(&glink->idr_lock, flags);
 394         ret = idr_alloc_cyclic(&glink->lcids, channel,
 395                                RPM_GLINK_CID_MIN, RPM_GLINK_CID_MAX,
 396                                GFP_ATOMIC);
 397         spin_unlock_irqrestore(&glink->idr_lock, flags);
 398         if (ret < 0)
 399                 return ret;
 400
 401         channel->lcid = ret;
 402
 403         req.msg.cmd = cpu_to_le16(RPM_CMD_OPEN);
 404         req.msg.param1 = cpu_to_le16(channel->lcid);
 405         req.msg.param2 = cpu_to_le32(name_len);
 406         strcpy(req.name, channel->name);
 407
 408         ret = qcom_glink_tx(glink, &req, req_len, NULL, 0, true);
 409         if (ret)
 410                 goto remove_idr;
 411
 412         return 0;
 413
 414 remove_idr:
 415         spin_lock_irqsave(&glink->idr_lock, flags);
 416         idr_remove(&glink->lcids, channel->lcid);
 417         channel->lcid = 0;
 418         spin_unlock_irqrestore(&glink->idr_lock, flags);
 419
 420         return ret;
 421 }
 422
 423 static void qcom_glink_send_close_req(struct qcom_glink *glink,
 424                                       struct glink_channel *channel)
 425 {
 426         struct glink_msg req;
 427
 428         req.cmd = cpu_to_le16(RPM_CMD_CLOSE);
 429         req.param1 = cpu_to_le16(channel->lcid);
 430         req.param2 = 0;
 431
 432         qcom_glink_tx(glink, &req, sizeof(req), NULL, 0, true);
 433 }
 434
 435 static void qcom_glink_send_close_ack(struct qcom_glink *glink,
 436                                       unsigned int rcid)
 437 {
 438         struct glink_msg req;
 439
 440         req.cmd = cpu_to_le16(RPM_CMD_CLOSE_ACK);
 441         req.param1 = cpu_to_le16(rcid);
 442         req.param2 = 0;
 443
 444         qcom_glink_tx(glink, &req, sizeof(req), NULL, 0, true);
 445 }
 446
 447 static void qcom_glink_rx_done_work(struct work_struct *work)
 448 {
 449         struct glink_channel *channel = container_of(work, struct glink_channel,
 450                                                      intent_work);
 451         struct qcom_glink *glink = channel->glink;
 452         struct glink_core_rx_intent *intent, *tmp;
 453         struct {
 454                 u16 id;
 455                 u16 lcid;
 456                 u32 liid;
 457         } __packed cmd;
 458
 459         unsigned int cid = channel->lcid;
 460         unsigned int iid;
 461         bool reuse;
 462         unsigned long flags;
 463
 464         spin_lock_irqsave(&channel->intent_lock, flags);
 465         list_for_each_entry_safe(intent, tmp, &channel->done_intents, node) {
 466                 list_del(&intent->node);
 467                 spin_unlock_irqrestore(&channel->intent_lock, flags);
 468                 iid = intent->id;
 469                 reuse = intent->reuse;
 470
 471                 cmd.id = reuse ? RPM_CMD_RX_DONE_W_REUSE : RPM_CMD_RX_DONE;
 472                 cmd.lcid = cid;
 473                 cmd.liid = iid;
 474
 475                 qcom_glink_tx(glink, &cmd, sizeof(cmd), NULL, 0, true);
 476                 if (!reuse) {
 477                         kfree(intent->data);
 478                         kfree(intent);
 479                 }
 480                 spin_lock_irqsave(&channel->intent_lock, flags);
 481         }
 482         spin_unlock_irqrestore(&channel->intent_lock, flags);
 483 }
 484
 485 static void qcom_glink_rx_done(struct qcom_glink *glink,
 486                                struct glink_channel *channel,
 487                                struct glink_core_rx_intent *intent)
 488 {
 489         /* We don't send RX_DONE to intentless systems */
 490         if (glink->intentless) {
 491                 kfree(intent->data);
 492                 kfree(intent);
 493                 return;
 494         }
 495
 496         /* Take it off the tree of receive intents */
 497         if (!intent->reuse) {
 498                 spin_lock(&channel->intent_lock);
 499                 idr_remove(&channel->liids, intent->id);
 500                 spin_unlock(&channel->intent_lock);
 501         }
 502
 503         /* Schedule the sending of a rx_done indication */
 504         spin_lock(&channel->intent_lock);
 505         list_add_tail(&intent->node, &channel->done_intents);
 506         spin_unlock(&channel->intent_lock);
 507
 508         schedule_work(&channel->intent_work);
 509 }
 510
 511 /**
 512  * qcom_glink_receive_version() - receive version/features from remote system
 513  *
 514  * @glink:      pointer to transport interface
 515  * @r_version:  remote version
 516  * @r_features: remote features
 517  *
 518  * This function is called in response to a remote-initiated version/feature
 519  * negotiation sequence.
 520  */
 521 static void qcom_glink_receive_version(struct qcom_glink *glink,
 522                                        u32 version,
 523                                        u32 features)
 524 {
 525         switch (version) {
 526         case 0:
 527                 break;
 528         case GLINK_VERSION_1:
 529                 glink->features &= features;
 530                 /* FALLTHROUGH */
 531         default:
 532                 qcom_glink_send_version_ack(glink);
 533                 break;
 534         }
 535 }
 536
 537 /**
 538  * qcom_glink_receive_version_ack() - receive negotiation ack from remote system
 539  *
 540  * @glink:      pointer to transport interface
 541  * @r_version:  remote version response
 542  * @r_features: remote features response
 543  *
 544  * This function is called in response to a local-initiated version/feature
 545  * negotiation sequence and is the counter-offer from the remote side based
 546  * upon the initial version and feature set requested.
 547  */
 548 static void qcom_glink_receive_version_ack(struct qcom_glink *glink,
 549                                            u32 version,
 550                                            u32 features)
 551 {
 552         switch (version) {
 553         case 0:
 554                 /* Version negotiation failed */
 555                 break;
 556         case GLINK_VERSION_1:
 557                 if (features == glink->features)
 558                         break;
 559
 560                 glink->features &= features;
 561                 /* FALLTHROUGH */
 562         default:
 563                 qcom_glink_send_version(glink);
 564                 break;
 565         }
 566 }
 567
 568 /**
 569  * qcom_glink_send_intent_req_ack() - convert an rx intent request ack cmd to
 570                                       wire format and transmit
 571  * @glink:      The transport to transmit on.
 572  * @channel:    The glink channel
 573  * @granted:    The request response to encode.
 574  *
 575  * Return: 0 on success or standard Linux error code.
 576  */
 577 static int qcom_glink_send_intent_req_ack(struct qcom_glink *glink,
 578                                           struct glink_channel *channel,
 579                                           bool granted)
 580 {
 581         struct glink_msg msg;
 582
 583         msg.cmd = cpu_to_le16(RPM_CMD_RX_INTENT_REQ_ACK);
 584         msg.param1 = cpu_to_le16(channel->lcid);
 585         msg.param2 = cpu_to_le32(granted);
 586
 587         qcom_glink_tx(glink, &msg, sizeof(msg), NULL, 0, true);
 588
 589         return 0;
 590 }
 591
 592 /**
 593  * qcom_glink_advertise_intent - convert an rx intent cmd to wire format and
 594  *                         transmit
 595  * @glink:      The transport to transmit on.
 596  * @channel:    The local channel
 597  * @size:       The intent to pass on to remote.
 598  *
 599  * Return: 0 on success or standard Linux error code.
 600  */
 601 static int qcom_glink_advertise_intent(struct qcom_glink *glink,
 602                                        struct glink_channel *channel,
 603                                        struct glink_core_rx_intent *intent)
 604 {
 605         struct command {
 606                 u16 id;
 607                 u16 lcid;
 608                 u32 count;
 609                 u32 size;
 610                 u32 liid;
 611         } __packed;
 612         struct command cmd;
 613
 614         cmd.id = cpu_to_le16(RPM_CMD_INTENT);
 615         cmd.lcid = cpu_to_le16(channel->lcid);
 616         cmd.count = cpu_to_le32(1);
 617         cmd.size = cpu_to_le32(intent->size);
 618         cmd.liid = cpu_to_le32(intent->id);
 619
 620         qcom_glink_tx(glink, &cmd, sizeof(cmd), NULL, 0, true);
 621
 622         return 0;
 623 }
 624
 625 static struct glink_core_rx_intent *
 626 qcom_glink_alloc_intent(struct qcom_glink *glink,
 627                         struct glink_channel *channel,
 628                         size_t size,
 629                         bool reuseable)
 630 {
 631         struct glink_core_rx_intent *intent;
 632         int ret;
 633         unsigned long flags;
 634
 635         intent = kzalloc(sizeof(*intent), GFP_KERNEL);
 636         if (!intent)
 637                 return NULL;
 638
 639         intent->data = kzalloc(size, GFP_KERNEL);
 640         if (!intent->data)
 641                 goto free_intent;
 642
 643         spin_lock_irqsave(&channel->intent_lock, flags);
 644         ret = idr_alloc_cyclic(&channel->liids, intent, 1, -1, GFP_ATOMIC);
 645         if (ret < 0) {
 646                 spin_unlock_irqrestore(&channel->intent_lock, flags);
 647                 goto free_data;
 648         }
 649         spin_unlock_irqrestore(&channel->intent_lock, flags);
 650
 651         intent->id = ret;
 652         intent->size = size;
 653         intent->reuse = reuseable;
 654
 655         return intent;
 656
 657 free_data:
 658         kfree(intent->data);
 659 free_intent:
 660         kfree(intent);
 661         return NULL;
 662 }
 663
 664 static void qcom_glink_handle_rx_done(struct qcom_glink *glink,
 665                                       u32 cid, uint32_t iid,
 666                                       bool reuse)
 667 {
 668         struct glink_core_rx_intent *intent;
 669         struct glink_channel *channel;
 670         unsigned long flags;
 671
 672         spin_lock_irqsave(&glink->idr_lock, flags);
 673         channel = idr_find(&glink->rcids, cid);
 674         spin_unlock_irqrestore(&glink->idr_lock, flags);
 675         if (!channel) {
 676                 dev_err(glink->dev, "invalid channel id received\n");
 677                 return;
 678         }
 679
 680         spin_lock_irqsave(&channel->intent_lock, flags);
 681         intent = idr_find(&channel->riids, iid);
 682
 683         if (!intent) {
 684                 spin_unlock_irqrestore(&channel->intent_lock, flags);
 685                 dev_err(glink->dev, "invalid intent id received\n");
 686                 return;
 687         }
 688
 689         intent->in_use = false;
 690
 691         if (!reuse) {
 692                 idr_remove(&channel->riids, intent->id);
 693                 kfree(intent);
 694         }
 695         spin_unlock_irqrestore(&channel->intent_lock, flags);
 696 }
 697
 698 /**
 699  * qcom_glink_handle_intent_req() - Receive a request for rx_intent
 700  *                                          from remote side
 701  * if_ptr:      Pointer to the transport interface
 702  * rcid:        Remote channel ID
 703  * size:        size of the intent
 704  *
 705  * The function searches for the local channel to which the request for
 706  * rx_intent has arrived and allocates and notifies the remote back
 707  */
 708 static void qcom_glink_handle_intent_req(struct qcom_glink *glink,
 709                                          u32 cid, size_t size)
 710 {
 711         struct glink_core_rx_intent *intent;
 712         struct glink_channel *channel;
 713         unsigned long flags;
 714
 715         spin_lock_irqsave(&glink->idr_lock, flags);
 716         channel = idr_find(&glink->rcids, cid);
 717         spin_unlock_irqrestore(&glink->idr_lock, flags);
 718
 719         if (!channel) {
 720                 pr_err("%s channel not found for cid %d\n", __func__, cid);
 721                 return;
 722         }
 723
 724         intent = qcom_glink_alloc_intent(glink, channel, size, false);
 725         if (intent)
 726                 qcom_glink_advertise_intent(glink, channel, intent);
 727
 728         qcom_glink_send_intent_req_ack(glink, channel, !!intent);
 729 }
 730
 731 static int qcom_glink_rx_defer(struct qcom_glink *glink, size_t extra)
 732 {
 733         struct glink_defer_cmd *dcmd;
 734
 735         extra = ALIGN(extra, 8);
 736
 737         if (qcom_glink_rx_avail(glink) < sizeof(struct glink_msg) + extra) {
 738                 dev_dbg(glink->dev, "Insufficient data in rx fifo");
 739                 return -ENXIO;
 740         }
 741
 742         dcmd = kzalloc(sizeof(*dcmd) + extra, GFP_ATOMIC);
 743         if (!dcmd)
 744                 return -ENOMEM;
 745
 746         INIT_LIST_HEAD(&dcmd->node);
 747
 748         qcom_glink_rx_peak(glink, &dcmd->msg, 0, sizeof(dcmd->msg) + extra);
 749
 750         spin_lock(&glink->rx_lock);
 751         list_add_tail(&dcmd->node, &glink->rx_queue);
 752         spin_unlock(&glink->rx_lock);
 753
 754         schedule_work(&glink->rx_work);
 755         qcom_glink_rx_advance(glink, sizeof(dcmd->msg) + extra);
 756
 757         return 0;
 758 }
 759
 760 static int qcom_glink_rx_data(struct qcom_glink *glink, size_t avail)
 761 {
 762         struct glink_core_rx_intent *intent;
 763         struct glink_channel *channel;
 764         struct {
 765                 struct glink_msg msg;
 766                 __le32 chunk_size;
 767                 __le32 left_size;
 768         } __packed hdr;
 769         unsigned int chunk_size;
 770         unsigned int left_size;
 771         unsigned int rcid;
 772         unsigned int liid;
 773         int ret = 0;
 774         unsigned long flags;
 775
 776         if (avail < sizeof(hdr)) {
 777                 dev_dbg(glink->dev, "Not enough data in fifo\n");
 778                 return -EAGAIN;
 779         }
 780
 781         qcom_glink_rx_peak(glink, &hdr, 0, sizeof(hdr));
 782         chunk_size = le32_to_cpu(hdr.chunk_size);
 783         left_size = le32_to_cpu(hdr.left_size);
 784
 785         if (avail < sizeof(hdr) + chunk_size) {
 786                 dev_dbg(glink->dev, "Payload not yet in fifo\n");
 787                 return -EAGAIN;
 788         }
 789
 790         if (WARN(chunk_size % 4, "Incoming data must be word aligned\n"))
 791                 return -EINVAL;
 792
 793         rcid = le16_to_cpu(hdr.msg.param1);
 794         spin_lock_irqsave(&glink->idr_lock, flags);
 795         channel = idr_find(&glink->rcids, rcid);
 796         spin_unlock_irqrestore(&glink->idr_lock, flags);
 797         if (!channel) {
 798                 dev_dbg(glink->dev, "Data on non-existing channel\n");
 799
 800                 /* Drop the message */
 801                 goto advance_rx;
 802         }
 803
 804         if (glink->intentless) {
 805                 /* Might have an ongoing, fragmented, message to append */
 806                 if (!channel->buf) {
 807                         intent = kzalloc(sizeof(*intent), GFP_ATOMIC);
 808                         if (!intent)
 809                                 return -ENOMEM;
 810
 811                         intent->data = kmalloc(chunk_size + left_size,
 812                                                GFP_ATOMIC);
 813                         if (!intent->data) {
 814                                 kfree(intent);
 815                                 return -ENOMEM;
 816                         }
 817
 818                         intent->id = 0xbabababa;
 819                         intent->size = chunk_size + left_size;
 820                         intent->offset = 0;
 821
 822                         channel->buf = intent;
 823                 } else {
 824                         intent = channel->buf;
 825                 }
 826         } else {
 827                 liid = le32_to_cpu(hdr.msg.param2);
 828
 829                 spin_lock_irqsave(&channel->intent_lock, flags);
 830                 intent = idr_find(&channel->liids, liid);
 831                 spin_unlock_irqrestore(&channel->intent_lock, flags);
 832
 833                 if (!intent) {
 834                         dev_err(glink->dev,
 835                                 "no intent found for channel %s intent %d",
 836                                 channel->name, liid);
 837                         goto advance_rx;
 838                 }
 839         }
 840
 841         if (intent->size - intent->offset < chunk_size) {
 842                 dev_err(glink->dev, "Insufficient space in intent\n");
 843
 844                 /* The packet header lied, drop payload */
 845                 goto advance_rx;
 846         }
 847
 848         qcom_glink_rx_peak(glink, intent->data + intent->offset,
 849                            sizeof(hdr), chunk_size);
 850         intent->offset += chunk_size;
 851
 852         /* Handle message when no fragments remain to be received */
 853         if (!left_size) {
 854                 spin_lock(&channel->recv_lock);
 855                 if (channel->ept.cb) {
 856                         channel->ept.cb(channel->ept.rpdev,
 857                                         intent->data,
 858                                         intent->offset,
 859                                         channel->ept.priv,
 860                                         RPMSG_ADDR_ANY);
 861                 }
 862                 spin_unlock(&channel->recv_lock);
 863
 864                 intent->offset = 0;
 865                 channel->buf = NULL;
 866
 867                 qcom_glink_rx_done(glink, channel, intent);
 868         }
 869
 870 advance_rx:
 871         qcom_glink_rx_advance(glink, ALIGN(sizeof(hdr) + chunk_size, 8));
 872
 873         return ret;
 874 }
 875
 876 static void qcom_glink_handle_intent(struct qcom_glink *glink,
 877                                      unsigned int cid,
 878                                      unsigned int count,
 879                                      size_t avail)
 880 {
 881         struct glink_core_rx_intent *intent;
 882         struct glink_channel *channel;
 883         struct intent_pair {
 884                 __le32 size;
 885                 __le32 iid;
 886         };
 887
 888         struct {
 889                 struct glink_msg msg;
 890                 struct intent_pair intents[];
 891         } __packed * msg;
 892
 893         const size_t msglen = sizeof(*msg) + sizeof(struct intent_pair) * count;
 894         int ret;
 895         int i;
 896         unsigned long flags;
 897
 898         if (avail < msglen) {
 899                 dev_dbg(glink->dev, "Not enough data in fifo\n");
 900                 return;
 901         }
 902
 903         spin_lock_irqsave(&glink->idr_lock, flags);
 904         channel = idr_find(&glink->rcids, cid);
 905         spin_unlock_irqrestore(&glink->idr_lock, flags);
 906         if (!channel) {
 907                 dev_err(glink->dev, "intents for non-existing channel\n");
 908                 return;
 909         }
 910
 911         msg = kmalloc(msglen, GFP_ATOMIC);
 912         if (!msg)
 913                 return;
 914
 915         qcom_glink_rx_peak(glink, msg, 0, msglen);
 916
 917         for (i = 0; i < count; ++i) {
 918                 intent = kzalloc(sizeof(*intent), GFP_ATOMIC);
 919                 if (!intent)
 920                         break;
 921
 922                 intent->id = le32_to_cpu(msg->intents[i].iid);
 923                 intent->size = le32_to_cpu(msg->intents[i].size);
 924
 925                 spin_lock_irqsave(&channel->intent_lock, flags);
 926                 ret = idr_alloc(&channel->riids, intent,
 927                                 intent->id, intent->id + 1, GFP_ATOMIC);
 928                 spin_unlock_irqrestore(&channel->intent_lock, flags);
 929
 930                 if (ret < 0)
 931                         dev_err(glink->dev, "failed to store remote intent\n");
 932         }
 933
 934         kfree(msg);
 935         qcom_glink_rx_advance(glink, ALIGN(msglen, 8));
 936 }
 937
 938 static int qcom_glink_rx_open_ack(struct qcom_glink *glink, unsigned int lcid)
 939 {
 940         struct glink_channel *channel;
 941
 942         spin_lock(&glink->idr_lock);
 943         channel = idr_find(&glink->lcids, lcid);
 944         spin_unlock(&glink->idr_lock);
 945         if (!channel) {
 946                 dev_err(glink->dev, "Invalid open ack packet\n");
 947                 return -EINVAL;
 948         }
 949
 950         complete(&channel->open_ack);
 951
 952         return 0;
 953 }
 954
 955 static irqreturn_t qcom_glink_native_intr(int irq, void *data)
 956 {
 957         struct qcom_glink *glink = data;
 958         struct glink_msg msg;
 959         unsigned int param1;
 960         unsigned int param2;
 961         unsigned int avail;
 962         unsigned int cmd;
 963         int ret = 0;
 964
 965         for (;;) {
 966                 avail = qcom_glink_rx_avail(glink);
 967                 if (avail < sizeof(msg))
 968                         break;
 969
 970                 qcom_glink_rx_peak(glink, &msg, 0, sizeof(msg));
 971
 972                 cmd = le16_to_cpu(msg.cmd);
 973                 param1 = le16_to_cpu(msg.param1);
 974                 param2 = le32_to_cpu(msg.param2);
 975
 976                 switch (cmd) {
 977                 case RPM_CMD_VERSION:
 978                 case RPM_CMD_VERSION_ACK:
 979                 case RPM_CMD_CLOSE:
 980                 case RPM_CMD_CLOSE_ACK:
 981                 case RPM_CMD_RX_INTENT_REQ:
 982                         ret = qcom_glink_rx_defer(glink, 0);
 983                         break;
 984                 case RPM_CMD_OPEN_ACK:
 985                         ret = qcom_glink_rx_open_ack(glink, param1);
 986                         qcom_glink_rx_advance(glink, ALIGN(sizeof(msg), 8));
 987                         break;
 988                 case RPM_CMD_OPEN:
 989                         ret = qcom_glink_rx_defer(glink, param2);
 990                         break;
 991                 case RPM_CMD_TX_DATA:
 992                 case RPM_CMD_TX_DATA_CONT:
 993                         ret = qcom_glink_rx_data(glink, avail);
 994                         break;
 995                 case RPM_CMD_READ_NOTIF:
 996                         qcom_glink_rx_advance(glink, ALIGN(sizeof(msg), 8));
 997
 998                         mbox_send_message(glink->mbox_chan, NULL);
 999                         mbox_client_txdone(glink->mbox_chan, 0);
1000                         break;
1001                 case RPM_CMD_INTENT:
1002                         qcom_glink_handle_intent(glink, param1, param2, avail);
1003                         break;
1004                 case RPM_CMD_RX_DONE:
1005                         qcom_glink_handle_rx_done(glink, param1, param2, false);
1006                         qcom_glink_rx_advance(glink, ALIGN(sizeof(msg), 8));
1007                         break;
1008                 case RPM_CMD_RX_DONE_W_REUSE:
1009                         qcom_glink_handle_rx_done(glink, param1, param2, true);
1010                         qcom_glink_rx_advance(glink, ALIGN(sizeof(msg), 8));
1011                         break;
1012                 case RPM_CMD_RX_INTENT_REQ_ACK:
1013                         qcom_glink_handle_intent_req_ack(glink, param1, param2);
1014                         qcom_glink_rx_advance(glink, ALIGN(sizeof(msg), 8));
1015                         break;
1016                 default:
1017                         dev_err(glink->dev, "unhandled rx cmd: %d\n", cmd);
1018                         ret = -EINVAL;
1019                         break;
1020                 }
1021
1022                 if (ret)
1023                         break;
1024         }
1025
1026         return IRQ_HANDLED;
1027 }
1028
1029 /* Locally initiated rpmsg_create_ept */
1030 static struct glink_channel *qcom_glink_create_local(struct qcom_glink *glink,
1031                                                      const char *name)
1032 {
1033         struct glink_channel *channel;
1034         int ret;
1035         unsigned long flags;
1036
1037         channel = qcom_glink_alloc_channel(glink, name);
1038         if (IS_ERR(channel))
1039                 return ERR_CAST(channel);
1040
1041         ret = qcom_glink_send_open_req(glink, channel);
1042         if (ret)
1043                 goto release_channel;
1044
1045         ret = wait_for_completion_timeout(&channel->open_ack, 5 * HZ);
1046         if (!ret)
1047                 goto err_timeout;
1048
1049         ret = wait_for_completion_timeout(&channel->open_req, 5 * HZ);
1050         if (!ret)
1051                 goto err_timeout;
1052
1053         qcom_glink_send_open_ack(glink, channel);
1054
1055         return channel;
1056
1057 err_timeout:
1058         /* qcom_glink_send_open_req() did register the channel in lcids*/
1059         spin_lock_irqsave(&glink->idr_lock, flags);
1060         idr_remove(&glink->lcids, channel->lcid);
1061         spin_unlock_irqrestore(&glink->idr_lock, flags);
1062
1063 release_channel:
1064         /* Release qcom_glink_send_open_req() reference */
1065         kref_put(&channel->refcount, qcom_glink_channel_release);
1066         /* Release qcom_glink_alloc_channel() reference */
1067         kref_put(&channel->refcount, qcom_glink_channel_release);
1068
1069         return ERR_PTR(-ETIMEDOUT);
1070 }
1071
1072 /* Remote initiated rpmsg_create_ept */
1073 static int qcom_glink_create_remote(struct qcom_glink *glink,
1074                                     struct glink_channel *channel)
1075 {
1076         int ret;
1077
1078         qcom_glink_send_open_ack(glink, channel);
1079
1080         ret = qcom_glink_send_open_req(glink, channel);
1081         if (ret)
1082                 goto close_link;
1083
1084         ret = wait_for_completion_timeout(&channel->open_ack, 5 * HZ);
1085         if (!ret) {
1086                 ret = -ETIMEDOUT;
1087                 goto close_link;
1088         }
1089
1090         return 0;
1091
1092 close_link:
1093         /*
1094          * Send a close request to "undo" our open-ack. The close-ack will
1095          * release the last reference.
1096          */
1097         qcom_glink_send_close_req(glink, channel);
1098
1099         /* Release qcom_glink_send_open_req() reference */
1100         kref_put(&channel->refcount, qcom_glink_channel_release);
1101
1102         return ret;
1103 }
1104
1105 static struct rpmsg_endpoint *qcom_glink_create_ept(struct rpmsg_device *rpdev,
1106                                                     rpmsg_rx_cb_t cb,
1107                                                     void *priv,
1108                                                     struct rpmsg_channel_info
1109                                                                         chinfo)
1110 {
1111         struct glink_channel *parent = to_glink_channel(rpdev->ept);
1112         struct glink_channel *channel;
1113         struct qcom_glink *glink = parent->glink;
1114         struct rpmsg_endpoint *ept;
1115         const char *name = chinfo.name;
1116         int cid;
1117         int ret;
1118         unsigned long flags;
1119
1120         spin_lock_irqsave(&glink->idr_lock, flags);
1121         idr_for_each_entry(&glink->rcids, channel, cid) {
1122                 if (!strcmp(channel->name, name))
1123                         break;
1124         }
1125         spin_unlock_irqrestore(&glink->idr_lock, flags);
1126
1127         if (!channel) {
1128                 channel = qcom_glink_create_local(glink, name);
1129                 if (IS_ERR(channel))
1130                         return NULL;
1131         } else {
1132                 ret = qcom_glink_create_remote(glink, channel);
1133                 if (ret)
1134                         return NULL;
1135         }
1136
1137         ept = &channel->ept;
1138         ept->rpdev = rpdev;
1139         ept->cb = cb;
1140         ept->priv = priv;
1141         ept->ops = &glink_endpoint_ops;
1142
1143         return ept;
1144 }
1145
1146 static int qcom_glink_announce_create(struct rpmsg_device *rpdev)
1147 {
1148         struct glink_channel *channel = to_glink_channel(rpdev->ept);
1149         struct device_node *np = rpdev->dev.of_node;
1150         struct qcom_glink *glink = channel->glink;
1151         struct glink_core_rx_intent *intent;
1152         const struct property *prop = NULL;
1153         __be32 defaults[] = { cpu_to_be32(SZ_1K), cpu_to_be32(5) };
1154         int num_intents;
1155         int num_groups = 1;
1156         __be32 *val = defaults;
1157         int size;
1158
1159         if (glink->intentless)
1160                 return 0;
1161
1162         prop = of_find_property(np, "qcom,intents", NULL);
1163         if (prop) {
1164                 val = prop->value;
1165                 num_groups = prop->length / sizeof(u32) / 2;
1166         }
1167
1168         /* Channel is now open, advertise base set of intents */
1169         while (num_groups--) {
1170                 size = be32_to_cpup(val++);
1171                 num_intents = be32_to_cpup(val++);
1172                 while (num_intents--) {
1173                         intent = qcom_glink_alloc_intent(glink, channel, size,
1174                                                          true);
1175                         if (!intent)
1176                                 break;
1177
1178                         qcom_glink_advertise_intent(glink, channel, intent);
1179                 }
1180         }
1181         return 0;
1182 }
1183
1184 static void qcom_glink_destroy_ept(struct rpmsg_endpoint *ept)
1185 {
1186         struct glink_channel *channel = to_glink_channel(ept);
1187         struct qcom_glink *glink = channel->glink;
1188         unsigned long flags;
1189
1190         spin_lock_irqsave(&channel->recv_lock, flags);
1191         channel->ept.cb = NULL;
1192         spin_unlock_irqrestore(&channel->recv_lock, flags);
1193
1194         /* Decouple the potential rpdev from the channel */
1195         channel->rpdev = NULL;
1196
1197         qcom_glink_send_close_req(glink, channel);
1198 }
1199
1200 static int qcom_glink_request_intent(struct qcom_glink *glink,
1201                                      struct glink_channel *channel,
1202                                      size_t size)
1203 {
1204         struct {
1205                 u16 id;
1206                 u16 cid;
1207                 u32 size;
1208         } __packed cmd;
1209
1210         int ret;
1211
1212         mutex_lock(&channel->intent_req_lock);
1213
1214         reinit_completion(&channel->intent_req_comp);
1215
1216         cmd.id = RPM_CMD_RX_INTENT_REQ;
1217         cmd.cid = channel->lcid;
1218         cmd.size = size;
1219
1220         ret = qcom_glink_tx(glink, &cmd, sizeof(cmd), NULL, 0, true);
1221         if (ret)
1222                 goto unlock;
1223
1224         ret = wait_for_completion_timeout(&channel->intent_req_comp, 10 * HZ);
1225         if (!ret) {
1226                 dev_err(glink->dev, "intent request timed out\n");
1227                 ret = -ETIMEDOUT;
1228         } else {
1229                 ret = channel->intent_req_result ? 0 : -ECANCELED;
1230         }
1231
1232 unlock:
1233         mutex_unlock(&channel->intent_req_lock);
1234         return ret;
1235 }
1236
1237 static int __qcom_glink_send(struct glink_channel *channel,
1238                              void *data, int len, bool wait)
1239 {
1240         struct qcom_glink *glink = channel->glink;
1241         struct glink_core_rx_intent *intent = NULL;
1242         struct glink_core_rx_intent *tmp;
1243         int iid = 0;
1244         struct {
1245                 struct glink_msg msg;
1246                 __le32 chunk_size;
1247                 __le32 left_size;
1248         } __packed req;
1249         int ret;
1250         unsigned long flags;
1251
1252         if (!glink->intentless) {
1253                 while (!intent) {
1254                         spin_lock_irqsave(&channel->intent_lock, flags);
1255                         idr_for_each_entry(&channel->riids, tmp, iid) {
1256                                 if (tmp->size >= len && !tmp->in_use) {
1257                                         if (!intent)
1258                                                 intent = tmp;
1259                                         else if (intent->size > tmp->size)
1260                                                 intent = tmp;
1261                                         if (intent->size == len)
1262                                                 break;
1263                                 }
1264                         }
1265                         if (intent)
1266                                 intent->in_use = true;
1267                         spin_unlock_irqrestore(&channel->intent_lock, flags);
1268
1269                         /* We found an available intent */
1270                         if (intent)
1271                                 break;
1272
1273                         if (!wait)
1274                                 return -EBUSY;
1275
1276                         ret = qcom_glink_request_intent(glink, channel, len);
1277                         if (ret < 0)
1278                                 return ret;
1279                 }
1280
1281                 iid = intent->id;
1282         }
1283
1284         req.msg.cmd = cpu_to_le16(RPM_CMD_TX_DATA);
1285         req.msg.param1 = cpu_to_le16(channel->lcid);
1286         req.msg.param2 = cpu_to_le32(iid);
1287         req.chunk_size = cpu_to_le32(len);
1288         req.left_size = cpu_to_le32(0);
1289
1290         ret = qcom_glink_tx(glink, &req, sizeof(req), data, len, wait);
1291
1292         /* Mark intent available if we failed */
1293         if (ret && intent)
1294                 intent->in_use = false;
1295
1296         return ret;
1297 }
1298
1299 static int qcom_glink_send(struct rpmsg_endpoint *ept, void *data, int len)
1300 {
1301         struct glink_channel *channel = to_glink_channel(ept);
1302
1303         return __qcom_glink_send(channel, data, len, true);
1304 }
1305
1306 static int qcom_glink_trysend(struct rpmsg_endpoint *ept, void *data, int len)
1307 {
1308         struct glink_channel *channel = to_glink_channel(ept);
1309
1310         return __qcom_glink_send(channel, data, len, false);
1311 }
1312
1313 /*
1314  * Finds the device_node for the glink child interested in this channel.
1315  */
1316 static struct device_node *qcom_glink_match_channel(struct device_node *node,
1317                                                     const char *channel)
1318 {
1319         struct device_node *child;
1320         const char *name;
1321         const char *key;
1322         int ret;
1323
1324         for_each_available_child_of_node(node, child) {
1325                 key = "qcom,glink-channels";
1326                 ret = of_property_read_string(child, key, &name);
1327                 if (ret)
1328                         continue;
1329
1330                 if (strcmp(name, channel) == 0)
1331                         return child;
1332         }
1333
1334         return NULL;
1335 }
1336
1337 static const struct rpmsg_device_ops glink_device_ops = {
1338         .create_ept = qcom_glink_create_ept,
1339         .announce_create = qcom_glink_announce_create,
1340 };
1341
1342 static const struct rpmsg_endpoint_ops glink_endpoint_ops = {
1343         .destroy_ept = qcom_glink_destroy_ept,
1344         .send = qcom_glink_send,
1345         .trysend = qcom_glink_trysend,
1346 };
1347
1348 static void qcom_glink_rpdev_release(struct device *dev)
1349 {
1350         struct rpmsg_device *rpdev = to_rpmsg_device(dev);
1351         struct glink_channel *channel = to_glink_channel(rpdev->ept);
1352
1353         channel->rpdev = NULL;
1354         kfree(rpdev);
1355 }
1356
1357 static int qcom_glink_rx_open(struct qcom_glink *glink, unsigned int rcid,
1358                               char *name)
1359 {
1360         struct glink_channel *channel;
1361         struct rpmsg_device *rpdev;
1362         bool create_device = false;
1363         struct device_node *node;
1364         int lcid;
1365         int ret;
1366         unsigned long flags;
1367
1368         spin_lock_irqsave(&glink->idr_lock, flags);
1369         idr_for_each_entry(&glink->lcids, channel, lcid) {
1370                 if (!strcmp(channel->name, name))
1371                         break;
1372         }
1373         spin_unlock_irqrestore(&glink->idr_lock, flags);
1374
1375         if (!channel) {
1376                 channel = qcom_glink_alloc_channel(glink, name);
1377                 if (IS_ERR(channel))
1378                         return PTR_ERR(channel);
1379
1380                 /* The opening dance was initiated by the remote */
1381                 create_device = true;
1382         }
1383
1384         spin_lock_irqsave(&glink->idr_lock, flags);
1385         ret = idr_alloc(&glink->rcids, channel, rcid, rcid + 1, GFP_ATOMIC);
1386         if (ret < 0) {
1387                 dev_err(glink->dev, "Unable to insert channel into rcid list\n");
1388                 spin_unlock_irqrestore(&glink->idr_lock, flags);
1389                 goto free_channel;
1390         }
1391         channel->rcid = ret;
1392         spin_unlock_irqrestore(&glink->idr_lock, flags);
1393
1394         complete(&channel->open_req);
1395
1396         if (create_device) {
1397                 rpdev = kzalloc(sizeof(*rpdev), GFP_KERNEL);
1398                 if (!rpdev) {
1399                         ret = -ENOMEM;
1400                         goto rcid_remove;
1401                 }
1402
1403                 rpdev->ept = &channel->ept;
1404                 strncpy(rpdev->id.name, name, RPMSG_NAME_SIZE);
1405                 rpdev->src = RPMSG_ADDR_ANY;
1406                 rpdev->dst = RPMSG_ADDR_ANY;
1407                 rpdev->ops = &glink_device_ops;
1408
1409                 node = qcom_glink_match_channel(glink->dev->of_node, name);
1410                 rpdev->dev.of_node = node;
1411                 rpdev->dev.parent = glink->dev;
1412                 rpdev->dev.release = qcom_glink_rpdev_release;
1413
1414                 ret = rpmsg_register_device(rpdev);
1415                 if (ret)
1416                         goto free_rpdev;
1417
1418                 channel->rpdev = rpdev;
1419         }
1420
1421         return 0;
1422
1423 free_rpdev:
1424         kfree(rpdev);
1425 rcid_remove:
1426         spin_lock_irqsave(&glink->idr_lock, flags);
1427         idr_remove(&glink->rcids, channel->rcid);
1428         channel->rcid = 0;
1429         spin_unlock_irqrestore(&glink->idr_lock, flags);
1430 free_channel:
1431         /* Release the reference, iff we took it */
1432         if (create_device)
1433                 kref_put(&channel->refcount, qcom_glink_channel_release);
1434
1435         return ret;
1436 }
1437
1438 static void qcom_glink_rx_close(struct qcom_glink *glink, unsigned int rcid)
1439 {
1440         struct rpmsg_channel_info chinfo;
1441         struct glink_channel *channel;
1442         unsigned long flags;
1443
1444         spin_lock_irqsave(&glink->idr_lock, flags);
1445         channel = idr_find(&glink->rcids, rcid);
1446         spin_unlock_irqrestore(&glink->idr_lock, flags);
1447         if (WARN(!channel, "close request on unknown channel\n"))
1448                 return;
1449
1450         /* cancel pending rx_done work */
1451         cancel_work_sync(&channel->intent_work);
1452
1453         if (channel->rpdev) {
1454                 strncpy(chinfo.name, channel->name, sizeof(chinfo.name));
1455                 chinfo.src = RPMSG_ADDR_ANY;
1456                 chinfo.dst = RPMSG_ADDR_ANY;
1457
1458                 rpmsg_unregister_device(glink->dev, &chinfo);
1459         }
1460
1461         qcom_glink_send_close_ack(glink, channel->rcid);
1462
1463         spin_lock_irqsave(&glink->idr_lock, flags);
1464         idr_remove(&glink->rcids, channel->rcid);
1465         channel->rcid = 0;
1466         spin_unlock_irqrestore(&glink->idr_lock, flags);
1467
1468         kref_put(&channel->refcount, qcom_glink_channel_release);
1469 }
1470
1471 static void qcom_glink_rx_close_ack(struct qcom_glink *glink, unsigned int lcid)
1472 {
1473         struct glink_channel *channel;
1474         unsigned long flags;
1475
1476         spin_lock_irqsave(&glink->idr_lock, flags);
1477         channel = idr_find(&glink->lcids, lcid);
1478         if (WARN(!channel, "close ack on unknown channel\n")) {
1479                 spin_unlock_irqrestore(&glink->idr_lock, flags);
1480                 return;
1481         }
1482
1483         idr_remove(&glink->lcids, channel->lcid);
1484         channel->lcid = 0;
1485         spin_unlock_irqrestore(&glink->idr_lock, flags);
1486
1487         kref_put(&channel->refcount, qcom_glink_channel_release);
1488 }
1489
1490 static void qcom_glink_work(struct work_struct *work)
1491 {
1492         struct qcom_glink *glink = container_of(work, struct qcom_glink,
1493                                                 rx_work);
1494         struct glink_defer_cmd *dcmd;
1495         struct glink_msg *msg;
1496         unsigned long flags;
1497         unsigned int param1;
1498         unsigned int param2;
1499         unsigned int cmd;
1500
1501         for (;;) {
1502                 spin_lock_irqsave(&glink->rx_lock, flags);
1503                 if (list_empty(&glink->rx_queue)) {
1504                         spin_unlock_irqrestore(&glink->rx_lock, flags);
1505                         break;
1506                 }
1507                 dcmd = list_first_entry(&glink->rx_queue,
1508                                         struct glink_defer_cmd, node);
1509                 list_del(&dcmd->node);
1510                 spin_unlock_irqrestore(&glink->rx_lock, flags);
1511
1512                 msg = &dcmd->msg;
1513                 cmd = le16_to_cpu(msg->cmd);
1514                 param1 = le16_to_cpu(msg->param1);
1515                 param2 = le32_to_cpu(msg->param2);
1516
1517                 switch (cmd) {
1518                 case RPM_CMD_VERSION:
1519                         qcom_glink_receive_version(glink, param1, param2);
1520                         break;
1521                 case RPM_CMD_VERSION_ACK:
1522                         qcom_glink_receive_version_ack(glink, param1, param2);
1523                         break;
1524                 case RPM_CMD_OPEN:
1525                         qcom_glink_rx_open(glink, param1, msg->data);
1526                         break;
1527                 case RPM_CMD_CLOSE:
1528                         qcom_glink_rx_close(glink, param1);
1529                         break;
1530                 case RPM_CMD_CLOSE_ACK:
1531                         qcom_glink_rx_close_ack(glink, param1);
1532                         break;
1533                 case RPM_CMD_RX_INTENT_REQ:
1534                         qcom_glink_handle_intent_req(glink, param1, param2);
1535                         break;
1536                 default:
1537                         WARN(1, "Unknown defer object %d\n", cmd);
1538                         break;
1539                 }
1540
1541                 kfree(dcmd);
1542         }
1543 }
1544
1545 struct qcom_glink *qcom_glink_native_probe(struct device *dev,
1546                                            unsigned long features,
1547                                            struct qcom_glink_pipe *rx,
1548                                            struct qcom_glink_pipe *tx,
1549                                            bool intentless)
1550 {
1551         int irq;
1552         int ret;
1553         struct qcom_glink *glink;
1554
1555         glink = devm_kzalloc(dev, sizeof(*glink), GFP_KERNEL);
1556         if (!glink)
1557                 return ERR_PTR(-ENOMEM);
1558
1559         glink->dev = dev;
1560         glink->tx_pipe = tx;
1561         glink->rx_pipe = rx;
1562
1563         glink->features = features;
1564         glink->intentless = intentless;
1565
1566         spin_lock_init(&glink->tx_lock);
1567         spin_lock_init(&glink->rx_lock);
1568         INIT_LIST_HEAD(&glink->rx_queue);
1569         INIT_WORK(&glink->rx_work, qcom_glink_work);
1570
1571         spin_lock_init(&glink->idr_lock);
1572         idr_init(&glink->lcids);
1573         idr_init(&glink->rcids);
1574
1575         glink->mbox_client.dev = dev;
1576         glink->mbox_client.knows_txdone = true;
1577         glink->mbox_chan = mbox_request_channel(&glink->mbox_client, 0);
1578         if (IS_ERR(glink->mbox_chan)) {
1579                 if (PTR_ERR(glink->mbox_chan) != -EPROBE_DEFER)
1580                         dev_err(dev, "failed to acquire IPC channel\n");
1581                 return ERR_CAST(glink->mbox_chan);
1582         }
1583
1584         irq = of_irq_get(dev->of_node, 0);
1585         ret = devm_request_irq(dev, irq,
1586                                qcom_glink_native_intr,
1587                                IRQF_NO_SUSPEND | IRQF_SHARED,
1588                                "glink-native", glink);
1589         if (ret) {
1590                 dev_err(dev, "failed to request IRQ\n");
1591                 return ERR_PTR(ret);
1592         }
1593
1594         glink->irq = irq;
1595
1596         ret = qcom_glink_send_version(glink);
1597         if (ret)
1598                 return ERR_PTR(ret);
1599
1600         return glink;
1601 }
1602 EXPORT_SYMBOL_GPL(qcom_glink_native_probe);
1603
1604 static int qcom_glink_remove_device(struct device *dev, void *data)
1605 {
1606         device_unregister(dev);
1607
1608         return 0;
1609 }
1610
1611 void qcom_glink_native_remove(struct qcom_glink *glink)
1612 {
1613         struct glink_channel *channel;
1614         int cid;
1615         int ret;
1616         unsigned long flags;
1617
1618         disable_irq(glink->irq);
1619         cancel_work_sync(&glink->rx_work);
1620
1621         ret = device_for_each_child(glink->dev, NULL, qcom_glink_remove_device);
1622         if (ret)
1623                 dev_warn(glink->dev, "Can't remove GLINK devices: %d\n", ret);
1624
1625         spin_lock_irqsave(&glink->idr_lock, flags);
1626         /* Release any defunct local channels, waiting for close-ack */
1627         idr_for_each_entry(&glink->lcids, channel, cid)
1628                 kref_put(&channel->refcount, qcom_glink_channel_release);
1629
1630         idr_destroy(&glink->lcids);
1631         idr_destroy(&glink->rcids);
1632         spin_unlock_irqrestore(&glink->idr_lock, flags);
1633         mbox_free_channel(glink->mbox_chan);
1634 }
1635 EXPORT_SYMBOL_GPL(qcom_glink_native_remove);
1636
1637 void qcom_glink_native_unregister(struct qcom_glink *glink)
1638 {
1639         device_unregister(glink->dev);
1640 }
1641 EXPORT_SYMBOL_GPL(qcom_glink_native_unregister);
1642
1643 MODULE_DESCRIPTION("Qualcomm GLINK driver");
1644 MODULE_LICENSE("GPL v2");