drivers/hv/channel.c

   1 /*
   2  * Copyright (c) 2009, Microsoft Corporation.
   3  *
   4  * This program is free software; you can redistribute it and/or modify it
   5  * under the terms and conditions of the GNU General Public License,
   6  * version 2, as published by the Free Software Foundation.
   7  *
   8  * This program is distributed in the hope it will be useful, but WITHOUT
   9  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  10  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  11  * more details.
  12  *
  13  * You should have received a copy of the GNU General Public License along with
  14  * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
  15  * Place - Suite 330, Boston, MA 02111-1307 USA.
  16  *
  17  * Authors:
  18  *   Haiyang Zhang <haiyangz@microsoft.com>
  19  *   Hank Janssen  <hjanssen@microsoft.com>
  20  */
  21 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  22
  23 #include <linux/kernel.h>
  24 #include <linux/sched.h>
  25 #include <linux/wait.h>
  26 #include <linux/mm.h>
  27 #include <linux/slab.h>
  28 #include <linux/module.h>
  29 #include <linux/hyperv.h>
  30 #include <linux/uio.h>
  31 #include <linux/interrupt.h>
  32 #include <asm/page.h>
  33
  34 #include "hyperv_vmbus.h"
  35
  36 #define NUM_PAGES_SPANNED(addr, len) \
  37 ((PAGE_ALIGN(addr + len) >> PAGE_SHIFT) - (addr >> PAGE_SHIFT))
  38
  39 static unsigned long virt_to_hvpfn(void *addr)
  40 {
  41         unsigned long paddr;
  42
  43         if (is_vmalloc_addr(addr))
  44                 paddr = page_to_phys(vmalloc_to_page(addr)) +
  45                                          offset_in_page(addr);
  46         else
  47                 paddr = __pa(addr);
  48
  49         return  paddr >> PAGE_SHIFT;
  50 }
  51
  52 /*
  53  * vmbus_setevent- Trigger an event notification on the specified
  54  * channel.
  55  */
  56 void vmbus_setevent(struct vmbus_channel *channel)
  57 {
  58         struct hv_monitor_page *monitorpage;
  59
  60         trace_vmbus_setevent(channel);
  61
  62         /*
  63          * For channels marked as in "low latency" mode
  64          * bypass the monitor page mechanism.
  65          */
  66         if (channel->offermsg.monitor_allocated && !channel->low_latency) {
  67                 vmbus_send_interrupt(channel->offermsg.child_relid);
  68
  69                 /* Get the child to parent monitor page */
  70                 monitorpage = vmbus_connection.monitor_pages[1];
  71
  72                 sync_set_bit(channel->monitor_bit,
  73                         (unsigned long *)&monitorpage->trigger_group
  74                                         [channel->monitor_grp].pending);
  75
  76         } else {
  77                 vmbus_set_event(channel);
  78         }
  79 }
  80 EXPORT_SYMBOL_GPL(vmbus_setevent);
  81
  82 /* vmbus_free_ring - drop mapping of ring buffer */
  83 void vmbus_free_ring(struct vmbus_channel *channel)
  84 {
  85         hv_ringbuffer_cleanup(&channel->outbound);
  86         hv_ringbuffer_cleanup(&channel->inbound);
  87
  88         if (channel->ringbuffer_page) {
  89                 __free_pages(channel->ringbuffer_page,
  90                              get_order(channel->ringbuffer_pagecount
  91                                        << PAGE_SHIFT));
  92                 channel->ringbuffer_page = NULL;
  93         }
  94 }
  95 EXPORT_SYMBOL_GPL(vmbus_free_ring);
  96
  97 /* vmbus_alloc_ring - allocate and map pages for ring buffer */
  98 int vmbus_alloc_ring(struct vmbus_channel *newchannel,
  99                      u32 send_size, u32 recv_size)
 100 {
 101         struct page *page;
 102         int order;
 103
 104         if (send_size % PAGE_SIZE || recv_size % PAGE_SIZE)
 105                 return -EINVAL;
 106
 107         /* Allocate the ring buffer */
 108         order = get_order(send_size + recv_size);
 109         page = alloc_pages_node(cpu_to_node(newchannel->target_cpu),
 110                                 GFP_KERNEL|__GFP_ZERO, order);
 111
 112         if (!page)
 113                 page = alloc_pages(GFP_KERNEL|__GFP_ZERO, order);
 114
 115         if (!page)
 116                 return -ENOMEM;
 117
 118         newchannel->ringbuffer_page = page;
 119         newchannel->ringbuffer_pagecount = (send_size + recv_size) >> PAGE_SHIFT;
 120         newchannel->ringbuffer_send_offset = send_size >> PAGE_SHIFT;
 121
 122         return 0;
 123 }
 124 EXPORT_SYMBOL_GPL(vmbus_alloc_ring);
 125
 126 static int __vmbus_open(struct vmbus_channel *newchannel,
 127                        void *userdata, u32 userdatalen,
 128                        void (*onchannelcallback)(void *context), void *context)
 129 {
 130         struct vmbus_channel_open_channel *open_msg;
 131         struct vmbus_channel_msginfo *open_info = NULL;
 132         struct page *page = newchannel->ringbuffer_page;
 133         u32 send_pages, recv_pages;
 134         unsigned long flags;
 135         int err;
 136
 137         if (userdatalen > MAX_USER_DEFINED_BYTES)
 138                 return -EINVAL;
 139
 140         send_pages = newchannel->ringbuffer_send_offset;
 141         recv_pages = newchannel->ringbuffer_pagecount - send_pages;
 142
 143         spin_lock_irqsave(&newchannel->lock, flags);
 144         if (newchannel->state != CHANNEL_OPEN_STATE) {
 145                 spin_unlock_irqrestore(&newchannel->lock, flags);
 146                 return -EINVAL;
 147         }
 148         spin_unlock_irqrestore(&newchannel->lock, flags);
 149
 150         newchannel->state = CHANNEL_OPENING_STATE;
 151         newchannel->onchannel_callback = onchannelcallback;
 152         newchannel->channel_callback_context = context;
 153
 154         err = hv_ringbuffer_init(&newchannel->outbound, page, send_pages);
 155         if (err)
 156                 goto error_clean_ring;
 157
 158         err = hv_ringbuffer_init(&newchannel->inbound,
 159                                  &page[send_pages], recv_pages);
 160         if (err)
 161                 goto error_clean_ring;
 162
 163         /* Establish the gpadl for the ring buffer */
 164         newchannel->ringbuffer_gpadlhandle = 0;
 165
 166         err = vmbus_establish_gpadl(newchannel,
 167                                     page_address(newchannel->ringbuffer_page),
 168                                     (send_pages + recv_pages) << PAGE_SHIFT,
 169                                     &newchannel->ringbuffer_gpadlhandle);
 170         if (err)
 171                 goto error_clean_ring;
 172
 173         /* Create and init the channel open message */
 174         open_info = kmalloc(sizeof(*open_info) +
 175                            sizeof(struct vmbus_channel_open_channel),
 176                            GFP_KERNEL);
 177         if (!open_info) {
 178                 err = -ENOMEM;
 179                 goto error_free_gpadl;
 180         }
 181
 182         init_completion(&open_info->waitevent);
 183         open_info->waiting_channel = newchannel;
 184
 185         open_msg = (struct vmbus_channel_open_channel *)open_info->msg;
 186         open_msg->header.msgtype = CHANNELMSG_OPENCHANNEL;
 187         open_msg->openid = newchannel->offermsg.child_relid;
 188         open_msg->child_relid = newchannel->offermsg.child_relid;
 189         open_msg->ringbuffer_gpadlhandle = newchannel->ringbuffer_gpadlhandle;
 190         open_msg->downstream_ringbuffer_pageoffset = newchannel->ringbuffer_send_offset;
 191         open_msg->target_vp = newchannel->target_vp;
 192
 193         if (userdatalen)
 194                 memcpy(open_msg->userdata, userdata, userdatalen);
 195
 196         spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags);
 197         list_add_tail(&open_info->msglistentry,
 198                       &vmbus_connection.chn_msg_list);
 199         spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags);
 200
 201         if (newchannel->rescind) {
 202                 err = -ENODEV;
 203                 goto error_free_info;
 204         }
 205
 206         err = vmbus_post_msg(open_msg,
 207                              sizeof(struct vmbus_channel_open_channel), true);
 208
 209         trace_vmbus_open(open_msg, err);
 210
 211         if (err != 0)
 212                 goto error_clean_msglist;
 213
 214         wait_for_completion(&open_info->waitevent);
 215
 216         spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags);
 217         list_del(&open_info->msglistentry);
 218         spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags);
 219
 220         if (newchannel->rescind) {
 221                 err = -ENODEV;
 222                 goto error_free_info;
 223         }
 224
 225         if (open_info->response.open_result.status) {
 226                 err = -EAGAIN;
 227                 goto error_free_info;
 228         }
 229
 230         newchannel->state = CHANNEL_OPENED_STATE;
 231         kfree(open_info);
 232         return 0;
 233
 234 error_clean_msglist:
 235         spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags);
 236         list_del(&open_info->msglistentry);
 237         spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags);
 238 error_free_info:
 239         kfree(open_info);
 240 error_free_gpadl:
 241         vmbus_teardown_gpadl(newchannel, newchannel->ringbuffer_gpadlhandle);
 242         newchannel->ringbuffer_gpadlhandle = 0;
 243 error_clean_ring:
 244         hv_ringbuffer_cleanup(&newchannel->outbound);
 245         hv_ringbuffer_cleanup(&newchannel->inbound);
 246         newchannel->state = CHANNEL_OPEN_STATE;
 247         return err;
 248 }
 249
 250 /*
 251  * vmbus_connect_ring - Open the channel but reuse ring buffer
 252  */
 253 int vmbus_connect_ring(struct vmbus_channel *newchannel,
 254                        void (*onchannelcallback)(void *context), void *context)
 255 {
 256         return  __vmbus_open(newchannel, NULL, 0, onchannelcallback, context);
 257 }
 258 EXPORT_SYMBOL_GPL(vmbus_connect_ring);
 259
 260 /*
 261  * vmbus_open - Open the specified channel.
 262  */
 263 int vmbus_open(struct vmbus_channel *newchannel,
 264                u32 send_ringbuffer_size, u32 recv_ringbuffer_size,
 265                void *userdata, u32 userdatalen,
 266                void (*onchannelcallback)(void *context), void *context)
 267 {
 268         int err;
 269
 270         err = vmbus_alloc_ring(newchannel, send_ringbuffer_size,
 271                                recv_ringbuffer_size);
 272         if (err)
 273                 return err;
 274
 275         err = __vmbus_open(newchannel, userdata, userdatalen,
 276                            onchannelcallback, context);
 277         if (err)
 278                 vmbus_free_ring(newchannel);
 279
 280         return err;
 281 }
 282 EXPORT_SYMBOL_GPL(vmbus_open);
 283
 284 /* Used for Hyper-V Socket: a guest client's connect() to the host */
 285 int vmbus_send_tl_connect_request(const uuid_le *shv_guest_servie_id,
 286                                   const uuid_le *shv_host_servie_id)
 287 {
 288         struct vmbus_channel_tl_connect_request conn_msg;
 289         int ret;
 290
 291         memset(&conn_msg, 0, sizeof(conn_msg));
 292         conn_msg.header.msgtype = CHANNELMSG_TL_CONNECT_REQUEST;
 293         conn_msg.guest_endpoint_id = *shv_guest_servie_id;
 294         conn_msg.host_service_id = *shv_host_servie_id;
 295
 296         ret = vmbus_post_msg(&conn_msg, sizeof(conn_msg), true);
 297
 298         trace_vmbus_send_tl_connect_request(&conn_msg, ret);
 299
 300         return ret;
 301 }
 302 EXPORT_SYMBOL_GPL(vmbus_send_tl_connect_request);
 303
 304 /*
 305  * create_gpadl_header - Creates a gpadl for the specified buffer
 306  */
 307 static int create_gpadl_header(void *kbuffer, u32 size,
 308                                struct vmbus_channel_msginfo **msginfo)
 309 {
 310         int i;
 311         int pagecount;
 312         struct vmbus_channel_gpadl_header *gpadl_header;
 313         struct vmbus_channel_gpadl_body *gpadl_body;
 314         struct vmbus_channel_msginfo *msgheader;
 315         struct vmbus_channel_msginfo *msgbody = NULL;
 316         u32 msgsize;
 317
 318         int pfnsum, pfncount, pfnleft, pfncurr, pfnsize;
 319
 320         pagecount = size >> PAGE_SHIFT;
 321
 322         /* do we need a gpadl body msg */
 323         pfnsize = MAX_SIZE_CHANNEL_MESSAGE -
 324                   sizeof(struct vmbus_channel_gpadl_header) -
 325                   sizeof(struct gpa_range);
 326         pfncount = pfnsize / sizeof(u64);
 327
 328         if (pagecount > pfncount) {
 329                 /* we need a gpadl body */
 330                 /* fill in the header */
 331                 msgsize = sizeof(struct vmbus_channel_msginfo) +
 332                           sizeof(struct vmbus_channel_gpadl_header) +
 333                           sizeof(struct gpa_range) + pfncount * sizeof(u64);
 334                 msgheader =  kzalloc(msgsize, GFP_KERNEL);
 335                 if (!msgheader)
 336                         goto nomem;
 337
 338                 INIT_LIST_HEAD(&msgheader->submsglist);
 339                 msgheader->msgsize = msgsize;
 340
 341                 gpadl_header = (struct vmbus_channel_gpadl_header *)
 342                         msgheader->msg;
 343                 gpadl_header->rangecount = 1;
 344                 gpadl_header->range_buflen = sizeof(struct gpa_range) +
 345                                          pagecount * sizeof(u64);
 346                 gpadl_header->range[0].byte_offset = 0;
 347                 gpadl_header->range[0].byte_count = size;
 348                 for (i = 0; i < pfncount; i++)
 349                         gpadl_header->range[0].pfn_array[i] = virt_to_hvpfn(
 350                                 kbuffer + PAGE_SIZE * i);
 351                 *msginfo = msgheader;
 352
 353                 pfnsum = pfncount;
 354                 pfnleft = pagecount - pfncount;
 355
 356                 /* how many pfns can we fit */
 357                 pfnsize = MAX_SIZE_CHANNEL_MESSAGE -
 358                           sizeof(struct vmbus_channel_gpadl_body);
 359                 pfncount = pfnsize / sizeof(u64);
 360
 361                 /* fill in the body */
 362                 while (pfnleft) {
 363                         if (pfnleft > pfncount)
 364                                 pfncurr = pfncount;
 365                         else
 366                                 pfncurr = pfnleft;
 367
 368                         msgsize = sizeof(struct vmbus_channel_msginfo) +
 369                                   sizeof(struct vmbus_channel_gpadl_body) +
 370                                   pfncurr * sizeof(u64);
 371                         msgbody = kzalloc(msgsize, GFP_KERNEL);
 372
 373                         if (!msgbody) {
 374                                 struct vmbus_channel_msginfo *pos = NULL;
 375                                 struct vmbus_channel_msginfo *tmp = NULL;
 376                                 /*
 377                                  * Free up all the allocated messages.
 378                                  */
 379                                 list_for_each_entry_safe(pos, tmp,
 380                                         &msgheader->submsglist,
 381                                         msglistentry) {
 382
 383                                         list_del(&pos->msglistentry);
 384                                         kfree(pos);
 385                                 }
 386
 387                                 goto nomem;
 388                         }
 389
 390                         msgbody->msgsize = msgsize;
 391                         gpadl_body =
 392                                 (struct vmbus_channel_gpadl_body *)msgbody->msg;
 393
 394                         /*
 395                          * Gpadl is u32 and we are using a pointer which could
 396                          * be 64-bit
 397                          * This is governed by the guest/host protocol and
 398                          * so the hypervisor guarantees that this is ok.
 399                          */
 400                         for (i = 0; i < pfncurr; i++)
 401                                 gpadl_body->pfn[i] = virt_to_hvpfn(
 402                                         kbuffer + PAGE_SIZE * (pfnsum + i));
 403
 404                         /* add to msg header */
 405                         list_add_tail(&msgbody->msglistentry,
 406                                       &msgheader->submsglist);
 407                         pfnsum += pfncurr;
 408                         pfnleft -= pfncurr;
 409                 }
 410         } else {
 411                 /* everything fits in a header */
 412                 msgsize = sizeof(struct vmbus_channel_msginfo) +
 413                           sizeof(struct vmbus_channel_gpadl_header) +
 414                           sizeof(struct gpa_range) + pagecount * sizeof(u64);
 415                 msgheader = kzalloc(msgsize, GFP_KERNEL);
 416                 if (msgheader == NULL)
 417                         goto nomem;
 418
 419                 INIT_LIST_HEAD(&msgheader->submsglist);
 420                 msgheader->msgsize = msgsize;
 421
 422                 gpadl_header = (struct vmbus_channel_gpadl_header *)
 423                         msgheader->msg;
 424                 gpadl_header->rangecount = 1;
 425                 gpadl_header->range_buflen = sizeof(struct gpa_range) +
 426                                          pagecount * sizeof(u64);
 427                 gpadl_header->range[0].byte_offset = 0;
 428                 gpadl_header->range[0].byte_count = size;
 429                 for (i = 0; i < pagecount; i++)
 430                         gpadl_header->range[0].pfn_array[i] = virt_to_hvpfn(
 431                                 kbuffer + PAGE_SIZE * i);
 432
 433                 *msginfo = msgheader;
 434         }
 435
 436         return 0;
 437 nomem:
 438         kfree(msgheader);
 439         kfree(msgbody);
 440         return -ENOMEM;
 441 }
 442
 443 /*
 444  * vmbus_establish_gpadl - Establish a GPADL for the specified buffer
 445  *
 446  * @channel: a channel
 447  * @kbuffer: from kmalloc or vmalloc
 448  * @size: page-size multiple
 449  * @gpadl_handle: some funky thing
 450  */
 451 int vmbus_establish_gpadl(struct vmbus_channel *channel, void *kbuffer,
 452                                u32 size, u32 *gpadl_handle)
 453 {
 454         struct vmbus_channel_gpadl_header *gpadlmsg;
 455         struct vmbus_channel_gpadl_body *gpadl_body;
 456         struct vmbus_channel_msginfo *msginfo = NULL;
 457         struct vmbus_channel_msginfo *submsginfo, *tmp;
 458         struct list_head *curr;
 459         u32 next_gpadl_handle;
 460         unsigned long flags;
 461         int ret = 0;
 462
 463         next_gpadl_handle =
 464                 (atomic_inc_return(&vmbus_connection.next_gpadl_handle) - 1);
 465
 466         ret = create_gpadl_header(kbuffer, size, &msginfo);
 467         if (ret)
 468                 return ret;
 469
 470         init_completion(&msginfo->waitevent);
 471         msginfo->waiting_channel = channel;
 472
 473         gpadlmsg = (struct vmbus_channel_gpadl_header *)msginfo->msg;
 474         gpadlmsg->header.msgtype = CHANNELMSG_GPADL_HEADER;
 475         gpadlmsg->child_relid = channel->offermsg.child_relid;
 476         gpadlmsg->gpadl = next_gpadl_handle;
 477
 478
 479         spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags);
 480         list_add_tail(&msginfo->msglistentry,
 481                       &vmbus_connection.chn_msg_list);
 482
 483         spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags);
 484
 485         if (channel->rescind) {
 486                 ret = -ENODEV;
 487                 goto cleanup;
 488         }
 489
 490         ret = vmbus_post_msg(gpadlmsg, msginfo->msgsize -
 491                              sizeof(*msginfo), true);
 492
 493         trace_vmbus_establish_gpadl_header(gpadlmsg, ret);
 494
 495         if (ret != 0)
 496                 goto cleanup;
 497
 498         list_for_each(curr, &msginfo->submsglist) {
 499                 submsginfo = (struct vmbus_channel_msginfo *)curr;
 500                 gpadl_body =
 501                         (struct vmbus_channel_gpadl_body *)submsginfo->msg;
 502
 503                 gpadl_body->header.msgtype =
 504                         CHANNELMSG_GPADL_BODY;
 505                 gpadl_body->gpadl = next_gpadl_handle;
 506
 507                 ret = vmbus_post_msg(gpadl_body,
 508                                      submsginfo->msgsize - sizeof(*submsginfo),
 509                                      true);
 510
 511                 trace_vmbus_establish_gpadl_body(gpadl_body, ret);
 512
 513                 if (ret != 0)
 514                         goto cleanup;
 515
 516         }
 517         wait_for_completion(&msginfo->waitevent);
 518
 519         if (msginfo->response.gpadl_created.creation_status != 0) {
 520                 pr_err("Failed to establish GPADL: err = 0x%x\n",
 521                        msginfo->response.gpadl_created.creation_status);
 522
 523                 ret = -EDQUOT;
 524                 goto cleanup;
 525         }
 526
 527         if (channel->rescind) {
 528                 ret = -ENODEV;
 529                 goto cleanup;
 530         }
 531
 532         /* At this point, we received the gpadl created msg */
 533         *gpadl_handle = gpadlmsg->gpadl;
 534
 535 cleanup:
 536         spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags);
 537         list_del(&msginfo->msglistentry);
 538         spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags);
 539         list_for_each_entry_safe(submsginfo, tmp, &msginfo->submsglist,
 540                                  msglistentry) {
 541                 kfree(submsginfo);
 542         }
 543
 544         kfree(msginfo);
 545         return ret;
 546 }
 547 EXPORT_SYMBOL_GPL(vmbus_establish_gpadl);
 548
 549 /*
 550  * vmbus_teardown_gpadl -Teardown the specified GPADL handle
 551  */
 552 int vmbus_teardown_gpadl(struct vmbus_channel *channel, u32 gpadl_handle)
 553 {
 554         struct vmbus_channel_gpadl_teardown *msg;
 555         struct vmbus_channel_msginfo *info;
 556         unsigned long flags;
 557         int ret;
 558
 559         info = kmalloc(sizeof(*info) +
 560                        sizeof(struct vmbus_channel_gpadl_teardown), GFP_KERNEL);
 561         if (!info)
 562                 return -ENOMEM;
 563
 564         init_completion(&info->waitevent);
 565         info->waiting_channel = channel;
 566
 567         msg = (struct vmbus_channel_gpadl_teardown *)info->msg;
 568
 569         msg->header.msgtype = CHANNELMSG_GPADL_TEARDOWN;
 570         msg->child_relid = channel->offermsg.child_relid;
 571         msg->gpadl = gpadl_handle;
 572
 573         spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags);
 574         list_add_tail(&info->msglistentry,
 575                       &vmbus_connection.chn_msg_list);
 576         spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags);
 577
 578         if (channel->rescind)
 579                 goto post_msg_err;
 580
 581         ret = vmbus_post_msg(msg, sizeof(struct vmbus_channel_gpadl_teardown),
 582                              true);
 583
 584         trace_vmbus_teardown_gpadl(msg, ret);
 585
 586         if (ret)
 587                 goto post_msg_err;
 588
 589         wait_for_completion(&info->waitevent);
 590
 591 post_msg_err:
 592         /*
 593          * If the channel has been rescinded;
 594          * we will be awakened by the rescind
 595          * handler; set the error code to zero so we don't leak memory.
 596          */
 597         if (channel->rescind)
 598                 ret = 0;
 599
 600         spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags);
 601         list_del(&info->msglistentry);
 602         spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags);
 603
 604         kfree(info);
 605         return ret;
 606 }
 607 EXPORT_SYMBOL_GPL(vmbus_teardown_gpadl);
 608
 609 static void reset_channel_cb(void *arg)
 610 {
 611         struct vmbus_channel *channel = arg;
 612
 613         channel->onchannel_callback = NULL;
 614 }
 615
 616 void vmbus_reset_channel_cb(struct vmbus_channel *channel)
 617 {
 618         /*
 619          * vmbus_on_event(), running in the per-channel tasklet, can race
 620          * with vmbus_close_internal() in the case of SMP guest, e.g., when
 621          * the former is accessing channel->inbound.ring_buffer, the latter
 622          * could be freeing the ring_buffer pages, so here we must stop it
 623          * first.
 624          */
 625         tasklet_disable(&channel->callback_event);
 626
 627         channel->sc_creation_callback = NULL;
 628
 629         /* Stop the callback asap */
 630         if (channel->target_cpu != get_cpu()) {
 631                 put_cpu();
 632                 smp_call_function_single(channel->target_cpu, reset_channel_cb,
 633                                          channel, true);
 634         } else {
 635                 reset_channel_cb(channel);
 636                 put_cpu();
 637         }
 638
 639         /* Re-enable tasklet for use on re-open */
 640         tasklet_enable(&channel->callback_event);
 641 }
 642
 643 static int vmbus_close_internal(struct vmbus_channel *channel)
 644 {
 645         struct vmbus_channel_close_channel *msg;
 646         int ret;
 647
 648         vmbus_reset_channel_cb(channel);
 649
 650         /*
 651          * In case a device driver's probe() fails (e.g.,
 652          * util_probe() -> vmbus_open() returns -ENOMEM) and the device is
 653          * rescinded later (e.g., we dynamically disable an Integrated Service
 654          * in Hyper-V Manager), the driver's remove() invokes vmbus_close():
 655          * here we should skip most of the below cleanup work.
 656          */
 657         if (channel->state != CHANNEL_OPENED_STATE)
 658                 return -EINVAL;
 659
 660         channel->state = CHANNEL_OPEN_STATE;
 661
 662         /* Send a closing message */
 663
 664         msg = &channel->close_msg.msg;
 665
 666         msg->header.msgtype = CHANNELMSG_CLOSECHANNEL;
 667         msg->child_relid = channel->offermsg.child_relid;
 668
 669         ret = vmbus_post_msg(msg, sizeof(struct vmbus_channel_close_channel),
 670                              true);
 671
 672         trace_vmbus_close_internal(msg, ret);
 673
 674         if (ret) {
 675                 pr_err("Close failed: close post msg return is %d\n", ret);
 676                 /*
 677                  * If we failed to post the close msg,
 678                  * it is perhaps better to leak memory.
 679                  */
 680         }
 681
 682         /* Tear down the gpadl for the channel's ring buffer */
 683         else if (channel->ringbuffer_gpadlhandle) {
 684                 ret = vmbus_teardown_gpadl(channel,
 685                                            channel->ringbuffer_gpadlhandle);
 686                 if (ret) {
 687                         pr_err("Close failed: teardown gpadl return %d\n", ret);
 688                         /*
 689                          * If we failed to teardown gpadl,
 690                          * it is perhaps better to leak memory.
 691                          */
 692                 }
 693
 694                 channel->ringbuffer_gpadlhandle = 0;
 695         }
 696
 697         return ret;
 698 }
 699
 700 /* disconnect ring - close all channels */
 701 int vmbus_disconnect_ring(struct vmbus_channel *channel)
 702 {
 703         struct vmbus_channel *cur_channel, *tmp;
 704         int ret;
 705
 706         if (channel->primary_channel != NULL)
 707                 return -EINVAL;
 708
 709         list_for_each_entry_safe(cur_channel, tmp, &channel->sc_list, sc_list) {
 710                 if (cur_channel->rescind)
 711                         wait_for_completion(&cur_channel->rescind_event);
 712
 713                 mutex_lock(&vmbus_connection.channel_mutex);
 714                 if (vmbus_close_internal(cur_channel) == 0) {
 715                         vmbus_free_ring(cur_channel);
 716
 717                         if (cur_channel->rescind)
 718                                 hv_process_channel_removal(cur_channel);
 719                 }
 720                 mutex_unlock(&vmbus_connection.channel_mutex);
 721         }
 722
 723         /*
 724          * Now close the primary.
 725          */
 726         mutex_lock(&vmbus_connection.channel_mutex);
 727         ret = vmbus_close_internal(channel);
 728         mutex_unlock(&vmbus_connection.channel_mutex);
 729
 730         return ret;
 731 }
 732 EXPORT_SYMBOL_GPL(vmbus_disconnect_ring);
 733
 734 /*
 735  * vmbus_close - Close the specified channel
 736  */
 737 void vmbus_close(struct vmbus_channel *channel)
 738 {
 739         if (vmbus_disconnect_ring(channel) == 0)
 740                 vmbus_free_ring(channel);
 741 }
 742 EXPORT_SYMBOL_GPL(vmbus_close);
 743
 744 /**
 745  * vmbus_sendpacket() - Send the specified buffer on the given channel
 746  * @channel: Pointer to vmbus_channel structure
 747  * @buffer: Pointer to the buffer you want to send the data from.
 748  * @bufferlen: Maximum size of what the buffer holds.
 749  * @requestid: Identifier of the request
 750  * @type: Type of packet that is being sent e.g. negotiate, time
 751  *        packet etc.
 752  * @flags: 0 or VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED
 753  *
 754  * Sends data in @buffer directly to Hyper-V via the vmbus.
 755  * This will send the data unparsed to Hyper-V.
 756  *
 757  * Mainly used by Hyper-V drivers.
 758  */
 759 int vmbus_sendpacket(struct vmbus_channel *channel, void *buffer,
 760                            u32 bufferlen, u64 requestid,
 761                            enum vmbus_packet_type type, u32 flags)
 762 {
 763         struct vmpacket_descriptor desc;
 764         u32 packetlen = sizeof(struct vmpacket_descriptor) + bufferlen;
 765         u32 packetlen_aligned = ALIGN(packetlen, sizeof(u64));
 766         struct kvec bufferlist[3];
 767         u64 aligned_data = 0;
 768         int num_vecs = ((bufferlen != 0) ? 3 : 1);
 769
 770
 771         /* Setup the descriptor */
 772         desc.type = type; /* VmbusPacketTypeDataInBand; */
 773         desc.flags = flags; /* VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED; */
 774         /* in 8-bytes granularity */
 775         desc.offset8 = sizeof(struct vmpacket_descriptor) >> 3;
 776         desc.len8 = (u16)(packetlen_aligned >> 3);
 777         desc.trans_id = requestid;
 778
 779         bufferlist[0].iov_base = &desc;
 780         bufferlist[0].iov_len = sizeof(struct vmpacket_descriptor);
 781         bufferlist[1].iov_base = buffer;
 782         bufferlist[1].iov_len = bufferlen;
 783         bufferlist[2].iov_base = &aligned_data;
 784         bufferlist[2].iov_len = (packetlen_aligned - packetlen);
 785
 786         return hv_ringbuffer_write(channel, bufferlist, num_vecs);
 787 }
 788 EXPORT_SYMBOL(vmbus_sendpacket);
 789
 790 /*
 791  * vmbus_sendpacket_pagebuffer - Send a range of single-page buffer
 792  * packets using a GPADL Direct packet type. This interface allows you
 793  * to control notifying the host. This will be useful for sending
 794  * batched data. Also the sender can control the send flags
 795  * explicitly.
 796  */
 797 int vmbus_sendpacket_pagebuffer(struct vmbus_channel *channel,
 798                                 struct hv_page_buffer pagebuffers[],
 799                                 u32 pagecount, void *buffer, u32 bufferlen,
 800                                 u64 requestid)
 801 {
 802         int i;
 803         struct vmbus_channel_packet_page_buffer desc;
 804         u32 descsize;
 805         u32 packetlen;
 806         u32 packetlen_aligned;
 807         struct kvec bufferlist[3];
 808         u64 aligned_data = 0;
 809
 810         if (pagecount > MAX_PAGE_BUFFER_COUNT)
 811                 return -EINVAL;
 812
 813         /*
 814          * Adjust the size down since vmbus_channel_packet_page_buffer is the
 815          * largest size we support
 816          */
 817         descsize = sizeof(struct vmbus_channel_packet_page_buffer) -
 818                           ((MAX_PAGE_BUFFER_COUNT - pagecount) *
 819                           sizeof(struct hv_page_buffer));
 820         packetlen = descsize + bufferlen;
 821         packetlen_aligned = ALIGN(packetlen, sizeof(u64));
 822
 823         /* Setup the descriptor */
 824         desc.type = VM_PKT_DATA_USING_GPA_DIRECT;
 825         desc.flags = VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED;
 826         desc.dataoffset8 = descsize >> 3; /* in 8-bytes granularity */
 827         desc.length8 = (u16)(packetlen_aligned >> 3);
 828         desc.transactionid = requestid;
 829         desc.reserved = 0;
 830         desc.rangecount = pagecount;
 831
 832         for (i = 0; i < pagecount; i++) {
 833                 desc.range[i].len = pagebuffers[i].len;
 834                 desc.range[i].offset = pagebuffers[i].offset;
 835                 desc.range[i].pfn        = pagebuffers[i].pfn;
 836         }
 837
 838         bufferlist[0].iov_base = &desc;
 839         bufferlist[0].iov_len = descsize;
 840         bufferlist[1].iov_base = buffer;
 841         bufferlist[1].iov_len = bufferlen;
 842         bufferlist[2].iov_base = &aligned_data;
 843         bufferlist[2].iov_len = (packetlen_aligned - packetlen);
 844
 845         return hv_ringbuffer_write(channel, bufferlist, 3);
 846 }
 847 EXPORT_SYMBOL_GPL(vmbus_sendpacket_pagebuffer);
 848
 849 /*
 850  * vmbus_sendpacket_multipagebuffer - Send a multi-page buffer packet
 851  * using a GPADL Direct packet type.
 852  * The buffer includes the vmbus descriptor.
 853  */
 854 int vmbus_sendpacket_mpb_desc(struct vmbus_channel *channel,
 855                               struct vmbus_packet_mpb_array *desc,
 856                               u32 desc_size,
 857                               void *buffer, u32 bufferlen, u64 requestid)
 858 {
 859         u32 packetlen;
 860         u32 packetlen_aligned;
 861         struct kvec bufferlist[3];
 862         u64 aligned_data = 0;
 863
 864         packetlen = desc_size + bufferlen;
 865         packetlen_aligned = ALIGN(packetlen, sizeof(u64));
 866
 867         /* Setup the descriptor */
 868         desc->type = VM_PKT_DATA_USING_GPA_DIRECT;
 869         desc->flags = VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED;
 870         desc->dataoffset8 = desc_size >> 3; /* in 8-bytes granularity */
 871         desc->length8 = (u16)(packetlen_aligned >> 3);
 872         desc->transactionid = requestid;
 873         desc->reserved = 0;
 874         desc->rangecount = 1;
 875
 876         bufferlist[0].iov_base = desc;
 877         bufferlist[0].iov_len = desc_size;
 878         bufferlist[1].iov_base = buffer;
 879         bufferlist[1].iov_len = bufferlen;
 880         bufferlist[2].iov_base = &aligned_data;
 881         bufferlist[2].iov_len = (packetlen_aligned - packetlen);
 882
 883         return hv_ringbuffer_write(channel, bufferlist, 3);
 884 }
 885 EXPORT_SYMBOL_GPL(vmbus_sendpacket_mpb_desc);
 886
 887 /**
 888  * __vmbus_recvpacket() - Retrieve the user packet on the specified channel
 889  * @channel: Pointer to vmbus_channel structure
 890  * @buffer: Pointer to the buffer you want to receive the data into.
 891  * @bufferlen: Maximum size of what the buffer can hold.
 892  * @buffer_actual_len: The actual size of the data after it was received.
 893  * @requestid: Identifier of the request
 894  * @raw: true means keep the vmpacket_descriptor header in the received data.
 895  *
 896  * Receives directly from the hyper-v vmbus and puts the data it received
 897  * into Buffer. This will receive the data unparsed from hyper-v.
 898  *
 899  * Mainly used by Hyper-V drivers.
 900  */
 901 static inline int
 902 __vmbus_recvpacket(struct vmbus_channel *channel, void *buffer,
 903                    u32 bufferlen, u32 *buffer_actual_len, u64 *requestid,
 904                    bool raw)
 905 {
 906         return hv_ringbuffer_read(channel, buffer, bufferlen,
 907                                   buffer_actual_len, requestid, raw);
 908
 909 }
 910
 911 int vmbus_recvpacket(struct vmbus_channel *channel, void *buffer,
 912                      u32 bufferlen, u32 *buffer_actual_len,
 913                      u64 *requestid)
 914 {
 915         return __vmbus_recvpacket(channel, buffer, bufferlen,
 916                                   buffer_actual_len, requestid, false);
 917 }
 918 EXPORT_SYMBOL(vmbus_recvpacket);
 919
 920 /*
 921  * vmbus_recvpacket_raw - Retrieve the raw packet on the specified channel
 922  */
 923 int vmbus_recvpacket_raw(struct vmbus_channel *channel, void *buffer,
 924                               u32 bufferlen, u32 *buffer_actual_len,
 925                               u64 *requestid)
 926 {
 927         return __vmbus_recvpacket(channel, buffer, bufferlen,
 928                                   buffer_actual_len, requestid, true);
 929 }
 930 EXPORT_SYMBOL_GPL(vmbus_recvpacket_raw);