treewide: Replace GPLv2 boilerplate/reference with SPDX - rule 157
[sfrench/cifs-2.6.git] / drivers / xen / pvcalls-front.c
1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3  * (c) 2017 Stefano Stabellini <stefano@aporeto.com>
4  */
5
6 #include <linux/module.h>
7 #include <linux/net.h>
8 #include <linux/socket.h>
9
10 #include <net/sock.h>
11
12 #include <xen/events.h>
13 #include <xen/grant_table.h>
14 #include <xen/xen.h>
15 #include <xen/xenbus.h>
16 #include <xen/interface/io/pvcalls.h>
17
18 #include "pvcalls-front.h"
19
20 #define PVCALLS_INVALID_ID UINT_MAX
21 #define PVCALLS_RING_ORDER XENBUS_MAX_RING_GRANT_ORDER
22 #define PVCALLS_NR_RSP_PER_RING __CONST_RING_SIZE(xen_pvcalls, XEN_PAGE_SIZE)
23 #define PVCALLS_FRONT_MAX_SPIN 5000
24
25 static struct proto pvcalls_proto = {
26         .name   = "PVCalls",
27         .owner  = THIS_MODULE,
28         .obj_size = sizeof(struct sock),
29 };
30
31 struct pvcalls_bedata {
32         struct xen_pvcalls_front_ring ring;
33         grant_ref_t ref;
34         int irq;
35
36         struct list_head socket_mappings;
37         spinlock_t socket_lock;
38
39         wait_queue_head_t inflight_req;
40         struct xen_pvcalls_response rsp[PVCALLS_NR_RSP_PER_RING];
41 };
42 /* Only one front/back connection supported. */
43 static struct xenbus_device *pvcalls_front_dev;
44 static atomic_t pvcalls_refcount;
45
46 /* first increment refcount, then proceed */
47 #define pvcalls_enter() {               \
48         atomic_inc(&pvcalls_refcount);      \
49 }
50
51 /* first complete other operations, then decrement refcount */
52 #define pvcalls_exit() {                \
53         atomic_dec(&pvcalls_refcount);      \
54 }
55
56 struct sock_mapping {
57         bool active_socket;
58         struct list_head list;
59         struct socket *sock;
60         atomic_t refcount;
61         union {
62                 struct {
63                         int irq;
64                         grant_ref_t ref;
65                         struct pvcalls_data_intf *ring;
66                         struct pvcalls_data data;
67                         struct mutex in_mutex;
68                         struct mutex out_mutex;
69
70                         wait_queue_head_t inflight_conn_req;
71                 } active;
72                 struct {
73                 /*
74                  * Socket status, needs to be 64-bit aligned due to the
75                  * test_and_* functions which have this requirement on arm64.
76                  */
77 #define PVCALLS_STATUS_UNINITALIZED  0
78 #define PVCALLS_STATUS_BIND          1
79 #define PVCALLS_STATUS_LISTEN        2
80                         uint8_t status __attribute__((aligned(8)));
81                 /*
82                  * Internal state-machine flags.
83                  * Only one accept operation can be inflight for a socket.
84                  * Only one poll operation can be inflight for a given socket.
85                  * flags needs to be 64-bit aligned due to the test_and_*
86                  * functions which have this requirement on arm64.
87                  */
88 #define PVCALLS_FLAG_ACCEPT_INFLIGHT 0
89 #define PVCALLS_FLAG_POLL_INFLIGHT   1
90 #define PVCALLS_FLAG_POLL_RET        2
91                         uint8_t flags __attribute__((aligned(8)));
92                         uint32_t inflight_req_id;
93                         struct sock_mapping *accept_map;
94                         wait_queue_head_t inflight_accept_req;
95                 } passive;
96         };
97 };
98
99 static inline struct sock_mapping *pvcalls_enter_sock(struct socket *sock)
100 {
101         struct sock_mapping *map;
102
103         if (!pvcalls_front_dev ||
104                 dev_get_drvdata(&pvcalls_front_dev->dev) == NULL)
105                 return ERR_PTR(-ENOTCONN);
106
107         map = (struct sock_mapping *)sock->sk->sk_send_head;
108         if (map == NULL)
109                 return ERR_PTR(-ENOTSOCK);
110
111         pvcalls_enter();
112         atomic_inc(&map->refcount);
113         return map;
114 }
115
116 static inline void pvcalls_exit_sock(struct socket *sock)
117 {
118         struct sock_mapping *map;
119
120         map = (struct sock_mapping *)sock->sk->sk_send_head;
121         atomic_dec(&map->refcount);
122         pvcalls_exit();
123 }
124
125 static inline int get_request(struct pvcalls_bedata *bedata, int *req_id)
126 {
127         *req_id = bedata->ring.req_prod_pvt & (RING_SIZE(&bedata->ring) - 1);
128         if (RING_FULL(&bedata->ring) ||
129             bedata->rsp[*req_id].req_id != PVCALLS_INVALID_ID)
130                 return -EAGAIN;
131         return 0;
132 }
133
134 static bool pvcalls_front_write_todo(struct sock_mapping *map)
135 {
136         struct pvcalls_data_intf *intf = map->active.ring;
137         RING_IDX cons, prod, size = XEN_FLEX_RING_SIZE(PVCALLS_RING_ORDER);
138         int32_t error;
139
140         error = intf->out_error;
141         if (error == -ENOTCONN)
142                 return false;
143         if (error != 0)
144                 return true;
145
146         cons = intf->out_cons;
147         prod = intf->out_prod;
148         return !!(size - pvcalls_queued(prod, cons, size));
149 }
150
151 static bool pvcalls_front_read_todo(struct sock_mapping *map)
152 {
153         struct pvcalls_data_intf *intf = map->active.ring;
154         RING_IDX cons, prod;
155         int32_t error;
156
157         cons = intf->in_cons;
158         prod = intf->in_prod;
159         error = intf->in_error;
160         return (error != 0 ||
161                 pvcalls_queued(prod, cons,
162                                XEN_FLEX_RING_SIZE(PVCALLS_RING_ORDER)) != 0);
163 }
164
165 static irqreturn_t pvcalls_front_event_handler(int irq, void *dev_id)
166 {
167         struct xenbus_device *dev = dev_id;
168         struct pvcalls_bedata *bedata;
169         struct xen_pvcalls_response *rsp;
170         uint8_t *src, *dst;
171         int req_id = 0, more = 0, done = 0;
172
173         if (dev == NULL)
174                 return IRQ_HANDLED;
175
176         pvcalls_enter();
177         bedata = dev_get_drvdata(&dev->dev);
178         if (bedata == NULL) {
179                 pvcalls_exit();
180                 return IRQ_HANDLED;
181         }
182
183 again:
184         while (RING_HAS_UNCONSUMED_RESPONSES(&bedata->ring)) {
185                 rsp = RING_GET_RESPONSE(&bedata->ring, bedata->ring.rsp_cons);
186
187                 req_id = rsp->req_id;
188                 if (rsp->cmd == PVCALLS_POLL) {
189                         struct sock_mapping *map = (struct sock_mapping *)(uintptr_t)
190                                                    rsp->u.poll.id;
191
192                         clear_bit(PVCALLS_FLAG_POLL_INFLIGHT,
193                                   (void *)&map->passive.flags);
194                         /*
195                          * clear INFLIGHT, then set RET. It pairs with
196                          * the checks at the beginning of
197                          * pvcalls_front_poll_passive.
198                          */
199                         smp_wmb();
200                         set_bit(PVCALLS_FLAG_POLL_RET,
201                                 (void *)&map->passive.flags);
202                 } else {
203                         dst = (uint8_t *)&bedata->rsp[req_id] +
204                               sizeof(rsp->req_id);
205                         src = (uint8_t *)rsp + sizeof(rsp->req_id);
206                         memcpy(dst, src, sizeof(*rsp) - sizeof(rsp->req_id));
207                         /*
208                          * First copy the rest of the data, then req_id. It is
209                          * paired with the barrier when accessing bedata->rsp.
210                          */
211                         smp_wmb();
212                         bedata->rsp[req_id].req_id = req_id;
213                 }
214
215                 done = 1;
216                 bedata->ring.rsp_cons++;
217         }
218
219         RING_FINAL_CHECK_FOR_RESPONSES(&bedata->ring, more);
220         if (more)
221                 goto again;
222         if (done)
223                 wake_up(&bedata->inflight_req);
224         pvcalls_exit();
225         return IRQ_HANDLED;
226 }
227
228 static void pvcalls_front_free_map(struct pvcalls_bedata *bedata,
229                                    struct sock_mapping *map)
230 {
231         int i;
232
233         unbind_from_irqhandler(map->active.irq, map);
234
235         spin_lock(&bedata->socket_lock);
236         if (!list_empty(&map->list))
237                 list_del_init(&map->list);
238         spin_unlock(&bedata->socket_lock);
239
240         for (i = 0; i < (1 << PVCALLS_RING_ORDER); i++)
241                 gnttab_end_foreign_access(map->active.ring->ref[i], 0, 0);
242         gnttab_end_foreign_access(map->active.ref, 0, 0);
243         free_page((unsigned long)map->active.ring);
244
245         kfree(map);
246 }
247
248 static irqreturn_t pvcalls_front_conn_handler(int irq, void *sock_map)
249 {
250         struct sock_mapping *map = sock_map;
251
252         if (map == NULL)
253                 return IRQ_HANDLED;
254
255         wake_up_interruptible(&map->active.inflight_conn_req);
256
257         return IRQ_HANDLED;
258 }
259
260 int pvcalls_front_socket(struct socket *sock)
261 {
262         struct pvcalls_bedata *bedata;
263         struct sock_mapping *map = NULL;
264         struct xen_pvcalls_request *req;
265         int notify, req_id, ret;
266
267         /*
268          * PVCalls only supports domain AF_INET,
269          * type SOCK_STREAM and protocol 0 sockets for now.
270          *
271          * Check socket type here, AF_INET and protocol checks are done
272          * by the caller.
273          */
274         if (sock->type != SOCK_STREAM)
275                 return -EOPNOTSUPP;
276
277         pvcalls_enter();
278         if (!pvcalls_front_dev) {
279                 pvcalls_exit();
280                 return -EACCES;
281         }
282         bedata = dev_get_drvdata(&pvcalls_front_dev->dev);
283
284         map = kzalloc(sizeof(*map), GFP_KERNEL);
285         if (map == NULL) {
286                 pvcalls_exit();
287                 return -ENOMEM;
288         }
289
290         spin_lock(&bedata->socket_lock);
291
292         ret = get_request(bedata, &req_id);
293         if (ret < 0) {
294                 kfree(map);
295                 spin_unlock(&bedata->socket_lock);
296                 pvcalls_exit();
297                 return ret;
298         }
299
300         /*
301          * sock->sk->sk_send_head is not used for ip sockets: reuse the
302          * field to store a pointer to the struct sock_mapping
303          * corresponding to the socket. This way, we can easily get the
304          * struct sock_mapping from the struct socket.
305          */
306         sock->sk->sk_send_head = (void *)map;
307         list_add_tail(&map->list, &bedata->socket_mappings);
308
309         req = RING_GET_REQUEST(&bedata->ring, req_id);
310         req->req_id = req_id;
311         req->cmd = PVCALLS_SOCKET;
312         req->u.socket.id = (uintptr_t) map;
313         req->u.socket.domain = AF_INET;
314         req->u.socket.type = SOCK_STREAM;
315         req->u.socket.protocol = IPPROTO_IP;
316
317         bedata->ring.req_prod_pvt++;
318         RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&bedata->ring, notify);
319         spin_unlock(&bedata->socket_lock);
320         if (notify)
321                 notify_remote_via_irq(bedata->irq);
322
323         wait_event(bedata->inflight_req,
324                    READ_ONCE(bedata->rsp[req_id].req_id) == req_id);
325
326         /* read req_id, then the content */
327         smp_rmb();
328         ret = bedata->rsp[req_id].ret;
329         bedata->rsp[req_id].req_id = PVCALLS_INVALID_ID;
330
331         pvcalls_exit();
332         return ret;
333 }
334
335 static void free_active_ring(struct sock_mapping *map)
336 {
337         if (!map->active.ring)
338                 return;
339
340         free_pages((unsigned long)map->active.data.in,
341                         map->active.ring->ring_order);
342         free_page((unsigned long)map->active.ring);
343 }
344
345 static int alloc_active_ring(struct sock_mapping *map)
346 {
347         void *bytes;
348
349         map->active.ring = (struct pvcalls_data_intf *)
350                 get_zeroed_page(GFP_KERNEL);
351         if (!map->active.ring)
352                 goto out;
353
354         map->active.ring->ring_order = PVCALLS_RING_ORDER;
355         bytes = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,
356                                         PVCALLS_RING_ORDER);
357         if (!bytes)
358                 goto out;
359
360         map->active.data.in = bytes;
361         map->active.data.out = bytes +
362                 XEN_FLEX_RING_SIZE(PVCALLS_RING_ORDER);
363
364         return 0;
365
366 out:
367         free_active_ring(map);
368         return -ENOMEM;
369 }
370
371 static int create_active(struct sock_mapping *map, int *evtchn)
372 {
373         void *bytes;
374         int ret = -ENOMEM, irq = -1, i;
375
376         *evtchn = -1;
377         init_waitqueue_head(&map->active.inflight_conn_req);
378
379         bytes = map->active.data.in;
380         for (i = 0; i < (1 << PVCALLS_RING_ORDER); i++)
381                 map->active.ring->ref[i] = gnttab_grant_foreign_access(
382                         pvcalls_front_dev->otherend_id,
383                         pfn_to_gfn(virt_to_pfn(bytes) + i), 0);
384
385         map->active.ref = gnttab_grant_foreign_access(
386                 pvcalls_front_dev->otherend_id,
387                 pfn_to_gfn(virt_to_pfn((void *)map->active.ring)), 0);
388
389         ret = xenbus_alloc_evtchn(pvcalls_front_dev, evtchn);
390         if (ret)
391                 goto out_error;
392         irq = bind_evtchn_to_irqhandler(*evtchn, pvcalls_front_conn_handler,
393                                         0, "pvcalls-frontend", map);
394         if (irq < 0) {
395                 ret = irq;
396                 goto out_error;
397         }
398
399         map->active.irq = irq;
400         map->active_socket = true;
401         mutex_init(&map->active.in_mutex);
402         mutex_init(&map->active.out_mutex);
403
404         return 0;
405
406 out_error:
407         if (*evtchn >= 0)
408                 xenbus_free_evtchn(pvcalls_front_dev, *evtchn);
409         return ret;
410 }
411
412 int pvcalls_front_connect(struct socket *sock, struct sockaddr *addr,
413                                 int addr_len, int flags)
414 {
415         struct pvcalls_bedata *bedata;
416         struct sock_mapping *map = NULL;
417         struct xen_pvcalls_request *req;
418         int notify, req_id, ret, evtchn;
419
420         if (addr->sa_family != AF_INET || sock->type != SOCK_STREAM)
421                 return -EOPNOTSUPP;
422
423         map = pvcalls_enter_sock(sock);
424         if (IS_ERR(map))
425                 return PTR_ERR(map);
426
427         bedata = dev_get_drvdata(&pvcalls_front_dev->dev);
428         ret = alloc_active_ring(map);
429         if (ret < 0) {
430                 pvcalls_exit_sock(sock);
431                 return ret;
432         }
433
434         spin_lock(&bedata->socket_lock);
435         ret = get_request(bedata, &req_id);
436         if (ret < 0) {
437                 spin_unlock(&bedata->socket_lock);
438                 free_active_ring(map);
439                 pvcalls_exit_sock(sock);
440                 return ret;
441         }
442         ret = create_active(map, &evtchn);
443         if (ret < 0) {
444                 spin_unlock(&bedata->socket_lock);
445                 free_active_ring(map);
446                 pvcalls_exit_sock(sock);
447                 return ret;
448         }
449
450         req = RING_GET_REQUEST(&bedata->ring, req_id);
451         req->req_id = req_id;
452         req->cmd = PVCALLS_CONNECT;
453         req->u.connect.id = (uintptr_t)map;
454         req->u.connect.len = addr_len;
455         req->u.connect.flags = flags;
456         req->u.connect.ref = map->active.ref;
457         req->u.connect.evtchn = evtchn;
458         memcpy(req->u.connect.addr, addr, sizeof(*addr));
459
460         map->sock = sock;
461
462         bedata->ring.req_prod_pvt++;
463         RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&bedata->ring, notify);
464         spin_unlock(&bedata->socket_lock);
465
466         if (notify)
467                 notify_remote_via_irq(bedata->irq);
468
469         wait_event(bedata->inflight_req,
470                    READ_ONCE(bedata->rsp[req_id].req_id) == req_id);
471
472         /* read req_id, then the content */
473         smp_rmb();
474         ret = bedata->rsp[req_id].ret;
475         bedata->rsp[req_id].req_id = PVCALLS_INVALID_ID;
476         pvcalls_exit_sock(sock);
477         return ret;
478 }
479
480 static int __write_ring(struct pvcalls_data_intf *intf,
481                         struct pvcalls_data *data,
482                         struct iov_iter *msg_iter,
483                         int len)
484 {
485         RING_IDX cons, prod, size, masked_prod, masked_cons;
486         RING_IDX array_size = XEN_FLEX_RING_SIZE(PVCALLS_RING_ORDER);
487         int32_t error;
488
489         error = intf->out_error;
490         if (error < 0)
491                 return error;
492         cons = intf->out_cons;
493         prod = intf->out_prod;
494         /* read indexes before continuing */
495         virt_mb();
496
497         size = pvcalls_queued(prod, cons, array_size);
498         if (size > array_size)
499                 return -EINVAL;
500         if (size == array_size)
501                 return 0;
502         if (len > array_size - size)
503                 len = array_size - size;
504
505         masked_prod = pvcalls_mask(prod, array_size);
506         masked_cons = pvcalls_mask(cons, array_size);
507
508         if (masked_prod < masked_cons) {
509                 len = copy_from_iter(data->out + masked_prod, len, msg_iter);
510         } else {
511                 if (len > array_size - masked_prod) {
512                         int ret = copy_from_iter(data->out + masked_prod,
513                                        array_size - masked_prod, msg_iter);
514                         if (ret != array_size - masked_prod) {
515                                 len = ret;
516                                 goto out;
517                         }
518                         len = ret + copy_from_iter(data->out, len - ret, msg_iter);
519                 } else {
520                         len = copy_from_iter(data->out + masked_prod, len, msg_iter);
521                 }
522         }
523 out:
524         /* write to ring before updating pointer */
525         virt_wmb();
526         intf->out_prod += len;
527
528         return len;
529 }
530
531 int pvcalls_front_sendmsg(struct socket *sock, struct msghdr *msg,
532                           size_t len)
533 {
534         struct pvcalls_bedata *bedata;
535         struct sock_mapping *map;
536         int sent, tot_sent = 0;
537         int count = 0, flags;
538
539         flags = msg->msg_flags;
540         if (flags & (MSG_CONFIRM|MSG_DONTROUTE|MSG_EOR|MSG_OOB))
541                 return -EOPNOTSUPP;
542
543         map = pvcalls_enter_sock(sock);
544         if (IS_ERR(map))
545                 return PTR_ERR(map);
546         bedata = dev_get_drvdata(&pvcalls_front_dev->dev);
547
548         mutex_lock(&map->active.out_mutex);
549         if ((flags & MSG_DONTWAIT) && !pvcalls_front_write_todo(map)) {
550                 mutex_unlock(&map->active.out_mutex);
551                 pvcalls_exit_sock(sock);
552                 return -EAGAIN;
553         }
554         if (len > INT_MAX)
555                 len = INT_MAX;
556
557 again:
558         count++;
559         sent = __write_ring(map->active.ring,
560                             &map->active.data, &msg->msg_iter,
561                             len);
562         if (sent > 0) {
563                 len -= sent;
564                 tot_sent += sent;
565                 notify_remote_via_irq(map->active.irq);
566         }
567         if (sent >= 0 && len > 0 && count < PVCALLS_FRONT_MAX_SPIN)
568                 goto again;
569         if (sent < 0)
570                 tot_sent = sent;
571
572         mutex_unlock(&map->active.out_mutex);
573         pvcalls_exit_sock(sock);
574         return tot_sent;
575 }
576
577 static int __read_ring(struct pvcalls_data_intf *intf,
578                        struct pvcalls_data *data,
579                        struct iov_iter *msg_iter,
580                        size_t len, int flags)
581 {
582         RING_IDX cons, prod, size, masked_prod, masked_cons;
583         RING_IDX array_size = XEN_FLEX_RING_SIZE(PVCALLS_RING_ORDER);
584         int32_t error;
585
586         cons = intf->in_cons;
587         prod = intf->in_prod;
588         error = intf->in_error;
589         /* get pointers before reading from the ring */
590         virt_rmb();
591
592         size = pvcalls_queued(prod, cons, array_size);
593         masked_prod = pvcalls_mask(prod, array_size);
594         masked_cons = pvcalls_mask(cons, array_size);
595
596         if (size == 0)
597                 return error ?: size;
598
599         if (len > size)
600                 len = size;
601
602         if (masked_prod > masked_cons) {
603                 len = copy_to_iter(data->in + masked_cons, len, msg_iter);
604         } else {
605                 if (len > (array_size - masked_cons)) {
606                         int ret = copy_to_iter(data->in + masked_cons,
607                                      array_size - masked_cons, msg_iter);
608                         if (ret != array_size - masked_cons) {
609                                 len = ret;
610                                 goto out;
611                         }
612                         len = ret + copy_to_iter(data->in, len - ret, msg_iter);
613                 } else {
614                         len = copy_to_iter(data->in + masked_cons, len, msg_iter);
615                 }
616         }
617 out:
618         /* read data from the ring before increasing the index */
619         virt_mb();
620         if (!(flags & MSG_PEEK))
621                 intf->in_cons += len;
622
623         return len;
624 }
625
626 int pvcalls_front_recvmsg(struct socket *sock, struct msghdr *msg, size_t len,
627                      int flags)
628 {
629         struct pvcalls_bedata *bedata;
630         int ret;
631         struct sock_mapping *map;
632
633         if (flags & (MSG_CMSG_CLOEXEC|MSG_ERRQUEUE|MSG_OOB|MSG_TRUNC))
634                 return -EOPNOTSUPP;
635
636         map = pvcalls_enter_sock(sock);
637         if (IS_ERR(map))
638                 return PTR_ERR(map);
639         bedata = dev_get_drvdata(&pvcalls_front_dev->dev);
640
641         mutex_lock(&map->active.in_mutex);
642         if (len > XEN_FLEX_RING_SIZE(PVCALLS_RING_ORDER))
643                 len = XEN_FLEX_RING_SIZE(PVCALLS_RING_ORDER);
644
645         while (!(flags & MSG_DONTWAIT) && !pvcalls_front_read_todo(map)) {
646                 wait_event_interruptible(map->active.inflight_conn_req,
647                                          pvcalls_front_read_todo(map));
648         }
649         ret = __read_ring(map->active.ring, &map->active.data,
650                           &msg->msg_iter, len, flags);
651
652         if (ret > 0)
653                 notify_remote_via_irq(map->active.irq);
654         if (ret == 0)
655                 ret = (flags & MSG_DONTWAIT) ? -EAGAIN : 0;
656         if (ret == -ENOTCONN)
657                 ret = 0;
658
659         mutex_unlock(&map->active.in_mutex);
660         pvcalls_exit_sock(sock);
661         return ret;
662 }
663
664 int pvcalls_front_bind(struct socket *sock, struct sockaddr *addr, int addr_len)
665 {
666         struct pvcalls_bedata *bedata;
667         struct sock_mapping *map = NULL;
668         struct xen_pvcalls_request *req;
669         int notify, req_id, ret;
670
671         if (addr->sa_family != AF_INET || sock->type != SOCK_STREAM)
672                 return -EOPNOTSUPP;
673
674         map = pvcalls_enter_sock(sock);
675         if (IS_ERR(map))
676                 return PTR_ERR(map);
677         bedata = dev_get_drvdata(&pvcalls_front_dev->dev);
678
679         spin_lock(&bedata->socket_lock);
680         ret = get_request(bedata, &req_id);
681         if (ret < 0) {
682                 spin_unlock(&bedata->socket_lock);
683                 pvcalls_exit_sock(sock);
684                 return ret;
685         }
686         req = RING_GET_REQUEST(&bedata->ring, req_id);
687         req->req_id = req_id;
688         map->sock = sock;
689         req->cmd = PVCALLS_BIND;
690         req->u.bind.id = (uintptr_t)map;
691         memcpy(req->u.bind.addr, addr, sizeof(*addr));
692         req->u.bind.len = addr_len;
693
694         init_waitqueue_head(&map->passive.inflight_accept_req);
695
696         map->active_socket = false;
697
698         bedata->ring.req_prod_pvt++;
699         RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&bedata->ring, notify);
700         spin_unlock(&bedata->socket_lock);
701         if (notify)
702                 notify_remote_via_irq(bedata->irq);
703
704         wait_event(bedata->inflight_req,
705                    READ_ONCE(bedata->rsp[req_id].req_id) == req_id);
706
707         /* read req_id, then the content */
708         smp_rmb();
709         ret = bedata->rsp[req_id].ret;
710         bedata->rsp[req_id].req_id = PVCALLS_INVALID_ID;
711
712         map->passive.status = PVCALLS_STATUS_BIND;
713         pvcalls_exit_sock(sock);
714         return 0;
715 }
716
717 int pvcalls_front_listen(struct socket *sock, int backlog)
718 {
719         struct pvcalls_bedata *bedata;
720         struct sock_mapping *map;
721         struct xen_pvcalls_request *req;
722         int notify, req_id, ret;
723
724         map = pvcalls_enter_sock(sock);
725         if (IS_ERR(map))
726                 return PTR_ERR(map);
727         bedata = dev_get_drvdata(&pvcalls_front_dev->dev);
728
729         if (map->passive.status != PVCALLS_STATUS_BIND) {
730                 pvcalls_exit_sock(sock);
731                 return -EOPNOTSUPP;
732         }
733
734         spin_lock(&bedata->socket_lock);
735         ret = get_request(bedata, &req_id);
736         if (ret < 0) {
737                 spin_unlock(&bedata->socket_lock);
738                 pvcalls_exit_sock(sock);
739                 return ret;
740         }
741         req = RING_GET_REQUEST(&bedata->ring, req_id);
742         req->req_id = req_id;
743         req->cmd = PVCALLS_LISTEN;
744         req->u.listen.id = (uintptr_t) map;
745         req->u.listen.backlog = backlog;
746
747         bedata->ring.req_prod_pvt++;
748         RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&bedata->ring, notify);
749         spin_unlock(&bedata->socket_lock);
750         if (notify)
751                 notify_remote_via_irq(bedata->irq);
752
753         wait_event(bedata->inflight_req,
754                    READ_ONCE(bedata->rsp[req_id].req_id) == req_id);
755
756         /* read req_id, then the content */
757         smp_rmb();
758         ret = bedata->rsp[req_id].ret;
759         bedata->rsp[req_id].req_id = PVCALLS_INVALID_ID;
760
761         map->passive.status = PVCALLS_STATUS_LISTEN;
762         pvcalls_exit_sock(sock);
763         return ret;
764 }
765
766 int pvcalls_front_accept(struct socket *sock, struct socket *newsock, int flags)
767 {
768         struct pvcalls_bedata *bedata;
769         struct sock_mapping *map;
770         struct sock_mapping *map2 = NULL;
771         struct xen_pvcalls_request *req;
772         int notify, req_id, ret, evtchn, nonblock;
773
774         map = pvcalls_enter_sock(sock);
775         if (IS_ERR(map))
776                 return PTR_ERR(map);
777         bedata = dev_get_drvdata(&pvcalls_front_dev->dev);
778
779         if (map->passive.status != PVCALLS_STATUS_LISTEN) {
780                 pvcalls_exit_sock(sock);
781                 return -EINVAL;
782         }
783
784         nonblock = flags & SOCK_NONBLOCK;
785         /*
786          * Backend only supports 1 inflight accept request, will return
787          * errors for the others
788          */
789         if (test_and_set_bit(PVCALLS_FLAG_ACCEPT_INFLIGHT,
790                              (void *)&map->passive.flags)) {
791                 req_id = READ_ONCE(map->passive.inflight_req_id);
792                 if (req_id != PVCALLS_INVALID_ID &&
793                     READ_ONCE(bedata->rsp[req_id].req_id) == req_id) {
794                         map2 = map->passive.accept_map;
795                         goto received;
796                 }
797                 if (nonblock) {
798                         pvcalls_exit_sock(sock);
799                         return -EAGAIN;
800                 }
801                 if (wait_event_interruptible(map->passive.inflight_accept_req,
802                         !test_and_set_bit(PVCALLS_FLAG_ACCEPT_INFLIGHT,
803                                           (void *)&map->passive.flags))) {
804                         pvcalls_exit_sock(sock);
805                         return -EINTR;
806                 }
807         }
808
809         map2 = kzalloc(sizeof(*map2), GFP_KERNEL);
810         if (map2 == NULL) {
811                 clear_bit(PVCALLS_FLAG_ACCEPT_INFLIGHT,
812                           (void *)&map->passive.flags);
813                 pvcalls_exit_sock(sock);
814                 return -ENOMEM;
815         }
816         ret = alloc_active_ring(map2);
817         if (ret < 0) {
818                 clear_bit(PVCALLS_FLAG_ACCEPT_INFLIGHT,
819                                 (void *)&map->passive.flags);
820                 kfree(map2);
821                 pvcalls_exit_sock(sock);
822                 return ret;
823         }
824         spin_lock(&bedata->socket_lock);
825         ret = get_request(bedata, &req_id);
826         if (ret < 0) {
827                 clear_bit(PVCALLS_FLAG_ACCEPT_INFLIGHT,
828                           (void *)&map->passive.flags);
829                 spin_unlock(&bedata->socket_lock);
830                 free_active_ring(map2);
831                 kfree(map2);
832                 pvcalls_exit_sock(sock);
833                 return ret;
834         }
835
836         ret = create_active(map2, &evtchn);
837         if (ret < 0) {
838                 free_active_ring(map2);
839                 kfree(map2);
840                 clear_bit(PVCALLS_FLAG_ACCEPT_INFLIGHT,
841                           (void *)&map->passive.flags);
842                 spin_unlock(&bedata->socket_lock);
843                 pvcalls_exit_sock(sock);
844                 return ret;
845         }
846         list_add_tail(&map2->list, &bedata->socket_mappings);
847
848         req = RING_GET_REQUEST(&bedata->ring, req_id);
849         req->req_id = req_id;
850         req->cmd = PVCALLS_ACCEPT;
851         req->u.accept.id = (uintptr_t) map;
852         req->u.accept.ref = map2->active.ref;
853         req->u.accept.id_new = (uintptr_t) map2;
854         req->u.accept.evtchn = evtchn;
855         map->passive.accept_map = map2;
856
857         bedata->ring.req_prod_pvt++;
858         RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&bedata->ring, notify);
859         spin_unlock(&bedata->socket_lock);
860         if (notify)
861                 notify_remote_via_irq(bedata->irq);
862         /* We could check if we have received a response before returning. */
863         if (nonblock) {
864                 WRITE_ONCE(map->passive.inflight_req_id, req_id);
865                 pvcalls_exit_sock(sock);
866                 return -EAGAIN;
867         }
868
869         if (wait_event_interruptible(bedata->inflight_req,
870                 READ_ONCE(bedata->rsp[req_id].req_id) == req_id)) {
871                 pvcalls_exit_sock(sock);
872                 return -EINTR;
873         }
874         /* read req_id, then the content */
875         smp_rmb();
876
877 received:
878         map2->sock = newsock;
879         newsock->sk = sk_alloc(sock_net(sock->sk), PF_INET, GFP_KERNEL, &pvcalls_proto, false);
880         if (!newsock->sk) {
881                 bedata->rsp[req_id].req_id = PVCALLS_INVALID_ID;
882                 map->passive.inflight_req_id = PVCALLS_INVALID_ID;
883                 clear_bit(PVCALLS_FLAG_ACCEPT_INFLIGHT,
884                           (void *)&map->passive.flags);
885                 pvcalls_front_free_map(bedata, map2);
886                 pvcalls_exit_sock(sock);
887                 return -ENOMEM;
888         }
889         newsock->sk->sk_send_head = (void *)map2;
890
891         ret = bedata->rsp[req_id].ret;
892         bedata->rsp[req_id].req_id = PVCALLS_INVALID_ID;
893         map->passive.inflight_req_id = PVCALLS_INVALID_ID;
894
895         clear_bit(PVCALLS_FLAG_ACCEPT_INFLIGHT, (void *)&map->passive.flags);
896         wake_up(&map->passive.inflight_accept_req);
897
898         pvcalls_exit_sock(sock);
899         return ret;
900 }
901
902 static __poll_t pvcalls_front_poll_passive(struct file *file,
903                                                struct pvcalls_bedata *bedata,
904                                                struct sock_mapping *map,
905                                                poll_table *wait)
906 {
907         int notify, req_id, ret;
908         struct xen_pvcalls_request *req;
909
910         if (test_bit(PVCALLS_FLAG_ACCEPT_INFLIGHT,
911                      (void *)&map->passive.flags)) {
912                 uint32_t req_id = READ_ONCE(map->passive.inflight_req_id);
913
914                 if (req_id != PVCALLS_INVALID_ID &&
915                     READ_ONCE(bedata->rsp[req_id].req_id) == req_id)
916                         return EPOLLIN | EPOLLRDNORM;
917
918                 poll_wait(file, &map->passive.inflight_accept_req, wait);
919                 return 0;
920         }
921
922         if (test_and_clear_bit(PVCALLS_FLAG_POLL_RET,
923                                (void *)&map->passive.flags))
924                 return EPOLLIN | EPOLLRDNORM;
925
926         /*
927          * First check RET, then INFLIGHT. No barriers necessary to
928          * ensure execution ordering because of the conditional
929          * instructions creating control dependencies.
930          */
931
932         if (test_and_set_bit(PVCALLS_FLAG_POLL_INFLIGHT,
933                              (void *)&map->passive.flags)) {
934                 poll_wait(file, &bedata->inflight_req, wait);
935                 return 0;
936         }
937
938         spin_lock(&bedata->socket_lock);
939         ret = get_request(bedata, &req_id);
940         if (ret < 0) {
941                 spin_unlock(&bedata->socket_lock);
942                 return ret;
943         }
944         req = RING_GET_REQUEST(&bedata->ring, req_id);
945         req->req_id = req_id;
946         req->cmd = PVCALLS_POLL;
947         req->u.poll.id = (uintptr_t) map;
948
949         bedata->ring.req_prod_pvt++;
950         RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&bedata->ring, notify);
951         spin_unlock(&bedata->socket_lock);
952         if (notify)
953                 notify_remote_via_irq(bedata->irq);
954
955         poll_wait(file, &bedata->inflight_req, wait);
956         return 0;
957 }
958
959 static __poll_t pvcalls_front_poll_active(struct file *file,
960                                               struct pvcalls_bedata *bedata,
961                                               struct sock_mapping *map,
962                                               poll_table *wait)
963 {
964         __poll_t mask = 0;
965         int32_t in_error, out_error;
966         struct pvcalls_data_intf *intf = map->active.ring;
967
968         out_error = intf->out_error;
969         in_error = intf->in_error;
970
971         poll_wait(file, &map->active.inflight_conn_req, wait);
972         if (pvcalls_front_write_todo(map))
973                 mask |= EPOLLOUT | EPOLLWRNORM;
974         if (pvcalls_front_read_todo(map))
975                 mask |= EPOLLIN | EPOLLRDNORM;
976         if (in_error != 0 || out_error != 0)
977                 mask |= EPOLLERR;
978
979         return mask;
980 }
981
982 __poll_t pvcalls_front_poll(struct file *file, struct socket *sock,
983                                poll_table *wait)
984 {
985         struct pvcalls_bedata *bedata;
986         struct sock_mapping *map;
987         __poll_t ret;
988
989         map = pvcalls_enter_sock(sock);
990         if (IS_ERR(map))
991                 return EPOLLNVAL;
992         bedata = dev_get_drvdata(&pvcalls_front_dev->dev);
993
994         if (map->active_socket)
995                 ret = pvcalls_front_poll_active(file, bedata, map, wait);
996         else
997                 ret = pvcalls_front_poll_passive(file, bedata, map, wait);
998         pvcalls_exit_sock(sock);
999         return ret;
1000 }
1001
1002 int pvcalls_front_release(struct socket *sock)
1003 {
1004         struct pvcalls_bedata *bedata;
1005         struct sock_mapping *map;
1006         int req_id, notify, ret;
1007         struct xen_pvcalls_request *req;
1008
1009         if (sock->sk == NULL)
1010                 return 0;
1011
1012         map = pvcalls_enter_sock(sock);
1013         if (IS_ERR(map)) {
1014                 if (PTR_ERR(map) == -ENOTCONN)
1015                         return -EIO;
1016                 else
1017                         return 0;
1018         }
1019         bedata = dev_get_drvdata(&pvcalls_front_dev->dev);
1020
1021         spin_lock(&bedata->socket_lock);
1022         ret = get_request(bedata, &req_id);
1023         if (ret < 0) {
1024                 spin_unlock(&bedata->socket_lock);
1025                 pvcalls_exit_sock(sock);
1026                 return ret;
1027         }
1028         sock->sk->sk_send_head = NULL;
1029
1030         req = RING_GET_REQUEST(&bedata->ring, req_id);
1031         req->req_id = req_id;
1032         req->cmd = PVCALLS_RELEASE;
1033         req->u.release.id = (uintptr_t)map;
1034
1035         bedata->ring.req_prod_pvt++;
1036         RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&bedata->ring, notify);
1037         spin_unlock(&bedata->socket_lock);
1038         if (notify)
1039                 notify_remote_via_irq(bedata->irq);
1040
1041         wait_event(bedata->inflight_req,
1042                    READ_ONCE(bedata->rsp[req_id].req_id) == req_id);
1043
1044         if (map->active_socket) {
1045                 /*
1046                  * Set in_error and wake up inflight_conn_req to force
1047                  * recvmsg waiters to exit.
1048                  */
1049                 map->active.ring->in_error = -EBADF;
1050                 wake_up_interruptible(&map->active.inflight_conn_req);
1051
1052                 /*
1053                  * We need to make sure that sendmsg/recvmsg on this socket have
1054                  * not started before we've cleared sk_send_head here. The
1055                  * easiest way to guarantee this is to see that no pvcalls
1056                  * (other than us) is in progress on this socket.
1057                  */
1058                 while (atomic_read(&map->refcount) > 1)
1059                         cpu_relax();
1060
1061                 pvcalls_front_free_map(bedata, map);
1062         } else {
1063                 wake_up(&bedata->inflight_req);
1064                 wake_up(&map->passive.inflight_accept_req);
1065
1066                 while (atomic_read(&map->refcount) > 1)
1067                         cpu_relax();
1068
1069                 spin_lock(&bedata->socket_lock);
1070                 list_del(&map->list);
1071                 spin_unlock(&bedata->socket_lock);
1072                 if (READ_ONCE(map->passive.inflight_req_id) != PVCALLS_INVALID_ID &&
1073                         READ_ONCE(map->passive.inflight_req_id) != 0) {
1074                         pvcalls_front_free_map(bedata,
1075                                                map->passive.accept_map);
1076                 }
1077                 kfree(map);
1078         }
1079         WRITE_ONCE(bedata->rsp[req_id].req_id, PVCALLS_INVALID_ID);
1080
1081         pvcalls_exit();
1082         return 0;
1083 }
1084
1085 static const struct xenbus_device_id pvcalls_front_ids[] = {
1086         { "pvcalls" },
1087         { "" }
1088 };
1089
1090 static int pvcalls_front_remove(struct xenbus_device *dev)
1091 {
1092         struct pvcalls_bedata *bedata;
1093         struct sock_mapping *map = NULL, *n;
1094
1095         bedata = dev_get_drvdata(&pvcalls_front_dev->dev);
1096         dev_set_drvdata(&dev->dev, NULL);
1097         pvcalls_front_dev = NULL;
1098         if (bedata->irq >= 0)
1099                 unbind_from_irqhandler(bedata->irq, dev);
1100
1101         list_for_each_entry_safe(map, n, &bedata->socket_mappings, list) {
1102                 map->sock->sk->sk_send_head = NULL;
1103                 if (map->active_socket) {
1104                         map->active.ring->in_error = -EBADF;
1105                         wake_up_interruptible(&map->active.inflight_conn_req);
1106                 }
1107         }
1108
1109         smp_mb();
1110         while (atomic_read(&pvcalls_refcount) > 0)
1111                 cpu_relax();
1112         list_for_each_entry_safe(map, n, &bedata->socket_mappings, list) {
1113                 if (map->active_socket) {
1114                         /* No need to lock, refcount is 0 */
1115                         pvcalls_front_free_map(bedata, map);
1116                 } else {
1117                         list_del(&map->list);
1118                         kfree(map);
1119                 }
1120         }
1121         if (bedata->ref != -1)
1122                 gnttab_end_foreign_access(bedata->ref, 0, 0);
1123         kfree(bedata->ring.sring);
1124         kfree(bedata);
1125         xenbus_switch_state(dev, XenbusStateClosed);
1126         return 0;
1127 }
1128
1129 static int pvcalls_front_probe(struct xenbus_device *dev,
1130                           const struct xenbus_device_id *id)
1131 {
1132         int ret = -ENOMEM, evtchn, i;
1133         unsigned int max_page_order, function_calls, len;
1134         char *versions;
1135         grant_ref_t gref_head = 0;
1136         struct xenbus_transaction xbt;
1137         struct pvcalls_bedata *bedata = NULL;
1138         struct xen_pvcalls_sring *sring;
1139
1140         if (pvcalls_front_dev != NULL) {
1141                 dev_err(&dev->dev, "only one PV Calls connection supported\n");
1142                 return -EINVAL;
1143         }
1144
1145         versions = xenbus_read(XBT_NIL, dev->otherend, "versions", &len);
1146         if (IS_ERR(versions))
1147                 return PTR_ERR(versions);
1148         if (!len)
1149                 return -EINVAL;
1150         if (strcmp(versions, "1")) {
1151                 kfree(versions);
1152                 return -EINVAL;
1153         }
1154         kfree(versions);
1155         max_page_order = xenbus_read_unsigned(dev->otherend,
1156                                               "max-page-order", 0);
1157         if (max_page_order < PVCALLS_RING_ORDER)
1158                 return -ENODEV;
1159         function_calls = xenbus_read_unsigned(dev->otherend,
1160                                               "function-calls", 0);
1161         /* See XENBUS_FUNCTIONS_CALLS in pvcalls.h */
1162         if (function_calls != 1)
1163                 return -ENODEV;
1164         pr_info("%s max-page-order is %u\n", __func__, max_page_order);
1165
1166         bedata = kzalloc(sizeof(struct pvcalls_bedata), GFP_KERNEL);
1167         if (!bedata)
1168                 return -ENOMEM;
1169
1170         dev_set_drvdata(&dev->dev, bedata);
1171         pvcalls_front_dev = dev;
1172         init_waitqueue_head(&bedata->inflight_req);
1173         INIT_LIST_HEAD(&bedata->socket_mappings);
1174         spin_lock_init(&bedata->socket_lock);
1175         bedata->irq = -1;
1176         bedata->ref = -1;
1177
1178         for (i = 0; i < PVCALLS_NR_RSP_PER_RING; i++)
1179                 bedata->rsp[i].req_id = PVCALLS_INVALID_ID;
1180
1181         sring = (struct xen_pvcalls_sring *) __get_free_page(GFP_KERNEL |
1182                                                              __GFP_ZERO);
1183         if (!sring)
1184                 goto error;
1185         SHARED_RING_INIT(sring);
1186         FRONT_RING_INIT(&bedata->ring, sring, XEN_PAGE_SIZE);
1187
1188         ret = xenbus_alloc_evtchn(dev, &evtchn);
1189         if (ret)
1190                 goto error;
1191
1192         bedata->irq = bind_evtchn_to_irqhandler(evtchn,
1193                                                 pvcalls_front_event_handler,
1194                                                 0, "pvcalls-frontend", dev);
1195         if (bedata->irq < 0) {
1196                 ret = bedata->irq;
1197                 goto error;
1198         }
1199
1200         ret = gnttab_alloc_grant_references(1, &gref_head);
1201         if (ret < 0)
1202                 goto error;
1203         ret = gnttab_claim_grant_reference(&gref_head);
1204         if (ret < 0)
1205                 goto error;
1206         bedata->ref = ret;
1207         gnttab_grant_foreign_access_ref(bedata->ref, dev->otherend_id,
1208                                         virt_to_gfn((void *)sring), 0);
1209
1210  again:
1211         ret = xenbus_transaction_start(&xbt);
1212         if (ret) {
1213                 xenbus_dev_fatal(dev, ret, "starting transaction");
1214                 goto error;
1215         }
1216         ret = xenbus_printf(xbt, dev->nodename, "version", "%u", 1);
1217         if (ret)
1218                 goto error_xenbus;
1219         ret = xenbus_printf(xbt, dev->nodename, "ring-ref", "%d", bedata->ref);
1220         if (ret)
1221                 goto error_xenbus;
1222         ret = xenbus_printf(xbt, dev->nodename, "port", "%u",
1223                             evtchn);
1224         if (ret)
1225                 goto error_xenbus;
1226         ret = xenbus_transaction_end(xbt, 0);
1227         if (ret) {
1228                 if (ret == -EAGAIN)
1229                         goto again;
1230                 xenbus_dev_fatal(dev, ret, "completing transaction");
1231                 goto error;
1232         }
1233         xenbus_switch_state(dev, XenbusStateInitialised);
1234
1235         return 0;
1236
1237  error_xenbus:
1238         xenbus_transaction_end(xbt, 1);
1239         xenbus_dev_fatal(dev, ret, "writing xenstore");
1240  error:
1241         pvcalls_front_remove(dev);
1242         return ret;
1243 }
1244
1245 static void pvcalls_front_changed(struct xenbus_device *dev,
1246                             enum xenbus_state backend_state)
1247 {
1248         switch (backend_state) {
1249         case XenbusStateReconfiguring:
1250         case XenbusStateReconfigured:
1251         case XenbusStateInitialising:
1252         case XenbusStateInitialised:
1253         case XenbusStateUnknown:
1254                 break;
1255
1256         case XenbusStateInitWait:
1257                 break;
1258
1259         case XenbusStateConnected:
1260                 xenbus_switch_state(dev, XenbusStateConnected);
1261                 break;
1262
1263         case XenbusStateClosed:
1264                 if (dev->state == XenbusStateClosed)
1265                         break;
1266                 /* Missed the backend's CLOSING state */
1267                 /* fall through */
1268         case XenbusStateClosing:
1269                 xenbus_frontend_closed(dev);
1270                 break;
1271         }
1272 }
1273
1274 static struct xenbus_driver pvcalls_front_driver = {
1275         .ids = pvcalls_front_ids,
1276         .probe = pvcalls_front_probe,
1277         .remove = pvcalls_front_remove,
1278         .otherend_changed = pvcalls_front_changed,
1279 };
1280
1281 static int __init pvcalls_frontend_init(void)
1282 {
1283         if (!xen_domain())
1284                 return -ENODEV;
1285
1286         pr_info("Initialising Xen pvcalls frontend driver\n");
1287
1288         return xenbus_register_frontend(&pvcalls_front_driver);
1289 }
1290
1291 module_init(pvcalls_frontend_init);
1292
1293 MODULE_DESCRIPTION("Xen PV Calls frontend driver");
1294 MODULE_AUTHOR("Stefano Stabellini <sstabellini@kernel.org>");
1295 MODULE_LICENSE("GPL");