1 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
3 * Copyright (c) 2014 Raspberry Pi (Trading) Ltd. All rights reserved.
4 * Copyright (c) 2010-2012 Broadcom. All rights reserved.
7 #include <linux/cdev.h>
9 #include <linux/device.h>
10 #include <linux/slab.h>
11 #include <linux/compat.h>
12 #include <linux/miscdevice.h>
14 #include "vchiq_core.h"
15 #include "vchiq_ioctl.h"
16 #include "vchiq_arm.h"
17 #include "vchiq_debugfs.h"
19 static const char *const ioctl_names[] = {
25 "QUEUE_BULK_TRANSMIT",
40 static_assert(ARRAY_SIZE(ioctl_names) == (VCHIQ_IOC_MAX + 1));
43 user_service_free(void *userdata)
48 static void close_delivered(struct user_service *user_service)
50 vchiq_log_debug(user_service->service->state->dev, VCHIQ_ARM,
52 __func__, user_service->service->handle);
54 if (user_service->close_pending) {
55 /* Allow the underlying service to be culled */
56 vchiq_service_put(user_service->service);
58 /* Wake the user-thread blocked in close_ or remove_service */
59 complete(&user_service->close_event);
61 user_service->close_pending = 0;
65 struct vchiq_io_copy_callback_context {
66 struct vchiq_element *element;
67 size_t element_offset;
68 unsigned long elements_to_go;
71 static ssize_t vchiq_ioc_copy_element_data(void *context, void *dest,
72 size_t offset, size_t maxsize)
74 struct vchiq_io_copy_callback_context *cc = context;
75 size_t total_bytes_copied = 0;
76 size_t bytes_this_round;
78 while (total_bytes_copied < maxsize) {
79 if (!cc->elements_to_go)
80 return total_bytes_copied;
82 if (!cc->element->size) {
85 cc->element_offset = 0;
89 bytes_this_round = min(cc->element->size - cc->element_offset,
90 maxsize - total_bytes_copied);
92 if (copy_from_user(dest + total_bytes_copied,
93 cc->element->data + cc->element_offset,
97 cc->element_offset += bytes_this_round;
98 total_bytes_copied += bytes_this_round;
100 if (cc->element_offset == cc->element->size) {
101 cc->elements_to_go--;
103 cc->element_offset = 0;
111 vchiq_ioc_queue_message(struct vchiq_instance *instance, unsigned int handle,
112 struct vchiq_element *elements, unsigned long count)
114 struct vchiq_io_copy_callback_context context;
117 size_t total_size = 0;
119 context.element = elements;
120 context.element_offset = 0;
121 context.elements_to_go = count;
123 for (i = 0; i < count; i++) {
124 if (!elements[i].data && elements[i].size != 0)
127 total_size += elements[i].size;
130 status = vchiq_queue_message(instance, handle, vchiq_ioc_copy_element_data,
131 &context, total_size);
133 if (status == -EINVAL)
135 else if (status == -EAGAIN)
140 static int vchiq_ioc_create_service(struct vchiq_instance *instance,
141 struct vchiq_create_service *args)
143 struct user_service *user_service = NULL;
144 struct vchiq_service *service;
146 struct vchiq_service_params_kernel params;
149 if (args->is_open && !instance->connected)
152 user_service = kmalloc(sizeof(*user_service), GFP_KERNEL);
157 srvstate = VCHIQ_SRVSTATE_OPENING;
159 srvstate = instance->connected ?
160 VCHIQ_SRVSTATE_LISTENING : VCHIQ_SRVSTATE_HIDDEN;
163 params = (struct vchiq_service_params_kernel) {
164 .fourcc = args->params.fourcc,
165 .callback = service_callback,
166 .userdata = user_service,
167 .version = args->params.version,
168 .version_min = args->params.version_min,
170 service = vchiq_add_service_internal(instance->state, ¶ms,
178 user_service->service = service;
179 user_service->userdata = args->params.userdata;
180 user_service->instance = instance;
181 user_service->is_vchi = (args->is_vchi != 0);
182 user_service->dequeue_pending = 0;
183 user_service->close_pending = 0;
184 user_service->message_available_pos = instance->completion_remove - 1;
185 user_service->msg_insert = 0;
186 user_service->msg_remove = 0;
187 init_completion(&user_service->insert_event);
188 init_completion(&user_service->remove_event);
189 init_completion(&user_service->close_event);
192 status = vchiq_open_service_internal(service, instance->pid);
194 vchiq_remove_service(instance, service->handle);
195 return (status == -EAGAIN) ?
199 args->handle = service->handle;
204 static int vchiq_ioc_dequeue_message(struct vchiq_instance *instance,
205 struct vchiq_dequeue_message *args)
207 struct user_service *user_service;
208 struct vchiq_service *service;
209 struct vchiq_header *header;
212 DEBUG_INITIALISE(g_state.local);
213 DEBUG_TRACE(DEQUEUE_MESSAGE_LINE);
214 service = find_service_for_instance(instance, args->handle);
218 user_service = (struct user_service *)service->base.userdata;
219 if (user_service->is_vchi == 0) {
224 spin_lock(&msg_queue_spinlock);
225 if (user_service->msg_remove == user_service->msg_insert) {
226 if (!args->blocking) {
227 spin_unlock(&msg_queue_spinlock);
228 DEBUG_TRACE(DEQUEUE_MESSAGE_LINE);
232 user_service->dequeue_pending = 1;
235 spin_unlock(&msg_queue_spinlock);
236 DEBUG_TRACE(DEQUEUE_MESSAGE_LINE);
237 if (wait_for_completion_interruptible(&user_service->insert_event)) {
238 vchiq_log_debug(service->state->dev, VCHIQ_ARM,
239 "DEQUEUE_MESSAGE interrupted");
243 spin_lock(&msg_queue_spinlock);
244 } while (user_service->msg_remove == user_service->msg_insert);
250 if (WARN_ON_ONCE((int)(user_service->msg_insert -
251 user_service->msg_remove) < 0)) {
252 spin_unlock(&msg_queue_spinlock);
257 header = user_service->msg_queue[user_service->msg_remove &
258 (MSG_QUEUE_SIZE - 1)];
259 user_service->msg_remove++;
260 spin_unlock(&msg_queue_spinlock);
262 complete(&user_service->remove_event);
265 } else if (header->size <= args->bufsize) {
266 /* Copy to user space if msgbuf is not NULL */
267 if (!args->buf || (copy_to_user(args->buf, header->data, header->size) == 0)) {
269 vchiq_release_message(instance, service->handle, header);
274 vchiq_log_error(service->state->dev, VCHIQ_ARM,
275 "header %pK: bufsize %x < size %x",
276 header, args->bufsize, header->size);
277 WARN(1, "invalid size\n");
280 DEBUG_TRACE(DEQUEUE_MESSAGE_LINE);
282 vchiq_service_put(service);
286 static int vchiq_irq_queue_bulk_tx_rx(struct vchiq_instance *instance,
287 struct vchiq_queue_bulk_transfer *args,
288 enum vchiq_bulk_dir dir,
289 enum vchiq_bulk_mode __user *mode)
291 struct vchiq_service *service;
292 struct bulk_waiter_node *waiter = NULL, *iter;
297 service = find_service_for_instance(instance, args->handle);
301 if (args->mode == VCHIQ_BULK_MODE_BLOCKING) {
302 waiter = kzalloc(sizeof(*waiter), GFP_KERNEL);
308 userdata = &waiter->bulk_waiter;
309 } else if (args->mode == VCHIQ_BULK_MODE_WAITING) {
310 mutex_lock(&instance->bulk_waiter_list_mutex);
311 list_for_each_entry(iter, &instance->bulk_waiter_list,
313 if (iter->pid == current->pid) {
314 list_del(&iter->list);
319 mutex_unlock(&instance->bulk_waiter_list_mutex);
321 vchiq_log_error(service->state->dev, VCHIQ_ARM,
322 "no bulk_waiter found for pid %d", current->pid);
326 vchiq_log_debug(service->state->dev, VCHIQ_ARM,
327 "found bulk_waiter %pK for pid %d", waiter, current->pid);
328 userdata = &waiter->bulk_waiter;
330 userdata = args->userdata;
333 status = vchiq_bulk_transfer(instance, args->handle, NULL, args->data, args->size,
334 userdata, args->mode, dir);
341 if ((status != -EAGAIN) || fatal_signal_pending(current) ||
342 !waiter->bulk_waiter.bulk) {
343 if (waiter->bulk_waiter.bulk) {
344 /* Cancel the signal when the transfer completes. */
345 spin_lock(&bulk_waiter_spinlock);
346 waiter->bulk_waiter.bulk->userdata = NULL;
347 spin_unlock(&bulk_waiter_spinlock);
352 const enum vchiq_bulk_mode mode_waiting =
353 VCHIQ_BULK_MODE_WAITING;
354 waiter->pid = current->pid;
355 mutex_lock(&instance->bulk_waiter_list_mutex);
356 list_add(&waiter->list, &instance->bulk_waiter_list);
357 mutex_unlock(&instance->bulk_waiter_list_mutex);
358 vchiq_log_debug(service->state->dev, VCHIQ_ARM,
359 "saved bulk_waiter %pK for pid %d", waiter, current->pid);
361 ret = put_user(mode_waiting, mode);
364 vchiq_service_put(service);
367 else if (status == -EINVAL)
369 else if (status == -EAGAIN)
374 /* read a user pointer value from an array pointers in user space */
375 static inline int vchiq_get_user_ptr(void __user **buf, void __user *ubuf, int index)
379 if (in_compat_syscall()) {
381 compat_uptr_t __user *uptr = ubuf;
383 ret = get_user(ptr32, uptr + index);
387 *buf = compat_ptr(ptr32);
389 uintptr_t ptr, __user *uptr = ubuf;
391 ret = get_user(ptr, uptr + index);
396 *buf = (void __user *)ptr;
402 struct vchiq_completion_data32 {
403 enum vchiq_reason reason;
404 compat_uptr_t header;
405 compat_uptr_t service_userdata;
406 compat_uptr_t bulk_userdata;
409 static int vchiq_put_completion(struct vchiq_completion_data __user *buf,
410 struct vchiq_completion_data *completion,
413 struct vchiq_completion_data32 __user *buf32 = (void __user *)buf;
415 if (in_compat_syscall()) {
416 struct vchiq_completion_data32 tmp = {
417 .reason = completion->reason,
418 .header = ptr_to_compat(completion->header),
419 .service_userdata = ptr_to_compat(completion->service_userdata),
420 .bulk_userdata = ptr_to_compat(completion->bulk_userdata),
422 if (copy_to_user(&buf32[index], &tmp, sizeof(tmp)))
425 if (copy_to_user(&buf[index], completion, sizeof(*completion)))
432 static int vchiq_ioc_await_completion(struct vchiq_instance *instance,
433 struct vchiq_await_completion *args,
434 int __user *msgbufcountp)
440 DEBUG_INITIALISE(g_state.local);
442 DEBUG_TRACE(AWAIT_COMPLETION_LINE);
443 if (!instance->connected)
446 mutex_lock(&instance->completion_mutex);
448 DEBUG_TRACE(AWAIT_COMPLETION_LINE);
449 while ((instance->completion_remove == instance->completion_insert) && !instance->closing) {
452 DEBUG_TRACE(AWAIT_COMPLETION_LINE);
453 mutex_unlock(&instance->completion_mutex);
454 rc = wait_for_completion_interruptible(&instance->insert_event);
455 mutex_lock(&instance->completion_mutex);
457 DEBUG_TRACE(AWAIT_COMPLETION_LINE);
458 vchiq_log_debug(instance->state->dev, VCHIQ_ARM,
459 "AWAIT_COMPLETION interrupted");
464 DEBUG_TRACE(AWAIT_COMPLETION_LINE);
466 msgbufcount = args->msgbufcount;
467 remove = instance->completion_remove;
469 for (ret = 0; ret < args->count; ret++) {
470 struct vchiq_completion_data_kernel *completion;
471 struct vchiq_completion_data user_completion;
472 struct vchiq_service *service;
473 struct user_service *user_service;
474 struct vchiq_header *header;
476 if (remove == instance->completion_insert)
479 completion = &instance->completions[remove & (MAX_COMPLETIONS - 1)];
482 * A read memory barrier is needed to stop
483 * prefetch of a stale completion record
487 service = completion->service_userdata;
488 user_service = service->base.userdata;
490 memset(&user_completion, 0, sizeof(user_completion));
491 user_completion = (struct vchiq_completion_data) {
492 .reason = completion->reason,
493 .service_userdata = user_service->userdata,
496 header = completion->header;
501 msglen = header->size + sizeof(struct vchiq_header);
502 /* This must be a VCHIQ-style service */
503 if (args->msgbufsize < msglen) {
504 vchiq_log_error(service->state->dev, VCHIQ_ARM,
505 "header %pK: msgbufsize %x < msglen %x",
506 header, args->msgbufsize, msglen);
507 WARN(1, "invalid message size\n");
512 if (msgbufcount <= 0)
513 /* Stall here for lack of a buffer for the message. */
515 /* Get the pointer from user space */
517 if (vchiq_get_user_ptr(&msgbuf, args->msgbufs,
524 /* Copy the message to user space */
525 if (copy_to_user(msgbuf, header, msglen)) {
531 /* Now it has been copied, the message can be released. */
532 vchiq_release_message(instance, service->handle, header);
534 /* The completion must point to the msgbuf. */
535 user_completion.header = msgbuf;
538 if ((completion->reason == VCHIQ_SERVICE_CLOSED) &&
539 !instance->use_close_delivered)
540 vchiq_service_put(service);
543 * FIXME: address space mismatch, does bulk_userdata
544 * actually point to user or kernel memory?
546 user_completion.bulk_userdata = completion->bulk_userdata;
548 if (vchiq_put_completion(args->buf, &user_completion, ret)) {
555 * Ensure that the above copy has completed
556 * before advancing the remove pointer.
560 instance->completion_remove = remove;
563 if (msgbufcount != args->msgbufcount) {
564 if (put_user(msgbufcount, msgbufcountp))
569 complete(&instance->remove_event);
570 mutex_unlock(&instance->completion_mutex);
571 DEBUG_TRACE(AWAIT_COMPLETION_LINE);
577 vchiq_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
579 struct vchiq_instance *instance = file->private_data;
581 struct vchiq_service *service = NULL;
585 vchiq_log_trace(instance->state->dev, VCHIQ_ARM,
586 "%s - instance %pK, cmd %s, arg %lx", __func__, instance,
587 ((_IOC_TYPE(cmd) == VCHIQ_IOC_MAGIC) && (_IOC_NR(cmd) <= VCHIQ_IOC_MAX)) ?
588 ioctl_names[_IOC_NR(cmd)] : "<invalid>", arg);
591 case VCHIQ_IOC_SHUTDOWN:
592 if (!instance->connected)
595 /* Remove all services */
597 while ((service = next_service_by_instance(instance->state,
599 status = vchiq_remove_service(instance, service->handle);
600 vchiq_service_put(service);
607 /* Wake the completion thread and ask it to exit */
608 instance->closing = 1;
609 complete(&instance->insert_event);
614 case VCHIQ_IOC_CONNECT:
615 if (instance->connected) {
619 rc = mutex_lock_killable(&instance->state->mutex);
621 vchiq_log_error(instance->state->dev, VCHIQ_ARM,
622 "vchiq: connect: could not lock mutex for state %d: %d",
623 instance->state->id, rc);
627 status = vchiq_connect_internal(instance->state, instance);
628 mutex_unlock(&instance->state->mutex);
631 instance->connected = 1;
633 vchiq_log_error(instance->state->dev, VCHIQ_ARM,
634 "vchiq: could not connect: %d", status);
637 case VCHIQ_IOC_CREATE_SERVICE: {
638 struct vchiq_create_service __user *argp;
639 struct vchiq_create_service args;
641 argp = (void __user *)arg;
642 if (copy_from_user(&args, argp, sizeof(args))) {
647 ret = vchiq_ioc_create_service(instance, &args);
651 if (put_user(args.handle, &argp->handle)) {
652 vchiq_remove_service(instance, args.handle);
657 case VCHIQ_IOC_CLOSE_SERVICE:
658 case VCHIQ_IOC_REMOVE_SERVICE: {
659 unsigned int handle = (unsigned int)arg;
660 struct user_service *user_service;
662 service = find_service_for_instance(instance, handle);
668 user_service = service->base.userdata;
671 * close_pending is false on first entry, and when the
672 * wait in vchiq_close_service has been interrupted.
674 if (!user_service->close_pending) {
675 status = (cmd == VCHIQ_IOC_CLOSE_SERVICE) ?
676 vchiq_close_service(instance, service->handle) :
677 vchiq_remove_service(instance, service->handle);
683 * close_pending is true once the underlying service
684 * has been closed until the client library calls the
685 * CLOSE_DELIVERED ioctl, signalling close_event.
687 if (user_service->close_pending &&
688 wait_for_completion_interruptible(&user_service->close_event))
693 case VCHIQ_IOC_USE_SERVICE:
694 case VCHIQ_IOC_RELEASE_SERVICE: {
695 unsigned int handle = (unsigned int)arg;
697 service = find_service_for_instance(instance, handle);
699 ret = (cmd == VCHIQ_IOC_USE_SERVICE) ?
700 vchiq_use_service_internal(service) :
701 vchiq_release_service_internal(service);
703 vchiq_log_error(instance->state->dev, VCHIQ_SUSPEND,
704 "%s: cmd %s returned error %ld for service %p4cc:%03d",
705 __func__, (cmd == VCHIQ_IOC_USE_SERVICE) ?
706 "VCHIQ_IOC_USE_SERVICE" :
707 "VCHIQ_IOC_RELEASE_SERVICE",
708 ret, &service->base.fourcc,
716 case VCHIQ_IOC_QUEUE_MESSAGE: {
717 struct vchiq_queue_message args;
719 if (copy_from_user(&args, (const void __user *)arg,
725 service = find_service_for_instance(instance, args.handle);
727 if (service && (args.count <= MAX_ELEMENTS)) {
728 /* Copy elements into kernel space */
729 struct vchiq_element elements[MAX_ELEMENTS];
731 if (copy_from_user(elements, args.elements,
732 args.count * sizeof(struct vchiq_element)) == 0)
733 ret = vchiq_ioc_queue_message(instance, args.handle, elements,
742 case VCHIQ_IOC_QUEUE_BULK_TRANSMIT:
743 case VCHIQ_IOC_QUEUE_BULK_RECEIVE: {
744 struct vchiq_queue_bulk_transfer args;
745 struct vchiq_queue_bulk_transfer __user *argp;
747 enum vchiq_bulk_dir dir =
748 (cmd == VCHIQ_IOC_QUEUE_BULK_TRANSMIT) ?
749 VCHIQ_BULK_TRANSMIT : VCHIQ_BULK_RECEIVE;
751 argp = (void __user *)arg;
752 if (copy_from_user(&args, argp, sizeof(args))) {
757 ret = vchiq_irq_queue_bulk_tx_rx(instance, &args,
761 case VCHIQ_IOC_AWAIT_COMPLETION: {
762 struct vchiq_await_completion args;
763 struct vchiq_await_completion __user *argp;
765 argp = (void __user *)arg;
766 if (copy_from_user(&args, argp, sizeof(args))) {
771 ret = vchiq_ioc_await_completion(instance, &args,
775 case VCHIQ_IOC_DEQUEUE_MESSAGE: {
776 struct vchiq_dequeue_message args;
778 if (copy_from_user(&args, (const void __user *)arg,
784 ret = vchiq_ioc_dequeue_message(instance, &args);
787 case VCHIQ_IOC_GET_CLIENT_ID: {
788 unsigned int handle = (unsigned int)arg;
790 ret = vchiq_get_client_id(instance, handle);
793 case VCHIQ_IOC_GET_CONFIG: {
794 struct vchiq_get_config args;
795 struct vchiq_config config;
797 if (copy_from_user(&args, (const void __user *)arg,
802 if (args.config_size > sizeof(config)) {
807 vchiq_get_config(&config);
808 if (copy_to_user(args.pconfig, &config, args.config_size)) {
814 case VCHIQ_IOC_SET_SERVICE_OPTION: {
815 struct vchiq_set_service_option args;
817 if (copy_from_user(&args, (const void __user *)arg,
823 service = find_service_for_instance(instance, args.handle);
829 ret = vchiq_set_service_option(instance, args.handle, args.option,
833 case VCHIQ_IOC_LIB_VERSION: {
834 unsigned int lib_version = (unsigned int)arg;
836 if (lib_version < VCHIQ_VERSION_MIN)
838 else if (lib_version >= VCHIQ_VERSION_CLOSE_DELIVERED)
839 instance->use_close_delivered = 1;
842 case VCHIQ_IOC_CLOSE_DELIVERED: {
843 unsigned int handle = (unsigned int)arg;
845 service = find_closed_service_for_instance(instance, handle);
847 struct user_service *user_service =
848 (struct user_service *)service->base.userdata;
849 close_delivered(user_service);
861 vchiq_service_put(service);
864 if (status == -EINVAL)
866 else if (status == -EAGAIN)
870 if (!status && (ret < 0) && (ret != -EINTR) && (ret != -EWOULDBLOCK)) {
871 vchiq_log_debug(instance->state->dev, VCHIQ_ARM,
872 " ioctl instance %pK, cmd %s -> status %d, %ld",
873 instance, (_IOC_NR(cmd) <= VCHIQ_IOC_MAX) ?
874 ioctl_names[_IOC_NR(cmd)] : "<invalid>", status, ret);
876 vchiq_log_trace(instance->state->dev, VCHIQ_ARM,
877 " ioctl instance %pK, cmd %s -> status %d, %ld",
878 instance, (_IOC_NR(cmd) <= VCHIQ_IOC_MAX) ?
879 ioctl_names[_IOC_NR(cmd)] : "<invalid>", status, ret);
885 #if defined(CONFIG_COMPAT)
887 struct vchiq_service_params32 {
889 compat_uptr_t callback;
890 compat_uptr_t userdata;
891 short version; /* Increment for non-trivial changes */
892 short version_min; /* Update for incompatible changes */
895 struct vchiq_create_service32 {
896 struct vchiq_service_params32 params;
899 unsigned int handle; /* OUT */
902 #define VCHIQ_IOC_CREATE_SERVICE32 \
903 _IOWR(VCHIQ_IOC_MAGIC, 2, struct vchiq_create_service32)
906 vchiq_compat_ioctl_create_service(struct file *file, unsigned int cmd,
907 struct vchiq_create_service32 __user *ptrargs32)
909 struct vchiq_create_service args;
910 struct vchiq_create_service32 args32;
911 struct vchiq_instance *instance = file->private_data;
914 if (copy_from_user(&args32, ptrargs32, sizeof(args32)))
917 args = (struct vchiq_create_service) {
919 .fourcc = args32.params.fourcc,
920 .callback = compat_ptr(args32.params.callback),
921 .userdata = compat_ptr(args32.params.userdata),
922 .version = args32.params.version,
923 .version_min = args32.params.version_min,
925 .is_open = args32.is_open,
926 .is_vchi = args32.is_vchi,
927 .handle = args32.handle,
930 ret = vchiq_ioc_create_service(instance, &args);
934 if (put_user(args.handle, &ptrargs32->handle)) {
935 vchiq_remove_service(instance, args.handle);
942 struct vchiq_element32 {
947 struct vchiq_queue_message32 {
950 compat_uptr_t elements;
953 #define VCHIQ_IOC_QUEUE_MESSAGE32 \
954 _IOW(VCHIQ_IOC_MAGIC, 4, struct vchiq_queue_message32)
957 vchiq_compat_ioctl_queue_message(struct file *file,
959 struct vchiq_queue_message32 __user *arg)
961 struct vchiq_queue_message args;
962 struct vchiq_queue_message32 args32;
963 struct vchiq_service *service;
964 struct vchiq_instance *instance = file->private_data;
967 if (copy_from_user(&args32, arg, sizeof(args32)))
970 args = (struct vchiq_queue_message) {
971 .handle = args32.handle,
972 .count = args32.count,
973 .elements = compat_ptr(args32.elements),
976 if (args32.count > MAX_ELEMENTS)
979 service = find_service_for_instance(instance, args.handle);
983 if (args32.elements && args32.count) {
984 struct vchiq_element32 element32[MAX_ELEMENTS];
985 struct vchiq_element elements[MAX_ELEMENTS];
988 if (copy_from_user(&element32, args.elements,
989 sizeof(element32))) {
990 vchiq_service_put(service);
994 for (count = 0; count < args32.count; count++) {
995 elements[count].data =
996 compat_ptr(element32[count].data);
997 elements[count].size = element32[count].size;
999 ret = vchiq_ioc_queue_message(instance, args.handle, elements,
1004 vchiq_service_put(service);
1009 struct vchiq_queue_bulk_transfer32 {
1010 unsigned int handle;
1013 compat_uptr_t userdata;
1014 enum vchiq_bulk_mode mode;
1017 #define VCHIQ_IOC_QUEUE_BULK_TRANSMIT32 \
1018 _IOWR(VCHIQ_IOC_MAGIC, 5, struct vchiq_queue_bulk_transfer32)
1019 #define VCHIQ_IOC_QUEUE_BULK_RECEIVE32 \
1020 _IOWR(VCHIQ_IOC_MAGIC, 6, struct vchiq_queue_bulk_transfer32)
1023 vchiq_compat_ioctl_queue_bulk(struct file *file,
1025 struct vchiq_queue_bulk_transfer32 __user *argp)
1027 struct vchiq_queue_bulk_transfer32 args32;
1028 struct vchiq_queue_bulk_transfer args;
1029 enum vchiq_bulk_dir dir = (cmd == VCHIQ_IOC_QUEUE_BULK_TRANSMIT32) ?
1030 VCHIQ_BULK_TRANSMIT : VCHIQ_BULK_RECEIVE;
1032 if (copy_from_user(&args32, argp, sizeof(args32)))
1035 args = (struct vchiq_queue_bulk_transfer) {
1036 .handle = args32.handle,
1037 .data = compat_ptr(args32.data),
1038 .size = args32.size,
1039 .userdata = compat_ptr(args32.userdata),
1040 .mode = args32.mode,
1043 return vchiq_irq_queue_bulk_tx_rx(file->private_data, &args,
1047 struct vchiq_await_completion32 {
1050 unsigned int msgbufsize;
1051 unsigned int msgbufcount; /* IN/OUT */
1052 compat_uptr_t msgbufs;
1055 #define VCHIQ_IOC_AWAIT_COMPLETION32 \
1056 _IOWR(VCHIQ_IOC_MAGIC, 7, struct vchiq_await_completion32)
1059 vchiq_compat_ioctl_await_completion(struct file *file,
1061 struct vchiq_await_completion32 __user *argp)
1063 struct vchiq_await_completion args;
1064 struct vchiq_await_completion32 args32;
1066 if (copy_from_user(&args32, argp, sizeof(args32)))
1069 args = (struct vchiq_await_completion) {
1070 .count = args32.count,
1071 .buf = compat_ptr(args32.buf),
1072 .msgbufsize = args32.msgbufsize,
1073 .msgbufcount = args32.msgbufcount,
1074 .msgbufs = compat_ptr(args32.msgbufs),
1077 return vchiq_ioc_await_completion(file->private_data, &args,
1078 &argp->msgbufcount);
1081 struct vchiq_dequeue_message32 {
1082 unsigned int handle;
1084 unsigned int bufsize;
1088 #define VCHIQ_IOC_DEQUEUE_MESSAGE32 \
1089 _IOWR(VCHIQ_IOC_MAGIC, 8, struct vchiq_dequeue_message32)
1092 vchiq_compat_ioctl_dequeue_message(struct file *file,
1094 struct vchiq_dequeue_message32 __user *arg)
1096 struct vchiq_dequeue_message32 args32;
1097 struct vchiq_dequeue_message args;
1099 if (copy_from_user(&args32, arg, sizeof(args32)))
1102 args = (struct vchiq_dequeue_message) {
1103 .handle = args32.handle,
1104 .blocking = args32.blocking,
1105 .bufsize = args32.bufsize,
1106 .buf = compat_ptr(args32.buf),
1109 return vchiq_ioc_dequeue_message(file->private_data, &args);
1112 struct vchiq_get_config32 {
1113 unsigned int config_size;
1114 compat_uptr_t pconfig;
1117 #define VCHIQ_IOC_GET_CONFIG32 \
1118 _IOWR(VCHIQ_IOC_MAGIC, 10, struct vchiq_get_config32)
1121 vchiq_compat_ioctl_get_config(struct file *file,
1123 struct vchiq_get_config32 __user *arg)
1125 struct vchiq_get_config32 args32;
1126 struct vchiq_config config;
1129 if (copy_from_user(&args32, arg, sizeof(args32)))
1131 if (args32.config_size > sizeof(config))
1134 vchiq_get_config(&config);
1135 ptr = compat_ptr(args32.pconfig);
1136 if (copy_to_user(ptr, &config, args32.config_size))
1143 vchiq_compat_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
1145 void __user *argp = compat_ptr(arg);
1148 case VCHIQ_IOC_CREATE_SERVICE32:
1149 return vchiq_compat_ioctl_create_service(file, cmd, argp);
1150 case VCHIQ_IOC_QUEUE_MESSAGE32:
1151 return vchiq_compat_ioctl_queue_message(file, cmd, argp);
1152 case VCHIQ_IOC_QUEUE_BULK_TRANSMIT32:
1153 case VCHIQ_IOC_QUEUE_BULK_RECEIVE32:
1154 return vchiq_compat_ioctl_queue_bulk(file, cmd, argp);
1155 case VCHIQ_IOC_AWAIT_COMPLETION32:
1156 return vchiq_compat_ioctl_await_completion(file, cmd, argp);
1157 case VCHIQ_IOC_DEQUEUE_MESSAGE32:
1158 return vchiq_compat_ioctl_dequeue_message(file, cmd, argp);
1159 case VCHIQ_IOC_GET_CONFIG32:
1160 return vchiq_compat_ioctl_get_config(file, cmd, argp);
1162 return vchiq_ioctl(file, cmd, (unsigned long)argp);
1168 static int vchiq_open(struct inode *inode, struct file *file)
1170 struct vchiq_state *state = vchiq_get_state();
1171 struct vchiq_instance *instance;
1173 vchiq_log_debug(state->dev, VCHIQ_ARM, "vchiq_open");
1176 vchiq_log_error(state->dev, VCHIQ_ARM,
1177 "vchiq has no connection to VideoCore");
1181 instance = kzalloc(sizeof(*instance), GFP_KERNEL);
1185 instance->state = state;
1186 instance->pid = current->tgid;
1188 vchiq_debugfs_add_instance(instance);
1190 init_completion(&instance->insert_event);
1191 init_completion(&instance->remove_event);
1192 mutex_init(&instance->completion_mutex);
1193 mutex_init(&instance->bulk_waiter_list_mutex);
1194 INIT_LIST_HEAD(&instance->bulk_waiter_list);
1196 file->private_data = instance;
1201 static int vchiq_release(struct inode *inode, struct file *file)
1203 struct vchiq_instance *instance = file->private_data;
1204 struct vchiq_state *state = vchiq_get_state();
1205 struct vchiq_service *service;
1209 vchiq_log_debug(state->dev, VCHIQ_ARM, "%s: instance=%lx", __func__,
1210 (unsigned long)instance);
1217 /* Ensure videocore is awake to allow termination. */
1218 vchiq_use_internal(instance->state, NULL, USE_TYPE_VCHIQ);
1220 mutex_lock(&instance->completion_mutex);
1222 /* Wake the completion thread and ask it to exit */
1223 instance->closing = 1;
1224 complete(&instance->insert_event);
1226 mutex_unlock(&instance->completion_mutex);
1228 /* Wake the slot handler if the completion queue is full. */
1229 complete(&instance->remove_event);
1231 /* Mark all services for termination... */
1233 while ((service = next_service_by_instance(state, instance, &i))) {
1234 struct user_service *user_service = service->base.userdata;
1236 /* Wake the slot handler if the msg queue is full. */
1237 complete(&user_service->remove_event);
1239 vchiq_terminate_service_internal(service);
1240 vchiq_service_put(service);
1243 /* ...and wait for them to die */
1245 while ((service = next_service_by_instance(state, instance, &i))) {
1246 struct user_service *user_service = service->base.userdata;
1248 wait_for_completion(&service->remove_event);
1250 if (WARN_ON(service->srvstate != VCHIQ_SRVSTATE_FREE)) {
1251 vchiq_service_put(service);
1255 spin_lock(&msg_queue_spinlock);
1257 while (user_service->msg_remove != user_service->msg_insert) {
1258 struct vchiq_header *header;
1259 int m = user_service->msg_remove & (MSG_QUEUE_SIZE - 1);
1261 header = user_service->msg_queue[m];
1262 user_service->msg_remove++;
1263 spin_unlock(&msg_queue_spinlock);
1266 vchiq_release_message(instance, service->handle, header);
1267 spin_lock(&msg_queue_spinlock);
1270 spin_unlock(&msg_queue_spinlock);
1272 vchiq_service_put(service);
1275 /* Release any closed services */
1276 while (instance->completion_remove != instance->completion_insert) {
1277 struct vchiq_completion_data_kernel *completion;
1278 struct vchiq_service *service;
1280 completion = &instance->completions[instance->completion_remove
1281 & (MAX_COMPLETIONS - 1)];
1282 service = completion->service_userdata;
1283 if (completion->reason == VCHIQ_SERVICE_CLOSED) {
1284 struct user_service *user_service =
1285 service->base.userdata;
1287 /* Wake any blocked user-thread */
1288 if (instance->use_close_delivered)
1289 complete(&user_service->close_event);
1290 vchiq_service_put(service);
1292 instance->completion_remove++;
1295 /* Release the PEER service count. */
1296 vchiq_release_internal(instance->state, NULL);
1298 free_bulk_waiter(instance);
1300 vchiq_debugfs_remove_instance(instance);
1303 file->private_data = NULL;
1310 vchiq_read(struct file *file, char __user *buf, size_t count, loff_t *ppos)
1312 struct dump_context context;
1317 context.space = count;
1318 context.offset = *ppos;
1320 err = vchiq_dump_state(&context, &g_state);
1324 *ppos += context.actual;
1326 return context.actual;
1329 static const struct file_operations
1331 .owner = THIS_MODULE,
1332 .unlocked_ioctl = vchiq_ioctl,
1333 #if defined(CONFIG_COMPAT)
1334 .compat_ioctl = vchiq_compat_ioctl,
1337 .release = vchiq_release,
1341 static struct miscdevice vchiq_miscdev = {
1342 .fops = &vchiq_fops,
1343 .minor = MISC_DYNAMIC_MINOR,
1349 * vchiq_register_chrdev - Register the char driver for vchiq
1350 * and create the necessary class and
1351 * device files in userspace.
1352 * @parent The parent of the char device.
1354 * Returns 0 on success else returns the error code.
1356 int vchiq_register_chrdev(struct device *parent)
1358 vchiq_miscdev.parent = parent;
1360 return misc_register(&vchiq_miscdev);
1364 * vchiq_deregister_chrdev - Deregister and cleanup the vchiq char
1365 * driver and device files
1367 void vchiq_deregister_chrdev(void)
1369 misc_deregister(&vchiq_miscdev);