drivers/gpu/drm/drm_syncobj.c

   1 /*
   2  * Copyright 2017 Red Hat
   3  * Parts ported from amdgpu (fence wait code).
   4  * Copyright 2016 Advanced Micro Devices, Inc.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the "Software"),
   8  * to deal in the Software without restriction, including without limitation
   9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  10  * and/or sell copies of the Software, and to permit persons to whom the
  11  * Software is furnished to do so, subject to the following conditions:
  12  *
  13  * The above copyright notice and this permission notice (including the next
  14  * paragraph) shall be included in all copies or substantial portions of the
  15  * Software.
  16  *
  17  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  18  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  19  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  20  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  21  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  22  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  23  * IN THE SOFTWARE.
  24  *
  25  * Authors:
  26  *
  27  */
  28
  29 /**
  30  * DOC: Overview
  31  *
  32  * DRM synchronisation objects (syncobj, see struct &drm_syncobj) are
  33  * persistent objects that contain an optional fence. The fence can be updated
  34  * with a new fence, or be NULL.
  35  *
  36  * syncobj's can be waited upon, where it will wait for the underlying
  37  * fence.
  38  *
  39  * syncobj's can be export to fd's and back, these fd's are opaque and
  40  * have no other use case, except passing the syncobj between processes.
  41  *
  42  * Their primary use-case is to implement Vulkan fences and semaphores.
  43  *
  44  * syncobj have a kref reference count, but also have an optional file.
  45  * The file is only created once the syncobj is exported.
  46  * The file takes a reference on the kref.
  47  */
  48
  49 #include <drm/drmP.h>
  50 #include <linux/file.h>
  51 #include <linux/fs.h>
  52 #include <linux/anon_inodes.h>
  53 #include <linux/sync_file.h>
  54 #include <linux/sched/signal.h>
  55
  56 #include "drm_internal.h"
  57 #include <drm/drm_syncobj.h>
  58
  59 /**
  60  * drm_syncobj_find - lookup and reference a sync object.
  61  * @file_private: drm file private pointer
  62  * @handle: sync object handle to lookup.
  63  *
  64  * Returns a reference to the syncobj pointed to by handle or NULL. The
  65  * reference must be released by calling drm_syncobj_put().
  66  */
  67 struct drm_syncobj *drm_syncobj_find(struct drm_file *file_private,
  68                                      u32 handle)
  69 {
  70         struct drm_syncobj *syncobj;
  71
  72         spin_lock(&file_private->syncobj_table_lock);
  73
  74         /* Check if we currently have a reference on the object */
  75         syncobj = idr_find(&file_private->syncobj_idr, handle);
  76         if (syncobj)
  77                 drm_syncobj_get(syncobj);
  78
  79         spin_unlock(&file_private->syncobj_table_lock);
  80
  81         return syncobj;
  82 }
  83 EXPORT_SYMBOL(drm_syncobj_find);
  84
  85 static void drm_syncobj_add_callback_locked(struct drm_syncobj *syncobj,
  86                                             struct drm_syncobj_cb *cb,
  87                                             drm_syncobj_func_t func)
  88 {
  89         cb->func = func;
  90         list_add_tail(&cb->node, &syncobj->cb_list);
  91 }
  92
  93 static int drm_syncobj_fence_get_or_add_callback(struct drm_syncobj *syncobj,
  94                                                  struct dma_fence **fence,
  95                                                  struct drm_syncobj_cb *cb,
  96                                                  drm_syncobj_func_t func)
  97 {
  98         int ret;
  99
 100         *fence = drm_syncobj_fence_get(syncobj);
 101         if (*fence)
 102                 return 1;
 103
 104         spin_lock(&syncobj->lock);
 105         /* We've already tried once to get a fence and failed.  Now that we
 106          * have the lock, try one more time just to be sure we don't add a
 107          * callback when a fence has already been set.
 108          */
 109         if (syncobj->fence) {
 110                 *fence = dma_fence_get(rcu_dereference_protected(syncobj->fence,
 111                                                                  lockdep_is_held(&syncobj->lock)));
 112                 ret = 1;
 113         } else {
 114                 *fence = NULL;
 115                 drm_syncobj_add_callback_locked(syncobj, cb, func);
 116                 ret = 0;
 117         }
 118         spin_unlock(&syncobj->lock);
 119
 120         return ret;
 121 }
 122
 123 void drm_syncobj_add_callback(struct drm_syncobj *syncobj,
 124                               struct drm_syncobj_cb *cb,
 125                               drm_syncobj_func_t func)
 126 {
 127         spin_lock(&syncobj->lock);
 128         drm_syncobj_add_callback_locked(syncobj, cb, func);
 129         spin_unlock(&syncobj->lock);
 130 }
 131
 132 void drm_syncobj_remove_callback(struct drm_syncobj *syncobj,
 133                                  struct drm_syncobj_cb *cb)
 134 {
 135         spin_lock(&syncobj->lock);
 136         list_del_init(&cb->node);
 137         spin_unlock(&syncobj->lock);
 138 }
 139
 140 /**
 141  * drm_syncobj_replace_fence - replace fence in a sync object.
 142  * @syncobj: Sync object to replace fence in
 143  * @fence: fence to install in sync file.
 144  *
 145  * This replaces the fence on a sync object.
 146  */
 147 void drm_syncobj_replace_fence(struct drm_syncobj *syncobj,
 148                                struct dma_fence *fence)
 149 {
 150         struct dma_fence *old_fence;
 151         struct drm_syncobj_cb *cur, *tmp;
 152
 153         if (fence)
 154                 dma_fence_get(fence);
 155
 156         spin_lock(&syncobj->lock);
 157
 158         old_fence = rcu_dereference_protected(syncobj->fence,
 159                                               lockdep_is_held(&syncobj->lock));
 160         rcu_assign_pointer(syncobj->fence, fence);
 161
 162         if (fence != old_fence) {
 163                 list_for_each_entry_safe(cur, tmp, &syncobj->cb_list, node) {
 164                         list_del_init(&cur->node);
 165                         cur->func(syncobj, cur);
 166                 }
 167         }
 168
 169         spin_unlock(&syncobj->lock);
 170
 171         dma_fence_put(old_fence);
 172 }
 173 EXPORT_SYMBOL(drm_syncobj_replace_fence);
 174
 175 /**
 176  * drm_syncobj_assign_null_handle - assign a stub fence to the sync object
 177  * @syncobj: sync object to assign the fence on
 178  *
 179  * Assign a already signaled stub fence to the sync object.
 180  */
 181 static void drm_syncobj_assign_null_handle(struct drm_syncobj *syncobj)
 182 {
 183         struct dma_fence *fence = dma_fence_get_stub();
 184
 185         drm_syncobj_replace_fence(syncobj, fence);
 186         dma_fence_put(fence);
 187 }
 188
 189 /**
 190  * drm_syncobj_find_fence - lookup and reference the fence in a sync object
 191  * @file_private: drm file private pointer
 192  * @handle: sync object handle to lookup.
 193  * @point: timeline point
 194  * @flags: DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT or not
 195  * @fence: out parameter for the fence
 196  *
 197  * This is just a convenience function that combines drm_syncobj_find() and
 198  * drm_syncobj_fence_get().
 199  *
 200  * Returns 0 on success or a negative error value on failure. On success @fence
 201  * contains a reference to the fence, which must be released by calling
 202  * dma_fence_put().
 203  */
 204 int drm_syncobj_find_fence(struct drm_file *file_private,
 205                            u32 handle, u64 point, u64 flags,
 206                            struct dma_fence **fence)
 207 {
 208         struct drm_syncobj *syncobj = drm_syncobj_find(file_private, handle);
 209         int ret = 0;
 210
 211         if (!syncobj)
 212                 return -ENOENT;
 213
 214         *fence = drm_syncobj_fence_get(syncobj);
 215         if (!*fence) {
 216                 ret = -EINVAL;
 217         }
 218         drm_syncobj_put(syncobj);
 219         return ret;
 220 }
 221 EXPORT_SYMBOL(drm_syncobj_find_fence);
 222
 223 /**
 224  * drm_syncobj_free - free a sync object.
 225  * @kref: kref to free.
 226  *
 227  * Only to be called from kref_put in drm_syncobj_put.
 228  */
 229 void drm_syncobj_free(struct kref *kref)
 230 {
 231         struct drm_syncobj *syncobj = container_of(kref,
 232                                                    struct drm_syncobj,
 233                                                    refcount);
 234         drm_syncobj_replace_fence(syncobj, NULL);
 235         kfree(syncobj);
 236 }
 237 EXPORT_SYMBOL(drm_syncobj_free);
 238
 239 /**
 240  * drm_syncobj_create - create a new syncobj
 241  * @out_syncobj: returned syncobj
 242  * @flags: DRM_SYNCOBJ_* flags
 243  * @fence: if non-NULL, the syncobj will represent this fence
 244  *
 245  * This is the first function to create a sync object. After creating, drivers
 246  * probably want to make it available to userspace, either through
 247  * drm_syncobj_get_handle() or drm_syncobj_get_fd().
 248  *
 249  * Returns 0 on success or a negative error value on failure.
 250  */
 251 int drm_syncobj_create(struct drm_syncobj **out_syncobj, uint32_t flags,
 252                        struct dma_fence *fence)
 253 {
 254         struct drm_syncobj *syncobj;
 255
 256         syncobj = kzalloc(sizeof(struct drm_syncobj), GFP_KERNEL);
 257         if (!syncobj)
 258                 return -ENOMEM;
 259
 260         kref_init(&syncobj->refcount);
 261         INIT_LIST_HEAD(&syncobj->cb_list);
 262         spin_lock_init(&syncobj->lock);
 263
 264         if (flags & DRM_SYNCOBJ_CREATE_SIGNALED)
 265                 drm_syncobj_assign_null_handle(syncobj);
 266
 267         if (fence)
 268                 drm_syncobj_replace_fence(syncobj, fence);
 269
 270         *out_syncobj = syncobj;
 271         return 0;
 272 }
 273 EXPORT_SYMBOL(drm_syncobj_create);
 274
 275 /**
 276  * drm_syncobj_get_handle - get a handle from a syncobj
 277  * @file_private: drm file private pointer
 278  * @syncobj: Sync object to export
 279  * @handle: out parameter with the new handle
 280  *
 281  * Exports a sync object created with drm_syncobj_create() as a handle on
 282  * @file_private to userspace.
 283  *
 284  * Returns 0 on success or a negative error value on failure.
 285  */
 286 int drm_syncobj_get_handle(struct drm_file *file_private,
 287                            struct drm_syncobj *syncobj, u32 *handle)
 288 {
 289         int ret;
 290
 291         /* take a reference to put in the idr */
 292         drm_syncobj_get(syncobj);
 293
 294         idr_preload(GFP_KERNEL);
 295         spin_lock(&file_private->syncobj_table_lock);
 296         ret = idr_alloc(&file_private->syncobj_idr, syncobj, 1, 0, GFP_NOWAIT);
 297         spin_unlock(&file_private->syncobj_table_lock);
 298
 299         idr_preload_end();
 300
 301         if (ret < 0) {
 302                 drm_syncobj_put(syncobj);
 303                 return ret;
 304         }
 305
 306         *handle = ret;
 307         return 0;
 308 }
 309 EXPORT_SYMBOL(drm_syncobj_get_handle);
 310
 311 static int drm_syncobj_create_as_handle(struct drm_file *file_private,
 312                                         u32 *handle, uint32_t flags)
 313 {
 314         int ret;
 315         struct drm_syncobj *syncobj;
 316
 317         ret = drm_syncobj_create(&syncobj, flags, NULL);
 318         if (ret)
 319                 return ret;
 320
 321         ret = drm_syncobj_get_handle(file_private, syncobj, handle);
 322         drm_syncobj_put(syncobj);
 323         return ret;
 324 }
 325
 326 static int drm_syncobj_destroy(struct drm_file *file_private,
 327                                u32 handle)
 328 {
 329         struct drm_syncobj *syncobj;
 330
 331         spin_lock(&file_private->syncobj_table_lock);
 332         syncobj = idr_remove(&file_private->syncobj_idr, handle);
 333         spin_unlock(&file_private->syncobj_table_lock);
 334
 335         if (!syncobj)
 336                 return -EINVAL;
 337
 338         drm_syncobj_put(syncobj);
 339         return 0;
 340 }
 341
 342 static int drm_syncobj_file_release(struct inode *inode, struct file *file)
 343 {
 344         struct drm_syncobj *syncobj = file->private_data;
 345
 346         drm_syncobj_put(syncobj);
 347         return 0;
 348 }
 349
 350 static const struct file_operations drm_syncobj_file_fops = {
 351         .release = drm_syncobj_file_release,
 352 };
 353
 354 /**
 355  * drm_syncobj_get_fd - get a file descriptor from a syncobj
 356  * @syncobj: Sync object to export
 357  * @p_fd: out parameter with the new file descriptor
 358  *
 359  * Exports a sync object created with drm_syncobj_create() as a file descriptor.
 360  *
 361  * Returns 0 on success or a negative error value on failure.
 362  */
 363 int drm_syncobj_get_fd(struct drm_syncobj *syncobj, int *p_fd)
 364 {
 365         struct file *file;
 366         int fd;
 367
 368         fd = get_unused_fd_flags(O_CLOEXEC);
 369         if (fd < 0)
 370                 return fd;
 371
 372         file = anon_inode_getfile("syncobj_file",
 373                                   &drm_syncobj_file_fops,
 374                                   syncobj, 0);
 375         if (IS_ERR(file)) {
 376                 put_unused_fd(fd);
 377                 return PTR_ERR(file);
 378         }
 379
 380         drm_syncobj_get(syncobj);
 381         fd_install(fd, file);
 382
 383         *p_fd = fd;
 384         return 0;
 385 }
 386 EXPORT_SYMBOL(drm_syncobj_get_fd);
 387
 388 static int drm_syncobj_handle_to_fd(struct drm_file *file_private,
 389                                     u32 handle, int *p_fd)
 390 {
 391         struct drm_syncobj *syncobj = drm_syncobj_find(file_private, handle);
 392         int ret;
 393
 394         if (!syncobj)
 395                 return -EINVAL;
 396
 397         ret = drm_syncobj_get_fd(syncobj, p_fd);
 398         drm_syncobj_put(syncobj);
 399         return ret;
 400 }
 401
 402 static int drm_syncobj_fd_to_handle(struct drm_file *file_private,
 403                                     int fd, u32 *handle)
 404 {
 405         struct drm_syncobj *syncobj;
 406         struct file *file;
 407         int ret;
 408
 409         file = fget(fd);
 410         if (!file)
 411                 return -EINVAL;
 412
 413         if (file->f_op != &drm_syncobj_file_fops) {
 414                 fput(file);
 415                 return -EINVAL;
 416         }
 417
 418         /* take a reference to put in the idr */
 419         syncobj = file->private_data;
 420         drm_syncobj_get(syncobj);
 421
 422         idr_preload(GFP_KERNEL);
 423         spin_lock(&file_private->syncobj_table_lock);
 424         ret = idr_alloc(&file_private->syncobj_idr, syncobj, 1, 0, GFP_NOWAIT);
 425         spin_unlock(&file_private->syncobj_table_lock);
 426         idr_preload_end();
 427
 428         if (ret > 0) {
 429                 *handle = ret;
 430                 ret = 0;
 431         } else
 432                 drm_syncobj_put(syncobj);
 433
 434         fput(file);
 435         return ret;
 436 }
 437
 438 static int drm_syncobj_import_sync_file_fence(struct drm_file *file_private,
 439                                               int fd, int handle)
 440 {
 441         struct dma_fence *fence = sync_file_get_fence(fd);
 442         struct drm_syncobj *syncobj;
 443
 444         if (!fence)
 445                 return -EINVAL;
 446
 447         syncobj = drm_syncobj_find(file_private, handle);
 448         if (!syncobj) {
 449                 dma_fence_put(fence);
 450                 return -ENOENT;
 451         }
 452
 453         drm_syncobj_replace_fence(syncobj, fence);
 454         dma_fence_put(fence);
 455         drm_syncobj_put(syncobj);
 456         return 0;
 457 }
 458
 459 static int drm_syncobj_export_sync_file(struct drm_file *file_private,
 460                                         int handle, int *p_fd)
 461 {
 462         int ret;
 463         struct dma_fence *fence;
 464         struct sync_file *sync_file;
 465         int fd = get_unused_fd_flags(O_CLOEXEC);
 466
 467         if (fd < 0)
 468                 return fd;
 469
 470         ret = drm_syncobj_find_fence(file_private, handle, 0, 0, &fence);
 471         if (ret)
 472                 goto err_put_fd;
 473
 474         sync_file = sync_file_create(fence);
 475
 476         dma_fence_put(fence);
 477
 478         if (!sync_file) {
 479                 ret = -EINVAL;
 480                 goto err_put_fd;
 481         }
 482
 483         fd_install(fd, sync_file->file);
 484
 485         *p_fd = fd;
 486         return 0;
 487 err_put_fd:
 488         put_unused_fd(fd);
 489         return ret;
 490 }
 491 /**
 492  * drm_syncobj_open - initalizes syncobj file-private structures at devnode open time
 493  * @file_private: drm file-private structure to set up
 494  *
 495  * Called at device open time, sets up the structure for handling refcounting
 496  * of sync objects.
 497  */
 498 void
 499 drm_syncobj_open(struct drm_file *file_private)
 500 {
 501         idr_init_base(&file_private->syncobj_idr, 1);
 502         spin_lock_init(&file_private->syncobj_table_lock);
 503 }
 504
 505 static int
 506 drm_syncobj_release_handle(int id, void *ptr, void *data)
 507 {
 508         struct drm_syncobj *syncobj = ptr;
 509
 510         drm_syncobj_put(syncobj);
 511         return 0;
 512 }
 513
 514 /**
 515  * drm_syncobj_release - release file-private sync object resources
 516  * @file_private: drm file-private structure to clean up
 517  *
 518  * Called at close time when the filp is going away.
 519  *
 520  * Releases any remaining references on objects by this filp.
 521  */
 522 void
 523 drm_syncobj_release(struct drm_file *file_private)
 524 {
 525         idr_for_each(&file_private->syncobj_idr,
 526                      &drm_syncobj_release_handle, file_private);
 527         idr_destroy(&file_private->syncobj_idr);
 528 }
 529
 530 int
 531 drm_syncobj_create_ioctl(struct drm_device *dev, void *data,
 532                          struct drm_file *file_private)
 533 {
 534         struct drm_syncobj_create *args = data;
 535
 536         if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
 537                 return -EOPNOTSUPP;
 538
 539         /* no valid flags yet */
 540         if (args->flags & ~DRM_SYNCOBJ_CREATE_SIGNALED)
 541                 return -EINVAL;
 542
 543         return drm_syncobj_create_as_handle(file_private,
 544                                             &args->handle, args->flags);
 545 }
 546
 547 int
 548 drm_syncobj_destroy_ioctl(struct drm_device *dev, void *data,
 549                           struct drm_file *file_private)
 550 {
 551         struct drm_syncobj_destroy *args = data;
 552
 553         if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
 554                 return -EOPNOTSUPP;
 555
 556         /* make sure padding is empty */
 557         if (args->pad)
 558                 return -EINVAL;
 559         return drm_syncobj_destroy(file_private, args->handle);
 560 }
 561
 562 int
 563 drm_syncobj_handle_to_fd_ioctl(struct drm_device *dev, void *data,
 564                                    struct drm_file *file_private)
 565 {
 566         struct drm_syncobj_handle *args = data;
 567
 568         if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
 569                 return -EOPNOTSUPP;
 570
 571         if (args->pad)
 572                 return -EINVAL;
 573
 574         if (args->flags != 0 &&
 575             args->flags != DRM_SYNCOBJ_HANDLE_TO_FD_FLAGS_EXPORT_SYNC_FILE)
 576                 return -EINVAL;
 577
 578         if (args->flags & DRM_SYNCOBJ_HANDLE_TO_FD_FLAGS_EXPORT_SYNC_FILE)
 579                 return drm_syncobj_export_sync_file(file_private, args->handle,
 580                                                     &args->fd);
 581
 582         return drm_syncobj_handle_to_fd(file_private, args->handle,
 583                                         &args->fd);
 584 }
 585
 586 int
 587 drm_syncobj_fd_to_handle_ioctl(struct drm_device *dev, void *data,
 588                                    struct drm_file *file_private)
 589 {
 590         struct drm_syncobj_handle *args = data;
 591
 592         if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
 593                 return -EOPNOTSUPP;
 594
 595         if (args->pad)
 596                 return -EINVAL;
 597
 598         if (args->flags != 0 &&
 599             args->flags != DRM_SYNCOBJ_FD_TO_HANDLE_FLAGS_IMPORT_SYNC_FILE)
 600                 return -EINVAL;
 601
 602         if (args->flags & DRM_SYNCOBJ_FD_TO_HANDLE_FLAGS_IMPORT_SYNC_FILE)
 603                 return drm_syncobj_import_sync_file_fence(file_private,
 604                                                           args->fd,
 605                                                           args->handle);
 606
 607         return drm_syncobj_fd_to_handle(file_private, args->fd,
 608                                         &args->handle);
 609 }
 610
 611 struct syncobj_wait_entry {
 612         struct task_struct *task;
 613         struct dma_fence *fence;
 614         struct dma_fence_cb fence_cb;
 615         struct drm_syncobj_cb syncobj_cb;
 616 };
 617
 618 static void syncobj_wait_fence_func(struct dma_fence *fence,
 619                                     struct dma_fence_cb *cb)
 620 {
 621         struct syncobj_wait_entry *wait =
 622                 container_of(cb, struct syncobj_wait_entry, fence_cb);
 623
 624         wake_up_process(wait->task);
 625 }
 626
 627 static void syncobj_wait_syncobj_func(struct drm_syncobj *syncobj,
 628                                       struct drm_syncobj_cb *cb)
 629 {
 630         struct syncobj_wait_entry *wait =
 631                 container_of(cb, struct syncobj_wait_entry, syncobj_cb);
 632
 633         /* This happens inside the syncobj lock */
 634         wait->fence = dma_fence_get(rcu_dereference_protected(syncobj->fence,
 635                                                               lockdep_is_held(&syncobj->lock)));
 636         wake_up_process(wait->task);
 637 }
 638
 639 static signed long drm_syncobj_array_wait_timeout(struct drm_syncobj **syncobjs,
 640                                                   uint32_t count,
 641                                                   uint32_t flags,
 642                                                   signed long timeout,
 643                                                   uint32_t *idx)
 644 {
 645         struct syncobj_wait_entry *entries;
 646         struct dma_fence *fence;
 647         uint32_t signaled_count, i;
 648
 649         entries = kcalloc(count, sizeof(*entries), GFP_KERNEL);
 650         if (!entries)
 651                 return -ENOMEM;
 652
 653         /* Walk the list of sync objects and initialize entries.  We do
 654          * this up-front so that we can properly return -EINVAL if there is
 655          * a syncobj with a missing fence and then never have the chance of
 656          * returning -EINVAL again.
 657          */
 658         signaled_count = 0;
 659         for (i = 0; i < count; ++i) {
 660                 entries[i].task = current;
 661                 entries[i].fence = drm_syncobj_fence_get(syncobjs[i]);
 662                 if (!entries[i].fence) {
 663                         if (flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT) {
 664                                 continue;
 665                         } else {
 666                                 timeout = -EINVAL;
 667                                 goto cleanup_entries;
 668                         }
 669                 }
 670
 671                 if (dma_fence_is_signaled(entries[i].fence)) {
 672                         if (signaled_count == 0 && idx)
 673                                 *idx = i;
 674                         signaled_count++;
 675                 }
 676         }
 677
 678         if (signaled_count == count ||
 679             (signaled_count > 0 &&
 680              !(flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL)))
 681                 goto cleanup_entries;
 682
 683         /* There's a very annoying laxness in the dma_fence API here, in
 684          * that backends are not required to automatically report when a
 685          * fence is signaled prior to fence->ops->enable_signaling() being
 686          * called.  So here if we fail to match signaled_count, we need to
 687          * fallthough and try a 0 timeout wait!
 688          */
 689
 690         if (flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT) {
 691                 for (i = 0; i < count; ++i) {
 692                         drm_syncobj_fence_get_or_add_callback(syncobjs[i],
 693                                                               &entries[i].fence,
 694                                                               &entries[i].syncobj_cb,
 695                                                               syncobj_wait_syncobj_func);
 696                 }
 697         }
 698
 699         do {
 700                 set_current_state(TASK_INTERRUPTIBLE);
 701
 702                 signaled_count = 0;
 703                 for (i = 0; i < count; ++i) {
 704                         fence = entries[i].fence;
 705                         if (!fence)
 706                                 continue;
 707
 708                         if (dma_fence_is_signaled(fence) ||
 709                             (!entries[i].fence_cb.func &&
 710                              dma_fence_add_callback(fence,
 711                                                     &entries[i].fence_cb,
 712                                                     syncobj_wait_fence_func))) {
 713                                 /* The fence has been signaled */
 714                                 if (flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL) {
 715                                         signaled_count++;
 716                                 } else {
 717                                         if (idx)
 718                                                 *idx = i;
 719                                         goto done_waiting;
 720                                 }
 721                         }
 722                 }
 723
 724                 if (signaled_count == count)
 725                         goto done_waiting;
 726
 727                 if (timeout == 0) {
 728                         timeout = -ETIME;
 729                         goto done_waiting;
 730                 }
 731
 732                 if (signal_pending(current)) {
 733                         timeout = -ERESTARTSYS;
 734                         goto done_waiting;
 735                 }
 736
 737                 timeout = schedule_timeout(timeout);
 738         } while (1);
 739
 740 done_waiting:
 741         __set_current_state(TASK_RUNNING);
 742
 743 cleanup_entries:
 744         for (i = 0; i < count; ++i) {
 745                 if (entries[i].syncobj_cb.func)
 746                         drm_syncobj_remove_callback(syncobjs[i],
 747                                                     &entries[i].syncobj_cb);
 748                 if (entries[i].fence_cb.func)
 749                         dma_fence_remove_callback(entries[i].fence,
 750                                                   &entries[i].fence_cb);
 751                 dma_fence_put(entries[i].fence);
 752         }
 753         kfree(entries);
 754
 755         return timeout;
 756 }
 757
 758 /**
 759  * drm_timeout_abs_to_jiffies - calculate jiffies timeout from absolute value
 760  *
 761  * @timeout_nsec: timeout nsec component in ns, 0 for poll
 762  *
 763  * Calculate the timeout in jiffies from an absolute time in sec/nsec.
 764  */
 765 static signed long drm_timeout_abs_to_jiffies(int64_t timeout_nsec)
 766 {
 767         ktime_t abs_timeout, now;
 768         u64 timeout_ns, timeout_jiffies64;
 769
 770         /* make 0 timeout means poll - absolute 0 doesn't seem valid */
 771         if (timeout_nsec == 0)
 772                 return 0;
 773
 774         abs_timeout = ns_to_ktime(timeout_nsec);
 775         now = ktime_get();
 776
 777         if (!ktime_after(abs_timeout, now))
 778                 return 0;
 779
 780         timeout_ns = ktime_to_ns(ktime_sub(abs_timeout, now));
 781
 782         timeout_jiffies64 = nsecs_to_jiffies64(timeout_ns);
 783         /*  clamp timeout to avoid infinite timeout */
 784         if (timeout_jiffies64 >= MAX_SCHEDULE_TIMEOUT - 1)
 785                 return MAX_SCHEDULE_TIMEOUT - 1;
 786
 787         return timeout_jiffies64 + 1;
 788 }
 789
 790 static int drm_syncobj_array_wait(struct drm_device *dev,
 791                                   struct drm_file *file_private,
 792                                   struct drm_syncobj_wait *wait,
 793                                   struct drm_syncobj **syncobjs)
 794 {
 795         signed long timeout = drm_timeout_abs_to_jiffies(wait->timeout_nsec);
 796         uint32_t first = ~0;
 797
 798         timeout = drm_syncobj_array_wait_timeout(syncobjs,
 799                                                  wait->count_handles,
 800                                                  wait->flags,
 801                                                  timeout, &first);
 802         if (timeout < 0)
 803                 return timeout;
 804
 805         wait->first_signaled = first;
 806         return 0;
 807 }
 808
 809 static int drm_syncobj_array_find(struct drm_file *file_private,
 810                                   void __user *user_handles,
 811                                   uint32_t count_handles,
 812                                   struct drm_syncobj ***syncobjs_out)
 813 {
 814         uint32_t i, *handles;
 815         struct drm_syncobj **syncobjs;
 816         int ret;
 817
 818         handles = kmalloc_array(count_handles, sizeof(*handles), GFP_KERNEL);
 819         if (handles == NULL)
 820                 return -ENOMEM;
 821
 822         if (copy_from_user(handles, user_handles,
 823                            sizeof(uint32_t) * count_handles)) {
 824                 ret = -EFAULT;
 825                 goto err_free_handles;
 826         }
 827
 828         syncobjs = kmalloc_array(count_handles, sizeof(*syncobjs), GFP_KERNEL);
 829         if (syncobjs == NULL) {
 830                 ret = -ENOMEM;
 831                 goto err_free_handles;
 832         }
 833
 834         for (i = 0; i < count_handles; i++) {
 835                 syncobjs[i] = drm_syncobj_find(file_private, handles[i]);
 836                 if (!syncobjs[i]) {
 837                         ret = -ENOENT;
 838                         goto err_put_syncobjs;
 839                 }
 840         }
 841
 842         kfree(handles);
 843         *syncobjs_out = syncobjs;
 844         return 0;
 845
 846 err_put_syncobjs:
 847         while (i-- > 0)
 848                 drm_syncobj_put(syncobjs[i]);
 849         kfree(syncobjs);
 850 err_free_handles:
 851         kfree(handles);
 852
 853         return ret;
 854 }
 855
 856 static void drm_syncobj_array_free(struct drm_syncobj **syncobjs,
 857                                    uint32_t count)
 858 {
 859         uint32_t i;
 860         for (i = 0; i < count; i++)
 861                 drm_syncobj_put(syncobjs[i]);
 862         kfree(syncobjs);
 863 }
 864
 865 int
 866 drm_syncobj_wait_ioctl(struct drm_device *dev, void *data,
 867                        struct drm_file *file_private)
 868 {
 869         struct drm_syncobj_wait *args = data;
 870         struct drm_syncobj **syncobjs;
 871         int ret = 0;
 872
 873         if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
 874                 return -EOPNOTSUPP;
 875
 876         if (args->flags & ~(DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL |
 877                             DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT))
 878                 return -EINVAL;
 879
 880         if (args->count_handles == 0)
 881                 return -EINVAL;
 882
 883         ret = drm_syncobj_array_find(file_private,
 884                                      u64_to_user_ptr(args->handles),
 885                                      args->count_handles,
 886                                      &syncobjs);
 887         if (ret < 0)
 888                 return ret;
 889
 890         ret = drm_syncobj_array_wait(dev, file_private,
 891                                      args, syncobjs);
 892
 893         drm_syncobj_array_free(syncobjs, args->count_handles);
 894
 895         return ret;
 896 }
 897
 898 int
 899 drm_syncobj_reset_ioctl(struct drm_device *dev, void *data,
 900                         struct drm_file *file_private)
 901 {
 902         struct drm_syncobj_array *args = data;
 903         struct drm_syncobj **syncobjs;
 904         uint32_t i;
 905         int ret;
 906
 907         if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
 908                 return -EOPNOTSUPP;
 909
 910         if (args->pad != 0)
 911                 return -EINVAL;
 912
 913         if (args->count_handles == 0)
 914                 return -EINVAL;
 915
 916         ret = drm_syncobj_array_find(file_private,
 917                                      u64_to_user_ptr(args->handles),
 918                                      args->count_handles,
 919                                      &syncobjs);
 920         if (ret < 0)
 921                 return ret;
 922
 923         for (i = 0; i < args->count_handles; i++)
 924                 drm_syncobj_replace_fence(syncobjs[i], NULL);
 925
 926         drm_syncobj_array_free(syncobjs, args->count_handles);
 927
 928         return 0;
 929 }
 930
 931 int
 932 drm_syncobj_signal_ioctl(struct drm_device *dev, void *data,
 933                          struct drm_file *file_private)
 934 {
 935         struct drm_syncobj_array *args = data;
 936         struct drm_syncobj **syncobjs;
 937         uint32_t i;
 938         int ret;
 939
 940         if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
 941                 return -EOPNOTSUPP;
 942
 943         if (args->pad != 0)
 944                 return -EINVAL;
 945
 946         if (args->count_handles == 0)
 947                 return -EINVAL;
 948
 949         ret = drm_syncobj_array_find(file_private,
 950                                      u64_to_user_ptr(args->handles),
 951                                      args->count_handles,
 952                                      &syncobjs);
 953         if (ret < 0)
 954                 return ret;
 955
 956         for (i = 0; i < args->count_handles; i++)
 957                 drm_syncobj_assign_null_handle(syncobjs[i]);
 958
 959         drm_syncobj_array_free(syncobjs, args->count_handles);
 960
 961         return ret;
 962 }