drivers/gpu/drm/i915/i915_sw_fence.c

   1 /*
   2  * SPDX-License-Identifier: MIT
   3  *
   4  * (C) Copyright 2016 Intel Corporation
   5  */
   6
   7 #include <linux/slab.h>
   8 #include <linux/dma-fence.h>
   9 #include <linux/irq_work.h>
  10 #include <linux/reservation.h>
  11
  12 #include "i915_sw_fence.h"
  13 #include "i915_selftest.h"
  14
  15 #define I915_SW_FENCE_FLAG_ALLOC BIT(3) /* after WQ_FLAG_* for safety */
  16
  17 static DEFINE_SPINLOCK(i915_sw_fence_lock);
  18
  19 enum {
  20         DEBUG_FENCE_IDLE = 0,
  21         DEBUG_FENCE_NOTIFY,
  22 };
  23
  24 static void *i915_sw_fence_debug_hint(void *addr)
  25 {
  26         return (void *)(((struct i915_sw_fence *)addr)->flags & I915_SW_FENCE_MASK);
  27 }
  28
  29 #ifdef CONFIG_DRM_I915_SW_FENCE_DEBUG_OBJECTS
  30
  31 static struct debug_obj_descr i915_sw_fence_debug_descr = {
  32         .name = "i915_sw_fence",
  33         .debug_hint = i915_sw_fence_debug_hint,
  34 };
  35
  36 static inline void debug_fence_init(struct i915_sw_fence *fence)
  37 {
  38         debug_object_init(fence, &i915_sw_fence_debug_descr);
  39 }
  40
  41 static inline void debug_fence_init_onstack(struct i915_sw_fence *fence)
  42 {
  43         debug_object_init_on_stack(fence, &i915_sw_fence_debug_descr);
  44 }
  45
  46 static inline void debug_fence_activate(struct i915_sw_fence *fence)
  47 {
  48         debug_object_activate(fence, &i915_sw_fence_debug_descr);
  49 }
  50
  51 static inline void debug_fence_set_state(struct i915_sw_fence *fence,
  52                                          int old, int new)
  53 {
  54         debug_object_active_state(fence, &i915_sw_fence_debug_descr, old, new);
  55 }
  56
  57 static inline void debug_fence_deactivate(struct i915_sw_fence *fence)
  58 {
  59         debug_object_deactivate(fence, &i915_sw_fence_debug_descr);
  60 }
  61
  62 static inline void debug_fence_destroy(struct i915_sw_fence *fence)
  63 {
  64         debug_object_destroy(fence, &i915_sw_fence_debug_descr);
  65 }
  66
  67 static inline void debug_fence_free(struct i915_sw_fence *fence)
  68 {
  69         debug_object_free(fence, &i915_sw_fence_debug_descr);
  70         smp_wmb(); /* flush the change in state before reallocation */
  71 }
  72
  73 static inline void debug_fence_assert(struct i915_sw_fence *fence)
  74 {
  75         debug_object_assert_init(fence, &i915_sw_fence_debug_descr);
  76 }
  77
  78 #else
  79
  80 static inline void debug_fence_init(struct i915_sw_fence *fence)
  81 {
  82 }
  83
  84 static inline void debug_fence_init_onstack(struct i915_sw_fence *fence)
  85 {
  86 }
  87
  88 static inline void debug_fence_activate(struct i915_sw_fence *fence)
  89 {
  90 }
  91
  92 static inline void debug_fence_set_state(struct i915_sw_fence *fence,
  93                                          int old, int new)
  94 {
  95 }
  96
  97 static inline void debug_fence_deactivate(struct i915_sw_fence *fence)
  98 {
  99 }
 100
 101 static inline void debug_fence_destroy(struct i915_sw_fence *fence)
 102 {
 103 }
 104
 105 static inline void debug_fence_free(struct i915_sw_fence *fence)
 106 {
 107 }
 108
 109 static inline void debug_fence_assert(struct i915_sw_fence *fence)
 110 {
 111 }
 112
 113 #endif
 114
 115 static int __i915_sw_fence_notify(struct i915_sw_fence *fence,
 116                                   enum i915_sw_fence_notify state)
 117 {
 118         i915_sw_fence_notify_t fn;
 119
 120         fn = (i915_sw_fence_notify_t)(fence->flags & I915_SW_FENCE_MASK);
 121         return fn(fence, state);
 122 }
 123
 124 #ifdef CONFIG_DRM_I915_SW_FENCE_DEBUG_OBJECTS
 125 void i915_sw_fence_fini(struct i915_sw_fence *fence)
 126 {
 127         debug_fence_free(fence);
 128 }
 129 #endif
 130
 131 static void __i915_sw_fence_wake_up_all(struct i915_sw_fence *fence,
 132                                         struct list_head *continuation)
 133 {
 134         wait_queue_head_t *x = &fence->wait;
 135         wait_queue_entry_t *pos, *next;
 136         unsigned long flags;
 137
 138         debug_fence_deactivate(fence);
 139         atomic_set_release(&fence->pending, -1); /* 0 -> -1 [done] */
 140
 141         /*
 142          * To prevent unbounded recursion as we traverse the graph of
 143          * i915_sw_fences, we move the entry list from this, the next ready
 144          * fence, to the tail of the original fence's entry list
 145          * (and so added to the list to be woken).
 146          */
 147
 148         spin_lock_irqsave_nested(&x->lock, flags, 1 + !!continuation);
 149         if (continuation) {
 150                 list_for_each_entry_safe(pos, next, &x->head, entry) {
 151                         if (pos->func == autoremove_wake_function)
 152                                 pos->func(pos, TASK_NORMAL, 0, continuation);
 153                         else
 154                                 list_move_tail(&pos->entry, continuation);
 155                 }
 156         } else {
 157                 LIST_HEAD(extra);
 158
 159                 do {
 160                         list_for_each_entry_safe(pos, next, &x->head, entry)
 161                                 pos->func(pos, TASK_NORMAL, 0, &extra);
 162
 163                         if (list_empty(&extra))
 164                                 break;
 165
 166                         list_splice_tail_init(&extra, &x->head);
 167                 } while (1);
 168         }
 169         spin_unlock_irqrestore(&x->lock, flags);
 170
 171         debug_fence_assert(fence);
 172 }
 173
 174 static void __i915_sw_fence_complete(struct i915_sw_fence *fence,
 175                                      struct list_head *continuation)
 176 {
 177         debug_fence_assert(fence);
 178
 179         if (!atomic_dec_and_test(&fence->pending))
 180                 return;
 181
 182         debug_fence_set_state(fence, DEBUG_FENCE_IDLE, DEBUG_FENCE_NOTIFY);
 183
 184         if (__i915_sw_fence_notify(fence, FENCE_COMPLETE) != NOTIFY_DONE)
 185                 return;
 186
 187         debug_fence_set_state(fence, DEBUG_FENCE_NOTIFY, DEBUG_FENCE_IDLE);
 188
 189         __i915_sw_fence_wake_up_all(fence, continuation);
 190
 191         debug_fence_destroy(fence);
 192         __i915_sw_fence_notify(fence, FENCE_FREE);
 193 }
 194
 195 static void i915_sw_fence_complete(struct i915_sw_fence *fence)
 196 {
 197         debug_fence_assert(fence);
 198
 199         if (WARN_ON(i915_sw_fence_done(fence)))
 200                 return;
 201
 202         __i915_sw_fence_complete(fence, NULL);
 203 }
 204
 205 static void i915_sw_fence_await(struct i915_sw_fence *fence)
 206 {
 207         debug_fence_assert(fence);
 208         WARN_ON(atomic_inc_return(&fence->pending) <= 1);
 209 }
 210
 211 void __i915_sw_fence_init(struct i915_sw_fence *fence,
 212                           i915_sw_fence_notify_t fn,
 213                           const char *name,
 214                           struct lock_class_key *key)
 215 {
 216         BUG_ON(!fn || (unsigned long)fn & ~I915_SW_FENCE_MASK);
 217
 218         debug_fence_init(fence);
 219
 220         __init_waitqueue_head(&fence->wait, name, key);
 221         atomic_set(&fence->pending, 1);
 222         fence->flags = (unsigned long)fn;
 223 }
 224
 225 void i915_sw_fence_commit(struct i915_sw_fence *fence)
 226 {
 227         debug_fence_activate(fence);
 228         i915_sw_fence_complete(fence);
 229 }
 230
 231 static int i915_sw_fence_wake(wait_queue_entry_t *wq, unsigned mode, int flags, void *key)
 232 {
 233         list_del(&wq->entry);
 234         __i915_sw_fence_complete(wq->private, key);
 235
 236         if (wq->flags & I915_SW_FENCE_FLAG_ALLOC)
 237                 kfree(wq);
 238         return 0;
 239 }
 240
 241 static bool __i915_sw_fence_check_if_after(struct i915_sw_fence *fence,
 242                                     const struct i915_sw_fence * const signaler)
 243 {
 244         wait_queue_entry_t *wq;
 245
 246         if (__test_and_set_bit(I915_SW_FENCE_CHECKED_BIT, &fence->flags))
 247                 return false;
 248
 249         if (fence == signaler)
 250                 return true;
 251
 252         list_for_each_entry(wq, &fence->wait.head, entry) {
 253                 if (wq->func != i915_sw_fence_wake)
 254                         continue;
 255
 256                 if (__i915_sw_fence_check_if_after(wq->private, signaler))
 257                         return true;
 258         }
 259
 260         return false;
 261 }
 262
 263 static void __i915_sw_fence_clear_checked_bit(struct i915_sw_fence *fence)
 264 {
 265         wait_queue_entry_t *wq;
 266
 267         if (!__test_and_clear_bit(I915_SW_FENCE_CHECKED_BIT, &fence->flags))
 268                 return;
 269
 270         list_for_each_entry(wq, &fence->wait.head, entry) {
 271                 if (wq->func != i915_sw_fence_wake)
 272                         continue;
 273
 274                 __i915_sw_fence_clear_checked_bit(wq->private);
 275         }
 276 }
 277
 278 static bool i915_sw_fence_check_if_after(struct i915_sw_fence *fence,
 279                                   const struct i915_sw_fence * const signaler)
 280 {
 281         unsigned long flags;
 282         bool err;
 283
 284         if (!IS_ENABLED(CONFIG_DRM_I915_SW_FENCE_CHECK_DAG))
 285                 return false;
 286
 287         spin_lock_irqsave(&i915_sw_fence_lock, flags);
 288         err = __i915_sw_fence_check_if_after(fence, signaler);
 289         __i915_sw_fence_clear_checked_bit(fence);
 290         spin_unlock_irqrestore(&i915_sw_fence_lock, flags);
 291
 292         return err;
 293 }
 294
 295 static int __i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence,
 296                                           struct i915_sw_fence *signaler,
 297                                           wait_queue_entry_t *wq, gfp_t gfp)
 298 {
 299         unsigned long flags;
 300         int pending;
 301
 302         debug_fence_assert(fence);
 303         might_sleep_if(gfpflags_allow_blocking(gfp));
 304
 305         if (i915_sw_fence_done(signaler))
 306                 return 0;
 307
 308         debug_fence_assert(signaler);
 309
 310         /* The dependency graph must be acyclic. */
 311         if (unlikely(i915_sw_fence_check_if_after(fence, signaler)))
 312                 return -EINVAL;
 313
 314         pending = 0;
 315         if (!wq) {
 316                 wq = kmalloc(sizeof(*wq), gfp);
 317                 if (!wq) {
 318                         if (!gfpflags_allow_blocking(gfp))
 319                                 return -ENOMEM;
 320
 321                         i915_sw_fence_wait(signaler);
 322                         return 0;
 323                 }
 324
 325                 pending |= I915_SW_FENCE_FLAG_ALLOC;
 326         }
 327
 328         INIT_LIST_HEAD(&wq->entry);
 329         wq->flags = pending;
 330         wq->func = i915_sw_fence_wake;
 331         wq->private = fence;
 332
 333         i915_sw_fence_await(fence);
 334
 335         spin_lock_irqsave(&signaler->wait.lock, flags);
 336         if (likely(!i915_sw_fence_done(signaler))) {
 337                 __add_wait_queue_entry_tail(&signaler->wait, wq);
 338                 pending = 1;
 339         } else {
 340                 i915_sw_fence_wake(wq, 0, 0, NULL);
 341                 pending = 0;
 342         }
 343         spin_unlock_irqrestore(&signaler->wait.lock, flags);
 344
 345         return pending;
 346 }
 347
 348 int i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence,
 349                                  struct i915_sw_fence *signaler,
 350                                  wait_queue_entry_t *wq)
 351 {
 352         return __i915_sw_fence_await_sw_fence(fence, signaler, wq, 0);
 353 }
 354
 355 int i915_sw_fence_await_sw_fence_gfp(struct i915_sw_fence *fence,
 356                                      struct i915_sw_fence *signaler,
 357                                      gfp_t gfp)
 358 {
 359         return __i915_sw_fence_await_sw_fence(fence, signaler, NULL, gfp);
 360 }
 361
 362 struct i915_sw_dma_fence_cb {
 363         struct dma_fence_cb base;
 364         struct i915_sw_fence *fence;
 365 };
 366
 367 struct i915_sw_dma_fence_cb_timer {
 368         struct i915_sw_dma_fence_cb base;
 369         struct dma_fence *dma;
 370         struct timer_list timer;
 371         struct irq_work work;
 372         struct rcu_head rcu;
 373 };
 374
 375 static void dma_i915_sw_fence_wake(struct dma_fence *dma,
 376                                    struct dma_fence_cb *data)
 377 {
 378         struct i915_sw_dma_fence_cb *cb = container_of(data, typeof(*cb), base);
 379
 380         i915_sw_fence_complete(cb->fence);
 381         kfree(cb);
 382 }
 383
 384 static void timer_i915_sw_fence_wake(struct timer_list *t)
 385 {
 386         struct i915_sw_dma_fence_cb_timer *cb = from_timer(cb, t, timer);
 387         struct i915_sw_fence *fence;
 388
 389         fence = xchg(&cb->base.fence, NULL);
 390         if (!fence)
 391                 return;
 392
 393         pr_notice("Asynchronous wait on fence %s:%s:%llx timed out (hint:%pS)\n",
 394                   cb->dma->ops->get_driver_name(cb->dma),
 395                   cb->dma->ops->get_timeline_name(cb->dma),
 396                   cb->dma->seqno,
 397                   i915_sw_fence_debug_hint(fence));
 398
 399         i915_sw_fence_complete(fence);
 400 }
 401
 402 static void dma_i915_sw_fence_wake_timer(struct dma_fence *dma,
 403                                          struct dma_fence_cb *data)
 404 {
 405         struct i915_sw_dma_fence_cb_timer *cb =
 406                 container_of(data, typeof(*cb), base.base);
 407         struct i915_sw_fence *fence;
 408
 409         fence = xchg(&cb->base.fence, NULL);
 410         if (fence)
 411                 i915_sw_fence_complete(fence);
 412
 413         irq_work_queue(&cb->work);
 414 }
 415
 416 static void irq_i915_sw_fence_work(struct irq_work *wrk)
 417 {
 418         struct i915_sw_dma_fence_cb_timer *cb =
 419                 container_of(wrk, typeof(*cb), work);
 420
 421         del_timer_sync(&cb->timer);
 422         dma_fence_put(cb->dma);
 423
 424         kfree_rcu(cb, rcu);
 425 }
 426
 427 int i915_sw_fence_await_dma_fence(struct i915_sw_fence *fence,
 428                                   struct dma_fence *dma,
 429                                   unsigned long timeout,
 430                                   gfp_t gfp)
 431 {
 432         struct i915_sw_dma_fence_cb *cb;
 433         dma_fence_func_t func;
 434         int ret;
 435
 436         debug_fence_assert(fence);
 437         might_sleep_if(gfpflags_allow_blocking(gfp));
 438
 439         if (dma_fence_is_signaled(dma))
 440                 return 0;
 441
 442         cb = kmalloc(timeout ?
 443                      sizeof(struct i915_sw_dma_fence_cb_timer) :
 444                      sizeof(struct i915_sw_dma_fence_cb),
 445                      gfp);
 446         if (!cb) {
 447                 if (!gfpflags_allow_blocking(gfp))
 448                         return -ENOMEM;
 449
 450                 return dma_fence_wait(dma, false);
 451         }
 452
 453         cb->fence = fence;
 454         i915_sw_fence_await(fence);
 455
 456         func = dma_i915_sw_fence_wake;
 457         if (timeout) {
 458                 struct i915_sw_dma_fence_cb_timer *timer =
 459                         container_of(cb, typeof(*timer), base);
 460
 461                 timer->dma = dma_fence_get(dma);
 462                 init_irq_work(&timer->work, irq_i915_sw_fence_work);
 463
 464                 timer_setup(&timer->timer,
 465                             timer_i915_sw_fence_wake, TIMER_IRQSAFE);
 466                 mod_timer(&timer->timer, round_jiffies_up(jiffies + timeout));
 467
 468                 func = dma_i915_sw_fence_wake_timer;
 469         }
 470
 471         ret = dma_fence_add_callback(dma, &cb->base, func);
 472         if (ret == 0) {
 473                 ret = 1;
 474         } else {
 475                 func(dma, &cb->base);
 476                 if (ret == -ENOENT) /* fence already signaled */
 477                         ret = 0;
 478         }
 479
 480         return ret;
 481 }
 482
 483 int i915_sw_fence_await_reservation(struct i915_sw_fence *fence,
 484                                     struct reservation_object *resv,
 485                                     const struct dma_fence_ops *exclude,
 486                                     bool write,
 487                                     unsigned long timeout,
 488                                     gfp_t gfp)
 489 {
 490         struct dma_fence *excl;
 491         int ret = 0, pending;
 492
 493         debug_fence_assert(fence);
 494         might_sleep_if(gfpflags_allow_blocking(gfp));
 495
 496         if (write) {
 497                 struct dma_fence **shared;
 498                 unsigned int count, i;
 499
 500                 ret = reservation_object_get_fences_rcu(resv,
 501                                                         &excl, &count, &shared);
 502                 if (ret)
 503                         return ret;
 504
 505                 for (i = 0; i < count; i++) {
 506                         if (shared[i]->ops == exclude)
 507                                 continue;
 508
 509                         pending = i915_sw_fence_await_dma_fence(fence,
 510                                                                 shared[i],
 511                                                                 timeout,
 512                                                                 gfp);
 513                         if (pending < 0) {
 514                                 ret = pending;
 515                                 break;
 516                         }
 517
 518                         ret |= pending;
 519                 }
 520
 521                 for (i = 0; i < count; i++)
 522                         dma_fence_put(shared[i]);
 523                 kfree(shared);
 524         } else {
 525                 excl = reservation_object_get_excl_rcu(resv);
 526         }
 527
 528         if (ret >= 0 && excl && excl->ops != exclude) {
 529                 pending = i915_sw_fence_await_dma_fence(fence,
 530                                                         excl,
 531                                                         timeout,
 532                                                         gfp);
 533                 if (pending < 0)
 534                         ret = pending;
 535                 else
 536                         ret |= pending;
 537         }
 538
 539         dma_fence_put(excl);
 540
 541         return ret;
 542 }
 543
 544 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
 545 #include "selftests/lib_sw_fence.c"
 546 #include "selftests/i915_sw_fence.c"
 547 #endif