drivers/gpu/drm/i915/i915_gem.c

   1 /*
   2  * Copyright © 2008-2015 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Eric Anholt <eric@anholt.net>
  25  *
  26  */
  27
  28 #include <drm/drmP.h>
  29 #include <drm/drm_vma_manager.h>
  30 #include <drm/i915_drm.h>
  31 #include "i915_drv.h"
  32 #include "i915_vgpu.h"
  33 #include "i915_trace.h"
  34 #include "intel_drv.h"
  35 #include "intel_frontbuffer.h"
  36 #include "intel_mocs.h"
  37 #include <linux/dma-fence-array.h>
  38 #include <linux/reservation.h>
  39 #include <linux/shmem_fs.h>
  40 #include <linux/slab.h>
  41 #include <linux/swap.h>
  42 #include <linux/pci.h>
  43 #include <linux/dma-buf.h>
  44
  45 static void i915_gem_flush_free_objects(struct drm_i915_private *i915);
  46 static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj);
  47 static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj);
  48
  49 static bool cpu_cache_is_coherent(struct drm_device *dev,
  50                                   enum i915_cache_level level)
  51 {
  52         return HAS_LLC(to_i915(dev)) || level != I915_CACHE_NONE;
  53 }
  54
  55 static bool cpu_write_needs_clflush(struct drm_i915_gem_object *obj)
  56 {
  57         if (obj->base.write_domain == I915_GEM_DOMAIN_CPU)
  58                 return false;
  59
  60         if (!cpu_cache_is_coherent(obj->base.dev, obj->cache_level))
  61                 return true;
  62
  63         return obj->pin_display;
  64 }
  65
  66 static int
  67 insert_mappable_node(struct i915_ggtt *ggtt,
  68                      struct drm_mm_node *node, u32 size)
  69 {
  70         memset(node, 0, sizeof(*node));
  71         return drm_mm_insert_node_in_range_generic(&ggtt->base.mm, node,
  72                                                    size, 0, -1,
  73                                                    0, ggtt->mappable_end,
  74                                                    DRM_MM_SEARCH_DEFAULT,
  75                                                    DRM_MM_CREATE_DEFAULT);
  76 }
  77
  78 static void
  79 remove_mappable_node(struct drm_mm_node *node)
  80 {
  81         drm_mm_remove_node(node);
  82 }
  83
  84 /* some bookkeeping */
  85 static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv,
  86                                   u64 size)
  87 {
  88         spin_lock(&dev_priv->mm.object_stat_lock);
  89         dev_priv->mm.object_count++;
  90         dev_priv->mm.object_memory += size;
  91         spin_unlock(&dev_priv->mm.object_stat_lock);
  92 }
  93
  94 static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv,
  95                                      u64 size)
  96 {
  97         spin_lock(&dev_priv->mm.object_stat_lock);
  98         dev_priv->mm.object_count--;
  99         dev_priv->mm.object_memory -= size;
 100         spin_unlock(&dev_priv->mm.object_stat_lock);
 101 }
 102
 103 static int
 104 i915_gem_wait_for_error(struct i915_gpu_error *error)
 105 {
 106         int ret;
 107
 108         might_sleep();
 109
 110         if (!i915_reset_in_progress(error))
 111                 return 0;
 112
 113         /*
 114          * Only wait 10 seconds for the gpu reset to complete to avoid hanging
 115          * userspace. If it takes that long something really bad is going on and
 116          * we should simply try to bail out and fail as gracefully as possible.
 117          */
 118         ret = wait_event_interruptible_timeout(error->reset_queue,
 119                                                !i915_reset_in_progress(error),
 120                                                I915_RESET_TIMEOUT);
 121         if (ret == 0) {
 122                 DRM_ERROR("Timed out waiting for the gpu reset to complete\n");
 123                 return -EIO;
 124         } else if (ret < 0) {
 125                 return ret;
 126         } else {
 127                 return 0;
 128         }
 129 }
 130
 131 int i915_mutex_lock_interruptible(struct drm_device *dev)
 132 {
 133         struct drm_i915_private *dev_priv = to_i915(dev);
 134         int ret;
 135
 136         ret = i915_gem_wait_for_error(&dev_priv->gpu_error);
 137         if (ret)
 138                 return ret;
 139
 140         ret = mutex_lock_interruptible(&dev->struct_mutex);
 141         if (ret)
 142                 return ret;
 143
 144         return 0;
 145 }
 146
 147 int
 148 i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
 149                             struct drm_file *file)
 150 {
 151         struct drm_i915_private *dev_priv = to_i915(dev);
 152         struct i915_ggtt *ggtt = &dev_priv->ggtt;
 153         struct drm_i915_gem_get_aperture *args = data;
 154         struct i915_vma *vma;
 155         size_t pinned;
 156
 157         pinned = 0;
 158         mutex_lock(&dev->struct_mutex);
 159         list_for_each_entry(vma, &ggtt->base.active_list, vm_link)
 160                 if (i915_vma_is_pinned(vma))
 161                         pinned += vma->node.size;
 162         list_for_each_entry(vma, &ggtt->base.inactive_list, vm_link)
 163                 if (i915_vma_is_pinned(vma))
 164                         pinned += vma->node.size;
 165         mutex_unlock(&dev->struct_mutex);
 166
 167         args->aper_size = ggtt->base.total;
 168         args->aper_available_size = args->aper_size - pinned;
 169
 170         return 0;
 171 }
 172
 173 static struct sg_table *
 174 i915_gem_object_get_pages_phys(struct drm_i915_gem_object *obj)
 175 {
 176         struct address_space *mapping = obj->base.filp->f_mapping;
 177         drm_dma_handle_t *phys;
 178         struct sg_table *st;
 179         struct scatterlist *sg;
 180         char *vaddr;
 181         int i;
 182
 183         if (WARN_ON(i915_gem_object_needs_bit17_swizzle(obj)))
 184                 return ERR_PTR(-EINVAL);
 185
 186         /* Always aligning to the object size, allows a single allocation
 187          * to handle all possible callers, and given typical object sizes,
 188          * the alignment of the buddy allocation will naturally match.
 189          */
 190         phys = drm_pci_alloc(obj->base.dev,
 191                              obj->base.size,
 192                              roundup_pow_of_two(obj->base.size));
 193         if (!phys)
 194                 return ERR_PTR(-ENOMEM);
 195
 196         vaddr = phys->vaddr;
 197         for (i = 0; i < obj->base.size / PAGE_SIZE; i++) {
 198                 struct page *page;
 199                 char *src;
 200
 201                 page = shmem_read_mapping_page(mapping, i);
 202                 if (IS_ERR(page)) {
 203                         st = ERR_CAST(page);
 204                         goto err_phys;
 205                 }
 206
 207                 src = kmap_atomic(page);
 208                 memcpy(vaddr, src, PAGE_SIZE);
 209                 drm_clflush_virt_range(vaddr, PAGE_SIZE);
 210                 kunmap_atomic(src);
 211
 212                 put_page(page);
 213                 vaddr += PAGE_SIZE;
 214         }
 215
 216         i915_gem_chipset_flush(to_i915(obj->base.dev));
 217
 218         st = kmalloc(sizeof(*st), GFP_KERNEL);
 219         if (!st) {
 220                 st = ERR_PTR(-ENOMEM);
 221                 goto err_phys;
 222         }
 223
 224         if (sg_alloc_table(st, 1, GFP_KERNEL)) {
 225                 kfree(st);
 226                 st = ERR_PTR(-ENOMEM);
 227                 goto err_phys;
 228         }
 229
 230         sg = st->sgl;
 231         sg->offset = 0;
 232         sg->length = obj->base.size;
 233
 234         sg_dma_address(sg) = phys->busaddr;
 235         sg_dma_len(sg) = obj->base.size;
 236
 237         obj->phys_handle = phys;
 238         return st;
 239
 240 err_phys:
 241         drm_pci_free(obj->base.dev, phys);
 242         return st;
 243 }
 244
 245 static void
 246 __i915_gem_object_release_shmem(struct drm_i915_gem_object *obj,
 247                                 struct sg_table *pages,
 248                                 bool needs_clflush)
 249 {
 250         GEM_BUG_ON(obj->mm.madv == __I915_MADV_PURGED);
 251
 252         if (obj->mm.madv == I915_MADV_DONTNEED)
 253                 obj->mm.dirty = false;
 254
 255         if (needs_clflush &&
 256             (obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0 &&
 257             !cpu_cache_is_coherent(obj->base.dev, obj->cache_level))
 258                 drm_clflush_sg(pages);
 259
 260         obj->base.read_domains = I915_GEM_DOMAIN_CPU;
 261         obj->base.write_domain = I915_GEM_DOMAIN_CPU;
 262 }
 263
 264 static void
 265 i915_gem_object_put_pages_phys(struct drm_i915_gem_object *obj,
 266                                struct sg_table *pages)
 267 {
 268         __i915_gem_object_release_shmem(obj, pages, false);
 269
 270         if (obj->mm.dirty) {
 271                 struct address_space *mapping = obj->base.filp->f_mapping;
 272                 char *vaddr = obj->phys_handle->vaddr;
 273                 int i;
 274
 275                 for (i = 0; i < obj->base.size / PAGE_SIZE; i++) {
 276                         struct page *page;
 277                         char *dst;
 278
 279                         page = shmem_read_mapping_page(mapping, i);
 280                         if (IS_ERR(page))
 281                                 continue;
 282
 283                         dst = kmap_atomic(page);
 284                         drm_clflush_virt_range(vaddr, PAGE_SIZE);
 285                         memcpy(dst, vaddr, PAGE_SIZE);
 286                         kunmap_atomic(dst);
 287
 288                         set_page_dirty(page);
 289                         if (obj->mm.madv == I915_MADV_WILLNEED)
 290                                 mark_page_accessed(page);
 291                         put_page(page);
 292                         vaddr += PAGE_SIZE;
 293                 }
 294                 obj->mm.dirty = false;
 295         }
 296
 297         sg_free_table(pages);
 298         kfree(pages);
 299
 300         drm_pci_free(obj->base.dev, obj->phys_handle);
 301 }
 302
 303 static void
 304 i915_gem_object_release_phys(struct drm_i915_gem_object *obj)
 305 {
 306         i915_gem_object_unpin_pages(obj);
 307 }
 308
 309 static const struct drm_i915_gem_object_ops i915_gem_phys_ops = {
 310         .get_pages = i915_gem_object_get_pages_phys,
 311         .put_pages = i915_gem_object_put_pages_phys,
 312         .release = i915_gem_object_release_phys,
 313 };
 314
 315 int i915_gem_object_unbind(struct drm_i915_gem_object *obj)
 316 {
 317         struct i915_vma *vma;
 318         LIST_HEAD(still_in_list);
 319         int ret;
 320
 321         lockdep_assert_held(&obj->base.dev->struct_mutex);
 322
 323         /* Closed vma are removed from the obj->vma_list - but they may
 324          * still have an active binding on the object. To remove those we
 325          * must wait for all rendering to complete to the object (as unbinding
 326          * must anyway), and retire the requests.
 327          */
 328         ret = i915_gem_object_wait(obj,
 329                                    I915_WAIT_INTERRUPTIBLE |
 330                                    I915_WAIT_LOCKED |
 331                                    I915_WAIT_ALL,
 332                                    MAX_SCHEDULE_TIMEOUT,
 333                                    NULL);
 334         if (ret)
 335                 return ret;
 336
 337         i915_gem_retire_requests(to_i915(obj->base.dev));
 338
 339         while ((vma = list_first_entry_or_null(&obj->vma_list,
 340                                                struct i915_vma,
 341                                                obj_link))) {
 342                 list_move_tail(&vma->obj_link, &still_in_list);
 343                 ret = i915_vma_unbind(vma);
 344                 if (ret)
 345                         break;
 346         }
 347         list_splice(&still_in_list, &obj->vma_list);
 348
 349         return ret;
 350 }
 351
 352 static long
 353 i915_gem_object_wait_fence(struct dma_fence *fence,
 354                            unsigned int flags,
 355                            long timeout,
 356                            struct intel_rps_client *rps)
 357 {
 358         struct drm_i915_gem_request *rq;
 359
 360         BUILD_BUG_ON(I915_WAIT_INTERRUPTIBLE != 0x1);
 361
 362         if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags))
 363                 return timeout;
 364
 365         if (!dma_fence_is_i915(fence))
 366                 return dma_fence_wait_timeout(fence,
 367                                               flags & I915_WAIT_INTERRUPTIBLE,
 368                                               timeout);
 369
 370         rq = to_request(fence);
 371         if (i915_gem_request_completed(rq))
 372                 goto out;
 373
 374         /* This client is about to stall waiting for the GPU. In many cases
 375          * this is undesirable and limits the throughput of the system, as
 376          * many clients cannot continue processing user input/output whilst
 377          * blocked. RPS autotuning may take tens of milliseconds to respond
 378          * to the GPU load and thus incurs additional latency for the client.
 379          * We can circumvent that by promoting the GPU frequency to maximum
 380          * before we wait. This makes the GPU throttle up much more quickly
 381          * (good for benchmarks and user experience, e.g. window animations),
 382          * but at a cost of spending more power processing the workload
 383          * (bad for battery). Not all clients even want their results
 384          * immediately and for them we should just let the GPU select its own
 385          * frequency to maximise efficiency. To prevent a single client from
 386          * forcing the clocks too high for the whole system, we only allow
 387          * each client to waitboost once in a busy period.
 388          */
 389         if (rps) {
 390                 if (INTEL_GEN(rq->i915) >= 6)
 391                         gen6_rps_boost(rq->i915, rps, rq->emitted_jiffies);
 392                 else
 393                         rps = NULL;
 394         }
 395
 396         timeout = i915_wait_request(rq, flags, timeout);
 397
 398 out:
 399         if (flags & I915_WAIT_LOCKED && i915_gem_request_completed(rq))
 400                 i915_gem_request_retire_upto(rq);
 401
 402         if (rps && rq->global_seqno == intel_engine_last_submit(rq->engine)) {
 403                 /* The GPU is now idle and this client has stalled.
 404                  * Since no other client has submitted a request in the
 405                  * meantime, assume that this client is the only one
 406                  * supplying work to the GPU but is unable to keep that
 407                  * work supplied because it is waiting. Since the GPU is
 408                  * then never kept fully busy, RPS autoclocking will
 409                  * keep the clocks relatively low, causing further delays.
 410                  * Compensate by giving the synchronous client credit for
 411                  * a waitboost next time.
 412                  */
 413                 spin_lock(&rq->i915->rps.client_lock);
 414                 list_del_init(&rps->link);
 415                 spin_unlock(&rq->i915->rps.client_lock);
 416         }
 417
 418         return timeout;
 419 }
 420
 421 static long
 422 i915_gem_object_wait_reservation(struct reservation_object *resv,
 423                                  unsigned int flags,
 424                                  long timeout,
 425                                  struct intel_rps_client *rps)
 426 {
 427         struct dma_fence *excl;
 428
 429         if (flags & I915_WAIT_ALL) {
 430                 struct dma_fence **shared;
 431                 unsigned int count, i;
 432                 int ret;
 433
 434                 ret = reservation_object_get_fences_rcu(resv,
 435                                                         &excl, &count, &shared);
 436                 if (ret)
 437                         return ret;
 438
 439                 for (i = 0; i < count; i++) {
 440                         timeout = i915_gem_object_wait_fence(shared[i],
 441                                                              flags, timeout,
 442                                                              rps);
 443                         if (timeout <= 0)
 444                                 break;
 445
 446                         dma_fence_put(shared[i]);
 447                 }
 448
 449                 for (; i < count; i++)
 450                         dma_fence_put(shared[i]);
 451                 kfree(shared);
 452         } else {
 453                 excl = reservation_object_get_excl_rcu(resv);
 454         }
 455
 456         if (excl && timeout > 0)
 457                 timeout = i915_gem_object_wait_fence(excl, flags, timeout, rps);
 458
 459         dma_fence_put(excl);
 460
 461         return timeout;
 462 }
 463
 464 static void __fence_set_priority(struct dma_fence *fence, int prio)
 465 {
 466         struct drm_i915_gem_request *rq;
 467         struct intel_engine_cs *engine;
 468
 469         if (!dma_fence_is_i915(fence))
 470                 return;
 471
 472         rq = to_request(fence);
 473         engine = rq->engine;
 474         if (!engine->schedule)
 475                 return;
 476
 477         engine->schedule(rq, prio);
 478 }
 479
 480 static void fence_set_priority(struct dma_fence *fence, int prio)
 481 {
 482         /* Recurse once into a fence-array */
 483         if (dma_fence_is_array(fence)) {
 484                 struct dma_fence_array *array = to_dma_fence_array(fence);
 485                 int i;
 486
 487                 for (i = 0; i < array->num_fences; i++)
 488                         __fence_set_priority(array->fences[i], prio);
 489         } else {
 490                 __fence_set_priority(fence, prio);
 491         }
 492 }
 493
 494 int
 495 i915_gem_object_wait_priority(struct drm_i915_gem_object *obj,
 496                               unsigned int flags,
 497                               int prio)
 498 {
 499         struct dma_fence *excl;
 500
 501         if (flags & I915_WAIT_ALL) {
 502                 struct dma_fence **shared;
 503                 unsigned int count, i;
 504                 int ret;
 505
 506                 ret = reservation_object_get_fences_rcu(obj->resv,
 507                                                         &excl, &count, &shared);
 508                 if (ret)
 509                         return ret;
 510
 511                 for (i = 0; i < count; i++) {
 512                         fence_set_priority(shared[i], prio);
 513                         dma_fence_put(shared[i]);
 514                 }
 515
 516                 kfree(shared);
 517         } else {
 518                 excl = reservation_object_get_excl_rcu(obj->resv);
 519         }
 520
 521         if (excl) {
 522                 fence_set_priority(excl, prio);
 523                 dma_fence_put(excl);
 524         }
 525         return 0;
 526 }
 527
 528 /**
 529  * Waits for rendering to the object to be completed
 530  * @obj: i915 gem object
 531  * @flags: how to wait (under a lock, for all rendering or just for writes etc)
 532  * @timeout: how long to wait
 533  * @rps: client (user process) to charge for any waitboosting
 534  */
 535 int
 536 i915_gem_object_wait(struct drm_i915_gem_object *obj,
 537                      unsigned int flags,
 538                      long timeout,
 539                      struct intel_rps_client *rps)
 540 {
 541         might_sleep();
 542 #if IS_ENABLED(CONFIG_LOCKDEP)
 543         GEM_BUG_ON(debug_locks &&
 544                    !!lockdep_is_held(&obj->base.dev->struct_mutex) !=
 545                    !!(flags & I915_WAIT_LOCKED));
 546 #endif
 547         GEM_BUG_ON(timeout < 0);
 548
 549         timeout = i915_gem_object_wait_reservation(obj->resv,
 550                                                    flags, timeout,
 551                                                    rps);
 552         return timeout < 0 ? timeout : 0;
 553 }
 554
 555 static struct intel_rps_client *to_rps_client(struct drm_file *file)
 556 {
 557         struct drm_i915_file_private *fpriv = file->driver_priv;
 558
 559         return &fpriv->rps;
 560 }
 561
 562 int
 563 i915_gem_object_attach_phys(struct drm_i915_gem_object *obj,
 564                             int align)
 565 {
 566         int ret;
 567
 568         if (align > obj->base.size)
 569                 return -EINVAL;
 570
 571         if (obj->ops == &i915_gem_phys_ops)
 572                 return 0;
 573
 574         if (obj->mm.madv != I915_MADV_WILLNEED)
 575                 return -EFAULT;
 576
 577         if (obj->base.filp == NULL)
 578                 return -EINVAL;
 579
 580         ret = i915_gem_object_unbind(obj);
 581         if (ret)
 582                 return ret;
 583
 584         __i915_gem_object_put_pages(obj, I915_MM_NORMAL);
 585         if (obj->mm.pages)
 586                 return -EBUSY;
 587
 588         obj->ops = &i915_gem_phys_ops;
 589
 590         return i915_gem_object_pin_pages(obj);
 591 }
 592
 593 static int
 594 i915_gem_phys_pwrite(struct drm_i915_gem_object *obj,
 595                      struct drm_i915_gem_pwrite *args,
 596                      struct drm_file *file)
 597 {
 598         struct drm_device *dev = obj->base.dev;
 599         void *vaddr = obj->phys_handle->vaddr + args->offset;
 600         char __user *user_data = u64_to_user_ptr(args->data_ptr);
 601         int ret;
 602
 603         /* We manually control the domain here and pretend that it
 604          * remains coherent i.e. in the GTT domain, like shmem_pwrite.
 605          */
 606         lockdep_assert_held(&obj->base.dev->struct_mutex);
 607         ret = i915_gem_object_wait(obj,
 608                                    I915_WAIT_INTERRUPTIBLE |
 609                                    I915_WAIT_LOCKED |
 610                                    I915_WAIT_ALL,
 611                                    MAX_SCHEDULE_TIMEOUT,
 612                                    to_rps_client(file));
 613         if (ret)
 614                 return ret;
 615
 616         intel_fb_obj_invalidate(obj, ORIGIN_CPU);
 617         if (__copy_from_user_inatomic_nocache(vaddr, user_data, args->size)) {
 618                 unsigned long unwritten;
 619
 620                 /* The physical object once assigned is fixed for the lifetime
 621                  * of the obj, so we can safely drop the lock and continue
 622                  * to access vaddr.
 623                  */
 624                 mutex_unlock(&dev->struct_mutex);
 625                 unwritten = copy_from_user(vaddr, user_data, args->size);
 626                 mutex_lock(&dev->struct_mutex);
 627                 if (unwritten) {
 628                         ret = -EFAULT;
 629                         goto out;
 630                 }
 631         }
 632
 633         drm_clflush_virt_range(vaddr, args->size);
 634         i915_gem_chipset_flush(to_i915(dev));
 635
 636 out:
 637         intel_fb_obj_flush(obj, false, ORIGIN_CPU);
 638         return ret;
 639 }
 640
 641 void *i915_gem_object_alloc(struct drm_device *dev)
 642 {
 643         struct drm_i915_private *dev_priv = to_i915(dev);
 644         return kmem_cache_zalloc(dev_priv->objects, GFP_KERNEL);
 645 }
 646
 647 void i915_gem_object_free(struct drm_i915_gem_object *obj)
 648 {
 649         struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
 650         kmem_cache_free(dev_priv->objects, obj);
 651 }
 652
 653 static int
 654 i915_gem_create(struct drm_file *file,
 655                 struct drm_device *dev,
 656                 uint64_t size,
 657                 uint32_t *handle_p)
 658 {
 659         struct drm_i915_gem_object *obj;
 660         int ret;
 661         u32 handle;
 662
 663         size = roundup(size, PAGE_SIZE);
 664         if (size == 0)
 665                 return -EINVAL;
 666
 667         /* Allocate the new object */
 668         obj = i915_gem_object_create(dev, size);
 669         if (IS_ERR(obj))
 670                 return PTR_ERR(obj);
 671
 672         ret = drm_gem_handle_create(file, &obj->base, &handle);
 673         /* drop reference from allocate - handle holds it now */
 674         i915_gem_object_put(obj);
 675         if (ret)
 676                 return ret;
 677
 678         *handle_p = handle;
 679         return 0;
 680 }
 681
 682 int
 683 i915_gem_dumb_create(struct drm_file *file,
 684                      struct drm_device *dev,
 685                      struct drm_mode_create_dumb *args)
 686 {
 687         /* have to work out size/pitch and return them */
 688         args->pitch = ALIGN(args->width * DIV_ROUND_UP(args->bpp, 8), 64);
 689         args->size = args->pitch * args->height;
 690         return i915_gem_create(file, dev,
 691                                args->size, &args->handle);
 692 }
 693
 694 /**
 695  * Creates a new mm object and returns a handle to it.
 696  * @dev: drm device pointer
 697  * @data: ioctl data blob
 698  * @file: drm file pointer
 699  */
 700 int
 701 i915_gem_create_ioctl(struct drm_device *dev, void *data,
 702                       struct drm_file *file)
 703 {
 704         struct drm_i915_gem_create *args = data;
 705
 706         i915_gem_flush_free_objects(to_i915(dev));
 707
 708         return i915_gem_create(file, dev,
 709                                args->size, &args->handle);
 710 }
 711
 712 static inline int
 713 __copy_to_user_swizzled(char __user *cpu_vaddr,
 714                         const char *gpu_vaddr, int gpu_offset,
 715                         int length)
 716 {
 717         int ret, cpu_offset = 0;
 718
 719         while (length > 0) {
 720                 int cacheline_end = ALIGN(gpu_offset + 1, 64);
 721                 int this_length = min(cacheline_end - gpu_offset, length);
 722                 int swizzled_gpu_offset = gpu_offset ^ 64;
 723
 724                 ret = __copy_to_user(cpu_vaddr + cpu_offset,
 725                                      gpu_vaddr + swizzled_gpu_offset,
 726                                      this_length);
 727                 if (ret)
 728                         return ret + length;
 729
 730                 cpu_offset += this_length;
 731                 gpu_offset += this_length;
 732                 length -= this_length;
 733         }
 734
 735         return 0;
 736 }
 737
 738 static inline int
 739 __copy_from_user_swizzled(char *gpu_vaddr, int gpu_offset,
 740                           const char __user *cpu_vaddr,
 741                           int length)
 742 {
 743         int ret, cpu_offset = 0;
 744
 745         while (length > 0) {
 746                 int cacheline_end = ALIGN(gpu_offset + 1, 64);
 747                 int this_length = min(cacheline_end - gpu_offset, length);
 748                 int swizzled_gpu_offset = gpu_offset ^ 64;
 749
 750                 ret = __copy_from_user(gpu_vaddr + swizzled_gpu_offset,
 751                                        cpu_vaddr + cpu_offset,
 752                                        this_length);
 753                 if (ret)
 754                         return ret + length;
 755
 756                 cpu_offset += this_length;
 757                 gpu_offset += this_length;
 758                 length -= this_length;
 759         }
 760
 761         return 0;
 762 }
 763
 764 /*
 765  * Pins the specified object's pages and synchronizes the object with
 766  * GPU accesses. Sets needs_clflush to non-zero if the caller should
 767  * flush the object from the CPU cache.
 768  */
 769 int i915_gem_obj_prepare_shmem_read(struct drm_i915_gem_object *obj,
 770                                     unsigned int *needs_clflush)
 771 {
 772         int ret;
 773
 774         lockdep_assert_held(&obj->base.dev->struct_mutex);
 775
 776         *needs_clflush = 0;
 777         if (!i915_gem_object_has_struct_page(obj))
 778                 return -ENODEV;
 779
 780         ret = i915_gem_object_wait(obj,
 781                                    I915_WAIT_INTERRUPTIBLE |
 782                                    I915_WAIT_LOCKED,
 783                                    MAX_SCHEDULE_TIMEOUT,
 784                                    NULL);
 785         if (ret)
 786                 return ret;
 787
 788         ret = i915_gem_object_pin_pages(obj);
 789         if (ret)
 790                 return ret;
 791
 792         i915_gem_object_flush_gtt_write_domain(obj);
 793
 794         /* If we're not in the cpu read domain, set ourself into the gtt
 795          * read domain and manually flush cachelines (if required). This
 796          * optimizes for the case when the gpu will dirty the data
 797          * anyway again before the next pread happens.
 798          */
 799         if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU))
 800                 *needs_clflush = !cpu_cache_is_coherent(obj->base.dev,
 801                                                         obj->cache_level);
 802
 803         if (*needs_clflush && !static_cpu_has(X86_FEATURE_CLFLUSH)) {
 804                 ret = i915_gem_object_set_to_cpu_domain(obj, false);
 805                 if (ret)
 806                         goto err_unpin;
 807
 808                 *needs_clflush = 0;
 809         }
 810
 811         /* return with the pages pinned */
 812         return 0;
 813
 814 err_unpin:
 815         i915_gem_object_unpin_pages(obj);
 816         return ret;
 817 }
 818
 819 int i915_gem_obj_prepare_shmem_write(struct drm_i915_gem_object *obj,
 820                                      unsigned int *needs_clflush)
 821 {
 822         int ret;
 823
 824         lockdep_assert_held(&obj->base.dev->struct_mutex);
 825
 826         *needs_clflush = 0;
 827         if (!i915_gem_object_has_struct_page(obj))
 828                 return -ENODEV;
 829
 830         ret = i915_gem_object_wait(obj,
 831                                    I915_WAIT_INTERRUPTIBLE |
 832                                    I915_WAIT_LOCKED |
 833                                    I915_WAIT_ALL,
 834                                    MAX_SCHEDULE_TIMEOUT,
 835                                    NULL);
 836         if (ret)
 837                 return ret;
 838
 839         ret = i915_gem_object_pin_pages(obj);
 840         if (ret)
 841                 return ret;
 842
 843         i915_gem_object_flush_gtt_write_domain(obj);
 844
 845         /* If we're not in the cpu write domain, set ourself into the
 846          * gtt write domain and manually flush cachelines (as required).
 847          * This optimizes for the case when the gpu will use the data
 848          * right away and we therefore have to clflush anyway.
 849          */
 850         if (obj->base.write_domain != I915_GEM_DOMAIN_CPU)
 851                 *needs_clflush |= cpu_write_needs_clflush(obj) << 1;
 852
 853         /* Same trick applies to invalidate partially written cachelines read
 854          * before writing.
 855          */
 856         if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU))
 857                 *needs_clflush |= !cpu_cache_is_coherent(obj->base.dev,
 858                                                          obj->cache_level);
 859
 860         if (*needs_clflush && !static_cpu_has(X86_FEATURE_CLFLUSH)) {
 861                 ret = i915_gem_object_set_to_cpu_domain(obj, true);
 862                 if (ret)
 863                         goto err_unpin;
 864
 865                 *needs_clflush = 0;
 866         }
 867
 868         if ((*needs_clflush & CLFLUSH_AFTER) == 0)
 869                 obj->cache_dirty = true;
 870
 871         intel_fb_obj_invalidate(obj, ORIGIN_CPU);
 872         obj->mm.dirty = true;
 873         /* return with the pages pinned */
 874         return 0;
 875
 876 err_unpin:
 877         i915_gem_object_unpin_pages(obj);
 878         return ret;
 879 }
 880
 881 static void
 882 shmem_clflush_swizzled_range(char *addr, unsigned long length,
 883                              bool swizzled)
 884 {
 885         if (unlikely(swizzled)) {
 886                 unsigned long start = (unsigned long) addr;
 887                 unsigned long end = (unsigned long) addr + length;
 888
 889                 /* For swizzling simply ensure that we always flush both
 890                  * channels. Lame, but simple and it works. Swizzled
 891                  * pwrite/pread is far from a hotpath - current userspace
 892                  * doesn't use it at all. */
 893                 start = round_down(start, 128);
 894                 end = round_up(end, 128);
 895
 896                 drm_clflush_virt_range((void *)start, end - start);
 897         } else {
 898                 drm_clflush_virt_range(addr, length);
 899         }
 900
 901 }
 902
 903 /* Only difference to the fast-path function is that this can handle bit17
 904  * and uses non-atomic copy and kmap functions. */
 905 static int
 906 shmem_pread_slow(struct page *page, int offset, int length,
 907                  char __user *user_data,
 908                  bool page_do_bit17_swizzling, bool needs_clflush)
 909 {
 910         char *vaddr;
 911         int ret;
 912
 913         vaddr = kmap(page);
 914         if (needs_clflush)
 915                 shmem_clflush_swizzled_range(vaddr + offset, length,
 916                                              page_do_bit17_swizzling);
 917
 918         if (page_do_bit17_swizzling)
 919                 ret = __copy_to_user_swizzled(user_data, vaddr, offset, length);
 920         else
 921                 ret = __copy_to_user(user_data, vaddr + offset, length);
 922         kunmap(page);
 923
 924         return ret ? - EFAULT : 0;
 925 }
 926
 927 static int
 928 shmem_pread(struct page *page, int offset, int length, char __user *user_data,
 929             bool page_do_bit17_swizzling, bool needs_clflush)
 930 {
 931         int ret;
 932
 933         ret = -ENODEV;
 934         if (!page_do_bit17_swizzling) {
 935                 char *vaddr = kmap_atomic(page);
 936
 937                 if (needs_clflush)
 938                         drm_clflush_virt_range(vaddr + offset, length);
 939                 ret = __copy_to_user_inatomic(user_data, vaddr + offset, length);
 940                 kunmap_atomic(vaddr);
 941         }
 942         if (ret == 0)
 943                 return 0;
 944
 945         return shmem_pread_slow(page, offset, length, user_data,
 946                                 page_do_bit17_swizzling, needs_clflush);
 947 }
 948
 949 static int
 950 i915_gem_shmem_pread(struct drm_i915_gem_object *obj,
 951                      struct drm_i915_gem_pread *args)
 952 {
 953         char __user *user_data;
 954         u64 remain;
 955         unsigned int obj_do_bit17_swizzling;
 956         unsigned int needs_clflush;
 957         unsigned int idx, offset;
 958         int ret;
 959
 960         obj_do_bit17_swizzling = 0;
 961         if (i915_gem_object_needs_bit17_swizzle(obj))
 962                 obj_do_bit17_swizzling = BIT(17);
 963
 964         ret = mutex_lock_interruptible(&obj->base.dev->struct_mutex);
 965         if (ret)
 966                 return ret;
 967
 968         ret = i915_gem_obj_prepare_shmem_read(obj, &needs_clflush);
 969         mutex_unlock(&obj->base.dev->struct_mutex);
 970         if (ret)
 971                 return ret;
 972
 973         remain = args->size;
 974         user_data = u64_to_user_ptr(args->data_ptr);
 975         offset = offset_in_page(args->offset);
 976         for (idx = args->offset >> PAGE_SHIFT; remain; idx++) {
 977                 struct page *page = i915_gem_object_get_page(obj, idx);
 978                 int length;
 979
 980                 length = remain;
 981                 if (offset + length > PAGE_SIZE)
 982                         length = PAGE_SIZE - offset;
 983
 984                 ret = shmem_pread(page, offset, length, user_data,
 985                                   page_to_phys(page) & obj_do_bit17_swizzling,
 986                                   needs_clflush);
 987                 if (ret)
 988                         break;
 989
 990                 remain -= length;
 991                 user_data += length;
 992                 offset = 0;
 993         }
 994
 995         i915_gem_obj_finish_shmem_access(obj);
 996         return ret;
 997 }
 998
 999 static inline bool
1000 gtt_user_read(struct io_mapping *mapping,
1001               loff_t base, int offset,
1002               char __user *user_data, int length)
1003 {
1004         void *vaddr;
1005         unsigned long unwritten;
1006
1007         /* We can use the cpu mem copy function because this is X86. */
1008         vaddr = (void __force *)io_mapping_map_atomic_wc(mapping, base);
1009         unwritten = __copy_to_user_inatomic(user_data, vaddr + offset, length);
1010         io_mapping_unmap_atomic(vaddr);
1011         if (unwritten) {
1012                 vaddr = (void __force *)
1013                         io_mapping_map_wc(mapping, base, PAGE_SIZE);
1014                 unwritten = copy_to_user(user_data, vaddr + offset, length);
1015                 io_mapping_unmap(vaddr);
1016         }
1017         return unwritten;
1018 }
1019
1020 static int
1021 i915_gem_gtt_pread(struct drm_i915_gem_object *obj,
1022                    const struct drm_i915_gem_pread *args)
1023 {
1024         struct drm_i915_private *i915 = to_i915(obj->base.dev);
1025         struct i915_ggtt *ggtt = &i915->ggtt;
1026         struct drm_mm_node node;
1027         struct i915_vma *vma;
1028         void __user *user_data;
1029         u64 remain, offset;
1030         int ret;
1031
1032         ret = mutex_lock_interruptible(&i915->drm.struct_mutex);
1033         if (ret)
1034                 return ret;
1035
1036         intel_runtime_pm_get(i915);
1037         vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0,
1038                                        PIN_MAPPABLE | PIN_NONBLOCK);
1039         if (!IS_ERR(vma)) {
1040                 node.start = i915_ggtt_offset(vma);
1041                 node.allocated = false;
1042                 ret = i915_vma_put_fence(vma);
1043                 if (ret) {
1044                         i915_vma_unpin(vma);
1045                         vma = ERR_PTR(ret);
1046                 }
1047         }
1048         if (IS_ERR(vma)) {
1049                 ret = insert_mappable_node(ggtt, &node, PAGE_SIZE);
1050                 if (ret)
1051                         goto out_unlock;
1052                 GEM_BUG_ON(!node.allocated);
1053         }
1054
1055         ret = i915_gem_object_set_to_gtt_domain(obj, false);
1056         if (ret)
1057                 goto out_unpin;
1058
1059         mutex_unlock(&i915->drm.struct_mutex);
1060
1061         user_data = u64_to_user_ptr(args->data_ptr);
1062         remain = args->size;
1063         offset = args->offset;
1064
1065         while (remain > 0) {
1066                 /* Operation in this page
1067                  *
1068                  * page_base = page offset within aperture
1069                  * page_offset = offset within page
1070                  * page_length = bytes to copy for this page
1071                  */
1072                 u32 page_base = node.start;
1073                 unsigned page_offset = offset_in_page(offset);
1074                 unsigned page_length = PAGE_SIZE - page_offset;
1075                 page_length = remain < page_length ? remain : page_length;
1076                 if (node.allocated) {
1077                         wmb();
1078                         ggtt->base.insert_page(&ggtt->base,
1079                                                i915_gem_object_get_dma_address(obj, offset >> PAGE_SHIFT),
1080                                                node.start, I915_CACHE_NONE, 0);
1081                         wmb();
1082                 } else {
1083                         page_base += offset & PAGE_MASK;
1084                 }
1085
1086                 if (gtt_user_read(&ggtt->mappable, page_base, page_offset,
1087                                   user_data, page_length)) {
1088                         ret = -EFAULT;
1089                         break;
1090                 }
1091
1092                 remain -= page_length;
1093                 user_data += page_length;
1094                 offset += page_length;
1095         }
1096
1097         mutex_lock(&i915->drm.struct_mutex);
1098 out_unpin:
1099         if (node.allocated) {
1100                 wmb();
1101                 ggtt->base.clear_range(&ggtt->base,
1102                                        node.start, node.size);
1103                 remove_mappable_node(&node);
1104         } else {
1105                 i915_vma_unpin(vma);
1106         }
1107 out_unlock:
1108         intel_runtime_pm_put(i915);
1109         mutex_unlock(&i915->drm.struct_mutex);
1110
1111         return ret;
1112 }
1113
1114 /**
1115  * Reads data from the object referenced by handle.
1116  * @dev: drm device pointer
1117  * @data: ioctl data blob
1118  * @file: drm file pointer
1119  *
1120  * On error, the contents of *data are undefined.
1121  */
1122 int
1123 i915_gem_pread_ioctl(struct drm_device *dev, void *data,
1124                      struct drm_file *file)
1125 {
1126         struct drm_i915_gem_pread *args = data;
1127         struct drm_i915_gem_object *obj;
1128         int ret;
1129
1130         if (args->size == 0)
1131                 return 0;
1132
1133         if (!access_ok(VERIFY_WRITE,
1134                        u64_to_user_ptr(args->data_ptr),
1135                        args->size))
1136                 return -EFAULT;
1137
1138         obj = i915_gem_object_lookup(file, args->handle);
1139         if (!obj)
1140                 return -ENOENT;
1141
1142         /* Bounds check source.  */
1143         if (args->offset > obj->base.size ||
1144             args->size > obj->base.size - args->offset) {
1145                 ret = -EINVAL;
1146                 goto out;
1147         }
1148
1149         trace_i915_gem_object_pread(obj, args->offset, args->size);
1150
1151         ret = i915_gem_object_wait(obj,
1152                                    I915_WAIT_INTERRUPTIBLE,
1153                                    MAX_SCHEDULE_TIMEOUT,
1154                                    to_rps_client(file));
1155         if (ret)
1156                 goto out;
1157
1158         ret = i915_gem_object_pin_pages(obj);
1159         if (ret)
1160                 goto out;
1161
1162         ret = i915_gem_shmem_pread(obj, args);
1163         if (ret == -EFAULT || ret == -ENODEV)
1164                 ret = i915_gem_gtt_pread(obj, args);
1165
1166         i915_gem_object_unpin_pages(obj);
1167 out:
1168         i915_gem_object_put(obj);
1169         return ret;
1170 }
1171
1172 /* This is the fast write path which cannot handle
1173  * page faults in the source data
1174  */
1175
1176 static inline bool
1177 ggtt_write(struct io_mapping *mapping,
1178            loff_t base, int offset,
1179            char __user *user_data, int length)
1180 {
1181         void *vaddr;
1182         unsigned long unwritten;
1183
1184         /* We can use the cpu mem copy function because this is X86. */
1185         vaddr = (void __force *)io_mapping_map_atomic_wc(mapping, base);
1186         unwritten = __copy_from_user_inatomic_nocache(vaddr + offset,
1187                                                       user_data, length);
1188         io_mapping_unmap_atomic(vaddr);
1189         if (unwritten) {
1190                 vaddr = (void __force *)
1191                         io_mapping_map_wc(mapping, base, PAGE_SIZE);
1192                 unwritten = copy_from_user(vaddr + offset, user_data, length);
1193                 io_mapping_unmap(vaddr);
1194         }
1195
1196         return unwritten;
1197 }
1198
1199 /**
1200  * This is the fast pwrite path, where we copy the data directly from the
1201  * user into the GTT, uncached.
1202  * @obj: i915 GEM object
1203  * @args: pwrite arguments structure
1204  */
1205 static int
1206 i915_gem_gtt_pwrite_fast(struct drm_i915_gem_object *obj,
1207                          const struct drm_i915_gem_pwrite *args)
1208 {
1209         struct drm_i915_private *i915 = to_i915(obj->base.dev);
1210         struct i915_ggtt *ggtt = &i915->ggtt;
1211         struct drm_mm_node node;
1212         struct i915_vma *vma;
1213         u64 remain, offset;
1214         void __user *user_data;
1215         int ret;
1216
1217         ret = mutex_lock_interruptible(&i915->drm.struct_mutex);
1218         if (ret)
1219                 return ret;
1220
1221         intel_runtime_pm_get(i915);
1222         vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0,
1223                                        PIN_MAPPABLE | PIN_NONBLOCK);
1224         if (!IS_ERR(vma)) {
1225                 node.start = i915_ggtt_offset(vma);
1226                 node.allocated = false;
1227                 ret = i915_vma_put_fence(vma);
1228                 if (ret) {
1229                         i915_vma_unpin(vma);
1230                         vma = ERR_PTR(ret);
1231                 }
1232         }
1233         if (IS_ERR(vma)) {
1234                 ret = insert_mappable_node(ggtt, &node, PAGE_SIZE);
1235                 if (ret)
1236                         goto out_unlock;
1237                 GEM_BUG_ON(!node.allocated);
1238         }
1239
1240         ret = i915_gem_object_set_to_gtt_domain(obj, true);
1241         if (ret)
1242                 goto out_unpin;
1243
1244         mutex_unlock(&i915->drm.struct_mutex);
1245
1246         intel_fb_obj_invalidate(obj, ORIGIN_CPU);
1247
1248         user_data = u64_to_user_ptr(args->data_ptr);
1249         offset = args->offset;
1250         remain = args->size;
1251         while (remain) {
1252                 /* Operation in this page
1253                  *
1254                  * page_base = page offset within aperture
1255                  * page_offset = offset within page
1256                  * page_length = bytes to copy for this page
1257                  */
1258                 u32 page_base = node.start;
1259                 unsigned int page_offset = offset_in_page(offset);
1260                 unsigned int page_length = PAGE_SIZE - page_offset;
1261                 page_length = remain < page_length ? remain : page_length;
1262                 if (node.allocated) {
1263                         wmb(); /* flush the write before we modify the GGTT */
1264                         ggtt->base.insert_page(&ggtt->base,
1265                                                i915_gem_object_get_dma_address(obj, offset >> PAGE_SHIFT),
1266                                                node.start, I915_CACHE_NONE, 0);
1267                         wmb(); /* flush modifications to the GGTT (insert_page) */
1268                 } else {
1269                         page_base += offset & PAGE_MASK;
1270                 }
1271                 /* If we get a fault while copying data, then (presumably) our
1272                  * source page isn't available.  Return the error and we'll
1273                  * retry in the slow path.
1274                  * If the object is non-shmem backed, we retry again with the
1275                  * path that handles page fault.
1276                  */
1277                 if (ggtt_write(&ggtt->mappable, page_base, page_offset,
1278                                user_data, page_length)) {
1279                         ret = -EFAULT;
1280                         break;
1281                 }
1282
1283                 remain -= page_length;
1284                 user_data += page_length;
1285                 offset += page_length;
1286         }
1287         intel_fb_obj_flush(obj, false, ORIGIN_CPU);
1288
1289         mutex_lock(&i915->drm.struct_mutex);
1290 out_unpin:
1291         if (node.allocated) {
1292                 wmb();
1293                 ggtt->base.clear_range(&ggtt->base,
1294                                        node.start, node.size);
1295                 remove_mappable_node(&node);
1296         } else {
1297                 i915_vma_unpin(vma);
1298         }
1299 out_unlock:
1300         intel_runtime_pm_put(i915);
1301         mutex_unlock(&i915->drm.struct_mutex);
1302         return ret;
1303 }
1304
1305 static int
1306 shmem_pwrite_slow(struct page *page, int offset, int length,
1307                   char __user *user_data,
1308                   bool page_do_bit17_swizzling,
1309                   bool needs_clflush_before,
1310                   bool needs_clflush_after)
1311 {
1312         char *vaddr;
1313         int ret;
1314
1315         vaddr = kmap(page);
1316         if (unlikely(needs_clflush_before || page_do_bit17_swizzling))
1317                 shmem_clflush_swizzled_range(vaddr + offset, length,
1318                                              page_do_bit17_swizzling);
1319         if (page_do_bit17_swizzling)
1320                 ret = __copy_from_user_swizzled(vaddr, offset, user_data,
1321                                                 length);
1322         else
1323                 ret = __copy_from_user(vaddr + offset, user_data, length);
1324         if (needs_clflush_after)
1325                 shmem_clflush_swizzled_range(vaddr + offset, length,
1326                                              page_do_bit17_swizzling);
1327         kunmap(page);
1328
1329         return ret ? -EFAULT : 0;
1330 }
1331
1332 /* Per-page copy function for the shmem pwrite fastpath.
1333  * Flushes invalid cachelines before writing to the target if
1334  * needs_clflush_before is set and flushes out any written cachelines after
1335  * writing if needs_clflush is set.
1336  */
1337 static int
1338 shmem_pwrite(struct page *page, int offset, int len, char __user *user_data,
1339              bool page_do_bit17_swizzling,
1340              bool needs_clflush_before,
1341              bool needs_clflush_after)
1342 {
1343         int ret;
1344
1345         ret = -ENODEV;
1346         if (!page_do_bit17_swizzling) {
1347                 char *vaddr = kmap_atomic(page);
1348
1349                 if (needs_clflush_before)
1350                         drm_clflush_virt_range(vaddr + offset, len);
1351                 ret = __copy_from_user_inatomic(vaddr + offset, user_data, len);
1352                 if (needs_clflush_after)
1353                         drm_clflush_virt_range(vaddr + offset, len);
1354
1355                 kunmap_atomic(vaddr);
1356         }
1357         if (ret == 0)
1358                 return ret;
1359
1360         return shmem_pwrite_slow(page, offset, len, user_data,
1361                                  page_do_bit17_swizzling,
1362                                  needs_clflush_before,
1363                                  needs_clflush_after);
1364 }
1365
1366 static int
1367 i915_gem_shmem_pwrite(struct drm_i915_gem_object *obj,
1368                       const struct drm_i915_gem_pwrite *args)
1369 {
1370         struct drm_i915_private *i915 = to_i915(obj->base.dev);
1371         void __user *user_data;
1372         u64 remain;
1373         unsigned int obj_do_bit17_swizzling;
1374         unsigned int partial_cacheline_write;
1375         unsigned int needs_clflush;
1376         unsigned int offset, idx;
1377         int ret;
1378
1379         ret = mutex_lock_interruptible(&i915->drm.struct_mutex);
1380         if (ret)
1381                 return ret;
1382
1383         ret = i915_gem_obj_prepare_shmem_write(obj, &needs_clflush);
1384         mutex_unlock(&i915->drm.struct_mutex);
1385         if (ret)
1386                 return ret;
1387
1388         obj_do_bit17_swizzling = 0;
1389         if (i915_gem_object_needs_bit17_swizzle(obj))
1390                 obj_do_bit17_swizzling = BIT(17);
1391
1392         /* If we don't overwrite a cacheline completely we need to be
1393          * careful to have up-to-date data by first clflushing. Don't
1394          * overcomplicate things and flush the entire patch.
1395          */
1396         partial_cacheline_write = 0;
1397         if (needs_clflush & CLFLUSH_BEFORE)
1398                 partial_cacheline_write = boot_cpu_data.x86_clflush_size - 1;
1399
1400         user_data = u64_to_user_ptr(args->data_ptr);
1401         remain = args->size;
1402         offset = offset_in_page(args->offset);
1403         for (idx = args->offset >> PAGE_SHIFT; remain; idx++) {
1404                 struct page *page = i915_gem_object_get_page(obj, idx);
1405                 int length;
1406
1407                 length = remain;
1408                 if (offset + length > PAGE_SIZE)
1409                         length = PAGE_SIZE - offset;
1410
1411                 ret = shmem_pwrite(page, offset, length, user_data,
1412                                    page_to_phys(page) & obj_do_bit17_swizzling,
1413                                    (offset | length) & partial_cacheline_write,
1414                                    needs_clflush & CLFLUSH_AFTER);
1415                 if (ret)
1416                         break;
1417
1418                 remain -= length;
1419                 user_data += length;
1420                 offset = 0;
1421         }
1422
1423         intel_fb_obj_flush(obj, false, ORIGIN_CPU);
1424         i915_gem_obj_finish_shmem_access(obj);
1425         return ret;
1426 }
1427
1428 /**
1429  * Writes data to the object referenced by handle.
1430  * @dev: drm device
1431  * @data: ioctl data blob
1432  * @file: drm file
1433  *
1434  * On error, the contents of the buffer that were to be modified are undefined.
1435  */
1436 int
1437 i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
1438                       struct drm_file *file)
1439 {
1440         struct drm_i915_gem_pwrite *args = data;
1441         struct drm_i915_gem_object *obj;
1442         int ret;
1443
1444         if (args->size == 0)
1445                 return 0;
1446
1447         if (!access_ok(VERIFY_READ,
1448                        u64_to_user_ptr(args->data_ptr),
1449                        args->size))
1450                 return -EFAULT;
1451
1452         obj = i915_gem_object_lookup(file, args->handle);
1453         if (!obj)
1454                 return -ENOENT;
1455
1456         /* Bounds check destination. */
1457         if (args->offset > obj->base.size ||
1458             args->size > obj->base.size - args->offset) {
1459                 ret = -EINVAL;
1460                 goto err;
1461         }
1462
1463         trace_i915_gem_object_pwrite(obj, args->offset, args->size);
1464
1465         ret = i915_gem_object_wait(obj,
1466                                    I915_WAIT_INTERRUPTIBLE |
1467                                    I915_WAIT_ALL,
1468                                    MAX_SCHEDULE_TIMEOUT,
1469                                    to_rps_client(file));
1470         if (ret)
1471                 goto err;
1472
1473         ret = i915_gem_object_pin_pages(obj);
1474         if (ret)
1475                 goto err;
1476
1477         ret = -EFAULT;
1478         /* We can only do the GTT pwrite on untiled buffers, as otherwise
1479          * it would end up going through the fenced access, and we'll get
1480          * different detiling behavior between reading and writing.
1481          * pread/pwrite currently are reading and writing from the CPU
1482          * perspective, requiring manual detiling by the client.
1483          */
1484         if (!i915_gem_object_has_struct_page(obj) ||
1485             cpu_write_needs_clflush(obj))
1486                 /* Note that the gtt paths might fail with non-page-backed user
1487                  * pointers (e.g. gtt mappings when moving data between
1488                  * textures). Fallback to the shmem path in that case.
1489                  */
1490                 ret = i915_gem_gtt_pwrite_fast(obj, args);
1491
1492         if (ret == -EFAULT || ret == -ENOSPC) {
1493                 if (obj->phys_handle)
1494                         ret = i915_gem_phys_pwrite(obj, args, file);
1495                 else
1496                         ret = i915_gem_shmem_pwrite(obj, args);
1497         }
1498
1499         i915_gem_object_unpin_pages(obj);
1500 err:
1501         i915_gem_object_put(obj);
1502         return ret;
1503 }
1504
1505 static inline enum fb_op_origin
1506 write_origin(struct drm_i915_gem_object *obj, unsigned domain)
1507 {
1508         return (domain == I915_GEM_DOMAIN_GTT ?
1509                 obj->frontbuffer_ggtt_origin : ORIGIN_CPU);
1510 }
1511
1512 static void i915_gem_object_bump_inactive_ggtt(struct drm_i915_gem_object *obj)
1513 {
1514         struct drm_i915_private *i915;
1515         struct list_head *list;
1516         struct i915_vma *vma;
1517
1518         list_for_each_entry(vma, &obj->vma_list, obj_link) {
1519                 if (!i915_vma_is_ggtt(vma))
1520                         continue;
1521
1522                 if (i915_vma_is_active(vma))
1523                         continue;
1524
1525                 if (!drm_mm_node_allocated(&vma->node))
1526                         continue;
1527
1528                 list_move_tail(&vma->vm_link, &vma->vm->inactive_list);
1529         }
1530
1531         i915 = to_i915(obj->base.dev);
1532         list = obj->bind_count ? &i915->mm.bound_list : &i915->mm.unbound_list;
1533         list_move_tail(&obj->global_link, list);
1534 }
1535
1536 /**
1537  * Called when user space prepares to use an object with the CPU, either
1538  * through the mmap ioctl's mapping or a GTT mapping.
1539  * @dev: drm device
1540  * @data: ioctl data blob
1541  * @file: drm file
1542  */
1543 int
1544 i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
1545                           struct drm_file *file)
1546 {
1547         struct drm_i915_gem_set_domain *args = data;
1548         struct drm_i915_gem_object *obj;
1549         uint32_t read_domains = args->read_domains;
1550         uint32_t write_domain = args->write_domain;
1551         int err;
1552
1553         /* Only handle setting domains to types used by the CPU. */
1554         if ((write_domain | read_domains) & I915_GEM_GPU_DOMAINS)
1555                 return -EINVAL;
1556
1557         /* Having something in the write domain implies it's in the read
1558          * domain, and only that read domain.  Enforce that in the request.
1559          */
1560         if (write_domain != 0 && read_domains != write_domain)
1561                 return -EINVAL;
1562
1563         obj = i915_gem_object_lookup(file, args->handle);
1564         if (!obj)
1565                 return -ENOENT;
1566
1567         /* Try to flush the object off the GPU without holding the lock.
1568          * We will repeat the flush holding the lock in the normal manner
1569          * to catch cases where we are gazumped.
1570          */
1571         err = i915_gem_object_wait(obj,
1572                                    I915_WAIT_INTERRUPTIBLE |
1573                                    (write_domain ? I915_WAIT_ALL : 0),
1574                                    MAX_SCHEDULE_TIMEOUT,
1575                                    to_rps_client(file));
1576         if (err)
1577                 goto out;
1578
1579         /* Flush and acquire obj->pages so that we are coherent through
1580          * direct access in memory with previous cached writes through
1581          * shmemfs and that our cache domain tracking remains valid.
1582          * For example, if the obj->filp was moved to swap without us
1583          * being notified and releasing the pages, we would mistakenly
1584          * continue to assume that the obj remained out of the CPU cached
1585          * domain.
1586          */
1587         err = i915_gem_object_pin_pages(obj);
1588         if (err)
1589                 goto out;
1590
1591         err = i915_mutex_lock_interruptible(dev);
1592         if (err)
1593                 goto out_unpin;
1594
1595         if (read_domains & I915_GEM_DOMAIN_GTT)
1596                 err = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
1597         else
1598                 err = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
1599
1600         /* And bump the LRU for this access */
1601         i915_gem_object_bump_inactive_ggtt(obj);
1602
1603         mutex_unlock(&dev->struct_mutex);
1604
1605         if (write_domain != 0)
1606                 intel_fb_obj_invalidate(obj, write_origin(obj, write_domain));
1607
1608 out_unpin:
1609         i915_gem_object_unpin_pages(obj);
1610 out:
1611         i915_gem_object_put(obj);
1612         return err;
1613 }
1614
1615 /**
1616  * Called when user space has done writes to this buffer
1617  * @dev: drm device
1618  * @data: ioctl data blob
1619  * @file: drm file
1620  */
1621 int
1622 i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
1623                          struct drm_file *file)
1624 {
1625         struct drm_i915_gem_sw_finish *args = data;
1626         struct drm_i915_gem_object *obj;
1627         int err = 0;
1628
1629         obj = i915_gem_object_lookup(file, args->handle);
1630         if (!obj)
1631                 return -ENOENT;
1632
1633         /* Pinned buffers may be scanout, so flush the cache */
1634         if (READ_ONCE(obj->pin_display)) {
1635                 err = i915_mutex_lock_interruptible(dev);
1636                 if (!err) {
1637                         i915_gem_object_flush_cpu_write_domain(obj);
1638                         mutex_unlock(&dev->struct_mutex);
1639                 }
1640         }
1641
1642         i915_gem_object_put(obj);
1643         return err;
1644 }
1645
1646 /**
1647  * i915_gem_mmap_ioctl - Maps the contents of an object, returning the address
1648  *                       it is mapped to.
1649  * @dev: drm device
1650  * @data: ioctl data blob
1651  * @file: drm file
1652  *
1653  * While the mapping holds a reference on the contents of the object, it doesn't
1654  * imply a ref on the object itself.
1655  *
1656  * IMPORTANT:
1657  *
1658  * DRM driver writers who look a this function as an example for how to do GEM
1659  * mmap support, please don't implement mmap support like here. The modern way
1660  * to implement DRM mmap support is with an mmap offset ioctl (like
1661  * i915_gem_mmap_gtt) and then using the mmap syscall on the DRM fd directly.
1662  * That way debug tooling like valgrind will understand what's going on, hiding
1663  * the mmap call in a driver private ioctl will break that. The i915 driver only
1664  * does cpu mmaps this way because we didn't know better.
1665  */
1666 int
1667 i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
1668                     struct drm_file *file)
1669 {
1670         struct drm_i915_gem_mmap *args = data;
1671         struct drm_i915_gem_object *obj;
1672         unsigned long addr;
1673
1674         if (args->flags & ~(I915_MMAP_WC))
1675                 return -EINVAL;
1676
1677         if (args->flags & I915_MMAP_WC && !boot_cpu_has(X86_FEATURE_PAT))
1678                 return -ENODEV;
1679
1680         obj = i915_gem_object_lookup(file, args->handle);
1681         if (!obj)
1682                 return -ENOENT;
1683
1684         /* prime objects have no backing filp to GEM mmap
1685          * pages from.
1686          */
1687         if (!obj->base.filp) {
1688                 i915_gem_object_put(obj);
1689                 return -EINVAL;
1690         }
1691
1692         addr = vm_mmap(obj->base.filp, 0, args->size,
1693                        PROT_READ | PROT_WRITE, MAP_SHARED,
1694                        args->offset);
1695         if (args->flags & I915_MMAP_WC) {
1696                 struct mm_struct *mm = current->mm;
1697                 struct vm_area_struct *vma;
1698
1699                 if (down_write_killable(&mm->mmap_sem)) {
1700                         i915_gem_object_put(obj);
1701                         return -EINTR;
1702                 }
1703                 vma = find_vma(mm, addr);
1704                 if (vma)
1705                         vma->vm_page_prot =
1706                                 pgprot_writecombine(vm_get_page_prot(vma->vm_flags));
1707                 else
1708                         addr = -ENOMEM;
1709                 up_write(&mm->mmap_sem);
1710
1711                 /* This may race, but that's ok, it only gets set */
1712                 WRITE_ONCE(obj->frontbuffer_ggtt_origin, ORIGIN_CPU);
1713         }
1714         i915_gem_object_put(obj);
1715         if (IS_ERR((void *)addr))
1716                 return addr;
1717
1718         args->addr_ptr = (uint64_t) addr;
1719
1720         return 0;
1721 }
1722
1723 static unsigned int tile_row_pages(struct drm_i915_gem_object *obj)
1724 {
1725         u64 size;
1726
1727         size = i915_gem_object_get_stride(obj);
1728         size *= i915_gem_object_get_tiling(obj) == I915_TILING_Y ? 32 : 8;
1729
1730         return size >> PAGE_SHIFT;
1731 }
1732
1733 /**
1734  * i915_gem_mmap_gtt_version - report the current feature set for GTT mmaps
1735  *
1736  * A history of the GTT mmap interface:
1737  *
1738  * 0 - Everything had to fit into the GTT. Both parties of a memcpy had to
1739  *     aligned and suitable for fencing, and still fit into the available
1740  *     mappable space left by the pinned display objects. A classic problem
1741  *     we called the page-fault-of-doom where we would ping-pong between
1742  *     two objects that could not fit inside the GTT and so the memcpy
1743  *     would page one object in at the expense of the other between every
1744  *     single byte.
1745  *
1746  * 1 - Objects can be any size, and have any compatible fencing (X Y, or none
1747  *     as set via i915_gem_set_tiling() [DRM_I915_GEM_SET_TILING]). If the
1748  *     object is too large for the available space (or simply too large
1749  *     for the mappable aperture!), a view is created instead and faulted
1750  *     into userspace. (This view is aligned and sized appropriately for
1751  *     fenced access.)
1752  *
1753  * Restrictions:
1754  *
1755  *  * snoopable objects cannot be accessed via the GTT. It can cause machine
1756  *    hangs on some architectures, corruption on others. An attempt to service
1757  *    a GTT page fault from a snoopable object will generate a SIGBUS.
1758  *
1759  *  * the object must be able to fit into RAM (physical memory, though no
1760  *    limited to the mappable aperture).
1761  *
1762  *
1763  * Caveats:
1764  *
1765  *  * a new GTT page fault will synchronize rendering from the GPU and flush
1766  *    all data to system memory. Subsequent access will not be synchronized.
1767  *
1768  *  * all mappings are revoked on runtime device suspend.
1769  *
1770  *  * there are only 8, 16 or 32 fence registers to share between all users
1771  *    (older machines require fence register for display and blitter access
1772  *    as well). Contention of the fence registers will cause the previous users
1773  *    to be unmapped and any new access will generate new page faults.
1774  *
1775  *  * running out of memory while servicing a fault may generate a SIGBUS,
1776  *    rather than the expected SIGSEGV.
1777  */
1778 int i915_gem_mmap_gtt_version(void)
1779 {
1780         return 1;
1781 }
1782
1783 /**
1784  * i915_gem_fault - fault a page into the GTT
1785  * @area: CPU VMA in question
1786  * @vmf: fault info
1787  *
1788  * The fault handler is set up by drm_gem_mmap() when a object is GTT mapped
1789  * from userspace.  The fault handler takes care of binding the object to
1790  * the GTT (if needed), allocating and programming a fence register (again,
1791  * only if needed based on whether the old reg is still valid or the object
1792  * is tiled) and inserting a new PTE into the faulting process.
1793  *
1794  * Note that the faulting process may involve evicting existing objects
1795  * from the GTT and/or fence registers to make room.  So performance may
1796  * suffer if the GTT working set is large or there are few fence registers
1797  * left.
1798  *
1799  * The current feature set supported by i915_gem_fault() and thus GTT mmaps
1800  * is exposed via I915_PARAM_MMAP_GTT_VERSION (see i915_gem_mmap_gtt_version).
1801  */
1802 int i915_gem_fault(struct vm_area_struct *area, struct vm_fault *vmf)
1803 {
1804 #define MIN_CHUNK_PAGES ((1 << 20) >> PAGE_SHIFT) /* 1 MiB */
1805         struct drm_i915_gem_object *obj = to_intel_bo(area->vm_private_data);
1806         struct drm_device *dev = obj->base.dev;
1807         struct drm_i915_private *dev_priv = to_i915(dev);
1808         struct i915_ggtt *ggtt = &dev_priv->ggtt;
1809         bool write = !!(vmf->flags & FAULT_FLAG_WRITE);
1810         struct i915_vma *vma;
1811         pgoff_t page_offset;
1812         unsigned int flags;
1813         int ret;
1814
1815         /* We don't use vmf->pgoff since that has the fake offset */
1816         page_offset = (vmf->address - area->vm_start) >> PAGE_SHIFT;
1817
1818         trace_i915_gem_object_fault(obj, page_offset, true, write);
1819
1820         /* Try to flush the object off the GPU first without holding the lock.
1821          * Upon acquiring the lock, we will perform our sanity checks and then
1822          * repeat the flush holding the lock in the normal manner to catch cases
1823          * where we are gazumped.
1824          */
1825         ret = i915_gem_object_wait(obj,
1826                                    I915_WAIT_INTERRUPTIBLE,
1827                                    MAX_SCHEDULE_TIMEOUT,
1828                                    NULL);
1829         if (ret)
1830                 goto err;
1831
1832         ret = i915_gem_object_pin_pages(obj);
1833         if (ret)
1834                 goto err;
1835
1836         intel_runtime_pm_get(dev_priv);
1837
1838         ret = i915_mutex_lock_interruptible(dev);
1839         if (ret)
1840                 goto err_rpm;
1841
1842         /* Access to snoopable pages through the GTT is incoherent. */
1843         if (obj->cache_level != I915_CACHE_NONE && !HAS_LLC(dev_priv)) {
1844                 ret = -EFAULT;
1845                 goto err_unlock;
1846         }
1847
1848         /* If the object is smaller than a couple of partial vma, it is
1849          * not worth only creating a single partial vma - we may as well
1850          * clear enough space for the full object.
1851          */
1852         flags = PIN_MAPPABLE;
1853         if (obj->base.size > 2 * MIN_CHUNK_PAGES << PAGE_SHIFT)
1854                 flags |= PIN_NONBLOCK | PIN_NONFAULT;
1855
1856         /* Now pin it into the GTT as needed */
1857         vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0, flags);
1858         if (IS_ERR(vma)) {
1859                 struct i915_ggtt_view view;
1860                 unsigned int chunk_size;
1861
1862                 /* Use a partial view if it is bigger than available space */
1863                 chunk_size = MIN_CHUNK_PAGES;
1864                 if (i915_gem_object_is_tiled(obj))
1865                         chunk_size = roundup(chunk_size, tile_row_pages(obj));
1866
1867                 memset(&view, 0, sizeof(view));
1868                 view.type = I915_GGTT_VIEW_PARTIAL;
1869                 view.params.partial.offset = rounddown(page_offset, chunk_size);
1870                 view.params.partial.size =
1871                         min_t(unsigned int, chunk_size,
1872                               vma_pages(area) - view.params.partial.offset);
1873
1874                 /* If the partial covers the entire object, just create a
1875                  * normal VMA.
1876                  */
1877                 if (chunk_size >= obj->base.size >> PAGE_SHIFT)
1878                         view.type = I915_GGTT_VIEW_NORMAL;
1879
1880                 /* Userspace is now writing through an untracked VMA, abandon
1881                  * all hope that the hardware is able to track future writes.
1882                  */
1883                 obj->frontbuffer_ggtt_origin = ORIGIN_CPU;
1884
1885                 vma = i915_gem_object_ggtt_pin(obj, &view, 0, 0, PIN_MAPPABLE);
1886         }
1887         if (IS_ERR(vma)) {
1888                 ret = PTR_ERR(vma);
1889                 goto err_unlock;
1890         }
1891
1892         ret = i915_gem_object_set_to_gtt_domain(obj, write);
1893         if (ret)
1894                 goto err_unpin;
1895
1896         ret = i915_vma_get_fence(vma);
1897         if (ret)
1898                 goto err_unpin;
1899
1900         /* Mark as being mmapped into userspace for later revocation */
1901         assert_rpm_wakelock_held(dev_priv);
1902         if (list_empty(&obj->userfault_link))
1903                 list_add(&obj->userfault_link, &dev_priv->mm.userfault_list);
1904
1905         /* Finally, remap it using the new GTT offset */
1906         ret = remap_io_mapping(area,
1907                                area->vm_start + (vma->ggtt_view.params.partial.offset << PAGE_SHIFT),
1908                                (ggtt->mappable_base + vma->node.start) >> PAGE_SHIFT,
1909                                min_t(u64, vma->size, area->vm_end - area->vm_start),
1910                                &ggtt->mappable);
1911
1912 err_unpin:
1913         __i915_vma_unpin(vma);
1914 err_unlock:
1915         mutex_unlock(&dev->struct_mutex);
1916 err_rpm:
1917         intel_runtime_pm_put(dev_priv);
1918         i915_gem_object_unpin_pages(obj);
1919 err:
1920         switch (ret) {
1921         case -EIO:
1922                 /*
1923                  * We eat errors when the gpu is terminally wedged to avoid
1924                  * userspace unduly crashing (gl has no provisions for mmaps to
1925                  * fail). But any other -EIO isn't ours (e.g. swap in failure)
1926                  * and so needs to be reported.
1927                  */
1928                 if (!i915_terminally_wedged(&dev_priv->gpu_error)) {
1929                         ret = VM_FAULT_SIGBUS;
1930                         break;
1931                 }
1932         case -EAGAIN:
1933                 /*
1934                  * EAGAIN means the gpu is hung and we'll wait for the error
1935                  * handler to reset everything when re-faulting in
1936                  * i915_mutex_lock_interruptible.
1937                  */
1938         case 0:
1939         case -ERESTARTSYS:
1940         case -EINTR:
1941         case -EBUSY:
1942                 /*
1943                  * EBUSY is ok: this just means that another thread
1944                  * already did the job.
1945                  */
1946                 ret = VM_FAULT_NOPAGE;
1947                 break;
1948         case -ENOMEM:
1949                 ret = VM_FAULT_OOM;
1950                 break;
1951         case -ENOSPC:
1952         case -EFAULT:
1953                 ret = VM_FAULT_SIGBUS;
1954                 break;
1955         default:
1956                 WARN_ONCE(ret, "unhandled error in i915_gem_fault: %i\n", ret);
1957                 ret = VM_FAULT_SIGBUS;
1958                 break;
1959         }
1960         return ret;
1961 }
1962
1963 /**
1964  * i915_gem_release_mmap - remove physical page mappings
1965  * @obj: obj in question
1966  *
1967  * Preserve the reservation of the mmapping with the DRM core code, but
1968  * relinquish ownership of the pages back to the system.
1969  *
1970  * It is vital that we remove the page mapping if we have mapped a tiled
1971  * object through the GTT and then lose the fence register due to
1972  * resource pressure. Similarly if the object has been moved out of the
1973  * aperture, than pages mapped into userspace must be revoked. Removing the
1974  * mapping will then trigger a page fault on the next user access, allowing
1975  * fixup by i915_gem_fault().
1976  */
1977 void
1978 i915_gem_release_mmap(struct drm_i915_gem_object *obj)
1979 {
1980         struct drm_i915_private *i915 = to_i915(obj->base.dev);
1981
1982         /* Serialisation between user GTT access and our code depends upon
1983          * revoking the CPU's PTE whilst the mutex is held. The next user
1984          * pagefault then has to wait until we release the mutex.
1985          *
1986          * Note that RPM complicates somewhat by adding an additional
1987          * requirement that operations to the GGTT be made holding the RPM
1988          * wakeref.
1989          */
1990         lockdep_assert_held(&i915->drm.struct_mutex);
1991         intel_runtime_pm_get(i915);
1992
1993         if (list_empty(&obj->userfault_link))
1994                 goto out;
1995
1996         list_del_init(&obj->userfault_link);
1997         drm_vma_node_unmap(&obj->base.vma_node,
1998                            obj->base.dev->anon_inode->i_mapping);
1999
2000         /* Ensure that the CPU's PTE are revoked and there are not outstanding
2001          * memory transactions from userspace before we return. The TLB
2002          * flushing implied above by changing the PTE above *should* be
2003          * sufficient, an extra barrier here just provides us with a bit
2004          * of paranoid documentation about our requirement to serialise
2005          * memory writes before touching registers / GSM.
2006          */
2007         wmb();
2008
2009 out:
2010         intel_runtime_pm_put(i915);
2011 }
2012
2013 void i915_gem_runtime_suspend(struct drm_i915_private *dev_priv)
2014 {
2015         struct drm_i915_gem_object *obj, *on;
2016         int i;
2017
2018         /*
2019          * Only called during RPM suspend. All users of the userfault_list
2020          * must be holding an RPM wakeref to ensure that this can not
2021          * run concurrently with themselves (and use the struct_mutex for
2022          * protection between themselves).
2023          */
2024
2025         list_for_each_entry_safe(obj, on,
2026                                  &dev_priv->mm.userfault_list, userfault_link) {
2027                 list_del_init(&obj->userfault_link);
2028                 drm_vma_node_unmap(&obj->base.vma_node,
2029                                    obj->base.dev->anon_inode->i_mapping);
2030         }
2031
2032         /* The fence will be lost when the device powers down. If any were
2033          * in use by hardware (i.e. they are pinned), we should not be powering
2034          * down! All other fences will be reacquired by the user upon waking.
2035          */
2036         for (i = 0; i < dev_priv->num_fence_regs; i++) {
2037                 struct drm_i915_fence_reg *reg = &dev_priv->fence_regs[i];
2038
2039                 if (WARN_ON(reg->pin_count))
2040                         continue;
2041
2042                 if (!reg->vma)
2043                         continue;
2044
2045                 GEM_BUG_ON(!list_empty(&reg->vma->obj->userfault_link));
2046                 reg->dirty = true;
2047         }
2048 }
2049
2050 /**
2051  * i915_gem_get_ggtt_size - return required global GTT size for an object
2052  * @dev_priv: i915 device
2053  * @size: object size
2054  * @tiling_mode: tiling mode
2055  *
2056  * Return the required global GTT size for an object, taking into account
2057  * potential fence register mapping.
2058  */
2059 u64 i915_gem_get_ggtt_size(struct drm_i915_private *dev_priv,
2060                            u64 size, int tiling_mode)
2061 {
2062         u64 ggtt_size;
2063
2064         GEM_BUG_ON(size == 0);
2065
2066         if (INTEL_GEN(dev_priv) >= 4 ||
2067             tiling_mode == I915_TILING_NONE)
2068                 return size;
2069
2070         /* Previous chips need a power-of-two fence region when tiling */
2071         if (IS_GEN3(dev_priv))
2072                 ggtt_size = 1024*1024;
2073         else
2074                 ggtt_size = 512*1024;
2075
2076         while (ggtt_size < size)
2077                 ggtt_size <<= 1;
2078
2079         return ggtt_size;
2080 }
2081
2082 /**
2083  * i915_gem_get_ggtt_alignment - return required global GTT alignment
2084  * @dev_priv: i915 device
2085  * @size: object size
2086  * @tiling_mode: tiling mode
2087  * @fenced: is fenced alignment required or not
2088  *
2089  * Return the required global GTT alignment for an object, taking into account
2090  * potential fence register mapping.
2091  */
2092 u64 i915_gem_get_ggtt_alignment(struct drm_i915_private *dev_priv, u64 size,
2093                                 int tiling_mode, bool fenced)
2094 {
2095         GEM_BUG_ON(size == 0);
2096
2097         /*
2098          * Minimum alignment is 4k (GTT page size), but might be greater
2099          * if a fence register is needed for the object.
2100          */
2101         if (INTEL_GEN(dev_priv) >= 4 || (!fenced && IS_G33(dev_priv)) ||
2102             tiling_mode == I915_TILING_NONE)
2103                 return 4096;
2104
2105         /*
2106          * Previous chips need to be aligned to the size of the smallest
2107          * fence register that can contain the object.
2108          */
2109         return i915_gem_get_ggtt_size(dev_priv, size, tiling_mode);
2110 }
2111
2112 static int i915_gem_object_create_mmap_offset(struct drm_i915_gem_object *obj)
2113 {
2114         struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
2115         int err;
2116
2117         err = drm_gem_create_mmap_offset(&obj->base);
2118         if (!err)
2119                 return 0;
2120
2121         /* We can idle the GPU locklessly to flush stale objects, but in order
2122          * to claim that space for ourselves, we need to take the big
2123          * struct_mutex to free the requests+objects and allocate our slot.
2124          */
2125         err = i915_gem_wait_for_idle(dev_priv, I915_WAIT_INTERRUPTIBLE);
2126         if (err)
2127                 return err;
2128
2129         err = i915_mutex_lock_interruptible(&dev_priv->drm);
2130         if (!err) {
2131                 i915_gem_retire_requests(dev_priv);
2132                 err = drm_gem_create_mmap_offset(&obj->base);
2133                 mutex_unlock(&dev_priv->drm.struct_mutex);
2134         }
2135
2136         return err;
2137 }
2138
2139 static void i915_gem_object_free_mmap_offset(struct drm_i915_gem_object *obj)
2140 {
2141         drm_gem_free_mmap_offset(&obj->base);
2142 }
2143
2144 int
2145 i915_gem_mmap_gtt(struct drm_file *file,
2146                   struct drm_device *dev,
2147                   uint32_t handle,
2148                   uint64_t *offset)
2149 {
2150         struct drm_i915_gem_object *obj;
2151         int ret;
2152
2153         obj = i915_gem_object_lookup(file, handle);
2154         if (!obj)
2155                 return -ENOENT;
2156
2157         ret = i915_gem_object_create_mmap_offset(obj);
2158         if (ret == 0)
2159                 *offset = drm_vma_node_offset_addr(&obj->base.vma_node);
2160
2161         i915_gem_object_put(obj);
2162         return ret;
2163 }
2164
2165 /**
2166  * i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing
2167  * @dev: DRM device
2168  * @data: GTT mapping ioctl data
2169  * @file: GEM object info
2170  *
2171  * Simply returns the fake offset to userspace so it can mmap it.
2172  * The mmap call will end up in drm_gem_mmap(), which will set things
2173  * up so we can get faults in the handler above.
2174  *
2175  * The fault handler will take care of binding the object into the GTT
2176  * (since it may have been evicted to make room for something), allocating
2177  * a fence register, and mapping the appropriate aperture address into
2178  * userspace.
2179  */
2180 int
2181 i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
2182                         struct drm_file *file)
2183 {
2184         struct drm_i915_gem_mmap_gtt *args = data;
2185
2186         return i915_gem_mmap_gtt(file, dev, args->handle, &args->offset);
2187 }
2188
2189 /* Immediately discard the backing storage */
2190 static void
2191 i915_gem_object_truncate(struct drm_i915_gem_object *obj)
2192 {
2193         i915_gem_object_free_mmap_offset(obj);
2194
2195         if (obj->base.filp == NULL)
2196                 return;
2197
2198         /* Our goal here is to return as much of the memory as
2199          * is possible back to the system as we are called from OOM.
2200          * To do this we must instruct the shmfs to drop all of its
2201          * backing pages, *now*.
2202          */
2203         shmem_truncate_range(file_inode(obj->base.filp), 0, (loff_t)-1);
2204         obj->mm.madv = __I915_MADV_PURGED;
2205 }
2206
2207 /* Try to discard unwanted pages */
2208 void __i915_gem_object_invalidate(struct drm_i915_gem_object *obj)
2209 {
2210         struct address_space *mapping;
2211
2212         lockdep_assert_held(&obj->mm.lock);
2213         GEM_BUG_ON(obj->mm.pages);
2214
2215         switch (obj->mm.madv) {
2216         case I915_MADV_DONTNEED:
2217                 i915_gem_object_truncate(obj);
2218         case __I915_MADV_PURGED:
2219                 return;
2220         }
2221
2222         if (obj->base.filp == NULL)
2223                 return;
2224
2225         mapping = obj->base.filp->f_mapping,
2226         invalidate_mapping_pages(mapping, 0, (loff_t)-1);
2227 }
2228
2229 static void
2230 i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj,
2231                               struct sg_table *pages)
2232 {
2233         struct sgt_iter sgt_iter;
2234         struct page *page;
2235
2236         __i915_gem_object_release_shmem(obj, pages, true);
2237
2238         i915_gem_gtt_finish_pages(obj, pages);
2239
2240         if (i915_gem_object_needs_bit17_swizzle(obj))
2241                 i915_gem_object_save_bit_17_swizzle(obj, pages);
2242
2243         for_each_sgt_page(page, sgt_iter, pages) {
2244                 if (obj->mm.dirty)
2245                         set_page_dirty(page);
2246
2247                 if (obj->mm.madv == I915_MADV_WILLNEED)
2248                         mark_page_accessed(page);
2249
2250                 put_page(page);
2251         }
2252         obj->mm.dirty = false;
2253
2254         sg_free_table(pages);
2255         kfree(pages);
2256 }
2257
2258 static void __i915_gem_object_reset_page_iter(struct drm_i915_gem_object *obj)
2259 {
2260         struct radix_tree_iter iter;
2261         void **slot;
2262
2263         radix_tree_for_each_slot(slot, &obj->mm.get_page.radix, &iter, 0)
2264                 radix_tree_delete(&obj->mm.get_page.radix, iter.index);
2265 }
2266
2267 void __i915_gem_object_put_pages(struct drm_i915_gem_object *obj,
2268                                  enum i915_mm_subclass subclass)
2269 {
2270         struct sg_table *pages;
2271
2272         if (i915_gem_object_has_pinned_pages(obj))
2273                 return;
2274
2275         GEM_BUG_ON(obj->bind_count);
2276         if (!READ_ONCE(obj->mm.pages))
2277                 return;
2278
2279         /* May be called by shrinker from within get_pages() (on another bo) */
2280         mutex_lock_nested(&obj->mm.lock, subclass);
2281         if (unlikely(atomic_read(&obj->mm.pages_pin_count)))
2282                 goto unlock;
2283
2284         /* ->put_pages might need to allocate memory for the bit17 swizzle
2285          * array, hence protect them from being reaped by removing them from gtt
2286          * lists early. */
2287         pages = fetch_and_zero(&obj->mm.pages);
2288         GEM_BUG_ON(!pages);
2289
2290         if (obj->mm.mapping) {
2291                 void *ptr;
2292
2293                 ptr = ptr_mask_bits(obj->mm.mapping);
2294                 if (is_vmalloc_addr(ptr))
2295                         vunmap(ptr);
2296                 else
2297                         kunmap(kmap_to_page(ptr));
2298
2299                 obj->mm.mapping = NULL;
2300         }
2301
2302         __i915_gem_object_reset_page_iter(obj);
2303
2304         obj->ops->put_pages(obj, pages);
2305 unlock:
2306         mutex_unlock(&obj->mm.lock);
2307 }
2308
2309 static void i915_sg_trim(struct sg_table *orig_st)
2310 {
2311         struct sg_table new_st;
2312         struct scatterlist *sg, *new_sg;
2313         unsigned int i;
2314
2315         if (orig_st->nents == orig_st->orig_nents)
2316                 return;
2317
2318         if (sg_alloc_table(&new_st, orig_st->nents, GFP_KERNEL | __GFP_NOWARN))
2319                 return;
2320
2321         new_sg = new_st.sgl;
2322         for_each_sg(orig_st->sgl, sg, orig_st->nents, i) {
2323                 sg_set_page(new_sg, sg_page(sg), sg->length, 0);
2324                 /* called before being DMA mapped, no need to copy sg->dma_* */
2325                 new_sg = sg_next(new_sg);
2326         }
2327
2328         sg_free_table(orig_st);
2329
2330         *orig_st = new_st;
2331 }
2332
2333 static struct sg_table *
2334 i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj)
2335 {
2336         struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
2337         const unsigned long page_count = obj->base.size / PAGE_SIZE;
2338         unsigned long i;
2339         struct address_space *mapping;
2340         struct sg_table *st;
2341         struct scatterlist *sg;
2342         struct sgt_iter sgt_iter;
2343         struct page *page;
2344         unsigned long last_pfn = 0;     /* suppress gcc warning */
2345         unsigned int max_segment;
2346         int ret;
2347         gfp_t gfp;
2348
2349         /* Assert that the object is not currently in any GPU domain. As it
2350          * wasn't in the GTT, there shouldn't be any way it could have been in
2351          * a GPU cache
2352          */
2353         GEM_BUG_ON(obj->base.read_domains & I915_GEM_GPU_DOMAINS);
2354         GEM_BUG_ON(obj->base.write_domain & I915_GEM_GPU_DOMAINS);
2355
2356         max_segment = swiotlb_max_segment();
2357         if (!max_segment)
2358                 max_segment = rounddown(UINT_MAX, PAGE_SIZE);
2359
2360         st = kmalloc(sizeof(*st), GFP_KERNEL);
2361         if (st == NULL)
2362                 return ERR_PTR(-ENOMEM);
2363
2364 rebuild_st:
2365         if (sg_alloc_table(st, page_count, GFP_KERNEL)) {
2366                 kfree(st);
2367                 return ERR_PTR(-ENOMEM);
2368         }
2369
2370         /* Get the list of pages out of our struct file.  They'll be pinned
2371          * at this point until we release them.
2372          *
2373          * Fail silently without starting the shrinker
2374          */
2375         mapping = obj->base.filp->f_mapping;
2376         gfp = mapping_gfp_constraint(mapping, ~(__GFP_IO | __GFP_RECLAIM));
2377         gfp |= __GFP_NORETRY | __GFP_NOWARN;
2378         sg = st->sgl;
2379         st->nents = 0;
2380         for (i = 0; i < page_count; i++) {
2381                 page = shmem_read_mapping_page_gfp(mapping, i, gfp);
2382                 if (IS_ERR(page)) {
2383                         i915_gem_shrink(dev_priv,
2384                                         page_count,
2385                                         I915_SHRINK_BOUND |
2386                                         I915_SHRINK_UNBOUND |
2387                                         I915_SHRINK_PURGEABLE);
2388                         page = shmem_read_mapping_page_gfp(mapping, i, gfp);
2389                 }
2390                 if (IS_ERR(page)) {
2391                         /* We've tried hard to allocate the memory by reaping
2392                          * our own buffer, now let the real VM do its job and
2393                          * go down in flames if truly OOM.
2394                          */
2395                         page = shmem_read_mapping_page(mapping, i);
2396                         if (IS_ERR(page)) {
2397                                 ret = PTR_ERR(page);
2398                                 goto err_sg;
2399                         }
2400                 }
2401                 if (!i ||
2402                     sg->length >= max_segment ||
2403                     page_to_pfn(page) != last_pfn + 1) {
2404                         if (i)
2405                                 sg = sg_next(sg);
2406                         st->nents++;
2407                         sg_set_page(sg, page, PAGE_SIZE, 0);
2408                 } else {
2409                         sg->length += PAGE_SIZE;
2410                 }
2411                 last_pfn = page_to_pfn(page);
2412
2413                 /* Check that the i965g/gm workaround works. */
2414                 WARN_ON((gfp & __GFP_DMA32) && (last_pfn >= 0x00100000UL));
2415         }
2416         if (sg) /* loop terminated early; short sg table */
2417                 sg_mark_end(sg);
2418
2419         /* Trim unused sg entries to avoid wasting memory. */
2420         i915_sg_trim(st);
2421
2422         ret = i915_gem_gtt_prepare_pages(obj, st);
2423         if (ret) {
2424                 /* DMA remapping failed? One possible cause is that
2425                  * it could not reserve enough large entries, asking
2426                  * for PAGE_SIZE chunks instead may be helpful.
2427                  */
2428                 if (max_segment > PAGE_SIZE) {
2429                         for_each_sgt_page(page, sgt_iter, st)
2430                                 put_page(page);
2431                         sg_free_table(st);
2432
2433                         max_segment = PAGE_SIZE;
2434                         goto rebuild_st;
2435                 } else {
2436                         dev_warn(&dev_priv->drm.pdev->dev,
2437                                  "Failed to DMA remap %lu pages\n",
2438                                  page_count);
2439                         goto err_pages;
2440                 }
2441         }
2442
2443         if (i915_gem_object_needs_bit17_swizzle(obj))
2444                 i915_gem_object_do_bit_17_swizzle(obj, st);
2445
2446         return st;
2447
2448 err_sg:
2449         sg_mark_end(sg);
2450 err_pages:
2451         for_each_sgt_page(page, sgt_iter, st)
2452                 put_page(page);
2453         sg_free_table(st);
2454         kfree(st);
2455
2456         /* shmemfs first checks if there is enough memory to allocate the page
2457          * and reports ENOSPC should there be insufficient, along with the usual
2458          * ENOMEM for a genuine allocation failure.
2459          *
2460          * We use ENOSPC in our driver to mean that we have run out of aperture
2461          * space and so want to translate the error from shmemfs back to our
2462          * usual understanding of ENOMEM.
2463          */
2464         if (ret == -ENOSPC)
2465                 ret = -ENOMEM;
2466
2467         return ERR_PTR(ret);
2468 }
2469
2470 void __i915_gem_object_set_pages(struct drm_i915_gem_object *obj,
2471                                  struct sg_table *pages)
2472 {
2473         lockdep_assert_held(&obj->mm.lock);
2474
2475         obj->mm.get_page.sg_pos = pages->sgl;
2476         obj->mm.get_page.sg_idx = 0;
2477
2478         obj->mm.pages = pages;
2479
2480         if (i915_gem_object_is_tiled(obj) &&
2481             to_i915(obj->base.dev)->quirks & QUIRK_PIN_SWIZZLED_PAGES) {
2482                 GEM_BUG_ON(obj->mm.quirked);
2483                 __i915_gem_object_pin_pages(obj);
2484                 obj->mm.quirked = true;
2485         }
2486 }
2487
2488 static int ____i915_gem_object_get_pages(struct drm_i915_gem_object *obj)
2489 {
2490         struct sg_table *pages;
2491
2492         GEM_BUG_ON(i915_gem_object_has_pinned_pages(obj));
2493
2494         if (unlikely(obj->mm.madv != I915_MADV_WILLNEED)) {
2495                 DRM_DEBUG("Attempting to obtain a purgeable object\n");
2496                 return -EFAULT;
2497         }
2498
2499         pages = obj->ops->get_pages(obj);
2500         if (unlikely(IS_ERR(pages)))
2501                 return PTR_ERR(pages);
2502
2503         __i915_gem_object_set_pages(obj, pages);
2504         return 0;
2505 }
2506
2507 /* Ensure that the associated pages are gathered from the backing storage
2508  * and pinned into our object. i915_gem_object_pin_pages() may be called
2509  * multiple times before they are released by a single call to
2510  * i915_gem_object_unpin_pages() - once the pages are no longer referenced
2511  * either as a result of memory pressure (reaping pages under the shrinker)
2512  * or as the object is itself released.
2513  */
2514 int __i915_gem_object_get_pages(struct drm_i915_gem_object *obj)
2515 {
2516         int err;
2517
2518         err = mutex_lock_interruptible(&obj->mm.lock);
2519         if (err)
2520                 return err;
2521
2522         if (unlikely(!obj->mm.pages)) {
2523                 err = ____i915_gem_object_get_pages(obj);
2524                 if (err)
2525                         goto unlock;
2526
2527                 smp_mb__before_atomic();
2528         }
2529         atomic_inc(&obj->mm.pages_pin_count);
2530
2531 unlock:
2532         mutex_unlock(&obj->mm.lock);
2533         return err;
2534 }
2535
2536 /* The 'mapping' part of i915_gem_object_pin_map() below */
2537 static void *i915_gem_object_map(const struct drm_i915_gem_object *obj,
2538                                  enum i915_map_type type)
2539 {
2540         unsigned long n_pages = obj->base.size >> PAGE_SHIFT;
2541         struct sg_table *sgt = obj->mm.pages;
2542         struct sgt_iter sgt_iter;
2543         struct page *page;
2544         struct page *stack_pages[32];
2545         struct page **pages = stack_pages;
2546         unsigned long i = 0;
2547         pgprot_t pgprot;
2548         void *addr;
2549
2550         /* A single page can always be kmapped */
2551         if (n_pages == 1 && type == I915_MAP_WB)
2552                 return kmap(sg_page(sgt->sgl));
2553
2554         if (n_pages > ARRAY_SIZE(stack_pages)) {
2555                 /* Too big for stack -- allocate temporary array instead */
2556                 pages = drm_malloc_gfp(n_pages, sizeof(*pages), GFP_TEMPORARY);
2557                 if (!pages)
2558                         return NULL;
2559         }
2560
2561         for_each_sgt_page(page, sgt_iter, sgt)
2562                 pages[i++] = page;
2563
2564         /* Check that we have the expected number of pages */
2565         GEM_BUG_ON(i != n_pages);
2566
2567         switch (type) {
2568         case I915_MAP_WB:
2569                 pgprot = PAGE_KERNEL;
2570                 break;
2571         case I915_MAP_WC:
2572                 pgprot = pgprot_writecombine(PAGE_KERNEL_IO);
2573                 break;
2574         }
2575         addr = vmap(pages, n_pages, 0, pgprot);
2576
2577         if (pages != stack_pages)
2578                 drm_free_large(pages);
2579
2580         return addr;
2581 }
2582
2583 /* get, pin, and map the pages of the object into kernel space */
2584 void *i915_gem_object_pin_map(struct drm_i915_gem_object *obj,
2585                               enum i915_map_type type)
2586 {
2587         enum i915_map_type has_type;
2588         bool pinned;
2589         void *ptr;
2590         int ret;
2591
2592         GEM_BUG_ON(!i915_gem_object_has_struct_page(obj));
2593
2594         ret = mutex_lock_interruptible(&obj->mm.lock);
2595         if (ret)
2596                 return ERR_PTR(ret);
2597
2598         pinned = true;
2599         if (!atomic_inc_not_zero(&obj->mm.pages_pin_count)) {
2600                 if (unlikely(!obj->mm.pages)) {
2601                         ret = ____i915_gem_object_get_pages(obj);
2602                         if (ret)
2603                                 goto err_unlock;
2604
2605                         smp_mb__before_atomic();
2606                 }
2607                 atomic_inc(&obj->mm.pages_pin_count);
2608                 pinned = false;
2609         }
2610         GEM_BUG_ON(!obj->mm.pages);
2611
2612         ptr = ptr_unpack_bits(obj->mm.mapping, has_type);
2613         if (ptr && has_type != type) {
2614                 if (pinned) {
2615                         ret = -EBUSY;
2616                         goto err_unpin;
2617                 }
2618
2619                 if (is_vmalloc_addr(ptr))
2620                         vunmap(ptr);
2621                 else
2622                         kunmap(kmap_to_page(ptr));
2623
2624                 ptr = obj->mm.mapping = NULL;
2625         }
2626
2627         if (!ptr) {
2628                 ptr = i915_gem_object_map(obj, type);
2629                 if (!ptr) {
2630                         ret = -ENOMEM;
2631                         goto err_unpin;
2632                 }
2633
2634                 obj->mm.mapping = ptr_pack_bits(ptr, type);
2635         }
2636
2637 out_unlock:
2638         mutex_unlock(&obj->mm.lock);
2639         return ptr;
2640
2641 err_unpin:
2642         atomic_dec(&obj->mm.pages_pin_count);
2643 err_unlock:
2644         ptr = ERR_PTR(ret);
2645         goto out_unlock;
2646 }
2647
2648 static bool i915_context_is_banned(const struct i915_gem_context *ctx)
2649 {
2650         unsigned long elapsed;
2651
2652         if (ctx->hang_stats.banned)
2653                 return true;
2654
2655         elapsed = get_seconds() - ctx->hang_stats.guilty_ts;
2656         if (ctx->hang_stats.ban_period_seconds &&
2657             elapsed <= ctx->hang_stats.ban_period_seconds) {
2658                 DRM_DEBUG("context hanging too fast, banning!\n");
2659                 return true;
2660         }
2661
2662         return false;
2663 }
2664
2665 static void i915_set_reset_status(struct i915_gem_context *ctx,
2666                                   const bool guilty)
2667 {
2668         struct i915_ctx_hang_stats *hs = &ctx->hang_stats;
2669
2670         if (guilty) {
2671                 hs->banned = i915_context_is_banned(ctx);
2672                 hs->batch_active++;
2673                 hs->guilty_ts = get_seconds();
2674         } else {
2675                 hs->batch_pending++;
2676         }
2677 }
2678
2679 struct drm_i915_gem_request *
2680 i915_gem_find_active_request(struct intel_engine_cs *engine)
2681 {
2682         struct drm_i915_gem_request *request;
2683
2684         /* We are called by the error capture and reset at a random
2685          * point in time. In particular, note that neither is crucially
2686          * ordered with an interrupt. After a hang, the GPU is dead and we
2687          * assume that no more writes can happen (we waited long enough for
2688          * all writes that were in transaction to be flushed) - adding an
2689          * extra delay for a recent interrupt is pointless. Hence, we do
2690          * not need an engine->irq_seqno_barrier() before the seqno reads.
2691          */
2692         list_for_each_entry(request, &engine->timeline->requests, link) {
2693                 if (__i915_gem_request_completed(request))
2694                         continue;
2695
2696                 return request;
2697         }
2698
2699         return NULL;
2700 }
2701
2702 static void reset_request(struct drm_i915_gem_request *request)
2703 {
2704         void *vaddr = request->ring->vaddr;
2705         u32 head;
2706
2707         /* As this request likely depends on state from the lost
2708          * context, clear out all the user operations leaving the
2709          * breadcrumb at the end (so we get the fence notifications).
2710          */
2711         head = request->head;
2712         if (request->postfix < head) {
2713                 memset(vaddr + head, 0, request->ring->size - head);
2714                 head = 0;
2715         }
2716         memset(vaddr + head, 0, request->postfix - head);
2717 }
2718
2719 static void i915_gem_reset_engine(struct intel_engine_cs *engine)
2720 {
2721         struct drm_i915_gem_request *request;
2722         struct i915_gem_context *incomplete_ctx;
2723         struct intel_timeline *timeline;
2724         unsigned long flags;
2725         bool ring_hung;
2726
2727         if (engine->irq_seqno_barrier)
2728                 engine->irq_seqno_barrier(engine);
2729
2730         request = i915_gem_find_active_request(engine);
2731         if (!request)
2732                 return;
2733
2734         ring_hung = engine->hangcheck.score >= HANGCHECK_SCORE_RING_HUNG;
2735         if (engine->hangcheck.seqno != intel_engine_get_seqno(engine))
2736                 ring_hung = false;
2737
2738         i915_set_reset_status(request->ctx, ring_hung);
2739         if (!ring_hung)
2740                 return;
2741
2742         DRM_DEBUG_DRIVER("resetting %s to restart from tail of request 0x%x\n",
2743                          engine->name, request->global_seqno);
2744
2745         /* Setup the CS to resume from the breadcrumb of the hung request */
2746         engine->reset_hw(engine, request);
2747
2748         /* Users of the default context do not rely on logical state
2749          * preserved between batches. They have to emit full state on
2750          * every batch and so it is safe to execute queued requests following
2751          * the hang.
2752          *
2753          * Other contexts preserve state, now corrupt. We want to skip all
2754          * queued requests that reference the corrupt context.
2755          */
2756         incomplete_ctx = request->ctx;
2757         if (i915_gem_context_is_default(incomplete_ctx))
2758                 return;
2759
2760         timeline = i915_gem_context_lookup_timeline(incomplete_ctx, engine);
2761
2762         spin_lock_irqsave(&engine->timeline->lock, flags);
2763         spin_lock(&timeline->lock);
2764
2765         list_for_each_entry_continue(request, &engine->timeline->requests, link)
2766                 if (request->ctx == incomplete_ctx)
2767                         reset_request(request);
2768
2769         list_for_each_entry(request, &timeline->requests, link)
2770                 reset_request(request);
2771
2772         spin_unlock(&timeline->lock);
2773         spin_unlock_irqrestore(&engine->timeline->lock, flags);
2774 }
2775
2776 void i915_gem_reset(struct drm_i915_private *dev_priv)
2777 {
2778         struct intel_engine_cs *engine;
2779         enum intel_engine_id id;
2780
2781         lockdep_assert_held(&dev_priv->drm.struct_mutex);
2782
2783         i915_gem_retire_requests(dev_priv);
2784
2785         for_each_engine(engine, dev_priv, id)
2786                 i915_gem_reset_engine(engine);
2787
2788         i915_gem_restore_fences(dev_priv);
2789
2790         if (dev_priv->gt.awake) {
2791                 intel_sanitize_gt_powersave(dev_priv);
2792                 intel_enable_gt_powersave(dev_priv);
2793                 if (INTEL_GEN(dev_priv) >= 6)
2794                         gen6_rps_busy(dev_priv);
2795         }
2796 }
2797
2798 static void nop_submit_request(struct drm_i915_gem_request *request)
2799 {
2800         i915_gem_request_submit(request);
2801         intel_engine_init_global_seqno(request->engine, request->global_seqno);
2802 }
2803
2804 static void i915_gem_cleanup_engine(struct intel_engine_cs *engine)
2805 {
2806         engine->submit_request = nop_submit_request;
2807
2808         /* Mark all pending requests as complete so that any concurrent
2809          * (lockless) lookup doesn't try and wait upon the request as we
2810          * reset it.
2811          */
2812         intel_engine_init_global_seqno(engine,
2813                                        intel_engine_last_submit(engine));
2814
2815         /*
2816          * Clear the execlists queue up before freeing the requests, as those
2817          * are the ones that keep the context and ringbuffer backing objects
2818          * pinned in place.
2819          */
2820
2821         if (i915.enable_execlists) {
2822                 unsigned long flags;
2823
2824                 spin_lock_irqsave(&engine->timeline->lock, flags);
2825
2826                 i915_gem_request_put(engine->execlist_port[0].request);
2827                 i915_gem_request_put(engine->execlist_port[1].request);
2828                 memset(engine->execlist_port, 0, sizeof(engine->execlist_port));
2829                 engine->execlist_queue = RB_ROOT;
2830                 engine->execlist_first = NULL;
2831
2832                 spin_unlock_irqrestore(&engine->timeline->lock, flags);
2833         }
2834 }
2835
2836 void i915_gem_set_wedged(struct drm_i915_private *dev_priv)
2837 {
2838         struct intel_engine_cs *engine;
2839         enum intel_engine_id id;
2840
2841         lockdep_assert_held(&dev_priv->drm.struct_mutex);
2842         set_bit(I915_WEDGED, &dev_priv->gpu_error.flags);
2843
2844         i915_gem_context_lost(dev_priv);
2845         for_each_engine(engine, dev_priv, id)
2846                 i915_gem_cleanup_engine(engine);
2847         mod_delayed_work(dev_priv->wq, &dev_priv->gt.idle_work, 0);
2848
2849         i915_gem_retire_requests(dev_priv);
2850 }
2851
2852 static void
2853 i915_gem_retire_work_handler(struct work_struct *work)
2854 {
2855         struct drm_i915_private *dev_priv =
2856                 container_of(work, typeof(*dev_priv), gt.retire_work.work);
2857         struct drm_device *dev = &dev_priv->drm;
2858
2859         /* Come back later if the device is busy... */
2860         if (mutex_trylock(&dev->struct_mutex)) {
2861                 i915_gem_retire_requests(dev_priv);
2862                 mutex_unlock(&dev->struct_mutex);
2863         }
2864
2865         /* Keep the retire handler running until we are finally idle.
2866          * We do not need to do this test under locking as in the worst-case
2867          * we queue the retire worker once too often.
2868          */
2869         if (READ_ONCE(dev_priv->gt.awake)) {
2870                 i915_queue_hangcheck(dev_priv);
2871                 queue_delayed_work(dev_priv->wq,
2872                                    &dev_priv->gt.retire_work,
2873                                    round_jiffies_up_relative(HZ));
2874         }
2875 }
2876
2877 static void
2878 i915_gem_idle_work_handler(struct work_struct *work)
2879 {
2880         struct drm_i915_private *dev_priv =
2881                 container_of(work, typeof(*dev_priv), gt.idle_work.work);
2882         struct drm_device *dev = &dev_priv->drm;
2883         struct intel_engine_cs *engine;
2884         enum intel_engine_id id;
2885         bool rearm_hangcheck;
2886
2887         if (!READ_ONCE(dev_priv->gt.awake))
2888                 return;
2889
2890         /*
2891          * Wait for last execlists context complete, but bail out in case a
2892          * new request is submitted.
2893          */
2894         wait_for(READ_ONCE(dev_priv->gt.active_requests) ||
2895                  intel_execlists_idle(dev_priv), 10);
2896
2897         if (READ_ONCE(dev_priv->gt.active_requests))
2898                 return;
2899
2900         rearm_hangcheck =
2901                 cancel_delayed_work_sync(&dev_priv->gpu_error.hangcheck_work);
2902
2903         if (!mutex_trylock(&dev->struct_mutex)) {
2904                 /* Currently busy, come back later */
2905                 mod_delayed_work(dev_priv->wq,
2906                                  &dev_priv->gt.idle_work,
2907                                  msecs_to_jiffies(50));
2908                 goto out_rearm;
2909         }
2910
2911         /*
2912          * New request retired after this work handler started, extend active
2913          * period until next instance of the work.
2914          */
2915         if (work_pending(work))
2916                 goto out_unlock;
2917
2918         if (dev_priv->gt.active_requests)
2919                 goto out_unlock;
2920
2921         if (wait_for(intel_execlists_idle(dev_priv), 10))
2922                 DRM_ERROR("Timeout waiting for engines to idle\n");
2923
2924         for_each_engine(engine, dev_priv, id)
2925                 i915_gem_batch_pool_fini(&engine->batch_pool);
2926
2927         GEM_BUG_ON(!dev_priv->gt.awake);
2928         dev_priv->gt.awake = false;
2929         rearm_hangcheck = false;
2930
2931         if (INTEL_GEN(dev_priv) >= 6)
2932                 gen6_rps_idle(dev_priv);
2933         intel_runtime_pm_put(dev_priv);
2934 out_unlock:
2935         mutex_unlock(&dev->struct_mutex);
2936
2937 out_rearm:
2938         if (rearm_hangcheck) {
2939                 GEM_BUG_ON(!dev_priv->gt.awake);
2940                 i915_queue_hangcheck(dev_priv);
2941         }
2942 }
2943
2944 void i915_gem_close_object(struct drm_gem_object *gem, struct drm_file *file)
2945 {
2946         struct drm_i915_gem_object *obj = to_intel_bo(gem);
2947         struct drm_i915_file_private *fpriv = file->driver_priv;
2948         struct i915_vma *vma, *vn;
2949
2950         mutex_lock(&obj->base.dev->struct_mutex);
2951         list_for_each_entry_safe(vma, vn, &obj->vma_list, obj_link)
2952                 if (vma->vm->file == fpriv)
2953                         i915_vma_close(vma);
2954
2955         if (i915_gem_object_is_active(obj) &&
2956             !i915_gem_object_has_active_reference(obj)) {
2957                 i915_gem_object_set_active_reference(obj);
2958                 i915_gem_object_get(obj);
2959         }
2960         mutex_unlock(&obj->base.dev->struct_mutex);
2961 }
2962
2963 static unsigned long to_wait_timeout(s64 timeout_ns)
2964 {
2965         if (timeout_ns < 0)
2966                 return MAX_SCHEDULE_TIMEOUT;
2967
2968         if (timeout_ns == 0)
2969                 return 0;
2970
2971         return nsecs_to_jiffies_timeout(timeout_ns);
2972 }
2973
2974 /**
2975  * i915_gem_wait_ioctl - implements DRM_IOCTL_I915_GEM_WAIT
2976  * @dev: drm device pointer
2977  * @data: ioctl data blob
2978  * @file: drm file pointer
2979  *
2980  * Returns 0 if successful, else an error is returned with the remaining time in
2981  * the timeout parameter.
2982  *  -ETIME: object is still busy after timeout
2983  *  -ERESTARTSYS: signal interrupted the wait
2984  *  -ENONENT: object doesn't exist
2985  * Also possible, but rare:
2986  *  -EAGAIN: GPU wedged
2987  *  -ENOMEM: damn
2988  *  -ENODEV: Internal IRQ fail
2989  *  -E?: The add request failed
2990  *
2991  * The wait ioctl with a timeout of 0 reimplements the busy ioctl. With any
2992  * non-zero timeout parameter the wait ioctl will wait for the given number of
2993  * nanoseconds on an object becoming unbusy. Since the wait itself does so
2994  * without holding struct_mutex the object may become re-busied before this
2995  * function completes. A similar but shorter * race condition exists in the busy
2996  * ioctl
2997  */
2998 int
2999 i915_gem_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
3000 {
3001         struct drm_i915_gem_wait *args = data;
3002         struct drm_i915_gem_object *obj;
3003         ktime_t start;
3004         long ret;
3005
3006         if (args->flags != 0)
3007                 return -EINVAL;
3008
3009         obj = i915_gem_object_lookup(file, args->bo_handle);
3010         if (!obj)
3011                 return -ENOENT;
3012
3013         start = ktime_get();
3014
3015         ret = i915_gem_object_wait(obj,
3016                                    I915_WAIT_INTERRUPTIBLE | I915_WAIT_ALL,
3017                                    to_wait_timeout(args->timeout_ns),
3018                                    to_rps_client(file));
3019
3020         if (args->timeout_ns > 0) {
3021                 args->timeout_ns -= ktime_to_ns(ktime_sub(ktime_get(), start));
3022                 if (args->timeout_ns < 0)
3023                         args->timeout_ns = 0;
3024         }
3025
3026         i915_gem_object_put(obj);
3027         return ret;
3028 }
3029
3030 static int wait_for_timeline(struct i915_gem_timeline *tl, unsigned int flags)
3031 {
3032         int ret, i;
3033
3034         for (i = 0; i < ARRAY_SIZE(tl->engine); i++) {
3035                 ret = i915_gem_active_wait(&tl->engine[i].last_request, flags);
3036                 if (ret)
3037                         return ret;
3038         }
3039
3040         return 0;
3041 }
3042
3043 int i915_gem_wait_for_idle(struct drm_i915_private *i915, unsigned int flags)
3044 {
3045         int ret;
3046
3047         if (flags & I915_WAIT_LOCKED) {
3048                 struct i915_gem_timeline *tl;
3049
3050                 lockdep_assert_held(&i915->drm.struct_mutex);
3051
3052                 list_for_each_entry(tl, &i915->gt.timelines, link) {
3053                         ret = wait_for_timeline(tl, flags);
3054                         if (ret)
3055                                 return ret;
3056                 }
3057         } else {
3058                 ret = wait_for_timeline(&i915->gt.global_timeline, flags);
3059                 if (ret)
3060                         return ret;
3061         }
3062
3063         return 0;
3064 }
3065
3066 void i915_gem_clflush_object(struct drm_i915_gem_object *obj,
3067                              bool force)
3068 {
3069         /* If we don't have a page list set up, then we're not pinned
3070          * to GPU, and we can ignore the cache flush because it'll happen
3071          * again at bind time.
3072          */
3073         if (!obj->mm.pages)
3074                 return;
3075
3076         /*
3077          * Stolen memory is always coherent with the GPU as it is explicitly
3078          * marked as wc by the system, or the system is cache-coherent.
3079          */
3080         if (obj->stolen || obj->phys_handle)
3081                 return;
3082
3083         /* If the GPU is snooping the contents of the CPU cache,
3084          * we do not need to manually clear the CPU cache lines.  However,
3085          * the caches are only snooped when the render cache is
3086          * flushed/invalidated.  As we always have to emit invalidations
3087          * and flushes when moving into and out of the RENDER domain, correct
3088          * snooping behaviour occurs naturally as the result of our domain
3089          * tracking.
3090          */
3091         if (!force && cpu_cache_is_coherent(obj->base.dev, obj->cache_level)) {
3092                 obj->cache_dirty = true;
3093                 return;
3094         }
3095
3096         trace_i915_gem_object_clflush(obj);
3097         drm_clflush_sg(obj->mm.pages);
3098         obj->cache_dirty = false;
3099 }
3100
3101 /** Flushes the GTT write domain for the object if it's dirty. */
3102 static void
3103 i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj)
3104 {
3105         struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
3106
3107         if (obj->base.write_domain != I915_GEM_DOMAIN_GTT)
3108                 return;
3109
3110         /* No actual flushing is required for the GTT write domain.  Writes
3111          * to it "immediately" go to main memory as far as we know, so there's
3112          * no chipset flush.  It also doesn't land in render cache.
3113          *
3114          * However, we do have to enforce the order so that all writes through
3115          * the GTT land before any writes to the device, such as updates to
3116          * the GATT itself.
3117          *
3118          * We also have to wait a bit for the writes to land from the GTT.
3119          * An uncached read (i.e. mmio) seems to be ideal for the round-trip
3120          * timing. This issue has only been observed when switching quickly
3121          * between GTT writes and CPU reads from inside the kernel on recent hw,
3122          * and it appears to only affect discrete GTT blocks (i.e. on LLC
3123          * system agents we cannot reproduce this behaviour).
3124          */
3125         wmb();
3126         if (INTEL_GEN(dev_priv) >= 6 && !HAS_LLC(dev_priv))
3127                 POSTING_READ(RING_ACTHD(dev_priv->engine[RCS]->mmio_base));
3128
3129         intel_fb_obj_flush(obj, false, write_origin(obj, I915_GEM_DOMAIN_GTT));
3130
3131         obj->base.write_domain = 0;
3132         trace_i915_gem_object_change_domain(obj,
3133                                             obj->base.read_domains,
3134                                             I915_GEM_DOMAIN_GTT);
3135 }
3136
3137 /** Flushes the CPU write domain for the object if it's dirty. */
3138 static void
3139 i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj)
3140 {
3141         if (obj->base.write_domain != I915_GEM_DOMAIN_CPU)
3142                 return;
3143
3144         i915_gem_clflush_object(obj, obj->pin_display);
3145         intel_fb_obj_flush(obj, false, ORIGIN_CPU);
3146
3147         obj->base.write_domain = 0;
3148         trace_i915_gem_object_change_domain(obj,
3149                                             obj->base.read_domains,
3150                                             I915_GEM_DOMAIN_CPU);
3151 }
3152
3153 /**
3154  * Moves a single object to the GTT read, and possibly write domain.
3155  * @obj: object to act on
3156  * @write: ask for write access or read only
3157  *
3158  * This function returns when the move is complete, including waiting on
3159  * flushes to occur.
3160  */
3161 int
3162 i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
3163 {
3164         uint32_t old_write_domain, old_read_domains;
3165         int ret;
3166
3167         lockdep_assert_held(&obj->base.dev->struct_mutex);
3168
3169         ret = i915_gem_object_wait(obj,
3170                                    I915_WAIT_INTERRUPTIBLE |
3171                                    I915_WAIT_LOCKED |
3172                                    (write ? I915_WAIT_ALL : 0),
3173                                    MAX_SCHEDULE_TIMEOUT,
3174                                    NULL);
3175         if (ret)
3176                 return ret;
3177
3178         if (obj->base.write_domain == I915_GEM_DOMAIN_GTT)
3179                 return 0;
3180
3181         /* Flush and acquire obj->pages so that we are coherent through
3182          * direct access in memory with previous cached writes through
3183          * shmemfs and that our cache domain tracking remains valid.
3184          * For example, if the obj->filp was moved to swap without us
3185          * being notified and releasing the pages, we would mistakenly
3186          * continue to assume that the obj remained out of the CPU cached
3187          * domain.
3188          */
3189         ret = i915_gem_object_pin_pages(obj);
3190         if (ret)
3191                 return ret;
3192
3193         i915_gem_object_flush_cpu_write_domain(obj);
3194
3195         /* Serialise direct access to this object with the barriers for
3196          * coherent writes from the GPU, by effectively invalidating the
3197          * GTT domain upon first access.
3198          */
3199         if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
3200                 mb();
3201
3202         old_write_domain = obj->base.write_domain;
3203         old_read_domains = obj->base.read_domains;
3204
3205         /* It should now be out of any other write domains, and we can update
3206          * the domain values for our changes.
3207          */
3208         GEM_BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
3209         obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
3210         if (write) {
3211                 obj->base.read_domains = I915_GEM_DOMAIN_GTT;
3212                 obj->base.write_domain = I915_GEM_DOMAIN_GTT;
3213                 obj->mm.dirty = true;
3214         }
3215
3216         trace_i915_gem_object_change_domain(obj,
3217                                             old_read_domains,
3218                                             old_write_domain);
3219
3220         i915_gem_object_unpin_pages(obj);
3221         return 0;
3222 }
3223
3224 /**
3225  * Changes the cache-level of an object across all VMA.
3226  * @obj: object to act on
3227  * @cache_level: new cache level to set for the object
3228  *
3229  * After this function returns, the object will be in the new cache-level
3230  * across all GTT and the contents of the backing storage will be coherent,
3231  * with respect to the new cache-level. In order to keep the backing storage
3232  * coherent for all users, we only allow a single cache level to be set
3233  * globally on the object and prevent it from being changed whilst the
3234  * hardware is reading from the object. That is if the object is currently
3235  * on the scanout it will be set to uncached (or equivalent display
3236  * cache coherency) and all non-MOCS GPU access will also be uncached so
3237  * that all direct access to the scanout remains coherent.
3238  */
3239 int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
3240                                     enum i915_cache_level cache_level)
3241 {
3242         struct i915_vma *vma;
3243         int ret;
3244
3245         lockdep_assert_held(&obj->base.dev->struct_mutex);
3246
3247         if (obj->cache_level == cache_level)
3248                 return 0;
3249
3250         /* Inspect the list of currently bound VMA and unbind any that would
3251          * be invalid given the new cache-level. This is principally to
3252          * catch the issue of the CS prefetch crossing page boundaries and
3253          * reading an invalid PTE on older architectures.
3254          */
3255 restart:
3256         list_for_each_entry(vma, &obj->vma_list, obj_link) {
3257                 if (!drm_mm_node_allocated(&vma->node))
3258                         continue;
3259
3260                 if (i915_vma_is_pinned(vma)) {
3261                         DRM_DEBUG("can not change the cache level of pinned objects\n");
3262                         return -EBUSY;
3263                 }
3264
3265                 if (i915_gem_valid_gtt_space(vma, cache_level))
3266                         continue;
3267
3268                 ret = i915_vma_unbind(vma);
3269                 if (ret)
3270                         return ret;
3271
3272                 /* As unbinding may affect other elements in the
3273                  * obj->vma_list (due to side-effects from retiring
3274                  * an active vma), play safe and restart the iterator.
3275                  */
3276                 goto restart;
3277         }
3278
3279         /* We can reuse the existing drm_mm nodes but need to change the
3280          * cache-level on the PTE. We could simply unbind them all and
3281          * rebind with the correct cache-level on next use. However since
3282          * we already have a valid slot, dma mapping, pages etc, we may as
3283          * rewrite the PTE in the belief that doing so tramples upon less
3284          * state and so involves less work.
3285          */
3286         if (obj->bind_count) {
3287                 /* Before we change the PTE, the GPU must not be accessing it.
3288                  * If we wait upon the object, we know that all the bound
3289                  * VMA are no longer active.
3290                  */
3291                 ret = i915_gem_object_wait(obj,
3292                                            I915_WAIT_INTERRUPTIBLE |
3293                                            I915_WAIT_LOCKED |
3294                                            I915_WAIT_ALL,
3295                                            MAX_SCHEDULE_TIMEOUT,
3296                                            NULL);
3297                 if (ret)
3298                         return ret;
3299
3300                 if (!HAS_LLC(to_i915(obj->base.dev)) &&
3301                     cache_level != I915_CACHE_NONE) {
3302                         /* Access to snoopable pages through the GTT is
3303                          * incoherent and on some machines causes a hard
3304                          * lockup. Relinquish the CPU mmaping to force
3305                          * userspace to refault in the pages and we can
3306                          * then double check if the GTT mapping is still
3307                          * valid for that pointer access.
3308                          */
3309                         i915_gem_release_mmap(obj);
3310
3311                         /* As we no longer need a fence for GTT access,
3312                          * we can relinquish it now (and so prevent having
3313                          * to steal a fence from someone else on the next
3314                          * fence request). Note GPU activity would have
3315                          * dropped the fence as all snoopable access is
3316                          * supposed to be linear.
3317                          */
3318                         list_for_each_entry(vma, &obj->vma_list, obj_link) {
3319                                 ret = i915_vma_put_fence(vma);
3320                                 if (ret)
3321                                         return ret;
3322                         }
3323                 } else {
3324                         /* We either have incoherent backing store and
3325                          * so no GTT access or the architecture is fully
3326                          * coherent. In such cases, existing GTT mmaps
3327                          * ignore the cache bit in the PTE and we can
3328                          * rewrite it without confusing the GPU or having
3329                          * to force userspace to fault back in its mmaps.
3330                          */
3331                 }
3332
3333                 list_for_each_entry(vma, &obj->vma_list, obj_link) {
3334                         if (!drm_mm_node_allocated(&vma->node))
3335                                 continue;
3336
3337                         ret = i915_vma_bind(vma, cache_level, PIN_UPDATE);
3338                         if (ret)
3339                                 return ret;
3340                 }
3341         }
3342
3343         if (obj->base.write_domain == I915_GEM_DOMAIN_CPU &&
3344             cpu_cache_is_coherent(obj->base.dev, obj->cache_level))
3345                 obj->cache_dirty = true;
3346
3347         list_for_each_entry(vma, &obj->vma_list, obj_link)
3348                 vma->node.color = cache_level;
3349         obj->cache_level = cache_level;
3350
3351         return 0;
3352 }
3353
3354 int i915_gem_get_caching_ioctl(struct drm_device *dev, void *data,
3355                                struct drm_file *file)
3356 {
3357         struct drm_i915_gem_caching *args = data;
3358         struct drm_i915_gem_object *obj;
3359         int err = 0;
3360
3361         rcu_read_lock();
3362         obj = i915_gem_object_lookup_rcu(file, args->handle);
3363         if (!obj) {
3364                 err = -ENOENT;
3365                 goto out;
3366         }
3367
3368         switch (obj->cache_level) {
3369         case I915_CACHE_LLC:
3370         case I915_CACHE_L3_LLC:
3371                 args->caching = I915_CACHING_CACHED;
3372                 break;
3373
3374         case I915_CACHE_WT:
3375                 args->caching = I915_CACHING_DISPLAY;
3376                 break;
3377
3378         default:
3379                 args->caching = I915_CACHING_NONE;
3380                 break;
3381         }
3382 out:
3383         rcu_read_unlock();
3384         return err;
3385 }
3386
3387 int i915_gem_set_caching_ioctl(struct drm_device *dev, void *data,
3388                                struct drm_file *file)
3389 {
3390         struct drm_i915_private *i915 = to_i915(dev);
3391         struct drm_i915_gem_caching *args = data;
3392         struct drm_i915_gem_object *obj;
3393         enum i915_cache_level level;
3394         int ret;
3395
3396         switch (args->caching) {
3397         case I915_CACHING_NONE:
3398                 level = I915_CACHE_NONE;
3399                 break;
3400         case I915_CACHING_CACHED:
3401                 /*
3402                  * Due to a HW issue on BXT A stepping, GPU stores via a
3403                  * snooped mapping may leave stale data in a corresponding CPU
3404                  * cacheline, whereas normally such cachelines would get
3405                  * invalidated.
3406                  */
3407                 if (!HAS_LLC(i915) && !HAS_SNOOP(i915))
3408                         return -ENODEV;
3409
3410                 level = I915_CACHE_LLC;
3411                 break;
3412         case I915_CACHING_DISPLAY:
3413                 level = HAS_WT(i915) ? I915_CACHE_WT : I915_CACHE_NONE;
3414                 break;
3415         default:
3416                 return -EINVAL;
3417         }
3418
3419         ret = i915_mutex_lock_interruptible(dev);
3420         if (ret)
3421                 return ret;
3422
3423         obj = i915_gem_object_lookup(file, args->handle);
3424         if (!obj) {
3425                 ret = -ENOENT;
3426                 goto unlock;
3427         }
3428
3429         ret = i915_gem_object_set_cache_level(obj, level);
3430         i915_gem_object_put(obj);
3431 unlock:
3432         mutex_unlock(&dev->struct_mutex);
3433         return ret;
3434 }
3435
3436 /*
3437  * Prepare buffer for display plane (scanout, cursors, etc).
3438  * Can be called from an uninterruptible phase (modesetting) and allows
3439  * any flushes to be pipelined (for pageflips).
3440  */
3441 struct i915_vma *
3442 i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
3443                                      u32 alignment,
3444                                      const struct i915_ggtt_view *view)
3445 {
3446         struct i915_vma *vma;
3447         u32 old_read_domains, old_write_domain;
3448         int ret;
3449
3450         lockdep_assert_held(&obj->base.dev->struct_mutex);
3451
3452         /* Mark the pin_display early so that we account for the
3453          * display coherency whilst setting up the cache domains.
3454          */
3455         obj->pin_display++;
3456
3457         /* The display engine is not coherent with the LLC cache on gen6.  As
3458          * a result, we make sure that the pinning that is about to occur is
3459          * done with uncached PTEs. This is lowest common denominator for all
3460          * chipsets.
3461          *
3462          * However for gen6+, we could do better by using the GFDT bit instead
3463          * of uncaching, which would allow us to flush all the LLC-cached data
3464          * with that bit in the PTE to main memory with just one PIPE_CONTROL.
3465          */
3466         ret = i915_gem_object_set_cache_level(obj,
3467                                               HAS_WT(to_i915(obj->base.dev)) ?
3468                                               I915_CACHE_WT : I915_CACHE_NONE);
3469         if (ret) {
3470                 vma = ERR_PTR(ret);
3471                 goto err_unpin_display;
3472         }
3473
3474         /* As the user may map the buffer once pinned in the display plane
3475          * (e.g. libkms for the bootup splash), we have to ensure that we
3476          * always use map_and_fenceable for all scanout buffers. However,
3477          * it may simply be too big to fit into mappable, in which case
3478          * put it anyway and hope that userspace can cope (but always first
3479          * try to preserve the existing ABI).
3480          */
3481         vma = ERR_PTR(-ENOSPC);
3482         if (view->type == I915_GGTT_VIEW_NORMAL)
3483                 vma = i915_gem_object_ggtt_pin(obj, view, 0, alignment,
3484                                                PIN_MAPPABLE | PIN_NONBLOCK);
3485         if (IS_ERR(vma)) {
3486                 struct drm_i915_private *i915 = to_i915(obj->base.dev);
3487                 unsigned int flags;
3488
3489                 /* Valleyview is definitely limited to scanning out the first
3490                  * 512MiB. Lets presume this behaviour was inherited from the
3491                  * g4x display engine and that all earlier gen are similarly
3492                  * limited. Testing suggests that it is a little more
3493                  * complicated than this. For example, Cherryview appears quite
3494                  * happy to scanout from anywhere within its global aperture.
3495                  */
3496                 flags = 0;
3497                 if (HAS_GMCH_DISPLAY(i915))
3498                         flags = PIN_MAPPABLE;
3499                 vma = i915_gem_object_ggtt_pin(obj, view, 0, alignment, flags);
3500         }
3501         if (IS_ERR(vma))
3502                 goto err_unpin_display;
3503
3504         vma->display_alignment = max_t(u64, vma->display_alignment, alignment);
3505
3506         /* Treat this as an end-of-frame, like intel_user_framebuffer_dirty() */
3507         if (obj->cache_dirty) {
3508                 i915_gem_clflush_object(obj, true);
3509                 intel_fb_obj_flush(obj, false, ORIGIN_DIRTYFB);
3510         }
3511
3512         old_write_domain = obj->base.write_domain;
3513         old_read_domains = obj->base.read_domains;
3514
3515         /* It should now be out of any other write domains, and we can update
3516          * the domain values for our changes.
3517          */
3518         obj->base.write_domain = 0;
3519         obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
3520
3521         trace_i915_gem_object_change_domain(obj,
3522                                             old_read_domains,
3523                                             old_write_domain);
3524
3525         return vma;
3526
3527 err_unpin_display:
3528         obj->pin_display--;
3529         return vma;
3530 }
3531
3532 void
3533 i915_gem_object_unpin_from_display_plane(struct i915_vma *vma)
3534 {
3535         lockdep_assert_held(&vma->vm->dev->struct_mutex);
3536
3537         if (WARN_ON(vma->obj->pin_display == 0))
3538                 return;
3539
3540         if (--vma->obj->pin_display == 0)
3541                 vma->display_alignment = 0;
3542
3543         /* Bump the LRU to try and avoid premature eviction whilst flipping  */
3544         if (!i915_vma_is_active(vma))
3545                 list_move_tail(&vma->vm_link, &vma->vm->inactive_list);
3546
3547         i915_vma_unpin(vma);
3548 }
3549
3550 /**
3551  * Moves a single object to the CPU read, and possibly write domain.
3552  * @obj: object to act on
3553  * @write: requesting write or read-only access
3554  *
3555  * This function returns when the move is complete, including waiting on
3556  * flushes to occur.
3557  */
3558 int
3559 i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)
3560 {
3561         uint32_t old_write_domain, old_read_domains;
3562         int ret;
3563
3564         lockdep_assert_held(&obj->base.dev->struct_mutex);
3565
3566         ret = i915_gem_object_wait(obj,
3567                                    I915_WAIT_INTERRUPTIBLE |
3568                                    I915_WAIT_LOCKED |
3569                                    (write ? I915_WAIT_ALL : 0),
3570                                    MAX_SCHEDULE_TIMEOUT,
3571                                    NULL);
3572         if (ret)
3573                 return ret;
3574
3575         if (obj->base.write_domain == I915_GEM_DOMAIN_CPU)
3576                 return 0;
3577
3578         i915_gem_object_flush_gtt_write_domain(obj);
3579
3580         old_write_domain = obj->base.write_domain;
3581         old_read_domains = obj->base.read_domains;
3582
3583         /* Flush the CPU cache if it's still invalid. */
3584         if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) {
3585                 i915_gem_clflush_object(obj, false);
3586
3587                 obj->base.read_domains |= I915_GEM_DOMAIN_CPU;
3588         }
3589
3590         /* It should now be out of any other write domains, and we can update
3591          * the domain values for our changes.
3592          */
3593         GEM_BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
3594
3595         /* If we're writing through the CPU, then the GPU read domains will
3596          * need to be invalidated at next use.
3597          */
3598         if (write) {
3599                 obj->base.read_domains = I915_GEM_DOMAIN_CPU;
3600                 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
3601         }
3602
3603         trace_i915_gem_object_change_domain(obj,
3604                                             old_read_domains,
3605                                             old_write_domain);
3606
3607         return 0;
3608 }
3609
3610 /* Throttle our rendering by waiting until the ring has completed our requests
3611  * emitted over 20 msec ago.
3612  *
3613  * Note that if we were to use the current jiffies each time around the loop,
3614  * we wouldn't escape the function with any frames outstanding if the time to
3615  * render a frame was over 20ms.
3616  *
3617  * This should get us reasonable parallelism between CPU and GPU but also
3618  * relatively low latency when blocking on a particular request to finish.
3619  */
3620 static int
3621 i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file)
3622 {
3623         struct drm_i915_private *dev_priv = to_i915(dev);
3624         struct drm_i915_file_private *file_priv = file->driver_priv;
3625         unsigned long recent_enough = jiffies - DRM_I915_THROTTLE_JIFFIES;
3626         struct drm_i915_gem_request *request, *target = NULL;
3627         long ret;
3628
3629         /* ABI: return -EIO if already wedged */
3630         if (i915_terminally_wedged(&dev_priv->gpu_error))
3631                 return -EIO;
3632
3633         spin_lock(&file_priv->mm.lock);
3634         list_for_each_entry(request, &file_priv->mm.request_list, client_list) {
3635                 if (time_after_eq(request->emitted_jiffies, recent_enough))
3636                         break;
3637
3638                 /*
3639                  * Note that the request might not have been submitted yet.
3640                  * In which case emitted_jiffies will be zero.
3641                  */
3642                 if (!request->emitted_jiffies)
3643                         continue;
3644
3645                 target = request;
3646         }
3647         if (target)
3648                 i915_gem_request_get(target);
3649         spin_unlock(&file_priv->mm.lock);
3650
3651         if (target == NULL)
3652                 return 0;
3653
3654         ret = i915_wait_request(target,
3655                                 I915_WAIT_INTERRUPTIBLE,
3656                                 MAX_SCHEDULE_TIMEOUT);
3657         i915_gem_request_put(target);
3658
3659         return ret < 0 ? ret : 0;
3660 }
3661
3662 struct i915_vma *
3663 i915_gem_object_ggtt_pin(struct drm_i915_gem_object *obj,
3664                          const struct i915_ggtt_view *view,
3665                          u64 size,
3666                          u64 alignment,
3667                          u64 flags)
3668 {
3669         struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
3670         struct i915_address_space *vm = &dev_priv->ggtt.base;
3671         struct i915_vma *vma;
3672         int ret;
3673
3674         lockdep_assert_held(&obj->base.dev->struct_mutex);
3675
3676         vma = i915_gem_obj_lookup_or_create_vma(obj, vm, view);
3677         if (IS_ERR(vma))
3678                 return vma;
3679
3680         if (i915_vma_misplaced(vma, size, alignment, flags)) {
3681                 if (flags & PIN_NONBLOCK &&
3682                     (i915_vma_is_pinned(vma) || i915_vma_is_active(vma)))
3683                         return ERR_PTR(-ENOSPC);
3684
3685                 if (flags & PIN_MAPPABLE) {
3686                         u32 fence_size;
3687
3688                         fence_size = i915_gem_get_ggtt_size(dev_priv, vma->size,
3689                                                             i915_gem_object_get_tiling(obj));
3690                         /* If the required space is larger than the available
3691                          * aperture, we will not able to find a slot for the
3692                          * object and unbinding the object now will be in
3693                          * vain. Worse, doing so may cause us to ping-pong
3694                          * the object in and out of the Global GTT and
3695                          * waste a lot of cycles under the mutex.
3696                          */
3697                         if (fence_size > dev_priv->ggtt.mappable_end)
3698                                 return ERR_PTR(-E2BIG);
3699
3700                         /* If NONBLOCK is set the caller is optimistically
3701                          * trying to cache the full object within the mappable
3702                          * aperture, and *must* have a fallback in place for
3703                          * situations where we cannot bind the object. We
3704                          * can be a little more lax here and use the fallback
3705                          * more often to avoid costly migrations of ourselves
3706                          * and other objects within the aperture.
3707                          *
3708                          * Half-the-aperture is used as a simple heuristic.
3709                          * More interesting would to do search for a free
3710                          * block prior to making the commitment to unbind.
3711                          * That caters for the self-harm case, and with a
3712                          * little more heuristics (e.g. NOFAULT, NOEVICT)
3713                          * we could try to minimise harm to others.
3714                          */
3715                         if (flags & PIN_NONBLOCK &&
3716                             fence_size > dev_priv->ggtt.mappable_end / 2)
3717                                 return ERR_PTR(-ENOSPC);
3718                 }
3719
3720                 WARN(i915_vma_is_pinned(vma),
3721                      "bo is already pinned in ggtt with incorrect alignment:"
3722                      " offset=%08x, req.alignment=%llx,"
3723                      " req.map_and_fenceable=%d, vma->map_and_fenceable=%d\n",
3724                      i915_ggtt_offset(vma), alignment,
3725                      !!(flags & PIN_MAPPABLE),
3726                      i915_vma_is_map_and_fenceable(vma));
3727                 ret = i915_vma_unbind(vma);
3728                 if (ret)
3729                         return ERR_PTR(ret);
3730         }
3731
3732         ret = i915_vma_pin(vma, size, alignment, flags | PIN_GLOBAL);
3733         if (ret)
3734                 return ERR_PTR(ret);
3735
3736         return vma;
3737 }
3738
3739 static __always_inline unsigned int __busy_read_flag(unsigned int id)
3740 {
3741         /* Note that we could alias engines in the execbuf API, but
3742          * that would be very unwise as it prevents userspace from
3743          * fine control over engine selection. Ahem.
3744          *
3745          * This should be something like EXEC_MAX_ENGINE instead of
3746          * I915_NUM_ENGINES.
3747          */
3748         BUILD_BUG_ON(I915_NUM_ENGINES > 16);
3749         return 0x10000 << id;
3750 }
3751
3752 static __always_inline unsigned int __busy_write_id(unsigned int id)
3753 {
3754         /* The uABI guarantees an active writer is also amongst the read
3755          * engines. This would be true if we accessed the activity tracking
3756          * under the lock, but as we perform the lookup of the object and
3757          * its activity locklessly we can not guarantee that the last_write
3758          * being active implies that we have set the same engine flag from
3759          * last_read - hence we always set both read and write busy for
3760          * last_write.
3761          */
3762         return id | __busy_read_flag(id);
3763 }
3764
3765 static __always_inline unsigned int
3766 __busy_set_if_active(const struct dma_fence *fence,
3767                      unsigned int (*flag)(unsigned int id))
3768 {
3769         struct drm_i915_gem_request *rq;
3770
3771         /* We have to check the current hw status of the fence as the uABI
3772          * guarantees forward progress. We could rely on the idle worker
3773          * to eventually flush us, but to minimise latency just ask the
3774          * hardware.
3775          *
3776          * Note we only report on the status of native fences.
3777          */
3778         if (!dma_fence_is_i915(fence))
3779                 return 0;
3780
3781         /* opencode to_request() in order to avoid const warnings */
3782         rq = container_of(fence, struct drm_i915_gem_request, fence);
3783         if (i915_gem_request_completed(rq))
3784                 return 0;
3785
3786         return flag(rq->engine->exec_id);
3787 }
3788
3789 static __always_inline unsigned int
3790 busy_check_reader(const struct dma_fence *fence)
3791 {
3792         return __busy_set_if_active(fence, __busy_read_flag);
3793 }
3794
3795 static __always_inline unsigned int
3796 busy_check_writer(const struct dma_fence *fence)
3797 {
3798         if (!fence)
3799                 return 0;
3800
3801         return __busy_set_if_active(fence, __busy_write_id);
3802 }
3803
3804 int
3805 i915_gem_busy_ioctl(struct drm_device *dev, void *data,
3806                     struct drm_file *file)
3807 {
3808         struct drm_i915_gem_busy *args = data;
3809         struct drm_i915_gem_object *obj;
3810         struct reservation_object_list *list;
3811         unsigned int seq;
3812         int err;
3813
3814         err = -ENOENT;
3815         rcu_read_lock();
3816         obj = i915_gem_object_lookup_rcu(file, args->handle);
3817         if (!obj)
3818                 goto out;
3819
3820         /* A discrepancy here is that we do not report the status of
3821          * non-i915 fences, i.e. even though we may report the object as idle,
3822          * a call to set-domain may still stall waiting for foreign rendering.
3823          * This also means that wait-ioctl may report an object as busy,
3824          * where busy-ioctl considers it idle.
3825          *
3826          * We trade the ability to warn of foreign fences to report on which
3827          * i915 engines are active for the object.
3828          *
3829          * Alternatively, we can trade that extra information on read/write
3830          * activity with
3831          *      args->busy =
3832          *              !reservation_object_test_signaled_rcu(obj->resv, true);
3833          * to report the overall busyness. This is what the wait-ioctl does.
3834          *
3835          */
3836 retry:
3837         seq = raw_read_seqcount(&obj->resv->seq);
3838
3839         /* Translate the exclusive fence to the READ *and* WRITE engine */
3840         args->busy = busy_check_writer(rcu_dereference(obj->resv->fence_excl));
3841
3842         /* Translate shared fences to READ set of engines */
3843         list = rcu_dereference(obj->resv->fence);
3844         if (list) {
3845                 unsigned int shared_count = list->shared_count, i;
3846
3847                 for (i = 0; i < shared_count; ++i) {
3848                         struct dma_fence *fence =
3849                                 rcu_dereference(list->shared[i]);
3850
3851                         args->busy |= busy_check_reader(fence);
3852                 }
3853         }
3854
3855         if (args->busy && read_seqcount_retry(&obj->resv->seq, seq))
3856                 goto retry;
3857
3858         err = 0;
3859 out:
3860         rcu_read_unlock();
3861         return err;
3862 }
3863
3864 int
3865 i915_gem_throttle_ioctl(struct drm_device *dev, void *data,
3866                         struct drm_file *file_priv)
3867 {
3868         return i915_gem_ring_throttle(dev, file_priv);
3869 }
3870
3871 int
3872 i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
3873                        struct drm_file *file_priv)
3874 {
3875         struct drm_i915_private *dev_priv = to_i915(dev);
3876         struct drm_i915_gem_madvise *args = data;
3877         struct drm_i915_gem_object *obj;
3878         int err;
3879
3880         switch (args->madv) {
3881         case I915_MADV_DONTNEED:
3882         case I915_MADV_WILLNEED:
3883             break;
3884         default:
3885             return -EINVAL;
3886         }
3887
3888         obj = i915_gem_object_lookup(file_priv, args->handle);
3889         if (!obj)
3890                 return -ENOENT;
3891
3892         err = mutex_lock_interruptible(&obj->mm.lock);
3893         if (err)
3894                 goto out;
3895
3896         if (obj->mm.pages &&
3897             i915_gem_object_is_tiled(obj) &&
3898             dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES) {
3899                 if (obj->mm.madv == I915_MADV_WILLNEED) {
3900                         GEM_BUG_ON(!obj->mm.quirked);
3901                         __i915_gem_object_unpin_pages(obj);
3902                         obj->mm.quirked = false;
3903                 }
3904                 if (args->madv == I915_MADV_WILLNEED) {
3905                         GEM_BUG_ON(obj->mm.quirked);
3906                         __i915_gem_object_pin_pages(obj);
3907                         obj->mm.quirked = true;
3908                 }
3909         }
3910
3911         if (obj->mm.madv != __I915_MADV_PURGED)
3912                 obj->mm.madv = args->madv;
3913
3914         /* if the object is no longer attached, discard its backing storage */
3915         if (obj->mm.madv == I915_MADV_DONTNEED && !obj->mm.pages)
3916                 i915_gem_object_truncate(obj);
3917
3918         args->retained = obj->mm.madv != __I915_MADV_PURGED;
3919         mutex_unlock(&obj->mm.lock);
3920
3921 out:
3922         i915_gem_object_put(obj);
3923         return err;
3924 }
3925
3926 static void
3927 frontbuffer_retire(struct i915_gem_active *active,
3928                    struct drm_i915_gem_request *request)
3929 {
3930         struct drm_i915_gem_object *obj =
3931                 container_of(active, typeof(*obj), frontbuffer_write);
3932
3933         intel_fb_obj_flush(obj, true, ORIGIN_CS);
3934 }
3935
3936 void i915_gem_object_init(struct drm_i915_gem_object *obj,
3937                           const struct drm_i915_gem_object_ops *ops)
3938 {
3939         mutex_init(&obj->mm.lock);
3940
3941         INIT_LIST_HEAD(&obj->global_link);
3942         INIT_LIST_HEAD(&obj->userfault_link);
3943         INIT_LIST_HEAD(&obj->obj_exec_link);
3944         INIT_LIST_HEAD(&obj->vma_list);
3945         INIT_LIST_HEAD(&obj->batch_pool_link);
3946
3947         obj->ops = ops;
3948
3949         reservation_object_init(&obj->__builtin_resv);
3950         obj->resv = &obj->__builtin_resv;
3951
3952         obj->frontbuffer_ggtt_origin = ORIGIN_GTT;
3953         init_request_active(&obj->frontbuffer_write, frontbuffer_retire);
3954
3955         obj->mm.madv = I915_MADV_WILLNEED;
3956         INIT_RADIX_TREE(&obj->mm.get_page.radix, GFP_KERNEL | __GFP_NOWARN);
3957         mutex_init(&obj->mm.get_page.lock);
3958
3959         i915_gem_info_add_obj(to_i915(obj->base.dev), obj->base.size);
3960 }
3961
3962 static const struct drm_i915_gem_object_ops i915_gem_object_ops = {
3963         .flags = I915_GEM_OBJECT_HAS_STRUCT_PAGE |
3964                  I915_GEM_OBJECT_IS_SHRINKABLE,
3965         .get_pages = i915_gem_object_get_pages_gtt,
3966         .put_pages = i915_gem_object_put_pages_gtt,
3967 };
3968
3969 /* Note we don't consider signbits :| */
3970 #define overflows_type(x, T) \
3971         (sizeof(x) > sizeof(T) && (x) >> (sizeof(T) * BITS_PER_BYTE))
3972
3973 struct drm_i915_gem_object *
3974 i915_gem_object_create(struct drm_device *dev, u64 size)
3975 {
3976         struct drm_i915_private *dev_priv = to_i915(dev);
3977         struct drm_i915_gem_object *obj;
3978         struct address_space *mapping;
3979         gfp_t mask;
3980         int ret;
3981
3982         /* There is a prevalence of the assumption that we fit the object's
3983          * page count inside a 32bit _signed_ variable. Let's document this and
3984          * catch if we ever need to fix it. In the meantime, if you do spot
3985          * such a local variable, please consider fixing!
3986          */
3987         if (WARN_ON(size >> PAGE_SHIFT > INT_MAX))
3988                 return ERR_PTR(-E2BIG);
3989
3990         if (overflows_type(size, obj->base.size))
3991                 return ERR_PTR(-E2BIG);
3992
3993         obj = i915_gem_object_alloc(dev);
3994         if (obj == NULL)
3995                 return ERR_PTR(-ENOMEM);
3996
3997         ret = drm_gem_object_init(dev, &obj->base, size);
3998         if (ret)
3999                 goto fail;
4000
4001         mask = GFP_HIGHUSER | __GFP_RECLAIMABLE;
4002         if (IS_CRESTLINE(dev_priv) || IS_BROADWATER(dev_priv)) {
4003                 /* 965gm cannot relocate objects above 4GiB. */
4004                 mask &= ~__GFP_HIGHMEM;
4005                 mask |= __GFP_DMA32;
4006         }
4007
4008         mapping = obj->base.filp->f_mapping;
4009         mapping_set_gfp_mask(mapping, mask);
4010
4011         i915_gem_object_init(obj, &i915_gem_object_ops);
4012
4013         obj->base.write_domain = I915_GEM_DOMAIN_CPU;
4014         obj->base.read_domains = I915_GEM_DOMAIN_CPU;
4015
4016         if (HAS_LLC(dev_priv)) {
4017                 /* On some devices, we can have the GPU use the LLC (the CPU
4018                  * cache) for about a 10% performance improvement
4019                  * compared to uncached.  Graphics requests other than
4020                  * display scanout are coherent with the CPU in
4021                  * accessing this cache.  This means in this mode we
4022                  * don't need to clflush on the CPU side, and on the
4023                  * GPU side we only need to flush internal caches to
4024                  * get data visible to the CPU.
4025                  *
4026                  * However, we maintain the display planes as UC, and so
4027                  * need to rebind when first used as such.
4028                  */
4029                 obj->cache_level = I915_CACHE_LLC;
4030         } else
4031                 obj->cache_level = I915_CACHE_NONE;
4032
4033         trace_i915_gem_object_create(obj);
4034
4035         return obj;
4036
4037 fail:
4038         i915_gem_object_free(obj);
4039         return ERR_PTR(ret);
4040 }
4041
4042 static bool discard_backing_storage(struct drm_i915_gem_object *obj)
4043 {
4044         /* If we are the last user of the backing storage (be it shmemfs
4045          * pages or stolen etc), we know that the pages are going to be
4046          * immediately released. In this case, we can then skip copying
4047          * back the contents from the GPU.
4048          */
4049
4050         if (obj->mm.madv != I915_MADV_WILLNEED)
4051                 return false;
4052
4053         if (obj->base.filp == NULL)
4054                 return true;
4055
4056         /* At first glance, this looks racy, but then again so would be
4057          * userspace racing mmap against close. However, the first external
4058          * reference to the filp can only be obtained through the
4059          * i915_gem_mmap_ioctl() which safeguards us against the user
4060          * acquiring such a reference whilst we are in the middle of
4061          * freeing the object.
4062          */
4063         return atomic_long_read(&obj->base.filp->f_count) == 1;
4064 }
4065
4066 static void __i915_gem_free_objects(struct drm_i915_private *i915,
4067                                     struct llist_node *freed)
4068 {
4069         struct drm_i915_gem_object *obj, *on;
4070
4071         mutex_lock(&i915->drm.struct_mutex);
4072         intel_runtime_pm_get(i915);
4073         llist_for_each_entry(obj, freed, freed) {
4074                 struct i915_vma *vma, *vn;
4075
4076                 trace_i915_gem_object_destroy(obj);
4077
4078                 GEM_BUG_ON(i915_gem_object_is_active(obj));
4079                 list_for_each_entry_safe(vma, vn,
4080                                          &obj->vma_list, obj_link) {
4081                         GEM_BUG_ON(!i915_vma_is_ggtt(vma));
4082                         GEM_BUG_ON(i915_vma_is_active(vma));
4083                         vma->flags &= ~I915_VMA_PIN_MASK;
4084                         i915_vma_close(vma);
4085                 }
4086                 GEM_BUG_ON(!list_empty(&obj->vma_list));
4087                 GEM_BUG_ON(!RB_EMPTY_ROOT(&obj->vma_tree));
4088
4089                 list_del(&obj->global_link);
4090         }
4091         intel_runtime_pm_put(i915);
4092         mutex_unlock(&i915->drm.struct_mutex);
4093
4094         llist_for_each_entry_safe(obj, on, freed, freed) {
4095                 GEM_BUG_ON(obj->bind_count);
4096                 GEM_BUG_ON(atomic_read(&obj->frontbuffer_bits));
4097
4098                 if (obj->ops->release)
4099                         obj->ops->release(obj);
4100
4101                 if (WARN_ON(i915_gem_object_has_pinned_pages(obj)))
4102                         atomic_set(&obj->mm.pages_pin_count, 0);
4103                 __i915_gem_object_put_pages(obj, I915_MM_NORMAL);
4104                 GEM_BUG_ON(obj->mm.pages);
4105
4106                 if (obj->base.import_attach)
4107                         drm_prime_gem_destroy(&obj->base, NULL);
4108
4109                 reservation_object_fini(&obj->__builtin_resv);
4110                 drm_gem_object_release(&obj->base);
4111                 i915_gem_info_remove_obj(i915, obj->base.size);
4112
4113                 kfree(obj->bit_17);
4114                 i915_gem_object_free(obj);
4115         }
4116 }
4117
4118 static void i915_gem_flush_free_objects(struct drm_i915_private *i915)
4119 {
4120         struct llist_node *freed;
4121
4122         freed = llist_del_all(&i915->mm.free_list);
4123         if (unlikely(freed))
4124                 __i915_gem_free_objects(i915, freed);
4125 }
4126
4127 static void __i915_gem_free_work(struct work_struct *work)
4128 {
4129         struct drm_i915_private *i915 =
4130                 container_of(work, struct drm_i915_private, mm.free_work);
4131         struct llist_node *freed;
4132
4133         /* All file-owned VMA should have been released by this point through
4134          * i915_gem_close_object(), or earlier by i915_gem_context_close().
4135          * However, the object may also be bound into the global GTT (e.g.
4136          * older GPUs without per-process support, or for direct access through
4137          * the GTT either for the user or for scanout). Those VMA still need to
4138          * unbound now.
4139          */
4140
4141         while ((freed = llist_del_all(&i915->mm.free_list)))
4142                 __i915_gem_free_objects(i915, freed);
4143 }
4144
4145 static void __i915_gem_free_object_rcu(struct rcu_head *head)
4146 {
4147         struct drm_i915_gem_object *obj =
4148                 container_of(head, typeof(*obj), rcu);
4149         struct drm_i915_private *i915 = to_i915(obj->base.dev);
4150
4151         /* We can't simply use call_rcu() from i915_gem_free_object()
4152          * as we need to block whilst unbinding, and the call_rcu
4153          * task may be called from softirq context. So we take a
4154          * detour through a worker.
4155          */
4156         if (llist_add(&obj->freed, &i915->mm.free_list))
4157                 schedule_work(&i915->mm.free_work);
4158 }
4159
4160 void i915_gem_free_object(struct drm_gem_object *gem_obj)
4161 {
4162         struct drm_i915_gem_object *obj = to_intel_bo(gem_obj);
4163
4164         if (obj->mm.quirked)
4165                 __i915_gem_object_unpin_pages(obj);
4166
4167         if (discard_backing_storage(obj))
4168                 obj->mm.madv = I915_MADV_DONTNEED;
4169
4170         /* Before we free the object, make sure any pure RCU-only
4171          * read-side critical sections are complete, e.g.
4172          * i915_gem_busy_ioctl(). For the corresponding synchronized
4173          * lookup see i915_gem_object_lookup_rcu().
4174          */
4175         call_rcu(&obj->rcu, __i915_gem_free_object_rcu);
4176 }
4177
4178 void __i915_gem_object_release_unless_active(struct drm_i915_gem_object *obj)
4179 {
4180         lockdep_assert_held(&obj->base.dev->struct_mutex);
4181
4182         GEM_BUG_ON(i915_gem_object_has_active_reference(obj));
4183         if (i915_gem_object_is_active(obj))
4184                 i915_gem_object_set_active_reference(obj);
4185         else
4186                 i915_gem_object_put(obj);
4187 }
4188
4189 static void assert_kernel_context_is_current(struct drm_i915_private *dev_priv)
4190 {
4191         struct intel_engine_cs *engine;
4192         enum intel_engine_id id;
4193
4194         for_each_engine(engine, dev_priv, id)
4195                 GEM_BUG_ON(engine->last_context != dev_priv->kernel_context);
4196 }
4197
4198 int i915_gem_suspend(struct drm_device *dev)
4199 {
4200         struct drm_i915_private *dev_priv = to_i915(dev);
4201         int ret;
4202
4203         intel_suspend_gt_powersave(dev_priv);
4204
4205         mutex_lock(&dev->struct_mutex);
4206
4207         /* We have to flush all the executing contexts to main memory so
4208          * that they can saved in the hibernation image. To ensure the last
4209          * context image is coherent, we have to switch away from it. That
4210          * leaves the dev_priv->kernel_context still active when
4211          * we actually suspend, and its image in memory may not match the GPU
4212          * state. Fortunately, the kernel_context is disposable and we do
4213          * not rely on its state.
4214          */
4215         ret = i915_gem_switch_to_kernel_context(dev_priv);
4216         if (ret)
4217                 goto err;
4218
4219         ret = i915_gem_wait_for_idle(dev_priv,
4220                                      I915_WAIT_INTERRUPTIBLE |
4221                                      I915_WAIT_LOCKED);
4222         if (ret)
4223                 goto err;
4224
4225         i915_gem_retire_requests(dev_priv);
4226         GEM_BUG_ON(dev_priv->gt.active_requests);
4227
4228         assert_kernel_context_is_current(dev_priv);
4229         i915_gem_context_lost(dev_priv);
4230         mutex_unlock(&dev->struct_mutex);
4231
4232         cancel_delayed_work_sync(&dev_priv->gpu_error.hangcheck_work);
4233         cancel_delayed_work_sync(&dev_priv->gt.retire_work);
4234         flush_delayed_work(&dev_priv->gt.idle_work);
4235         flush_work(&dev_priv->mm.free_work);
4236
4237         /* Assert that we sucessfully flushed all the work and
4238          * reset the GPU back to its idle, low power state.
4239          */
4240         WARN_ON(dev_priv->gt.awake);
4241         WARN_ON(!intel_execlists_idle(dev_priv));
4242
4243         /*
4244          * Neither the BIOS, ourselves or any other kernel
4245          * expects the system to be in execlists mode on startup,
4246          * so we need to reset the GPU back to legacy mode. And the only
4247          * known way to disable logical contexts is through a GPU reset.
4248          *
4249          * So in order to leave the system in a known default configuration,
4250          * always reset the GPU upon unload and suspend. Afterwards we then
4251          * clean up the GEM state tracking, flushing off the requests and
4252          * leaving the system in a known idle state.
4253          *
4254          * Note that is of the upmost importance that the GPU is idle and
4255          * all stray writes are flushed *before* we dismantle the backing
4256          * storage for the pinned objects.
4257          *
4258          * However, since we are uncertain that resetting the GPU on older
4259          * machines is a good idea, we don't - just in case it leaves the
4260          * machine in an unusable condition.
4261          */
4262         if (HAS_HW_CONTEXTS(dev_priv)) {
4263                 int reset = intel_gpu_reset(dev_priv, ALL_ENGINES);
4264                 WARN_ON(reset && reset != -ENODEV);
4265         }
4266
4267         return 0;
4268
4269 err:
4270         mutex_unlock(&dev->struct_mutex);
4271         return ret;
4272 }
4273
4274 void i915_gem_resume(struct drm_device *dev)
4275 {
4276         struct drm_i915_private *dev_priv = to_i915(dev);
4277
4278         WARN_ON(dev_priv->gt.awake);
4279
4280         mutex_lock(&dev->struct_mutex);
4281         i915_gem_restore_gtt_mappings(dev_priv);
4282
4283         /* As we didn't flush the kernel context before suspend, we cannot
4284          * guarantee that the context image is complete. So let's just reset
4285          * it and start again.
4286          */
4287         dev_priv->gt.resume(dev_priv);
4288
4289         mutex_unlock(&dev->struct_mutex);
4290 }
4291
4292 void i915_gem_init_swizzling(struct drm_i915_private *dev_priv)
4293 {
4294         if (INTEL_GEN(dev_priv) < 5 ||
4295             dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_NONE)
4296                 return;
4297
4298         I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) |
4299                                  DISP_TILE_SURFACE_SWIZZLING);
4300
4301         if (IS_GEN5(dev_priv))
4302                 return;
4303
4304         I915_WRITE(TILECTL, I915_READ(TILECTL) | TILECTL_SWZCTL);
4305         if (IS_GEN6(dev_priv))
4306                 I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_SNB));
4307         else if (IS_GEN7(dev_priv))
4308                 I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_IVB));
4309         else if (IS_GEN8(dev_priv))
4310                 I915_WRITE(GAMTARBMODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_BDW));
4311         else
4312                 BUG();
4313 }
4314
4315 static void init_unused_ring(struct drm_i915_private *dev_priv, u32 base)
4316 {
4317         I915_WRITE(RING_CTL(base), 0);
4318         I915_WRITE(RING_HEAD(base), 0);
4319         I915_WRITE(RING_TAIL(base), 0);
4320         I915_WRITE(RING_START(base), 0);
4321 }
4322
4323 static void init_unused_rings(struct drm_i915_private *dev_priv)
4324 {
4325         if (IS_I830(dev_priv)) {
4326                 init_unused_ring(dev_priv, PRB1_BASE);
4327                 init_unused_ring(dev_priv, SRB0_BASE);
4328                 init_unused_ring(dev_priv, SRB1_BASE);
4329                 init_unused_ring(dev_priv, SRB2_BASE);
4330                 init_unused_ring(dev_priv, SRB3_BASE);
4331         } else if (IS_GEN2(dev_priv)) {
4332                 init_unused_ring(dev_priv, SRB0_BASE);
4333                 init_unused_ring(dev_priv, SRB1_BASE);
4334         } else if (IS_GEN3(dev_priv)) {
4335                 init_unused_ring(dev_priv, PRB1_BASE);
4336                 init_unused_ring(dev_priv, PRB2_BASE);
4337         }
4338 }
4339
4340 int
4341 i915_gem_init_hw(struct drm_device *dev)
4342 {
4343         struct drm_i915_private *dev_priv = to_i915(dev);
4344         struct intel_engine_cs *engine;
4345         enum intel_engine_id id;
4346         int ret;
4347
4348         dev_priv->gt.last_init_time = ktime_get();
4349
4350         /* Double layer security blanket, see i915_gem_init() */
4351         intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
4352
4353         if (HAS_EDRAM(dev_priv) && INTEL_GEN(dev_priv) < 9)
4354                 I915_WRITE(HSW_IDICR, I915_READ(HSW_IDICR) | IDIHASHMSK(0xf));
4355
4356         if (IS_HASWELL(dev_priv))
4357                 I915_WRITE(MI_PREDICATE_RESULT_2, IS_HSW_GT3(dev_priv) ?
4358                            LOWER_SLICE_ENABLED : LOWER_SLICE_DISABLED);
4359
4360         if (HAS_PCH_NOP(dev_priv)) {
4361                 if (IS_IVYBRIDGE(dev_priv)) {
4362                         u32 temp = I915_READ(GEN7_MSG_CTL);
4363                         temp &= ~(WAIT_FOR_PCH_FLR_ACK | WAIT_FOR_PCH_RESET_ACK);
4364                         I915_WRITE(GEN7_MSG_CTL, temp);
4365                 } else if (INTEL_GEN(dev_priv) >= 7) {
4366                         u32 temp = I915_READ(HSW_NDE_RSTWRN_OPT);
4367                         temp &= ~RESET_PCH_HANDSHAKE_ENABLE;
4368                         I915_WRITE(HSW_NDE_RSTWRN_OPT, temp);
4369                 }
4370         }
4371
4372         i915_gem_init_swizzling(dev_priv);
4373
4374         /*
4375          * At least 830 can leave some of the unused rings
4376          * "active" (ie. head != tail) after resume which
4377          * will prevent c3 entry. Makes sure all unused rings
4378          * are totally idle.
4379          */
4380         init_unused_rings(dev_priv);
4381
4382         BUG_ON(!dev_priv->kernel_context);
4383
4384         ret = i915_ppgtt_init_hw(dev_priv);
4385         if (ret) {
4386                 DRM_ERROR("PPGTT enable HW failed %d\n", ret);
4387                 goto out;
4388         }
4389
4390         /* Need to do basic initialisation of all rings first: */
4391         for_each_engine(engine, dev_priv, id) {
4392                 ret = engine->init_hw(engine);
4393                 if (ret)
4394                         goto out;
4395         }
4396
4397         intel_mocs_init_l3cc_table(dev);
4398
4399         /* We can't enable contexts until all firmware is loaded */
4400         ret = intel_guc_setup(dev);
4401         if (ret)
4402                 goto out;
4403
4404 out:
4405         intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
4406         return ret;
4407 }
4408
4409 bool intel_sanitize_semaphores(struct drm_i915_private *dev_priv, int value)
4410 {
4411         if (INTEL_INFO(dev_priv)->gen < 6)
4412                 return false;
4413
4414         /* TODO: make semaphores and Execlists play nicely together */
4415         if (i915.enable_execlists)
4416                 return false;
4417
4418         if (value >= 0)
4419                 return value;
4420
4421 #ifdef CONFIG_INTEL_IOMMU
4422         /* Enable semaphores on SNB when IO remapping is off */
4423         if (INTEL_INFO(dev_priv)->gen == 6 && intel_iommu_gfx_mapped)
4424                 return false;
4425 #endif
4426
4427         return true;
4428 }
4429
4430 int i915_gem_init(struct drm_device *dev)
4431 {
4432         struct drm_i915_private *dev_priv = to_i915(dev);
4433         int ret;
4434
4435         mutex_lock(&dev->struct_mutex);
4436
4437         if (!i915.enable_execlists) {
4438                 dev_priv->gt.resume = intel_legacy_submission_resume;
4439                 dev_priv->gt.cleanup_engine = intel_engine_cleanup;
4440         } else {
4441                 dev_priv->gt.resume = intel_lr_context_resume;
4442                 dev_priv->gt.cleanup_engine = intel_logical_ring_cleanup;
4443         }
4444
4445         /* This is just a security blanket to placate dragons.
4446          * On some systems, we very sporadically observe that the first TLBs
4447          * used by the CS may be stale, despite us poking the TLB reset. If
4448          * we hold the forcewake during initialisation these problems
4449          * just magically go away.
4450          */
4451         intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
4452
4453         i915_gem_init_userptr(dev_priv);
4454
4455         ret = i915_gem_init_ggtt(dev_priv);
4456         if (ret)
4457                 goto out_unlock;
4458
4459         ret = i915_gem_context_init(dev);
4460         if (ret)
4461                 goto out_unlock;
4462
4463         ret = intel_engines_init(dev);
4464         if (ret)
4465                 goto out_unlock;
4466
4467         ret = i915_gem_init_hw(dev);
4468         if (ret == -EIO) {
4469                 /* Allow engine initialisation to fail by marking the GPU as
4470                  * wedged. But we only want to do this where the GPU is angry,
4471                  * for all other failure, such as an allocation failure, bail.
4472                  */
4473                 DRM_ERROR("Failed to initialize GPU, declaring it wedged\n");
4474                 i915_gem_set_wedged(dev_priv);
4475                 ret = 0;
4476         }
4477
4478 out_unlock:
4479         intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
4480         mutex_unlock(&dev->struct_mutex);
4481
4482         return ret;
4483 }
4484
4485 void
4486 i915_gem_cleanup_engines(struct drm_device *dev)
4487 {
4488         struct drm_i915_private *dev_priv = to_i915(dev);
4489         struct intel_engine_cs *engine;
4490         enum intel_engine_id id;
4491
4492         for_each_engine(engine, dev_priv, id)
4493                 dev_priv->gt.cleanup_engine(engine);
4494 }
4495
4496 void
4497 i915_gem_load_init_fences(struct drm_i915_private *dev_priv)
4498 {
4499         int i;
4500
4501         if (INTEL_INFO(dev_priv)->gen >= 7 && !IS_VALLEYVIEW(dev_priv) &&
4502             !IS_CHERRYVIEW(dev_priv))
4503                 dev_priv->num_fence_regs = 32;
4504         else if (INTEL_INFO(dev_priv)->gen >= 4 || IS_I945G(dev_priv) ||
4505                  IS_I945GM(dev_priv) || IS_G33(dev_priv))
4506                 dev_priv->num_fence_regs = 16;
4507         else
4508                 dev_priv->num_fence_regs = 8;
4509
4510         if (intel_vgpu_active(dev_priv))
4511                 dev_priv->num_fence_regs =
4512                                 I915_READ(vgtif_reg(avail_rs.fence_num));
4513
4514         /* Initialize fence registers to zero */
4515         for (i = 0; i < dev_priv->num_fence_regs; i++) {
4516                 struct drm_i915_fence_reg *fence = &dev_priv->fence_regs[i];
4517
4518                 fence->i915 = dev_priv;
4519                 fence->id = i;
4520                 list_add_tail(&fence->link, &dev_priv->mm.fence_list);
4521         }
4522         i915_gem_restore_fences(dev_priv);
4523
4524         i915_gem_detect_bit_6_swizzle(dev_priv);
4525 }
4526
4527 int
4528 i915_gem_load_init(struct drm_device *dev)
4529 {
4530         struct drm_i915_private *dev_priv = to_i915(dev);
4531         int err = -ENOMEM;
4532
4533         dev_priv->objects = KMEM_CACHE(drm_i915_gem_object, SLAB_HWCACHE_ALIGN);
4534         if (!dev_priv->objects)
4535                 goto err_out;
4536
4537         dev_priv->vmas = KMEM_CACHE(i915_vma, SLAB_HWCACHE_ALIGN);
4538         if (!dev_priv->vmas)
4539                 goto err_objects;
4540
4541         dev_priv->requests = KMEM_CACHE(drm_i915_gem_request,
4542                                         SLAB_HWCACHE_ALIGN |
4543                                         SLAB_RECLAIM_ACCOUNT |
4544                                         SLAB_DESTROY_BY_RCU);
4545         if (!dev_priv->requests)
4546                 goto err_vmas;
4547
4548         dev_priv->dependencies = KMEM_CACHE(i915_dependency,
4549                                             SLAB_HWCACHE_ALIGN |
4550                                             SLAB_RECLAIM_ACCOUNT);
4551         if (!dev_priv->dependencies)
4552                 goto err_requests;
4553
4554         mutex_lock(&dev_priv->drm.struct_mutex);
4555         INIT_LIST_HEAD(&dev_priv->gt.timelines);
4556         err = i915_gem_timeline_init__global(dev_priv);
4557         mutex_unlock(&dev_priv->drm.struct_mutex);
4558         if (err)
4559                 goto err_dependencies;
4560
4561         INIT_LIST_HEAD(&dev_priv->context_list);
4562         INIT_WORK(&dev_priv->mm.free_work, __i915_gem_free_work);
4563         init_llist_head(&dev_priv->mm.free_list);
4564         INIT_LIST_HEAD(&dev_priv->mm.unbound_list);
4565         INIT_LIST_HEAD(&dev_priv->mm.bound_list);
4566         INIT_LIST_HEAD(&dev_priv->mm.fence_list);
4567         INIT_LIST_HEAD(&dev_priv->mm.userfault_list);
4568         INIT_DELAYED_WORK(&dev_priv->gt.retire_work,
4569                           i915_gem_retire_work_handler);
4570         INIT_DELAYED_WORK(&dev_priv->gt.idle_work,
4571                           i915_gem_idle_work_handler);
4572         init_waitqueue_head(&dev_priv->gpu_error.wait_queue);
4573         init_waitqueue_head(&dev_priv->gpu_error.reset_queue);
4574
4575         dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL;
4576
4577         init_waitqueue_head(&dev_priv->pending_flip_queue);
4578
4579         dev_priv->mm.interruptible = true;
4580
4581         atomic_set(&dev_priv->mm.bsd_engine_dispatch_index, 0);
4582
4583         spin_lock_init(&dev_priv->fb_tracking.lock);
4584
4585         return 0;
4586
4587 err_dependencies:
4588         kmem_cache_destroy(dev_priv->dependencies);
4589 err_requests:
4590         kmem_cache_destroy(dev_priv->requests);
4591 err_vmas:
4592         kmem_cache_destroy(dev_priv->vmas);
4593 err_objects:
4594         kmem_cache_destroy(dev_priv->objects);
4595 err_out:
4596         return err;
4597 }
4598
4599 void i915_gem_load_cleanup(struct drm_device *dev)
4600 {
4601         struct drm_i915_private *dev_priv = to_i915(dev);
4602
4603         WARN_ON(!llist_empty(&dev_priv->mm.free_list));
4604
4605         mutex_lock(&dev_priv->drm.struct_mutex);
4606         i915_gem_timeline_fini(&dev_priv->gt.global_timeline);
4607         WARN_ON(!list_empty(&dev_priv->gt.timelines));
4608         mutex_unlock(&dev_priv->drm.struct_mutex);
4609
4610         kmem_cache_destroy(dev_priv->dependencies);
4611         kmem_cache_destroy(dev_priv->requests);
4612         kmem_cache_destroy(dev_priv->vmas);
4613         kmem_cache_destroy(dev_priv->objects);
4614
4615         /* And ensure that our DESTROY_BY_RCU slabs are truly destroyed */
4616         rcu_barrier();
4617 }
4618
4619 int i915_gem_freeze(struct drm_i915_private *dev_priv)
4620 {
4621         intel_runtime_pm_get(dev_priv);
4622
4623         mutex_lock(&dev_priv->drm.struct_mutex);
4624         i915_gem_shrink_all(dev_priv);
4625         mutex_unlock(&dev_priv->drm.struct_mutex);
4626
4627         intel_runtime_pm_put(dev_priv);
4628
4629         return 0;
4630 }
4631
4632 int i915_gem_freeze_late(struct drm_i915_private *dev_priv)
4633 {
4634         struct drm_i915_gem_object *obj;
4635         struct list_head *phases[] = {
4636                 &dev_priv->mm.unbound_list,
4637                 &dev_priv->mm.bound_list,
4638                 NULL
4639         }, **p;
4640
4641         /* Called just before we write the hibernation image.
4642          *
4643          * We need to update the domain tracking to reflect that the CPU
4644          * will be accessing all the pages to create and restore from the
4645          * hibernation, and so upon restoration those pages will be in the
4646          * CPU domain.
4647          *
4648          * To make sure the hibernation image contains the latest state,
4649          * we update that state just before writing out the image.
4650          *
4651          * To try and reduce the hibernation image, we manually shrink
4652          * the objects as well.
4653          */
4654
4655         mutex_lock(&dev_priv->drm.struct_mutex);
4656         i915_gem_shrink(dev_priv, -1UL, I915_SHRINK_UNBOUND);
4657
4658         for (p = phases; *p; p++) {
4659                 list_for_each_entry(obj, *p, global_link) {
4660                         obj->base.read_domains = I915_GEM_DOMAIN_CPU;
4661                         obj->base.write_domain = I915_GEM_DOMAIN_CPU;
4662                 }
4663         }
4664         mutex_unlock(&dev_priv->drm.struct_mutex);
4665
4666         return 0;
4667 }
4668
4669 void i915_gem_release(struct drm_device *dev, struct drm_file *file)
4670 {
4671         struct drm_i915_file_private *file_priv = file->driver_priv;
4672         struct drm_i915_gem_request *request;
4673
4674         /* Clean up our request list when the client is going away, so that
4675          * later retire_requests won't dereference our soon-to-be-gone
4676          * file_priv.
4677          */
4678         spin_lock(&file_priv->mm.lock);
4679         list_for_each_entry(request, &file_priv->mm.request_list, client_list)
4680                 request->file_priv = NULL;
4681         spin_unlock(&file_priv->mm.lock);
4682
4683         if (!list_empty(&file_priv->rps.link)) {
4684                 spin_lock(&to_i915(dev)->rps.client_lock);
4685                 list_del(&file_priv->rps.link);
4686                 spin_unlock(&to_i915(dev)->rps.client_lock);
4687         }
4688 }
4689
4690 int i915_gem_open(struct drm_device *dev, struct drm_file *file)
4691 {
4692         struct drm_i915_file_private *file_priv;
4693         int ret;
4694
4695         DRM_DEBUG("\n");
4696
4697         file_priv = kzalloc(sizeof(*file_priv), GFP_KERNEL);
4698         if (!file_priv)
4699                 return -ENOMEM;
4700
4701         file->driver_priv = file_priv;
4702         file_priv->dev_priv = to_i915(dev);
4703         file_priv->file = file;
4704         INIT_LIST_HEAD(&file_priv->rps.link);
4705
4706         spin_lock_init(&file_priv->mm.lock);
4707         INIT_LIST_HEAD(&file_priv->mm.request_list);
4708
4709         file_priv->bsd_engine = -1;
4710
4711         ret = i915_gem_context_open(dev, file);
4712         if (ret)
4713                 kfree(file_priv);
4714
4715         return ret;
4716 }
4717
4718 /**
4719  * i915_gem_track_fb - update frontbuffer tracking
4720  * @old: current GEM buffer for the frontbuffer slots
4721  * @new: new GEM buffer for the frontbuffer slots
4722  * @frontbuffer_bits: bitmask of frontbuffer slots
4723  *
4724  * This updates the frontbuffer tracking bits @frontbuffer_bits by clearing them
4725  * from @old and setting them in @new. Both @old and @new can be NULL.
4726  */
4727 void i915_gem_track_fb(struct drm_i915_gem_object *old,
4728                        struct drm_i915_gem_object *new,
4729                        unsigned frontbuffer_bits)
4730 {
4731         /* Control of individual bits within the mask are guarded by
4732          * the owning plane->mutex, i.e. we can never see concurrent
4733          * manipulation of individual bits. But since the bitfield as a whole
4734          * is updated using RMW, we need to use atomics in order to update
4735          * the bits.
4736          */
4737         BUILD_BUG_ON(INTEL_FRONTBUFFER_BITS_PER_PIPE * I915_MAX_PIPES >
4738                      sizeof(atomic_t) * BITS_PER_BYTE);
4739
4740         if (old) {
4741                 WARN_ON(!(atomic_read(&old->frontbuffer_bits) & frontbuffer_bits));
4742                 atomic_andnot(frontbuffer_bits, &old->frontbuffer_bits);
4743         }
4744
4745         if (new) {
4746                 WARN_ON(atomic_read(&new->frontbuffer_bits) & frontbuffer_bits);
4747                 atomic_or(frontbuffer_bits, &new->frontbuffer_bits);
4748         }
4749 }
4750
4751 /* Allocate a new GEM object and fill it with the supplied data */
4752 struct drm_i915_gem_object *
4753 i915_gem_object_create_from_data(struct drm_device *dev,
4754                                  const void *data, size_t size)
4755 {
4756         struct drm_i915_gem_object *obj;
4757         struct sg_table *sg;
4758         size_t bytes;
4759         int ret;
4760
4761         obj = i915_gem_object_create(dev, round_up(size, PAGE_SIZE));
4762         if (IS_ERR(obj))
4763                 return obj;
4764
4765         ret = i915_gem_object_set_to_cpu_domain(obj, true);
4766         if (ret)
4767                 goto fail;
4768
4769         ret = i915_gem_object_pin_pages(obj);
4770         if (ret)
4771                 goto fail;
4772
4773         sg = obj->mm.pages;
4774         bytes = sg_copy_from_buffer(sg->sgl, sg->nents, (void *)data, size);
4775         obj->mm.dirty = true; /* Backing store is now out of date */
4776         i915_gem_object_unpin_pages(obj);
4777
4778         if (WARN_ON(bytes != size)) {
4779                 DRM_ERROR("Incomplete copy, wrote %zu of %zu", bytes, size);
4780                 ret = -EFAULT;
4781                 goto fail;
4782         }
4783
4784         return obj;
4785
4786 fail:
4787         i915_gem_object_put(obj);
4788         return ERR_PTR(ret);
4789 }
4790
4791 struct scatterlist *
4792 i915_gem_object_get_sg(struct drm_i915_gem_object *obj,
4793                        unsigned int n,
4794                        unsigned int *offset)
4795 {
4796         struct i915_gem_object_page_iter *iter = &obj->mm.get_page;
4797         struct scatterlist *sg;
4798         unsigned int idx, count;
4799
4800         might_sleep();
4801         GEM_BUG_ON(n >= obj->base.size >> PAGE_SHIFT);
4802         GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj));
4803
4804         /* As we iterate forward through the sg, we record each entry in a
4805          * radixtree for quick repeated (backwards) lookups. If we have seen
4806          * this index previously, we will have an entry for it.
4807          *
4808          * Initial lookup is O(N), but this is amortized to O(1) for
4809          * sequential page access (where each new request is consecutive
4810          * to the previous one). Repeated lookups are O(lg(obj->base.size)),
4811          * i.e. O(1) with a large constant!
4812          */
4813         if (n < READ_ONCE(iter->sg_idx))
4814                 goto lookup;
4815
4816         mutex_lock(&iter->lock);
4817
4818         /* We prefer to reuse the last sg so that repeated lookup of this
4819          * (or the subsequent) sg are fast - comparing against the last
4820          * sg is faster than going through the radixtree.
4821          */
4822
4823         sg = iter->sg_pos;
4824         idx = iter->sg_idx;
4825         count = __sg_page_count(sg);
4826
4827         while (idx + count <= n) {
4828                 unsigned long exception, i;
4829                 int ret;
4830
4831                 /* If we cannot allocate and insert this entry, or the
4832                  * individual pages from this range, cancel updating the
4833                  * sg_idx so that on this lookup we are forced to linearly
4834                  * scan onwards, but on future lookups we will try the
4835                  * insertion again (in which case we need to be careful of
4836                  * the error return reporting that we have already inserted
4837                  * this index).
4838                  */
4839                 ret = radix_tree_insert(&iter->radix, idx, sg);
4840                 if (ret && ret != -EEXIST)
4841                         goto scan;
4842
4843                 exception =
4844                         RADIX_TREE_EXCEPTIONAL_ENTRY |
4845                         idx << RADIX_TREE_EXCEPTIONAL_SHIFT;
4846                 for (i = 1; i < count; i++) {
4847                         ret = radix_tree_insert(&iter->radix, idx + i,
4848                                                 (void *)exception);
4849                         if (ret && ret != -EEXIST)
4850                                 goto scan;
4851                 }
4852
4853                 idx += count;
4854                 sg = ____sg_next(sg);
4855                 count = __sg_page_count(sg);
4856         }
4857
4858 scan:
4859         iter->sg_pos = sg;
4860         iter->sg_idx = idx;
4861
4862         mutex_unlock(&iter->lock);
4863
4864         if (unlikely(n < idx)) /* insertion completed by another thread */
4865                 goto lookup;
4866
4867         /* In case we failed to insert the entry into the radixtree, we need
4868          * to look beyond the current sg.
4869          */
4870         while (idx + count <= n) {
4871                 idx += count;
4872                 sg = ____sg_next(sg);
4873                 count = __sg_page_count(sg);
4874         }
4875
4876         *offset = n - idx;
4877         return sg;
4878
4879 lookup:
4880         rcu_read_lock();
4881
4882         sg = radix_tree_lookup(&iter->radix, n);
4883         GEM_BUG_ON(!sg);
4884
4885         /* If this index is in the middle of multi-page sg entry,
4886          * the radixtree will contain an exceptional entry that points
4887          * to the start of that range. We will return the pointer to
4888          * the base page and the offset of this page within the
4889          * sg entry's range.
4890          */
4891         *offset = 0;
4892         if (unlikely(radix_tree_exception(sg))) {
4893                 unsigned long base =
4894                         (unsigned long)sg >> RADIX_TREE_EXCEPTIONAL_SHIFT;
4895
4896                 sg = radix_tree_lookup(&iter->radix, base);
4897                 GEM_BUG_ON(!sg);
4898
4899                 *offset = n - base;
4900         }
4901
4902         rcu_read_unlock();
4903
4904         return sg;
4905 }
4906
4907 struct page *
4908 i915_gem_object_get_page(struct drm_i915_gem_object *obj, unsigned int n)
4909 {
4910         struct scatterlist *sg;
4911         unsigned int offset;
4912
4913         GEM_BUG_ON(!i915_gem_object_has_struct_page(obj));
4914
4915         sg = i915_gem_object_get_sg(obj, n, &offset);
4916         return nth_page(sg_page(sg), offset);
4917 }
4918
4919 /* Like i915_gem_object_get_page(), but mark the returned page dirty */
4920 struct page *
4921 i915_gem_object_get_dirty_page(struct drm_i915_gem_object *obj,
4922                                unsigned int n)
4923 {
4924         struct page *page;
4925
4926         page = i915_gem_object_get_page(obj, n);
4927         if (!obj->mm.dirty)
4928                 set_page_dirty(page);
4929
4930         return page;
4931 }
4932
4933 dma_addr_t
4934 i915_gem_object_get_dma_address(struct drm_i915_gem_object *obj,
4935                                 unsigned long n)
4936 {
4937         struct scatterlist *sg;
4938         unsigned int offset;
4939
4940         sg = i915_gem_object_get_sg(obj, n, &offset);
4941         return sg_dma_address(sg) + (offset << PAGE_SHIFT);
4942 }