f678017c4d3127cfaf2ae021818f700ea60f808a
[sfrench/cifs-2.6.git] / drivers / gpu / drm / i915 / i915_gem.c
1 /*
2  * Copyright © 2008 Intel Corporation
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice (including the next
12  * paragraph) shall be included in all copies or substantial portions of the
13  * Software.
14  *
15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21  * IN THE SOFTWARE.
22  *
23  * Authors:
24  *    Eric Anholt <eric@anholt.net>
25  *
26  */
27
28 #include <drm/drmP.h>
29 #include <drm/drm_vma_manager.h>
30 #include <drm/i915_drm.h>
31 #include "i915_drv.h"
32 #include "i915_trace.h"
33 #include "intel_drv.h"
34 #include <linux/oom.h>
35 #include <linux/shmem_fs.h>
36 #include <linux/slab.h>
37 #include <linux/swap.h>
38 #include <linux/pci.h>
39 #include <linux/dma-buf.h>
40
41 static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj);
42 static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj,
43                                                    bool force);
44 static __must_check int
45 i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj,
46                                bool readonly);
47 static void
48 i915_gem_object_retire(struct drm_i915_gem_object *obj);
49
50 static void i915_gem_write_fence(struct drm_device *dev, int reg,
51                                  struct drm_i915_gem_object *obj);
52 static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj,
53                                          struct drm_i915_fence_reg *fence,
54                                          bool enable);
55
56 static unsigned long i915_gem_shrinker_count(struct shrinker *shrinker,
57                                              struct shrink_control *sc);
58 static unsigned long i915_gem_shrinker_scan(struct shrinker *shrinker,
59                                             struct shrink_control *sc);
60 static int i915_gem_shrinker_oom(struct notifier_block *nb,
61                                  unsigned long event,
62                                  void *ptr);
63 static unsigned long i915_gem_shrink_all(struct drm_i915_private *dev_priv);
64
65 static bool cpu_cache_is_coherent(struct drm_device *dev,
66                                   enum i915_cache_level level)
67 {
68         return HAS_LLC(dev) || level != I915_CACHE_NONE;
69 }
70
71 static bool cpu_write_needs_clflush(struct drm_i915_gem_object *obj)
72 {
73         if (!cpu_cache_is_coherent(obj->base.dev, obj->cache_level))
74                 return true;
75
76         return obj->pin_display;
77 }
78
79 static inline void i915_gem_object_fence_lost(struct drm_i915_gem_object *obj)
80 {
81         if (obj->tiling_mode)
82                 i915_gem_release_mmap(obj);
83
84         /* As we do not have an associated fence register, we will force
85          * a tiling change if we ever need to acquire one.
86          */
87         obj->fence_dirty = false;
88         obj->fence_reg = I915_FENCE_REG_NONE;
89 }
90
91 /* some bookkeeping */
92 static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv,
93                                   size_t size)
94 {
95         spin_lock(&dev_priv->mm.object_stat_lock);
96         dev_priv->mm.object_count++;
97         dev_priv->mm.object_memory += size;
98         spin_unlock(&dev_priv->mm.object_stat_lock);
99 }
100
101 static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv,
102                                      size_t size)
103 {
104         spin_lock(&dev_priv->mm.object_stat_lock);
105         dev_priv->mm.object_count--;
106         dev_priv->mm.object_memory -= size;
107         spin_unlock(&dev_priv->mm.object_stat_lock);
108 }
109
110 static int
111 i915_gem_wait_for_error(struct i915_gpu_error *error)
112 {
113         int ret;
114
115 #define EXIT_COND (!i915_reset_in_progress(error) || \
116                    i915_terminally_wedged(error))
117         if (EXIT_COND)
118                 return 0;
119
120         /*
121          * Only wait 10 seconds for the gpu reset to complete to avoid hanging
122          * userspace. If it takes that long something really bad is going on and
123          * we should simply try to bail out and fail as gracefully as possible.
124          */
125         ret = wait_event_interruptible_timeout(error->reset_queue,
126                                                EXIT_COND,
127                                                10*HZ);
128         if (ret == 0) {
129                 DRM_ERROR("Timed out waiting for the gpu reset to complete\n");
130                 return -EIO;
131         } else if (ret < 0) {
132                 return ret;
133         }
134 #undef EXIT_COND
135
136         return 0;
137 }
138
139 int i915_mutex_lock_interruptible(struct drm_device *dev)
140 {
141         struct drm_i915_private *dev_priv = dev->dev_private;
142         int ret;
143
144         ret = i915_gem_wait_for_error(&dev_priv->gpu_error);
145         if (ret)
146                 return ret;
147
148         ret = mutex_lock_interruptible(&dev->struct_mutex);
149         if (ret)
150                 return ret;
151
152         WARN_ON(i915_verify_lists(dev));
153         return 0;
154 }
155
156 static inline bool
157 i915_gem_object_is_inactive(struct drm_i915_gem_object *obj)
158 {
159         return i915_gem_obj_bound_any(obj) && !obj->active;
160 }
161
162 int
163 i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
164                             struct drm_file *file)
165 {
166         struct drm_i915_private *dev_priv = dev->dev_private;
167         struct drm_i915_gem_get_aperture *args = data;
168         struct drm_i915_gem_object *obj;
169         size_t pinned;
170
171         pinned = 0;
172         mutex_lock(&dev->struct_mutex);
173         list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list)
174                 if (i915_gem_obj_is_pinned(obj))
175                         pinned += i915_gem_obj_ggtt_size(obj);
176         mutex_unlock(&dev->struct_mutex);
177
178         args->aper_size = dev_priv->gtt.base.total;
179         args->aper_available_size = args->aper_size - pinned;
180
181         return 0;
182 }
183
184 static int
185 i915_gem_object_get_pages_phys(struct drm_i915_gem_object *obj)
186 {
187         struct address_space *mapping = file_inode(obj->base.filp)->i_mapping;
188         char *vaddr = obj->phys_handle->vaddr;
189         struct sg_table *st;
190         struct scatterlist *sg;
191         int i;
192
193         if (WARN_ON(i915_gem_object_needs_bit17_swizzle(obj)))
194                 return -EINVAL;
195
196         for (i = 0; i < obj->base.size / PAGE_SIZE; i++) {
197                 struct page *page;
198                 char *src;
199
200                 page = shmem_read_mapping_page(mapping, i);
201                 if (IS_ERR(page))
202                         return PTR_ERR(page);
203
204                 src = kmap_atomic(page);
205                 memcpy(vaddr, src, PAGE_SIZE);
206                 drm_clflush_virt_range(vaddr, PAGE_SIZE);
207                 kunmap_atomic(src);
208
209                 page_cache_release(page);
210                 vaddr += PAGE_SIZE;
211         }
212
213         i915_gem_chipset_flush(obj->base.dev);
214
215         st = kmalloc(sizeof(*st), GFP_KERNEL);
216         if (st == NULL)
217                 return -ENOMEM;
218
219         if (sg_alloc_table(st, 1, GFP_KERNEL)) {
220                 kfree(st);
221                 return -ENOMEM;
222         }
223
224         sg = st->sgl;
225         sg->offset = 0;
226         sg->length = obj->base.size;
227
228         sg_dma_address(sg) = obj->phys_handle->busaddr;
229         sg_dma_len(sg) = obj->base.size;
230
231         obj->pages = st;
232         obj->has_dma_mapping = true;
233         return 0;
234 }
235
236 static void
237 i915_gem_object_put_pages_phys(struct drm_i915_gem_object *obj)
238 {
239         int ret;
240
241         BUG_ON(obj->madv == __I915_MADV_PURGED);
242
243         ret = i915_gem_object_set_to_cpu_domain(obj, true);
244         if (ret) {
245                 /* In the event of a disaster, abandon all caches and
246                  * hope for the best.
247                  */
248                 WARN_ON(ret != -EIO);
249                 obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU;
250         }
251
252         if (obj->madv == I915_MADV_DONTNEED)
253                 obj->dirty = 0;
254
255         if (obj->dirty) {
256                 struct address_space *mapping = file_inode(obj->base.filp)->i_mapping;
257                 char *vaddr = obj->phys_handle->vaddr;
258                 int i;
259
260                 for (i = 0; i < obj->base.size / PAGE_SIZE; i++) {
261                         struct page *page;
262                         char *dst;
263
264                         page = shmem_read_mapping_page(mapping, i);
265                         if (IS_ERR(page))
266                                 continue;
267
268                         dst = kmap_atomic(page);
269                         drm_clflush_virt_range(vaddr, PAGE_SIZE);
270                         memcpy(dst, vaddr, PAGE_SIZE);
271                         kunmap_atomic(dst);
272
273                         set_page_dirty(page);
274                         if (obj->madv == I915_MADV_WILLNEED)
275                                 mark_page_accessed(page);
276                         page_cache_release(page);
277                         vaddr += PAGE_SIZE;
278                 }
279                 obj->dirty = 0;
280         }
281
282         sg_free_table(obj->pages);
283         kfree(obj->pages);
284
285         obj->has_dma_mapping = false;
286 }
287
288 static void
289 i915_gem_object_release_phys(struct drm_i915_gem_object *obj)
290 {
291         drm_pci_free(obj->base.dev, obj->phys_handle);
292 }
293
294 static const struct drm_i915_gem_object_ops i915_gem_phys_ops = {
295         .get_pages = i915_gem_object_get_pages_phys,
296         .put_pages = i915_gem_object_put_pages_phys,
297         .release = i915_gem_object_release_phys,
298 };
299
300 static int
301 drop_pages(struct drm_i915_gem_object *obj)
302 {
303         struct i915_vma *vma, *next;
304         int ret;
305
306         drm_gem_object_reference(&obj->base);
307         list_for_each_entry_safe(vma, next, &obj->vma_list, vma_link)
308                 if (i915_vma_unbind(vma))
309                         break;
310
311         ret = i915_gem_object_put_pages(obj);
312         drm_gem_object_unreference(&obj->base);
313
314         return ret;
315 }
316
317 int
318 i915_gem_object_attach_phys(struct drm_i915_gem_object *obj,
319                             int align)
320 {
321         drm_dma_handle_t *phys;
322         int ret;
323
324         if (obj->phys_handle) {
325                 if ((unsigned long)obj->phys_handle->vaddr & (align -1))
326                         return -EBUSY;
327
328                 return 0;
329         }
330
331         if (obj->madv != I915_MADV_WILLNEED)
332                 return -EFAULT;
333
334         if (obj->base.filp == NULL)
335                 return -EINVAL;
336
337         ret = drop_pages(obj);
338         if (ret)
339                 return ret;
340
341         /* create a new object */
342         phys = drm_pci_alloc(obj->base.dev, obj->base.size, align);
343         if (!phys)
344                 return -ENOMEM;
345
346         obj->phys_handle = phys;
347         obj->ops = &i915_gem_phys_ops;
348
349         return i915_gem_object_get_pages(obj);
350 }
351
352 static int
353 i915_gem_phys_pwrite(struct drm_i915_gem_object *obj,
354                      struct drm_i915_gem_pwrite *args,
355                      struct drm_file *file_priv)
356 {
357         struct drm_device *dev = obj->base.dev;
358         void *vaddr = obj->phys_handle->vaddr + args->offset;
359         char __user *user_data = to_user_ptr(args->data_ptr);
360         int ret;
361
362         /* We manually control the domain here and pretend that it
363          * remains coherent i.e. in the GTT domain, like shmem_pwrite.
364          */
365         ret = i915_gem_object_wait_rendering(obj, false);
366         if (ret)
367                 return ret;
368
369         if (__copy_from_user_inatomic_nocache(vaddr, user_data, args->size)) {
370                 unsigned long unwritten;
371
372                 /* The physical object once assigned is fixed for the lifetime
373                  * of the obj, so we can safely drop the lock and continue
374                  * to access vaddr.
375                  */
376                 mutex_unlock(&dev->struct_mutex);
377                 unwritten = copy_from_user(vaddr, user_data, args->size);
378                 mutex_lock(&dev->struct_mutex);
379                 if (unwritten)
380                         return -EFAULT;
381         }
382
383         drm_clflush_virt_range(vaddr, args->size);
384         i915_gem_chipset_flush(dev);
385         return 0;
386 }
387
388 void *i915_gem_object_alloc(struct drm_device *dev)
389 {
390         struct drm_i915_private *dev_priv = dev->dev_private;
391         return kmem_cache_zalloc(dev_priv->slab, GFP_KERNEL);
392 }
393
394 void i915_gem_object_free(struct drm_i915_gem_object *obj)
395 {
396         struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
397         kmem_cache_free(dev_priv->slab, obj);
398 }
399
400 static int
401 i915_gem_create(struct drm_file *file,
402                 struct drm_device *dev,
403                 uint64_t size,
404                 bool dumb,
405                 uint32_t *handle_p)
406 {
407         struct drm_i915_gem_object *obj;
408         int ret;
409         u32 handle;
410
411         size = roundup(size, PAGE_SIZE);
412         if (size == 0)
413                 return -EINVAL;
414
415         /* Allocate the new object */
416         obj = i915_gem_alloc_object(dev, size);
417         if (obj == NULL)
418                 return -ENOMEM;
419
420         obj->base.dumb = dumb;
421         ret = drm_gem_handle_create(file, &obj->base, &handle);
422         /* drop reference from allocate - handle holds it now */
423         drm_gem_object_unreference_unlocked(&obj->base);
424         if (ret)
425                 return ret;
426
427         *handle_p = handle;
428         return 0;
429 }
430
431 int
432 i915_gem_dumb_create(struct drm_file *file,
433                      struct drm_device *dev,
434                      struct drm_mode_create_dumb *args)
435 {
436         /* have to work out size/pitch and return them */
437         args->pitch = ALIGN(args->width * DIV_ROUND_UP(args->bpp, 8), 64);
438         args->size = args->pitch * args->height;
439         return i915_gem_create(file, dev,
440                                args->size, true, &args->handle);
441 }
442
443 /**
444  * Creates a new mm object and returns a handle to it.
445  */
446 int
447 i915_gem_create_ioctl(struct drm_device *dev, void *data,
448                       struct drm_file *file)
449 {
450         struct drm_i915_gem_create *args = data;
451
452         return i915_gem_create(file, dev,
453                                args->size, false, &args->handle);
454 }
455
456 static inline int
457 __copy_to_user_swizzled(char __user *cpu_vaddr,
458                         const char *gpu_vaddr, int gpu_offset,
459                         int length)
460 {
461         int ret, cpu_offset = 0;
462
463         while (length > 0) {
464                 int cacheline_end = ALIGN(gpu_offset + 1, 64);
465                 int this_length = min(cacheline_end - gpu_offset, length);
466                 int swizzled_gpu_offset = gpu_offset ^ 64;
467
468                 ret = __copy_to_user(cpu_vaddr + cpu_offset,
469                                      gpu_vaddr + swizzled_gpu_offset,
470                                      this_length);
471                 if (ret)
472                         return ret + length;
473
474                 cpu_offset += this_length;
475                 gpu_offset += this_length;
476                 length -= this_length;
477         }
478
479         return 0;
480 }
481
482 static inline int
483 __copy_from_user_swizzled(char *gpu_vaddr, int gpu_offset,
484                           const char __user *cpu_vaddr,
485                           int length)
486 {
487         int ret, cpu_offset = 0;
488
489         while (length > 0) {
490                 int cacheline_end = ALIGN(gpu_offset + 1, 64);
491                 int this_length = min(cacheline_end - gpu_offset, length);
492                 int swizzled_gpu_offset = gpu_offset ^ 64;
493
494                 ret = __copy_from_user(gpu_vaddr + swizzled_gpu_offset,
495                                        cpu_vaddr + cpu_offset,
496                                        this_length);
497                 if (ret)
498                         return ret + length;
499
500                 cpu_offset += this_length;
501                 gpu_offset += this_length;
502                 length -= this_length;
503         }
504
505         return 0;
506 }
507
508 /*
509  * Pins the specified object's pages and synchronizes the object with
510  * GPU accesses. Sets needs_clflush to non-zero if the caller should
511  * flush the object from the CPU cache.
512  */
513 int i915_gem_obj_prepare_shmem_read(struct drm_i915_gem_object *obj,
514                                     int *needs_clflush)
515 {
516         int ret;
517
518         *needs_clflush = 0;
519
520         if (!obj->base.filp)
521                 return -EINVAL;
522
523         if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU)) {
524                 /* If we're not in the cpu read domain, set ourself into the gtt
525                  * read domain and manually flush cachelines (if required). This
526                  * optimizes for the case when the gpu will dirty the data
527                  * anyway again before the next pread happens. */
528                 *needs_clflush = !cpu_cache_is_coherent(obj->base.dev,
529                                                         obj->cache_level);
530                 ret = i915_gem_object_wait_rendering(obj, true);
531                 if (ret)
532                         return ret;
533
534                 i915_gem_object_retire(obj);
535         }
536
537         ret = i915_gem_object_get_pages(obj);
538         if (ret)
539                 return ret;
540
541         i915_gem_object_pin_pages(obj);
542
543         return ret;
544 }
545
546 /* Per-page copy function for the shmem pread fastpath.
547  * Flushes invalid cachelines before reading the target if
548  * needs_clflush is set. */
549 static int
550 shmem_pread_fast(struct page *page, int shmem_page_offset, int page_length,
551                  char __user *user_data,
552                  bool page_do_bit17_swizzling, bool needs_clflush)
553 {
554         char *vaddr;
555         int ret;
556
557         if (unlikely(page_do_bit17_swizzling))
558                 return -EINVAL;
559
560         vaddr = kmap_atomic(page);
561         if (needs_clflush)
562                 drm_clflush_virt_range(vaddr + shmem_page_offset,
563                                        page_length);
564         ret = __copy_to_user_inatomic(user_data,
565                                       vaddr + shmem_page_offset,
566                                       page_length);
567         kunmap_atomic(vaddr);
568
569         return ret ? -EFAULT : 0;
570 }
571
572 static void
573 shmem_clflush_swizzled_range(char *addr, unsigned long length,
574                              bool swizzled)
575 {
576         if (unlikely(swizzled)) {
577                 unsigned long start = (unsigned long) addr;
578                 unsigned long end = (unsigned long) addr + length;
579
580                 /* For swizzling simply ensure that we always flush both
581                  * channels. Lame, but simple and it works. Swizzled
582                  * pwrite/pread is far from a hotpath - current userspace
583                  * doesn't use it at all. */
584                 start = round_down(start, 128);
585                 end = round_up(end, 128);
586
587                 drm_clflush_virt_range((void *)start, end - start);
588         } else {
589                 drm_clflush_virt_range(addr, length);
590         }
591
592 }
593
594 /* Only difference to the fast-path function is that this can handle bit17
595  * and uses non-atomic copy and kmap functions. */
596 static int
597 shmem_pread_slow(struct page *page, int shmem_page_offset, int page_length,
598                  char __user *user_data,
599                  bool page_do_bit17_swizzling, bool needs_clflush)
600 {
601         char *vaddr;
602         int ret;
603
604         vaddr = kmap(page);
605         if (needs_clflush)
606                 shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
607                                              page_length,
608                                              page_do_bit17_swizzling);
609
610         if (page_do_bit17_swizzling)
611                 ret = __copy_to_user_swizzled(user_data,
612                                               vaddr, shmem_page_offset,
613                                               page_length);
614         else
615                 ret = __copy_to_user(user_data,
616                                      vaddr + shmem_page_offset,
617                                      page_length);
618         kunmap(page);
619
620         return ret ? - EFAULT : 0;
621 }
622
623 static int
624 i915_gem_shmem_pread(struct drm_device *dev,
625                      struct drm_i915_gem_object *obj,
626                      struct drm_i915_gem_pread *args,
627                      struct drm_file *file)
628 {
629         char __user *user_data;
630         ssize_t remain;
631         loff_t offset;
632         int shmem_page_offset, page_length, ret = 0;
633         int obj_do_bit17_swizzling, page_do_bit17_swizzling;
634         int prefaulted = 0;
635         int needs_clflush = 0;
636         struct sg_page_iter sg_iter;
637
638         user_data = to_user_ptr(args->data_ptr);
639         remain = args->size;
640
641         obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
642
643         ret = i915_gem_obj_prepare_shmem_read(obj, &needs_clflush);
644         if (ret)
645                 return ret;
646
647         offset = args->offset;
648
649         for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents,
650                          offset >> PAGE_SHIFT) {
651                 struct page *page = sg_page_iter_page(&sg_iter);
652
653                 if (remain <= 0)
654                         break;
655
656                 /* Operation in this page
657                  *
658                  * shmem_page_offset = offset within page in shmem file
659                  * page_length = bytes to copy for this page
660                  */
661                 shmem_page_offset = offset_in_page(offset);
662                 page_length = remain;
663                 if ((shmem_page_offset + page_length) > PAGE_SIZE)
664                         page_length = PAGE_SIZE - shmem_page_offset;
665
666                 page_do_bit17_swizzling = obj_do_bit17_swizzling &&
667                         (page_to_phys(page) & (1 << 17)) != 0;
668
669                 ret = shmem_pread_fast(page, shmem_page_offset, page_length,
670                                        user_data, page_do_bit17_swizzling,
671                                        needs_clflush);
672                 if (ret == 0)
673                         goto next_page;
674
675                 mutex_unlock(&dev->struct_mutex);
676
677                 if (likely(!i915.prefault_disable) && !prefaulted) {
678                         ret = fault_in_multipages_writeable(user_data, remain);
679                         /* Userspace is tricking us, but we've already clobbered
680                          * its pages with the prefault and promised to write the
681                          * data up to the first fault. Hence ignore any errors
682                          * and just continue. */
683                         (void)ret;
684                         prefaulted = 1;
685                 }
686
687                 ret = shmem_pread_slow(page, shmem_page_offset, page_length,
688                                        user_data, page_do_bit17_swizzling,
689                                        needs_clflush);
690
691                 mutex_lock(&dev->struct_mutex);
692
693                 if (ret)
694                         goto out;
695
696 next_page:
697                 remain -= page_length;
698                 user_data += page_length;
699                 offset += page_length;
700         }
701
702 out:
703         i915_gem_object_unpin_pages(obj);
704
705         return ret;
706 }
707
708 /**
709  * Reads data from the object referenced by handle.
710  *
711  * On error, the contents of *data are undefined.
712  */
713 int
714 i915_gem_pread_ioctl(struct drm_device *dev, void *data,
715                      struct drm_file *file)
716 {
717         struct drm_i915_gem_pread *args = data;
718         struct drm_i915_gem_object *obj;
719         int ret = 0;
720
721         if (args->size == 0)
722                 return 0;
723
724         if (!access_ok(VERIFY_WRITE,
725                        to_user_ptr(args->data_ptr),
726                        args->size))
727                 return -EFAULT;
728
729         ret = i915_mutex_lock_interruptible(dev);
730         if (ret)
731                 return ret;
732
733         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
734         if (&obj->base == NULL) {
735                 ret = -ENOENT;
736                 goto unlock;
737         }
738
739         /* Bounds check source.  */
740         if (args->offset > obj->base.size ||
741             args->size > obj->base.size - args->offset) {
742                 ret = -EINVAL;
743                 goto out;
744         }
745
746         /* prime objects have no backing filp to GEM pread/pwrite
747          * pages from.
748          */
749         if (!obj->base.filp) {
750                 ret = -EINVAL;
751                 goto out;
752         }
753
754         trace_i915_gem_object_pread(obj, args->offset, args->size);
755
756         ret = i915_gem_shmem_pread(dev, obj, args, file);
757
758 out:
759         drm_gem_object_unreference(&obj->base);
760 unlock:
761         mutex_unlock(&dev->struct_mutex);
762         return ret;
763 }
764
765 /* This is the fast write path which cannot handle
766  * page faults in the source data
767  */
768
769 static inline int
770 fast_user_write(struct io_mapping *mapping,
771                 loff_t page_base, int page_offset,
772                 char __user *user_data,
773                 int length)
774 {
775         void __iomem *vaddr_atomic;
776         void *vaddr;
777         unsigned long unwritten;
778
779         vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base);
780         /* We can use the cpu mem copy function because this is X86. */
781         vaddr = (void __force*)vaddr_atomic + page_offset;
782         unwritten = __copy_from_user_inatomic_nocache(vaddr,
783                                                       user_data, length);
784         io_mapping_unmap_atomic(vaddr_atomic);
785         return unwritten;
786 }
787
788 /**
789  * This is the fast pwrite path, where we copy the data directly from the
790  * user into the GTT, uncached.
791  */
792 static int
793 i915_gem_gtt_pwrite_fast(struct drm_device *dev,
794                          struct drm_i915_gem_object *obj,
795                          struct drm_i915_gem_pwrite *args,
796                          struct drm_file *file)
797 {
798         struct drm_i915_private *dev_priv = dev->dev_private;
799         ssize_t remain;
800         loff_t offset, page_base;
801         char __user *user_data;
802         int page_offset, page_length, ret;
803
804         ret = i915_gem_obj_ggtt_pin(obj, 0, PIN_MAPPABLE | PIN_NONBLOCK);
805         if (ret)
806                 goto out;
807
808         ret = i915_gem_object_set_to_gtt_domain(obj, true);
809         if (ret)
810                 goto out_unpin;
811
812         ret = i915_gem_object_put_fence(obj);
813         if (ret)
814                 goto out_unpin;
815
816         user_data = to_user_ptr(args->data_ptr);
817         remain = args->size;
818
819         offset = i915_gem_obj_ggtt_offset(obj) + args->offset;
820
821         while (remain > 0) {
822                 /* Operation in this page
823                  *
824                  * page_base = page offset within aperture
825                  * page_offset = offset within page
826                  * page_length = bytes to copy for this page
827                  */
828                 page_base = offset & PAGE_MASK;
829                 page_offset = offset_in_page(offset);
830                 page_length = remain;
831                 if ((page_offset + remain) > PAGE_SIZE)
832                         page_length = PAGE_SIZE - page_offset;
833
834                 /* If we get a fault while copying data, then (presumably) our
835                  * source page isn't available.  Return the error and we'll
836                  * retry in the slow path.
837                  */
838                 if (fast_user_write(dev_priv->gtt.mappable, page_base,
839                                     page_offset, user_data, page_length)) {
840                         ret = -EFAULT;
841                         goto out_unpin;
842                 }
843
844                 remain -= page_length;
845                 user_data += page_length;
846                 offset += page_length;
847         }
848
849 out_unpin:
850         i915_gem_object_ggtt_unpin(obj);
851 out:
852         return ret;
853 }
854
855 /* Per-page copy function for the shmem pwrite fastpath.
856  * Flushes invalid cachelines before writing to the target if
857  * needs_clflush_before is set and flushes out any written cachelines after
858  * writing if needs_clflush is set. */
859 static int
860 shmem_pwrite_fast(struct page *page, int shmem_page_offset, int page_length,
861                   char __user *user_data,
862                   bool page_do_bit17_swizzling,
863                   bool needs_clflush_before,
864                   bool needs_clflush_after)
865 {
866         char *vaddr;
867         int ret;
868
869         if (unlikely(page_do_bit17_swizzling))
870                 return -EINVAL;
871
872         vaddr = kmap_atomic(page);
873         if (needs_clflush_before)
874                 drm_clflush_virt_range(vaddr + shmem_page_offset,
875                                        page_length);
876         ret = __copy_from_user_inatomic(vaddr + shmem_page_offset,
877                                         user_data, page_length);
878         if (needs_clflush_after)
879                 drm_clflush_virt_range(vaddr + shmem_page_offset,
880                                        page_length);
881         kunmap_atomic(vaddr);
882
883         return ret ? -EFAULT : 0;
884 }
885
886 /* Only difference to the fast-path function is that this can handle bit17
887  * and uses non-atomic copy and kmap functions. */
888 static int
889 shmem_pwrite_slow(struct page *page, int shmem_page_offset, int page_length,
890                   char __user *user_data,
891                   bool page_do_bit17_swizzling,
892                   bool needs_clflush_before,
893                   bool needs_clflush_after)
894 {
895         char *vaddr;
896         int ret;
897
898         vaddr = kmap(page);
899         if (unlikely(needs_clflush_before || page_do_bit17_swizzling))
900                 shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
901                                              page_length,
902                                              page_do_bit17_swizzling);
903         if (page_do_bit17_swizzling)
904                 ret = __copy_from_user_swizzled(vaddr, shmem_page_offset,
905                                                 user_data,
906                                                 page_length);
907         else
908                 ret = __copy_from_user(vaddr + shmem_page_offset,
909                                        user_data,
910                                        page_length);
911         if (needs_clflush_after)
912                 shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
913                                              page_length,
914                                              page_do_bit17_swizzling);
915         kunmap(page);
916
917         return ret ? -EFAULT : 0;
918 }
919
920 static int
921 i915_gem_shmem_pwrite(struct drm_device *dev,
922                       struct drm_i915_gem_object *obj,
923                       struct drm_i915_gem_pwrite *args,
924                       struct drm_file *file)
925 {
926         ssize_t remain;
927         loff_t offset;
928         char __user *user_data;
929         int shmem_page_offset, page_length, ret = 0;
930         int obj_do_bit17_swizzling, page_do_bit17_swizzling;
931         int hit_slowpath = 0;
932         int needs_clflush_after = 0;
933         int needs_clflush_before = 0;
934         struct sg_page_iter sg_iter;
935
936         user_data = to_user_ptr(args->data_ptr);
937         remain = args->size;
938
939         obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
940
941         if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
942                 /* If we're not in the cpu write domain, set ourself into the gtt
943                  * write domain and manually flush cachelines (if required). This
944                  * optimizes for the case when the gpu will use the data
945                  * right away and we therefore have to clflush anyway. */
946                 needs_clflush_after = cpu_write_needs_clflush(obj);
947                 ret = i915_gem_object_wait_rendering(obj, false);
948                 if (ret)
949                         return ret;
950
951                 i915_gem_object_retire(obj);
952         }
953         /* Same trick applies to invalidate partially written cachelines read
954          * before writing. */
955         if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0)
956                 needs_clflush_before =
957                         !cpu_cache_is_coherent(dev, obj->cache_level);
958
959         ret = i915_gem_object_get_pages(obj);
960         if (ret)
961                 return ret;
962
963         i915_gem_object_pin_pages(obj);
964
965         offset = args->offset;
966         obj->dirty = 1;
967
968         for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents,
969                          offset >> PAGE_SHIFT) {
970                 struct page *page = sg_page_iter_page(&sg_iter);
971                 int partial_cacheline_write;
972
973                 if (remain <= 0)
974                         break;
975
976                 /* Operation in this page
977                  *
978                  * shmem_page_offset = offset within page in shmem file
979                  * page_length = bytes to copy for this page
980                  */
981                 shmem_page_offset = offset_in_page(offset);
982
983                 page_length = remain;
984                 if ((shmem_page_offset + page_length) > PAGE_SIZE)
985                         page_length = PAGE_SIZE - shmem_page_offset;
986
987                 /* If we don't overwrite a cacheline completely we need to be
988                  * careful to have up-to-date data by first clflushing. Don't
989                  * overcomplicate things and flush the entire patch. */
990                 partial_cacheline_write = needs_clflush_before &&
991                         ((shmem_page_offset | page_length)
992                                 & (boot_cpu_data.x86_clflush_size - 1));
993
994                 page_do_bit17_swizzling = obj_do_bit17_swizzling &&
995                         (page_to_phys(page) & (1 << 17)) != 0;
996
997                 ret = shmem_pwrite_fast(page, shmem_page_offset, page_length,
998                                         user_data, page_do_bit17_swizzling,
999                                         partial_cacheline_write,
1000                                         needs_clflush_after);
1001                 if (ret == 0)
1002                         goto next_page;
1003
1004                 hit_slowpath = 1;
1005                 mutex_unlock(&dev->struct_mutex);
1006                 ret = shmem_pwrite_slow(page, shmem_page_offset, page_length,
1007                                         user_data, page_do_bit17_swizzling,
1008                                         partial_cacheline_write,
1009                                         needs_clflush_after);
1010
1011                 mutex_lock(&dev->struct_mutex);
1012
1013                 if (ret)
1014                         goto out;
1015
1016 next_page:
1017                 remain -= page_length;
1018                 user_data += page_length;
1019                 offset += page_length;
1020         }
1021
1022 out:
1023         i915_gem_object_unpin_pages(obj);
1024
1025         if (hit_slowpath) {
1026                 /*
1027                  * Fixup: Flush cpu caches in case we didn't flush the dirty
1028                  * cachelines in-line while writing and the object moved
1029                  * out of the cpu write domain while we've dropped the lock.
1030                  */
1031                 if (!needs_clflush_after &&
1032                     obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
1033                         if (i915_gem_clflush_object(obj, obj->pin_display))
1034                                 i915_gem_chipset_flush(dev);
1035                 }
1036         }
1037
1038         if (needs_clflush_after)
1039                 i915_gem_chipset_flush(dev);
1040
1041         return ret;
1042 }
1043
1044 /**
1045  * Writes data to the object referenced by handle.
1046  *
1047  * On error, the contents of the buffer that were to be modified are undefined.
1048  */
1049 int
1050 i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
1051                       struct drm_file *file)
1052 {
1053         struct drm_i915_gem_pwrite *args = data;
1054         struct drm_i915_gem_object *obj;
1055         int ret;
1056
1057         if (args->size == 0)
1058                 return 0;
1059
1060         if (!access_ok(VERIFY_READ,
1061                        to_user_ptr(args->data_ptr),
1062                        args->size))
1063                 return -EFAULT;
1064
1065         if (likely(!i915.prefault_disable)) {
1066                 ret = fault_in_multipages_readable(to_user_ptr(args->data_ptr),
1067                                                    args->size);
1068                 if (ret)
1069                         return -EFAULT;
1070         }
1071
1072         ret = i915_mutex_lock_interruptible(dev);
1073         if (ret)
1074                 return ret;
1075
1076         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
1077         if (&obj->base == NULL) {
1078                 ret = -ENOENT;
1079                 goto unlock;
1080         }
1081
1082         /* Bounds check destination. */
1083         if (args->offset > obj->base.size ||
1084             args->size > obj->base.size - args->offset) {
1085                 ret = -EINVAL;
1086                 goto out;
1087         }
1088
1089         /* prime objects have no backing filp to GEM pread/pwrite
1090          * pages from.
1091          */
1092         if (!obj->base.filp) {
1093                 ret = -EINVAL;
1094                 goto out;
1095         }
1096
1097         trace_i915_gem_object_pwrite(obj, args->offset, args->size);
1098
1099         ret = -EFAULT;
1100         /* We can only do the GTT pwrite on untiled buffers, as otherwise
1101          * it would end up going through the fenced access, and we'll get
1102          * different detiling behavior between reading and writing.
1103          * pread/pwrite currently are reading and writing from the CPU
1104          * perspective, requiring manual detiling by the client.
1105          */
1106         if (obj->tiling_mode == I915_TILING_NONE &&
1107             obj->base.write_domain != I915_GEM_DOMAIN_CPU &&
1108             cpu_write_needs_clflush(obj)) {
1109                 ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file);
1110                 /* Note that the gtt paths might fail with non-page-backed user
1111                  * pointers (e.g. gtt mappings when moving data between
1112                  * textures). Fallback to the shmem path in that case. */
1113         }
1114
1115         if (ret == -EFAULT || ret == -ENOSPC) {
1116                 if (obj->phys_handle)
1117                         ret = i915_gem_phys_pwrite(obj, args, file);
1118                 else
1119                         ret = i915_gem_shmem_pwrite(dev, obj, args, file);
1120         }
1121
1122 out:
1123         drm_gem_object_unreference(&obj->base);
1124 unlock:
1125         mutex_unlock(&dev->struct_mutex);
1126         return ret;
1127 }
1128
1129 int
1130 i915_gem_check_wedge(struct i915_gpu_error *error,
1131                      bool interruptible)
1132 {
1133         if (i915_reset_in_progress(error)) {
1134                 /* Non-interruptible callers can't handle -EAGAIN, hence return
1135                  * -EIO unconditionally for these. */
1136                 if (!interruptible)
1137                         return -EIO;
1138
1139                 /* Recovery complete, but the reset failed ... */
1140                 if (i915_terminally_wedged(error))
1141                         return -EIO;
1142
1143                 /*
1144                  * Check if GPU Reset is in progress - we need intel_ring_begin
1145                  * to work properly to reinit the hw state while the gpu is
1146                  * still marked as reset-in-progress. Handle this with a flag.
1147                  */
1148                 if (!error->reload_in_reset)
1149                         return -EAGAIN;
1150         }
1151
1152         return 0;
1153 }
1154
1155 /*
1156  * Compare arbitrary request against outstanding lazy request. Emit on match.
1157  */
1158 int
1159 i915_gem_check_olr(struct drm_i915_gem_request *req)
1160 {
1161         int ret;
1162
1163         WARN_ON(!mutex_is_locked(&req->ring->dev->struct_mutex));
1164
1165         ret = 0;
1166         if (req == req->ring->outstanding_lazy_request)
1167                 ret = i915_add_request(req->ring);
1168
1169         return ret;
1170 }
1171
1172 static void fake_irq(unsigned long data)
1173 {
1174         wake_up_process((struct task_struct *)data);
1175 }
1176
1177 static bool missed_irq(struct drm_i915_private *dev_priv,
1178                        struct intel_engine_cs *ring)
1179 {
1180         return test_bit(ring->id, &dev_priv->gpu_error.missed_irq_rings);
1181 }
1182
1183 static bool can_wait_boost(struct drm_i915_file_private *file_priv)
1184 {
1185         if (file_priv == NULL)
1186                 return true;
1187
1188         return !atomic_xchg(&file_priv->rps_wait_boost, true);
1189 }
1190
1191 /**
1192  * __i915_wait_request - wait until execution of request has finished
1193  * @req: duh!
1194  * @reset_counter: reset sequence associated with the given request
1195  * @interruptible: do an interruptible wait (normally yes)
1196  * @timeout: in - how long to wait (NULL forever); out - how much time remaining
1197  *
1198  * Note: It is of utmost importance that the passed in seqno and reset_counter
1199  * values have been read by the caller in an smp safe manner. Where read-side
1200  * locks are involved, it is sufficient to read the reset_counter before
1201  * unlocking the lock that protects the seqno. For lockless tricks, the
1202  * reset_counter _must_ be read before, and an appropriate smp_rmb must be
1203  * inserted.
1204  *
1205  * Returns 0 if the request was found within the alloted time. Else returns the
1206  * errno with remaining time filled in timeout argument.
1207  */
1208 int __i915_wait_request(struct drm_i915_gem_request *req,
1209                         unsigned reset_counter,
1210                         bool interruptible,
1211                         s64 *timeout,
1212                         struct drm_i915_file_private *file_priv)
1213 {
1214         struct intel_engine_cs *ring = i915_gem_request_get_ring(req);
1215         struct drm_device *dev = ring->dev;
1216         struct drm_i915_private *dev_priv = dev->dev_private;
1217         const bool irq_test_in_progress =
1218                 ACCESS_ONCE(dev_priv->gpu_error.test_irq_rings) & intel_ring_flag(ring);
1219         DEFINE_WAIT(wait);
1220         unsigned long timeout_expire;
1221         s64 before, now;
1222         int ret;
1223
1224         WARN(!intel_irqs_enabled(dev_priv), "IRQs disabled");
1225
1226         if (i915_gem_request_completed(req, true))
1227                 return 0;
1228
1229         timeout_expire = timeout ? jiffies + nsecs_to_jiffies((u64)*timeout) : 0;
1230
1231         if (INTEL_INFO(dev)->gen >= 6 && ring->id == RCS && can_wait_boost(file_priv)) {
1232                 gen6_rps_boost(dev_priv);
1233                 if (file_priv)
1234                         mod_delayed_work(dev_priv->wq,
1235                                          &file_priv->mm.idle_work,
1236                                          msecs_to_jiffies(100));
1237         }
1238
1239         if (!irq_test_in_progress && WARN_ON(!ring->irq_get(ring)))
1240                 return -ENODEV;
1241
1242         /* Record current time in case interrupted by signal, or wedged */
1243         trace_i915_gem_request_wait_begin(req);
1244         before = ktime_get_raw_ns();
1245         for (;;) {
1246                 struct timer_list timer;
1247
1248                 prepare_to_wait(&ring->irq_queue, &wait,
1249                                 interruptible ? TASK_INTERRUPTIBLE : TASK_UNINTERRUPTIBLE);
1250
1251                 /* We need to check whether any gpu reset happened in between
1252                  * the caller grabbing the seqno and now ... */
1253                 if (reset_counter != atomic_read(&dev_priv->gpu_error.reset_counter)) {
1254                         /* ... but upgrade the -EAGAIN to an -EIO if the gpu
1255                          * is truely gone. */
1256                         ret = i915_gem_check_wedge(&dev_priv->gpu_error, interruptible);
1257                         if (ret == 0)
1258                                 ret = -EAGAIN;
1259                         break;
1260                 }
1261
1262                 if (i915_gem_request_completed(req, false)) {
1263                         ret = 0;
1264                         break;
1265                 }
1266
1267                 if (interruptible && signal_pending(current)) {
1268                         ret = -ERESTARTSYS;
1269                         break;
1270                 }
1271
1272                 if (timeout && time_after_eq(jiffies, timeout_expire)) {
1273                         ret = -ETIME;
1274                         break;
1275                 }
1276
1277                 timer.function = NULL;
1278                 if (timeout || missed_irq(dev_priv, ring)) {
1279                         unsigned long expire;
1280
1281                         setup_timer_on_stack(&timer, fake_irq, (unsigned long)current);
1282                         expire = missed_irq(dev_priv, ring) ? jiffies + 1 : timeout_expire;
1283                         mod_timer(&timer, expire);
1284                 }
1285
1286                 io_schedule();
1287
1288                 if (timer.function) {
1289                         del_singleshot_timer_sync(&timer);
1290                         destroy_timer_on_stack(&timer);
1291                 }
1292         }
1293         now = ktime_get_raw_ns();
1294         trace_i915_gem_request_wait_end(req);
1295
1296         if (!irq_test_in_progress)
1297                 ring->irq_put(ring);
1298
1299         finish_wait(&ring->irq_queue, &wait);
1300
1301         if (timeout) {
1302                 s64 tres = *timeout - (now - before);
1303
1304                 *timeout = tres < 0 ? 0 : tres;
1305         }
1306
1307         return ret;
1308 }
1309
1310 /**
1311  * Waits for a request to be signaled, and cleans up the
1312  * request and object lists appropriately for that event.
1313  */
1314 int
1315 i915_wait_request(struct drm_i915_gem_request *req)
1316 {
1317         struct drm_device *dev;
1318         struct drm_i915_private *dev_priv;
1319         bool interruptible;
1320         unsigned reset_counter;
1321         int ret;
1322
1323         BUG_ON(req == NULL);
1324
1325         dev = req->ring->dev;
1326         dev_priv = dev->dev_private;
1327         interruptible = dev_priv->mm.interruptible;
1328
1329         BUG_ON(!mutex_is_locked(&dev->struct_mutex));
1330
1331         ret = i915_gem_check_wedge(&dev_priv->gpu_error, interruptible);
1332         if (ret)
1333                 return ret;
1334
1335         ret = i915_gem_check_olr(req);
1336         if (ret)
1337                 return ret;
1338
1339         reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
1340         i915_gem_request_reference(req);
1341         ret = __i915_wait_request(req, reset_counter,
1342                                   interruptible, NULL, NULL);
1343         i915_gem_request_unreference(req);
1344         return ret;
1345 }
1346
1347 static int
1348 i915_gem_object_wait_rendering__tail(struct drm_i915_gem_object *obj)
1349 {
1350         if (!obj->active)
1351                 return 0;
1352
1353         /* Manually manage the write flush as we may have not yet
1354          * retired the buffer.
1355          *
1356          * Note that the last_write_req is always the earlier of
1357          * the two (read/write) requests, so if we haved successfully waited,
1358          * we know we have passed the last write.
1359          */
1360         i915_gem_request_assign(&obj->last_write_req, NULL);
1361
1362         return 0;
1363 }
1364
1365 /**
1366  * Ensures that all rendering to the object has completed and the object is
1367  * safe to unbind from the GTT or access from the CPU.
1368  */
1369 static __must_check int
1370 i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj,
1371                                bool readonly)
1372 {
1373         struct drm_i915_gem_request *req;
1374         int ret;
1375
1376         req = readonly ? obj->last_write_req : obj->last_read_req;
1377         if (!req)
1378                 return 0;
1379
1380         ret = i915_wait_request(req);
1381         if (ret)
1382                 return ret;
1383
1384         return i915_gem_object_wait_rendering__tail(obj);
1385 }
1386
1387 /* A nonblocking variant of the above wait. This is a highly dangerous routine
1388  * as the object state may change during this call.
1389  */
1390 static __must_check int
1391 i915_gem_object_wait_rendering__nonblocking(struct drm_i915_gem_object *obj,
1392                                             struct drm_i915_file_private *file_priv,
1393                                             bool readonly)
1394 {
1395         struct drm_i915_gem_request *req;
1396         struct drm_device *dev = obj->base.dev;
1397         struct drm_i915_private *dev_priv = dev->dev_private;
1398         unsigned reset_counter;
1399         int ret;
1400
1401         BUG_ON(!mutex_is_locked(&dev->struct_mutex));
1402         BUG_ON(!dev_priv->mm.interruptible);
1403
1404         req = readonly ? obj->last_write_req : obj->last_read_req;
1405         if (!req)
1406                 return 0;
1407
1408         ret = i915_gem_check_wedge(&dev_priv->gpu_error, true);
1409         if (ret)
1410                 return ret;
1411
1412         ret = i915_gem_check_olr(req);
1413         if (ret)
1414                 return ret;
1415
1416         reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
1417         i915_gem_request_reference(req);
1418         mutex_unlock(&dev->struct_mutex);
1419         ret = __i915_wait_request(req, reset_counter, true, NULL, file_priv);
1420         mutex_lock(&dev->struct_mutex);
1421         i915_gem_request_unreference(req);
1422         if (ret)
1423                 return ret;
1424
1425         return i915_gem_object_wait_rendering__tail(obj);
1426 }
1427
1428 /**
1429  * Called when user space prepares to use an object with the CPU, either
1430  * through the mmap ioctl's mapping or a GTT mapping.
1431  */
1432 int
1433 i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
1434                           struct drm_file *file)
1435 {
1436         struct drm_i915_gem_set_domain *args = data;
1437         struct drm_i915_gem_object *obj;
1438         uint32_t read_domains = args->read_domains;
1439         uint32_t write_domain = args->write_domain;
1440         int ret;
1441
1442         /* Only handle setting domains to types used by the CPU. */
1443         if (write_domain & I915_GEM_GPU_DOMAINS)
1444                 return -EINVAL;
1445
1446         if (read_domains & I915_GEM_GPU_DOMAINS)
1447                 return -EINVAL;
1448
1449         /* Having something in the write domain implies it's in the read
1450          * domain, and only that read domain.  Enforce that in the request.
1451          */
1452         if (write_domain != 0 && read_domains != write_domain)
1453                 return -EINVAL;
1454
1455         ret = i915_mutex_lock_interruptible(dev);
1456         if (ret)
1457                 return ret;
1458
1459         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
1460         if (&obj->base == NULL) {
1461                 ret = -ENOENT;
1462                 goto unlock;
1463         }
1464
1465         /* Try to flush the object off the GPU without holding the lock.
1466          * We will repeat the flush holding the lock in the normal manner
1467          * to catch cases where we are gazumped.
1468          */
1469         ret = i915_gem_object_wait_rendering__nonblocking(obj,
1470                                                           file->driver_priv,
1471                                                           !write_domain);
1472         if (ret)
1473                 goto unref;
1474
1475         if (read_domains & I915_GEM_DOMAIN_GTT) {
1476                 ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
1477
1478                 /* Silently promote "you're not bound, there was nothing to do"
1479                  * to success, since the client was just asking us to
1480                  * make sure everything was done.
1481                  */
1482                 if (ret == -EINVAL)
1483                         ret = 0;
1484         } else {
1485                 ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
1486         }
1487
1488 unref:
1489         drm_gem_object_unreference(&obj->base);
1490 unlock:
1491         mutex_unlock(&dev->struct_mutex);
1492         return ret;
1493 }
1494
1495 /**
1496  * Called when user space has done writes to this buffer
1497  */
1498 int
1499 i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
1500                          struct drm_file *file)
1501 {
1502         struct drm_i915_gem_sw_finish *args = data;
1503         struct drm_i915_gem_object *obj;
1504         int ret = 0;
1505
1506         ret = i915_mutex_lock_interruptible(dev);
1507         if (ret)
1508                 return ret;
1509
1510         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
1511         if (&obj->base == NULL) {
1512                 ret = -ENOENT;
1513                 goto unlock;
1514         }
1515
1516         /* Pinned buffers may be scanout, so flush the cache */
1517         if (obj->pin_display)
1518                 i915_gem_object_flush_cpu_write_domain(obj, true);
1519
1520         drm_gem_object_unreference(&obj->base);
1521 unlock:
1522         mutex_unlock(&dev->struct_mutex);
1523         return ret;
1524 }
1525
1526 /**
1527  * Maps the contents of an object, returning the address it is mapped
1528  * into.
1529  *
1530  * While the mapping holds a reference on the contents of the object, it doesn't
1531  * imply a ref on the object itself.
1532  *
1533  * IMPORTANT:
1534  *
1535  * DRM driver writers who look a this function as an example for how to do GEM
1536  * mmap support, please don't implement mmap support like here. The modern way
1537  * to implement DRM mmap support is with an mmap offset ioctl (like
1538  * i915_gem_mmap_gtt) and then using the mmap syscall on the DRM fd directly.
1539  * That way debug tooling like valgrind will understand what's going on, hiding
1540  * the mmap call in a driver private ioctl will break that. The i915 driver only
1541  * does cpu mmaps this way because we didn't know better.
1542  */
1543 int
1544 i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
1545                     struct drm_file *file)
1546 {
1547         struct drm_i915_gem_mmap *args = data;
1548         struct drm_gem_object *obj;
1549         unsigned long addr;
1550
1551         obj = drm_gem_object_lookup(dev, file, args->handle);
1552         if (obj == NULL)
1553                 return -ENOENT;
1554
1555         /* prime objects have no backing filp to GEM mmap
1556          * pages from.
1557          */
1558         if (!obj->filp) {
1559                 drm_gem_object_unreference_unlocked(obj);
1560                 return -EINVAL;
1561         }
1562
1563         addr = vm_mmap(obj->filp, 0, args->size,
1564                        PROT_READ | PROT_WRITE, MAP_SHARED,
1565                        args->offset);
1566         drm_gem_object_unreference_unlocked(obj);
1567         if (IS_ERR((void *)addr))
1568                 return addr;
1569
1570         args->addr_ptr = (uint64_t) addr;
1571
1572         return 0;
1573 }
1574
1575 /**
1576  * i915_gem_fault - fault a page into the GTT
1577  * vma: VMA in question
1578  * vmf: fault info
1579  *
1580  * The fault handler is set up by drm_gem_mmap() when a object is GTT mapped
1581  * from userspace.  The fault handler takes care of binding the object to
1582  * the GTT (if needed), allocating and programming a fence register (again,
1583  * only if needed based on whether the old reg is still valid or the object
1584  * is tiled) and inserting a new PTE into the faulting process.
1585  *
1586  * Note that the faulting process may involve evicting existing objects
1587  * from the GTT and/or fence registers to make room.  So performance may
1588  * suffer if the GTT working set is large or there are few fence registers
1589  * left.
1590  */
1591 int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1592 {
1593         struct drm_i915_gem_object *obj = to_intel_bo(vma->vm_private_data);
1594         struct drm_device *dev = obj->base.dev;
1595         struct drm_i915_private *dev_priv = dev->dev_private;
1596         pgoff_t page_offset;
1597         unsigned long pfn;
1598         int ret = 0;
1599         bool write = !!(vmf->flags & FAULT_FLAG_WRITE);
1600
1601         intel_runtime_pm_get(dev_priv);
1602
1603         /* We don't use vmf->pgoff since that has the fake offset */
1604         page_offset = ((unsigned long)vmf->virtual_address - vma->vm_start) >>
1605                 PAGE_SHIFT;
1606
1607         ret = i915_mutex_lock_interruptible(dev);
1608         if (ret)
1609                 goto out;
1610
1611         trace_i915_gem_object_fault(obj, page_offset, true, write);
1612
1613         /* Try to flush the object off the GPU first without holding the lock.
1614          * Upon reacquiring the lock, we will perform our sanity checks and then
1615          * repeat the flush holding the lock in the normal manner to catch cases
1616          * where we are gazumped.
1617          */
1618         ret = i915_gem_object_wait_rendering__nonblocking(obj, NULL, !write);
1619         if (ret)
1620                 goto unlock;
1621
1622         /* Access to snoopable pages through the GTT is incoherent. */
1623         if (obj->cache_level != I915_CACHE_NONE && !HAS_LLC(dev)) {
1624                 ret = -EFAULT;
1625                 goto unlock;
1626         }
1627
1628         /* Now bind it into the GTT if needed */
1629         ret = i915_gem_obj_ggtt_pin(obj, 0, PIN_MAPPABLE);
1630         if (ret)
1631                 goto unlock;
1632
1633         ret = i915_gem_object_set_to_gtt_domain(obj, write);
1634         if (ret)
1635                 goto unpin;
1636
1637         ret = i915_gem_object_get_fence(obj);
1638         if (ret)
1639                 goto unpin;
1640
1641         /* Finally, remap it using the new GTT offset */
1642         pfn = dev_priv->gtt.mappable_base + i915_gem_obj_ggtt_offset(obj);
1643         pfn >>= PAGE_SHIFT;
1644
1645         if (!obj->fault_mappable) {
1646                 unsigned long size = min_t(unsigned long,
1647                                            vma->vm_end - vma->vm_start,
1648                                            obj->base.size);
1649                 int i;
1650
1651                 for (i = 0; i < size >> PAGE_SHIFT; i++) {
1652                         ret = vm_insert_pfn(vma,
1653                                             (unsigned long)vma->vm_start + i * PAGE_SIZE,
1654                                             pfn + i);
1655                         if (ret)
1656                                 break;
1657                 }
1658
1659                 obj->fault_mappable = true;
1660         } else
1661                 ret = vm_insert_pfn(vma,
1662                                     (unsigned long)vmf->virtual_address,
1663                                     pfn + page_offset);
1664 unpin:
1665         i915_gem_object_ggtt_unpin(obj);
1666 unlock:
1667         mutex_unlock(&dev->struct_mutex);
1668 out:
1669         switch (ret) {
1670         case -EIO:
1671                 /*
1672                  * We eat errors when the gpu is terminally wedged to avoid
1673                  * userspace unduly crashing (gl has no provisions for mmaps to
1674                  * fail). But any other -EIO isn't ours (e.g. swap in failure)
1675                  * and so needs to be reported.
1676                  */
1677                 if (!i915_terminally_wedged(&dev_priv->gpu_error)) {
1678                         ret = VM_FAULT_SIGBUS;
1679                         break;
1680                 }
1681         case -EAGAIN:
1682                 /*
1683                  * EAGAIN means the gpu is hung and we'll wait for the error
1684                  * handler to reset everything when re-faulting in
1685                  * i915_mutex_lock_interruptible.
1686                  */
1687         case 0:
1688         case -ERESTARTSYS:
1689         case -EINTR:
1690         case -EBUSY:
1691                 /*
1692                  * EBUSY is ok: this just means that another thread
1693                  * already did the job.
1694                  */
1695                 ret = VM_FAULT_NOPAGE;
1696                 break;
1697         case -ENOMEM:
1698                 ret = VM_FAULT_OOM;
1699                 break;
1700         case -ENOSPC:
1701         case -EFAULT:
1702                 ret = VM_FAULT_SIGBUS;
1703                 break;
1704         default:
1705                 WARN_ONCE(ret, "unhandled error in i915_gem_fault: %i\n", ret);
1706                 ret = VM_FAULT_SIGBUS;
1707                 break;
1708         }
1709
1710         intel_runtime_pm_put(dev_priv);
1711         return ret;
1712 }
1713
1714 /**
1715  * i915_gem_release_mmap - remove physical page mappings
1716  * @obj: obj in question
1717  *
1718  * Preserve the reservation of the mmapping with the DRM core code, but
1719  * relinquish ownership of the pages back to the system.
1720  *
1721  * It is vital that we remove the page mapping if we have mapped a tiled
1722  * object through the GTT and then lose the fence register due to
1723  * resource pressure. Similarly if the object has been moved out of the
1724  * aperture, than pages mapped into userspace must be revoked. Removing the
1725  * mapping will then trigger a page fault on the next user access, allowing
1726  * fixup by i915_gem_fault().
1727  */
1728 void
1729 i915_gem_release_mmap(struct drm_i915_gem_object *obj)
1730 {
1731         if (!obj->fault_mappable)
1732                 return;
1733
1734         drm_vma_node_unmap(&obj->base.vma_node,
1735                            obj->base.dev->anon_inode->i_mapping);
1736         obj->fault_mappable = false;
1737 }
1738
1739 void
1740 i915_gem_release_all_mmaps(struct drm_i915_private *dev_priv)
1741 {
1742         struct drm_i915_gem_object *obj;
1743
1744         list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list)
1745                 i915_gem_release_mmap(obj);
1746 }
1747
1748 uint32_t
1749 i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, int tiling_mode)
1750 {
1751         uint32_t gtt_size;
1752
1753         if (INTEL_INFO(dev)->gen >= 4 ||
1754             tiling_mode == I915_TILING_NONE)
1755                 return size;
1756
1757         /* Previous chips need a power-of-two fence region when tiling */
1758         if (INTEL_INFO(dev)->gen == 3)
1759                 gtt_size = 1024*1024;
1760         else
1761                 gtt_size = 512*1024;
1762
1763         while (gtt_size < size)
1764                 gtt_size <<= 1;
1765
1766         return gtt_size;
1767 }
1768
1769 /**
1770  * i915_gem_get_gtt_alignment - return required GTT alignment for an object
1771  * @obj: object to check
1772  *
1773  * Return the required GTT alignment for an object, taking into account
1774  * potential fence register mapping.
1775  */
1776 uint32_t
1777 i915_gem_get_gtt_alignment(struct drm_device *dev, uint32_t size,
1778                            int tiling_mode, bool fenced)
1779 {
1780         /*
1781          * Minimum alignment is 4k (GTT page size), but might be greater
1782          * if a fence register is needed for the object.
1783          */
1784         if (INTEL_INFO(dev)->gen >= 4 || (!fenced && IS_G33(dev)) ||
1785             tiling_mode == I915_TILING_NONE)
1786                 return 4096;
1787
1788         /*
1789          * Previous chips need to be aligned to the size of the smallest
1790          * fence register that can contain the object.
1791          */
1792         return i915_gem_get_gtt_size(dev, size, tiling_mode);
1793 }
1794
1795 static int i915_gem_object_create_mmap_offset(struct drm_i915_gem_object *obj)
1796 {
1797         struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
1798         int ret;
1799
1800         if (drm_vma_node_has_offset(&obj->base.vma_node))
1801                 return 0;
1802
1803         dev_priv->mm.shrinker_no_lock_stealing = true;
1804
1805         ret = drm_gem_create_mmap_offset(&obj->base);
1806         if (ret != -ENOSPC)
1807                 goto out;
1808
1809         /* Badly fragmented mmap space? The only way we can recover
1810          * space is by destroying unwanted objects. We can't randomly release
1811          * mmap_offsets as userspace expects them to be persistent for the
1812          * lifetime of the objects. The closest we can is to release the
1813          * offsets on purgeable objects by truncating it and marking it purged,
1814          * which prevents userspace from ever using that object again.
1815          */
1816         i915_gem_shrink(dev_priv,
1817                         obj->base.size >> PAGE_SHIFT,
1818                         I915_SHRINK_BOUND |
1819                         I915_SHRINK_UNBOUND |
1820                         I915_SHRINK_PURGEABLE);
1821         ret = drm_gem_create_mmap_offset(&obj->base);
1822         if (ret != -ENOSPC)
1823                 goto out;
1824
1825         i915_gem_shrink_all(dev_priv);
1826         ret = drm_gem_create_mmap_offset(&obj->base);
1827 out:
1828         dev_priv->mm.shrinker_no_lock_stealing = false;
1829
1830         return ret;
1831 }
1832
1833 static void i915_gem_object_free_mmap_offset(struct drm_i915_gem_object *obj)
1834 {
1835         drm_gem_free_mmap_offset(&obj->base);
1836 }
1837
1838 static int
1839 i915_gem_mmap_gtt(struct drm_file *file,
1840                   struct drm_device *dev,
1841                   uint32_t handle, bool dumb,
1842                   uint64_t *offset)
1843 {
1844         struct drm_i915_private *dev_priv = dev->dev_private;
1845         struct drm_i915_gem_object *obj;
1846         int ret;
1847
1848         ret = i915_mutex_lock_interruptible(dev);
1849         if (ret)
1850                 return ret;
1851
1852         obj = to_intel_bo(drm_gem_object_lookup(dev, file, handle));
1853         if (&obj->base == NULL) {
1854                 ret = -ENOENT;
1855                 goto unlock;
1856         }
1857
1858         /*
1859          * We don't allow dumb mmaps on objects created using another
1860          * interface.
1861          */
1862         WARN_ONCE(dumb && !(obj->base.dumb || obj->base.import_attach),
1863                   "Illegal dumb map of accelerated buffer.\n");
1864
1865         if (obj->base.size > dev_priv->gtt.mappable_end) {
1866                 ret = -E2BIG;
1867                 goto out;
1868         }
1869
1870         if (obj->madv != I915_MADV_WILLNEED) {
1871                 DRM_DEBUG("Attempting to mmap a purgeable buffer\n");
1872                 ret = -EFAULT;
1873                 goto out;
1874         }
1875
1876         ret = i915_gem_object_create_mmap_offset(obj);
1877         if (ret)
1878                 goto out;
1879
1880         *offset = drm_vma_node_offset_addr(&obj->base.vma_node);
1881
1882 out:
1883         drm_gem_object_unreference(&obj->base);
1884 unlock:
1885         mutex_unlock(&dev->struct_mutex);
1886         return ret;
1887 }
1888
1889 int
1890 i915_gem_dumb_map_offset(struct drm_file *file,
1891                          struct drm_device *dev,
1892                          uint32_t handle,
1893                          uint64_t *offset)
1894 {
1895         return i915_gem_mmap_gtt(file, dev, handle, true, offset);
1896 }
1897
1898 /**
1899  * i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing
1900  * @dev: DRM device
1901  * @data: GTT mapping ioctl data
1902  * @file: GEM object info
1903  *
1904  * Simply returns the fake offset to userspace so it can mmap it.
1905  * The mmap call will end up in drm_gem_mmap(), which will set things
1906  * up so we can get faults in the handler above.
1907  *
1908  * The fault handler will take care of binding the object into the GTT
1909  * (since it may have been evicted to make room for something), allocating
1910  * a fence register, and mapping the appropriate aperture address into
1911  * userspace.
1912  */
1913 int
1914 i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
1915                         struct drm_file *file)
1916 {
1917         struct drm_i915_gem_mmap_gtt *args = data;
1918
1919         return i915_gem_mmap_gtt(file, dev, args->handle, false, &args->offset);
1920 }
1921
1922 static inline int
1923 i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj)
1924 {
1925         return obj->madv == I915_MADV_DONTNEED;
1926 }
1927
1928 /* Immediately discard the backing storage */
1929 static void
1930 i915_gem_object_truncate(struct drm_i915_gem_object *obj)
1931 {
1932         i915_gem_object_free_mmap_offset(obj);
1933
1934         if (obj->base.filp == NULL)
1935                 return;
1936
1937         /* Our goal here is to return as much of the memory as
1938          * is possible back to the system as we are called from OOM.
1939          * To do this we must instruct the shmfs to drop all of its
1940          * backing pages, *now*.
1941          */
1942         shmem_truncate_range(file_inode(obj->base.filp), 0, (loff_t)-1);
1943         obj->madv = __I915_MADV_PURGED;
1944 }
1945
1946 /* Try to discard unwanted pages */
1947 static void
1948 i915_gem_object_invalidate(struct drm_i915_gem_object *obj)
1949 {
1950         struct address_space *mapping;
1951
1952         switch (obj->madv) {
1953         case I915_MADV_DONTNEED:
1954                 i915_gem_object_truncate(obj);
1955         case __I915_MADV_PURGED:
1956                 return;
1957         }
1958
1959         if (obj->base.filp == NULL)
1960                 return;
1961
1962         mapping = file_inode(obj->base.filp)->i_mapping,
1963         invalidate_mapping_pages(mapping, 0, (loff_t)-1);
1964 }
1965
1966 static void
1967 i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj)
1968 {
1969         struct sg_page_iter sg_iter;
1970         int ret;
1971
1972         BUG_ON(obj->madv == __I915_MADV_PURGED);
1973
1974         ret = i915_gem_object_set_to_cpu_domain(obj, true);
1975         if (ret) {
1976                 /* In the event of a disaster, abandon all caches and
1977                  * hope for the best.
1978                  */
1979                 WARN_ON(ret != -EIO);
1980                 i915_gem_clflush_object(obj, true);
1981                 obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU;
1982         }
1983
1984         if (i915_gem_object_needs_bit17_swizzle(obj))
1985                 i915_gem_object_save_bit_17_swizzle(obj);
1986
1987         if (obj->madv == I915_MADV_DONTNEED)
1988                 obj->dirty = 0;
1989
1990         for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents, 0) {
1991                 struct page *page = sg_page_iter_page(&sg_iter);
1992
1993                 if (obj->dirty)
1994                         set_page_dirty(page);
1995
1996                 if (obj->madv == I915_MADV_WILLNEED)
1997                         mark_page_accessed(page);
1998
1999                 page_cache_release(page);
2000         }
2001         obj->dirty = 0;
2002
2003         sg_free_table(obj->pages);
2004         kfree(obj->pages);
2005 }
2006
2007 int
2008 i915_gem_object_put_pages(struct drm_i915_gem_object *obj)
2009 {
2010         const struct drm_i915_gem_object_ops *ops = obj->ops;
2011
2012         if (obj->pages == NULL)
2013                 return 0;
2014
2015         if (obj->pages_pin_count)
2016                 return -EBUSY;
2017
2018         BUG_ON(i915_gem_obj_bound_any(obj));
2019
2020         /* ->put_pages might need to allocate memory for the bit17 swizzle
2021          * array, hence protect them from being reaped by removing them from gtt
2022          * lists early. */
2023         list_del(&obj->global_list);
2024
2025         ops->put_pages(obj);
2026         obj->pages = NULL;
2027
2028         i915_gem_object_invalidate(obj);
2029
2030         return 0;
2031 }
2032
2033 unsigned long
2034 i915_gem_shrink(struct drm_i915_private *dev_priv,
2035                 long target, unsigned flags)
2036 {
2037         const struct {
2038                 struct list_head *list;
2039                 unsigned int bit;
2040         } phases[] = {
2041                 { &dev_priv->mm.unbound_list, I915_SHRINK_UNBOUND },
2042                 { &dev_priv->mm.bound_list, I915_SHRINK_BOUND },
2043                 { NULL, 0 },
2044         }, *phase;
2045         unsigned long count = 0;
2046
2047         /*
2048          * As we may completely rewrite the (un)bound list whilst unbinding
2049          * (due to retiring requests) we have to strictly process only
2050          * one element of the list at the time, and recheck the list
2051          * on every iteration.
2052          *
2053          * In particular, we must hold a reference whilst removing the
2054          * object as we may end up waiting for and/or retiring the objects.
2055          * This might release the final reference (held by the active list)
2056          * and result in the object being freed from under us. This is
2057          * similar to the precautions the eviction code must take whilst
2058          * removing objects.
2059          *
2060          * Also note that although these lists do not hold a reference to
2061          * the object we can safely grab one here: The final object
2062          * unreferencing and the bound_list are both protected by the
2063          * dev->struct_mutex and so we won't ever be able to observe an
2064          * object on the bound_list with a reference count equals 0.
2065          */
2066         for (phase = phases; phase->list; phase++) {
2067                 struct list_head still_in_list;
2068
2069                 if ((flags & phase->bit) == 0)
2070                         continue;
2071
2072                 INIT_LIST_HEAD(&still_in_list);
2073                 while (count < target && !list_empty(phase->list)) {
2074                         struct drm_i915_gem_object *obj;
2075                         struct i915_vma *vma, *v;
2076
2077                         obj = list_first_entry(phase->list,
2078                                                typeof(*obj), global_list);
2079                         list_move_tail(&obj->global_list, &still_in_list);
2080
2081                         if (flags & I915_SHRINK_PURGEABLE &&
2082                             !i915_gem_object_is_purgeable(obj))
2083                                 continue;
2084
2085                         drm_gem_object_reference(&obj->base);
2086
2087                         /* For the unbound phase, this should be a no-op! */
2088                         list_for_each_entry_safe(vma, v,
2089                                                  &obj->vma_list, vma_link)
2090                                 if (i915_vma_unbind(vma))
2091                                         break;
2092
2093                         if (i915_gem_object_put_pages(obj) == 0)
2094                                 count += obj->base.size >> PAGE_SHIFT;
2095
2096                         drm_gem_object_unreference(&obj->base);
2097                 }
2098                 list_splice(&still_in_list, phase->list);
2099         }
2100
2101         return count;
2102 }
2103
2104 static unsigned long
2105 i915_gem_shrink_all(struct drm_i915_private *dev_priv)
2106 {
2107         i915_gem_evict_everything(dev_priv->dev);
2108         return i915_gem_shrink(dev_priv, LONG_MAX,
2109                                I915_SHRINK_BOUND | I915_SHRINK_UNBOUND);
2110 }
2111
2112 static int
2113 i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj)
2114 {
2115         struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
2116         int page_count, i;
2117         struct address_space *mapping;
2118         struct sg_table *st;
2119         struct scatterlist *sg;
2120         struct sg_page_iter sg_iter;
2121         struct page *page;
2122         unsigned long last_pfn = 0;     /* suppress gcc warning */
2123         gfp_t gfp;
2124
2125         /* Assert that the object is not currently in any GPU domain. As it
2126          * wasn't in the GTT, there shouldn't be any way it could have been in
2127          * a GPU cache
2128          */
2129         BUG_ON(obj->base.read_domains & I915_GEM_GPU_DOMAINS);
2130         BUG_ON(obj->base.write_domain & I915_GEM_GPU_DOMAINS);
2131
2132         st = kmalloc(sizeof(*st), GFP_KERNEL);
2133         if (st == NULL)
2134                 return -ENOMEM;
2135
2136         page_count = obj->base.size / PAGE_SIZE;
2137         if (sg_alloc_table(st, page_count, GFP_KERNEL)) {
2138                 kfree(st);
2139                 return -ENOMEM;
2140         }
2141
2142         /* Get the list of pages out of our struct file.  They'll be pinned
2143          * at this point until we release them.
2144          *
2145          * Fail silently without starting the shrinker
2146          */
2147         mapping = file_inode(obj->base.filp)->i_mapping;
2148         gfp = mapping_gfp_mask(mapping);
2149         gfp |= __GFP_NORETRY | __GFP_NOWARN | __GFP_NO_KSWAPD;
2150         gfp &= ~(__GFP_IO | __GFP_WAIT);
2151         sg = st->sgl;
2152         st->nents = 0;
2153         for (i = 0; i < page_count; i++) {
2154                 page = shmem_read_mapping_page_gfp(mapping, i, gfp);
2155                 if (IS_ERR(page)) {
2156                         i915_gem_shrink(dev_priv,
2157                                         page_count,
2158                                         I915_SHRINK_BOUND |
2159                                         I915_SHRINK_UNBOUND |
2160                                         I915_SHRINK_PURGEABLE);
2161                         page = shmem_read_mapping_page_gfp(mapping, i, gfp);
2162                 }
2163                 if (IS_ERR(page)) {
2164                         /* We've tried hard to allocate the memory by reaping
2165                          * our own buffer, now let the real VM do its job and
2166                          * go down in flames if truly OOM.
2167                          */
2168                         i915_gem_shrink_all(dev_priv);
2169                         page = shmem_read_mapping_page(mapping, i);
2170                         if (IS_ERR(page))
2171                                 goto err_pages;
2172                 }
2173 #ifdef CONFIG_SWIOTLB
2174                 if (swiotlb_nr_tbl()) {
2175                         st->nents++;
2176                         sg_set_page(sg, page, PAGE_SIZE, 0);
2177                         sg = sg_next(sg);
2178                         continue;
2179                 }
2180 #endif
2181                 if (!i || page_to_pfn(page) != last_pfn + 1) {
2182                         if (i)
2183                                 sg = sg_next(sg);
2184                         st->nents++;
2185                         sg_set_page(sg, page, PAGE_SIZE, 0);
2186                 } else {
2187                         sg->length += PAGE_SIZE;
2188                 }
2189                 last_pfn = page_to_pfn(page);
2190
2191                 /* Check that the i965g/gm workaround works. */
2192                 WARN_ON((gfp & __GFP_DMA32) && (last_pfn >= 0x00100000UL));
2193         }
2194 #ifdef CONFIG_SWIOTLB
2195         if (!swiotlb_nr_tbl())
2196 #endif
2197                 sg_mark_end(sg);
2198         obj->pages = st;
2199
2200         if (i915_gem_object_needs_bit17_swizzle(obj))
2201                 i915_gem_object_do_bit_17_swizzle(obj);
2202
2203         if (obj->tiling_mode != I915_TILING_NONE &&
2204             dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES)
2205                 i915_gem_object_pin_pages(obj);
2206
2207         return 0;
2208
2209 err_pages:
2210         sg_mark_end(sg);
2211         for_each_sg_page(st->sgl, &sg_iter, st->nents, 0)
2212                 page_cache_release(sg_page_iter_page(&sg_iter));
2213         sg_free_table(st);
2214         kfree(st);
2215
2216         /* shmemfs first checks if there is enough memory to allocate the page
2217          * and reports ENOSPC should there be insufficient, along with the usual
2218          * ENOMEM for a genuine allocation failure.
2219          *
2220          * We use ENOSPC in our driver to mean that we have run out of aperture
2221          * space and so want to translate the error from shmemfs back to our
2222          * usual understanding of ENOMEM.
2223          */
2224         if (PTR_ERR(page) == -ENOSPC)
2225                 return -ENOMEM;
2226         else
2227                 return PTR_ERR(page);
2228 }
2229
2230 /* Ensure that the associated pages are gathered from the backing storage
2231  * and pinned into our object. i915_gem_object_get_pages() may be called
2232  * multiple times before they are released by a single call to
2233  * i915_gem_object_put_pages() - once the pages are no longer referenced
2234  * either as a result of memory pressure (reaping pages under the shrinker)
2235  * or as the object is itself released.
2236  */
2237 int
2238 i915_gem_object_get_pages(struct drm_i915_gem_object *obj)
2239 {
2240         struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
2241         const struct drm_i915_gem_object_ops *ops = obj->ops;
2242         int ret;
2243
2244         if (obj->pages)
2245                 return 0;
2246
2247         if (obj->madv != I915_MADV_WILLNEED) {
2248                 DRM_DEBUG("Attempting to obtain a purgeable object\n");
2249                 return -EFAULT;
2250         }
2251
2252         BUG_ON(obj->pages_pin_count);
2253
2254         ret = ops->get_pages(obj);
2255         if (ret)
2256                 return ret;
2257
2258         list_add_tail(&obj->global_list, &dev_priv->mm.unbound_list);
2259         return 0;
2260 }
2261
2262 static void
2263 i915_gem_object_move_to_active(struct drm_i915_gem_object *obj,
2264                                struct intel_engine_cs *ring)
2265 {
2266         struct drm_i915_gem_request *req;
2267         struct intel_engine_cs *old_ring;
2268
2269         BUG_ON(ring == NULL);
2270
2271         req = intel_ring_get_request(ring);
2272         old_ring = i915_gem_request_get_ring(obj->last_read_req);
2273
2274         if (old_ring != ring && obj->last_write_req) {
2275                 /* Keep the request relative to the current ring */
2276                 i915_gem_request_assign(&obj->last_write_req, req);
2277         }
2278
2279         /* Add a reference if we're newly entering the active list. */
2280         if (!obj->active) {
2281                 drm_gem_object_reference(&obj->base);
2282                 obj->active = 1;
2283         }
2284
2285         list_move_tail(&obj->ring_list, &ring->active_list);
2286
2287         i915_gem_request_assign(&obj->last_read_req, req);
2288 }
2289
2290 void i915_vma_move_to_active(struct i915_vma *vma,
2291                              struct intel_engine_cs *ring)
2292 {
2293         list_move_tail(&vma->mm_list, &vma->vm->active_list);
2294         return i915_gem_object_move_to_active(vma->obj, ring);
2295 }
2296
2297 static void
2298 i915_gem_object_move_to_inactive(struct drm_i915_gem_object *obj)
2299 {
2300         struct i915_vma *vma;
2301
2302         BUG_ON(obj->base.write_domain & ~I915_GEM_GPU_DOMAINS);
2303         BUG_ON(!obj->active);
2304
2305         list_for_each_entry(vma, &obj->vma_list, vma_link) {
2306                 if (!list_empty(&vma->mm_list))
2307                         list_move_tail(&vma->mm_list, &vma->vm->inactive_list);
2308         }
2309
2310         intel_fb_obj_flush(obj, true);
2311
2312         list_del_init(&obj->ring_list);
2313
2314         i915_gem_request_assign(&obj->last_read_req, NULL);
2315         i915_gem_request_assign(&obj->last_write_req, NULL);
2316         obj->base.write_domain = 0;
2317
2318         i915_gem_request_assign(&obj->last_fenced_req, NULL);
2319
2320         obj->active = 0;
2321         drm_gem_object_unreference(&obj->base);
2322
2323         WARN_ON(i915_verify_lists(dev));
2324 }
2325
2326 static void
2327 i915_gem_object_retire(struct drm_i915_gem_object *obj)
2328 {
2329         if (obj->last_read_req == NULL)
2330                 return;
2331
2332         if (i915_gem_request_completed(obj->last_read_req, true))
2333                 i915_gem_object_move_to_inactive(obj);
2334 }
2335
2336 static int
2337 i915_gem_init_seqno(struct drm_device *dev, u32 seqno)
2338 {
2339         struct drm_i915_private *dev_priv = dev->dev_private;
2340         struct intel_engine_cs *ring;
2341         int ret, i, j;
2342
2343         /* Carefully retire all requests without writing to the rings */
2344         for_each_ring(ring, dev_priv, i) {
2345                 ret = intel_ring_idle(ring);
2346                 if (ret)
2347                         return ret;
2348         }
2349         i915_gem_retire_requests(dev);
2350
2351         /* Finally reset hw state */
2352         for_each_ring(ring, dev_priv, i) {
2353                 intel_ring_init_seqno(ring, seqno);
2354
2355                 for (j = 0; j < ARRAY_SIZE(ring->semaphore.sync_seqno); j++)
2356                         ring->semaphore.sync_seqno[j] = 0;
2357         }
2358
2359         return 0;
2360 }
2361
2362 int i915_gem_set_seqno(struct drm_device *dev, u32 seqno)
2363 {
2364         struct drm_i915_private *dev_priv = dev->dev_private;
2365         int ret;
2366
2367         if (seqno == 0)
2368                 return -EINVAL;
2369
2370         /* HWS page needs to be set less than what we
2371          * will inject to ring
2372          */
2373         ret = i915_gem_init_seqno(dev, seqno - 1);
2374         if (ret)
2375                 return ret;
2376
2377         /* Carefully set the last_seqno value so that wrap
2378          * detection still works
2379          */
2380         dev_priv->next_seqno = seqno;
2381         dev_priv->last_seqno = seqno - 1;
2382         if (dev_priv->last_seqno == 0)
2383                 dev_priv->last_seqno--;
2384
2385         return 0;
2386 }
2387
2388 int
2389 i915_gem_get_seqno(struct drm_device *dev, u32 *seqno)
2390 {
2391         struct drm_i915_private *dev_priv = dev->dev_private;
2392
2393         /* reserve 0 for non-seqno */
2394         if (dev_priv->next_seqno == 0) {
2395                 int ret = i915_gem_init_seqno(dev, 0);
2396                 if (ret)
2397                         return ret;
2398
2399                 dev_priv->next_seqno = 1;
2400         }
2401
2402         *seqno = dev_priv->last_seqno = dev_priv->next_seqno++;
2403         return 0;
2404 }
2405
2406 int __i915_add_request(struct intel_engine_cs *ring,
2407                        struct drm_file *file,
2408                        struct drm_i915_gem_object *obj)
2409 {
2410         struct drm_i915_private *dev_priv = ring->dev->dev_private;
2411         struct drm_i915_gem_request *request;
2412         struct intel_ringbuffer *ringbuf;
2413         u32 request_ring_position, request_start;
2414         int ret;
2415
2416         request = ring->outstanding_lazy_request;
2417         if (WARN_ON(request == NULL))
2418                 return -ENOMEM;
2419
2420         if (i915.enable_execlists) {
2421                 struct intel_context *ctx = request->ctx;
2422                 ringbuf = ctx->engine[ring->id].ringbuf;
2423         } else
2424                 ringbuf = ring->buffer;
2425
2426         request_start = intel_ring_get_tail(ringbuf);
2427         /*
2428          * Emit any outstanding flushes - execbuf can fail to emit the flush
2429          * after having emitted the batchbuffer command. Hence we need to fix
2430          * things up similar to emitting the lazy request. The difference here
2431          * is that the flush _must_ happen before the next request, no matter
2432          * what.
2433          */
2434         if (i915.enable_execlists) {
2435                 ret = logical_ring_flush_all_caches(ringbuf);
2436                 if (ret)
2437                         return ret;
2438         } else {
2439                 ret = intel_ring_flush_all_caches(ring);
2440                 if (ret)
2441                         return ret;
2442         }
2443
2444         /* Record the position of the start of the request so that
2445          * should we detect the updated seqno part-way through the
2446          * GPU processing the request, we never over-estimate the
2447          * position of the head.
2448          */
2449         request_ring_position = intel_ring_get_tail(ringbuf);
2450
2451         if (i915.enable_execlists) {
2452                 ret = ring->emit_request(ringbuf);
2453                 if (ret)
2454                         return ret;
2455         } else {
2456                 ret = ring->add_request(ring);
2457                 if (ret)
2458                         return ret;
2459         }
2460
2461         request->head = request_start;
2462         request->tail = request_ring_position;
2463
2464         /* Whilst this request exists, batch_obj will be on the
2465          * active_list, and so will hold the active reference. Only when this
2466          * request is retired will the the batch_obj be moved onto the
2467          * inactive_list and lose its active reference. Hence we do not need
2468          * to explicitly hold another reference here.
2469          */
2470         request->batch_obj = obj;
2471
2472         if (!i915.enable_execlists) {
2473                 /* Hold a reference to the current context so that we can inspect
2474                  * it later in case a hangcheck error event fires.
2475                  */
2476                 request->ctx = ring->last_context;
2477                 if (request->ctx)
2478                         i915_gem_context_reference(request->ctx);
2479         }
2480
2481         request->emitted_jiffies = jiffies;
2482         list_add_tail(&request->list, &ring->request_list);
2483         request->file_priv = NULL;
2484
2485         if (file) {
2486                 struct drm_i915_file_private *file_priv = file->driver_priv;
2487
2488                 spin_lock(&file_priv->mm.lock);
2489                 request->file_priv = file_priv;
2490                 list_add_tail(&request->client_list,
2491                               &file_priv->mm.request_list);
2492                 spin_unlock(&file_priv->mm.lock);
2493         }
2494
2495         trace_i915_gem_request_add(request);
2496         ring->outstanding_lazy_request = NULL;
2497
2498         i915_queue_hangcheck(ring->dev);
2499
2500         cancel_delayed_work_sync(&dev_priv->mm.idle_work);
2501         queue_delayed_work(dev_priv->wq,
2502                            &dev_priv->mm.retire_work,
2503                            round_jiffies_up_relative(HZ));
2504         intel_mark_busy(dev_priv->dev);
2505
2506         return 0;
2507 }
2508
2509 static inline void
2510 i915_gem_request_remove_from_client(struct drm_i915_gem_request *request)
2511 {
2512         struct drm_i915_file_private *file_priv = request->file_priv;
2513
2514         if (!file_priv)
2515                 return;
2516
2517         spin_lock(&file_priv->mm.lock);
2518         list_del(&request->client_list);
2519         request->file_priv = NULL;
2520         spin_unlock(&file_priv->mm.lock);
2521 }
2522
2523 static bool i915_context_is_banned(struct drm_i915_private *dev_priv,
2524                                    const struct intel_context *ctx)
2525 {
2526         unsigned long elapsed;
2527
2528         elapsed = get_seconds() - ctx->hang_stats.guilty_ts;
2529
2530         if (ctx->hang_stats.banned)
2531                 return true;
2532
2533         if (elapsed <= DRM_I915_CTX_BAN_PERIOD) {
2534                 if (!i915_gem_context_is_default(ctx)) {
2535                         DRM_DEBUG("context hanging too fast, banning!\n");
2536                         return true;
2537                 } else if (i915_stop_ring_allow_ban(dev_priv)) {
2538                         if (i915_stop_ring_allow_warn(dev_priv))
2539                                 DRM_ERROR("gpu hanging too fast, banning!\n");
2540                         return true;
2541                 }
2542         }
2543
2544         return false;
2545 }
2546
2547 static void i915_set_reset_status(struct drm_i915_private *dev_priv,
2548                                   struct intel_context *ctx,
2549                                   const bool guilty)
2550 {
2551         struct i915_ctx_hang_stats *hs;
2552
2553         if (WARN_ON(!ctx))
2554                 return;
2555
2556         hs = &ctx->hang_stats;
2557
2558         if (guilty) {
2559                 hs->banned = i915_context_is_banned(dev_priv, ctx);
2560                 hs->batch_active++;
2561                 hs->guilty_ts = get_seconds();
2562         } else {
2563                 hs->batch_pending++;
2564         }
2565 }
2566
2567 static void i915_gem_free_request(struct drm_i915_gem_request *request)
2568 {
2569         list_del(&request->list);
2570         i915_gem_request_remove_from_client(request);
2571
2572         i915_gem_request_unreference(request);
2573 }
2574
2575 void i915_gem_request_free(struct kref *req_ref)
2576 {
2577         struct drm_i915_gem_request *req = container_of(req_ref,
2578                                                  typeof(*req), ref);
2579         struct intel_context *ctx = req->ctx;
2580
2581         if (ctx) {
2582                 if (i915.enable_execlists) {
2583                         struct intel_engine_cs *ring = req->ring;
2584
2585                         if (ctx != ring->default_context)
2586                                 intel_lr_context_unpin(ring, ctx);
2587                 }
2588
2589                 i915_gem_context_unreference(ctx);
2590         }
2591
2592         kfree(req);
2593 }
2594
2595 struct drm_i915_gem_request *
2596 i915_gem_find_active_request(struct intel_engine_cs *ring)
2597 {
2598         struct drm_i915_gem_request *request;
2599
2600         list_for_each_entry(request, &ring->request_list, list) {
2601                 if (i915_gem_request_completed(request, false))
2602                         continue;
2603
2604                 return request;
2605         }
2606
2607         return NULL;
2608 }
2609
2610 static void i915_gem_reset_ring_status(struct drm_i915_private *dev_priv,
2611                                        struct intel_engine_cs *ring)
2612 {
2613         struct drm_i915_gem_request *request;
2614         bool ring_hung;
2615
2616         request = i915_gem_find_active_request(ring);
2617
2618         if (request == NULL)
2619                 return;
2620
2621         ring_hung = ring->hangcheck.score >= HANGCHECK_SCORE_RING_HUNG;
2622
2623         i915_set_reset_status(dev_priv, request->ctx, ring_hung);
2624
2625         list_for_each_entry_continue(request, &ring->request_list, list)
2626                 i915_set_reset_status(dev_priv, request->ctx, false);
2627 }
2628
2629 static void i915_gem_reset_ring_cleanup(struct drm_i915_private *dev_priv,
2630                                         struct intel_engine_cs *ring)
2631 {
2632         while (!list_empty(&ring->active_list)) {
2633                 struct drm_i915_gem_object *obj;
2634
2635                 obj = list_first_entry(&ring->active_list,
2636                                        struct drm_i915_gem_object,
2637                                        ring_list);
2638
2639                 i915_gem_object_move_to_inactive(obj);
2640         }
2641
2642         /*
2643          * Clear the execlists queue up before freeing the requests, as those
2644          * are the ones that keep the context and ringbuffer backing objects
2645          * pinned in place.
2646          */
2647         while (!list_empty(&ring->execlist_queue)) {
2648                 struct intel_ctx_submit_request *submit_req;
2649
2650                 submit_req = list_first_entry(&ring->execlist_queue,
2651                                 struct intel_ctx_submit_request,
2652                                 execlist_link);
2653                 list_del(&submit_req->execlist_link);
2654                 intel_runtime_pm_put(dev_priv);
2655                 i915_gem_context_unreference(submit_req->ctx);
2656                 kfree(submit_req);
2657         }
2658
2659         /*
2660          * We must free the requests after all the corresponding objects have
2661          * been moved off active lists. Which is the same order as the normal
2662          * retire_requests function does. This is important if object hold
2663          * implicit references on things like e.g. ppgtt address spaces through
2664          * the request.
2665          */
2666         while (!list_empty(&ring->request_list)) {
2667                 struct drm_i915_gem_request *request;
2668
2669                 request = list_first_entry(&ring->request_list,
2670                                            struct drm_i915_gem_request,
2671                                            list);
2672
2673                 i915_gem_free_request(request);
2674         }
2675
2676         /* This may not have been flushed before the reset, so clean it now */
2677         i915_gem_request_assign(&ring->outstanding_lazy_request, NULL);
2678 }
2679
2680 void i915_gem_restore_fences(struct drm_device *dev)
2681 {
2682         struct drm_i915_private *dev_priv = dev->dev_private;
2683         int i;
2684
2685         for (i = 0; i < dev_priv->num_fence_regs; i++) {
2686                 struct drm_i915_fence_reg *reg = &dev_priv->fence_regs[i];
2687
2688                 /*
2689                  * Commit delayed tiling changes if we have an object still
2690                  * attached to the fence, otherwise just clear the fence.
2691                  */
2692                 if (reg->obj) {
2693                         i915_gem_object_update_fence(reg->obj, reg,
2694                                                      reg->obj->tiling_mode);
2695                 } else {
2696                         i915_gem_write_fence(dev, i, NULL);
2697                 }
2698         }
2699 }
2700
2701 void i915_gem_reset(struct drm_device *dev)
2702 {
2703         struct drm_i915_private *dev_priv = dev->dev_private;
2704         struct intel_engine_cs *ring;
2705         int i;
2706
2707         /*
2708          * Before we free the objects from the requests, we need to inspect
2709          * them for finding the guilty party. As the requests only borrow
2710          * their reference to the objects, the inspection must be done first.
2711          */
2712         for_each_ring(ring, dev_priv, i)
2713                 i915_gem_reset_ring_status(dev_priv, ring);
2714
2715         for_each_ring(ring, dev_priv, i)
2716                 i915_gem_reset_ring_cleanup(dev_priv, ring);
2717
2718         i915_gem_context_reset(dev);
2719
2720         i915_gem_restore_fences(dev);
2721 }
2722
2723 /**
2724  * This function clears the request list as sequence numbers are passed.
2725  */
2726 void
2727 i915_gem_retire_requests_ring(struct intel_engine_cs *ring)
2728 {
2729         if (list_empty(&ring->request_list))
2730                 return;
2731
2732         WARN_ON(i915_verify_lists(ring->dev));
2733
2734         /* Move any buffers on the active list that are no longer referenced
2735          * by the ringbuffer to the flushing/inactive lists as appropriate,
2736          * before we free the context associated with the requests.
2737          */
2738         while (!list_empty(&ring->active_list)) {
2739                 struct drm_i915_gem_object *obj;
2740
2741                 obj = list_first_entry(&ring->active_list,
2742                                       struct drm_i915_gem_object,
2743                                       ring_list);
2744
2745                 if (!i915_gem_request_completed(obj->last_read_req, true))
2746                         break;
2747
2748                 i915_gem_object_move_to_inactive(obj);
2749         }
2750
2751
2752         while (!list_empty(&ring->request_list)) {
2753                 struct drm_i915_gem_request *request;
2754                 struct intel_ringbuffer *ringbuf;
2755
2756                 request = list_first_entry(&ring->request_list,
2757                                            struct drm_i915_gem_request,
2758                                            list);
2759
2760                 if (!i915_gem_request_completed(request, true))
2761                         break;
2762
2763                 trace_i915_gem_request_retire(request);
2764
2765                 /* This is one of the few common intersection points
2766                  * between legacy ringbuffer submission and execlists:
2767                  * we need to tell them apart in order to find the correct
2768                  * ringbuffer to which the request belongs to.
2769                  */
2770                 if (i915.enable_execlists) {
2771                         struct intel_context *ctx = request->ctx;
2772                         ringbuf = ctx->engine[ring->id].ringbuf;
2773                 } else
2774                         ringbuf = ring->buffer;
2775
2776                 /* We know the GPU must have read the request to have
2777                  * sent us the seqno + interrupt, so use the position
2778                  * of tail of the request to update the last known position
2779                  * of the GPU head.
2780                  */
2781                 ringbuf->last_retired_head = request->tail;
2782
2783                 i915_gem_free_request(request);
2784         }
2785
2786         if (unlikely(ring->trace_irq_req &&
2787                      i915_gem_request_completed(ring->trace_irq_req, true))) {
2788                 ring->irq_put(ring);
2789                 i915_gem_request_assign(&ring->trace_irq_req, NULL);
2790         }
2791
2792         WARN_ON(i915_verify_lists(ring->dev));
2793 }
2794
2795 bool
2796 i915_gem_retire_requests(struct drm_device *dev)
2797 {
2798         struct drm_i915_private *dev_priv = dev->dev_private;
2799         struct intel_engine_cs *ring;
2800         bool idle = true;
2801         int i;
2802
2803         for_each_ring(ring, dev_priv, i) {
2804                 i915_gem_retire_requests_ring(ring);
2805                 idle &= list_empty(&ring->request_list);
2806                 if (i915.enable_execlists) {
2807                         unsigned long flags;
2808
2809                         spin_lock_irqsave(&ring->execlist_lock, flags);
2810                         idle &= list_empty(&ring->execlist_queue);
2811                         spin_unlock_irqrestore(&ring->execlist_lock, flags);
2812
2813                         intel_execlists_retire_requests(ring);
2814                 }
2815         }
2816
2817         if (idle)
2818                 mod_delayed_work(dev_priv->wq,
2819                                    &dev_priv->mm.idle_work,
2820                                    msecs_to_jiffies(100));
2821
2822         return idle;
2823 }
2824
2825 static void
2826 i915_gem_retire_work_handler(struct work_struct *work)
2827 {
2828         struct drm_i915_private *dev_priv =
2829                 container_of(work, typeof(*dev_priv), mm.retire_work.work);
2830         struct drm_device *dev = dev_priv->dev;
2831         bool idle;
2832
2833         /* Come back later if the device is busy... */
2834         idle = false;
2835         if (mutex_trylock(&dev->struct_mutex)) {
2836                 idle = i915_gem_retire_requests(dev);
2837                 mutex_unlock(&dev->struct_mutex);
2838         }
2839         if (!idle)
2840                 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work,
2841                                    round_jiffies_up_relative(HZ));
2842 }
2843
2844 static void
2845 i915_gem_idle_work_handler(struct work_struct *work)
2846 {
2847         struct drm_i915_private *dev_priv =
2848                 container_of(work, typeof(*dev_priv), mm.idle_work.work);
2849
2850         intel_mark_idle(dev_priv->dev);
2851 }
2852
2853 /**
2854  * Ensures that an object will eventually get non-busy by flushing any required
2855  * write domains, emitting any outstanding lazy request and retiring and
2856  * completed requests.
2857  */
2858 static int
2859 i915_gem_object_flush_active(struct drm_i915_gem_object *obj)
2860 {
2861         struct intel_engine_cs *ring;
2862         int ret;
2863
2864         if (obj->active) {
2865                 ring = i915_gem_request_get_ring(obj->last_read_req);
2866
2867                 ret = i915_gem_check_olr(obj->last_read_req);
2868                 if (ret)
2869                         return ret;
2870
2871                 i915_gem_retire_requests_ring(ring);
2872         }
2873
2874         return 0;
2875 }
2876
2877 /**
2878  * i915_gem_wait_ioctl - implements DRM_IOCTL_I915_GEM_WAIT
2879  * @DRM_IOCTL_ARGS: standard ioctl arguments
2880  *
2881  * Returns 0 if successful, else an error is returned with the remaining time in
2882  * the timeout parameter.
2883  *  -ETIME: object is still busy after timeout
2884  *  -ERESTARTSYS: signal interrupted the wait
2885  *  -ENONENT: object doesn't exist
2886  * Also possible, but rare:
2887  *  -EAGAIN: GPU wedged
2888  *  -ENOMEM: damn
2889  *  -ENODEV: Internal IRQ fail
2890  *  -E?: The add request failed
2891  *
2892  * The wait ioctl with a timeout of 0 reimplements the busy ioctl. With any
2893  * non-zero timeout parameter the wait ioctl will wait for the given number of
2894  * nanoseconds on an object becoming unbusy. Since the wait itself does so
2895  * without holding struct_mutex the object may become re-busied before this
2896  * function completes. A similar but shorter * race condition exists in the busy
2897  * ioctl
2898  */
2899 int
2900 i915_gem_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
2901 {
2902         struct drm_i915_private *dev_priv = dev->dev_private;
2903         struct drm_i915_gem_wait *args = data;
2904         struct drm_i915_gem_object *obj;
2905         struct drm_i915_gem_request *req;
2906         unsigned reset_counter;
2907         int ret = 0;
2908
2909         if (args->flags != 0)
2910                 return -EINVAL;
2911
2912         ret = i915_mutex_lock_interruptible(dev);
2913         if (ret)
2914                 return ret;
2915
2916         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->bo_handle));
2917         if (&obj->base == NULL) {
2918                 mutex_unlock(&dev->struct_mutex);
2919                 return -ENOENT;
2920         }
2921
2922         /* Need to make sure the object gets inactive eventually. */
2923         ret = i915_gem_object_flush_active(obj);
2924         if (ret)
2925                 goto out;
2926
2927         if (!obj->active || !obj->last_read_req)
2928                 goto out;
2929
2930         req = obj->last_read_req;
2931
2932         /* Do this after OLR check to make sure we make forward progress polling
2933          * on this IOCTL with a timeout <=0 (like busy ioctl)
2934          */
2935         if (args->timeout_ns <= 0) {
2936                 ret = -ETIME;
2937                 goto out;
2938         }
2939
2940         drm_gem_object_unreference(&obj->base);
2941         reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
2942         i915_gem_request_reference(req);
2943         mutex_unlock(&dev->struct_mutex);
2944
2945         ret = __i915_wait_request(req, reset_counter, true, &args->timeout_ns,
2946                                   file->driver_priv);
2947         mutex_lock(&dev->struct_mutex);
2948         i915_gem_request_unreference(req);
2949         mutex_unlock(&dev->struct_mutex);
2950         return ret;
2951
2952 out:
2953         drm_gem_object_unreference(&obj->base);
2954         mutex_unlock(&dev->struct_mutex);
2955         return ret;
2956 }
2957
2958 /**
2959  * i915_gem_object_sync - sync an object to a ring.
2960  *
2961  * @obj: object which may be in use on another ring.
2962  * @to: ring we wish to use the object on. May be NULL.
2963  *
2964  * This code is meant to abstract object synchronization with the GPU.
2965  * Calling with NULL implies synchronizing the object with the CPU
2966  * rather than a particular GPU ring.
2967  *
2968  * Returns 0 if successful, else propagates up the lower layer error.
2969  */
2970 int
2971 i915_gem_object_sync(struct drm_i915_gem_object *obj,
2972                      struct intel_engine_cs *to)
2973 {
2974         struct intel_engine_cs *from;
2975         u32 seqno;
2976         int ret, idx;
2977
2978         from = i915_gem_request_get_ring(obj->last_read_req);
2979
2980         if (from == NULL || to == from)
2981                 return 0;
2982
2983         if (to == NULL || !i915_semaphore_is_enabled(obj->base.dev))
2984                 return i915_gem_object_wait_rendering(obj, false);
2985
2986         idx = intel_ring_sync_index(from, to);
2987
2988         seqno = i915_gem_request_get_seqno(obj->last_read_req);
2989         /* Optimization: Avoid semaphore sync when we are sure we already
2990          * waited for an object with higher seqno */
2991         if (seqno <= from->semaphore.sync_seqno[idx])
2992                 return 0;
2993
2994         ret = i915_gem_check_olr(obj->last_read_req);
2995         if (ret)
2996                 return ret;
2997
2998         trace_i915_gem_ring_sync_to(from, to, obj->last_read_req);
2999         ret = to->semaphore.sync_to(to, from, seqno);
3000         if (!ret)
3001                 /* We use last_read_req because sync_to()
3002                  * might have just caused seqno wrap under
3003                  * the radar.
3004                  */
3005                 from->semaphore.sync_seqno[idx] =
3006                                 i915_gem_request_get_seqno(obj->last_read_req);
3007
3008         return ret;
3009 }
3010
3011 static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj)
3012 {
3013         u32 old_write_domain, old_read_domains;
3014
3015         /* Force a pagefault for domain tracking on next user access */
3016         i915_gem_release_mmap(obj);
3017
3018         if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
3019                 return;
3020
3021         /* Wait for any direct GTT access to complete */
3022         mb();
3023
3024         old_read_domains = obj->base.read_domains;
3025         old_write_domain = obj->base.write_domain;
3026
3027         obj->base.read_domains &= ~I915_GEM_DOMAIN_GTT;
3028         obj->base.write_domain &= ~I915_GEM_DOMAIN_GTT;
3029
3030         trace_i915_gem_object_change_domain(obj,
3031                                             old_read_domains,
3032                                             old_write_domain);
3033 }
3034
3035 int i915_vma_unbind(struct i915_vma *vma)
3036 {
3037         struct drm_i915_gem_object *obj = vma->obj;
3038         struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
3039         int ret;
3040
3041         if (list_empty(&vma->vma_link))
3042                 return 0;
3043
3044         if (!drm_mm_node_allocated(&vma->node)) {
3045                 i915_gem_vma_destroy(vma);
3046                 return 0;
3047         }
3048
3049         if (vma->pin_count)
3050                 return -EBUSY;
3051
3052         BUG_ON(obj->pages == NULL);
3053
3054         ret = i915_gem_object_finish_gpu(obj);
3055         if (ret)
3056                 return ret;
3057         /* Continue on if we fail due to EIO, the GPU is hung so we
3058          * should be safe and we need to cleanup or else we might
3059          * cause memory corruption through use-after-free.
3060          */
3061
3062         if (i915_is_ggtt(vma->vm) &&
3063             vma->ggtt_view.type == I915_GGTT_VIEW_NORMAL) {
3064                 i915_gem_object_finish_gtt(obj);
3065
3066                 /* release the fence reg _after_ flushing */
3067                 ret = i915_gem_object_put_fence(obj);
3068                 if (ret)
3069                         return ret;
3070         }
3071
3072         trace_i915_vma_unbind(vma);
3073
3074         vma->unbind_vma(vma);
3075
3076         list_del_init(&vma->mm_list);
3077         if (i915_is_ggtt(vma->vm)) {
3078                 if (vma->ggtt_view.type == I915_GGTT_VIEW_NORMAL) {
3079                         obj->map_and_fenceable = false;
3080                 } else if (vma->ggtt_view.pages) {
3081                         sg_free_table(vma->ggtt_view.pages);
3082                         kfree(vma->ggtt_view.pages);
3083                         vma->ggtt_view.pages = NULL;
3084                 }
3085         }
3086
3087         drm_mm_remove_node(&vma->node);
3088         i915_gem_vma_destroy(vma);
3089
3090         /* Since the unbound list is global, only move to that list if
3091          * no more VMAs exist. */
3092         if (list_empty(&obj->vma_list)) {
3093                 /* Throw away the active reference before
3094                  * moving to the unbound list. */
3095                 i915_gem_object_retire(obj);
3096
3097                 i915_gem_gtt_finish_object(obj);
3098                 list_move_tail(&obj->global_list, &dev_priv->mm.unbound_list);
3099         }
3100
3101         /* And finally now the object is completely decoupled from this vma,
3102          * we can drop its hold on the backing storage and allow it to be
3103          * reaped by the shrinker.
3104          */
3105         i915_gem_object_unpin_pages(obj);
3106
3107         return 0;
3108 }
3109
3110 int i915_gpu_idle(struct drm_device *dev)
3111 {
3112         struct drm_i915_private *dev_priv = dev->dev_private;
3113         struct intel_engine_cs *ring;
3114         int ret, i;
3115
3116         /* Flush everything onto the inactive list. */
3117         for_each_ring(ring, dev_priv, i) {
3118                 if (!i915.enable_execlists) {
3119                         ret = i915_switch_context(ring, ring->default_context);
3120                         if (ret)
3121                                 return ret;
3122                 }
3123
3124                 ret = intel_ring_idle(ring);
3125                 if (ret)
3126                         return ret;
3127         }
3128
3129         return 0;
3130 }
3131
3132 static void i965_write_fence_reg(struct drm_device *dev, int reg,
3133                                  struct drm_i915_gem_object *obj)
3134 {
3135         struct drm_i915_private *dev_priv = dev->dev_private;
3136         int fence_reg;
3137         int fence_pitch_shift;
3138
3139         if (INTEL_INFO(dev)->gen >= 6) {
3140                 fence_reg = FENCE_REG_SANDYBRIDGE_0;
3141                 fence_pitch_shift = SANDYBRIDGE_FENCE_PITCH_SHIFT;
3142         } else {
3143                 fence_reg = FENCE_REG_965_0;
3144                 fence_pitch_shift = I965_FENCE_PITCH_SHIFT;
3145         }
3146
3147         fence_reg += reg * 8;
3148
3149         /* To w/a incoherency with non-atomic 64-bit register updates,
3150          * we split the 64-bit update into two 32-bit writes. In order
3151          * for a partial fence not to be evaluated between writes, we
3152          * precede the update with write to turn off the fence register,
3153          * and only enable the fence as the last step.
3154          *
3155          * For extra levels of paranoia, we make sure each step lands
3156          * before applying the next step.
3157          */
3158         I915_WRITE(fence_reg, 0);
3159         POSTING_READ(fence_reg);
3160
3161         if (obj) {
3162                 u32 size = i915_gem_obj_ggtt_size(obj);
3163                 uint64_t val;
3164
3165                 val = (uint64_t)((i915_gem_obj_ggtt_offset(obj) + size - 4096) &
3166                                  0xfffff000) << 32;
3167                 val |= i915_gem_obj_ggtt_offset(obj) & 0xfffff000;
3168                 val |= (uint64_t)((obj->stride / 128) - 1) << fence_pitch_shift;
3169                 if (obj->tiling_mode == I915_TILING_Y)
3170                         val |= 1 << I965_FENCE_TILING_Y_SHIFT;
3171                 val |= I965_FENCE_REG_VALID;
3172
3173                 I915_WRITE(fence_reg + 4, val >> 32);
3174                 POSTING_READ(fence_reg + 4);
3175
3176                 I915_WRITE(fence_reg + 0, val);
3177                 POSTING_READ(fence_reg);
3178         } else {
3179                 I915_WRITE(fence_reg + 4, 0);
3180                 POSTING_READ(fence_reg + 4);
3181         }
3182 }
3183
3184 static void i915_write_fence_reg(struct drm_device *dev, int reg,
3185                                  struct drm_i915_gem_object *obj)
3186 {
3187         struct drm_i915_private *dev_priv = dev->dev_private;
3188         u32 val;
3189
3190         if (obj) {
3191                 u32 size = i915_gem_obj_ggtt_size(obj);
3192                 int pitch_val;
3193                 int tile_width;
3194
3195                 WARN((i915_gem_obj_ggtt_offset(obj) & ~I915_FENCE_START_MASK) ||
3196                      (size & -size) != size ||
3197                      (i915_gem_obj_ggtt_offset(obj) & (size - 1)),
3198                      "object 0x%08lx [fenceable? %d] not 1M or pot-size (0x%08x) aligned\n",
3199                      i915_gem_obj_ggtt_offset(obj), obj->map_and_fenceable, size);
3200
3201                 if (obj->tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev))
3202 &n