3044fb324c8e9e255ab20d867af94af71e11bc11
[sfrench/cifs-2.6.git] / drivers / gpu / drm / i915 / i915_gem.c
1 /*
2  * Copyright © 2008 Intel Corporation
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice (including the next
12  * paragraph) shall be included in all copies or substantial portions of the
13  * Software.
14  *
15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21  * IN THE SOFTWARE.
22  *
23  * Authors:
24  *    Eric Anholt <eric@anholt.net>
25  *
26  */
27
28 #include <drm/drmP.h>
29 #include <drm/drm_vma_manager.h>
30 #include <drm/i915_drm.h>
31 #include "i915_drv.h"
32 #include "i915_trace.h"
33 #include "intel_drv.h"
34 #include <linux/oom.h>
35 #include <linux/shmem_fs.h>
36 #include <linux/slab.h>
37 #include <linux/swap.h>
38 #include <linux/pci.h>
39 #include <linux/dma-buf.h>
40
41 static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj);
42 static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj,
43                                                    bool force);
44 static __must_check int
45 i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj,
46                                bool readonly);
47 static void
48 i915_gem_object_retire(struct drm_i915_gem_object *obj);
49
50 static void i915_gem_write_fence(struct drm_device *dev, int reg,
51                                  struct drm_i915_gem_object *obj);
52 static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj,
53                                          struct drm_i915_fence_reg *fence,
54                                          bool enable);
55
56 static unsigned long i915_gem_shrinker_count(struct shrinker *shrinker,
57                                              struct shrink_control *sc);
58 static unsigned long i915_gem_shrinker_scan(struct shrinker *shrinker,
59                                             struct shrink_control *sc);
60 static int i915_gem_shrinker_oom(struct notifier_block *nb,
61                                  unsigned long event,
62                                  void *ptr);
63 static unsigned long i915_gem_shrink_all(struct drm_i915_private *dev_priv);
64
65 static bool cpu_cache_is_coherent(struct drm_device *dev,
66                                   enum i915_cache_level level)
67 {
68         return HAS_LLC(dev) || level != I915_CACHE_NONE;
69 }
70
71 static bool cpu_write_needs_clflush(struct drm_i915_gem_object *obj)
72 {
73         if (!cpu_cache_is_coherent(obj->base.dev, obj->cache_level))
74                 return true;
75
76         return obj->pin_display;
77 }
78
79 static inline void i915_gem_object_fence_lost(struct drm_i915_gem_object *obj)
80 {
81         if (obj->tiling_mode)
82                 i915_gem_release_mmap(obj);
83
84         /* As we do not have an associated fence register, we will force
85          * a tiling change if we ever need to acquire one.
86          */
87         obj->fence_dirty = false;
88         obj->fence_reg = I915_FENCE_REG_NONE;
89 }
90
91 /* some bookkeeping */
92 static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv,
93                                   size_t size)
94 {
95         spin_lock(&dev_priv->mm.object_stat_lock);
96         dev_priv->mm.object_count++;
97         dev_priv->mm.object_memory += size;
98         spin_unlock(&dev_priv->mm.object_stat_lock);
99 }
100
101 static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv,
102                                      size_t size)
103 {
104         spin_lock(&dev_priv->mm.object_stat_lock);
105         dev_priv->mm.object_count--;
106         dev_priv->mm.object_memory -= size;
107         spin_unlock(&dev_priv->mm.object_stat_lock);
108 }
109
110 static int
111 i915_gem_wait_for_error(struct i915_gpu_error *error)
112 {
113         int ret;
114
115 #define EXIT_COND (!i915_reset_in_progress(error) || \
116                    i915_terminally_wedged(error))
117         if (EXIT_COND)
118                 return 0;
119
120         /*
121          * Only wait 10 seconds for the gpu reset to complete to avoid hanging
122          * userspace. If it takes that long something really bad is going on and
123          * we should simply try to bail out and fail as gracefully as possible.
124          */
125         ret = wait_event_interruptible_timeout(error->reset_queue,
126                                                EXIT_COND,
127                                                10*HZ);
128         if (ret == 0) {
129                 DRM_ERROR("Timed out waiting for the gpu reset to complete\n");
130                 return -EIO;
131         } else if (ret < 0) {
132                 return ret;
133         }
134 #undef EXIT_COND
135
136         return 0;
137 }
138
139 int i915_mutex_lock_interruptible(struct drm_device *dev)
140 {
141         struct drm_i915_private *dev_priv = dev->dev_private;
142         int ret;
143
144         ret = i915_gem_wait_for_error(&dev_priv->gpu_error);
145         if (ret)
146                 return ret;
147
148         ret = mutex_lock_interruptible(&dev->struct_mutex);
149         if (ret)
150                 return ret;
151
152         WARN_ON(i915_verify_lists(dev));
153         return 0;
154 }
155
156 static inline bool
157 i915_gem_object_is_inactive(struct drm_i915_gem_object *obj)
158 {
159         return i915_gem_obj_bound_any(obj) && !obj->active;
160 }
161
162 int
163 i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
164                             struct drm_file *file)
165 {
166         struct drm_i915_private *dev_priv = dev->dev_private;
167         struct drm_i915_gem_get_aperture *args = data;
168         struct drm_i915_gem_object *obj;
169         size_t pinned;
170
171         pinned = 0;
172         mutex_lock(&dev->struct_mutex);
173         list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list)
174                 if (i915_gem_obj_is_pinned(obj))
175                         pinned += i915_gem_obj_ggtt_size(obj);
176         mutex_unlock(&dev->struct_mutex);
177
178         args->aper_size = dev_priv->gtt.base.total;
179         args->aper_available_size = args->aper_size - pinned;
180
181         return 0;
182 }
183
184 static int
185 i915_gem_object_get_pages_phys(struct drm_i915_gem_object *obj)
186 {
187         struct address_space *mapping = file_inode(obj->base.filp)->i_mapping;
188         char *vaddr = obj->phys_handle->vaddr;
189         struct sg_table *st;
190         struct scatterlist *sg;
191         int i;
192
193         if (WARN_ON(i915_gem_object_needs_bit17_swizzle(obj)))
194                 return -EINVAL;
195
196         for (i = 0; i < obj->base.size / PAGE_SIZE; i++) {
197                 struct page *page;
198                 char *src;
199
200                 page = shmem_read_mapping_page(mapping, i);
201                 if (IS_ERR(page))
202                         return PTR_ERR(page);
203
204                 src = kmap_atomic(page);
205                 memcpy(vaddr, src, PAGE_SIZE);
206                 drm_clflush_virt_range(vaddr, PAGE_SIZE);
207                 kunmap_atomic(src);
208
209                 page_cache_release(page);
210                 vaddr += PAGE_SIZE;
211         }
212
213         i915_gem_chipset_flush(obj->base.dev);
214
215         st = kmalloc(sizeof(*st), GFP_KERNEL);
216         if (st == NULL)
217                 return -ENOMEM;
218
219         if (sg_alloc_table(st, 1, GFP_KERNEL)) {
220                 kfree(st);
221                 return -ENOMEM;
222         }
223
224         sg = st->sgl;
225         sg->offset = 0;
226         sg->length = obj->base.size;
227
228         sg_dma_address(sg) = obj->phys_handle->busaddr;
229         sg_dma_len(sg) = obj->base.size;
230
231         obj->pages = st;
232         obj->has_dma_mapping = true;
233         return 0;
234 }
235
236 static void
237 i915_gem_object_put_pages_phys(struct drm_i915_gem_object *obj)
238 {
239         int ret;
240
241         BUG_ON(obj->madv == __I915_MADV_PURGED);
242
243         ret = i915_gem_object_set_to_cpu_domain(obj, true);
244         if (ret) {
245                 /* In the event of a disaster, abandon all caches and
246                  * hope for the best.
247                  */
248                 WARN_ON(ret != -EIO);
249                 obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU;
250         }
251
252         if (obj->madv == I915_MADV_DONTNEED)
253                 obj->dirty = 0;
254
255         if (obj->dirty) {
256                 struct address_space *mapping = file_inode(obj->base.filp)->i_mapping;
257                 char *vaddr = obj->phys_handle->vaddr;
258                 int i;
259
260                 for (i = 0; i < obj->base.size / PAGE_SIZE; i++) {
261                         struct page *page;
262                         char *dst;
263
264                         page = shmem_read_mapping_page(mapping, i);
265                         if (IS_ERR(page))
266                                 continue;
267
268                         dst = kmap_atomic(page);
269                         drm_clflush_virt_range(vaddr, PAGE_SIZE);
270                         memcpy(dst, vaddr, PAGE_SIZE);
271                         kunmap_atomic(dst);
272
273                         set_page_dirty(page);
274                         if (obj->madv == I915_MADV_WILLNEED)
275                                 mark_page_accessed(page);
276                         page_cache_release(page);
277                         vaddr += PAGE_SIZE;
278                 }
279                 obj->dirty = 0;
280         }
281
282         sg_free_table(obj->pages);
283         kfree(obj->pages);
284
285         obj->has_dma_mapping = false;
286 }
287
288 static void
289 i915_gem_object_release_phys(struct drm_i915_gem_object *obj)
290 {
291         drm_pci_free(obj->base.dev, obj->phys_handle);
292 }
293
294 static const struct drm_i915_gem_object_ops i915_gem_phys_ops = {
295         .get_pages = i915_gem_object_get_pages_phys,
296         .put_pages = i915_gem_object_put_pages_phys,
297         .release = i915_gem_object_release_phys,
298 };
299
300 static int
301 drop_pages(struct drm_i915_gem_object *obj)
302 {
303         struct i915_vma *vma, *next;
304         int ret;
305
306         drm_gem_object_reference(&obj->base);
307         list_for_each_entry_safe(vma, next, &obj->vma_list, vma_link)
308                 if (i915_vma_unbind(vma))
309                         break;
310
311         ret = i915_gem_object_put_pages(obj);
312         drm_gem_object_unreference(&obj->base);
313
314         return ret;
315 }
316
317 int
318 i915_gem_object_attach_phys(struct drm_i915_gem_object *obj,
319                             int align)
320 {
321         drm_dma_handle_t *phys;
322         int ret;
323
324         if (obj->phys_handle) {
325                 if ((unsigned long)obj->phys_handle->vaddr & (align -1))
326                         return -EBUSY;
327
328                 return 0;
329         }
330
331         if (obj->madv != I915_MADV_WILLNEED)
332                 return -EFAULT;
333
334         if (obj->base.filp == NULL)
335                 return -EINVAL;
336
337         ret = drop_pages(obj);
338         if (ret)
339                 return ret;
340
341         /* create a new object */
342         phys = drm_pci_alloc(obj->base.dev, obj->base.size, align);
343         if (!phys)
344                 return -ENOMEM;
345
346         obj->phys_handle = phys;
347         obj->ops = &i915_gem_phys_ops;
348
349         return i915_gem_object_get_pages(obj);
350 }
351
352 static int
353 i915_gem_phys_pwrite(struct drm_i915_gem_object *obj,
354                      struct drm_i915_gem_pwrite *args,
355                      struct drm_file *file_priv)
356 {
357         struct drm_device *dev = obj->base.dev;
358         void *vaddr = obj->phys_handle->vaddr + args->offset;
359         char __user *user_data = to_user_ptr(args->data_ptr);
360         int ret;
361
362         /* We manually control the domain here and pretend that it
363          * remains coherent i.e. in the GTT domain, like shmem_pwrite.
364          */
365         ret = i915_gem_object_wait_rendering(obj, false);
366         if (ret)
367                 return ret;
368
369         if (__copy_from_user_inatomic_nocache(vaddr, user_data, args->size)) {
370                 unsigned long unwritten;
371
372                 /* The physical object once assigned is fixed for the lifetime
373                  * of the obj, so we can safely drop the lock and continue
374                  * to access vaddr.
375                  */
376                 mutex_unlock(&dev->struct_mutex);
377                 unwritten = copy_from_user(vaddr, user_data, args->size);
378                 mutex_lock(&dev->struct_mutex);
379                 if (unwritten)
380                         return -EFAULT;
381         }
382
383         drm_clflush_virt_range(vaddr, args->size);
384         i915_gem_chipset_flush(dev);
385         return 0;
386 }
387
388 void *i915_gem_object_alloc(struct drm_device *dev)
389 {
390         struct drm_i915_private *dev_priv = dev->dev_private;
391         return kmem_cache_zalloc(dev_priv->slab, GFP_KERNEL);
392 }
393
394 void i915_gem_object_free(struct drm_i915_gem_object *obj)
395 {
396         struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
397         kmem_cache_free(dev_priv->slab, obj);
398 }
399
400 static int
401 i915_gem_create(struct drm_file *file,
402                 struct drm_device *dev,
403                 uint64_t size,
404                 uint32_t *handle_p)
405 {
406         struct drm_i915_gem_object *obj;
407         int ret;
408         u32 handle;
409
410         size = roundup(size, PAGE_SIZE);
411         if (size == 0)
412                 return -EINVAL;
413
414         /* Allocate the new object */
415         obj = i915_gem_alloc_object(dev, size);
416         if (obj == NULL)
417                 return -ENOMEM;
418
419         ret = drm_gem_handle_create(file, &obj->base, &handle);
420         /* drop reference from allocate - handle holds it now */
421         drm_gem_object_unreference_unlocked(&obj->base);
422         if (ret)
423                 return ret;
424
425         *handle_p = handle;
426         return 0;
427 }
428
429 int
430 i915_gem_dumb_create(struct drm_file *file,
431                      struct drm_device *dev,
432                      struct drm_mode_create_dumb *args)
433 {
434         /* have to work out size/pitch and return them */
435         args->pitch = ALIGN(args->width * DIV_ROUND_UP(args->bpp, 8), 64);
436         args->size = args->pitch * args->height;
437         return i915_gem_create(file, dev,
438                                args->size, &args->handle);
439 }
440
441 /**
442  * Creates a new mm object and returns a handle to it.
443  */
444 int
445 i915_gem_create_ioctl(struct drm_device *dev, void *data,
446                       struct drm_file *file)
447 {
448         struct drm_i915_gem_create *args = data;
449
450         return i915_gem_create(file, dev,
451                                args->size, &args->handle);
452 }
453
454 static inline int
455 __copy_to_user_swizzled(char __user *cpu_vaddr,
456                         const char *gpu_vaddr, int gpu_offset,
457                         int length)
458 {
459         int ret, cpu_offset = 0;
460
461         while (length > 0) {
462                 int cacheline_end = ALIGN(gpu_offset + 1, 64);
463                 int this_length = min(cacheline_end - gpu_offset, length);
464                 int swizzled_gpu_offset = gpu_offset ^ 64;
465
466                 ret = __copy_to_user(cpu_vaddr + cpu_offset,
467                                      gpu_vaddr + swizzled_gpu_offset,
468                                      this_length);
469                 if (ret)
470                         return ret + length;
471
472                 cpu_offset += this_length;
473                 gpu_offset += this_length;
474                 length -= this_length;
475         }
476
477         return 0;
478 }
479
480 static inline int
481 __copy_from_user_swizzled(char *gpu_vaddr, int gpu_offset,
482                           const char __user *cpu_vaddr,
483                           int length)
484 {
485         int ret, cpu_offset = 0;
486
487         while (length > 0) {
488                 int cacheline_end = ALIGN(gpu_offset + 1, 64);
489                 int this_length = min(cacheline_end - gpu_offset, length);
490                 int swizzled_gpu_offset = gpu_offset ^ 64;
491
492                 ret = __copy_from_user(gpu_vaddr + swizzled_gpu_offset,
493                                        cpu_vaddr + cpu_offset,
494                                        this_length);
495                 if (ret)
496                         return ret + length;
497
498                 cpu_offset += this_length;
499                 gpu_offset += this_length;
500                 length -= this_length;
501         }
502
503         return 0;
504 }
505
506 /*
507  * Pins the specified object's pages and synchronizes the object with
508  * GPU accesses. Sets needs_clflush to non-zero if the caller should
509  * flush the object from the CPU cache.
510  */
511 int i915_gem_obj_prepare_shmem_read(struct drm_i915_gem_object *obj,
512                                     int *needs_clflush)
513 {
514         int ret;
515
516         *needs_clflush = 0;
517
518         if (!obj->base.filp)
519                 return -EINVAL;
520
521         if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU)) {
522                 /* If we're not in the cpu read domain, set ourself into the gtt
523                  * read domain and manually flush cachelines (if required). This
524                  * optimizes for the case when the gpu will dirty the data
525                  * anyway again before the next pread happens. */
526                 *needs_clflush = !cpu_cache_is_coherent(obj->base.dev,
527                                                         obj->cache_level);
528                 ret = i915_gem_object_wait_rendering(obj, true);
529                 if (ret)
530                         return ret;
531
532                 i915_gem_object_retire(obj);
533         }
534
535         ret = i915_gem_object_get_pages(obj);
536         if (ret)
537                 return ret;
538
539         i915_gem_object_pin_pages(obj);
540
541         return ret;
542 }
543
544 /* Per-page copy function for the shmem pread fastpath.
545  * Flushes invalid cachelines before reading the target if
546  * needs_clflush is set. */
547 static int
548 shmem_pread_fast(struct page *page, int shmem_page_offset, int page_length,
549                  char __user *user_data,
550                  bool page_do_bit17_swizzling, bool needs_clflush)
551 {
552         char *vaddr;
553         int ret;
554
555         if (unlikely(page_do_bit17_swizzling))
556                 return -EINVAL;
557
558         vaddr = kmap_atomic(page);
559         if (needs_clflush)
560                 drm_clflush_virt_range(vaddr + shmem_page_offset,
561                                        page_length);
562         ret = __copy_to_user_inatomic(user_data,
563                                       vaddr + shmem_page_offset,
564                                       page_length);
565         kunmap_atomic(vaddr);
566
567         return ret ? -EFAULT : 0;
568 }
569
570 static void
571 shmem_clflush_swizzled_range(char *addr, unsigned long length,
572                              bool swizzled)
573 {
574         if (unlikely(swizzled)) {
575                 unsigned long start = (unsigned long) addr;
576                 unsigned long end = (unsigned long) addr + length;
577
578                 /* For swizzling simply ensure that we always flush both
579                  * channels. Lame, but simple and it works. Swizzled
580                  * pwrite/pread is far from a hotpath - current userspace
581                  * doesn't use it at all. */
582                 start = round_down(start, 128);
583                 end = round_up(end, 128);
584
585                 drm_clflush_virt_range((void *)start, end - start);
586         } else {
587                 drm_clflush_virt_range(addr, length);
588         }
589
590 }
591
592 /* Only difference to the fast-path function is that this can handle bit17
593  * and uses non-atomic copy and kmap functions. */
594 static int
595 shmem_pread_slow(struct page *page, int shmem_page_offset, int page_length,
596                  char __user *user_data,
597                  bool page_do_bit17_swizzling, bool needs_clflush)
598 {
599         char *vaddr;
600         int ret;
601
602         vaddr = kmap(page);
603         if (needs_clflush)
604                 shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
605                                              page_length,
606                                              page_do_bit17_swizzling);
607
608         if (page_do_bit17_swizzling)
609                 ret = __copy_to_user_swizzled(user_data,
610                                               vaddr, shmem_page_offset,
611                                               page_length);
612         else
613                 ret = __copy_to_user(user_data,
614                                      vaddr + shmem_page_offset,
615                                      page_length);
616         kunmap(page);
617
618         return ret ? - EFAULT : 0;
619 }
620
621 static int
622 i915_gem_shmem_pread(struct drm_device *dev,
623                      struct drm_i915_gem_object *obj,
624                      struct drm_i915_gem_pread *args,
625                      struct drm_file *file)
626 {
627         char __user *user_data;
628         ssize_t remain;
629         loff_t offset;
630         int shmem_page_offset, page_length, ret = 0;
631         int obj_do_bit17_swizzling, page_do_bit17_swizzling;
632         int prefaulted = 0;
633         int needs_clflush = 0;
634         struct sg_page_iter sg_iter;
635
636         user_data = to_user_ptr(args->data_ptr);
637         remain = args->size;
638
639         obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
640
641         ret = i915_gem_obj_prepare_shmem_read(obj, &needs_clflush);
642         if (ret)
643                 return ret;
644
645         offset = args->offset;
646
647         for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents,
648                          offset >> PAGE_SHIFT) {
649                 struct page *page = sg_page_iter_page(&sg_iter);
650
651                 if (remain <= 0)
652                         break;
653
654                 /* Operation in this page
655                  *
656                  * shmem_page_offset = offset within page in shmem file
657                  * page_length = bytes to copy for this page
658                  */
659                 shmem_page_offset = offset_in_page(offset);
660                 page_length = remain;
661                 if ((shmem_page_offset + page_length) > PAGE_SIZE)
662                         page_length = PAGE_SIZE - shmem_page_offset;
663
664                 page_do_bit17_swizzling = obj_do_bit17_swizzling &&
665                         (page_to_phys(page) & (1 << 17)) != 0;
666
667                 ret = shmem_pread_fast(page, shmem_page_offset, page_length,
668                                        user_data, page_do_bit17_swizzling,
669                                        needs_clflush);
670                 if (ret == 0)
671                         goto next_page;
672
673                 mutex_unlock(&dev->struct_mutex);
674
675                 if (likely(!i915.prefault_disable) && !prefaulted) {
676                         ret = fault_in_multipages_writeable(user_data, remain);
677                         /* Userspace is tricking us, but we've already clobbered
678                          * its pages with the prefault and promised to write the
679                          * data up to the first fault. Hence ignore any errors
680                          * and just continue. */
681                         (void)ret;
682                         prefaulted = 1;
683                 }
684
685                 ret = shmem_pread_slow(page, shmem_page_offset, page_length,
686                                        user_data, page_do_bit17_swizzling,
687                                        needs_clflush);
688
689                 mutex_lock(&dev->struct_mutex);
690
691                 if (ret)
692                         goto out;
693
694 next_page:
695                 remain -= page_length;
696                 user_data += page_length;
697                 offset += page_length;
698         }
699
700 out:
701         i915_gem_object_unpin_pages(obj);
702
703         return ret;
704 }
705
706 /**
707  * Reads data from the object referenced by handle.
708  *
709  * On error, the contents of *data are undefined.
710  */
711 int
712 i915_gem_pread_ioctl(struct drm_device *dev, void *data,
713                      struct drm_file *file)
714 {
715         struct drm_i915_gem_pread *args = data;
716         struct drm_i915_gem_object *obj;
717         int ret = 0;
718
719         if (args->size == 0)
720                 return 0;
721
722         if (!access_ok(VERIFY_WRITE,
723                        to_user_ptr(args->data_ptr),
724                        args->size))
725                 return -EFAULT;
726
727         ret = i915_mutex_lock_interruptible(dev);
728         if (ret)
729                 return ret;
730
731         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
732         if (&obj->base == NULL) {
733                 ret = -ENOENT;
734                 goto unlock;
735         }
736
737         /* Bounds check source.  */
738         if (args->offset > obj->base.size ||
739             args->size > obj->base.size - args->offset) {
740                 ret = -EINVAL;
741                 goto out;
742         }
743
744         /* prime objects have no backing filp to GEM pread/pwrite
745          * pages from.
746          */
747         if (!obj->base.filp) {
748                 ret = -EINVAL;
749                 goto out;
750         }
751
752         trace_i915_gem_object_pread(obj, args->offset, args->size);
753
754         ret = i915_gem_shmem_pread(dev, obj, args, file);
755
756 out:
757         drm_gem_object_unreference(&obj->base);
758 unlock:
759         mutex_unlock(&dev->struct_mutex);
760         return ret;
761 }
762
763 /* This is the fast write path which cannot handle
764  * page faults in the source data
765  */
766
767 static inline int
768 fast_user_write(struct io_mapping *mapping,
769                 loff_t page_base, int page_offset,
770                 char __user *user_data,
771                 int length)
772 {
773         void __iomem *vaddr_atomic;
774         void *vaddr;
775         unsigned long unwritten;
776
777         vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base);
778         /* We can use the cpu mem copy function because this is X86. */
779         vaddr = (void __force*)vaddr_atomic + page_offset;
780         unwritten = __copy_from_user_inatomic_nocache(vaddr,
781                                                       user_data, length);
782         io_mapping_unmap_atomic(vaddr_atomic);
783         return unwritten;
784 }
785
786 /**
787  * This is the fast pwrite path, where we copy the data directly from the
788  * user into the GTT, uncached.
789  */
790 static int
791 i915_gem_gtt_pwrite_fast(struct drm_device *dev,
792                          struct drm_i915_gem_object *obj,
793                          struct drm_i915_gem_pwrite *args,
794                          struct drm_file *file)
795 {
796         struct drm_i915_private *dev_priv = dev->dev_private;
797         ssize_t remain;
798         loff_t offset, page_base;
799         char __user *user_data;
800         int page_offset, page_length, ret;
801
802         ret = i915_gem_obj_ggtt_pin(obj, 0, PIN_MAPPABLE | PIN_NONBLOCK);
803         if (ret)
804                 goto out;
805
806         ret = i915_gem_object_set_to_gtt_domain(obj, true);
807         if (ret)
808                 goto out_unpin;
809
810         ret = i915_gem_object_put_fence(obj);
811         if (ret)
812                 goto out_unpin;
813
814         user_data = to_user_ptr(args->data_ptr);
815         remain = args->size;
816
817         offset = i915_gem_obj_ggtt_offset(obj) + args->offset;
818
819         while (remain > 0) {
820                 /* Operation in this page
821                  *
822                  * page_base = page offset within aperture
823                  * page_offset = offset within page
824                  * page_length = bytes to copy for this page
825                  */
826                 page_base = offset & PAGE_MASK;
827                 page_offset = offset_in_page(offset);
828                 page_length = remain;
829                 if ((page_offset + remain) > PAGE_SIZE)
830                         page_length = PAGE_SIZE - page_offset;
831
832                 /* If we get a fault while copying data, then (presumably) our
833                  * source page isn't available.  Return the error and we'll
834                  * retry in the slow path.
835                  */
836                 if (fast_user_write(dev_priv->gtt.mappable, page_base,
837                                     page_offset, user_data, page_length)) {
838                         ret = -EFAULT;
839                         goto out_unpin;
840                 }
841
842                 remain -= page_length;
843                 user_data += page_length;
844                 offset += page_length;
845         }
846
847 out_unpin:
848         i915_gem_object_ggtt_unpin(obj);
849 out:
850         return ret;
851 }
852
853 /* Per-page copy function for the shmem pwrite fastpath.
854  * Flushes invalid cachelines before writing to the target if
855  * needs_clflush_before is set and flushes out any written cachelines after
856  * writing if needs_clflush is set. */
857 static int
858 shmem_pwrite_fast(struct page *page, int shmem_page_offset, int page_length,
859                   char __user *user_data,
860                   bool page_do_bit17_swizzling,
861                   bool needs_clflush_before,
862                   bool needs_clflush_after)
863 {
864         char *vaddr;
865         int ret;
866
867         if (unlikely(page_do_bit17_swizzling))
868                 return -EINVAL;
869
870         vaddr = kmap_atomic(page);
871         if (needs_clflush_before)
872                 drm_clflush_virt_range(vaddr + shmem_page_offset,
873                                        page_length);
874         ret = __copy_from_user_inatomic(vaddr + shmem_page_offset,
875                                         user_data, page_length);
876         if (needs_clflush_after)
877                 drm_clflush_virt_range(vaddr + shmem_page_offset,
878                                        page_length);
879         kunmap_atomic(vaddr);
880
881         return ret ? -EFAULT : 0;
882 }
883
884 /* Only difference to the fast-path function is that this can handle bit17
885  * and uses non-atomic copy and kmap functions. */
886 static int
887 shmem_pwrite_slow(struct page *page, int shmem_page_offset, int page_length,
888                   char __user *user_data,
889                   bool page_do_bit17_swizzling,
890                   bool needs_clflush_before,
891                   bool needs_clflush_after)
892 {
893         char *vaddr;
894         int ret;
895
896         vaddr = kmap(page);
897         if (unlikely(needs_clflush_before || page_do_bit17_swizzling))
898                 shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
899                                              page_length,
900                                              page_do_bit17_swizzling);
901         if (page_do_bit17_swizzling)
902                 ret = __copy_from_user_swizzled(vaddr, shmem_page_offset,
903                                                 user_data,
904                                                 page_length);
905         else
906                 ret = __copy_from_user(vaddr + shmem_page_offset,
907                                        user_data,
908                                        page_length);
909         if (needs_clflush_after)
910                 shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
911                                              page_length,
912                                              page_do_bit17_swizzling);
913         kunmap(page);
914
915         return ret ? -EFAULT : 0;
916 }
917
918 static int
919 i915_gem_shmem_pwrite(struct drm_device *dev,
920                       struct drm_i915_gem_object *obj,
921                       struct drm_i915_gem_pwrite *args,
922                       struct drm_file *file)
923 {
924         ssize_t remain;
925         loff_t offset;
926         char __user *user_data;
927         int shmem_page_offset, page_length, ret = 0;
928         int obj_do_bit17_swizzling, page_do_bit17_swizzling;
929         int hit_slowpath = 0;
930         int needs_clflush_after = 0;
931         int needs_clflush_before = 0;
932         struct sg_page_iter sg_iter;
933
934         user_data = to_user_ptr(args->data_ptr);
935         remain = args->size;
936
937         obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
938
939         if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
940                 /* If we're not in the cpu write domain, set ourself into the gtt
941                  * write domain and manually flush cachelines (if required). This
942                  * optimizes for the case when the gpu will use the data
943                  * right away and we therefore have to clflush anyway. */
944                 needs_clflush_after = cpu_write_needs_clflush(obj);
945                 ret = i915_gem_object_wait_rendering(obj, false);
946                 if (ret)
947                         return ret;
948
949                 i915_gem_object_retire(obj);
950         }
951         /* Same trick applies to invalidate partially written cachelines read
952          * before writing. */
953         if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0)
954                 needs_clflush_before =
955                         !cpu_cache_is_coherent(dev, obj->cache_level);
956
957         ret = i915_gem_object_get_pages(obj);
958         if (ret)
959                 return ret;
960
961         i915_gem_object_pin_pages(obj);
962
963         offset = args->offset;
964         obj->dirty = 1;
965
966         for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents,
967                          offset >> PAGE_SHIFT) {
968                 struct page *page = sg_page_iter_page(&sg_iter);
969                 int partial_cacheline_write;
970
971                 if (remain <= 0)
972                         break;
973
974                 /* Operation in this page
975                  *
976                  * shmem_page_offset = offset within page in shmem file
977                  * page_length = bytes to copy for this page
978                  */
979                 shmem_page_offset = offset_in_page(offset);
980
981                 page_length = remain;
982                 if ((shmem_page_offset + page_length) > PAGE_SIZE)
983                         page_length = PAGE_SIZE - shmem_page_offset;
984
985                 /* If we don't overwrite a cacheline completely we need to be
986                  * careful to have up-to-date data by first clflushing. Don't
987                  * overcomplicate things and flush the entire patch. */
988                 partial_cacheline_write = needs_clflush_before &&
989                         ((shmem_page_offset | page_length)
990                                 & (boot_cpu_data.x86_clflush_size - 1));
991
992                 page_do_bit17_swizzling = obj_do_bit17_swizzling &&
993                         (page_to_phys(page) & (1 << 17)) != 0;
994
995                 ret = shmem_pwrite_fast(page, shmem_page_offset, page_length,
996                                         user_data, page_do_bit17_swizzling,
997                                         partial_cacheline_write,
998                                         needs_clflush_after);
999                 if (ret == 0)
1000                         goto next_page;
1001
1002                 hit_slowpath = 1;
1003                 mutex_unlock(&dev->struct_mutex);
1004                 ret = shmem_pwrite_slow(page, shmem_page_offset, page_length,
1005                                         user_data, page_do_bit17_swizzling,
1006                                         partial_cacheline_write,
1007                                         needs_clflush_after);
1008
1009                 mutex_lock(&dev->struct_mutex);
1010
1011                 if (ret)
1012                         goto out;
1013
1014 next_page:
1015                 remain -= page_length;
1016                 user_data += page_length;
1017                 offset += page_length;
1018         }
1019
1020 out:
1021         i915_gem_object_unpin_pages(obj);
1022
1023         if (hit_slowpath) {
1024                 /*
1025                  * Fixup: Flush cpu caches in case we didn't flush the dirty
1026                  * cachelines in-line while writing and the object moved
1027                  * out of the cpu write domain while we've dropped the lock.
1028                  */
1029                 if (!needs_clflush_after &&
1030                     obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
1031                         if (i915_gem_clflush_object(obj, obj->pin_display))
1032                                 i915_gem_chipset_flush(dev);
1033                 }
1034         }
1035
1036         if (needs_clflush_after)
1037                 i915_gem_chipset_flush(dev);
1038
1039         return ret;
1040 }
1041
1042 /**
1043  * Writes data to the object referenced by handle.
1044  *
1045  * On error, the contents of the buffer that were to be modified are undefined.
1046  */
1047 int
1048 i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
1049                       struct drm_file *file)
1050 {
1051         struct drm_i915_gem_pwrite *args = data;
1052         struct drm_i915_gem_object *obj;
1053         int ret;
1054
1055         if (args->size == 0)
1056                 return 0;
1057
1058         if (!access_ok(VERIFY_READ,
1059                        to_user_ptr(args->data_ptr),
1060                        args->size))
1061                 return -EFAULT;
1062
1063         if (likely(!i915.prefault_disable)) {
1064                 ret = fault_in_multipages_readable(to_user_ptr(args->data_ptr),
1065                                                    args->size);
1066                 if (ret)
1067                         return -EFAULT;
1068         }
1069
1070         ret = i915_mutex_lock_interruptible(dev);
1071         if (ret)
1072                 return ret;
1073
1074         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
1075         if (&obj->base == NULL) {
1076                 ret = -ENOENT;
1077                 goto unlock;
1078         }
1079
1080         /* Bounds check destination. */
1081         if (args->offset > obj->base.size ||
1082             args->size > obj->base.size - args->offset) {
1083                 ret = -EINVAL;
1084                 goto out;
1085         }
1086
1087         /* prime objects have no backing filp to GEM pread/pwrite
1088          * pages from.
1089          */
1090         if (!obj->base.filp) {
1091                 ret = -EINVAL;
1092                 goto out;
1093         }
1094
1095         trace_i915_gem_object_pwrite(obj, args->offset, args->size);
1096
1097         ret = -EFAULT;
1098         /* We can only do the GTT pwrite on untiled buffers, as otherwise
1099          * it would end up going through the fenced access, and we'll get
1100          * different detiling behavior between reading and writing.
1101          * pread/pwrite currently are reading and writing from the CPU
1102          * perspective, requiring manual detiling by the client.
1103          */
1104         if (obj->tiling_mode == I915_TILING_NONE &&
1105             obj->base.write_domain != I915_GEM_DOMAIN_CPU &&
1106             cpu_write_needs_clflush(obj)) {
1107                 ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file);
1108                 /* Note that the gtt paths might fail with non-page-backed user
1109                  * pointers (e.g. gtt mappings when moving data between
1110                  * textures). Fallback to the shmem path in that case. */
1111         }
1112
1113         if (ret == -EFAULT || ret == -ENOSPC) {
1114                 if (obj->phys_handle)
1115                         ret = i915_gem_phys_pwrite(obj, args, file);
1116                 else
1117                         ret = i915_gem_shmem_pwrite(dev, obj, args, file);
1118         }
1119
1120 out:
1121         drm_gem_object_unreference(&obj->base);
1122 unlock:
1123         mutex_unlock(&dev->struct_mutex);
1124         return ret;
1125 }
1126
1127 int
1128 i915_gem_check_wedge(struct i915_gpu_error *error,
1129                      bool interruptible)
1130 {
1131         if (i915_reset_in_progress(error)) {
1132                 /* Non-interruptible callers can't handle -EAGAIN, hence return
1133                  * -EIO unconditionally for these. */
1134                 if (!interruptible)
1135                         return -EIO;
1136
1137                 /* Recovery complete, but the reset failed ... */
1138                 if (i915_terminally_wedged(error))
1139                         return -EIO;
1140
1141                 /*
1142                  * Check if GPU Reset is in progress - we need intel_ring_begin
1143                  * to work properly to reinit the hw state while the gpu is
1144                  * still marked as reset-in-progress. Handle this with a flag.
1145                  */
1146                 if (!error->reload_in_reset)
1147                         return -EAGAIN;
1148         }
1149
1150         return 0;
1151 }
1152
1153 /*
1154  * Compare arbitrary request against outstanding lazy request. Emit on match.
1155  */
1156 int
1157 i915_gem_check_olr(struct drm_i915_gem_request *req)
1158 {
1159         int ret;
1160
1161         WARN_ON(!mutex_is_locked(&req->ring->dev->struct_mutex));
1162
1163         ret = 0;
1164         if (req == req->ring->outstanding_lazy_request)
1165                 ret = i915_add_request(req->ring);
1166
1167         return ret;
1168 }
1169
1170 static void fake_irq(unsigned long data)
1171 {
1172         wake_up_process((struct task_struct *)data);
1173 }
1174
1175 static bool missed_irq(struct drm_i915_private *dev_priv,
1176                        struct intel_engine_cs *ring)
1177 {
1178         return test_bit(ring->id, &dev_priv->gpu_error.missed_irq_rings);
1179 }
1180
1181 static bool can_wait_boost(struct drm_i915_file_private *file_priv)
1182 {
1183         if (file_priv == NULL)
1184                 return true;
1185
1186         return !atomic_xchg(&file_priv->rps_wait_boost, true);
1187 }
1188
1189 /**
1190  * __i915_wait_request - wait until execution of request has finished
1191  * @req: duh!
1192  * @reset_counter: reset sequence associated with the given request
1193  * @interruptible: do an interruptible wait (normally yes)
1194  * @timeout: in - how long to wait (NULL forever); out - how much time remaining
1195  *
1196  * Note: It is of utmost importance that the passed in seqno and reset_counter
1197  * values have been read by the caller in an smp safe manner. Where read-side
1198  * locks are involved, it is sufficient to read the reset_counter before
1199  * unlocking the lock that protects the seqno. For lockless tricks, the
1200  * reset_counter _must_ be read before, and an appropriate smp_rmb must be
1201  * inserted.
1202  *
1203  * Returns 0 if the request was found within the alloted time. Else returns the
1204  * errno with remaining time filled in timeout argument.
1205  */
1206 int __i915_wait_request(struct drm_i915_gem_request *req,
1207                         unsigned reset_counter,
1208                         bool interruptible,
1209                         s64 *timeout,
1210                         struct drm_i915_file_private *file_priv)
1211 {
1212         struct intel_engine_cs *ring = i915_gem_request_get_ring(req);
1213         struct drm_device *dev = ring->dev;
1214         struct drm_i915_private *dev_priv = dev->dev_private;
1215         const bool irq_test_in_progress =
1216                 ACCESS_ONCE(dev_priv->gpu_error.test_irq_rings) & intel_ring_flag(ring);
1217         DEFINE_WAIT(wait);
1218         unsigned long timeout_expire;
1219         s64 before, now;
1220         int ret;
1221
1222         WARN(!intel_irqs_enabled(dev_priv), "IRQs disabled");
1223
1224         if (i915_gem_request_completed(req, true))
1225                 return 0;
1226
1227         timeout_expire = timeout ?
1228                 jiffies + nsecs_to_jiffies_timeout((u64)*timeout) : 0;
1229
1230         if (INTEL_INFO(dev)->gen >= 6 && ring->id == RCS && can_wait_boost(file_priv)) {
1231                 gen6_rps_boost(dev_priv);
1232                 if (file_priv)
1233                         mod_delayed_work(dev_priv->wq,
1234                                          &file_priv->mm.idle_work,
1235                                          msecs_to_jiffies(100));
1236         }
1237
1238         if (!irq_test_in_progress && WARN_ON(!ring->irq_get(ring)))
1239                 return -ENODEV;
1240
1241         /* Record current time in case interrupted by signal, or wedged */
1242         trace_i915_gem_request_wait_begin(req);
1243         before = ktime_get_raw_ns();
1244         for (;;) {
1245                 struct timer_list timer;
1246
1247                 prepare_to_wait(&ring->irq_queue, &wait,
1248                                 interruptible ? TASK_INTERRUPTIBLE : TASK_UNINTERRUPTIBLE);
1249
1250                 /* We need to check whether any gpu reset happened in between
1251                  * the caller grabbing the seqno and now ... */
1252                 if (reset_counter != atomic_read(&dev_priv->gpu_error.reset_counter)) {
1253                         /* ... but upgrade the -EAGAIN to an -EIO if the gpu
1254                          * is truely gone. */
1255                         ret = i915_gem_check_wedge(&dev_priv->gpu_error, interruptible);
1256                         if (ret == 0)
1257                                 ret = -EAGAIN;
1258                         break;
1259                 }
1260
1261                 if (i915_gem_request_completed(req, false)) {
1262                         ret = 0;
1263                         break;
1264                 }
1265
1266                 if (interruptible && signal_pending(current)) {
1267                         ret = -ERESTARTSYS;
1268                         break;
1269                 }
1270
1271                 if (timeout && time_after_eq(jiffies, timeout_expire)) {
1272                         ret = -ETIME;
1273                         break;
1274                 }
1275
1276                 timer.function = NULL;
1277                 if (timeout || missed_irq(dev_priv, ring)) {
1278                         unsigned long expire;
1279
1280                         setup_timer_on_stack(&timer, fake_irq, (unsigned long)current);
1281                         expire = missed_irq(dev_priv, ring) ? jiffies + 1 : timeout_expire;
1282                         mod_timer(&timer, expire);
1283                 }
1284
1285                 io_schedule();
1286
1287                 if (timer.function) {
1288                         del_singleshot_timer_sync(&timer);
1289                         destroy_timer_on_stack(&timer);
1290                 }
1291         }
1292         now = ktime_get_raw_ns();
1293         trace_i915_gem_request_wait_end(req);
1294
1295         if (!irq_test_in_progress)
1296                 ring->irq_put(ring);
1297
1298         finish_wait(&ring->irq_queue, &wait);
1299
1300         if (timeout) {
1301                 s64 tres = *timeout - (now - before);
1302
1303                 *timeout = tres < 0 ? 0 : tres;
1304
1305                 /*
1306                  * Apparently ktime isn't accurate enough and occasionally has a
1307                  * bit of mismatch in the jiffies<->nsecs<->ktime loop. So patch
1308                  * things up to make the test happy. We allow up to 1 jiffy.
1309                  *
1310                  * This is a regrssion from the timespec->ktime conversion.
1311                  */
1312                 if (ret == -ETIME && *timeout < jiffies_to_usecs(1)*1000)
1313                         *timeout = 0;
1314         }
1315
1316         return ret;
1317 }
1318
1319 /**
1320  * Waits for a request to be signaled, and cleans up the
1321  * request and object lists appropriately for that event.
1322  */
1323 int
1324 i915_wait_request(struct drm_i915_gem_request *req)
1325 {
1326         struct drm_device *dev;
1327         struct drm_i915_private *dev_priv;
1328         bool interruptible;
1329         unsigned reset_counter;
1330         int ret;
1331
1332         BUG_ON(req == NULL);
1333
1334         dev = req->ring->dev;
1335         dev_priv = dev->dev_private;
1336         interruptible = dev_priv->mm.interruptible;
1337
1338         BUG_ON(!mutex_is_locked(&dev->struct_mutex));
1339
1340         ret = i915_gem_check_wedge(&dev_priv->gpu_error, interruptible);
1341         if (ret)
1342                 return ret;
1343
1344         ret = i915_gem_check_olr(req);
1345         if (ret)
1346                 return ret;
1347
1348         reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
1349         i915_gem_request_reference(req);
1350         ret = __i915_wait_request(req, reset_counter,
1351                                   interruptible, NULL, NULL);
1352         i915_gem_request_unreference(req);
1353         return ret;
1354 }
1355
1356 static int
1357 i915_gem_object_wait_rendering__tail(struct drm_i915_gem_object *obj)
1358 {
1359         if (!obj->active)
1360                 return 0;
1361
1362         /* Manually manage the write flush as we may have not yet
1363          * retired the buffer.
1364          *
1365          * Note that the last_write_req is always the earlier of
1366          * the two (read/write) requests, so if we haved successfully waited,
1367          * we know we have passed the last write.
1368          */
1369         i915_gem_request_assign(&obj->last_write_req, NULL);
1370
1371         return 0;
1372 }
1373
1374 /**
1375  * Ensures that all rendering to the object has completed and the object is
1376  * safe to unbind from the GTT or access from the CPU.
1377  */
1378 static __must_check int
1379 i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj,
1380                                bool readonly)
1381 {
1382         struct drm_i915_gem_request *req;
1383         int ret;
1384
1385         req = readonly ? obj->last_write_req : obj->last_read_req;
1386         if (!req)
1387                 return 0;
1388
1389         ret = i915_wait_request(req);
1390         if (ret)
1391                 return ret;
1392
1393         return i915_gem_object_wait_rendering__tail(obj);
1394 }
1395
1396 /* A nonblocking variant of the above wait. This is a highly dangerous routine
1397  * as the object state may change during this call.
1398  */
1399 static __must_check int
1400 i915_gem_object_wait_rendering__nonblocking(struct drm_i915_gem_object *obj,
1401                                             struct drm_i915_file_private *file_priv,
1402                                             bool readonly)
1403 {
1404         struct drm_i915_gem_request *req;
1405         struct drm_device *dev = obj->base.dev;
1406         struct drm_i915_private *dev_priv = dev->dev_private;
1407         unsigned reset_counter;
1408         int ret;
1409
1410         BUG_ON(!mutex_is_locked(&dev->struct_mutex));
1411         BUG_ON(!dev_priv->mm.interruptible);
1412
1413         req = readonly ? obj->last_write_req : obj->last_read_req;
1414         if (!req)
1415                 return 0;
1416
1417         ret = i915_gem_check_wedge(&dev_priv->gpu_error, true);
1418         if (ret)
1419                 return ret;
1420
1421         ret = i915_gem_check_olr(req);
1422         if (ret)
1423                 return ret;
1424
1425         reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
1426         i915_gem_request_reference(req);
1427         mutex_unlock(&dev->struct_mutex);
1428         ret = __i915_wait_request(req, reset_counter, true, NULL, file_priv);
1429         mutex_lock(&dev->struct_mutex);
1430         i915_gem_request_unreference(req);
1431         if (ret)
1432                 return ret;
1433
1434         return i915_gem_object_wait_rendering__tail(obj);
1435 }
1436
1437 /**
1438  * Called when user space prepares to use an object with the CPU, either
1439  * through the mmap ioctl's mapping or a GTT mapping.
1440  */
1441 int
1442 i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
1443                           struct drm_file *file)
1444 {
1445         struct drm_i915_gem_set_domain *args = data;
1446         struct drm_i915_gem_object *obj;
1447         uint32_t read_domains = args->read_domains;
1448         uint32_t write_domain = args->write_domain;
1449         int ret;
1450
1451         /* Only handle setting domains to types used by the CPU. */
1452         if (write_domain & I915_GEM_GPU_DOMAINS)
1453                 return -EINVAL;
1454
1455         if (read_domains & I915_GEM_GPU_DOMAINS)
1456                 return -EINVAL;
1457
1458         /* Having something in the write domain implies it's in the read
1459          * domain, and only that read domain.  Enforce that in the request.
1460          */
1461         if (write_domain != 0 && read_domains != write_domain)
1462                 return -EINVAL;
1463
1464         ret = i915_mutex_lock_interruptible(dev);
1465         if (ret)
1466                 return ret;
1467
1468         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
1469         if (&obj->base == NULL) {
1470                 ret = -ENOENT;
1471                 goto unlock;
1472         }
1473
1474         /* Try to flush the object off the GPU without holding the lock.
1475          * We will repeat the flush holding the lock in the normal manner
1476          * to catch cases where we are gazumped.
1477          */
1478         ret = i915_gem_object_wait_rendering__nonblocking(obj,
1479                                                           file->driver_priv,
1480                                                           !write_domain);
1481         if (ret)
1482                 goto unref;
1483
1484         if (read_domains & I915_GEM_DOMAIN_GTT) {
1485                 ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
1486
1487                 /* Silently promote "you're not bound, there was nothing to do"
1488                  * to success, since the client was just asking us to
1489                  * make sure everything was done.
1490                  */
1491                 if (ret == -EINVAL)
1492                         ret = 0;
1493         } else {
1494                 ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
1495         }
1496
1497 unref:
1498         drm_gem_object_unreference(&obj->base);
1499 unlock:
1500         mutex_unlock(&dev->struct_mutex);
1501         return ret;
1502 }
1503
1504 /**
1505  * Called when user space has done writes to this buffer
1506  */
1507 int
1508 i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
1509                          struct drm_file *file)
1510 {
1511         struct drm_i915_gem_sw_finish *args = data;
1512         struct drm_i915_gem_object *obj;
1513         int ret = 0;
1514
1515         ret = i915_mutex_lock_interruptible(dev);
1516         if (ret)
1517                 return ret;
1518
1519         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
1520         if (&obj->base == NULL) {
1521                 ret = -ENOENT;
1522                 goto unlock;
1523         }
1524
1525         /* Pinned buffers may be scanout, so flush the cache */
1526         if (obj->pin_display)
1527                 i915_gem_object_flush_cpu_write_domain(obj, true);
1528
1529         drm_gem_object_unreference(&obj->base);
1530 unlock:
1531         mutex_unlock(&dev->struct_mutex);
1532         return ret;
1533 }
1534
1535 /**
1536  * Maps the contents of an object, returning the address it is mapped
1537  * into.
1538  *
1539  * While the mapping holds a reference on the contents of the object, it doesn't
1540  * imply a ref on the object itself.
1541  *
1542  * IMPORTANT:
1543  *
1544  * DRM driver writers who look a this function as an example for how to do GEM
1545  * mmap support, please don't implement mmap support like here. The modern way
1546  * to implement DRM mmap support is with an mmap offset ioctl (like
1547  * i915_gem_mmap_gtt) and then using the mmap syscall on the DRM fd directly.
1548  * That way debug tooling like valgrind will understand what's going on, hiding
1549  * the mmap call in a driver private ioctl will break that. The i915 driver only
1550  * does cpu mmaps this way because we didn't know better.
1551  */
1552 int
1553 i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
1554                     struct drm_file *file)
1555 {
1556         struct drm_i915_gem_mmap *args = data;
1557         struct drm_gem_object *obj;
1558         unsigned long addr;
1559
1560         obj = drm_gem_object_lookup(dev, file, args->handle);
1561         if (obj == NULL)
1562                 return -ENOENT;
1563
1564         /* prime objects have no backing filp to GEM mmap
1565          * pages from.
1566          */
1567         if (!obj->filp) {
1568                 drm_gem_object_unreference_unlocked(obj);
1569                 return -EINVAL;
1570         }
1571
1572         addr = vm_mmap(obj->filp, 0, args->size,
1573                        PROT_READ | PROT_WRITE, MAP_SHARED,
1574                        args->offset);
1575         drm_gem_object_unreference_unlocked(obj);
1576         if (IS_ERR((void *)addr))
1577                 return addr;
1578
1579         args->addr_ptr = (uint64_t) addr;
1580
1581         return 0;
1582 }
1583
1584 /**
1585  * i915_gem_fault - fault a page into the GTT
1586  * vma: VMA in question
1587  * vmf: fault info
1588  *
1589  * The fault handler is set up by drm_gem_mmap() when a object is GTT mapped
1590  * from userspace.  The fault handler takes care of binding the object to
1591  * the GTT (if needed), allocating and programming a fence register (again,
1592  * only if needed based on whether the old reg is still valid or the object
1593  * is tiled) and inserting a new PTE into the faulting process.
1594  *
1595  * Note that the faulting process may involve evicting existing objects
1596  * from the GTT and/or fence registers to make room.  So performance may
1597  * suffer if the GTT working set is large or there are few fence registers
1598  * left.
1599  */
1600 int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1601 {
1602         struct drm_i915_gem_object *obj = to_intel_bo(vma->vm_private_data);
1603         struct drm_device *dev = obj->base.dev;
1604         struct drm_i915_private *dev_priv = dev->dev_private;
1605         pgoff_t page_offset;
1606         unsigned long pfn;
1607         int ret = 0;
1608         bool write = !!(vmf->flags & FAULT_FLAG_WRITE);
1609
1610         intel_runtime_pm_get(dev_priv);
1611
1612         /* We don't use vmf->pgoff since that has the fake offset */
1613         page_offset = ((unsigned long)vmf->virtual_address - vma->vm_start) >>
1614                 PAGE_SHIFT;
1615
1616         ret = i915_mutex_lock_interruptible(dev);
1617         if (ret)
1618                 goto out;
1619
1620         trace_i915_gem_object_fault(obj, page_offset, true, write);
1621
1622         /* Try to flush the object off the GPU first without holding the lock.
1623          * Upon reacquiring the lock, we will perform our sanity checks and then
1624          * repeat the flush holding the lock in the normal manner to catch cases
1625          * where we are gazumped.
1626          */
1627         ret = i915_gem_object_wait_rendering__nonblocking(obj, NULL, !write);
1628         if (ret)
1629                 goto unlock;
1630
1631         /* Access to snoopable pages through the GTT is incoherent. */
1632         if (obj->cache_level != I915_CACHE_NONE && !HAS_LLC(dev)) {
1633                 ret = -EFAULT;
1634                 goto unlock;
1635         }
1636
1637         /* Now bind it into the GTT if needed */
1638         ret = i915_gem_obj_ggtt_pin(obj, 0, PIN_MAPPABLE);
1639         if (ret)
1640                 goto unlock;
1641
1642         ret = i915_gem_object_set_to_gtt_domain(obj, write);
1643         if (ret)
1644                 goto unpin;
1645
1646         ret = i915_gem_object_get_fence(obj);
1647         if (ret)
1648                 goto unpin;
1649
1650         /* Finally, remap it using the new GTT offset */
1651         pfn = dev_priv->gtt.mappable_base + i915_gem_obj_ggtt_offset(obj);
1652         pfn >>= PAGE_SHIFT;
1653
1654         if (!obj->fault_mappable) {
1655                 unsigned long size = min_t(unsigned long,
1656                                            vma->vm_end - vma->vm_start,
1657                                            obj->base.size);
1658                 int i;
1659
1660                 for (i = 0; i < size >> PAGE_SHIFT; i++) {
1661                         ret = vm_insert_pfn(vma,
1662                                             (unsigned long)vma->vm_start + i * PAGE_SIZE,
1663                                             pfn + i);
1664                         if (ret)
1665                                 break;
1666                 }
1667
1668                 obj->fault_mappable = true;
1669         } else
1670                 ret = vm_insert_pfn(vma,
1671                                     (unsigned long)vmf->virtual_address,
1672                                     pfn + page_offset);
1673 unpin:
1674         i915_gem_object_ggtt_unpin(obj);
1675 unlock:
1676         mutex_unlock(&dev->struct_mutex);
1677 out:
1678         switch (ret) {
1679         case -EIO:
1680                 /*
1681                  * We eat errors when the gpu is terminally wedged to avoid
1682                  * userspace unduly crashing (gl has no provisions for mmaps to
1683                  * fail). But any other -EIO isn't ours (e.g. swap in failure)
1684                  * and so needs to be reported.
1685                  */
1686                 if (!i915_terminally_wedged(&dev_priv->gpu_error)) {
1687                         ret = VM_FAULT_SIGBUS;
1688                         break;
1689                 }
1690         case -EAGAIN:
1691                 /*
1692                  * EAGAIN means the gpu is hung and we'll wait for the error
1693                  * handler to reset everything when re-faulting in
1694                  * i915_mutex_lock_interruptible.
1695                  */
1696         case 0:
1697         case -ERESTARTSYS:
1698         case -EINTR:
1699         case -EBUSY:
1700                 /*
1701                  * EBUSY is ok: this just means that another thread
1702                  * already did the job.
1703                  */
1704                 ret = VM_FAULT_NOPAGE;
1705                 break;
1706         case -ENOMEM:
1707                 ret = VM_FAULT_OOM;
1708                 break;
1709         case -ENOSPC:
1710         case -EFAULT:
1711                 ret = VM_FAULT_SIGBUS;
1712                 break;
1713         default:
1714                 WARN_ONCE(ret, "unhandled error in i915_gem_fault: %i\n", ret);
1715                 ret = VM_FAULT_SIGBUS;
1716                 break;
1717         }
1718
1719         intel_runtime_pm_put(dev_priv);
1720         return ret;
1721 }
1722
1723 /**
1724  * i915_gem_release_mmap - remove physical page mappings
1725  * @obj: obj in question
1726  *
1727  * Preserve the reservation of the mmapping with the DRM core code, but
1728  * relinquish ownership of the pages back to the system.
1729  *
1730  * It is vital that we remove the page mapping if we have mapped a tiled
1731  * object through the GTT and then lose the fence register due to
1732  * resource pressure. Similarly if the object has been moved out of the
1733  * aperture, than pages mapped into userspace must be revoked. Removing the
1734  * mapping will then trigger a page fault on the next user access, allowing
1735  * fixup by i915_gem_fault().
1736  */
1737 void
1738 i915_gem_release_mmap(struct drm_i915_gem_object *obj)
1739 {
1740         if (!obj->fault_mappable)
1741                 return;
1742
1743         drm_vma_node_unmap(&obj->base.vma_node,
1744                            obj->base.dev->anon_inode->i_mapping);
1745         obj->fault_mappable = false;
1746 }
1747
1748 void
1749 i915_gem_release_all_mmaps(struct drm_i915_private *dev_priv)
1750 {
1751         struct drm_i915_gem_object *obj;
1752
1753         list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list)
1754                 i915_gem_release_mmap(obj);
1755 }
1756
1757 uint32_t
1758 i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, int tiling_mode)
1759 {
1760         uint32_t gtt_size;
1761
1762         if (INTEL_INFO(dev)->gen >= 4 ||
1763             tiling_mode == I915_TILING_NONE)
1764                 return size;
1765
1766         /* Previous chips need a power-of-two fence region when tiling */
1767         if (INTEL_INFO(dev)->gen == 3)
1768                 gtt_size = 1024*1024;
1769         else
1770                 gtt_size = 512*1024;
1771
1772         while (gtt_size < size)
1773                 gtt_size <<= 1;
1774
1775         return gtt_size;
1776 }
1777
1778 /**
1779  * i915_gem_get_gtt_alignment - return required GTT alignment for an object
1780  * @obj: object to check
1781  *
1782  * Return the required GTT alignment for an object, taking into account
1783  * potential fence register mapping.
1784  */
1785 uint32_t
1786 i915_gem_get_gtt_alignment(struct drm_device *dev, uint32_t size,
1787                            int tiling_mode, bool fenced)
1788 {
1789         /*
1790          * Minimum alignment is 4k (GTT page size), but might be greater
1791          * if a fence register is needed for the object.
1792          */
1793         if (INTEL_INFO(dev)->gen >= 4 || (!fenced && IS_G33(dev)) ||
1794             tiling_mode == I915_TILING_NONE)
1795                 return 4096;
1796
1797         /*
1798          * Previous chips need to be aligned to the size of the smallest
1799          * fence register that can contain the object.
1800          */
1801         return i915_gem_get_gtt_size(dev, size, tiling_mode);
1802 }
1803
1804 static int i915_gem_object_create_mmap_offset(struct drm_i915_gem_object *obj)
1805 {
1806         struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
1807         int ret;
1808
1809         if (drm_vma_node_has_offset(&obj->base.vma_node))
1810                 return 0;
1811
1812         dev_priv->mm.shrinker_no_lock_stealing = true;
1813
1814         ret = drm_gem_create_mmap_offset(&obj->base);
1815         if (ret != -ENOSPC)
1816                 goto out;
1817
1818         /* Badly fragmented mmap space? The only way we can recover
1819          * space is by destroying unwanted objects. We can't randomly release
1820          * mmap_offsets as userspace expects them to be persistent for the
1821          * lifetime of the objects. The closest we can is to release the
1822          * offsets on purgeable objects by truncating it and marking it purged,
1823          * which prevents userspace from ever using that object again.
1824          */
1825         i915_gem_shrink(dev_priv,
1826                         obj->base.size >> PAGE_SHIFT,
1827                         I915_SHRINK_BOUND |
1828                         I915_SHRINK_UNBOUND |
1829                         I915_SHRINK_PURGEABLE);
1830         ret = drm_gem_create_mmap_offset(&obj->base);
1831         if (ret != -ENOSPC)
1832                 goto out;
1833
1834         i915_gem_shrink_all(dev_priv);
1835         ret = drm_gem_create_mmap_offset(&obj->base);
1836 out:
1837         dev_priv->mm.shrinker_no_lock_stealing = false;
1838
1839         return ret;
1840 }
1841
1842 static void i915_gem_object_free_mmap_offset(struct drm_i915_gem_object *obj)
1843 {
1844         drm_gem_free_mmap_offset(&obj->base);
1845 }
1846
1847 int
1848 i915_gem_mmap_gtt(struct drm_file *file,
1849                   struct drm_device *dev,
1850                   uint32_t handle,
1851                   uint64_t *offset)
1852 {
1853         struct drm_i915_private *dev_priv = dev->dev_private;
1854         struct drm_i915_gem_object *obj;
1855         int ret;
1856
1857         ret = i915_mutex_lock_interruptible(dev);
1858         if (ret)
1859                 return ret;
1860
1861         obj = to_intel_bo(drm_gem_object_lookup(dev, file, handle));
1862         if (&obj->base == NULL) {
1863                 ret = -ENOENT;
1864                 goto unlock;
1865         }
1866
1867         if (obj->base.size > dev_priv->gtt.mappable_end) {
1868                 ret = -E2BIG;
1869                 goto out;
1870         }
1871
1872         if (obj->madv != I915_MADV_WILLNEED) {
1873                 DRM_DEBUG("Attempting to mmap a purgeable buffer\n");
1874                 ret = -EFAULT;
1875                 goto out;
1876         }
1877
1878         ret = i915_gem_object_create_mmap_offset(obj);
1879         if (ret)
1880                 goto out;
1881
1882         *offset = drm_vma_node_offset_addr(&obj->base.vma_node);
1883
1884 out:
1885         drm_gem_object_unreference(&obj->base);
1886 unlock:
1887         mutex_unlock(&dev->struct_mutex);
1888         return ret;
1889 }
1890
1891 /**
1892  * i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing
1893  * @dev: DRM device
1894  * @data: GTT mapping ioctl data
1895  * @file: GEM object info
1896  *
1897  * Simply returns the fake offset to userspace so it can mmap it.
1898  * The mmap call will end up in drm_gem_mmap(), which will set things
1899  * up so we can get faults in the handler above.
1900  *
1901  * The fault handler will take care of binding the object into the GTT
1902  * (since it may have been evicted to make room for something), allocating
1903  * a fence register, and mapping the appropriate aperture address into
1904  * userspace.
1905  */
1906 int
1907 i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
1908                         struct drm_file *file)
1909 {
1910         struct drm_i915_gem_mmap_gtt *args = data;
1911
1912         return i915_gem_mmap_gtt(file, dev, args->handle, &args->offset);
1913 }
1914
1915 static inline int
1916 i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj)
1917 {
1918         return obj->madv == I915_MADV_DONTNEED;
1919 }
1920
1921 /* Immediately discard the backing storage */
1922 static void
1923 i915_gem_object_truncate(struct drm_i915_gem_object *obj)
1924 {
1925         i915_gem_object_free_mmap_offset(obj);
1926
1927         if (obj->base.filp == NULL)
1928                 return;
1929
1930         /* Our goal here is to return as much of the memory as
1931          * is possible back to the system as we are called from OOM.
1932          * To do this we must instruct the shmfs to drop all of its
1933          * backing pages, *now*.
1934          */
1935         shmem_truncate_range(file_inode(obj->base.filp), 0, (loff_t)-1);
1936         obj->madv = __I915_MADV_PURGED;
1937 }
1938
1939 /* Try to discard unwanted pages */
1940 static void
1941 i915_gem_object_invalidate(struct drm_i915_gem_object *obj)
1942 {
1943         struct address_space *mapping;
1944
1945         switch (obj->madv) {
1946         case I915_MADV_DONTNEED:
1947                 i915_gem_object_truncate(obj);
1948         case __I915_MADV_PURGED:
1949                 return;
1950         }
1951
1952         if (obj->base.filp == NULL)
1953                 return;
1954
1955         mapping = file_inode(obj->base.filp)->i_mapping,
1956         invalidate_mapping_pages(mapping, 0, (loff_t)-1);
1957 }
1958
1959 static void
1960 i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj)
1961 {
1962         struct sg_page_iter sg_iter;
1963         int ret;
1964
1965         BUG_ON(obj->madv == __I915_MADV_PURGED);
1966
1967         ret = i915_gem_object_set_to_cpu_domain(obj, true);
1968         if (ret) {
1969                 /* In the event of a disaster, abandon all caches and
1970                  * hope for the best.
1971                  */
1972                 WARN_ON(ret != -EIO);
1973                 i915_gem_clflush_object(obj, true);
1974                 obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU;
1975         }
1976
1977         if (i915_gem_object_needs_bit17_swizzle(obj))
1978                 i915_gem_object_save_bit_17_swizzle(obj);
1979
1980         if (obj->madv == I915_MADV_DONTNEED)
1981                 obj->dirty = 0;
1982
1983         for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents, 0) {
1984                 struct page *page = sg_page_iter_page(&sg_iter);
1985
1986                 if (obj->dirty)
1987                         set_page_dirty(page);
1988
1989                 if (obj->madv == I915_MADV_WILLNEED)
1990                         mark_page_accessed(page);
1991
1992                 page_cache_release(page);
1993         }
1994         obj->dirty = 0;
1995
1996         sg_free_table(obj->pages);
1997         kfree(obj->pages);
1998 }
1999
2000 int
2001 i915_gem_object_put_pages(struct drm_i915_gem_object *obj)
2002 {
2003         const struct drm_i915_gem_object_ops *ops = obj->ops;
2004
2005         if (obj->pages == NULL)
2006                 return 0;
2007
2008         if (obj->pages_pin_count)
2009                 return -EBUSY;
2010
2011         BUG_ON(i915_gem_obj_bound_any(obj));
2012
2013         /* ->put_pages might need to allocate memory for the bit17 swizzle
2014          * array, hence protect them from being reaped by removing them from gtt
2015          * lists early. */
2016         list_del(&obj->global_list);
2017
2018         ops->put_pages(obj);
2019         obj->pages = NULL;
2020
2021         i915_gem_object_invalidate(obj);
2022
2023         return 0;
2024 }
2025
2026 unsigned long
2027 i915_gem_shrink(struct drm_i915_private *dev_priv,
2028                 long target, unsigned flags)
2029 {
2030         const struct {
2031                 struct list_head *list;
2032                 unsigned int bit;
2033         } phases[] = {
2034                 { &dev_priv->mm.unbound_list, I915_SHRINK_UNBOUND },
2035                 { &dev_priv->mm.bound_list, I915_SHRINK_BOUND },
2036                 { NULL, 0 },
2037         }, *phase;
2038         unsigned long count = 0;
2039
2040         /*
2041          * As we may completely rewrite the (un)bound list whilst unbinding
2042          * (due to retiring requests) we have to strictly process only
2043          * one element of the list at the time, and recheck the list
2044          * on every iteration.
2045          *
2046          * In particular, we must hold a reference whilst removing the
2047          * object as we may end up waiting for and/or retiring the objects.
2048          * This might release the final reference (held by the active list)
2049          * and result in the object being freed from under us. This is
2050          * similar to the precautions the eviction code must take whilst
2051          * removing objects.
2052          *
2053          * Also note that although these lists do not hold a reference to
2054          * the object we can safely grab one here: The final object
2055          * unreferencing and the bound_list are both protected by the
2056          * dev->struct_mutex and so we won't ever be able to observe an
2057          * object on the bound_list with a reference count equals 0.
2058          */
2059         for (phase = phases; phase->list; phase++) {
2060                 struct list_head still_in_list;
2061
2062                 if ((flags & phase->bit) == 0)
2063                         continue;
2064
2065                 INIT_LIST_HEAD(&still_in_list);
2066                 while (count < target && !list_empty(phase->list)) {
2067                         struct drm_i915_gem_object *obj;
2068                         struct i915_vma *vma, *v;
2069
2070                         obj = list_first_entry(phase->list,
2071                                                typeof(*obj), global_list);
2072                         list_move_tail(&obj->global_list, &still_in_list);
2073
2074                         if (flags & I915_SHRINK_PURGEABLE &&
2075                             !i915_gem_object_is_purgeable(obj))
2076                                 continue;
2077
2078                         drm_gem_object_reference(&obj->base);
2079
2080                         /* For the unbound phase, this should be a no-op! */
2081                         list_for_each_entry_safe(vma, v,
2082                                                  &obj->vma_list, vma_link)
2083                                 if (i915_vma_unbind(vma))
2084                                         break;
2085
2086                         if (i915_gem_object_put_pages(obj) == 0)
2087                                 count += obj->base.size >> PAGE_SHIFT;
2088
2089                         drm_gem_object_unreference(&obj->base);
2090                 }
2091                 list_splice(&still_in_list, phase->list);
2092         }
2093
2094         return count;
2095 }
2096
2097 static unsigned long
2098 i915_gem_shrink_all(struct drm_i915_private *dev_priv)
2099 {
2100         i915_gem_evict_everything(dev_priv->dev);
2101         return i915_gem_shrink(dev_priv, LONG_MAX,
2102                                I915_SHRINK_BOUND | I915_SHRINK_UNBOUND);
2103 }
2104
2105 static int
2106 i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj)
2107 {
2108         struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
2109         int page_count, i;
2110         struct address_space *mapping;
2111         struct sg_table *st;
2112         struct scatterlist *sg;
2113         struct sg_page_iter sg_iter;
2114         struct page *page;
2115         unsigned long last_pfn = 0;     /* suppress gcc warning */
2116         gfp_t gfp;
2117
2118         /* Assert that the object is not currently in any GPU domain. As it
2119          * wasn't in the GTT, there shouldn't be any way it could have been in
2120          * a GPU cache
2121          */
2122         BUG_ON(obj->base.read_domains & I915_GEM_GPU_DOMAINS);
2123         BUG_ON(obj->base.write_domain & I915_GEM_GPU_DOMAINS);
2124
2125         st = kmalloc(sizeof(*st), GFP_KERNEL);
2126         if (st == NULL)
2127                 return -ENOMEM;
2128
2129         page_count = obj->base.size / PAGE_SIZE;
2130         if (sg_alloc_table(st, page_count, GFP_KERNEL)) {
2131                 kfree(st);
2132                 return -ENOMEM;
2133         }
2134
2135         /* Get the list of pages out of our struct file.  They'll be pinned
2136          * at this point until we release them.
2137          *
2138          * Fail silently without starting the shrinker
2139          */
2140         mapping = file_inode(obj->base.filp)->i_mapping;
2141         gfp = mapping_gfp_mask(mapping);
2142         gfp |= __GFP_NORETRY | __GFP_NOWARN | __GFP_NO_KSWAPD;
2143         gfp &= ~(__GFP_IO | __GFP_WAIT);
2144         sg = st->sgl;
2145         st->nents = 0;
2146         for (i = 0; i < page_count; i++) {
2147                 page = shmem_read_mapping_page_gfp(mapping, i, gfp);
2148                 if (IS_ERR(page)) {
2149                         i915_gem_shrink(dev_priv,
2150                                         page_count,
2151                                         I915_SHRINK_BOUND |
2152                                         I915_SHRINK_UNBOUND |
2153                                         I915_SHRINK_PURGEABLE);
2154                         page = shmem_read_mapping_page_gfp(mapping, i, gfp);
2155                 }
2156                 if (IS_ERR(page)) {
2157                         /* We've tried hard to allocate the memory by reaping
2158                          * our own buffer, now let the real VM do its job and
2159                          * go down in flames if truly OOM.
2160                          */
2161                         i915_gem_shrink_all(dev_priv);
2162                         page = shmem_read_mapping_page(mapping, i);
2163                         if (IS_ERR(page))
2164                                 goto err_pages;
2165                 }
2166 #ifdef CONFIG_SWIOTLB
2167                 if (swiotlb_nr_tbl()) {
2168                         st->nents++;
2169                         sg_set_page(sg, page, PAGE_SIZE, 0);
2170                         sg = sg_next(sg);
2171                         continue;
2172                 }
2173 #endif
2174                 if (!i || page_to_pfn(page) != last_pfn + 1) {
2175                         if (i)
2176                                 sg = sg_next(sg);
2177                         st->nents++;
2178                         sg_set_page(sg, page, PAGE_SIZE, 0);
2179                 } else {
2180                         sg->length += PAGE_SIZE;
2181                 }
2182                 last_pfn = page_to_pfn(page);
2183
2184                 /* Check that the i965g/gm workaround works. */
2185                 WARN_ON((gfp & __GFP_DMA32) && (last_pfn >= 0x00100000UL));
2186         }
2187 #ifdef CONFIG_SWIOTLB
2188         if (!swiotlb_nr_tbl())
2189 #endif
2190                 sg_mark_end(sg);
2191         obj->pages = st;
2192
2193         if (i915_gem_object_needs_bit17_swizzle(obj))
2194                 i915_gem_object_do_bit_17_swizzle(obj);
2195
2196         if (obj->tiling_mode != I915_TILING_NONE &&
2197             dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES)
2198                 i915_gem_object_pin_pages(obj);
2199
2200         return 0;
2201
2202 err_pages:
2203         sg_mark_end(sg);
2204         for_each_sg_page(st->sgl, &sg_iter, st->nents, 0)
2205                 page_cache_release(sg_page_iter_page(&sg_iter));
2206         sg_free_table(st);
2207         kfree(st);
2208
2209         /* shmemfs first checks if there is enough memory to allocate the page
2210          * and reports ENOSPC should there be insufficient, along with the usual
2211          * ENOMEM for a genuine allocation failure.
2212          *
2213          * We use ENOSPC in our driver to mean that we have run out of aperture
2214          * space and so want to translate the error from shmemfs back to our
2215          * usual understanding of ENOMEM.
2216          */
2217         if (PTR_ERR(page) == -ENOSPC)
2218                 return -ENOMEM;
2219         else
2220                 return PTR_ERR(page);
2221 }
2222
2223 /* Ensure that the associated pages are gathered from the backing storage
2224  * and pinned into our object. i915_gem_object_get_pages() may be called
2225  * multiple times before they are released by a single call to
2226  * i915_gem_object_put_pages() - once the pages are no longer referenced
2227  * either as a result of memory pressure (reaping pages under the shrinker)
2228  * or as the object is itself released.
2229  */
2230 int
2231 i915_gem_object_get_pages(struct drm_i915_gem_object *obj)
2232 {
2233         struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
2234         const struct drm_i915_gem_object_ops *ops = obj->ops;
2235         int ret;
2236
2237         if (obj->pages)
2238                 return 0;
2239
2240         if (obj->madv != I915_MADV_WILLNEED) {
2241                 DRM_DEBUG("Attempting to obtain a purgeable object\n");
2242                 return -EFAULT;
2243         }
2244
2245         BUG_ON(obj->pages_pin_count);
2246
2247         ret = ops->get_pages(obj);
2248         if (ret)
2249                 return ret;
2250
2251         list_add_tail(&obj->global_list, &dev_priv->mm.unbound_list);
2252         return 0;
2253 }
2254
2255 static void
2256 i915_gem_object_move_to_active(struct drm_i915_gem_object *obj,
2257                                struct intel_engine_cs *ring)
2258 {
2259         struct drm_i915_gem_request *req;
2260         struct intel_engine_cs *old_ring;
2261
2262         BUG_ON(ring == NULL);
2263
2264         req = intel_ring_get_request(ring);
2265         old_ring = i915_gem_request_get_ring(obj->last_read_req);
2266
2267         if (old_ring != ring && obj->last_write_req) {
2268                 /* Keep the request relative to the current ring */
2269                 i915_gem_request_assign(&obj->last_write_req, req);
2270         }
2271
2272         /* Add a reference if we're newly entering the active list. */
2273         if (!obj->active) {
2274                 drm_gem_object_reference(&obj->base);
2275                 obj->active = 1;
2276         }
2277
2278         list_move_tail(&obj->ring_list, &ring->active_list);
2279
2280         i915_gem_request_assign(&obj->last_read_req, req);
2281 }
2282
2283 void i915_vma_move_to_active(struct i915_vma *vma,
2284                              struct intel_engine_cs *ring)
2285 {
2286         list_move_tail(&vma->mm_list, &vma->vm->active_list);
2287         return i915_gem_object_move_to_active(vma->obj, ring);
2288 }
2289
2290 static void
2291 i915_gem_object_move_to_inactive(struct drm_i915_gem_object *obj)
2292 {
2293         struct i915_vma *vma;
2294
2295         BUG_ON(obj->base.write_domain & ~I915_GEM_GPU_DOMAINS);
2296         BUG_ON(!obj->active);
2297
2298         list_for_each_entry(vma, &obj->vma_list, vma_link) {
2299                 if (!list_empty(&vma->mm_list))
2300                         list_move_tail(&vma->mm_list, &vma->vm->inactive_list);
2301         }
2302
2303         intel_fb_obj_flush(obj, true);
2304
2305         list_del_init(&obj->ring_list);
2306
2307         i915_gem_request_assign(&obj->last_read_req, NULL);
2308         i915_gem_request_assign(&obj->last_write_req, NULL);
2309         obj->base.write_domain = 0;
2310
2311         i915_gem_request_assign(&obj->last_fenced_req, NULL);
2312
2313         obj->active = 0;
2314         drm_gem_object_unreference(&obj->base);
2315
2316         WARN_ON(i915_verify_lists(dev));
2317 }
2318
2319 static void
2320 i915_gem_object_retire(struct drm_i915_gem_object *obj)
2321 {
2322         if (obj->last_read_req == NULL)
2323                 return;
2324
2325         if (i915_gem_request_completed(obj->last_read_req, true))
2326                 i915_gem_object_move_to_inactive(obj);
2327 }
2328
2329 static int
2330 i915_gem_init_seqno(struct drm_device *dev, u32 seqno)
2331 {
2332         struct drm_i915_private *dev_priv = dev->dev_private;
2333         struct intel_engine_cs *ring;
2334         int ret, i, j;
2335
2336         /* Carefully retire all requests without writing to the rings */
2337         for_each_ring(ring, dev_priv, i) {
2338                 ret = intel_ring_idle(ring);
2339                 if (ret)
2340                         return ret;
2341         }
2342         i915_gem_retire_requests(dev);
2343
2344         /* Finally reset hw state */
2345         for_each_ring(ring, dev_priv, i) {
2346                 intel_ring_init_seqno(ring, seqno);
2347
2348                 for (j = 0; j < ARRAY_SIZE(ring->semaphore.sync_seqno); j++)
2349                         ring->semaphore.sync_seqno[j] = 0;
2350         }
2351
2352         return 0;
2353 }
2354
2355 int i915_gem_set_seqno(struct drm_device *dev, u32 seqno)
2356 {
2357         struct drm_i915_private *dev_priv = dev->dev_private;
2358         int ret;
2359
2360         if (seqno == 0)
2361                 return -EINVAL;
2362
2363         /* HWS page needs to be set less than what we
2364          * will inject to ring
2365          */
2366         ret = i915_gem_init_seqno(dev, seqno - 1);
2367         if (ret)
2368                 return ret;
2369
2370         /* Carefully set the last_seqno value so that wrap
2371          * detection still works
2372          */
2373         dev_priv->next_seqno = seqno;
2374         dev_priv->last_seqno = seqno - 1;
2375         if (dev_priv->last_seqno == 0)
2376                 dev_priv->last_seqno--;
2377
2378         return 0;
2379 }
2380
2381 int
2382 i915_gem_get_seqno(struct drm_device *dev, u32 *seqno)
2383 {
2384         struct drm_i915_private *dev_priv = dev->dev_private;
2385
2386         /* reserve 0 for non-seqno */
2387         if (dev_priv->next_seqno == 0) {
2388                 int ret = i915_gem_init_seqno(dev, 0);
2389                 if (ret)
2390                         return ret;
2391
2392                 dev_priv->next_seqno = 1;
2393         }
2394
2395         *seqno = dev_priv->last_seqno = dev_priv->next_seqno++;
2396         return 0;
2397 }
2398
2399 int __i915_add_request(struct intel_engine_cs *ring,
2400                        struct drm_file *file,
2401                        struct drm_i915_gem_object *obj)
2402 {
2403         struct drm_i915_private *dev_priv = ring->dev->dev_private;
2404         struct drm_i915_gem_request *request;
2405         struct intel_ringbuffer *ringbuf;
2406         u32 request_ring_position, request_start;
2407         int ret;
2408
2409         request = ring->outstanding_lazy_request;
2410         if (WARN_ON(request == NULL))
2411                 return -ENOMEM;
2412
2413         if (i915.enable_execlists) {
2414                 struct intel_context *ctx = request->ctx;
2415                 ringbuf = ctx->engine[ring->id].ringbuf;
2416         } else
2417                 ringbuf = ring->buffer;
2418
2419         request_start = intel_ring_get_tail(ringbuf);
2420         /*
2421          * Emit any outstanding flushes - execbuf can fail to emit the flush
2422          * after having emitted the batchbuffer command. Hence we need to fix
2423          * things up similar to emitting the lazy request. The difference here
2424          * is that the flush _must_ happen before the next request, no matter
2425          * what.
2426          */
2427         if (i915.enable_execlists) {
2428                 ret = logical_ring_flush_all_caches(ringbuf);
2429                 if (ret)
2430                         return ret;
2431         } else {
2432                 ret = intel_ring_flush_all_caches(ring);
2433                 if (ret)
2434                         return ret;
2435         }
2436
2437         /* Record the position of the start of the request so that
2438          * should we detect the updated seqno part-way through the
2439          * GPU processing the request, we never over-estimate the
2440          * position of the head.
2441          */
2442         request_ring_position = intel_ring_get_tail(ringbuf);
2443
2444         if (i915.enable_execlists) {
2445                 ret = ring->emit_request(ringbuf);
2446                 if (ret)
2447                         return ret;
2448         } else {
2449                 ret = ring->add_request(ring);
2450                 if (ret)
2451                         return ret;
2452         }
2453
2454         request->head = request_start;
2455         request->tail = request_ring_position;
2456
2457         /* Whilst this request exists, batch_obj will be on the
2458          * active_list, and so will hold the active reference. Only when this
2459          * request is retired will the the batch_obj be moved onto the
2460          * inactive_list and lose its active reference. Hence we do not need
2461          * to explicitly hold another reference here.
2462          */
2463         request->batch_obj = obj;
2464
2465         if (!i915.enable_execlists) {
2466                 /* Hold a reference to the current context so that we can inspect
2467                  * it later in case a hangcheck error event fires.
2468                  */
2469                 request->ctx = ring->last_context;
2470                 if (request->ctx)
2471                         i915_gem_context_reference(request->ctx);
2472         }
2473
2474         request->emitted_jiffies = jiffies;
2475         list_add_tail(&request->list, &ring->request_list);
2476         request->file_priv = NULL;
2477
2478         if (file) {
2479                 struct drm_i915_file_private *file_priv = file->driver_priv;
2480
2481                 spin_lock(&file_priv->mm.lock);
2482                 request->file_priv = file_priv;
2483                 list_add_tail(&request->client_list,
2484                               &file_priv->mm.request_list);
2485                 spin_unlock(&file_priv->mm.lock);
2486         }
2487
2488         trace_i915_gem_request_add(request);
2489         ring->outstanding_lazy_request = NULL;
2490
2491         i915_queue_hangcheck(ring->dev);
2492
2493         cancel_delayed_work_sync(&dev_priv->mm.idle_work);
2494         queue_delayed_work(dev_priv->wq,
2495                            &dev_priv->mm.retire_work,
2496                            round_jiffies_up_relative(HZ));
2497         intel_mark_busy(dev_priv->dev);
2498
2499         return 0;
2500 }
2501
2502 static inline void
2503 i915_gem_request_remove_from_client(struct drm_i915_gem_request *request)
2504 {
2505         struct drm_i915_file_private *file_priv = request->file_priv;
2506
2507         if (!file_priv)
2508                 return;
2509
2510         spin_lock(&file_priv->mm.lock);
2511         list_del(&request->client_list);
2512         request->file_priv = NULL;
2513         spin_unlock(&file_priv->mm.lock);
2514 }
2515
2516 static bool i915_context_is_banned(struct drm_i915_private *dev_priv,
2517                                    const struct intel_context *ctx)
2518 {
2519         unsigned long elapsed;
2520
2521         elapsed = get_seconds() - ctx->hang_stats.guilty_ts;
2522
2523         if (ctx->hang_stats.banned)
2524                 return true;
2525
2526         if (elapsed <= DRM_I915_CTX_BAN_PERIOD) {
2527                 if (!i915_gem_context_is_default(ctx)) {
2528                         DRM_DEBUG("context hanging too fast, banning!\n");
2529                         return true;
2530                 } else if (i915_stop_ring_allow_ban(dev_priv)) {
2531                         if (i915_stop_ring_allow_warn(dev_priv))
2532                                 DRM_ERROR("gpu hanging too fast, banning!\n");
2533                         return true;
2534                 }
2535         }
2536
2537         return false;
2538 }
2539
2540 static void i915_set_reset_status(struct drm_i915_private *dev_priv,
2541                                   struct intel_context *ctx,
2542                                   const bool guilty)
2543 {
2544         struct i915_ctx_hang_stats *hs;
2545
2546         if (WARN_ON(!ctx))
2547                 return;
2548
2549         hs = &ctx->hang_stats;
2550
2551         if (guilty) {
2552                 hs->banned = i915_context_is_banned(dev_priv, ctx);
2553                 hs->batch_active++;
2554                 hs->guilty_ts = get_seconds();
2555         } else {
2556                 hs->batch_pending++;
2557         }
2558 }
2559
2560 static void i915_gem_free_request(struct drm_i915_gem_request *request)
2561 {
2562         list_del(&request->list);
2563         i915_gem_request_remove_from_client(request);
2564
2565         i915_gem_request_unreference(request);
2566 }
2567
2568 void i915_gem_request_free(struct kref *req_ref)
2569 {
2570         struct drm_i915_gem_request *req = container_of(req_ref,
2571                                                  typeof(*req), ref);
2572         struct intel_context *ctx = req->ctx;
2573
2574         if (ctx) {
2575                 if (i915.enable_execlists) {
2576                         struct intel_engine_cs *ring = req->ring;
2577
2578                         if (ctx != ring->default_context)
2579                                 intel_lr_context_unpin(ring, ctx);
2580                 }
2581
2582                 i915_gem_context_unreference(ctx);
2583         }
2584
2585         kfree(req);
2586 }
2587
2588 struct drm_i915_gem_request *
2589 i915_gem_find_active_request(struct intel_engine_cs *ring)
2590 {
2591         struct drm_i915_gem_request *request;
2592
2593         list_for_each_entry(request, &ring->request_list, list) {
2594                 if (i915_gem_request_completed(request, false))
2595                         continue;
2596
2597                 return request;
2598         }
2599
2600         return NULL;
2601 }
2602
2603 static void i915_gem_reset_ring_status(struct drm_i915_private *dev_priv,
2604                                        struct intel_engine_cs *ring)
2605 {
2606         struct drm_i915_gem_request *request;
2607         bool ring_hung;
2608
2609         request = i915_gem_find_active_request(ring);
2610
2611         if (request == NULL)
2612                 return;
2613
2614         ring_hung = ring->hangcheck.score >= HANGCHECK_SCORE_RING_HUNG;
2615
2616         i915_set_reset_status(dev_priv, request->ctx, ring_hung);
2617
2618         list_for_each_entry_continue(request, &ring->request_list, list)
2619                 i915_set_reset_status(dev_priv, request->ctx, false);
2620 }
2621
2622 static void i915_gem_reset_ring_cleanup(struct drm_i915_private *dev_priv,
2623                                         struct intel_engine_cs *ring)
2624 {
2625         while (!list_empty(&ring->active_list)) {
2626                 struct drm_i915_gem_object *obj;
2627
2628                 obj = list_first_entry(&ring->active_list,
2629                                        struct drm_i915_gem_object,
2630                                        ring_list);
2631
2632                 i915_gem_object_move_to_inactive(obj);
2633         }
2634
2635         /*
2636          * Clear the execlists queue up before freeing the requests, as those
2637          * are the ones that keep the context and ringbuffer backing objects
2638          * pinned in place.
2639          */
2640         while (!list_empty(&ring->execlist_queue)) {
2641                 struct intel_ctx_submit_request *submit_req;
2642
2643                 submit_req = list_first_entry(&ring->execlist_queue,
2644                                 struct intel_ctx_submit_request,
2645                                 execlist_link);
2646                 list_del(&submit_req->execlist_link);
2647                 intel_runtime_pm_put(dev_priv);
2648                 i915_gem_context_unreference(submit_req->ctx);
2649                 kfree(submit_req);
2650         }
2651
2652         /*
2653          * We must free the requests after all the corresponding objects have
2654          * been moved off active lists. Which is the same order as the normal
2655          * retire_requests function does. This is important if object hold
2656          * implicit references on things like e.g. ppgtt address spaces through
2657          * the request.
2658          */
2659         while (!list_empty(&ring->request_list)) {
2660                 struct drm_i915_gem_request *request;
2661
2662                 request = list_first_entry(&ring->request_list,
2663                                            struct drm_i915_gem_request,
2664                                            list);
2665
2666                 i915_gem_free_request(request);
2667         }
2668
2669         /* This may not have been flushed before the reset, so clean it now */
2670         i915_gem_request_assign(&ring->outstanding_lazy_request, NULL);
2671 }
2672
2673 void i915_gem_restore_fences(struct drm_device *dev)
2674 {
2675         struct drm_i915_private *dev_priv = dev->dev_private;
2676         int i;
2677
2678         for (i = 0; i < dev_priv->num_fence_regs; i++) {
2679                 struct drm_i915_fence_reg *reg = &dev_priv->fence_regs[i];
2680
2681                 /*
2682                  * Commit delayed tiling changes if we have an object still
2683                  * attached to the fence, otherwise just clear the fence.
2684                  */
2685                 if (reg->obj) {
2686                         i915_gem_object_update_fence(reg->obj, reg,
2687                                                      reg->obj->tiling_mode);
2688                 } else {
2689                         i915_gem_write_fence(dev, i, NULL);
2690                 }
2691         }
2692 }
2693
2694 void i915_gem_reset(struct drm_device *dev)
2695 {
2696         struct drm_i915_private *dev_priv = dev->dev_private;
2697         struct intel_engine_cs *ring;
2698         int i;
2699
2700         /*
2701          * Before we free the objects from the requests, we need to inspect
2702          * them for finding the guilty party. As the requests only borrow
2703          * their reference to the objects, the inspection must be done first.
2704          */
2705         for_each_ring(ring, dev_priv, i)
2706                 i915_gem_reset_ring_status(dev_priv, ring);
2707
2708         for_each_ring(ring, dev_priv, i)
2709                 i915_gem_reset_ring_cleanup(dev_priv, ring);
2710
2711         i915_gem_context_reset(dev);
2712
2713         i915_gem_restore_fences(dev);
2714 }
2715
2716 /**
2717  * This function clears the request list as sequence numbers are passed.
2718  */
2719 void
2720 i915_gem_retire_requests_ring(struct intel_engine_cs *ring)
2721 {
2722         if (list_empty(&ring->request_list))
2723                 return;
2724
2725         WARN_ON(i915_verify_lists(ring->dev));
2726
2727         /* Move any buffers on the active list that are no longer referenced
2728          * by the ringbuffer to the flushing/inactive lists as appropriate,
2729          * before we free the context associated with the requests.
2730          */
2731         while (!list_empty(&ring->active_list)) {
2732                 struct drm_i915_gem_object *obj;
2733
2734                 obj = list_first_entry(&ring->active_list,
2735                                       struct drm_i915_gem_object,
2736                                       ring_list);
2737
2738                 if (!i915_gem_request_completed(obj->last_read_req, true))
2739                         break;
2740
2741                 i915_gem_object_move_to_inactive(obj);
2742         }
2743
2744
2745         while (!list_empty(&ring->request_list)) {
2746                 struct drm_i915_gem_request *request;
2747                 struct intel_ringbuffer *ringbuf;
2748
2749                 request = list_first_entry(&ring->request_list,
2750                                            struct drm_i915_gem_request,
2751                                            list);
2752
2753                 if (!i915_gem_request_completed(request, true))
2754                         break;
2755
2756                 trace_i915_gem_request_retire(request);
2757
2758                 /* This is one of the few common intersection points
2759                  * between legacy ringbuffer submission and execlists:
2760                  * we need to tell them apart in order to find the correct
2761                  * ringbuffer to which the request belongs to.
2762                  */
2763                 if (i915.enable_execlists) {
2764                         struct intel_context *ctx = request->ctx;
2765                         ringbuf = ctx->engine[ring->id].ringbuf;
2766                 } else
2767                         ringbuf = ring->buffer;
2768
2769                 /* We know the GPU must have read the request to have
2770                  * sent us the seqno + interrupt, so use the position
2771                  * of tail of the request to update the last known position
2772                  * of the GPU head.
2773                  */
2774                 ringbuf->last_retired_head = request->tail;
2775
2776                 i915_gem_free_request(request);
2777         }
2778
2779         if (unlikely(ring->trace_irq_req &&
2780                      i915_gem_request_completed(ring->trace_irq_req, true))) {
2781                 ring->irq_put(ring);
2782                 i915_gem_request_assign(&ring->trace_irq_req, NULL);
2783         }
2784
2785         WARN_ON(i915_verify_lists(ring->dev));
2786 }
2787
2788 bool
2789 i915_gem_retire_requests(struct drm_device *dev)
2790 {
2791         struct drm_i915_private *dev_priv = dev->dev_private;
2792         struct intel_engine_cs *ring;
2793         bool idle = true;
2794         int i;
2795
2796         for_each_ring(ring, dev_priv, i) {
2797                 i915_gem_retire_requests_ring(ring);
2798                 idle &= list_empty(&ring->request_list);
2799                 if (i915.enable_execlists) {
2800                         unsigned long flags;
2801
2802                         spin_lock_irqsave(&ring->execlist_lock, flags);
2803                         idle &= list_empty(&ring->execlist_queue);
2804                         spin_unlock_irqrestore(&ring->execlist_lock, flags);
2805
2806                         intel_execlists_retire_requests(ring);
2807                 }
2808         }
2809
2810         if (idle)
2811                 mod_delayed_work(dev_priv->wq,
2812                                    &dev_priv->mm.idle_work,
2813                                    msecs_to_jiffies(100));
2814
2815         return idle;
2816 }
2817
2818 static void
2819 i915_gem_retire_work_handler(struct work_struct *work)
2820 {
2821         struct drm_i915_private *dev_priv =
2822                 container_of(work, typeof(*dev_priv), mm.retire_work.work);
2823         struct drm_device *dev = dev_priv->dev;
2824         bool idle;
2825
2826         /* Come back later if the device is busy... */
2827         idle = false;
2828         if (mutex_trylock(&dev->struct_mutex)) {
2829                 idle = i915_gem_retire_requests(dev);
2830                 mutex_unlock(&dev->struct_mutex);
2831         }
2832         if (!idle)
2833                 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work,
2834                                    round_jiffies_up_relative(HZ));
2835 }
2836
2837 static void
2838 i915_gem_idle_work_handler(struct work_struct *work)
2839 {
2840         struct drm_i915_private *dev_priv =
2841                 container_of(work, typeof(*dev_priv), mm.idle_work.work);
2842
2843         intel_mark_idle(dev_priv->dev);
2844 }
2845
2846 /**
2847  * Ensures that an object will eventually get non-busy by flushing any required
2848  * write domains, emitting any outstanding lazy request and retiring and
2849  * completed requests.
2850  */
2851 static int
2852 i915_gem_object_flush_active(struct drm_i915_gem_object *obj)
2853 {
2854         struct intel_engine_cs *ring;
2855         int ret;
2856
2857         if (obj->active) {
2858                 ring = i915_gem_request_get_ring(obj->last_read_req);
2859
2860                 ret = i915_gem_check_olr(obj->last_read_req);
2861                 if (ret)
2862                         return ret;
2863
2864                 i915_gem_retire_requests_ring(ring);
2865         }
2866
2867         return 0;
2868 }
2869
2870 /**
2871  * i915_gem_wait_ioctl - implements DRM_IOCTL_I915_GEM_WAIT
2872  * @DRM_IOCTL_ARGS: standard ioctl arguments
2873  *
2874  * Returns 0 if successful, else an error is returned with the remaining time in
2875  * the timeout parameter.
2876  *  -ETIME: object is still busy after timeout
2877  *  -ERESTARTSYS: signal interrupted the wait
2878  *  -ENONENT: object doesn't exist
2879  * Also possible, but rare:
2880  *  -EAGAIN: GPU wedged
2881  *  -ENOMEM: damn
2882  *  -ENODEV: Internal IRQ fail
2883  *  -E?: The add request failed
2884  *
2885  * The wait ioctl with a timeout of 0 reimplements the busy ioctl. With any
2886  * non-zero timeout parameter the wait ioctl will wait for the given number of
2887  * nanoseconds on an object becoming unbusy. Since the wait itself does so
2888  * without holding struct_mutex the object may become re-busied before this
2889  * function completes. A similar but shorter * race condition exists in the busy
2890  * ioctl
2891  */
2892 int
2893 i915_gem_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
2894 {
2895         struct drm_i915_private *dev_priv = dev->dev_private;
2896         struct drm_i915_gem_wait *args = data;
2897         struct drm_i915_gem_object *obj;
2898         struct drm_i915_gem_request *req;
2899         unsigned reset_counter;
2900         int ret = 0;
2901
2902         if (args->flags != 0)
2903                 return -EINVAL;
2904
2905         ret = i915_mutex_lock_interruptible(dev);
2906         if (ret)
2907                 return ret;
2908
2909         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->bo_handle));
2910         if (&obj->base == NULL) {
2911                 mutex_unlock(&dev->struct_mutex);
2912                 return -ENOENT;
2913         }
2914
2915         /* Need to make sure the object gets inactive eventually. */
2916         ret = i915_gem_object_flush_active(obj);
2917         if (ret)
2918                 goto out;
2919
2920         if (!obj->active || !obj->last_read_req)
2921                 goto out;
2922
2923         req = obj->last_read_req;
2924
2925         /* Do this after OLR check to make sure we make forward progress polling
2926          * on this IOCTL with a timeout <=0 (like busy ioctl)
2927          */
2928         if (args->timeout_ns <= 0) {
2929                 ret = -ETIME;
2930                 goto out;
2931         }
2932
2933         drm_gem_object_unreference(&obj->base);
2934         reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
2935         i915_gem_request_reference(req);
2936         mutex_unlock(&dev->struct_mutex);
2937
2938         ret = __i915_wait_request(req, reset_counter, true, &args->timeout_ns,
2939                                   file->driver_priv);
2940         mutex_lock(&dev->struct_mutex);
2941         i915_gem_request_unreference(req);
2942         mutex_unlock(&dev->struct_mutex);
2943         return ret;
2944
2945 out:
2946         drm_gem_object_unreference(&obj->base);
2947         mutex_unlock(&dev->struct_mutex);
2948         return ret;
2949 }
2950
2951 /**
2952  * i915_gem_object_sync - sync an object to a ring.
2953  *
2954  * @obj: object which may be in use on another ring.
2955  * @to: ring we wish to use the object on. May be NULL.
2956  *
2957  * This code is meant to abstract object synchronization with the GPU.
2958  * Calling with NULL implies synchronizing the object with the CPU
2959  * rather than a particular GPU ring.
2960  *
2961  * Returns 0 if successful, else propagates up the lower layer error.
2962  */
2963 int
2964 i915_gem_object_sync(struct drm_i915_gem_object *obj,
2965                      struct intel_engine_cs *to)
2966 {
2967         struct intel_engine_cs *from;
2968         u32 seqno;
2969         int ret, idx;
2970
2971         from = i915_gem_request_get_ring(obj->last_read_req);
2972
2973         if (from == NULL || to == from)
2974                 return 0;
2975
2976         if (to == NULL || !i915_semaphore_is_enabled(obj->base.dev))
2977                 return i915_gem_object_wait_rendering(obj, false);
2978
2979         idx = intel_ring_sync_index(from, to);
2980
2981         seqno = i915_gem_request_get_seqno(obj->last_read_req);
2982         /* Optimization: Avoid semaphore sync when we are sure we already
2983          * waited for an object with higher seqno */
2984         if (seqno <= from->semaphore.sync_seqno[idx])
2985                 return 0;
2986
2987         ret = i915_gem_check_olr(obj->last_read_req);
2988         if (ret)
2989                 return ret;
2990
2991         trace_i915_gem_ring_sync_to(from, to, obj->last_read_req);
2992         ret = to->semaphore.sync_to(to, from, seqno);
2993         if (!ret)
2994                 /* We use last_read_req because sync_to()
2995                  * might have just caused seqno wrap under
2996                  * the radar.
2997                  */
2998                 from->semaphore.sync_seqno[idx] =
2999                                 i915_gem_request_get_seqno(obj->last_read_req);
3000
3001         return ret;
3002 }
3003
3004 static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj)
3005 {
3006         u32 old_write_domain, old_read_domains;
3007
3008         /* Force a pagefault for domain tracking on next user access */
3009         i915_gem_release_mmap(obj);
3010
3011         if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
3012                 return;
3013
3014         /* Wait for any direct GTT access to complete */
3015         mb();
3016
3017         old_read_domains = obj->base.read_domains;
3018         old_write_domain = obj->base.write_domain;
3019
3020         obj->base.read_domains &= ~I915_GEM_DOMAIN_GTT;
3021         obj->base.write_domain &= ~I915_GEM_DOMAIN_GTT;
3022
3023         trace_i915_gem_object_change_domain(obj,
3024                                             old_read_domains,
3025                                             old_write_domain);
3026 }
3027
3028 int i915_vma_unbind(struct i915_vma *vma)
3029 {
3030         struct drm_i915_gem_object *obj = vma->obj;
3031         struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
3032         int ret;
3033
3034         if (list_empty(&vma->vma_link))
3035                 return 0;
3036
3037         if (!drm_mm_node_allocated(&vma->node)) {
3038                 i915_gem_vma_destroy(vma);
3039                 return 0;
3040         }
3041
3042         if (vma->pin_count)
3043                 return -EBUSY;
3044
3045         BUG_ON(obj->pages == NULL);
3046
3047         ret = i915_gem_object_finish_gpu(obj);
3048         if (ret)
3049                 return ret;
3050         /* Continue on if we fail due to EIO, the GPU is hung so we
3051          * should be safe and we need to cleanup or else we might
3052          * cause memory corruption through use-after-free.
3053          */
3054
3055         if (i915_is_ggtt(vma->vm) &&
3056             vma->ggtt_view.type == I915_GGTT_VIEW_NORMAL) {
3057                 i915_gem_object_finish_gtt(obj);
3058
3059                 /* release the fence reg _after_ flushing */
3060                 ret = i915_gem_object_put_fence(obj);
3061                 if (ret)
3062                         return ret;
3063         }
3064
3065         trace_i915_vma_unbind(vma);
3066
3067         vma->unbind_vma(vma);
3068
3069         list_del_init(&vma->mm_list);
3070         if (i915_is_ggtt(vma->vm)) {
3071                 if (vma->ggtt_view.type == I915_GGTT_VIEW_NORMAL) {
3072                         obj->map_and_fenceable = false;
3073                 } else if (vma->ggtt_view.pages) {
3074                         sg_free_table(vma->ggtt_view.pages);
3075                         kfree(vma->ggtt_view.pages);
3076                         vma->ggtt_view.pages = NULL;
3077                 }
3078         }
3079
3080         drm_mm_remove_node(&vma->node);
3081         i915_gem_vma_destroy(vma);
3082
3083         /* Since the unbound list is global, only move to that list if
3084          * no more VMAs exist. */
3085         if (list_empty(&obj->vma_list)) {
3086                 /* Throw away the active reference before
3087                  * moving to the unbound list. */
3088                 i915_gem_object_retire(obj);
3089
3090                 i915_gem_gtt_finish_object(obj);
3091                 list_move_tail(&obj->global_list, &dev_priv->mm.unbound_list);
3092         }
3093
3094         /* And finally now the object is completely decoupled from this vma,
3095          * we can drop its hold on the backing storage and allow it to be
3096          * reaped by the shrinker.
3097          */
3098         i915_gem_object_unpin_pages(obj);
3099
3100         return 0;
3101 }
3102
3103 int i915_gpu_idle(struct drm_device *dev)
3104 {
3105         struct drm_i915_private *dev_priv = dev->dev_private;
3106         struct intel_engine_cs *ring;
3107         int ret, i;
3108
3109         /* Flush everything onto the inactive list. */
3110         for_each_ring(ring, dev_priv, i) {
3111                 if (!i915.enable_execlists) {
3112                         ret = i915_switch_context(ring, ring->default_context);
3113                         if (ret)
3114                                 return ret;
3115                 }
3116
3117                 ret = intel_ring_idle(ring);
3118                 if (ret)
3119                         return ret;
3120         }
3121
3122         return 0;
3123 }
3124
3125 static void i965_write_fence_reg(struct drm_device *dev, int reg,
3126                                  struct drm_i915_gem_object *obj)
3127 {
3128         struct drm_i915_private *dev_priv = dev->dev_private;
3129         int fence_reg;
3130         int fence_pitch_shift;
3131
3132         if (INTEL_INFO(dev)->gen >= 6) {
3133                 fence_reg = FENCE_REG_SANDYBRIDGE_0;
3134                 fence_pitch_shift = SANDYBRIDGE_FENCE_PITCH_SHIFT;
3135         } else {
3136                 fence_reg = FENCE_REG_965_0;
3137                 fence_pitch_shift = I965_FENCE_PITCH_SHIFT;
3138         }
3139
3140         fence_reg += reg * 8;
3141
3142         /* To w/a incoherency with non-atomic 64-bit register updates,
3143          * we split the 64-bit update into two 32-bit writes. In order
3144          * for a partial fence not to be evaluated between writes, we
3145          * precede the update with write to turn off the fence register,
3146          * and only enable the fence as the last step.
3147          *
3148          * For extra levels of paranoia, we make sure each step lands
3149          * before applying the next step.
3150          */
3151         I915_WRITE(fence_reg, 0);
3152         POSTING_READ(fence_reg);
3153
3154         if (obj) {
3155                 u32 size = i915_gem_obj_ggtt_size(obj);
3156                 uint64_t val;
3157
3158                 val = (uint64_t)((i915_gem_obj_ggtt_offset(obj) + size - 4096) &
3159                                  0xfffff000) << 32;
3160                 val |= i915_gem_obj_ggtt_offset(obj) & 0xfffff000;
3161                 val |= (uint64_t)((obj->stride / 128) - 1) << fence_pitch_shift;
3162                 if (obj->tiling_mode == I915_TILING_Y)
3163                         val |= 1 << I965_FENCE_TILING_Y_SHIFT;
3164                 val |= I965_FENCE_REG_VALID;
3165
3166                 I915_WRITE(fence_reg + 4, val >> 32);
3167                 POSTING_READ(fence_reg + 4);
3168
3169                 I915_WRITE(fence_reg + 0, val);
3170                 POSTING_READ(fence_reg);
3171         } else {
3172                 I915_WRITE(fence_reg + 4, 0);
3173                 POSTING_READ(fence_reg + 4);
3174         }
3175 }
3176
3177 static void i915_write_fence_reg(struct drm_device *dev, int reg,
3178                                  struct drm_i915_gem_object *obj)
3179 {
3180         struct drm_i915_private *dev_priv = dev->dev_private;
3181         u32 val;
3182
3183         if (obj) {
3184                 u32 size = i915_gem_obj_ggtt_size(obj);
3185                 int pitch_val;
3186                 int tile_width;
3187
3188                 WARN((i915_gem_obj_ggtt_offset(obj) & ~I915_FENCE_START_MASK) ||
3189                      (size & -size) != size ||
3190                      (i915_gem_obj_ggtt_offset(obj) & (size - 1)),
3191                      "object 0x%08lx [fenceable? %d] not 1M or pot-size (0x%08x) aligned\n",
3192                      i915_gem_obj_ggtt_offset(obj), obj->map_and_fenceable, size);
3193
3194                 if (obj->tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev))
3195                         tile_width = 128;
3196                 else
3197                         tile_width = 512;
3198
3199                 /* Note: pitch better be a power of two tile widths */