drm/i915: use shmem_truncate_range
[sfrench/cifs-2.6.git] / drivers / gpu / drm / i915 / i915_gem.c
1 /*
2  * Copyright © 2008 Intel Corporation
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice (including the next
12  * paragraph) shall be included in all copies or substantial portions of the
13  * Software.
14  *
15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21  * IN THE SOFTWARE.
22  *
23  * Authors:
24  *    Eric Anholt <eric@anholt.net>
25  *
26  */
27
28 #include "drmP.h"
29 #include "drm.h"
30 #include "i915_drm.h"
31 #include "i915_drv.h"
32 #include "i915_trace.h"
33 #include "intel_drv.h"
34 #include <linux/shmem_fs.h>
35 #include <linux/slab.h>
36 #include <linux/swap.h>
37 #include <linux/pci.h>
38
39 static __must_check int i915_gem_object_flush_gpu_write_domain(struct drm_i915_gem_object *obj);
40 static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj);
41 static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj);
42 static __must_check int i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj,
43                                                           bool write);
44 static __must_check int i915_gem_object_set_cpu_read_domain_range(struct drm_i915_gem_object *obj,
45                                                                   uint64_t offset,
46                                                                   uint64_t size);
47 static void i915_gem_object_set_to_full_cpu_read_domain(struct drm_i915_gem_object *obj);
48 static __must_check int i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
49                                                     unsigned alignment,
50                                                     bool map_and_fenceable);
51 static void i915_gem_clear_fence_reg(struct drm_device *dev,
52                                      struct drm_i915_fence_reg *reg);
53 static int i915_gem_phys_pwrite(struct drm_device *dev,
54                                 struct drm_i915_gem_object *obj,
55                                 struct drm_i915_gem_pwrite *args,
56                                 struct drm_file *file);
57 static void i915_gem_free_object_tail(struct drm_i915_gem_object *obj);
58
59 static int i915_gem_inactive_shrink(struct shrinker *shrinker,
60                                     struct shrink_control *sc);
61
62 /* some bookkeeping */
63 static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv,
64                                   size_t size)
65 {
66         dev_priv->mm.object_count++;
67         dev_priv->mm.object_memory += size;
68 }
69
70 static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv,
71                                      size_t size)
72 {
73         dev_priv->mm.object_count--;
74         dev_priv->mm.object_memory -= size;
75 }
76
77 static int
78 i915_gem_wait_for_error(struct drm_device *dev)
79 {
80         struct drm_i915_private *dev_priv = dev->dev_private;
81         struct completion *x = &dev_priv->error_completion;
82         unsigned long flags;
83         int ret;
84
85         if (!atomic_read(&dev_priv->mm.wedged))
86                 return 0;
87
88         ret = wait_for_completion_interruptible(x);
89         if (ret)
90                 return ret;
91
92         if (atomic_read(&dev_priv->mm.wedged)) {
93                 /* GPU is hung, bump the completion count to account for
94                  * the token we just consumed so that we never hit zero and
95                  * end up waiting upon a subsequent completion event that
96                  * will never happen.
97                  */
98                 spin_lock_irqsave(&x->wait.lock, flags);
99                 x->done++;
100                 spin_unlock_irqrestore(&x->wait.lock, flags);
101         }
102         return 0;
103 }
104
105 int i915_mutex_lock_interruptible(struct drm_device *dev)
106 {
107         int ret;
108
109         ret = i915_gem_wait_for_error(dev);
110         if (ret)
111                 return ret;
112
113         ret = mutex_lock_interruptible(&dev->struct_mutex);
114         if (ret)
115                 return ret;
116
117         WARN_ON(i915_verify_lists(dev));
118         return 0;
119 }
120
121 static inline bool
122 i915_gem_object_is_inactive(struct drm_i915_gem_object *obj)
123 {
124         return obj->gtt_space && !obj->active && obj->pin_count == 0;
125 }
126
127 void i915_gem_do_init(struct drm_device *dev,
128                       unsigned long start,
129                       unsigned long mappable_end,
130                       unsigned long end)
131 {
132         drm_i915_private_t *dev_priv = dev->dev_private;
133
134         drm_mm_init(&dev_priv->mm.gtt_space, start, end - start);
135
136         dev_priv->mm.gtt_start = start;
137         dev_priv->mm.gtt_mappable_end = mappable_end;
138         dev_priv->mm.gtt_end = end;
139         dev_priv->mm.gtt_total = end - start;
140         dev_priv->mm.mappable_gtt_total = min(end, mappable_end) - start;
141
142         /* Take over this portion of the GTT */
143         intel_gtt_clear_range(start / PAGE_SIZE, (end-start) / PAGE_SIZE);
144 }
145
146 int
147 i915_gem_init_ioctl(struct drm_device *dev, void *data,
148                     struct drm_file *file)
149 {
150         struct drm_i915_gem_init *args = data;
151
152         if (args->gtt_start >= args->gtt_end ||
153             (args->gtt_end | args->gtt_start) & (PAGE_SIZE - 1))
154                 return -EINVAL;
155
156         mutex_lock(&dev->struct_mutex);
157         i915_gem_do_init(dev, args->gtt_start, args->gtt_end, args->gtt_end);
158         mutex_unlock(&dev->struct_mutex);
159
160         return 0;
161 }
162
163 int
164 i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
165                             struct drm_file *file)
166 {
167         struct drm_i915_private *dev_priv = dev->dev_private;
168         struct drm_i915_gem_get_aperture *args = data;
169         struct drm_i915_gem_object *obj;
170         size_t pinned;
171
172         if (!(dev->driver->driver_features & DRIVER_GEM))
173                 return -ENODEV;
174
175         pinned = 0;
176         mutex_lock(&dev->struct_mutex);
177         list_for_each_entry(obj, &dev_priv->mm.pinned_list, mm_list)
178                 pinned += obj->gtt_space->size;
179         mutex_unlock(&dev->struct_mutex);
180
181         args->aper_size = dev_priv->mm.gtt_total;
182         args->aper_available_size = args->aper_size -pinned;
183
184         return 0;
185 }
186
187 static int
188 i915_gem_create(struct drm_file *file,
189                 struct drm_device *dev,
190                 uint64_t size,
191                 uint32_t *handle_p)
192 {
193         struct drm_i915_gem_object *obj;
194         int ret;
195         u32 handle;
196
197         size = roundup(size, PAGE_SIZE);
198
199         /* Allocate the new object */
200         obj = i915_gem_alloc_object(dev, size);
201         if (obj == NULL)
202                 return -ENOMEM;
203
204         ret = drm_gem_handle_create(file, &obj->base, &handle);
205         if (ret) {
206                 drm_gem_object_release(&obj->base);
207                 i915_gem_info_remove_obj(dev->dev_private, obj->base.size);
208                 kfree(obj);
209                 return ret;
210         }
211
212         /* drop reference from allocate - handle holds it now */
213         drm_gem_object_unreference(&obj->base);
214         trace_i915_gem_object_create(obj);
215
216         *handle_p = handle;
217         return 0;
218 }
219
220 int
221 i915_gem_dumb_create(struct drm_file *file,
222                      struct drm_device *dev,
223                      struct drm_mode_create_dumb *args)
224 {
225         /* have to work out size/pitch and return them */
226         args->pitch = ALIGN(args->width * ((args->bpp + 7) / 8), 64);
227         args->size = args->pitch * args->height;
228         return i915_gem_create(file, dev,
229                                args->size, &args->handle);
230 }
231
232 int i915_gem_dumb_destroy(struct drm_file *file,
233                           struct drm_device *dev,
234                           uint32_t handle)
235 {
236         return drm_gem_handle_delete(file, handle);
237 }
238
239 /**
240  * Creates a new mm object and returns a handle to it.
241  */
242 int
243 i915_gem_create_ioctl(struct drm_device *dev, void *data,
244                       struct drm_file *file)
245 {
246         struct drm_i915_gem_create *args = data;
247         return i915_gem_create(file, dev,
248                                args->size, &args->handle);
249 }
250
251 static int i915_gem_object_needs_bit17_swizzle(struct drm_i915_gem_object *obj)
252 {
253         drm_i915_private_t *dev_priv = obj->base.dev->dev_private;
254
255         return dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 &&
256                 obj->tiling_mode != I915_TILING_NONE;
257 }
258
259 static inline void
260 slow_shmem_copy(struct page *dst_page,
261                 int dst_offset,
262                 struct page *src_page,
263                 int src_offset,
264                 int length)
265 {
266         char *dst_vaddr, *src_vaddr;
267
268         dst_vaddr = kmap(dst_page);
269         src_vaddr = kmap(src_page);
270
271         memcpy(dst_vaddr + dst_offset, src_vaddr + src_offset, length);
272
273         kunmap(src_page);
274         kunmap(dst_page);
275 }
276
277 static inline void
278 slow_shmem_bit17_copy(struct page *gpu_page,
279                       int gpu_offset,
280                       struct page *cpu_page,
281                       int cpu_offset,
282                       int length,
283                       int is_read)
284 {
285         char *gpu_vaddr, *cpu_vaddr;
286
287         /* Use the unswizzled path if this page isn't affected. */
288         if ((page_to_phys(gpu_page) & (1 << 17)) == 0) {
289                 if (is_read)
290                         return slow_shmem_copy(cpu_page, cpu_offset,
291                                                gpu_page, gpu_offset, length);
292                 else
293                         return slow_shmem_copy(gpu_page, gpu_offset,
294                                                cpu_page, cpu_offset, length);
295         }
296
297         gpu_vaddr = kmap(gpu_page);
298         cpu_vaddr = kmap(cpu_page);
299
300         /* Copy the data, XORing A6 with A17 (1). The user already knows he's
301          * XORing with the other bits (A9 for Y, A9 and A10 for X)
302          */
303         while (length > 0) {
304                 int cacheline_end = ALIGN(gpu_offset + 1, 64);
305                 int this_length = min(cacheline_end - gpu_offset, length);
306                 int swizzled_gpu_offset = gpu_offset ^ 64;
307
308                 if (is_read) {
309                         memcpy(cpu_vaddr + cpu_offset,
310                                gpu_vaddr + swizzled_gpu_offset,
311                                this_length);
312                 } else {
313                         memcpy(gpu_vaddr + swizzled_gpu_offset,
314                                cpu_vaddr + cpu_offset,
315                                this_length);
316                 }
317                 cpu_offset += this_length;
318                 gpu_offset += this_length;
319                 length -= this_length;
320         }
321
322         kunmap(cpu_page);
323         kunmap(gpu_page);
324 }
325
326 /**
327  * This is the fast shmem pread path, which attempts to copy_from_user directly
328  * from the backing pages of the object to the user's address space.  On a
329  * fault, it fails so we can fall back to i915_gem_shmem_pwrite_slow().
330  */
331 static int
332 i915_gem_shmem_pread_fast(struct drm_device *dev,
333                           struct drm_i915_gem_object *obj,
334                           struct drm_i915_gem_pread *args,
335                           struct drm_file *file)
336 {
337         struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
338         ssize_t remain;
339         loff_t offset;
340         char __user *user_data;
341         int page_offset, page_length;
342
343         user_data = (char __user *) (uintptr_t) args->data_ptr;
344         remain = args->size;
345
346         offset = args->offset;
347
348         while (remain > 0) {
349                 struct page *page;
350                 char *vaddr;
351                 int ret;
352
353                 /* Operation in this page
354                  *
355                  * page_offset = offset within page
356                  * page_length = bytes to copy for this page
357                  */
358                 page_offset = offset_in_page(offset);
359                 page_length = remain;
360                 if ((page_offset + remain) > PAGE_SIZE)
361                         page_length = PAGE_SIZE - page_offset;
362
363                 page = shmem_read_mapping_page(mapping, offset >> PAGE_SHIFT);
364                 if (IS_ERR(page))
365                         return PTR_ERR(page);
366
367                 vaddr = kmap_atomic(page);
368                 ret = __copy_to_user_inatomic(user_data,
369                                               vaddr + page_offset,
370                                               page_length);
371                 kunmap_atomic(vaddr);
372
373                 mark_page_accessed(page);
374                 page_cache_release(page);
375                 if (ret)
376                         return -EFAULT;
377
378                 remain -= page_length;
379                 user_data += page_length;
380                 offset += page_length;
381         }
382
383         return 0;
384 }
385
386 /**
387  * This is the fallback shmem pread path, which allocates temporary storage
388  * in kernel space to copy_to_user into outside of the struct_mutex, so we
389  * can copy out of the object's backing pages while holding the struct mutex
390  * and not take page faults.
391  */
392 static int
393 i915_gem_shmem_pread_slow(struct drm_device *dev,
394                           struct drm_i915_gem_object *obj,
395                           struct drm_i915_gem_pread *args,
396                           struct drm_file *file)
397 {
398         struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
399         struct mm_struct *mm = current->mm;
400         struct page **user_pages;
401         ssize_t remain;
402         loff_t offset, pinned_pages, i;
403         loff_t first_data_page, last_data_page, num_pages;
404         int shmem_page_offset;
405         int data_page_index, data_page_offset;
406         int page_length;
407         int ret;
408         uint64_t data_ptr = args->data_ptr;
409         int do_bit17_swizzling;
410
411         remain = args->size;
412
413         /* Pin the user pages containing the data.  We can't fault while
414          * holding the struct mutex, yet we want to hold it while
415          * dereferencing the user data.
416          */
417         first_data_page = data_ptr / PAGE_SIZE;
418         last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
419         num_pages = last_data_page - first_data_page + 1;
420
421         user_pages = drm_malloc_ab(num_pages, sizeof(struct page *));
422         if (user_pages == NULL)
423                 return -ENOMEM;
424
425         mutex_unlock(&dev->struct_mutex);
426         down_read(&mm->mmap_sem);
427         pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
428                                       num_pages, 1, 0, user_pages, NULL);
429         up_read(&mm->mmap_sem);
430         mutex_lock(&dev->struct_mutex);
431         if (pinned_pages < num_pages) {
432                 ret = -EFAULT;
433                 goto out;
434         }
435
436         ret = i915_gem_object_set_cpu_read_domain_range(obj,
437                                                         args->offset,
438                                                         args->size);
439         if (ret)
440                 goto out;
441
442         do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
443
444         offset = args->offset;
445
446         while (remain > 0) {
447                 struct page *page;
448
449                 /* Operation in this page
450                  *
451                  * shmem_page_offset = offset within page in shmem file
452                  * data_page_index = page number in get_user_pages return
453                  * data_page_offset = offset with data_page_index page.
454                  * page_length = bytes to copy for this page
455                  */
456                 shmem_page_offset = offset_in_page(offset);
457                 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
458                 data_page_offset = offset_in_page(data_ptr);
459
460                 page_length = remain;
461                 if ((shmem_page_offset + page_length) > PAGE_SIZE)
462                         page_length = PAGE_SIZE - shmem_page_offset;
463                 if ((data_page_offset + page_length) > PAGE_SIZE)
464                         page_length = PAGE_SIZE - data_page_offset;
465
466                 page = shmem_read_mapping_page(mapping, offset >> PAGE_SHIFT);
467                 if (IS_ERR(page)) {
468                         ret = PTR_ERR(page);
469                         goto out;
470                 }
471
472                 if (do_bit17_swizzling) {
473                         slow_shmem_bit17_copy(page,
474                                               shmem_page_offset,
475                                               user_pages[data_page_index],
476                                               data_page_offset,
477                                               page_length,
478                                               1);
479                 } else {
480                         slow_shmem_copy(user_pages[data_page_index],
481                                         data_page_offset,
482                                         page,
483                                         shmem_page_offset,
484                                         page_length);
485                 }
486
487                 mark_page_accessed(page);
488                 page_cache_release(page);
489
490                 remain -= page_length;
491                 data_ptr += page_length;
492                 offset += page_length;
493         }
494
495 out:
496         for (i = 0; i < pinned_pages; i++) {
497                 SetPageDirty(user_pages[i]);
498                 mark_page_accessed(user_pages[i]);
499                 page_cache_release(user_pages[i]);
500         }
501         drm_free_large(user_pages);
502
503         return ret;
504 }
505
506 /**
507  * Reads data from the object referenced by handle.
508  *
509  * On error, the contents of *data are undefined.
510  */
511 int
512 i915_gem_pread_ioctl(struct drm_device *dev, void *data,
513                      struct drm_file *file)
514 {
515         struct drm_i915_gem_pread *args = data;
516         struct drm_i915_gem_object *obj;
517         int ret = 0;
518
519         if (args->size == 0)
520                 return 0;
521
522         if (!access_ok(VERIFY_WRITE,
523                        (char __user *)(uintptr_t)args->data_ptr,
524                        args->size))
525                 return -EFAULT;
526
527         ret = fault_in_pages_writeable((char __user *)(uintptr_t)args->data_ptr,
528                                        args->size);
529         if (ret)
530                 return -EFAULT;
531
532         ret = i915_mutex_lock_interruptible(dev);
533         if (ret)
534                 return ret;
535
536         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
537         if (&obj->base == NULL) {
538                 ret = -ENOENT;
539                 goto unlock;
540         }
541
542         /* Bounds check source.  */
543         if (args->offset > obj->base.size ||
544             args->size > obj->base.size - args->offset) {
545                 ret = -EINVAL;
546                 goto out;
547         }
548
549         trace_i915_gem_object_pread(obj, args->offset, args->size);
550
551         ret = i915_gem_object_set_cpu_read_domain_range(obj,
552                                                         args->offset,
553                                                         args->size);
554         if (ret)
555                 goto out;
556
557         ret = -EFAULT;
558         if (!i915_gem_object_needs_bit17_swizzle(obj))
559                 ret = i915_gem_shmem_pread_fast(dev, obj, args, file);
560         if (ret == -EFAULT)
561                 ret = i915_gem_shmem_pread_slow(dev, obj, args, file);
562
563 out:
564         drm_gem_object_unreference(&obj->base);
565 unlock:
566         mutex_unlock(&dev->struct_mutex);
567         return ret;
568 }
569
570 /* This is the fast write path which cannot handle
571  * page faults in the source data
572  */
573
574 static inline int
575 fast_user_write(struct io_mapping *mapping,
576                 loff_t page_base, int page_offset,
577                 char __user *user_data,
578                 int length)
579 {
580         char *vaddr_atomic;
581         unsigned long unwritten;
582
583         vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base);
584         unwritten = __copy_from_user_inatomic_nocache(vaddr_atomic + page_offset,
585                                                       user_data, length);
586         io_mapping_unmap_atomic(vaddr_atomic);
587         return unwritten;
588 }
589
590 /* Here's the write path which can sleep for
591  * page faults
592  */
593
594 static inline void
595 slow_kernel_write(struct io_mapping *mapping,
596                   loff_t gtt_base, int gtt_offset,
597                   struct page *user_page, int user_offset,
598                   int length)
599 {
600         char __iomem *dst_vaddr;
601         char *src_vaddr;
602
603         dst_vaddr = io_mapping_map_wc(mapping, gtt_base);
604         src_vaddr = kmap(user_page);
605
606         memcpy_toio(dst_vaddr + gtt_offset,
607                     src_vaddr + user_offset,
608                     length);
609
610         kunmap(user_page);
611         io_mapping_unmap(dst_vaddr);
612 }
613
614 /**
615  * This is the fast pwrite path, where we copy the data directly from the
616  * user into the GTT, uncached.
617  */
618 static int
619 i915_gem_gtt_pwrite_fast(struct drm_device *dev,
620                          struct drm_i915_gem_object *obj,
621                          struct drm_i915_gem_pwrite *args,
622                          struct drm_file *file)
623 {
624         drm_i915_private_t *dev_priv = dev->dev_private;
625         ssize_t remain;
626         loff_t offset, page_base;
627         char __user *user_data;
628         int page_offset, page_length;
629
630         user_data = (char __user *) (uintptr_t) args->data_ptr;
631         remain = args->size;
632
633         offset = obj->gtt_offset + args->offset;
634
635         while (remain > 0) {
636                 /* Operation in this page
637                  *
638                  * page_base = page offset within aperture
639                  * page_offset = offset within page
640                  * page_length = bytes to copy for this page
641                  */
642                 page_base = offset & PAGE_MASK;
643                 page_offset = offset_in_page(offset);
644                 page_length = remain;
645                 if ((page_offset + remain) > PAGE_SIZE)
646                         page_length = PAGE_SIZE - page_offset;
647
648                 /* If we get a fault while copying data, then (presumably) our
649                  * source page isn't available.  Return the error and we'll
650                  * retry in the slow path.
651                  */
652                 if (fast_user_write(dev_priv->mm.gtt_mapping, page_base,
653                                     page_offset, user_data, page_length))
654                         return -EFAULT;
655
656                 remain -= page_length;
657                 user_data += page_length;
658                 offset += page_length;
659         }
660
661         return 0;
662 }
663
664 /**
665  * This is the fallback GTT pwrite path, which uses get_user_pages to pin
666  * the memory and maps it using kmap_atomic for copying.
667  *
668  * This code resulted in x11perf -rgb10text consuming about 10% more CPU
669  * than using i915_gem_gtt_pwrite_fast on a G45 (32-bit).
670  */
671 static int
672 i915_gem_gtt_pwrite_slow(struct drm_device *dev,
673                          struct drm_i915_gem_object *obj,
674                          struct drm_i915_gem_pwrite *args,
675                          struct drm_file *file)
676 {
677         drm_i915_private_t *dev_priv = dev->dev_private;
678         ssize_t remain;
679         loff_t gtt_page_base, offset;
680         loff_t first_data_page, last_data_page, num_pages;
681         loff_t pinned_pages, i;
682         struct page **user_pages;
683         struct mm_struct *mm = current->mm;
684         int gtt_page_offset, data_page_offset, data_page_index, page_length;
685         int ret;
686         uint64_t data_ptr = args->data_ptr;
687
688         remain = args->size;
689
690         /* Pin the user pages containing the data.  We can't fault while
691          * holding the struct mutex, and all of the pwrite implementations
692          * want to hold it while dereferencing the user data.
693          */
694         first_data_page = data_ptr / PAGE_SIZE;
695         last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
696         num_pages = last_data_page - first_data_page + 1;
697
698         user_pages = drm_malloc_ab(num_pages, sizeof(struct page *));
699         if (user_pages == NULL)
700                 return -ENOMEM;
701
702         mutex_unlock(&dev->struct_mutex);
703         down_read(&mm->mmap_sem);
704         pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
705                                       num_pages, 0, 0, user_pages, NULL);
706         up_read(&mm->mmap_sem);
707         mutex_lock(&dev->struct_mutex);
708         if (pinned_pages < num_pages) {
709                 ret = -EFAULT;
710                 goto out_unpin_pages;
711         }
712
713         ret = i915_gem_object_set_to_gtt_domain(obj, true);
714         if (ret)
715                 goto out_unpin_pages;
716
717         ret = i915_gem_object_put_fence(obj);
718         if (ret)
719                 goto out_unpin_pages;
720
721         offset = obj->gtt_offset + args->offset;
722
723         while (remain > 0) {
724                 /* Operation in this page
725                  *
726                  * gtt_page_base = page offset within aperture
727                  * gtt_page_offset = offset within page in aperture
728                  * data_page_index = page number in get_user_pages return
729                  * data_page_offset = offset with data_page_index page.
730                  * page_length = bytes to copy for this page
731                  */
732                 gtt_page_base = offset & PAGE_MASK;
733                 gtt_page_offset = offset_in_page(offset);
734                 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
735                 data_page_offset = offset_in_page(data_ptr);
736
737                 page_length = remain;
738                 if ((gtt_page_offset + page_length) > PAGE_SIZE)
739                         page_length = PAGE_SIZE - gtt_page_offset;
740                 if ((data_page_offset + page_length) > PAGE_SIZE)
741                         page_length = PAGE_SIZE - data_page_offset;
742
743                 slow_kernel_write(dev_priv->mm.gtt_mapping,
744                                   gtt_page_base, gtt_page_offset,
745                                   user_pages[data_page_index],
746                                   data_page_offset,
747                                   page_length);
748
749                 remain -= page_length;
750                 offset += page_length;
751                 data_ptr += page_length;
752         }
753
754 out_unpin_pages:
755         for (i = 0; i < pinned_pages; i++)
756                 page_cache_release(user_pages[i]);
757         drm_free_large(user_pages);
758
759         return ret;
760 }
761
762 /**
763  * This is the fast shmem pwrite path, which attempts to directly
764  * copy_from_user into the kmapped pages backing the object.
765  */
766 static int
767 i915_gem_shmem_pwrite_fast(struct drm_device *dev,
768                            struct drm_i915_gem_object *obj,
769                            struct drm_i915_gem_pwrite *args,
770                            struct drm_file *file)
771 {
772         struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
773         ssize_t remain;
774         loff_t offset;
775         char __user *user_data;
776         int page_offset, page_length;
777
778         user_data = (char __user *) (uintptr_t) args->data_ptr;
779         remain = args->size;
780
781         offset = args->offset;
782         obj->dirty = 1;
783
784         while (remain > 0) {
785                 struct page *page;
786                 char *vaddr;
787                 int ret;
788
789                 /* Operation in this page
790                  *
791                  * page_offset = offset within page
792                  * page_length = bytes to copy for this page
793                  */
794                 page_offset = offset_in_page(offset);
795                 page_length = remain;
796                 if ((page_offset + remain) > PAGE_SIZE)
797                         page_length = PAGE_SIZE - page_offset;
798
799                 page = shmem_read_mapping_page(mapping, offset >> PAGE_SHIFT);
800                 if (IS_ERR(page))
801                         return PTR_ERR(page);
802
803                 vaddr = kmap_atomic(page, KM_USER0);
804                 ret = __copy_from_user_inatomic(vaddr + page_offset,
805                                                 user_data,
806                                                 page_length);
807                 kunmap_atomic(vaddr, KM_USER0);
808
809                 set_page_dirty(page);
810                 mark_page_accessed(page);
811                 page_cache_release(page);
812
813                 /* If we get a fault while copying data, then (presumably) our
814                  * source page isn't available.  Return the error and we'll
815                  * retry in the slow path.
816                  */
817                 if (ret)
818                         return -EFAULT;
819
820                 remain -= page_length;
821                 user_data += page_length;
822                 offset += page_length;
823         }
824
825         return 0;
826 }
827
828 /**
829  * This is the fallback shmem pwrite path, which uses get_user_pages to pin
830  * the memory and maps it using kmap_atomic for copying.
831  *
832  * This avoids taking mmap_sem for faulting on the user's address while the
833  * struct_mutex is held.
834  */
835 static int
836 i915_gem_shmem_pwrite_slow(struct drm_device *dev,
837                            struct drm_i915_gem_object *obj,
838                            struct drm_i915_gem_pwrite *args,
839                            struct drm_file *file)
840 {
841         struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
842         struct mm_struct *mm = current->mm;
843         struct page **user_pages;
844         ssize_t remain;
845         loff_t offset, pinned_pages, i;
846         loff_t first_data_page, last_data_page, num_pages;
847         int shmem_page_offset;
848         int data_page_index,  data_page_offset;
849         int page_length;
850         int ret;
851         uint64_t data_ptr = args->data_ptr;
852         int do_bit17_swizzling;
853
854         remain = args->size;
855
856         /* Pin the user pages containing the data.  We can't fault while
857          * holding the struct mutex, and all of the pwrite implementations
858          * want to hold it while dereferencing the user data.
859          */
860         first_data_page = data_ptr / PAGE_SIZE;
861         last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
862         num_pages = last_data_page - first_data_page + 1;
863
864         user_pages = drm_malloc_ab(num_pages, sizeof(struct page *));
865         if (user_pages == NULL)
866                 return -ENOMEM;
867
868         mutex_unlock(&dev->struct_mutex);
869         down_read(&mm->mmap_sem);
870         pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
871                                       num_pages, 0, 0, user_pages, NULL);
872         up_read(&mm->mmap_sem);
873         mutex_lock(&dev->struct_mutex);
874         if (pinned_pages < num_pages) {
875                 ret = -EFAULT;
876                 goto out;
877         }
878
879         ret = i915_gem_object_set_to_cpu_domain(obj, 1);
880         if (ret)
881                 goto out;
882
883         do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
884
885         offset = args->offset;
886         obj->dirty = 1;
887
888         while (remain > 0) {
889                 struct page *page;
890
891                 /* Operation in this page
892                  *
893                  * shmem_page_offset = offset within page in shmem file
894                  * data_page_index = page number in get_user_pages return
895                  * data_page_offset = offset with data_page_index page.
896                  * page_length = bytes to copy for this page
897                  */
898                 shmem_page_offset = offset_in_page(offset);
899                 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
900                 data_page_offset = offset_in_page(data_ptr);
901
902                 page_length = remain;
903                 if ((shmem_page_offset + page_length) > PAGE_SIZE)
904                         page_length = PAGE_SIZE - shmem_page_offset;
905                 if ((data_page_offset + page_length) > PAGE_SIZE)
906                         page_length = PAGE_SIZE - data_page_offset;
907
908                 page = shmem_read_mapping_page(mapping, offset >> PAGE_SHIFT);
909                 if (IS_ERR(page)) {
910                         ret = PTR_ERR(page);
911                         goto out;
912                 }
913
914                 if (do_bit17_swizzling) {
915                         slow_shmem_bit17_copy(page,
916                                               shmem_page_offset,
917                                               user_pages[data_page_index],
918                                               data_page_offset,
919                                               page_length,
920                                               0);
921                 } else {
922                         slow_shmem_copy(page,
923                                         shmem_page_offset,
924                                         user_pages[data_page_index],
925                                         data_page_offset,
926                                         page_length);
927                 }
928
929                 set_page_dirty(page);
930                 mark_page_accessed(page);
931                 page_cache_release(page);
932
933                 remain -= page_length;
934                 data_ptr += page_length;
935                 offset += page_length;
936         }
937
938 out:
939         for (i = 0; i < pinned_pages; i++)
940                 page_cache_release(user_pages[i]);
941         drm_free_large(user_pages);
942
943         return ret;
944 }
945
946 /**
947  * Writes data to the object referenced by handle.
948  *
949  * On error, the contents of the buffer that were to be modified are undefined.
950  */
951 int
952 i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
953                       struct drm_file *file)
954 {
955         struct drm_i915_gem_pwrite *args = data;
956         struct drm_i915_gem_object *obj;
957         int ret;
958
959         if (args->size == 0)
960                 return 0;
961
962         if (!access_ok(VERIFY_READ,
963                        (char __user *)(uintptr_t)args->data_ptr,
964                        args->size))
965                 return -EFAULT;
966
967         ret = fault_in_pages_readable((char __user *)(uintptr_t)args->data_ptr,
968                                       args->size);
969         if (ret)
970                 return -EFAULT;
971
972         ret = i915_mutex_lock_interruptible(dev);
973         if (ret)
974                 return ret;
975
976         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
977         if (&obj->base == NULL) {
978                 ret = -ENOENT;
979                 goto unlock;
980         }
981
982         /* Bounds check destination. */
983         if (args->offset > obj->base.size ||
984             args->size > obj->base.size - args->offset) {
985                 ret = -EINVAL;
986                 goto out;
987         }
988
989         trace_i915_gem_object_pwrite(obj, args->offset, args->size);
990
991         /* We can only do the GTT pwrite on untiled buffers, as otherwise
992          * it would end up going through the fenced access, and we'll get
993          * different detiling behavior between reading and writing.
994          * pread/pwrite currently are reading and writing from the CPU
995          * perspective, requiring manual detiling by the client.
996          */
997         if (obj->phys_obj)
998                 ret = i915_gem_phys_pwrite(dev, obj, args, file);
999         else if (obj->gtt_space &&
1000                  obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
1001                 ret = i915_gem_object_pin(obj, 0, true);
1002                 if (ret)
1003                         goto out;
1004
1005                 ret = i915_gem_object_set_to_gtt_domain(obj, true);
1006                 if (ret)
1007                         goto out_unpin;
1008
1009                 ret = i915_gem_object_put_fence(obj);
1010                 if (ret)
1011                         goto out_unpin;
1012
1013                 ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file);
1014                 if (ret == -EFAULT)
1015                         ret = i915_gem_gtt_pwrite_slow(dev, obj, args, file);
1016
1017 out_unpin:
1018                 i915_gem_object_unpin(obj);
1019         } else {
1020                 ret = i915_gem_object_set_to_cpu_domain(obj, 1);
1021                 if (ret)
1022                         goto out;
1023
1024                 ret = -EFAULT;
1025                 if (!i915_gem_object_needs_bit17_swizzle(obj))
1026                         ret = i915_gem_shmem_pwrite_fast(dev, obj, args, file);
1027                 if (ret == -EFAULT)
1028                         ret = i915_gem_shmem_pwrite_slow(dev, obj, args, file);
1029         }
1030
1031 out:
1032         drm_gem_object_unreference(&obj->base);
1033 unlock:
1034         mutex_unlock(&dev->struct_mutex);
1035         return ret;
1036 }
1037
1038 /**
1039  * Called when user space prepares to use an object with the CPU, either
1040  * through the mmap ioctl's mapping or a GTT mapping.
1041  */
1042 int
1043 i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
1044                           struct drm_file *file)
1045 {
1046         struct drm_i915_gem_set_domain *args = data;
1047         struct drm_i915_gem_object *obj;
1048         uint32_t read_domains = args->read_domains;
1049         uint32_t write_domain = args->write_domain;
1050         int ret;
1051
1052         if (!(dev->driver->driver_features & DRIVER_GEM))
1053                 return -ENODEV;
1054
1055         /* Only handle setting domains to types used by the CPU. */
1056         if (write_domain & I915_GEM_GPU_DOMAINS)
1057                 return -EINVAL;
1058
1059         if (read_domains & I915_GEM_GPU_DOMAINS)
1060                 return -EINVAL;
1061
1062         /* Having something in the write domain implies it's in the read
1063          * domain, and only that read domain.  Enforce that in the request.
1064          */
1065         if (write_domain != 0 && read_domains != write_domain)
1066                 return -EINVAL;
1067
1068         ret = i915_mutex_lock_interruptible(dev);
1069         if (ret)
1070                 return ret;
1071
1072         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
1073         if (&obj->base == NULL) {
1074                 ret = -ENOENT;
1075                 goto unlock;
1076         }
1077
1078         if (read_domains & I915_GEM_DOMAIN_GTT) {
1079                 ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
1080
1081                 /* Silently promote "you're not bound, there was nothing to do"
1082                  * to success, since the client was just asking us to
1083                  * make sure everything was done.
1084                  */
1085                 if (ret == -EINVAL)
1086                         ret = 0;
1087         } else {
1088                 ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
1089         }
1090
1091         drm_gem_object_unreference(&obj->base);
1092 unlock:
1093         mutex_unlock(&dev->struct_mutex);
1094         return ret;
1095 }
1096
1097 /**
1098  * Called when user space has done writes to this buffer
1099  */
1100 int
1101 i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
1102                          struct drm_file *file)
1103 {
1104         struct drm_i915_gem_sw_finish *args = data;
1105         struct drm_i915_gem_object *obj;
1106         int ret = 0;
1107
1108         if (!(dev->driver->driver_features & DRIVER_GEM))
1109                 return -ENODEV;
1110
1111         ret = i915_mutex_lock_interruptible(dev);
1112         if (ret)
1113                 return ret;
1114
1115         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
1116         if (&obj->base == NULL) {
1117                 ret = -ENOENT;
1118                 goto unlock;
1119         }
1120
1121         /* Pinned buffers may be scanout, so flush the cache */
1122         if (obj->pin_count)
1123                 i915_gem_object_flush_cpu_write_domain(obj);
1124
1125         drm_gem_object_unreference(&obj->base);
1126 unlock:
1127         mutex_unlock(&dev->struct_mutex);
1128         return ret;
1129 }
1130
1131 /**
1132  * Maps the contents of an object, returning the address it is mapped
1133  * into.
1134  *
1135  * While the mapping holds a reference on the contents of the object, it doesn't
1136  * imply a ref on the object itself.
1137  */
1138 int
1139 i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
1140                     struct drm_file *file)
1141 {
1142         struct drm_i915_private *dev_priv = dev->dev_private;
1143         struct drm_i915_gem_mmap *args = data;
1144         struct drm_gem_object *obj;
1145         unsigned long addr;
1146
1147         if (!(dev->driver->driver_features & DRIVER_GEM))
1148                 return -ENODEV;
1149
1150         obj = drm_gem_object_lookup(dev, file, args->handle);
1151         if (obj == NULL)
1152                 return -ENOENT;
1153
1154         if (obj->size > dev_priv->mm.gtt_mappable_end) {
1155                 drm_gem_object_unreference_unlocked(obj);
1156                 return -E2BIG;
1157         }
1158
1159         down_write(&current->mm->mmap_sem);
1160         addr = do_mmap(obj->filp, 0, args->size,
1161                        PROT_READ | PROT_WRITE, MAP_SHARED,
1162                        args->offset);
1163         up_write(&current->mm->mmap_sem);
1164         drm_gem_object_unreference_unlocked(obj);
1165         if (IS_ERR((void *)addr))
1166                 return addr;
1167
1168         args->addr_ptr = (uint64_t) addr;
1169
1170         return 0;
1171 }
1172
1173 /**
1174  * i915_gem_fault - fault a page into the GTT
1175  * vma: VMA in question
1176  * vmf: fault info
1177  *
1178  * The fault handler is set up by drm_gem_mmap() when a object is GTT mapped
1179  * from userspace.  The fault handler takes care of binding the object to
1180  * the GTT (if needed), allocating and programming a fence register (again,
1181  * only if needed based on whether the old reg is still valid or the object
1182  * is tiled) and inserting a new PTE into the faulting process.
1183  *
1184  * Note that the faulting process may involve evicting existing objects
1185  * from the GTT and/or fence registers to make room.  So performance may
1186  * suffer if the GTT working set is large or there are few fence registers
1187  * left.
1188  */
1189 int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1190 {
1191         struct drm_i915_gem_object *obj = to_intel_bo(vma->vm_private_data);
1192         struct drm_device *dev = obj->base.dev;
1193         drm_i915_private_t *dev_priv = dev->dev_private;
1194         pgoff_t page_offset;
1195         unsigned long pfn;
1196         int ret = 0;
1197         bool write = !!(vmf->flags & FAULT_FLAG_WRITE);
1198
1199         /* We don't use vmf->pgoff since that has the fake offset */
1200         page_offset = ((unsigned long)vmf->virtual_address - vma->vm_start) >>
1201                 PAGE_SHIFT;
1202
1203         ret = i915_mutex_lock_interruptible(dev);
1204         if (ret)
1205                 goto out;
1206
1207         trace_i915_gem_object_fault(obj, page_offset, true, write);
1208
1209         /* Now bind it into the GTT if needed */
1210         if (!obj->map_and_fenceable) {
1211                 ret = i915_gem_object_unbind(obj);
1212                 if (ret)
1213                         goto unlock;
1214         }
1215         if (!obj->gtt_space) {
1216                 ret = i915_gem_object_bind_to_gtt(obj, 0, true);
1217                 if (ret)
1218                         goto unlock;
1219
1220                 ret = i915_gem_object_set_to_gtt_domain(obj, write);
1221                 if (ret)
1222                         goto unlock;
1223         }
1224
1225         if (obj->tiling_mode == I915_TILING_NONE)
1226                 ret = i915_gem_object_put_fence(obj);
1227         else
1228                 ret = i915_gem_object_get_fence(obj, NULL);
1229         if (ret)
1230                 goto unlock;
1231
1232         if (i915_gem_object_is_inactive(obj))
1233                 list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
1234
1235         obj->fault_mappable = true;
1236
1237         pfn = ((dev->agp->base + obj->gtt_offset) >> PAGE_SHIFT) +
1238                 page_offset;
1239
1240         /* Finally, remap it using the new GTT offset */
1241         ret = vm_insert_pfn(vma, (unsigned long)vmf->virtual_address, pfn);
1242 unlock:
1243         mutex_unlock(&dev->struct_mutex);
1244 out:
1245         switch (ret) {
1246         case -EIO:
1247         case -EAGAIN:
1248                 /* Give the error handler a chance to run and move the
1249                  * objects off the GPU active list. Next time we service the
1250                  * fault, we should be able to transition the page into the
1251                  * GTT without touching the GPU (and so avoid further
1252                  * EIO/EGAIN). If the GPU is wedged, then there is no issue
1253                  * with coherency, just lost writes.
1254                  */
1255                 set_need_resched();
1256         case 0:
1257         case -ERESTARTSYS:
1258         case -EINTR:
1259                 return VM_FAULT_NOPAGE;
1260         case -ENOMEM:
1261                 return VM_FAULT_OOM;
1262         default:
1263                 return VM_FAULT_SIGBUS;
1264         }
1265 }
1266
1267 /**
1268  * i915_gem_create_mmap_offset - create a fake mmap offset for an object
1269  * @obj: obj in question
1270  *
1271  * GEM memory mapping works by handing back to userspace a fake mmap offset
1272  * it can use in a subsequent mmap(2) call.  The DRM core code then looks
1273  * up the object based on the offset and sets up the various memory mapping
1274  * structures.
1275  *
1276  * This routine allocates and attaches a fake offset for @obj.
1277  */
1278 static int
1279 i915_gem_create_mmap_offset(struct drm_i915_gem_object *obj)
1280 {
1281         struct drm_device *dev = obj->base.dev;
1282         struct drm_gem_mm *mm = dev->mm_private;
1283         struct drm_map_list *list;
1284         struct drm_local_map *map;
1285         int ret = 0;
1286
1287         /* Set the object up for mmap'ing */
1288         list = &obj->base.map_list;
1289         list->map = kzalloc(sizeof(struct drm_map_list), GFP_KERNEL);
1290         if (!list->map)
1291                 return -ENOMEM;
1292
1293         map = list->map;
1294         map->type = _DRM_GEM;
1295         map->size = obj->base.size;
1296         map->handle = obj;
1297
1298         /* Get a DRM GEM mmap offset allocated... */
1299         list->file_offset_node = drm_mm_search_free(&mm->offset_manager,
1300                                                     obj->base.size / PAGE_SIZE,
1301                                                     0, 0);
1302         if (!list->file_offset_node) {
1303                 DRM_ERROR("failed to allocate offset for bo %d\n",
1304                           obj->base.name);
1305                 ret = -ENOSPC;
1306                 goto out_free_list;
1307         }
1308
1309         list->file_offset_node = drm_mm_get_block(list->file_offset_node,
1310                                                   obj->base.size / PAGE_SIZE,
1311                                                   0);
1312         if (!list->file_offset_node) {
1313                 ret = -ENOMEM;
1314                 goto out_free_list;
1315         }
1316
1317         list->hash.key = list->file_offset_node->start;
1318         ret = drm_ht_insert_item(&mm->offset_hash, &list->hash);
1319         if (ret) {
1320                 DRM_ERROR("failed to add to map hash\n");
1321                 goto out_free_mm;
1322         }
1323
1324         return 0;
1325
1326 out_free_mm:
1327         drm_mm_put_block(list->file_offset_node);
1328 out_free_list:
1329         kfree(list->map);
1330         list->map = NULL;
1331
1332         return ret;
1333 }
1334
1335 /**
1336  * i915_gem_release_mmap - remove physical page mappings
1337  * @obj: obj in question
1338  *
1339  * Preserve the reservation of the mmapping with the DRM core code, but
1340  * relinquish ownership of the pages back to the system.
1341  *
1342  * It is vital that we remove the page mapping if we have mapped a tiled
1343  * object through the GTT and then lose the fence register due to
1344  * resource pressure. Similarly if the object has been moved out of the
1345  * aperture, than pages mapped into userspace must be revoked. Removing the
1346  * mapping will then trigger a page fault on the next user access, allowing
1347  * fixup by i915_gem_fault().
1348  */
1349 void
1350 i915_gem_release_mmap(struct drm_i915_gem_object *obj)
1351 {
1352         if (!obj->fault_mappable)
1353                 return;
1354
1355         if (obj->base.dev->dev_mapping)
1356                 unmap_mapping_range(obj->base.dev->dev_mapping,
1357                                     (loff_t)obj->base.map_list.hash.key<<PAGE_SHIFT,
1358                                     obj->base.size, 1);
1359
1360         obj->fault_mappable = false;
1361 }
1362
1363 static void
1364 i915_gem_free_mmap_offset(struct drm_i915_gem_object *obj)
1365 {
1366         struct drm_device *dev = obj->base.dev;
1367         struct drm_gem_mm *mm = dev->mm_private;
1368         struct drm_map_list *list = &obj->base.map_list;
1369
1370         drm_ht_remove_item(&mm->offset_hash, &list->hash);
1371         drm_mm_put_block(list->file_offset_node);
1372         kfree(list->map);
1373         list->map = NULL;
1374 }
1375
1376 static uint32_t
1377 i915_gem_get_gtt_size(struct drm_i915_gem_object *obj)
1378 {
1379         struct drm_device *dev = obj->base.dev;
1380         uint32_t size;
1381
1382         if (INTEL_INFO(dev)->gen >= 4 ||
1383             obj->tiling_mode == I915_TILING_NONE)
1384                 return obj->base.size;
1385
1386         /* Previous chips need a power-of-two fence region when tiling */
1387         if (INTEL_INFO(dev)->gen == 3)
1388                 size = 1024*1024;
1389         else
1390                 size = 512*1024;
1391
1392         while (size < obj->base.size)
1393                 size <<= 1;
1394
1395         return size;
1396 }
1397
1398 /**
1399  * i915_gem_get_gtt_alignment - return required GTT alignment for an object
1400  * @obj: object to check
1401  *
1402  * Return the required GTT alignment for an object, taking into account
1403  * potential fence register mapping.
1404  */
1405 static uint32_t
1406 i915_gem_get_gtt_alignment(struct drm_i915_gem_object *obj)
1407 {
1408         struct drm_device *dev = obj->base.dev;
1409
1410         /*
1411          * Minimum alignment is 4k (GTT page size), but might be greater
1412          * if a fence register is needed for the object.
1413          */
1414         if (INTEL_INFO(dev)->gen >= 4 ||
1415             obj->tiling_mode == I915_TILING_NONE)
1416                 return 4096;
1417
1418         /*
1419          * Previous chips need to be aligned to the size of the smallest
1420          * fence register that can contain the object.
1421          */
1422         return i915_gem_get_gtt_size(obj);
1423 }
1424
1425 /**
1426  * i915_gem_get_unfenced_gtt_alignment - return required GTT alignment for an
1427  *                                       unfenced object
1428  * @obj: object to check
1429  *
1430  * Return the required GTT alignment for an object, only taking into account
1431  * unfenced tiled surface requirements.
1432  */
1433 uint32_t
1434 i915_gem_get_unfenced_gtt_alignment(struct drm_i915_gem_object *obj)
1435 {
1436         struct drm_device *dev = obj->base.dev;
1437         int tile_height;
1438
1439         /*
1440          * Minimum alignment is 4k (GTT page size) for sane hw.
1441          */
1442         if (INTEL_INFO(dev)->gen >= 4 || IS_G33(dev) ||
1443             obj->tiling_mode == I915_TILING_NONE)
1444                 return 4096;
1445
1446         /*
1447          * Older chips need unfenced tiled buffers to be aligned to the left
1448          * edge of an even tile row (where tile rows are counted as if the bo is
1449          * placed in a fenced gtt region).
1450          */
1451         if (IS_GEN2(dev))
1452                 tile_height = 16;
1453         else if (obj->tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev))
1454                 tile_height = 32;
1455         else
1456                 tile_height = 8;
1457
1458         return tile_height * obj->stride * 2;
1459 }
1460
1461 int
1462 i915_gem_mmap_gtt(struct drm_file *file,
1463                   struct drm_device *dev,
1464                   uint32_t handle,
1465                   uint64_t *offset)
1466 {
1467         struct drm_i915_private *dev_priv = dev->dev_private;
1468         struct drm_i915_gem_object *obj;
1469         int ret;
1470
1471         if (!(dev->driver->driver_features & DRIVER_GEM))
1472                 return -ENODEV;
1473
1474         ret = i915_mutex_lock_interruptible(dev);
1475         if (ret)
1476                 return ret;
1477
1478         obj = to_intel_bo(drm_gem_object_lookup(dev, file, handle));
1479         if (&obj->base == NULL) {
1480                 ret = -ENOENT;
1481                 goto unlock;
1482         }
1483
1484         if (obj->base.size > dev_priv->mm.gtt_mappable_end) {
1485                 ret = -E2BIG;
1486                 goto unlock;
1487         }
1488
1489         if (obj->madv != I915_MADV_WILLNEED) {
1490                 DRM_ERROR("Attempting to mmap a purgeable buffer\n");
1491                 ret = -EINVAL;
1492                 goto out;
1493         }
1494
1495         if (!obj->base.map_list.map) {
1496                 ret = i915_gem_create_mmap_offset(obj);
1497                 if (ret)
1498                         goto out;
1499         }
1500
1501         *offset = (u64)obj->base.map_list.hash.key << PAGE_SHIFT;
1502
1503 out:
1504         drm_gem_object_unreference(&obj->base);
1505 unlock:
1506         mutex_unlock(&dev->struct_mutex);
1507         return ret;
1508 }
1509
1510 /**
1511  * i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing
1512  * @dev: DRM device
1513  * @data: GTT mapping ioctl data
1514  * @file: GEM object info
1515  *
1516  * Simply returns the fake offset to userspace so it can mmap it.
1517  * The mmap call will end up in drm_gem_mmap(), which will set things
1518  * up so we can get faults in the handler above.
1519  *
1520  * The fault handler will take care of binding the object into the GTT
1521  * (since it may have been evicted to make room for something), allocating
1522  * a fence register, and mapping the appropriate aperture address into
1523  * userspace.
1524  */
1525 int
1526 i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
1527                         struct drm_file *file)
1528 {
1529         struct drm_i915_gem_mmap_gtt *args = data;
1530
1531         if (!(dev->driver->driver_features & DRIVER_GEM))
1532                 return -ENODEV;
1533
1534         return i915_gem_mmap_gtt(file, dev, args->handle, &args->offset);
1535 }
1536
1537
1538 static int
1539 i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj,
1540                               gfp_t gfpmask)
1541 {
1542         int page_count, i;
1543         struct address_space *mapping;
1544         struct inode *inode;
1545         struct page *page;
1546
1547         /* Get the list of pages out of our struct file.  They'll be pinned
1548          * at this point until we release them.
1549          */
1550         page_count = obj->base.size / PAGE_SIZE;
1551         BUG_ON(obj->pages != NULL);
1552         obj->pages = drm_malloc_ab(page_count, sizeof(struct page *));
1553         if (obj->pages == NULL)
1554                 return -ENOMEM;
1555
1556         inode = obj->base.filp->f_path.dentry->d_inode;
1557         mapping = inode->i_mapping;
1558         gfpmask |= mapping_gfp_mask(mapping);
1559
1560         for (i = 0; i < page_count; i++) {
1561                 page = shmem_read_mapping_page_gfp(mapping, i, gfpmask);
1562                 if (IS_ERR(page))
1563                         goto err_pages;
1564
1565                 obj->pages[i] = page;
1566         }
1567
1568         if (obj->tiling_mode != I915_TILING_NONE)
1569                 i915_gem_object_do_bit_17_swizzle(obj);
1570
1571         return 0;
1572
1573 err_pages:
1574         while (i--)
1575                 page_cache_release(obj->pages[i]);
1576
1577         drm_free_large(obj->pages);
1578         obj->pages = NULL;
1579         return PTR_ERR(page);
1580 }
1581
1582 static void
1583 i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj)
1584 {
1585         int page_count = obj->base.size / PAGE_SIZE;
1586         int i;
1587
1588         BUG_ON(obj->madv == __I915_MADV_PURGED);
1589
1590         if (obj->tiling_mode != I915_TILING_NONE)
1591                 i915_gem_object_save_bit_17_swizzle(obj);
1592
1593         if (obj->madv == I915_MADV_DONTNEED)
1594                 obj->dirty = 0;
1595
1596         for (i = 0; i < page_count; i++) {
1597                 if (obj->dirty)
1598                         set_page_dirty(obj->pages[i]);
1599
1600                 if (obj->madv == I915_MADV_WILLNEED)
1601                         mark_page_accessed(obj->pages[i]);
1602
1603                 page_cache_release(obj->pages[i]);
1604         }
1605         obj->dirty = 0;
1606
1607         drm_free_large(obj->pages);
1608         obj->pages = NULL;
1609 }
1610
1611 void
1612 i915_gem_object_move_to_active(struct drm_i915_gem_object *obj,
1613                                struct intel_ring_buffer *ring,
1614                                u32 seqno)
1615 {
1616         struct drm_device *dev = obj->base.dev;
1617         struct drm_i915_private *dev_priv = dev->dev_private;
1618
1619         BUG_ON(ring == NULL);
1620         obj->ring = ring;
1621
1622         /* Add a reference if we're newly entering the active list. */
1623         if (!obj->active) {
1624                 drm_gem_object_reference(&obj->base);
1625                 obj->active = 1;
1626         }
1627
1628         /* Move from whatever list we were on to the tail of execution. */
1629         list_move_tail(&obj->mm_list, &dev_priv->mm.active_list);
1630         list_move_tail(&obj->ring_list, &ring->active_list);
1631
1632         obj->last_rendering_seqno = seqno;
1633         if (obj->fenced_gpu_access) {
1634                 struct drm_i915_fence_reg *reg;
1635
1636                 BUG_ON(obj->fence_reg == I915_FENCE_REG_NONE);
1637
1638                 obj->last_fenced_seqno = seqno;
1639                 obj->last_fenced_ring = ring;
1640
1641                 reg = &dev_priv->fence_regs[obj->fence_reg];
1642                 list_move_tail(&reg->lru_list, &dev_priv->mm.fence_list);
1643         }
1644 }
1645
1646 static void
1647 i915_gem_object_move_off_active(struct drm_i915_gem_object *obj)
1648 {
1649         list_del_init(&obj->ring_list);
1650         obj->last_rendering_seqno = 0;
1651 }
1652
1653 static void
1654 i915_gem_object_move_to_flushing(struct drm_i915_gem_object *obj)
1655 {
1656         struct drm_device *dev = obj->base.dev;
1657         drm_i915_private_t *dev_priv = dev->dev_private;
1658
1659         BUG_ON(!obj->active);
1660         list_move_tail(&obj->mm_list, &dev_priv->mm.flushing_list);
1661
1662         i915_gem_object_move_off_active(obj);
1663 }
1664
1665 static void
1666 i915_gem_object_move_to_inactive(struct drm_i915_gem_object *obj)
1667 {
1668         struct drm_device *dev = obj->base.dev;
1669         struct drm_i915_private *dev_priv = dev->dev_private;
1670
1671         if (obj->pin_count != 0)
1672                 list_move_tail(&obj->mm_list, &dev_priv->mm.pinned_list);
1673         else
1674                 list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
1675
1676         BUG_ON(!list_empty(&obj->gpu_write_list));
1677         BUG_ON(!obj->active);
1678         obj->ring = NULL;
1679
1680         i915_gem_object_move_off_active(obj);
1681         obj->fenced_gpu_access = false;
1682
1683         obj->active = 0;
1684         obj->pending_gpu_write = false;
1685         drm_gem_object_unreference(&obj->base);
1686
1687         WARN_ON(i915_verify_lists(dev));
1688 }
1689
1690 /* Immediately discard the backing storage */
1691 static void
1692 i915_gem_object_truncate(struct drm_i915_gem_object *obj)
1693 {
1694         struct inode *inode;
1695
1696         /* Our goal here is to return as much of the memory as
1697          * is possible back to the system as we are called from OOM.
1698          * To do this we must instruct the shmfs to drop all of its
1699          * backing pages, *now*.
1700          */
1701         inode = obj->base.filp->f_path.dentry->d_inode;
1702         shmem_truncate_range(inode, 0, (loff_t)-1);
1703
1704         obj->madv = __I915_MADV_PURGED;
1705 }
1706
1707 static inline int
1708 i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj)
1709 {
1710         return obj->madv == I915_MADV_DONTNEED;
1711 }
1712
1713 static void
1714 i915_gem_process_flushing_list(struct intel_ring_buffer *ring,
1715                                uint32_t flush_domains)
1716 {
1717         struct drm_i915_gem_object *obj, *next;
1718
1719         list_for_each_entry_safe(obj, next,
1720                                  &ring->gpu_write_list,
1721                                  gpu_write_list) {
1722                 if (obj->base.write_domain & flush_domains) {
1723                         uint32_t old_write_domain = obj->base.write_domain;
1724
1725                         obj->base.write_domain = 0;
1726                         list_del_init(&obj->gpu_write_list);
1727                         i915_gem_object_move_to_active(obj, ring,
1728                                                        i915_gem_next_request_seqno(ring));
1729
1730                         trace_i915_gem_object_change_domain(obj,
1731                                                             obj->base.read_domains,
1732                                                             old_write_domain);
1733                 }
1734         }
1735 }
1736
1737 int
1738 i915_add_request(struct intel_ring_buffer *ring,
1739                  struct drm_file *file,
1740                  struct drm_i915_gem_request *request)
1741 {
1742         drm_i915_private_t *dev_priv = ring->dev->dev_private;
1743         uint32_t seqno;
1744         int was_empty;
1745         int ret;
1746
1747         BUG_ON(request == NULL);
1748
1749         ret = ring->add_request(ring, &seqno);
1750         if (ret)
1751             return ret;
1752
1753         trace_i915_gem_request_add(ring, seqno);
1754
1755         request->seqno = seqno;
1756         request->ring = ring;
1757         request->emitted_jiffies = jiffies;
1758         was_empty = list_empty(&ring->request_list);
1759         list_add_tail(&request->list, &ring->request_list);
1760
1761         if (file) {
1762                 struct drm_i915_file_private *file_priv = file->driver_priv;
1763
1764                 spin_lock(&file_priv->mm.lock);
1765                 request->file_priv = file_priv;
1766                 list_add_tail(&request->client_list,
1767                               &file_priv->mm.request_list);
1768                 spin_unlock(&file_priv->mm.lock);
1769         }
1770
1771         ring->outstanding_lazy_request = false;
1772
1773         if (!dev_priv->mm.suspended) {
1774                 mod_timer(&dev_priv->hangcheck_timer,
1775                           jiffies + msecs_to_jiffies(DRM_I915_HANGCHECK_PERIOD));
1776                 if (was_empty)
1777                         queue_delayed_work(dev_priv->wq,
1778                                            &dev_priv->mm.retire_work, HZ);
1779         }
1780         return 0;
1781 }
1782
1783 static inline void
1784 i915_gem_request_remove_from_client(struct drm_i915_gem_request *request)
1785 {
1786         struct drm_i915_file_private *file_priv = request->file_priv;
1787
1788         if (!file_priv)
1789                 return;
1790
1791         spin_lock(&file_priv->mm.lock);
1792         if (request->file_priv) {
1793                 list_del(&request->client_list);
1794                 request->file_priv = NULL;
1795         }
1796         spin_unlock(&file_priv->mm.lock);
1797 }
1798
1799 static void i915_gem_reset_ring_lists(struct drm_i915_private *dev_priv,
1800                                       struct intel_ring_buffer *ring)
1801 {
1802         while (!list_empty(&ring->request_list)) {
1803                 struct drm_i915_gem_request *request;
1804
1805                 request = list_first_entry(&ring->request_list,
1806                                            struct drm_i915_gem_request,
1807                                            list);
1808
1809                 list_del(&request->list);
1810                 i915_gem_request_remove_from_client(request);
1811                 kfree(request);
1812         }
1813
1814         while (!list_empty(&ring->active_list)) {
1815                 struct drm_i915_gem_object *obj;
1816
1817                 obj = list_first_entry(&ring->active_list,
1818                                        struct drm_i915_gem_object,
1819                                        ring_list);
1820
1821                 obj->base.write_domain = 0;
1822                 list_del_init(&obj->gpu_write_list);
1823                 i915_gem_object_move_to_inactive(obj);
1824         }
1825 }
1826
1827 static void i915_gem_reset_fences(struct drm_device *dev)
1828 {
1829         struct drm_i915_private *dev_priv = dev->dev_private;
1830         int i;
1831
1832         for (i = 0; i < 16; i++) {
1833                 struct drm_i915_fence_reg *reg = &dev_priv->fence_regs[i];
1834                 struct drm_i915_gem_object *obj = reg->obj;
1835
1836                 if (!obj)
1837                         continue;
1838
1839                 if (obj->tiling_mode)
1840                         i915_gem_release_mmap(obj);
1841
1842                 reg->obj->fence_reg = I915_FENCE_REG_NONE;
1843                 reg->obj->fenced_gpu_access = false;
1844                 reg->obj->last_fenced_seqno = 0;
1845                 reg->obj->last_fenced_ring = NULL;
1846                 i915_gem_clear_fence_reg(dev, reg);
1847         }
1848 }
1849
1850 void i915_gem_reset(struct drm_device *dev)
1851 {
1852         struct drm_i915_private *dev_priv = dev->dev_private;
1853         struct drm_i915_gem_object *obj;
1854         int i;
1855
1856         for (i = 0; i < I915_NUM_RINGS; i++)
1857                 i915_gem_reset_ring_lists(dev_priv, &dev_priv->ring[i]);
1858
1859         /* Remove anything from the flushing lists. The GPU cache is likely
1860          * to be lost on reset along with the data, so simply move the
1861          * lost bo to the inactive list.
1862          */
1863         while (!list_empty(&dev_priv->mm.flushing_list)) {
1864                 obj= list_first_entry(&dev_priv->mm.flushing_list,
1865                                       struct drm_i915_gem_object,
1866                                       mm_list);
1867
1868                 obj->base.write_domain = 0;
1869                 list_del_init(&obj->gpu_write_list);
1870                 i915_gem_object_move_to_inactive(obj);
1871         }
1872
1873         /* Move everything out of the GPU domains to ensure we do any
1874          * necessary invalidation upon reuse.
1875          */
1876         list_for_each_entry(obj,
1877                             &dev_priv->mm.inactive_list,
1878                             mm_list)
1879         {
1880                 obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
1881         }
1882
1883         /* The fence registers are invalidated so clear them out */
1884         i915_gem_reset_fences(dev);
1885 }
1886
1887 /**
1888  * This function clears the request list as sequence numbers are passed.
1889  */
1890 static void
1891 i915_gem_retire_requests_ring(struct intel_ring_buffer *ring)
1892 {
1893         uint32_t seqno;
1894         int i;
1895
1896         if (list_empty(&ring->request_list))
1897                 return;
1898
1899         WARN_ON(i915_verify_lists(ring->dev));
1900
1901         seqno = ring->get_seqno(ring);
1902
1903         for (i = 0; i < ARRAY_SIZE(ring->sync_seqno); i++)
1904                 if (seqno >= ring->sync_seqno[i])
1905                         ring->sync_seqno[i] = 0;
1906
1907         while (!list_empty(&ring->request_list)) {
1908                 struct drm_i915_gem_request *request;
1909
1910                 request = list_first_entry(&ring->request_list,
1911                                            struct drm_i915_gem_request,
1912                                            list);
1913
1914                 if (!i915_seqno_passed(seqno, request->seqno))
1915                         break;
1916
1917                 trace_i915_gem_request_retire(ring, request->seqno);
1918
1919                 list_del(&request->list);
1920                 i915_gem_request_remove_from_client(request);
1921                 kfree(request);
1922         }
1923
1924         /* Move any buffers on the active list that are no longer referenced
1925          * by the ringbuffer to the flushing/inactive lists as appropriate.
1926          */
1927         while (!list_empty(&ring->active_list)) {
1928                 struct drm_i915_gem_object *obj;
1929
1930                 obj= list_first_entry(&ring->active_list,
1931                                       struct drm_i915_gem_object,
1932                                       ring_list);
1933
1934                 if (!i915_seqno_passed(seqno, obj->last_rendering_seqno))
1935                         break;
1936
1937                 if (obj->base.write_domain != 0)
1938                         i915_gem_object_move_to_flushing(obj);
1939                 else
1940                         i915_gem_object_move_to_inactive(obj);
1941         }
1942
1943         if (unlikely(ring->trace_irq_seqno &&
1944                      i915_seqno_passed(seqno, ring->trace_irq_seqno))) {
1945                 ring->irq_put(ring);
1946                 ring->trace_irq_seqno = 0;
1947         }
1948
1949         WARN_ON(i915_verify_lists(ring->dev));
1950 }
1951
1952 void
1953 i915_gem_retire_requests(struct drm_device *dev)
1954 {
1955         drm_i915_private_t *dev_priv = dev->dev_private;
1956         int i;
1957
1958         if (!list_empty(&dev_priv->mm.deferred_free_list)) {
1959             struct drm_i915_gem_object *obj, *next;
1960
1961             /* We must be careful that during unbind() we do not
1962              * accidentally infinitely recurse into retire requests.
1963              * Currently:
1964              *   retire -> free -> unbind -> wait -> retire_ring
1965              */
1966             list_for_each_entry_safe(obj, next,
1967                                      &dev_priv->mm.deferred_free_list,
1968                                      mm_list)
1969                     i915_gem_free_object_tail(obj);
1970         }
1971
1972         for (i = 0; i < I915_NUM_RINGS; i++)
1973                 i915_gem_retire_requests_ring(&dev_priv->ring[i]);
1974 }
1975
1976 static void
1977 i915_gem_retire_work_handler(struct work_struct *work)
1978 {
1979         drm_i915_private_t *dev_priv;
1980         struct drm_device *dev;
1981         bool idle;
1982         int i;
1983
1984         dev_priv = container_of(work, drm_i915_private_t,
1985                                 mm.retire_work.work);
1986         dev = dev_priv->dev;
1987
1988         /* Come back later if the device is busy... */
1989         if (!mutex_trylock(&dev->struct_mutex)) {
1990                 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
1991                 return;
1992         }
1993
1994         i915_gem_retire_requests(dev);
1995
1996         /* Send a periodic flush down the ring so we don't hold onto GEM
1997          * objects indefinitely.
1998          */
1999         idle = true;
2000         for (i = 0; i < I915_NUM_RINGS; i++) {
2001                 struct intel_ring_buffer *ring = &dev_priv->ring[i];
2002
2003                 if (!list_empty(&ring->gpu_write_list)) {
2004                         struct drm_i915_gem_request *request;
2005                         int ret;
2006
2007                         ret = i915_gem_flush_ring(ring,
2008                                                   0, I915_GEM_GPU_DOMAINS);
2009                         request = kzalloc(sizeof(*request), GFP_KERNEL);
2010                         if (ret || request == NULL ||
2011                             i915_add_request(ring, NULL, request))
2012                             kfree(request);
2013                 }
2014
2015                 idle &= list_empty(&ring->request_list);
2016         }
2017
2018         if (!dev_priv->mm.suspended && !idle)
2019                 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
2020
2021         mutex_unlock(&dev->struct_mutex);
2022 }
2023
2024 /**
2025  * Waits for a sequence number to be signaled, and cleans up the
2026  * request and object lists appropriately for that event.
2027  */
2028 int
2029 i915_wait_request(struct intel_ring_buffer *ring,
2030                   uint32_t seqno)
2031 {
2032         drm_i915_private_t *dev_priv = ring->dev->dev_private;
2033         u32 ier;
2034         int ret = 0;
2035
2036         BUG_ON(seqno == 0);
2037
2038         if (atomic_read(&dev_priv->mm.wedged)) {
2039                 struct completion *x = &dev_priv->error_completion;
2040                 bool recovery_complete;
2041                 unsigned long flags;
2042
2043                 /* Give the error handler a chance to run. */
2044                 spin_lock_irqsave(&x->wait.lock, flags);
2045                 recovery_complete = x->done > 0;
2046                 spin_unlock_irqrestore(&x->wait.lock, flags);
2047
2048                 return recovery_complete ? -EIO : -EAGAIN;
2049         }
2050
2051         if (seqno == ring->outstanding_lazy_request) {
2052                 struct drm_i915_gem_request *request;
2053
2054                 request = kzalloc(sizeof(*request), GFP_KERNEL);
2055                 if (request == NULL)
2056                         return -ENOMEM;
2057
2058                 ret = i915_add_request(ring, NULL, request);
2059                 if (ret) {
2060                         kfree(request);
2061                         return ret;
2062                 }
2063
2064                 seqno = request->seqno;
2065         }
2066
2067         if (!i915_seqno_passed(ring->get_seqno(ring), seqno)) {
2068                 if (HAS_PCH_SPLIT(ring->dev))
2069                         ier = I915_READ(DEIER) | I915_READ(GTIER);
2070                 else
2071                         ier = I915_READ(IER);
2072                 if (!ier) {
2073                         DRM_ERROR("something (likely vbetool) disabled "
2074                                   "interrupts, re-enabling\n");
2075                         i915_driver_irq_preinstall(ring->dev);
2076                         i915_driver_irq_postinstall(ring->dev);
2077                 }
2078
2079                 trace_i915_gem_request_wait_begin(ring, seqno);
2080
2081                 ring->waiting_seqno = seqno;
2082                 if (ring->irq_get(ring)) {
2083                         if (dev_priv->mm.interruptible)
2084                                 ret = wait_event_interruptible(ring->irq_queue,
2085                                                                i915_seqno_passed(ring->get_seqno(ring), seqno)
2086                                                                || atomic_read(&dev_priv->mm.wedged));
2087                         else
2088                                 wait_event(ring->irq_queue,
2089                                            i915_seqno_passed(ring->get_seqno(ring), seqno)
2090                                            || atomic_read(&dev_priv->mm.wedged));
2091
2092                         ring->irq_put(ring);
2093                 } else if (wait_for(i915_seqno_passed(ring->get_seqno(ring),
2094                                                       seqno) ||
2095                                     atomic_read(&dev_priv->mm.wedged), 3000))
2096                         ret = -EBUSY;
2097                 ring->waiting_seqno = 0;
2098
2099                 trace_i915_gem_request_wait_end(ring, seqno);
2100         }
2101         if (atomic_read(&dev_priv->mm.wedged))
2102                 ret = -EAGAIN;
2103
2104         if (ret && ret != -ERESTARTSYS)
2105                 DRM_ERROR("%s returns %d (awaiting %d at %d, next %d)\n",
2106                           __func__, ret, seqno, ring->get_seqno(ring),
2107                           dev_priv->next_seqno);
2108
2109         /* Directly dispatch request retiring.  While we have the work queue
2110          * to handle this, the waiter on a request often wants an associated
2111          * buffer to have made it to the inactive list, and we would need
2112          * a separate wait queue to handle that.
2113          */
2114         if (ret == 0)
2115                 i915_gem_retire_requests_ring(ring);
2116
2117         return ret;
2118 }
2119
2120 /**
2121  * Ensures that all rendering to the object has completed and the object is
2122  * safe to unbind from the GTT or access from the CPU.
2123  */
2124 int
2125 i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj)
2126 {
2127         int ret;
2128
2129         /* This function only exists to support waiting for existing rendering,
2130          * not for emitting required flushes.
2131          */
2132         BUG_ON((obj->base.write_domain & I915_GEM_GPU_DOMAINS) != 0);
2133
2134         /* If there is rendering queued on the buffer being evicted, wait for
2135          * it.
2136          */
2137         if (obj->active) {
2138                 ret = i915_wait_request(obj->ring, obj->last_rendering_seqno);
2139                 if (ret)
2140                         return ret;
2141         }
2142
2143         return 0;
2144 }
2145
2146 /**
2147  * Unbinds an object from the GTT aperture.
2148  */
2149 int
2150 i915_gem_object_unbind(struct drm_i915_gem_object *obj)
2151 {
2152         int ret = 0;
2153
2154         if (obj->gtt_space == NULL)
2155                 return 0;
2156
2157         if (obj->pin_count != 0) {
2158                 DRM_ERROR("Attempting to unbind pinned buffer\n");
2159                 return -EINVAL;
2160         }
2161
2162         /* blow away mappings if mapped through GTT */
2163         i915_gem_release_mmap(obj);
2164
2165         /* Move the object to the CPU domain to ensure that
2166          * any possible CPU writes while it's not in the GTT
2167          * are flushed when we go to remap it. This will
2168          * also ensure that all pending GPU writes are finished
2169          * before we unbind.
2170          */
2171         ret = i915_gem_object_set_to_cpu_domain(obj, 1);
2172         if (ret == -ERESTARTSYS)
2173                 return ret;
2174         /* Continue on if we fail due to EIO, the GPU is hung so we
2175          * should be safe and we need to cleanup or else we might
2176          * cause memory corruption through use-after-free.
2177          */
2178         if (ret) {
2179                 i915_gem_clflush_object(obj);
2180                 obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU;
2181         }
2182
2183         /* release the fence reg _after_ flushing */
2184         ret = i915_gem_object_put_fence(obj);
2185         if (ret == -ERESTARTSYS)
2186                 return ret;
2187
2188         trace_i915_gem_object_unbind(obj);
2189
2190         i915_gem_gtt_unbind_object(obj);
2191         i915_gem_object_put_pages_gtt(obj);
2192
2193         list_del_init(&obj->gtt_list);
2194         list_del_init(&obj->mm_list);
2195         /* Avoid an unnecessary call to unbind on rebind. */
2196         obj->map_and_fenceable = true;
2197
2198         drm_mm_put_block(obj->gtt_space);
2199         obj->gtt_space = NULL;
2200         obj->gtt_offset = 0;
2201
2202         if (i915_gem_object_is_purgeable(obj))
2203                 i915_gem_object_truncate(obj);
2204
2205         return ret;
2206 }
2207
2208 int
2209 i915_gem_flush_ring(struct intel_ring_buffer *ring,
2210                     uint32_t invalidate_domains,
2211                     uint32_t flush_domains)
2212 {
2213         int ret;
2214
2215         if (((invalidate_domains | flush_domains) & I915_GEM_GPU_DOMAINS) == 0)
2216                 return 0;
2217
2218         trace_i915_gem_ring_flush(ring, invalidate_domains, flush_domains);
2219
2220         ret = ring->flush(ring, invalidate_domains, flush_domains);
2221         if (ret)
2222                 return ret;
2223
2224         if (flush_domains & I915_GEM_GPU_DOMAINS)
2225                 i915_gem_process_flushing_list(ring, flush_domains);
2226
2227         return 0;
2228 }
2229
2230 static int i915_ring_idle(struct intel_ring_buffer *ring)
2231 {
2232         int ret;
2233
2234         if (list_empty(&ring->gpu_write_list) && list_empty(&ring->active_list))
2235                 return 0;
2236
2237         if (!list_empty(&ring->gpu_write_list)) {
2238                 ret = i915_gem_flush_ring(ring,
2239                                     I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS);
2240                 if (ret)
2241                         return ret;
2242         }
2243
2244         return i915_wait_request(ring, i915_gem_next_request_seqno(ring));
2245 }
2246
2247 int
2248 i915_gpu_idle(struct drm_device *dev)
2249 {
2250         drm_i915_private_t *dev_priv = dev->dev_private;
2251         bool lists_empty;
2252         int ret, i;
2253
2254         lists_empty = (list_empty(&dev_priv->mm.flushing_list) &&
2255                        list_empty(&dev_priv->mm.active_list));
2256         if (lists_empty)
2257                 return 0;
2258
2259         /* Flush everything onto the inactive list. */
2260         for (i = 0; i < I915_NUM_RINGS; i++) {
2261                 ret = i915_ring_idle(&dev_priv->ring[i]);
2262                 if (ret)
2263                         return ret;
2264         }
2265
2266         return 0;
2267 }
2268
2269 static int sandybridge_write_fence_reg(struct drm_i915_gem_object *obj,
2270                                        struct intel_ring_buffer *pipelined)
2271 {
2272         struct drm_device *dev = obj->base.dev;
2273         drm_i915_private_t *dev_priv = dev->dev_private;
2274         u32 size = obj->gtt_space->size;
2275         int regnum = obj->fence_reg;
2276         uint64_t val;
2277
2278         val = (uint64_t)((obj->gtt_offset + size - 4096) &
2279                          0xfffff000) << 32;
2280         val |= obj->gtt_offset & 0xfffff000;
2281         val |= (uint64_t)((obj->stride / 128) - 1) <<
2282                 SANDYBRIDGE_FENCE_PITCH_SHIFT;
2283
2284         if (obj->tiling_mode == I915_TILING_Y)
2285                 val |= 1 << I965_FENCE_TILING_Y_SHIFT;
2286         val |= I965_FENCE_REG_VALID;
2287
2288         if (pipelined) {
2289                 int ret = intel_ring_begin(pipelined, 6);
2290                 if (ret)
2291                         return ret;
2292
2293                 intel_ring_emit(pipelined, MI_NOOP);
2294                 intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(2));
2295                 intel_ring_emit(pipelined, FENCE_REG_SANDYBRIDGE_0 + regnum*8);
2296                 intel_ring_emit(pipelined, (u32)val);
2297                 intel_ring_emit(pipelined, FENCE_REG_SANDYBRIDGE_0 + regnum*8 + 4);
2298                 intel_ring_emit(pipelined, (u32)(val >> 32));
2299                 intel_ring_advance(pipelined);
2300         } else
2301                 I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + regnum * 8, val);
2302
2303         return 0;
2304 }
2305
2306 static int i965_write_fence_reg(struct drm_i915_gem_object *obj,
2307                                 struct intel_ring_buffer *pipelined)
2308 {
2309         struct drm_device *dev = obj->base.dev;
2310         drm_i915_private_t *dev_priv = dev->dev_private;
2311         u32 size = obj->gtt_space->size;
2312         int regnum = obj->fence_reg;
2313         uint64_t val;
2314
2315         val = (uint64_t)((obj->gtt_offset + size - 4096) &
2316                     0xfffff000) << 32;
2317         val |= obj->gtt_offset & 0xfffff000;
2318         val |= ((obj->stride / 128) - 1) << I965_FENCE_PITCH_SHIFT;
2319         if (obj->tiling_mode == I915_TILING_Y)
2320                 val |= 1 << I965_FENCE_TILING_Y_SHIFT;
2321         val |= I965_FENCE_REG_VALID;
2322
2323         if (pipelined) {
2324                 int ret = intel_ring_begin(pipelined, 6);
2325                 if (ret)
2326                         return ret;
2327
2328                 intel_ring_emit(pipelined, MI_NOOP);
2329                 intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(2));
2330                 intel_ring_emit(pipelined, FENCE_REG_965_0 + regnum*8);
2331                 intel_ring_emit(pipelined, (u32)val);
2332                 intel_ring_emit(pipelined, FENCE_REG_965_0 + regnum*8 + 4);
2333                 intel_ring_emit(pipelined, (u32)(val >> 32));
2334                 intel_ring_advance(pipelined);
2335         } else
2336                 I915_WRITE64(FENCE_REG_965_0 + regnum * 8, val);
2337
2338         return 0;
2339 }
2340
2341 static int i915_write_fence_reg(struct drm_i915_gem_object *obj,
2342                                 struct intel_ring_buffer *pipelined)
2343 {
2344         struct drm_device *dev = obj->base.dev;
2345         drm_i915_private_t *dev_priv = dev->dev_private;
2346         u32 size = obj->gtt_space->size;
2347         u32 fence_reg, val, pitch_val;
2348         int tile_width;
2349
2350         if (WARN((obj->gtt_offset & ~I915_FENCE_START_MASK) ||
2351                  (size & -size) != size ||
2352                  (obj->gtt_offset & (size - 1)),
2353                  "object 0x%08x [fenceable? %d] not 1M or pot-size (0x%08x) aligned\n",
2354                  obj->gtt_offset, obj->map_and_fenceable, size))
2355                 return -EINVAL;
2356
2357         if (obj->tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev))
2358                 tile_width = 128;
2359         else
2360                 tile_width = 512;
2361
2362         /* Note: pitch better be a power of two tile widths */
2363         pitch_val = obj->stride / tile_width;
2364         pitch_val = ffs(pitch_val) - 1;
2365
2366         val = obj->gtt_offset;
2367         if (obj->tiling_mode == I915_TILING_Y)
2368                 val |= 1 << I830_FENCE_TILING_Y_SHIFT;
2369         val |= I915_FENCE_SIZE_BITS(size);
2370         val |= pitch_val << I830_FENCE_PITCH_SHIFT;
2371         val |= I830_FENCE_REG_VALID;
2372
2373         fence_reg = obj->fence_reg;
2374         if (fence_reg < 8)
2375                 fence_reg = FENCE_REG_830_0 + fence_reg * 4;
2376         else
2377                 fence_reg = FENCE_REG_945_8 + (fence_reg - 8) * 4;
2378
2379         if (pipelined) {
2380                 int ret = intel_ring_begin(pipelined, 4);
2381                 if (ret)
2382                         return ret;
2383
2384                 intel_ring_emit(pipelined, MI_NOOP);
2385                 intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(1));
2386                 intel_ring_emit(pipelined, fence_reg);
2387                 intel_ring_emit(pipelined, val);
2388                 intel_ring_advance(pipelined);
2389         } else
2390                 I915_WRITE(fence_reg, val);
2391
2392         return 0;
2393 }
2394
2395 static int i830_write_fence_reg(struct drm_i915_gem_object *obj,
2396                                 struct intel_ring_buffer *pipelined)
2397 {
2398         struct drm_device *dev = obj->base.dev;
2399         drm_i915_private_t *dev_priv = dev->dev_private;
2400         u32 size = obj->gtt_space->size;
2401         int regnum = obj->fence_reg;
2402         uint32_t val;
2403         uint32_t pitch_val;
2404
2405         if (WARN((obj->gtt_offset & ~I830_FENCE_START_MASK) ||
2406                  (size & -size) != size ||
2407                  (obj->gtt_offset & (size - 1)),
2408                  "object 0x%08x not 512K or pot-size 0x%08x aligned\n",
2409                  obj->gtt_offset, size))
2410                 return -EINVAL;
2411
2412         pitch_val = obj->stride / 128;
2413         pitch_val = ffs(pitch_val) - 1;
2414
2415         val = obj->gtt_offset;
2416         if (obj->tiling_mode == I915_TILING_Y)
2417                 val |= 1 << I830_FENCE_TILING_Y_SHIFT;
2418         val |= I830_FENCE_SIZE_BITS(size);
2419         val |= pitch_val << I830_FENCE_PITCH_SHIFT;
2420         val |= I830_FENCE_REG_VALID;
2421
2422         if (pipelined) {
2423                 int ret = intel_ring_begin(pipelined, 4);
2424                 if (ret)
2425                         return ret;
2426
2427                 intel_ring_emit(pipelined, MI_NOOP);
2428                 intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(1));
2429                 intel_ring_emit(pipelined, FENCE_REG_830_0 + regnum*4);
2430                 intel_ring_emit(pipelined, val);
2431                 intel_ring_advance(pipelined);
2432         } else
2433                 I915_WRITE(FENCE_REG_830_0 + regnum * 4, val);
2434
2435         return 0;
2436 }
2437
2438 static bool ring_passed_seqno(struct intel_ring_buffer *ring, u32 seqno)
2439 {
2440         return i915_seqno_passed(ring->get_seqno(ring), seqno);
2441 }
2442
2443 static int
2444 i915_gem_object_flush_fence(struct drm_i915_gem_object *obj,
2445                             struct intel_ring_buffer *pipelined)
2446 {
2447         int ret;
2448
2449         if (obj->fenced_gpu_access) {
2450                 if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) {
2451                         ret = i915_gem_flush_ring(obj->last_fenced_ring,
2452                                                   0, obj->base.write_domain);
2453                         if (ret)
2454                                 return ret;
2455                 }
2456
2457                 obj->fenced_gpu_access = false;
2458         }
2459
2460         if (obj->last_fenced_seqno && pipelined != obj->last_fenced_ring) {
2461                 if (!ring_passed_seqno(obj->last_fenced_ring,
2462                                        obj->last_fenced_seqno)) {
2463                         ret = i915_wait_request(obj->last_fenced_ring,
2464                                                 obj->last_fenced_seqno);
2465                         if (ret)
2466                                 return ret;
2467                 }
2468
2469                 obj->last_fenced_seqno = 0;
2470                 obj->last_fenced_ring = NULL;
2471         }
2472
2473         /* Ensure that all CPU reads are completed before installing a fence
2474          * and all writes before removing the fence.
2475          */
2476         if (obj->base.read_domains & I915_GEM_DOMAIN_GTT)
2477                 mb();
2478
2479         return 0;
2480 }
2481
2482 int
2483 i915_gem_object_put_fence(struct drm_i915_gem_object *obj)
2484 {
2485         int ret;
2486
2487         if (obj->tiling_mode)
2488                 i915_gem_release_mmap(obj);
2489
2490         ret = i915_gem_object_flush_fence(obj, NULL);
2491         if (ret)
2492                 return ret;
2493
2494         if (obj->fence_reg != I915_FENCE_REG_NONE) {
2495                 struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
2496                 i915_gem_clear_fence_reg(obj->base.dev,
2497                                          &dev_priv->fence_regs[obj->fence_reg]);
2498
2499                 obj->fence_reg = I915_FENCE_REG_NONE;
2500         }
2501
2502         return 0;
2503 }
2504
2505 static struct drm_i915_fence_reg *
2506 i915_find_fence_reg(struct drm_device *dev,
2507                     struct intel_ring_buffer *pipelined)
2508 {
2509         struct drm_i915_private *dev_priv = dev->dev_private;
2510         struct drm_i915_fence_reg *reg, *first, *avail;
2511         int i;
2512
2513         /* First try to find a free reg */
2514         avail = NULL;
2515         for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) {
2516                 reg = &dev_priv->fence_regs[i];
2517                 if (!reg->obj)
2518                         return reg;
2519
2520                 if (!reg->obj->pin_count)
2521                         avail = reg;
2522         }
2523
2524         if (avail == NULL)
2525                 return NULL;
2526
2527         /* None available, try to steal one or wait for a user to finish */
2528         avail = first = NULL;
2529         list_for_each_entry(reg, &dev_priv->mm.fence_list, lru_list) {
2530                 if (reg->obj->pin_count)
2531                         continue;
2532
2533                 if (first == NULL)
2534                         first = reg;
2535
2536                 if (!pipelined ||
2537                     !reg->obj->last_fenced_ring ||
2538                     reg->obj->last_fenced_ring == pipelined) {
2539                         avail = reg;
2540                         break;
2541                 }
2542         }
2543
2544         if (avail == NULL)
2545                 avail = first;
2546
2547         return avail;
2548 }
2549
2550 /**
2551  * i915_gem_object_get_fence - set up a fence reg for an object
2552  * @obj: object to map through a fence reg
2553  * @pipelined: ring on which to queue the change, or NULL for CPU access
2554  * @interruptible: must we wait uninterruptibly for the register to retire?
2555  *
2556  * When mapping objects through the GTT, userspace wants to be able to write
2557  * to them without having to worry about swizzling if the object is tiled.
2558  *
2559  * This function walks the fence regs looking for a free one for @obj,
2560  * stealing one if it can't find any.
2561  *
2562  * It then sets up the reg based on the object's properties: address, pitch
2563  * and tiling format.
2564  */
2565 int
2566 i915_gem_object_get_fence(struct drm_i915_gem_object *obj,
2567                           struct intel_ring_buffer *pipelined)
2568 {
2569         struct drm_device *dev = obj->base.dev;
2570         struct drm_i915_private *dev_priv = dev->dev_private;
2571         struct drm_i915_fence_reg *reg;
2572         int ret;
2573
2574         /* XXX disable pipelining. There are bugs. Shocking. */
2575         pipelined = NULL;
2576
2577         /* Just update our place in the LRU if our fence is getting reused. */
2578         if (obj->fence_reg != I915_FENCE_REG_NONE) {
2579                 reg = &dev_priv->fence_regs[obj->fence_reg];
2580                 list_move_tail(&reg->lru_list, &dev_priv->mm.fence_list);
2581
2582                 if (obj->tiling_changed) {
2583                         ret = i915_gem_object_flush_fence(obj, pipelined);
2584                         if (ret)
2585                                 return ret;
2586
2587                         if (!obj->fenced_gpu_access && !obj->last_fenced_seqno)
2588                                 pipelined = NULL;
2589
2590                         if (pipelined) {
2591                                 reg->setup_seqno =
2592                                         i915_gem_next_request_seqno(pipelined);
2593                                 obj->last_fenced_seqno = reg->setup_seqno;
2594                                 obj->last_fenced_ring = pipelined;
2595                         }
2596
2597                         goto update;
2598                 }
2599
2600                 if (!pipelined) {
2601                         if (reg->setup_seqno) {
2602                                 if (!ring_passed_seqno(obj->last_fenced_ring,
2603                                                        reg->setup_seqno)) {
2604                                         ret = i915_wait_request(obj->last_fenced_ring,
2605                                                                 reg->setup_seqno);
2606                                         if (ret)
2607                                                 return ret;
2608                                 }
2609
2610                                 reg->setup_seqno = 0;
2611                         }
2612                 } else if (obj->last_fenced_ring &&
2613                            obj->last_fenced_ring != pipelined) {
2614                         ret = i915_gem_object_flush_fence(obj, pipelined);
2615                         if (ret)
2616                                 return ret;
2617                 }
2618
2619                 return 0;
2620         }
2621
2622         reg = i915_find_fence_reg(dev, pipelined);
2623         if (reg == NULL)
2624                 return -ENOSPC;
2625
2626         ret = i915_gem_object_flush_fence(obj, pipelined);
2627         if (ret)
2628                 return ret;
2629
2630         if (reg->obj) {
2631                 struct drm_i915_gem_object *old = reg->obj;
2632
2633                 drm_gem_object_reference(&old->base);
2634
2635                 if (old->tiling_mode)
2636                         i915_gem_release_mmap(old);
2637
2638                 ret = i915_gem_object_flush_fence(old, pipelined);
2639                 if (ret) {
2640                         drm_gem_object_unreference(&old->base);
2641                         return ret;
2642                 }
2643
2644                 if (old->last_fenced_seqno == 0 && obj->last_fenced_seqno == 0)
2645                         pipelined = NULL;
2646
2647                 old->fence_reg = I915_FENCE_REG_NONE;
2648                 old->last_fenced_ring = pipelined;
2649                 old->last_fenced_seqno =
2650                         pipelined ? i915_gem_next_request_seqno(pipelined) : 0;
2651
2652                 drm_gem_object_unreference(&old->base);
2653         } else if (obj->last_fenced_seqno == 0)
2654                 pipelined = NULL;
2655
2656         reg->obj = obj;
2657         list_move_tail(&reg->lru_list, &dev_priv->mm.fence_list);
2658         obj->fence_reg = reg - dev_priv->fence_regs;
2659         obj->last_fenced_ring = pipelined;
2660
2661         reg->setup_seqno =
2662                 pipelined ? i915_gem_next_request_seqno(pipelined) : 0;
2663         obj->last_fenced_seqno = reg->setup_seqno;
2664
2665 update:
2666         obj->tiling_changed = false;
2667         switch (INTEL_INFO(dev)->gen) {
2668         case 7:
2669         case 6:
2670                 ret = sandybridge_write_fence_reg(obj, pipelined);
2671                 break;
2672         case 5:
2673         case 4:
2674                 ret = i965_write_fence_reg(obj, pipelined);
2675                 break;
2676         case 3:
2677                 ret = i915_write_fence_reg(obj, pipelined);
2678                 break;
2679         case 2:
2680                 ret = i830_write_fence_reg(obj, pipelined);
2681                 break;
2682         }
2683
2684         return ret;
2685 }
2686
2687 /**
2688  * i915_gem_clear_fence_reg - clear out fence register info
2689  * @obj: object to clear
2690  *
2691  * Zeroes out the fence register itself and clears out the associated
2692  * data structures in dev_priv and obj.
2693  */
2694 static void
2695 i915_gem_clear_fence_reg(struct drm_device *dev,
2696                          struct drm_i915_fence_reg *reg)
2697 {
2698         drm_i915_private_t *dev_priv = dev->dev_private;
2699         uint32_t fence_reg = reg - dev_priv->fence_regs;
2700
2701         switch (INTEL_INFO(dev)->gen) {
2702         case 7:
2703         case 6:
2704                 I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + fence_reg*8, 0);
2705                 break;
2706         case 5:
2707         case 4:
2708                 I915_WRITE64(FENCE_REG_965_0 + fence_reg*8, 0);
2709                 break;
2710         case 3:
2711                 if (fence_reg >= 8)
2712                         fence_reg = FENCE_REG_945_8 + (fence_reg - 8) * 4;
2713                 else
2714         case 2:
2715                         fence_reg = FENCE_REG_830_0 + fence_reg * 4;
2716
2717                 I915_WRITE(fence_reg, 0);
2718                 break;
2719         }
2720
2721         list_del_init(&reg->lru_list);
2722         reg->obj = NULL;
2723         reg->setup_seqno = 0;
2724 }
2725
2726 /**
2727  * Finds free space in the GTT aperture and binds the object there.
2728  */
2729 static int
2730 i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
2731                             unsigned alignment,
2732                             bool map_and_fenceable)
2733 {
2734         struct drm_device *dev = obj->base.dev;
2735         drm_i915_private_t *dev_priv = dev->dev_private;
2736         struct drm_mm_node *free_space;
2737         gfp_t gfpmask = __GFP_NORETRY | __GFP_NOWARN;
2738         u32 size, fence_size, fence_alignment, unfenced_alignment;
2739         bool mappable, fenceable;
2740         int ret;
2741
2742         if (obj->madv != I915_MADV_WILLNEED) {
2743                 DRM_ERROR("Attempting to bind a purgeable object\n");
2744                 return -EINVAL;
2745         }
2746
2747         fence_size = i915_gem_get_gtt_size(obj);
2748         fence_alignment = i915_gem_get_gtt_alignment(obj);
2749         unfenced_alignment = i915_gem_get_unfenced_gtt_alignment(obj);
2750
2751         if (alignment == 0)
2752                 alignment = map_and_fenceable ? fence_alignment :
2753                                                 unfenced_alignment;
2754         if (map_and_fenceable && alignment & (fence_alignment - 1)) {
2755                 DRM_ERROR("Invalid object alignment requested %u\n", alignment);
2756                 return -EINVAL;
2757         }
2758
2759         size = map_and_fenceable ? fence_size : obj->base.size;
2760
2761         /* If the object is bigger than the entire aperture, reject it early
2762          * before evicting everything in a vain attempt to find space.
2763          */
2764         if (obj->base.size >
2765             (map_and_fenceable ? dev_priv->mm.gtt_mappable_end : dev_priv->mm.gtt_total)) {
2766                 DRM_ERROR("Attempting to bind an object larger than the aperture\n");
2767                 return -E2BIG;
2768         }
2769
2770  search_free:
2771         if (map_and_fenceable)
2772                 free_space =
2773                         drm_mm_search_free_in_range(&dev_priv->mm.gtt_space,
2774                                                     size, alignment, 0,
2775                                                     dev_priv->mm.gtt_mappable_end,
2776                                                     0);
2777         else
2778                 free_space = drm_mm_search_free(&dev_priv->mm.gtt_space,
2779                                                 size, alignment, 0);
2780
2781         if (free_space != NULL) {
2782                 if (map_and_fenceable)
2783                         obj->gtt_space =
2784                                 drm_mm_get_block_range_generic(free_space,
2785                                                                size, alignment, 0,
2786                                                                dev_priv->mm.gtt_mappable_end,
2787                                                                0);
2788                 else
2789                         obj->gtt_space =
2790                                 drm_mm_get_block(free_space, size, alignment);
2791         }
2792         if (obj->gtt_space == NULL) {
2793                 /* If the gtt is empty and we're still having trouble
2794                  * fitting our object in, we're out of memory.
2795                  */
2796                 ret = i915_gem_evict_something(dev, size, alignment,
2797                                                map_and_fenceable);
2798                 if (ret)
2799                         return ret;
2800
2801                 goto search_free;
2802         }
2803
2804         ret = i915_gem_object_get_pages_gtt(obj, gfpmask);
2805         if (ret) {
2806                 drm_mm_put_block(obj->gtt_space);
2807                 obj->gtt_space = NULL;
2808
2809                 if (ret == -ENOMEM) {
2810                         /* first try to reclaim some memory by clearing the GTT */
2811                         ret = i915_gem_evict_everything(dev, false);
2812                         if (ret) {
2813                                 /* now try to shrink everyone else */
2814                                 if (gfpmask) {
2815                                         gfpmask = 0;
2816                                         goto search_free;
2817                                 }
2818
2819                                 return -ENOMEM;
2820                         }
2821
2822                         goto search_free;
2823                 }
2824
2825                 return ret;
2826         }
2827
2828         ret = i915_gem_gtt_bind_object(obj);
2829         if (ret) {
2830                 i915_gem_object_put_pages_gtt(obj);
2831                 drm_mm_put_block(obj->gtt_space);
2832                 obj->gtt_space = NULL;
2833
2834                 if (i915_gem_evict_everything(dev, false))
2835                         return ret;
2836
2837                 goto search_free;
2838         }
2839
2840         list_add_tail(&obj->gtt_list, &dev_priv->mm.gtt_list);
2841         list_add_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
2842
2843         /* Assert that the object is not currently in any GPU domain. As it
2844          * wasn't in the GTT, there shouldn't be any way it could have been in
2845          * a GPU cache
2846          */
2847         BUG_ON(obj->base.read_domains & I915_GEM_GPU_DOMAINS);
2848         BUG_ON(obj->base.write_domain & I915_GEM_GPU_DOMAINS);
2849
2850         obj->gtt_offset = obj->gtt_space->start;
2851
2852         fenceable =
2853                 obj->gtt_space->size == fence_size &&
2854                 (obj->gtt_space->start & (fence_alignment -1)) == 0;
2855
2856         mappable =
2857                 obj->gtt_offset + obj->base.size <= dev_priv->mm.gtt_mappable_end;
2858
2859         obj->map_and_fenceable = mappable && fenceable;
2860
2861         trace_i915_gem_object_bind(obj, map_and_fenceable);
2862         return 0;
2863 }
2864
2865 void
2866 i915_gem_clflush_object(struct drm_i915_gem_object *obj)
2867 {
2868         /* If we don't have a page list set up, then we're not pinned
2869          * to GPU, and we can ignore the cache flush because it'll happen
2870          * again at bind time.
2871          */
2872         if (obj->pages == NULL)
2873                 return;
2874
2875         /* If the GPU is snooping the contents of the CPU cache,
2876          * we do not need to manually clear the CPU cache lines.  However,
2877          * the caches are only snooped when the render cache is
2878          * flushed/invalidated.  As we always have to emit invalidations
2879          * and flushes when moving into and out of the RENDER domain, correct
2880          * snooping behaviour occurs naturally as the result of our domain
2881          * tracking.
2882          */
2883         if (obj->cache_level != I915_CACHE_NONE)
2884                 return;
2885
2886         trace_i915_gem_object_clflush(obj);
2887
2888         drm_clflush_pages(obj->pages, obj->base.size / PAGE_SIZE);
2889 }
2890
2891 /** Flushes any GPU write domain for the object if it's dirty. */
2892 static int
2893 i915_gem_object_flush_gpu_write_domain(struct drm_i915_gem_object *obj)
2894 {
2895         if ((obj->base.write_domain & I915_GEM_GPU_DOMAINS) == 0)
2896                 return 0;
2897
2898         /* Queue the GPU write cache flushing we need. */
2899         return i915_gem_flush_ring(obj->ring, 0, obj->base.write_domain);
2900 }
2901
2902 /** Flushes the GTT write domain for the object if it's dirty. */
2903 static void
2904 i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj)
2905 {
2906         uint32_t old_write_domain;
2907
2908         if (obj->base.write_domain != I915_GEM_DOMAIN_GTT)
2909                 return;
2910
2911         /* No actual flushing is required for the GTT write domain.  Writes
2912          * to it immediately go to main memory as far as we know, so there's
2913          * no chipset flush.  It also doesn't land in render cache.
2914          *
2915          * However, we do have to enforce the order so that all writes through
2916          * the GTT land before any writes to the device, such as updates to
2917          * the GATT itself.
2918          */
2919         wmb();
2920
2921         old_write_domain = obj->base.write_domain;
2922         obj->base.write_domain = 0;
2923
2924         trace_i915_gem_object_change_domain(obj,
2925                                             obj->base.read_domains,
2926                                             old_write_domain);
2927 }
2928
2929 /** Flushes the CPU write domain for the object if it's dirty. */
2930 static void
2931 i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj)
2932 {
2933         uint32_t old_write_domain;
2934
2935         if (obj->base.write_domain != I915_GEM_DOMAIN_CPU)
2936                 return;
2937
2938         i915_gem_clflush_object(obj);
2939         intel_gtt_chipset_flush();
2940         old_write_domain = obj->base.write_domain;
2941         obj->base.write_domain = 0;
2942
2943         trace_i915_gem_object_change_domain(obj,
2944                                             obj->base.read_domains,
2945                                             old_write_domain);
2946 }
2947
2948 /**
2949  * Moves a single object to the GTT read, and possibly write domain.
2950  *
2951  * This function returns when the move is complete, including waiting on
2952  * flushes to occur.
2953  */
2954 int
2955 i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
2956 {
2957         uint32_t old_write_domain, old_read_domains;
2958         int ret;
2959
2960         /* Not valid to be called on unbound objects. */
2961         if (obj->gtt_space == NULL)
2962                 return -EINVAL;
2963
2964         if (obj->base.write_domain == I915_GEM_DOMAIN_GTT)
2965                 return 0;
2966
2967         ret = i915_gem_object_flush_gpu_write_domain(obj);
2968         if (ret)
2969                 return ret;
2970
2971         if (obj->pending_gpu_write || write) {
2972                 ret = i915_gem_object_wait_rendering(obj);
2973                 if (ret)
2974                         return ret;
2975         }
2976
2977         i915_gem_object_flush_cpu_write_domain(obj);
2978
2979         old_write_domain = obj->base.write_domain;
2980         old_read_domains = obj->base.read_domains;
2981
2982         /* It should now be out of any other write domains, and we can update
2983          * the domain values for our changes.
2984          */
2985         BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
2986         obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
2987         if (write) {
2988                 obj->base.read_domains = I915_GEM_DOMAIN_GTT;
2989                 obj->base.write_domain = I915_GEM_DOMAIN_GTT;
2990                 obj->dirty = 1;
2991         }
2992
2993         trace_i915_gem_object_change_domain(obj,
2994                                             old_read_domains,
2995                                             old_write_domain);
2996
2997         return 0;
2998 }
2999
3000 /*
3001  * Prepare buffer for display plane. Use uninterruptible for possible flush
3002  * wait, as in modesetting process we're not supposed to be interrupted.
3003  */
3004 int
3005 i915_gem_object_set_to_display_plane(struct drm_i915_gem_object *obj,
3006                                      struct intel_ring_buffer *pipelined)
3007 {
3008         uint32_t old_read_domains;
3009         int ret;
3010
3011         /* Not valid to be called on unbound objects. */
3012         if (obj->gtt_space == NULL)
3013                 return -EINVAL;
3014
3015         ret = i915_gem_object_flush_gpu_write_domain(obj);
3016         if (ret)
3017                 return ret;
3018
3019
3020         /* Currently, we are always called from an non-interruptible context. */
3021         if (pipelined != obj->ring) {
3022                 ret = i915_gem_object_wait_rendering(obj);
3023                 if (ret)
3024                         return ret;
3025         }
3026
3027         i915_gem_object_flush_cpu_write_domain(obj);
3028
3029         old_read_domains = obj->base.read_domains;
3030         obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
3031
3032         trace_i915_gem_object_change_domain(obj,
3033                                             old_read_domains,
3034                                             obj->base.write_domain);
3035
3036         return 0;
3037 }
3038
3039 int
3040 i915_gem_object_flush_gpu(struct drm_i915_gem_object *obj)
3041 {
3042         int ret;
3043
3044         if (!obj->active)
3045                 return 0;
3046
3047         if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) {
3048                 ret = i915_gem_flush_ring(obj->ring, 0, obj->base.write_domain);
3049                 if (ret)
3050                         return ret;
3051         }
3052
3053         return i915_gem_object_wait_rendering(obj);
3054 }
3055
3056 /**
3057  * Moves a single object to the CPU read, and possibly write domain.
3058  *
3059  * This function returns when the move is complete, including waiting on
3060  * flushes to occur.
3061  */
3062 static int
3063 i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)
3064 {
3065         uint32_t old_write_domain, old_read_domains;
3066         int ret;
3067
3068         if (obj->base.write_domain == I915_GEM_DOMAIN_CPU)
3069                 return 0;
3070
3071         ret = i915_gem_object_flush_gpu_write_domain(obj);
3072         if (ret)
3073                 return ret;
3074
3075         ret = i915_gem_object_wait_rendering(obj);
3076         if (ret)
3077                 return ret;
3078
3079         i915_gem_object_flush_gtt_write_domain(obj);
3080
3081         /* If we have a partially-valid cache of the object in the CPU,
3082          * finish invalidating it and free the per-page flags.
3083          */
3084         i915_gem_object_set_to_full_cpu_read_domain(obj);
3085
3086         old_write_domain = obj->base.write_domain;
3087         old_read_domains = obj->base.read_domains;
3088
3089         /* Flush the CPU cache if it's still invalid. */
3090         if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) {
3091                 i915_gem_clflush_object(obj);
3092
3093                 obj->base.read_domains |= I915_GEM_DOMAIN_CPU;
3094         }
3095
3096         /* It should now be out of any other write domains, and we can update
3097          * the domain values for our changes.
3098          */
3099         BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
3100
3101         /* If we're writing through the CPU, then the GPU read domains will
3102          * need to be invalidated at next use.
3103          */
3104         if (write) {
3105                 obj->base.read_domains = I915_GEM_DOMAIN_CPU;
3106                 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
3107         }
3108
3109         trace_i915_gem_object_change_domain(obj,
3110                                             old_read_domains,
3111                                             old_write_domain);
3112
3113         return 0;
3114 }
3115
3116 /**
3117  * Moves the object from a partially CPU read to a full one.
3118  *
3119  * Note that this only resolves i915_gem_object_set_cpu_read_domain_range(),
3120  * and doesn't handle transitioning from !(read_domains & I915_GEM_DOMAIN_CPU).
3121  */
3122 static void
3123 i915_gem_object_set_to_full_cpu_read_domain(struct drm_i915_gem_object *obj)
3124 {
3125         if (!obj->page_cpu_valid)
3126                 return;
3127
3128         /* If we're partially in the CPU read domain, finish moving it in.
3129          */
3130         if (obj->base.read_domains & I915_GEM_DOMAIN_CPU) {
3131                 int i;
3132
3133                 for (i = 0; i <= (obj->base.size - 1) / PAGE_SIZE; i++) {
3134                         if (obj->page_cpu_valid[i])
3135                                 continue;
3136                         drm_clflush_pages(obj->pages + i, 1);
3137                 }
3138         }
3139
3140         /* Free the page_cpu_valid mappings which are now stale, whether
3141          * or not we've got I915_GEM_DOMAIN_CPU.
3142          */
3143         kfree(obj->page_cpu_valid);
3144         obj->page_cpu_valid = NULL;
3145 }
3146
3147 /**
3148  * Set the CPU read domain on a range of the object.
3149  *
3150  * The object ends up with I915_GEM_DOMAIN_CPU in its read flags although it's
3151  * not entirely valid.  The page_cpu_valid member of the object flags which
3152  * pages have been flushed, and will be respected by
3153  * i915_gem_object_set_to_cpu_domain() if it's called on to get a valid mapping
3154  * of the whole object.
3155  *
3156  * This function returns when the move is complete, including waiting on
3157  * flushes to occur.
3158  */
3159 static int
3160 i915_gem_object_set_cpu_read_domain_range(struct drm_i915_gem_object *obj,
3161                                           uint64_t offset, uint64_t size)
3162 {
3163         uint32_t old_read_domains;
3164         int i, ret;
3165
3166         if (offset == 0 && size == obj->base.size)
3167                 return i915_gem_object_set_to_cpu_domain(obj, 0);
3168