2 * Copyright © 2016 Intel Corporation
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
25 #include "../i915_selftest.h"
27 #include "mock_gem_device.h"
28 #include "huge_gem_object.h"
30 static int igt_gem_object(void *arg)
32 struct drm_i915_private *i915 = arg;
33 struct drm_i915_gem_object *obj;
36 /* Basic test to ensure we can create an object */
38 obj = i915_gem_object_create(i915, PAGE_SIZE);
41 pr_err("i915_gem_object_create failed, err=%d\n", err);
46 i915_gem_object_put(obj);
51 static int igt_phys_object(void *arg)
53 struct drm_i915_private *i915 = arg;
54 struct drm_i915_gem_object *obj;
57 /* Create an object and bind it to a contiguous set of physical pages,
58 * i.e. exercise the i915_gem_object_phys API.
61 obj = i915_gem_object_create(i915, PAGE_SIZE);
64 pr_err("i915_gem_object_create failed, err=%d\n", err);
68 mutex_lock(&i915->drm.struct_mutex);
69 err = i915_gem_object_attach_phys(obj, PAGE_SIZE);
70 mutex_unlock(&i915->drm.struct_mutex);
72 pr_err("i915_gem_object_attach_phys failed, err=%d\n", err);
76 if (obj->ops != &i915_gem_phys_ops) {
77 pr_err("i915_gem_object_attach_phys did not create a phys object\n");
82 if (!atomic_read(&obj->mm.pages_pin_count)) {
83 pr_err("i915_gem_object_attach_phys did not pin its phys pages\n");
88 /* Make the object dirty so that put_pages must do copy back the data */
89 mutex_lock(&i915->drm.struct_mutex);
90 err = i915_gem_object_set_to_gtt_domain(obj, true);
91 mutex_unlock(&i915->drm.struct_mutex);
93 pr_err("i915_gem_object_set_to_gtt_domain failed with err=%d\n",
99 i915_gem_object_put(obj);
104 static int igt_gem_huge(void *arg)
106 const unsigned int nreal = 509; /* just to be awkward */
107 struct drm_i915_private *i915 = arg;
108 struct drm_i915_gem_object *obj;
112 /* Basic sanitycheck of our huge fake object allocation */
114 obj = huge_gem_object(i915,
116 i915->ggtt.vm.total + PAGE_SIZE);
120 err = i915_gem_object_pin_pages(obj);
122 pr_err("Failed to allocate %u pages (%lu total), err=%d\n",
123 nreal, obj->base.size / PAGE_SIZE, err);
127 for (n = 0; n < obj->base.size / PAGE_SIZE; n++) {
128 if (i915_gem_object_get_page(obj, n) !=
129 i915_gem_object_get_page(obj, n % nreal)) {
130 pr_err("Page lookup mismatch at index %u [%u]\n",
138 i915_gem_object_unpin_pages(obj);
140 i915_gem_object_put(obj);
150 unsigned int swizzle;
153 static u64 swizzle_bit(unsigned int bit, u64 offset)
155 return (offset & BIT_ULL(bit)) >> (bit - 6);
158 static u64 tiled_offset(const struct tile *tile, u64 v)
162 if (tile->tiling == I915_TILING_NONE)
165 y = div64_u64_rem(v, tile->stride, &x);
166 v = div64_u64_rem(y, tile->height, &y) * tile->stride * tile->height;
168 if (tile->tiling == I915_TILING_X) {
169 v += y * tile->width;
170 v += div64_u64_rem(x, tile->width, &x) << tile->size;
172 } else if (tile->width == 128) {
173 const unsigned int ytile_span = 16;
174 const unsigned int ytile_height = 512;
177 v += div64_u64_rem(x, ytile_span, &x) * ytile_height;
180 const unsigned int ytile_span = 32;
181 const unsigned int ytile_height = 256;
184 v += div64_u64_rem(x, ytile_span, &x) * ytile_height;
188 switch (tile->swizzle) {
189 case I915_BIT_6_SWIZZLE_9:
190 v ^= swizzle_bit(9, v);
192 case I915_BIT_6_SWIZZLE_9_10:
193 v ^= swizzle_bit(9, v) ^ swizzle_bit(10, v);
195 case I915_BIT_6_SWIZZLE_9_11:
196 v ^= swizzle_bit(9, v) ^ swizzle_bit(11, v);
198 case I915_BIT_6_SWIZZLE_9_10_11:
199 v ^= swizzle_bit(9, v) ^ swizzle_bit(10, v) ^ swizzle_bit(11, v);
206 static int check_partial_mapping(struct drm_i915_gem_object *obj,
207 const struct tile *tile,
208 unsigned long end_time)
210 const unsigned int nreal = obj->scratch / PAGE_SIZE;
211 const unsigned long npages = obj->base.size / PAGE_SIZE;
212 struct i915_vma *vma;
216 if (igt_timeout(end_time,
217 "%s: timed out before tiling=%d stride=%d\n",
218 __func__, tile->tiling, tile->stride))
221 err = i915_gem_object_set_tiling(obj, tile->tiling, tile->stride);
223 pr_err("Failed to set tiling mode=%u, stride=%u, err=%d\n",
224 tile->tiling, tile->stride, err);
228 GEM_BUG_ON(i915_gem_object_get_tiling(obj) != tile->tiling);
229 GEM_BUG_ON(i915_gem_object_get_stride(obj) != tile->stride);
231 for_each_prime_number_from(page, 1, npages) {
232 struct i915_ggtt_view view =
233 compute_partial_view(obj, page, MIN_CHUNK_PAGES);
240 GEM_BUG_ON(view.partial.size > nreal);
243 err = i915_gem_object_set_to_gtt_domain(obj, true);
245 pr_err("Failed to flush to GTT write domain; err=%d\n",
250 vma = i915_gem_object_ggtt_pin(obj, &view, 0, 0, PIN_MAPPABLE);
252 pr_err("Failed to pin partial view: offset=%lu; err=%d\n",
253 page, (int)PTR_ERR(vma));
257 n = page - view.partial.offset;
258 GEM_BUG_ON(n >= view.partial.size);
260 io = i915_vma_pin_iomap(vma);
263 pr_err("Failed to iomap partial view: offset=%lu; err=%d\n",
264 page, (int)PTR_ERR(io));
268 iowrite32(page, io + n * PAGE_SIZE/sizeof(*io));
269 i915_vma_unpin_iomap(vma);
271 offset = tiled_offset(tile, page << PAGE_SHIFT);
272 if (offset >= obj->base.size)
275 flush_write_domain(obj, ~I915_GEM_DOMAIN_CPU);
277 p = i915_gem_object_get_page(obj, offset >> PAGE_SHIFT);
278 cpu = kmap(p) + offset_in_page(offset);
279 drm_clflush_virt_range(cpu, sizeof(*cpu));
280 if (*cpu != (u32)page) {
281 pr_err("Partial view for %lu [%u] (offset=%llu, size=%u [%llu, row size %u], fence=%d, tiling=%d, stride=%d) misalignment, expected write to page (%llu + %u [0x%llx]) of 0x%x, found 0x%x\n",
285 vma->size >> PAGE_SHIFT,
286 tile->tiling ? tile_row_pages(obj) : 0,
287 vma->fence ? vma->fence->id : -1, tile->tiling, tile->stride,
288 offset >> PAGE_SHIFT,
289 (unsigned int)offset_in_page(offset),
295 drm_clflush_virt_range(cpu, sizeof(*cpu));
300 i915_vma_destroy(vma);
306 static int igt_partial_tiling(void *arg)
308 const unsigned int nreal = 1 << 12; /* largest tile row x2 */
309 struct drm_i915_private *i915 = arg;
310 struct drm_i915_gem_object *obj;
311 intel_wakeref_t wakeref;
315 /* We want to check the page mapping and fencing of a large object
316 * mmapped through the GTT. The object we create is larger than can
317 * possibly be mmaped as a whole, and so we must use partial GGTT vma.
318 * We then check that a write through each partial GGTT vma ends up
319 * in the right set of pages within the object, and with the expected
320 * tiling, which we verify by manual swizzling.
323 obj = huge_gem_object(i915,
325 (1 + next_prime_number(i915->ggtt.vm.total >> PAGE_SHIFT)) << PAGE_SHIFT);
329 err = i915_gem_object_pin_pages(obj);
331 pr_err("Failed to allocate %u pages (%lu total), err=%d\n",
332 nreal, obj->base.size / PAGE_SIZE, err);
336 mutex_lock(&i915->drm.struct_mutex);
337 wakeref = intel_runtime_pm_get(i915);
347 tile.swizzle = I915_BIT_6_SWIZZLE_NONE;
348 tile.tiling = I915_TILING_NONE;
350 err = check_partial_mapping(obj, &tile, end);
351 if (err && err != -EINTR)
355 for (tiling = I915_TILING_X; tiling <= I915_TILING_Y; tiling++) {
357 unsigned int max_pitch;
361 if (i915->quirks & QUIRK_PIN_SWIZZLED_PAGES)
363 * The swizzling pattern is actually unknown as it
364 * varies based on physical address of each page.
365 * See i915_gem_detect_bit_6_swizzle().
369 tile.tiling = tiling;
372 tile.swizzle = i915->mm.bit_6_swizzle_x;
375 tile.swizzle = i915->mm.bit_6_swizzle_y;
379 GEM_BUG_ON(tile.swizzle == I915_BIT_6_SWIZZLE_UNKNOWN);
380 if (tile.swizzle == I915_BIT_6_SWIZZLE_9_17 ||
381 tile.swizzle == I915_BIT_6_SWIZZLE_9_10_17)
384 if (INTEL_GEN(i915) <= 2) {
388 } else if (tile.tiling == I915_TILING_Y &&
389 HAS_128_BYTE_Y_TILING(i915)) {
399 if (INTEL_GEN(i915) < 4)
400 max_pitch = 8192 / tile.width;
401 else if (INTEL_GEN(i915) < 7)
402 max_pitch = 128 * I965_FENCE_MAX_PITCH_VAL / tile.width;
404 max_pitch = 128 * GEN7_FENCE_MAX_PITCH_VAL / tile.width;
406 for (pitch = max_pitch; pitch; pitch >>= 1) {
407 tile.stride = tile.width * pitch;
408 err = check_partial_mapping(obj, &tile, end);
414 if (pitch > 2 && INTEL_GEN(i915) >= 4) {
415 tile.stride = tile.width * (pitch - 1);
416 err = check_partial_mapping(obj, &tile, end);
423 if (pitch < max_pitch && INTEL_GEN(i915) >= 4) {
424 tile.stride = tile.width * (pitch + 1);
425 err = check_partial_mapping(obj, &tile, end);
433 if (INTEL_GEN(i915) >= 4) {
434 for_each_prime_number(pitch, max_pitch) {
435 tile.stride = tile.width * pitch;
436 err = check_partial_mapping(obj, &tile, end);
448 intel_runtime_pm_put(i915, wakeref);
449 mutex_unlock(&i915->drm.struct_mutex);
450 i915_gem_object_unpin_pages(obj);
452 i915_gem_object_put(obj);
456 static int make_obj_busy(struct drm_i915_gem_object *obj)
458 struct drm_i915_private *i915 = to_i915(obj->base.dev);
459 struct i915_request *rq;
460 struct i915_vma *vma;
463 vma = i915_vma_instance(obj, &i915->ggtt.vm, NULL);
467 err = i915_vma_pin(vma, 0, 0, PIN_USER);
471 rq = i915_request_alloc(i915->engine[RCS], i915->kernel_context);
477 err = i915_vma_move_to_active(vma, rq, EXEC_OBJECT_WRITE);
479 i915_request_add(rq);
481 __i915_gem_object_release_unless_active(obj);
487 static bool assert_mmap_offset(struct drm_i915_private *i915,
491 struct drm_i915_gem_object *obj;
494 obj = i915_gem_object_create_internal(i915, size);
498 err = i915_gem_object_create_mmap_offset(obj);
499 i915_gem_object_put(obj);
501 return err == expected;
504 static void disable_retire_worker(struct drm_i915_private *i915)
506 i915_gem_shrinker_unregister(i915);
508 mutex_lock(&i915->drm.struct_mutex);
509 if (!i915->gt.active_requests++) {
510 intel_wakeref_t wakeref;
512 with_intel_runtime_pm(i915, wakeref)
513 i915_gem_unpark(i915);
515 mutex_unlock(&i915->drm.struct_mutex);
517 cancel_delayed_work_sync(&i915->gt.retire_work);
518 cancel_delayed_work_sync(&i915->gt.idle_work);
521 static int igt_mmap_offset_exhaustion(void *arg)
523 struct drm_i915_private *i915 = arg;
524 struct drm_mm *mm = &i915->drm.vma_offset_manager->vm_addr_space_mm;
525 struct drm_i915_gem_object *obj;
526 struct drm_mm_node resv, *hole;
527 u64 hole_start, hole_end;
530 /* Disable background reaper */
531 disable_retire_worker(i915);
532 GEM_BUG_ON(!i915->gt.awake);
534 /* Trim the device mmap space to only a page */
535 memset(&resv, 0, sizeof(resv));
536 drm_mm_for_each_hole(hole, mm, hole_start, hole_end) {
537 resv.start = hole_start;
538 resv.size = hole_end - hole_start - 1; /* PAGE_SIZE units */
539 err = drm_mm_reserve_node(mm, &resv);
541 pr_err("Failed to trim VMA manager, err=%d\n", err);
548 if (!assert_mmap_offset(i915, PAGE_SIZE, 0)) {
549 pr_err("Unable to insert object into single page hole\n");
555 if (!assert_mmap_offset(i915, 2*PAGE_SIZE, -ENOSPC)) {
556 pr_err("Unexpectedly succeeded in inserting too large object into single page hole\n");
561 /* Fill the hole, further allocation attempts should then fail */
562 obj = i915_gem_object_create_internal(i915, PAGE_SIZE);
568 err = i915_gem_object_create_mmap_offset(obj);
570 pr_err("Unable to insert object into reclaimed hole\n");
574 if (!assert_mmap_offset(i915, PAGE_SIZE, -ENOSPC)) {
575 pr_err("Unexpectedly succeeded in inserting object into no holes!\n");
580 i915_gem_object_put(obj);
582 /* Now fill with busy dead objects that we expect to reap */
583 for (loop = 0; loop < 3; loop++) {
584 intel_wakeref_t wakeref;
586 if (i915_terminally_wedged(&i915->gpu_error))
589 obj = i915_gem_object_create_internal(i915, PAGE_SIZE);
596 mutex_lock(&i915->drm.struct_mutex);
597 with_intel_runtime_pm(i915, wakeref)
598 err = make_obj_busy(obj);
599 mutex_unlock(&i915->drm.struct_mutex);
601 pr_err("[loop %d] Failed to busy the object\n", loop);
605 /* NB we rely on the _active_ reference to access obj now */
606 GEM_BUG_ON(!i915_gem_object_is_active(obj));
607 err = i915_gem_object_create_mmap_offset(obj);
609 pr_err("[loop %d] i915_gem_object_create_mmap_offset failed with err=%d\n",
616 drm_mm_remove_node(&resv);
618 mutex_lock(&i915->drm.struct_mutex);
619 if (--i915->gt.active_requests)
620 queue_delayed_work(i915->wq, &i915->gt.retire_work, 0);
622 queue_delayed_work(i915->wq, &i915->gt.idle_work, 0);
623 mutex_unlock(&i915->drm.struct_mutex);
624 i915_gem_shrinker_register(i915);
627 i915_gem_object_put(obj);
631 int i915_gem_object_mock_selftests(void)
633 static const struct i915_subtest tests[] = {
634 SUBTEST(igt_gem_object),
635 SUBTEST(igt_phys_object),
637 struct drm_i915_private *i915;
640 i915 = mock_gem_device();
644 err = i915_subtests(tests, i915);
646 drm_dev_put(&i915->drm);
650 int i915_gem_object_live_selftests(struct drm_i915_private *i915)
652 static const struct i915_subtest tests[] = {
653 SUBTEST(igt_gem_huge),
654 SUBTEST(igt_partial_tiling),
655 SUBTEST(igt_mmap_offset_exhaustion),
658 return i915_subtests(tests, i915);