drivers/gpu/drm/i915/selftests/i915_gem_gtt.c

   1 /*
   2  * Copyright © 2016 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  */
  24
  25 #include <linux/list_sort.h>
  26 #include <linux/prime_numbers.h>
  27
  28 #include "../i915_selftest.h"
  29 #include "i915_random.h"
  30
  31 #include "mock_context.h"
  32 #include "mock_drm.h"
  33 #include "mock_gem_device.h"
  34
  35 static void fake_free_pages(struct drm_i915_gem_object *obj,
  36                             struct sg_table *pages)
  37 {
  38         sg_free_table(pages);
  39         kfree(pages);
  40 }
  41
  42 static int fake_get_pages(struct drm_i915_gem_object *obj)
  43 {
  44 #define GFP (GFP_KERNEL | __GFP_NOWARN | __GFP_NORETRY)
  45 #define PFN_BIAS 0x1000
  46         struct sg_table *pages;
  47         struct scatterlist *sg;
  48         unsigned int sg_page_sizes;
  49         typeof(obj->base.size) rem;
  50
  51         pages = kmalloc(sizeof(*pages), GFP);
  52         if (!pages)
  53                 return -ENOMEM;
  54
  55         rem = round_up(obj->base.size, BIT(31)) >> 31;
  56         if (sg_alloc_table(pages, rem, GFP)) {
  57                 kfree(pages);
  58                 return -ENOMEM;
  59         }
  60
  61         sg_page_sizes = 0;
  62         rem = obj->base.size;
  63         for (sg = pages->sgl; sg; sg = sg_next(sg)) {
  64                 unsigned long len = min_t(typeof(rem), rem, BIT(31));
  65
  66                 GEM_BUG_ON(!len);
  67                 sg_set_page(sg, pfn_to_page(PFN_BIAS), len, 0);
  68                 sg_dma_address(sg) = page_to_phys(sg_page(sg));
  69                 sg_dma_len(sg) = len;
  70                 sg_page_sizes |= len;
  71
  72                 rem -= len;
  73         }
  74         GEM_BUG_ON(rem);
  75
  76         obj->mm.madv = I915_MADV_DONTNEED;
  77
  78         __i915_gem_object_set_pages(obj, pages, sg_page_sizes);
  79
  80         return 0;
  81 #undef GFP
  82 }
  83
  84 static void fake_put_pages(struct drm_i915_gem_object *obj,
  85                            struct sg_table *pages)
  86 {
  87         fake_free_pages(obj, pages);
  88         obj->mm.dirty = false;
  89         obj->mm.madv = I915_MADV_WILLNEED;
  90 }
  91
  92 static const struct drm_i915_gem_object_ops fake_ops = {
  93         .flags = I915_GEM_OBJECT_IS_SHRINKABLE,
  94         .get_pages = fake_get_pages,
  95         .put_pages = fake_put_pages,
  96 };
  97
  98 static struct drm_i915_gem_object *
  99 fake_dma_object(struct drm_i915_private *i915, u64 size)
 100 {
 101         struct drm_i915_gem_object *obj;
 102
 103         GEM_BUG_ON(!size);
 104         GEM_BUG_ON(!IS_ALIGNED(size, I915_GTT_PAGE_SIZE));
 105
 106         if (overflows_type(size, obj->base.size))
 107                 return ERR_PTR(-E2BIG);
 108
 109         obj = i915_gem_object_alloc(i915);
 110         if (!obj)
 111                 goto err;
 112
 113         drm_gem_private_object_init(&i915->drm, &obj->base, size);
 114         i915_gem_object_init(obj, &fake_ops);
 115
 116         obj->base.write_domain = I915_GEM_DOMAIN_CPU;
 117         obj->base.read_domains = I915_GEM_DOMAIN_CPU;
 118         obj->cache_level = I915_CACHE_NONE;
 119
 120         /* Preallocate the "backing storage" */
 121         if (i915_gem_object_pin_pages(obj))
 122                 goto err_obj;
 123
 124         i915_gem_object_unpin_pages(obj);
 125         return obj;
 126
 127 err_obj:
 128         i915_gem_object_put(obj);
 129 err:
 130         return ERR_PTR(-ENOMEM);
 131 }
 132
 133 static int igt_ppgtt_alloc(void *arg)
 134 {
 135         struct drm_i915_private *dev_priv = arg;
 136         struct i915_hw_ppgtt *ppgtt;
 137         u64 size, last;
 138         int err;
 139
 140         /* Allocate a ppggt and try to fill the entire range */
 141
 142         if (!USES_PPGTT(dev_priv))
 143                 return 0;
 144
 145         ppgtt = kzalloc(sizeof(*ppgtt), GFP_KERNEL);
 146         if (!ppgtt)
 147                 return -ENOMEM;
 148
 149         mutex_lock(&dev_priv->drm.struct_mutex);
 150         err = __hw_ppgtt_init(ppgtt, dev_priv);
 151         if (err)
 152                 goto err_ppgtt;
 153
 154         if (!ppgtt->base.allocate_va_range)
 155                 goto err_ppgtt_cleanup;
 156
 157         /* Check we can allocate the entire range */
 158         for (size = 4096;
 159              size <= ppgtt->base.total;
 160              size <<= 2) {
 161                 err = ppgtt->base.allocate_va_range(&ppgtt->base, 0, size);
 162                 if (err) {
 163                         if (err == -ENOMEM) {
 164                                 pr_info("[1] Ran out of memory for va_range [0 + %llx] [bit %d]\n",
 165                                         size, ilog2(size));
 166                                 err = 0; /* virtual space too large! */
 167                         }
 168                         goto err_ppgtt_cleanup;
 169                 }
 170
 171                 ppgtt->base.clear_range(&ppgtt->base, 0, size);
 172         }
 173
 174         /* Check we can incrementally allocate the entire range */
 175         for (last = 0, size = 4096;
 176              size <= ppgtt->base.total;
 177              last = size, size <<= 2) {
 178                 err = ppgtt->base.allocate_va_range(&ppgtt->base,
 179                                                     last, size - last);
 180                 if (err) {
 181                         if (err == -ENOMEM) {
 182                                 pr_info("[2] Ran out of memory for va_range [%llx + %llx] [bit %d]\n",
 183                                         last, size - last, ilog2(size));
 184                                 err = 0; /* virtual space too large! */
 185                         }
 186                         goto err_ppgtt_cleanup;
 187                 }
 188         }
 189
 190 err_ppgtt_cleanup:
 191         ppgtt->base.cleanup(&ppgtt->base);
 192 err_ppgtt:
 193         mutex_unlock(&dev_priv->drm.struct_mutex);
 194         kfree(ppgtt);
 195         return err;
 196 }
 197
 198 static int lowlevel_hole(struct drm_i915_private *i915,
 199                          struct i915_address_space *vm,
 200                          u64 hole_start, u64 hole_end,
 201                          unsigned long end_time)
 202 {
 203         I915_RND_STATE(seed_prng);
 204         unsigned int size;
 205         struct i915_vma mock_vma;
 206
 207         memset(&mock_vma, 0, sizeof(struct i915_vma));
 208
 209         /* Keep creating larger objects until one cannot fit into the hole */
 210         for (size = 12; (hole_end - hole_start) >> size; size++) {
 211                 I915_RND_SUBSTATE(prng, seed_prng);
 212                 struct drm_i915_gem_object *obj;
 213                 unsigned int *order, count, n;
 214                 u64 hole_size;
 215
 216                 hole_size = (hole_end - hole_start) >> size;
 217                 if (hole_size > KMALLOC_MAX_SIZE / sizeof(u32))
 218                         hole_size = KMALLOC_MAX_SIZE / sizeof(u32);
 219                 count = hole_size;
 220                 do {
 221                         count >>= 1;
 222                         order = i915_random_order(count, &prng);
 223                 } while (!order && count);
 224                 if (!order)
 225                         break;
 226
 227                 GEM_BUG_ON(count * BIT_ULL(size) > vm->total);
 228                 GEM_BUG_ON(hole_start + count * BIT_ULL(size) > hole_end);
 229
 230                 /* Ignore allocation failures (i.e. don't report them as
 231                  * a test failure) as we are purposefully allocating very
 232                  * large objects without checking that we have sufficient
 233                  * memory. We expect to hit -ENOMEM.
 234                  */
 235
 236                 obj = fake_dma_object(i915, BIT_ULL(size));
 237                 if (IS_ERR(obj)) {
 238                         kfree(order);
 239                         break;
 240                 }
 241
 242                 GEM_BUG_ON(obj->base.size != BIT_ULL(size));
 243
 244                 if (i915_gem_object_pin_pages(obj)) {
 245                         i915_gem_object_put(obj);
 246                         kfree(order);
 247                         break;
 248                 }
 249
 250                 for (n = 0; n < count; n++) {
 251                         u64 addr = hole_start + order[n] * BIT_ULL(size);
 252
 253                         GEM_BUG_ON(addr + BIT_ULL(size) > vm->total);
 254
 255                         if (igt_timeout(end_time,
 256                                         "%s timed out before %d/%d\n",
 257                                         __func__, n, count)) {
 258                                 hole_end = hole_start; /* quit */
 259                                 break;
 260                         }
 261
 262                         if (vm->allocate_va_range &&
 263                             vm->allocate_va_range(vm, addr, BIT_ULL(size)))
 264                                 break;
 265
 266                         mock_vma.pages = obj->mm.pages;
 267                         mock_vma.node.size = BIT_ULL(size);
 268                         mock_vma.node.start = addr;
 269
 270                         vm->insert_entries(vm, &mock_vma, I915_CACHE_NONE, 0);
 271                 }
 272                 count = n;
 273
 274                 i915_random_reorder(order, count, &prng);
 275                 for (n = 0; n < count; n++) {
 276                         u64 addr = hole_start + order[n] * BIT_ULL(size);
 277
 278                         GEM_BUG_ON(addr + BIT_ULL(size) > vm->total);
 279                         vm->clear_range(vm, addr, BIT_ULL(size));
 280                 }
 281
 282                 i915_gem_object_unpin_pages(obj);
 283                 i915_gem_object_put(obj);
 284
 285                 kfree(order);
 286         }
 287
 288         return 0;
 289 }
 290
 291 static void close_object_list(struct list_head *objects,
 292                               struct i915_address_space *vm)
 293 {
 294         struct drm_i915_gem_object *obj, *on;
 295         int ignored;
 296
 297         list_for_each_entry_safe(obj, on, objects, st_link) {
 298                 struct i915_vma *vma;
 299
 300                 vma = i915_vma_instance(obj, vm, NULL);
 301                 if (!IS_ERR(vma))
 302                         ignored = i915_vma_unbind(vma);
 303                 /* Only ppgtt vma may be closed before the object is freed */
 304                 if (!IS_ERR(vma) && !i915_vma_is_ggtt(vma))
 305                         i915_vma_close(vma);
 306
 307                 list_del(&obj->st_link);
 308                 i915_gem_object_put(obj);
 309         }
 310 }
 311
 312 static int fill_hole(struct drm_i915_private *i915,
 313                      struct i915_address_space *vm,
 314                      u64 hole_start, u64 hole_end,
 315                      unsigned long end_time)
 316 {
 317         const u64 hole_size = hole_end - hole_start;
 318         struct drm_i915_gem_object *obj;
 319         const unsigned long max_pages =
 320                 min_t(u64, ULONG_MAX - 1, hole_size/2 >> PAGE_SHIFT);
 321         const unsigned long max_step = max(int_sqrt(max_pages), 2UL);
 322         unsigned long npages, prime, flags;
 323         struct i915_vma *vma;
 324         LIST_HEAD(objects);
 325         int err;
 326
 327         /* Try binding many VMA working inwards from either edge */
 328
 329         flags = PIN_OFFSET_FIXED | PIN_USER;
 330         if (i915_is_ggtt(vm))
 331                 flags |= PIN_GLOBAL;
 332
 333         for_each_prime_number_from(prime, 2, max_step) {
 334                 for (npages = 1; npages <= max_pages; npages *= prime) {
 335                         const u64 full_size = npages << PAGE_SHIFT;
 336                         const struct {
 337                                 const char *name;
 338                                 u64 offset;
 339                                 int step;
 340                         } phases[] = {
 341                                 { "top-down", hole_end, -1, },
 342                                 { "bottom-up", hole_start, 1, },
 343                                 { }
 344                         }, *p;
 345
 346                         obj = fake_dma_object(i915, full_size);
 347                         if (IS_ERR(obj))
 348                                 break;
 349
 350                         list_add(&obj->st_link, &objects);
 351
 352                         /* Align differing sized objects against the edges, and
 353                          * check we don't walk off into the void when binding
 354                          * them into the GTT.
 355                          */
 356                         for (p = phases; p->name; p++) {
 357                                 u64 offset;
 358
 359                                 offset = p->offset;
 360                                 list_for_each_entry(obj, &objects, st_link) {
 361                                         vma = i915_vma_instance(obj, vm, NULL);
 362                                         if (IS_ERR(vma))
 363                                                 continue;
 364
 365                                         if (p->step < 0) {
 366                                                 if (offset < hole_start + obj->base.size)
 367                                                         break;
 368                                                 offset -= obj->base.size;
 369                                         }
 370
 371                                         err = i915_vma_pin(vma, 0, 0, offset | flags);
 372                                         if (err) {
 373                                                 pr_err("%s(%s) pin (forward) failed with err=%d on size=%lu pages (prime=%lu), offset=%llx\n",
 374                                                        __func__, p->name, err, npages, prime, offset);
 375                                                 goto err;
 376                                         }
 377
 378                                         if (!drm_mm_node_allocated(&vma->node) ||
 379                                             i915_vma_misplaced(vma, 0, 0, offset | flags)) {
 380                                                 pr_err("%s(%s) (forward) insert failed: vma.node=%llx + %llx [allocated? %d], expected offset %llx\n",
 381                                                        __func__, p->name, vma->node.start, vma->node.size, drm_mm_node_allocated(&vma->node),
 382                                                        offset);
 383                                                 err = -EINVAL;
 384                                                 goto err;
 385                                         }
 386
 387                                         i915_vma_unpin(vma);
 388
 389                                         if (p->step > 0) {
 390                                                 if (offset + obj->base.size > hole_end)
 391                                                         break;
 392                                                 offset += obj->base.size;
 393                                         }
 394                                 }
 395
 396                                 offset = p->offset;
 397                                 list_for_each_entry(obj, &objects, st_link) {
 398                                         vma = i915_vma_instance(obj, vm, NULL);
 399                                         if (IS_ERR(vma))
 400                                                 continue;
 401
 402                                         if (p->step < 0) {
 403                                                 if (offset < hole_start + obj->base.size)
 404                                                         break;
 405                                                 offset -= obj->base.size;
 406                                         }
 407
 408                                         if (!drm_mm_node_allocated(&vma->node) ||
 409                                             i915_vma_misplaced(vma, 0, 0, offset | flags)) {
 410                                                 pr_err("%s(%s) (forward) moved vma.node=%llx + %llx, expected offset %llx\n",
 411                                                        __func__, p->name, vma->node.start, vma->node.size,
 412                                                        offset);
 413                                                 err = -EINVAL;
 414                                                 goto err;
 415                                         }
 416
 417                                         err = i915_vma_unbind(vma);
 418                                         if (err) {
 419                                                 pr_err("%s(%s) (forward) unbind of vma.node=%llx + %llx failed with err=%d\n",
 420                                                        __func__, p->name, vma->node.start, vma->node.size,
 421                                                        err);
 422                                                 goto err;
 423                                         }
 424
 425                                         if (p->step > 0) {
 426                                                 if (offset + obj->base.size > hole_end)
 427                                                         break;
 428                                                 offset += obj->base.size;
 429                                         }
 430                                 }
 431
 432                                 offset = p->offset;
 433                                 list_for_each_entry_reverse(obj, &objects, st_link) {
 434                                         vma = i915_vma_instance(obj, vm, NULL);
 435                                         if (IS_ERR(vma))
 436                                                 continue;
 437
 438                                         if (p->step < 0) {
 439                                                 if (offset < hole_start + obj->base.size)
 440                                                         break;
 441                                                 offset -= obj->base.size;
 442                                         }
 443
 444                                         err = i915_vma_pin(vma, 0, 0, offset | flags);
 445                                         if (err) {
 446                                                 pr_err("%s(%s) pin (backward) failed with err=%d on size=%lu pages (prime=%lu), offset=%llx\n",
 447                                                        __func__, p->name, err, npages, prime, offset);
 448                                                 goto err;
 449                                         }
 450
 451                                         if (!drm_mm_node_allocated(&vma->node) ||
 452                                             i915_vma_misplaced(vma, 0, 0, offset | flags)) {
 453                                                 pr_err("%s(%s) (backward) insert failed: vma.node=%llx + %llx [allocated? %d], expected offset %llx\n",
 454                                                        __func__, p->name, vma->node.start, vma->node.size, drm_mm_node_allocated(&vma->node),
 455                                                        offset);
 456                                                 err = -EINVAL;
 457                                                 goto err;
 458                                         }
 459
 460                                         i915_vma_unpin(vma);
 461
 462                                         if (p->step > 0) {
 463                                                 if (offset + obj->base.size > hole_end)
 464                                                         break;
 465                                                 offset += obj->base.size;
 466                                         }
 467                                 }
 468
 469                                 offset = p->offset;
 470                                 list_for_each_entry_reverse(obj, &objects, st_link) {
 471                                         vma = i915_vma_instance(obj, vm, NULL);
 472                                         if (IS_ERR(vma))
 473                                                 continue;
 474
 475                                         if (p->step < 0) {
 476                                                 if (offset < hole_start + obj->base.size)
 477                                                         break;
 478                                                 offset -= obj->base.size;
 479                                         }
 480
 481                                         if (!drm_mm_node_allocated(&vma->node) ||
 482                                             i915_vma_misplaced(vma, 0, 0, offset | flags)) {
 483                                                 pr_err("%s(%s) (backward) moved vma.node=%llx + %llx [allocated? %d], expected offset %llx\n",
 484                                                        __func__, p->name, vma->node.start, vma->node.size, drm_mm_node_allocated(&vma->node),
 485                                                        offset);
 486                                                 err = -EINVAL;
 487                                                 goto err;
 488                                         }
 489
 490                                         err = i915_vma_unbind(vma);
 491                                         if (err) {
 492                                                 pr_err("%s(%s) (backward) unbind of vma.node=%llx + %llx failed with err=%d\n",
 493                                                        __func__, p->name, vma->node.start, vma->node.size,
 494                                                        err);
 495                                                 goto err;
 496                                         }
 497
 498                                         if (p->step > 0) {
 499                                                 if (offset + obj->base.size > hole_end)
 500                                                         break;
 501                                                 offset += obj->base.size;
 502                                         }
 503                                 }
 504                         }
 505
 506                         if (igt_timeout(end_time, "%s timed out (npages=%lu, prime=%lu)\n",
 507                                         __func__, npages, prime)) {
 508                                 err = -EINTR;
 509                                 goto err;
 510                         }
 511                 }
 512
 513                 close_object_list(&objects, vm);
 514         }
 515
 516         return 0;
 517
 518 err:
 519         close_object_list(&objects, vm);
 520         return err;
 521 }
 522
 523 static int walk_hole(struct drm_i915_private *i915,
 524                      struct i915_address_space *vm,
 525                      u64 hole_start, u64 hole_end,
 526                      unsigned long end_time)
 527 {
 528         const u64 hole_size = hole_end - hole_start;
 529         const unsigned long max_pages =
 530                 min_t(u64, ULONG_MAX - 1, hole_size >> PAGE_SHIFT);
 531         unsigned long flags;
 532         u64 size;
 533
 534         /* Try binding a single VMA in different positions within the hole */
 535
 536         flags = PIN_OFFSET_FIXED | PIN_USER;
 537         if (i915_is_ggtt(vm))
 538                 flags |= PIN_GLOBAL;
 539
 540         for_each_prime_number_from(size, 1, max_pages) {
 541                 struct drm_i915_gem_object *obj;
 542                 struct i915_vma *vma;
 543                 u64 addr;
 544                 int err = 0;
 545
 546                 obj = fake_dma_object(i915, size << PAGE_SHIFT);
 547                 if (IS_ERR(obj))
 548                         break;
 549
 550                 vma = i915_vma_instance(obj, vm, NULL);
 551                 if (IS_ERR(vma)) {
 552                         err = PTR_ERR(vma);
 553                         goto err_put;
 554                 }
 555
 556                 for (addr = hole_start;
 557                      addr + obj->base.size < hole_end;
 558                      addr += obj->base.size) {
 559                         err = i915_vma_pin(vma, 0, 0, addr | flags);
 560                         if (err) {
 561                                 pr_err("%s bind failed at %llx + %llx [hole %llx- %llx] with err=%d\n",
 562                                        __func__, addr, vma->size,
 563                                        hole_start, hole_end, err);
 564                                 goto err_close;
 565                         }
 566                         i915_vma_unpin(vma);
 567
 568                         if (!drm_mm_node_allocated(&vma->node) ||
 569                             i915_vma_misplaced(vma, 0, 0, addr | flags)) {
 570                                 pr_err("%s incorrect at %llx + %llx\n",
 571                                        __func__, addr, vma->size);
 572                                 err = -EINVAL;
 573                                 goto err_close;
 574                         }
 575
 576                         err = i915_vma_unbind(vma);
 577                         if (err) {
 578                                 pr_err("%s unbind failed at %llx + %llx  with err=%d\n",
 579                                        __func__, addr, vma->size, err);
 580                                 goto err_close;
 581                         }
 582
 583                         GEM_BUG_ON(drm_mm_node_allocated(&vma->node));
 584
 585                         if (igt_timeout(end_time,
 586                                         "%s timed out at %llx\n",
 587                                         __func__, addr)) {
 588                                 err = -EINTR;
 589                                 goto err_close;
 590                         }
 591                 }
 592
 593 err_close:
 594                 if (!i915_vma_is_ggtt(vma))
 595                         i915_vma_close(vma);
 596 err_put:
 597                 i915_gem_object_put(obj);
 598                 if (err)
 599                         return err;
 600         }
 601
 602         return 0;
 603 }
 604
 605 static int pot_hole(struct drm_i915_private *i915,
 606                     struct i915_address_space *vm,
 607                     u64 hole_start, u64 hole_end,
 608                     unsigned long end_time)
 609 {
 610         struct drm_i915_gem_object *obj;
 611         struct i915_vma *vma;
 612         unsigned long flags;
 613         unsigned int pot;
 614         int err = 0;
 615
 616         flags = PIN_OFFSET_FIXED | PIN_USER;
 617         if (i915_is_ggtt(vm))
 618                 flags |= PIN_GLOBAL;
 619
 620         obj = i915_gem_object_create_internal(i915, 2 * I915_GTT_PAGE_SIZE);
 621         if (IS_ERR(obj))
 622                 return PTR_ERR(obj);
 623
 624         vma = i915_vma_instance(obj, vm, NULL);
 625         if (IS_ERR(vma)) {
 626                 err = PTR_ERR(vma);
 627                 goto err_obj;
 628         }
 629
 630         /* Insert a pair of pages across every pot boundary within the hole */
 631         for (pot = fls64(hole_end - 1) - 1;
 632              pot > ilog2(2 * I915_GTT_PAGE_SIZE);
 633              pot--) {
 634                 u64 step = BIT_ULL(pot);
 635                 u64 addr;
 636
 637                 for (addr = round_up(hole_start + I915_GTT_PAGE_SIZE, step) - I915_GTT_PAGE_SIZE;
 638                      addr <= round_down(hole_end - 2*I915_GTT_PAGE_SIZE, step) - I915_GTT_PAGE_SIZE;
 639                      addr += step) {
 640                         err = i915_vma_pin(vma, 0, 0, addr | flags);
 641                         if (err) {
 642                                 pr_err("%s failed to pin object at %llx in hole [%llx - %llx], with err=%d\n",
 643                                        __func__,
 644                                        addr,
 645                                        hole_start, hole_end,
 646                                        err);
 647                                 goto err;
 648                         }
 649
 650                         if (!drm_mm_node_allocated(&vma->node) ||
 651                             i915_vma_misplaced(vma, 0, 0, addr | flags)) {
 652                                 pr_err("%s incorrect at %llx + %llx\n",
 653                                        __func__, addr, vma->size);
 654                                 i915_vma_unpin(vma);
 655                                 err = i915_vma_unbind(vma);
 656                                 err = -EINVAL;
 657                                 goto err;
 658                         }
 659
 660                         i915_vma_unpin(vma);
 661                         err = i915_vma_unbind(vma);
 662                         GEM_BUG_ON(err);
 663                 }
 664
 665                 if (igt_timeout(end_time,
 666                                 "%s timed out after %d/%d\n",
 667                                 __func__, pot, fls64(hole_end - 1) - 1)) {
 668                         err = -EINTR;
 669                         goto err;
 670                 }
 671         }
 672
 673 err:
 674         if (!i915_vma_is_ggtt(vma))
 675                 i915_vma_close(vma);
 676 err_obj:
 677         i915_gem_object_put(obj);
 678         return err;
 679 }
 680
 681 static int drunk_hole(struct drm_i915_private *i915,
 682                       struct i915_address_space *vm,
 683                       u64 hole_start, u64 hole_end,
 684                       unsigned long end_time)
 685 {
 686         I915_RND_STATE(prng);
 687         unsigned int size;
 688         unsigned long flags;
 689
 690         flags = PIN_OFFSET_FIXED | PIN_USER;
 691         if (i915_is_ggtt(vm))
 692                 flags |= PIN_GLOBAL;
 693
 694         /* Keep creating larger objects until one cannot fit into the hole */
 695         for (size = 12; (hole_end - hole_start) >> size; size++) {
 696                 struct drm_i915_gem_object *obj;
 697                 unsigned int *order, count, n;
 698                 struct i915_vma *vma;
 699                 u64 hole_size;
 700                 int err;
 701
 702                 hole_size = (hole_end - hole_start) >> size;
 703                 if (hole_size > KMALLOC_MAX_SIZE / sizeof(u32))
 704                         hole_size = KMALLOC_MAX_SIZE / sizeof(u32);
 705                 count = hole_size;
 706                 do {
 707                         count >>= 1;
 708                         order = i915_random_order(count, &prng);
 709                 } while (!order && count);
 710                 if (!order)
 711                         break;
 712
 713                 /* Ignore allocation failures (i.e. don't report them as
 714                  * a test failure) as we are purposefully allocating very
 715                  * large objects without checking that we have sufficient
 716                  * memory. We expect to hit -ENOMEM.
 717                  */
 718
 719                 obj = fake_dma_object(i915, BIT_ULL(size));
 720                 if (IS_ERR(obj)) {
 721                         kfree(order);
 722                         break;
 723                 }
 724
 725                 vma = i915_vma_instance(obj, vm, NULL);
 726                 if (IS_ERR(vma)) {
 727                         err = PTR_ERR(vma);
 728                         goto err_obj;
 729                 }
 730
 731                 GEM_BUG_ON(vma->size != BIT_ULL(size));
 732
 733                 for (n = 0; n < count; n++) {
 734                         u64 addr = hole_start + order[n] * BIT_ULL(size);
 735
 736                         err = i915_vma_pin(vma, 0, 0, addr | flags);
 737                         if (err) {
 738                                 pr_err("%s failed to pin object at %llx + %llx in hole [%llx - %llx], with err=%d\n",
 739                                        __func__,
 740                                        addr, BIT_ULL(size),
 741                                        hole_start, hole_end,
 742                                        err);
 743                                 goto err;
 744                         }
 745
 746                         if (!drm_mm_node_allocated(&vma->node) ||
 747                             i915_vma_misplaced(vma, 0, 0, addr | flags)) {
 748                                 pr_err("%s incorrect at %llx + %llx\n",
 749                                        __func__, addr, BIT_ULL(size));
 750                                 i915_vma_unpin(vma);
 751                                 err = i915_vma_unbind(vma);
 752                                 err = -EINVAL;
 753                                 goto err;
 754                         }
 755
 756                         i915_vma_unpin(vma);
 757                         err = i915_vma_unbind(vma);
 758                         GEM_BUG_ON(err);
 759
 760                         if (igt_timeout(end_time,
 761                                         "%s timed out after %d/%d\n",
 762                                         __func__, n, count)) {
 763                                 err = -EINTR;
 764                                 goto err;
 765                         }
 766                 }
 767
 768 err:
 769                 if (!i915_vma_is_ggtt(vma))
 770                         i915_vma_close(vma);
 771 err_obj:
 772                 i915_gem_object_put(obj);
 773                 kfree(order);
 774                 if (err)
 775                         return err;
 776         }
 777
 778         return 0;
 779 }
 780
 781 static int __shrink_hole(struct drm_i915_private *i915,
 782                          struct i915_address_space *vm,
 783                          u64 hole_start, u64 hole_end,
 784                          unsigned long end_time)
 785 {
 786         struct drm_i915_gem_object *obj;
 787         unsigned long flags = PIN_OFFSET_FIXED | PIN_USER;
 788         unsigned int order = 12;
 789         LIST_HEAD(objects);
 790         int err = 0;
 791         u64 addr;
 792
 793         /* Keep creating larger objects until one cannot fit into the hole */
 794         for (addr = hole_start; addr < hole_end; ) {
 795                 struct i915_vma *vma;
 796                 u64 size = BIT_ULL(order++);
 797
 798                 size = min(size, hole_end - addr);
 799                 obj = fake_dma_object(i915, size);
 800                 if (IS_ERR(obj)) {
 801                         err = PTR_ERR(obj);
 802                         break;
 803                 }
 804
 805                 list_add(&obj->st_link, &objects);
 806
 807                 vma = i915_vma_instance(obj, vm, NULL);
 808                 if (IS_ERR(vma)) {
 809                         err = PTR_ERR(vma);
 810                         break;
 811                 }
 812
 813                 GEM_BUG_ON(vma->size != size);
 814
 815                 err = i915_vma_pin(vma, 0, 0, addr | flags);
 816                 if (err) {
 817                         pr_err("%s failed to pin object at %llx + %llx in hole [%llx - %llx], with err=%d\n",
 818                                __func__, addr, size, hole_start, hole_end, err);
 819                         break;
 820                 }
 821
 822                 if (!drm_mm_node_allocated(&vma->node) ||
 823                     i915_vma_misplaced(vma, 0, 0, addr | flags)) {
 824                         pr_err("%s incorrect at %llx + %llx\n",
 825                                __func__, addr, size);
 826                         i915_vma_unpin(vma);
 827                         err = i915_vma_unbind(vma);
 828                         err = -EINVAL;
 829                         break;
 830                 }
 831
 832                 i915_vma_unpin(vma);
 833                 addr += size;
 834
 835                 if (igt_timeout(end_time,
 836                                 "%s timed out at ofset %llx [%llx - %llx]\n",
 837                                 __func__, addr, hole_start, hole_end)) {
 838                         err = -EINTR;
 839                         break;
 840                 }
 841         }
 842
 843         close_object_list(&objects, vm);
 844         return err;
 845 }
 846
 847 static int shrink_hole(struct drm_i915_private *i915,
 848                        struct i915_address_space *vm,
 849                        u64 hole_start, u64 hole_end,
 850                        unsigned long end_time)
 851 {
 852         unsigned long prime;
 853         int err;
 854
 855         vm->fault_attr.probability = 999;
 856         atomic_set(&vm->fault_attr.times, -1);
 857
 858         for_each_prime_number_from(prime, 0, ULONG_MAX - 1) {
 859                 vm->fault_attr.interval = prime;
 860                 err = __shrink_hole(i915, vm, hole_start, hole_end, end_time);
 861                 if (err)
 862                         break;
 863         }
 864
 865         memset(&vm->fault_attr, 0, sizeof(vm->fault_attr));
 866
 867         return err;
 868 }
 869
 870 static int exercise_ppgtt(struct drm_i915_private *dev_priv,
 871                           int (*func)(struct drm_i915_private *i915,
 872                                       struct i915_address_space *vm,
 873                                       u64 hole_start, u64 hole_end,
 874                                       unsigned long end_time))
 875 {
 876         struct drm_file *file;
 877         struct i915_hw_ppgtt *ppgtt;
 878         IGT_TIMEOUT(end_time);
 879         int err;
 880
 881         if (!USES_FULL_PPGTT(dev_priv))
 882                 return 0;
 883
 884         file = mock_file(dev_priv);
 885         if (IS_ERR(file))
 886                 return PTR_ERR(file);
 887
 888         mutex_lock(&dev_priv->drm.struct_mutex);
 889         ppgtt = i915_ppgtt_create(dev_priv, file->driver_priv, "mock");
 890         if (IS_ERR(ppgtt)) {
 891                 err = PTR_ERR(ppgtt);
 892                 goto out_unlock;
 893         }
 894         GEM_BUG_ON(offset_in_page(ppgtt->base.total));
 895         GEM_BUG_ON(ppgtt->base.closed);
 896
 897         err = func(dev_priv, &ppgtt->base, 0, ppgtt->base.total, end_time);
 898
 899         i915_ppgtt_close(&ppgtt->base);
 900         i915_ppgtt_put(ppgtt);
 901 out_unlock:
 902         mutex_unlock(&dev_priv->drm.struct_mutex);
 903
 904         mock_file_free(dev_priv, file);
 905         return err;
 906 }
 907
 908 static int igt_ppgtt_fill(void *arg)
 909 {
 910         return exercise_ppgtt(arg, fill_hole);
 911 }
 912
 913 static int igt_ppgtt_walk(void *arg)
 914 {
 915         return exercise_ppgtt(arg, walk_hole);
 916 }
 917
 918 static int igt_ppgtt_pot(void *arg)
 919 {
 920         return exercise_ppgtt(arg, pot_hole);
 921 }
 922
 923 static int igt_ppgtt_drunk(void *arg)
 924 {
 925         return exercise_ppgtt(arg, drunk_hole);
 926 }
 927
 928 static int igt_ppgtt_lowlevel(void *arg)
 929 {
 930         return exercise_ppgtt(arg, lowlevel_hole);
 931 }
 932
 933 static int igt_ppgtt_shrink(void *arg)
 934 {
 935         return exercise_ppgtt(arg, shrink_hole);
 936 }
 937
 938 static int sort_holes(void *priv, struct list_head *A, struct list_head *B)
 939 {
 940         struct drm_mm_node *a = list_entry(A, typeof(*a), hole_stack);
 941         struct drm_mm_node *b = list_entry(B, typeof(*b), hole_stack);
 942
 943         if (a->start < b->start)
 944                 return -1;
 945         else
 946                 return 1;
 947 }
 948
 949 static int exercise_ggtt(struct drm_i915_private *i915,
 950                          int (*func)(struct drm_i915_private *i915,
 951                                      struct i915_address_space *vm,
 952                                      u64 hole_start, u64 hole_end,
 953                                      unsigned long end_time))
 954 {
 955         struct i915_ggtt *ggtt = &i915->ggtt;
 956         u64 hole_start, hole_end, last = 0;
 957         struct drm_mm_node *node;
 958         IGT_TIMEOUT(end_time);
 959         int err;
 960
 961         mutex_lock(&i915->drm.struct_mutex);
 962 restart:
 963         list_sort(NULL, &ggtt->base.mm.hole_stack, sort_holes);
 964         drm_mm_for_each_hole(node, &ggtt->base.mm, hole_start, hole_end) {
 965                 if (hole_start < last)
 966                         continue;
 967
 968                 if (ggtt->base.mm.color_adjust)
 969                         ggtt->base.mm.color_adjust(node, 0,
 970                                                    &hole_start, &hole_end);
 971                 if (hole_start >= hole_end)
 972                         continue;
 973
 974                 err = func(i915, &ggtt->base, hole_start, hole_end, end_time);
 975                 if (err)
 976                         break;
 977
 978                 /* As we have manipulated the drm_mm, the list may be corrupt */
 979                 last = hole_end;
 980                 goto restart;
 981         }
 982         mutex_unlock(&i915->drm.struct_mutex);
 983
 984         return err;
 985 }
 986
 987 static int igt_ggtt_fill(void *arg)
 988 {
 989         return exercise_ggtt(arg, fill_hole);
 990 }
 991
 992 static int igt_ggtt_walk(void *arg)
 993 {
 994         return exercise_ggtt(arg, walk_hole);
 995 }
 996
 997 static int igt_ggtt_pot(void *arg)
 998 {
 999         return exercise_ggtt(arg, pot_hole);
1000 }
1001
1002 static int igt_ggtt_drunk(void *arg)
1003 {
1004         return exercise_ggtt(arg, drunk_hole);
1005 }
1006
1007 static int igt_ggtt_lowlevel(void *arg)
1008 {
1009         return exercise_ggtt(arg, lowlevel_hole);
1010 }
1011
1012 static int igt_ggtt_page(void *arg)
1013 {
1014         const unsigned int count = PAGE_SIZE/sizeof(u32);
1015         I915_RND_STATE(prng);
1016         struct drm_i915_private *i915 = arg;
1017         struct i915_ggtt *ggtt = &i915->ggtt;
1018         struct drm_i915_gem_object *obj;
1019         struct drm_mm_node tmp;
1020         unsigned int *order, n;
1021         int err;
1022
1023         mutex_lock(&i915->drm.struct_mutex);
1024
1025         obj = i915_gem_object_create_internal(i915, PAGE_SIZE);
1026         if (IS_ERR(obj)) {
1027                 err = PTR_ERR(obj);
1028                 goto out_unlock;
1029         }
1030
1031         err = i915_gem_object_pin_pages(obj);
1032         if (err)
1033                 goto out_free;
1034
1035         memset(&tmp, 0, sizeof(tmp));
1036         err = drm_mm_insert_node_in_range(&ggtt->base.mm, &tmp,
1037                                           1024 * PAGE_SIZE, 0,
1038                                           I915_COLOR_UNEVICTABLE,
1039                                           0, ggtt->mappable_end,
1040                                           DRM_MM_INSERT_LOW);
1041         if (err)
1042                 goto out_unpin;
1043
1044         order = i915_random_order(count, &prng);
1045         if (!order) {
1046                 err = -ENOMEM;
1047                 goto out_remove;
1048         }
1049
1050         for (n = 0; n < count; n++) {
1051                 u64 offset = tmp.start + order[n] * PAGE_SIZE;
1052                 u32 __iomem *vaddr;
1053
1054                 ggtt->base.insert_page(&ggtt->base,
1055                                        i915_gem_object_get_dma_address(obj, 0),
1056                                        offset, I915_CACHE_NONE, 0);
1057
1058                 vaddr = io_mapping_map_atomic_wc(&ggtt->mappable, offset);
1059                 iowrite32(n, vaddr + n);
1060                 io_mapping_unmap_atomic(vaddr);
1061
1062                 wmb();
1063                 ggtt->base.clear_range(&ggtt->base, offset, PAGE_SIZE);
1064         }
1065
1066         i915_random_reorder(order, count, &prng);
1067         for (n = 0; n < count; n++) {
1068                 u64 offset = tmp.start + order[n] * PAGE_SIZE;
1069                 u32 __iomem *vaddr;
1070                 u32 val;
1071
1072                 ggtt->base.insert_page(&ggtt->base,
1073                                        i915_gem_object_get_dma_address(obj, 0),
1074                                        offset, I915_CACHE_NONE, 0);
1075
1076                 vaddr = io_mapping_map_atomic_wc(&ggtt->mappable, offset);
1077                 val = ioread32(vaddr + n);
1078                 io_mapping_unmap_atomic(vaddr);
1079
1080                 ggtt->base.clear_range(&ggtt->base, offset, PAGE_SIZE);
1081
1082                 if (val != n) {
1083                         pr_err("insert page failed: found %d, expected %d\n",
1084                                val, n);
1085                         err = -EINVAL;
1086                         break;
1087                 }
1088         }
1089
1090         kfree(order);
1091 out_remove:
1092         drm_mm_remove_node(&tmp);
1093 out_unpin:
1094         i915_gem_object_unpin_pages(obj);
1095 out_free:
1096         i915_gem_object_put(obj);
1097 out_unlock:
1098         mutex_unlock(&i915->drm.struct_mutex);
1099         return err;
1100 }
1101
1102 static void track_vma_bind(struct i915_vma *vma)
1103 {
1104         struct drm_i915_gem_object *obj = vma->obj;
1105
1106         obj->bind_count++; /* track for eviction later */
1107         __i915_gem_object_pin_pages(obj);
1108
1109         vma->pages = obj->mm.pages;
1110         list_move_tail(&vma->vm_link, &vma->vm->inactive_list);
1111 }
1112
1113 static int exercise_mock(struct drm_i915_private *i915,
1114                          int (*func)(struct drm_i915_private *i915,
1115                                      struct i915_address_space *vm,
1116                                      u64 hole_start, u64 hole_end,
1117                                      unsigned long end_time))
1118 {
1119         struct i915_gem_context *ctx;
1120         struct i915_hw_ppgtt *ppgtt;
1121         IGT_TIMEOUT(end_time);
1122         int err;
1123
1124         ctx = mock_context(i915, "mock");
1125         if (!ctx)
1126                 return -ENOMEM;
1127
1128         ppgtt = ctx->ppgtt;
1129         GEM_BUG_ON(!ppgtt);
1130
1131         err = func(i915, &ppgtt->base, 0, ppgtt->base.total, end_time);
1132
1133         mock_context_close(ctx);
1134         return err;
1135 }
1136
1137 static int igt_mock_fill(void *arg)
1138 {
1139         return exercise_mock(arg, fill_hole);
1140 }
1141
1142 static int igt_mock_walk(void *arg)
1143 {
1144         return exercise_mock(arg, walk_hole);
1145 }
1146
1147 static int igt_mock_pot(void *arg)
1148 {
1149         return exercise_mock(arg, pot_hole);
1150 }
1151
1152 static int igt_mock_drunk(void *arg)
1153 {
1154         return exercise_mock(arg, drunk_hole);
1155 }
1156
1157 static int igt_gtt_reserve(void *arg)
1158 {
1159         struct drm_i915_private *i915 = arg;
1160         struct drm_i915_gem_object *obj, *on;
1161         LIST_HEAD(objects);
1162         u64 total;
1163         int err;
1164
1165         /* i915_gem_gtt_reserve() tries to reserve the precise range
1166          * for the node, and evicts if it has to. So our test checks that
1167          * it can give us the requsted space and prevent overlaps.
1168          */
1169
1170         /* Start by filling the GGTT */
1171         for (total = 0;
1172              total + 2*I915_GTT_PAGE_SIZE <= i915->ggtt.base.total;
1173              total += 2*I915_GTT_PAGE_SIZE) {
1174                 struct i915_vma *vma;
1175
1176                 obj = i915_gem_object_create_internal(i915, 2*PAGE_SIZE);
1177                 if (IS_ERR(obj)) {
1178                         err = PTR_ERR(obj);
1179                         goto out;
1180                 }
1181
1182                 err = i915_gem_object_pin_pages(obj);
1183                 if (err) {
1184                         i915_gem_object_put(obj);
1185                         goto out;
1186                 }
1187
1188                 list_add(&obj->st_link, &objects);
1189
1190                 vma = i915_vma_instance(obj, &i915->ggtt.base, NULL);
1191                 if (IS_ERR(vma)) {
1192                         err = PTR_ERR(vma);
1193                         goto out;
1194                 }
1195
1196                 err = i915_gem_gtt_reserve(&i915->ggtt.base, &vma->node,
1197                                            obj->base.size,
1198                                            total,
1199                                            obj->cache_level,
1200                                            0);
1201                 if (err) {
1202                         pr_err("i915_gem_gtt_reserve (pass 1) failed at %llu/%llu with err=%d\n",
1203                                total, i915->ggtt.base.total, err);
1204                         goto out;
1205                 }
1206                 track_vma_bind(vma);
1207
1208                 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
1209                 if (vma->node.start != total ||
1210                     vma->node.size != 2*I915_GTT_PAGE_SIZE) {
1211                         pr_err("i915_gem_gtt_reserve (pass 1) placement failed, found (%llx + %llx), expected (%llx + %lx)\n",
1212                                vma->node.start, vma->node.size,
1213                                total, 2*I915_GTT_PAGE_SIZE);
1214                         err = -EINVAL;
1215                         goto out;
1216                 }
1217         }
1218
1219         /* Now we start forcing evictions */
1220         for (total = I915_GTT_PAGE_SIZE;
1221              total + 2*I915_GTT_PAGE_SIZE <= i915->ggtt.base.total;
1222              total += 2*I915_GTT_PAGE_SIZE) {
1223                 struct i915_vma *vma;
1224
1225                 obj = i915_gem_object_create_internal(i915, 2*PAGE_SIZE);
1226                 if (IS_ERR(obj)) {
1227                         err = PTR_ERR(obj);
1228                         goto out;
1229                 }
1230
1231                 err = i915_gem_object_pin_pages(obj);
1232                 if (err) {
1233                         i915_gem_object_put(obj);
1234                         goto out;
1235                 }
1236
1237                 list_add(&obj->st_link, &objects);
1238
1239                 vma = i915_vma_instance(obj, &i915->ggtt.base, NULL);
1240                 if (IS_ERR(vma)) {
1241                         err = PTR_ERR(vma);
1242                         goto out;
1243                 }
1244
1245                 err = i915_gem_gtt_reserve(&i915->ggtt.base, &vma->node,
1246                                            obj->base.size,
1247                                            total,
1248                                            obj->cache_level,
1249                                            0);
1250                 if (err) {
1251                         pr_err("i915_gem_gtt_reserve (pass 2) failed at %llu/%llu with err=%d\n",
1252                                total, i915->ggtt.base.total, err);
1253                         goto out;
1254                 }
1255                 track_vma_bind(vma);
1256
1257                 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
1258                 if (vma->node.start != total ||
1259                     vma->node.size != 2*I915_GTT_PAGE_SIZE) {
1260                         pr_err("i915_gem_gtt_reserve (pass 2) placement failed, found (%llx + %llx), expected (%llx + %lx)\n",
1261                                vma->node.start, vma->node.size,
1262                                total, 2*I915_GTT_PAGE_SIZE);
1263                         err = -EINVAL;
1264                         goto out;
1265                 }
1266         }
1267
1268         /* And then try at random */
1269         list_for_each_entry_safe(obj, on, &objects, st_link) {
1270                 struct i915_vma *vma;
1271                 u64 offset;
1272
1273                 vma = i915_vma_instance(obj, &i915->ggtt.base, NULL);
1274                 if (IS_ERR(vma)) {
1275                         err = PTR_ERR(vma);
1276                         goto out;
1277                 }
1278
1279                 err = i915_vma_unbind(vma);
1280                 if (err) {
1281                         pr_err("i915_vma_unbind failed with err=%d!\n", err);
1282                         goto out;
1283                 }
1284
1285                 offset = random_offset(0, i915->ggtt.base.total,
1286                                        2*I915_GTT_PAGE_SIZE,
1287                                        I915_GTT_MIN_ALIGNMENT);
1288
1289                 err = i915_gem_gtt_reserve(&i915->ggtt.base, &vma->node,
1290                                            obj->base.size,
1291                                            offset,
1292                                            obj->cache_level,
1293                                            0);
1294                 if (err) {
1295                         pr_err("i915_gem_gtt_reserve (pass 3) failed at %llu/%llu with err=%d\n",
1296                                total, i915->ggtt.base.total, err);
1297                         goto out;
1298                 }
1299                 track_vma_bind(vma);
1300
1301                 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
1302                 if (vma->node.start != offset ||
1303                     vma->node.size != 2*I915_GTT_PAGE_SIZE) {
1304                         pr_err("i915_gem_gtt_reserve (pass 3) placement failed, found (%llx + %llx), expected (%llx + %lx)\n",
1305                                vma->node.start, vma->node.size,
1306                                offset, 2*I915_GTT_PAGE_SIZE);
1307                         err = -EINVAL;
1308                         goto out;
1309                 }
1310         }
1311
1312 out:
1313         list_for_each_entry_safe(obj, on, &objects, st_link) {
1314                 i915_gem_object_unpin_pages(obj);
1315                 i915_gem_object_put(obj);
1316         }
1317         return err;
1318 }
1319
1320 static int igt_gtt_insert(void *arg)
1321 {
1322         struct drm_i915_private *i915 = arg;
1323         struct drm_i915_gem_object *obj, *on;
1324         struct drm_mm_node tmp = {};
1325         const struct invalid_insert {
1326                 u64 size;
1327                 u64 alignment;
1328                 u64 start, end;
1329         } invalid_insert[] = {
1330                 {
1331                         i915->ggtt.base.total + I915_GTT_PAGE_SIZE, 0,
1332                         0, i915->ggtt.base.total,
1333                 },
1334                 {
1335                         2*I915_GTT_PAGE_SIZE, 0,
1336                         0, I915_GTT_PAGE_SIZE,
1337                 },
1338                 {
1339                         -(u64)I915_GTT_PAGE_SIZE, 0,
1340                         0, 4*I915_GTT_PAGE_SIZE,
1341                 },
1342                 {
1343                         -(u64)2*I915_GTT_PAGE_SIZE, 2*I915_GTT_PAGE_SIZE,
1344                         0, 4*I915_GTT_PAGE_SIZE,
1345                 },
1346                 {
1347                         I915_GTT_PAGE_SIZE, I915_GTT_MIN_ALIGNMENT << 1,
1348                         I915_GTT_MIN_ALIGNMENT, I915_GTT_MIN_ALIGNMENT << 1,
1349                 },
1350                 {}
1351         }, *ii;
1352         LIST_HEAD(objects);
1353         u64 total;
1354         int err;
1355
1356         /* i915_gem_gtt_insert() tries to allocate some free space in the GTT
1357          * to the node, evicting if required.
1358          */
1359
1360         /* Check a couple of obviously invalid requests */
1361         for (ii = invalid_insert; ii->size; ii++) {
1362                 err = i915_gem_gtt_insert(&i915->ggtt.base, &tmp,
1363                                           ii->size, ii->alignment,
1364                                           I915_COLOR_UNEVICTABLE,
1365                                           ii->start, ii->end,
1366                                           0);
1367                 if (err != -ENOSPC) {
1368                         pr_err("Invalid i915_gem_gtt_insert(.size=%llx, .alignment=%llx, .start=%llx, .end=%llx) succeeded (err=%d)\n",
1369                                ii->size, ii->alignment, ii->start, ii->end,
1370                                err);
1371                         return -EINVAL;
1372                 }
1373         }
1374
1375         /* Start by filling the GGTT */
1376         for (total = 0;
1377              total + I915_GTT_PAGE_SIZE <= i915->ggtt.base.total;
1378              total += I915_GTT_PAGE_SIZE) {
1379                 struct i915_vma *vma;
1380
1381                 obj = i915_gem_object_create_internal(i915, I915_GTT_PAGE_SIZE);
1382                 if (IS_ERR(obj)) {
1383                         err = PTR_ERR(obj);
1384                         goto out;
1385                 }
1386
1387                 err = i915_gem_object_pin_pages(obj);
1388                 if (err) {
1389                         i915_gem_object_put(obj);
1390                         goto out;
1391                 }
1392
1393                 list_add(&obj->st_link, &objects);
1394
1395                 vma = i915_vma_instance(obj, &i915->ggtt.base, NULL);
1396                 if (IS_ERR(vma)) {
1397                         err = PTR_ERR(vma);
1398                         goto out;
1399                 }
1400
1401                 err = i915_gem_gtt_insert(&i915->ggtt.base, &vma->node,
1402                                           obj->base.size, 0, obj->cache_level,
1403                                           0, i915->ggtt.base.total,
1404                                           0);
1405                 if (err == -ENOSPC) {
1406                         /* maxed out the GGTT space */
1407                         i915_gem_object_put(obj);
1408                         break;
1409                 }
1410                 if (err) {
1411                         pr_err("i915_gem_gtt_insert (pass 1) failed at %llu/%llu with err=%d\n",
1412                                total, i915->ggtt.base.total, err);
1413                         goto out;
1414                 }
1415                 track_vma_bind(vma);
1416                 __i915_vma_pin(vma);
1417
1418                 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
1419         }
1420
1421         list_for_each_entry(obj, &objects, st_link) {
1422                 struct i915_vma *vma;
1423
1424                 vma = i915_vma_instance(obj, &i915->ggtt.base, NULL);
1425                 if (IS_ERR(vma)) {
1426                         err = PTR_ERR(vma);
1427                         goto out;
1428                 }
1429
1430                 if (!drm_mm_node_allocated(&vma->node)) {
1431                         pr_err("VMA was unexpectedly evicted!\n");
1432                         err = -EINVAL;
1433                         goto out;
1434                 }
1435
1436                 __i915_vma_unpin(vma);
1437         }
1438
1439         /* If we then reinsert, we should find the same hole */
1440         list_for_each_entry_safe(obj, on, &objects, st_link) {
1441                 struct i915_vma *vma;
1442                 u64 offset;
1443
1444                 vma = i915_vma_instance(obj, &i915->ggtt.base, NULL);
1445                 if (IS_ERR(vma)) {
1446                         err = PTR_ERR(vma);
1447                         goto out;
1448                 }
1449
1450                 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
1451                 offset = vma->node.start;
1452
1453                 err = i915_vma_unbind(vma);
1454                 if (err) {
1455                         pr_err("i915_vma_unbind failed with err=%d!\n", err);
1456                         goto out;
1457                 }
1458
1459                 err = i915_gem_gtt_insert(&i915->ggtt.base, &vma->node,
1460                                           obj->base.size, 0, obj->cache_level,
1461                                           0, i915->ggtt.base.total,
1462                                           0);
1463                 if (err) {
1464                         pr_err("i915_gem_gtt_insert (pass 2) failed at %llu/%llu with err=%d\n",
1465                                total, i915->ggtt.base.total, err);
1466                         goto out;
1467                 }
1468                 track_vma_bind(vma);
1469
1470                 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
1471                 if (vma->node.start != offset) {
1472                         pr_err("i915_gem_gtt_insert did not return node to its previous location (the only hole), expected address %llx, found %llx\n",
1473                                offset, vma->node.start);
1474                         err = -EINVAL;
1475                         goto out;
1476                 }
1477         }
1478
1479         /* And then force evictions */
1480         for (total = 0;
1481              total + 2*I915_GTT_PAGE_SIZE <= i915->ggtt.base.total;
1482              total += 2*I915_GTT_PAGE_SIZE) {
1483                 struct i915_vma *vma;
1484
1485                 obj = i915_gem_object_create_internal(i915, 2*I915_GTT_PAGE_SIZE);
1486                 if (IS_ERR(obj)) {
1487                         err = PTR_ERR(obj);
1488                         goto out;
1489                 }
1490
1491                 err = i915_gem_object_pin_pages(obj);
1492                 if (err) {
1493                         i915_gem_object_put(obj);
1494                         goto out;
1495                 }
1496
1497                 list_add(&obj->st_link, &objects);
1498
1499                 vma = i915_vma_instance(obj, &i915->ggtt.base, NULL);
1500                 if (IS_ERR(vma)) {
1501                         err = PTR_ERR(vma);
1502                         goto out;
1503                 }
1504
1505                 err = i915_gem_gtt_insert(&i915->ggtt.base, &vma->node,
1506                                           obj->base.size, 0, obj->cache_level,
1507                                           0, i915->ggtt.base.total,
1508                                           0);
1509                 if (err) {
1510                         pr_err("i915_gem_gtt_insert (pass 3) failed at %llu/%llu with err=%d\n",
1511                                total, i915->ggtt.base.total, err);
1512                         goto out;
1513                 }
1514                 track_vma_bind(vma);
1515
1516                 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
1517         }
1518
1519 out:
1520         list_for_each_entry_safe(obj, on, &objects, st_link) {
1521                 i915_gem_object_unpin_pages(obj);
1522                 i915_gem_object_put(obj);
1523         }
1524         return err;
1525 }
1526
1527 int i915_gem_gtt_mock_selftests(void)
1528 {
1529         static const struct i915_subtest tests[] = {
1530                 SUBTEST(igt_mock_drunk),
1531                 SUBTEST(igt_mock_walk),
1532                 SUBTEST(igt_mock_pot),
1533                 SUBTEST(igt_mock_fill),
1534                 SUBTEST(igt_gtt_reserve),
1535                 SUBTEST(igt_gtt_insert),
1536         };
1537         struct drm_i915_private *i915;
1538         int err;
1539
1540         i915 = mock_gem_device();
1541         if (!i915)
1542                 return -ENOMEM;
1543
1544         mutex_lock(&i915->drm.struct_mutex);
1545         err = i915_subtests(tests, i915);
1546         mutex_unlock(&i915->drm.struct_mutex);
1547
1548         drm_dev_unref(&i915->drm);
1549         return err;
1550 }
1551
1552 int i915_gem_gtt_live_selftests(struct drm_i915_private *i915)
1553 {
1554         static const struct i915_subtest tests[] = {
1555                 SUBTEST(igt_ppgtt_alloc),
1556                 SUBTEST(igt_ppgtt_lowlevel),
1557                 SUBTEST(igt_ppgtt_drunk),
1558                 SUBTEST(igt_ppgtt_walk),
1559                 SUBTEST(igt_ppgtt_pot),
1560                 SUBTEST(igt_ppgtt_fill),
1561                 SUBTEST(igt_ppgtt_shrink),
1562                 SUBTEST(igt_ggtt_lowlevel),
1563                 SUBTEST(igt_ggtt_drunk),
1564                 SUBTEST(igt_ggtt_walk),
1565                 SUBTEST(igt_ggtt_pot),
1566                 SUBTEST(igt_ggtt_fill),
1567                 SUBTEST(igt_ggtt_page),
1568         };
1569
1570         GEM_BUG_ON(offset_in_page(i915->ggtt.base.total));
1571
1572         return i915_subtests(tests, i915);
1573 }