drivers/gpu/drm/i915/selftests/i915_gem_gtt.c

   1 /*
   2  * Copyright © 2016 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  */
  24
  25 #include <linux/list_sort.h>
  26 #include <linux/prime_numbers.h>
  27
  28 #include "../i915_selftest.h"
  29 #include "i915_random.h"
  30
  31 #include "mock_context.h"
  32 #include "mock_drm.h"
  33 #include "mock_gem_device.h"
  34
  35 static void fake_free_pages(struct drm_i915_gem_object *obj,
  36                             struct sg_table *pages)
  37 {
  38         sg_free_table(pages);
  39         kfree(pages);
  40 }
  41
  42 static struct sg_table *
  43 fake_get_pages(struct drm_i915_gem_object *obj)
  44 {
  45 #define GFP (GFP_KERNEL | __GFP_NOWARN | __GFP_NORETRY)
  46 #define PFN_BIAS 0x1000
  47         struct sg_table *pages;
  48         struct scatterlist *sg;
  49         typeof(obj->base.size) rem;
  50
  51         pages = kmalloc(sizeof(*pages), GFP);
  52         if (!pages)
  53                 return ERR_PTR(-ENOMEM);
  54
  55         rem = round_up(obj->base.size, BIT(31)) >> 31;
  56         if (sg_alloc_table(pages, rem, GFP)) {
  57                 kfree(pages);
  58                 return ERR_PTR(-ENOMEM);
  59         }
  60
  61         rem = obj->base.size;
  62         for (sg = pages->sgl; sg; sg = sg_next(sg)) {
  63                 unsigned long len = min_t(typeof(rem), rem, BIT(31));
  64
  65                 GEM_BUG_ON(!len);
  66                 sg_set_page(sg, pfn_to_page(PFN_BIAS), len, 0);
  67                 sg_dma_address(sg) = page_to_phys(sg_page(sg));
  68                 sg_dma_len(sg) = len;
  69
  70                 rem -= len;
  71         }
  72         GEM_BUG_ON(rem);
  73
  74         obj->mm.madv = I915_MADV_DONTNEED;
  75         return pages;
  76 #undef GFP
  77 }
  78
  79 static void fake_put_pages(struct drm_i915_gem_object *obj,
  80                            struct sg_table *pages)
  81 {
  82         fake_free_pages(obj, pages);
  83         obj->mm.dirty = false;
  84         obj->mm.madv = I915_MADV_WILLNEED;
  85 }
  86
  87 static const struct drm_i915_gem_object_ops fake_ops = {
  88         .flags = I915_GEM_OBJECT_IS_SHRINKABLE,
  89         .get_pages = fake_get_pages,
  90         .put_pages = fake_put_pages,
  91 };
  92
  93 static struct drm_i915_gem_object *
  94 fake_dma_object(struct drm_i915_private *i915, u64 size)
  95 {
  96         struct drm_i915_gem_object *obj;
  97
  98         GEM_BUG_ON(!size);
  99         GEM_BUG_ON(!IS_ALIGNED(size, I915_GTT_PAGE_SIZE));
 100
 101         if (overflows_type(size, obj->base.size))
 102                 return ERR_PTR(-E2BIG);
 103
 104         obj = i915_gem_object_alloc(i915);
 105         if (!obj)
 106                 goto err;
 107
 108         drm_gem_private_object_init(&i915->drm, &obj->base, size);
 109         i915_gem_object_init(obj, &fake_ops);
 110
 111         obj->base.write_domain = I915_GEM_DOMAIN_CPU;
 112         obj->base.read_domains = I915_GEM_DOMAIN_CPU;
 113         obj->cache_level = I915_CACHE_NONE;
 114
 115         /* Preallocate the "backing storage" */
 116         if (i915_gem_object_pin_pages(obj))
 117                 goto err_obj;
 118
 119         i915_gem_object_unpin_pages(obj);
 120         return obj;
 121
 122 err_obj:
 123         i915_gem_object_put(obj);
 124 err:
 125         return ERR_PTR(-ENOMEM);
 126 }
 127
 128 static int igt_ppgtt_alloc(void *arg)
 129 {
 130         struct drm_i915_private *dev_priv = arg;
 131         struct i915_hw_ppgtt *ppgtt;
 132         u64 size, last;
 133         int err;
 134
 135         /* Allocate a ppggt and try to fill the entire range */
 136
 137         if (!USES_PPGTT(dev_priv))
 138                 return 0;
 139
 140         ppgtt = kzalloc(sizeof(*ppgtt), GFP_KERNEL);
 141         if (!ppgtt)
 142                 return -ENOMEM;
 143
 144         mutex_lock(&dev_priv->drm.struct_mutex);
 145         err = __hw_ppgtt_init(ppgtt, dev_priv);
 146         if (err)
 147                 goto err_ppgtt;
 148
 149         if (!ppgtt->base.allocate_va_range)
 150                 goto err_ppgtt_cleanup;
 151
 152         /* Check we can allocate the entire range */
 153         for (size = 4096;
 154              size <= ppgtt->base.total;
 155              size <<= 2) {
 156                 err = ppgtt->base.allocate_va_range(&ppgtt->base, 0, size);
 157                 if (err) {
 158                         if (err == -ENOMEM) {
 159                                 pr_info("[1] Ran out of memory for va_range [0 + %llx] [bit %d]\n",
 160                                         size, ilog2(size));
 161                                 err = 0; /* virtual space too large! */
 162                         }
 163                         goto err_ppgtt_cleanup;
 164                 }
 165
 166                 ppgtt->base.clear_range(&ppgtt->base, 0, size);
 167         }
 168
 169         /* Check we can incrementally allocate the entire range */
 170         for (last = 0, size = 4096;
 171              size <= ppgtt->base.total;
 172              last = size, size <<= 2) {
 173                 err = ppgtt->base.allocate_va_range(&ppgtt->base,
 174                                                     last, size - last);
 175                 if (err) {
 176                         if (err == -ENOMEM) {
 177                                 pr_info("[2] Ran out of memory for va_range [%llx + %llx] [bit %d]\n",
 178                                         last, size - last, ilog2(size));
 179                                 err = 0; /* virtual space too large! */
 180                         }
 181                         goto err_ppgtt_cleanup;
 182                 }
 183         }
 184
 185 err_ppgtt_cleanup:
 186         ppgtt->base.cleanup(&ppgtt->base);
 187 err_ppgtt:
 188         mutex_unlock(&dev_priv->drm.struct_mutex);
 189         kfree(ppgtt);
 190         return err;
 191 }
 192
 193 static int lowlevel_hole(struct drm_i915_private *i915,
 194                          struct i915_address_space *vm,
 195                          u64 hole_start, u64 hole_end,
 196                          unsigned long end_time)
 197 {
 198         I915_RND_STATE(seed_prng);
 199         unsigned int size;
 200         struct i915_vma mock_vma;
 201
 202         memset(&mock_vma, 0, sizeof(struct i915_vma));
 203
 204         /* Keep creating larger objects until one cannot fit into the hole */
 205         for (size = 12; (hole_end - hole_start) >> size; size++) {
 206                 I915_RND_SUBSTATE(prng, seed_prng);
 207                 struct drm_i915_gem_object *obj;
 208                 unsigned int *order, count, n;
 209                 u64 hole_size;
 210
 211                 hole_size = (hole_end - hole_start) >> size;
 212                 if (hole_size > KMALLOC_MAX_SIZE / sizeof(u32))
 213                         hole_size = KMALLOC_MAX_SIZE / sizeof(u32);
 214                 count = hole_size;
 215                 do {
 216                         count >>= 1;
 217                         order = i915_random_order(count, &prng);
 218                 } while (!order && count);
 219                 if (!order)
 220                         break;
 221
 222                 GEM_BUG_ON(count * BIT_ULL(size) > vm->total);
 223                 GEM_BUG_ON(hole_start + count * BIT_ULL(size) > hole_end);
 224
 225                 /* Ignore allocation failures (i.e. don't report them as
 226                  * a test failure) as we are purposefully allocating very
 227                  * large objects without checking that we have sufficient
 228                  * memory. We expect to hit -ENOMEM.
 229                  */
 230
 231                 obj = fake_dma_object(i915, BIT_ULL(size));
 232                 if (IS_ERR(obj)) {
 233                         kfree(order);
 234                         break;
 235                 }
 236
 237                 GEM_BUG_ON(obj->base.size != BIT_ULL(size));
 238
 239                 if (i915_gem_object_pin_pages(obj)) {
 240                         i915_gem_object_put(obj);
 241                         kfree(order);
 242                         break;
 243                 }
 244
 245                 for (n = 0; n < count; n++) {
 246                         u64 addr = hole_start + order[n] * BIT_ULL(size);
 247
 248                         GEM_BUG_ON(addr + BIT_ULL(size) > vm->total);
 249
 250                         if (igt_timeout(end_time,
 251                                         "%s timed out before %d/%d\n",
 252                                         __func__, n, count)) {
 253                                 hole_end = hole_start; /* quit */
 254                                 break;
 255                         }
 256
 257                         if (vm->allocate_va_range &&
 258                             vm->allocate_va_range(vm, addr, BIT_ULL(size)))
 259                                 break;
 260
 261                         mock_vma.pages = obj->mm.pages;
 262                         mock_vma.node.size = BIT_ULL(size);
 263                         mock_vma.node.start = addr;
 264
 265                         vm->insert_entries(vm, &mock_vma, I915_CACHE_NONE, 0);
 266                 }
 267                 count = n;
 268
 269                 i915_random_reorder(order, count, &prng);
 270                 for (n = 0; n < count; n++) {
 271                         u64 addr = hole_start + order[n] * BIT_ULL(size);
 272
 273                         GEM_BUG_ON(addr + BIT_ULL(size) > vm->total);
 274                         vm->clear_range(vm, addr, BIT_ULL(size));
 275                 }
 276
 277                 i915_gem_object_unpin_pages(obj);
 278                 i915_gem_object_put(obj);
 279
 280                 kfree(order);
 281         }
 282
 283         return 0;
 284 }
 285
 286 static void close_object_list(struct list_head *objects,
 287                               struct i915_address_space *vm)
 288 {
 289         struct drm_i915_gem_object *obj, *on;
 290         int ignored;
 291
 292         list_for_each_entry_safe(obj, on, objects, st_link) {
 293                 struct i915_vma *vma;
 294
 295                 vma = i915_vma_instance(obj, vm, NULL);
 296                 if (!IS_ERR(vma))
 297                         ignored = i915_vma_unbind(vma);
 298                 /* Only ppgtt vma may be closed before the object is freed */
 299                 if (!IS_ERR(vma) && !i915_vma_is_ggtt(vma))
 300                         i915_vma_close(vma);
 301
 302                 list_del(&obj->st_link);
 303                 i915_gem_object_put(obj);
 304         }
 305 }
 306
 307 static int fill_hole(struct drm_i915_private *i915,
 308                      struct i915_address_space *vm,
 309                      u64 hole_start, u64 hole_end,
 310                      unsigned long end_time)
 311 {
 312         const u64 hole_size = hole_end - hole_start;
 313         struct drm_i915_gem_object *obj;
 314         const unsigned long max_pages =
 315                 min_t(u64, ULONG_MAX - 1, hole_size/2 >> PAGE_SHIFT);
 316         const unsigned long max_step = max(int_sqrt(max_pages), 2UL);
 317         unsigned long npages, prime, flags;
 318         struct i915_vma *vma;
 319         LIST_HEAD(objects);
 320         int err;
 321
 322         /* Try binding many VMA working inwards from either edge */
 323
 324         flags = PIN_OFFSET_FIXED | PIN_USER;
 325         if (i915_is_ggtt(vm))
 326                 flags |= PIN_GLOBAL;
 327
 328         for_each_prime_number_from(prime, 2, max_step) {
 329                 for (npages = 1; npages <= max_pages; npages *= prime) {
 330                         const u64 full_size = npages << PAGE_SHIFT;
 331                         const struct {
 332                                 const char *name;
 333                                 u64 offset;
 334                                 int step;
 335                         } phases[] = {
 336                                 { "top-down", hole_end, -1, },
 337                                 { "bottom-up", hole_start, 1, },
 338                                 { }
 339                         }, *p;
 340
 341                         obj = fake_dma_object(i915, full_size);
 342                         if (IS_ERR(obj))
 343                                 break;
 344
 345                         list_add(&obj->st_link, &objects);
 346
 347                         /* Align differing sized objects against the edges, and
 348                          * check we don't walk off into the void when binding
 349                          * them into the GTT.
 350                          */
 351                         for (p = phases; p->name; p++) {
 352                                 u64 offset;
 353
 354                                 offset = p->offset;
 355                                 list_for_each_entry(obj, &objects, st_link) {
 356                                         vma = i915_vma_instance(obj, vm, NULL);
 357                                         if (IS_ERR(vma))
 358                                                 continue;
 359
 360                                         if (p->step < 0) {
 361                                                 if (offset < hole_start + obj->base.size)
 362                                                         break;
 363                                                 offset -= obj->base.size;
 364                                         }
 365
 366                                         err = i915_vma_pin(vma, 0, 0, offset | flags);
 367                                         if (err) {
 368                                                 pr_err("%s(%s) pin (forward) failed with err=%d on size=%lu pages (prime=%lu), offset=%llx\n",
 369                                                        __func__, p->name, err, npages, prime, offset);
 370                                                 goto err;
 371                                         }
 372
 373                                         if (!drm_mm_node_allocated(&vma->node) ||
 374                                             i915_vma_misplaced(vma, 0, 0, offset | flags)) {
 375                                                 pr_err("%s(%s) (forward) insert failed: vma.node=%llx + %llx [allocated? %d], expected offset %llx\n",
 376                                                        __func__, p->name, vma->node.start, vma->node.size, drm_mm_node_allocated(&vma->node),
 377                                                        offset);
 378                                                 err = -EINVAL;
 379                                                 goto err;
 380                                         }
 381
 382                                         i915_vma_unpin(vma);
 383
 384                                         if (p->step > 0) {
 385                                                 if (offset + obj->base.size > hole_end)
 386                                                         break;
 387                                                 offset += obj->base.size;
 388                                         }
 389                                 }
 390
 391                                 offset = p->offset;
 392                                 list_for_each_entry(obj, &objects, st_link) {
 393                                         vma = i915_vma_instance(obj, vm, NULL);
 394                                         if (IS_ERR(vma))
 395                                                 continue;
 396
 397                                         if (p->step < 0) {
 398                                                 if (offset < hole_start + obj->base.size)
 399                                                         break;
 400                                                 offset -= obj->base.size;
 401                                         }
 402
 403                                         if (!drm_mm_node_allocated(&vma->node) ||
 404                                             i915_vma_misplaced(vma, 0, 0, offset | flags)) {
 405                                                 pr_err("%s(%s) (forward) moved vma.node=%llx + %llx, expected offset %llx\n",
 406                                                        __func__, p->name, vma->node.start, vma->node.size,
 407                                                        offset);
 408                                                 err = -EINVAL;
 409                                                 goto err;
 410                                         }
 411
 412                                         err = i915_vma_unbind(vma);
 413                                         if (err) {
 414                                                 pr_err("%s(%s) (forward) unbind of vma.node=%llx + %llx failed with err=%d\n",
 415                                                        __func__, p->name, vma->node.start, vma->node.size,
 416                                                        err);
 417                                                 goto err;
 418                                         }
 419
 420                                         if (p->step > 0) {
 421                                                 if (offset + obj->base.size > hole_end)
 422                                                         break;
 423                                                 offset += obj->base.size;
 424                                         }
 425                                 }
 426
 427                                 offset = p->offset;
 428                                 list_for_each_entry_reverse(obj, &objects, st_link) {
 429                                         vma = i915_vma_instance(obj, vm, NULL);
 430                                         if (IS_ERR(vma))
 431                                                 continue;
 432
 433                                         if (p->step < 0) {
 434                                                 if (offset < hole_start + obj->base.size)
 435                                                         break;
 436                                                 offset -= obj->base.size;
 437                                         }
 438
 439                                         err = i915_vma_pin(vma, 0, 0, offset | flags);
 440                                         if (err) {
 441                                                 pr_err("%s(%s) pin (backward) failed with err=%d on size=%lu pages (prime=%lu), offset=%llx\n",
 442                                                        __func__, p->name, err, npages, prime, offset);
 443                                                 goto err;
 444                                         }
 445
 446                                         if (!drm_mm_node_allocated(&vma->node) ||
 447                                             i915_vma_misplaced(vma, 0, 0, offset | flags)) {
 448                                                 pr_err("%s(%s) (backward) insert failed: vma.node=%llx + %llx [allocated? %d], expected offset %llx\n",
 449                                                        __func__, p->name, vma->node.start, vma->node.size, drm_mm_node_allocated(&vma->node),
 450                                                        offset);
 451                                                 err = -EINVAL;
 452                                                 goto err;
 453                                         }
 454
 455                                         i915_vma_unpin(vma);
 456
 457                                         if (p->step > 0) {
 458                                                 if (offset + obj->base.size > hole_end)
 459                                                         break;
 460                                                 offset += obj->base.size;
 461                                         }
 462                                 }
 463
 464                                 offset = p->offset;
 465                                 list_for_each_entry_reverse(obj, &objects, st_link) {
 466                                         vma = i915_vma_instance(obj, vm, NULL);
 467                                         if (IS_ERR(vma))
 468                                                 continue;
 469
 470                                         if (p->step < 0) {
 471                                                 if (offset < hole_start + obj->base.size)
 472                                                         break;
 473                                                 offset -= obj->base.size;
 474                                         }
 475
 476                                         if (!drm_mm_node_allocated(&vma->node) ||
 477                                             i915_vma_misplaced(vma, 0, 0, offset | flags)) {
 478                                                 pr_err("%s(%s) (backward) moved vma.node=%llx + %llx [allocated? %d], expected offset %llx\n",
 479                                                        __func__, p->name, vma->node.start, vma->node.size, drm_mm_node_allocated(&vma->node),
 480                                                        offset);
 481                                                 err = -EINVAL;
 482                                                 goto err;
 483                                         }
 484
 485                                         err = i915_vma_unbind(vma);
 486                                         if (err) {
 487                                                 pr_err("%s(%s) (backward) unbind of vma.node=%llx + %llx failed with err=%d\n",
 488                                                        __func__, p->name, vma->node.start, vma->node.size,
 489                                                        err);
 490                                                 goto err;
 491                                         }
 492
 493                                         if (p->step > 0) {
 494                                                 if (offset + obj->base.size > hole_end)
 495                                                         break;
 496                                                 offset += obj->base.size;
 497                                         }
 498                                 }
 499                         }
 500
 501                         if (igt_timeout(end_time, "%s timed out (npages=%lu, prime=%lu)\n",
 502                                         __func__, npages, prime)) {
 503                                 err = -EINTR;
 504                                 goto err;
 505                         }
 506                 }
 507
 508                 close_object_list(&objects, vm);
 509         }
 510
 511         return 0;
 512
 513 err:
 514         close_object_list(&objects, vm);
 515         return err;
 516 }
 517
 518 static int walk_hole(struct drm_i915_private *i915,
 519                      struct i915_address_space *vm,
 520                      u64 hole_start, u64 hole_end,
 521                      unsigned long end_time)
 522 {
 523         const u64 hole_size = hole_end - hole_start;
 524         const unsigned long max_pages =
 525                 min_t(u64, ULONG_MAX - 1, hole_size >> PAGE_SHIFT);
 526         unsigned long flags;
 527         u64 size;
 528
 529         /* Try binding a single VMA in different positions within the hole */
 530
 531         flags = PIN_OFFSET_FIXED | PIN_USER;
 532         if (i915_is_ggtt(vm))
 533                 flags |= PIN_GLOBAL;
 534
 535         for_each_prime_number_from(size, 1, max_pages) {
 536                 struct drm_i915_gem_object *obj;
 537                 struct i915_vma *vma;
 538                 u64 addr;
 539                 int err = 0;
 540
 541                 obj = fake_dma_object(i915, size << PAGE_SHIFT);
 542                 if (IS_ERR(obj))
 543                         break;
 544
 545                 vma = i915_vma_instance(obj, vm, NULL);
 546                 if (IS_ERR(vma)) {
 547                         err = PTR_ERR(vma);
 548                         goto err_put;
 549                 }
 550
 551                 for (addr = hole_start;
 552                      addr + obj->base.size < hole_end;
 553                      addr += obj->base.size) {
 554                         err = i915_vma_pin(vma, 0, 0, addr | flags);
 555                         if (err) {
 556                                 pr_err("%s bind failed at %llx + %llx [hole %llx- %llx] with err=%d\n",
 557                                        __func__, addr, vma->size,
 558                                        hole_start, hole_end, err);
 559                                 goto err_close;
 560                         }
 561                         i915_vma_unpin(vma);
 562
 563                         if (!drm_mm_node_allocated(&vma->node) ||
 564                             i915_vma_misplaced(vma, 0, 0, addr | flags)) {
 565                                 pr_err("%s incorrect at %llx + %llx\n",
 566                                        __func__, addr, vma->size);
 567                                 err = -EINVAL;
 568                                 goto err_close;
 569                         }
 570
 571                         err = i915_vma_unbind(vma);
 572                         if (err) {
 573                                 pr_err("%s unbind failed at %llx + %llx  with err=%d\n",
 574                                        __func__, addr, vma->size, err);
 575                                 goto err_close;
 576                         }
 577
 578                         GEM_BUG_ON(drm_mm_node_allocated(&vma->node));
 579
 580                         if (igt_timeout(end_time,
 581                                         "%s timed out at %llx\n",
 582                                         __func__, addr)) {
 583                                 err = -EINTR;
 584                                 goto err_close;
 585                         }
 586                 }
 587
 588 err_close:
 589                 if (!i915_vma_is_ggtt(vma))
 590                         i915_vma_close(vma);
 591 err_put:
 592                 i915_gem_object_put(obj);
 593                 if (err)
 594                         return err;
 595         }
 596
 597         return 0;
 598 }
 599
 600 static int pot_hole(struct drm_i915_private *i915,
 601                     struct i915_address_space *vm,
 602                     u64 hole_start, u64 hole_end,
 603                     unsigned long end_time)
 604 {
 605         struct drm_i915_gem_object *obj;
 606         struct i915_vma *vma;
 607         unsigned long flags;
 608         unsigned int pot;
 609         int err = 0;
 610
 611         flags = PIN_OFFSET_FIXED | PIN_USER;
 612         if (i915_is_ggtt(vm))
 613                 flags |= PIN_GLOBAL;
 614
 615         obj = i915_gem_object_create_internal(i915, 2 * I915_GTT_PAGE_SIZE);
 616         if (IS_ERR(obj))
 617                 return PTR_ERR(obj);
 618
 619         vma = i915_vma_instance(obj, vm, NULL);
 620         if (IS_ERR(vma)) {
 621                 err = PTR_ERR(vma);
 622                 goto err_obj;
 623         }
 624
 625         /* Insert a pair of pages across every pot boundary within the hole */
 626         for (pot = fls64(hole_end - 1) - 1;
 627              pot > ilog2(2 * I915_GTT_PAGE_SIZE);
 628              pot--) {
 629                 u64 step = BIT_ULL(pot);
 630                 u64 addr;
 631
 632                 for (addr = round_up(hole_start + I915_GTT_PAGE_SIZE, step) - I915_GTT_PAGE_SIZE;
 633                      addr <= round_down(hole_end - 2*I915_GTT_PAGE_SIZE, step) - I915_GTT_PAGE_SIZE;
 634                      addr += step) {
 635                         err = i915_vma_pin(vma, 0, 0, addr | flags);
 636                         if (err) {
 637                                 pr_err("%s failed to pin object at %llx in hole [%llx - %llx], with err=%d\n",
 638                                        __func__,
 639                                        addr,
 640                                        hole_start, hole_end,
 641                                        err);
 642                                 goto err;
 643                         }
 644
 645                         if (!drm_mm_node_allocated(&vma->node) ||
 646                             i915_vma_misplaced(vma, 0, 0, addr | flags)) {
 647                                 pr_err("%s incorrect at %llx + %llx\n",
 648                                        __func__, addr, vma->size);
 649                                 i915_vma_unpin(vma);
 650                                 err = i915_vma_unbind(vma);
 651                                 err = -EINVAL;
 652                                 goto err;
 653                         }
 654
 655                         i915_vma_unpin(vma);
 656                         err = i915_vma_unbind(vma);
 657                         GEM_BUG_ON(err);
 658                 }
 659
 660                 if (igt_timeout(end_time,
 661                                 "%s timed out after %d/%d\n",
 662                                 __func__, pot, fls64(hole_end - 1) - 1)) {
 663                         err = -EINTR;
 664                         goto err;
 665                 }
 666         }
 667
 668 err:
 669         if (!i915_vma_is_ggtt(vma))
 670                 i915_vma_close(vma);
 671 err_obj:
 672         i915_gem_object_put(obj);
 673         return err;
 674 }
 675
 676 static int drunk_hole(struct drm_i915_private *i915,
 677                       struct i915_address_space *vm,
 678                       u64 hole_start, u64 hole_end,
 679                       unsigned long end_time)
 680 {
 681         I915_RND_STATE(prng);
 682         unsigned int size;
 683         unsigned long flags;
 684
 685         flags = PIN_OFFSET_FIXED | PIN_USER;
 686         if (i915_is_ggtt(vm))
 687                 flags |= PIN_GLOBAL;
 688
 689         /* Keep creating larger objects until one cannot fit into the hole */
 690         for (size = 12; (hole_end - hole_start) >> size; size++) {
 691                 struct drm_i915_gem_object *obj;
 692                 unsigned int *order, count, n;
 693                 struct i915_vma *vma;
 694                 u64 hole_size;
 695                 int err;
 696
 697                 hole_size = (hole_end - hole_start) >> size;
 698                 if (hole_size > KMALLOC_MAX_SIZE / sizeof(u32))
 699                         hole_size = KMALLOC_MAX_SIZE / sizeof(u32);
 700                 count = hole_size;
 701                 do {
 702                         count >>= 1;
 703                         order = i915_random_order(count, &prng);
 704                 } while (!order && count);
 705                 if (!order)
 706                         break;
 707
 708                 /* Ignore allocation failures (i.e. don't report them as
 709                  * a test failure) as we are purposefully allocating very
 710                  * large objects without checking that we have sufficient
 711                  * memory. We expect to hit -ENOMEM.
 712                  */
 713
 714                 obj = fake_dma_object(i915, BIT_ULL(size));
 715                 if (IS_ERR(obj)) {
 716                         kfree(order);
 717                         break;
 718                 }
 719
 720                 vma = i915_vma_instance(obj, vm, NULL);
 721                 if (IS_ERR(vma)) {
 722                         err = PTR_ERR(vma);
 723                         goto err_obj;
 724                 }
 725
 726                 GEM_BUG_ON(vma->size != BIT_ULL(size));
 727
 728                 for (n = 0; n < count; n++) {
 729                         u64 addr = hole_start + order[n] * BIT_ULL(size);
 730
 731                         err = i915_vma_pin(vma, 0, 0, addr | flags);
 732                         if (err) {
 733                                 pr_err("%s failed to pin object at %llx + %llx in hole [%llx - %llx], with err=%d\n",
 734                                        __func__,
 735                                        addr, BIT_ULL(size),
 736                                        hole_start, hole_end,
 737                                        err);
 738                                 goto err;
 739                         }
 740
 741                         if (!drm_mm_node_allocated(&vma->node) ||
 742                             i915_vma_misplaced(vma, 0, 0, addr | flags)) {
 743                                 pr_err("%s incorrect at %llx + %llx\n",
 744                                        __func__, addr, BIT_ULL(size));
 745                                 i915_vma_unpin(vma);
 746                                 err = i915_vma_unbind(vma);
 747                                 err = -EINVAL;
 748                                 goto err;
 749                         }
 750
 751                         i915_vma_unpin(vma);
 752                         err = i915_vma_unbind(vma);
 753                         GEM_BUG_ON(err);
 754
 755                         if (igt_timeout(end_time,
 756                                         "%s timed out after %d/%d\n",
 757                                         __func__, n, count)) {
 758                                 err = -EINTR;
 759                                 goto err;
 760                         }
 761                 }
 762
 763 err:
 764                 if (!i915_vma_is_ggtt(vma))
 765                         i915_vma_close(vma);
 766 err_obj:
 767                 i915_gem_object_put(obj);
 768                 kfree(order);
 769                 if (err)
 770                         return err;
 771         }
 772
 773         return 0;
 774 }
 775
 776 static int __shrink_hole(struct drm_i915_private *i915,
 777                          struct i915_address_space *vm,
 778                          u64 hole_start, u64 hole_end,
 779                          unsigned long end_time)
 780 {
 781         struct drm_i915_gem_object *obj;
 782         unsigned long flags = PIN_OFFSET_FIXED | PIN_USER;
 783         unsigned int order = 12;
 784         LIST_HEAD(objects);
 785         int err = 0;
 786         u64 addr;
 787
 788         /* Keep creating larger objects until one cannot fit into the hole */
 789         for (addr = hole_start; addr < hole_end; ) {
 790                 struct i915_vma *vma;
 791                 u64 size = BIT_ULL(order++);
 792
 793                 size = min(size, hole_end - addr);
 794                 obj = fake_dma_object(i915, size);
 795                 if (IS_ERR(obj)) {
 796                         err = PTR_ERR(obj);
 797                         break;
 798                 }
 799
 800                 list_add(&obj->st_link, &objects);
 801
 802                 vma = i915_vma_instance(obj, vm, NULL);
 803                 if (IS_ERR(vma)) {
 804                         err = PTR_ERR(vma);
 805                         break;
 806                 }
 807
 808                 GEM_BUG_ON(vma->size != size);
 809
 810                 err = i915_vma_pin(vma, 0, 0, addr | flags);
 811                 if (err) {
 812                         pr_err("%s failed to pin object at %llx + %llx in hole [%llx - %llx], with err=%d\n",
 813                                __func__, addr, size, hole_start, hole_end, err);
 814                         break;
 815                 }
 816
 817                 if (!drm_mm_node_allocated(&vma->node) ||
 818                     i915_vma_misplaced(vma, 0, 0, addr | flags)) {
 819                         pr_err("%s incorrect at %llx + %llx\n",
 820                                __func__, addr, size);
 821                         i915_vma_unpin(vma);
 822                         err = i915_vma_unbind(vma);
 823                         err = -EINVAL;
 824                         break;
 825                 }
 826
 827                 i915_vma_unpin(vma);
 828                 addr += size;
 829
 830                 if (igt_timeout(end_time,
 831                                 "%s timed out at ofset %llx [%llx - %llx]\n",
 832                                 __func__, addr, hole_start, hole_end)) {
 833                         err = -EINTR;
 834                         break;
 835                 }
 836         }
 837
 838         close_object_list(&objects, vm);
 839         return err;
 840 }
 841
 842 static int shrink_hole(struct drm_i915_private *i915,
 843                        struct i915_address_space *vm,
 844                        u64 hole_start, u64 hole_end,
 845                        unsigned long end_time)
 846 {
 847         unsigned long prime;
 848         int err;
 849
 850         vm->fault_attr.probability = 999;
 851         atomic_set(&vm->fault_attr.times, -1);
 852
 853         for_each_prime_number_from(prime, 0, ULONG_MAX - 1) {
 854                 vm->fault_attr.interval = prime;
 855                 err = __shrink_hole(i915, vm, hole_start, hole_end, end_time);
 856                 if (err)
 857                         break;
 858         }
 859
 860         memset(&vm->fault_attr, 0, sizeof(vm->fault_attr));
 861
 862         return err;
 863 }
 864
 865 static int exercise_ppgtt(struct drm_i915_private *dev_priv,
 866                           int (*func)(struct drm_i915_private *i915,
 867                                       struct i915_address_space *vm,
 868                                       u64 hole_start, u64 hole_end,
 869                                       unsigned long end_time))
 870 {
 871         struct drm_file *file;
 872         struct i915_hw_ppgtt *ppgtt;
 873         IGT_TIMEOUT(end_time);
 874         int err;
 875
 876         if (!USES_FULL_PPGTT(dev_priv))
 877                 return 0;
 878
 879         file = mock_file(dev_priv);
 880         if (IS_ERR(file))
 881                 return PTR_ERR(file);
 882
 883         mutex_lock(&dev_priv->drm.struct_mutex);
 884         ppgtt = i915_ppgtt_create(dev_priv, file->driver_priv, "mock");
 885         if (IS_ERR(ppgtt)) {
 886                 err = PTR_ERR(ppgtt);
 887                 goto out_unlock;
 888         }
 889         GEM_BUG_ON(offset_in_page(ppgtt->base.total));
 890         GEM_BUG_ON(ppgtt->base.closed);
 891
 892         err = func(dev_priv, &ppgtt->base, 0, ppgtt->base.total, end_time);
 893
 894         i915_ppgtt_close(&ppgtt->base);
 895         i915_ppgtt_put(ppgtt);
 896 out_unlock:
 897         mutex_unlock(&dev_priv->drm.struct_mutex);
 898
 899         mock_file_free(dev_priv, file);
 900         return err;
 901 }
 902
 903 static int igt_ppgtt_fill(void *arg)
 904 {
 905         return exercise_ppgtt(arg, fill_hole);
 906 }
 907
 908 static int igt_ppgtt_walk(void *arg)
 909 {
 910         return exercise_ppgtt(arg, walk_hole);
 911 }
 912
 913 static int igt_ppgtt_pot(void *arg)
 914 {
 915         return exercise_ppgtt(arg, pot_hole);
 916 }
 917
 918 static int igt_ppgtt_drunk(void *arg)
 919 {
 920         return exercise_ppgtt(arg, drunk_hole);
 921 }
 922
 923 static int igt_ppgtt_lowlevel(void *arg)
 924 {
 925         return exercise_ppgtt(arg, lowlevel_hole);
 926 }
 927
 928 static int igt_ppgtt_shrink(void *arg)
 929 {
 930         return exercise_ppgtt(arg, shrink_hole);
 931 }
 932
 933 static int sort_holes(void *priv, struct list_head *A, struct list_head *B)
 934 {
 935         struct drm_mm_node *a = list_entry(A, typeof(*a), hole_stack);
 936         struct drm_mm_node *b = list_entry(B, typeof(*b), hole_stack);
 937
 938         if (a->start < b->start)
 939                 return -1;
 940         else
 941                 return 1;
 942 }
 943
 944 static int exercise_ggtt(struct drm_i915_private *i915,
 945                          int (*func)(struct drm_i915_private *i915,
 946                                      struct i915_address_space *vm,
 947                                      u64 hole_start, u64 hole_end,
 948                                      unsigned long end_time))
 949 {
 950         struct i915_ggtt *ggtt = &i915->ggtt;
 951         u64 hole_start, hole_end, last = 0;
 952         struct drm_mm_node *node;
 953         IGT_TIMEOUT(end_time);
 954         int err;
 955
 956         mutex_lock(&i915->drm.struct_mutex);
 957 restart:
 958         list_sort(NULL, &ggtt->base.mm.hole_stack, sort_holes);
 959         drm_mm_for_each_hole(node, &ggtt->base.mm, hole_start, hole_end) {
 960                 if (hole_start < last)
 961                         continue;
 962
 963                 if (ggtt->base.mm.color_adjust)
 964                         ggtt->base.mm.color_adjust(node, 0,
 965                                                    &hole_start, &hole_end);
 966                 if (hole_start >= hole_end)
 967                         continue;
 968
 969                 err = func(i915, &ggtt->base, hole_start, hole_end, end_time);
 970                 if (err)
 971                         break;
 972
 973                 /* As we have manipulated the drm_mm, the list may be corrupt */
 974                 last = hole_end;
 975                 goto restart;
 976         }
 977         mutex_unlock(&i915->drm.struct_mutex);
 978
 979         return err;
 980 }
 981
 982 static int igt_ggtt_fill(void *arg)
 983 {
 984         return exercise_ggtt(arg, fill_hole);
 985 }
 986
 987 static int igt_ggtt_walk(void *arg)
 988 {
 989         return exercise_ggtt(arg, walk_hole);
 990 }
 991
 992 static int igt_ggtt_pot(void *arg)
 993 {
 994         return exercise_ggtt(arg, pot_hole);
 995 }
 996
 997 static int igt_ggtt_drunk(void *arg)
 998 {
 999         return exercise_ggtt(arg, drunk_hole);
1000 }
1001
1002 static int igt_ggtt_lowlevel(void *arg)
1003 {
1004         return exercise_ggtt(arg, lowlevel_hole);
1005 }
1006
1007 static int igt_ggtt_page(void *arg)
1008 {
1009         const unsigned int count = PAGE_SIZE/sizeof(u32);
1010         I915_RND_STATE(prng);
1011         struct drm_i915_private *i915 = arg;
1012         struct i915_ggtt *ggtt = &i915->ggtt;
1013         struct drm_i915_gem_object *obj;
1014         struct drm_mm_node tmp;
1015         unsigned int *order, n;
1016         int err;
1017
1018         mutex_lock(&i915->drm.struct_mutex);
1019
1020         obj = i915_gem_object_create_internal(i915, PAGE_SIZE);
1021         if (IS_ERR(obj)) {
1022                 err = PTR_ERR(obj);
1023                 goto out_unlock;
1024         }
1025
1026         err = i915_gem_object_pin_pages(obj);
1027         if (err)
1028                 goto out_free;
1029
1030         memset(&tmp, 0, sizeof(tmp));
1031         err = drm_mm_insert_node_in_range(&ggtt->base.mm, &tmp,
1032                                           1024 * PAGE_SIZE, 0,
1033                                           I915_COLOR_UNEVICTABLE,
1034                                           0, ggtt->mappable_end,
1035                                           DRM_MM_INSERT_LOW);
1036         if (err)
1037                 goto out_unpin;
1038
1039         order = i915_random_order(count, &prng);
1040         if (!order) {
1041                 err = -ENOMEM;
1042                 goto out_remove;
1043         }
1044
1045         for (n = 0; n < count; n++) {
1046                 u64 offset = tmp.start + order[n] * PAGE_SIZE;
1047                 u32 __iomem *vaddr;
1048
1049                 ggtt->base.insert_page(&ggtt->base,
1050                                        i915_gem_object_get_dma_address(obj, 0),
1051                                        offset, I915_CACHE_NONE, 0);
1052
1053                 vaddr = io_mapping_map_atomic_wc(&ggtt->mappable, offset);
1054                 iowrite32(n, vaddr + n);
1055                 io_mapping_unmap_atomic(vaddr);
1056
1057                 wmb();
1058                 ggtt->base.clear_range(&ggtt->base, offset, PAGE_SIZE);
1059         }
1060
1061         i915_random_reorder(order, count, &prng);
1062         for (n = 0; n < count; n++) {
1063                 u64 offset = tmp.start + order[n] * PAGE_SIZE;
1064                 u32 __iomem *vaddr;
1065                 u32 val;
1066
1067                 ggtt->base.insert_page(&ggtt->base,
1068                                        i915_gem_object_get_dma_address(obj, 0),
1069                                        offset, I915_CACHE_NONE, 0);
1070
1071                 vaddr = io_mapping_map_atomic_wc(&ggtt->mappable, offset);
1072                 val = ioread32(vaddr + n);
1073                 io_mapping_unmap_atomic(vaddr);
1074
1075                 ggtt->base.clear_range(&ggtt->base, offset, PAGE_SIZE);
1076
1077                 if (val != n) {
1078                         pr_err("insert page failed: found %d, expected %d\n",
1079                                val, n);
1080                         err = -EINVAL;
1081                         break;
1082                 }
1083         }
1084
1085         kfree(order);
1086 out_remove:
1087         drm_mm_remove_node(&tmp);
1088 out_unpin:
1089         i915_gem_object_unpin_pages(obj);
1090 out_free:
1091         i915_gem_object_put(obj);
1092 out_unlock:
1093         mutex_unlock(&i915->drm.struct_mutex);
1094         return err;
1095 }
1096
1097 static void track_vma_bind(struct i915_vma *vma)
1098 {
1099         struct drm_i915_gem_object *obj = vma->obj;
1100
1101         obj->bind_count++; /* track for eviction later */
1102         __i915_gem_object_pin_pages(obj);
1103
1104         vma->pages = obj->mm.pages;
1105         list_move_tail(&vma->vm_link, &vma->vm->inactive_list);
1106 }
1107
1108 static int exercise_mock(struct drm_i915_private *i915,
1109                          int (*func)(struct drm_i915_private *i915,
1110                                      struct i915_address_space *vm,
1111                                      u64 hole_start, u64 hole_end,
1112                                      unsigned long end_time))
1113 {
1114         struct i915_gem_context *ctx;
1115         struct i915_hw_ppgtt *ppgtt;
1116         IGT_TIMEOUT(end_time);
1117         int err;
1118
1119         ctx = mock_context(i915, "mock");
1120         if (!ctx)
1121                 return -ENOMEM;
1122
1123         ppgtt = ctx->ppgtt;
1124         GEM_BUG_ON(!ppgtt);
1125
1126         err = func(i915, &ppgtt->base, 0, ppgtt->base.total, end_time);
1127
1128         mock_context_close(ctx);
1129         return err;
1130 }
1131
1132 static int igt_mock_fill(void *arg)
1133 {
1134         return exercise_mock(arg, fill_hole);
1135 }
1136
1137 static int igt_mock_walk(void *arg)
1138 {
1139         return exercise_mock(arg, walk_hole);
1140 }
1141
1142 static int igt_mock_pot(void *arg)
1143 {
1144         return exercise_mock(arg, pot_hole);
1145 }
1146
1147 static int igt_mock_drunk(void *arg)
1148 {
1149         return exercise_mock(arg, drunk_hole);
1150 }
1151
1152 static int igt_gtt_reserve(void *arg)
1153 {
1154         struct drm_i915_private *i915 = arg;
1155         struct drm_i915_gem_object *obj, *on;
1156         LIST_HEAD(objects);
1157         u64 total;
1158         int err;
1159
1160         /* i915_gem_gtt_reserve() tries to reserve the precise range
1161          * for the node, and evicts if it has to. So our test checks that
1162          * it can give us the requsted space and prevent overlaps.
1163          */
1164
1165         /* Start by filling the GGTT */
1166         for (total = 0;
1167              total + 2*I915_GTT_PAGE_SIZE <= i915->ggtt.base.total;
1168              total += 2*I915_GTT_PAGE_SIZE) {
1169                 struct i915_vma *vma;
1170
1171                 obj = i915_gem_object_create_internal(i915, 2*PAGE_SIZE);
1172                 if (IS_ERR(obj)) {
1173                         err = PTR_ERR(obj);
1174                         goto out;
1175                 }
1176
1177                 err = i915_gem_object_pin_pages(obj);
1178                 if (err) {
1179                         i915_gem_object_put(obj);
1180                         goto out;
1181                 }
1182
1183                 list_add(&obj->st_link, &objects);
1184
1185                 vma = i915_vma_instance(obj, &i915->ggtt.base, NULL);
1186                 if (IS_ERR(vma)) {
1187                         err = PTR_ERR(vma);
1188                         goto out;
1189                 }
1190
1191                 err = i915_gem_gtt_reserve(&i915->ggtt.base, &vma->node,
1192                                            obj->base.size,
1193                                            total,
1194                                            obj->cache_level,
1195                                            0);
1196                 if (err) {
1197                         pr_err("i915_gem_gtt_reserve (pass 1) failed at %llu/%llu with err=%d\n",
1198                                total, i915->ggtt.base.total, err);
1199                         goto out;
1200                 }
1201                 track_vma_bind(vma);
1202
1203                 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
1204                 if (vma->node.start != total ||
1205                     vma->node.size != 2*I915_GTT_PAGE_SIZE) {
1206                         pr_err("i915_gem_gtt_reserve (pass 1) placement failed, found (%llx + %llx), expected (%llx + %lx)\n",
1207                                vma->node.start, vma->node.size,
1208                                total, 2*I915_GTT_PAGE_SIZE);
1209                         err = -EINVAL;
1210                         goto out;
1211                 }
1212         }
1213
1214         /* Now we start forcing evictions */
1215         for (total = I915_GTT_PAGE_SIZE;
1216              total + 2*I915_GTT_PAGE_SIZE <= i915->ggtt.base.total;
1217              total += 2*I915_GTT_PAGE_SIZE) {
1218                 struct i915_vma *vma;
1219
1220                 obj = i915_gem_object_create_internal(i915, 2*PAGE_SIZE);
1221                 if (IS_ERR(obj)) {
1222                         err = PTR_ERR(obj);
1223                         goto out;
1224                 }
1225
1226                 err = i915_gem_object_pin_pages(obj);
1227                 if (err) {
1228                         i915_gem_object_put(obj);
1229                         goto out;
1230                 }
1231
1232                 list_add(&obj->st_link, &objects);
1233
1234                 vma = i915_vma_instance(obj, &i915->ggtt.base, NULL);
1235                 if (IS_ERR(vma)) {
1236                         err = PTR_ERR(vma);
1237                         goto out;
1238                 }
1239
1240                 err = i915_gem_gtt_reserve(&i915->ggtt.base, &vma->node,
1241                                            obj->base.size,
1242                                            total,
1243                                            obj->cache_level,
1244                                            0);
1245                 if (err) {
1246                         pr_err("i915_gem_gtt_reserve (pass 2) failed at %llu/%llu with err=%d\n",
1247                                total, i915->ggtt.base.total, err);
1248                         goto out;
1249                 }
1250                 track_vma_bind(vma);
1251
1252                 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
1253                 if (vma->node.start != total ||
1254                     vma->node.size != 2*I915_GTT_PAGE_SIZE) {
1255                         pr_err("i915_gem_gtt_reserve (pass 2) placement failed, found (%llx + %llx), expected (%llx + %lx)\n",
1256                                vma->node.start, vma->node.size,
1257                                total, 2*I915_GTT_PAGE_SIZE);
1258                         err = -EINVAL;
1259                         goto out;
1260                 }
1261         }
1262
1263         /* And then try at random */
1264         list_for_each_entry_safe(obj, on, &objects, st_link) {
1265                 struct i915_vma *vma;
1266                 u64 offset;
1267
1268                 vma = i915_vma_instance(obj, &i915->ggtt.base, NULL);
1269                 if (IS_ERR(vma)) {
1270                         err = PTR_ERR(vma);
1271                         goto out;
1272                 }
1273
1274                 err = i915_vma_unbind(vma);
1275                 if (err) {
1276                         pr_err("i915_vma_unbind failed with err=%d!\n", err);
1277                         goto out;
1278                 }
1279
1280                 offset = random_offset(0, i915->ggtt.base.total,
1281                                        2*I915_GTT_PAGE_SIZE,
1282                                        I915_GTT_MIN_ALIGNMENT);
1283
1284                 err = i915_gem_gtt_reserve(&i915->ggtt.base, &vma->node,
1285                                            obj->base.size,
1286                                            offset,
1287                                            obj->cache_level,
1288                                            0);
1289                 if (err) {
1290                         pr_err("i915_gem_gtt_reserve (pass 3) failed at %llu/%llu with err=%d\n",
1291                                total, i915->ggtt.base.total, err);
1292                         goto out;
1293                 }
1294                 track_vma_bind(vma);
1295
1296                 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
1297                 if (vma->node.start != offset ||
1298                     vma->node.size != 2*I915_GTT_PAGE_SIZE) {
1299                         pr_err("i915_gem_gtt_reserve (pass 3) placement failed, found (%llx + %llx), expected (%llx + %lx)\n",
1300                                vma->node.start, vma->node.size,
1301                                offset, 2*I915_GTT_PAGE_SIZE);
1302                         err = -EINVAL;
1303                         goto out;
1304                 }
1305         }
1306
1307 out:
1308         list_for_each_entry_safe(obj, on, &objects, st_link) {
1309                 i915_gem_object_unpin_pages(obj);
1310                 i915_gem_object_put(obj);
1311         }
1312         return err;
1313 }
1314
1315 static int igt_gtt_insert(void *arg)
1316 {
1317         struct drm_i915_private *i915 = arg;
1318         struct drm_i915_gem_object *obj, *on;
1319         struct drm_mm_node tmp = {};
1320         const struct invalid_insert {
1321                 u64 size;
1322                 u64 alignment;
1323                 u64 start, end;
1324         } invalid_insert[] = {
1325                 {
1326                         i915->ggtt.base.total + I915_GTT_PAGE_SIZE, 0,
1327                         0, i915->ggtt.base.total,
1328                 },
1329                 {
1330                         2*I915_GTT_PAGE_SIZE, 0,
1331                         0, I915_GTT_PAGE_SIZE,
1332                 },
1333                 {
1334                         -(u64)I915_GTT_PAGE_SIZE, 0,
1335                         0, 4*I915_GTT_PAGE_SIZE,
1336                 },
1337                 {
1338                         -(u64)2*I915_GTT_PAGE_SIZE, 2*I915_GTT_PAGE_SIZE,
1339                         0, 4*I915_GTT_PAGE_SIZE,
1340                 },
1341                 {
1342                         I915_GTT_PAGE_SIZE, I915_GTT_MIN_ALIGNMENT << 1,
1343                         I915_GTT_MIN_ALIGNMENT, I915_GTT_MIN_ALIGNMENT << 1,
1344                 },
1345                 {}
1346         }, *ii;
1347         LIST_HEAD(objects);
1348         u64 total;
1349         int err;
1350
1351         /* i915_gem_gtt_insert() tries to allocate some free space in the GTT
1352          * to the node, evicting if required.
1353          */
1354
1355         /* Check a couple of obviously invalid requests */
1356         for (ii = invalid_insert; ii->size; ii++) {
1357                 err = i915_gem_gtt_insert(&i915->ggtt.base, &tmp,
1358                                           ii->size, ii->alignment,
1359                                           I915_COLOR_UNEVICTABLE,
1360                                           ii->start, ii->end,
1361                                           0);
1362                 if (err != -ENOSPC) {
1363                         pr_err("Invalid i915_gem_gtt_insert(.size=%llx, .alignment=%llx, .start=%llx, .end=%llx) succeeded (err=%d)\n",
1364                                ii->size, ii->alignment, ii->start, ii->end,
1365                                err);
1366                         return -EINVAL;
1367                 }
1368         }
1369
1370         /* Start by filling the GGTT */
1371         for (total = 0;
1372              total + I915_GTT_PAGE_SIZE <= i915->ggtt.base.total;
1373              total += I915_GTT_PAGE_SIZE) {
1374                 struct i915_vma *vma;
1375
1376                 obj = i915_gem_object_create_internal(i915, I915_GTT_PAGE_SIZE);
1377                 if (IS_ERR(obj)) {
1378                         err = PTR_ERR(obj);
1379                         goto out;
1380                 }
1381
1382                 err = i915_gem_object_pin_pages(obj);
1383                 if (err) {
1384                         i915_gem_object_put(obj);
1385                         goto out;
1386                 }
1387
1388                 list_add(&obj->st_link, &objects);
1389
1390                 vma = i915_vma_instance(obj, &i915->ggtt.base, NULL);
1391                 if (IS_ERR(vma)) {
1392                         err = PTR_ERR(vma);
1393                         goto out;
1394                 }
1395
1396                 err = i915_gem_gtt_insert(&i915->ggtt.base, &vma->node,
1397                                           obj->base.size, 0, obj->cache_level,
1398                                           0, i915->ggtt.base.total,
1399                                           0);
1400                 if (err == -ENOSPC) {
1401                         /* maxed out the GGTT space */
1402                         i915_gem_object_put(obj);
1403                         break;
1404                 }
1405                 if (err) {
1406                         pr_err("i915_gem_gtt_insert (pass 1) failed at %llu/%llu with err=%d\n",
1407                                total, i915->ggtt.base.total, err);
1408                         goto out;
1409                 }
1410                 track_vma_bind(vma);
1411                 __i915_vma_pin(vma);
1412
1413                 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
1414         }
1415
1416         list_for_each_entry(obj, &objects, st_link) {
1417                 struct i915_vma *vma;
1418
1419                 vma = i915_vma_instance(obj, &i915->ggtt.base, NULL);
1420                 if (IS_ERR(vma)) {
1421                         err = PTR_ERR(vma);
1422                         goto out;
1423                 }
1424
1425                 if (!drm_mm_node_allocated(&vma->node)) {
1426                         pr_err("VMA was unexpectedly evicted!\n");
1427                         err = -EINVAL;
1428                         goto out;
1429                 }
1430
1431                 __i915_vma_unpin(vma);
1432         }
1433
1434         /* If we then reinsert, we should find the same hole */
1435         list_for_each_entry_safe(obj, on, &objects, st_link) {
1436                 struct i915_vma *vma;
1437                 u64 offset;
1438
1439                 vma = i915_vma_instance(obj, &i915->ggtt.base, NULL);
1440                 if (IS_ERR(vma)) {
1441                         err = PTR_ERR(vma);
1442                         goto out;
1443                 }
1444
1445                 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
1446                 offset = vma->node.start;
1447
1448                 err = i915_vma_unbind(vma);
1449                 if (err) {
1450                         pr_err("i915_vma_unbind failed with err=%d!\n", err);
1451                         goto out;
1452                 }
1453
1454                 err = i915_gem_gtt_insert(&i915->ggtt.base, &vma->node,
1455                                           obj->base.size, 0, obj->cache_level,
1456                                           0, i915->ggtt.base.total,
1457                                           0);
1458                 if (err) {
1459                         pr_err("i915_gem_gtt_insert (pass 2) failed at %llu/%llu with err=%d\n",
1460                                total, i915->ggtt.base.total, err);
1461                         goto out;
1462                 }
1463                 track_vma_bind(vma);
1464
1465                 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
1466                 if (vma->node.start != offset) {
1467                         pr_err("i915_gem_gtt_insert did not return node to its previous location (the only hole), expected address %llx, found %llx\n",
1468                                offset, vma->node.start);
1469                         err = -EINVAL;
1470                         goto out;
1471                 }
1472         }
1473
1474         /* And then force evictions */
1475         for (total = 0;
1476              total + 2*I915_GTT_PAGE_SIZE <= i915->ggtt.base.total;
1477              total += 2*I915_GTT_PAGE_SIZE) {
1478                 struct i915_vma *vma;
1479
1480                 obj = i915_gem_object_create_internal(i915, 2*I915_GTT_PAGE_SIZE);
1481                 if (IS_ERR(obj)) {
1482                         err = PTR_ERR(obj);
1483                         goto out;
1484                 }
1485
1486                 err = i915_gem_object_pin_pages(obj);
1487                 if (err) {
1488                         i915_gem_object_put(obj);
1489                         goto out;
1490                 }
1491
1492                 list_add(&obj->st_link, &objects);
1493
1494                 vma = i915_vma_instance(obj, &i915->ggtt.base, NULL);
1495                 if (IS_ERR(vma)) {
1496                         err = PTR_ERR(vma);
1497                         goto out;
1498                 }
1499
1500                 err = i915_gem_gtt_insert(&i915->ggtt.base, &vma->node,
1501                                           obj->base.size, 0, obj->cache_level,
1502                                           0, i915->ggtt.base.total,
1503                                           0);
1504                 if (err) {
1505                         pr_err("i915_gem_gtt_insert (pass 3) failed at %llu/%llu with err=%d\n",
1506                                total, i915->ggtt.base.total, err);
1507                         goto out;
1508                 }
1509                 track_vma_bind(vma);
1510
1511                 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
1512         }
1513
1514 out:
1515         list_for_each_entry_safe(obj, on, &objects, st_link) {
1516                 i915_gem_object_unpin_pages(obj);
1517                 i915_gem_object_put(obj);
1518         }
1519         return err;
1520 }
1521
1522 int i915_gem_gtt_mock_selftests(void)
1523 {
1524         static const struct i915_subtest tests[] = {
1525                 SUBTEST(igt_mock_drunk),
1526                 SUBTEST(igt_mock_walk),
1527                 SUBTEST(igt_mock_pot),
1528                 SUBTEST(igt_mock_fill),
1529                 SUBTEST(igt_gtt_reserve),
1530                 SUBTEST(igt_gtt_insert),
1531         };
1532         struct drm_i915_private *i915;
1533         int err;
1534
1535         i915 = mock_gem_device();
1536         if (!i915)
1537                 return -ENOMEM;
1538
1539         mutex_lock(&i915->drm.struct_mutex);
1540         err = i915_subtests(tests, i915);
1541         mutex_unlock(&i915->drm.struct_mutex);
1542
1543         drm_dev_unref(&i915->drm);
1544         return err;
1545 }
1546
1547 int i915_gem_gtt_live_selftests(struct drm_i915_private *i915)
1548 {
1549         static const struct i915_subtest tests[] = {
1550                 SUBTEST(igt_ppgtt_alloc),
1551                 SUBTEST(igt_ppgtt_lowlevel),
1552                 SUBTEST(igt_ppgtt_drunk),
1553                 SUBTEST(igt_ppgtt_walk),
1554                 SUBTEST(igt_ppgtt_pot),
1555                 SUBTEST(igt_ppgtt_fill),
1556                 SUBTEST(igt_ppgtt_shrink),
1557                 SUBTEST(igt_ggtt_lowlevel),
1558                 SUBTEST(igt_ggtt_drunk),
1559                 SUBTEST(igt_ggtt_walk),
1560                 SUBTEST(igt_ggtt_pot),
1561                 SUBTEST(igt_ggtt_fill),
1562                 SUBTEST(igt_ggtt_page),
1563         };
1564
1565         GEM_BUG_ON(offset_in_page(i915->ggtt.base.total));
1566
1567         return i915_subtests(tests, i915);
1568 }