drivers/gpu/drm/i915/selftests/huge_pages.c

   1 /*
   2  * Copyright © 2017 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  */
  24
  25 #include "../i915_selftest.h"
  26
  27 #include <linux/prime_numbers.h>
  28
  29 #include "mock_drm.h"
  30 #include "i915_random.h"
  31
  32 static const unsigned int page_sizes[] = {
  33         I915_GTT_PAGE_SIZE_2M,
  34         I915_GTT_PAGE_SIZE_64K,
  35         I915_GTT_PAGE_SIZE_4K,
  36 };
  37
  38 static unsigned int get_largest_page_size(struct drm_i915_private *i915,
  39                                           u64 rem)
  40 {
  41         int i;
  42
  43         for (i = 0; i < ARRAY_SIZE(page_sizes); ++i) {
  44                 unsigned int page_size = page_sizes[i];
  45
  46                 if (HAS_PAGE_SIZES(i915, page_size) && rem >= page_size)
  47                         return page_size;
  48         }
  49
  50         return 0;
  51 }
  52
  53 static void huge_pages_free_pages(struct sg_table *st)
  54 {
  55         struct scatterlist *sg;
  56
  57         for (sg = st->sgl; sg; sg = __sg_next(sg)) {
  58                 if (sg_page(sg))
  59                         __free_pages(sg_page(sg), get_order(sg->length));
  60         }
  61
  62         sg_free_table(st);
  63         kfree(st);
  64 }
  65
  66 static int get_huge_pages(struct drm_i915_gem_object *obj)
  67 {
  68 #define GFP (GFP_KERNEL | __GFP_NOWARN | __GFP_NORETRY)
  69         unsigned int page_mask = obj->mm.page_mask;
  70         struct sg_table *st;
  71         struct scatterlist *sg;
  72         unsigned int sg_page_sizes;
  73         u64 rem;
  74
  75         st = kmalloc(sizeof(*st), GFP);
  76         if (!st)
  77                 return -ENOMEM;
  78
  79         if (sg_alloc_table(st, obj->base.size >> PAGE_SHIFT, GFP)) {
  80                 kfree(st);
  81                 return -ENOMEM;
  82         }
  83
  84         rem = obj->base.size;
  85         sg = st->sgl;
  86         st->nents = 0;
  87         sg_page_sizes = 0;
  88
  89         /*
  90          * Our goal here is simple, we want to greedily fill the object from
  91          * largest to smallest page-size, while ensuring that we use *every*
  92          * page-size as per the given page-mask.
  93          */
  94         do {
  95                 unsigned int bit = ilog2(page_mask);
  96                 unsigned int page_size = BIT(bit);
  97                 int order = get_order(page_size);
  98
  99                 do {
 100                         struct page *page;
 101
 102                         GEM_BUG_ON(order >= MAX_ORDER);
 103                         page = alloc_pages(GFP | __GFP_ZERO, order);
 104                         if (!page)
 105                                 goto err;
 106
 107                         sg_set_page(sg, page, page_size, 0);
 108                         sg_page_sizes |= page_size;
 109                         st->nents++;
 110
 111                         rem -= page_size;
 112                         if (!rem) {
 113                                 sg_mark_end(sg);
 114                                 break;
 115                         }
 116
 117                         sg = __sg_next(sg);
 118                 } while ((rem - ((page_size-1) & page_mask)) >= page_size);
 119
 120                 page_mask &= (page_size-1);
 121         } while (page_mask);
 122
 123         if (i915_gem_gtt_prepare_pages(obj, st))
 124                 goto err;
 125
 126         obj->mm.madv = I915_MADV_DONTNEED;
 127
 128         GEM_BUG_ON(sg_page_sizes != obj->mm.page_mask);
 129         __i915_gem_object_set_pages(obj, st, sg_page_sizes);
 130
 131         return 0;
 132
 133 err:
 134         sg_set_page(sg, NULL, 0, 0);
 135         sg_mark_end(sg);
 136         huge_pages_free_pages(st);
 137
 138         return -ENOMEM;
 139 }
 140
 141 static void put_huge_pages(struct drm_i915_gem_object *obj,
 142                            struct sg_table *pages)
 143 {
 144         i915_gem_gtt_finish_pages(obj, pages);
 145         huge_pages_free_pages(pages);
 146
 147         obj->mm.dirty = false;
 148         obj->mm.madv = I915_MADV_WILLNEED;
 149 }
 150
 151 static const struct drm_i915_gem_object_ops huge_page_ops = {
 152         .flags = I915_GEM_OBJECT_HAS_STRUCT_PAGE |
 153                  I915_GEM_OBJECT_IS_SHRINKABLE,
 154         .get_pages = get_huge_pages,
 155         .put_pages = put_huge_pages,
 156 };
 157
 158 static struct drm_i915_gem_object *
 159 huge_pages_object(struct drm_i915_private *i915,
 160                   u64 size,
 161                   unsigned int page_mask)
 162 {
 163         struct drm_i915_gem_object *obj;
 164
 165         GEM_BUG_ON(!size);
 166         GEM_BUG_ON(!IS_ALIGNED(size, BIT(__ffs(page_mask))));
 167
 168         if (size >> PAGE_SHIFT > INT_MAX)
 169                 return ERR_PTR(-E2BIG);
 170
 171         if (overflows_type(size, obj->base.size))
 172                 return ERR_PTR(-E2BIG);
 173
 174         obj = i915_gem_object_alloc(i915);
 175         if (!obj)
 176                 return ERR_PTR(-ENOMEM);
 177
 178         drm_gem_private_object_init(&i915->drm, &obj->base, size);
 179         i915_gem_object_init(obj, &huge_page_ops);
 180
 181         obj->write_domain = I915_GEM_DOMAIN_CPU;
 182         obj->read_domains = I915_GEM_DOMAIN_CPU;
 183         obj->cache_level = I915_CACHE_NONE;
 184
 185         obj->mm.page_mask = page_mask;
 186
 187         return obj;
 188 }
 189
 190 static int fake_get_huge_pages(struct drm_i915_gem_object *obj)
 191 {
 192         struct drm_i915_private *i915 = to_i915(obj->base.dev);
 193         const u64 max_len = rounddown_pow_of_two(UINT_MAX);
 194         struct sg_table *st;
 195         struct scatterlist *sg;
 196         unsigned int sg_page_sizes;
 197         u64 rem;
 198
 199         st = kmalloc(sizeof(*st), GFP);
 200         if (!st)
 201                 return -ENOMEM;
 202
 203         if (sg_alloc_table(st, obj->base.size >> PAGE_SHIFT, GFP)) {
 204                 kfree(st);
 205                 return -ENOMEM;
 206         }
 207
 208         /* Use optimal page sized chunks to fill in the sg table */
 209         rem = obj->base.size;
 210         sg = st->sgl;
 211         st->nents = 0;
 212         sg_page_sizes = 0;
 213         do {
 214                 unsigned int page_size = get_largest_page_size(i915, rem);
 215                 unsigned int len = min(page_size * div_u64(rem, page_size),
 216                                        max_len);
 217
 218                 GEM_BUG_ON(!page_size);
 219
 220                 sg->offset = 0;
 221                 sg->length = len;
 222                 sg_dma_len(sg) = len;
 223                 sg_dma_address(sg) = page_size;
 224
 225                 sg_page_sizes |= len;
 226
 227                 st->nents++;
 228
 229                 rem -= len;
 230                 if (!rem) {
 231                         sg_mark_end(sg);
 232                         break;
 233                 }
 234
 235                 sg = sg_next(sg);
 236         } while (1);
 237
 238         i915_sg_trim(st);
 239
 240         obj->mm.madv = I915_MADV_DONTNEED;
 241
 242         __i915_gem_object_set_pages(obj, st, sg_page_sizes);
 243
 244         return 0;
 245 }
 246
 247 static int fake_get_huge_pages_single(struct drm_i915_gem_object *obj)
 248 {
 249         struct drm_i915_private *i915 = to_i915(obj->base.dev);
 250         struct sg_table *st;
 251         struct scatterlist *sg;
 252         unsigned int page_size;
 253
 254         st = kmalloc(sizeof(*st), GFP);
 255         if (!st)
 256                 return -ENOMEM;
 257
 258         if (sg_alloc_table(st, 1, GFP)) {
 259                 kfree(st);
 260                 return -ENOMEM;
 261         }
 262
 263         sg = st->sgl;
 264         st->nents = 1;
 265
 266         page_size = get_largest_page_size(i915, obj->base.size);
 267         GEM_BUG_ON(!page_size);
 268
 269         sg->offset = 0;
 270         sg->length = obj->base.size;
 271         sg_dma_len(sg) = obj->base.size;
 272         sg_dma_address(sg) = page_size;
 273
 274         obj->mm.madv = I915_MADV_DONTNEED;
 275
 276         __i915_gem_object_set_pages(obj, st, sg->length);
 277
 278         return 0;
 279 #undef GFP
 280 }
 281
 282 static void fake_free_huge_pages(struct drm_i915_gem_object *obj,
 283                                  struct sg_table *pages)
 284 {
 285         sg_free_table(pages);
 286         kfree(pages);
 287 }
 288
 289 static void fake_put_huge_pages(struct drm_i915_gem_object *obj,
 290                                 struct sg_table *pages)
 291 {
 292         fake_free_huge_pages(obj, pages);
 293         obj->mm.dirty = false;
 294         obj->mm.madv = I915_MADV_WILLNEED;
 295 }
 296
 297 static const struct drm_i915_gem_object_ops fake_ops = {
 298         .flags = I915_GEM_OBJECT_IS_SHRINKABLE,
 299         .get_pages = fake_get_huge_pages,
 300         .put_pages = fake_put_huge_pages,
 301 };
 302
 303 static const struct drm_i915_gem_object_ops fake_ops_single = {
 304         .flags = I915_GEM_OBJECT_IS_SHRINKABLE,
 305         .get_pages = fake_get_huge_pages_single,
 306         .put_pages = fake_put_huge_pages,
 307 };
 308
 309 static struct drm_i915_gem_object *
 310 fake_huge_pages_object(struct drm_i915_private *i915, u64 size, bool single)
 311 {
 312         struct drm_i915_gem_object *obj;
 313
 314         GEM_BUG_ON(!size);
 315         GEM_BUG_ON(!IS_ALIGNED(size, I915_GTT_PAGE_SIZE));
 316
 317         if (size >> PAGE_SHIFT > UINT_MAX)
 318                 return ERR_PTR(-E2BIG);
 319
 320         if (overflows_type(size, obj->base.size))
 321                 return ERR_PTR(-E2BIG);
 322
 323         obj = i915_gem_object_alloc(i915);
 324         if (!obj)
 325                 return ERR_PTR(-ENOMEM);
 326
 327         drm_gem_private_object_init(&i915->drm, &obj->base, size);
 328
 329         if (single)
 330                 i915_gem_object_init(obj, &fake_ops_single);
 331         else
 332                 i915_gem_object_init(obj, &fake_ops);
 333
 334         obj->write_domain = I915_GEM_DOMAIN_CPU;
 335         obj->read_domains = I915_GEM_DOMAIN_CPU;
 336         obj->cache_level = I915_CACHE_NONE;
 337
 338         return obj;
 339 }
 340
 341 static int igt_check_page_sizes(struct i915_vma *vma)
 342 {
 343         struct drm_i915_private *i915 = vma->vm->i915;
 344         unsigned int supported = INTEL_INFO(i915)->page_sizes;
 345         struct drm_i915_gem_object *obj = vma->obj;
 346         int err = 0;
 347
 348         if (!HAS_PAGE_SIZES(i915, vma->page_sizes.sg)) {
 349                 pr_err("unsupported page_sizes.sg=%u, supported=%u\n",
 350                        vma->page_sizes.sg & ~supported, supported);
 351                 err = -EINVAL;
 352         }
 353
 354         if (!HAS_PAGE_SIZES(i915, vma->page_sizes.gtt)) {
 355                 pr_err("unsupported page_sizes.gtt=%u, supported=%u\n",
 356                        vma->page_sizes.gtt & ~supported, supported);
 357                 err = -EINVAL;
 358         }
 359
 360         if (vma->page_sizes.phys != obj->mm.page_sizes.phys) {
 361                 pr_err("vma->page_sizes.phys(%u) != obj->mm.page_sizes.phys(%u)\n",
 362                        vma->page_sizes.phys, obj->mm.page_sizes.phys);
 363                 err = -EINVAL;
 364         }
 365
 366         if (vma->page_sizes.sg != obj->mm.page_sizes.sg) {
 367                 pr_err("vma->page_sizes.sg(%u) != obj->mm.page_sizes.sg(%u)\n",
 368                        vma->page_sizes.sg, obj->mm.page_sizes.sg);
 369                 err = -EINVAL;
 370         }
 371
 372         if (obj->mm.page_sizes.gtt) {
 373                 pr_err("obj->page_sizes.gtt(%u) should never be set\n",
 374                        obj->mm.page_sizes.gtt);
 375                 err = -EINVAL;
 376         }
 377
 378         return err;
 379 }
 380
 381 static int igt_mock_exhaust_device_supported_pages(void *arg)
 382 {
 383         struct i915_hw_ppgtt *ppgtt = arg;
 384         struct drm_i915_private *i915 = ppgtt->vm.i915;
 385         unsigned int saved_mask = INTEL_INFO(i915)->page_sizes;
 386         struct drm_i915_gem_object *obj;
 387         struct i915_vma *vma;
 388         int i, j, single;
 389         int err;
 390
 391         /*
 392          * Sanity check creating objects with every valid page support
 393          * combination for our mock device.
 394          */
 395
 396         for (i = 1; i < BIT(ARRAY_SIZE(page_sizes)); i++) {
 397                 unsigned int combination = 0;
 398
 399                 for (j = 0; j < ARRAY_SIZE(page_sizes); j++) {
 400                         if (i & BIT(j))
 401                                 combination |= page_sizes[j];
 402                 }
 403
 404                 mkwrite_device_info(i915)->page_sizes = combination;
 405
 406                 for (single = 0; single <= 1; ++single) {
 407                         obj = fake_huge_pages_object(i915, combination, !!single);
 408                         if (IS_ERR(obj)) {
 409                                 err = PTR_ERR(obj);
 410                                 goto out_device;
 411                         }
 412
 413                         if (obj->base.size != combination) {
 414                                 pr_err("obj->base.size=%zu, expected=%u\n",
 415                                        obj->base.size, combination);
 416                                 err = -EINVAL;
 417                                 goto out_put;
 418                         }
 419
 420                         vma = i915_vma_instance(obj, &ppgtt->vm, NULL);
 421                         if (IS_ERR(vma)) {
 422                                 err = PTR_ERR(vma);
 423                                 goto out_put;
 424                         }
 425
 426                         err = i915_vma_pin(vma, 0, 0, PIN_USER);
 427                         if (err)
 428                                 goto out_close;
 429
 430                         err = igt_check_page_sizes(vma);
 431
 432                         if (vma->page_sizes.sg != combination) {
 433                                 pr_err("page_sizes.sg=%u, expected=%u\n",
 434                                        vma->page_sizes.sg, combination);
 435                                 err = -EINVAL;
 436                         }
 437
 438                         i915_vma_unpin(vma);
 439                         i915_vma_close(vma);
 440
 441                         i915_gem_object_put(obj);
 442
 443                         if (err)
 444                                 goto out_device;
 445                 }
 446         }
 447
 448         goto out_device;
 449
 450 out_close:
 451         i915_vma_close(vma);
 452 out_put:
 453         i915_gem_object_put(obj);
 454 out_device:
 455         mkwrite_device_info(i915)->page_sizes = saved_mask;
 456
 457         return err;
 458 }
 459
 460 static int igt_mock_ppgtt_misaligned_dma(void *arg)
 461 {
 462         struct i915_hw_ppgtt *ppgtt = arg;
 463         struct drm_i915_private *i915 = ppgtt->vm.i915;
 464         unsigned long supported = INTEL_INFO(i915)->page_sizes;
 465         struct drm_i915_gem_object *obj;
 466         int bit;
 467         int err;
 468
 469         /*
 470          * Sanity check dma misalignment for huge pages -- the dma addresses we
 471          * insert into the paging structures need to always respect the page
 472          * size alignment.
 473          */
 474
 475         bit = ilog2(I915_GTT_PAGE_SIZE_64K);
 476
 477         for_each_set_bit_from(bit, &supported,
 478                               ilog2(I915_GTT_MAX_PAGE_SIZE) + 1) {
 479                 IGT_TIMEOUT(end_time);
 480                 unsigned int page_size = BIT(bit);
 481                 unsigned int flags = PIN_USER | PIN_OFFSET_FIXED;
 482                 unsigned int offset;
 483                 unsigned int size =
 484                         round_up(page_size, I915_GTT_PAGE_SIZE_2M) << 1;
 485                 struct i915_vma *vma;
 486
 487                 obj = fake_huge_pages_object(i915, size, true);
 488                 if (IS_ERR(obj))
 489                         return PTR_ERR(obj);
 490
 491                 if (obj->base.size != size) {
 492                         pr_err("obj->base.size=%zu, expected=%u\n",
 493                                obj->base.size, size);
 494                         err = -EINVAL;
 495                         goto out_put;
 496                 }
 497
 498                 err = i915_gem_object_pin_pages(obj);
 499                 if (err)
 500                         goto out_put;
 501
 502                 /* Force the page size for this object */
 503                 obj->mm.page_sizes.sg = page_size;
 504
 505                 vma = i915_vma_instance(obj, &ppgtt->vm, NULL);
 506                 if (IS_ERR(vma)) {
 507                         err = PTR_ERR(vma);
 508                         goto out_unpin;
 509                 }
 510
 511                 err = i915_vma_pin(vma, 0, 0, flags);
 512                 if (err) {
 513                         i915_vma_close(vma);
 514                         goto out_unpin;
 515                 }
 516
 517
 518                 err = igt_check_page_sizes(vma);
 519
 520                 if (vma->page_sizes.gtt != page_size) {
 521                         pr_err("page_sizes.gtt=%u, expected %u\n",
 522                                vma->page_sizes.gtt, page_size);
 523                         err = -EINVAL;
 524                 }
 525
 526                 i915_vma_unpin(vma);
 527
 528                 if (err) {
 529                         i915_vma_close(vma);
 530                         goto out_unpin;
 531                 }
 532
 533                 /*
 534                  * Try all the other valid offsets until the next
 535                  * boundary -- should always fall back to using 4K
 536                  * pages.
 537                  */
 538                 for (offset = 4096; offset < page_size; offset += 4096) {
 539                         err = i915_vma_unbind(vma);
 540                         if (err) {
 541                                 i915_vma_close(vma);
 542                                 goto out_unpin;
 543                         }
 544
 545                         err = i915_vma_pin(vma, 0, 0, flags | offset);
 546                         if (err) {
 547                                 i915_vma_close(vma);
 548                                 goto out_unpin;
 549                         }
 550
 551                         err = igt_check_page_sizes(vma);
 552
 553                         if (vma->page_sizes.gtt != I915_GTT_PAGE_SIZE_4K) {
 554                                 pr_err("page_sizes.gtt=%u, expected %llu\n",
 555                                        vma->page_sizes.gtt, I915_GTT_PAGE_SIZE_4K);
 556                                 err = -EINVAL;
 557                         }
 558
 559                         i915_vma_unpin(vma);
 560
 561                         if (err) {
 562                                 i915_vma_close(vma);
 563                                 goto out_unpin;
 564                         }
 565
 566                         if (igt_timeout(end_time,
 567                                         "%s timed out at offset %x with page-size %x\n",
 568                                         __func__, offset, page_size))
 569                                 break;
 570                 }
 571
 572                 i915_vma_close(vma);
 573
 574                 i915_gem_object_unpin_pages(obj);
 575                 __i915_gem_object_put_pages(obj, I915_MM_NORMAL);
 576                 i915_gem_object_put(obj);
 577         }
 578
 579         return 0;
 580
 581 out_unpin:
 582         i915_gem_object_unpin_pages(obj);
 583 out_put:
 584         i915_gem_object_put(obj);
 585
 586         return err;
 587 }
 588
 589 static void close_object_list(struct list_head *objects,
 590                               struct i915_hw_ppgtt *ppgtt)
 591 {
 592         struct drm_i915_gem_object *obj, *on;
 593
 594         list_for_each_entry_safe(obj, on, objects, st_link) {
 595                 struct i915_vma *vma;
 596
 597                 vma = i915_vma_instance(obj, &ppgtt->vm, NULL);
 598                 if (!IS_ERR(vma))
 599                         i915_vma_close(vma);
 600
 601                 list_del(&obj->st_link);
 602                 i915_gem_object_unpin_pages(obj);
 603                 __i915_gem_object_put_pages(obj, I915_MM_NORMAL);
 604                 i915_gem_object_put(obj);
 605         }
 606 }
 607
 608 static int igt_mock_ppgtt_huge_fill(void *arg)
 609 {
 610         struct i915_hw_ppgtt *ppgtt = arg;
 611         struct drm_i915_private *i915 = ppgtt->vm.i915;
 612         unsigned long max_pages = ppgtt->vm.total >> PAGE_SHIFT;
 613         unsigned long page_num;
 614         bool single = false;
 615         LIST_HEAD(objects);
 616         IGT_TIMEOUT(end_time);
 617         int err = -ENODEV;
 618
 619         for_each_prime_number_from(page_num, 1, max_pages) {
 620                 struct drm_i915_gem_object *obj;
 621                 u64 size = page_num << PAGE_SHIFT;
 622                 struct i915_vma *vma;
 623                 unsigned int expected_gtt = 0;
 624                 int i;
 625
 626                 obj = fake_huge_pages_object(i915, size, single);
 627                 if (IS_ERR(obj)) {
 628                         err = PTR_ERR(obj);
 629                         break;
 630                 }
 631
 632                 if (obj->base.size != size) {
 633                         pr_err("obj->base.size=%zd, expected=%llu\n",
 634                                obj->base.size, size);
 635                         i915_gem_object_put(obj);
 636                         err = -EINVAL;
 637                         break;
 638                 }
 639
 640                 err = i915_gem_object_pin_pages(obj);
 641                 if (err) {
 642                         i915_gem_object_put(obj);
 643                         break;
 644                 }
 645
 646                 list_add(&obj->st_link, &objects);
 647
 648                 vma = i915_vma_instance(obj, &ppgtt->vm, NULL);
 649                 if (IS_ERR(vma)) {
 650                         err = PTR_ERR(vma);
 651                         break;
 652                 }
 653
 654                 err = i915_vma_pin(vma, 0, 0, PIN_USER);
 655                 if (err)
 656                         break;
 657
 658                 err = igt_check_page_sizes(vma);
 659                 if (err) {
 660                         i915_vma_unpin(vma);
 661                         break;
 662                 }
 663
 664                 /*
 665                  * Figure out the expected gtt page size knowing that we go from
 666                  * largest to smallest page size sg chunks, and that we align to
 667                  * the largest page size.
 668                  */
 669                 for (i = 0; i < ARRAY_SIZE(page_sizes); ++i) {
 670                         unsigned int page_size = page_sizes[i];
 671
 672                         if (HAS_PAGE_SIZES(i915, page_size) &&
 673                             size >= page_size) {
 674                                 expected_gtt |= page_size;
 675                                 size &= page_size-1;
 676                         }
 677                 }
 678
 679                 GEM_BUG_ON(!expected_gtt);
 680                 GEM_BUG_ON(size);
 681
 682                 if (expected_gtt & I915_GTT_PAGE_SIZE_4K)
 683                         expected_gtt &= ~I915_GTT_PAGE_SIZE_64K;
 684
 685                 i915_vma_unpin(vma);
 686
 687                 if (vma->page_sizes.sg & I915_GTT_PAGE_SIZE_64K) {
 688                         if (!IS_ALIGNED(vma->node.start,
 689                                         I915_GTT_PAGE_SIZE_2M)) {
 690                                 pr_err("node.start(%llx) not aligned to 2M\n",
 691                                        vma->node.start);
 692                                 err = -EINVAL;
 693                                 break;
 694                         }
 695
 696                         if (!IS_ALIGNED(vma->node.size,
 697                                         I915_GTT_PAGE_SIZE_2M)) {
 698                                 pr_err("node.size(%llx) not aligned to 2M\n",
 699                                        vma->node.size);
 700                                 err = -EINVAL;
 701                                 break;
 702                         }
 703                 }
 704
 705                 if (vma->page_sizes.gtt != expected_gtt) {
 706                         pr_err("gtt=%u, expected=%u, size=%zd, single=%s\n",
 707                                vma->page_sizes.gtt, expected_gtt,
 708                                obj->base.size, yesno(!!single));
 709                         err = -EINVAL;
 710                         break;
 711                 }
 712
 713                 if (igt_timeout(end_time,
 714                                 "%s timed out at size %zd\n",
 715                                 __func__, obj->base.size))
 716                         break;
 717
 718                 single = !single;
 719         }
 720
 721         close_object_list(&objects, ppgtt);
 722
 723         if (err == -ENOMEM || err == -ENOSPC)
 724                 err = 0;
 725
 726         return err;
 727 }
 728
 729 static int igt_mock_ppgtt_64K(void *arg)
 730 {
 731         struct i915_hw_ppgtt *ppgtt = arg;
 732         struct drm_i915_private *i915 = ppgtt->vm.i915;
 733         struct drm_i915_gem_object *obj;
 734         const struct object_info {
 735                 unsigned int size;
 736                 unsigned int gtt;
 737                 unsigned int offset;
 738         } objects[] = {
 739                 /* Cases with forced padding/alignment */
 740                 {
 741                         .size = SZ_64K,
 742                         .gtt = I915_GTT_PAGE_SIZE_64K,
 743                         .offset = 0,
 744                 },
 745                 {
 746                         .size = SZ_64K + SZ_4K,
 747                         .gtt = I915_GTT_PAGE_SIZE_4K,
 748                         .offset = 0,
 749                 },
 750                 {
 751                         .size = SZ_64K - SZ_4K,
 752                         .gtt = I915_GTT_PAGE_SIZE_4K,
 753                         .offset = 0,
 754                 },
 755                 {
 756                         .size = SZ_2M,
 757                         .gtt = I915_GTT_PAGE_SIZE_64K,
 758                         .offset = 0,
 759                 },
 760                 {
 761                         .size = SZ_2M - SZ_4K,
 762                         .gtt = I915_GTT_PAGE_SIZE_4K,
 763                         .offset = 0,
 764                 },
 765                 {
 766                         .size = SZ_2M + SZ_4K,
 767                         .gtt = I915_GTT_PAGE_SIZE_64K | I915_GTT_PAGE_SIZE_4K,
 768                         .offset = 0,
 769                 },
 770                 {
 771                         .size = SZ_2M + SZ_64K,
 772                         .gtt = I915_GTT_PAGE_SIZE_64K,
 773                         .offset = 0,
 774                 },
 775                 {
 776                         .size = SZ_2M - SZ_64K,
 777                         .gtt = I915_GTT_PAGE_SIZE_64K,
 778                         .offset = 0,
 779                 },
 780                 /* Try without any forced padding/alignment */
 781                 {
 782                         .size = SZ_64K,
 783                         .offset = SZ_2M,
 784                         .gtt = I915_GTT_PAGE_SIZE_4K,
 785                 },
 786                 {
 787                         .size = SZ_128K,
 788                         .offset = SZ_2M - SZ_64K,
 789                         .gtt = I915_GTT_PAGE_SIZE_4K,
 790                 },
 791         };
 792         struct i915_vma *vma;
 793         int i, single;
 794         int err;
 795
 796         /*
 797          * Sanity check some of the trickiness with 64K pages -- either we can
 798          * safely mark the whole page-table(2M block) as 64K, or we have to
 799          * always fallback to 4K.
 800          */
 801
 802         if (!HAS_PAGE_SIZES(i915, I915_GTT_PAGE_SIZE_64K))
 803                 return 0;
 804
 805         for (i = 0; i < ARRAY_SIZE(objects); ++i) {
 806                 unsigned int size = objects[i].size;
 807                 unsigned int expected_gtt = objects[i].gtt;
 808                 unsigned int offset = objects[i].offset;
 809                 unsigned int flags = PIN_USER;
 810
 811                 for (single = 0; single <= 1; single++) {
 812                         obj = fake_huge_pages_object(i915, size, !!single);
 813                         if (IS_ERR(obj))
 814                                 return PTR_ERR(obj);
 815
 816                         err = i915_gem_object_pin_pages(obj);
 817                         if (err)
 818                                 goto out_object_put;
 819
 820                         /*
 821                          * Disable 2M pages -- We only want to use 64K/4K pages
 822                          * for this test.
 823                          */
 824                         obj->mm.page_sizes.sg &= ~I915_GTT_PAGE_SIZE_2M;
 825
 826                         vma = i915_vma_instance(obj, &ppgtt->vm, NULL);
 827                         if (IS_ERR(vma)) {
 828                                 err = PTR_ERR(vma);
 829                                 goto out_object_unpin;
 830                         }
 831
 832                         if (offset)
 833                                 flags |= PIN_OFFSET_FIXED | offset;
 834
 835                         err = i915_vma_pin(vma, 0, 0, flags);
 836                         if (err)
 837                                 goto out_vma_close;
 838
 839                         err = igt_check_page_sizes(vma);
 840                         if (err)
 841                                 goto out_vma_unpin;
 842
 843                         if (!offset && vma->page_sizes.sg & I915_GTT_PAGE_SIZE_64K) {
 844                                 if (!IS_ALIGNED(vma->node.start,
 845                                                 I915_GTT_PAGE_SIZE_2M)) {
 846                                         pr_err("node.start(%llx) not aligned to 2M\n",
 847                                                vma->node.start);
 848                                         err = -EINVAL;
 849                                         goto out_vma_unpin;
 850                                 }
 851
 852                                 if (!IS_ALIGNED(vma->node.size,
 853                                                 I915_GTT_PAGE_SIZE_2M)) {
 854                                         pr_err("node.size(%llx) not aligned to 2M\n",
 855                                                vma->node.size);
 856                                         err = -EINVAL;
 857                                         goto out_vma_unpin;
 858                                 }
 859                         }
 860
 861                         if (vma->page_sizes.gtt != expected_gtt) {
 862                                 pr_err("gtt=%u, expected=%u, i=%d, single=%s\n",
 863                                        vma->page_sizes.gtt, expected_gtt, i,
 864                                        yesno(!!single));
 865                                 err = -EINVAL;
 866                                 goto out_vma_unpin;
 867                         }
 868
 869                         i915_vma_unpin(vma);
 870                         i915_vma_close(vma);
 871
 872                         i915_gem_object_unpin_pages(obj);
 873                         __i915_gem_object_put_pages(obj, I915_MM_NORMAL);
 874                         i915_gem_object_put(obj);
 875                 }
 876         }
 877
 878         return 0;
 879
 880 out_vma_unpin:
 881         i915_vma_unpin(vma);
 882 out_vma_close:
 883         i915_vma_close(vma);
 884 out_object_unpin:
 885         i915_gem_object_unpin_pages(obj);
 886 out_object_put:
 887         i915_gem_object_put(obj);
 888
 889         return err;
 890 }
 891
 892 static struct i915_vma *
 893 gpu_write_dw(struct i915_vma *vma, u64 offset, u32 val)
 894 {
 895         struct drm_i915_private *i915 = vma->vm->i915;
 896         const int gen = INTEL_GEN(i915);
 897         unsigned int count = vma->size >> PAGE_SHIFT;
 898         struct drm_i915_gem_object *obj;
 899         struct i915_vma *batch;
 900         unsigned int size;
 901         u32 *cmd;
 902         int n;
 903         int err;
 904
 905         size = (1 + 4 * count) * sizeof(u32);
 906         size = round_up(size, PAGE_SIZE);
 907         obj = i915_gem_object_create_internal(i915, size);
 908         if (IS_ERR(obj))
 909                 return ERR_CAST(obj);
 910
 911         err = i915_gem_object_set_to_wc_domain(obj, true);
 912         if (err)
 913                 goto err;
 914
 915         cmd = i915_gem_object_pin_map(obj, I915_MAP_WC);
 916         if (IS_ERR(cmd)) {
 917                 err = PTR_ERR(cmd);
 918                 goto err;
 919         }
 920
 921         offset += vma->node.start;
 922
 923         for (n = 0; n < count; n++) {
 924                 if (gen >= 8) {
 925                         *cmd++ = MI_STORE_DWORD_IMM_GEN4;
 926                         *cmd++ = lower_32_bits(offset);
 927                         *cmd++ = upper_32_bits(offset);
 928                         *cmd++ = val;
 929                 } else if (gen >= 4) {
 930                         *cmd++ = MI_STORE_DWORD_IMM_GEN4 |
 931                                 (gen < 6 ? MI_USE_GGTT : 0);
 932                         *cmd++ = 0;
 933                         *cmd++ = offset;
 934                         *cmd++ = val;
 935                 } else {
 936                         *cmd++ = MI_STORE_DWORD_IMM | MI_MEM_VIRTUAL;
 937                         *cmd++ = offset;
 938                         *cmd++ = val;
 939                 }
 940
 941                 offset += PAGE_SIZE;
 942         }
 943
 944         *cmd = MI_BATCH_BUFFER_END;
 945         i915_gem_chipset_flush(i915);
 946
 947         i915_gem_object_unpin_map(obj);
 948
 949         batch = i915_vma_instance(obj, vma->vm, NULL);
 950         if (IS_ERR(batch)) {
 951                 err = PTR_ERR(batch);
 952                 goto err;
 953         }
 954
 955         err = i915_vma_pin(batch, 0, 0, PIN_USER);
 956         if (err)
 957                 goto err;
 958
 959         return batch;
 960
 961 err:
 962         i915_gem_object_put(obj);
 963
 964         return ERR_PTR(err);
 965 }
 966
 967 static int gpu_write(struct i915_vma *vma,
 968                      struct i915_gem_context *ctx,
 969                      struct intel_engine_cs *engine,
 970                      u32 dword,
 971                      u32 value)
 972 {
 973         struct i915_request *rq;
 974         struct i915_vma *batch;
 975         int err;
 976
 977         GEM_BUG_ON(!intel_engine_can_store_dword(engine));
 978
 979         err = i915_gem_object_set_to_gtt_domain(vma->obj, true);
 980         if (err)
 981                 return err;
 982
 983         batch = gpu_write_dw(vma, dword * sizeof(u32), value);
 984         if (IS_ERR(batch))
 985                 return PTR_ERR(batch);
 986
 987         rq = i915_request_alloc(engine, ctx);
 988         if (IS_ERR(rq)) {
 989                 err = PTR_ERR(rq);
 990                 goto err_batch;
 991         }
 992
 993         err = i915_vma_move_to_active(batch, rq, 0);
 994         if (err)
 995                 goto err_request;
 996
 997         i915_gem_object_set_active_reference(batch->obj);
 998
 999         err = i915_vma_move_to_active(vma, rq, EXEC_OBJECT_WRITE);
1000         if (err)
1001                 goto err_request;
1002
1003         err = engine->emit_bb_start(rq,
1004                                     batch->node.start, batch->node.size,
1005                                     0);
1006 err_request:
1007         if (err)
1008                 i915_request_skip(rq, err);
1009         i915_request_add(rq);
1010 err_batch:
1011         i915_vma_unpin(batch);
1012         i915_vma_close(batch);
1013
1014         return err;
1015 }
1016
1017 static int cpu_check(struct drm_i915_gem_object *obj, u32 dword, u32 val)
1018 {
1019         unsigned int needs_flush;
1020         unsigned long n;
1021         int err;
1022
1023         err = i915_gem_obj_prepare_shmem_read(obj, &needs_flush);
1024         if (err)
1025                 return err;
1026
1027         for (n = 0; n < obj->base.size >> PAGE_SHIFT; ++n) {
1028                 u32 *ptr = kmap_atomic(i915_gem_object_get_page(obj, n));
1029
1030                 if (needs_flush & CLFLUSH_BEFORE)
1031                         drm_clflush_virt_range(ptr, PAGE_SIZE);
1032
1033                 if (ptr[dword] != val) {
1034                         pr_err("n=%lu ptr[%u]=%u, val=%u\n",
1035                                n, dword, ptr[dword], val);
1036                         kunmap_atomic(ptr);
1037                         err = -EINVAL;
1038                         break;
1039                 }
1040
1041                 kunmap_atomic(ptr);
1042         }
1043
1044         i915_gem_obj_finish_shmem_access(obj);
1045
1046         return err;
1047 }
1048
1049 static int __igt_write_huge(struct i915_gem_context *ctx,
1050                             struct intel_engine_cs *engine,
1051                             struct drm_i915_gem_object *obj,
1052                             u64 size, u64 offset,
1053                             u32 dword, u32 val)
1054 {
1055         struct drm_i915_private *i915 = to_i915(obj->base.dev);
1056         struct i915_address_space *vm =
1057                 ctx->ppgtt ? &ctx->ppgtt->vm : &i915->ggtt.vm;
1058         unsigned int flags = PIN_USER | PIN_OFFSET_FIXED;
1059         struct i915_vma *vma;
1060         int err;
1061
1062         vma = i915_vma_instance(obj, vm, NULL);
1063         if (IS_ERR(vma))
1064                 return PTR_ERR(vma);
1065
1066         err = i915_vma_unbind(vma);
1067         if (err)
1068                 goto out_vma_close;
1069
1070         err = i915_vma_pin(vma, size, 0, flags | offset);
1071         if (err) {
1072                 /*
1073                  * The ggtt may have some pages reserved so
1074                  * refrain from erroring out.
1075                  */
1076                 if (err == -ENOSPC && i915_is_ggtt(vm))
1077                         err = 0;
1078
1079                 goto out_vma_close;
1080         }
1081
1082         err = igt_check_page_sizes(vma);
1083         if (err)
1084                 goto out_vma_unpin;
1085
1086         err = gpu_write(vma, ctx, engine, dword, val);
1087         if (err) {
1088                 pr_err("gpu-write failed at offset=%llx\n", offset);
1089                 goto out_vma_unpin;
1090         }
1091
1092         err = cpu_check(obj, dword, val);
1093         if (err) {
1094                 pr_err("cpu-check failed at offset=%llx\n", offset);
1095                 goto out_vma_unpin;
1096         }
1097
1098 out_vma_unpin:
1099         i915_vma_unpin(vma);
1100 out_vma_close:
1101         i915_vma_destroy(vma);
1102
1103         return err;
1104 }
1105
1106 static int igt_write_huge(struct i915_gem_context *ctx,
1107                           struct drm_i915_gem_object *obj)
1108 {
1109         struct drm_i915_private *i915 = to_i915(obj->base.dev);
1110         struct i915_address_space *vm =
1111                 ctx->ppgtt ? &ctx->ppgtt->vm : &i915->ggtt.vm;
1112         static struct intel_engine_cs *engines[I915_NUM_ENGINES];
1113         struct intel_engine_cs *engine;
1114         I915_RND_STATE(prng);
1115         IGT_TIMEOUT(end_time);
1116         unsigned int max_page_size;
1117         unsigned int id;
1118         u64 max;
1119         u64 num;
1120         u64 size;
1121         int *order;
1122         int i, n;
1123         int err = 0;
1124
1125         GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj));
1126
1127         size = obj->base.size;
1128         if (obj->mm.page_sizes.sg & I915_GTT_PAGE_SIZE_64K)
1129                 size = round_up(size, I915_GTT_PAGE_SIZE_2M);
1130
1131         max_page_size = rounddown_pow_of_two(obj->mm.page_sizes.sg);
1132         max = div_u64((vm->total - size), max_page_size);
1133
1134         n = 0;
1135         for_each_engine(engine, i915, id) {
1136                 if (!intel_engine_can_store_dword(engine)) {
1137                         pr_info("store-dword-imm not supported on engine=%u\n",
1138                                 id);
1139                         continue;
1140                 }
1141                 engines[n++] = engine;
1142         }
1143
1144         if (!n)
1145                 return 0;
1146
1147         /*
1148          * To keep things interesting when alternating between engines in our
1149          * randomized order, lets also make feeding to the same engine a few
1150          * times in succession a possibility by enlarging the permutation array.
1151          */
1152         order = i915_random_order(n * I915_NUM_ENGINES, &prng);
1153         if (!order)
1154                 return -ENOMEM;
1155
1156         /*
1157          * Try various offsets in an ascending/descending fashion until we
1158          * timeout -- we want to avoid issues hidden by effectively always using
1159          * offset = 0.
1160          */
1161         i = 0;
1162         for_each_prime_number_from(num, 0, max) {
1163                 u64 offset_low = num * max_page_size;
1164                 u64 offset_high = (max - num) * max_page_size;
1165                 u32 dword = offset_in_page(num) / 4;
1166
1167                 engine = engines[order[i] % n];
1168                 i = (i + 1) % (n * I915_NUM_ENGINES);
1169
1170                 /*
1171                  * In order to utilize 64K pages we need to both pad the vma
1172                  * size and ensure the vma offset is at the start of the pt
1173                  * boundary, however to improve coverage we opt for testing both
1174                  * aligned and unaligned offsets.
1175                  */
1176                 if (obj->mm.page_sizes.sg & I915_GTT_PAGE_SIZE_64K)
1177                         offset_low = round_down(offset_low,
1178                                                 I915_GTT_PAGE_SIZE_2M);
1179
1180                 err = __igt_write_huge(ctx, engine, obj, size, offset_low,
1181                                        dword, num + 1);
1182                 if (err)
1183                         break;
1184
1185                 err = __igt_write_huge(ctx, engine, obj, size, offset_high,
1186                                        dword, num + 1);
1187                 if (err)
1188                         break;
1189
1190                 if (igt_timeout(end_time,
1191                                 "%s timed out on engine=%u, offset_low=%llx offset_high=%llx, max_page_size=%x\n",
1192                                 __func__, engine->id, offset_low, offset_high,
1193                                 max_page_size))
1194                         break;
1195         }
1196
1197         kfree(order);
1198
1199         return err;
1200 }
1201
1202 static int igt_ppgtt_exhaust_huge(void *arg)
1203 {
1204         struct i915_gem_context *ctx = arg;
1205         struct drm_i915_private *i915 = ctx->i915;
1206         unsigned long supported = INTEL_INFO(i915)->page_sizes;
1207         static unsigned int pages[ARRAY_SIZE(page_sizes)];
1208         struct drm_i915_gem_object *obj;
1209         unsigned int size_mask;
1210         unsigned int page_mask;
1211         int n, i;
1212         int err = -ENODEV;
1213
1214         if (supported == I915_GTT_PAGE_SIZE_4K)
1215                 return 0;
1216
1217         /*
1218          * Sanity check creating objects with a varying mix of page sizes --
1219          * ensuring that our writes lands in the right place.
1220          */
1221
1222         n = 0;
1223         for_each_set_bit(i, &supported, ilog2(I915_GTT_MAX_PAGE_SIZE) + 1)
1224                 pages[n++] = BIT(i);
1225
1226         for (size_mask = 2; size_mask < BIT(n); size_mask++) {
1227                 unsigned int size = 0;
1228
1229                 for (i = 0; i < n; i++) {
1230                         if (size_mask & BIT(i))
1231                                 size |= pages[i];
1232                 }
1233
1234                 /*
1235                  * For our page mask we want to enumerate all the page-size
1236                  * combinations which will fit into our chosen object size.
1237                  */
1238                 for (page_mask = 2; page_mask <= size_mask; page_mask++) {
1239                         unsigned int page_sizes = 0;
1240
1241                         for (i = 0; i < n; i++) {
1242                                 if (page_mask & BIT(i))
1243                                         page_sizes |= pages[i];
1244                         }
1245
1246                         /*
1247                          * Ensure that we can actually fill the given object
1248                          * with our chosen page mask.
1249                          */
1250                         if (!IS_ALIGNED(size, BIT(__ffs(page_sizes))))
1251                                 continue;
1252
1253                         obj = huge_pages_object(i915, size, page_sizes);
1254                         if (IS_ERR(obj)) {
1255                                 err = PTR_ERR(obj);
1256                                 goto out_device;
1257                         }
1258
1259                         err = i915_gem_object_pin_pages(obj);
1260                         if (err) {
1261                                 i915_gem_object_put(obj);
1262
1263                                 if (err == -ENOMEM) {
1264                                         pr_info("unable to get pages, size=%u, pages=%u\n",
1265                                                 size, page_sizes);
1266                                         err = 0;
1267                                         break;
1268                                 }
1269
1270                                 pr_err("pin_pages failed, size=%u, pages=%u\n",
1271                                        size_mask, page_mask);
1272
1273                                 goto out_device;
1274                         }
1275
1276                         /* Force the page-size for the gtt insertion */
1277                         obj->mm.page_sizes.sg = page_sizes;
1278
1279                         err = igt_write_huge(ctx, obj);
1280                         if (err) {
1281                                 pr_err("exhaust write-huge failed with size=%u\n",
1282                                        size);
1283                                 goto out_unpin;
1284                         }
1285
1286                         i915_gem_object_unpin_pages(obj);
1287                         __i915_gem_object_put_pages(obj, I915_MM_NORMAL);
1288                         i915_gem_object_put(obj);
1289                 }
1290         }
1291
1292         goto out_device;
1293
1294 out_unpin:
1295         i915_gem_object_unpin_pages(obj);
1296         i915_gem_object_put(obj);
1297 out_device:
1298         mkwrite_device_info(i915)->page_sizes = supported;
1299
1300         return err;
1301 }
1302
1303 static int igt_ppgtt_internal_huge(void *arg)
1304 {
1305         struct i915_gem_context *ctx = arg;
1306         struct drm_i915_private *i915 = ctx->i915;
1307         struct drm_i915_gem_object *obj;
1308         static const unsigned int sizes[] = {
1309                 SZ_64K,
1310                 SZ_128K,
1311                 SZ_256K,
1312                 SZ_512K,
1313                 SZ_1M,
1314                 SZ_2M,
1315         };
1316         int i;
1317         int err;
1318
1319         /*
1320          * Sanity check that the HW uses huge pages correctly through internal
1321          * -- ensure that our writes land in the right place.
1322          */
1323
1324         for (i = 0; i < ARRAY_SIZE(sizes); ++i) {
1325                 unsigned int size = sizes[i];
1326
1327                 obj = i915_gem_object_create_internal(i915, size);
1328                 if (IS_ERR(obj))
1329                         return PTR_ERR(obj);
1330
1331                 err = i915_gem_object_pin_pages(obj);
1332                 if (err)
1333                         goto out_put;
1334
1335                 if (obj->mm.page_sizes.phys < I915_GTT_PAGE_SIZE_64K) {
1336                         pr_info("internal unable to allocate huge-page(s) with size=%u\n",
1337                                 size);
1338                         goto out_unpin;
1339                 }
1340
1341                 err = igt_write_huge(ctx, obj);
1342                 if (err) {
1343                         pr_err("internal write-huge failed with size=%u\n",
1344                                size);
1345                         goto out_unpin;
1346                 }
1347
1348                 i915_gem_object_unpin_pages(obj);
1349                 __i915_gem_object_put_pages(obj, I915_MM_NORMAL);
1350                 i915_gem_object_put(obj);
1351         }
1352
1353         return 0;
1354
1355 out_unpin:
1356         i915_gem_object_unpin_pages(obj);
1357 out_put:
1358         i915_gem_object_put(obj);
1359
1360         return err;
1361 }
1362
1363 static inline bool igt_can_allocate_thp(struct drm_i915_private *i915)
1364 {
1365         return i915->mm.gemfs && has_transparent_hugepage();
1366 }
1367
1368 static int igt_ppgtt_gemfs_huge(void *arg)
1369 {
1370         struct i915_gem_context *ctx = arg;
1371         struct drm_i915_private *i915 = ctx->i915;
1372         struct drm_i915_gem_object *obj;
1373         static const unsigned int sizes[] = {
1374                 SZ_2M,
1375                 SZ_4M,
1376                 SZ_8M,
1377                 SZ_16M,
1378                 SZ_32M,
1379         };
1380         int i;
1381         int err;
1382
1383         /*
1384          * Sanity check that the HW uses huge pages correctly through gemfs --
1385          * ensure that our writes land in the right place.
1386          */
1387
1388         if (!igt_can_allocate_thp(i915)) {
1389                 pr_info("missing THP support, skipping\n");
1390                 return 0;
1391         }
1392
1393         for (i = 0; i < ARRAY_SIZE(sizes); ++i) {
1394                 unsigned int size = sizes[i];
1395
1396                 obj = i915_gem_object_create(i915, size);
1397                 if (IS_ERR(obj))
1398                         return PTR_ERR(obj);
1399
1400                 err = i915_gem_object_pin_pages(obj);
1401                 if (err)
1402                         goto out_put;
1403
1404                 if (obj->mm.page_sizes.phys < I915_GTT_PAGE_SIZE_2M) {
1405                         pr_info("finishing test early, gemfs unable to allocate huge-page(s) with size=%u\n",
1406                                 size);
1407                         goto out_unpin;
1408                 }
1409
1410                 err = igt_write_huge(ctx, obj);
1411                 if (err) {
1412                         pr_err("gemfs write-huge failed with size=%u\n",
1413                                size);
1414                         goto out_unpin;
1415                 }
1416
1417                 i915_gem_object_unpin_pages(obj);
1418                 __i915_gem_object_put_pages(obj, I915_MM_NORMAL);
1419                 i915_gem_object_put(obj);
1420         }
1421
1422         return 0;
1423
1424 out_unpin:
1425         i915_gem_object_unpin_pages(obj);
1426 out_put:
1427         i915_gem_object_put(obj);
1428
1429         return err;
1430 }
1431
1432 static int igt_ppgtt_pin_update(void *arg)
1433 {
1434         struct i915_gem_context *ctx = arg;
1435         struct drm_i915_private *dev_priv = ctx->i915;
1436         unsigned long supported = INTEL_INFO(dev_priv)->page_sizes;
1437         struct i915_hw_ppgtt *ppgtt = ctx->ppgtt;
1438         struct drm_i915_gem_object *obj;
1439         struct i915_vma *vma;
1440         unsigned int flags = PIN_USER | PIN_OFFSET_FIXED;
1441         int first, last;
1442         int err;
1443
1444         /*
1445          * Make sure there's no funny business when doing a PIN_UPDATE -- in the
1446          * past we had a subtle issue with being able to incorrectly do multiple
1447          * alloc va ranges on the same object when doing a PIN_UPDATE, which
1448          * resulted in some pretty nasty bugs, though only when using
1449          * huge-gtt-pages.
1450          */
1451
1452         if (!ppgtt || !i915_vm_is_48bit(&ppgtt->vm)) {
1453                 pr_info("48b PPGTT not supported, skipping\n");
1454                 return 0;
1455         }
1456
1457         first = ilog2(I915_GTT_PAGE_SIZE_64K);
1458         last = ilog2(I915_GTT_PAGE_SIZE_2M);
1459
1460         for_each_set_bit_from(first, &supported, last + 1) {
1461                 unsigned int page_size = BIT(first);
1462
1463                 obj = i915_gem_object_create_internal(dev_priv, page_size);
1464                 if (IS_ERR(obj))
1465                         return PTR_ERR(obj);
1466
1467                 vma = i915_vma_instance(obj, &ppgtt->vm, NULL);
1468                 if (IS_ERR(vma)) {
1469                         err = PTR_ERR(vma);
1470                         goto out_put;
1471                 }
1472
1473                 err = i915_vma_pin(vma, SZ_2M, 0, flags);
1474                 if (err)
1475                         goto out_close;
1476
1477                 if (vma->page_sizes.sg < page_size) {
1478                         pr_info("Unable to allocate page-size %x, finishing test early\n",
1479                                 page_size);
1480                         goto out_unpin;
1481                 }
1482
1483                 err = igt_check_page_sizes(vma);
1484                 if (err)
1485                         goto out_unpin;
1486
1487                 if (vma->page_sizes.gtt != page_size) {
1488                         dma_addr_t addr = i915_gem_object_get_dma_address(obj, 0);
1489
1490                         /*
1491                          * The only valid reason for this to ever fail would be
1492                          * if the dma-mapper screwed us over when we did the
1493                          * dma_map_sg(), since it has the final say over the dma
1494                          * address.
1495                          */
1496                         if (IS_ALIGNED(addr, page_size)) {
1497                                 pr_err("page_sizes.gtt=%u, expected=%u\n",
1498                                        vma->page_sizes.gtt, page_size);
1499                                 err = -EINVAL;
1500                         } else {
1501                                 pr_info("dma address misaligned, finishing test early\n");
1502                         }
1503
1504                         goto out_unpin;
1505                 }
1506
1507                 err = i915_vma_bind(vma, I915_CACHE_NONE, PIN_UPDATE);
1508                 if (err)
1509                         goto out_unpin;
1510
1511                 i915_vma_unpin(vma);
1512                 i915_vma_close(vma);
1513
1514                 i915_gem_object_put(obj);
1515         }
1516
1517         obj = i915_gem_object_create_internal(dev_priv, PAGE_SIZE);
1518         if (IS_ERR(obj))
1519                 return PTR_ERR(obj);
1520
1521         vma = i915_vma_instance(obj, &ppgtt->vm, NULL);
1522         if (IS_ERR(vma)) {
1523                 err = PTR_ERR(vma);
1524                 goto out_put;
1525         }
1526
1527         err = i915_vma_pin(vma, 0, 0, flags);
1528         if (err)
1529                 goto out_close;
1530
1531         /*
1532          * Make sure we don't end up with something like where the pde is still
1533          * pointing to the 2M page, and the pt we just filled-in is dangling --
1534          * we can check this by writing to the first page where it would then
1535          * land in the now stale 2M page.
1536          */
1537
1538         err = gpu_write(vma, ctx, dev_priv->engine[RCS], 0, 0xdeadbeaf);
1539         if (err)
1540                 goto out_unpin;
1541
1542         err = cpu_check(obj, 0, 0xdeadbeaf);
1543
1544 out_unpin:
1545         i915_vma_unpin(vma);
1546 out_close:
1547         i915_vma_close(vma);
1548 out_put:
1549         i915_gem_object_put(obj);
1550
1551         return err;
1552 }
1553
1554 static int igt_tmpfs_fallback(void *arg)
1555 {
1556         struct i915_gem_context *ctx = arg;
1557         struct drm_i915_private *i915 = ctx->i915;
1558         struct vfsmount *gemfs = i915->mm.gemfs;
1559         struct i915_address_space *vm =
1560                 ctx->ppgtt ? &ctx->ppgtt->vm : &i915->ggtt.vm;
1561         struct drm_i915_gem_object *obj;
1562         struct i915_vma *vma;
1563         u32 *vaddr;
1564         int err = 0;
1565
1566         /*
1567          * Make sure that we don't burst into a ball of flames upon falling back
1568          * to tmpfs, which we rely on if on the off-chance we encouter a failure
1569          * when setting up gemfs.
1570          */
1571
1572         i915->mm.gemfs = NULL;
1573
1574         obj = i915_gem_object_create(i915, PAGE_SIZE);
1575         if (IS_ERR(obj)) {
1576                 err = PTR_ERR(obj);
1577                 goto out_restore;
1578         }
1579
1580         vaddr = i915_gem_object_pin_map(obj, I915_MAP_WB);
1581         if (IS_ERR(vaddr)) {
1582                 err = PTR_ERR(vaddr);
1583                 goto out_put;
1584         }
1585         *vaddr = 0xdeadbeaf;
1586
1587         i915_gem_object_unpin_map(obj);
1588
1589         vma = i915_vma_instance(obj, vm, NULL);
1590         if (IS_ERR(vma)) {
1591                 err = PTR_ERR(vma);
1592                 goto out_put;
1593         }
1594
1595         err = i915_vma_pin(vma, 0, 0, PIN_USER);
1596         if (err)
1597                 goto out_close;
1598
1599         err = igt_check_page_sizes(vma);
1600
1601         i915_vma_unpin(vma);
1602 out_close:
1603         i915_vma_close(vma);
1604 out_put:
1605         i915_gem_object_put(obj);
1606 out_restore:
1607         i915->mm.gemfs = gemfs;
1608
1609         return err;
1610 }
1611
1612 static int igt_shrink_thp(void *arg)
1613 {
1614         struct i915_gem_context *ctx = arg;
1615         struct drm_i915_private *i915 = ctx->i915;
1616         struct i915_address_space *vm =
1617                 ctx->ppgtt ? &ctx->ppgtt->vm : &i915->ggtt.vm;
1618         struct drm_i915_gem_object *obj;
1619         struct i915_vma *vma;
1620         unsigned int flags = PIN_USER;
1621         int err;
1622
1623         /*
1624          * Sanity check shrinking huge-paged object -- make sure nothing blows
1625          * up.
1626          */
1627
1628         if (!igt_can_allocate_thp(i915)) {
1629                 pr_info("missing THP support, skipping\n");
1630                 return 0;
1631         }
1632
1633         obj = i915_gem_object_create(i915, SZ_2M);
1634         if (IS_ERR(obj))
1635                 return PTR_ERR(obj);
1636
1637         vma = i915_vma_instance(obj, vm, NULL);
1638         if (IS_ERR(vma)) {
1639                 err = PTR_ERR(vma);
1640                 goto out_put;
1641         }
1642
1643         err = i915_vma_pin(vma, 0, 0, flags);
1644         if (err)
1645                 goto out_close;
1646
1647         if (obj->mm.page_sizes.phys < I915_GTT_PAGE_SIZE_2M) {
1648                 pr_info("failed to allocate THP, finishing test early\n");
1649                 goto out_unpin;
1650         }
1651
1652         err = igt_check_page_sizes(vma);
1653         if (err)
1654                 goto out_unpin;
1655
1656         err = gpu_write(vma, ctx, i915->engine[RCS], 0, 0xdeadbeaf);
1657         if (err)
1658                 goto out_unpin;
1659
1660         i915_vma_unpin(vma);
1661
1662         /*
1663          * Now that the pages are *unpinned* shrink-all should invoke
1664          * shmem to truncate our pages.
1665          */
1666         i915_gem_shrink_all(i915);
1667         if (i915_gem_object_has_pages(obj)) {
1668                 pr_err("shrink-all didn't truncate the pages\n");
1669                 err = -EINVAL;
1670                 goto out_close;
1671         }
1672
1673         if (obj->mm.page_sizes.sg || obj->mm.page_sizes.phys) {
1674                 pr_err("residual page-size bits left\n");
1675                 err = -EINVAL;
1676                 goto out_close;
1677         }
1678
1679         err = i915_vma_pin(vma, 0, 0, flags);
1680         if (err)
1681                 goto out_close;
1682
1683         err = cpu_check(obj, 0, 0xdeadbeaf);
1684
1685 out_unpin:
1686         i915_vma_unpin(vma);
1687 out_close:
1688         i915_vma_close(vma);
1689 out_put:
1690         i915_gem_object_put(obj);
1691
1692         return err;
1693 }
1694
1695 int i915_gem_huge_page_mock_selftests(void)
1696 {
1697         static const struct i915_subtest tests[] = {
1698                 SUBTEST(igt_mock_exhaust_device_supported_pages),
1699                 SUBTEST(igt_mock_ppgtt_misaligned_dma),
1700                 SUBTEST(igt_mock_ppgtt_huge_fill),
1701                 SUBTEST(igt_mock_ppgtt_64K),
1702         };
1703         struct drm_i915_private *dev_priv;
1704         struct i915_hw_ppgtt *ppgtt;
1705         int err;
1706
1707         dev_priv = mock_gem_device();
1708         if (!dev_priv)
1709                 return -ENOMEM;
1710
1711         /* Pretend to be a device which supports the 48b PPGTT */
1712         mkwrite_device_info(dev_priv)->ppgtt = INTEL_PPGTT_FULL_4LVL;
1713
1714         mutex_lock(&dev_priv->drm.struct_mutex);
1715         ppgtt = i915_ppgtt_create(dev_priv, ERR_PTR(-ENODEV));
1716         if (IS_ERR(ppgtt)) {
1717                 err = PTR_ERR(ppgtt);
1718                 goto out_unlock;
1719         }
1720
1721         if (!i915_vm_is_48bit(&ppgtt->vm)) {
1722                 pr_err("failed to create 48b PPGTT\n");
1723                 err = -EINVAL;
1724                 goto out_close;
1725         }
1726
1727         /* If we were ever hit this then it's time to mock the 64K scratch */
1728         if (!i915_vm_has_scratch_64K(&ppgtt->vm)) {
1729                 pr_err("PPGTT missing 64K scratch page\n");
1730                 err = -EINVAL;
1731                 goto out_close;
1732         }
1733
1734         err = i915_subtests(tests, ppgtt);
1735
1736 out_close:
1737         i915_ppgtt_close(&ppgtt->vm);
1738         i915_ppgtt_put(ppgtt);
1739
1740 out_unlock:
1741         mutex_unlock(&dev_priv->drm.struct_mutex);
1742         drm_dev_put(&dev_priv->drm);
1743
1744         return err;
1745 }
1746
1747 int i915_gem_huge_page_live_selftests(struct drm_i915_private *dev_priv)
1748 {
1749         static const struct i915_subtest tests[] = {
1750                 SUBTEST(igt_shrink_thp),
1751                 SUBTEST(igt_ppgtt_pin_update),
1752                 SUBTEST(igt_tmpfs_fallback),
1753                 SUBTEST(igt_ppgtt_exhaust_huge),
1754                 SUBTEST(igt_ppgtt_gemfs_huge),
1755                 SUBTEST(igt_ppgtt_internal_huge),
1756         };
1757         struct drm_file *file;
1758         struct i915_gem_context *ctx;
1759         intel_wakeref_t wakeref;
1760         int err;
1761
1762         if (!HAS_PPGTT(dev_priv)) {
1763                 pr_info("PPGTT not supported, skipping live-selftests\n");
1764                 return 0;
1765         }
1766
1767         if (i915_terminally_wedged(&dev_priv->gpu_error))
1768                 return 0;
1769
1770         file = mock_file(dev_priv);
1771         if (IS_ERR(file))
1772                 return PTR_ERR(file);
1773
1774         mutex_lock(&dev_priv->drm.struct_mutex);
1775         wakeref = intel_runtime_pm_get(dev_priv);
1776
1777         ctx = live_context(dev_priv, file);
1778         if (IS_ERR(ctx)) {
1779                 err = PTR_ERR(ctx);
1780                 goto out_unlock;
1781         }
1782
1783         if (ctx->ppgtt)
1784                 ctx->ppgtt->vm.scrub_64K = true;
1785
1786         err = i915_subtests(tests, ctx);
1787
1788 out_unlock:
1789         intel_runtime_pm_put(dev_priv, wakeref);
1790         mutex_unlock(&dev_priv->drm.struct_mutex);
1791
1792         mock_file_free(dev_priv, file);
1793
1794         return err;
1795 }