drivers/gpu/drm/i915/gem/selftests/huge_pages.c

   1 /*
   2  * SPDX-License-Identifier: MIT
   3  *
   4  * Copyright © 2017 Intel Corporation
   5  */
   6
   7 #include <linux/prime_numbers.h>
   8
   9 #include "i915_selftest.h"
  10
  11 #include "gem/i915_gem_region.h"
  12 #include "gem/i915_gem_lmem.h"
  13 #include "gem/i915_gem_pm.h"
  14
  15 #include "gt/intel_gt.h"
  16
  17 #include "igt_gem_utils.h"
  18 #include "mock_context.h"
  19
  20 #include "selftests/mock_drm.h"
  21 #include "selftests/mock_gem_device.h"
  22 #include "selftests/mock_region.h"
  23 #include "selftests/i915_random.h"
  24
  25 static const unsigned int page_sizes[] = {
  26         I915_GTT_PAGE_SIZE_2M,
  27         I915_GTT_PAGE_SIZE_64K,
  28         I915_GTT_PAGE_SIZE_4K,
  29 };
  30
  31 static unsigned int get_largest_page_size(struct drm_i915_private *i915,
  32                                           u64 rem)
  33 {
  34         int i;
  35
  36         for (i = 0; i < ARRAY_SIZE(page_sizes); ++i) {
  37                 unsigned int page_size = page_sizes[i];
  38
  39                 if (HAS_PAGE_SIZES(i915, page_size) && rem >= page_size)
  40                         return page_size;
  41         }
  42
  43         return 0;
  44 }
  45
  46 static void huge_pages_free_pages(struct sg_table *st)
  47 {
  48         struct scatterlist *sg;
  49
  50         for (sg = st->sgl; sg; sg = __sg_next(sg)) {
  51                 if (sg_page(sg))
  52                         __free_pages(sg_page(sg), get_order(sg->length));
  53         }
  54
  55         sg_free_table(st);
  56         kfree(st);
  57 }
  58
  59 static int get_huge_pages(struct drm_i915_gem_object *obj)
  60 {
  61 #define GFP (GFP_KERNEL | __GFP_NOWARN | __GFP_NORETRY)
  62         unsigned int page_mask = obj->mm.page_mask;
  63         struct sg_table *st;
  64         struct scatterlist *sg;
  65         unsigned int sg_page_sizes;
  66         u64 rem;
  67
  68         st = kmalloc(sizeof(*st), GFP);
  69         if (!st)
  70                 return -ENOMEM;
  71
  72         if (sg_alloc_table(st, obj->base.size >> PAGE_SHIFT, GFP)) {
  73                 kfree(st);
  74                 return -ENOMEM;
  75         }
  76
  77         rem = obj->base.size;
  78         sg = st->sgl;
  79         st->nents = 0;
  80         sg_page_sizes = 0;
  81
  82         /*
  83          * Our goal here is simple, we want to greedily fill the object from
  84          * largest to smallest page-size, while ensuring that we use *every*
  85          * page-size as per the given page-mask.
  86          */
  87         do {
  88                 unsigned int bit = ilog2(page_mask);
  89                 unsigned int page_size = BIT(bit);
  90                 int order = get_order(page_size);
  91
  92                 do {
  93                         struct page *page;
  94
  95                         GEM_BUG_ON(order >= MAX_ORDER);
  96                         page = alloc_pages(GFP | __GFP_ZERO, order);
  97                         if (!page)
  98                                 goto err;
  99
 100                         sg_set_page(sg, page, page_size, 0);
 101                         sg_page_sizes |= page_size;
 102                         st->nents++;
 103
 104                         rem -= page_size;
 105                         if (!rem) {
 106                                 sg_mark_end(sg);
 107                                 break;
 108                         }
 109
 110                         sg = __sg_next(sg);
 111                 } while ((rem - ((page_size-1) & page_mask)) >= page_size);
 112
 113                 page_mask &= (page_size-1);
 114         } while (page_mask);
 115
 116         if (i915_gem_gtt_prepare_pages(obj, st))
 117                 goto err;
 118
 119         GEM_BUG_ON(sg_page_sizes != obj->mm.page_mask);
 120         __i915_gem_object_set_pages(obj, st, sg_page_sizes);
 121
 122         return 0;
 123
 124 err:
 125         sg_set_page(sg, NULL, 0, 0);
 126         sg_mark_end(sg);
 127         huge_pages_free_pages(st);
 128
 129         return -ENOMEM;
 130 }
 131
 132 static void put_huge_pages(struct drm_i915_gem_object *obj,
 133                            struct sg_table *pages)
 134 {
 135         i915_gem_gtt_finish_pages(obj, pages);
 136         huge_pages_free_pages(pages);
 137
 138         obj->mm.dirty = false;
 139 }
 140
 141 static const struct drm_i915_gem_object_ops huge_page_ops = {
 142         .flags = I915_GEM_OBJECT_HAS_STRUCT_PAGE |
 143                  I915_GEM_OBJECT_IS_SHRINKABLE,
 144         .get_pages = get_huge_pages,
 145         .put_pages = put_huge_pages,
 146 };
 147
 148 static struct drm_i915_gem_object *
 149 huge_pages_object(struct drm_i915_private *i915,
 150                   u64 size,
 151                   unsigned int page_mask)
 152 {
 153         static struct lock_class_key lock_class;
 154         struct drm_i915_gem_object *obj;
 155
 156         GEM_BUG_ON(!size);
 157         GEM_BUG_ON(!IS_ALIGNED(size, BIT(__ffs(page_mask))));
 158
 159         if (size >> PAGE_SHIFT > INT_MAX)
 160                 return ERR_PTR(-E2BIG);
 161
 162         if (overflows_type(size, obj->base.size))
 163                 return ERR_PTR(-E2BIG);
 164
 165         obj = i915_gem_object_alloc();
 166         if (!obj)
 167                 return ERR_PTR(-ENOMEM);
 168
 169         drm_gem_private_object_init(&i915->drm, &obj->base, size);
 170         i915_gem_object_init(obj, &huge_page_ops, &lock_class);
 171
 172         i915_gem_object_set_volatile(obj);
 173
 174         obj->write_domain = I915_GEM_DOMAIN_CPU;
 175         obj->read_domains = I915_GEM_DOMAIN_CPU;
 176         obj->cache_level = I915_CACHE_NONE;
 177
 178         obj->mm.page_mask = page_mask;
 179
 180         return obj;
 181 }
 182
 183 static int fake_get_huge_pages(struct drm_i915_gem_object *obj)
 184 {
 185         struct drm_i915_private *i915 = to_i915(obj->base.dev);
 186         const u64 max_len = rounddown_pow_of_two(UINT_MAX);
 187         struct sg_table *st;
 188         struct scatterlist *sg;
 189         unsigned int sg_page_sizes;
 190         u64 rem;
 191
 192         st = kmalloc(sizeof(*st), GFP);
 193         if (!st)
 194                 return -ENOMEM;
 195
 196         if (sg_alloc_table(st, obj->base.size >> PAGE_SHIFT, GFP)) {
 197                 kfree(st);
 198                 return -ENOMEM;
 199         }
 200
 201         /* Use optimal page sized chunks to fill in the sg table */
 202         rem = obj->base.size;
 203         sg = st->sgl;
 204         st->nents = 0;
 205         sg_page_sizes = 0;
 206         do {
 207                 unsigned int page_size = get_largest_page_size(i915, rem);
 208                 unsigned int len = min(page_size * div_u64(rem, page_size),
 209                                        max_len);
 210
 211                 GEM_BUG_ON(!page_size);
 212
 213                 sg->offset = 0;
 214                 sg->length = len;
 215                 sg_dma_len(sg) = len;
 216                 sg_dma_address(sg) = page_size;
 217
 218                 sg_page_sizes |= len;
 219
 220                 st->nents++;
 221
 222                 rem -= len;
 223                 if (!rem) {
 224                         sg_mark_end(sg);
 225                         break;
 226                 }
 227
 228                 sg = sg_next(sg);
 229         } while (1);
 230
 231         i915_sg_trim(st);
 232
 233         __i915_gem_object_set_pages(obj, st, sg_page_sizes);
 234
 235         return 0;
 236 }
 237
 238 static int fake_get_huge_pages_single(struct drm_i915_gem_object *obj)
 239 {
 240         struct drm_i915_private *i915 = to_i915(obj->base.dev);
 241         struct sg_table *st;
 242         struct scatterlist *sg;
 243         unsigned int page_size;
 244
 245         st = kmalloc(sizeof(*st), GFP);
 246         if (!st)
 247                 return -ENOMEM;
 248
 249         if (sg_alloc_table(st, 1, GFP)) {
 250                 kfree(st);
 251                 return -ENOMEM;
 252         }
 253
 254         sg = st->sgl;
 255         st->nents = 1;
 256
 257         page_size = get_largest_page_size(i915, obj->base.size);
 258         GEM_BUG_ON(!page_size);
 259
 260         sg->offset = 0;
 261         sg->length = obj->base.size;
 262         sg_dma_len(sg) = obj->base.size;
 263         sg_dma_address(sg) = page_size;
 264
 265         __i915_gem_object_set_pages(obj, st, sg->length);
 266
 267         return 0;
 268 #undef GFP
 269 }
 270
 271 static void fake_free_huge_pages(struct drm_i915_gem_object *obj,
 272                                  struct sg_table *pages)
 273 {
 274         sg_free_table(pages);
 275         kfree(pages);
 276 }
 277
 278 static void fake_put_huge_pages(struct drm_i915_gem_object *obj,
 279                                 struct sg_table *pages)
 280 {
 281         fake_free_huge_pages(obj, pages);
 282         obj->mm.dirty = false;
 283 }
 284
 285 static const struct drm_i915_gem_object_ops fake_ops = {
 286         .flags = I915_GEM_OBJECT_IS_SHRINKABLE,
 287         .get_pages = fake_get_huge_pages,
 288         .put_pages = fake_put_huge_pages,
 289 };
 290
 291 static const struct drm_i915_gem_object_ops fake_ops_single = {
 292         .flags = I915_GEM_OBJECT_IS_SHRINKABLE,
 293         .get_pages = fake_get_huge_pages_single,
 294         .put_pages = fake_put_huge_pages,
 295 };
 296
 297 static struct drm_i915_gem_object *
 298 fake_huge_pages_object(struct drm_i915_private *i915, u64 size, bool single)
 299 {
 300         static struct lock_class_key lock_class;
 301         struct drm_i915_gem_object *obj;
 302
 303         GEM_BUG_ON(!size);
 304         GEM_BUG_ON(!IS_ALIGNED(size, I915_GTT_PAGE_SIZE));
 305
 306         if (size >> PAGE_SHIFT > UINT_MAX)
 307                 return ERR_PTR(-E2BIG);
 308
 309         if (overflows_type(size, obj->base.size))
 310                 return ERR_PTR(-E2BIG);
 311
 312         obj = i915_gem_object_alloc();
 313         if (!obj)
 314                 return ERR_PTR(-ENOMEM);
 315
 316         drm_gem_private_object_init(&i915->drm, &obj->base, size);
 317
 318         if (single)
 319                 i915_gem_object_init(obj, &fake_ops_single, &lock_class);
 320         else
 321                 i915_gem_object_init(obj, &fake_ops, &lock_class);
 322
 323         i915_gem_object_set_volatile(obj);
 324
 325         obj->write_domain = I915_GEM_DOMAIN_CPU;
 326         obj->read_domains = I915_GEM_DOMAIN_CPU;
 327         obj->cache_level = I915_CACHE_NONE;
 328
 329         return obj;
 330 }
 331
 332 static int igt_check_page_sizes(struct i915_vma *vma)
 333 {
 334         struct drm_i915_private *i915 = vma->vm->i915;
 335         unsigned int supported = INTEL_INFO(i915)->page_sizes;
 336         struct drm_i915_gem_object *obj = vma->obj;
 337         int err;
 338
 339         /* We have to wait for the async bind to complete before our asserts */
 340         err = i915_vma_sync(vma);
 341         if (err)
 342                 return err;
 343
 344         if (!HAS_PAGE_SIZES(i915, vma->page_sizes.sg)) {
 345                 pr_err("unsupported page_sizes.sg=%u, supported=%u\n",
 346                        vma->page_sizes.sg & ~supported, supported);
 347                 err = -EINVAL;
 348         }
 349
 350         if (!HAS_PAGE_SIZES(i915, vma->page_sizes.gtt)) {
 351                 pr_err("unsupported page_sizes.gtt=%u, supported=%u\n",
 352                        vma->page_sizes.gtt & ~supported, supported);
 353                 err = -EINVAL;
 354         }
 355
 356         if (vma->page_sizes.phys != obj->mm.page_sizes.phys) {
 357                 pr_err("vma->page_sizes.phys(%u) != obj->mm.page_sizes.phys(%u)\n",
 358                        vma->page_sizes.phys, obj->mm.page_sizes.phys);
 359                 err = -EINVAL;
 360         }
 361
 362         if (vma->page_sizes.sg != obj->mm.page_sizes.sg) {
 363                 pr_err("vma->page_sizes.sg(%u) != obj->mm.page_sizes.sg(%u)\n",
 364                        vma->page_sizes.sg, obj->mm.page_sizes.sg);
 365                 err = -EINVAL;
 366         }
 367
 368         if (obj->mm.page_sizes.gtt) {
 369                 pr_err("obj->page_sizes.gtt(%u) should never be set\n",
 370                        obj->mm.page_sizes.gtt);
 371                 err = -EINVAL;
 372         }
 373
 374         return err;
 375 }
 376
 377 static int igt_mock_exhaust_device_supported_pages(void *arg)
 378 {
 379         struct i915_ppgtt *ppgtt = arg;
 380         struct drm_i915_private *i915 = ppgtt->vm.i915;
 381         unsigned int saved_mask = INTEL_INFO(i915)->page_sizes;
 382         struct drm_i915_gem_object *obj;
 383         struct i915_vma *vma;
 384         int i, j, single;
 385         int err;
 386
 387         /*
 388          * Sanity check creating objects with every valid page support
 389          * combination for our mock device.
 390          */
 391
 392         for (i = 1; i < BIT(ARRAY_SIZE(page_sizes)); i++) {
 393                 unsigned int combination = 0;
 394
 395                 for (j = 0; j < ARRAY_SIZE(page_sizes); j++) {
 396                         if (i & BIT(j))
 397                                 combination |= page_sizes[j];
 398                 }
 399
 400                 mkwrite_device_info(i915)->page_sizes = combination;
 401
 402                 for (single = 0; single <= 1; ++single) {
 403                         obj = fake_huge_pages_object(i915, combination, !!single);
 404                         if (IS_ERR(obj)) {
 405                                 err = PTR_ERR(obj);
 406                                 goto out_device;
 407                         }
 408
 409                         if (obj->base.size != combination) {
 410                                 pr_err("obj->base.size=%zu, expected=%u\n",
 411                                        obj->base.size, combination);
 412                                 err = -EINVAL;
 413                                 goto out_put;
 414                         }
 415
 416                         vma = i915_vma_instance(obj, &ppgtt->vm, NULL);
 417                         if (IS_ERR(vma)) {
 418                                 err = PTR_ERR(vma);
 419                                 goto out_put;
 420                         }
 421
 422                         err = i915_vma_pin(vma, 0, 0, PIN_USER);
 423                         if (err)
 424                                 goto out_close;
 425
 426                         err = igt_check_page_sizes(vma);
 427
 428                         if (vma->page_sizes.sg != combination) {
 429                                 pr_err("page_sizes.sg=%u, expected=%u\n",
 430                                        vma->page_sizes.sg, combination);
 431                                 err = -EINVAL;
 432                         }
 433
 434                         i915_vma_unpin(vma);
 435                         i915_vma_close(vma);
 436
 437                         i915_gem_object_put(obj);
 438
 439                         if (err)
 440                                 goto out_device;
 441                 }
 442         }
 443
 444         goto out_device;
 445
 446 out_close:
 447         i915_vma_close(vma);
 448 out_put:
 449         i915_gem_object_put(obj);
 450 out_device:
 451         mkwrite_device_info(i915)->page_sizes = saved_mask;
 452
 453         return err;
 454 }
 455
 456 static int igt_mock_memory_region_huge_pages(void *arg)
 457 {
 458         const unsigned int flags[] = { 0, I915_BO_ALLOC_CONTIGUOUS };
 459         struct i915_ppgtt *ppgtt = arg;
 460         struct drm_i915_private *i915 = ppgtt->vm.i915;
 461         unsigned long supported = INTEL_INFO(i915)->page_sizes;
 462         struct intel_memory_region *mem;
 463         struct drm_i915_gem_object *obj;
 464         struct i915_vma *vma;
 465         int bit;
 466         int err = 0;
 467
 468         mem = mock_region_create(i915, 0, SZ_2G, I915_GTT_PAGE_SIZE_4K, 0);
 469         if (IS_ERR(mem)) {
 470                 pr_err("%s failed to create memory region\n", __func__);
 471                 return PTR_ERR(mem);
 472         }
 473
 474         for_each_set_bit(bit, &supported, ilog2(I915_GTT_MAX_PAGE_SIZE) + 1) {
 475                 unsigned int page_size = BIT(bit);
 476                 resource_size_t phys;
 477                 int i;
 478
 479                 for (i = 0; i < ARRAY_SIZE(flags); ++i) {
 480                         obj = i915_gem_object_create_region(mem, page_size,
 481                                                             flags[i]);
 482                         if (IS_ERR(obj)) {
 483                                 err = PTR_ERR(obj);
 484                                 goto out_region;
 485                         }
 486
 487                         vma = i915_vma_instance(obj, &ppgtt->vm, NULL);
 488                         if (IS_ERR(vma)) {
 489                                 err = PTR_ERR(vma);
 490                                 goto out_put;
 491                         }
 492
 493                         err = i915_vma_pin(vma, 0, 0, PIN_USER);
 494                         if (err)
 495                                 goto out_close;
 496
 497                         err = igt_check_page_sizes(vma);
 498                         if (err)
 499                                 goto out_unpin;
 500
 501                         phys = i915_gem_object_get_dma_address(obj, 0);
 502                         if (!IS_ALIGNED(phys, page_size)) {
 503                                 pr_err("%s addr misaligned(%pa) page_size=%u\n",
 504                                        __func__, &phys, page_size);
 505                                 err = -EINVAL;
 506                                 goto out_unpin;
 507                         }
 508
 509                         if (vma->page_sizes.gtt != page_size) {
 510                                 pr_err("%s page_sizes.gtt=%u, expected=%u\n",
 511                                        __func__, vma->page_sizes.gtt,
 512                                        page_size);
 513                                 err = -EINVAL;
 514                                 goto out_unpin;
 515                         }
 516
 517                         i915_vma_unpin(vma);
 518                         i915_vma_close(vma);
 519
 520                         __i915_gem_object_put_pages(obj);
 521                         i915_gem_object_put(obj);
 522                 }
 523         }
 524
 525         goto out_region;
 526
 527 out_unpin:
 528         i915_vma_unpin(vma);
 529 out_close:
 530         i915_vma_close(vma);
 531 out_put:
 532         i915_gem_object_put(obj);
 533 out_region:
 534         intel_memory_region_put(mem);
 535         return err;
 536 }
 537
 538 static int igt_mock_ppgtt_misaligned_dma(void *arg)
 539 {
 540         struct i915_ppgtt *ppgtt = arg;
 541         struct drm_i915_private *i915 = ppgtt->vm.i915;
 542         unsigned long supported = INTEL_INFO(i915)->page_sizes;
 543         struct drm_i915_gem_object *obj;
 544         int bit;
 545         int err;
 546
 547         /*
 548          * Sanity check dma misalignment for huge pages -- the dma addresses we
 549          * insert into the paging structures need to always respect the page
 550          * size alignment.
 551          */
 552
 553         bit = ilog2(I915_GTT_PAGE_SIZE_64K);
 554
 555         for_each_set_bit_from(bit, &supported,
 556                               ilog2(I915_GTT_MAX_PAGE_SIZE) + 1) {
 557                 IGT_TIMEOUT(end_time);
 558                 unsigned int page_size = BIT(bit);
 559                 unsigned int flags = PIN_USER | PIN_OFFSET_FIXED;
 560                 unsigned int offset;
 561                 unsigned int size =
 562                         round_up(page_size, I915_GTT_PAGE_SIZE_2M) << 1;
 563                 struct i915_vma *vma;
 564
 565                 obj = fake_huge_pages_object(i915, size, true);
 566                 if (IS_ERR(obj))
 567                         return PTR_ERR(obj);
 568
 569                 if (obj->base.size != size) {
 570                         pr_err("obj->base.size=%zu, expected=%u\n",
 571                                obj->base.size, size);
 572                         err = -EINVAL;
 573                         goto out_put;
 574                 }
 575
 576                 err = i915_gem_object_pin_pages(obj);
 577                 if (err)
 578                         goto out_put;
 579
 580                 /* Force the page size for this object */
 581                 obj->mm.page_sizes.sg = page_size;
 582
 583                 vma = i915_vma_instance(obj, &ppgtt->vm, NULL);
 584                 if (IS_ERR(vma)) {
 585                         err = PTR_ERR(vma);
 586                         goto out_unpin;
 587                 }
 588
 589                 err = i915_vma_pin(vma, 0, 0, flags);
 590                 if (err) {
 591                         i915_vma_close(vma);
 592                         goto out_unpin;
 593                 }
 594
 595
 596                 err = igt_check_page_sizes(vma);
 597
 598                 if (vma->page_sizes.gtt != page_size) {
 599                         pr_err("page_sizes.gtt=%u, expected %u\n",
 600                                vma->page_sizes.gtt, page_size);
 601                         err = -EINVAL;
 602                 }
 603
 604                 i915_vma_unpin(vma);
 605
 606                 if (err) {
 607                         i915_vma_close(vma);
 608                         goto out_unpin;
 609                 }
 610
 611                 /*
 612                  * Try all the other valid offsets until the next
 613                  * boundary -- should always fall back to using 4K
 614                  * pages.
 615                  */
 616                 for (offset = 4096; offset < page_size; offset += 4096) {
 617                         err = i915_vma_unbind(vma);
 618                         if (err) {
 619                                 i915_vma_close(vma);
 620                                 goto out_unpin;
 621                         }
 622
 623                         err = i915_vma_pin(vma, 0, 0, flags | offset);
 624                         if (err) {
 625                                 i915_vma_close(vma);
 626                                 goto out_unpin;
 627                         }
 628
 629                         err = igt_check_page_sizes(vma);
 630
 631                         if (vma->page_sizes.gtt != I915_GTT_PAGE_SIZE_4K) {
 632                                 pr_err("page_sizes.gtt=%u, expected %llu\n",
 633                                        vma->page_sizes.gtt, I915_GTT_PAGE_SIZE_4K);
 634                                 err = -EINVAL;
 635                         }
 636
 637                         i915_vma_unpin(vma);
 638
 639                         if (err) {
 640                                 i915_vma_close(vma);
 641                                 goto out_unpin;
 642                         }
 643
 644                         if (igt_timeout(end_time,
 645                                         "%s timed out at offset %x with page-size %x\n",
 646                                         __func__, offset, page_size))
 647                                 break;
 648                 }
 649
 650                 i915_vma_close(vma);
 651
 652                 i915_gem_object_unpin_pages(obj);
 653                 __i915_gem_object_put_pages(obj);
 654                 i915_gem_object_put(obj);
 655         }
 656
 657         return 0;
 658
 659 out_unpin:
 660         i915_gem_object_unpin_pages(obj);
 661 out_put:
 662         i915_gem_object_put(obj);
 663
 664         return err;
 665 }
 666
 667 static void close_object_list(struct list_head *objects,
 668                               struct i915_ppgtt *ppgtt)
 669 {
 670         struct drm_i915_gem_object *obj, *on;
 671
 672         list_for_each_entry_safe(obj, on, objects, st_link) {
 673                 struct i915_vma *vma;
 674
 675                 vma = i915_vma_instance(obj, &ppgtt->vm, NULL);
 676                 if (!IS_ERR(vma))
 677                         i915_vma_close(vma);
 678
 679                 list_del(&obj->st_link);
 680                 i915_gem_object_unpin_pages(obj);
 681                 __i915_gem_object_put_pages(obj);
 682                 i915_gem_object_put(obj);
 683         }
 684 }
 685
 686 static int igt_mock_ppgtt_huge_fill(void *arg)
 687 {
 688         struct i915_ppgtt *ppgtt = arg;
 689         struct drm_i915_private *i915 = ppgtt->vm.i915;
 690         unsigned long max_pages = ppgtt->vm.total >> PAGE_SHIFT;
 691         unsigned long page_num;
 692         bool single = false;
 693         LIST_HEAD(objects);
 694         IGT_TIMEOUT(end_time);
 695         int err = -ENODEV;
 696
 697         for_each_prime_number_from(page_num, 1, max_pages) {
 698                 struct drm_i915_gem_object *obj;
 699                 u64 size = page_num << PAGE_SHIFT;
 700                 struct i915_vma *vma;
 701                 unsigned int expected_gtt = 0;
 702                 int i;
 703
 704                 obj = fake_huge_pages_object(i915, size, single);
 705                 if (IS_ERR(obj)) {
 706                         err = PTR_ERR(obj);
 707                         break;
 708                 }
 709
 710                 if (obj->base.size != size) {
 711                         pr_err("obj->base.size=%zd, expected=%llu\n",
 712                                obj->base.size, size);
 713                         i915_gem_object_put(obj);
 714                         err = -EINVAL;
 715                         break;
 716                 }
 717
 718                 err = i915_gem_object_pin_pages(obj);
 719                 if (err) {
 720                         i915_gem_object_put(obj);
 721                         break;
 722                 }
 723
 724                 list_add(&obj->st_link, &objects);
 725
 726                 vma = i915_vma_instance(obj, &ppgtt->vm, NULL);
 727                 if (IS_ERR(vma)) {
 728                         err = PTR_ERR(vma);
 729                         break;
 730                 }
 731
 732                 err = i915_vma_pin(vma, 0, 0, PIN_USER);
 733                 if (err)
 734                         break;
 735
 736                 err = igt_check_page_sizes(vma);
 737                 if (err) {
 738                         i915_vma_unpin(vma);
 739                         break;
 740                 }
 741
 742                 /*
 743                  * Figure out the expected gtt page size knowing that we go from
 744                  * largest to smallest page size sg chunks, and that we align to
 745                  * the largest page size.
 746                  */
 747                 for (i = 0; i < ARRAY_SIZE(page_sizes); ++i) {
 748                         unsigned int page_size = page_sizes[i];
 749
 750                         if (HAS_PAGE_SIZES(i915, page_size) &&
 751                             size >= page_size) {
 752                                 expected_gtt |= page_size;
 753                                 size &= page_size-1;
 754                         }
 755                 }
 756
 757                 GEM_BUG_ON(!expected_gtt);
 758                 GEM_BUG_ON(size);
 759
 760                 if (expected_gtt & I915_GTT_PAGE_SIZE_4K)
 761                         expected_gtt &= ~I915_GTT_PAGE_SIZE_64K;
 762
 763                 i915_vma_unpin(vma);
 764
 765                 if (vma->page_sizes.sg & I915_GTT_PAGE_SIZE_64K) {
 766                         if (!IS_ALIGNED(vma->node.start,
 767                                         I915_GTT_PAGE_SIZE_2M)) {
 768                                 pr_err("node.start(%llx) not aligned to 2M\n",
 769                                        vma->node.start);
 770                                 err = -EINVAL;
 771                                 break;
 772                         }
 773
 774                         if (!IS_ALIGNED(vma->node.size,
 775                                         I915_GTT_PAGE_SIZE_2M)) {
 776                                 pr_err("node.size(%llx) not aligned to 2M\n",
 777                                        vma->node.size);
 778                                 err = -EINVAL;
 779                                 break;
 780                         }
 781                 }
 782
 783                 if (vma->page_sizes.gtt != expected_gtt) {
 784                         pr_err("gtt=%u, expected=%u, size=%zd, single=%s\n",
 785                                vma->page_sizes.gtt, expected_gtt,
 786                                obj->base.size, yesno(!!single));
 787                         err = -EINVAL;
 788                         break;
 789                 }
 790
 791                 if (igt_timeout(end_time,
 792                                 "%s timed out at size %zd\n",
 793                                 __func__, obj->base.size))
 794                         break;
 795
 796                 single = !single;
 797         }
 798
 799         close_object_list(&objects, ppgtt);
 800
 801         if (err == -ENOMEM || err == -ENOSPC)
 802                 err = 0;
 803
 804         return err;
 805 }
 806
 807 static int igt_mock_ppgtt_64K(void *arg)
 808 {
 809         struct i915_ppgtt *ppgtt = arg;
 810         struct drm_i915_private *i915 = ppgtt->vm.i915;
 811         struct drm_i915_gem_object *obj;
 812         const struct object_info {
 813                 unsigned int size;
 814                 unsigned int gtt;
 815                 unsigned int offset;
 816         } objects[] = {
 817                 /* Cases with forced padding/alignment */
 818                 {
 819                         .size = SZ_64K,
 820                         .gtt = I915_GTT_PAGE_SIZE_64K,
 821                         .offset = 0,
 822                 },
 823                 {
 824                         .size = SZ_64K + SZ_4K,
 825                         .gtt = I915_GTT_PAGE_SIZE_4K,
 826                         .offset = 0,
 827                 },
 828                 {
 829                         .size = SZ_64K - SZ_4K,
 830                         .gtt = I915_GTT_PAGE_SIZE_4K,
 831                         .offset = 0,
 832                 },
 833                 {
 834                         .size = SZ_2M,
 835                         .gtt = I915_GTT_PAGE_SIZE_64K,
 836                         .offset = 0,
 837                 },
 838                 {
 839                         .size = SZ_2M - SZ_4K,
 840                         .gtt = I915_GTT_PAGE_SIZE_4K,
 841                         .offset = 0,
 842                 },
 843                 {
 844                         .size = SZ_2M + SZ_4K,
 845                         .gtt = I915_GTT_PAGE_SIZE_64K | I915_GTT_PAGE_SIZE_4K,
 846                         .offset = 0,
 847                 },
 848                 {
 849                         .size = SZ_2M + SZ_64K,
 850                         .gtt = I915_GTT_PAGE_SIZE_64K,
 851                         .offset = 0,
 852                 },
 853                 {
 854                         .size = SZ_2M - SZ_64K,
 855                         .gtt = I915_GTT_PAGE_SIZE_64K,
 856                         .offset = 0,
 857                 },
 858                 /* Try without any forced padding/alignment */
 859                 {
 860                         .size = SZ_64K,
 861                         .offset = SZ_2M,
 862                         .gtt = I915_GTT_PAGE_SIZE_4K,
 863                 },
 864                 {
 865                         .size = SZ_128K,
 866                         .offset = SZ_2M - SZ_64K,
 867                         .gtt = I915_GTT_PAGE_SIZE_4K,
 868                 },
 869         };
 870         struct i915_vma *vma;
 871         int i, single;
 872         int err;
 873
 874         /*
 875          * Sanity check some of the trickiness with 64K pages -- either we can
 876          * safely mark the whole page-table(2M block) as 64K, or we have to
 877          * always fallback to 4K.
 878          */
 879
 880         if (!HAS_PAGE_SIZES(i915, I915_GTT_PAGE_SIZE_64K))
 881                 return 0;
 882
 883         for (i = 0; i < ARRAY_SIZE(objects); ++i) {
 884                 unsigned int size = objects[i].size;
 885                 unsigned int expected_gtt = objects[i].gtt;
 886                 unsigned int offset = objects[i].offset;
 887                 unsigned int flags = PIN_USER;
 888
 889                 for (single = 0; single <= 1; single++) {
 890                         obj = fake_huge_pages_object(i915, size, !!single);
 891                         if (IS_ERR(obj))
 892                                 return PTR_ERR(obj);
 893
 894                         err = i915_gem_object_pin_pages(obj);
 895                         if (err)
 896                                 goto out_object_put;
 897
 898                         /*
 899                          * Disable 2M pages -- We only want to use 64K/4K pages
 900                          * for this test.
 901                          */
 902                         obj->mm.page_sizes.sg &= ~I915_GTT_PAGE_SIZE_2M;
 903
 904                         vma = i915_vma_instance(obj, &ppgtt->vm, NULL);
 905                         if (IS_ERR(vma)) {
 906                                 err = PTR_ERR(vma);
 907                                 goto out_object_unpin;
 908                         }
 909
 910                         if (offset)
 911                                 flags |= PIN_OFFSET_FIXED | offset;
 912
 913                         err = i915_vma_pin(vma, 0, 0, flags);
 914                         if (err)
 915                                 goto out_vma_close;
 916
 917                         err = igt_check_page_sizes(vma);
 918                         if (err)
 919                                 goto out_vma_unpin;
 920
 921                         if (!offset && vma->page_sizes.sg & I915_GTT_PAGE_SIZE_64K) {
 922                                 if (!IS_ALIGNED(vma->node.start,
 923                                                 I915_GTT_PAGE_SIZE_2M)) {
 924                                         pr_err("node.start(%llx) not aligned to 2M\n",
 925                                                vma->node.start);
 926                                         err = -EINVAL;
 927                                         goto out_vma_unpin;
 928                                 }
 929
 930                                 if (!IS_ALIGNED(vma->node.size,
 931                                                 I915_GTT_PAGE_SIZE_2M)) {
 932                                         pr_err("node.size(%llx) not aligned to 2M\n",
 933                                                vma->node.size);
 934                                         err = -EINVAL;
 935                                         goto out_vma_unpin;
 936                                 }
 937                         }
 938
 939                         if (vma->page_sizes.gtt != expected_gtt) {
 940                                 pr_err("gtt=%u, expected=%u, i=%d, single=%s\n",
 941                                        vma->page_sizes.gtt, expected_gtt, i,
 942                                        yesno(!!single));
 943                                 err = -EINVAL;
 944                                 goto out_vma_unpin;
 945                         }
 946
 947                         i915_vma_unpin(vma);
 948                         i915_vma_close(vma);
 949
 950                         i915_gem_object_unpin_pages(obj);
 951                         __i915_gem_object_put_pages(obj);
 952                         i915_gem_object_put(obj);
 953                 }
 954         }
 955
 956         return 0;
 957
 958 out_vma_unpin:
 959         i915_vma_unpin(vma);
 960 out_vma_close:
 961         i915_vma_close(vma);
 962 out_object_unpin:
 963         i915_gem_object_unpin_pages(obj);
 964 out_object_put:
 965         i915_gem_object_put(obj);
 966
 967         return err;
 968 }
 969
 970 static int gpu_write(struct intel_context *ce,
 971                      struct i915_vma *vma,
 972                      u32 dw,
 973                      u32 val)
 974 {
 975         int err;
 976
 977         i915_gem_object_lock(vma->obj);
 978         err = i915_gem_object_set_to_gtt_domain(vma->obj, true);
 979         i915_gem_object_unlock(vma->obj);
 980         if (err)
 981                 return err;
 982
 983         return igt_gpu_fill_dw(ce, vma, dw * sizeof(u32),
 984                                vma->size >> PAGE_SHIFT, val);
 985 }
 986
 987 static int
 988 __cpu_check_shmem(struct drm_i915_gem_object *obj, u32 dword, u32 val)
 989 {
 990         unsigned int needs_flush;
 991         unsigned long n;
 992         int err;
 993
 994         err = i915_gem_object_prepare_read(obj, &needs_flush);
 995         if (err)
 996                 return err;
 997
 998         for (n = 0; n < obj->base.size >> PAGE_SHIFT; ++n) {
 999                 u32 *ptr = kmap_atomic(i915_gem_object_get_page(obj, n));
1000
1001                 if (needs_flush & CLFLUSH_BEFORE)
1002                         drm_clflush_virt_range(ptr, PAGE_SIZE);
1003
1004                 if (ptr[dword] != val) {
1005                         pr_err("n=%lu ptr[%u]=%u, val=%u\n",
1006                                n, dword, ptr[dword], val);
1007                         kunmap_atomic(ptr);
1008                         err = -EINVAL;
1009                         break;
1010                 }
1011
1012                 kunmap_atomic(ptr);
1013         }
1014
1015         i915_gem_object_finish_access(obj);
1016
1017         return err;
1018 }
1019
1020 static int __cpu_check_vmap(struct drm_i915_gem_object *obj, u32 dword, u32 val)
1021 {
1022         unsigned long n = obj->base.size >> PAGE_SHIFT;
1023         u32 *ptr;
1024         int err;
1025
1026         err = i915_gem_object_wait(obj, 0, MAX_SCHEDULE_TIMEOUT);
1027         if (err)
1028                 return err;
1029
1030         ptr = i915_gem_object_pin_map(obj, I915_MAP_WC);
1031         if (IS_ERR(ptr))
1032                 return PTR_ERR(ptr);
1033
1034         ptr += dword;
1035         while (n--) {
1036                 if (*ptr != val) {
1037                         pr_err("base[%u]=%08x, val=%08x\n",
1038                                dword, *ptr, val);
1039                         err = -EINVAL;
1040                         break;
1041                 }
1042
1043                 ptr += PAGE_SIZE / sizeof(*ptr);
1044         }
1045
1046         i915_gem_object_unpin_map(obj);
1047         return err;
1048 }
1049
1050 static int cpu_check(struct drm_i915_gem_object *obj, u32 dword, u32 val)
1051 {
1052         if (i915_gem_object_has_struct_page(obj))
1053                 return __cpu_check_shmem(obj, dword, val);
1054         else
1055                 return __cpu_check_vmap(obj, dword, val);
1056 }
1057
1058 static int __igt_write_huge(struct intel_context *ce,
1059                             struct drm_i915_gem_object *obj,
1060                             u64 size, u64 offset,
1061                             u32 dword, u32 val)
1062 {
1063         unsigned int flags = PIN_USER | PIN_OFFSET_FIXED;
1064         struct i915_vma *vma;
1065         int err;
1066
1067         vma = i915_vma_instance(obj, ce->vm, NULL);
1068         if (IS_ERR(vma))
1069                 return PTR_ERR(vma);
1070
1071         err = i915_vma_unbind(vma);
1072         if (err)
1073                 goto out_vma_close;
1074
1075         err = i915_vma_pin(vma, size, 0, flags | offset);
1076         if (err) {
1077                 /*
1078                  * The ggtt may have some pages reserved so
1079                  * refrain from erroring out.
1080                  */
1081                 if (err == -ENOSPC && i915_is_ggtt(ce->vm))
1082                         err = 0;
1083
1084                 goto out_vma_close;
1085         }
1086
1087         err = igt_check_page_sizes(vma);
1088         if (err)
1089                 goto out_vma_unpin;
1090
1091         err = gpu_write(ce, vma, dword, val);
1092         if (err) {
1093                 pr_err("gpu-write failed at offset=%llx\n", offset);
1094                 goto out_vma_unpin;
1095         }
1096
1097         err = cpu_check(obj, dword, val);
1098         if (err) {
1099                 pr_err("cpu-check failed at offset=%llx\n", offset);
1100                 goto out_vma_unpin;
1101         }
1102
1103 out_vma_unpin:
1104         i915_vma_unpin(vma);
1105 out_vma_close:
1106         __i915_vma_put(vma);
1107         return err;
1108 }
1109
1110 static int igt_write_huge(struct i915_gem_context *ctx,
1111                           struct drm_i915_gem_object *obj)
1112 {
1113         struct i915_gem_engines *engines;
1114         struct i915_gem_engines_iter it;
1115         struct intel_context *ce;
1116         I915_RND_STATE(prng);
1117         IGT_TIMEOUT(end_time);
1118         unsigned int max_page_size;
1119         unsigned int count;
1120         u64 max;
1121         u64 num;
1122         u64 size;
1123         int *order;
1124         int i, n;
1125         int err = 0;
1126
1127         GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj));
1128
1129         size = obj->base.size;
1130         if (obj->mm.page_sizes.sg & I915_GTT_PAGE_SIZE_64K)
1131                 size = round_up(size, I915_GTT_PAGE_SIZE_2M);
1132
1133         n = 0;
1134         count = 0;
1135         max = U64_MAX;
1136         for_each_gem_engine(ce, i915_gem_context_lock_engines(ctx), it) {
1137                 count++;
1138                 if (!intel_engine_can_store_dword(ce->engine))
1139                         continue;
1140
1141                 max = min(max, ce->vm->total);
1142                 n++;
1143         }
1144         i915_gem_context_unlock_engines(ctx);
1145         if (!n)
1146                 return 0;
1147
1148         /*
1149          * To keep things interesting when alternating between engines in our
1150          * randomized order, lets also make feeding to the same engine a few
1151          * times in succession a possibility by enlarging the permutation array.
1152          */
1153         order = i915_random_order(count * count, &prng);
1154         if (!order)
1155                 return -ENOMEM;
1156
1157         max_page_size = rounddown_pow_of_two(obj->mm.page_sizes.sg);
1158         max = div_u64(max - size, max_page_size);
1159
1160         /*
1161          * Try various offsets in an ascending/descending fashion until we
1162          * timeout -- we want to avoid issues hidden by effectively always using
1163          * offset = 0.
1164          */
1165         i = 0;
1166         engines = i915_gem_context_lock_engines(ctx);
1167         for_each_prime_number_from(num, 0, max) {
1168                 u64 offset_low = num * max_page_size;
1169                 u64 offset_high = (max - num) * max_page_size;
1170                 u32 dword = offset_in_page(num) / 4;
1171                 struct intel_context *ce;
1172
1173                 ce = engines->engines[order[i] % engines->num_engines];
1174                 i = (i + 1) % (count * count);
1175                 if (!ce || !intel_engine_can_store_dword(ce->engine))
1176                         continue;
1177
1178                 /*
1179                  * In order to utilize 64K pages we need to both pad the vma
1180                  * size and ensure the vma offset is at the start of the pt
1181                  * boundary, however to improve coverage we opt for testing both
1182                  * aligned and unaligned offsets.
1183                  */
1184                 if (obj->mm.page_sizes.sg & I915_GTT_PAGE_SIZE_64K)
1185                         offset_low = round_down(offset_low,
1186                                                 I915_GTT_PAGE_SIZE_2M);
1187
1188                 err = __igt_write_huge(ce, obj, size, offset_low,
1189                                        dword, num + 1);
1190                 if (err)
1191                         break;
1192
1193                 err = __igt_write_huge(ce, obj, size, offset_high,
1194                                        dword, num + 1);
1195                 if (err)
1196                         break;
1197
1198                 if (igt_timeout(end_time,
1199                                 "%s timed out on %s, offset_low=%llx offset_high=%llx, max_page_size=%x\n",
1200                                 __func__, ce->engine->name, offset_low, offset_high,
1201                                 max_page_size))
1202                         break;
1203         }
1204         i915_gem_context_unlock_engines(ctx);
1205
1206         kfree(order);
1207
1208         return err;
1209 }
1210
1211 static int igt_ppgtt_exhaust_huge(void *arg)
1212 {
1213         struct i915_gem_context *ctx = arg;
1214         struct drm_i915_private *i915 = ctx->i915;
1215         unsigned long supported = INTEL_INFO(i915)->page_sizes;
1216         static unsigned int pages[ARRAY_SIZE(page_sizes)];
1217         struct drm_i915_gem_object *obj;
1218         unsigned int size_mask;
1219         unsigned int page_mask;
1220         int n, i;
1221         int err = -ENODEV;
1222
1223         if (supported == I915_GTT_PAGE_SIZE_4K)
1224                 return 0;
1225
1226         /*
1227          * Sanity check creating objects with a varying mix of page sizes --
1228          * ensuring that our writes lands in the right place.
1229          */
1230
1231         n = 0;
1232         for_each_set_bit(i, &supported, ilog2(I915_GTT_MAX_PAGE_SIZE) + 1)
1233                 pages[n++] = BIT(i);
1234
1235         for (size_mask = 2; size_mask < BIT(n); size_mask++) {
1236                 unsigned int size = 0;
1237
1238                 for (i = 0; i < n; i++) {
1239                         if (size_mask & BIT(i))
1240                                 size |= pages[i];
1241                 }
1242
1243                 /*
1244                  * For our page mask we want to enumerate all the page-size
1245                  * combinations which will fit into our chosen object size.
1246                  */
1247                 for (page_mask = 2; page_mask <= size_mask; page_mask++) {
1248                         unsigned int page_sizes = 0;
1249
1250                         for (i = 0; i < n; i++) {
1251                                 if (page_mask & BIT(i))
1252                                         page_sizes |= pages[i];
1253                         }
1254
1255                         /*
1256                          * Ensure that we can actually fill the given object
1257                          * with our chosen page mask.
1258                          */
1259                         if (!IS_ALIGNED(size, BIT(__ffs(page_sizes))))
1260                                 continue;
1261
1262                         obj = huge_pages_object(i915, size, page_sizes);
1263                         if (IS_ERR(obj)) {
1264                                 err = PTR_ERR(obj);
1265                                 goto out_device;
1266                         }
1267
1268                         err = i915_gem_object_pin_pages(obj);
1269                         if (err) {
1270                                 i915_gem_object_put(obj);
1271
1272                                 if (err == -ENOMEM) {
1273                                         pr_info("unable to get pages, size=%u, pages=%u\n",
1274                                                 size, page_sizes);
1275                                         err = 0;
1276                                         break;
1277                                 }
1278
1279                                 pr_err("pin_pages failed, size=%u, pages=%u\n",
1280                                        size_mask, page_mask);
1281
1282                                 goto out_device;
1283                         }
1284
1285                         /* Force the page-size for the gtt insertion */
1286                         obj->mm.page_sizes.sg = page_sizes;
1287
1288                         err = igt_write_huge(ctx, obj);
1289                         if (err) {
1290                                 pr_err("exhaust write-huge failed with size=%u\n",
1291                                        size);
1292                                 goto out_unpin;
1293                         }
1294
1295                         i915_gem_object_unpin_pages(obj);
1296                         __i915_gem_object_put_pages(obj);
1297                         i915_gem_object_put(obj);
1298                 }
1299         }
1300
1301         goto out_device;
1302
1303 out_unpin:
1304         i915_gem_object_unpin_pages(obj);
1305         i915_gem_object_put(obj);
1306 out_device:
1307         mkwrite_device_info(i915)->page_sizes = supported;
1308
1309         return err;
1310 }
1311
1312 typedef struct drm_i915_gem_object *
1313 (*igt_create_fn)(struct drm_i915_private *i915, u32 size, u32 flags);
1314
1315 static inline bool igt_can_allocate_thp(struct drm_i915_private *i915)
1316 {
1317         return i915->mm.gemfs && has_transparent_hugepage();
1318 }
1319
1320 static struct drm_i915_gem_object *
1321 igt_create_shmem(struct drm_i915_private *i915, u32 size, u32 flags)
1322 {
1323         if (!igt_can_allocate_thp(i915)) {
1324                 pr_info("%s missing THP support, skipping\n", __func__);
1325                 return ERR_PTR(-ENODEV);
1326         }
1327
1328         return i915_gem_object_create_shmem(i915, size);
1329 }
1330
1331 static struct drm_i915_gem_object *
1332 igt_create_internal(struct drm_i915_private *i915, u32 size, u32 flags)
1333 {
1334         return i915_gem_object_create_internal(i915, size);
1335 }
1336
1337 static struct drm_i915_gem_object *
1338 igt_create_system(struct drm_i915_private *i915, u32 size, u32 flags)
1339 {
1340         return huge_pages_object(i915, size, size);
1341 }
1342
1343 static struct drm_i915_gem_object *
1344 igt_create_local(struct drm_i915_private *i915, u32 size, u32 flags)
1345 {
1346         return i915_gem_object_create_lmem(i915, size, flags);
1347 }
1348
1349 static u32 igt_random_size(struct rnd_state *prng,
1350                            u32 min_page_size,
1351                            u32 max_page_size)
1352 {
1353         u64 mask;
1354         u32 size;
1355
1356         GEM_BUG_ON(!is_power_of_2(min_page_size));
1357         GEM_BUG_ON(!is_power_of_2(max_page_size));
1358         GEM_BUG_ON(min_page_size < PAGE_SIZE);
1359         GEM_BUG_ON(min_page_size > max_page_size);
1360
1361         mask = ((max_page_size << 1ULL) - 1) & PAGE_MASK;
1362         size = prandom_u32_state(prng) & mask;
1363         if (size < min_page_size)
1364                 size |= min_page_size;
1365
1366         return size;
1367 }
1368
1369 static int igt_ppgtt_smoke_huge(void *arg)
1370 {
1371         struct i915_gem_context *ctx = arg;
1372         struct drm_i915_private *i915 = ctx->i915;
1373         struct drm_i915_gem_object *obj;
1374         I915_RND_STATE(prng);
1375         struct {
1376                 igt_create_fn fn;
1377                 u32 min;
1378                 u32 max;
1379         } backends[] = {
1380                 { igt_create_internal, SZ_64K, SZ_2M,  },
1381                 { igt_create_shmem,    SZ_64K, SZ_32M, },
1382                 { igt_create_local,    SZ_64K, SZ_1G,  },
1383         };
1384         int err;
1385         int i;
1386
1387         /*
1388          * Sanity check that the HW uses huge pages correctly through our
1389          * various backends -- ensure that our writes land in the right place.
1390          */
1391
1392         for (i = 0; i < ARRAY_SIZE(backends); ++i) {
1393                 u32 min = backends[i].min;
1394                 u32 max = backends[i].max;
1395                 u32 size = max;
1396 try_again:
1397                 size = igt_random_size(&prng, min, rounddown_pow_of_two(size));
1398
1399                 obj = backends[i].fn(i915, size, 0);
1400                 if (IS_ERR(obj)) {
1401                         err = PTR_ERR(obj);
1402                         if (err == -E2BIG) {
1403                                 size >>= 1;
1404                                 goto try_again;
1405                         } else if (err == -ENODEV) {
1406                                 err = 0;
1407                                 continue;
1408                         }
1409
1410                         return err;
1411                 }
1412
1413                 err = i915_gem_object_pin_pages(obj);
1414                 if (err) {
1415                         if (err == -ENXIO || err == -E2BIG) {
1416                                 i915_gem_object_put(obj);
1417                                 size >>= 1;
1418                                 goto try_again;
1419                         }
1420                         goto out_put;
1421                 }
1422
1423                 if (obj->mm.page_sizes.phys < min) {
1424                         pr_info("%s unable to allocate huge-page(s) with size=%u, i=%d\n",
1425                                 __func__, size, i);
1426                         err = -ENOMEM;
1427                         goto out_unpin;
1428                 }
1429
1430                 err = igt_write_huge(ctx, obj);
1431                 if (err) {
1432                         pr_err("%s write-huge failed with size=%u, i=%d\n",
1433                                __func__, size, i);
1434                 }
1435 out_unpin:
1436                 i915_gem_object_unpin_pages(obj);
1437                 __i915_gem_object_put_pages(obj);
1438 out_put:
1439                 i915_gem_object_put(obj);
1440
1441                 if (err == -ENOMEM || err == -ENXIO)
1442                         err = 0;
1443
1444                 if (err)
1445                         break;
1446
1447                 cond_resched();
1448         }
1449
1450         return err;
1451 }
1452
1453 static int igt_ppgtt_sanity_check(void *arg)
1454 {
1455         struct i915_gem_context *ctx = arg;
1456         struct drm_i915_private *i915 = ctx->i915;
1457         unsigned int supported = INTEL_INFO(i915)->page_sizes;
1458         struct {
1459                 igt_create_fn fn;
1460                 unsigned int flags;
1461         } backends[] = {
1462                 { igt_create_system, 0,                        },
1463                 { igt_create_local,  I915_BO_ALLOC_CONTIGUOUS, },
1464         };
1465         struct {
1466                 u32 size;
1467                 u32 pages;
1468         } combos[] = {
1469                 { SZ_64K,               SZ_64K          },
1470                 { SZ_2M,                SZ_2M           },
1471                 { SZ_2M,                SZ_64K          },
1472                 { SZ_2M - SZ_64K,       SZ_64K          },
1473                 { SZ_2M - SZ_4K,        SZ_64K | SZ_4K  },
1474                 { SZ_2M + SZ_4K,        SZ_64K | SZ_4K  },
1475                 { SZ_2M + SZ_4K,        SZ_2M  | SZ_4K  },
1476                 { SZ_2M + SZ_64K,       SZ_2M  | SZ_64K },
1477         };
1478         int i, j;
1479         int err;
1480
1481         if (supported == I915_GTT_PAGE_SIZE_4K)
1482                 return 0;
1483
1484         /*
1485          * Sanity check that the HW behaves with a limited set of combinations.
1486          * We already have a bunch of randomised testing, which should give us
1487          * a decent amount of variation between runs, however we should keep
1488          * this to limit the chances of introducing a temporary regression, by
1489          * testing the most obvious cases that might make something blow up.
1490          */
1491
1492         for (i = 0; i < ARRAY_SIZE(backends); ++i) {
1493                 for (j = 0; j < ARRAY_SIZE(combos); ++j) {
1494                         struct drm_i915_gem_object *obj;
1495                         u32 size = combos[j].size;
1496                         u32 pages = combos[j].pages;
1497
1498                         obj = backends[i].fn(i915, size, backends[i].flags);
1499                         if (IS_ERR(obj)) {
1500                                 err = PTR_ERR(obj);
1501                                 if (err == -ENODEV) {
1502                                         pr_info("Device lacks local memory, skipping\n");
1503                                         err = 0;
1504                                         break;
1505                                 }
1506
1507                                 return err;
1508                         }
1509
1510                         err = i915_gem_object_pin_pages(obj);
1511                         if (err) {
1512                                 i915_gem_object_put(obj);
1513                                 goto out;
1514                         }
1515
1516                         GEM_BUG_ON(pages > obj->base.size);
1517                         pages = pages & supported;
1518
1519                         if (pages)
1520                                 obj->mm.page_sizes.sg = pages;
1521
1522                         err = igt_write_huge(ctx, obj);
1523
1524                         i915_gem_object_unpin_pages(obj);
1525                         __i915_gem_object_put_pages(obj);
1526                         i915_gem_object_put(obj);
1527
1528                         if (err) {
1529                                 pr_err("%s write-huge failed with size=%u pages=%u i=%d, j=%d\n",
1530                                        __func__, size, pages, i, j);
1531                                 goto out;
1532                         }
1533                 }
1534
1535                 cond_resched();
1536         }
1537
1538 out:
1539         if (err == -ENOMEM)
1540                 err = 0;
1541
1542         return err;
1543 }
1544
1545 static int igt_ppgtt_pin_update(void *arg)
1546 {
1547         struct i915_gem_context *ctx = arg;
1548         struct drm_i915_private *dev_priv = ctx->i915;
1549         unsigned long supported = INTEL_INFO(dev_priv)->page_sizes;
1550         struct drm_i915_gem_object *obj;
1551         struct i915_gem_engines_iter it;
1552         struct i915_address_space *vm;
1553         struct intel_context *ce;
1554         struct i915_vma *vma;
1555         unsigned int flags = PIN_USER | PIN_OFFSET_FIXED;
1556         unsigned int n;
1557         int first, last;
1558         int err = 0;
1559
1560         /*
1561          * Make sure there's no funny business when doing a PIN_UPDATE -- in the
1562          * past we had a subtle issue with being able to incorrectly do multiple
1563          * alloc va ranges on the same object when doing a PIN_UPDATE, which
1564          * resulted in some pretty nasty bugs, though only when using
1565          * huge-gtt-pages.
1566          */
1567
1568         vm = i915_gem_context_get_vm_rcu(ctx);
1569         if (!i915_vm_is_4lvl(vm)) {
1570                 pr_info("48b PPGTT not supported, skipping\n");
1571                 goto out_vm;
1572         }
1573
1574         first = ilog2(I915_GTT_PAGE_SIZE_64K);
1575         last = ilog2(I915_GTT_PAGE_SIZE_2M);
1576
1577         for_each_set_bit_from(first, &supported, last + 1) {
1578                 unsigned int page_size = BIT(first);
1579
1580                 obj = i915_gem_object_create_internal(dev_priv, page_size);
1581                 if (IS_ERR(obj))
1582                         return PTR_ERR(obj);
1583
1584                 vma = i915_vma_instance(obj, vm, NULL);
1585                 if (IS_ERR(vma)) {
1586                         err = PTR_ERR(vma);
1587                         goto out_put;
1588                 }
1589
1590                 err = i915_vma_pin(vma, SZ_2M, 0, flags);
1591                 if (err)
1592                         goto out_close;
1593
1594                 if (vma->page_sizes.sg < page_size) {
1595                         pr_info("Unable to allocate page-size %x, finishing test early\n",
1596                                 page_size);
1597                         goto out_unpin;
1598                 }
1599
1600                 err = igt_check_page_sizes(vma);
1601                 if (err)
1602                         goto out_unpin;
1603
1604                 if (vma->page_sizes.gtt != page_size) {
1605                         dma_addr_t addr = i915_gem_object_get_dma_address(obj, 0);
1606
1607                         /*
1608                          * The only valid reason for this to ever fail would be
1609                          * if the dma-mapper screwed us over when we did the
1610                          * dma_map_sg(), since it has the final say over the dma
1611                          * address.
1612                          */
1613                         if (IS_ALIGNED(addr, page_size)) {
1614                                 pr_err("page_sizes.gtt=%u, expected=%u\n",
1615                                        vma->page_sizes.gtt, page_size);
1616                                 err = -EINVAL;
1617                         } else {
1618                                 pr_info("dma address misaligned, finishing test early\n");
1619                         }
1620
1621                         goto out_unpin;
1622                 }
1623
1624                 err = i915_vma_bind(vma, I915_CACHE_NONE, PIN_UPDATE, NULL);
1625                 if (err)
1626                         goto out_unpin;
1627
1628                 i915_vma_unpin(vma);
1629                 i915_vma_close(vma);
1630
1631                 i915_gem_object_put(obj);
1632         }
1633
1634         obj = i915_gem_object_create_internal(dev_priv, PAGE_SIZE);
1635         if (IS_ERR(obj))
1636                 return PTR_ERR(obj);
1637
1638         vma = i915_vma_instance(obj, vm, NULL);
1639         if (IS_ERR(vma)) {
1640                 err = PTR_ERR(vma);
1641                 goto out_put;
1642         }
1643
1644         err = i915_vma_pin(vma, 0, 0, flags);
1645         if (err)
1646                 goto out_close;
1647
1648         /*
1649          * Make sure we don't end up with something like where the pde is still
1650          * pointing to the 2M page, and the pt we just filled-in is dangling --
1651          * we can check this by writing to the first page where it would then
1652          * land in the now stale 2M page.
1653          */
1654
1655         n = 0;
1656         for_each_gem_engine(ce, i915_gem_context_lock_engines(ctx), it) {
1657                 if (!intel_engine_can_store_dword(ce->engine))
1658                         continue;
1659
1660                 err = gpu_write(ce, vma, n++, 0xdeadbeaf);
1661                 if (err)
1662                         break;
1663         }
1664         i915_gem_context_unlock_engines(ctx);
1665         if (err)
1666                 goto out_unpin;
1667
1668         while (n--) {
1669                 err = cpu_check(obj, n, 0xdeadbeaf);
1670                 if (err)
1671                         goto out_unpin;
1672         }
1673
1674 out_unpin:
1675         i915_vma_unpin(vma);
1676 out_close:
1677         i915_vma_close(vma);
1678 out_put:
1679         i915_gem_object_put(obj);
1680 out_vm:
1681         i915_vm_put(vm);
1682
1683         return err;
1684 }
1685
1686 static int igt_tmpfs_fallback(void *arg)
1687 {
1688         struct i915_gem_context *ctx = arg;
1689         struct drm_i915_private *i915 = ctx->i915;
1690         struct vfsmount *gemfs = i915->mm.gemfs;
1691         struct i915_address_space *vm = i915_gem_context_get_vm_rcu(ctx);
1692         struct drm_i915_gem_object *obj;
1693         struct i915_vma *vma;
1694         u32 *vaddr;
1695         int err = 0;
1696
1697         /*
1698          * Make sure that we don't burst into a ball of flames upon falling back
1699          * to tmpfs, which we rely on if on the off-chance we encouter a failure
1700          * when setting up gemfs.
1701          */
1702
1703         i915->mm.gemfs = NULL;
1704
1705         obj = i915_gem_object_create_shmem(i915, PAGE_SIZE);
1706         if (IS_ERR(obj)) {
1707                 err = PTR_ERR(obj);
1708                 goto out_restore;
1709         }
1710
1711         vaddr = i915_gem_object_pin_map(obj, I915_MAP_WB);
1712         if (IS_ERR(vaddr)) {
1713                 err = PTR_ERR(vaddr);
1714                 goto out_put;
1715         }
1716         *vaddr = 0xdeadbeaf;
1717
1718         __i915_gem_object_flush_map(obj, 0, 64);
1719         i915_gem_object_unpin_map(obj);
1720
1721         vma = i915_vma_instance(obj, vm, NULL);
1722         if (IS_ERR(vma)) {
1723                 err = PTR_ERR(vma);
1724                 goto out_put;
1725         }
1726
1727         err = i915_vma_pin(vma, 0, 0, PIN_USER);
1728         if (err)
1729                 goto out_close;
1730
1731         err = igt_check_page_sizes(vma);
1732
1733         i915_vma_unpin(vma);
1734 out_close:
1735         i915_vma_close(vma);
1736 out_put:
1737         i915_gem_object_put(obj);
1738 out_restore:
1739         i915->mm.gemfs = gemfs;
1740
1741         i915_vm_put(vm);
1742         return err;
1743 }
1744
1745 static int igt_shrink_thp(void *arg)
1746 {
1747         struct i915_gem_context *ctx = arg;
1748         struct drm_i915_private *i915 = ctx->i915;
1749         struct i915_address_space *vm = i915_gem_context_get_vm_rcu(ctx);
1750         struct drm_i915_gem_object *obj;
1751         struct i915_gem_engines_iter it;
1752         struct intel_context *ce;
1753         struct i915_vma *vma;
1754         unsigned int flags = PIN_USER;
1755         unsigned int n;
1756         int err = 0;
1757
1758         /*
1759          * Sanity check shrinking huge-paged object -- make sure nothing blows
1760          * up.
1761          */
1762
1763         if (!igt_can_allocate_thp(i915)) {
1764                 pr_info("missing THP support, skipping\n");
1765                 goto out_vm;
1766         }
1767
1768         obj = i915_gem_object_create_shmem(i915, SZ_2M);
1769         if (IS_ERR(obj)) {
1770                 err = PTR_ERR(obj);
1771                 goto out_vm;
1772         }
1773
1774         vma = i915_vma_instance(obj, vm, NULL);
1775         if (IS_ERR(vma)) {
1776                 err = PTR_ERR(vma);
1777                 goto out_put;
1778         }
1779
1780         err = i915_vma_pin(vma, 0, 0, flags);
1781         if (err)
1782                 goto out_close;
1783
1784         if (obj->mm.page_sizes.phys < I915_GTT_PAGE_SIZE_2M) {
1785                 pr_info("failed to allocate THP, finishing test early\n");
1786                 goto out_unpin;
1787         }
1788
1789         err = igt_check_page_sizes(vma);
1790         if (err)
1791                 goto out_unpin;
1792
1793         n = 0;
1794
1795         for_each_gem_engine(ce, i915_gem_context_lock_engines(ctx), it) {
1796                 if (!intel_engine_can_store_dword(ce->engine))
1797                         continue;
1798
1799                 err = gpu_write(ce, vma, n++, 0xdeadbeaf);
1800                 if (err)
1801                         break;
1802         }
1803         i915_gem_context_unlock_engines(ctx);
1804         i915_vma_unpin(vma);
1805         if (err)
1806                 goto out_close;
1807
1808         /*
1809          * Now that the pages are *unpinned* shrink-all should invoke
1810          * shmem to truncate our pages.
1811          */
1812         i915_gem_shrink_all(i915);
1813         if (i915_gem_object_has_pages(obj)) {
1814                 pr_err("shrink-all didn't truncate the pages\n");
1815                 err = -EINVAL;
1816                 goto out_close;
1817         }
1818
1819         if (obj->mm.page_sizes.sg || obj->mm.page_sizes.phys) {
1820                 pr_err("residual page-size bits left\n");
1821                 err = -EINVAL;
1822                 goto out_close;
1823         }
1824
1825         err = i915_vma_pin(vma, 0, 0, flags);
1826         if (err)
1827                 goto out_close;
1828
1829         while (n--) {
1830                 err = cpu_check(obj, n, 0xdeadbeaf);
1831                 if (err)
1832                         break;
1833         }
1834
1835 out_unpin:
1836         i915_vma_unpin(vma);
1837 out_close:
1838         i915_vma_close(vma);
1839 out_put:
1840         i915_gem_object_put(obj);
1841 out_vm:
1842         i915_vm_put(vm);
1843
1844         return err;
1845 }
1846
1847 int i915_gem_huge_page_mock_selftests(void)
1848 {
1849         static const struct i915_subtest tests[] = {
1850                 SUBTEST(igt_mock_exhaust_device_supported_pages),
1851                 SUBTEST(igt_mock_memory_region_huge_pages),
1852                 SUBTEST(igt_mock_ppgtt_misaligned_dma),
1853                 SUBTEST(igt_mock_ppgtt_huge_fill),
1854                 SUBTEST(igt_mock_ppgtt_64K),
1855         };
1856         struct drm_i915_private *dev_priv;
1857         struct i915_ppgtt *ppgtt;
1858         int err;
1859
1860         dev_priv = mock_gem_device();
1861         if (!dev_priv)
1862                 return -ENOMEM;
1863
1864         /* Pretend to be a device which supports the 48b PPGTT */
1865         mkwrite_device_info(dev_priv)->ppgtt_type = INTEL_PPGTT_FULL;
1866         mkwrite_device_info(dev_priv)->ppgtt_size = 48;
1867
1868         ppgtt = i915_ppgtt_create(&dev_priv->gt);
1869         if (IS_ERR(ppgtt)) {
1870                 err = PTR_ERR(ppgtt);
1871                 goto out_unlock;
1872         }
1873
1874         if (!i915_vm_is_4lvl(&ppgtt->vm)) {
1875                 pr_err("failed to create 48b PPGTT\n");
1876                 err = -EINVAL;
1877                 goto out_close;
1878         }
1879
1880         /* If we were ever hit this then it's time to mock the 64K scratch */
1881         if (!i915_vm_has_scratch_64K(&ppgtt->vm)) {
1882                 pr_err("PPGTT missing 64K scratch page\n");
1883                 err = -EINVAL;
1884                 goto out_close;
1885         }
1886
1887         err = i915_subtests(tests, ppgtt);
1888
1889 out_close:
1890         i915_vm_put(&ppgtt->vm);
1891
1892 out_unlock:
1893         drm_dev_put(&dev_priv->drm);
1894         return err;
1895 }
1896
1897 int i915_gem_huge_page_live_selftests(struct drm_i915_private *i915)
1898 {
1899         static const struct i915_subtest tests[] = {
1900                 SUBTEST(igt_shrink_thp),
1901                 SUBTEST(igt_ppgtt_pin_update),
1902                 SUBTEST(igt_tmpfs_fallback),
1903                 SUBTEST(igt_ppgtt_exhaust_huge),
1904                 SUBTEST(igt_ppgtt_smoke_huge),
1905                 SUBTEST(igt_ppgtt_sanity_check),
1906         };
1907         struct i915_gem_context *ctx;
1908         struct i915_address_space *vm;
1909         struct file *file;
1910         int err;
1911
1912         if (!HAS_PPGTT(i915)) {
1913                 pr_info("PPGTT not supported, skipping live-selftests\n");
1914                 return 0;
1915         }
1916
1917         if (intel_gt_is_wedged(&i915->gt))
1918                 return 0;
1919
1920         file = mock_file(i915);
1921         if (IS_ERR(file))
1922                 return PTR_ERR(file);
1923
1924         ctx = live_context(i915, file);
1925         if (IS_ERR(ctx)) {
1926                 err = PTR_ERR(ctx);
1927                 goto out_file;
1928         }
1929
1930         mutex_lock(&ctx->mutex);
1931         vm = i915_gem_context_vm(ctx);
1932         if (vm)
1933                 WRITE_ONCE(vm->scrub_64K, true);
1934         mutex_unlock(&ctx->mutex);
1935
1936         err = i915_subtests(tests, ctx);
1937
1938 out_file:
1939         fput(file);
1940         return err;
1941 }