drivers/gpu/drm/nouveau/nouveau_uvmm.c

   1 // SPDX-License-Identifier: MIT
   2
   3 /*
   4  * Locking:
   5  *
   6  * The uvmm mutex protects any operations on the GPU VA space provided by the
   7  * DRM GPU VA manager.
   8  *
   9  * The GEMs dma_resv lock protects the GEMs GPUVA list, hence link/unlink of a
  10  * mapping to it's backing GEM must be performed under this lock.
  11  *
  12  * Actual map/unmap operations within the fence signalling critical path are
  13  * protected by installing DMA fences to the corresponding GEMs DMA
  14  * reservations, such that concurrent BO moves, which itself walk the GEMs GPUVA
  15  * list in order to map/unmap it's entries, can't occur concurrently.
  16  *
  17  * Accessing the DRM_GPUVA_INVALIDATED flag doesn't need any separate
  18  * protection, since there are no accesses other than from BO move callbacks
  19  * and from the fence signalling critical path, which are already protected by
  20  * the corresponding GEMs DMA reservation fence.
  21  */
  22
  23 #include "nouveau_drv.h"
  24 #include "nouveau_gem.h"
  25 #include "nouveau_mem.h"
  26 #include "nouveau_uvmm.h"
  27
  28 #include <nvif/vmm.h>
  29 #include <nvif/mem.h>
  30
  31 #include <nvif/class.h>
  32 #include <nvif/if000c.h>
  33 #include <nvif/if900d.h>
  34
  35 #define NOUVEAU_VA_SPACE_BITS           47 /* FIXME */
  36 #define NOUVEAU_VA_SPACE_START          0x0
  37 #define NOUVEAU_VA_SPACE_END            (1ULL << NOUVEAU_VA_SPACE_BITS)
  38
  39 #define list_last_op(_ops) list_last_entry(_ops, struct bind_job_op, entry)
  40 #define list_prev_op(_op) list_prev_entry(_op, entry)
  41 #define list_for_each_op(_op, _ops) list_for_each_entry(_op, _ops, entry)
  42 #define list_for_each_op_from_reverse(_op, _ops) \
  43         list_for_each_entry_from_reverse(_op, _ops, entry)
  44 #define list_for_each_op_safe(_op, _n, _ops) list_for_each_entry_safe(_op, _n, _ops, entry)
  45
  46 enum vm_bind_op {
  47         OP_MAP = DRM_NOUVEAU_VM_BIND_OP_MAP,
  48         OP_UNMAP = DRM_NOUVEAU_VM_BIND_OP_UNMAP,
  49         OP_MAP_SPARSE,
  50         OP_UNMAP_SPARSE,
  51 };
  52
  53 struct nouveau_uvma_prealloc {
  54         struct nouveau_uvma *map;
  55         struct nouveau_uvma *prev;
  56         struct nouveau_uvma *next;
  57 };
  58
  59 struct bind_job_op {
  60         struct list_head entry;
  61
  62         enum vm_bind_op op;
  63         u32 flags;
  64
  65         struct drm_gpuvm_bo *vm_bo;
  66
  67         struct {
  68                 u64 addr;
  69                 u64 range;
  70         } va;
  71
  72         struct {
  73                 u32 handle;
  74                 u64 offset;
  75                 struct drm_gem_object *obj;
  76         } gem;
  77
  78         struct nouveau_uvma_region *reg;
  79         struct nouveau_uvma_prealloc new;
  80         struct drm_gpuva_ops *ops;
  81 };
  82
  83 struct uvmm_map_args {
  84         struct nouveau_uvma_region *region;
  85         u64 addr;
  86         u64 range;
  87         u8 kind;
  88 };
  89
  90 static int
  91 nouveau_uvmm_vmm_sparse_ref(struct nouveau_uvmm *uvmm,
  92                             u64 addr, u64 range)
  93 {
  94         struct nvif_vmm *vmm = &uvmm->vmm.vmm;
  95
  96         return nvif_vmm_raw_sparse(vmm, addr, range, true);
  97 }
  98
  99 static int
 100 nouveau_uvmm_vmm_sparse_unref(struct nouveau_uvmm *uvmm,
 101                               u64 addr, u64 range)
 102 {
 103         struct nvif_vmm *vmm = &uvmm->vmm.vmm;
 104
 105         return nvif_vmm_raw_sparse(vmm, addr, range, false);
 106 }
 107
 108 static int
 109 nouveau_uvmm_vmm_get(struct nouveau_uvmm *uvmm,
 110                      u64 addr, u64 range)
 111 {
 112         struct nvif_vmm *vmm = &uvmm->vmm.vmm;
 113
 114         return nvif_vmm_raw_get(vmm, addr, range, PAGE_SHIFT);
 115 }
 116
 117 static int
 118 nouveau_uvmm_vmm_put(struct nouveau_uvmm *uvmm,
 119                      u64 addr, u64 range)
 120 {
 121         struct nvif_vmm *vmm = &uvmm->vmm.vmm;
 122
 123         return nvif_vmm_raw_put(vmm, addr, range, PAGE_SHIFT);
 124 }
 125
 126 static int
 127 nouveau_uvmm_vmm_unmap(struct nouveau_uvmm *uvmm,
 128                        u64 addr, u64 range, bool sparse)
 129 {
 130         struct nvif_vmm *vmm = &uvmm->vmm.vmm;
 131
 132         return nvif_vmm_raw_unmap(vmm, addr, range, PAGE_SHIFT, sparse);
 133 }
 134
 135 static int
 136 nouveau_uvmm_vmm_map(struct nouveau_uvmm *uvmm,
 137                      u64 addr, u64 range,
 138                      u64 bo_offset, u8 kind,
 139                      struct nouveau_mem *mem)
 140 {
 141         struct nvif_vmm *vmm = &uvmm->vmm.vmm;
 142         union {
 143                 struct gf100_vmm_map_v0 gf100;
 144         } args;
 145         u32 argc = 0;
 146
 147         switch (vmm->object.oclass) {
 148         case NVIF_CLASS_VMM_GF100:
 149         case NVIF_CLASS_VMM_GM200:
 150         case NVIF_CLASS_VMM_GP100:
 151                 args.gf100.version = 0;
 152                 if (mem->mem.type & NVIF_MEM_VRAM)
 153                         args.gf100.vol = 0;
 154                 else
 155                         args.gf100.vol = 1;
 156                 args.gf100.ro = 0;
 157                 args.gf100.priv = 0;
 158                 args.gf100.kind = kind;
 159                 argc = sizeof(args.gf100);
 160                 break;
 161         default:
 162                 WARN_ON(1);
 163                 return -ENOSYS;
 164         }
 165
 166         return nvif_vmm_raw_map(vmm, addr, range, PAGE_SHIFT,
 167                                 &args, argc,
 168                                 &mem->mem, bo_offset);
 169 }
 170
 171 static int
 172 nouveau_uvma_region_sparse_unref(struct nouveau_uvma_region *reg)
 173 {
 174         u64 addr = reg->va.addr;
 175         u64 range = reg->va.range;
 176
 177         return nouveau_uvmm_vmm_sparse_unref(reg->uvmm, addr, range);
 178 }
 179
 180 static int
 181 nouveau_uvma_vmm_put(struct nouveau_uvma *uvma)
 182 {
 183         u64 addr = uvma->va.va.addr;
 184         u64 range = uvma->va.va.range;
 185
 186         return nouveau_uvmm_vmm_put(to_uvmm(uvma), addr, range);
 187 }
 188
 189 static int
 190 nouveau_uvma_map(struct nouveau_uvma *uvma,
 191                  struct nouveau_mem *mem)
 192 {
 193         u64 addr = uvma->va.va.addr;
 194         u64 offset = uvma->va.gem.offset;
 195         u64 range = uvma->va.va.range;
 196
 197         return nouveau_uvmm_vmm_map(to_uvmm(uvma), addr, range,
 198                                     offset, uvma->kind, mem);
 199 }
 200
 201 static int
 202 nouveau_uvma_unmap(struct nouveau_uvma *uvma)
 203 {
 204         u64 addr = uvma->va.va.addr;
 205         u64 range = uvma->va.va.range;
 206         bool sparse = !!uvma->region;
 207
 208         if (drm_gpuva_invalidated(&uvma->va))
 209                 return 0;
 210
 211         return nouveau_uvmm_vmm_unmap(to_uvmm(uvma), addr, range, sparse);
 212 }
 213
 214 static int
 215 nouveau_uvma_alloc(struct nouveau_uvma **puvma)
 216 {
 217         *puvma = kzalloc(sizeof(**puvma), GFP_KERNEL);
 218         if (!*puvma)
 219                 return -ENOMEM;
 220
 221         return 0;
 222 }
 223
 224 static void
 225 nouveau_uvma_free(struct nouveau_uvma *uvma)
 226 {
 227         kfree(uvma);
 228 }
 229
 230 static void
 231 nouveau_uvma_gem_get(struct nouveau_uvma *uvma)
 232 {
 233         drm_gem_object_get(uvma->va.gem.obj);
 234 }
 235
 236 static void
 237 nouveau_uvma_gem_put(struct nouveau_uvma *uvma)
 238 {
 239         drm_gem_object_put(uvma->va.gem.obj);
 240 }
 241
 242 static int
 243 nouveau_uvma_region_alloc(struct nouveau_uvma_region **preg)
 244 {
 245         *preg = kzalloc(sizeof(**preg), GFP_KERNEL);
 246         if (!*preg)
 247                 return -ENOMEM;
 248
 249         kref_init(&(*preg)->kref);
 250
 251         return 0;
 252 }
 253
 254 static void
 255 nouveau_uvma_region_free(struct kref *kref)
 256 {
 257         struct nouveau_uvma_region *reg =
 258                 container_of(kref, struct nouveau_uvma_region, kref);
 259
 260         kfree(reg);
 261 }
 262
 263 static void
 264 nouveau_uvma_region_get(struct nouveau_uvma_region *reg)
 265 {
 266         kref_get(&reg->kref);
 267 }
 268
 269 static void
 270 nouveau_uvma_region_put(struct nouveau_uvma_region *reg)
 271 {
 272         kref_put(&reg->kref, nouveau_uvma_region_free);
 273 }
 274
 275 static int
 276 __nouveau_uvma_region_insert(struct nouveau_uvmm *uvmm,
 277                              struct nouveau_uvma_region *reg)
 278 {
 279         u64 addr = reg->va.addr;
 280         u64 range = reg->va.range;
 281         u64 last = addr + range - 1;
 282         MA_STATE(mas, &uvmm->region_mt, addr, addr);
 283
 284         if (unlikely(mas_walk(&mas)))
 285                 return -EEXIST;
 286
 287         if (unlikely(mas.last < last))
 288                 return -EEXIST;
 289
 290         mas.index = addr;
 291         mas.last = last;
 292
 293         mas_store_gfp(&mas, reg, GFP_KERNEL);
 294
 295         reg->uvmm = uvmm;
 296
 297         return 0;
 298 }
 299
 300 static int
 301 nouveau_uvma_region_insert(struct nouveau_uvmm *uvmm,
 302                            struct nouveau_uvma_region *reg,
 303                            u64 addr, u64 range)
 304 {
 305         int ret;
 306
 307         reg->uvmm = uvmm;
 308         reg->va.addr = addr;
 309         reg->va.range = range;
 310
 311         ret = __nouveau_uvma_region_insert(uvmm, reg);
 312         if (ret)
 313                 return ret;
 314
 315         return 0;
 316 }
 317
 318 static void
 319 nouveau_uvma_region_remove(struct nouveau_uvma_region *reg)
 320 {
 321         struct nouveau_uvmm *uvmm = reg->uvmm;
 322         MA_STATE(mas, &uvmm->region_mt, reg->va.addr, 0);
 323
 324         mas_erase(&mas);
 325 }
 326
 327 static int
 328 nouveau_uvma_region_create(struct nouveau_uvmm *uvmm,
 329                            u64 addr, u64 range)
 330 {
 331         struct nouveau_uvma_region *reg;
 332         int ret;
 333
 334         if (!drm_gpuvm_interval_empty(&uvmm->base, addr, range))
 335                 return -ENOSPC;
 336
 337         ret = nouveau_uvma_region_alloc(&reg);
 338         if (ret)
 339                 return ret;
 340
 341         ret = nouveau_uvma_region_insert(uvmm, reg, addr, range);
 342         if (ret)
 343                 goto err_free_region;
 344
 345         ret = nouveau_uvmm_vmm_sparse_ref(uvmm, addr, range);
 346         if (ret)
 347                 goto err_region_remove;
 348
 349         return 0;
 350
 351 err_region_remove:
 352         nouveau_uvma_region_remove(reg);
 353 err_free_region:
 354         nouveau_uvma_region_put(reg);
 355         return ret;
 356 }
 357
 358 static struct nouveau_uvma_region *
 359 nouveau_uvma_region_find_first(struct nouveau_uvmm *uvmm,
 360                                u64 addr, u64 range)
 361 {
 362         MA_STATE(mas, &uvmm->region_mt, addr, 0);
 363
 364         return mas_find(&mas, addr + range - 1);
 365 }
 366
 367 static struct nouveau_uvma_region *
 368 nouveau_uvma_region_find(struct nouveau_uvmm *uvmm,
 369                          u64 addr, u64 range)
 370 {
 371         struct nouveau_uvma_region *reg;
 372
 373         reg = nouveau_uvma_region_find_first(uvmm, addr, range);
 374         if (!reg)
 375                 return NULL;
 376
 377         if (reg->va.addr != addr ||
 378             reg->va.range != range)
 379                 return NULL;
 380
 381         return reg;
 382 }
 383
 384 static bool
 385 nouveau_uvma_region_empty(struct nouveau_uvma_region *reg)
 386 {
 387         struct nouveau_uvmm *uvmm = reg->uvmm;
 388
 389         return drm_gpuvm_interval_empty(&uvmm->base,
 390                                         reg->va.addr,
 391                                         reg->va.range);
 392 }
 393
 394 static int
 395 __nouveau_uvma_region_destroy(struct nouveau_uvma_region *reg)
 396 {
 397         struct nouveau_uvmm *uvmm = reg->uvmm;
 398         u64 addr = reg->va.addr;
 399         u64 range = reg->va.range;
 400
 401         if (!nouveau_uvma_region_empty(reg))
 402                 return -EBUSY;
 403
 404         nouveau_uvma_region_remove(reg);
 405         nouveau_uvmm_vmm_sparse_unref(uvmm, addr, range);
 406         nouveau_uvma_region_put(reg);
 407
 408         return 0;
 409 }
 410
 411 static int
 412 nouveau_uvma_region_destroy(struct nouveau_uvmm *uvmm,
 413                             u64 addr, u64 range)
 414 {
 415         struct nouveau_uvma_region *reg;
 416
 417         reg = nouveau_uvma_region_find(uvmm, addr, range);
 418         if (!reg)
 419                 return -ENOENT;
 420
 421         return __nouveau_uvma_region_destroy(reg);
 422 }
 423
 424 static void
 425 nouveau_uvma_region_dirty(struct nouveau_uvma_region *reg)
 426 {
 427
 428         init_completion(&reg->complete);
 429         reg->dirty = true;
 430 }
 431
 432 static void
 433 nouveau_uvma_region_complete(struct nouveau_uvma_region *reg)
 434 {
 435         complete_all(&reg->complete);
 436 }
 437
 438 static void
 439 op_map_prepare_unwind(struct nouveau_uvma *uvma)
 440 {
 441         struct drm_gpuva *va = &uvma->va;
 442         nouveau_uvma_gem_put(uvma);
 443         drm_gpuva_remove(va);
 444         nouveau_uvma_free(uvma);
 445 }
 446
 447 static void
 448 op_unmap_prepare_unwind(struct drm_gpuva *va)
 449 {
 450         drm_gpuva_insert(va->vm, va);
 451 }
 452
 453 static void
 454 nouveau_uvmm_sm_prepare_unwind(struct nouveau_uvmm *uvmm,
 455                                struct nouveau_uvma_prealloc *new,
 456                                struct drm_gpuva_ops *ops,
 457                                struct drm_gpuva_op *last,
 458                                struct uvmm_map_args *args)
 459 {
 460         struct drm_gpuva_op *op = last;
 461         u64 vmm_get_start = args ? args->addr : 0;
 462         u64 vmm_get_end = args ? args->addr + args->range : 0;
 463
 464         /* Unwind GPUVA space. */
 465         drm_gpuva_for_each_op_from_reverse(op, ops) {
 466                 switch (op->op) {
 467                 case DRM_GPUVA_OP_MAP:
 468                         op_map_prepare_unwind(new->map);
 469                         break;
 470                 case DRM_GPUVA_OP_REMAP: {
 471                         struct drm_gpuva_op_remap *r = &op->remap;
 472                         struct drm_gpuva *va = r->unmap->va;
 473
 474                         if (r->next)
 475                                 op_map_prepare_unwind(new->next);
 476
 477                         if (r->prev)
 478                                 op_map_prepare_unwind(new->prev);
 479
 480                         op_unmap_prepare_unwind(va);
 481                         break;
 482                 }
 483                 case DRM_GPUVA_OP_UNMAP:
 484                         op_unmap_prepare_unwind(op->unmap.va);
 485                         break;
 486                 default:
 487                         break;
 488                 }
 489         }
 490
 491         /* Unmap operation don't allocate page tables, hence skip the following
 492          * page table unwind.
 493          */
 494         if (!args)
 495                 return;
 496
 497         drm_gpuva_for_each_op(op, ops) {
 498                 switch (op->op) {
 499                 case DRM_GPUVA_OP_MAP: {
 500                         u64 vmm_get_range = vmm_get_end - vmm_get_start;
 501
 502                         if (vmm_get_range)
 503                                 nouveau_uvmm_vmm_put(uvmm, vmm_get_start,
 504                                                      vmm_get_range);
 505                         break;
 506                 }
 507                 case DRM_GPUVA_OP_REMAP: {
 508                         struct drm_gpuva_op_remap *r = &op->remap;
 509                         struct drm_gpuva *va = r->unmap->va;
 510                         u64 ustart = va->va.addr;
 511                         u64 urange = va->va.range;
 512                         u64 uend = ustart + urange;
 513
 514                         if (r->prev)
 515                                 vmm_get_start = uend;
 516
 517                         if (r->next)
 518                                 vmm_get_end = ustart;
 519
 520                         if (r->prev && r->next)
 521                                 vmm_get_start = vmm_get_end = 0;
 522
 523                         break;
 524                 }
 525                 case DRM_GPUVA_OP_UNMAP: {
 526                         struct drm_gpuva_op_unmap *u = &op->unmap;
 527                         struct drm_gpuva *va = u->va;
 528                         u64 ustart = va->va.addr;
 529                         u64 urange = va->va.range;
 530                         u64 uend = ustart + urange;
 531
 532                         /* Nothing to do for mappings we merge with. */
 533                         if (uend == vmm_get_start ||
 534                             ustart == vmm_get_end)
 535                                 break;
 536
 537                         if (ustart > vmm_get_start) {
 538                                 u64 vmm_get_range = ustart - vmm_get_start;
 539
 540                                 nouveau_uvmm_vmm_put(uvmm, vmm_get_start,
 541                                                      vmm_get_range);
 542                         }
 543                         vmm_get_start = uend;
 544                         break;
 545                 }
 546                 default:
 547                         break;
 548                 }
 549
 550                 if (op == last)
 551                         break;
 552         }
 553 }
 554
 555 static void
 556 nouveau_uvmm_sm_map_prepare_unwind(struct nouveau_uvmm *uvmm,
 557                                    struct nouveau_uvma_prealloc *new,
 558                                    struct drm_gpuva_ops *ops,
 559                                    u64 addr, u64 range)
 560 {
 561         struct drm_gpuva_op *last = drm_gpuva_last_op(ops);
 562         struct uvmm_map_args args = {
 563                 .addr = addr,
 564                 .range = range,
 565         };
 566
 567         nouveau_uvmm_sm_prepare_unwind(uvmm, new, ops, last, &args);
 568 }
 569
 570 static void
 571 nouveau_uvmm_sm_unmap_prepare_unwind(struct nouveau_uvmm *uvmm,
 572                                      struct nouveau_uvma_prealloc *new,
 573                                      struct drm_gpuva_ops *ops)
 574 {
 575         struct drm_gpuva_op *last = drm_gpuva_last_op(ops);
 576
 577         nouveau_uvmm_sm_prepare_unwind(uvmm, new, ops, last, NULL);
 578 }
 579
 580 static int
 581 op_map_prepare(struct nouveau_uvmm *uvmm,
 582                struct nouveau_uvma **puvma,
 583                struct drm_gpuva_op_map *op,
 584                struct uvmm_map_args *args)
 585 {
 586         struct nouveau_uvma *uvma;
 587         int ret;
 588
 589         ret = nouveau_uvma_alloc(&uvma);
 590         if (ret)
 591                 return ret;
 592
 593         uvma->region = args->region;
 594         uvma->kind = args->kind;
 595
 596         drm_gpuva_map(&uvmm->base, &uvma->va, op);
 597
 598         /* Keep a reference until this uvma is destroyed. */
 599         nouveau_uvma_gem_get(uvma);
 600
 601         *puvma = uvma;
 602         return 0;
 603 }
 604
 605 static void
 606 op_unmap_prepare(struct drm_gpuva_op_unmap *u)
 607 {
 608         drm_gpuva_unmap(u);
 609 }
 610
 611 /*
 612  * Note: @args should not be NULL when calling for a map operation.
 613  */
 614 static int
 615 nouveau_uvmm_sm_prepare(struct nouveau_uvmm *uvmm,
 616                         struct nouveau_uvma_prealloc *new,
 617                         struct drm_gpuva_ops *ops,
 618                         struct uvmm_map_args *args)
 619 {
 620         struct drm_gpuva_op *op;
 621         u64 vmm_get_start = args ? args->addr : 0;
 622         u64 vmm_get_end = args ? args->addr + args->range : 0;
 623         int ret;
 624
 625         drm_gpuva_for_each_op(op, ops) {
 626                 switch (op->op) {
 627                 case DRM_GPUVA_OP_MAP: {
 628                         u64 vmm_get_range = vmm_get_end - vmm_get_start;
 629
 630                         ret = op_map_prepare(uvmm, &new->map, &op->map, args);
 631                         if (ret)
 632                                 goto unwind;
 633
 634                         if (vmm_get_range) {
 635                                 ret = nouveau_uvmm_vmm_get(uvmm, vmm_get_start,
 636                                                            vmm_get_range);
 637                                 if (ret) {
 638                                         op_map_prepare_unwind(new->map);
 639                                         goto unwind;
 640                                 }
 641                         }
 642
 643                         break;
 644                 }
 645                 case DRM_GPUVA_OP_REMAP: {
 646                         struct drm_gpuva_op_remap *r = &op->remap;
 647                         struct drm_gpuva *va = r->unmap->va;
 648                         struct uvmm_map_args remap_args = {
 649                                 .kind = uvma_from_va(va)->kind,
 650                                 .region = uvma_from_va(va)->region,
 651                         };
 652                         u64 ustart = va->va.addr;
 653                         u64 urange = va->va.range;
 654                         u64 uend = ustart + urange;
 655
 656                         op_unmap_prepare(r->unmap);
 657
 658                         if (r->prev) {
 659                                 ret = op_map_prepare(uvmm, &new->prev, r->prev,
 660                                                      &remap_args);
 661                                 if (ret)
 662                                         goto unwind;
 663
 664                                 if (args)
 665                                         vmm_get_start = uend;
 666                         }
 667
 668                         if (r->next) {
 669                                 ret = op_map_prepare(uvmm, &new->next, r->next,
 670                                                      &remap_args);
 671                                 if (ret) {
 672                                         if (r->prev)
 673                                                 op_map_prepare_unwind(new->prev);
 674                                         goto unwind;
 675                                 }
 676
 677                                 if (args)
 678                                         vmm_get_end = ustart;
 679                         }
 680
 681                         if (args && (r->prev && r->next))
 682                                 vmm_get_start = vmm_get_end = 0;
 683
 684                         break;
 685                 }
 686                 case DRM_GPUVA_OP_UNMAP: {
 687                         struct drm_gpuva_op_unmap *u = &op->unmap;
 688                         struct drm_gpuva *va = u->va;
 689                         u64 ustart = va->va.addr;
 690                         u64 urange = va->va.range;
 691                         u64 uend = ustart + urange;
 692
 693                         op_unmap_prepare(u);
 694
 695                         if (!args)
 696                                 break;
 697
 698                         /* Nothing to do for mappings we merge with. */
 699                         if (uend == vmm_get_start ||
 700                             ustart == vmm_get_end)
 701                                 break;
 702
 703                         if (ustart > vmm_get_start) {
 704                                 u64 vmm_get_range = ustart - vmm_get_start;
 705
 706                                 ret = nouveau_uvmm_vmm_get(uvmm, vmm_get_start,
 707                                                            vmm_get_range);
 708                                 if (ret) {
 709                                         op_unmap_prepare_unwind(va);
 710                                         goto unwind;
 711                                 }
 712                         }
 713                         vmm_get_start = uend;
 714
 715                         break;
 716                 }
 717                 default:
 718                         ret = -EINVAL;
 719                         goto unwind;
 720                 }
 721         }
 722
 723         return 0;
 724
 725 unwind:
 726         if (op != drm_gpuva_first_op(ops))
 727                 nouveau_uvmm_sm_prepare_unwind(uvmm, new, ops,
 728                                                drm_gpuva_prev_op(op),
 729                                                args);
 730         return ret;
 731 }
 732
 733 static int
 734 nouveau_uvmm_sm_map_prepare(struct nouveau_uvmm *uvmm,
 735                             struct nouveau_uvma_prealloc *new,
 736                             struct nouveau_uvma_region *region,
 737                             struct drm_gpuva_ops *ops,
 738                             u64 addr, u64 range, u8 kind)
 739 {
 740         struct uvmm_map_args args = {
 741                 .region = region,
 742                 .addr = addr,
 743                 .range = range,
 744                 .kind = kind,
 745         };
 746
 747         return nouveau_uvmm_sm_prepare(uvmm, new, ops, &args);
 748 }
 749
 750 static int
 751 nouveau_uvmm_sm_unmap_prepare(struct nouveau_uvmm *uvmm,
 752                               struct nouveau_uvma_prealloc *new,
 753                               struct drm_gpuva_ops *ops)
 754 {
 755         return nouveau_uvmm_sm_prepare(uvmm, new, ops, NULL);
 756 }
 757
 758 static struct drm_gem_object *
 759 op_gem_obj(struct drm_gpuva_op *op)
 760 {
 761         switch (op->op) {
 762         case DRM_GPUVA_OP_MAP:
 763                 return op->map.gem.obj;
 764         case DRM_GPUVA_OP_REMAP:
 765                 /* Actually, we're looking for the GEMs backing remap.prev and
 766                  * remap.next, but since this is a remap they're identical to
 767                  * the GEM backing the unmapped GPUVA.
 768                  */
 769                 return op->remap.unmap->va->gem.obj;
 770         case DRM_GPUVA_OP_UNMAP:
 771                 return op->unmap.va->gem.obj;
 772         default:
 773                 WARN(1, "Unknown operation.\n");
 774                 return NULL;
 775         }
 776 }
 777
 778 static void
 779 op_map(struct nouveau_uvma *uvma)
 780 {
 781         struct nouveau_bo *nvbo = nouveau_gem_object(uvma->va.gem.obj);
 782
 783         nouveau_uvma_map(uvma, nouveau_mem(nvbo->bo.resource));
 784 }
 785
 786 static void
 787 op_unmap(struct drm_gpuva_op_unmap *u)
 788 {
 789         struct drm_gpuva *va = u->va;
 790         struct nouveau_uvma *uvma = uvma_from_va(va);
 791
 792         /* nouveau_uvma_unmap() does not unmap if backing BO is evicted. */
 793         if (!u->keep)
 794                 nouveau_uvma_unmap(uvma);
 795 }
 796
 797 static void
 798 op_unmap_range(struct drm_gpuva_op_unmap *u,
 799                u64 addr, u64 range)
 800 {
 801         struct nouveau_uvma *uvma = uvma_from_va(u->va);
 802         bool sparse = !!uvma->region;
 803
 804         if (!drm_gpuva_invalidated(u->va))
 805                 nouveau_uvmm_vmm_unmap(to_uvmm(uvma), addr, range, sparse);
 806 }
 807
 808 static void
 809 op_remap(struct drm_gpuva_op_remap *r,
 810          struct nouveau_uvma_prealloc *new)
 811 {
 812         struct drm_gpuva_op_unmap *u = r->unmap;
 813         struct nouveau_uvma *uvma = uvma_from_va(u->va);
 814         u64 addr = uvma->va.va.addr;
 815         u64 range = uvma->va.va.range;
 816
 817         if (r->prev)
 818                 addr = r->prev->va.addr + r->prev->va.range;
 819
 820         if (r->next)
 821                 range = r->next->va.addr - addr;
 822
 823         op_unmap_range(u, addr, range);
 824 }
 825
 826 static int
 827 nouveau_uvmm_sm(struct nouveau_uvmm *uvmm,
 828                 struct nouveau_uvma_prealloc *new,
 829                 struct drm_gpuva_ops *ops)
 830 {
 831         struct drm_gpuva_op *op;
 832
 833         drm_gpuva_for_each_op(op, ops) {
 834                 switch (op->op) {
 835                 case DRM_GPUVA_OP_MAP:
 836                         op_map(new->map);
 837                         break;
 838                 case DRM_GPUVA_OP_REMAP:
 839                         op_remap(&op->remap, new);
 840                         break;
 841                 case DRM_GPUVA_OP_UNMAP:
 842                         op_unmap(&op->unmap);
 843                         break;
 844                 default:
 845                         break;
 846                 }
 847         }
 848
 849         return 0;
 850 }
 851
 852 static int
 853 nouveau_uvmm_sm_map(struct nouveau_uvmm *uvmm,
 854                     struct nouveau_uvma_prealloc *new,
 855                     struct drm_gpuva_ops *ops)
 856 {
 857         return nouveau_uvmm_sm(uvmm, new, ops);
 858 }
 859
 860 static int
 861 nouveau_uvmm_sm_unmap(struct nouveau_uvmm *uvmm,
 862                       struct nouveau_uvma_prealloc *new,
 863                       struct drm_gpuva_ops *ops)
 864 {
 865         return nouveau_uvmm_sm(uvmm, new, ops);
 866 }
 867
 868 static void
 869 nouveau_uvmm_sm_cleanup(struct nouveau_uvmm *uvmm,
 870                         struct nouveau_uvma_prealloc *new,
 871                         struct drm_gpuva_ops *ops, bool unmap)
 872 {
 873         struct drm_gpuva_op *op;
 874
 875         drm_gpuva_for_each_op(op, ops) {
 876                 switch (op->op) {
 877                 case DRM_GPUVA_OP_MAP:
 878                         break;
 879                 case DRM_GPUVA_OP_REMAP: {
 880                         struct drm_gpuva_op_remap *r = &op->remap;
 881                         struct drm_gpuva_op_map *p = r->prev;
 882                         struct drm_gpuva_op_map *n = r->next;
 883                         struct drm_gpuva *va = r->unmap->va;
 884                         struct nouveau_uvma *uvma = uvma_from_va(va);
 885
 886                         if (unmap) {
 887                                 u64 addr = va->va.addr;
 888                                 u64 end = addr + va->va.range;
 889
 890                                 if (p)
 891                                         addr = p->va.addr + p->va.range;
 892
 893                                 if (n)
 894                                         end = n->va.addr;
 895
 896                                 nouveau_uvmm_vmm_put(uvmm, addr, end - addr);
 897                         }
 898
 899                         nouveau_uvma_gem_put(uvma);
 900                         nouveau_uvma_free(uvma);
 901                         break;
 902                 }
 903                 case DRM_GPUVA_OP_UNMAP: {
 904                         struct drm_gpuva_op_unmap *u = &op->unmap;
 905                         struct drm_gpuva *va = u->va;
 906                         struct nouveau_uvma *uvma = uvma_from_va(va);
 907
 908                         if (unmap)
 909                                 nouveau_uvma_vmm_put(uvma);
 910
 911                         nouveau_uvma_gem_put(uvma);
 912                         nouveau_uvma_free(uvma);
 913                         break;
 914                 }
 915                 default:
 916                         break;
 917                 }
 918         }
 919 }
 920
 921 static void
 922 nouveau_uvmm_sm_map_cleanup(struct nouveau_uvmm *uvmm,
 923                             struct nouveau_uvma_prealloc *new,
 924                             struct drm_gpuva_ops *ops)
 925 {
 926         nouveau_uvmm_sm_cleanup(uvmm, new, ops, false);
 927 }
 928
 929 static void
 930 nouveau_uvmm_sm_unmap_cleanup(struct nouveau_uvmm *uvmm,
 931                               struct nouveau_uvma_prealloc *new,
 932                               struct drm_gpuva_ops *ops)
 933 {
 934         nouveau_uvmm_sm_cleanup(uvmm, new, ops, true);
 935 }
 936
 937 static int
 938 nouveau_uvmm_validate_range(struct nouveau_uvmm *uvmm, u64 addr, u64 range)
 939 {
 940         if (addr & ~PAGE_MASK)
 941                 return -EINVAL;
 942
 943         if (range & ~PAGE_MASK)
 944                 return -EINVAL;
 945
 946         if (!drm_gpuvm_range_valid(&uvmm->base, addr, range))
 947                 return -EINVAL;
 948
 949         return 0;
 950 }
 951
 952 static int
 953 nouveau_uvmm_bind_job_alloc(struct nouveau_uvmm_bind_job **pjob)
 954 {
 955         *pjob = kzalloc(sizeof(**pjob), GFP_KERNEL);
 956         if (!*pjob)
 957                 return -ENOMEM;
 958
 959         kref_init(&(*pjob)->kref);
 960
 961         return 0;
 962 }
 963
 964 static void
 965 nouveau_uvmm_bind_job_free(struct kref *kref)
 966 {
 967         struct nouveau_uvmm_bind_job *job =
 968                 container_of(kref, struct nouveau_uvmm_bind_job, kref);
 969         struct bind_job_op *op, *next;
 970
 971         list_for_each_op_safe(op, next, &job->ops) {
 972                 list_del(&op->entry);
 973                 kfree(op);
 974         }
 975
 976         nouveau_job_free(&job->base);
 977         kfree(job);
 978 }
 979
 980 static void
 981 nouveau_uvmm_bind_job_get(struct nouveau_uvmm_bind_job *job)
 982 {
 983         kref_get(&job->kref);
 984 }
 985
 986 static void
 987 nouveau_uvmm_bind_job_put(struct nouveau_uvmm_bind_job *job)
 988 {
 989         kref_put(&job->kref, nouveau_uvmm_bind_job_free);
 990 }
 991
 992 static int
 993 bind_validate_op(struct nouveau_job *job,
 994                  struct bind_job_op *op)
 995 {
 996         struct nouveau_uvmm *uvmm = nouveau_cli_uvmm(job->cli);
 997         struct drm_gem_object *obj = op->gem.obj;
 998
 999         if (op->op == OP_MAP) {
1000                 if (op->gem.offset & ~PAGE_MASK)
1001                         return -EINVAL;
1002
1003                 if (obj->size <= op->gem.offset)
1004                         return -EINVAL;
1005
1006                 if (op->va.range > (obj->size - op->gem.offset))
1007                         return -EINVAL;
1008         }
1009
1010         return nouveau_uvmm_validate_range(uvmm, op->va.addr, op->va.range);
1011 }
1012
1013 static void
1014 bind_validate_map_sparse(struct nouveau_job *job, u64 addr, u64 range)
1015 {
1016         struct nouveau_sched *sched = job->sched;
1017         struct nouveau_job *__job;
1018         struct bind_job_op *op;
1019         u64 end = addr + range;
1020
1021 again:
1022         spin_lock(&sched->job.list.lock);
1023         list_for_each_entry(__job, &sched->job.list.head, entry) {
1024                 struct nouveau_uvmm_bind_job *bind_job = to_uvmm_bind_job(__job);
1025
1026                 list_for_each_op(op, &bind_job->ops) {
1027                         if (op->op == OP_UNMAP) {
1028                                 u64 op_addr = op->va.addr;
1029                                 u64 op_end = op_addr + op->va.range;
1030
1031                                 if (!(end <= op_addr || addr >= op_end)) {
1032                                         nouveau_uvmm_bind_job_get(bind_job);
1033                                         spin_unlock(&sched->job.list.lock);
1034                                         wait_for_completion(&bind_job->complete);
1035                                         nouveau_uvmm_bind_job_put(bind_job);
1036                                         goto again;
1037                                 }
1038                         }
1039                 }
1040         }
1041         spin_unlock(&sched->job.list.lock);
1042 }
1043
1044 static int
1045 bind_validate_map_common(struct nouveau_job *job, u64 addr, u64 range,
1046                          bool sparse)
1047 {
1048         struct nouveau_uvmm *uvmm = nouveau_cli_uvmm(job->cli);
1049         struct nouveau_uvma_region *reg;
1050         u64 reg_addr, reg_end;
1051         u64 end = addr + range;
1052
1053 again:
1054         nouveau_uvmm_lock(uvmm);
1055         reg = nouveau_uvma_region_find_first(uvmm, addr, range);
1056         if (!reg) {
1057                 nouveau_uvmm_unlock(uvmm);
1058                 return 0;
1059         }
1060
1061         /* Generally, job submits are serialized, hence only
1062          * dirty regions can be modified concurrently.
1063          */
1064         if (reg->dirty) {
1065                 nouveau_uvma_region_get(reg);
1066                 nouveau_uvmm_unlock(uvmm);
1067                 wait_for_completion(&reg->complete);
1068                 nouveau_uvma_region_put(reg);
1069                 goto again;
1070         }
1071         nouveau_uvmm_unlock(uvmm);
1072
1073         if (sparse)
1074                 return -ENOSPC;
1075
1076         reg_addr = reg->va.addr;
1077         reg_end = reg_addr + reg->va.range;
1078
1079         /* Make sure the mapping is either outside of a
1080          * region or fully enclosed by a region.
1081          */
1082         if (reg_addr > addr || reg_end < end)
1083                 return -ENOSPC;
1084
1085         return 0;
1086 }
1087
1088 static int
1089 bind_validate_region(struct nouveau_job *job)
1090 {
1091         struct nouveau_uvmm_bind_job *bind_job = to_uvmm_bind_job(job);
1092         struct bind_job_op *op;
1093         int ret;
1094
1095         list_for_each_op(op, &bind_job->ops) {
1096                 u64 op_addr = op->va.addr;
1097                 u64 op_range = op->va.range;
1098                 bool sparse = false;
1099
1100                 switch (op->op) {
1101                 case OP_MAP_SPARSE:
1102                         sparse = true;
1103                         bind_validate_map_sparse(job, op_addr, op_range);
1104                         fallthrough;
1105                 case OP_MAP:
1106                         ret = bind_validate_map_common(job, op_addr, op_range,
1107                                                        sparse);
1108                         if (ret)
1109                                 return ret;
1110                         break;
1111                 default:
1112                         break;
1113                 }
1114         }
1115
1116         return 0;
1117 }
1118
1119 static void
1120 bind_link_gpuvas(struct bind_job_op *bop)
1121 {
1122         struct nouveau_uvma_prealloc *new = &bop->new;
1123         struct drm_gpuvm_bo *vm_bo = bop->vm_bo;
1124         struct drm_gpuva_ops *ops = bop->ops;
1125         struct drm_gpuva_op *op;
1126
1127         drm_gpuva_for_each_op(op, ops) {
1128                 switch (op->op) {
1129                 case DRM_GPUVA_OP_MAP:
1130                         drm_gpuva_link(&new->map->va, vm_bo);
1131                         break;
1132                 case DRM_GPUVA_OP_REMAP: {
1133                         struct drm_gpuva *va = op->remap.unmap->va;
1134
1135                         if (op->remap.prev)
1136                                 drm_gpuva_link(&new->prev->va, va->vm_bo);
1137                         if (op->remap.next)
1138                                 drm_gpuva_link(&new->next->va, va->vm_bo);
1139                         drm_gpuva_unlink(va);
1140                         break;
1141                 }
1142                 case DRM_GPUVA_OP_UNMAP:
1143                         drm_gpuva_unlink(op->unmap.va);
1144                         break;
1145                 default:
1146                         break;
1147                 }
1148         }
1149 }
1150
1151 static int
1152 bind_lock_validate(struct nouveau_job *job, struct drm_exec *exec,
1153                    unsigned int num_fences)
1154 {
1155         struct nouveau_uvmm_bind_job *bind_job = to_uvmm_bind_job(job);
1156         struct bind_job_op *op;
1157         int ret;
1158
1159         list_for_each_op(op, &bind_job->ops) {
1160                 struct drm_gpuva_op *va_op;
1161
1162                 if (!op->ops)
1163                         continue;
1164
1165                 drm_gpuva_for_each_op(va_op, op->ops) {
1166                         struct drm_gem_object *obj = op_gem_obj(va_op);
1167
1168                         if (unlikely(!obj))
1169                                 continue;
1170
1171                         ret = drm_exec_prepare_obj(exec, obj, num_fences);
1172                         if (ret)
1173                                 return ret;
1174
1175                         /* Don't validate GEMs backing mappings we're about to
1176                          * unmap, it's not worth the effort.
1177                          */
1178                         if (va_op->op == DRM_GPUVA_OP_UNMAP)
1179                                 continue;
1180
1181                         ret = nouveau_bo_validate(nouveau_gem_object(obj),
1182                                                   true, false);
1183                         if (ret)
1184                                 return ret;
1185                 }
1186         }
1187
1188         return 0;
1189 }
1190
1191 static int
1192 nouveau_uvmm_bind_job_submit(struct nouveau_job *job,
1193                              struct drm_gpuvm_exec *vme)
1194 {
1195         struct nouveau_uvmm *uvmm = nouveau_cli_uvmm(job->cli);
1196         struct nouveau_uvmm_bind_job *bind_job = to_uvmm_bind_job(job);
1197         struct drm_exec *exec = &vme->exec;
1198         struct bind_job_op *op;
1199         int ret;
1200
1201         list_for_each_op(op, &bind_job->ops) {
1202                 if (op->op == OP_MAP) {
1203                         struct drm_gem_object *obj = op->gem.obj =
1204                                 drm_gem_object_lookup(job->file_priv,
1205                                                       op->gem.handle);
1206                         if (!obj)
1207                                 return -ENOENT;
1208
1209                         dma_resv_lock(obj->resv, NULL);
1210                         op->vm_bo = drm_gpuvm_bo_obtain(&uvmm->base, obj);
1211                         dma_resv_unlock(obj->resv);
1212                         if (IS_ERR(op->vm_bo))
1213                                 return PTR_ERR(op->vm_bo);
1214
1215                         drm_gpuvm_bo_extobj_add(op->vm_bo);
1216                 }
1217
1218                 ret = bind_validate_op(job, op);
1219                 if (ret)
1220                         return ret;
1221         }
1222
1223         /* If a sparse region or mapping overlaps a dirty region, we need to
1224          * wait for the region to complete the unbind process. This is due to
1225          * how page table management is currently implemented. A future
1226          * implementation might change this.
1227          */
1228         ret = bind_validate_region(job);
1229         if (ret)
1230                 return ret;
1231
1232         /* Once we start modifying the GPU VA space we need to keep holding the
1233          * uvmm lock until we can't fail anymore. This is due to the set of GPU
1234          * VA space changes must appear atomically and we need to be able to
1235          * unwind all GPU VA space changes on failure.
1236          */
1237         nouveau_uvmm_lock(uvmm);
1238
1239         list_for_each_op(op, &bind_job->ops) {
1240                 switch (op->op) {
1241                 case OP_MAP_SPARSE:
1242                         ret = nouveau_uvma_region_create(uvmm,
1243                                                          op->va.addr,
1244                                                          op->va.range);
1245                         if (ret)
1246                                 goto unwind_continue;
1247
1248                         break;
1249                 case OP_UNMAP_SPARSE:
1250                         op->reg = nouveau_uvma_region_find(uvmm, op->va.addr,
1251                                                            op->va.range);
1252                         if (!op->reg || op->reg->dirty) {
1253                                 ret = -ENOENT;
1254                                 goto unwind_continue;
1255                         }
1256
1257                         op->ops = drm_gpuvm_sm_unmap_ops_create(&uvmm->base,
1258                                                                 op->va.addr,
1259                                                                 op->va.range);
1260                         if (IS_ERR(op->ops)) {
1261                                 ret = PTR_ERR(op->ops);
1262                                 goto unwind_continue;
1263                         }
1264
1265                         ret = nouveau_uvmm_sm_unmap_prepare(uvmm, &op->new,
1266                                                             op->ops);
1267                         if (ret) {
1268                                 drm_gpuva_ops_free(&uvmm->base, op->ops);
1269                                 op->ops = NULL;
1270                                 op->reg = NULL;
1271                                 goto unwind_continue;
1272                         }
1273
1274                         nouveau_uvma_region_dirty(op->reg);
1275
1276                         break;
1277                 case OP_MAP: {
1278                         struct nouveau_uvma_region *reg;
1279
1280                         reg = nouveau_uvma_region_find_first(uvmm,
1281                                                              op->va.addr,
1282                                                              op->va.range);
1283                         if (reg) {
1284                                 u64 reg_addr = reg->va.addr;
1285                                 u64 reg_end = reg_addr + reg->va.range;
1286                                 u64 op_addr = op->va.addr;
1287                                 u64 op_end = op_addr + op->va.range;
1288
1289                                 if (unlikely(reg->dirty)) {
1290                                         ret = -EINVAL;
1291                                         goto unwind_continue;
1292                                 }
1293
1294                                 /* Make sure the mapping is either outside of a
1295                                  * region or fully enclosed by a region.
1296                                  */
1297                                 if (reg_addr > op_addr || reg_end < op_end) {
1298                                         ret = -ENOSPC;
1299                                         goto unwind_continue;
1300                                 }
1301                         }
1302
1303                         op->ops = drm_gpuvm_sm_map_ops_create(&uvmm->base,
1304                                                               op->va.addr,
1305                                                               op->va.range,
1306                                                               op->gem.obj,
1307                                                               op->gem.offset);
1308                         if (IS_ERR(op->ops)) {
1309                                 ret = PTR_ERR(op->ops);
1310                                 goto unwind_continue;
1311                         }
1312
1313                         ret = nouveau_uvmm_sm_map_prepare(uvmm, &op->new,
1314                                                           reg, op->ops,
1315                                                           op->va.addr,
1316                                                           op->va.range,
1317                                                           op->flags & 0xff);
1318                         if (ret) {
1319                                 drm_gpuva_ops_free(&uvmm->base, op->ops);
1320                                 op->ops = NULL;
1321                                 goto unwind_continue;
1322                         }
1323
1324                         break;
1325                 }
1326                 case OP_UNMAP:
1327                         op->ops = drm_gpuvm_sm_unmap_ops_create(&uvmm->base,
1328                                                                 op->va.addr,
1329                                                                 op->va.range);
1330                         if (IS_ERR(op->ops)) {
1331                                 ret = PTR_ERR(op->ops);
1332                                 goto unwind_continue;
1333                         }
1334
1335                         ret = nouveau_uvmm_sm_unmap_prepare(uvmm, &op->new,
1336                                                             op->ops);
1337                         if (ret) {
1338                                 drm_gpuva_ops_free(&uvmm->base, op->ops);
1339                                 op->ops = NULL;
1340                                 goto unwind_continue;
1341                         }
1342
1343                         break;
1344                 default:
1345                         ret = -EINVAL;
1346                         goto unwind_continue;
1347                 }
1348         }
1349
1350         drm_exec_init(exec, vme->flags, 0);
1351         drm_exec_until_all_locked(exec) {
1352                 ret = bind_lock_validate(job, exec, vme->num_fences);
1353                 drm_exec_retry_on_contention(exec);
1354                 if (ret) {
1355                         op = list_last_op(&bind_job->ops);
1356                         goto unwind;
1357                 }
1358         }
1359
1360         /* Link and unlink GPUVAs while holding the dma_resv lock.
1361          *
1362          * As long as we validate() all GEMs and add fences to all GEMs DMA
1363          * reservations backing map and remap operations we can be sure there
1364          * won't be any concurrent (in)validations during job execution, hence
1365          * we're safe to check drm_gpuva_invalidated() within the fence
1366          * signalling critical path without holding a separate lock.
1367          *
1368          * GPUVAs about to be unmapped are safe as well, since they're unlinked
1369          * already.
1370          *
1371          * GEMs from map and remap operations must be validated before linking
1372          * their corresponding mappings to prevent the actual PT update to
1373          * happen right away in validate() rather than asynchronously as
1374          * intended.
1375          *
1376          * Note that after linking and unlinking the GPUVAs in this loop this
1377          * function cannot fail anymore, hence there is no need for an unwind
1378          * path.
1379          */
1380         list_for_each_op(op, &bind_job->ops) {
1381                 switch (op->op) {
1382                 case OP_UNMAP_SPARSE:
1383                 case OP_MAP:
1384                 case OP_UNMAP:
1385                         bind_link_gpuvas(op);
1386                         break;
1387                 default:
1388                         break;
1389                 }
1390         }
1391         nouveau_uvmm_unlock(uvmm);
1392
1393         return 0;
1394
1395 unwind_continue:
1396         op = list_prev_op(op);
1397 unwind:
1398         list_for_each_op_from_reverse(op, &bind_job->ops) {
1399                 switch (op->op) {
1400                 case OP_MAP_SPARSE:
1401                         nouveau_uvma_region_destroy(uvmm, op->va.addr,
1402                                                     op->va.range);
1403                         break;
1404                 case OP_UNMAP_SPARSE:
1405                         __nouveau_uvma_region_insert(uvmm, op->reg);
1406                         nouveau_uvmm_sm_unmap_prepare_unwind(uvmm, &op->new,
1407                                                              op->ops);
1408                         break;
1409                 case OP_MAP:
1410                         nouveau_uvmm_sm_map_prepare_unwind(uvmm, &op->new,
1411                                                            op->ops,
1412                                                            op->va.addr,
1413                                                            op->va.range);
1414                         break;
1415                 case OP_UNMAP:
1416                         nouveau_uvmm_sm_unmap_prepare_unwind(uvmm, &op->new,
1417                                                              op->ops);
1418                         break;
1419                 }
1420
1421                 drm_gpuva_ops_free(&uvmm->base, op->ops);
1422                 op->ops = NULL;
1423                 op->reg = NULL;
1424         }
1425
1426         nouveau_uvmm_unlock(uvmm);
1427         drm_gpuvm_exec_unlock(vme);
1428         return ret;
1429 }
1430
1431 static void
1432 nouveau_uvmm_bind_job_armed_submit(struct nouveau_job *job,
1433                                    struct drm_gpuvm_exec *vme)
1434 {
1435         drm_gpuvm_exec_resv_add_fence(vme, job->done_fence,
1436                                       job->resv_usage, job->resv_usage);
1437         drm_gpuvm_exec_unlock(vme);
1438 }
1439
1440 static struct dma_fence *
1441 nouveau_uvmm_bind_job_run(struct nouveau_job *job)
1442 {
1443         struct nouveau_uvmm_bind_job *bind_job = to_uvmm_bind_job(job);
1444         struct nouveau_uvmm *uvmm = nouveau_cli_uvmm(job->cli);
1445         struct bind_job_op *op;
1446         int ret = 0;
1447
1448         list_for_each_op(op, &bind_job->ops) {
1449                 switch (op->op) {
1450                 case OP_MAP_SPARSE:
1451                         /* noop */
1452                         break;
1453                 case OP_MAP:
1454                         ret = nouveau_uvmm_sm_map(uvmm, &op->new, op->ops);
1455                         if (ret)
1456                                 goto out;
1457                         break;
1458                 case OP_UNMAP_SPARSE:
1459                         fallthrough;
1460                 case OP_UNMAP:
1461                         ret = nouveau_uvmm_sm_unmap(uvmm, &op->new, op->ops);
1462                         if (ret)
1463                                 goto out;
1464                         break;
1465                 }
1466         }
1467
1468 out:
1469         if (ret)
1470                 NV_PRINTK(err, job->cli, "bind job failed: %d\n", ret);
1471         return ERR_PTR(ret);
1472 }
1473
1474 static void
1475 nouveau_uvmm_bind_job_cleanup(struct nouveau_job *job)
1476 {
1477         struct nouveau_uvmm_bind_job *bind_job = to_uvmm_bind_job(job);
1478         struct nouveau_uvmm *uvmm = nouveau_cli_uvmm(job->cli);
1479         struct bind_job_op *op;
1480
1481         list_for_each_op(op, &bind_job->ops) {
1482                 struct drm_gem_object *obj = op->gem.obj;
1483
1484                 /* When nouveau_uvmm_bind_job_submit() fails op->ops and op->reg
1485                  * will be NULL, hence skip the cleanup.
1486                  */
1487                 switch (op->op) {
1488                 case OP_MAP_SPARSE:
1489                         /* noop */
1490                         break;
1491                 case OP_UNMAP_SPARSE:
1492                         if (!IS_ERR_OR_NULL(op->ops))
1493                                 nouveau_uvmm_sm_unmap_cleanup(uvmm, &op->new,
1494                                                               op->ops);
1495
1496                         if (op->reg) {
1497                                 nouveau_uvma_region_sparse_unref(op->reg);
1498                                 nouveau_uvmm_lock(uvmm);
1499                                 nouveau_uvma_region_remove(op->reg);
1500                                 nouveau_uvmm_unlock(uvmm);
1501                                 nouveau_uvma_region_complete(op->reg);
1502                                 nouveau_uvma_region_put(op->reg);
1503                         }
1504
1505                         break;
1506                 case OP_MAP:
1507                         if (!IS_ERR_OR_NULL(op->ops))
1508                                 nouveau_uvmm_sm_map_cleanup(uvmm, &op->new,
1509                                                             op->ops);
1510                         break;
1511                 case OP_UNMAP:
1512                         if (!IS_ERR_OR_NULL(op->ops))
1513                                 nouveau_uvmm_sm_unmap_cleanup(uvmm, &op->new,
1514                                                               op->ops);
1515                         break;
1516                 }
1517
1518                 if (!IS_ERR_OR_NULL(op->ops))
1519                         drm_gpuva_ops_free(&uvmm->base, op->ops);
1520
1521                 if (!IS_ERR_OR_NULL(op->vm_bo)) {
1522                         dma_resv_lock(obj->resv, NULL);
1523                         drm_gpuvm_bo_put(op->vm_bo);
1524                         dma_resv_unlock(obj->resv);
1525                 }
1526
1527                 if (obj)
1528                         drm_gem_object_put(obj);
1529         }
1530
1531         nouveau_job_done(job);
1532         complete_all(&bind_job->complete);
1533
1534         nouveau_uvmm_bind_job_put(bind_job);
1535 }
1536
1537 static struct nouveau_job_ops nouveau_bind_job_ops = {
1538         .submit = nouveau_uvmm_bind_job_submit,
1539         .armed_submit = nouveau_uvmm_bind_job_armed_submit,
1540         .run = nouveau_uvmm_bind_job_run,
1541         .free = nouveau_uvmm_bind_job_cleanup,
1542 };
1543
1544 static int
1545 bind_job_op_from_uop(struct bind_job_op **pop,
1546                      struct drm_nouveau_vm_bind_op *uop)
1547 {
1548         struct bind_job_op *op;
1549
1550         op = *pop = kzalloc(sizeof(*op), GFP_KERNEL);
1551         if (!op)
1552                 return -ENOMEM;
1553
1554         switch (uop->op) {
1555         case OP_MAP:
1556                 op->op = uop->flags & DRM_NOUVEAU_VM_BIND_SPARSE ?
1557                          OP_MAP_SPARSE : OP_MAP;
1558                 break;
1559         case OP_UNMAP:
1560                 op->op = uop->flags & DRM_NOUVEAU_VM_BIND_SPARSE ?
1561                          OP_UNMAP_SPARSE : OP_UNMAP;
1562                 break;
1563         default:
1564                 op->op = uop->op;
1565                 break;
1566         }
1567
1568         op->flags = uop->flags;
1569         op->va.addr = uop->addr;
1570         op->va.range = uop->range;
1571         op->gem.handle = uop->handle;
1572         op->gem.offset = uop->bo_offset;
1573
1574         return 0;
1575 }
1576
1577 static void
1578 bind_job_ops_free(struct list_head *ops)
1579 {
1580         struct bind_job_op *op, *next;
1581
1582         list_for_each_op_safe(op, next, ops) {
1583                 list_del(&op->entry);
1584                 kfree(op);
1585         }
1586 }
1587
1588 static int
1589 nouveau_uvmm_bind_job_init(struct nouveau_uvmm_bind_job **pjob,
1590                            struct nouveau_uvmm_bind_job_args *__args)
1591 {
1592         struct nouveau_uvmm_bind_job *job;
1593         struct nouveau_job_args args = {};
1594         struct bind_job_op *op;
1595         int i, ret;
1596
1597         ret = nouveau_uvmm_bind_job_alloc(&job);
1598         if (ret)
1599                 return ret;
1600
1601         INIT_LIST_HEAD(&job->ops);
1602
1603         for (i = 0; i < __args->op.count; i++) {
1604                 ret = bind_job_op_from_uop(&op, &__args->op.s[i]);
1605                 if (ret)
1606                         goto err_free;
1607
1608                 list_add_tail(&op->entry, &job->ops);
1609         }
1610
1611         init_completion(&job->complete);
1612
1613         args.file_priv = __args->file_priv;
1614
1615         args.sched = __args->sched;
1616         args.credits = 1;
1617
1618         args.in_sync.count = __args->in_sync.count;
1619         args.in_sync.s = __args->in_sync.s;
1620
1621         args.out_sync.count = __args->out_sync.count;
1622         args.out_sync.s = __args->out_sync.s;
1623
1624         args.sync = !(__args->flags & DRM_NOUVEAU_VM_BIND_RUN_ASYNC);
1625         args.ops = &nouveau_bind_job_ops;
1626         args.resv_usage = DMA_RESV_USAGE_BOOKKEEP;
1627
1628         ret = nouveau_job_init(&job->base, &args);
1629         if (ret)
1630                 goto err_free;
1631
1632         *pjob = job;
1633         return 0;
1634
1635 err_free:
1636         bind_job_ops_free(&job->ops);
1637         kfree(job);
1638         *pjob = NULL;
1639
1640         return ret;
1641 }
1642
1643 static int
1644 nouveau_uvmm_vm_bind(struct nouveau_uvmm_bind_job_args *args)
1645 {
1646         struct nouveau_uvmm_bind_job *job;
1647         int ret;
1648
1649         ret = nouveau_uvmm_bind_job_init(&job, args);
1650         if (ret)
1651                 return ret;
1652
1653         ret = nouveau_job_submit(&job->base);
1654         if (ret)
1655                 goto err_job_fini;
1656
1657         return 0;
1658
1659 err_job_fini:
1660         nouveau_job_fini(&job->base);
1661         return ret;
1662 }
1663
1664 static int
1665 nouveau_uvmm_vm_bind_ucopy(struct nouveau_uvmm_bind_job_args *args,
1666                            struct drm_nouveau_vm_bind *req)
1667 {
1668         struct drm_nouveau_sync **s;
1669         u32 inc = req->wait_count;
1670         u64 ins = req->wait_ptr;
1671         u32 outc = req->sig_count;
1672         u64 outs = req->sig_ptr;
1673         u32 opc = req->op_count;
1674         u64 ops = req->op_ptr;
1675         int ret;
1676
1677         args->flags = req->flags;
1678
1679         if (opc) {
1680                 args->op.count = opc;
1681                 args->op.s = u_memcpya(ops, opc,
1682                                        sizeof(*args->op.s));
1683                 if (IS_ERR(args->op.s))
1684                         return PTR_ERR(args->op.s);
1685         }
1686
1687         if (inc) {
1688                 s = &args->in_sync.s;
1689
1690                 args->in_sync.count = inc;
1691                 *s = u_memcpya(ins, inc, sizeof(**s));
1692                 if (IS_ERR(*s)) {
1693                         ret = PTR_ERR(*s);
1694                         goto err_free_ops;
1695                 }
1696         }
1697
1698         if (outc) {
1699                 s = &args->out_sync.s;
1700
1701                 args->out_sync.count = outc;
1702                 *s = u_memcpya(outs, outc, sizeof(**s));
1703                 if (IS_ERR(*s)) {
1704                         ret = PTR_ERR(*s);
1705                         goto err_free_ins;
1706                 }
1707         }
1708
1709         return 0;
1710
1711 err_free_ops:
1712         u_free(args->op.s);
1713 err_free_ins:
1714         u_free(args->in_sync.s);
1715         return ret;
1716 }
1717
1718 static void
1719 nouveau_uvmm_vm_bind_ufree(struct nouveau_uvmm_bind_job_args *args)
1720 {
1721         u_free(args->op.s);
1722         u_free(args->in_sync.s);
1723         u_free(args->out_sync.s);
1724 }
1725
1726 int
1727 nouveau_uvmm_ioctl_vm_bind(struct drm_device *dev,
1728                            void *data,
1729                            struct drm_file *file_priv)
1730 {
1731         struct nouveau_cli *cli = nouveau_cli(file_priv);
1732         struct nouveau_uvmm_bind_job_args args = {};
1733         struct drm_nouveau_vm_bind *req = data;
1734         int ret = 0;
1735
1736         if (unlikely(!nouveau_cli_uvmm_locked(cli)))
1737                 return -ENOSYS;
1738
1739         ret = nouveau_uvmm_vm_bind_ucopy(&args, req);
1740         if (ret)
1741                 return ret;
1742
1743         args.sched = cli->sched;
1744         args.file_priv = file_priv;
1745
1746         ret = nouveau_uvmm_vm_bind(&args);
1747         if (ret)
1748                 goto out_free_args;
1749
1750 out_free_args:
1751         nouveau_uvmm_vm_bind_ufree(&args);
1752         return ret;
1753 }
1754
1755 void
1756 nouveau_uvmm_bo_map_all(struct nouveau_bo *nvbo, struct nouveau_mem *mem)
1757 {
1758         struct drm_gem_object *obj = &nvbo->bo.base;
1759         struct drm_gpuvm_bo *vm_bo;
1760         struct drm_gpuva *va;
1761
1762         dma_resv_assert_held(obj->resv);
1763
1764         drm_gem_for_each_gpuvm_bo(vm_bo, obj) {
1765                 drm_gpuvm_bo_for_each_va(va, vm_bo) {
1766                         struct nouveau_uvma *uvma = uvma_from_va(va);
1767
1768                         nouveau_uvma_map(uvma, mem);
1769                         drm_gpuva_invalidate(va, false);
1770                 }
1771         }
1772 }
1773
1774 void
1775 nouveau_uvmm_bo_unmap_all(struct nouveau_bo *nvbo)
1776 {
1777         struct drm_gem_object *obj = &nvbo->bo.base;
1778         struct drm_gpuvm_bo *vm_bo;
1779         struct drm_gpuva *va;
1780
1781         dma_resv_assert_held(obj->resv);
1782
1783         drm_gem_for_each_gpuvm_bo(vm_bo, obj) {
1784                 drm_gpuvm_bo_for_each_va(va, vm_bo) {
1785                         struct nouveau_uvma *uvma = uvma_from_va(va);
1786
1787                         nouveau_uvma_unmap(uvma);
1788                         drm_gpuva_invalidate(va, true);
1789                 }
1790         }
1791 }
1792
1793 static void
1794 nouveau_uvmm_free(struct drm_gpuvm *gpuvm)
1795 {
1796         struct nouveau_uvmm *uvmm = uvmm_from_gpuvm(gpuvm);
1797
1798         kfree(uvmm);
1799 }
1800
1801 static int
1802 nouveau_uvmm_bo_validate(struct drm_gpuvm_bo *vm_bo, struct drm_exec *exec)
1803 {
1804         struct nouveau_bo *nvbo = nouveau_gem_object(vm_bo->obj);
1805
1806         return nouveau_bo_validate(nvbo, true, false);
1807 }
1808
1809 static const struct drm_gpuvm_ops gpuvm_ops = {
1810         .vm_free = nouveau_uvmm_free,
1811         .vm_bo_validate = nouveau_uvmm_bo_validate,
1812 };
1813
1814 int
1815 nouveau_uvmm_ioctl_vm_init(struct drm_device *dev,
1816                            void *data,
1817                            struct drm_file *file_priv)
1818 {
1819         struct nouveau_uvmm *uvmm;
1820         struct nouveau_cli *cli = nouveau_cli(file_priv);
1821         struct drm_device *drm = cli->drm->dev;
1822         struct drm_gem_object *r_obj;
1823         struct drm_nouveau_vm_init *init = data;
1824         u64 kernel_managed_end;
1825         int ret;
1826
1827         if (check_add_overflow(init->kernel_managed_addr,
1828                                init->kernel_managed_size,
1829                                &kernel_managed_end))
1830                 return -EINVAL;
1831
1832         if (kernel_managed_end > NOUVEAU_VA_SPACE_END)
1833                 return -EINVAL;
1834
1835         mutex_lock(&cli->mutex);
1836
1837         if (unlikely(cli->uvmm.disabled)) {
1838                 ret = -ENOSYS;
1839                 goto out_unlock;
1840         }
1841
1842         uvmm = kzalloc(sizeof(*uvmm), GFP_KERNEL);
1843         if (!uvmm) {
1844                 ret = -ENOMEM;
1845                 goto out_unlock;
1846         }
1847
1848         r_obj = drm_gpuvm_resv_object_alloc(drm);
1849         if (!r_obj) {
1850                 kfree(uvmm);
1851                 ret = -ENOMEM;
1852                 goto out_unlock;
1853         }
1854
1855         mutex_init(&uvmm->mutex);
1856         mt_init_flags(&uvmm->region_mt, MT_FLAGS_LOCK_EXTERN);
1857         mt_set_external_lock(&uvmm->region_mt, &uvmm->mutex);
1858
1859         drm_gpuvm_init(&uvmm->base, cli->name, 0, drm, r_obj,
1860                        NOUVEAU_VA_SPACE_START,
1861                        NOUVEAU_VA_SPACE_END,
1862                        init->kernel_managed_addr,
1863                        init->kernel_managed_size,
1864                        &gpuvm_ops);
1865         /* GPUVM takes care from here on. */
1866         drm_gem_object_put(r_obj);
1867
1868         ret = nvif_vmm_ctor(&cli->mmu, "uvmm",
1869                             cli->vmm.vmm.object.oclass, RAW,
1870                             init->kernel_managed_addr,
1871                             init->kernel_managed_size,
1872                             NULL, 0, &uvmm->vmm.vmm);
1873         if (ret)
1874                 goto out_gpuvm_fini;
1875
1876         uvmm->vmm.cli = cli;
1877         cli->uvmm.ptr = uvmm;
1878         mutex_unlock(&cli->mutex);
1879
1880         return 0;
1881
1882 out_gpuvm_fini:
1883         drm_gpuvm_put(&uvmm->base);
1884 out_unlock:
1885         mutex_unlock(&cli->mutex);
1886         return ret;
1887 }
1888
1889 void
1890 nouveau_uvmm_fini(struct nouveau_uvmm *uvmm)
1891 {
1892         MA_STATE(mas, &uvmm->region_mt, 0, 0);
1893         struct nouveau_uvma_region *reg;
1894         struct nouveau_cli *cli = uvmm->vmm.cli;
1895         struct drm_gpuva *va, *next;
1896
1897         nouveau_uvmm_lock(uvmm);
1898         drm_gpuvm_for_each_va_safe(va, next, &uvmm->base) {
1899                 struct nouveau_uvma *uvma = uvma_from_va(va);
1900                 struct drm_gem_object *obj = va->gem.obj;
1901
1902                 if (unlikely(va == &uvmm->base.kernel_alloc_node))
1903                         continue;
1904
1905                 drm_gpuva_remove(va);
1906
1907                 dma_resv_lock(obj->resv, NULL);
1908                 drm_gpuva_unlink(va);
1909                 dma_resv_unlock(obj->resv);
1910
1911                 nouveau_uvma_unmap(uvma);
1912                 nouveau_uvma_vmm_put(uvma);
1913
1914                 nouveau_uvma_gem_put(uvma);
1915                 nouveau_uvma_free(uvma);
1916         }
1917
1918         mas_for_each(&mas, reg, ULONG_MAX) {
1919                 mas_erase(&mas);
1920                 nouveau_uvma_region_sparse_unref(reg);
1921                 nouveau_uvma_region_put(reg);
1922         }
1923
1924         WARN(!mtree_empty(&uvmm->region_mt),
1925              "nouveau_uvma_region tree not empty, potentially leaking memory.");
1926         __mt_destroy(&uvmm->region_mt);
1927         nouveau_uvmm_unlock(uvmm);
1928
1929         mutex_lock(&cli->mutex);
1930         nouveau_vmm_fini(&uvmm->vmm);
1931         drm_gpuvm_put(&uvmm->base);
1932         mutex_unlock(&cli->mutex);
1933 }