drivers/gpu/drm/vmwgfx/vmwgfx_resource.c

   1 // SPDX-License-Identifier: GPL-2.0 OR MIT
   2 /**************************************************************************
   3  *
   4  * Copyright 2009-2023 VMware, Inc., Palo Alto, CA., USA
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the
   8  * "Software"), to deal in the Software without restriction, including
   9  * without limitation the rights to use, copy, modify, merge, publish,
  10  * distribute, sub license, and/or sell copies of the Software, and to
  11  * permit persons to whom the Software is furnished to do so, subject to
  12  * the following conditions:
  13  *
  14  * The above copyright notice and this permission notice (including the
  15  * next paragraph) shall be included in all copies or substantial portions
  16  * of the Software.
  17  *
  18  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  19  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  20  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
  21  * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
  22  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
  23  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
  24  * USE OR OTHER DEALINGS IN THE SOFTWARE.
  25  *
  26  **************************************************************************/
  27
  28 #include <drm/ttm/ttm_placement.h>
  29
  30 #include "vmwgfx_binding.h"
  31 #include "vmwgfx_bo.h"
  32 #include "vmwgfx_drv.h"
  33 #include "vmwgfx_resource_priv.h"
  34
  35 #define VMW_RES_EVICT_ERR_COUNT 10
  36
  37 /**
  38  * vmw_resource_mob_attach - Mark a resource as attached to its backing mob
  39  * @res: The resource
  40  */
  41 void vmw_resource_mob_attach(struct vmw_resource *res)
  42 {
  43         struct vmw_bo *backup = res->backup;
  44         struct rb_node **new = &backup->res_tree.rb_node, *parent = NULL;
  45
  46         dma_resv_assert_held(res->backup->base.base.resv);
  47         res->used_prio = (res->res_dirty) ? res->func->dirty_prio :
  48                 res->func->prio;
  49
  50         while (*new) {
  51                 struct vmw_resource *this =
  52                         container_of(*new, struct vmw_resource, mob_node);
  53
  54                 parent = *new;
  55                 new = (res->backup_offset < this->backup_offset) ?
  56                         &((*new)->rb_left) : &((*new)->rb_right);
  57         }
  58
  59         rb_link_node(&res->mob_node, parent, new);
  60         rb_insert_color(&res->mob_node, &backup->res_tree);
  61
  62         vmw_bo_prio_add(backup, res->used_prio);
  63 }
  64
  65 /**
  66  * vmw_resource_mob_detach - Mark a resource as detached from its backing mob
  67  * @res: The resource
  68  */
  69 void vmw_resource_mob_detach(struct vmw_resource *res)
  70 {
  71         struct vmw_bo *backup = res->backup;
  72
  73         dma_resv_assert_held(backup->base.base.resv);
  74         if (vmw_resource_mob_attached(res)) {
  75                 rb_erase(&res->mob_node, &backup->res_tree);
  76                 RB_CLEAR_NODE(&res->mob_node);
  77                 vmw_bo_prio_del(backup, res->used_prio);
  78         }
  79 }
  80
  81 struct vmw_resource *vmw_resource_reference(struct vmw_resource *res)
  82 {
  83         kref_get(&res->kref);
  84         return res;
  85 }
  86
  87 struct vmw_resource *
  88 vmw_resource_reference_unless_doomed(struct vmw_resource *res)
  89 {
  90         return kref_get_unless_zero(&res->kref) ? res : NULL;
  91 }
  92
  93 /**
  94  * vmw_resource_release_id - release a resource id to the id manager.
  95  *
  96  * @res: Pointer to the resource.
  97  *
  98  * Release the resource id to the resource id manager and set it to -1
  99  */
 100 void vmw_resource_release_id(struct vmw_resource *res)
 101 {
 102         struct vmw_private *dev_priv = res->dev_priv;
 103         struct idr *idr = &dev_priv->res_idr[res->func->res_type];
 104
 105         spin_lock(&dev_priv->resource_lock);
 106         if (res->id != -1)
 107                 idr_remove(idr, res->id);
 108         res->id = -1;
 109         spin_unlock(&dev_priv->resource_lock);
 110 }
 111
 112 static void vmw_resource_release(struct kref *kref)
 113 {
 114         struct vmw_resource *res =
 115             container_of(kref, struct vmw_resource, kref);
 116         struct vmw_private *dev_priv = res->dev_priv;
 117         int id;
 118         int ret;
 119         struct idr *idr = &dev_priv->res_idr[res->func->res_type];
 120
 121         spin_lock(&dev_priv->resource_lock);
 122         list_del_init(&res->lru_head);
 123         spin_unlock(&dev_priv->resource_lock);
 124         if (res->backup) {
 125                 struct ttm_buffer_object *bo = &res->backup->base;
 126
 127                 ret = ttm_bo_reserve(bo, false, false, NULL);
 128                 BUG_ON(ret);
 129                 if (vmw_resource_mob_attached(res) &&
 130                     res->func->unbind != NULL) {
 131                         struct ttm_validate_buffer val_buf;
 132
 133                         val_buf.bo = bo;
 134                         val_buf.num_shared = 0;
 135                         res->func->unbind(res, false, &val_buf);
 136                 }
 137                 res->backup_dirty = false;
 138                 vmw_resource_mob_detach(res);
 139                 if (res->dirty)
 140                         res->func->dirty_free(res);
 141                 if (res->coherent)
 142                         vmw_bo_dirty_release(res->backup);
 143                 ttm_bo_unreserve(bo);
 144                 vmw_bo_unreference(&res->backup);
 145         }
 146
 147         if (likely(res->hw_destroy != NULL)) {
 148                 mutex_lock(&dev_priv->binding_mutex);
 149                 vmw_binding_res_list_kill(&res->binding_head);
 150                 mutex_unlock(&dev_priv->binding_mutex);
 151                 res->hw_destroy(res);
 152         }
 153
 154         id = res->id;
 155         if (res->res_free != NULL)
 156                 res->res_free(res);
 157         else
 158                 kfree(res);
 159
 160         spin_lock(&dev_priv->resource_lock);
 161         if (id != -1)
 162                 idr_remove(idr, id);
 163         spin_unlock(&dev_priv->resource_lock);
 164 }
 165
 166 void vmw_resource_unreference(struct vmw_resource **p_res)
 167 {
 168         struct vmw_resource *res = *p_res;
 169
 170         *p_res = NULL;
 171         kref_put(&res->kref, vmw_resource_release);
 172 }
 173
 174
 175 /**
 176  * vmw_resource_alloc_id - release a resource id to the id manager.
 177  *
 178  * @res: Pointer to the resource.
 179  *
 180  * Allocate the lowest free resource from the resource manager, and set
 181  * @res->id to that id. Returns 0 on success and -ENOMEM on failure.
 182  */
 183 int vmw_resource_alloc_id(struct vmw_resource *res)
 184 {
 185         struct vmw_private *dev_priv = res->dev_priv;
 186         int ret;
 187         struct idr *idr = &dev_priv->res_idr[res->func->res_type];
 188
 189         BUG_ON(res->id != -1);
 190
 191         idr_preload(GFP_KERNEL);
 192         spin_lock(&dev_priv->resource_lock);
 193
 194         ret = idr_alloc(idr, res, 1, 0, GFP_NOWAIT);
 195         if (ret >= 0)
 196                 res->id = ret;
 197
 198         spin_unlock(&dev_priv->resource_lock);
 199         idr_preload_end();
 200         return ret < 0 ? ret : 0;
 201 }
 202
 203 /**
 204  * vmw_resource_init - initialize a struct vmw_resource
 205  *
 206  * @dev_priv:       Pointer to a device private struct.
 207  * @res:            The struct vmw_resource to initialize.
 208  * @delay_id:       Boolean whether to defer device id allocation until
 209  *                  the first validation.
 210  * @res_free:       Resource destructor.
 211  * @func:           Resource function table.
 212  */
 213 int vmw_resource_init(struct vmw_private *dev_priv, struct vmw_resource *res,
 214                       bool delay_id,
 215                       void (*res_free) (struct vmw_resource *res),
 216                       const struct vmw_res_func *func)
 217 {
 218         kref_init(&res->kref);
 219         res->hw_destroy = NULL;
 220         res->res_free = res_free;
 221         res->dev_priv = dev_priv;
 222         res->func = func;
 223         RB_CLEAR_NODE(&res->mob_node);
 224         INIT_LIST_HEAD(&res->lru_head);
 225         INIT_LIST_HEAD(&res->binding_head);
 226         res->id = -1;
 227         res->backup = NULL;
 228         res->backup_offset = 0;
 229         res->backup_dirty = false;
 230         res->res_dirty = false;
 231         res->coherent = false;
 232         res->used_prio = 3;
 233         res->dirty = NULL;
 234         if (delay_id)
 235                 return 0;
 236         else
 237                 return vmw_resource_alloc_id(res);
 238 }
 239
 240
 241 /**
 242  * vmw_user_resource_lookup_handle - lookup a struct resource from a
 243  * TTM user-space handle and perform basic type checks
 244  *
 245  * @dev_priv:     Pointer to a device private struct
 246  * @tfile:        Pointer to a struct ttm_object_file identifying the caller
 247  * @handle:       The TTM user-space handle
 248  * @converter:    Pointer to an object describing the resource type
 249  * @p_res:        On successful return the location pointed to will contain
 250  *                a pointer to a refcounted struct vmw_resource.
 251  *
 252  * If the handle can't be found or is associated with an incorrect resource
 253  * type, -EINVAL will be returned.
 254  */
 255 int vmw_user_resource_lookup_handle(struct vmw_private *dev_priv,
 256                                     struct ttm_object_file *tfile,
 257                                     uint32_t handle,
 258                                     const struct vmw_user_resource_conv
 259                                     *converter,
 260                                     struct vmw_resource **p_res)
 261 {
 262         struct ttm_base_object *base;
 263         struct vmw_resource *res;
 264         int ret = -EINVAL;
 265
 266         base = ttm_base_object_lookup(tfile, handle);
 267         if (unlikely(base == NULL))
 268                 return -EINVAL;
 269
 270         if (unlikely(ttm_base_object_type(base) != converter->object_type))
 271                 goto out_bad_resource;
 272
 273         res = converter->base_obj_to_res(base);
 274         kref_get(&res->kref);
 275
 276         *p_res = res;
 277         ret = 0;
 278
 279 out_bad_resource:
 280         ttm_base_object_unref(&base);
 281
 282         return ret;
 283 }
 284
 285 /*
 286  * Helper function that looks either a surface or bo.
 287  *
 288  * The pointer this pointed at by out_surf and out_buf needs to be null.
 289  */
 290 int vmw_user_lookup_handle(struct vmw_private *dev_priv,
 291                            struct drm_file *filp,
 292                            uint32_t handle,
 293                            struct vmw_surface **out_surf,
 294                            struct vmw_bo **out_buf)
 295 {
 296         struct ttm_object_file *tfile = vmw_fpriv(filp)->tfile;
 297         struct vmw_resource *res;
 298         int ret;
 299
 300         BUG_ON(*out_surf || *out_buf);
 301
 302         ret = vmw_user_resource_lookup_handle(dev_priv, tfile, handle,
 303                                               user_surface_converter,
 304                                               &res);
 305         if (!ret) {
 306                 *out_surf = vmw_res_to_srf(res);
 307                 return 0;
 308         }
 309
 310         *out_surf = NULL;
 311         ret = vmw_user_bo_lookup(filp, handle, out_buf);
 312         return ret;
 313 }
 314
 315 /**
 316  * vmw_resource_buf_alloc - Allocate a backup buffer for a resource.
 317  *
 318  * @res:            The resource for which to allocate a backup buffer.
 319  * @interruptible:  Whether any sleeps during allocation should be
 320  *                  performed while interruptible.
 321  */
 322 static int vmw_resource_buf_alloc(struct vmw_resource *res,
 323                                   bool interruptible)
 324 {
 325         unsigned long size = PFN_ALIGN(res->backup_size);
 326         struct vmw_bo *backup;
 327         int ret;
 328
 329         if (likely(res->backup)) {
 330                 BUG_ON(res->backup->base.base.size < size);
 331                 return 0;
 332         }
 333
 334         ret = vmw_bo_create(res->dev_priv, res->backup_size,
 335                             res->func->domain, res->func->busy_domain,
 336                             interruptible, false, &backup);
 337         if (unlikely(ret != 0))
 338                 goto out_no_bo;
 339
 340         res->backup = backup;
 341
 342 out_no_bo:
 343         return ret;
 344 }
 345
 346 /**
 347  * vmw_resource_do_validate - Make a resource up-to-date and visible
 348  *                            to the device.
 349  *
 350  * @res:            The resource to make visible to the device.
 351  * @val_buf:        Information about a buffer possibly
 352  *                  containing backup data if a bind operation is needed.
 353  * @dirtying:       Transfer dirty regions.
 354  *
 355  * On hardware resource shortage, this function returns -EBUSY and
 356  * should be retried once resources have been freed up.
 357  */
 358 static int vmw_resource_do_validate(struct vmw_resource *res,
 359                                     struct ttm_validate_buffer *val_buf,
 360                                     bool dirtying)
 361 {
 362         int ret = 0;
 363         const struct vmw_res_func *func = res->func;
 364
 365         if (unlikely(res->id == -1)) {
 366                 ret = func->create(res);
 367                 if (unlikely(ret != 0))
 368                         return ret;
 369         }
 370
 371         if (func->bind &&
 372             ((func->needs_backup && !vmw_resource_mob_attached(res) &&
 373               val_buf->bo != NULL) ||
 374              (!func->needs_backup && val_buf->bo != NULL))) {
 375                 ret = func->bind(res, val_buf);
 376                 if (unlikely(ret != 0))
 377                         goto out_bind_failed;
 378                 if (func->needs_backup)
 379                         vmw_resource_mob_attach(res);
 380         }
 381
 382         /*
 383          * Handle the case where the backup mob is marked coherent but
 384          * the resource isn't.
 385          */
 386         if (func->dirty_alloc && vmw_resource_mob_attached(res) &&
 387             !res->coherent) {
 388                 if (res->backup->dirty && !res->dirty) {
 389                         ret = func->dirty_alloc(res);
 390                         if (ret)
 391                                 return ret;
 392                 } else if (!res->backup->dirty && res->dirty) {
 393                         func->dirty_free(res);
 394                 }
 395         }
 396
 397         /*
 398          * Transfer the dirty regions to the resource and update
 399          * the resource.
 400          */
 401         if (res->dirty) {
 402                 if (dirtying && !res->res_dirty) {
 403                         pgoff_t start = res->backup_offset >> PAGE_SHIFT;
 404                         pgoff_t end = __KERNEL_DIV_ROUND_UP
 405                                 (res->backup_offset + res->backup_size,
 406                                  PAGE_SIZE);
 407
 408                         vmw_bo_dirty_unmap(res->backup, start, end);
 409                 }
 410
 411                 vmw_bo_dirty_transfer_to_res(res);
 412                 return func->dirty_sync(res);
 413         }
 414
 415         return 0;
 416
 417 out_bind_failed:
 418         func->destroy(res);
 419
 420         return ret;
 421 }
 422
 423 /**
 424  * vmw_resource_unreserve - Unreserve a resource previously reserved for
 425  * command submission.
 426  *
 427  * @res:               Pointer to the struct vmw_resource to unreserve.
 428  * @dirty_set:         Change dirty status of the resource.
 429  * @dirty:             When changing dirty status indicates the new status.
 430  * @switch_backup:     Backup buffer has been switched.
 431  * @new_backup:        Pointer to new backup buffer if command submission
 432  *                     switched. May be NULL.
 433  * @new_backup_offset: New backup offset if @switch_backup is true.
 434  *
 435  * Currently unreserving a resource means putting it back on the device's
 436  * resource lru list, so that it can be evicted if necessary.
 437  */
 438 void vmw_resource_unreserve(struct vmw_resource *res,
 439                             bool dirty_set,
 440                             bool dirty,
 441                             bool switch_backup,
 442                             struct vmw_bo *new_backup,
 443                             unsigned long new_backup_offset)
 444 {
 445         struct vmw_private *dev_priv = res->dev_priv;
 446
 447         if (!list_empty(&res->lru_head))
 448                 return;
 449
 450         if (switch_backup && new_backup != res->backup) {
 451                 if (res->backup) {
 452                         vmw_resource_mob_detach(res);
 453                         if (res->coherent)
 454                                 vmw_bo_dirty_release(res->backup);
 455                         vmw_bo_unreference(&res->backup);
 456                 }
 457
 458                 if (new_backup) {
 459                         res->backup = vmw_bo_reference(new_backup);
 460
 461                         /*
 462                          * The validation code should already have added a
 463                          * dirty tracker here.
 464                          */
 465                         WARN_ON(res->coherent && !new_backup->dirty);
 466
 467                         vmw_resource_mob_attach(res);
 468                 } else {
 469                         res->backup = NULL;
 470                 }
 471         } else if (switch_backup && res->coherent) {
 472                 vmw_bo_dirty_release(res->backup);
 473         }
 474
 475         if (switch_backup)
 476                 res->backup_offset = new_backup_offset;
 477
 478         if (dirty_set)
 479                 res->res_dirty = dirty;
 480
 481         if (!res->func->may_evict || res->id == -1 || res->pin_count)
 482                 return;
 483
 484         spin_lock(&dev_priv->resource_lock);
 485         list_add_tail(&res->lru_head,
 486                       &res->dev_priv->res_lru[res->func->res_type]);
 487         spin_unlock(&dev_priv->resource_lock);
 488 }
 489
 490 /**
 491  * vmw_resource_check_buffer - Check whether a backup buffer is needed
 492  *                             for a resource and in that case, allocate
 493  *                             one, reserve and validate it.
 494  *
 495  * @ticket:         The ww acquire context to use, or NULL if trylocking.
 496  * @res:            The resource for which to allocate a backup buffer.
 497  * @interruptible:  Whether any sleeps during allocation should be
 498  *                  performed while interruptible.
 499  * @val_buf:        On successful return contains data about the
 500  *                  reserved and validated backup buffer.
 501  */
 502 static int
 503 vmw_resource_check_buffer(struct ww_acquire_ctx *ticket,
 504                           struct vmw_resource *res,
 505                           bool interruptible,
 506                           struct ttm_validate_buffer *val_buf)
 507 {
 508         struct ttm_operation_ctx ctx = { true, false };
 509         struct list_head val_list;
 510         bool backup_dirty = false;
 511         int ret;
 512
 513         if (unlikely(res->backup == NULL)) {
 514                 ret = vmw_resource_buf_alloc(res, interruptible);
 515                 if (unlikely(ret != 0))
 516                         return ret;
 517         }
 518
 519         INIT_LIST_HEAD(&val_list);
 520         ttm_bo_get(&res->backup->base);
 521         val_buf->bo = &res->backup->base;
 522         val_buf->num_shared = 0;
 523         list_add_tail(&val_buf->head, &val_list);
 524         ret = ttm_eu_reserve_buffers(ticket, &val_list, interruptible, NULL);
 525         if (unlikely(ret != 0))
 526                 goto out_no_reserve;
 527
 528         if (res->func->needs_backup && !vmw_resource_mob_attached(res))
 529                 return 0;
 530
 531         backup_dirty = res->backup_dirty;
 532         vmw_bo_placement_set(res->backup, res->func->domain,
 533                              res->func->busy_domain);
 534         ret = ttm_bo_validate(&res->backup->base,
 535                               &res->backup->placement,
 536                               &ctx);
 537
 538         if (unlikely(ret != 0))
 539                 goto out_no_validate;
 540
 541         return 0;
 542
 543 out_no_validate:
 544         ttm_eu_backoff_reservation(ticket, &val_list);
 545 out_no_reserve:
 546         ttm_bo_put(val_buf->bo);
 547         val_buf->bo = NULL;
 548         if (backup_dirty)
 549                 vmw_bo_unreference(&res->backup);
 550
 551         return ret;
 552 }
 553
 554 /*
 555  * vmw_resource_reserve - Reserve a resource for command submission
 556  *
 557  * @res:            The resource to reserve.
 558  *
 559  * This function takes the resource off the LRU list and make sure
 560  * a backup buffer is present for guest-backed resources. However,
 561  * the buffer may not be bound to the resource at this point.
 562  *
 563  */
 564 int vmw_resource_reserve(struct vmw_resource *res, bool interruptible,
 565                          bool no_backup)
 566 {
 567         struct vmw_private *dev_priv = res->dev_priv;
 568         int ret;
 569
 570         spin_lock(&dev_priv->resource_lock);
 571         list_del_init(&res->lru_head);
 572         spin_unlock(&dev_priv->resource_lock);
 573
 574         if (res->func->needs_backup && res->backup == NULL &&
 575             !no_backup) {
 576                 ret = vmw_resource_buf_alloc(res, interruptible);
 577                 if (unlikely(ret != 0)) {
 578                         DRM_ERROR("Failed to allocate a backup buffer "
 579                                   "of size %lu. bytes\n",
 580                                   (unsigned long) res->backup_size);
 581                         return ret;
 582                 }
 583         }
 584
 585         return 0;
 586 }
 587
 588 /**
 589  * vmw_resource_backoff_reservation - Unreserve and unreference a
 590  *                                    backup buffer
 591  *.
 592  * @ticket:         The ww acquire ctx used for reservation.
 593  * @val_buf:        Backup buffer information.
 594  */
 595 static void
 596 vmw_resource_backoff_reservation(struct ww_acquire_ctx *ticket,
 597                                  struct ttm_validate_buffer *val_buf)
 598 {
 599         struct list_head val_list;
 600
 601         if (likely(val_buf->bo == NULL))
 602                 return;
 603
 604         INIT_LIST_HEAD(&val_list);
 605         list_add_tail(&val_buf->head, &val_list);
 606         ttm_eu_backoff_reservation(ticket, &val_list);
 607         ttm_bo_put(val_buf->bo);
 608         val_buf->bo = NULL;
 609 }
 610
 611 /**
 612  * vmw_resource_do_evict - Evict a resource, and transfer its data
 613  *                         to a backup buffer.
 614  *
 615  * @ticket:         The ww acquire ticket to use, or NULL if trylocking.
 616  * @res:            The resource to evict.
 617  * @interruptible:  Whether to wait interruptible.
 618  */
 619 static int vmw_resource_do_evict(struct ww_acquire_ctx *ticket,
 620                                  struct vmw_resource *res, bool interruptible)
 621 {
 622         struct ttm_validate_buffer val_buf;
 623         const struct vmw_res_func *func = res->func;
 624         int ret;
 625
 626         BUG_ON(!func->may_evict);
 627
 628         val_buf.bo = NULL;
 629         val_buf.num_shared = 0;
 630         ret = vmw_resource_check_buffer(ticket, res, interruptible, &val_buf);
 631         if (unlikely(ret != 0))
 632                 return ret;
 633
 634         if (unlikely(func->unbind != NULL &&
 635                      (!func->needs_backup || vmw_resource_mob_attached(res)))) {
 636                 ret = func->unbind(res, res->res_dirty, &val_buf);
 637                 if (unlikely(ret != 0))
 638                         goto out_no_unbind;
 639                 vmw_resource_mob_detach(res);
 640         }
 641         ret = func->destroy(res);
 642         res->backup_dirty = true;
 643         res->res_dirty = false;
 644 out_no_unbind:
 645         vmw_resource_backoff_reservation(ticket, &val_buf);
 646
 647         return ret;
 648 }
 649
 650
 651 /**
 652  * vmw_resource_validate - Make a resource up-to-date and visible
 653  *                         to the device.
 654  * @res: The resource to make visible to the device.
 655  * @intr: Perform waits interruptible if possible.
 656  * @dirtying: Pending GPU operation will dirty the resource
 657  *
 658  * On successful return, any backup DMA buffer pointed to by @res->backup will
 659  * be reserved and validated.
 660  * On hardware resource shortage, this function will repeatedly evict
 661  * resources of the same type until the validation succeeds.
 662  *
 663  * Return: Zero on success, -ERESTARTSYS if interrupted, negative error code
 664  * on failure.
 665  */
 666 int vmw_resource_validate(struct vmw_resource *res, bool intr,
 667                           bool dirtying)
 668 {
 669         int ret;
 670         struct vmw_resource *evict_res;
 671         struct vmw_private *dev_priv = res->dev_priv;
 672         struct list_head *lru_list = &dev_priv->res_lru[res->func->res_type];
 673         struct ttm_validate_buffer val_buf;
 674         unsigned err_count = 0;
 675
 676         if (!res->func->create)
 677                 return 0;
 678
 679         val_buf.bo = NULL;
 680         val_buf.num_shared = 0;
 681         if (res->backup)
 682                 val_buf.bo = &res->backup->base;
 683         do {
 684                 ret = vmw_resource_do_validate(res, &val_buf, dirtying);
 685                 if (likely(ret != -EBUSY))
 686                         break;
 687
 688                 spin_lock(&dev_priv->resource_lock);
 689                 if (list_empty(lru_list) || !res->func->may_evict) {
 690                         DRM_ERROR("Out of device device resources "
 691                                   "for %s.\n", res->func->type_name);
 692                         ret = -EBUSY;
 693                         spin_unlock(&dev_priv->resource_lock);
 694                         break;
 695                 }
 696
 697                 evict_res = vmw_resource_reference
 698                         (list_first_entry(lru_list, struct vmw_resource,
 699                                           lru_head));
 700                 list_del_init(&evict_res->lru_head);
 701
 702                 spin_unlock(&dev_priv->resource_lock);
 703
 704                 /* Trylock backup buffers with a NULL ticket. */
 705                 ret = vmw_resource_do_evict(NULL, evict_res, intr);
 706                 if (unlikely(ret != 0)) {
 707                         spin_lock(&dev_priv->resource_lock);
 708                         list_add_tail(&evict_res->lru_head, lru_list);
 709                         spin_unlock(&dev_priv->resource_lock);
 710                         if (ret == -ERESTARTSYS ||
 711                             ++err_count > VMW_RES_EVICT_ERR_COUNT) {
 712                                 vmw_resource_unreference(&evict_res);
 713                                 goto out_no_validate;
 714                         }
 715                 }
 716
 717                 vmw_resource_unreference(&evict_res);
 718         } while (1);
 719
 720         if (unlikely(ret != 0))
 721                 goto out_no_validate;
 722         else if (!res->func->needs_backup && res->backup) {
 723                 WARN_ON_ONCE(vmw_resource_mob_attached(res));
 724                 vmw_bo_unreference(&res->backup);
 725         }
 726
 727         return 0;
 728
 729 out_no_validate:
 730         return ret;
 731 }
 732
 733
 734 /**
 735  * vmw_resource_unbind_list
 736  *
 737  * @vbo: Pointer to the current backing MOB.
 738  *
 739  * Evicts the Guest Backed hardware resource if the backup
 740  * buffer is being moved out of MOB memory.
 741  * Note that this function will not race with the resource
 742  * validation code, since resource validation and eviction
 743  * both require the backup buffer to be reserved.
 744  */
 745 void vmw_resource_unbind_list(struct vmw_bo *vbo)
 746 {
 747         struct ttm_validate_buffer val_buf = {
 748                 .bo = &vbo->base,
 749                 .num_shared = 0
 750         };
 751
 752         dma_resv_assert_held(vbo->base.base.resv);
 753         while (!RB_EMPTY_ROOT(&vbo->res_tree)) {
 754                 struct rb_node *node = vbo->res_tree.rb_node;
 755                 struct vmw_resource *res =
 756                         container_of(node, struct vmw_resource, mob_node);
 757
 758                 if (!WARN_ON_ONCE(!res->func->unbind))
 759                         (void) res->func->unbind(res, res->res_dirty, &val_buf);
 760
 761                 res->backup_dirty = true;
 762                 res->res_dirty = false;
 763                 vmw_resource_mob_detach(res);
 764         }
 765
 766         (void) ttm_bo_wait(&vbo->base, false, false);
 767 }
 768
 769
 770 /**
 771  * vmw_query_readback_all - Read back cached query states
 772  *
 773  * @dx_query_mob: Buffer containing the DX query MOB
 774  *
 775  * Read back cached states from the device if they exist.  This function
 776  * assumes binding_mutex is held.
 777  */
 778 int vmw_query_readback_all(struct vmw_bo *dx_query_mob)
 779 {
 780         struct vmw_resource *dx_query_ctx;
 781         struct vmw_private *dev_priv;
 782         struct {
 783                 SVGA3dCmdHeader header;
 784                 SVGA3dCmdDXReadbackAllQuery body;
 785         } *cmd;
 786
 787
 788         /* No query bound, so do nothing */
 789         if (!dx_query_mob || !dx_query_mob->dx_query_ctx)
 790                 return 0;
 791
 792         dx_query_ctx = dx_query_mob->dx_query_ctx;
 793         dev_priv     = dx_query_ctx->dev_priv;
 794
 795         cmd = VMW_CMD_CTX_RESERVE(dev_priv, sizeof(*cmd), dx_query_ctx->id);
 796         if (unlikely(cmd == NULL))
 797                 return -ENOMEM;
 798
 799         cmd->header.id   = SVGA_3D_CMD_DX_READBACK_ALL_QUERY;
 800         cmd->header.size = sizeof(cmd->body);
 801         cmd->body.cid    = dx_query_ctx->id;
 802
 803         vmw_cmd_commit(dev_priv, sizeof(*cmd));
 804
 805         /* Triggers a rebind the next time affected context is bound */
 806         dx_query_mob->dx_query_ctx = NULL;
 807
 808         return 0;
 809 }
 810
 811
 812
 813 /**
 814  * vmw_query_move_notify - Read back cached query states
 815  *
 816  * @bo: The TTM buffer object about to move.
 817  * @old_mem: The memory region @bo is moving from.
 818  * @new_mem: The memory region @bo is moving to.
 819  *
 820  * Called before the query MOB is swapped out to read back cached query
 821  * states from the device.
 822  */
 823 void vmw_query_move_notify(struct ttm_buffer_object *bo,
 824                            struct ttm_resource *old_mem,
 825                            struct ttm_resource *new_mem)
 826 {
 827         struct vmw_bo *dx_query_mob;
 828         struct ttm_device *bdev = bo->bdev;
 829         struct vmw_private *dev_priv;
 830
 831         dev_priv = container_of(bdev, struct vmw_private, bdev);
 832
 833         mutex_lock(&dev_priv->binding_mutex);
 834
 835         /* If BO is being moved from MOB to system memory */
 836         if (new_mem->mem_type == TTM_PL_SYSTEM &&
 837             old_mem->mem_type == VMW_PL_MOB) {
 838                 struct vmw_fence_obj *fence;
 839
 840                 dx_query_mob = container_of(bo, struct vmw_bo, base);
 841                 if (!dx_query_mob || !dx_query_mob->dx_query_ctx) {
 842                         mutex_unlock(&dev_priv->binding_mutex);
 843                         return;
 844                 }
 845
 846                 (void) vmw_query_readback_all(dx_query_mob);
 847                 mutex_unlock(&dev_priv->binding_mutex);
 848
 849                 /* Create a fence and attach the BO to it */
 850                 (void) vmw_execbuf_fence_commands(NULL, dev_priv, &fence, NULL);
 851                 vmw_bo_fence_single(bo, fence);
 852
 853                 if (fence != NULL)
 854                         vmw_fence_obj_unreference(&fence);
 855
 856                 (void) ttm_bo_wait(bo, false, false);
 857         } else
 858                 mutex_unlock(&dev_priv->binding_mutex);
 859 }
 860
 861 /**
 862  * vmw_resource_needs_backup - Return whether a resource needs a backup buffer.
 863  *
 864  * @res:            The resource being queried.
 865  */
 866 bool vmw_resource_needs_backup(const struct vmw_resource *res)
 867 {
 868         return res->func->needs_backup;
 869 }
 870
 871 /**
 872  * vmw_resource_evict_type - Evict all resources of a specific type
 873  *
 874  * @dev_priv:       Pointer to a device private struct
 875  * @type:           The resource type to evict
 876  *
 877  * To avoid thrashing starvation or as part of the hibernation sequence,
 878  * try to evict all evictable resources of a specific type.
 879  */
 880 static void vmw_resource_evict_type(struct vmw_private *dev_priv,
 881                                     enum vmw_res_type type)
 882 {
 883         struct list_head *lru_list = &dev_priv->res_lru[type];
 884         struct vmw_resource *evict_res;
 885         unsigned err_count = 0;
 886         int ret;
 887         struct ww_acquire_ctx ticket;
 888
 889         do {
 890                 spin_lock(&dev_priv->resource_lock);
 891
 892                 if (list_empty(lru_list))
 893                         goto out_unlock;
 894
 895                 evict_res = vmw_resource_reference(
 896                         list_first_entry(lru_list, struct vmw_resource,
 897                                          lru_head));
 898                 list_del_init(&evict_res->lru_head);
 899                 spin_unlock(&dev_priv->resource_lock);
 900
 901                 /* Wait lock backup buffers with a ticket. */
 902                 ret = vmw_resource_do_evict(&ticket, evict_res, false);
 903                 if (unlikely(ret != 0)) {
 904                         spin_lock(&dev_priv->resource_lock);
 905                         list_add_tail(&evict_res->lru_head, lru_list);
 906                         spin_unlock(&dev_priv->resource_lock);
 907                         if (++err_count > VMW_RES_EVICT_ERR_COUNT) {
 908                                 vmw_resource_unreference(&evict_res);
 909                                 return;
 910                         }
 911                 }
 912
 913                 vmw_resource_unreference(&evict_res);
 914         } while (1);
 915
 916 out_unlock:
 917         spin_unlock(&dev_priv->resource_lock);
 918 }
 919
 920 /**
 921  * vmw_resource_evict_all - Evict all evictable resources
 922  *
 923  * @dev_priv:       Pointer to a device private struct
 924  *
 925  * To avoid thrashing starvation or as part of the hibernation sequence,
 926  * evict all evictable resources. In particular this means that all
 927  * guest-backed resources that are registered with the device are
 928  * evicted and the OTable becomes clean.
 929  */
 930 void vmw_resource_evict_all(struct vmw_private *dev_priv)
 931 {
 932         enum vmw_res_type type;
 933
 934         mutex_lock(&dev_priv->cmdbuf_mutex);
 935
 936         for (type = 0; type < vmw_res_max; ++type)
 937                 vmw_resource_evict_type(dev_priv, type);
 938
 939         mutex_unlock(&dev_priv->cmdbuf_mutex);
 940 }
 941
 942 /*
 943  * vmw_resource_pin - Add a pin reference on a resource
 944  *
 945  * @res: The resource to add a pin reference on
 946  *
 947  * This function adds a pin reference, and if needed validates the resource.
 948  * Having a pin reference means that the resource can never be evicted, and
 949  * its id will never change as long as there is a pin reference.
 950  * This function returns 0 on success and a negative error code on failure.
 951  */
 952 int vmw_resource_pin(struct vmw_resource *res, bool interruptible)
 953 {
 954         struct ttm_operation_ctx ctx = { interruptible, false };
 955         struct vmw_private *dev_priv = res->dev_priv;
 956         int ret;
 957
 958         mutex_lock(&dev_priv->cmdbuf_mutex);
 959         ret = vmw_resource_reserve(res, interruptible, false);
 960         if (ret)
 961                 goto out_no_reserve;
 962
 963         if (res->pin_count == 0) {
 964                 struct vmw_bo *vbo = NULL;
 965
 966                 if (res->backup) {
 967                         vbo = res->backup;
 968
 969                         ret = ttm_bo_reserve(&vbo->base, interruptible, false, NULL);
 970                         if (ret)
 971                                 goto out_no_validate;
 972                         if (!vbo->base.pin_count) {
 973                                 vmw_bo_placement_set(vbo,
 974                                                      res->func->domain,
 975                                                      res->func->busy_domain);
 976                                 ret = ttm_bo_validate
 977                                         (&vbo->base,
 978                                          &vbo->placement,
 979                                          &ctx);
 980                                 if (ret) {
 981                                         ttm_bo_unreserve(&vbo->base);
 982                                         goto out_no_validate;
 983                                 }
 984                         }
 985
 986                         /* Do we really need to pin the MOB as well? */
 987                         vmw_bo_pin_reserved(vbo, true);
 988                 }
 989                 ret = vmw_resource_validate(res, interruptible, true);
 990                 if (vbo)
 991                         ttm_bo_unreserve(&vbo->base);
 992                 if (ret)
 993                         goto out_no_validate;
 994         }
 995         res->pin_count++;
 996
 997 out_no_validate:
 998         vmw_resource_unreserve(res, false, false, false, NULL, 0UL);
 999 out_no_reserve:
1000         mutex_unlock(&dev_priv->cmdbuf_mutex);
1001
1002         return ret;
1003 }
1004
1005 /**
1006  * vmw_resource_unpin - Remove a pin reference from a resource
1007  *
1008  * @res: The resource to remove a pin reference from
1009  *
1010  * Having a pin reference means that the resource can never be evicted, and
1011  * its id will never change as long as there is a pin reference.
1012  */
1013 void vmw_resource_unpin(struct vmw_resource *res)
1014 {
1015         struct vmw_private *dev_priv = res->dev_priv;
1016         int ret;
1017
1018         mutex_lock(&dev_priv->cmdbuf_mutex);
1019
1020         ret = vmw_resource_reserve(res, false, true);
1021         WARN_ON(ret);
1022
1023         WARN_ON(res->pin_count == 0);
1024         if (--res->pin_count == 0 && res->backup) {
1025                 struct vmw_bo *vbo = res->backup;
1026
1027                 (void) ttm_bo_reserve(&vbo->base, false, false, NULL);
1028                 vmw_bo_pin_reserved(vbo, false);
1029                 ttm_bo_unreserve(&vbo->base);
1030         }
1031
1032         vmw_resource_unreserve(res, false, false, false, NULL, 0UL);
1033
1034         mutex_unlock(&dev_priv->cmdbuf_mutex);
1035 }
1036
1037 /**
1038  * vmw_res_type - Return the resource type
1039  *
1040  * @res: Pointer to the resource
1041  */
1042 enum vmw_res_type vmw_res_type(const struct vmw_resource *res)
1043 {
1044         return res->func->res_type;
1045 }
1046
1047 /**
1048  * vmw_resource_dirty_update - Update a resource's dirty tracker with a
1049  * sequential range of touched backing store memory.
1050  * @res: The resource.
1051  * @start: The first page touched.
1052  * @end: The last page touched + 1.
1053  */
1054 void vmw_resource_dirty_update(struct vmw_resource *res, pgoff_t start,
1055                                pgoff_t end)
1056 {
1057         if (res->dirty)
1058                 res->func->dirty_range_add(res, start << PAGE_SHIFT,
1059                                            end << PAGE_SHIFT);
1060 }
1061
1062 /**
1063  * vmw_resources_clean - Clean resources intersecting a mob range
1064  * @vbo: The mob buffer object
1065  * @start: The mob page offset starting the range
1066  * @end: The mob page offset ending the range
1067  * @num_prefault: Returns how many pages including the first have been
1068  * cleaned and are ok to prefault
1069  */
1070 int vmw_resources_clean(struct vmw_bo *vbo, pgoff_t start,
1071                         pgoff_t end, pgoff_t *num_prefault)
1072 {
1073         struct rb_node *cur = vbo->res_tree.rb_node;
1074         struct vmw_resource *found = NULL;
1075         unsigned long res_start = start << PAGE_SHIFT;
1076         unsigned long res_end = end << PAGE_SHIFT;
1077         unsigned long last_cleaned = 0;
1078
1079         /*
1080          * Find the resource with lowest backup_offset that intersects the
1081          * range.
1082          */
1083         while (cur) {
1084                 struct vmw_resource *cur_res =
1085                         container_of(cur, struct vmw_resource, mob_node);
1086
1087                 if (cur_res->backup_offset >= res_end) {
1088                         cur = cur->rb_left;
1089                 } else if (cur_res->backup_offset + cur_res->backup_size <=
1090                            res_start) {
1091                         cur = cur->rb_right;
1092                 } else {
1093                         found = cur_res;
1094                         cur = cur->rb_left;
1095                         /* Continue to look for resources with lower offsets */
1096                 }
1097         }
1098
1099         /*
1100          * In order of increasing backup_offset, clean dirty resources
1101          * intersecting the range.
1102          */
1103         while (found) {
1104                 if (found->res_dirty) {
1105                         int ret;
1106
1107                         if (!found->func->clean)
1108                                 return -EINVAL;
1109
1110                         ret = found->func->clean(found);
1111                         if (ret)
1112                                 return ret;
1113
1114                         found->res_dirty = false;
1115                 }
1116                 last_cleaned = found->backup_offset + found->backup_size;
1117                 cur = rb_next(&found->mob_node);
1118                 if (!cur)
1119                         break;
1120
1121                 found = container_of(cur, struct vmw_resource, mob_node);
1122                 if (found->backup_offset >= res_end)
1123                         break;
1124         }
1125
1126         /*
1127          * Set number of pages allowed prefaulting and fence the buffer object
1128          */
1129         *num_prefault = 1;
1130         if (last_cleaned > res_start) {
1131                 struct ttm_buffer_object *bo = &vbo->base;
1132
1133                 *num_prefault = __KERNEL_DIV_ROUND_UP(last_cleaned - res_start,
1134                                                       PAGE_SIZE);
1135                 vmw_bo_fence_single(bo, NULL);
1136         }
1137
1138         return 0;
1139 }