drivers/gpu/drm/vmwgfx/vmwgfx_resource.c

   1 // SPDX-License-Identifier: GPL-2.0 OR MIT
   2 /**************************************************************************
   3  *
   4  * Copyright 2009-2015 VMware, Inc., Palo Alto, CA., USA
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the
   8  * "Software"), to deal in the Software without restriction, including
   9  * without limitation the rights to use, copy, modify, merge, publish,
  10  * distribute, sub license, and/or sell copies of the Software, and to
  11  * permit persons to whom the Software is furnished to do so, subject to
  12  * the following conditions:
  13  *
  14  * The above copyright notice and this permission notice (including the
  15  * next paragraph) shall be included in all copies or substantial portions
  16  * of the Software.
  17  *
  18  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  19  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  20  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
  21  * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
  22  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
  23  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
  24  * USE OR OTHER DEALINGS IN THE SOFTWARE.
  25  *
  26  **************************************************************************/
  27
  28 #include "vmwgfx_drv.h"
  29 #include <drm/vmwgfx_drm.h>
  30 #include <drm/ttm/ttm_placement.h>
  31 #include <drm/drmP.h>
  32 #include "vmwgfx_resource_priv.h"
  33 #include "vmwgfx_binding.h"
  34
  35 #define VMW_RES_EVICT_ERR_COUNT 10
  36
  37 /**
  38  * vmw_resource_mob_attach - Mark a resource as attached to its backing mob
  39  * @res: The resource
  40  */
  41 void vmw_resource_mob_attach(struct vmw_resource *res)
  42 {
  43         struct vmw_buffer_object *backup = res->backup;
  44         struct rb_node **new = &backup->res_tree.rb_node, *parent = NULL;
  45
  46         lockdep_assert_held(&backup->base.resv->lock.base);
  47         res->used_prio = (res->res_dirty) ? res->func->dirty_prio :
  48                 res->func->prio;
  49
  50         while (*new) {
  51                 struct vmw_resource *this =
  52                         container_of(*new, struct vmw_resource, mob_node);
  53
  54                 parent = *new;
  55                 new = (res->backup_offset < this->backup_offset) ?
  56                         &((*new)->rb_left) : &((*new)->rb_right);
  57         }
  58
  59         rb_link_node(&res->mob_node, parent, new);
  60         rb_insert_color(&res->mob_node, &backup->res_tree);
  61
  62         vmw_bo_prio_add(backup, res->used_prio);
  63 }
  64
  65 /**
  66  * vmw_resource_mob_detach - Mark a resource as detached from its backing mob
  67  * @res: The resource
  68  */
  69 void vmw_resource_mob_detach(struct vmw_resource *res)
  70 {
  71         struct vmw_buffer_object *backup = res->backup;
  72
  73         lockdep_assert_held(&backup->base.resv->lock.base);
  74         if (vmw_resource_mob_attached(res)) {
  75                 rb_erase(&res->mob_node, &backup->res_tree);
  76                 RB_CLEAR_NODE(&res->mob_node);
  77                 vmw_bo_prio_del(backup, res->used_prio);
  78         }
  79 }
  80
  81
  82 struct vmw_resource *vmw_resource_reference(struct vmw_resource *res)
  83 {
  84         kref_get(&res->kref);
  85         return res;
  86 }
  87
  88 struct vmw_resource *
  89 vmw_resource_reference_unless_doomed(struct vmw_resource *res)
  90 {
  91         return kref_get_unless_zero(&res->kref) ? res : NULL;
  92 }
  93
  94 /**
  95  * vmw_resource_release_id - release a resource id to the id manager.
  96  *
  97  * @res: Pointer to the resource.
  98  *
  99  * Release the resource id to the resource id manager and set it to -1
 100  */
 101 void vmw_resource_release_id(struct vmw_resource *res)
 102 {
 103         struct vmw_private *dev_priv = res->dev_priv;
 104         struct idr *idr = &dev_priv->res_idr[res->func->res_type];
 105
 106         spin_lock(&dev_priv->resource_lock);
 107         if (res->id != -1)
 108                 idr_remove(idr, res->id);
 109         res->id = -1;
 110         spin_unlock(&dev_priv->resource_lock);
 111 }
 112
 113 static void vmw_resource_release(struct kref *kref)
 114 {
 115         struct vmw_resource *res =
 116             container_of(kref, struct vmw_resource, kref);
 117         struct vmw_private *dev_priv = res->dev_priv;
 118         int id;
 119         struct idr *idr = &dev_priv->res_idr[res->func->res_type];
 120
 121         spin_lock(&dev_priv->resource_lock);
 122         list_del_init(&res->lru_head);
 123         spin_unlock(&dev_priv->resource_lock);
 124         if (res->backup) {
 125                 struct ttm_buffer_object *bo = &res->backup->base;
 126
 127                 ttm_bo_reserve(bo, false, false, NULL);
 128                 if (vmw_resource_mob_attached(res) &&
 129                     res->func->unbind != NULL) {
 130                         struct ttm_validate_buffer val_buf;
 131
 132                         val_buf.bo = bo;
 133                         val_buf.num_shared = 0;
 134                         res->func->unbind(res, false, &val_buf);
 135                 }
 136                 res->backup_dirty = false;
 137                 vmw_resource_mob_detach(res);
 138                 if (res->dirty)
 139                         res->func->dirty_free(res);
 140                 if (res->coherent)
 141                         vmw_bo_dirty_release(res->backup);
 142                 ttm_bo_unreserve(bo);
 143                 vmw_bo_unreference(&res->backup);
 144         }
 145
 146         if (likely(res->hw_destroy != NULL)) {
 147                 mutex_lock(&dev_priv->binding_mutex);
 148                 vmw_binding_res_list_kill(&res->binding_head);
 149                 mutex_unlock(&dev_priv->binding_mutex);
 150                 res->hw_destroy(res);
 151         }
 152
 153         id = res->id;
 154         if (res->res_free != NULL)
 155                 res->res_free(res);
 156         else
 157                 kfree(res);
 158
 159         spin_lock(&dev_priv->resource_lock);
 160         if (id != -1)
 161                 idr_remove(idr, id);
 162         spin_unlock(&dev_priv->resource_lock);
 163 }
 164
 165 void vmw_resource_unreference(struct vmw_resource **p_res)
 166 {
 167         struct vmw_resource *res = *p_res;
 168
 169         *p_res = NULL;
 170         kref_put(&res->kref, vmw_resource_release);
 171 }
 172
 173
 174 /**
 175  * vmw_resource_alloc_id - release a resource id to the id manager.
 176  *
 177  * @res: Pointer to the resource.
 178  *
 179  * Allocate the lowest free resource from the resource manager, and set
 180  * @res->id to that id. Returns 0 on success and -ENOMEM on failure.
 181  */
 182 int vmw_resource_alloc_id(struct vmw_resource *res)
 183 {
 184         struct vmw_private *dev_priv = res->dev_priv;
 185         int ret;
 186         struct idr *idr = &dev_priv->res_idr[res->func->res_type];
 187
 188         BUG_ON(res->id != -1);
 189
 190         idr_preload(GFP_KERNEL);
 191         spin_lock(&dev_priv->resource_lock);
 192
 193         ret = idr_alloc(idr, res, 1, 0, GFP_NOWAIT);
 194         if (ret >= 0)
 195                 res->id = ret;
 196
 197         spin_unlock(&dev_priv->resource_lock);
 198         idr_preload_end();
 199         return ret < 0 ? ret : 0;
 200 }
 201
 202 /**
 203  * vmw_resource_init - initialize a struct vmw_resource
 204  *
 205  * @dev_priv:       Pointer to a device private struct.
 206  * @res:            The struct vmw_resource to initialize.
 207  * @obj_type:       Resource object type.
 208  * @delay_id:       Boolean whether to defer device id allocation until
 209  *                  the first validation.
 210  * @res_free:       Resource destructor.
 211  * @func:           Resource function table.
 212  */
 213 int vmw_resource_init(struct vmw_private *dev_priv, struct vmw_resource *res,
 214                       bool delay_id,
 215                       void (*res_free) (struct vmw_resource *res),
 216                       const struct vmw_res_func *func)
 217 {
 218         kref_init(&res->kref);
 219         res->hw_destroy = NULL;
 220         res->res_free = res_free;
 221         res->dev_priv = dev_priv;
 222         res->func = func;
 223         RB_CLEAR_NODE(&res->mob_node);
 224         INIT_LIST_HEAD(&res->lru_head);
 225         INIT_LIST_HEAD(&res->binding_head);
 226         res->id = -1;
 227         res->backup = NULL;
 228         res->backup_offset = 0;
 229         res->backup_dirty = false;
 230         res->res_dirty = false;
 231         res->coherent = false;
 232         res->used_prio = 3;
 233         res->dirty = NULL;
 234         if (delay_id)
 235                 return 0;
 236         else
 237                 return vmw_resource_alloc_id(res);
 238 }
 239
 240
 241 /**
 242  * vmw_user_resource_lookup_handle - lookup a struct resource from a
 243  * TTM user-space handle and perform basic type checks
 244  *
 245  * @dev_priv:     Pointer to a device private struct
 246  * @tfile:        Pointer to a struct ttm_object_file identifying the caller
 247  * @handle:       The TTM user-space handle
 248  * @converter:    Pointer to an object describing the resource type
 249  * @p_res:        On successful return the location pointed to will contain
 250  *                a pointer to a refcounted struct vmw_resource.
 251  *
 252  * If the handle can't be found or is associated with an incorrect resource
 253  * type, -EINVAL will be returned.
 254  */
 255 int vmw_user_resource_lookup_handle(struct vmw_private *dev_priv,
 256                                     struct ttm_object_file *tfile,
 257                                     uint32_t handle,
 258                                     const struct vmw_user_resource_conv
 259                                     *converter,
 260                                     struct vmw_resource **p_res)
 261 {
 262         struct ttm_base_object *base;
 263         struct vmw_resource *res;
 264         int ret = -EINVAL;
 265
 266         base = ttm_base_object_lookup(tfile, handle);
 267         if (unlikely(base == NULL))
 268                 return -EINVAL;
 269
 270         if (unlikely(ttm_base_object_type(base) != converter->object_type))
 271                 goto out_bad_resource;
 272
 273         res = converter->base_obj_to_res(base);
 274         kref_get(&res->kref);
 275
 276         *p_res = res;
 277         ret = 0;
 278
 279 out_bad_resource:
 280         ttm_base_object_unref(&base);
 281
 282         return ret;
 283 }
 284
 285 /**
 286  * vmw_user_resource_lookup_handle - lookup a struct resource from a
 287  * TTM user-space handle and perform basic type checks
 288  *
 289  * @dev_priv:     Pointer to a device private struct
 290  * @tfile:        Pointer to a struct ttm_object_file identifying the caller
 291  * @handle:       The TTM user-space handle
 292  * @converter:    Pointer to an object describing the resource type
 293  * @p_res:        On successful return the location pointed to will contain
 294  *                a pointer to a refcounted struct vmw_resource.
 295  *
 296  * If the handle can't be found or is associated with an incorrect resource
 297  * type, -EINVAL will be returned.
 298  */
 299 struct vmw_resource *
 300 vmw_user_resource_noref_lookup_handle(struct vmw_private *dev_priv,
 301                                       struct ttm_object_file *tfile,
 302                                       uint32_t handle,
 303                                       const struct vmw_user_resource_conv
 304                                       *converter)
 305 {
 306         struct ttm_base_object *base;
 307
 308         base = ttm_base_object_noref_lookup(tfile, handle);
 309         if (!base)
 310                 return ERR_PTR(-ESRCH);
 311
 312         if (unlikely(ttm_base_object_type(base) != converter->object_type)) {
 313                 ttm_base_object_noref_release();
 314                 return ERR_PTR(-EINVAL);
 315         }
 316
 317         return converter->base_obj_to_res(base);
 318 }
 319
 320 /**
 321  * Helper function that looks either a surface or bo.
 322  *
 323  * The pointer this pointed at by out_surf and out_buf needs to be null.
 324  */
 325 int vmw_user_lookup_handle(struct vmw_private *dev_priv,
 326                            struct ttm_object_file *tfile,
 327                            uint32_t handle,
 328                            struct vmw_surface **out_surf,
 329                            struct vmw_buffer_object **out_buf)
 330 {
 331         struct vmw_resource *res;
 332         int ret;
 333
 334         BUG_ON(*out_surf || *out_buf);
 335
 336         ret = vmw_user_resource_lookup_handle(dev_priv, tfile, handle,
 337                                               user_surface_converter,
 338                                               &res);
 339         if (!ret) {
 340                 *out_surf = vmw_res_to_srf(res);
 341                 return 0;
 342         }
 343
 344         *out_surf = NULL;
 345         ret = vmw_user_bo_lookup(tfile, handle, out_buf, NULL);
 346         return ret;
 347 }
 348
 349 /**
 350  * vmw_resource_buf_alloc - Allocate a backup buffer for a resource.
 351  *
 352  * @res:            The resource for which to allocate a backup buffer.
 353  * @interruptible:  Whether any sleeps during allocation should be
 354  *                  performed while interruptible.
 355  */
 356 static int vmw_resource_buf_alloc(struct vmw_resource *res,
 357                                   bool interruptible)
 358 {
 359         unsigned long size =
 360                 (res->backup_size + PAGE_SIZE - 1) & PAGE_MASK;
 361         struct vmw_buffer_object *backup;
 362         int ret;
 363
 364         if (likely(res->backup)) {
 365                 BUG_ON(res->backup->base.num_pages * PAGE_SIZE < size);
 366                 return 0;
 367         }
 368
 369         backup = kzalloc(sizeof(*backup), GFP_KERNEL);
 370         if (unlikely(!backup))
 371                 return -ENOMEM;
 372
 373         ret = vmw_bo_init(res->dev_priv, backup, res->backup_size,
 374                               res->func->backup_placement,
 375                               interruptible,
 376                               &vmw_bo_bo_free);
 377         if (unlikely(ret != 0))
 378                 goto out_no_bo;
 379
 380         res->backup = backup;
 381
 382 out_no_bo:
 383         return ret;
 384 }
 385
 386 /**
 387  * vmw_resource_do_validate - Make a resource up-to-date and visible
 388  *                            to the device.
 389  *
 390  * @res:            The resource to make visible to the device.
 391  * @val_buf:        Information about a buffer possibly
 392  *                  containing backup data if a bind operation is needed.
 393  *
 394  * On hardware resource shortage, this function returns -EBUSY and
 395  * should be retried once resources have been freed up.
 396  */
 397 static int vmw_resource_do_validate(struct vmw_resource *res,
 398                                     struct ttm_validate_buffer *val_buf,
 399                                     bool dirtying)
 400 {
 401         int ret = 0;
 402         const struct vmw_res_func *func = res->func;
 403
 404         if (unlikely(res->id == -1)) {
 405                 ret = func->create(res);
 406                 if (unlikely(ret != 0))
 407                         return ret;
 408         }
 409
 410         if (func->bind &&
 411             ((func->needs_backup && !vmw_resource_mob_attached(res) &&
 412               val_buf->bo != NULL) ||
 413              (!func->needs_backup && val_buf->bo != NULL))) {
 414                 ret = func->bind(res, val_buf);
 415                 if (unlikely(ret != 0))
 416                         goto out_bind_failed;
 417                 if (func->needs_backup)
 418                         vmw_resource_mob_attach(res);
 419         }
 420
 421         /*
 422          * Handle the case where the backup mob is marked coherent but
 423          * the resource isn't.
 424          */
 425         if (func->dirty_alloc && vmw_resource_mob_attached(res) &&
 426             !res->coherent) {
 427                 if (res->backup->dirty && !res->dirty) {
 428                         ret = func->dirty_alloc(res);
 429                         if (ret)
 430                                 return ret;
 431                 } else if (!res->backup->dirty && res->dirty) {
 432                         func->dirty_free(res);
 433                 }
 434         }
 435
 436         /*
 437          * Transfer the dirty regions to the resource and update
 438          * the resource.
 439          */
 440         if (res->dirty) {
 441                 if (dirtying && !res->res_dirty) {
 442                         pgoff_t start = res->backup_offset >> PAGE_SHIFT;
 443                         pgoff_t end = __KERNEL_DIV_ROUND_UP
 444                                 (res->backup_offset + res->backup_size,
 445                                  PAGE_SIZE);
 446
 447                         vmw_bo_dirty_unmap(res->backup, start, end);
 448                 }
 449
 450                 vmw_bo_dirty_transfer_to_res(res);
 451                 return func->dirty_sync(res);
 452         }
 453
 454         return 0;
 455
 456 out_bind_failed:
 457         func->destroy(res);
 458
 459         return ret;
 460 }
 461
 462 /**
 463  * vmw_resource_unreserve - Unreserve a resource previously reserved for
 464  * command submission.
 465  *
 466  * @res:               Pointer to the struct vmw_resource to unreserve.
 467  * @dirty_set:         Change dirty status of the resource.
 468  * @dirty:             When changing dirty status indicates the new status.
 469  * @switch_backup:     Backup buffer has been switched.
 470  * @new_backup:        Pointer to new backup buffer if command submission
 471  *                     switched. May be NULL.
 472  * @new_backup_offset: New backup offset if @switch_backup is true.
 473  *
 474  * Currently unreserving a resource means putting it back on the device's
 475  * resource lru list, so that it can be evicted if necessary.
 476  */
 477 void vmw_resource_unreserve(struct vmw_resource *res,
 478                             bool dirty_set,
 479                             bool dirty,
 480                             bool switch_backup,
 481                             struct vmw_buffer_object *new_backup,
 482                             unsigned long new_backup_offset)
 483 {
 484         struct vmw_private *dev_priv = res->dev_priv;
 485
 486         if (!list_empty(&res->lru_head))
 487                 return;
 488
 489         if (switch_backup && new_backup != res->backup) {
 490                 if (res->backup) {
 491                         vmw_resource_mob_detach(res);
 492                         if (res->coherent)
 493                                 vmw_bo_dirty_release(res->backup);
 494                         vmw_bo_unreference(&res->backup);
 495                 }
 496
 497                 if (new_backup) {
 498                         res->backup = vmw_bo_reference(new_backup);
 499
 500                         /*
 501                          * The validation code should already have added a
 502                          * dirty tracker here.
 503                          */
 504                         WARN_ON(res->coherent && !new_backup->dirty);
 505
 506                         vmw_resource_mob_attach(res);
 507                 } else {
 508                         res->backup = NULL;
 509                 }
 510         } else if (switch_backup && res->coherent) {
 511                 vmw_bo_dirty_release(res->backup);
 512         }
 513
 514         if (switch_backup)
 515                 res->backup_offset = new_backup_offset;
 516
 517         if (dirty_set)
 518                 res->res_dirty = dirty;
 519
 520         if (!res->func->may_evict || res->id == -1 || res->pin_count)
 521                 return;
 522
 523         spin_lock(&dev_priv->resource_lock);
 524         list_add_tail(&res->lru_head,
 525                       &res->dev_priv->res_lru[res->func->res_type]);
 526         spin_unlock(&dev_priv->resource_lock);
 527 }
 528
 529 /**
 530  * vmw_resource_check_buffer - Check whether a backup buffer is needed
 531  *                             for a resource and in that case, allocate
 532  *                             one, reserve and validate it.
 533  *
 534  * @ticket:         The ww aqcquire context to use, or NULL if trylocking.
 535  * @res:            The resource for which to allocate a backup buffer.
 536  * @interruptible:  Whether any sleeps during allocation should be
 537  *                  performed while interruptible.
 538  * @val_buf:        On successful return contains data about the
 539  *                  reserved and validated backup buffer.
 540  */
 541 static int
 542 vmw_resource_check_buffer(struct ww_acquire_ctx *ticket,
 543                           struct vmw_resource *res,
 544                           bool interruptible,
 545                           struct ttm_validate_buffer *val_buf)
 546 {
 547         struct ttm_operation_ctx ctx = { true, false };
 548         struct list_head val_list;
 549         bool backup_dirty = false;
 550         int ret;
 551
 552         if (unlikely(res->backup == NULL)) {
 553                 ret = vmw_resource_buf_alloc(res, interruptible);
 554                 if (unlikely(ret != 0))
 555                         return ret;
 556         }
 557
 558         INIT_LIST_HEAD(&val_list);
 559         ttm_bo_get(&res->backup->base);
 560         val_buf->bo = &res->backup->base;
 561         val_buf->num_shared = 0;
 562         list_add_tail(&val_buf->head, &val_list);
 563         ret = ttm_eu_reserve_buffers(ticket, &val_list, interruptible, NULL,
 564                                      true);
 565         if (unlikely(ret != 0))
 566                 goto out_no_reserve;
 567
 568         if (res->func->needs_backup && !vmw_resource_mob_attached(res))
 569                 return 0;
 570
 571         backup_dirty = res->backup_dirty;
 572         ret = ttm_bo_validate(&res->backup->base,
 573                               res->func->backup_placement,
 574                               &ctx);
 575
 576         if (unlikely(ret != 0))
 577                 goto out_no_validate;
 578
 579         return 0;
 580
 581 out_no_validate:
 582         ttm_eu_backoff_reservation(ticket, &val_list);
 583 out_no_reserve:
 584         ttm_bo_put(val_buf->bo);
 585         val_buf->bo = NULL;
 586         if (backup_dirty)
 587                 vmw_bo_unreference(&res->backup);
 588
 589         return ret;
 590 }
 591
 592 /**
 593  * vmw_resource_reserve - Reserve a resource for command submission
 594  *
 595  * @res:            The resource to reserve.
 596  *
 597  * This function takes the resource off the LRU list and make sure
 598  * a backup buffer is present for guest-backed resources. However,
 599  * the buffer may not be bound to the resource at this point.
 600  *
 601  */
 602 int vmw_resource_reserve(struct vmw_resource *res, bool interruptible,
 603                          bool no_backup)
 604 {
 605         struct vmw_private *dev_priv = res->dev_priv;
 606         int ret;
 607
 608         spin_lock(&dev_priv->resource_lock);
 609         list_del_init(&res->lru_head);
 610         spin_unlock(&dev_priv->resource_lock);
 611
 612         if (res->func->needs_backup && res->backup == NULL &&
 613             !no_backup) {
 614                 ret = vmw_resource_buf_alloc(res, interruptible);
 615                 if (unlikely(ret != 0)) {
 616                         DRM_ERROR("Failed to allocate a backup buffer "
 617                                   "of size %lu. bytes\n",
 618                                   (unsigned long) res->backup_size);
 619                         return ret;
 620                 }
 621         }
 622
 623         return 0;
 624 }
 625
 626 /**
 627  * vmw_resource_backoff_reservation - Unreserve and unreference a
 628  *                                    backup buffer
 629  *.
 630  * @ticket:         The ww acquire ctx used for reservation.
 631  * @val_buf:        Backup buffer information.
 632  */
 633 static void
 634 vmw_resource_backoff_reservation(struct ww_acquire_ctx *ticket,
 635                                  struct ttm_validate_buffer *val_buf)
 636 {
 637         struct list_head val_list;
 638
 639         if (likely(val_buf->bo == NULL))
 640                 return;
 641
 642         INIT_LIST_HEAD(&val_list);
 643         list_add_tail(&val_buf->head, &val_list);
 644         ttm_eu_backoff_reservation(ticket, &val_list);
 645         ttm_bo_put(val_buf->bo);
 646         val_buf->bo = NULL;
 647 }
 648
 649 /**
 650  * vmw_resource_do_evict - Evict a resource, and transfer its data
 651  *                         to a backup buffer.
 652  *
 653  * @ticket:         The ww acquire ticket to use, or NULL if trylocking.
 654  * @res:            The resource to evict.
 655  * @interruptible:  Whether to wait interruptible.
 656  */
 657 static int vmw_resource_do_evict(struct ww_acquire_ctx *ticket,
 658                                  struct vmw_resource *res, bool interruptible)
 659 {
 660         struct ttm_validate_buffer val_buf;
 661         const struct vmw_res_func *func = res->func;
 662         int ret;
 663
 664         BUG_ON(!func->may_evict);
 665
 666         val_buf.bo = NULL;
 667         val_buf.num_shared = 0;
 668         ret = vmw_resource_check_buffer(ticket, res, interruptible, &val_buf);
 669         if (unlikely(ret != 0))
 670                 return ret;
 671
 672         if (unlikely(func->unbind != NULL &&
 673                      (!func->needs_backup || vmw_resource_mob_attached(res)))) {
 674                 ret = func->unbind(res, res->res_dirty, &val_buf);
 675                 if (unlikely(ret != 0))
 676                         goto out_no_unbind;
 677                 vmw_resource_mob_detach(res);
 678         }
 679         ret = func->destroy(res);
 680         res->backup_dirty = true;
 681         res->res_dirty = false;
 682 out_no_unbind:
 683         vmw_resource_backoff_reservation(ticket, &val_buf);
 684
 685         return ret;
 686 }
 687
 688
 689 /**
 690  * vmw_resource_validate - Make a resource up-to-date and visible
 691  *                         to the device.
 692  * @res: The resource to make visible to the device.
 693  * @intr: Perform waits interruptible if possible.
 694  * @dirtying: Pending GPU operation will dirty the resource
 695  *
 696  * On succesful return, any backup DMA buffer pointed to by @res->backup will
 697  * be reserved and validated.
 698  * On hardware resource shortage, this function will repeatedly evict
 699  * resources of the same type until the validation succeeds.
 700  *
 701  * Return: Zero on success, -ERESTARTSYS if interrupted, negative error code
 702  * on failure.
 703  */
 704 int vmw_resource_validate(struct vmw_resource *res, bool intr,
 705                           bool dirtying)
 706 {
 707         int ret;
 708         struct vmw_resource *evict_res;
 709         struct vmw_private *dev_priv = res->dev_priv;
 710         struct list_head *lru_list = &dev_priv->res_lru[res->func->res_type];
 711         struct ttm_validate_buffer val_buf;
 712         unsigned err_count = 0;
 713
 714         if (!res->func->create)
 715                 return 0;
 716
 717         val_buf.bo = NULL;
 718         val_buf.num_shared = 0;
 719         if (res->backup)
 720                 val_buf.bo = &res->backup->base;
 721         do {
 722                 ret = vmw_resource_do_validate(res, &val_buf, dirtying);
 723                 if (likely(ret != -EBUSY))
 724                         break;
 725
 726                 spin_lock(&dev_priv->resource_lock);
 727                 if (list_empty(lru_list) || !res->func->may_evict) {
 728                         DRM_ERROR("Out of device device resources "
 729                                   "for %s.\n", res->func->type_name);
 730                         ret = -EBUSY;
 731                         spin_unlock(&dev_priv->resource_lock);
 732                         break;
 733                 }
 734
 735                 evict_res = vmw_resource_reference
 736                         (list_first_entry(lru_list, struct vmw_resource,
 737                                           lru_head));
 738                 list_del_init(&evict_res->lru_head);
 739
 740                 spin_unlock(&dev_priv->resource_lock);
 741
 742                 /* Trylock backup buffers with a NULL ticket. */
 743                 ret = vmw_resource_do_evict(NULL, evict_res, intr);
 744                 if (unlikely(ret != 0)) {
 745                         spin_lock(&dev_priv->resource_lock);
 746                         list_add_tail(&evict_res->lru_head, lru_list);
 747                         spin_unlock(&dev_priv->resource_lock);
 748                         if (ret == -ERESTARTSYS ||
 749                             ++err_count > VMW_RES_EVICT_ERR_COUNT) {
 750                                 vmw_resource_unreference(&evict_res);
 751                                 goto out_no_validate;
 752                         }
 753                 }
 754
 755                 vmw_resource_unreference(&evict_res);
 756         } while (1);
 757
 758         if (unlikely(ret != 0))
 759                 goto out_no_validate;
 760         else if (!res->func->needs_backup && res->backup) {
 761                 WARN_ON_ONCE(vmw_resource_mob_attached(res));
 762                 vmw_bo_unreference(&res->backup);
 763         }
 764
 765         return 0;
 766
 767 out_no_validate:
 768         return ret;
 769 }
 770
 771
 772 /**
 773  * vmw_resource_unbind_list
 774  *
 775  * @vbo: Pointer to the current backing MOB.
 776  *
 777  * Evicts the Guest Backed hardware resource if the backup
 778  * buffer is being moved out of MOB memory.
 779  * Note that this function will not race with the resource
 780  * validation code, since resource validation and eviction
 781  * both require the backup buffer to be reserved.
 782  */
 783 void vmw_resource_unbind_list(struct vmw_buffer_object *vbo)
 784 {
 785         struct ttm_validate_buffer val_buf = {
 786                 .bo = &vbo->base,
 787                 .num_shared = 0
 788         };
 789
 790         lockdep_assert_held(&vbo->base.resv->lock.base);
 791         while (!RB_EMPTY_ROOT(&vbo->res_tree)) {
 792                 struct rb_node *node = vbo->res_tree.rb_node;
 793                 struct vmw_resource *res =
 794                         container_of(node, struct vmw_resource, mob_node);
 795
 796                 if (!WARN_ON_ONCE(!res->func->unbind))
 797                         (void) res->func->unbind(res, res->res_dirty, &val_buf);
 798
 799                 res->backup_dirty = true;
 800                 res->res_dirty = false;
 801                 vmw_resource_mob_detach(res);
 802         }
 803
 804         (void) ttm_bo_wait(&vbo->base, false, false);
 805 }
 806
 807
 808 /**
 809  * vmw_query_readback_all - Read back cached query states
 810  *
 811  * @dx_query_mob: Buffer containing the DX query MOB
 812  *
 813  * Read back cached states from the device if they exist.  This function
 814  * assumings binding_mutex is held.
 815  */
 816 int vmw_query_readback_all(struct vmw_buffer_object *dx_query_mob)
 817 {
 818         struct vmw_resource *dx_query_ctx;
 819         struct vmw_private *dev_priv;
 820         struct {
 821                 SVGA3dCmdHeader header;
 822                 SVGA3dCmdDXReadbackAllQuery body;
 823         } *cmd;
 824
 825
 826         /* No query bound, so do nothing */
 827         if (!dx_query_mob || !dx_query_mob->dx_query_ctx)
 828                 return 0;
 829
 830         dx_query_ctx = dx_query_mob->dx_query_ctx;
 831         dev_priv     = dx_query_ctx->dev_priv;
 832
 833         cmd = VMW_FIFO_RESERVE_DX(dev_priv, sizeof(*cmd), dx_query_ctx->id);
 834         if (unlikely(cmd == NULL))
 835                 return -ENOMEM;
 836
 837         cmd->header.id   = SVGA_3D_CMD_DX_READBACK_ALL_QUERY;
 838         cmd->header.size = sizeof(cmd->body);
 839         cmd->body.cid    = dx_query_ctx->id;
 840
 841         vmw_fifo_commit(dev_priv, sizeof(*cmd));
 842
 843         /* Triggers a rebind the next time affected context is bound */
 844         dx_query_mob->dx_query_ctx = NULL;
 845
 846         return 0;
 847 }
 848
 849
 850
 851 /**
 852  * vmw_query_move_notify - Read back cached query states
 853  *
 854  * @bo: The TTM buffer object about to move.
 855  * @mem: The memory region @bo is moving to.
 856  *
 857  * Called before the query MOB is swapped out to read back cached query
 858  * states from the device.
 859  */
 860 void vmw_query_move_notify(struct ttm_buffer_object *bo,
 861                            struct ttm_mem_reg *mem)
 862 {
 863         struct vmw_buffer_object *dx_query_mob;
 864         struct ttm_bo_device *bdev = bo->bdev;
 865         struct vmw_private *dev_priv;
 866
 867
 868         dev_priv = container_of(bdev, struct vmw_private, bdev);
 869
 870         mutex_lock(&dev_priv->binding_mutex);
 871
 872         dx_query_mob = container_of(bo, struct vmw_buffer_object, base);
 873         if (mem == NULL || !dx_query_mob || !dx_query_mob->dx_query_ctx) {
 874                 mutex_unlock(&dev_priv->binding_mutex);
 875                 return;
 876         }
 877
 878         /* If BO is being moved from MOB to system memory */
 879         if (mem->mem_type == TTM_PL_SYSTEM && bo->mem.mem_type == VMW_PL_MOB) {
 880                 struct vmw_fence_obj *fence;
 881
 882                 (void) vmw_query_readback_all(dx_query_mob);
 883                 mutex_unlock(&dev_priv->binding_mutex);
 884
 885                 /* Create a fence and attach the BO to it */
 886                 (void) vmw_execbuf_fence_commands(NULL, dev_priv, &fence, NULL);
 887                 vmw_bo_fence_single(bo, fence);
 888
 889                 if (fence != NULL)
 890                         vmw_fence_obj_unreference(&fence);
 891
 892                 (void) ttm_bo_wait(bo, false, false);
 893         } else
 894                 mutex_unlock(&dev_priv->binding_mutex);
 895
 896 }
 897
 898 /**
 899  * vmw_resource_needs_backup - Return whether a resource needs a backup buffer.
 900  *
 901  * @res:            The resource being queried.
 902  */
 903 bool vmw_resource_needs_backup(const struct vmw_resource *res)
 904 {
 905         return res->func->needs_backup;
 906 }
 907
 908 /**
 909  * vmw_resource_evict_type - Evict all resources of a specific type
 910  *
 911  * @dev_priv:       Pointer to a device private struct
 912  * @type:           The resource type to evict
 913  *
 914  * To avoid thrashing starvation or as part of the hibernation sequence,
 915  * try to evict all evictable resources of a specific type.
 916  */
 917 static void vmw_resource_evict_type(struct vmw_private *dev_priv,
 918                                     enum vmw_res_type type)
 919 {
 920         struct list_head *lru_list = &dev_priv->res_lru[type];
 921         struct vmw_resource *evict_res;
 922         unsigned err_count = 0;
 923         int ret;
 924         struct ww_acquire_ctx ticket;
 925
 926         do {
 927                 spin_lock(&dev_priv->resource_lock);
 928
 929                 if (list_empty(lru_list))
 930                         goto out_unlock;
 931
 932                 evict_res = vmw_resource_reference(
 933                         list_first_entry(lru_list, struct vmw_resource,
 934                                          lru_head));
 935                 list_del_init(&evict_res->lru_head);
 936                 spin_unlock(&dev_priv->resource_lock);
 937
 938                 /* Wait lock backup buffers with a ticket. */
 939                 ret = vmw_resource_do_evict(&ticket, evict_res, false);
 940                 if (unlikely(ret != 0)) {
 941                         spin_lock(&dev_priv->resource_lock);
 942                         list_add_tail(&evict_res->lru_head, lru_list);
 943                         spin_unlock(&dev_priv->resource_lock);
 944                         if (++err_count > VMW_RES_EVICT_ERR_COUNT) {
 945                                 vmw_resource_unreference(&evict_res);
 946                                 return;
 947                         }
 948                 }
 949
 950                 vmw_resource_unreference(&evict_res);
 951         } while (1);
 952
 953 out_unlock:
 954         spin_unlock(&dev_priv->resource_lock);
 955 }
 956
 957 /**
 958  * vmw_resource_evict_all - Evict all evictable resources
 959  *
 960  * @dev_priv:       Pointer to a device private struct
 961  *
 962  * To avoid thrashing starvation or as part of the hibernation sequence,
 963  * evict all evictable resources. In particular this means that all
 964  * guest-backed resources that are registered with the device are
 965  * evicted and the OTable becomes clean.
 966  */
 967 void vmw_resource_evict_all(struct vmw_private *dev_priv)
 968 {
 969         enum vmw_res_type type;
 970
 971         mutex_lock(&dev_priv->cmdbuf_mutex);
 972
 973         for (type = 0; type < vmw_res_max; ++type)
 974                 vmw_resource_evict_type(dev_priv, type);
 975
 976         mutex_unlock(&dev_priv->cmdbuf_mutex);
 977 }
 978
 979 /**
 980  * vmw_resource_pin - Add a pin reference on a resource
 981  *
 982  * @res: The resource to add a pin reference on
 983  *
 984  * This function adds a pin reference, and if needed validates the resource.
 985  * Having a pin reference means that the resource can never be evicted, and
 986  * its id will never change as long as there is a pin reference.
 987  * This function returns 0 on success and a negative error code on failure.
 988  */
 989 int vmw_resource_pin(struct vmw_resource *res, bool interruptible)
 990 {
 991         struct ttm_operation_ctx ctx = { interruptible, false };
 992         struct vmw_private *dev_priv = res->dev_priv;
 993         int ret;
 994
 995         ttm_write_lock(&dev_priv->reservation_sem, interruptible);
 996         mutex_lock(&dev_priv->cmdbuf_mutex);
 997         ret = vmw_resource_reserve(res, interruptible, false);
 998         if (ret)
 999                 goto out_no_reserve;
1000
1001         if (res->pin_count == 0) {
1002                 struct vmw_buffer_object *vbo = NULL;
1003
1004                 if (res->backup) {
1005                         vbo = res->backup;
1006
1007                         ttm_bo_reserve(&vbo->base, interruptible, false, NULL);
1008                         if (!vbo->pin_count) {
1009                                 ret = ttm_bo_validate
1010                                         (&vbo->base,
1011                                          res->func->backup_placement,
1012                                          &ctx);
1013                                 if (ret) {
1014                                         ttm_bo_unreserve(&vbo->base);
1015                                         goto out_no_validate;
1016                                 }
1017                         }
1018
1019                         /* Do we really need to pin the MOB as well? */
1020                         vmw_bo_pin_reserved(vbo, true);
1021                 }
1022                 ret = vmw_resource_validate(res, interruptible, true);
1023                 if (vbo)
1024                         ttm_bo_unreserve(&vbo->base);
1025                 if (ret)
1026                         goto out_no_validate;
1027         }
1028         res->pin_count++;
1029
1030 out_no_validate:
1031         vmw_resource_unreserve(res, false, false, false, NULL, 0UL);
1032 out_no_reserve:
1033         mutex_unlock(&dev_priv->cmdbuf_mutex);
1034         ttm_write_unlock(&dev_priv->reservation_sem);
1035
1036         return ret;
1037 }
1038
1039 /**
1040  * vmw_resource_unpin - Remove a pin reference from a resource
1041  *
1042  * @res: The resource to remove a pin reference from
1043  *
1044  * Having a pin reference means that the resource can never be evicted, and
1045  * its id will never change as long as there is a pin reference.
1046  */
1047 void vmw_resource_unpin(struct vmw_resource *res)
1048 {
1049         struct vmw_private *dev_priv = res->dev_priv;
1050         int ret;
1051
1052         (void) ttm_read_lock(&dev_priv->reservation_sem, false);
1053         mutex_lock(&dev_priv->cmdbuf_mutex);
1054
1055         ret = vmw_resource_reserve(res, false, true);
1056         WARN_ON(ret);
1057
1058         WARN_ON(res->pin_count == 0);
1059         if (--res->pin_count == 0 && res->backup) {
1060                 struct vmw_buffer_object *vbo = res->backup;
1061
1062                 (void) ttm_bo_reserve(&vbo->base, false, false, NULL);
1063                 vmw_bo_pin_reserved(vbo, false);
1064                 ttm_bo_unreserve(&vbo->base);
1065         }
1066
1067         vmw_resource_unreserve(res, false, false, false, NULL, 0UL);
1068
1069         mutex_unlock(&dev_priv->cmdbuf_mutex);
1070         ttm_read_unlock(&dev_priv->reservation_sem);
1071 }
1072
1073 /**
1074  * vmw_res_type - Return the resource type
1075  *
1076  * @res: Pointer to the resource
1077  */
1078 enum vmw_res_type vmw_res_type(const struct vmw_resource *res)
1079 {
1080         return res->func->res_type;
1081 }
1082
1083 /**
1084  * vmw_resource_update_dirty - Update a resource's dirty tracker with a
1085  * sequential range of touched backing store memory.
1086  * @res: The resource.
1087  * @start: The first page touched.
1088  * @end: The last page touched + 1.
1089  */
1090 void vmw_resource_dirty_update(struct vmw_resource *res, pgoff_t start,
1091                                pgoff_t end)
1092 {
1093         if (res->dirty)
1094                 res->func->dirty_range_add(res, start << PAGE_SHIFT,
1095                                            end << PAGE_SHIFT);
1096 }
1097
1098 /**
1099  * vmw_resources_clean - Clean resources intersecting a mob range
1100  * @vbo: The mob buffer object
1101  * @start: The mob page offset starting the range
1102  * @end: The mob page offset ending the range
1103  * @num_prefault: Returns how many pages including the first have been
1104  * cleaned and are ok to prefault
1105  */
1106 int vmw_resources_clean(struct vmw_buffer_object *vbo, pgoff_t start,
1107                         pgoff_t end, pgoff_t *num_prefault)
1108 {
1109         struct rb_node *cur = vbo->res_tree.rb_node;
1110         struct vmw_resource *found = NULL;
1111         unsigned long res_start = start << PAGE_SHIFT;
1112         unsigned long res_end = end << PAGE_SHIFT;
1113         unsigned long last_cleaned = 0;
1114
1115         /*
1116          * Find the resource with lowest backup_offset that intersects the
1117          * range.
1118          */
1119         while (cur) {
1120                 struct vmw_resource *cur_res =
1121                         container_of(cur, struct vmw_resource, mob_node);
1122
1123                 if (cur_res->backup_offset >= res_end) {
1124                         cur = cur->rb_left;
1125                 } else if (cur_res->backup_offset + cur_res->backup_size <=
1126                            res_start) {
1127                         cur = cur->rb_right;
1128                 } else {
1129                         found = cur_res;
1130                         cur = cur->rb_left;
1131                         /* Continue to look for resources with lower offsets */
1132                 }
1133         }
1134
1135         /*
1136          * In order of increasing backup_offset, clean dirty resorces
1137          * intersecting the range.
1138          */
1139         while (found) {
1140                 if (found->res_dirty) {
1141                         int ret;
1142
1143                         if (!found->func->clean)
1144                                 return -EINVAL;
1145
1146                         ret = found->func->clean(found);
1147                         if (ret)
1148                                 return ret;
1149
1150                         found->res_dirty = false;
1151                 }
1152                 last_cleaned = found->backup_offset + found->backup_size;
1153                 cur = rb_next(&found->mob_node);
1154                 if (!cur)
1155                         break;
1156
1157                 found = container_of(cur, struct vmw_resource, mob_node);
1158                 if (found->backup_offset >= res_end)
1159                         break;
1160         }
1161
1162         /*
1163          * Set number of pages allowed prefaulting and fence the buffer object
1164          */
1165         *num_prefault = 1;
1166         if (last_cleaned > res_start) {
1167                 struct ttm_buffer_object *bo = &vbo->base;
1168
1169                 *num_prefault = __KERNEL_DIV_ROUND_UP(last_cleaned - res_start,
1170                                                       PAGE_SIZE);
1171                 vmw_bo_fence_single(bo, NULL);
1172                 if (bo->moving)
1173                         dma_fence_put(bo->moving);
1174                 bo->moving = dma_fence_get
1175                         (reservation_object_get_excl(bo->resv));
1176         }
1177
1178         return 0;
1179 }