drivers/gpu/drm/xe/xe_gt_pagefault.c

   1 // SPDX-License-Identifier: MIT
   2 /*
   3  * Copyright © 2022 Intel Corporation
   4  */
   5
   6 #include "xe_gt_pagefault.h"
   7
   8 #include <linux/bitfield.h>
   9 #include <linux/circ_buf.h>
  10
  11 #include <drm/drm_exec.h>
  12 #include <drm/drm_managed.h>
  13 #include <drm/ttm/ttm_execbuf_util.h>
  14
  15 #include "abi/guc_actions_abi.h"
  16 #include "xe_bo.h"
  17 #include "xe_gt.h"
  18 #include "xe_gt_tlb_invalidation.h"
  19 #include "xe_guc.h"
  20 #include "xe_guc_ct.h"
  21 #include "xe_migrate.h"
  22 #include "xe_pt.h"
  23 #include "xe_trace.h"
  24 #include "xe_vm.h"
  25
  26 struct pagefault {
  27         u64 page_addr;
  28         u32 asid;
  29         u16 pdata;
  30         u8 vfid;
  31         u8 access_type;
  32         u8 fault_type;
  33         u8 fault_level;
  34         u8 engine_class;
  35         u8 engine_instance;
  36         u8 fault_unsuccessful;
  37         bool trva_fault;
  38 };
  39
  40 enum access_type {
  41         ACCESS_TYPE_READ = 0,
  42         ACCESS_TYPE_WRITE = 1,
  43         ACCESS_TYPE_ATOMIC = 2,
  44         ACCESS_TYPE_RESERVED = 3,
  45 };
  46
  47 enum fault_type {
  48         NOT_PRESENT = 0,
  49         WRITE_ACCESS_VIOLATION = 1,
  50         ATOMIC_ACCESS_VIOLATION = 2,
  51 };
  52
  53 struct acc {
  54         u64 va_range_base;
  55         u32 asid;
  56         u32 sub_granularity;
  57         u8 granularity;
  58         u8 vfid;
  59         u8 access_type;
  60         u8 engine_class;
  61         u8 engine_instance;
  62 };
  63
  64 static bool access_is_atomic(enum access_type access_type)
  65 {
  66         return access_type == ACCESS_TYPE_ATOMIC;
  67 }
  68
  69 static bool vma_is_valid(struct xe_tile *tile, struct xe_vma *vma)
  70 {
  71         return BIT(tile->id) & vma->tile_present &&
  72                 !(BIT(tile->id) & vma->tile_invalidated);
  73 }
  74
  75 static bool vma_matches(struct xe_vma *vma, u64 page_addr)
  76 {
  77         if (page_addr > xe_vma_end(vma) - 1 ||
  78             page_addr + SZ_4K - 1 < xe_vma_start(vma))
  79                 return false;
  80
  81         return true;
  82 }
  83
  84 static struct xe_vma *lookup_vma(struct xe_vm *vm, u64 page_addr)
  85 {
  86         struct xe_vma *vma = NULL;
  87
  88         if (vm->usm.last_fault_vma) {   /* Fast lookup */
  89                 if (vma_matches(vm->usm.last_fault_vma, page_addr))
  90                         vma = vm->usm.last_fault_vma;
  91         }
  92         if (!vma)
  93                 vma = xe_vm_find_overlapping_vma(vm, page_addr, SZ_4K);
  94
  95         return vma;
  96 }
  97
  98 static int xe_pf_begin(struct drm_exec *exec, struct xe_vma *vma,
  99                        bool atomic, unsigned int id)
 100 {
 101         struct xe_bo *bo = xe_vma_bo(vma);
 102         struct xe_vm *vm = xe_vma_vm(vma);
 103         int err;
 104
 105         err = xe_vm_lock_vma(exec, vma);
 106         if (err)
 107                 return err;
 108
 109         if (atomic && IS_DGFX(vm->xe)) {
 110                 if (xe_vma_is_userptr(vma)) {
 111                         err = -EACCES;
 112                         return err;
 113                 }
 114
 115                 /* Migrate to VRAM, move should invalidate the VMA first */
 116                 err = xe_bo_migrate(bo, XE_PL_VRAM0 + id);
 117                 if (err)
 118                         return err;
 119         } else if (bo) {
 120                 /* Create backing store if needed */
 121                 err = xe_bo_validate(bo, vm, true);
 122                 if (err)
 123                         return err;
 124         }
 125
 126         return 0;
 127 }
 128
 129 static int handle_pagefault(struct xe_gt *gt, struct pagefault *pf)
 130 {
 131         struct xe_device *xe = gt_to_xe(gt);
 132         struct xe_tile *tile = gt_to_tile(gt);
 133         struct drm_exec exec;
 134         struct xe_vm *vm;
 135         struct xe_vma *vma = NULL;
 136         struct dma_fence *fence;
 137         bool write_locked;
 138         int ret = 0;
 139         bool atomic;
 140
 141         /* SW isn't expected to handle TRTT faults */
 142         if (pf->trva_fault)
 143                 return -EFAULT;
 144
 145         /* ASID to VM */
 146         mutex_lock(&xe->usm.lock);
 147         vm = xa_load(&xe->usm.asid_to_vm, pf->asid);
 148         if (vm && xe_vm_in_fault_mode(vm))
 149                 xe_vm_get(vm);
 150         else
 151                 vm = NULL;
 152         mutex_unlock(&xe->usm.lock);
 153         if (!vm)
 154                 return -EINVAL;
 155
 156 retry_userptr:
 157         /*
 158          * TODO: Avoid exclusive lock if VM doesn't have userptrs, or
 159          * start out read-locked?
 160          */
 161         down_write(&vm->lock);
 162         write_locked = true;
 163         vma = lookup_vma(vm, pf->page_addr);
 164         if (!vma) {
 165                 ret = -EINVAL;
 166                 goto unlock_vm;
 167         }
 168
 169         if (!xe_vma_is_userptr(vma) ||
 170             !xe_vma_userptr_check_repin(to_userptr_vma(vma))) {
 171                 downgrade_write(&vm->lock);
 172                 write_locked = false;
 173         }
 174
 175         trace_xe_vma_pagefault(vma);
 176
 177         atomic = access_is_atomic(pf->access_type);
 178
 179         /* Check if VMA is valid */
 180         if (vma_is_valid(tile, vma) && !atomic)
 181                 goto unlock_vm;
 182
 183         /* TODO: Validate fault */
 184
 185         if (xe_vma_is_userptr(vma) && write_locked) {
 186                 struct xe_userptr_vma *uvma = to_userptr_vma(vma);
 187
 188                 spin_lock(&vm->userptr.invalidated_lock);
 189                 list_del_init(&uvma->userptr.invalidate_link);
 190                 spin_unlock(&vm->userptr.invalidated_lock);
 191
 192                 ret = xe_vma_userptr_pin_pages(uvma);
 193                 if (ret)
 194                         goto unlock_vm;
 195
 196                 downgrade_write(&vm->lock);
 197                 write_locked = false;
 198         }
 199
 200         /* Lock VM and BOs dma-resv */
 201         drm_exec_init(&exec, 0, 0);
 202         drm_exec_until_all_locked(&exec) {
 203                 ret = xe_pf_begin(&exec, vma, atomic, tile->id);
 204                 drm_exec_retry_on_contention(&exec);
 205                 if (ret)
 206                         goto unlock_dma_resv;
 207         }
 208
 209         /* Bind VMA only to the GT that has faulted */
 210         trace_xe_vma_pf_bind(vma);
 211         fence = __xe_pt_bind_vma(tile, vma, xe_tile_migrate_engine(tile), NULL, 0,
 212                                  vma->tile_present & BIT(tile->id));
 213         if (IS_ERR(fence)) {
 214                 ret = PTR_ERR(fence);
 215                 goto unlock_dma_resv;
 216         }
 217
 218         /*
 219          * XXX: Should we drop the lock before waiting? This only helps if doing
 220          * GPU binds which is currently only done if we have to wait for more
 221          * than 10ms on a move.
 222          */
 223         dma_fence_wait(fence, false);
 224         dma_fence_put(fence);
 225
 226         if (xe_vma_is_userptr(vma))
 227                 ret = xe_vma_userptr_check_repin(to_userptr_vma(vma));
 228         vma->tile_invalidated &= ~BIT(tile->id);
 229
 230 unlock_dma_resv:
 231         drm_exec_fini(&exec);
 232 unlock_vm:
 233         if (!ret)
 234                 vm->usm.last_fault_vma = vma;
 235         if (write_locked)
 236                 up_write(&vm->lock);
 237         else
 238                 up_read(&vm->lock);
 239         if (ret == -EAGAIN)
 240                 goto retry_userptr;
 241
 242         if (!ret) {
 243                 ret = xe_gt_tlb_invalidation_vma(gt, NULL, vma);
 244                 if (ret >= 0)
 245                         ret = 0;
 246         }
 247         xe_vm_put(vm);
 248
 249         return ret;
 250 }
 251
 252 static int send_pagefault_reply(struct xe_guc *guc,
 253                                 struct xe_guc_pagefault_reply *reply)
 254 {
 255         u32 action[] = {
 256                 XE_GUC_ACTION_PAGE_FAULT_RES_DESC,
 257                 reply->dw0,
 258                 reply->dw1,
 259         };
 260
 261         return xe_guc_ct_send(&guc->ct, action, ARRAY_SIZE(action), 0, 0);
 262 }
 263
 264 static void print_pagefault(struct xe_device *xe, struct pagefault *pf)
 265 {
 266         drm_dbg(&xe->drm, "\n\tASID: %d\n"
 267                  "\tVFID: %d\n"
 268                  "\tPDATA: 0x%04x\n"
 269                  "\tFaulted Address: 0x%08x%08x\n"
 270                  "\tFaultType: %d\n"
 271                  "\tAccessType: %d\n"
 272                  "\tFaultLevel: %d\n"
 273                  "\tEngineClass: %d\n"
 274                  "\tEngineInstance: %d\n",
 275                  pf->asid, pf->vfid, pf->pdata, upper_32_bits(pf->page_addr),
 276                  lower_32_bits(pf->page_addr),
 277                  pf->fault_type, pf->access_type, pf->fault_level,
 278                  pf->engine_class, pf->engine_instance);
 279 }
 280
 281 #define PF_MSG_LEN_DW   4
 282
 283 static bool get_pagefault(struct pf_queue *pf_queue, struct pagefault *pf)
 284 {
 285         const struct xe_guc_pagefault_desc *desc;
 286         bool ret = false;
 287
 288         spin_lock_irq(&pf_queue->lock);
 289         if (pf_queue->tail != pf_queue->head) {
 290                 desc = (const struct xe_guc_pagefault_desc *)
 291                         (pf_queue->data + pf_queue->tail);
 292
 293                 pf->fault_level = FIELD_GET(PFD_FAULT_LEVEL, desc->dw0);
 294                 pf->trva_fault = FIELD_GET(XE2_PFD_TRVA_FAULT, desc->dw0);
 295                 pf->engine_class = FIELD_GET(PFD_ENG_CLASS, desc->dw0);
 296                 pf->engine_instance = FIELD_GET(PFD_ENG_INSTANCE, desc->dw0);
 297                 pf->pdata = FIELD_GET(PFD_PDATA_HI, desc->dw1) <<
 298                         PFD_PDATA_HI_SHIFT;
 299                 pf->pdata |= FIELD_GET(PFD_PDATA_LO, desc->dw0);
 300                 pf->asid = FIELD_GET(PFD_ASID, desc->dw1);
 301                 pf->vfid = FIELD_GET(PFD_VFID, desc->dw2);
 302                 pf->access_type = FIELD_GET(PFD_ACCESS_TYPE, desc->dw2);
 303                 pf->fault_type = FIELD_GET(PFD_FAULT_TYPE, desc->dw2);
 304                 pf->page_addr = (u64)(FIELD_GET(PFD_VIRTUAL_ADDR_HI, desc->dw3)) <<
 305                         PFD_VIRTUAL_ADDR_HI_SHIFT;
 306                 pf->page_addr |= FIELD_GET(PFD_VIRTUAL_ADDR_LO, desc->dw2) <<
 307                         PFD_VIRTUAL_ADDR_LO_SHIFT;
 308
 309                 pf_queue->tail = (pf_queue->tail + PF_MSG_LEN_DW) %
 310                         PF_QUEUE_NUM_DW;
 311                 ret = true;
 312         }
 313         spin_unlock_irq(&pf_queue->lock);
 314
 315         return ret;
 316 }
 317
 318 static bool pf_queue_full(struct pf_queue *pf_queue)
 319 {
 320         lockdep_assert_held(&pf_queue->lock);
 321
 322         return CIRC_SPACE(pf_queue->head, pf_queue->tail, PF_QUEUE_NUM_DW) <=
 323                 PF_MSG_LEN_DW;
 324 }
 325
 326 int xe_guc_pagefault_handler(struct xe_guc *guc, u32 *msg, u32 len)
 327 {
 328         struct xe_gt *gt = guc_to_gt(guc);
 329         struct xe_device *xe = gt_to_xe(gt);
 330         struct pf_queue *pf_queue;
 331         unsigned long flags;
 332         u32 asid;
 333         bool full;
 334
 335         /*
 336          * The below logic doesn't work unless PF_QUEUE_NUM_DW % PF_MSG_LEN_DW == 0
 337          */
 338         BUILD_BUG_ON(PF_QUEUE_NUM_DW % PF_MSG_LEN_DW);
 339
 340         if (unlikely(len != PF_MSG_LEN_DW))
 341                 return -EPROTO;
 342
 343         asid = FIELD_GET(PFD_ASID, msg[1]);
 344         pf_queue = gt->usm.pf_queue + (asid % NUM_PF_QUEUE);
 345
 346         spin_lock_irqsave(&pf_queue->lock, flags);
 347         full = pf_queue_full(pf_queue);
 348         if (!full) {
 349                 memcpy(pf_queue->data + pf_queue->head, msg, len * sizeof(u32));
 350                 pf_queue->head = (pf_queue->head + len) % PF_QUEUE_NUM_DW;
 351                 queue_work(gt->usm.pf_wq, &pf_queue->worker);
 352         } else {
 353                 drm_warn(&xe->drm, "PF Queue full, shouldn't be possible");
 354         }
 355         spin_unlock_irqrestore(&pf_queue->lock, flags);
 356
 357         return full ? -ENOSPC : 0;
 358 }
 359
 360 #define USM_QUEUE_MAX_RUNTIME_MS        20
 361
 362 static void pf_queue_work_func(struct work_struct *w)
 363 {
 364         struct pf_queue *pf_queue = container_of(w, struct pf_queue, worker);
 365         struct xe_gt *gt = pf_queue->gt;
 366         struct xe_device *xe = gt_to_xe(gt);
 367         struct xe_guc_pagefault_reply reply = {};
 368         struct pagefault pf = {};
 369         unsigned long threshold;
 370         int ret;
 371
 372         threshold = jiffies + msecs_to_jiffies(USM_QUEUE_MAX_RUNTIME_MS);
 373
 374         while (get_pagefault(pf_queue, &pf)) {
 375                 ret = handle_pagefault(gt, &pf);
 376                 if (unlikely(ret)) {
 377                         print_pagefault(xe, &pf);
 378                         pf.fault_unsuccessful = 1;
 379                         drm_dbg(&xe->drm, "Fault response: Unsuccessful %d\n", ret);
 380                 }
 381
 382                 reply.dw0 = FIELD_PREP(PFR_VALID, 1) |
 383                         FIELD_PREP(PFR_SUCCESS, pf.fault_unsuccessful) |
 384                         FIELD_PREP(PFR_REPLY, PFR_ACCESS) |
 385                         FIELD_PREP(PFR_DESC_TYPE, FAULT_RESPONSE_DESC) |
 386                         FIELD_PREP(PFR_ASID, pf.asid);
 387
 388                 reply.dw1 = FIELD_PREP(PFR_VFID, pf.vfid) |
 389                         FIELD_PREP(PFR_ENG_INSTANCE, pf.engine_instance) |
 390                         FIELD_PREP(PFR_ENG_CLASS, pf.engine_class) |
 391                         FIELD_PREP(PFR_PDATA, pf.pdata);
 392
 393                 send_pagefault_reply(&gt->uc.guc, &reply);
 394
 395                 if (time_after(jiffies, threshold) &&
 396                     pf_queue->tail != pf_queue->head) {
 397                         queue_work(gt->usm.pf_wq, w);
 398                         break;
 399                 }
 400         }
 401 }
 402
 403 static void acc_queue_work_func(struct work_struct *w);
 404
 405 int xe_gt_pagefault_init(struct xe_gt *gt)
 406 {
 407         struct xe_device *xe = gt_to_xe(gt);
 408         int i;
 409
 410         if (!xe->info.has_usm)
 411                 return 0;
 412
 413         for (i = 0; i < NUM_PF_QUEUE; ++i) {
 414                 gt->usm.pf_queue[i].gt = gt;
 415                 spin_lock_init(&gt->usm.pf_queue[i].lock);
 416                 INIT_WORK(&gt->usm.pf_queue[i].worker, pf_queue_work_func);
 417         }
 418         for (i = 0; i < NUM_ACC_QUEUE; ++i) {
 419                 gt->usm.acc_queue[i].gt = gt;
 420                 spin_lock_init(&gt->usm.acc_queue[i].lock);
 421                 INIT_WORK(&gt->usm.acc_queue[i].worker, acc_queue_work_func);
 422         }
 423
 424         gt->usm.pf_wq = alloc_workqueue("xe_gt_page_fault_work_queue",
 425                                         WQ_UNBOUND | WQ_HIGHPRI, NUM_PF_QUEUE);
 426         if (!gt->usm.pf_wq)
 427                 return -ENOMEM;
 428
 429         gt->usm.acc_wq = alloc_workqueue("xe_gt_access_counter_work_queue",
 430                                          WQ_UNBOUND | WQ_HIGHPRI,
 431                                          NUM_ACC_QUEUE);
 432         if (!gt->usm.acc_wq)
 433                 return -ENOMEM;
 434
 435         return 0;
 436 }
 437
 438 void xe_gt_pagefault_reset(struct xe_gt *gt)
 439 {
 440         struct xe_device *xe = gt_to_xe(gt);
 441         int i;
 442
 443         if (!xe->info.has_usm)
 444                 return;
 445
 446         for (i = 0; i < NUM_PF_QUEUE; ++i) {
 447                 spin_lock_irq(&gt->usm.pf_queue[i].lock);
 448                 gt->usm.pf_queue[i].head = 0;
 449                 gt->usm.pf_queue[i].tail = 0;
 450                 spin_unlock_irq(&gt->usm.pf_queue[i].lock);
 451         }
 452
 453         for (i = 0; i < NUM_ACC_QUEUE; ++i) {
 454                 spin_lock(&gt->usm.acc_queue[i].lock);
 455                 gt->usm.acc_queue[i].head = 0;
 456                 gt->usm.acc_queue[i].tail = 0;
 457                 spin_unlock(&gt->usm.acc_queue[i].lock);
 458         }
 459 }
 460
 461 static int granularity_in_byte(int val)
 462 {
 463         switch (val) {
 464         case 0:
 465                 return SZ_128K;
 466         case 1:
 467                 return SZ_2M;
 468         case 2:
 469                 return SZ_16M;
 470         case 3:
 471                 return SZ_64M;
 472         default:
 473                 return 0;
 474         }
 475 }
 476
 477 static int sub_granularity_in_byte(int val)
 478 {
 479         return (granularity_in_byte(val) / 32);
 480 }
 481
 482 static void print_acc(struct xe_device *xe, struct acc *acc)
 483 {
 484         drm_warn(&xe->drm, "Access counter request:\n"
 485                  "\tType: %s\n"
 486                  "\tASID: %d\n"
 487                  "\tVFID: %d\n"
 488                  "\tEngine: %d:%d\n"
 489                  "\tGranularity: 0x%x KB Region/ %d KB sub-granularity\n"
 490                  "\tSub_Granularity Vector: 0x%08x\n"
 491                  "\tVA Range base: 0x%016llx\n",
 492                  acc->access_type ? "AC_NTFY_VAL" : "AC_TRIG_VAL",
 493                  acc->asid, acc->vfid, acc->engine_class, acc->engine_instance,
 494                  granularity_in_byte(acc->granularity) / SZ_1K,
 495                  sub_granularity_in_byte(acc->granularity) / SZ_1K,
 496                  acc->sub_granularity, acc->va_range_base);
 497 }
 498
 499 static struct xe_vma *get_acc_vma(struct xe_vm *vm, struct acc *acc)
 500 {
 501         u64 page_va = acc->va_range_base + (ffs(acc->sub_granularity) - 1) *
 502                 sub_granularity_in_byte(acc->granularity);
 503
 504         return xe_vm_find_overlapping_vma(vm, page_va, SZ_4K);
 505 }
 506
 507 static int handle_acc(struct xe_gt *gt, struct acc *acc)
 508 {
 509         struct xe_device *xe = gt_to_xe(gt);
 510         struct xe_tile *tile = gt_to_tile(gt);
 511         struct drm_exec exec;
 512         struct xe_vm *vm;
 513         struct xe_vma *vma;
 514         int ret = 0;
 515
 516         /* We only support ACC_TRIGGER at the moment */
 517         if (acc->access_type != ACC_TRIGGER)
 518                 return -EINVAL;
 519
 520         /* ASID to VM */
 521         mutex_lock(&xe->usm.lock);
 522         vm = xa_load(&xe->usm.asid_to_vm, acc->asid);
 523         if (vm)
 524                 xe_vm_get(vm);
 525         mutex_unlock(&xe->usm.lock);
 526         if (!vm || !xe_vm_in_fault_mode(vm))
 527                 return -EINVAL;
 528
 529         down_read(&vm->lock);
 530
 531         /* Lookup VMA */
 532         vma = get_acc_vma(vm, acc);
 533         if (!vma) {
 534                 ret = -EINVAL;
 535                 goto unlock_vm;
 536         }
 537
 538         trace_xe_vma_acc(vma);
 539
 540         /* Userptr or null can't be migrated, nothing to do */
 541         if (xe_vma_has_no_bo(vma))
 542                 goto unlock_vm;
 543
 544         /* Lock VM and BOs dma-resv */
 545         drm_exec_init(&exec, 0, 0);
 546         drm_exec_until_all_locked(&exec) {
 547                 ret = xe_pf_begin(&exec, vma, true, tile->id);
 548                 drm_exec_retry_on_contention(&exec);
 549                 if (ret)
 550                         break;
 551         }
 552
 553         drm_exec_fini(&exec);
 554 unlock_vm:
 555         up_read(&vm->lock);
 556         xe_vm_put(vm);
 557
 558         return ret;
 559 }
 560
 561 #define make_u64(hi__, low__)  ((u64)(hi__) << 32 | (u64)(low__))
 562
 563 #define ACC_MSG_LEN_DW        4
 564
 565 static bool get_acc(struct acc_queue *acc_queue, struct acc *acc)
 566 {
 567         const struct xe_guc_acc_desc *desc;
 568         bool ret = false;
 569
 570         spin_lock(&acc_queue->lock);
 571         if (acc_queue->tail != acc_queue->head) {
 572                 desc = (const struct xe_guc_acc_desc *)
 573                         (acc_queue->data + acc_queue->tail);
 574
 575                 acc->granularity = FIELD_GET(ACC_GRANULARITY, desc->dw2);
 576                 acc->sub_granularity = FIELD_GET(ACC_SUBG_HI, desc->dw1) << 31 |
 577                         FIELD_GET(ACC_SUBG_LO, desc->dw0);
 578                 acc->engine_class = FIELD_GET(ACC_ENG_CLASS, desc->dw1);
 579                 acc->engine_instance = FIELD_GET(ACC_ENG_INSTANCE, desc->dw1);
 580                 acc->asid =  FIELD_GET(ACC_ASID, desc->dw1);
 581                 acc->vfid =  FIELD_GET(ACC_VFID, desc->dw2);
 582                 acc->access_type = FIELD_GET(ACC_TYPE, desc->dw0);
 583                 acc->va_range_base = make_u64(desc->dw3 & ACC_VIRTUAL_ADDR_RANGE_HI,
 584                                               desc->dw2 & ACC_VIRTUAL_ADDR_RANGE_LO);
 585
 586                 acc_queue->tail = (acc_queue->tail + ACC_MSG_LEN_DW) %
 587                                   ACC_QUEUE_NUM_DW;
 588                 ret = true;
 589         }
 590         spin_unlock(&acc_queue->lock);
 591
 592         return ret;
 593 }
 594
 595 static void acc_queue_work_func(struct work_struct *w)
 596 {
 597         struct acc_queue *acc_queue = container_of(w, struct acc_queue, worker);
 598         struct xe_gt *gt = acc_queue->gt;
 599         struct xe_device *xe = gt_to_xe(gt);
 600         struct acc acc = {};
 601         unsigned long threshold;
 602         int ret;
 603
 604         threshold = jiffies + msecs_to_jiffies(USM_QUEUE_MAX_RUNTIME_MS);
 605
 606         while (get_acc(acc_queue, &acc)) {
 607                 ret = handle_acc(gt, &acc);
 608                 if (unlikely(ret)) {
 609                         print_acc(xe, &acc);
 610                         drm_warn(&xe->drm, "ACC: Unsuccessful %d\n", ret);
 611                 }
 612
 613                 if (time_after(jiffies, threshold) &&
 614                     acc_queue->tail != acc_queue->head) {
 615                         queue_work(gt->usm.acc_wq, w);
 616                         break;
 617                 }
 618         }
 619 }
 620
 621 static bool acc_queue_full(struct acc_queue *acc_queue)
 622 {
 623         lockdep_assert_held(&acc_queue->lock);
 624
 625         return CIRC_SPACE(acc_queue->head, acc_queue->tail, ACC_QUEUE_NUM_DW) <=
 626                 ACC_MSG_LEN_DW;
 627 }
 628
 629 int xe_guc_access_counter_notify_handler(struct xe_guc *guc, u32 *msg, u32 len)
 630 {
 631         struct xe_gt *gt = guc_to_gt(guc);
 632         struct acc_queue *acc_queue;
 633         u32 asid;
 634         bool full;
 635
 636         /*
 637          * The below logic doesn't work unless ACC_QUEUE_NUM_DW % ACC_MSG_LEN_DW == 0
 638          */
 639         BUILD_BUG_ON(ACC_QUEUE_NUM_DW % ACC_MSG_LEN_DW);
 640
 641         if (unlikely(len != ACC_MSG_LEN_DW))
 642                 return -EPROTO;
 643
 644         asid = FIELD_GET(ACC_ASID, msg[1]);
 645         acc_queue = &gt->usm.acc_queue[asid % NUM_ACC_QUEUE];
 646
 647         spin_lock(&acc_queue->lock);
 648         full = acc_queue_full(acc_queue);
 649         if (!full) {
 650                 memcpy(acc_queue->data + acc_queue->head, msg,
 651                        len * sizeof(u32));
 652                 acc_queue->head = (acc_queue->head + len) % ACC_QUEUE_NUM_DW;
 653                 queue_work(gt->usm.acc_wq, &acc_queue->worker);
 654         } else {
 655                 drm_warn(&gt_to_xe(gt)->drm, "ACC Queue full, dropping ACC");
 656         }
 657         spin_unlock(&acc_queue->lock);
 658
 659         return full ? -ENOSPC : 0;
 660 }