drivers/gpu/drm/i915/gt/intel_engine_cs.c

   1 /*
   2  * Copyright © 2016 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  */
  24
  25 #include <drm/drm_print.h>
  26
  27 #include "gem/i915_gem_context.h"
  28
  29 #include "i915_drv.h"
  30
  31 #include "intel_engine.h"
  32 #include "intel_engine_pm.h"
  33 #include "intel_context.h"
  34 #include "intel_lrc.h"
  35 #include "intel_reset.h"
  36
  37 /* Haswell does have the CXT_SIZE register however it does not appear to be
  38  * valid. Now, docs explain in dwords what is in the context object. The full
  39  * size is 70720 bytes, however, the power context and execlist context will
  40  * never be saved (power context is stored elsewhere, and execlists don't work
  41  * on HSW) - so the final size, including the extra state required for the
  42  * Resource Streamer, is 66944 bytes, which rounds to 17 pages.
  43  */
  44 #define HSW_CXT_TOTAL_SIZE              (17 * PAGE_SIZE)
  45
  46 #define DEFAULT_LR_CONTEXT_RENDER_SIZE  (22 * PAGE_SIZE)
  47 #define GEN8_LR_CONTEXT_RENDER_SIZE     (20 * PAGE_SIZE)
  48 #define GEN9_LR_CONTEXT_RENDER_SIZE     (22 * PAGE_SIZE)
  49 #define GEN10_LR_CONTEXT_RENDER_SIZE    (18 * PAGE_SIZE)
  50 #define GEN11_LR_CONTEXT_RENDER_SIZE    (14 * PAGE_SIZE)
  51
  52 #define GEN8_LR_CONTEXT_OTHER_SIZE      ( 2 * PAGE_SIZE)
  53
  54 struct engine_class_info {
  55         const char *name;
  56         u8 uabi_class;
  57 };
  58
  59 static const struct engine_class_info intel_engine_classes[] = {
  60         [RENDER_CLASS] = {
  61                 .name = "rcs",
  62                 .uabi_class = I915_ENGINE_CLASS_RENDER,
  63         },
  64         [COPY_ENGINE_CLASS] = {
  65                 .name = "bcs",
  66                 .uabi_class = I915_ENGINE_CLASS_COPY,
  67         },
  68         [VIDEO_DECODE_CLASS] = {
  69                 .name = "vcs",
  70                 .uabi_class = I915_ENGINE_CLASS_VIDEO,
  71         },
  72         [VIDEO_ENHANCEMENT_CLASS] = {
  73                 .name = "vecs",
  74                 .uabi_class = I915_ENGINE_CLASS_VIDEO_ENHANCE,
  75         },
  76 };
  77
  78 #define MAX_MMIO_BASES 3
  79 struct engine_info {
  80         unsigned int hw_id;
  81         u8 class;
  82         u8 instance;
  83         /* mmio bases table *must* be sorted in reverse gen order */
  84         struct engine_mmio_base {
  85                 u32 gen : 8;
  86                 u32 base : 24;
  87         } mmio_bases[MAX_MMIO_BASES];
  88 };
  89
  90 static const struct engine_info intel_engines[] = {
  91         [RCS0] = {
  92                 .hw_id = RCS0_HW,
  93                 .class = RENDER_CLASS,
  94                 .instance = 0,
  95                 .mmio_bases = {
  96                         { .gen = 1, .base = RENDER_RING_BASE }
  97                 },
  98         },
  99         [BCS0] = {
 100                 .hw_id = BCS0_HW,
 101                 .class = COPY_ENGINE_CLASS,
 102                 .instance = 0,
 103                 .mmio_bases = {
 104                         { .gen = 6, .base = BLT_RING_BASE }
 105                 },
 106         },
 107         [VCS0] = {
 108                 .hw_id = VCS0_HW,
 109                 .class = VIDEO_DECODE_CLASS,
 110                 .instance = 0,
 111                 .mmio_bases = {
 112                         { .gen = 11, .base = GEN11_BSD_RING_BASE },
 113                         { .gen = 6, .base = GEN6_BSD_RING_BASE },
 114                         { .gen = 4, .base = BSD_RING_BASE }
 115                 },
 116         },
 117         [VCS1] = {
 118                 .hw_id = VCS1_HW,
 119                 .class = VIDEO_DECODE_CLASS,
 120                 .instance = 1,
 121                 .mmio_bases = {
 122                         { .gen = 11, .base = GEN11_BSD2_RING_BASE },
 123                         { .gen = 8, .base = GEN8_BSD2_RING_BASE }
 124                 },
 125         },
 126         [VCS2] = {
 127                 .hw_id = VCS2_HW,
 128                 .class = VIDEO_DECODE_CLASS,
 129                 .instance = 2,
 130                 .mmio_bases = {
 131                         { .gen = 11, .base = GEN11_BSD3_RING_BASE }
 132                 },
 133         },
 134         [VCS3] = {
 135                 .hw_id = VCS3_HW,
 136                 .class = VIDEO_DECODE_CLASS,
 137                 .instance = 3,
 138                 .mmio_bases = {
 139                         { .gen = 11, .base = GEN11_BSD4_RING_BASE }
 140                 },
 141         },
 142         [VECS0] = {
 143                 .hw_id = VECS0_HW,
 144                 .class = VIDEO_ENHANCEMENT_CLASS,
 145                 .instance = 0,
 146                 .mmio_bases = {
 147                         { .gen = 11, .base = GEN11_VEBOX_RING_BASE },
 148                         { .gen = 7, .base = VEBOX_RING_BASE }
 149                 },
 150         },
 151         [VECS1] = {
 152                 .hw_id = VECS1_HW,
 153                 .class = VIDEO_ENHANCEMENT_CLASS,
 154                 .instance = 1,
 155                 .mmio_bases = {
 156                         { .gen = 11, .base = GEN11_VEBOX2_RING_BASE }
 157                 },
 158         },
 159 };
 160
 161 /**
 162  * intel_engine_context_size() - return the size of the context for an engine
 163  * @dev_priv: i915 device private
 164  * @class: engine class
 165  *
 166  * Each engine class may require a different amount of space for a context
 167  * image.
 168  *
 169  * Return: size (in bytes) of an engine class specific context image
 170  *
 171  * Note: this size includes the HWSP, which is part of the context image
 172  * in LRC mode, but does not include the "shared data page" used with
 173  * GuC submission. The caller should account for this if using the GuC.
 174  */
 175 u32 intel_engine_context_size(struct drm_i915_private *dev_priv, u8 class)
 176 {
 177         u32 cxt_size;
 178
 179         BUILD_BUG_ON(I915_GTT_PAGE_SIZE != PAGE_SIZE);
 180
 181         switch (class) {
 182         case RENDER_CLASS:
 183                 switch (INTEL_GEN(dev_priv)) {
 184                 default:
 185                         MISSING_CASE(INTEL_GEN(dev_priv));
 186                         return DEFAULT_LR_CONTEXT_RENDER_SIZE;
 187                 case 11:
 188                         return GEN11_LR_CONTEXT_RENDER_SIZE;
 189                 case 10:
 190                         return GEN10_LR_CONTEXT_RENDER_SIZE;
 191                 case 9:
 192                         return GEN9_LR_CONTEXT_RENDER_SIZE;
 193                 case 8:
 194                         return GEN8_LR_CONTEXT_RENDER_SIZE;
 195                 case 7:
 196                         if (IS_HASWELL(dev_priv))
 197                                 return HSW_CXT_TOTAL_SIZE;
 198
 199                         cxt_size = I915_READ(GEN7_CXT_SIZE);
 200                         return round_up(GEN7_CXT_TOTAL_SIZE(cxt_size) * 64,
 201                                         PAGE_SIZE);
 202                 case 6:
 203                         cxt_size = I915_READ(CXT_SIZE);
 204                         return round_up(GEN6_CXT_TOTAL_SIZE(cxt_size) * 64,
 205                                         PAGE_SIZE);
 206                 case 5:
 207                 case 4:
 208                         /*
 209                          * There is a discrepancy here between the size reported
 210                          * by the register and the size of the context layout
 211                          * in the docs. Both are described as authorative!
 212                          *
 213                          * The discrepancy is on the order of a few cachelines,
 214                          * but the total is under one page (4k), which is our
 215                          * minimum allocation anyway so it should all come
 216                          * out in the wash.
 217                          */
 218                         cxt_size = I915_READ(CXT_SIZE) + 1;
 219                         DRM_DEBUG_DRIVER("gen%d CXT_SIZE = %d bytes [0x%08x]\n",
 220                                          INTEL_GEN(dev_priv),
 221                                          cxt_size * 64,
 222                                          cxt_size - 1);
 223                         return round_up(cxt_size * 64, PAGE_SIZE);
 224                 case 3:
 225                 case 2:
 226                 /* For the special day when i810 gets merged. */
 227                 case 1:
 228                         return 0;
 229                 }
 230                 break;
 231         default:
 232                 MISSING_CASE(class);
 233                 /* fall through */
 234         case VIDEO_DECODE_CLASS:
 235         case VIDEO_ENHANCEMENT_CLASS:
 236         case COPY_ENGINE_CLASS:
 237                 if (INTEL_GEN(dev_priv) < 8)
 238                         return 0;
 239                 return GEN8_LR_CONTEXT_OTHER_SIZE;
 240         }
 241 }
 242
 243 static u32 __engine_mmio_base(struct drm_i915_private *i915,
 244                               const struct engine_mmio_base *bases)
 245 {
 246         int i;
 247
 248         for (i = 0; i < MAX_MMIO_BASES; i++)
 249                 if (INTEL_GEN(i915) >= bases[i].gen)
 250                         break;
 251
 252         GEM_BUG_ON(i == MAX_MMIO_BASES);
 253         GEM_BUG_ON(!bases[i].base);
 254
 255         return bases[i].base;
 256 }
 257
 258 static void __sprint_engine_name(char *name, const struct engine_info *info)
 259 {
 260         WARN_ON(snprintf(name, INTEL_ENGINE_CS_MAX_NAME, "%s%u",
 261                          intel_engine_classes[info->class].name,
 262                          info->instance) >= INTEL_ENGINE_CS_MAX_NAME);
 263 }
 264
 265 void intel_engine_set_hwsp_writemask(struct intel_engine_cs *engine, u32 mask)
 266 {
 267         /*
 268          * Though they added more rings on g4x/ilk, they did not add
 269          * per-engine HWSTAM until gen6.
 270          */
 271         if (INTEL_GEN(engine->i915) < 6 && engine->class != RENDER_CLASS)
 272                 return;
 273
 274         if (INTEL_GEN(engine->i915) >= 3)
 275                 ENGINE_WRITE(engine, RING_HWSTAM, mask);
 276         else
 277                 ENGINE_WRITE16(engine, RING_HWSTAM, mask);
 278 }
 279
 280 static void intel_engine_sanitize_mmio(struct intel_engine_cs *engine)
 281 {
 282         /* Mask off all writes into the unknown HWSP */
 283         intel_engine_set_hwsp_writemask(engine, ~0u);
 284 }
 285
 286 static int
 287 intel_engine_setup(struct drm_i915_private *dev_priv,
 288                    enum intel_engine_id id)
 289 {
 290         const struct engine_info *info = &intel_engines[id];
 291         struct intel_engine_cs *engine;
 292
 293         GEM_BUG_ON(info->class >= ARRAY_SIZE(intel_engine_classes));
 294
 295         BUILD_BUG_ON(MAX_ENGINE_CLASS >= BIT(GEN11_ENGINE_CLASS_WIDTH));
 296         BUILD_BUG_ON(MAX_ENGINE_INSTANCE >= BIT(GEN11_ENGINE_INSTANCE_WIDTH));
 297
 298         if (GEM_DEBUG_WARN_ON(info->class > MAX_ENGINE_CLASS))
 299                 return -EINVAL;
 300
 301         if (GEM_DEBUG_WARN_ON(info->instance > MAX_ENGINE_INSTANCE))
 302                 return -EINVAL;
 303
 304         if (GEM_DEBUG_WARN_ON(dev_priv->engine_class[info->class][info->instance]))
 305                 return -EINVAL;
 306
 307         GEM_BUG_ON(dev_priv->engine[id]);
 308         engine = kzalloc(sizeof(*engine), GFP_KERNEL);
 309         if (!engine)
 310                 return -ENOMEM;
 311
 312         BUILD_BUG_ON(BITS_PER_TYPE(engine->mask) < I915_NUM_ENGINES);
 313
 314         engine->id = id;
 315         engine->mask = BIT(id);
 316         engine->i915 = dev_priv;
 317         engine->uncore = &dev_priv->uncore;
 318         __sprint_engine_name(engine->name, info);
 319         engine->hw_id = engine->guc_id = info->hw_id;
 320         engine->mmio_base = __engine_mmio_base(dev_priv, info->mmio_bases);
 321         engine->class = info->class;
 322         engine->instance = info->instance;
 323
 324         /*
 325          * To be overridden by the backend on setup. However to facilitate
 326          * cleanup on error during setup, we always provide the destroy vfunc.
 327          */
 328         engine->destroy = (typeof(engine->destroy))kfree;
 329
 330         engine->uabi_class = intel_engine_classes[info->class].uabi_class;
 331
 332         engine->context_size = intel_engine_context_size(dev_priv,
 333                                                          engine->class);
 334         if (WARN_ON(engine->context_size > BIT(20)))
 335                 engine->context_size = 0;
 336         if (engine->context_size)
 337                 DRIVER_CAPS(dev_priv)->has_logical_contexts = true;
 338
 339         /* Nothing to do here, execute in order of dependencies */
 340         engine->schedule = NULL;
 341
 342         seqlock_init(&engine->stats.lock);
 343
 344         ATOMIC_INIT_NOTIFIER_HEAD(&engine->context_status_notifier);
 345
 346         /* Scrub mmio state on takeover */
 347         intel_engine_sanitize_mmio(engine);
 348
 349         dev_priv->engine_class[info->class][info->instance] = engine;
 350         dev_priv->engine[id] = engine;
 351         return 0;
 352 }
 353
 354 static void __setup_engine_capabilities(struct intel_engine_cs *engine)
 355 {
 356         struct drm_i915_private *i915 = engine->i915;
 357
 358         if (engine->class == VIDEO_DECODE_CLASS) {
 359                 /*
 360                  * HEVC support is present on first engine instance
 361                  * before Gen11 and on all instances afterwards.
 362                  */
 363                 if (INTEL_GEN(i915) >= 11 ||
 364                     (INTEL_GEN(i915) >= 9 && engine->instance == 0))
 365                         engine->uabi_capabilities |=
 366                                 I915_VIDEO_CLASS_CAPABILITY_HEVC;
 367
 368                 /*
 369                  * SFC block is present only on even logical engine
 370                  * instances.
 371                  */
 372                 if ((INTEL_GEN(i915) >= 11 &&
 373                      RUNTIME_INFO(i915)->vdbox_sfc_access & engine->mask) ||
 374                     (INTEL_GEN(i915) >= 9 && engine->instance == 0))
 375                         engine->uabi_capabilities |=
 376                                 I915_VIDEO_AND_ENHANCE_CLASS_CAPABILITY_SFC;
 377         } else if (engine->class == VIDEO_ENHANCEMENT_CLASS) {
 378                 if (INTEL_GEN(i915) >= 9)
 379                         engine->uabi_capabilities |=
 380                                 I915_VIDEO_AND_ENHANCE_CLASS_CAPABILITY_SFC;
 381         }
 382 }
 383
 384 static void intel_setup_engine_capabilities(struct drm_i915_private *i915)
 385 {
 386         struct intel_engine_cs *engine;
 387         enum intel_engine_id id;
 388
 389         for_each_engine(engine, i915, id)
 390                 __setup_engine_capabilities(engine);
 391 }
 392
 393 /**
 394  * intel_engines_cleanup() - free the resources allocated for Command Streamers
 395  * @i915: the i915 devic
 396  */
 397 void intel_engines_cleanup(struct drm_i915_private *i915)
 398 {
 399         struct intel_engine_cs *engine;
 400         enum intel_engine_id id;
 401
 402         for_each_engine(engine, i915, id) {
 403                 engine->destroy(engine);
 404                 i915->engine[id] = NULL;
 405         }
 406 }
 407
 408 /**
 409  * intel_engines_init_mmio() - allocate and prepare the Engine Command Streamers
 410  * @i915: the i915 device
 411  *
 412  * Return: non-zero if the initialization failed.
 413  */
 414 int intel_engines_init_mmio(struct drm_i915_private *i915)
 415 {
 416         struct intel_device_info *device_info = mkwrite_device_info(i915);
 417         const unsigned int engine_mask = INTEL_INFO(i915)->engine_mask;
 418         unsigned int mask = 0;
 419         unsigned int i;
 420         int err;
 421
 422         WARN_ON(engine_mask == 0);
 423         WARN_ON(engine_mask &
 424                 GENMASK(BITS_PER_TYPE(mask) - 1, I915_NUM_ENGINES));
 425
 426         if (i915_inject_load_failure())
 427                 return -ENODEV;
 428
 429         for (i = 0; i < ARRAY_SIZE(intel_engines); i++) {
 430                 if (!HAS_ENGINE(i915, i))
 431                         continue;
 432
 433                 err = intel_engine_setup(i915, i);
 434                 if (err)
 435                         goto cleanup;
 436
 437                 mask |= BIT(i);
 438         }
 439
 440         /*
 441          * Catch failures to update intel_engines table when the new engines
 442          * are added to the driver by a warning and disabling the forgotten
 443          * engines.
 444          */
 445         if (WARN_ON(mask != engine_mask))
 446                 device_info->engine_mask = mask;
 447
 448         /* We always presume we have at least RCS available for later probing */
 449         if (WARN_ON(!HAS_ENGINE(i915, RCS0))) {
 450                 err = -ENODEV;
 451                 goto cleanup;
 452         }
 453
 454         RUNTIME_INFO(i915)->num_engines = hweight32(mask);
 455
 456         i915_check_and_clear_faults(i915);
 457
 458         intel_setup_engine_capabilities(i915);
 459
 460         return 0;
 461
 462 cleanup:
 463         intel_engines_cleanup(i915);
 464         return err;
 465 }
 466
 467 /**
 468  * intel_engines_init() - init the Engine Command Streamers
 469  * @i915: i915 device private
 470  *
 471  * Return: non-zero if the initialization failed.
 472  */
 473 int intel_engines_init(struct drm_i915_private *i915)
 474 {
 475         int (*init)(struct intel_engine_cs *engine);
 476         struct intel_engine_cs *engine;
 477         enum intel_engine_id id;
 478         int err;
 479
 480         if (HAS_EXECLISTS(i915))
 481                 init = intel_execlists_submission_init;
 482         else
 483                 init = intel_ring_submission_init;
 484
 485         for_each_engine(engine, i915, id) {
 486                 err = init(engine);
 487                 if (err)
 488                         goto cleanup;
 489         }
 490
 491         return 0;
 492
 493 cleanup:
 494         intel_engines_cleanup(i915);
 495         return err;
 496 }
 497
 498 static void intel_engine_init_batch_pool(struct intel_engine_cs *engine)
 499 {
 500         i915_gem_batch_pool_init(&engine->batch_pool, engine);
 501 }
 502
 503 void intel_engine_init_execlists(struct intel_engine_cs *engine)
 504 {
 505         struct intel_engine_execlists * const execlists = &engine->execlists;
 506
 507         execlists->port_mask = 1;
 508         GEM_BUG_ON(!is_power_of_2(execlists_num_ports(execlists)));
 509         GEM_BUG_ON(execlists_num_ports(execlists) > EXECLIST_MAX_PORTS);
 510
 511         execlists->queue_priority_hint = INT_MIN;
 512         execlists->queue = RB_ROOT_CACHED;
 513 }
 514
 515 static void cleanup_status_page(struct intel_engine_cs *engine)
 516 {
 517         struct i915_vma *vma;
 518
 519         /* Prevent writes into HWSP after returning the page to the system */
 520         intel_engine_set_hwsp_writemask(engine, ~0u);
 521
 522         vma = fetch_and_zero(&engine->status_page.vma);
 523         if (!vma)
 524                 return;
 525
 526         if (!HWS_NEEDS_PHYSICAL(engine->i915))
 527                 i915_vma_unpin(vma);
 528
 529         i915_gem_object_unpin_map(vma->obj);
 530         i915_gem_object_put(vma->obj);
 531 }
 532
 533 static int pin_ggtt_status_page(struct intel_engine_cs *engine,
 534                                 struct i915_vma *vma)
 535 {
 536         unsigned int flags;
 537
 538         flags = PIN_GLOBAL;
 539         if (!HAS_LLC(engine->i915))
 540                 /*
 541                  * On g33, we cannot place HWS above 256MiB, so
 542                  * restrict its pinning to the low mappable arena.
 543                  * Though this restriction is not documented for
 544                  * gen4, gen5, or byt, they also behave similarly
 545                  * and hang if the HWS is placed at the top of the
 546                  * GTT. To generalise, it appears that all !llc
 547                  * platforms have issues with us placing the HWS
 548                  * above the mappable region (even though we never
 549                  * actually map it).
 550                  */
 551                 flags |= PIN_MAPPABLE;
 552         else
 553                 flags |= PIN_HIGH;
 554
 555         return i915_vma_pin(vma, 0, 0, flags);
 556 }
 557
 558 static int init_status_page(struct intel_engine_cs *engine)
 559 {
 560         struct drm_i915_gem_object *obj;
 561         struct i915_vma *vma;
 562         void *vaddr;
 563         int ret;
 564
 565         /*
 566          * Though the HWS register does support 36bit addresses, historically
 567          * we have had hangs and corruption reported due to wild writes if
 568          * the HWS is placed above 4G. We only allow objects to be allocated
 569          * in GFP_DMA32 for i965, and no earlier physical address users had
 570          * access to more than 4G.
 571          */
 572         obj = i915_gem_object_create_internal(engine->i915, PAGE_SIZE);
 573         if (IS_ERR(obj)) {
 574                 DRM_ERROR("Failed to allocate status page\n");
 575                 return PTR_ERR(obj);
 576         }
 577
 578         i915_gem_object_set_cache_coherency(obj, I915_CACHE_LLC);
 579
 580         vma = i915_vma_instance(obj, &engine->i915->ggtt.vm, NULL);
 581         if (IS_ERR(vma)) {
 582                 ret = PTR_ERR(vma);
 583                 goto err;
 584         }
 585
 586         vaddr = i915_gem_object_pin_map(obj, I915_MAP_WB);
 587         if (IS_ERR(vaddr)) {
 588                 ret = PTR_ERR(vaddr);
 589                 goto err;
 590         }
 591
 592         engine->status_page.addr = memset(vaddr, 0, PAGE_SIZE);
 593         engine->status_page.vma = vma;
 594
 595         if (!HWS_NEEDS_PHYSICAL(engine->i915)) {
 596                 ret = pin_ggtt_status_page(engine, vma);
 597                 if (ret)
 598                         goto err_unpin;
 599         }
 600
 601         return 0;
 602
 603 err_unpin:
 604         i915_gem_object_unpin_map(obj);
 605 err:
 606         i915_gem_object_put(obj);
 607         return ret;
 608 }
 609
 610 static int intel_engine_setup_common(struct intel_engine_cs *engine)
 611 {
 612         int err;
 613
 614         init_llist_head(&engine->barrier_tasks);
 615
 616         err = init_status_page(engine);
 617         if (err)
 618                 return err;
 619
 620         intel_engine_init_active(engine, ENGINE_PHYSICAL);
 621         intel_engine_init_breadcrumbs(engine);
 622         intel_engine_init_execlists(engine);
 623         intel_engine_init_hangcheck(engine);
 624         intel_engine_init_batch_pool(engine);
 625         intel_engine_init_cmd_parser(engine);
 626         intel_engine_init__pm(engine);
 627
 628         /* Use the whole device by default */
 629         engine->sseu =
 630                 intel_sseu_from_device_info(&RUNTIME_INFO(engine->i915)->sseu);
 631
 632         return 0;
 633 }
 634
 635 /**
 636  * intel_engines_setup- setup engine state not requiring hw access
 637  * @i915: Device to setup.
 638  *
 639  * Initializes engine structure members shared between legacy and execlists
 640  * submission modes which do not require hardware access.
 641  *
 642  * Typically done early in the submission mode specific engine setup stage.
 643  */
 644 int intel_engines_setup(struct drm_i915_private *i915)
 645 {
 646         int (*setup)(struct intel_engine_cs *engine);
 647         struct intel_engine_cs *engine;
 648         enum intel_engine_id id;
 649         int err;
 650
 651         if (HAS_EXECLISTS(i915))
 652                 setup = intel_execlists_submission_setup;
 653         else
 654                 setup = intel_ring_submission_setup;
 655
 656         for_each_engine(engine, i915, id) {
 657                 err = intel_engine_setup_common(engine);
 658                 if (err)
 659                         goto cleanup;
 660
 661                 err = setup(engine);
 662                 if (err)
 663                         goto cleanup;
 664
 665                 /* We expect the backend to take control over its state */
 666                 GEM_BUG_ON(engine->destroy == (typeof(engine->destroy))kfree);
 667
 668                 GEM_BUG_ON(!engine->cops);
 669         }
 670
 671         return 0;
 672
 673 cleanup:
 674         intel_engines_cleanup(i915);
 675         return err;
 676 }
 677
 678 void intel_engines_set_scheduler_caps(struct drm_i915_private *i915)
 679 {
 680         static const struct {
 681                 u8 engine;
 682                 u8 sched;
 683         } map[] = {
 684 #define MAP(x, y) { ilog2(I915_ENGINE_HAS_##x), ilog2(I915_SCHEDULER_CAP_##y) }
 685                 MAP(PREEMPTION, PREEMPTION),
 686                 MAP(SEMAPHORES, SEMAPHORES),
 687 #undef MAP
 688         };
 689         struct intel_engine_cs *engine;
 690         enum intel_engine_id id;
 691         u32 enabled, disabled;
 692
 693         enabled = 0;
 694         disabled = 0;
 695         for_each_engine(engine, i915, id) { /* all engines must agree! */
 696                 int i;
 697
 698                 if (engine->schedule)
 699                         enabled |= (I915_SCHEDULER_CAP_ENABLED |
 700                                     I915_SCHEDULER_CAP_PRIORITY);
 701                 else
 702                         disabled |= (I915_SCHEDULER_CAP_ENABLED |
 703                                      I915_SCHEDULER_CAP_PRIORITY);
 704
 705                 for (i = 0; i < ARRAY_SIZE(map); i++) {
 706                         if (engine->flags & BIT(map[i].engine))
 707                                 enabled |= BIT(map[i].sched);
 708                         else
 709                                 disabled |= BIT(map[i].sched);
 710                 }
 711         }
 712
 713         i915->caps.scheduler = enabled & ~disabled;
 714         if (!(i915->caps.scheduler & I915_SCHEDULER_CAP_ENABLED))
 715                 i915->caps.scheduler = 0;
 716 }
 717
 718 struct measure_breadcrumb {
 719         struct i915_request rq;
 720         struct i915_timeline timeline;
 721         struct intel_ring ring;
 722         u32 cs[1024];
 723 };
 724
 725 static int measure_breadcrumb_dw(struct intel_engine_cs *engine)
 726 {
 727         struct measure_breadcrumb *frame;
 728         int dw = -ENOMEM;
 729
 730         GEM_BUG_ON(!engine->i915->gt.scratch);
 731
 732         frame = kzalloc(sizeof(*frame), GFP_KERNEL);
 733         if (!frame)
 734                 return -ENOMEM;
 735
 736         if (i915_timeline_init(engine->i915,
 737                                &frame->timeline,
 738                                engine->status_page.vma))
 739                 goto out_frame;
 740
 741         INIT_LIST_HEAD(&frame->ring.request_list);
 742         frame->ring.timeline = &frame->timeline;
 743         frame->ring.vaddr = frame->cs;
 744         frame->ring.size = sizeof(frame->cs);
 745         frame->ring.effective_size = frame->ring.size;
 746         intel_ring_update_space(&frame->ring);
 747
 748         frame->rq.i915 = engine->i915;
 749         frame->rq.engine = engine;
 750         frame->rq.ring = &frame->ring;
 751         frame->rq.timeline = &frame->timeline;
 752
 753         dw = i915_timeline_pin(&frame->timeline);
 754         if (dw < 0)
 755                 goto out_timeline;
 756
 757         dw = engine->emit_fini_breadcrumb(&frame->rq, frame->cs) - frame->cs;
 758         GEM_BUG_ON(dw & 1); /* RING_TAIL must be qword aligned */
 759
 760         i915_timeline_unpin(&frame->timeline);
 761
 762 out_timeline:
 763         i915_timeline_fini(&frame->timeline);
 764 out_frame:
 765         kfree(frame);
 766         return dw;
 767 }
 768
 769 static int pin_context(struct i915_gem_context *ctx,
 770                        struct intel_engine_cs *engine,
 771                        struct intel_context **out)
 772 {
 773         struct intel_context *ce;
 774         int err;
 775
 776         ce = i915_gem_context_get_engine(ctx, engine->id);
 777         if (IS_ERR(ce))
 778                 return PTR_ERR(ce);
 779
 780         err = intel_context_pin(ce);
 781         intel_context_put(ce);
 782         if (err)
 783                 return err;
 784
 785         *out = ce;
 786         return 0;
 787 }
 788
 789 void
 790 intel_engine_init_active(struct intel_engine_cs *engine, unsigned int subclass)
 791 {
 792         INIT_LIST_HEAD(&engine->active.requests);
 793
 794         spin_lock_init(&engine->active.lock);
 795         lockdep_set_subclass(&engine->active.lock, subclass);
 796
 797         /*
 798          * Due to an interesting quirk in lockdep's internal debug tracking,
 799          * after setting a subclass we must ensure the lock is used. Otherwise,
 800          * nr_unused_locks is incremented once too often.
 801          */
 802 #ifdef CONFIG_DEBUG_LOCK_ALLOC
 803         local_irq_disable();
 804         lock_map_acquire(&engine->active.lock.dep_map);
 805         lock_map_release(&engine->active.lock.dep_map);
 806         local_irq_enable();
 807 #endif
 808 }
 809
 810 /**
 811  * intel_engines_init_common - initialize cengine state which might require hw access
 812  * @engine: Engine to initialize.
 813  *
 814  * Initializes @engine@ structure members shared between legacy and execlists
 815  * submission modes which do require hardware access.
 816  *
 817  * Typcally done at later stages of submission mode specific engine setup.
 818  *
 819  * Returns zero on success or an error code on failure.
 820  */
 821 int intel_engine_init_common(struct intel_engine_cs *engine)
 822 {
 823         struct drm_i915_private *i915 = engine->i915;
 824         int ret;
 825
 826         /* We may need to do things with the shrinker which
 827          * require us to immediately switch back to the default
 828          * context. This can cause a problem as pinning the
 829          * default context also requires GTT space which may not
 830          * be available. To avoid this we always pin the default
 831          * context.
 832          */
 833         ret = pin_context(i915->kernel_context, engine,
 834                           &engine->kernel_context);
 835         if (ret)
 836                 return ret;
 837
 838         /*
 839          * Similarly the preempt context must always be available so that
 840          * we can interrupt the engine at any time. However, as preemption
 841          * is optional, we allow it to fail.
 842          */
 843         if (i915->preempt_context)
 844                 pin_context(i915->preempt_context, engine,
 845                             &engine->preempt_context);
 846
 847         ret = measure_breadcrumb_dw(engine);
 848         if (ret < 0)
 849                 goto err_unpin;
 850
 851         engine->emit_fini_breadcrumb_dw = ret;
 852
 853         engine->set_default_submission(engine);
 854
 855         return 0;
 856
 857 err_unpin:
 858         if (engine->preempt_context)
 859                 intel_context_unpin(engine->preempt_context);
 860         intel_context_unpin(engine->kernel_context);
 861         return ret;
 862 }
 863
 864 /**
 865  * intel_engines_cleanup_common - cleans up the engine state created by
 866  *                                the common initiailizers.
 867  * @engine: Engine to cleanup.
 868  *
 869  * This cleans up everything created by the common helpers.
 870  */
 871 void intel_engine_cleanup_common(struct intel_engine_cs *engine)
 872 {
 873         GEM_BUG_ON(!list_empty(&engine->active.requests));
 874
 875         cleanup_status_page(engine);
 876
 877         intel_engine_fini_breadcrumbs(engine);
 878         intel_engine_cleanup_cmd_parser(engine);
 879         i915_gem_batch_pool_fini(&engine->batch_pool);
 880
 881         if (engine->default_state)
 882                 i915_gem_object_put(engine->default_state);
 883
 884         if (engine->preempt_context)
 885                 intel_context_unpin(engine->preempt_context);
 886         intel_context_unpin(engine->kernel_context);
 887         GEM_BUG_ON(!llist_empty(&engine->barrier_tasks));
 888
 889         intel_wa_list_free(&engine->ctx_wa_list);
 890         intel_wa_list_free(&engine->wa_list);
 891         intel_wa_list_free(&engine->whitelist);
 892 }
 893
 894 u64 intel_engine_get_active_head(const struct intel_engine_cs *engine)
 895 {
 896         struct drm_i915_private *i915 = engine->i915;
 897
 898         u64 acthd;
 899
 900         if (INTEL_GEN(i915) >= 8)
 901                 acthd = ENGINE_READ64(engine, RING_ACTHD, RING_ACTHD_UDW);
 902         else if (INTEL_GEN(i915) >= 4)
 903                 acthd = ENGINE_READ(engine, RING_ACTHD);
 904         else
 905                 acthd = ENGINE_READ(engine, ACTHD);
 906
 907         return acthd;
 908 }
 909
 910 u64 intel_engine_get_last_batch_head(const struct intel_engine_cs *engine)
 911 {
 912         u64 bbaddr;
 913
 914         if (INTEL_GEN(engine->i915) >= 8)
 915                 bbaddr = ENGINE_READ64(engine, RING_BBADDR, RING_BBADDR_UDW);
 916         else
 917                 bbaddr = ENGINE_READ(engine, RING_BBADDR);
 918
 919         return bbaddr;
 920 }
 921
 922 int intel_engine_stop_cs(struct intel_engine_cs *engine)
 923 {
 924         struct intel_uncore *uncore = engine->uncore;
 925         const u32 base = engine->mmio_base;
 926         const i915_reg_t mode = RING_MI_MODE(base);
 927         int err;
 928
 929         if (INTEL_GEN(engine->i915) < 3)
 930                 return -ENODEV;
 931
 932         GEM_TRACE("%s\n", engine->name);
 933
 934         intel_uncore_write_fw(uncore, mode, _MASKED_BIT_ENABLE(STOP_RING));
 935
 936         err = 0;
 937         if (__intel_wait_for_register_fw(uncore,
 938                                          mode, MODE_IDLE, MODE_IDLE,
 939                                          1000, 0,
 940                                          NULL)) {
 941                 GEM_TRACE("%s: timed out on STOP_RING -> IDLE\n", engine->name);
 942                 err = -ETIMEDOUT;
 943         }
 944
 945         /* A final mmio read to let GPU writes be hopefully flushed to memory */
 946         intel_uncore_posting_read_fw(uncore, mode);
 947
 948         return err;
 949 }
 950
 951 void intel_engine_cancel_stop_cs(struct intel_engine_cs *engine)
 952 {
 953         GEM_TRACE("%s\n", engine->name);
 954
 955         ENGINE_WRITE_FW(engine, RING_MI_MODE, _MASKED_BIT_DISABLE(STOP_RING));
 956 }
 957
 958 const char *i915_cache_level_str(struct drm_i915_private *i915, int type)
 959 {
 960         switch (type) {
 961         case I915_CACHE_NONE: return " uncached";
 962         case I915_CACHE_LLC: return HAS_LLC(i915) ? " LLC" : " snooped";
 963         case I915_CACHE_L3_LLC: return " L3+LLC";
 964         case I915_CACHE_WT: return " WT";
 965         default: return "";
 966         }
 967 }
 968
 969 u32 intel_calculate_mcr_s_ss_select(struct drm_i915_private *dev_priv)
 970 {
 971         const struct sseu_dev_info *sseu = &RUNTIME_INFO(dev_priv)->sseu;
 972         u32 mcr_s_ss_select;
 973         u32 slice = fls(sseu->slice_mask);
 974         u32 subslice = fls(sseu->subslice_mask[slice]);
 975
 976         if (IS_GEN(dev_priv, 10))
 977                 mcr_s_ss_select = GEN8_MCR_SLICE(slice) |
 978                                   GEN8_MCR_SUBSLICE(subslice);
 979         else if (INTEL_GEN(dev_priv) >= 11)
 980                 mcr_s_ss_select = GEN11_MCR_SLICE(slice) |
 981                                   GEN11_MCR_SUBSLICE(subslice);
 982         else
 983                 mcr_s_ss_select = 0;
 984
 985         return mcr_s_ss_select;
 986 }
 987
 988 static u32
 989 read_subslice_reg(struct intel_engine_cs *engine, int slice, int subslice,
 990                   i915_reg_t reg)
 991 {
 992         struct drm_i915_private *i915 = engine->i915;
 993         struct intel_uncore *uncore = engine->uncore;
 994         u32 mcr_slice_subslice_mask;
 995         u32 mcr_slice_subslice_select;
 996         u32 default_mcr_s_ss_select;
 997         u32 mcr;
 998         u32 ret;
 999         enum forcewake_domains fw_domains;
1000
1001         if (INTEL_GEN(i915) >= 11) {
1002                 mcr_slice_subslice_mask = GEN11_MCR_SLICE_MASK |
1003                                           GEN11_MCR_SUBSLICE_MASK;
1004                 mcr_slice_subslice_select = GEN11_MCR_SLICE(slice) |
1005                                             GEN11_MCR_SUBSLICE(subslice);
1006         } else {
1007                 mcr_slice_subslice_mask = GEN8_MCR_SLICE_MASK |
1008                                           GEN8_MCR_SUBSLICE_MASK;
1009                 mcr_slice_subslice_select = GEN8_MCR_SLICE(slice) |
1010                                             GEN8_MCR_SUBSLICE(subslice);
1011         }
1012
1013         default_mcr_s_ss_select = intel_calculate_mcr_s_ss_select(i915);
1014
1015         fw_domains = intel_uncore_forcewake_for_reg(uncore, reg,
1016                                                     FW_REG_READ);
1017         fw_domains |= intel_uncore_forcewake_for_reg(uncore,
1018                                                      GEN8_MCR_SELECTOR,
1019                                                      FW_REG_READ | FW_REG_WRITE);
1020
1021         spin_lock_irq(&uncore->lock);
1022         intel_uncore_forcewake_get__locked(uncore, fw_domains);
1023
1024         mcr = intel_uncore_read_fw(uncore, GEN8_MCR_SELECTOR);
1025
1026         WARN_ON_ONCE((mcr & mcr_slice_subslice_mask) !=
1027                      default_mcr_s_ss_select);
1028
1029         mcr &= ~mcr_slice_subslice_mask;
1030         mcr |= mcr_slice_subslice_select;
1031         intel_uncore_write_fw(uncore, GEN8_MCR_SELECTOR, mcr);
1032
1033         ret = intel_uncore_read_fw(uncore, reg);
1034
1035         mcr &= ~mcr_slice_subslice_mask;
1036         mcr |= default_mcr_s_ss_select;
1037
1038         intel_uncore_write_fw(uncore, GEN8_MCR_SELECTOR, mcr);
1039
1040         intel_uncore_forcewake_put__locked(uncore, fw_domains);
1041         spin_unlock_irq(&uncore->lock);
1042
1043         return ret;
1044 }
1045
1046 /* NB: please notice the memset */
1047 void intel_engine_get_instdone(struct intel_engine_cs *engine,
1048                                struct intel_instdone *instdone)
1049 {
1050         struct drm_i915_private *i915 = engine->i915;
1051         struct intel_uncore *uncore = engine->uncore;
1052         u32 mmio_base = engine->mmio_base;
1053         int slice;
1054         int subslice;
1055
1056         memset(instdone, 0, sizeof(*instdone));
1057
1058         switch (INTEL_GEN(i915)) {
1059         default:
1060                 instdone->instdone =
1061                         intel_uncore_read(uncore, RING_INSTDONE(mmio_base));
1062
1063                 if (engine->id != RCS0)
1064                         break;
1065
1066                 instdone->slice_common =
1067                         intel_uncore_read(uncore, GEN7_SC_INSTDONE);
1068                 for_each_instdone_slice_subslice(i915, slice, subslice) {
1069                         instdone->sampler[slice][subslice] =
1070                                 read_subslice_reg(engine, slice, subslice,
1071                                                   GEN7_SAMPLER_INSTDONE);
1072                         instdone->row[slice][subslice] =
1073                                 read_subslice_reg(engine, slice, subslice,
1074                                                   GEN7_ROW_INSTDONE);
1075                 }
1076                 break;
1077         case 7:
1078                 instdone->instdone =
1079                         intel_uncore_read(uncore, RING_INSTDONE(mmio_base));
1080
1081                 if (engine->id != RCS0)
1082                         break;
1083
1084                 instdone->slice_common =
1085                         intel_uncore_read(uncore, GEN7_SC_INSTDONE);
1086                 instdone->sampler[0][0] =
1087                         intel_uncore_read(uncore, GEN7_SAMPLER_INSTDONE);
1088                 instdone->row[0][0] =
1089                         intel_uncore_read(uncore, GEN7_ROW_INSTDONE);
1090
1091                 break;
1092         case 6:
1093         case 5:
1094         case 4:
1095                 instdone->instdone =
1096                         intel_uncore_read(uncore, RING_INSTDONE(mmio_base));
1097                 if (engine->id == RCS0)
1098                         /* HACK: Using the wrong struct member */
1099                         instdone->slice_common =
1100                                 intel_uncore_read(uncore, GEN4_INSTDONE1);
1101                 break;
1102         case 3:
1103         case 2:
1104                 instdone->instdone = intel_uncore_read(uncore, GEN2_INSTDONE);
1105                 break;
1106         }
1107 }
1108
1109 static bool ring_is_idle(struct intel_engine_cs *engine)
1110 {
1111         struct drm_i915_private *dev_priv = engine->i915;
1112         intel_wakeref_t wakeref;
1113         bool idle = true;
1114
1115         if (I915_SELFTEST_ONLY(!engine->mmio_base))
1116                 return true;
1117
1118         /* If the whole device is asleep, the engine must be idle */
1119         wakeref = intel_runtime_pm_get_if_in_use(&dev_priv->runtime_pm);
1120         if (!wakeref)
1121                 return true;
1122
1123         /* First check that no commands are left in the ring */
1124         if ((ENGINE_READ(engine, RING_HEAD) & HEAD_ADDR) !=
1125             (ENGINE_READ(engine, RING_TAIL) & TAIL_ADDR))
1126                 idle = false;
1127
1128         /* No bit for gen2, so assume the CS parser is idle */
1129         if (INTEL_GEN(dev_priv) > 2 &&
1130             !(ENGINE_READ(engine, RING_MI_MODE) & MODE_IDLE))
1131                 idle = false;
1132
1133         intel_runtime_pm_put(&dev_priv->runtime_pm, wakeref);
1134
1135         return idle;
1136 }
1137
1138 /**
1139  * intel_engine_is_idle() - Report if the engine has finished process all work
1140  * @engine: the intel_engine_cs
1141  *
1142  * Return true if there are no requests pending, nothing left to be submitted
1143  * to hardware, and that the engine is idle.
1144  */
1145 bool intel_engine_is_idle(struct intel_engine_cs *engine)
1146 {
1147         /* More white lies, if wedged, hw state is inconsistent */
1148         if (i915_reset_failed(engine->i915))
1149                 return true;
1150
1151         if (!intel_wakeref_active(&engine->wakeref))
1152                 return true;
1153
1154         /* Waiting to drain ELSP? */
1155         if (READ_ONCE(engine->execlists.active)) {
1156                 struct tasklet_struct *t = &engine->execlists.tasklet;
1157
1158                 synchronize_hardirq(engine->i915->drm.irq);
1159
1160                 local_bh_disable();
1161                 if (tasklet_trylock(t)) {
1162                         /* Must wait for any GPU reset in progress. */
1163                         if (__tasklet_is_enabled(t))
1164                                 t->func(t->data);
1165                         tasklet_unlock(t);
1166                 }
1167                 local_bh_enable();
1168
1169                 /* Otherwise flush the tasklet if it was on another cpu */
1170                 tasklet_unlock_wait(t);
1171
1172                 if (READ_ONCE(engine->execlists.active))
1173                         return false;
1174         }
1175
1176         /* ELSP is empty, but there are ready requests? E.g. after reset */
1177         if (!RB_EMPTY_ROOT(&engine->execlists.queue.rb_root))
1178                 return false;
1179
1180         /* Ring stopped? */
1181         return ring_is_idle(engine);
1182 }
1183
1184 bool intel_engines_are_idle(struct drm_i915_private *i915)
1185 {
1186         struct intel_engine_cs *engine;
1187         enum intel_engine_id id;
1188
1189         /*
1190          * If the driver is wedged, HW state may be very inconsistent and
1191          * report that it is still busy, even though we have stopped using it.
1192          */
1193         if (i915_reset_failed(i915))
1194                 return true;
1195
1196         /* Already parked (and passed an idleness test); must still be idle */
1197         if (!READ_ONCE(i915->gt.awake))
1198                 return true;
1199
1200         for_each_engine(engine, i915, id) {
1201                 if (!intel_engine_is_idle(engine))
1202                         return false;
1203         }
1204
1205         return true;
1206 }
1207
1208 void intel_engines_reset_default_submission(struct drm_i915_private *i915)
1209 {
1210         struct intel_engine_cs *engine;
1211         enum intel_engine_id id;
1212
1213         for_each_engine(engine, i915, id)
1214                 engine->set_default_submission(engine);
1215 }
1216
1217 bool intel_engine_can_store_dword(struct intel_engine_cs *engine)
1218 {
1219         switch (INTEL_GEN(engine->i915)) {
1220         case 2:
1221                 return false; /* uses physical not virtual addresses */
1222         case 3:
1223                 /* maybe only uses physical not virtual addresses */
1224                 return !(IS_I915G(engine->i915) || IS_I915GM(engine->i915));
1225         case 6:
1226                 return engine->class != VIDEO_DECODE_CLASS; /* b0rked */
1227         default:
1228                 return true;
1229         }
1230 }
1231
1232 unsigned int intel_engines_has_context_isolation(struct drm_i915_private *i915)
1233 {
1234         struct intel_engine_cs *engine;
1235         enum intel_engine_id id;
1236         unsigned int which;
1237
1238         which = 0;
1239         for_each_engine(engine, i915, id)
1240                 if (engine->default_state)
1241                         which |= BIT(engine->uabi_class);
1242
1243         return which;
1244 }
1245
1246 static int print_sched_attr(struct drm_i915_private *i915,
1247                             const struct i915_sched_attr *attr,
1248                             char *buf, int x, int len)
1249 {
1250         if (attr->priority == I915_PRIORITY_INVALID)
1251                 return x;
1252
1253         x += snprintf(buf + x, len - x,
1254                       " prio=%d", attr->priority);
1255
1256         return x;
1257 }
1258
1259 static void print_request(struct drm_printer *m,
1260                           struct i915_request *rq,
1261                           const char *prefix)
1262 {
1263         const char *name = rq->fence.ops->get_timeline_name(&rq->fence);
1264         char buf[80] = "";
1265         int x = 0;
1266
1267         x = print_sched_attr(rq->i915, &rq->sched.attr, buf, x, sizeof(buf));
1268
1269         drm_printf(m, "%s %llx:%llx%s%s %s @ %dms: %s\n",
1270                    prefix,
1271                    rq->fence.context, rq->fence.seqno,
1272                    i915_request_completed(rq) ? "!" :
1273                    i915_request_started(rq) ? "*" :
1274                    "",
1275                    test_bit(DMA_FENCE_FLAG_SIGNALED_BIT,
1276                             &rq->fence.flags) ? "+" :
1277                    test_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT,
1278                             &rq->fence.flags) ? "-" :
1279                    "",
1280                    buf,
1281                    jiffies_to_msecs(jiffies - rq->emitted_jiffies),
1282                    name);
1283 }
1284
1285 static void hexdump(struct drm_printer *m, const void *buf, size_t len)
1286 {
1287         const size_t rowsize = 8 * sizeof(u32);
1288         const void *prev = NULL;
1289         bool skip = false;
1290         size_t pos;
1291
1292         for (pos = 0; pos < len; pos += rowsize) {
1293                 char line[128];
1294
1295                 if (prev && !memcmp(prev, buf + pos, rowsize)) {
1296                         if (!skip) {
1297                                 drm_printf(m, "*\n");
1298                                 skip = true;
1299                         }
1300                         continue;
1301                 }
1302
1303                 WARN_ON_ONCE(hex_dump_to_buffer(buf + pos, len - pos,
1304                                                 rowsize, sizeof(u32),
1305                                                 line, sizeof(line),
1306                                                 false) >= sizeof(line));
1307                 drm_printf(m, "[%04zx] %s\n", pos, line);
1308
1309                 prev = buf + pos;
1310                 skip = false;
1311         }
1312 }
1313
1314 static void intel_engine_print_registers(struct intel_engine_cs *engine,
1315                                          struct drm_printer *m)
1316 {
1317         struct drm_i915_private *dev_priv = engine->i915;
1318         const struct intel_engine_execlists * const execlists =
1319                 &engine->execlists;
1320         unsigned long flags;
1321         u64 addr;
1322
1323         if (engine->id == RCS0 && IS_GEN_RANGE(dev_priv, 4, 7))
1324                 drm_printf(m, "\tCCID: 0x%08x\n", ENGINE_READ(engine, CCID));
1325         drm_printf(m, "\tRING_START: 0x%08x\n",
1326                    ENGINE_READ(engine, RING_START));
1327         drm_printf(m, "\tRING_HEAD:  0x%08x\n",
1328                    ENGINE_READ(engine, RING_HEAD) & HEAD_ADDR);
1329         drm_printf(m, "\tRING_TAIL:  0x%08x\n",
1330                    ENGINE_READ(engine, RING_TAIL) & TAIL_ADDR);
1331         drm_printf(m, "\tRING_CTL:   0x%08x%s\n",
1332                    ENGINE_READ(engine, RING_CTL),
1333                    ENGINE_READ(engine, RING_CTL) & (RING_WAIT | RING_WAIT_SEMAPHORE) ? " [waiting]" : "");
1334         if (INTEL_GEN(engine->i915) > 2) {
1335                 drm_printf(m, "\tRING_MODE:  0x%08x%s\n",
1336                            ENGINE_READ(engine, RING_MI_MODE),
1337                            ENGINE_READ(engine, RING_MI_MODE) & (MODE_IDLE) ? " [idle]" : "");
1338         }
1339
1340         if (INTEL_GEN(dev_priv) >= 6) {
1341                 drm_printf(m, "\tRING_IMR: %08x\n",
1342                            ENGINE_READ(engine, RING_IMR));
1343         }
1344
1345         addr = intel_engine_get_active_head(engine);
1346         drm_printf(m, "\tACTHD:  0x%08x_%08x\n",
1347                    upper_32_bits(addr), lower_32_bits(addr));
1348         addr = intel_engine_get_last_batch_head(engine);
1349         drm_printf(m, "\tBBADDR: 0x%08x_%08x\n",
1350                    upper_32_bits(addr), lower_32_bits(addr));
1351         if (INTEL_GEN(dev_priv) >= 8)
1352                 addr = ENGINE_READ64(engine, RING_DMA_FADD, RING_DMA_FADD_UDW);
1353         else if (INTEL_GEN(dev_priv) >= 4)
1354                 addr = ENGINE_READ(engine, RING_DMA_FADD);
1355         else
1356                 addr = ENGINE_READ(engine, DMA_FADD_I8XX);
1357         drm_printf(m, "\tDMA_FADDR: 0x%08x_%08x\n",
1358                    upper_32_bits(addr), lower_32_bits(addr));
1359         if (INTEL_GEN(dev_priv) >= 4) {
1360                 drm_printf(m, "\tIPEIR: 0x%08x\n",
1361                            ENGINE_READ(engine, RING_IPEIR));
1362                 drm_printf(m, "\tIPEHR: 0x%08x\n",
1363                            ENGINE_READ(engine, RING_IPEHR));
1364         } else {
1365                 drm_printf(m, "\tIPEIR: 0x%08x\n", ENGINE_READ(engine, IPEIR));
1366                 drm_printf(m, "\tIPEHR: 0x%08x\n", ENGINE_READ(engine, IPEHR));
1367         }
1368
1369         if (HAS_EXECLISTS(dev_priv)) {
1370                 const u32 *hws =
1371                         &engine->status_page.addr[I915_HWS_CSB_BUF0_INDEX];
1372                 const u8 num_entries = execlists->csb_size;
1373                 unsigned int idx;
1374                 u8 read, write;
1375
1376                 drm_printf(m, "\tExeclist status: 0x%08x %08x, entries %u\n",
1377                            ENGINE_READ(engine, RING_EXECLIST_STATUS_LO),
1378                            ENGINE_READ(engine, RING_EXECLIST_STATUS_HI),
1379                            num_entries);
1380
1381                 read = execlists->csb_head;
1382                 write = READ_ONCE(*execlists->csb_write);
1383
1384                 drm_printf(m, "\tExeclist CSB read %d, write %d, tasklet queued? %s (%s)\n",
1385                            read, write,
1386                            yesno(test_bit(TASKLET_STATE_SCHED,
1387                                           &engine->execlists.tasklet.state)),
1388                            enableddisabled(!atomic_read(&engine->execlists.tasklet.count)));
1389                 if (read >= num_entries)
1390                         read = 0;
1391                 if (write >= num_entries)
1392                         write = 0;
1393                 if (read > write)
1394                         write += num_entries;
1395                 while (read < write) {
1396                         idx = ++read % num_entries;
1397                         drm_printf(m, "\tExeclist CSB[%d]: 0x%08x, context: %d\n",
1398                                    idx, hws[idx * 2], hws[idx * 2 + 1]);
1399                 }
1400
1401                 spin_lock_irqsave(&engine->active.lock, flags);
1402                 for (idx = 0; idx < execlists_num_ports(execlists); idx++) {
1403                         struct i915_request *rq;
1404                         unsigned int count;
1405                         char hdr[80];
1406
1407                         rq = port_unpack(&execlists->port[idx], &count);
1408                         if (!rq) {
1409                                 drm_printf(m, "\t\tELSP[%d] idle\n", idx);
1410                         } else if (!i915_request_signaled(rq)) {
1411                                 snprintf(hdr, sizeof(hdr),
1412                                          "\t\tELSP[%d] count=%d, ring:{start:%08x, hwsp:%08x, seqno:%08x}, rq: ",
1413                                          idx, count,
1414                                          i915_ggtt_offset(rq->ring->vma),
1415                                          rq->timeline->hwsp_offset,
1416                                          hwsp_seqno(rq));
1417                                 print_request(m, rq, hdr);
1418                         } else {
1419                                 print_request(m, rq, "\t\tELSP[%d] rq: ");
1420                         }
1421                 }
1422                 drm_printf(m, "\t\tHW active? 0x%x\n", execlists->active);
1423                 spin_unlock_irqrestore(&engine->active.lock, flags);
1424         } else if (INTEL_GEN(dev_priv) > 6) {
1425                 drm_printf(m, "\tPP_DIR_BASE: 0x%08x\n",
1426                            ENGINE_READ(engine, RING_PP_DIR_BASE));
1427                 drm_printf(m, "\tPP_DIR_BASE_READ: 0x%08x\n",
1428                            ENGINE_READ(engine, RING_PP_DIR_BASE_READ));
1429                 drm_printf(m, "\tPP_DIR_DCLV: 0x%08x\n",
1430                            ENGINE_READ(engine, RING_PP_DIR_DCLV));
1431         }
1432 }
1433
1434 static void print_request_ring(struct drm_printer *m, struct i915_request *rq)
1435 {
1436         void *ring;
1437         int size;
1438
1439         drm_printf(m,
1440                    "[head %04x, postfix %04x, tail %04x, batch 0x%08x_%08x]:\n",
1441                    rq->head, rq->postfix, rq->tail,
1442                    rq->batch ? upper_32_bits(rq->batch->node.start) : ~0u,
1443                    rq->batch ? lower_32_bits(rq->batch->node.start) : ~0u);
1444
1445         size = rq->tail - rq->head;
1446         if (rq->tail < rq->head)
1447                 size += rq->ring->size;
1448
1449         ring = kmalloc(size, GFP_ATOMIC);
1450         if (ring) {
1451                 const void *vaddr = rq->ring->vaddr;
1452                 unsigned int head = rq->head;
1453                 unsigned int len = 0;
1454
1455                 if (rq->tail < head) {
1456                         len = rq->ring->size - head;
1457                         memcpy(ring, vaddr + head, len);
1458                         head = 0;
1459                 }
1460                 memcpy(ring + len, vaddr + head, size - len);
1461
1462                 hexdump(m, ring, size);
1463                 kfree(ring);
1464         }
1465 }
1466
1467 void intel_engine_dump(struct intel_engine_cs *engine,
1468                        struct drm_printer *m,
1469                        const char *header, ...)
1470 {
1471         struct i915_gpu_error * const error = &engine->i915->gpu_error;
1472         struct i915_request *rq;
1473         intel_wakeref_t wakeref;
1474
1475         if (header) {
1476                 va_list ap;
1477
1478                 va_start(ap, header);
1479                 drm_vprintf(m, header, &ap);
1480                 va_end(ap);
1481         }
1482
1483         if (i915_reset_failed(engine->i915))
1484                 drm_printf(m, "*** WEDGED ***\n");
1485
1486         drm_printf(m, "\tAwake? %d\n", atomic_read(&engine->wakeref.count));
1487         drm_printf(m, "\tHangcheck: %d ms ago\n",
1488                    jiffies_to_msecs(jiffies - engine->hangcheck.action_timestamp));
1489         drm_printf(m, "\tReset count: %d (global %d)\n",
1490                    i915_reset_engine_count(error, engine),
1491                    i915_reset_count(error));
1492
1493         rcu_read_lock();
1494
1495         drm_printf(m, "\tRequests:\n");
1496
1497         rq = intel_engine_find_active_request(engine);
1498         if (rq) {
1499                 print_request(m, rq, "\t\tactive ");
1500
1501                 drm_printf(m, "\t\tring->start:  0x%08x\n",
1502                            i915_ggtt_offset(rq->ring->vma));
1503                 drm_printf(m, "\t\tring->head:   0x%08x\n",
1504                            rq->ring->head);
1505                 drm_printf(m, "\t\tring->tail:   0x%08x\n",
1506                            rq->ring->tail);
1507                 drm_printf(m, "\t\tring->emit:   0x%08x\n",
1508                            rq->ring->emit);
1509                 drm_printf(m, "\t\tring->space:  0x%08x\n",
1510                            rq->ring->space);
1511                 drm_printf(m, "\t\tring->hwsp:   0x%08x\n",
1512                            rq->timeline->hwsp_offset);
1513
1514                 print_request_ring(m, rq);
1515         }
1516
1517         rcu_read_unlock();
1518
1519         wakeref = intel_runtime_pm_get_if_in_use(&engine->i915->runtime_pm);
1520         if (wakeref) {
1521                 intel_engine_print_registers(engine, m);
1522                 intel_runtime_pm_put(&engine->i915->runtime_pm, wakeref);
1523         } else {
1524                 drm_printf(m, "\tDevice is asleep; skipping register dump\n");
1525         }
1526
1527         intel_execlists_show_requests(engine, m, print_request, 8);
1528
1529         drm_printf(m, "HWSP:\n");
1530         hexdump(m, engine->status_page.addr, PAGE_SIZE);
1531
1532         drm_printf(m, "Idle? %s\n", yesno(intel_engine_is_idle(engine)));
1533
1534         intel_engine_print_breadcrumbs(engine, m);
1535 }
1536
1537 static u8 user_class_map[] = {
1538         [I915_ENGINE_CLASS_RENDER] = RENDER_CLASS,
1539         [I915_ENGINE_CLASS_COPY] = COPY_ENGINE_CLASS,
1540         [I915_ENGINE_CLASS_VIDEO] = VIDEO_DECODE_CLASS,
1541         [I915_ENGINE_CLASS_VIDEO_ENHANCE] = VIDEO_ENHANCEMENT_CLASS,
1542 };
1543
1544 struct intel_engine_cs *
1545 intel_engine_lookup_user(struct drm_i915_private *i915, u8 class, u8 instance)
1546 {
1547         if (class >= ARRAY_SIZE(user_class_map))
1548                 return NULL;
1549
1550         class = user_class_map[class];
1551
1552         GEM_BUG_ON(class > MAX_ENGINE_CLASS);
1553
1554         if (instance > MAX_ENGINE_INSTANCE)
1555                 return NULL;
1556
1557         return i915->engine_class[class][instance];
1558 }
1559
1560 /**
1561  * intel_enable_engine_stats() - Enable engine busy tracking on engine
1562  * @engine: engine to enable stats collection
1563  *
1564  * Start collecting the engine busyness data for @engine.
1565  *
1566  * Returns 0 on success or a negative error code.
1567  */
1568 int intel_enable_engine_stats(struct intel_engine_cs *engine)
1569 {
1570         struct intel_engine_execlists *execlists = &engine->execlists;
1571         unsigned long flags;
1572         int err = 0;
1573
1574         if (!intel_engine_supports_stats(engine))
1575                 return -ENODEV;
1576
1577         spin_lock_irqsave(&engine->active.lock, flags);
1578         write_seqlock(&engine->stats.lock);
1579
1580         if (unlikely(engine->stats.enabled == ~0)) {
1581                 err = -EBUSY;
1582                 goto unlock;
1583         }
1584
1585         if (engine->stats.enabled++ == 0) {
1586                 const struct execlist_port *port = execlists->port;
1587                 unsigned int num_ports = execlists_num_ports(execlists);
1588
1589                 engine->stats.enabled_at = ktime_get();
1590
1591                 /* XXX submission method oblivious? */
1592                 while (num_ports-- && port_isset(port)) {
1593                         engine->stats.active++;
1594                         port++;
1595                 }
1596
1597                 if (engine->stats.active)
1598                         engine->stats.start = engine->stats.enabled_at;
1599         }
1600
1601 unlock:
1602         write_sequnlock(&engine->stats.lock);
1603         spin_unlock_irqrestore(&engine->active.lock, flags);
1604
1605         return err;
1606 }
1607
1608 static ktime_t __intel_engine_get_busy_time(struct intel_engine_cs *engine)
1609 {
1610         ktime_t total = engine->stats.total;
1611
1612         /*
1613          * If the engine is executing something at the moment
1614          * add it to the total.
1615          */
1616         if (engine->stats.active)
1617                 total = ktime_add(total,
1618                                   ktime_sub(ktime_get(), engine->stats.start));
1619
1620         return total;
1621 }
1622
1623 /**
1624  * intel_engine_get_busy_time() - Return current accumulated engine busyness
1625  * @engine: engine to report on
1626  *
1627  * Returns accumulated time @engine was busy since engine stats were enabled.
1628  */
1629 ktime_t intel_engine_get_busy_time(struct intel_engine_cs *engine)
1630 {
1631         unsigned int seq;
1632         ktime_t total;
1633
1634         do {
1635                 seq = read_seqbegin(&engine->stats.lock);
1636                 total = __intel_engine_get_busy_time(engine);
1637         } while (read_seqretry(&engine->stats.lock, seq));
1638
1639         return total;
1640 }
1641
1642 /**
1643  * intel_disable_engine_stats() - Disable engine busy tracking on engine
1644  * @engine: engine to disable stats collection
1645  *
1646  * Stops collecting the engine busyness data for @engine.
1647  */
1648 void intel_disable_engine_stats(struct intel_engine_cs *engine)
1649 {
1650         unsigned long flags;
1651
1652         if (!intel_engine_supports_stats(engine))
1653                 return;
1654
1655         write_seqlock_irqsave(&engine->stats.lock, flags);
1656         WARN_ON_ONCE(engine->stats.enabled == 0);
1657         if (--engine->stats.enabled == 0) {
1658                 engine->stats.total = __intel_engine_get_busy_time(engine);
1659                 engine->stats.active = 0;
1660         }
1661         write_sequnlock_irqrestore(&engine->stats.lock, flags);
1662 }
1663
1664 static bool match_ring(struct i915_request *rq)
1665 {
1666         u32 ring = ENGINE_READ(rq->engine, RING_START);
1667
1668         return ring == i915_ggtt_offset(rq->ring->vma);
1669 }
1670
1671 struct i915_request *
1672 intel_engine_find_active_request(struct intel_engine_cs *engine)
1673 {
1674         struct i915_request *request, *active = NULL;
1675         unsigned long flags;
1676
1677         /*
1678          * We are called by the error capture, reset and to dump engine
1679          * state at random points in time. In particular, note that neither is
1680          * crucially ordered with an interrupt. After a hang, the GPU is dead
1681          * and we assume that no more writes can happen (we waited long enough
1682          * for all writes that were in transaction to be flushed) - adding an
1683          * extra delay for a recent interrupt is pointless. Hence, we do
1684          * not need an engine->irq_seqno_barrier() before the seqno reads.
1685          * At all other times, we must assume the GPU is still running, but
1686          * we only care about the snapshot of this moment.
1687          */
1688         spin_lock_irqsave(&engine->active.lock, flags);
1689         list_for_each_entry(request, &engine->active.requests, sched.link) {
1690                 if (i915_request_completed(request))
1691                         continue;
1692
1693                 if (!i915_request_started(request))
1694                         continue;
1695
1696                 /* More than one preemptible request may match! */
1697                 if (!match_ring(request))
1698                         continue;
1699
1700                 active = request;
1701                 break;
1702         }
1703         spin_unlock_irqrestore(&engine->active.lock, flags);
1704
1705         return active;
1706 }
1707
1708 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
1709 #include "selftest_engine_cs.c"
1710 #endif