ab5bf4e2e28e21f3daa7072a37cb6895aba8233c
[sfrench/cifs-2.6.git] / drivers / gpu / drm / i915 / intel_engine_cs.c
1 /*
2  * Copyright © 2016 Intel Corporation
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice (including the next
12  * paragraph) shall be included in all copies or substantial portions of the
13  * Software.
14  *
15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21  * IN THE SOFTWARE.
22  *
23  */
24
25 #include <drm/drm_print.h>
26
27 #include "i915_drv.h"
28 #include "i915_vgpu.h"
29 #include "intel_ringbuffer.h"
30 #include "intel_lrc.h"
31
32 /* Haswell does have the CXT_SIZE register however it does not appear to be
33  * valid. Now, docs explain in dwords what is in the context object. The full
34  * size is 70720 bytes, however, the power context and execlist context will
35  * never be saved (power context is stored elsewhere, and execlists don't work
36  * on HSW) - so the final size, including the extra state required for the
37  * Resource Streamer, is 66944 bytes, which rounds to 17 pages.
38  */
39 #define HSW_CXT_TOTAL_SIZE              (17 * PAGE_SIZE)
40 /* Same as Haswell, but 72064 bytes now. */
41 #define GEN8_CXT_TOTAL_SIZE             (18 * PAGE_SIZE)
42
43 #define GEN8_LR_CONTEXT_RENDER_SIZE     (20 * PAGE_SIZE)
44 #define GEN9_LR_CONTEXT_RENDER_SIZE     (22 * PAGE_SIZE)
45 #define GEN10_LR_CONTEXT_RENDER_SIZE    (18 * PAGE_SIZE)
46
47 #define GEN8_LR_CONTEXT_OTHER_SIZE      ( 2 * PAGE_SIZE)
48
49 struct engine_class_info {
50         const char *name;
51         int (*init_legacy)(struct intel_engine_cs *engine);
52         int (*init_execlists)(struct intel_engine_cs *engine);
53 };
54
55 static const struct engine_class_info intel_engine_classes[] = {
56         [RENDER_CLASS] = {
57                 .name = "rcs",
58                 .init_execlists = logical_render_ring_init,
59                 .init_legacy = intel_init_render_ring_buffer,
60         },
61         [COPY_ENGINE_CLASS] = {
62                 .name = "bcs",
63                 .init_execlists = logical_xcs_ring_init,
64                 .init_legacy = intel_init_blt_ring_buffer,
65         },
66         [VIDEO_DECODE_CLASS] = {
67                 .name = "vcs",
68                 .init_execlists = logical_xcs_ring_init,
69                 .init_legacy = intel_init_bsd_ring_buffer,
70         },
71         [VIDEO_ENHANCEMENT_CLASS] = {
72                 .name = "vecs",
73                 .init_execlists = logical_xcs_ring_init,
74                 .init_legacy = intel_init_vebox_ring_buffer,
75         },
76 };
77
78 struct engine_info {
79         unsigned int hw_id;
80         unsigned int uabi_id;
81         u8 class;
82         u8 instance;
83         u32 mmio_base;
84         unsigned irq_shift;
85 };
86
87 static const struct engine_info intel_engines[] = {
88         [RCS] = {
89                 .hw_id = RCS_HW,
90                 .uabi_id = I915_EXEC_RENDER,
91                 .class = RENDER_CLASS,
92                 .instance = 0,
93                 .mmio_base = RENDER_RING_BASE,
94                 .irq_shift = GEN8_RCS_IRQ_SHIFT,
95         },
96         [BCS] = {
97                 .hw_id = BCS_HW,
98                 .uabi_id = I915_EXEC_BLT,
99                 .class = COPY_ENGINE_CLASS,
100                 .instance = 0,
101                 .mmio_base = BLT_RING_BASE,
102                 .irq_shift = GEN8_BCS_IRQ_SHIFT,
103         },
104         [VCS] = {
105                 .hw_id = VCS_HW,
106                 .uabi_id = I915_EXEC_BSD,
107                 .class = VIDEO_DECODE_CLASS,
108                 .instance = 0,
109                 .mmio_base = GEN6_BSD_RING_BASE,
110                 .irq_shift = GEN8_VCS1_IRQ_SHIFT,
111         },
112         [VCS2] = {
113                 .hw_id = VCS2_HW,
114                 .uabi_id = I915_EXEC_BSD,
115                 .class = VIDEO_DECODE_CLASS,
116                 .instance = 1,
117                 .mmio_base = GEN8_BSD2_RING_BASE,
118                 .irq_shift = GEN8_VCS2_IRQ_SHIFT,
119         },
120         [VECS] = {
121                 .hw_id = VECS_HW,
122                 .uabi_id = I915_EXEC_VEBOX,
123                 .class = VIDEO_ENHANCEMENT_CLASS,
124                 .instance = 0,
125                 .mmio_base = VEBOX_RING_BASE,
126                 .irq_shift = GEN8_VECS_IRQ_SHIFT,
127         },
128 };
129
130 /**
131  * ___intel_engine_context_size() - return the size of the context for an engine
132  * @dev_priv: i915 device private
133  * @class: engine class
134  *
135  * Each engine class may require a different amount of space for a context
136  * image.
137  *
138  * Return: size (in bytes) of an engine class specific context image
139  *
140  * Note: this size includes the HWSP, which is part of the context image
141  * in LRC mode, but does not include the "shared data page" used with
142  * GuC submission. The caller should account for this if using the GuC.
143  */
144 static u32
145 __intel_engine_context_size(struct drm_i915_private *dev_priv, u8 class)
146 {
147         u32 cxt_size;
148
149         BUILD_BUG_ON(I915_GTT_PAGE_SIZE != PAGE_SIZE);
150
151         switch (class) {
152         case RENDER_CLASS:
153                 switch (INTEL_GEN(dev_priv)) {
154                 default:
155                         MISSING_CASE(INTEL_GEN(dev_priv));
156                 case 10:
157                         return GEN10_LR_CONTEXT_RENDER_SIZE;
158                 case 9:
159                         return GEN9_LR_CONTEXT_RENDER_SIZE;
160                 case 8:
161                         return i915_modparams.enable_execlists ?
162                                GEN8_LR_CONTEXT_RENDER_SIZE :
163                                GEN8_CXT_TOTAL_SIZE;
164                 case 7:
165                         if (IS_HASWELL(dev_priv))
166                                 return HSW_CXT_TOTAL_SIZE;
167
168                         cxt_size = I915_READ(GEN7_CXT_SIZE);
169                         return round_up(GEN7_CXT_TOTAL_SIZE(cxt_size) * 64,
170                                         PAGE_SIZE);
171                 case 6:
172                         cxt_size = I915_READ(CXT_SIZE);
173                         return round_up(GEN6_CXT_TOTAL_SIZE(cxt_size) * 64,
174                                         PAGE_SIZE);
175                 case 5:
176                 case 4:
177                 case 3:
178                 case 2:
179                 /* For the special day when i810 gets merged. */
180                 case 1:
181                         return 0;
182                 }
183                 break;
184         default:
185                 MISSING_CASE(class);
186         case VIDEO_DECODE_CLASS:
187         case VIDEO_ENHANCEMENT_CLASS:
188         case COPY_ENGINE_CLASS:
189                 if (INTEL_GEN(dev_priv) < 8)
190                         return 0;
191                 return GEN8_LR_CONTEXT_OTHER_SIZE;
192         }
193 }
194
195 static int
196 intel_engine_setup(struct drm_i915_private *dev_priv,
197                    enum intel_engine_id id)
198 {
199         const struct engine_info *info = &intel_engines[id];
200         const struct engine_class_info *class_info;
201         struct intel_engine_cs *engine;
202
203         GEM_BUG_ON(info->class >= ARRAY_SIZE(intel_engine_classes));
204         class_info = &intel_engine_classes[info->class];
205
206         GEM_BUG_ON(dev_priv->engine[id]);
207         engine = kzalloc(sizeof(*engine), GFP_KERNEL);
208         if (!engine)
209                 return -ENOMEM;
210
211         engine->id = id;
212         engine->i915 = dev_priv;
213         WARN_ON(snprintf(engine->name, sizeof(engine->name), "%s%u",
214                          class_info->name, info->instance) >=
215                 sizeof(engine->name));
216         engine->uabi_id = info->uabi_id;
217         engine->hw_id = engine->guc_id = info->hw_id;
218         engine->mmio_base = info->mmio_base;
219         engine->irq_shift = info->irq_shift;
220         engine->class = info->class;
221         engine->instance = info->instance;
222
223         engine->context_size = __intel_engine_context_size(dev_priv,
224                                                            engine->class);
225         if (WARN_ON(engine->context_size > BIT(20)))
226                 engine->context_size = 0;
227
228         /* Nothing to do here, execute in order of dependencies */
229         engine->schedule = NULL;
230
231         ATOMIC_INIT_NOTIFIER_HEAD(&engine->context_status_notifier);
232
233         dev_priv->engine[id] = engine;
234         return 0;
235 }
236
237 /**
238  * intel_engines_init_mmio() - allocate and prepare the Engine Command Streamers
239  * @dev_priv: i915 device private
240  *
241  * Return: non-zero if the initialization failed.
242  */
243 int intel_engines_init_mmio(struct drm_i915_private *dev_priv)
244 {
245         struct intel_device_info *device_info = mkwrite_device_info(dev_priv);
246         const unsigned int ring_mask = INTEL_INFO(dev_priv)->ring_mask;
247         struct intel_engine_cs *engine;
248         enum intel_engine_id id;
249         unsigned int mask = 0;
250         unsigned int i;
251         int err;
252
253         WARN_ON(ring_mask == 0);
254         WARN_ON(ring_mask &
255                 GENMASK(sizeof(mask) * BITS_PER_BYTE - 1, I915_NUM_ENGINES));
256
257         for (i = 0; i < ARRAY_SIZE(intel_engines); i++) {
258                 if (!HAS_ENGINE(dev_priv, i))
259                         continue;
260
261                 err = intel_engine_setup(dev_priv, i);
262                 if (err)
263                         goto cleanup;
264
265                 mask |= ENGINE_MASK(i);
266         }
267
268         /*
269          * Catch failures to update intel_engines table when the new engines
270          * are added to the driver by a warning and disabling the forgotten
271          * engines.
272          */
273         if (WARN_ON(mask != ring_mask))
274                 device_info->ring_mask = mask;
275
276         /* We always presume we have at least RCS available for later probing */
277         if (WARN_ON(!HAS_ENGINE(dev_priv, RCS))) {
278                 err = -ENODEV;
279                 goto cleanup;
280         }
281
282         device_info->num_rings = hweight32(mask);
283
284         return 0;
285
286 cleanup:
287         for_each_engine(engine, dev_priv, id)
288                 kfree(engine);
289         return err;
290 }
291
292 /**
293  * intel_engines_init() - init the Engine Command Streamers
294  * @dev_priv: i915 device private
295  *
296  * Return: non-zero if the initialization failed.
297  */
298 int intel_engines_init(struct drm_i915_private *dev_priv)
299 {
300         struct intel_engine_cs *engine;
301         enum intel_engine_id id, err_id;
302         int err;
303
304         for_each_engine(engine, dev_priv, id) {
305                 const struct engine_class_info *class_info =
306                         &intel_engine_classes[engine->class];
307                 int (*init)(struct intel_engine_cs *engine);
308
309                 if (i915_modparams.enable_execlists)
310                         init = class_info->init_execlists;
311                 else
312                         init = class_info->init_legacy;
313
314                 err = -EINVAL;
315                 err_id = id;
316
317                 if (GEM_WARN_ON(!init))
318                         goto cleanup;
319
320                 err = init(engine);
321                 if (err)
322                         goto cleanup;
323
324                 GEM_BUG_ON(!engine->submit_request);
325         }
326
327         return 0;
328
329 cleanup:
330         for_each_engine(engine, dev_priv, id) {
331                 if (id >= err_id) {
332                         kfree(engine);
333                         dev_priv->engine[id] = NULL;
334                 } else {
335                         dev_priv->gt.cleanup_engine(engine);
336                 }
337         }
338         return err;
339 }
340
341 void intel_engine_init_global_seqno(struct intel_engine_cs *engine, u32 seqno)
342 {
343         struct drm_i915_private *dev_priv = engine->i915;
344
345         /* Our semaphore implementation is strictly monotonic (i.e. we proceed
346          * so long as the semaphore value in the register/page is greater
347          * than the sync value), so whenever we reset the seqno,
348          * so long as we reset the tracking semaphore value to 0, it will
349          * always be before the next request's seqno. If we don't reset
350          * the semaphore value, then when the seqno moves backwards all
351          * future waits will complete instantly (causing rendering corruption).
352          */
353         if (IS_GEN6(dev_priv) || IS_GEN7(dev_priv)) {
354                 I915_WRITE(RING_SYNC_0(engine->mmio_base), 0);
355                 I915_WRITE(RING_SYNC_1(engine->mmio_base), 0);
356                 if (HAS_VEBOX(dev_priv))
357                         I915_WRITE(RING_SYNC_2(engine->mmio_base), 0);
358         }
359         if (dev_priv->semaphore) {
360                 struct page *page = i915_vma_first_page(dev_priv->semaphore);
361                 void *semaphores;
362
363                 /* Semaphores are in noncoherent memory, flush to be safe */
364                 semaphores = kmap_atomic(page);
365                 memset(semaphores + GEN8_SEMAPHORE_OFFSET(engine->id, 0),
366                        0, I915_NUM_ENGINES * gen8_semaphore_seqno_size);
367                 drm_clflush_virt_range(semaphores + GEN8_SEMAPHORE_OFFSET(engine->id, 0),
368                                        I915_NUM_ENGINES * gen8_semaphore_seqno_size);
369                 kunmap_atomic(semaphores);
370         }
371
372         intel_write_status_page(engine, I915_GEM_HWS_INDEX, seqno);
373         clear_bit(ENGINE_IRQ_BREADCRUMB, &engine->irq_posted);
374
375         /* After manually advancing the seqno, fake the interrupt in case
376          * there are any waiters for that seqno.
377          */
378         intel_engine_wakeup(engine);
379
380         GEM_BUG_ON(intel_engine_get_seqno(engine) != seqno);
381 }
382
383 static void intel_engine_init_timeline(struct intel_engine_cs *engine)
384 {
385         engine->timeline = &engine->i915->gt.global_timeline.engine[engine->id];
386 }
387
388 static bool csb_force_mmio(struct drm_i915_private *i915)
389 {
390         /*
391          * IOMMU adds unpredictable latency causing the CSB write (from the
392          * GPU into the HWSP) to only be visible some time after the interrupt
393          * (missed breadcrumb syndrome).
394          */
395         if (intel_vtd_active())
396                 return true;
397
398         /* Older GVT emulation depends upon intercepting CSB mmio */
399         if (intel_vgpu_active(i915) && !intel_vgpu_has_hwsp_emulation(i915))
400                 return true;
401
402         return false;
403 }
404
405 static void intel_engine_init_execlist(struct intel_engine_cs *engine)
406 {
407         struct intel_engine_execlists * const execlists = &engine->execlists;
408
409         execlists->csb_use_mmio = csb_force_mmio(engine->i915);
410
411         execlists->port_mask = 1;
412         BUILD_BUG_ON_NOT_POWER_OF_2(execlists_num_ports(execlists));
413         GEM_BUG_ON(execlists_num_ports(execlists) > EXECLIST_MAX_PORTS);
414
415         execlists->queue = RB_ROOT;
416         execlists->first = NULL;
417 }
418
419 /**
420  * intel_engines_setup_common - setup engine state not requiring hw access
421  * @engine: Engine to setup.
422  *
423  * Initializes @engine@ structure members shared between legacy and execlists
424  * submission modes which do not require hardware access.
425  *
426  * Typically done early in the submission mode specific engine setup stage.
427  */
428 void intel_engine_setup_common(struct intel_engine_cs *engine)
429 {
430         intel_engine_init_execlist(engine);
431
432         intel_engine_init_timeline(engine);
433         intel_engine_init_hangcheck(engine);
434         i915_gem_batch_pool_init(engine, &engine->batch_pool);
435
436         intel_engine_init_cmd_parser(engine);
437 }
438
439 int intel_engine_create_scratch(struct intel_engine_cs *engine, int size)
440 {
441         struct drm_i915_gem_object *obj;
442         struct i915_vma *vma;
443         int ret;
444
445         WARN_ON(engine->scratch);
446
447         obj = i915_gem_object_create_stolen(engine->i915, size);
448         if (!obj)
449                 obj = i915_gem_object_create_internal(engine->i915, size);
450         if (IS_ERR(obj)) {
451                 DRM_ERROR("Failed to allocate scratch page\n");
452                 return PTR_ERR(obj);
453         }
454
455         vma = i915_vma_instance(obj, &engine->i915->ggtt.base, NULL);
456         if (IS_ERR(vma)) {
457                 ret = PTR_ERR(vma);
458                 goto err_unref;
459         }
460
461         ret = i915_vma_pin(vma, 0, 4096, PIN_GLOBAL | PIN_HIGH);
462         if (ret)
463                 goto err_unref;
464
465         engine->scratch = vma;
466         DRM_DEBUG_DRIVER("%s pipe control offset: 0x%08x\n",
467                          engine->name, i915_ggtt_offset(vma));
468         return 0;
469
470 err_unref:
471         i915_gem_object_put(obj);
472         return ret;
473 }
474
475 static void intel_engine_cleanup_scratch(struct intel_engine_cs *engine)
476 {
477         i915_vma_unpin_and_release(&engine->scratch);
478 }
479
480 static void cleanup_phys_status_page(struct intel_engine_cs *engine)
481 {
482         struct drm_i915_private *dev_priv = engine->i915;
483
484         if (!dev_priv->status_page_dmah)
485                 return;
486
487         drm_pci_free(&dev_priv->drm, dev_priv->status_page_dmah);
488         engine->status_page.page_addr = NULL;
489 }
490
491 static void cleanup_status_page(struct intel_engine_cs *engine)
492 {
493         struct i915_vma *vma;
494         struct drm_i915_gem_object *obj;
495
496         vma = fetch_and_zero(&engine->status_page.vma);
497         if (!vma)
498                 return;
499
500         obj = vma->obj;
501
502         i915_vma_unpin(vma);
503         i915_vma_close(vma);
504
505         i915_gem_object_unpin_map(obj);
506         __i915_gem_object_release_unless_active(obj);
507 }
508
509 static int init_status_page(struct intel_engine_cs *engine)
510 {
511         struct drm_i915_gem_object *obj;
512         struct i915_vma *vma;
513         unsigned int flags;
514         void *vaddr;
515         int ret;
516
517         obj = i915_gem_object_create_internal(engine->i915, PAGE_SIZE);
518         if (IS_ERR(obj)) {
519                 DRM_ERROR("Failed to allocate status page\n");
520                 return PTR_ERR(obj);
521         }
522
523         ret = i915_gem_object_set_cache_level(obj, I915_CACHE_LLC);
524         if (ret)
525                 goto err;
526
527         vma = i915_vma_instance(obj, &engine->i915->ggtt.base, NULL);
528         if (IS_ERR(vma)) {
529                 ret = PTR_ERR(vma);
530                 goto err;
531         }
532
533         flags = PIN_GLOBAL;
534         if (!HAS_LLC(engine->i915))
535                 /* On g33, we cannot place HWS above 256MiB, so
536                  * restrict its pinning to the low mappable arena.
537                  * Though this restriction is not documented for
538                  * gen4, gen5, or byt, they also behave similarly
539                  * and hang if the HWS is placed at the top of the
540                  * GTT. To generalise, it appears that all !llc
541                  * platforms have issues with us placing the HWS
542                  * above the mappable region (even though we never
543                  * actually map it).
544                  */
545                 flags |= PIN_MAPPABLE;
546         else
547                 flags |= PIN_HIGH;
548         ret = i915_vma_pin(vma, 0, 4096, flags);
549         if (ret)
550                 goto err;
551
552         vaddr = i915_gem_object_pin_map(obj, I915_MAP_WB);
553         if (IS_ERR(vaddr)) {
554                 ret = PTR_ERR(vaddr);
555                 goto err_unpin;
556         }
557
558         engine->status_page.vma = vma;
559         engine->status_page.ggtt_offset = i915_ggtt_offset(vma);
560         engine->status_page.page_addr = memset(vaddr, 0, PAGE_SIZE);
561
562         DRM_DEBUG_DRIVER("%s hws offset: 0x%08x\n",
563                          engine->name, i915_ggtt_offset(vma));
564         return 0;
565
566 err_unpin:
567         i915_vma_unpin(vma);
568 err:
569         i915_gem_object_put(obj);
570         return ret;
571 }
572
573 static int init_phys_status_page(struct intel_engine_cs *engine)
574 {
575         struct drm_i915_private *dev_priv = engine->i915;
576
577         GEM_BUG_ON(engine->id != RCS);
578
579         dev_priv->status_page_dmah =
580                 drm_pci_alloc(&dev_priv->drm, PAGE_SIZE, PAGE_SIZE);
581         if (!dev_priv->status_page_dmah)
582                 return -ENOMEM;
583
584         engine->status_page.page_addr = dev_priv->status_page_dmah->vaddr;
585         memset(engine->status_page.page_addr, 0, PAGE_SIZE);
586
587         return 0;
588 }
589
590 /**
591  * intel_engines_init_common - initialize cengine state which might require hw access
592  * @engine: Engine to initialize.
593  *
594  * Initializes @engine@ structure members shared between legacy and execlists
595  * submission modes which do require hardware access.
596  *
597  * Typcally done at later stages of submission mode specific engine setup.
598  *
599  * Returns zero on success or an error code on failure.
600  */
601 int intel_engine_init_common(struct intel_engine_cs *engine)
602 {
603         struct intel_ring *ring;
604         int ret;
605
606         engine->set_default_submission(engine);
607
608         /* We may need to do things with the shrinker which
609          * require us to immediately switch back to the default
610          * context. This can cause a problem as pinning the
611          * default context also requires GTT space which may not
612          * be available. To avoid this we always pin the default
613          * context.
614          */
615         ring = engine->context_pin(engine, engine->i915->kernel_context);
616         if (IS_ERR(ring))
617                 return PTR_ERR(ring);
618
619         /*
620          * Similarly the preempt context must always be available so that
621          * we can interrupt the engine at any time.
622          */
623         if (INTEL_INFO(engine->i915)->has_logical_ring_preemption) {
624                 ring = engine->context_pin(engine,
625                                            engine->i915->preempt_context);
626                 if (IS_ERR(ring)) {
627                         ret = PTR_ERR(ring);
628                         goto err_unpin_kernel;
629                 }
630         }
631
632         ret = intel_engine_init_breadcrumbs(engine);
633         if (ret)
634                 goto err_unpin_preempt;
635
636         ret = i915_gem_render_state_init(engine);
637         if (ret)
638                 goto err_breadcrumbs;
639
640         if (HWS_NEEDS_PHYSICAL(engine->i915))
641                 ret = init_phys_status_page(engine);
642         else
643                 ret = init_status_page(engine);
644         if (ret)
645                 goto err_rs_fini;
646
647         return 0;
648
649 err_rs_fini:
650         i915_gem_render_state_fini(engine);
651 err_breadcrumbs:
652         intel_engine_fini_breadcrumbs(engine);
653 err_unpin_preempt:
654         if (INTEL_INFO(engine->i915)->has_logical_ring_preemption)
655                 engine->context_unpin(engine, engine->i915->preempt_context);
656 err_unpin_kernel:
657         engine->context_unpin(engine, engine->i915->kernel_context);
658         return ret;
659 }
660
661 /**
662  * intel_engines_cleanup_common - cleans up the engine state created by
663  *                                the common initiailizers.
664  * @engine: Engine to cleanup.
665  *
666  * This cleans up everything created by the common helpers.
667  */
668 void intel_engine_cleanup_common(struct intel_engine_cs *engine)
669 {
670         intel_engine_cleanup_scratch(engine);
671
672         if (HWS_NEEDS_PHYSICAL(engine->i915))
673                 cleanup_phys_status_page(engine);
674         else
675                 cleanup_status_page(engine);
676
677         i915_gem_render_state_fini(engine);
678         intel_engine_fini_breadcrumbs(engine);
679         intel_engine_cleanup_cmd_parser(engine);
680         i915_gem_batch_pool_fini(&engine->batch_pool);
681
682         if (INTEL_INFO(engine->i915)->has_logical_ring_preemption)
683                 engine->context_unpin(engine, engine->i915->preempt_context);
684         engine->context_unpin(engine, engine->i915->kernel_context);
685 }
686
687 u64 intel_engine_get_active_head(struct intel_engine_cs *engine)
688 {
689         struct drm_i915_private *dev_priv = engine->i915;
690         u64 acthd;
691
692         if (INTEL_GEN(dev_priv) >= 8)
693                 acthd = I915_READ64_2x32(RING_ACTHD(engine->mmio_base),
694                                          RING_ACTHD_UDW(engine->mmio_base));
695         else if (INTEL_GEN(dev_priv) >= 4)
696                 acthd = I915_READ(RING_ACTHD(engine->mmio_base));
697         else
698                 acthd = I915_READ(ACTHD);
699
700         return acthd;
701 }
702
703 u64 intel_engine_get_last_batch_head(struct intel_engine_cs *engine)
704 {
705         struct drm_i915_private *dev_priv = engine->i915;
706         u64 bbaddr;
707
708         if (INTEL_GEN(dev_priv) >= 8)
709                 bbaddr = I915_READ64_2x32(RING_BBADDR(engine->mmio_base),
710                                           RING_BBADDR_UDW(engine->mmio_base));
711         else
712                 bbaddr = I915_READ(RING_BBADDR(engine->mmio_base));
713
714         return bbaddr;
715 }
716
717 const char *i915_cache_level_str(struct drm_i915_private *i915, int type)
718 {
719         switch (type) {
720         case I915_CACHE_NONE: return " uncached";
721         case I915_CACHE_LLC: return HAS_LLC(i915) ? " LLC" : " snooped";
722         case I915_CACHE_L3_LLC: return " L3+LLC";
723         case I915_CACHE_WT: return " WT";
724         default: return "";
725         }
726 }
727
728 static inline uint32_t
729 read_subslice_reg(struct drm_i915_private *dev_priv, int slice,
730                   int subslice, i915_reg_t reg)
731 {
732         uint32_t mcr;
733         uint32_t ret;
734         enum forcewake_domains fw_domains;
735
736         fw_domains = intel_uncore_forcewake_for_reg(dev_priv, reg,
737                                                     FW_REG_READ);
738         fw_domains |= intel_uncore_forcewake_for_reg(dev_priv,
739                                                      GEN8_MCR_SELECTOR,
740                                                      FW_REG_READ | FW_REG_WRITE);
741
742         spin_lock_irq(&dev_priv->uncore.lock);
743         intel_uncore_forcewake_get__locked(dev_priv, fw_domains);
744
745         mcr = I915_READ_FW(GEN8_MCR_SELECTOR);
746         /*
747          * The HW expects the slice and sublice selectors to be reset to 0
748          * after reading out the registers.
749          */
750         WARN_ON_ONCE(mcr & (GEN8_MCR_SLICE_MASK | GEN8_MCR_SUBSLICE_MASK));
751         mcr &= ~(GEN8_MCR_SLICE_MASK | GEN8_MCR_SUBSLICE_MASK);
752         mcr |= GEN8_MCR_SLICE(slice) | GEN8_MCR_SUBSLICE(subslice);
753         I915_WRITE_FW(GEN8_MCR_SELECTOR, mcr);
754
755         ret = I915_READ_FW(reg);
756
757         mcr &= ~(GEN8_MCR_SLICE_MASK | GEN8_MCR_SUBSLICE_MASK);
758         I915_WRITE_FW(GEN8_MCR_SELECTOR, mcr);
759
760         intel_uncore_forcewake_put__locked(dev_priv, fw_domains);
761         spin_unlock_irq(&dev_priv->uncore.lock);
762
763         return ret;
764 }
765
766 /* NB: please notice the memset */
767 void intel_engine_get_instdone(struct intel_engine_cs *engine,
768                                struct intel_instdone *instdone)
769 {
770         struct drm_i915_private *dev_priv = engine->i915;
771         u32 mmio_base = engine->mmio_base;
772         int slice;
773         int subslice;
774
775         memset(instdone, 0, sizeof(*instdone));
776
777         switch (INTEL_GEN(dev_priv)) {
778         default:
779                 instdone->instdone = I915_READ(RING_INSTDONE(mmio_base));
780
781                 if (engine->id != RCS)
782                         break;
783
784                 instdone->slice_common = I915_READ(GEN7_SC_INSTDONE);
785                 for_each_instdone_slice_subslice(dev_priv, slice, subslice) {
786                         instdone->sampler[slice][subslice] =
787                                 read_subslice_reg(dev_priv, slice, subslice,
788                                                   GEN7_SAMPLER_INSTDONE);
789                         instdone->row[slice][subslice] =
790                                 read_subslice_reg(dev_priv, slice, subslice,
791                                                   GEN7_ROW_INSTDONE);
792                 }
793                 break;
794         case 7:
795                 instdone->instdone = I915_READ(RING_INSTDONE(mmio_base));
796
797                 if (engine->id != RCS)
798                         break;
799
800                 instdone->slice_common = I915_READ(GEN7_SC_INSTDONE);
801                 instdone->sampler[0][0] = I915_READ(GEN7_SAMPLER_INSTDONE);
802                 instdone->row[0][0] = I915_READ(GEN7_ROW_INSTDONE);
803
804                 break;
805         case 6:
806         case 5:
807         case 4:
808                 instdone->instdone = I915_READ(RING_INSTDONE(mmio_base));
809
810                 if (engine->id == RCS)
811                         /* HACK: Using the wrong struct member */
812                         instdone->slice_common = I915_READ(GEN4_INSTDONE1);
813                 break;
814         case 3:
815         case 2:
816                 instdone->instdone = I915_READ(GEN2_INSTDONE);
817                 break;
818         }
819 }
820
821 static int wa_add(struct drm_i915_private *dev_priv,
822                   i915_reg_t addr,
823                   const u32 mask, const u32 val)
824 {
825         const u32 idx = dev_priv->workarounds.count;
826
827         if (WARN_ON(idx >= I915_MAX_WA_REGS))
828                 return -ENOSPC;
829
830         dev_priv->workarounds.reg[idx].addr = addr;
831         dev_priv->workarounds.reg[idx].value = val;
832         dev_priv->workarounds.reg[idx].mask = mask;
833
834         dev_priv->workarounds.count++;
835
836         return 0;
837 }
838
839 #define WA_REG(addr, mask, val) do { \
840                 const int r = wa_add(dev_priv, (addr), (mask), (val)); \
841                 if (r) \
842                         return r; \
843         } while (0)
844
845 #define WA_SET_BIT_MASKED(addr, mask) \
846         WA_REG(addr, (mask), _MASKED_BIT_ENABLE(mask))
847
848 #define WA_CLR_BIT_MASKED(addr, mask) \
849         WA_REG(addr, (mask), _MASKED_BIT_DISABLE(mask))
850
851 #define WA_SET_FIELD_MASKED(addr, mask, value) \
852         WA_REG(addr, mask, _MASKED_FIELD(mask, value))
853
854 static int wa_ring_whitelist_reg(struct intel_engine_cs *engine,
855                                  i915_reg_t reg)
856 {
857         struct drm_i915_private *dev_priv = engine->i915;
858         struct i915_workarounds *wa = &dev_priv->workarounds;
859         const uint32_t index = wa->hw_whitelist_count[engine->id];
860
861         if (WARN_ON(index >= RING_MAX_NONPRIV_SLOTS))
862                 return -EINVAL;
863
864         I915_WRITE(RING_FORCE_TO_NONPRIV(engine->mmio_base, index),
865                    i915_mmio_reg_offset(reg));
866         wa->hw_whitelist_count[engine->id]++;
867
868         return 0;
869 }
870
871 static int gen8_init_workarounds(struct intel_engine_cs *engine)
872 {
873         struct drm_i915_private *dev_priv = engine->i915;
874
875         WA_SET_BIT_MASKED(INSTPM, INSTPM_FORCE_ORDERING);
876
877         /* WaDisableAsyncFlipPerfMode:bdw,chv */
878         WA_SET_BIT_MASKED(MI_MODE, ASYNC_FLIP_PERF_DISABLE);
879
880         /* WaDisablePartialInstShootdown:bdw,chv */
881         WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN,
882                           PARTIAL_INSTRUCTION_SHOOTDOWN_DISABLE);
883
884         /* Use Force Non-Coherent whenever executing a 3D context. This is a
885          * workaround for for a possible hang in the unlikely event a TLB
886          * invalidation occurs during a PSD flush.
887          */
888         /* WaForceEnableNonCoherent:bdw,chv */
889         /* WaHdcDisableFetchWhenMasked:bdw,chv */
890         WA_SET_BIT_MASKED(HDC_CHICKEN0,
891                           HDC_DONOT_FETCH_MEM_WHEN_MASKED |
892                           HDC_FORCE_NON_COHERENT);
893
894         /* From the Haswell PRM, Command Reference: Registers, CACHE_MODE_0:
895          * "The Hierarchical Z RAW Stall Optimization allows non-overlapping
896          *  polygons in the same 8x4 pixel/sample area to be processed without
897          *  stalling waiting for the earlier ones to write to Hierarchical Z
898          *  buffer."
899          *
900          * This optimization is off by default for BDW and CHV; turn it on.
901          */
902         WA_CLR_BIT_MASKED(CACHE_MODE_0_GEN7, HIZ_RAW_STALL_OPT_DISABLE);
903
904         /* Wa4x4STCOptimizationDisable:bdw,chv */
905         WA_SET_BIT_MASKED(CACHE_MODE_1, GEN8_4x4_STC_OPTIMIZATION_DISABLE);
906
907         /*
908          * BSpec recommends 8x4 when MSAA is used,
909          * however in practice 16x4 seems fastest.
910          *
911          * Note that PS/WM thread counts depend on the WIZ hashing
912          * disable bit, which we don't touch here, but it's good
913          * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
914          */
915         WA_SET_FIELD_MASKED(GEN7_GT_MODE,
916                             GEN6_WIZ_HASHING_MASK,
917                             GEN6_WIZ_HASHING_16x4);
918
919         return 0;
920 }
921
922 static int bdw_init_workarounds(struct intel_engine_cs *engine)
923 {
924         struct drm_i915_private *dev_priv = engine->i915;
925         int ret;
926
927         ret = gen8_init_workarounds(engine);
928         if (ret)
929                 return ret;
930
931         /* WaDisableThreadStallDopClockGating:bdw (pre-production) */
932         WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN, STALL_DOP_GATING_DISABLE);
933
934         /* WaDisableDopClockGating:bdw
935          *
936          * Also see the related UCGTCL1 write in broadwell_init_clock_gating()
937          * to disable EUTC clock gating.
938          */
939         WA_SET_BIT_MASKED(GEN7_ROW_CHICKEN2,
940                           DOP_CLOCK_GATING_DISABLE);
941
942         WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN3,
943                           GEN8_SAMPLER_POWER_BYPASS_DIS);
944
945         WA_SET_BIT_MASKED(HDC_CHICKEN0,
946                           /* WaForceContextSaveRestoreNonCoherent:bdw */
947                           HDC_FORCE_CONTEXT_SAVE_RESTORE_NON_COHERENT |
948                           /* WaDisableFenceDestinationToSLM:bdw (pre-prod) */
949                           (IS_BDW_GT3(dev_priv) ? HDC_FENCE_DEST_SLM_DISABLE : 0));
950
951         return 0;
952 }
953
954 static int chv_init_workarounds(struct intel_engine_cs *engine)
955 {
956         struct drm_i915_private *dev_priv = engine->i915;
957         int ret;
958
959         ret = gen8_init_workarounds(engine);
960         if (ret)
961                 return ret;
962
963         /* WaDisableThreadStallDopClockGating:chv */
964         WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN, STALL_DOP_GATING_DISABLE);
965
966         /* Improve HiZ throughput on CHV. */
967         WA_SET_BIT_MASKED(HIZ_CHICKEN, CHV_HZ_8X8_MODE_IN_1X);
968
969         return 0;
970 }
971
972 static int gen9_init_workarounds(struct intel_engine_cs *engine)
973 {
974         struct drm_i915_private *dev_priv = engine->i915;
975         int ret;
976
977         /* WaConextSwitchWithConcurrentTLBInvalidate:skl,bxt,kbl,glk,cfl */
978         I915_WRITE(GEN9_CSFE_CHICKEN1_RCS, _MASKED_BIT_ENABLE(GEN9_PREEMPT_GPGPU_SYNC_SWITCH_DISABLE));
979
980         /* WaEnableLbsSlaRetryTimerDecrement:skl,bxt,kbl,glk,cfl */
981         I915_WRITE(BDW_SCRATCH1, I915_READ(BDW_SCRATCH1) |
982                    GEN9_LBS_SLA_RETRY_TIMER_DECREMENT_ENABLE);
983
984         /* WaDisableKillLogic:bxt,skl,kbl */
985         if (!IS_COFFEELAKE(dev_priv))
986                 I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) |
987                            ECOCHK_DIS_TLB);
988
989         if (HAS_LLC(dev_priv)) {
990                 /* WaCompressedResourceSamplerPbeMediaNewHashMode:skl,kbl
991                  *
992                  * Must match Display Engine. See
993                  * WaCompressedResourceDisplayNewHashMode.
994                  */
995                 WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
996                                   GEN9_PBE_COMPRESSED_HASH_SELECTION);
997                 WA_SET_BIT_MASKED(GEN9_HALF_SLICE_CHICKEN7,
998                                   GEN9_SAMPLER_HASH_COMPRESSED_READ_ADDR);
999
1000                 I915_WRITE(MMCD_MISC_CTRL,
1001                            I915_READ(MMCD_MISC_CTRL) |
1002                            MMCD_PCLA |
1003                            MMCD_HOTSPOT_EN);
1004         }
1005
1006         /* WaClearFlowControlGpgpuContextSave:skl,bxt,kbl,glk,cfl */
1007         /* WaDisablePartialInstShootdown:skl,bxt,kbl,glk,cfl */
1008         WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN,
1009                           FLOW_CONTROL_ENABLE |
1010                           PARTIAL_INSTRUCTION_SHOOTDOWN_DISABLE);
1011
1012         /* Syncing dependencies between camera and graphics:skl,bxt,kbl */
1013         if (!IS_COFFEELAKE(dev_priv))
1014                 WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN3,
1015                                   GEN9_DISABLE_OCL_OOB_SUPPRESS_LOGIC);
1016
1017         /* WaDisableDgMirrorFixInHalfSliceChicken5:bxt */
1018         if (IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1))
1019                 WA_CLR_BIT_MASKED(GEN9_HALF_SLICE_CHICKEN5,
1020                                   GEN9_DG_MIRROR_FIX_ENABLE);
1021
1022         /* WaSetDisablePixMaskCammingAndRhwoInCommonSliceChicken:bxt */
1023         if (IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1)) {
1024                 WA_SET_BIT_MASKED(GEN7_COMMON_SLICE_CHICKEN1,
1025                                   GEN9_RHWO_OPTIMIZATION_DISABLE);
1026                 /*
1027                  * WA also requires GEN9_SLICE_COMMON_ECO_CHICKEN0[14:14] to be set
1028                  * but we do that in per ctx batchbuffer as there is an issue
1029                  * with this register not getting restored on ctx restore
1030                  */
1031         }
1032
1033         /* WaEnableYV12BugFixInHalfSliceChicken7:skl,bxt,kbl,glk,cfl */
1034         /* WaEnableSamplerGPGPUPreemptionSupport:skl,bxt,kbl,cfl */
1035         WA_SET_BIT_MASKED(GEN9_HALF_SLICE_CHICKEN7,
1036                           GEN9_ENABLE_YV12_BUGFIX |
1037                           GEN9_ENABLE_GPGPU_PREEMPTION);
1038
1039         /* Wa4x4STCOptimizationDisable:skl,bxt,kbl,glk,cfl */
1040         /* WaDisablePartialResolveInVc:skl,bxt,kbl,cfl */
1041         WA_SET_BIT_MASKED(CACHE_MODE_1, (GEN8_4x4_STC_OPTIMIZATION_DISABLE |
1042                                          GEN9_PARTIAL_RESOLVE_IN_VC_DISABLE));
1043
1044         /* WaCcsTlbPrefetchDisable:skl,bxt,kbl,glk,cfl */
1045         WA_CLR_BIT_MASKED(GEN9_HALF_SLICE_CHICKEN5,
1046                           GEN9_CCS_TLB_PREFETCH_ENABLE);
1047
1048         /* WaDisableMaskBasedCammingInRCC:bxt */
1049         if (IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1))
1050                 WA_SET_BIT_MASKED(SLICE_ECO_CHICKEN0,
1051                                   PIXEL_MASK_CAMMING_DISABLE);
1052
1053         /* WaForceContextSaveRestoreNonCoherent:skl,bxt,kbl,cfl */
1054         WA_SET_BIT_MASKED(HDC_CHICKEN0,
1055                           HDC_FORCE_CONTEXT_SAVE_RESTORE_NON_COHERENT |
1056                           HDC_FORCE_CSR_NON_COHERENT_OVR_DISABLE);
1057
1058         /* WaForceEnableNonCoherent and WaDisableHDCInvalidation are
1059          * both tied to WaForceContextSaveRestoreNonCoherent
1060          * in some hsds for skl. We keep the tie for all gen9. The
1061          * documentation is a bit hazy and so we want to get common behaviour,
1062          * even though there is no clear evidence we would need both on kbl/bxt.
1063          * This area has been source of system hangs so we play it safe
1064          * and mimic the skl regardless of what bspec says.
1065          *
1066          * Use Force Non-Coherent whenever executing a 3D context. This
1067          * is a workaround for a possible hang in the unlikely event
1068          * a TLB invalidation occurs during a PSD flush.
1069          */
1070
1071         /* WaForceEnableNonCoherent:skl,bxt,kbl,cfl */
1072         WA_SET_BIT_MASKED(HDC_CHICKEN0,
1073                           HDC_FORCE_NON_COHERENT);
1074
1075         /* WaDisableHDCInvalidation:skl,bxt,kbl,cfl */
1076         I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) |
1077                    BDW_DISABLE_HDC_INVALIDATION);
1078
1079         /* WaDisableSamplerPowerBypassForSOPingPong:skl,bxt,kbl,cfl */
1080         if (IS_SKYLAKE(dev_priv) ||
1081             IS_KABYLAKE(dev_priv) ||
1082             IS_COFFEELAKE(dev_priv) ||
1083             IS_BXT_REVID(dev_priv, 0, BXT_REVID_B0))
1084                 WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN3,
1085                                   GEN8_SAMPLER_POWER_BYPASS_DIS);
1086
1087         /* WaDisableSTUnitPowerOptimization:skl,bxt,kbl,glk,cfl */
1088         WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN2, GEN8_ST_PO_DISABLE);
1089
1090         /* WaOCLCoherentLineFlush:skl,bxt,kbl,cfl */
1091         I915_WRITE(GEN8_L3SQCREG4, (I915_READ(GEN8_L3SQCREG4) |
1092                                     GEN8_LQSC_FLUSH_COHERENT_LINES));
1093
1094         /*
1095          * Supporting preemption with fine-granularity requires changes in the
1096          * batch buffer programming. Since we can't break old userspace, we
1097          * need to set our default preemption level to safe value. Userspace is
1098          * still able to use more fine-grained preemption levels, since in
1099          * WaEnablePreemptionGranularityControlByUMD we're whitelisting the
1100          * per-ctx register. As such, WaDisable{3D,GPGPU}MidCmdPreemption are
1101          * not real HW workarounds, but merely a way to start using preemption
1102          * while maintaining old contract with userspace.
1103          */
1104
1105         /* WaDisable3DMidCmdPreemption:skl,bxt,glk,cfl,[cnl] */
1106         WA_CLR_BIT_MASKED(GEN8_CS_CHICKEN1, GEN9_PREEMPT_3D_OBJECT_LEVEL);
1107
1108         /* WaDisableGPGPUMidCmdPreemption:skl,bxt,blk,cfl,[cnl] */
1109         WA_SET_FIELD_MASKED(GEN8_CS_CHICKEN1, GEN9_PREEMPT_GPGPU_LEVEL_MASK,
1110                             GEN9_PREEMPT_GPGPU_COMMAND_LEVEL);
1111
1112         /* WaVFEStateAfterPipeControlwithMediaStateClear:skl,bxt,glk,cfl */
1113         ret = wa_ring_whitelist_reg(engine, GEN9_CTX_PREEMPT_REG);
1114         if (ret)
1115                 return ret;
1116
1117         /* WaEnablePreemptionGranularityControlByUMD:skl,bxt,kbl,cfl,[cnl] */
1118         I915_WRITE(GEN7_FF_SLICE_CS_CHICKEN1,
1119                    _MASKED_BIT_ENABLE(GEN9_FFSC_PERCTX_PREEMPT_CTRL));
1120         ret = wa_ring_whitelist_reg(engine, GEN8_CS_CHICKEN1);
1121         if (ret)
1122                 return ret;
1123
1124         /* WaAllowUMDToModifyHDCChicken1:skl,bxt,kbl,glk,cfl */
1125         ret = wa_ring_whitelist_reg(engine, GEN8_HDC_CHICKEN1);
1126         if (ret)
1127                 return ret;
1128
1129         return 0;
1130 }
1131
1132 static int skl_tune_iz_hashing(struct intel_engine_cs *engine)
1133 {
1134         struct drm_i915_private *dev_priv = engine->i915;
1135         u8 vals[3] = { 0, 0, 0 };
1136         unsigned int i;
1137
1138         for (i = 0; i < 3; i++) {
1139                 u8 ss;
1140
1141                 /*
1142                  * Only consider slices where one, and only one, subslice has 7
1143                  * EUs
1144                  */
1145                 if (!is_power_of_2(INTEL_INFO(dev_priv)->sseu.subslice_7eu[i]))
1146                         continue;
1147
1148                 /*
1149                  * subslice_7eu[i] != 0 (because of the check above) and
1150                  * ss_max == 4 (maximum number of subslices possible per slice)
1151                  *
1152                  * ->    0 <= ss <= 3;
1153                  */
1154                 ss = ffs(INTEL_INFO(dev_priv)->sseu.subslice_7eu[i]) - 1;
1155                 vals[i] = 3 - ss;
1156         }
1157
1158         if (vals[0] == 0 && vals[1] == 0 && vals[2] == 0)
1159                 return 0;
1160
1161         /* Tune IZ hashing. See intel_device_info_runtime_init() */
1162         WA_SET_FIELD_MASKED(GEN7_GT_MODE,
1163                             GEN9_IZ_HASHING_MASK(2) |
1164                             GEN9_IZ_HASHING_MASK(1) |
1165                             GEN9_IZ_HASHING_MASK(0),
1166                             GEN9_IZ_HASHING(2, vals[2]) |
1167                             GEN9_IZ_HASHING(1, vals[1]) |
1168                             GEN9_IZ_HASHING(0, vals[0]));
1169
1170         return 0;
1171 }
1172
1173 static int skl_init_workarounds(struct intel_engine_cs *engine)
1174 {
1175         struct drm_i915_private *dev_priv = engine->i915;
1176         int ret;
1177
1178         ret = gen9_init_workarounds(engine);
1179         if (ret)
1180                 return ret;
1181
1182         /* WaEnableGapsTsvCreditFix:skl */
1183         I915_WRITE(GEN8_GARBCNTL, (I915_READ(GEN8_GARBCNTL) |
1184                                    GEN9_GAPS_TSV_CREDIT_DISABLE));
1185
1186         /* WaDisableGafsUnitClkGating:skl */
1187         I915_WRITE(GEN7_UCGCTL4, (I915_READ(GEN7_UCGCTL4) |
1188                                   GEN8_EU_GAUNIT_CLOCK_GATE_DISABLE));
1189
1190         /* WaInPlaceDecompressionHang:skl */
1191         if (IS_SKL_REVID(dev_priv, SKL_REVID_H0, REVID_FOREVER))
1192                 I915_WRITE(GEN9_GAMT_ECO_REG_RW_IA,
1193                            (I915_READ(GEN9_GAMT_ECO_REG_RW_IA) |
1194                             GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS));
1195
1196         /* WaDisableLSQCROPERFforOCL:skl */
1197         ret = wa_ring_whitelist_reg(engine, GEN8_L3SQCREG4);
1198         if (ret)
1199                 return ret;
1200
1201         return skl_tune_iz_hashing(engine);
1202 }
1203
1204 static int bxt_init_workarounds(struct intel_engine_cs *engine)
1205 {
1206         struct drm_i915_private *dev_priv = engine->i915;
1207         int ret;
1208
1209         ret = gen9_init_workarounds(engine);
1210         if (ret)
1211                 return ret;
1212
1213         /* WaStoreMultiplePTEenable:bxt */
1214         /* This is a requirement according to Hardware specification */
1215         if (IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1))
1216                 I915_WRITE(TILECTL, I915_READ(TILECTL) | TILECTL_TLBPF);
1217
1218         /* WaSetClckGatingDisableMedia:bxt */
1219         if (IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1)) {
1220                 I915_WRITE(GEN7_MISCCPCTL, (I915_READ(GEN7_MISCCPCTL) &
1221                                             ~GEN8_DOP_CLOCK_GATE_MEDIA_ENABLE));
1222         }
1223
1224         /* WaDisableThreadStallDopClockGating:bxt */
1225         WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN,
1226                           STALL_DOP_GATING_DISABLE);
1227
1228         /* WaDisablePooledEuLoadBalancingFix:bxt */
1229         if (IS_BXT_REVID(dev_priv, BXT_REVID_B0, REVID_FOREVER)) {
1230                 I915_WRITE(FF_SLICE_CS_CHICKEN2,
1231                            _MASKED_BIT_ENABLE(GEN9_POOLED_EU_LOAD_BALANCING_FIX_DISABLE));
1232         }
1233
1234         /* WaDisableSbeCacheDispatchPortSharing:bxt */
1235         if (IS_BXT_REVID(dev_priv, 0, BXT_REVID_B0)) {
1236                 WA_SET_BIT_MASKED(
1237                         GEN7_HALF_SLICE_CHICKEN1,
1238                         GEN7_SBE_SS_CACHE_DISPATCH_PORT_SHARING_DISABLE);
1239         }
1240
1241         /* WaDisableObjectLevelPreemptionForTrifanOrPolygon:bxt */
1242         /* WaDisableObjectLevelPreemptionForInstancedDraw:bxt */
1243         /* WaDisableObjectLevelPreemtionForInstanceId:bxt */
1244         /* WaDisableLSQCROPERFforOCL:bxt */
1245         if (IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1)) {
1246                 ret = wa_ring_whitelist_reg(engine, GEN9_CS_DEBUG_MODE1);
1247                 if (ret)
1248                         return ret;
1249
1250                 ret = wa_ring_whitelist_reg(engine, GEN8_L3SQCREG4);
1251                 if (ret)
1252                         return ret;
1253         }
1254
1255         /* WaProgramL3SqcReg1DefaultForPerf:bxt */
1256         if (IS_BXT_REVID(dev_priv, BXT_REVID_B0, REVID_FOREVER)) {
1257                 u32 val = I915_READ(GEN8_L3SQCREG1);
1258                 val &= ~L3_PRIO_CREDITS_MASK;
1259                 val |= L3_GENERAL_PRIO_CREDITS(62) | L3_HIGH_PRIO_CREDITS(2);
1260                 I915_WRITE(GEN8_L3SQCREG1, val);
1261         }
1262
1263         /* WaToEnableHwFixForPushConstHWBug:bxt */
1264         if (IS_BXT_REVID(dev_priv, BXT_REVID_C0, REVID_FOREVER))
1265                 WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
1266                                   GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
1267
1268         /* WaInPlaceDecompressionHang:bxt */
1269         if (IS_BXT_REVID(dev_priv, BXT_REVID_C0, REVID_FOREVER))
1270                 I915_WRITE(GEN9_GAMT_ECO_REG_RW_IA,
1271                            (I915_READ(GEN9_GAMT_ECO_REG_RW_IA) |
1272                             GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS));
1273
1274         return 0;
1275 }
1276
1277 static int cnl_init_workarounds(struct intel_engine_cs *engine)
1278 {
1279         struct drm_i915_private *dev_priv = engine->i915;
1280         int ret;
1281
1282         /* WaDisableI2mCycleOnWRPort:cnl (pre-prod) */
1283         if (IS_CNL_REVID(dev_priv, CNL_REVID_B0, CNL_REVID_B0))
1284                 I915_WRITE(GAMT_CHKN_BIT_REG,
1285                            (I915_READ(GAMT_CHKN_BIT_REG) |
1286                             GAMT_CHKN_DISABLE_I2M_CYCLE_ON_WR_PORT));
1287
1288         /* WaForceContextSaveRestoreNonCoherent:cnl */
1289         WA_SET_BIT_MASKED(CNL_HDC_CHICKEN0,
1290                           HDC_FORCE_CONTEXT_SAVE_RESTORE_NON_COHERENT);
1291
1292         /* WaThrottleEUPerfToAvoidTDBackPressure:cnl(pre-prod) */
1293         if (IS_CNL_REVID(dev_priv, CNL_REVID_B0, CNL_REVID_B0))
1294                 WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN, THROTTLE_12_5);
1295
1296         /* WaDisableReplayBufferBankArbitrationOptimization:cnl */
1297         WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
1298                           GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
1299
1300         /* WaDisableEnhancedSBEVertexCaching:cnl (pre-prod) */
1301         if (IS_CNL_REVID(dev_priv, 0, CNL_REVID_B0))
1302                 WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
1303                                   GEN8_CSC2_SBE_VUE_CACHE_CONSERVATIVE);
1304
1305         /* WaInPlaceDecompressionHang:cnl */
1306         I915_WRITE(GEN9_GAMT_ECO_REG_RW_IA,
1307                    (I915_READ(GEN9_GAMT_ECO_REG_RW_IA) |
1308                     GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS));
1309
1310         /* WaPushConstantDereferenceHoldDisable:cnl */
1311         WA_SET_BIT_MASKED(GEN7_ROW_CHICKEN2, PUSH_CONSTANT_DEREF_DISABLE);
1312
1313         /* FtrEnableFastAnisoL1BankingFix: cnl */
1314         WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN3, CNL_FAST_ANISO_L1_BANKING_FIX);
1315
1316         /* WaDisable3DMidCmdPreemption:cnl */
1317         WA_CLR_BIT_MASKED(GEN8_CS_CHICKEN1, GEN9_PREEMPT_3D_OBJECT_LEVEL);
1318
1319         /* WaDisableGPGPUMidCmdPreemption:cnl */
1320         WA_SET_FIELD_MASKED(GEN8_CS_CHICKEN1, GEN9_PREEMPT_GPGPU_LEVEL_MASK,
1321                             GEN9_PREEMPT_GPGPU_COMMAND_LEVEL);
1322
1323         /* WaEnablePreemptionGranularityControlByUMD:cnl */
1324         I915_WRITE(GEN7_FF_SLICE_CS_CHICKEN1,
1325                    _MASKED_BIT_ENABLE(GEN9_FFSC_PERCTX_PREEMPT_CTRL));
1326         ret= wa_ring_whitelist_reg(engine, GEN8_CS_CHICKEN1);
1327         if (ret)
1328                 return ret;
1329
1330         return 0;
1331 }
1332
1333 static int kbl_init_workarounds(struct intel_engine_cs *engine)
1334 {
1335         struct drm_i915_private *dev_priv = engine->i915;
1336         int ret;
1337
1338         ret = gen9_init_workarounds(engine);
1339         if (ret)
1340                 return ret;
1341
1342         /* WaEnableGapsTsvCreditFix:kbl */
1343         I915_WRITE(GEN8_GARBCNTL, (I915_READ(GEN8_GARBCNTL) |
1344                                    GEN9_GAPS_TSV_CREDIT_DISABLE));
1345
1346         /* WaDisableDynamicCreditSharing:kbl */
1347         if (IS_KBL_REVID(dev_priv, 0, KBL_REVID_B0))
1348                 I915_WRITE(GAMT_CHKN_BIT_REG,
1349                            (I915_READ(GAMT_CHKN_BIT_REG) |
1350                             GAMT_CHKN_DISABLE_DYNAMIC_CREDIT_SHARING));
1351
1352         /* WaDisableFenceDestinationToSLM:kbl (pre-prod) */
1353         if (IS_KBL_REVID(dev_priv, KBL_REVID_A0, KBL_REVID_A0))
1354                 WA_SET_BIT_MASKED(HDC_CHICKEN0,
1355                                   HDC_FENCE_DEST_SLM_DISABLE);
1356
1357         /* WaToEnableHwFixForPushConstHWBug:kbl */
1358         if (IS_KBL_REVID(dev_priv, KBL_REVID_C0, REVID_FOREVER))
1359                 WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
1360                                   GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
1361
1362         /* WaDisableGafsUnitClkGating:kbl */
1363         I915_WRITE(GEN7_UCGCTL4, (I915_READ(GEN7_UCGCTL4) |
1364                                   GEN8_EU_GAUNIT_CLOCK_GATE_DISABLE));
1365
1366         /* WaDisableSbeCacheDispatchPortSharing:kbl */
1367         WA_SET_BIT_MASKED(
1368                 GEN7_HALF_SLICE_CHICKEN1,
1369                 GEN7_SBE_SS_CACHE_DISPATCH_PORT_SHARING_DISABLE);
1370
1371         /* WaInPlaceDecompressionHang:kbl */
1372         I915_WRITE(GEN9_GAMT_ECO_REG_RW_IA,
1373                    (I915_READ(GEN9_GAMT_ECO_REG_RW_IA) |
1374                     GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS));
1375
1376         /* WaDisableLSQCROPERFforOCL:kbl */
1377         ret = wa_ring_whitelist_reg(engine, GEN8_L3SQCREG4);
1378         if (ret)
1379                 return ret;
1380
1381         return 0;
1382 }
1383
1384 static int glk_init_workarounds(struct intel_engine_cs *engine)
1385 {
1386         struct drm_i915_private *dev_priv = engine->i915;
1387         int ret;
1388
1389         ret = gen9_init_workarounds(engine);
1390         if (ret)
1391                 return ret;
1392
1393         /* WaToEnableHwFixForPushConstHWBug:glk */
1394         WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
1395                           GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
1396
1397         return 0;
1398 }
1399
1400 static int cfl_init_workarounds(struct intel_engine_cs *engine)
1401 {
1402         struct drm_i915_private *dev_priv = engine->i915;
1403         int ret;
1404
1405         ret = gen9_init_workarounds(engine);
1406         if (ret)
1407                 return ret;
1408
1409         /* WaEnableGapsTsvCreditFix:cfl */
1410         I915_WRITE(GEN8_GARBCNTL, (I915_READ(GEN8_GARBCNTL) |
1411                                    GEN9_GAPS_TSV_CREDIT_DISABLE));
1412
1413         /* WaToEnableHwFixForPushConstHWBug:cfl */
1414         WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
1415                           GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
1416
1417         /* WaDisableGafsUnitClkGating:cfl */
1418         I915_WRITE(GEN7_UCGCTL4, (I915_READ(GEN7_UCGCTL4) |
1419                                   GEN8_EU_GAUNIT_CLOCK_GATE_DISABLE));
1420
1421         /* WaDisableSbeCacheDispatchPortSharing:cfl */
1422         WA_SET_BIT_MASKED(
1423                 GEN7_HALF_SLICE_CHICKEN1,
1424                 GEN7_SBE_SS_CACHE_DISPATCH_PORT_SHARING_DISABLE);
1425
1426         /* WaInPlaceDecompressionHang:cfl */
1427         I915_WRITE(GEN9_GAMT_ECO_REG_RW_IA,
1428                    (I915_READ(GEN9_GAMT_ECO_REG_RW_IA) |
1429                     GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS));
1430
1431         return 0;
1432 }
1433
1434 int init_workarounds_ring(struct intel_engine_cs *engine)
1435 {
1436         struct drm_i915_private *dev_priv = engine->i915;
1437         int err;
1438
1439         WARN_ON(engine->id != RCS);
1440
1441         dev_priv->workarounds.count = 0;
1442         dev_priv->workarounds.hw_whitelist_count[engine->id] = 0;
1443
1444         if (IS_BROADWELL(dev_priv))
1445                 err = bdw_init_workarounds(engine);
1446         else if (IS_CHERRYVIEW(dev_priv))
1447                 err = chv_init_workarounds(engine);
1448         else if (IS_SKYLAKE(dev_priv))
1449                 err =  skl_init_workarounds(engine);
1450         else if (IS_BROXTON(dev_priv))
1451                 err = bxt_init_workarounds(engine);
1452         else if (IS_KABYLAKE(dev_priv))
1453                 err = kbl_init_workarounds(engine);
1454         else if (IS_GEMINILAKE(dev_priv))
1455                 err =  glk_init_workarounds(engine);
1456         else if (IS_COFFEELAKE(dev_priv))
1457                 err = cfl_init_workarounds(engine);
1458         else if (IS_CANNONLAKE(dev_priv))
1459                 err = cnl_init_workarounds(engine);
1460         else
1461                 err = 0;
1462         if (err)
1463                 return err;
1464
1465         DRM_DEBUG_DRIVER("%s: Number of context specific w/a: %d\n",
1466                          engine->name, dev_priv->workarounds.count);
1467         return 0;
1468 }
1469
1470 int intel_ring_workarounds_emit(struct drm_i915_gem_request *req)
1471 {
1472         struct i915_workarounds *w = &req->i915->workarounds;
1473         u32 *cs;
1474         int ret, i;
1475
1476         if (w->count == 0)
1477                 return 0;
1478
1479         ret = req->engine->emit_flush(req, EMIT_BARRIER);
1480         if (ret)
1481                 return ret;
1482
1483         cs = intel_ring_begin(req, (w->count * 2 + 2));
1484         if (IS_ERR(cs))
1485                 return PTR_ERR(cs);
1486
1487         *cs++ = MI_LOAD_REGISTER_IMM(w->count);
1488         for (i = 0; i < w->count; i++) {
1489                 *cs++ = i915_mmio_reg_offset(w->reg[i].addr);
1490                 *cs++ = w->reg[i].value;
1491         }
1492         *cs++ = MI_NOOP;
1493
1494         intel_ring_advance(req, cs);
1495
1496         ret = req->engine->emit_flush(req, EMIT_BARRIER);
1497         if (ret)
1498                 return ret;
1499
1500         return 0;
1501 }
1502
1503 static bool ring_is_idle(struct intel_engine_cs *engine)
1504 {
1505         struct drm_i915_private *dev_priv = engine->i915;
1506         bool idle = true;
1507
1508         intel_runtime_pm_get(dev_priv);
1509
1510         /* First check that no commands are left in the ring */
1511         if ((I915_READ_HEAD(engine) & HEAD_ADDR) !=
1512             (I915_READ_TAIL(engine) & TAIL_ADDR))
1513                 idle = false;
1514
1515         /* No bit for gen2, so assume the CS parser is idle */
1516         if (INTEL_GEN(dev_priv) > 2 && !(I915_READ_MODE(engine) & MODE_IDLE))
1517                 idle = false;
1518
1519         intel_runtime_pm_put(dev_priv);
1520
1521         return idle;
1522 }
1523
1524 /**
1525  * intel_engine_is_idle() - Report if the engine has finished process all work
1526  * @engine: the intel_engine_cs
1527  *
1528  * Return true if there are no requests pending, nothing left to be submitted
1529  * to hardware, and that the engine is idle.
1530  */
1531 bool intel_engine_is_idle(struct intel_engine_cs *engine)
1532 {
1533         struct drm_i915_private *dev_priv = engine->i915;
1534
1535         /* More white lies, if wedged, hw state is inconsistent */
1536         if (i915_terminally_wedged(&dev_priv->gpu_error))
1537                 return true;
1538
1539         /* Any inflight/incomplete requests? */
1540         if (!i915_seqno_passed(intel_engine_get_seqno(engine),
1541                                intel_engine_last_submit(engine)))
1542                 return false;
1543
1544         if (I915_SELFTEST_ONLY(engine->breadcrumbs.mock))
1545                 return true;
1546
1547         /* Interrupt/tasklet pending? */
1548         if (test_bit(ENGINE_IRQ_EXECLIST, &engine->irq_posted))
1549                 return false;
1550
1551         /* Waiting to drain ELSP? */
1552         if (READ_ONCE(engine->execlists.active))
1553                 return false;
1554
1555         /* ELSP is empty, but there are ready requests? */
1556         if (READ_ONCE(engine->execlists.first))
1557                 return false;
1558
1559         /* Ring stopped? */
1560         if (!ring_is_idle(engine))
1561                 return false;
1562
1563         return true;
1564 }
1565
1566 bool intel_engines_are_idle(struct drm_i915_private *dev_priv)
1567 {
1568         struct intel_engine_cs *engine;
1569         enum intel_engine_id id;
1570
1571         if (READ_ONCE(dev_priv->gt.active_requests))
1572                 return false;
1573
1574         /* If the driver is wedged, HW state may be very inconsistent and
1575          * report that it is still busy, even though we have stopped using it.
1576          */
1577         if (i915_terminally_wedged(&dev_priv->gpu_error))
1578                 return true;
1579
1580         for_each_engine(engine, dev_priv, id) {
1581                 if (!intel_engine_is_idle(engine))
1582                         return false;
1583         }
1584
1585         return true;
1586 }
1587
1588 void intel_engines_reset_default_submission(struct drm_i915_private *i915)
1589 {
1590         struct intel_engine_cs *engine;
1591         enum intel_engine_id id;
1592
1593         for_each_engine(engine, i915, id)
1594                 engine->set_default_submission(engine);
1595 }
1596
1597 void intel_engines_mark_idle(struct drm_i915_private *i915)
1598 {
1599         struct intel_engine_cs *engine;
1600         enum intel_engine_id id;
1601
1602         for_each_engine(engine, i915, id) {
1603                 intel_engine_disarm_breadcrumbs(engine);
1604                 i915_gem_batch_pool_fini(&engine->batch_pool);
1605                 tasklet_kill(&engine->execlists.irq_tasklet);
1606                 engine->execlists.no_priolist = false;
1607         }
1608 }
1609
1610 bool intel_engine_can_store_dword(struct intel_engine_cs *engine)
1611 {
1612         switch (INTEL_GEN(engine->i915)) {
1613         case 2:
1614                 return false; /* uses physical not virtual addresses */
1615         case 3:
1616                 /* maybe only uses physical not virtual addresses */
1617                 return !(IS_I915G(engine->i915) || IS_I915GM(engine->i915));
1618         case 6:
1619                 return engine->class != VIDEO_DECODE_CLASS; /* b0rked */
1620         default:
1621                 return true;
1622         }
1623 }
1624
1625 static void print_request(struct drm_printer *m,
1626                           struct drm_i915_gem_request *rq,
1627                           const char *prefix)
1628 {
1629         drm_printf(m, "%s%x%s [%x:%x] prio=%d @ %dms: %s\n", prefix,
1630                    rq->global_seqno,
1631                    i915_gem_request_completed(rq) ? "!" : "",
1632                    rq->ctx->hw_id, rq->fence.seqno,
1633                    rq->priotree.priority,
1634                    jiffies_to_msecs(jiffies - rq->emitted_jiffies),
1635                    rq->timeline->common->name);
1636 }
1637
1638 void intel_engine_dump(struct intel_engine_cs *engine, struct drm_printer *m)
1639 {
1640         struct intel_breadcrumbs * const b = &engine->breadcrumbs;
1641         const struct intel_engine_execlists * const execlists = &engine->execlists;
1642         struct i915_gpu_error * const error = &engine->i915->gpu_error;
1643         struct drm_i915_private *dev_priv = engine->i915;
1644         struct drm_i915_gem_request *rq;
1645         struct rb_node *rb;
1646         u64 addr;
1647
1648         drm_printf(m, "%s\n", engine->name);
1649         drm_printf(m, "\tcurrent seqno %x, last %x, hangcheck %x [%d ms], inflight %d\n",
1650                    intel_engine_get_seqno(engine),
1651                    intel_engine_last_submit(engine),
1652                    engine->hangcheck.seqno,
1653                    jiffies_to_msecs(jiffies - engine->hangcheck.action_timestamp),
1654                    engine->timeline->inflight_seqnos);
1655         drm_printf(m, "\tReset count: %d\n",
1656                    i915_reset_engine_count(error, engine));
1657
1658         rcu_read_lock();
1659
1660         drm_printf(m, "\tRequests:\n");
1661
1662         rq = list_first_entry(&engine->timeline->requests,
1663                               struct drm_i915_gem_request, link);
1664         if (&rq->link != &engine->timeline->requests)
1665                 print_request(m, rq, "\t\tfirst  ");
1666
1667         rq = list_last_entry(&engine->timeline->requests,
1668                              struct drm_i915_gem_request, link);
1669         if (&rq->link != &engine->timeline->requests)
1670                 print_request(m, rq, "\t\tlast   ");
1671
1672         rq = i915_gem_find_active_request(engine);
1673         if (rq) {
1674                 print_request(m, rq, "\t\tactive ");
1675                 drm_printf(m,
1676                            "\t\t[head %04x, postfix %04x, tail %04x, batch 0x%08x_%08x]\n",
1677                            rq->head, rq->postfix, rq->tail,
1678                            rq->batch ? upper_32_bits(rq->batch->node.start) : ~0u,
1679                            rq->batch ? lower_32_bits(rq->batch->node.start) : ~0u);
1680         }
1681
1682         drm_printf(m, "\tRING_START: 0x%08x [0x%08x]\n",
1683                    I915_READ(RING_START(engine->mmio_base)),
1684                    rq ? i915_ggtt_offset(rq->ring->vma) : 0);
1685         drm_printf(m, "\tRING_HEAD:  0x%08x [0x%08x]\n",
1686                    I915_READ(RING_HEAD(engine->mmio_base)) & HEAD_ADDR,
1687                    rq ? rq->ring->head : 0);
1688         drm_printf(m, "\tRING_TAIL:  0x%08x [0x%08x]\n",
1689                    I915_READ(RING_TAIL(engine->mmio_base)) & TAIL_ADDR,
1690                    rq ? rq->ring->tail : 0);
1691         drm_printf(m, "\tRING_CTL:   0x%08x [%s]\n",
1692                    I915_READ(RING_CTL(engine->mmio_base)),
1693                    I915_READ(RING_CTL(engine->mmio_base)) & (RING_WAIT | RING_WAIT_SEMAPHORE) ? "waiting" : "");
1694
1695         rcu_read_unlock();
1696
1697         addr = intel_engine_get_active_head(engine);
1698         drm_printf(m, "\tACTHD:  0x%08x_%08x\n",
1699                    upper_32_bits(addr), lower_32_bits(addr));
1700         addr = intel_engine_get_last_batch_head(engine);
1701         drm_printf(m, "\tBBADDR: 0x%08x_%08x\n",
1702                    upper_32_bits(addr), lower_32_bits(addr));
1703
1704         if (i915_modparams.enable_execlists) {
1705                 const u32 *hws = &engine->status_page.page_addr[I915_HWS_CSB_BUF0_INDEX];
1706                 u32 ptr, read, write;
1707                 unsigned int idx;
1708
1709                 drm_printf(m, "\tExeclist status: 0x%08x %08x\n",
1710                            I915_READ(RING_EXECLIST_STATUS_LO(engine)),
1711                            I915_READ(RING_EXECLIST_STATUS_HI(engine)));
1712
1713                 ptr = I915_READ(RING_CONTEXT_STATUS_PTR(engine));
1714                 read = GEN8_CSB_READ_PTR(ptr);
1715                 write = GEN8_CSB_WRITE_PTR(ptr);
1716                 drm_printf(m, "\tExeclist CSB read %d [%d cached], write %d [%d from hws], interrupt posted? %s\n",
1717                            read, execlists->csb_head,
1718                            write,
1719                            intel_read_status_page(engine, intel_hws_csb_write_index(engine->i915)),
1720                            yesno(test_bit(ENGINE_IRQ_EXECLIST,
1721                                           &engine->irq_posted)));
1722                 if (read >= GEN8_CSB_ENTRIES)
1723                         read = 0;
1724                 if (write >= GEN8_CSB_ENTRIES)
1725                         write = 0;
1726                 if (read > write)
1727                         write += GEN8_CSB_ENTRIES;
1728                 while (read < write) {
1729                         idx = ++read % GEN8_CSB_ENTRIES;
1730                         drm_printf(m, "\tExeclist CSB[%d]: 0x%08x [0x%08x in hwsp], context: %d [%d in hwsp]\n",
1731                                    idx,
1732                                    I915_READ(RING_CONTEXT_STATUS_BUF_LO(engine, idx)),
1733                                    hws[idx * 2],
1734                                    I915_READ(RING_CONTEXT_STATUS_BUF_HI(engine, idx)),
1735                                    hws[idx * 2 + 1]);
1736                 }
1737
1738                 rcu_read_lock();
1739                 for (idx = 0; idx < execlists_num_ports(execlists); idx++) {
1740                         unsigned int count;
1741
1742                         rq = port_unpack(&execlists->port[idx], &count);
1743                         if (rq) {
1744                                 drm_printf(m, "\t\tELSP[%d] count=%d, ",
1745                                            idx, count);
1746                                 print_request(m, rq, "rq: ");
1747                         } else {
1748                                 drm_printf(m, "\t\tELSP[%d] idle\n",
1749                                            idx);
1750                         }
1751                 }
1752                 drm_printf(m, "\t\tHW active? 0x%x\n", execlists->active);
1753                 rcu_read_unlock();
1754         } else if (INTEL_GEN(dev_priv) > 6) {
1755                 drm_printf(m, "\tPP_DIR_BASE: 0x%08x\n",
1756                            I915_READ(RING_PP_DIR_BASE(engine)));
1757                 drm_printf(m, "\tPP_DIR_BASE_READ: 0x%08x\n",
1758                            I915_READ(RING_PP_DIR_BASE_READ(engine)));
1759                 drm_printf(m, "\tPP_DIR_DCLV: 0x%08x\n",
1760                            I915_READ(RING_PP_DIR_DCLV(engine)));
1761         }
1762
1763         spin_lock_irq(&engine->timeline->lock);
1764         list_for_each_entry(rq, &engine->timeline->requests, link)
1765                 print_request(m, rq, "\t\tE ");
1766         for (rb = execlists->first; rb; rb = rb_next(rb)) {
1767                 struct i915_priolist *p =
1768                         rb_entry(rb, typeof(*p), node);
1769
1770                 list_for_each_entry(rq, &p->requests, priotree.link)
1771                         print_request(m, rq, "\t\tQ ");
1772         }
1773         spin_unlock_irq(&engine->timeline->lock);
1774
1775         spin_lock_irq(&b->rb_lock);
1776         for (rb = rb_first(&b->waiters); rb; rb = rb_next(rb)) {
1777                 struct intel_wait *w = rb_entry(rb, typeof(*w), node);
1778
1779                 drm_printf(m, "\t%s [%d] waiting for %x\n",
1780                            w->tsk->comm, w->tsk->pid, w->seqno);
1781         }
1782         spin_unlock_irq(&b->rb_lock);
1783
1784         drm_printf(m, "\n");
1785 }
1786
1787 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
1788 #include "selftests/mock_engine.c"
1789 #endif