Merge tag 'drm-intel-next-2019-03-20' of git://anongit.freedesktop.org/drm/drm-intel...
authorDave Airlie <airlied@redhat.com>
Mon, 25 Mar 2019 20:15:27 +0000 (06:15 +1000)
committerDave Airlie <airlied@redhat.com>
Mon, 25 Mar 2019 20:15:27 +0000 (06:15 +1000)
UAPI Changes:
- Report an error early instead of SIGBUS later when mmap beyond BO size

Core Changes:
- This includes backmerge of drm-next and two merges of Maarten's
  topic/hdr-formats

Driver Changes:
- Add Comet Lake (Gen9) PCI IDs to Coffee Lake ID list (Anusha)
- Add missing ICL PCI ID (Jose)
- Fix legacy gamma mode for ICL (Ville)
- Assume eDP is present on port A when there is no VBT (Thomas)
- Corrections to eDP training patterns (Jose)
- Fix PSR2 selective update corruption after PSR1 setup (Jose)
- Fix CRC mismatch error for DP link layer compliance (Aditya)
- Fix CNL DPLL readout and clean up code (Ville)
- Turn off the CUS when turning off a HDR plane (Ville)
- Avoid a race with execlist tasklet during race (Chris)
- Add missing CSC readout and clean up code (Ville)
- Avoid unnecessary wakeref during debugfs/drop_caches/set (Chris, Caz)
- Hold references to ring/HW context/context explicitly when used (Chris)

- Assume next platforms inherit old platform (Rodrigo)
- Use HWS indices rather than addresses for breadcrumbs (Chris)
- Add REG_BIT/REG_GENMASK and REG_FIELD_PREP macros (Jani)
- Convert crept in C99 types to kernel fixed size types (Jani)
- Avoid passing full dev_priv in forcewake functions (Daniele)
- Reset GuC on GPU reset (Sujaritha)
- Rework MG and Combo PLLs to vfuncs (Lucas)
- Explicitly track ppGTT size (Chris, Bob)
- Coding style improvements and code modularization (Ville)
- Selftest and debugging improvements (Chris)

Signed-off-by: Dave Airlie <airlied@redhat.com>
# Conflicts:
# drivers/gpu/drm/i915/intel_hdmi.c
From: Joonas Lahtinen <joonas.lahtinen@linux.intel.com>
Link: https://patchwork.freedesktop.org/patch/msgid/20190325124925.GA12726@jlahtine-desk.ger.corp.intel.com
151 files changed:
drivers/gpu/drm/i915/Makefile
drivers/gpu/drm/i915/gvt/cmd_parser.c
drivers/gpu/drm/i915/gvt/dmabuf.c
drivers/gpu/drm/i915/gvt/execlist.c
drivers/gpu/drm/i915/gvt/handlers.c
drivers/gpu/drm/i915/gvt/interrupt.c
drivers/gpu/drm/i915/gvt/mmio_context.c
drivers/gpu/drm/i915/gvt/scheduler.c
drivers/gpu/drm/i915/gvt/vgpu.c
drivers/gpu/drm/i915/i915_active.c
drivers/gpu/drm/i915/i915_active.h
drivers/gpu/drm/i915/i915_cmd_parser.c
drivers/gpu/drm/i915/i915_debugfs.c
drivers/gpu/drm/i915/i915_drv.c
drivers/gpu/drm/i915/i915_drv.h
drivers/gpu/drm/i915/i915_gem.c
drivers/gpu/drm/i915/i915_gem.h
drivers/gpu/drm/i915/i915_gem_context.c
drivers/gpu/drm/i915/i915_gem_context.h
drivers/gpu/drm/i915/i915_gem_context_types.h [new file with mode: 0644]
drivers/gpu/drm/i915/i915_gem_dmabuf.c
drivers/gpu/drm/i915/i915_gem_evict.c
drivers/gpu/drm/i915/i915_gem_execbuffer.c
drivers/gpu/drm/i915/i915_gem_fence_reg.c
drivers/gpu/drm/i915/i915_gem_gtt.c
drivers/gpu/drm/i915/i915_gem_gtt.h
drivers/gpu/drm/i915/i915_gem_internal.c
drivers/gpu/drm/i915/i915_gem_object.c
drivers/gpu/drm/i915/i915_gem_object.h
drivers/gpu/drm/i915/i915_gem_render_state.c
drivers/gpu/drm/i915/i915_gem_stolen.c
drivers/gpu/drm/i915/i915_gem_userptr.c
drivers/gpu/drm/i915/i915_globals.c [new file with mode: 0644]
drivers/gpu/drm/i915/i915_globals.h [new file with mode: 0644]
drivers/gpu/drm/i915/i915_gpu_error.c
drivers/gpu/drm/i915/i915_gpu_error.h
drivers/gpu/drm/i915/i915_irq.c
drivers/gpu/drm/i915/i915_pci.c
drivers/gpu/drm/i915/i915_perf.c
drivers/gpu/drm/i915/i915_pmu.c
drivers/gpu/drm/i915/i915_pvinfo.h
drivers/gpu/drm/i915/i915_query.c
drivers/gpu/drm/i915/i915_reg.h
drivers/gpu/drm/i915/i915_request.c
drivers/gpu/drm/i915/i915_request.h
drivers/gpu/drm/i915/i915_reset.c
drivers/gpu/drm/i915/i915_reset.h
drivers/gpu/drm/i915/i915_scheduler.c
drivers/gpu/drm/i915/i915_scheduler.h
drivers/gpu/drm/i915/i915_sw_fence.c
drivers/gpu/drm/i915/i915_sw_fence.h
drivers/gpu/drm/i915/i915_timeline.c
drivers/gpu/drm/i915/i915_timeline.h
drivers/gpu/drm/i915/i915_timeline_types.h [new file with mode: 0644]
drivers/gpu/drm/i915/i915_trace.h
drivers/gpu/drm/i915/i915_vgpu.c
drivers/gpu/drm/i915/i915_vgpu.h
drivers/gpu/drm/i915/i915_vma.c
drivers/gpu/drm/i915/i915_vma.h
drivers/gpu/drm/i915/icl_dsi.c
drivers/gpu/drm/i915/intel_atomic_plane.c
drivers/gpu/drm/i915/intel_audio.c
drivers/gpu/drm/i915/intel_bios.c
drivers/gpu/drm/i915/intel_cdclk.c
drivers/gpu/drm/i915/intel_color.c
drivers/gpu/drm/i915/intel_connector.c
drivers/gpu/drm/i915/intel_context.c [new file with mode: 0644]
drivers/gpu/drm/i915/intel_context.h [new file with mode: 0644]
drivers/gpu/drm/i915/intel_context_types.h [new file with mode: 0644]
drivers/gpu/drm/i915/intel_ddi.c
drivers/gpu/drm/i915/intel_device_info.c
drivers/gpu/drm/i915/intel_device_info.h
drivers/gpu/drm/i915/intel_display.c
drivers/gpu/drm/i915/intel_dp.c
drivers/gpu/drm/i915/intel_dpll_mgr.c
drivers/gpu/drm/i915/intel_dpll_mgr.h
drivers/gpu/drm/i915/intel_drv.h
drivers/gpu/drm/i915/intel_dsi_vbt.c
drivers/gpu/drm/i915/intel_engine_cs.c
drivers/gpu/drm/i915/intel_engine_types.h [new file with mode: 0644]
drivers/gpu/drm/i915/intel_gpu_commands.h
drivers/gpu/drm/i915/intel_guc.c
drivers/gpu/drm/i915/intel_guc.h
drivers/gpu/drm/i915/intel_guc_ads.c
drivers/gpu/drm/i915/intel_guc_ct.c
drivers/gpu/drm/i915/intel_guc_ct.h
drivers/gpu/drm/i915/intel_guc_log.c
drivers/gpu/drm/i915/intel_guc_submission.c
drivers/gpu/drm/i915/intel_hangcheck.c
drivers/gpu/drm/i915/intel_hdcp.c
drivers/gpu/drm/i915/intel_hdmi.c
drivers/gpu/drm/i915/intel_lrc.c
drivers/gpu/drm/i915/intel_lspcon.c
drivers/gpu/drm/i915/intel_lvds.c
drivers/gpu/drm/i915/intel_mocs.c
drivers/gpu/drm/i915/intel_overlay.c
drivers/gpu/drm/i915/intel_panel.c
drivers/gpu/drm/i915/intel_pipe_crc.c
drivers/gpu/drm/i915/intel_pm.c
drivers/gpu/drm/i915/intel_psr.c
drivers/gpu/drm/i915/intel_ringbuffer.c
drivers/gpu/drm/i915/intel_ringbuffer.h
drivers/gpu/drm/i915/intel_runtime_pm.c
drivers/gpu/drm/i915/intel_sdvo.c
drivers/gpu/drm/i915/intel_sprite.c
drivers/gpu/drm/i915/intel_uc.c
drivers/gpu/drm/i915/intel_uc.h
drivers/gpu/drm/i915/intel_uncore.c
drivers/gpu/drm/i915/intel_uncore.h
drivers/gpu/drm/i915/intel_vbt_defs.h
drivers/gpu/drm/i915/intel_workarounds.c
drivers/gpu/drm/i915/intel_workarounds.h
drivers/gpu/drm/i915/intel_workarounds_types.h [new file with mode: 0644]
drivers/gpu/drm/i915/selftests/huge_gem_object.c
drivers/gpu/drm/i915/selftests/huge_pages.c
drivers/gpu/drm/i915/selftests/i915_active.c
drivers/gpu/drm/i915/selftests/i915_gem.c
drivers/gpu/drm/i915/selftests/i915_gem_coherency.c
drivers/gpu/drm/i915/selftests/i915_gem_context.c
drivers/gpu/drm/i915/selftests/i915_gem_evict.c
drivers/gpu/drm/i915/selftests/i915_gem_gtt.c
drivers/gpu/drm/i915/selftests/i915_gem_object.c
drivers/gpu/drm/i915/selftests/i915_request.c
drivers/gpu/drm/i915/selftests/i915_selftest.c
drivers/gpu/drm/i915/selftests/i915_sw_fence.c
drivers/gpu/drm/i915/selftests/i915_timeline.c
drivers/gpu/drm/i915/selftests/i915_vma.c
drivers/gpu/drm/i915/selftests/igt_flush_test.c
drivers/gpu/drm/i915/selftests/igt_spinner.c
drivers/gpu/drm/i915/selftests/intel_guc.c
drivers/gpu/drm/i915/selftests/intel_hangcheck.c
drivers/gpu/drm/i915/selftests/intel_lrc.c
drivers/gpu/drm/i915/selftests/intel_workarounds.c
drivers/gpu/drm/i915/selftests/mock_context.c
drivers/gpu/drm/i915/selftests/mock_engine.c
drivers/gpu/drm/i915/selftests/mock_gem_device.c
drivers/gpu/drm/i915/selftests/mock_request.c
drivers/gpu/drm/i915/selftests/mock_request.h
drivers/gpu/drm/i915/selftests/mock_timeline.c
drivers/gpu/drm/i915/test_i915_active_types_standalone.c [new file with mode: 0644]
drivers/gpu/drm/i915/test_i915_gem_context_types_standalone.c [new file with mode: 0644]
drivers/gpu/drm/i915/test_i915_timeline_types_standalone.c [new file with mode: 0644]
drivers/gpu/drm/i915/test_intel_context_types_standalone.c [new file with mode: 0644]
drivers/gpu/drm/i915/test_intel_engine_types_standalone.c [new file with mode: 0644]
drivers/gpu/drm/i915/test_intel_workarounds_types_standalone.c [new file with mode: 0644]
include/drm/drm_audio_component.h
include/drm/drm_hdcp.h
include/drm/i915_pciids.h
include/sound/hdaudio.h
include/uapi/drm/i915_drm.h
sound/hda/hdac_component.c

index 1787e1299b1b2a8ec6647473aa64c4c03efd71f0..68fecf355471474999e9796baf110fd1dc321db4 100644 (file)
@@ -56,6 +56,15 @@ i915-$(CONFIG_COMPAT)   += i915_ioc32.o
 i915-$(CONFIG_DEBUG_FS) += i915_debugfs.o intel_pipe_crc.o
 i915-$(CONFIG_PERF_EVENTS) += i915_pmu.o
 
+# Test the headers are compilable as standalone units
+i915-$(CONFIG_DRM_I915_WERROR) += \
+       test_i915_active_types_standalone.o \
+       test_i915_gem_context_types_standalone.o \
+       test_i915_timeline_types_standalone.o \
+       test_intel_context_types_standalone.o \
+       test_intel_engine_types_standalone.o \
+       test_intel_workarounds_types_standalone.o
+
 # GEM code
 i915-y += \
          i915_active.o \
@@ -77,6 +86,7 @@ i915-y += \
          i915_gem_tiling.o \
          i915_gem_userptr.o \
          i915_gemfs.o \
+         i915_globals.o \
          i915_query.o \
          i915_request.o \
          i915_scheduler.o \
@@ -84,6 +94,7 @@ i915-y += \
          i915_trace_points.o \
          i915_vma.o \
          intel_breadcrumbs.o \
+         intel_context.o \
          intel_engine_cs.o \
          intel_hangcheck.o \
          intel_lrc.o \
index 35b4ec3f7618b887e5661d0d652cca99b6ed02c6..cf4a1ecf6853659f6c231b07e1d5149c5f480555 100644 (file)
@@ -391,12 +391,12 @@ struct cmd_info {
 #define F_POST_HANDLE  (1<<2)
        u32 flag;
 
-#define R_RCS  (1 << RCS)
-#define R_VCS1  (1 << VCS)
-#define R_VCS2  (1 << VCS2)
+#define R_RCS  BIT(RCS0)
+#define R_VCS1  BIT(VCS0)
+#define R_VCS2  BIT(VCS1)
 #define R_VCS  (R_VCS1 | R_VCS2)
-#define R_BCS  (1 << BCS)
-#define R_VECS (1 << VECS)
+#define R_BCS  BIT(BCS0)
+#define R_VECS BIT(VECS0)
 #define R_ALL (R_RCS | R_VCS | R_BCS | R_VECS)
        /* rings that support this cmd: BLT/RCS/VCS/VECS */
        u16 rings;
@@ -558,7 +558,7 @@ static const struct decode_info decode_info_vebox = {
 };
 
 static const struct decode_info *ring_decode_info[I915_NUM_ENGINES][8] = {
-       [RCS] = {
+       [RCS0] = {
                &decode_info_mi,
                NULL,
                NULL,
@@ -569,7 +569,7 @@ static const struct decode_info *ring_decode_info[I915_NUM_ENGINES][8] = {
                NULL,
        },
 
-       [VCS] = {
+       [VCS0] = {
                &decode_info_mi,
                NULL,
                NULL,
@@ -580,7 +580,7 @@ static const struct decode_info *ring_decode_info[I915_NUM_ENGINES][8] = {
                NULL,
        },
 
-       [BCS] = {
+       [BCS0] = {
                &decode_info_mi,
                NULL,
                &decode_info_2d,
@@ -591,7 +591,7 @@ static const struct decode_info *ring_decode_info[I915_NUM_ENGINES][8] = {
                NULL,
        },
 
-       [VECS] = {
+       [VECS0] = {
                &decode_info_mi,
                NULL,
                NULL,
@@ -602,7 +602,7 @@ static const struct decode_info *ring_decode_info[I915_NUM_ENGINES][8] = {
                NULL,
        },
 
-       [VCS2] = {
+       [VCS1] = {
                &decode_info_mi,
                NULL,
                NULL,
@@ -631,8 +631,7 @@ static inline const struct cmd_info *find_cmd_entry(struct intel_gvt *gvt,
        struct cmd_entry *e;
 
        hash_for_each_possible(gvt->cmd_table, e, hlist, opcode) {
-               if ((opcode == e->info->opcode) &&
-                               (e->info->rings & (1 << ring_id)))
+               if (opcode == e->info->opcode && e->info->rings & BIT(ring_id))
                        return e->info;
        }
        return NULL;
@@ -943,15 +942,12 @@ static int cmd_handler_lri(struct parser_exec_state *s)
        struct intel_gvt *gvt = s->vgpu->gvt;
 
        for (i = 1; i < cmd_len; i += 2) {
-               if (IS_BROADWELL(gvt->dev_priv) &&
-                               (s->ring_id != RCS)) {
-                       if (s->ring_id == BCS &&
-                                       cmd_reg(s, i) ==
-                                       i915_mmio_reg_offset(DERRMR))
+               if (IS_BROADWELL(gvt->dev_priv) && s->ring_id != RCS0) {
+                       if (s->ring_id == BCS0 &&
+                           cmd_reg(s, i) == i915_mmio_reg_offset(DERRMR))
                                ret |= 0;
                        else
-                               ret |= (cmd_reg_inhibit(s, i)) ?
-                                       -EBADRQC : 0;
+                               ret |= cmd_reg_inhibit(s, i) ? -EBADRQC : 0;
                }
                if (ret)
                        break;
@@ -1047,27 +1043,27 @@ struct cmd_interrupt_event {
 };
 
 static struct cmd_interrupt_event cmd_interrupt_events[] = {
-       [RCS] = {
+       [RCS0] = {
                .pipe_control_notify = RCS_PIPE_CONTROL,
                .mi_flush_dw = INTEL_GVT_EVENT_RESERVED,
                .mi_user_interrupt = RCS_MI_USER_INTERRUPT,
        },
-       [BCS] = {
+       [BCS0] = {
                .pipe_control_notify = INTEL_GVT_EVENT_RESERVED,
                .mi_flush_dw = BCS_MI_FLUSH_DW,
                .mi_user_interrupt = BCS_MI_USER_INTERRUPT,
        },
-       [VCS] = {
+       [VCS0] = {
                .pipe_control_notify = INTEL_GVT_EVENT_RESERVED,
                .mi_flush_dw = VCS_MI_FLUSH_DW,
                .mi_user_interrupt = VCS_MI_USER_INTERRUPT,
        },
-       [VCS2] = {
+       [VCS1] = {
                .pipe_control_notify = INTEL_GVT_EVENT_RESERVED,
                .mi_flush_dw = VCS2_MI_FLUSH_DW,
                .mi_user_interrupt = VCS2_MI_USER_INTERRUPT,
        },
-       [VECS] = {
+       [VECS0] = {
                .pipe_control_notify = INTEL_GVT_EVENT_RESERVED,
                .mi_flush_dw = VECS_MI_FLUSH_DW,
                .mi_user_interrupt = VECS_MI_USER_INTERRUPT,
index 3e7e2b80c8579017cecdda478bc6166e1f46e061..f27edf17b4ab6ce089db97fa28ec44894f5ed7bc 100644 (file)
@@ -153,7 +153,7 @@ static struct drm_i915_gem_object *vgpu_create_gem(struct drm_device *dev,
        struct drm_i915_private *dev_priv = to_i915(dev);
        struct drm_i915_gem_object *obj;
 
-       obj = i915_gem_object_alloc(dev_priv);
+       obj = i915_gem_object_alloc();
        if (obj == NULL)
                return NULL;
 
index 70494e394d2cbd0c2dceba4a8d466dfb11ec7658..1a93472cb34eff6170643a1aa76cc33943661b5f 100644 (file)
                ((a)->lrca == (b)->lrca))
 
 static int context_switch_events[] = {
-       [RCS] = RCS_AS_CONTEXT_SWITCH,
-       [BCS] = BCS_AS_CONTEXT_SWITCH,
-       [VCS] = VCS_AS_CONTEXT_SWITCH,
-       [VCS2] = VCS2_AS_CONTEXT_SWITCH,
-       [VECS] = VECS_AS_CONTEXT_SWITCH,
+       [RCS0]  = RCS_AS_CONTEXT_SWITCH,
+       [BCS0]  = BCS_AS_CONTEXT_SWITCH,
+       [VCS0]  = VCS_AS_CONTEXT_SWITCH,
+       [VCS1]  = VCS2_AS_CONTEXT_SWITCH,
+       [VECS0] = VECS_AS_CONTEXT_SWITCH,
 };
 
-static int ring_id_to_context_switch_event(int ring_id)
+static int ring_id_to_context_switch_event(unsigned int ring_id)
 {
-       if (WARN_ON(ring_id < RCS ||
-                   ring_id >= ARRAY_SIZE(context_switch_events)))
+       if (WARN_ON(ring_id >= ARRAY_SIZE(context_switch_events)))
                return -EINVAL;
 
        return context_switch_events[ring_id];
@@ -411,7 +410,7 @@ static int complete_execlist_workload(struct intel_vgpu_workload *workload)
        gvt_dbg_el("complete workload %p status %d\n", workload,
                        workload->status);
 
-       if (workload->status || (vgpu->resetting_eng & ENGINE_MASK(ring_id)))
+       if (workload->status || (vgpu->resetting_eng & BIT(ring_id)))
                goto out;
 
        if (!list_empty(workload_q_head(vgpu, ring_id))) {
index bc64b810e0d5300c9cc6ea812f1a0800f3d2f6da..b596cb42e24eb83621fdddf3d7934086824e5e17 100644 (file)
@@ -323,25 +323,25 @@ static int gdrst_mmio_write(struct intel_vgpu *vgpu, unsigned int offset,
        } else {
                if (data & GEN6_GRDOM_RENDER) {
                        gvt_dbg_mmio("vgpu%d: request RCS reset\n", vgpu->id);
-                       engine_mask |= (1 << RCS);
+                       engine_mask |= BIT(RCS0);
                }
                if (data & GEN6_GRDOM_MEDIA) {
                        gvt_dbg_mmio("vgpu%d: request VCS reset\n", vgpu->id);
-                       engine_mask |= (1 << VCS);
+                       engine_mask |= BIT(VCS0);
                }
                if (data & GEN6_GRDOM_BLT) {
                        gvt_dbg_mmio("vgpu%d: request BCS Reset\n", vgpu->id);
-                       engine_mask |= (1 << BCS);
+                       engine_mask |= BIT(BCS0);
                }
                if (data & GEN6_GRDOM_VECS) {
                        gvt_dbg_mmio("vgpu%d: request VECS Reset\n", vgpu->id);
-                       engine_mask |= (1 << VECS);
+                       engine_mask |= BIT(VECS0);
                }
                if (data & GEN8_GRDOM_MEDIA2) {
                        gvt_dbg_mmio("vgpu%d: request VCS2 Reset\n", vgpu->id);
-                       if (HAS_BSD2(vgpu->gvt->dev_priv))
-                               engine_mask |= (1 << VCS2);
+                       engine_mask |= BIT(VCS1);
                }
+               engine_mask &= INTEL_INFO(vgpu->gvt->dev_priv)->engine_mask;
        }
 
        /* vgpu_lock already hold by emulate mmio r/w */
@@ -1704,7 +1704,7 @@ static int ring_mode_mmio_write(struct intel_vgpu *vgpu, unsigned int offset,
                        return 0;
 
                ret = intel_vgpu_select_submission_ops(vgpu,
-                              ENGINE_MASK(ring_id),
+                              BIT(ring_id),
                               INTEL_VGPU_EXECLIST_SUBMISSION);
                if (ret)
                        return ret;
@@ -1724,19 +1724,19 @@ static int gvt_reg_tlb_control_handler(struct intel_vgpu *vgpu,
 
        switch (offset) {
        case 0x4260:
-               id = RCS;
+               id = RCS0;
                break;
        case 0x4264:
-               id = VCS;
+               id = VCS0;
                break;
        case 0x4268:
-               id = VCS2;
+               id = VCS1;
                break;
        case 0x426c:
-               id = BCS;
+               id = BCS0;
                break;
        case 0x4270:
-               id = VECS;
+               id = VECS0;
                break;
        default:
                return -EINVAL;
@@ -1793,7 +1793,7 @@ static int ring_reset_ctl_write(struct intel_vgpu *vgpu,
        MMIO_F(prefix(BLT_RING_BASE), s, f, am, rm, d, r, w); \
        MMIO_F(prefix(GEN6_BSD_RING_BASE), s, f, am, rm, d, r, w); \
        MMIO_F(prefix(VEBOX_RING_BASE), s, f, am, rm, d, r, w); \
-       if (HAS_BSD2(dev_priv)) \
+       if (HAS_ENGINE(dev_priv, VCS1)) \
                MMIO_F(prefix(GEN8_BSD2_RING_BASE), s, f, am, rm, d, r, w); \
 } while (0)
 
index 67125c5eec6eb59c7502065ae2c4a3698b390822..951681813230fbcd10052a07efe80c51c4ea716c 100644 (file)
@@ -536,7 +536,7 @@ static void gen8_init_irq(
        SET_BIT_INFO(irq, 4, VCS_MI_FLUSH_DW, INTEL_GVT_IRQ_INFO_GT1);
        SET_BIT_INFO(irq, 8, VCS_AS_CONTEXT_SWITCH, INTEL_GVT_IRQ_INFO_GT1);
 
-       if (HAS_BSD2(gvt->dev_priv)) {
+       if (HAS_ENGINE(gvt->dev_priv, VCS1)) {
                SET_BIT_INFO(irq, 16, VCS2_MI_USER_INTERRUPT,
                        INTEL_GVT_IRQ_INFO_GT1);
                SET_BIT_INFO(irq, 20, VCS2_MI_FLUSH_DW,
index 7d84cfb9051ac886579648ac7bb2cc5e2a70b3fa..f64c76dd11d4f61038ac48bf02ceaa3caeaf02c3 100644 (file)
 
 /* Raw offset is appened to each line for convenience. */
 static struct engine_mmio gen8_engine_mmio_list[] __cacheline_aligned = {
-       {RCS, GFX_MODE_GEN7, 0xffff, false}, /* 0x229c */
-       {RCS, GEN9_CTX_PREEMPT_REG, 0x0, false}, /* 0x2248 */
-       {RCS, HWSTAM, 0x0, false}, /* 0x2098 */
-       {RCS, INSTPM, 0xffff, true}, /* 0x20c0 */
-       {RCS, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 0), 0, false}, /* 0x24d0 */
-       {RCS, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 1), 0, false}, /* 0x24d4 */
-       {RCS, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 2), 0, false}, /* 0x24d8 */
-       {RCS, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 3), 0, false}, /* 0x24dc */
-       {RCS, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 4), 0, false}, /* 0x24e0 */
-       {RCS, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 5), 0, false}, /* 0x24e4 */
-       {RCS, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 6), 0, false}, /* 0x24e8 */
-       {RCS, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 7), 0, false}, /* 0x24ec */
-       {RCS, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 8), 0, false}, /* 0x24f0 */
-       {RCS, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 9), 0, false}, /* 0x24f4 */
-       {RCS, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 10), 0, false}, /* 0x24f8 */
-       {RCS, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 11), 0, false}, /* 0x24fc */
-       {RCS, CACHE_MODE_1, 0xffff, true}, /* 0x7004 */
-       {RCS, GEN7_GT_MODE, 0xffff, true}, /* 0x7008 */
-       {RCS, CACHE_MODE_0_GEN7, 0xffff, true}, /* 0x7000 */
-       {RCS, GEN7_COMMON_SLICE_CHICKEN1, 0xffff, true}, /* 0x7010 */
-       {RCS, HDC_CHICKEN0, 0xffff, true}, /* 0x7300 */
-       {RCS, VF_GUARDBAND, 0xffff, true}, /* 0x83a4 */
-
-       {BCS, RING_GFX_MODE(BLT_RING_BASE), 0xffff, false}, /* 0x2229c */
-       {BCS, RING_MI_MODE(BLT_RING_BASE), 0xffff, false}, /* 0x2209c */
-       {BCS, RING_INSTPM(BLT_RING_BASE), 0xffff, false}, /* 0x220c0 */
-       {BCS, RING_HWSTAM(BLT_RING_BASE), 0x0, false}, /* 0x22098 */
-       {BCS, RING_EXCC(BLT_RING_BASE), 0x0, false}, /* 0x22028 */
-       {RCS, INVALID_MMIO_REG, 0, false } /* Terminated */
+       {RCS0, GFX_MODE_GEN7, 0xffff, false}, /* 0x229c */
+       {RCS0, GEN9_CTX_PREEMPT_REG, 0x0, false}, /* 0x2248 */
+       {RCS0, HWSTAM, 0x0, false}, /* 0x2098 */
+       {RCS0, INSTPM, 0xffff, true}, /* 0x20c0 */
+       {RCS0, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 0), 0, false}, /* 0x24d0 */
+       {RCS0, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 1), 0, false}, /* 0x24d4 */
+       {RCS0, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 2), 0, false}, /* 0x24d8 */
+       {RCS0, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 3), 0, false}, /* 0x24dc */
+       {RCS0, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 4), 0, false}, /* 0x24e0 */
+       {RCS0, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 5), 0, false}, /* 0x24e4 */
+       {RCS0, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 6), 0, false}, /* 0x24e8 */
+       {RCS0, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 7), 0, false}, /* 0x24ec */
+       {RCS0, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 8), 0, false}, /* 0x24f0 */
+       {RCS0, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 9), 0, false}, /* 0x24f4 */
+       {RCS0, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 10), 0, false}, /* 0x24f8 */
+       {RCS0, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 11), 0, false}, /* 0x24fc */
+       {RCS0, CACHE_MODE_1, 0xffff, true}, /* 0x7004 */
+       {RCS0, GEN7_GT_MODE, 0xffff, true}, /* 0x7008 */
+       {RCS0, CACHE_MODE_0_GEN7, 0xffff, true}, /* 0x7000 */
+       {RCS0, GEN7_COMMON_SLICE_CHICKEN1, 0xffff, true}, /* 0x7010 */
+       {RCS0, HDC_CHICKEN0, 0xffff, true}, /* 0x7300 */
+       {RCS0, VF_GUARDBAND, 0xffff, true}, /* 0x83a4 */
+
+       {BCS0, RING_GFX_MODE(BLT_RING_BASE), 0xffff, false}, /* 0x2229c */
+       {BCS0, RING_MI_MODE(BLT_RING_BASE), 0xffff, false}, /* 0x2209c */
+       {BCS0, RING_INSTPM(BLT_RING_BASE), 0xffff, false}, /* 0x220c0 */
+       {BCS0, RING_HWSTAM(BLT_RING_BASE), 0x0, false}, /* 0x22098 */
+       {BCS0, RING_EXCC(BLT_RING_BASE), 0x0, false}, /* 0x22028 */
+       {RCS0, INVALID_MMIO_REG, 0, false } /* Terminated */
 };
 
 static struct engine_mmio gen9_engine_mmio_list[] __cacheline_aligned = {
-       {RCS, GFX_MODE_GEN7, 0xffff, false}, /* 0x229c */
-       {RCS, GEN9_CTX_PREEMPT_REG, 0x0, false}, /* 0x2248 */
-       {RCS, HWSTAM, 0x0, false}, /* 0x2098 */
-       {RCS, INSTPM, 0xffff, true}, /* 0x20c0 */
-       {RCS, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 0), 0, false}, /* 0x24d0 */
-       {RCS, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 1), 0, false}, /* 0x24d4 */
-       {RCS, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 2), 0, false}, /* 0x24d8 */
-       {RCS, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 3), 0, false}, /* 0x24dc */
-       {RCS, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 4), 0, false}, /* 0x24e0 */
-       {RCS, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 5), 0, false}, /* 0x24e4 */
-       {RCS, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 6), 0, false}, /* 0x24e8 */
-       {RCS, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 7), 0, false}, /* 0x24ec */
-       {RCS, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 8), 0, false}, /* 0x24f0 */
-       {RCS, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 9), 0, false}, /* 0x24f4 */
-       {RCS, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 10), 0, false}, /* 0x24f8 */
-       {RCS, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 11), 0, false}, /* 0x24fc */
-       {RCS, CACHE_MODE_1, 0xffff, true}, /* 0x7004 */
-       {RCS, GEN7_GT_MODE, 0xffff, true}, /* 0x7008 */
-       {RCS, CACHE_MODE_0_GEN7, 0xffff, true}, /* 0x7000 */
-       {RCS, GEN7_COMMON_SLICE_CHICKEN1, 0xffff, true}, /* 0x7010 */
-       {RCS, HDC_CHICKEN0, 0xffff, true}, /* 0x7300 */
-       {RCS, VF_GUARDBAND, 0xffff, true}, /* 0x83a4 */
-
-       {RCS, GEN8_PRIVATE_PAT_LO, 0, false}, /* 0x40e0 */
-       {RCS, GEN8_PRIVATE_PAT_HI, 0, false}, /* 0x40e4 */
-       {RCS, GEN8_CS_CHICKEN1, 0xffff, true}, /* 0x2580 */
-       {RCS, COMMON_SLICE_CHICKEN2, 0xffff, true}, /* 0x7014 */
-       {RCS, GEN9_CS_DEBUG_MODE1, 0xffff, false}, /* 0x20ec */
-       {RCS, GEN8_L3SQCREG4, 0, false}, /* 0xb118 */
-       {RCS, GEN7_HALF_SLICE_CHICKEN1, 0xffff, true}, /* 0xe100 */
-       {RCS, HALF_SLICE_CHICKEN2, 0xffff, true}, /* 0xe180 */
-       {RCS, HALF_SLICE_CHICKEN3, 0xffff, true}, /* 0xe184 */
-       {RCS, GEN9_HALF_SLICE_CHICKEN5, 0xffff, true}, /* 0xe188 */
-       {RCS, GEN9_HALF_SLICE_CHICKEN7, 0xffff, true}, /* 0xe194 */
-       {RCS, GEN8_ROW_CHICKEN, 0xffff, true}, /* 0xe4f0 */
-       {RCS, TRVATTL3PTRDW(0), 0, false}, /* 0x4de0 */
-       {RCS, TRVATTL3PTRDW(1), 0, false}, /* 0x4de4 */
-       {RCS, TRNULLDETCT, 0, false}, /* 0x4de8 */
-       {RCS, TRINVTILEDETCT, 0, false}, /* 0x4dec */
-       {RCS, TRVADR, 0, false}, /* 0x4df0 */
-       {RCS, TRTTE, 0, false}, /* 0x4df4 */
-
-       {BCS, RING_GFX_MODE(BLT_RING_BASE), 0xffff, false}, /* 0x2229c */
-       {BCS, RING_MI_MODE(BLT_RING_BASE), 0xffff, false}, /* 0x2209c */
-       {BCS, RING_INSTPM(BLT_RING_BASE), 0xffff, false}, /* 0x220c0 */
-       {BCS, RING_HWSTAM(BLT_RING_BASE), 0x0, false}, /* 0x22098 */
-       {BCS, RING_EXCC(BLT_RING_BASE), 0x0, false}, /* 0x22028 */
-
-       {VCS2, RING_EXCC(GEN8_BSD2_RING_BASE), 0xffff, false}, /* 0x1c028 */
-
-       {VECS, RING_EXCC(VEBOX_RING_BASE), 0xffff, false}, /* 0x1a028 */
-
-       {RCS, GEN8_HDC_CHICKEN1, 0xffff, true}, /* 0x7304 */
-       {RCS, GEN9_CTX_PREEMPT_REG, 0x0, false}, /* 0x2248 */
-       {RCS, GEN7_UCGCTL4, 0x0, false}, /* 0x940c */
-       {RCS, GAMT_CHKN_BIT_REG, 0x0, false}, /* 0x4ab8 */
-
-       {RCS, GEN9_GAMT_ECO_REG_RW_IA, 0x0, false}, /* 0x4ab0 */
-       {RCS, GEN9_CSFE_CHICKEN1_RCS, 0xffff, false}, /* 0x20d4 */
-
-       {RCS, GEN8_GARBCNTL, 0x0, false}, /* 0xb004 */
-       {RCS, GEN7_FF_THREAD_MODE, 0x0, false}, /* 0x20a0 */
-       {RCS, FF_SLICE_CS_CHICKEN2, 0xffff, false}, /* 0x20e4 */
-       {RCS, INVALID_MMIO_REG, 0, false } /* Terminated */
+       {RCS0, GFX_MODE_GEN7, 0xffff, false}, /* 0x229c */
+       {RCS0, GEN9_CTX_PREEMPT_REG, 0x0, false}, /* 0x2248 */
+       {RCS0, HWSTAM, 0x0, false}, /* 0x2098 */
+       {RCS0, INSTPM, 0xffff, true}, /* 0x20c0 */
+       {RCS0, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 0), 0, false}, /* 0x24d0 */
+       {RCS0, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 1), 0, false}, /* 0x24d4 */
+       {RCS0, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 2), 0, false}, /* 0x24d8 */
+       {RCS0, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 3), 0, false}, /* 0x24dc */
+       {RCS0, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 4), 0, false}, /* 0x24e0 */
+       {RCS0, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 5), 0, false}, /* 0x24e4 */
+       {RCS0, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 6), 0, false}, /* 0x24e8 */
+       {RCS0, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 7), 0, false}, /* 0x24ec */
+       {RCS0, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 8), 0, false}, /* 0x24f0 */
+       {RCS0, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 9), 0, false}, /* 0x24f4 */
+       {RCS0, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 10), 0, false}, /* 0x24f8 */
+       {RCS0, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 11), 0, false}, /* 0x24fc */
+       {RCS0, CACHE_MODE_1, 0xffff, true}, /* 0x7004 */
+       {RCS0, GEN7_GT_MODE, 0xffff, true}, /* 0x7008 */
+       {RCS0, CACHE_MODE_0_GEN7, 0xffff, true}, /* 0x7000 */
+       {RCS0, GEN7_COMMON_SLICE_CHICKEN1, 0xffff, true}, /* 0x7010 */
+       {RCS0, HDC_CHICKEN0, 0xffff, true}, /* 0x7300 */
+       {RCS0, VF_GUARDBAND, 0xffff, true}, /* 0x83a4 */
+
+       {RCS0, GEN8_PRIVATE_PAT_LO, 0, false}, /* 0x40e0 */
+       {RCS0, GEN8_PRIVATE_PAT_HI, 0, false}, /* 0x40e4 */
+       {RCS0, GEN8_CS_CHICKEN1, 0xffff, true}, /* 0x2580 */
+       {RCS0, COMMON_SLICE_CHICKEN2, 0xffff, true}, /* 0x7014 */
+       {RCS0, GEN9_CS_DEBUG_MODE1, 0xffff, false}, /* 0x20ec */
+       {RCS0, GEN8_L3SQCREG4, 0, false}, /* 0xb118 */
+       {RCS0, GEN7_HALF_SLICE_CHICKEN1, 0xffff, true}, /* 0xe100 */
+       {RCS0, HALF_SLICE_CHICKEN2, 0xffff, true}, /* 0xe180 */
+       {RCS0, HALF_SLICE_CHICKEN3, 0xffff, true}, /* 0xe184 */
+       {RCS0, GEN9_HALF_SLICE_CHICKEN5, 0xffff, true}, /* 0xe188 */
+       {RCS0, GEN9_HALF_SLICE_CHICKEN7, 0xffff, true}, /* 0xe194 */
+       {RCS0, GEN8_ROW_CHICKEN, 0xffff, true}, /* 0xe4f0 */
+       {RCS0, TRVATTL3PTRDW(0), 0, false}, /* 0x4de0 */
+       {RCS0, TRVATTL3PTRDW(1), 0, false}, /* 0x4de4 */
+       {RCS0, TRNULLDETCT, 0, false}, /* 0x4de8 */
+       {RCS0, TRINVTILEDETCT, 0, false}, /* 0x4dec */
+       {RCS0, TRVADR, 0, false}, /* 0x4df0 */
+       {RCS0, TRTTE, 0, false}, /* 0x4df4 */
+
+       {BCS0, RING_GFX_MODE(BLT_RING_BASE), 0xffff, false}, /* 0x2229c */
+       {BCS0, RING_MI_MODE(BLT_RING_BASE), 0xffff, false}, /* 0x2209c */
+       {BCS0, RING_INSTPM(BLT_RING_BASE), 0xffff, false}, /* 0x220c0 */
+       {BCS0, RING_HWSTAM(BLT_RING_BASE), 0x0, false}, /* 0x22098 */
+       {BCS0, RING_EXCC(BLT_RING_BASE), 0x0, false}, /* 0x22028 */
+
+       {VCS1, RING_EXCC(GEN8_BSD2_RING_BASE), 0xffff, false}, /* 0x1c028 */
+
+       {VECS0, RING_EXCC(VEBOX_RING_BASE), 0xffff, false}, /* 0x1a028 */
+
+       {RCS0, GEN8_HDC_CHICKEN1, 0xffff, true}, /* 0x7304 */
+       {RCS0, GEN9_CTX_PREEMPT_REG, 0x0, false}, /* 0x2248 */
+       {RCS0, GEN7_UCGCTL4, 0x0, false}, /* 0x940c */
+       {RCS0, GAMT_CHKN_BIT_REG, 0x0, false}, /* 0x4ab8 */
+
+       {RCS0, GEN9_GAMT_ECO_REG_RW_IA, 0x0, false}, /* 0x4ab0 */
+       {RCS0, GEN9_CSFE_CHICKEN1_RCS, 0xffff, false}, /* 0x20d4 */
+
+       {RCS0, GEN8_GARBCNTL, 0x0, false}, /* 0xb004 */
+       {RCS0, GEN7_FF_THREAD_MODE, 0x0, false}, /* 0x20a0 */
+       {RCS0, FF_SLICE_CS_CHICKEN2, 0xffff, false}, /* 0x20e4 */
+       {RCS0, INVALID_MMIO_REG, 0, false } /* Terminated */
 };
 
 static struct {
@@ -149,11 +149,11 @@ static void load_render_mocs(struct drm_i915_private *dev_priv)
 {
        i915_reg_t offset;
        u32 regs[] = {
-               [RCS] = 0xc800,
-               [VCS] = 0xc900,
-               [VCS2] = 0xca00,
-               [BCS] = 0xcc00,
-               [VECS] = 0xcb00,
+               [RCS0]  = 0xc800,
+               [VCS0]  = 0xc900,
+               [VCS1]  = 0xca00,
+               [BCS0]  = 0xcc00,
+               [VECS0] = 0xcb00,
        };
        int ring_id, i;
 
@@ -301,7 +301,7 @@ int intel_vgpu_restore_inhibit_context(struct intel_vgpu *vgpu,
                goto out;
 
        /* no MOCS register in context except render engine */
-       if (req->engine->id != RCS)
+       if (req->engine->id != RCS0)
                goto out;
 
        ret = restore_render_mocs_control_for_inhibit(vgpu, req);
@@ -331,11 +331,11 @@ static void handle_tlb_pending_event(struct intel_vgpu *vgpu, int ring_id)
        enum forcewake_domains fw;
        i915_reg_t reg;
        u32 regs[] = {
-               [RCS] = 0x4260,
-               [VCS] = 0x4264,
-               [VCS2] = 0x4268,
-               [BCS] = 0x426c,
-               [VECS] = 0x4270,
+               [RCS0]  = 0x4260,
+               [VCS0]  = 0x4264,
+               [VCS1]  = 0x4268,
+               [BCS0]  = 0x426c,
+               [VECS0] = 0x4270,
        };
 
        if (WARN_ON(ring_id >= ARRAY_SIZE(regs)))
@@ -353,7 +353,7 @@ static void handle_tlb_pending_event(struct intel_vgpu *vgpu, int ring_id)
         */
        fw = intel_uncore_forcewake_for_reg(dev_priv, reg,
                                            FW_REG_READ | FW_REG_WRITE);
-       if (ring_id == RCS && (INTEL_GEN(dev_priv) >= 9))
+       if (ring_id == RCS0 && INTEL_GEN(dev_priv) >= 9)
                fw |= FORCEWAKE_RENDER;
 
        intel_uncore_forcewake_get(dev_priv, fw);
@@ -378,11 +378,11 @@ static void switch_mocs(struct intel_vgpu *pre, struct intel_vgpu *next,
        u32 old_v, new_v;
 
        u32 regs[] = {
-               [RCS] = 0xc800,
-               [VCS] = 0xc900,
-               [VCS2] = 0xca00,
-               [BCS] = 0xcc00,
-               [VECS] = 0xcb00,
+               [RCS0]  = 0xc800,
+               [VCS0]  = 0xc900,
+               [VCS1]  = 0xca00,
+               [BCS0]  = 0xcc00,
+               [VECS0] = 0xcb00,
        };
        int i;
 
@@ -390,8 +390,10 @@ static void switch_mocs(struct intel_vgpu *pre, struct intel_vgpu *next,
        if (WARN_ON(ring_id >= ARRAY_SIZE(regs)))
                return;
 
-       if ((IS_KABYLAKE(dev_priv)  || IS_BROXTON(dev_priv)
-               || IS_COFFEELAKE(dev_priv)) && ring_id == RCS)
+       if (ring_id == RCS0 &&
+           (IS_KABYLAKE(dev_priv) ||
+            IS_BROXTON(dev_priv) ||
+            IS_COFFEELAKE(dev_priv)))
                return;
 
        if (!pre && !gen9_render_mocs.initialized)
@@ -414,7 +416,7 @@ static void switch_mocs(struct intel_vgpu *pre, struct intel_vgpu *next,
                offset.reg += 4;
        }
 
-       if (ring_id == RCS) {
+       if (ring_id == RCS0) {
                l3_offset.reg = 0xb020;
                for (i = 0; i < GEN9_MOCS_SIZE / 2; i++) {
                        if (pre)
@@ -492,7 +494,8 @@ static void switch_mmio(struct intel_vgpu *pre,
                         * itself.
                         */
                        if (mmio->in_context &&
-                           !is_inhibit_context(&s->shadow_ctx->__engine[ring_id]))
+                           !is_inhibit_context(intel_context_lookup(s->shadow_ctx,
+                                                                    dev_priv->engine[ring_id])))
                                continue;
 
                        if (mmio->mask)
index 1bb8f936fdaa75f2ee738bdf3235a247fac90fe8..7550e09939aea6d4b757d7f1c6e16338e10f35a1 100644 (file)
@@ -93,7 +93,7 @@ static void sr_oa_regs(struct intel_vgpu_workload *workload,
                i915_mmio_reg_offset(EU_PERF_CNTL6),
        };
 
-       if (workload->ring_id != RCS)
+       if (workload->ring_id != RCS0)
                return;
 
        if (save) {
@@ -149,7 +149,7 @@ static int populate_shadow_context(struct intel_vgpu_workload *workload)
        COPY_REG_MASKED(ctx_ctrl);
        COPY_REG(ctx_timestamp);
 
-       if (ring_id == RCS) {
+       if (ring_id == RCS0) {
                COPY_REG(bb_per_ctx_ptr);
                COPY_REG(rcs_indirect_ctx);
                COPY_REG(rcs_indirect_ctx_offset);
@@ -177,7 +177,7 @@ static int populate_shadow_context(struct intel_vgpu_workload *workload)
 
        context_page_num = context_page_num >> PAGE_SHIFT;
 
-       if (IS_BROADWELL(gvt->dev_priv) && ring_id == RCS)
+       if (IS_BROADWELL(gvt->dev_priv) && ring_id == RCS0)
                context_page_num = 19;
 
        i = 2;
@@ -440,8 +440,7 @@ int intel_gvt_scan_and_shadow_workload(struct intel_vgpu_workload *workload)
        if (ret)
                goto err_unpin;
 
-       if ((workload->ring_id == RCS) &&
-           (workload->wa_ctx.indirect_ctx.size != 0)) {
+       if (workload->ring_id == RCS0 && workload->wa_ctx.indirect_ctx.size) {
                ret = intel_gvt_scan_and_shadow_wa_ctx(&workload->wa_ctx);
                if (ret)
                        goto err_shadow;
@@ -791,7 +790,7 @@ static void update_guest_context(struct intel_vgpu_workload *workload)
        context_page_num = rq->engine->context_size;
        context_page_num = context_page_num >> PAGE_SHIFT;
 
-       if (IS_BROADWELL(gvt->dev_priv) && rq->engine->id == RCS)
+       if (IS_BROADWELL(gvt->dev_priv) && rq->engine->id == RCS0)
                context_page_num = 19;
 
        i = 2;
@@ -891,8 +890,8 @@ static void complete_current_workload(struct intel_gvt *gvt, int ring_id)
                                workload->status = 0;
                }
 
-               if (!workload->status && !(vgpu->resetting_eng &
-                                          ENGINE_MASK(ring_id))) {
+               if (!workload->status &&
+                   !(vgpu->resetting_eng & BIT(ring_id))) {
                        update_guest_context(workload);
 
                        for_each_set_bit(event, workload->pending_events,
@@ -915,7 +914,7 @@ static void complete_current_workload(struct intel_gvt *gvt, int ring_id)
 
        list_del_init(&workload->list);
 
-       if (workload->status || (vgpu->resetting_eng & ENGINE_MASK(ring_id))) {
+       if (workload->status || vgpu->resetting_eng & BIT(ring_id)) {
                /* if workload->status is not successful means HW GPU
                 * has occurred GPU hang or something wrong with i915/GVT,
                 * and GVT won't inject context switch interrupt to guest.
@@ -929,7 +928,7 @@ static void complete_current_workload(struct intel_gvt *gvt, int ring_id)
                 * cleaned up during the resetting process later, so doing
                 * the workload clean up here doesn't have any impact.
                 **/
-               intel_vgpu_clean_workloads(vgpu, ENGINE_MASK(ring_id));
+               intel_vgpu_clean_workloads(vgpu, BIT(ring_id));
        }
 
        workload->complete(workload);
@@ -1102,9 +1101,9 @@ i915_context_ppgtt_root_restore(struct intel_vgpu_submission *s)
        struct i915_hw_ppgtt *i915_ppgtt = s->shadow_ctx->ppgtt;
        int i;
 
-       if (i915_vm_is_48bit(&i915_ppgtt->vm))
+       if (i915_vm_is_4lvl(&i915_ppgtt->vm)) {
                px_dma(&i915_ppgtt->pml4) = s->i915_context_pml4;
-       else {
+       else {
                for (i = 0; i < GEN8_3LVL_PDPES; i++)
                        px_dma(i915_ppgtt->pdp.page_directory[i]) =
                                                s->i915_context_pdps[i];
@@ -1155,7 +1154,7 @@ i915_context_ppgtt_root_save(struct intel_vgpu_submission *s)
        struct i915_hw_ppgtt *i915_ppgtt = s->shadow_ctx->ppgtt;
        int i;
 
-       if (i915_vm_is_48bit(&i915_ppgtt->vm))
+       if (i915_vm_is_4lvl(&i915_ppgtt->vm))
                s->i915_context_pml4 = px_dma(&i915_ppgtt->pml4);
        else {
                for (i = 0; i < GEN8_3LVL_PDPES; i++)
@@ -1438,7 +1437,7 @@ intel_vgpu_create_workload(struct intel_vgpu *vgpu, int ring_id,
        workload->rb_start = start;
        workload->rb_ctl = ctl;
 
-       if (ring_id == RCS) {
+       if (ring_id == RCS0) {
                intel_gvt_hypervisor_read_gpa(vgpu, ring_context_gpa +
                        RING_CTX_OFF(bb_per_ctx_ptr.val), &per_ctx, 4);
                intel_gvt_hypervisor_read_gpa(vgpu, ring_context_gpa +
index 720e2b10adaa10f8ccc4c7472da4e300eb4b296f..314e40121e47e0d770fc0a6039bd8bcf4947baae 100644 (file)
@@ -44,7 +44,7 @@ void populate_pvinfo_page(struct intel_vgpu *vgpu)
        vgpu_vreg_t(vgpu, vgtif_reg(display_ready)) = 0;
        vgpu_vreg_t(vgpu, vgtif_reg(vgt_id)) = vgpu->id;
 
-       vgpu_vreg_t(vgpu, vgtif_reg(vgt_caps)) = VGT_CAPS_FULL_48BIT_PPGTT;
+       vgpu_vreg_t(vgpu, vgtif_reg(vgt_caps)) = VGT_CAPS_FULL_PPGTT;
        vgpu_vreg_t(vgpu, vgtif_reg(vgt_caps)) |= VGT_CAPS_HWSP_EMULATION;
        vgpu_vreg_t(vgpu, vgtif_reg(vgt_caps)) |= VGT_CAPS_HUGE_GTT;
 
index db7bb5bd5adde4863df871b71e1279d633ea74f6..863ae12707bac4b853655ef912da8eb2df920012 100644 (file)
@@ -6,6 +6,7 @@
 
 #include "i915_drv.h"
 #include "i915_active.h"
+#include "i915_globals.h"
 
 #define BKL(ref) (&(ref)->i915->drm.struct_mutex)
 
@@ -17,6 +18,7 @@
  * nodes from a local slab cache to hopefully reduce the fragmentation.
  */
 static struct i915_global_active {
+       struct i915_global base;
        struct kmem_cache *slab_cache;
 } global;
 
@@ -285,16 +287,27 @@ void i915_active_retire_noop(struct i915_active_request *active,
 #include "selftests/i915_active.c"
 #endif
 
+static void i915_global_active_shrink(void)
+{
+       kmem_cache_shrink(global.slab_cache);
+}
+
+static void i915_global_active_exit(void)
+{
+       kmem_cache_destroy(global.slab_cache);
+}
+
+static struct i915_global_active global = { {
+       .shrink = i915_global_active_shrink,
+       .exit = i915_global_active_exit,
+} };
+
 int __init i915_global_active_init(void)
 {
        global.slab_cache = KMEM_CACHE(active_node, SLAB_HWCACHE_ALIGN);
        if (!global.slab_cache)
                return -ENOMEM;
 
+       i915_global_register(&global.base);
        return 0;
 }
-
-void __exit i915_global_active_exit(void)
-{
-       kmem_cache_destroy(global.slab_cache);
-}
index 12b5c1d287d196de138abbcf7ce473654d7a88f9..7d758719ce39d03dc7b0741786030de8978cebf6 100644 (file)
@@ -108,19 +108,6 @@ i915_active_request_set_retire_fn(struct i915_active_request *active,
        active->retire = fn ?: i915_active_retire_noop;
 }
 
-static inline struct i915_request *
-__i915_active_request_peek(const struct i915_active_request *active)
-{
-       /*
-        * Inside the error capture (running with the driver in an unknown
-        * state), we want to bend the rules slightly (a lot).
-        *
-        * Work is in progress to make it safer, in the meantime this keeps
-        * the known issue from spamming the logs.
-        */
-       return rcu_dereference_protected(active->request, 1);
-}
-
 /**
  * i915_active_request_raw - return the active request
  * @active - the active tracker
@@ -419,7 +406,4 @@ void i915_active_fini(struct i915_active *ref);
 static inline void i915_active_fini(struct i915_active *ref) { }
 #endif
 
-int i915_global_active_init(void);
-void i915_global_active_exit(void);
-
 #endif /* _I915_ACTIVE_H_ */
index 33e8eed64423af5f00fac38c6ca88dccd7b7911e..503d548a55f7d5003b3f676e12be6da0b81e4721 100644 (file)
@@ -868,8 +868,8 @@ void intel_engine_init_cmd_parser(struct intel_engine_cs *engine)
        if (!IS_GEN(engine->i915, 7))
                return;
 
-       switch (engine->id) {
-       case RCS:
+       switch (engine->class) {
+       case RENDER_CLASS:
                if (IS_HASWELL(engine->i915)) {
                        cmd_tables = hsw_render_ring_cmds;
                        cmd_table_count =
@@ -889,12 +889,12 @@ void intel_engine_init_cmd_parser(struct intel_engine_cs *engine)
 
                engine->get_cmd_length_mask = gen7_render_get_cmd_length_mask;
                break;
-       case VCS:
+       case VIDEO_DECODE_CLASS:
                cmd_tables = gen7_video_cmds;
                cmd_table_count = ARRAY_SIZE(gen7_video_cmds);
                engine->get_cmd_length_mask = gen7_bsd_get_cmd_length_mask;
                break;
-       case BCS:
+       case COPY_ENGINE_CLASS:
                if (IS_HASWELL(engine->i915)) {
                        cmd_tables = hsw_blt_ring_cmds;
                        cmd_table_count = ARRAY_SIZE(hsw_blt_ring_cmds);
@@ -913,14 +913,14 @@ void intel_engine_init_cmd_parser(struct intel_engine_cs *engine)
 
                engine->get_cmd_length_mask = gen7_blt_get_cmd_length_mask;
                break;
-       case VECS:
+       case VIDEO_ENHANCEMENT_CLASS:
                cmd_tables = hsw_vebox_cmds;
                cmd_table_count = ARRAY_SIZE(hsw_vebox_cmds);
                /* VECS can use the same length_mask function as VCS */
                engine->get_cmd_length_mask = gen7_bsd_get_cmd_length_mask;
                break;
        default:
-               MISSING_CASE(engine->id);
+               MISSING_CASE(engine->class);
                return;
        }
 
index 0bd890c04fe4f7c911bd9bde1a79af11af08ff5c..08683dca7775032d2fb204a0163fa7b72b34f238 100644 (file)
@@ -388,12 +388,9 @@ static void print_context_stats(struct seq_file *m,
        struct i915_gem_context *ctx;
 
        list_for_each_entry(ctx, &i915->contexts.list, link) {
-               struct intel_engine_cs *engine;
-               enum intel_engine_id id;
-
-               for_each_engine(engine, i915, id) {
-                       struct intel_context *ce = to_intel_context(ctx, engine);
+               struct intel_context *ce;
 
+               list_for_each_entry(ce, &ctx->active_engines, active_link) {
                        if (ce->state)
                                per_file_stats(0, ce->state->obj, &kstats);
                        if (ce->ring)
@@ -1281,14 +1278,11 @@ static int i915_hangcheck_info(struct seq_file *m, void *unused)
        intel_wakeref_t wakeref;
        enum intel_engine_id id;
 
+       seq_printf(m, "Reset flags: %lx\n", dev_priv->gpu_error.flags);
        if (test_bit(I915_WEDGED, &dev_priv->gpu_error.flags))
-               seq_puts(m, "Wedged\n");
+               seq_puts(m, "\tWedged\n");
        if (test_bit(I915_RESET_BACKOFF, &dev_priv->gpu_error.flags))
-               seq_puts(m, "Reset in progress: struct_mutex backoff\n");
-       if (waitqueue_active(&dev_priv->gpu_error.wait_queue))
-               seq_puts(m, "Waiter holding struct mutex\n");
-       if (waitqueue_active(&dev_priv->gpu_error.reset_queue))
-               seq_puts(m, "struct_mutex blocked for reset\n");
+               seq_puts(m, "\tDevice (global) reset in progress\n");
 
        if (!i915_modparams.enable_hangcheck) {
                seq_puts(m, "Hangcheck disabled\n");
@@ -1298,10 +1292,10 @@ static int i915_hangcheck_info(struct seq_file *m, void *unused)
        with_intel_runtime_pm(dev_priv, wakeref) {
                for_each_engine(engine, dev_priv, id) {
                        acthd[id] = intel_engine_get_active_head(engine);
-                       seqno[id] = intel_engine_get_seqno(engine);
+                       seqno[id] = intel_engine_get_hangcheck_seqno(engine);
                }
 
-               intel_engine_get_instdone(dev_priv->engine[RCS], &instdone);
+               intel_engine_get_instdone(dev_priv->engine[RCS0], &instdone);
        }
 
        if (timer_pending(&dev_priv->gpu_error.hangcheck_work.timer))
@@ -1318,8 +1312,9 @@ static int i915_hangcheck_info(struct seq_file *m, void *unused)
        for_each_engine(engine, dev_priv, id) {
                seq_printf(m, "%s:\n", engine->name);
                seq_printf(m, "\tseqno = %x [current %x, last %x], %dms ago\n",
-                          engine->hangcheck.seqno, seqno[id],
-                          intel_engine_last_submit(engine),
+                          engine->hangcheck.last_seqno,
+                          seqno[id],
+                          engine->hangcheck.next_seqno,
                           jiffies_to_msecs(jiffies -
                                            engine->hangcheck.action_timestamp));
 
@@ -1327,7 +1322,7 @@ static int i915_hangcheck_info(struct seq_file *m, void *unused)
                           (long long)engine->hangcheck.acthd,
                           (long long)acthd[id]);
 
-               if (engine->id == RCS) {
+               if (engine->id == RCS0) {
                        seq_puts(m, "\tinstdone read =\n");
 
                        i915_instdone_info(dev_priv, m, &instdone);
@@ -1882,9 +1877,7 @@ static int i915_context_status(struct seq_file *m, void *unused)
 {
        struct drm_i915_private *dev_priv = node_to_i915(m->private);
        struct drm_device *dev = &dev_priv->drm;
-       struct intel_engine_cs *engine;
        struct i915_gem_context *ctx;
-       enum intel_engine_id id;
        int ret;
 
        ret = mutex_lock_interruptible(&dev->struct_mutex);
@@ -1892,6 +1885,8 @@ static int i915_context_status(struct seq_file *m, void *unused)
                return ret;
 
        list_for_each_entry(ctx, &dev_priv->contexts.list, link) {
+               struct intel_context *ce;
+
                seq_puts(m, "HW context ");
                if (!list_empty(&ctx->hw_id_link))
                        seq_printf(m, "%x [pin %u]", ctx->hw_id,
@@ -1914,11 +1909,8 @@ static int i915_context_status(struct seq_file *m, void *unused)
                seq_putc(m, ctx->remap_slice ? 'R' : 'r');
                seq_putc(m, '\n');
 
-               for_each_engine(engine, dev_priv, id) {
-                       struct intel_context *ce =
-                               to_intel_context(ctx, engine);
-
-                       seq_printf(m, "%s: ", engine->name);
+               list_for_each_entry(ce, &ctx->active_engines, active_link) {
+                       seq_printf(m, "%s: ", ce->engine->name);
                        if (ce->state)
                                describe_obj(m, ce->state->obj);
                        if (ce->ring)
@@ -2023,11 +2015,9 @@ static const char *rps_power_to_str(unsigned int power)
 static int i915_rps_boost_info(struct seq_file *m, void *data)
 {
        struct drm_i915_private *dev_priv = node_to_i915(m->private);
-       struct drm_device *dev = &dev_priv->drm;
        struct intel_rps *rps = &dev_priv->gt_pm.rps;
        u32 act_freq = rps->cur_freq;
        intel_wakeref_t wakeref;
-       struct drm_file *file;
 
        with_intel_runtime_pm_if_in_use(dev_priv, wakeref) {
                if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
@@ -2061,22 +2051,7 @@ static int i915_rps_boost_info(struct seq_file *m, void *data)
                   intel_gpu_freq(dev_priv, rps->efficient_freq),
                   intel_gpu_freq(dev_priv, rps->boost_freq));
 
-       mutex_lock(&dev->filelist_mutex);
-       list_for_each_entry_reverse(file, &dev->filelist, lhead) {
-               struct drm_i915_file_private *file_priv = file->driver_priv;
-               struct task_struct *task;
-
-               rcu_read_lock();
-               task = pid_task(file->pid, PIDTYPE_PID);
-               seq_printf(m, "%s [%d]: %d boosts\n",
-                          task ? task->comm : "<unknown>",
-                          task ? task->pid : -1,
-                          atomic_read(&file_priv->rps_client.boosts));
-               rcu_read_unlock();
-       }
-       seq_printf(m, "Kernel (anonymous) boosts: %d\n",
-                  atomic_read(&rps->boosts));
-       mutex_unlock(&dev->filelist_mutex);
+       seq_printf(m, "Wait boosts: %d\n", atomic_read(&rps->boosts));
 
        if (INTEL_GEN(dev_priv) >= 6 &&
            rps->enabled &&
@@ -2607,7 +2582,6 @@ static int
 i915_edp_psr_debug_set(void *data, u64 val)
 {
        struct drm_i915_private *dev_priv = data;
-       struct drm_modeset_acquire_ctx ctx;
        intel_wakeref_t wakeref;
        int ret;
 
@@ -2618,18 +2592,7 @@ i915_edp_psr_debug_set(void *data, u64 val)
 
        wakeref = intel_runtime_pm_get(dev_priv);
 
-       drm_modeset_acquire_init(&ctx, DRM_MODESET_ACQUIRE_INTERRUPTIBLE);
-
-retry:
-       ret = intel_psr_set_debugfs_mode(dev_priv, &ctx, val);
-       if (ret == -EDEADLK) {
-               ret = drm_modeset_backoff(&ctx);
-               if (!ret)
-                       goto retry;
-       }
-
-       drm_modeset_drop_locks(&ctx);
-       drm_modeset_acquire_fini(&ctx);
+       ret = intel_psr_debug_set(dev_priv, val);
 
        intel_runtime_pm_put(dev_priv, wakeref);
 
@@ -2686,8 +2649,7 @@ static int i915_runtime_pm_status(struct seq_file *m, void *unused)
        seq_printf(m, "Runtime power status: %s\n",
                   enableddisabled(!dev_priv->power_domains.wakeref));
 
-       seq_printf(m, "GPU idle: %s (epoch %u)\n",
-                  yesno(!dev_priv->gt.awake), dev_priv->gt.epoch);
+       seq_printf(m, "GPU idle: %s\n", yesno(!dev_priv->gt.awake));
        seq_printf(m, "IRQs disabled: %s\n",
                   yesno(!intel_irqs_enabled(dev_priv)));
 #ifdef CONFIG_PM
@@ -3123,8 +3085,7 @@ static int i915_engine_info(struct seq_file *m, void *unused)
 
        wakeref = intel_runtime_pm_get(dev_priv);
 
-       seq_printf(m, "GT awake? %s (epoch %u)\n",
-                  yesno(dev_priv->gt.awake), dev_priv->gt.epoch);
+       seq_printf(m, "GT awake? %s\n", yesno(dev_priv->gt.awake));
        seq_printf(m, "Global active requests: %d\n",
                   dev_priv->gt.active_requests);
        seq_printf(m, "CS timestamp frequency: %u kHz\n",
@@ -3211,7 +3172,7 @@ static int i915_shared_dplls_info(struct seq_file *m, void *unused)
 static int i915_wa_registers(struct seq_file *m, void *unused)
 {
        struct drm_i915_private *i915 = node_to_i915(m->private);
-       const struct i915_wa_list *wal = &i915->engine[RCS]->ctx_wa_list;
+       const struct i915_wa_list *wal = &i915->engine[RCS0]->ctx_wa_list;
        struct i915_wa *wa;
        unsigned int i;
 
@@ -3865,11 +3826,18 @@ static const struct file_operations i915_cur_wm_latency_fops = {
 static int
 i915_wedged_get(void *data, u64 *val)
 {
-       struct drm_i915_private *dev_priv = data;
+       int ret = i915_terminally_wedged(data);
 
-       *val = i915_terminally_wedged(&dev_priv->gpu_error);
-
-       return 0;
+       switch (ret) {
+       case -EIO:
+               *val = 1;
+               return 0;
+       case 0:
+               *val = 0;
+               return 0;
+       default:
+               return ret;
+       }
 }
 
 static int
@@ -3877,16 +3845,9 @@ i915_wedged_set(void *data, u64 val)
 {
        struct drm_i915_private *i915 = data;
 
-       /*
-        * There is no safeguard against this debugfs entry colliding
-        * with the hangcheck calling same i915_handle_error() in
-        * parallel, causing an explosion. For now we assume that the
-        * test harness is responsible enough not to inject gpu hangs
-        * while it is writing to 'i915_wedged'
-        */
-
-       if (i915_reset_backoff(&i915->gpu_error))
-               return -EAGAIN;
+       /* Flush any previous reset before applying for a new one */
+       wait_event(i915->gpu_error.reset_queue,
+                  !test_bit(I915_RESET_BACKOFF, &i915->gpu_error.flags));
 
        i915_handle_error(i915, val, I915_ERROR_CAPTURE,
                          "Manually set wedged engine mask = %llx", val);
@@ -3927,12 +3888,9 @@ static int
 i915_drop_caches_set(void *data, u64 val)
 {
        struct drm_i915_private *i915 = data;
-       intel_wakeref_t wakeref;
-       int ret = 0;
 
        DRM_DEBUG("Dropping caches: 0x%08llx [0x%08llx]\n",
                  val, val & DROP_ALL);
-       wakeref = intel_runtime_pm_get(i915);
 
        if (val & DROP_RESET_ACTIVE &&
            wait_for(intel_engines_are_idle(i915), I915_IDLE_ENGINES_TIMEOUT))
@@ -3941,9 +3899,11 @@ i915_drop_caches_set(void *data, u64 val)
        /* No need to check and wait for gpu resets, only libdrm auto-restarts
         * on ioctls on -EAGAIN. */
        if (val & (DROP_ACTIVE | DROP_RETIRE | DROP_RESET_SEQNO)) {
+               int ret;
+
                ret = mutex_lock_interruptible(&i915->drm.struct_mutex);
                if (ret)
-                       goto out;
+                       return ret;
 
                if (val & DROP_ACTIVE)
                        ret = i915_gem_wait_for_idle(i915,
@@ -3957,7 +3917,7 @@ i915_drop_caches_set(void *data, u64 val)
                mutex_unlock(&i915->drm.struct_mutex);
        }
 
-       if (val & DROP_RESET_ACTIVE && i915_terminally_wedged(&i915->gpu_error))
+       if (val & DROP_RESET_ACTIVE && i915_terminally_wedged(i915))
                i915_handle_error(i915, ALL_ENGINES, 0, NULL);
 
        fs_reclaim_acquire(GFP_KERNEL);
@@ -3982,10 +3942,7 @@ i915_drop_caches_set(void *data, u64 val)
        if (val & DROP_FREED)
                i915_gem_drain_freed_objects(i915);
 
-out:
-       intel_runtime_pm_put(i915, wakeref);
-
-       return ret;
+       return 0;
 }
 
 DEFINE_SIMPLE_ATTRIBUTE(i915_drop_caches_fops,
index 9df65d386d11b40349df879fec224f0990a40b62..9e380cd317dcd0faf2a7bdfa9ed78316db208969 100644 (file)
@@ -188,6 +188,11 @@ intel_pch_type(const struct drm_i915_private *dev_priv, unsigned short id)
                DRM_DEBUG_KMS("Found Cannon Lake LP PCH (CNP-LP)\n");
                WARN_ON(!IS_CANNONLAKE(dev_priv) && !IS_COFFEELAKE(dev_priv));
                return PCH_CNP;
+       case INTEL_PCH_CMP_DEVICE_ID_TYPE:
+               DRM_DEBUG_KMS("Found Comet Lake PCH (CMP)\n");
+               WARN_ON(!IS_COFFEELAKE(dev_priv));
+               /* CometPoint is CNP Compatible */
+               return PCH_CNP;
        case INTEL_PCH_ICP_DEVICE_ID_TYPE:
                DRM_DEBUG_KMS("Found Ice Lake PCH\n");
                WARN_ON(!IS_ICELAKE(dev_priv));
@@ -219,20 +224,20 @@ intel_virt_detect_pch(const struct drm_i915_private *dev_priv)
         * make an educated guess as to which PCH is really there.
         */
 
-       if (IS_GEN(dev_priv, 5))
-               id = INTEL_PCH_IBX_DEVICE_ID_TYPE;
-       else if (IS_GEN(dev_priv, 6) || IS_IVYBRIDGE(dev_priv))
-               id = INTEL_PCH_CPT_DEVICE_ID_TYPE;
+       if (IS_ICELAKE(dev_priv))
+               id = INTEL_PCH_ICP_DEVICE_ID_TYPE;
+       else if (IS_CANNONLAKE(dev_priv) || IS_COFFEELAKE(dev_priv))
+               id = INTEL_PCH_CNP_DEVICE_ID_TYPE;
+       else if (IS_KABYLAKE(dev_priv) || IS_SKYLAKE(dev_priv))
+               id = INTEL_PCH_SPT_DEVICE_ID_TYPE;
        else if (IS_HSW_ULT(dev_priv) || IS_BDW_ULT(dev_priv))
                id = INTEL_PCH_LPT_LP_DEVICE_ID_TYPE;
        else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
                id = INTEL_PCH_LPT_DEVICE_ID_TYPE;
-       else if (IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv))
-               id = INTEL_PCH_SPT_DEVICE_ID_TYPE;
-       else if (IS_COFFEELAKE(dev_priv) || IS_CANNONLAKE(dev_priv))
-               id = INTEL_PCH_CNP_DEVICE_ID_TYPE;
-       else if (IS_ICELAKE(dev_priv))
-               id = INTEL_PCH_ICP_DEVICE_ID_TYPE;
+       else if (IS_GEN(dev_priv, 6) || IS_IVYBRIDGE(dev_priv))
+               id = INTEL_PCH_CPT_DEVICE_ID_TYPE;
+       else if (IS_GEN(dev_priv, 5))
+               id = INTEL_PCH_IBX_DEVICE_ID_TYPE;
 
        if (id)
                DRM_DEBUG_KMS("Assuming PCH ID %04x\n", id);
@@ -330,16 +335,16 @@ static int i915_getparam_ioctl(struct drm_device *dev, void *data,
                value = dev_priv->overlay ? 1 : 0;
                break;
        case I915_PARAM_HAS_BSD:
-               value = !!dev_priv->engine[VCS];
+               value = !!dev_priv->engine[VCS0];
                break;
        case I915_PARAM_HAS_BLT:
-               value = !!dev_priv->engine[BCS];
+               value = !!dev_priv->engine[BCS0];
                break;
        case I915_PARAM_HAS_VEBOX:
-               value = !!dev_priv->engine[VECS];
+               value = !!dev_priv->engine[VECS0];
                break;
        case I915_PARAM_HAS_BSD2:
-               value = !!dev_priv->engine[VCS2];
+               value = !!dev_priv->engine[VCS1];
                break;
        case I915_PARAM_HAS_LLC:
                value = HAS_LLC(dev_priv);
@@ -348,10 +353,10 @@ static int i915_getparam_ioctl(struct drm_device *dev, void *data,
                value = HAS_WT(dev_priv);
                break;
        case I915_PARAM_HAS_ALIASING_PPGTT:
-               value = min_t(int, INTEL_PPGTT(dev_priv), I915_GEM_PPGTT_FULL);
+               value = INTEL_PPGTT(dev_priv);
                break;
        case I915_PARAM_HAS_SEMAPHORES:
-               value = 0;
+               value = !!(dev_priv->caps.scheduler & I915_SCHEDULER_CAP_SEMAPHORES);
                break;
        case I915_PARAM_HAS_SECURE_BATCHES:
                value = capable(CAP_SYS_ADMIN);
@@ -714,8 +719,7 @@ static int i915_load_modeset_init(struct drm_device *dev)
        return 0;
 
 cleanup_gem:
-       if (i915_gem_suspend(dev_priv))
-               DRM_ERROR("failed to idle hardware; continuing to unload!\n");
+       i915_gem_suspend(dev_priv);
        i915_gem_fini(dev_priv);
 cleanup_modeset:
        intel_modeset_cleanup(dev);
@@ -873,6 +877,7 @@ static int i915_driver_init_early(struct drm_i915_private *dev_priv)
        mutex_init(&dev_priv->av_mutex);
        mutex_init(&dev_priv->wm.wm_mutex);
        mutex_init(&dev_priv->pps_mutex);
+       mutex_init(&dev_priv->hdcp_comp_mutex);
 
        i915_memcpy_init_early(dev_priv);
        intel_runtime_pm_init_early(dev_priv);
@@ -1034,110 +1039,180 @@ static void intel_sanitize_options(struct drm_i915_private *dev_priv)
        intel_gvt_sanitize_options(dev_priv);
 }
 
-static enum dram_rank skl_get_dimm_rank(u8 size, u32 rank)
+#define DRAM_TYPE_STR(type) [INTEL_DRAM_ ## type] = #type
+
+static const char *intel_dram_type_str(enum intel_dram_type type)
 {
-       if (size == 0)
-               return I915_DRAM_RANK_INVALID;
-       if (rank == SKL_DRAM_RANK_SINGLE)
-               return I915_DRAM_RANK_SINGLE;
-       else if (rank == SKL_DRAM_RANK_DUAL)
-               return I915_DRAM_RANK_DUAL;
+       static const char * const str[] = {
+               DRAM_TYPE_STR(UNKNOWN),
+               DRAM_TYPE_STR(DDR3),
+               DRAM_TYPE_STR(DDR4),
+               DRAM_TYPE_STR(LPDDR3),
+               DRAM_TYPE_STR(LPDDR4),
+       };
 
-       return I915_DRAM_RANK_INVALID;
+       if (type >= ARRAY_SIZE(str))
+               type = INTEL_DRAM_UNKNOWN;
+
+       return str[type];
 }
 
-static bool
-skl_is_16gb_dimm(enum dram_rank rank, u8 size, u8 width)
+#undef DRAM_TYPE_STR
+
+static int intel_dimm_num_devices(const struct dram_dimm_info *dimm)
 {
-       if (rank == I915_DRAM_RANK_SINGLE && width == 8 && size == 16)
-               return true;
-       else if (rank == I915_DRAM_RANK_DUAL && width == 8 && size == 32)
-               return true;
-       else if (rank == SKL_DRAM_RANK_SINGLE && width == 16 && size == 8)
-               return true;
-       else if (rank == SKL_DRAM_RANK_DUAL && width == 16 && size == 16)
-               return true;
+       return dimm->ranks * 64 / (dimm->width ?: 1);
+}
 
-       return false;
+/* Returns total GB for the whole DIMM */
+static int skl_get_dimm_size(u16 val)
+{
+       return val & SKL_DRAM_SIZE_MASK;
 }
 
-static int
-skl_dram_get_channel_info(struct dram_channel_info *ch, u32 val)
+static int skl_get_dimm_width(u16 val)
 {
-       u32 tmp_l, tmp_s;
-       u32 s_val = val >> SKL_DRAM_S_SHIFT;
+       if (skl_get_dimm_size(val) == 0)
+               return 0;
 
-       if (!val)
-               return -EINVAL;
+       switch (val & SKL_DRAM_WIDTH_MASK) {
+       case SKL_DRAM_WIDTH_X8:
+       case SKL_DRAM_WIDTH_X16:
+       case SKL_DRAM_WIDTH_X32:
+               val = (val & SKL_DRAM_WIDTH_MASK) >> SKL_DRAM_WIDTH_SHIFT;
+               return 8 << val;
+       default:
+               MISSING_CASE(val);
+               return 0;
+       }
+}
+
+static int skl_get_dimm_ranks(u16 val)
+{
+       if (skl_get_dimm_size(val) == 0)
+               return 0;
+
+       val = (val & SKL_DRAM_RANK_MASK) >> SKL_DRAM_RANK_SHIFT;
+
+       return val + 1;
+}
 
-       tmp_l = val & SKL_DRAM_SIZE_MASK;
-       tmp_s = s_val & SKL_DRAM_SIZE_MASK;
+/* Returns total GB for the whole DIMM */
+static int cnl_get_dimm_size(u16 val)
+{
+       return (val & CNL_DRAM_SIZE_MASK) / 2;
+}
+
+static int cnl_get_dimm_width(u16 val)
+{
+       if (cnl_get_dimm_size(val) == 0)
+               return 0;
 
-       if (tmp_l == 0 && tmp_s == 0)
+       switch (val & CNL_DRAM_WIDTH_MASK) {
+       case CNL_DRAM_WIDTH_X8:
+       case CNL_DRAM_WIDTH_X16:
+       case CNL_DRAM_WIDTH_X32:
+               val = (val & CNL_DRAM_WIDTH_MASK) >> CNL_DRAM_WIDTH_SHIFT;
+               return 8 << val;
+       default:
+               MISSING_CASE(val);
+               return 0;
+       }
+}
+
+static int cnl_get_dimm_ranks(u16 val)
+{
+       if (cnl_get_dimm_size(val) == 0)
+               return 0;
+
+       val = (val & CNL_DRAM_RANK_MASK) >> CNL_DRAM_RANK_SHIFT;
+
+       return val + 1;
+}
+
+static bool
+skl_is_16gb_dimm(const struct dram_dimm_info *dimm)
+{
+       /* Convert total GB to Gb per DRAM device */
+       return 8 * dimm->size / (intel_dimm_num_devices(dimm) ?: 1) == 16;
+}
+
+static void
+skl_dram_get_dimm_info(struct drm_i915_private *dev_priv,
+                      struct dram_dimm_info *dimm,
+                      int channel, char dimm_name, u16 val)
+{
+       if (INTEL_GEN(dev_priv) >= 10) {
+               dimm->size = cnl_get_dimm_size(val);
+               dimm->width = cnl_get_dimm_width(val);
+               dimm->ranks = cnl_get_dimm_ranks(val);
+       } else {
+               dimm->size = skl_get_dimm_size(val);
+               dimm->width = skl_get_dimm_width(val);
+               dimm->ranks = skl_get_dimm_ranks(val);
+       }
+
+       DRM_DEBUG_KMS("CH%u DIMM %c size: %u GB, width: X%u, ranks: %u, 16Gb DIMMs: %s\n",
+                     channel, dimm_name, dimm->size, dimm->width, dimm->ranks,
+                     yesno(skl_is_16gb_dimm(dimm)));
+}
+
+static int
+skl_dram_get_channel_info(struct drm_i915_private *dev_priv,
+                         struct dram_channel_info *ch,
+                         int channel, u32 val)
+{
+       skl_dram_get_dimm_info(dev_priv, &ch->dimm_l,
+                              channel, 'L', val & 0xffff);
+       skl_dram_get_dimm_info(dev_priv, &ch->dimm_s,
+                              channel, 'S', val >> 16);
+
+       if (ch->dimm_l.size == 0 && ch->dimm_s.size == 0) {
+               DRM_DEBUG_KMS("CH%u not populated\n", channel);
                return -EINVAL;
+       }
 
-       ch->l_info.size = tmp_l;
-       ch->s_info.size = tmp_s;
-
-       tmp_l = (val & SKL_DRAM_WIDTH_MASK) >> SKL_DRAM_WIDTH_SHIFT;
-       tmp_s = (s_val & SKL_DRAM_WIDTH_MASK) >> SKL_DRAM_WIDTH_SHIFT;
-       ch->l_info.width = (1 << tmp_l) * 8;
-       ch->s_info.width = (1 << tmp_s) * 8;
-
-       tmp_l = val & SKL_DRAM_RANK_MASK;
-       tmp_s = s_val & SKL_DRAM_RANK_MASK;
-       ch->l_info.rank = skl_get_dimm_rank(ch->l_info.size, tmp_l);
-       ch->s_info.rank = skl_get_dimm_rank(ch->s_info.size, tmp_s);
-
-       if (ch->l_info.rank == I915_DRAM_RANK_DUAL ||
-           ch->s_info.rank == I915_DRAM_RANK_DUAL)
-               ch->rank = I915_DRAM_RANK_DUAL;
-       else if (ch->l_info.rank == I915_DRAM_RANK_SINGLE &&
-                ch->s_info.rank == I915_DRAM_RANK_SINGLE)
-               ch->rank = I915_DRAM_RANK_DUAL;
+       if (ch->dimm_l.ranks == 2 || ch->dimm_s.ranks == 2)
+               ch->ranks = 2;
+       else if (ch->dimm_l.ranks == 1 && ch->dimm_s.ranks == 1)
+               ch->ranks = 2;
        else
-               ch->rank = I915_DRAM_RANK_SINGLE;
+               ch->ranks = 1;
 
-       ch->is_16gb_dimm = skl_is_16gb_dimm(ch->l_info.rank, ch->l_info.size,
-                                           ch->l_info.width) ||
-                          skl_is_16gb_dimm(ch->s_info.rank, ch->s_info.size,
-                                           ch->s_info.width);
+       ch->is_16gb_dimm =
+               skl_is_16gb_dimm(&ch->dimm_l) ||
+               skl_is_16gb_dimm(&ch->dimm_s);
 
-       DRM_DEBUG_KMS("(size:width:rank) L(%dGB:X%d:%s) S(%dGB:X%d:%s)\n",
-                     ch->l_info.size, ch->l_info.width,
-                     ch->l_info.rank ? "dual" : "single",
-                     ch->s_info.size, ch->s_info.width,
-                     ch->s_info.rank ? "dual" : "single");
+       DRM_DEBUG_KMS("CH%u ranks: %u, 16Gb DIMMs: %s\n",
+                     channel, ch->ranks, yesno(ch->is_16gb_dimm));
 
        return 0;
 }
 
 static bool
-intel_is_dram_symmetric(u32 val_ch0, u32 val_ch1,
-                       struct dram_channel_info *ch0)
+intel_is_dram_symmetric(const struct dram_channel_info *ch0,
+                       const struct dram_channel_info *ch1)
 {
-       return (val_ch0 == val_ch1 &&
-               (ch0->s_info.size == 0 ||
-                (ch0->l_info.size == ch0->s_info.size &&
-                 ch0->l_info.width == ch0->s_info.width &&
-                 ch0->l_info.rank == ch0->s_info.rank)));
+       return !memcmp(ch0, ch1, sizeof(*ch0)) &&
+               (ch0->dimm_s.size == 0 ||
+                !memcmp(&ch0->dimm_l, &ch0->dimm_s, sizeof(ch0->dimm_l)));
 }
 
 static int
 skl_dram_get_channels_info(struct drm_i915_private *dev_priv)
 {
        struct dram_info *dram_info = &dev_priv->dram_info;
-       struct dram_channel_info ch0, ch1;
-       u32 val_ch0, val_ch1;
+       struct dram_channel_info ch0 = {}, ch1 = {};
+       u32 val;
        int ret;
 
-       val_ch0 = I915_READ(SKL_MAD_DIMM_CH0_0_0_0_MCHBAR_MCMAIN);
-       ret = skl_dram_get_channel_info(&ch0, val_ch0);
+       val = I915_READ(SKL_MAD_DIMM_CH0_0_0_0_MCHBAR_MCMAIN);
+       ret = skl_dram_get_channel_info(dev_priv, &ch0, 0, val);
        if (ret == 0)
                dram_info->num_channels++;
 
-       val_ch1 = I915_READ(SKL_MAD_DIMM_CH1_0_0_0_MCHBAR_MCMAIN);
-       ret = skl_dram_get_channel_info(&ch1, val_ch1);
+       val = I915_READ(SKL_MAD_DIMM_CH1_0_0_0_MCHBAR_MCMAIN);
+       ret = skl_dram_get_channel_info(dev_priv, &ch1, 1, val);
        if (ret == 0)
                dram_info->num_channels++;
 
@@ -1151,28 +1226,47 @@ skl_dram_get_channels_info(struct drm_i915_private *dev_priv)
         * will be same as if single rank memory, so consider single rank
         * memory.
         */
-       if (ch0.rank == I915_DRAM_RANK_SINGLE ||
-           ch1.rank == I915_DRAM_RANK_SINGLE)
-               dram_info->rank = I915_DRAM_RANK_SINGLE;
+       if (ch0.ranks == 1 || ch1.ranks == 1)
+               dram_info->ranks = 1;
        else
-               dram_info->rank = max(ch0.rank, ch1.rank);
+               dram_info->ranks = max(ch0.ranks, ch1.ranks);
 
-       if (dram_info->rank == I915_DRAM_RANK_INVALID) {
+       if (dram_info->ranks == 0) {
                DRM_INFO("couldn't get memory rank information\n");
                return -EINVAL;
        }
 
        dram_info->is_16gb_dimm = ch0.is_16gb_dimm || ch1.is_16gb_dimm;
 
-       dev_priv->dram_info.symmetric_memory = intel_is_dram_symmetric(val_ch0,
-                                                                      val_ch1,
-                                                                      &ch0);
+       dram_info->symmetric_memory = intel_is_dram_symmetric(&ch0, &ch1);
 
-       DRM_DEBUG_KMS("memory configuration is %sSymmetric memory\n",
-                     dev_priv->dram_info.symmetric_memory ? "" : "not ");
+       DRM_DEBUG_KMS("Memory configuration is symmetric? %s\n",
+                     yesno(dram_info->symmetric_memory));
        return 0;
 }
 
+static enum intel_dram_type
+skl_get_dram_type(struct drm_i915_private *dev_priv)
+{
+       u32 val;
+
+       val = I915_READ(SKL_MAD_INTER_CHANNEL_0_0_0_MCHBAR_MCMAIN);
+
+       switch (val & SKL_DRAM_DDR_TYPE_MASK) {
+       case SKL_DRAM_DDR_TYPE_DDR3:
+               return INTEL_DRAM_DDR3;
+       case SKL_DRAM_DDR_TYPE_DDR4:
+               return INTEL_DRAM_DDR4;
+       case SKL_DRAM_DDR_TYPE_LPDDR3:
+               return INTEL_DRAM_LPDDR3;
+       case SKL_DRAM_DDR_TYPE_LPDDR4:
+               return INTEL_DRAM_LPDDR4;
+       default:
+               MISSING_CASE(val);
+               return INTEL_DRAM_UNKNOWN;
+       }
+}
+
 static int
 skl_get_dram_info(struct drm_i915_private *dev_priv)
 {
@@ -1180,6 +1274,9 @@ skl_get_dram_info(struct drm_i915_private *dev_priv)
        u32 mem_freq_khz, val;
        int ret;
 
+       dram_info->type = skl_get_dram_type(dev_priv);
+       DRM_DEBUG_KMS("DRAM type: %s\n", intel_dram_type_str(dram_info->type));
+
        ret = skl_dram_get_channels_info(dev_priv);
        if (ret)
                return ret;
@@ -1200,6 +1297,85 @@ skl_get_dram_info(struct drm_i915_private *dev_priv)
        return 0;
 }
 
+/* Returns Gb per DRAM device */
+static int bxt_get_dimm_size(u32 val)
+{
+       switch (val & BXT_DRAM_SIZE_MASK) {
+       case BXT_DRAM_SIZE_4GBIT:
+               return 4;
+       case BXT_DRAM_SIZE_6GBIT:
+               return 6;
+       case BXT_DRAM_SIZE_8GBIT:
+               return 8;
+       case BXT_DRAM_SIZE_12GBIT:
+               return 12;
+       case BXT_DRAM_SIZE_16GBIT:
+               return 16;
+       default:
+               MISSING_CASE(val);
+               return 0;
+       }
+}
+
+static int bxt_get_dimm_width(u32 val)
+{
+       if (!bxt_get_dimm_size(val))
+               return 0;
+
+       val = (val & BXT_DRAM_WIDTH_MASK) >> BXT_DRAM_WIDTH_SHIFT;
+
+       return 8 << val;
+}
+
+static int bxt_get_dimm_ranks(u32 val)
+{
+       if (!bxt_get_dimm_size(val))
+               return 0;
+
+       switch (val & BXT_DRAM_RANK_MASK) {
+       case BXT_DRAM_RANK_SINGLE:
+               return 1;
+       case BXT_DRAM_RANK_DUAL:
+               return 2;
+       default:
+               MISSING_CASE(val);
+               return 0;
+       }
+}
+
+static enum intel_dram_type bxt_get_dimm_type(u32 val)
+{
+       if (!bxt_get_dimm_size(val))
+               return INTEL_DRAM_UNKNOWN;
+
+       switch (val & BXT_DRAM_TYPE_MASK) {
+       case BXT_DRAM_TYPE_DDR3:
+               return INTEL_DRAM_DDR3;
+       case BXT_DRAM_TYPE_LPDDR3:
+               return INTEL_DRAM_LPDDR3;
+       case BXT_DRAM_TYPE_DDR4:
+               return INTEL_DRAM_DDR4;
+       case BXT_DRAM_TYPE_LPDDR4:
+               return INTEL_DRAM_LPDDR4;
+       default:
+               MISSING_CASE(val);
+               return INTEL_DRAM_UNKNOWN;
+       }
+}
+
+static void bxt_get_dimm_info(struct dram_dimm_info *dimm,
+                             u32 val)
+{
+       dimm->width = bxt_get_dimm_width(val);
+       dimm->ranks = bxt_get_dimm_ranks(val);
+
+       /*
+        * Size in register is Gb per DRAM device. Convert to total
+        * GB to match the way we report this for non-LP platforms.
+        */
+       dimm->size = bxt_get_dimm_size(val) * intel_dimm_num_devices(dimm) / 8;
+}
+
 static int
 bxt_get_dram_info(struct drm_i915_private *dev_priv)
 {
@@ -1228,57 +1404,44 @@ bxt_get_dram_info(struct drm_i915_private *dev_priv)
         * Now read each DUNIT8/9/10/11 to check the rank of each dimms.
         */
        for (i = BXT_D_CR_DRP0_DUNIT_START; i <= BXT_D_CR_DRP0_DUNIT_END; i++) {
-               u8 size, width;
-               enum dram_rank rank;
-               u32 tmp;
+               struct dram_dimm_info dimm;
+               enum intel_dram_type type;
 
                val = I915_READ(BXT_D_CR_DRP0_DUNIT(i));
                if (val == 0xFFFFFFFF)
                        continue;
 
                dram_info->num_channels++;
-               tmp = val & BXT_DRAM_RANK_MASK;
-
-               if (tmp == BXT_DRAM_RANK_SINGLE)
-                       rank = I915_DRAM_RANK_SINGLE;
-               else if (tmp == BXT_DRAM_RANK_DUAL)
-                       rank = I915_DRAM_RANK_DUAL;
-               else
-                       rank = I915_DRAM_RANK_INVALID;
-
-               tmp = val & BXT_DRAM_SIZE_MASK;
-               if (tmp == BXT_DRAM_SIZE_4GB)
-                       size = 4;
-               else if (tmp == BXT_DRAM_SIZE_6GB)
-                       size = 6;
-               else if (tmp == BXT_DRAM_SIZE_8GB)
-                       size = 8;
-               else if (tmp == BXT_DRAM_SIZE_12GB)
-                       size = 12;
-               else if (tmp == BXT_DRAM_SIZE_16GB)
-                       size = 16;
-               else
-                       size = 0;
-
-               tmp = (val & BXT_DRAM_WIDTH_MASK) >> BXT_DRAM_WIDTH_SHIFT;
-               width = (1 << tmp) * 8;
-               DRM_DEBUG_KMS("dram size:%dGB width:X%d rank:%s\n", size,
-                             width, rank == I915_DRAM_RANK_SINGLE ? "single" :
-                             rank == I915_DRAM_RANK_DUAL ? "dual" : "unknown");
+
+               bxt_get_dimm_info(&dimm, val);
+               type = bxt_get_dimm_type(val);
+
+               WARN_ON(type != INTEL_DRAM_UNKNOWN &&
+                       dram_info->type != INTEL_DRAM_UNKNOWN &&
+                       dram_info->type != type);
+
+               DRM_DEBUG_KMS("CH%u DIMM size: %u GB, width: X%u, ranks: %u, type: %s\n",
+                             i - BXT_D_CR_DRP0_DUNIT_START,
+                             dimm.size, dimm.width, dimm.ranks,
+                             intel_dram_type_str(type));
 
                /*
                 * If any of the channel is single rank channel,
                 * worst case output will be same as if single rank
                 * memory, so consider single rank memory.
                 */
-               if (dram_info->rank == I915_DRAM_RANK_INVALID)
-                       dram_info->rank = rank;
-               else if (rank == I915_DRAM_RANK_SINGLE)
-                       dram_info->rank = I915_DRAM_RANK_SINGLE;
+               if (dram_info->ranks == 0)
+                       dram_info->ranks = dimm.ranks;
+               else if (dimm.ranks == 1)
+                       dram_info->ranks = 1;
+
+               if (type != INTEL_DRAM_UNKNOWN)
+                       dram_info->type = type;
        }
 
-       if (dram_info->rank == I915_DRAM_RANK_INVALID) {
-               DRM_INFO("couldn't get memory rank information\n");
+       if (dram_info->type == INTEL_DRAM_UNKNOWN ||
+           dram_info->ranks == 0) {
+               DRM_INFO("couldn't get memory information\n");
                return -EINVAL;
        }
 
@@ -1290,14 +1453,8 @@ static void
 intel_get_dram_info(struct drm_i915_private *dev_priv)
 {
        struct dram_info *dram_info = &dev_priv->dram_info;
-       char bandwidth_str[32];
        int ret;
 
-       dram_info->valid = false;
-       dram_info->rank = I915_DRAM_RANK_INVALID;
-       dram_info->bandwidth_kbps = 0;
-       dram_info->num_channels = 0;
-
        /*
         * Assume 16Gb DIMMs are present until proven otherwise.
         * This is only used for the level 0 watermark latency
@@ -1305,28 +1462,22 @@ intel_get_dram_info(struct drm_i915_private *dev_priv)
         */
        dram_info->is_16gb_dimm = !IS_GEN9_LP(dev_priv);
 
-       if (INTEL_GEN(dev_priv) < 9 || IS_GEMINILAKE(dev_priv))
+       if (INTEL_GEN(dev_priv) < 9)
                return;
 
-       /* Need to calculate bandwidth only for Gen9 */
-       if (IS_BROXTON(dev_priv))
+       if (IS_GEN9_LP(dev_priv))
                ret = bxt_get_dram_info(dev_priv);
-       else if (IS_GEN(dev_priv, 9))
-               ret = skl_get_dram_info(dev_priv);
        else
-               ret = skl_dram_get_channels_info(dev_priv);
+               ret = skl_get_dram_info(dev_priv);
        if (ret)
                return;
 
-       if (dram_info->bandwidth_kbps)
-               sprintf(bandwidth_str, "%d KBps", dram_info->bandwidth_kbps);
-       else
-               sprintf(bandwidth_str, "unknown");
-       DRM_DEBUG_KMS("DRAM bandwidth:%s, total-channels: %u\n",
-                     bandwidth_str, dram_info->num_channels);
-       DRM_DEBUG_KMS("DRAM rank: %s rank 16GB-dimm:%s\n",
-                     (dram_info->rank == I915_DRAM_RANK_DUAL) ?
-                     "dual" : "single", yesno(dram_info->is_16gb_dimm));
+       DRM_DEBUG_KMS("DRAM bandwidth: %u kBps, channels: %u\n",
+                     dram_info->bandwidth_kbps,
+                     dram_info->num_channels);
+
+       DRM_DEBUG_KMS("DRAM ranks: %u, 16Gb DIMMs: %s\n",
+                     dram_info->ranks, yesno(dram_info->is_16gb_dimm));
 }
 
 /**
@@ -1348,7 +1499,7 @@ static int i915_driver_init_hw(struct drm_i915_private *dev_priv)
 
        if (HAS_PPGTT(dev_priv)) {
                if (intel_vgpu_active(dev_priv) &&
-                   !intel_vgpu_has_full_48bit_ppgtt(dev_priv)) {
+                   !intel_vgpu_has_full_ppgtt(dev_priv)) {
                        i915_report_error(dev_priv,
                                          "incompatible vGPU found, support for isolated ppGTT required\n");
                        return -ENXIO;
@@ -1753,8 +1904,7 @@ void i915_driver_unload(struct drm_device *dev)
        /* Flush any external code that still may be under the RCU lock */
        synchronize_rcu();
 
-       if (i915_gem_suspend(dev_priv))
-               DRM_ERROR("failed to idle hardware; continuing to unload!\n");
+       i915_gem_suspend(dev_priv);
 
        drm_atomic_helper_shutdown(dev);
 
@@ -1862,7 +2012,6 @@ static bool suspend_to_idle(struct drm_i915_private *dev_priv)
 static int i915_drm_prepare(struct drm_device *dev)
 {
        struct drm_i915_private *i915 = to_i915(dev);
-       int err;
 
        /*
         * NB intel_display_suspend() may issue new requests after we've
@@ -1870,12 +2019,9 @@ static int i915_drm_prepare(struct drm_device *dev)
         * split out that work and pull it forward so that after point,
         * the GPU is not woken again.
         */
-       err = i915_gem_suspend(i915);
-       if (err)
-               dev_err(&i915->drm.pdev->dev,
-                       "GEM idle failed, suspend/resume might fail\n");
+       i915_gem_suspend(i915);
 
-       return err;
+       return 0;
 }
 
 static int i915_drm_suspend(struct drm_device *dev)
index 9adc7bb9e69ccfec96e468f95435b83e084ffcce..363b2d3e4d50b214633f2be811f04c5d8d2d6892 100644 (file)
@@ -55,6 +55,7 @@
 #include <drm/drm_util.h>
 #include <drm/drm_dsc.h>
 #include <drm/drm_connector.h>
+#include <drm/i915_mei_hdcp_interface.h>
 
 #include "i915_fixed.h"
 #include "i915_params.h"
@@ -91,8 +92,8 @@
 
 #define DRIVER_NAME            "i915"
 #define DRIVER_DESC            "Intel Graphics"
-#define DRIVER_DATE            "20190207"
-#define DRIVER_TIMESTAMP       1549572331
+#define DRIVER_DATE            "20190320"
+#define DRIVER_TIMESTAMP       1553069028
 
 /* Use I915_STATE_WARN(x) and I915_STATE_WARN_ON() (rather than WARN() and
  * WARN_ON()) for hw state sanity checks to check for unexpected conditions
@@ -217,10 +218,6 @@ struct drm_i915_file_private {
        } mm;
        struct idr context_idr;
 
-       struct intel_rps_client {
-               atomic_t boosts;
-       } rps_client;
-
        unsigned int bsd_engine;
 
 /*
@@ -508,7 +505,7 @@ struct i915_psr {
 
        u32 debug;
        bool sink_support;
-       bool prepared, enabled;
+       bool enabled;
        struct intel_dp *dp;
        enum pipe pipe;
        bool active;
@@ -526,16 +523,22 @@ struct i915_psr {
        u16 su_x_granularity;
 };
 
+/*
+ * Sorted by south display engine compatibility.
+ * If the new PCH comes with a south display engine that is not
+ * inherited from the latest item, please do not add it to the
+ * end. Instead, add it right after its "parent" PCH.
+ */
 enum intel_pch {
+       PCH_NOP = -1,   /* PCH without south display */
        PCH_NONE = 0,   /* No PCH present */
        PCH_IBX,        /* Ibexpeak PCH */
        PCH_CPT,        /* Cougarpoint/Pantherpoint PCH */
        PCH_LPT,        /* Lynxpoint/Wildcatpoint PCH */
        PCH_SPT,        /* Sunrisepoint PCH */
        PCH_KBP,        /* Kaby Lake PCH */
-       PCH_CNP,        /* Cannon Lake PCH */
+       PCH_CNP,        /* Cannon/Comet Lake PCH */
        PCH_ICP,        /* Ice Lake PCH */
-       PCH_NOP,        /* PCH without south display */
 };
 
 enum intel_sbi_destination {
@@ -1009,6 +1012,7 @@ struct intel_vbt_data {
                enum psr_lines_to_wait lines_to_wait;
                int tp1_wakeup_time_us;
                int tp2_tp3_wakeup_time_us;
+               int psr2_tp2_tp3_wakeup_time_us;
        } psr;
 
        struct {
@@ -1130,6 +1134,7 @@ struct skl_wm_level {
        u16 plane_res_b;
        u8 plane_res_l;
        bool plane_en;
+       bool ignore_lines;
 };
 
 /* Stores plane specific WM parameters */
@@ -1200,7 +1205,11 @@ enum intel_pipe_crc_source {
        INTEL_PIPE_CRC_SOURCE_NONE,
        INTEL_PIPE_CRC_SOURCE_PLANE1,
        INTEL_PIPE_CRC_SOURCE_PLANE2,
-       INTEL_PIPE_CRC_SOURCE_PF,
+       INTEL_PIPE_CRC_SOURCE_PLANE3,
+       INTEL_PIPE_CRC_SOURCE_PLANE4,
+       INTEL_PIPE_CRC_SOURCE_PLANE5,
+       INTEL_PIPE_CRC_SOURCE_PLANE6,
+       INTEL_PIPE_CRC_SOURCE_PLANE7,
        INTEL_PIPE_CRC_SOURCE_PIPE,
        /* TV/DP on pre-gen5/vlv can't use the pipe source. */
        INTEL_PIPE_CRC_SOURCE_TV,
@@ -1468,13 +1477,6 @@ struct intel_cdclk_state {
 struct drm_i915_private {
        struct drm_device drm;
 
-       struct kmem_cache *objects;
-       struct kmem_cache *vmas;
-       struct kmem_cache *luts;
-       struct kmem_cache *requests;
-       struct kmem_cache *dependencies;
-       struct kmem_cache *priorities;
-
        const struct intel_device_info __info; /* Use INTEL_INFO() to access. */
        struct intel_runtime_info __runtime; /* Use RUNTIME_INFO() to access. */
        struct intel_driver_caps caps;
@@ -1831,13 +1833,16 @@ struct drm_i915_private {
                bool valid;
                bool is_16gb_dimm;
                u8 num_channels;
-               enum dram_rank {
-                       I915_DRAM_RANK_INVALID = 0,
-                       I915_DRAM_RANK_SINGLE,
-                       I915_DRAM_RANK_DUAL
-               } rank;
+               u8 ranks;
                u32 bandwidth_kbps;
                bool symmetric_memory;
+               enum intel_dram_type {
+                       INTEL_DRAM_UNKNOWN,
+                       INTEL_DRAM_DDR3,
+                       INTEL_DRAM_DDR4,
+                       INTEL_DRAM_LPDDR3,
+                       INTEL_DRAM_LPDDR4
+               } type;
        } dram_info;
 
        struct i915_runtime_pm runtime_pm;
@@ -1997,6 +2002,7 @@ struct drm_i915_private {
                        struct list_head hwsp_free_list;
                } timelines;
 
+               intel_engine_mask_t active_engines;
                struct list_head active_rings;
                struct list_head closed_vma;
                u32 active_requests;
@@ -2010,12 +2016,6 @@ struct drm_i915_private {
                 */
                intel_wakeref_t awake;
 
-               /**
-                * The number of times we have woken up.
-                */
-               unsigned int epoch;
-#define I915_EPOCH_INVALID 0
-
                /**
                 * We leave the user IRQ off as much as possible,
                 * but this means that requests will finish and never
@@ -2055,18 +2055,25 @@ struct drm_i915_private {
 
        struct i915_pmu pmu;
 
+       struct i915_hdcp_comp_master *hdcp_master;
+       bool hdcp_comp_added;
+
+       /* Mutex to protect the above hdcp component related values. */
+       struct mutex hdcp_comp_mutex;
+
        /*
         * NOTE: This is the dri1/ums dungeon, don't add stuff here. Your patch
         * will be rejected. Instead look for a better place.
         */
 };
 
+struct dram_dimm_info {
+       u8 size, width, ranks;
+};
+
 struct dram_channel_info {
-       struct info {
-               u8 size, width;
-               enum dram_rank rank;
-       } l_info, s_info;
-       enum dram_rank rank;
+       struct dram_dimm_info dimm_l, dimm_s;
+       u8 ranks;
        bool is_16gb_dimm;
 };
 
@@ -2104,7 +2111,7 @@ static inline struct drm_i915_private *huc_to_i915(struct intel_huc *huc)
 
 /* Iterator over subset of engines selected by mask */
 #define for_each_engine_masked(engine__, dev_priv__, mask__, tmp__) \
-       for ((tmp__) = (mask__) & INTEL_INFO(dev_priv__)->ring_mask; \
+       for ((tmp__) = (mask__) & INTEL_INFO(dev_priv__)->engine_mask; \
             (tmp__) ? \
             ((engine__) = (dev_priv__)->engine[__mask_next_bit(tmp__)]), 1 : \
             0;)
@@ -2425,24 +2432,8 @@ static inline unsigned int i915_sg_segment_size(void)
 #define IS_GEN9_LP(dev_priv)   (IS_GEN(dev_priv, 9) && IS_LP(dev_priv))
 #define IS_GEN9_BC(dev_priv)   (IS_GEN(dev_priv, 9) && !IS_LP(dev_priv))
 
-#define ENGINE_MASK(id)        BIT(id)
-#define RENDER_RING    ENGINE_MASK(RCS)
-#define BSD_RING       ENGINE_MASK(VCS)
-#define BLT_RING       ENGINE_MASK(BCS)
-#define VEBOX_RING     ENGINE_MASK(VECS)
-#define BSD2_RING      ENGINE_MASK(VCS2)
-#define BSD3_RING      ENGINE_MASK(VCS3)
-#define BSD4_RING      ENGINE_MASK(VCS4)
-#define VEBOX2_RING    ENGINE_MASK(VECS2)
-#define ALL_ENGINES    (~0)
-
-#define HAS_ENGINE(dev_priv, id) \
-       (!!(INTEL_INFO(dev_priv)->ring_mask & ENGINE_MASK(id)))
-
-#define HAS_BSD(dev_priv)      HAS_ENGINE(dev_priv, VCS)
-#define HAS_BSD2(dev_priv)     HAS_ENGINE(dev_priv, VCS2)
-#define HAS_BLT(dev_priv)      HAS_ENGINE(dev_priv, BCS)
-#define HAS_VEBOX(dev_priv)    HAS_ENGINE(dev_priv, VECS)
+#define ALL_ENGINES    (~0u)
+#define HAS_ENGINE(dev_priv, id) (INTEL_INFO(dev_priv)->engine_mask & BIT(id))
 
 #define HAS_LLC(dev_priv)      (INTEL_INFO(dev_priv)->has_llc)
 #define HAS_SNOOP(dev_priv)    (INTEL_INFO(dev_priv)->has_snoop)
@@ -2461,13 +2452,11 @@ static inline unsigned int i915_sg_segment_size(void)
 
 #define HAS_EXECLISTS(dev_priv) HAS_LOGICAL_RING_CONTEXTS(dev_priv)
 
-#define INTEL_PPGTT(dev_priv) (INTEL_INFO(dev_priv)->ppgtt)
+#define INTEL_PPGTT(dev_priv) (INTEL_INFO(dev_priv)->ppgtt_type)
 #define HAS_PPGTT(dev_priv) \
        (INTEL_PPGTT(dev_priv) != INTEL_PPGTT_NONE)
 #define HAS_FULL_PPGTT(dev_priv) \
        (INTEL_PPGTT(dev_priv) >= INTEL_PPGTT_FULL)
-#define HAS_FULL_48BIT_PPGTT(dev_priv) \
-       (INTEL_PPGTT(dev_priv) >= INTEL_PPGTT_FULL_4LVL)
 
 #define HAS_PAGE_SIZES(dev_priv, sizes) ({ \
        GEM_BUG_ON((sizes) == 0); \
@@ -2511,6 +2500,7 @@ static inline unsigned int i915_sg_segment_size(void)
 #define HAS_DDI(dev_priv)               (INTEL_INFO(dev_priv)->display.has_ddi)
 #define HAS_FPGA_DBG_UNCLAIMED(dev_priv) (INTEL_INFO(dev_priv)->has_fpga_dbg)
 #define HAS_PSR(dev_priv)               (INTEL_INFO(dev_priv)->display.has_psr)
+#define HAS_TRANSCODER_EDP(dev_priv)    (INTEL_INFO(dev_priv)->trans_offsets[TRANSCODER_EDP] != 0)
 
 #define HAS_RC6(dev_priv)               (INTEL_INFO(dev_priv)->has_rc6)
 #define HAS_RC6p(dev_priv)              (INTEL_INFO(dev_priv)->has_rc6p)
@@ -2557,6 +2547,7 @@ static inline unsigned int i915_sg_segment_size(void)
 #define INTEL_PCH_KBP_DEVICE_ID_TYPE           0xA280
 #define INTEL_PCH_CNP_DEVICE_ID_TYPE           0xA300
 #define INTEL_PCH_CNP_LP_DEVICE_ID_TYPE                0x9D80
+#define INTEL_PCH_CMP_DEVICE_ID_TYPE           0x0280
 #define INTEL_PCH_ICP_DEVICE_ID_TYPE           0x3480
 #define INTEL_PCH_P2X_DEVICE_ID_TYPE           0x7100
 #define INTEL_PCH_P3X_DEVICE_ID_TYPE           0x7000
@@ -2566,8 +2557,6 @@ static inline unsigned int i915_sg_segment_size(void)
 #define INTEL_PCH_ID(dev_priv) ((dev_priv)->pch_id)
 #define HAS_PCH_ICP(dev_priv) (INTEL_PCH_TYPE(dev_priv) == PCH_ICP)
 #define HAS_PCH_CNP(dev_priv) (INTEL_PCH_TYPE(dev_priv) == PCH_CNP)
-#define HAS_PCH_CNP_LP(dev_priv) \
-       (INTEL_PCH_ID(dev_priv) == INTEL_PCH_CNP_LP_DEVICE_ID_TYPE)
 #define HAS_PCH_KBP(dev_priv) (INTEL_PCH_TYPE(dev_priv) == PCH_KBP)
 #define HAS_PCH_SPT(dev_priv) (INTEL_PCH_TYPE(dev_priv) == PCH_SPT)
 #define HAS_PCH_LPT(dev_priv) (INTEL_PCH_TYPE(dev_priv) == PCH_LPT)
@@ -2799,8 +2788,6 @@ void i915_gem_load_init_fences(struct drm_i915_private *dev_priv);
 int i915_gem_freeze(struct drm_i915_private *dev_priv);
 int i915_gem_freeze_late(struct drm_i915_private *dev_priv);
 
-void *i915_gem_object_alloc(struct drm_i915_private *dev_priv);
-void i915_gem_object_free(struct drm_i915_gem_object *obj);
 void i915_gem_object_init(struct drm_i915_gem_object *obj,
                         const struct drm_i915_gem_object_ops *ops);
 struct drm_i915_gem_object *
@@ -3001,7 +2988,12 @@ i915_gem_obj_finish_shmem_access(struct drm_i915_gem_object *obj)
        i915_gem_object_unpin_pages(obj);
 }
 
-int __must_check i915_mutex_lock_interruptible(struct drm_device *dev);
+static inline int __must_check
+i915_mutex_lock_interruptible(struct drm_device *dev)
+{
+       return mutex_lock_interruptible(&dev->struct_mutex);
+}
+
 int i915_gem_dumb_create(struct drm_file *file_priv,
                         struct drm_device *dev,
                         struct drm_mode_create_dumb *args);
@@ -3015,22 +3007,14 @@ void i915_gem_track_fb(struct drm_i915_gem_object *old,
 
 int __must_check i915_gem_set_global_seqno(struct drm_device *dev, u32 seqno);
 
-struct i915_request *
-i915_gem_find_active_request(struct intel_engine_cs *engine);
-
-static inline bool i915_reset_backoff(struct i915_gpu_error *error)
-{
-       return unlikely(test_bit(I915_RESET_BACKOFF, &error->flags));
-}
-
-static inline bool i915_terminally_wedged(struct i915_gpu_error *error)
+static inline bool __i915_wedged(struct i915_gpu_error *error)
 {
        return unlikely(test_bit(I915_WEDGED, &error->flags));
 }
 
-static inline bool i915_reset_backoff_or_wedged(struct i915_gpu_error *error)
+static inline bool i915_reset_failed(struct drm_i915_private *i915)
 {
-       return i915_reset_backoff(error) | i915_terminally_wedged(error);
+       return __i915_wedged(&i915->gpu_error);
 }
 
 static inline u32 i915_reset_count(struct i915_gpu_error *error)
@@ -3055,14 +3039,13 @@ void i915_gem_fini(struct drm_i915_private *dev_priv);
 void i915_gem_cleanup_engines(struct drm_i915_private *dev_priv);
 int i915_gem_wait_for_idle(struct drm_i915_private *dev_priv,
                           unsigned int flags, long timeout);
-int __must_check i915_gem_suspend(struct drm_i915_private *dev_priv);
+void i915_gem_suspend(struct drm_i915_private *dev_priv);
 void i915_gem_suspend_late(struct drm_i915_private *dev_priv);
 void i915_gem_resume(struct drm_i915_private *dev_priv);
 vm_fault_t i915_gem_fault(struct vm_fault *vmf);
 int i915_gem_object_wait(struct drm_i915_gem_object *obj,
                         unsigned int flags,
-                        long timeout,
-                        struct intel_rps_client *rps);
+                        long timeout);
 int i915_gem_object_wait_priority(struct drm_i915_gem_object *obj,
                                  unsigned int flags,
                                  const struct i915_sched_attr *attr);
@@ -3105,7 +3088,6 @@ struct drm_i915_fence_reg *
 i915_reserve_fence(struct drm_i915_private *dev_priv);
 void i915_unreserve_fence(struct drm_i915_fence_reg *fence);
 
-void i915_gem_revoke_fences(struct drm_i915_private *dev_priv);
 void i915_gem_restore_fences(struct drm_i915_private *dev_priv);
 
 void i915_gem_detect_bit_6_swizzle(struct drm_i915_private *dev_priv);
@@ -3141,7 +3123,7 @@ int i915_perf_add_config_ioctl(struct drm_device *dev, void *data,
 int i915_perf_remove_config_ioctl(struct drm_device *dev, void *data,
                                  struct drm_file *file);
 void i915_oa_init_reg_state(struct intel_engine_cs *engine,
-                           struct i915_gem_context *ctx,
+                           struct intel_context *ce,
                            u32 *reg_state);
 
 /* i915_gem_evict.c */
index 8558e81fdc2af85dd52486b7c8d55580fe997373..b7086c8d472639b2f8de608d37a501b8643c2f19 100644 (file)
@@ -42,6 +42,7 @@
 #include "i915_drv.h"
 #include "i915_gem_clflush.h"
 #include "i915_gemfs.h"
+#include "i915_globals.h"
 #include "i915_reset.h"
 #include "i915_trace.h"
 #include "i915_vgpu.h"
@@ -100,48 +101,7 @@ static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv,
        spin_unlock(&dev_priv->mm.object_stat_lock);
 }
 
-static int
-i915_gem_wait_for_error(struct i915_gpu_error *error)
-{
-       int ret;
-
-       might_sleep();
-
-       /*
-        * Only wait 10 seconds for the gpu reset to complete to avoid hanging
-        * userspace. If it takes that long something really bad is going on and
-        * we should simply try to bail out and fail as gracefully as possible.
-        */
-       ret = wait_event_interruptible_timeout(error->reset_queue,
-                                              !i915_reset_backoff(error),
-                                              I915_RESET_TIMEOUT);
-       if (ret == 0) {
-               DRM_ERROR("Timed out waiting for the gpu reset to complete\n");
-               return -EIO;
-       } else if (ret < 0) {
-               return ret;
-       } else {
-               return 0;
-       }
-}
-
-int i915_mutex_lock_interruptible(struct drm_device *dev)
-{
-       struct drm_i915_private *dev_priv = to_i915(dev);
-       int ret;
-
-       ret = i915_gem_wait_for_error(&dev_priv->gpu_error);
-       if (ret)
-               return ret;
-
-       ret = mutex_lock_interruptible(&dev->struct_mutex);
-       if (ret)
-               return ret;
-
-       return 0;
-}
-
-static u32 __i915_gem_park(struct drm_i915_private *i915)
+static void __i915_gem_park(struct drm_i915_private *i915)
 {
        intel_wakeref_t wakeref;
 
@@ -152,9 +112,7 @@ static u32 __i915_gem_park(struct drm_i915_private *i915)
        GEM_BUG_ON(!list_empty(&i915->gt.active_rings));
 
        if (!i915->gt.awake)
-               return I915_EPOCH_INVALID;
-
-       GEM_BUG_ON(i915->gt.epoch == I915_EPOCH_INVALID);
+               return;
 
        /*
         * Be paranoid and flush a concurrent interrupt to make sure
@@ -183,7 +141,7 @@ static u32 __i915_gem_park(struct drm_i915_private *i915)
 
        intel_display_power_put(i915, POWER_DOMAIN_GT_IRQ, wakeref);
 
-       return i915->gt.epoch;
+       i915_globals_park();
 }
 
 void i915_gem_park(struct drm_i915_private *i915)
@@ -225,8 +183,7 @@ void i915_gem_unpark(struct drm_i915_private *i915)
        i915->gt.awake = intel_display_power_get(i915, POWER_DOMAIN_GT_IRQ);
        GEM_BUG_ON(!i915->gt.awake);
 
-       if (unlikely(++i915->gt.epoch == 0)) /* keep 0 as invalid */
-               i915->gt.epoch = 1;
+       i915_globals_unpark();
 
        intel_enable_gt_powersave(i915);
        i915_update_gfx_val(i915);
@@ -459,8 +416,7 @@ int i915_gem_object_unbind(struct drm_i915_gem_object *obj)
 static long
 i915_gem_object_wait_fence(struct dma_fence *fence,
                           unsigned int flags,
-                          long timeout,
-                          struct intel_rps_client *rps_client)
+                          long timeout)
 {
        struct i915_request *rq;
 
@@ -478,27 +434,6 @@ i915_gem_object_wait_fence(struct dma_fence *fence,
        if (i915_request_completed(rq))
                goto out;
 
-       /*
-        * This client is about to stall waiting for the GPU. In many cases
-        * this is undesirable and limits the throughput of the system, as
-        * many clients cannot continue processing user input/output whilst
-        * blocked. RPS autotuning may take tens of milliseconds to respond
-        * to the GPU load and thus incurs additional latency for the client.
-        * We can circumvent that by promoting the GPU frequency to maximum
-        * before we wait. This makes the GPU throttle up much more quickly
-        * (good for benchmarks and user experience, e.g. window animations),
-        * but at a cost of spending more power processing the workload
-        * (bad for battery). Not all clients even want their results
-        * immediately and for them we should just let the GPU select its own
-        * frequency to maximise efficiency. To prevent a single client from
-        * forcing the clocks too high for the whole system, we only allow
-        * each client to waitboost once in a busy period.
-        */
-       if (rps_client && !i915_request_started(rq)) {
-               if (INTEL_GEN(rq->i915) >= 6)
-                       gen6_rps_boost(rq, rps_client);
-       }
-
        timeout = i915_request_wait(rq, flags, timeout);
 
 out:
@@ -511,8 +446,7 @@ out:
 static long
 i915_gem_object_wait_reservation(struct reservation_object *resv,
                                 unsigned int flags,
-                                long timeout,
-                                struct intel_rps_client *rps_client)
+                                long timeout)
 {
        unsigned int seq = __read_seqcount_begin(&resv->seq);
        struct dma_fence *excl;
@@ -530,8 +464,7 @@ i915_gem_object_wait_reservation(struct reservation_object *resv,
 
                for (i = 0; i < count; i++) {
                        timeout = i915_gem_object_wait_fence(shared[i],
-                                                            flags, timeout,
-                                                            rps_client);
+                                                            flags, timeout);
                        if (timeout < 0)
                                break;
 
@@ -557,8 +490,7 @@ i915_gem_object_wait_reservation(struct reservation_object *resv,
        }
 
        if (excl && timeout >= 0)
-               timeout = i915_gem_object_wait_fence(excl, flags, timeout,
-                                                    rps_client);
+               timeout = i915_gem_object_wait_fence(excl, flags, timeout);
 
        dma_fence_put(excl);
 
@@ -652,30 +584,19 @@ i915_gem_object_wait_priority(struct drm_i915_gem_object *obj,
  * @obj: i915 gem object
  * @flags: how to wait (under a lock, for all rendering or just for writes etc)
  * @timeout: how long to wait
- * @rps_client: client (user process) to charge for any waitboosting
  */
 int
 i915_gem_object_wait(struct drm_i915_gem_object *obj,
                     unsigned int flags,
-                    long timeout,
-                    struct intel_rps_client *rps_client)
+                    long timeout)
 {
        might_sleep();
        GEM_BUG_ON(timeout < 0);
 
-       timeout = i915_gem_object_wait_reservation(obj->resv,
-                                                  flags, timeout,
-                                                  rps_client);
+       timeout = i915_gem_object_wait_reservation(obj->resv, flags, timeout);
        return timeout < 0 ? timeout : 0;
 }
 
-static struct intel_rps_client *to_rps_client(struct drm_file *file)
-{
-       struct drm_i915_file_private *fpriv = file->driver_priv;
-
-       return &fpriv->rps_client;
-}
-
 static int
 i915_gem_phys_pwrite(struct drm_i915_gem_object *obj,
                     struct drm_i915_gem_pwrite *args,
@@ -698,17 +619,6 @@ i915_gem_phys_pwrite(struct drm_i915_gem_object *obj,
        return 0;
 }
 
-void *i915_gem_object_alloc(struct drm_i915_private *dev_priv)
-{
-       return kmem_cache_zalloc(dev_priv->objects, GFP_KERNEL);
-}
-
-void i915_gem_object_free(struct drm_i915_gem_object *obj)
-{
-       struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
-       kmem_cache_free(dev_priv->objects, obj);
-}
-
 static int
 i915_gem_create(struct drm_file *file,
                struct drm_i915_private *dev_priv,
@@ -881,8 +791,7 @@ int i915_gem_obj_prepare_shmem_read(struct drm_i915_gem_object *obj,
        ret = i915_gem_object_wait(obj,
                                   I915_WAIT_INTERRUPTIBLE |
                                   I915_WAIT_LOCKED,
-                                  MAX_SCHEDULE_TIMEOUT,
-                                  NULL);
+                                  MAX_SCHEDULE_TIMEOUT);
        if (ret)
                return ret;
 
@@ -934,8 +843,7 @@ int i915_gem_obj_prepare_shmem_write(struct drm_i915_gem_object *obj,
                                   I915_WAIT_INTERRUPTIBLE |
                                   I915_WAIT_LOCKED |
                                   I915_WAIT_ALL,
-                                  MAX_SCHEDULE_TIMEOUT,
-                                  NULL);
+                                  MAX_SCHEDULE_TIMEOUT);
        if (ret)
                return ret;
 
@@ -1197,8 +1105,7 @@ i915_gem_pread_ioctl(struct drm_device *dev, void *data,
 
        ret = i915_gem_object_wait(obj,
                                   I915_WAIT_INTERRUPTIBLE,
-                                  MAX_SCHEDULE_TIMEOUT,
-                                  to_rps_client(file));
+                                  MAX_SCHEDULE_TIMEOUT);
        if (ret)
                goto out;
 
@@ -1497,8 +1404,7 @@ i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
        ret = i915_gem_object_wait(obj,
                                   I915_WAIT_INTERRUPTIBLE |
                                   I915_WAIT_ALL,
-                                  MAX_SCHEDULE_TIMEOUT,
-                                  to_rps_client(file));
+                                  MAX_SCHEDULE_TIMEOUT);
        if (ret)
                goto err;
 
@@ -1596,8 +1502,7 @@ i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
                                   I915_WAIT_INTERRUPTIBLE |
                                   I915_WAIT_PRIORITY |
                                   (write_domain ? I915_WAIT_ALL : 0),
-                                  MAX_SCHEDULE_TIMEOUT,
-                                  to_rps_client(file));
+                                  MAX_SCHEDULE_TIMEOUT);
        if (err)
                goto out;
 
@@ -1891,6 +1796,7 @@ vm_fault_t i915_gem_fault(struct vm_fault *vmf)
        intel_wakeref_t wakeref;
        struct i915_vma *vma;
        pgoff_t page_offset;
+       int srcu;
        int ret;
 
        /* Sanity check that we allow writing into this object */
@@ -1909,8 +1815,7 @@ vm_fault_t i915_gem_fault(struct vm_fault *vmf)
         */
        ret = i915_gem_object_wait(obj,
                                   I915_WAIT_INTERRUPTIBLE,
-                                  MAX_SCHEDULE_TIMEOUT,
-                                  NULL);
+                                  MAX_SCHEDULE_TIMEOUT);
        if (ret)
                goto err;
 
@@ -1920,9 +1825,15 @@ vm_fault_t i915_gem_fault(struct vm_fault *vmf)
 
        wakeref = intel_runtime_pm_get(dev_priv);
 
+       srcu = i915_reset_trylock(dev_priv);
+       if (srcu < 0) {
+               ret = srcu;
+               goto err_rpm;
+       }
+
        ret = i915_mutex_lock_interruptible(dev);
        if (ret)
-               goto err_rpm;
+               goto err_reset;
 
        /* Access to snoopable pages through the GTT is incoherent. */
        if (obj->cache_level != I915_CACHE_NONE && !HAS_LLC(dev_priv)) {
@@ -1930,7 +1841,6 @@ vm_fault_t i915_gem_fault(struct vm_fault *vmf)
                goto err_unlock;
        }
 
-
        /* Now pin it into the GTT as needed */
        vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0,
                                       PIN_MAPPABLE |
@@ -1995,6 +1905,8 @@ err_unpin:
        __i915_vma_unpin(vma);
 err_unlock:
        mutex_unlock(&dev->struct_mutex);
+err_reset:
+       i915_reset_unlock(dev_priv, srcu);
 err_rpm:
        intel_runtime_pm_put(dev_priv, wakeref);
        i915_gem_object_unpin_pages(obj);
@@ -2007,7 +1919,7 @@ err:
                 * fail). But any other -EIO isn't ours (e.g. swap in failure)
                 * and so needs to be reported.
                 */
-               if (!i915_terminally_wedged(&dev_priv->gpu_error))
+               if (!i915_terminally_wedged(dev_priv))
                        return VM_FAULT_SIGBUS;
                /* else: fall through */
        case -EAGAIN:
@@ -2488,7 +2400,7 @@ rebuild_st:
                do {
                        cond_resched();
                        page = shmem_read_mapping_page_gfp(mapping, i, gfp);
-                       if (likely(!IS_ERR(page)))
+                       if (!IS_ERR(page))
                                break;
 
                        if (!*s) {
@@ -2895,51 +2807,6 @@ i915_gem_object_pwrite_gtt(struct drm_i915_gem_object *obj,
        return 0;
 }
 
-static bool match_ring(struct i915_request *rq)
-{
-       struct drm_i915_private *dev_priv = rq->i915;
-       u32 ring = I915_READ(RING_START(rq->engine->mmio_base));
-
-       return ring == i915_ggtt_offset(rq->ring->vma);
-}
-
-struct i915_request *
-i915_gem_find_active_request(struct intel_engine_cs *engine)
-{
-       struct i915_request *request, *active = NULL;
-       unsigned long flags;
-
-       /*
-        * We are called by the error capture, reset and to dump engine
-        * state at random points in time. In particular, note that neither is
-        * crucially ordered with an interrupt. After a hang, the GPU is dead
-        * and we assume that no more writes can happen (we waited long enough
-        * for all writes that were in transaction to be flushed) - adding an
-        * extra delay for a recent interrupt is pointless. Hence, we do
-        * not need an engine->irq_seqno_barrier() before the seqno reads.
-        * At all other times, we must assume the GPU is still running, but
-        * we only care about the snapshot of this moment.
-        */
-       spin_lock_irqsave(&engine->timeline.lock, flags);
-       list_for_each_entry(request, &engine->timeline.requests, link) {
-               if (i915_request_completed(request))
-                       continue;
-
-               if (!i915_request_started(request))
-                       break;
-
-               /* More than one preemptible request may match! */
-               if (!match_ring(request))
-                       break;
-
-               active = request;
-               break;
-       }
-       spin_unlock_irqrestore(&engine->timeline.lock, flags);
-
-       return active;
-}
-
 static void
 i915_gem_retire_work_handler(struct work_struct *work)
 {
@@ -2964,180 +2831,105 @@ i915_gem_retire_work_handler(struct work_struct *work)
                                   round_jiffies_up_relative(HZ));
 }
 
-static void shrink_caches(struct drm_i915_private *i915)
+static bool switch_to_kernel_context_sync(struct drm_i915_private *i915,
+                                         unsigned long mask)
 {
-       /*
-        * kmem_cache_shrink() discards empty slabs and reorders partially
-        * filled slabs to prioritise allocating from the mostly full slabs,
-        * with the aim of reducing fragmentation.
-        */
-       kmem_cache_shrink(i915->priorities);
-       kmem_cache_shrink(i915->dependencies);
-       kmem_cache_shrink(i915->requests);
-       kmem_cache_shrink(i915->luts);
-       kmem_cache_shrink(i915->vmas);
-       kmem_cache_shrink(i915->objects);
-}
-
-struct sleep_rcu_work {
-       union {
-               struct rcu_head rcu;
-               struct work_struct work;
-       };
-       struct drm_i915_private *i915;
-       unsigned int epoch;
-};
+       bool result = true;
 
-static inline bool
-same_epoch(struct drm_i915_private *i915, unsigned int epoch)
-{
        /*
-        * There is a small chance that the epoch wrapped since we started
-        * sleeping. If we assume that epoch is at least a u32, then it will
-        * take at least 2^32 * 100ms for it to wrap, or about 326 years.
+        * Even if we fail to switch, give whatever is running a small chance
+        * to save itself before we report the failure. Yes, this may be a
+        * false positive due to e.g. ENOMEM, caveat emptor!
         */
-       return epoch == READ_ONCE(i915->gt.epoch);
-}
-
-static void __sleep_work(struct work_struct *work)
-{
-       struct sleep_rcu_work *s = container_of(work, typeof(*s), work);
-       struct drm_i915_private *i915 = s->i915;
-       unsigned int epoch = s->epoch;
-
-       kfree(s);
-       if (same_epoch(i915, epoch))
-               shrink_caches(i915);
-}
+       if (i915_gem_switch_to_kernel_context(i915, mask))
+               result = false;
 
-static void __sleep_rcu(struct rcu_head *rcu)
-{
-       struct sleep_rcu_work *s = container_of(rcu, typeof(*s), rcu);
-       struct drm_i915_private *i915 = s->i915;
-
-       destroy_rcu_head(&s->rcu);
+       if (i915_gem_wait_for_idle(i915,
+                                  I915_WAIT_LOCKED |
+                                  I915_WAIT_FOR_IDLE_BOOST,
+                                  I915_GEM_IDLE_TIMEOUT))
+               result = false;
+
+       if (!result) {
+               if (i915_modparams.reset) { /* XXX hide warning from gem_eio */
+                       dev_err(i915->drm.dev,
+                               "Failed to idle engines, declaring wedged!\n");
+                       GEM_TRACE_DUMP();
+               }
 
-       if (same_epoch(i915, s->epoch)) {
-               INIT_WORK(&s->work, __sleep_work);
-               queue_work(i915->wq, &s->work);
-       } else {
-               kfree(s);
+               /* Forcibly cancel outstanding work and leave the gpu quiet. */
+               i915_gem_set_wedged(i915);
        }
-}
 
-static inline bool
-new_requests_since_last_retire(const struct drm_i915_private *i915)
-{
-       return (READ_ONCE(i915->gt.active_requests) ||
-               work_pending(&i915->gt.idle_work.work));
+       i915_retire_requests(i915); /* ensure we flush after wedging */
+       return result;
 }
 
-static void assert_kernel_context_is_current(struct drm_i915_private *i915)
+static bool load_power_context(struct drm_i915_private *i915)
 {
-       struct intel_engine_cs *engine;
-       enum intel_engine_id id;
+       /* Force loading the kernel context on all engines */
+       if (!switch_to_kernel_context_sync(i915, ALL_ENGINES))
+               return false;
 
-       if (i915_terminally_wedged(&i915->gpu_error))
-               return;
+       /*
+        * Immediately park the GPU so that we enable powersaving and
+        * treat it as idle. The next time we issue a request, we will
+        * unpark and start using the engine->pinned_default_state, otherwise
+        * it is in limbo and an early reset may fail.
+        */
+       __i915_gem_park(i915);
 
-       GEM_BUG_ON(i915->gt.active_requests);
-       for_each_engine(engine, i915, id) {
-               GEM_BUG_ON(__i915_active_request_peek(&engine->timeline.last_request));
-               GEM_BUG_ON(engine->last_retired_context !=
-                          to_intel_context(i915->kernel_context, engine));
-       }
+       return true;
 }
 
 static void
 i915_gem_idle_work_handler(struct work_struct *work)
 {
-       struct drm_i915_private *dev_priv =
-               container_of(work, typeof(*dev_priv), gt.idle_work.work);
-       unsigned int epoch = I915_EPOCH_INVALID;
+       struct drm_i915_private *i915 =
+               container_of(work, typeof(*i915), gt.idle_work.work);
        bool rearm_hangcheck;
 
-       if (!READ_ONCE(dev_priv->gt.awake))
+       if (!READ_ONCE(i915->gt.awake))
                return;
 
-       if (READ_ONCE(dev_priv->gt.active_requests))
+       if (READ_ONCE(i915->gt.active_requests))
                return;
 
-       /*
-        * Flush out the last user context, leaving only the pinned
-        * kernel context resident. When we are idling on the kernel_context,
-        * no more new requests (with a context switch) are emitted and we
-        * can finally rest. A consequence is that the idle work handler is
-        * always called at least twice before idling (and if the system is
-        * idle that implies a round trip through the retire worker).
-        */
-       mutex_lock(&dev_priv->drm.struct_mutex);
-       i915_gem_switch_to_kernel_context(dev_priv);
-       mutex_unlock(&dev_priv->drm.struct_mutex);
-
-       GEM_TRACE("active_requests=%d (after switch-to-kernel-context)\n",
-                 READ_ONCE(dev_priv->gt.active_requests));
-
-       /*
-        * Wait for last execlists context complete, but bail out in case a
-        * new request is submitted. As we don't trust the hardware, we
-        * continue on if the wait times out. This is necessary to allow
-        * the machine to suspend even if the hardware dies, and we will
-        * try to recover in resume (after depriving the hardware of power,
-        * it may be in a better mmod).
-        */
-       __wait_for(if (new_requests_since_last_retire(dev_priv)) return,
-                  intel_engines_are_idle(dev_priv),
-                  I915_IDLE_ENGINES_TIMEOUT * 1000,
-                  10, 500);
-
        rearm_hangcheck =
-               cancel_delayed_work_sync(&dev_priv->gpu_error.hangcheck_work);
+               cancel_delayed_work_sync(&i915->gpu_error.hangcheck_work);
 
-       if (!mutex_trylock(&dev_priv->drm.struct_mutex)) {
+       if (!mutex_trylock(&i915->drm.struct_mutex)) {
                /* Currently busy, come back later */
-               mod_delayed_work(dev_priv->wq,
-                                &dev_priv->gt.idle_work,
+               mod_delayed_work(i915->wq,
+                                &i915->gt.idle_work,
                                 msecs_to_jiffies(50));
                goto out_rearm;
        }
 
        /*
-        * New request retired after this work handler started, extend active
-        * period until next instance of the work.
+        * Flush out the last user context, leaving only the pinned
+        * kernel context resident. Should anything unfortunate happen
+        * while we are idle (such as the GPU being power cycled), no users
+        * will be harmed.
         */
-       if (new_requests_since_last_retire(dev_priv))
-               goto out_unlock;
+       if (!work_pending(&i915->gt.idle_work.work) &&
+           !i915->gt.active_requests) {
+               ++i915->gt.active_requests; /* don't requeue idle */
 
-       epoch = __i915_gem_park(dev_priv);
+               switch_to_kernel_context_sync(i915, i915->gt.active_engines);
 
-       assert_kernel_context_is_current(dev_priv);
+               if (!--i915->gt.active_requests) {
+                       __i915_gem_park(i915);
+                       rearm_hangcheck = false;
+               }
+       }
 
-       rearm_hangcheck = false;
-out_unlock:
-       mutex_unlock(&dev_priv->drm.struct_mutex);
+       mutex_unlock(&i915->drm.struct_mutex);
 
 out_rearm:
        if (rearm_hangcheck) {
-               GEM_BUG_ON(!dev_priv->gt.awake);
-               i915_queue_hangcheck(dev_priv);
-       }
-
-       /*
-        * When we are idle, it is an opportune time to reap our caches.
-        * However, we have many objects that utilise RCU and the ordered
-        * i915->wq that this work is executing on. To try and flush any
-        * pending frees now we are idle, we first wait for an RCU grace
-        * period, and then queue a task (that will run last on the wq) to
-        * shrink and re-optimize the caches.
-        */
-       if (same_epoch(dev_priv, epoch)) {
-               struct sleep_rcu_work *s = kmalloc(sizeof(*s), GFP_KERNEL);
-               if (s) {
-                       init_rcu_head(&s->rcu);
-                       s->i915 = dev_priv;
-                       s->epoch = epoch;
-                       call_rcu(&s->rcu, __sleep_rcu);
-               }
+               GEM_BUG_ON(!i915->gt.awake);
+               i915_queue_hangcheck(i915);
        }
 }
 
@@ -3171,7 +2963,7 @@ void i915_gem_close_object(struct drm_gem_object *gem, struct drm_file *file)
                list_del(&lut->obj_link);
                list_del(&lut->ctx_link);
 
-               kmem_cache_free(i915->luts, lut);
+               i915_lut_handle_free(lut);
                __i915_gem_object_release_unless_active(obj);
        }
 
@@ -3234,8 +3026,7 @@ i915_gem_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
                                   I915_WAIT_INTERRUPTIBLE |
                                   I915_WAIT_PRIORITY |
                                   I915_WAIT_ALL,
-                                  to_wait_timeout(args->timeout_ns),
-                                  to_rps_client(file));
+                                  to_wait_timeout(args->timeout_ns));
 
        if (args->timeout_ns > 0) {
                args->timeout_ns -= ktime_to_ns(ktime_sub(ktime_get(), start));
@@ -3304,7 +3095,7 @@ wait_for_timelines(struct drm_i915_private *i915,
                 * stalls, so allow the gpu to boost to maximum clocks.
                 */
                if (flags & I915_WAIT_FOR_IDLE_BOOST)
-                       gen6_rps_boost(rq, NULL);
+                       gen6_rps_boost(rq);
 
                timeout = i915_request_wait(rq, flags, timeout);
                i915_request_put(rq);
@@ -3340,19 +3131,11 @@ int i915_gem_wait_for_idle(struct drm_i915_private *i915,
 
                lockdep_assert_held(&i915->drm.struct_mutex);
 
-               if (GEM_SHOW_DEBUG() && !timeout) {
-                       /* Presume that timeout was non-zero to begin with! */
-                       dev_warn(&i915->drm.pdev->dev,
-                                "Missed idle-completion interrupt!\n");
-                       GEM_TRACE_DUMP();
-               }
-
                err = wait_for_engines(i915);
                if (err)
                        return err;
 
                i915_retire_requests(i915);
-               GEM_BUG_ON(i915->gt.active_requests);
        }
 
        return 0;
@@ -3399,8 +3182,7 @@ i915_gem_object_set_to_wc_domain(struct drm_i915_gem_object *obj, bool write)
                                   I915_WAIT_INTERRUPTIBLE |
                                   I915_WAIT_LOCKED |
                                   (write ? I915_WAIT_ALL : 0),
-                                  MAX_SCHEDULE_TIMEOUT,
-                                  NULL);
+                                  MAX_SCHEDULE_TIMEOUT);
        if (ret)
                return ret;
 
@@ -3462,8 +3244,7 @@ i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
                                   I915_WAIT_INTERRUPTIBLE |
                                   I915_WAIT_LOCKED |
                                   (write ? I915_WAIT_ALL : 0),
-                                  MAX_SCHEDULE_TIMEOUT,
-                                  NULL);
+                                  MAX_SCHEDULE_TIMEOUT);
        if (ret)
                return ret;
 
@@ -3578,8 +3359,7 @@ restart:
                                           I915_WAIT_INTERRUPTIBLE |
                                           I915_WAIT_LOCKED |
                                           I915_WAIT_ALL,
-                                          MAX_SCHEDULE_TIMEOUT,
-                                          NULL);
+                                          MAX_SCHEDULE_TIMEOUT);
                if (ret)
                        return ret;
 
@@ -3717,8 +3497,7 @@ int i915_gem_set_caching_ioctl(struct drm_device *dev, void *data,
 
        ret = i915_gem_object_wait(obj,
                                   I915_WAIT_INTERRUPTIBLE,
-                                  MAX_SCHEDULE_TIMEOUT,
-                                  to_rps_client(file));
+                                  MAX_SCHEDULE_TIMEOUT);
        if (ret)
                goto out;
 
@@ -3844,8 +3623,7 @@ i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)
                                   I915_WAIT_INTERRUPTIBLE |
                                   I915_WAIT_LOCKED |
                                   (write ? I915_WAIT_ALL : 0),
-                                  MAX_SCHEDULE_TIMEOUT,
-                                  NULL);
+                                  MAX_SCHEDULE_TIMEOUT);
        if (ret)
                return ret;
 
@@ -3891,8 +3669,9 @@ i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file)
        long ret;
 
        /* ABI: return -EIO if already wedged */
-       if (i915_terminally_wedged(&dev_priv->gpu_error))
-               return -EIO;
+       ret = i915_terminally_wedged(dev_priv);
+       if (ret)
+               return ret;
 
        spin_lock(&file_priv->mm.lock);
        list_for_each_entry(request, &file_priv->mm.request_list, client_link) {
@@ -3968,7 +3747,7 @@ i915_gem_object_ggtt_pin(struct drm_i915_gem_object *obj,
        }
 
        vma = i915_vma_instance(obj, vm, view);
-       if (unlikely(IS_ERR(vma)))
+       if (IS_ERR(vma))
                return vma;
 
        if (i915_vma_misplaced(vma, size, alignment, flags)) {
@@ -4002,20 +3781,17 @@ i915_gem_object_ggtt_pin(struct drm_i915_gem_object *obj,
 
 static __always_inline unsigned int __busy_read_flag(unsigned int id)
 {
-       /* Note that we could alias engines in the execbuf API, but
-        * that would be very unwise as it prevents userspace from
-        * fine control over engine selection. Ahem.
-        *
-        * This should be something like EXEC_MAX_ENGINE instead of
-        * I915_NUM_ENGINES.
-        */
-       BUILD_BUG_ON(I915_NUM_ENGINES > 16);
+       if (id == I915_ENGINE_CLASS_INVALID)
+               return 0xffff0000;
+
+       GEM_BUG_ON(id >= 16);
        return 0x10000 << id;
 }
 
 static __always_inline unsigned int __busy_write_id(unsigned int id)
 {
-       /* The uABI guarantees an active writer is also amongst the read
+       /*
+        * The uABI guarantees an active writer is also amongst the read
         * engines. This would be true if we accessed the activity tracking
         * under the lock, but as we perform the lookup of the object and
         * its activity locklessly we can not guarantee that the last_write
@@ -4023,16 +3799,20 @@ static __always_inline unsigned int __busy_write_id(unsigned int id)
         * last_read - hence we always set both read and write busy for
         * last_write.
         */
-       return id | __busy_read_flag(id);
+       if (id == I915_ENGINE_CLASS_INVALID)
+               return 0xffffffff;
+
+       return (id + 1) | __busy_read_flag(id);
 }
 
 static __always_inline unsigned int
 __busy_set_if_active(const struct dma_fence *fence,
                     unsigned int (*flag)(unsigned int id))
 {
-       struct i915_request *rq;
+       const struct i915_request *rq;
 
-       /* We have to check the current hw status of the fence as the uABI
+       /*
+        * We have to check the current hw status of the fence as the uABI
         * guarantees forward progress. We could rely on the idle worker
         * to eventually flush us, but to minimise latency just ask the
         * hardware.
@@ -4043,11 +3823,11 @@ __busy_set_if_active(const struct dma_fence *fence,
                return 0;
 
        /* opencode to_request() in order to avoid const warnings */
-       rq = container_of(fence, struct i915_request, fence);
+       rq = container_of(fence, const struct i915_request, fence);
        if (i915_request_completed(rq))
                return 0;
 
-       return flag(rq->engine->uabi_id);
+       return flag(rq->engine->uabi_class);
 }
 
 static __always_inline unsigned int
@@ -4081,7 +3861,8 @@ i915_gem_busy_ioctl(struct drm_device *dev, void *data,
        if (!obj)
                goto out;
 
-       /* A discrepancy here is that we do not report the status of
+       /*
+        * A discrepancy here is that we do not report the status of
         * non-i915 fences, i.e. even though we may report the object as idle,
         * a call to set-domain may still stall waiting for foreign rendering.
         * This also means that wait-ioctl may report an object as busy,
@@ -4281,7 +4062,7 @@ i915_gem_object_create(struct drm_i915_private *dev_priv, u64 size)
        if (overflows_type(size, obj->base.size))
                return ERR_PTR(-E2BIG);
 
-       obj = i915_gem_object_alloc(dev_priv);
+       obj = i915_gem_object_alloc();
        if (obj == NULL)
                return ERR_PTR(-ENOMEM);
 
@@ -4545,7 +4326,7 @@ void i915_gem_sanitize(struct drm_i915_private *i915)
         * back to defaults, recovering from whatever wedged state we left it
         * in and so worth trying to use the device once more.
         */
-       if (i915_terminally_wedged(&i915->gpu_error))
+       if (i915_terminally_wedged(i915))
                i915_gem_unset_wedged(i915);
 
        /*
@@ -4566,10 +4347,9 @@ void i915_gem_sanitize(struct drm_i915_private *i915)
        mutex_unlock(&i915->drm.struct_mutex);
 }
 
-int i915_gem_suspend(struct drm_i915_private *i915)
+void i915_gem_suspend(struct drm_i915_private *i915)
 {
        intel_wakeref_t wakeref;
-       int ret;
 
        GEM_TRACE("\n");
 
@@ -4589,22 +4369,7 @@ int i915_gem_suspend(struct drm_i915_private *i915)
         * state. Fortunately, the kernel_context is disposable and we do
         * not rely on its state.
         */
-       if (!i915_terminally_wedged(&i915->gpu_error)) {
-               ret = i915_gem_switch_to_kernel_context(i915);
-               if (ret)
-                       goto err_unlock;
-
-               ret = i915_gem_wait_for_idle(i915,
-                                            I915_WAIT_INTERRUPTIBLE |
-                                            I915_WAIT_LOCKED |
-                                            I915_WAIT_FOR_IDLE_BOOST,
-                                            MAX_SCHEDULE_TIMEOUT);
-               if (ret && ret != -EIO)
-                       goto err_unlock;
-
-               assert_kernel_context_is_current(i915);
-       }
-       i915_retire_requests(i915); /* ensure we flush after wedging */
+       switch_to_kernel_context_sync(i915, i915->gt.active_engines);
 
        mutex_unlock(&i915->drm.struct_mutex);
        i915_reset_flush(i915);
@@ -4617,23 +4382,13 @@ int i915_gem_suspend(struct drm_i915_private *i915)
         */
        drain_delayed_work(&i915->gt.idle_work);
 
-       intel_uc_suspend(i915);
-
        /*
         * Assert that we successfully flushed all the work and
         * reset the GPU back to its idle, low power state.
         */
-       WARN_ON(i915->gt.awake);
-       if (WARN_ON(!intel_engines_are_idle(i915)))
-               i915_gem_set_wedged(i915); /* no hope, discard everything */
-
-       intel_runtime_pm_put(i915, wakeref);
-       return 0;
+       GEM_BUG_ON(i915->gt.awake);
 
-err_unlock:
-       mutex_unlock(&i915->drm.struct_mutex);
        intel_runtime_pm_put(i915, wakeref);
-       return ret;
 }
 
 void i915_gem_suspend_late(struct drm_i915_private *i915)
@@ -4701,7 +4456,7 @@ void i915_gem_resume(struct drm_i915_private *i915)
        intel_uc_resume(i915);
 
        /* Always reload a context for powersaving. */
-       if (i915_gem_switch_to_kernel_context(i915))
+       if (!load_power_context(i915))
                goto err_wedged;
 
 out_unlock:
@@ -4710,8 +4465,9 @@ out_unlock:
        return;
 
 err_wedged:
-       if (!i915_terminally_wedged(&i915->gpu_error)) {
-               DRM_ERROR("failed to re-initialize GPU, declaring wedged!\n");
+       if (!i915_reset_failed(i915)) {
+               dev_err(i915->drm.dev,
+                       "Failed to re-initialize GPU, declaring it wedged!\n");
                i915_gem_set_wedged(i915);
        }
        goto out_unlock;
@@ -4781,6 +4537,8 @@ static int __i915_gem_restart_engines(void *data)
                }
        }
 
+       intel_engines_set_scheduler_caps(i915);
+
        return 0;
 }
 
@@ -4816,10 +4574,9 @@ int i915_gem_init_hw(struct drm_i915_private *dev_priv)
        init_unused_rings(dev_priv);
 
        BUG_ON(!dev_priv->kernel_context);
-       if (i915_terminally_wedged(&dev_priv->gpu_error)) {
-               ret = -EIO;
+       ret = i915_terminally_wedged(dev_priv);
+       if (ret)
                goto out;
-       }
 
        ret = i915_ppgtt_init_hw(dev_priv);
        if (ret) {
@@ -4864,7 +4621,7 @@ static int __intel_engines_record_defaults(struct drm_i915_private *i915)
        struct i915_gem_context *ctx;
        struct intel_engine_cs *engine;
        enum intel_engine_id id;
-       int err;
+       int err = 0;
 
        /*
         * As we reset the gpu during very early sanitisation, the current
@@ -4897,36 +4654,27 @@ static int __intel_engines_record_defaults(struct drm_i915_private *i915)
                        goto err_active;
        }
 
-       err = i915_gem_switch_to_kernel_context(i915);
-       if (err)
-               goto err_active;
-
-       if (i915_gem_wait_for_idle(i915, I915_WAIT_LOCKED, HZ / 5)) {
-               i915_gem_set_wedged(i915);
-               err = -EIO; /* Caller will declare us wedged */
+       /* Flush the default context image to memory, and enable powersaving. */
+       if (!load_power_context(i915)) {
+               err = -EIO;
                goto err_active;
        }
 
-       assert_kernel_context_is_current(i915);
-
-       /*
-        * Immediately park the GPU so that we enable powersaving and
-        * treat it as idle. The next time we issue a request, we will
-        * unpark and start using the engine->pinned_default_state, otherwise
-        * it is in limbo and an early reset may fail.
-        */
-       __i915_gem_park(i915);
-
        for_each_engine(engine, i915, id) {
+               struct intel_context *ce;
                struct i915_vma *state;
                void *vaddr;
 
-               GEM_BUG_ON(to_intel_context(ctx, engine)->pin_count);
+               ce = intel_context_lookup(ctx, engine);
+               if (!ce)
+                       continue;
 
-               state = to_intel_context(ctx, engine)->state;
+               state = ce->state;
                if (!state)
                        continue;
 
+               GEM_BUG_ON(intel_context_is_pinned(ce));
+
                /*
                 * As we will hold a reference to the logical state, it will
                 * not be torn down with the context, and importantly the
@@ -4982,19 +4730,10 @@ out_ctx:
 err_active:
        /*
         * If we have to abandon now, we expect the engines to be idle
-        * and ready to be torn-down. First try to flush any remaining
-        * request, ensure we are pointing at the kernel context and
-        * then remove it.
+        * and ready to be torn-down. The quickest way we can accomplish
+        * this is by declaring ourselves wedged.
         */
-       if (WARN_ON(i915_gem_switch_to_kernel_context(i915)))
-               goto out_ctx;
-
-       if (WARN_ON(i915_gem_wait_for_idle(i915,
-                                          I915_WAIT_LOCKED,
-                                          MAX_SCHEDULE_TIMEOUT)))
-               goto out_ctx;
-
-       i915_gem_contexts_lost(i915);
+       i915_gem_set_wedged(i915);
        goto out_ctx;
 }
 
@@ -5152,7 +4891,7 @@ int i915_gem_init(struct drm_i915_private *dev_priv)
 err_init_hw:
        mutex_unlock(&dev_priv->drm.struct_mutex);
 
-       WARN_ON(i915_gem_suspend(dev_priv));
+       i915_gem_suspend(dev_priv);
        i915_gem_suspend_late(dev_priv);
 
        i915_gem_drain_workqueue(dev_priv);
@@ -5192,7 +4931,7 @@ err_uc_misc:
                 * wedged. But we only want to do this where the GPU is angry,
                 * for all other failure, such as an allocation failure, bail.
                 */
-               if (!i915_terminally_wedged(&dev_priv->gpu_error)) {
+               if (!i915_reset_failed(dev_priv)) {
                        i915_load_error(dev_priv,
                                        "Failed to initialize GPU, declaring it wedged!\n");
                        i915_gem_set_wedged(dev_priv);
@@ -5305,36 +5044,7 @@ static void i915_gem_init__mm(struct drm_i915_private *i915)
 
 int i915_gem_init_early(struct drm_i915_private *dev_priv)
 {
-       int err = -ENOMEM;
-
-       dev_priv->objects = KMEM_CACHE(drm_i915_gem_object, SLAB_HWCACHE_ALIGN);
-       if (!dev_priv->objects)
-               goto err_out;
-
-       dev_priv->vmas = KMEM_CACHE(i915_vma, SLAB_HWCACHE_ALIGN);
-       if (!dev_priv->vmas)
-               goto err_objects;
-
-       dev_priv->luts = KMEM_CACHE(i915_lut_handle, 0);
-       if (!dev_priv->luts)
-               goto err_vmas;
-
-       dev_priv->requests = KMEM_CACHE(i915_request,
-                                       SLAB_HWCACHE_ALIGN |
-                                       SLAB_RECLAIM_ACCOUNT |
-                                       SLAB_TYPESAFE_BY_RCU);
-       if (!dev_priv->requests)
-               goto err_luts;
-
-       dev_priv->dependencies = KMEM_CACHE(i915_dependency,
-                                           SLAB_HWCACHE_ALIGN |
-                                           SLAB_RECLAIM_ACCOUNT);
-       if (!dev_priv->dependencies)
-               goto err_requests;
-
-       dev_priv->priorities = KMEM_CACHE(i915_priolist, SLAB_HWCACHE_ALIGN);
-       if (!dev_priv->priorities)
-               goto err_dependencies;
+       int err;
 
        INIT_LIST_HEAD(&dev_priv->gt.active_rings);
        INIT_LIST_HEAD(&dev_priv->gt.closed_vma);
@@ -5348,6 +5058,7 @@ int i915_gem_init_early(struct drm_i915_private *dev_priv)
        init_waitqueue_head(&dev_priv->gpu_error.wait_queue);
        init_waitqueue_head(&dev_priv->gpu_error.reset_queue);
        mutex_init(&dev_priv->gpu_error.wedge_mutex);
+       init_srcu_struct(&dev_priv->gpu_error.reset_backoff_srcu);
 
        atomic_set(&dev_priv->mm.bsd_engine_dispatch_index, 0);
 
@@ -5358,19 +5069,6 @@ int i915_gem_init_early(struct drm_i915_private *dev_priv)
                DRM_NOTE("Unable to create a private tmpfs mount, hugepage support will be disabled(%d).\n", err);
 
        return 0;
-
-err_dependencies:
-       kmem_cache_destroy(dev_priv->dependencies);
-err_requests:
-       kmem_cache_destroy(dev_priv->requests);
-err_luts:
-       kmem_cache_destroy(dev_priv->luts);
-err_vmas:
-       kmem_cache_destroy(dev_priv->vmas);
-err_objects:
-       kmem_cache_destroy(dev_priv->objects);
-err_out:
-       return err;
 }
 
 void i915_gem_cleanup_early(struct drm_i915_private *dev_priv)
@@ -5380,15 +5078,7 @@ void i915_gem_cleanup_early(struct drm_i915_private *dev_priv)
        GEM_BUG_ON(atomic_read(&dev_priv->mm.free_count));
        WARN_ON(dev_priv->mm.object_count);
 
-       kmem_cache_destroy(dev_priv->priorities);
-       kmem_cache_destroy(dev_priv->dependencies);
-       kmem_cache_destroy(dev_priv->requests);
-       kmem_cache_destroy(dev_priv->luts);
-       kmem_cache_destroy(dev_priv->vmas);
-       kmem_cache_destroy(dev_priv->objects);
-
-       /* And ensure that our DESTROY_BY_RCU slabs are truly destroyed */
-       rcu_barrier();
+       cleanup_srcu_struct(&dev_priv->gpu_error.reset_backoff_srcu);
 
        i915_gemfs_fini(dev_priv);
 }
index b0e4b976880c0bf6699197f4f0215e15289c23f4..5c073fe736640bf0902874628cb3072b2a277d19 100644 (file)
@@ -75,12 +75,14 @@ struct drm_i915_private;
 
 #define I915_NUM_ENGINES 8
 
+#define I915_GEM_IDLE_TIMEOUT (HZ / 5)
+
 void i915_gem_park(struct drm_i915_private *i915);
 void i915_gem_unpark(struct drm_i915_private *i915);
 
 static inline void __tasklet_disable_sync_once(struct tasklet_struct *t)
 {
-       if (atomic_inc_return(&t->count) == 1)
+       if (!atomic_fetch_inc(&t->count))
                tasklet_unlock_wait(t);
 }
 
@@ -89,4 +91,9 @@ static inline bool __tasklet_is_enabled(const struct tasklet_struct *t)
        return !atomic_read(&t->count);
 }
 
+static inline bool __tasklet_enable(struct tasklet_struct *t)
+{
+       return atomic_dec_and_test(&t->count);
+}
+
 #endif /* __I915_GEM_H__ */
index 280813a4bf82a6fdd223c1b6f8dcbcdbe2980e12..d776d43707e0be72488a7fd1ea314afe53fcac91 100644 (file)
 #include <linux/log2.h>
 #include <drm/i915_drm.h>
 #include "i915_drv.h"
+#include "i915_globals.h"
 #include "i915_trace.h"
 #include "intel_lrc_reg.h"
 #include "intel_workarounds.h"
 
 #define ALL_L3_SLICES(dev) (1 << NUM_L3_SLICES(dev)) - 1
 
+static struct i915_global_gem_context {
+       struct i915_global base;
+       struct kmem_cache *slab_luts;
+} global;
+
+struct i915_lut_handle *i915_lut_handle_alloc(void)
+{
+       return kmem_cache_alloc(global.slab_luts, GFP_KERNEL);
+}
+
+void i915_lut_handle_free(struct i915_lut_handle *lut)
+{
+       return kmem_cache_free(global.slab_luts, lut);
+}
+
 static void lut_close(struct i915_gem_context *ctx)
 {
        struct i915_lut_handle *lut, *ln;
@@ -102,7 +118,7 @@ static void lut_close(struct i915_gem_context *ctx)
 
        list_for_each_entry_safe(lut, ln, &ctx->handles_list, ctx_link) {
                list_del(&lut->obj_link);
-               kmem_cache_free(ctx->i915->luts, lut);
+               i915_lut_handle_free(lut);
        }
 
        rcu_read_lock();
@@ -206,25 +222,23 @@ static void release_hw_id(struct i915_gem_context *ctx)
 
 static void i915_gem_context_free(struct i915_gem_context *ctx)
 {
-       unsigned int n;
+       struct intel_context *it, *n;
 
        lockdep_assert_held(&ctx->i915->drm.struct_mutex);
        GEM_BUG_ON(!i915_gem_context_is_closed(ctx));
+       GEM_BUG_ON(!list_empty(&ctx->active_engines));
 
        release_hw_id(ctx);
        i915_ppgtt_put(ctx->ppgtt);
 
-       for (n = 0; n < ARRAY_SIZE(ctx->__engine); n++) {
-               struct intel_context *ce = &ctx->__engine[n];
-
-               if (ce->ops)
-                       ce->ops->destroy(ce);
-       }
+       rbtree_postorder_for_each_entry_safe(it, n, &ctx->hw_contexts, node)
+               intel_context_put(it);
 
        kfree(ctx->name);
        put_pid(ctx->pid);
 
        list_del(&ctx->link);
+       mutex_destroy(&ctx->mutex);
 
        kfree_rcu(ctx, rcu);
 }
@@ -307,7 +321,7 @@ static u32 default_desc_template(const struct drm_i915_private *i915,
        desc = GEN8_CTX_VALID | GEN8_CTX_PRIVILEGE;
 
        address_mode = INTEL_LEGACY_32B_CONTEXT;
-       if (ppgtt && i915_vm_is_48bit(&ppgtt->vm))
+       if (ppgtt && i915_vm_is_4lvl(&ppgtt->vm))
                address_mode = INTEL_LEGACY_64B_CONTEXT;
        desc |= address_mode << GEN8_CTX_ADDRESSING_MODE_SHIFT;
 
@@ -322,39 +336,13 @@ static u32 default_desc_template(const struct drm_i915_private *i915,
        return desc;
 }
 
-static void intel_context_retire(struct i915_active_request *active,
-                                struct i915_request *rq)
-{
-       struct intel_context *ce =
-               container_of(active, typeof(*ce), active_tracker);
-
-       intel_context_unpin(ce);
-}
-
-void
-intel_context_init(struct intel_context *ce,
-                  struct i915_gem_context *ctx,
-                  struct intel_engine_cs *engine)
-{
-       ce->gem_context = ctx;
-
-       INIT_LIST_HEAD(&ce->signal_link);
-       INIT_LIST_HEAD(&ce->signals);
-
-       /* Use the whole device by default */
-       ce->sseu = intel_device_default_sseu(ctx->i915);
-
-       i915_active_request_init(&ce->active_tracker,
-                                NULL, intel_context_retire);
-}
-
 static struct i915_gem_context *
 __create_hw_context(struct drm_i915_private *dev_priv,
                    struct drm_i915_file_private *file_priv)
 {
        struct i915_gem_context *ctx;
-       unsigned int n;
        int ret;
+       int i;
 
        ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
        if (ctx == NULL)
@@ -364,9 +352,11 @@ __create_hw_context(struct drm_i915_private *dev_priv,
        list_add_tail(&ctx->link, &dev_priv->contexts.list);
        ctx->i915 = dev_priv;
        ctx->sched.priority = I915_USER_PRIORITY(I915_PRIORITY_NORMAL);
+       INIT_LIST_HEAD(&ctx->active_engines);
+       mutex_init(&ctx->mutex);
 
-       for (n = 0; n < ARRAY_SIZE(ctx->__engine); n++)
-               intel_context_init(&ctx->__engine[n], ctx, dev_priv->engine[n]);
+       ctx->hw_contexts = RB_ROOT;
+       spin_lock_init(&ctx->hw_contexts_lock);
 
        INIT_RADIX_TREE(&ctx->handles_vma, GFP_KERNEL);
        INIT_LIST_HEAD(&ctx->handles_list);
@@ -401,10 +391,15 @@ __create_hw_context(struct drm_i915_private *dev_priv,
        ctx->remap_slice = ALL_L3_SLICES(dev_priv);
 
        i915_gem_context_set_bannable(ctx);
+       i915_gem_context_set_recoverable(ctx);
+
        ctx->ring_size = 4 * PAGE_SIZE;
        ctx->desc_template =
                default_desc_template(dev_priv, dev_priv->mm.aliasing_ppgtt);
 
+       for (i = 0; i < ARRAY_SIZE(ctx->hang_timestamp); i++)
+               ctx->hang_timestamp[i] = jiffies - CONTEXT_FAST_HANG_JIFFIES;
+
        return ctx;
 
 err_pid:
@@ -563,7 +558,7 @@ int i915_gem_contexts_init(struct drm_i915_private *dev_priv)
        GEM_BUG_ON(dev_priv->kernel_context);
        GEM_BUG_ON(dev_priv->preempt_context);
 
-       intel_engine_init_ctx_wa(dev_priv->engine[RCS]);
+       intel_engine_init_ctx_wa(dev_priv->engine[RCS0]);
        init_contexts(dev_priv);
 
        /* lowest priority; idle task */
@@ -682,81 +677,98 @@ last_request_on_engine(struct i915_timeline *timeline,
        return NULL;
 }
 
-static bool engine_has_kernel_context_barrier(struct intel_engine_cs *engine)
+struct context_barrier_task {
+       struct i915_active base;
+       void (*task)(void *data);
+       void *data;
+};
+
+static void cb_retire(struct i915_active *base)
 {
-       struct drm_i915_private *i915 = engine->i915;
-       const struct intel_context * const ce =
-               to_intel_context(i915->kernel_context, engine);
-       struct i915_timeline *barrier = ce->ring->timeline;
-       struct intel_ring *ring;
-       bool any_active = false;
+       struct context_barrier_task *cb = container_of(base, typeof(*cb), base);
+
+       if (cb->task)
+               cb->task(cb->data);
+
+       i915_active_fini(&cb->base);
+       kfree(cb);
+}
+
+I915_SELFTEST_DECLARE(static unsigned long context_barrier_inject_fault);
+static int context_barrier_task(struct i915_gem_context *ctx,
+                               unsigned long engines,
+                               void (*task)(void *data),
+                               void *data)
+{
+       struct drm_i915_private *i915 = ctx->i915;
+       struct context_barrier_task *cb;
+       struct intel_context *ce;
+       intel_wakeref_t wakeref;
+       int err = 0;
 
        lockdep_assert_held(&i915->drm.struct_mutex);
-       list_for_each_entry(ring, &i915->gt.active_rings, active_link) {
-               struct i915_request *rq;
+       GEM_BUG_ON(!task);
 
-               rq = last_request_on_engine(ring->timeline, engine);
-               if (!rq)
-                       continue;
+       cb = kmalloc(sizeof(*cb), GFP_KERNEL);
+       if (!cb)
+               return -ENOMEM;
 
-               any_active = true;
+       i915_active_init(i915, &cb->base, cb_retire);
+       i915_active_acquire(&cb->base);
+
+       wakeref = intel_runtime_pm_get(i915);
+       list_for_each_entry(ce, &ctx->active_engines, active_link) {
+               struct intel_engine_cs *engine = ce->engine;
+               struct i915_request *rq;
 
-               if (rq->hw_context == ce)
+               if (!(ce->engine->mask & engines))
                        continue;
 
-               /*
-                * Was this request submitted after the previous
-                * switch-to-kernel-context?
-                */
-               if (!i915_timeline_sync_is_later(barrier, &rq->fence)) {
-                       GEM_TRACE("%s needs barrier for %llx:%lld\n",
-                                 ring->timeline->name,
-                                 rq->fence.context,
-                                 rq->fence.seqno);
-                       return false;
+               if (I915_SELFTEST_ONLY(context_barrier_inject_fault &
+                                      engine->mask)) {
+                       err = -ENXIO;
+                       break;
+               }
+
+               rq = i915_request_alloc(engine, ctx);
+               if (IS_ERR(rq)) {
+                       err = PTR_ERR(rq);
+                       break;
                }
 
-               GEM_TRACE("%s has barrier after %llx:%lld\n",
-                         ring->timeline->name,
-                         rq->fence.context,
-                         rq->fence.seqno);
+               err = i915_active_ref(&cb->base, rq->fence.context, rq);
+               i915_request_add(rq);
+               if (err)
+                       break;
        }
+       intel_runtime_pm_put(i915, wakeref);
 
-       /*
-        * If any other timeline was still active and behind the last barrier,
-        * then our last switch-to-kernel-context must still be queued and
-        * will run last (leaving the engine in the kernel context when it
-        * eventually idles).
-        */
-       if (any_active)
-               return true;
+       cb->task = err ? NULL : task; /* caller needs to unwind instead */
+       cb->data = data;
 
-       /* The engine is idle; check that it is idling in the kernel context. */
-       return engine->last_retired_context == ce;
+       i915_active_release(&cb->base);
+
+       return err;
 }
 
-int i915_gem_switch_to_kernel_context(struct drm_i915_private *i915)
+int i915_gem_switch_to_kernel_context(struct drm_i915_private *i915,
+                                     unsigned long mask)
 {
        struct intel_engine_cs *engine;
-       enum intel_engine_id id;
 
        GEM_TRACE("awake?=%s\n", yesno(i915->gt.awake));
 
        lockdep_assert_held(&i915->drm.struct_mutex);
        GEM_BUG_ON(!i915->kernel_context);
 
-       i915_retire_requests(i915);
+       /* Inoperable, so presume the GPU is safely pointing into the void! */
+       if (i915_terminally_wedged(i915))
+               return 0;
 
-       for_each_engine(engine, i915, id) {
+       for_each_engine_masked(engine, i915, mask, mask) {
                struct intel_ring *ring;
                struct i915_request *rq;
 
-               GEM_BUG_ON(!to_intel_context(i915->kernel_context, engine));
-               if (engine_has_kernel_context_barrier(engine))
-                       continue;
-
-               GEM_TRACE("emit barrier on %s\n", engine->name);
-
                rq = i915_request_alloc(engine, i915->kernel_context);
                if (IS_ERR(rq))
                        return PTR_ERR(rq);
@@ -779,7 +791,6 @@ int i915_gem_switch_to_kernel_context(struct drm_i915_private *i915)
                        i915_sw_fence_await_sw_fence_gfp(&rq->submit,
                                                         &prev->submit,
                                                         I915_FENCE_GFP);
-                       i915_timeline_sync_set(rq->timeline, &prev->fence);
                }
 
                i915_request_add(rq);
@@ -796,18 +807,22 @@ static bool client_is_banned(struct drm_i915_file_private *file_priv)
 int i915_gem_context_create_ioctl(struct drm_device *dev, void *data,
                                  struct drm_file *file)
 {
-       struct drm_i915_private *dev_priv = to_i915(dev);
+       struct drm_i915_private *i915 = to_i915(dev);
        struct drm_i915_gem_context_create *args = data;
        struct drm_i915_file_private *file_priv = file->driver_priv;
        struct i915_gem_context *ctx;
        int ret;
 
-       if (!DRIVER_CAPS(dev_priv)->has_logical_contexts)
+       if (!DRIVER_CAPS(i915)->has_logical_contexts)
                return -ENODEV;
 
        if (args->pad != 0)
                return -EINVAL;
 
+       ret = i915_terminally_wedged(i915);
+       if (ret)
+               return ret;
+
        if (client_is_banned(file_priv)) {
                DRM_DEBUG("client %s[%d] banned from creating ctx\n",
                          current->comm,
@@ -820,7 +835,7 @@ int i915_gem_context_create_ioctl(struct drm_device *dev, void *data,
        if (ret)
                return ret;
 
-       ctx = i915_gem_create_context(dev_priv, file_priv);
+       ctx = i915_gem_create_context(i915, file_priv);
        mutex_unlock(&dev->struct_mutex);
        if (IS_ERR(ctx))
                return PTR_ERR(ctx);
@@ -869,7 +884,6 @@ static int get_sseu(struct i915_gem_context *ctx,
        struct drm_i915_gem_context_param_sseu user_sseu;
        struct intel_engine_cs *engine;
        struct intel_context *ce;
-       int ret;
 
        if (args->size == 0)
                goto out;
@@ -889,19 +903,16 @@ static int get_sseu(struct i915_gem_context *ctx,
        if (!engine)
                return -EINVAL;
 
-       /* Only use for mutex here is to serialize get_param and set_param. */
-       ret = mutex_lock_interruptible(&ctx->i915->drm.struct_mutex);
-       if (ret)
-               return ret;
-
-       ce = to_intel_context(ctx, engine);
+       ce = intel_context_pin_lock(ctx, engine); /* serialises with set_sseu */
+       if (IS_ERR(ce))
+               return PTR_ERR(ce);
 
        user_sseu.slice_mask = ce->sseu.slice_mask;
        user_sseu.subslice_mask = ce->sseu.subslice_mask;
        user_sseu.min_eus_per_subslice = ce->sseu.min_eus_per_subslice;
        user_sseu.max_eus_per_subslice = ce->sseu.max_eus_per_subslice;
 
-       mutex_unlock(&ctx->i915->drm.struct_mutex);
+       intel_context_pin_unlock(ce);
 
        if (copy_to_user(u64_to_user_ptr(args->value), &user_sseu,
                         sizeof(user_sseu)))
@@ -951,6 +962,10 @@ int i915_gem_context_getparam_ioctl(struct drm_device *dev, void *data,
                args->size = 0;
                args->value = i915_gem_context_is_bannable(ctx);
                break;
+       case I915_CONTEXT_PARAM_RECOVERABLE:
+               args->size = 0;
+               args->value = i915_gem_context_is_recoverable(ctx);
+               break;
        case I915_CONTEXT_PARAM_PRIORITY:
                args->size = 0;
                args->value = ctx->sched.priority >> I915_USER_PRIORITY_SHIFT;
@@ -993,23 +1008,28 @@ static int gen8_emit_rpcs_config(struct i915_request *rq,
 }
 
 static int
-gen8_modify_rpcs_gpu(struct intel_context *ce,
-                    struct intel_engine_cs *engine,
-                    struct intel_sseu sseu)
+gen8_modify_rpcs(struct intel_context *ce, struct intel_sseu sseu)
 {
-       struct drm_i915_private *i915 = engine->i915;
+       struct drm_i915_private *i915 = ce->engine->i915;
        struct i915_request *rq, *prev;
        intel_wakeref_t wakeref;
        int ret;
 
-       GEM_BUG_ON(!ce->pin_count);
+       lockdep_assert_held(&ce->pin_mutex);
 
-       lockdep_assert_held(&i915->drm.struct_mutex);
+       /*
+        * If the context is not idle, we have to submit an ordered request to
+        * modify its context image via the kernel context (writing to our own
+        * image, or into the registers directory, does not stick). Pristine
+        * and idle contexts will be configured on pinning.
+        */
+       if (!intel_context_is_pinned(ce))
+               return 0;
 
        /* Submitting requests etc needs the hw awake. */
        wakeref = intel_runtime_pm_get(i915);
 
-       rq = i915_request_alloc(engine, i915->kernel_context);
+       rq = i915_request_alloc(ce->engine, i915->kernel_context);
        if (IS_ERR(rq)) {
                ret = PTR_ERR(rq);
                goto out_put;
@@ -1057,27 +1077,26 @@ __i915_gem_context_reconfigure_sseu(struct i915_gem_context *ctx,
                                    struct intel_engine_cs *engine,
                                    struct intel_sseu sseu)
 {
-       struct intel_context *ce = to_intel_context(ctx, engine);
+       struct intel_context *ce;
        int ret = 0;
 
        GEM_BUG_ON(INTEL_GEN(ctx->i915) < 8);
-       GEM_BUG_ON(engine->id != RCS);
+       GEM_BUG_ON(engine->id != RCS0);
+
+       ce = intel_context_pin_lock(ctx, engine);
+       if (IS_ERR(ce))
+               return PTR_ERR(ce);
 
        /* Nothing to do if unmodified. */
        if (!memcmp(&ce->sseu, &sseu, sizeof(sseu)))
-               return 0;
-
-       /*
-        * If context is not idle we have to submit an ordered request to modify
-        * its context image via the kernel context. Pristine and idle contexts
-        * will be configured on pinning.
-        */
-       if (ce->pin_count)
-               ret = gen8_modify_rpcs_gpu(ce, engine, sseu);
+               goto unlock;
 
+       ret = gen8_modify_rpcs(ce, sseu);
        if (!ret)
                ce->sseu = sseu;
 
+unlock:
+       intel_context_pin_unlock(ce);
        return ret;
 }
 
@@ -1285,6 +1304,15 @@ int i915_gem_context_setparam_ioctl(struct drm_device *dev, void *data,
                        i915_gem_context_clear_bannable(ctx);
                break;
 
+       case I915_CONTEXT_PARAM_RECOVERABLE:
+               if (args->size)
+                       ret = -EINVAL;
+               else if (args->value)
+                       i915_gem_context_set_recoverable(ctx);
+               else
+                       i915_gem_context_clear_recoverable(ctx);
+               break;
+
        case I915_CONTEXT_PARAM_PRIORITY:
                {
                        s64 priority = args->value;
@@ -1385,3 +1413,28 @@ out_unlock:
 #include "selftests/mock_context.c"
 #include "selftests/i915_gem_context.c"
 #endif
+
+static void i915_global_gem_context_shrink(void)
+{
+       kmem_cache_shrink(global.slab_luts);
+}
+
+static void i915_global_gem_context_exit(void)
+{
+       kmem_cache_destroy(global.slab_luts);
+}
+
+static struct i915_global_gem_context global = { {
+       .shrink = i915_global_gem_context_shrink,
+       .exit = i915_global_gem_context_exit,
+} };
+
+int __init i915_global_gem_context_init(void)
+{
+       global.slab_luts = KMEM_CACHE(i915_lut_handle, 0);
+       if (!global.slab_luts)
+               return -ENOMEM;
+
+       i915_global_register(&global.base);
+       return 0;
+}
index ca150a764c24d48547106108692e1ac5b2e9ed82..5a32c4b4816f87e144300beb14b06458c0445a35 100644 (file)
 #ifndef __I915_GEM_CONTEXT_H__
 #define __I915_GEM_CONTEXT_H__
 
-#include <linux/bitops.h>
-#include <linux/list.h>
-#include <linux/radix-tree.h>
+#include "i915_gem_context_types.h"
 
 #include "i915_gem.h"
 #include "i915_scheduler.h"
+#include "intel_context.h"
 #include "intel_device_info.h"
 
-struct pid;
-
 struct drm_device;
 struct drm_file;
 
-struct drm_i915_private;
-struct drm_i915_file_private;
-struct i915_hw_ppgtt;
-struct i915_request;
-struct i915_vma;
-struct intel_ring;
-
-#define DEFAULT_CONTEXT_HANDLE 0
-
-struct intel_context;
-
-struct intel_context_ops {
-       void (*unpin)(struct intel_context *ce);
-       void (*destroy)(struct intel_context *ce);
-};
-
-/*
- * Powergating configuration for a particular (context,engine).
- */
-struct intel_sseu {
-       u8 slice_mask;
-       u8 subslice_mask;
-       u8 min_eus_per_subslice;
-       u8 max_eus_per_subslice;
-};
-
-/**
- * struct i915_gem_context - client state
- *
- * The struct i915_gem_context represents the combined view of the driver and
- * logical hardware state for a particular client.
- */
-struct i915_gem_context {
-       /** i915: i915 device backpointer */
-       struct drm_i915_private *i915;
-
-       /** file_priv: owning file descriptor */
-       struct drm_i915_file_private *file_priv;
-
-       /**
-        * @ppgtt: unique address space (GTT)
-        *
-        * In full-ppgtt mode, each context has its own address space ensuring
-        * complete seperation of one client from all others.
-        *
-        * In other modes, this is a NULL pointer with the expectation that
-        * the caller uses the shared global GTT.
-        */
-       struct i915_hw_ppgtt *ppgtt;
-
-       /**
-        * @pid: process id of creator
-        *
-        * Note that who created the context may not be the principle user,
-        * as the context may be shared across a local socket. However,
-        * that should only affect the default context, all contexts created
-        * explicitly by the client are expected to be isolated.
-        */
-       struct pid *pid;
-
-       /**
-        * @name: arbitrary name
-        *
-        * A name is constructed for the context from the creator's process
-        * name, pid and user handle in order to uniquely identify the
-        * context in messages.
-        */
-       const char *name;
-
-       /** link: place with &drm_i915_private.context_list */
-       struct list_head link;
-       struct llist_node free_link;
-
-       /**
-        * @ref: reference count
-        *
-        * A reference to a context is held by both the client who created it
-        * and on each request submitted to the hardware using the request
-        * (to ensure the hardware has access to the state until it has
-        * finished all pending writes). See i915_gem_context_get() and
-        * i915_gem_context_put() for access.
-        */
-       struct kref ref;
-
-       /**
-        * @rcu: rcu_head for deferred freeing.
-        */
-       struct rcu_head rcu;
-
-       /**
-        * @user_flags: small set of booleans controlled by the user
-        */
-       unsigned long user_flags;
-#define UCONTEXT_NO_ZEROMAP            0
-#define UCONTEXT_NO_ERROR_CAPTURE      1
-#define UCONTEXT_BANNABLE              2
-
-       /**
-        * @flags: small set of booleans
-        */
-       unsigned long flags;
-#define CONTEXT_BANNED                 0
-#define CONTEXT_CLOSED                 1
-#define CONTEXT_FORCE_SINGLE_SUBMISSION        2
-
-       /**
-        * @hw_id: - unique identifier for the context
-        *
-        * The hardware needs to uniquely identify the context for a few
-        * functions like fault reporting, PASID, scheduling. The
-        * &drm_i915_private.context_hw_ida is used to assign a unqiue
-        * id for the lifetime of the context.
-        *
-        * @hw_id_pin_count: - number of times this context had been pinned
-        * for use (should be, at most, once per engine).
-        *
-        * @hw_id_link: - all contexts with an assigned id are tracked
-        * for possible repossession.
-        */
-       unsigned int hw_id;
-       atomic_t hw_id_pin_count;
-       struct list_head hw_id_link;
-
-       /**
-        * @user_handle: userspace identifier
-        *
-        * A unique per-file identifier is generated from
-        * &drm_i915_file_private.contexts.
-        */
-       u32 user_handle;
-
-       struct i915_sched_attr sched;
-
-       /** engine: per-engine logical HW state */
-       struct intel_context {
-               struct i915_gem_context *gem_context;
-               struct intel_engine_cs *active;
-               struct list_head signal_link;
-               struct list_head signals;
-               struct i915_vma *state;
-               struct intel_ring *ring;
-               u32 *lrc_reg_state;
-               u64 lrc_desc;
-               int pin_count;
-
-               /**
-                * active_tracker: Active tracker for the external rq activity
-                * on this intel_context object.
-                */
-               struct i915_active_request active_tracker;
-
-               const struct intel_context_ops *ops;
-
-               /** sseu: Control eu/slice partitioning */
-               struct intel_sseu sseu;
-       } __engine[I915_NUM_ENGINES];
-
-       /** ring_size: size for allocating the per-engine ring buffer */
-       u32 ring_size;
-       /** desc_template: invariant fields for the HW context descriptor */
-       u32 desc_template;
-
-       /** guilty_count: How many times this context has caused a GPU hang. */
-       atomic_t guilty_count;
-       /**
-        * @active_count: How many times this context was active during a GPU
-        * hang, but did not cause it.
-        */
-       atomic_t active_count;
-
-#define CONTEXT_SCORE_GUILTY           10
-#define CONTEXT_SCORE_BAN_THRESHOLD    40
-       /** ban_score: Accumulated score of all hangs caused by this context. */
-       atomic_t ban_score;
-
-       /** remap_slice: Bitmask of cache lines that need remapping */
-       u8 remap_slice;
-
-       /** handles_vma: rbtree to look up our context specific obj/vma for
-        * the user handle. (user handles are per fd, but the binding is
-        * per vm, which may be one per context or shared with the global GTT)
-        */
-       struct radix_tree_root handles_vma;
-
-       /** handles_list: reverse list of all the rbtree entries in use for
-        * this context, which allows us to free all the allocations on
-        * context close.
-        */
-       struct list_head handles_list;
-};
-
 static inline bool i915_gem_context_is_closed(const struct i915_gem_context *ctx)
 {
        return test_bit(CONTEXT_CLOSED, &ctx->flags);
@@ -270,6 +76,21 @@ static inline void i915_gem_context_clear_bannable(struct i915_gem_context *ctx)
        clear_bit(UCONTEXT_BANNABLE, &ctx->user_flags);
 }
 
+static inline bool i915_gem_context_is_recoverable(const struct i915_gem_context *ctx)
+{
+       return test_bit(UCONTEXT_RECOVERABLE, &ctx->user_flags);
+}
+
+static inline void i915_gem_context_set_recoverable(struct i915_gem_context *ctx)
+{
+       set_bit(UCONTEXT_RECOVERABLE, &ctx->user_flags);
+}
+
+static inline void i915_gem_context_clear_recoverable(struct i915_gem_context *ctx)
+{
+       clear_bit(UCONTEXT_RECOVERABLE, &ctx->user_flags);
+}
+
 static inline bool i915_gem_context_is_banned(const struct i915_gem_context *ctx)
 {
        return test_bit(CONTEXT_BANNED, &ctx->flags);
@@ -315,35 +136,6 @@ static inline bool i915_gem_context_is_kernel(struct i915_gem_context *ctx)
        return !ctx->file_priv;
 }
 
-static inline struct intel_context *
-to_intel_context(struct i915_gem_context *ctx,
-                const struct intel_engine_cs *engine)
-{
-       return &ctx->__engine[engine->id];
-}
-
-static inline struct intel_context *
-intel_context_pin(struct i915_gem_context *ctx, struct intel_engine_cs *engine)
-{
-       return engine->context_pin(engine, ctx);
-}
-
-static inline void __intel_context_pin(struct intel_context *ce)
-{
-       GEM_BUG_ON(!ce->pin_count);
-       ce->pin_count++;
-}
-
-static inline void intel_context_unpin(struct intel_context *ce)
-{
-       GEM_BUG_ON(!ce->pin_count);
-       if (--ce->pin_count)
-               return;
-
-       GEM_BUG_ON(!ce->ops);
-       ce->ops->unpin(ce);
-}
-
 /* i915_gem_context.c */
 int __must_check i915_gem_contexts_init(struct drm_i915_private *dev_priv);
 void i915_gem_contexts_lost(struct drm_i915_private *dev_priv);
@@ -354,7 +146,8 @@ int i915_gem_context_open(struct drm_i915_private *i915,
 void i915_gem_context_close(struct drm_file *file);
 
 int i915_switch_context(struct i915_request *rq);
-int i915_gem_switch_to_kernel_context(struct drm_i915_private *dev_priv);
+int i915_gem_switch_to_kernel_context(struct drm_i915_private *i915,
+                                     unsigned long engine_mask);
 
 void i915_gem_context_release(struct kref *ctx_ref);
 struct i915_gem_context *
@@ -386,8 +179,7 @@ static inline void i915_gem_context_put(struct i915_gem_context *ctx)
        kref_put(&ctx->ref, i915_gem_context_release);
 }
 
-void intel_context_init(struct intel_context *ce,
-                       struct i915_gem_context *ctx,
-                       struct intel_engine_cs *engine);
+struct i915_lut_handle *i915_lut_handle_alloc(void);
+void i915_lut_handle_free(struct i915_lut_handle *lut);
 
 #endif /* !__I915_GEM_CONTEXT_H__ */
diff --git a/drivers/gpu/drm/i915/i915_gem_context_types.h b/drivers/gpu/drm/i915/i915_gem_context_types.h
new file mode 100644 (file)
index 0000000..2bf1973
--- /dev/null
@@ -0,0 +1,182 @@
+/*
+ * SPDX-License-Identifier: MIT
+ *
+ * Copyright © 2019 Intel Corporation
+ */
+
+#ifndef __I915_GEM_CONTEXT_TYPES_H__
+#define __I915_GEM_CONTEXT_TYPES_H__
+
+#include <linux/atomic.h>
+#include <linux/list.h>
+#include <linux/llist.h>
+#include <linux/kref.h>
+#include <linux/mutex.h>
+#include <linux/radix-tree.h>
+#include <linux/rbtree.h>
+#include <linux/rcupdate.h>
+#include <linux/types.h>
+
+#include "i915_scheduler.h"
+#include "intel_context_types.h"
+
+struct pid;
+
+struct drm_i915_private;
+struct drm_i915_file_private;
+struct i915_hw_ppgtt;
+struct i915_timeline;
+struct intel_ring;
+
+/**
+ * struct i915_gem_context - client state
+ *
+ * The struct i915_gem_context represents the combined view of the driver and
+ * logical hardware state for a particular client.
+ */
+struct i915_gem_context {
+       /** i915: i915 device backpointer */
+       struct drm_i915_private *i915;
+
+       /** file_priv: owning file descriptor */
+       struct drm_i915_file_private *file_priv;
+
+       /**
+        * @ppgtt: unique address space (GTT)
+        *
+        * In full-ppgtt mode, each context has its own address space ensuring
+        * complete seperation of one client from all others.
+        *
+        * In other modes, this is a NULL pointer with the expectation that
+        * the caller uses the shared global GTT.
+        */
+       struct i915_hw_ppgtt *ppgtt;
+
+       /**
+        * @pid: process id of creator
+        *
+        * Note that who created the context may not be the principle user,
+        * as the context may be shared across a local socket. However,
+        * that should only affect the default context, all contexts created
+        * explicitly by the client are expected to be isolated.
+        */
+       struct pid *pid;
+
+       /**
+        * @name: arbitrary name
+        *
+        * A name is constructed for the context from the creator's process
+        * name, pid and user handle in order to uniquely identify the
+        * context in messages.
+        */
+       const char *name;
+
+       /** link: place with &drm_i915_private.context_list */
+       struct list_head link;
+       struct llist_node free_link;
+
+       /**
+        * @ref: reference count
+        *
+        * A reference to a context is held by both the client who created it
+        * and on each request submitted to the hardware using the request
+        * (to ensure the hardware has access to the state until it has
+        * finished all pending writes). See i915_gem_context_get() and
+        * i915_gem_context_put() for access.
+        */
+       struct kref ref;
+
+       /**
+        * @rcu: rcu_head for deferred freeing.
+        */
+       struct rcu_head rcu;
+
+       /**
+        * @user_flags: small set of booleans controlled by the user
+        */
+       unsigned long user_flags;
+#define UCONTEXT_NO_ZEROMAP            0
+#define UCONTEXT_NO_ERROR_CAPTURE      1
+#define UCONTEXT_BANNABLE              2
+#define UCONTEXT_RECOVERABLE           3
+
+       /**
+        * @flags: small set of booleans
+        */
+       unsigned long flags;
+#define CONTEXT_BANNED                 0
+#define CONTEXT_CLOSED                 1
+#define CONTEXT_FORCE_SINGLE_SUBMISSION        2
+
+       /**
+        * @hw_id: - unique identifier for the context
+        *
+        * The hardware needs to uniquely identify the context for a few
+        * functions like fault reporting, PASID, scheduling. The
+        * &drm_i915_private.context_hw_ida is used to assign a unqiue
+        * id for the lifetime of the context.
+        *
+        * @hw_id_pin_count: - number of times this context had been pinned
+        * for use (should be, at most, once per engine).
+        *
+        * @hw_id_link: - all contexts with an assigned id are tracked
+        * for possible repossession.
+        */
+       unsigned int hw_id;
+       atomic_t hw_id_pin_count;
+       struct list_head hw_id_link;
+
+       struct list_head active_engines;
+       struct mutex mutex;
+
+       /**
+        * @user_handle: userspace identifier
+        *
+        * A unique per-file identifier is generated from
+        * &drm_i915_file_private.contexts.
+        */
+       u32 user_handle;
+#define DEFAULT_CONTEXT_HANDLE 0
+
+       struct i915_sched_attr sched;
+
+       /** hw_contexts: per-engine logical HW state */
+       struct rb_root hw_contexts;
+       spinlock_t hw_contexts_lock;
+
+       /** ring_size: size for allocating the per-engine ring buffer */
+       u32 ring_size;
+       /** desc_template: invariant fields for the HW context descriptor */
+       u32 desc_template;
+
+       /** guilty_count: How many times this context has caused a GPU hang. */
+       atomic_t guilty_count;
+       /**
+        * @active_count: How many times this context was active during a GPU
+        * hang, but did not cause it.
+        */
+       atomic_t active_count;
+
+       /**
+        * @hang_timestamp: The last time(s) this context caused a GPU hang
+        */
+       unsigned long hang_timestamp[2];
+#define CONTEXT_FAST_HANG_JIFFIES (120 * HZ) /* 3 hangs within 120s? Banned! */
+
+       /** remap_slice: Bitmask of cache lines that need remapping */
+       u8 remap_slice;
+
+       /** handles_vma: rbtree to look up our context specific obj/vma for
+        * the user handle. (user handles are per fd, but the binding is
+        * per vm, which may be one per context or shared with the global GTT)
+        */
+       struct radix_tree_root handles_vma;
+
+       /** handles_list: reverse list of all the rbtree entries in use for
+        * this context, which allows us to free all the allocations on
+        * context close.
+        */
+       struct list_head handles_list;
+};
+
+#endif /* __I915_GEM_CONTEXT_TYPES_H__ */
index 02f7298bfe57cda5ad1d344adb0ff99079267c00..33181678990ec764985dd9593b09f47989455f87 100644 (file)
@@ -300,7 +300,7 @@ struct drm_gem_object *i915_gem_prime_import(struct drm_device *dev,
 
        get_dma_buf(dma_buf);
 
-       obj = i915_gem_object_alloc(to_i915(dev));
+       obj = i915_gem_object_alloc();
        if (obj == NULL) {
                ret = -ENOMEM;
                goto fail_detach;
index 68d74c50ac392dba58ea511386912790399c2a13..060f5903544a042427f195e95c0bf057b129a029 100644 (file)
@@ -38,31 +38,21 @@ I915_SELFTEST_DECLARE(static struct igt_evict_ctl {
 
 static bool ggtt_is_idle(struct drm_i915_private *i915)
 {
-       struct intel_engine_cs *engine;
-       enum intel_engine_id id;
-
-       if (i915->gt.active_requests)
-              return false;
-
-       for_each_engine(engine, i915, id) {
-              if (!intel_engine_has_kernel_context(engine))
-                      return false;
-       }
-
-       return true;
+       return !i915->gt.active_requests;
 }
 
 static int ggtt_flush(struct drm_i915_private *i915)
 {
        int err;
 
-       /* Not everything in the GGTT is tracked via vma (otherwise we
+       /*
+        * Not everything in the GGTT is tracked via vma (otherwise we
         * could evict as required with minimal stalling) so we are forced
         * to idle the GPU and explicitly retire outstanding requests in
         * the hopes that we can then remove contexts and the like only
         * bound by their active reference.
         */
-       err = i915_gem_switch_to_kernel_context(i915);
+       err = i915_gem_switch_to_kernel_context(i915, i915->gt.active_engines);
        if (err)
                return err;
 
index 02adcaf6ebea69086aa07be57fed55b347636c94..ee6d301a96277616788dbe94365149d50c5c7338 100644 (file)
@@ -794,8 +794,8 @@ static int eb_wait_for_ring(const struct i915_execbuffer *eb)
         * keeping all of their resources pinned.
         */
 
-       ce = to_intel_context(eb->ctx, eb->engine);
-       if (!ce->ring) /* first use, assume empty! */
+       ce = intel_context_lookup(eb->ctx, eb->engine);
+       if (!ce || !ce->ring) /* first use, assume empty! */
                return 0;
 
        rq = __eb_wait_for_ring(ce->ring);
@@ -849,12 +849,12 @@ static int eb_lookup_vmas(struct i915_execbuffer *eb)
                }
 
                vma = i915_vma_instance(obj, eb->vm, NULL);
-               if (unlikely(IS_ERR(vma))) {
+               if (IS_ERR(vma)) {
                        err = PTR_ERR(vma);
                        goto err_obj;
                }
 
-               lut = kmem_cache_alloc(eb->i915->luts, GFP_KERNEL);
+               lut = i915_lut_handle_alloc();
                if (unlikely(!lut)) {
                        err = -ENOMEM;
                        goto err_obj;
@@ -862,7 +862,7 @@ static int eb_lookup_vmas(struct i915_execbuffer *eb)
 
                err = radix_tree_insert(handles_vma, handle, vma);
                if (unlikely(err)) {
-                       kmem_cache_free(eb->i915->luts, lut);
+                       i915_lut_handle_free(lut);
                        goto err_obj;
                }
 
@@ -1957,7 +1957,7 @@ static int i915_reset_gen7_sol_offsets(struct i915_request *rq)
        u32 *cs;
        int i;
 
-       if (!IS_GEN(rq->i915, 7) || rq->engine->id != RCS) {
+       if (!IS_GEN(rq->i915, 7) || rq->engine->id != RCS0) {
                DRM_DEBUG("sol reset is gen7/rcs only\n");
                return -EINVAL;
        }
@@ -2082,11 +2082,11 @@ gen8_dispatch_bsd_engine(struct drm_i915_private *dev_priv,
 #define I915_USER_RINGS (4)
 
 static const enum intel_engine_id user_ring_map[I915_USER_RINGS + 1] = {
-       [I915_EXEC_DEFAULT]     = RCS,
-       [I915_EXEC_RENDER]      = RCS,
-       [I915_EXEC_BLT]         = BCS,
-       [I915_EXEC_BSD]         = VCS,
-       [I915_EXEC_VEBOX]       = VECS
+       [I915_EXEC_DEFAULT]     = RCS0,
+       [I915_EXEC_RENDER]      = RCS0,
+       [I915_EXEC_BLT]         = BCS0,
+       [I915_EXEC_BSD]         = VCS0,
+       [I915_EXEC_VEBOX]       = VECS0
 };
 
 static struct intel_engine_cs *
@@ -2109,7 +2109,7 @@ eb_select_engine(struct drm_i915_private *dev_priv,
                return NULL;
        }
 
-       if (user_ring_id == I915_EXEC_BSD && HAS_BSD2(dev_priv)) {
+       if (user_ring_id == I915_EXEC_BSD && HAS_ENGINE(dev_priv, VCS1)) {
                unsigned int bsd_idx = args->flags & I915_EXEC_BSD_MASK;
 
                if (bsd_idx == I915_EXEC_BSD_DEFAULT) {
@@ -2312,10 +2312,6 @@ i915_gem_do_execbuffer(struct drm_device *dev,
        if (args->flags & I915_EXEC_IS_PINNED)
                eb.batch_flags |= I915_DISPATCH_PINNED;
 
-       eb.engine = eb_select_engine(eb.i915, file, args);
-       if (!eb.engine)
-               return -EINVAL;
-
        if (args->flags & I915_EXEC_FENCE_IN) {
                in_fence = sync_file_get_fence(lower_32_bits(args->rsvd2));
                if (!in_fence)
@@ -2340,6 +2336,12 @@ i915_gem_do_execbuffer(struct drm_device *dev,
        if (unlikely(err))
                goto err_destroy;
 
+       eb.engine = eb_select_engine(eb.i915, file, args);
+       if (!eb.engine) {
+               err = -EINVAL;
+               goto err_engine;
+       }
+
        /*
         * Take a local wakeref for preparing to dispatch the execbuf as
         * we expect to access the hardware fairly frequently in the
@@ -2505,6 +2507,7 @@ err_unlock:
        mutex_unlock(&dev->struct_mutex);
 err_rpm:
        intel_runtime_pm_put(eb.i915, wakeref);
+err_engine:
        i915_gem_context_put(eb.ctx);
 err_destroy:
        eb_destroy(&eb);
index e037e94792f3530e202e66c8d0b5c915bf49e454..65624b8e4d159a783937190d1ad5d68a368b51ec 100644 (file)
@@ -210,6 +210,7 @@ static int fence_update(struct drm_i915_fence_reg *fence,
                        struct i915_vma *vma)
 {
        intel_wakeref_t wakeref;
+       struct i915_vma *old;
        int ret;
 
        if (vma) {
@@ -229,49 +230,55 @@ static int fence_update(struct drm_i915_fence_reg *fence,
                        return ret;
        }
 
-       if (fence->vma) {
-               struct i915_vma *old = fence->vma;
-
+       old = xchg(&fence->vma, NULL);
+       if (old) {
                ret = i915_active_request_retire(&old->last_fence,
                                             &old->obj->base.dev->struct_mutex);
-               if (ret)
+               if (ret) {
+                       fence->vma = old;
                        return ret;
+               }
 
                i915_vma_flush_writes(old);
-       }
 
-       if (fence->vma && fence->vma != vma) {
-               /* Ensure that all userspace CPU access is completed before
+               /*
+                * Ensure that all userspace CPU access is completed before
                 * stealing the fence.
                 */
-               GEM_BUG_ON(fence->vma->fence != fence);
-               i915_vma_revoke_mmap(fence->vma);
-
-               fence->vma->fence = NULL;
-               fence->vma = NULL;
+               if (old != vma) {
+                       GEM_BUG_ON(old->fence != fence);
+                       i915_vma_revoke_mmap(old);
+                       old->fence = NULL;
+               }
 
                list_move(&fence->link, &fence->i915->mm.fence_list);
        }
 
-       /* We only need to update the register itself if the device is awake.
+       /*
+        * We only need to update the register itself if the device is awake.
         * If the device is currently powered down, we will defer the write
         * to the runtime resume, see i915_gem_restore_fences().
+        *
+        * This only works for removing the fence register, on acquisition
+        * the caller must hold the rpm wakeref. The fence register must
+        * be cleared before we can use any other fences to ensure that
+        * the new fences do not overlap the elided clears, confusing HW.
         */
        wakeref = intel_runtime_pm_get_if_in_use(fence->i915);
-       if (wakeref) {
-               fence_write(fence, vma);
-               intel_runtime_pm_put(fence->i915, wakeref);
+       if (!wakeref) {
+               GEM_BUG_ON(vma);
+               return 0;
        }
 
-       if (vma) {
-               if (fence->vma != vma) {
-                       vma->fence = fence;
-                       fence->vma = vma;
-               }
+       WRITE_ONCE(fence->vma, vma);
+       fence_write(fence, vma);
 
+       if (vma) {
+               vma->fence = fence;
                list_move_tail(&fence->link, &fence->i915->mm.fence_list);
        }
 
+       intel_runtime_pm_put(fence->i915, wakeref);
        return 0;
 }
 
@@ -435,32 +442,6 @@ void i915_unreserve_fence(struct drm_i915_fence_reg *fence)
        list_add(&fence->link, &fence->i915->mm.fence_list);
 }
 
-/**
- * i915_gem_revoke_fences - revoke fence state
- * @dev_priv: i915 device private
- *
- * Removes all GTT mmappings via the fence registers. This forces any user
- * of the fence to reacquire that fence before continuing with their access.
- * One use is during GPU reset where the fence register is lost and we need to
- * revoke concurrent userspace access via GTT mmaps until the hardware has been
- * reset and the fence registers have been restored.
- */
-void i915_gem_revoke_fences(struct drm_i915_private *dev_priv)
-{
-       int i;
-
-       lockdep_assert_held(&dev_priv->drm.struct_mutex);
-
-       for (i = 0; i < dev_priv->num_fence_regs; i++) {
-               struct drm_i915_fence_reg *fence = &dev_priv->fence_regs[i];
-
-               GEM_BUG_ON(fence->vma && fence->vma->fence != fence);
-
-               if (fence->vma)
-                       i915_vma_revoke_mmap(fence->vma);
-       }
-}
-
 /**
  * i915_gem_restore_fences - restore fence state
  * @dev_priv: i915 device private
@@ -473,9 +454,10 @@ void i915_gem_restore_fences(struct drm_i915_private *dev_priv)
 {
        int i;
 
+       rcu_read_lock(); /* keep obj alive as we dereference */
        for (i = 0; i < dev_priv->num_fence_regs; i++) {
                struct drm_i915_fence_reg *reg = &dev_priv->fence_regs[i];
-               struct i915_vma *vma = reg->vma;
+               struct i915_vma *vma = READ_ONCE(reg->vma);
 
                GEM_BUG_ON(vma && vma->fence != reg);
 
@@ -483,18 +465,12 @@ void i915_gem_restore_fences(struct drm_i915_private *dev_priv)
                 * Commit delayed tiling changes if we have an object still
                 * attached to the fence, otherwise just clear the fence.
                 */
-               if (vma && !i915_gem_object_is_tiled(vma->obj)) {
-                       GEM_BUG_ON(!reg->dirty);
-                       GEM_BUG_ON(i915_vma_has_userfault(vma));
-
-                       list_move(&reg->link, &dev_priv->mm.fence_list);
-                       vma->fence = NULL;
+               if (vma && !i915_gem_object_is_tiled(vma->obj))
                        vma = NULL;
-               }
 
                fence_write(reg, vma);
-               reg->vma = vma;
        }
+       rcu_read_unlock();
 }
 
 /**
index d646d37eec2f8a751a3a7f1642bae26841d2b24f..b8055c8d4e719a2dca74eba377f740093cdbe3bd 100644 (file)
@@ -584,7 +584,7 @@ setup_scratch_page(struct i915_address_space *vm, gfp_t gfp)
         * for all.
         */
        size = I915_GTT_PAGE_SIZE_4K;
-       if (i915_vm_is_48bit(vm) &&
+       if (i915_vm_is_4lvl(vm) &&
            HAS_PAGE_SIZES(vm->i915, I915_GTT_PAGE_SIZE_64K)) {
                size = I915_GTT_PAGE_SIZE_64K;
                gfp |= __GFP_NOWARN;
@@ -613,7 +613,7 @@ setup_scratch_page(struct i915_address_space *vm, gfp_t gfp)
 
                vm->scratch_page.page = page;
                vm->scratch_page.daddr = addr;
-               vm->scratch_page.order = order;
+               vm->scratch_order = order;
                return 0;
 
 unmap_page:
@@ -632,10 +632,11 @@ skip:
 static void cleanup_scratch_page(struct i915_address_space *vm)
 {
        struct i915_page_dma *p = &vm->scratch_page;
+       int order = vm->scratch_order;
 
-       dma_unmap_page(vm->dma, p->daddr, BIT(p->order) << PAGE_SHIFT,
+       dma_unmap_page(vm->dma, p->daddr, BIT(order) << PAGE_SHIFT,
                       PCI_DMA_BIDIRECTIONAL);
-       __free_pages(p->page, p->order);
+       __free_pages(p->page, order);
 }
 
 static struct i915_page_table *alloc_pt(struct i915_address_space *vm)
@@ -726,18 +727,13 @@ static void __pdp_fini(struct i915_page_directory_pointer *pdp)
        pdp->page_directory = NULL;
 }
 
-static inline bool use_4lvl(const struct i915_address_space *vm)
-{
-       return i915_vm_is_48bit(vm);
-}
-
 static struct i915_page_directory_pointer *
 alloc_pdp(struct i915_address_space *vm)
 {
        struct i915_page_directory_pointer *pdp;
        int ret = -ENOMEM;
 
-       GEM_BUG_ON(!use_4lvl(vm));
+       GEM_BUG_ON(!i915_vm_is_4lvl(vm));
 
        pdp = kzalloc(sizeof(*pdp), GFP_KERNEL);
        if (!pdp)
@@ -766,7 +762,7 @@ static void free_pdp(struct i915_address_space *vm,
 {
        __pdp_fini(pdp);
 
-       if (!use_4lvl(vm))
+       if (!i915_vm_is_4lvl(vm))
                return;
 
        cleanup_px(vm, pdp);
@@ -791,14 +787,15 @@ static void gen8_initialize_pml4(struct i915_address_space *vm,
        memset_p((void **)pml4->pdps, vm->scratch_pdp, GEN8_PML4ES_PER_PML4);
 }
 
-/* PDE TLBs are a pain to invalidate on GEN8+. When we modify
+/*
+ * PDE TLBs are a pain to invalidate on GEN8+. When we modify
  * the page table structures, we mark them dirty so that
  * context switching/execlist queuing code takes extra steps
  * to ensure that tlbs are flushed.
  */
 static void mark_tlbs_dirty(struct i915_hw_ppgtt *ppgtt)
 {
-       ppgtt->pd_dirty_rings = INTEL_INFO(ppgtt->vm.i915)->ring_mask;
+       ppgtt->pd_dirty_engines = ALL_ENGINES;
 }
 
 /* Removes entries from a single page table, releasing it if it's empty.
@@ -809,8 +806,6 @@ static bool gen8_ppgtt_clear_pt(const struct i915_address_space *vm,
                                u64 start, u64 length)
 {
        unsigned int num_entries = gen8_pte_count(start, length);
-       unsigned int pte = gen8_pte_index(start);
-       unsigned int pte_end = pte + num_entries;
        gen8_pte_t *vaddr;
 
        GEM_BUG_ON(num_entries > pt->used_ptes);
@@ -820,8 +815,7 @@ static bool gen8_ppgtt_clear_pt(const struct i915_address_space *vm,
                return true;
 
        vaddr = kmap_atomic_px(pt);
-       while (pte < pte_end)
-               vaddr[pte++] = vm->scratch_pte;
+       memset64(vaddr + gen8_pte_index(start), vm->scratch_pte, num_entries);
        kunmap_atomic(vaddr);
 
        return false;
@@ -872,7 +866,7 @@ static void gen8_ppgtt_set_pdpe(struct i915_address_space *vm,
        gen8_ppgtt_pdpe_t *vaddr;
 
        pdp->page_directory[pdpe] = pd;
-       if (!use_4lvl(vm))
+       if (!i915_vm_is_4lvl(vm))
                return;
 
        vaddr = kmap_atomic_px(pdp);
@@ -937,7 +931,7 @@ static void gen8_ppgtt_clear_4lvl(struct i915_address_space *vm,
        struct i915_page_directory_pointer *pdp;
        unsigned int pml4e;
 
-       GEM_BUG_ON(!use_4lvl(vm));
+       GEM_BUG_ON(!i915_vm_is_4lvl(vm));
 
        gen8_for_each_pml4e(pdp, pml4, start, length, pml4e) {
                GEM_BUG_ON(pdp == vm->scratch_pdp);
@@ -1219,7 +1213,7 @@ static int gen8_init_scratch(struct i915_address_space *vm)
 
                GEM_BUG_ON(!clone->has_read_only);
 
-               vm->scratch_page.order = clone->scratch_page.order;
+               vm->scratch_order = clone->scratch_order;
                vm->scratch_pte = clone->scratch_pte;
                vm->scratch_pt  = clone->scratch_pt;
                vm->scratch_pd  = clone->scratch_pd;
@@ -1248,7 +1242,7 @@ static int gen8_init_scratch(struct i915_address_space *vm)
                goto free_pt;
        }
 
-       if (use_4lvl(vm)) {
+       if (i915_vm_is_4lvl(vm)) {
                vm->scratch_pdp = alloc_pdp(vm);
                if (IS_ERR(vm->scratch_pdp)) {
                        ret = PTR_ERR(vm->scratch_pdp);
@@ -1258,7 +1252,7 @@ static int gen8_init_scratch(struct i915_address_space *vm)
 
        gen8_initialize_pt(vm, vm->scratch_pt);
        gen8_initialize_pd(vm, vm->scratch_pd);
-       if (use_4lvl(vm))
+       if (i915_vm_is_4lvl(vm))
                gen8_initialize_pdp(vm, vm->scratch_pdp);
 
        return 0;
@@ -1280,7 +1274,7 @@ static int gen8_ppgtt_notify_vgt(struct i915_hw_ppgtt *ppgtt, bool create)
        enum vgt_g2v_type msg;
        int i;
 
-       if (use_4lvl(vm)) {
+       if (i915_vm_is_4lvl(vm)) {
                const u64 daddr = px_dma(&ppgtt->pml4);
 
                I915_WRITE(vgtif_reg(pdp[0].lo), lower_32_bits(daddr));
@@ -1310,7 +1304,7 @@ static void gen8_free_scratch(struct i915_address_space *vm)
        if (!vm->scratch_page.daddr)
                return;
 
-       if (use_4lvl(vm))
+       if (i915_vm_is_4lvl(vm))
                free_pdp(vm, vm->scratch_pdp);
        free_pd(vm, vm->scratch_pd);
        free_pt(vm, vm->scratch_pt);
@@ -1356,7 +1350,7 @@ static void gen8_ppgtt_cleanup(struct i915_address_space *vm)
        if (intel_vgpu_active(dev_priv))
                gen8_ppgtt_notify_vgt(ppgtt, false);
 
-       if (use_4lvl(vm))
+       if (i915_vm_is_4lvl(vm))
                gen8_ppgtt_cleanup_4lvl(ppgtt);
        else
                gen8_ppgtt_cleanup_3lvl(&ppgtt->vm, &ppgtt->pdp);
@@ -1519,6 +1513,23 @@ unwind:
        return -ENOMEM;
 }
 
+static void ppgtt_init(struct drm_i915_private *i915,
+                      struct i915_hw_ppgtt *ppgtt)
+{
+       kref_init(&ppgtt->ref);
+
+       ppgtt->vm.i915 = i915;
+       ppgtt->vm.dma = &i915->drm.pdev->dev;
+       ppgtt->vm.total = BIT_ULL(INTEL_INFO(i915)->ppgtt_size);
+
+       i915_address_space_init(&ppgtt->vm, VM_CLASS_PPGTT);
+
+       ppgtt->vm.vma_ops.bind_vma    = ppgtt_bind_vma;
+       ppgtt->vm.vma_ops.unbind_vma  = ppgtt_unbind_vma;
+       ppgtt->vm.vma_ops.set_pages   = ppgtt_set_pages;
+       ppgtt->vm.vma_ops.clear_pages = clear_pages;
+}
+
 /*
  * GEN8 legacy ppgtt programming is accomplished through a max 4 PDP registers
  * with a net effect resembling a 2-level page table in normal x86 terms. Each
@@ -1535,20 +1546,11 @@ static struct i915_hw_ppgtt *gen8_ppgtt_create(struct drm_i915_private *i915)
        if (!ppgtt)
                return ERR_PTR(-ENOMEM);
 
-       kref_init(&ppgtt->ref);
-
-       ppgtt->vm.i915 = i915;
-       ppgtt->vm.dma = &i915->drm.pdev->dev;
-
-       ppgtt->vm.total = HAS_FULL_48BIT_PPGTT(i915) ?
-               1ULL << 48 :
-               1ULL << 32;
+       ppgtt_init(i915, ppgtt);
 
        /* From bdw, there is support for read-only pages in the PPGTT. */
        ppgtt->vm.has_read_only = true;
 
-       i915_address_space_init(&ppgtt->vm, VM_CLASS_PPGTT);
-
        /* There are only few exceptions for gen >=6. chv and bxt.
         * And we are not sure about the latter so play safe for now.
         */
@@ -1559,7 +1561,7 @@ static struct i915_hw_ppgtt *gen8_ppgtt_create(struct drm_i915_private *i915)
        if (err)
                goto err_free;
 
-       if (use_4lvl(&ppgtt->vm)) {
+       if (i915_vm_is_4lvl(&ppgtt->vm)) {
                err = setup_px(&ppgtt->vm, &ppgtt->pml4);
                if (err)
                        goto err_scratch;
@@ -1592,11 +1594,6 @@ static struct i915_hw_ppgtt *gen8_ppgtt_create(struct drm_i915_private *i915)
 
        ppgtt->vm.cleanup = gen8_ppgtt_cleanup;
 
-       ppgtt->vm.vma_ops.bind_vma    = ppgtt_bind_vma;
-       ppgtt->vm.vma_ops.unbind_vma  = ppgtt_unbind_vma;
-       ppgtt->vm.vma_ops.set_pages   = ppgtt_set_pages;
-       ppgtt->vm.vma_ops.clear_pages = clear_pages;
-
        return ppgtt;
 
 err_scratch:
@@ -1672,8 +1669,7 @@ static void gen6_ppgtt_clear_range(struct i915_address_space *vm,
 
        while (num_entries) {
                struct i915_page_table *pt = ppgtt->base.pd.page_table[pde++];
-               const unsigned int end = min(pte + num_entries, GEN6_PTES);
-               const unsigned int count = end - pte;
+               const unsigned int count = min(num_entries, GEN6_PTES - pte);
                gen6_pte_t *vaddr;
 
                GEM_BUG_ON(pt == vm->scratch_pt);
@@ -1693,9 +1689,7 @@ static void gen6_ppgtt_clear_range(struct i915_address_space *vm,
                 */
 
                vaddr = kmap_atomic_px(pt);
-               do {
-                       vaddr[pte++] = scratch_pte;
-               } while (pte < end);
+               memset32(vaddr + pte, scratch_pte, count);
                kunmap_atomic(vaddr);
 
                pte = 0;
@@ -1913,7 +1907,7 @@ static struct i915_vma *pd_vma_create(struct gen6_hw_ppgtt *ppgtt, int size)
        GEM_BUG_ON(!IS_ALIGNED(size, I915_GTT_PAGE_SIZE));
        GEM_BUG_ON(size > ggtt->vm.total);
 
-       vma = kmem_cache_zalloc(i915->vmas, GFP_KERNEL);
+       vma = i915_vma_alloc();
        if (!vma)
                return ERR_PTR(-ENOMEM);
 
@@ -1991,25 +1985,13 @@ static struct i915_hw_ppgtt *gen6_ppgtt_create(struct drm_i915_private *i915)
        if (!ppgtt)
                return ERR_PTR(-ENOMEM);
 
-       kref_init(&ppgtt->base.ref);
-
-       ppgtt->base.vm.i915 = i915;
-       ppgtt->base.vm.dma = &i915->drm.pdev->dev;
-
-       ppgtt->base.vm.total = I915_PDES * GEN6_PTES * I915_GTT_PAGE_SIZE;
-
-       i915_address_space_init(&ppgtt->base.vm, VM_CLASS_PPGTT);
+       ppgtt_init(i915, &ppgtt->base);
 
        ppgtt->base.vm.allocate_va_range = gen6_alloc_va_range;
        ppgtt->base.vm.clear_range = gen6_ppgtt_clear_range;
        ppgtt->base.vm.insert_entries = gen6_ppgtt_insert_entries;
        ppgtt->base.vm.cleanup = gen6_ppgtt_cleanup;
 
-       ppgtt->base.vm.vma_ops.bind_vma    = ppgtt_bind_vma;
-       ppgtt->base.vm.vma_ops.unbind_vma  = ppgtt_unbind_vma;
-       ppgtt->base.vm.vma_ops.set_pages   = ppgtt_set_pages;
-       ppgtt->base.vm.vma_ops.clear_pages = clear_pages;
-
        ppgtt->base.vm.pte_encode = ggtt->vm.pte_encode;
 
        err = gen6_ppgtt_init_scratch(ppgtt);
@@ -3701,7 +3683,7 @@ i915_get_ggtt_vma_pages(struct i915_vma *vma)
        }
 
        ret = 0;
-       if (unlikely(IS_ERR(vma->pages))) {
+       if (IS_ERR(vma->pages)) {
                ret = PTR_ERR(vma->pages);
                vma->pages = NULL;
                DRM_ERROR("Failed to get pages for VMA view type %u (%d)!\n",
index 03ade71b8d9a046a5c5581f38eb967018e3ab1a3..35f21a2ae36c6ab7fdc2a20215afeaa1053eff02 100644 (file)
@@ -213,7 +213,6 @@ struct i915_vma;
 
 struct i915_page_dma {
        struct page *page;
-       int order;
        union {
                dma_addr_t daddr;
 
@@ -293,6 +292,7 @@ struct i915_address_space {
 #define VM_CLASS_PPGTT 1
 
        u64 scratch_pte;
+       int scratch_order;
        struct i915_page_dma scratch_page;
        struct i915_page_table *scratch_pt;
        struct i915_page_directory *scratch_pd;
@@ -348,7 +348,7 @@ struct i915_address_space {
 #define i915_is_ggtt(vm) ((vm)->is_ggtt)
 
 static inline bool
-i915_vm_is_48bit(const struct i915_address_space *vm)
+i915_vm_is_4lvl(const struct i915_address_space *vm)
 {
        return (vm->total - 1) >> 32;
 }
@@ -356,7 +356,7 @@ i915_vm_is_48bit(const struct i915_address_space *vm)
 static inline bool
 i915_vm_has_scratch_64K(struct i915_address_space *vm)
 {
-       return vm->scratch_page.order == get_order(I915_GTT_PAGE_SIZE_64K);
+       return vm->scratch_order == get_order(I915_GTT_PAGE_SIZE_64K);
 }
 
 /* The Graphics Translation Table is the way in which GEN hardware translates a
@@ -390,7 +390,7 @@ struct i915_hw_ppgtt {
        struct i915_address_space vm;
        struct kref ref;
 
-       unsigned long pd_dirty_rings;
+       unsigned long pd_dirty_engines;
        union {
                struct i915_pml4 pml4;          /* GEN8+ & 48b PPGTT */
                struct i915_page_directory_pointer pdp; /* GEN8+ */
@@ -488,7 +488,7 @@ static inline u32 gen6_pde_index(u32 addr)
 static inline unsigned int
 i915_pdpes_per_pdp(const struct i915_address_space *vm)
 {
-       if (i915_vm_is_48bit(vm))
+       if (i915_vm_is_4lvl(vm))
                return GEN8_PML4ES_PER_PML4;
 
        return GEN8_3LVL_PDPES;
index fddde1033e747ee65f0914114975f9d8d29c88de..ab627ed1269c701a80670c848825321ee9137bb0 100644 (file)
@@ -193,7 +193,7 @@ i915_gem_object_create_internal(struct drm_i915_private *i915,
        if (overflows_type(size, obj->base.size))
                return ERR_PTR(-E2BIG);
 
-       obj = i915_gem_object_alloc(i915);
+       obj = i915_gem_object_alloc();
        if (!obj)
                return ERR_PTR(-ENOMEM);
 
index aab8cdd80e6d339eb950409a5f235cd9b43a98e4..ac6a5ab8458665ae24c5645188a90be293612516 100644 (file)
 
 #include "i915_drv.h"
 #include "i915_gem_object.h"
+#include "i915_globals.h"
+
+static struct i915_global_object {
+       struct i915_global base;
+       struct kmem_cache *slab_objects;
+} global;
+
+struct drm_i915_gem_object *i915_gem_object_alloc(void)
+{
+       return kmem_cache_zalloc(global.slab_objects, GFP_KERNEL);
+}
+
+void i915_gem_object_free(struct drm_i915_gem_object *obj)
+{
+       return kmem_cache_free(global.slab_objects, obj);
+}
 
 /**
  * Mark up the object's coherency levels for a given cache_level
@@ -46,3 +62,29 @@ void i915_gem_object_set_cache_coherency(struct drm_i915_gem_object *obj,
        obj->cache_dirty =
                !(obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_WRITE);
 }
+
+static void i915_global_objects_shrink(void)
+{
+       kmem_cache_shrink(global.slab_objects);
+}
+
+static void i915_global_objects_exit(void)
+{
+       kmem_cache_destroy(global.slab_objects);
+}
+
+static struct i915_global_object global = { {
+       .shrink = i915_global_objects_shrink,
+       .exit = i915_global_objects_exit,
+} };
+
+int __init i915_global_objects_init(void)
+{
+       global.slab_objects =
+               KMEM_CACHE(drm_i915_gem_object, SLAB_HWCACHE_ALIGN);
+       if (!global.slab_objects)
+               return -ENOMEM;
+
+       i915_global_register(&global.base);
+       return 0;
+}
index fab040331cdb250d44b5791833da357fa5c82206..1a24dc97e4fdb0abe820e5fd3ffcb399bfc1ac2d 100644 (file)
@@ -304,6 +304,9 @@ to_intel_bo(struct drm_gem_object *gem)
        return container_of(gem, struct drm_i915_gem_object, base);
 }
 
+struct drm_i915_gem_object *i915_gem_object_alloc(void);
+void i915_gem_object_free(struct drm_i915_gem_object *obj);
+
 /**
  * i915_gem_object_lookup_rcu - look up a temporary GEM object from its handle
  * @filp: DRM file private date
@@ -500,4 +503,3 @@ void i915_gem_object_set_cache_coherency(struct drm_i915_gem_object *obj,
 void i915_gem_object_flush_if_display(struct drm_i915_gem_object *obj);
 
 #endif
-
index 90baf9086d0a49f0fd2667a20ec337f520361051..91196348c68c2b9a267f614c6639b8c2853f4757 100644 (file)
@@ -42,7 +42,7 @@ struct intel_render_state {
 static const struct intel_renderstate_rodata *
 render_state_get_rodata(const struct intel_engine_cs *engine)
 {
-       if (engine->id != RCS)
+       if (engine->id != RCS0)
                return NULL;
 
        switch (INTEL_GEN(engine->i915)) {
index 74a9661479ca54b8c04e6141e14d992585aa0d52..0a8082cfc761a967776b55b140c2800e5f6ce572 100644 (file)
@@ -565,7 +565,7 @@ _i915_gem_object_create_stolen(struct drm_i915_private *dev_priv,
        struct drm_i915_gem_object *obj;
        unsigned int cache_level;
 
-       obj = i915_gem_object_alloc(dev_priv);
+       obj = i915_gem_object_alloc();
        if (obj == NULL)
                return NULL;
 
index 1d3f9a31ad61921c17923029712f7a943e664bd6..ad0087127144e6c334d0f66a4c32ccff9a111a11 100644 (file)
@@ -795,7 +795,7 @@ i915_gem_userptr_ioctl(struct drm_device *dev,
                        return -ENODEV;
        }
 
-       obj = i915_gem_object_alloc(dev_priv);
+       obj = i915_gem_object_alloc();
        if (obj == NULL)
                return -ENOMEM;
 
diff --git a/drivers/gpu/drm/i915/i915_globals.c b/drivers/gpu/drm/i915/i915_globals.c
new file mode 100644 (file)
index 0000000..2f5c72e
--- /dev/null
@@ -0,0 +1,135 @@
+/*
+ * SPDX-License-Identifier: MIT
+ *
+ * Copyright © 2019 Intel Corporation
+ */
+
+#include <linux/slab.h>
+#include <linux/workqueue.h>
+
+#include "i915_active.h"
+#include "i915_gem_context.h"
+#include "i915_gem_object.h"
+#include "i915_globals.h"
+#include "i915_request.h"
+#include "i915_scheduler.h"
+#include "i915_vma.h"
+
+static LIST_HEAD(globals);
+
+void __init i915_global_register(struct i915_global *global)
+{
+       GEM_BUG_ON(!global->shrink);
+       GEM_BUG_ON(!global->exit);
+
+       list_add_tail(&global->link, &globals);
+}
+
+static void __i915_globals_cleanup(void)
+{
+       struct i915_global *global, *next;
+
+       list_for_each_entry_safe_reverse(global, next, &globals, link)
+               global->exit();
+}
+
+static __initconst int (* const initfn[])(void) = {
+       i915_global_active_init,
+       i915_global_context_init,
+       i915_global_gem_context_init,
+       i915_global_objects_init,
+       i915_global_request_init,
+       i915_global_scheduler_init,
+       i915_global_vma_init,
+};
+
+int __init i915_globals_init(void)
+{
+       int i;
+
+       for (i = 0; i < ARRAY_SIZE(initfn); i++) {
+               int err;
+
+               err = initfn[i]();
+               if (err) {
+                       __i915_globals_cleanup();
+                       return err;
+               }
+       }
+
+       return 0;
+}
+
+static void i915_globals_shrink(void)
+{
+       struct i915_global *global;
+
+       /*
+        * kmem_cache_shrink() discards empty slabs and reorders partially
+        * filled slabs to prioritise allocating from the mostly full slabs,
+        * with the aim of reducing fragmentation.
+        */
+       list_for_each_entry(global, &globals, link)
+               global->shrink();
+}
+
+static atomic_t active;
+static atomic_t epoch;
+struct park_work {
+       struct rcu_work work;
+       int epoch;
+};
+
+static void __i915_globals_park(struct work_struct *work)
+{
+       struct park_work *wrk = container_of(work, typeof(*wrk), work.work);
+
+       /* Confirm nothing woke up in the last grace period */
+       if (wrk->epoch == atomic_read(&epoch))
+               i915_globals_shrink();
+
+       kfree(wrk);
+}
+
+void i915_globals_park(void)
+{
+       struct park_work *wrk;
+
+       /*
+        * Defer shrinking the global slab caches (and other work) until
+        * after a RCU grace period has completed with no activity. This
+        * is to try and reduce the latency impact on the consumers caused
+        * by us shrinking the caches the same time as they are trying to
+        * allocate, with the assumption being that if we idle long enough
+        * for an RCU grace period to elapse since the last use, it is likely
+        * to be longer until we need the caches again.
+        */
+       if (!atomic_dec_and_test(&active))
+               return;
+
+       wrk = kmalloc(sizeof(*wrk), GFP_KERNEL);
+       if (!wrk)
+               return;
+
+       wrk->epoch = atomic_inc_return(&epoch);
+       INIT_RCU_WORK(&wrk->work, __i915_globals_park);
+       queue_rcu_work(system_wq, &wrk->work);
+}
+
+void i915_globals_unpark(void)
+{
+       atomic_inc(&epoch);
+       atomic_inc(&active);
+}
+
+void __exit i915_globals_exit(void)
+{
+       /* Flush any residual park_work */
+       rcu_barrier();
+       flush_scheduled_work();
+
+       __i915_globals_cleanup();
+
+       /* And ensure that our DESTROY_BY_RCU slabs are truly destroyed */
+       rcu_barrier();
+}
diff --git a/drivers/gpu/drm/i915/i915_globals.h b/drivers/gpu/drm/i915/i915_globals.h
new file mode 100644 (file)
index 0000000..04c1ce1
--- /dev/null
@@ -0,0 +1,35 @@
+/*
+ * SPDX-License-Identifier: MIT
+ *
+ * Copyright © 2019 Intel Corporation
+ */
+
+#ifndef _I915_GLOBALS_H_
+#define _I915_GLOBALS_H_
+
+typedef void (*i915_global_func_t)(void);
+
+struct i915_global {
+       struct list_head link;
+
+       i915_global_func_t shrink;
+       i915_global_func_t exit;
+};
+
+void i915_global_register(struct i915_global *global);
+
+int i915_globals_init(void);
+void i915_globals_park(void);
+void i915_globals_unpark(void);
+void i915_globals_exit(void);
+
+/* constructors */
+int i915_global_active_init(void);
+int i915_global_context_init(void);
+int i915_global_gem_context_init(void);
+int i915_global_objects_init(void);
+int i915_global_request_init(void);
+int i915_global_scheduler_init(void);
+int i915_global_vma_init(void);
+
+#endif /* _I915_GLOBALS_H_ */
index aa6791255252f1800b2609285fb399653625b000..26bac517e38360ea9f1e4113690b4c41d07045b8 100644 (file)
@@ -380,19 +380,16 @@ static void print_error_buffers(struct drm_i915_error_state_buf *m,
        err_printf(m, "%s [%d]:\n", name, count);
 
        while (count--) {
-               err_printf(m, "    %08x_%08x %8u %02x %02x %02x",
+               err_printf(m, "    %08x_%08x %8u %02x %02x",
                           upper_32_bits(err->gtt_offset),
                           lower_32_bits(err->gtt_offset),
                           err->size,
                           err->read_domains,
-                          err->write_domain,
-                          err->wseqno);
+                          err->write_domain);
                err_puts(m, tiling_flag(err->tiling));
                err_puts(m, dirty_flag(err->dirty));
                err_puts(m, purgeable_flag(err->purgeable));
                err_puts(m, err->userptr ? " userptr" : "");
-               err_puts(m, err->engine != -1 ? " " : "");
-               err_puts(m, engine_name(m->i915, err->engine));
                err_puts(m, i915_cache_level_str(m->i915, err->cache_level));
 
                if (err->name)
@@ -414,7 +411,7 @@ static void error_print_instdone(struct drm_i915_error_state_buf *m,
        err_printf(m, "  INSTDONE: 0x%08x\n",
                   ee->instdone.instdone);
 
-       if (ee->engine_id != RCS || INTEL_GEN(m->i915) <= 3)
+       if (ee->engine_id != RCS0 || INTEL_GEN(m->i915) <= 3)
                return;
 
        err_printf(m, "  SC_INSTDONE: 0x%08x\n",
@@ -434,11 +431,6 @@ static void error_print_instdone(struct drm_i915_error_state_buf *m,
                           ee->instdone.row[slice][subslice]);
 }
 
-static const char *bannable(const struct drm_i915_error_context *ctx)
-{
-       return ctx->bannable ? "" : " (unbannable)";
-}
-
 static void error_print_request(struct drm_i915_error_state_buf *m,
                                const char *prefix,
                                const struct drm_i915_error_request *erq,
@@ -447,9 +439,8 @@ static void error_print_request(struct drm_i915_error_state_buf *m,
        if (!erq->seqno)
                return;
 
-       err_printf(m, "%s pid %d, ban score %d, seqno %8x:%08x%s%s, prio %d, emitted %dms, start %08x, head %08x, tail %08x\n",
-                  prefix, erq->pid, erq->ban_score,
-                  erq->context, erq->seqno,
+       err_printf(m, "%s pid %d, seqno %8x:%08x%s%s, prio %d, emitted %dms, start %08x, head %08x, tail %08x\n",
+                  prefix, erq->pid, erq->context, erq->seqno,
                   test_bit(DMA_FENCE_FLAG_SIGNALED_BIT,
                            &erq->flags) ? "!" : "",
                   test_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT,
@@ -463,10 +454,9 @@ static void error_print_context(struct drm_i915_error_state_buf *m,
                                const char *header,
                                const struct drm_i915_error_context *ctx)
 {
-       err_printf(m, "%s%s[%d] user_handle %d hw_id %d, prio %d, ban score %d%s guilty %d active %d\n",
+       err_printf(m, "%s%s[%d] user_handle %d hw_id %d, prio %d, guilty %d active %d\n",
                   header, ctx->comm, ctx->pid, ctx->handle, ctx->hw_id,
-                  ctx->sched_attr.priority, ctx->ban_score, bannable(ctx),
-                  ctx->guilty, ctx->active);
+                  ctx->sched_attr.priority, ctx->guilty, ctx->active);
 }
 
 static void error_print_engine(struct drm_i915_error_state_buf *m,
@@ -512,13 +502,6 @@ static void error_print_engine(struct drm_i915_error_state_buf *m,
        if (INTEL_GEN(m->i915) >= 6) {
                err_printf(m, "  RC PSMI: 0x%08x\n", ee->rc_psmi);
                err_printf(m, "  FAULT_REG: 0x%08x\n", ee->fault_reg);
-               err_printf(m, "  SYNC_0: 0x%08x\n",
-                          ee->semaphore_mboxes[0]);
-               err_printf(m, "  SYNC_1: 0x%08x\n",
-                          ee->semaphore_mboxes[1]);
-               if (HAS_VEBOX(m->i915))
-                       err_printf(m, "  SYNC_2: 0x%08x\n",
-                                  ee->semaphore_mboxes[2]);
        }
        if (HAS_PPGTT(m->i915)) {
                err_printf(m, "  GFX_MODE: 0x%08x\n", ee->vm_info.gfx_mode);
@@ -533,8 +516,6 @@ static void error_print_engine(struct drm_i915_error_state_buf *m,
                                   ee->vm_info.pp_dir_base);
                }
        }
-       err_printf(m, "  seqno: 0x%08x\n", ee->seqno);
-       err_printf(m, "  last_seqno: 0x%08x\n", ee->last_seqno);
        err_printf(m, "  ring->head: 0x%08x\n", ee->cpu_ring_head);
        err_printf(m, "  ring->tail: 0x%08x\n", ee->cpu_ring_tail);
        err_printf(m, "  hangcheck timestamp: %dms (%lu%s)\n",
@@ -688,12 +669,10 @@ static void __err_print_to_sgl(struct drm_i915_error_state_buf *m,
                if (!error->engine[i].context.pid)
                        continue;
 
-               err_printf(m, "Active process (on ring %s): %s [%d], score %d%s\n",
+               err_printf(m, "Active process (on ring %s): %s [%d]\n",
                           engine_name(m->i915, i),
                           error->engine[i].context.comm,
-                          error->engine[i].context.pid,
-                          error->engine[i].context.ban_score,
-                          bannable(&error->engine[i].context));
+                          error->engine[i].context.pid);
        }
        err_printf(m, "Reset count: %u\n", error->reset_count);
        err_printf(m, "Suspend count: %u\n", error->suspend_count);
@@ -779,13 +758,11 @@ static void __err_print_to_sgl(struct drm_i915_error_state_buf *m,
                if (obj) {
                        err_puts(m, m->i915->engine[i]->name);
                        if (ee->context.pid)
-                               err_printf(m, " (submitted by %s [%d], ctx %d [%d], score %d%s)",
+                               err_printf(m, " (submitted by %s [%d], ctx %d [%d])",
                                           ee->context.comm,
                                           ee->context.pid,
                                           ee->context.handle,
-                                          ee->context.hw_id,
-                                          ee->context.ban_score,
-                                          bannable(&ee->context));
+                                          ee->context.hw_id);
                        err_printf(m, " --- gtt_offset = 0x%08x %08x\n",
                                   upper_32_bits(obj->gtt_offset),
                                   lower_32_bits(obj->gtt_offset));
@@ -1061,27 +1038,6 @@ i915_error_object_create(struct drm_i915_private *i915,
        return dst;
 }
 
-/* The error capture is special as tries to run underneath the normal
- * locking rules - so we use the raw version of the i915_active_request lookup.
- */
-static inline u32
-__active_get_seqno(struct i915_active_request *active)
-{
-       struct i915_request *request;
-
-       request = __i915_active_request_peek(active);
-       return request ? request->global_seqno : 0;
-}
-
-static inline int
-__active_get_engine_id(struct i915_active_request *active)
-{
-       struct i915_request *request;
-
-       request = __i915_active_request_peek(active);
-       return request ? request->engine->id : -1;
-}
-
 static void capture_bo(struct drm_i915_error_buffer *err,
                       struct i915_vma *vma)
 {
@@ -1090,9 +1046,6 @@ static void capture_bo(struct drm_i915_error_buffer *err,
        err->size = obj->base.size;
        err->name = obj->base.name;
 
-       err->wseqno = __active_get_seqno(&obj->frontbuffer_write);
-       err->engine = __active_get_engine_id(&obj->frontbuffer_write);
-
        err->gtt_offset = vma->node.start;
        err->read_domains = obj->read_domains;
        err->write_domain = obj->write_domain;
@@ -1178,18 +1131,6 @@ static void gem_record_fences(struct i915_gpu_state *error)
        error->nfence = i;
 }
 
-static void gen6_record_semaphore_state(struct intel_engine_cs *engine,
-                                       struct drm_i915_error_engine *ee)
-{
-       struct drm_i915_private *dev_priv = engine->i915;
-
-       ee->semaphore_mboxes[0] = I915_READ(RING_SYNC_0(engine->mmio_base));
-       ee->semaphore_mboxes[1] = I915_READ(RING_SYNC_1(engine->mmio_base));
-       if (HAS_VEBOX(dev_priv))
-               ee->semaphore_mboxes[2] =
-                       I915_READ(RING_SYNC_2(engine->mmio_base));
-}
-
 static void error_record_engine_registers(struct i915_gpu_state *error,
                                          struct intel_engine_cs *engine,
                                          struct drm_i915_error_engine *ee)
@@ -1198,12 +1139,10 @@ static void error_record_engine_registers(struct i915_gpu_state *error,
 
        if (INTEL_GEN(dev_priv) >= 6) {
                ee->rc_psmi = I915_READ(RING_PSMI_CTL(engine->mmio_base));
-               if (INTEL_GEN(dev_priv) >= 8) {
+               if (INTEL_GEN(dev_priv) >= 8)
                        ee->fault_reg = I915_READ(GEN8_RING_FAULT_REG);
-               } else {
-                       gen6_record_semaphore_state(engine, ee);
+               else
                        ee->fault_reg = I915_READ(RING_FAULT_REG(engine));
-               }
        }
 
        if (INTEL_GEN(dev_priv) >= 4) {
@@ -1227,8 +1166,6 @@ static void error_record_engine_registers(struct i915_gpu_state *error,
 
        ee->instpm = I915_READ(RING_INSTPM(engine->mmio_base));
        ee->acthd = intel_engine_get_active_head(engine);
-       ee->seqno = intel_engine_get_seqno(engine);
-       ee->last_seqno = intel_engine_last_submit(engine);
        ee->start = I915_READ_START(engine);
        ee->head = I915_READ_HEAD(engine);
        ee->tail = I915_READ_TAIL(engine);
@@ -1242,16 +1179,17 @@ static void error_record_engine_registers(struct i915_gpu_state *error,
                if (IS_GEN(dev_priv, 7)) {
                        switch (engine->id) {
                        default:
-                       case RCS:
+                               MISSING_CASE(engine->id);
+                       case RCS0:
                                mmio = RENDER_HWS_PGA_GEN7;
                                break;
-                       case BCS:
+                       case BCS0:
                                mmio = BLT_HWS_PGA_GEN7;
                                break;
-                       case VCS:
+                       case VCS0:
                                mmio = BSD_HWS_PGA_GEN7;
                                break;
-                       case VECS:
+                       case VECS0:
                                mmio = VEBOX_HWS_PGA_GEN7;
                                break;
                        }
@@ -1299,10 +1237,9 @@ static void record_request(struct i915_request *request,
        struct i915_gem_context *ctx = request->gem_context;
 
        erq->flags = request->fence.flags;
-       erq->context = ctx->hw_id;
+       erq->context = request->fence.context;
+       erq->seqno = request->fence.seqno;
        erq->sched_attr = request->sched.attr;
-       erq->ban_score = atomic_read(&ctx->ban_score);
-       erq->seqno = request->global_seqno;
        erq->jiffies = request->emitted_jiffies;
        erq->start = i915_ggtt_offset(request->ring->vma);
        erq->head = request->head;
@@ -1396,8 +1333,6 @@ static void record_context(struct drm_i915_error_context *e,
        e->handle = ctx->user_handle;
        e->hw_id = ctx->hw_id;
        e->sched_attr = ctx->sched;
-       e->ban_score = atomic_read(&ctx->ban_score);
-       e->bannable = i915_gem_context_is_bannable(ctx);
        e->guilty = atomic_read(&ctx->guilty_count);
        e->active = atomic_read(&ctx->active_count);
 }
@@ -1476,7 +1411,7 @@ static void gem_record_rings(struct i915_gpu_state *error)
                error_record_engine_registers(error, engine, ee);
                error_record_engine_execlists(engine, ee);
 
-               request = i915_gem_find_active_request(engine);
+               request = intel_engine_find_active_request(engine);
                if (request) {
                        struct i915_gem_context *ctx = request->gem_context;
                        struct intel_ring *ring;
index 53b1f22dd365689c048df41fcc1ea8a4c148fa06..99d6b7b270c2033bc2d8fa6771c51615482a90bb 100644 (file)
@@ -94,8 +94,6 @@ struct i915_gpu_state {
                u32 cpu_ring_head;
                u32 cpu_ring_tail;
 
-               u32 last_seqno;
-
                /* Register state */
                u32 start;
                u32 tail;
@@ -108,13 +106,11 @@ struct i915_gpu_state {
                u32 bbstate;
                u32 instpm;
                u32 instps;
-               u32 seqno;
                u64 bbaddr;
                u64 acthd;
                u32 fault_reg;
                u64 faddr;
                u32 rc_psmi; /* sleep state */
-               u32 semaphore_mboxes[I915_NUM_ENGINES - 1];
                struct intel_instdone instdone;
 
                struct drm_i915_error_context {
@@ -122,10 +118,8 @@ struct i915_gpu_state {
                        pid_t pid;
                        u32 handle;
                        u32 hw_id;
-                       int ban_score;
                        int active;
                        int guilty;
-                       bool bannable;
                        struct i915_sched_attr sched_attr;
                } context;
 
@@ -149,7 +143,6 @@ struct i915_gpu_state {
                        long jiffies;
                        pid_t pid;
                        u32 context;
-                       int ban_score;
                        u32 seqno;
                        u32 start;
                        u32 head;
@@ -170,7 +163,6 @@ struct i915_gpu_state {
        struct drm_i915_error_buffer {
                u32 size;
                u32 name;
-               u32 wseqno;
                u64 gtt_offset;
                u32 read_domains;
                u32 write_domain;
@@ -179,7 +171,6 @@ struct i915_gpu_state {
                u32 dirty:1;
                u32 purgeable:1;
                u32 userptr:1;
-               s32 engine:4;
                u32 cache_level:3;
        } *active_bo[I915_NUM_ENGINES], *pinned_bo;
        u32 active_bo_count[I915_NUM_ENGINES], pinned_bo_count;
@@ -204,39 +195,13 @@ struct i915_gpu_error {
 
        atomic_t pending_fb_pin;
 
-       /**
-        * State variable controlling the reset flow and count
-        *
-        * This is a counter which gets incremented when reset is triggered,
-        *
-        * Before the reset commences, the I915_RESET_BACKOFF bit is set
-        * meaning that any waiters holding onto the struct_mutex should
-        * relinquish the lock immediately in order for the reset to start.
-        *
-        * If reset is not completed successfully, the I915_WEDGE bit is
-        * set meaning that hardware is terminally sour and there is no
-        * recovery. All waiters on the reset_queue will be woken when
-        * that happens.
-        *
-        * This counter is used by the wait_seqno code to notice that reset
-        * event happened and it needs to restart the entire ioctl (since most
-        * likely the seqno it waited for won't ever signal anytime soon).
-        *
-        * This is important for lock-free wait paths, where no contended lock
-        * naturally enforces the correct ordering between the bail-out of the
-        * waiter and the gpu reset work code.
-        */
-       unsigned long reset_count;
-
        /**
         * flags: Control various stages of the GPU reset
         *
-        * #I915_RESET_BACKOFF - When we start a reset, we want to stop any
-        * other users acquiring the struct_mutex. To do this we set the
-        * #I915_RESET_BACKOFF bit in the error flags when we detect a reset
-        * and then check for that bit before acquiring the struct_mutex (in
-        * i915_mutex_lock_interruptible()?). I915_RESET_BACKOFF serves a
-        * secondary role in preventing two concurrent global reset attempts.
+        * #I915_RESET_BACKOFF - When we start a global reset, we need to
+        * serialise with any other users attempting to do the same, and
+        * any global resources that may be clobber by the reset (such as
+        * FENCE registers).
         *
         * #I915_RESET_ENGINE[num_engines] - Since the driver doesn't need to
         * acquire the struct_mutex to reset an engine, we need an explicit
@@ -255,6 +220,9 @@ struct i915_gpu_error {
 #define I915_RESET_ENGINE      2
 #define I915_WEDGED            (BITS_PER_LONG - 1)
 
+       /** Number of times the device has been reset (global) */
+       u32 reset_count;
+
        /** Number of times an engine has been reset */
        u32 reset_engine_count[I915_NUM_ENGINES];
 
@@ -272,6 +240,8 @@ struct i915_gpu_error {
         */
        wait_queue_head_t reset_queue;
 
+       struct srcu_struct reset_backoff_srcu;
+
        struct i915_gpu_restart *restart;
 };
 
index 441d2674b2725227116b5cd852779e86920f5fa0..82d487189a3406e57ab7865d8fe82d680444f541 100644 (file)
@@ -1288,6 +1288,18 @@ static void gen6_pm_rps_work(struct work_struct *work)
 
        rps->last_adj = adj;
 
+       /*
+        * Limit deboosting and boosting to keep ourselves at the extremes
+        * when in the respective power modes (i.e. slowly decrease frequencies
+        * while in the HIGH_POWER zone and slowly increase frequencies while
+        * in the LOW_POWER zone). On idle, we will hit the timeout and drop
+        * to the next level quickly, and conversely if busy we expect to
+        * hit a waitboost and rapidly switch into max power.
+        */
+       if ((adj < 0 && rps->power.mode == HIGH_POWER) ||
+           (adj > 0 && rps->power.mode == LOW_POWER))
+               rps->last_adj = 0;
+
        /* sysfs frequency interfaces may have snuck in while servicing the
         * interrupt
         */
@@ -1415,20 +1427,20 @@ static void ilk_gt_irq_handler(struct drm_i915_private *dev_priv,
                               u32 gt_iir)
 {
        if (gt_iir & GT_RENDER_USER_INTERRUPT)
-               intel_engine_breadcrumbs_irq(dev_priv->engine[RCS]);
+               intel_engine_breadcrumbs_irq(dev_priv->engine[RCS0]);
        if (gt_iir & ILK_BSD_USER_INTERRUPT)
-               intel_engine_breadcrumbs_irq(dev_priv->engine[VCS]);
+               intel_engine_breadcrumbs_irq(dev_priv->engine[VCS0]);
 }
 
 static void snb_gt_irq_handler(struct drm_i915_private *dev_priv,
                               u32 gt_iir)
 {
        if (gt_iir & GT_RENDER_USER_INTERRUPT)
-               intel_engine_breadcrumbs_irq(dev_priv->engine[RCS]);
+               intel_engine_breadcrumbs_irq(dev_priv->engine[RCS0]);
        if (gt_iir & GT_BSD_USER_INTERRUPT)
-               intel_engine_breadcrumbs_irq(dev_priv->engine[VCS]);
+               intel_engine_breadcrumbs_irq(dev_priv->engine[VCS0]);
        if (gt_iir & GT_BLT_USER_INTERRUPT)
-               intel_engine_breadcrumbs_irq(dev_priv->engine[BCS]);
+               intel_engine_breadcrumbs_irq(dev_priv->engine[BCS0]);
 
        if (gt_iir & (GT_BLT_CS_ERROR_INTERRUPT |
                      GT_BSD_CS_ERROR_INTERRUPT |
@@ -1463,8 +1475,8 @@ static void gen8_gt_irq_ack(struct drm_i915_private *i915,
 
 #define GEN8_GT_IRQS (GEN8_GT_RCS_IRQ | \
                      GEN8_GT_BCS_IRQ | \
+                     GEN8_GT_VCS0_IRQ | \
                      GEN8_GT_VCS1_IRQ | \
-                     GEN8_GT_VCS2_IRQ | \
                      GEN8_GT_VECS_IRQ | \
                      GEN8_GT_PM_IRQ | \
                      GEN8_GT_GUC_IRQ)
@@ -1475,7 +1487,7 @@ static void gen8_gt_irq_ack(struct drm_i915_private *i915,
                        raw_reg_write(regs, GEN8_GT_IIR(0), gt_iir[0]);
        }
 
-       if (master_ctl & (GEN8_GT_VCS1_IRQ | GEN8_GT_VCS2_IRQ)) {
+       if (master_ctl & (GEN8_GT_VCS0_IRQ | GEN8_GT_VCS1_IRQ)) {
                gt_iir[1] = raw_reg_read(regs, GEN8_GT_IIR(1));
                if (likely(gt_iir[1]))
                        raw_reg_write(regs, GEN8_GT_IIR(1), gt_iir[1]);
@@ -1498,21 +1510,21 @@ static void gen8_gt_irq_handler(struct drm_i915_private *i915,
                                u32 master_ctl, u32 gt_iir[4])
 {
        if (master_ctl & (GEN8_GT_RCS_IRQ | GEN8_GT_BCS_IRQ)) {
-               gen8_cs_irq_handler(i915->engine[RCS],
+               gen8_cs_irq_handler(i915->engine[RCS0],
                                    gt_iir[0] >> GEN8_RCS_IRQ_SHIFT);
-               gen8_cs_irq_handler(i915->engine[BCS],
+               gen8_cs_irq_handler(i915->engine[BCS0],
                                    gt_iir[0] >> GEN8_BCS_IRQ_SHIFT);
        }
 
-       if (master_ctl & (GEN8_GT_VCS1_IRQ | GEN8_GT_VCS2_IRQ)) {
-               gen8_cs_irq_handler(i915->engine[VCS],
+       if (master_ctl & (GEN8_GT_VCS0_IRQ | GEN8_GT_VCS1_IRQ)) {
+               gen8_cs_irq_handler(i915->engine[VCS0],
+                                   gt_iir[1] >> GEN8_VCS0_IRQ_SHIFT);
+               gen8_cs_irq_handler(i915->engine[VCS1],
                                    gt_iir[1] >> GEN8_VCS1_IRQ_SHIFT);
-               gen8_cs_irq_handler(i915->engine[VCS2],
-                                   gt_iir[1] >> GEN8_VCS2_IRQ_SHIFT);
        }
 
        if (master_ctl & GEN8_GT_VECS_IRQ) {
-               gen8_cs_irq_handler(i915->engine[VECS],
+               gen8_cs_irq_handler(i915->engine[VECS0],
                                    gt_iir[3] >> GEN8_VECS_IRQ_SHIFT);
        }
 
@@ -1693,7 +1705,9 @@ static void display_pipe_crc_irq_handler(struct drm_i915_private *dev_priv,
 {
        struct intel_pipe_crc *pipe_crc = &dev_priv->pipe_crc[pipe];
        struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
-       u32 crcs[5];
+       u32 crcs[5] = { crc0, crc1, crc2, crc3, crc4 };
+
+       trace_intel_pipe_crc(crtc, crcs);
 
        spin_lock(&pipe_crc->lock);
        /*
@@ -1712,11 +1726,6 @@ static void display_pipe_crc_irq_handler(struct drm_i915_private *dev_priv,
        }
        spin_unlock(&pipe_crc->lock);
 
-       crcs[0] = crc0;
-       crcs[1] = crc1;
-       crcs[2] = crc2;
-       crcs[3] = crc3;
-       crcs[4] = crc4;
        drm_crtc_add_crc_entry(&crtc->base, true,
                                drm_crtc_accurate_vblank_count(&crtc->base),
                                crcs);
@@ -1792,13 +1801,11 @@ static void gen6_rps_irq_handler(struct drm_i915_private *dev_priv, u32 pm_iir)
        if (INTEL_GEN(dev_priv) >= 8)
                return;
 
-       if (HAS_VEBOX(dev_priv)) {
-               if (pm_iir & PM_VEBOX_USER_INTERRUPT)
-                       intel_engine_breadcrumbs_irq(dev_priv->engine[VECS]);
+       if (pm_iir & PM_VEBOX_USER_INTERRUPT)
+               intel_engine_breadcrumbs_irq(dev_priv->engine[VECS0]);
 
-               if (pm_iir & PM_VEBOX_CS_ERROR_INTERRUPT)
-                       DRM_DEBUG("Command parser error, pm_iir 0x%08x\n", pm_iir);
-       }
+       if (pm_iir & PM_VEBOX_CS_ERROR_INTERRUPT)
+               DRM_DEBUG("Command parser error, pm_iir 0x%08x\n", pm_iir);
 }
 
 static void gen9_guc_irq_handler(struct drm_i915_private *dev_priv, u32 gt_iir)
@@ -2667,6 +2674,25 @@ static void gen11_hpd_irq_handler(struct drm_i915_private *dev_priv, u32 iir)
                DRM_ERROR("Unexpected DE HPD interrupt 0x%08x\n", iir);
 }
 
+static u32 gen8_de_port_aux_mask(struct drm_i915_private *dev_priv)
+{
+       u32 mask = GEN8_AUX_CHANNEL_A;
+
+       if (INTEL_GEN(dev_priv) >= 9)
+               mask |= GEN9_AUX_CHANNEL_B |
+                       GEN9_AUX_CHANNEL_C |
+                       GEN9_AUX_CHANNEL_D;
+
+       if (IS_CNL_WITH_PORT_F(dev_priv))
+               mask |= CNL_AUX_CHANNEL_F;
+
+       if (INTEL_GEN(dev_priv) >= 11)
+               mask |= ICL_AUX_CHANNEL_E |
+                       CNL_AUX_CHANNEL_F;
+
+       return mask;
+}
+
 static irqreturn_t
 gen8_de_irq_handler(struct drm_i915_private *dev_priv, u32 master_ctl)
 {
@@ -2722,20 +2748,7 @@ gen8_de_irq_handler(struct drm_i915_private *dev_priv, u32 master_ctl)
                        I915_WRITE(GEN8_DE_PORT_IIR, iir);
                        ret = IRQ_HANDLED;
 
-                       tmp_mask = GEN8_AUX_CHANNEL_A;
-                       if (INTEL_GEN(dev_priv) >= 9)
-                               tmp_mask |= GEN9_AUX_CHANNEL_B |
-                                           GEN9_AUX_CHANNEL_C |
-                                           GEN9_AUX_CHANNEL_D;
-
-                       if (INTEL_GEN(dev_priv) >= 11)
-                               tmp_mask |= ICL_AUX_CHANNEL_E;
-
-                       if (IS_CNL_WITH_PORT_F(dev_priv) ||
-                           INTEL_GEN(dev_priv) >= 11)
-                               tmp_mask |= CNL_AUX_CHANNEL_F;
-
-                       if (iir & tmp_mask) {
+                       if (iir & gen8_de_port_aux_mask(dev_priv)) {
                                dp_aux_irq_handler(dev_priv);
                                found = true;
                        }
@@ -2816,11 +2829,9 @@ gen8_de_irq_handler(struct drm_i915_private *dev_priv, u32 master_ctl)
                        I915_WRITE(SDEIIR, iir);
                        ret = IRQ_HANDLED;
 
-                       if (HAS_PCH_ICP(dev_priv))
+                       if (INTEL_PCH_TYPE(dev_priv) >= PCH_ICP)
                                icp_irq_handler(dev_priv, iir);
-                       else if (HAS_PCH_SPT(dev_priv) ||
-                                HAS_PCH_KBP(dev_priv) ||
-                                HAS_PCH_CNP(dev_priv))
+                       else if (INTEL_PCH_TYPE(dev_priv) >= PCH_SPT)
                                spt_irq_handler(dev_priv, iir);
                        else
                                cpt_irq_handler(dev_priv, iir);
@@ -3402,7 +3413,7 @@ static void gen11_irq_reset(struct drm_device *dev)
        GEN3_IRQ_RESET(GEN11_GU_MISC_);
        GEN3_IRQ_RESET(GEN8_PCU_);
 
-       if (HAS_PCH_ICP(dev_priv))
+       if (INTEL_PCH_TYPE(dev_priv) >= PCH_ICP)
                GEN3_IRQ_RESET(SDE);
 }
 
@@ -3583,7 +3594,7 @@ static void gen11_hpd_irq_setup(struct drm_i915_private *dev_priv)
 
        gen11_hpd_detection_setup(dev_priv);
 
-       if (HAS_PCH_ICP(dev_priv))
+       if (INTEL_PCH_TYPE(dev_priv) >= PCH_ICP)
                icp_hpd_irq_setup(dev_priv);
 }
 
@@ -3767,7 +3778,7 @@ static void gen5_gt_irq_postinstall(struct drm_device *dev)
                 * RPS interrupts will get enabled/disabled on demand when RPS
                 * itself is enabled/disabled.
                 */
-               if (HAS_VEBOX(dev_priv)) {
+               if (HAS_ENGINE(dev_priv, VECS0)) {
                        pm_irqs |= PM_VEBOX_USER_INTERRUPT;
                        dev_priv->pm_ier |= PM_VEBOX_USER_INTERRUPT;
                }
@@ -3879,18 +3890,21 @@ static void gen8_gt_irq_postinstall(struct drm_i915_private *dev_priv)
 {
        /* These are interrupts we'll toggle with the ring mask register */
        u32 gt_interrupts[] = {
-               GT_RENDER_USER_INTERRUPT << GEN8_RCS_IRQ_SHIFT |
-                       GT_CONTEXT_SWITCH_INTERRUPT << GEN8_RCS_IRQ_SHIFT |
-                       GT_RENDER_USER_INTERRUPT << GEN8_BCS_IRQ_SHIFT |
-                       GT_CONTEXT_SWITCH_INTERRUPT << GEN8_BCS_IRQ_SHIFT,
-               GT_RENDER_USER_INTERRUPT << GEN8_VCS1_IRQ_SHIFT |
-                       GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VCS1_IRQ_SHIFT |
-                       GT_RENDER_USER_INTERRUPT << GEN8_VCS2_IRQ_SHIFT |
-                       GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VCS2_IRQ_SHIFT,
+               (GT_RENDER_USER_INTERRUPT << GEN8_RCS_IRQ_SHIFT |
+                GT_CONTEXT_SWITCH_INTERRUPT << GEN8_RCS_IRQ_SHIFT |
+                GT_RENDER_USER_INTERRUPT << GEN8_BCS_IRQ_SHIFT |
+                GT_CONTEXT_SWITCH_INTERRUPT << GEN8_BCS_IRQ_SHIFT),
+
+               (GT_RENDER_USER_INTERRUPT << GEN8_VCS0_IRQ_SHIFT |
+                GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VCS0_IRQ_SHIFT |
+                GT_RENDER_USER_INTERRUPT << GEN8_VCS1_IRQ_SHIFT |
+                GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VCS1_IRQ_SHIFT),
+
                0,
-               GT_RENDER_USER_INTERRUPT << GEN8_VECS_IRQ_SHIFT |
-                       GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VECS_IRQ_SHIFT
-               };
+
+               (GT_RENDER_USER_INTERRUPT << GEN8_VECS_IRQ_SHIFT |
+                GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VECS_IRQ_SHIFT)
+       };
 
        dev_priv->pm_ier = 0x0;
        dev_priv->pm_imr = ~dev_priv->pm_ier;
@@ -4036,7 +4050,7 @@ static int gen11_irq_postinstall(struct drm_device *dev)
        struct drm_i915_private *dev_priv = dev->dev_private;
        u32 gu_misc_masked = GEN11_GU_MISC_GSE;
 
-       if (HAS_PCH_ICP(dev_priv))
+       if (INTEL_PCH_TYPE(dev_priv) >= PCH_ICP)
                icp_irq_postinstall(dev);
 
        gen11_gt_irq_postinstall(dev_priv);
@@ -4218,7 +4232,7 @@ static irqreturn_t i8xx_irq_handler(int irq, void *arg)
                I915_WRITE16(IIR, iir);
 
                if (iir & I915_USER_INTERRUPT)
-                       intel_engine_breadcrumbs_irq(dev_priv->engine[RCS]);
+                       intel_engine_breadcrumbs_irq(dev_priv->engine[RCS0]);
 
                if (iir & I915_MASTER_ERROR_INTERRUPT)
                        i8xx_error_irq_handler(dev_priv, eir, eir_stuck);
@@ -4326,7 +4340,7 @@ static irqreturn_t i915_irq_handler(int irq, void *arg)
                I915_WRITE(IIR, iir);
 
                if (iir & I915_USER_INTERRUPT)
-                       intel_engine_breadcrumbs_irq(dev_priv->engine[RCS]);
+                       intel_engine_breadcrumbs_irq(dev_priv->engine[RCS0]);
 
                if (iir & I915_MASTER_ERROR_INTERRUPT)
                        i9xx_error_irq_handler(dev_priv, eir, eir_stuck);
@@ -4471,10 +4485,10 @@ static irqreturn_t i965_irq_handler(int irq, void *arg)
                I915_WRITE(IIR, iir);
 
                if (iir & I915_USER_INTERRUPT)
-                       intel_engine_breadcrumbs_irq(dev_priv->engine[RCS]);
+                       intel_engine_breadcrumbs_irq(dev_priv->engine[RCS0]);
 
                if (iir & I915_BSD_USER_INTERRUPT)
-                       intel_engine_breadcrumbs_irq(dev_priv->engine[VCS]);
+                       intel_engine_breadcrumbs_irq(dev_priv->engine[VCS0]);
 
                if (iir & I915_MASTER_ERROR_INTERRUPT)
                        i9xx_error_irq_handler(dev_priv, eir, eir_stuck);
@@ -4605,8 +4619,7 @@ void intel_irq_init(struct drm_i915_private *dev_priv)
                dev->driver->disable_vblank = gen8_disable_vblank;
                if (IS_GEN9_LP(dev_priv))
                        dev_priv->display.hpd_irq_setup = bxt_hpd_irq_setup;
-               else if (HAS_PCH_SPT(dev_priv) || HAS_PCH_KBP(dev_priv) ||
-                        HAS_PCH_CNP(dev_priv))
+               else if (INTEL_PCH_TYPE(dev_priv) >= PCH_SPT)
                        dev_priv->display.hpd_irq_setup = spt_hpd_irq_setup;
                else
                        dev_priv->display.hpd_irq_setup = ilk_hpd_irq_setup;
index 66f82f3f050f34aa1a6756800b2436ceab50c9ab..7a6054eadb8e99604140d019086ecc5093c3915a 100644 (file)
 
 #include <drm/drm_drv.h>
 
-#include "i915_active.h"
 #include "i915_drv.h"
+#include "i915_globals.h"
 #include "i915_selftest.h"
 
 #define PLATFORM(x) .platform = (x), .platform_mask = BIT(x)
 #define GEN(x) .gen = (x), .gen_mask = BIT((x) - 1)
 
-#define GEN_DEFAULT_PIPEOFFSETS \
+#define I845_PIPE_OFFSETS \
+       .pipe_offsets = { \
+               [TRANSCODER_A] = PIPE_A_OFFSET, \
+       }, \
+       .trans_offsets = { \
+               [TRANSCODER_A] = TRANSCODER_A_OFFSET, \
+       }
+
+#define I9XX_PIPE_OFFSETS \
+       .pipe_offsets = { \
+               [TRANSCODER_A] = PIPE_A_OFFSET, \
+               [TRANSCODER_B] = PIPE_B_OFFSET, \
+       }, \
+       .trans_offsets = { \
+               [TRANSCODER_A] = TRANSCODER_A_OFFSET, \
+               [TRANSCODER_B] = TRANSCODER_B_OFFSET, \
+       }
+
+#define IVB_PIPE_OFFSETS \
+       .pipe_offsets = { \
+               [TRANSCODER_A] = PIPE_A_OFFSET, \
+               [TRANSCODER_B] = PIPE_B_OFFSET, \
+               [TRANSCODER_C] = PIPE_C_OFFSET, \
+       }, \
+       .trans_offsets = { \
+               [TRANSCODER_A] = TRANSCODER_A_OFFSET, \
+               [TRANSCODER_B] = TRANSCODER_B_OFFSET, \
+               [TRANSCODER_C] = TRANSCODER_C_OFFSET, \
+       }
+
+#define HSW_PIPE_OFFSETS \
        .pipe_offsets = { \
                [TRANSCODER_A] = PIPE_A_OFFSET, \
                [TRANSCODER_B] = PIPE_B_OFFSET, \
@@ -49,7 +79,7 @@
                [TRANSCODER_EDP] = TRANSCODER_EDP_OFFSET, \
        }
 
-#define GEN_CHV_PIPEOFFSETS \