Need MST sideband message transaction to power up/down nodes.
Signed-off-by: Jani Nikula <jani.nikula@intel.com>
select DRM_PANEL
select DRM_MIPI_DSI
select RELAY
+ select IRQ_WORK
# i915 depends on ACPI_VIDEO when ACPI is enabled
# but for select to work, need to select ACPI_VIDEO's dependencies, ick
select BACKLIGHT_LCD_SUPPORT if ACPI
i915_oa_bxt.o \
i915_oa_kblgt2.o \
i915_oa_kblgt3.o \
- i915_oa_glk.o
+ i915_oa_glk.o \
+ i915_oa_cflgt2.o
ifeq ($(CONFIG_DRM_I915_GVT),y)
i915-y += intel_gvt.o
if (WARN_ON(bytes > 4))
return -EINVAL;
- if (WARN_ON(offset + bytes > INTEL_GVT_MAX_CFG_SPACE_SZ))
+ if (WARN_ON(offset + bytes > vgpu->gvt->device_info.cfg_space_size))
return -EINVAL;
memcpy(p_data, vgpu_cfg_space(vgpu) + offset, bytes);
static int map_aperture(struct intel_vgpu *vgpu, bool map)
{
- u64 first_gfn, first_mfn;
+ phys_addr_t aperture_pa = vgpu_aperture_pa_base(vgpu);
+ unsigned long aperture_sz = vgpu_aperture_sz(vgpu);
+ u64 first_gfn;
u64 val;
int ret;
if (map == vgpu->cfg_space.bar[INTEL_GVT_PCI_BAR_APERTURE].tracked)
return 0;
+ if (map) {
+ vgpu->gm.aperture_va = memremap(aperture_pa, aperture_sz,
+ MEMREMAP_WC);
+ if (!vgpu->gm.aperture_va)
+ return -ENOMEM;
+ } else {
+ memunmap(vgpu->gm.aperture_va);
+ vgpu->gm.aperture_va = NULL;
+ }
+
val = vgpu_cfg_space(vgpu)[PCI_BASE_ADDRESS_2];
if (val & PCI_BASE_ADDRESS_MEM_TYPE_64)
val = *(u64 *)(vgpu_cfg_space(vgpu) + PCI_BASE_ADDRESS_2);
val = *(u32 *)(vgpu_cfg_space(vgpu) + PCI_BASE_ADDRESS_2);
first_gfn = (val + vgpu_aperture_offset(vgpu)) >> PAGE_SHIFT;
- first_mfn = vgpu_aperture_pa_base(vgpu) >> PAGE_SHIFT;
ret = intel_gvt_hypervisor_map_gfn_to_mfn(vgpu, first_gfn,
- first_mfn,
- vgpu_aperture_sz(vgpu) >>
- PAGE_SHIFT, map);
- if (ret)
+ aperture_pa >> PAGE_SHIFT,
+ aperture_sz >> PAGE_SHIFT,
+ map);
+ if (ret) {
+ memunmap(vgpu->gm.aperture_va);
+ vgpu->gm.aperture_va = NULL;
return ret;
+ }
vgpu->cfg_space.bar[INTEL_GVT_PCI_BAR_APERTURE].tracked = map;
return 0;
if (WARN_ON(bytes > 4))
return -EINVAL;
- if (WARN_ON(offset + bytes > INTEL_GVT_MAX_CFG_SPACE_SZ))
+ if (WARN_ON(offset + bytes > vgpu->gvt->device_info.cfg_space_size))
return -EINVAL;
/* First check if it's PCI_COMMAND */
return 1;
}
-static uint32_t find_bb_size(struct parser_exec_state *s)
+static int find_bb_size(struct parser_exec_state *s)
{
unsigned long gma = 0;
struct cmd_info *info;
- uint32_t bb_size = 0;
+ int bb_size = 0;
uint32_t cmd_len = 0;
bool met_bb_end = false;
struct intel_vgpu *vgpu = s->vgpu;
/* get the size of the batch buffer */
bb_size = find_bb_size(s);
+ if (bb_size < 0)
+ return -EINVAL;
/* allocate shadow batch buffer */
entry_obj = kmalloc(sizeof(*entry_obj), GFP_KERNEL);
{
struct intel_vgpu *vgpu = workload->vgpu;
unsigned long gma_head, gma_tail, gma_top, guest_rb_size;
- u32 *cs;
+ void *shadow_ring_buffer_va;
+ int ring_id = workload->ring_id;
int ret;
guest_rb_size = _RING_CTL_BUF_SIZE(workload->rb_ctl);
gma_tail = workload->rb_start + workload->rb_tail;
gma_top = workload->rb_start + guest_rb_size;
- /* allocate shadow ring buffer */
- cs = intel_ring_begin(workload->req, workload->rb_len / sizeof(u32));
- if (IS_ERR(cs))
- return PTR_ERR(cs);
+ if (workload->rb_len > vgpu->reserve_ring_buffer_size[ring_id]) {
+ void *va = vgpu->reserve_ring_buffer_va[ring_id];
+ /* realloc the new ring buffer if needed */
+ vgpu->reserve_ring_buffer_va[ring_id] =
+ krealloc(va, workload->rb_len, GFP_KERNEL);
+ if (!vgpu->reserve_ring_buffer_va[ring_id]) {
+ gvt_vgpu_err("fail to alloc reserve ring buffer\n");
+ return -ENOMEM;
+ }
+ vgpu->reserve_ring_buffer_size[ring_id] = workload->rb_len;
+ }
+
+ shadow_ring_buffer_va = vgpu->reserve_ring_buffer_va[ring_id];
/* get shadow ring buffer va */
- workload->shadow_ring_buffer_va = cs;
+ workload->shadow_ring_buffer_va = shadow_ring_buffer_va;
/* head > tail --> copy head <-> top */
if (gma_head > gma_tail) {
ret = copy_gma_to_hva(vgpu, vgpu->gtt.ggtt_mm,
- gma_head, gma_top, cs);
+ gma_head, gma_top, shadow_ring_buffer_va);
if (ret < 0) {
gvt_vgpu_err("fail to copy guest ring buffer\n");
return ret;
}
- cs += ret / sizeof(u32);
+ shadow_ring_buffer_va += ret;
gma_head = workload->rb_start;
}
/* copy head or start <-> tail */
- ret = copy_gma_to_hva(vgpu, vgpu->gtt.ggtt_mm, gma_head, gma_tail, cs);
+ ret = copy_gma_to_hva(vgpu, vgpu->gtt.ggtt_mm, gma_head, gma_tail,
+ shadow_ring_buffer_va);
if (ret < 0) {
gvt_vgpu_err("fail to copy guest ring buffer\n");
return ret;
}
- cs += ret / sizeof(u32);
- intel_ring_advance(workload->req, cs);
return 0;
}
#define get_desc_from_elsp_dwords(ed, i) \
((struct execlist_ctx_descriptor_format *)&((ed)->data[i * 2]))
-static void prepare_shadow_batch_buffer(struct intel_vgpu_workload *workload)
+static int prepare_shadow_batch_buffer(struct intel_vgpu_workload *workload)
{
const int gmadr_bytes = workload->vgpu->gvt->device_info.gmadr_bytes_in_cmd;
struct intel_shadow_bb_entry *entry_obj;
vma = i915_gem_object_ggtt_pin(entry_obj->obj, NULL, 0, 4, 0);
if (IS_ERR(vma)) {
- return;
+ return PTR_ERR(vma);
}
/* FIXME: we are not tracking our pinned VMA leaving it
if (gmadr_bytes == 8)
entry_obj->bb_start_cmd_va[2] = 0;
}
+ return 0;
}
static int update_wa_ctx_2_shadow_ctx(struct intel_shadow_wa_ctx *wa_ctx)
return 0;
}
-static void prepare_shadow_wa_ctx(struct intel_shadow_wa_ctx *wa_ctx)
+static int prepare_shadow_wa_ctx(struct intel_shadow_wa_ctx *wa_ctx)
{
struct i915_vma *vma;
unsigned char *per_ctx_va =
wa_ctx->indirect_ctx.size;
if (wa_ctx->indirect_ctx.size == 0)
- return;
+ return 0;
vma = i915_gem_object_ggtt_pin(wa_ctx->indirect_ctx.obj, NULL,
0, CACHELINE_BYTES, 0);
if (IS_ERR(vma)) {
- return;
+ return PTR_ERR(vma);
}
/* FIXME: we are not tracking our pinned VMA leaving it
memset(per_ctx_va, 0, CACHELINE_BYTES);
update_wa_ctx_2_shadow_ctx(wa_ctx);
-}
-
-static int prepare_execlist_workload(struct intel_vgpu_workload *workload)
-{
- struct intel_vgpu *vgpu = workload->vgpu;
- struct execlist_ctx_descriptor_format ctx[2];
- int ring_id = workload->ring_id;
-
- intel_vgpu_pin_mm(workload->shadow_mm);
- intel_vgpu_sync_oos_pages(workload->vgpu);
- intel_vgpu_flush_post_shadow(workload->vgpu);
- prepare_shadow_batch_buffer(workload);
- prepare_shadow_wa_ctx(&workload->wa_ctx);
- if (!workload->emulate_schedule_in)
- return 0;
-
- ctx[0] = *get_desc_from_elsp_dwords(&workload->elsp_dwords, 1);
- ctx[1] = *get_desc_from_elsp_dwords(&workload->elsp_dwords, 0);
-
- return emulate_execlist_schedule_in(&vgpu->execlist[ring_id], ctx);
+ return 0;
}
static void release_shadow_batch_buffer(struct intel_vgpu_workload *workload)
}
}
-static void release_shadow_wa_ctx(struct intel_shadow_wa_ctx *wa_ctx)
+static int prepare_execlist_workload(struct intel_vgpu_workload *workload)
{
- if (!wa_ctx->indirect_ctx.obj)
- return;
+ struct intel_vgpu *vgpu = workload->vgpu;
+ struct execlist_ctx_descriptor_format ctx[2];
+ int ring_id = workload->ring_id;
+ int ret;
+
+ ret = intel_vgpu_pin_mm(workload->shadow_mm);
+ if (ret) {
+ gvt_vgpu_err("fail to vgpu pin mm\n");
+ goto out;
+ }
+
+ ret = intel_vgpu_sync_oos_pages(workload->vgpu);
+ if (ret) {
+ gvt_vgpu_err("fail to vgpu sync oos pages\n");
+ goto err_unpin_mm;
+ }
- i915_gem_object_unpin_map(wa_ctx->indirect_ctx.obj);
- i915_gem_object_put(wa_ctx->indirect_ctx.obj);
+ ret = intel_vgpu_flush_post_shadow(workload->vgpu);
+ if (ret) {
+ gvt_vgpu_err("fail to flush post shadow\n");
+ goto err_unpin_mm;
+ }
+
+ ret = prepare_shadow_batch_buffer(workload);
+ if (ret) {
+ gvt_vgpu_err("fail to prepare_shadow_batch_buffer\n");
+ goto err_unpin_mm;
+ }
+
+ ret = prepare_shadow_wa_ctx(&workload->wa_ctx);
+ if (ret) {
+ gvt_vgpu_err("fail to prepare_shadow_wa_ctx\n");
+ goto err_shadow_batch;
+ }
+
+ if (!workload->emulate_schedule_in)
+ return 0;
+
+ ctx[0] = *get_desc_from_elsp_dwords(&workload->elsp_dwords, 1);
+ ctx[1] = *get_desc_from_elsp_dwords(&workload->elsp_dwords, 0);
+
+ ret = emulate_execlist_schedule_in(&vgpu->execlist[ring_id], ctx);
+ if (!ret)
+ goto out;
+ else
+ gvt_vgpu_err("fail to emulate execlist schedule in\n");
+
+ release_shadow_wa_ctx(&workload->wa_ctx);
+err_shadow_batch:
+ release_shadow_batch_buffer(workload);
+err_unpin_mm:
+ intel_vgpu_unpin_mm(workload->shadow_mm);
+out:
+ return ret;
}
static int complete_execlist_workload(struct intel_vgpu_workload *workload)
gvt_dbg_el("complete workload %p status %d\n", workload,
workload->status);
- release_shadow_batch_buffer(workload);
- release_shadow_wa_ctx(&workload->wa_ctx);
+ if (!workload->status) {
+ release_shadow_batch_buffer(workload);
+ release_shadow_wa_ctx(&workload->wa_ctx);
+ }
if (workload->status || (vgpu->resetting_eng & ENGINE_MASK(ring_id))) {
/* if workload->status is not successful means HW GPU
void intel_vgpu_clean_execlist(struct intel_vgpu *vgpu)
{
+ enum intel_engine_id i;
+ struct intel_engine_cs *engine;
+
clean_workloads(vgpu, ALL_ENGINES);
kmem_cache_destroy(vgpu->workloads);
+
+ for_each_engine(engine, vgpu->gvt->dev_priv, i) {
+ kfree(vgpu->reserve_ring_buffer_va[i]);
+ vgpu->reserve_ring_buffer_va[i] = NULL;
+ vgpu->reserve_ring_buffer_size[i] = 0;
+ }
+
}
+#define RESERVE_RING_BUFFER_SIZE ((1 * PAGE_SIZE)/8)
int intel_vgpu_init_execlist(struct intel_vgpu *vgpu)
{
enum intel_engine_id i;
if (!vgpu->workloads)
return -ENOMEM;
+ /* each ring has a shadow ring buffer until vgpu destroyed */
+ for_each_engine(engine, vgpu->gvt->dev_priv, i) {
+ vgpu->reserve_ring_buffer_va[i] =
+ kmalloc(RESERVE_RING_BUFFER_SIZE, GFP_KERNEL);
+ if (!vgpu->reserve_ring_buffer_va[i]) {
+ gvt_vgpu_err("fail to alloc reserve ring buffer\n");
+ goto out;
+ }
+ vgpu->reserve_ring_buffer_size[i] = RESERVE_RING_BUFFER_SIZE;
+ }
return 0;
+out:
+ for_each_engine(engine, vgpu->gvt->dev_priv, i) {
+ if (vgpu->reserve_ring_buffer_size[i]) {
+ kfree(vgpu->reserve_ring_buffer_va[i]);
+ vgpu->reserve_ring_buffer_va[i] = NULL;
+ vgpu->reserve_ring_buffer_size[i] = 0;
+ }
+ }
+ return -ENOMEM;
}
void intel_vgpu_reset_execlist(struct intel_vgpu *vgpu,
if (WARN_ON(mm->type != INTEL_GVT_MM_PPGTT))
return 0;
- atomic_inc(&mm->pincount);
-
if (!mm->shadowed) {
ret = shadow_mm(mm);
if (ret)
return ret;
}
+ atomic_inc(&mm->pincount);
list_del_init(&mm->lru_list);
list_add_tail(&mm->lru_list, &mm->vgpu->gvt->gtt.mm_lru_list_head);
return 0;
*/
se.val64 |= _PAGE_PRESENT | _PAGE_RW;
if (type == GTT_TYPE_PPGTT_PDE_PT)
- se.val64 |= PPAT_CACHED_INDEX;
+ se.val64 |= PPAT_CACHED;
for (i = 0; i < page_entry_num; i++)
ops->set_entry(scratch_pt, &se, i, false, 0, vgpu);
if (IS_BROADWELL(gvt->dev_priv) || IS_SKYLAKE(gvt->dev_priv)
|| IS_KABYLAKE(gvt->dev_priv)) {
info->max_support_vgpus = 8;
- info->cfg_space_size = 256;
+ info->cfg_space_size = PCI_CFG_SPACE_EXP_SIZE;
info->mmio_size = 2 * 1024 * 1024;
info->mmio_bar = 0;
info->gtt_start_offset = 8 * 1024 * 1024;
struct intel_vgpu_gm {
u64 aperture_sz;
u64 hidden_sz;
+ void *aperture_va;
struct drm_mm_node low_gm_node;
struct drm_mm_node high_gm_node;
};
bool disable_warn_untrack;
};
-#define INTEL_GVT_MAX_CFG_SPACE_SZ 256
#define INTEL_GVT_MAX_BAR_NUM 4
struct intel_vgpu_pci_bar {
};
struct intel_vgpu_cfg_space {
- unsigned char virtual_cfg_space[INTEL_GVT_MAX_CFG_SPACE_SZ];
+ unsigned char virtual_cfg_space[PCI_CFG_SPACE_EXP_SIZE];
struct intel_vgpu_pci_bar bar[INTEL_GVT_MAX_BAR_NUM];
};
struct list_head workload_q_head[I915_NUM_ENGINES];
struct kmem_cache *workloads;
atomic_t running_workload_num;
+ /* 1/2K for each reserve ring buffer */
+ void *reserve_ring_buffer_va[I915_NUM_ENGINES];
+ int reserve_ring_buffer_size[I915_NUM_ENGINES];
DECLARE_BITMAP(tlb_handle_pending, I915_NUM_ENGINES);
struct i915_gem_context *shadow_ctx;
DECLARE_BITMAP(shadow_ctx_desc_updated, I915_NUM_ENGINES);
int intel_vgpu_emulate_cfg_write(struct intel_vgpu *vgpu, unsigned int offset,
void *p_data, unsigned int bytes);
+static inline u64 intel_vgpu_get_bar_gpa(struct intel_vgpu *vgpu, int bar)
+{
+ /* We are 64bit bar. */
+ return (*(u64 *)(vgpu->cfg_space.virtual_cfg_space + bar)) &
+ PCI_BASE_ADDRESS_MEM_MASK;
+}
+
void intel_gvt_clean_opregion(struct intel_gvt *gvt);
int intel_gvt_init_opregion(struct intel_gvt *gvt);
__intel_vgpu_release(vgpu);
}
-static uint64_t intel_vgpu_get_bar0_addr(struct intel_vgpu *vgpu)
+static uint64_t intel_vgpu_get_bar_addr(struct intel_vgpu *vgpu, int bar)
{
u32 start_lo, start_hi;
u32 mem_type;
- int pos = PCI_BASE_ADDRESS_0;
- start_lo = (*(u32 *)(vgpu->cfg_space.virtual_cfg_space + pos)) &
+ start_lo = (*(u32 *)(vgpu->cfg_space.virtual_cfg_space + bar)) &
PCI_BASE_ADDRESS_MEM_MASK;
- mem_type = (*(u32 *)(vgpu->cfg_space.virtual_cfg_space + pos)) &
+ mem_type = (*(u32 *)(vgpu->cfg_space.virtual_cfg_space + bar)) &
PCI_BASE_ADDRESS_MEM_TYPE_MASK;
switch (mem_type) {
case PCI_BASE_ADDRESS_MEM_TYPE_64:
start_hi = (*(u32 *)(vgpu->cfg_space.virtual_cfg_space
- + pos + 4));
+ + bar + 4));
break;
case PCI_BASE_ADDRESS_MEM_TYPE_32:
case PCI_BASE_ADDRESS_MEM_TYPE_1M:
return ((u64)start_hi << 32) | start_lo;
}
+static int intel_vgpu_bar_rw(struct intel_vgpu *vgpu, int bar, uint64_t off,
+ void *buf, unsigned int count, bool is_write)
+{
+ uint64_t bar_start = intel_vgpu_get_bar_addr(vgpu, bar);
+ int ret;
+
+ if (is_write)
+ ret = intel_gvt_ops->emulate_mmio_write(vgpu,
+ bar_start + off, buf, count);
+ else
+ ret = intel_gvt_ops->emulate_mmio_read(vgpu,
+ bar_start + off, buf, count);
+ return ret;
+}
+
static ssize_t intel_vgpu_rw(struct mdev_device *mdev, char *buf,
size_t count, loff_t *ppos, bool is_write)
{
buf, count);
break;
case VFIO_PCI_BAR0_REGION_INDEX:
- case VFIO_PCI_BAR1_REGION_INDEX:
- if (is_write) {
- uint64_t bar0_start = intel_vgpu_get_bar0_addr(vgpu);
-
- ret = intel_gvt_ops->emulate_mmio_write(vgpu,
- bar0_start + pos, buf, count);
- } else {
- uint64_t bar0_start = intel_vgpu_get_bar0_addr(vgpu);
-
- ret = intel_gvt_ops->emulate_mmio_read(vgpu,
- bar0_start + pos, buf, count);
- }
+ ret = intel_vgpu_bar_rw(vgpu, PCI_BASE_ADDRESS_0, pos,
+ buf, count, is_write);
break;
case VFIO_PCI_BAR2_REGION_INDEX:
+ ret = intel_vgpu_bar_rw(vgpu, PCI_BASE_ADDRESS_2, pos,
+ buf, count, is_write);
+ break;
+ case VFIO_PCI_BAR1_REGION_INDEX:
case VFIO_PCI_BAR3_REGION_INDEX:
case VFIO_PCI_BAR4_REGION_INDEX:
case VFIO_PCI_BAR5_REGION_INDEX:
switch (info.index) {
case VFIO_PCI_CONFIG_REGION_INDEX:
info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
- info.size = INTEL_GVT_MAX_CFG_SPACE_SZ;
+ info.size = vgpu->gvt->device_info.cfg_space_size;
info.flags = VFIO_REGION_INFO_FLAG_READ |
VFIO_REGION_INFO_FLAG_WRITE;
break;
*/
int intel_vgpu_gpa_to_mmio_offset(struct intel_vgpu *vgpu, u64 gpa)
{
- u64 gttmmio_gpa = *(u64 *)(vgpu_cfg_space(vgpu) + PCI_BASE_ADDRESS_0) &
- ~GENMASK(3, 0);
+ u64 gttmmio_gpa = intel_vgpu_get_bar_gpa(vgpu, PCI_BASE_ADDRESS_0);
return gpa - gttmmio_gpa;
}
(reg >= gvt->device_info.gtt_start_offset \
&& reg < gvt->device_info.gtt_start_offset + gvt_ggtt_sz(gvt))
+static bool vgpu_gpa_is_aperture(struct intel_vgpu *vgpu, uint64_t gpa)
+{
+ u64 aperture_gpa = intel_vgpu_get_bar_gpa(vgpu, PCI_BASE_ADDRESS_2);
+ u64 aperture_sz = vgpu_aperture_sz(vgpu);
+
+ return gpa >= aperture_gpa && gpa < aperture_gpa + aperture_sz;
+}
+
+static int vgpu_aperture_rw(struct intel_vgpu *vgpu, uint64_t gpa,
+ void *pdata, unsigned int size, bool is_read)
+{
+ u64 aperture_gpa = intel_vgpu_get_bar_gpa(vgpu, PCI_BASE_ADDRESS_2);
+ u64 offset = gpa - aperture_gpa;
+
+ if (!vgpu_gpa_is_aperture(vgpu, gpa + size - 1)) {
+ gvt_vgpu_err("Aperture rw out of range, offset %llx, size %d\n",
+ offset, size);
+ return -EINVAL;
+ }
+
+ if (!vgpu->gm.aperture_va) {
+ gvt_vgpu_err("BAR is not enabled\n");
+ return -ENXIO;
+ }
+
+ if (is_read)
+ memcpy(pdata, vgpu->gm.aperture_va + offset, size);
+ else
+ memcpy(vgpu->gm.aperture_va + offset, pdata, size);
+ return 0;
+}
+
static void failsafe_emulate_mmio_rw(struct intel_vgpu *vgpu, uint64_t pa,
void *p_data, unsigned int bytes, bool read)
{
}
mutex_lock(&gvt->lock);
+ if (vgpu_gpa_is_aperture(vgpu, pa)) {
+ ret = vgpu_aperture_rw(vgpu, pa, p_data, bytes, true);
+ mutex_unlock(&gvt->lock);
+ return ret;
+ }
+
if (atomic_read(&vgpu->gtt.n_write_protected_guest_page)) {
struct intel_vgpu_guest_page *gp;
mutex_lock(&gvt->lock);
+ if (vgpu_gpa_is_aperture(vgpu, pa)) {
+ ret = vgpu_aperture_rw(vgpu, pa, p_data, bytes, false);
+ mutex_unlock(&gvt->lock);
+ return ret;
+ }
+
if (atomic_read(&vgpu->gtt.n_write_protected_guest_page)) {
struct intel_vgpu_guest_page *gp;
*/
if (mmio->in_context &&
((ctx_ctrl & inhibit_mask) != inhibit_mask) &&
- i915.enable_execlists)
+ i915_modparams.enable_execlists)
continue;
if (mmio->mask)
return -EINVAL;
}
- page = i915_gem_object_get_page(ctx_obj, LRC_PPHWSP_PN + i);
+ page = i915_gem_object_get_page(ctx_obj, LRC_HEADER_PAGES + i);
dst = kmap(page);
intel_gvt_hypervisor_read_gpa(vgpu, context_gpa, dst,
GTT_PAGE_SIZE);
ce->lrc_desc = desc;
}
+static int copy_workload_to_ring_buffer(struct intel_vgpu_workload *workload)
+{
+ struct intel_vgpu *vgpu = workload->vgpu;
+ void *shadow_ring_buffer_va;
+ u32 *cs;
+
+ /* allocate shadow ring buffer */
+ cs = intel_ring_begin(workload->req, workload->rb_len / sizeof(u32));
+ if (IS_ERR(cs)) {
+ gvt_vgpu_err("fail to alloc size =%ld shadow ring buffer\n",
+ workload->rb_len);
+ return PTR_ERR(cs);
+ }
+
+ shadow_ring_buffer_va = workload->shadow_ring_buffer_va;
+
+ /* get shadow ring buffer va */
+ workload->shadow_ring_buffer_va = cs;
+
+ memcpy(cs, shadow_ring_buffer_va,
+ workload->rb_len);
+
+ cs += workload->rb_len / sizeof(u32);
+ intel_ring_advance(workload->req, cs);
+
+ return 0;
+}
+
+void release_shadow_wa_ctx(struct intel_shadow_wa_ctx *wa_ctx)
+{
+ if (!wa_ctx->indirect_ctx.obj)
+ return;
+
+ i915_gem_object_unpin_map(wa_ctx->indirect_ctx.obj);
+ i915_gem_object_put(wa_ctx->indirect_ctx.obj);
+}
+
/**
* intel_gvt_scan_and_shadow_workload - audit the workload by scanning and
* shadow it as well, include ringbuffer,wa_ctx and ctx.
int ring_id = workload->ring_id;
struct i915_gem_context *shadow_ctx = workload->vgpu->shadow_ctx;
struct drm_i915_private *dev_priv = workload->vgpu->gvt->dev_priv;
+ struct intel_engine_cs *engine = dev_priv->engine[ring_id];
struct drm_i915_gem_request *rq;
struct intel_vgpu *vgpu = workload->vgpu;
+ struct intel_ring *ring;
int ret;
lockdep_assert_held(&dev_priv->drm.struct_mutex);
shadow_context_descriptor_update(shadow_ctx,
dev_priv->engine[ring_id]);
- rq = i915_gem_request_alloc(dev_priv->engine[ring_id], shadow_ctx);
- if (IS_ERR(rq)) {
- gvt_vgpu_err("fail to allocate gem request\n");
- ret = PTR_ERR(rq);
- goto out;
- }
-
- gvt_dbg_sched("ring id %d get i915 gem request %p\n", ring_id, rq);
-
- workload->req = i915_gem_request_get(rq);
-
ret = intel_gvt_scan_and_shadow_ringbuffer(workload);
if (ret)
- goto out;
+ goto err_scan;
if ((workload->ring_id == RCS) &&
(workload->wa_ctx.indirect_ctx.size != 0)) {
ret = intel_gvt_scan_and_shadow_wa_ctx(&workload->wa_ctx);
if (ret)
- goto out;
+ goto err_scan;
+ }
+
+ /* pin shadow context by gvt even the shadow context will be pinned
+ * when i915 alloc request. That is because gvt will update the guest
+ * context from shadow context when workload is completed, and at that
+ * moment, i915 may already unpined the shadow context to make the
+ * shadow_ctx pages invalid. So gvt need to pin itself. After update
+ * the guest context, gvt can unpin the shadow_ctx safely.
+ */
+ ring = engine->context_pin(engine, shadow_ctx);
+ if (IS_ERR(ring)) {
+ ret = PTR_ERR(ring);
+ gvt_vgpu_err("fail to pin shadow context\n");
+ goto err_shadow;
}
ret = populate_shadow_context(workload);
if (ret)
- goto out;
+ goto err_unpin;
+ rq = i915_gem_request_alloc(dev_priv->engine[ring_id], shadow_ctx);
+ if (IS_ERR(rq)) {
+ gvt_vgpu_err("fail to allocate gem request\n");
+ ret = PTR_ERR(rq);
+ goto err_unpin;
+ }
+
+ gvt_dbg_sched("ring id %d get i915 gem request %p\n", ring_id, rq);
+
+ workload->req = i915_gem_request_get(rq);
+ ret = copy_workload_to_ring_buffer(workload);
+ if (ret)
+ goto err_unpin;
workload->shadowed = true;
+ return 0;
-out:
+err_unpin:
+ engine->context_unpin(engine, shadow_ctx);
+err_shadow:
+ release_shadow_wa_ctx(&workload->wa_ctx);
+err_scan:
return ret;
}
struct i915_gem_context *shadow_ctx = workload->vgpu->shadow_ctx;
struct drm_i915_private *dev_priv = workload->vgpu->gvt->dev_priv;
struct intel_engine_cs *engine = dev_priv->engine[ring_id];
- struct intel_vgpu *vgpu = workload->vgpu;
- struct intel_ring *ring;
int ret = 0;
gvt_dbg_sched("ring id %d prepare to dispatch workload %p\n",
if (workload->prepare) {
ret = workload->prepare(workload);
- if (ret)
+ if (ret) {
+ engine->context_unpin(engine, shadow_ctx);
goto out;
- }
-
- /* pin shadow context by gvt even the shadow context will be pinned
- * when i915 alloc request. That is because gvt will update the guest
- * context from shadow context when workload is completed, and at that
- * moment, i915 may already unpined the shadow context to make the
- * shadow_ctx pages invalid. So gvt need to pin itself. After update
- * the guest context, gvt can unpin the shadow_ctx safely.
- */
- ring = engine->context_pin(engine, shadow_ctx);
- if (IS_ERR(ring)) {
- ret = PTR_ERR(ring);
- gvt_vgpu_err("fail to pin shadow context\n");
- goto out;
+ }
}
out:
return;
}
- page = i915_gem_object_get_page(ctx_obj, LRC_PPHWSP_PN + i);
+ page = i915_gem_object_get_page(ctx_obj, LRC_HEADER_PAGES + i);
src = kmap(page);
intel_gvt_hypervisor_write_gpa(vgpu, context_gpa, src,
GTT_PAGE_SIZE);
void intel_vgpu_clean_gvt_context(struct intel_vgpu *vgpu);
+void release_shadow_wa_ctx(struct intel_shadow_wa_ctx *wa_ctx);
#endif
#undef PRINT_FLAG
kernel_param_lock(THIS_MODULE);
-#define PRINT_PARAM(T, x) seq_print_param(m, #x, #T, &i915.x);
+#define PRINT_PARAM(T, x, ...) seq_print_param(m, #x, #T, &i915_modparams.x);
I915_PARAMS_FOR_EACH(PRINT_PARAM);
#undef PRINT_PARAM
kernel_param_unlock(THIS_MODULE);
if (waitqueue_active(&dev_priv->gpu_error.reset_queue))
seq_puts(m, "struct_mutex blocked for reset\n");
- if (!i915.enable_hangcheck) {
+ if (!i915_modparams.enable_hangcheck) {
seq_puts(m, "Hangcheck disabled\n");
return 0;
}
struct intel_uncore_forcewake_domain *fw_domain;
unsigned int tmp;
+ seq_printf(m, "user.bypass_count = %u\n",
+ i915->uncore.user_forcewake.count);
+
for_each_fw_domain(fw_domain, i915, tmp)
seq_printf(m, "%s.wake_count = %u\n",
intel_uncore_forcewake_domain_to_str(fw_domain->id),
intel_runtime_pm_get(dev_priv);
seq_printf(m, "Enabled by kernel parameter: %s\n",
- yesno(i915.enable_ips));
+ yesno(i915_modparams.enable_ips));
if (INTEL_GEN(dev_priv) >= 8) {
seq_puts(m, "Currently: unknown\n");
enum intel_engine_id id;
int ret;
- if (!i915.enable_execlists) {
+ if (!i915_modparams.enable_execlists) {
seq_printf(m, "Logical Ring Contexts are disabled\n");
return 0;
}
seq_printf(m, "\tPriority %d, GuC stage index: %u, PD offset 0x%x\n",
client->priority, client->stage_id, client->proc_desc_offset);
- seq_printf(m, "\tDoorbell id %d, offset: 0x%lx, cookie 0x%x\n",
- client->doorbell_id, client->doorbell_offset, client->doorbell_cookie);
- seq_printf(m, "\tWQ size %d, offset: 0x%x, tail %d\n",
- client->wq_size, client->wq_offset, client->wq_tail);
-
- seq_printf(m, "\tWork queue full: %u\n", client->no_wq_space);
+ seq_printf(m, "\tDoorbell id %d, offset: 0x%lx\n",
+ client->doorbell_id, client->doorbell_offset);
for_each_engine(engine, dev_priv, id) {
u64 submissions = client->submissions[id];
if (!dev_priv->guc.log.vma)
return -EINVAL;
- *val = i915.guc_log_level;
+ *val = i915_modparams.guc_log_level;
return 0;
}
seq_printf(m, "\tBBADDR: 0x%08x_%08x\n",
upper_32_bits(addr), lower_32_bits(addr));
- if (i915.enable_execlists) {
+ if (i915_modparams.enable_execlists) {
+ const u32 *hws = &engine->status_page.page_addr[I915_HWS_CSB_BUF0_INDEX];
+ struct intel_engine_execlists * const execlists = &engine->execlists;
u32 ptr, read, write;
unsigned int idx;
ptr = I915_READ(RING_CONTEXT_STATUS_PTR(engine));
read = GEN8_CSB_READ_PTR(ptr);
write = GEN8_CSB_WRITE_PTR(ptr);
- seq_printf(m, "\tExeclist CSB read %d, write %d, interrupt posted? %s\n",
- read, write,
+ seq_printf(m, "\tExeclist CSB read %d [%d cached], write %d [%d from hws], interrupt posted? %s\n",
+ read, execlists->csb_head,
+ write,
+ intel_read_status_page(engine, intel_hws_csb_write_index(engine->i915)),
yesno(test_bit(ENGINE_IRQ_EXECLIST,
&engine->irq_posted)));
if (read >= GEN8_CSB_ENTRIES)
write += GEN8_CSB_ENTRIES;
while (read < write) {
idx = ++read % GEN8_CSB_ENTRIES;
- seq_printf(m, "\tExeclist CSB[%d]: 0x%08x, context: %d\n",
+ seq_printf(m, "\tExeclist CSB[%d]: 0x%08x [0x%08x in hwsp], context: %d [%d in hwsp]\n",
idx,
I915_READ(RING_CONTEXT_STATUS_BUF_LO(engine, idx)),
- I915_READ(RING_CONTEXT_STATUS_BUF_HI(engine, idx)));
+ hws[idx * 2],
+ I915_READ(RING_CONTEXT_STATUS_BUF_HI(engine, idx)),
+ hws[idx * 2 + 1]);
}
rcu_read_lock();
- for (idx = 0; idx < ARRAY_SIZE(engine->execlist_port); idx++) {
+ for (idx = 0; idx < execlists_num_ports(execlists); idx++) {
unsigned int count;
- rq = port_unpack(&engine->execlist_port[idx],
- &count);
+ rq = port_unpack(&execlists->port[idx], &count);
if (rq) {
seq_printf(m, "\t\tELSP[%d] count=%d, ",
idx, count);
rcu_read_unlock();
spin_lock_irq(&engine->timeline->lock);
- for (rb = engine->execlist_first; rb; rb = rb_next(rb)){
+ for (rb = execlists->first; rb; rb = rb_next(rb)) {
struct i915_priolist *p =
rb_entry(rb, typeof(*p), node);
enum intel_engine_id id;
int j, ret;
- if (!i915.semaphores) {
+ if (!i915_modparams.semaphores) {
seq_puts(m, "Semaphores are disabled\n");
return 0;
}
return 0;
}
+static int i915_ipc_status_show(struct seq_file *m, void *data)
+{
+ struct drm_i915_private *dev_priv = m->private;
+
+ seq_printf(m, "Isochronous Priority Control: %s\n",
+ yesno(dev_priv->ipc_enabled));
+ return 0;
+}
+
+static int i915_ipc_status_open(struct inode *inode, struct file *file)
+{
+ struct drm_i915_private *dev_priv = inode->i_private;
+
+ if (!HAS_IPC(dev_priv))
+ return -ENODEV;
+
+ return single_open(file, i915_ipc_status_show, dev_priv);
+}
+
+static ssize_t i915_ipc_status_write(struct file *file, const char __user *ubuf,
+ size_t len, loff_t *offp)
+{
+ struct seq_file *m = file->private_data;
+ struct drm_i915_private *dev_priv = m->private;
+ int ret;
+ bool enable;
+
+ ret = kstrtobool_from_user(ubuf, len, &enable);
+ if (ret < 0)
+ return ret;
+
+ intel_runtime_pm_get(dev_priv);
+ if (!dev_priv->ipc_enabled && enable)
+ DRM_INFO("Enabling IPC: WM will be proper only after next commit\n");
+ dev_priv->wm.distrust_bios_wm = true;
+ dev_priv->ipc_enabled = enable;
+ intel_enable_ipc(dev_priv);
+ intel_runtime_pm_put(dev_priv);
+
+ return len;
+}
+
+static const struct file_operations i915_ipc_status_fops = {
+ .owner = THIS_MODULE,
+ .open = i915_ipc_status_open,
+ .read = seq_read,
+ .llseek = seq_lseek,
+ .release = single_release,
+ .write = i915_ipc_status_write
+};
+
static int i915_ddb_info(struct seq_file *m, void *unused)
{
struct drm_i915_private *dev_priv = node_to_i915(m->private);
static int i915_forcewake_open(struct inode *inode, struct file *file)
{
- struct drm_i915_private *dev_priv = inode->i_private;
+ struct drm_i915_private *i915 = inode->i_private;
- if (INTEL_GEN(dev_priv) < 6)
+ if (INTEL_GEN(i915) < 6)
return 0;
- intel_runtime_pm_get(dev_priv);
- intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
+ intel_runtime_pm_get(i915);
+ intel_uncore_forcewake_user_get(i915);
return 0;
}
static int i915_forcewake_release(struct inode *inode, struct file *file)
{
- struct drm_i915_private *dev_priv = inode->i_private;
+ struct drm_i915_private *i915 = inode->i_private;
- if (INTEL_GEN(dev_priv) < 6)
+ if (INTEL_GEN(i915) < 6)
return 0;
- intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
- intel_runtime_pm_put(dev_priv);
+ intel_uncore_forcewake_user_put(i915);
+ intel_runtime_pm_put(i915);
return 0;
}
{"i915_dp_test_type", &i915_displayport_test_type_fops},
{"i915_dp_test_active", &i915_displayport_test_active_fops},
{"i915_guc_log_control", &i915_guc_log_control_fops},
- {"i915_hpd_storm_ctl", &i915_hpd_storm_ctl_fops}
+ {"i915_hpd_storm_ctl", &i915_hpd_storm_ctl_fops},
+ {"i915_ipc_status", &i915_ipc_status_fops}
};
int i915_debugfs_register(struct drm_i915_private *dev_priv)
bool __i915_inject_load_failure(const char *func, int line)
{
- if (i915_load_fail_count >= i915.inject_load_failure)
+ if (i915_load_fail_count >= i915_modparams.inject_load_failure)
return false;
- if (++i915_load_fail_count == i915.inject_load_failure) {
+ if (++i915_load_fail_count == i915_modparams.inject_load_failure) {
DRM_INFO("Injecting failure at checkpoint %u [%s:%d]\n",
- i915.inject_load_failure, func, line);
+ i915_modparams.inject_load_failure, func, line);
return true;
}
static bool i915_error_injected(struct drm_i915_private *dev_priv)
{
- return i915.inject_load_failure &&
- i915_load_fail_count == i915.inject_load_failure;
+ return i915_modparams.inject_load_failure &&
+ i915_load_fail_count == i915_modparams.inject_load_failure;
}
#define i915_load_error(dev_priv, fmt, ...) \
value = USES_PPGTT(dev_priv);
break;
case I915_PARAM_HAS_SEMAPHORES:
- value = i915.semaphores;
+ value = i915_modparams.semaphores;
break;
case I915_PARAM_HAS_SECURE_BATCHES:
value = capable(CAP_SYS_ADMIN);
return -ENODEV;
break;
case I915_PARAM_HAS_GPU_RESET:
- value = i915.enable_hangcheck && intel_has_gpu_reset(dev_priv);
+ value = i915_modparams.enable_hangcheck &&
+ intel_has_gpu_reset(dev_priv);
if (value && intel_has_reset_engine(dev_priv))
value = 2;
break;
memcpy(device_info, match_info, sizeof(*device_info));
device_info->device_id = dev_priv->drm.pdev->device;
+ BUILD_BUG_ON(INTEL_MAX_PLATFORMS >
+ sizeof(device_info->platform_mask) * BITS_PER_BYTE);
+ device_info->platform_mask = BIT(device_info->platform);
+
BUG_ON(device_info->gen > sizeof(device_info->gen_mask) * BITS_PER_BYTE);
device_info->gen_mask = BIT(device_info->gen - 1);
static void intel_sanitize_options(struct drm_i915_private *dev_priv)
{
- i915.enable_execlists =
+ i915_modparams.enable_execlists =
intel_sanitize_enable_execlists(dev_priv,
- i915.enable_execlists);
+ i915_modparams.enable_execlists);
/*
* i915.enable_ppgtt is read-only, so do an early pass to validate the
* do this now so that we can print out any log messages once rather
* than every time we check intel_enable_ppgtt().
*/
- i915.enable_ppgtt =
- intel_sanitize_enable_ppgtt(dev_priv, i915.enable_ppgtt);
- DRM_DEBUG_DRIVER("ppgtt mode: %i\n", i915.enable_ppgtt);
+ i915_modparams.enable_ppgtt =
+ intel_sanitize_enable_ppgtt(dev_priv,
+ i915_modparams.enable_ppgtt);
+ DRM_DEBUG_DRIVER("ppgtt mode: %i\n", i915_modparams.enable_ppgtt);
- i915.semaphores = intel_sanitize_semaphores(dev_priv, i915.semaphores);
- DRM_DEBUG_DRIVER("use GPU semaphores? %s\n", yesno(i915.semaphores));
+ i915_modparams.semaphores =
+ intel_sanitize_semaphores(dev_priv, i915_modparams.semaphores);
+ DRM_DEBUG_DRIVER("use GPU semaphores? %s\n",
+ yesno(i915_modparams.semaphores));
intel_uc_sanitize_options(dev_priv);
int ret;
/* Enable nuclear pageflip on ILK+ */
- if (!i915.nuclear_pageflip && match_info->gen < 5)
+ if (!i915_modparams.nuclear_pageflip && match_info->gen < 5)
driver.driver_features &= ~DRIVER_ATOMIC;
ret = -ENOMEM;
intel_runtime_pm_enable(dev_priv);
- dev_priv->ipc_enabled = false;
+ intel_init_ipc(dev_priv);
if (IS_ENABLED(CONFIG_DRM_I915_DEBUG))
DRM_INFO("DRM_I915_DEBUG enabled\n");
if (!IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv))
intel_hpd_init(dev_priv);
+ intel_enable_ipc(dev_priv);
+
enable_rpm_wakeref_asserts(dev_priv);
if (ret)
#define I915_STATE_WARN(condition, format...) ({ \
int __ret_warn_on = !!(condition); \
if (unlikely(__ret_warn_on)) \
- if (!WARN(i915.verbose_state_checks, format)) \
+ if (!WARN(i915_modparams.verbose_state_checks, format)) \
DRM_ERROR(format); \
unlikely(__ret_warn_on); \
})
{
uint_fixed_16_16_t fp;
- WARN_ON(val >> 16);
+ WARN_ON(val > U16_MAX);
fp.val = val << 16;
return fp;
static inline uint_fixed_16_16_t clamp_u64_to_fixed16(uint64_t val)
{
uint_fixed_16_16_t fp;
- WARN_ON(val >> 32);
- fp.val = clamp_t(uint32_t, val, 0, ~0);
+ WARN_ON(val > U32_MAX);
+ fp.val = (uint32_t) val;
return fp;
}
intermediate_val = (uint64_t) val * mul.val;
intermediate_val = DIV_ROUND_UP_ULL(intermediate_val, 1 << 16);
- WARN_ON(intermediate_val >> 32);
- return clamp_t(uint32_t, intermediate_val, 0, ~0);
+ WARN_ON(intermediate_val > U32_MAX);
+ return (uint32_t) intermediate_val;
}
static inline uint_fixed_16_16_t mul_fixed16(uint_fixed_16_16_t val,
interm_val = (uint64_t)val << 16;
interm_val = DIV_ROUND_UP_ULL(interm_val, d.val);
- WARN_ON(interm_val >> 32);
- return clamp_t(uint32_t, interm_val, 0, ~0);
+ WARN_ON(interm_val > U32_MAX);
+ return (uint32_t) interm_val;
}
static inline uint_fixed_16_16_t mul_u32_fixed16(uint32_t val,
func(has_fpga_dbg); \
func(has_full_ppgtt); \
func(has_full_48bit_ppgtt); \
- func(has_gmbus_irq); \
func(has_gmch_display); \
func(has_guc); \
func(has_guc_ct); \
func(cursor_needs_physical); \
func(hws_needs_physical); \
func(overlay_needs_physical); \
- func(supports_tv);
+ func(supports_tv); \
+ func(has_ipc);
struct sseu_dev_info {
u8 slice_mask;
};
struct intel_device_info {
- u32 display_mmio_offset;
u16 device_id;
- u8 num_pipes;
- u8 num_sprites[I915_MAX_PIPES];
- u8 num_scalers[I915_MAX_PIPES];
- u8 gen;
u16 gen_mask;
- enum intel_platform platform;
+
+ u8 gen;
u8 gt; /* GT number, 0 if undefined */
- u8 ring_mask; /* Rings supported by the HW */
u8 num_rings;
+ u8 ring_mask; /* Rings supported by the HW */
+
+ enum intel_platform platform;
+ u32 platform_mask;
+
+ u32 display_mmio_offset;
+
+ u8 num_pipes;
+ u8 num_sprites[I915_MAX_PIPES];
+ u8 num_scalers[I915_MAX_PIPES];
+
#define DEFINE_FLAG(name) u8 name:1
DEV_INFO_FOR_EACH_FLAG(DEFINE_FLAG);
#undef DEFINE_FLAG
u16 ddb_size; /* in blocks */
+
/* Register offsets for the various display pipes and transcoders */
int pipe_offsets[I915_MAX_TRANSCODERS];
int trans_offsets[I915_MAX_TRANSCODERS];
u32 seqno;
u32 head;
u32 tail;
- } *requests, execlist[2];
+ } *requests, execlist[EXECLIST_MAX_PORTS];
+ unsigned int num_ports;
struct drm_i915_error_waiter {
char comm[TASK_COMM_LEN];
bool y_cord_support;
bool colorimetry_support;
bool alpm;
+
+ void (*enable_source)(struct intel_dp *,
+ const struct intel_crtc_state *);
+ void (*disable_source)(struct intel_dp *,
+ const struct intel_crtc_state *);
+ void (*enable_sink)(struct intel_dp *);
+ void (*activate)(struct intel_dp *);
+ void (*setup_vsc)(struct intel_dp *, const struct intel_crtc_state *);
};
enum intel_pch {
uint8_t plane_res_l;
};
+/* Stores plane specific WM parameters */
+struct skl_wm_params {
+ bool x_tiled, y_tiled;
+ bool rc_surface;
+ uint32_t width;
+ uint8_t cpp;
+ uint32_t plane_pixel_rate;
+ uint32_t y_min_scanlines;
+ uint32_t plane_bytes_per_line;
+ uint_fixed_16_16_t plane_blocks_per_line;
+ uint_fixed_16_16_t y_tile_minimum;
+ uint32_t linetime_us;
+};
+
/*
* This struct helps tracking the state needed for runtime PM, which puts the
* device in PCI D3 state. Notice that when this happens, nothing on the
DECLARE_HASHTABLE(mm_structs, 7);
struct mutex mm_lock;
+ struct intel_ppat ppat;
+
/* Kernel Modesetting */
struct intel_crtc *plane_to_crtc_mapping[I915_MAX_PIPES];
#define for_each_sgt_dma(__dmap, __iter, __sgt) \
for ((__iter) = __sgt_iter((__sgt)->sgl, true); \
((__dmap) = (__iter).dma + (__iter).curr); \
- (((__iter).curr += PAGE_SIZE) < (__iter).max) || \
- ((__iter) = __sgt_iter(__sg_next((__iter).sgp), true), 0))
+ (((__iter).curr += PAGE_SIZE) >= (__iter).max) ? \
+ (__iter) = __sgt_iter(__sg_next((__iter).sgp), true), 0 : 0)
/**
* for_each_sgt_page - iterate over the pages of the given sg_table
for ((__iter) = __sgt_iter((__sgt)->sgl, false); \
((__pp) = (__iter).pfn == 0 ? NULL : \
pfn_to_page((__iter).pfn + ((__iter).curr >> PAGE_SHIFT))); \
- (((__iter).curr += PAGE_SIZE) < (__iter).max) || \
- ((__iter) = __sgt_iter(__sg_next((__iter).sgp), false), 0))
+ (((__iter).curr += PAGE_SIZE) >= (__iter).max) ? \
+ (__iter) = __sgt_iter(__sg_next((__iter).sgp), false), 0 : 0)
+
+static inline unsigned int i915_sg_segment_size(void)
+{
+ unsigned int size = swiotlb_max_segment();
+
+ if (size == 0)
+ return SCATTERLIST_MAX_SEGMENT;
+
+ size = rounddown(size, PAGE_SIZE);
+ /* swiotlb_max_segment_size can return 1 byte when it means one page. */
+ if (size < PAGE_SIZE)
+ size = PAGE_SIZE;
+
+ return size;
+}
static inline const struct intel_device_info *
intel_info(const struct drm_i915_private *dev_priv)
#define INTEL_REVID(dev_priv) ((dev_priv)->drm.pdev->revision)
#define GEN_FOREVER (0)
+
+#define INTEL_GEN_MASK(s, e) ( \
+ BUILD_BUG_ON_ZERO(!__builtin_constant_p(s)) + \
+ BUILD_BUG_ON_ZERO(!__builtin_constant_p(e)) + \
+ GENMASK((e) != GEN_FOREVER ? (e) - 1 : BITS_PER_LONG - 1, \
+ (s) != GEN_FOREVER ? (s) - 1 : 0) \
+)
+
/*
* Returns true if Gen is in inclusive range [Start, End].
*
* Use GEN_FOREVER for unbound start and or end.
*/
-#define IS_GEN(dev_priv, s, e) ({ \
- unsigned int __s = (s), __e = (e); \
- BUILD_BUG_ON(!__builtin_constant_p(s)); \
- BUILD_BUG_ON(!__builtin_constant_p(e)); \
- if ((__s) != GEN_FOREVER) \
- __s = (s) - 1; \
- if ((__e) == GEN_FOREVER) \
- __e = BITS_PER_LONG - 1; \
- else \
- __e = (e) - 1; \
- !!((dev_priv)->info.gen_mask & GENMASK((__e), (__s))); \
-})
+#define IS_GEN(dev_priv, s, e) \
+ (!!((dev_priv)->info.gen_mask & INTEL_GEN_MASK((s), (e))))
/*
* Return true if revision is in range [since,until] inclusive.
#define IS_REVID(p, since, until) \
(INTEL_REVID(p) >= (since) && INTEL_REVID(p) <= (until))
-#define IS_I830(dev_priv) ((dev_priv)->info.platform == INTEL_I830)
-#define IS_I845G(dev_priv) ((dev_priv)->info.platform == INTEL_I845G)
-#define IS_I85X(dev_priv) ((dev_priv)->info.platform == INTEL_I85X)
-#define IS_I865G(dev_priv) ((dev_priv)->info.platform == INTEL_I865G)
-#define IS_I915G(dev_priv) ((dev_priv)->info.platform == INTEL_I915G)
-#define IS_I915GM(dev_priv) ((dev_priv)->info.platform == INTEL_I915GM)
-#define IS_I945G(dev_priv) ((dev_priv)->info.platform == INTEL_I945G)
-#define IS_I945GM(dev_priv) ((dev_priv)->info.platform == INTEL_I945GM)
-#define IS_I965G(dev_priv) ((dev_priv)->info.platform == INTEL_I965G)
-#define IS_I965GM(dev_priv) ((dev_priv)->info.platform == INTEL_I965GM)
-#define IS_G45(dev_priv) ((dev_priv)->info.platform == INTEL_G45)
-#define IS_GM45(dev_priv) ((dev_priv)->info.platform == INTEL_GM45)
+#define IS_PLATFORM(dev_priv, p) ((dev_priv)->info.platform_mask & BIT(p))
+
+#define IS_I830(dev_priv) IS_PLATFORM(dev_priv, INTEL_I830)
+#define IS_I845G(dev_priv) IS_PLATFORM(dev_priv, INTEL_I845G)
+#define IS_I85X(dev_priv) IS_PLATFORM(dev_priv, INTEL_I85X)
+#define IS_I865G(dev_priv) IS_PLATFORM(dev_priv, INTEL_I865G)
+#define IS_I915G(dev_priv) IS_PLATFORM(dev_priv, INTEL_I915G)
+#define IS_I915GM(dev_priv) IS_PLATFORM(dev_priv, INTEL_I915GM)
+#define IS_I945G(dev_priv) IS_PLATFORM(dev_priv, INTEL_I945G)
+#define IS_I945GM(dev_priv) IS_PLATFORM(dev_priv, INTEL_I945GM)
+#define IS_I965G(dev_priv) IS_PLATFORM(dev_priv, INTEL_I965G)
+#define IS_I965GM(dev_priv) IS_PLATFORM(dev_priv, INTEL_I965GM)
+#define IS_G45(dev_priv) IS_PLATFORM(dev_priv, INTEL_G45)
+#define IS_GM45(dev_priv) IS_PLATFORM(dev_priv, INTEL_GM45)
#define IS_G4X(dev_priv) (IS_G45(dev_priv) || IS_GM45(dev_priv))
#define IS_PINEVIEW_G(dev_priv) (INTEL_DEVID(dev_priv) == 0xa001)
#define IS_PINEVIEW_M(dev_priv) (INTEL_DEVID(dev_priv) == 0xa011)
-#define IS_PINEVIEW(dev_priv) ((dev_priv)->info.platform == INTEL_PINEVIEW)
-#define IS_G33(dev_priv) ((dev_priv)->info.platform == INTEL_G33)
+#define IS_PINEVIEW(dev_priv) IS_PLATFORM(dev_priv, INTEL_PINEVIEW)
+#define IS_G33(dev_priv) IS_PLATFORM(dev_priv, INTEL_G33)
#define IS_IRONLAKE_M(dev_priv) (INTEL_DEVID(dev_priv) == 0x0046)
-#define IS_IVYBRIDGE(dev_priv) ((dev_priv)->info.platform == INTEL_IVYBRIDGE)
+#define IS_IVYBRIDGE(dev_priv) IS_PLATFORM(dev_priv, INTEL_IVYBRIDGE)
#define IS_IVB_GT1(dev_priv) (IS_IVYBRIDGE(dev_priv) && \
(dev_priv)->info.gt == 1)
-#define IS_VALLEYVIEW(dev_priv) ((dev_priv)->info.platform == INTEL_VALLEYVIEW)
-#define IS_CHERRYVIEW(dev_priv) ((dev_priv)->info.platform == INTEL_CHERRYVIEW)
-#define IS_HASWELL(dev_priv) ((dev_priv)->info.platform == INTEL_HASWELL)
-#define IS_BROADWELL(dev_priv) ((dev_priv)->info.platform == INTEL_BROADWELL)
-#define IS_SKYLAKE(dev_priv) ((dev_priv)->info.platform == INTEL_SKYLAKE)
-#define IS_BROXTON(dev_priv) ((dev_priv)->info.platform == INTEL_BROXTON)
-#define IS_KABYLAKE(dev_priv) ((dev_priv)->info.platform == INTEL_KABYLAKE)
-#define IS_GEMINILAKE(dev_priv) ((dev_priv)->info.platform == INTEL_GEMINILAKE)
-#define IS_COFFEELAKE(dev_priv) ((dev_priv)->info.platform == INTEL_COFFEELAKE)
-#define IS_CANNONLAKE(dev_priv) ((dev_priv)->info.platform == INTEL_CANNONLAKE)
+#define IS_VALLEYVIEW(dev_priv) IS_PLATFORM(dev_priv, INTEL_VALLEYVIEW)
+#define IS_CHERRYVIEW(dev_priv) IS_PLATFORM(dev_priv, INTEL_CHERRYVIEW)
+#define IS_HASWELL(dev_priv) IS_PLATFORM(dev_priv, INTEL_HASWELL)
+#define IS_BROADWELL(dev_priv) IS_PLATFORM(dev_priv, INTEL_BROADWELL)
+#define IS_SKYLAKE(dev_priv) IS_PLATFORM(dev_priv, INTEL_SKYLAKE)
+#define IS_BROXTON(dev_priv) IS_PLATFORM(dev_priv, INTEL_BROXTON)
+#define IS_KABYLAKE(dev_priv) IS_PLATFORM(dev_priv, INTEL_KABYLAKE)
+#define IS_GEMINILAKE(dev_priv) IS_PLATFORM(dev_priv, INTEL_GEMINILAKE)
+#define IS_COFFEELAKE(dev_priv) IS_PLATFORM(dev_priv, INTEL_COFFEELAKE)
+#define IS_CANNONLAKE(dev_priv) IS_PLATFORM(dev_priv, INTEL_CANNONLAKE)
#define IS_MOBILE(dev_priv) ((dev_priv)->info.is_mobile)
#define IS_HSW_EARLY_SDV(dev_priv) (IS_HASWELL(dev_priv) && \
(INTEL_DEVID(dev_priv) & 0xFF00) == 0x0C00)
(dev_priv)->info.gt == 3)
#define IS_CFL_ULT(dev_priv) (IS_COFFEELAKE(dev_priv) && \
(INTEL_DEVID(dev_priv) & 0x00F0) == 0x00A0)
+#define IS_CFL_GT2(dev_priv) (IS_COFFEELAKE(dev_priv) && \
+ (dev_priv)->info.gt == 2)
#define IS_ALPHA_SUPPORT(intel_info) ((intel_info)->is_alpha_support)
#define HAS_LOGICAL_RING_CONTEXTS(dev_priv) \
((dev_priv)->info.has_logical_ring_contexts)
-#define USES_PPGTT(dev_priv) (i915.enable_ppgtt)
-#define USES_FULL_PPGTT(dev_priv) (i915.enable_ppgtt >= 2)
-#define USES_FULL_48BIT_PPGTT(dev_priv) (i915.enable_ppgtt == 3)
+#define USES_PPGTT(dev_priv) (i915_modparams.enable_ppgtt)
+#define USES_FULL_PPGTT(dev_priv) (i915_modparams.enable_ppgtt >= 2)
+#define USES_FULL_48BIT_PPGTT(dev_priv) (i915_modparams.enable_ppgtt == 3)
#define HAS_OVERLAY(dev_priv) ((dev_priv)->info.has_overlay)
#define OVERLAY_NEEDS_PHYSICAL(dev_priv) \
* even when in MSI mode. This results in spurious interrupt warnings if the
* legacy irq no. is shared with another device. The kernel then disables that
* interrupt source and so prevents the other device from working properly.
+ *
+ * Since we don't enable MSI anymore on gen4, we can always use GMBUS/AUX
+ * interrupts.
*/
-#define HAS_AUX_IRQ(dev_priv) ((dev_priv)->info.gen >= 5)
-#define HAS_GMBUS_IRQ(dev_priv) ((dev_priv)->info.has_gmbus_irq)
+#define HAS_AUX_IRQ(dev_priv) true
+#define HAS_GMBUS_IRQ(dev_priv) (INTEL_GEN(dev_priv) >= 4)
/* With the 945 and later, Y tiling got adjusted so that it was 32 128-byte
* rows, which changed the alignment requirements and fence programming.
#define HAS_RUNTIME_PM(dev_priv) ((dev_priv)->info.has_runtime_pm)
#define HAS_64BIT_RELOC(dev_priv) ((dev_priv)->info.has_64bit_reloc)
+#define HAS_IPC(dev_priv) ((dev_priv)->info.has_ipc)
+
/*
* For now, anything with a GuC requires uCode loading, and then supports
* command submission once loaded. But these are logically independent
{
unsigned long delay;
- if (unlikely(!i915.enable_hangcheck))
+ if (unlikely(!i915_modparams.enable_hangcheck))
return;
/* Don't continually defer the hangcheck so that it is always run at
return dev_priv->vgpu.active;
}
+u32 i915_pipestat_enable_mask(struct drm_i915_private *dev_priv,
+ enum pipe pipe);
void
i915_enable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe,
u32 status_mask);
unsigned long addr, unsigned long pfn, unsigned long size,
struct io_mapping *iomap);
+static inline int intel_hws_csb_write_index(struct drm_i915_private *i915)
+{
+ if (INTEL_GEN(i915) >= 10)
+ return CNL_HWS_CSB_WRITE_INDEX;
+ else
+ return I915_HWS_CSB_WRITE_INDEX;
+}
+
#endif
* the alignment of the buddy allocation will naturally match.
*/
phys = drm_pci_alloc(obj->base.dev,
- obj->base.size,
+ roundup_pow_of_two(obj->base.size),
roundup_pow_of_two(obj->base.size));
if (!phys)
return ERR_PTR(-ENOMEM);
switch (obj->base.write_domain) {
case I915_GEM_DOMAIN_GTT:
- if (INTEL_GEN(dev_priv) >= 6 && !HAS_LLC(dev_priv)) {
+ if (!HAS_LLC(dev_priv)) {
intel_runtime_pm_get(dev_priv);
spin_lock_irq(&dev_priv->uncore.lock);
- POSTING_READ_FW(RING_ACTHD(dev_priv->engine[RCS]->mmio_base));
+ POSTING_READ_FW(RING_HEAD(dev_priv->engine[RCS]->mmio_base));
spin_unlock_irq(&dev_priv->uncore.lock);
intel_runtime_pm_put(dev_priv);
}
struct sgt_iter sgt_iter;
struct page *page;
unsigned long last_pfn = 0; /* suppress gcc warning */
- unsigned int max_segment;
+ unsigned int max_segment = i915_sg_segment_size();
gfp_t noreclaim;
int ret;
GEM_BUG_ON(obj->base.read_domains & I915_GEM_GPU_DOMAINS);
GEM_BUG_ON(obj->base.write_domain & I915_GEM_GPU_DOMAINS);
- max_segment = swiotlb_max_segment();
- if (!max_segment)
- max_segment = rounddown(UINT_MAX, PAGE_SIZE);
-
st = kmalloc(sizeof(*st), GFP_KERNEL);
if (st == NULL)
return ERR_PTR(-ENOMEM);
* Turning off the engine->irq_tasklet until the reset is over
* prevents the race.
*/
- tasklet_kill(&engine->irq_tasklet);
- tasklet_disable(&engine->irq_tasklet);
+ tasklet_kill(&engine->execlists.irq_tasklet);
+ tasklet_disable(&engine->execlists.irq_tasklet);
if (engine->irq_seqno_barrier)
engine->irq_seqno_barrier(engine);
void i915_gem_reset_finish_engine(struct intel_engine_cs *engine)
{
- tasklet_enable(&engine->irq_tasklet);
+ tasklet_enable(&engine->execlists.irq_tasklet);
kthread_unpark(engine->breadcrumbs.signaler);
}
static void engine_set_wedged(struct intel_engine_cs *engine)
{
- struct drm_i915_gem_request *request;
- unsigned long flags;
-
/* We need to be sure that no thread is running the old callback as
* we install the nop handler (otherwise we would submit a request
* to hardware that will never complete). In order to prevent this
engine->submit_request = nop_submit_request;
/* Mark all executing requests as skipped */
- spin_lock_irqsave(&engine->timeline->lock, flags);
- list_for_each_entry(request, &engine->timeline->requests, link)
- if (!i915_gem_request_completed(request))
- dma_fence_set_error(&request->fence, -EIO);
- spin_unlock_irqrestore(&engine->timeline->lock, flags);
-
- /*
- * Clear the execlists queue up before freeing the requests, as those
- * are the ones that keep the context and ringbuffer backing objects
- * pinned in place.
- */
-
- if (i915.enable_execlists) {
- struct execlist_port *port = engine->execlist_port;
- unsigned long flags;
- unsigned int n;
-
- spin_lock_irqsave(&engine->timeline->lock, flags);
-
- for (n = 0; n < ARRAY_SIZE(engine->execlist_port); n++)
- i915_gem_request_put(port_request(&port[n]));
- memset(engine->execlist_port, 0, sizeof(engine->execlist_port));
- engine->execlist_queue = RB_ROOT;
- engine->execlist_first = NULL;
-
- spin_unlock_irqrestore(&engine->timeline->lock, flags);
-
- /* The port is checked prior to scheduling a tasklet, but
- * just in case we have suspended the tasklet to do the
- * wedging make sure that when it wakes, it decides there
- * is no work to do by clearing the irq_posted bit.
- */
- clear_bit(ENGINE_IRQ_EXECLIST, &engine->irq_posted);
- }
+ engine->cancel_requests(engine);
/* Mark all pending requests as complete so that any concurrent
* (lockless) lookup doesn't try and wait upon the request as we
return false;
/* TODO: make semaphores and Execlists play nicely together */
- if (i915.enable_execlists)
+ if (i915_modparams.enable_execlists)
return false;
if (value >= 0)
dev_priv->mm.unordered_timeline = dma_fence_context_alloc(1);
- if (!i915.enable_execlists) {
+ if (!i915_modparams.enable_execlists) {
dev_priv->gt.resume = intel_legacy_submission_resume;
dev_priv->gt.cleanup_engine = intel_engine_cleanup;
} else {
* present or not in use we still need a small bias as ring wraparound
* at offset 0 sometimes hangs. No idea why.
*/
- if (HAS_GUC(dev_priv) && i915.enable_guc_loading)
+ if (HAS_GUC(dev_priv) && i915_modparams.enable_guc_loading)
ctx->ggtt_offset_bias = GUC_WOPCM_TOP;
else
ctx->ggtt_offset_bias = I915_GTT_PAGE_SIZE;
i915_gem_context_set_closed(ctx); /* not user accessible */
i915_gem_context_clear_bannable(ctx);
i915_gem_context_set_force_single_submission(ctx);
- if (!i915.enable_guc_submission)
+ if (!i915_modparams.enable_guc_submission)
ctx->ring_size = 512 * PAGE_SIZE; /* Max ring buffer size */
GEM_BUG_ON(i915_gem_context_is_kernel(ctx));
if (intel_vgpu_active(dev_priv) &&
HAS_LOGICAL_RING_CONTEXTS(dev_priv)) {
- if (!i915.enable_execlists) {
+ if (!i915_modparams.enable_execlists) {
DRM_INFO("Only EXECLIST mode is supported in vgpu.\n");
return -EINVAL;
}
}
/* Force the GPU state to be restored on enabling */
- if (!i915.enable_execlists) {
+ if (!i915_modparams.enable_execlists) {
struct i915_gem_context *ctx;
list_for_each_entry(ctx, &dev_priv->contexts.list, link) {
enum intel_engine_id id;
const int num_rings =
/* Use an extended w/a on gen7 if signalling from other rings */
- (i915.semaphores && INTEL_GEN(dev_priv) == 7) ?
+ (i915_modparams.semaphores && INTEL_GEN(dev_priv) == 7) ?
INTEL_INFO(dev_priv)->num_rings - 1 :
0;
int len;
struct intel_engine_cs *engine = req->engine;
lockdep_assert_held(&req->i915->drm.struct_mutex);
- if (i915.enable_execlists)
+ if (i915_modparams.enable_execlists)
return 0;
if (!req->ctx->engine[engine->id].state) {
#define __EXEC_HAS_RELOC BIT(31)
#define __EXEC_VALIDATED BIT(30)
+#define __EXEC_INTERNAL_FLAGS (~0u << 30)
#define UPDATE PIN_OFFSET_FIXED
#define BATCH_OFFSET_BIAS (256*1024)
static int eb_lookup_vmas(struct i915_execbuffer *eb)
{
struct radix_tree_root *handles_vma = &eb->ctx->handles_vma;
- struct drm_i915_gem_object *uninitialized_var(obj);
+ struct drm_i915_gem_object *obj;
unsigned int i;
int err;
goto err_obj;
}
+ /* transfer ref to ctx */
vma->open_count++;
list_add(&lut->obj_link, &obj->lut_list);
list_add(&lut->ctx_link, &eb->ctx->handles_list);
lut->ctx = eb->ctx;
lut->handle = handle;
- /* transfer ref to ctx */
- obj = NULL;
-
add_vma:
err = eb_add_vma(eb, i, vma);
if (unlikely(err))
- goto err_obj;
+ goto err_vma;
GEM_BUG_ON(vma != eb->vma[i]);
GEM_BUG_ON(vma->exec_flags != &eb->flags[i]);
return eb_reserve(eb);
err_obj:
- if (obj)
- i915_gem_object_put(obj);
+ i915_gem_object_put(obj);
err_vma:
eb->vma[i] = NULL;
return err;
const unsigned int count = eb->buffer_count;
unsigned int i;
- if (unlikely(i915.prefault_disable))
+ if (unlikely(i915_modparams.prefault_disable))
return 0;
for (i = 0; i < count; i++) {
int out_fence_fd = -1;
int err;
+ BUILD_BUG_ON(__EXEC_INTERNAL_FLAGS & ~__I915_EXEC_ILLEGAL_FLAGS);
BUILD_BUG_ON(__EXEC_OBJECT_INTERNAL_FLAGS &
~__EXEC_OBJECT_UNKNOWN_FLAGS);
return 0;
}
- if (INTEL_GEN(dev_priv) >= 8 && i915.enable_execlists) {
+ if (INTEL_GEN(dev_priv) >= 8 && i915_modparams.enable_execlists) {
if (has_full_48bit_ppgtt)
return 3;
switch (level) {
case I915_CACHE_NONE:
- pte |= PPAT_UNCACHED_INDEX;
+ pte |= PPAT_UNCACHED;
break;
case I915_CACHE_WT:
- pte |= PPAT_DISPLAY_ELLC_INDEX;
+ pte |= PPAT_DISPLAY_ELLC;
break;
default:
- pte |= PPAT_CACHED_INDEX;
+ pte |= PPAT_CACHED;
break;
}
gen8_pde_t pde = _PAGE_PRESENT | _PAGE_RW;
pde |= addr;
if (level != I915_CACHE_NONE)
- pde |= PPAT_CACHED_PDE_INDEX;
+ pde |= PPAT_CACHED_PDE;
else
- pde |= PPAT_UNCACHED_INDEX;
+ pde |= PPAT_UNCACHED;
return pde;
}
unsigned int pde;
gen8_for_each_pde(pt, pd, start, length, pde) {
+ int count = gen8_pte_count(start, length);
+
if (pt == vm->scratch_pt) {
pt = alloc_pt(vm);
if (IS_ERR(pt))
goto unwind;
- gen8_initialize_pt(vm, pt);
+ if (count < GEN8_PTES)
+ gen8_initialize_pt(vm, pt);
gen8_ppgtt_set_pde(vm, pd, pt, pde);
pd->used_pdes++;
GEM_BUG_ON(pd->used_pdes > I915_PDES);
}
- pt->used_ptes += gen8_pte_count(start, length);
+ pt->used_ptes += count;
}
return 0;
/* In the case of execlists, PPGTT is enabled by the context descriptor
* and the PDPs are contained within the context itself. We don't
* need to do anything here. */
- if (i915.enable_execlists)
+ if (i915_modparams.enable_execlists)
return 0;
if (!USES_PPGTT(dev_priv))
return 0;
}
-static void cnl_setup_private_ppat(struct drm_i915_private *dev_priv)
+static struct intel_ppat_entry *
+__alloc_ppat_entry(struct intel_ppat *ppat, unsigned int index, u8 value)
+{
+ struct intel_ppat_entry *entry = &ppat->entries[index];
+
+ GEM_BUG_ON(index >= ppat->max_entries);
+ GEM_BUG_ON(test_bit(index, ppat->used));
+
+ entry->ppat = ppat;
+ entry->value = value;
+ kref_init(&entry->ref);
+ set_bit(index, ppat->used);
+ set_bit(index, ppat->dirty);
+
+ return entry;
+}
+
+static void __free_ppat_entry(struct intel_ppat_entry *entry)
+{
+ struct intel_ppat *ppat = entry->ppat;
+ unsigned int index = entry - ppat->entries;
+
+ GEM_BUG_ON(index >= ppat->max_entries);
+ GEM_BUG_ON(!test_bit(index, ppat->used));
+
+ entry->value = ppat->clear_value;
+ clear_bit(index, ppat->used);
+ set_bit(index, ppat->dirty);
+}
+
+/**
+ * intel_ppat_get - get a usable PPAT entry
+ * @i915: i915 device instance
+ * @value: the PPAT value required by the caller
+ *
+ * The function tries to search if there is an existing PPAT entry which
+ * matches with the required value. If perfectly matched, the existing PPAT
+ * entry will be used. If only partially matched, it will try to check if
+ * there is any available PPAT index. If yes, it will allocate a new PPAT
+ * index for the required entry and update the HW. If not, the partially
+ * matched entry will be used.
+ */
+const struct intel_ppat_entry *
+intel_ppat_get(struct drm_i915_private *i915, u8 value)
+{
+ struct intel_ppat *ppat = &i915->ppat;
+ struct intel_ppat_entry *entry;
+ unsigned int scanned, best_score;
+ int i;
+
+ GEM_BUG_ON(!ppat->max_entries);
+
+ scanned = best_score = 0;
+ for_each_set_bit(i, ppat->used, ppat->max_entries) {
+ unsigned int score;
+
+ score = ppat->match(ppat->entries[i].value, value);
+ if (score > best_score) {
+ entry = &ppat->entries[i];
+ if (score == INTEL_PPAT_PERFECT_MATCH) {
+ kref_get(&entry->ref);
+ return entry;
+ }
+ best_score = score;
+ }
+ scanned++;
+ }
+
+ if (scanned == ppat->max_entries) {
+ if (!best_score)
+ return ERR_PTR(-ENOSPC);
+
+ kref_get(&entry->ref);
+ return entry;
+ }
+
+ i = find_first_zero_bit(ppat->used, ppat->max_entries);
+ entry = __alloc_ppat_entry(ppat, i, value);
+ ppat->update_hw(i915);
+ return entry;
+}
+
+static void release_ppat(struct kref *kref)
+{
+ struct intel_ppat_entry *entry =
+ container_of(kref, struct intel_ppat_entry, ref);
+ struct drm_i915_private *i915 = entry->ppat->i915;
+
+ __free_ppat_entry(entry);
+ entry->ppat->update_hw(i915);
+}
+
+/**
+ * intel_ppat_put - put back the PPAT entry got from intel_ppat_get()
+ * @entry: an intel PPAT entry
+ *
+ * Put back the PPAT entry got from intel_ppat_get(). If the PPAT index of the
+ * entry is dynamically allocated, its reference count will be decreased. Once
+ * the reference count becomes into zero, the PPAT index becomes free again.
+ */
+void intel_ppat_put(const struct intel_ppat_entry *entry)
+{
+ struct intel_ppat *ppat = entry->ppat;
+ unsigned int index = entry - ppat->entries;
+
+ GEM_BUG_ON(!ppat->max_entries);
+
+ kref_put(&ppat->entries[index].ref, release_ppat);
+}
+
+static void cnl_private_pat_update_hw(struct drm_i915_private *dev_priv)
+{
+ struct intel_ppat *ppat = &dev_priv->ppat;
+ int i;
+
+ for_each_set_bit(i, ppat->dirty, ppat->max_entries) {
+ I915_WRITE(GEN10_PAT_INDEX(i), ppat->entries[i].value);
+ clear_bit(i, ppat->dirty);
+ }
+}
+
+static void bdw_private_pat_update_hw(struct drm_i915_private *dev_priv)
+{
+ struct intel_ppat *ppat = &dev_priv->ppat;
+ u64 pat = 0;
+ int i;
+
+ for (i = 0; i < ppat->max_entries; i++)
+ pat |= GEN8_PPAT(i, ppat->entries[i].value);
+
+ bitmap_clear(ppat->dirty, 0, ppat->max_entries);
+
+ I915_WRITE(GEN8_PRIVATE_PAT_LO, lower_32_bits(pat));
+ I915_WRITE(GEN8_PRIVATE_PAT_HI, upper_32_bits(pat));
+}
+
+static unsigned int bdw_private_pat_match(u8 src, u8 dst)
{
+ unsigned int score = 0;
+ enum {
+ AGE_MATCH = BIT(0),
+ TC_MATCH = BIT(1),
+ CA_MATCH = BIT(2),
+ };
+
+ /* Cache attribute has to be matched. */
+ if (GEN8_PPAT_GET_CA(src) != GEN8_PPAT_GET_CA(dst))
+ return 0;
+
+ score |= CA_MATCH;
+
+ if (GEN8_PPAT_GET_TC(src) == GEN8_PPAT_GET_TC(dst))
+ score |= TC_MATCH;
+
+ if (GEN8_PPAT_GET_AGE(src) == GEN8_PPAT_GET_AGE(dst))
+ score |= AGE_MATCH;
+
+ if (score == (AGE_MATCH | TC_MATCH | CA_MATCH))
+ return INTEL_PPAT_PERFECT_MATCH;
+
+ return score;
+}
+
+static unsigned int chv_private_pat_match(u8 src, u8 dst)
+{
+ return (CHV_PPAT_GET_SNOOP(src) == CHV_PPAT_GET_SNOOP(dst)) ?
+ INTEL_PPAT_PERFECT_MATCH : 0;
+}
+
+static void cnl_setup_private_ppat(struct intel_ppat *ppat)
+{
+ ppat->max_entries = 8;
+ ppat->update_hw = cnl_private_pat_update_hw;
+ ppat->match = bdw_private_pat_match;
+ ppat->clear_value = GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(3);
+
/* XXX: spec is unclear if this is still needed for CNL+ */
- if (!USES_PPGTT(dev_priv)) {
- I915_WRITE(GEN10_PAT_INDEX(0), GEN8_PPAT_UC);
+ if (!USES_PPGTT(ppat->i915)) {
+ __alloc_ppat_entry(ppat, 0, GEN8_PPAT_UC);
return;
}
- I915_WRITE(GEN10_PAT_INDEX(0), GEN8_PPAT_WB | GEN8_PPAT_LLC);
- I915_WRITE(GEN10_PAT_INDEX(1), GEN8_PPAT_WC | GEN8_PPAT_LLCELLC);
- I915_WRITE(GEN10_PAT_INDEX(2), GEN8_PPAT_WT | GEN8_PPAT_LLCELLC);
- I915_WRITE(GEN10_PAT_INDEX(3), GEN8_PPAT_UC);
- I915_WRITE(GEN10_PAT_INDEX(4), GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(0));
- I915_WRITE(GEN10_PAT_INDEX(5), GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(1));
- I915_WRITE(GEN10_PAT_INDEX(6), GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(2));
- I915_WRITE(GEN10_PAT_INDEX(7), GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(3));
+ __alloc_ppat_entry(ppat, 0, GEN8_PPAT_WB | GEN8_PPAT_LLC);
+ __alloc_ppat_entry(ppat, 1, GEN8_PPAT_WC | GEN8_PPAT_LLCELLC);
+ __alloc_ppat_entry(ppat, 2, GEN8_PPAT_WT | GEN8_PPAT_LLCELLC);
+ __alloc_ppat_entry(ppat, 3, GEN8_PPAT_UC);
+ __alloc_ppat_entry(ppat, 4, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(0));
+ __alloc_ppat_entry(ppat, 5, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(1));
+ __alloc_ppat_entry(ppat, 6, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(2));
+ __alloc_ppat_entry(ppat, 7, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(3));
}
/* The GGTT and PPGTT need a private PPAT setup in order to handle cacheability
* bits. When using advanced contexts each context stores its own PAT, but
* writing this data shouldn't be harmful even in those cases. */
-static void bdw_setup_private_ppat(struct drm_i915_private *dev_priv)
+static void bdw_setup_private_ppat(struct intel_ppat *ppat)
{
- u64 pat;
+ ppat->max_entries = 8;
+ ppat->update_hw = bdw_private_pat_update_hw;
+ ppat->match = bdw_private_pat_match;
+ ppat->clear_value = GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(3);
- pat = GEN8_PPAT(0, GEN8_PPAT_WB | GEN8_PPAT_LLC) | /* for normal objects, no eLLC */
- GEN8_PPAT(1, GEN8_PPAT_WC | GEN8_PPAT_LLCELLC) | /* for something pointing to ptes? */
- GEN8_PPAT(2, GEN8_PPAT_WT | GEN8_PPAT_LLCELLC) | /* for scanout with eLLC */
- GEN8_PPAT(3, GEN8_PPAT_UC) | /* Uncached objects, mostly for scanout */
- GEN8_PPAT(4, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(0)) |
- GEN8_PPAT(5, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(1)) |
- GEN8_PPAT(6, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(2)) |
- GEN8_PPAT(7, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(3));
-
- if (!USES_PPGTT(dev_priv))
+ if (!USES_PPGTT(ppat->i915)) {
/* Spec: "For GGTT, there is NO pat_sel[2:0] from the entry,
* so RTL will always use the value corresponding to
* pat_sel = 000".
* So we can still hold onto all our assumptions wrt cpu
* clflushing on LLC machines.
*/
- pat = GEN8_PPAT(0, GEN8_PPAT_UC);
+ __alloc_ppat_entry(ppat, 0, GEN8_PPAT_UC);
+ return;
+ }
- /* XXX: spec defines this as 2 distinct registers. It's unclear if a 64b
- * write would work. */
- I915_WRITE(GEN8_PRIVATE_PAT_LO, pat);
- I915_WRITE(GEN8_PRIVATE_PAT_HI, pat >> 32);
+ __alloc_ppat_entry(ppat, 0, GEN8_PPAT_WB | GEN8_PPAT_LLC); /* for normal objects, no eLLC */
+ __alloc_ppat_entry(ppat, 1, GEN8_PPAT_WC | GEN8_PPAT_LLCELLC); /* for something pointing to ptes? */
+ __alloc_ppat_entry(ppat, 2, GEN8_PPAT_WT | GEN8_PPAT_LLCELLC); /* for scanout with eLLC */
+ __alloc_ppat_entry(ppat, 3, GEN8_PPAT_UC); /* Uncached objects, mostly for scanout */
+ __alloc_ppat_entry(ppat, 4, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(0));
+ __alloc_ppat_entry(ppat, 5, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(1));
+ __alloc_ppat_entry(ppat, 6, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(2));
+ __alloc_ppat_entry(ppat, 7, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(3));
}
-static void chv_setup_private_ppat(struct drm_i915_private *dev_priv)
+static void chv_setup_private_ppat(struct intel_ppat *ppat)
{
- u64 pat;
+ ppat->max_entries = 8;
+ ppat->update_hw = bdw_private_pat_update_hw;
+ ppat->match = chv_private_pat_match;
+ ppat->clear_value = CHV_PPAT_SNOOP;
/*
* Map WB on BDW to snooped on CHV.
* Which means we must set the snoop bit in PAT entry 0
* in order to keep the global status page working.
*/
- pat = GEN8_PPAT(0, CHV_PPAT_SNOOP) |
- GEN8_PPAT(1, 0) |
- GEN8_PPAT(2, 0) |
- GEN8_PPAT(3, 0) |
- GEN8_PPAT(4, CHV_PPAT_SNOOP) |
- GEN8_PPAT(5, CHV_PPAT_SNOOP) |
- GEN8_PPAT(6, CHV_PPAT_SNOOP) |
- GEN8_PPAT(7, CHV_PPAT_SNOOP);
- I915_WRITE(GEN8_PRIVATE_PAT_LO, pat);
- I915_WRITE(GEN8_PRIVATE_PAT_HI, pat >> 32);
+ __alloc_ppat_entry(ppat, 0, CHV_PPAT_SNOOP);
+ __alloc_ppat_entry(ppat, 1, 0);
+ __alloc_ppat_entry(ppat, 2, 0);
+ __alloc_ppat_entry(ppat, 3, 0);
+ __alloc_ppat_entry(ppat, 4, CHV_PPAT_SNOOP);
+ __alloc_ppat_entry(ppat, 5, CHV_PPAT_SNOOP);
+ __alloc_ppat_entry(ppat, 6, CHV_PPAT_SNOOP);
+ __alloc_ppat_entry(ppat, 7, CHV_PPAT_SNOOP);
}
static void gen6_gmch_remove(struct i915_address_space *vm)
cleanup_scratch_page(vm);
}
+static void setup_private_pat(struct drm_i915_private *dev_priv)
+{
+ struct intel_ppat *ppat = &dev_priv->ppat;
+ int i;
+
+ ppat->i915 = dev_priv;
+
+ if (INTEL_GEN(dev_priv) >= 10)
+ cnl_setup_private_ppat(ppat);
+ else if (IS_CHERRYVIEW(dev_priv) || IS_GEN9_LP(dev_priv))
+ chv_setup_private_ppat(ppat);
+ else
+ bdw_setup_private_ppat(ppat);
+
+ GEM_BUG_ON(ppat->max_entries > INTEL_MAX_PPAT_ENTRIES);
+
+ for_each_clear_bit(i, ppat->used, ppat->max_entries) {
+ ppat->entries[i].value = ppat->clear_value;
+ ppat->entries[i].ppat = ppat;
+ set_bit(i, ppat->dirty);
+ }
+
+ ppat->update_hw(dev_priv);
+}
+
static int gen8_gmch_probe(struct i915_ggtt *ggtt)
{
struct drm_i915_private *dev_priv = ggtt->base.i915;
}
ggtt->base.total = (size / sizeof(gen8_pte_t)) << PAGE_SHIFT;
-
- if (INTEL_GEN(dev_priv) >= 10)
- cnl_setup_private_ppat(dev_priv);
- else if (IS_CHERRYVIEW(dev_priv) || IS_GEN9_LP(dev_priv))
- chv_setup_private_ppat(dev_priv);
- else
- bdw_setup_private_ppat(dev_priv);
-
ggtt->base.cleanup = gen6_gmch_remove;
ggtt->base.bind_vma = ggtt_bind_vma;
ggtt->base.unbind_vma = ggtt_unbind_vma;
ggtt->invalidate = gen6_ggtt_invalidate;
+ setup_private_pat(dev_priv);
+
return ggtt_probe_common(ggtt, size);
}
* currently don't have any bits spare to pass in this upper
* restriction!
*/
- if (HAS_GUC(dev_priv) && i915.enable_guc_loading) {
+ if (HAS_GUC(dev_priv) && i915_modparams.enable_guc_loading) {
ggtt->base.total = min_t(u64, ggtt->base.total, GUC_GGTT_TOP);
ggtt->mappable_end = min(ggtt->mappable_end, ggtt->base.total);
}
ggtt->base.closed = false;
if (INTEL_GEN(dev_priv) >= 8) {
- if (INTEL_GEN(dev_priv) >= 10)
- cnl_setup_private_ppat(dev_priv);
- else if (IS_CHERRYVIEW(dev_priv) || IS_GEN9_LP(dev_priv))
- chv_setup_private_ppat(dev_priv);
- else
- bdw_setup_private_ppat(dev_priv);
+ struct intel_ppat *ppat = &dev_priv->ppat;
+ bitmap_set(ppat->dirty, 0, ppat->max_entries);
+ dev_priv->ppat.update_hw(dev_priv);
return;
}
* tables */
#define GEN8_PDPE_MASK 0x1ff
-#define PPAT_UNCACHED_INDEX (_PAGE_PWT | _PAGE_PCD)
-#define PPAT_CACHED_PDE_INDEX 0 /* WB LLC */
-#define PPAT_CACHED_INDEX _PAGE_PAT /* WB LLCeLLC */
-#define PPAT_DISPLAY_ELLC_INDEX _PAGE_PCD /* WT eLLC */
+#define PPAT_UNCACHED (_PAGE_PWT | _PAGE_PCD)
+#define PPAT_CACHED_PDE 0 /* WB LLC */
+#define PPAT_CACHED _PAGE_PAT /* WB LLCeLLC */
+#define PPAT_DISPLAY_ELLC _PAGE_PCD /* WT eLLC */
#define CHV_PPAT_SNOOP (1<<6)
-#define GEN8_PPAT_AGE(x) (x<<4)
+#define GEN8_PPAT_AGE(x) ((x)<<4)
#define GEN8_PPAT_LLCeLLC (3<<2)
#define GEN8_PPAT_LLCELLC (2<<2)
#define GEN8_PPAT_LLC (1<<2)
#define GEN8_PPAT_ELLC_OVERRIDE (0<<2)
#define GEN8_PPAT(i, x) ((u64)(x) << ((i) * 8))
+#define GEN8_PPAT_GET_CA(x) ((x) & 3)
+#define GEN8_PPAT_GET_TC(x) ((x) & (3 << 2))
+#define GEN8_PPAT_GET_AGE(x) ((x) & (3 << 4))
+#define CHV_PPAT_GET_SNOOP(x) ((x) & (1 << 6))
+
struct sg_table;
struct intel_rotation_info {
return container_of(vm, struct i915_ggtt, base);
}
+#define INTEL_MAX_PPAT_ENTRIES 8
+#define INTEL_PPAT_PERFECT_MATCH (~0U)
+
+struct intel_ppat;
+
+struct intel_ppat_entry {
+ struct intel_ppat *ppat;
+ struct kref ref;
+ u8 value;
+};
+
+struct intel_ppat {
+ struct intel_ppat_entry entries[INTEL_MAX_PPAT_ENTRIES];
+ DECLARE_BITMAP(used, INTEL_MAX_PPAT_ENTRIES);
+ DECLARE_BITMAP(dirty, INTEL_MAX_PPAT_ENTRIES);
+ unsigned int max_entries;
+ u8 clear_value;
+ /*
+ * Return a score to show how two PPAT values match,
+ * a INTEL_PPAT_PERFECT_MATCH indicates a perfect match
+ */
+ unsigned int (*match)(u8 src, u8 dst);
+ void (*update_hw)(struct drm_i915_private *i915);
+
+ struct drm_i915_private *i915;
+};
+
+const struct intel_ppat_entry *
+intel_ppat_get(struct drm_i915_private *i915, u8 value);
+void intel_ppat_put(const struct intel_ppat_entry *entry);
+
int i915_gem_init_aliasing_ppgtt(struct drm_i915_private *i915);
void i915_gem_fini_aliasing_ppgtt(struct drm_i915_private *i915);
return this_cpu != cpu;
}
-bool __i915_spin_request(const struct drm_i915_gem_request *req,
- u32 seqno, int state, unsigned long timeout_us)
+static bool __i915_spin_request(const struct drm_i915_gem_request *req,
+ u32 seqno, int state, unsigned long timeout_us)
{
struct intel_engine_cs *engine = req->engine;
unsigned int irq, cpu;
+ GEM_BUG_ON(!seqno);
+
+ /*
+ * Only wait for the request if we know it is likely to complete.
+ *
+ * We don't track the timestamps around requests, nor the average
+ * request length, so we do not have a good indicator that this
+ * request will complete within the timeout. What we do know is the
+ * order in which requests are executed by the engine and so we can
+ * tell if the request has started. If the request hasn't started yet,
+ * it is a fair assumption that it will not complete within our
+ * relatively short timeout.
+ */
+ if (!i915_seqno_passed(intel_engine_get_seqno(engine), seqno - 1))
+ return false;
+
/* When waiting for high frequency requests, e.g. during synchronous
* rendering split between the CPU and GPU, the finite amount of time
* required to set up the irq and wait upon it limits the response
irq = atomic_read(&engine->irq_count);
timeout_us += local_clock_us(&cpu);
do {
- if (seqno != i915_gem_request_global_seqno(req))
- break;
-
- if (i915_seqno_passed(intel_engine_get_seqno(req->engine),
- seqno))
- return true;
+ if (i915_seqno_passed(intel_engine_get_seqno(engine), seqno))
+ return seqno == i915_gem_request_global_seqno(req);
/* Seqno are meant to be ordered *before* the interrupt. If
* we see an interrupt without a corresponding seqno advance,
GEM_BUG_ON(!i915_sw_fence_signaled(&req->submit));
/* Optimistic short spin before touching IRQs */
- if (i915_spin_request(req, state, 5))
+ if (__i915_spin_request(req, wait.seqno, state, 5))
goto complete;
set_current_state(state);
continue;
/* Only spin if we know the GPU is processing this request */
- if (i915_spin_request(req, state, 2))
+ if (__i915_spin_request(req, wait.seqno, state, 2))
break;
if (!intel_wait_check_request(&wait, req)) {
return (s32)(seq1 - seq2) >= 0;
}
-static inline bool
-__i915_gem_request_started(const struct drm_i915_gem_request *req, u32 seqno)
-{
- GEM_BUG_ON(!seqno);
- return i915_seqno_passed(intel_engine_get_seqno(req->engine),
- seqno - 1);
-}
-
-static inline bool
-i915_gem_request_started(const struct drm_i915_gem_request *req)
-{
- u32 seqno;
-
- seqno = i915_gem_request_global_seqno(req);
- if (!seqno)
- return false;
-
- return __i915_gem_request_started(req, seqno);
-}
-
static inline bool
__i915_gem_request_completed(const struct drm_i915_gem_request *req, u32 seqno)
{
return __i915_gem_request_completed(req, seqno);
}
-bool __i915_spin_request(const struct drm_i915_gem_request *request,
- u32 seqno, int state, unsigned long timeout_us);
-static inline bool i915_spin_request(const struct drm_i915_gem_request *request,
- int state, unsigned long timeout_us)
-{
- u32 seqno;
-
- seqno = i915_gem_request_global_seqno(request);
- if (!seqno)
- return 0;
-
- return (__i915_gem_request_started(request, seqno) &&
- __i915_spin_request(request, seqno, state, timeout_us));
-}
-
/* We treat requests as fences. This is not be to confused with our
* "fence registers" but pipeline synchronisation objects ala GL_ARB_sync.
* We use the fences to synchronize access from the CPU with activity on the
struct task_struct *task;
};
-#if IS_ENABLED(CONFIG_SWIOTLB)
-#define swiotlb_active() swiotlb_nr_tbl()
-#else
-#define swiotlb_active() 0
-#endif
-
-static int
-st_set_pages(struct sg_table **st, struct page **pvec, int num_pages)
-{
- struct scatterlist *sg;
- int ret, n;
-
- *st = kmalloc(sizeof(**st), GFP_KERNEL);
- if (*st == NULL)
- return -ENOMEM;
-
- if (swiotlb_active()) {
- ret = sg_alloc_table(*st, num_pages, GFP_KERNEL);
- if (ret)
- goto err;
-
- for_each_sg((*st)->sgl, sg, num_pages, n)
- sg_set_page(sg, pvec[n], PAGE_SIZE, 0);
- } else {
- ret = sg_alloc_table_from_pages(*st, pvec, num_pages,
- 0, num_pages << PAGE_SHIFT,
- GFP_KERNEL);
- if (ret)
- goto err;
- }
-
- return 0;
-
-err:
- kfree(*st);
- *st = NULL;
- return ret;
-}
-
static struct sg_table *
-__i915_gem_userptr_set_pages(struct drm_i915_gem_object *obj,
- struct page **pvec, int num_pages)
+__i915_gem_userptr_alloc_pages(struct drm_i915_gem_object *obj,
+ struct page **pvec, int num_pages)
{
- struct sg_table *pages;
+ unsigned int max_segment = i915_sg_segment_size();
+ struct sg_table *st;
int ret;
- ret = st_set_pages(&pages, pvec, num_pages);
- if (ret)
+ st = kmalloc(sizeof(*st), GFP_KERNEL);
+ if (!st)
+ return ERR_PTR(-ENOMEM);
+
+alloc_table:
+ ret = __sg_alloc_table_from_pages(st, pvec, num_pages,
+ 0, num_pages << PAGE_SHIFT,
+ max_segment,
+ GFP_KERNEL);
+ if (ret) {
+ kfree(st);
return ERR_PTR(ret);
+ }
- ret = i915_gem_gtt_prepare_pages(obj, pages);
+ ret = i915_gem_gtt_prepare_pages(obj, st);
if (ret) {
- sg_free_table(pages);
- kfree(pages);
+ sg_free_table(st);
+
+ if (max_segment > PAGE_SIZE) {
+ max_segment = PAGE_SIZE;
+ goto alloc_table;
+ }
+
+ kfree(st);
return ERR_PTR(ret);
}
- return pages;
+ return st;
}
static int
struct sg_table *pages = ERR_PTR(ret);
if (pinned == npages) {
- pages = __i915_gem_userptr_set_pages(obj, pvec, npages);
+ pages = __i915_gem_userptr_alloc_pages(obj, pvec,
+ npages);
if (!IS_ERR(pages)) {
__i915_gem_object_set_pages(obj, pages);
pinned = 0;
pages = __i915_gem_userptr_get_pages_schedule(obj);
active = pages == ERR_PTR(-EAGAIN);
} else {
- pages = __i915_gem_userptr_set_pages(obj, pvec, num_pages);
+ pages = __i915_gem_userptr_alloc_pages(obj, pvec, num_pages);
active = !IS_ERR(pages);
}
if (active)
hash_init(dev_priv->mm_structs);
dev_priv->mm.userptr_wq =
- alloc_workqueue("i915-userptr-acquire", WQ_HIGHPRI, 0);
+ alloc_workqueue("i915-userptr-acquire",
+ WQ_HIGHPRI | WQ_MEM_RECLAIM,
+ 0);
if (!dev_priv->mm.userptr_wq)
return -ENOMEM;
static void error_print_engine(struct drm_i915_error_state_buf *m,
const struct drm_i915_error_engine *ee)
{
+ int n;
+
err_printf(m, "%s command stream:\n", engine_str(ee->engine_id));
err_printf(m, " START: 0x%08x\n", ee->start);
err_printf(m, " HEAD: 0x%08x [0x%08x]\n", ee->head, ee->rq_head);
jiffies_to_msecs(jiffies - ee->hangcheck_timestamp));
err_printf(m, " engine reset count: %u\n", ee->reset_count);
- error_print_request(m, " ELSP[0]: ", &ee->execlist[0]);
- error_print_request(m, " ELSP[1]: ", &ee->execlist[1]);
+ for (n = 0; n < ee->num_ports; n++) {
+ err_printf(m, " ELSP[%d]:", n);
+ error_print_request(m, " ", &ee->execlist[n]);
+ }
+
error_print_context(m, " Active context: ", &ee->context);
}
static void err_print_params(struct drm_i915_error_state_buf *m,
const struct i915_params *p)
{
-#define PRINT(T, x) err_print_param(m, #x, #T, &p->x);
+#define PRINT(T, x, ...) err_print_param(m, #x, #T, &p->x);
I915_PARAMS_FOR_EACH(PRINT);
#undef PRINT
}
kfree(error->overlay);
kfree(error->display);
-#define FREE(T, x) free_param(#T, &error->params.x);
+#define FREE(T, x, ...) free_param(#T, &error->params.x);
I915_PARAMS_FOR_EACH(FREE);
#undef FREE
static void error_record_engine_execlists(struct intel_engine_cs *engine,
struct drm_i915_error_engine *ee)
{
- const struct execlist_port *port = engine->execlist_port;
+ const struct intel_engine_execlists * const execlists = &engine->execlists;
unsigned int n;
- for (n = 0; n < ARRAY_SIZE(engine->execlist_port); n++) {
- struct drm_i915_gem_request *rq = port_request(&port[n]);
+ for (n = 0; n < execlists_num_ports(execlists); n++) {
+ struct drm_i915_gem_request *rq = port_request(&execlists->port[n]);
if (!rq)
break;
record_request(rq, &ee->execlist[n]);
}
+
+ ee->num_ports = n;
}
static void record_context(struct drm_i915_error_context *e,
struct i915_gpu_state *error)
{
/* Capturing log buf contents won't be useful if logging was disabled */
- if (!dev_priv->guc.log.vma || (i915.guc_log_level < 0))
+ if (!dev_priv->guc.log.vma || (i915_modparams.guc_log_level < 0))
return;
error->guc_log = i915_error_object_create(dev_priv,
ktime_to_timeval(ktime_sub(ktime_get(),
error->i915->gt.last_init_time));
- error->params = i915;
-#define DUP(T, x) dup_param(#T, &error->params.x);
+ error->params = i915_modparams;
+#define DUP(T, x, ...) dup_param(#T, &error->params.x);
I915_PARAMS_FOR_EACH(DUP);
#undef DUP
struct i915_gpu_state *error;
unsigned long flags;
- if (!i915.error_capture)
+ if (!i915_modparams.error_capture)
return;
if (READ_ONCE(dev_priv->gpu_error.first_error))
doorbell = __get_doorbell(client);
doorbell->db_status = GUC_DOORBELL_ENABLED;
- doorbell->cookie = client->doorbell_cookie;
+ doorbell->cookie = 0;
err = __guc_allocate_doorbell(client->guc, client->stage_id);
- if (err) {
+ if (err)
doorbell->db_status = GUC_DOORBELL_DISABLED;
- doorbell->cookie = 0;
- }
+
return err;
}
desc->db_base_addr = 0;
desc->stage_id = client->stage_id;
- desc->wq_size_bytes = client->wq_size;
+ desc->wq_size_bytes = GUC_WQ_SIZE;
desc->wq_status = WQ_STATUS_ACTIVE;
desc->priority = client->priority;
}
desc->db_trigger_cpu = (uintptr_t)__get_doorbell(client);
desc->db_trigger_uk = gfx_addr + client->doorbell_offset;
desc->process_desc = gfx_addr + client->proc_desc_offset;
- desc->wq_addr = gfx_addr + client->wq_offset;
- desc->wq_size = client->wq_size;
+ desc->wq_addr = gfx_addr + GUC_DB_SIZE;
+ desc->wq_size = GUC_WQ_SIZE;
desc->desc_private = (uintptr_t)client;
}
memset(desc, 0, sizeof(*desc));
}
-/**
- * i915_guc_wq_reserve() - reserve space in the GuC's workqueue
- * @request: request associated with the commands
- *
- * Return: 0 if space is available
- * -EAGAIN if space is not currently available
- *
- * This function must be called (and must return 0) before a request
- * is submitted to the GuC via i915_guc_submit() below. Once a result
- * of 0 has been returned, it must be balanced by a corresponding
- * call to submit().
- *
- * Reservation allows the caller to determine in advance that space
- * will be available for the next submission before committing resources
- * to it, and helps avoid late failures with complicated recovery paths.
- */
-int i915_guc_wq_reserve(struct drm_i915_gem_request *request)
-{
- const size_t wqi_size = sizeof(struct guc_wq_item);
- struct i915_guc_client *client = request->i915->guc.execbuf_client;
- struct guc_process_desc *desc = __get_process_desc(client);
- u32 freespace;
- int ret;
-
- spin_lock_irq(&client->wq_lock);
- freespace = CIRC_SPACE(client->wq_tail, desc->head, client->wq_size);
- freespace -= client->wq_rsvd;
- if (likely(freespace >= wqi_size)) {
- client->wq_rsvd += wqi_size;
- ret = 0;
- } else {
- client->no_wq_space++;
- ret = -EAGAIN;
- }
- spin_unlock_irq(&client->wq_lock);
-
- return ret;
-}
-
-static void guc_client_update_wq_rsvd(struct i915_guc_client *client, int size)
-{
- unsigned long flags;
-
- spin_lock_irqsave(&client->wq_lock, flags);
- client->wq_rsvd += size;
- spin_unlock_irqrestore(&client->wq_lock, flags);
-}
-
-void i915_guc_wq_unreserve(struct drm_i915_gem_request *request)
-{
- const int wqi_size = sizeof(struct guc_wq_item);
- struct i915_guc_client *client = request->i915->guc.execbuf_client;
-
- GEM_BUG_ON(READ_ONCE(client->wq_rsvd) < wqi_size);
- guc_client_update_wq_rsvd(client, -wqi_size);
-}
-
/* Construct a Work Item and append it to the GuC's Work Queue */
static void guc_wq_item_append(struct i915_guc_client *client,
struct drm_i915_gem_request *rq)
{
/* wqi_len is in DWords, and does not include the one-word header */
const size_t wqi_size = sizeof(struct guc_wq_item);
- const u32 wqi_len = wqi_size/sizeof(u32) - 1;
+ const u32 wqi_len = wqi_size / sizeof(u32) - 1;
struct intel_engine_cs *engine = rq->engine;
+ struct i915_gem_context *ctx = rq->ctx;
struct guc_process_desc *desc = __get_process_desc(client);
struct guc_wq_item *wqi;
- u32 freespace, tail, wq_off;
+ u32 ring_tail, wq_off;
- /* Free space is guaranteed, see i915_guc_wq_reserve() above */
- freespace = CIRC_SPACE(client->wq_tail, desc->head, client->wq_size);
- GEM_BUG_ON(freespace < wqi_size);
+ lockdep_assert_held(&client->wq_lock);
- /* The GuC firmware wants the tail index in QWords, not bytes */
- tail = intel_ring_set_tail(rq->ring, rq->tail) >> 3;
- GEM_BUG_ON(tail > WQ_RING_TAIL_MAX);
+ ring_tail = intel_ring_set_tail(rq->ring, rq->tail) / sizeof(u64);
+ GEM_BUG_ON(ring_tail > WQ_RING_TAIL_MAX);
/* For now workqueue item is 4 DWs; workqueue buffer is 2 pages. So we
* should not have the case where structure wqi is across page, neither
* workqueue buffer dw by dw.
*/
BUILD_BUG_ON(wqi_size != 16);
- GEM_BUG_ON(client->wq_rsvd < wqi_size);
- /* postincrement WQ tail for next time */
- wq_off = client->wq_tail;
+ /* Free space is guaranteed. */
+ wq_off = READ_ONCE(desc->tail);
+ GEM_BUG_ON(CIRC_SPACE(wq_off, READ_ONCE(desc->head),
+ GUC_WQ_SIZE) < wqi_size);
GEM_BUG_ON(wq_off & (wqi_size - 1));
- client->wq_tail += wqi_size;
- client->wq_tail &= client->wq_size - 1;
- client->wq_rsvd -= wqi_size;
/* WQ starts from the page after doorbell / process_desc */
wqi = client->vaddr + wq_off + GUC_DB_SIZE;
/* Now fill in the 4-word work queue item */
wqi->header = WQ_TYPE_INORDER |
- (wqi_len << WQ_LEN_SHIFT) |
- (engine->guc_id << WQ_TARGET_SHIFT) |
- WQ_NO_WCFLUSH_WAIT;
+ (wqi_len << WQ_LEN_SHIFT) |
+ (engine->guc_id << WQ_TARGET_SHIFT) |
+ WQ_NO_WCFLUSH_WAIT;
- /* The GuC wants only the low-order word of the context descriptor */
- wqi->context_desc = (u32)intel_lr_context_descriptor(rq->ctx, engine);
+ wqi->context_desc = lower_32_bits(intel_lr_context_descriptor(ctx, engine));
- wqi->submit_element_info = tail << WQ_RING_TAIL_SHIFT;
+ wqi->submit_element_info = ring_tail << WQ_RING_TAIL_SHIFT;
wqi->fence_id = rq->global_seqno;
+
+ /* Postincrement WQ tail for next time. */
+ WRITE_ONCE(desc->tail, (wq_off + wqi_size) & (GUC_WQ_SIZE - 1));
}
static void guc_reset_wq(struct i915_guc_client *client)
desc->head = 0;
desc->tail = 0;
-
- client->wq_tail = 0;
}
-static int guc_ring_doorbell(struct i915_guc_client *client)
+static void guc_ring_doorbell(struct i915_guc_client *client)
{
- struct guc_process_desc *desc = __get_process_desc(client);
- union guc_doorbell_qw db_cmp, db_exc, db_ret;
- union guc_doorbell_qw *db;
- int attempt = 2, ret = -EAGAIN;
-
- /* Update the tail so it is visible to GuC */
- desc->tail = client->wq_tail;
-
- /* current cookie */
- db_cmp.db_status = GUC_DOORBELL_ENABLED;
- db_cmp.cookie = client->doorbell_cookie;
+ struct guc_doorbell_info *db;
+ u32 cookie;
- /* cookie to be updated */
- db_exc.db_status = GUC_DOORBELL_ENABLED;
- db_exc.cookie = client->doorbell_cookie + 1;
- if (db_exc.cookie == 0)
- db_exc.cookie = 1;
+ lockdep_assert_held(&client->wq_lock);
/* pointer of current doorbell cacheline */
- db = (union guc_doorbell_qw *)__get_doorbell(client);
-
- while (attempt--) {
- /* lets ring the doorbell */
- db_ret.value_qw = atomic64_cmpxchg((atomic64_t *)db,
- db_cmp.value_qw, db_exc.value_qw);
-
- /* if the exchange was successfully executed */
- if (db_ret.value_qw == db_cmp.value_qw) {
- /* db was successfully rung */
- client->doorbell_cookie = db_exc.cookie;
- ret = 0;
- break;
- }
-
- /* XXX: doorbell was lost and need to acquire it again */
- if (db_ret.db_status == GUC_DOORBELL_DISABLED)
- break;
+ db = __get_doorbell(client);
- DRM_WARN("Cookie mismatch. Expected %d, found %d\n",
- db_cmp.cookie, db_ret.cookie);
-
- /* update the cookie to newly read cookie from GuC */
- db_cmp.cookie = db_ret.cookie;
- db_exc.cookie = db_ret.cookie + 1;
- if (db_exc.cookie == 0)
- db_exc.cookie = 1;
- }
+ /* we're not expecting the doorbell cookie to change behind our back */
+ cookie = READ_ONCE(db->cookie);
+ WARN_ON_ONCE(xchg(&db->cookie, cookie + 1) != cookie);
- return ret;
+ /* XXX: doorbell was lost and need to acquire it again */
+ GEM_BUG_ON(db->db_status != GUC_DOORBELL_ENABLED);
}
/**
- * __i915_guc_submit() - Submit commands through GuC
- * @rq: request associated with the commands
- *
- * The caller must have already called i915_guc_wq_reserve() above with
- * a result of 0 (success), guaranteeing that there is space in the work
- * queue for the new request, so enqueuing the item cannot fail.
- *
- * Bad Things Will Happen if the caller violates this protocol e.g. calls
- * submit() when _reserve() says there's no space, or calls _submit()
- * a different number of times from (successful) calls to _reserve().
+ * i915_guc_submit() - Submit commands through GuC
+ * @engine: engine associated with the commands
*
* The only error here arises if the doorbell hardware isn't functioning
* as expected, which really shouln't happen.
*/
-static void __i915_guc_submit(struct drm_i915_gem_request *rq)
+static void i915_guc_submit(struct intel_engine_cs *engine)
{
- struct drm_i915_private *dev_priv = rq->i915;
- struct intel_engine_cs *engine = rq->engine;
- unsigned int engine_id = engine->id;
- struct intel_guc *guc = &rq->i915->guc;
+ struct drm_i915_private *dev_priv = engine->i915;
+ struct intel_guc *guc = &dev_priv->guc;
struct i915_guc_client *client = guc->execbuf_client;
- unsigned long flags;
- int b_ret;
+ struct intel_engine_execlists * const execlists = &engine->execlists;
+ struct execlist_port *port = execlists->port;
+ const unsigned int engine_id = engine->id;
+ unsigned int n;
- /* WA to flush out the pending GMADR writes to ring buffer. */
- if (i915_vma_is_map_and_fenceable(rq->ring->vma))
- POSTING_READ_FW(GUC_STATUS);
+ for (n = 0; n < ARRAY_SIZE(execlists->port); n++) {
+ struct drm_i915_gem_request *rq;
+ unsigned int count;
- spin_lock_irqsave(&client->wq_lock, flags);
+ rq = port_unpack(&port[n], &count);
+ if (rq && count == 0) {
+ port_set(&port[n], port_pack(rq, ++count));
- guc_wq_item_append(client, rq);
- b_ret = guc_ring_doorbell(client);
+ if (i915_vma_is_map_and_fenceable(rq->ring->vma))
+ POSTING_READ_FW(GUC_STATUS);
- client->submissions[engine_id] += 1;
+ spin_lock(&client->wq_lock);
- spin_unlock_irqrestore(&client->wq_lock, flags);
-}
+ guc_wq_item_append(client, rq);
+ guc_ring_doorbell(client);
-static void i915_guc_submit(struct drm_i915_gem_request *rq)
-{
- __i915_gem_request_submit(rq);
- __i915_guc_submit(rq);
+ client->submissions[engine_id] += 1;
+
+ spin_unlock(&client->wq_lock);
+ }
+ }
}
static void nested_enable_signaling(struct drm_i915_gem_request *rq)
if (port_isset(port))
i915_gem_request_put(port_request(port));
- port_set(port, i915_gem_request_get(rq));
+ port_set(port, port_pack(i915_gem_request_get(rq), port_count(port)));
nested_enable_signaling(rq);
}
-static bool i915_guc_dequeue(struct intel_engine_cs *engine)
+static void i915_guc_dequeue(struct intel_engine_cs *engine)
{
- struct execlist_port *port = engine->execlist_port;
- struct drm_i915_gem_request *last = port_request(port);
- struct rb_node *rb;
+ struct intel_engine_execlists * const execlists = &engine->execlists;
+ struct execlist_port *port = execlists->port;
+ struct drm_i915_gem_request *last = NULL;
+ const struct execlist_port * const last_port =
+ &execlists->port[execlists->port_mask];
bool submit = false;
+ struct rb_node *rb;
+
+ if (port_isset(port))
+ port++;
spin_lock_irq(&engine->timeline->lock);
- rb = engine->execlist_first;
- GEM_BUG_ON(rb_first(&engine->execlist_queue) != rb);
+ rb = execlists->first;
+ GEM_BUG_ON(rb_first(&execlists->queue) != rb);
while (rb) {
struct i915_priolist *p = rb_entry(rb, typeof(*p), node);
struct drm_i915_gem_request *rq, *rn;
list_for_each_entry_safe(rq, rn, &p->requests, priotree.link) {
if (last && rq->ctx != last->ctx) {
- if (port != engine->execlist_port) {
+ if (port == last_port) {
__list_del_many(&p->requests,
&rq->priotree.link);
goto done;
INIT_LIST_HEAD(&rq->priotree.link);
rq->priotree.priority = INT_MAX;
- i915_guc_submit(rq);
- trace_i915_gem_request_in(rq, port_index(port, engine));
+ __i915_gem_request_submit(rq);
+ trace_i915_gem_request_in(rq, port_index(port, execlists));
last = rq;
submit = true;
}
rb = rb_next(rb);
- rb_erase(&p->node, &engine->execlist_queue);
+ rb_erase(&p->node, &execlists->queue);
INIT_LIST_HEAD(&p->requests);
if (p->priority != I915_PRIORITY_NORMAL)
kmem_cache_free(engine->i915->priorities, p);
}
done:
- engine->execlist_first = rb;
- if (submit)
+ execlists->first = rb;
+ if (submit) {
port_assign(port, last);
+ i915_guc_submit(engine);
+ }
spin_unlock_irq(&engine->timeline->lock);
-
- return submit;
}
static void i915_guc_irq_handler(unsigned long data)
{
- struct intel_engine_cs *engine = (struct intel_engine_cs *)data;
- struct execlist_port *port = engine->execlist_port;
+ struct intel_engine_cs * const engine = (struct intel_engine_cs *)data;
+ struct intel_engine_execlists * const execlists = &engine->execlists;
+ struct execlist_port *port = execlists->port;
+ const struct execlist_port * const last_port =
+ &execlists->port[execlists->port_mask];
struct drm_i915_gem_request *rq;
- bool submit;
- do {
- rq = port_request(&port[0]);
- while (rq && i915_gem_request_completed(rq)) {
- trace_i915_gem_request_out(rq);
- i915_gem_request_put(rq);
+ rq = port_request(&port[0]);
+ while (rq && i915_gem_request_completed(rq)) {
+ trace_i915_gem_request_out(rq);
+ i915_gem_request_put(rq);
- port[0] = port[1];
- memset(&port[1], 0, sizeof(port[1]));
+ execlists_port_complete(execlists, port);
- rq = port_request(&port[0]);
- }
+ rq = port_request(&port[0]);
+ }
- submit = false;
- if (!port_count(&port[1]))
- submit = i915_guc_dequeue(engine);
- } while (submit);
+ if (!port_isset(last_port))
+ i915_guc_dequeue(engine);
}
/*
client->engines = engines;
client->priority = priority;
client->doorbell_id = GUC_DOORBELL_INVALID;
- client->wq_offset = GUC_DB_SIZE;
- client->wq_size = GUC_WQ_SIZE;
spin_lock_init(&client->wq_lock);
ret = ida_simple_get(&guc->stage_ids, 0, GUC_MAX_STAGE_DESCRIPTORS,
kfree(client);
}
+static void guc_policy_init(struct guc_policy *policy)
+{
+ policy->execution_quantum = POLICY_DEFAULT_EXECUTION_QUANTUM_US;
+ policy->preemption_time = POLICY_DEFAULT_PREEMPTION_TIME_US;
+ policy->fault_time = POLICY_DEFAULT_FAULT_TIME_US;
+ policy->policy_flags = 0;
+}
+
static void guc_policies_init(struct guc_policies *policies)
{
struct guc_policy *policy;
u32 p, i;
- policies->dpc_promote_time = 500000;
+ policies->dpc_promote_time = POLICY_DEFAULT_DPC_PROMOTE_TIME_US;
policies->max_num_work_items = POLICY_MAX_NUM_WI;
for (p = 0; p < GUC_CLIENT_PRIORITY_NUM; p++) {
for (i = GUC_RENDER_ENGINE; i < GUC_MAX_ENGINES_NUM; i++) {
policy = &policies->policy[p][i];
- policy->execution_quantum = 1000000;
- policy->preemption_time = 500000;
- policy->fault_time = 250000;
- policy->policy_flags = 0;
+ guc_policy_init(policy);
}
}
policies->is_valid = 1;
}
+/*
+ * The first 80 dwords of the register state context, containing the
+ * execlists and ppgtt registers.
+ */
+#define LR_HW_CONTEXT_SIZE (80 * sizeof(u32))
+
static int guc_ads_create(struct intel_guc *guc)
{
struct drm_i915_private *dev_priv = guc_to_i915(guc);
} __packed *blob;
struct intel_engine_cs *engine;
enum intel_engine_id id;
+ const u32 skipped_offset = LRC_HEADER_PAGES * PAGE_SIZE;
+ const u32 skipped_size = LRC_PPHWSP_SZ * PAGE_SIZE + LR_HW_CONTEXT_SIZE;
u32 base;
GEM_BUG_ON(guc->ads_vma);
* engines after a reset. Here we use the Render ring default
* context, which must already exist and be pinned in the GGTT,
* so its address won't change after we've told the GuC where
- * to find it.
+ * to find it. Note that we have to skip our header (1 page),
+ * because our GuC shared data is there.
*/
blob->ads.golden_context_lrca =
- dev_priv->engine[RCS]->status_page.ggtt_offset;
+ guc_ggtt_offset(dev_priv->kernel_context->engine[RCS].state) + skipped_offset;
+ /*
+ * The GuC expects us to exclude the portion of the context image that
+ * it skips from the size it is to read. It starts reading from after
+ * the execlist context (so skipping the first page [PPHWSP] and 80
+ * dwords). Weird guc is weird.
+ */
for_each_engine(engine, dev_priv, id)
- blob->ads.eng_state_size[engine->guc_id] = engine->context_size;
+ blob->ads.eng_state_size[engine->guc_id] = engine->context_size - skipped_size;
base = guc_ggtt_offset(vma);
blob->ads.scheduler_policies = base + ptr_offset(blob, policies);
enum intel_engine_id id;
int err;
+ /*
+ * We're using GuC work items for submitting work through GuC. Since
+ * we're coalescing multiple requests from a single context into a
+ * single work item prior to assigning it to execlist_port, we can
+ * never have more work items than the total number of ports (for all
+ * engines). The GuC firmware is controlling the HEAD of work queue,
+ * and it is guaranteed that it will remove the work item from the
+ * queue before our request is completed.
+ */
+ BUILD_BUG_ON(ARRAY_SIZE(engine->execlists.port) *
+ sizeof(struct guc_wq_item) *
+ I915_NUM_ENGINES > GUC_WQ_SIZE);
+
if (!client) {
client = guc_client_alloc(dev_priv,
INTEL_INFO(dev_priv)->ring_mask,
guc_interrupts_capture(dev_priv);
for_each_engine(engine, dev_priv, id) {
- const int wqi_size = sizeof(struct guc_wq_item);
- struct drm_i915_gem_request *rq;
-
+ struct intel_engine_execlists * const execlists = &engine->execlists;
/* The tasklet was initialised by execlists, and may be in
* a state of flux (across a reset) and so we just want to
* take over the callback without changing any other state
* in the tasklet.
*/
- engine->irq_tasklet.func = i915_guc_irq_handler;
+ execlists->irq_tasklet.func = i915_guc_irq_handler;
clear_bit(ENGINE_IRQ_EXECLIST, &engine->irq_posted);
-
- /* Replay the current set of previously submitted requests */
- spin_lock_irq(&engine->timeline->lock);
- list_for_each_entry(rq, &engine->timeline->requests, link) {
- guc_client_update_wq_rsvd(client, wqi_size);
- __i915_guc_submit(rq);
- }
- spin_unlock_irq(&engine->timeline->lock);
+ tasklet_schedule(&execlists->irq_tasklet);
}
return 0;
/* any value greater than GUC_POWER_D0 */
data[1] = GUC_POWER_D1;
/* first page is shared data with GuC */
- data[2] = guc_ggtt_offset(ctx->engine[RCS].state);
+ data[2] = guc_ggtt_offset(ctx->engine[RCS].state) + LRC_GUCSHR_PN * PAGE_SIZE;
return intel_guc_send(guc, data, ARRAY_SIZE(data));
}
if (guc->fw.load_status != INTEL_UC_FIRMWARE_SUCCESS)
return 0;
- if (i915.guc_log_level >= 0)
+ if (i915_modparams.guc_log_level >= 0)
gen9_enable_guc_interrupts(dev_priv);
ctx = dev_priv->kernel_context;
data[0] = INTEL_GUC_ACTION_EXIT_S_STATE;
data[1] = GUC_POWER_D0;
/* first page is shared data with GuC */
- data[2] = guc_ggtt_offset(ctx->engine[RCS].state);
+ data[2] = guc_ggtt_offset(ctx->engine[RCS].state) + LRC_GUCSHR_PN * PAGE_SIZE;
return intel_guc_send(guc, data, ARRAY_SIZE(data));
}
POSTING_READ(GEN8_##type##_IIR(which)); \
} while (0)
-#define GEN5_IRQ_RESET(type) do { \
+#define GEN3_IRQ_RESET(type) do { \
I915_WRITE(type##IMR, 0xffffffff); \
POSTING_READ(type##IMR); \
I915_WRITE(type##IER, 0); \
POSTING_READ(type##IIR); \
} while (0)
+#define GEN2_IRQ_RESET(type) do { \
+ I915_WRITE16(type##IMR, 0xffff); \
+ POSTING_READ16(type##IMR); \
+ I915_WRITE16(type##IER, 0); \
+ I915_WRITE16(type##IIR, 0xffff); \
+ POSTING_READ16(type##IIR); \
+ I915_WRITE16(type##IIR, 0xffff); \
+ POSTING_READ16(type##IIR); \
+} while (0)
+
/*
* We should clear IMR at preinstall/uninstall, and just check at postinstall.
*/
-static void gen5_assert_iir_is_zero(struct drm_i915_private *dev_priv,
+static void gen3_assert_iir_is_zero(struct drm_i915_private *dev_priv,
i915_reg_t reg)
{
u32 val = I915_READ(reg);
POSTING_READ(reg);
}
+static void gen2_assert_iir_is_zero(struct drm_i915_private *dev_priv,
+ i915_reg_t reg)
+{
+ u16 val = I915_READ16(reg);
+
+ if (val == 0)
+ return;
+
+ WARN(1, "Interrupt register 0x%x is not zero: 0x%08x\n",
+ i915_mmio_reg_offset(reg), val);
+ I915_WRITE16(reg, 0xffff);
+ POSTING_READ16(reg);
+ I915_WRITE16(reg, 0xffff);
+ POSTING_READ16(reg);
+}
+
#define GEN8_IRQ_INIT_NDX(type, which, imr_val, ier_val) do { \
- gen5_assert_iir_is_zero(dev_priv, GEN8_##type##_IIR(which)); \
+ gen3_assert_iir_is_zero(dev_priv, GEN8_##type##_IIR(which)); \
I915_WRITE(GEN8_##type##_IER(which), (ier_val)); \
I915_WRITE(GEN8_##type##_IMR(which), (imr_val)); \
POSTING_READ(GEN8_##type##_IMR(which)); \
} while (0)
-#define GEN5_IRQ_INIT(type, imr_val, ier_val) do { \
- gen5_assert_iir_is_zero(dev_priv, type##IIR); \
+#define GEN3_IRQ_INIT(type, imr_val, ier_val) do { \
+ gen3_assert_iir_is_zero(dev_priv, type##IIR); \
I915_WRITE(type##IER, (ier_val)); \
I915_WRITE(type##IMR, (imr_val)); \
POSTING_READ(type##IMR); \
} while (0)
+#define GEN2_IRQ_INIT(type, imr_val, ier_val) do { \
+ gen2_assert_iir_is_zero(dev_priv, type##IIR); \
+ I915_WRITE16(type##IER, (ier_val)); \
+ I915_WRITE16(type##IMR, (imr_val)); \
+ POSTING_READ16(type##IMR); \
+} while (0)
+
static void gen6_rps_irq_handler(struct drm_i915_private *dev_priv, u32 pm_iir);
static void gen9_guc_irq_handler(struct drm_i915_private *dev_priv, u32 pm_iir);
POSTING_READ(SDEIMR);
}
-static void
-__i915_enable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe,
- u32 enable_mask, u32 status_mask)
+u32 i915_pipestat_enable_mask(struct drm_i915_private *dev_priv,
+ enum pipe pipe)
{
- i915_reg_t reg = PIPESTAT(pipe);
- u32 pipestat = I915_READ(reg) & PIPESTAT_INT_ENABLE_MASK;
-
- lockdep_assert_held(&dev_priv->irq_lock);
- WARN_ON(!intel_irqs_enabled(dev_priv));
-
- if (WARN_ONCE(enable_mask & ~PIPESTAT_INT_ENABLE_MASK ||
- status_mask & ~PIPESTAT_INT_STATUS_MASK,
- "pipe %c: enable_mask=0x%x, status_mask=0x%x\n",
- pipe_name(pipe), enable_mask, status_mask))
- return;
-
- if ((pipestat & enable_mask) == enable_mask)
- return;
-
- dev_priv->pipestat_irq_mask[pipe] |= status_mask;
-
- /* Enable the interrupt, clear any pending status */
- pipestat |= enable_mask | status_mask;
- I915_WRITE(reg, pipestat);
- POSTING_READ(reg);
-}
-
-static void
-__i915_disable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe,
- u32 enable_mask, u32 status_mask)
-{
- i915_reg_t reg = PIPESTAT(pipe);
- u32 pipestat = I915_READ(reg) & PIPESTAT_INT_ENABLE_MASK;
+ u32 status_mask = dev_priv->pipestat_irq_mask[pipe];
+ u32 enable_mask = status_mask << 16;
lockdep_assert_held(&dev_priv->irq_lock);
- WARN_ON(!intel_irqs_enabled(dev_priv));
-
- if (WARN_ONCE(enable_mask & ~PIPESTAT_INT_ENABLE_MASK ||
- status_mask & ~PIPESTAT_INT_STATUS_MASK,
- "pipe %c: enable_mask=0x%x, status_mask=0x%x\n",
- pipe_name(pipe), enable_mask, status_mask))
- return;
-
- if ((pipestat & enable_mask) == 0)
- return;
-
- dev_priv->pipestat_irq_mask[pipe] &= ~status_mask;
- pipestat &= ~enable_mask;
- I915_WRITE(reg, pipestat);
- POSTING_READ(reg);
-}
-
-static u32 vlv_get_pipestat_enable_mask(struct drm_device *dev, u32 status_mask)
-{
- u32 enable_mask = status_mask << 16;
+ if (INTEL_GEN(dev_priv) < 5)
+ goto out;
/*
* On pipe A we don't support the PSR interrupt yet,
if (status_mask & SPRITE1_FLIP_DONE_INT_STATUS_VLV)
enable_mask |= SPRITE1_FLIP_DONE_INT_EN_VLV;
+out:
+ WARN_ONCE(enable_mask & ~PIPESTAT_INT_ENABLE_MASK ||
+ status_mask & ~PIPESTAT_INT_STATUS_MASK,
+ "pipe %c: enable_mask=0x%x, status_mask=0x%x\n",
+ pipe_name(pipe), enable_mask, status_mask);
+
return enable_mask;
}
-void
-i915_enable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe,
- u32 status_mask)
+void i915_enable_pipestat(struct drm_i915_private *dev_priv,
+ enum pipe pipe, u32 status_mask)
{
+ i915_reg_t reg = PIPESTAT(pipe);
u32 enable_mask;
- if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
- enable_mask = vlv_get_pipestat_enable_mask(&dev_priv->drm,
- status_mask);
- else
- enable_mask = status_mask << 16;
- __i915_enable_pipestat(dev_priv, pipe, enable_mask, status_mask);
+ WARN_ONCE(status_mask & ~PIPESTAT_INT_STATUS_MASK,
+ "pipe %c: status_mask=0x%x\n",
+ pipe_name(pipe), status_mask);
+
+ lockdep_assert_held(&dev_priv->irq_lock);
+ WARN_ON(!intel_irqs_enabled(dev_priv));
+
+ if ((dev_priv->pipestat_irq_mask[pipe] & status_mask) == status_mask)
+ return;
+
+ dev_priv->pipestat_irq_mask[pipe] |= status_mask;
+ enable_mask = i915_pipestat_enable_mask(dev_priv, pipe);
+
+ I915_WRITE(reg, enable_mask | status_mask);
+ POSTING_READ(reg);
}
-void
-i915_disable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe,
- u32 status_mask)
+void i915_disable_pipestat(struct drm_i915_private *dev_priv,
+ enum pipe pipe, u32 status_mask)
{
+ i915_reg_t reg = PIPESTAT(pipe);
u32 enable_mask;
- if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
- enable_mask = vlv_get_pipestat_enable_mask(&dev_priv->drm,
- status_mask);
- else
- enable_mask = status_mask << 16;
- __i915_disable_pipestat(dev_priv, pipe, enable_mask, status_mask);
+ WARN_ONCE(status_mask & ~PIPESTAT_INT_STATUS_MASK,
+ "pipe %c: status_mask=0x%x\n",
+ pipe_name(pipe), status_mask);
+
+ lockdep_assert_held(&dev_priv->irq_lock);
+ WARN_ON(!intel_irqs_enabled(dev_priv));
+
+ if ((dev_priv->pipestat_irq_mask[pipe] & status_mask) == 0)
+ return;
+
+ dev_priv->pipestat_irq_mask[pipe] &= ~status_mask;
+ enable_mask = i915_pipestat_enable_mask(dev_priv, pipe);
+
+ I915_WRITE(reg, enable_mask | status_mask);
+ POSTING_READ(reg);
}
/**
return I915_READ(PIPE_FRMCOUNT_G4X(pipe));
}
+/*
+ * On certain encoders on certain platforms, pipe
+ * scanline register will not work to get the scanline,
+ * since the timings are driven from the PORT or issues
+ * with scanline register updates.
+ * This function will use Framestamp and current
+ * timestamp registers to calculate the scanline.
+ */
+static u32 __intel_get_crtc_scanline_from_timestamp(struct intel_crtc *crtc)
+{
+ struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
+ struct drm_vblank_crtc *vblank =
+ &crtc->base.dev->vblank[drm_crtc_index(&crtc->base)];
+ const struct drm_display_mode *mode = &vblank->hwmode;
+ u32 vblank_start = mode->crtc_vblank_start;
+ u32 vtotal = mode->crtc_vtotal;
+ u32 htotal = mode->crtc_htotal;
+ u32 clock = mode->crtc_clock;
+ u32 scanline, scan_prev_time, scan_curr_time, scan_post_time;
+
+ /*
+ * To avoid the race condition where we might cross into the
+ * next vblank just between the PIPE_FRMTMSTMP and TIMESTAMP_CTR
+ * reads. We make sure we read PIPE_FRMTMSTMP and TIMESTAMP_CTR
+ * during the same frame.
+ */
+ do {
+ /*
+ * This field provides read back of the display
+ * pipe frame time stamp. The time stamp value
+ * is sampled at every start of vertical blank.
+ */
+ scan_prev_time = I915_READ_FW(PIPE_FRMTMSTMP(crtc->pipe));
+
+ /*
+ * The TIMESTAMP_CTR register has the current
+ * time stamp value.
+ */
+ scan_curr_time = I915_READ_FW(IVB_TIMESTAMP_CTR);
+
+ scan_post_time = I915_READ_FW(PIPE_FRMTMSTMP(crtc->pipe));
+ } while (scan_post_time != scan_prev_time);
+
+ scanline = div_u64(mul_u32_u32(scan_curr_time - scan_prev_time,
+ clock), 1000 * htotal);
+ scanline = min(scanline, vtotal - 1);
+ scanline = (scanline + vblank_start) % vtotal;
+
+ return scanline;
+}
+
/* I915_READ_FW, only for fast reads of display block, no need for forcewake etc. */
static int __intel_get_crtc_scanline(struct intel_crtc *crtc)
{
vblank = &crtc->base.dev->vblank[drm_crtc_index(&crtc->base)];
mode = &vblank->hwmode;
+ if (mode->private_flags & I915_MODE_FLAG_GET_SCANLINE_FROM_TIMESTAMP)
+ return __intel_get_crtc_scanline_from_timestamp(crtc);
+
vtotal = mode->crtc_vtotal;
if (mode->flags & DRM_MODE_FLAG_INTERLACE)
vtotal /= 2;
spin_lock(&engine->breadcrumbs.irq_lock);
wait = engine->breadcrumbs.irq_wait;
if (wait) {
+ bool wakeup = engine->irq_seqno_barrier;
+
/* We use a callback from the dma-fence to submit
* requests after waiting on our own requests. To
* ensure minimum delay in queuing the next request to
* and many waiters.
*/
if (i915_seqno_passed(intel_engine_get_seqno(engine),
- wait->seqno) &&
- !test_bit(DMA_FENCE_FLAG_SIGNALED_BIT,
- &wait->request->fence.flags))
- rq = i915_gem_request_get(wait->request);
+ wait->seqno)) {
+ struct drm_i915_gem_request *waiter = wait->request;
+
+ wakeup = true;
+ if (!test_bit(DMA_FENCE_FLAG_SIGNALED_BIT,
+ &waiter->fence.flags) &&
+ intel_wait_check_request(wait, waiter))
+ rq = i915_gem_request_get(waiter);
+ }
- wake_up_process(wait->tsk);
+ if (wakeup)
+ wake_up_process(wait->tsk);
} else {
__intel_engine_disarm_breadcrumbs(engine);
}
static void
gen8_cs_irq_handler(struct intel_engine_cs *engine, u32 iir, int test_shift)
{
+ struct intel_engine_execlists * const execlists = &engine->execlists;
bool tasklet = false;
if (iir & (GT_CONTEXT_SWITCH_INTERRUPT << test_shift)) {
- if (port_count(&engine->execlist_port[0])) {
+ if (port_count(&execlists->port[0])) {
__set_bit(ENGINE_IRQ_EXECLIST, &engine->irq_posted);
tasklet = true;
}
if (iir & (GT_RENDER_USER_INTERRUPT << test_shift)) {
notify_ring(engine);
- tasklet |= i915.enable_guc_submission;
+ tasklet |= i915_modparams.enable_guc_submission;
}
if (tasklet)
- tasklet_hi_schedule(&engine->irq_tasklet);
+ tasklet_hi_schedule(&execlists->irq_tasklet);
}
static irqreturn_t gen8_gt_irq_ack(struct drm_i915_private *dev_priv,
}
}
-static void valleyview_pipestat_irq_ack(struct drm_i915_private *dev_priv,
- u32 iir, u32 pipe_stats[I915_MAX_PIPES])
+static void i9xx_pipestat_irq_reset(struct drm_i915_private *dev_priv)
+{
+ enum pipe pipe;
+
+ for_each_pipe(dev_priv, pipe) {
+ I915_WRITE(PIPESTAT(pipe),
+ PIPESTAT_INT_STATUS_MASK |
+ PIPE_FIFO_UNDERRUN_STATUS);
+
+ dev_priv->pipestat_irq_mask[pipe] = 0;
+ }
+}
+
+static void i9xx_pipestat_irq_ack(struct drm_i915_private *dev_priv,
+ u32 iir, u32 pipe_stats[I915_MAX_PIPES])
{
int pipe;
for_each_pipe(dev_priv, pipe) {
i915_reg_t reg;
- u32 mask, iir_bit = 0;
+ u32 status_mask, enable_mask, iir_bit = 0;
/*
* PIPESTAT bits get signalled even when the interrupt is
*/
/* fifo underruns are filterered in the underrun handler. */
- mask = PIPE_FIFO_UNDERRUN_STATUS;
+ status_mask = PIPE_FIFO_UNDERRUN_STATUS;
switch (pipe) {
case PIPE_A:
break;
}
if (iir & iir_bit)
- mask |= dev_priv->pipestat_irq_mask[pipe];
+ status_mask |= dev_priv->pipestat_irq_mask[pipe];
- if (!mask)
+ if (!status_mask)
continue;
reg = PIPESTAT(pipe);
- mask |= PIPESTAT_INT_ENABLE_MASK;
- pipe_stats[pipe] = I915_READ(reg) & mask;
+ pipe_stats[pipe] = I915_READ(reg) & status_mask;
+ enable_mask = i915_pipestat_enable_mask(dev_priv, pipe);
/*
* Clear the PIPE*STAT regs before the IIR
*/
- if (pipe_stats[pipe] & (PIPE_FIFO_UNDERRUN_STATUS |
- PIPESTAT_INT_STATUS_MASK))
- I915_WRITE(reg, pipe_stats[pipe]);
+ if (pipe_stats[pipe])
+ I915_WRITE(reg, enable_mask | pipe_stats[pipe]);
}
spin_unlock(&dev_priv->irq_lock);
}
+static void i8xx_pipestat_irq_handler(struct drm_i915_private *dev_priv,
+ u16 iir, u32 pipe_stats[I915_MAX_PIPES])
+{
+ enum pipe pipe;
+
+ for_each_pipe(dev_priv, pipe) {
+ if (pipe_stats[pipe] & PIPE_VBLANK_INTERRUPT_STATUS)
+ drm_handle_vblank(&dev_priv->drm, pipe);
+
+ if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
+ i9xx_pipe_crc_irq_handler(dev_priv, pipe);
+
+ if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
+ intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
+ }
+}
+
+static void i915_pipestat_irq_handler(struct drm_i915_private *dev_priv,
+ u32 iir, u32 pipe_stats[I915_MAX_PIPES])
+{
+ bool blc_event = false;
+ enum pipe pipe;
+
+ for_each_pipe(dev_priv, pipe) {
+ if (pipe_stats[pipe] & PIPE_VBLANK_INTERRUPT_STATUS)
+ drm_handle_vblank(&dev_priv->drm, pipe);
+
+ if (pipe_stats[pipe] & PIPE_LEGACY_BLC_EVENT_STATUS)
+ blc_event = true;
+
+ if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
+ i9xx_pipe_crc_irq_handler(dev_priv, pipe);
+
+ if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
+ intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
+ }
+
+ if (blc_event || (iir & I915_ASLE_INTERRUPT))
+ intel_opregion_asle_intr(dev_priv);
+}
+
+static void i965_pipestat_irq_handler(struct drm_i915_private *dev_priv,
+ u32 iir, u32 pipe_stats[I915_MAX_PIPES])
+{
+ bool blc_event = false;
+ enum pipe pipe;
+
+ for_each_pipe(dev_priv, pipe) {
+ if (pipe_stats[pipe] & PIPE_START_VBLANK_INTERRUPT_STATUS)
+ drm_handle_vblank(&dev_priv->drm, pipe);
+
+ if (pipe_stats[pipe] & PIPE_LEGACY_BLC_EVENT_STATUS)
+ blc_event = true;
+
+ if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
+ i9xx_pipe_crc_irq_handler(dev_priv, pipe);
+
+ if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
+ intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
+ }
+
+ if (blc_event || (iir & I915_ASLE_INTERRUPT))
+ intel_opregion_asle_intr(dev_priv);
+
+ if (pipe_stats[0] & PIPE_GMBUS_INTERRUPT_STATUS)
+ gmbus_irq_handler(dev_priv);
+}
+
static void valleyview_pipestat_irq_handler(struct drm_i915_private *dev_priv,
u32 pipe_stats[I915_MAX_PIPES])
{
/* Call regardless, as some status bits might not be
* signalled in iir */
- valleyview_pipestat_irq_ack(dev_priv, iir, pipe_stats);
+ i9xx_pipestat_irq_ack(dev_priv, iir, pipe_stats);
if (iir & (I915_LPE_PIPE_A_INTERRUPT |
I915_LPE_PIPE_B_INTERRUPT))
/* Call regardless, as some status bits might not be
* signalled in iir */
- valleyview_pipestat_irq_ack(dev_priv, iir, pipe_stats);
+ i9xx_pipestat_irq_ack(dev_priv, iir, pipe_stats);
if (iir & (I915_LPE_PIPE_A_INTERRUPT |
I915_LPE_PIPE_B_INTERRUPT |
if (HAS_PCH_NOP(dev_priv))
return;
- GEN5_IRQ_RESET(SDE);
+ GEN3_IRQ_RESET(SDE);
if (HAS_PCH_CPT(dev_priv) || HAS_PCH_LPT(dev_priv))
I915_WRITE(SERR_INT, 0xffffffff);
static void gen5_gt_irq_reset(struct drm_i915_private *dev_priv)
{
- GEN5_IRQ_RESET(GT);
+ GEN3_IRQ_RESET(GT);
if (INTEL_GEN(dev_priv) >= 6)
- GEN5_IRQ_RESET(GEN6_PM);
+ GEN3_IRQ_RESET(GEN6_PM);
}
static void vlv_display_irq_reset(struct drm_i915_private *dev_priv)
{
- enum pipe pipe;
-
if (IS_CHERRYVIEW(dev_priv))
I915_WRITE(DPINVGTT, DPINVGTT_STATUS_MASK_CHV);
else
i915_hotplug_interrupt_update_locked(dev_priv, 0xffffffff, 0);
I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
- for_each_pipe(dev_priv, pipe) {
- I915_WRITE(PIPESTAT(pipe),
- PIPE_FIFO_UNDERRUN_STATUS |
- PIPESTAT_INT_STATUS_MASK);
- dev_priv->pipestat_irq_mask[pipe] = 0;
- }
+ i9xx_pipestat_irq_reset(dev_priv);
- GEN5_IRQ_RESET(VLV_);
+ GEN3_IRQ_RESET(VLV_);
dev_priv->irq_mask = ~0;
}
u32 enable_mask;
enum pipe pipe;
- pipestat_mask = PLANE_FLIP_DONE_INT_STATUS_VLV |
- PIPE_CRC_DONE_INTERRUPT_STATUS;
+ pipestat_mask = PIPE_CRC_DONE_INTERRUPT_STATUS;
i915_enable_pipestat(dev_priv, PIPE_A, PIPE_GMBUS_INTERRUPT_STATUS);
for_each_pipe(dev_priv, pipe)
dev_priv->irq_mask = ~enable_mask;
- GEN5_IRQ_INIT(VLV_, dev_priv->irq_mask, enable_mask);
+ GEN3_IRQ_INIT(VLV_, dev_priv->irq_mask, enable_mask);
}
/* drm_dma.h hooks
{
struct drm_i915_private *dev_priv = to_i915(dev);
- I915_WRITE(HWSTAM, 0xffffffff);
+ if (IS_GEN5(dev_priv))
+ I915_WRITE(HWSTAM, 0xffffffff);
- GEN5_IRQ_RESET(DE);
+ GEN3_IRQ_RESET(DE);
if (IS_GEN7(dev_priv))
I915_WRITE(GEN7_ERR_INT, 0xffffffff);
ibx_irq_reset(dev_priv);
}
-static void valleyview_irq_preinstall(struct drm_device *dev)
+static void valleyview_irq_reset(struct drm_device *dev)
{
struct drm_i915_private *dev_priv = to_i915(dev);
POWER_DOMAIN_PIPE(pipe)))
GEN8_IRQ_RESET_NDX(DE_PIPE, pipe);
- GEN5_IRQ_RESET(GEN8_DE_PORT_);
- GEN5_IRQ_RESET(GEN8_DE_MISC_);
- GEN5_IRQ_RESET(GEN8_PCU_);
+ GEN3_IRQ_RESET(GEN8_DE_PORT_);
+ GEN3_IRQ_RESET(GEN8_DE_MISC_);
+ GEN3_IRQ_RESET(GEN8_PCU_);
if (HAS_PCH_SPLIT(dev_priv))
ibx_irq_reset(dev_priv);
synchronize_irq(dev_priv->drm.irq);
}
-static void cherryview_irq_preinstall(struct drm_device *dev)
+static void cherryview_irq_reset(struct drm_device *dev)
{
struct drm_i915_private *dev_priv = to_i915(dev);
gen8_gt_irq_reset(dev_priv);
- GEN5_IRQ_RESET(GEN8_PCU_);
+ GEN3_IRQ_RESET(GEN8_PCU_);
spin_lock_irq(&dev_priv->irq_lock);
if (dev_priv->display_irqs_enabled)
static void spt_hpd_detection_setup(struct drm_i915_private *dev_priv)
{
- u32 hotplug;
+ u32 val, hotplug;
+
+ /* Display WA #1179 WaHardHangonHotPlug: cnp */
+ if (HAS_PCH_CNP(dev_priv)) {
+ val = I915_READ(SOUTH_CHICKEN1);
+ val &= ~CHASSIS_CLK_REQ_DURATION_MASK;
+ val |= CHASSIS_CLK_REQ_DURATION(0xf);
+ I915_WRITE(SOUTH_CHICKEN1, val);
+ }
/* Enable digital hotplug on the PCH */
hotplug = I915_READ(PCH_PORT_HOTPLUG);
if (HAS_PCH_IBX(dev_priv))
mask = SDE_GMBUS | SDE_AUX_MASK | SDE_POISON;
- else
+ else if (HAS_PCH_CPT(dev_priv) || HAS_PCH_LPT(dev_priv))
mask = SDE_GMBUS_CPT | SDE_AUX_MASK_CPT;
+ else
+ mask = SDE_GMBUS_CPT;
- gen5_assert_iir_is_zero(dev_priv, SDEIIR);
+ gen3_assert_iir_is_zero(dev_priv, SDEIIR);
I915_WRITE(SDEIMR, ~mask);
if (HAS_PCH_IBX(dev_priv) || HAS_PCH_CPT(dev_priv) ||
gt_irqs |= GT_BLT_USER_INTERRUPT | GT_BSD_USER_INTERRUPT;
}
- GEN5_IRQ_INIT(GT, dev_priv->gt_irq_mask, gt_irqs);
+ GEN3_IRQ_INIT(GT, dev_priv->gt_irq_mask, gt_irqs);
if (INTEL_GEN(dev_priv) >= 6) {
/*
}
dev_priv->pm_imr = 0xffffffff;
- GEN5_IRQ_INIT(GEN6_PM, dev_priv->pm_imr, pm_irqs);
+ GEN3_IRQ_INIT(GEN6_PM, dev_priv->pm_imr, pm_irqs);
}
}
if (INTEL_GEN(dev_priv) >= 7) {
display_mask = (DE_MASTER_IRQ_CONTROL | DE_GSE_IVB |
- DE_PCH_EVENT_IVB | DE_PLANEC_FLIP_DONE_IVB |
- DE_PLANEB_FLIP_DONE_IVB |
- DE_PLANEA_FLIP_DONE_IVB | DE_AUX_CHANNEL_A_IVB);
+ DE_PCH_EVENT_IVB | DE_AUX_CHANNEL_A_IVB);
extra_mask = (DE_PIPEC_VBLANK_IVB | DE_PIPEB_VBLANK_IVB |
DE_PIPEA_VBLANK_IVB | DE_ERR_INT_IVB |
DE_DP_A_HOTPLUG_IVB);
} else {
display_mask = (DE_MASTER_IRQ_CONTROL | DE_GSE | DE_PCH_EVENT |
- DE_PLANEA_FLIP_DONE | DE_PLANEB_FLIP_DONE |
- DE_AUX_CHANNEL_A |
- DE_PIPEB_CRC_DONE | DE_PIPEA_CRC_DONE |
- DE_POISON);
+ DE_AUX_CHANNEL_A | DE_PIPEB_CRC_DONE |
+ DE_PIPEA_CRC_DONE | DE_POISON);
extra_mask = (DE_PIPEA_VBLANK | DE_PIPEB_VBLANK | DE_PCU_EVENT |
DE_PIPEB_FIFO_UNDERRUN | DE_PIPEA_FIFO_UNDERRUN |
DE_DP_A_HOTPLUG);
dev_priv->irq_mask = ~display_mask;
- I915_WRITE(HWSTAM, 0xeffe);
-
ibx_irq_pre_postinstall(dev);
- GEN5_IRQ_INIT(DE, dev_priv->irq_mask, display_mask | extra_mask);
+ GEN3_IRQ_INIT(DE, dev_priv->irq_mask, display_mask | extra_mask);
gen5_gt_irq_postinstall(dev);
enum pipe pipe;
if (INTEL_GEN(dev_priv) >= 9) {
- de_pipe_masked |= GEN9_PIPE_PLANE1_FLIP_DONE |
- GEN9_DE_PIPE_IRQ_FAULT_ERRORS;
+ de_pipe_masked |= GEN9_DE_PIPE_IRQ_FAULT_ERRORS;
de_port_masked |= GEN9_AUX_CHANNEL_B | GEN9_AUX_CHANNEL_C |
GEN9_AUX_CHANNEL_D;
if (IS_GEN9_LP(dev_priv))
de_port_masked |= BXT_DE_PORT_GMBUS;
} else {
- de_pipe_masked |= GEN8_PIPE_PRIMARY_FLIP_DONE |
- GEN8_DE_PIPE_IRQ_FAULT_ERRORS;
+ de_pipe_masked |= GEN8_DE_PIPE_IRQ_FAULT_ERRORS;
}
de_pipe_enables = de_pipe_masked | GEN8_PIPE_VBLANK |
dev_priv->de_irq_mask[pipe],
de_pipe_enables);
- GEN5_IRQ_INIT(GEN8_DE_PORT_, ~de_port_masked, de_port_enables);
- GEN5_IRQ_INIT(GEN8_DE_MISC_, ~de_misc_masked, de_misc_masked);
+ GEN3_IRQ_INIT(GEN8_DE_PORT_, ~de_port_masked, de_port_enables);
+ GEN3_IRQ_INIT(GEN8_DE_MISC_, ~de_misc_masked, de_misc_masked);
if (IS_GEN9_LP(dev_priv))
bxt_hpd_detection_setup(dev_priv);
return 0;
}
-static void gen8_irq_uninstall(struct drm_device *dev)
-{
- struct drm_i915_private *dev_priv = to_i915(dev);
-
- if (!dev_priv)
- return;
-
- gen8_irq_reset(dev);
-}
-
-static void valleyview_irq_uninstall(struct drm_device *dev)
-{
- struct drm_i915_private *dev_priv = to_i915(dev);
-
- if (!dev_priv)
- return;
-
- I915_WRITE(VLV_MASTER_IER, 0);
- POSTING_READ(VLV_MASTER_IER);
-
- gen5_gt_irq_reset(dev_priv);
-
- I915_WRITE(HWSTAM, 0xffffffff);
-
- spin_lock_irq(&dev_priv->irq_lock);
- if (dev_priv->display_irqs_enabled)
- vlv_display_irq_reset(dev_priv);
- spin_unlock_irq(&dev_priv->irq_lock);
-}
-
-static void cherryview_irq_uninstall(struct drm_device *dev)
+static void i8xx_irq_reset(struct drm_device *dev)
{
struct drm_i915_private *dev_priv = to_i915(dev);
- if (!dev_priv)
- return;
-
- I915_WRITE(GEN8_MASTER_IRQ, 0);
- POSTING_READ(GEN8_MASTER_IRQ);
-
- gen8_gt_irq_reset(dev_priv);
-
- GEN5_IRQ_RESET(GEN8_PCU_);
-
- spin_lock_irq(&dev_priv->irq_lock);
- if (dev_priv->display_irqs_enabled)
- vlv_display_irq_reset(dev_priv);
- spin_unlock_irq(&dev_priv->irq_lock);
-}
-
-static void ironlake_irq_uninstall(struct drm_device *dev)
-{
- struct drm_i915_private *dev_priv = to_i915(dev);
+ i9xx_pipestat_irq_reset(dev_priv);
- if (!dev_priv)
- return;
-
- ironlake_irq_reset(dev);
-}
-
-static void i8xx_irq_preinstall(struct drm_device * dev)
-{
- struct drm_i915_private *dev_priv = to_i915(dev);
- int pipe;
+ I915_WRITE16(HWSTAM, 0xffff);
- for_each_pipe(dev_priv, pipe)
- I915_WRITE(PIPESTAT(pipe), 0);
- I915_WRITE16(IMR, 0xffff);
- I915_WRITE16(IER, 0x0);
- POSTING_READ16(IER);
+ GEN2_IRQ_RESET();
}
static int i8xx_irq_postinstall(struct drm_device *dev)
{
struct drm_i915_private *dev_priv = to_i915(dev);
+ u16 enable_mask;
- I915_WRITE16(EMR,
- ~(I915_ERROR_PAGE_TABLE | I915_ERROR_MEMORY_REFRESH));
+ I915_WRITE16(EMR, ~(I915_ERROR_PAGE_TABLE |
+ I915_ERROR_MEMORY_REFRESH));
/* Unmask the interrupts that we always want on. */
dev_priv->irq_mask =
~(I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
- I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
- I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
- I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT);
- I915_WRITE16(IMR, dev_priv->irq_mask);
+ I915_DISPLAY_PIPE_B_EVENT_INTERRUPT);
+
+ enable_mask =
+ I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
+ I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
+ I915_USER_INTERRUPT;
- I915_WRITE16(IER,
- I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
- I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
- I915_USER_INTERRUPT);
- POSTING_READ16(IER);
+ GEN2_IRQ_INIT(, dev_priv->irq_mask, enable_mask);
/* Interrupt setup is already guaranteed to be single-threaded, this is
* just to make the assert_spin_locked check happy. */
return 0;
}
-/*
- * Returns true when a page flip has completed.
- */
static irqreturn_t i8xx_irq_handler(int irq, void *arg)
{
struct drm_device *dev = arg;
struct drm_i915_private *dev_priv = to_i915(dev);
- u16 iir, new_iir;
- u32 pipe_stats[2];
- int pipe;
- irqreturn_t ret;
+ irqreturn_t ret = IRQ_NONE;
if (!intel_irqs_enabled(dev_priv))
return IRQ_NONE;
/* IRQs are synced during runtime_suspend, we don't require a wakeref */
disable_rpm_wakeref_asserts(dev_priv);
- ret = IRQ_NONE;
- iir = I915_READ16(IIR);
- if (iir == 0)
- goto out;
+ do {
+ u32 pipe_stats[I915_MAX_PIPES] = {};
+ u16 iir;
- while (iir) {
- /* Can't rely on pipestat interrupt bit in iir as it might
- * have been cleared after the pipestat interrupt was received.
- * It doesn't set the bit in iir again, but it still produces
- * interrupts (for non-MSI).
- */
- spin_lock(&dev_priv->irq_lock);
- if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
- DRM_DEBUG("Command parser error, iir 0x%08x\n", iir);
+ iir = I915_READ16(IIR);
+ if (iir == 0)
+ break;
- for_each_pipe(dev_priv, pipe) {
- i915_reg_t reg = PIPESTAT(pipe);
- pipe_stats[pipe] = I915_READ(reg);
+ ret = IRQ_HANDLED;
- /*
- * Clear the PIPE*STAT regs before the IIR
- */
- if (pipe_stats[pipe] & 0x8000ffff)
- I915_WRITE(reg, pipe_stats[pipe]);
- }
- spin_unlock(&dev_priv->irq_lock);
+ /* Call regardless, as some status bits might not be
+ * signalled in iir */
+ i9xx_pipestat_irq_ack(dev_priv, iir, pipe_stats);
I915_WRITE16(IIR, iir);
- new_iir = I915_READ16(IIR); /* Flush posted writes */
if (iir & I915_USER_INTERRUPT)
notify_ring(dev_priv->engine[RCS]);
- for_each_pipe(dev_priv, pipe) {
- int plane = pipe;
- if (HAS_FBC(dev_priv))
- plane = !plane;
-
- if (pipe_stats[pipe] & PIPE_VBLANK_INTERRUPT_STATUS)
- drm_handle_vblank(&dev_priv->drm, pipe);
-
- if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
- i9xx_pipe_crc_irq_handler(dev_priv, pipe);
-
- if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
- intel_cpu_fifo_underrun_irq_handler(dev_priv,
- pipe);
- }
+ if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
+ DRM_DEBUG("Command parser error, iir 0x%08x\n", iir);
- iir = new_iir;
- }
- ret = IRQ_HANDLED;
+ i8xx_pipestat_irq_handler(dev_priv, iir, pipe_stats);
+ } while (0);
-out:
enable_rpm_wakeref_asserts(dev_priv);
return ret;
}
-static void i8xx_irq_uninstall(struct drm_device * dev)
-{
- struct drm_i915_private *dev_priv = to_i915(dev);
- int pipe;
-
- for_each_pipe(dev_priv, pipe) {
- /* Clear enable bits; then clear status bits */
- I915_WRITE(PIPESTAT(pipe), 0);
- I915_WRITE(PIPESTAT(pipe), I915_READ(PIPESTAT(pipe)));
- }
- I915_WRITE16(IMR, 0xffff);
- I915_WRITE16(IER, 0x0);
- I915_WRITE16(IIR, I915_READ16(IIR));
-}
-
-static void i915_irq_preinstall(struct drm_device * dev)
+static void i915_irq_reset(struct drm_device *dev)
{
struct drm_i915_private *dev_priv = to_i915(dev);
- int pipe;
if (I915_HAS_HOTPLUG(dev_priv)) {
i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
}
- I915_WRITE16(HWSTAM, 0xeffe);
- for_each_pipe(dev_priv, pipe)
- I915_WRITE(PIPESTAT(pipe), 0);
- I915_WRITE(IMR, 0xffffffff);
- I915_WRITE(IER, 0x0);
- POSTING_READ(IER);
+ i9xx_pipestat_irq_reset(dev_priv);
+
+ I915_WRITE(HWSTAM, 0xffffffff);
+
+ GEN3_IRQ_RESET();
}
static int i915_irq_postinstall(struct drm_device *dev)
struct drm_i915_private *dev_priv = to_i915(dev);
u32 enable_mask;
- I915_WRITE(EMR, ~(I915_ERROR_PAGE_TABLE | I915_ERROR_MEMORY_REFRESH));
+ I915_WRITE(EMR, ~(I915_ERROR_PAGE_TABLE |
+ I915_ERROR_MEMORY_REFRESH));
/* Unmask the interrupts that we always want on. */
dev_priv->irq_mask =
~(I915_ASLE_INTERRUPT |
I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
- I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
- I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
- I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT);
+ I915_DISPLAY_PIPE_B_EVENT_INTERRUPT);
enable_mask =
I915_ASLE_INTERRUPT |
I915_USER_INTERRUPT;
if (I915_HAS_HOTPLUG(dev_priv)) {
- i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
- POSTING_READ(PORT_HOTPLUG_EN);
-
/* Enable in IER... */
enable_mask |= I915_DISPLAY_PORT_INTERRUPT;
/* and unmask in IMR */
dev_priv->irq_mask &= ~I915_DISPLAY_PORT_INTERRUPT;
}
- I915_WRITE(IMR, dev_priv->irq_mask);
- I915_WRITE(IER, enable_mask);
- POSTING_READ(IER);
-
- i915_enable_asle_pipestat(dev_priv);
+ GEN3_IRQ_INIT(, dev_priv->irq_mask, enable_mask);
/* Interrupt setup is already guaranteed to be single-threaded, this is
* just to make the assert_spin_locked check happy. */
i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS);
spin_unlock_irq(&dev_priv->irq_lock);
+ i915_enable_asle_pipestat(dev_priv);
+
return 0;
}
{
struct drm_device *dev = arg;
struct drm_i915_private *dev_priv = to_i915(dev);
- u32 iir, new_iir, pipe_stats[I915_MAX_PIPES];
- int pipe, ret = IRQ_NONE;
+ irqreturn_t ret = IRQ_NONE;
if (!intel_irqs_enabled(dev_priv))
return IRQ_NONE;
/* IRQs are synced during runtime_suspend, we don't require a wakeref */
disable_rpm_wakeref_asserts(dev_priv);
- iir = I915_READ(IIR);
do {
- bool irq_received = (iir) != 0;
- bool blc_event = false;
-
- /* Can't rely on pipestat interrupt bit in iir as it might
- * have been cleared after the pipestat interrupt was received.
- * It doesn't set the bit in iir again, but it still produces
- * interrupts (for non-MSI).
- */
- spin_lock(&dev_priv->irq_lock);
- if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
- DRM_DEBUG("Command parser error, iir 0x%08x\n", iir);
-
- for_each_pipe(dev_priv, pipe) {
- i915_reg_t reg = PIPESTAT(pipe);
- pipe_stats[pipe] = I915_READ(reg);
-
- /* Clear the PIPE*STAT regs before the IIR */
- if (pipe_stats[pipe] & 0x8000ffff) {
- I915_WRITE(reg, pipe_stats[pipe]);
- irq_received = true;
- }
- }
- spin_unlock(&dev_priv->irq_lock);
+ u32 pipe_stats[I915_MAX_PIPES] = {};
+ u32 hotplug_status = 0;
+ u32 iir;
- if (!irq_received)
+ iir = I915_READ(IIR);
+ if (iir == 0)
break;
- /* Consume port. Then clear IIR or we'll miss events */
+ ret = IRQ_HANDLED;
+
if (I915_HAS_HOTPLUG(dev_priv) &&
- iir & I915_DISPLAY_PORT_INTERRUPT) {
- u32 hotplug_status = i9xx_hpd_irq_ack(dev_priv);
- if (hotplug_status)
- i9xx_hpd_irq_handler(dev_priv, hotplug_status);
- }
+ iir & I915_DISPLAY_PORT_INTERRUPT)
+ hotplug_status = i9xx_hpd_irq_ack(dev_priv);
+
+ /* Call regardless, as some status bits might not be
+ * signalled in iir */
+ i9xx_pipestat_irq_ack(dev_priv, iir, pipe_stats);
I915_WRITE(IIR, iir);
- new_iir = I915_READ(IIR); /* Flush posted writes */
if (iir & I915_USER_INTERRUPT)
notify_ring(dev_priv->engine[RCS]);
- for_each_pipe(dev_priv, pipe) {
- int plane = pipe;
- if (HAS_FBC(dev_priv))
- plane = !plane;
-
- if (pipe_stats[pipe] & PIPE_VBLANK_INTERRUPT_STATUS)
- drm_handle_vblank(&dev_priv->drm, pipe);
-
- if (pipe_stats[pipe] & PIPE_LEGACY_BLC_EVENT_STATUS)
- blc_event = true;
-
- if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
- i9xx_pipe_crc_irq_handler(dev_priv, pipe);
-
- if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
- intel_cpu_fifo_underrun_irq_handler(dev_priv,
- pipe);
- }
+ if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
+ DRM_DEBUG("Command parser error, iir 0x%08x\n", iir);
- if (blc_event || (iir & I915_ASLE_INTERRUPT))
- intel_opregion_asle_intr(dev_priv);
+ if (hotplug_status)
+ i9xx_hpd_irq_handler(dev_priv, hotplug_status);
- /* With MSI, interrupts are only generated when iir
- * transitions from zero to nonzero. If another bit got
- * set while we were handling the existing iir bits, then
- * we would never get another interrupt.
- *
- * This is fine on non-MSI as well, as if we hit this path
- * we avoid exiting the interrupt handler only to generate
- * another one.
- *
- * Note that for MSI this could cause a stray interrupt report
- * if an interrupt landed in the time between writing IIR and
- * the posting read. This should be rare enough to never
- * trigger the 99% of 100,000 interrupts test for disabling
- * stray interrupts.
- */
- ret = IRQ_HANDLED;
- iir = new_iir;
- } while (iir);
+ i915_pipestat_irq_handler(dev_priv, iir, pipe_stats);
+ } while (0);
enable_rpm_wakeref_asserts(dev_priv);
return ret;
}
-static void i915_irq_uninstall(struct drm_device * dev)
-{
- struct drm_i915_private *dev_priv = to_i915(dev);
- int pipe;
-
- if (I915_HAS_HOTPLUG(dev_priv)) {
- i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
- I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
- }
-
- I915_WRITE16(HWSTAM, 0xffff);
- for_each_pipe(dev_priv, pipe) {
- /* Clear enable bits; then clear status bits */
- I915_WRITE(PIPESTAT(pipe), 0);
- I915_WRITE(PIPESTAT(pipe), I915_READ(PIPESTAT(pipe)));
- }
- I915_WRITE(IMR, 0xffffffff);
- I915_WRITE(IER, 0x0);
-
- I915_WRITE(IIR, I915_READ(IIR));
-}
-
-static void i965_irq_preinstall(struct drm_device * dev)
+static void i965_irq_reset(struct drm_device *dev)
{
struct drm_i915_private *dev_priv = to_i915(dev);
- int pipe;
i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
- I915_WRITE(HWSTAM, 0xeffe);
- for_each_pipe(dev_priv, pipe)
- I915_WRITE(PIPESTAT(pipe), 0);
- I915_WRITE(IMR, 0xffffffff);
- I915_WRITE(IER, 0x0);
- POSTING_READ(IER);
+ i9xx_pipestat_irq_reset(dev_priv);
+
+ I915_WRITE(HWSTAM, 0xffffffff);
+
+ GEN3_IRQ_RESET();
}
static int i965_irq_postinstall(struct drm_device *dev)
u32 enable_mask;
u32 error_mask;
- /* Unmask the interrupts that we always want on. */
- dev_priv->irq_mask = ~(I915_ASLE_INTERRUPT |
- I915_DISPLAY_PORT_INTERRUPT |
- I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
- I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
- I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
- I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT |
- I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT);
-
- enable_mask = ~dev_priv->irq_mask;
- enable_mask &= ~(I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
- I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT);
- enable_mask |= I915_USER_INTERRUPT;
-
- if (IS_G4X(dev_priv))
- enable_mask |= I915_BSD_USER_INTERRUPT;
-
- /* Interrupt setup is already guaranteed to be single-threaded, this is
- * just to make the assert_spin_locked check happy. */
- spin_lock_irq(&dev_priv->irq_lock);
- i915_enable_pipestat(dev_priv, PIPE_A, PIPE_GMBUS_INTERRUPT_STATUS);
- i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS);
- i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS);
- spin_unlock_irq(&dev_priv->irq_lock);
-
/*
* Enable some error detection, note the instruction error mask
* bit is reserved, so we leave it masked.
}
I915_WRITE(EMR, error_mask);
- I915_WRITE(IMR, dev_priv->irq_mask);
- I915_WRITE(IER, enable_mask);
- POSTING_READ(IER);
+ /* Unmask the interrupts that we always want on. */
+ dev_priv->irq_mask =
+ ~(I915_ASLE_INTERRUPT |
+ I915_DISPLAY_PORT_INTERRUPT |
+ I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
+ I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
+ I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT);
- i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
- POSTING_READ(PORT_HOTPLUG_EN);
+ enable_mask =
+ I915_ASLE_INTERRUPT |
+ I915_DISPLAY_PORT_INTERRUPT |
+ I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
+ I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
+ I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT |
+ I915_USER_INTERRUPT;
+
+ if (IS_G4X(dev_priv))
+ enable_mask |= I915_BSD_USER_INTERRUPT;
+
+ GEN3_IRQ_INIT(, dev_priv->irq_mask, enable_mask);
+
+ /* Interrupt setup is already guaranteed to be single-threaded, this is
+ * just to make the assert_spin_locked check happy. */
+ spin_lock_irq(&dev_priv->irq_lock);
+ i915_enable_pipestat(dev_priv, PIPE_A, PIPE_GMBUS_INTERRUPT_STATUS);
+ i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS);
+ i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS);
+ spin_unlock_irq(&dev_priv->irq_lock);
i915_enable_asle_pipestat(dev_priv);
{
struct drm_device *dev = arg;
struct drm_i915_private *dev_priv = to_i915(dev);
- u32 iir, new_iir;
- u32 pipe_stats[I915_MAX_PIPES];
- int ret = IRQ_NONE, pipe;
+ irqreturn_t ret = IRQ_NONE;
if (!intel_irqs_enabled(dev_priv))
return IRQ_NONE;
/* IRQs are synced during runtime_suspend, we don't require a wakeref */
disable_rpm_wakeref_asserts(dev_priv);
- iir = I915_READ(IIR);
-
- for (;;) {
- bool irq_received = (iir) != 0;
- bool blc_event = false;
-
- /* Can't rely on pipestat interrupt bit in iir as it might
- * have been cleared after the pipestat interrupt was received.
- * It doesn't set the bit in iir again, but it still produces
- * interrupts (for non-MSI).
- */
- spin_lock(&dev_priv->irq_lock);
- if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
- DRM_DEBUG("Command parser error, iir 0x%08x\n", iir);
-
- for_each_pipe(dev_priv, pipe) {
- i915_reg_t reg = PIPESTAT(pipe);
- pipe_stats[pipe] = I915_READ(reg);
-
- /*
- * Clear the PIPE*STAT regs before the IIR
- */
- if (pipe_stats[pipe] & 0x8000ffff) {
- I915_WRITE(reg, pipe_stats[pipe]);
- irq_received = true;
- }
- }
- spin_unlock(&dev_priv->irq_lock);
+ do {
+ u32 pipe_stats[I915_MAX_PIPES] = {};
+ u32 hotplug_status = 0;
+ u32 iir;
- if (!irq_received)
+ iir = I915_READ(IIR);
+ if (iir == 0)
break;
ret = IRQ_HANDLED;
- /* Consume port. Then clear IIR or we'll miss events */
- if (iir & I915_DISPLAY_PORT_INTERRUPT) {
- u32 hotplug_status = i9xx_hpd_irq_ack(dev_priv);
- if (hotplug_status)
- i9xx_hpd_irq_handler(dev_priv, hotplug_status);
- }
+ if (iir & I915_DISPLAY_PORT_INTERRUPT)
+ hotplug_status = i9xx_hpd_irq_ack(dev_priv);
+
+ /* Call regardless, as some status bits might not be
+ * signalled in iir */
+ i9xx_pipestat_irq_ack(dev_priv, iir, pipe_stats);
I915_WRITE(IIR, iir);
- new_iir = I915_READ(IIR); /* Flush posted writes */
if (iir & I915_USER_INTERRUPT)
notify_ring(dev_priv->engine[RCS]);
+
if (iir & I915_BSD_USER_INTERRUPT)
notify_ring(dev_priv->engine[VCS]);
- for_each_pipe(dev_priv, pipe) {
- if (pipe_stats[pipe] & PIPE_START_VBLANK_INTERRUPT_STATUS)
- drm_handle_vblank(&dev_priv->drm, pipe);
-
- if (pipe_stats[pipe] & PIPE_LEGACY_BLC_EVENT_STATUS)
- blc_event = true;
-
- if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
- i9xx_pipe_crc_irq_handler(dev_priv, pipe);
-
- if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
- intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
- }
-
- if (blc_event || (iir & I915_ASLE_INTERRUPT))
- intel_opregion_asle_intr(dev_priv);
+ if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
+ DRM_DEBUG("Command parser error, iir 0x%08x\n", iir);
- if (pipe_stats[0] & PIPE_GMBUS_INTERRUPT_STATUS)
- gmbus_irq_handler(dev_priv);
+ if (hotplug_status)
+ i9xx_hpd_irq_handler(dev_priv, hotplug_status);
- /* With MSI, interrupts are only generated when iir
- * transitions from zero to nonzero. If another bit got
- * set while we were handling the existing iir bits, then
- * we would never get another interrupt.
- *
- * This is fine on non-MSI as well, as if we hit this path
- * we avoid exiting the interrupt handler only to generate
- * another one.
- *
- * Note that for MSI this could cause a stray interrupt report
- * if an interrupt landed in the time between writing IIR and
- * the posting read. This should be rare enough to never
- * trigger the 99% of 100,000 interrupts test for disabling
- * stray interrupts.
- */
- iir = new_iir;
- }
+ i965_pipestat_irq_handler(dev_priv, iir, pipe_stats);
+ } while (0);
enable_rpm_wakeref_asserts(dev_priv);
return ret;
}
-static void i965_irq_uninstall(struct drm_device * dev)
-{
- struct drm_i915_private *dev_priv = to_i915(dev);
- int pipe;
-
- if (!dev_priv)
- return;
-
- i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
- I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
-
- I915_WRITE(HWSTAM, 0xffffffff);
- for_each_pipe(dev_priv, pipe)
- I915_WRITE(PIPESTAT(pipe), 0);
- I915_WRITE(IMR, 0xffffffff);
- I915_WRITE(IER, 0x0);
-
- for_each_pipe(dev_priv, pipe)
- I915_WRITE(PIPESTAT(pipe),
- I915_READ(PIPESTAT(pipe)) & 0x8000ffff);
- I915_WRITE(IIR, I915_READ(IIR));
-}
-
/**
* intel_irq_init - initializes irq support
* @dev_priv: i915 device instance
if (IS_CHERRYVIEW(dev_priv)) {
dev->driver->irq_handler = cherryview_irq_handler;
- dev->driver->irq_preinstall = cherryview_irq_preinstall;
+ dev->driver->irq_preinstall = cherryview_irq_reset;
dev->driver->irq_postinstall = cherryview_irq_postinstall;
- dev->driver->irq_uninstall = cherryview_irq_uninstall;
+ dev->driver->irq_uninstall = cherryview_irq_reset;
dev->driver->enable_vblank = i965_enable_vblank;
dev->driver->disable_vblank = i965_disable_vblank;
dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup;
} else if (IS_VALLEYVIEW(dev_priv)) {
dev->driver->irq_handler = valleyview_irq_handler;
- dev->driver->irq_preinstall = valleyview_irq_preinstall;
+ dev->driver->irq_preinstall = valleyview_irq_reset;
dev->driver->irq_postinstall = valleyview_irq_postinstall;
- dev->driver->irq_uninstall = valleyview_irq_uninstall;
+ dev->driver->irq_uninstall = valleyview_irq_reset;
dev->driver->enable_vblank = i965_enable_vblank;
dev->driver->disable_vblank = i965_disable_vblank;
dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup;
dev->driver->irq_handler = gen8_irq_handler;
dev->driver->irq_preinstall = gen8_irq_reset;
dev->driver->irq_postinstall = gen8_irq_postinstall;
- dev->driver->irq_uninstall = gen8_irq_uninstall;
+ dev->driver->irq_uninstall = gen8_irq_reset;
dev->driver->enable_vblank = gen8_enable_vblank;
dev->driver->disable_vblank = gen8_disable_vblank;
if (IS_GEN9_LP(dev_priv))
dev->driver->irq_handler = ironlake_irq_handler;
dev->driver->irq_preinstall = ironlake_irq_reset;
dev->driver->irq_postinstall = ironlake_irq_postinstall;
- dev->driver->irq_uninstall = ironlake_irq_uninstall;
+ dev->driver->irq_uninstall = ironlake_irq_reset;
dev->driver->enable_vblank = ironlake_enable_vblank;
dev->driver->disable_vblank = ironlake_disable_vblank;
dev_priv->display.hpd_irq_setup = ilk_hpd_irq_setup;
} else {
if (IS_GEN2(dev_priv)) {
- dev->driver->irq_preinstall = i8xx_irq_preinstall;
+ dev->driver->irq_preinstall = i8xx_irq_reset;
dev->driver->irq_postinstall = i8xx_irq_postinstall;
dev->driver->irq_handler = i8xx_irq_handler;
- dev->driver->irq_uninstall = i8xx_irq_uninstall;
+ dev->driver->irq_uninstall = i8xx_irq_reset;
dev->driver->enable_vblank = i8xx_enable_vblank;
dev->driver->disable_vblank = i8xx_disable_vblank;
} else if (IS_GEN3(dev_priv)) {
- dev->driver->irq_preinstall = i915_irq_preinstall;
+ dev->driver->irq_preinstall = i915_irq_reset;
dev->driver->irq_postinstall = i915_irq_postinstall;
- dev->driver->irq_uninstall = i915_irq_uninstall;
+ dev->driver->irq_uninstall = i915_irq_reset;
dev->driver->irq_handler = i915_irq_handler;
dev->driver->enable_vblank = i8xx_enable_vblank;
dev->driver->disable_vblank = i8xx_disable_vblank;
} else {
- dev->driver->irq_preinstall = i965_irq_preinstall;
+ dev->driver->irq_preinstall = i965_irq_reset;
dev->driver->irq_postinstall = i965_irq_postinstall;
- dev->driver->irq_uninstall = i965_irq_uninstall;
+ dev->driver->irq_uninstall = i965_irq_reset;
dev->driver->irq_handler = i965_irq_handler;
dev->driver->enable_vblank = i965_enable_vblank;
dev->driver->disable_vblank = i965_disable_vblank;
--- /dev/null
+/*
+ * Autogenerated file by GPU Top : https://github.com/rib/gputop
+ * DO NOT EDIT manually!
+ *
+ *
+ * Copyright (c) 2015 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ *
+ */
+
+#include <linux/sysfs.h>
+
+#include "i915_drv.h"
+#include "i915_oa_cflgt2.h"
+
+static const struct i915_oa_reg b_counter_config_test_oa[] = {
+ { _MMIO(0x2740), 0x00000000 },
+ { _MMIO(0x2744), 0x00800000 },
+ { _MMIO(0x2714), 0xf0800000 },
+ { _MMIO(0x2710), 0x00000000 },
+ { _MMIO(0x2724), 0xf0800000 },
+ { _MMIO(0x2720), 0x00000000 },
+ { _MMIO(0x2770), 0x00000004 },
+ { _MMIO(0x2774), 0x00000000 },
+ { _MMIO(0x2778), 0x00000003 },
+ { _MMIO(0x277c), 0x00000000 },
+ { _MMIO(0x2780), 0x00000007 },
+ { _MMIO(0x2784), 0x00000000 },
+ { _MMIO(0x2788), 0x00100002 },
+ { _MMIO(0x278c), 0x0000fff7 },
+ { _MMIO(0x2790), 0x00100002 },
+ { _MMIO(0x2794), 0x0000ffcf },
+ { _MMIO(0x2798), 0x00100082 },
+ { _MMIO(0x279c), 0x0000ffef },
+ { _MMIO(0x27a0), 0x001000c2 },
+ { _MMIO(0x27a4), 0x0000ffe7 },
+ { _MMIO(0x27a8), 0x00100001 },
+ { _MMIO(0x27ac), 0x0000ffe7 },
+};
+
+static const struct i915_oa_reg flex_eu_config_test_oa[] = {
+};
+
+static const struct i915_oa_reg mux_config_test_oa[] = {
+ { _MMIO(0x9840), 0x00000080 },
+ { _MMIO(0x9888), 0x11810000 },
+ { _MMIO(0x9888), 0x07810013 },
+ { _MMIO(0x9888), 0x1f810000 },
+ { _MMIO(0x9888), 0x1d810000 },
+ { _MMIO(0x9888), 0x1b930040 },
+ { _MMIO(0x9888), 0x07e54000 },
+ { _MMIO(0x9888), 0x1f908000 },
+ { _MMIO(0x9888), 0x11900000 },
+ { _MMIO(0x9888), 0x37900000 },
+ { _MMIO(0x9888), 0x53900000 },
+ { _MMIO(0x9888), 0x45900000 },
+ { _MMIO(0x9888), 0x33900000 },
+};
+
+static ssize_t
+show_test_oa_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+ return sprintf(buf, "1\n");
+}
+
+void
+i915_perf_load_test_config_cflgt2(struct drm_i915_private *dev_priv)
+{
+ strncpy(dev_priv->perf.oa.test_config.uuid,
+ "74fb4902-d3d3-4237-9e90-cbdc68d0a446",
+ UUID_STRING_LEN);
+ dev_priv->perf.oa.test_config.id = 1;
+
+ dev_priv->perf.oa.test_config.mux_regs = mux_config_test_oa;
+ dev_priv->perf.oa.test_config.mux_regs_len = ARRAY_SIZE(mux_config_test_oa);
+
+ dev_priv->perf.oa.test_config.b_counter_regs = b_counter_config_test_oa;
+ dev_priv->perf.oa.test_config.b_counter_regs_len = ARRAY_SIZE(b_counter_config_test_oa);
+
+ dev_priv->perf.oa.test_config.flex_regs = flex_eu_config_test_oa;
+ dev_priv->perf.oa.test_config.flex_regs_len = ARRAY_SIZE(flex_eu_config_test_oa);
+
+ dev_priv->perf.oa.test_config.sysfs_metric.name = "74fb4902-d3d3-4237-9e90-cbdc68d0a446";
+ dev_priv->perf.oa.test_config.sysfs_metric.attrs = dev_priv->perf.oa.test_config.attrs;
+
+ dev_priv->perf.oa.test_config.attrs[0] = &dev_priv->perf.oa.test_config.sysfs_metric_id.attr;
+
+ dev_priv->perf.oa.test_config.sysfs_metric_id.attr.name = "id";
+ dev_priv->perf.oa.test_config.sysfs_metric_id.attr.mode = 0444;
+ dev_priv->perf.oa.test_config.sysfs_metric_id.show = show_test_oa_id;
+}
--- /dev/null
+/*
+ * Autogenerated file by GPU Top : https://github.com/rib/gputop
+ * DO NOT EDIT manually!
+ *
+ *
+ * Copyright (c) 2015 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ *
+ */
+
+#ifndef __I915_OA_CFLGT2_H__
+#define __I915_OA_CFLGT2_H__
+
+extern void i915_perf_load_test_config_cflgt2(struct drm_i915_private *dev_priv);
+
+#endif
#include "i915_params.h"
#include "i915_drv.h"
-struct i915_params i915 __read_mostly = {
- .modeset = -1,
- .panel_ignore_lid = 1,
- .semaphores = -1,
- .lvds_channel_mode = 0,
- .panel_use_ssc = -1,
- .vbt_sdvo_panel_type = -1,
- .enable_rc6 = -1,
- .enable_dc = -1,
- .enable_fbc = -1,
- .enable_execlists = -1,
- .enable_hangcheck = true,
- .enable_ppgtt = -1,
- .enable_psr = -1,
- .alpha_support = IS_ENABLED(CONFIG_DRM_I915_ALPHA_SUPPORT),
- .disable_power_well = -1,
- .enable_ips = 1,
- .fastboot = 0,
- .prefault_disable = 0,
- .load_detect_test = 0,
- .force_reset_modeset_test = 0,
- .reset = 2,
- .error_capture = true,
- .invert_brightness = 0,
- .disable_display = 0,
- .enable_cmd_parser = true,
- .use_mmio_flip = 0,
- .mmio_debug = 0,
- .verbose_state_checks = 1,
- .nuclear_pageflip = 0,
- .edp_vswing = 0,
- .enable_guc_loading = 0,
- .enable_guc_submission = 0,
- .guc_log_level = -1,
- .guc_firmware_path = NULL,
- .huc_firmware_path = NULL,
- .enable_dp_mst = true,
- .inject_load_failure = 0,
- .enable_dpcd_backlight = false,
- .enable_gvt = false,
+#define i915_param_named(name, T, perm, desc) \
+ module_param_named(name, i915_modparams.name, T, perm); \
+ MODULE_PARM_DESC(name, desc)
+#define i915_param_named_unsafe(name, T, perm, desc) \
+ module_param_named_unsafe(name, i915_modparams.name, T, perm); \
+ MODULE_PARM_DESC(name, desc)
+
+struct i915_params i915_modparams __read_mostly = {
+#define MEMBER(T, member, value) .member = (value),
+ I915_PARAMS_FOR_EACH(MEMBER)
+#undef MEMBER
};
-module_param_named(modeset, i915.modeset, int, 0400);
-MODULE_PARM_DESC(modeset,
+i915_param_named(modeset, int, 0400,
"Use kernel modesetting [KMS] (0=disable, "
"1=on, -1=force vga console preference [default])");
-module_param_named_unsafe(panel_ignore_lid, i915.panel_ignore_lid, int, 0600);
-MODULE_PARM_DESC(panel_ignore_lid,
+i915_param_named_unsafe(panel_ignore_lid, int, 0600,
"Override lid status (0=autodetect, 1=autodetect disabled [default], "
"-1=force lid closed, -2=force lid open)");
-module_param_named_unsafe(semaphores, i915.semaphores, int, 0400);
-MODULE_PARM_DESC(semaphores,
+i915_param_named_unsafe(semaphores, int, 0400,
"Use semaphores for inter-ring sync "
"(default: -1 (use per-chip defaults))");
-module_param_named_unsafe(enable_rc6, i915.enable_rc6, int, 0400);
-MODULE_PARM_DESC(enable_rc6,
+i915_param_named_unsafe(enable_rc6, int, 0400,
"Enable power-saving render C-state 6. "
"Different stages can be selected via bitmask values "
"(0 = disable; 1 = enable rc6; 2 = enable deep rc6; 4 = enable deepest rc6). "
"For example, 3 would enable rc6 and deep rc6, and 7 would enable everything. "
"default: -1 (use per-chip default)");
-module_param_named_unsafe(enable_dc, i915.enable_dc, int, 0400);
-MODULE_PARM_DESC(enable_dc,
+i915_param_named_unsafe(enable_dc, int, 0400,
"Enable power-saving display C-states. "
"(-1=auto [default]; 0=disable; 1=up to DC5; 2=up to DC6)");
-module_param_named_unsafe(enable_fbc, i915.enable_fbc, int, 0600);
-MODULE_PARM_DESC(enable_fbc,
+i915_param_named_unsafe(enable_fbc, int, 0600,
"Enable frame buffer compression for power savings "
"(default: -1 (use per-chip default))");
-module_param_named_unsafe(lvds_channel_mode, i915.lvds_channel_mode, int, 0400);
-MODULE_PARM_DESC(lvds_channel_mode,
+i915_param_named_unsafe(lvds_channel_mode, int, 0400,
"Specify LVDS channel mode "
"(0=probe BIOS [default], 1=single-channel, 2=dual-channel)");
-module_param_named_unsafe(lvds_use_ssc, i915.panel_use_ssc, int, 0600);
-MODULE_PARM_DESC(lvds_use_ssc,
+i915_param_named_unsafe(panel_use_ssc, int, 0600,
"Use Spread Spectrum Clock with panels [LVDS/eDP] "
"(default: auto from VBT)");
-module_param_named_unsafe(vbt_sdvo_panel_type, i915.vbt_sdvo_panel_type, int, 0400);
-MODULE_PARM_DESC(vbt_sdvo_panel_type,
+i915_param_named_unsafe(vbt_sdvo_panel_type, int, 0400,
"Override/Ignore selection of SDVO panel mode in the VBT "
"(-2=ignore, -1=auto [default], index in VBT BIOS table)");
-module_param_named_unsafe(reset, i915.reset, int, 0600);
-MODULE_PARM_DESC(reset, "Attempt GPU resets (0=disabled, 1=full gpu reset, 2=engine reset [default])");
+i915_param_named_unsafe(reset, int, 0600,
+ "Attempt GPU resets (0=disabled, 1=full gpu reset, 2=engine reset [default])");
-module_param_named_unsafe(vbt_firmware, i915.vbt_firmware, charp, 0400);
-MODULE_PARM_DESC(vbt_firmware,
- "Load VBT from specified file under /lib/firmware");
+i915_param_named_unsafe(vbt_firmware, charp, 0400,
+ "Load VBT from specified file under /lib/firmware");
#if IS_ENABLED(CONFIG_DRM_I915_CAPTURE_ERROR)
-module_param_named(error_capture, i915.error_capture, bool, 0600);
-MODULE_PARM_DESC(error_capture,
+i915_param_named(error_capture, bool, 0600,
"Record the GPU state following a hang. "
"This information in /sys/class/drm/card<N>/error is vital for "
"triaging and debugging hangs.");
#endif
-module_param_named_unsafe(enable_hangcheck, i915.enable_hangcheck, bool, 0644);
-MODULE_PARM_DESC(enable_hangcheck,
+i915_param_named_unsafe(enable_hangcheck, bool, 0644,
"Periodically check GPU activity for detecting hangs. "
"WARNING: Disabling this can cause system wide hangs. "
"(default: true)");
-module_param_named_unsafe(enable_ppgtt, i915.enable_ppgtt, int, 0400);
-MODULE_PARM_DESC(enable_ppgtt,
+i915_param_named_unsafe(enable_ppgtt, int, 0400,
"Override PPGTT usage. "
"(-1=auto [default], 0=disabled, 1=aliasing, 2=full, 3=full with extended address space)");
-module_param_named_unsafe(enable_execlists, i915.enable_execlists, int, 0400);
-MODULE_PARM_DESC(enable_execlists,
+i915_param_named_unsafe(enable_execlists, int, 0400,
"Override execlists usage. "
"(-1=auto [default], 0=disabled, 1=enabled)");
-module_param_named_unsafe(enable_psr, i915.enable_psr, int, 0600);
-MODULE_PARM_DESC(enable_psr, "Enable PSR "
- "(0=disabled, 1=enabled - link mode chosen per-platform, 2=force link-standby mode, 3=force link-off mode) "
- "Default: -1 (use per-chip default)");
+i915_param_named_unsafe(enable_psr, int, 0600,
+ "Enable PSR "
+ "(0=disabled, 1=enabled - link mode chosen per-platform, 2=force link-standby mode, 3=force link-off mode) "
+ "Default: -1 (use per-chip default)");
-module_param_named_unsafe(alpha_support, i915.alpha_support, bool, 0400);
-MODULE_PARM_DESC(alpha_support,
+i915_param_named_unsafe(alpha_support, bool, 0400,
"Enable alpha quality driver support for latest hardware. "
"See also CONFIG_DRM_I915_ALPHA_SUPPORT.");
-module_param_named_unsafe(disable_power_well, i915.disable_power_well, int, 0400);
-MODULE_PARM_DESC(disable_power_well,
+i915_param_named_unsafe(disable_power_well, int, 0400,
"Disable display power wells when possible "
"(-1=auto [default], 0=power wells always on, 1=power wells disabled when possible)");
-module_param_named_unsafe(enable_ips, i915.enable_ips, int, 0600);
-MODULE_PARM_DESC(enable_ips, "Enable IPS (default: true)");
+i915_param_named_unsafe(enable_ips, int, 0600, "Enable IPS (default: true)");
-module_param_named(fastboot, i915.fastboot, bool, 0600);
-MODULE_PARM_DESC(fastboot,
+i915_param_named(fastboot, bool, 0600,
"Try to skip unnecessary mode sets at boot time (default: false)");
-module_param_named_unsafe(prefault_disable, i915.prefault_disable, bool, 0600);
-MODULE_PARM_DESC(prefault_disable,
+i915_param_named_unsafe(prefault_disable, bool, 0600,
"Disable page prefaulting for pread/pwrite/reloc (default:false). "
"For developers only.");
-module_param_named_unsafe(load_detect_test, i915.load_detect_test, bool, 0600);
-MODULE_PARM_DESC(load_detect_test,
+i915_param_named_unsafe(load_detect_test, bool, 0600,
"Force-enable the VGA load detect code for testing (default:false). "
"For developers only.");
-module_param_named_unsafe(force_reset_modeset_test, i915.force_reset_modeset_test, bool, 0600);
-MODULE_PARM_DESC(force_reset_modeset_test,
+i915_param_named_unsafe(force_reset_modeset_test, bool, 0600,
"Force a modeset during gpu reset for testing (default:false). "
"For developers only.");
-module_param_named_unsafe(invert_brightness, i915.invert_brightness, int, 0600);
-MODULE_PARM_DESC(invert_brightness,
+i915_param_named_unsafe(invert_brightness, int, 0600,
"Invert backlight brightness "
"(-1 force normal, 0 machine defaults, 1 force inversion), please "
"report PCI device ID, subsystem vendor and subsystem device ID "
"to dri-devel@lists.freedesktop.org, if your machine needs it. "
"It will then be included in an upcoming module version.");
-module_param_named(disable_display, i915.disable_display, bool, 0400);
-MODULE_PARM_DESC(disable_display, "Disable display (default: false)");
+i915_param_named(disable_display, bool, 0400,
+ "Disable display (default: false)");
-module_param_named_unsafe(enable_cmd_parser, i915.enable_cmd_parser, bool, 0400);
-MODULE_PARM_DESC(enable_cmd_parser,
- "Enable command parsing (true=enabled [default], false=disabled)");
+i915_param_named_unsafe(enable_cmd_parser, bool, 0400,
+ "Enable command parsing (true=enabled [default], false=disabled)");
-module_param_named_unsafe(use_mmio_flip, i915.use_mmio_flip, int, 0600);
-MODULE_PARM_DESC(use_mmio_flip,
- "use MMIO flips (-1=never, 0=driver discretion [default], 1=always)");
+i915_param_named_unsafe(use_mmio_flip, int, 0600,
+ "use MMIO flips (-1=never, 0=driver discretion [default], 1=always)");
-module_param_named(mmio_debug, i915.mmio_debug, int, 0600);
-MODULE_PARM_DESC(mmio_debug,
+i915_param_named(mmio_debug, int, 0600,
"Enable the MMIO debug code for the first N failures (default: off). "
"This may negatively affect performance.");
-module_param_named(verbose_state_checks, i915.verbose_state_checks, bool, 0600);
-MODULE_PARM_DESC(verbose_state_checks,
+i915_param_named(verbose_state_checks, bool, 0600,
"Enable verbose logs (ie. WARN_ON()) in case of unexpected hw state conditions.");
-module_param_named_unsafe(nuclear_pageflip, i915.nuclear_pageflip, bool, 0400);
-MODULE_PARM_DESC(nuclear_pageflip,
- "Force enable atomic functionality on platforms that don't have full support yet.");
+i915_param_named_unsafe(nuclear_pageflip, bool, 0400,
+ "Force enable atomic functionality on platforms that don't have full support yet.");
/* WA to get away with the default setting in VBT for early platforms.Will be removed */
-module_param_named_unsafe(edp_vswing, i915.edp_vswing, int, 0400);
-MODULE_PARM_DESC(edp_vswing,
- "Ignore/Override vswing pre-emph table selection from VBT "
- "(0=use value from vbt [default], 1=low power swing(200mV),"
- "2=default swing(400mV))");
-
-module_param_named_unsafe(enable_guc_loading, i915.enable_guc_loading, int, 0400);
-MODULE_PARM_DESC(enable_guc_loading,
- "Enable GuC firmware loading "
- "(-1=auto, 0=never [default], 1=if available, 2=required)");
-
-module_param_named_unsafe(enable_guc_submission, i915.enable_guc_submission, int, 0400);
-MODULE_PARM_DESC(enable_guc_submission,
- "Enable GuC submission "
- "(-1=auto, 0=never [default], 1=if available, 2=required)");
-
-module_param_named(guc_log_level, i915.guc_log_level, int, 0400);
-MODULE_PARM_DESC(guc_log_level,
+i915_param_named_unsafe(edp_vswing, int, 0400,
+ "Ignore/Override vswing pre-emph table selection from VBT "
+ "(0=use value from vbt [default], 1=low power swing(200mV),"
+ "2=default swing(400mV))");
+
+i915_param_named_unsafe(enable_guc_loading, int, 0400,
+ "Enable GuC firmware loading "
+ "(-1=auto, 0=never [default], 1=if available, 2=required)");
+
+i915_param_named_unsafe(enable_guc_submission, int, 0400,
+ "Enable GuC submission "
+ "(-1=auto, 0=never [default], 1=if available, 2=required)");
+
+i915_param_named(guc_log_level, int, 0400,
"GuC firmware logging level (-1:disabled (default), 0-3:enabled)");
-module_param_named_unsafe(guc_firmware_path, i915.guc_firmware_path, charp, 0400);
-MODULE_PARM_DESC(guc_firmware_path,
+i915_param_named_unsafe(guc_firmware_path, charp, 0400,
"GuC firmware path to use instead of the default one");
-module_param_named_unsafe(huc_firmware_path, i915.huc_firmware_path, charp, 0400);
-MODULE_PARM_DESC(huc_firmware_path,
+i915_param_named_unsafe(huc_firmware_path, charp, 0400,
"HuC firmware path to use instead of the default one");
-module_param_named_unsafe(enable_dp_mst, i915.enable_dp_mst, bool, 0600);
-MODULE_PARM_DESC(enable_dp_mst,
+i915_param_named_unsafe(enable_dp_mst, bool, 0600,
"Enable multi-stream transport (MST) for new DisplayPort sinks. (default: true)");
-module_param_named_unsafe(inject_load_failure, i915.inject_load_failure, uint, 0400);
-MODULE_PARM_DESC(inject_load_failure,
+
+i915_param_named_unsafe(inject_load_failure, uint, 0400,
"Force an error after a number of failure check points (0:disabled (default), N:force failure at the Nth failure check point)");
-module_param_named(enable_dpcd_backlight, i915.enable_dpcd_backlight, bool, 0600);
-MODULE_PARM_DESC(enable_dpcd_backlight,
+
+i915_param_named(enable_dpcd_backlight, bool, 0600,
"Enable support for DPCD backlight control (default:false)");
-module_param_named(enable_gvt, i915.enable_gvt, bool, 0400);
-MODULE_PARM_DESC(enable_gvt,
+i915_param_named(enable_gvt, bool, 0400,
"Enable support for Intel GVT-g graphics virtualization host support(default:false)");
#include <linux/cache.h> /* for __read_mostly */
-#define I915_PARAMS_FOR_EACH(func) \
- func(char *, vbt_firmware); \
- func(int, modeset); \
- func(int, panel_ignore_lid); \
- func(int, semaphores); \
- func(int, lvds_channel_mode); \
- func(int, panel_use_ssc); \
- func(int, vbt_sdvo_panel_type); \
- func(int, enable_rc6); \
- func(int, enable_dc); \
- func(int, enable_fbc); \
- func(int, enable_ppgtt); \
- func(int, enable_execlists); \
- func(int, enable_psr); \
- func(int, disable_power_well); \
- func(int, enable_ips); \
- func(int, invert_brightness); \
- func(int, enable_guc_loading); \
- func(int, enable_guc_submission); \
- func(int, guc_log_level); \
- func(char *, guc_firmware_path); \
- func(char *, huc_firmware_path); \
- func(int, use_mmio_flip); \
- func(int, mmio_debug); \
- func(int, edp_vswing); \
- func(int, reset); \
- func(unsigned int, inject_load_failure); \
+#define I915_PARAMS_FOR_EACH(param) \
+ param(char *, vbt_firmware, NULL) \
+ param(int, modeset, -1) \
+ param(int, panel_ignore_lid, 1) \
+ param(int, semaphores, -1) \
+ param(int, lvds_channel_mode, 0) \
+ param(int, panel_use_ssc, -1) \
+ param(int, vbt_sdvo_panel_type, -1) \
+ param(int, enable_rc6, -1) \
+ param(int, enable_dc, -1) \
+ param(int, enable_fbc, -1) \
+ param(int, enable_ppgtt, -1) \
+ param(int, enable_execlists, -1) \
+ param(int, enable_psr, -1) \
+ param(int, disable_power_well, -1) \
+ param(int, enable_ips, 1) \
+ param(int, invert_brightness, 0) \
+ param(int, enable_guc_loading, 0) \
+ param(int, enable_guc_submission, 0) \
+ param(int, guc_log_level, -1) \
+ param(char *, guc_firmware_path, NULL) \
+ param(char *, huc_firmware_path, NULL) \
+ param(int, use_mmio_flip, 0) \
+ param(int, mmio_debug, 0) \
+ param(int, edp_vswing, 0) \
+ param(int, reset, 2) \
+ param(unsigned int, inject_load_failure, 0) \
/* leave bools at the end to not create holes */ \
- func(bool, alpha_support); \
- func(bool, enable_cmd_parser); \
- func(bool, enable_hangcheck); \
- func(bool, fastboot); \
- func(bool, prefault_disable); \
- func(bool, load_detect_test); \
- func(bool, force_reset_modeset_test); \
- func(bool, error_capture); \
- func(bool, disable_display); \
- func(bool, verbose_state_checks); \
- func(bool, nuclear_pageflip); \
- func(bool, enable_dp_mst); \
- func(bool, enable_dpcd_backlight); \
- func(bool, enable_gvt)
+ param(bool, alpha_support, IS_ENABLED(CONFIG_DRM_I915_ALPHA_SUPPORT)) \
+ param(bool, enable_cmd_parser, true) \
+ param(bool, enable_hangcheck, true) \
+ param(bool, fastboot, false) \
+ param(bool, prefault_disable, false) \
+ param(bool, load_detect_test, false) \
+ param(bool, force_reset_modeset_test, false) \
+ param(bool, error_capture, true) \
+ param(bool, disable_display, false) \
+ param(bool, verbose_state_checks, true) \
+ param(bool, nuclear_pageflip, false) \
+ param(bool, enable_dp_mst, true) \
+ param(bool, enable_dpcd_backlight, false) \
+ param(bool, enable_gvt, false)
-#define MEMBER(T, member) T member
+#define MEMBER(T, member, ...) T member;
struct i915_params {
I915_PARAMS_FOR_EACH(MEMBER);
};
#undef MEMBER
-extern struct i915_params i915 __read_mostly;
+extern struct i915_params i915_modparams __read_mostly;
#endif
.platform = INTEL_I965G,
.has_overlay = 1,
.hws_needs_physical = 1,
+ .has_snoop = false,
};
static const struct intel_device_info intel_i965gm_info __initconst = {
.has_overlay = 1,
.supports_tv = 1,
.hws_needs_physical = 1,
+ .has_snoop = false,
};
static const struct intel_device_info intel_g45_info __initconst = {
#define GEN5_FEATURES \
.gen = 5, .num_pipes = 2, \
.has_hotplug = 1, \
- .has_gmbus_irq = 1, \
.ring_mask = RENDER_RING | BSD_RING, \
.has_snoop = true, \
GEN_DEFAULT_PIPEOFFSETS, \
.has_llc = 1, \
.has_rc6 = 1, \
.has_rc6p = 1, \
- .has_gmbus_irq = 1, \
.has_aliasing_ppgtt = 1, \
GEN_DEFAULT_PIPEOFFSETS, \
CURSOR_OFFSETS
.has_llc = 1, \
.has_rc6 = 1, \
.has_rc6p = 1, \
- .has_gmbus_irq = 1, \
.has_aliasing_ppgtt = 1, \
.has_full_ppgtt = 1, \
GEN_DEFAULT_PIPEOFFSETS, \
.has_psr = 1,
.has_runtime_pm = 1,
.has_rc6 = 1,
- .has_gmbus_irq = 1,
.has_gmch_display = 1,
.has_hotplug = 1,
.has_aliasing_ppgtt = 1,
.has_runtime_pm = 1,
.has_resource_streamer = 1,
.has_rc6 = 1,
- .has_gmbus_irq = 1,
.has_logical_ring_contexts = 1,
.has_gmch_display = 1,
.has_aliasing_ppgtt = 1,
.has_resource_streamer = 1, \
.has_rc6 = 1, \
.has_dp_mst = 1, \
- .has_gmbus_irq = 1, \
.has_logical_ring_contexts = 1, \
.has_guc = 1, \
.has_aliasing_ppgtt = 1, \
.has_full_48bit_ppgtt = 1, \
.has_reset_engine = 1, \
.has_snoop = true, \
+ .has_ipc = 1, \
GEN_DEFAULT_PIPEOFFSETS, \
IVB_CURSOR_OFFSETS, \
BDW_COLORS
.platform = INTEL_KABYLAKE, \
.has_csr = 1, \
.has_guc = 1, \
+ .has_ipc = 1, \
.ddb_size = 896
static const struct intel_device_info intel_kabylake_gt1_info __initconst = {
};
#define CFL_PLATFORM \
- .is_alpha_support = 1, \
BDW_FEATURES, \
.gen = 9, \
.platform = INTEL_COFFEELAKE, \
.has_csr = 1, \
.has_guc = 1, \
+ .has_ipc = 1, \
.ddb_size = 896
static const struct intel_device_info intel_coffeelake_gt1_info __initconst = {
.gt = 2,
.ddb_size = 1024,
.has_csr = 1,
+ .has_ipc = 1,
.color = { .degamma_lut_size = 0, .gamma_lut_size = 1024 }
};
(struct intel_device_info *) ent->driver_data;
int err;
- if (IS_ALPHA_SUPPORT(intel_info) && !i915.alpha_support) {
+ if (IS_ALPHA_SUPPORT(intel_info) && !i915_modparams.alpha_support) {
DRM_INFO("The driver support for your hardware in this kernel version is alpha quality\n"
"See CONFIG_DRM_I915_ALPHA_SUPPORT or i915.alpha_support module parameter\n"
"to enable support in this kernel version, or check for kernel updates.\n");
* vga_text_mode_force boot option.
*/
- if (i915.modeset == 0)
+ if (i915_modparams.modeset == 0)
use_kms = false;
- if (vgacon_text_force() && i915.modeset == -1)
+ if (vgacon_text_force() && i915_modparams.modeset == -1)
use_kms = false;
if (!use_kms) {
#include "i915_oa_kblgt2.h"
#include "i915_oa_kblgt3.h"
#include "i915_oa_glk.h"
+#include "i915_oa_cflgt2.h"
/* HW requires this to be a power of two, between 128k and 16M, though driver
* is currently generally designed assuming the largest 16M size is used such
{
struct drm_i915_private *dev_priv = stream->dev_priv;
- if (i915.enable_execlists)
+ if (i915_modparams.enable_execlists)
dev_priv->perf.oa.specific_ctx_id = stream->ctx->hw_id;
else {
struct intel_engine_cs *engine = dev_priv->engine[RCS];
{
struct drm_i915_private *dev_priv = stream->dev_priv;
- if (i915.enable_execlists) {
+ if (i915_modparams.enable_execlists) {
dev_priv->perf.oa.specific_ctx_id = INVALID_CTX_ID;
} else {
struct intel_engine_cs *engine = dev_priv->engine[RCS];
* be read back from automatically triggered reports, as part of the
* RPT_ID field.
*/
- if (IS_SKYLAKE(dev_priv) || IS_BROXTON(dev_priv) ||
- IS_KABYLAKE(dev_priv) || IS_GEMINILAKE(dev_priv)) {
+ if (IS_GEN9(dev_priv)) {
I915_WRITE(GEN8_OA_DEBUG,
_MASKED_BIT_ENABLE(GEN9_OA_DEBUG_DISABLE_CLK_RATIO_REPORTS |
GEN9_OA_DEBUG_INCLUDE_CLK_RATIO));
i915_perf_load_test_config_kblgt3(dev_priv);
} else if (IS_GEMINILAKE(dev_priv)) {
i915_perf_load_test_config_glk(dev_priv);
+ } else if (IS_COFFEELAKE(dev_priv)) {
+ if (IS_CFL_GT2(dev_priv))
+ i915_perf_load_test_config_cflgt2(dev_priv);
}
if (dev_priv->perf.oa.test_config.id == 0)
dev_priv->perf.oa.timestamp_frequency = 12500000;
dev_priv->perf.oa.oa_formats = hsw_oa_formats;
- } else if (i915.enable_execlists) {
+ } else if (i915_modparams.enable_execlists) {
/* Note: that although we could theoretically also support the
* legacy ringbuffer mode on BDW (and earlier iterations of
* this driver, before upstreaming did this) it didn't seem
break;
case INTEL_SKYLAKE:
case INTEL_KABYLAKE:
+ case INTEL_COFFEELAKE:
dev_priv->perf.oa.timestamp_frequency = 12000000;
break;
default:
#define DONE_REG _MMIO(0x40b0)
#define GEN8_PRIVATE_PAT_LO _MMIO(0x40e0)
#define GEN8_PRIVATE_PAT_HI _MMIO(0x40e0 + 4)
-#define GEN10_PAT_INDEX(index) _MMIO(0x40e0 + index*4)
+#define GEN10_PAT_INDEX(index) _MMIO(0x40e0 + (index)*4)
#define BSD_HWS_PGA_GEN7 _MMIO(0x04180)
#define BLT_HWS_PGA_GEN7 _MMIO(0x04280)
#define VEBOX_HWS_PGA_GEN7 _MMIO(0x04380)
#define GEN9_F2_SS_DIS_SHIFT 20
#define GEN9_F2_SS_DIS_MASK (0xf << GEN9_F2_SS_DIS_SHIFT)
+#define GEN10_F2_S_ENA_SHIFT 22
+#define GEN10_F2_S_ENA_MASK (0x3f << GEN10_F2_S_ENA_SHIFT)
+#define GEN10_F2_SS_DIS_SHIFT 18
+#define GEN10_F2_SS_DIS_MASK (0xf << GEN10_F2_SS_DIS_SHIFT)
+
#define GEN8_EU_DISABLE0 _MMIO(0x9134)
#define GEN8_EU_DIS0_S0_MASK 0xffffff
#define GEN8_EU_DIS0_S1_SHIFT 24
#define GEN9_EU_DISABLE(slice) _MMIO(0x9134 + (slice)*0x4)
+#define GEN10_EU_DISABLE3 _MMIO(0x9140)
+#define GEN10_EU_DIS_SS_MASK 0xff
+
#define GEN6_BSD_SLEEP_PSMI_CONTROL _MMIO(0x12050)
#define GEN6_BSD_SLEEP_MSG_DISABLE (1 << 0)
#define GEN6_BSD_SLEEP_FLUSH_DISABLE (1 << 2)
# define CHICKEN3_DGMG_DONE_FIX_DISABLE (1 << 2)
#define CHICKEN_PAR1_1 _MMIO(0x42080)
-#define SKL_RC_HASH_OUTSIDE (1 << 15)
+#define SKL_DE_COMPRESSED_HASH_MODE (1 << 15)
#define DPA_MASK_VBLANK_SRD (1 << 15)
#define FORCE_ARB_IDLE_PLANES (1 << 14)
#define SKL_EDP_PSR_FIX_RDWRAP (1 << 3)
#define DISP_FBC_WM_DIS (1<<15)
#define DISP_ARB_CTL2 _MMIO(0x45004)
#define DISP_DATA_PARTITION_5_6 (1<<6)
+#define DISP_IPC_ENABLE (1<<3)
#define DBUF_CTL _MMIO(0x45008)
#define DBUF_POWER_REQUEST (1<<31)
#define DBUF_POWER_STATE (1<<30)
# define GEN7_CSC1_RHWO_OPT_DISABLE_IN_RCC ((1<<10) | (1<<26))
# define GEN9_RHWO_OPTIMIZATION_DISABLE (1<<14)
#define COMMON_SLICE_CHICKEN2 _MMIO(0x7014)
+# define GEN9_PBE_COMPRESSED_HASH_SELECTION (1<<13)
# define GEN9_DISABLE_GATHER_AT_SET_SHADER_COMMON_SLICE (1<<12)
# define GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION (1<<8)
# define GEN8_CSC2_SBE_VUE_CACHE_CONSERVATIVE (1<<0)
#define FDI_PHASE_SYNC_OVR(pipe) (1<<(FDIA_PHASE_SYNC_SHIFT_OVR - ((pipe) * 2)))
#define FDI_PHASE_SYNC_EN(pipe) (1<<(FDIA_PHASE_SYNC_SHIFT_EN - ((pipe) * 2)))
#define FDI_BC_BIFURCATION_SELECT (1 << 12)
+#define CHASSIS_CLK_REQ_DURATION_MASK (0xf << 8)
+#define CHASSIS_CLK_REQ_DURATION(x) ((x) << 8)
#define SPT_PWM_GRANULARITY (1<<0)
#define SOUTH_CHICKEN2 _MMIO(0xc2004)
#define FDI_MPHY_IOSFSB_RESET_STATUS (1<<13)
#define GEN7_PCODE_TIMEOUT 0x2
#define GEN7_PCODE_ILLEGAL_DATA 0x3
#define GEN7_PCODE_MIN_FREQ_TABLE_GT_RATIO_OUT_OF_RANGE 0x10
-#define GEN6_PCODE_WRITE_RC6VIDS 0x4
-#define GEN6_PCODE_READ_RC6VIDS 0x5
+#define GEN6_PCODE_WRITE_RC6VIDS 0x4
+#define GEN6_PCODE_READ_RC6VIDS 0x5
#define GEN6_ENCODE_RC6_VID(mv) (((mv) - 245) / 5)
#define GEN6_DECODE_RC6_VID(vids) (((vids) * 5) + 245)
#define BDW_PCODE_DISPLAY_FREQ_CHANGE_REQ 0x18
#define GEN6_PCODE_WRITE_D_COMP 0x11
#define HSW_PCODE_DE_WRITE_FREQ_REQ 0x17
#define DISPLAY_IPS_CONTROL 0x19
-#define HSW_PCODE_DYNAMIC_DUTY_CYCLE_CONTROL 0x1A
+ /* See also IPS_CTL */
+#define IPS_PCODE_CONTROL (1 << 30)
+#define HSW_PCODE_DYNAMIC_DUTY_CYCLE_CONTROL 0x1A
#define GEN9_PCODE_SAGV_CONTROL 0x21
#define GEN9_SAGV_DISABLE 0x0
#define GEN9_SAGV_IS_DISABLED 0x1
#define GEN8_SAMPLER_POWER_BYPASS_DIS (1<<1)
#define GEN9_HALF_SLICE_CHICKEN7 _MMIO(0xe194)
+#define GEN9_SAMPLER_HASH_COMPRESSED_READ_ADDR (1<<8)
#define GEN9_ENABLE_YV12_BUGFIX (1<<4)
#define GEN9_ENABLE_GPGPU_PREEMPTION (1<<2)
#define DPLL_CFGCR0_LINK_RATE_3240 (6 << 25)
#define DPLL_CFGCR0_LINK_RATE_4050 (7 << 25)
#define DPLL_CFGCR0_DCO_FRACTION_MASK (0x7fff << 10)
-#define DPLL_CFGCR0_DCO_FRAC_SHIFT (10)
+#define DPLL_CFGCR0_DCO_FRACTION_SHIFT (10)
#define DPLL_CFGCR0_DCO_FRACTION(x) ((x) << 10)
#define DPLL_CFGCR0_DCO_INTEGER_MASK (0x3ff)
#define CNL_DPLL_CFGCR0(pll) _MMIO_PLL(pll, _CNL_DPLL0_CFGCR0, _CNL_DPLL1_CFGCR0)
#define MIPIO_TXESC_CLK_DIV2 _MMIO(0x160008)
#define GLK_TX_ESC_CLK_DIV2_MASK 0x3FF
+/* Gen4+ Timestamp and Pipe Frame time stamp registers */
+#define GEN4_TIMESTAMP _MMIO(0x2358)
+#define ILK_TIMESTAMP_HI _MMIO(0x70070)
+#define IVB_TIMESTAMP_CTR _MMIO(0x44070)
+
+#define _PIPE_FRMTMSTMP_A 0x70048
+#define PIPE_FRMTMSTMP(pipe) \
+ _MMIO_PIPE2(pipe, _PIPE_FRMTMSTMP_A)
+
/* BXT MIPI clock controls */
#define BXT_MAX_VAR_OUTPUT_KHZ 39500
#define GEN9_L3_LRA_1_GPGPU_DEFAULT_VALUE_SKL 0x67F1427F /* " " */
#define GEN9_L3_LRA_1_GPGPU_DEFAULT_VALUE_BXT 0x5FF101FF /* " " */
+#define MMCD_MISC_CTRL _MMIO(0x4ddc) /* skl+ */
+#define MMCD_PCLA (1 << 31)
+#define MMCD_HOTSPOT_EN (1 << 27)
+
#endif /* _I915_REG_H_ */
#include <linux/slab.h>
#include <linux/dma-fence.h>
+#include <linux/irq_work.h>
#include <linux/reservation.h>
#include "i915_sw_fence.h"
struct i915_sw_fence *fence;
struct dma_fence *dma;
struct timer_list timer;
+ struct irq_work work;
};
static void timer_i915_sw_fence_wake(unsigned long data)
{
struct i915_sw_dma_fence_cb *cb = (struct i915_sw_dma_fence_cb *)data;
+ struct i915_sw_fence *fence;
+
+ fence = xchg(&cb->fence, NULL);
+ if (!fence)
+ return;
pr_warn("asynchronous wait on fence %s:%s:%x timed out\n",
cb->dma->ops->get_driver_name(cb->dma),
cb->dma->ops->get_timeline_name(cb->dma),
cb->dma->seqno);
- dma_fence_put(cb->dma);
- cb->dma = NULL;
- i915_sw_fence_complete(cb->fence);
- cb->timer.function = NULL;
+ i915_sw_fence_complete(fence);
}
static void dma_i915_sw_fence_wake(struct dma_fence *dma,
struct dma_fence_cb *data)
{
struct i915_sw_dma_fence_cb *cb = container_of(data, typeof(*cb), base);
+ struct i915_sw_fence *fence;
+
+ fence = xchg(&cb->fence, NULL);
+ if (fence)
+ i915_sw_fence_complete(fence);
+
+ irq_work_queue(&cb->work);
+}
+
+static void irq_i915_sw_fence_work(struct irq_work *wrk)
+{
+ struct i915_sw_dma_fence_cb *cb = container_of(wrk, typeof(*cb), work);
del_timer_sync(&cb->timer);
- if (cb->timer.function)
- i915_sw_fence_complete(cb->fence);
dma_fence_put(cb->dma);
kfree(cb);
__setup_timer(&cb->timer,
timer_i915_sw_fence_wake, (unsigned long)cb,
TIMER_IRQSAFE);
+ init_irq_work(&cb->work, irq_i915_sw_fence_work);
if (timeout) {
cb->dma = dma_fence_get(dma);
mod_timer(&cb->timer, round_jiffies_up(jiffies + timeout));
connector->encoder->base.id,
connector->encoder->name);
- /* ELD Conn_Type */
- connector->eld[5] &= ~(3 << 2);
- if (intel_crtc_has_dp_encoder(crtc_state))
- connector->eld[5] |= (1 << 2);
-
connector->eld[6] = drm_av_sync_delay(connector, adjusted_mode) / 2;
if (dev_priv->display.audio_codec_enable)
struct drm_display_mode *panel_fixed_mode;
int index;
- index = i915.vbt_sdvo_panel_type;
+ index = i915_modparams.vbt_sdvo_panel_type;
if (index == -2) {
DRM_DEBUG_KMS("Ignore SDVO panel mode from BIOS VBT tables.\n");
return;
uint8_t vswing;
/* Don't read from VBT if module parameter has valid value*/
- if (i915.edp_vswing) {
- dev_priv->vbt.edp.low_vswing = i915.edp_vswing == 1;
+ if (i915_modparams.edp_vswing) {
+ dev_priv->vbt.edp.low_vswing =
+ i915_modparams.edp_vswing == 1;
} else {
vswing = (edp->edp_vswing_preemph >> (panel_type * 4)) & 0xF;
dev_priv->vbt.edp.low_vswing = vswing == 0;
is_hdmi = is_dvi && (child->device_type & DEVICE_TYPE_NOT_HDMI_OUTPUT) == 0;
is_edp = is_dp && (child->device_type & DEVICE_TYPE_INTERNAL_CONNECTOR);
+ if (port == PORT_A && is_dvi) {
+ DRM_DEBUG_KMS("VBT claims port A supports DVI%s, ignoring\n",
+ is_hdmi ? "/HDMI" : "");
+ is_dvi = false;
+ is_hdmi = false;
+ }
+
info->supports_dvi = is_dvi;
info->supports_hdmi = is_hdmi;
info->supports_dp = is_dp;
static noinline void missed_breadcrumb(struct intel_engine_cs *engine)
{
- DRM_DEBUG_DRIVER("%s missed breadcrumb at %pF, irq posted? %s, current seqno=%x, last=%x\n",
+ DRM_DEBUG_DRIVER("%s missed breadcrumb at %pS, irq posted? %s, current seqno=%x, last=%x\n",
engine->name, __builtin_return_address(0),
yesno(test_bit(ENGINE_IRQ_BREADCRUMB,
&engine->irq_posted)),
val |= LCPLL_CD_SOURCE_FCLK;
I915_WRITE(LCPLL_CTL, val);
+ /*
+ * According to the spec, it should be enough to poll for this 1 us.
+ * However, extensive testing shows that this can take longer.
+ */
if (wait_for_us(I915_READ(LCPLL_CTL) &
- LCPLL_CD_SOURCE_FCLK_DONE, 1))
+ LCPLL_CD_SOURCE_FCLK_DONE, 100))
DRM_ERROR("Switching to FCLK failed\n");
val = I915_READ(LCPLL_CTL);
* broken monitor (without edid) to work behind a broken kvm (that fails
* to have the right resistors for HP detection) needs to fix this up.
* For now just bail out. */
- if (I915_HAS_HOTPLUG(dev_priv) && !i915.load_detect_test) {
+ if (I915_HAS_HOTPLUG(dev_priv) && !i915_modparams.load_detect_test) {
status = connector_status_disconnected;
goto out;
}
else if (INTEL_GEN(dev_priv) < 4)
status = intel_crt_load_detect(crt,
to_intel_crtc(connector->state->crtc)->pipe);
- else if (i915.load_detect_test)
+ else if (i915_modparams.load_detect_test)
status = connector_status_disconnected;
else
status = connector_status_unknown;
}
fw_size = dev_priv->csr.dmc_fw_size;
+ assert_rpm_wakelock_held(dev_priv);
+
+ preempt_disable();
+
for (i = 0; i < fw_size; i++)
- I915_WRITE(CSR_PROGRAM(i), payload[i]);
+ I915_WRITE_FW(CSR_PROGRAM(i), payload[i]);
+
+ preempt_enable();
for (i = 0; i < dev_priv->csr.mmio_count; i++) {
I915_WRITE(dev_priv->csr.mmioaddr[i],
};
struct bxt_ddi_buf_trans {
- u32 margin; /* swing value */
- u32 scale; /* scale value */
- u32 enable; /* scale enable */
- u32 deemphasis;
+ u8 margin; /* swing value */
+ u8 scale; /* scale value */
+ u8 enable; /* scale enable */
+ u8 deemphasis;
bool default_index; /* true if the entry represents default value */
};
};
struct cnl_ddi_buf_trans {
- u32 dw2_swing_sel;
- u32 dw7_n_scalar;
- u32 dw4_cursor_coeff;
- u32 dw4_post_cursor_2;
- u32 dw4_post_cursor_1;
+ u8 dw2_swing_sel;
+ u8 dw7_n_scalar;
+ u8 dw4_cursor_coeff;
+ u8 dw4_post_cursor_2;
+ u8 dw4_post_cursor_1;
};
/* Voltage Swing Programming for VccIO 0.85V for DP */
dco_freq = (cfgcr0 & DPLL_CFGCR0_DCO_INTEGER_MASK) * ref_clock;
dco_freq += (((cfgcr0 & DPLL_CFGCR0_DCO_FRACTION_MASK) >>
- DPLL_CFGCR0_DCO_FRAC_SHIFT) * ref_clock) / 0x8000;
+ DPLL_CFGCR0_DCO_FRACTION_SHIFT) * ref_clock) / 0x8000;
return dco_freq / (p0 * p1 * p2 * 5);
}
val |= RCOMP_SCALAR(0x98);
I915_WRITE(CNL_PORT_TX_DW2_GRP(port), val);
- /* Program PORT_TX_DW4 */
+ /* Program PORT_TX_DW4 */
/* We cannot write to GRP. It would overrite individual loadgen */
for (ln = 0; ln < 4; ln++) {
val = I915_READ(CNL_PORT_TX_DW4_LN(port, ln));
I915_WRITE(CNL_PORT_TX_DW4_LN(port, ln), val);
}
- /* Program PORT_TX_DW5 */
+ /* Program PORT_TX_DW5 */
/* All DW5 values are fixed for every table entry */
val = I915_READ(CNL_PORT_TX_DW5_LN0(port));
val &= ~RTERM_SELECT_MASK;
val |= TAP3_DISABLE;
I915_WRITE(CNL_PORT_TX_DW5_GRP(port), val);
- /* Program PORT_TX_DW7 */
+ /* Program PORT_TX_DW7 */
val = I915_READ(CNL_PORT_TX_DW7_LN0(port));
val &= ~N_SCALAR_MASK;
val |= N_SCALAR(ddi_translations[level].dw7_n_scalar);
#undef PRINT_FLAG
}
+static void gen10_sseu_info_init(struct drm_i915_private *dev_priv)
+{
+ struct sseu_dev_info *sseu = &mkwrite_device_info(dev_priv)->sseu;
+ const u32 fuse2 = I915_READ(GEN8_FUSE2);
+
+ sseu->slice_mask = (fuse2 & GEN10_F2_S_ENA_MASK) >>
+ GEN10_F2_S_ENA_SHIFT;
+ sseu->subslice_mask = (1 << 4) - 1;
+ sseu->subslice_mask &= ~((fuse2 & GEN10_F2_SS_DIS_MASK) >>
+ GEN10_F2_SS_DIS_SHIFT);
+
+ sseu->eu_total = hweight32(~I915_READ(GEN8_EU_DISABLE0));
+ sseu->eu_total += hweight32(~I915_READ(GEN8_EU_DISABLE1));
+ sseu->eu_total += hweight32(~I915_READ(GEN8_EU_DISABLE2));
+ sseu->eu_total += hweight8(~(I915_READ(GEN10_EU_DISABLE3) &
+ GEN10_EU_DIS_SS_MASK));
+
+ /*
+ * CNL is expected to always have a uniform distribution
+ * of EU across subslices with the exception that any one
+ * EU in any one subslice may be fused off for die
+ * recovery.
+ */
+ sseu->eu_per_subslice = sseu_subslice_total(sseu) ?
+ DIV_ROUND_UP(sseu->eu_total,
+ sseu_subslice_total(sseu)) : 0;
+
+ /* No restrictions on Power Gating */
+ sseu->has_slice_pg = 1;
+ sseu->has_subslice_pg = 1;
+ sseu->has_eu_pg = 1;
+}
+
static void cherryview_sseu_info_init(struct drm_i915_private *dev_priv)
{
struct sseu_dev_info *sseu = &mkwrite_device_info(dev_priv)->sseu;
info->num_sprites[pipe] = 1;
}
- if (i915.disable_display) {
+ if (i915_modparams.disable_display) {
DRM_INFO("Display disabled (module parameter)\n");
info->num_pipes = 0;
} else if (info->num_pipes > 0 &&
cherryview_sseu_info_init(dev_priv);
else if (IS_BROADWELL(dev_priv))
broadwell_sseu_info_init(dev_priv);
- else if (INTEL_INFO(dev_priv)->gen >= 9)
+ else if (INTEL_GEN(dev_priv) == 9)
gen9_sseu_info_init(dev_priv);
-
- WARN_ON(info->has_snoop != !info->has_llc);
+ else if (INTEL_GEN(dev_priv) >= 10)
+ gen10_sseu_info_init(dev_priv);
DRM_DEBUG_DRIVER("slice mask: %04x\n", info->sseu.slice_mask);
DRM_DEBUG_DRIVER("slice total: %u\n", hweight8(info->sseu.slice_mask));
/* reset doesn't touch the display */
- if (!i915.force_reset_modeset_test &&
+ if (!i915_modparams.force_reset_modeset_test &&
!gpu_reset_clobbers_display(dev_priv))
return;
int ret;
/* reset doesn't touch the display */
- if (!i915.force_reset_modeset_test &&
+ if (!i915_modparams.force_reset_modeset_test &&
!gpu_reset_clobbers_display(dev_priv))
return;
assert_plane_enabled(dev_priv, crtc->plane);
if (IS_BROADWELL(dev_priv)) {
mutex_lock(&dev_priv->rps.hw_lock);
- WARN_ON(sandybridge_pcode_write(dev_priv, DISPLAY_IPS_CONTROL, 0xc0000000));
+ WARN_ON(sandybridge_pcode_write(dev_priv, DISPLAY_IPS_CONTROL,
+ IPS_ENABLE | IPS_PCODE_CONTROL));
mutex_unlock(&dev_priv->rps.hw_lock);
/* Quoting Art Runyan: "its not safe to expect any particular
* value in IPS_CTL bit 31 after enabling IPS through the
struct drm_device *dev = crtc->base.dev;
struct drm_i915_private *dev_priv = to_i915(dev);
- pipe_config->ips_enabled = i915.enable_ips &&
+ pipe_config->ips_enabled = i915_modparams.enable_ips &&
hsw_crtc_supports_ips(crtc) &&
pipe_config_supports_ips(dev_priv, pipe_config);
}
static inline bool intel_panel_use_ssc(struct drm_i915_private *dev_priv)
{
- if (i915.panel_use_ssc >= 0)
- return i915.panel_use_ssc != 0;
+ if (i915_modparams.panel_use_ssc >= 0)
+ return i915_modparams.panel_use_ssc != 0;
return dev_priv->vbt.lvds_use_ssc
&& !(dev_priv->quirks & QUIRK_LVDS_SSC_DISABLE);
}
pipe_config->gamma_mode =
I915_READ(GAMMA_MODE(crtc->pipe)) & GAMMA_MODE_MODE_MASK;
- if (IS_BROADWELL(dev_priv) || dev_priv->info.gen >= 9) {
+ if (IS_BROADWELL(dev_priv) || INTEL_GEN(dev_priv) >= 9) {
u32 tmp = I915_READ(PIPEMISC(crtc->pipe));
bool clrspace_yuv = tmp & PIPEMISC_OUTPUT_COLORSPACE_YUV;
- if (IS_GEMINILAKE(dev_priv) || dev_priv->info.gen >= 10) {
+ if (IS_GEMINILAKE(dev_priv) || INTEL_GEN(dev_priv) >= 10) {
bool blend_mode_420 = tmp &
PIPEMISC_YUV420_MODE_FULL_BLEND;
if (!m_n->link_n)
return 0;
- return div_u64((u64)m_n->link_m * link_freq, m_n->link_n);
+ return div_u64(mul_u32_u32(m_n->link_m, link_freq), m_n->link_n);
}
static void ironlake_pch_clock_get(struct intel_crtc *crtc,
return ret;
}
- if (i915.fastboot &&
+ if (i915_modparams.fastboot &&
intel_pipe_config_compare(dev_priv,
to_intel_crtc_state(old_crtc_state),
pipe_config, true)) {
struct drm_crtc_state *old_crtc_state, *new_crtc_state;
struct drm_crtc *crtc;
struct intel_crtc_state *intel_cstate;
- bool hw_check = intel_state->modeset;
u64 put_domains[I915_MAX_PIPES] = {};
int i;
if (needs_modeset(new_crtc_state) ||
to_intel_crtc_state(new_crtc_state)->update_pipe) {
- hw_check = true;
put_domains[to_intel_crtc(crtc)->pipe] =
modeset_get_crtc_power_domains(crtc,
dev_priv->display.fdi_link_train = hsw_fdi_link_train;
}
- if (dev_priv->info.gen >= 9)
+ if (INTEL_GEN(dev_priv) >= 9)
dev_priv->display.update_crtcs = skl_update_crtcs;
else
dev_priv->display.update_crtcs = intel_update_crtcs;
if (!ret)
ret = __intel_display_resume(dev, state, &ctx);
+ intel_enable_ipc(dev_priv);
drm_modeset_drop_locks(&ctx);
drm_modeset_acquire_fini(&ctx);
#include "i915_drv.h"
#define DP_LINK_CHECK_TIMEOUT (10 * 1000)
+#define DP_DPRX_ESI_LEN 14
/* Compliance test status bits */
#define INTEL_DP_RESOLUTION_SHIFT_MASK 0
const struct drm_connector_state *old_conn_state)
{
struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base);
- struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
if (old_crtc_state->has_audio)
intel_audio_codec_disable(encoder);
- if (HAS_PSR(dev_priv) && !HAS_DDI(dev_priv))
- intel_psr_disable(intel_dp, old_crtc_state);
-
/* Make sure the panel is off before trying to change the mode. But also
* ensure that we have vdd while we switch off the panel. */
intel_edp_panel_vdd_on(intel_dp);
intel_edp_backlight_off(old_conn_state);
intel_dp_sink_dpms(intel_dp, DRM_MODE_DPMS_OFF);
intel_edp_panel_off(intel_dp);
+}
+
+static void g4x_disable_dp(struct intel_encoder *encoder,
+ const struct intel_crtc_state *old_crtc_state,
+ const struct drm_connector_state *old_conn_state)
+{
+ struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base);
+
+ intel_disable_dp(encoder, old_crtc_state, old_conn_state);
/* disable the port before the pipe on g4x */
- if (INTEL_GEN(dev_priv) < 5)
- intel_dp_link_down(intel_dp);
+ intel_dp_link_down(intel_dp);
+}
+
+static void ilk_disable_dp(struct intel_encoder *encoder,
+ const struct intel_crtc_state *old_crtc_state,
+ const struct drm_connector_state *old_conn_state)
+{
+ intel_disable_dp(encoder, old_crtc_state, old_conn_state);
+}
+
+static void vlv_disable_dp(struct intel_encoder *encoder,
+ const struct intel_crtc_state *old_crtc_state,
+ const struct drm_connector_state *old_conn_state)
+{
+ struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base);
+
+ intel_psr_disable(intel_dp, old_crtc_state);
+
+ intel_disable_dp(encoder, old_crtc_state, old_conn_state);
}
static void ilk_post_disable_dp(struct intel_encoder *encoder,
{
u8 mstm_cap;
- if (!i915.enable_dp_mst)
+ if (!i915_modparams.enable_dp_mst)
return false;
if (!intel_dp->can_mst)
static void
intel_dp_configure_mst(struct intel_dp *intel_dp)
{
- if (!i915.enable_dp_mst)
+ if (!i915_modparams.enable_dp_mst)
return;
if (!intel_dp->can_mst)
static bool
intel_dp_get_sink_irq_esi(struct intel_dp *intel_dp, u8 *sink_irq_vector)
{
- int ret;
-
- ret = drm_dp_dpcd_read(&intel_dp->aux,
- DP_SINK_COUNT_ESI,
- sink_irq_vector, 14);
- if (ret != 14)
- return false;
-
- return true;
+ return drm_dp_dpcd_read(&intel_dp->aux, DP_SINK_COUNT_ESI,
+ sink_irq_vector, DP_DPRX_ESI_LEN) ==
+ DP_DPRX_ESI_LEN;
}
static uint8_t intel_dp_autotest_link_training(struct intel_dp *intel_dp)
bool bret;
if (intel_dp->is_mst) {
- u8 esi[16] = { 0 };
+ u8 esi[DP_DPRX_ESI_LEN] = { 0 };
int ret = 0;
int retry;
bool handled;
if (intel_encoder->type != INTEL_OUTPUT_EDP)
intel_encoder->type = INTEL_OUTPUT_DP;
- DRM_DEBUG_KMS("Display Port TPS3 support: source %s, sink %s\n",
- yesno(intel_dp_source_supports_hbr2(intel_dp)),
- yesno(drm_dp_tps3_supported(intel_dp->dpcd)));
-
if (intel_dp->reset_link_params) {
/* Initial max link lane count */
intel_dp->max_link_lane_count = intel_dp_max_common_lane_count(intel_dp);
return;
}
- /*
- * FIXME: This needs proper synchronization with psr state for some
- * platforms that cannot have PSR and DRRS enabled at the same time.
- */
-
dig_port = dp_to_dig_port(intel_dp);
encoder = &dig_port->base;
intel_crtc = to_intel_crtc(encoder->base.crtc);
return;
}
+ if (dev_priv->psr.enabled) {
+ DRM_DEBUG_KMS("PSR enabled. Not enabling DRRS.\n");
+ return;
+ }
+
mutex_lock(&dev_priv->drrs.mutex);
if (WARN_ON(dev_priv->drrs.dp)) {
DRM_ERROR("DRRS already enabled\n");
goto err_encoder_init;
intel_encoder->compute_config = intel_dp_compute_config;
- intel_encoder->disable = intel_disable_dp;
intel_encoder->get_hw_state = intel_dp_get_hw_state;
intel_encoder->get_config = intel_dp_get_config;
intel_encoder->suspend = intel_dp_encoder_suspend;
intel_encoder->pre_pll_enable = chv_dp_pre_pll_enable;
intel_encoder->pre_enable = chv_pre_enable_dp;
intel_encoder->enable = vlv_enable_dp;
+ intel_encoder->disable = vlv_disable_dp;
intel_encoder->post_disable = chv_post_disable_dp;
intel_encoder->post_pll_disable = chv_dp_post_pll_disable;
} else if (IS_VALLEYVIEW(dev_priv)) {
intel_encoder->pre_pll_enable = vlv_dp_pre_pll_enable;
intel_encoder->pre_enable = vlv_pre_enable_dp;
intel_encoder->enable = vlv_enable_dp;
+ intel_encoder->disable = vlv_disable_dp;
intel_encoder->post_disable = vlv_post_disable_dp;
+ } else if (INTEL_GEN(dev_priv) >= 5) {
+ intel_encoder->pre_enable = g4x_pre_enable_dp;
+ intel_encoder->enable = g4x_enable_dp;
+ intel_encoder->disable = ilk_disable_dp;
+ intel_encoder->post_disable = ilk_post_disable_dp;
} else {
intel_encoder->pre_enable = g4x_pre_enable_dp;
intel_encoder->enable = g4x_enable_dp;
- if (INTEL_GEN(dev_priv) >= 5)
- intel_encoder->post_disable = ilk_post_disable_dp;
+ intel_encoder->disable = g4x_disable_dp;
}
intel_dig_port->port = port;
{
struct intel_panel *panel = &intel_connector->panel;
- if (!i915.enable_dpcd_backlight)
+ if (!i915_modparams.enable_dpcd_backlight)
return -ENODEV;
if (!intel_dp_aux_display_control_capable(intel_connector))
to_intel_connector(old_conn_state->connector);
int ret;
- DRM_DEBUG_KMS("%d\n", intel_dp->active_mst_links);
+ DRM_DEBUG_KMS("active links %d\n", intel_dp->active_mst_links);
drm_dp_mst_reset_vcpi_slots(&intel_dp->mst_mgr, connector->port);
struct intel_connector *connector =
to_intel_connector(old_conn_state->connector);
- DRM_DEBUG_KMS("%d\n", intel_dp->active_mst_links);
-
/* this can fail */
drm_dp_check_act_status(&intel_dp->mst_mgr);
/* and this can also fail */
intel_dp_sink_dpms(intel_dp, DRM_MODE_DPMS_OFF);
}
+ DRM_DEBUG_KMS("active links %d\n", intel_dp->active_mst_links);
}
static void intel_mst_pre_enable_dp(struct intel_encoder *encoder,
connector->encoder = encoder;
intel_mst->connector = connector;
- DRM_DEBUG_KMS("%d\n", intel_dp->active_mst_links);
+ DRM_DEBUG_KMS("active links %d\n", intel_dp->active_mst_links);
if (intel_dp->active_mst_links == 0)
intel_dig_port->base.pre_enable(&intel_dig_port->base,
enum port port = intel_dig_port->port;
int ret;
- DRM_DEBUG_KMS("%d\n", intel_dp->active_mst_links);
+ DRM_DEBUG_KMS("active links %d\n", intel_dp->active_mst_links);
if (intel_wait_for_register(dev_priv,
DP_TP_STATUS(port),
struct intel_connector *intel_connector = to_intel_connector(connector);
struct drm_i915_private *dev_priv = to_i915(connector->dev);
+ DRM_DEBUG_KMS("[CONNECTOR:%d:%s]\n", connector->base.id, connector->name);
drm_connector_unregister(connector);
if (dev_priv->fbdev)
drm_modeset_unlock(&connector->dev->mode_config.connection_mutex);
drm_connector_unreference(connector);
- DRM_DEBUG_KMS("\n");
}
static void intel_dp_mst_hotplug(struct drm_dp_mst_topology_mgr *mgr)
/* drm_mode->private_flags */
#define I915_MODE_FLAG_INHERITED 1
+/* Flag to get scanline using frame time stamps */
+#define I915_MODE_FLAG_GET_SCANLINE_FROM_TIMESTAMP (1<<1)
struct intel_pipe_wm {
struct intel_wm_level wm[5];
int sanitize_rc6_option(struct drm_i915_private *dev_priv, int enable_rc6);
int skl_check_pipe_max_pixel_rate(struct intel_crtc *intel_crtc,
struct intel_crtc_state *cstate);
+void intel_init_ipc(struct drm_i915_private *dev_priv);
+void intel_enable_ipc(struct drm_i915_private *dev_priv);
static inline int intel_enable_rc6(void)
{
- return i915.enable_rc6;
+ return i915_modparams.enable_rc6;
}
/* intel_sdvo.c */
/* XXX: old code skips write if control unchanged */
if (cmd == I915_READ(MIPI_DPI_CONTROL(port)))
- DRM_ERROR("Same special packet %02x twice in a row.\n", cmd);
+ DRM_DEBUG_KMS("Same special packet %02x twice in a row.\n", cmd);
I915_WRITE(MIPI_DPI_CONTROL(port), cmd);
adjusted_mode->flags = 0;
if (IS_GEN9_LP(dev_priv)) {
+ /* Enable Frame time stamp based scanline reporting */
+ adjusted_mode->private_flags |=
+ I915_MODE_FLAG_GET_SCANLINE_FROM_TIMESTAMP;
+
/* Dual link goes to DSI transcoder A. */
if (intel_dsi->ports == BIT(PORT_C))
pipe_config->cpu_transcoder = TRANSCODER_DSI_C;
pixel_format_from_register_bits(fmt));
bpp = pipe_config->pipe_bpp;
+ /* Enable Frame time stamo based scanline reporting */
+ adjusted_mode->private_flags |=
+ I915_MODE_FLAG_GET_SCANLINE_FROM_TIMESTAMP;
+
/* In terms of pixels */
adjusted_mode->crtc_hdisplay =
I915_READ(BXT_MIPI_TRANS_HACTIVE(port));
#define GEN8_LR_CONTEXT_RENDER_SIZE (20 * PAGE_SIZE)
#define GEN9_LR_CONTEXT_RENDER_SIZE (22 * PAGE_SIZE)
+#define GEN10_LR_CONTEXT_RENDER_SIZE (19 * PAGE_SIZE)
#define GEN8_LR_CONTEXT_OTHER_SIZE ( 2 * PAGE_SIZE)
default:
MISSING_CASE(INTEL_GEN(dev_priv));
case 10:
+ return GEN10_LR_CONTEXT_RENDER_SIZE;
case 9:
return GEN9_LR_CONTEXT_RENDER_SIZE;
case 8:
- return i915.enable_execlists ?
+ return i915_modparams.enable_execlists ?
GEN8_LR_CONTEXT_RENDER_SIZE :
GEN8_CXT_TOTAL_SIZE;
case 7:
&intel_engine_classes[engine->class];
int (*init)(struct intel_engine_cs *engine);
- if (i915.enable_execlists)
+ if (i915_modparams.enable_execlists)
init = class_info->init_execlists;
else
init = class_info->init_legacy;
engine->timeline = &engine->i915->gt.global_timeline.engine[engine->id];
}
+static bool csb_force_mmio(struct drm_i915_private *i915)
+{
+ /* GVT emulation depends upon intercepting CSB mmio */
+ if (intel_vgpu_active(i915))
+ return true;
+
+ /*
+ * IOMMU adds unpredictable latency causing the CSB write (from the
+ * GPU into the HWSP) to only be visible some time after the interrupt
+ * (missed breadcrumb syndrome).
+ */
+ if (intel_vtd_active())
+ return true;
+
+ return false;
+}
+
+static void intel_engine_init_execlist(struct intel_engine_cs *engine)
+{
+ struct intel_engine_execlists * const execlists = &engine->execlists;
+
+ execlists->csb_use_mmio = csb_force_mmio(engine->i915);
+
+ execlists->port_mask = 1;
+ BUILD_BUG_ON_NOT_POWER_OF_2(execlists_num_ports(execlists));
+ GEM_BUG_ON(execlists_num_ports(execlists) > EXECLIST_MAX_PORTS);
+
+ execlists->queue = RB_ROOT;
+ execlists->first = NULL;
+}
+
/**
* intel_engines_setup_common - setup engine state not requiring hw access
* @engine: Engine to setup.
*/
void intel_engine_setup_common(struct intel_engine_cs *engine)
{
- engine->execlist_queue = RB_ROOT;
- engine->execlist_first = NULL;
+ intel_engine_init_execlist(engine);
intel_engine_init_timeline(engine);
intel_engine_init_hangcheck(engine);
i915_vma_unpin_and_release(&engine->scratch);
}
+static void cleanup_phys_status_page(struct intel_engine_cs *engine)
+{
+ struct drm_i915_private *dev_priv = engine->i915;
+
+ if (!dev_priv->status_page_dmah)
+ return;
+
+ drm_pci_free(&dev_priv->drm, dev_priv->status_page_dmah);
+ engine->status_page.page_addr = NULL;
+}
+
+static void cleanup_status_page(struct intel_engine_cs *engine)
+{
+ struct i915_vma *vma;
+ struct drm_i915_gem_object *obj;
+
+ vma = fetch_and_zero(&engine->status_page.vma);
+ if (!vma)
+ return;
+
+ obj = vma->obj;
+
+ i915_vma_unpin(vma);
+ i915_vma_close(vma);
+
+ i915_gem_object_unpin_map(obj);
+ __i915_gem_object_release_unless_active(obj);
+}
+
+static int init_status_page(struct intel_engine_cs *engine)
+{
+ struct drm_i915_gem_object *obj;
+ struct i915_vma *vma;
+ unsigned int flags;
+ void *vaddr;
+ int ret;
+
+ obj = i915_gem_object_create_internal(engine->i915, PAGE_SIZE);
+ if (IS_ERR(obj)) {
+ DRM_ERROR("Failed to allocate status page\n");
+ return PTR_ERR(obj);
+ }
+
+ ret = i915_gem_object_set_cache_level(obj, I915_CACHE_LLC);
+ if (ret)
+ goto err;
+
+ vma = i915_vma_instance(obj, &engine->i915->ggtt.base, NULL);
+ if (IS_ERR(vma)) {
+ ret = PTR_ERR(vma);
+ goto err;
+ }
+
+ flags = PIN_GLOBAL;
+ if (!HAS_LLC(engine->i915))
+ /* On g33, we cannot place HWS above 256MiB, so
+ * restrict its pinning to the low mappable arena.
+ * Though this restriction is not documented for
+ * gen4, gen5, or byt, they also behave similarly
+ * and hang if the HWS is placed at the top of the
+ * GTT. To generalise, it appears that all !llc
+ * platforms have issues with us placing the HWS
+ * above the mappable region (even though we never
+ * actually map it).
+ */
+ flags |= PIN_MAPPABLE;
+ else
+ flags |= PIN_HIGH;
+ ret = i915_vma_pin(vma, 0, 4096, flags);
+ if (ret)
+ goto err;
+
+ vaddr = i915_gem_object_pin_map(obj, I915_MAP_WB);
+ if (IS_ERR(vaddr)) {
+ ret = PTR_ERR(vaddr);
+ goto err_unpin;
+ }
+
+ engine->status_page.vma = vma;
+ engine->status_page.ggtt_offset = i915_ggtt_offset(vma);
+ engine->status_page.page_addr = memset(vaddr, 0, PAGE_SIZE);
+
+ DRM_DEBUG_DRIVER("%s hws offset: 0x%08x\n",
+ engine->name, i915_ggtt_offset(vma));
+ return 0;
+
+err_unpin:
+ i915_vma_unpin(vma);
+err:
+ i915_gem_object_put(obj);
+ return ret;
+}
+
+static int init_phys_status_page(struct intel_engine_cs *engine)
+{
+ struct drm_i915_private *dev_priv = engine->i915;
+
+ GEM_BUG_ON(engine->id != RCS);
+
+ dev_priv->status_page_dmah =
+ drm_pci_alloc(&dev_priv->drm, PAGE_SIZE, PAGE_SIZE);
+ if (!dev_priv->status_page_dmah)
+ return -ENOMEM;
+
+ engine->status_page.page_addr = dev_priv->status_page_dmah->vaddr;
+ memset(engine->status_page.page_addr, 0, PAGE_SIZE);
+
+ return 0;
+}
+
/**
* intel_engines_init_common - initialize cengine state which might require hw access
* @engine: Engine to initialize.
ret = i915_gem_render_state_init(engine);
if (ret)
- goto err_unpin;
+ goto err_breadcrumbs;
+
+ if (HWS_NEEDS_PHYSICAL(engine->i915))
+ ret = init_phys_status_page(engine);
+ else
+ ret = init_status_page(engine);
+ if (ret)
+ goto err_rs_fini;
return 0;
+err_rs_fini:
+ i915_gem_render_state_fini(engine);
+err_breadcrumbs:
+ intel_engine_fini_breadcrumbs(engine);
err_unpin:
engine->context_unpin(engine, engine->i915->kernel_context);
return ret;
{
intel_engine_cleanup_scratch(engine);
+ if (HWS_NEEDS_PHYSICAL(engine->i915))
+ cleanup_phys_status_page(engine);
+ else
+ cleanup_status_page(engine);
+
i915_gem_render_state_fini(engine);
intel_engine_fini_breadcrumbs(engine);
intel_engine_cleanup_cmd_parser(engine);
I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) |
ECOCHK_DIS_TLB);
+ if (HAS_LLC(dev_priv)) {
+ /* WaCompressedResourceSamplerPbeMediaNewHashMode:skl,kbl
+ *
+ * Must match Display Engine. See
+ * WaCompressedResourceDisplayNewHashMode.
+ */
+ WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
+ GEN9_PBE_COMPRESSED_HASH_SELECTION);
+ WA_SET_BIT_MASKED(GEN9_HALF_SLICE_CHICKEN7,
+ GEN9_SAMPLER_HASH_COMPRESSED_READ_ADDR);
+ WA_SET_BIT(MMCD_MISC_CTRL, MMCD_PCLA | MMCD_HOTSPOT_EN);
+ }
+
/* WaClearFlowControlGpgpuContextSave:skl,bxt,kbl,glk,cfl */
/* WaDisablePartialInstShootdown:skl,bxt,kbl,glk,cfl */
WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN,
GEN9_GAPS_TSV_CREDIT_DISABLE));
/* WaDisableGafsUnitClkGating:skl */
- WA_SET_BIT(GEN7_UCGCTL4, GEN8_EU_GAUNIT_CLOCK_GATE_DISABLE);
+ I915_WRITE(GEN7_UCGCTL4, (I915_READ(GEN7_UCGCTL4) |
+ GEN8_EU_GAUNIT_CLOCK_GATE_DISABLE));
/* WaInPlaceDecompressionHang:skl */
if (IS_SKL_REVID(dev_priv, SKL_REVID_H0, REVID_FOREVER))
- WA_SET_BIT(GEN9_GAMT_ECO_REG_RW_IA,
- GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
+ I915_WRITE(GEN9_GAMT_ECO_REG_RW_IA,
+ (I915_READ(GEN9_GAMT_ECO_REG_RW_IA) |
+ GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS));
/* WaDisableLSQCROPERFforOCL:skl */
ret = wa_ring_whitelist_reg(engine, GEN8_L3SQCREG4);
/* WaDisablePooledEuLoadBalancingFix:bxt */
if (IS_BXT_REVID(dev_priv, BXT_REVID_B0, REVID_FOREVER)) {
- WA_SET_BIT_MASKED(FF_SLICE_CS_CHICKEN2,
- GEN9_POOLED_EU_LOAD_BALANCING_FIX_DISABLE);
+ I915_WRITE(FF_SLICE_CS_CHICKEN2,
+ _MASKED_BIT_ENABLE(GEN9_POOLED_EU_LOAD_BALANCING_FIX_DISABLE));
}
/* WaDisableSbeCacheDispatchPortSharing:bxt */
/* WaInPlaceDecompressionHang:bxt */
if (IS_BXT_REVID(dev_priv, BXT_REVID_C0, REVID_FOREVER))
- WA_SET_BIT(GEN9_GAMT_ECO_REG_RW_IA,
- GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
+ I915_WRITE(GEN9_GAMT_ECO_REG_RW_IA,
+ (I915_READ(GEN9_GAMT_ECO_REG_RW_IA) |
+ GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS));
return 0;
}
struct drm_i915_private *dev_priv = engine->i915;
int ret;
- /* WaDisableI2mCycleOnWRPort: cnl (pre-prod) */
+ /* WaDisableI2mCycleOnWRPort:cnl (pre-prod) */
if (IS_CNL_REVID(dev_priv, CNL_REVID_B0, CNL_REVID_B0))
- WA_SET_BIT(GAMT_CHKN_BIT_REG,
- GAMT_CHKN_DISABLE_I2M_CYCLE_ON_WR_PORT);
+ I915_WRITE(GAMT_CHKN_BIT_REG,
+ (I915_READ(GAMT_CHKN_BIT_REG) |
+ GAMT_CHKN_DISABLE_I2M_CYCLE_ON_WR_PORT));
/* WaForceContextSaveRestoreNonCoherent:cnl */
WA_SET_BIT_MASKED(CNL_HDC_CHICKEN0,
GEN8_CSC2_SBE_VUE_CACHE_CONSERVATIVE);
/* WaInPlaceDecompressionHang:cnl */
- WA_SET_BIT(GEN9_GAMT_ECO_REG_RW_IA,
- GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
+ I915_WRITE(GEN9_GAMT_ECO_REG_RW_IA,
+ (I915_READ(GEN9_GAMT_ECO_REG_RW_IA) |
+ GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS));
/* WaPushConstantDereferenceHoldDisable:cnl */
- WA_SET_BIT(GEN7_ROW_CHICKEN2, PUSH_CONSTANT_DEREF_DISABLE);
+ WA_SET_BIT_MASKED(GEN7_ROW_CHICKEN2, PUSH_CONSTANT_DEREF_DISABLE);
/* FtrEnableFastAnisoL1BankingFix: cnl */
WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN3, CNL_FAST_ANISO_L1_BANKING_FIX);
/* WaDisableDynamicCreditSharing:kbl */
if (IS_KBL_REVID(dev_priv, 0, KBL_REVID_B0))
- WA_SET_BIT(GAMT_CHKN_BIT_REG,
- GAMT_CHKN_DISABLE_DYNAMIC_CREDIT_SHARING);
+ I915_WRITE(GAMT_CHKN_BIT_REG,
+ (I915_READ(GAMT_CHKN_BIT_REG) |
+ GAMT_CHKN_DISABLE_DYNAMIC_CREDIT_SHARING));
/* WaDisableFenceDestinationToSLM:kbl (pre-prod) */
if (IS_KBL_REVID(dev_priv, KBL_REVID_A0, KBL_REVID_A0))
GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
/* WaDisableGafsUnitClkGating:kbl */
- WA_SET_BIT(GEN7_UCGCTL4, GEN8_EU_GAUNIT_CLOCK_GATE_DISABLE);
+ I915_WRITE(GEN7_UCGCTL4, (I915_READ(GEN7_UCGCTL4) |
+ GEN8_EU_GAUNIT_CLOCK_GATE_DISABLE));
/* WaDisableSbeCacheDispatchPortSharing:kbl */
WA_SET_BIT_MASKED(
GEN7_SBE_SS_CACHE_DISPATCH_PORT_SHARING_DISABLE);
/* WaInPlaceDecompressionHang:kbl */
- WA_SET_BIT(GEN9_GAMT_ECO_REG_RW_IA,
- GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
+ I915_WRITE(GEN9_GAMT_ECO_REG_RW_IA,
+ (I915_READ(GEN9_GAMT_ECO_REG_RW_IA) |
+ GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS));
/* WaDisableLSQCROPERFforOCL:kbl */
ret = wa_ring_whitelist_reg(engine, GEN8_L3SQCREG4);
GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
/* WaDisableGafsUnitClkGating:cfl */
- WA_SET_BIT(GEN7_UCGCTL4, GEN8_EU_GAUNIT_CLOCK_GATE_DISABLE);
+ I915_WRITE(GEN7_UCGCTL4, (I915_READ(GEN7_UCGCTL4) |
+ GEN8_EU_GAUNIT_CLOCK_GATE_DISABLE));
/* WaDisableSbeCacheDispatchPortSharing:cfl */
WA_SET_BIT_MASKED(
GEN7_SBE_SS_CACHE_DISPATCH_PORT_SHARING_DISABLE);
/* WaInPlaceDecompressionHang:cfl */
- WA_SET_BIT(GEN9_GAMT_ECO_REG_RW_IA,
- GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
+ I915_WRITE(GEN9_GAMT_ECO_REG_RW_IA,
+ (I915_READ(GEN9_GAMT_ECO_REG_RW_IA) |
+ GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS));
return 0;
}
return false;
/* Both ports drained, no more ELSP submission? */
- if (port_request(&engine->execlist_port[0]))
+ if (port_request(&engine->execlists.port[0]))
return false;
/* ELSP is empty, but there are ready requests? */
- if (READ_ONCE(engine->execlist_first))
+ if (READ_ONCE(engine->execlists.first))
return false;
/* Ring stopped? */
for_each_engine(engine, i915, id) {
intel_engine_disarm_breadcrumbs(engine);
i915_gem_batch_pool_fini(&engine->batch_pool);
- tasklet_kill(&engine->irq_tasklet);
- engine->no_priolist = false;
+ tasklet_kill(&engine->execlists.irq_tasklet);
+ engine->execlists.no_priolist = false;
}
}
return false;
}
- if (!i915.enable_fbc) {
+ if (!i915_modparams.enable_fbc) {
fbc->no_fbc_reason = "disabled per module param or by default";
return false;
}
*/
static int intel_sanitize_fbc_option(struct drm_i915_private *dev_priv)
{
- if (i915.enable_fbc >= 0)
- return !!i915.enable_fbc;
+ if (i915_modparams.enable_fbc >= 0)
+ return !!i915_modparams.enable_fbc;
if (!HAS_FBC(dev_priv))
return 0;
if (need_fbc_vtd_wa(dev_priv))
mkwrite_device_info(dev_priv)->has_fbc = false;
- i915.enable_fbc = intel_sanitize_fbc_option(dev_priv);
- DRM_DEBUG_KMS("Sanitized enable_fbc value: %d\n", i915.enable_fbc);
+ i915_modparams.enable_fbc = intel_sanitize_fbc_option(dev_priv);
+ DRM_DEBUG_KMS("Sanitized enable_fbc value: %d\n",
+ i915_modparams.enable_fbc);
if (!HAS_FBC(dev_priv)) {
fbc->no_fbc_reason = "unsupported by this chipset";
{
struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
i915_reg_t reg = PIPESTAT(crtc->pipe);
- u32 pipestat = I915_READ(reg) & 0xffff0000;
+ u32 enable_mask;
lockdep_assert_held(&dev_priv->irq_lock);
- if ((pipestat & PIPE_FIFO_UNDERRUN_STATUS) == 0)
+ if ((I915_READ(reg) & PIPE_FIFO_UNDERRUN_STATUS) == 0)
return;
- I915_WRITE(reg, pipestat | PIPE_FIFO_UNDERRUN_STATUS);
+ enable_mask = i915_pipestat_enable_mask(dev_priv, crtc->pipe);
+ I915_WRITE(reg, enable_mask | PIPE_FIFO_UNDERRUN_STATUS);
POSTING_READ(reg);
trace_intel_cpu_fifo_underrun(dev_priv, crtc->pipe);
{
struct drm_i915_private *dev_priv = to_i915(dev);
i915_reg_t reg = PIPESTAT(pipe);
- u32 pipestat = I915_READ(reg) & 0xffff0000;
lockdep_assert_held(&dev_priv->irq_lock);
if (enable) {
- I915_WRITE(reg, pipestat | PIPE_FIFO_UNDERRUN_STATUS);
+ u32 enable_mask = i915_pipestat_enable_mask(dev_priv, pipe);
+
+ I915_WRITE(reg, enable_mask | PIPE_FIFO_UNDERRUN_STATUS);
POSTING_READ(reg);
} else {
- if (old && pipestat & PIPE_FIFO_UNDERRUN_STATUS)
+ if (old && I915_READ(reg) & PIPE_FIFO_UNDERRUN_STATUS)
DRM_ERROR("pipe %c underrun\n", pipe_name(pipe));
}
}
#define WQ_LEN_SHIFT 16
#define WQ_NO_WCFLUSH_WAIT (1 << 27)
#define WQ_PRESENT_WORKLOAD (1 << 28)
-#define WQ_WORKLOAD_SHIFT 29
-#define WQ_WORKLOAD_GENERAL (0 << WQ_WORKLOAD_SHIFT)
-#define WQ_WORKLOAD_GPGPU (1 << WQ_WORKLOAD_SHIFT)
-#define WQ_WORKLOAD_TOUCH (2 << WQ_WORKLOAD_SHIFT)
#define WQ_RING_TAIL_SHIFT 20
#define WQ_RING_TAIL_MAX 0x7FF /* 2^11 QWords */
/* Preempt to idle on quantum expiry */
#define POLICY_PREEMPT_TO_IDLE (1<<1)
-#define POLICY_MAX_NUM_WI 15
+#define POLICY_MAX_NUM_WI 15
+#define POLICY_DEFAULT_DPC_PROMOTE_TIME_US 500000
+#define POLICY_DEFAULT_EXECUTION_QUANTUM_US 1000000
+#define POLICY_DEFAULT_PREEMPTION_TIME_US 500000
+#define POLICY_DEFAULT_FAULT_TIME_US 250000
struct guc_policy {
/* Time for one workload to execute. (in micro seconds) */
params[GUC_CTL_LOG_PARAMS] = guc->log.flags;
- if (i915.guc_log_level >= 0) {
+ if (i915_modparams.guc_log_level >= 0) {
params[GUC_CTL_DEBUG] =
- i915.guc_log_level << GUC_LOG_VERBOSITY_SHIFT;
+ i915_modparams.guc_log_level << GUC_LOG_VERBOSITY_SHIFT;
} else
params[GUC_CTL_DEBUG] = GUC_LOG_DISABLED;
/* If GuC submission is enabled, set up additional parameters here */
- if (i915.enable_guc_submission) {
+ if (i915_modparams.enable_guc_submission) {
u32 ads = guc_ggtt_offset(guc->ads_vma) >> PAGE_SHIFT;
u32 pgs = guc_ggtt_offset(dev_priv->guc.stage_desc_pool);
u32 ctx_in_16 = GUC_MAX_STAGE_DESCRIPTORS / 16;
guc->fw.load_status = INTEL_UC_FIRMWARE_SUCCESS;
DRM_INFO("GuC %s (firmware %s [version %u.%u])\n",
- i915.enable_guc_submission ? "submission enabled" : "loaded",
+ i915_modparams.enable_guc_submission ? "submission enabled" :
+ "loaded",
guc->fw.path,
guc->fw.major_ver_found, guc->fw.minor_ver_found);
guc->fw.load_status = INTEL_UC_FIRMWARE_NONE;
guc->fw.type = INTEL_UC_FW_TYPE_GUC;
- if (i915.guc_firmware_path) {
- guc->fw.path = i915.guc_firmware_path;
+ if (i915_modparams.guc_firmware_path) {
+ guc->fw.path = i915_modparams.guc_firmware_path;
guc->fw.major_ver_wanted = 0;
guc->fw.minor_ver_wanted = 0;
} else if (IS_SKYLAKE(dev_priv)) {
struct dentry *log_dir;
int ret;
- if (i915.guc_log_level < 0)
+ if (i915_modparams.guc_log_level < 0)
return 0;
/* For now create the log file in /sys/kernel/debug/dri/0 dir */
guc_log_runtime_destroy(guc);
err:
/* logging will remain off */
- i915.guc_log_level = -1;
+ i915_modparams.guc_log_level = -1;
return ret;
}
{
struct drm_i915_private *dev_priv = guc_to_i915(guc);
- if (!i915.enable_guc_submission || (i915.guc_log_level < 0))
+ if (!i915_modparams.enable_guc_submission ||
+ (i915_modparams.guc_log_level < 0))
return;
/* First disable the interrupts, will be renabled afterwards */
GEM_BUG_ON(guc->log.vma);
- if (i915.guc_log_level > GUC_LOG_VERBOSITY_MAX)
- i915.guc_log_level = GUC_LOG_VERBOSITY_MAX;
+ if (i915_modparams.guc_log_level > GUC_LOG_VERBOSITY_MAX)
+ i915_modparams.guc_log_level = GUC_LOG_VERBOSITY_MAX;
/* The first page is to save log buffer state. Allocate one
* extra page for others in case for overlap */
guc->log.vma = vma;
- if (i915.guc_log_level >= 0) {
+ if (i915_modparams.guc_log_level >= 0) {
ret = guc_log_runtime_create(guc);
if (ret < 0)
goto err_vma;
i915_vma_unpin_and_release(&guc->log.vma);
err:
/* logging will be off */
- i915.guc_log_level = -1;
+ i915_modparams.guc_log_level = -1;
return ret;
}
return -EINVAL;
/* This combination doesn't make sense & won't have any effect */
- if (!log_param.logging_enabled && (i915.guc_log_level < 0))
+ if (!log_param.logging_enabled && (i915_modparams.guc_log_level < 0))
return 0;
ret = guc_log_control(guc, log_param.value);
}
if (log_param.logging_enabled) {
- i915.guc_log_level = log_param.verbosity;
+ i915_modparams.guc_log_level = log_param.verbosity;
/* If log_level was set as -1 at boot time, then the relay channel file
* wouldn't have been created by now and interrupts also would not have
guc_flush_logs(guc);
/* As logging is disabled, update log level to reflect that */
- i915.guc_log_level = -1;
+ i915_modparams.guc_log_level = -1;
}
return ret;
void i915_guc_log_register(struct drm_i915_private *dev_priv)
{
- if (!i915.enable_guc_submission || i915.guc_log_level < 0)
+ if (!i915_modparams.enable_guc_submission ||
+ (i915_modparams.guc_log_level < 0))
return;
mutex_lock(&dev_priv->drm.struct_mutex);
void i915_guc_log_unregister(struct drm_i915_private *dev_priv)
{
- if (!i915.enable_guc_submission)
+ if (!i915_modparams.enable_guc_submission)
return;
mutex_lock(&dev_priv->drm.struct_mutex);
*/
void intel_gvt_sanitize_options(struct drm_i915_private *dev_priv)
{
- if (!i915.enable_gvt)
+ if (!i915_modparams.enable_gvt)
return;
if (intel_vgpu_active(dev_priv)) {
return;
bail:
- i915.enable_gvt = 0;
+ i915_modparams.enable_gvt = 0;
}
/**
{
int ret;
- if (!i915.enable_gvt) {
+ if (!i915_modparams.enable_gvt) {
DRM_DEBUG_DRIVER("GVT-g is disabled by kernel params\n");
return 0;
}
- if (!i915.enable_execlists) {
+ if (!i915_modparams.enable_execlists) {
DRM_ERROR("i915 GVT-g loading failed due to disabled execlists mode\n");
return -EIO;
}
- if (i915.enable_guc_submission) {
+ if (i915_modparams.enable_guc_submission) {
DRM_ERROR("i915 GVT-g loading failed due to Graphics virtualization is not yet supported with GuC submission\n");
return -EIO;
}
return 0;
bail:
- i915.enable_gvt = 0;
+ i915_modparams.enable_gvt = 0;
return 0;
}
unsigned int hung = 0, stuck = 0;
int busy_count = 0;
- if (!i915.enable_hangcheck)
+ if (!i915_modparams.enable_hangcheck)
return;
if (!READ_ONCE(dev_priv->gt.awake))
huc->fw.load_status = INTEL_UC_FIRMWARE_NONE;
huc->fw.type = INTEL_UC_FW_TYPE_HUC;
- if (i915.huc_firmware_path) {
- huc->fw.path = i915.huc_firmware_path;
+ if (i915_modparams.huc_firmware_path) {
+ huc->fw.path = i915_modparams.huc_firmware_path;
huc->fw.major_ver_wanted = 0;
huc->fw.minor_ver_wanted = 0;
} else if (IS_SKYLAKE(dev_priv)) {
}
/**
- * intel_guc_auth_huc() - authenticate ucode
- * @dev_priv: the drm_i915_device
+ * intel_huc_auth() - Authenticate HuC uCode
+ * @huc: intel_huc structure
+ *
+ * Called after HuC and GuC firmware loading during intel_uc_init_hw().
*
- * Triggers a HuC fw authentication request to the GuC via intel_guc_action_
- * authenticate_huc interface.
+ * This function pins HuC firmware image object into GGTT.
+ * Then it invokes GuC action to authenticate passing the offset to RSA
+ * signature through intel_guc_auth_huc(). It then waits for 50ms for
+ * firmware verification ACK and unpins the object.
*/
-void intel_guc_auth_huc(struct drm_i915_private *dev_priv)
+void intel_huc_auth(struct intel_huc *huc)
{
- struct intel_guc *guc = &dev_priv->guc;
- struct intel_huc *huc = &dev_priv->huc;
+ struct drm_i915_private *i915 = huc_to_i915(huc);
+ struct intel_guc *guc = &i915->guc;
struct i915_vma *vma;
int ret;
- u32 data[2];
if (huc->fw.load_status != INTEL_UC_FIRMWARE_SUCCESS)
return;
return;
}
- /* Specify auth action and where public signature is. */
- data[0] = INTEL_GUC_ACTION_AUTHENTICATE_HUC;
- data[1] = guc_ggtt_offset(vma) + huc->fw.rsa_offset;
-
- ret = intel_guc_send(guc, data, ARRAY_SIZE(data));
+ ret = intel_guc_auth_huc(guc,
+ guc_ggtt_offset(vma) + huc->fw.rsa_offset);
if (ret) {
DRM_ERROR("HuC: GuC did not ack Auth request %d\n", ret);
goto out;
}
/* Check authentication status, it should be done by now */
- ret = intel_wait_for_register(dev_priv,
- HUC_STATUS2,
- HUC_FW_VERIFIED,
- HUC_FW_VERIFIED,
- 50);
-
+ ret = intel_wait_for_register(i915,
+ HUC_STATUS2,
+ HUC_FW_VERIFIED,
+ HUC_FW_VERIFIED,
+ 50);
if (ret) {
DRM_ERROR("HuC: Authentication failed %d\n", ret);
goto out;
out:
i915_vma_unpin(vma);
}
-
if (HAS_LOGICAL_RING_CONTEXTS(dev_priv) &&
USES_PPGTT(dev_priv) &&
- i915.use_mmio_flip >= 0)
+ i915_modparams.use_mmio_flip >= 0)
return 1;
return 0;
BUILD_BUG_ON(MAX_CONTEXT_HW_ID > (1<<GEN8_CTX_ID_WIDTH));
desc = ctx->desc_template; /* bits 0-11 */
- desc |= i915_ggtt_offset(ce->state) + LRC_PPHWSP_PN * PAGE_SIZE;
+ desc |= i915_ggtt_offset(ce->state) + LRC_HEADER_PAGES * PAGE_SIZE;
/* bits 12-31 */
desc |= (u64)ctx->hw_id << GEN8_CTX_ID_SHIFT; /* bits 32-52 */
ce->lrc_desc = desc;
}
-uint64_t intel_lr_context_descriptor(struct i915_gem_context *ctx,
- struct intel_engine_cs *engine)
+static struct i915_priolist *
+lookup_priolist(struct intel_engine_cs *engine,
+ struct i915_priotree *pt,
+ int prio)
{
- return ctx->engine[engine->id].lrc_desc;
+ struct intel_engine_execlists * const execlists = &engine->execlists;
+ struct i915_priolist *p;
+ struct rb_node **parent, *rb;
+ bool first = true;
+
+ if (unlikely(execlists->no_priolist))
+ prio = I915_PRIORITY_NORMAL;
+
+find_priolist:
+ /* most positive priority is scheduled first, equal priorities fifo */
+ rb = NULL;
+ parent = &execlists->queue.rb_node;
+ while (*parent) {
+ rb = *parent;
+ p = rb_entry(rb, typeof(*p), node);
+ if (prio > p->priority) {
+ parent = &rb->rb_left;
+ } else if (prio < p->priority) {
+ parent = &rb->rb_right;
+ first = false;
+ } else {
+ return p;
+ }
+ }
+
+ if (prio == I915_PRIORITY_NORMAL) {
+ p = &execlists->default_priolist;
+ } else {
+ p = kmem_cache_alloc(engine->i915->priorities, GFP_ATOMIC);
+ /* Convert an allocation failure to a priority bump */
+ if (unlikely(!p)) {
+ prio = I915_PRIORITY_NORMAL; /* recurses just once */
+
+ /* To maintain ordering with all rendering, after an
+ * allocation failure we have to disable all scheduling.
+ * Requests will then be executed in fifo, and schedule
+ * will ensure that dependencies are emitted in fifo.
+ * There will be still some reordering with existing
+ * requests, so if userspace lied about their
+ * dependencies that reordering may be visible.
+ */
+ execlists->no_priolist = true;
+ goto find_priolist;
+ }
+ }
+
+ p->priority = prio;
+ INIT_LIST_HEAD(&p->requests);
+ rb_link_node(&p->node, rb, parent);
+ rb_insert_color(&p->node, &execlists->queue);
+
+ if (first)
+ execlists->first = &p->node;
+
+ return ptr_pack_bits(p, first, 1);
}
static inline void
static void execlists_submit_ports(struct intel_engine_cs *engine)
{
- struct execlist_port *port = engine->execlist_port;
+ struct execlist_port *port = engine->execlists.port;
u32 __iomem *elsp =
engine->i915->regs + i915_mmio_reg_offset(RING_ELSP(engine));
unsigned int n;
- for (n = ARRAY_SIZE(engine->execlist_port); n--; ) {
+ for (n = execlists_num_ports(&engine->execlists); n--; ) {
struct drm_i915_gem_request *rq;
unsigned int count;
u64 desc;
static void execlists_dequeue(struct intel_engine_cs *engine)
{
struct drm_i915_gem_request *last;
- struct execlist_port *port = engine->execlist_port;
+ struct intel_engine_execlists * const execlists = &engine->execlists;
+ struct execlist_port *port = execlists->port;
+ const struct execlist_port * const last_port =
+ &execlists->port[execlists->port_mask];
struct rb_node *rb;
bool submit = false;
*/
last->tail = last->wa_tail;
- GEM_BUG_ON(port_isset(&port[1]));
-
/* Hardware submission is through 2 ports. Conceptually each port
* has a (RING_START, RING_HEAD, RING_TAIL) tuple. RING_START is
* static for a context, and unique to each, so we only execute
*/
spin_lock_irq(&engine->timeline->lock);
- rb = engine->execlist_first;
- GEM_BUG_ON(rb_first(&engine->execlist_queue) != rb);
+ rb = execlists->first;
+ GEM_BUG_ON(rb_first(&execlists->queue) != rb);
while (rb) {
struct i915_priolist *p = rb_entry(rb, typeof(*p), node);
struct drm_i915_gem_request *rq, *rn;
* combine this request with the last, then we
* are done.
*/
- if (port != engine->execlist_port) {
+ if (port == last_port) {
__list_del_many(&p->requests,
&rq->priotree.link);
goto done;
if (submit)
port_assign(port, last);
port++;
+
+ GEM_BUG_ON(port_isset(port));
}
INIT_LIST_HEAD(&rq->priotree.link);
rq->priotree.priority = INT_MAX;
__i915_gem_request_submit(rq);
- trace_i915_gem_request_in(rq, port_index(port, engine));
+ trace_i915_gem_request_in(rq, port_index(port, execlists));
last = rq;
submit = true;
}
rb = rb_next(rb);
- rb_erase(&p->node, &engine->execlist_queue);
+ rb_erase(&p->node, &execlists->queue);
INIT_LIST_HEAD(&p->requests);
if (p->priority != I915_PRIORITY_NORMAL)
kmem_cache_free(engine->i915->priorities, p);
}
done:
- engine->execlist_first = rb;
+ execlists->first = rb;
if (submit)
port_assign(port, last);
spin_unlock_irq(&engine->timeline->lock);
execlists_submit_ports(engine);
}
+static void
+execlist_cancel_port_requests(struct intel_engine_execlists *execlists)
+{
+ struct execlist_port *port = execlists->port;
+ unsigned int num_ports = ARRAY_SIZE(execlists->port);
+
+ while (num_ports-- && port_isset(port)) {
+ struct drm_i915_gem_request *rq = port_request(port);
+
+ execlists_context_status_change(rq, INTEL_CONTEXT_SCHEDULE_OUT);
+ i915_gem_request_put(rq);
+
+ memset(port, 0, sizeof(*port));
+ port++;
+ }
+}
+
+static void execlists_cancel_requests(struct intel_engine_cs *engine)
+{
+ struct intel_engine_execlists * const execlists = &engine->execlists;
+ struct drm_i915_gem_request *rq, *rn;
+ struct rb_node *rb;
+ unsigned long flags;
+
+ spin_lock_irqsave(&engine->timeline->lock, flags);
+
+ /* Cancel the requests on the HW and clear the ELSP tracker. */
+ execlist_cancel_port_requests(execlists);
+
+ /* Mark all executing requests as skipped. */
+ list_for_each_entry(rq, &engine->timeline->requests, link) {
+ GEM_BUG_ON(!rq->global_seqno);
+ if (!i915_gem_request_completed(rq))
+ dma_fence_set_error(&rq->fence, -EIO);
+ }
+
+ /* Flush the queued requests to the timeline list (for retiring). */
+ rb = execlists->first;
+ while (rb) {
+ struct i915_priolist *p = rb_entry(rb, typeof(*p), node);
+
+ list_for_each_entry_safe(rq, rn, &p->requests, priotree.link) {
+ INIT_LIST_HEAD(&rq->priotree.link);
+ rq->priotree.priority = INT_MAX;
+
+ dma_fence_set_error(&rq->fence, -EIO);
+ __i915_gem_request_submit(rq);
+ }
+
+ rb = rb_next(rb);
+ rb_erase(&p->node, &execlists->queue);
+ INIT_LIST_HEAD(&p->requests);
+ if (p->priority != I915_PRIORITY_NORMAL)
+ kmem_cache_free(engine->i915->priorities, p);
+ }
+
+ /* Remaining _unready_ requests will be nop'ed when submitted */
+
+
+ execlists->queue = RB_ROOT;
+ execlists->first = NULL;
+ GEM_BUG_ON(port_isset(execlists->port));
+
+ /*
+ * The port is checked prior to scheduling a tasklet, but
+ * just in case we have suspended the tasklet to do the
+ * wedging make sure that when it wakes, it decides there
+ * is no work to do by clearing the irq_posted bit.
+ */
+ clear_bit(ENGINE_IRQ_EXECLIST, &engine->irq_posted);
+
+ spin_unlock_irqrestore(&engine->timeline->lock, flags);
+}
+
static bool execlists_elsp_ready(const struct intel_engine_cs *engine)
{
- const struct execlist_port *port = engine->execlist_port;
+ const struct execlist_port *port = engine->execlists.port;
return port_count(&port[0]) + port_count(&port[1]) < 2;
}
*/
static void intel_lrc_irq_handler(unsigned long data)
{
- struct intel_engine_cs *engine = (struct intel_engine_cs *)data;
- struct execlist_port *port = engine->execlist_port;
+ struct intel_engine_cs * const engine = (struct intel_engine_cs *)data;
+ struct intel_engine_execlists * const execlists = &engine->execlists;
+ struct execlist_port *port = execlists->port;
struct drm_i915_private *dev_priv = engine->i915;
/* We can skip acquiring intel_runtime_pm_get() here as it was taken
*/
GEM_BUG_ON(!dev_priv->gt.awake);
- intel_uncore_forcewake_get(dev_priv, engine->fw_domains);
+ intel_uncore_forcewake_get(dev_priv, execlists->fw_domains);
/* Prefer doing test_and_clear_bit() as a two stage operation to avoid
* imposing the cost of a locked atomic transaction when submitting a
* new request (outside of the context-switch interrupt).
*/
while (test_bit(ENGINE_IRQ_EXECLIST, &engine->irq_posted)) {
- u32 __iomem *csb_mmio =
- dev_priv->regs + i915_mmio_reg_offset(RING_CONTEXT_STATUS_PTR(engine));
- u32 __iomem *buf =
- dev_priv->regs + i915_mmio_reg_offset(RING_CONTEXT_STATUS_BUF_LO(engine, 0));
+ /* The HWSP contains a (cacheable) mirror of the CSB */
+ const u32 *buf =
+ &engine->status_page.page_addr[I915_HWS_CSB_BUF0_INDEX];
unsigned int head, tail;
+ /* However GVT emulation depends upon intercepting CSB mmio */
+ if (unlikely(execlists->csb_use_mmio)) {
+ buf = (u32 * __force)
+ (dev_priv->regs + i915_mmio_reg_offset(RING_CONTEXT_STATUS_BUF_LO(engine, 0)));
+ execlists->csb_head = -1; /* force mmio read of CSB ptrs */
+ }
+
/* The write will be ordered by the uncached read (itself
* a memory barrier), so we do not need another in the form
* of a locked instruction. The race between the interrupt
* is set and we do a new loop.
*/
__clear_bit(ENGINE_IRQ_EXECLIST, &engine->irq_posted);
- head = readl(csb_mmio);
- tail = GEN8_CSB_WRITE_PTR(head);
- head = GEN8_CSB_READ_PTR(head);
+ if (unlikely(execlists->csb_head == -1)) { /* following a reset */
+ head = readl(dev_priv->regs + i915_mmio_reg_offset(RING_CONTEXT_STATUS_PTR(engine)));
+ tail = GEN8_CSB_WRITE_PTR(head);
+ head = GEN8_CSB_READ_PTR(head);
+ execlists->csb_head = head;
+ } else {
+ const int write_idx =
+ intel_hws_csb_write_index(dev_priv) -
+ I915_HWS_CSB_BUF0_INDEX;
+
+ head = execlists->csb_head;
+ tail = READ_ONCE(buf[write_idx]);
+ }
+
while (head != tail) {
struct drm_i915_gem_request *rq;
unsigned int status;
* status notifier.
*/
- status = readl(buf + 2 * head);
+ status = READ_ONCE(buf[2 * head]); /* maybe mmio! */
if (!(status & GEN8_CTX_STATUS_COMPLETED_MASK))
continue;
/* Check the context/desc id for this event matches */
- GEM_DEBUG_BUG_ON(readl(buf + 2 * head + 1) !=
- port->context_id);
+ GEM_DEBUG_BUG_ON(buf[2 * head + 1] != port->context_id);
rq = port_unpack(port, &count);
GEM_BUG_ON(count == 0);
trace_i915_gem_request_out(rq);
i915_gem_request_put(rq);
- port[0] = port[1];
- memset(&port[1], 0, sizeof(port[1]));
+ execlists_port_complete(execlists, port);
} else {
port_set(port, port_pack(rq, count));
}
!(status & GEN8_CTX_STATUS_ACTIVE_IDLE));
}
- writel(_MASKED_FIELD(GEN8_CSB_READ_PTR_MASK, head << 8),
- csb_mmio);
+ if (head != execlists->csb_head) {
+ execlists->csb_head = head;
+ writel(_MASKED_FIELD(GEN8_CSB_READ_PTR_MASK, head << 8),
+ dev_priv->regs + i915_mmio_reg_offset(RING_CONTEXT_STATUS_PTR(engine)));
+ }
}
if (execlists_elsp_ready(engine))
execlists_dequeue(engine);
- intel_uncore_forcewake_put(dev_priv, engine->fw_domains);
+ intel_uncore_forcewake_put(dev_priv, execlists->fw_domains);
}
-static bool
-insert_request(struct intel_engine_cs *engine,
- struct i915_priotree *pt,
- int prio)
+static void insert_request(struct intel_engine_cs *engine,
+ struct i915_priotree *pt,
+ int prio)
{
- struct i915_priolist *p;
- struct rb_node **parent, *rb;
- bool first = true;
+ struct i915_priolist *p = lookup_priolist(engine, pt, prio);
- if (unlikely(engine->no_priolist))
- prio = I915_PRIORITY_NORMAL;
-
-find_priolist:
- /* most positive priority is scheduled first, equal priorities fifo */
- rb = NULL;
- parent = &engine->execlist_queue.rb_node;
- while (*parent) {
- rb = *parent;
- p = rb_entry(rb, typeof(*p), node);
- if (prio > p->priority) {
- parent = &rb->rb_left;
- } else if (prio < p->priority) {
- parent = &rb->rb_right;
- first = false;
- } else {
- list_add_tail(&pt->link, &p->requests);
- return false;
- }
- }
-
- if (prio == I915_PRIORITY_NORMAL) {
- p = &engine->default_priolist;
- } else {
- p = kmem_cache_alloc(engine->i915->priorities, GFP_ATOMIC);
- /* Convert an allocation failure to a priority bump */
- if (unlikely(!p)) {
- prio = I915_PRIORITY_NORMAL; /* recurses just once */
-
- /* To maintain ordering with all rendering, after an
- * allocation failure we have to disable all scheduling.
- * Requests will then be executed in fifo, and schedule
- * will ensure that dependencies are emitted in fifo.
- * There will be still some reordering with existing
- * requests, so if userspace lied about their
- * dependencies that reordering may be visible.
- */
- engine->no_priolist = true;
- goto find_priolist;
- }
- }
-
- p->priority = prio;
- rb_link_node(&p->node, rb, parent);
- rb_insert_color(&p->node, &engine->execlist_queue);
-
- INIT_LIST_HEAD(&p->requests);
- list_add_tail(&pt->link, &p->requests);
-
- if (first)
- engine->execlist_first = &p->node;
-
- return first;
+ list_add_tail(&pt->link, &ptr_mask_bits(p, 1)->requests);
+ if (ptr_unmask_bits(p, 1) && execlists_elsp_ready(engine))
+ tasklet_hi_schedule(&engine->execlists.irq_tasklet);
}
static void execlists_submit_request(struct drm_i915_gem_request *request)
/* Will be called from irq-context when using foreign fences. */
spin_lock_irqsave(&engine->timeline->lock, flags);
- if (insert_request(engine,
- &request->priotree,
- request->priotree.priority)) {
- if (execlists_elsp_ready(engine))
- tasklet_hi_schedule(&engine->irq_tasklet);
- }
+ insert_request(engine, &request->priotree, request->priotree.priority);
- GEM_BUG_ON(!engine->execlist_first);
+ GEM_BUG_ON(!engine->execlists.first);
GEM_BUG_ON(list_empty(&request->priotree.link));
spin_unlock_irqrestore(&engine->timeline->lock, flags);
*/
request->reserved_space += EXECLISTS_REQUEST_SIZE;
- if (i915.enable_guc_submission) {
- /*
- * Check that the GuC has space for the request before
- * going any further, as the i915_add_request() call
- * later on mustn't fail ...
- */
- ret = i915_guc_wq_reserve(request);
- if (ret)
- goto err;
- }
-
cs = intel_ring_begin(request, 0);
- if (IS_ERR(cs)) {
- ret = PTR_ERR(cs);
- goto err_unreserve;
- }
+ if (IS_ERR(cs))
+ return PTR_ERR(cs);
if (!ce->initialised) {
ret = engine->init_context(request);
if (ret)
- goto err_unreserve;
+ return ret;
ce->initialised = true;
}
request->reserved_space -= EXECLISTS_REQUEST_SIZE;
return 0;
-
-err_unreserve:
- if (i915.enable_guc_submission)
- i915_guc_wq_unreserve(request);
-err:
- return ret;
}
/*
return batch;
}
-static u32 *gen9_init_perctx_bb(struct intel_engine_cs *engine, u32 *batch)
-{
- *batch++ = MI_BATCH_BUFFER_END;
-
- return batch;
-}
-
#define CTX_WA_BB_OBJ_SIZE (PAGE_SIZE)
static int lrc_setup_wa_ctx(struct intel_engine_cs *engine)
return 0;
case 9:
wa_bb_fn[0] = gen9_init_indirectctx_bb;
- wa_bb_fn[1] = gen9_init_perctx_bb;
+ wa_bb_fn[1] = NULL;
break;
case 8:
wa_bb_fn[0] = gen8_init_indirectctx_bb;
ret = -EINVAL;
break;
}
- batch_ptr = wa_bb_fn[i](engine, batch_ptr);
+ if (wa_bb_fn[i])
+ batch_ptr = wa_bb_fn[i](engine, batch_ptr);
wa_bb[i]->size = batch_ptr - (batch + wa_bb[i]->offset);
}
static int gen8_init_common_ring(struct intel_engine_cs *engine)
{
struct drm_i915_private *dev_priv = engine->i915;
- struct execlist_port *port = engine->execlist_port;
- unsigned int n;
- bool submit;
+ struct intel_engine_execlists * const execlists = &engine->execlists;
int ret;
ret = intel_mocs_init_engine(engine);
I915_WRITE(GEN8_GT_IIR(gtiir[engine->id]),
GT_CONTEXT_SWITCH_INTERRUPT << engine->irq_shift);
clear_bit(ENGINE_IRQ_EXECLIST, &engine->irq_posted);
+ execlists->csb_head = -1;
/* After a GPU reset, we may have requests to replay */
- submit = false;
- for (n = 0; n < ARRAY_SIZE(engine->execlist_port); n++) {
- if (!port_isset(&port[n]))
- break;
-
- DRM_DEBUG_DRIVER("Restarting %s:%d from 0x%x\n",
- engine->name, n,
- port_request(&port[n])->global_seqno);
-
- /* Discard the current inflight count */
- port_set(&port[n], port_request(&port[n]));
- submit = true;
- }
-
- if (submit && !i915.enable_guc_submission)
- execlists_submit_ports(engine);
+ if (!i915_modparams.enable_guc_submission && execlists->first)
+ tasklet_schedule(&execlists->irq_tasklet);
return 0;
}
static void reset_common_ring(struct intel_engine_cs *engine,
struct drm_i915_gem_request *request)
{
- struct execlist_port *port = engine->execlist_port;
+ struct intel_engine_execlists * const execlists = &engine->execlists;
+ struct drm_i915_gem_request *rq, *rn;
struct intel_context *ce;
- unsigned int n;
+ unsigned long flags;
+
+ spin_lock_irqsave(&engine->timeline->lock, flags);
/*
* Catch up with any missed context-switch interrupts.
* guessing the missed context-switch events by looking at what
* requests were completed.
*/
- if (!request) {
- for (n = 0; n < ARRAY_SIZE(engine->execlist_port); n++)
- i915_gem_request_put(port_request(&port[n]));
- memset(engine->execlist_port, 0, sizeof(engine->execlist_port));
- return;
- }
+ execlist_cancel_port_requests(execlists);
- if (request->ctx != port_request(port)->ctx) {
- i915_gem_request_put(port_request(port));
- port[0] = port[1];
- memset(&port[1], 0, sizeof(port[1]));
+ /* Push back any incomplete requests for replay after the reset. */
+ list_for_each_entry_safe_reverse(rq, rn,
+ &engine->timeline->requests, link) {
+ struct i915_priolist *p;
+
+ if (i915_gem_request_completed(rq))
+ break;
+
+ __i915_gem_request_unsubmit(rq);
+
+ p = lookup_priolist(engine,
+ &rq->priotree,
+ rq->priotree.priority);
+ list_add(&rq->priotree.link,
+ &ptr_mask_bits(p, 1)->requests);
}
- GEM_BUG_ON(request->ctx != port_request(port)->ctx);
+ spin_unlock_irqrestore(&engine->timeline->lock, flags);
/* If the request was innocent, we leave the request in the ELSP
* and will try to replay it on restarting. The context image may
* and have to at least restore the RING register in the context
* image back to the expected values to skip over the guilty request.
*/
- if (request->fence.error != -EIO)
+ if (!request || request->fence.error != -EIO)
return;
/* We want a simple context + ring to execute the breadcrumb update.
* Tasklet cannot be active at this point due intel_mark_active/idle
* so this is just for documentation.
*/
- if (WARN_ON(test_bit(TASKLET_STATE_SCHED, &engine->irq_tasklet.state)))
- tasklet_kill(&engine->irq_tasklet);
+ if (WARN_ON(test_bit(TASKLET_STATE_SCHED, &engine->execlists.irq_tasklet.state)))
+ tasklet_kill(&engine->execlists.irq_tasklet);
dev_priv = engine->i915;
if (engine->cleanup)
engine->cleanup(engine);
- if (engine->status_page.vma) {
- i915_gem_object_unpin_map(engine->status_page.vma->obj);
- engine->status_page.vma = NULL;
- }
-
intel_engine_cleanup_common(engine);
lrc_destroy_wa_ctx(engine);
static void execlists_set_default_submission(struct intel_engine_cs *engine)
{
engine->submit_request = execlists_submit_request;
+ engine->cancel_requests = execlists_cancel_requests;
engine->schedule = execlists_schedule;
- engine->irq_tasklet.func = intel_lrc_irq_handler;
+ engine->execlists.irq_tasklet.func = intel_lrc_irq_handler;
}
static void
engine->irq_keep_mask = GT_CONTEXT_SWITCH_INTERRUPT << shift;
}
-static int
-lrc_setup_hws(struct intel_engine_cs *engine, struct i915_vma *vma)
-{
- const int hws_offset = LRC_PPHWSP_PN * PAGE_SIZE;
- void *hws;
-
- /* The HWSP is part of the default context object in LRC mode. */
- hws = i915_gem_object_pin_map(vma->obj, I915_MAP_WB);
- if (IS_ERR(hws))
- return PTR_ERR(hws);
-
- engine->status_page.page_addr = hws + hws_offset;
- engine->status_page.ggtt_offset = i915_ggtt_offset(vma) + hws_offset;
- engine->status_page.vma = vma;
-
- return 0;
-}
-
static void
logical_ring_setup(struct intel_engine_cs *engine)
{
RING_CONTEXT_STATUS_BUF_BASE(engine),
FW_REG_READ);
- engine->fw_domains = fw_domains;
+ engine->execlists.fw_domains = fw_domains;
- tasklet_init(&engine->irq_tasklet,
+ tasklet_init(&engine->execlists.irq_tasklet,
intel_lrc_irq_handler, (unsigned long)engine);
logical_ring_default_vfuncs(engine);
logical_ring_default_irqs(engine);
}
-static int
-logical_ring_init(struct intel_engine_cs *engine)
+static int logical_ring_init(struct intel_engine_cs *engine)
{
- struct i915_gem_context *dctx = engine->i915->kernel_context;
int ret;
ret = intel_engine_init_common(engine);
if (ret)
goto error;
- /* And setup the hardware status page. */
- ret = lrc_setup_hws(engine, dctx->engine[engine->id].state);
- if (ret) {
- DRM_ERROR("Failed to set up hws %s: %d\n", engine->name, ret);
- goto error;
- }
-
return 0;
error:
CTX_REG(regs, CTX_SECOND_BB_HEAD_L, RING_SBBADDR(base), 0);
CTX_REG(regs, CTX_SECOND_BB_STATE, RING_SBBSTATE(base), 0);
if (rcs) {
- CTX_REG(regs, CTX_BB_PER_CTX_PTR, RING_BB_PER_CTX_PTR(base), 0);
+ struct i915_ctx_workarounds *wa_ctx = &engine->wa_ctx;
+
CTX_REG(regs, CTX_RCS_INDIRECT_CTX, RING_INDIRECT_CTX(base), 0);
CTX_REG(regs, CTX_RCS_INDIRECT_CTX_OFFSET,
RING_INDIRECT_CTX_OFFSET(base), 0);
-
- if (engine->wa_ctx.vma) {
- struct i915_ctx_workarounds *wa_ctx = &engine->wa_ctx;
+ if (wa_ctx->indirect_ctx.size) {
u32 ggtt_offset = i915_ggtt_offset(wa_ctx->vma);
regs[CTX_RCS_INDIRECT_CTX + 1] =
regs[CTX_RCS_INDIRECT_CTX_OFFSET + 1] =
intel_lr_indirect_ctx_offset(engine) << 6;
+ }
+
+ CTX_REG(regs, CTX_BB_PER_CTX_PTR, RING_BB_PER_CTX_PTR(base), 0);
+ if (wa_ctx->per_ctx.size) {
+ u32 ggtt_offset = i915_ggtt_offset(wa_ctx->vma);
regs[CTX_BB_PER_CTX_PTR + 1] =
(ggtt_offset + wa_ctx->per_ctx.offset) | 0x01;
context_size = round_up(engine->context_size, I915_GTT_PAGE_SIZE);
- /* One extra page as the sharing data between driver and GuC */
- context_size += PAGE_SIZE * LRC_PPHWSP_PN;
+ /*
+ * Before the actual start of the context image, we insert a few pages
+ * for our own use and for sharing with the GuC.
+ */
+ context_size += LRC_HEADER_PAGES * PAGE_SIZE;
ctx_obj = i915_gem_object_create(ctx->i915, context_size);
if (IS_ERR(ctx_obj)) {
#define _INTEL_LRC_H_
#include "intel_ringbuffer.h"
+#include "i915_gem_context.h"
#define GEN8_LR_CONTEXT_ALIGN I915_GTT_MIN_ALIGNMENT
/* Logical Ring Contexts */
-/* One extra page is added before LRC for GuC as shared data */
+/*
+ * We allocate a header at the start of the context image for our own
+ * use, therefore the actual location of the logical state is offset
+ * from the start of the VMA. The layout is
+ *
+ * | [guc] | [hwsp] [logical state] |
+ * |<- our header ->|<- context image ->|
+ *
+ */
+/* The first page is used for sharing data with the GuC */
#define LRC_GUCSHR_PN (0)
-#define LRC_PPHWSP_PN (LRC_GUCSHR_PN + 1)
-#define LRC_STATE_PN (LRC_PPHWSP_PN + 1)
+#define LRC_GUCSHR_SZ (1)
+/* At the start of the context image is its per-process HWS page */
+#define LRC_PPHWSP_PN (LRC_GUCSHR_PN + LRC_GUCSHR_SZ)
+#define LRC_PPHWSP_SZ (1)
+/* Finally we have the logical state for the context */
+#define LRC_STATE_PN (LRC_PPHWSP_PN + LRC_PPHWSP_SZ)
+
+/*
+ * Currently we include the PPHWSP in __intel_engine_context_size() so
+ * the size of the header is synonymous with the start of the PPHWSP.
+ */
+#define LRC_HEADER_PAGES LRC_PPHWSP_PN
struct drm_i915_private;
struct i915_gem_context;
void intel_lr_context_resume(struct drm_i915_private *dev_priv);
-uint64_t intel_lr_context_descriptor(struct i915_gem_context *ctx,
- struct intel_engine_cs *engine);
+
+static inline uint64_t
+intel_lr_context_descriptor(struct i915_gem_context *ctx,
+ struct intel_engine_cs *engine)
+{
+ return ctx->engine[engine->id].lrc_desc;
+}
+
/* Execlists */
int intel_sanitize_enable_execlists(struct drm_i915_private *dev_priv,
struct drm_i915_private *dev_priv = to_i915(dev);
/* use the module option value if specified */
- if (i915.lvds_channel_mode > 0)
- return i915.lvds_channel_mode == 2;
+ if (i915_modparams.lvds_channel_mode > 0)
+ return i915_modparams.lvds_channel_mode == 2;
/* single channel LVDS is limited to 112 MHz */
if (lvds_encoder->attached_connector->base.panel.fixed_mode->clock
#include "intel_drv.h"
#include "i915_drv.h"
+static void intel_connector_update_eld_conn_type(struct drm_connector *connector)
+{
+ u8 conn_type;
+
+ if (connector->connector_type == DRM_MODE_CONNECTOR_DisplayPort ||
+ connector->connector_type == DRM_MODE_CONNECTOR_eDP) {
+ conn_type = DRM_ELD_CONN_TYPE_DP;
+ } else {
+ conn_type = DRM_ELD_CONN_TYPE_HDMI;
+ }
+
+ connector->eld[DRM_ELD_SAD_COUNT_CONN_TYPE] &= ~DRM_ELD_CONN_TYPE_MASK;
+ connector->eld[DRM_ELD_SAD_COUNT_CONN_TYPE] |= conn_type;
+}
+
/**
* intel_connector_update_modes - update connector from edid
* @connector: DRM connector device to use
ret = drm_add_edid_modes(connector, edid);
drm_edid_to_eld(connector, edid);
+ intel_connector_update_eld_conn_type(connector);
+
return ret;
}
{
struct intel_opregion *opregion = &dev_priv->opregion;
const struct firmware *fw = NULL;
- const char *name = i915.vbt_firmware;
+ const char *name = i915_modparams.vbt_firmware;
int ret;
if (!name || !*name)
intel_panel_detect(struct drm_i915_private *dev_priv)
{
/* Assume that the BIOS does not lie through the OpRegion... */
- if (!i915.panel_ignore_lid && dev_priv->opregion.lid_state) {
+ if (!i915_modparams.panel_ignore_lid && dev_priv->opregion.lid_state) {
return *dev_priv->opregion.lid_state & 0x1 ?
connector_status_connected :
connector_status_disconnected;
}
- switch (i915.panel_ignore_lid) {
+ switch (i915_modparams.panel_ignore_lid) {
case -2:
return connector_status_connected;
case -1:
WARN_ON(panel->backlight.max == 0);
- if (i915.invert_brightness < 0)
+ if (i915_modparams.invert_brightness < 0)
return val;
- if (i915.invert_brightness > 0 ||
+ if (i915_modparams.invert_brightness > 0 ||
dev_priv->quirks & QUIRK_INVERT_BRIGHTNESS) {
return panel->backlight.max - val + panel->backlight.min;
}
static void gen9_init_clock_gating(struct drm_i915_private *dev_priv)
{
+ if (HAS_LLC(dev_priv)) {
+ /*
+ * WaCompressedResourceDisplayNewHashMode:skl,kbl
+ * Display WA#0390: skl,kbl
+ *
+ * Must match Sampler, Pixel Back End, and Media. See
+ * WaCompressedResourceSamplerPbeMediaNewHashMode.
+ */
+ I915_WRITE(CHICKEN_PAR1_1,
+ I915_READ(CHICKEN_PAR1_1) |
+ SKL_DE_COMPRESSED_HASH_MODE);
+ }
+
/* See Bspec note for PSR2_CTL bit 31, Wa#828:skl,bxt,kbl,cfl */
I915_WRITE(CHICKEN_PAR1_1,
I915_READ(CHICKEN_PAR1_1) | SKL_EDP_PSR_FIX_RDWRAP);
- /*
- * Display WA#0390: skl,bxt,kbl,glk
- *
- * Must match Sampler, Pixel Back End, and Media
- * (0xE194 bit 8, 0x7014 bit 13, 0x4DDC bits 27 and 31).
- *
- * Including bits outside the page in the hash would
- * require 2 (or 4?) MiB alignment of resources. Just
- * assume the defaul hashing mode which only uses bits
- * within the page.
- */
- I915_WRITE(CHICKEN_PAR1_1,
- I915_READ(CHICKEN_PAR1_1) & ~SKL_RC_HASH_OUTSIDE);
-
I915_WRITE(GEN8_CONFIG0,
I915_READ(GEN8_CONFIG0) | GEN9_DEFAULT_FIXES);
downscale_amount);
}
-static int skl_compute_plane_wm(const struct drm_i915_private *dev_priv,
- struct intel_crtc_state *cstate,
- const struct intel_plane_state *intel_pstate,
- uint16_t ddb_allocation,
- int level,
- uint16_t *out_blocks, /* out */
- uint8_t *out_lines, /* out */
- bool *enabled /* out */)
+static int
+skl_compute_plane_wm_params(const struct drm_i915_private *dev_priv,
+ struct intel_crtc_state *cstate,
+ const struct intel_plane_state *intel_pstate,
+ struct skl_wm_params *wp)
{
struct intel_plane *plane = to_intel_plane(intel_pstate->base.plane);
const struct drm_plane_state *pstate = &intel_pstate->base;
const struct drm_framebuffer *fb = pstate->fb;
- uint32_t latency = dev_priv->wm.skl_latency[level];
- uint_fixed_16_16_t method1, method2;
- uint_fixed_16_16_t plane_blocks_per_line;
- uint_fixed_16_16_t selected_result;
uint32_t interm_pbpl;
- uint32_t plane_bytes_per_line;
- uint32_t res_blocks, res_lines;
- uint8_t cpp;
- uint32_t width = 0;
- uint32_t plane_pixel_rate;
- uint_fixed_16_16_t y_tile_minimum;
- uint32_t y_min_scanlines;
struct intel_atomic_state *state =
to_intel_atomic_state(cstate->base.state);
bool apply_memory_bw_wa = skl_needs_memory_bw_wa(state);
- bool y_tiled, x_tiled;
- if (latency == 0 ||
- !intel_wm_plane_visible(cstate, intel_pstate)) {
- *enabled = false;
+ if (!intel_wm_plane_visible(cstate, intel_pstate))
return 0;
- }
- y_tiled = fb->modifier == I915_FORMAT_MOD_Y_TILED ||
- fb->modifier == I915_FORMAT_MOD_Yf_TILED ||
- fb->modifier == I915_FORMAT_MOD_Y_TILED_CCS ||
- fb->modifier == I915_FORMAT_MOD_Yf_TILED_CCS;
- x_tiled = fb->modifier == I915_FORMAT_MOD_X_TILED;
-
- /* Display WA #1141: kbl,cfl */
- if ((IS_KABYLAKE(dev_priv) || IS_COFFEELAKE(dev_priv)) &&
- dev_priv->ipc_enabled)
- latency += 4;
-
- if (apply_memory_bw_wa && x_tiled)
- latency += 15;
+ wp->y_tiled = fb->modifier == I915_FORMAT_MOD_Y_TILED ||
+ fb->modifier == I915_FORMAT_MOD_Yf_TILED ||
+ fb->modifier == I915_FORMAT_MOD_Y_TILED_CCS ||
+ fb->modifier == I915_FORMAT_MOD_Yf_TILED_CCS;
+ wp->x_tiled = fb->modifier == I915_FORMAT_MOD_X_TILED;
+ wp->rc_surface = fb->modifier == I915_FORMAT_MOD_Y_TILED_CCS ||
+ fb->modifier == I915_FORMAT_MOD_Yf_TILED_CCS;
if (plane->id == PLANE_CURSOR) {
- width = intel_pstate->base.crtc_w;
+ wp->width = intel_pstate->base.crtc_w;
} else {
/*
* Src coordinates are already rotated by 270 degrees for
* the 90/270 degree plane rotation cases (to match the
* GTT mapping), hence no need to account for rotation here.
*/
- width = drm_rect_width(&intel_pstate->base.src) >> 16;
+ wp->width = drm_rect_width(&intel_pstate->base.src) >> 16;
}
- cpp = (fb->format->format == DRM_FORMAT_NV12) ? fb->format->cpp[1] :
- fb->format->cpp[0];
- plane_pixel_rate = skl_adjusted_plane_pixel_rate(cstate, intel_pstate);
+ wp->cpp = (fb->format->format == DRM_FORMAT_NV12) ? fb->format->cpp[1] :
+ fb->format->cpp[0];
+ wp->plane_pixel_rate = skl_adjusted_plane_pixel_rate(cstate,
+ intel_pstate);
if (drm_rotation_90_or_270(pstate->rotation)) {
- switch (cpp) {
+ switch (wp->cpp) {
case 1:
- y_min_scanlines = 16;
+ wp->y_min_scanlines = 16;
break;
case 2:
- y_min_scanlines = 8;
+ wp->y_min_scanlines = 8;
break;
case 4:
- y_min_scanlines = 4;
+ wp->y_min_scanlines = 4;
break;
default:
- MISSING_CASE(cpp);
+ MISSING_CASE(wp->cpp);
return -EINVAL;
}
} else {
- y_min_scanlines = 4;
+ wp->y_min_scanlines = 4;
}
if (apply_memory_bw_wa)
- y_min_scanlines *= 2;
+ wp->y_min_scanlines *= 2;
- plane_bytes_per_line = width * cpp;
- if (y_tiled) {
- interm_pbpl = DIV_ROUND_UP(plane_bytes_per_line *
- y_min_scanlines, 512);
+ wp->plane_bytes_per_line = wp->width * wp->cpp;
+ if (wp->y_tiled) {
+ interm_pbpl = DIV_ROUND_UP(wp->plane_bytes_per_line *
+ wp->y_min_scanlines, 512);
if (INTEL_GEN(dev_priv) >= 10)
interm_pbpl++;
- plane_blocks_per_line = div_fixed16(interm_pbpl,
- y_min_scanlines);
- } else if (x_tiled && INTEL_GEN(dev_priv) == 9) {
- interm_pbpl = DIV_ROUND_UP(plane_bytes_per_line, 512);
- plane_blocks_per_line = u32_to_fixed16(interm_pbpl);
+ wp->plane_blocks_per_line = div_fixed16(interm_pbpl,
+ wp->y_min_scanlines);
+ } else if (wp->x_tiled && IS_GEN9(dev_priv)) {
+ interm_pbpl = DIV_ROUND_UP(wp->plane_bytes_per_line, 512);
+ wp->plane_blocks_per_line = u32_to_fixed16(interm_pbpl);
} else {
- interm_pbpl = DIV_ROUND_UP(plane_bytes_per_line, 512) + 1;
- plane_blocks_per_line = u32_to_fixed16(interm_pbpl);
+ interm_pbpl = DIV_ROUND_UP(wp->plane_bytes_per_line, 512) + 1;
+ wp->plane_blocks_per_line = u32_to_fixed16(interm_pbpl);
+ }
+
+ wp->y_tile_minimum = mul_u32_fixed16(wp->y_min_scanlines,
+ wp->plane_blocks_per_line);
+ wp->linetime_us = fixed16_to_u32_round_up(
+ intel_get_linetime_us(cstate));
+
+ return 0;
+}
+
+static int skl_compute_plane_wm(const struct drm_i915_private *dev_priv,
+ struct intel_crtc_state *cstate,
+ const struct intel_plane_state *intel_pstate,
+ uint16_t ddb_allocation,
+ int level,
+ const struct skl_wm_params *wp,
+ uint16_t *out_blocks, /* out */
+ uint8_t *out_lines, /* out */
+ bool *enabled /* out */)
+{
+ const struct drm_plane_state *pstate = &intel_pstate->base;
+ uint32_t latency = dev_priv->wm.skl_latency[level];
+ uint_fixed_16_16_t method1, method2;
+ uint_fixed_16_16_t selected_result;
+ uint32_t res_blocks, res_lines;
+ struct intel_atomic_state *state =
+ to_intel_atomic_state(cstate->base.state);
+ bool apply_memory_bw_wa = skl_needs_memory_bw_wa(state);
+
+ if (latency == 0 ||
+ !intel_wm_plane_visible(cstate, intel_pstate)) {
+ *enabled = false;
+ return 0;
}
- method1 = skl_wm_method1(dev_priv, plane_pixel_rate, cpp, latency);
- method2 = skl_wm_method2(plane_pixel_rate,
+ /* Display WA #1141: kbl,cfl */
+ if ((IS_KABYLAKE(dev_priv) || IS_COFFEELAKE(dev_priv) ||
+ IS_CNL_REVID(dev_priv, CNL_REVID_A0, CNL_REVID_B0)) &&
+ dev_priv->ipc_enabled)
+ latency += 4;
+
+ if (apply_memory_bw_wa && wp->x_tiled)
+ latency += 15;
+
+ method1 = skl_wm_method1(dev_priv, wp->plane_pixel_rate,
+ wp->cpp, latency);
+ method2 = skl_wm_method2(wp->plane_pixel_rate,
cstate->base.adjusted_mode.crtc_htotal,
latency,
- plane_blocks_per_line);
+ wp->plane_blocks_per_line);
- y_tile_minimum = mul_u32_fixed16(y_min_scanlines,
- plane_blocks_per_line);
-
- if (y_tiled) {
- selected_result = max_fixed16(method2, y_tile_minimum);
+ if (wp->y_tiled) {
+ selected_result = max_fixed16(method2, wp->y_tile_minimum);
} else {
- uint32_t linetime_us;
-
- linetime_us = fixed16_to_u32_round_up(
- intel_get_linetime_us(cstate));
- if ((cpp * cstate->base.adjusted_mode.crtc_htotal / 512 < 1) &&
- (plane_bytes_per_line / 512 < 1))
+ if ((wp->cpp * cstate->base.adjusted_mode.crtc_htotal /
+ 512 < 1) && (wp->plane_bytes_per_line / 512 < 1))
selected_result = method2;
else if (ddb_allocation >=
- fixed16_to_u32_round_up(plane_blocks_per_line))
+ fixed16_to_u32_round_up(wp->plane_blocks_per_line))
selected_result = min_fixed16(method1, method2);
- else if (latency >= linetime_us)
+ else if (latency >= wp->linetime_us)
selected_result = min_fixed16(method1, method2);
else
selected_result = method1;
res_blocks = fixed16_to_u32_round_up(selected_result) + 1;
res_lines = div_round_up_fixed16(selected_result,
- plane_blocks_per_line);
+ wp->plane_blocks_per_line);
/* Display WA #1125: skl,bxt,kbl,glk */
- if (level == 0 &&
- (fb->modifier == I915_FORMAT_MOD_Y_TILED_CCS ||
- fb->modifier == I915_FORMAT_MOD_Yf_TILED_CCS))
- res_blocks += fixed16_to_u32_round_up(y_tile_minimum);
+ if (level == 0 && wp->rc_surface)
+ res_blocks += fixed16_to_u32_round_up(wp->y_tile_minimum);
/* Display WA #1126: skl,bxt,kbl,glk */
if (level >= 1 && level <= 7) {
- if (y_tiled) {
- res_blocks += fixed16_to_u32_round_up(y_tile_minimum);
- res_lines += y_min_scanlines;
+ if (wp->y_tiled) {
+ res_blocks += fixed16_to_u32_round_up(
+ wp->y_tile_minimum);
+ res_lines += wp->y_min_scanlines;
} else {
res_blocks++;
}
struct skl_ddb_allocation *ddb,
struct intel_crtc_state *cstate,
const struct intel_plane_state *intel_pstate,
+ const struct skl_wm_params *wm_params,
struct skl_plane_wm *wm)
{
struct intel_crtc *intel_crtc = to_intel_crtc(cstate->base.crtc);
intel_pstate,
ddb_blocks,
level,
+ wm_params,
&result->plane_res_b,
&result->plane_res_l,
&result->plane_en);
linetime_wm = fixed16_to_u32_round_up(mul_u32_fixed16(8, linetime_us));
- /* Display WA #1135: bxt. */
- if (IS_BROXTON(dev_priv) && dev_priv->ipc_enabled)
- linetime_wm = DIV_ROUND_UP(linetime_wm, 2);
+ /* Display WA #1135: bxt:ALL GLK:ALL */
+ if ((IS_BROXTON(dev_priv) || IS_GEMINILAKE(dev_priv)) &&
+ dev_priv->ipc_enabled)
+ linetime_wm /= 2;
return linetime_wm;
}
static void skl_compute_transition_wm(struct intel_crtc_state *cstate,
+ struct skl_wm_params *wp,
+ struct skl_wm_level *wm_l0,
+ uint16_t ddb_allocation,
struct skl_wm_level *trans_wm /* out */)
{
+ struct drm_device *dev = cstate->base.crtc->dev;
+ const struct drm_i915_private *dev_priv = to_i915(dev);
+ uint16_t trans_min, trans_y_tile_min;
+ const uint16_t trans_amount = 10; /* This is configurable amount */
+ uint16_t trans_offset_b, res_blocks;
+
if (!cstate->base.active)
+ goto exit;
+
+ /* Transition WM are not recommended by HW team for GEN9 */
+ if (INTEL_GEN(dev_priv) <= 9)
+ goto exit;
+
+ /* Transition WM don't make any sense if ipc is disabled */
+ if (!dev_priv->ipc_enabled)
+ goto exit;
+
+ if (INTEL_GEN(dev_priv) >= 10)
+ trans_min = 4;
+
+ trans_offset_b = trans_min + trans_amount;
+
+ if (wp->y_tiled) {
+ trans_y_tile_min = (uint16_t) mul_round_up_u32_fixed16(2,
+ wp->y_tile_minimum);
+ res_blocks = max(wm_l0->plane_res_b, trans_y_tile_min) +
+ trans_offset_b;
+ } else {
+ res_blocks = wm_l0->plane_res_b + trans_offset_b;
+
+ /* WA BUG:1938466 add one block for non y-tile planes */
+ if (IS_CNL_REVID(dev_priv, CNL_REVID_A0, CNL_REVID_A0))
+ res_blocks += 1;
+
+ }
+
+ res_blocks += 1;
+
+ if (res_blocks < ddb_allocation) {
+ trans_wm->plane_res_b = res_blocks;
+ trans_wm->plane_en = true;
return;
+ }
- /* Until we know more, just disable transition WMs */
+exit:
trans_wm->plane_en = false;
}
const struct intel_plane_state *intel_pstate =
to_intel_plane_state(pstate);
enum plane_id plane_id = to_intel_plane(plane)->id;
+ struct skl_wm_params wm_params;
+ enum pipe pipe = to_intel_crtc(cstate->base.crtc)->pipe;
+ uint16_t ddb_blocks;
wm = &pipe_wm->planes[plane_id];
+ ddb_blocks = skl_ddb_entry_size(&ddb->plane[pipe][plane_id]);
+ memset(&wm_params, 0, sizeof(struct skl_wm_params));
+
+ ret = skl_compute_plane_wm_params(dev_priv, cstate,
+ intel_pstate, &wm_params);
+ if (ret)
+ return ret;
ret = skl_compute_wm_levels(dev_priv, ddb, cstate,
- intel_pstate, wm);
+ intel_pstate, &wm_params, wm);
if (ret)
return ret;
- skl_compute_transition_wm(cstate, &wm->trans_wm);
+ skl_compute_transition_wm(cstate, &wm_params, &wm->wm[0],
+ ddb_blocks, &wm->trans_wm);
}
pipe_wm->linetime = skl_compute_linetime_wm(cstate);
dev_priv->display.update_wm(crtc);
}
+void intel_enable_ipc(struct drm_i915_private *dev_priv)
+{
+ u32 val;
+
+ val = I915_READ(DISP_ARB_CTL2);
+
+ if (dev_priv->ipc_enabled)
+ val |= DISP_IPC_ENABLE;
+ else
+ val &= ~DISP_IPC_ENABLE;
+
+ I915_WRITE(DISP_ARB_CTL2, val);
+}
+
+void intel_init_ipc(struct drm_i915_private *dev_priv)
+{
+ dev_priv->ipc_enabled = false;
+ if (!HAS_IPC(dev_priv))
+ return;
+
+ dev_priv->ipc_enabled = true;
+ intel_enable_ipc(dev_priv);
+}
+
/*
* Lock protecting IPS related data structures
*/
* RPM depends on RC6 to save restore the GT HW context, so make RC6 a
* requirement.
*/
- if (!i915.enable_rc6) {
+ if (!i915_modparams.enable_rc6) {
DRM_INFO("RC6 disabled, disabling runtime PM support\n");
intel_runtime_pm_get(dev_priv);
}
if (IS_VALLEYVIEW(dev_priv))
valleyview_cleanup_gt_powersave(dev_priv);
- if (!i915.enable_rc6)
+ if (!i915_modparams.enable_rc6)
intel_runtime_pm_put(dev_priv);
}
if (IS_ERR(req))
goto unlock;
- if (!i915.enable_execlists && i915_switch_context(req) == 0)
+ if (!i915_modparams.enable_execlists && i915_switch_context(req) == 0)
rcs->init_context(req);
/* Mark the device busy, calling intel_enable_gt_powersave() */
return;
/* Wa #1181 */
- I915_WRITE(SOUTH_DSPCLK_GATE_D, CNP_PWM_CGE_GATING_DISABLE);
+ I915_WRITE(SOUTH_DSPCLK_GATE_D, I915_READ(SOUTH_DSPCLK_GATE_D) |
+ CNP_PWM_CGE_GATING_DISABLE);
}
static void cnl_init_clock_gating(struct drm_i915_private *dev_priv)
I915_WRITE(VLV_VSCSDP(crtc->pipe), val);
}
-static void skl_psr_setup_su_vsc(struct intel_dp *intel_dp,
- const struct intel_crtc_state *crtc_state)
+static void hsw_psr_setup_vsc(struct intel_dp *intel_dp,
+ const struct intel_crtc_state *crtc_state)
{
struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
struct drm_i915_private *dev_priv = to_i915(intel_dig_port->base.base.dev);
struct edp_vsc_psr psr_vsc;
- /* Prepare VSC Header for SU as per EDP 1.4 spec, Table 6.11 */
- memset(&psr_vsc, 0, sizeof(psr_vsc));
- psr_vsc.sdp_header.HB0 = 0;
- psr_vsc.sdp_header.HB1 = 0x7;
- if (dev_priv->psr.colorimetry_support &&
- dev_priv->psr.y_cord_support) {
- psr_vsc.sdp_header.HB2 = 0x5;
- psr_vsc.sdp_header.HB3 = 0x13;
- } else if (dev_priv->psr.y_cord_support) {
- psr_vsc.sdp_header.HB2 = 0x4;
- psr_vsc.sdp_header.HB3 = 0xe;
+ if (dev_priv->psr.psr2_support) {
+ /* Prepare VSC Header for SU as per EDP 1.4 spec, Table 6.11 */
+ memset(&psr_vsc, 0, sizeof(psr_vsc));
+ psr_vsc.sdp_header.HB0 = 0;
+ psr_vsc.sdp_header.HB1 = 0x7;
+ if (dev_priv->psr.colorimetry_support &&
+ dev_priv->psr.y_cord_support) {
+ psr_vsc.sdp_header.HB2 = 0x5;
+ psr_vsc.sdp_header.HB3 = 0x13;
+ } else if (dev_priv->psr.y_cord_support) {
+ psr_vsc.sdp_header.HB2 = 0x4;
+ psr_vsc.sdp_header.HB3 = 0xe;
+ } else {
+ psr_vsc.sdp_header.HB2 = 0x3;
+ psr_vsc.sdp_header.HB3 = 0xc;
+ }
} else {
- psr_vsc.sdp_header.HB2 = 0x3;
- psr_vsc.sdp_header.HB3 = 0xc;
+ /* Prepare VSC packet as per EDP 1.3 spec, Table 3.10 */
+ memset(&psr_vsc, 0, sizeof(psr_vsc));
+ psr_vsc.sdp_header.HB0 = 0;
+ psr_vsc.sdp_header.HB1 = 0x7;
+ psr_vsc.sdp_header.HB2 = 0x2;
+ psr_vsc.sdp_header.HB3 = 0x8;
}
intel_psr_write_vsc(intel_dp, &psr_vsc);
}
-static void hsw_psr_setup_vsc(struct intel_dp *intel_dp,
- const struct intel_crtc_state *crtc_state)
-{
- struct edp_vsc_psr psr_vsc;
-
- /* Prepare VSC packet as per EDP 1.3 spec, Table 3.10 */
- memset(&psr_vsc, 0, sizeof(psr_vsc));
- psr_vsc.sdp_header.HB0 = 0;
- psr_vsc.sdp_header.HB1 = 0x7;
- psr_vsc.sdp_header.HB2 = 0x2;
- psr_vsc.sdp_header.HB3 = 0x8;
- intel_psr_write_vsc(intel_dp, &psr_vsc);
-}
-
static void vlv_psr_enable_sink(struct intel_dp *intel_dp)
{
drm_dp_dpcd_writeb(&intel_dp->aux, DP_PSR_EN_CFG,
struct drm_i915_private *dev_priv = to_i915(dig_port->base.base.dev);
struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
- /* Transition from PSR_state 0 to PSR_state 1, i.e. PSR Inactive */
+ /* Transition from PSR_state 0 (disabled) to PSR_state 1 (inactive) */
I915_WRITE(VLV_PSRCTL(crtc->pipe),
VLV_EDP_PSR_MODE_SW_TIMER |
VLV_EDP_PSR_SRC_TRANSMITTER_STATE |
struct drm_crtc *crtc = dig_port->base.base.crtc;
enum pipe pipe = to_intel_crtc(crtc)->pipe;
- /* Let's do the transition from PSR_state 1 to PSR_state 2
- * that is PSR transition to active - static frame transmission.
- * Then Hardware is responsible for the transition to PSR_state 3
- * that is PSR active - no Remote Frame Buffer (RFB) update.
+ /*
+ * Let's do the transition from PSR_state 1 (inactive) to
+ * PSR_state 2 (transition to active - static frame transmission).
+ * Then Hardware is responsible for the transition to
+ * PSR_state 3 (active - no Remote Frame Buffer (RFB) update).
*/
I915_WRITE(VLV_PSRCTL(pipe), I915_READ(VLV_PSRCTL(pipe)) |
VLV_EDP_PSR_ACTIVE_ENTRY);
}
-static void intel_enable_source_psr1(struct intel_dp *intel_dp)
+static void hsw_activate_psr1(struct intel_dp *intel_dp)
{
struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);
struct drm_device *dev = dig_port->base.base.dev;
I915_WRITE(EDP_PSR_CTL, val);
}
-static void intel_enable_source_psr2(struct intel_dp *intel_dp)
+static void hsw_activate_psr2(struct intel_dp *intel_dp)
{
struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);
struct drm_device *dev = dig_port->base.base.dev;
I915_WRITE(EDP_PSR2_CTL, val);
}
-static void hsw_psr_enable_source(struct intel_dp *intel_dp)
+static void hsw_psr_activate(struct intel_dp *intel_dp)
{
struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);
struct drm_device *dev = dig_port->base.base.dev;
struct drm_i915_private *dev_priv = to_i915(dev);
+ /* On HSW+ after we enable PSR on source it will activate it
+ * as soon as it match configure idle_frame count. So
+ * we just actually enable it here on activation time.
+ */
+
/* psr1 and psr2 are mutually exclusive.*/
if (dev_priv->psr.psr2_support)
- intel_enable_source_psr2(intel_dp);
+ hsw_activate_psr2(intel_dp);
else
- intel_enable_source_psr1(intel_dp);
+ hsw_activate_psr1(intel_dp);
}
static bool intel_psr_match_conditions(struct intel_dp *intel_dp)
return false;
}
- if (!i915.enable_psr) {
+ if (!i915_modparams.enable_psr) {
DRM_DEBUG_KMS("PSR disable by flag\n");
return false;
}
WARN_ON(dev_priv->psr.active);
lockdep_assert_held(&dev_priv->psr.lock);
- /* Enable/Re-enable PSR on the host */
- if (HAS_DDI(dev_priv))
- /* On HSW+ after we enable PSR on source it will activate it
- * as soon as it match configure idle_frame count. So
- * we just actually enable it here on activation time.
- */
- hsw_psr_enable_source(intel_dp);
- else
- vlv_psr_activate(intel_dp);
-
+ dev_priv->psr.activate(intel_dp);
dev_priv->psr.active = true;
}
+static void hsw_psr_enable_source(struct intel_dp *intel_dp,
+ const struct intel_crtc_state *crtc_state)
+{
+ struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);
+ struct drm_device *dev = dig_port->base.base.dev;
+ struct drm_i915_private *dev_priv = to_i915(dev);
+ enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
+ u32 chicken;
+
+ if (dev_priv->psr.psr2_support) {
+ chicken = PSR2_VSC_ENABLE_PROG_HEADER;
+ if (dev_priv->psr.y_cord_support)
+ chicken |= PSR2_ADD_VERTICAL_LINE_COUNT;
+ I915_WRITE(CHICKEN_TRANS(cpu_transcoder), chicken);
+
+ I915_WRITE(EDP_PSR_DEBUG_CTL,
+ EDP_PSR_DEBUG_MASK_MEMUP |
+ EDP_PSR_DEBUG_MASK_HPD |
+ EDP_PSR_DEBUG_MASK_LPSP |
+ EDP_PSR_DEBUG_MASK_MAX_SLEEP |
+ EDP_PSR_DEBUG_MASK_DISP_REG_WRITE);
+ } else {
+ /*
+ * Per Spec: Avoid continuous PSR exit by masking MEMUP
+ * and HPD. also mask LPSP to avoid dependency on other
+ * drivers that might block runtime_pm besides
+ * preventing other hw tracking issues now we can rely
+ * on frontbuffer tracking.
+ */
+ I915_WRITE(EDP_PSR_DEBUG_CTL,
+ EDP_PSR_DEBUG_MASK_MEMUP |
+ EDP_PSR_DEBUG_MASK_HPD |
+ EDP_PSR_DEBUG_MASK_LPSP);
+ }
+}
+
/**
* intel_psr_enable - Enable PSR
* @intel_dp: Intel DP
struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
struct drm_device *dev = intel_dig_port->base.base.dev;
struct drm_i915_private *dev_priv = to_i915(dev);
- enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
- u32 chicken;
- if (!HAS_PSR(dev_priv)) {
- DRM_DEBUG_KMS("PSR not supported on this platform\n");
+ if (!HAS_PSR(dev_priv))
return;
- }
if (!is_edp_psr(intel_dp)) {
DRM_DEBUG_KMS("PSR not supported by this panel\n");
return;
}
+ WARN_ON(dev_priv->drrs.dp);
mutex_lock(&dev_priv->psr.lock);
if (dev_priv->psr.enabled) {
DRM_DEBUG_KMS("PSR already in use\n");
dev_priv->psr.busy_frontbuffer_bits = 0;
- if (HAS_DDI(dev_priv)) {
- if (dev_priv->psr.psr2_support) {
- skl_psr_setup_su_vsc(intel_dp, crtc_state);
-
- chicken = PSR2_VSC_ENABLE_PROG_HEADER;
- if (dev_priv->psr.y_cord_support)
- chicken |= PSR2_ADD_VERTICAL_LINE_COUNT;
- I915_WRITE(CHICKEN_TRANS(cpu_transcoder), chicken);
-
- I915_WRITE(EDP_PSR_DEBUG_CTL,
- EDP_PSR_DEBUG_MASK_MEMUP |
- EDP_PSR_DEBUG_MASK_HPD |
- EDP_PSR_DEBUG_MASK_LPSP |
- EDP_PSR_DEBUG_MASK_MAX_SLEEP |
- EDP_PSR_DEBUG_MASK_DISP_REG_WRITE);
- } else {
- /* set up vsc header for psr1 */
- hsw_psr_setup_vsc(intel_dp, crtc_state);
-
- /*
- * Per Spec: Avoid continuous PSR exit by masking MEMUP
- * and HPD. also mask LPSP to avoid dependency on other
- * drivers that might block runtime_pm besides
- * preventing other hw tracking issues now we can rely
- * on frontbuffer tracking.
- */
- I915_WRITE(EDP_PSR_DEBUG_CTL,
- EDP_PSR_DEBUG_MASK_MEMUP |
- EDP_PSR_DEBUG_MASK_HPD |
- EDP_PSR_DEBUG_MASK_LPSP);
- }
-
- /* Enable PSR on the panel */
- hsw_psr_enable_sink(intel_dp);
+ dev_priv->psr.setup_vsc(intel_dp, crtc_state);
+ dev_priv->psr.enable_sink(intel_dp);
+ dev_priv->psr.enable_source(intel_dp, crtc_state);
+ dev_priv->psr.enabled = intel_dp;
- if (INTEL_GEN(dev_priv) >= 9)
- intel_psr_activate(intel_dp);
+ if (INTEL_GEN(dev_priv) >= 9) {
+ intel_psr_activate(intel_dp);
} else {
- vlv_psr_setup_vsc(intel_dp, crtc_state);
-
- /* Enable PSR on the panel */
- vlv_psr_enable_sink(intel_dp);
-
- /* On HSW+ enable_source also means go to PSR entry/active
- * state as soon as idle_frame achieved and here would be
- * to soon. However on VLV enable_source just enable PSR
- * but let it on inactive state. So we might do this prior
- * to active transition, i.e. here.
+ /*
+ * FIXME: Activation should happen immediately since this
+ * function is just called after pipe is fully trained and
+ * enabled.
+ * However on some platforms we face issues when first
+ * activation follows a modeset so quickly.
+ * - On VLV/CHV we get bank screen on first activation
+ * - On HSW/BDW we get a recoverable frozen screen until
+ * next exit-activate sequence.
*/
- vlv_psr_enable_source(intel_dp, crtc_state);
- }
-
- /*
- * FIXME: Activation should happen immediately since this function
- * is just called after pipe is fully trained and enabled.
- * However on every platform we face issues when first activation
- * follows a modeset so quickly.
- * - On VLV/CHV we get bank screen on first activation
- * - On HSW/BDW we get a recoverable frozen screen until next
- * exit-activate sequence.
- */
- if (INTEL_GEN(dev_priv) < 9)
schedule_delayed_work(&dev_priv->psr.work,
msecs_to_jiffies(intel_dp->panel_power_cycle_delay * 5));
+ }
- dev_priv->psr.enabled = intel_dp;
unlock:
mutex_unlock(&dev_priv->psr.lock);
}
uint32_t val;
if (dev_priv->psr.active) {
- /* Put VLV PSR back to PSR_state 0 that is PSR Disabled. */
+ /* Put VLV PSR back to PSR_state 0 (disabled). */
if (intel_wait_for_register(dev_priv,
VLV_PSRSTAT(crtc->pipe),
VLV_EDP_PSR_IN_TRANS,
struct drm_device *dev = intel_dig_port->base.base.dev;
struct drm_i915_private *dev_priv = to_i915(dev);
+ if (!HAS_PSR(dev_priv))
+ return;
+
mutex_lock(&dev_priv->psr.lock);
if (!dev_priv->psr.enabled) {
mutex_unlock(&dev_priv->psr.lock);
return;
}
- /* Disable PSR on Source */
- if (HAS_DDI(dev_priv))
- hsw_psr_disable(intel_dp, old_crtc_state);
- else
- vlv_psr_disable(intel_dp, old_crtc_state);
+ dev_priv->psr.disable_source(intel_dp, old_crtc_state);
/* Disable PSR on Sink */
drm_dp_dpcd_writeb(&intel_dp->aux, DP_PSR_EN_CFG, 0);
} else {
val = I915_READ(VLV_PSRCTL(pipe));
- /* Here we do the transition from PSR_state 3 to PSR_state 5
- * directly once PSR State 4 that is active with single frame
- * update can be skipped. PSR_state 5 that is PSR exit then
- * Hardware is responsible to transition back to PSR_state 1
- * that is PSR inactive. Same state after
- * vlv_edp_psr_enable_source.
+ /*
+ * Here we do the transition drirectly from
+ * PSR_state 3 (active - no Remote Frame Buffer (RFB) update) to
+ * PSR_state 5 (exit).
+ * PSR State 4 (active with single frame update) can be skipped.
+ * On PSR_state 5 (exit) Hardware is responsible to transition
+ * back to PSR_state 1 (inactive).
+ * Now we are at Same state after vlv_psr_enable_source.
*/
val &= ~VLV_EDP_PSR_ACTIVE_ENTRY;
I915_WRITE(VLV_PSRCTL(pipe), val);
- /* Send AUX wake up - Spec says after transitioning to PSR
+ /*
+ * Send AUX wake up - Spec says after transitioning to PSR
* active we have to send AUX wake up by writing 01h in DPCD
* 600h of sink device.
* XXX: This might slow down the transition, but without this
enum pipe pipe;
u32 val;
+ if (!HAS_PSR(dev_priv))
+ return;
+
/*
* Single frame update is already supported on BDW+ but it requires
* many W/A and it isn't really needed.
struct drm_crtc *crtc;
enum pipe pipe;
+ if (!HAS_PSR(dev_priv))
+ return;
+
mutex_lock(&dev_priv->psr.lock);
if (!dev_priv->psr.enabled) {
mutex_unlock(&dev_priv->psr.lock);
struct drm_crtc *crtc;
enum pipe pipe;
+ if (!HAS_PSR(dev_priv))
+ return;
+
mutex_lock(&dev_priv->psr.lock);
if (!dev_priv->psr.enabled) {
mutex_unlock(&dev_priv->psr.lock);
*/
void intel_psr_init(struct drm_i915_private *dev_priv)
{
+ if (!HAS_PSR(dev_priv))
+ return;
+
dev_priv->psr_mmio_base = IS_HASWELL(dev_priv) ?
HSW_EDP_PSR_BASE : BDW_EDP_PSR_BASE;
/* Per platform default: all disabled. */
- if (i915.enable_psr == -1)
- i915.enable_psr = 0;
+ if (i915_modparams.enable_psr == -1)
+ i915_modparams.enable_psr = 0;
/* Set link_standby x link_off defaults */
if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
dev_priv->psr.link_standby = dev_priv->vbt.psr.full_link;
/* Override link_standby x link_off defaults */
- if (i915.enable_psr == 2 && !dev_priv->psr.link_standby) {
+ if (i915_modparams.enable_psr == 2 && !dev_priv->psr.link_standby) {
DRM_DEBUG_KMS("PSR: Forcing link standby\n");
dev_priv->psr.link_standby = true;
}
- if (i915.enable_psr == 3 && dev_priv->psr.link_standby) {
+ if (i915_modparams.enable_psr == 3 && dev_priv->psr.link_standby) {
DRM_DEBUG_KMS("PSR: Forcing main link off\n");
dev_priv->psr.link_standby = false;
}
INIT_DELAYED_WORK(&dev_priv->psr.work, intel_psr_work);
mutex_init(&dev_priv->psr.lock);
+
+ if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
+ dev_priv->psr.enable_source = vlv_psr_enable_source;
+ dev_priv->psr.disable_source = vlv_psr_disable;
+ dev_priv->psr.enable_sink = vlv_psr_enable_sink;
+ dev_priv->psr.activate = vlv_psr_activate;
+ dev_priv->psr.setup_vsc = vlv_psr_setup_vsc;
+ } else {
+ dev_priv->psr.enable_source = hsw_psr_enable_source;
+ dev_priv->psr.disable_source = hsw_psr_disable;
+ dev_priv->psr.enable_sink = hsw_psr_enable_sink;
+ dev_priv->psr.activate = hsw_psr_activate;
+ dev_priv->psr.setup_vsc = hsw_psr_setup_vsc;
+ }
}
*/
if (IS_GEN7(dev_priv)) {
switch (engine->id) {
+ /*
+ * No more rings exist on Gen7. Default case is only to shut up
+ * gcc switch check warning.
+ */
+ default:
+ GEM_BUG_ON(engine->id);
case RCS:
mmio = RENDER_HWS_PGA_GEN7;
break;
case BCS:
mmio = BLT_HWS_PGA_GEN7;
break;
- /*
- * VCS2 actually doesn't exist on Gen7. Only shut up
- * gcc switch check warning
- */
- case VCS2:
case VCS:
mmio = BSD_HWS_PGA_GEN7;
break;
mmio = RING_HWS_PGA(engine->mmio_base);
}
+ if (INTEL_GEN(dev_priv) >= 6)
+ I915_WRITE(RING_HWSTAM(engine->mmio_base), 0xffffffff);
+
I915_WRITE(mmio, engine->status_page.ggtt_offset);
POSTING_READ(mmio);
return cs;
}
+static void cancel_requests(struct intel_engine_cs *engine)
+{
+ struct drm_i915_gem_request *request;
+ unsigned long flags;
+
+ spin_lock_irqsave(&engine->timeline->lock, flags);
+
+ /* Mark all submitted requests as skipped. */
+ list_for_each_entry(request, &engine->timeline->requests, link) {
+ GEM_BUG_ON(!request->global_seqno);
+ if (!i915_gem_request_completed(request))
+ dma_fence_set_error(&request->fence, -EIO);
+ }
+ /* Remaining _unready_ requests will be nop'ed when submitted */
+
+ spin_unlock_irqrestore(&engine->timeline->lock, flags);
+}
+
static void i9xx_submit_request(struct drm_i915_gem_request *request)
{
struct drm_i915_private *dev_priv = request->i915;
return 0;
}
-static void cleanup_phys_status_page(struct intel_engine_cs *engine)
-{
- struct drm_i915_private *dev_priv = engine->i915;
-
- if (!dev_priv->status_page_dmah)
- return;
-
- drm_pci_free(&dev_priv->drm, dev_priv->status_page_dmah);
- engine->status_page.page_addr = NULL;
-}
-
-static void cleanup_status_page(struct intel_engine_cs *engine)
-{
- struct i915_vma *vma;
- struct drm_i915_gem_object *obj;
-
- vma = fetch_and_zero(&engine->status_page.vma);
- if (!vma)
- return;
-
- obj = vma->obj;
-
- i915_vma_unpin(vma);
- i915_vma_close(vma);
-
- i915_gem_object_unpin_map(obj);
- __i915_gem_object_release_unless_active(obj);
-}
-
-static int init_status_page(struct intel_engine_cs *engine)
-{
- struct drm_i915_gem_object *obj;
- struct i915_vma *vma;
- unsigned int flags;
- void *vaddr;
- int ret;
-
- obj = i915_gem_object_create_internal(engine->i915, PAGE_SIZE);
- if (IS_ERR(obj)) {
- DRM_ERROR("Failed to allocate status page\n");
- return PTR_ERR(obj);
- }
- ret = i915_gem_object_set_cache_level(obj, I915_CACHE_LLC);
- if (ret)
- goto err;
-
- vma = i915_vma_instance(obj, &engine->i915->ggtt.base, NULL);
- if (IS_ERR(vma)) {
- ret = PTR_ERR(vma);
- goto err;
- }
-
- flags = PIN_GLOBAL;
- if (!HAS_LLC(engine->i915))
- /* On g33, we cannot place HWS above 256MiB, so
- * restrict its pinning to the low mappable arena.
- * Though this restriction is not documented for
- * gen4, gen5, or byt, they also behave similarly
- * and hang if the HWS is placed at the top of the
- * GTT. To generalise, it appears that all !llc
- * platforms have issues with us placing the HWS
- * above the mappable region (even though we never
- * actualy map it).
- */
- flags |= PIN_MAPPABLE;
- ret = i915_vma_pin(vma, 0, 4096, flags);
- if (ret)
- goto err;
-
- vaddr = i915_gem_object_pin_map(obj, I915_MAP_WB);
- if (IS_ERR(vaddr)) {
- ret = PTR_ERR(vaddr);
- goto err_unpin;
- }
-
- engine->status_page.vma = vma;
- engine->status_page.ggtt_offset = i915_ggtt_offset(vma);
- engine->status_page.page_addr = memset(vaddr, 0, PAGE_SIZE);
-
- DRM_DEBUG_DRIVER("%s hws offset: 0x%08x\n",
- engine->name, i915_ggtt_offset(vma));
- return 0;
-
-err_unpin:
- i915_vma_unpin(vma);
-err:
- i915_gem_object_put(obj);
- return ret;
-}
-
-static int init_phys_status_page(struct intel_engine_cs *engine)
-{
- struct drm_i915_private *dev_priv = engine->i915;
-
- GEM_BUG_ON(engine->id != RCS);
-
- dev_priv->status_page_dmah =
- drm_pci_alloc(&dev_priv->drm, PAGE_SIZE, PAGE_SIZE);
- if (!dev_priv->status_page_dmah)
- return -ENOMEM;
-
- engine->status_page.page_addr = dev_priv->status_page_dmah->vaddr;
- memset(engine->status_page.page_addr, 0, PAGE_SIZE);
-
- return 0;
-}
int intel_ring_pin(struct intel_ring *ring,
struct drm_i915_private *i915,
if (err)
goto err;
- if (HWS_NEEDS_PHYSICAL(engine->i915))
- err = init_phys_status_page(engine);
- else
- err = init_status_page(engine);
- if (err)
- goto err;
-
ring = intel_engine_create_ring(engine, 32 * PAGE_SIZE);
if (IS_ERR(ring)) {
err = PTR_ERR(ring);
- goto err_hws;
+ goto err;
}
/* Ring wraparound at offset 0 sometimes hangs. No idea why. */
err_ring:
intel_ring_free(ring);
-err_hws:
- if (HWS_NEEDS_PHYSICAL(engine->i915))
- cleanup_phys_status_page(engine);
- else
- cleanup_status_page(engine);
err:
intel_engine_cleanup_common(engine);
return err;
if (engine->cleanup)
engine->cleanup(engine);
- if (HWS_NEEDS_PHYSICAL(dev_priv))
- cleanup_phys_status_page(engine);
- else
- cleanup_status_page(engine);
-
intel_engine_cleanup_common(engine);
dev_priv->engine[engine->id] = NULL;
struct drm_i915_gem_object *obj;
int ret, i;
- if (!i915.semaphores)
+ if (!i915_modparams.semaphores)
return;
if (INTEL_GEN(dev_priv) >= 8 && !dev_priv->semaphore) {
i915_gem_object_put(obj);
err:
DRM_DEBUG_DRIVER("Failed to allocate space for semaphores, disabling\n");
- i915.semaphores = 0;
+ i915_modparams.semaphores = 0;
}
static void intel_ring_init_irq(struct drm_i915_private *dev_priv,
static void i9xx_set_default_submission(struct intel_engine_cs *engine)
{
engine->submit_request = i9xx_submit_request;
+ engine->cancel_requests = cancel_requests;
}
static void gen6_bsd_set_default_submission(struct intel_engine_cs *engine)
{
engine->submit_request = gen6_bsd_submit_request;
+ engine->cancel_requests = cancel_requests;
}
static void intel_ring_default_vfuncs(struct drm_i915_private *dev_priv,
engine->emit_breadcrumb = i9xx_emit_breadcrumb;
engine->emit_breadcrumb_sz = i9xx_emit_breadcrumb_sz;
- if (i915.semaphores) {
+ if (i915_modparams.semaphores) {
int num_rings;
engine->emit_breadcrumb = gen6_sema_emit_breadcrumb;
engine->emit_breadcrumb = gen8_render_emit_breadcrumb;
engine->emit_breadcrumb_sz = gen8_render_emit_breadcrumb_sz;
engine->emit_flush = gen8_render_ring_flush;
- if (i915.semaphores) {
+ if (i915_modparams.semaphores) {
int num_rings;
engine->semaphore.signal = gen8_rcs_signal;
int priority;
};
+/**
+ * struct intel_engine_execlists - execlist submission queue and port state
+ *
+ * The struct intel_engine_execlists represents the combined logical state of
+ * driver and the hardware state for execlist mode of submission.
+ */
+struct intel_engine_execlists {
+ /**
+ * @irq_tasklet: softirq tasklet for bottom handler
+ */
+ struct tasklet_struct irq_tasklet;
+
+ /**
+ * @default_priolist: priority list for I915_PRIORITY_NORMAL
+ */
+ struct i915_priolist default_priolist;
+
+ /**
+ * @no_priolist: priority lists disabled
+ */
+ bool no_priolist;
+
+ /**
+ * @port: execlist port states
+ *
+ * For each hardware ELSP (ExecList Submission Port) we keep
+ * track of the last request and the number of times we submitted
+ * that port to hw. We then count the number of times the hw reports
+ * a context completion or preemption. As only one context can
+ * be active on hw, we limit resubmission of context to port[0]. This
+ * is called Lite Restore, of the context.
+ */
+ struct execlist_port {
+ /**
+ * @request_count: combined request and submission count
+ */
+ struct drm_i915_gem_request *request_count;
+#define EXECLIST_COUNT_BITS 2
+#define port_request(p) ptr_mask_bits((p)->request_count, EXECLIST_COUNT_BITS)
+#define port_count(p) ptr_unmask_bits((p)->request_count, EXECLIST_COUNT_BITS)
+#define port_pack(rq, count) ptr_pack_bits(rq, count, EXECLIST_COUNT_BITS)
+#define port_unpack(p, count) ptr_unpack_bits((p)->request_count, count, EXECLIST_COUNT_BITS)
+#define port_set(p, packed) ((p)->request_count = (packed))
+#define port_isset(p) ((p)->request_count)
+#define port_index(p, execlists) ((p) - (execlists)->port)
+
+ /**
+ * @context_id: context ID for port
+ */
+ GEM_DEBUG_DECL(u32 context_id);
+
+#define EXECLIST_MAX_PORTS 2
+ } port[EXECLIST_MAX_PORTS];
+
+ /**
+ * @port_mask: number of execlist ports - 1
+ */
+ unsigned int port_mask;
+
+ /**
+ * @queue: queue of requests, in priority lists
+ */
+ struct rb_root queue;
+
+ /**
+ * @first: leftmost level in priority @queue
+ */
+ struct rb_node *first;
+
+ /**
+ * @fw_domains: forcewake domains for irq tasklet
+ */
+ unsigned int fw_domains;
+
+ /**
+ * @csb_head: context status buffer head
+ */
+ unsigned int csb_head;
+
+ /**
+ * @csb_use_mmio: access csb through mmio, instead of hwsp
+ */
+ bool csb_use_mmio;
+};
+
#define INTEL_ENGINE_CS_MAX_NAME 8
struct intel_engine_cs {
void (*schedule)(struct drm_i915_gem_request *request,
int priority);
+ /*
+ * Cancel all requests on the hardware, or queued for execution.
+ * This should only cancel the ready requests that have been
+ * submitted to the engine (via the engine->submit_request callback).
+ * This is called when marking the device as wedged.
+ */
+ void (*cancel_requests)(struct intel_engine_cs *engine);
+
/* Some chipsets are not quite as coherent as advertised and need
* an expensive kick to force a true read of the up-to-date seqno.
* However, the up-to-date seqno is not always required and the last
u32 *(*signal)(struct drm_i915_gem_request *req, u32 *cs);
} semaphore;
- /* Execlists */
- struct tasklet_struct irq_tasklet;
- struct i915_priolist default_priolist;
- bool no_priolist;
- struct execlist_port {
- struct drm_i915_gem_request *request_count;
-#define EXECLIST_COUNT_BITS 2
-#define port_request(p) ptr_mask_bits((p)->request_count, EXECLIST_COUNT_BITS)
-#define port_count(p) ptr_unmask_bits((p)->request_count, EXECLIST_COUNT_BITS)
-#define port_pack(rq, count) ptr_pack_bits(rq, count, EXECLIST_COUNT_BITS)
-#define port_unpack(p, count) ptr_unpack_bits((p)->request_count, count, EXECLIST_COUNT_BITS)
-#define port_set(p, packed) ((p)->request_count = (packed))
-#define port_isset(p) ((p)->request_count)
-#define port_index(p, e) ((p) - (e)->execlist_port)
- GEM_DEBUG_DECL(u32 context_id);
- } execlist_port[2];
- struct rb_root execlist_queue;
- struct rb_node *execlist_first;
- unsigned int fw_domains;
+ struct intel_engine_execlists execlists;
/* Contexts are pinned whilst they are active on the GPU. The last
* context executed remains active whilst the GPU is idle - the
u32 (*get_cmd_length_mask)(u32 cmd_header);
};
+static inline unsigned int
+execlists_num_ports(const struct intel_engine_execlists * const execlists)
+{
+ return execlists->port_mask + 1;
+}
+
+static inline void
+execlists_port_complete(struct intel_engine_execlists * const execlists,
+ struct execlist_port * const port)
+{
+ const unsigned int m = execlists->port_mask;
+
+ GEM_BUG_ON(port_index(port, execlists) != 0);
+
+ memmove(port, port + 1, m * sizeof(struct execlist_port));
+ memset(port + m, 0, sizeof(struct execlist_port));
+}
+
static inline unsigned int
intel_engine_flag(const struct intel_engine_cs *engine)
{
#define I915_GEM_HWS_SCRATCH_INDEX 0x40
#define I915_GEM_HWS_SCRATCH_ADDR (I915_GEM_HWS_SCRATCH_INDEX << MI_STORE_DWORD_INDEX_SHIFT)
+#define I915_HWS_CSB_BUF0_INDEX 0x10
+#define I915_HWS_CSB_WRITE_INDEX 0x1f
+#define CNL_HWS_CSB_WRITE_INDEX 0x2f
+
struct intel_ring *
intel_engine_create_ring(struct intel_engine_cs *engine, int size);
int intel_ring_pin(struct intel_ring *ring,
mask = 0;
}
- if (!i915.disable_power_well)
+ if (!i915_modparams.disable_power_well)
max_dc = 0;
if (enable_dc >= 0 && enable_dc <= max_dc) {
{
struct i915_power_domains *power_domains = &dev_priv->power_domains;
- i915.disable_power_well = sanitize_disable_power_well_option(dev_priv,
- i915.disable_power_well);
- dev_priv->csr.allowed_dc_mask = get_allowed_dc_mask(dev_priv,
- i915.enable_dc);
+ i915_modparams.disable_power_well =
+ sanitize_disable_power_well_option(dev_priv,
+ i915_modparams.disable_power_well);
+ dev_priv->csr.allowed_dc_mask =
+ get_allowed_dc_mask(dev_priv, i915_modparams.enable_dc);
BUILD_BUG_ON(POWER_DOMAIN_NUM > 64);
intel_display_set_init_power(dev_priv, true);
/* Remove the refcount we took to keep power well support disabled. */
- if (!i915.disable_power_well)
+ if (!i915_modparams.disable_power_well)
intel_display_power_put(dev_priv, POWER_DOMAIN_INIT);
/*
/* For now, we need the power well to be always enabled. */
intel_display_set_init_power(dev_priv, true);
/* Disable power support if the user asked so. */
- if (!i915.disable_power_well)
+ if (!i915_modparams.disable_power_well)
intel_display_power_get(dev_priv, POWER_DOMAIN_INIT);
intel_power_domains_sync_hw(dev_priv);
power_domains->initializing = false;
* Even if power well support was disabled we still want to disable
* power wells while we are system suspended.
*/
- if (!i915.disable_power_well)
+ if (!i915_modparams.disable_power_well)
intel_display_power_put(dev_priv, POWER_DOMAIN_INIT);
if (IS_CANNONLAKE(dev_priv))
mode_ptr->vsync_end = mode_ptr->vsync_start + 1;
mode_ptr->vtotal = vactive_s + 33;
- tmp = (u64) tv_mode->refresh * mode_ptr->vtotal;
+ tmp = mul_u32_u32(tv_mode->refresh, mode_ptr->vtotal);
tmp *= mode_ptr->htotal;
tmp = div_u64(tmp, 1000000);
mode_ptr->clock = (int) tmp;
void intel_uc_sanitize_options(struct drm_i915_private *dev_priv)
{
if (!HAS_GUC(dev_priv)) {
- if (i915.enable_guc_loading > 0 ||
- i915.enable_guc_submission > 0)
+ if (i915_modparams.enable_guc_loading > 0 ||
+ i915_modparams.enable_guc_submission > 0)
DRM_INFO("Ignoring GuC options, no hardware\n");
- i915.enable_guc_loading = 0;
- i915.enable_guc_submission = 0;
+ i915_modparams.enable_guc_loading = 0;
+ i915_modparams.enable_guc_submission = 0;
return;
}
/* A negative value means "use platform default" */
- if (i915.enable_guc_loading < 0)
- i915.enable_guc_loading = HAS_GUC_UCODE(dev_priv);
+ if (i915_modparams.enable_guc_loading < 0)
+ i915_modparams.enable_guc_loading = HAS_GUC_UCODE(dev_priv);
/* Verify firmware version */
- if (i915.enable_guc_loading) {
+ if (i915_modparams.enable_guc_loading) {
if (HAS_HUC_UCODE(dev_priv))
intel_huc_select_fw(&dev_priv->huc);
if (intel_guc_select_fw(&dev_priv->guc))
- i915.enable_guc_loading = 0;
+ i915_modparams.enable_guc_loading = 0;
}
/* Can't enable guc submission without guc loaded */
- if (!i915.enable_guc_loading)
- i915.enable_guc_submission = 0;
+ if (!i915_modparams.enable_guc_loading)
+ i915_modparams.enable_guc_submission = 0;
/* A negative value means "use platform default" */
- if (i915.enable_guc_submission < 0)
- i915.enable_guc_submission = HAS_GUC_SCHED(dev_priv);
+ if (i915_modparams.enable_guc_submission < 0)
+ i915_modparams.enable_guc_submission = HAS_GUC_SCHED(dev_priv);
}
static void gen8_guc_raise_irq(struct intel_guc *guc)
static void guc_capture_load_err_log(struct intel_guc *guc)
{
- if (!guc->log.vma || i915.guc_log_level < 0)
+ if (!guc->log.vma || i915_modparams.guc_log_level < 0)
return;
if (!guc->load_err_log)
guc->send = intel_guc_send_nop;
}
+/**
+ * intel_guc_auth_huc() - Send action to GuC to authenticate HuC ucode
+ * @guc: intel_guc structure
+ * @rsa_offset: rsa offset w.r.t ggtt base of huc vma
+ *
+ * Triggers a HuC firmware authentication request to the GuC via intel_guc_send
+ * INTEL_GUC_ACTION_AUTHENTICATE_HUC interface. This function is invoked by
+ * intel_huc_auth().
+ *
+ * Return: non-zero code on error
+ */
+int intel_guc_auth_huc(struct intel_guc *guc, u32 rsa_offset)
+{
+ u32 action[] = {
+ INTEL_GUC_ACTION_AUTHENTICATE_HUC,
+ rsa_offset
+ };
+
+ return intel_guc_send(guc, action, ARRAY_SIZE(action));
+}
+
int intel_uc_init_hw(struct drm_i915_private *dev_priv)
{
struct intel_guc *guc = &dev_priv->guc;
int ret, attempts;
- if (!i915.enable_guc_loading)
+ if (!i915_modparams.enable_guc_loading)
return 0;
guc_disable_communication(guc);
/* We need to notify the guc whenever we change the GGTT */
i915_ggtt_enable_guc(dev_priv);
- if (i915.enable_guc_submission) {
+ if (i915_modparams.enable_guc_submission) {
/*
* This is stuff we need to have available at fw load time
* if we are planning to enable submission later
if (ret)
goto err_log_capture;
- intel_guc_auth_huc(dev_priv);
- if (i915.enable_guc_submission) {
- if (i915.guc_log_level >= 0)
+ intel_huc_auth(&dev_priv->huc);
+ if (i915_modparams.enable_guc_submission) {
+ if (i915_modparams.guc_log_level >= 0)
gen9_enable_guc_interrupts(dev_priv);
ret = i915_guc_submission_enable(dev_priv);
err_log_capture:
guc_capture_load_err_log(guc);
err_submission:
- if (i915.enable_guc_submission)
+ if (i915_modparams.enable_guc_submission)
i915_guc_submission_fini(dev_priv);
err_guc:
i915_ggtt_disable_guc(dev_priv);
DRM_ERROR("GuC init failed\n");
- if (i915.enable_guc_loading > 1 || i915.enable_guc_submission > 1)
+ if (i915_modparams.enable_guc_loading > 1 ||
+ i915_modparams.enable_guc_submission > 1)
ret = -EIO;
else
ret = 0;
- if (i915.enable_guc_submission) {
- i915.enable_guc_submission = 0;
+ if (i915_modparams.enable_guc_submission) {
+ i915_modparams.enable_guc_submission = 0;
DRM_NOTE("Falling back from GuC submission to execlist mode\n");
}
- i915.enable_guc_loading = 0;
+ i915_modparams.enable_guc_loading = 0;
DRM_NOTE("GuC firmware loading disabled\n");
return ret;
{
guc_free_load_err_log(&dev_priv->guc);
- if (!i915.enable_guc_loading)
+ if (!i915_modparams.enable_guc_loading)
return;
- if (i915.enable_guc_submission)
+ if (i915_modparams.enable_guc_submission)
i915_guc_submission_disable(dev_priv);
guc_disable_communication(&dev_priv->guc);
- if (i915.enable_guc_submission) {
+ if (i915_modparams.enable_guc_submission) {
gen9_disable_guc_interrupts(dev_priv);
i915_guc_submission_fini(dev_priv);
}
* GuC). The subsequent pages of the client object constitute the work
* queue (a circular array of work items), again described in the process
* descriptor. Work queue pages are mapped momentarily as required.
- *
- * We also keep a few statistics on failures. Ideally, these should all
- * be zero!
- * no_wq_space: times that the submission pre-check found no space was
- * available in the work queue (note, the queue is shared,
- * not per-engine). It is OK for this to be nonzero, but
- * it should not be huge!
- * b_fail: failed to ring the doorbell. This should never happen, unless
- * somehow the hardware misbehaves, or maybe if the GuC firmware
- * crashes? We probably need to reset the GPU to recover.
- * retcode: errno from last guc_submit()
*/
struct i915_guc_client {
struct i915_vma *vma;
u16 doorbell_id;
unsigned long doorbell_offset;
- u32 doorbell_cookie;
spinlock_t wq_lock;
- uint32_t wq_offset;
- uint32_t wq_size;
- uint32_t wq_tail;
- uint32_t wq_rsvd;
- uint32_t no_wq_space;
-
/* Per-engine counts of GuC submissions */
uint64_t submissions[I915_NUM_ENGINES];
};
int intel_guc_sample_forcewake(struct intel_guc *guc);
int intel_guc_send_nop(struct intel_guc *guc, const u32 *action, u32 len);
int intel_guc_send_mmio(struct intel_guc *guc, const u32 *action, u32 len);
+int intel_guc_auth_huc(struct intel_guc *guc, u32 rsa_offset);
static inline int intel_guc_send(struct intel_guc *guc, const u32 *action, u32 len)
{
/* i915_guc_submission.c */
int i915_guc_submission_init(struct drm_i915_private *dev_priv);
int i915_guc_submission_enable(struct drm_i915_private *dev_priv);
-int i915_guc_wq_reserve(struct drm_i915_gem_request *rq);
-void i915_guc_wq_unreserve(struct drm_i915_gem_request *request);
void i915_guc_submission_disable(struct drm_i915_private *dev_priv);
void i915_guc_submission_fini(struct drm_i915_private *dev_priv);
struct i915_vma *intel_guc_allocate_vma(struct intel_guc *guc, u32 size);
/* intel_huc.c */
void intel_huc_select_fw(struct intel_huc *huc);
void intel_huc_init_hw(struct intel_huc *huc);
-void intel_guc_auth_huc(struct drm_i915_private *dev_priv);
+void intel_huc_auth(struct intel_huc *huc);
#endif
void intel_uncore_sanitize(struct drm_i915_private *dev_priv)
{
- i915.enable_rc6 = sanitize_rc6_option(dev_priv, i915.enable_rc6);
+ i915_modparams.enable_rc6 =
+ sanitize_rc6_option(dev_priv, i915_modparams.enable_rc6);
/* BIOS often leaves RC6 enabled, but disable it for hw init */
intel_sanitize_gt_powersave(dev_priv);
spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
}
+/**
+ * intel_uncore_forcewake_user_get - claim forcewake on behalf of userspace
+ * @dev_priv: i915 device instance
+ *
+ * This function is a wrapper around intel_uncore_forcewake_get() to acquire
+ * the GT powerwell and in the process disable our debugging for the
+ * duration of userspace's bypass.
+ */
+void intel_uncore_forcewake_user_get(struct drm_i915_private *dev_priv)
+{
+ spin_lock_irq(&dev_priv->uncore.lock);
+ if (!dev_priv->uncore.user_forcewake.count++) {
+ intel_uncore_forcewake_get__locked(dev_priv, FORCEWAKE_ALL);
+
+ /* Save and disable mmio debugging for the user bypass */
+ dev_priv->uncore.user_forcewake.saved_mmio_check =
+ dev_priv->uncore.unclaimed_mmio_check;
+ dev_priv->uncore.user_forcewake.saved_mmio_debug =
+ i915_modparams.mmio_debug;
+
+ dev_priv->uncore.unclaimed_mmio_check = 0;
+ i915_modparams.mmio_debug = 0;
+ }
+ spin_unlock_irq(&dev_priv->uncore.lock);
+}
+
+/**
+ * intel_uncore_forcewake_user_put - release forcewake on behalf of userspace
+ * @dev_priv: i915 device instance
+ *
+ * This function complements intel_uncore_forcewake_user_get() and releases
+ * the GT powerwell taken on behalf of the userspace bypass.
+ */
+void intel_uncore_forcewake_user_put(struct drm_i915_private *dev_priv)
+{
+ spin_lock_irq(&dev_priv->uncore.lock);
+ if (!--dev_priv->uncore.user_forcewake.count) {
+ if (intel_uncore_unclaimed_mmio(dev_priv))
+ dev_info(dev_priv->drm.dev,
+ "Invalid mmio detected during user access\n");
+
+ dev_priv->uncore.unclaimed_mmio_check =
+ dev_priv->uncore.user_forcewake.saved_mmio_check;
+ i915_modparams.mmio_debug =
+ dev_priv->uncore.user_forcewake.saved_mmio_debug;
+
+ intel_uncore_forcewake_put__locked(dev_priv, FORCEWAKE_ALL);
+ }
+ spin_unlock_irq(&dev_priv->uncore.lock);
+}
+
/**
* intel_uncore_forcewake_get__locked - grab forcewake domain references
* @dev_priv: i915 device instance
"Unclaimed %s register 0x%x\n",
read ? "read from" : "write to",
i915_mmio_reg_offset(reg)))
- i915.mmio_debug--; /* Only report the first N failures */
+ /* Only report the first N failures */
+ i915_modparams.mmio_debug--;
}
static inline void
const bool read,
const bool before)
{
- if (likely(!i915.mmio_debug))
+ if (likely(!i915_modparams.mmio_debug))
return;
__unclaimed_reg_debug(dev_priv, reg, read, before);
intel_uncore_forcewake_reset(dev_priv, false);
}
-#define GEN_RANGE(l, h) GENMASK((h) - 1, (l) - 1)
-
-static const struct register_whitelist {
- i915_reg_t offset_ldw, offset_udw;
- uint32_t size;
- /* supported gens, 0x10 for 4, 0x30 for 4 and 5, etc. */
- uint32_t gen_bitmask;
-} whitelist[] = {
- { .offset_ldw = RING_TIMESTAMP(RENDER_RING_BASE),
- .offset_udw = RING_TIMESTAMP_UDW(RENDER_RING_BASE),
- .size = 8, .gen_bitmask = GEN_RANGE(4, 10) },
-};
+static const struct reg_whitelist {
+ i915_reg_t offset_ldw;
+ i915_reg_t offset_udw;
+ u16 gen_mask;
+ u8 size;
+} reg_read_whitelist[] = { {
+ .offset_ldw = RING_TIMESTAMP(RENDER_RING_BASE),
+ .offset_udw = RING_TIMESTAMP_UDW(RENDER_RING_BASE),
+ .gen_mask = INTEL_GEN_MASK(4, 10),
+ .size = 8
+} };
int i915_reg_read_ioctl(struct drm_device *dev,
void *data, struct drm_file *file)
{
struct drm_i915_private *dev_priv = to_i915(dev);
struct drm_i915_reg_read *reg = data;
- struct register_whitelist const *entry = whitelist;
- unsigned size;
- i915_reg_t offset_ldw, offset_udw;
- int i, ret = 0;
-
- for (i = 0; i < ARRAY_SIZE(whitelist); i++, entry++) {
- if (i915_mmio_reg_offset(entry->offset_ldw) == (reg->offset & -entry->size) &&
- (INTEL_INFO(dev_priv)->gen_mask & entry->gen_bitmask))
+ struct reg_whitelist const *entry;
+ unsigned int flags;
+ int remain;
+ int ret = 0;
+
+ entry = reg_read_whitelist;
+ remain = ARRAY_SIZE(reg_read_whitelist);
+ while (remain) {
+ u32 entry_offset = i915_mmio_reg_offset(entry->offset_ldw);
+
+ GEM_BUG_ON(!is_power_of_2(entry->size));
+ GEM_BUG_ON(entry->size > 8);
+ GEM_BUG_ON(entry_offset & (entry->size - 1));
+
+ if (INTEL_INFO(dev_priv)->gen_mask & entry->gen_mask &&
+ entry_offset == (reg->offset & -entry->size))
break;
+ entry++;
+ remain--;
}
- if (i == ARRAY_SIZE(whitelist))
+ if (!remain)
return -EINVAL;
- /* We use the low bits to encode extra flags as the register should
- * be naturally aligned (and those that are not so aligned merely
- * limit the available flags for that register).
- */
- offset_ldw = entry->offset_ldw;
- offset_udw = entry->offset_udw;
- size = entry->size;
- size |= reg->offset ^ i915_mmio_reg_offset(offset_ldw);
+ flags = reg->offset & (entry->size - 1);
intel_runtime_pm_get(dev_priv);
-
- switch (size) {
- case 8 | 1:
- reg->val = I915_READ64_2x32(offset_ldw, offset_udw);
- break;
- case 8:
- reg->val = I915_READ64(offset_ldw);
- break;
- case 4:
- reg->val = I915_READ(offset_ldw);
- break;
- case 2:
- reg->val = I915_READ16(offset_ldw);
- break;
- case 1:
- reg->val = I915_READ8(offset_ldw);
- break;
- default:
+ if (entry->size == 8 && flags == I915_REG_READ_8B_WA)
+ reg->val = I915_READ64_2x32(entry->offset_ldw,
+ entry->offset_udw);
+ else if (entry->size == 8 && flags == 0)
+ reg->val = I915_READ64(entry->offset_ldw);
+ else if (entry->size == 4 && flags == 0)
+ reg->val = I915_READ(entry->offset_ldw);
+ else if (entry->size == 2 && flags == 0)
+ reg->val = I915_READ16(entry->offset_ldw);
+ else if (entry->size == 1 && flags == 0)
+ reg->val = I915_READ8(entry->offset_ldw);
+ else
ret = -EINVAL;
- goto out;
- }
-
-out:
intel_runtime_pm_put(dev_priv);
+
return ret;
}
-static void gen3_stop_rings(struct drm_i915_private *dev_priv)
+static void gen3_stop_engine(struct intel_engine_cs *engine)
+{
+ struct drm_i915_private *dev_priv = engine->i915;
+ const u32 base = engine->mmio_base;
+ const i915_reg_t mode = RING_MI_MODE(base);
+
+ I915_WRITE_FW(mode, _MASKED_BIT_ENABLE(STOP_RING));
+ if (intel_wait_for_register_fw(dev_priv,
+ mode,
+ MODE_IDLE,
+ MODE_IDLE,
+ 500))
+ DRM_DEBUG_DRIVER("%s: timed out on STOP_RING\n",
+ engine->name);
+
+ I915_WRITE_FW(RING_CTL(base), 0);
+ I915_WRITE_FW(RING_HEAD(base), 0);
+ I915_WRITE_FW(RING_TAIL(base), 0);
+
+ /* Check acts as a post */
+ if (I915_READ_FW(RING_HEAD(base)) != 0)
+ DRM_DEBUG_DRIVER("%s: ring head not parked\n",
+ engine->name);
+}
+
+static void i915_stop_engines(struct drm_i915_private *dev_priv,
+ unsigned engine_mask)
{
struct intel_engine_cs *engine;
enum intel_engine_id id;
- for_each_engine(engine, dev_priv, id) {
- const u32 base = engine->mmio_base;
- const i915_reg_t mode = RING_MI_MODE(base);
-
- I915_WRITE_FW(mode, _MASKED_BIT_ENABLE(STOP_RING));
- if (intel_wait_for_register_fw(dev_priv,
- mode,
- MODE_IDLE,
- MODE_IDLE,
- 500))
- DRM_DEBUG_DRIVER("%s: timed out on STOP_RING\n",
- engine->name);
-
- I915_WRITE_FW(RING_CTL(base), 0);
- I915_WRITE_FW(RING_HEAD(base), 0);
- I915_WRITE_FW(RING_TAIL(base), 0);
-
- /* Check acts as a post */
- if (I915_READ_FW(RING_HEAD(base)) != 0)
- DRM_DEBUG_DRIVER("%s: ring head not parked\n",
- engine->name);
- }
+ for_each_engine_masked(engine, dev_priv, engine_mask, id)
+ gen3_stop_engine(engine);
}
static bool i915_reset_complete(struct pci_dev *pdev)
{
struct pci_dev *pdev = dev_priv->drm.pdev;
- /* Stop engines before we reset; see g4x_do_reset() below for why. */
- gen3_stop_rings(dev_priv);
-
pci_write_config_byte(pdev, I915_GDRST, GRDOM_RESET_ENABLE);
return wait_for(g4x_reset_complete(pdev), 500);
}
I915_READ(VDECCLK_GATE_D) | VCP_UNIT_CLOCK_GATE_DISABLE);
POSTING_READ(VDECCLK_GATE_D);
- /* We stop engines, otherwise we might get failed reset and a
- * dead gpu (on elk).
- * WaMediaResetMainRingCleanup:ctg,elk (presumably)
- */
- gen3_stop_rings(dev_priv);
-
pci_write_config_byte(pdev, I915_GDRST,
GRDOM_MEDIA | GRDOM_RESET_ENABLE);
ret = wait_for(g4x_reset_complete(pdev), 500);
static reset_func intel_get_gpu_reset(struct drm_i915_private *dev_priv)
{
- if (!i915.reset)
+ if (!i915_modparams.reset)
return NULL;
if (INTEL_INFO(dev_priv)->gen >= 8)
*/
intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
for (retry = 0; retry < 3; retry++) {
+
+ /* We stop engines, otherwise we might get failed reset and a
+ * dead gpu (on elk). Also as modern gpu as kbl can suffer
+ * from system hang if batchbuffer is progressing when
+ * the reset is issued, regardless of READY_TO_RESET ack.
+ * Thus assume it is best to stop engines on all gens
+ * where we have a gpu reset.
+ *
+ * WaMediaResetMainRingCleanup:ctg,elk (presumably)
+ *
+ * FIXME: Wa for more modern gens needs to be validated
+ */
+ i915_stop_engines(dev_priv, engine_mask);
+
ret = reset(dev_priv, engine_mask);
if (ret != -ETIMEDOUT)
break;
{
return (dev_priv->info.has_reset_engine &&
!dev_priv->guc.execbuf_client &&
- i915.reset >= 2);
+ i915_modparams.reset >= 2);
}
int intel_guc_reset(struct drm_i915_private *dev_priv)
bool
intel_uncore_arm_unclaimed_mmio_detection(struct drm_i915_private *dev_priv)
{
- if (unlikely(i915.mmio_debug ||
+ if (unlikely(i915_modparams.mmio_debug ||
dev_priv->uncore.unclaimed_mmio_check <= 0))
return false;
DRM_DEBUG("Unclaimed register detected, "
"enabling oneshot unclaimed register reporting. "
"Please use i915.mmio_debug=N for more information.\n");
- i915.mmio_debug++;
+ i915_modparams.mmio_debug++;
dev_priv->uncore.unclaimed_mmio_check--;
return true;
}
i915_reg_t reg_ack;
} fw_domain[FW_DOMAIN_ID_COUNT];
+ struct {
+ unsigned int count;
+
+ int saved_mmio_check;
+ int saved_mmio_debug;
+ } user_forcewake;
+
int unclaimed_mmio_check;
};
void intel_uncore_forcewake_put__locked(struct drm_i915_private *dev_priv,
enum forcewake_domains domains);
+void intel_uncore_forcewake_user_get(struct drm_i915_private *dev_priv);
+void intel_uncore_forcewake_user_put(struct drm_i915_private *dev_priv);
+
int intel_wait_for_register(struct drm_i915_private *dev_priv,
i915_reg_t reg,
u32 mask,
static unsigned int random_engine(struct rnd_state *rnd)
{
- return ((u64)prandom_u32_state(rnd) * I915_NUM_ENGINES) >> 32;
+ return i915_prandom_u32_max_state(I915_NUM_ENGINES, rnd);
}
static int bench_sync(void *arg)
return x;
}
-static inline u32 i915_prandom_u32_max_state(u32 ep_ro, struct rnd_state *state)
-{
- return upper_32_bits((u64)prandom_u32_state(state) * ep_ro);
-}
-
void i915_random_reorder(unsigned int *order, unsigned int count,
struct rnd_state *state)
{
u64 i915_prandom_u64_state(struct rnd_state *rnd);
+static inline u32 i915_prandom_u32_max_state(u32 ep_ro, struct rnd_state *state)
+{
+ return upper_32_bits(mul_u32_u32(prandom_u32_state(state), ep_ro));
+}
+
unsigned int *i915_random_order(unsigned int count,
struct rnd_state *state);
void i915_random_reorder(unsigned int *order,
__i915_add_request(rq, true);
if (!wait_for_hang(&h, rq)) {
- pr_err("Failed to start request %x\n", rq->fence.seqno);
+ pr_err("Failed to start request %x, at %x\n",
+ rq->fence.seqno, hws_seqno(&h, rq));
+
+ i915_reset(i915, 0);
+ i915_gem_set_wedged(i915);
+
err = -EIO;
goto out_rq;
}
__i915_add_request(rq, true);
if (!wait_for_hang(&h, prev)) {
- pr_err("Failed to start request %x\n",
- prev->fence.seqno);
+ pr_err("Failed to start request %x, at %x\n",
+ prev->fence.seqno, hws_seqno(&h, prev));
i915_gem_request_put(rq);
i915_gem_request_put(prev);
+
+ i915_reset(i915, 0);
+ i915_gem_set_wedged(i915);
+
err = -EIO;
goto fini;
}
__i915_add_request(rq, true);
if (!wait_for_hang(&h, rq)) {
- pr_err("Failed to start request %x\n", rq->fence.seqno);
+ pr_err("Failed to start request %x, at %x\n",
+ rq->fence.seqno, hws_seqno(&h, rq));
+
+ i915_reset(i915, 0);
+ i915_gem_set_wedged(i915);
+
err = -EIO;
goto err_request;
}
int intel_hangcheck_live_selftests(struct drm_i915_private *i915)
{
static const struct i915_subtest tests[] = {
+ SUBTEST(igt_global_reset), /* attempt to recover GPU first */
SUBTEST(igt_hang_sanitycheck),
- SUBTEST(igt_global_reset),
SUBTEST(igt_reset_engine),
SUBTEST(igt_reset_active_engines),
SUBTEST(igt_wait_reset),
dev_set_name(&pdev->dev, "mock");
dma_coerce_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
+#if IS_ENABLED(CONFIG_IOMMU_API)
+ /* hack to disable iommu for the fake device; force identity mapping */
+ pdev->dev.archdata.iommu = (void *)-1;
+#endif
+
dev_pm_domain_set(&pdev->dev, &pm_domain);
pm_runtime_enable(&pdev->dev);
pm_runtime_dont_use_autosuspend(&pdev->dev);
{
struct vb2_dc_buf *buf;
struct frame_vector *vec;
- unsigned long offset;
+ unsigned int offset;
int n_pages, i;
int ret = 0;
struct sg_table *sgt;
buf->dev = dev;
buf->dma_dir = dma_dir;
- offset = vaddr & ~PAGE_MASK;
+ offset = lower_32_bits(offset_in_page(vaddr));
vec = vb2_create_framevec(vaddr, size, dma_dir == DMA_FROM_DEVICE ||
dma_dir == DMA_BIDIRECTIONAL);
if (IS_ERR(vec)) {
* offset within the internal buffer specified by handle parameter.
*/
if (xfer->loc_addr) {
- unsigned long offset;
+ unsigned int offset;
long pinned;
- offset = (unsigned long)(uintptr_t)xfer->loc_addr & ~PAGE_MASK;
+ offset = lower_32_bits(offset_in_page(xfer->loc_addr));
nr_pages = PAGE_ALIGN(xfer->length + offset) >> PAGE_SHIFT;
page_list = kmalloc_array(nr_pages,
#define INTEL_KBL_GT1_IDS(info) \
INTEL_VGA_DEVICE(0x5913, info), /* ULT GT1.5 */ \
INTEL_VGA_DEVICE(0x5915, info), /* ULX GT1.5 */ \
- INTEL_VGA_DEVICE(0x5917, info), /* DT GT1.5 */ \
INTEL_VGA_DEVICE(0x5906, info), /* ULT GT1 */ \
INTEL_VGA_DEVICE(0x590E, info), /* ULX GT1 */ \
INTEL_VGA_DEVICE(0x5902, info), /* DT GT1 */ \
#define INTEL_KBL_GT2_IDS(info) \
INTEL_VGA_DEVICE(0x5916, info), /* ULT GT2 */ \
+ INTEL_VGA_DEVICE(0x5917, info), /* Mobile GT2 */ \
INTEL_VGA_DEVICE(0x5921, info), /* ULT GT2F */ \
INTEL_VGA_DEVICE(0x591E, info), /* ULX GT2 */ \
INTEL_VGA_DEVICE(0x5912, info), /* DT GT2 */ \
#endif
};
+/*
+ * Since the above length field is an unsigned int, below we define the maximum
+ * length in bytes that can be stored in one scatterlist entry.
+ */
+#define SCATTERLIST_MAX_SEGMENT (UINT_MAX & PAGE_MASK)
+
/*
* These macros should be used after a dma_map_sg call has been done
* to get bus addresses of each of the SG entries and their lengths.
int __sg_alloc_table(struct sg_table *, unsigned int, unsigned int,
struct scatterlist *, gfp_t, sg_alloc_fn *);
int sg_alloc_table(struct sg_table *, unsigned int, gfp_t);
-int sg_alloc_table_from_pages(struct sg_table *sgt,
- struct page **pages, unsigned int n_pages,
- unsigned long offset, unsigned long size,
- gfp_t gfp_mask);
+int __sg_alloc_table_from_pages(struct sg_table *sgt, struct page **pages,
+ unsigned int n_pages, unsigned int offset,
+ unsigned long size, unsigned int max_segment,
+ gfp_t gfp_mask);
+int sg_alloc_table_from_pages(struct sg_table *sgt, struct page **pages,
+ unsigned int n_pages, unsigned int offset,
+ unsigned long size, gfp_t gfp_mask);
size_t sg_copy_buffer(struct scatterlist *sgl, unsigned int nents, void *buf,
size_t buflen, off_t skip, bool to_buffer);
* be specified
*/
__u64 offset;
+#define I915_REG_READ_8B_WA BIT(0)
+
__u64 val; /* Return value */
};
/* Known registers:
*
* Render engine timestamp - 0x2358 + 64bit - gen7+
* - Note this register returns an invalid value if using the default
- * single instruction 8byte read, in order to workaround that use
- * offset (0x2538 | 1) instead.
+ * single instruction 8byte read, in order to workaround that pass
+ * flag I915_REG_READ_8B_WA in offset field.
*
*/
__u32 n_boolean_regs;
__u32 n_flex_regs;
+ /*
+ * These fields are pointers to tuples of u32 values (register
+ * address, value). For example the expected length of the buffer
+ * pointed by mux_regs_ptr is (2 * sizeof(u32) * n_mux_regs).
+ */
__u64 mux_regs_ptr;
__u64 boolean_regs_ptr;
__u64 flex_regs_ptr;
EXPORT_SYMBOL(sg_alloc_table);
/**
- * sg_alloc_table_from_pages - Allocate and initialize an sg table from
- * an array of pages
- * @sgt: The sg table header to use
- * @pages: Pointer to an array of page pointers
- * @n_pages: Number of pages in the pages array
- * @offset: Offset from start of the first page to the start of a buffer
- * @size: Number of valid bytes in the buffer (after offset)
- * @gfp_mask: GFP allocation mask
+ * __sg_alloc_table_from_pages - Allocate and initialize an sg table from
+ * an array of pages
+ * @sgt: The sg table header to use
+ * @pages: Pointer to an array of page pointers
+ * @n_pages: Number of pages in the pages array
+ * @offset: Offset from start of the first page to the start of a buffer
+ * @size: Number of valid bytes in the buffer (after offset)
+ * @max_segment: Maximum size of a scatterlist node in bytes (page aligned)
+ * @gfp_mask: GFP allocation mask
*
* Description:
* Allocate and initialize an sg table from a list of pages. Contiguous
- * ranges of the pages are squashed into a single scatterlist node. A user
- * may provide an offset at a start and a size of valid data in a buffer
- * specified by the page array. The returned sg table is released by
- * sg_free_table.
+ * ranges of the pages are squashed into a single scatterlist node up to the
+ * maximum size specified in @max_segment. An user may provide an offset at a
+ * start and a size of valid data in a buffer specified by the page array.
+ * The returned sg table is released by sg_free_table.
*
* Returns:
* 0 on success, negative error on failure
*/
-int sg_alloc_table_from_pages(struct sg_table *sgt,
- struct page **pages, unsigned int n_pages,
- unsigned long offset, unsigned long size,
- gfp_t gfp_mask)
+int __sg_alloc_table_from_pages(struct sg_table *sgt, struct page **pages,
+ unsigned int n_pages, unsigned int offset,
+ unsigned long size, unsigned int max_segment,
+ gfp_t gfp_mask)
{
- unsigned int chunks;
- unsigned int i;
- unsigned int cur_page;
+ unsigned int chunks, cur_page, seg_len, i;
int ret;
struct scatterlist *s;
+ if (WARN_ON(!max_segment || offset_in_page(max_segment)))
+ return -EINVAL;
+
/* compute number of contiguous chunks */
chunks = 1;
- for (i = 1; i < n_pages; ++i)
- if (page_to_pfn(pages[i]) != page_to_pfn(pages[i - 1]) + 1)
- ++chunks;
+ seg_len = 0;
+ for (i = 1; i < n_pages; i++) {
+ seg_len += PAGE_SIZE;
+ if (seg_len >= max_segment ||
+ page_to_pfn(pages[i]) != page_to_pfn(pages[i - 1]) + 1) {
+ chunks++;
+ seg_len = 0;
+ }
+ }
ret = sg_alloc_table(sgt, chunks, gfp_mask);
if (unlikely(ret))
/* merging chunks and putting them into the scatterlist */
cur_page = 0;
for_each_sg(sgt->sgl, s, sgt->orig_nents, i) {
- unsigned long chunk_size;
- unsigned int j;
+ unsigned int j, chunk_size;
/* look for the end of the current chunk */
- for (j = cur_page + 1; j < n_pages; ++j)
- if (page_to_pfn(pages[j]) !=
+ seg_len = 0;
+ for (j = cur_page + 1; j < n_pages; j++) {
+ seg_len += PAGE_SIZE;
+ if (seg_len >= max_segment ||
+ page_to_pfn(pages[j]) !=
page_to_pfn(pages[j - 1]) + 1)
break;
+ }
chunk_size = ((j - cur_page) << PAGE_SHIFT) - offset;
- sg_set_page(s, pages[cur_page], min(size, chunk_size), offset);
+ sg_set_page(s, pages[cur_page],
+ min_t(unsigned long, size, chunk_size), offset);
size -= chunk_size;
offset = 0;
cur_page = j;
return 0;
}
+EXPORT_SYMBOL(__sg_alloc_table_from_pages);
+
+/**
+ * sg_alloc_table_from_pages - Allocate and initialize an sg table from
+ * an array of pages
+ * @sgt: The sg table header to use
+ * @pages: Pointer to an array of page pointers
+ * @n_pages: Number of pages in the pages array
+ * @offset: Offset from start of the first page to the start of a buffer
+ * @size: Number of valid bytes in the buffer (after offset)
+ * @gfp_mask: GFP allocation mask
+ *
+ * Description:
+ * Allocate and initialize an sg table from a list of pages. Contiguous
+ * ranges of the pages are squashed into a single scatterlist node. A user
+ * may provide an offset at a start and a size of valid data in a buffer
+ * specified by the page array. The returned sg table is released by
+ * sg_free_table.
+ *
+ * Returns:
+ * 0 on success, negative error on failure
+ */
+int sg_alloc_table_from_pages(struct sg_table *sgt, struct page **pages,
+ unsigned int n_pages, unsigned int offset,
+ unsigned long size, gfp_t gfp_mask)
+{
+ return __sg_alloc_table_from_pages(sgt, pages, n_pages, offset, size,
+ SCATTERLIST_MAX_SEGMENT, gfp_mask);
+}
EXPORT_SYMBOL(sg_alloc_table_from_pages);
void __sg_page_iter_start(struct sg_page_iter *piter,
--- /dev/null
+CFLAGS += -I. -I../../include -g -O2 -Wall -fsanitize=address
+LDFLAGS += -fsanitize=address -fsanitize=undefined
+TARGETS = main
+OFILES = main.o scatterlist.o
+
+ifeq ($(BUILD), 32)
+ CFLAGS += -m32
+ LDFLAGS += -m32
+endif
+
+targets: include $(TARGETS)
+
+main: $(OFILES)
+
+clean:
+ $(RM) $(TARGETS) $(OFILES) scatterlist.c linux/scatterlist.h linux/highmem.h linux/kmemleak.h asm/io.h
+ @rmdir asm
+
+scatterlist.c: ../../../lib/scatterlist.c
+ @sed -e 's/^static //' -e 's/__always_inline //' -e 's/inline //' < $< > $@
+
+.PHONY: include
+
+include: ../../../include/linux/scatterlist.h
+ @mkdir -p linux
+ @mkdir -p asm
+ @touch asm/io.h
+ @touch linux/highmem.h
+ @touch linux/kmemleak.h
+ @cp $< linux/scatterlist.h
--- /dev/null
+#ifndef _LINUX_MM_H
+#define _LINUX_MM_H
+
+#include <assert.h>
+#include <string.h>
+#include <stdlib.h>
+#include <errno.h>
+#include <limits.h>
+#include <stdio.h>
+
+typedef unsigned long dma_addr_t;
+
+#define unlikely
+
+#define BUG_ON(x) assert(!(x))
+
+#define WARN_ON(condition) ({ \
+ int __ret_warn_on = !!(condition); \
+ unlikely(__ret_warn_on); \
+})
+
+#define WARN_ON_ONCE(condition) ({ \
+ int __ret_warn_on = !!(condition); \
+ if (unlikely(__ret_warn_on)) \
+ assert(0); \
+ unlikely(__ret_warn_on); \
+})
+
+#define PAGE_SIZE (4096)
+#define PAGE_SHIFT (12)
+#define PAGE_MASK (~(PAGE_SIZE-1))
+
+#define __ALIGN_KERNEL(x, a) __ALIGN_KERNEL_MASK(x, (typeof(x))(a) - 1)
+#define __ALIGN_KERNEL_MASK(x, mask) (((x) + (mask)) & ~(mask))
+#define ALIGN(x, a) __ALIGN_KERNEL((x), (a))
+
+#define PAGE_ALIGN(addr) ALIGN(addr, PAGE_SIZE)
+
+#define offset_in_page(p) ((unsigned long)(p) & ~PAGE_MASK)
+
+#define virt_to_page(x) ((void *)x)
+#define page_address(x) ((void *)x)
+
+static inline unsigned long page_to_phys(struct page *page)
+{
+ assert(0);
+
+ return 0;
+}
+
+#define page_to_pfn(page) ((unsigned long)(page) / PAGE_SIZE)
+#define pfn_to_page(pfn) (void *)((pfn) * PAGE_SIZE)
+#define nth_page(page,n) pfn_to_page(page_to_pfn((page)) + (n))
+
+#define __min(t1, t2, min1, min2, x, y) ({ \
+ t1 min1 = (x); \
+ t2 min2 = (y); \
+ (void) (&min1 == &min2); \
+ min1 < min2 ? min1 : min2; })
+
+#define ___PASTE(a,b) a##b
+#define __PASTE(a,b) ___PASTE(a,b)
+
+#define __UNIQUE_ID(prefix) __PASTE(__PASTE(__UNIQUE_ID_, prefix), __COUNTER__)
+
+#define min(x, y) \
+ __min(typeof(x), typeof(y), \
+ __UNIQUE_ID(min1_), __UNIQUE_ID(min2_), \
+ x, y)
+
+#define min_t(type, x, y) \
+ __min(type, type, \
+ __UNIQUE_ID(min1_), __UNIQUE_ID(min2_), \
+ x, y)
+
+#define preemptible() (1)
+
+static inline void *kmap(struct page *page)
+{
+ assert(0);
+
+ return NULL;
+}
+
+static inline void *kmap_atomic(struct page *page)
+{
+ assert(0);
+
+ return NULL;
+}
+
+static inline void kunmap(void *addr)
+{
+ assert(0);
+}
+
+static inline void kunmap_atomic(void *addr)
+{
+ assert(0);
+}
+
+static inline unsigned long __get_free_page(unsigned int flags)
+{
+ return (unsigned long)malloc(PAGE_SIZE);
+}
+
+static inline void free_page(unsigned long page)
+{
+ free((void *)page);
+}
+
+static inline void *kmalloc(unsigned int size, unsigned int flags)
+{
+ return malloc(size);
+}
+
+#define kfree(x) free(x)
+
+#define kmemleak_alloc(a, b, c, d)
+#define kmemleak_free(a)
+
+#define PageSlab(p) (0)
+#define flush_kernel_dcache_page(p)
+
+#endif
--- /dev/null
+#include <stdio.h>
+#include <assert.h>
+
+#include <linux/scatterlist.h>
+
+#define MAX_PAGES (64)
+
+static void set_pages(struct page **pages, const unsigned *array, unsigned num)
+{
+ unsigned int i;
+
+ assert(num < MAX_PAGES);
+ for (i = 0; i < num; i++)
+ pages[i] = (struct page *)(unsigned long)
+ ((1 + array[i]) * PAGE_SIZE);
+}
+
+#define pfn(...) (unsigned []){ __VA_ARGS__ }
+
+int main(void)
+{
+ const unsigned int sgmax = SCATTERLIST_MAX_SEGMENT;
+ struct test {
+ int alloc_ret;
+ unsigned num_pages;
+ unsigned *pfn;
+ unsigned size;
+ unsigned int max_seg;
+ unsigned int expected_segments;
+ } *test, tests[] = {
+ { -EINVAL, 1, pfn(0), PAGE_SIZE, PAGE_SIZE + 1, 1 },
+ { -EINVAL, 1, pfn(0), PAGE_SIZE, 0, 1 },
+ { -EINVAL, 1, pfn(0), PAGE_SIZE, sgmax + 1, 1 },
+ { 0, 1, pfn(0), PAGE_SIZE, sgmax, 1 },
+ { 0, 1, pfn(0), 1, sgmax, 1 },
+ { 0, 2, pfn(0, 1), 2 * PAGE_SIZE, sgmax, 1 },
+ { 0, 2, pfn(1, 0), 2 * PAGE_SIZE, sgmax, 2 },
+ { 0, 3, pfn(0, 1, 2), 3 * PAGE_SIZE, sgmax, 1 },
+ { 0, 3, pfn(0, 2, 1), 3 * PAGE_SIZE, sgmax, 3 },
+ { 0, 3, pfn(0, 1, 3), 3 * PAGE_SIZE, sgmax, 2 },
+ { 0, 3, pfn(1, 2, 4), 3 * PAGE_SIZE, sgmax, 2 },
+ { 0, 3, pfn(1, 3, 4), 3 * PAGE_SIZE, sgmax, 2 },
+ { 0, 4, pfn(0, 1, 3, 4), 4 * PAGE_SIZE, sgmax, 2 },
+ { 0, 5, pfn(0, 1, 3, 4, 5), 5 * PAGE_SIZE, sgmax, 2 },
+ { 0, 5, pfn(0, 1, 3, 4, 6), 5 * PAGE_SIZE, sgmax, 3 },
+ { 0, 5, pfn(0, 1, 2, 3, 4), 5 * PAGE_SIZE, sgmax, 1 },
+ { 0, 5, pfn(0, 1, 2, 3, 4), 5 * PAGE_SIZE, 2 * PAGE_SIZE, 3 },
+ { 0, 6, pfn(0, 1, 2, 3, 4, 5), 6 * PAGE_SIZE, 2 * PAGE_SIZE, 3 },
+ { 0, 6, pfn(0, 2, 3, 4, 5, 6), 6 * PAGE_SIZE, 2 * PAGE_SIZE, 4 },
+ { 0, 6, pfn(0, 1, 3, 4, 5, 6), 6 * PAGE_SIZE, 2 * PAGE_SIZE, 3 },
+ { 0, 0, NULL, 0, 0, 0 },
+ };
+ unsigned int i;
+
+ for (i = 0, test = tests; test->expected_segments; test++, i++) {
+ struct page *pages[MAX_PAGES];
+ struct sg_table st;
+ int ret;
+
+ set_pages(pages, test->pfn, test->num_pages);
+
+ ret = __sg_alloc_table_from_pages(&st, pages, test->num_pages,
+ 0, test->size, test->max_seg,
+ GFP_KERNEL);
+ assert(ret == test->alloc_ret);
+
+ if (test->alloc_ret)
+ continue;
+
+ assert(st.nents == test->expected_segments);
+ assert(st.orig_nents == test->expected_segments);
+
+ sg_free_table(&st);
+ }
+
+ assert(i == (sizeof(tests) / sizeof(tests[0])) - 1);
+
+ return 0;
+}
git.samba.org / sfrench / cifs-2.6.git / commitdiff