From cd167748adeef1a9e77e54520ade18c520bb11f2 Mon Sep 17 00:00:00 2001 From: Daniel Vetter Date: Wed, 6 Aug 2014 15:04:45 +0200 Subject: [PATCH] Backport drm/i915 from ickle's requests branch to 3.17.0 I've checked out git://people.freedesktop.org/~ickle/linux-2.6 commit dcee630f5775a1f2268deaa2a3cd011cf8ba941f Author: Chris Wilson Date: Thu Aug 28 10:48:47 2014 +0100 drm/i915: s/seqno/request/ tracking inside objects then removed drivers/gpu/drm/i915 from a Fedora 3.17.0 and put in the drm/i915 from Chris's tree. The remaining work is the following: * Patch to compile with drm and kernel core from 3.17.0 instead of Chris's tree. * Remove unused features that didn't exist in 3.17.0 * Check all changes to ensure that I've not pulled in something dangerous. --- drivers/gpu/drm/i915/Makefile | 4 +- drivers/gpu/drm/i915/i915_cmd_parser.c | 160 +- drivers/gpu/drm/i915/i915_debugfs.c | 432 ++-- drivers/gpu/drm/i915/i915_dma.c | 979 +------- drivers/gpu/drm/i915/i915_drv.c | 229 +- drivers/gpu/drm/i915/i915_drv.h | 592 +++-- drivers/gpu/drm/i915/i915_gem.c | 1832 ++++++-------- drivers/gpu/drm/i915/i915_gem_context.c | 682 +++--- drivers/gpu/drm/i915/i915_gem_debug.c | 118 - drivers/gpu/drm/i915/i915_gem_evict.c | 4 +- drivers/gpu/drm/i915/i915_gem_execbuffer.c | 694 +++--- drivers/gpu/drm/i915/i915_gem_gtt.c | 408 ++-- drivers/gpu/drm/i915/i915_gem_gtt.h | 36 +- drivers/gpu/drm/i915/i915_gem_render_state.c | 35 +- drivers/gpu/drm/i915/i915_gem_request.c | 666 ++++++ drivers/gpu/drm/i915/i915_gem_stolen.c | 13 +- drivers/gpu/drm/i915/i915_gem_tiling.c | 11 +- drivers/gpu/drm/i915/i915_gpu_error.c | 526 ++-- drivers/gpu/drm/i915/i915_irq.c | 677 +++--- drivers/gpu/drm/i915/i915_reg.h | 287 ++- drivers/gpu/drm/i915/i915_sysfs.c | 2 +- drivers/gpu/drm/i915/i915_trace.h | 215 +- drivers/gpu/drm/i915/intel_bios.c | 13 +- drivers/gpu/drm/i915/intel_bios.h | 13 +- drivers/gpu/drm/i915/intel_ddi.c | 344 +-- drivers/gpu/drm/i915/intel_display.c | 2024 +++++++++------- drivers/gpu/drm/i915/intel_dp.c | 1059 ++++++--- drivers/gpu/drm/i915/intel_drv.h | 61 +- drivers/gpu/drm/i915/intel_dsi.c | 40 +- drivers/gpu/drm/i915/intel_dsi.h | 2 + drivers/gpu/drm/i915/intel_dsi_cmd.c | 2 +- drivers/gpu/drm/i915/intel_dsi_panel_vbt.c | 38 +- drivers/gpu/drm/i915/intel_dsi_pll.c | 17 +- drivers/gpu/drm/i915/intel_dvo.c | 11 +- drivers/gpu/drm/i915/intel_fbdev.c | 83 +- drivers/gpu/drm/i915/intel_hdmi.c | 170 +- drivers/gpu/drm/i915/intel_lvds.c | 3 +- drivers/gpu/drm/i915/intel_opregion.c | 16 +- drivers/gpu/drm/i915/intel_overlay.c | 200 +- drivers/gpu/drm/i915/intel_panel.c | 59 +- drivers/gpu/drm/i915/intel_pm.c | 1152 ++++++--- drivers/gpu/drm/i915/intel_ringbuffer.c | 3291 +++++++++++++------------- drivers/gpu/drm/i915/intel_ringbuffer.h | 379 +-- drivers/gpu/drm/i915/intel_sprite.c | 349 ++- drivers/gpu/drm/i915/intel_tv.c | 9 +- drivers/gpu/drm/i915/intel_uncore.c | 23 +- 46 files changed, 9728 insertions(+), 8232 deletions(-) delete mode 100644 drivers/gpu/drm/i915/i915_gem_debug.c create mode 100644 drivers/gpu/drm/i915/i915_gem_request.c diff --git a/drivers/gpu/drm/i915/Makefile b/drivers/gpu/drm/i915/Makefile index 91bd167..5e8dc28 100644 --- a/drivers/gpu/drm/i915/Makefile +++ b/drivers/gpu/drm/i915/Makefile @@ -17,14 +17,14 @@ i915-$(CONFIG_DEBUG_FS) += i915_debugfs.o # GEM code i915-y += i915_cmd_parser.o \ + i915_gem.o \ i915_gem_context.o \ i915_gem_render_state.o \ - i915_gem_debug.o \ i915_gem_dmabuf.o \ i915_gem_evict.o \ i915_gem_execbuffer.o \ i915_gem_gtt.o \ - i915_gem.o \ + i915_gem_request.o \ i915_gem_stolen.o \ i915_gem_tiling.o \ i915_gem_userptr.o \ diff --git a/drivers/gpu/drm/i915/i915_cmd_parser.c b/drivers/gpu/drm/i915/i915_cmd_parser.c index 4b7ed52..408e0bd 100644 --- a/drivers/gpu/drm/i915/i915_cmd_parser.c +++ b/drivers/gpu/drm/i915/i915_cmd_parser.c @@ -501,7 +501,7 @@ static u32 gen7_blt_get_cmd_length_mask(u32 cmd_header) return 0; } -static bool validate_cmds_sorted(struct intel_engine_cs *ring, +static bool validate_cmds_sorted(struct intel_engine_cs *engine, const struct drm_i915_cmd_table *cmd_tables, int cmd_table_count) { @@ -523,7 +523,7 @@ static bool validate_cmds_sorted(struct intel_engine_cs *ring, if (curr < previous) { DRM_ERROR("CMD: table not sorted ring=%d table=%d entry=%d cmd=0x%08X prev=0x%08X\n", - ring->id, i, j, curr, previous); + engine->id, i, j, curr, previous); ret = false; } @@ -555,11 +555,11 @@ static bool check_sorted(int ring_id, const u32 *reg_table, int reg_count) return ret; } -static bool validate_regs_sorted(struct intel_engine_cs *ring) +static bool validate_regs_sorted(struct intel_engine_cs *engine) { - return check_sorted(ring->id, ring->reg_table, ring->reg_count) && - check_sorted(ring->id, ring->master_reg_table, - ring->master_reg_count); + return check_sorted(engine->id, engine->reg_table, engine->reg_count) && + check_sorted(engine->id, engine->master_reg_table, + engine->master_reg_count); } struct cmd_node { @@ -583,13 +583,13 @@ struct cmd_node { */ #define CMD_HASH_MASK STD_MI_OPCODE_MASK -static int init_hash_table(struct intel_engine_cs *ring, +static int init_hash_table(struct intel_engine_cs *engine, const struct drm_i915_cmd_table *cmd_tables, int cmd_table_count) { int i, j; - hash_init(ring->cmd_hash); + hash_init(engine->cmd_hash); for (i = 0; i < cmd_table_count; i++) { const struct drm_i915_cmd_table *table = &cmd_tables[i]; @@ -604,7 +604,7 @@ static int init_hash_table(struct intel_engine_cs *ring, return -ENOMEM; desc_node->desc = desc; - hash_add(ring->cmd_hash, &desc_node->node, + hash_add(engine->cmd_hash, &desc_node->node, desc->cmd.value & CMD_HASH_MASK); } } @@ -612,21 +612,21 @@ static int init_hash_table(struct intel_engine_cs *ring, return 0; } -static void fini_hash_table(struct intel_engine_cs *ring) +static void fini_hash_table(struct intel_engine_cs *engine) { struct hlist_node *tmp; struct cmd_node *desc_node; int i; - hash_for_each_safe(ring->cmd_hash, i, tmp, desc_node, node) { + hash_for_each_safe(engine->cmd_hash, i, tmp, desc_node, node) { hash_del(&desc_node->node); kfree(desc_node); } } /** - * i915_cmd_parser_init_ring() - set cmd parser related fields for a ringbuffer - * @ring: the ringbuffer to initialize + * i915_cmd_parser_init_engine() - set cmd parser related fields for a ringbuffer + * @engine: the ringbuffer to initialize * * Optionally initializes fields related to batch buffer command parsing in the * struct intel_engine_cs based on whether the platform requires software @@ -634,18 +634,18 @@ static void fini_hash_table(struct intel_engine_cs *ring) * * Return: non-zero if initialization fails */ -int i915_cmd_parser_init_ring(struct intel_engine_cs *ring) +int i915_cmd_parser_init_engine(struct intel_engine_cs *engine) { const struct drm_i915_cmd_table *cmd_tables; int cmd_table_count; int ret; - if (!IS_GEN7(ring->dev)) + if (!IS_GEN7(engine->i915)) return 0; - switch (ring->id) { + switch (engine->id) { case RCS: - if (IS_HASWELL(ring->dev)) { + if (IS_HASWELL(engine->i915)) { cmd_tables = hsw_render_ring_cmds; cmd_table_count = ARRAY_SIZE(hsw_render_ring_cmds); @@ -654,26 +654,26 @@ int i915_cmd_parser_init_ring(struct intel_engine_cs *ring) cmd_table_count = ARRAY_SIZE(gen7_render_cmds); } - ring->reg_table = gen7_render_regs; - ring->reg_count = ARRAY_SIZE(gen7_render_regs); + engine->reg_table = gen7_render_regs; + engine->reg_count = ARRAY_SIZE(gen7_render_regs); - if (IS_HASWELL(ring->dev)) { - ring->master_reg_table = hsw_master_regs; - ring->master_reg_count = ARRAY_SIZE(hsw_master_regs); + if (IS_HASWELL(engine->i915)) { + engine->master_reg_table = hsw_master_regs; + engine->master_reg_count = ARRAY_SIZE(hsw_master_regs); } else { - ring->master_reg_table = ivb_master_regs; - ring->master_reg_count = ARRAY_SIZE(ivb_master_regs); + engine->master_reg_table = ivb_master_regs; + engine->master_reg_count = ARRAY_SIZE(ivb_master_regs); } - ring->get_cmd_length_mask = gen7_render_get_cmd_length_mask; + engine->get_cmd_length_mask = gen7_render_get_cmd_length_mask; break; case VCS: cmd_tables = gen7_video_cmds; cmd_table_count = ARRAY_SIZE(gen7_video_cmds); - ring->get_cmd_length_mask = gen7_bsd_get_cmd_length_mask; + engine->get_cmd_length_mask = gen7_bsd_get_cmd_length_mask; break; case BCS: - if (IS_HASWELL(ring->dev)) { + if (IS_HASWELL(engine->i915)) { cmd_tables = hsw_blt_ring_cmds; cmd_table_count = ARRAY_SIZE(hsw_blt_ring_cmds); } else { @@ -681,70 +681,68 @@ int i915_cmd_parser_init_ring(struct intel_engine_cs *ring) cmd_table_count = ARRAY_SIZE(gen7_blt_cmds); } - ring->reg_table = gen7_blt_regs; - ring->reg_count = ARRAY_SIZE(gen7_blt_regs); + engine->reg_table = gen7_blt_regs; + engine->reg_count = ARRAY_SIZE(gen7_blt_regs); - if (IS_HASWELL(ring->dev)) { - ring->master_reg_table = hsw_master_regs; - ring->master_reg_count = ARRAY_SIZE(hsw_master_regs); + if (IS_HASWELL(engine->i915)) { + engine->master_reg_table = hsw_master_regs; + engine->master_reg_count = ARRAY_SIZE(hsw_master_regs); } else { - ring->master_reg_table = ivb_master_regs; - ring->master_reg_count = ARRAY_SIZE(ivb_master_regs); + engine->master_reg_table = ivb_master_regs; + engine->master_reg_count = ARRAY_SIZE(ivb_master_regs); } - ring->get_cmd_length_mask = gen7_blt_get_cmd_length_mask; + engine->get_cmd_length_mask = gen7_blt_get_cmd_length_mask; break; case VECS: cmd_tables = hsw_vebox_cmds; cmd_table_count = ARRAY_SIZE(hsw_vebox_cmds); /* VECS can use the same length_mask function as VCS */ - ring->get_cmd_length_mask = gen7_bsd_get_cmd_length_mask; + engine->get_cmd_length_mask = gen7_bsd_get_cmd_length_mask; break; default: - DRM_ERROR("CMD: cmd_parser_init with unknown ring: %d\n", - ring->id); + DRM_ERROR("CMD: cmd_parser_init with unknown engine: %d\n", + engine->id); BUG(); } - BUG_ON(!validate_cmds_sorted(ring, cmd_tables, cmd_table_count)); - BUG_ON(!validate_regs_sorted(ring)); + BUG_ON(!validate_cmds_sorted(engine, cmd_tables, cmd_table_count)); + BUG_ON(!validate_regs_sorted(engine)); - if (hash_empty(ring->cmd_hash)) { - ret = init_hash_table(ring, cmd_tables, cmd_table_count); - if (ret) { - DRM_ERROR("CMD: cmd_parser_init failed!\n"); - fini_hash_table(ring); - return ret; - } + ret = init_hash_table(engine, cmd_tables, cmd_table_count); + if (ret) { + DRM_ERROR("CMD: cmd_parser_init failed!\n"); + fini_hash_table(engine); + return ret; } - ring->needs_cmd_parser = true; + engine->needs_cmd_parser = true; return 0; } /** - * i915_cmd_parser_fini_ring() - clean up cmd parser related fields - * @ring: the ringbuffer to clean up + * i915_cmd_parser_fini_engine() - clean up cmd parser related fields + * @engine: the ringbuffer to clean up * * Releases any resources related to command parsing that may have been - * initialized for the specified ring. + * initialized for the specified engine. */ -void i915_cmd_parser_fini_ring(struct intel_engine_cs *ring) +void i915_cmd_parser_fini_engine(struct intel_engine_cs *engine) { - if (!ring->needs_cmd_parser) + if (!engine->needs_cmd_parser) return; - fini_hash_table(ring); + fini_hash_table(engine); } static const struct drm_i915_cmd_descriptor* -find_cmd_in_table(struct intel_engine_cs *ring, +find_cmd_in_table(struct intel_engine_cs *engine, u32 cmd_header) { struct cmd_node *desc_node; - hash_for_each_possible(ring->cmd_hash, desc_node, node, + hash_for_each_possible(engine->cmd_hash, desc_node, node, cmd_header & CMD_HASH_MASK) { const struct drm_i915_cmd_descriptor *desc = desc_node->desc; u32 masked_cmd = desc->cmd.mask & cmd_header; @@ -761,23 +759,23 @@ find_cmd_in_table(struct intel_engine_cs *ring, * Returns a pointer to a descriptor for the command specified by cmd_header. * * The caller must supply space for a default descriptor via the default_desc - * parameter. If no descriptor for the specified command exists in the ring's + * parameter. If no descriptor for the specified command exists in the engine's * command parser tables, this function fills in default_desc based on the - * ring's default length encoding and returns default_desc. + * engine's default length encoding and returns default_desc. */ static const struct drm_i915_cmd_descriptor* -find_cmd(struct intel_engine_cs *ring, +find_cmd(struct intel_engine_cs *engine, u32 cmd_header, struct drm_i915_cmd_descriptor *default_desc) { const struct drm_i915_cmd_descriptor *desc; u32 mask; - desc = find_cmd_in_table(ring, cmd_header); + desc = find_cmd_in_table(engine, cmd_header); if (desc) return desc; - mask = ring->get_cmd_length_mask(cmd_header); + mask = engine->get_cmd_length_mask(cmd_header); if (!mask) return NULL; @@ -834,19 +832,17 @@ finish: } /** - * i915_needs_cmd_parser() - should a given ring use software command parsing? - * @ring: the ring in question + * i915_needs_cmd_parser() - should a given engine use software command parsing? + * @engine: the engine in question * * Only certain platforms require software batch buffer command parsing, and * only when enabled via module paramter. * - * Return: true if the ring requires software command parsing + * Return: true if the engine requires software command parsing */ -bool i915_needs_cmd_parser(struct intel_engine_cs *ring) +bool i915_needs_cmd_parser(struct intel_engine_cs *engine) { - struct drm_i915_private *dev_priv = ring->dev->dev_private; - - if (!ring->needs_cmd_parser) + if (!engine->needs_cmd_parser) return false; /* @@ -854,13 +850,13 @@ bool i915_needs_cmd_parser(struct intel_engine_cs *ring) * disabled. That will cause all of the parser's PPGTT checks to * fail. For now, disable parsing when PPGTT is off. */ - if (!dev_priv->mm.aliasing_ppgtt) + if (USES_PPGTT(engine->dev)) return false; return (i915.enable_cmd_parser == 1); } -static bool check_cmd(const struct intel_engine_cs *ring, +static bool check_cmd(const struct intel_engine_cs *engine, const struct drm_i915_cmd_descriptor *desc, const u32 *cmd, const bool is_master, @@ -897,16 +893,16 @@ static bool check_cmd(const struct intel_engine_cs *ring, *oacontrol_set = (cmd[2] != 0); } - if (!valid_reg(ring->reg_table, - ring->reg_count, reg_addr)) { + if (!valid_reg(engine->reg_table, + engine->reg_count, reg_addr)) { if (!is_master || - !valid_reg(ring->master_reg_table, - ring->master_reg_count, + !valid_reg(engine->master_reg_table, + engine->master_reg_count, reg_addr)) { - DRM_DEBUG_DRIVER("CMD: Rejected register 0x%08X in command: 0x%08X (ring=%d)\n", + DRM_DEBUG_DRIVER("CMD: Rejected register 0x%08X in command: 0x%08X (engine=%d)\n", reg_addr, *cmd, - ring->id); + engine->id); return false; } } @@ -935,11 +931,11 @@ static bool check_cmd(const struct intel_engine_cs *ring, desc->bits[i].mask; if (dword != desc->bits[i].expected) { - DRM_DEBUG_DRIVER("CMD: Rejected command 0x%08X for bitmask 0x%08X (exp=0x%08X act=0x%08X) (ring=%d)\n", + DRM_DEBUG_DRIVER("CMD: Rejected command 0x%08X for bitmask 0x%08X (exp=0x%08X act=0x%08X) (engine=%d)\n", *cmd, desc->bits[i].mask, desc->bits[i].expected, - dword, ring->id); + dword, engine->id); return false; } } @@ -952,7 +948,7 @@ static bool check_cmd(const struct intel_engine_cs *ring, /** * i915_parse_cmds() - parse a submitted batch buffer for privilege violations - * @ring: the ring on which the batch is to execute + * @engine: the engine on which the batch is to execute * @batch_obj: the batch buffer in question * @batch_start_offset: byte offset in the batch at which execution starts * @is_master: is the submitting process the drm master? @@ -962,7 +958,7 @@ static bool check_cmd(const struct intel_engine_cs *ring, * * Return: non-zero if the parser finds violations or otherwise fails */ -int i915_parse_cmds(struct intel_engine_cs *ring, +int i915_parse_cmds(struct intel_engine_cs *engine, struct drm_i915_gem_object *batch_obj, u32 batch_start_offset, bool is_master) @@ -999,7 +995,7 @@ int i915_parse_cmds(struct intel_engine_cs *ring, if (*cmd == MI_BATCH_BUFFER_END) break; - desc = find_cmd(ring, *cmd, &default_desc); + desc = find_cmd(engine, *cmd, &default_desc); if (!desc) { DRM_DEBUG_DRIVER("CMD: Unrecognized command: 0x%08X\n", *cmd); @@ -1021,7 +1017,7 @@ int i915_parse_cmds(struct intel_engine_cs *ring, break; } - if (!check_cmd(ring, desc, cmd, is_master, &oacontrol_set)) { + if (!check_cmd(engine, desc, cmd, is_master, &oacontrol_set)) { ret = -EINVAL; break; } diff --git a/drivers/gpu/drm/i915/i915_debugfs.c b/drivers/gpu/drm/i915/i915_debugfs.c index 9e737b7..d5f237e 100644 --- a/drivers/gpu/drm/i915/i915_debugfs.c +++ b/drivers/gpu/drm/i915/i915_debugfs.c @@ -123,20 +123,23 @@ static void describe_obj(struct seq_file *m, struct drm_i915_gem_object *obj) { struct i915_vma *vma; - int pin_count = 0; + int pin_count = 0, n; - seq_printf(m, "%pK: %s%s%s %8zdKiB %02x %02x %u %u %u%s%s%s", + seq_printf(m, "%pK: %s%s%s %8zdKiB %02x %02x [", &obj->base, get_pin_flag(obj), get_tiling_flag(obj), get_global_flag(obj), obj->base.size / 1024, obj->base.read_domains, - obj->base.write_domain, - obj->last_read_seqno, - obj->last_write_seqno, - obj->last_fenced_seqno, - i915_cache_level_str(obj->cache_level), + obj->base.write_domain); + for (n = 0; n < ARRAY_SIZE(obj->last_read); n++) + seq_printf(m, " %x", + i915_request_seqno(obj->last_read[n].request)); + seq_printf(m, " ] %x %x%s%s%s", + i915_request_seqno(obj->last_write.request), + i915_request_seqno(obj->last_fence.request), + i915_cache_level_str(to_i915(obj->base.dev), obj->cache_level), obj->dirty ? " dirty" : "", obj->madv == I915_MADV_DONTNEED ? " purgeable" : ""); if (obj->base.name) @@ -168,15 +171,15 @@ describe_obj(struct seq_file *m, struct drm_i915_gem_object *obj) *t = '\0'; seq_printf(m, " (%s mappable)", s); } - if (obj->ring != NULL) - seq_printf(m, " (%s)", obj->ring->name); + if (obj->last_write.request) + seq_printf(m, " (%s)", obj->last_write.request->engine->name); if (obj->frontbuffer_bits) seq_printf(m, " (frontbuffer: 0x%03x)", obj->frontbuffer_bits); } static void describe_ctx(struct seq_file *m, struct intel_context *ctx) { - seq_putc(m, ctx->legacy_hw_ctx.initialized ? 'I' : 'i'); + seq_putc(m, ctx->ring[RCS].initialized ? 'I' : 'i'); seq_putc(m, ctx->remap_slice ? 'R' : 'r'); seq_putc(m, ' '); } @@ -333,10 +336,10 @@ static int per_file_stats(int id, void *ptr, void *data) } ppgtt = container_of(vma->vm, struct i915_hw_ppgtt, base); - if (ppgtt->ctx && ppgtt->ctx->file_priv != stats->file_priv) + if (ppgtt->file_priv != stats->file_priv) continue; - if (obj->ring) /* XXX per-vma statistic */ + if (obj->active) /* XXX per-vma statistic */ stats->active += obj->base.size; else stats->inactive += obj->base.size; @@ -346,7 +349,7 @@ static int per_file_stats(int id, void *ptr, void *data) } else { if (i915_gem_obj_ggtt_bound(obj)) { stats->global += obj->base.size; - if (obj->ring) + if (obj->active) stats->active += obj->base.size; else stats->inactive += obj->base.size; @@ -515,7 +518,7 @@ static int i915_gem_pageflip_info(struct seq_file *m, void *data) { struct drm_info_node *node = m->private; struct drm_device *dev = node->minor->dev; - unsigned long flags; + struct drm_i915_private *dev_priv = dev->dev_private; struct intel_crtc *crtc; int ret; @@ -528,12 +531,14 @@ static int i915_gem_pageflip_info(struct seq_file *m, void *data) const char plane = plane_name(crtc->plane); struct intel_unpin_work *work; - spin_lock_irqsave(&dev->event_lock, flags); + spin_lock_irq(&dev->event_lock); work = crtc->unpin_work; if (work == NULL) { seq_printf(m, "No flip due on pipe %c (plane %c)\n", pipe, plane); } else { + u32 addr; + if (atomic_read(&work->pending) < INTEL_FLIP_COMPLETE) { seq_printf(m, "Flip queued on pipe %c (plane %c)\n", pipe, plane); @@ -541,26 +546,38 @@ static int i915_gem_pageflip_info(struct seq_file *m, void *data) seq_printf(m, "Flip pending (waiting for vsync) on pipe %c (plane %c)\n", pipe, plane); } + if (work->flip_queued_request) { + struct i915_gem_request *rq = + work->flip_queued_request; + seq_printf(m, "Flip queued on %s at seqno %u, next seqno %u [current breadcrumb %u], completed? %d\n", + rq->engine->name, + rq->seqno, rq->i915->next_seqno, + rq->engine->get_seqno(rq->engine), + __i915_request_complete__wa(rq)); + } else + seq_printf(m, "Flip not associated with any ring\n"); + seq_printf(m, "Flip queued on frame %d, (was ready on frame %d), now %d\n", + work->flip_queued_vblank, + work->flip_ready_vblank, + drm_vblank_count(dev, crtc->pipe)); if (work->enable_stall_check) seq_puts(m, "Stall check enabled, "); else seq_puts(m, "Stall check waiting for page flip ioctl, "); seq_printf(m, "%d prepares\n", atomic_read(&work->pending)); - if (work->old_fb_obj) { - struct drm_i915_gem_object *obj = work->old_fb_obj; - if (obj) - seq_printf(m, "Old framebuffer gtt_offset 0x%08lx\n", - i915_gem_obj_ggtt_offset(obj)); - } + if (INTEL_INFO(dev)->gen >= 4) + addr = I915_HI_DISPBASE(I915_READ(DSPSURF(crtc->plane))); + else + addr = I915_READ(DSPADDR(crtc->plane)); + seq_printf(m, "Current scanout address 0x%08x\n", addr); + if (work->pending_flip_obj) { - struct drm_i915_gem_object *obj = work->pending_flip_obj; - if (obj) - seq_printf(m, "New framebuffer gtt_offset 0x%08lx\n", - i915_gem_obj_ggtt_offset(obj)); + seq_printf(m, "New framebuffer address 0x%08lx\n", (long)work->gtt_offset); + seq_printf(m, "MMIO update completed? %d\n", addr == work->gtt_offset); } } - spin_unlock_irqrestore(&dev->event_lock, flags); + spin_unlock_irq(&dev->event_lock); } mutex_unlock(&dev->struct_mutex); @@ -573,8 +590,8 @@ static int i915_gem_request_info(struct seq_file *m, void *data) struct drm_info_node *node = m->private; struct drm_device *dev = node->minor->dev; struct drm_i915_private *dev_priv = dev->dev_private; - struct intel_engine_cs *ring; - struct drm_i915_gem_request *gem_request; + struct intel_engine_cs *engine; + struct i915_gem_request *rq; int ret, count, i; ret = mutex_lock_interruptible(&dev->struct_mutex); @@ -582,17 +599,15 @@ static int i915_gem_request_info(struct seq_file *m, void *data) return ret; count = 0; - for_each_ring(ring, dev_priv, i) { - if (list_empty(&ring->request_list)) + for_each_engine(engine, dev_priv, i) { + if (list_empty(&engine->requests)) continue; - seq_printf(m, "%s requests:\n", ring->name); - list_for_each_entry(gem_request, - &ring->request_list, - list) { + seq_printf(m, "%s requests:\n", engine->name); + list_for_each_entry(rq, &engine->requests, engine_list) { seq_printf(m, " %d @ %d\n", - gem_request->seqno, - (int) (jiffies - gem_request->emitted_jiffies)); + rq->seqno, + (int)(jiffies - rq->emitted_jiffies)); } count++; } @@ -604,13 +619,17 @@ static int i915_gem_request_info(struct seq_file *m, void *data) return 0; } -static void i915_ring_seqno_info(struct seq_file *m, - struct intel_engine_cs *ring) +static void i915_engine_seqno_info(struct seq_file *m, + struct intel_engine_cs *engine) { - if (ring->get_seqno) { - seq_printf(m, "Current sequence (%s): %u\n", - ring->name, ring->get_seqno(ring, false)); - } + seq_printf(m, "Current sequence (%s): seqno=%u, tag=%u [last breadcrumb %u, last request %u], next seqno=%u, next tag=%u\n", + engine->name, + engine->get_seqno(engine), + engine->tag, + engine->breadcrumb[engine->id], + engine->last_request ? engine->last_request->seqno : 0, + engine->i915->next_seqno, + engine->next_tag); } static int i915_gem_seqno_info(struct seq_file *m, void *data) @@ -618,7 +637,7 @@ static int i915_gem_seqno_info(struct seq_file *m, void *data) struct drm_info_node *node = m->private; struct drm_device *dev = node->minor->dev; struct drm_i915_private *dev_priv = dev->dev_private; - struct intel_engine_cs *ring; + struct intel_engine_cs *engine; int ret, i; ret = mutex_lock_interruptible(&dev->struct_mutex); @@ -626,8 +645,8 @@ static int i915_gem_seqno_info(struct seq_file *m, void *data) return ret; intel_runtime_pm_get(dev_priv); - for_each_ring(ring, dev_priv, i) - i915_ring_seqno_info(m, ring); + for_each_engine(engine, dev_priv, i) + i915_engine_seqno_info(m, engine); intel_runtime_pm_put(dev_priv); mutex_unlock(&dev->struct_mutex); @@ -641,7 +660,7 @@ static int i915_interrupt_info(struct seq_file *m, void *data) struct drm_info_node *node = m->private; struct drm_device *dev = node->minor->dev; struct drm_i915_private *dev_priv = dev->dev_private; - struct intel_engine_cs *ring; + struct intel_engine_cs *engine; int ret, i, pipe; ret = mutex_lock_interruptible(&dev->struct_mutex); @@ -650,7 +669,6 @@ static int i915_interrupt_info(struct seq_file *m, void *data) intel_runtime_pm_get(dev_priv); if (IS_CHERRYVIEW(dev)) { - int i; seq_printf(m, "Master Interrupt Control:\t%08x\n", I915_READ(GEN8_MASTER_IRQ)); @@ -662,7 +680,7 @@ static int i915_interrupt_info(struct seq_file *m, void *data) I915_READ(VLV_IIR_RW)); seq_printf(m, "Display IMR:\t%08x\n", I915_READ(VLV_IMR)); - for_each_pipe(pipe) + for_each_pipe(dev_priv, pipe) seq_printf(m, "Pipe %c stat:\t%08x\n", pipe_name(pipe), I915_READ(PIPESTAT(pipe))); @@ -702,7 +720,13 @@ static int i915_interrupt_info(struct seq_file *m, void *data) i, I915_READ(GEN8_GT_IER(i))); } - for_each_pipe(pipe) { + for_each_pipe(dev_priv, pipe) { + if (!intel_display_power_enabled(dev_priv, + POWER_DOMAIN_PIPE(pipe))) { + seq_printf(m, "Pipe %c power disabled\n", + pipe_name(pipe)); + continue; + } seq_printf(m, "Pipe %c IMR:\t%08x\n", pipe_name(pipe), I915_READ(GEN8_DE_PIPE_IMR(pipe))); @@ -743,7 +767,7 @@ static int i915_interrupt_info(struct seq_file *m, void *data) I915_READ(VLV_IIR_RW)); seq_printf(m, "Display IMR:\t%08x\n", I915_READ(VLV_IMR)); - for_each_pipe(pipe) + for_each_pipe(dev_priv, pipe) seq_printf(m, "Pipe %c stat:\t%08x\n", pipe_name(pipe), I915_READ(PIPESTAT(pipe))); @@ -779,7 +803,7 @@ static int i915_interrupt_info(struct seq_file *m, void *data) I915_READ(IIR)); seq_printf(m, "Interrupt mask: %08x\n", I915_READ(IMR)); - for_each_pipe(pipe) + for_each_pipe(dev_priv, pipe) seq_printf(m, "Pipe %c stat: %08x\n", pipe_name(pipe), I915_READ(PIPESTAT(pipe))); @@ -803,13 +827,13 @@ static int i915_interrupt_info(struct seq_file *m, void *data) seq_printf(m, "Graphics Interrupt mask: %08x\n", I915_READ(GTIMR)); } - for_each_ring(ring, dev_priv, i) { + for_each_engine(engine, dev_priv, i) { if (INTEL_INFO(dev)->gen >= 6) { seq_printf(m, "Graphics Interrupt mask (%s): %08x\n", - ring->name, I915_READ_IMR(ring)); + engine->name, I915_READ_IMR(engine)); } - i915_ring_seqno_info(m, ring); + i915_engine_seqno_info(m, engine); } intel_runtime_pm_put(dev_priv); mutex_unlock(&dev->struct_mutex); @@ -851,12 +875,12 @@ static int i915_hws_info(struct seq_file *m, void *data) struct drm_info_node *node = m->private; struct drm_device *dev = node->minor->dev; struct drm_i915_private *dev_priv = dev->dev_private; - struct intel_engine_cs *ring; + struct intel_engine_cs *engine; const u32 *hws; int i; - ring = &dev_priv->ring[(uintptr_t)node->info_ent->data]; - hws = ring->status_page.page_addr; + engine = &dev_priv->engine[(uintptr_t)node->info_ent->data]; + hws = engine->status_page.page_addr; if (hws == NULL) return 0; @@ -927,7 +951,7 @@ static ssize_t i915_error_state_read(struct file *file, char __user *userbuf, ssize_t ret_count = 0; int ret; - ret = i915_error_state_buf_init(&error_str, count, *pos); + ret = i915_error_state_buf_init(&error_str, to_i915(error_priv->dev), count, *pos); if (ret) return ret; @@ -980,7 +1004,7 @@ i915_next_seqno_set(void *data, u64 val) struct drm_device *dev = data; int ret; - ret = mutex_lock_interruptible(&dev->struct_mutex); + ret = i915_mutex_lock_interruptible(dev); if (ret) return ret; @@ -1024,6 +1048,7 @@ static int i915_frequency_info(struct seq_file *m, void *unused) u32 rpstat, cagf, reqf; u32 rpupei, rpcurup, rpprevup; u32 rpdownei, rpcurdown, rpprevdown; + u32 pm_ier, pm_imr, pm_isr, pm_iir, pm_mask; int max_freq; /* RPSTAT1 is in the GT power well */ @@ -1061,12 +1086,21 @@ static int i915_frequency_info(struct seq_file *m, void *unused) gen6_gt_force_wake_put(dev_priv, FORCEWAKE_ALL); mutex_unlock(&dev->struct_mutex); + if (IS_GEN6(dev) || IS_GEN7(dev)) { + pm_ier = I915_READ(GEN6_PMIER); + pm_imr = I915_READ(GEN6_PMIMR); + pm_isr = I915_READ(GEN6_PMISR); + pm_iir = I915_READ(GEN6_PMIIR); + pm_mask = I915_READ(GEN6_PMINTRMSK); + } else { + pm_ier = I915_READ(GEN8_GT_IER(2)); + pm_imr = I915_READ(GEN8_GT_IMR(2)); + pm_isr = I915_READ(GEN8_GT_ISR(2)); + pm_iir = I915_READ(GEN8_GT_IIR(2)); + pm_mask = I915_READ(GEN6_PMINTRMSK); + } seq_printf(m, "PM IER=0x%08x IMR=0x%08x ISR=0x%08x IIR=0x%08x, MASK=0x%08x\n", - I915_READ(GEN6_PMIER), - I915_READ(GEN6_PMIMR), - I915_READ(GEN6_PMISR), - I915_READ(GEN6_PMIIR), - I915_READ(GEN6_PMINTRMSK)); + pm_ier, pm_imr, pm_isr, pm_iir, pm_mask); seq_printf(m, "GT_PERF_STATUS: 0x%08x\n", gt_perf_status); seq_printf(m, "Render p-state ratio: %d\n", (gt_perf_status & 0xff00) >> 8); @@ -1365,7 +1399,7 @@ static int i915_drpc_info(struct seq_file *m, void *unused) if (IS_VALLEYVIEW(dev)) return vlv_drpc_info(m); - else if (IS_GEN6(dev) || IS_GEN7(dev)) + else if (INTEL_INFO(dev)->gen >= 6) return gen6_drpc_info(m); else return ironlake_drpc_info(m); @@ -1433,6 +1467,47 @@ static int i915_fbc_status(struct seq_file *m, void *unused) return 0; } +static int i915_fbc_fc_get(void *data, u64 *val) +{ + struct drm_device *dev = data; + struct drm_i915_private *dev_priv = dev->dev_private; + + if (INTEL_INFO(dev)->gen < 7 || !HAS_FBC(dev)) + return -ENODEV; + + drm_modeset_lock_all(dev); + *val = dev_priv->fbc.false_color; + drm_modeset_unlock_all(dev); + + return 0; +} + +static int i915_fbc_fc_set(void *data, u64 val) +{ + struct drm_device *dev = data; + struct drm_i915_private *dev_priv = dev->dev_private; + u32 reg; + + if (INTEL_INFO(dev)->gen < 7 || !HAS_FBC(dev)) + return -ENODEV; + + drm_modeset_lock_all(dev); + + reg = I915_READ(ILK_DPFC_CONTROL); + dev_priv->fbc.false_color = val; + + I915_WRITE(ILK_DPFC_CONTROL, val ? + (reg | FBC_CTL_FALSE_COLOR) : + (reg & ~FBC_CTL_FALSE_COLOR)); + + drm_modeset_unlock_all(dev); + return 0; +} + +DEFINE_SIMPLE_ATTRIBUTE(i915_fbc_fc_fops, + i915_fbc_fc_get, i915_fbc_fc_set, + "%llu\n"); + static int i915_ips_status(struct seq_file *m, void *unused) { struct drm_info_node *node = m->private; @@ -1630,12 +1705,18 @@ static int i915_gem_framebuffer_info(struct seq_file *m, void *data) return 0; } +static void describe_ring(struct seq_file *m, struct intel_ringbuffer *ring) +{ + seq_printf(m, " (ringbuffer, space: %d, head: %u, tail: %u, last head: %d)", + ring->space, ring->head, ring->tail, ring->retired_head); +} + static int i915_context_status(struct seq_file *m, void *unused) { struct drm_info_node *node = m->private; struct drm_device *dev = node->minor->dev; struct drm_i915_private *dev_priv = dev->dev_private; - struct intel_engine_cs *ring; + struct intel_engine_cs *engine; struct intel_context *ctx; int ret, i; @@ -1649,23 +1730,28 @@ static int i915_context_status(struct seq_file *m, void *unused) seq_putc(m, '\n'); } - if (dev_priv->ips.renderctx) { - seq_puts(m, "render context "); - describe_obj(m, dev_priv->ips.renderctx); - seq_putc(m, '\n'); - } - list_for_each_entry(ctx, &dev_priv->context_list, link) { - if (ctx->legacy_hw_ctx.rcs_state == NULL) - continue; - seq_puts(m, "HW context "); describe_ctx(m, ctx); - for_each_ring(ring, dev_priv, i) - if (ring->default_context == ctx) - seq_printf(m, "(default context %s) ", ring->name); + for_each_engine(engine, dev_priv, i) { + if (engine->default_context == ctx) + seq_printf(m, "(default context %s) ", + engine->name); + } + + seq_putc(m, '\n'); + for_each_engine(engine, dev_priv, i) { + struct drm_i915_gem_object *obj = ctx->ring[i].state; + struct intel_ringbuffer *ring = ctx->ring[i].ring; + + seq_printf(m, "%s: ", engine->name); + if (obj) + describe_obj(m, obj); + if (ring) + describe_ring(m, ring); + seq_putc(m, '\n'); + } - describe_obj(m, ctx->legacy_hw_ctx.rcs_state); seq_putc(m, '\n'); } @@ -1774,7 +1860,13 @@ static int per_file_ctx(int id, void *ptr, void *data) { struct intel_context *ctx = ptr; struct seq_file *m = data; - struct i915_hw_ppgtt *ppgtt = ctx_to_ppgtt(ctx); + struct i915_hw_ppgtt *ppgtt = ctx->ppgtt; + + if (!ppgtt) { + seq_printf(m, " no ppgtt for context %d\n", + ctx->user_handle); + return 0; + } if (i915_gem_context_is_default(ctx)) seq_puts(m, " default context:\n"); @@ -1788,7 +1880,7 @@ static int per_file_ctx(int id, void *ptr, void *data) static void gen8_ppgtt_info(struct seq_file *m, struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; - struct intel_engine_cs *ring; + struct intel_engine_cs *engine; struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt; int unused, i; @@ -1797,13 +1889,13 @@ static void gen8_ppgtt_info(struct seq_file *m, struct drm_device *dev) seq_printf(m, "Page directories: %d\n", ppgtt->num_pd_pages); seq_printf(m, "Page tables: %d\n", ppgtt->num_pd_entries); - for_each_ring(ring, dev_priv, unused) { - seq_printf(m, "%s\n", ring->name); + for_each_engine(engine, dev_priv, unused) { + seq_printf(m, "%s\n", engine->name); for (i = 0; i < 4; i++) { u32 offset = 0x270 + i * 8; - u64 pdp = I915_READ(ring->mmio_base + offset + 4); + u64 pdp = I915_READ(engine->mmio_base + offset + 4); pdp <<= 32; - pdp |= I915_READ(ring->mmio_base + offset); + pdp |= I915_READ(engine->mmio_base + offset); seq_printf(m, "\tPDP%d 0x%016llx\n", i, pdp); } } @@ -1812,20 +1904,20 @@ static void gen8_ppgtt_info(struct seq_file *m, struct drm_device *dev) static void gen6_ppgtt_info(struct seq_file *m, struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; - struct intel_engine_cs *ring; + struct intel_engine_cs *engine; struct drm_file *file; int i; if (INTEL_INFO(dev)->gen == 6) seq_printf(m, "GFX_MODE: 0x%08x\n", I915_READ(GFX_MODE)); - for_each_ring(ring, dev_priv, i) { - seq_printf(m, "%s\n", ring->name); + for_each_engine(engine, dev_priv, i) { + seq_printf(m, "%s\n", engine->name); if (INTEL_INFO(dev)->gen == 7) - seq_printf(m, "GFX_MODE: 0x%08x\n", I915_READ(RING_MODE_GEN7(ring))); - seq_printf(m, "PP_DIR_BASE: 0x%08x\n", I915_READ(RING_PP_DIR_BASE(ring))); - seq_printf(m, "PP_DIR_BASE_READ: 0x%08x\n", I915_READ(RING_PP_DIR_BASE_READ(ring))); - seq_printf(m, "PP_DIR_DCLV: 0x%08x\n", I915_READ(RING_PP_DIR_DCLV(ring))); + seq_printf(m, "GFX_MODE: 0x%08x\n", I915_READ(RING_MODE_GEN7(engine))); + seq_printf(m, "PP_DIR_BASE: 0x%08x\n", I915_READ(RING_PP_DIR_BASE(engine))); + seq_printf(m, "PP_DIR_BASE_READ: 0x%08x\n", I915_READ(RING_PP_DIR_BASE_READ(engine))); + seq_printf(m, "PP_DIR_DCLV: 0x%08x\n", I915_READ(RING_PP_DIR_DCLV(engine))); } if (dev_priv->mm.aliasing_ppgtt) { struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt; @@ -1834,8 +1926,7 @@ static void gen6_ppgtt_info(struct seq_file *m, struct drm_device *dev) seq_printf(m, "pd gtt offset: 0x%08x\n", ppgtt->pd_offset); ppgtt->debug_dump(ppgtt, m); - } else - return; + } list_for_each_entry_reverse(file, &dev->filelist, lhead) { struct drm_i915_file_private *file_priv = file->driver_priv; @@ -2313,67 +2404,62 @@ static int i915_semaphore_status(struct seq_file *m, void *unused) struct drm_info_node *node = (struct drm_info_node *) m->private; struct drm_device *dev = node->minor->dev; struct drm_i915_private *dev_priv = dev->dev_private; - struct intel_engine_cs *ring; + struct intel_engine_cs *engine; int num_rings = hweight32(INTEL_INFO(dev)->ring_mask); int i, j, ret; - if (!i915_semaphore_is_enabled(dev)) { - seq_puts(m, "Semaphores are disabled\n"); - return 0; - } - ret = mutex_lock_interruptible(&dev->struct_mutex); if (ret) return ret; intel_runtime_pm_get(dev_priv); - if (IS_BROADWELL(dev)) { - struct page *page; - uint64_t *seqno; + seq_puts(m, " Last breadcrumb:"); + for_each_engine(engine, dev_priv, i) + for (j = 0; j < num_rings; j++) + seq_printf(m, "0x%08x\n", + engine->breadcrumb[j]); + seq_putc(m, '\n'); - page = i915_gem_object_get_page(dev_priv->semaphore_obj, 0); + if (engine->semaphore.wait) { + if (IS_BROADWELL(dev)) { + struct page *page; + uint64_t *seqno; - seqno = (uint64_t *)kmap_atomic(page); - for_each_ring(ring, dev_priv, i) { - uint64_t offset; + page = i915_gem_object_get_page(dev_priv->semaphore_obj, 0); - seq_printf(m, "%s\n", ring->name); + seqno = (uint64_t *)kmap_atomic(page); + for_each_engine(engine, dev_priv, i) { + uint64_t offset; - seq_puts(m, " Last signal:"); - for (j = 0; j < num_rings; j++) { - offset = i * I915_NUM_RINGS + j; - seq_printf(m, "0x%08llx (0x%02llx) ", - seqno[offset], offset * 8); - } - seq_putc(m, '\n'); + seq_printf(m, "%s\n", engine->name); - seq_puts(m, " Last wait: "); - for (j = 0; j < num_rings; j++) { - offset = i + (j * I915_NUM_RINGS); - seq_printf(m, "0x%08llx (0x%02llx) ", - seqno[offset], offset * 8); - } - seq_putc(m, '\n'); + seq_puts(m, " Last signal:"); + for (j = 0; j < num_rings; j++) { + offset = i * I915_NUM_ENGINES + j; + seq_printf(m, "0x%08llx (0x%02llx) ", + seqno[offset], offset * 8); + } + seq_putc(m, '\n'); - } - kunmap_atomic(seqno); - } else { - seq_puts(m, " Last signal:"); - for_each_ring(ring, dev_priv, i) - for (j = 0; j < num_rings; j++) - seq_printf(m, "0x%08x\n", - I915_READ(ring->semaphore.mbox.signal[j])); - seq_putc(m, '\n'); - } + seq_puts(m, " Last wait: "); + for (j = 0; j < num_rings; j++) { + offset = i + (j * I915_NUM_ENGINES); + seq_printf(m, "0x%08llx (0x%02llx) ", + seqno[offset], offset * 8); + } + seq_putc(m, '\n'); - seq_puts(m, "\nSync seqno:\n"); - for_each_ring(ring, dev_priv, i) { - for (j = 0; j < num_rings; j++) { - seq_printf(m, " 0x%08x ", ring->semaphore.sync_seqno[j]); + } + kunmap_atomic(seqno); + } else { + seq_puts(m, " Last signal:"); + for_each_engine(engine, dev_priv, i) + for (j = 0; j < num_rings; j++) + seq_printf(m, "0x%08x\n", + I915_READ(engine->semaphore.mbox.signal[j])); + seq_putc(m, '\n'); } - seq_putc(m, '\n'); } - seq_putc(m, '\n'); intel_runtime_pm_put(dev_priv); mutex_unlock(&dev->struct_mutex); @@ -2406,6 +2492,40 @@ static int i915_shared_dplls_info(struct seq_file *m, void *unused) return 0; } +static int i915_wa_registers(struct seq_file *m, void *unused) +{ + int i; + int ret; + struct drm_info_node *node = (struct drm_info_node *) m->private; + struct drm_device *dev = node->minor->dev; + struct drm_i915_private *dev_priv = dev->dev_private; + + ret = mutex_lock_interruptible(&dev->struct_mutex); + if (ret) + return ret; + + intel_runtime_pm_get(dev_priv); + + seq_printf(m, "Workarounds applied: %d\n", dev_priv->num_wa_regs); + for (i = 0; i < dev_priv->num_wa_regs; ++i) { + u32 addr, mask; + + addr = dev_priv->intel_wa_regs[i].addr; + mask = dev_priv->intel_wa_regs[i].mask; + dev_priv->intel_wa_regs[i].value = I915_READ(addr) | mask; + if (dev_priv->intel_wa_regs[i].addr) + seq_printf(m, "0x%X: 0x%08X, mask: 0x%08X\n", + dev_priv->intel_wa_regs[i].addr, + dev_priv->intel_wa_regs[i].value, + dev_priv->intel_wa_regs[i].mask); + } + + intel_runtime_pm_put(dev_priv); + mutex_unlock(&dev->struct_mutex); + + return 0; +} + struct pipe_crc_info { const char *name; struct drm_device *dev; @@ -2667,8 +2787,7 @@ static int i9xx_pipe_crc_auto_source(struct drm_device *dev, enum pipe pipe, *source = INTEL_PIPE_CRC_SOURCE_PIPE; drm_modeset_lock_all(dev); - list_for_each_entry(encoder, &dev->mode_config.encoder_list, - base.head) { + for_each_intel_encoder(dev, encoder) { if (!encoder->base.crtc) continue; @@ -3557,9 +3676,6 @@ i915_drop_caches_set(void *data, u64 val) { struct drm_device *dev = data; struct drm_i915_private *dev_priv = dev->dev_private; - struct drm_i915_gem_object *obj, *next; - struct i915_address_space *vm; - struct i915_vma *vma, *x; int ret; DRM_DEBUG("Dropping caches: 0x%08llx\n", val); @@ -3579,29 +3695,11 @@ i915_drop_caches_set(void *data, u64 val) if (val & (DROP_RETIRE | DROP_ACTIVE)) i915_gem_retire_requests(dev); - if (val & DROP_BOUND) { - list_for_each_entry(vm, &dev_priv->vm_list, global_link) { - list_for_each_entry_safe(vma, x, &vm->inactive_list, - mm_list) { - if (vma->pin_count) - continue; + if (val & DROP_BOUND) + i915_gem_shrink(dev_priv, LONG_MAX, I915_SHRINK_BOUND); - ret = i915_vma_unbind(vma); - if (ret) - goto unlock; - } - } - } - - if (val & DROP_UNBOUND) { - list_for_each_entry_safe(obj, next, &dev_priv->mm.unbound_list, - global_list) - if (obj->pages_pin_count == 0) { - ret = i915_gem_object_put_pages(obj); - if (ret) - goto unlock; - } - } + if (val & DROP_UNBOUND) + i915_gem_shrink(dev_priv, LONG_MAX, I915_SHRINK_UNBOUND); unlock: mutex_unlock(&dev->struct_mutex); @@ -3936,6 +4034,7 @@ static const struct drm_info_list i915_debugfs_list[] = { {"i915_semaphore_status", i915_semaphore_status, 0}, {"i915_shared_dplls_info", i915_shared_dplls_info, 0}, {"i915_dp_mst_info", i915_dp_mst_info, 0}, + {"i915_wa_registers", i915_wa_registers, 0}, }; #define I915_DEBUGFS_ENTRIES ARRAY_SIZE(i915_debugfs_list) @@ -3957,6 +4056,7 @@ static const struct i915_debugfs_files { {"i915_pri_wm_latency", &i915_pri_wm_latency_fops}, {"i915_spr_wm_latency", &i915_spr_wm_latency_fops}, {"i915_cur_wm_latency", &i915_cur_wm_latency_fops}, + {"i915_fbc_false_color", &i915_fbc_fc_fops}, }; void intel_display_crc_init(struct drm_device *dev) @@ -3964,7 +4064,7 @@ void intel_display_crc_init(struct drm_device *dev) struct drm_i915_private *dev_priv = dev->dev_private; enum pipe pipe; - for_each_pipe(pipe) { + for_each_pipe(dev_priv, pipe) { struct intel_pipe_crc *pipe_crc = &dev_priv->pipe_crc[pipe]; pipe_crc->opened = false; diff --git a/drivers/gpu/drm/i915/i915_dma.c b/drivers/gpu/drm/i915/i915_dma.c index 9933c26..31577d6 100644 --- a/drivers/gpu/drm/i915/i915_dma.c +++ b/drivers/gpu/drm/i915/i915_dma.c @@ -28,6 +28,7 @@ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt +#include #include #include #include @@ -48,883 +49,58 @@ #include #include -#define LP_RING(d) (&((struct drm_i915_private *)(d))->ring[RCS]) - -#define BEGIN_LP_RING(n) \ - intel_ring_begin(LP_RING(dev_priv), (n)) - -#define OUT_RING(x) \ - intel_ring_emit(LP_RING(dev_priv), x) - -#define ADVANCE_LP_RING() \ - __intel_ring_advance(LP_RING(dev_priv)) - -/** - * Lock test for when it's just for synchronization of ring access. - * - * In that case, we don't need to do it when GEM is initialized as nobody else - * has access to the ring. - */ -#define RING_LOCK_TEST_WITH_RETURN(dev, file) do { \ - if (LP_RING(dev->dev_private)->buffer->obj == NULL) \ - LOCK_TEST_WITH_RETURN(dev, file); \ -} while (0) - -static inline u32 -intel_read_legacy_status_page(struct drm_i915_private *dev_priv, int reg) -{ - if (I915_NEED_GFX_HWS(dev_priv->dev)) - return ioread32(dev_priv->dri1.gfx_hws_cpu_addr + reg); - else - return intel_read_status_page(LP_RING(dev_priv), reg); -} - -#define READ_HWSP(dev_priv, reg) intel_read_legacy_status_page(dev_priv, reg) -#define READ_BREADCRUMB(dev_priv) READ_HWSP(dev_priv, I915_BREADCRUMB_INDEX) -#define I915_BREADCRUMB_INDEX 0x21 - -void i915_update_dri1_breadcrumb(struct drm_device *dev) -{ - struct drm_i915_private *dev_priv = dev->dev_private; - struct drm_i915_master_private *master_priv; - - /* - * The dri breadcrumb update races against the drm master disappearing. - * Instead of trying to fix this (this is by far not the only ums issue) - * just don't do the update in kms mode. - */ - if (drm_core_check_feature(dev, DRIVER_MODESET)) - return; - - if (dev->primary->master) { - master_priv = dev->primary->master->driver_priv; - if (master_priv->sarea_priv) - master_priv->sarea_priv->last_dispatch = - READ_BREADCRUMB(dev_priv); - } -} - -static void i915_write_hws_pga(struct drm_device *dev) -{ - struct drm_i915_private *dev_priv = dev->dev_private; - u32 addr; - - addr = dev_priv->status_page_dmah->busaddr; - if (INTEL_INFO(dev)->gen >= 4) - addr |= (dev_priv->status_page_dmah->busaddr >> 28) & 0xf0; - I915_WRITE(HWS_PGA, addr); -} - -/** - * Frees the hardware status page, whether it's a physical address or a virtual - * address set up by the X Server. - */ -static void i915_free_hws(struct drm_device *dev) -{ - struct drm_i915_private *dev_priv = dev->dev_private; - struct intel_engine_cs *ring = LP_RING(dev_priv); - - if (dev_priv->status_page_dmah) { - drm_pci_free(dev, dev_priv->status_page_dmah); - dev_priv->status_page_dmah = NULL; - } - - if (ring->status_page.gfx_addr) { - ring->status_page.gfx_addr = 0; - iounmap(dev_priv->dri1.gfx_hws_cpu_addr); - } - - /* Need to rewrite hardware status page */ - I915_WRITE(HWS_PGA, 0x1ffff000); -} - -void i915_kernel_lost_context(struct drm_device *dev) -{ - struct drm_i915_private *dev_priv = dev->dev_private; - struct drm_i915_master_private *master_priv; - struct intel_engine_cs *ring = LP_RING(dev_priv); - struct intel_ringbuffer *ringbuf = ring->buffer; - - /* - * We should never lose context on the ring with modesetting - * as we don't expose it to userspace - */ - if (drm_core_check_feature(dev, DRIVER_MODESET)) - return; - - ringbuf->head = I915_READ_HEAD(ring) & HEAD_ADDR; - ringbuf->tail = I915_READ_TAIL(ring) & TAIL_ADDR; - ringbuf->space = ringbuf->head - (ringbuf->tail + I915_RING_FREE_SPACE); - if (ringbuf->space < 0) - ringbuf->space += ringbuf->size; - - if (!dev->primary->master) - return; - - master_priv = dev->primary->master->driver_priv; - if (ringbuf->head == ringbuf->tail && master_priv->sarea_priv) - master_priv->sarea_priv->perf_boxes |= I915_BOX_RING_EMPTY; -} - -static int i915_dma_cleanup(struct drm_device *dev) -{ - struct drm_i915_private *dev_priv = dev->dev_private; - int i; - - /* Make sure interrupts are disabled here because the uninstall ioctl - * may not have been called from userspace and after dev_private - * is freed, it's too late. - */ - if (dev->irq_enabled) - drm_irq_uninstall(dev); - - mutex_lock(&dev->struct_mutex); - for (i = 0; i < I915_NUM_RINGS; i++) - intel_cleanup_ring_buffer(&dev_priv->ring[i]); - mutex_unlock(&dev->struct_mutex); - - /* Clear the HWS virtual address at teardown */ - if (I915_NEED_GFX_HWS(dev)) - i915_free_hws(dev); - - return 0; -} - -static int i915_initialize(struct drm_device *dev, drm_i915_init_t *init) -{ - struct drm_i915_private *dev_priv = dev->dev_private; - struct drm_i915_master_private *master_priv = dev->primary->master->driver_priv; - int ret; - - master_priv->sarea = drm_getsarea(dev); - if (master_priv->sarea) { - master_priv->sarea_priv = (drm_i915_sarea_t *) - ((u8 *)master_priv->sarea->handle + init->sarea_priv_offset); - } else { - DRM_DEBUG_DRIVER("sarea not found assuming DRI2 userspace\n"); - } - - if (init->ring_size != 0) { - if (LP_RING(dev_priv)->buffer->obj != NULL) { - i915_dma_cleanup(dev); - DRM_ERROR("Client tried to initialize ringbuffer in " - "GEM mode\n"); - return -EINVAL; - } - - ret = intel_render_ring_init_dri(dev, - init->ring_start, - init->ring_size); - if (ret) { - i915_dma_cleanup(dev); - return ret; - } - } - - dev_priv->dri1.cpp = init->cpp; - dev_priv->dri1.back_offset = init->back_offset; - dev_priv->dri1.front_offset = init->front_offset; - dev_priv->dri1.current_page = 0; - if (master_priv->sarea_priv) - master_priv->sarea_priv->pf_current_page = 0; - - /* Allow hardware batchbuffers unless told otherwise. - */ - dev_priv->dri1.allow_batchbuffer = 1; - - return 0; -} - -static int i915_dma_resume(struct drm_device *dev) -{ - struct drm_i915_private *dev_priv = dev->dev_private; - struct intel_engine_cs *ring = LP_RING(dev_priv); - - DRM_DEBUG_DRIVER("%s\n", __func__); - - if (ring->buffer->virtual_start == NULL) { - DRM_ERROR("can not ioremap virtual address for" - " ring buffer\n"); - return -ENOMEM; - } - - /* Program Hardware Status Page */ - if (!ring->status_page.page_addr) { - DRM_ERROR("Can not find hardware status page\n"); - return -EINVAL; - } - DRM_DEBUG_DRIVER("hw status page @ %p\n", - ring->status_page.page_addr); - if (ring->status_page.gfx_addr != 0) - intel_ring_setup_status_page(ring); - else - i915_write_hws_pga(dev); - - DRM_DEBUG_DRIVER("Enabled hardware status page\n"); - - return 0; -} - static int i915_dma_init(struct drm_device *dev, void *data, struct drm_file *file_priv) { - drm_i915_init_t *init = data; - int retcode = 0; - - if (drm_core_check_feature(dev, DRIVER_MODESET)) - return -ENODEV; - - switch (init->func) { - case I915_INIT_DMA: - retcode = i915_initialize(dev, init); - break; - case I915_CLEANUP_DMA: - retcode = i915_dma_cleanup(dev); - break; - case I915_RESUME_DMA: - retcode = i915_dma_resume(dev); - break; - default: - retcode = -EINVAL; - break; - } - - return retcode; -} - -/* Implement basically the same security restrictions as hardware does - * for MI_BATCH_NON_SECURE. These can be made stricter at any time. - * - * Most of the calculations below involve calculating the size of a - * particular instruction. It's important to get the size right as - * that tells us where the next instruction to check is. Any illegal - * instruction detected will be given a size of zero, which is a - * signal to abort the rest of the buffer. - */ -static int validate_cmd(int cmd) -{ - switch (((cmd >> 29) & 0x7)) { - case 0x0: - switch ((cmd >> 23) & 0x3f) { - case 0x0: - return 1; /* MI_NOOP */ - case 0x4: - return 1; /* MI_FLUSH */ - default: - return 0; /* disallow everything else */ - } - break; - case 0x1: - return 0; /* reserved */ - case 0x2: - return (cmd & 0xff) + 2; /* 2d commands */ - case 0x3: - if (((cmd >> 24) & 0x1f) <= 0x18) - return 1; - - switch ((cmd >> 24) & 0x1f) { - case 0x1c: - return 1; - case 0x1d: - switch ((cmd >> 16) & 0xff) { - case 0x3: - return (cmd & 0x1f) + 2; - case 0x4: - return (cmd & 0xf) + 2; - default: - return (cmd & 0xffff) + 2; - } - case 0x1e: - if (cmd & (1 << 23)) - return (cmd & 0xffff) + 1; - else - return 1; - case 0x1f: - if ((cmd & (1 << 23)) == 0) /* inline vertices */ - return (cmd & 0x1ffff) + 2; - else if (cmd & (1 << 17)) /* indirect random */ - if ((cmd & 0xffff) == 0) - return 0; /* unknown length, too hard */ - else - return (((cmd & 0xffff) + 1) / 2) + 1; - else - return 2; /* indirect sequential */ - default: - return 0; - } - default: - return 0; - } - - return 0; -} - -static int i915_emit_cmds(struct drm_device *dev, int *buffer, int dwords) -{ - struct drm_i915_private *dev_priv = dev->dev_private; - int i, ret; - - if ((dwords+1) * sizeof(int) >= LP_RING(dev_priv)->buffer->size - 8) - return -EINVAL; - - for (i = 0; i < dwords;) { - int sz = validate_cmd(buffer[i]); - - if (sz == 0 || i + sz > dwords) - return -EINVAL; - i += sz; - } - - ret = BEGIN_LP_RING((dwords+1)&~1); - if (ret) - return ret; - - for (i = 0; i < dwords; i++) - OUT_RING(buffer[i]); - if (dwords & 1) - OUT_RING(0); - - ADVANCE_LP_RING(); - - return 0; -} - -int -i915_emit_box(struct drm_device *dev, - struct drm_clip_rect *box, - int DR1, int DR4) -{ - struct drm_i915_private *dev_priv = dev->dev_private; - int ret; - - if (box->y2 <= box->y1 || box->x2 <= box->x1 || - box->y2 <= 0 || box->x2 <= 0) { - DRM_ERROR("Bad box %d,%d..%d,%d\n", - box->x1, box->y1, box->x2, box->y2); - return -EINVAL; - } - - if (INTEL_INFO(dev)->gen >= 4) { - ret = BEGIN_LP_RING(4); - if (ret) - return ret; - - OUT_RING(GFX_OP_DRAWRECT_INFO_I965); - OUT_RING((box->x1 & 0xffff) | (box->y1 << 16)); - OUT_RING(((box->x2 - 1) & 0xffff) | ((box->y2 - 1) << 16)); - OUT_RING(DR4); - } else { - ret = BEGIN_LP_RING(6); - if (ret) - return ret; - - OUT_RING(GFX_OP_DRAWRECT_INFO); - OUT_RING(DR1); - OUT_RING((box->x1 & 0xffff) | (box->y1 << 16)); - OUT_RING(((box->x2 - 1) & 0xffff) | ((box->y2 - 1) << 16)); - OUT_RING(DR4); - OUT_RING(0); - } - ADVANCE_LP_RING(); - - return 0; -} - -/* XXX: Emitting the counter should really be moved to part of the IRQ - * emit. For now, do it in both places: - */ - -static void i915_emit_breadcrumb(struct drm_device *dev) -{ - struct drm_i915_private *dev_priv = dev->dev_private; - struct drm_i915_master_private *master_priv = dev->primary->master->driver_priv; - - dev_priv->dri1.counter++; - if (dev_priv->dri1.counter > 0x7FFFFFFFUL) - dev_priv->dri1.counter = 0; - if (master_priv->sarea_priv) - master_priv->sarea_priv->last_enqueue = dev_priv->dri1.counter; - - if (BEGIN_LP_RING(4) == 0) { - OUT_RING(MI_STORE_DWORD_INDEX); - OUT_RING(I915_BREADCRUMB_INDEX << MI_STORE_DWORD_INDEX_SHIFT); - OUT_RING(dev_priv->dri1.counter); - OUT_RING(0); - ADVANCE_LP_RING(); - } -} - -static int i915_dispatch_cmdbuffer(struct drm_device *dev, - drm_i915_cmdbuffer_t *cmd, - struct drm_clip_rect *cliprects, - void *cmdbuf) -{ - int nbox = cmd->num_cliprects; - int i = 0, count, ret; - - if (cmd->sz & 0x3) { - DRM_ERROR("alignment"); - return -EINVAL; - } - - i915_kernel_lost_context(dev); - - count = nbox ? nbox : 1; - - for (i = 0; i < count; i++) { - if (i < nbox) { - ret = i915_emit_box(dev, &cliprects[i], - cmd->DR1, cmd->DR4); - if (ret) - return ret; - } - - ret = i915_emit_cmds(dev, cmdbuf, cmd->sz / 4); - if (ret) - return ret; - } - - i915_emit_breadcrumb(dev); - return 0; -} - -static int i915_dispatch_batchbuffer(struct drm_device *dev, - drm_i915_batchbuffer_t *batch, - struct drm_clip_rect *cliprects) -{ - struct drm_i915_private *dev_priv = dev->dev_private; - int nbox = batch->num_cliprects; - int i, count, ret; - - if ((batch->start | batch->used) & 0x7) { - DRM_ERROR("alignment"); - return -EINVAL; - } - - i915_kernel_lost_context(dev); - - count = nbox ? nbox : 1; - for (i = 0; i < count; i++) { - if (i < nbox) { - ret = i915_emit_box(dev, &cliprects[i], - batch->DR1, batch->DR4); - if (ret) - return ret; - } - - if (!IS_I830(dev) && !IS_845G(dev)) { - ret = BEGIN_LP_RING(2); - if (ret) - return ret; - - if (INTEL_INFO(dev)->gen >= 4) { - OUT_RING(MI_BATCH_BUFFER_START | (2 << 6) | MI_BATCH_NON_SECURE_I965); - OUT_RING(batch->start); - } else { - OUT_RING(MI_BATCH_BUFFER_START | (2 << 6)); - OUT_RING(batch->start | MI_BATCH_NON_SECURE); - } - } else { - ret = BEGIN_LP_RING(4); - if (ret) - return ret; - - OUT_RING(MI_BATCH_BUFFER); - OUT_RING(batch->start | MI_BATCH_NON_SECURE); - OUT_RING(batch->start + batch->used - 4); - OUT_RING(0); - } - ADVANCE_LP_RING(); - } - - - if (IS_G4X(dev) || IS_GEN5(dev)) { - if (BEGIN_LP_RING(2) == 0) { - OUT_RING(MI_FLUSH | MI_NO_WRITE_FLUSH | MI_INVALIDATE_ISP); - OUT_RING(MI_NOOP); - ADVANCE_LP_RING(); - } - } - - i915_emit_breadcrumb(dev); - return 0; -} - -static int i915_dispatch_flip(struct drm_device *dev) -{ - struct drm_i915_private *dev_priv = dev->dev_private; - struct drm_i915_master_private *master_priv = - dev->primary->master->driver_priv; - int ret; - - if (!master_priv->sarea_priv) - return -EINVAL; - - DRM_DEBUG_DRIVER("%s: page=%d pfCurrentPage=%d\n", - __func__, - dev_priv->dri1.current_page, - master_priv->sarea_priv->pf_current_page); - - i915_kernel_lost_context(dev); - - ret = BEGIN_LP_RING(10); - if (ret) - return ret; - - OUT_RING(MI_FLUSH | MI_READ_FLUSH); - OUT_RING(0); - - OUT_RING(CMD_OP_DISPLAYBUFFER_INFO | ASYNC_FLIP); - OUT_RING(0); - if (dev_priv->dri1.current_page == 0) { - OUT_RING(dev_priv->dri1.back_offset); - dev_priv->dri1.current_page = 1; - } else { - OUT_RING(dev_priv->dri1.front_offset); - dev_priv->dri1.current_page = 0; - } - OUT_RING(0); - - OUT_RING(MI_WAIT_FOR_EVENT | MI_WAIT_FOR_PLANE_A_FLIP); - OUT_RING(0); - - ADVANCE_LP_RING(); - - master_priv->sarea_priv->last_enqueue = dev_priv->dri1.counter++; - - if (BEGIN_LP_RING(4) == 0) { - OUT_RING(MI_STORE_DWORD_INDEX); - OUT_RING(I915_BREADCRUMB_INDEX << MI_STORE_DWORD_INDEX_SHIFT); - OUT_RING(dev_priv->dri1.counter); - OUT_RING(0); - ADVANCE_LP_RING(); - } - - master_priv->sarea_priv->pf_current_page = dev_priv->dri1.current_page; - return 0; -} - -static int i915_quiescent(struct drm_device *dev) -{ - i915_kernel_lost_context(dev); - return intel_ring_idle(LP_RING(dev->dev_private)); + return -ENODEV; } static int i915_flush_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv) { - int ret; - - if (drm_core_check_feature(dev, DRIVER_MODESET)) - return -ENODEV; - - RING_LOCK_TEST_WITH_RETURN(dev, file_priv); - - mutex_lock(&dev->struct_mutex); - ret = i915_quiescent(dev); - mutex_unlock(&dev->struct_mutex); - - return ret; + return -ENODEV; } static int i915_batchbuffer(struct drm_device *dev, void *data, struct drm_file *file_priv) { - struct drm_i915_private *dev_priv = dev->dev_private; - struct drm_i915_master_private *master_priv; - drm_i915_sarea_t *sarea_priv; - drm_i915_batchbuffer_t *batch = data; - int ret; - struct drm_clip_rect *cliprects = NULL; - - if (drm_core_check_feature(dev, DRIVER_MODESET)) - return -ENODEV; - - master_priv = dev->primary->master->driver_priv; - sarea_priv = (drm_i915_sarea_t *) master_priv->sarea_priv; - - if (!dev_priv->dri1.allow_batchbuffer) { - DRM_ERROR("Batchbuffer ioctl disabled\n"); - return -EINVAL; - } - - DRM_DEBUG_DRIVER("i915 batchbuffer, start %x used %d cliprects %d\n", - batch->start, batch->used, batch->num_cliprects); - - RING_LOCK_TEST_WITH_RETURN(dev, file_priv); - - if (batch->num_cliprects < 0) - return -EINVAL; - - if (batch->num_cliprects) { - cliprects = kcalloc(batch->num_cliprects, - sizeof(*cliprects), - GFP_KERNEL); - if (cliprects == NULL) - return -ENOMEM; - - ret = copy_from_user(cliprects, batch->cliprects, - batch->num_cliprects * - sizeof(struct drm_clip_rect)); - if (ret != 0) { - ret = -EFAULT; - goto fail_free; - } - } - - mutex_lock(&dev->struct_mutex); - ret = i915_dispatch_batchbuffer(dev, batch, cliprects); - mutex_unlock(&dev->struct_mutex); - - if (sarea_priv) - sarea_priv->last_dispatch = READ_BREADCRUMB(dev_priv); - -fail_free: - kfree(cliprects); - - return ret; + return -ENODEV; } static int i915_cmdbuffer(struct drm_device *dev, void *data, struct drm_file *file_priv) { - struct drm_i915_private *dev_priv = dev->dev_private; - struct drm_i915_master_private *master_priv; - drm_i915_sarea_t *sarea_priv; - drm_i915_cmdbuffer_t *cmdbuf = data; - struct drm_clip_rect *cliprects = NULL; - void *batch_data; - int ret; - - DRM_DEBUG_DRIVER("i915 cmdbuffer, buf %p sz %d cliprects %d\n", - cmdbuf->buf, cmdbuf->sz, cmdbuf->num_cliprects); - - if (drm_core_check_feature(dev, DRIVER_MODESET)) - return -ENODEV; - - master_priv = dev->primary->master->driver_priv; - sarea_priv = (drm_i915_sarea_t *) master_priv->sarea_priv; - - RING_LOCK_TEST_WITH_RETURN(dev, file_priv); - - if (cmdbuf->num_cliprects < 0) - return -EINVAL; - - batch_data = kmalloc(cmdbuf->sz, GFP_KERNEL); - if (batch_data == NULL) - return -ENOMEM; - - ret = copy_from_user(batch_data, cmdbuf->buf, cmdbuf->sz); - if (ret != 0) { - ret = -EFAULT; - goto fail_batch_free; - } - - if (cmdbuf->num_cliprects) { - cliprects = kcalloc(cmdbuf->num_cliprects, - sizeof(*cliprects), GFP_KERNEL); - if (cliprects == NULL) { - ret = -ENOMEM; - goto fail_batch_free; - } - - ret = copy_from_user(cliprects, cmdbuf->cliprects, - cmdbuf->num_cliprects * - sizeof(struct drm_clip_rect)); - if (ret != 0) { - ret = -EFAULT; - goto fail_clip_free; - } - } - - mutex_lock(&dev->struct_mutex); - ret = i915_dispatch_cmdbuffer(dev, cmdbuf, cliprects, batch_data); - mutex_unlock(&dev->struct_mutex); - if (ret) { - DRM_ERROR("i915_dispatch_cmdbuffer failed\n"); - goto fail_clip_free; - } - - if (sarea_priv) - sarea_priv->last_dispatch = READ_BREADCRUMB(dev_priv); - -fail_clip_free: - kfree(cliprects); -fail_batch_free: - kfree(batch_data); - - return ret; -} - -static int i915_emit_irq(struct drm_device *dev) -{ - struct drm_i915_private *dev_priv = dev->dev_private; - struct drm_i915_master_private *master_priv = dev->primary->master->driver_priv; - - i915_kernel_lost_context(dev); - - DRM_DEBUG_DRIVER("\n"); - - dev_priv->dri1.counter++; - if (dev_priv->dri1.counter > 0x7FFFFFFFUL) - dev_priv->dri1.counter = 1; - if (master_priv->sarea_priv) - master_priv->sarea_priv->last_enqueue = dev_priv->dri1.counter; - - if (BEGIN_LP_RING(4) == 0) { - OUT_RING(MI_STORE_DWORD_INDEX); - OUT_RING(I915_BREADCRUMB_INDEX << MI_STORE_DWORD_INDEX_SHIFT); - OUT_RING(dev_priv->dri1.counter); - OUT_RING(MI_USER_INTERRUPT); - ADVANCE_LP_RING(); - } - - return dev_priv->dri1.counter; -} - -static int i915_wait_irq(struct drm_device *dev, int irq_nr) -{ - struct drm_i915_private *dev_priv = dev->dev_private; - struct drm_i915_master_private *master_priv = dev->primary->master->driver_priv; - int ret = 0; - struct intel_engine_cs *ring = LP_RING(dev_priv); - - DRM_DEBUG_DRIVER("irq_nr=%d breadcrumb=%d\n", irq_nr, - READ_BREADCRUMB(dev_priv)); - - if (READ_BREADCRUMB(dev_priv) >= irq_nr) { - if (master_priv->sarea_priv) - master_priv->sarea_priv->last_dispatch = READ_BREADCRUMB(dev_priv); - return 0; - } - - if (master_priv->sarea_priv) - master_priv->sarea_priv->perf_boxes |= I915_BOX_WAIT; - - if (ring->irq_get(ring)) { - DRM_WAIT_ON(ret, ring->irq_queue, 3 * HZ, - READ_BREADCRUMB(dev_priv) >= irq_nr); - ring->irq_put(ring); - } else if (wait_for(READ_BREADCRUMB(dev_priv) >= irq_nr, 3000)) - ret = -EBUSY; - - if (ret == -EBUSY) { - DRM_ERROR("EBUSY -- rec: %d emitted: %d\n", - READ_BREADCRUMB(dev_priv), (int)dev_priv->dri1.counter); - } - - return ret; + return -ENODEV; } -/* Needs the lock as it touches the ring. - */ static int i915_irq_emit(struct drm_device *dev, void *data, struct drm_file *file_priv) { - struct drm_i915_private *dev_priv = dev->dev_private; - drm_i915_irq_emit_t *emit = data; - int result; - - if (drm_core_check_feature(dev, DRIVER_MODESET)) - return -ENODEV; - - if (!dev_priv || !LP_RING(dev_priv)->buffer->virtual_start) { - DRM_ERROR("called with no initialization\n"); - return -EINVAL; - } - - RING_LOCK_TEST_WITH_RETURN(dev, file_priv); - - mutex_lock(&dev->struct_mutex); - result = i915_emit_irq(dev); - mutex_unlock(&dev->struct_mutex); - - if (copy_to_user(emit->irq_seq, &result, sizeof(int))) { - DRM_ERROR("copy_to_user\n"); - return -EFAULT; - } - - return 0; + return -ENODEV; } -/* Doesn't need the hardware lock. - */ static int i915_irq_wait(struct drm_device *dev, void *data, struct drm_file *file_priv) { - struct drm_i915_private *dev_priv = dev->dev_private; - drm_i915_irq_wait_t *irqwait = data; - - if (drm_core_check_feature(dev, DRIVER_MODESET)) - return -ENODEV; - - if (!dev_priv) { - DRM_ERROR("called with no initialization\n"); - return -EINVAL; - } - - return i915_wait_irq(dev, irqwait->irq_seq); + return -ENODEV; } static int i915_vblank_pipe_get(struct drm_device *dev, void *data, struct drm_file *file_priv) { - struct drm_i915_private *dev_priv = dev->dev_private; - drm_i915_vblank_pipe_t *pipe = data; - - if (drm_core_check_feature(dev, DRIVER_MODESET)) - return -ENODEV; - - if (!dev_priv) { - DRM_ERROR("called with no initialization\n"); - return -EINVAL; - } - - pipe->pipe = DRM_I915_VBLANK_PIPE_A | DRM_I915_VBLANK_PIPE_B; - - return 0; + return -ENODEV; } -/** - * Schedule buffer swap at given vertical blank. - */ static int i915_vblank_swap(struct drm_device *dev, void *data, struct drm_file *file_priv) { - /* The delayed swap mechanism was fundamentally racy, and has been - * removed. The model was that the client requested a delayed flip/swap - * from the kernel, then waited for vblank before continuing to perform - * rendering. The problem was that the kernel might wake the client - * up before it dispatched the vblank swap (since the lock has to be - * held while touching the ringbuffer), in which case the client would - * clear and start the next frame before the swap occurred, and - * flicker would occur in addition to likely missing the vblank. - * - * In the absence of this ioctl, userland falls back to a correct path - * of waiting for a vblank, then dispatching the swap on its own. - * Context switching to userland and back is plenty fast enough for - * meeting the requirements of vblank swapping. - */ - return -EINVAL; + return -ENODEV; } static int i915_flip_bufs(struct drm_device *dev, void *data, struct drm_file *file_priv) { - int ret; - - if (drm_core_check_feature(dev, DRIVER_MODESET)) - return -ENODEV; - - DRM_DEBUG_DRIVER("%s\n", __func__); - - RING_LOCK_TEST_WITH_RETURN(dev, file_priv); - - mutex_lock(&dev->struct_mutex); - ret = i915_dispatch_flip(dev); - mutex_unlock(&dev->struct_mutex); - - return ret; + return -ENODEV; } static int i915_getparam(struct drm_device *dev, void *data, @@ -941,14 +117,11 @@ static int i915_getparam(struct drm_device *dev, void *data, switch (param->param) { case I915_PARAM_IRQ_ACTIVE: - value = dev->pdev->irq ? 1 : 0; - break; + return -ENODEV; case I915_PARAM_ALLOW_BATCHBUFFER: - value = dev_priv->dri1.allow_batchbuffer ? 1 : 0; - break; + return -ENODEV; case I915_PARAM_LAST_DISPATCH: - value = READ_BREADCRUMB(dev_priv); - break; + return -ENODEV; case I915_PARAM_CHIPSET_ID: value = dev->pdev->device; break; @@ -969,13 +142,13 @@ static int i915_getparam(struct drm_device *dev, void *data, value = 1; break; case I915_PARAM_HAS_BSD: - value = intel_ring_initialized(&dev_priv->ring[VCS]); + value = intel_engine_initialized(&dev_priv->engine[VCS]); break; case I915_PARAM_HAS_BLT: - value = intel_ring_initialized(&dev_priv->ring[BCS]); + value = intel_engine_initialized(&dev_priv->engine[BCS]); break; case I915_PARAM_HAS_VEBOX: - value = intel_ring_initialized(&dev_priv->ring[VECS]); + value = intel_engine_initialized(&dev_priv->engine[VECS]); break; case I915_PARAM_HAS_RELAXED_FENCING: value = 1; @@ -999,13 +172,13 @@ static int i915_getparam(struct drm_device *dev, void *data, value = HAS_WT(dev); break; case I915_PARAM_HAS_ALIASING_PPGTT: - value = dev_priv->mm.aliasing_ppgtt || USES_FULL_PPGTT(dev); + value = USES_PPGTT(dev); break; case I915_PARAM_HAS_WAIT_TIMEOUT: value = 1; break; case I915_PARAM_HAS_SEMAPHORES: - value = i915_semaphore_is_enabled(dev); + value = RCS_ENGINE(dev_priv)->semaphore.wait != NULL; break; case I915_PARAM_HAS_PRIME_VMAP_FLUSH: value = 1; @@ -1051,12 +224,10 @@ static int i915_setparam(struct drm_device *dev, void *data, switch (param->param) { case I915_SETPARAM_USE_MI_BATCHBUFFER_START: - break; case I915_SETPARAM_TEX_LRU_LOG_GRANULARITY: - break; case I915_SETPARAM_ALLOW_BATCHBUFFER: - dev_priv->dri1.allow_batchbuffer = param->value ? 1 : 0; - break; + return -ENODEV; + case I915_SETPARAM_NUM_USED_FENCES: if (param->value > dev_priv->num_fence_regs || param->value < 0) @@ -1076,49 +247,7 @@ static int i915_setparam(struct drm_device *dev, void *data, static int i915_set_status_page(struct drm_device *dev, void *data, struct drm_file *file_priv) { - struct drm_i915_private *dev_priv = dev->dev_private; - drm_i915_hws_addr_t *hws = data; - struct intel_engine_cs *ring; - - if (drm_core_check_feature(dev, DRIVER_MODESET)) - return -ENODEV; - - if (!I915_NEED_GFX_HWS(dev)) - return -EINVAL; - - if (!dev_priv) { - DRM_ERROR("called with no initialization\n"); - return -EINVAL; - } - - if (drm_core_check_feature(dev, DRIVER_MODESET)) { - WARN(1, "tried to set status page when mode setting active\n"); - return 0; - } - - DRM_DEBUG_DRIVER("set status page addr 0x%08x\n", (u32)hws->addr); - - ring = LP_RING(dev_priv); - ring->status_page.gfx_addr = hws->addr & (0x1ffff<<12); - - dev_priv->dri1.gfx_hws_cpu_addr = - ioremap_wc(dev_priv->gtt.mappable_base + hws->addr, 4096); - if (dev_priv->dri1.gfx_hws_cpu_addr == NULL) { - i915_dma_cleanup(dev); - ring->status_page.gfx_addr = 0; - DRM_ERROR("can not ioremap virtual address for" - " G33 hw status page\n"); - return -ENOMEM; - } - - memset_io(dev_priv->dri1.gfx_hws_cpu_addr, 0, PAGE_SIZE); - I915_WRITE(HWS_PGA, ring->status_page.gfx_addr); - - DRM_DEBUG_DRIVER("load hws HWS_PGA with gfx mem 0x%x\n", - ring->status_page.gfx_addr); - DRM_DEBUG_DRIVER("load hws at %p\n", - ring->status_page.page_addr); - return 0; + return -ENODEV; } static int i915_get_bridge_dev(struct drm_device *dev) @@ -1355,8 +484,6 @@ static int i915_load_modeset_init(struct drm_device *dev) if (ret) goto cleanup_irq; - INIT_WORK(&dev_priv->console_resume_work, intel_console_resume); - intel_modeset_gem_init(dev); /* Always safe in the mode setting case. */ @@ -1382,7 +509,7 @@ static int i915_load_modeset_init(struct drm_device *dev) * scanning against hotplug events. Hence do this first and ignore the * tiny window where we will loose hotplug notifactions. */ - intel_fbdev_initial_config(dev); + async_schedule(intel_fbdev_initial_config, dev_priv); drm_kms_helper_poll_init(dev); @@ -1390,10 +517,8 @@ static int i915_load_modeset_init(struct drm_device *dev) cleanup_gem: mutex_lock(&dev->struct_mutex); - i915_gem_cleanup_ringbuffer(dev); - i915_gem_context_fini(dev); + i915_gem_fini(dev); mutex_unlock(&dev->struct_mutex); - WARN_ON(dev_priv->mm.aliasing_ppgtt); cleanup_irq: drm_irq_uninstall(dev); cleanup_gem_stolen: @@ -1406,30 +531,6 @@ out: return ret; } -int i915_master_create(struct drm_device *dev, struct drm_master *master) -{ - struct drm_i915_master_private *master_priv; - - master_priv = kzalloc(sizeof(*master_priv), GFP_KERNEL); - if (!master_priv) - return -ENOMEM; - - master->driver_priv = master_priv; - return 0; -} - -void i915_master_destroy(struct drm_device *dev, struct drm_master *master) -{ - struct drm_i915_master_private *master_priv = master->driver_priv; - - if (!master_priv) - return; - - kfree(master_priv); - - master->driver_priv = NULL; -} - #if IS_ENABLED(CONFIG_FB) static int i915_kick_out_firmware_fb(struct drm_i915_private *dev_priv) { @@ -1536,10 +637,10 @@ static void intel_device_info_runtime_init(struct drm_device *dev) info = (struct intel_device_info *)&dev_priv->info; if (IS_VALLEYVIEW(dev)) - for_each_pipe(pipe) + for_each_pipe(dev_priv, pipe) info->num_sprites[pipe] = 2; else - for_each_pipe(pipe) + for_each_pipe(dev_priv, pipe) info->num_sprites[pipe] = 1; if (i915.disable_display) { @@ -1601,6 +702,8 @@ int i915_driver_load(struct drm_device *dev, unsigned long flags) if (!drm_core_check_feature(dev, DRIVER_MODESET) && !dev->agp) return -EINVAL; + BUILD_BUG_ON(I915_NUM_ENGINES >= (1 << I915_NUM_ENGINE_BITS)); + dev_priv = kzalloc(sizeof(*dev_priv), GFP_KERNEL); if (dev_priv == NULL) return -ENOMEM; @@ -1608,13 +711,14 @@ int i915_driver_load(struct drm_device *dev, unsigned long flags) dev->dev_private = dev_priv; dev_priv->dev = dev; - /* copy initial configuration to dev_priv->info */ + /* Setup the write-once "constant" device info */ device_info = (struct intel_device_info *)&dev_priv->info; - *device_info = *info; + memcpy(device_info, info, sizeof(dev_priv->info)); + device_info->device_id = dev->pdev->device; spin_lock_init(&dev_priv->irq_lock); spin_lock_init(&dev_priv->gpu_error.lock); - spin_lock_init(&dev_priv->backlight_lock); + mutex_init(&dev_priv->backlight_lock); spin_lock_init(&dev_priv->uncore.lock); spin_lock_init(&dev_priv->mm.object_stat_lock); spin_lock_init(&dev_priv->mmio_flip_lock); @@ -1822,7 +926,7 @@ out_mtrrfree: arch_phys_wc_del(dev_priv->gtt.mtrr); io_mapping_free(dev_priv->gtt.mappable); out_gtt: - dev_priv->gtt.base.cleanup(&dev_priv->gtt.base); + i915_global_gtt_cleanup(dev); out_regs: intel_uncore_fini(dev); pci_iounmap(dev->pdev, dev_priv->regs); @@ -1869,7 +973,6 @@ int i915_driver_unload(struct drm_device *dev) if (drm_core_check_feature(dev, DRIVER_MODESET)) { intel_fbdev_fini(dev); intel_modeset_cleanup(dev); - cancel_work_sync(&dev_priv->console_resume_work); /* * free the memory space allocated for the child device @@ -1900,18 +1003,11 @@ int i915_driver_unload(struct drm_device *dev) flush_workqueue(dev_priv->wq); mutex_lock(&dev->struct_mutex); - i915_gem_cleanup_ringbuffer(dev); - i915_gem_context_fini(dev); - WARN_ON(dev_priv->mm.aliasing_ppgtt); + i915_gem_fini(dev); mutex_unlock(&dev->struct_mutex); i915_gem_cleanup_stolen(dev); - - if (!I915_NEED_GFX_HWS(dev)) - i915_free_hws(dev); } - WARN_ON(!list_empty(&dev_priv->vm_list)); - drm_vblank_cleanup(dev); intel_teardown_gmbus(dev); @@ -1921,7 +1017,7 @@ int i915_driver_unload(struct drm_device *dev) destroy_workqueue(dev_priv->wq); pm_qos_remove_request(&dev_priv->pm_qos); - dev_priv->gtt.base.cleanup(&dev_priv->gtt.base); + i915_global_gtt_cleanup(dev); intel_uncore_fini(dev); if (dev_priv->regs != NULL) @@ -1976,8 +1072,6 @@ void i915_driver_lastclose(struct drm_device *dev) } i915_gem_lastclose(dev); - - i915_dma_cleanup(dev); } void i915_driver_preclose(struct drm_device *dev, struct drm_file *file) @@ -1986,14 +1080,15 @@ void i915_driver_preclose(struct drm_device *dev, struct drm_file *file) i915_gem_context_close(dev, file); i915_gem_release(dev, file); mutex_unlock(&dev->struct_mutex); + + if (drm_core_check_feature(dev, DRIVER_MODESET)) + intel_modeset_preclose(dev, file); } void i915_driver_postclose(struct drm_device *dev, struct drm_file *file) { struct drm_i915_file_private *file_priv = file->driver_priv; - if (file_priv && file_priv->bsd_ring) - file_priv->bsd_ring = NULL; kfree(file_priv); } diff --git a/drivers/gpu/drm/i915/i915_drv.c b/drivers/gpu/drm/i915/i915_drv.c index e27cdbe..67084ee 100644 --- a/drivers/gpu/drm/i915/i915_drv.c +++ b/drivers/gpu/drm/i915/i915_drv.c @@ -473,26 +473,6 @@ void intel_detect_pch(struct drm_device *dev) pci_dev_put(pch); } -bool i915_semaphore_is_enabled(struct drm_device *dev) -{ - if (INTEL_INFO(dev)->gen < 6) - return false; - - if (i915.semaphores >= 0) - return i915.semaphores; - - /* Until we get further testing... */ - if (IS_GEN8(dev)) - return false; - -#ifdef CONFIG_INTEL_IOMMU - /* Enable semaphores on SNB when IO remapping is off */ - if (INTEL_INFO(dev)->gen == 6 && intel_iommu_gfx_mapped) - return false; -#endif - - return true; -} void intel_hpd_cancel_work(struct drm_i915_private *dev_priv) { @@ -505,7 +485,7 @@ void intel_hpd_cancel_work(struct drm_i915_private *dev_priv) spin_unlock_irq(&dev_priv->irq_lock); cancel_work_sync(&dev_priv->dig_port_work); - cancel_work_sync(&dev_priv->hotplug_work); + cancel_delayed_work_sync(&dev_priv->hotplug_work); cancel_delayed_work_sync(&dev_priv->hotplug_reenable_work); } @@ -524,6 +504,10 @@ static void intel_suspend_encoders(struct drm_i915_private *dev_priv) drm_modeset_unlock_all(dev); } +static int intel_suspend_complete(struct drm_i915_private *dev_priv); +static int intel_resume_prepare(struct drm_i915_private *dev_priv, + bool rpm_resume); + static int i915_drm_freeze(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; @@ -591,9 +575,7 @@ static int i915_drm_freeze(struct drm_device *dev) intel_uncore_forcewake_reset(dev, false); intel_opregion_fini(dev); - console_lock(); - intel_fbdev_set_suspend(dev, FBINFO_STATE_SUSPENDED); - console_unlock(); + intel_fbdev_set_suspend(dev, FBINFO_STATE_SUSPENDED, true); dev_priv->suspend_count++; @@ -632,30 +614,20 @@ int i915_suspend(struct drm_device *dev, pm_message_t state) return 0; } -void intel_console_resume(struct work_struct *work) -{ - struct drm_i915_private *dev_priv = - container_of(work, struct drm_i915_private, - console_resume_work); - struct drm_device *dev = dev_priv->dev; - - console_lock(); - intel_fbdev_set_suspend(dev, FBINFO_STATE_RUNNING); - console_unlock(); -} - static int i915_drm_thaw_early(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; + int ret; - if (IS_HASWELL(dev) || IS_BROADWELL(dev)) - hsw_disable_pc8(dev_priv); + ret = intel_resume_prepare(dev_priv, false); + if (ret) + DRM_ERROR("Resume prepare failed: %d,Continuing resume\n", ret); intel_uncore_early_sanitize(dev, true); intel_uncore_sanitize(dev); intel_power_domains_init_hw(dev_priv); - return 0; + return ret; } static int __i915_drm_thaw(struct drm_device *dev, bool restore_gtt_mappings) @@ -684,16 +656,16 @@ static int __i915_drm_thaw(struct drm_device *dev, bool restore_gtt_mappings) } mutex_unlock(&dev->struct_mutex); + /* We need working interrupts for modeset enabling ... */ intel_runtime_pm_restore_interrupts(dev); intel_modeset_init_hw(dev); { - unsigned long irqflags; - spin_lock_irqsave(&dev_priv->irq_lock, irqflags); + spin_lock_irq(&dev_priv->irq_lock); if (dev_priv->display.hpd_irq_setup) dev_priv->display.hpd_irq_setup(dev); - spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags); + spin_unlock_irq(&dev_priv->irq_lock); } intel_dp_mst_resume(dev); @@ -714,17 +686,7 @@ static int __i915_drm_thaw(struct drm_device *dev, bool restore_gtt_mappings) intel_opregion_init(dev); - /* - * The console lock can be pretty contented on resume due - * to all the printk activity. Try to keep it out of the hot - * path of resume if possible. - */ - if (console_trylock()) { - intel_fbdev_set_suspend(dev, FBINFO_STATE_RUNNING); - console_unlock(); - } else { - schedule_work(&dev_priv->console_resume_work); - } + intel_fbdev_set_suspend(dev, FBINFO_STATE_RUNNING, false); mutex_lock(&dev_priv->modeset_restore_lock); dev_priv->modeset_restore = MODESET_DONE; @@ -809,7 +771,6 @@ static int i915_resume_legacy(struct drm_device *dev) int i915_reset(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; - bool simulated; int ret; if (!i915.reset) @@ -817,14 +778,16 @@ int i915_reset(struct drm_device *dev) mutex_lock(&dev->struct_mutex); - i915_gem_reset(dev); - - simulated = dev_priv->gpu_error.stop_rings != 0; - ret = intel_gpu_reset(dev); + /* Clear the reset counter. Before anyone else + * can grab the mutex, we will declare whether or + * not the GPU is wedged. + */ + atomic_inc(&dev_priv->gpu_error.reset_counter); + /* Also reset the gpu hangman. */ - if (simulated) { + if (dev_priv->gpu_error.stop_rings) { DRM_INFO("Simulated gpu hang, resetting stop_rings\n"); dev_priv->gpu_error.stop_rings = 0; if (ret == -ENODEV) { @@ -834,6 +797,8 @@ int i915_reset(struct drm_device *dev) } } + i915_gem_reset(dev); + if (ret) { DRM_ERROR("Failed to reset chip: %i\n", ret); mutex_unlock(&dev->struct_mutex); @@ -857,7 +822,6 @@ int i915_reset(struct drm_device *dev) if (drm_core_check_feature(dev, DRIVER_MODESET) || !dev_priv->ums.mm_suspended) { dev_priv->ums.mm_suspended = 0; - ret = i915_gem_init_hw(dev); mutex_unlock(&dev->struct_mutex); if (ret) { @@ -879,8 +843,6 @@ int i915_reset(struct drm_device *dev) */ if (INTEL_INFO(dev)->gen > 5) intel_reset_gt_powersave(dev); - - intel_hpd_init(dev); } else { mutex_unlock(&dev->struct_mutex); } @@ -941,6 +903,7 @@ static int i915_pm_suspend_late(struct device *dev) struct pci_dev *pdev = to_pci_dev(dev); struct drm_device *drm_dev = pci_get_drvdata(pdev); struct drm_i915_private *dev_priv = drm_dev->dev_private; + int ret; /* * We have a suspedn ordering issue with the snd-hda driver also @@ -954,13 +917,16 @@ static int i915_pm_suspend_late(struct device *dev) if (drm_dev->switch_power_state == DRM_SWITCH_POWER_OFF) return 0; - if (IS_HASWELL(drm_dev) || IS_BROADWELL(drm_dev)) - hsw_enable_pc8(dev_priv); + ret = intel_suspend_complete(dev_priv); - pci_disable_device(pdev); - pci_set_power_state(pdev, PCI_D3hot); + if (ret) + DRM_ERROR("Suspend complete failed: %d\n", ret); + else { + pci_disable_device(pdev); + pci_set_power_state(pdev, PCI_D3hot); + } - return 0; + return ret; } static int i915_pm_resume_early(struct device *dev) @@ -1016,23 +982,26 @@ static int i915_pm_poweroff(struct device *dev) return i915_drm_freeze(drm_dev); } -static int hsw_runtime_suspend(struct drm_i915_private *dev_priv) +static int hsw_suspend_complete(struct drm_i915_private *dev_priv) { hsw_enable_pc8(dev_priv); return 0; } -static int snb_runtime_resume(struct drm_i915_private *dev_priv) +static int snb_resume_prepare(struct drm_i915_private *dev_priv, + bool rpm_resume) { struct drm_device *dev = dev_priv->dev; - intel_init_pch_refclk(dev); + if (rpm_resume) + intel_init_pch_refclk(dev); return 0; } -static int hsw_runtime_resume(struct drm_i915_private *dev_priv) +static int hsw_resume_prepare(struct drm_i915_private *dev_priv, + bool rpm_resume) { hsw_disable_pc8(dev_priv); @@ -1328,7 +1297,7 @@ static void vlv_check_no_gt_access(struct drm_i915_private *dev_priv) I915_WRITE(VLV_GTLC_PW_STATUS, VLV_GTLC_ALLOWWAKEERR); } -static int vlv_runtime_suspend(struct drm_i915_private *dev_priv) +static int vlv_suspend_complete(struct drm_i915_private *dev_priv) { u32 mask; int err; @@ -1368,7 +1337,8 @@ err1: return err; } -static int vlv_runtime_resume(struct drm_i915_private *dev_priv) +static int vlv_resume_prepare(struct drm_i915_private *dev_priv, + bool rpm_resume) { struct drm_device *dev = dev_priv->dev; int err; @@ -1393,8 +1363,10 @@ static int vlv_runtime_resume(struct drm_i915_private *dev_priv) vlv_check_no_gt_access(dev_priv); - intel_init_clock_gating(dev); - i915_gem_restore_fences(dev); + if (rpm_resume) { + intel_init_clock_gating(dev); + i915_gem_restore_fences(dev); + } return ret; } @@ -1409,7 +1381,9 @@ static int intel_runtime_suspend(struct device *device) if (WARN_ON_ONCE(!(dev_priv->rps.enabled && intel_enable_rc6(dev)))) return -ENODEV; - WARN_ON(!HAS_RUNTIME_PM(dev)); + if (WARN_ON_ONCE(!HAS_RUNTIME_PM(dev))) + return -ENODEV; + assert_force_wake_inactive(dev_priv); DRM_DEBUG_KMS("Suspending device\n"); @@ -1446,17 +1420,7 @@ static int intel_runtime_suspend(struct device *device) cancel_work_sync(&dev_priv->rps.work); intel_runtime_pm_disable_interrupts(dev); - if (IS_GEN6(dev)) { - ret = 0; - } else if (IS_HASWELL(dev) || IS_BROADWELL(dev)) { - ret = hsw_runtime_suspend(dev_priv); - } else if (IS_VALLEYVIEW(dev)) { - ret = vlv_runtime_suspend(dev_priv); - } else { - ret = -ENODEV; - WARN_ON(1); - } - + ret = intel_suspend_complete(dev_priv); if (ret) { DRM_ERROR("Runtime suspend failed, disabling it (%d)\n", ret); intel_runtime_pm_restore_interrupts(dev); @@ -1468,13 +1432,29 @@ static int intel_runtime_suspend(struct device *device) dev_priv->pm.suspended = true; /* - * current versions of firmware which depend on this opregion - * notification have repurposed the D1 definition to mean - * "runtime suspended" vs. what you would normally expect (D3) - * to distinguish it from notifications that might be sent - * via the suspend path. + * FIXME: We really should find a document that references the arguments + * used below! */ - intel_opregion_notify_adapter(dev, PCI_D1); + if (IS_HASWELL(dev)) { + /* + * current versions of firmware which depend on this opregion + * notification have repurposed the D1 definition to mean + * "runtime suspended" vs. what you would normally expect (D3) + * to distinguish it from notifications that might be sent via + * the suspend path. + */ + intel_opregion_notify_adapter(dev, PCI_D1); + } else { + /* + * On Broadwell, if we use PCI_D1 the PCH DDI ports will stop + * being detected, and the call we do at intel_runtime_resume() + * won't be able to restore them. Since PCI_D3hot matches the + * actual specification and appears to be working, use it. Let's + * assume the other non-Haswell platforms will stay the same as + * Broadwell. + */ + intel_opregion_notify_adapter(dev, PCI_D3hot); + } DRM_DEBUG_KMS("Device suspended\n"); return 0; @@ -1487,24 +1467,15 @@ static int intel_runtime_resume(struct device *device) struct drm_i915_private *dev_priv = dev->dev_private; int ret; - WARN_ON(!HAS_RUNTIME_PM(dev)); + if (WARN_ON_ONCE(!HAS_RUNTIME_PM(dev))) + return -ENODEV; DRM_DEBUG_KMS("Resuming device\n"); intel_opregion_notify_adapter(dev, PCI_D0); dev_priv->pm.suspended = false; - if (IS_GEN6(dev)) { - ret = snb_runtime_resume(dev_priv); - } else if (IS_HASWELL(dev) || IS_BROADWELL(dev)) { - ret = hsw_runtime_resume(dev_priv); - } else if (IS_VALLEYVIEW(dev)) { - ret = vlv_runtime_resume(dev_priv); - } else { - WARN_ON(1); - ret = -ENODEV; - } - + ret = intel_resume_prepare(dev_priv, true); /* * No point of rolling back things in case of an error, as the best * we can do is to hope that things will still work (and disable RPM). @@ -1523,6 +1494,48 @@ static int intel_runtime_resume(struct device *device) return ret; } +/* + * This function implements common functionality of runtime and system + * suspend sequence. + */ +static int intel_suspend_complete(struct drm_i915_private *dev_priv) +{ + struct drm_device *dev = dev_priv->dev; + int ret; + + if (IS_HASWELL(dev) || IS_BROADWELL(dev)) + ret = hsw_suspend_complete(dev_priv); + else if (IS_VALLEYVIEW(dev)) + ret = vlv_suspend_complete(dev_priv); + else + ret = 0; + + return ret; +} + +/* + * This function implements common functionality of runtime and system + * resume sequence. Variable rpm_resume used for implementing different + * code paths. + */ +static int intel_resume_prepare(struct drm_i915_private *dev_priv, + bool rpm_resume) +{ + struct drm_device *dev = dev_priv->dev; + int ret; + + if (IS_GEN6(dev)) + ret = snb_resume_prepare(dev_priv, rpm_resume); + else if (IS_HASWELL(dev) || IS_BROADWELL(dev)) + ret = hsw_resume_prepare(dev_priv, rpm_resume); + else if (IS_VALLEYVIEW(dev)) + ret = vlv_resume_prepare(dev_priv, rpm_resume); + else + ret = 0; + + return ret; +} + static const struct dev_pm_ops i915_pm_ops = { .suspend = i915_pm_suspend, .suspend_late = i915_pm_suspend_late, @@ -1578,8 +1591,6 @@ static struct drm_driver driver = { .resume = i915_resume_legacy, .device_is_agp = i915_driver_device_is_agp, - .master_create = i915_master_create, - .master_destroy = i915_master_destroy, #if defined(CONFIG_DEBUG_FS) .debugfs_init = i915_debugfs_init, .debugfs_cleanup = i915_debugfs_cleanup, @@ -1663,6 +1674,8 @@ static void __exit i915_exit(void) module_init(i915_init); module_exit(i915_exit); -MODULE_AUTHOR(DRIVER_AUTHOR); +MODULE_AUTHOR("Tungsten Graphics, Inc."); +MODULE_AUTHOR("Intel Corporation"); + MODULE_DESCRIPTION(DRIVER_DESC); MODULE_LICENSE("GPL and additional rights"); diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h index 3524306..57f03b0 100644 --- a/drivers/gpu/drm/i915/i915_drv.h +++ b/drivers/gpu/drm/i915/i915_drv.h @@ -49,11 +49,9 @@ /* General customization: */ -#define DRIVER_AUTHOR "Tungsten Graphics, Inc." - #define DRIVER_NAME "i915" #define DRIVER_DESC "Intel Graphics" -#define DRIVER_DATE "20140725" +#define DRIVER_DATE "20140905" enum pipe { INVALID_PIPE = -1, @@ -162,7 +160,10 @@ enum hpd_pin { I915_GEM_DOMAIN_INSTRUCTION | \ I915_GEM_DOMAIN_VERTEX) -#define for_each_pipe(p) for ((p) = 0; (p) < INTEL_INFO(dev)->num_pipes; (p)++) +#define for_each_pipe(__dev_priv, __p) \ + for ((__p) = 0; (__p) < INTEL_INFO(__dev_priv)->num_pipes; (__p)++) +#define for_each_plane(pipe, p) \ + for ((p) = 0; (p) < INTEL_INFO(dev)->num_sprites[(pipe)] + 1; (p)++) #define for_each_sprite(p, s) for ((s) = 0; (s) < INTEL_INFO(dev)->num_sprites[(p)]; (s)++) #define for_each_crtc(dev, crtc) \ @@ -171,6 +172,11 @@ enum hpd_pin { #define for_each_intel_crtc(dev, intel_crtc) \ list_for_each_entry(intel_crtc, &dev->mode_config.crtc_list, base.head) +#define for_each_intel_encoder(dev, intel_encoder) \ + list_for_each_entry(intel_encoder, \ + &(dev)->mode_config.encoder_list, \ + base.head) + #define for_each_encoder_on_crtc(dev, __crtc, intel_encoder) \ list_for_each_entry((intel_encoder), &(dev)->mode_config.encoder_list, base.head) \ if ((intel_encoder)->base.crtc == (__crtc)) @@ -186,6 +192,7 @@ enum hpd_pin { struct drm_i915_private; struct i915_mm_struct; struct i915_mmu_object; +struct i915_gem_request; enum intel_dpll_id { DPLL_ID_PRIVATE = -1, /* non-shared dpll in use */ @@ -198,10 +205,13 @@ enum intel_dpll_id { #define I915_NUM_PLLS 2 struct intel_dpll_hw_state { + /* i9xx, pch plls */ uint32_t dpll; uint32_t dpll_md; uint32_t fp0; uint32_t fp1; + + /* hsw, bdw */ uint32_t wrpll; }; @@ -277,10 +287,6 @@ struct intel_opregion { struct intel_overlay; struct intel_overlay_error_state; -struct drm_i915_master_private { - drm_local_map_t *sarea; - struct _drm_i915_sarea *sarea_priv; -}; #define I915_FENCE_REG_NONE -1 #define I915_MAX_NUM_FENCES 32 /* 32 fences + sign bit for FENCE_REG_NONE */ @@ -316,6 +322,7 @@ struct drm_i915_error_state { u32 pgtbl_er; u32 ier; u32 gtier[4]; + u32 gtimr[4]; u32 ccid; u32 derrmr; u32 forcewake; @@ -333,23 +340,26 @@ struct drm_i915_error_state { struct drm_i915_error_object *semaphore_obj; struct drm_i915_error_ring { + int id; bool valid; /* Software tracked state */ bool waiting; int hangcheck_score; - enum intel_ring_hangcheck_action hangcheck_action; + enum intel_engine_hangcheck_action hangcheck_action; int num_requests; /* our own tracking of ring head and tail */ u32 cpu_ring_head; u32 cpu_ring_tail; - - u32 semaphore_seqno[I915_NUM_RINGS - 1]; + u32 interrupts; + u32 irq_count; /* Register state */ u32 tail; u32 head; + u32 start; u32 ctl; + u32 mode; u32 hws; u32 ipeir; u32 ipehr; @@ -357,13 +367,15 @@ struct drm_i915_error_state { u32 bbstate; u32 instpm; u32 instps; - u32 seqno; + u32 seqno, request, tag, hangcheck; + u32 breadcrumb[I915_NUM_ENGINES]; u64 bbaddr; u64 acthd; u32 fault_reg; u64 faddr; u32 rc_psmi; /* sleep state */ - u32 semaphore_mboxes[I915_NUM_RINGS - 1]; + u32 semaphore_mboxes[I915_NUM_ENGINES]; + u32 semaphore_sync[I915_NUM_ENGINES]; struct drm_i915_error_object { int page_count; @@ -373,8 +385,14 @@ struct drm_i915_error_state { struct drm_i915_error_request { long jiffies; - u32 seqno; + long pid; + u32 batch; + u32 head; u32 tail; + u32 seqno; + u32 breadcrumb[I915_NUM_ENGINES]; + u32 complete; + u32 tag; } *requests; struct { @@ -387,11 +405,12 @@ struct drm_i915_error_state { pid_t pid; char comm[TASK_COMM_LEN]; - } ring[I915_NUM_RINGS]; + } ring[I915_NUM_ENGINES]; + struct drm_i915_error_buffer { u32 size; u32 name; - u32 rseqno, wseqno; + u32 rseqno[I915_NUM_ENGINES], wseqno, fseqno; u32 gtt_offset; u32 read_domains; u32 write_domain; @@ -406,6 +425,7 @@ struct drm_i915_error_state { } **active_bo, **pinned_bo; u32 *active_bo_count, *pinned_bo_count; + u32 vm_count; }; struct intel_connector; @@ -462,10 +482,10 @@ struct drm_i915_display_funcs { struct drm_display_mode *mode); void (*fdi_link_train)(struct drm_crtc *crtc); void (*init_clock_gating)(struct drm_device *dev); - int (*queue_flip)(struct drm_device *dev, struct drm_crtc *crtc, + int (*queue_flip)(struct i915_gem_request *rq, + struct intel_crtc *crtc, struct drm_framebuffer *fb, struct drm_i915_gem_object *obj, - struct intel_engine_cs *ring, uint32_t flags); void (*update_primary_plane)(struct drm_crtc *crtc, struct drm_framebuffer *fb, @@ -551,6 +571,7 @@ struct intel_uncore { struct intel_device_info { u32 display_mmio_offset; + u16 device_id; u8 num_pipes:3; u8 num_sprites[I915_MAX_PIPES]; u8 gen; @@ -586,6 +607,11 @@ struct i915_ctx_hang_stats { /* Time when this context was last blamed for a GPU reset */ unsigned long guilty_ts; + /* If the contexts causes a second GPU hang within this time, + * it is permanently banned from submitting any more work. + */ + unsigned long ban_period_seconds; + /* This context is banned to submit more work */ bool banned; }; @@ -611,16 +637,18 @@ struct i915_ctx_hang_stats { */ struct intel_context { struct kref ref; + struct drm_i915_private *i915; int user_handle; uint8_t remap_slice; struct drm_i915_file_private *file_priv; struct i915_ctx_hang_stats hang_stats; - struct i915_address_space *vm; + struct i915_hw_ppgtt *ppgtt; - struct { - struct drm_i915_gem_object *rcs_state; + struct intel_engine_context { + struct intel_ringbuffer *ring; + struct drm_i915_gem_object *state; bool initialized; - } legacy_hw_ctx; + } ring[I915_NUM_ENGINES]; struct list_head link; }; @@ -635,6 +663,8 @@ struct i915_fbc { struct drm_mm_node compressed_fb; struct drm_mm_node *compressed_llb; + bool false_color; + struct intel_fbc_work { struct delayed_work work; struct drm_crtc *crtc; @@ -688,6 +718,7 @@ enum intel_sbi_destination { #define QUIRK_LVDS_SSC_DISABLE (1<<1) #define QUIRK_INVERT_BRIGHTNESS (1<<2) #define QUIRK_BACKLIGHT_PRESENT (1<<3) +#define QUIRK_PIPEB_FORCE (1<<4) struct intel_fbdev; struct intel_fbc_work; @@ -917,6 +948,23 @@ struct intel_rps_ei { u32 media_c0; }; +struct intel_rps_bdw_cal { + u32 it_threshold_pct; /* interrupt, in percentage */ + u32 eval_interval; /* evaluation interval, in us */ + u32 last_ts; + u32 last_c0; + bool is_up; +}; + +struct intel_rps_bdw_turbo { + struct intel_rps_bdw_cal up; + struct intel_rps_bdw_cal down; + struct timer_list flip_timer; + u32 timeout; + atomic_t flip_received; + struct work_struct work_max_freq; +}; + struct intel_gen6_power_mgmt { /* work and pm_iir are protected by dev_priv->irq_lock */ struct work_struct work; @@ -950,6 +998,9 @@ struct intel_gen6_power_mgmt { bool enabled; struct delayed_work delayed_resume_work; + bool is_bdw_sw_turbo; /* Switch of BDW software turbo */ + struct intel_rps_bdw_turbo sw_turbo; /* Calculate RP interrupt timing */ + /* manual wa residency calculations */ struct intel_rps_ei up_ei, down_ei; @@ -982,7 +1033,6 @@ struct intel_ilk_power_mgmt { int r_t; struct drm_i915_gem_object *pwrctx; - struct drm_i915_gem_object *renderctx; }; struct drm_i915_private; @@ -1042,19 +1092,6 @@ struct i915_power_domains { struct i915_power_well *power_wells; }; -struct i915_dri1_state { - unsigned allow_batchbuffer : 1; - u32 __iomem *gfx_hws_cpu_addr; - - unsigned int cpp; - int back_offset; - int front_offset; - int current_page; - int page_flipping; - - uint32_t counter; -}; - struct i915_ums_state { /** * Flag if the X Server, and thus DRM, is not currently in @@ -1147,6 +1184,7 @@ struct i915_gem_mm { }; struct drm_i915_error_state_buf { + struct drm_i915_private *i915; unsigned bytes; unsigned size; int err; @@ -1213,7 +1251,7 @@ struct i915_gpu_error { /* Userspace knobs for gpu hang simulation; * combines both a ring mask, and extra flags */ - u32 stop_rings; + unsigned long stop_rings; #define I915_STOP_RING_ALLOW_BAN (1 << 31) #define I915_STOP_RING_ALLOW_WARN (1 << 30) @@ -1228,6 +1266,12 @@ enum modeset_restore { }; struct ddi_vbt_port_info { + /* + * This is an index in the HDMI/DVI DDI buffer translation table. + * The special value HDMI_LEVEL_SHIFT_UNKNOWN means the VBT didn't + * populate this field. + */ +#define HDMI_LEVEL_SHIFT_UNKNOWN 0xff uint8_t hdmi_level_shift; uint8_t supports_dvi:1; @@ -1417,11 +1461,12 @@ struct drm_i915_private { wait_queue_head_t gmbus_wait_queue; struct pci_dev *bridge_dev; - struct intel_engine_cs ring[I915_NUM_RINGS]; + struct intel_engine_cs engine[I915_NUM_ENGINES]; + struct intel_context *default_context; struct drm_i915_gem_object *semaphore_obj; - uint32_t last_seqno, next_seqno; + uint32_t next_seqno; - drm_dma_handle_t *status_page_dmah; + struct drm_dma_handle *status_page_dmah; struct resource mch_res; /* protects the irq masks */ @@ -1448,7 +1493,7 @@ struct drm_i915_private { u32 pm_rps_events; u32 pipestat_irq_mask[I915_MAX_PIPES]; - struct work_struct hotplug_work; + struct delayed_work hotplug_work; struct { unsigned long hpd_last_jiffies; int hpd_cnt; @@ -1470,11 +1515,14 @@ struct drm_i915_private { struct intel_overlay *overlay; /* backlight registers and fields in struct intel_panel */ - spinlock_t backlight_lock; + struct mutex backlight_lock; /* LVDS info */ bool no_aux_handshake; + /* protects panel power sequencer state */ + struct mutex pps_mutex; + struct drm_i915_fence_reg fence_regs[I915_MAX_NUM_FENCES]; /* assume 965 */ int fence_reg_start; /* 4 if userland hasn't ioctl'd us yet */ int num_fence_regs; /* 8 on pre-965, 16 otherwise */ @@ -1526,6 +1574,20 @@ struct drm_i915_private { struct intel_shared_dpll shared_dplls[I915_NUM_PLLS]; int dpio_phy_iosf_port[I915_NUM_PHYS_VLV]; + /* + * workarounds are currently applied at different places and + * changes are being done to consolidate them so exact count is + * not clear at this point, use a max value for now. + */ +#define I915_MAX_WA_REGS 16 + struct { + u32 addr; + u32 value; + /* bitmask representing WA bits */ + u32 mask; + } intel_wa_regs[I915_MAX_WA_REGS]; + u32 num_wa_regs; + /* Reclocking support */ bool render_reclock_avail; bool lvds_downclock_avail; @@ -1561,14 +1623,9 @@ struct drm_i915_private { #ifdef CONFIG_DRM_I915_FBDEV /* list of fbdev register on this device */ struct intel_fbdev *fbdev; + struct work_struct fbdev_suspend_work; #endif - /* - * The console may be contended at resume, but we don't - * want it to block on it. - */ - struct work_struct console_resume_work; - struct drm_property *broadcast_rgb_property; struct drm_property *force_audio_property; @@ -1614,12 +1671,10 @@ struct drm_i915_private { */ struct workqueue_struct *dp_wq; - /* Old dri1 support infrastructure, beware the dragons ya fools entering - * here! */ - struct i915_dri1_state dri1; + uint32_t bios_vgacntr; + /* Old ums support infrastructure, same warning applies. */ struct i915_ums_state ums; - /* * NOTE: This is the dri1/ums dungeon, don't add stuff here. Your patch * will be rejected. Instead look for a better place. @@ -1632,9 +1687,11 @@ static inline struct drm_i915_private *to_i915(const struct drm_device *dev) } /* Iterate over initialised rings */ -#define for_each_ring(ring__, dev_priv__, i__) \ - for ((i__) = 0; (i__) < I915_NUM_RINGS; (i__)++) \ - if (((ring__) = &(dev_priv__)->ring[(i__)]), intel_ring_initialized((ring__))) +#define for_each_engine(engine__, dev_priv__, i__) \ + for ((i__) = 0; (i__) < I915_NUM_ENGINES; (i__)++) \ + if (((engine__) = &(dev_priv__)->engine[(i__)]), intel_engine_initialized((engine__))) + +#define RCS_ENGINE(x) (&__I915__(x)->engine[RCS]) enum hdmi_force_audio { HDMI_AUDIO_OFF_DVI = -2, /* no aux data for HDMI-DVI converter */ @@ -1699,16 +1756,15 @@ struct drm_i915_gem_object { struct drm_mm_node *stolen; struct list_head global_list; - struct list_head ring_list; /** Used in execbuf to temporarily hold a ref */ struct list_head obj_exec_link; /** * This is set if the object is on the active lists (has pending - * rendering and so a non-zero seqno), and is not set if it i s on - * inactive (ready to be unbound) list. + * rendering and so a submitted request), and is not set if it is on + * inactive (ready to be unbound) list. We track activity per engine. */ - unsigned int active:1; + unsigned int active:I915_NUM_ENGINE_BITS; /** * This is set if the object has been written to since last bound @@ -1761,13 +1817,6 @@ struct drm_i915_gem_object { * Only honoured if hardware has relevant pte bit */ unsigned long gt_ro:1; - - /* - * Is the GPU currently using a fence to access this buffer, - */ - unsigned int pending_fenced_gpu_access:1; - unsigned int fenced_gpu_access:1; - unsigned int cache_level:3; unsigned int has_aliasing_ppgtt_mapping:1; @@ -1783,13 +1832,11 @@ struct drm_i915_gem_object { void *dma_buf_vmapping; int vmapping_count; - struct intel_engine_cs *ring; - - /** Breadcrumb of last rendering to the buffer. */ - uint32_t last_read_seqno; - uint32_t last_write_seqno; - /** Breadcrumb of last fenced GPU access to the buffer. */ - uint32_t last_fenced_seqno; + /** Breadcrumbs of last rendering to the buffer. */ + struct { + struct i915_gem_request *request; + struct list_head engine_list; + } last_write, last_read[I915_NUM_ENGINES], last_fence; /** Current tiling stride for the object, if it's tiled. */ uint32_t stride; @@ -1805,7 +1852,7 @@ struct drm_i915_gem_object { struct drm_file *pin_filp; /** for phy allocated objects */ - drm_dma_handle_t *phys_handle; + struct drm_dma_handle *phys_handle; union { struct i915_gem_userptr { @@ -1827,44 +1874,13 @@ void i915_gem_track_fb(struct drm_i915_gem_object *old, unsigned frontbuffer_bits); /** - * Request queue structure. - * - * The request queue allows us to note sequence numbers that have been emitted - * and may be associated with active buffers to be retired. - * - * By keeping this list, we can avoid having to do questionable - * sequence-number comparisons on buffer last_rendering_seqnos, and associate - * an emission time with seqnos for tracking how far ahead of the GPU we are. + * Returns true if seq1 is later than seq2. */ -struct drm_i915_gem_request { - /** On Which ring this request was generated */ - struct intel_engine_cs *ring; - - /** GEM sequence number associated with this request. */ - uint32_t seqno; - - /** Position in the ringbuffer of the start of the request */ - u32 head; - - /** Position in the ringbuffer of the end of the request */ - u32 tail; - - /** Context related to this request */ - struct intel_context *ctx; - - /** Batch buffer related to this request if any */ - struct drm_i915_gem_object *batch_obj; - - /** Time at which this request was emitted, in jiffies. */ - unsigned long emitted_jiffies; - - /** global list entry for this request */ - struct list_head list; - - struct drm_i915_file_private *file_priv; - /** file_priv list entry for this request */ - struct list_head client_list; -}; +static inline bool +__i915_seqno_passed(uint32_t seq1, uint32_t seq2) +{ + return (int32_t)(seq1 - seq2) >= 0; +} struct drm_i915_file_private { struct drm_i915_private *dev_priv; @@ -1878,7 +1894,7 @@ struct drm_i915_file_private { struct idr context_idr; atomic_t rps_wait_boost; - struct intel_engine_cs *bsd_ring; + struct intel_engine_cs *bsd_engine; }; /* @@ -1971,51 +1987,63 @@ struct drm_i915_cmd_table { int count; }; -#define INTEL_INFO(dev) (&to_i915(dev)->info) - -#define IS_I830(dev) ((dev)->pdev->device == 0x3577) -#define IS_845G(dev) ((dev)->pdev->device == 0x2562) +/* Note that the (struct drm_i915_private *) cast is just to shut up gcc. */ +#define __I915__(p) ({ \ + struct drm_i915_private *__p; \ + if (__builtin_types_compatible_p(typeof(*p), struct drm_i915_private)) \ + __p = (struct drm_i915_private *)p; \ + else if (__builtin_types_compatible_p(typeof(*p), struct drm_device)) \ + __p = to_i915((struct drm_device *)p); \ + else \ + BUILD_BUG(); \ + __p; \ +}) +#define INTEL_INFO(p) (&__I915__(p)->info) +#define INTEL_DEVID(p) (INTEL_INFO(p)->device_id) + +#define IS_I830(dev) (INTEL_DEVID(dev) == 0x3577) +#define IS_845G(dev) (INTEL_DEVID(dev) == 0x2562) #define IS_I85X(dev) (INTEL_INFO(dev)->is_i85x) -#define IS_I865G(dev) ((dev)->pdev->device == 0x2572) +#define IS_I865G(dev) (INTEL_DEVID(dev) == 0x2572) #define IS_I915G(dev) (INTEL_INFO(dev)->is_i915g) -#define IS_I915GM(dev) ((dev)->pdev->device == 0x2592) -#define IS_I945G(dev) ((dev)->pdev->device == 0x2772) +#define IS_I915GM(dev) (INTEL_DEVID(dev) == 0x2592) +#define IS_I945G(dev) (INTEL_DEVID(dev) == 0x2772) #define IS_I945GM(dev) (INTEL_INFO(dev)->is_i945gm) #define IS_BROADWATER(dev) (INTEL_INFO(dev)->is_broadwater) #define IS_CRESTLINE(dev) (INTEL_INFO(dev)->is_crestline) -#define IS_GM45(dev) ((dev)->pdev->device == 0x2A42) +#define IS_GM45(dev) (INTEL_DEVID(dev) == 0x2A42) #define IS_G4X(dev) (INTEL_INFO(dev)->is_g4x) -#define IS_PINEVIEW_G(dev) ((dev)->pdev->device == 0xa001) -#define IS_PINEVIEW_M(dev) ((dev)->pdev->device == 0xa011) +#define IS_PINEVIEW_G(dev) (INTEL_DEVID(dev) == 0xa001) +#define IS_PINEVIEW_M(dev) (INTEL_DEVID(dev) == 0xa011) #define IS_PINEVIEW(dev) (INTEL_INFO(dev)->is_pineview) #define IS_G33(dev) (INTEL_INFO(dev)->is_g33) -#define IS_IRONLAKE_M(dev) ((dev)->pdev->device == 0x0046) +#define IS_IRONLAKE_M(dev) (INTEL_DEVID(dev) == 0x0046) #define IS_IVYBRIDGE(dev) (INTEL_INFO(dev)->is_ivybridge) -#define IS_IVB_GT1(dev) ((dev)->pdev->device == 0x0156 || \ - (dev)->pdev->device == 0x0152 || \ - (dev)->pdev->device == 0x015a) -#define IS_SNB_GT1(dev) ((dev)->pdev->device == 0x0102 || \ - (dev)->pdev->device == 0x0106 || \ - (dev)->pdev->device == 0x010A) +#define IS_IVB_GT1(dev) (INTEL_DEVID(dev) == 0x0156 || \ + INTEL_DEVID(dev) == 0x0152 || \ + INTEL_DEVID(dev) == 0x015a) +#define IS_SNB_GT1(dev) (INTEL_DEVID(dev) == 0x0102 || \ + INTEL_DEVID(dev) == 0x0106 || \ + INTEL_DEVID(dev) == 0x010A) #define IS_VALLEYVIEW(dev) (INTEL_INFO(dev)->is_valleyview) #define IS_CHERRYVIEW(dev) (INTEL_INFO(dev)->is_valleyview && IS_GEN8(dev)) #define IS_HASWELL(dev) (INTEL_INFO(dev)->is_haswell) #define IS_BROADWELL(dev) (!INTEL_INFO(dev)->is_valleyview && IS_GEN8(dev)) #define IS_MOBILE(dev) (INTEL_INFO(dev)->is_mobile) #define IS_HSW_EARLY_SDV(dev) (IS_HASWELL(dev) && \ - ((dev)->pdev->device & 0xFF00) == 0x0C00) + (INTEL_DEVID(dev) & 0xFF00) == 0x0C00) #define IS_BDW_ULT(dev) (IS_BROADWELL(dev) && \ - (((dev)->pdev->device & 0xf) == 0x2 || \ - ((dev)->pdev->device & 0xf) == 0x6 || \ - ((dev)->pdev->device & 0xf) == 0xe)) + ((INTEL_DEVID(dev) & 0xf) == 0x2 || \ + (INTEL_DEVID(dev) & 0xf) == 0x6 || \ + (INTEL_DEVID(dev) & 0xf) == 0xe)) #define IS_HSW_ULT(dev) (IS_HASWELL(dev) && \ - ((dev)->pdev->device & 0xFF00) == 0x0A00) + (INTEL_DEVID(dev) & 0xFF00) == 0x0A00) #define IS_ULT(dev) (IS_HSW_ULT(dev) || IS_BDW_ULT(dev)) #define IS_HSW_GT3(dev) (IS_HASWELL(dev) && \ - ((dev)->pdev->device & 0x00F0) == 0x0020) + (INTEL_DEVID(dev) & 0x00F0) == 0x0020) /* ULX machines are also considered ULT. */ -#define IS_HSW_ULX(dev) ((dev)->pdev->device == 0x0A0E || \ - (dev)->pdev->device == 0x0A1E) +#define IS_HSW_ULX(dev) (INTEL_DEVID(dev) == 0x0A0E || \ + INTEL_DEVID(dev) == 0x0A1E) #define IS_PRELIMINARY_HW(intel_info) ((intel_info)->is_preliminary) /* @@ -2046,11 +2074,12 @@ struct drm_i915_cmd_table { to_i915(dev)->ellc_size) #define I915_NEED_GFX_HWS(dev) (INTEL_INFO(dev)->need_gfx_hws) -#define HAS_HW_CONTEXTS(dev) (INTEL_INFO(dev)->gen >= 6) +#define HAS_HW_CONTEXTS(dev) (INTEL_INFO(dev)->gen >= 5) +#define HAS_LOGICAL_RING_CONTEXTS(dev) (INTEL_INFO(dev)->gen >= 8) #define HAS_ALIASING_PPGTT(dev) (INTEL_INFO(dev)->gen >= 6) #define HAS_PPGTT(dev) (INTEL_INFO(dev)->gen >= 7 && !IS_GEN8(dev)) -#define USES_PPGTT(dev) intel_enable_ppgtt(dev, false) -#define USES_FULL_PPGTT(dev) intel_enable_ppgtt(dev, true) +#define USES_PPGTT(dev) (i915.enable_ppgtt) +#define USES_FULL_PPGTT(dev) (i915.enable_ppgtt == 2) #define HAS_OVERLAY(dev) (INTEL_INFO(dev)->has_overlay) #define OVERLAY_NEEDS_PHYSICAL(dev) (INTEL_INFO(dev)->overlay_needs_physical) @@ -2152,9 +2181,7 @@ struct i915_params { }; extern struct i915_params i915 __read_mostly; - /* i915_dma.c */ -void i915_update_dri1_breadcrumb(struct drm_device *dev); -extern void i915_kernel_lost_context(struct drm_device * dev); +/* i915_dma.c */ extern int i915_driver_load(struct drm_device *, unsigned long flags); extern int i915_driver_unload(struct drm_device *); extern int i915_driver_open(struct drm_device *dev, struct drm_file *file); @@ -2168,9 +2195,6 @@ extern int i915_driver_device_is_agp(struct drm_device * dev); extern long i915_compat_ioctl(struct file *filp, unsigned int cmd, unsigned long arg); #endif -extern int i915_emit_box(struct drm_device *dev, - struct drm_clip_rect *box, - int DR1, int DR4); extern int intel_gpu_reset(struct drm_device *dev); extern int i915_reset(struct drm_device *dev); extern unsigned long i915_chipset_val(struct drm_i915_private *dev_priv); @@ -2180,8 +2204,6 @@ extern void i915_update_gfx_val(struct drm_i915_private *dev_priv); int vlv_force_gfx_clock(struct drm_i915_private *dev_priv, bool on); void intel_hpd_cancel_work(struct drm_i915_private *dev_priv); -extern void intel_console_resume(struct work_struct *work); - /* i915_irq.c */ void i915_queue_hangcheck(struct drm_device *dev); __printf(3, 4) @@ -2263,6 +2285,12 @@ int i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data, int i915_gem_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv); void i915_gem_load(struct drm_device *dev); +unsigned long i915_gem_shrink(struct drm_i915_private *dev_priv, + long target, + unsigned flags); +#define I915_SHRINK_PURGEABLE 0x1 +#define I915_SHRINK_UNBOUND 0x2 +#define I915_SHRINK_BOUND 0x4 void *i915_gem_object_alloc(struct drm_device *dev); void i915_gem_object_free(struct drm_i915_gem_object *obj); void i915_gem_object_init(struct drm_i915_gem_object *obj, @@ -2315,22 +2343,12 @@ static inline void i915_gem_object_unpin_pages(struct drm_i915_gem_object *obj) int __must_check i915_mutex_lock_interruptible(struct drm_device *dev); int i915_gem_object_sync(struct drm_i915_gem_object *obj, - struct intel_engine_cs *to); -void i915_vma_move_to_active(struct i915_vma *vma, - struct intel_engine_cs *ring); + struct i915_gem_request *rq); int i915_gem_dumb_create(struct drm_file *file_priv, struct drm_device *dev, struct drm_mode_create_dumb *args); int i915_gem_mmap_gtt(struct drm_file *file_priv, struct drm_device *dev, uint32_t handle, uint64_t *offset); -/** - * Returns true if seq1 is later than seq2. - */ -static inline bool -i915_seqno_passed(uint32_t seq1, uint32_t seq2) -{ - return (int32_t)(seq1 - seq2) >= 0; -} int __must_check i915_gem_get_seqno(struct drm_device *dev, u32 *seqno); int __must_check i915_gem_set_seqno(struct drm_device *dev, u32 seqno); @@ -2340,24 +2358,33 @@ int __must_check i915_gem_object_put_fence(struct drm_i915_gem_object *obj); bool i915_gem_object_pin_fence(struct drm_i915_gem_object *obj); void i915_gem_object_unpin_fence(struct drm_i915_gem_object *obj); -struct drm_i915_gem_request * -i915_gem_find_active_request(struct intel_engine_cs *ring); - bool i915_gem_retire_requests(struct drm_device *dev); -void i915_gem_retire_requests_ring(struct intel_engine_cs *ring); -int __must_check i915_gem_check_wedge(struct i915_gpu_error *error, - bool interruptible); -int __must_check i915_gem_check_olr(struct intel_engine_cs *ring, u32 seqno); +void i915_gem_retire_requests__engine(struct intel_engine_cs *engine); + +static inline bool __i915_reset_in_progress(unsigned x) +{ + return unlikely(x & I915_RESET_IN_PROGRESS_FLAG); +} static inline bool i915_reset_in_progress(struct i915_gpu_error *error) { - return unlikely(atomic_read(&error->reset_counter) - & (I915_RESET_IN_PROGRESS_FLAG | I915_WEDGED)); + return __i915_reset_in_progress(atomic_read(&error->reset_counter)); +} + +static inline bool __i915_terminally_wedged(unsigned x) +{ + return unlikely(x & I915_WEDGED); } static inline bool i915_terminally_wedged(struct i915_gpu_error *error) { - return atomic_read(&error->reset_counter) & I915_WEDGED; + return __i915_terminally_wedged(atomic_read(&error->reset_counter)); +} + +static inline bool i915_recovery_pending(struct i915_gpu_error *error) +{ + unsigned x = atomic_read(&error->reset_counter); + return __i915_reset_in_progress(x) && !__i915_terminally_wedged(x); } static inline u32 i915_reset_count(struct i915_gpu_error *error) @@ -2382,19 +2409,10 @@ bool i915_gem_clflush_object(struct drm_i915_gem_object *obj, bool force); int __must_check i915_gem_object_finish_gpu(struct drm_i915_gem_object *obj); int __must_check i915_gem_init(struct drm_device *dev); int __must_check i915_gem_init_hw(struct drm_device *dev); -int i915_gem_l3_remap(struct intel_engine_cs *ring, int slice); +void i915_gem_fini(struct drm_device *dev); void i915_gem_init_swizzling(struct drm_device *dev); -void i915_gem_cleanup_ringbuffer(struct drm_device *dev); int __must_check i915_gpu_idle(struct drm_device *dev); int __must_check i915_gem_suspend(struct drm_device *dev); -int __i915_add_request(struct intel_engine_cs *ring, - struct drm_file *file, - struct drm_i915_gem_object *batch_obj, - u32 *seqno); -#define i915_add_request(ring, seqno) \ - __i915_add_request(ring, NULL, NULL, seqno) -int __must_check i915_wait_seqno(struct intel_engine_cs *ring, - uint32_t seqno); int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf); int __must_check i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, @@ -2404,7 +2422,7 @@ i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write); int __must_check i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj, u32 alignment, - struct intel_engine_cs *pipelined); + struct i915_gem_request *pipelined); void i915_gem_object_unpin_from_display_plane(struct drm_i915_gem_object *obj); int i915_gem_object_attach_phys(struct drm_i915_gem_object *obj, int align); @@ -2451,30 +2469,36 @@ static inline bool i915_gem_obj_is_pinned(struct drm_i915_gem_object *obj) { } /* Some GGTT VM helpers */ -#define obj_to_ggtt(obj) \ - (&((struct drm_i915_private *)(obj)->base.dev->dev_private)->gtt.base) +#define i915_obj_to_ggtt(obj) (&to_i915((obj)->base.dev)->gtt.base) static inline bool i915_is_ggtt(struct i915_address_space *vm) { - struct i915_address_space *ggtt = - &((struct drm_i915_private *)(vm)->dev->dev_private)->gtt.base; - return vm == ggtt; + return vm == &to_i915(vm->dev)->gtt.base; } +static inline struct i915_hw_ppgtt * +i915_vm_to_ppgtt(struct i915_address_space *vm) +{ + WARN_ON(i915_is_ggtt(vm)); + + return container_of(vm, struct i915_hw_ppgtt, base); +} + + static inline bool i915_gem_obj_ggtt_bound(struct drm_i915_gem_object *obj) { - return i915_gem_obj_bound(obj, obj_to_ggtt(obj)); + return i915_gem_obj_bound(obj, i915_obj_to_ggtt(obj)); } static inline unsigned long i915_gem_obj_ggtt_offset(struct drm_i915_gem_object *obj) { - return i915_gem_obj_offset(obj, obj_to_ggtt(obj)); + return i915_gem_obj_offset(obj, i915_obj_to_ggtt(obj)); } static inline unsigned long i915_gem_obj_ggtt_size(struct drm_i915_gem_object *obj) { - return i915_gem_obj_size(obj, obj_to_ggtt(obj)); + return i915_gem_obj_size(obj, i915_obj_to_ggtt(obj)); } static inline int __must_check @@ -2482,7 +2506,8 @@ i915_gem_obj_ggtt_pin(struct drm_i915_gem_object *obj, uint32_t alignment, unsigned flags) { - return i915_gem_object_pin(obj, obj_to_ggtt(obj), alignment, flags | PIN_GLOBAL); + return i915_gem_object_pin(obj, i915_obj_to_ggtt(obj), + alignment, flags | PIN_GLOBAL); } static inline int @@ -2494,18 +2519,19 @@ i915_gem_object_ggtt_unbind(struct drm_i915_gem_object *obj) void i915_gem_object_ggtt_unpin(struct drm_i915_gem_object *obj); /* i915_gem_context.c */ -#define ctx_to_ppgtt(ctx) container_of((ctx)->vm, struct i915_hw_ppgtt, base) int __must_check i915_gem_context_init(struct drm_device *dev); void i915_gem_context_fini(struct drm_device *dev); -void i915_gem_context_reset(struct drm_device *dev); int i915_gem_context_open(struct drm_device *dev, struct drm_file *file); int i915_gem_context_enable(struct drm_i915_private *dev_priv); void i915_gem_context_close(struct drm_device *dev, struct drm_file *file); -int i915_switch_context(struct intel_engine_cs *ring, - struct intel_context *to); +int i915_request_switch_context(struct i915_gem_request *rq); +void i915_request_switch_context__commit(struct i915_gem_request *rq); +void i915_request_switch_context__undo(struct i915_gem_request *rq); struct intel_context * i915_gem_context_get(struct drm_i915_file_private *file_priv, u32 id); void i915_gem_context_free(struct kref *ctx_ref); +struct drm_i915_gem_object * +i915_gem_alloc_context_obj(struct drm_device *dev, size_t size); static inline void i915_gem_context_reference(struct intel_context *ctx) { kref_get(&ctx->ref); @@ -2527,7 +2553,7 @@ int i915_gem_context_destroy_ioctl(struct drm_device *dev, void *data, struct drm_file *file); /* i915_gem_render_state.c */ -int i915_gem_render_state_init(struct intel_engine_cs *ring); +int i915_gem_render_state_init(struct i915_gem_request *rq); /* i915_gem_evict.c */ int __must_check i915_gem_evict_something(struct drm_device *dev, struct i915_address_space *vm, @@ -2547,6 +2573,160 @@ static inline void i915_gem_chipset_flush(struct drm_device *dev) intel_gtt_chipset_flush(); } +/* i915_gem_request.c */ + +/** + * Request queue structure. + * + * The request queue allows us to note sequence numbers that have been emitted + * and may be associated with active buffers to be retired. + * + * By keeping this list, we can avoid having to do questionable + * sequence-number comparisons on buffer last_rendering_seqnos, and associate + * an emission time with seqnos for tracking how far ahead of the GPU we are. + */ +struct i915_gem_request { + struct kref kref; + + /** On which ring/engine/ctx this request was generated */ + struct drm_i915_private *i915; + struct intel_context *ctx; + struct intel_engine_cs *engine; + struct intel_ringbuffer *ring; + + /** How many GPU resets ago was this request first constructed? */ + unsigned reset_counter; + + /** GEM sequence number/breadcrumb associated with this request. */ + u32 seqno; + u32 breadcrumb[I915_NUM_ENGINES]; + u32 semaphore[I915_NUM_ENGINES]; + + /** Position in the ringbuffer of the request */ + u32 head, tail; + + /** Batch buffer and objects related to this request if any */ + struct i915_vma *batch; + struct list_head vmas; + + /** Time at which this request was emitted, in jiffies. */ + unsigned long emitted_jiffies; + + /** global list entry for this request */ + struct list_head engine_list; + struct list_head breadcrumb_link; + + struct drm_i915_file_private *file_priv; + /** file_priv list entry for this request */ + struct list_head client_list; + + u16 tag; + unsigned remap_l3:8; + unsigned pending_flush:4; + bool outstanding:1; + bool has_ctx_switch:1; + + bool completed; /* kept separate for atomicity */ +}; + +static inline struct intel_engine_cs *i915_request_engine(struct i915_gem_request *rq) +{ + return rq ? rq->engine : NULL; +} + +static inline int i915_request_engine_id(struct i915_gem_request *rq) +{ + return rq ? rq->engine->id : -1; +} + +static inline u32 i915_request_seqno(struct i915_gem_request *rq) +{ + return rq ? rq->seqno : 0; +} + +bool __i915_request_complete__wa(struct i915_gem_request *rq); + +static inline bool +i915_request_complete(struct i915_gem_request *rq) +{ + if (!rq->completed && rq->engine->is_complete(rq)) { + trace_i915_gem_request_complete(rq); + rq->completed = true; + } + return rq->completed; +} + +static inline struct i915_gem_request * +i915_request_get(struct i915_gem_request *rq) +{ + if (rq) + kref_get(&rq->kref); + return rq; +} + +void __i915_request_free(struct kref *kref); + +static inline void +i915_request_put(struct i915_gem_request *rq) +{ + if (rq == NULL) + return; + + lockdep_assert_held(&rq->i915->dev->struct_mutex); + kref_put(&rq->kref, __i915_request_free); +} + +static inline void +i915_request_put__unlocked(struct i915_gem_request *rq) +{ + if (!atomic_add_unless(&rq->kref.refcount, -1, 1)) { + struct drm_device *dev = rq->i915->dev; + + mutex_lock(&dev->struct_mutex); + if (likely(atomic_dec_and_test(&rq->kref.refcount))) + __i915_request_free(&rq->kref); + mutex_unlock(&dev->struct_mutex); + } +} + +int __must_check +i915_request_add_vma(struct i915_gem_request *rq, + struct i915_vma *vma, + unsigned fenced); +#define VMA_IS_FENCED 0x1 +#define VMA_HAS_FENCE 0x2 +int __must_check +i915_request_emit_flush(struct i915_gem_request *rq, + unsigned flags); +int __must_check +__i915_request_emit_breadcrumb(struct i915_gem_request *rq, int id); +static inline int __must_check +i915_request_emit_breadcrumb(struct i915_gem_request *rq) +{ + return __i915_request_emit_breadcrumb(rq, rq->engine->id); +} +static inline int __must_check +i915_request_emit_semaphore(struct i915_gem_request *rq, int id) +{ + return __i915_request_emit_breadcrumb(rq, id); +} +int __must_check +i915_request_emit_batchbuffer(struct i915_gem_request *rq, + struct i915_vma *batch, + uint64_t start, uint32_t len, + unsigned flags); +int __must_check +i915_request_commit(struct i915_gem_request *rq); +struct i915_gem_request * +i915_request_get_breadcrumb(struct i915_gem_request *rq); +int __must_check +i915_request_wait(struct i915_gem_request *rq); +int __i915_request_wait(struct i915_gem_request *rq, + bool interruptible, + s64 *timeout, + struct drm_i915_file_private *file); +void i915_request_retire(struct i915_gem_request *rq); + /* i915_gem_stolen.c */ int i915_gem_init_stolen(struct drm_device *dev); int i915_gem_stolen_setup_compression(struct drm_device *dev, int size, int fb_cpp); @@ -2573,13 +2753,6 @@ void i915_gem_detect_bit_6_swizzle(struct drm_device *dev); void i915_gem_object_do_bit_17_swizzle(struct drm_i915_gem_object *obj); void i915_gem_object_save_bit_17_swizzle(struct drm_i915_gem_object *obj); -/* i915_gem_debug.c */ -#if WATCH_LISTS -int i915_verify_lists(struct drm_device *dev); -#else -#define i915_verify_lists(dev) 0 -#endif - /* i915_debugfs.c */ int i915_debugfs_init(struct drm_minor *minor); void i915_debugfs_cleanup(struct drm_minor *minor); @@ -2595,6 +2768,7 @@ void i915_error_printf(struct drm_i915_error_state_buf *e, const char *f, ...); int i915_error_state_to_str(struct drm_i915_error_state_buf *estr, const struct i915_error_state_file_priv *error); int i915_error_state_buf_init(struct drm_i915_error_state_buf *eb, + struct drm_i915_private *i915, size_t count, loff_t pos); static inline void i915_error_state_buf_release( struct drm_i915_error_state_buf *eb) @@ -2609,14 +2783,14 @@ void i915_error_state_put(struct i915_error_state_file_priv *error_priv); void i915_destroy_error_state(struct drm_device *dev); void i915_get_extra_instdone(struct drm_device *dev, uint32_t *instdone); -const char *i915_cache_level_str(int type); +const char *i915_cache_level_str(struct drm_i915_private *i915, int type); /* i915_cmd_parser.c */ int i915_cmd_parser_get_version(void); -int i915_cmd_parser_init_ring(struct intel_engine_cs *ring); -void i915_cmd_parser_fini_ring(struct intel_engine_cs *ring); -bool i915_needs_cmd_parser(struct intel_engine_cs *ring); -int i915_parse_cmds(struct intel_engine_cs *ring, +int i915_cmd_parser_init_engine(struct intel_engine_cs *engine); +void i915_cmd_parser_fini_engine(struct intel_engine_cs *engine); +bool i915_needs_cmd_parser(struct intel_engine_cs *engine); +int i915_parse_cmds(struct intel_engine_cs *engine, struct drm_i915_gem_object *batch_obj, u32 batch_start_offset, bool is_master); @@ -2701,10 +2875,13 @@ extern void intel_modeset_setup_hw_state(struct drm_device *dev, extern void i915_redisable_vga(struct drm_device *dev); extern void i915_redisable_vga_power_on(struct drm_device *dev); extern bool intel_fbc_enabled(struct drm_device *dev); +extern void gen8_fbc_sw_flush(struct drm_device *dev, u32 value); extern void intel_disable_fbc(struct drm_device *dev); extern bool ironlake_set_drps(struct drm_device *dev, u8 val); extern void intel_init_pch_refclk(struct drm_device *dev); extern void gen6_set_rps(struct drm_device *dev, u8 val); +extern void bdw_software_turbo(struct drm_device *dev); +extern void gen8_flip_interrupt(struct drm_device *dev); extern void valleyview_set_rps(struct drm_device *dev, u8 val); extern void intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable); @@ -2712,14 +2889,11 @@ extern void intel_detect_pch(struct drm_device *dev); extern int intel_trans_dp_port_sel(struct drm_crtc *crtc); extern int intel_enable_rc6(const struct drm_device *dev); -extern bool i915_semaphore_is_enabled(struct drm_device *dev); int i915_reg_read_ioctl(struct drm_device *dev, void *data, struct drm_file *file); int i915_get_reset_stats_ioctl(struct drm_device *dev, void *data, struct drm_file *file); -void intel_notify_mmio_flip(struct intel_engine_cs *ring); - /* overlay */ extern struct intel_overlay_error_state *intel_overlay_capture_error_state(struct drm_device *dev); extern void intel_overlay_print_error_state(struct drm_i915_error_state_buf *e, diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c index ad55b06..98a78a4 100644 --- a/drivers/gpu/drm/i915/i915_gem.c +++ b/drivers/gpu/drm/i915/i915_gem.c @@ -38,15 +38,14 @@ #include #include +#define RQ_BUG_ON(x) BUG_ON(x) + static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj); static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj, bool force); static __must_check int i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj, bool readonly); -static void -i915_gem_object_retire(struct drm_i915_gem_object *obj); - static void i915_gem_write_fence(struct drm_device *dev, int reg, struct drm_i915_gem_object *obj); static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj, @@ -60,7 +59,6 @@ static unsigned long i915_gem_shrinker_scan(struct shrinker *shrinker, static int i915_gem_shrinker_oom(struct notifier_block *nb, unsigned long event, void *ptr); -static unsigned long i915_gem_purge(struct drm_i915_private *dev_priv, long target); static unsigned long i915_gem_shrink_all(struct drm_i915_private *dev_priv); static bool cpu_cache_is_coherent(struct drm_device *dev, @@ -108,23 +106,108 @@ static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv, spin_unlock(&dev_priv->mm.object_stat_lock); } +static void +i915_gem_object_retire__write(struct drm_i915_gem_object *obj) +{ + RQ_BUG_ON(obj->active == 0); + RQ_BUG_ON(obj->last_write.request == NULL); + RQ_BUG_ON(list_empty(&obj->last_write.engine_list)); + + intel_fb_obj_flush(obj, true); + list_del_init(&obj->last_write.engine_list); + i915_request_put(obj->last_write.request); + obj->last_write.request = NULL; +} + +static void +i915_gem_object_retire__fence(struct drm_i915_gem_object *obj) +{ + RQ_BUG_ON(obj->active == 0); + RQ_BUG_ON(obj->last_fence.request == NULL); + RQ_BUG_ON(list_empty(&obj->last_fence.engine_list)); + + list_del_init(&obj->last_fence.engine_list); + i915_request_put(obj->last_fence.request); + obj->last_fence.request = NULL; +} + +static void +i915_gem_object_retire__read(struct drm_i915_gem_object *obj, + struct intel_engine_cs *engine) +{ + struct i915_vma *vma; + + RQ_BUG_ON(obj->active == 0); + RQ_BUG_ON(obj->last_read[engine->id].request == NULL); + RQ_BUG_ON(list_empty(&obj->last_read[engine->id].engine_list)); + + list_del_init(&obj->last_read[engine->id].engine_list); + i915_request_put(obj->last_read[engine->id].request); + obj->last_read[engine->id].request = NULL; + + if (obj->last_write.request && + obj->last_write.request->engine == engine) + i915_gem_object_retire__write(obj); + + if (obj->last_fence.request && + obj->last_fence.request->engine == engine) + i915_gem_object_retire__fence(obj); + + if (--obj->active) + return; + + RQ_BUG_ON(obj->last_write.request); + RQ_BUG_ON(obj->last_fence.request); + + list_for_each_entry(vma, &obj->vma_list, vma_link) + if (!list_empty(&vma->mm_list)) + list_move_tail(&vma->mm_list, &vma->vm->inactive_list); + + drm_gem_object_unreference(&obj->base); +} + +static void +i915_gem_object_retire(struct drm_i915_gem_object *obj) +{ + struct i915_gem_request *rq; + int i; + + /* We should only be called from code paths where we know we + * hold both the active reference *and* a user reference. + * Therefore we can safely access the object after retiring as + * we will hold a second reference and not free the object. + */ + + rq = obj->last_write.request; + if (rq && i915_request_complete(rq)) + i915_gem_object_retire__write(obj); + + rq = obj->last_fence.request; + if (rq && i915_request_complete(rq)) + i915_gem_object_retire__fence(obj); + + for (i = 0; i < I915_NUM_ENGINES; i++) { + rq = obj->last_read[i].request; + if (rq && i915_request_complete(rq)) + i915_gem_object_retire__read(obj, rq->engine); + } + + if (!obj->active) + i915_gem_retire_requests(obj->base.dev); +} + static int i915_gem_wait_for_error(struct i915_gpu_error *error) { int ret; -#define EXIT_COND (!i915_reset_in_progress(error) || \ - i915_terminally_wedged(error)) - if (EXIT_COND) - return 0; - /* * Only wait 10 seconds for the gpu reset to complete to avoid hanging * userspace. If it takes that long something really bad is going on and * we should simply try to bail out and fail as gracefully as possible. */ ret = wait_event_interruptible_timeout(error->reset_queue, - EXIT_COND, + !i915_recovery_pending(error), 10*HZ); if (ret == 0) { DRM_ERROR("Timed out waiting for the gpu reset to complete\n"); @@ -132,7 +215,6 @@ i915_gem_wait_for_error(struct i915_gpu_error *error) } else if (ret < 0) { return ret; } -#undef EXIT_COND return 0; } @@ -150,7 +232,6 @@ int i915_mutex_lock_interruptible(struct drm_device *dev) if (ret) return ret; - WARN_ON(i915_verify_lists(dev)); return 0; } @@ -474,8 +555,6 @@ int i915_gem_obj_prepare_shmem_read(struct drm_i915_gem_object *obj, ret = i915_gem_object_wait_rendering(obj, true); if (ret) return ret; - - i915_gem_object_retire(obj); } ret = i915_gem_object_get_pages(obj); @@ -891,8 +970,6 @@ i915_gem_shmem_pwrite(struct drm_device *dev, ret = i915_gem_object_wait_rendering(obj, false); if (ret) return ret; - - i915_gem_object_retire(obj); } /* Same trick applies to invalidate partially written cachelines read * before writing. */ @@ -1071,229 +1148,6 @@ unlock: return ret; } -int -i915_gem_check_wedge(struct i915_gpu_error *error, - bool interruptible) -{ - if (i915_reset_in_progress(error)) { - /* Non-interruptible callers can't handle -EAGAIN, hence return - * -EIO unconditionally for these. */ - if (!interruptible) - return -EIO; - - /* Recovery complete, but the reset failed ... */ - if (i915_terminally_wedged(error)) - return -EIO; - - return -EAGAIN; - } - - return 0; -} - -/* - * Compare seqno against outstanding lazy request. Emit a request if they are - * equal. - */ -int -i915_gem_check_olr(struct intel_engine_cs *ring, u32 seqno) -{ - int ret; - - BUG_ON(!mutex_is_locked(&ring->dev->struct_mutex)); - - ret = 0; - if (seqno == ring->outstanding_lazy_seqno) - ret = i915_add_request(ring, NULL); - - return ret; -} - -static void fake_irq(unsigned long data) -{ - wake_up_process((struct task_struct *)data); -} - -static bool missed_irq(struct drm_i915_private *dev_priv, - struct intel_engine_cs *ring) -{ - return test_bit(ring->id, &dev_priv->gpu_error.missed_irq_rings); -} - -static bool can_wait_boost(struct drm_i915_file_private *file_priv) -{ - if (file_priv == NULL) - return true; - - return !atomic_xchg(&file_priv->rps_wait_boost, true); -} - -/** - * __wait_seqno - wait until execution of seqno has finished - * @ring: the ring expected to report seqno - * @seqno: duh! - * @reset_counter: reset sequence associated with the given seqno - * @interruptible: do an interruptible wait (normally yes) - * @timeout: in - how long to wait (NULL forever); out - how much time remaining - * - * Note: It is of utmost importance that the passed in seqno and reset_counter - * values have been read by the caller in an smp safe manner. Where read-side - * locks are involved, it is sufficient to read the reset_counter before - * unlocking the lock that protects the seqno. For lockless tricks, the - * reset_counter _must_ be read before, and an appropriate smp_rmb must be - * inserted. - * - * Returns 0 if the seqno was found within the alloted time. Else returns the - * errno with remaining time filled in timeout argument. - */ -static int __wait_seqno(struct intel_engine_cs *ring, u32 seqno, - unsigned reset_counter, - bool interruptible, - s64 *timeout, - struct drm_i915_file_private *file_priv) -{ - struct drm_device *dev = ring->dev; - struct drm_i915_private *dev_priv = dev->dev_private; - const bool irq_test_in_progress = - ACCESS_ONCE(dev_priv->gpu_error.test_irq_rings) & intel_ring_flag(ring); - DEFINE_WAIT(wait); - unsigned long timeout_expire; - s64 before, now; - int ret; - - WARN(!intel_irqs_enabled(dev_priv), "IRQs disabled"); - - if (i915_seqno_passed(ring->get_seqno(ring, true), seqno)) - return 0; - - timeout_expire = timeout ? jiffies + nsecs_to_jiffies((u64)*timeout) : 0; - - if (INTEL_INFO(dev)->gen >= 6 && ring->id == RCS && can_wait_boost(file_priv)) { - gen6_rps_boost(dev_priv); - if (file_priv) - mod_delayed_work(dev_priv->wq, - &file_priv->mm.idle_work, - msecs_to_jiffies(100)); - } - - if (!irq_test_in_progress && WARN_ON(!ring->irq_get(ring))) - return -ENODEV; - - /* Record current time in case interrupted by signal, or wedged */ - trace_i915_gem_request_wait_begin(ring, seqno); - before = ktime_get_raw_ns(); - for (;;) { - struct timer_list timer; - - prepare_to_wait(&ring->irq_queue, &wait, - interruptible ? TASK_INTERRUPTIBLE : TASK_UNINTERRUPTIBLE); - - /* We need to check whether any gpu reset happened in between - * the caller grabbing the seqno and now ... */ - if (reset_counter != atomic_read(&dev_priv->gpu_error.reset_counter)) { - /* ... but upgrade the -EAGAIN to an -EIO if the gpu - * is truely gone. */ - ret = i915_gem_check_wedge(&dev_priv->gpu_error, interruptible); - if (ret == 0) - ret = -EAGAIN; - break; - } - - if (i915_seqno_passed(ring->get_seqno(ring, false), seqno)) { - ret = 0; - break; - } - - if (interruptible && signal_pending(current)) { - ret = -ERESTARTSYS; - break; - } - - if (timeout && time_after_eq(jiffies, timeout_expire)) { - ret = -ETIME; - break; - } - - timer.function = NULL; - if (timeout || missed_irq(dev_priv, ring)) { - unsigned long expire; - - setup_timer_on_stack(&timer, fake_irq, (unsigned long)current); - expire = missed_irq(dev_priv, ring) ? jiffies + 1 : timeout_expire; - mod_timer(&timer, expire); - } - - io_schedule(); - - if (timer.function) { - del_singleshot_timer_sync(&timer); - destroy_timer_on_stack(&timer); - } - } - now = ktime_get_raw_ns(); - trace_i915_gem_request_wait_end(ring, seqno); - - if (!irq_test_in_progress) - ring->irq_put(ring); - - finish_wait(&ring->irq_queue, &wait); - - if (timeout) { - s64 tres = *timeout - (now - before); - - *timeout = tres < 0 ? 0 : tres; - } - - return ret; -} - -/** - * Waits for a sequence number to be signaled, and cleans up the - * request and object lists appropriately for that event. - */ -int -i915_wait_seqno(struct intel_engine_cs *ring, uint32_t seqno) -{ - struct drm_device *dev = ring->dev; - struct drm_i915_private *dev_priv = dev->dev_private; - bool interruptible = dev_priv->mm.interruptible; - int ret; - - BUG_ON(!mutex_is_locked(&dev->struct_mutex)); - BUG_ON(seqno == 0); - - ret = i915_gem_check_wedge(&dev_priv->gpu_error, interruptible); - if (ret) - return ret; - - ret = i915_gem_check_olr(ring, seqno); - if (ret) - return ret; - - return __wait_seqno(ring, seqno, - atomic_read(&dev_priv->gpu_error.reset_counter), - interruptible, NULL, NULL); -} - -static int -i915_gem_object_wait_rendering__tail(struct drm_i915_gem_object *obj, - struct intel_engine_cs *ring) -{ - if (!obj->active) - return 0; - - /* Manually manage the write flush as we may have not yet - * retired the buffer. - * - * Note that the last_write_seqno is always the earlier of - * the two (read/write) seqno, so if we haved successfully waited, - * we know we have passed the last write. - */ - obj->last_write_seqno = 0; - - return 0; -} - /** * Ensures that all rendering to the object has completed and the object is * safe to unbind from the GTT or access from the CPU. @@ -1302,19 +1156,30 @@ static __must_check int i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj, bool readonly) { - struct intel_engine_cs *ring = obj->ring; - u32 seqno; - int ret; + int i, ret; - seqno = readonly ? obj->last_write_seqno : obj->last_read_seqno; - if (seqno == 0) + if (!obj->active) return 0; - ret = i915_wait_seqno(ring, seqno); - if (ret) - return ret; + if (readonly) { + if (obj->last_write.request) { + ret = i915_request_wait(obj->last_write.request); + if (ret) + return ret; + } + } else { + for (i = 0; i < I915_NUM_ENGINES; i++) { + if (obj->last_read[i].request == NULL) + continue; + + ret = i915_request_wait(obj->last_read[i].request); + if (ret) + return ret; + } + } - return i915_gem_object_wait_rendering__tail(obj, ring); + i915_gem_object_retire(obj); + return 0; } /* A nonblocking variant of the above wait. This is a highly dangerous routine @@ -1327,34 +1192,51 @@ i915_gem_object_wait_rendering__nonblocking(struct drm_i915_gem_object *obj, { struct drm_device *dev = obj->base.dev; struct drm_i915_private *dev_priv = dev->dev_private; - struct intel_engine_cs *ring = obj->ring; - unsigned reset_counter; - u32 seqno; - int ret; + struct i915_gem_request *rq[I915_NUM_ENGINES] = {}; + int i, n, ret = 0; BUG_ON(!mutex_is_locked(&dev->struct_mutex)); BUG_ON(!dev_priv->mm.interruptible); - seqno = readonly ? obj->last_write_seqno : obj->last_read_seqno; - if (seqno == 0) + n = 0; + if (readonly) { + if (obj->last_write.request) { + rq[n] = i915_request_get_breadcrumb(obj->last_write.request); + if (IS_ERR(rq[n])) + return PTR_ERR(rq[n]); + n++; + } + } else { + for (i = 0; i < I915_NUM_ENGINES; i++) { + if (obj->last_read[i].request == NULL) + continue; + + rq[n] = i915_request_get_breadcrumb(obj->last_read[i].request); + if (IS_ERR(rq[n])) { + ret = PTR_ERR(rq[n]); + goto out; + } + n++; + } + } + if (n == 0) return 0; - ret = i915_gem_check_wedge(&dev_priv->gpu_error, true); - if (ret) - return ret; + mutex_unlock(&dev->struct_mutex); - ret = i915_gem_check_olr(ring, seqno); - if (ret) - return ret; + for (i = 0; i < n; i++) { + ret = __i915_request_wait(rq[i], true, NULL, file_priv); + if (ret) + break; + } - reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter); - mutex_unlock(&dev->struct_mutex); - ret = __wait_seqno(ring, seqno, reset_counter, true, NULL, file_priv); mutex_lock(&dev->struct_mutex); - if (ret) - return ret; - return i915_gem_object_wait_rendering__tail(obj, ring); +out: + for (i = 0; i < n; i++) + i915_request_put(rq[i]); + + return ret; } /** @@ -1735,7 +1617,11 @@ static int i915_gem_object_create_mmap_offset(struct drm_i915_gem_object *obj) * offsets on purgeable objects by truncating it and marking it purged, * which prevents userspace from ever using that object again. */ - i915_gem_purge(dev_priv, obj->base.size >> PAGE_SHIFT); + i915_gem_shrink(dev_priv, + obj->base.size >> PAGE_SHIFT, + I915_SHRINK_BOUND | + I915_SHRINK_UNBOUND | + I915_SHRINK_PURGEABLE); ret = drm_gem_create_mmap_offset(&obj->base); if (ret != -ENOSPC) goto out; @@ -1932,12 +1818,11 @@ i915_gem_object_put_pages(struct drm_i915_gem_object *obj) return 0; } -static unsigned long -__i915_gem_shrink(struct drm_i915_private *dev_priv, long target, - bool purgeable_only) +unsigned long +i915_gem_shrink(struct drm_i915_private *dev_priv, + long target, unsigned flags) { - struct list_head still_in_list; - struct drm_i915_gem_object *obj; + const bool purgeable_only = flags & I915_SHRINK_PURGEABLE; unsigned long count = 0; /* @@ -1959,62 +1844,68 @@ __i915_gem_shrink(struct drm_i915_private *dev_priv, long target, * dev->struct_mutex and so we won't ever be able to observe an * object on the bound_list with a reference count equals 0. */ - INIT_LIST_HEAD(&still_in_list); - while (count < target && !list_empty(&dev_priv->mm.unbound_list)) { - obj = list_first_entry(&dev_priv->mm.unbound_list, - typeof(*obj), global_list); - list_move_tail(&obj->global_list, &still_in_list); + if (flags & I915_SHRINK_UNBOUND) { + struct list_head still_in_list; - if (!i915_gem_object_is_purgeable(obj) && purgeable_only) - continue; + INIT_LIST_HEAD(&still_in_list); + while (count < target && !list_empty(&dev_priv->mm.unbound_list)) { + struct drm_i915_gem_object *obj; - drm_gem_object_reference(&obj->base); + obj = list_first_entry(&dev_priv->mm.unbound_list, + typeof(*obj), global_list); + list_move_tail(&obj->global_list, &still_in_list); - if (i915_gem_object_put_pages(obj) == 0) - count += obj->base.size >> PAGE_SHIFT; + if (!i915_gem_object_is_purgeable(obj) && purgeable_only) + continue; + + drm_gem_object_reference(&obj->base); + + if (i915_gem_object_put_pages(obj) == 0) + count += obj->base.size >> PAGE_SHIFT; - drm_gem_object_unreference(&obj->base); + drm_gem_object_unreference(&obj->base); + } + list_splice(&still_in_list, &dev_priv->mm.unbound_list); } - list_splice(&still_in_list, &dev_priv->mm.unbound_list); - INIT_LIST_HEAD(&still_in_list); - while (count < target && !list_empty(&dev_priv->mm.bound_list)) { - struct i915_vma *vma, *v; + if (flags & I915_SHRINK_BOUND) { + struct list_head still_in_list; - obj = list_first_entry(&dev_priv->mm.bound_list, - typeof(*obj), global_list); - list_move_tail(&obj->global_list, &still_in_list); + INIT_LIST_HEAD(&still_in_list); + while (count < target && !list_empty(&dev_priv->mm.bound_list)) { + struct drm_i915_gem_object *obj; + struct i915_vma *vma, *v; - if (!i915_gem_object_is_purgeable(obj) && purgeable_only) - continue; + obj = list_first_entry(&dev_priv->mm.bound_list, + typeof(*obj), global_list); + list_move_tail(&obj->global_list, &still_in_list); - drm_gem_object_reference(&obj->base); + if (!i915_gem_object_is_purgeable(obj) && purgeable_only) + continue; - list_for_each_entry_safe(vma, v, &obj->vma_list, vma_link) - if (i915_vma_unbind(vma)) - break; + drm_gem_object_reference(&obj->base); - if (i915_gem_object_put_pages(obj) == 0) - count += obj->base.size >> PAGE_SHIFT; + list_for_each_entry_safe(vma, v, &obj->vma_list, vma_link) + if (i915_vma_unbind(vma)) + break; - drm_gem_object_unreference(&obj->base); + if (i915_gem_object_put_pages(obj) == 0) + count += obj->base.size >> PAGE_SHIFT; + + drm_gem_object_unreference(&obj->base); + } + list_splice(&still_in_list, &dev_priv->mm.bound_list); } - list_splice(&still_in_list, &dev_priv->mm.bound_list); return count; } static unsigned long -i915_gem_purge(struct drm_i915_private *dev_priv, long target) -{ - return __i915_gem_shrink(dev_priv, target, true); -} - -static unsigned long i915_gem_shrink_all(struct drm_i915_private *dev_priv) { i915_gem_evict_everything(dev_priv->dev); - return __i915_gem_shrink(dev_priv, LONG_MAX, false); + return i915_gem_shrink(dev_priv, LONG_MAX, + I915_SHRINK_BOUND | I915_SHRINK_UNBOUND); } static int @@ -2061,7 +1952,11 @@ i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj) for (i = 0; i < page_count; i++) { page = shmem_read_mapping_page_gfp(mapping, i, gfp); if (IS_ERR(page)) { - i915_gem_purge(dev_priv, page_count); + i915_gem_shrink(dev_priv, + page_count, + I915_SHRINK_BOUND | + I915_SHRINK_UNBOUND | + I915_SHRINK_PURGEABLE); page = shmem_read_mapping_page_gfp(mapping, i, gfp); } if (IS_ERR(page)) { @@ -2159,405 +2054,75 @@ i915_gem_object_get_pages(struct drm_i915_gem_object *obj) return 0; } -static void -i915_gem_object_move_to_active(struct drm_i915_gem_object *obj, - struct intel_engine_cs *ring) +int i915_gem_set_seqno(struct drm_device *dev, u32 seqno) { - struct drm_device *dev = obj->base.dev; struct drm_i915_private *dev_priv = dev->dev_private; - u32 seqno = intel_ring_get_seqno(ring); - - BUG_ON(ring == NULL); - if (obj->ring != ring && obj->last_write_seqno) { - /* Keep the seqno relative to the current ring */ - obj->last_write_seqno = seqno; - } - obj->ring = ring; - - /* Add a reference if we're newly entering the active list. */ - if (!obj->active) { - drm_gem_object_reference(&obj->base); - obj->active = 1; - } - - list_move_tail(&obj->ring_list, &ring->active_list); + struct intel_engine_cs *signaller, *waiter; + int ret, i, j; - obj->last_read_seqno = seqno; + if (seqno == 0) + return -EINVAL; - if (obj->fenced_gpu_access) { - obj->last_fenced_seqno = seqno; + if (seqno == dev_priv->next_seqno) + return 0; - /* Bump MRU to take account of the delayed flush */ - if (obj->fence_reg != I915_FENCE_REG_NONE) { - struct drm_i915_fence_reg *reg; + do { + /* Flush the breadcrumbs */ + ret = i915_gpu_idle(dev); + if (ret) + return ret; - reg = &dev_priv->fence_regs[obj->fence_reg]; - list_move_tail(®->lru_list, - &dev_priv->mm.fence_list); - } - } -} + if (!i915_gem_retire_requests(dev)) + return -EIO; -void i915_vma_move_to_active(struct i915_vma *vma, - struct intel_engine_cs *ring) -{ - list_move_tail(&vma->mm_list, &vma->vm->active_list); - return i915_gem_object_move_to_active(vma->obj, ring); -} + /* Update all semaphores to the current value */ + for_each_engine(signaller, to_i915(dev), i) { + struct i915_gem_request *rq; -static void -i915_gem_object_move_to_inactive(struct drm_i915_gem_object *obj) -{ - struct drm_i915_private *dev_priv = obj->base.dev->dev_private; - struct i915_address_space *vm; - struct i915_vma *vma; + if (!signaller->semaphore.signal) + continue; - BUG_ON(obj->base.write_domain & ~I915_GEM_GPU_DOMAINS); - BUG_ON(!obj->active); + rq = intel_engine_alloc_request(signaller, + signaller->default_context); + if (IS_ERR(rq)) + return PTR_ERR(rq); - list_for_each_entry(vm, &dev_priv->vm_list, global_link) { - vma = i915_gem_obj_to_vma(obj, vm); - if (vma && !list_empty(&vma->mm_list)) - list_move_tail(&vma->mm_list, &vm->inactive_list); - } + for_each_engine(waiter, to_i915(dev), j) { + if (signaller == waiter) + continue; - intel_fb_obj_flush(obj, true); + if (!waiter->semaphore.wait) + continue; - list_del_init(&obj->ring_list); - obj->ring = NULL; + ret = i915_request_emit_semaphore(rq, waiter->id); + if (ret) + break; + } - obj->last_read_seqno = 0; - obj->last_write_seqno = 0; - obj->base.write_domain = 0; + if (ret == 0) + ret = i915_request_commit(rq); + i915_request_put(rq); + if (ret) + return ret; + } - obj->last_fenced_seqno = 0; - obj->fenced_gpu_access = false; + /* We can only roll seqno forwards across a wraparound. + * This ship is not for turning! + */ + if (!__i915_seqno_passed(dev_priv->next_seqno, seqno)) + break; - obj->active = 0; - drm_gem_object_unreference(&obj->base); + dev_priv->next_seqno += 0x40000000; + }while (1); - WARN_ON(i915_verify_lists(dev)); + dev_priv->next_seqno = seqno; + return 0; } -static void -i915_gem_object_retire(struct drm_i915_gem_object *obj) +void i915_gem_restore_fences(struct drm_device *dev) { - struct intel_engine_cs *ring = obj->ring; - - if (ring == NULL) - return; - - if (i915_seqno_passed(ring->get_seqno(ring, true), - obj->last_read_seqno)) - i915_gem_object_move_to_inactive(obj); -} - -static int -i915_gem_init_seqno(struct drm_device *dev, u32 seqno) -{ - struct drm_i915_private *dev_priv = dev->dev_private; - struct intel_engine_cs *ring; - int ret, i, j; - - /* Carefully retire all requests without writing to the rings */ - for_each_ring(ring, dev_priv, i) { - ret = intel_ring_idle(ring); - if (ret) - return ret; - } - i915_gem_retire_requests(dev); - - /* Finally reset hw state */ - for_each_ring(ring, dev_priv, i) { - intel_ring_init_seqno(ring, seqno); - - for (j = 0; j < ARRAY_SIZE(ring->semaphore.sync_seqno); j++) - ring->semaphore.sync_seqno[j] = 0; - } - - return 0; -} - -int i915_gem_set_seqno(struct drm_device *dev, u32 seqno) -{ - struct drm_i915_private *dev_priv = dev->dev_private; - int ret; - - if (seqno == 0) - return -EINVAL; - - /* HWS page needs to be set less than what we - * will inject to ring - */ - ret = i915_gem_init_seqno(dev, seqno - 1); - if (ret) - return ret; - - /* Carefully set the last_seqno value so that wrap - * detection still works - */ - dev_priv->next_seqno = seqno; - dev_priv->last_seqno = seqno - 1; - if (dev_priv->last_seqno == 0) - dev_priv->last_seqno--; - - return 0; -} - -int -i915_gem_get_seqno(struct drm_device *dev, u32 *seqno) -{ - struct drm_i915_private *dev_priv = dev->dev_private; - - /* reserve 0 for non-seqno */ - if (dev_priv->next_seqno == 0) { - int ret = i915_gem_init_seqno(dev, 0); - if (ret) - return ret; - - dev_priv->next_seqno = 1; - } - - *seqno = dev_priv->last_seqno = dev_priv->next_seqno++; - return 0; -} - -int __i915_add_request(struct intel_engine_cs *ring, - struct drm_file *file, - struct drm_i915_gem_object *obj, - u32 *out_seqno) -{ - struct drm_i915_private *dev_priv = ring->dev->dev_private; - struct drm_i915_gem_request *request; - u32 request_ring_position, request_start; - int ret; - - request_start = intel_ring_get_tail(ring->buffer); - /* - * Emit any outstanding flushes - execbuf can fail to emit the flush - * after having emitted the batchbuffer command. Hence we need to fix - * things up similar to emitting the lazy request. The difference here - * is that the flush _must_ happen before the next request, no matter - * what. - */ - ret = intel_ring_flush_all_caches(ring); - if (ret) - return ret; - - request = ring->preallocated_lazy_request; - if (WARN_ON(request == NULL)) - return -ENOMEM; - - /* Record the position of the start of the request so that - * should we detect the updated seqno part-way through the - * GPU processing the request, we never over-estimate the - * position of the head. - */ - request_ring_position = intel_ring_get_tail(ring->buffer); - - ret = ring->add_request(ring); - if (ret) - return ret; - - request->seqno = intel_ring_get_seqno(ring); - request->ring = ring; - request->head = request_start; - request->tail = request_ring_position; - - /* Whilst this request exists, batch_obj will be on the - * active_list, and so will hold the active reference. Only when this - * request is retired will the the batch_obj be moved onto the - * inactive_list and lose its active reference. Hence we do not need - * to explicitly hold another reference here. - */ - request->batch_obj = obj; - - /* Hold a reference to the current context so that we can inspect - * it later in case a hangcheck error event fires. - */ - request->ctx = ring->last_context; - if (request->ctx) - i915_gem_context_reference(request->ctx); - - request->emitted_jiffies = jiffies; - list_add_tail(&request->list, &ring->request_list); - request->file_priv = NULL; - - if (file) { - struct drm_i915_file_private *file_priv = file->driver_priv; - - spin_lock(&file_priv->mm.lock); - request->file_priv = file_priv; - list_add_tail(&request->client_list, - &file_priv->mm.request_list); - spin_unlock(&file_priv->mm.lock); - } - - trace_i915_gem_request_add(ring, request->seqno); - ring->outstanding_lazy_seqno = 0; - ring->preallocated_lazy_request = NULL; - - if (!dev_priv->ums.mm_suspended) { - i915_queue_hangcheck(ring->dev); - - cancel_delayed_work_sync(&dev_priv->mm.idle_work); - queue_delayed_work(dev_priv->wq, - &dev_priv->mm.retire_work, - round_jiffies_up_relative(HZ)); - intel_mark_busy(dev_priv->dev); - } - - if (out_seqno) - *out_seqno = request->seqno; - return 0; -} - -static inline void -i915_gem_request_remove_from_client(struct drm_i915_gem_request *request) -{ - struct drm_i915_file_private *file_priv = request->file_priv; - - if (!file_priv) - return; - - spin_lock(&file_priv->mm.lock); - list_del(&request->client_list); - request->file_priv = NULL; - spin_unlock(&file_priv->mm.lock); -} - -static bool i915_context_is_banned(struct drm_i915_private *dev_priv, - const struct intel_context *ctx) -{ - unsigned long elapsed; - - elapsed = get_seconds() - ctx->hang_stats.guilty_ts; - - if (ctx->hang_stats.banned) - return true; - - if (elapsed <= DRM_I915_CTX_BAN_PERIOD) { - if (!i915_gem_context_is_default(ctx)) { - DRM_DEBUG("context hanging too fast, banning!\n"); - return true; - } else if (i915_stop_ring_allow_ban(dev_priv)) { - if (i915_stop_ring_allow_warn(dev_priv)) - DRM_ERROR("gpu hanging too fast, banning!\n"); - return true; - } - } - - return false; -} - -static void i915_set_reset_status(struct drm_i915_private *dev_priv, - struct intel_context *ctx, - const bool guilty) -{ - struct i915_ctx_hang_stats *hs; - - if (WARN_ON(!ctx)) - return; - - hs = &ctx->hang_stats; - - if (guilty) { - hs->banned = i915_context_is_banned(dev_priv, ctx); - hs->batch_active++; - hs->guilty_ts = get_seconds(); - } else { - hs->batch_pending++; - } -} - -static void i915_gem_free_request(struct drm_i915_gem_request *request) -{ - list_del(&request->list); - i915_gem_request_remove_from_client(request); - - if (request->ctx) - i915_gem_context_unreference(request->ctx); - - kfree(request); -} - -struct drm_i915_gem_request * -i915_gem_find_active_request(struct intel_engine_cs *ring) -{ - struct drm_i915_gem_request *request; - u32 completed_seqno; - - completed_seqno = ring->get_seqno(ring, false); - - list_for_each_entry(request, &ring->request_list, list) { - if (i915_seqno_passed(completed_seqno, request->seqno)) - continue; - - return request; - } - - return NULL; -} - -static void i915_gem_reset_ring_status(struct drm_i915_private *dev_priv, - struct intel_engine_cs *ring) -{ - struct drm_i915_gem_request *request; - bool ring_hung; - - request = i915_gem_find_active_request(ring); - - if (request == NULL) - return; - - ring_hung = ring->hangcheck.score >= HANGCHECK_SCORE_RING_HUNG; - - i915_set_reset_status(dev_priv, request->ctx, ring_hung); - - list_for_each_entry_continue(request, &ring->request_list, list) - i915_set_reset_status(dev_priv, request->ctx, false); -} - -static void i915_gem_reset_ring_cleanup(struct drm_i915_private *dev_priv, - struct intel_engine_cs *ring) -{ - while (!list_empty(&ring->active_list)) { - struct drm_i915_gem_object *obj; - - obj = list_first_entry(&ring->active_list, - struct drm_i915_gem_object, - ring_list); - - i915_gem_object_move_to_inactive(obj); - } - - /* - * We must free the requests after all the corresponding objects have - * been moved off active lists. Which is the same order as the normal - * retire_requests function does. This is important if object hold - * implicit references on things like e.g. ppgtt address spaces through - * the request. - */ - while (!list_empty(&ring->request_list)) { - struct drm_i915_gem_request *request; - - request = list_first_entry(&ring->request_list, - struct drm_i915_gem_request, - list); - - i915_gem_free_request(request); - } - - /* These may not have been flush before the reset, do so now */ - kfree(ring->preallocated_lazy_request); - ring->preallocated_lazy_request = NULL; - ring->outstanding_lazy_seqno = 0; -} - -void i915_gem_restore_fences(struct drm_device *dev) -{ - struct drm_i915_private *dev_priv = dev->dev_private; - int i; + struct drm_i915_private *dev_priv = dev->dev_private; + int i; for (i = 0; i < dev_priv->num_fence_regs; i++) { struct drm_i915_fence_reg *reg = &dev_priv->fence_regs[i]; @@ -2578,21 +2143,25 @@ void i915_gem_restore_fences(struct drm_device *dev) void i915_gem_reset(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; - struct intel_engine_cs *ring; + struct intel_engine_cs *engine; int i; - /* - * Before we free the objects from the requests, we need to inspect - * them for finding the guilty party. As the requests only borrow - * their reference to the objects, the inspection must be done first. - */ - for_each_ring(ring, dev_priv, i) - i915_gem_reset_ring_status(dev_priv, ring); + for_each_engine(engine, dev_priv, i) { + /* Clearing the read list will also clear the write + * and fence lists, 3 birds with one stone. + */ + while (!list_empty(&engine->read_list)) { + struct drm_i915_gem_object *obj; - for_each_ring(ring, dev_priv, i) - i915_gem_reset_ring_cleanup(dev_priv, ring); + obj = list_first_entry(&engine->read_list, + struct drm_i915_gem_object, + last_read[i].engine_list); - i915_gem_context_reset(dev); + i915_gem_object_retire__read(obj, engine); + } + + intel_engine_reset(engine); + } i915_gem_restore_fences(dev); } @@ -2601,82 +2170,71 @@ void i915_gem_reset(struct drm_device *dev) * This function clears the request list as sequence numbers are passed. */ void -i915_gem_retire_requests_ring(struct intel_engine_cs *ring) +i915_gem_retire_requests__engine(struct intel_engine_cs *engine) { - uint32_t seqno; - - if (list_empty(&ring->request_list)) + if (engine->last_request == NULL) return; - WARN_ON(i915_verify_lists(ring->dev)); - - seqno = ring->get_seqno(ring, true); + if (!intel_engine_retire(engine, engine->get_seqno(engine))) + return; - /* Move any buffers on the active list that are no longer referenced - * by the ringbuffer to the flushing/inactive lists as appropriate, - * before we free the context associated with the requests. - */ - while (!list_empty(&ring->active_list)) { + while (!list_empty(&engine->read_list)) { struct drm_i915_gem_object *obj; - obj = list_first_entry(&ring->active_list, - struct drm_i915_gem_object, - ring_list); + obj = list_first_entry(&engine->read_list, + struct drm_i915_gem_object, + last_read[engine->id].engine_list); - if (!i915_seqno_passed(seqno, obj->last_read_seqno)) + if (!obj->last_read[engine->id].request->completed) break; - i915_gem_object_move_to_inactive(obj); + i915_gem_object_retire__read(obj, engine); } + while (!list_empty(&engine->fence_list)) { + struct drm_i915_gem_object *obj; - while (!list_empty(&ring->request_list)) { - struct drm_i915_gem_request *request; - - request = list_first_entry(&ring->request_list, - struct drm_i915_gem_request, - list); + obj = list_first_entry(&engine->fence_list, + struct drm_i915_gem_object, + last_fence.engine_list); - if (!i915_seqno_passed(seqno, request->seqno)) + if (!obj->last_fence.request->completed) break; - trace_i915_gem_request_retire(ring, request->seqno); - /* We know the GPU must have read the request to have - * sent us the seqno + interrupt, so use the position - * of tail of the request to update the last known position - * of the GPU head. - */ - ring->buffer->last_retired_head = request->tail; - - i915_gem_free_request(request); + i915_gem_object_retire__fence(obj); } - if (unlikely(ring->trace_irq_seqno && - i915_seqno_passed(seqno, ring->trace_irq_seqno))) { - ring->irq_put(ring); - ring->trace_irq_seqno = 0; - } + while (!list_empty(&engine->write_list)) { + struct drm_i915_gem_object *obj; - WARN_ON(i915_verify_lists(ring->dev)); + obj = list_first_entry(&engine->write_list, + struct drm_i915_gem_object, + last_write.engine_list); + + if (!obj->last_write.request->completed) + break; + + i915_gem_object_retire__write(obj); + } } bool i915_gem_retire_requests(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; - struct intel_engine_cs *ring; + struct intel_engine_cs *engine; bool idle = true; int i; - for_each_ring(ring, dev_priv, i) { - i915_gem_retire_requests_ring(ring); - idle &= list_empty(&ring->request_list); + for_each_engine(engine, dev_priv, i) { + i915_gem_retire_requests__engine(engine); + idle &= engine->last_request == NULL; } if (idle) mod_delayed_work(dev_priv->wq, - &dev_priv->mm.idle_work, - msecs_to_jiffies(100)); + &dev_priv->mm.idle_work, + msecs_to_jiffies(100)); return idle; } @@ -2696,7 +2254,8 @@ i915_gem_retire_work_handler(struct work_struct *work) mutex_unlock(&dev->struct_mutex); } if (!idle) - queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, + queue_delayed_work(dev_priv->wq, + &dev_priv->mm.retire_work, round_jiffies_up_relative(HZ)); } @@ -2719,14 +2278,16 @@ i915_gem_object_flush_active(struct drm_i915_gem_object *obj) { int ret; - if (obj->active) { - ret = i915_gem_check_olr(obj->ring, obj->last_read_seqno); + if (!obj->active) + return 0; + + if (obj->last_write.request) { + ret = i915_request_emit_breadcrumb(obj->last_write.request); if (ret) return ret; - - i915_gem_retire_requests_ring(obj->ring); } + i915_gem_object_retire(obj); return 0; } @@ -2755,13 +2316,10 @@ i915_gem_object_flush_active(struct drm_i915_gem_object *obj) int i915_gem_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file) { - struct drm_i915_private *dev_priv = dev->dev_private; struct drm_i915_gem_wait *args = data; struct drm_i915_gem_object *obj; - struct intel_engine_cs *ring = NULL; - unsigned reset_counter; - u32 seqno = 0; - int ret = 0; + struct i915_gem_request *rq[I915_NUM_ENGINES] = {}; + int i, n, ret = 0; ret = i915_mutex_lock_interruptible(dev); if (ret) @@ -2778,13 +2336,8 @@ i915_gem_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file) if (ret) goto out; - if (obj->active) { - seqno = obj->last_read_seqno; - ring = obj->ring; - } - - if (seqno == 0) - goto out; + if (!obj->active) + goto out; /* Do this after OLR check to make sure we make forward progress polling * on this IOCTL with a timeout <=0 (like busy ioctl) @@ -2794,12 +2347,31 @@ i915_gem_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file) goto out; } + for (i = n = 0; i < I915_NUM_ENGINES; i++) { + if (obj->last_read[i].request == NULL) + continue; + + rq[n] = i915_request_get_breadcrumb(obj->last_read[i].request); + if (IS_ERR(rq[n])) { + ret = PTR_ERR(rq[n]); + break; + } + n++; + } + drm_gem_object_unreference(&obj->base); - reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter); mutex_unlock(&dev->struct_mutex); - return __wait_seqno(ring, seqno, reset_counter, true, &args->timeout_ns, - file->driver_priv); + for (i = 0; i < n; i++) { + if (ret == 0) + ret = __i915_request_wait(rq[i], true, + &args->timeout_ns, + file->driver_priv); + + i915_request_put__unlocked(rq[i]); + } + + return ret; out: drm_gem_object_unreference(&obj->base); @@ -2807,6 +2379,49 @@ out: return ret; } +static int +__i915_request_sync(struct i915_gem_request *waiter, + struct i915_gem_request *signaller, + bool *retire) +{ + int ret; + + if (signaller == NULL || i915_request_complete(signaller)) + return 0; + + if (waiter == NULL) + goto wait; + + /* XXX still true with execlists? */ + if (waiter->engine == signaller->engine) + return 0; + + if (!waiter->engine->semaphore.wait) + goto wait; + + /* Try to emit only one wait per request per ring */ + if (waiter->semaphore[signaller->engine->id] && + __i915_seqno_passed(waiter->semaphore[signaller->engine->id], + signaller->seqno)) + return 0; + + ret = i915_request_emit_semaphore(signaller, waiter->engine->id); + if (ret) + goto wait; + + trace_i915_gem_ring_wait(signaller, waiter); + if (waiter->engine->semaphore.wait(waiter, signaller)) + goto wait; + + waiter->pending_flush &= ~I915_COMMAND_BARRIER; + waiter->semaphore[signaller->engine->id] = signaller->breadcrumb[waiter->engine->id]; + return 0; + +wait: + *retire = true; + return i915_request_wait(signaller); +} + /** * i915_gem_object_sync - sync an object to a ring. * @@ -2821,38 +2436,23 @@ out: */ int i915_gem_object_sync(struct drm_i915_gem_object *obj, - struct intel_engine_cs *to) + struct i915_gem_request *rq) { - struct intel_engine_cs *from = obj->ring; - u32 seqno; - int ret, idx; - - if (from == NULL || to == from) - return 0; - - if (to == NULL || !i915_semaphore_is_enabled(obj->base.dev)) - return i915_gem_object_wait_rendering(obj, false); - - idx = intel_ring_sync_index(from, to); - - seqno = obj->last_read_seqno; - /* Optimization: Avoid semaphore sync when we are sure we already - * waited for an object with higher seqno */ - if (seqno <= from->semaphore.sync_seqno[idx]) - return 0; + int ret = 0, i; + bool retire = false; - ret = i915_gem_check_olr(obj->ring, seqno); - if (ret) - return ret; + if (obj->base.pending_write_domain == 0) { + ret = __i915_request_sync(rq, obj->last_write.request, &retire); + } else { + for (i = 0; i < I915_NUM_ENGINES; i++) { + ret = __i915_request_sync(rq, obj->last_read[i].request, &retire); + if (ret) + break; + } + } - trace_i915_gem_ring_sync_to(from, to, seqno); - ret = to->semaphore.sync_to(to, from, seqno); - if (!ret) - /* We use last_read_seqno because sync_to() - * might have just caused seqno wrap under - * the radar. - */ - from->semaphore.sync_seqno[idx] = obj->last_read_seqno; + if (retire) + i915_gem_object_retire(obj); return ret; } @@ -2922,9 +2522,8 @@ int i915_vma_unbind(struct i915_vma *vma) vma->unbind_vma(vma); list_del_init(&vma->mm_list); - /* Avoid an unnecessary call to unbind on rebind. */ if (i915_is_ggtt(vma->vm)) - obj->map_and_fenceable = true; + obj->map_and_fenceable = false; drm_mm_remove_node(&vma->node); i915_gem_vma_destroy(vma); @@ -2947,17 +2546,22 @@ int i915_vma_unbind(struct i915_vma *vma) int i915_gpu_idle(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; - struct intel_engine_cs *ring; - int ret, i; + struct intel_engine_cs *engine; + int i; - /* Flush everything onto the inactive list. */ - for_each_ring(ring, dev_priv, i) { - ret = i915_switch_context(ring, ring->default_context); - if (ret) - return ret; + /* Flush everything including contexts onto the inactive list. */ + for_each_engine(engine, to_i915(dev), i) { + struct i915_gem_request *rq; + int ret; + + rq = intel_engine_alloc_request(engine, + engine->default_context); + if (IS_ERR(rq)) + return PTR_ERR(rq); + + ret = i915_request_wait(rq); + i915_request_put(rq); - ret = intel_ring_idle(ring); if (ret) return ret; } @@ -3161,15 +2765,16 @@ static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj, static int i915_gem_object_wait_fence(struct drm_i915_gem_object *obj) { - if (obj->last_fenced_seqno) { - int ret = i915_wait_seqno(obj->ring, obj->last_fenced_seqno); - if (ret) - return ret; + int ret; - obj->last_fenced_seqno = 0; - } + if (obj->last_fence.request == NULL) + return 0; - obj->fenced_gpu_access = false; + ret = i915_request_wait(obj->last_fence.request); + if (ret) + return ret; + + i915_gem_object_retire__fence(obj); return 0; } @@ -3276,6 +2881,9 @@ i915_gem_object_get_fence(struct drm_i915_gem_object *obj) return 0; } } else if (enable) { + if (WARN_ON(!obj->map_and_fenceable)) + return -EINVAL; + reg = i915_find_fence_reg(dev); if (IS_ERR(reg)) return PTR_ERR(reg); @@ -3297,17 +2905,20 @@ i915_gem_object_get_fence(struct drm_i915_gem_object *obj) return 0; } -static bool i915_gem_valid_gtt_space(struct drm_device *dev, - struct drm_mm_node *gtt_space, +static bool i915_gem_valid_gtt_space(struct i915_vma *vma, unsigned long cache_level) { + struct drm_mm_node *gtt_space = &vma->node; struct drm_mm_node *other; - /* On non-LLC machines we have to be careful when putting differing - * types of snoopable memory together to avoid the prefetcher - * crossing memory domains and dying. + /* + * On some machines we have to be careful when putting differing types + * of snoopable memory together to avoid the prefetcher crossing memory + * domains and dying. During vm initialisation, we decide whether or not + * these constraints apply and set the drm_mm.color_adjust + * appropriately. */ - if (HAS_LLC(dev)) + if (vma->vm->mm.color_adjust == NULL) return true; if (!drm_mm_node_allocated(gtt_space)) @@ -3327,46 +2938,6 @@ static bool i915_gem_valid_gtt_space(struct drm_device *dev, return true; } -static void i915_gem_verify_gtt(struct drm_device *dev) -{ -#if WATCH_GTT - struct drm_i915_private *dev_priv = dev->dev_private; - struct drm_i915_gem_object *obj; - int err = 0; - - list_for_each_entry(obj, &dev_priv->mm.gtt_list, global_list) { - if (obj->gtt_space == NULL) { - printk(KERN_ERR "object found on GTT list with no space reserved\n"); - err++; - continue; - } - - if (obj->cache_level != obj->gtt_space->color) { - printk(KERN_ERR "object reserved space [%08lx, %08lx] with wrong color, cache_level=%x, color=%lx\n", - i915_gem_obj_ggtt_offset(obj), - i915_gem_obj_ggtt_offset(obj) + i915_gem_obj_ggtt_size(obj), - obj->cache_level, - obj->gtt_space->color); - err++; - continue; - } - - if (!i915_gem_valid_gtt_space(dev, - obj->gtt_space, - obj->cache_level)) { - printk(KERN_ERR "invalid GTT space found at [%08lx, %08lx] - color=%x\n", - i915_gem_obj_ggtt_offset(obj), - i915_gem_obj_ggtt_offset(obj) + i915_gem_obj_ggtt_size(obj), - obj->cache_level); - err++; - continue; - } - } - - WARN_ON(err); -#endif -} - /** * Finds free space in the GTT aperture and binds the object there. */ @@ -3445,8 +3016,7 @@ search_free: goto err_free_vma; } - if (WARN_ON(!i915_gem_valid_gtt_space(dev, &vma->node, - obj->cache_level))) { + if (WARN_ON(!i915_gem_valid_gtt_space(vma, obj->cache_level))) { ret = -EINVAL; goto err_remove_node; } @@ -3476,7 +3046,6 @@ search_free: vma->bind_vma(vma, obj->cache_level, flags & (PIN_MAPPABLE | PIN_GLOBAL) ? GLOBAL_BIND : 0); - i915_gem_verify_gtt(dev); return vma; err_remove_node: @@ -3586,11 +3155,12 @@ int i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write) { struct drm_i915_private *dev_priv = obj->base.dev->dev_private; + struct i915_vma *vma = i915_gem_obj_to_ggtt(obj); uint32_t old_write_domain, old_read_domains; int ret; /* Not valid to be called on unbound objects. */ - if (!i915_gem_obj_bound_any(obj)) + if (vma == NULL) return -EINVAL; if (obj->base.write_domain == I915_GEM_DOMAIN_GTT) @@ -3600,7 +3170,6 @@ i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write) if (ret) return ret; - i915_gem_object_retire(obj); i915_gem_object_flush_cpu_write_domain(obj, false); /* Serialise direct access to this object with the barriers for @@ -3619,26 +3188,20 @@ i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write) BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) != 0); obj->base.read_domains |= I915_GEM_DOMAIN_GTT; if (write) { + intel_fb_obj_invalidate(obj, NULL); obj->base.read_domains = I915_GEM_DOMAIN_GTT; obj->base.write_domain = I915_GEM_DOMAIN_GTT; obj->dirty = 1; } - if (write) - intel_fb_obj_invalidate(obj, NULL); - trace_i915_gem_object_change_domain(obj, old_read_domains, old_write_domain); /* And bump the LRU for this access */ - if (i915_gem_object_is_inactive(obj)) { - struct i915_vma *vma = i915_gem_obj_to_ggtt(obj); - if (vma) - list_move_tail(&vma->mm_list, - &dev_priv->gtt.base.inactive_list); - - } + if (i915_gem_object_is_inactive(obj)) + list_move_tail(&vma->mm_list, + &dev_priv->gtt.base.inactive_list); return 0; } @@ -3659,7 +3222,7 @@ int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj, } list_for_each_entry_safe(vma, next, &obj->vma_list, vma_link) { - if (!i915_gem_valid_gtt_space(dev, &vma->node, cache_level)) { + if (!i915_gem_valid_gtt_space(vma, cache_level)) { ret = i915_vma_unbind(vma); if (ret) return ret; @@ -3702,7 +3265,6 @@ int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj, * in obj->write_domain and have been skipping the clflushes. * Just set it to the CPU cache for now. */ - i915_gem_object_retire(obj); WARN_ON(obj->base.write_domain & ~I915_GEM_DOMAIN_CPU); old_read_domains = obj->base.read_domains; @@ -3716,7 +3278,6 @@ int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj, old_write_domain); } - i915_gem_verify_gtt(dev); return 0; } @@ -3802,9 +3363,6 @@ static bool is_pin_display(struct drm_i915_gem_object *obj) { struct i915_vma *vma; - if (list_empty(&obj->vma_list)) - return false; - vma = i915_gem_obj_to_ggtt(obj); if (!vma) return false; @@ -3831,17 +3389,15 @@ static bool is_pin_display(struct drm_i915_gem_object *obj) int i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj, u32 alignment, - struct intel_engine_cs *pipelined) + struct i915_gem_request *pipelined) { u32 old_read_domains, old_write_domain; bool was_pin_display; int ret; - if (pipelined != obj->ring) { - ret = i915_gem_object_sync(obj, pipelined); - if (ret) - return ret; - } + ret = i915_gem_object_sync(obj, pipelined); + if (ret) + return ret; /* Mark the pin_display early so that we account for the * display coherency whilst setting up the cache domains. @@ -3937,7 +3493,6 @@ i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write) if (ret) return ret; - i915_gem_object_retire(obj); i915_gem_object_flush_gtt_write_domain(obj); old_write_domain = obj->base.write_domain; @@ -3959,13 +3514,11 @@ i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write) * need to be invalidated at next use. */ if (write) { + intel_fb_obj_invalidate(obj, NULL); obj->base.read_domains = I915_GEM_DOMAIN_CPU; obj->base.write_domain = I915_GEM_DOMAIN_CPU; } - if (write) - intel_fb_obj_invalidate(obj, NULL); - trace_i915_gem_object_change_domain(obj, old_read_domains, old_write_domain); @@ -3973,57 +3526,6 @@ i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write) return 0; } -/* Throttle our rendering by waiting until the ring has completed our requests - * emitted over 20 msec ago. - * - * Note that if we were to use the current jiffies each time around the loop, - * we wouldn't escape the function with any frames outstanding if the time to - * render a frame was over 20ms. - * - * This should get us reasonable parallelism between CPU and GPU but also - * relatively low latency when blocking on a particular request to finish. - */ -static int -i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file) -{ - struct drm_i915_private *dev_priv = dev->dev_private; - struct drm_i915_file_private *file_priv = file->driver_priv; - unsigned long recent_enough = jiffies - msecs_to_jiffies(20); - struct drm_i915_gem_request *request; - struct intel_engine_cs *ring = NULL; - unsigned reset_counter; - u32 seqno = 0; - int ret; - - ret = i915_gem_wait_for_error(&dev_priv->gpu_error); - if (ret) - return ret; - - ret = i915_gem_check_wedge(&dev_priv->gpu_error, false); - if (ret) - return ret; - - spin_lock(&file_priv->mm.lock); - list_for_each_entry(request, &file_priv->mm.request_list, client_list) { - if (time_after_eq(request->emitted_jiffies, recent_enough)) - break; - - ring = request->ring; - seqno = request->seqno; - } - reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter); - spin_unlock(&file_priv->mm.lock); - - if (seqno == 0) - return 0; - - ret = __wait_seqno(ring, seqno, reset_counter, true, NULL, NULL); - if (ret == 0) - queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 0); - - return ret; -} - static bool i915_vma_misplaced(struct i915_vma *vma, uint32_t alignment, uint64_t flags) { @@ -4234,7 +3736,7 @@ i915_gem_busy_ioctl(struct drm_device *dev, void *data, { struct drm_i915_gem_busy *args = data; struct drm_i915_gem_object *obj; - int ret; + int ret, i; ret = i915_mutex_lock_interruptible(dev); if (ret) @@ -4253,10 +3755,16 @@ i915_gem_busy_ioctl(struct drm_device *dev, void *data, */ ret = i915_gem_object_flush_active(obj); - args->busy = obj->active; - if (obj->ring) { - BUILD_BUG_ON(I915_NUM_RINGS > 16); - args->busy |= intel_ring_flag(obj->ring) << 16; + args->busy = 0; + if (obj->active) { + BUILD_BUG_ON(I915_NUM_ENGINES > 16); + args->busy |= 1; + for (i = 0; i < I915_NUM_ENGINES; i++) { + if (obj->last_read[i].request == NULL) + continue; + + args->busy |= 1 << (16 + i); + } } drm_gem_object_unreference(&obj->base); @@ -4265,11 +3773,58 @@ unlock: return ret; } +/* Throttle our rendering by waiting until the ring has completed our requests + * emitted over 20 msec ago. + * + * Note that if we were to use the current jiffies each time around the loop, + * we wouldn't escape the function with any frames outstanding if the time to + * render a frame was over 20ms. + * + * This should get us reasonable parallelism between CPU and GPU but also + * relatively low latency when blocking on a particular request to finish. + */ int i915_gem_throttle_ioctl(struct drm_device *dev, void *data, - struct drm_file *file_priv) + struct drm_file *file) { - return i915_gem_ring_throttle(dev, file_priv); + struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_file_private *file_priv = file->driver_priv; + unsigned long recent_enough = jiffies - msecs_to_jiffies(20); + struct i915_gem_request *rq, *tmp; + int ret; + + ret = i915_gem_wait_for_error(&dev_priv->gpu_error); + if (ret) + return ret; + + /* used for querying whethering the GPU is wedged by legacy userspace */ + if (i915_terminally_wedged(&dev_priv->gpu_error)) + return -EIO; + + spin_lock(&file_priv->mm.lock); + rq = NULL; + list_for_each_entry(tmp, &file_priv->mm.request_list, client_list) { + if (time_after_eq(tmp->emitted_jiffies, recent_enough)) + break; + rq = tmp; + } + rq = i915_request_get(rq); + spin_unlock(&file_priv->mm.lock); + + if (rq != NULL) { + if (rq->breadcrumb[rq->engine->id] == 0) { + ret = i915_mutex_lock_interruptible(dev); + if (ret == 0) { + ret = i915_request_emit_breadcrumb(rq); + mutex_unlock(&dev->struct_mutex); + } + } + if (ret == 0) + ret = __i915_request_wait(rq, true, NULL, NULL); + i915_request_put__unlocked(rq); + } + + return ret; } int @@ -4322,8 +3877,13 @@ unlock: void i915_gem_object_init(struct drm_i915_gem_object *obj, const struct drm_i915_gem_object_ops *ops) { + int i; + INIT_LIST_HEAD(&obj->global_list); - INIT_LIST_HEAD(&obj->ring_list); + INIT_LIST_HEAD(&obj->last_fence.engine_list); + INIT_LIST_HEAD(&obj->last_write.engine_list); + for (i = 0; i < I915_NUM_ENGINES; i++) + INIT_LIST_HEAD(&obj->last_read[i].engine_list); INIT_LIST_HEAD(&obj->obj_exec_link); INIT_LIST_HEAD(&obj->vma_list); @@ -4331,8 +3891,6 @@ void i915_gem_object_init(struct drm_i915_gem_object *obj, obj->fence_reg = I915_FENCE_REG_NONE; obj->madv = I915_MADV_WILLNEED; - /* Avoid an unnecessary call to unbind on the first bind. */ - obj->map_and_fenceable = true; i915_gem_info_add_obj(obj->base.dev->dev_private, obj->base.size); } @@ -4493,26 +4051,156 @@ struct i915_vma *i915_gem_obj_to_vma(struct drm_i915_gem_object *obj, void i915_gem_vma_destroy(struct i915_vma *vma) { + struct i915_address_space *vm = NULL; WARN_ON(vma->node.allocated); /* Keep the vma as a placeholder in the execbuffer reservation lists */ if (!list_empty(&vma->exec_list)) return; + vm = vma->vm; + + if (!i915_is_ggtt(vm)) + i915_ppgtt_put(i915_vm_to_ppgtt(vm)); + list_del(&vma->vma_link); kfree(vma); } static void -i915_gem_stop_ringbuffers(struct drm_device *dev) +i915_gem_cleanup_engines(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; - struct intel_engine_cs *ring; int i; - for_each_ring(ring, dev_priv, i) - intel_stop_ring_buffer(ring); + /* Not the regular for_each_engine so we can cleanup a failed setup */ + for (i =0; i < I915_NUM_ENGINES; i++) { + struct intel_engine_cs *engine = &to_i915(dev)->engine[i]; + + if (engine->i915 == NULL) + continue; + + intel_engine_cleanup(engine); + } +} + +static int +i915_gem_resume_engines(struct drm_device *dev) +{ + struct intel_engine_cs *engine; + int i, ret; + + for_each_engine(engine, to_i915(dev), i) { + ret = intel_engine_resume(engine); + if (ret) + return ret; + } + + return 0; +} + +static int +i915_gem_suspend_engines(struct drm_device *dev) +{ + struct intel_engine_cs *engine; + int i, ret; + + for_each_engine(engine, to_i915(dev), i) { + ret = intel_engine_suspend(engine); + if (ret) + return ret; + } + + return 0; +} + +static bool +intel_enable_blt(struct drm_i915_private *dev_priv) +{ + if (!HAS_BLT(dev_priv)) + return false; + + /* The blitter was dysfunctional on early prototypes */ + if (IS_GEN6(dev_priv) && dev_priv->dev->pdev->revision < 8) { + DRM_INFO("BLT not supported on this pre-production hardware;" + " graphics performance will be degraded.\n"); + return false; + } + + return true; +} + +static void stop_unused_ring(struct drm_i915_private *dev_priv, u32 base) +{ + I915_WRITE(RING_CTL(base), 0); + I915_WRITE(RING_HEAD(base), 0); + I915_WRITE(RING_TAIL(base), 0); + I915_WRITE(RING_START(base), 0); +} + +static void stop_unused_rings(struct drm_i915_private *dev_priv) +{ + if (IS_I830(dev_priv)) { + stop_unused_ring(dev_priv, PRB1_BASE); + stop_unused_ring(dev_priv, SRB0_BASE); + stop_unused_ring(dev_priv, SRB1_BASE); + stop_unused_ring(dev_priv, SRB2_BASE); + stop_unused_ring(dev_priv, SRB3_BASE); + } else if (IS_GEN2(dev_priv)) { + stop_unused_ring(dev_priv, SRB0_BASE); + stop_unused_ring(dev_priv, SRB1_BASE); + } else if (IS_GEN3(dev_priv)) { + stop_unused_ring(dev_priv, PRB1_BASE); + stop_unused_ring(dev_priv, PRB2_BASE); + } +} + +static int i915_gem_setup_engines(struct drm_device *dev) +{ + struct drm_i915_private *dev_priv = dev->dev_private; + int ret; + + /* + * At least 830 can leave some of the unused rings + * "active" (ie. head != tail) after resume which + * will prevent c3 entry. Makes sure all unused rings + * are totally idle. + */ + stop_unused_rings(dev_priv); + + ret = intel_init_render_engine(dev_priv); + if (ret) + goto cleanup; + + if (HAS_BSD(dev_priv)) { + ret = intel_init_bsd_engine(dev_priv); + if (ret) + goto cleanup; + } + + if (intel_enable_blt(dev_priv)) { + ret = intel_init_blt_engine(dev_priv); + if (ret) + goto cleanup; + } + + if (HAS_VEBOX(dev_priv)) { + ret = intel_init_vebox_engine(dev_priv); + if (ret) + goto cleanup; + } + + if (HAS_BSD2(dev_priv)) { + ret = intel_init_bsd2_engine(dev_priv); + if (ret) + goto cleanup; + } + + return 0; + +cleanup: + i915_gem_cleanup_engines(dev); + return ret; } int @@ -4535,8 +4223,9 @@ i915_gem_suspend(struct drm_device *dev) if (!drm_core_check_feature(dev, DRIVER_MODESET)) i915_gem_evict_everything(dev); - i915_kernel_lost_context(dev); - i915_gem_stop_ringbuffers(dev); + ret = i915_gem_suspend_engines(dev); + if (ret) + goto err; /* Hack! Don't let anybody do execbuf while we don't control the chip. * We need to replace this with a semaphore, or something. @@ -4557,37 +4246,6 @@ err: return ret; } -int i915_gem_l3_remap(struct intel_engine_cs *ring, int slice) -{ - struct drm_device *dev = ring->dev; - struct drm_i915_private *dev_priv = dev->dev_private; - u32 reg_base = GEN7_L3LOG_BASE + (slice * 0x200); - u32 *remap_info = dev_priv->l3_parity.remap_info[slice]; - int i, ret; - - if (!HAS_L3_DPF(dev) || !remap_info) - return 0; - - ret = intel_ring_begin(ring, GEN7_L3LOG_SIZE / 4 * 3); - if (ret) - return ret; - - /* - * Note: We do not worry about the concurrent register cacheline hang - * here because no other code should access these registers other than - * at initialization time. - */ - for (i = 0; i < GEN7_L3LOG_SIZE; i += 4) { - intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1)); - intel_ring_emit(ring, reg_base + i); - intel_ring_emit(ring, remap_info[i/4]); - } - - intel_ring_advance(ring); - - return ret; -} - void i915_gem_init_swizzling(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; @@ -4613,80 +4271,11 @@ void i915_gem_init_swizzling(struct drm_device *dev) BUG(); } -static bool -intel_enable_blt(struct drm_device *dev) -{ - if (!HAS_BLT(dev)) - return false; - - /* The blitter was dysfunctional on early prototypes */ - if (IS_GEN6(dev) && dev->pdev->revision < 8) { - DRM_INFO("BLT not supported on this pre-production hardware;" - " graphics performance will be degraded.\n"); - return false; - } - - return true; -} - -static int i915_gem_init_rings(struct drm_device *dev) -{ - struct drm_i915_private *dev_priv = dev->dev_private; - int ret; - - ret = intel_init_render_ring_buffer(dev); - if (ret) - return ret; - - if (HAS_BSD(dev)) { - ret = intel_init_bsd_ring_buffer(dev); - if (ret) - goto cleanup_render_ring; - } - - if (intel_enable_blt(dev)) { - ret = intel_init_blt_ring_buffer(dev); - if (ret) - goto cleanup_bsd_ring; - } - - if (HAS_VEBOX(dev)) { - ret = intel_init_vebox_ring_buffer(dev); - if (ret) - goto cleanup_blt_ring; - } - - if (HAS_BSD2(dev)) { - ret = intel_init_bsd2_ring_buffer(dev); - if (ret) - goto cleanup_vebox_ring; - } - - ret = i915_gem_set_seqno(dev, ((u32)~0 - 0x1000)); - if (ret) - goto cleanup_bsd2_ring; - - return 0; - -cleanup_bsd2_ring: - intel_cleanup_ring_buffer(&dev_priv->ring[VCS2]); -cleanup_vebox_ring: - intel_cleanup_ring_buffer(&dev_priv->ring[VECS]); -cleanup_blt_ring: - intel_cleanup_ring_buffer(&dev_priv->ring[BCS]); -cleanup_bsd_ring: - intel_cleanup_ring_buffer(&dev_priv->ring[VCS]); -cleanup_render_ring: - intel_cleanup_ring_buffer(&dev_priv->ring[RCS]); - - return ret; -} - int i915_gem_init_hw(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; - int ret, i; + int ret; if (INTEL_INFO(dev)->gen < 6 && !intel_enable_gtt()) return -EIO; @@ -4712,25 +4301,11 @@ i915_gem_init_hw(struct drm_device *dev) i915_gem_init_swizzling(dev); - ret = i915_gem_init_rings(dev); - if (ret) - return ret; - - for (i = 0; i < NUM_L3_SLICES(dev); i++) - i915_gem_l3_remap(&dev_priv->ring[RCS], i); - - /* - * XXX: Contexts should only be initialized once. Doing a switch to the - * default context switch however is something we'd like to do after - * reset or thaw (the latter may not actually be necessary for HW, but - * goes with our code better). Context switching requires rings (for - * the do_switch), but before enabling PPGTT. So don't move this. - */ - ret = i915_gem_context_enable(dev_priv); - if (ret && ret != -EIO) { - DRM_ERROR("Context enable failed %d\n", ret); - i915_gem_cleanup_ringbuffer(dev); - } + ret = i915_ppgtt_init_hw(dev); + if (ret == 0) + ret = i915_gem_context_enable(dev_priv); + if (ret == 0) + ret = i915_gem_resume_engines(dev); return ret; } @@ -4750,16 +4325,20 @@ int i915_gem_init(struct drm_device *dev) DRM_DEBUG_DRIVER("allow wake ack timed out\n"); } - i915_gem_init_userptr(dev); - i915_gem_init_global_gtt(dev); - - ret = i915_gem_context_init(dev); + ret = i915_gem_init_userptr(dev); if (ret) { mutex_unlock(&dev->struct_mutex); return ret; } - ret = i915_gem_init_hw(dev); + i915_gem_init_global_gtt(dev); + + gen6_gt_force_wake_get(dev_priv, FORCEWAKE_ALL); + ret = i915_gem_setup_engines(dev); + if (ret == 0) + ret = i915_gem_context_init(dev); + if (ret == 0) + ret = i915_gem_init_hw(dev); if (ret == -EIO) { /* Allow ring initialisation to fail by marking the GPU as * wedged. But we only want to do this where the GPU is angry, @@ -4769,23 +4348,16 @@ int i915_gem_init(struct drm_device *dev) atomic_set_mask(I915_WEDGED, &dev_priv->gpu_error.reset_counter); ret = 0; } + gen6_gt_force_wake_put(dev_priv, FORCEWAKE_ALL); mutex_unlock(&dev->struct_mutex); - /* Allow hardware batchbuffers unless told otherwise, but not for KMS. */ - if (!drm_core_check_feature(dev, DRIVER_MODESET)) - dev_priv->dri1.allow_batchbuffer = 1; return ret; } -void -i915_gem_cleanup_ringbuffer(struct drm_device *dev) +void i915_gem_fini(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; - struct intel_engine_cs *ring; - int i; - - for_each_ring(ring, dev_priv, i) - intel_cleanup_ring_buffer(ring); + i915_gem_context_fini(dev); + i915_gem_cleanup_engines(dev); } int @@ -4804,26 +4376,12 @@ i915_gem_entervt_ioctl(struct drm_device *dev, void *data, } mutex_lock(&dev->struct_mutex); - dev_priv->ums.mm_suspended = 0; - ret = i915_gem_init_hw(dev); - if (ret != 0) { - mutex_unlock(&dev->struct_mutex); - return ret; - } - + if (ret == 0) + ret = drm_irq_install(dev, dev->pdev->irq); + if (ret == 0) + dev_priv->ums.mm_suspended = 0; BUG_ON(!list_empty(&dev_priv->gtt.base.active_list)); - - ret = drm_irq_install(dev, dev->pdev->irq); - if (ret) - goto cleanup_ringbuffer; - mutex_unlock(&dev->struct_mutex); - - return 0; - -cleanup_ringbuffer: - i915_gem_cleanup_ringbuffer(dev); - dev_priv->ums.mm_suspended = 1; mutex_unlock(&dev->struct_mutex); return ret; @@ -4857,10 +4415,13 @@ i915_gem_lastclose(struct drm_device *dev) } static void -init_ring_lists(struct intel_engine_cs *ring) +init_null_engine(struct intel_engine_cs *engine) { - INIT_LIST_HEAD(&ring->active_list); - INIT_LIST_HEAD(&ring->request_list); + INIT_LIST_HEAD(&engine->read_list); + INIT_LIST_HEAD(&engine->write_list); + INIT_LIST_HEAD(&engine->fence_list); + INIT_LIST_HEAD(&engine->requests); + INIT_LIST_HEAD(&engine->rings); } void i915_init_vm(struct drm_i915_private *dev_priv, @@ -4894,8 +4455,8 @@ i915_gem_load(struct drm_device *dev) INIT_LIST_HEAD(&dev_priv->mm.unbound_list); INIT_LIST_HEAD(&dev_priv->mm.bound_list); INIT_LIST_HEAD(&dev_priv->mm.fence_list); - for (i = 0; i < I915_NUM_RINGS; i++) - init_ring_lists(&dev_priv->ring[i]); + for (i = 0; i < I915_NUM_ENGINES; i++) + init_null_engine(&dev_priv->engine[i]); for (i = 0; i < I915_MAX_NUM_FENCES; i++) INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list); INIT_DELAYED_WORK(&dev_priv->mm.retire_work, @@ -4955,13 +4516,13 @@ void i915_gem_release(struct drm_device *dev, struct drm_file *file) */ spin_lock(&file_priv->mm.lock); while (!list_empty(&file_priv->mm.request_list)) { - struct drm_i915_gem_request *request; + struct i915_gem_request *rq; - request = list_first_entry(&file_priv->mm.request_list, - struct drm_i915_gem_request, - client_list); - list_del(&request->client_list); - request->file_priv = NULL; + rq = list_first_entry(&file_priv->mm.request_list, + struct i915_gem_request, + client_list); + list_del(&rq->client_list); + rq->file_priv = NULL; } spin_unlock(&file_priv->mm.lock); } @@ -5097,9 +4658,7 @@ unsigned long i915_gem_obj_offset(struct drm_i915_gem_object *o, struct drm_i915_private *dev_priv = o->base.dev->dev_private; struct i915_vma *vma; - if (!dev_priv->mm.aliasing_ppgtt || - vm == &dev_priv->mm.aliasing_ppgtt->base) - vm = &dev_priv->gtt.base; + WARN_ON(vm == &dev_priv->mm.aliasing_ppgtt->base); list_for_each_entry(vma, &o->vma_list, vma_link) { if (vma->vm == vm) @@ -5140,9 +4699,7 @@ unsigned long i915_gem_obj_size(struct drm_i915_gem_object *o, struct drm_i915_private *dev_priv = o->base.dev->dev_private; struct i915_vma *vma; - if (!dev_priv->mm.aliasing_ppgtt || - vm == &dev_priv->mm.aliasing_ppgtt->base) - vm = &dev_priv->gtt.base; + WARN_ON(vm == &dev_priv->mm.aliasing_ppgtt->base); BUG_ON(list_empty(&o->vma_list)); @@ -5165,11 +4722,16 @@ i915_gem_shrinker_scan(struct shrinker *shrinker, struct shrink_control *sc) if (!i915_gem_shrinker_lock(dev, &unlock)) return SHRINK_STOP; - freed = i915_gem_purge(dev_priv, sc->nr_to_scan); + freed = i915_gem_shrink(dev_priv, + sc->nr_to_scan, + I915_SHRINK_BOUND | + I915_SHRINK_UNBOUND | + I915_SHRINK_PURGEABLE); if (freed < sc->nr_to_scan) - freed += __i915_gem_shrink(dev_priv, - sc->nr_to_scan - freed, - false); + freed += i915_gem_shrink(dev_priv, + sc->nr_to_scan - freed, + I915_SHRINK_BOUND | + I915_SHRINK_UNBOUND); if (unlock) mutex_unlock(&dev->struct_mutex); @@ -5247,14 +4809,8 @@ struct i915_vma *i915_gem_obj_to_ggtt(struct drm_i915_gem_object *obj) { struct i915_vma *vma; - /* This WARN has probably outlived its usefulness (callers already - * WARN if they don't find the GGTT vma they expect). When removing, - * remember to remove the pre-check in is_pin_display() as well */ - if (WARN_ON(list_empty(&obj->vma_list))) - return NULL; - vma = list_first_entry(&obj->vma_list, typeof(*vma), vma_link); - if (vma->vm != obj_to_ggtt(obj)) + if (vma->vm != i915_obj_to_ggtt(obj)) return NULL; return vma; diff --git a/drivers/gpu/drm/i915/i915_gem_context.c b/drivers/gpu/drm/i915/i915_gem_context.c index 3b99390..841056c 100644 --- a/drivers/gpu/drm/i915/i915_gem_context.c +++ b/drivers/gpu/drm/i915/i915_gem_context.c @@ -96,53 +96,9 @@ #define GEN6_CONTEXT_ALIGN (64<<10) #define GEN7_CONTEXT_ALIGN 4096 -static void do_ppgtt_cleanup(struct i915_hw_ppgtt *ppgtt) +static size_t get_context_alignment(struct drm_i915_private *i915) { - struct drm_device *dev = ppgtt->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; - struct i915_address_space *vm = &ppgtt->base; - - if (ppgtt == dev_priv->mm.aliasing_ppgtt || - (list_empty(&vm->active_list) && list_empty(&vm->inactive_list))) { - ppgtt->base.cleanup(&ppgtt->base); - return; - } - - /* - * Make sure vmas are unbound before we take down the drm_mm - * - * FIXME: Proper refcounting should take care of this, this shouldn't be - * needed at all. - */ - if (!list_empty(&vm->active_list)) { - struct i915_vma *vma; - - list_for_each_entry(vma, &vm->active_list, mm_list) - if (WARN_ON(list_empty(&vma->vma_link) || - list_is_singular(&vma->vma_link))) - break; - - i915_gem_evict_vm(&ppgtt->base, true); - } else { - i915_gem_retire_requests(dev); - i915_gem_evict_vm(&ppgtt->base, false); - } - - ppgtt->base.cleanup(&ppgtt->base); -} - -static void ppgtt_release(struct kref *kref) -{ - struct i915_hw_ppgtt *ppgtt = - container_of(kref, struct i915_hw_ppgtt, ref); - - do_ppgtt_cleanup(ppgtt); - kfree(ppgtt); -} - -static size_t get_context_alignment(struct drm_device *dev) -{ - if (IS_GEN6(dev)) + if (IS_GEN6(i915)) return GEN6_CONTEXT_ALIGN; return GEN7_CONTEXT_ALIGN; @@ -155,6 +111,9 @@ static int get_context_size(struct drm_device *dev) u32 reg; switch (INTEL_INFO(dev)->gen) { + case 5: + ret = ILK_CXT_TOTAL_SIZE; + break; case 6: reg = I915_READ(CXT_SIZE); ret = GEN6_CXT_TOTAL_SIZE(reg) * 64; @@ -178,25 +137,27 @@ static int get_context_size(struct drm_device *dev) void i915_gem_context_free(struct kref *ctx_ref) { - struct intel_context *ctx = container_of(ctx_ref, - typeof(*ctx), ref); - struct i915_hw_ppgtt *ppgtt = NULL; - - if (ctx->legacy_hw_ctx.rcs_state) { - /* We refcount even the aliasing PPGTT to keep the code symmetric */ - if (USES_PPGTT(ctx->legacy_hw_ctx.rcs_state->base.dev)) - ppgtt = ctx_to_ppgtt(ctx); + struct intel_context *ctx = + container_of(ctx_ref, typeof(*ctx), ref); + struct drm_i915_private *dev_priv = ctx->i915; + int i; + + i915_ppgtt_put(ctx->ppgtt); + + for (i = 0; i < I915_NUM_ENGINES; i++) { + if (intel_engine_initialized(&dev_priv->engine[i]) && + ctx->ring[i].ring != NULL) + dev_priv->engine[i].put_ring(ctx->ring[i].ring, ctx); + + if (ctx->ring[i].state != NULL) + drm_gem_object_unreference(&ctx->ring[i].state->base); } - if (ppgtt) - kref_put(&ppgtt->ref, ppgtt_release); - if (ctx->legacy_hw_ctx.rcs_state) - drm_gem_object_unreference(&ctx->legacy_hw_ctx.rcs_state->base); list_del(&ctx->link); kfree(ctx); } -static struct drm_i915_gem_object * +struct drm_i915_gem_object * i915_gem_alloc_context_obj(struct drm_device *dev, size_t size) { struct drm_i915_gem_object *obj; @@ -226,49 +187,32 @@ i915_gem_alloc_context_obj(struct drm_device *dev, size_t size) return obj; } -static struct i915_hw_ppgtt * -create_vm_for_ctx(struct drm_device *dev, struct intel_context *ctx) -{ - struct i915_hw_ppgtt *ppgtt; - int ret; - - ppgtt = kzalloc(sizeof(*ppgtt), GFP_KERNEL); - if (!ppgtt) - return ERR_PTR(-ENOMEM); - - ret = i915_gem_init_ppgtt(dev, ppgtt); - if (ret) { - kfree(ppgtt); - return ERR_PTR(ret); - } - - ppgtt->ctx = ctx; - return ppgtt; -} - static struct intel_context * -__create_hw_context(struct drm_device *dev, - struct drm_i915_file_private *file_priv) +i915_gem_create_context(struct drm_device *dev, + struct drm_i915_file_private *file_priv) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_context *ctx; int ret; + BUG_ON(!mutex_is_locked(&dev->struct_mutex)); + ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); if (ctx == NULL) return ERR_PTR(-ENOMEM); kref_init(&ctx->ref); list_add_tail(&ctx->link, &dev_priv->context_list); + ctx->i915 = dev_priv; if (dev_priv->hw_context_size) { struct drm_i915_gem_object *obj = - i915_gem_alloc_context_obj(dev, dev_priv->hw_context_size); + i915_gem_alloc_context_obj(dev, dev_priv->hw_context_size); if (IS_ERR(obj)) { ret = PTR_ERR(obj); - goto err_out; + goto err; } - ctx->legacy_hw_ctx.rcs_state = obj; + ctx->ring[RCS].state = obj; } /* Default context will never have a file_priv */ @@ -276,7 +220,7 @@ __create_hw_context(struct drm_device *dev, ret = idr_alloc(&file_priv->context_idr, ctx, DEFAULT_CONTEXT_HANDLE, 0, GFP_KERNEL); if (ret < 0) - goto err_out; + goto err; } else ret = DEFAULT_CONTEXT_HANDLE; @@ -287,135 +231,33 @@ __create_hw_context(struct drm_device *dev, * is no remap info, it will be a NOP. */ ctx->remap_slice = (1 << NUM_L3_SLICES(dev)) - 1; - return ctx; - -err_out: - i915_gem_context_unreference(ctx); - return ERR_PTR(ret); -} - -/** - * The default context needs to exist per ring that uses contexts. It stores the - * context state of the GPU for applications that don't utilize HW contexts, as - * well as an idle case. - */ -static struct intel_context * -i915_gem_create_context(struct drm_device *dev, - struct drm_i915_file_private *file_priv, - bool create_vm) -{ - const bool is_global_default_ctx = file_priv == NULL; - struct drm_i915_private *dev_priv = dev->dev_private; - struct intel_context *ctx; - int ret = 0; - - BUG_ON(!mutex_is_locked(&dev->struct_mutex)); + ctx->hang_stats.ban_period_seconds = DRM_I915_CTX_BAN_PERIOD; - ctx = __create_hw_context(dev, file_priv); - if (IS_ERR(ctx)) - return ctx; - - if (is_global_default_ctx && ctx->legacy_hw_ctx.rcs_state) { - /* We may need to do things with the shrinker which - * require us to immediately switch back to the default - * context. This can cause a problem as pinning the - * default context also requires GTT space which may not - * be available. To avoid this we always pin the default - * context. - */ - ret = i915_gem_obj_ggtt_pin(ctx->legacy_hw_ctx.rcs_state, - get_context_alignment(dev), 0); - if (ret) { - DRM_DEBUG_DRIVER("Couldn't pin %d\n", ret); - goto err_destroy; - } - } - - if (create_vm) { - struct i915_hw_ppgtt *ppgtt = create_vm_for_ctx(dev, ctx); + if (USES_FULL_PPGTT(dev)) { + struct i915_hw_ppgtt *ppgtt = i915_ppgtt_create(dev, file_priv); if (IS_ERR_OR_NULL(ppgtt)) { DRM_DEBUG_DRIVER("PPGTT setup failed (%ld)\n", PTR_ERR(ppgtt)); ret = PTR_ERR(ppgtt); - goto err_unpin; - } else - ctx->vm = &ppgtt->base; - - /* This case is reserved for the global default context and - * should only happen once. */ - if (is_global_default_ctx) { - if (WARN_ON(dev_priv->mm.aliasing_ppgtt)) { - ret = -EEXIST; - goto err_unpin; - } - - dev_priv->mm.aliasing_ppgtt = ppgtt; + goto err; } - } else if (USES_PPGTT(dev)) { - /* For platforms which only have aliasing PPGTT, we fake the - * address space and refcounting. */ - ctx->vm = &dev_priv->mm.aliasing_ppgtt->base; - kref_get(&dev_priv->mm.aliasing_ppgtt->ref); - } else - ctx->vm = &dev_priv->gtt.base; + + ctx->ppgtt = ppgtt; + } return ctx; -err_unpin: - if (is_global_default_ctx && ctx->legacy_hw_ctx.rcs_state) - i915_gem_object_ggtt_unpin(ctx->legacy_hw_ctx.rcs_state); -err_destroy: +err: i915_gem_context_unreference(ctx); return ERR_PTR(ret); } -void i915_gem_context_reset(struct drm_device *dev) -{ - struct drm_i915_private *dev_priv = dev->dev_private; - int i; - - /* Prevent the hardware from restoring the last context (which hung) on - * the next switch */ - for (i = 0; i < I915_NUM_RINGS; i++) { - struct intel_engine_cs *ring = &dev_priv->ring[i]; - struct intel_context *dctx = ring->default_context; - struct intel_context *lctx = ring->last_context; - - /* Do a fake switch to the default context */ - if (lctx == dctx) - continue; - - if (!lctx) - continue; - - if (dctx->legacy_hw_ctx.rcs_state && i == RCS) { - WARN_ON(i915_gem_obj_ggtt_pin(dctx->legacy_hw_ctx.rcs_state, - get_context_alignment(dev), 0)); - /* Fake a finish/inactive */ - dctx->legacy_hw_ctx.rcs_state->base.write_domain = 0; - dctx->legacy_hw_ctx.rcs_state->active = 0; - } - - if (lctx->legacy_hw_ctx.rcs_state && i == RCS) - i915_gem_object_ggtt_unpin(lctx->legacy_hw_ctx.rcs_state); - - i915_gem_context_unreference(lctx); - i915_gem_context_reference(dctx); - ring->last_context = dctx; - } -} - int i915_gem_context_init(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; struct intel_context *ctx; - int i; - - /* Init should only be called once per module load. Eventually the - * restriction on the context_disabled check can be loosened. */ - if (WARN_ON(dev_priv->ring[RCS].default_context)) - return 0; + int i, ret; if (HAS_HW_CONTEXTS(dev)) { dev_priv->hw_context_size = round_up(get_context_size(dev), 4096); @@ -426,86 +268,111 @@ int i915_gem_context_init(struct drm_device *dev) } } - ctx = i915_gem_create_context(dev, NULL, USES_PPGTT(dev)); + /** + * The default context needs to exist per ring that uses contexts. + * It stores the context state of the GPU for applications that don't + * utilize HW contexts or per-process VM, as well as an idle case. + */ + ctx = i915_gem_create_context(dev, NULL); if (IS_ERR(ctx)) { DRM_ERROR("Failed to create default global context (error %ld)\n", PTR_ERR(ctx)); return PTR_ERR(ctx); } - /* NB: RCS will hold a ref for all rings */ - for (i = 0; i < I915_NUM_RINGS; i++) - dev_priv->ring[i].default_context = ctx; + if (dev_priv->hw_context_size) { + /* We may need to do things with the shrinker which + * require us to immediately switch back to the default + * context. This can cause a problem as pinning the + * default context also requires GTT space which may not + * be available. To avoid this we always pin the default + * context. + */ + ret = i915_gem_obj_ggtt_pin(ctx->ring[RCS].state, + get_context_alignment(dev_priv), 0); + if (ret) { + DRM_ERROR("Failed to pin global default context\n"); + i915_gem_context_unreference(ctx); + return ret; + } + } + + for (i = 0; i < I915_NUM_ENGINES; i++) { + struct intel_engine_cs *engine = &dev_priv->engine[i]; + + if (engine->i915 == NULL) + continue; + + engine->default_context = ctx; + i915_gem_context_reference(ctx); + } + + dev_priv->default_context = ctx; - DRM_DEBUG_DRIVER("%s context support initialized\n", dev_priv->hw_context_size ? "HW" : "fake"); + DRM_DEBUG_DRIVER("%s context support initialized\n", + dev_priv->hw_context_size ? "HW" : "fake"); return 0; } void i915_gem_context_fini(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; - struct intel_context *dctx = dev_priv->ring[RCS].default_context; + struct intel_engine_cs *engine; int i; - if (dctx->legacy_hw_ctx.rcs_state) { + if (dev_priv->hw_context_size) /* The only known way to stop the gpu from accessing the hw context is * to reset it. Do this as the very last operation to avoid confusing * other code, leading to spurious errors. */ intel_gpu_reset(dev); - /* When default context is created and switched to, base object refcount - * will be 2 (+1 from object creation and +1 from do_switch()). - * i915_gem_context_fini() will be called after gpu_idle() has switched - * to default context. So we need to unreference the base object once - * to offset the do_switch part, so that i915_gem_context_unreference() - * can then free the base object correctly. */ - WARN_ON(!dev_priv->ring[RCS].last_context); - if (dev_priv->ring[RCS].last_context == dctx) { - /* Fake switch to NULL context */ - WARN_ON(dctx->legacy_hw_ctx.rcs_state->active); - i915_gem_object_ggtt_unpin(dctx->legacy_hw_ctx.rcs_state); - i915_gem_context_unreference(dctx); - dev_priv->ring[RCS].last_context = NULL; - } - - i915_gem_object_ggtt_unpin(dctx->legacy_hw_ctx.rcs_state); + for_each_engine(engine, dev_priv, i) { + i915_gem_context_unreference(engine->default_context); + engine->default_context = NULL; } - for (i = 0; i < I915_NUM_RINGS; i++) { - struct intel_engine_cs *ring = &dev_priv->ring[i]; - - if (ring->last_context) - i915_gem_context_unreference(ring->last_context); - - ring->default_context = NULL; - ring->last_context = NULL; + if (dev_priv->default_context) { + if (dev_priv->hw_context_size) + i915_gem_object_ggtt_unpin(dev_priv->default_context->ring[RCS].state); + i915_gem_context_unreference(dev_priv->default_context); + dev_priv->default_context = NULL; } - - i915_gem_context_unreference(dctx); } int i915_gem_context_enable(struct drm_i915_private *dev_priv) { - struct intel_engine_cs *ring; + struct intel_engine_cs *engine; int ret, i; - /* This is the only place the aliasing PPGTT gets enabled, which means - * it has to happen before we bail on reset */ - if (dev_priv->mm.aliasing_ppgtt) { - struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt; - ppgtt->enable(ppgtt); - } + for_each_engine(engine, dev_priv, i) { + struct intel_context *ctx = engine->default_context; + struct i915_gem_request *rq; - /* FIXME: We should make this work, even in reset */ - if (i915_reset_in_progress(&dev_priv->gpu_error)) - return 0; + if (HAS_L3_DPF(dev_priv)) + ctx->remap_slice = (1 << NUM_L3_SLICES(dev_priv)) - 1; - BUG_ON(!dev_priv->ring[RCS].default_context); + rq = intel_engine_alloc_request(engine, ctx); + if (IS_ERR(rq)) { + ret = PTR_ERR(rq); + goto err; + } - for_each_ring(ring, dev_priv, i) { - ret = i915_switch_context(ring, ring->default_context); - if (ret) + ret = 0; + /* + * Workarounds applied in this fn are part of register state context, + * they need to be re-initialized followed by gpu reset, suspend/resume, + * module reload. + */ + if (engine->init_context) + ret = engine->init_context(rq); + if (ret == 0) + ret = i915_request_commit(rq); + i915_request_put(rq); + if (ret) { +err: + DRM_ERROR("failed to enabled contexts (%s): %d\n", engine->name, ret); return ret; + } } return 0; @@ -515,7 +382,9 @@ static int context_idr_cleanup(int id, void *p, void *data) { struct intel_context *ctx = p; + ctx->file_priv = NULL; i915_gem_context_unreference(ctx); + return 0; } @@ -527,7 +396,7 @@ int i915_gem_context_open(struct drm_device *dev, struct drm_file *file) idr_init(&file_priv->context_idr); mutex_lock(&dev->struct_mutex); - ctx = i915_gem_create_context(dev, file_priv, USES_FULL_PPGTT(dev)); + ctx = i915_gem_create_context(dev, file_priv); mutex_unlock(&dev->struct_mutex); if (IS_ERR(ctx)) { @@ -559,147 +428,212 @@ i915_gem_context_get(struct drm_i915_file_private *file_priv, u32 id) } static inline int -mi_set_context(struct intel_engine_cs *ring, - struct intel_context *new_context, - u32 hw_flags) +mi_set_context(struct i915_gem_request *rq, + struct intel_engine_context *new_context, + u32 flags) { - int ret; + struct intel_ringbuffer *ring; + int len; /* w/a: If Flush TLB Invalidation Mode is enabled, driver must do a TLB * invalidation prior to MI_SET_CONTEXT. On GEN6 we don't set the value - * explicitly, so we rely on the value at ring init, stored in + * explicitly, so we rely on the value at engine init, stored in * itlb_before_ctx_switch. */ - if (IS_GEN6(ring->dev)) { - ret = ring->flush(ring, I915_GEM_GPU_DOMAINS, 0); - if (ret) - return ret; + if (IS_GEN6(rq->i915)) + rq->pending_flush |= I915_INVALIDATE_CACHES; + + len = 3; + switch (INTEL_INFO(rq->i915)->gen) { + case 8: + case 7: + case 5: len += 2; + break; } - ret = intel_ring_begin(ring, 6); - if (ret) - return ret; + ring = intel_ring_begin(rq, len); + if (IS_ERR(ring)) + return PTR_ERR(ring); - /* WaProgramMiArbOnOffAroundMiSetContext:ivb,vlv,hsw,bdw,chv */ - if (INTEL_INFO(ring->dev)->gen >= 7) + switch (INTEL_INFO(rq->i915)->gen) { + case 8: + case 7: + /* WaProgramMiArbOnOffAroundMiSetContext:ivb,vlv,hsw,bdw,chv */ intel_ring_emit(ring, MI_ARB_ON_OFF | MI_ARB_DISABLE); - else - intel_ring_emit(ring, MI_NOOP); + break; + case 5: + intel_ring_emit(ring, MI_SUSPEND_FLUSH | MI_SUSPEND_FLUSH_EN); + break; + } - intel_ring_emit(ring, MI_NOOP); intel_ring_emit(ring, MI_SET_CONTEXT); - intel_ring_emit(ring, i915_gem_obj_ggtt_offset(new_context->legacy_hw_ctx.rcs_state) | + intel_ring_emit(ring, + i915_gem_obj_ggtt_offset(new_context->state) | MI_MM_SPACE_GTT | - MI_SAVE_EXT_STATE_EN | - MI_RESTORE_EXT_STATE_EN | - hw_flags); + flags); /* * w/a: MI_SET_CONTEXT must always be followed by MI_NOOP * WaMiSetContext_Hang:snb,ivb,vlv */ intel_ring_emit(ring, MI_NOOP); - if (INTEL_INFO(ring->dev)->gen >= 7) + switch (INTEL_INFO(rq->i915)->gen) { + case 8: + case 7: intel_ring_emit(ring, MI_ARB_ON_OFF | MI_ARB_ENABLE); - else - intel_ring_emit(ring, MI_NOOP); + break; + case 5: + intel_ring_emit(ring, MI_SUSPEND_FLUSH); + break; + } intel_ring_advance(ring); - return ret; + rq->pending_flush &= ~I915_COMMAND_BARRIER; + return 0; } -static int do_switch(struct intel_engine_cs *ring, - struct intel_context *to) +static int l3_remap(struct i915_gem_request *rq, int slice) { - struct drm_i915_private *dev_priv = ring->dev->dev_private; - struct intel_context *from = ring->last_context; - struct i915_hw_ppgtt *ppgtt = ctx_to_ppgtt(to); - u32 hw_flags = 0; - bool uninitialized = false; - int ret, i; + const u32 reg_base = GEN7_L3LOG_BASE + (slice * 0x200); + const u32 *remap_info; + struct intel_ringbuffer *ring; + int i; + + remap_info = rq->i915->l3_parity.remap_info[slice]; + if (remap_info == NULL) + return 0; + + ring = intel_ring_begin(rq, GEN7_L3LOG_SIZE / 4 * 3); + if (IS_ERR(ring)) + return PTR_ERR(ring); - if (from != NULL && ring == &dev_priv->ring[RCS]) { - BUG_ON(from->legacy_hw_ctx.rcs_state == NULL); - BUG_ON(!i915_gem_obj_is_pinned(from->legacy_hw_ctx.rcs_state)); + /* + * Note: We do not worry about the concurrent register cacheline hang + * here because no other code should access these registers other than + * at initialization time. + */ + for (i = 0; i < GEN7_L3LOG_SIZE; i += 4) { + intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1)); + intel_ring_emit(ring, reg_base + i); + intel_ring_emit(ring, remap_info[i/4]); } - if (from == to && !to->remap_slice) + intel_ring_advance(ring); + return 0; +} + +/** + * i915_request_switch_context() - perform a GPU context switch. + * @rq: request and ring/ctx for which we'll execute the context switch + * + * The context life cycle is simple. The context refcount is incremented and + * decremented by 1 and create and destroy. If the context is in use by the GPU, + * it will have a refoucnt > 1. This allows us to destroy the context abstract + * object while letting the normal object tracking destroy the backing BO. + */ +int i915_request_switch_context(struct i915_gem_request *rq) +{ + struct intel_context *to = rq->ctx; + struct intel_engine_context *ctx = &to->ring[rq->engine->id]; + struct intel_context *from; + int ret, i; + + lockdep_assert_held(&rq->i915->dev->struct_mutex); + + if (rq->ring->last_context == to && !to->remap_slice) return 0; - /* Trying to pin first makes error handling easier. */ - if (ring == &dev_priv->ring[RCS]) { - ret = i915_gem_obj_ggtt_pin(to->legacy_hw_ctx.rcs_state, - get_context_alignment(ring->dev), 0); + if (ctx->state != NULL) { + /* Trying to pin first makes error handling easier. */ + ret = i915_gem_obj_ggtt_pin(ctx->state, + get_context_alignment(rq->i915), 0); if (ret) return ret; - } - - /* - * Pin can switch back to the default context if we end up calling into - * evict_everything - as a last ditch gtt defrag effort that also - * switches to the default context. Hence we need to reload from here. - */ - from = ring->last_context; - if (USES_FULL_PPGTT(ring->dev)) { - ret = ppgtt->switch_mm(ppgtt, ring, false); + /* + * Clear this page out of any CPU caches for coherent swap-in/out. Note + * that thanks to write = false in this call and us not setting any gpu + * write domains when putting a context object onto the active list + * (when switching away from it), this won't block. + * + * XXX: We need a real interface to do this instead of trickery. + */ + ret = i915_gem_object_set_to_gtt_domain(ctx->state, false); if (ret) goto unpin_out; } - if (ring != &dev_priv->ring[RCS]) { - if (from) - i915_gem_context_unreference(from); - goto done; - } + if (ctx->state != NULL) { + u32 flags; - /* - * Clear this page out of any CPU caches for coherent swap-in/out. Note - * that thanks to write = false in this call and us not setting any gpu - * write domains when putting a context object onto the active list - * (when switching away from it), this won't block. - * - * XXX: We need a real interface to do this instead of trickery. - */ - ret = i915_gem_object_set_to_gtt_domain(to->legacy_hw_ctx.rcs_state, false); - if (ret) - goto unpin_out; + if (!ctx->state->has_global_gtt_mapping) { + struct i915_vma *vma = + i915_gem_obj_to_vma(ctx->state, + &rq->i915->gtt.base); + vma->bind_vma(vma, + ctx->state->cache_level, + GLOBAL_BIND); + } - if (!to->legacy_hw_ctx.rcs_state->has_global_gtt_mapping) { - struct i915_vma *vma = i915_gem_obj_to_vma(to->legacy_hw_ctx.rcs_state, - &dev_priv->gtt.base); - vma->bind_vma(vma, to->legacy_hw_ctx.rcs_state->cache_level, GLOBAL_BIND); - } + flags = 0; + if (!ctx->initialized || i915_gem_context_is_default(to)) + flags |= MI_RESTORE_INHIBIT; + + /* These flags are for resource streamer on HSW+ */ + if (!IS_HASWELL(rq->i915) && INTEL_INFO(rq->i915)->gen < 8) { + if (ctx->initialized) + flags |= MI_RESTORE_EXT_STATE_EN; + flags |= MI_SAVE_EXT_STATE_EN; + } - if (!to->legacy_hw_ctx.initialized || i915_gem_context_is_default(to)) - hw_flags |= MI_RESTORE_INHIBIT; + trace_i915_gem_ring_switch_context(rq->engine, to, flags); + ret = mi_set_context(rq, ctx, flags); + if (ret) + goto unpin_out; - ret = mi_set_context(ring, to, hw_flags); - if (ret) - goto unpin_out; + rq->pending_flush &= ~I915_COMMAND_BARRIER; + } + + if (to->ppgtt) { + ret = to->ppgtt->switch_mm(rq, to->ppgtt); + if (ret) + goto unpin_out; + } for (i = 0; i < MAX_L3_SLICES; i++) { - if (!(to->remap_slice & (1<remap_slice & (1 << i))) continue; - ret = i915_gem_l3_remap(ring, i); /* If it failed, try again next round */ - if (ret) - DRM_DEBUG_DRIVER("L3 remapping failed\n"); - else - to->remap_slice &= ~(1<remap_l3 |= 1 << i; } + /* + * Pin can switch back to the default context if we end up calling into + * evict_everything - as a last ditch gtt defrag effort that also + * switches to the default context. Hence we need to reload from here. + */ + from = rq->ring->last_context; + /* The backing object for the context is done after switching to the * *next* context. Therefore we cannot retire the previous context until * the next context has already started running. In fact, the below code * is a bit suboptimal because the retiring can occur simply after the * MI_SET_CONTEXT instead of when the next seqno has completed. */ - if (from != NULL) { - from->legacy_hw_ctx.rcs_state->base.read_domains = I915_GEM_DOMAIN_INSTRUCTION; - i915_vma_move_to_active(i915_gem_obj_to_ggtt(from->legacy_hw_ctx.rcs_state), ring); + if (from && from->ring[rq->engine->id].state) { + struct drm_i915_gem_object *from_obj = from->ring[rq->engine->id].state; + + from_obj->base.pending_read_domains = I915_GEM_DOMAIN_INSTRUCTION; + /* obj is kept alive until the next request by its active ref */ + ret = i915_request_add_vma(rq, + i915_gem_obj_to_ggtt(from_obj), + 0); + if (ret) + goto unpin_out; + /* As long as MI_SET_CONTEXT is serializing, ie. it flushes the * whole damn pipeline, we don't need to explicitly mark the * object dirty. The only exception is that the context must be @@ -707,68 +641,60 @@ static int do_switch(struct intel_engine_cs *ring, * able to defer doing this until we know the object would be * swapped, but there is no way to do that yet. */ - from->legacy_hw_ctx.rcs_state->dirty = 1; - BUG_ON(from->legacy_hw_ctx.rcs_state->ring != ring); - - /* obj is kept alive until the next request by its active ref */ - i915_gem_object_ggtt_unpin(from->legacy_hw_ctx.rcs_state); - i915_gem_context_unreference(from); - } - - uninitialized = !to->legacy_hw_ctx.initialized && from == NULL; - to->legacy_hw_ctx.initialized = true; - -done: - i915_gem_context_reference(to); - ring->last_context = to; - - if (uninitialized) { - ret = i915_gem_render_state_init(ring); - if (ret) - DRM_ERROR("init render state: %d\n", ret); + from_obj->dirty = 1; } + rq->has_ctx_switch = true; return 0; unpin_out: - if (ring->id == RCS) - i915_gem_object_ggtt_unpin(to->legacy_hw_ctx.rcs_state); + if (ctx->state) + i915_gem_object_ggtt_unpin(ctx->state); return ret; } /** - * i915_switch_context() - perform a GPU context switch. - * @ring: ring for which we'll execute the context switch - * @to: the context to switch to - * - * The context life cycle is simple. The context refcount is incremented and - * decremented by 1 and create and destroy. If the context is in use by the GPU, - * it will have a refoucnt > 1. This allows us to destroy the context abstract - * object while letting the normal object tracking destroy the backing BO. + * i915_request_switch_context__commit() - commit the context sitch + * @rq: request for which we have executed the context switch */ -int i915_switch_context(struct intel_engine_cs *ring, - struct intel_context *to) +void i915_request_switch_context__commit(struct i915_gem_request *rq) { - struct drm_i915_private *dev_priv = ring->dev->dev_private; + struct intel_context *ctx; - WARN_ON(!mutex_is_locked(&dev_priv->dev->struct_mutex)); + lockdep_assert_held(&rq->i915->dev->struct_mutex); - if (to->legacy_hw_ctx.rcs_state == NULL) { /* We have the fake context */ - if (to != ring->last_context) { - i915_gem_context_reference(to); - if (ring->last_context) - i915_gem_context_unreference(ring->last_context); - ring->last_context = to; - } - return 0; - } + if (!rq->has_ctx_switch) + return; + + ctx = rq->ring->last_context; + if (ctx && ctx->ring[rq->engine->id].state) + i915_gem_object_ggtt_unpin(ctx->ring[rq->engine->id].state); + + ctx = rq->ctx; + ctx->remap_slice &= ~rq->remap_l3; + ctx->ring[rq->engine->id].initialized = true; + + rq->has_ctx_switch = false; +} + +/** + * i915_request_switch_context__undo() - unwind the context sitch + * @rq: request for which we have executed the context switch + */ +void i915_request_switch_context__undo(struct i915_gem_request *rq) +{ + lockdep_assert_held(&rq->i915->dev->struct_mutex); + + if (!rq->has_ctx_switch) + return; - return do_switch(ring, to); + if (rq->ctx->ring[rq->engine->id].state) + i915_gem_object_ggtt_unpin(rq->ctx->ring[rq->engine->id].state); } -static bool hw_context_enabled(struct drm_device *dev) +static bool contexts_enabled(struct drm_i915_private *dev_priv) { - return to_i915(dev)->hw_context_size; + return dev_priv->hw_context_size; } int i915_gem_context_create_ioctl(struct drm_device *dev, void *data, @@ -779,14 +705,14 @@ int i915_gem_context_create_ioctl(struct drm_device *dev, void *data, struct intel_context *ctx; int ret; - if (!hw_context_enabled(dev)) + if (!contexts_enabled(to_i915(dev))) return -ENODEV; ret = i915_mutex_lock_interruptible(dev); if (ret) return ret; - ctx = i915_gem_create_context(dev, file_priv, USES_FULL_PPGTT(dev)); + ctx = i915_gem_create_context(dev, file_priv); mutex_unlock(&dev->struct_mutex); if (IS_ERR(ctx)) return PTR_ERR(ctx); diff --git a/drivers/gpu/drm/i915/i915_gem_debug.c b/drivers/gpu/drm/i915/i915_gem_debug.c deleted file mode 100644 index f462d1b..0000000 --- a/drivers/gpu/drm/i915/i915_gem_debug.c +++ /dev/null @@ -1,118 +0,0 @@ -/* - * Copyright © 2008 Intel Corporation - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice (including the next - * paragraph) shall be included in all copies or substantial portions of the - * Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - * - * Authors: - * Keith Packard - * - */ - -#include -#include -#include "i915_drv.h" - -#if WATCH_LISTS -int -i915_verify_lists(struct drm_device *dev) -{ - static int warned; - struct drm_i915_private *dev_priv = dev->dev_private; - struct drm_i915_gem_object *obj; - int err = 0; - - if (warned) - return 0; - - list_for_each_entry(obj, &dev_priv->render_ring.active_list, list) { - if (obj->base.dev != dev || - !atomic_read(&obj->base.refcount.refcount)) { - DRM_ERROR("freed render active %p\n", obj); - err++; - break; - } else if (!obj->active || - (obj->base.read_domains & I915_GEM_GPU_DOMAINS) == 0) { - DRM_ERROR("invalid render active %p (a %d r %x)\n", - obj, - obj->active, - obj->base.read_domains); - err++; - } else if (obj->base.write_domain && list_empty(&obj->gpu_write_list)) { - DRM_ERROR("invalid render active %p (w %x, gwl %d)\n", - obj, - obj->base.write_domain, - !list_empty(&obj->gpu_write_list)); - err++; - } - } - - list_for_each_entry(obj, &dev_priv->mm.flushing_list, list) { - if (obj->base.dev != dev || - !atomic_read(&obj->base.refcount.refcount)) { - DRM_ERROR("freed flushing %p\n", obj); - err++; - break; - } else if (!obj->active || - (obj->base.write_domain & I915_GEM_GPU_DOMAINS) == 0 || - list_empty(&obj->gpu_write_list)) { - DRM_ERROR("invalid flushing %p (a %d w %x gwl %d)\n", - obj, - obj->active, - obj->base.write_domain, - !list_empty(&obj->gpu_write_list)); - err++; - } - } - - list_for_each_entry(obj, &dev_priv->mm.gpu_write_list, gpu_write_list) { - if (obj->base.dev != dev || - !atomic_read(&obj->base.refcount.refcount)) { - DRM_ERROR("freed gpu write %p\n", obj); - err++; - break; - } else if (!obj->active || - (obj->base.write_domain & I915_GEM_GPU_DOMAINS) == 0) { - DRM_ERROR("invalid gpu write %p (a %d w %x)\n", - obj, - obj->active, - obj->base.write_domain); - err++; - } - } - - list_for_each_entry(obj, &i915_gtt_vm->inactive_list, list) { - if (obj->base.dev != dev || - !atomic_read(&obj->base.refcount.refcount)) { - DRM_ERROR("freed inactive %p\n", obj); - err++; - break; - } else if (obj->pin_count || obj->active || - (obj->base.write_domain & I915_GEM_GPU_DOMAINS)) { - DRM_ERROR("invalid inactive %p (p %d a %d w %x)\n", - obj, - obj->pin_count, obj->active, - obj->base.write_domain); - err++; - } - } - - return warned = err; -} -#endif /* WATCH_LIST */ diff --git a/drivers/gpu/drm/i915/i915_gem_evict.c b/drivers/gpu/drm/i915/i915_gem_evict.c index bbf4b12..886ff2e 100644 --- a/drivers/gpu/drm/i915/i915_gem_evict.c +++ b/drivers/gpu/drm/i915/i915_gem_evict.c @@ -243,7 +243,7 @@ int i915_gem_evict_everything(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; - struct i915_address_space *vm; + struct i915_address_space *vm, *v; bool lists_empty = true; int ret; @@ -270,7 +270,7 @@ i915_gem_evict_everything(struct drm_device *dev) i915_gem_retire_requests(dev); /* Having flushed everything, unbind() should never raise an error */ - list_for_each_entry(vm, &dev_priv->vm_list, global_link) + list_for_each_entry_safe(vm, v, &dev_priv->vm_list, global_link) WARN_ON(i915_gem_evict_vm(vm, false)); return 0; diff --git a/drivers/gpu/drm/i915/i915_gem_execbuffer.c b/drivers/gpu/drm/i915/i915_gem_execbuffer.c index 60998fc..182c0b3 100644 --- a/drivers/gpu/drm/i915/i915_gem_execbuffer.c +++ b/drivers/gpu/drm/i915/i915_gem_execbuffer.c @@ -35,12 +35,14 @@ #define __EXEC_OBJECT_HAS_PIN (1<<31) #define __EXEC_OBJECT_HAS_FENCE (1<<30) +#define __EXEC_OBJECT_NEEDS_MAP (1<<29) #define __EXEC_OBJECT_NEEDS_BIAS (1<<28) #define BATCH_OFFSET_BIAS (256*1024) struct eb_vmas { struct list_head vmas; + struct i915_vma *batch; int and; union { struct i915_vma *lut[0]; @@ -87,6 +89,26 @@ eb_reset(struct eb_vmas *eb) memset(eb->buckets, 0, (eb->and+1)*sizeof(struct hlist_head)); } +static struct i915_vma * +eb_get_batch(struct eb_vmas *eb) +{ + struct i915_vma *vma = + list_entry(eb->vmas.prev, typeof(*vma), exec_list); + + /* + * SNA is doing fancy tricks with compressing batch buffers, which leads + * to negative relocation deltas. Usually that works out ok since the + * relocate address is still positive, except when the batch is placed + * very low in the GTT. Ensure this doesn't happen. + * + * Note that actual hangs have only been observed on gen7, but for + * paranoia do it everywhere. + */ + vma->exec_entry->flags |= __EXEC_OBJECT_NEEDS_BIAS; + + return vma; +} + static int eb_lookup_vmas(struct eb_vmas *eb, struct drm_i915_gem_exec_object2 *exec, @@ -94,7 +116,6 @@ eb_lookup_vmas(struct eb_vmas *eb, struct i915_address_space *vm, struct drm_file *file) { - struct drm_i915_private *dev_priv = vm->dev->dev_private; struct drm_i915_gem_object *obj; struct list_head objects; int i, ret; @@ -129,20 +150,6 @@ eb_lookup_vmas(struct eb_vmas *eb, i = 0; while (!list_empty(&objects)) { struct i915_vma *vma; - struct i915_address_space *bind_vm = vm; - - if (exec[i].flags & EXEC_OBJECT_NEEDS_GTT && - USES_FULL_PPGTT(vm->dev)) { - ret = -EINVAL; - goto err; - } - - /* If we have secure dispatch, or the userspace assures us that - * they know what they're doing, use the GGTT VM. - */ - if (((args->flags & I915_EXEC_SECURE) && - (i == (args->buffer_count - 1)))) - bind_vm = &dev_priv->gtt.base; obj = list_first_entry(&objects, struct drm_i915_gem_object, @@ -156,7 +163,7 @@ eb_lookup_vmas(struct eb_vmas *eb, * from the (obj, vm) we don't run the risk of creating * duplicated vmas for the same vm. */ - vma = i915_gem_obj_lookup_or_create_vma(obj, bind_vm); + vma = i915_gem_obj_lookup_or_create_vma(obj, vm); if (IS_ERR(vma)) { DRM_DEBUG("Failed to lookup VMA\n"); ret = PTR_ERR(vma); @@ -179,6 +186,9 @@ eb_lookup_vmas(struct eb_vmas *eb, ++i; } + /* take note of the batch buffer before we might reorder the lists */ + eb->batch = eb_get_batch(eb); + return 0; @@ -270,7 +280,7 @@ relocate_entry_cpu(struct drm_i915_gem_object *obj, { struct drm_device *dev = obj->base.dev; uint32_t page_offset = offset_in_page(reloc->offset); - uint64_t delta = reloc->delta + target_offset; + uint64_t delta = (int)reloc->delta + target_offset; char *vaddr; int ret; @@ -306,8 +316,8 @@ relocate_entry_gtt(struct drm_i915_gem_object *obj, { struct drm_device *dev = obj->base.dev; struct drm_i915_private *dev_priv = dev->dev_private; - uint64_t delta = reloc->delta + target_offset; - uint32_t __iomem *reloc_entry; + uint64_t delta = (int)reloc->delta + target_offset; + uint64_t offset; void __iomem *reloc_page; int ret; @@ -320,25 +330,24 @@ relocate_entry_gtt(struct drm_i915_gem_object *obj, return ret; /* Map the page containing the relocation we're going to perform. */ - reloc->offset += i915_gem_obj_ggtt_offset(obj); + offset = i915_gem_obj_ggtt_offset(obj); + offset += reloc->offset; reloc_page = io_mapping_map_atomic_wc(dev_priv->gtt.mappable, - reloc->offset & PAGE_MASK); - reloc_entry = (uint32_t __iomem *) - (reloc_page + offset_in_page(reloc->offset)); - iowrite32(lower_32_bits(delta), reloc_entry); + offset & PAGE_MASK); + iowrite32(lower_32_bits(delta), reloc_page + offset_in_page(offset)); if (INTEL_INFO(dev)->gen >= 8) { - reloc_entry += 1; + offset += sizeof(uint32_t); - if (offset_in_page(reloc->offset + sizeof(uint32_t)) == 0) { + if (offset_in_page(offset) == 0) { io_mapping_unmap_atomic(reloc_page); - reloc_page = io_mapping_map_atomic_wc( - dev_priv->gtt.mappable, - reloc->offset + sizeof(uint32_t)); - reloc_entry = reloc_page; + reloc_page = + io_mapping_map_atomic_wc(dev_priv->gtt.mappable, + offset); } - iowrite32(upper_32_bits(delta), reloc_entry); + iowrite32(upper_32_bits(delta), + reloc_page + offset_in_page(offset)); } io_mapping_unmap_atomic(reloc_page); @@ -437,13 +446,11 @@ i915_gem_execbuffer_relocate_entry(struct drm_i915_gem_object *obj, ret = relocate_entry_cpu(obj, reloc, target_offset); else ret = relocate_entry_gtt(obj, reloc, target_offset); - if (ret) return ret; /* and update the user's relocation entry */ reloc->presumed_offset = target_offset; - return 0; } @@ -535,34 +542,18 @@ i915_gem_execbuffer_relocate(struct eb_vmas *eb) } static int -need_reloc_mappable(struct i915_vma *vma) -{ - struct drm_i915_gem_exec_object2 *entry = vma->exec_entry; - return entry->relocation_count && !use_cpu_reloc(vma->obj) && - i915_is_ggtt(vma->vm); -} - -static int i915_gem_execbuffer_reserve_vma(struct i915_vma *vma, - struct intel_engine_cs *ring, + struct intel_engine_cs *engine, bool *need_reloc) { struct drm_i915_gem_object *obj = vma->obj; struct drm_i915_gem_exec_object2 *entry = vma->exec_entry; - bool has_fenced_gpu_access = INTEL_INFO(ring->dev)->gen < 4; - bool need_fence; uint64_t flags; int ret; flags = 0; - - need_fence = - has_fenced_gpu_access && - entry->flags & EXEC_OBJECT_NEEDS_FENCE && - obj->tiling_mode != I915_TILING_NONE; - if (need_fence || need_reloc_mappable(vma)) + if (entry->flags & __EXEC_OBJECT_NEEDS_MAP) flags |= PIN_MAPPABLE; - if (entry->flags & EXEC_OBJECT_NEEDS_GTT) flags |= PIN_GLOBAL; if (entry->flags & __EXEC_OBJECT_NEEDS_BIAS) @@ -574,17 +565,13 @@ i915_gem_execbuffer_reserve_vma(struct i915_vma *vma, entry->flags |= __EXEC_OBJECT_HAS_PIN; - if (has_fenced_gpu_access) { - if (entry->flags & EXEC_OBJECT_NEEDS_FENCE) { - ret = i915_gem_object_get_fence(obj); - if (ret) - return ret; - - if (i915_gem_object_pin_fence(obj)) - entry->flags |= __EXEC_OBJECT_HAS_FENCE; + if (entry->flags & EXEC_OBJECT_NEEDS_FENCE) { + ret = i915_gem_object_get_fence(obj); + if (ret) + return ret; - obj->pending_fenced_gpu_access = true; - } + if (i915_gem_object_pin_fence(obj)) + entry->flags |= __EXEC_OBJECT_HAS_FENCE; } if (entry->offset != vma->node.start) { @@ -601,26 +588,40 @@ i915_gem_execbuffer_reserve_vma(struct i915_vma *vma, } static bool -eb_vma_misplaced(struct i915_vma *vma, bool has_fenced_gpu_access) +need_reloc_mappable(struct i915_vma *vma) { struct drm_i915_gem_exec_object2 *entry = vma->exec_entry; - struct drm_i915_gem_object *obj = vma->obj; - bool need_fence, need_mappable; - need_fence = - has_fenced_gpu_access && - entry->flags & EXEC_OBJECT_NEEDS_FENCE && - obj->tiling_mode != I915_TILING_NONE; - need_mappable = need_fence || need_reloc_mappable(vma); + if (entry->relocation_count == 0) + return false; + + if (!i915_is_ggtt(vma->vm)) + return false; + + /* See also use_cpu_reloc() */ + if (HAS_LLC(vma->obj->base.dev)) + return false; + + if (vma->obj->base.write_domain == I915_GEM_DOMAIN_CPU) + return false; + + return true; +} - WARN_ON((need_mappable || need_fence) && +static bool +eb_vma_misplaced(struct i915_vma *vma) +{ + struct drm_i915_gem_exec_object2 *entry = vma->exec_entry; + struct drm_i915_gem_object *obj = vma->obj; + + WARN_ON(entry->flags & __EXEC_OBJECT_NEEDS_MAP && !i915_is_ggtt(vma->vm)); if (entry->alignment && vma->node.start & (entry->alignment - 1)) return true; - if (need_mappable && !obj->map_and_fenceable) + if (entry->flags & __EXEC_OBJECT_NEEDS_MAP && !obj->map_and_fenceable) return true; if (entry->flags & __EXEC_OBJECT_NEEDS_BIAS && @@ -631,7 +632,7 @@ eb_vma_misplaced(struct i915_vma *vma, bool has_fenced_gpu_access) } static int -i915_gem_execbuffer_reserve(struct intel_engine_cs *ring, +i915_gem_execbuffer_reserve(struct intel_engine_cs *engine, struct list_head *vmas, bool *need_relocs) { @@ -639,13 +640,10 @@ i915_gem_execbuffer_reserve(struct intel_engine_cs *ring, struct i915_vma *vma; struct i915_address_space *vm; struct list_head ordered_vmas; - bool has_fenced_gpu_access = INTEL_INFO(ring->dev)->gen < 4; + bool has_fenced_gpu_access = INTEL_INFO(engine->i915)->gen < 4; int retry; - if (list_empty(vmas)) - return 0; - - i915_gem_retire_requests_ring(ring); + i915_gem_retire_requests__engine(engine); vm = list_first_entry(vmas, struct i915_vma, exec_list)->vm; @@ -658,20 +656,21 @@ i915_gem_execbuffer_reserve(struct intel_engine_cs *ring, obj = vma->obj; entry = vma->exec_entry; + if (!has_fenced_gpu_access) + entry->flags &= ~EXEC_OBJECT_NEEDS_FENCE; need_fence = - has_fenced_gpu_access && entry->flags & EXEC_OBJECT_NEEDS_FENCE && obj->tiling_mode != I915_TILING_NONE; need_mappable = need_fence || need_reloc_mappable(vma); - if (need_mappable) + if (need_mappable) { + entry->flags |= __EXEC_OBJECT_NEEDS_MAP; list_move(&vma->exec_list, &ordered_vmas); - else + } else list_move_tail(&vma->exec_list, &ordered_vmas); obj->base.pending_read_domains = I915_GEM_GPU_DOMAINS & ~I915_GEM_DOMAIN_COMMAND; obj->base.pending_write_domain = 0; - obj->pending_fenced_gpu_access = false; } list_splice(&ordered_vmas, vmas); @@ -696,10 +695,10 @@ i915_gem_execbuffer_reserve(struct intel_engine_cs *ring, if (!drm_mm_node_allocated(&vma->node)) continue; - if (eb_vma_misplaced(vma, has_fenced_gpu_access)) + if (eb_vma_misplaced(vma)) ret = i915_vma_unbind(vma); else - ret = i915_gem_execbuffer_reserve_vma(vma, ring, need_relocs); + ret = i915_gem_execbuffer_reserve_vma(vma, engine, need_relocs); if (ret) goto err; } @@ -709,7 +708,7 @@ i915_gem_execbuffer_reserve(struct intel_engine_cs *ring, if (drm_mm_node_allocated(&vma->node)) continue; - ret = i915_gem_execbuffer_reserve_vma(vma, ring, need_relocs); + ret = i915_gem_execbuffer_reserve_vma(vma, engine, need_relocs); if (ret) goto err; } @@ -729,10 +728,10 @@ err: } static int -i915_gem_execbuffer_relocate_slow(struct drm_device *dev, +i915_gem_execbuffer_relocate_slow(struct drm_i915_private *i915, struct drm_i915_gem_execbuffer2 *args, struct drm_file *file, - struct intel_engine_cs *ring, + struct intel_engine_cs *engine, struct eb_vmas *eb, struct drm_i915_gem_exec_object2 *exec) { @@ -744,9 +743,6 @@ i915_gem_execbuffer_relocate_slow(struct drm_device *dev, int i, total, ret; unsigned count = args->buffer_count; - if (WARN_ON(list_empty(&eb->vmas))) - return 0; - vm = list_first_entry(&eb->vmas, struct i915_vma, exec_list)->vm; /* We may process another execbuffer during the unlock... */ @@ -757,7 +753,7 @@ i915_gem_execbuffer_relocate_slow(struct drm_device *dev, drm_gem_object_unreference(&vma->obj->base); } - mutex_unlock(&dev->struct_mutex); + mutex_unlock(&i915->dev->struct_mutex); total = 0; for (i = 0; i < count; i++) @@ -768,7 +764,7 @@ i915_gem_execbuffer_relocate_slow(struct drm_device *dev, if (reloc == NULL || reloc_offset == NULL) { drm_free_large(reloc); drm_free_large(reloc_offset); - mutex_lock(&dev->struct_mutex); + mutex_lock(&i915->dev->struct_mutex); return -ENOMEM; } @@ -783,7 +779,7 @@ i915_gem_execbuffer_relocate_slow(struct drm_device *dev, if (copy_from_user(reloc+total, user_relocs, exec[i].relocation_count * sizeof(*reloc))) { ret = -EFAULT; - mutex_lock(&dev->struct_mutex); + mutex_lock(&i915->dev->struct_mutex); goto err; } @@ -801,7 +797,7 @@ i915_gem_execbuffer_relocate_slow(struct drm_device *dev, &invalid_offset, sizeof(invalid_offset))) { ret = -EFAULT; - mutex_lock(&dev->struct_mutex); + mutex_lock(&i915->dev->struct_mutex); goto err; } } @@ -810,9 +806,9 @@ i915_gem_execbuffer_relocate_slow(struct drm_device *dev, total += exec[i].relocation_count; } - ret = i915_mutex_lock_interruptible(dev); + ret = i915_mutex_lock_interruptible(i915->dev); if (ret) { - mutex_lock(&dev->struct_mutex); + mutex_lock(&i915->dev->struct_mutex); goto err; } @@ -823,7 +819,7 @@ i915_gem_execbuffer_relocate_slow(struct drm_device *dev, goto err; need_relocs = (args->flags & I915_EXEC_NO_RELOC) == 0; - ret = i915_gem_execbuffer_reserve(ring, &eb->vmas, &need_relocs); + ret = i915_gem_execbuffer_reserve(engine, &eb->vmas, &need_relocs); if (ret) goto err; @@ -848,17 +844,19 @@ err: } static int -i915_gem_execbuffer_move_to_gpu(struct intel_engine_cs *ring, - struct list_head *vmas) +vmas_move_to_rq(struct eb_vmas *eb, + struct i915_gem_request *rq) { struct i915_vma *vma; uint32_t flush_domains = 0; bool flush_chipset = false; int ret; - list_for_each_entry(vma, vmas, exec_list) { + /* 1: flush/serialise damage from other sources */ + list_for_each_entry(vma, &eb->vmas, exec_list) { struct drm_i915_gem_object *obj = vma->obj; - ret = i915_gem_object_sync(obj, ring); + + ret = i915_gem_object_sync(obj, rq); if (ret) return ret; @@ -866,18 +864,39 @@ i915_gem_execbuffer_move_to_gpu(struct intel_engine_cs *ring, flush_chipset |= i915_gem_clflush_object(obj, false); flush_domains |= obj->base.write_domain; + if (obj->last_read[rq->engine->id].request == NULL) + rq->pending_flush |= I915_INVALIDATE_CACHES; } if (flush_chipset) - i915_gem_chipset_flush(ring->dev); + i915_gem_chipset_flush(rq->i915->dev); if (flush_domains & I915_GEM_DOMAIN_GTT) wmb(); - /* Unconditionally invalidate gpu caches and ensure that we do flush - * any residual writes from the previous batch. - */ - return intel_ring_invalidate_all_caches(ring); + /* 2: invalidate the caches from this ring after emitting semaphores */ + ret = i915_request_emit_flush(rq, I915_INVALIDATE_CACHES); + if (ret) + return ret; + + /* 3: track flushes and objects for this rq */ + list_for_each_entry(vma, &eb->vmas, exec_list) { + struct drm_i915_gem_exec_object2 *entry = vma->exec_entry; + unsigned fenced; + + fenced = 0; + if (entry->flags & EXEC_OBJECT_NEEDS_FENCE) { + fenced |= VMA_IS_FENCED; + if (entry->flags & __EXEC_OBJECT_HAS_FENCE) + fenced |= VMA_HAS_FENCE; + } + + ret = i915_request_add_vma(rq, vma, fenced); + if (ret) + return ret; + } + + return 0; } static bool @@ -890,18 +909,24 @@ i915_gem_check_execbuffer(struct drm_i915_gem_execbuffer2 *exec) } static int -validate_exec_list(struct drm_i915_gem_exec_object2 *exec, +validate_exec_list(struct drm_i915_private *dev_priv, + struct drm_i915_gem_exec_object2 *exec, int count) { - int i; unsigned relocs_total = 0; unsigned relocs_max = UINT_MAX / sizeof(struct drm_i915_gem_relocation_entry); + unsigned invalid_flags; + int i; + + invalid_flags = __EXEC_OBJECT_UNKNOWN_FLAGS; + if (USES_FULL_PPGTT(dev_priv)) + invalid_flags |= EXEC_OBJECT_NEEDS_GTT; for (i = 0; i < count; i++) { char __user *ptr = to_user_ptr(exec[i].relocs_ptr); int length; /* limited by fault_in_pages_readable() */ - if (exec[i].flags & __EXEC_OBJECT_UNKNOWN_FLAGS) + if (exec[i].flags & invalid_flags) return -EINVAL; /* First check for malicious input causing overflow in @@ -932,13 +957,14 @@ validate_exec_list(struct drm_i915_gem_exec_object2 *exec, } static struct intel_context * -i915_gem_validate_context(struct drm_device *dev, struct drm_file *file, - struct intel_engine_cs *ring, const u32 ctx_id) +i915_gem_validate_context(struct drm_file *file, + struct intel_engine_cs *engine, + const u32 ctx_id) { struct intel_context *ctx = NULL; struct i915_ctx_hang_stats *hs; - if (ring->id != RCS && ctx_id != DEFAULT_CONTEXT_HANDLE) + if (engine->id != RCS && ctx_id != DEFAULT_CONTEXT_HANDLE) return ERR_PTR(-EINVAL); ctx = i915_gem_context_get(file->driver_priv, ctx_id); @@ -954,109 +980,152 @@ i915_gem_validate_context(struct drm_device *dev, struct drm_file *file, return ctx; } -static void -i915_gem_execbuffer_move_to_active(struct list_head *vmas, - struct intel_engine_cs *ring) +static int +reset_sol_offsets(struct i915_gem_request *rq) { - struct i915_vma *vma; + struct intel_ringbuffer *ring; + int i; - list_for_each_entry(vma, vmas, exec_list) { - struct drm_i915_gem_object *obj = vma->obj; - u32 old_read = obj->base.read_domains; - u32 old_write = obj->base.write_domain; + if (!IS_GEN7(rq->i915) || rq->engine->id != RCS) { + DRM_DEBUG("sol reset is gen7/rcs only\n"); + return -EINVAL; + } - obj->base.write_domain = obj->base.pending_write_domain; - if (obj->base.write_domain == 0) - obj->base.pending_read_domains |= obj->base.read_domains; - obj->base.read_domains = obj->base.pending_read_domains; - obj->fenced_gpu_access = obj->pending_fenced_gpu_access; + ring = intel_ring_begin(rq, 4 * 3); + if (IS_ERR(ring)) + return PTR_ERR(ring); - i915_vma_move_to_active(vma, ring); - if (obj->base.write_domain) { - obj->dirty = 1; - obj->last_write_seqno = intel_ring_get_seqno(ring); + for (i = 0; i < 4; i++) { + intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1)); + intel_ring_emit(ring, GEN7_SO_WRITE_OFFSET(i)); + intel_ring_emit(ring, 0); + } - intel_fb_obj_invalidate(obj, ring); + intel_ring_advance(ring); + return 0; +} - /* update for the implicit flush after a batch */ - obj->base.write_domain &= ~I915_GEM_GPU_DOMAINS; - } +static int +emit_box(struct i915_gem_request *rq, + struct drm_clip_rect *box, + int DR1, int DR4) +{ + struct intel_ringbuffer *ring; - trace_i915_gem_object_change_domain(obj, old_read, old_write); + if (box->y2 <= box->y1 || box->x2 <= box->x1 || + box->y2 <= 0 || box->x2 <= 0) { + DRM_DEBUG("Bad box %d,%d..%d,%d\n", + box->x1, box->y1, box->x2, box->y2); + return -EINVAL; } -} -static void -i915_gem_execbuffer_retire_commands(struct drm_device *dev, - struct drm_file *file, - struct intel_engine_cs *ring, - struct drm_i915_gem_object *obj) -{ - /* Unconditionally force add_request to emit a full flush. */ - ring->gpu_caches_dirty = true; + if (INTEL_INFO(rq->i915)->gen >= 4) { + ring = intel_ring_begin(rq, 4); + if (IS_ERR(ring)) + return PTR_ERR(ring); - /* Add a breadcrumb for the completion of the batch buffer */ - (void)__i915_add_request(ring, file, obj, NULL); + intel_ring_emit(ring, GFX_OP_DRAWRECT_INFO_I965); + } else { + ring = intel_ring_begin(rq, 5); + if (IS_ERR(ring)) + return PTR_ERR(ring); + + intel_ring_emit(ring, GFX_OP_DRAWRECT_INFO); + intel_ring_emit(ring, DR1); + } + intel_ring_emit(ring, (box->x1 & 0xffff) | box->y1 << 16); + intel_ring_emit(ring, ((box->x2 - 1) & 0xffff) | (box->y2 - 1) << 16); + intel_ring_emit(ring, DR4); + intel_ring_advance(ring); + + return 0; } -static int -i915_reset_gen7_sol_offsets(struct drm_device *dev, - struct intel_engine_cs *ring) +static int set_contants_base(struct i915_gem_request *rq, + struct drm_i915_gem_execbuffer2 *args) { - struct drm_i915_private *dev_priv = dev->dev_private; - int ret, i; + int mode = args->flags & I915_EXEC_CONSTANTS_MASK; + u32 mask = I915_EXEC_CONSTANTS_MASK; - if (!IS_GEN7(dev) || ring != &dev_priv->ring[RCS]) { - DRM_DEBUG("sol reset is gen7/rcs only\n"); + switch (mode) { + case I915_EXEC_CONSTANTS_REL_GENERAL: + case I915_EXEC_CONSTANTS_ABSOLUTE: + case I915_EXEC_CONSTANTS_REL_SURFACE: + if (mode != 0 && rq->engine->id != RCS) { + DRM_DEBUG("non-0 rel constants mode on non-RCS\n"); + return -EINVAL; + } + + if (mode != rq->engine->i915->relative_constants_mode) { + if (INTEL_INFO(rq->engine->i915)->gen < 4) { + DRM_DEBUG("no rel constants on pre-gen4\n"); + return -EINVAL; + } + + if (INTEL_INFO(rq->engine->i915)->gen > 5 && + mode == I915_EXEC_CONSTANTS_REL_SURFACE) { + DRM_DEBUG("rel surface constants mode invalid on gen5+\n"); + return -EINVAL; + } + + /* The HW changed the meaning on this bit on gen6 */ + if (INTEL_INFO(rq->i915)->gen >= 6) + mask &= ~I915_EXEC_CONSTANTS_REL_SURFACE; + } + break; + default: + DRM_DEBUG("execbuf with unknown constants: %d\n", mode); return -EINVAL; } - ret = intel_ring_begin(ring, 4 * 3); - if (ret) - return ret; + /* XXX INSTPM is per-context not global etc */ + if (rq->engine->id == RCS && mode != rq->i915->relative_constants_mode) { + struct intel_ringbuffer *ring; + + ring = intel_ring_begin(rq, 3); + if (IS_ERR(ring)) + return PTR_ERR(ring); - for (i = 0; i < 4; i++) { intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1)); - intel_ring_emit(ring, GEN7_SO_WRITE_OFFSET(i)); - intel_ring_emit(ring, 0); - } + intel_ring_emit(ring, INSTPM); + intel_ring_emit(ring, mask << 16 | mode); + intel_ring_advance(ring); - intel_ring_advance(ring); + rq->i915->relative_constants_mode = mode; + } return 0; } static int -legacy_ringbuffer_submission(struct drm_device *dev, struct drm_file *file, - struct intel_engine_cs *ring, - struct intel_context *ctx, - struct drm_i915_gem_execbuffer2 *args, - struct list_head *vmas, - struct drm_i915_gem_object *batch_obj, - u64 exec_start, u32 flags) +submit_execbuf(struct intel_engine_cs *engine, + struct intel_context *ctx, + struct drm_i915_gem_execbuffer2 *args, + struct eb_vmas *eb, + u64 exec_start, u32 flags) { struct drm_clip_rect *cliprects = NULL; - struct drm_i915_private *dev_priv = dev->dev_private; - u64 exec_len; - int instp_mode; - u32 instp_mask; - int i, ret = 0; + struct i915_gem_request *rq = NULL; + int i, ret; if (args->num_cliprects != 0) { - if (ring != &dev_priv->ring[RCS]) { + if (engine->id != RCS) { DRM_DEBUG("clip rectangles are only valid with the render ring\n"); - return -EINVAL; + ret = -EINVAL; + goto out; } - if (INTEL_INFO(dev)->gen >= 5) { + if (INTEL_INFO(engine->i915)->gen >= 5) { DRM_DEBUG("clip rectangles are only valid on pre-gen5\n"); - return -EINVAL; + ret = -EINVAL; + goto out; } if (args->num_cliprects > UINT_MAX / sizeof(*cliprects)) { DRM_DEBUG("execbuf with %u cliprects\n", args->num_cliprects); - return -EINVAL; + ret = -EINVAL; + goto out; } cliprects = kcalloc(args->num_cliprects, @@ -1064,14 +1133,14 @@ legacy_ringbuffer_submission(struct drm_device *dev, struct drm_file *file, GFP_KERNEL); if (cliprects == NULL) { ret = -ENOMEM; - goto error; + goto out; } if (copy_from_user(cliprects, to_user_ptr(args->cliprects_ptr), sizeof(*cliprects)*args->num_cliprects)) { ret = -EFAULT; - goto error; + goto out_cliprects; } } else { if (args->DR4 == 0xffffffff) { @@ -1081,172 +1150,115 @@ legacy_ringbuffer_submission(struct drm_device *dev, struct drm_file *file, if (args->DR1 || args->DR4 || args->cliprects_ptr) { DRM_DEBUG("0 cliprects but dirt in cliprects fields\n"); - return -EINVAL; - } - } - - ret = i915_gem_execbuffer_move_to_gpu(ring, vmas); - if (ret) - goto error; - - ret = i915_switch_context(ring, ctx); - if (ret) - goto error; - - instp_mode = args->flags & I915_EXEC_CONSTANTS_MASK; - instp_mask = I915_EXEC_CONSTANTS_MASK; - switch (instp_mode) { - case I915_EXEC_CONSTANTS_REL_GENERAL: - case I915_EXEC_CONSTANTS_ABSOLUTE: - case I915_EXEC_CONSTANTS_REL_SURFACE: - if (instp_mode != 0 && ring != &dev_priv->ring[RCS]) { - DRM_DEBUG("non-0 rel constants mode on non-RCS\n"); ret = -EINVAL; - goto error; - } - - if (instp_mode != dev_priv->relative_constants_mode) { - if (INTEL_INFO(dev)->gen < 4) { - DRM_DEBUG("no rel constants on pre-gen4\n"); - ret = -EINVAL; - goto error; - } - - if (INTEL_INFO(dev)->gen > 5 && - instp_mode == I915_EXEC_CONSTANTS_REL_SURFACE) { - DRM_DEBUG("rel surface constants mode invalid on gen5+\n"); - ret = -EINVAL; - goto error; - } - - /* The HW changed the meaning on this bit on gen6 */ - if (INTEL_INFO(dev)->gen >= 6) - instp_mask &= ~I915_EXEC_CONSTANTS_REL_SURFACE; + goto out; } - break; - default: - DRM_DEBUG("execbuf with unknown constants: %d\n", instp_mode); - ret = -EINVAL; - goto error; } - if (ring == &dev_priv->ring[RCS] && - instp_mode != dev_priv->relative_constants_mode) { - ret = intel_ring_begin(ring, 4); - if (ret) - goto error; + rq = intel_engine_alloc_request(engine, ctx); + if (IS_ERR(rq)) { + ret = PTR_ERR(rq); + goto out_cliprects; + } - intel_ring_emit(ring, MI_NOOP); - intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1)); - intel_ring_emit(ring, INSTPM); - intel_ring_emit(ring, instp_mask << 16 | instp_mode); - intel_ring_advance(ring); + ret = vmas_move_to_rq(eb, rq); + if (ret) + goto out_rq; - dev_priv->relative_constants_mode = instp_mode; - } + ret = set_contants_base(rq, args); + if (ret) + goto out_rq; if (args->flags & I915_EXEC_GEN7_SOL_RESET) { - ret = i915_reset_gen7_sol_offsets(dev, ring); + ret = reset_sol_offsets(rq); if (ret) - goto error; + goto out_rq; } - exec_len = args->batch_len; if (cliprects) { for (i = 0; i < args->num_cliprects; i++) { - ret = i915_emit_box(dev, &cliprects[i], - args->DR1, args->DR4); + ret = emit_box(rq, &cliprects[i], + args->DR1, args->DR4); if (ret) - goto error; + goto out_rq; - ret = ring->dispatch_execbuffer(ring, - exec_start, exec_len, - flags); + ret = i915_request_emit_batchbuffer(rq, eb->batch, + exec_start, + args->batch_len, + flags); if (ret) - goto error; + goto out_rq; } } else { - ret = ring->dispatch_execbuffer(ring, - exec_start, exec_len, - flags); + ret = i915_request_emit_batchbuffer(rq, eb->batch, + exec_start, + args->batch_len, + flags); if (ret) - return ret; + goto out_rq; } - trace_i915_gem_ring_dispatch(ring, intel_ring_get_seqno(ring), flags); + ret = i915_request_commit(rq); + if (ret) + goto out_rq; + + i915_queue_hangcheck(rq->i915->dev); - i915_gem_execbuffer_move_to_active(vmas, ring); - i915_gem_execbuffer_retire_commands(dev, file, ring, batch_obj); + cancel_delayed_work_sync(&rq->i915->mm.idle_work); + queue_delayed_work(rq->i915->wq, + &rq->i915->mm.retire_work, + round_jiffies_up_relative(HZ)); + intel_mark_busy(rq->i915->dev); -error: +out_rq: + i915_request_put(rq); +out_cliprects: kfree(cliprects); +out: return ret; } /** * Find one BSD ring to dispatch the corresponding BSD command. - * The Ring ID is returned. */ -static int gen8_dispatch_bsd_ring(struct drm_device *dev, - struct drm_file *file) +static struct intel_engine_cs * +gen8_select_bsd_engine(struct drm_i915_private *dev_priv, + struct drm_file *file) { - struct drm_i915_private *dev_priv = dev->dev_private; struct drm_i915_file_private *file_priv = file->driver_priv; - /* Check whether the file_priv is using one ring */ - if (file_priv->bsd_ring) - return file_priv->bsd_ring->id; - else { - /* If no, use the ping-pong mechanism to select one ring */ - int ring_id; + /* Use the ping-pong mechanism to select one ring for this client */ + if (file_priv->bsd_engine == NULL) { + int id; - mutex_lock(&dev->struct_mutex); + mutex_lock(&dev_priv->dev->struct_mutex); if (dev_priv->mm.bsd_ring_dispatch_index == 0) { - ring_id = VCS; + id = VCS; dev_priv->mm.bsd_ring_dispatch_index = 1; } else { - ring_id = VCS2; + id = VCS2; dev_priv->mm.bsd_ring_dispatch_index = 0; } - file_priv->bsd_ring = &dev_priv->ring[ring_id]; - mutex_unlock(&dev->struct_mutex); - return ring_id; + file_priv->bsd_engine = &dev_priv->engine[id]; + mutex_unlock(&dev_priv->dev->struct_mutex); } -} - -static struct drm_i915_gem_object * -eb_get_batch(struct eb_vmas *eb) -{ - struct i915_vma *vma = list_entry(eb->vmas.prev, typeof(*vma), exec_list); - /* - * SNA is doing fancy tricks with compressing batch buffers, which leads - * to negative relocation deltas. Usually that works out ok since the - * relocate address is still positive, except when the batch is placed - * very low in the GTT. Ensure this doesn't happen. - * - * Note that actual hangs have only been observed on gen7, but for - * paranoia do it everywhere. - */ - vma->exec_entry->flags |= __EXEC_OBJECT_NEEDS_BIAS; - - return vma->obj; + return file_priv->bsd_engine; } static int -i915_gem_do_execbuffer(struct drm_device *dev, void *data, +i915_gem_do_execbuffer(struct drm_i915_private *dev_priv, void *data, struct drm_file *file, struct drm_i915_gem_execbuffer2 *args, struct drm_i915_gem_exec_object2 *exec) { - struct drm_i915_private *dev_priv = dev->dev_private; struct eb_vmas *eb; - struct drm_i915_gem_object *batch_obj; - struct intel_engine_cs *ring; + struct intel_engine_cs *engine; struct intel_context *ctx; struct i915_address_space *vm; const u32 ctx_id = i915_execbuffer2_get_context_id(*args); u64 exec_start = args->batch_start_offset; + struct drm_i915_gem_object *batch; u32 flags; int ret; bool need_relocs; @@ -1254,7 +1266,7 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data, if (!i915_gem_check_execbuffer(args)) return -EINVAL; - ret = validate_exec_list(exec, args->buffer_count); + ret = validate_exec_list(dev_priv, exec, args->buffer_count); if (ret) return ret; @@ -1275,18 +1287,16 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data, } if ((args->flags & I915_EXEC_RING_MASK) == I915_EXEC_DEFAULT) - ring = &dev_priv->ring[RCS]; + engine = &dev_priv->engine[RCS]; else if ((args->flags & I915_EXEC_RING_MASK) == I915_EXEC_BSD) { - if (HAS_BSD2(dev)) { - int ring_id; - ring_id = gen8_dispatch_bsd_ring(dev, file); - ring = &dev_priv->ring[ring_id]; - } else - ring = &dev_priv->ring[VCS]; + if (HAS_BSD2(dev_priv)) + engine = gen8_select_bsd_engine(dev_priv, file); + else + engine = &dev_priv->engine[VCS]; } else - ring = &dev_priv->ring[(args->flags & I915_EXEC_RING_MASK) - 1]; + engine = &dev_priv->engine[(args->flags & I915_EXEC_RING_MASK) - 1]; - if (!intel_ring_initialized(ring)) { + if (!intel_engine_initialized(engine)) { DRM_DEBUG("execbuf with invalid ring: %d\n", (int)(args->flags & I915_EXEC_RING_MASK)); return -EINVAL; @@ -1299,33 +1309,34 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data, intel_runtime_pm_get(dev_priv); - ret = i915_mutex_lock_interruptible(dev); + ret = i915_mutex_lock_interruptible(dev_priv->dev); if (ret) goto pre_mutex_err; if (dev_priv->ums.mm_suspended) { - mutex_unlock(&dev->struct_mutex); + mutex_unlock(&dev_priv->dev->struct_mutex); ret = -EBUSY; goto pre_mutex_err; } - ctx = i915_gem_validate_context(dev, file, ring, ctx_id); + ctx = i915_gem_validate_context(file, engine, ctx_id); if (IS_ERR(ctx)) { - mutex_unlock(&dev->struct_mutex); + mutex_unlock(&dev_priv->dev->struct_mutex); ret = PTR_ERR(ctx); goto pre_mutex_err; } i915_gem_context_reference(ctx); - vm = ctx->vm; - if (!USES_FULL_PPGTT(dev)) + if (ctx->ppgtt) + vm = &ctx->ppgtt->base; + else vm = &dev_priv->gtt.base; eb = eb_create(args); if (eb == NULL) { i915_gem_context_unreference(ctx); - mutex_unlock(&dev->struct_mutex); + mutex_unlock(&dev_priv->dev->struct_mutex); ret = -ENOMEM; goto pre_mutex_err; } @@ -1335,12 +1346,9 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data, if (ret) goto err; - /* take note of the batch buffer before we might reorder the lists */ - batch_obj = eb_get_batch(eb); - /* Move the objects en-masse into the GTT, evicting if necessary. */ need_relocs = (args->flags & I915_EXEC_NO_RELOC) == 0; - ret = i915_gem_execbuffer_reserve(ring, &eb->vmas, &need_relocs); + ret = i915_gem_execbuffer_reserve(engine, &eb->vmas, &need_relocs); if (ret) goto err; @@ -1349,25 +1357,25 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data, ret = i915_gem_execbuffer_relocate(eb); if (ret) { if (ret == -EFAULT) { - ret = i915_gem_execbuffer_relocate_slow(dev, args, file, ring, + ret = i915_gem_execbuffer_relocate_slow(dev_priv, args, file, engine, eb, exec); - BUG_ON(!mutex_is_locked(&dev->struct_mutex)); + BUG_ON(!mutex_is_locked(&dev_priv->dev->struct_mutex)); } if (ret) goto err; } /* Set the pending read domains for the batch buffer to COMMAND */ - if (batch_obj->base.pending_write_domain) { + batch = eb->batch->obj; + if (batch->base.pending_write_domain) { DRM_DEBUG("Attempting to use self-modifying batch buffer\n"); ret = -EINVAL; goto err; } - batch_obj->base.pending_read_domains |= I915_GEM_DOMAIN_COMMAND; + batch->base.pending_read_domains |= I915_GEM_DOMAIN_COMMAND; - if (i915_needs_cmd_parser(ring)) { - ret = i915_parse_cmds(ring, - batch_obj, + if (i915_needs_cmd_parser(engine)) { + ret = i915_parse_cmds(engine, batch, args->batch_start_offset, file->is_master); if (ret) @@ -1386,31 +1394,41 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data, /* snb/ivb/vlv conflate the "batch in ppgtt" bit with the "non-secure * batch" bit. Hence we need to pin secure batches into the global gtt. * hsw should have this fixed, but bdw mucks it up again. */ - if (flags & I915_DISPATCH_SECURE && - !batch_obj->has_global_gtt_mapping) { - /* When we have multiple VMs, we'll need to make sure that we - * allocate space first */ - struct i915_vma *vma = i915_gem_obj_to_ggtt(batch_obj); - BUG_ON(!vma); - vma->bind_vma(vma, batch_obj->cache_level, GLOBAL_BIND); - } + if (flags & I915_DISPATCH_SECURE) { + /* + * So on first glance it looks freaky that we pin the batch here + * outside of the reservation loop. But: + * - The batch is already pinned into the relevant ppgtt, so we + * already have the backing storage fully allocated. + * - No other BO uses the global gtt (well contexts, but meh), + * so we don't really have issues with mutliple objects not + * fitting due to fragmentation. + * So this is actually safe. + */ + ret = i915_gem_obj_ggtt_pin(batch, 0, 0); + if (ret) + goto err; - if (flags & I915_DISPATCH_SECURE) - exec_start += i915_gem_obj_ggtt_offset(batch_obj); - else - exec_start += i915_gem_obj_offset(batch_obj, vm); + exec_start += i915_gem_obj_ggtt_offset(batch); + } else + exec_start += i915_gem_obj_offset(batch, vm); - ret = legacy_ringbuffer_submission(dev, file, ring, ctx, - args, &eb->vmas, batch_obj, exec_start, flags); - if (ret) - goto err; + ret = submit_execbuf(engine, ctx, args, eb, exec_start, flags); + /* + * FIXME: We crucially rely upon the active tracking for the (ppgtt) + * batch vma for correctness. For less ugly and less fragility this + * needs to be adjusted to also track the ggtt batch vma properly as + * active. + */ + if (flags & I915_DISPATCH_SECURE) + i915_gem_object_ggtt_unpin(batch); err: /* the request owns the ref now */ i915_gem_context_unreference(ctx); eb_destroy(eb); - mutex_unlock(&dev->struct_mutex); + mutex_unlock(&dev_priv->dev->struct_mutex); pre_mutex_err: /* intel_gpu_busy should also get a ref, so it will free when the device @@ -1482,7 +1500,7 @@ i915_gem_execbuffer(struct drm_device *dev, void *data, exec2.flags = I915_EXEC_RENDER; i915_execbuffer2_set_context_id(exec2, 0); - ret = i915_gem_do_execbuffer(dev, data, file, &exec2, exec2_list); + ret = i915_gem_do_execbuffer(to_i915(dev), data, file, &exec2, exec2_list); if (!ret) { struct drm_i915_gem_exec_object __user *user_exec_list = to_user_ptr(args->buffers_ptr); @@ -1546,7 +1564,7 @@ i915_gem_execbuffer2(struct drm_device *dev, void *data, return -EFAULT; } - ret = i915_gem_do_execbuffer(dev, data, file, args, exec2_list); + ret = i915_gem_do_execbuffer(to_i915(dev), data, file, args, exec2_list); if (!ret) { /* Copy the new buffer offsets back to the user's exec list. */ struct drm_i915_gem_exec_object2 __user *user_exec_list = diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.c b/drivers/gpu/drm/i915/i915_gem_gtt.c index e42925f..91f6962 100644 --- a/drivers/gpu/drm/i915/i915_gem_gtt.c +++ b/drivers/gpu/drm/i915/i915_gem_gtt.c @@ -33,17 +33,6 @@ static void bdw_setup_private_ppat(struct drm_i915_private *dev_priv); static void chv_setup_private_ppat(struct drm_i915_private *dev_priv); -bool intel_enable_ppgtt(struct drm_device *dev, bool full) -{ - if (i915.enable_ppgtt == 0) - return false; - - if (i915.enable_ppgtt == 1 && full) - return false; - - return true; -} - static int sanitize_enable_ppgtt(struct drm_device *dev, int enable_ppgtt) { if (enable_ppgtt == 0 || !HAS_ALIASING_PPGTT(dev)) @@ -70,7 +59,7 @@ static int sanitize_enable_ppgtt(struct drm_device *dev, int enable_ppgtt) return 0; } - return HAS_ALIASING_PPGTT(dev) ? 1 : 0; + return HAS_PPGTT(dev) ? 2 : HAS_ALIASING_PPGTT(dev) ? 1 : 0; } @@ -78,7 +67,6 @@ static void ppgtt_bind_vma(struct i915_vma *vma, enum i915_cache_level cache_level, u32 flags); static void ppgtt_unbind_vma(struct i915_vma *vma); -static int gen8_ppgtt_enable(struct i915_hw_ppgtt *ppgtt); static inline gen8_gtt_pte_t gen8_pte_encode(dma_addr_t addr, enum i915_cache_level level, @@ -215,38 +203,28 @@ static gen6_gtt_pte_t iris_pte_encode(dma_addr_t addr, } /* Broadwell Page Directory Pointer Descriptors */ -static int gen8_write_pdp(struct intel_engine_cs *ring, unsigned entry, - uint64_t val, bool synchronous) +static int gen8_write_pdp(struct i915_gem_request *rq, unsigned entry, uint64_t val) { - struct drm_i915_private *dev_priv = ring->dev->dev_private; - int ret; + struct intel_ringbuffer *ring; BUG_ON(entry >= 4); - if (synchronous) { - I915_WRITE(GEN8_RING_PDP_UDW(ring, entry), val >> 32); - I915_WRITE(GEN8_RING_PDP_LDW(ring, entry), (u32)val); - return 0; - } - - ret = intel_ring_begin(ring, 6); - if (ret) - return ret; + ring = intel_ring_begin(rq, 5); + if (IS_ERR(ring)) + return PTR_ERR(ring); - intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1)); - intel_ring_emit(ring, GEN8_RING_PDP_UDW(ring, entry)); + intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(2)); + intel_ring_emit(ring, GEN8_RING_PDP_UDW(rq->engine, entry)); intel_ring_emit(ring, (u32)(val >> 32)); - intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1)); - intel_ring_emit(ring, GEN8_RING_PDP_LDW(ring, entry)); + intel_ring_emit(ring, GEN8_RING_PDP_LDW(rq->engine, entry)); intel_ring_emit(ring, (u32)(val)); intel_ring_advance(ring); return 0; } -static int gen8_mm_switch(struct i915_hw_ppgtt *ppgtt, - struct intel_engine_cs *ring, - bool synchronous) +static int gen8_mm_switch(struct i915_gem_request *rq, + struct i915_hw_ppgtt *ppgtt) { int i, ret; @@ -255,7 +233,7 @@ static int gen8_mm_switch(struct i915_hw_ppgtt *ppgtt, for (i = used_pd - 1; i >= 0; i--) { dma_addr_t addr = ppgtt->pd_dma_addr[i]; - ret = gen8_write_pdp(ring, i, addr, synchronous); + ret = gen8_write_pdp(rq, i, addr); if (ret) return ret; } @@ -403,9 +381,6 @@ static void gen8_ppgtt_cleanup(struct i915_address_space *vm) struct i915_hw_ppgtt *ppgtt = container_of(vm, struct i915_hw_ppgtt, base); - list_del(&vm->global_link); - drm_mm_takedown(&vm->mm); - gen8_ppgtt_unmap_pages(ppgtt); gen8_ppgtt_free(ppgtt); } @@ -615,7 +590,6 @@ static int gen8_ppgtt_init(struct i915_hw_ppgtt *ppgtt, uint64_t size) kunmap_atomic(pd_vaddr); } - ppgtt->enable = gen8_ppgtt_enable; ppgtt->switch_mm = gen8_mm_switch; ppgtt->base.clear_range = gen8_ppgtt_clear_range; ppgtt->base.insert_entries = gen8_ppgtt_insert_entries; @@ -723,154 +697,53 @@ static uint32_t get_pd_offset(struct i915_hw_ppgtt *ppgtt) return (ppgtt->pd_offset / 64) << 16; } -static int hsw_mm_switch(struct i915_hw_ppgtt *ppgtt, - struct intel_engine_cs *ring, - bool synchronous) +static int gen7_mm_switch(struct i915_gem_request *rq, + struct i915_hw_ppgtt *ppgtt) { - struct drm_device *dev = ppgtt->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct intel_ringbuffer *ring; int ret; - /* If we're in reset, we can assume the GPU is sufficiently idle to - * manually frob these bits. Ideally we could use the ring functions, - * except our error handling makes it quite difficult (can't use - * intel_ring_begin, ring->flush, or intel_ring_advance) - * - * FIXME: We should try not to special case reset - */ - if (synchronous || - i915_reset_in_progress(&dev_priv->gpu_error)) { - WARN_ON(ppgtt != dev_priv->mm.aliasing_ppgtt); - I915_WRITE(RING_PP_DIR_DCLV(ring), PP_DIR_DCLV_2G); - I915_WRITE(RING_PP_DIR_BASE(ring), get_pd_offset(ppgtt)); - POSTING_READ(RING_PP_DIR_BASE(ring)); - return 0; - } - - /* NB: TLBs must be flushed and invalidated before a switch */ - ret = ring->flush(ring, I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS); + ret = i915_request_emit_flush(rq, I915_COMMAND_BARRIER); if (ret) return ret; - ret = intel_ring_begin(ring, 6); - if (ret) - return ret; + ring = intel_ring_begin(rq, 5); + if (IS_ERR(ring)) + return PTR_ERR(ring); intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(2)); - intel_ring_emit(ring, RING_PP_DIR_DCLV(ring)); + intel_ring_emit(ring, RING_PP_DIR_DCLV(rq->engine)); intel_ring_emit(ring, PP_DIR_DCLV_2G); - intel_ring_emit(ring, RING_PP_DIR_BASE(ring)); + intel_ring_emit(ring, RING_PP_DIR_BASE(rq->engine)); intel_ring_emit(ring, get_pd_offset(ppgtt)); - intel_ring_emit(ring, MI_NOOP); intel_ring_advance(ring); - return 0; -} - -static int gen7_mm_switch(struct i915_hw_ppgtt *ppgtt, - struct intel_engine_cs *ring, - bool synchronous) -{ - struct drm_device *dev = ppgtt->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; - int ret; - - /* If we're in reset, we can assume the GPU is sufficiently idle to - * manually frob these bits. Ideally we could use the ring functions, - * except our error handling makes it quite difficult (can't use - * intel_ring_begin, ring->flush, or intel_ring_advance) - * - * FIXME: We should try not to special case reset - */ - if (synchronous || - i915_reset_in_progress(&dev_priv->gpu_error)) { - WARN_ON(ppgtt != dev_priv->mm.aliasing_ppgtt); - I915_WRITE(RING_PP_DIR_DCLV(ring), PP_DIR_DCLV_2G); - I915_WRITE(RING_PP_DIR_BASE(ring), get_pd_offset(ppgtt)); - POSTING_READ(RING_PP_DIR_BASE(ring)); - return 0; - } - - /* NB: TLBs must be flushed and invalidated before a switch */ - ret = ring->flush(ring, I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS); - if (ret) - return ret; - - ret = intel_ring_begin(ring, 6); - if (ret) - return ret; - - intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(2)); - intel_ring_emit(ring, RING_PP_DIR_DCLV(ring)); - intel_ring_emit(ring, PP_DIR_DCLV_2G); - intel_ring_emit(ring, RING_PP_DIR_BASE(ring)); - intel_ring_emit(ring, get_pd_offset(ppgtt)); - intel_ring_emit(ring, MI_NOOP); - intel_ring_advance(ring); - - /* XXX: RCS is the only one to auto invalidate the TLBs? */ - if (ring->id != RCS) { - ret = ring->flush(ring, I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS); - if (ret) - return ret; - } + rq->pending_flush |= I915_INVALIDATE_CACHES; return 0; } -static int gen6_mm_switch(struct i915_hw_ppgtt *ppgtt, - struct intel_engine_cs *ring, - bool synchronous) +static int gen6_mm_switch(struct i915_gem_request *rq, + struct i915_hw_ppgtt *ppgtt) { - struct drm_device *dev = ppgtt->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; - - if (!synchronous) - return 0; - - I915_WRITE(RING_PP_DIR_DCLV(ring), PP_DIR_DCLV_2G); - I915_WRITE(RING_PP_DIR_BASE(ring), get_pd_offset(ppgtt)); - - POSTING_READ(RING_PP_DIR_DCLV(ring)); - - return 0; + return -ENODEV; } -static int gen8_ppgtt_enable(struct i915_hw_ppgtt *ppgtt) +static void gen8_ppgtt_enable(struct drm_device *dev) { - struct drm_device *dev = ppgtt->base.dev; struct drm_i915_private *dev_priv = dev->dev_private; - struct intel_engine_cs *ring; - int j, ret; + struct intel_engine_cs *engine; + int j; - for_each_ring(ring, dev_priv, j) { - I915_WRITE(RING_MODE_GEN7(ring), + for_each_engine(engine, dev_priv, j) + I915_WRITE(RING_MODE_GEN7(engine), _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE)); - - /* We promise to do a switch later with FULL PPGTT. If this is - * aliasing, this is the one and only switch we'll do */ - if (USES_FULL_PPGTT(dev)) - continue; - - ret = ppgtt->switch_mm(ppgtt, ring, true); - if (ret) - goto err_out; - } - - return 0; - -err_out: - for_each_ring(ring, dev_priv, j) - I915_WRITE(RING_MODE_GEN7(ring), - _MASKED_BIT_DISABLE(GFX_PPGTT_ENABLE)); - return ret; } -static int gen7_ppgtt_enable(struct i915_hw_ppgtt *ppgtt) +static void gen7_ppgtt_enable(struct drm_device *dev) { - struct drm_device *dev = ppgtt->base.dev; struct drm_i915_private *dev_priv = dev->dev_private; - struct intel_engine_cs *ring; + struct intel_engine_cs *engine; uint32_t ecochk, ecobits; int i; @@ -886,31 +759,25 @@ static int gen7_ppgtt_enable(struct i915_hw_ppgtt *ppgtt) } I915_WRITE(GAM_ECOCHK, ecochk); - for_each_ring(ring, dev_priv, i) { - int ret; + for_each_engine(engine, dev_priv, i) { /* GFX_MODE is per-ring on gen7+ */ - I915_WRITE(RING_MODE_GEN7(ring), + I915_WRITE(RING_MODE_GEN7(engine), _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE)); - /* We promise to do a switch later with FULL PPGTT. If this is - * aliasing, this is the one and only switch we'll do */ - if (USES_FULL_PPGTT(dev)) - continue; + if (dev_priv->mm.aliasing_ppgtt) { + I915_WRITE(RING_PP_DIR_DCLV(engine), PP_DIR_DCLV_2G); + I915_WRITE(RING_PP_DIR_BASE(engine), get_pd_offset(dev_priv->mm.aliasing_ppgtt)); + } - ret = ppgtt->switch_mm(ppgtt, ring, true); - if (ret) - return ret; } - - return 0; + POSTING_READ(RING_PP_DIR_DCLV(RCS_ENGINE(dev_priv))); } -static int gen6_ppgtt_enable(struct i915_hw_ppgtt *ppgtt) +static void gen6_ppgtt_enable(struct drm_device *dev) { - struct drm_device *dev = ppgtt->base.dev; struct drm_i915_private *dev_priv = dev->dev_private; - struct intel_engine_cs *ring; uint32_t ecochk, gab_ctl, ecobits; + struct intel_engine_cs *engine; int i; ecobits = I915_READ(GAC_ECO_BITS); @@ -925,13 +792,14 @@ static int gen6_ppgtt_enable(struct i915_hw_ppgtt *ppgtt) I915_WRITE(GFX_MODE, _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE)); - for_each_ring(ring, dev_priv, i) { - int ret = ppgtt->switch_mm(ppgtt, ring, true); - if (ret) - return ret; - } + for_each_engine(engine, dev_priv, i) { + if (dev_priv->mm.aliasing_ppgtt) { + I915_WRITE(RING_PP_DIR_DCLV(engine), PP_DIR_DCLV_2G); + I915_WRITE(RING_PP_DIR_BASE(engine), get_pd_offset(dev_priv->mm.aliasing_ppgtt)); + } - return 0; + } + POSTING_READ(RING_PP_DIR_DCLV(RCS_ENGINE(dev_priv))); } /* PPGTT support for Sandybdrige/Gen6 and later */ @@ -1029,8 +897,6 @@ static void gen6_ppgtt_cleanup(struct i915_address_space *vm) struct i915_hw_ppgtt *ppgtt = container_of(vm, struct i915_hw_ppgtt, base); - list_del(&vm->global_link); - drm_mm_takedown(&ppgtt->base.mm); drm_mm_remove_node(&ppgtt->node); gen6_ppgtt_unmap_pages(ppgtt); @@ -1151,13 +1017,8 @@ static int gen6_ppgtt_init(struct i915_hw_ppgtt *ppgtt) ppgtt->base.pte_encode = dev_priv->gtt.base.pte_encode; if (IS_GEN6(dev)) { - ppgtt->enable = gen6_ppgtt_enable; ppgtt->switch_mm = gen6_mm_switch; - } else if (IS_HASWELL(dev)) { - ppgtt->enable = gen7_ppgtt_enable; - ppgtt->switch_mm = hsw_mm_switch; } else if (IS_GEN7(dev)) { - ppgtt->enable = gen7_ppgtt_enable; ppgtt->switch_mm = gen7_mm_switch; } else BUG(); @@ -1188,40 +1049,96 @@ static int gen6_ppgtt_init(struct i915_hw_ppgtt *ppgtt) ppgtt->node.size >> 20, ppgtt->node.start / PAGE_SIZE); + gen6_write_pdes(ppgtt); + DRM_DEBUG("Adding PPGTT at offset %x\n", + ppgtt->pd_offset << 10); + return 0; } -int i915_gem_init_ppgtt(struct drm_device *dev, struct i915_hw_ppgtt *ppgtt) +static int __hw_ppgtt_init(struct drm_device *dev, struct i915_hw_ppgtt *ppgtt) { struct drm_i915_private *dev_priv = dev->dev_private; - int ret = 0; ppgtt->base.dev = dev; ppgtt->base.scratch = dev_priv->gtt.base.scratch; if (INTEL_INFO(dev)->gen < 8) - ret = gen6_ppgtt_init(ppgtt); + return gen6_ppgtt_init(ppgtt); else if (IS_GEN8(dev)) - ret = gen8_ppgtt_init(ppgtt, dev_priv->gtt.base.total); + return gen8_ppgtt_init(ppgtt, dev_priv->gtt.base.total); else BUG(); +} +int i915_ppgtt_init(struct drm_device *dev, struct i915_hw_ppgtt *ppgtt) +{ + struct drm_i915_private *dev_priv = dev->dev_private; + int ret = 0; - if (!ret) { - struct drm_i915_private *dev_priv = dev->dev_private; + ret = __hw_ppgtt_init(dev, ppgtt); + if (ret == 0) { kref_init(&ppgtt->ref); drm_mm_init(&ppgtt->base.mm, ppgtt->base.start, ppgtt->base.total); i915_init_vm(dev_priv, &ppgtt->base); - if (INTEL_INFO(dev)->gen < 8) { - gen6_write_pdes(ppgtt); - DRM_DEBUG("Adding PPGTT at offset %x\n", - ppgtt->pd_offset << 10); - } } return ret; } +int i915_ppgtt_init_hw(struct drm_device *dev) +{ + if (!USES_PPGTT(dev)) + return 0; + + if (IS_GEN6(dev)) + gen6_ppgtt_enable(dev); + else if (IS_GEN7(dev)) + gen7_ppgtt_enable(dev); + else if (INTEL_INFO(dev)->gen >= 8) + gen8_ppgtt_enable(dev); + else + WARN_ON(1); + + return 0; +} +struct i915_hw_ppgtt * +i915_ppgtt_create(struct drm_device *dev, struct drm_i915_file_private *fpriv) +{ + struct i915_hw_ppgtt *ppgtt; + int ret; + + ppgtt = kzalloc(sizeof(*ppgtt), GFP_KERNEL); + if (!ppgtt) + return ERR_PTR(-ENOMEM); + + ret = i915_ppgtt_init(dev, ppgtt); + if (ret) { + kfree(ppgtt); + return ERR_PTR(ret); + } + + ppgtt->file_priv = fpriv; + + return ppgtt; +} + +void i915_ppgtt_release(struct kref *kref) +{ + struct i915_hw_ppgtt *ppgtt = + container_of(kref, struct i915_hw_ppgtt, ref); + + /* vmas should already be unbound */ + WARN_ON(!list_empty(&ppgtt->base.active_list)); + WARN_ON(!list_empty(&ppgtt->base.inactive_list)); + + list_del(&ppgtt->base.global_link); + drm_mm_takedown(&ppgtt->base.mm); + + ppgtt->base.cleanup(&ppgtt->base); + kfree(ppgtt); +} + static void ppgtt_bind_vma(struct i915_vma *vma, enum i915_cache_level cache_level, @@ -1233,6 +1150,7 @@ ppgtt_bind_vma(struct i915_vma *vma, vma->vm->insert_entries(vma->vm, vma->obj->pages, vma->node.start, cache_level, flags); + vma->vm->dirty = true; } static void ppgtt_unbind_vma(struct i915_vma *vma) @@ -1284,15 +1202,15 @@ static void undo_idling(struct drm_i915_private *dev_priv, bool interruptible) void i915_check_and_clear_faults(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; - struct intel_engine_cs *ring; + struct intel_engine_cs *engine; int i; if (INTEL_INFO(dev)->gen < 6) return; - for_each_ring(ring, dev_priv, i) { + for_each_engine(engine, dev_priv, i) { u32 fault_reg; - fault_reg = I915_READ(RING_FAULT_REG(ring)); + fault_reg = I915_READ(RING_FAULT_REG(engine)); if (fault_reg & RING_FAULT_VALID) { DRM_DEBUG_DRIVER("Unexpected fault\n" "\tAddr: 0x%08lx\\n" @@ -1303,21 +1221,11 @@ void i915_check_and_clear_faults(struct drm_device *dev) fault_reg & RING_FAULT_GTTSEL_MASK ? "GGTT" : "PPGTT", RING_FAULT_SRCID(fault_reg), RING_FAULT_FAULT_TYPE(fault_reg)); - I915_WRITE(RING_FAULT_REG(ring), + I915_WRITE(RING_FAULT_REG(engine), fault_reg & ~RING_FAULT_VALID); } } - POSTING_READ(RING_FAULT_REG(&dev_priv->ring[RCS])); -} - -static void i915_ggtt_flush(struct drm_i915_private *dev_priv) -{ - if (INTEL_INFO(dev_priv->dev)->gen < 6) { - intel_gtt_chipset_flush(); - } else { - I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN); - POSTING_READ(GFX_FLSH_CNTL_GEN6); - } + POSTING_READ(RING_FAULT_REG(RCS_ENGINE(dev_priv))); } void i915_gem_suspend_gtt_mappings(struct drm_device *dev) @@ -1336,8 +1244,6 @@ void i915_gem_suspend_gtt_mappings(struct drm_device *dev) dev_priv->gtt.base.start, dev_priv->gtt.base.total, true); - - i915_ggtt_flush(dev_priv); } void i915_gem_restore_gtt_mappings(struct drm_device *dev) @@ -1390,7 +1296,7 @@ void i915_gem_restore_gtt_mappings(struct drm_device *dev) gen6_write_pdes(container_of(vm, struct i915_hw_ppgtt, base)); } - i915_ggtt_flush(dev_priv); + i915_gem_chipset_flush(dev); } int i915_gem_gtt_prepare_object(struct drm_i915_gem_object *obj) @@ -1562,6 +1468,7 @@ static void i915_ggtt_bind_vma(struct i915_vma *vma, BUG_ON(!i915_is_ggtt(vma->vm)); intel_gtt_insert_sg_entries(vma->obj->pages, entry, flags); + vma->vm->dirty = true; vma->obj->has_global_gtt_mapping = 1; } @@ -1572,6 +1479,7 @@ static void i915_ggtt_clear_range(struct i915_address_space *vm, { unsigned first_entry = start >> PAGE_SHIFT; unsigned num_entries = length >> PAGE_SHIFT; + intel_gtt_clear_range(first_entry, num_entries); } @@ -1614,6 +1522,7 @@ static void ggtt_bind_vma(struct i915_vma *vma, vma->vm->insert_entries(vma->vm, obj->pages, vma->node.start, cache_level, flags); + vma->vm->dirty = true; obj->has_global_gtt_mapping = 1; } } @@ -1626,6 +1535,7 @@ static void ggtt_bind_vma(struct i915_vma *vma, vma->obj->pages, vma->node.start, cache_level, flags); + vma->vm->dirty = true; vma->obj->has_aliasing_ppgtt_mapping = 1; } } @@ -1687,10 +1597,10 @@ static void i915_gtt_color_adjust(struct drm_mm_node *node, } } -void i915_gem_setup_global_gtt(struct drm_device *dev, - unsigned long start, - unsigned long mappable_end, - unsigned long end) +int i915_gem_setup_global_gtt(struct drm_device *dev, + unsigned long start, + unsigned long mappable_end, + unsigned long end) { /* Let GEM Manage all of the aperture. * @@ -1706,6 +1616,7 @@ void i915_gem_setup_global_gtt(struct drm_device *dev, struct drm_mm_node *entry; struct drm_i915_gem_object *obj; unsigned long hole_start, hole_end; + int ret; BUG_ON(mappable_end > end); @@ -1717,14 +1628,16 @@ void i915_gem_setup_global_gtt(struct drm_device *dev, /* Mark any preallocated objects as occupied */ list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) { struct i915_vma *vma = i915_gem_obj_to_vma(obj, ggtt_vm); - int ret; + DRM_DEBUG_KMS("reserving preallocated space: %lx + %zx\n", i915_gem_obj_ggtt_offset(obj), obj->base.size); WARN_ON(i915_gem_obj_ggtt_bound(obj)); ret = drm_mm_reserve_node(&ggtt_vm->mm, &vma->node); - if (ret) - DRM_DEBUG_KMS("Reservation failed\n"); + if (ret) { + DRM_DEBUG_KMS("Reservation failed: %i\n", ret); + return ret; + } obj->has_global_gtt_mapping = 1; } @@ -1741,6 +1654,22 @@ void i915_gem_setup_global_gtt(struct drm_device *dev, /* And finally clear the reserved guard page */ ggtt_vm->clear_range(ggtt_vm, end - PAGE_SIZE, PAGE_SIZE, true); + + if (USES_PPGTT(dev) && !USES_FULL_PPGTT(dev)) { + struct i915_hw_ppgtt *ppgtt; + + ppgtt = kzalloc(sizeof(*ppgtt), GFP_KERNEL); + if (!ppgtt) + return -ENOMEM; + + ret = __hw_ppgtt_init(dev, ppgtt); + if (ret != 0) + return ret; + + dev_priv->mm.aliasing_ppgtt = ppgtt; + } + + return 0; } void i915_gem_init_global_gtt(struct drm_device *dev) @@ -1754,6 +1683,25 @@ void i915_gem_init_global_gtt(struct drm_device *dev) i915_gem_setup_global_gtt(dev, 0, mappable_size, gtt_size); } +void i915_global_gtt_cleanup(struct drm_device *dev) +{ + struct drm_i915_private *dev_priv = dev->dev_private; + struct i915_address_space *vm = &dev_priv->gtt.base; + + if (dev_priv->mm.aliasing_ppgtt) { + struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt; + + ppgtt->base.cleanup(&ppgtt->base); + } + + if (drm_mm_initialized(&vm->mm)) { + drm_mm_takedown(&vm->mm); + list_del(&vm->global_link); + } + + vm->cleanup(vm); +} + static int setup_scratch_page(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; @@ -1763,7 +1711,6 @@ static int setup_scratch_page(struct drm_device *dev) page = alloc_page(GFP_KERNEL | GFP_DMA32 | __GFP_ZERO); if (page == NULL) return -ENOMEM; - get_page(page); set_pages_uc(page, 1); #ifdef CONFIG_INTEL_IOMMU @@ -1788,7 +1735,6 @@ static void teardown_scratch_page(struct drm_device *dev) set_pages_wb(page, 1); pci_unmap_page(dev->pdev, dev_priv->gtt.base.scratch.addr, PAGE_SIZE, PCI_DMA_BIDIRECTIONAL); - put_page(page); __free_page(page); } @@ -2022,10 +1968,6 @@ static void gen6_gmch_remove(struct i915_address_space *vm) struct i915_gtt *gtt = container_of(vm, struct i915_gtt, base); - if (drm_mm_initialized(&vm->mm)) { - drm_mm_takedown(&vm->mm); - list_del(&vm->global_link); - } iounmap(gtt->gsm); teardown_scratch_page(vm->dev); } @@ -2058,10 +2000,6 @@ static int i915_gmch_probe(struct drm_device *dev, static void i915_gmch_remove(struct i915_address_space *vm) { - if (drm_mm_initialized(&vm->mm)) { - drm_mm_takedown(&vm->mm); - list_del(&vm->global_link); - } intel_gmch_remove(); } @@ -2160,8 +2098,10 @@ static struct i915_vma *__i915_gem_vma_create(struct drm_i915_gem_object *obj, /* Keep GGTT vmas first to make debug easier */ if (i915_is_ggtt(vm)) list_add(&vma->vma_link, &obj->vma_list); - else + else { list_add_tail(&vma->vma_link, &obj->vma_list); + i915_ppgtt_get(i915_vm_to_ppgtt(vm)); + } return vma; } diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.h b/drivers/gpu/drm/i915/i915_gem_gtt.h index 8d6f7c1..208b96b 100644 --- a/drivers/gpu/drm/i915/i915_gem_gtt.h +++ b/drivers/gpu/drm/i915/i915_gem_gtt.h @@ -34,6 +34,8 @@ #ifndef __I915_GEM_GTT_H__ #define __I915_GEM_GTT_H__ +struct drm_i915_file_private; + typedef uint32_t gen6_gtt_pte_t; typedef uint64_t gen8_gtt_pte_t; typedef gen8_gtt_pte_t gen8_ppgtt_pde_t; @@ -167,6 +169,8 @@ struct i915_address_space { unsigned long start; /* Start offset always 0 for dri2 */ size_t total; /* size addr space maps (ex. 2GB for ggtt) */ + bool dirty; + struct { dma_addr_t addr; struct page *page; @@ -258,22 +262,36 @@ struct i915_hw_ppgtt { dma_addr_t *gen8_pt_dma_addr[4]; }; - struct intel_context *ctx; + struct drm_i915_file_private *file_priv; int (*enable)(struct i915_hw_ppgtt *ppgtt); - int (*switch_mm)(struct i915_hw_ppgtt *ppgtt, - struct intel_engine_cs *ring, - bool synchronous); + int (*switch_mm)(struct i915_gem_request *rq, + struct i915_hw_ppgtt *ppgtt); void (*debug_dump)(struct i915_hw_ppgtt *ppgtt, struct seq_file *m); }; int i915_gem_gtt_init(struct drm_device *dev); void i915_gem_init_global_gtt(struct drm_device *dev); -void i915_gem_setup_global_gtt(struct drm_device *dev, unsigned long start, - unsigned long mappable_end, unsigned long end); - -bool intel_enable_ppgtt(struct drm_device *dev, bool full); -int i915_gem_init_ppgtt(struct drm_device *dev, struct i915_hw_ppgtt *ppgtt); +int i915_gem_setup_global_gtt(struct drm_device *dev, unsigned long start, + unsigned long mappable_end, unsigned long end); +void i915_global_gtt_cleanup(struct drm_device *dev); + + +int i915_ppgtt_init(struct drm_device *dev, struct i915_hw_ppgtt *ppgtt); +int i915_ppgtt_init_hw(struct drm_device *dev); +void i915_ppgtt_release(struct kref *kref); +struct i915_hw_ppgtt *i915_ppgtt_create(struct drm_device *dev, + struct drm_i915_file_private *fpriv); +static inline void i915_ppgtt_get(struct i915_hw_ppgtt *ppgtt) +{ + if (ppgtt) + kref_get(&ppgtt->ref); +} +static inline void i915_ppgtt_put(struct i915_hw_ppgtt *ppgtt) +{ + if (ppgtt) + kref_put(&ppgtt->ref, i915_ppgtt_release); +} void i915_check_and_clear_faults(struct drm_device *dev); void i915_gem_suspend_gtt_mappings(struct drm_device *dev); diff --git a/drivers/gpu/drm/i915/i915_gem_render_state.c b/drivers/gpu/drm/i915/i915_gem_render_state.c index e60be3f..fffd26d 100644 --- a/drivers/gpu/drm/i915/i915_gem_render_state.c +++ b/drivers/gpu/drm/i915/i915_gem_render_state.c @@ -36,7 +36,7 @@ struct render_state { }; static const struct intel_renderstate_rodata * -render_state_get_rodata(struct drm_device *dev, const int gen) +render_state_get_rodata(const int gen) { switch (gen) { case 6: @@ -50,19 +50,19 @@ render_state_get_rodata(struct drm_device *dev, const int gen) return NULL; } -static int render_state_init(struct render_state *so, struct drm_device *dev) +static int render_state_init(struct render_state *so, struct i915_gem_request *rq) { int ret; - so->gen = INTEL_INFO(dev)->gen; - so->rodata = render_state_get_rodata(dev, so->gen); + so->gen = INTEL_INFO(rq->i915)->gen; + so->rodata = render_state_get_rodata(so->gen); if (so->rodata == NULL) return 0; if (so->rodata->batch_items * 4 > 4096) return -EINVAL; - so->obj = i915_gem_alloc_object(dev, 4096); + so->obj = i915_gem_alloc_object(rq->i915->dev, 4096); if (so->obj == NULL) return -ENOMEM; @@ -115,10 +115,6 @@ static int render_state_setup(struct render_state *so) } kunmap(page); - ret = i915_gem_object_set_to_gtt_domain(so->obj, false); - if (ret) - return ret; - if (rodata->reloc[reloc_index] != -1) { DRM_ERROR("only %d relocs resolved\n", reloc_index); return -EINVAL; @@ -133,15 +129,15 @@ static void render_state_fini(struct render_state *so) drm_gem_object_unreference(&so->obj->base); } -int i915_gem_render_state_init(struct intel_engine_cs *ring) +int i915_gem_render_state_init(struct i915_gem_request *rq) { struct render_state so; int ret; - if (WARN_ON(ring->id != RCS)) + if (WARN_ON(rq->engine->id != RCS)) return -ENOENT; - ret = render_state_init(&so, ring->dev); + ret = render_state_init(&so, rq); if (ret) return ret; @@ -152,16 +148,19 @@ int i915_gem_render_state_init(struct intel_engine_cs *ring) if (ret) goto out; - ret = ring->dispatch_execbuffer(ring, - so.ggtt_offset, - so.rodata->batch_items * 4, - I915_DISPATCH_SECURE); + if (i915_gem_clflush_object(so.obj, false)) + i915_gem_chipset_flush(rq->i915->dev); + + ret = i915_request_emit_batchbuffer(rq, NULL, + so.ggtt_offset, + so.rodata->batch_items * 4, + I915_DISPATCH_SECURE); if (ret) goto out; - i915_vma_move_to_active(i915_gem_obj_to_ggtt(so.obj), ring); + so.obj->base.pending_read_domains = I915_GEM_DOMAIN_COMMAND; + ret = i915_request_add_vma(rq, i915_gem_obj_to_ggtt(so.obj), 0); - ret = __i915_add_request(ring, NULL, so.obj, NULL); /* __i915_add_request moves object to inactive if it fails */ out: render_state_fini(&so); diff --git a/drivers/gpu/drm/i915/i915_gem_request.c b/drivers/gpu/drm/i915/i915_gem_request.c new file mode 100644 index 0000000..c85b526 --- /dev/null +++ b/drivers/gpu/drm/i915/i915_gem_request.c @@ -0,0 +1,666 @@ +/* + * Copyright © 2014 Intel Corporation + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the next + * paragraph) shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + * + */ + +#include +#include "i915_drv.h" +#include +#include "i915_trace.h" +#include "intel_drv.h" + +struct i915_gem_request__vma { + struct list_head link; + struct i915_vma *vma; + struct drm_i915_gem_object *obj; + u32 write, fence; +}; + +static bool check_reset(struct i915_gem_request *rq) +{ + unsigned reset = atomic_read(&rq->i915->gpu_error.reset_counter); + return likely(reset == rq->reset_counter); +} + +int +i915_request_add_vma(struct i915_gem_request *rq, + struct i915_vma *vma, + unsigned fenced) +{ + struct drm_i915_gem_object *obj = vma->obj; + u32 old_read = obj->base.read_domains; + u32 old_write = obj->base.write_domain; + struct i915_gem_request__vma *ref; + + lockdep_assert_held(&rq->i915->dev->struct_mutex); + BUG_ON(!rq->outstanding); + + obj->base.write_domain = obj->base.pending_write_domain; + if (obj->base.write_domain == 0) + obj->base.pending_read_domains |= obj->base.read_domains; + obj->base.read_domains = obj->base.pending_read_domains; + + obj->base.pending_read_domains = 0; + obj->base.pending_write_domain = 0; + + trace_i915_gem_object_change_domain(obj, old_read, old_write); + if (obj->base.read_domains == 0) + return 0; + + ref = kmalloc(sizeof(*ref), GFP_KERNEL); + if (ref == NULL) + return -ENOMEM; + + list_add(&ref->link, &rq->vmas); + ref->vma = vma; + ref->obj = obj; + drm_gem_object_reference(&obj->base); + ref->write = obj->base.write_domain & I915_GEM_GPU_DOMAINS; + ref->fence = fenced; + + if (ref->write) { + rq->pending_flush |= I915_FLUSH_CACHES; + intel_fb_obj_invalidate(obj, rq); + } + + /* update for the implicit flush after the rq */ + obj->base.write_domain &= ~I915_GEM_GPU_DOMAINS; + return 0; +} + +static void vma_free(struct i915_gem_request__vma *ref) +{ + drm_gem_object_unreference(&ref->obj->base); + list_del(&ref->link); + kfree(ref); +} + +int +i915_request_emit_flush(struct i915_gem_request *rq, + unsigned flags) +{ + struct intel_engine_cs *engine = rq->engine; + int ret; + + lockdep_assert_held(&rq->i915->dev->struct_mutex); + BUG_ON(!rq->outstanding); + + if ((flags & rq->pending_flush) == 0) + return 0; + + trace_i915_gem_request_emit_flush(rq); + ret = engine->emit_flush(rq, rq->pending_flush); + if (ret) + return ret; + + rq->pending_flush = 0; + return 0; +} + +int +__i915_request_emit_breadcrumb(struct i915_gem_request *rq, int id) +{ + struct intel_engine_cs *engine = rq->engine; + u32 seqno; + int ret; + + lockdep_assert_held(&rq->i915->dev->struct_mutex); + + if (rq->breadcrumb[id]) + return 0; + + if (rq->outstanding) { + ret = i915_request_emit_flush(rq, I915_COMMAND_BARRIER); + if (ret) + return ret; + + trace_i915_gem_request_emit_breadcrumb(rq); + if (id == engine->id) + ret = engine->emit_breadcrumb(rq); + else + ret = engine->semaphore.signal(rq, id); + if (ret) + return ret; + + seqno = rq->seqno; + } else if (engine->breadcrumb[id] == 0 || + __i915_seqno_passed(rq->seqno, engine->breadcrumb[id])) { + struct i915_gem_request *tmp; + + tmp = intel_engine_alloc_request(engine, + rq->ring->last_context); + if (IS_ERR(tmp)) + return PTR_ERR(tmp); + + /* Masquerade as a continuation of the earlier request */ + tmp->reset_counter = rq->reset_counter; + + ret = __i915_request_emit_breadcrumb(tmp, id); + if (ret == 0 && id != engine->id) { + /* semaphores are unstable across a wrap */ + if (tmp->seqno < engine->breadcrumb[id]) + ret = i915_request_wait(tmp); + } + if (ret == 0) + ret = i915_request_commit(tmp); + + i915_request_put(tmp); + if (ret) + return ret; + + seqno = tmp->seqno; + } else + seqno = engine->breadcrumb[id]; + + rq->breadcrumb[id] = seqno; + return 0; +} + +int +i915_request_emit_batchbuffer(struct i915_gem_request *rq, + struct i915_vma *batch, + uint64_t start, uint32_t len, + unsigned flags) +{ + struct intel_engine_cs *engine = rq->engine; + int ret; + + lockdep_assert_held(&rq->i915->dev->struct_mutex); + BUG_ON(!rq->outstanding); + BUG_ON(rq->breadcrumb[rq->engine->id]); + + trace_i915_gem_request_emit_batch(rq); + rq->batch = batch; + + ret = engine->emit_batchbuffer(rq, start, len, flags); + if (ret) + return ret; + + /* We track the associated batch vma for debugging and error capture. + * Whilst this request exists, the batch obj will be on the active_list, + * and so will hold the active reference. Only when this request is + * retired will the the batch be moved onto the inactive_list and lose + * its active reference. Hence we do not need to explicitly hold + * another reference here. + */ + rq->pending_flush |= I915_COMMAND_BARRIER; + return 0; +} + +/* Track the batches submitted by clients for throttling */ +static void +add_to_client(struct i915_gem_request *rq) +{ + struct drm_i915_file_private *file_priv = rq->ctx->file_priv; + + if (file_priv) { + spin_lock(&file_priv->mm.lock); + list_add_tail(&rq->client_list, + &file_priv->mm.request_list); + rq->file_priv = file_priv; + spin_unlock(&file_priv->mm.lock); + } +} + +static void +remove_from_client(struct i915_gem_request *rq) +{ + struct drm_i915_file_private *file_priv = rq->file_priv; + + if (!file_priv) + return; + + spin_lock(&file_priv->mm.lock); + if (rq->file_priv) { + list_del(&rq->client_list); + rq->file_priv = NULL; + } + spin_unlock(&file_priv->mm.lock); +} + +/* Activity tracking on the object so that we can serialise CPU access to + * the object's memory with the GPU. + */ +static void +add_to_obj(struct i915_gem_request *rq, + struct i915_gem_request__vma *ref) +{ + struct drm_i915_gem_object *obj = ref->obj; + struct intel_engine_cs *engine = rq->engine; + + /* Add a reference if we're newly entering the active list. */ + if (obj->last_read[engine->id].request == NULL && obj->active++ == 0) + drm_gem_object_reference(&obj->base); + + if (ref->write) { + obj->dirty = 1; + i915_request_put(obj->last_write.request); + obj->last_write.request = i915_request_get(rq); + list_move_tail(&obj->last_write.engine_list, + &engine->write_list); + + if (obj->active > 1) { + int i; + + for (i = 0; i < I915_NUM_ENGINES; i++) { + if (obj->last_read[i].request == NULL) + continue; + + list_del_init(&obj->last_read[i].engine_list); + i915_request_put(obj->last_read[i].request); + obj->last_read[i].request = NULL; + } + + obj->active = 1; + } + } + + if (ref->fence & VMA_IS_FENCED) { + i915_request_put(obj->last_fence.request); + obj->last_fence.request = i915_request_get(rq); + list_move_tail(&obj->last_fence.engine_list, + &engine->fence_list); + if (ref->fence & VMA_HAS_FENCE) + list_move_tail(&rq->i915->fence_regs[obj->fence_reg].lru_list, + &rq->i915->mm.fence_list); + } + + i915_request_put(obj->last_read[engine->id].request); + obj->last_read[engine->id].request = i915_request_get(rq); + list_move_tail(&obj->last_read[engine->id].engine_list, + &engine->read_list); + + list_move_tail(&ref->vma->mm_list, &ref->vma->vm->active_list); +} + +static bool leave_breadcrumb(struct i915_gem_request *rq) +{ + if (rq->breadcrumb[rq->engine->id]) + return false; + + /* Auto-report HEAD every 4k to make sure that we can always wait on + * some available ring space in the future. This also caps the + * latency of future waits for missed breadcrumbs. + */ + if (__intel_ring_space(rq->ring->tail, rq->ring->breadcrumb_tail, + rq->ring->size, 0) >= PAGE_SIZE) + return true; + + return false; +} + +static bool simulated_hang(struct intel_engine_cs *engine) +{ + return test_and_clear_bit(engine->id, + &engine->i915->gpu_error.stop_rings); +} + +int i915_request_commit(struct i915_gem_request *rq) +{ + int ret, n; + + lockdep_assert_held(&rq->i915->dev->struct_mutex); + + if (!rq->outstanding) + return 0; + + if (rq->head == rq->ring->tail) { + rq->completed = true; + goto done; + } + + if (simulated_hang(rq->engine)) + i915_handle_error(rq->i915->dev, true, "Simulated hang"); + + if (!check_reset(rq)) + return rq->i915->mm.interruptible ? -EAGAIN : -EIO; + + if (leave_breadcrumb(rq)) { + ret = i915_request_emit_breadcrumb(rq); + if (ret) + return ret; + } + + /* TAIL must be aligned to a qword */ + if ((rq->ring->tail / sizeof (uint32_t)) & 1) { + intel_ring_emit(rq->ring, MI_NOOP); + intel_ring_advance(rq->ring); + } + rq->tail = rq->ring->tail; + rq->emitted_jiffies = jiffies; + + intel_runtime_pm_get(rq->i915); + + trace_i915_gem_request_commit(rq); + ret = rq->engine->add_request(rq); + if (ret) { + intel_runtime_pm_put(rq->i915); + return ret; + } + + i915_request_get(rq); + + rq->outstanding = false; + if (rq->breadcrumb[rq->engine->id]) { + list_add_tail(&rq->breadcrumb_link, &rq->ring->breadcrumbs); + rq->ring->breadcrumb_tail = rq->tail; + } + + memcpy(rq->engine->semaphore.sync, + rq->semaphore, + sizeof(rq->semaphore)); + for (n = 0; n < ARRAY_SIZE(rq->breadcrumb); n++) + if (rq->breadcrumb[n]) + rq->engine->breadcrumb[n] = rq->breadcrumb[n]; + + rq->ring->pending_flush = rq->pending_flush; + + if (rq->batch) { + add_to_client(rq); + while (!list_empty(&rq->vmas)) { + struct i915_gem_request__vma *ref = + list_first_entry(&rq->vmas, typeof(*ref), link); + + add_to_obj(rq, ref); + vma_free(ref); + } + rq->batch->vm->dirty = false; + } + + i915_request_switch_context__commit(rq); + + rq->engine->last_request = rq; +done: + rq->ring->last_context = rq->ctx; + return 0; +} + +static void fake_irq(unsigned long data) +{ + wake_up_process((struct task_struct *)data); +} + +static bool missed_irq(struct i915_gem_request *rq) +{ + return test_bit(rq->engine->id, &rq->i915->gpu_error.missed_irq_rings); +} + +static bool can_wait_boost(struct drm_i915_file_private *file_priv) +{ + if (file_priv == NULL) + return true; + + return !atomic_xchg(&file_priv->rps_wait_boost, true); +} + +bool __i915_request_complete__wa(struct i915_gem_request *rq) +{ + struct drm_i915_private *dev_priv = rq->i915; + unsigned head, tail; + + if (i915_request_complete(rq)) + return true; + + /* With execlists, we rely on interrupts to track request completion */ + if (rq->engine->execlists_enabled) + return false; + + /* As we may not emit a breadcrumb with every request, we + * often have unflushed requests. In the event of an emergency, + * just assume that if the RING_HEAD has reached the tail, then + * the request is complete. However, note that the RING_HEAD + * advances before the instruction completes, so this is quite lax, + * and should only be used carefully. + * + * As we treat this as only an advisory completion, we forgo + * marking the request as actually complete. + */ + head = __intel_ring_space(I915_READ_HEAD(rq->engine) & HEAD_ADDR, + rq->ring->tail, rq->ring->size, 0); + tail = __intel_ring_space(rq->tail, + rq->ring->tail, rq->ring->size, 0); + return head >= tail; +} + +/** + * __i915_request_wait - wait until execution of request has finished + * @request: the request to wait upon + * @interruptible: do an interruptible wait (normally yes) + * @timeout_ns: in - how long to wait (NULL forever); out - how much time remaining + * + * Returns 0 if the request was completed within the alloted time. Else returns the + * errno with remaining time filled in timeout argument. + */ +int __i915_request_wait(struct i915_gem_request *rq, + bool interruptible, + s64 *timeout_ns, + struct drm_i915_file_private *file_priv) +{ + const bool irq_test_in_progress = + ACCESS_ONCE(rq->i915->gpu_error.test_irq_rings) & intel_engine_flag(rq->engine); + DEFINE_WAIT(wait); + unsigned long timeout_expire; + unsigned long before, now; + int ret = 0; + + WARN(!intel_irqs_enabled(rq->i915), "IRQs disabled"); + + if (i915_request_complete(rq)) + return 0; + + timeout_expire = timeout_ns ? jiffies + nsecs_to_jiffies((u64)*timeout_ns) : 0; + + if (INTEL_INFO(rq->i915)->gen >= 6 && rq->engine->id == RCS && can_wait_boost(file_priv)) { + gen6_rps_boost(rq->i915); + if (file_priv) + mod_delayed_work(rq->i915->wq, + &file_priv->mm.idle_work, + msecs_to_jiffies(100)); + } + + if (!irq_test_in_progress) { + if (WARN_ON(!intel_irqs_enabled(rq->i915))) + return -ENODEV; + + rq->engine->irq_get(rq->engine); + } + + /* Record current time in case interrupted by signal, or wedged */ + trace_i915_gem_request_wait_begin(rq); + before = jiffies; + for (;;) { + struct timer_list timer; + + prepare_to_wait(&rq->engine->irq_queue, &wait, + interruptible ? TASK_INTERRUPTIBLE : TASK_UNINTERRUPTIBLE); + + if (!check_reset(rq)) + break; + + rq->engine->irq_barrier(rq->engine); + + if (i915_request_complete(rq)) + break; + + if (timeout_ns && time_after_eq(jiffies, timeout_expire)) { + ret = -ETIME; + break; + } + + if (interruptible && signal_pending(current)) { + ret = -ERESTARTSYS; + break; + } + + /* Paranoid kick of hangcheck so that we never wait forever */ + i915_queue_hangcheck(rq->i915->dev); + + timer.function = NULL; + if (timeout_ns || missed_irq(rq)) { + unsigned long expire; + + setup_timer_on_stack(&timer, fake_irq, (unsigned long)current); + expire = missed_irq(rq) ? jiffies + 1 : timeout_expire; + mod_timer(&timer, expire); + } + + io_schedule(); + + if (timer.function) { + del_singleshot_timer_sync(&timer); + destroy_timer_on_stack(&timer); + } + } + now = jiffies; + trace_i915_gem_request_wait_end(rq); + + if (!irq_test_in_progress) + rq->engine->irq_put(rq->engine); + + finish_wait(&rq->engine->irq_queue, &wait); + + if (timeout_ns) { + s64 tres = *timeout_ns - jiffies_to_nsecs(now - before); + *timeout_ns = tres <= 0 ? 0 : tres; + } + + return ret; +} + +struct i915_gem_request * +i915_request_get_breadcrumb(struct i915_gem_request *rq) +{ + struct list_head *list; + u32 seqno; + int ret; + + /* Writes are only coherent from the cpu (in the general case) when + * the interrupt following the write to memory is complete. That is + * when the breadcrumb after the write request is complete. + * + * Reads are only complete when then command streamer barrier is + * passed. + * + * In both cases, the CPU needs to wait upon the subsequent breadcrumb, + * which ensures that all pending flushes have been emitted and are + * complete, before reporting that the request is finished and + * the CPU's view of memory is coherent with the GPU. + */ + + ret = i915_request_emit_breadcrumb(rq); + if (ret) + return ERR_PTR(ret); + + ret = i915_request_commit(rq); + if (ret) + return ERR_PTR(ret); + + if (!list_empty(&rq->breadcrumb_link)) + return i915_request_get(rq); + + seqno = rq->breadcrumb[rq->engine->id]; + list = &rq->ring->breadcrumbs; + list_for_each_entry_reverse(rq, list, breadcrumb_link) { + if (rq->seqno == seqno) + return i915_request_get(rq); + } + + return ERR_PTR(-EIO); +} + +int +i915_request_wait(struct i915_gem_request *rq) +{ + int ret; + + lockdep_assert_held(&rq->i915->dev->struct_mutex); + + rq = i915_request_get_breadcrumb(rq); + if (IS_ERR(rq)) + return PTR_ERR(rq); + + ret = __i915_request_wait(rq, rq->i915->mm.interruptible, + NULL, NULL); + i915_request_put(rq); + + return ret; +} + +void +i915_request_retire(struct i915_gem_request *rq) +{ + lockdep_assert_held(&rq->i915->dev->struct_mutex); + + if (!rq->completed) { + trace_i915_gem_request_complete(rq); + rq->completed = true; + } + trace_i915_gem_request_retire(rq); + + /* We know the GPU must have read the request to have + * sent us the seqno + interrupt, we can use the position + * of tail of the request to update the last known position + * of the GPU head. + */ + if (!list_empty(&rq->breadcrumb_link)) + rq->ring->retired_head = rq->tail; + + rq->batch = NULL; + + /* We need to protect against simultaneous hangcheck/capture */ + spin_lock(&rq->engine->lock); + if (rq->engine->last_request == rq) + rq->engine->last_request = NULL; + list_del(&rq->engine_list); + spin_unlock(&rq->engine->lock); + + list_del(&rq->breadcrumb_link); + remove_from_client(rq); + + intel_runtime_pm_put(rq->i915); + i915_request_put(rq); +} + +void +__i915_request_free(struct kref *kref) +{ + struct i915_gem_request *rq = container_of(kref, struct i915_gem_request, kref); + + lockdep_assert_held(&rq->i915->dev->struct_mutex); + + if (rq->outstanding) { + /* Rollback this partial transaction as we never committed + * the request to the hardware queue. + */ + rq->ring->tail = rq->head; + rq->ring->space = intel_ring_space(rq->ring); + } + + while (!list_empty(&rq->vmas)) + vma_free(list_first_entry(&rq->vmas, + struct i915_gem_request__vma, + link)); + + i915_request_switch_context__undo(rq); + i915_gem_context_unreference(rq->ctx); + kfree(rq); +} diff --git a/drivers/gpu/drm/i915/i915_gem_stolen.c b/drivers/gpu/drm/i915/i915_gem_stolen.c index 21c025a..85fda6b 100644 --- a/drivers/gpu/drm/i915/i915_gem_stolen.c +++ b/drivers/gpu/drm/i915/i915_gem_stolen.c @@ -289,6 +289,7 @@ void i915_gem_cleanup_stolen(struct drm_device *dev) int i915_gem_init_stolen(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; + u32 tmp; int bios_reserved = 0; #ifdef CONFIG_INTEL_IOMMU @@ -308,8 +309,16 @@ int i915_gem_init_stolen(struct drm_device *dev) DRM_DEBUG_KMS("found %zd bytes of stolen memory at %08lx\n", dev_priv->gtt.stolen_size, dev_priv->mm.stolen_base); - if (IS_VALLEYVIEW(dev)) - bios_reserved = 1024*1024; /* top 1M on VLV/BYT */ + if (INTEL_INFO(dev)->gen >= 8) { + tmp = I915_READ(GEN7_BIOS_RESERVED); + tmp >>= GEN8_BIOS_RESERVED_SHIFT; + tmp &= GEN8_BIOS_RESERVED_MASK; + bios_reserved = (1024*1024) << tmp; + } else if (IS_GEN7(dev)) { + tmp = I915_READ(GEN7_BIOS_RESERVED); + bios_reserved = tmp & GEN7_BIOS_RESERVED_256K ? + 256*1024 : 1024*1024; + } if (WARN_ON(bios_reserved > dev_priv->gtt.stolen_size)) return 0; diff --git a/drivers/gpu/drm/i915/i915_gem_tiling.c b/drivers/gpu/drm/i915/i915_gem_tiling.c index cb150e8..a48355c 100644 --- a/drivers/gpu/drm/i915/i915_gem_tiling.c +++ b/drivers/gpu/drm/i915/i915_gem_tiling.c @@ -91,7 +91,14 @@ i915_gem_detect_bit_6_swizzle(struct drm_device *dev) uint32_t swizzle_x = I915_BIT_6_SWIZZLE_UNKNOWN; uint32_t swizzle_y = I915_BIT_6_SWIZZLE_UNKNOWN; - if (IS_VALLEYVIEW(dev)) { + if (INTEL_INFO(dev)->gen >= 8 || IS_VALLEYVIEW(dev)) { + /* + * On BDW+, swizzling is not used. We leave the CPU memory + * controller in charge of optimizing memory accesses without + * the extra address manipulation GPU side. + * + * VLV and CHV don't have GPU swizzling. + */ swizzle_x = I915_BIT_6_SWIZZLE_NONE; swizzle_y = I915_BIT_6_SWIZZLE_NONE; } else if (INTEL_INFO(dev)->gen >= 6) { @@ -376,7 +383,7 @@ i915_gem_set_tiling(struct drm_device *dev, void *data, if (ret == 0) { obj->fence_dirty = - obj->fenced_gpu_access || + obj->last_fence.request || obj->fence_reg != I915_FENCE_REG_NONE; obj->tiling_mode = args->tiling_mode; diff --git a/drivers/gpu/drm/i915/i915_gpu_error.c b/drivers/gpu/drm/i915/i915_gpu_error.c index eab41f9..1f320c8 100644 --- a/drivers/gpu/drm/i915/i915_gpu_error.c +++ b/drivers/gpu/drm/i915/i915_gpu_error.c @@ -192,15 +192,18 @@ static void print_error_buffers(struct drm_i915_error_state_buf *m, struct drm_i915_error_buffer *err, int count) { - err_printf(m, "%s [%d]:\n", name, count); + int n; + err_printf(m, " %s [%d]:\n", name, count); while (count--) { - err_printf(m, " %08x %8u %02x %02x %x %x", + err_printf(m, " %08x %8u %02x %02x [", err->gtt_offset, err->size, err->read_domains, - err->write_domain, - err->rseqno, err->wseqno); + err->write_domain); + for (n = 0; n < ARRAY_SIZE(err->rseqno); n++) + err_printf(m, " %x", err->rseqno[n]); + err_printf(m, " ] %x %x ", err->wseqno, err->fseqno); err_puts(m, pin_flag(err->pinned)); err_puts(m, tiling_flag(err->tiling)); err_puts(m, dirty_flag(err->dirty)); @@ -208,7 +211,7 @@ static void print_error_buffers(struct drm_i915_error_state_buf *m, err_puts(m, err->userptr ? " userptr" : ""); err_puts(m, err->ring != -1 ? " " : ""); err_puts(m, ring_str(err->ring)); - err_puts(m, i915_cache_level_str(err->cache_level)); + err_puts(m, i915_cache_level_str(m->i915, err->cache_level)); if (err->name) err_printf(m, " (name: %d)", err->name); @@ -220,11 +223,13 @@ static void print_error_buffers(struct drm_i915_error_state_buf *m, } } -static const char *hangcheck_action_to_str(enum intel_ring_hangcheck_action a) +static const char *hangcheck_action_to_str(enum intel_engine_hangcheck_action a) { switch (a) { case HANGCHECK_IDLE: return "idle"; + case HANGCHECK_IDLE_WAITERS: + return "idle (with waiters)"; case HANGCHECK_WAIT: return "wait"; case HANGCHECK_ACTIVE: @@ -244,13 +249,19 @@ static void i915_ring_error_state(struct drm_i915_error_state_buf *m, struct drm_device *dev, struct drm_i915_error_ring *ring) { + int n; + if (!ring->valid) return; - err_printf(m, " HEAD: 0x%08x\n", ring->head); - err_printf(m, " TAIL: 0x%08x\n", ring->tail); - err_printf(m, " CTL: 0x%08x\n", ring->ctl); - err_printf(m, " HWS: 0x%08x\n", ring->hws); + err_printf(m, "%s command stream:\n", ring_str(ring->id)); + + err_printf(m, " START: 0x%08x\n", ring->start); + err_printf(m, " HEAD: 0x%08x\n", ring->head); + err_printf(m, " TAIL: 0x%08x\n", ring->tail); + err_printf(m, " CTL: 0x%08x\n", ring->ctl); + err_printf(m, " MODE: 0x%08x [idle? %d]\n", ring->mode, !!(ring->mode & MODE_IDLE)); + err_printf(m, " HWS: 0x%08x\n", ring->hws); err_printf(m, " ACTHD: 0x%08x %08x\n", (u32)(ring->acthd>>32), (u32)ring->acthd); err_printf(m, " IPEIR: 0x%08x\n", ring->ipeir); err_printf(m, " IPEHR: 0x%08x\n", ring->ipehr); @@ -266,17 +277,13 @@ static void i915_ring_error_state(struct drm_i915_error_state_buf *m, if (INTEL_INFO(dev)->gen >= 6) { err_printf(m, " RC PSMI: 0x%08x\n", ring->rc_psmi); err_printf(m, " FAULT_REG: 0x%08x\n", ring->fault_reg); - err_printf(m, " SYNC_0: 0x%08x [last synced 0x%08x]\n", - ring->semaphore_mboxes[0], - ring->semaphore_seqno[0]); - err_printf(m, " SYNC_1: 0x%08x [last synced 0x%08x]\n", - ring->semaphore_mboxes[1], - ring->semaphore_seqno[1]); - if (HAS_VEBOX(dev)) { - err_printf(m, " SYNC_2: 0x%08x [last synced 0x%08x]\n", - ring->semaphore_mboxes[2], - ring->semaphore_seqno[2]); - } + err_printf(m, " SYNC_0: 0x%08x\n", + ring->semaphore_mboxes[0]); + err_printf(m, " SYNC_1: 0x%08x\n", + ring->semaphore_mboxes[1]); + if (HAS_VEBOX(dev)) + err_printf(m, " SYNC_2: 0x%08x\n", + ring->semaphore_mboxes[2]); } if (USES_PPGTT(dev)) { err_printf(m, " GFX_MODE: 0x%08x\n", ring->vm_info.gfx_mode); @@ -291,8 +298,20 @@ static void i915_ring_error_state(struct drm_i915_error_state_buf *m, ring->vm_info.pp_dir_base); } } - err_printf(m, " seqno: 0x%08x\n", ring->seqno); - err_printf(m, " waiting: %s\n", yesno(ring->waiting)); + err_printf(m, " tag: 0x%04x\n", ring->tag); + err_printf(m, " seqno: 0x%08x [hangcheck 0x%08x, breadcrumb 0x%08x, request 0x%08x]\n", + ring->seqno, ring->hangcheck, ring->breadcrumb[ring->id], ring->request); + err_printf(m, " sem.signal: ["); + for (n = 0; n < ARRAY_SIZE(ring->breadcrumb); n++) + err_printf(m, " %s%08x", n == ring->id ? "*" : "", ring->breadcrumb[n]); + err_printf(m, " ]\n"); + err_printf(m, " sem.waited: ["); + for (n = 0; n < ARRAY_SIZE(ring->semaphore_sync); n++) + err_printf(m, " %s%08x", n == ring->id ? "*" : "", ring->semaphore_sync[n]); + err_printf(m, " ]\n"); + err_printf(m, " waiting: %s [irq count %d]\n", + yesno(ring->waiting), ring->irq_count); + err_printf(m, " interrupts: %d\n", ring->interrupts); err_printf(m, " ring->head: 0x%08x\n", ring->cpu_ring_head); err_printf(m, " ring->tail: 0x%08x\n", ring->cpu_ring_tail); err_printf(m, " hangcheck: %s [%d]\n", @@ -362,11 +381,16 @@ int i915_error_state_to_str(struct drm_i915_error_state_buf *m, err_printf(m, "EIR: 0x%08x\n", error->eir); err_printf(m, "IER: 0x%08x\n", error->ier); if (INTEL_INFO(dev)->gen >= 8) { - for (i = 0; i < 4; i++) + for (i = 0; i < 4; i++) { err_printf(m, "GTIER gt %d: 0x%08x\n", i, error->gtier[i]); - } else if (HAS_PCH_SPLIT(dev) || IS_VALLEYVIEW(dev)) + err_printf(m, "GTIMR gt %d: 0x%08x\n", i, + error->gtimr[i]); + } + } else if (HAS_PCH_SPLIT(dev) || IS_VALLEYVIEW(dev)) { err_printf(m, "GTIER: 0x%08x\n", error->gtier[0]); + err_printf(m, "GTIMR: 0x%08x\n", error->gtimr[0]); + } err_printf(m, "PGTBL_ER: 0x%08x\n", error->pgtbl_er); err_printf(m, "FORCEWAKE: 0x%08x\n", error->forcewake); err_printf(m, "DERRMR: 0x%08x\n", error->derrmr); @@ -388,64 +412,69 @@ int i915_error_state_to_str(struct drm_i915_error_state_buf *m, if (INTEL_INFO(dev)->gen == 7) err_printf(m, "ERR_INT: 0x%08x\n", error->err_int); - for (i = 0; i < ARRAY_SIZE(error->ring); i++) { - err_printf(m, "%s command stream:\n", ring_str(i)); + for (i = 0; i < ARRAY_SIZE(error->ring); i++) i915_ring_error_state(m, dev, &error->ring[i]); - } - if (error->active_bo) + for (i = 0; i < error->vm_count; i++) { + err_printf(m, "vm[%d]\n", i); + print_error_buffers(m, "Active", - error->active_bo[0], - error->active_bo_count[0]); + error->active_bo[i], + error->active_bo_count[i]); - if (error->pinned_bo) print_error_buffers(m, "Pinned", - error->pinned_bo[0], - error->pinned_bo_count[0]); + error->pinned_bo[i], + error->pinned_bo_count[i]); + } for (i = 0; i < ARRAY_SIZE(error->ring); i++) { - obj = error->ring[i].batchbuffer; + const struct drm_i915_error_ring *ering = &error->ring[i]; + const char *name = dev_priv->engine[ering->id].name; + + obj = ering->batchbuffer; if (obj) { - err_puts(m, dev_priv->ring[i].name); - if (error->ring[i].pid != -1) + err_puts(m, name); + if (ering->pid != -1) err_printf(m, " (submitted by %s [%d])", - error->ring[i].comm, - error->ring[i].pid); + ering->comm, ering->pid); err_printf(m, " --- gtt_offset = 0x%08x\n", obj->gtt_offset); print_error_obj(m, obj); } - obj = error->ring[i].wa_batchbuffer; + obj = ering->wa_batchbuffer; if (obj) { err_printf(m, "%s (w/a) --- gtt_offset = 0x%08x\n", - dev_priv->ring[i].name, obj->gtt_offset); + name, obj->gtt_offset); print_error_obj(m, obj); } - if (error->ring[i].num_requests) { + if (ering->num_requests) { err_printf(m, "%s --- %d requests\n", - dev_priv->ring[i].name, - error->ring[i].num_requests); - for (j = 0; j < error->ring[i].num_requests; j++) { - err_printf(m, " seqno 0x%08x, emitted %ld, tail 0x%08x\n", - error->ring[i].requests[j].seqno, - error->ring[i].requests[j].jiffies, - error->ring[i].requests[j].tail); + name, ering->num_requests); + for (j = 0; j < ering->num_requests; j++) { + err_printf(m, " pid %ld, seqno 0x%08x, tag 0x%04x, emitted %dms ago (at %ld jiffies), head 0x%08x, tail 0x%08x, batch 0x%08x, complete? %d\n", + ering->requests[j].pid, + ering->requests[j].seqno, + ering->requests[j].tag, + jiffies_to_usecs(jiffies - ering->requests[j].jiffies) / 1000, + ering->requests[j].jiffies, + ering->requests[j].head, + ering->requests[j].tail, + ering->requests[j].batch, + ering->requests[j].complete); } } - if ((obj = error->ring[i].ringbuffer)) { + if ((obj = ering->ringbuffer)) { err_printf(m, "%s --- ringbuffer = 0x%08x\n", - dev_priv->ring[i].name, - obj->gtt_offset); + name, obj->gtt_offset); print_error_obj(m, obj); } - if ((obj = error->ring[i].hws_page)) { + if ((obj = ering->hws_page)) { err_printf(m, "%s --- HW Status = 0x%08x\n", - dev_priv->ring[i].name, - obj->gtt_offset); + name, obj->gtt_offset); offset = 0; for (elt = 0; elt < PAGE_SIZE/16; elt += 4) { err_printf(m, "[%04x] %08x %08x %08x %08x\n", @@ -460,8 +489,7 @@ int i915_error_state_to_str(struct drm_i915_error_state_buf *m, if ((obj = error->ring[i].ctx)) { err_printf(m, "%s --- HW Context = 0x%08x\n", - dev_priv->ring[i].name, - obj->gtt_offset); + name, obj->gtt_offset); print_error_obj(m, obj); } } @@ -492,9 +520,11 @@ out: } int i915_error_state_buf_init(struct drm_i915_error_state_buf *ebuf, + struct drm_i915_private *i915, size_t count, loff_t pos) { memset(ebuf, 0, sizeof(*ebuf)); + ebuf->i915 = i915; /* We need to have enough room to store any i915_error_state printf * so that we can move it to start position. @@ -556,24 +586,55 @@ static void i915_error_state_free(struct kref *error_ref) } static struct drm_i915_error_object * -i915_error_object_create_sized(struct drm_i915_private *dev_priv, - struct drm_i915_gem_object *src, - struct i915_address_space *vm, - const int num_pages) +i915_error_object_create(struct drm_i915_private *dev_priv, + struct i915_vma *vma) { + struct drm_i915_gem_object *src; struct drm_i915_error_object *dst; - int i; + int num_pages; + bool use_ggtt; + int i = 0; u32 reloc_offset; - if (src == NULL || src->pages == NULL) + if (vma == NULL) return NULL; + src = vma->obj; + if (src->pages == NULL) + return NULL; + + num_pages = src->base.size >> PAGE_SHIFT; + dst = kmalloc(sizeof(*dst) + num_pages * sizeof(u32 *), GFP_ATOMIC); if (dst == NULL) return NULL; - reloc_offset = dst->gtt_offset = i915_gem_obj_offset(src, vm); - for (i = 0; i < num_pages; i++) { + dst->gtt_offset = vma->node.start; + + reloc_offset = dst->gtt_offset; + use_ggtt = (src->cache_level == I915_CACHE_NONE && + i915_is_ggtt(vma->vm) && + src->has_global_gtt_mapping && + reloc_offset + num_pages * PAGE_SIZE <= dev_priv->gtt.mappable_end); + + /* Cannot access stolen address directly, try to use the aperture */ + if (src->stolen) { + use_ggtt = true; + + if (!src->has_global_gtt_mapping) + goto unwind; + + reloc_offset = i915_gem_obj_ggtt_offset(src); + if (reloc_offset + num_pages * PAGE_SIZE > dev_priv->gtt.mappable_end) + goto unwind; + } + + /* Cannot access snooped pages through the aperture */ + if (use_ggtt && src->cache_level != I915_CACHE_NONE && !HAS_LLC(dev_priv->dev)) + goto unwind; + + dst->page_count = num_pages; + while (num_pages--) { unsigned long flags; void *d; @@ -582,10 +643,7 @@ i915_error_object_create_sized(struct drm_i915_private *dev_priv, goto unwind; local_irq_save(flags); - if (src->cache_level == I915_CACHE_NONE && - reloc_offset < dev_priv->gtt.mappable_end && - src->has_global_gtt_mapping && - i915_is_ggtt(vm)) { + if (use_ggtt) { void __iomem *s; /* Simply ignore tiling or any overlapping fence. @@ -597,14 +655,6 @@ i915_error_object_create_sized(struct drm_i915_private *dev_priv, reloc_offset); memcpy_fromio(d, s, PAGE_SIZE); io_mapping_unmap_atomic(s); - } else if (src->stolen) { - unsigned long offset; - - offset = dev_priv->mm.stolen_base; - offset += src->stolen->start; - offset += i << PAGE_SHIFT; - - memcpy_fromio(d, (void __iomem *) offset, PAGE_SIZE); } else { struct page *page; void *s; @@ -621,11 +671,9 @@ i915_error_object_create_sized(struct drm_i915_private *dev_priv, } local_irq_restore(flags); - dst->pages[i] = d; - + dst->pages[i++] = d; reloc_offset += PAGE_SIZE; } - dst->page_count = num_pages; return dst; @@ -635,22 +683,32 @@ unwind: kfree(dst); return NULL; } -#define i915_error_object_create(dev_priv, src, vm) \ - i915_error_object_create_sized((dev_priv), (src), (vm), \ - (src)->base.size>>PAGE_SHIFT) -#define i915_error_ggtt_object_create(dev_priv, src) \ - i915_error_object_create_sized((dev_priv), (src), &(dev_priv)->gtt.base, \ - (src)->base.size>>PAGE_SHIFT) +static inline struct drm_i915_error_object * +i915_error_ggtt_object_create(struct drm_i915_private *i915, + struct drm_i915_gem_object *src) +{ + if (src == NULL) + return NULL; + + return i915_error_object_create(i915, + i915_gem_obj_to_vma(src, + &i915->gtt.base)); +} static void capture_bo(struct drm_i915_error_buffer *err, - struct drm_i915_gem_object *obj) + struct i915_vma *vma) { + struct drm_i915_gem_object *obj = vma->obj; + int n; + err->size = obj->base.size; err->name = obj->base.name; - err->rseqno = obj->last_read_seqno; - err->wseqno = obj->last_write_seqno; - err->gtt_offset = i915_gem_obj_ggtt_offset(obj); + for (n = 0; n < ARRAY_SIZE(obj->last_read); n++) + err->rseqno[n] = i915_request_seqno(obj->last_read[n].request); + err->wseqno = i915_request_seqno(obj->last_write.request); + err->fseqno = i915_request_seqno(obj->last_fence.request); + err->gtt_offset = vma->node.start; err->read_domains = obj->base.read_domains; err->write_domain = obj->base.write_domain; err->fence_reg = obj->fence_reg; @@ -663,7 +721,7 @@ static void capture_bo(struct drm_i915_error_buffer *err, err->dirty = obj->dirty; err->purgeable = obj->madv != I915_MADV_WILLNEED; err->userptr = obj->userptr.mm != NULL; - err->ring = obj->ring ? obj->ring->id : -1; + err->ring = i915_request_engine_id(obj->last_write.request); err->cache_level = obj->cache_level; } @@ -674,7 +732,7 @@ static u32 capture_active_bo(struct drm_i915_error_buffer *err, int i = 0; list_for_each_entry(vma, head, mm_list) { - capture_bo(err++, vma->obj); + capture_bo(err++, vma); if (++i == count) break; } @@ -683,21 +741,27 @@ static u32 capture_active_bo(struct drm_i915_error_buffer *err, } static u32 capture_pinned_bo(struct drm_i915_error_buffer *err, - int count, struct list_head *head) + int count, struct list_head *head, + struct i915_address_space *vm) { struct drm_i915_gem_object *obj; - int i = 0; + struct drm_i915_error_buffer * const first = err; + struct drm_i915_error_buffer * const last = err + count; list_for_each_entry(obj, head, global_list) { - if (!i915_gem_obj_is_pinned(obj)) - continue; + struct i915_vma *vma; - capture_bo(err++, obj); - if (++i == count) + if (err == last) break; + + list_for_each_entry(vma, &obj->vma_list, vma_link) + if (vma->vm == vm && vma->pin_count > 0) { + capture_bo(err++, vma); + break; + } } - return i; + return err - first; } /* Generate a semi-unique error code. The code is not meant to have meaning, The @@ -721,7 +785,7 @@ static uint32_t i915_error_generate_code(struct drm_i915_private *dev_priv, * synchronization commands which almost always appear in the case * strictly a client bug. Use instdone to differentiate those some. */ - for (i = 0; i < I915_NUM_RINGS; i++) { + for (i = 0; i < I915_NUM_ENGINES; i++) { if (error->ring[i].hangcheck_action == HANGCHECK_HUNG) { if (ring_id) *ring_id = i; @@ -769,83 +833,77 @@ static void i915_gem_record_fences(struct drm_device *dev, static void gen8_record_semaphore_state(struct drm_i915_private *dev_priv, struct drm_i915_error_state *error, - struct intel_engine_cs *ring, + struct intel_engine_cs *engine, struct drm_i915_error_ring *ering) { struct intel_engine_cs *to; + u32 *mbox; int i; - if (!i915_semaphore_is_enabled(dev_priv->dev)) + if (dev_priv->semaphore_obj == NULL) return; - if (!error->semaphore_obj) + if (error->semaphore_obj == NULL) error->semaphore_obj = - i915_error_object_create(dev_priv, - dev_priv->semaphore_obj, - &dev_priv->gtt.base); - - for_each_ring(to, dev_priv, i) { - int idx; - u16 signal_offset; - u32 *tmp; + i915_error_ggtt_object_create(dev_priv, + dev_priv->semaphore_obj); + if (error->semaphore_obj == NULL) + return; - if (ring == to) + mbox = error->semaphore_obj->pages[0]; + for_each_engine(to, dev_priv, i) { + if (engine == to) continue; - signal_offset = (GEN8_SIGNAL_OFFSET(ring, i) & (PAGE_SIZE - 1)) - / 4; - tmp = error->semaphore_obj->pages[0]; - idx = intel_ring_sync_index(ring, to); - - ering->semaphore_mboxes[idx] = tmp[signal_offset]; - ering->semaphore_seqno[idx] = ring->semaphore.sync_seqno[idx]; + ering->semaphore_mboxes[i] = + mbox[GEN8_SEMAPHORE_OFFSET(dev_priv, + engine->id, + i) & (PAGE_SIZE - 1) / 4]; } } static void gen6_record_semaphore_state(struct drm_i915_private *dev_priv, - struct intel_engine_cs *ring, + struct intel_engine_cs *engine, struct drm_i915_error_ring *ering) { - ering->semaphore_mboxes[0] = I915_READ(RING_SYNC_0(ring->mmio_base)); - ering->semaphore_mboxes[1] = I915_READ(RING_SYNC_1(ring->mmio_base)); - ering->semaphore_seqno[0] = ring->semaphore.sync_seqno[0]; - ering->semaphore_seqno[1] = ring->semaphore.sync_seqno[1]; - + ering->semaphore_mboxes[0] = I915_READ(RING_SYNC_0(engine->mmio_base)); + ering->semaphore_mboxes[1] = I915_READ(RING_SYNC_1(engine->mmio_base)); if (HAS_VEBOX(dev_priv->dev)) { ering->semaphore_mboxes[2] = - I915_READ(RING_SYNC_2(ring->mmio_base)); - ering->semaphore_seqno[2] = ring->semaphore.sync_seqno[2]; + I915_READ(RING_SYNC_2(engine->mmio_base)); } } static void i915_record_ring_state(struct drm_device *dev, struct drm_i915_error_state *error, - struct intel_engine_cs *ring, + struct intel_engine_cs *engine, + struct i915_gem_request *rq, struct drm_i915_error_ring *ering) { struct drm_i915_private *dev_priv = dev->dev_private; + struct intel_ringbuffer *ring; if (INTEL_INFO(dev)->gen >= 6) { - ering->rc_psmi = I915_READ(ring->mmio_base + 0x50); - ering->fault_reg = I915_READ(RING_FAULT_REG(ring)); + ering->rc_psmi = I915_READ(engine->mmio_base + 0x50); + ering->fault_reg = I915_READ(RING_FAULT_REG(engine)); if (INTEL_INFO(dev)->gen >= 8) - gen8_record_semaphore_state(dev_priv, error, ring, ering); + gen8_record_semaphore_state(dev_priv, error, engine, ering); else - gen6_record_semaphore_state(dev_priv, ring, ering); + gen6_record_semaphore_state(dev_priv, engine, ering); } if (INTEL_INFO(dev)->gen >= 4) { - ering->faddr = I915_READ(RING_DMA_FADD(ring->mmio_base)); - ering->ipeir = I915_READ(RING_IPEIR(ring->mmio_base)); - ering->ipehr = I915_READ(RING_IPEHR(ring->mmio_base)); - ering->instdone = I915_READ(RING_INSTDONE(ring->mmio_base)); - ering->instps = I915_READ(RING_INSTPS(ring->mmio_base)); - ering->bbaddr = I915_READ(RING_BBADDR(ring->mmio_base)); + ering->faddr = I915_READ(RING_DMA_FADD(engine->mmio_base)); + ering->ipeir = I915_READ(RING_IPEIR(engine->mmio_base)); + ering->ipehr = I915_READ(RING_IPEHR(engine->mmio_base)); + ering->instdone = I915_READ(RING_INSTDONE(engine->mmio_base)); + ering->instps = I915_READ(RING_INSTPS(engine->mmio_base)); + ering->bbaddr = I915_READ(RING_BBADDR(engine->mmio_base)); if (INTEL_INFO(dev)->gen >= 8) { - ering->faddr |= (u64) I915_READ(RING_DMA_FADD_UDW(ring->mmio_base)) << 32; - ering->bbaddr |= (u64) I915_READ(RING_BBADDR_UDW(ring->mmio_base)) << 32; + ering->faddr |= (u64) I915_READ(RING_DMA_FADD_UDW(engine->mmio_base)) << 32; + ering->bbaddr |= (u64) I915_READ(RING_BBADDR_UDW(engine->mmio_base)) << 32; } - ering->bbstate = I915_READ(RING_BBSTATE(ring->mmio_base)); + ering->bbstate = I915_READ(RING_BBSTATE(engine->mmio_base)); } else { ering->faddr = I915_READ(DMA_FADD_I8XX); ering->ipeir = I915_READ(IPEIR); @@ -853,19 +911,29 @@ static void i915_record_ring_state(struct drm_device *dev, ering->instdone = I915_READ(INSTDONE); } - ering->waiting = waitqueue_active(&ring->irq_queue); - ering->instpm = I915_READ(RING_INSTPM(ring->mmio_base)); - ering->seqno = ring->get_seqno(ring, false); - ering->acthd = intel_ring_get_active_head(ring); - ering->head = I915_READ_HEAD(ring); - ering->tail = I915_READ_TAIL(ring); - ering->ctl = I915_READ_CTL(ring); + ering->waiting = waitqueue_active(&engine->irq_queue); + ering->instpm = I915_READ(RING_INSTPM(engine->mmio_base)); + ering->acthd = intel_engine_get_active_head(engine); + ering->seqno = engine->get_seqno(engine); + ering->request = engine->last_request ? engine->last_request->seqno : 0; + ering->hangcheck = engine->hangcheck.seqno; + memcpy(ering->breadcrumb, engine->breadcrumb, sizeof(ering->breadcrumb)); + memcpy(ering->semaphore_sync, engine->semaphore.sync, sizeof(ering->semaphore_sync)); + ering->tag = engine->tag; + ering->interrupts = atomic_read(&engine->interrupts); + ering->irq_count = engine->irq_refcount; + ering->start = I915_READ_START(engine); + ering->head = I915_READ_HEAD(engine); + ering->tail = I915_READ_TAIL(engine); + ering->ctl = I915_READ_CTL(engine); + if (!IS_GEN2(dev_priv)) + ering->mode = I915_READ_MODE(engine); if (I915_NEED_GFX_HWS(dev)) { int mmio; if (IS_GEN7(dev)) { - switch (ring->id) { + switch (engine->id) { default: case RCS: mmio = RENDER_HWS_PGA_GEN7; @@ -880,59 +948,67 @@ static void i915_record_ring_state(struct drm_device *dev, mmio = VEBOX_HWS_PGA_GEN7; break; } - } else if (IS_GEN6(ring->dev)) { - mmio = RING_HWS_PGA_GEN6(ring->mmio_base); + } else if (IS_GEN6(engine->i915)) { + mmio = RING_HWS_PGA_GEN6(engine->mmio_base); } else { /* XXX: gen8 returns to sanity */ - mmio = RING_HWS_PGA(ring->mmio_base); + mmio = RING_HWS_PGA(engine->mmio_base); } ering->hws = I915_READ(mmio); } - ering->cpu_ring_head = ring->buffer->head; - ering->cpu_ring_tail = ring->buffer->tail; + ring = rq ? rq->ctx->ring[engine->id].ring : engine->default_context->ring[engine->id].ring; + if (ring) { + ering->cpu_ring_head = ring->head; + ering->cpu_ring_tail = ring->tail; + ering->ringbuffer = + i915_error_ggtt_object_create(dev_priv, ring->obj); + } + + ering->hws_page = + i915_error_ggtt_object_create(dev_priv, + engine->status_page.obj); - ering->hangcheck_score = ring->hangcheck.score; - ering->hangcheck_action = ring->hangcheck.action; + ering->hangcheck_score = engine->hangcheck.score; + ering->hangcheck_action = engine->hangcheck.action; if (USES_PPGTT(dev)) { int i; - ering->vm_info.gfx_mode = I915_READ(RING_MODE_GEN7(ring)); + ering->vm_info.gfx_mode = I915_READ(RING_MODE_GEN7(engine)); switch (INTEL_INFO(dev)->gen) { case 8: for (i = 0; i < 4; i++) { ering->vm_info.pdp[i] = - I915_READ(GEN8_RING_PDP_UDW(ring, i)); + I915_READ(GEN8_RING_PDP_UDW(engine, i)); ering->vm_info.pdp[i] <<= 32; ering->vm_info.pdp[i] |= - I915_READ(GEN8_RING_PDP_LDW(ring, i)); + I915_READ(GEN8_RING_PDP_LDW(engine, i)); } break; case 7: ering->vm_info.pp_dir_base = - I915_READ(RING_PP_DIR_BASE(ring)); + I915_READ(RING_PP_DIR_BASE(engine)); break; case 6: ering->vm_info.pp_dir_base = - I915_READ(RING_PP_DIR_BASE_READ(ring)); + I915_READ(RING_PP_DIR_BASE_READ(engine)); break; } } } - -static void i915_gem_record_active_context(struct intel_engine_cs *ring, +static void i915_gem_record_active_context(struct intel_engine_cs *engine, struct drm_i915_error_state *error, struct drm_i915_error_ring *ering) { - struct drm_i915_private *dev_priv = ring->dev->dev_private; + struct drm_i915_private *dev_priv = engine->i915; struct drm_i915_gem_object *obj; /* Currently render ring is the only HW context user */ - if (ring->id != RCS || !error->ccid) + if (engine->id != RCS || !error->ccid) return; list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) { @@ -950,45 +1026,40 @@ static void i915_gem_record_rings(struct drm_device *dev, struct drm_i915_error_state *error) { struct drm_i915_private *dev_priv = dev->dev_private; - struct drm_i915_gem_request *request; + struct i915_gem_request *rq; int i, count; - for (i = 0; i < I915_NUM_RINGS; i++) { - struct intel_engine_cs *ring = &dev_priv->ring[i]; + for (i = 0; i < I915_NUM_ENGINES; i++) { + struct intel_engine_cs *engine = &dev_priv->engine[i]; error->ring[i].pid = -1; - if (ring->dev == NULL) + if (engine->i915 == NULL) continue; error->ring[i].valid = true; + error->ring[i].id = i; - i915_record_ring_state(dev, error, ring, &error->ring[i]); - - request = i915_gem_find_active_request(ring); - if (request) { + spin_lock(&engine->lock); + rq = intel_engine_find_active_batch(engine); + if (rq) { /* We need to copy these to an anonymous buffer * as the simplest method to avoid being overwritten * by userspace. */ error->ring[i].batchbuffer = - i915_error_object_create(dev_priv, - request->batch_obj, - request->ctx ? - request->ctx->vm : - &dev_priv->gtt.base); - - if (HAS_BROKEN_CS_TLB(dev_priv->dev) && - ring->scratch.obj) + i915_error_object_create(dev_priv, rq->batch); + + if (HAS_BROKEN_CS_TLB(dev_priv->dev)) error->ring[i].wa_batchbuffer = i915_error_ggtt_object_create(dev_priv, - ring->scratch.obj); + engine->scratch.obj); - if (request->file_priv) { + if (rq->file_priv) { struct task_struct *task; rcu_read_lock(); - task = pid_task(request->file_priv->file->pid, + task = pid_task(rq->file_priv->file->pid, PIDTYPE_PID); if (task) { strcpy(error->ring[i].comm, task->comm); @@ -998,17 +1069,12 @@ static void i915_gem_record_rings(struct drm_device *dev, } } - error->ring[i].ringbuffer = - i915_error_ggtt_object_create(dev_priv, ring->buffer->obj); - - if (ring->status_page.obj) - error->ring[i].hws_page = - i915_error_ggtt_object_create(dev_priv, ring->status_page.obj); + i915_record_ring_state(dev, error, engine, rq, &error->ring[i]); - i915_gem_record_active_context(ring, error, &error->ring[i]); + i915_gem_record_active_context(engine, error, &error->ring[i]); count = 0; - list_for_each_entry(request, &ring->request_list, list) + list_for_each_entry(rq, &engine->requests, engine_list) count++; error->ring[i].num_requests = count; @@ -1021,14 +1087,28 @@ static void i915_gem_record_rings(struct drm_device *dev, } count = 0; - list_for_each_entry(request, &ring->request_list, list) { + list_for_each_entry(rq, &engine->requests, engine_list) { struct drm_i915_error_request *erq; + struct task_struct *task; erq = &error->ring[i].requests[count++]; - erq->seqno = request->seqno; - erq->jiffies = request->emitted_jiffies; - erq->tail = request->tail; + erq->seqno = rq->seqno; + erq->jiffies = rq->emitted_jiffies; + erq->head = rq->head; + erq->tail = rq->tail; + erq->batch = 0; + if (rq->batch) + erq->batch = rq->batch->node.start; + memcpy(erq->breadcrumb, rq->breadcrumb, sizeof(rq->breadcrumb)); + erq->complete = i915_request_complete(rq); + erq->tag = rq->tag; + + rcu_read_lock(); + task = rq->file_priv ? pid_task(rq->file_priv->file->pid, PIDTYPE_PID) : NULL; + erq->pid = task ? task->pid : 0; + rcu_read_unlock(); } + spin_unlock(&engine->lock); } } @@ -1049,9 +1129,14 @@ static void i915_gem_capture_vm(struct drm_i915_private *dev_priv, list_for_each_entry(vma, &vm->active_list, mm_list) i++; error->active_bo_count[ndx] = i; - list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) - if (i915_gem_obj_is_pinned(obj)) - i++; + + list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) { + list_for_each_entry(vma, &obj->vma_list, vma_link) + if (vma->vm == vm && vma->pin_count > 0) { + i++; + break; + } + } error->pinned_bo_count[ndx] = i - error->active_bo_count[ndx]; if (i) { @@ -1070,7 +1155,7 @@ static void i915_gem_capture_vm(struct drm_i915_private *dev_priv, error->pinned_bo_count[ndx] = capture_pinned_bo(pinned_bo, error->pinned_bo_count[ndx], - &dev_priv->mm.bound_list); + &dev_priv->mm.bound_list, vm); error->active_bo[ndx] = active_bo; error->pinned_bo[ndx] = pinned_bo; } @@ -1091,8 +1176,25 @@ static void i915_gem_capture_buffers(struct drm_i915_private *dev_priv, error->pinned_bo_count = kcalloc(cnt, sizeof(*error->pinned_bo_count), GFP_ATOMIC); - list_for_each_entry(vm, &dev_priv->vm_list, global_link) - i915_gem_capture_vm(dev_priv, error, vm, i++); + if (error->active_bo == NULL || + error->pinned_bo == NULL || + error->active_bo_count == NULL || + error->pinned_bo_count == NULL) { + kfree(error->active_bo); + kfree(error->active_bo_count); + kfree(error->pinned_bo); + kfree(error->pinned_bo_count); + + error->active_bo = NULL; + error->active_bo_count = NULL; + error->pinned_bo = NULL; + error->pinned_bo_count = NULL; + } else { + list_for_each_entry(vm, &dev_priv->vm_list, global_link) + i915_gem_capture_vm(dev_priv, error, vm, i++); + + error->vm_count = cnt; + } } /* Capture all registers which don't fit into another category. */ @@ -1113,6 +1215,7 @@ static void i915_capture_reg_state(struct drm_i915_private *dev_priv, /* 1: Registers specific to a single generation */ if (IS_VALLEYVIEW(dev)) { error->gtier[0] = I915_READ(GTIER); + error->gtimr[0] = I915_READ(GTIMR); error->ier = I915_READ(VLV_IER); error->forcewake = I915_READ(FORCEWAKE_VLV); } @@ -1148,11 +1251,14 @@ static void i915_capture_reg_state(struct drm_i915_private *dev_priv, if (INTEL_INFO(dev)->gen >= 8) { error->ier = I915_READ(GEN8_DE_MISC_IER); - for (i = 0; i < 4; i++) + for (i = 0; i < 4; i++) { error->gtier[i] = I915_READ(GEN8_GT_IER(i)); + error->gtimr[i] = I915_READ(GEN8_GT_IMR(i)); + } } else if (HAS_PCH_SPLIT(dev)) { error->ier = I915_READ(DEIER); error->gtier[0] = I915_READ(GTIER); + error->gtimr[0] = I915_READ(GTIMR); } else if (IS_GEN2(dev)) { error->ier = I915_READ16(IER); } else if (!IS_VALLEYVIEW(dev)) { @@ -1264,13 +1370,12 @@ void i915_error_state_get(struct drm_device *dev, struct i915_error_state_file_priv *error_priv) { struct drm_i915_private *dev_priv = dev->dev_private; - unsigned long flags; - spin_lock_irqsave(&dev_priv->gpu_error.lock, flags); + spin_lock_irq(&dev_priv->gpu_error.lock); error_priv->error = dev_priv->gpu_error.first_error; if (error_priv->error) kref_get(&error_priv->error->ref); - spin_unlock_irqrestore(&dev_priv->gpu_error.lock, flags); + spin_unlock_irq(&dev_priv->gpu_error.lock); } @@ -1284,22 +1389,21 @@ void i915_destroy_error_state(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; struct drm_i915_error_state *error; - unsigned long flags; - spin_lock_irqsave(&dev_priv->gpu_error.lock, flags); + spin_lock_irq(&dev_priv->gpu_error.lock); error = dev_priv->gpu_error.first_error; dev_priv->gpu_error.first_error = NULL; - spin_unlock_irqrestore(&dev_priv->gpu_error.lock, flags); + spin_unlock_irq(&dev_priv->gpu_error.lock); if (error) kref_put(&error->ref, i915_error_state_free); } -const char *i915_cache_level_str(int type) +const char *i915_cache_level_str(struct drm_i915_private *i915, int type) { switch (type) { case I915_CACHE_NONE: return " uncached"; - case I915_CACHE_LLC: return " snooped or LLC"; + case I915_CACHE_LLC: return HAS_LLC(i915) ? " LLC" : " snooped"; case I915_CACHE_L3_LLC: return " L3+LLC"; case I915_CACHE_WT: return " WT"; default: return ""; diff --git a/drivers/gpu/drm/i915/i915_irq.c b/drivers/gpu/drm/i915/i915_irq.c index 0050ee9..348f207 100644 --- a/drivers/gpu/drm/i915/i915_irq.c +++ b/drivers/gpu/drm/i915/i915_irq.c @@ -151,7 +151,7 @@ ironlake_disable_display_irq(struct drm_i915_private *dev_priv, u32 mask) { assert_spin_locked(&dev_priv->irq_lock); - if (!intel_irqs_enabled(dev_priv)) + if (WARN_ON(!intel_irqs_enabled(dev_priv))) return; if ((dev_priv->irq_mask & mask) != mask) { @@ -238,7 +238,7 @@ static bool ivb_can_enable_err_int(struct drm_device *dev) assert_spin_locked(&dev_priv->irq_lock); - for_each_pipe(pipe) { + for_each_pipe(dev_priv, pipe) { crtc = to_intel_crtc(dev_priv->pipe_to_crtc_mapping[pipe]); if (crtc->cpu_fifo_underrun_disabled) @@ -296,7 +296,7 @@ static bool cpt_can_enable_serr_int(struct drm_device *dev) assert_spin_locked(&dev_priv->irq_lock); - for_each_pipe(pipe) { + for_each_pipe(dev_priv, pipe) { crtc = to_intel_crtc(dev_priv->pipe_to_crtc_mapping[pipe]); if (crtc->pch_fifo_underrun_disabled) @@ -310,9 +310,8 @@ void i9xx_check_fifo_underruns(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; struct intel_crtc *crtc; - unsigned long flags; - spin_lock_irqsave(&dev_priv->irq_lock, flags); + spin_lock_irq(&dev_priv->irq_lock); for_each_intel_crtc(dev, crtc) { u32 reg = PIPESTAT(crtc->pipe); @@ -331,7 +330,7 @@ void i9xx_check_fifo_underruns(struct drm_device *dev) DRM_ERROR("pipe %c underrun\n", pipe_name(crtc->pipe)); } - spin_unlock_irqrestore(&dev_priv->irq_lock, flags); + spin_unlock_irq(&dev_priv->irq_lock); } static void i9xx_set_fifo_underrun_reporting(struct drm_device *dev, @@ -497,7 +496,7 @@ static bool __intel_set_cpu_fifo_underrun_reporting(struct drm_device *dev, old = !intel_crtc->cpu_fifo_underrun_disabled; intel_crtc->cpu_fifo_underrun_disabled = !enable; - if (INTEL_INFO(dev)->gen < 5 || IS_VALLEYVIEW(dev)) + if (HAS_GMCH_DISPLAY(dev)) i9xx_set_fifo_underrun_reporting(dev, pipe, enable, old); else if (IS_GEN5(dev) || IS_GEN6(dev)) ironlake_set_fifo_underrun_reporting(dev, pipe, enable); @@ -589,6 +588,7 @@ __i915_enable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe, u32 pipestat = I915_READ(reg) & PIPESTAT_INT_ENABLE_MASK; assert_spin_locked(&dev_priv->irq_lock); + WARN_ON(!intel_irqs_enabled(dev_priv)); if (WARN_ONCE(enable_mask & ~PIPESTAT_INT_ENABLE_MASK || status_mask & ~PIPESTAT_INT_STATUS_MASK, @@ -615,6 +615,7 @@ __i915_disable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe, u32 pipestat = I915_READ(reg) & PIPESTAT_INT_ENABLE_MASK; assert_spin_locked(&dev_priv->irq_lock); + WARN_ON(!intel_irqs_enabled(dev_priv)); if (WARN_ONCE(enable_mask & ~PIPESTAT_INT_ENABLE_MASK || status_mask & ~PIPESTAT_INT_STATUS_MASK, @@ -694,19 +695,18 @@ i915_disable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe, static void i915_enable_asle_pipestat(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; - unsigned long irqflags; if (!dev_priv->opregion.asle || !IS_MOBILE(dev)) return; - spin_lock_irqsave(&dev_priv->irq_lock, irqflags); + spin_lock_irq(&dev_priv->irq_lock); i915_enable_pipestat(dev_priv, PIPE_B, PIPE_LEGACY_BLC_EVENT_STATUS); if (INTEL_INFO(dev)->gen >= 4) i915_enable_pipestat(dev_priv, PIPE_A, PIPE_LEGACY_BLC_EVENT_STATUS); - spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags); + spin_unlock_irq(&dev_priv->irq_lock); } /** @@ -1094,18 +1094,17 @@ static void i915_digport_work_func(struct work_struct *work) { struct drm_i915_private *dev_priv = container_of(work, struct drm_i915_private, dig_port_work); - unsigned long irqflags; u32 long_port_mask, short_port_mask; struct intel_digital_port *intel_dig_port; int i, ret; u32 old_bits = 0; - spin_lock_irqsave(&dev_priv->irq_lock, irqflags); + spin_lock_irq(&dev_priv->irq_lock); long_port_mask = dev_priv->long_hpd_port_mask; dev_priv->long_hpd_port_mask = 0; short_port_mask = dev_priv->short_hpd_port_mask; dev_priv->short_hpd_port_mask = 0; - spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags); + spin_unlock_irq(&dev_priv->irq_lock); for (i = 0; i < I915_MAX_PORTS; i++) { bool valid = false; @@ -1130,10 +1129,13 @@ static void i915_digport_work_func(struct work_struct *work) } if (old_bits) { - spin_lock_irqsave(&dev_priv->irq_lock, irqflags); + spin_lock_irq(&dev_priv->irq_lock); dev_priv->hpd_event_bits |= old_bits; - spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags); - schedule_work(&dev_priv->hotplug_work); + spin_unlock_irq(&dev_priv->irq_lock); + + mod_delayed_work(system_wq, + &dev_priv->hotplug_work, + msecs_to_jiffies(50)); } } @@ -1145,13 +1147,12 @@ static void i915_digport_work_func(struct work_struct *work) static void i915_hotplug_work_func(struct work_struct *work) { struct drm_i915_private *dev_priv = - container_of(work, struct drm_i915_private, hotplug_work); + container_of(work, struct drm_i915_private, hotplug_work.work); struct drm_device *dev = dev_priv->dev; struct drm_mode_config *mode_config = &dev->mode_config; struct intel_connector *intel_connector; struct intel_encoder *intel_encoder; struct drm_connector *connector; - unsigned long irqflags; bool hpd_disabled = false; bool changed = false; u32 hpd_event_bits; @@ -1159,7 +1160,7 @@ static void i915_hotplug_work_func(struct work_struct *work) mutex_lock(&mode_config->mutex); DRM_DEBUG_KMS("running encoder hotplug functions\n"); - spin_lock_irqsave(&dev_priv->irq_lock, irqflags); + spin_lock_irq(&dev_priv->irq_lock); hpd_event_bits = dev_priv->hpd_event_bits; dev_priv->hpd_event_bits = 0; @@ -1193,7 +1194,7 @@ static void i915_hotplug_work_func(struct work_struct *work) msecs_to_jiffies(I915_REENABLE_HOTPLUG_DELAY)); } - spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags); + spin_unlock_irq(&dev_priv->irq_lock); list_for_each_entry(connector, &mode_config->connector_list, head) { intel_connector = to_intel_connector(connector); @@ -1253,17 +1254,15 @@ static void ironlake_rps_change_irq_handler(struct drm_device *dev) } static void notify_ring(struct drm_device *dev, - struct intel_engine_cs *ring) + struct intel_engine_cs *engine) { - if (!intel_ring_initialized(ring)) + if (!intel_engine_initialized(engine)) return; - trace_i915_gem_request_complete(ring); - - if (drm_core_check_feature(dev, DRIVER_MODESET)) - intel_notify_mmio_flip(ring); + trace_i915_gem_ring_complete(engine); + atomic_inc(&engine->interrupts); - wake_up_all(&ring->irq_queue); + wake_up_all(&engine->irq_queue); i915_queue_hangcheck(dev); } @@ -1322,10 +1321,10 @@ static u32 vlv_c0_residency(struct drm_i915_private *dev_priv, * @dev_priv: DRM device private * */ -static u32 vlv_calc_delay_from_C0_counters(struct drm_i915_private *dev_priv) +static int vlv_calc_delay_from_C0_counters(struct drm_i915_private *dev_priv) { u32 residency_C0_up = 0, residency_C0_down = 0; - u8 new_delay, adj; + int new_delay, adj; dev_priv->rps.ei_interrupt_count++; @@ -1488,7 +1487,6 @@ static void ivybridge_parity_work(struct work_struct *work) u32 error_status, row, bank, subbank; char *parity_event[6]; uint32_t misccpctl; - unsigned long flags; uint8_t slice = 0; /* We must turn off DOP level clock gating to access the L3 registers. @@ -1547,9 +1545,9 @@ static void ivybridge_parity_work(struct work_struct *work) out: WARN_ON(dev_priv->l3_parity.which_slice); - spin_lock_irqsave(&dev_priv->irq_lock, flags); + spin_lock_irq(&dev_priv->irq_lock); gen5_enable_gt_irq(dev_priv, GT_PARITY_ERROR(dev_priv->dev)); - spin_unlock_irqrestore(&dev_priv->irq_lock, flags); + spin_unlock_irq(&dev_priv->irq_lock); mutex_unlock(&dev_priv->dev->struct_mutex); } @@ -1581,9 +1579,9 @@ static void ilk_gt_irq_handler(struct drm_device *dev, { if (gt_iir & (GT_RENDER_USER_INTERRUPT | GT_RENDER_PIPECTL_NOTIFY_INTERRUPT)) - notify_ring(dev, &dev_priv->ring[RCS]); + notify_ring(dev, &dev_priv->engine[RCS]); if (gt_iir & ILK_BSD_USER_INTERRUPT) - notify_ring(dev, &dev_priv->ring[VCS]); + notify_ring(dev, &dev_priv->engine[VCS]); } static void snb_gt_irq_handler(struct drm_device *dev, @@ -1593,11 +1591,11 @@ static void snb_gt_irq_handler(struct drm_device *dev, if (gt_iir & (GT_RENDER_USER_INTERRUPT | GT_RENDER_PIPECTL_NOTIFY_INTERRUPT)) - notify_ring(dev, &dev_priv->ring[RCS]); + notify_ring(dev, &dev_priv->engine[RCS]); if (gt_iir & GT_BSD_USER_INTERRUPT) - notify_ring(dev, &dev_priv->ring[VCS]); + notify_ring(dev, &dev_priv->engine[VCS]); if (gt_iir & GT_BLT_USER_INTERRUPT) - notify_ring(dev, &dev_priv->ring[BCS]); + notify_ring(dev, &dev_priv->engine[BCS]); if (gt_iir & (GT_BLT_CS_ERROR_INTERRUPT | GT_BSD_CS_ERROR_INTERRUPT | @@ -1627,6 +1625,7 @@ static irqreturn_t gen8_gt_irq_handler(struct drm_device *dev, struct drm_i915_private *dev_priv, u32 master_ctl) { + struct intel_engine_cs *engine; u32 rcs, bcs, vcs; uint32_t tmp = 0; irqreturn_t ret = IRQ_NONE; @@ -1636,12 +1635,16 @@ static irqreturn_t gen8_gt_irq_handler(struct drm_device *dev, if (tmp) { I915_WRITE(GEN8_GT_IIR(0), tmp); ret = IRQ_HANDLED; + rcs = tmp >> GEN8_RCS_IRQ_SHIFT; - bcs = tmp >> GEN8_BCS_IRQ_SHIFT; + engine = &dev_priv->engine[RCS]; if (rcs & GT_RENDER_USER_INTERRUPT) - notify_ring(dev, &dev_priv->ring[RCS]); + notify_ring(dev, engine); + + bcs = tmp >> GEN8_BCS_IRQ_SHIFT; + engine = &dev_priv->engine[BCS]; if (bcs & GT_RENDER_USER_INTERRUPT) - notify_ring(dev, &dev_priv->ring[BCS]); + notify_ring(dev, engine); } else DRM_ERROR("The master control interrupt lied (GT0)!\n"); } @@ -1651,12 +1654,16 @@ static irqreturn_t gen8_gt_irq_handler(struct drm_device *dev, if (tmp) { I915_WRITE(GEN8_GT_IIR(1), tmp); ret = IRQ_HANDLED; + vcs = tmp >> GEN8_VCS1_IRQ_SHIFT; + engine = &dev_priv->engine[VCS]; if (vcs & GT_RENDER_USER_INTERRUPT) - notify_ring(dev, &dev_priv->ring[VCS]); + notify_ring(dev, engine); + vcs = tmp >> GEN8_VCS2_IRQ_SHIFT; + engine = &dev_priv->engine[VCS2]; if (vcs & GT_RENDER_USER_INTERRUPT) - notify_ring(dev, &dev_priv->ring[VCS2]); + notify_ring(dev, engine); } else DRM_ERROR("The master control interrupt lied (GT1)!\n"); } @@ -1677,9 +1684,11 @@ static irqreturn_t gen8_gt_irq_handler(struct drm_device *dev, if (tmp) { I915_WRITE(GEN8_GT_IIR(3), tmp); ret = IRQ_HANDLED; + vcs = tmp >> GEN8_VECS_IRQ_SHIFT; + engine = &dev_priv->engine[VECS]; if (vcs & GT_RENDER_USER_INTERRUPT) - notify_ring(dev, &dev_priv->ring[VECS]); + notify_ring(dev, engine); } else DRM_ERROR("The master control interrupt lied (GT3)!\n"); } @@ -1772,7 +1781,9 @@ static inline void intel_hpd_irq_handler(struct drm_device *dev, long_hpd = (dig_hotplug_reg >> dig_shift) & PORTB_HOTPLUG_LONG_DETECT; } - DRM_DEBUG_DRIVER("digital hpd port %d %d\n", port, long_hpd); + DRM_DEBUG_DRIVER("digital hpd port %c - %s\n", + port_name(port), + long_hpd ? "long" : "short"); /* for long HPD pulses we want to have the digital queue happen, but we still want HPD storm detection to function. */ if (long_hpd) { @@ -1843,7 +1854,9 @@ static inline void intel_hpd_irq_handler(struct drm_device *dev, if (queue_dig) queue_work(dev_priv->dp_wq, &dev_priv->dig_port_work); if (queue_hp) - schedule_work(&dev_priv->hotplug_work); + mod_delayed_work(system_wq, + &dev_priv->hotplug_work, + msecs_to_jiffies(50)); } static void gmbus_irq_handler(struct drm_device *dev) @@ -1956,6 +1969,27 @@ static void i9xx_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe) res1, res2); } +void gen8_flip_interrupt(struct drm_device *dev) +{ + struct drm_i915_private *dev_priv = dev->dev_private; + + if (!dev_priv->rps.is_bdw_sw_turbo) + return; + + if(atomic_read(&dev_priv->rps.sw_turbo.flip_received)) { + mod_timer(&dev_priv->rps.sw_turbo.flip_timer, + usecs_to_jiffies(dev_priv->rps.sw_turbo.timeout) + jiffies); + } + else { + dev_priv->rps.sw_turbo.flip_timer.expires = + usecs_to_jiffies(dev_priv->rps.sw_turbo.timeout) + jiffies; + add_timer(&dev_priv->rps.sw_turbo.flip_timer); + atomic_set(&dev_priv->rps.sw_turbo.flip_received, true); + } + + bdw_software_turbo(dev); +} + /* The RPS events need forcewake, so we add them to a work queue and mask their * IMR bits until the work is done. Other interrupts can be processed without * the work queue. */ @@ -1972,7 +2006,7 @@ static void gen6_rps_irq_handler(struct drm_i915_private *dev_priv, u32 pm_iir) if (HAS_VEBOX(dev_priv->dev)) { if (pm_iir & PM_VEBOX_USER_INTERRUPT) - notify_ring(dev_priv->dev, &dev_priv->ring[VECS]); + notify_ring(dev_priv->dev, &dev_priv->engine[VECS]); if (pm_iir & PM_VEBOX_CS_ERROR_INTERRUPT) { i915_handle_error(dev_priv->dev, false, @@ -1984,14 +2018,9 @@ static void gen6_rps_irq_handler(struct drm_i915_private *dev_priv, u32 pm_iir) static bool intel_pipe_handle_vblank(struct drm_device *dev, enum pipe pipe) { - struct intel_crtc *crtc; - if (!drm_handle_vblank(dev, pipe)) return false; - crtc = to_intel_crtc(intel_get_crtc_for_pipe(dev, pipe)); - wake_up(&crtc->vbl_wait); - return true; } @@ -2002,7 +2031,7 @@ static void valleyview_pipestat_irq_handler(struct drm_device *dev, u32 iir) int pipe; spin_lock(&dev_priv->irq_lock); - for_each_pipe(pipe) { + for_each_pipe(dev_priv, pipe) { int reg; u32 mask, iir_bit = 0; @@ -2047,9 +2076,10 @@ static void valleyview_pipestat_irq_handler(struct drm_device *dev, u32 iir) } spin_unlock(&dev_priv->irq_lock); - for_each_pipe(pipe) { - if (pipe_stats[pipe] & PIPE_START_VBLANK_INTERRUPT_STATUS) - intel_pipe_handle_vblank(dev, pipe); + for_each_pipe(dev_priv, pipe) { + if (pipe_stats[pipe] & PIPE_START_VBLANK_INTERRUPT_STATUS && + intel_pipe_handle_vblank(dev, pipe)) + intel_check_page_flip(dev, pipe); if (pipe_stats[pipe] & PLANE_FLIP_DONE_INT_STATUS_VLV) { intel_prepare_page_flip(dev, pipe); @@ -2216,7 +2246,7 @@ static void ibx_irq_handler(struct drm_device *dev, u32 pch_iir) DRM_ERROR("PCH poison interrupt\n"); if (pch_iir & SDE_FDI_MASK) - for_each_pipe(pipe) + for_each_pipe(dev_priv, pipe) DRM_DEBUG_DRIVER(" pipe %c FDI IIR: 0x%08x\n", pipe_name(pipe), I915_READ(FDI_RX_IIR(pipe))); @@ -2247,7 +2277,7 @@ static void ivb_err_int_handler(struct drm_device *dev) if (err_int & ERR_INT_POISON) DRM_ERROR("Poison interrupt\n"); - for_each_pipe(pipe) { + for_each_pipe(dev_priv, pipe) { if (err_int & ERR_INT_FIFO_UNDERRUN(pipe)) { if (intel_set_cpu_fifo_underrun_reporting(dev, pipe, false)) @@ -2324,7 +2354,7 @@ static void cpt_irq_handler(struct drm_device *dev, u32 pch_iir) DRM_DEBUG_DRIVER("Audio CP change interrupt\n"); if (pch_iir & SDE_FDI_MASK_CPT) - for_each_pipe(pipe) + for_each_pipe(dev_priv, pipe) DRM_DEBUG_DRIVER(" pipe %c FDI IIR: 0x%08x\n", pipe_name(pipe), I915_READ(FDI_RX_IIR(pipe))); @@ -2347,9 +2377,10 @@ static void ilk_display_irq_handler(struct drm_device *dev, u32 de_iir) if (de_iir & DE_POISON) DRM_ERROR("Poison interrupt\n"); - for_each_pipe(pipe) { - if (de_iir & DE_PIPE_VBLANK(pipe)) - intel_pipe_handle_vblank(dev, pipe); + for_each_pipe(dev_priv, pipe) { + if (de_iir & DE_PIPE_VBLANK(pipe) && + intel_pipe_handle_vblank(dev, pipe)) + intel_check_page_flip(dev, pipe); if (de_iir & DE_PIPE_FIFO_UNDERRUN(pipe)) if (intel_set_cpu_fifo_underrun_reporting(dev, pipe, false)) @@ -2397,9 +2428,10 @@ static void ivb_display_irq_handler(struct drm_device *dev, u32 de_iir) if (de_iir & DE_GSE_IVB) intel_opregion_asle_intr(dev); - for_each_pipe(pipe) { - if (de_iir & (DE_PIPE_VBLANK_IVB(pipe))) - intel_pipe_handle_vblank(dev, pipe); + for_each_pipe(dev_priv, pipe) { + if (de_iir & (DE_PIPE_VBLANK_IVB(pipe)) && + intel_pipe_handle_vblank(dev, pipe)) + intel_check_page_flip(dev, pipe); /* plane/pipes map 1:1 on ilk+ */ if (de_iir & DE_PLANE_FLIP_DONE_IVB(pipe)) { @@ -2544,7 +2576,7 @@ static irqreturn_t gen8_irq_handler(int irq, void *arg) DRM_ERROR("The master control interrupt lied (DE PORT)!\n"); } - for_each_pipe(pipe) { + for_each_pipe(dev_priv, pipe) { uint32_t pipe_iir; if (!(master_ctl & GEN8_DE_PIPE_IRQ(pipe))) @@ -2554,8 +2586,9 @@ static irqreturn_t gen8_irq_handler(int irq, void *arg) if (pipe_iir) { ret = IRQ_HANDLED; I915_WRITE(GEN8_DE_PIPE_IIR(pipe), pipe_iir); - if (pipe_iir & GEN8_PIPE_VBLANK) - intel_pipe_handle_vblank(dev, pipe); + if (pipe_iir & GEN8_PIPE_VBLANK && + intel_pipe_handle_vblank(dev, pipe)) + intel_check_page_flip(dev, pipe); if (pipe_iir & GEN8_PIPE_PRIMARY_FLIP_DONE) { intel_prepare_page_flip(dev, pipe); @@ -2606,7 +2639,7 @@ static irqreturn_t gen8_irq_handler(int irq, void *arg) static void i915_error_wake_up(struct drm_i915_private *dev_priv, bool reset_completed) { - struct intel_engine_cs *ring; + struct intel_engine_cs *engine; int i; /* @@ -2617,8 +2650,8 @@ static void i915_error_wake_up(struct drm_i915_private *dev_priv, */ /* Wake up __wait_seqno, potentially holding dev->struct_mutex. */ - for_each_ring(ring, dev_priv, i) - wake_up_all(&ring->irq_queue); + for_each_engine(engine, dev_priv, i) + wake_up_all(&engine->irq_queue); /* Wake up intel_crtc_wait_for_pending_flips, holding crtc->mutex. */ wake_up_all(&dev_priv->pending_flip_queue); @@ -2662,7 +2695,7 @@ static void i915_error_work_func(struct work_struct *work) * the reset in-progress bit is only ever set by code outside of this * work we don't need to worry about any other races. */ - if (i915_reset_in_progress(error) && !i915_terminally_wedged(error)) { + if (i915_recovery_pending(error)) { DRM_DEBUG_DRIVER("resetting chip\n"); kobject_uevent_env(&dev->primary->kdev->kobj, KOBJ_CHANGE, reset_event); @@ -2698,9 +2731,7 @@ static void i915_error_work_func(struct work_struct *work) * updates before * the counter increment. */ - smp_mb__before_atomic(); - atomic_inc(&dev_priv->gpu_error.reset_counter); - + smp_mb__after_atomic(); kobject_uevent_env(&dev->primary->kdev->kobj, KOBJ_CHANGE, reset_done_event); } else { @@ -2763,7 +2794,7 @@ static void i915_report_and_clear_eir(struct drm_device *dev) if (eir & I915_ERROR_MEMORY_REFRESH) { pr_err("memory refresh error:\n"); - for_each_pipe(pipe) + for_each_pipe(dev_priv, pipe) pr_err("pipe %c stat: 0x%08x\n", pipe_name(pipe), I915_READ(PIPESTAT(pipe))); /* pipestat has already been acked */ @@ -2860,52 +2891,6 @@ void i915_handle_error(struct drm_device *dev, bool wedged, schedule_work(&dev_priv->gpu_error.work); } -static void __always_unused i915_pageflip_stall_check(struct drm_device *dev, int pipe) -{ - struct drm_i915_private *dev_priv = dev->dev_private; - struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe]; - struct intel_crtc *intel_crtc = to_intel_crtc(crtc); - struct drm_i915_gem_object *obj; - struct intel_unpin_work *work; - unsigned long flags; - bool stall_detected; - - /* Ignore early vblank irqs */ - if (intel_crtc == NULL) - return; - - spin_lock_irqsave(&dev->event_lock, flags); - work = intel_crtc->unpin_work; - - if (work == NULL || - atomic_read(&work->pending) >= INTEL_FLIP_COMPLETE || - !work->enable_stall_check) { - /* Either the pending flip IRQ arrived, or we're too early. Don't check */ - spin_unlock_irqrestore(&dev->event_lock, flags); - return; - } - - /* Potential stall - if we see that the flip has happened, assume a missed interrupt */ - obj = work->pending_flip_obj; - if (INTEL_INFO(dev)->gen >= 4) { - int dspsurf = DSPSURF(intel_crtc->plane); - stall_detected = I915_HI_DISPBASE(I915_READ(dspsurf)) == - i915_gem_obj_ggtt_offset(obj); - } else { - int dspaddr = DSPADDR(intel_crtc->plane); - stall_detected = I915_READ(dspaddr) == (i915_gem_obj_ggtt_offset(obj) + - crtc->y * crtc->primary->fb->pitches[0] + - crtc->x * crtc->primary->fb->bits_per_pixel/8); - } - - spin_unlock_irqrestore(&dev->event_lock, flags); - - if (stall_detected) { - DRM_DEBUG_DRIVER("Pageflip stall detected\n"); - intel_prepare_page_flip(dev, intel_crtc->plane); - } -} - /* Called from drm generic code, passed 'crtc' which * we use as a pipe index */ @@ -3031,24 +3016,28 @@ static void gen8_disable_vblank(struct drm_device *dev, int pipe) spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags); } -static u32 -ring_last_seqno(struct intel_engine_cs *ring) -{ - return list_entry(ring->request_list.prev, - struct drm_i915_gem_request, list)->seqno; -} - static bool -ring_idle(struct intel_engine_cs *ring, u32 seqno) +engine_idle(struct intel_engine_cs *engine) { - return (list_empty(&ring->request_list) || - i915_seqno_passed(seqno, ring_last_seqno(ring))); + bool ret = true; + + spin_lock(&engine->lock); + if (engine->last_request) { + /* poke to make sure we retire before we wake up again */ + queue_delayed_work(engine->i915->wq, + &engine->i915->mm.retire_work, + round_jiffies_up_relative(DRM_I915_HANGCHECK_JIFFIES/2)); + ret = __i915_request_complete__wa(engine->last_request); + } + spin_unlock(&engine->lock); + + return ret; } static bool -ipehr_is_semaphore_wait(struct drm_device *dev, u32 ipehr) +ipehr_is_semaphore_wait(struct drm_i915_private *i915, u32 ipehr) { - if (INTEL_INFO(dev)->gen >= 8) { + if (INTEL_INFO(i915)->gen >= 8) { return (ipehr >> 23) == 0x1c; } else { ipehr &= ~MI_SEMAPHORE_SYNC_MASK; @@ -3058,48 +3047,54 @@ ipehr_is_semaphore_wait(struct drm_device *dev, u32 ipehr) } static struct intel_engine_cs * -semaphore_wait_to_signaller_ring(struct intel_engine_cs *ring, u32 ipehr, u64 offset) +semaphore_wait_to_signaller_engine(struct intel_engine_cs *engine, u32 ipehr, u64 offset) { - struct drm_i915_private *dev_priv = ring->dev->dev_private; + struct drm_i915_private *dev_priv = engine->i915; struct intel_engine_cs *signaller; int i; if (INTEL_INFO(dev_priv->dev)->gen >= 8) { - for_each_ring(signaller, dev_priv, i) { - if (ring == signaller) + for_each_engine(signaller, dev_priv, i) { + if (engine == signaller) continue; - if (offset == signaller->semaphore.signal_ggtt[ring->id]) + if (offset == GEN8_SEMAPHORE_OFFSET(dev_priv, signaller->id, engine->id)) return signaller; } } else { u32 sync_bits = ipehr & MI_SEMAPHORE_SYNC_MASK; - for_each_ring(signaller, dev_priv, i) { - if(ring == signaller) + for_each_engine(signaller, dev_priv, i) { + if(engine == signaller) continue; - if (sync_bits == signaller->semaphore.mbox.wait[ring->id]) + if (sync_bits == signaller->semaphore.mbox.wait[engine->id]) return signaller; } } DRM_ERROR("No signaller ring found for ring %i, ipehr 0x%08x, offset 0x%016llx\n", - ring->id, ipehr, offset); + engine->id, ipehr, offset); return NULL; } static struct intel_engine_cs * -semaphore_waits_for(struct intel_engine_cs *ring, u32 *seqno) +semaphore_waits_for(struct intel_engine_cs *engine, u32 *seqno) { - struct drm_i915_private *dev_priv = ring->dev->dev_private; + struct drm_i915_private *dev_priv = engine->i915; + struct intel_ringbuffer *ring; u32 cmd, ipehr, head; u64 offset = 0; int i, backwards; - ipehr = I915_READ(RING_IPEHR(ring->mmio_base)); - if (!ipehr_is_semaphore_wait(ring->dev, ipehr)) + ipehr = I915_READ(RING_IPEHR(engine->mmio_base)); + if (!ipehr_is_semaphore_wait(engine->i915, ipehr)) + return NULL; + + /* XXX execlists */ + ring = engine->default_context->ring[RCS].ring; + if (ring == NULL) return NULL; /* @@ -3110,19 +3105,19 @@ semaphore_waits_for(struct intel_engine_cs *ring, u32 *seqno) * point at at batch, and semaphores are always emitted into the * ringbuffer itself. */ - head = I915_READ_HEAD(ring) & HEAD_ADDR; - backwards = (INTEL_INFO(ring->dev)->gen >= 8) ? 5 : 4; + head = I915_READ_HEAD(engine) & HEAD_ADDR; + backwards = (INTEL_INFO(dev_priv)->gen >= 8) ? 5 : 4; for (i = backwards; i; --i) { /* * Be paranoid and presume the hw has gone off into the wild - - * our ring is smaller than what the hardware (and hence + * our engine is smaller than what the hardware (and hence * HEAD_ADDR) allows. Also handles wrap-around. */ - head &= ring->buffer->size - 1; + head &= ring->size - 1; /* This here seems to blow up */ - cmd = ioread32(ring->buffer->virtual_start + head); + cmd = ioread32(ring->virtual_start + head); if (cmd == ipehr) break; @@ -3132,32 +3127,37 @@ semaphore_waits_for(struct intel_engine_cs *ring, u32 *seqno) if (!i) return NULL; - *seqno = ioread32(ring->buffer->virtual_start + head + 4) + 1; - if (INTEL_INFO(ring->dev)->gen >= 8) { - offset = ioread32(ring->buffer->virtual_start + head + 12); + *seqno = ioread32(ring->virtual_start + head + 4) + 1; + if (INTEL_INFO(dev_priv)->gen >= 8) { + offset = ioread32(ring->virtual_start + head + 12); offset <<= 32; - offset = ioread32(ring->buffer->virtual_start + head + 8); + offset = ioread32(ring->virtual_start + head + 8); } - return semaphore_wait_to_signaller_ring(ring, ipehr, offset); + return semaphore_wait_to_signaller_engine(engine, ipehr, offset); } -static int semaphore_passed(struct intel_engine_cs *ring) +static int semaphore_passed(struct intel_engine_cs *engine) { - struct drm_i915_private *dev_priv = ring->dev->dev_private; + struct drm_i915_private *dev_priv = engine->i915; struct intel_engine_cs *signaller; + struct i915_gem_request *rq; u32 seqno; - ring->hangcheck.deadlock++; + engine->hangcheck.deadlock++; + + if (engine->semaphore.wait == NULL) + return -1; - signaller = semaphore_waits_for(ring, &seqno); + signaller = semaphore_waits_for(engine, &seqno); if (signaller == NULL) return -1; /* Prevent pathological recursion due to driver bugs */ - if (signaller->hangcheck.deadlock >= I915_NUM_RINGS) + if (signaller->hangcheck.deadlock >= I915_NUM_ENGINES) return -1; - if (i915_seqno_passed(signaller->get_seqno(signaller, false), seqno)) + rq = intel_engine_seqno_to_request(engine, seqno); + if (rq == NULL || i915_request_complete(rq)) return 1; /* cursory check for an unkickable deadlock */ @@ -3170,30 +3170,29 @@ static int semaphore_passed(struct intel_engine_cs *ring) static void semaphore_clear_deadlocks(struct drm_i915_private *dev_priv) { - struct intel_engine_cs *ring; + struct intel_engine_cs *engine; int i; - for_each_ring(ring, dev_priv, i) - ring->hangcheck.deadlock = 0; + for_each_engine(engine, dev_priv, i) + engine->hangcheck.deadlock = 0; } -static enum intel_ring_hangcheck_action -ring_stuck(struct intel_engine_cs *ring, u64 acthd) +static enum intel_engine_hangcheck_action +engine_stuck(struct intel_engine_cs *engine, u64 acthd) { - struct drm_device *dev = ring->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = engine->i915; u32 tmp; - if (acthd != ring->hangcheck.acthd) { - if (acthd > ring->hangcheck.max_acthd) { - ring->hangcheck.max_acthd = acthd; + if (acthd != engine->hangcheck.acthd) { + if (acthd > engine->hangcheck.max_acthd) { + engine->hangcheck.max_acthd = acthd; return HANGCHECK_ACTIVE; } return HANGCHECK_ACTIVE_LOOP; } - if (IS_GEN2(dev)) + if (IS_GEN2(dev_priv)) return HANGCHECK_HUNG; /* Is the chip hanging on a WAIT_FOR_EVENT? @@ -3201,24 +3200,24 @@ ring_stuck(struct intel_engine_cs *ring, u64 acthd) * and break the hang. This should work on * all but the second generation chipsets. */ - tmp = I915_READ_CTL(ring); + tmp = I915_READ_CTL(engine); if (tmp & RING_WAIT) { - i915_handle_error(dev, false, + i915_handle_error(dev_priv->dev, false, "Kicking stuck wait on %s", - ring->name); - I915_WRITE_CTL(ring, tmp); + engine->name); + I915_WRITE_CTL(engine, tmp); return HANGCHECK_KICK; } - if (INTEL_INFO(dev)->gen >= 6 && tmp & RING_WAIT_SEMAPHORE) { - switch (semaphore_passed(ring)) { + if (INTEL_INFO(dev_priv)->gen >= 6 && tmp & RING_WAIT_SEMAPHORE) { + switch (semaphore_passed(engine)) { default: return HANGCHECK_HUNG; case 1: - i915_handle_error(dev, false, + i915_handle_error(dev_priv->dev, false, "Kicking stuck semaphore on %s", - ring->name); - I915_WRITE_CTL(ring, tmp); + engine->name); + I915_WRITE_CTL(engine, tmp); return HANGCHECK_KICK; case 0: return HANGCHECK_WAIT; @@ -3230,7 +3229,7 @@ ring_stuck(struct intel_engine_cs *ring, u64 acthd) /** * This is called when the chip hasn't reported back with completed - * batchbuffers in a long time. We keep track per ring seqno progress and + * batchbuffers in a long time. We keep track per engine seqno progress and * if there are no progress, hangcheck score for that ring is increased. * Further, acthd is inspected to see if the ring is stuck. On stuck case * we kick the ring. If we see no progress on three subsequent calls @@ -3238,12 +3237,11 @@ ring_stuck(struct intel_engine_cs *ring, u64 acthd) */ static void i915_hangcheck_elapsed(unsigned long data) { - struct drm_device *dev = (struct drm_device *)data; - struct drm_i915_private *dev_priv = dev->dev_private; - struct intel_engine_cs *ring; + struct drm_i915_private *dev_priv = (struct drm_i915_private *)data; + struct intel_engine_cs *engine; int i; int busy_count = 0, rings_hung = 0; - bool stuck[I915_NUM_RINGS] = { 0 }; + bool stuck[I915_NUM_ENGINES] = { 0 }; #define BUSY 1 #define KICK 5 #define HUNG 20 @@ -3251,104 +3249,108 @@ static void i915_hangcheck_elapsed(unsigned long data) if (!i915.enable_hangcheck) return; - for_each_ring(ring, dev_priv, i) { + for_each_engine(engine, dev_priv, i) { u64 acthd; u32 seqno; + u32 interrupts; bool busy = true; semaphore_clear_deadlocks(dev_priv); - seqno = ring->get_seqno(ring, false); - acthd = intel_ring_get_active_head(ring); - - if (ring->hangcheck.seqno == seqno) { - if (ring_idle(ring, seqno)) { - ring->hangcheck.action = HANGCHECK_IDLE; - - if (waitqueue_active(&ring->irq_queue)) { - /* Issue a wake-up to catch stuck h/w. */ - if (!test_and_set_bit(ring->id, &dev_priv->gpu_error.missed_irq_rings)) { - if (!(dev_priv->gpu_error.test_irq_rings & intel_ring_flag(ring))) - DRM_ERROR("Hangcheck timer elapsed... %s idle\n", - ring->name); - else - DRM_INFO("Fake missed irq on %s\n", - ring->name); - wake_up_all(&ring->irq_queue); - } - /* Safeguard against driver failure */ - ring->hangcheck.score += BUSY; - } else - busy = false; + acthd = intel_engine_get_active_head(engine); + seqno = engine->get_seqno(engine); + interrupts = atomic_read(&engine->interrupts); + + if (engine_idle(engine)) { + if (waitqueue_active(&engine->irq_queue)) { + /* Issue a wake-up to catch stuck h/w. */ + if (engine->hangcheck.action == HANGCHECK_IDLE_WAITERS && + engine->hangcheck.interrupts == interrupts && + !test_and_set_bit(engine->id, &dev_priv->gpu_error.missed_irq_rings)) { + if (!(dev_priv->gpu_error.test_irq_rings & intel_engine_flag(engine))) + DRM_ERROR("Hangcheck timer elapsed... %s idle\n", + engine->name); + else + DRM_INFO("Fake missed irq on %s\n", + engine->name); + wake_up_all(&engine->irq_queue); + } + + /* Safeguard against driver failure */ + engine->hangcheck.score += BUSY; + engine->hangcheck.action = HANGCHECK_IDLE_WAITERS; } else { - /* We always increment the hangcheck score - * if the ring is busy and still processing - * the same request, so that no single request - * can run indefinitely (such as a chain of - * batches). The only time we do not increment - * the hangcheck score on this ring, if this - * ring is in a legitimate wait for another - * ring. In that case the waiting ring is a - * victim and we want to be sure we catch the - * right culprit. Then every time we do kick - * the ring, add a small increment to the - * score so that we can catch a batch that is - * being repeatedly kicked and so responsible - * for stalling the machine. - */ - ring->hangcheck.action = ring_stuck(ring, - acthd); - - switch (ring->hangcheck.action) { + busy = false; + engine->hangcheck.action = HANGCHECK_IDLE; + } + } else if (engine->hangcheck.seqno == seqno) { + /* We always increment the hangcheck score + * if the ring is busy and still processing + * the same request, so that no single request + * can run indefinitely (such as a chain of + * batches). The only time we do not increment + * the hangcheck score on this ring, if this + * ring is in a legitimate wait for another + * ring. In that case the waiting ring is a + * victim and we want to be sure we catch the + * right culprit. Then every time we do kick + * the ring, add a small increment to the + * score so that we can catch a batch that is + * being repeatedly kicked and so responsible + * for stalling the machine. + */ + engine->hangcheck.action = engine_stuck(engine, acthd); + switch (engine->hangcheck.action) { case HANGCHECK_IDLE: + case HANGCHECK_IDLE_WAITERS: case HANGCHECK_WAIT: case HANGCHECK_ACTIVE: break; case HANGCHECK_ACTIVE_LOOP: - ring->hangcheck.score += BUSY; + engine->hangcheck.score += BUSY; break; case HANGCHECK_KICK: - ring->hangcheck.score += KICK; + engine->hangcheck.score += KICK; break; case HANGCHECK_HUNG: - ring->hangcheck.score += HUNG; + engine->hangcheck.score += HUNG; stuck[i] = true; break; - } } } else { - ring->hangcheck.action = HANGCHECK_ACTIVE; + engine->hangcheck.action = HANGCHECK_ACTIVE; /* Gradually reduce the count so that we catch DoS * attempts across multiple batches. */ - if (ring->hangcheck.score > 0) - ring->hangcheck.score--; + if (engine->hangcheck.score > 0) + engine->hangcheck.score--; - ring->hangcheck.acthd = ring->hangcheck.max_acthd = 0; + engine->hangcheck.acthd = engine->hangcheck.max_acthd = 0; } - ring->hangcheck.seqno = seqno; - ring->hangcheck.acthd = acthd; + engine->hangcheck.interrupts = interrupts; + engine->hangcheck.seqno = seqno; + engine->hangcheck.acthd = acthd; busy_count += busy; } - for_each_ring(ring, dev_priv, i) { - if (ring->hangcheck.score >= HANGCHECK_SCORE_RING_HUNG) { + for_each_engine(engine, dev_priv, i) { + if (engine->hangcheck.score >= HANGCHECK_SCORE_RING_HUNG) { DRM_INFO("%s on %s\n", stuck[i] ? "stuck" : "no progress", - ring->name); + engine->name); rings_hung++; } } if (rings_hung) - return i915_handle_error(dev, true, "Ring hung"); + return i915_handle_error(dev_priv->dev, true, "Ring hung"); if (busy_count) /* Reset timer case chip hangs without another request * being added */ - i915_queue_hangcheck(dev); + i915_queue_hangcheck(dev_priv->dev); } void i915_queue_hangcheck(struct drm_device *dev) @@ -3441,7 +3443,7 @@ static void valleyview_irq_preinstall(struct drm_device *dev) I915_WRITE(PORT_HOTPLUG_EN, 0); I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT)); - for_each_pipe(pipe) + for_each_pipe(dev_priv, pipe) I915_WRITE(PIPESTAT(pipe), 0xffff); I915_WRITE(VLV_IIR, 0xffffffff); I915_WRITE(VLV_IMR, 0xffffffff); @@ -3467,7 +3469,7 @@ static void gen8_irq_reset(struct drm_device *dev) gen8_gt_irq_reset(dev_priv); - for_each_pipe(pipe) + for_each_pipe(dev_priv, pipe) if (intel_display_power_enabled(dev_priv, POWER_DOMAIN_PIPE(pipe))) GEN8_IRQ_RESET_NDX(DE_PIPE, pipe); @@ -3481,14 +3483,12 @@ static void gen8_irq_reset(struct drm_device *dev) void gen8_irq_power_well_post_enable(struct drm_i915_private *dev_priv) { - unsigned long irqflags; - - spin_lock_irqsave(&dev_priv->irq_lock, irqflags); + spin_lock_irq(&dev_priv->irq_lock); GEN8_IRQ_INIT_NDX(DE_PIPE, PIPE_B, dev_priv->de_irq_mask[PIPE_B], ~dev_priv->de_irq_mask[PIPE_B]); GEN8_IRQ_INIT_NDX(DE_PIPE, PIPE_C, dev_priv->de_irq_mask[PIPE_C], ~dev_priv->de_irq_mask[PIPE_C]); - spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags); + spin_unlock_irq(&dev_priv->irq_lock); } static void cherryview_irq_preinstall(struct drm_device *dev) @@ -3510,7 +3510,7 @@ static void cherryview_irq_preinstall(struct drm_device *dev) I915_WRITE(PORT_HOTPLUG_EN, 0); I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT)); - for_each_pipe(pipe) + for_each_pipe(dev_priv, pipe) I915_WRITE(PIPESTAT(pipe), 0xffff); I915_WRITE(VLV_IMR, 0xffffffff); @@ -3522,18 +3522,17 @@ static void cherryview_irq_preinstall(struct drm_device *dev) static void ibx_hpd_irq_setup(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; - struct drm_mode_config *mode_config = &dev->mode_config; struct intel_encoder *intel_encoder; u32 hotplug_irqs, hotplug, enabled_irqs = 0; if (HAS_PCH_IBX(dev)) { hotplug_irqs = SDE_HOTPLUG_MASK; - list_for_each_entry(intel_encoder, &mode_config->encoder_list, base.head) + for_each_intel_encoder(dev, intel_encoder) if (dev_priv->hpd_stats[intel_encoder->hpd_pin].hpd_mark == HPD_ENABLED) enabled_irqs |= hpd_ibx[intel_encoder->hpd_pin]; } else { hotplug_irqs = SDE_HOTPLUG_MASK_CPT; - list_for_each_entry(intel_encoder, &mode_config->encoder_list, base.head) + for_each_intel_encoder(dev, intel_encoder) if (dev_priv->hpd_stats[intel_encoder->hpd_pin].hpd_mark == HPD_ENABLED) enabled_irqs |= hpd_cpt[intel_encoder->hpd_pin]; } @@ -3608,7 +3607,6 @@ static void gen5_gt_irq_postinstall(struct drm_device *dev) static int ironlake_irq_postinstall(struct drm_device *dev) { - unsigned long irqflags; struct drm_i915_private *dev_priv = dev->dev_private; u32 display_mask, extra_mask; @@ -3647,9 +3645,9 @@ static int ironlake_irq_postinstall(struct drm_device *dev) * spinlocking not required here for correctness since interrupt * setup is guaranteed to run in single-threaded context. But we * need it to make the assert_spin_locked happy. */ - spin_lock_irqsave(&dev_priv->irq_lock, irqflags); + spin_lock_irq(&dev_priv->irq_lock); ironlake_enable_display_irq(dev_priv, DE_PCU_EVENT); - spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags); + spin_unlock_irq(&dev_priv->irq_lock); } return 0; @@ -3725,7 +3723,7 @@ void valleyview_enable_display_irqs(struct drm_i915_private *dev_priv) dev_priv->display_irqs_enabled = true; - if (dev_priv->dev->irq_enabled) + if (intel_irqs_enabled(dev_priv)) valleyview_display_irqs_install(dev_priv); } @@ -3738,14 +3736,13 @@ void valleyview_disable_display_irqs(struct drm_i915_private *dev_priv) dev_priv->display_irqs_enabled = false; - if (dev_priv->dev->irq_enabled) + if (intel_irqs_enabled(dev_priv)) valleyview_display_irqs_uninstall(dev_priv); } static int valleyview_irq_postinstall(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; - unsigned long irqflags; dev_priv->irq_mask = ~0; @@ -3759,10 +3756,10 @@ static int valleyview_irq_postinstall(struct drm_device *dev) /* Interrupt setup is already guaranteed to be single-threaded, this is * just to make the assert_spin_locked check happy. */ - spin_lock_irqsave(&dev_priv->irq_lock, irqflags); + spin_lock_irq(&dev_priv->irq_lock); if (dev_priv->display_irqs_enabled) valleyview_display_irqs_install(dev_priv); - spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags); + spin_unlock_irq(&dev_priv->irq_lock); I915_WRITE(VLV_IIR, 0xffffffff); I915_WRITE(VLV_IIR, 0xffffffff); @@ -3782,28 +3779,31 @@ static int valleyview_irq_postinstall(struct drm_device *dev) static void gen8_gt_irq_postinstall(struct drm_i915_private *dev_priv) { - int i; - /* These are interrupts we'll toggle with the ring mask register */ uint32_t gt_interrupts[] = { GT_RENDER_USER_INTERRUPT << GEN8_RCS_IRQ_SHIFT | + GT_CONTEXT_SWITCH_INTERRUPT << GEN8_RCS_IRQ_SHIFT | GT_RENDER_L3_PARITY_ERROR_INTERRUPT | - GT_RENDER_USER_INTERRUPT << GEN8_BCS_IRQ_SHIFT, + GT_RENDER_USER_INTERRUPT << GEN8_BCS_IRQ_SHIFT | + GT_CONTEXT_SWITCH_INTERRUPT << GEN8_BCS_IRQ_SHIFT, GT_RENDER_USER_INTERRUPT << GEN8_VCS1_IRQ_SHIFT | - GT_RENDER_USER_INTERRUPT << GEN8_VCS2_IRQ_SHIFT, + GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VCS1_IRQ_SHIFT | + GT_RENDER_USER_INTERRUPT << GEN8_VCS2_IRQ_SHIFT | + GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VCS2_IRQ_SHIFT, 0, - GT_RENDER_USER_INTERRUPT << GEN8_VECS_IRQ_SHIFT + GT_RENDER_USER_INTERRUPT << GEN8_VECS_IRQ_SHIFT | + GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VECS_IRQ_SHIFT }; - for (i = 0; i < ARRAY_SIZE(gt_interrupts); i++) - GEN8_IRQ_INIT_NDX(GT, i, ~gt_interrupts[i], gt_interrupts[i]); - dev_priv->pm_irq_mask = 0xffffffff; + GEN8_IRQ_INIT_NDX(GT, 0, ~gt_interrupts[0], gt_interrupts[0]); + GEN8_IRQ_INIT_NDX(GT, 1, ~gt_interrupts[1], gt_interrupts[1]); + GEN8_IRQ_INIT_NDX(GT, 2, dev_priv->pm_irq_mask, dev_priv->pm_rps_events); + GEN8_IRQ_INIT_NDX(GT, 3, ~gt_interrupts[3], gt_interrupts[3]); } static void gen8_de_irq_postinstall(struct drm_i915_private *dev_priv) { - struct drm_device *dev = dev_priv->dev; uint32_t de_pipe_masked = GEN8_PIPE_PRIMARY_FLIP_DONE | GEN8_PIPE_CDCLK_CRC_DONE | GEN8_DE_PIPE_IRQ_FAULT_ERRORS; @@ -3814,7 +3814,7 @@ static void gen8_de_irq_postinstall(struct drm_i915_private *dev_priv) dev_priv->de_irq_mask[PIPE_B] = ~de_pipe_masked; dev_priv->de_irq_mask[PIPE_C] = ~de_pipe_masked; - for_each_pipe(pipe) + for_each_pipe(dev_priv, pipe) if (intel_display_power_enabled(dev_priv, POWER_DOMAIN_PIPE(pipe))) GEN8_IRQ_INIT_NDX(DE_PIPE, pipe, @@ -3850,7 +3850,6 @@ static int cherryview_irq_postinstall(struct drm_device *dev) I915_DISPLAY_PIPE_C_EVENT_INTERRUPT; u32 pipestat_enable = PLANE_FLIP_DONE_INT_STATUS_VLV | PIPE_CRC_DONE_INTERRUPT_STATUS; - unsigned long irqflags; int pipe; /* @@ -3859,14 +3858,14 @@ static int cherryview_irq_postinstall(struct drm_device *dev) */ dev_priv->irq_mask = ~enable_mask; - for_each_pipe(pipe) + for_each_pipe(dev_priv, pipe) I915_WRITE(PIPESTAT(pipe), 0xffff); - spin_lock_irqsave(&dev_priv->irq_lock, irqflags); + spin_lock_irq(&dev_priv->irq_lock); i915_enable_pipestat(dev_priv, PIPE_A, PIPE_GMBUS_INTERRUPT_STATUS); - for_each_pipe(pipe) + for_each_pipe(dev_priv, pipe) i915_enable_pipestat(dev_priv, pipe, pipestat_enable); - spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags); + spin_unlock_irq(&dev_priv->irq_lock); I915_WRITE(VLV_IIR, 0xffffffff); I915_WRITE(VLV_IMR, dev_priv->irq_mask); @@ -3893,7 +3892,6 @@ static void gen8_irq_uninstall(struct drm_device *dev) static void valleyview_irq_uninstall(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; - unsigned long irqflags; int pipe; if (!dev_priv) @@ -3901,17 +3899,19 @@ static void valleyview_irq_uninstall(struct drm_device *dev) I915_WRITE(VLV_MASTER_IER, 0); - for_each_pipe(pipe) + for_each_pipe(dev_priv, pipe) I915_WRITE(PIPESTAT(pipe), 0xffff); I915_WRITE(HWSTAM, 0xffffffff); I915_WRITE(PORT_HOTPLUG_EN, 0); I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT)); - spin_lock_irqsave(&dev_priv->irq_lock, irqflags); + /* Interrupt setup is already guaranteed to be single-threaded, this is + * just to make the assert_spin_locked check happy. */ + spin_lock_irq(&dev_priv->irq_lock); if (dev_priv->display_irqs_enabled) valleyview_display_irqs_uninstall(dev_priv); - spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags); + spin_unlock_irq(&dev_priv->irq_lock); dev_priv->irq_mask = 0; @@ -3963,7 +3963,7 @@ do { \ I915_WRITE(PORT_HOTPLUG_EN, 0); I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT)); - for_each_pipe(pipe) + for_each_pipe(dev_priv, pipe) I915_WRITE(PIPESTAT(pipe), 0xffff); I915_WRITE(VLV_IMR, 0xffffffff); @@ -3987,7 +3987,7 @@ static void i8xx_irq_preinstall(struct drm_device * dev) struct drm_i915_private *dev_priv = dev->dev_private; int pipe; - for_each_pipe(pipe) + for_each_pipe(dev_priv, pipe) I915_WRITE(PIPESTAT(pipe), 0); I915_WRITE16(IMR, 0xffff); I915_WRITE16(IER, 0x0); @@ -3997,7 +3997,6 @@ static void i8xx_irq_preinstall(struct drm_device * dev) static int i8xx_irq_postinstall(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; - unsigned long irqflags; I915_WRITE16(EMR, ~(I915_ERROR_PAGE_TABLE | I915_ERROR_MEMORY_REFRESH)); @@ -4020,10 +4019,10 @@ static int i8xx_irq_postinstall(struct drm_device *dev) /* Interrupt setup is already guaranteed to be single-threaded, this is * just to make the assert_spin_locked check happy. */ - spin_lock_irqsave(&dev_priv->irq_lock, irqflags); + spin_lock_irq(&dev_priv->irq_lock); i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS); i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS); - spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags); + spin_unlock_irq(&dev_priv->irq_lock); return 0; } @@ -4041,7 +4040,7 @@ static bool i8xx_handle_vblank(struct drm_device *dev, return false; if ((iir & flip_pending) == 0) - return false; + goto check_page_flip; intel_prepare_page_flip(dev, plane); @@ -4052,11 +4051,14 @@ static bool i8xx_handle_vblank(struct drm_device *dev, * an interrupt per se, we watch for the change at vblank. */ if (I915_READ16(ISR) & flip_pending) - return false; + goto check_page_flip; intel_finish_page_flip(dev, pipe); - return true; + +check_page_flip: + intel_check_page_flip(dev, pipe); + return false; } static irqreturn_t i8xx_irq_handler(int irq, void *arg) @@ -4065,7 +4067,6 @@ static irqreturn_t i8xx_irq_handler(int irq, void *arg) struct drm_i915_private *dev_priv = dev->dev_private; u16 iir, new_iir; u32 pipe_stats[2]; - unsigned long irqflags; int pipe; u16 flip_mask = I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT | @@ -4081,13 +4082,13 @@ static irqreturn_t i8xx_irq_handler(int irq, void *arg) * It doesn't set the bit in iir again, but it still produces * interrupts (for non-MSI). */ - spin_lock_irqsave(&dev_priv->irq_lock, irqflags); + spin_lock(&dev_priv->irq_lock); if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT) i915_handle_error(dev, false, "Command parser error, iir 0x%08x", iir); - for_each_pipe(pipe) { + for_each_pipe(dev_priv, pipe) { int reg = PIPESTAT(pipe); pipe_stats[pipe] = I915_READ(reg); @@ -4097,17 +4098,15 @@ static irqreturn_t i8xx_irq_handler(int irq, void *arg) if (pipe_stats[pipe] & 0x8000ffff) I915_WRITE(reg, pipe_stats[pipe]); } - spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags); + spin_unlock(&dev_priv->irq_lock); I915_WRITE16(IIR, iir & ~flip_mask); new_iir = I915_READ16(IIR); /* Flush posted writes */ - i915_update_dri1_breadcrumb(dev); - if (iir & I915_USER_INTERRUPT) - notify_ring(dev, &dev_priv->ring[RCS]); + notify_ring(dev, &dev_priv->engine[RCS]); - for_each_pipe(pipe) { + for_each_pipe(dev_priv, pipe) { int plane = pipe; if (HAS_FBC(dev)) plane = !plane; @@ -4135,7 +4134,7 @@ static void i8xx_irq_uninstall(struct drm_device * dev) struct drm_i915_private *dev_priv = dev->dev_private; int pipe; - for_each_pipe(pipe) { + for_each_pipe(dev_priv, pipe) { /* Clear enable bits; then clear status bits */ I915_WRITE(PIPESTAT(pipe), 0); I915_WRITE(PIPESTAT(pipe), I915_READ(PIPESTAT(pipe))); @@ -4156,7 +4155,7 @@ static void i915_irq_preinstall(struct drm_device * dev) } I915_WRITE16(HWSTAM, 0xeffe); - for_each_pipe(pipe) + for_each_pipe(dev_priv, pipe) I915_WRITE(PIPESTAT(pipe), 0); I915_WRITE(IMR, 0xffffffff); I915_WRITE(IER, 0x0); @@ -4167,7 +4166,6 @@ static int i915_irq_postinstall(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; u32 enable_mask; - unsigned long irqflags; I915_WRITE(EMR, ~(I915_ERROR_PAGE_TABLE | I915_ERROR_MEMORY_REFRESH)); @@ -4205,10 +4203,10 @@ static int i915_irq_postinstall(struct drm_device *dev) /* Interrupt setup is already guaranteed to be single-threaded, this is * just to make the assert_spin_locked check happy. */ - spin_lock_irqsave(&dev_priv->irq_lock, irqflags); + spin_lock_irq(&dev_priv->irq_lock); i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS); i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS); - spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags); + spin_unlock_irq(&dev_priv->irq_lock); return 0; } @@ -4226,7 +4224,7 @@ static bool i915_handle_vblank(struct drm_device *dev, return false; if ((iir & flip_pending) == 0) - return false; + goto check_page_flip; intel_prepare_page_flip(dev, plane); @@ -4237,11 +4235,14 @@ static bool i915_handle_vblank(struct drm_device *dev, * an interrupt per se, we watch for the change at vblank. */ if (I915_READ(ISR) & flip_pending) - return false; + goto check_page_flip; intel_finish_page_flip(dev, pipe); - return true; + +check_page_flip: + intel_check_page_flip(dev, pipe); + return false; } static irqreturn_t i915_irq_handler(int irq, void *arg) @@ -4249,7 +4250,6 @@ static irqreturn_t i915_irq_handler(int irq, void *arg) struct drm_device *dev = arg; struct drm_i915_private *dev_priv = dev->dev_private; u32 iir, new_iir, pipe_stats[I915_MAX_PIPES]; - unsigned long irqflags; u32 flip_mask = I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT | I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT; @@ -4265,13 +4265,13 @@ static irqreturn_t i915_irq_handler(int irq, void *arg) * It doesn't set the bit in iir again, but it still produces * interrupts (for non-MSI). */ - spin_lock_irqsave(&dev_priv->irq_lock, irqflags); + spin_lock(&dev_priv->irq_lock); if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT) i915_handle_error(dev, false, "Command parser error, iir 0x%08x", iir); - for_each_pipe(pipe) { + for_each_pipe(dev_priv, pipe) { int reg = PIPESTAT(pipe); pipe_stats[pipe] = I915_READ(reg); @@ -4281,7 +4281,7 @@ static irqreturn_t i915_irq_handler(int irq, void *arg) irq_received = true; } } - spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags); + spin_unlock(&dev_priv->irq_lock); if (!irq_received) break; @@ -4295,9 +4295,9 @@ static irqreturn_t i915_irq_handler(int irq, void *arg) new_iir = I915_READ(IIR); /* Flush posted writes */ if (iir & I915_USER_INTERRUPT) - notify_ring(dev, &dev_priv->ring[RCS]); + notify_ring(dev, &dev_priv->engine[RCS]); - for_each_pipe(pipe) { + for_each_pipe(dev_priv, pipe) { int plane = pipe; if (HAS_FBC(dev)) plane = !plane; @@ -4339,8 +4339,6 @@ static irqreturn_t i915_irq_handler(int irq, void *arg) iir = new_iir; } while (iir & ~flip_mask); - i915_update_dri1_breadcrumb(dev); - return ret; } @@ -4355,7 +4353,7 @@ static void i915_irq_uninstall(struct drm_device * dev) } I915_WRITE16(HWSTAM, 0xffff); - for_each_pipe(pipe) { + for_each_pipe(dev_priv, pipe) { /* Clear enable bits; then clear status bits */ I915_WRITE(PIPESTAT(pipe), 0); I915_WRITE(PIPESTAT(pipe), I915_READ(PIPESTAT(pipe))); @@ -4375,7 +4373,7 @@ static void i965_irq_preinstall(struct drm_device * dev) I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT)); I915_WRITE(HWSTAM, 0xeffe); - for_each_pipe(pipe) + for_each_pipe(dev_priv, pipe) I915_WRITE(PIPESTAT(pipe), 0); I915_WRITE(IMR, 0xffffffff); I915_WRITE(IER, 0x0); @@ -4387,7 +4385,6 @@ static int i965_irq_postinstall(struct drm_device *dev) struct drm_i915_private *dev_priv = dev->dev_private; u32 enable_mask; u32 error_mask; - unsigned long irqflags; /* Unmask the interrupts that we always want on. */ dev_priv->irq_mask = ~(I915_ASLE_INTERRUPT | @@ -4408,11 +4405,11 @@ static int i965_irq_postinstall(struct drm_device *dev) /* Interrupt setup is already guaranteed to be single-threaded, this is * just to make the assert_spin_locked check happy. */ - spin_lock_irqsave(&dev_priv->irq_lock, irqflags); + spin_lock_irq(&dev_priv->irq_lock); i915_enable_pipestat(dev_priv, PIPE_A, PIPE_GMBUS_INTERRUPT_STATUS); i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS); i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS); - spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags); + spin_unlock_irq(&dev_priv->irq_lock); /* * Enable some error detection, note the instruction error mask @@ -4444,7 +4441,6 @@ static int i965_irq_postinstall(struct drm_device *dev) static void i915_hpd_irq_setup(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; - struct drm_mode_config *mode_config = &dev->mode_config; struct intel_encoder *intel_encoder; u32 hotplug_en; @@ -4455,7 +4451,7 @@ static void i915_hpd_irq_setup(struct drm_device *dev) hotplug_en &= ~HOTPLUG_INT_EN_MASK; /* Note HDMI and DP share hotplug bits */ /* enable bits are the same for all generations */ - list_for_each_entry(intel_encoder, &mode_config->encoder_list, base.head) + for_each_intel_encoder(dev, intel_encoder) if (dev_priv->hpd_stats[intel_encoder->hpd_pin].hpd_mark == HPD_ENABLED) hotplug_en |= hpd_mask_i915[intel_encoder->hpd_pin]; /* Programming the CRT detection parameters tends @@ -4478,7 +4474,6 @@ static irqreturn_t i965_irq_handler(int irq, void *arg) struct drm_i915_private *dev_priv = dev->dev_private; u32 iir, new_iir; u32 pipe_stats[I915_MAX_PIPES]; - unsigned long irqflags; int ret = IRQ_NONE, pipe; u32 flip_mask = I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT | @@ -4495,13 +4490,13 @@ static irqreturn_t i965_irq_handler(int irq, void *arg) * It doesn't set the bit in iir again, but it still produces * interrupts (for non-MSI). */ - spin_lock_irqsave(&dev_priv->irq_lock, irqflags); + spin_lock(&dev_priv->irq_lock); if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT) i915_handle_error(dev, false, "Command parser error, iir 0x%08x", iir); - for_each_pipe(pipe) { + for_each_pipe(dev_priv, pipe) { int reg = PIPESTAT(pipe); pipe_stats[pipe] = I915_READ(reg); @@ -4513,7 +4508,7 @@ static irqreturn_t i965_irq_handler(int irq, void *arg) irq_received = true; } } - spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags); + spin_unlock(&dev_priv->irq_lock); if (!irq_received) break; @@ -4528,11 +4523,11 @@ static irqreturn_t i965_irq_handler(int irq, void *arg) new_iir = I915_READ(IIR); /* Flush posted writes */ if (iir & I915_USER_INTERRUPT) - notify_ring(dev, &dev_priv->ring[RCS]); + notify_ring(dev, &dev_priv->engine[RCS]); if (iir & I915_BSD_USER_INTERRUPT) - notify_ring(dev, &dev_priv->ring[VCS]); + notify_ring(dev, &dev_priv->engine[VCS]); - for_each_pipe(pipe) { + for_each_pipe(dev_priv, pipe) { if (pipe_stats[pipe] & PIPE_START_VBLANK_INTERRUPT_STATUS && i915_handle_vblank(dev, pipe, pipe, iir)) flip_mask &= ~DISPLAY_PLANE_FLIP_PENDING(pipe); @@ -4572,8 +4567,6 @@ static irqreturn_t i965_irq_handler(int irq, void *arg) iir = new_iir; } - i915_update_dri1_breadcrumb(dev); - return ret; } @@ -4589,30 +4582,29 @@ static void i965_irq_uninstall(struct drm_device * dev) I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT)); I915_WRITE(HWSTAM, 0xffffffff); - for_each_pipe(pipe) + for_each_pipe(dev_priv, pipe) I915_WRITE(PIPESTAT(pipe), 0); I915_WRITE(IMR, 0xffffffff); I915_WRITE(IER, 0x0); - for_each_pipe(pipe) + for_each_pipe(dev_priv, pipe) I915_WRITE(PIPESTAT(pipe), I915_READ(PIPESTAT(pipe)) & 0x8000ffff); I915_WRITE(IIR, I915_READ(IIR)); } -static void intel_hpd_irq_reenable(struct work_struct *work) +static void intel_hpd_irq_reenable_work(struct work_struct *work) { struct drm_i915_private *dev_priv = container_of(work, typeof(*dev_priv), hotplug_reenable_work.work); struct drm_device *dev = dev_priv->dev; struct drm_mode_config *mode_config = &dev->mode_config; - unsigned long irqflags; int i; intel_runtime_pm_get(dev_priv); - spin_lock_irqsave(&dev_priv->irq_lock, irqflags); + spin_lock_irq(&dev_priv->irq_lock); for (i = (HPD_NONE + 1); i < HPD_NUM_PINS; i++) { struct drm_connector *connector; @@ -4636,7 +4628,7 @@ static void intel_hpd_irq_reenable(struct work_struct *work) } if (dev_priv->display.hpd_irq_setup) dev_priv->display.hpd_irq_setup(dev); - spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags); + spin_unlock_irq(&dev_priv->irq_lock); intel_runtime_pm_put(dev_priv); } @@ -4645,24 +4637,24 @@ void intel_irq_init(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; - INIT_WORK(&dev_priv->hotplug_work, i915_hotplug_work_func); + INIT_DELAYED_WORK(&dev_priv->hotplug_work, i915_hotplug_work_func); INIT_WORK(&dev_priv->dig_port_work, i915_digport_work_func); INIT_WORK(&dev_priv->gpu_error.work, i915_error_work_func); INIT_WORK(&dev_priv->rps.work, gen6_pm_rps_work); INIT_WORK(&dev_priv->l3_parity.error_work, ivybridge_parity_work); /* Let's track the enabled rps events */ - if (IS_VALLEYVIEW(dev)) - /* WaGsvRC0ResidenncyMethod:VLV */ + if (IS_VALLEYVIEW(dev) && !IS_CHERRYVIEW(dev)) + /* WaGsvRC0ResidencyMethod:vlv */ dev_priv->pm_rps_events = GEN6_PM_RP_UP_EI_EXPIRED; else dev_priv->pm_rps_events = GEN6_PM_RPS_EVENTS; setup_timer(&dev_priv->gpu_error.hangcheck_timer, i915_hangcheck_elapsed, - (unsigned long) dev); + (unsigned long) dev_priv); INIT_DELAYED_WORK(&dev_priv->hotplug_reenable_work, - intel_hpd_irq_reenable); + intel_hpd_irq_reenable_work); pm_qos_add_request(&dev_priv->pm_qos, PM_QOS_CPU_DMA_LATENCY, PM_QOS_DEFAULT_VALUE); @@ -4746,7 +4738,6 @@ void intel_hpd_init(struct drm_device *dev) struct drm_i915_private *dev_priv = dev->dev_private; struct drm_mode_config *mode_config = &dev->mode_config; struct drm_connector *connector; - unsigned long irqflags; int i; for (i = 1; i < HPD_NUM_PINS; i++) { @@ -4764,10 +4755,10 @@ void intel_hpd_init(struct drm_device *dev) /* Interrupt setup is already guaranteed to be single-threaded, this is * just to make the assert_spin_locked checks happy. */ - spin_lock_irqsave(&dev_priv->irq_lock, irqflags); + spin_lock_irq(&dev_priv->irq_lock); if (dev_priv->display.hpd_irq_setup) dev_priv->display.hpd_irq_setup(dev); - spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags); + spin_unlock_irq(&dev_priv->irq_lock); } /* Disable interrupts so we can allow runtime PM. */ diff --git a/drivers/gpu/drm/i915/i915_reg.h b/drivers/gpu/drm/i915/i915_reg.h index f29b44c..44aaabd 100644 --- a/drivers/gpu/drm/i915/i915_reg.h +++ b/drivers/gpu/drm/i915/i915_reg.h @@ -143,6 +143,14 @@ #define GAB_CTL 0x24000 #define GAB_CTL_CONT_AFTER_PAGEFAULT (1<<8) +#define GEN7_BIOS_RESERVED 0x1082C0 +#define GEN7_BIOS_RESERVED_1M (0 << 5) +#define GEN7_BIOS_RESERVED_256K (1 << 5) +#define GEN8_BIOS_RESERVED_SHIFT 7 +#define GEN7_BIOS_RESERVED_MASK 0x1 +#define GEN8_BIOS_RESERVED_MASK 0x3 + + /* VGA stuff */ #define VGA_ST01_MDA 0x3ba @@ -272,6 +280,7 @@ #define MI_SEMAPHORE_POLL (1<<15) #define MI_SEMAPHORE_SAD_GTE_SDD (1<<12) #define MI_STORE_DWORD_IMM MI_INSTR(0x20, 1) +#define MI_STORE_DWORD_IMM_GEN8 MI_INSTR(0x20, 2) #define MI_MEM_VIRTUAL (1 << 22) /* 965+ only */ #define MI_STORE_DWORD_INDEX MI_INSTR(0x21, 1) #define MI_STORE_DWORD_INDEX_SHIFT 2 @@ -282,10 +291,11 @@ * address/value pairs. Don't overdue it, though, x <= 2^4 must hold! */ #define MI_LOAD_REGISTER_IMM(x) MI_INSTR(0x22, 2*(x)-1) +#define MI_LRI_FORCE_POSTED (1<<12) #define MI_STORE_REGISTER_MEM(x) MI_INSTR(0x24, 2*(x)-1) #define MI_STORE_REGISTER_MEM_GEN8(x) MI_INSTR(0x24, 3*(x)-1) #define MI_SRM_LRM_GLOBAL_GTT (1<<22) -#define MI_FLUSH_DW MI_INSTR(0x26, 1) /* for GEN6 */ +#define MI_FLUSH_DW MI_INSTR(0x26, 0) /* for GEN6 */ #define MI_FLUSH_DW_STORE_INDEX (1<<21) #define MI_INVALIDATE_TLB (1<<18) #define MI_FLUSH_DW_OP_STOREDW (1<<14) @@ -501,10 +511,26 @@ #define BUNIT_REG_BISOC 0x11 #define PUNIT_REG_DSPFREQ 0x36 +#define DSPFREQSTAT_SHIFT_CHV 24 +#define DSPFREQSTAT_MASK_CHV (0x1f << DSPFREQSTAT_SHIFT_CHV) +#define DSPFREQGUAR_SHIFT_CHV 8 +#define DSPFREQGUAR_MASK_CHV (0x1f << DSPFREQGUAR_SHIFT_CHV) #define DSPFREQSTAT_SHIFT 30 #define DSPFREQSTAT_MASK (0x3 << DSPFREQSTAT_SHIFT) #define DSPFREQGUAR_SHIFT 14 #define DSPFREQGUAR_MASK (0x3 << DSPFREQGUAR_SHIFT) +#define _DP_SSC(val, pipe) ((val) << (2 * (pipe))) +#define DP_SSC_MASK(pipe) _DP_SSC(0x3, (pipe)) +#define DP_SSC_PWR_ON(pipe) _DP_SSC(0x0, (pipe)) +#define DP_SSC_CLK_GATE(pipe) _DP_SSC(0x1, (pipe)) +#define DP_SSC_RESET(pipe) _DP_SSC(0x2, (pipe)) +#define DP_SSC_PWR_GATE(pipe) _DP_SSC(0x3, (pipe)) +#define _DP_SSS(val, pipe) ((val) << (2 * (pipe) + 16)) +#define DP_SSS_MASK(pipe) _DP_SSS(0x3, (pipe)) +#define DP_SSS_PWR_ON(pipe) _DP_SSS(0x0, (pipe)) +#define DP_SSS_CLK_GATE(pipe) _DP_SSS(0x1, (pipe)) +#define DP_SSS_RESET(pipe) _DP_SSS(0x2, (pipe)) +#define DP_SSS_PWR_GATE(pipe) _DP_SSS(0x3, (pipe)) /* See the PUNIT HAS v0.8 for the below bits */ enum punit_power_well { @@ -518,6 +544,11 @@ enum punit_power_well { PUNIT_POWER_WELL_DPIO_TX_C_LANES_23 = 9, PUNIT_POWER_WELL_DPIO_RX0 = 10, PUNIT_POWER_WELL_DPIO_RX1 = 11, + PUNIT_POWER_WELL_DPIO_CMN_D = 12, + /* FIXME: guesswork below */ + PUNIT_POWER_WELL_DPIO_TX_D_LANES_01 = 13, + PUNIT_POWER_WELL_DPIO_TX_D_LANES_23 = 14, + PUNIT_POWER_WELL_DPIO_RX2 = 15, PUNIT_POWER_WELL_NUM, }; @@ -838,8 +869,8 @@ enum punit_power_well { #define _VLV_TX_DW2_CH0 0x8288 #define _VLV_TX_DW2_CH1 0x8488 -#define DPIO_SWING_MARGIN_SHIFT 16 -#define DPIO_SWING_MARGIN_MASK (0xff << DPIO_SWING_MARGIN_SHIFT) +#define DPIO_SWING_MARGIN000_SHIFT 16 +#define DPIO_SWING_MARGIN000_MASK (0xff << DPIO_SWING_MARGIN000_SHIFT) #define DPIO_UNIQ_TRANS_SCALE_SHIFT 8 #define VLV_TX_DW2(ch) _PORT(ch, _VLV_TX_DW2_CH0, _VLV_TX_DW2_CH1) @@ -847,12 +878,16 @@ enum punit_power_well { #define _VLV_TX_DW3_CH1 0x848c /* The following bit for CHV phy */ #define DPIO_TX_UNIQ_TRANS_SCALE_EN (1<<27) +#define DPIO_SWING_MARGIN101_SHIFT 16 +#define DPIO_SWING_MARGIN101_MASK (0xff << DPIO_SWING_MARGIN101_SHIFT) #define VLV_TX_DW3(ch) _PORT(ch, _VLV_TX_DW3_CH0, _VLV_TX_DW3_CH1) #define _VLV_TX_DW4_CH0 0x8290 #define _VLV_TX_DW4_CH1 0x8490 #define DPIO_SWING_DEEMPH9P5_SHIFT 24 #define DPIO_SWING_DEEMPH9P5_MASK (0xff << DPIO_SWING_DEEMPH9P5_SHIFT) +#define DPIO_SWING_DEEMPH6P0_SHIFT 16 +#define DPIO_SWING_DEEMPH6P0_MASK (0xff << DPIO_SWING_DEEMPH6P0_SHIFT) #define VLV_TX_DW4(ch) _PORT(ch, _VLV_TX_DW4_CH0, _VLV_TX_DW4_CH1) #define _VLV_TX3_DW4_CH0 0x690 @@ -1003,6 +1038,13 @@ enum punit_power_well { #define PGTBL_ADDRESS_LO_MASK 0xfffff000 /* bits [31:12] */ #define PGTBL_ADDRESS_HI_MASK 0x000000f0 /* bits [35:32] (gen4) */ #define PGTBL_ER 0x02024 +#define PRB0_BASE (0x2030-0x30) +#define PRB1_BASE (0x2040-0x30) /* 830,gen3 */ +#define PRB2_BASE (0x2050-0x30) /* gen3 */ +#define SRB0_BASE (0x2100-0x30) /* gen2 */ +#define SRB1_BASE (0x2110-0x30) /* gen2 */ +#define SRB2_BASE (0x2120-0x30) /* 830 */ +#define SRB3_BASE (0x2130-0x30) /* 830 */ #define RENDER_RING_BASE 0x02000 #define BSD_RING_BASE 0x04000 #define GEN6_BSD_RING_BASE 0x12000 @@ -1064,6 +1106,7 @@ enum punit_power_well { #define RING_ACTHD_UDW(base) ((base)+0x5c) #define RING_NOPID(base) ((base)+0x94) #define RING_IMR(base) ((base)+0xa8) +#define RING_HWSTAM(base) ((base)+0x98) #define RING_TIMESTAMP(base) ((base)+0x358) #define TAIL_ADDR 0x001FFFF8 #define HEAD_WRAP_COUNT 0xFFE00000 @@ -1248,6 +1291,10 @@ enum punit_power_well { #define INSTPM_TLB_INVALIDATE (1<<9) #define INSTPM_SYNC_FLUSH (1<<5) #define ACTHD 0x020c8 +#define MEM_MODE 0x020cc +#define MEM_DISPLAY_B_TRICKLE_FEED_DISABLE (1<<3) /* 830 only */ +#define MEM_DISPLAY_A_TRICKLE_FEED_DISABLE (1<<2) /* 830/845 only */ +#define MEM_DISPLAY_TRICKLE_FEED_DISABLE (1<<2) /* 85x only */ #define FW_BLC 0x020d8 #define FW_BLC2 0x020dc #define FW_BLC_SELF 0x020e0 /* 915+ only */ @@ -1380,6 +1427,7 @@ enum punit_power_well { #define GT_BSD_CS_ERROR_INTERRUPT (1 << 15) #define GT_BSD_USER_INTERRUPT (1 << 12) #define GT_RENDER_L3_PARITY_ERROR_INTERRUPT_S1 (1 << 11) /* hsw+; rsvd on snb, ivb, vlv */ +#define GT_CONTEXT_SWITCH_INTERRUPT (1 << 8) #define GT_RENDER_L3_PARITY_ERROR_INTERRUPT (1 << 5) /* !snb */ #define GT_RENDER_PIPECTL_NOTIFY_INTERRUPT (1 << 4) #define GT_RENDER_CS_MASTER_ERROR_INTERRUPT (1 << 3) @@ -1519,6 +1567,7 @@ enum punit_power_well { /* Framebuffer compression for Ironlake */ #define ILK_DPFC_CB_BASE 0x43200 #define ILK_DPFC_CONTROL 0x43208 +#define FBC_CTL_FALSE_COLOR (1<<10) /* The bit 28-8 is reserved */ #define DPFC_RESERVED (0x1FFFFF00) #define ILK_DPFC_RECOMP_CTL 0x4320c @@ -1675,12 +1724,9 @@ enum punit_power_well { #define DPIO_PHY_STATUS (VLV_DISPLAY_BASE + 0x6240) #define DPLL_PORTD_READY_MASK (0xf) #define DISPLAY_PHY_CONTROL (VLV_DISPLAY_BASE + 0x60100) -#define PHY_COM_LANE_RESET_DEASSERT(phy, val) \ - ((phy == DPIO_PHY0) ? (val | 1) : (val | 2)) -#define PHY_COM_LANE_RESET_ASSERT(phy, val) \ - ((phy == DPIO_PHY0) ? (val & ~1) : (val & ~2)) +#define PHY_COM_LANE_RESET_DEASSERT(phy) (1 << (phy)) #define DISPLAY_PHY_STATUS (VLV_DISPLAY_BASE + 0x60104) -#define PHY_POWERGOOD(phy) ((phy == DPIO_PHY0) ? (1<<31) : (1<<30)) +#define PHY_POWERGOOD(phy) (((phy) == DPIO_PHY0) ? (1<<31) : (1<<30)) /* * The i830 generation, in LVDS mode, defines P1 as the bit number set within @@ -2261,6 +2307,7 @@ enum punit_power_well { * doesn't need saving on GT1 */ #define CXT_SIZE 0x21a0 +#define ILK_CXT_TOTAL_SIZE (1 * PAGE_SIZE) #define GEN6_CXT_POWER_SIZE(cxt_reg) ((cxt_reg >> 24) & 0x3f) #define GEN6_CXT_RING_SIZE(cxt_reg) ((cxt_reg >> 18) & 0x3f) #define GEN6_CXT_RENDER_SIZE(cxt_reg) ((cxt_reg >> 12) & 0x3f) @@ -3476,6 +3523,8 @@ enum punit_power_well { #define DP_LINK_TRAIN_OFF (3 << 28) #define DP_LINK_TRAIN_MASK (3 << 28) #define DP_LINK_TRAIN_SHIFT 28 +#define DP_LINK_TRAIN_PAT_3_CHV (1 << 14) +#define DP_LINK_TRAIN_MASK_CHV ((3 << 28)|(1<<14)) /* CPT Link training mode */ #define DP_LINK_TRAIN_PAT_1_CPT (0 << 8) @@ -3732,7 +3781,6 @@ enum punit_power_well { #define PIPE_VSYNC_INTERRUPT_STATUS (1UL<<9) #define PIPE_DISPLAY_LINE_COMPARE_STATUS (1UL<<8) #define PIPE_DPST_EVENT_STATUS (1UL<<7) -#define PIPE_LEGACY_BLC_EVENT_STATUS (1UL<<6) #define PIPE_A_PSR_STATUS_VLV (1UL<<6) #define PIPE_LEGACY_BLC_EVENT_STATUS (1UL<<6) #define PIPE_ODD_FIELD_INTERRUPT_STATUS (1UL<<5) @@ -3842,73 +3890,151 @@ enum punit_power_well { #define DSPARB_BEND_SHIFT 9 /* on 855 */ #define DSPARB_AEND_SHIFT 0 +/* pnv/gen4/g4x/vlv/chv */ #define DSPFW1 (dev_priv->info.display_mmio_offset + 0x70034) -#define DSPFW_SR_SHIFT 23 -#define DSPFW_SR_MASK (0x1ff<<23) -#define DSPFW_CURSORB_SHIFT 16 -#define DSPFW_CURSORB_MASK (0x3f<<16) -#define DSPFW_PLANEB_SHIFT 8 -#define DSPFW_PLANEB_MASK (0x7f<<8) -#define DSPFW_PLANEA_MASK (0x7f) +#define DSPFW_SR_SHIFT 23 +#define DSPFW_SR_MASK (0x1ff<<23) +#define DSPFW_CURSORB_SHIFT 16 +#define DSPFW_CURSORB_MASK (0x3f<<16) +#define DSPFW_PLANEB_SHIFT 8 +#define DSPFW_PLANEB_MASK (0x7f<<8) +#define DSPFW_PLANEB_MASK_VLV (0xff<<8) /* vlv/chv */ +#define DSPFW_PLANEA_SHIFT 0 +#define DSPFW_PLANEA_MASK (0x7f<<0) +#define DSPFW_PLANEA_MASK_VLV (0xff<<0) /* vlv/chv */ #define DSPFW2 (dev_priv->info.display_mmio_offset + 0x70038) -#define DSPFW_CURSORA_MASK 0x00003f00 -#define DSPFW_CURSORA_SHIFT 8 -#define DSPFW_PLANEC_MASK (0x7f) +#define DSPFW_FBC_SR_EN (1<<31) /* g4x */ +#define DSPFW_FBC_SR_SHIFT 28 +#define DSPFW_FBC_SR_MASK (0x7<<28) /* g4x */ +#define DSPFW_FBC_HPLL_SR_SHIFT 24 +#define DSPFW_FBC_HPLL_SR_MASK (0xf<<24) /* g4x */ +#define DSPFW_SPRITEB_SHIFT (16) +#define DSPFW_SPRITEB_MASK (0x7f<<16) /* g4x */ +#define DSPFW_SPRITEB_MASK_VLV (0xff<<16) /* vlv/chv */ +#define DSPFW_CURSORA_SHIFT 8 +#define DSPFW_CURSORA_MASK (0x3f<<8) +#define DSPFW_PLANEC_SHIFT_OLD 0 +#define DSPFW_PLANEC_MASK_OLD (0x7f<<0) /* pre-gen4 sprite C */ +#define DSPFW_SPRITEA_SHIFT 0 +#define DSPFW_SPRITEA_MASK (0x7f<<0) /* g4x */ +#define DSPFW_SPRITEA_MASK_VLV (0xff<<0) /* vlv/chv */ #define DSPFW3 (dev_priv->info.display_mmio_offset + 0x7003c) -#define DSPFW_HPLL_SR_EN (1<<31) -#define DSPFW_CURSOR_SR_SHIFT 24 +#define DSPFW_HPLL_SR_EN (1<<31) #define PINEVIEW_SELF_REFRESH_EN (1<<30) +#define DSPFW_CURSOR_SR_SHIFT 24 #define DSPFW_CURSOR_SR_MASK (0x3f<<24) #define DSPFW_HPLL_CURSOR_SHIFT 16 #define DSPFW_HPLL_CURSOR_MASK (0x3f<<16) -#define DSPFW_HPLL_SR_MASK (0x1ff) -#define DSPFW4 (dev_priv->info.display_mmio_offset + 0x70070) -#define DSPFW7 (dev_priv->info.display_mmio_offset + 0x7007c) +#define DSPFW_HPLL_SR_SHIFT 0 +#define DSPFW_HPLL_SR_MASK (0x1ff<<0) + +/* vlv/chv */ +#define DSPFW4 (VLV_DISPLAY_BASE + 0x70070) +#define DSPFW_SPRITEB_WM1_SHIFT 16 +#define DSPFW_SPRITEB_WM1_MASK (0xff<<16) +#define DSPFW_CURSORA_WM1_SHIFT 8 +#define DSPFW_CURSORA_WM1_MASK (0x3f<<8) +#define DSPFW_SPRITEA_WM1_SHIFT 0 +#define DSPFW_SPRITEA_WM1_MASK (0xff<<0) +#define DSPFW5 (VLV_DISPLAY_BASE + 0x70074) +#define DSPFW_PLANEB_WM1_SHIFT 24 +#define DSPFW_PLANEB_WM1_MASK (0xff<<24) +#define DSPFW_PLANEA_WM1_SHIFT 16 +#define DSPFW_PLANEA_WM1_MASK (0xff<<16) +#define DSPFW_CURSORB_WM1_SHIFT 8 +#define DSPFW_CURSORB_WM1_MASK (0x3f<<8) +#define DSPFW_CURSOR_SR_WM1_SHIFT 0 +#define DSPFW_CURSOR_SR_WM1_MASK (0x3f<<0) +#define DSPFW6 (VLV_DISPLAY_BASE + 0x70078) +#define DSPFW_SR_WM1_SHIFT 0 +#define DSPFW_SR_WM1_MASK (0x1ff<<0) +#define DSPFW7 (VLV_DISPLAY_BASE + 0x7007c) +#define DSPFW7_CHV (VLV_DISPLAY_BASE + 0x700b4) /* wtf #1? */ +#define DSPFW_SPRITED_WM1_SHIFT 24 +#define DSPFW_SPRITED_WM1_MASK (0xff<<24) +#define DSPFW_SPRITED_SHIFT 16 +#define DSPFW_SPRITED_MASK (0xff<<16) +#define DSPFW_SPRITEC_WM1_SHIFT 8 +#define DSPFW_SPRITEC_WM1_MASK (0xff<<8) +#define DSPFW_SPRITEC_SHIFT 0 +#define DSPFW_SPRITEC_MASK (0xff<<0) +#define DSPFW8_CHV (VLV_DISPLAY_BASE + 0x700b8) +#define DSPFW_SPRITEF_WM1_SHIFT 24 +#define DSPFW_SPRITEF_WM1_MASK (0xff<<24) +#define DSPFW_SPRITEF_SHIFT 16 +#define DSPFW_SPRITEF_MASK (0xff<<16) +#define DSPFW_SPRITEE_WM1_SHIFT 8 +#define DSPFW_SPRITEE_WM1_MASK (0xff<<8) +#define DSPFW_SPRITEE_SHIFT 0 +#define DSPFW_SPRITEE_MASK (0xff<<0) +#define DSPFW9_CHV (VLV_DISPLAY_BASE + 0x7007c) /* wtf #2? */ +#define DSPFW_PLANEC_WM1_SHIFT 24 +#define DSPFW_PLANEC_WM1_MASK (0xff<<24) +#define DSPFW_PLANEC_SHIFT 16 +#define DSPFW_PLANEC_MASK (0xff<<16) +#define DSPFW_CURSORC_WM1_SHIFT 8 +#define DSPFW_CURSORC_WM1_MASK (0x3f<<16) +#define DSPFW_CURSORC_SHIFT 0 +#define DSPFW_CURSORC_MASK (0x3f<<0) + +/* vlv/chv high order bits */ +#define DSPHOWM (VLV_DISPLAY_BASE + 0x70064) +#define DSPFW_SR_HI_SHIFT 24 +#define DSPFW_SR_HI_MASK (1<<24) +#define DSPFW_SPRITEF_HI_SHIFT 23 +#define DSPFW_SPRITEF_HI_MASK (1<<23) +#define DSPFW_SPRITEE_HI_SHIFT 22 +#define DSPFW_SPRITEE_HI_MASK (1<<22) +#define DSPFW_PLANEC_HI_SHIFT 21 +#define DSPFW_PLANEC_HI_MASK (1<<21) +#define DSPFW_SPRITED_HI_SHIFT 20 +#define DSPFW_SPRITED_HI_MASK (1<<20) +#define DSPFW_SPRITEC_HI_SHIFT 16 +#define DSPFW_SPRITEC_HI_MASK (1<<16) +#define DSPFW_PLANEB_HI_SHIFT 12 +#define DSPFW_PLANEB_HI_MASK (1<<12) +#define DSPFW_SPRITEB_HI_SHIFT 8 +#define DSPFW_SPRITEB_HI_MASK (1<<8) +#define DSPFW_SPRITEA_HI_SHIFT 4 +#define DSPFW_SPRITEA_HI_MASK (1<<4) +#define DSPFW_PLANEA_HI_SHIFT 0 +#define DSPFW_PLANEA_HI_MASK (1<<0) +#define DSPHOWM1 (VLV_DISPLAY_BASE + 0x70068) +#define DSPFW_SR_WM1_HI_SHIFT 24 +#define DSPFW_SR_WM1_HI_MASK (1<<24) +#define DSPFW_SPRITEF_WM1_HI_SHIFT 23 +#define DSPFW_SPRITEF_WM1_HI_MASK (1<<23) +#define DSPFW_SPRITEE_WM1_HI_SHIFT 22 +#define DSPFW_SPRITEE_WM1_HI_MASK (1<<22) +#define DSPFW_PLANEC_WM1_HI_SHIFT 21 +#define DSPFW_PLANEC_WM1_HI_MASK (1<<21) +#define DSPFW_SPRITED_WM1_HI_SHIFT 20 +#define DSPFW_SPRITED_WM1_HI_MASK (1<<20) +#define DSPFW_SPRITEC_WM1_HI_SHIFT 16 +#define DSPFW_SPRITEC_WM1_HI_MASK (1<<16) +#define DSPFW_PLANEB_WM1_HI_SHIFT 12 +#define DSPFW_PLANEB_WM1_HI_MASK (1<<12) +#define DSPFW_SPRITEB_WM1_HI_SHIFT 8 +#define DSPFW_SPRITEB_WM1_HI_MASK (1<<8) +#define DSPFW_SPRITEA_WM1_HI_SHIFT 4 +#define DSPFW_SPRITEA_WM1_HI_MASK (1<<4) +#define DSPFW_PLANEA_WM1_HI_SHIFT 0 +#define DSPFW_PLANEA_WM1_HI_MASK (1<<0) /* drain latency register values*/ #define DRAIN_LATENCY_PRECISION_32 32 #define DRAIN_LATENCY_PRECISION_64 64 -#define VLV_DDL1 (VLV_DISPLAY_BASE + 0x70050) -#define DDL_CURSORA_PRECISION_64 (1<<31) -#define DDL_CURSORA_PRECISION_32 (0<<31) -#define DDL_CURSORA_SHIFT 24 -#define DDL_SPRITEB_PRECISION_64 (1<<23) -#define DDL_SPRITEB_PRECISION_32 (0<<23) -#define DDL_SPRITEB_SHIFT 16 -#define DDL_SPRITEA_PRECISION_64 (1<<15) -#define DDL_SPRITEA_PRECISION_32 (0<<15) -#define DDL_SPRITEA_SHIFT 8 -#define DDL_PLANEA_PRECISION_64 (1<<7) -#define DDL_PLANEA_PRECISION_32 (0<<7) -#define DDL_PLANEA_SHIFT 0 - -#define VLV_DDL2 (VLV_DISPLAY_BASE + 0x70054) -#define DDL_CURSORB_PRECISION_64 (1<<31) -#define DDL_CURSORB_PRECISION_32 (0<<31) -#define DDL_CURSORB_SHIFT 24 -#define DDL_SPRITED_PRECISION_64 (1<<23) -#define DDL_SPRITED_PRECISION_32 (0<<23) -#define DDL_SPRITED_SHIFT 16 -#define DDL_SPRITEC_PRECISION_64 (1<<15) -#define DDL_SPRITEC_PRECISION_32 (0<<15) -#define DDL_SPRITEC_SHIFT 8 -#define DDL_PLANEB_PRECISION_64 (1<<7) -#define DDL_PLANEB_PRECISION_32 (0<<7) -#define DDL_PLANEB_SHIFT 0 - -#define VLV_DDL3 (VLV_DISPLAY_BASE + 0x70058) -#define DDL_CURSORC_PRECISION_64 (1<<31) -#define DDL_CURSORC_PRECISION_32 (0<<31) -#define DDL_CURSORC_SHIFT 24 -#define DDL_SPRITEF_PRECISION_64 (1<<23) -#define DDL_SPRITEF_PRECISION_32 (0<<23) -#define DDL_SPRITEF_SHIFT 16 -#define DDL_SPRITEE_PRECISION_64 (1<<15) -#define DDL_SPRITEE_PRECISION_32 (0<<15) -#define DDL_SPRITEE_SHIFT 8 -#define DDL_PLANEC_PRECISION_64 (1<<7) -#define DDL_PLANEC_PRECISION_32 (0<<7) -#define DDL_PLANEC_SHIFT 0 +#define VLV_DDL(pipe) (VLV_DISPLAY_BASE + 0x70050 + 4 * (pipe)) +#define DDL_CURSOR_PRECISION_64 (1<<31) +#define DDL_CURSOR_PRECISION_32 (0<<31) +#define DDL_CURSOR_SHIFT 24 +#define DDL_SPRITE_PRECISION_64(sprite) (1<<(15+8*(sprite))) +#define DDL_SPRITE_PRECISION_32(sprite) (0<<(15+8*(sprite))) +#define DDL_SPRITE_SHIFT(sprite) (8+8*(sprite)) +#define DDL_PLANE_PRECISION_64 (1<<7) +#define DDL_PLANE_PRECISION_32 (0<<7) +#define DDL_PLANE_SHIFT 0 +#define DRAIN_LATENCY_MASK 0x7f /* FIFO watermark sizes etc */ #define G4X_FIFO_LINE_SIZE 64 @@ -4026,7 +4152,8 @@ enum punit_power_well { /* Old style CUR*CNTR flags (desktop 8xx) */ #define CURSOR_ENABLE 0x80000000 #define CURSOR_GAMMA_ENABLE 0x40000000 -#define CURSOR_STRIDE_MASK 0x30000000 +#define CURSOR_STRIDE_SHIFT 28 +#define CURSOR_STRIDE(x) ((ffs(x)-9) << CURSOR_STRIDE_SHIFT) /* 256,512,1k,2k */ #define CURSOR_PIPE_CSC_ENABLE (1<<24) #define CURSOR_FORMAT_SHIFT 24 #define CURSOR_FORMAT_MASK (0x07 << CURSOR_FORMAT_SHIFT) @@ -4111,6 +4238,7 @@ enum punit_power_well { #define DISPPLANE_NO_LINE_DOUBLE 0 #define DISPPLANE_STEREO_POLARITY_FIRST 0 #define DISPPLANE_STEREO_POLARITY_SECOND (1<<18) +#define DISPPLANE_ROTATE_180 (1<<15) #define DISPPLANE_TRICKLE_FEED_DISABLE (1<<14) /* Ironlake */ #define DISPPLANE_TILED (1<<10) #define _DSPAADDR 0x70184 @@ -4195,6 +4323,7 @@ enum punit_power_well { #define DVS_YUV_ORDER_UYVY (1<<16) #define DVS_YUV_ORDER_YVYU (2<<16) #define DVS_YUV_ORDER_VYUY (3<<16) +#define DVS_ROTATE_180 (1<<15) #define DVS_DEST_KEY (1<<2) #define DVS_TRICKLE_FEED_DISABLE (1<<14) #define DVS_TILED (1<<10) @@ -4265,6 +4394,7 @@ enum punit_power_well { #define SPRITE_YUV_ORDER_UYVY (1<<16) #define SPRITE_YUV_ORDER_YVYU (2<<16) #define SPRITE_YUV_ORDER_VYUY (3<<16) +#define SPRITE_ROTATE_180 (1<<15) #define SPRITE_TRICKLE_FEED_DISABLE (1<<14) #define SPRITE_INT_GAMMA_ENABLE (1<<13) #define SPRITE_TILED (1<<10) @@ -4338,6 +4468,7 @@ enum punit_power_well { #define SP_YUV_ORDER_UYVY (1<<16) #define SP_YUV_ORDER_YVYU (2<<16) #define SP_YUV_ORDER_VYUY (3<<16) +#define SP_ROTATE_180 (1<<15) #define SP_TILED (1<<10) #define _SPALINOFF (VLV_DISPLAY_BASE + 0x72184) #define _SPASTRIDE (VLV_DISPLAY_BASE + 0x72188) @@ -5246,8 +5377,7 @@ enum punit_power_well { #define PIPEA_PP_STATUS (VLV_DISPLAY_BASE + 0x61200) #define PIPEA_PP_CONTROL (VLV_DISPLAY_BASE + 0x61204) #define PIPEA_PP_ON_DELAYS (VLV_DISPLAY_BASE + 0x61208) -#define PANEL_PORT_SELECT_DPB_VLV (1 << 30) -#define PANEL_PORT_SELECT_DPC_VLV (2 << 30) +#define PANEL_PORT_SELECT_VLV(port) ((port) << 30) #define PIPEA_PP_OFF_DELAYS (VLV_DISPLAY_BASE + 0x6120c) #define PIPEA_PP_DIVISOR (VLV_DISPLAY_BASE + 0x61210) @@ -5407,7 +5537,6 @@ enum punit_power_well { #define VLV_GTLC_ALLOWWAKEERR (1 << 1) #define VLV_GTLC_PW_MEDIA_STATUS_MASK (1 << 5) #define VLV_GTLC_PW_RENDER_STATUS_MASK (1 << 7) -#define VLV_GTLC_SURVIVABILITY_REG 0x130098 #define FORCEWAKE_MT 0xa188 /* multi-threaded */ #define FORCEWAKE_KERNEL 0x1 #define FORCEWAKE_USER 0x2 @@ -5457,6 +5586,10 @@ enum punit_power_well { #define GEN8_UCGCTL6 0x9430 #define GEN8_SDEUNIT_CLOCK_GATE_DISABLE (1<<14) +#define TIMESTAMP_CTR 0x44070 +#define FREQ_1_28_US(us) (((us) * 100) >> 7) +#define MCHBAR_PCU_C0 (MCHBAR_MIRROR_BASE_SNB + 0x5960) + #define GEN6_GFXPAUSE 0xA000 #define GEN6_RPNSWREQ 0xA008 #define GEN6_TURBO_DISABLE (1<<31) @@ -5545,12 +5678,6 @@ enum punit_power_well { GEN6_PM_RP_DOWN_THRESHOLD | \ GEN6_PM_RP_DOWN_TIMEOUT) -#define CHV_CZ_CLOCK_FREQ_MODE_200 200 -#define CHV_CZ_CLOCK_FREQ_MODE_267 267 -#define CHV_CZ_CLOCK_FREQ_MODE_320 320 -#define CHV_CZ_CLOCK_FREQ_MODE_333 333 -#define CHV_CZ_CLOCK_FREQ_MODE_400 400 - #define GEN7_GT_SCRATCH_BASE 0x4F100 #define GEN7_GT_SCRATCH_REG_NUM 8 @@ -5866,15 +5993,7 @@ enum punit_power_well { #define DDI_BUF_CTL_B 0x64100 #define DDI_BUF_CTL(port) _PORT(port, DDI_BUF_CTL_A, DDI_BUF_CTL_B) #define DDI_BUF_CTL_ENABLE (1<<31) -#define DDI_BUF_EMP_400MV_0DB_HSW (0<<24) /* Sel0 */ -#define DDI_BUF_EMP_400MV_3_5DB_HSW (1<<24) /* Sel1 */ -#define DDI_BUF_EMP_400MV_6DB_HSW (2<<24) /* Sel2 */ -#define DDI_BUF_EMP_400MV_9_5DB_HSW (3<<24) /* Sel3 */ -#define DDI_BUF_EMP_600MV_0DB_HSW (4<<24) /* Sel4 */ -#define DDI_BUF_EMP_600MV_3_5DB_HSW (5<<24) /* Sel5 */ -#define DDI_BUF_EMP_600MV_6DB_HSW (6<<24) /* Sel6 */ -#define DDI_BUF_EMP_800MV_0DB_HSW (7<<24) /* Sel7 */ -#define DDI_BUF_EMP_800MV_3_5DB_HSW (8<<24) /* Sel8 */ +#define DDI_BUF_TRANS_SELECT(n) ((n) << 24) #define DDI_BUF_EMP_MASK (0xf<<24) #define DDI_BUF_PORT_REVERSAL (1<<16) #define DDI_BUF_IS_IDLE (1<<7) diff --git a/drivers/gpu/drm/i915/i915_sysfs.c b/drivers/gpu/drm/i915/i915_sysfs.c index ae7fd8f..503847f 100644 --- a/drivers/gpu/drm/i915/i915_sysfs.c +++ b/drivers/gpu/drm/i915/i915_sysfs.c @@ -540,7 +540,7 @@ static ssize_t error_state_read(struct file *filp, struct kobject *kobj, memset(&error_priv, 0, sizeof(error_priv)); - ret = i915_error_state_buf_init(&error_str, count, off); + ret = i915_error_state_buf_init(&error_str, to_i915(dev), count, off); if (ret) return ret; diff --git a/drivers/gpu/drm/i915/i915_trace.h b/drivers/gpu/drm/i915/i915_trace.h index f5aa006..8bb51dc 100644 --- a/drivers/gpu/drm/i915/i915_trace.h +++ b/drivers/gpu/drm/i915/i915_trace.h @@ -325,11 +325,10 @@ TRACE_EVENT(i915_gem_evict_vm, TP_printk("dev=%d, vm=%p", __entry->dev, __entry->vm) ); -TRACE_EVENT(i915_gem_ring_sync_to, - TP_PROTO(struct intel_engine_cs *from, - struct intel_engine_cs *to, - u32 seqno), - TP_ARGS(from, to, seqno), +TRACE_EVENT(i915_gem_ring_wait, + TP_PROTO(struct i915_gem_request *waiter, + struct i915_gem_request *signaller), + TP_ARGS(waiter, signaller), TP_STRUCT__entry( __field(u32, dev) @@ -339,18 +338,40 @@ TRACE_EVENT(i915_gem_ring_sync_to, ), TP_fast_assign( - __entry->dev = from->dev->primary->index; - __entry->sync_from = from->id; - __entry->sync_to = to->id; - __entry->seqno = seqno; + __entry->dev = waiter->i915->dev->primary->index; + __entry->sync_from = waiter->engine->id; + __entry->sync_to = signaller->engine->id; + __entry->seqno = signaller->breadcrumb[waiter->engine->id]; ), - TP_printk("dev=%u, sync-from=%u, sync-to=%u, seqno=%u", + TP_printk("dev=%u, sync-from=%u, sync-to=%u, seqno=%x", __entry->dev, __entry->sync_from, __entry->sync_to, __entry->seqno) ); +TRACE_EVENT(i915_gem_ring_switch_context, + TP_PROTO(struct intel_engine_cs *engine, struct intel_context *ctx, u32 flags), + TP_ARGS(engine, ctx, flags), + + TP_STRUCT__entry( + __field(u32, dev) + __field(u32, ring) + __field(u32, ctx) + __field(u32, flags) + ), + + TP_fast_assign( + __entry->dev = engine->i915->dev->primary->index; + __entry->ring = engine->id; + __entry->ctx = ctx->file_priv ? ctx->user_handle : -1; + __entry->flags = flags; + ), + + TP_printk("dev=%u, ring=%u, ctx=%d, flags=0x%08x", + __entry->dev, __entry->ring, __entry->ctx, __entry->flags) +); + TRACE_EVENT(i915_gem_ring_dispatch, TP_PROTO(struct intel_engine_cs *ring, u32 seqno, u32 flags), TP_ARGS(ring, seqno, flags), @@ -363,66 +384,84 @@ TRACE_EVENT(i915_gem_ring_dispatch, ), TP_fast_assign( - __entry->dev = ring->dev->primary->index; + __entry->dev = ring->i915->dev->primary->index; __entry->ring = ring->id; __entry->seqno = seqno; __entry->flags = flags; i915_trace_irq_get(ring, seqno); ), - TP_printk("dev=%u, ring=%u, seqno=%u, flags=%x", + TP_printk("dev=%u, ring=%u, seqno=%x, flags=%x", __entry->dev, __entry->ring, __entry->seqno, __entry->flags) ); -TRACE_EVENT(i915_gem_ring_flush, - TP_PROTO(struct intel_engine_cs *ring, u32 invalidate, u32 flush), - TP_ARGS(ring, invalidate, flush), +TRACE_EVENT(intel_ringbuffer_begin, + TP_PROTO(struct intel_ringbuffer *ring, int need), + TP_ARGS(ring, need), TP_STRUCT__entry( __field(u32, dev) __field(u32, ring) - __field(u32, invalidate) - __field(u32, flush) + __field(u32, need) + __field(u32, space) ), TP_fast_assign( - __entry->dev = ring->dev->primary->index; - __entry->ring = ring->id; - __entry->invalidate = invalidate; - __entry->flush = flush; + __entry->dev = ring->engine->i915->dev->primary->index; + __entry->ring = ring->engine->id; + __entry->need = need; + __entry->space = intel_ring_space(ring); ), - TP_printk("dev=%u, ring=%x, invalidate=%04x, flush=%04x", - __entry->dev, __entry->ring, - __entry->invalidate, __entry->flush) + TP_printk("dev=%u, ring=%u, need=%u, space=%u", + __entry->dev, __entry->ring, __entry->need, __entry->space) ); -DECLARE_EVENT_CLASS(i915_gem_request, - TP_PROTO(struct intel_engine_cs *ring, u32 seqno), - TP_ARGS(ring, seqno), +TRACE_EVENT(intel_ringbuffer_wait, + TP_PROTO(struct intel_ringbuffer *ring, int need), + TP_ARGS(ring, need), TP_STRUCT__entry( __field(u32, dev) __field(u32, ring) - __field(u32, seqno) + __field(u32, need) + __field(u32, space) ), TP_fast_assign( - __entry->dev = ring->dev->primary->index; - __entry->ring = ring->id; - __entry->seqno = seqno; + __entry->dev = ring->engine->i915->dev->primary->index; + __entry->ring = ring->engine->id; + __entry->need = need; + __entry->space = intel_ring_space(ring); ), - TP_printk("dev=%u, ring=%u, seqno=%u", - __entry->dev, __entry->ring, __entry->seqno) + TP_printk("dev=%u, ring=%u, need=%u, space=%u", + __entry->dev, __entry->ring, __entry->need, __entry->space) ); -DEFINE_EVENT(i915_gem_request, i915_gem_request_add, - TP_PROTO(struct intel_engine_cs *ring, u32 seqno), - TP_ARGS(ring, seqno) +TRACE_EVENT(intel_ringbuffer_wrap, + TP_PROTO(struct intel_ringbuffer *ring, int rem), + TP_ARGS(ring, rem), + + TP_STRUCT__entry( + __field(u32, dev) + __field(u32, ring) + __field(u32, rem) + __field(u32, size) + ), + + TP_fast_assign( + __entry->dev = ring->engine->i915->dev->primary->index; + __entry->ring = ring->engine->id; + __entry->rem = rem; + __entry->size = ring->effective_size; + ), + + TP_printk("dev=%u, ring=%u, rem=%u, size=%u", + __entry->dev, __entry->ring, __entry->rem, __entry->size) ); -TRACE_EVENT(i915_gem_request_complete, +TRACE_EVENT(i915_gem_ring_complete, TP_PROTO(struct intel_engine_cs *ring), TP_ARGS(ring), @@ -433,23 +472,68 @@ TRACE_EVENT(i915_gem_request_complete, ), TP_fast_assign( - __entry->dev = ring->dev->primary->index; + __entry->dev = ring->i915->dev->primary->index; __entry->ring = ring->id; - __entry->seqno = ring->get_seqno(ring, false); + __entry->seqno = ring->get_seqno(ring); ), - TP_printk("dev=%u, ring=%u, seqno=%u", + TP_printk("dev=%u, ring=%u, seqno=%x", __entry->dev, __entry->ring, __entry->seqno) ); +DECLARE_EVENT_CLASS(i915_gem_request, + TP_PROTO(struct i915_gem_request *rq), + TP_ARGS(rq), + + TP_STRUCT__entry( + __field(u32, dev) + __field(u32, ring) + __field(u32, seqno) + ), + + TP_fast_assign( + __entry->dev = rq->i915->dev->primary->index; + __entry->ring = rq->engine->id; + __entry->seqno = rq->seqno; + ), + + TP_printk("dev=%u, ring=%u, seqno=%x", + __entry->dev, __entry->ring, __entry->seqno) +); + +DEFINE_EVENT(i915_gem_request, i915_gem_request_emit_flush, + TP_PROTO(struct i915_gem_request *rq), + TP_ARGS(rq) +); + +DEFINE_EVENT(i915_gem_request, i915_gem_request_emit_batch, + TP_PROTO(struct i915_gem_request *rq), + TP_ARGS(rq) +); + +DEFINE_EVENT(i915_gem_request, i915_gem_request_emit_breadcrumb, + TP_PROTO(struct i915_gem_request *rq), + TP_ARGS(rq) +); + +DEFINE_EVENT(i915_gem_request, i915_gem_request_commit, + TP_PROTO(struct i915_gem_request *rq), + TP_ARGS(rq) +); + +DEFINE_EVENT(i915_gem_request, i915_gem_request_complete, + TP_PROTO(struct i915_gem_request *rq), + TP_ARGS(rq) +); + DEFINE_EVENT(i915_gem_request, i915_gem_request_retire, - TP_PROTO(struct intel_engine_cs *ring, u32 seqno), - TP_ARGS(ring, seqno) + TP_PROTO(struct i915_gem_request *rq), + TP_ARGS(rq) ); TRACE_EVENT(i915_gem_request_wait_begin, - TP_PROTO(struct intel_engine_cs *ring, u32 seqno), - TP_ARGS(ring, seqno), + TP_PROTO(struct i915_gem_request *rq), + TP_ARGS(rq), TP_STRUCT__entry( __field(u32, dev) @@ -465,47 +549,38 @@ TRACE_EVENT(i915_gem_request_wait_begin, * less desirable. */ TP_fast_assign( - __entry->dev = ring->dev->primary->index; - __entry->ring = ring->id; - __entry->seqno = seqno; - __entry->blocking = mutex_is_locked(&ring->dev->struct_mutex); + __entry->dev = rq->i915->dev->primary->index; + __entry->ring = rq->engine->id; + __entry->seqno = rq->seqno; + __entry->blocking = mutex_is_locked(&rq->i915->dev->struct_mutex); ), - TP_printk("dev=%u, ring=%u, seqno=%u, blocking=%s", + TP_printk("dev=%u, ring=%u, seqno=%x, blocking?=%s", __entry->dev, __entry->ring, __entry->seqno, __entry->blocking ? "yes (NB)" : "no") ); -DEFINE_EVENT(i915_gem_request, i915_gem_request_wait_end, - TP_PROTO(struct intel_engine_cs *ring, u32 seqno), - TP_ARGS(ring, seqno) -); - -DECLARE_EVENT_CLASS(i915_ring, - TP_PROTO(struct intel_engine_cs *ring), - TP_ARGS(ring), +TRACE_EVENT(i915_gem_request_wait_end, + TP_PROTO(struct i915_gem_request *rq), + TP_ARGS(rq), TP_STRUCT__entry( __field(u32, dev) __field(u32, ring) + __field(u32, seqno) + __field(bool, completed) ), TP_fast_assign( - __entry->dev = ring->dev->primary->index; - __entry->ring = ring->id; + __entry->dev = rq->i915->dev->primary->index; + __entry->ring = rq->engine->id; + __entry->seqno = rq->seqno; + __entry->completed = rq->completed; ), - TP_printk("dev=%u, ring=%u", __entry->dev, __entry->ring) -); - -DEFINE_EVENT(i915_ring, i915_ring_wait_begin, - TP_PROTO(struct intel_engine_cs *ring), - TP_ARGS(ring) -); - -DEFINE_EVENT(i915_ring, i915_ring_wait_end, - TP_PROTO(struct intel_engine_cs *ring), - TP_ARGS(ring) + TP_printk("dev=%u, ring=%u, seqno=%x, completed=%s", + __entry->dev, __entry->ring, __entry->seqno, + __entry->completed ? "yes" : "no") ); TRACE_EVENT(i915_flip_request, diff --git a/drivers/gpu/drm/i915/intel_bios.c b/drivers/gpu/drm/i915/intel_bios.c index eee79e1..32d2233 100644 --- a/drivers/gpu/drm/i915/intel_bios.c +++ b/drivers/gpu/drm/i915/intel_bios.c @@ -976,12 +976,10 @@ static void parse_ddi_port(struct drm_i915_private *dev_priv, enum port port, if (bdb->version >= 158) { /* The VBT HDMI level shift values match the table we have. */ hdmi_level_shift = child->raw[7] & 0xF; - if (hdmi_level_shift < 0xC) { - DRM_DEBUG_KMS("VBT HDMI level shift for port %c: %d\n", - port_name(port), - hdmi_level_shift); - info->hdmi_level_shift = hdmi_level_shift; - } + DRM_DEBUG_KMS("VBT HDMI level shift for port %c: %d\n", + port_name(port), + hdmi_level_shift); + info->hdmi_level_shift = hdmi_level_shift; } } @@ -1114,8 +1112,7 @@ init_vbt_defaults(struct drm_i915_private *dev_priv) struct ddi_vbt_port_info *info = &dev_priv->vbt.ddi_port_info[port]; - /* Recommended BSpec default: 800mV 0dB. */ - info->hdmi_level_shift = 6; + info->hdmi_level_shift = HDMI_LEVEL_SHIFT_UNKNOWN; info->supports_dvi = (port != PORT_A && port != PORT_E); info->supports_hdmi = info->supports_dvi; diff --git a/drivers/gpu/drm/i915/intel_bios.h b/drivers/gpu/drm/i915/intel_bios.h index b986677..7603765 100644 --- a/drivers/gpu/drm/i915/intel_bios.h +++ b/drivers/gpu/drm/i915/intel_bios.h @@ -46,7 +46,7 @@ struct bdb_header { u16 version; /**< decimal */ u16 header_size; /**< in bytes */ u16 bdb_size; /**< in bytes */ -}; +} __packed; /* strictly speaking, this is a "skip" block, but it has interesting info */ struct vbios_data { @@ -252,7 +252,7 @@ union child_device_config { /* This one should also be safe to use anywhere, even without version * checks. */ struct common_child_dev_config common; -}; +} __packed; struct bdb_general_definitions { /* DDC GPIO */ @@ -802,7 +802,8 @@ struct mipi_config { u16 rsvd4; - u8 rsvd5[5]; + u8 rsvd5; + u32 target_burst_mode_freq; u32 dsi_ddr_clk; u32 bridge_ref_clk; @@ -887,12 +888,12 @@ struct mipi_pps_data { u16 bl_disable_delay; u16 panel_off_delay; u16 panel_power_cycle_delay; -}; +} __packed; struct bdb_mipi_config { struct mipi_config config[MAX_MIPI_CONFIGURATIONS]; struct mipi_pps_data pps[MAX_MIPI_CONFIGURATIONS]; -}; +} __packed; /* Block 53 contains MIPI sequences as needed by the panel * for enabling it. This block can be variable in size and @@ -901,7 +902,7 @@ struct bdb_mipi_config { struct bdb_mipi_sequence { u8 version; u8 data[0]; -}; +} __packed; /* MIPI Sequnece Block definitions */ enum mipi_seq { diff --git a/drivers/gpu/drm/i915/intel_ddi.c b/drivers/gpu/drm/i915/intel_ddi.c index 5db0b55..b63d4fa 100644 --- a/drivers/gpu/drm/i915/intel_ddi.c +++ b/drivers/gpu/drm/i915/intel_ddi.c @@ -28,87 +28,103 @@ #include "i915_drv.h" #include "intel_drv.h" +struct ddi_buf_trans { + u32 trans1; /* balance leg enable, de-emph level */ + u32 trans2; /* vref sel, vswing */ +}; + /* HDMI/DVI modes ignore everything but the last 2 items. So we share * them for both DP and FDI transports, allowing those ports to * automatically adapt to HDMI connections as well */ -static const u32 hsw_ddi_translations_dp[] = { - 0x00FFFFFF, 0x0006000E, /* DP parameters */ - 0x00D75FFF, 0x0005000A, - 0x00C30FFF, 0x00040006, - 0x80AAAFFF, 0x000B0000, - 0x00FFFFFF, 0x0005000A, - 0x00D75FFF, 0x000C0004, - 0x80C30FFF, 0x000B0000, - 0x00FFFFFF, 0x00040006, - 0x80D75FFF, 0x000B0000, +static const struct ddi_buf_trans hsw_ddi_translations_dp[] = { + { 0x00FFFFFF, 0x0006000E }, + { 0x00D75FFF, 0x0005000A }, + { 0x00C30FFF, 0x00040006 }, + { 0x80AAAFFF, 0x000B0000 }, + { 0x00FFFFFF, 0x0005000A }, + { 0x00D75FFF, 0x000C0004 }, + { 0x80C30FFF, 0x000B0000 }, + { 0x00FFFFFF, 0x00040006 }, + { 0x80D75FFF, 0x000B0000 }, }; -static const u32 hsw_ddi_translations_fdi[] = { - 0x00FFFFFF, 0x0007000E, /* FDI parameters */ - 0x00D75FFF, 0x000F000A, - 0x00C30FFF, 0x00060006, - 0x00AAAFFF, 0x001E0000, - 0x00FFFFFF, 0x000F000A, - 0x00D75FFF, 0x00160004, - 0x00C30FFF, 0x001E0000, - 0x00FFFFFF, 0x00060006, - 0x00D75FFF, 0x001E0000, +static const struct ddi_buf_trans hsw_ddi_translations_fdi[] = { + { 0x00FFFFFF, 0x0007000E }, + { 0x00D75FFF, 0x000F000A }, + { 0x00C30FFF, 0x00060006 }, + { 0x00AAAFFF, 0x001E0000 }, + { 0x00FFFFFF, 0x000F000A }, + { 0x00D75FFF, 0x00160004 }, + { 0x00C30FFF, 0x001E0000 }, + { 0x00FFFFFF, 0x00060006 }, + { 0x00D75FFF, 0x001E0000 }, }; -static const u32 hsw_ddi_translations_hdmi[] = { - /* Idx NT mV diff T mV diff db */ - 0x00FFFFFF, 0x0006000E, /* 0: 400 400 0 */ - 0x00E79FFF, 0x000E000C, /* 1: 400 500 2 */ - 0x00D75FFF, 0x0005000A, /* 2: 400 600 3.5 */ - 0x00FFFFFF, 0x0005000A, /* 3: 600 600 0 */ - 0x00E79FFF, 0x001D0007, /* 4: 600 750 2 */ - 0x00D75FFF, 0x000C0004, /* 5: 600 900 3.5 */ - 0x00FFFFFF, 0x00040006, /* 6: 800 800 0 */ - 0x80E79FFF, 0x00030002, /* 7: 800 1000 2 */ - 0x00FFFFFF, 0x00140005, /* 8: 850 850 0 */ - 0x00FFFFFF, 0x000C0004, /* 9: 900 900 0 */ - 0x00FFFFFF, 0x001C0003, /* 10: 950 950 0 */ - 0x80FFFFFF, 0x00030002, /* 11: 1000 1000 0 */ +static const struct ddi_buf_trans hsw_ddi_translations_hdmi[] = { + /* Idx NT mV d T mV d db */ + { 0x00FFFFFF, 0x0006000E }, /* 0: 400 400 0 */ + { 0x00E79FFF, 0x000E000C }, /* 1: 400 500 2 */ + { 0x00D75FFF, 0x0005000A }, /* 2: 400 600 3.5 */ + { 0x00FFFFFF, 0x0005000A }, /* 3: 600 600 0 */ + { 0x00E79FFF, 0x001D0007 }, /* 4: 600 750 2 */ + { 0x00D75FFF, 0x000C0004 }, /* 5: 600 900 3.5 */ + { 0x00FFFFFF, 0x00040006 }, /* 6: 800 800 0 */ + { 0x80E79FFF, 0x00030002 }, /* 7: 800 1000 2 */ + { 0x00FFFFFF, 0x00140005 }, /* 8: 850 850 0 */ + { 0x00FFFFFF, 0x000C0004 }, /* 9: 900 900 0 */ + { 0x00FFFFFF, 0x001C0003 }, /* 10: 950 950 0 */ + { 0x80FFFFFF, 0x00030002 }, /* 11: 1000 1000 0 */ }; -static const u32 bdw_ddi_translations_edp[] = { - 0x00FFFFFF, 0x00000012, /* eDP parameters */ - 0x00EBAFFF, 0x00020011, - 0x00C71FFF, 0x0006000F, - 0x00AAAFFF, 0x000E000A, - 0x00FFFFFF, 0x00020011, - 0x00DB6FFF, 0x0005000F, - 0x00BEEFFF, 0x000A000C, - 0x00FFFFFF, 0x0005000F, - 0x00DB6FFF, 0x000A000C, - 0x00FFFFFF, 0x00140006 /* HDMI parameters 800mV 0dB*/ +static const struct ddi_buf_trans bdw_ddi_translations_edp[] = { + { 0x00FFFFFF, 0x00000012 }, + { 0x00EBAFFF, 0x00020011 }, + { 0x00C71FFF, 0x0006000F }, + { 0x00AAAFFF, 0x000E000A }, + { 0x00FFFFFF, 0x00020011 }, + { 0x00DB6FFF, 0x0005000F }, + { 0x00BEEFFF, 0x000A000C }, + { 0x00FFFFFF, 0x0005000F }, + { 0x00DB6FFF, 0x000A000C }, }; -static const u32 bdw_ddi_translations_dp[] = { - 0x00FFFFFF, 0x0007000E, /* DP parameters */ - 0x00D75FFF, 0x000E000A, - 0x00BEFFFF, 0x00140006, - 0x80B2CFFF, 0x001B0002, - 0x00FFFFFF, 0x000E000A, - 0x00D75FFF, 0x00180004, - 0x80CB2FFF, 0x001B0002, - 0x00F7DFFF, 0x00180004, - 0x80D75FFF, 0x001B0002, - 0x00FFFFFF, 0x00140006 /* HDMI parameters 800mV 0dB*/ +static const struct ddi_buf_trans bdw_ddi_translations_dp[] = { + { 0x00FFFFFF, 0x0007000E }, + { 0x00D75FFF, 0x000E000A }, + { 0x00BEFFFF, 0x00140006 }, + { 0x80B2CFFF, 0x001B0002 }, + { 0x00FFFFFF, 0x000E000A }, + { 0x00D75FFF, 0x00180004 }, + { 0x80CB2FFF, 0x001B0002 }, + { 0x00F7DFFF, 0x00180004 }, + { 0x80D75FFF, 0x001B0002 }, }; -static const u32 bdw_ddi_translations_fdi[] = { - 0x00FFFFFF, 0x0001000E, /* FDI parameters */ - 0x00D75FFF, 0x0004000A, - 0x00C30FFF, 0x00070006, - 0x00AAAFFF, 0x000C0000, - 0x00FFFFFF, 0x0004000A, - 0x00D75FFF, 0x00090004, - 0x00C30FFF, 0x000C0000, - 0x00FFFFFF, 0x00070006, - 0x00D75FFF, 0x000C0000, - 0x00FFFFFF, 0x00140006 /* HDMI parameters 800mV 0dB*/ +static const struct ddi_buf_trans bdw_ddi_translations_fdi[] = { + { 0x00FFFFFF, 0x0001000E }, + { 0x00D75FFF, 0x0004000A }, + { 0x00C30FFF, 0x00070006 }, + { 0x00AAAFFF, 0x000C0000 }, + { 0x00FFFFFF, 0x0004000A }, + { 0x00D75FFF, 0x00090004 }, + { 0x00C30FFF, 0x000C0000 }, + { 0x00FFFFFF, 0x00070006 }, + { 0x00D75FFF, 0x000C0000 }, +}; + +static const struct ddi_buf_trans bdw_ddi_translations_hdmi[] = { + /* Idx NT mV d T mV df db */ + { 0x00FFFFFF, 0x0007000E }, /* 0: 400 400 0 */ + { 0x00D75FFF, 0x000E000A }, /* 1: 400 600 3.5 */ + { 0x00BEFFFF, 0x00140006 }, /* 2: 400 800 6 */ + { 0x00FFFFFF, 0x0009000D }, /* 3: 450 450 0 */ + { 0x00FFFFFF, 0x000E000A }, /* 4: 600 600 0 */ + { 0x00D7FFFF, 0x00140006 }, /* 5: 600 800 2.5 */ + { 0x80CB2FFF, 0x001B0002 }, /* 6: 600 1000 4.5 */ + { 0x00FFFFFF, 0x00140006 }, /* 7: 800 800 0 */ + { 0x80E79FFF, 0x001B0002 }, /* 8: 800 1000 2 */ + { 0x80FFFFFF, 0x001B0002 }, /* 9: 1000 1000 0 */ }; enum port intel_ddi_get_encoder_port(struct intel_encoder *intel_encoder) @@ -145,26 +161,36 @@ static void intel_prepare_ddi_buffers(struct drm_device *dev, enum port port) { struct drm_i915_private *dev_priv = dev->dev_private; u32 reg; - int i; + int i, n_hdmi_entries, hdmi_800mV_0dB; int hdmi_level = dev_priv->vbt.ddi_port_info[port].hdmi_level_shift; - const u32 *ddi_translations_fdi; - const u32 *ddi_translations_dp; - const u32 *ddi_translations_edp; - const u32 *ddi_translations; + const struct ddi_buf_trans *ddi_translations_fdi; + const struct ddi_buf_trans *ddi_translations_dp; + const struct ddi_buf_trans *ddi_translations_edp; + const struct ddi_buf_trans *ddi_translations_hdmi; + const struct ddi_buf_trans *ddi_translations; if (IS_BROADWELL(dev)) { ddi_translations_fdi = bdw_ddi_translations_fdi; ddi_translations_dp = bdw_ddi_translations_dp; ddi_translations_edp = bdw_ddi_translations_edp; + ddi_translations_hdmi = bdw_ddi_translations_hdmi; + n_hdmi_entries = ARRAY_SIZE(bdw_ddi_translations_hdmi); + hdmi_800mV_0dB = 7; } else if (IS_HASWELL(dev)) { ddi_translations_fdi = hsw_ddi_translations_fdi; ddi_translations_dp = hsw_ddi_translations_dp; ddi_translations_edp = hsw_ddi_translations_dp; + ddi_translations_hdmi = hsw_ddi_translations_hdmi; + n_hdmi_entries = ARRAY_SIZE(hsw_ddi_translations_hdmi); + hdmi_800mV_0dB = 6; } else { WARN(1, "ddi translation table missing\n"); ddi_translations_edp = bdw_ddi_translations_dp; ddi_translations_fdi = bdw_ddi_translations_fdi; ddi_translations_dp = bdw_ddi_translations_dp; + ddi_translations_hdmi = bdw_ddi_translations_hdmi; + n_hdmi_entries = ARRAY_SIZE(bdw_ddi_translations_hdmi); + hdmi_800mV_0dB = 7; } switch (port) { @@ -190,14 +216,22 @@ static void intel_prepare_ddi_buffers(struct drm_device *dev, enum port port) for (i = 0, reg = DDI_BUF_TRANS(port); i < ARRAY_SIZE(hsw_ddi_translations_fdi); i++) { - I915_WRITE(reg, ddi_translations[i]); + I915_WRITE(reg, ddi_translations[i].trans1); reg += 4; - } - /* Entry 9 is for HDMI: */ - for (i = 0; i < 2; i++) { - I915_WRITE(reg, hsw_ddi_translations_hdmi[hdmi_level * 2 + i]); + I915_WRITE(reg, ddi_translations[i].trans2); reg += 4; } + + /* Choose a good default if VBT is badly populated */ + if (hdmi_level == HDMI_LEVEL_SHIFT_UNKNOWN || + hdmi_level >= n_hdmi_entries) + hdmi_level = hdmi_800mV_0dB; + + /* Entry 9 is for HDMI: */ + I915_WRITE(reg, ddi_translations_hdmi[hdmi_level].trans1); + reg += 4; + I915_WRITE(reg, ddi_translations_hdmi[hdmi_level].trans2); + reg += 4; } /* Program DDI buffers translations for DP. By default, program ports A-D in DP @@ -214,18 +248,6 @@ void intel_prepare_ddi(struct drm_device *dev) intel_prepare_ddi_buffers(dev, port); } -static const long hsw_ddi_buf_ctl_values[] = { - DDI_BUF_EMP_400MV_0DB_HSW, - DDI_BUF_EMP_400MV_3_5DB_HSW, - DDI_BUF_EMP_400MV_6DB_HSW, - DDI_BUF_EMP_400MV_9_5DB_HSW, - DDI_BUF_EMP_600MV_0DB_HSW, - DDI_BUF_EMP_600MV_3_5DB_HSW, - DDI_BUF_EMP_600MV_6DB_HSW, - DDI_BUF_EMP_800MV_0DB_HSW, - DDI_BUF_EMP_800MV_3_5DB_HSW -}; - static void intel_wait_ddi_buf_idle(struct drm_i915_private *dev_priv, enum port port) { @@ -285,7 +307,7 @@ void hsw_fdi_link_train(struct drm_crtc *crtc) /* Start the training iterating through available voltages and emphasis, * testing each value twice. */ - for (i = 0; i < ARRAY_SIZE(hsw_ddi_buf_ctl_values) * 2; i++) { + for (i = 0; i < ARRAY_SIZE(hsw_ddi_translations_fdi) * 2; i++) { /* Configure DP_TP_CTL with auto-training */ I915_WRITE(DP_TP_CTL(PORT_E), DP_TP_CTL_FDI_AUTOTRAIN | @@ -300,7 +322,7 @@ void hsw_fdi_link_train(struct drm_crtc *crtc) I915_WRITE(DDI_BUF_CTL(PORT_E), DDI_BUF_CTL_ENABLE | ((intel_crtc->config.fdi_lanes - 1) << 1) | - hsw_ddi_buf_ctl_values[i / 2]); + DDI_BUF_TRANS_SELECT(i / 2)); POSTING_READ(DDI_BUF_CTL(PORT_E)); udelay(600); @@ -375,7 +397,7 @@ void intel_ddi_init_dp_buf_reg(struct intel_encoder *encoder) enc_to_dig_port(&encoder->base); intel_dp->DP = intel_dig_port->saved_port_bits | - DDI_BUF_CTL_ENABLE | DDI_BUF_EMP_400MV_0DB_HSW; + DDI_BUF_CTL_ENABLE | DDI_BUF_TRANS_SELECT(0); intel_dp->DP |= DDI_PORT_WIDTH(intel_dp->lane_count); } @@ -402,7 +424,7 @@ intel_ddi_get_crtc_encoder(struct drm_crtc *crtc) } #define LC_FREQ 2700 -#define LC_FREQ_2K (LC_FREQ * 2000) +#define LC_FREQ_2K U64_C(LC_FREQ * 2000) #define P_MIN 2 #define P_MAX 64 @@ -414,7 +436,11 @@ intel_ddi_get_crtc_encoder(struct drm_crtc *crtc) #define VCO_MIN 2400 #define VCO_MAX 4800 -#define ABS_DIFF(a, b) ((a > b) ? (a - b) : (b - a)) +#define abs_diff(a, b) ({ \ + typeof(a) __a = (a); \ + typeof(b) __b = (b); \ + (void) (&__a == &__b); \ + __a > __b ? (__a - __b) : (__b - __a); }) struct wrpll_rnp { unsigned p, n2, r2; @@ -524,9 +550,9 @@ static void wrpll_update_rnp(uint64_t freq2k, unsigned budget, */ a = freq2k * budget * p * r2; b = freq2k * budget * best->p * best->r2; - diff = ABS_DIFF((freq2k * p * r2), (LC_FREQ_2K * n2)); - diff_best = ABS_DIFF((freq2k * best->p * best->r2), - (LC_FREQ_2K * best->n2)); + diff = abs_diff(freq2k * p * r2, LC_FREQ_2K * n2); + diff_best = abs_diff(freq2k * best->p * best->r2, + LC_FREQ_2K * best->n2); c = 1000000 * diff; d = 1000000 * diff_best; @@ -587,8 +613,8 @@ static int intel_ddi_calc_wrpll_link(struct drm_i915_private *dev_priv, return (refclk * n * 100) / (p * r); } -void intel_ddi_clock_get(struct intel_encoder *encoder, - struct intel_crtc_config *pipe_config) +static void hsw_ddi_clock_get(struct intel_encoder *encoder, + struct intel_crtc_config *pipe_config) { struct drm_i915_private *dev_priv = encoder->base.dev->dev_private; int link_clock = 0; @@ -643,9 +669,15 @@ void intel_ddi_clock_get(struct intel_encoder *encoder, pipe_config->adjusted_mode.crtc_clock = pipe_config->port_clock; } +void intel_ddi_clock_get(struct intel_encoder *encoder, + struct intel_crtc_config *pipe_config) +{ + hsw_ddi_clock_get(encoder, pipe_config); +} + static void -intel_ddi_calculate_wrpll(int clock /* in Hz */, - unsigned *r2_out, unsigned *n2_out, unsigned *p_out) +hsw_ddi_calculate_wrpll(int clock /* in Hz */, + unsigned *r2_out, unsigned *n2_out, unsigned *p_out) { uint64_t freq2k; unsigned p, n2, r2; @@ -708,27 +740,17 @@ intel_ddi_calculate_wrpll(int clock /* in Hz */, *r2_out = best.r2; } -/* - * Tries to find a PLL for the CRTC. If it finds, it increases the refcount and - * stores it in intel_crtc->ddi_pll_sel, so other mode sets won't be able to - * steal the selected PLL. You need to call intel_ddi_pll_enable to actually - * enable the PLL. - */ -bool intel_ddi_pll_select(struct intel_crtc *intel_crtc) +static bool +hsw_ddi_pll_select(struct intel_crtc *intel_crtc, + struct intel_encoder *intel_encoder, + int clock) { - struct drm_crtc *crtc = &intel_crtc->base; - struct intel_encoder *intel_encoder = intel_ddi_get_crtc_encoder(crtc); - int type = intel_encoder->type; - int clock = intel_crtc->config.port_clock; - - intel_put_shared_dpll(intel_crtc); - - if (type == INTEL_OUTPUT_HDMI) { + if (intel_encoder->type == INTEL_OUTPUT_HDMI) { struct intel_shared_dpll *pll; uint32_t val; unsigned p, n2, r2; - intel_ddi_calculate_wrpll(clock * 1000, &r2, &n2, &p); + hsw_ddi_calculate_wrpll(clock * 1000, &r2, &n2, &p); val = WRPLL_PLL_ENABLE | WRPLL_PLL_LCPLL | WRPLL_DIVIDER_REFERENCE(r2) | WRPLL_DIVIDER_FEEDBACK(n2) | @@ -749,6 +771,25 @@ bool intel_ddi_pll_select(struct intel_crtc *intel_crtc) return true; } + +/* + * Tries to find a *shared* PLL for the CRTC and store it in + * intel_crtc->ddi_pll_sel. + * + * For private DPLLs, compute_config() should do the selection for us. This + * function should be folded into compute_config() eventually. + */ +bool intel_ddi_pll_select(struct intel_crtc *intel_crtc) +{ + struct drm_crtc *crtc = &intel_crtc->base; + struct intel_encoder *intel_encoder = intel_ddi_get_crtc_encoder(crtc); + int clock = intel_crtc->config.port_clock; + + intel_put_shared_dpll(intel_crtc); + + return hsw_ddi_pll_select(intel_crtc, intel_encoder, clock); +} + void intel_ddi_set_pipe_settings(struct drm_crtc *crtc) { struct drm_i915_private *dev_priv = crtc->dev->dev_private; @@ -1183,31 +1224,52 @@ static void intel_disable_ddi(struct intel_encoder *intel_encoder) } } -int intel_ddi_get_cdclk_freq(struct drm_i915_private *dev_priv) +static int bdw_get_cdclk_freq(struct drm_i915_private *dev_priv) +{ + uint32_t lcpll = I915_READ(LCPLL_CTL); + uint32_t freq = lcpll & LCPLL_CLK_FREQ_MASK; + + if (lcpll & LCPLL_CD_SOURCE_FCLK) + return 800000; + else if (I915_READ(FUSE_STRAP) & HSW_CDCLK_LIMIT) + return 450000; + else if (freq == LCPLL_CLK_FREQ_450) + return 450000; + else if (freq == LCPLL_CLK_FREQ_54O_BDW) + return 540000; + else if (freq == LCPLL_CLK_FREQ_337_5_BDW) + return 337500; + else + return 675000; +} + +static int hsw_get_cdclk_freq(struct drm_i915_private *dev_priv) { struct drm_device *dev = dev_priv->dev; uint32_t lcpll = I915_READ(LCPLL_CTL); uint32_t freq = lcpll & LCPLL_CLK_FREQ_MASK; - if (lcpll & LCPLL_CD_SOURCE_FCLK) { + if (lcpll & LCPLL_CD_SOURCE_FCLK) return 800000; - } else if (I915_READ(FUSE_STRAP) & HSW_CDCLK_LIMIT) { + else if (I915_READ(FUSE_STRAP) & HSW_CDCLK_LIMIT) return 450000; - } else if (freq == LCPLL_CLK_FREQ_450) { + else if (freq == LCPLL_CLK_FREQ_450) return 450000; - } else if (IS_HASWELL(dev)) { - if (IS_ULT(dev)) - return 337500; - else - return 540000; - } else { - if (freq == LCPLL_CLK_FREQ_54O_BDW) - return 540000; - else if (freq == LCPLL_CLK_FREQ_337_5_BDW) - return 337500; - else - return 675000; - } + else if (IS_ULT(dev)) + return 337500; + else + return 540000; +} + +int intel_ddi_get_cdclk_freq(struct drm_i915_private *dev_priv) +{ + struct drm_device *dev = dev_priv->dev; + + if (IS_BROADWELL(dev)) + return bdw_get_cdclk_freq(dev_priv); + + /* Haswell */ + return hsw_get_cdclk_freq(dev_priv); } static void hsw_ddi_pll_enable(struct drm_i915_private *dev_priv, @@ -1248,10 +1310,8 @@ static const char * const hsw_ddi_pll_names[] = { "WRPLL 2", }; -void intel_ddi_pll_init(struct drm_device *dev) +static void hsw_shared_dplls_init(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = dev->dev_private; - uint32_t val = I915_READ(LCPLL_CTL); int i; dev_priv->num_shared_dpll = 2; @@ -1264,6 +1324,14 @@ void intel_ddi_pll_init(struct drm_device *dev) dev_priv->shared_dplls[i].get_hw_state = hsw_ddi_pll_get_hw_state; } +} + +void intel_ddi_pll_init(struct drm_device *dev) +{ + struct drm_i915_private *dev_priv = dev->dev_private; + uint32_t val = I915_READ(LCPLL_CTL); + + hsw_shared_dplls_init(dev_priv); /* The LCPLL register should be turned on by the BIOS. For now let's * just check its state and print errors in case something is wrong. @@ -1444,7 +1512,7 @@ void intel_ddi_get_config(struct intel_encoder *encoder, dev_priv->vbt.edp_bpp = pipe_config->pipe_bpp; } - intel_ddi_clock_get(encoder, pipe_config); + hsw_ddi_clock_get(encoder, pipe_config); } static void intel_ddi_destroy(struct drm_encoder *encoder) diff --git a/drivers/gpu/drm/i915/intel_display.c b/drivers/gpu/drm/i915/intel_display.c index ee0ca36..add26d0 100644 --- a/drivers/gpu/drm/i915/intel_display.c +++ b/drivers/gpu/drm/i915/intel_display.c @@ -91,15 +91,16 @@ static int intel_framebuffer_init(struct drm_device *dev, struct intel_framebuffer *ifb, struct drm_mode_fb_cmd2 *mode_cmd, struct drm_i915_gem_object *obj); -static void intel_dp_set_m_n(struct intel_crtc *crtc); static void i9xx_set_pipeconf(struct intel_crtc *intel_crtc); static void intel_set_pipe_timings(struct intel_crtc *intel_crtc); static void intel_cpu_transcoder_set_m_n(struct intel_crtc *crtc, - struct intel_link_m_n *m_n); + struct intel_link_m_n *m_n, + struct intel_link_m_n *m2_n2); static void ironlake_set_pipeconf(struct drm_crtc *crtc); static void haswell_set_pipeconf(struct drm_crtc *crtc); static void intel_set_pipe_csc(struct drm_crtc *crtc); static void vlv_prepare_pll(struct intel_crtc *crtc); +static void chv_prepare_pll(struct intel_crtc *crtc); static struct intel_encoder *intel_find_encoder(struct intel_connector *connector, int pipe) { @@ -964,8 +965,7 @@ static bool pipe_dsl_stopped(struct drm_device *dev, enum pipe pipe) /* * intel_wait_for_pipe_off - wait for pipe to turn off - * @dev: drm device - * @pipe: pipe to wait for + * @crtc: crtc whose pipe to wait for * * After disabling a pipe, we can't wait for vblank in the usual way, * spinning on the vblank interrupt status bit, since we won't actually @@ -979,11 +979,12 @@ static bool pipe_dsl_stopped(struct drm_device *dev, enum pipe pipe) * ends up stopping at the start of the next frame). * */ -void intel_wait_for_pipe_off(struct drm_device *dev, int pipe) +static void intel_wait_for_pipe_off(struct intel_crtc *crtc) { + struct drm_device *dev = crtc->base.dev; struct drm_i915_private *dev_priv = dev->dev_private; - enum transcoder cpu_transcoder = intel_pipe_to_cpu_transcoder(dev_priv, - pipe); + enum transcoder cpu_transcoder = crtc->config.cpu_transcoder; + enum pipe pipe = crtc->pipe; if (INTEL_INFO(dev)->gen >= 4) { int reg = PIPECONF(cpu_transcoder); @@ -1192,27 +1193,40 @@ void assert_fdi_rx_pll(struct drm_i915_private *dev_priv, static void assert_panel_unlocked(struct drm_i915_private *dev_priv, enum pipe pipe) { - int pp_reg, lvds_reg; + struct drm_device *dev = dev_priv->dev; + int pp_reg; u32 val; enum pipe panel_pipe = PIPE_A; bool locked = true; - if (HAS_PCH_SPLIT(dev_priv->dev)) { + if (WARN_ON(HAS_DDI(dev))) + return; + + if (HAS_PCH_SPLIT(dev)) { + u32 port_sel; + pp_reg = PCH_PP_CONTROL; - lvds_reg = PCH_LVDS; + port_sel = I915_READ(PCH_PP_ON_DELAYS) & PANEL_PORT_SELECT_MASK; + + if (port_sel == PANEL_PORT_SELECT_LVDS && + I915_READ(PCH_LVDS) & LVDS_PIPEB_SELECT) + panel_pipe = PIPE_B; + /* XXX: else fix for eDP */ + } else if (IS_VALLEYVIEW(dev)) { + /* presumably write lock depends on pipe, not port select */ + pp_reg = VLV_PIPE_PP_CONTROL(pipe); + panel_pipe = pipe; } else { pp_reg = PP_CONTROL; - lvds_reg = LVDS; + if (I915_READ(LVDS) & LVDS_PIPEB_SELECT) + panel_pipe = PIPE_B; } val = I915_READ(pp_reg); if (!(val & PANEL_POWER_ON) || - ((val & PANEL_UNLOCK_REGS) == PANEL_UNLOCK_REGS)) + ((val & PANEL_UNLOCK_MASK) == PANEL_UNLOCK_REGS)) locked = false; - if (I915_READ(lvds_reg) & LVDS_PIPEB_SELECT) - panel_pipe = PIPE_B; - WARN(panel_pipe == pipe && locked, "panel assertion failure, pipe %c regs locked\n", pipe_name(pipe)); @@ -1245,8 +1259,9 @@ void assert_pipe(struct drm_i915_private *dev_priv, enum transcoder cpu_transcoder = intel_pipe_to_cpu_transcoder(dev_priv, pipe); - /* if we need the pipe A quirk it must be always on */ - if (pipe == PIPE_A && dev_priv->quirks & QUIRK_PIPEA_FORCE) + /* if we need the pipe quirk it must be always on */ + if ((pipe == PIPE_A && dev_priv->quirks & QUIRK_PIPEA_FORCE) || + (pipe == PIPE_B && dev_priv->quirks & QUIRK_PIPEB_FORCE)) state = true; if (!intel_display_power_enabled(dev_priv, @@ -1300,7 +1315,7 @@ static void assert_planes_disabled(struct drm_i915_private *dev_priv, } /* Need to check both planes against the pipe */ - for_each_pipe(i) { + for_each_pipe(dev_priv, i) { reg = DSPCNTR(i); val = I915_READ(reg); cur_pipe = (val & DISPPLANE_SEL_PIPE_MASK) >> @@ -1341,6 +1356,12 @@ static void assert_sprites_disabled(struct drm_i915_private *dev_priv, } } +static void assert_vblank_disabled(struct drm_crtc *crtc) +{ + if (drm_crtc_vblank_get(crtc) == 0) + drm_crtc_vblank_put(crtc); +} + static void ibx_assert_pch_refclk_enabled(struct drm_i915_private *dev_priv) { u32 val; @@ -1513,34 +1534,6 @@ static void intel_init_dpio(struct drm_device *dev) } } -static void intel_reset_dpio(struct drm_device *dev) -{ - struct drm_i915_private *dev_priv = dev->dev_private; - - if (IS_CHERRYVIEW(dev)) { - enum dpio_phy phy; - u32 val; - - for (phy = DPIO_PHY0; phy < I915_NUM_PHYS_VLV; phy++) { - /* Poll for phypwrgood signal */ - if (wait_for(I915_READ(DISPLAY_PHY_STATUS) & - PHY_POWERGOOD(phy), 1)) - DRM_ERROR("Display PHY %d is not power up\n", phy); - - /* - * Deassert common lane reset for PHY. - * - * This should only be done on init and resume from S3 - * with both PLLs disabled, or we risk losing DPIO and - * PLL synchronization. - */ - val = I915_READ(DISPLAY_PHY_CONTROL); - I915_WRITE(DISPLAY_PHY_CONTROL, - PHY_COM_LANE_RESET_DEASSERT(phy, val)); - } - } -} - static void vlv_enable_pll(struct intel_crtc *crtc) { struct drm_device *dev = crtc->base.dev; @@ -1554,7 +1547,7 @@ static void vlv_enable_pll(struct intel_crtc *crtc) BUG_ON(!IS_VALLEYVIEW(dev_priv->dev)); /* PLL is protected by panel, make sure we can write it */ - if (IS_MOBILE(dev_priv->dev) && !IS_I830(dev_priv->dev)) + if (IS_MOBILE(dev_priv->dev)) assert_panel_unlocked(dev_priv, crtc->pipe); I915_WRITE(reg, dpll); @@ -1617,6 +1610,18 @@ static void chv_enable_pll(struct intel_crtc *crtc) mutex_unlock(&dev_priv->dpio_lock); } +static int intel_num_dvo_pipes(struct drm_device *dev) +{ + struct intel_crtc *crtc; + int count = 0; + + for_each_intel_crtc(dev, crtc) + count += crtc->active && + intel_pipe_has_type(&crtc->base, INTEL_OUTPUT_DVO); + + return count; +} + static void i9xx_enable_pll(struct intel_crtc *crtc) { struct drm_device *dev = crtc->base.dev; @@ -1633,7 +1638,18 @@ static void i9xx_enable_pll(struct intel_crtc *crtc) if (IS_MOBILE(dev) && !IS_I830(dev)) assert_panel_unlocked(dev_priv, crtc->pipe); - I915_WRITE(reg, dpll); + /* Enable DVO 2x clock on both PLLs if necessary */ + if (IS_I830(dev) && intel_num_dvo_pipes(dev) > 0) { + /* + * It appears to be important that we don't enable this + * for the current pipe before otherwise configuring the + * PLL. No idea how this should be handled if multiple + * DVO outputs are enabled simultaneosly. + */ + dpll |= DPLL_DVO_2X_MODE; + I915_WRITE(DPLL(!crtc->pipe), + I915_READ(DPLL(!crtc->pipe)) | DPLL_DVO_2X_MODE); + } /* Wait for the clocks to stabilize. */ POSTING_READ(reg); @@ -1672,10 +1688,25 @@ static void i9xx_enable_pll(struct intel_crtc *crtc) * * Note! This is for pre-ILK only. */ -static void i9xx_disable_pll(struct drm_i915_private *dev_priv, enum pipe pipe) +static void i9xx_disable_pll(struct intel_crtc *crtc) { - /* Don't disable pipe A or pipe A PLLs if needed */ - if (pipe == PIPE_A && (dev_priv->quirks & QUIRK_PIPEA_FORCE)) + struct drm_device *dev = crtc->base.dev; + struct drm_i915_private *dev_priv = dev->dev_private; + enum pipe pipe = crtc->pipe; + + /* Disable DVO 2x clock on both PLLs if necessary */ + if (IS_I830(dev) && + intel_pipe_has_type(&crtc->base, INTEL_OUTPUT_DVO) && + intel_num_dvo_pipes(dev) == 1) { + I915_WRITE(DPLL(PIPE_B), + I915_READ(DPLL(PIPE_B)) & ~DPLL_DVO_2X_MODE); + I915_WRITE(DPLL(PIPE_A), + I915_READ(DPLL(PIPE_A)) & ~DPLL_DVO_2X_MODE); + } + + /* Don't disable pipe or pipe PLLs if needed */ + if ((pipe == PIPE_A && dev_priv->quirks & QUIRK_PIPEA_FORCE) || + (pipe == PIPE_B && dev_priv->quirks & QUIRK_PIPEB_FORCE)) return; /* Make sure the pipe isn't still relying on us */ @@ -1712,7 +1743,7 @@ static void chv_disable_pll(struct drm_i915_private *dev_priv, enum pipe pipe) assert_pipe_disabled(dev_priv, pipe); /* Set PLL en = 0 */ - val = DPLL_SSC_REF_CLOCK_CHV; + val = DPLL_SSC_REF_CLOCK_CHV | DPLL_REFA_CLK_ENABLE_VLV; if (pipe != PIPE_A) val |= DPLL_INTEGRATED_CRI_CLK_VLV; I915_WRITE(DPLL(pipe), val); @@ -1806,7 +1837,7 @@ static void intel_enable_shared_dpll(struct intel_crtc *crtc) if (WARN_ON(pll->refcount == 0)) return; - DRM_DEBUG_KMS("enable %s (active %d, on? %d)for crtc %d\n", + DRM_DEBUG_KMS("enable %s (active %d, on? %d) for crtc %d\n", pll->name, pll->active, pll->on, crtc->base.base.id); @@ -1824,7 +1855,7 @@ static void intel_enable_shared_dpll(struct intel_crtc *crtc) pll->on = true; } -void intel_disable_shared_dpll(struct intel_crtc *crtc) +static void intel_disable_shared_dpll(struct intel_crtc *crtc) { struct drm_device *dev = crtc->base.dev; struct drm_i915_private *dev_priv = dev->dev_private; @@ -1868,7 +1899,7 @@ static void ironlake_enable_pch_transcoder(struct drm_i915_private *dev_priv, uint32_t reg, val, pipeconf_val; /* PCH only available on ILK+ */ - BUG_ON(INTEL_INFO(dev)->gen < 5); + BUG_ON(!HAS_PCH_SPLIT(dev)); /* Make sure PCH DPLL is enabled */ assert_shared_dpll_enabled(dev_priv, @@ -1921,7 +1952,7 @@ static void lpt_enable_pch_transcoder(struct drm_i915_private *dev_priv, u32 val, pipeconf_val; /* PCH only available on ILK+ */ - BUG_ON(INTEL_INFO(dev_priv->dev)->gen < 5); + BUG_ON(!HAS_PCH_SPLIT(dev_priv->dev)); /* FDI must be feeding us bits for PCH ports */ assert_fdi_tx_enabled(dev_priv, (enum pipe) cpu_transcoder); @@ -2043,8 +2074,8 @@ static void intel_enable_pipe(struct intel_crtc *crtc) reg = PIPECONF(cpu_transcoder); val = I915_READ(reg); if (val & PIPECONF_ENABLE) { - WARN_ON(!(pipe == PIPE_A && - dev_priv->quirks & QUIRK_PIPEA_FORCE)); + WARN_ON(!((pipe == PIPE_A && dev_priv->quirks & QUIRK_PIPEA_FORCE) || + (pipe == PIPE_B && dev_priv->quirks & QUIRK_PIPEB_FORCE))); return; } @@ -2054,21 +2085,19 @@ static void intel_enable_pipe(struct intel_crtc *crtc) /** * intel_disable_pipe - disable a pipe, asserting requirements - * @dev_priv: i915 private structure - * @pipe: pipe to disable - * - * Disable @pipe, making sure that various hardware specific requirements - * are met, if applicable, e.g. plane disabled, panel fitter off, etc. + * @crtc: crtc whose pipes is to be disabled * - * @pipe should be %PIPE_A or %PIPE_B. + * Disable the pipe of @crtc, making sure that various hardware + * specific requirements are met, if applicable, e.g. plane + * disabled, panel fitter off, etc. * * Will wait until the pipe has shut down before returning. */ -static void intel_disable_pipe(struct drm_i915_private *dev_priv, - enum pipe pipe) +static void intel_disable_pipe(struct intel_crtc *crtc) { - enum transcoder cpu_transcoder = intel_pipe_to_cpu_transcoder(dev_priv, - pipe); + struct drm_i915_private *dev_priv = crtc->base.dev->dev_private; + enum transcoder cpu_transcoder = crtc->config.cpu_transcoder; + enum pipe pipe = crtc->pipe; int reg; u32 val; @@ -2080,17 +2109,26 @@ static void intel_disable_pipe(struct drm_i915_private *dev_priv, assert_cursor_disabled(dev_priv, pipe); assert_sprites_disabled(dev_priv, pipe); - /* Don't disable pipe A or pipe A PLLs if needed */ - if (pipe == PIPE_A && (dev_priv->quirks & QUIRK_PIPEA_FORCE)) - return; - reg = PIPECONF(cpu_transcoder); val = I915_READ(reg); if ((val & PIPECONF_ENABLE) == 0) return; - I915_WRITE(reg, val & ~PIPECONF_ENABLE); - intel_wait_for_pipe_off(dev_priv->dev, pipe); + /* + * Double wide has implications for planes + * so best keep it disabled when not needed. + */ + if (crtc->config.double_wide) + val &= ~PIPECONF_DOUBLE_WIDE; + + /* Don't disable pipe or pipe PLLs if needed */ + if (!(pipe == PIPE_A && dev_priv->quirks & QUIRK_PIPEA_FORCE) && + !(pipe == PIPE_B && dev_priv->quirks & QUIRK_PIPEB_FORCE)) + val &= ~PIPECONF_ENABLE; + + I915_WRITE(reg, val); + if ((val & PIPECONF_ENABLE) == 0) + intel_wait_for_pipe_off(crtc); } /* @@ -2109,35 +2147,28 @@ void intel_flush_primary_plane(struct drm_i915_private *dev_priv, /** * intel_enable_primary_hw_plane - enable the primary plane on a given pipe - * @dev_priv: i915 private structure - * @plane: plane to enable - * @pipe: pipe being fed + * @plane: plane to be enabled + * @crtc: crtc for the plane * - * Enable @plane on @pipe, making sure that @pipe is running first. + * Enable @plane on @crtc, making sure that the pipe is running first. */ -static void intel_enable_primary_hw_plane(struct drm_i915_private *dev_priv, - enum plane plane, enum pipe pipe) +static void intel_enable_primary_hw_plane(struct drm_plane *plane, + struct drm_crtc *crtc) { - struct drm_device *dev = dev_priv->dev; - struct intel_crtc *intel_crtc = - to_intel_crtc(dev_priv->pipe_to_crtc_mapping[pipe]); - int reg; - u32 val; + struct drm_device *dev = plane->dev; + struct drm_i915_private *dev_priv = dev->dev_private; + struct intel_crtc *intel_crtc = to_intel_crtc(crtc); /* If the pipe isn't enabled, we can't pump pixels and may hang */ - assert_pipe_enabled(dev_priv, pipe); + assert_pipe_enabled(dev_priv, intel_crtc->pipe); if (intel_crtc->primary_enabled) return; intel_crtc->primary_enabled = true; - reg = DSPCNTR(plane); - val = I915_READ(reg); - WARN_ON(val & DISPLAY_PLANE_ENABLE); - - I915_WRITE(reg, val | DISPLAY_PLANE_ENABLE); - intel_flush_primary_plane(dev_priv, plane); + dev_priv->display.update_primary_plane(crtc, plane->fb, + crtc->x, crtc->y); /* * BDW signals flip done immediately if the plane @@ -2150,31 +2181,27 @@ static void intel_enable_primary_hw_plane(struct drm_i915_private *dev_priv, /** * intel_disable_primary_hw_plane - disable the primary hardware plane - * @dev_priv: i915 private structure - * @plane: plane to disable - * @pipe: pipe consuming the data + * @plane: plane to be disabled + * @crtc: crtc for the plane * - * Disable @plane; should be an independent operation. + * Disable @plane on @crtc, making sure that the pipe is running first. */ -static void intel_disable_primary_hw_plane(struct drm_i915_private *dev_priv, - enum plane plane, enum pipe pipe) +static void intel_disable_primary_hw_plane(struct drm_plane *plane, + struct drm_crtc *crtc) { - struct intel_crtc *intel_crtc = - to_intel_crtc(dev_priv->pipe_to_crtc_mapping[pipe]); - int reg; - u32 val; + struct drm_device *dev = plane->dev; + struct drm_i915_private *dev_priv = dev->dev_private; + struct intel_crtc *intel_crtc = to_intel_crtc(crtc); + + assert_pipe_enabled(dev_priv, intel_crtc->pipe); if (!intel_crtc->primary_enabled) return; intel_crtc->primary_enabled = false; - reg = DSPCNTR(plane); - val = I915_READ(reg); - WARN_ON((val & DISPLAY_PLANE_ENABLE) == 0); - - I915_WRITE(reg, val & ~DISPLAY_PLANE_ENABLE); - intel_flush_primary_plane(dev_priv, plane); + dev_priv->display.update_primary_plane(crtc, plane->fb, + crtc->x, crtc->y); } static bool need_vtd_wa(struct drm_device *dev) @@ -2197,7 +2224,7 @@ static int intel_align_height(struct drm_device *dev, int height, bool tiled) int intel_pin_and_fence_fb_obj(struct drm_device *dev, struct drm_i915_gem_object *obj, - struct intel_engine_cs *pipelined) + struct i915_gem_request *pipelined) { struct drm_i915_private *dev_priv = dev->dev_private; u32 alignment; @@ -2422,16 +2449,46 @@ static void i9xx_update_primary_plane(struct drm_crtc *crtc, struct drm_device *dev = crtc->dev; struct drm_i915_private *dev_priv = dev->dev_private; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); - struct drm_i915_gem_object *obj = intel_fb_obj(fb); + struct drm_i915_gem_object *obj; int plane = intel_crtc->plane; unsigned long linear_offset; u32 dspcntr; - u32 reg; + u32 reg = DSPCNTR(plane); + int pixel_size; + + if (!intel_crtc->primary_enabled) { + I915_WRITE(reg, 0); + if (INTEL_INFO(dev)->gen >= 4) + I915_WRITE(DSPSURF(plane), 0); + else + I915_WRITE(DSPADDR(plane), 0); + POSTING_READ(reg); + return; + } + + obj = intel_fb_obj(fb); + if (WARN_ON(obj == NULL)) + return; + + pixel_size = drm_format_plane_cpp(fb->pixel_format, 0); + + dspcntr = DISPPLANE_GAMMA_ENABLE; + + dspcntr |= DISPLAY_PLANE_ENABLE; + + if (INTEL_INFO(dev)->gen < 4) { + if (intel_crtc->pipe == PIPE_B) + dspcntr |= DISPPLANE_SEL_PIPE_B; + + /* pipesrc and dspsize control the size that is scaled from, + * which should always be the user's requested size. + */ + I915_WRITE(DSPSIZE(plane), + ((intel_crtc->config.pipe_src_h - 1) << 16) | + (intel_crtc->config.pipe_src_w - 1)); + I915_WRITE(DSPPOS(plane), 0); + } - reg = DSPCNTR(plane); - dspcntr = I915_READ(reg); - /* Mask out pixel format bits in case we change it */ - dspcntr &= ~DISPPLANE_PIXFORMAT_MASK; switch (fb->pixel_format) { case DRM_FORMAT_C8: dspcntr |= DISPPLANE_8BPP; @@ -2463,30 +2520,40 @@ static void i9xx_update_primary_plane(struct drm_crtc *crtc, BUG(); } - if (INTEL_INFO(dev)->gen >= 4) { - if (obj->tiling_mode != I915_TILING_NONE) - dspcntr |= DISPPLANE_TILED; - else - dspcntr &= ~DISPPLANE_TILED; - } + if (INTEL_INFO(dev)->gen >= 4 && + obj->tiling_mode != I915_TILING_NONE) + dspcntr |= DISPPLANE_TILED; if (IS_G4X(dev)) dspcntr |= DISPPLANE_TRICKLE_FEED_DISABLE; - I915_WRITE(reg, dspcntr); - - linear_offset = y * fb->pitches[0] + x * (fb->bits_per_pixel / 8); + linear_offset = y * fb->pitches[0] + x * pixel_size; if (INTEL_INFO(dev)->gen >= 4) { intel_crtc->dspaddr_offset = intel_gen4_compute_page_offset(&x, &y, obj->tiling_mode, - fb->bits_per_pixel / 8, + pixel_size, fb->pitches[0]); linear_offset -= intel_crtc->dspaddr_offset; } else { intel_crtc->dspaddr_offset = linear_offset; } + if (to_intel_plane(crtc->primary)->rotation == BIT(DRM_ROTATE_180)) { + dspcntr |= DISPPLANE_ROTATE_180; + + x += (intel_crtc->config.pipe_src_w - 1); + y += (intel_crtc->config.pipe_src_h - 1); + + /* Finding the last pixel of the last line of the display + data and adding to linear_offset*/ + linear_offset += + (intel_crtc->config.pipe_src_h - 1) * fb->pitches[0] + + (intel_crtc->config.pipe_src_w - 1) * pixel_size; + } + + I915_WRITE(reg, dspcntr); + DRM_DEBUG_KMS("Writing base %08lX %08lX %d %d %d\n", i915_gem_obj_ggtt_offset(obj), linear_offset, x, y, fb->pitches[0]); @@ -2508,16 +2575,33 @@ static void ironlake_update_primary_plane(struct drm_crtc *crtc, struct drm_device *dev = crtc->dev; struct drm_i915_private *dev_priv = dev->dev_private; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); - struct drm_i915_gem_object *obj = intel_fb_obj(fb); + struct drm_i915_gem_object *obj; int plane = intel_crtc->plane; unsigned long linear_offset; u32 dspcntr; - u32 reg; + u32 reg = DSPCNTR(plane); + int pixel_size; + + if (!intel_crtc->primary_enabled) { + I915_WRITE(reg, 0); + I915_WRITE(DSPSURF(plane), 0); + POSTING_READ(reg); + return; + } + + obj = intel_fb_obj(fb); + if (WARN_ON(obj == NULL)) + return; + + pixel_size = drm_format_plane_cpp(fb->pixel_format, 0); + + dspcntr = DISPPLANE_GAMMA_ENABLE; + + dspcntr |= DISPLAY_PLANE_ENABLE; + + if (IS_HASWELL(dev) || IS_BROADWELL(dev)) + dspcntr |= DISPPLANE_PIPE_CSC_ENABLE; - reg = DSPCNTR(plane); - dspcntr = I915_READ(reg); - /* Mask out pixel format bits in case we change it */ - dspcntr &= ~DISPPLANE_PIXFORMAT_MASK; switch (fb->pixel_format) { case DRM_FORMAT_C8: dspcntr |= DISPPLANE_8BPP; @@ -2547,22 +2631,32 @@ static void ironlake_update_primary_plane(struct drm_crtc *crtc, if (obj->tiling_mode != I915_TILING_NONE) dspcntr |= DISPPLANE_TILED; - else - dspcntr &= ~DISPPLANE_TILED; - if (IS_HASWELL(dev) || IS_BROADWELL(dev)) - dspcntr &= ~DISPPLANE_TRICKLE_FEED_DISABLE; - else + if (!IS_HASWELL(dev) && !IS_BROADWELL(dev)) dspcntr |= DISPPLANE_TRICKLE_FEED_DISABLE; - I915_WRITE(reg, dspcntr); - - linear_offset = y * fb->pitches[0] + x * (fb->bits_per_pixel / 8); + linear_offset = y * fb->pitches[0] + x * pixel_size; intel_crtc->dspaddr_offset = intel_gen4_compute_page_offset(&x, &y, obj->tiling_mode, - fb->bits_per_pixel / 8, + pixel_size, fb->pitches[0]); linear_offset -= intel_crtc->dspaddr_offset; + if (to_intel_plane(crtc->primary)->rotation == BIT(DRM_ROTATE_180)) { + dspcntr |= DISPPLANE_ROTATE_180; + + if (!IS_HASWELL(dev) && !IS_BROADWELL(dev)) { + x += (intel_crtc->config.pipe_src_w - 1); + y += (intel_crtc->config.pipe_src_h - 1); + + /* Finding the last pixel of the last line of the display + data and adding to linear_offset*/ + linear_offset += + (intel_crtc->config.pipe_src_h - 1) * fb->pitches[0] + + (intel_crtc->config.pipe_src_w - 1) * pixel_size; + } + } + + I915_WRITE(reg, dspcntr); DRM_DEBUG_KMS("Writing base %08lX %08lX %d %d %d\n", i915_gem_obj_ggtt_offset(obj), linear_offset, x, y, @@ -2669,20 +2763,58 @@ static bool intel_crtc_has_pending_flip(struct drm_crtc *crtc) struct drm_device *dev = crtc->dev; struct drm_i915_private *dev_priv = dev->dev_private; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); - unsigned long flags; bool pending; if (i915_reset_in_progress(&dev_priv->gpu_error) || intel_crtc->reset_counter != atomic_read(&dev_priv->gpu_error.reset_counter)) return false; - spin_lock_irqsave(&dev->event_lock, flags); + spin_lock_irq(&dev->event_lock); pending = to_intel_crtc(crtc)->unpin_work != NULL; - spin_unlock_irqrestore(&dev->event_lock, flags); + spin_unlock_irq(&dev->event_lock); return pending; } +static void intel_update_pipe_size(struct intel_crtc *crtc) +{ + struct drm_device *dev = crtc->base.dev; + struct drm_i915_private *dev_priv = dev->dev_private; + const struct drm_display_mode *adjusted_mode; + + if (!i915.fastboot) + return; + + /* + * Update pipe size and adjust fitter if needed: the reason for this is + * that in compute_mode_changes we check the native mode (not the pfit + * mode) to see if we can flip rather than do a full mode set. In the + * fastboot case, we'll flip, but if we don't update the pipesrc and + * pfit state, we'll end up with a big fb scanned out into the wrong + * sized surface. + * + * To fix this properly, we need to hoist the checks up into + * compute_mode_changes (or above), check the actual pfit state and + * whether the platform allows pfit disable with pipe active, and only + * then update the pipesrc and pfit state, even on the flip path. + */ + + adjusted_mode = &crtc->config.adjusted_mode; + + I915_WRITE(PIPESRC(crtc->pipe), + ((adjusted_mode->crtc_hdisplay - 1) << 16) | + (adjusted_mode->crtc_vdisplay - 1)); + if (!crtc->config.pch_pfit.enabled && + (intel_pipe_has_type(&crtc->base, INTEL_OUTPUT_LVDS) || + intel_pipe_has_type(&crtc->base, INTEL_OUTPUT_EDP))) { + I915_WRITE(PF_CTL(crtc->pipe), 0); + I915_WRITE(PF_WIN_POS(crtc->pipe), 0); + I915_WRITE(PF_WIN_SZ(crtc->pipe), 0); + } + crtc->config.pipe_src_w = adjusted_mode->crtc_hdisplay; + crtc->config.pipe_src_h = adjusted_mode->crtc_vdisplay; +} + static int intel_pipe_set_base(struct drm_crtc *crtc, int x, int y, struct drm_framebuffer *fb) @@ -2725,36 +2857,7 @@ intel_pipe_set_base(struct drm_crtc *crtc, int x, int y, return ret; } - /* - * Update pipe size and adjust fitter if needed: the reason for this is - * that in compute_mode_changes we check the native mode (not the pfit - * mode) to see if we can flip rather than do a full mode set. In the - * fastboot case, we'll flip, but if we don't update the pipesrc and - * pfit state, we'll end up with a big fb scanned out into the wrong - * sized surface. - * - * To fix this properly, we need to hoist the checks up into - * compute_mode_changes (or above), check the actual pfit state and - * whether the platform allows pfit disable with pipe active, and only - * then update the pipesrc and pfit state, even on the flip path. - */ - if (i915.fastboot) { - const struct drm_display_mode *adjusted_mode = - &intel_crtc->config.adjusted_mode; - - I915_WRITE(PIPESRC(intel_crtc->pipe), - ((adjusted_mode->crtc_hdisplay - 1) << 16) | - (adjusted_mode->crtc_vdisplay - 1)); - if (!intel_crtc->config.pch_pfit.enabled && - (intel_pipe_has_type(crtc, INTEL_OUTPUT_LVDS) || - intel_pipe_has_type(crtc, INTEL_OUTPUT_EDP))) { - I915_WRITE(PF_CTL(intel_crtc->pipe), 0); - I915_WRITE(PF_WIN_POS(intel_crtc->pipe), 0); - I915_WRITE(PF_WIN_SZ(intel_crtc->pipe), 0); - } - intel_crtc->config.pipe_src_w = adjusted_mode->crtc_hdisplay; - intel_crtc->config.pipe_src_h = adjusted_mode->crtc_vdisplay; - } + intel_update_pipe_size(intel_crtc); dev_priv->display.update_primary_plane(crtc, fb, x, y); @@ -3346,23 +3449,53 @@ bool intel_has_pending_fb_unpin(struct drm_device *dev) return false; } +static void page_flip_completed(struct intel_crtc *intel_crtc) +{ + struct drm_i915_private *dev_priv = to_i915(intel_crtc->base.dev); + struct intel_unpin_work *work = intel_crtc->unpin_work; + + /* ensure that the unpin work is consistent wrt ->pending. */ + smp_rmb(); + intel_crtc->unpin_work = NULL; + + if (work->event) + drm_send_vblank_event(intel_crtc->base.dev, + intel_crtc->pipe, + work->event); + + drm_crtc_vblank_put(&intel_crtc->base); + + wake_up_all(&dev_priv->pending_flip_queue); + queue_work(dev_priv->wq, &work->work); + + trace_i915_flip_complete(intel_crtc->plane, + work->pending_flip_obj); +} + void intel_crtc_wait_for_pending_flips(struct drm_crtc *crtc) { struct drm_device *dev = crtc->dev; struct drm_i915_private *dev_priv = dev->dev_private; - if (crtc->primary->fb == NULL) - return; - WARN_ON(waitqueue_active(&dev_priv->pending_flip_queue)); + if (WARN_ON(wait_event_timeout(dev_priv->pending_flip_queue, + !intel_crtc_has_pending_flip(crtc), + 60*HZ) == 0)) { + struct intel_crtc *intel_crtc = to_intel_crtc(crtc); - WARN_ON(wait_event_timeout(dev_priv->pending_flip_queue, - !intel_crtc_has_pending_flip(crtc), - 60*HZ) == 0); + spin_lock_irq(&dev->event_lock); + if (intel_crtc->unpin_work) { + WARN_ONCE(1, "Removing stuck page flip\n"); + page_flip_completed(intel_crtc); + } + spin_unlock_irq(&dev->event_lock); + } - mutex_lock(&dev->struct_mutex); - intel_finish_fb(crtc->primary->fb); - mutex_unlock(&dev->struct_mutex); + if (crtc->primary->fb) { + mutex_lock(&dev->struct_mutex); + intel_finish_fb(crtc->primary->fb); + mutex_unlock(&dev->struct_mutex); + } } /* Program iCLKIP clock to the desired frequency */ @@ -3911,14 +4044,10 @@ static void intel_crtc_dpms_overlay(struct intel_crtc *intel_crtc, bool enable) static void intel_crtc_enable_planes(struct drm_crtc *crtc) { struct drm_device *dev = crtc->dev; - struct drm_i915_private *dev_priv = dev->dev_private; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); int pipe = intel_crtc->pipe; - int plane = intel_crtc->plane; - - drm_vblank_on(dev, pipe); - intel_enable_primary_hw_plane(dev_priv, plane, pipe); + intel_enable_primary_hw_plane(crtc->primary, crtc); intel_enable_planes(crtc); intel_crtc_update_cursor(crtc, true); intel_crtc_dpms_overlay(intel_crtc, true); @@ -3955,7 +4084,7 @@ static void intel_crtc_disable_planes(struct drm_crtc *crtc) intel_crtc_dpms_overlay(intel_crtc, false); intel_crtc_update_cursor(crtc, false); intel_disable_planes(crtc); - intel_disable_primary_hw_plane(dev_priv, plane, pipe); + intel_disable_primary_hw_plane(crtc->primary, crtc); /* * FIXME: Once we grow proper nuclear flip support out of this we need @@ -3963,8 +4092,6 @@ static void intel_crtc_disable_planes(struct drm_crtc *crtc) * consider this a flip to a NULL plane. */ intel_frontbuffer_flip(dev, INTEL_FRONTBUFFER_ALL_MASK(pipe)); - - drm_vblank_off(dev, pipe); } static void ironlake_crtc_enable(struct drm_crtc *crtc) @@ -3974,7 +4101,6 @@ static void ironlake_crtc_enable(struct drm_crtc *crtc) struct intel_crtc *intel_crtc = to_intel_crtc(crtc); struct intel_encoder *encoder; int pipe = intel_crtc->pipe; - enum plane plane = intel_crtc->plane; WARN_ON(!crtc->enabled); @@ -3991,18 +4117,11 @@ static void ironlake_crtc_enable(struct drm_crtc *crtc) if (intel_crtc->config.has_pch_encoder) { intel_cpu_transcoder_set_m_n(intel_crtc, - &intel_crtc->config.fdi_m_n); + &intel_crtc->config.fdi_m_n, NULL); } ironlake_set_pipeconf(crtc); - /* Set up the display plane register */ - I915_WRITE(DSPCNTR(plane), DISPPLANE_GAMMA_ENABLE); - POSTING_READ(DSPCNTR(plane)); - - dev_priv->display.update_primary_plane(crtc, crtc->primary->fb, - crtc->x, crtc->y); - intel_crtc->active = true; intel_set_cpu_fifo_underrun_reporting(dev, pipe, true); @@ -4042,6 +4161,9 @@ static void ironlake_crtc_enable(struct drm_crtc *crtc) if (HAS_PCH_CPT(dev)) cpt_verify_modeset(dev, intel_crtc->pipe); + assert_vblank_disabled(crtc); + drm_crtc_vblank_on(crtc); + intel_crtc_enable_planes(crtc); } @@ -4087,7 +4209,6 @@ static void haswell_crtc_enable(struct drm_crtc *crtc) struct intel_crtc *intel_crtc = to_intel_crtc(crtc); struct intel_encoder *encoder; int pipe = intel_crtc->pipe; - enum plane plane = intel_crtc->plane; WARN_ON(!crtc->enabled); @@ -4104,20 +4225,13 @@ static void haswell_crtc_enable(struct drm_crtc *crtc) if (intel_crtc->config.has_pch_encoder) { intel_cpu_transcoder_set_m_n(intel_crtc, - &intel_crtc->config.fdi_m_n); + &intel_crtc->config.fdi_m_n, NULL); } haswell_set_pipeconf(crtc); intel_set_pipe_csc(crtc); - /* Set up the display plane register */ - I915_WRITE(DSPCNTR(plane), DISPPLANE_GAMMA_ENABLE | DISPPLANE_PIPE_CSC_ENABLE); - POSTING_READ(DSPCNTR(plane)); - - dev_priv->display.update_primary_plane(crtc, crtc->primary->fb, - crtc->x, crtc->y); - intel_crtc->active = true; intel_set_cpu_fifo_underrun_reporting(dev, pipe, true); @@ -4157,6 +4271,9 @@ static void haswell_crtc_enable(struct drm_crtc *crtc) intel_opregion_notify_encoder(encoder, true); } + assert_vblank_disabled(crtc); + drm_crtc_vblank_on(crtc); + /* If we change the relative order between pipe/planes enabling, we need * to change the workaround. */ haswell_mode_set_planes_workaround(intel_crtc); @@ -4192,13 +4309,17 @@ static void ironlake_crtc_disable(struct drm_crtc *crtc) intel_crtc_disable_planes(crtc); + drm_crtc_vblank_off(crtc); + assert_vblank_disabled(crtc); + for_each_encoder_on_crtc(dev, crtc, encoder) encoder->disable(encoder); if (intel_crtc->config.has_pch_encoder) intel_set_pch_fifo_underrun_reporting(dev, pipe, false); - intel_disable_pipe(dev_priv, pipe); + intel_disable_pipe(intel_crtc); + ironlake_pfit_disable(intel_crtc); for_each_encoder_on_crtc(dev, crtc, encoder) @@ -4246,7 +4367,6 @@ static void haswell_crtc_disable(struct drm_crtc *crtc) struct drm_i915_private *dev_priv = dev->dev_private; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); struct intel_encoder *encoder; - int pipe = intel_crtc->pipe; enum transcoder cpu_transcoder = intel_crtc->config.cpu_transcoder; if (!intel_crtc->active) @@ -4254,6 +4374,9 @@ static void haswell_crtc_disable(struct drm_crtc *crtc) intel_crtc_disable_planes(crtc); + drm_crtc_vblank_off(crtc); + assert_vblank_disabled(crtc); + for_each_encoder_on_crtc(dev, crtc, encoder) { intel_opregion_notify_encoder(encoder, false); encoder->disable(encoder); @@ -4261,7 +4384,7 @@ static void haswell_crtc_disable(struct drm_crtc *crtc) if (intel_crtc->config.has_pch_encoder) intel_set_pch_fifo_underrun_reporting(dev, TRANSCODER_A, false); - intel_disable_pipe(dev_priv, pipe); + intel_disable_pipe(intel_crtc); if (intel_crtc->config.dp_encoder_is_mst) intel_ddi_set_vc_payload_alloc(crtc, false); @@ -4539,12 +4662,57 @@ static void valleyview_set_cdclk(struct drm_device *dev, int cdclk) vlv_update_cdclk(dev); } +static void cherryview_set_cdclk(struct drm_device *dev, int cdclk) +{ + struct drm_i915_private *dev_priv = dev->dev_private; + u32 val, cmd; + + WARN_ON(dev_priv->display.get_display_clock_speed(dev) != dev_priv->vlv_cdclk_freq); + + switch (cdclk) { + case 400000: + cmd = 3; + break; + case 333333: + case 320000: + cmd = 2; + break; + case 266667: + cmd = 1; + break; + case 200000: + cmd = 0; + break; + default: + WARN_ON(1); + return; + } + + mutex_lock(&dev_priv->rps.hw_lock); + val = vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ); + val &= ~DSPFREQGUAR_MASK_CHV; + val |= (cmd << DSPFREQGUAR_SHIFT_CHV); + vlv_punit_write(dev_priv, PUNIT_REG_DSPFREQ, val); + if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ) & + DSPFREQSTAT_MASK_CHV) == (cmd << DSPFREQSTAT_SHIFT_CHV), + 50)) { + DRM_ERROR("timed out waiting for CDclk change\n"); + } + mutex_unlock(&dev_priv->rps.hw_lock); + + vlv_update_cdclk(dev); +} + static int valleyview_calc_cdclk(struct drm_i915_private *dev_priv, int max_pixclk) { int vco = valleyview_get_vco(dev_priv); int freq_320 = (vco << 1) % 320000 != 0 ? 333333 : 320000; + /* FIXME: Punit isn't quite ready yet */ + if (IS_CHERRYVIEW(dev_priv->dev)) + return 400000; + /* * Really only a few cases to deal with, as only 4 CDclks are supported: * 200MHz @@ -4607,21 +4775,23 @@ static void valleyview_modeset_global_resources(struct drm_device *dev) int max_pixclk = intel_mode_max_pixclk(dev_priv); int req_cdclk = valleyview_calc_cdclk(dev_priv, max_pixclk); - if (req_cdclk != dev_priv->vlv_cdclk_freq) - valleyview_set_cdclk(dev, req_cdclk); + if (req_cdclk != dev_priv->vlv_cdclk_freq) { + if (IS_CHERRYVIEW(dev)) + cherryview_set_cdclk(dev, req_cdclk); + else + valleyview_set_cdclk(dev, req_cdclk); + } + modeset_update_crtc_power_domains(dev); } static void valleyview_crtc_enable(struct drm_crtc *crtc) { struct drm_device *dev = crtc->dev; - struct drm_i915_private *dev_priv = dev->dev_private; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); struct intel_encoder *encoder; int pipe = intel_crtc->pipe; - int plane = intel_crtc->plane; bool is_dsi; - u32 dspcntr; WARN_ON(!crtc->enabled); @@ -4630,33 +4800,20 @@ static void valleyview_crtc_enable(struct drm_crtc *crtc) is_dsi = intel_pipe_has_type(crtc, INTEL_OUTPUT_DSI); - if (!is_dsi && !IS_CHERRYVIEW(dev)) - vlv_prepare_pll(intel_crtc); - - /* Set up the display plane register */ - dspcntr = DISPPLANE_GAMMA_ENABLE; + if (!is_dsi) { + if (IS_CHERRYVIEW(dev)) + chv_prepare_pll(intel_crtc); + else + vlv_prepare_pll(intel_crtc); + } if (intel_crtc->config.has_dp_encoder) intel_dp_set_m_n(intel_crtc); intel_set_pipe_timings(intel_crtc); - /* pipesrc and dspsize control the size that is scaled from, - * which should always be the user's requested size. - */ - I915_WRITE(DSPSIZE(plane), - ((intel_crtc->config.pipe_src_h - 1) << 16) | - (intel_crtc->config.pipe_src_w - 1)); - I915_WRITE(DSPPOS(plane), 0); - i9xx_set_pipeconf(intel_crtc); - I915_WRITE(DSPCNTR(plane), dspcntr); - POSTING_READ(DSPCNTR(plane)); - - dev_priv->display.update_primary_plane(crtc, crtc->primary->fb, - crtc->x, crtc->y); - intel_crtc->active = true; intel_set_cpu_fifo_underrun_reporting(dev, pipe, true); @@ -4686,6 +4843,9 @@ static void valleyview_crtc_enable(struct drm_crtc *crtc) for_each_encoder_on_crtc(dev, crtc, encoder) encoder->enable(encoder); + assert_vblank_disabled(crtc); + drm_crtc_vblank_on(crtc); + intel_crtc_enable_planes(crtc); /* Underruns don't raise interrupts, so check manually. */ @@ -4704,49 +4864,24 @@ static void i9xx_set_pll_dividers(struct intel_crtc *crtc) static void i9xx_crtc_enable(struct drm_crtc *crtc) { struct drm_device *dev = crtc->dev; - struct drm_i915_private *dev_priv = dev->dev_private; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); struct intel_encoder *encoder; int pipe = intel_crtc->pipe; - int plane = intel_crtc->plane; - u32 dspcntr; WARN_ON(!crtc->enabled); if (intel_crtc->active) return; - i9xx_set_pll_dividers(intel_crtc); - - /* Set up the display plane register */ - dspcntr = DISPPLANE_GAMMA_ENABLE; - - if (pipe == 0) - dspcntr &= ~DISPPLANE_SEL_PIPE_MASK; - else - dspcntr |= DISPPLANE_SEL_PIPE_B; + i9xx_set_pll_dividers(intel_crtc); if (intel_crtc->config.has_dp_encoder) intel_dp_set_m_n(intel_crtc); intel_set_pipe_timings(intel_crtc); - /* pipesrc and dspsize control the size that is scaled from, - * which should always be the user's requested size. - */ - I915_WRITE(DSPSIZE(plane), - ((intel_crtc->config.pipe_src_h - 1) << 16) | - (intel_crtc->config.pipe_src_w - 1)); - I915_WRITE(DSPPOS(plane), 0); - i9xx_set_pipeconf(intel_crtc); - I915_WRITE(DSPCNTR(plane), dspcntr); - POSTING_READ(DSPCNTR(plane)); - - dev_priv->display.update_primary_plane(crtc, crtc->primary->fb, - crtc->x, crtc->y); - intel_crtc->active = true; if (!IS_GEN2(dev)) @@ -4768,6 +4903,9 @@ static void i9xx_crtc_enable(struct drm_crtc *crtc) for_each_encoder_on_crtc(dev, crtc, encoder) encoder->enable(encoder); + assert_vblank_disabled(crtc); + drm_crtc_vblank_on(crtc); + intel_crtc_enable_planes(crtc); /* @@ -4831,9 +4969,6 @@ static void i9xx_crtc_disable(struct drm_crtc *crtc) intel_set_memory_cxsr(dev_priv, false); intel_crtc_disable_planes(crtc); - for_each_encoder_on_crtc(dev, crtc, encoder) - encoder->disable(encoder); - /* * On gen2 planes are double buffered but the pipe isn't, so we must * wait for planes to fully turn off before disabling the pipe. @@ -4842,7 +4977,13 @@ static void i9xx_crtc_disable(struct drm_crtc *crtc) */ intel_wait_for_vblank(dev, pipe); - intel_disable_pipe(dev_priv, pipe); + drm_crtc_vblank_off(crtc); + assert_vblank_disabled(crtc); + + for_each_encoder_on_crtc(dev, crtc, encoder) + encoder->disable(encoder); + + intel_disable_pipe(intel_crtc); i9xx_pfit_disable(intel_crtc); @@ -4856,7 +4997,7 @@ static void i9xx_crtc_disable(struct drm_crtc *crtc) else if (IS_VALLEYVIEW(dev)) vlv_disable_pll(dev_priv, pipe); else - i9xx_disable_pll(dev_priv, pipe); + i9xx_disable_pll(intel_crtc); } if (!IS_GEN2(dev)) @@ -4874,36 +5015,6 @@ static void i9xx_crtc_off(struct drm_crtc *crtc) { } -static void intel_crtc_update_sarea(struct drm_crtc *crtc, - bool enabled) -{ - struct drm_device *dev = crtc->dev; - struct drm_i915_master_private *master_priv; - struct intel_crtc *intel_crtc = to_intel_crtc(crtc); - int pipe = intel_crtc->pipe; - - if (!dev->primary->master) - return; - - master_priv = dev->primary->master->driver_priv; - if (!master_priv->sarea_priv) - return; - - switch (pipe) { - case 0: - master_priv->sarea_priv->pipeA_w = enabled ? crtc->mode.hdisplay : 0; - master_priv->sarea_priv->pipeA_h = enabled ? crtc->mode.vdisplay : 0; - break; - case 1: - master_priv->sarea_priv->pipeB_w = enabled ? crtc->mode.hdisplay : 0; - master_priv->sarea_priv->pipeB_h = enabled ? crtc->mode.vdisplay : 0; - break; - default: - DRM_ERROR("Can't update pipe %c in SAREA\n", pipe_name(pipe)); - break; - } -} - /* Master function to enable/disable CRTC and corresponding power wells */ void intel_crtc_control(struct drm_crtc *crtc, bool enable) { @@ -4947,8 +5058,6 @@ void intel_crtc_update_dpms(struct drm_crtc *crtc) enable |= intel_encoder->connectors_active; intel_crtc_control(crtc, enable); - - intel_crtc_update_sarea(crtc, enable); } static void intel_crtc_disable(struct drm_crtc *crtc) @@ -4963,7 +5072,6 @@ static void intel_crtc_disable(struct drm_crtc *crtc) WARN_ON(!crtc->enabled); dev_priv->display.crtc_disable(crtc); - intel_crtc_update_sarea(crtc, false); dev_priv->display.off(crtc); if (crtc->primary->fb) { @@ -5190,12 +5298,29 @@ retry: return setup_ok ? 0 : -EINVAL; } +static bool pipe_config_supports_ips(struct drm_i915_private *dev_priv, + struct intel_crtc_config *pipe_config) +{ + if (pipe_config->pipe_bpp > 24) + return false; + + /* HSW can handle pixel rate up to cdclk? */ + if (IS_HASWELL(dev_priv->dev)) + return true; + + return ilk_pipe_pixel_rate(pipe_config) <= + intel_ddi_get_cdclk_freq(dev_priv) * 95 / 100; +} + static void hsw_compute_ips_config(struct intel_crtc *crtc, struct intel_crtc_config *pipe_config) { + struct drm_device *dev = crtc->base.dev; + struct drm_i915_private *dev_priv = dev->dev_private; + pipe_config->ips_enabled = i915.enable_ips && - hsw_crtc_supports_ips(crtc) && - pipe_config->pipe_bpp <= 24; + hsw_crtc_supports_ips(crtc) && + pipe_config_supports_ips(dev_priv, pipe_config); } static int intel_crtc_compute_config(struct intel_crtc *crtc, @@ -5275,6 +5400,10 @@ static int valleyview_get_display_clock_speed(struct drm_device *dev) u32 val; int divider; + /* FIXME: Punit isn't quite ready yet */ + if (IS_CHERRYVIEW(dev)) + return 400000; + mutex_lock(&dev_priv->dpio_lock); val = vlv_cck_read(dev_priv, CCK_DISPLAY_CLOCK_CONTROL); mutex_unlock(&dev_priv->dpio_lock); @@ -5519,7 +5648,8 @@ static void intel_pch_transcoder_set_m_n(struct intel_crtc *crtc, } static void intel_cpu_transcoder_set_m_n(struct intel_crtc *crtc, - struct intel_link_m_n *m_n) + struct intel_link_m_n *m_n, + struct intel_link_m_n *m2_n2) { struct drm_device *dev = crtc->base.dev; struct drm_i915_private *dev_priv = dev->dev_private; @@ -5531,6 +5661,18 @@ static void intel_cpu_transcoder_set_m_n(struct intel_crtc *crtc, I915_WRITE(PIPE_DATA_N1(transcoder), m_n->gmch_n); I915_WRITE(PIPE_LINK_M1(transcoder), m_n->link_m); I915_WRITE(PIPE_LINK_N1(transcoder), m_n->link_n); + /* M2_N2 registers to be set only for gen < 8 (M2_N2 available + * for gen < 8) and if DRRS is supported (to make sure the + * registers are not unnecessarily accessed). + */ + if (m2_n2 && INTEL_INFO(dev)->gen < 8 && + crtc->config.has_drrs) { + I915_WRITE(PIPE_DATA_M2(transcoder), + TU_SIZE(m2_n2->tu) | m2_n2->gmch_m); + I915_WRITE(PIPE_DATA_N2(transcoder), m2_n2->gmch_n); + I915_WRITE(PIPE_LINK_M2(transcoder), m2_n2->link_m); + I915_WRITE(PIPE_LINK_N2(transcoder), m2_n2->link_n); + } } else { I915_WRITE(PIPE_DATA_M_G4X(pipe), TU_SIZE(m_n->tu) | m_n->gmch_m); I915_WRITE(PIPE_DATA_N_G4X(pipe), m_n->gmch_n); @@ -5539,12 +5681,13 @@ static void intel_cpu_transcoder_set_m_n(struct intel_crtc *crtc, } } -static void intel_dp_set_m_n(struct intel_crtc *crtc) +void intel_dp_set_m_n(struct intel_crtc *crtc) { if (crtc->config.has_pch_encoder) intel_pch_transcoder_set_m_n(crtc, &crtc->config.dp_m_n); else - intel_cpu_transcoder_set_m_n(crtc, &crtc->config.dp_m_n); + intel_cpu_transcoder_set_m_n(crtc, &crtc->config.dp_m_n, + &crtc->config.dp_m2_n2); } static void vlv_update_pll(struct intel_crtc *crtc) @@ -5662,6 +5805,18 @@ static void vlv_prepare_pll(struct intel_crtc *crtc) static void chv_update_pll(struct intel_crtc *crtc) { + crtc->config.dpll_hw_state.dpll = DPLL_SSC_REF_CLOCK_CHV | + DPLL_REFA_CLK_ENABLE_VLV | DPLL_VGA_MODE_DIS | + DPLL_VCO_ENABLE; + if (crtc->pipe != PIPE_A) + crtc->config.dpll_hw_state.dpll |= DPLL_INTEGRATED_CRI_CLK_VLV; + + crtc->config.dpll_hw_state.dpll_md = + (crtc->config.pixel_multiplier - 1) << DPLL_MD_UDI_MULTIPLIER_SHIFT; +} + +static void chv_prepare_pll(struct intel_crtc *crtc) +{ struct drm_device *dev = crtc->base.dev; struct drm_i915_private *dev_priv = dev->dev_private; int pipe = crtc->pipe; @@ -5671,15 +5826,6 @@ static void chv_update_pll(struct intel_crtc *crtc) u32 bestn, bestm1, bestm2, bestp1, bestp2, bestm2_frac; int refclk; - crtc->config.dpll_hw_state.dpll = DPLL_SSC_REF_CLOCK_CHV | - DPLL_REFA_CLK_ENABLE_VLV | DPLL_VGA_MODE_DIS | - DPLL_VCO_ENABLE; - if (pipe != PIPE_A) - crtc->config.dpll_hw_state.dpll |= DPLL_INTEGRATED_CRI_CLK_VLV; - - crtc->config.dpll_hw_state.dpll_md = - (crtc->config.pixel_multiplier - 1) << DPLL_MD_UDI_MULTIPLIER_SHIFT; - bestn = crtc->config.dpll.n; bestm2_frac = crtc->config.dpll.m2 & 0x3fffff; bestm1 = crtc->config.dpll.m1; @@ -5839,7 +5985,7 @@ static void i8xx_update_pll(struct intel_crtc *crtc, dpll |= PLL_P2_DIVIDE_BY_4; } - if (intel_pipe_has_type(&crtc->base, INTEL_OUTPUT_DVO)) + if (!IS_I830(dev) && intel_pipe_has_type(&crtc->base, INTEL_OUTPUT_DVO)) dpll |= DPLL_DVO_2X_MODE; if (intel_pipe_has_type(&crtc->base, INTEL_OUTPUT_LVDS) && @@ -5990,9 +6136,9 @@ static void i9xx_set_pipeconf(struct intel_crtc *intel_crtc) pipeconf = 0; - if (dev_priv->quirks & QUIRK_PIPEA_FORCE && - I915_READ(PIPECONF(intel_crtc->pipe)) & PIPECONF_ENABLE) - pipeconf |= PIPECONF_ENABLE; + if ((intel_crtc->pipe == PIPE_A && dev_priv->quirks & QUIRK_PIPEA_FORCE) || + (intel_crtc->pipe == PIPE_B && dev_priv->quirks & QUIRK_PIPEB_FORCE)) + pipeconf |= I915_READ(PIPECONF(intel_crtc->pipe)) & PIPECONF_ENABLE; if (intel_crtc->config.double_wide) pipeconf |= PIPECONF_DOUBLE_WIDE; @@ -6235,7 +6381,7 @@ static void i9xx_get_plane_config(struct intel_crtc *crtc, crtc->base.primary->fb->height = ((val >> 0) & 0xfff) + 1; val = I915_READ(DSPSTRIDE(pipe)); - crtc->base.primary->fb->pitches[0] = val & 0xffffff80; + crtc->base.primary->fb->pitches[0] = val & 0xffffffc0; aligned_height = intel_align_height(dev, crtc->base.primary->fb->height, plane_config->tiled); @@ -6345,6 +6491,14 @@ static bool i9xx_get_pipe_config(struct intel_crtc *crtc, } pipe_config->dpll_hw_state.dpll = I915_READ(DPLL(crtc->pipe)); if (!IS_VALLEYVIEW(dev)) { + /* + * DPLL_DVO_2X_MODE must be enabled for both DPLLs + * on 830. Filter it out here so that we don't + * report errors due to that. + */ + if (IS_I830(dev)) + pipe_config->dpll_hw_state.dpll &= ~DPLL_DVO_2X_MODE; + pipe_config->dpll_hw_state.fp0 = I915_READ(FP0(crtc->pipe)); pipe_config->dpll_hw_state.fp1 = I915_READ(FP1(crtc->pipe)); } else { @@ -6367,7 +6521,6 @@ static bool i9xx_get_pipe_config(struct intel_crtc *crtc, static void ironlake_init_pch_refclk(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; - struct drm_mode_config *mode_config = &dev->mode_config; struct intel_encoder *encoder; u32 val, final; bool has_lvds = false; @@ -6377,8 +6530,7 @@ static void ironlake_init_pch_refclk(struct drm_device *dev) bool can_ssc = false; /* We need to take the global config into account */ - list_for_each_entry(encoder, &mode_config->encoder_list, - base.head) { + for_each_intel_encoder(dev, encoder) { switch (encoder->type) { case INTEL_OUTPUT_LVDS: has_panel = true; @@ -6685,11 +6837,10 @@ static void lpt_disable_clkout_dp(struct drm_device *dev) static void lpt_init_pch_refclk(struct drm_device *dev) { - struct drm_mode_config *mode_config = &dev->mode_config; struct intel_encoder *encoder; bool has_vga = false; - list_for_each_entry(encoder, &mode_config->encoder_list, base.head) { + for_each_intel_encoder(dev, encoder) { switch (encoder->type) { case INTEL_OUTPUT_ANALOG: has_vga = true; @@ -7145,7 +7296,8 @@ static void intel_pch_transcoder_get_m_n(struct intel_crtc *crtc, static void intel_cpu_transcoder_get_m_n(struct intel_crtc *crtc, enum transcoder transcoder, - struct intel_link_m_n *m_n) + struct intel_link_m_n *m_n, + struct intel_link_m_n *m2_n2) { struct drm_device *dev = crtc->base.dev; struct drm_i915_private *dev_priv = dev->dev_private; @@ -7159,6 +7311,20 @@ static void intel_cpu_transcoder_get_m_n(struct intel_crtc *crtc, m_n->gmch_n = I915_READ(PIPE_DATA_N1(transcoder)); m_n->tu = ((I915_READ(PIPE_DATA_M1(transcoder)) & TU_SIZE_MASK) >> TU_SIZE_SHIFT) + 1; + /* Read M2_N2 registers only for gen < 8 (M2_N2 available for + * gen < 8) and if DRRS is supported (to make sure the + * registers are not unnecessarily read). + */ + if (m2_n2 && INTEL_INFO(dev)->gen < 8 && + crtc->config.has_drrs) { + m2_n2->link_m = I915_READ(PIPE_LINK_M2(transcoder)); + m2_n2->link_n = I915_READ(PIPE_LINK_N2(transcoder)); + m2_n2->gmch_m = I915_READ(PIPE_DATA_M2(transcoder)) + & ~TU_SIZE_MASK; + m2_n2->gmch_n = I915_READ(PIPE_DATA_N2(transcoder)); + m2_n2->tu = ((I915_READ(PIPE_DATA_M2(transcoder)) + & TU_SIZE_MASK) >> TU_SIZE_SHIFT) + 1; + } } else { m_n->link_m = I915_READ(PIPE_LINK_M_G4X(pipe)); m_n->link_n = I915_READ(PIPE_LINK_N_G4X(pipe)); @@ -7177,14 +7343,15 @@ void intel_dp_get_m_n(struct intel_crtc *crtc, intel_pch_transcoder_get_m_n(crtc, &pipe_config->dp_m_n); else intel_cpu_transcoder_get_m_n(crtc, pipe_config->cpu_transcoder, - &pipe_config->dp_m_n); + &pipe_config->dp_m_n, + &pipe_config->dp_m2_n2); } static void ironlake_get_fdi_m_n_config(struct intel_crtc *crtc, struct intel_crtc_config *pipe_config) { intel_cpu_transcoder_get_m_n(crtc, pipe_config->cpu_transcoder, - &pipe_config->fdi_m_n); + &pipe_config->fdi_m_n, NULL); } static void ironlake_get_pfit_config(struct intel_crtc *crtc, @@ -7255,7 +7422,7 @@ static void ironlake_get_plane_config(struct intel_crtc *crtc, crtc->base.primary->fb->height = ((val >> 0) & 0xfff) + 1; val = I915_READ(DSPSTRIDE(pipe)); - crtc->base.primary->fb->pitches[0] = val & 0xffffff80; + crtc->base.primary->fb->pitches[0] = val & 0xffffffc0; aligned_height = intel_align_height(dev, crtc->base.primary->fb->height, plane_config->tiled); @@ -7472,7 +7639,6 @@ static void hsw_disable_lcpll(struct drm_i915_private *dev_priv, static void hsw_restore_lcpll(struct drm_i915_private *dev_priv) { uint32_t val; - unsigned long irqflags; val = I915_READ(LCPLL_CTL); @@ -7492,10 +7658,10 @@ static void hsw_restore_lcpll(struct drm_i915_private *dev_priv) * to call special forcewake code that doesn't touch runtime PM and * doesn't enable the forcewake delayed work. */ - spin_lock_irqsave(&dev_priv->uncore.lock, irqflags); + spin_lock_irq(&dev_priv->uncore.lock); if (dev_priv->uncore.forcewake_count++ == 0) dev_priv->uncore.funcs.force_wake_get(dev_priv, FORCEWAKE_ALL); - spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); + spin_unlock_irq(&dev_priv->uncore.lock); if (val & LCPLL_POWER_DOWN_ALLOW) { val &= ~LCPLL_POWER_DOWN_ALLOW; @@ -7526,10 +7692,10 @@ static void hsw_restore_lcpll(struct drm_i915_private *dev_priv) } /* See the big comment above. */ - spin_lock_irqsave(&dev_priv->uncore.lock, irqflags); + spin_lock_irq(&dev_priv->uncore.lock); if (--dev_priv->uncore.forcewake_count == 0) dev_priv->uncore.funcs.force_wake_put(dev_priv, FORCEWAKE_ALL); - spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); + spin_unlock_irq(&dev_priv->uncore.lock); } /* @@ -7615,6 +7781,22 @@ static int haswell_crtc_mode_set(struct drm_crtc *crtc, return 0; } +static void haswell_get_ddi_pll(struct drm_i915_private *dev_priv, + enum port port, + struct intel_crtc_config *pipe_config) +{ + pipe_config->ddi_pll_sel = I915_READ(PORT_CLK_SEL(port)); + + switch (pipe_config->ddi_pll_sel) { + case PORT_CLK_SEL_WRPLL1: + pipe_config->shared_dpll = DPLL_ID_WRPLL1; + break; + case PORT_CLK_SEL_WRPLL2: + pipe_config->shared_dpll = DPLL_ID_WRPLL2; + break; + } +} + static void haswell_get_ddi_port_state(struct intel_crtc *crtc, struct intel_crtc_config *pipe_config) { @@ -7628,16 +7810,7 @@ static void haswell_get_ddi_port_state(struct intel_crtc *crtc, port = (tmp & TRANS_DDI_PORT_MASK) >> TRANS_DDI_PORT_SHIFT; - pipe_config->ddi_pll_sel = I915_READ(PORT_CLK_SEL(port)); - - switch (pipe_config->ddi_pll_sel) { - case PORT_CLK_SEL_WRPLL1: - pipe_config->shared_dpll = DPLL_ID_WRPLL1; - break; - case PORT_CLK_SEL_WRPLL2: - pipe_config->shared_dpll = DPLL_ID_WRPLL2; - break; - } + haswell_get_ddi_pll(dev_priv, port, pipe_config); if (pipe_config->shared_dpll >= 0) { pll = &dev_priv->shared_dplls[pipe_config->shared_dpll]; @@ -8037,28 +8210,56 @@ static void i845_update_cursor(struct drm_crtc *crtc, u32 base) struct drm_device *dev = crtc->dev; struct drm_i915_private *dev_priv = dev->dev_private; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); - uint32_t cntl; + uint32_t cntl = 0, size = 0; - if (base != intel_crtc->cursor_base) { - /* On these chipsets we can only modify the base whilst - * the cursor is disabled. - */ - if (intel_crtc->cursor_cntl) { - I915_WRITE(_CURACNTR, 0); - POSTING_READ(_CURACNTR); - intel_crtc->cursor_cntl = 0; + if (base) { + unsigned int width = intel_crtc->cursor_width; + unsigned int height = intel_crtc->cursor_height; + unsigned int stride = roundup_pow_of_two(width) * 4; + + switch (stride) { + default: + WARN_ONCE(1, "Invalid cursor width/stride, width=%u, stride=%u\n", + width, stride); + stride = 256; + /* fallthrough */ + case 256: + case 512: + case 1024: + case 2048: + break; } + cntl |= CURSOR_ENABLE | + CURSOR_GAMMA_ENABLE | + CURSOR_FORMAT_ARGB | + CURSOR_STRIDE(stride); + + size = (height << 12) | width; + } + + if (intel_crtc->cursor_cntl != 0 && + (intel_crtc->cursor_base != base || + intel_crtc->cursor_size != size || + intel_crtc->cursor_cntl != cntl)) { + /* On these chipsets we can only modify the base/size/stride + * whilst the cursor is disabled. + */ + I915_WRITE(_CURACNTR, 0); + POSTING_READ(_CURACNTR); + intel_crtc->cursor_cntl = 0; + } + + if (intel_crtc->cursor_base != base) { I915_WRITE(_CURABASE, base); - POSTING_READ(_CURABASE); + intel_crtc->cursor_base = base; + } + + if (intel_crtc->cursor_size != size) { + I915_WRITE(CURSIZE, size); + intel_crtc->cursor_size = size; } - /* XXX width must be 64, stride 256 => 0x00 << 28 */ - cntl = 0; - if (base) - cntl = (CURSOR_ENABLE | - CURSOR_GAMMA_ENABLE | - CURSOR_FORMAT_ARGB); if (intel_crtc->cursor_cntl != cntl) { I915_WRITE(_CURACNTR, cntl); POSTING_READ(_CURACNTR); @@ -8092,46 +8293,10 @@ static void i9xx_update_cursor(struct drm_crtc *crtc, u32 base) return; } cntl |= pipe << 28; /* Connect to correct pipe */ - } - if (intel_crtc->cursor_cntl != cntl) { - I915_WRITE(CURCNTR(pipe), cntl); - POSTING_READ(CURCNTR(pipe)); - intel_crtc->cursor_cntl = cntl; - } - - /* and commit changes on next vblank */ - I915_WRITE(CURBASE(pipe), base); - POSTING_READ(CURBASE(pipe)); -} - -static void ivb_update_cursor(struct drm_crtc *crtc, u32 base) -{ - struct drm_device *dev = crtc->dev; - struct drm_i915_private *dev_priv = dev->dev_private; - struct intel_crtc *intel_crtc = to_intel_crtc(crtc); - int pipe = intel_crtc->pipe; - uint32_t cntl; - cntl = 0; - if (base) { - cntl = MCURSOR_GAMMA_ENABLE; - switch (intel_crtc->cursor_width) { - case 64: - cntl |= CURSOR_MODE_64_ARGB_AX; - break; - case 128: - cntl |= CURSOR_MODE_128_ARGB_AX; - break; - case 256: - cntl |= CURSOR_MODE_256_ARGB_AX; - break; - default: - WARN_ON(1); - return; - } + if (IS_HASWELL(dev) || IS_BROADWELL(dev)) + cntl |= CURSOR_PIPE_CSC_ENABLE; } - if (IS_HASWELL(dev) || IS_BROADWELL(dev)) - cntl |= CURSOR_PIPE_CSC_ENABLE; if (intel_crtc->cursor_cntl != cntl) { I915_WRITE(CURCNTR(pipe), cntl); @@ -8142,6 +8307,8 @@ static void ivb_update_cursor(struct drm_crtc *crtc, u32 base) /* and commit changes on next vblank */ I915_WRITE(CURBASE(pipe), base); POSTING_READ(CURBASE(pipe)); + + intel_crtc->cursor_base = base; } /* If no-part of the cursor is visible on the framebuffer, then the GPU may hang... */ @@ -8188,13 +8355,47 @@ static void intel_crtc_update_cursor(struct drm_crtc *crtc, I915_WRITE(CURPOS(pipe), pos); - if (IS_IVYBRIDGE(dev) || IS_HASWELL(dev) || IS_BROADWELL(dev)) - ivb_update_cursor(crtc, base); - else if (IS_845G(dev) || IS_I865G(dev)) + if (IS_845G(dev) || IS_I865G(dev)) i845_update_cursor(crtc, base); else i9xx_update_cursor(crtc, base); - intel_crtc->cursor_base = base; +} + +static bool cursor_size_ok(struct drm_device *dev, + uint32_t width, uint32_t height) +{ + if (width == 0 || height == 0) + return false; + + /* + * 845g/865g are special in that they are only limited by + * the width of their cursors, the height is arbitrary up to + * the precision of the register. Everything else requires + * square cursors, limited to a few power-of-two sizes. + */ + if (IS_845G(dev) || IS_I865G(dev)) { + if ((width & 63) != 0) + return false; + + if (width > (IS_845G(dev) ? 64 : 512)) + return false; + + if (height > 1023) + return false; + } else { + switch (width | height) { + case 256: + case 128: + if (IS_GEN2(dev)) + return false; + case 64: + break; + default: + return false; + } + } + + return true; } /* @@ -8209,10 +8410,10 @@ static int intel_crtc_cursor_set_obj(struct drm_crtc *crtc, uint32_t width, uint32_t height) { struct drm_device *dev = crtc->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_crtc *intel_crtc = to_intel_crtc(crtc); enum pipe pipe = intel_crtc->pipe; - unsigned old_width; + unsigned old_width, stride; uint32_t addr; int ret; @@ -8220,20 +8421,18 @@ static int intel_crtc_cursor_set_obj(struct drm_crtc *crtc, if (!obj) { DRM_DEBUG_KMS("cursor off\n"); addr = 0; - obj = NULL; mutex_lock(&dev->struct_mutex); goto finish; } /* Check for which cursor types we support */ - if (!((width == 64 && height == 64) || - (width == 128 && height == 128 && !IS_GEN2(dev)) || - (width == 256 && height == 256 && !IS_GEN2(dev)))) { + if (!cursor_size_ok(dev, width, height)) { DRM_DEBUG("Cursor dimension not supported\n"); return -EINVAL; } - if (obj->base.size < width * height * 4) { + stride = roundup_pow_of_two(width) * 4; + if (obj->base.size < stride * height) { DRM_DEBUG_KMS("buffer is too small\n"); ret = -ENOMEM; goto fail; @@ -8295,9 +8494,6 @@ static int intel_crtc_cursor_set_obj(struct drm_crtc *crtc, addr = obj->phys_handle->busaddr; } - if (IS_GEN2(dev)) - I915_WRITE(CURSIZE, (height << 12) | width); - finish: if (intel_crtc->cursor_bo) { if (!INTEL_INFO(dev)->cursor_needs_physical) @@ -8942,36 +9138,37 @@ out: */ static void intel_mark_fb_busy(struct drm_device *dev, unsigned frontbuffer_bits, - struct intel_engine_cs *ring) + struct i915_gem_request *rq) { + struct drm_i915_private *dev_priv = dev->dev_private; enum pipe pipe; if (!i915.powersave) return; - for_each_pipe(pipe) { + for_each_pipe(dev_priv, pipe) { if (!(frontbuffer_bits & INTEL_FRONTBUFFER_ALL_MASK(pipe))) continue; intel_increase_pllclock(dev, pipe); - if (ring && intel_fbc_enabled(dev)) - ring->fbc_dirty = true; + if (rq && intel_fbc_enabled(dev)) + rq->pending_flush |= I915_KICK_FBC; } } /** * intel_fb_obj_invalidate - invalidate frontbuffer object * @obj: GEM object to invalidate - * @ring: set for asynchronous rendering + * @rq: set for asynchronous rendering * * This function gets called every time rendering on the given object starts and * frontbuffer caching (fbc, low refresh rate for DRRS, panel self refresh) must - * be invalidated. If @ring is non-NULL any subsequent invalidation will be delayed + * be invalidated. If @engine is non-NULL any subsequent invalidation will be delayed * until the rendering completes or a flip on this frontbuffer plane is * scheduled. */ void intel_fb_obj_invalidate(struct drm_i915_gem_object *obj, - struct intel_engine_cs *ring) + struct i915_gem_request *rq) { struct drm_device *dev = obj->base.dev; struct drm_i915_private *dev_priv = dev->dev_private; @@ -8981,7 +9178,7 @@ void intel_fb_obj_invalidate(struct drm_i915_gem_object *obj, if (!obj->frontbuffer_bits) return; - if (ring) { + if (rq) { mutex_lock(&dev_priv->fb_tracking.lock); dev_priv->fb_tracking.busy_bits |= obj->frontbuffer_bits; @@ -8990,7 +9187,7 @@ void intel_fb_obj_invalidate(struct drm_i915_gem_object *obj, mutex_unlock(&dev_priv->fb_tracking.lock); } - intel_mark_fb_busy(dev, obj->frontbuffer_bits, ring); + intel_mark_fb_busy(dev, obj->frontbuffer_bits, rq); intel_edp_psr_invalidate(dev, obj->frontbuffer_bits); } @@ -9019,6 +9216,14 @@ void intel_frontbuffer_flush(struct drm_device *dev, intel_mark_fb_busy(dev, frontbuffer_bits, NULL); intel_edp_psr_flush(dev, frontbuffer_bits); + + /* + * FIXME: Unconditional fbc flushing here is a rather gross hack and + * needs to be reworked into a proper frontbuffer tracking scheme like + * psr employs. + */ + if (IS_BROADWELL(dev)) + gen8_fbc_sw_flush(dev, FBC_REND_CACHE_CLEAN); } /** @@ -9108,12 +9313,11 @@ static void intel_crtc_destroy(struct drm_crtc *crtc) struct intel_crtc *intel_crtc = to_intel_crtc(crtc); struct drm_device *dev = crtc->dev; struct intel_unpin_work *work; - unsigned long flags; - spin_lock_irqsave(&dev->event_lock, flags); + spin_lock_irq(&dev->event_lock); work = intel_crtc->unpin_work; intel_crtc->unpin_work = NULL; - spin_unlock_irqrestore(&dev->event_lock, flags); + spin_unlock_irq(&dev->event_lock); if (work) { cancel_work_sync(&work->work); @@ -9136,6 +9340,7 @@ static void intel_unpin_work_fn(struct work_struct *__work) intel_unpin_fb_obj(work->old_fb_obj); drm_gem_object_unreference(&work->pending_flip_obj->base); drm_gem_object_unreference(&work->old_fb_obj->base); + i915_request_put(work->flip_queued_request); intel_update_fbc(dev); mutex_unlock(&dev->struct_mutex); @@ -9151,7 +9356,6 @@ static void intel_unpin_work_fn(struct work_struct *__work) static void do_intel_finish_page_flip(struct drm_device *dev, struct drm_crtc *crtc) { - struct drm_i915_private *dev_priv = dev->dev_private; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); struct intel_unpin_work *work; unsigned long flags; @@ -9160,6 +9364,10 @@ static void do_intel_finish_page_flip(struct drm_device *dev, if (intel_crtc == NULL) return; + /* + * This is called both by irq handlers and the reset code (to complete + * lost pageflips) so needs the full irqsave spinlocks. + */ spin_lock_irqsave(&dev->event_lock, flags); work = intel_crtc->unpin_work; @@ -9171,23 +9379,9 @@ static void do_intel_finish_page_flip(struct drm_device *dev, return; } - /* and that the unpin work is consistent wrt ->pending. */ - smp_rmb(); - - intel_crtc->unpin_work = NULL; - - if (work->event) - drm_send_vblank_event(dev, intel_crtc->pipe, work->event); - - drm_crtc_vblank_put(crtc); + page_flip_completed(intel_crtc); spin_unlock_irqrestore(&dev->event_lock, flags); - - wake_up_all(&dev_priv->pending_flip_queue); - - queue_work(dev_priv->wq, &work->work); - - trace_i915_flip_complete(intel_crtc->plane, work->pending_flip_obj); } void intel_finish_page_flip(struct drm_device *dev, int pipe) @@ -9255,7 +9449,12 @@ void intel_prepare_page_flip(struct drm_device *dev, int plane) to_intel_crtc(dev_priv->plane_to_crtc_mapping[plane]); unsigned long flags; - /* NB: An MMIO update of the plane base pointer will also + + /* + * This is called both by irq handlers and the reset code (to complete + * lost pageflips) so needs the full irqsave spinlocks. + * + * NB: An MMIO update of the plane base pointer will also * generate a page-flip completion irq, i.e. every modeset * is also accompanied by a spurious intel_prepare_page_flip(). */ @@ -9274,97 +9473,86 @@ static inline void intel_mark_page_flip_active(struct intel_crtc *intel_crtc) smp_wmb(); } -static int intel_gen2_queue_flip(struct drm_device *dev, - struct drm_crtc *crtc, +static int intel_gen2_queue_flip(struct i915_gem_request *rq, + struct intel_crtc *crtc, struct drm_framebuffer *fb, struct drm_i915_gem_object *obj, - struct intel_engine_cs *ring, uint32_t flags) { - struct intel_crtc *intel_crtc = to_intel_crtc(crtc); + struct intel_ringbuffer *ring; u32 flip_mask; - int ret; - ret = intel_ring_begin(ring, 6); - if (ret) - return ret; + ring = intel_ring_begin(rq, 5); + if (IS_ERR(ring)) + return PTR_ERR(ring); /* Can't queue multiple flips, so wait for the previous * one to finish before executing the next. */ - if (intel_crtc->plane) + if (crtc->plane) flip_mask = MI_WAIT_FOR_PLANE_B_FLIP; else flip_mask = MI_WAIT_FOR_PLANE_A_FLIP; intel_ring_emit(ring, MI_WAIT_FOR_EVENT | flip_mask); - intel_ring_emit(ring, MI_NOOP); intel_ring_emit(ring, MI_DISPLAY_FLIP | - MI_DISPLAY_FLIP_PLANE(intel_crtc->plane)); + MI_DISPLAY_FLIP_PLANE(crtc->plane)); intel_ring_emit(ring, fb->pitches[0]); - intel_ring_emit(ring, intel_crtc->unpin_work->gtt_offset); + intel_ring_emit(ring, crtc->unpin_work->gtt_offset); intel_ring_emit(ring, 0); /* aux display base address, unused */ + intel_ring_advance(ring); - intel_mark_page_flip_active(intel_crtc); - __intel_ring_advance(ring); return 0; } -static int intel_gen3_queue_flip(struct drm_device *dev, - struct drm_crtc *crtc, +static int intel_gen3_queue_flip(struct i915_gem_request *rq, + struct intel_crtc *crtc, struct drm_framebuffer *fb, struct drm_i915_gem_object *obj, - struct intel_engine_cs *ring, uint32_t flags) { - struct intel_crtc *intel_crtc = to_intel_crtc(crtc); + struct intel_ringbuffer *ring; u32 flip_mask; - int ret; - ret = intel_ring_begin(ring, 6); - if (ret) - return ret; + ring = intel_ring_begin(rq, 4); + if (IS_ERR(ring)) + return PTR_ERR(ring); - if (intel_crtc->plane) + if (crtc->plane) flip_mask = MI_WAIT_FOR_PLANE_B_FLIP; else flip_mask = MI_WAIT_FOR_PLANE_A_FLIP; intel_ring_emit(ring, MI_WAIT_FOR_EVENT | flip_mask); - intel_ring_emit(ring, MI_NOOP); intel_ring_emit(ring, MI_DISPLAY_FLIP_I915 | - MI_DISPLAY_FLIP_PLANE(intel_crtc->plane)); + MI_DISPLAY_FLIP_PLANE(crtc->plane)); intel_ring_emit(ring, fb->pitches[0]); - intel_ring_emit(ring, intel_crtc->unpin_work->gtt_offset); - intel_ring_emit(ring, MI_NOOP); + intel_ring_emit(ring, crtc->unpin_work->gtt_offset); + intel_ring_advance(ring); - intel_mark_page_flip_active(intel_crtc); - __intel_ring_advance(ring); return 0; } -static int intel_gen4_queue_flip(struct drm_device *dev, - struct drm_crtc *crtc, +static int intel_gen4_queue_flip(struct i915_gem_request *rq, + struct intel_crtc *crtc, struct drm_framebuffer *fb, struct drm_i915_gem_object *obj, - struct intel_engine_cs *ring, uint32_t flags) { - struct drm_i915_private *dev_priv = dev->dev_private; - struct intel_crtc *intel_crtc = to_intel_crtc(crtc); + struct drm_i915_private *dev_priv = rq->i915; + struct intel_ringbuffer *ring; uint32_t pf, pipesrc; - int ret; - ret = intel_ring_begin(ring, 4); - if (ret) - return ret; + ring = intel_ring_begin(rq, 4); + if (IS_ERR(ring)) + return PTR_ERR(ring); /* i965+ uses the linear or tiled offsets from the * Display Registers (which do not change across a page-flip) * so we need only reprogram the base address. */ intel_ring_emit(ring, MI_DISPLAY_FLIP | - MI_DISPLAY_FLIP_PLANE(intel_crtc->plane)); + MI_DISPLAY_FLIP_PLANE(crtc->plane)); intel_ring_emit(ring, fb->pitches[0]); - intel_ring_emit(ring, intel_crtc->unpin_work->gtt_offset | + intel_ring_emit(ring, crtc->unpin_work->gtt_offset | obj->tiling_mode); /* XXX Enabling the panel-fitter across page-flip is so far @@ -9372,62 +9560,57 @@ static int intel_gen4_queue_flip(struct drm_device *dev, * pf = I915_READ(pipe == 0 ? PFA_CTL_1 : PFB_CTL_1) & PF_ENABLE; */ pf = 0; - pipesrc = I915_READ(PIPESRC(intel_crtc->pipe)) & 0x0fff0fff; + pipesrc = I915_READ(PIPESRC(crtc->pipe)) & 0x0fff0fff; intel_ring_emit(ring, pf | pipesrc); + intel_ring_advance(ring); - intel_mark_page_flip_active(intel_crtc); - __intel_ring_advance(ring); return 0; } -static int intel_gen6_queue_flip(struct drm_device *dev, - struct drm_crtc *crtc, +static int intel_gen6_queue_flip(struct i915_gem_request *rq, + struct intel_crtc *crtc, struct drm_framebuffer *fb, struct drm_i915_gem_object *obj, - struct intel_engine_cs *ring, uint32_t flags) { - struct drm_i915_private *dev_priv = dev->dev_private; - struct intel_crtc *intel_crtc = to_intel_crtc(crtc); + struct drm_i915_private *dev_priv = rq->i915; + struct intel_ringbuffer *ring; uint32_t pf, pipesrc; - int ret; - ret = intel_ring_begin(ring, 4); - if (ret) - return ret; + ring = intel_ring_begin(rq, 4); + if (IS_ERR(ring)) + return PTR_ERR(ring); intel_ring_emit(ring, MI_DISPLAY_FLIP | - MI_DISPLAY_FLIP_PLANE(intel_crtc->plane)); + MI_DISPLAY_FLIP_PLANE(crtc->plane)); intel_ring_emit(ring, fb->pitches[0] | obj->tiling_mode); - intel_ring_emit(ring, intel_crtc->unpin_work->gtt_offset); + intel_ring_emit(ring, crtc->unpin_work->gtt_offset); /* Contrary to the suggestions in the documentation, * "Enable Panel Fitter" does not seem to be required when page * flipping with a non-native mode, and worse causes a normal * modeset to fail. - * pf = I915_READ(PF_CTL(intel_crtc->pipe)) & PF_ENABLE; + * pf = I915_READ(PF_CTL(crtc->pipe)) & PF_ENABLE; */ pf = 0; - pipesrc = I915_READ(PIPESRC(intel_crtc->pipe)) & 0x0fff0fff; + pipesrc = I915_READ(PIPESRC(crtc->pipe)) & 0x0fff0fff; intel_ring_emit(ring, pf | pipesrc); + intel_ring_advance(ring); - intel_mark_page_flip_active(intel_crtc); - __intel_ring_advance(ring); return 0; } -static int intel_gen7_queue_flip(struct drm_device *dev, - struct drm_crtc *crtc, +static int intel_gen7_queue_flip(struct i915_gem_request *rq, + struct intel_crtc *crtc, struct drm_framebuffer *fb, struct drm_i915_gem_object *obj, - struct intel_engine_cs *ring, uint32_t flags) { - struct intel_crtc *intel_crtc = to_intel_crtc(crtc); + struct intel_ringbuffer *ring; uint32_t plane_bit = 0; int len, ret; - switch (intel_crtc->plane) { + switch (crtc->plane) { case PLANE_A: plane_bit = MI_DISPLAY_FLIP_IVB_PLANE_A; break; @@ -9442,16 +9625,16 @@ static int intel_gen7_queue_flip(struct drm_device *dev, return -ENODEV; } - len = 4; - if (ring->id == RCS) { + len = 3; + if (rq->engine->id == RCS) { len += 6; /* * On Gen 8, SRM is now taking an extra dword to accommodate * 48bits addresses, and we need a NOOP for the batch size to * stay even. */ - if (IS_GEN8(dev)) - len += 2; + if (IS_GEN8(rq->i915)) + len += 1; } /* @@ -9464,13 +9647,13 @@ static int intel_gen7_queue_flip(struct drm_device *dev, * then do the cacheline alignment, and finally emit the * MI_DISPLAY_FLIP. */ - ret = intel_ring_cacheline_align(ring); + ret = intel_ring_cacheline_align(rq); if (ret) return ret; - ret = intel_ring_begin(ring, len); - if (ret) - return ret; + ring = intel_ring_begin(rq, len); + if (IS_ERR(ring)) + return PTR_ERR(ring); /* Unmask the flip-done completion message. Note that the bspec says that * we should do this for both the BCS and RCS, and that we must not unmask @@ -9481,37 +9664,33 @@ static int intel_gen7_queue_flip(struct drm_device *dev, * for the RCS also doesn't appear to drop events. Setting the DERRMR * to zero does lead to lockups within MI_DISPLAY_FLIP. */ - if (ring->id == RCS) { + if (rq->engine->id == RCS) { intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1)); intel_ring_emit(ring, DERRMR); intel_ring_emit(ring, ~(DERRMR_PIPEA_PRI_FLIP_DONE | DERRMR_PIPEB_PRI_FLIP_DONE | DERRMR_PIPEC_PRI_FLIP_DONE)); - if (IS_GEN8(dev)) + if (IS_GEN8(rq->i915)) intel_ring_emit(ring, MI_STORE_REGISTER_MEM_GEN8(1) | MI_SRM_LRM_GLOBAL_GTT); else intel_ring_emit(ring, MI_STORE_REGISTER_MEM(1) | MI_SRM_LRM_GLOBAL_GTT); intel_ring_emit(ring, DERRMR); - intel_ring_emit(ring, ring->scratch.gtt_offset + 256); - if (IS_GEN8(dev)) { + intel_ring_emit(ring, rq->engine->scratch.gtt_offset + 256); + if (IS_GEN8(rq->i915)) intel_ring_emit(ring, 0); - intel_ring_emit(ring, MI_NOOP); - } } intel_ring_emit(ring, MI_DISPLAY_FLIP_I915 | plane_bit); - intel_ring_emit(ring, (fb->pitches[0] | obj->tiling_mode)); - intel_ring_emit(ring, intel_crtc->unpin_work->gtt_offset); - intel_ring_emit(ring, (MI_NOOP)); + intel_ring_emit(ring, fb->pitches[0] | obj->tiling_mode); + intel_ring_emit(ring, crtc->unpin_work->gtt_offset); + intel_ring_advance(ring); - intel_mark_page_flip_active(intel_crtc); - __intel_ring_advance(ring); return 0; } -static bool use_mmio_flip(struct intel_engine_cs *ring, +static bool use_mmio_flip(struct intel_engine_cs *engine, struct drm_i915_gem_object *obj) { /* @@ -9522,10 +9701,10 @@ static bool use_mmio_flip(struct intel_engine_cs *ring, * So using MMIO flips there would disrupt this mechanism. */ - if (ring == NULL) + if (engine == NULL) return true; - if (INTEL_INFO(ring->dev)->gen < 5) + if (INTEL_INFO(engine->i915)->gen < 5) return false; if (i915.use_mmio_flip < 0) @@ -9533,7 +9712,7 @@ static bool use_mmio_flip(struct intel_engine_cs *ring, else if (i915.use_mmio_flip > 0) return true; else - return ring != obj->ring; + return engine != i915_request_engine(obj->last_write.request); } static void intel_do_mmio_flip(struct intel_crtc *intel_crtc) @@ -9564,105 +9743,124 @@ static void intel_do_mmio_flip(struct intel_crtc *intel_crtc) POSTING_READ(DSPSURF(intel_crtc->plane)); } -static int intel_postpone_flip(struct drm_i915_gem_object *obj) -{ - struct intel_engine_cs *ring; - int ret; +struct flip_work { + struct work_struct work; + struct i915_gem_request *rq; + struct intel_crtc *crtc; +}; - lockdep_assert_held(&obj->base.dev->struct_mutex); +static void intel_mmio_flip_work(struct work_struct *work) +{ + struct flip_work *flip = container_of(work, struct flip_work, work); - if (!obj->last_write_seqno) - return 0; + if (__i915_request_wait(flip->rq, false, NULL, NULL) == 0) + intel_do_mmio_flip(flip->crtc); - ring = obj->ring; + i915_request_put__unlocked(flip->rq); + kfree(flip); +} - if (i915_seqno_passed(ring->get_seqno(ring, true), - obj->last_write_seqno)) - return 0; +static int intel_queue_mmio_flip(struct intel_crtc *crtc, + struct i915_gem_request *rq) +{ + struct flip_work *flip; - ret = i915_gem_check_olr(ring, obj->last_write_seqno); - if (ret) - return ret; + if (WARN_ON(crtc->mmio_flip)) + return -EBUSY; - if (WARN_ON(!ring->irq_get(ring))) + if (rq == NULL) { + intel_do_mmio_flip(crtc); return 0; + } - return 1; -} - -void intel_notify_mmio_flip(struct intel_engine_cs *ring) -{ - struct drm_i915_private *dev_priv = to_i915(ring->dev); - struct intel_crtc *intel_crtc; - unsigned long irq_flags; - u32 seqno; - - seqno = ring->get_seqno(ring, false); + if (i915_request_complete(rq)) { + intel_do_mmio_flip(crtc); + return 0; + } - spin_lock_irqsave(&dev_priv->mmio_flip_lock, irq_flags); - for_each_intel_crtc(ring->dev, intel_crtc) { - struct intel_mmio_flip *mmio_flip; + flip = kmalloc(sizeof(*flip), GFP_KERNEL); + if (flip == NULL) + return -ENOMEM; - mmio_flip = &intel_crtc->mmio_flip; - if (mmio_flip->seqno == 0) - continue; + INIT_WORK(&flip->work, intel_mmio_flip_work); + flip->crtc = crtc; + flip->rq = i915_request_get_breadcrumb(rq); + if (IS_ERR(flip->rq)) { + int ret = PTR_ERR(flip->rq); + kfree(flip); + return ret; + } - if (ring->id != mmio_flip->ring_id) - continue; + schedule_work(&flip->work); + return 0; +} - if (i915_seqno_passed(seqno, mmio_flip->seqno)) { - intel_do_mmio_flip(intel_crtc); - mmio_flip->seqno = 0; - ring->irq_put(ring); - } - } - spin_unlock_irqrestore(&dev_priv->mmio_flip_lock, irq_flags); +static int intel_default_queue_flip(struct i915_gem_request *rq, + struct intel_crtc *crtc, + struct drm_framebuffer *fb, + struct drm_i915_gem_object *obj, + uint32_t flags) +{ + return -ENODEV; } -static int intel_queue_mmio_flip(struct drm_device *dev, - struct drm_crtc *crtc, - struct drm_framebuffer *fb, - struct drm_i915_gem_object *obj, - struct intel_engine_cs *ring, - uint32_t flags) +static bool __intel_pageflip_stall_check(struct drm_device *dev, + struct drm_crtc *crtc) { struct drm_i915_private *dev_priv = dev->dev_private; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); - unsigned long irq_flags; - int ret; + struct intel_unpin_work *work = intel_crtc->unpin_work; + u32 addr; - if (WARN_ON(intel_crtc->mmio_flip.seqno)) - return -EBUSY; + if (atomic_read(&work->pending) >= INTEL_FLIP_COMPLETE) + return true; - ret = intel_postpone_flip(obj); - if (ret < 0) - return ret; - if (ret == 0) { - intel_do_mmio_flip(intel_crtc); - return 0; + if (!work->enable_stall_check) + return false; + + if (work->flip_ready_vblank == 0) { + if (work->flip_queued_request && + !i915_request_complete(work->flip_queued_request)) + return false; + + work->flip_ready_vblank = drm_vblank_count(dev, intel_crtc->pipe); } - spin_lock_irqsave(&dev_priv->mmio_flip_lock, irq_flags); - intel_crtc->mmio_flip.seqno = obj->last_write_seqno; - intel_crtc->mmio_flip.ring_id = obj->ring->id; - spin_unlock_irqrestore(&dev_priv->mmio_flip_lock, irq_flags); + if (drm_vblank_count(dev, intel_crtc->pipe) - work->flip_ready_vblank < 3) + return false; - /* - * Double check to catch cases where irq fired before - * mmio flip data was ready + /* Potential stall - if we see that the flip has happened, + * assume a missed interrupt. */ + if (INTEL_INFO(dev)->gen >= 4) + addr = I915_HI_DISPBASE(I915_READ(DSPSURF(intel_crtc->plane))); + else + addr = I915_READ(DSPADDR(intel_crtc->plane)); + + /* There is a potential issue here with a false positive after a flip + * to the same address. We could address this by checking for a + * non-incrementing frame counter. */ - intel_notify_mmio_flip(obj->ring); - return 0; + return addr == work->gtt_offset; } -static int intel_default_queue_flip(struct drm_device *dev, - struct drm_crtc *crtc, - struct drm_framebuffer *fb, - struct drm_i915_gem_object *obj, - struct intel_engine_cs *ring, - uint32_t flags) +void intel_check_page_flip(struct drm_device *dev, int pipe) { - return -ENODEV; + struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe]; + struct intel_crtc *intel_crtc = to_intel_crtc(crtc); + + WARN_ON(!in_irq()); + + if (crtc == NULL) + return; + + spin_lock(&dev->event_lock); + if (intel_crtc->unpin_work && __intel_pageflip_stall_check(dev, crtc)) { + WARN_ONCE(1, "Kicking stuck page flip: queued at %d, now %d\n", + intel_crtc->unpin_work->flip_queued_vblank, drm_vblank_count(dev, pipe)); + page_flip_completed(intel_crtc); + } + spin_unlock(&dev->event_lock); } static int intel_crtc_page_flip(struct drm_crtc *crtc, @@ -9677,10 +9875,13 @@ static int intel_crtc_page_flip(struct drm_crtc *crtc, struct intel_crtc *intel_crtc = to_intel_crtc(crtc); enum pipe pipe = intel_crtc->pipe; struct intel_unpin_work *work; - struct intel_engine_cs *ring; - unsigned long flags; + struct intel_engine_cs *engine; + struct i915_gem_request *rq; int ret; + //trigger software GT busyness calculation + gen8_flip_interrupt(dev); + /* * drm_mode_page_flip_ioctl() should already catch this, but double * check to be safe. In the future we may enable pageflipping from @@ -9719,17 +9920,25 @@ static int intel_crtc_page_flip(struct drm_crtc *crtc, goto free_work; /* We borrow the event spin lock for protecting unpin_work */ - spin_lock_irqsave(&dev->event_lock, flags); + spin_lock_irq(&dev->event_lock); if (intel_crtc->unpin_work) { - spin_unlock_irqrestore(&dev->event_lock, flags); - kfree(work); - drm_crtc_vblank_put(crtc); + /* Before declaring the flip queue wedged, check if + * the hardware completed the operation behind our backs. + */ + if (__intel_pageflip_stall_check(dev, crtc)) { + DRM_DEBUG_DRIVER("flip queue: previous flip completed, continuing\n"); + page_flip_completed(intel_crtc); + } else { + DRM_DEBUG_DRIVER("flip queue: crtc already busy\n"); + spin_unlock_irq(&dev->event_lock); - DRM_DEBUG_DRIVER("flip queue: crtc already busy\n"); - return -EBUSY; + drm_crtc_vblank_put(crtc); + kfree(work); + return -EBUSY; + } } intel_crtc->unpin_work = work; - spin_unlock_irqrestore(&dev->event_lock, flags); + spin_unlock_irq(&dev->event_lock); if (atomic_read(&intel_crtc->unpin_work_count) >= 2) flush_workqueue(dev_priv->wq); @@ -9746,8 +9955,6 @@ static int intel_crtc_page_flip(struct drm_crtc *crtc, work->pending_flip_obj = obj; - work->enable_stall_check = true; - atomic_inc(&intel_crtc->unpin_work_count); intel_crtc->reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter); @@ -9755,35 +9962,65 @@ static int intel_crtc_page_flip(struct drm_crtc *crtc, work->flip_count = I915_READ(PIPE_FLIPCOUNT_GM45(pipe)) + 1; if (IS_VALLEYVIEW(dev)) { - ring = &dev_priv->ring[BCS]; + engine = &dev_priv->engine[BCS]; if (obj->tiling_mode != work->old_fb_obj->tiling_mode) /* vlv: DISPLAY_FLIP fails to change tiling */ - ring = NULL; + engine = NULL; } else if (IS_IVYBRIDGE(dev)) { - ring = &dev_priv->ring[BCS]; + engine = &dev_priv->engine[BCS]; } else if (INTEL_INFO(dev)->gen >= 7) { - ring = obj->ring; - if (ring == NULL || ring->id != RCS) - ring = &dev_priv->ring[BCS]; + engine = i915_request_engine(obj->last_write.request); + if (engine == NULL || engine->id != RCS) + engine = &dev_priv->engine[BCS]; } else { - ring = &dev_priv->ring[RCS]; + engine = &dev_priv->engine[RCS]; } - ret = intel_pin_and_fence_fb_obj(dev, obj, ring); - if (ret) - goto cleanup_pending; + if (use_mmio_flip(engine, obj)) { + rq = i915_request_get(obj->last_write.request); - work->gtt_offset = - i915_gem_obj_ggtt_offset(obj) + intel_crtc->dspaddr_offset; + ret = intel_pin_and_fence_fb_obj(dev, obj, rq); + if (ret) + goto cleanup_rq; - if (use_mmio_flip(ring, obj)) - ret = intel_queue_mmio_flip(dev, crtc, fb, obj, ring, - page_flip_flags); - else - ret = dev_priv->display.queue_flip(dev, crtc, fb, obj, ring, - page_flip_flags); - if (ret) - goto cleanup_unpin; + work->gtt_offset = + i915_gem_obj_ggtt_offset(obj) + intel_crtc->dspaddr_offset; + + ret = intel_queue_mmio_flip(intel_crtc, rq); + if (ret) + goto cleanup_unpin; + } else { + struct intel_context *ctx = engine->default_context; + if (obj->last_write.request) + ctx = obj->last_write.request->ctx; + rq = intel_engine_alloc_request(engine, ctx); + if (IS_ERR(rq)) { + ret = PTR_ERR(rq); + goto cleanup_pending; + } + + ret = intel_pin_and_fence_fb_obj(dev, obj, rq); + if (ret) + goto cleanup_rq; + + work->gtt_offset = + i915_gem_obj_ggtt_offset(obj) + intel_crtc->dspaddr_offset; + + ret = dev_priv->display.queue_flip(rq, intel_crtc, fb, obj, + page_flip_flags); + if (ret) + goto cleanup_unpin; + + intel_mark_page_flip_active(intel_crtc); + + ret = i915_request_commit(rq); + if (ret) + goto cleanup_unpin; + } + + work->flip_queued_request = rq; + work->flip_queued_vblank = drm_vblank_count(dev, intel_crtc->pipe); + work->enable_stall_check = true; i915_gem_track_fb(work->old_fb_obj, obj, INTEL_FRONTBUFFER_PRIMARY(pipe)); @@ -9798,6 +10035,8 @@ static int intel_crtc_page_flip(struct drm_crtc *crtc, cleanup_unpin: intel_unpin_fb_obj(obj); +cleanup_rq: + i915_request_put(rq); cleanup_pending: atomic_dec(&intel_crtc->unpin_work_count); crtc->primary->fb = old_fb; @@ -9806,9 +10045,9 @@ cleanup_pending: mutex_unlock(&dev->struct_mutex); cleanup: - spin_lock_irqsave(&dev->event_lock, flags); + spin_lock_irq(&dev->event_lock); intel_crtc->unpin_work = NULL; - spin_unlock_irqrestore(&dev->event_lock, flags); + spin_unlock_irq(&dev->event_lock); drm_crtc_vblank_put(crtc); free_work: @@ -9818,8 +10057,11 @@ free_work: out_hang: intel_crtc_wait_for_pending_flips(crtc); ret = intel_pipe_set_base(crtc, crtc->x, crtc->y, fb); - if (ret == 0 && event) + if (ret == 0 && event) { + spin_lock_irq(&dev->event_lock); drm_send_vblank_event(dev, pipe, event); + spin_unlock_irq(&dev->event_lock); + } } return ret; } @@ -9847,8 +10089,7 @@ static void intel_modeset_update_staged_output_state(struct drm_device *dev) to_intel_encoder(connector->base.encoder); } - list_for_each_entry(encoder, &dev->mode_config.encoder_list, - base.head) { + for_each_intel_encoder(dev, encoder) { encoder->new_crtc = to_intel_crtc(encoder->base.crtc); } @@ -9879,8 +10120,7 @@ static void intel_modeset_commit_output_state(struct drm_device *dev) connector->base.encoder = &connector->new_encoder->base; } - list_for_each_entry(encoder, &dev->mode_config.encoder_list, - base.head) { + for_each_intel_encoder(dev, encoder) { encoder->base.crtc = &encoder->new_crtc->base; } @@ -10007,6 +10247,15 @@ static void intel_dump_pipe_config(struct intel_crtc *crtc, pipe_config->dp_m_n.gmch_m, pipe_config->dp_m_n.gmch_n, pipe_config->dp_m_n.link_m, pipe_config->dp_m_n.link_n, pipe_config->dp_m_n.tu); + + DRM_DEBUG_KMS("dp: %i, gmch_m2: %u, gmch_n2: %u, link_m2: %u, link_n2: %u, tu2: %u\n", + pipe_config->has_dp_encoder, + pipe_config->dp_m2_n2.gmch_m, + pipe_config->dp_m2_n2.gmch_n, + pipe_config->dp_m2_n2.link_m, + pipe_config->dp_m2_n2.link_n, + pipe_config->dp_m2_n2.tu); + DRM_DEBUG_KMS("requested mode:\n"); drm_mode_debug_printmodeline(&pipe_config->requested_mode); DRM_DEBUG_KMS("adjusted mode:\n"); @@ -10041,8 +10290,7 @@ static bool check_single_encoder_cloning(struct intel_crtc *crtc, struct drm_device *dev = crtc->base.dev; struct intel_encoder *source_encoder; - list_for_each_entry(source_encoder, - &dev->mode_config.encoder_list, base.head) { + for_each_intel_encoder(dev, source_encoder) { if (source_encoder->new_crtc != crtc) continue; @@ -10058,8 +10306,7 @@ static bool check_encoder_cloning(struct intel_crtc *crtc) struct drm_device *dev = crtc->base.dev; struct intel_encoder *encoder; - list_for_each_entry(encoder, - &dev->mode_config.encoder_list, base.head) { + for_each_intel_encoder(dev, encoder) { if (encoder->new_crtc != crtc) continue; @@ -10143,8 +10390,7 @@ encoder_retry: * adjust it according to limitations or connector properties, and also * a chance to reject the mode entirely. */ - list_for_each_entry(encoder, &dev->mode_config.encoder_list, - base.head) { + for_each_intel_encoder(dev, encoder) { if (&encoder->new_crtc->base != crtc) continue; @@ -10222,8 +10468,7 @@ intel_modeset_affected_pipes(struct drm_crtc *crtc, unsigned *modeset_pipes, 1 << connector->new_encoder->new_crtc->pipe; } - list_for_each_entry(encoder, &dev->mode_config.encoder_list, - base.head) { + for_each_intel_encoder(dev, encoder) { if (encoder->base.crtc == &encoder->new_crtc->base) continue; @@ -10297,8 +10542,7 @@ intel_modeset_update_state(struct drm_device *dev, unsigned prepare_pipes) struct intel_crtc *intel_crtc; struct drm_connector *connector; - list_for_each_entry(intel_encoder, &dev->mode_config.encoder_list, - base.head) { + for_each_intel_encoder(dev, intel_encoder) { if (!intel_encoder->base.crtc) continue; @@ -10387,6 +10631,22 @@ intel_pipe_config_compare(struct drm_device *dev, return false; \ } +/* This is required for BDW+ where there is only one set of registers for + * switching between high and low RR. + * This macro can be used whenever a comparison has to be made between one + * hw state and multiple sw state variables. + */ +#define PIPE_CONF_CHECK_I_ALT(name, alt_name) \ + if ((current_config->name != pipe_config->name) && \ + (current_config->alt_name != pipe_config->name)) { \ + DRM_ERROR("mismatch in " #name " " \ + "(expected %i or %i, found %i)\n", \ + current_config->name, \ + current_config->alt_name, \ + pipe_config->name); \ + return false; \ + } + #define PIPE_CONF_CHECK_FLAGS(name, mask) \ if ((current_config->name ^ pipe_config->name) & (mask)) { \ DRM_ERROR("mismatch in " #name "(" #mask ") " \ @@ -10419,11 +10679,28 @@ intel_pipe_config_compare(struct drm_device *dev, PIPE_CONF_CHECK_I(fdi_m_n.tu); PIPE_CONF_CHECK_I(has_dp_encoder); - PIPE_CONF_CHECK_I(dp_m_n.gmch_m); - PIPE_CONF_CHECK_I(dp_m_n.gmch_n); - PIPE_CONF_CHECK_I(dp_m_n.link_m); - PIPE_CONF_CHECK_I(dp_m_n.link_n); - PIPE_CONF_CHECK_I(dp_m_n.tu); + + if (INTEL_INFO(dev)->gen < 8) { + PIPE_CONF_CHECK_I(dp_m_n.gmch_m); + PIPE_CONF_CHECK_I(dp_m_n.gmch_n); + PIPE_CONF_CHECK_I(dp_m_n.link_m); + PIPE_CONF_CHECK_I(dp_m_n.link_n); + PIPE_CONF_CHECK_I(dp_m_n.tu); + + if (current_config->has_drrs) { + PIPE_CONF_CHECK_I(dp_m2_n2.gmch_m); + PIPE_CONF_CHECK_I(dp_m2_n2.gmch_n); + PIPE_CONF_CHECK_I(dp_m2_n2.link_m); + PIPE_CONF_CHECK_I(dp_m2_n2.link_n); + PIPE_CONF_CHECK_I(dp_m2_n2.tu); + } + } else { + PIPE_CONF_CHECK_I_ALT(dp_m_n.gmch_m, dp_m2_n2.gmch_m); + PIPE_CONF_CHECK_I_ALT(dp_m_n.gmch_n, dp_m2_n2.gmch_n); + PIPE_CONF_CHECK_I_ALT(dp_m_n.link_m, dp_m2_n2.link_m); + PIPE_CONF_CHECK_I_ALT(dp_m_n.link_n, dp_m2_n2.link_n); + PIPE_CONF_CHECK_I_ALT(dp_m_n.tu, dp_m2_n2.tu); + } PIPE_CONF_CHECK_I(adjusted_mode.crtc_hdisplay); PIPE_CONF_CHECK_I(adjusted_mode.crtc_htotal); @@ -10509,6 +10786,7 @@ intel_pipe_config_compare(struct drm_device *dev, #undef PIPE_CONF_CHECK_X #undef PIPE_CONF_CHECK_I +#undef PIPE_CONF_CHECK_I_ALT #undef PIPE_CONF_CHECK_FLAGS #undef PIPE_CONF_CHECK_CLOCK_FUZZY #undef PIPE_CONF_QUIRK @@ -10538,8 +10816,7 @@ check_encoder_state(struct drm_device *dev) struct intel_encoder *encoder; struct intel_connector *connector; - list_for_each_entry(encoder, &dev->mode_config.encoder_list, - base.head) { + for_each_intel_encoder(dev, encoder) { bool enabled = false; bool active = false; enum pipe pipe, tracked_pipe; @@ -10618,8 +10895,7 @@ check_crtc_state(struct drm_device *dev) WARN(crtc->active && !crtc->base.enabled, "active crtc, but not enabled in sw tracking\n"); - list_for_each_entry(encoder, &dev->mode_config.encoder_list, - base.head) { + for_each_intel_encoder(dev, encoder) { if (encoder->base.crtc != &crtc->base) continue; enabled = true; @@ -10637,12 +10913,12 @@ check_crtc_state(struct drm_device *dev) active = dev_priv->display.get_pipe_config(crtc, &pipe_config); - /* hw state is inconsistent with the pipe A quirk */ - if (crtc->pipe == PIPE_A && dev_priv->quirks & QUIRK_PIPEA_FORCE) + /* hw state is inconsistent with the pipe quirk */ + if ((crtc->pipe == PIPE_A && dev_priv->quirks & QUIRK_PIPEA_FORCE) || + (crtc->pipe == PIPE_B && dev_priv->quirks & QUIRK_PIPEB_FORCE)) active = crtc->active; - list_for_each_entry(encoder, &dev->mode_config.encoder_list, - base.head) { + for_each_intel_encoder(dev, encoder) { enum pipe pipe; if (encoder->base.crtc != &crtc->base) continue; @@ -10656,7 +10932,7 @@ check_crtc_state(struct drm_device *dev) if (active && !intel_pipe_config_compare(dev, &crtc->config, &pipe_config)) { - DRM_DEBUG_KMS("pipe state doesn't match!\n"); + WARN(1, "pipe state doesn't match!\n"); intel_dump_pipe_config(crtc, &pipe_config, "[hw state]"); intel_dump_pipe_config(crtc, &crtc->config, @@ -11010,7 +11286,7 @@ static void intel_set_config_restore_state(struct drm_device *dev, } count = 0; - list_for_each_entry(encoder, &dev->mode_config.encoder_list, base.head) { + for_each_intel_encoder(dev, encoder) { encoder->new_crtc = to_intel_crtc(config->save_encoder_crtcs[count++]); } @@ -11169,8 +11445,7 @@ intel_modeset_stage_output_state(struct drm_device *dev, } /* Check for any encoders that needs to be disabled. */ - list_for_each_entry(encoder, &dev->mode_config.encoder_list, - base.head) { + for_each_intel_encoder(dev, encoder) { int num_connectors = 0; list_for_each_entry(connector, &dev->mode_config.connector_list, @@ -11203,9 +11478,7 @@ intel_modeset_stage_output_state(struct drm_device *dev, for_each_intel_crtc(dev, crtc) { crtc->new_enabled = false; - list_for_each_entry(encoder, - &dev->mode_config.encoder_list, - base.head) { + for_each_intel_encoder(dev, encoder) { if (encoder->new_crtc == crtc) { crtc->new_enabled = true; break; @@ -11242,7 +11515,7 @@ static void disable_crtc_nofb(struct intel_crtc *crtc) connector->new_encoder = NULL; } - list_for_each_entry(encoder, &dev->mode_config.encoder_list, base.head) { + for_each_intel_encoder(dev, encoder) { if (encoder->new_crtc == crtc) encoder->new_crtc = NULL; } @@ -11305,7 +11578,6 @@ static int intel_crtc_set_config(struct drm_mode_set *set) ret = intel_set_mode(set->crtc, set->mode, set->x, set->y, set->fb); } else if (config->fb_changed) { - struct drm_i915_private *dev_priv = dev->dev_private; struct intel_crtc *intel_crtc = to_intel_crtc(set->crtc); intel_crtc_wait_for_pending_flips(set->crtc); @@ -11319,8 +11591,7 @@ static int intel_crtc_set_config(struct drm_mode_set *set) */ if (!intel_crtc->primary_enabled && ret == 0) { WARN_ON(!intel_crtc->active); - intel_enable_primary_hw_plane(dev_priv, intel_crtc->plane, - intel_crtc->pipe); + intel_enable_primary_hw_plane(set->crtc->primary, set->crtc); } /* @@ -11473,8 +11744,6 @@ static int intel_primary_plane_disable(struct drm_plane *plane) { struct drm_device *dev = plane->dev; - struct drm_i915_private *dev_priv = dev->dev_private; - struct intel_plane *intel_plane = to_intel_plane(plane); struct intel_crtc *intel_crtc; if (!plane->fb) @@ -11497,8 +11766,8 @@ intel_primary_plane_disable(struct drm_plane *plane) goto disable_unpin; intel_crtc_wait_for_pending_flips(plane->crtc); - intel_disable_primary_hw_plane(dev_priv, intel_plane->plane, - intel_plane->pipe); + intel_disable_primary_hw_plane(plane, plane->crtc); + disable_unpin: mutex_lock(&dev->struct_mutex); i915_gem_track_fb(intel_fb_obj(plane->fb), NULL, @@ -11511,75 +11780,37 @@ disable_unpin: } static int -intel_primary_plane_setplane(struct drm_plane *plane, struct drm_crtc *crtc, - struct drm_framebuffer *fb, int crtc_x, int crtc_y, - unsigned int crtc_w, unsigned int crtc_h, - uint32_t src_x, uint32_t src_y, - uint32_t src_w, uint32_t src_h) +intel_check_primary_plane(struct drm_plane *plane, + struct intel_plane_state *state) +{ + struct drm_crtc *crtc = state->crtc; + struct drm_framebuffer *fb = state->fb; + struct drm_rect *dest = &state->dst; + struct drm_rect *src = &state->src; + const struct drm_rect *clip = &state->clip; + + return drm_plane_helper_check_update(plane, crtc, fb, + src, dest, clip, + DRM_PLANE_HELPER_NO_SCALING, + DRM_PLANE_HELPER_NO_SCALING, + false, true, &state->visible); +} + +static int +intel_commit_primary_plane(struct drm_plane *plane, + struct intel_plane_state *state) { + struct drm_crtc *crtc = state->crtc; + struct drm_framebuffer *fb = state->fb; struct drm_device *dev = crtc->dev; struct drm_i915_private *dev_priv = dev->dev_private; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); - struct intel_plane *intel_plane = to_intel_plane(plane); struct drm_i915_gem_object *obj = intel_fb_obj(fb); struct drm_i915_gem_object *old_obj = intel_fb_obj(plane->fb); - struct drm_rect dest = { - /* integer pixels */ - .x1 = crtc_x, - .y1 = crtc_y, - .x2 = crtc_x + crtc_w, - .y2 = crtc_y + crtc_h, - }; - struct drm_rect src = { - /* 16.16 fixed point */ - .x1 = src_x, - .y1 = src_y, - .x2 = src_x + src_w, - .y2 = src_y + src_h, - }; - const struct drm_rect clip = { - /* integer pixels */ - .x2 = intel_crtc->active ? intel_crtc->config.pipe_src_w : 0, - .y2 = intel_crtc->active ? intel_crtc->config.pipe_src_h : 0, - }; - bool visible; + struct intel_plane *intel_plane = to_intel_plane(plane); + struct drm_rect *src = &state->src; int ret; - ret = drm_plane_helper_check_update(plane, crtc, fb, - &src, &dest, &clip, - DRM_PLANE_HELPER_NO_SCALING, - DRM_PLANE_HELPER_NO_SCALING, - false, true, &visible); - - if (ret) - return ret; - - /* - * If the CRTC isn't enabled, we're just pinning the framebuffer, - * updating the fb pointer, and returning without touching the - * hardware. This allows us to later do a drmModeSetCrtc with fb=-1 to - * turn on the display with all planes setup as desired. - */ - if (!crtc->enabled) { - mutex_lock(&dev->struct_mutex); - - /* - * If we already called setplane while the crtc was disabled, - * we may have an fb pinned; unpin it. - */ - if (plane->fb) - intel_unpin_fb_obj(old_obj); - - i915_gem_track_fb(old_obj, obj, - INTEL_FRONTBUFFER_PRIMARY(intel_crtc->pipe)); - - /* Pin and return without programming hardware */ - ret = intel_pin_and_fence_fb_obj(dev, obj, NULL); - mutex_unlock(&dev->struct_mutex); - - return ret; - } - intel_crtc_wait_for_pending_flips(crtc); /* @@ -11588,7 +11819,7 @@ intel_primary_plane_setplane(struct drm_plane *plane, struct drm_crtc *crtc, * happens if userspace explicitly disables the plane by passing fb=0 * because plane->fb still gets set and pinned. */ - if (!visible) { + if (!state->visible) { mutex_lock(&dev->struct_mutex); /* @@ -11607,9 +11838,7 @@ intel_primary_plane_setplane(struct drm_plane *plane, struct drm_crtc *crtc, INTEL_FRONTBUFFER_PRIMARY(intel_crtc->pipe)); if (intel_crtc->primary_enabled) - intel_disable_primary_hw_plane(dev_priv, - intel_plane->plane, - intel_plane->pipe); + intel_disable_primary_hw_plane(plane, crtc); if (plane->fb != fb) @@ -11618,16 +11847,85 @@ intel_primary_plane_setplane(struct drm_plane *plane, struct drm_crtc *crtc, mutex_unlock(&dev->struct_mutex); - return 0; + } else { + if (intel_crtc && intel_crtc->active && + intel_crtc->primary_enabled) { + /* + * FBC does not work on some platforms for rotated + * planes, so disable it when rotation is not 0 and + * update it when rotation is set back to 0. + * + * FIXME: This is redundant with the fbc update done in + * the primary plane enable function except that that + * one is done too late. We eventually need to unify + * this. + */ + if (INTEL_INFO(dev)->gen <= 4 && !IS_G4X(dev) && + dev_priv->fbc.plane == intel_crtc->plane && + intel_plane->rotation != BIT(DRM_ROTATE_0)) { + intel_disable_fbc(dev); + } + } + ret = intel_pipe_set_base(crtc, src->x1, src->y1, fb); + if (ret) + return ret; + + if (!intel_crtc->primary_enabled) + intel_enable_primary_hw_plane(plane, crtc); } - ret = intel_pipe_set_base(crtc, src.x1, src.y1, fb); + intel_plane->crtc_x = state->orig_dst.x1; + intel_plane->crtc_y = state->orig_dst.y1; + intel_plane->crtc_w = drm_rect_width(&state->orig_dst); + intel_plane->crtc_h = drm_rect_height(&state->orig_dst); + intel_plane->src_x = state->orig_src.x1; + intel_plane->src_y = state->orig_src.y1; + intel_plane->src_w = drm_rect_width(&state->orig_src); + intel_plane->src_h = drm_rect_height(&state->orig_src); + intel_plane->obj = obj; + + return 0; +} + +static int +intel_primary_plane_setplane(struct drm_plane *plane, struct drm_crtc *crtc, + struct drm_framebuffer *fb, int crtc_x, int crtc_y, + unsigned int crtc_w, unsigned int crtc_h, + uint32_t src_x, uint32_t src_y, + uint32_t src_w, uint32_t src_h) +{ + struct intel_plane_state state; + struct intel_crtc *intel_crtc = to_intel_crtc(crtc); + int ret; + + state.crtc = crtc; + state.fb = fb; + + /* sample coordinates in 16.16 fixed point */ + state.src.x1 = src_x; + state.src.x2 = src_x + src_w; + state.src.y1 = src_y; + state.src.y2 = src_y + src_h; + + /* integer pixels */ + state.dst.x1 = crtc_x; + state.dst.x2 = crtc_x + crtc_w; + state.dst.y1 = crtc_y; + state.dst.y2 = crtc_y + crtc_h; + + state.clip.x1 = 0; + state.clip.y1 = 0; + state.clip.x2 = intel_crtc->active ? intel_crtc->config.pipe_src_w : 0; + state.clip.y2 = intel_crtc->active ? intel_crtc->config.pipe_src_h : 0; + + state.orig_src = state.src; + state.orig_dst = state.dst; + + ret = intel_check_primary_plane(plane, &state); if (ret) return ret; - if (!intel_crtc->primary_enabled) - intel_enable_primary_hw_plane(dev_priv, intel_crtc->plane, - intel_crtc->pipe); + intel_commit_primary_plane(plane, &state); return 0; } @@ -11644,6 +11942,7 @@ static const struct drm_plane_funcs intel_primary_plane_funcs = { .update_plane = intel_primary_plane_setplane, .disable_plane = intel_primary_plane_disable, .destroy = intel_plane_destroy, + .set_property = intel_plane_set_property }; static struct drm_plane *intel_primary_plane_create(struct drm_device *dev, @@ -11661,6 +11960,7 @@ static struct drm_plane *intel_primary_plane_create(struct drm_device *dev, primary->max_downscale = 1; primary->pipe = pipe; primary->plane = pipe; + primary->rotation = BIT(DRM_ROTATE_0); if (HAS_FBC(dev) && INTEL_INFO(dev)->gen < 4) primary->plane = !pipe; @@ -11676,6 +11976,7 @@ static struct drm_plane *intel_primary_plane_create(struct drm_device *dev, &intel_primary_plane_funcs, intel_primary_formats, num_formats, DRM_PLANE_TYPE_PRIMARY); + return &primary->base; } @@ -11691,6 +11992,50 @@ intel_cursor_plane_disable(struct drm_plane *plane) } static int +intel_check_cursor_plane(struct drm_plane *plane, + struct intel_plane_state *state) +{ + struct drm_crtc *crtc = state->crtc; + struct drm_framebuffer *fb = state->fb; + struct drm_rect *dest = &state->dst; + struct drm_rect *src = &state->src; + const struct drm_rect *clip = &state->clip; + + return drm_plane_helper_check_update(plane, crtc, fb, + src, dest, clip, + DRM_PLANE_HELPER_NO_SCALING, + DRM_PLANE_HELPER_NO_SCALING, + true, true, &state->visible); +} + +static int +intel_commit_cursor_plane(struct drm_plane *plane, + struct intel_plane_state *state) +{ + struct drm_crtc *crtc = state->crtc; + struct drm_framebuffer *fb = state->fb; + struct intel_crtc *intel_crtc = to_intel_crtc(crtc); + struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb); + struct drm_i915_gem_object *obj = intel_fb->obj; + int crtc_w, crtc_h; + + crtc->cursor_x = state->orig_dst.x1; + crtc->cursor_y = state->orig_dst.y1; + if (fb != crtc->cursor->fb) { + crtc_w = drm_rect_width(&state->orig_dst); + crtc_h = drm_rect_height(&state->orig_dst); + return intel_crtc_cursor_set_obj(crtc, obj, crtc_w, crtc_h); + } else { + intel_crtc_update_cursor(crtc, state->visible); + + intel_frontbuffer_flip(crtc->dev, + INTEL_FRONTBUFFER_CURSOR(intel_crtc->pipe)); + + return 0; + } +} + +static int intel_cursor_plane_update(struct drm_plane *plane, struct drm_crtc *crtc, struct drm_framebuffer *fb, int crtc_x, int crtc_y, unsigned int crtc_w, unsigned int crtc_h, @@ -11698,47 +12043,39 @@ intel_cursor_plane_update(struct drm_plane *plane, struct drm_crtc *crtc, uint32_t src_w, uint32_t src_h) { struct intel_crtc *intel_crtc = to_intel_crtc(crtc); - struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb); - struct drm_i915_gem_object *obj = intel_fb->obj; - struct drm_rect dest = { - /* integer pixels */ - .x1 = crtc_x, - .y1 = crtc_y, - .x2 = crtc_x + crtc_w, - .y2 = crtc_y + crtc_h, - }; - struct drm_rect src = { - /* 16.16 fixed point */ - .x1 = src_x, - .y1 = src_y, - .x2 = src_x + src_w, - .y2 = src_y + src_h, - }; - const struct drm_rect clip = { - /* integer pixels */ - .x2 = intel_crtc->active ? intel_crtc->config.pipe_src_w : 0, - .y2 = intel_crtc->active ? intel_crtc->config.pipe_src_h : 0, - }; - bool visible; + struct intel_plane_state state; int ret; - ret = drm_plane_helper_check_update(plane, crtc, fb, - &src, &dest, &clip, - DRM_PLANE_HELPER_NO_SCALING, - DRM_PLANE_HELPER_NO_SCALING, - true, true, &visible); + state.crtc = crtc; + state.fb = fb; + + /* sample coordinates in 16.16 fixed point */ + state.src.x1 = src_x; + state.src.x2 = src_x + src_w; + state.src.y1 = src_y; + state.src.y2 = src_y + src_h; + + /* integer pixels */ + state.dst.x1 = crtc_x; + state.dst.x2 = crtc_x + crtc_w; + state.dst.y1 = crtc_y; + state.dst.y2 = crtc_y + crtc_h; + + state.clip.x1 = 0; + state.clip.y1 = 0; + state.clip.x2 = intel_crtc->active ? intel_crtc->config.pipe_src_w : 0; + state.clip.y2 = intel_crtc->active ? intel_crtc->config.pipe_src_h : 0; + + state.orig_src = state.src; + state.orig_dst = state.dst; + + ret = intel_check_cursor_plane(plane, &state); if (ret) return ret; - crtc->cursor_x = crtc_x; - crtc->cursor_y = crtc_y; - if (fb != crtc->cursor->fb) { - return intel_crtc_cursor_set_obj(crtc, obj, crtc_w, crtc_h); - } else { - intel_crtc_update_cursor(crtc, visible); - return 0; - } + return intel_commit_cursor_plane(plane, &state); } + static const struct drm_plane_funcs intel_cursor_plane_funcs = { .update_plane = intel_cursor_plane_update, .disable_plane = intel_cursor_plane_disable, @@ -11812,8 +12149,7 @@ static void intel_crtc_init(struct drm_device *dev, int pipe) intel_crtc->cursor_base = ~0; intel_crtc->cursor_cntl = ~0; - - init_waitqueue_head(&intel_crtc->vbl_wait); + intel_crtc->cursor_size = ~0; BUG_ON(pipe >= ARRAY_SIZE(dev_priv->plane_to_crtc_mapping) || dev_priv->plane_to_crtc_mapping[intel_crtc->plane] != NULL); @@ -11876,8 +12212,7 @@ static int intel_encoder_clones(struct intel_encoder *encoder) int index_mask = 0; int entry = 0; - list_for_each_entry(source_encoder, - &dev->mode_config.encoder_list, base.head) { + for_each_intel_encoder(dev, source_encoder) { if (encoders_cloneable(encoder, source_encoder)) index_mask |= (1 << entry); @@ -12066,7 +12401,7 @@ static void intel_setup_outputs(struct drm_device *dev) intel_edp_psr_init(dev); - list_for_each_entry(encoder, &dev->mode_config.encoder_list, base.head) { + for_each_intel_encoder(dev, encoder) { encoder->base.possible_crtcs = encoder->crtc_mask; encoder->base.possible_clones = intel_encoder_clones(encoder); @@ -12332,29 +12667,27 @@ static void intel_init_display(struct drm_device *dev) dev_priv->display.get_display_clock_speed = i830_get_display_clock_speed; - if (HAS_PCH_SPLIT(dev)) { - if (IS_GEN5(dev)) { - dev_priv->display.fdi_link_train = ironlake_fdi_link_train; - dev_priv->display.write_eld = ironlake_write_eld; - } else if (IS_GEN6(dev)) { - dev_priv->display.fdi_link_train = gen6_fdi_link_train; - dev_priv->display.write_eld = ironlake_write_eld; - dev_priv->display.modeset_global_resources = - snb_modeset_global_resources; - } else if (IS_IVYBRIDGE(dev)) { - /* FIXME: detect B0+ stepping and use auto training */ - dev_priv->display.fdi_link_train = ivb_manual_fdi_link_train; - dev_priv->display.write_eld = ironlake_write_eld; - dev_priv->display.modeset_global_resources = - ivb_modeset_global_resources; - } else if (IS_HASWELL(dev) || IS_GEN8(dev)) { - dev_priv->display.fdi_link_train = hsw_fdi_link_train; - dev_priv->display.write_eld = haswell_write_eld; - dev_priv->display.modeset_global_resources = - haswell_modeset_global_resources; - } - } else if (IS_G4X(dev)) { + if (IS_G4X(dev)) { dev_priv->display.write_eld = g4x_write_eld; + } else if (IS_GEN5(dev)) { + dev_priv->display.fdi_link_train = ironlake_fdi_link_train; + dev_priv->display.write_eld = ironlake_write_eld; + } else if (IS_GEN6(dev)) { + dev_priv->display.fdi_link_train = gen6_fdi_link_train; + dev_priv->display.write_eld = ironlake_write_eld; + dev_priv->display.modeset_global_resources = + snb_modeset_global_resources; + } else if (IS_IVYBRIDGE(dev)) { + /* FIXME: detect B0+ stepping and use auto training */ + dev_priv->display.fdi_link_train = ivb_manual_fdi_link_train; + dev_priv->display.write_eld = ironlake_write_eld; + dev_priv->display.modeset_global_resources = + ivb_modeset_global_resources; + } else if (IS_HASWELL(dev) || IS_BROADWELL(dev)) { + dev_priv->display.fdi_link_train = hsw_fdi_link_train; + dev_priv->display.write_eld = haswell_write_eld; + dev_priv->display.modeset_global_resources = + haswell_modeset_global_resources; } else if (IS_VALLEYVIEW(dev)) { dev_priv->display.modeset_global_resources = valleyview_modeset_global_resources; @@ -12388,6 +12721,8 @@ static void intel_init_display(struct drm_device *dev) } intel_panel_init_backlight_funcs(dev); + + mutex_init(&dev_priv->pps_mutex); } /* @@ -12403,6 +12738,14 @@ static void quirk_pipea_force(struct drm_device *dev) DRM_INFO("applying pipe a force quirk\n"); } +static void quirk_pipeb_force(struct drm_device *dev) +{ + struct drm_i915_private *dev_priv = dev->dev_private; + + dev_priv->quirks |= QUIRK_PIPEB_FORCE; + DRM_INFO("applying pipe b force quirk\n"); +} + /* * Some machines (Lenovo U160) do not work with SSC on LVDS for some reason */ @@ -12477,6 +12820,12 @@ static struct intel_quirk intel_quirks[] = { /* ThinkPad T60 needs pipe A force quirk (bug #16494) */ { 0x2782, 0x17aa, 0x201a, quirk_pipea_force }, + /* 830 needs to leave pipe A & dpll A up */ + { 0x3577, PCI_ANY_ID, PCI_ANY_ID, quirk_pipea_force }, + + /* 830 needs to leave pipe B & dpll B up */ + { 0x3577, PCI_ANY_ID, PCI_ANY_ID, quirk_pipeb_force }, + /* Lenovo U160 cannot use SSC on LVDS */ { 0x0046, 0x17aa, 0x3920, quirk_ssc_force_disable }, @@ -12550,7 +12899,11 @@ static void i915_disable_vga(struct drm_device *dev) vga_put(dev->pdev, VGA_RSRC_LEGACY_IO); udelay(300); - I915_WRITE(vga_reg, VGA_DISP_DISABLE); + /* + * Fujitsu-Siemens Lifebook S6010 (830) has problems resuming + * from S3 without preserving (some of?) the other bits. + */ + I915_WRITE(vga_reg, dev_priv->bios_vgacntr | VGA_DISP_DISABLE); POSTING_READ(vga_reg); } @@ -12563,8 +12916,6 @@ void intel_modeset_init_hw(struct drm_device *dev) intel_init_clock_gating(dev); - intel_reset_dpio(dev); - intel_enable_gt_powersave(dev); } @@ -12610,7 +12961,10 @@ void intel_modeset_init(struct drm_device *dev) dev->mode_config.max_height = 8192; } - if (IS_GEN2(dev)) { + if (IS_845G(dev) || IS_I865G(dev)) { + dev->mode_config.cursor_width = IS_845G(dev) ? 64 : 512; + dev->mode_config.cursor_height = 1023; + } else if (IS_GEN2(dev)) { dev->mode_config.cursor_width = GEN2_CURSOR_WIDTH; dev->mode_config.cursor_height = GEN2_CURSOR_HEIGHT; } else { @@ -12624,7 +12978,7 @@ void intel_modeset_init(struct drm_device *dev) INTEL_INFO(dev)->num_pipes, INTEL_INFO(dev)->num_pipes > 1 ? "s" : ""); - for_each_pipe(pipe) { + for_each_pipe(dev_priv, pipe) { intel_crtc_init(dev, pipe); for_each_sprite(pipe, sprite) { ret = intel_plane_init(dev, pipe, sprite); @@ -12635,10 +12989,11 @@ void intel_modeset_init(struct drm_device *dev) } intel_init_dpio(dev); - intel_reset_dpio(dev); intel_shared_dpll_init(dev); + /* save the BIOS value before clobbering it */ + dev_priv->bios_vgacntr = I915_READ(i915_vgacntrl_reg(dev)); /* Just disable it once at startup */ i915_disable_vga(dev); intel_setup_outputs(dev); @@ -12730,9 +13085,10 @@ static void intel_sanitize_crtc(struct intel_crtc *crtc) I915_WRITE(reg, I915_READ(reg) & ~PIPECONF_FRAME_START_DELAY_MASK); /* restore vblank interrupts to correct state */ - if (crtc->active) + if (crtc->active) { + update_scanline_offset(crtc); drm_vblank_on(dev, crtc->pipe); - else + } else drm_vblank_off(dev, crtc->pipe); /* We need to sanitize the plane -> pipe mapping first because this will @@ -12815,7 +13171,7 @@ static void intel_sanitize_crtc(struct intel_crtc *crtc) } } - if (crtc->active || IS_VALLEYVIEW(dev) || INTEL_INFO(dev)->gen < 5) { + if (crtc->active || HAS_GMCH_DISPLAY(dev)) { /* * We start out with underrun reporting disabled to avoid races. * For correct bookkeeping mark this on active crtcs. @@ -12831,8 +13187,6 @@ static void intel_sanitize_crtc(struct intel_crtc *crtc) */ crtc->cpu_fifo_underrun_disabled = true; crtc->pch_fifo_underrun_disabled = true; - - update_scanline_offset(crtc); } } @@ -12964,8 +13318,7 @@ static void intel_modeset_readout_hw_state(struct drm_device *dev) intel_display_power_get(dev_priv, POWER_DOMAIN_PLLS); } - list_for_each_entry(encoder, &dev->mode_config.encoder_list, - base.head) { + for_each_intel_encoder(dev, encoder) { pipe = 0; if (encoder->get_hw_state(encoder, &pipe)) { @@ -13029,12 +13382,11 @@ void intel_modeset_setup_hw_state(struct drm_device *dev, } /* HW state is read out, now we need to sanitize this mess. */ - list_for_each_entry(encoder, &dev->mode_config.encoder_list, - base.head) { + for_each_intel_encoder(dev, encoder) { intel_sanitize_encoder(encoder); } - for_each_pipe(pipe) { + for_each_pipe(dev_priv, pipe) { crtc = to_intel_crtc(dev_priv->pipe_to_crtc_mapping[pipe]); intel_sanitize_crtc(crtc); intel_dump_pipe_config(crtc, &crtc->config, "[setup_hw_state]"); @@ -13062,7 +13414,7 @@ void intel_modeset_setup_hw_state(struct drm_device *dev, * We need to use raw interfaces for restoring state to avoid * checking (bogus) intermediate states. */ - for_each_pipe(pipe) { + for_each_pipe(dev_priv, pipe) { struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe]; @@ -13283,7 +13635,7 @@ intel_display_capture_error_state(struct drm_device *dev) if (IS_HASWELL(dev) || IS_BROADWELL(dev)) error->power_well_driver = I915_READ(HSW_PWR_WELL_DRIVER); - for_each_pipe(i) { + for_each_pipe(dev_priv, i) { error->pipe[i].power_domain_on = intel_display_power_enabled_unlocked(dev_priv, POWER_DOMAIN_PIPE(i)); @@ -13347,6 +13699,7 @@ intel_display_print_error_state(struct drm_i915_error_state_buf *m, struct drm_device *dev, struct intel_display_error_state *error) { + struct drm_i915_private *dev_priv = dev->dev_private; int i; if (!error) @@ -13356,7 +13709,7 @@ intel_display_print_error_state(struct drm_i915_error_state_buf *m, if (IS_HASWELL(dev) || IS_BROADWELL(dev)) err_printf(m, "PWR_WELL_CTL2: %08x\n", error->power_well_driver); - for_each_pipe(i) { + for_each_pipe(dev_priv, i) { err_printf(m, "Pipe [%d]:\n", i); err_printf(m, " Power: %s\n", error->pipe[i].power_domain_on ? "on" : "off"); @@ -13397,3 +13750,24 @@ intel_display_print_error_state(struct drm_i915_error_state_buf *m, err_printf(m, " VSYNC: %08x\n", error->transcoder[i].vsync); } } + +void intel_modeset_preclose(struct drm_device *dev, struct drm_file *file) +{ + struct intel_crtc *crtc; + + for_each_intel_crtc(dev, crtc) { + struct intel_unpin_work *work; + + spin_lock_irq(&dev->event_lock); + + work = crtc->unpin_work; + + if (work && work->event && + work->event->base.file_priv == file) { + kfree(work->event); + work->event = NULL; + } + + spin_unlock_irq(&dev->event_lock); + } +} diff --git a/drivers/gpu/drm/i915/intel_dp.c b/drivers/gpu/drm/i915/intel_dp.c index fdff1d4..0994690 100644 --- a/drivers/gpu/drm/i915/intel_dp.c +++ b/drivers/gpu/drm/i915/intel_dp.c @@ -111,7 +111,7 @@ static struct intel_dp *intel_attached_dp(struct drm_connector *connector) } static void intel_dp_link_down(struct intel_dp *intel_dp); -static bool _edp_panel_vdd_on(struct intel_dp *intel_dp); +static bool edp_panel_vdd_on(struct intel_dp *intel_dp); static void edp_panel_vdd_off(struct intel_dp *intel_dp, bool sync); int @@ -290,32 +290,201 @@ intel_dp_init_panel_power_sequencer_registers(struct drm_device *dev, struct intel_dp *intel_dp, struct edp_power_seq *out); +static void pps_lock(struct intel_dp *intel_dp) +{ + struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp); + struct intel_encoder *encoder = &intel_dig_port->base; + struct drm_device *dev = encoder->base.dev; + struct drm_i915_private *dev_priv = dev->dev_private; + enum intel_display_power_domain power_domain; + + /* + * See vlv_power_sequencer_reset() why we need + * a power domain reference here. + */ + power_domain = intel_display_port_power_domain(encoder); + intel_display_power_get(dev_priv, power_domain); + + mutex_lock(&dev_priv->pps_mutex); +} + +static void pps_unlock(struct intel_dp *intel_dp) +{ + struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp); + struct intel_encoder *encoder = &intel_dig_port->base; + struct drm_device *dev = encoder->base.dev; + struct drm_i915_private *dev_priv = dev->dev_private; + enum intel_display_power_domain power_domain; + + mutex_unlock(&dev_priv->pps_mutex); + + power_domain = intel_display_port_power_domain(encoder); + intel_display_power_put(dev_priv, power_domain); +} + static enum pipe vlv_power_sequencer_pipe(struct intel_dp *intel_dp) { struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp); - struct drm_crtc *crtc = intel_dig_port->base.base.crtc; struct drm_device *dev = intel_dig_port->base.base.dev; struct drm_i915_private *dev_priv = dev->dev_private; - enum port port = intel_dig_port->port; - enum pipe pipe; + struct intel_encoder *encoder; + unsigned int pipes = (1 << PIPE_A) | (1 << PIPE_B); + struct edp_power_seq power_seq; + + lockdep_assert_held(&dev_priv->pps_mutex); - /* modeset should have pipe */ - if (crtc) - return to_intel_crtc(crtc)->pipe; + if (intel_dp->pps_pipe != INVALID_PIPE) + return intel_dp->pps_pipe; + + /* + * We don't have power sequencer currently. + * Pick one that's not used by other ports. + */ + list_for_each_entry(encoder, &dev->mode_config.encoder_list, + base.head) { + struct intel_dp *tmp; + + if (encoder->type != INTEL_OUTPUT_EDP) + continue; + + tmp = enc_to_intel_dp(&encoder->base); + + if (tmp->pps_pipe != INVALID_PIPE) + pipes &= ~(1 << tmp->pps_pipe); + } + + /* + * Didn't find one. This should not happen since there + * are two power sequencers and up to two eDP ports. + */ + if (WARN_ON(pipes == 0)) + return PIPE_A; + + intel_dp->pps_pipe = ffs(pipes) - 1; + + DRM_DEBUG_KMS("picked pipe %c power sequencer for port %c\n", + pipe_name(intel_dp->pps_pipe), + port_name(intel_dig_port->port)); + + /* init power sequencer on this pipe and port */ + intel_dp_init_panel_power_sequencer(dev, intel_dp, &power_seq); + intel_dp_init_panel_power_sequencer_registers(dev, intel_dp, + &power_seq); + + return intel_dp->pps_pipe; +} + +typedef bool (*vlv_pipe_check)(struct drm_i915_private *dev_priv, + enum pipe pipe); + +static bool vlv_pipe_has_pp_on(struct drm_i915_private *dev_priv, + enum pipe pipe) +{ + return I915_READ(VLV_PIPE_PP_STATUS(pipe)) & PP_ON; +} + +static bool vlv_pipe_has_vdd_on(struct drm_i915_private *dev_priv, + enum pipe pipe) +{ + return I915_READ(VLV_PIPE_PP_CONTROL(pipe)) & EDP_FORCE_VDD; +} + +static bool vlv_pipe_any(struct drm_i915_private *dev_priv, + enum pipe pipe) +{ + return true; +} + +static enum pipe +vlv_initial_pps_pipe(struct drm_i915_private *dev_priv, + enum port port, + vlv_pipe_check pipe_check) +{ + enum pipe pipe; - /* init time, try to find a pipe with this port selected */ for (pipe = PIPE_A; pipe <= PIPE_B; pipe++) { u32 port_sel = I915_READ(VLV_PIPE_PP_ON_DELAYS(pipe)) & PANEL_PORT_SELECT_MASK; - if (port_sel == PANEL_PORT_SELECT_DPB_VLV && port == PORT_B) - return pipe; - if (port_sel == PANEL_PORT_SELECT_DPC_VLV && port == PORT_C) - return pipe; + + if (port_sel != PANEL_PORT_SELECT_VLV(port)) + continue; + + if (!pipe_check(dev_priv, pipe)) + continue; + + return pipe; } - /* shrug */ - return PIPE_A; + return INVALID_PIPE; +} + +static void +vlv_initial_power_sequencer_setup(struct intel_dp *intel_dp) +{ + struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp); + struct drm_device *dev = intel_dig_port->base.base.dev; + struct drm_i915_private *dev_priv = dev->dev_private; + struct edp_power_seq power_seq; + enum port port = intel_dig_port->port; + + lockdep_assert_held(&dev_priv->pps_mutex); + + /* try to find a pipe with this port selected */ + /* first pick one where the panel is on */ + intel_dp->pps_pipe = vlv_initial_pps_pipe(dev_priv, port, + vlv_pipe_has_pp_on); + /* didn't find one? pick one where vdd is on */ + if (intel_dp->pps_pipe == INVALID_PIPE) + intel_dp->pps_pipe = vlv_initial_pps_pipe(dev_priv, port, + vlv_pipe_has_vdd_on); + /* didn't find one? pick one with just the correct port */ + if (intel_dp->pps_pipe == INVALID_PIPE) + intel_dp->pps_pipe = vlv_initial_pps_pipe(dev_priv, port, + vlv_pipe_any); + + /* didn't find one? just let vlv_power_sequencer_pipe() pick one when needed */ + if (intel_dp->pps_pipe == INVALID_PIPE) { + DRM_DEBUG_KMS("no initial power sequencer for port %c\n", + port_name(port)); + return; + } + + DRM_DEBUG_KMS("initial power sequencer for port %c: pipe %c\n", + port_name(port), pipe_name(intel_dp->pps_pipe)); + + intel_dp_init_panel_power_sequencer(dev, intel_dp, &power_seq); + intel_dp_init_panel_power_sequencer_registers(dev, intel_dp, + &power_seq); +} + +void vlv_power_sequencer_reset(struct drm_i915_private *dev_priv) +{ + struct drm_device *dev = dev_priv->dev; + struct intel_encoder *encoder; + + if (WARN_ON(!IS_VALLEYVIEW(dev))) + return; + + /* + * We can't grab pps_mutex here due to deadlock with power_domain + * mutex when power_domain functions are called while holding pps_mutex. + * That also means that in order to use pps_pipe the code needs to + * hold both a power domain reference and pps_mutex, and the power domain + * reference get/put must be done while _not_ holding pps_mutex. + * pps_{lock,unlock}() do these steps in the correct order, so one + * should use them always. + */ + + list_for_each_entry(encoder, &dev->mode_config.encoder_list, base.head) { + struct intel_dp *intel_dp; + + if (encoder->type != INTEL_OUTPUT_EDP) + continue; + + intel_dp = enc_to_intel_dp(&encoder->base); + intel_dp->pps_pipe = INVALID_PIPE; + } } static u32 _pp_ctrl_reg(struct intel_dp *intel_dp) @@ -349,12 +518,15 @@ static int edp_notify_handler(struct notifier_block *this, unsigned long code, struct drm_i915_private *dev_priv = dev->dev_private; u32 pp_div; u32 pp_ctrl_reg, pp_div_reg; - enum pipe pipe = vlv_power_sequencer_pipe(intel_dp); if (!is_edp(intel_dp) || code != SYS_RESTART) return 0; + pps_lock(intel_dp); + if (IS_VALLEYVIEW(dev)) { + enum pipe pipe = vlv_power_sequencer_pipe(intel_dp); + pp_ctrl_reg = VLV_PIPE_PP_CONTROL(pipe); pp_div_reg = VLV_PIPE_PP_DIVISOR(pipe); pp_div = I915_READ(pp_div_reg); @@ -366,6 +538,8 @@ static int edp_notify_handler(struct notifier_block *this, unsigned long code, msleep(intel_dp->panel_power_cycle_delay); } + pps_unlock(intel_dp); + return 0; } @@ -374,6 +548,8 @@ static bool edp_have_panel_power(struct intel_dp *intel_dp) struct drm_device *dev = intel_dp_to_dev(intel_dp); struct drm_i915_private *dev_priv = dev->dev_private; + lockdep_assert_held(&dev_priv->pps_mutex); + return (I915_READ(_pp_stat_reg(intel_dp)) & PP_ON) != 0; } @@ -381,13 +557,10 @@ static bool edp_have_panel_vdd(struct intel_dp *intel_dp) { struct drm_device *dev = intel_dp_to_dev(intel_dp); struct drm_i915_private *dev_priv = dev->dev_private; - struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp); - struct intel_encoder *intel_encoder = &intel_dig_port->base; - enum intel_display_power_domain power_domain; - power_domain = intel_display_port_power_domain(intel_encoder); - return intel_display_power_enabled(dev_priv, power_domain) && - (I915_READ(_pp_ctrl_reg(intel_dp)) & EDP_FORCE_VDD) != 0; + lockdep_assert_held(&dev_priv->pps_mutex); + + return I915_READ(_pp_ctrl_reg(intel_dp)) & EDP_FORCE_VDD; } static void @@ -535,7 +708,15 @@ intel_dp_aux_ch(struct intel_dp *intel_dp, bool has_aux_irq = HAS_AUX_IRQ(dev); bool vdd; - vdd = _edp_panel_vdd_on(intel_dp); + pps_lock(intel_dp); + + /* + * We will be called with VDD already enabled for dpcd/edid/oui reads. + * In such cases we want to leave VDD enabled and it's up to upper layers + * to turn it off. But for eg. i2c-dev access we need to turn it on/off + * ourselves. + */ + vdd = edp_panel_vdd_on(intel_dp); /* dp aux is extremely sensitive to irq latency, hence request the * lowest possible wakeup latency and so prevent the cpu from going into @@ -644,6 +825,8 @@ out: if (vdd) edp_panel_vdd_off(intel_dp, false); + pps_unlock(intel_dp); + return ret; } @@ -828,20 +1011,6 @@ intel_dp_set_clock(struct intel_encoder *encoder, } } -static void -intel_dp_set_m2_n2(struct intel_crtc *crtc, struct intel_link_m_n *m_n) -{ - struct drm_device *dev = crtc->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; - enum transcoder transcoder = crtc->config.cpu_transcoder; - - I915_WRITE(PIPE_DATA_M2(transcoder), - TU_SIZE(m_n->tu) | m_n->gmch_m); - I915_WRITE(PIPE_DATA_N2(transcoder), m_n->gmch_n); - I915_WRITE(PIPE_LINK_M2(transcoder), m_n->link_m); - I915_WRITE(PIPE_LINK_N2(transcoder), m_n->link_n); -} - bool intel_dp_compute_config(struct intel_encoder *encoder, struct intel_crtc_config *pipe_config) @@ -867,6 +1036,7 @@ intel_dp_compute_config(struct intel_encoder *encoder, pipe_config->has_pch_encoder = true; pipe_config->has_dp_encoder = true; + pipe_config->has_drrs = false; pipe_config->has_audio = intel_dp->has_audio; if (is_edp(intel_dp) && intel_connector->panel.fixed_mode) { @@ -898,23 +1068,15 @@ intel_dp_compute_config(struct intel_encoder *encoder, bpp = dev_priv->vbt.edp_bpp; } - if (IS_BROADWELL(dev)) { - /* Yes, it's an ugly hack. */ - min_lane_count = max_lane_count; - DRM_DEBUG_KMS("forcing lane count to max (%u) on BDW\n", - min_lane_count); - } else if (dev_priv->vbt.edp_lanes) { - min_lane_count = min(dev_priv->vbt.edp_lanes, - max_lane_count); - DRM_DEBUG_KMS("using min %u lanes per VBT\n", - min_lane_count); - } - - if (dev_priv->vbt.edp_rate) { - min_clock = min(dev_priv->vbt.edp_rate >> 3, max_clock); - DRM_DEBUG_KMS("using min %02x link bw per VBT\n", - bws[min_clock]); - } + /* + * Use the maximum clock and number of lanes the eDP panel + * advertizes being capable of. The panels are generally + * designed to support only a single clock and lane + * configuration, and typically these values correspond to the + * native resolution of the panel. + */ + min_lane_count = max_lane_count; + min_clock = max_clock; } for (; bpp >= 6*3; bpp -= 2*3) { @@ -970,13 +1132,14 @@ found: if (intel_connector->panel.downclock_mode != NULL && intel_dp->drrs_state.type == SEAMLESS_DRRS_SUPPORT) { + pipe_config->has_drrs = true; intel_link_compute_m_n(bpp, lane_count, intel_connector->panel.downclock_mode->clock, pipe_config->port_clock, &pipe_config->dp_m2_n2); } - if (HAS_DDI(dev)) + if (IS_HASWELL(dev) || IS_BROADWELL(dev)) hsw_dp_set_ddi_pll_sel(pipe_config, intel_dp->link_bw); else intel_dp_set_clock(encoder, pipe_config, intel_dp->link_bw); @@ -1110,6 +1273,8 @@ static void wait_panel_status(struct intel_dp *intel_dp, struct drm_i915_private *dev_priv = dev->dev_private; u32 pp_stat_reg, pp_ctrl_reg; + lockdep_assert_held(&dev_priv->pps_mutex); + pp_stat_reg = _pp_stat_reg(intel_dp); pp_ctrl_reg = _pp_ctrl_reg(intel_dp); @@ -1173,13 +1338,20 @@ static u32 ironlake_get_pp_control(struct intel_dp *intel_dp) struct drm_i915_private *dev_priv = dev->dev_private; u32 control; + lockdep_assert_held(&dev_priv->pps_mutex); + control = I915_READ(_pp_ctrl_reg(intel_dp)); control &= ~PANEL_UNLOCK_MASK; control |= PANEL_UNLOCK_REGS; return control; } -static bool _edp_panel_vdd_on(struct intel_dp *intel_dp) +/* + * Must be paired with edp_panel_vdd_off(). + * Must hold pps_mutex around the whole on/off sequence. + * Can be nested with intel_edp_panel_vdd_{on,off}() calls. + */ +static bool edp_panel_vdd_on(struct intel_dp *intel_dp) { struct drm_device *dev = intel_dp_to_dev(intel_dp); struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp); @@ -1190,6 +1362,8 @@ static bool _edp_panel_vdd_on(struct intel_dp *intel_dp) u32 pp_stat_reg, pp_ctrl_reg; bool need_to_disable = !intel_dp->want_panel_vdd; + lockdep_assert_held(&dev_priv->pps_mutex); + if (!is_edp(intel_dp)) return false; @@ -1227,62 +1401,76 @@ static bool _edp_panel_vdd_on(struct intel_dp *intel_dp) return need_to_disable; } +/* + * Must be paired with intel_edp_panel_vdd_off() or + * intel_edp_panel_off(). + * Nested calls to these functions are not allowed since + * we drop the lock. Caller must use some higher level + * locking to prevent nested calls from other threads. + */ void intel_edp_panel_vdd_on(struct intel_dp *intel_dp) { - if (is_edp(intel_dp)) { - bool vdd = _edp_panel_vdd_on(intel_dp); + bool vdd; - WARN(!vdd, "eDP VDD already requested on\n"); - } + if (!is_edp(intel_dp)) + return; + + pps_lock(intel_dp); + vdd = edp_panel_vdd_on(intel_dp); + pps_unlock(intel_dp); + + WARN(!vdd, "eDP VDD already requested on\n"); } static void edp_panel_vdd_off_sync(struct intel_dp *intel_dp) { struct drm_device *dev = intel_dp_to_dev(intel_dp); struct drm_i915_private *dev_priv = dev->dev_private; + struct intel_digital_port *intel_dig_port = + dp_to_dig_port(intel_dp); + struct intel_encoder *intel_encoder = &intel_dig_port->base; + enum intel_display_power_domain power_domain; u32 pp; u32 pp_stat_reg, pp_ctrl_reg; - WARN_ON(!drm_modeset_is_locked(&dev->mode_config.connection_mutex)); + lockdep_assert_held(&dev_priv->pps_mutex); - if (!intel_dp->want_panel_vdd && edp_have_panel_vdd(intel_dp)) { - struct intel_digital_port *intel_dig_port = - dp_to_dig_port(intel_dp); - struct intel_encoder *intel_encoder = &intel_dig_port->base; - enum intel_display_power_domain power_domain; + WARN_ON(intel_dp->want_panel_vdd); - DRM_DEBUG_KMS("Turning eDP VDD off\n"); + if (!edp_have_panel_vdd(intel_dp)) + return; - pp = ironlake_get_pp_control(intel_dp); - pp &= ~EDP_FORCE_VDD; + DRM_DEBUG_KMS("Turning eDP VDD off\n"); - pp_ctrl_reg = _pp_ctrl_reg(intel_dp); - pp_stat_reg = _pp_stat_reg(intel_dp); + pp = ironlake_get_pp_control(intel_dp); + pp &= ~EDP_FORCE_VDD; - I915_WRITE(pp_ctrl_reg, pp); - POSTING_READ(pp_ctrl_reg); + pp_ctrl_reg = _pp_ctrl_reg(intel_dp); + pp_stat_reg = _pp_stat_reg(intel_dp); - /* Make sure sequencer is idle before allowing subsequent activity */ - DRM_DEBUG_KMS("PP_STATUS: 0x%08x PP_CONTROL: 0x%08x\n", - I915_READ(pp_stat_reg), I915_READ(pp_ctrl_reg)); + I915_WRITE(pp_ctrl_reg, pp); + POSTING_READ(pp_ctrl_reg); - if ((pp & POWER_TARGET_ON) == 0) - intel_dp->last_power_cycle = jiffies; + /* Make sure sequencer is idle before allowing subsequent activity */ + DRM_DEBUG_KMS("PP_STATUS: 0x%08x PP_CONTROL: 0x%08x\n", + I915_READ(pp_stat_reg), I915_READ(pp_ctrl_reg)); - power_domain = intel_display_port_power_domain(intel_encoder); - intel_display_power_put(dev_priv, power_domain); - } + if ((pp & POWER_TARGET_ON) == 0) + intel_dp->last_power_cycle = jiffies; + + power_domain = intel_display_port_power_domain(intel_encoder); + intel_display_power_put(dev_priv, power_domain); } static void edp_panel_vdd_work(struct work_struct *__work) { struct intel_dp *intel_dp = container_of(to_delayed_work(__work), struct intel_dp, panel_vdd_work); - struct drm_device *dev = intel_dp_to_dev(intel_dp); - drm_modeset_lock(&dev->mode_config.connection_mutex, NULL); - edp_panel_vdd_off_sync(intel_dp); - drm_modeset_unlock(&dev->mode_config.connection_mutex); + pps_lock(intel_dp); + if (!intel_dp->want_panel_vdd) + edp_panel_vdd_off_sync(intel_dp); + pps_unlock(intel_dp); } static void edp_panel_vdd_schedule_off(struct intel_dp *intel_dp) @@ -1298,8 +1486,18 @@ static void edp_panel_vdd_schedule_off(struct intel_dp *intel_dp) schedule_delayed_work(&intel_dp->panel_vdd_work, delay); } +/* + * Must be paired with edp_panel_vdd_on(). + * Must hold pps_mutex around the whole on/off sequence. + * Can be nested with intel_edp_panel_vdd_{on,off}() calls. + */ static void edp_panel_vdd_off(struct intel_dp *intel_dp, bool sync) { + struct drm_i915_private *dev_priv = + intel_dp_to_dev(intel_dp)->dev_private; + + lockdep_assert_held(&dev_priv->pps_mutex); + if (!is_edp(intel_dp)) return; @@ -1313,6 +1511,22 @@ static void edp_panel_vdd_off(struct intel_dp *intel_dp, bool sync) edp_panel_vdd_schedule_off(intel_dp); } +/* + * Must be paired with intel_edp_panel_vdd_on(). + * Nested calls to these functions are not allowed since + * we drop the lock. Caller must use some higher level + * locking to prevent nested calls from other threads. + */ +static void intel_edp_panel_vdd_off(struct intel_dp *intel_dp, bool sync) +{ + if (!is_edp(intel_dp)) + return; + + pps_lock(intel_dp); + edp_panel_vdd_off(intel_dp, sync); + pps_unlock(intel_dp); +} + void intel_edp_panel_on(struct intel_dp *intel_dp) { struct drm_device *dev = intel_dp_to_dev(intel_dp); @@ -1325,9 +1539,11 @@ void intel_edp_panel_on(struct intel_dp *intel_dp) DRM_DEBUG_KMS("Turn eDP power on\n"); + pps_lock(intel_dp); + if (edp_have_panel_power(intel_dp)) { DRM_DEBUG_KMS("eDP power already on\n"); - return; + goto out; } wait_panel_power_cycle(intel_dp); @@ -1356,6 +1572,9 @@ void intel_edp_panel_on(struct intel_dp *intel_dp) I915_WRITE(pp_ctrl_reg, pp); POSTING_READ(pp_ctrl_reg); } + + out: + pps_unlock(intel_dp); } void intel_edp_panel_off(struct intel_dp *intel_dp) @@ -1373,6 +1592,8 @@ void intel_edp_panel_off(struct intel_dp *intel_dp) DRM_DEBUG_KMS("Turn eDP power off\n"); + pps_lock(intel_dp); + WARN(!intel_dp->want_panel_vdd, "Need VDD to turn off panel\n"); pp = ironlake_get_pp_control(intel_dp); @@ -1394,9 +1615,12 @@ void intel_edp_panel_off(struct intel_dp *intel_dp) /* We got a reference when we enabled the VDD. */ power_domain = intel_display_port_power_domain(intel_encoder); intel_display_power_put(dev_priv, power_domain); + + pps_unlock(intel_dp); } -void intel_edp_backlight_on(struct intel_dp *intel_dp) +/* Enable backlight in the panel power control. */ +static void _intel_edp_backlight_on(struct intel_dp *intel_dp) { struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp); struct drm_device *dev = intel_dig_port->base.base.dev; @@ -1404,13 +1628,6 @@ void intel_edp_backlight_on(struct intel_dp *intel_dp) u32 pp; u32 pp_ctrl_reg; - if (!is_edp(intel_dp)) - return; - - DRM_DEBUG_KMS("\n"); - - intel_panel_enable_backlight(intel_dp->attached_connector); - /* * If we enable the backlight right away following a panel power * on, we may see slight flicker as the panel syncs with the eDP @@ -1418,6 +1635,9 @@ void intel_edp_backlight_on(struct intel_dp *intel_dp) * allowing it to appear. */ wait_backlight_on(intel_dp); + + pps_lock(intel_dp); + pp = ironlake_get_pp_control(intel_dp); pp |= EDP_BLC_ENABLE; @@ -1425,9 +1645,24 @@ void intel_edp_backlight_on(struct intel_dp *intel_dp) I915_WRITE(pp_ctrl_reg, pp); POSTING_READ(pp_ctrl_reg); + + pps_unlock(intel_dp); } -void intel_edp_backlight_off(struct intel_dp *intel_dp) +/* Enable backlight PWM and backlight PP control. */ +void intel_edp_backlight_on(struct intel_dp *intel_dp) +{ + if (!is_edp(intel_dp)) + return; + + DRM_DEBUG_KMS("\n"); + + intel_panel_enable_backlight(intel_dp->attached_connector); + _intel_edp_backlight_on(intel_dp); +} + +/* Disable backlight in the panel power control. */ +static void _intel_edp_backlight_off(struct intel_dp *intel_dp) { struct drm_device *dev = intel_dp_to_dev(intel_dp); struct drm_i915_private *dev_priv = dev->dev_private; @@ -1437,7 +1672,8 @@ void intel_edp_backlight_off(struct intel_dp *intel_dp) if (!is_edp(intel_dp)) return; - DRM_DEBUG_KMS("\n"); + pps_lock(intel_dp); + pp = ironlake_get_pp_control(intel_dp); pp &= ~EDP_BLC_ENABLE; @@ -1445,13 +1681,51 @@ void intel_edp_backlight_off(struct intel_dp *intel_dp) I915_WRITE(pp_ctrl_reg, pp); POSTING_READ(pp_ctrl_reg); - intel_dp->last_backlight_off = jiffies; + pps_unlock(intel_dp); + + intel_dp->last_backlight_off = jiffies; edp_wait_backlight_off(intel_dp); +} +/* Disable backlight PP control and backlight PWM. */ +void intel_edp_backlight_off(struct intel_dp *intel_dp) +{ + if (!is_edp(intel_dp)) + return; + + DRM_DEBUG_KMS("\n"); + + _intel_edp_backlight_off(intel_dp); intel_panel_disable_backlight(intel_dp->attached_connector); } +/* + * Hook for controlling the panel power control backlight through the bl_power + * sysfs attribute. Take care to handle multiple calls. + */ +static void intel_edp_backlight_power(struct intel_connector *connector, + bool enable) +{ + struct intel_dp *intel_dp = intel_attached_dp(&connector->base); + bool is_enabled; + + pps_lock(intel_dp); + is_enabled = ironlake_get_pp_control(intel_dp) & EDP_BLC_ENABLE; + pps_unlock(intel_dp); + + if (is_enabled == enable) + return; + + DRM_DEBUG_KMS("panel power control backlight %s\n", + enable ? "enable" : "disable"); + + if (enable) + _intel_edp_backlight_on(intel_dp); + else + _intel_edp_backlight_off(intel_dp); +} + static void ironlake_edp_pll_on(struct intel_dp *intel_dp) { struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp); @@ -1515,8 +1789,6 @@ void intel_dp_sink_dpms(struct intel_dp *intel_dp, int mode) if (mode != DRM_MODE_DPMS_ON) { ret = drm_dp_dpcd_writeb(&intel_dp->aux, DP_SET_POWER, DP_SET_POWER_D3); - if (ret != 1) - DRM_DEBUG_DRIVER("failed to write sink power state\n"); } else { /* * When turning on, we need to retry for 1ms to give the sink @@ -1530,6 +1802,10 @@ void intel_dp_sink_dpms(struct intel_dp *intel_dp, int mode) msleep(1); } } + + if (ret != 1) + DRM_DEBUG_KMS("failed to %s sink power state\n", + mode == DRM_MODE_DPMS_ON ? "enable" : "disable"); } static bool intel_dp_get_hw_state(struct intel_encoder *encoder, @@ -1576,7 +1852,7 @@ static bool intel_dp_get_hw_state(struct intel_encoder *encoder, return true; } - for_each_pipe(i) { + for_each_pipe(dev_priv, i) { trans_dp = I915_READ(TRANS_DP_CTL(i)); if ((trans_dp & TRANS_DP_PORT_SEL_MASK) == trans_sel) { *pipe = i; @@ -2036,7 +2312,6 @@ void intel_edp_psr_init(struct drm_device *dev) static void intel_disable_dp(struct intel_encoder *encoder) { struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base); - enum port port = dp_to_dig_port(intel_dp)->port; struct drm_device *dev = encoder->base.dev; /* Make sure the panel is off before trying to change the mode. But also @@ -2046,21 +2321,19 @@ static void intel_disable_dp(struct intel_encoder *encoder) intel_dp_sink_dpms(intel_dp, DRM_MODE_DPMS_OFF); intel_edp_panel_off(intel_dp); - /* cpu edp my only be disable _after_ the cpu pipe/plane is disabled. */ - if (!(port == PORT_A || IS_VALLEYVIEW(dev))) + /* disable the port before the pipe on g4x */ + if (INTEL_INFO(dev)->gen < 5) intel_dp_link_down(intel_dp); } -static void g4x_post_disable_dp(struct intel_encoder *encoder) +static void ilk_post_disable_dp(struct intel_encoder *encoder) { struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base); enum port port = dp_to_dig_port(intel_dp)->port; - if (port != PORT_A) - return; - intel_dp_link_down(intel_dp); - ironlake_edp_pll_off(intel_dp); + if (port == PORT_A) + ironlake_edp_pll_off(intel_dp); } static void vlv_post_disable_dp(struct intel_encoder *encoder) @@ -2103,7 +2376,105 @@ static void chv_post_disable_dp(struct intel_encoder *encoder) val &= ~(DPIO_PCS_TX_LANE2_RESET | DPIO_PCS_TX_LANE1_RESET); vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW0(ch), val); - mutex_unlock(&dev_priv->dpio_lock); + mutex_unlock(&dev_priv->dpio_lock); +} + +static void +_intel_dp_set_link_train(struct intel_dp *intel_dp, + uint32_t *DP, + uint8_t dp_train_pat) +{ + struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp); + struct drm_device *dev = intel_dig_port->base.base.dev; + struct drm_i915_private *dev_priv = dev->dev_private; + enum port port = intel_dig_port->port; + + if (HAS_DDI(dev)) { + uint32_t temp = I915_READ(DP_TP_CTL(port)); + + if (dp_train_pat & DP_LINK_SCRAMBLING_DISABLE) + temp |= DP_TP_CTL_SCRAMBLE_DISABLE; + else + temp &= ~DP_TP_CTL_SCRAMBLE_DISABLE; + + temp &= ~DP_TP_CTL_LINK_TRAIN_MASK; + switch (dp_train_pat & DP_TRAINING_PATTERN_MASK) { + case DP_TRAINING_PATTERN_DISABLE: + temp |= DP_TP_CTL_LINK_TRAIN_NORMAL; + + break; + case DP_TRAINING_PATTERN_1: + temp |= DP_TP_CTL_LINK_TRAIN_PAT1; + break; + case DP_TRAINING_PATTERN_2: + temp |= DP_TP_CTL_LINK_TRAIN_PAT2; + break; + case DP_TRAINING_PATTERN_3: + temp |= DP_TP_CTL_LINK_TRAIN_PAT3; + break; + } + I915_WRITE(DP_TP_CTL(port), temp); + + } else if (HAS_PCH_CPT(dev) && (IS_GEN7(dev) || port != PORT_A)) { + *DP &= ~DP_LINK_TRAIN_MASK_CPT; + + switch (dp_train_pat & DP_TRAINING_PATTERN_MASK) { + case DP_TRAINING_PATTERN_DISABLE: + *DP |= DP_LINK_TRAIN_OFF_CPT; + break; + case DP_TRAINING_PATTERN_1: + *DP |= DP_LINK_TRAIN_PAT_1_CPT; + break; + case DP_TRAINING_PATTERN_2: + *DP |= DP_LINK_TRAIN_PAT_2_CPT; + break; + case DP_TRAINING_PATTERN_3: + DRM_ERROR("DP training pattern 3 not supported\n"); + *DP |= DP_LINK_TRAIN_PAT_2_CPT; + break; + } + + } else { + if (IS_CHERRYVIEW(dev)) + *DP &= ~DP_LINK_TRAIN_MASK_CHV; + else + *DP &= ~DP_LINK_TRAIN_MASK; + + switch (dp_train_pat & DP_TRAINING_PATTERN_MASK) { + case DP_TRAINING_PATTERN_DISABLE: + *DP |= DP_LINK_TRAIN_OFF; + break; + case DP_TRAINING_PATTERN_1: + *DP |= DP_LINK_TRAIN_PAT_1; + break; + case DP_TRAINING_PATTERN_2: + *DP |= DP_LINK_TRAIN_PAT_2; + break; + case DP_TRAINING_PATTERN_3: + if (IS_CHERRYVIEW(dev)) { + *DP |= DP_LINK_TRAIN_PAT_3_CHV; + } else { + DRM_ERROR("DP training pattern 3 not supported\n"); + *DP |= DP_LINK_TRAIN_PAT_2; + } + break; + } + } +} + +static void intel_dp_enable_port(struct intel_dp *intel_dp) +{ + struct drm_device *dev = intel_dp_to_dev(intel_dp); + struct drm_i915_private *dev_priv = dev->dev_private; + + intel_dp->DP |= DP_PORT_EN; + + /* enable with pattern 1 (as per spec) */ + _intel_dp_set_link_train(intel_dp, &intel_dp->DP, + DP_TRAINING_PATTERN_1); + + I915_WRITE(intel_dp->output_reg, intel_dp->DP); + POSTING_READ(intel_dp->output_reg); } static void intel_enable_dp(struct intel_encoder *encoder) @@ -2116,11 +2487,12 @@ static void intel_enable_dp(struct intel_encoder *encoder) if (WARN_ON(dp_reg & DP_PORT_EN)) return; + intel_dp_enable_port(intel_dp); intel_edp_panel_vdd_on(intel_dp); + intel_edp_panel_on(intel_dp); + intel_edp_panel_vdd_off(intel_dp, true); intel_dp_sink_dpms(intel_dp, DRM_MODE_DPMS_ON); intel_dp_start_link_train(intel_dp); - intel_edp_panel_on(intel_dp); - edp_panel_vdd_off(intel_dp, true); intel_dp_complete_link_train(intel_dp); intel_dp_stop_link_train(intel_dp); } @@ -2154,6 +2526,78 @@ static void g4x_pre_enable_dp(struct intel_encoder *encoder) } } +static void vlv_steal_power_sequencer(struct drm_device *dev, + enum pipe pipe) +{ + struct drm_i915_private *dev_priv = dev->dev_private; + struct intel_encoder *encoder; + + lockdep_assert_held(&dev_priv->pps_mutex); + + list_for_each_entry(encoder, &dev->mode_config.encoder_list, + base.head) { + struct intel_dp *intel_dp; + enum port port; + + if (encoder->type != INTEL_OUTPUT_EDP) + continue; + + intel_dp = enc_to_intel_dp(&encoder->base); + port = dp_to_dig_port(intel_dp)->port; + + if (intel_dp->pps_pipe != pipe) + continue; + + DRM_DEBUG_KMS("stealing pipe %c power sequencer from port %c\n", + pipe_name(pipe), port_name(port)); + + /* make sure vdd is off before we steal it */ + edp_panel_vdd_off_sync(intel_dp); + + intel_dp->pps_pipe = INVALID_PIPE; + } +} + +static void vlv_init_panel_power_sequencer(struct intel_dp *intel_dp) +{ + struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp); + struct intel_encoder *encoder = &intel_dig_port->base; + struct drm_device *dev = encoder->base.dev; + struct drm_i915_private *dev_priv = dev->dev_private; + struct intel_crtc *crtc = to_intel_crtc(encoder->base.crtc); + struct edp_power_seq power_seq; + + lockdep_assert_held(&dev_priv->pps_mutex); + + if (intel_dp->pps_pipe == crtc->pipe) + return; + + /* + * If another power sequencer was being used on this + * port previously make sure to turn off vdd there while + * we still have control of it. + */ + if (intel_dp->pps_pipe != INVALID_PIPE) + edp_panel_vdd_off_sync(intel_dp); + + /* + * We may be stealing the power + * sequencer from another port. + */ + vlv_steal_power_sequencer(dev, crtc->pipe); + + /* now it's all ours */ + intel_dp->pps_pipe = crtc->pipe; + + DRM_DEBUG_KMS("initializing pipe %c power sequencer for port %c\n", + pipe_name(intel_dp->pps_pipe), port_name(intel_dig_port->port)); + + /* init power sequencer on this pipe and port */ + intel_dp_init_panel_power_sequencer(dev, intel_dp, &power_seq); + intel_dp_init_panel_power_sequencer_registers(dev, intel_dp, + &power_seq); +} + static void vlv_pre_enable_dp(struct intel_encoder *encoder) { struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base); @@ -2163,7 +2607,6 @@ static void vlv_pre_enable_dp(struct intel_encoder *encoder) struct intel_crtc *intel_crtc = to_intel_crtc(encoder->base.crtc); enum dpio_channel port = vlv_dport_to_channel(dport); int pipe = intel_crtc->pipe; - struct edp_power_seq power_seq; u32 val; mutex_lock(&dev_priv->dpio_lock); @@ -2182,10 +2625,9 @@ static void vlv_pre_enable_dp(struct intel_encoder *encoder) mutex_unlock(&dev_priv->dpio_lock); if (is_edp(intel_dp)) { - /* init power sequencer on this pipe and port */ - intel_dp_init_panel_power_sequencer(dev, intel_dp, &power_seq); - intel_dp_init_panel_power_sequencer_registers(dev, intel_dp, - &power_seq); + pps_lock(intel_dp); + vlv_init_panel_power_sequencer(intel_dp); + pps_unlock(intel_dp); } intel_enable_dp(encoder); @@ -2229,7 +2671,6 @@ static void chv_pre_enable_dp(struct intel_encoder *encoder) struct intel_digital_port *dport = dp_to_dig_port(intel_dp); struct drm_device *dev = encoder->base.dev; struct drm_i915_private *dev_priv = dev->dev_private; - struct edp_power_seq power_seq; struct intel_crtc *intel_crtc = to_intel_crtc(encoder->base.crtc); enum dpio_channel ch = vlv_dport_to_channel(dport); @@ -2275,10 +2716,9 @@ static void chv_pre_enable_dp(struct intel_encoder *encoder) mutex_unlock(&dev_priv->dpio_lock); if (is_edp(intel_dp)) { - /* init power sequencer on this pipe and port */ - intel_dp_init_panel_power_sequencer(dev, intel_dp, &power_seq); - intel_dp_init_panel_power_sequencer_registers(dev, intel_dp, - &power_seq); + pps_lock(intel_dp); + vlv_init_panel_power_sequencer(intel_dp); + pps_unlock(intel_dp); } intel_enable_dp(encoder); @@ -2297,6 +2737,8 @@ static void chv_dp_pre_pll_enable(struct intel_encoder *encoder) enum pipe pipe = intel_crtc->pipe; u32 val; + intel_dp_prepare(encoder); + mutex_lock(&dev_priv->dpio_lock); /* program left/right clock distribution */ @@ -2663,8 +3105,8 @@ static uint32_t intel_chv_signal_levels(struct intel_dp *intel_dp) /* Program swing margin */ for (i = 0; i < 4; i++) { val = vlv_dpio_read(dev_priv, pipe, CHV_TX_DW2(ch, i)); - val &= ~DPIO_SWING_MARGIN_MASK; - val |= margin_reg_value << DPIO_SWING_MARGIN_SHIFT; + val &= ~DPIO_SWING_MARGIN000_MASK; + val |= margin_reg_value << DPIO_SWING_MARGIN000_SHIFT; vlv_dpio_write(dev_priv, pipe, CHV_TX_DW2(ch, i), val); } @@ -2856,29 +3298,29 @@ intel_hsw_signal_levels(uint8_t train_set) DP_TRAIN_PRE_EMPHASIS_MASK); switch (signal_levels) { case DP_TRAIN_VOLTAGE_SWING_400 | DP_TRAIN_PRE_EMPHASIS_0: - return DDI_BUF_EMP_400MV_0DB_HSW; + return DDI_BUF_TRANS_SELECT(0); case DP_TRAIN_VOLTAGE_SWING_400 | DP_TRAIN_PRE_EMPHASIS_3_5: - return DDI_BUF_EMP_400MV_3_5DB_HSW; + return DDI_BUF_TRANS_SELECT(1); case DP_TRAIN_VOLTAGE_SWING_400 | DP_TRAIN_PRE_EMPHASIS_6: - return DDI_BUF_EMP_400MV_6DB_HSW; + return DDI_BUF_TRANS_SELECT(2); case DP_TRAIN_VOLTAGE_SWING_400 | DP_TRAIN_PRE_EMPHASIS_9_5: - return DDI_BUF_EMP_400MV_9_5DB_HSW; + return DDI_BUF_TRANS_SELECT(3); case DP_TRAIN_VOLTAGE_SWING_600 | DP_TRAIN_PRE_EMPHASIS_0: - return DDI_BUF_EMP_600MV_0DB_HSW; + return DDI_BUF_TRANS_SELECT(4); case DP_TRAIN_VOLTAGE_SWING_600 | DP_TRAIN_PRE_EMPHASIS_3_5: - return DDI_BUF_EMP_600MV_3_5DB_HSW; + return DDI_BUF_TRANS_SELECT(5); case DP_TRAIN_VOLTAGE_SWING_600 | DP_TRAIN_PRE_EMPHASIS_6: - return DDI_BUF_EMP_600MV_6DB_HSW; + return DDI_BUF_TRANS_SELECT(6); case DP_TRAIN_VOLTAGE_SWING_800 | DP_TRAIN_PRE_EMPHASIS_0: - return DDI_BUF_EMP_800MV_0DB_HSW; + return DDI_BUF_TRANS_SELECT(7); case DP_TRAIN_VOLTAGE_SWING_800 | DP_TRAIN_PRE_EMPHASIS_3_5: - return DDI_BUF_EMP_800MV_3_5DB_HSW; + return DDI_BUF_TRANS_SELECT(8); default: DRM_DEBUG_KMS("Unsupported voltage swing/pre-emphasis level:" "0x%x\n", signal_levels); - return DDI_BUF_EMP_400MV_0DB_HSW; + return DDI_BUF_TRANS_SELECT(0); } } @@ -2925,74 +3367,10 @@ intel_dp_set_link_train(struct intel_dp *intel_dp, struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp); struct drm_device *dev = intel_dig_port->base.base.dev; struct drm_i915_private *dev_priv = dev->dev_private; - enum port port = intel_dig_port->port; uint8_t buf[sizeof(intel_dp->train_set) + 1]; int ret, len; - if (HAS_DDI(dev)) { - uint32_t temp = I915_READ(DP_TP_CTL(port)); - - if (dp_train_pat & DP_LINK_SCRAMBLING_DISABLE) - temp |= DP_TP_CTL_SCRAMBLE_DISABLE; - else - temp &= ~DP_TP_CTL_SCRAMBLE_DISABLE; - - temp &= ~DP_TP_CTL_LINK_TRAIN_MASK; - switch (dp_train_pat & DP_TRAINING_PATTERN_MASK) { - case DP_TRAINING_PATTERN_DISABLE: - temp |= DP_TP_CTL_LINK_TRAIN_NORMAL; - - break; - case DP_TRAINING_PATTERN_1: - temp |= DP_TP_CTL_LINK_TRAIN_PAT1; - break; - case DP_TRAINING_PATTERN_2: - temp |= DP_TP_CTL_LINK_TRAIN_PAT2; - break; - case DP_TRAINING_PATTERN_3: - temp |= DP_TP_CTL_LINK_TRAIN_PAT3; - break; - } - I915_WRITE(DP_TP_CTL(port), temp); - - } else if (HAS_PCH_CPT(dev) && (IS_GEN7(dev) || port != PORT_A)) { - *DP &= ~DP_LINK_TRAIN_MASK_CPT; - - switch (dp_train_pat & DP_TRAINING_PATTERN_MASK) { - case DP_TRAINING_PATTERN_DISABLE: - *DP |= DP_LINK_TRAIN_OFF_CPT; - break; - case DP_TRAINING_PATTERN_1: - *DP |= DP_LINK_TRAIN_PAT_1_CPT; - break; - case DP_TRAINING_PATTERN_2: - *DP |= DP_LINK_TRAIN_PAT_2_CPT; - break; - case DP_TRAINING_PATTERN_3: - DRM_ERROR("DP training pattern 3 not supported\n"); - *DP |= DP_LINK_TRAIN_PAT_2_CPT; - break; - } - - } else { - *DP &= ~DP_LINK_TRAIN_MASK; - - switch (dp_train_pat & DP_TRAINING_PATTERN_MASK) { - case DP_TRAINING_PATTERN_DISABLE: - *DP |= DP_LINK_TRAIN_OFF; - break; - case DP_TRAINING_PATTERN_1: - *DP |= DP_LINK_TRAIN_PAT_1; - break; - case DP_TRAINING_PATTERN_2: - *DP |= DP_LINK_TRAIN_PAT_2; - break; - case DP_TRAINING_PATTERN_3: - DRM_ERROR("DP training pattern 3 not supported\n"); - *DP |= DP_LINK_TRAIN_PAT_2; - break; - } - } + _intel_dp_set_link_train(intel_dp, DP, dp_train_pat); I915_WRITE(intel_dp->output_reg, *DP); POSTING_READ(intel_dp->output_reg); @@ -3276,7 +3654,10 @@ intel_dp_link_down(struct intel_dp *intel_dp) DP &= ~DP_LINK_TRAIN_MASK_CPT; I915_WRITE(intel_dp->output_reg, DP | DP_LINK_TRAIN_PAT_IDLE_CPT); } else { - DP &= ~DP_LINK_TRAIN_MASK; + if (IS_CHERRYVIEW(dev)) + DP &= ~DP_LINK_TRAIN_MASK_CHV; + else + DP &= ~DP_LINK_TRAIN_MASK; I915_WRITE(intel_dp->output_reg, DP | DP_LINK_TRAIN_PAT_IDLE); } POSTING_READ(intel_dp->output_reg); @@ -3322,15 +3703,11 @@ intel_dp_get_dpcd(struct intel_dp *intel_dp) struct drm_device *dev = dig_port->base.base.dev; struct drm_i915_private *dev_priv = dev->dev_private; - char dpcd_hex_dump[sizeof(intel_dp->dpcd) * 3]; - if (intel_dp_dpcd_read_wake(&intel_dp->aux, 0x000, intel_dp->dpcd, sizeof(intel_dp->dpcd)) < 0) return false; /* aux transfer failed */ - hex_dump_to_buffer(intel_dp->dpcd, sizeof(intel_dp->dpcd), - 32, 1, dpcd_hex_dump, sizeof(dpcd_hex_dump), false); - DRM_DEBUG_KMS("DPCD: %s\n", dpcd_hex_dump); + DRM_DEBUG_KMS("DPCD: %*ph\n", (int) sizeof(intel_dp->dpcd), intel_dp->dpcd); if (intel_dp->dpcd[DP_DPCD_REV] == 0) return false; /* DPCD not present */ @@ -3351,7 +3728,7 @@ intel_dp_get_dpcd(struct intel_dp *intel_dp) if (intel_dp->dpcd[DP_DPCD_REV] >= 0x12 && intel_dp->dpcd[DP_MAX_LANE_COUNT] & DP_TPS3_SUPPORTED) { intel_dp->use_tps3 = true; - DRM_DEBUG_KMS("Displayport TPS3 supported"); + DRM_DEBUG_KMS("Displayport TPS3 supported\n"); } else intel_dp->use_tps3 = false; @@ -3388,7 +3765,7 @@ intel_dp_probe_oui(struct intel_dp *intel_dp) DRM_DEBUG_KMS("Branch OUI: %02hx%02hx%02hx\n", buf[0], buf[1], buf[2]); - edp_panel_vdd_off(intel_dp, false); + intel_edp_panel_vdd_off(intel_dp, false); } static bool @@ -3402,7 +3779,7 @@ intel_dp_probe_mst(struct intel_dp *intel_dp) if (intel_dp->dpcd[DP_DPCD_REV] < 0x12) return false; - _edp_panel_vdd_on(intel_dp); + intel_edp_panel_vdd_on(intel_dp); if (intel_dp_dpcd_read_wake(&intel_dp->aux, DP_MSTM_CAP, buf, 1)) { if (buf[0] & DP_MST_CAP) { DRM_DEBUG_KMS("Sink is MST capable\n"); @@ -3412,7 +3789,7 @@ intel_dp_probe_mst(struct intel_dp *intel_dp) intel_dp->is_mst = false; } } - edp_panel_vdd_off(intel_dp, false); + intel_edp_panel_vdd_off(intel_dp, false); drm_dp_mst_topology_mgr_set_mst(&intel_dp->mst_mgr, intel_dp->is_mst); return intel_dp->is_mst; @@ -3427,21 +3804,21 @@ int intel_dp_sink_crc(struct intel_dp *intel_dp, u8 *crc) u8 buf[1]; if (drm_dp_dpcd_readb(&intel_dp->aux, DP_TEST_SINK_MISC, buf) < 0) - return -EAGAIN; + return -EIO; if (!(buf[0] & DP_TEST_CRC_SUPPORTED)) return -ENOTTY; if (drm_dp_dpcd_writeb(&intel_dp->aux, DP_TEST_SINK, DP_TEST_SINK_START) < 0) - return -EAGAIN; + return -EIO; /* Wait 2 vblanks to be sure we will have the correct CRC value */ intel_wait_for_vblank(dev, intel_crtc->pipe); intel_wait_for_vblank(dev, intel_crtc->pipe); if (drm_dp_dpcd_read(&intel_dp->aux, DP_TEST_CRC_R_CR, crc, 6) < 0) - return -EAGAIN; + return -EIO; drm_dp_dpcd_writeb(&intel_dp->aux, DP_TEST_SINK, 0); return 0; @@ -3644,20 +4021,24 @@ intel_dp_detect_dpcd(struct intel_dp *intel_dp) } static enum drm_connector_status +edp_detect(struct intel_dp *intel_dp) +{ + struct drm_device *dev = intel_dp_to_dev(intel_dp); + enum drm_connector_status status; + + status = intel_panel_detect(dev); + if (status == connector_status_unknown) + status = connector_status_connected; + + return status; +} + +static enum drm_connector_status ironlake_dp_detect(struct intel_dp *intel_dp) { struct drm_device *dev = intel_dp_to_dev(intel_dp); struct drm_i915_private *dev_priv = dev->dev_private; struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp); - enum drm_connector_status status; - - /* Can't disconnect eDP, but you can close the lid... */ - if (is_edp(intel_dp)) { - status = intel_panel_detect(dev); - if (status == connector_status_unknown) - status = connector_status_connected; - return status; - } if (!ibx_digital_port_connected(dev_priv, intel_dig_port)) return connector_status_disconnected; @@ -3733,9 +4114,9 @@ g4x_dp_detect(struct intel_dp *intel_dp) } static struct edid * -intel_dp_get_edid(struct drm_connector *connector, struct i2c_adapter *adapter) +intel_dp_get_edid(struct intel_dp *intel_dp) { - struct intel_connector *intel_connector = to_intel_connector(connector); + struct intel_connector *intel_connector = intel_dp->attached_connector; /* use cached edid if we have one */ if (intel_connector->edid) { @@ -3744,27 +4125,55 @@ intel_dp_get_edid(struct drm_connector *connector, struct i2c_adapter *adapter) return NULL; return drm_edid_duplicate(intel_connector->edid); - } + } else + return drm_get_edid(&intel_connector->base, + &intel_dp->aux.ddc); +} + +static void +intel_dp_set_edid(struct intel_dp *intel_dp) +{ + struct intel_connector *intel_connector = intel_dp->attached_connector; + struct edid *edid; - return drm_get_edid(connector, adapter); + edid = intel_dp_get_edid(intel_dp); + intel_connector->detect_edid = edid; + + if (intel_dp->force_audio != HDMI_AUDIO_AUTO) + intel_dp->has_audio = intel_dp->force_audio == HDMI_AUDIO_ON; + else + intel_dp->has_audio = drm_detect_monitor_audio(edid); } -static int -intel_dp_get_edid_modes(struct drm_connector *connector, struct i2c_adapter *adapter) +static void +intel_dp_unset_edid(struct intel_dp *intel_dp) { - struct intel_connector *intel_connector = to_intel_connector(connector); + struct intel_connector *intel_connector = intel_dp->attached_connector; - /* use cached edid if we have one */ - if (intel_connector->edid) { - /* invalid edid */ - if (IS_ERR(intel_connector->edid)) - return 0; + kfree(intel_connector->detect_edid); + intel_connector->detect_edid = NULL; - return intel_connector_update_modes(connector, - intel_connector->edid); - } + intel_dp->has_audio = false; +} + +static enum intel_display_power_domain +intel_dp_power_get(struct intel_dp *dp) +{ + struct intel_encoder *encoder = &dp_to_dig_port(dp)->base; + enum intel_display_power_domain power_domain; + + power_domain = intel_display_port_power_domain(encoder); + intel_display_power_get(to_i915(encoder->base.dev), power_domain); + + return power_domain; +} - return intel_ddc_get_modes(connector, adapter); +static void +intel_dp_power_put(struct intel_dp *dp, + enum intel_display_power_domain power_domain) +{ + struct intel_encoder *encoder = &dp_to_dig_port(dp)->base; + intel_display_power_put(to_i915(encoder->base.dev), power_domain); } static enum drm_connector_status @@ -3774,33 +4183,30 @@ intel_dp_detect(struct drm_connector *connector, bool force) struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp); struct intel_encoder *intel_encoder = &intel_dig_port->base; struct drm_device *dev = connector->dev; - struct drm_i915_private *dev_priv = dev->dev_private; enum drm_connector_status status; enum intel_display_power_domain power_domain; - struct edid *edid = NULL; bool ret; - power_domain = intel_display_port_power_domain(intel_encoder); - intel_display_power_get(dev_priv, power_domain); - DRM_DEBUG_KMS("[CONNECTOR:%d:%s]\n", connector->base.id, connector->name); + intel_dp_unset_edid(intel_dp); if (intel_dp->is_mst) { /* MST devices are disconnected from a monitor POV */ if (intel_encoder->type != INTEL_OUTPUT_EDP) intel_encoder->type = INTEL_OUTPUT_DISPLAYPORT; - status = connector_status_disconnected; - goto out; + return connector_status_disconnected; } - intel_dp->has_audio = false; + power_domain = intel_dp_power_get(intel_dp); - if (HAS_PCH_SPLIT(dev)) + /* Can't disconnect eDP, but you can close the lid... */ + if (is_edp(intel_dp)) + status = edp_detect(intel_dp); + else if (HAS_PCH_SPLIT(dev)) status = ironlake_dp_detect(intel_dp); else status = g4x_dp_detect(intel_dp); - if (status != connector_status_connected) goto out; @@ -3816,82 +4222,78 @@ intel_dp_detect(struct drm_connector *connector, bool force) goto out; } - if (intel_dp->force_audio != HDMI_AUDIO_AUTO) { - intel_dp->has_audio = (intel_dp->force_audio == HDMI_AUDIO_ON); - } else { - edid = intel_dp_get_edid(connector, &intel_dp->aux.ddc); - if (edid) { - intel_dp->has_audio = drm_detect_monitor_audio(edid); - kfree(edid); - } - } + intel_dp_set_edid(intel_dp); if (intel_encoder->type != INTEL_OUTPUT_EDP) intel_encoder->type = INTEL_OUTPUT_DISPLAYPORT; status = connector_status_connected; out: - intel_display_power_put(dev_priv, power_domain); + intel_dp_power_put(intel_dp, power_domain); return status; } -static int intel_dp_get_modes(struct drm_connector *connector) +static void +intel_dp_force(struct drm_connector *connector) { struct intel_dp *intel_dp = intel_attached_dp(connector); - struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp); - struct intel_encoder *intel_encoder = &intel_dig_port->base; - struct intel_connector *intel_connector = to_intel_connector(connector); - struct drm_device *dev = connector->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct intel_encoder *intel_encoder = &dp_to_dig_port(intel_dp)->base; enum intel_display_power_domain power_domain; - int ret; - /* We should parse the EDID data and find out if it has an audio sink - */ + DRM_DEBUG_KMS("[CONNECTOR:%d:%s]\n", + connector->base.id, connector->name); + intel_dp_unset_edid(intel_dp); - power_domain = intel_display_port_power_domain(intel_encoder); - intel_display_power_get(dev_priv, power_domain); + if (connector->status != connector_status_connected) + return; - ret = intel_dp_get_edid_modes(connector, &intel_dp->aux.ddc); - intel_display_power_put(dev_priv, power_domain); - if (ret) - return ret; + power_domain = intel_dp_power_get(intel_dp); + + intel_dp_set_edid(intel_dp); + + intel_dp_power_put(intel_dp, power_domain); + + if (intel_encoder->type != INTEL_OUTPUT_EDP) + intel_encoder->type = INTEL_OUTPUT_DISPLAYPORT; +} + +static int intel_dp_get_modes(struct drm_connector *connector) +{ + struct intel_connector *intel_connector = to_intel_connector(connector); + struct edid *edid; + + edid = intel_connector->detect_edid; + if (edid) { + int ret = intel_connector_update_modes(connector, edid); + if (ret) + return ret; + } /* if eDP has no EDID, fall back to fixed mode */ - if (is_edp(intel_dp) && intel_connector->panel.fixed_mode) { + if (is_edp(intel_attached_dp(connector)) && + intel_connector->panel.fixed_mode) { struct drm_display_mode *mode; - mode = drm_mode_duplicate(dev, + + mode = drm_mode_duplicate(connector->dev, intel_connector->panel.fixed_mode); if (mode) { drm_mode_probed_add(connector, mode); return 1; } } + return 0; } static bool intel_dp_detect_audio(struct drm_connector *connector) { - struct intel_dp *intel_dp = intel_attached_dp(connector); - struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp); - struct intel_encoder *intel_encoder = &intel_dig_port->base; - struct drm_device *dev = connector->dev; - struct drm_i915_private *dev_priv = dev->dev_private; - enum intel_display_power_domain power_domain; - struct edid *edid; bool has_audio = false; + struct edid *edid; - power_domain = intel_display_port_power_domain(intel_encoder); - intel_display_power_get(dev_priv, power_domain); - - edid = intel_dp_get_edid(connector, &intel_dp->aux.ddc); - if (edid) { + edid = to_intel_connector(connector)->detect_edid; + if (edid) has_audio = drm_detect_monitor_audio(edid); - kfree(edid); - } - - intel_display_power_put(dev_priv, power_domain); return has_audio; } @@ -3989,6 +4391,8 @@ intel_dp_connector_destroy(struct drm_connector *connector) { struct intel_connector *intel_connector = to_intel_connector(connector); + kfree(intel_connector->detect_edid); + if (!IS_ERR_OR_NULL(intel_connector->edid)) kfree(intel_connector->edid); @@ -4005,16 +4409,20 @@ void intel_dp_encoder_destroy(struct drm_encoder *encoder) { struct intel_digital_port *intel_dig_port = enc_to_dig_port(encoder); struct intel_dp *intel_dp = &intel_dig_port->dp; - struct drm_device *dev = intel_dp_to_dev(intel_dp); drm_dp_aux_unregister(&intel_dp->aux); intel_dp_mst_encoder_cleanup(intel_dig_port); drm_encoder_cleanup(encoder); if (is_edp(intel_dp)) { cancel_delayed_work_sync(&intel_dp->panel_vdd_work); - drm_modeset_lock(&dev->mode_config.connection_mutex, NULL); + /* + * vdd might still be enabled do to the delayed vdd off. + * Make sure vdd is actually turned off here. + */ + pps_lock(intel_dp); edp_panel_vdd_off_sync(intel_dp); - drm_modeset_unlock(&dev->mode_config.connection_mutex); + pps_unlock(intel_dp); + if (intel_dp->edp_notifier.notifier_call) { unregister_reboot_notifier(&intel_dp->edp_notifier); intel_dp->edp_notifier.notifier_call = NULL; @@ -4030,7 +4438,13 @@ static void intel_dp_encoder_suspend(struct intel_encoder *intel_encoder) if (!is_edp(intel_dp)) return; + /* + * vdd might still be enabled do to the delayed vdd off. + * Make sure vdd is actually turned off here. + */ + pps_lock(intel_dp); edp_panel_vdd_off_sync(intel_dp); + pps_unlock(intel_dp); } static void intel_dp_encoder_reset(struct drm_encoder *encoder) @@ -4041,6 +4455,7 @@ static void intel_dp_encoder_reset(struct drm_encoder *encoder) static const struct drm_connector_funcs intel_dp_connector_funcs = { .dpms = intel_connector_dpms, .detect = intel_dp_detect, + .force = intel_dp_force, .fill_modes = drm_helper_probe_single_connector_modes, .set_property = intel_dp_set_property, .destroy = intel_dp_connector_destroy, @@ -4076,7 +4491,8 @@ intel_dp_hpd_pulse(struct intel_digital_port *intel_dig_port, bool long_hpd) if (intel_dig_port->base.type != INTEL_OUTPUT_EDP) intel_dig_port->base.type = INTEL_OUTPUT_DISPLAYPORT; - DRM_DEBUG_KMS("got hpd irq on port %d - %s\n", intel_dig_port->port, + DRM_DEBUG_KMS("got hpd irq on port %c - %s\n", + port_name(intel_dig_port->port), long_hpd ? "long" : "short"); power_domain = intel_display_port_power_domain(intel_encoder); @@ -4216,6 +4632,8 @@ intel_dp_init_panel_power_sequencer(struct drm_device *dev, u32 pp_on, pp_off, pp_div, pp; int pp_ctrl_reg, pp_on_reg, pp_off_reg, pp_div_reg; + lockdep_assert_held(&dev_priv->pps_mutex); + if (HAS_PCH_SPLIT(dev)) { pp_ctrl_reg = PCH_PP_CONTROL; pp_on_reg = PCH_PP_ON_DELAYS; @@ -4315,6 +4733,9 @@ intel_dp_init_panel_power_sequencer_registers(struct drm_device *dev, u32 pp_on, pp_off, pp_div, port_sel = 0; int div = HAS_PCH_SPLIT(dev) ? intel_pch_rawclk(dev) : intel_hrawclk(dev); int pp_on_reg, pp_off_reg, pp_div_reg; + enum port port = dp_to_dig_port(intel_dp)->port; + + lockdep_assert_held(&dev_priv->pps_mutex); if (HAS_PCH_SPLIT(dev)) { pp_on_reg = PCH_PP_ON_DELAYS; @@ -4349,12 +4770,9 @@ intel_dp_init_panel_power_sequencer_registers(struct drm_device *dev, /* Haswell doesn't have any port selection bits for the panel * power sequencer any more. */ if (IS_VALLEYVIEW(dev)) { - if (dp_to_dig_port(intel_dp)->port == PORT_B) - port_sel = PANEL_PORT_SELECT_DPB_VLV; - else - port_sel = PANEL_PORT_SELECT_DPC_VLV; + port_sel = PANEL_PORT_SELECT_VLV(port); } else if (HAS_PCH_IBX(dev) || HAS_PCH_CPT(dev)) { - if (dp_to_dig_port(intel_dp)->port == PORT_A) + if (port == PORT_A) port_sel = PANEL_PORT_SELECT_DPA; else port_sel = PANEL_PORT_SELECT_DPD; @@ -4438,7 +4856,7 @@ void intel_dp_set_drrs_state(struct drm_device *dev, int refresh_rate) val = I915_READ(reg); if (index > DRRS_HIGH_RR) { val |= PIPECONF_EDP_RR_MODE_SWITCH; - intel_dp_set_m2_n2(intel_crtc, &config->dp_m2_n2); + intel_dp_set_m_n(intel_crtc); } else { val &= ~PIPECONF_EDP_RR_MODE_SWITCH; } @@ -4478,7 +4896,7 @@ intel_dp_drrs_init(struct intel_digital_port *intel_dig_port, } if (dev_priv->vbt.drrs_type != SEAMLESS_DRRS_SUPPORT) { - DRM_INFO("VBT doesn't support DRRS\n"); + DRM_DEBUG_KMS("VBT doesn't support DRRS\n"); return NULL; } @@ -4486,7 +4904,7 @@ intel_dp_drrs_init(struct intel_digital_port *intel_dig_port, (dev, fixed_mode, connector); if (!downclock_mode) { - DRM_INFO("DRRS not supported\n"); + DRM_DEBUG_KMS("DRRS not supported\n"); return NULL; } @@ -4497,7 +4915,7 @@ intel_dp_drrs_init(struct intel_digital_port *intel_dig_port, intel_dp->drrs_state.type = dev_priv->vbt.drrs_type; intel_dp->drrs_state.refresh_rate_type = DRRS_HIGH_RR; - DRM_INFO("seamless DRRS supported for eDP panel.\n"); + DRM_DEBUG_KMS("seamless DRRS supported for eDP panel.\n"); return downclock_mode; } @@ -4512,8 +4930,11 @@ void intel_edp_panel_vdd_sanitize(struct intel_encoder *intel_encoder) return; intel_dp = enc_to_intel_dp(&intel_encoder->base); + + pps_lock(intel_dp); + if (!edp_have_panel_vdd(intel_dp)) - return; + goto out; /* * The VDD bit needs a power domain reference, so if the bit is * already enabled when we boot or resume, grab this reference and @@ -4525,6 +4946,8 @@ void intel_edp_panel_vdd_sanitize(struct intel_encoder *intel_encoder) intel_display_power_get(dev_priv, power_domain); edp_panel_vdd_schedule_off(intel_dp); + out: + pps_unlock(intel_dp); } static bool intel_edp_init_connector(struct intel_dp *intel_dp, @@ -4552,7 +4975,7 @@ static bool intel_edp_init_connector(struct intel_dp *intel_dp, /* Cache DPCD and EDID for edp. */ intel_edp_panel_vdd_on(intel_dp); has_dpcd = intel_dp_get_dpcd(intel_dp); - edp_panel_vdd_off(intel_dp, false); + intel_edp_panel_vdd_off(intel_dp, false); if (has_dpcd) { if (intel_dp->dpcd[DP_DPCD_REV] >= 0x11) @@ -4566,7 +4989,9 @@ static bool intel_edp_init_connector(struct intel_dp *intel_dp, } /* We now know it's not a ghost, init power sequence regs. */ + pps_lock(intel_dp); intel_dp_init_panel_power_sequencer_registers(dev, intel_dp, power_seq); + pps_unlock(intel_dp); mutex_lock(&dev->mode_config.mutex); edid = drm_get_edid(connector, &intel_dp->aux.ddc); @@ -4610,6 +5035,7 @@ static bool intel_edp_init_connector(struct intel_dp *intel_dp, } intel_panel_init(&intel_connector->panel, fixed_mode, downclock_mode); + intel_connector->panel.backlight_power = intel_edp_backlight_power; intel_panel_setup_backlight(connector); return true; @@ -4628,6 +5054,8 @@ intel_dp_init_connector(struct intel_digital_port *intel_dig_port, struct edp_power_seq power_seq = { 0 }; int type; + intel_dp->pps_pipe = INVALID_PIPE; + /* intel_dp vfuncs */ if (IS_VALLEYVIEW(dev)) intel_dp->get_aux_clock_divider = vlv_get_aux_clock_divider; @@ -4698,8 +5126,15 @@ intel_dp_init_connector(struct intel_digital_port *intel_dig_port, } if (is_edp(intel_dp)) { - intel_dp_init_panel_power_timestamps(intel_dp); - intel_dp_init_panel_power_sequencer(dev, intel_dp, &power_seq); + pps_lock(intel_dp); + if (IS_VALLEYVIEW(dev)) { + vlv_initial_power_sequencer_setup(intel_dp); + } else { + intel_dp_init_panel_power_timestamps(intel_dp); + intel_dp_init_panel_power_sequencer(dev, intel_dp, + &power_seq); + } + pps_unlock(intel_dp); } intel_dp_aux_init(intel_dp, intel_connector); @@ -4707,7 +5142,8 @@ intel_dp_init_connector(struct intel_digital_port *intel_dig_port, /* init MST on ports that can support it */ if (IS_HASWELL(dev) || IS_BROADWELL(dev)) { if (port == PORT_B || port == PORT_C || port == PORT_D) { - intel_dp_mst_encoder_init(intel_dig_port, intel_connector->base.base.id); + intel_dp_mst_encoder_init(intel_dig_port, + intel_connector->base.base.id); } } @@ -4715,9 +5151,13 @@ intel_dp_init_connector(struct intel_digital_port *intel_dig_port, drm_dp_aux_unregister(&intel_dp->aux); if (is_edp(intel_dp)) { cancel_delayed_work_sync(&intel_dp->panel_vdd_work); - drm_modeset_lock(&dev->mode_config.connection_mutex, NULL); + /* + * vdd might still be enabled do to the delayed vdd off. + * Make sure vdd is actually turned off here. + */ + pps_lock(intel_dp); edp_panel_vdd_off_sync(intel_dp); - drm_modeset_unlock(&dev->mode_config.connection_mutex); + pps_unlock(intel_dp); } drm_connector_unregister(connector); drm_connector_cleanup(connector); @@ -4781,7 +5221,8 @@ intel_dp_init(struct drm_device *dev, int output_reg, enum port port) } else { intel_encoder->pre_enable = g4x_pre_enable_dp; intel_encoder->enable = g4x_enable_dp; - intel_encoder->post_disable = g4x_post_disable_dp; + if (INTEL_INFO(dev)->gen >= 5) + intel_encoder->post_disable = ilk_post_disable_dp; } intel_dig_port->port = port; diff --git a/drivers/gpu/drm/i915/intel_drv.h b/drivers/gpu/drm/i915/intel_drv.h index b8c8bbd..4ba59c2 100644 --- a/drivers/gpu/drm/i915/intel_drv.h +++ b/drivers/gpu/drm/i915/intel_drv.h @@ -25,6 +25,7 @@ #ifndef __INTEL_DRV_H__ #define __INTEL_DRV_H__ +#include #include #include #include @@ -33,6 +34,7 @@ #include #include #include +#include /** * _wait_for - magic (register) wait macro @@ -179,6 +181,8 @@ struct intel_panel { bool active_low_pwm; struct backlight_device *device; } backlight; + + void (*backlight_power)(struct intel_connector *, bool enable); }; struct intel_connector { @@ -211,6 +215,7 @@ struct intel_connector { /* Cached EDID for eDP and LVDS. May hold ERR_PTR for invalid EDID. */ struct edid *edid; + struct edid *detect_edid; /* since POLL and HPD connectors may use the same HPD line keep the native state of connector->polled in case hotplug storm detection changes it */ @@ -233,6 +238,17 @@ typedef struct dpll { int p; } intel_clock_t; +struct intel_plane_state { + struct drm_crtc *crtc; + struct drm_framebuffer *fb; + struct drm_rect src; + struct drm_rect dst; + struct drm_rect clip; + struct drm_rect orig_src; + struct drm_rect orig_dst; + bool visible; +}; + struct intel_plane_config { bool tiled; int size; @@ -330,6 +346,7 @@ struct intel_crtc_config { /* m2_n2 for eDP downclock */ struct intel_link_m_n dp_m2_n2; + bool has_drrs; /* * Frequence the dpll for the port should run at. Differs from the @@ -377,11 +394,6 @@ struct intel_pipe_wm { bool sprites_scaled; }; -struct intel_mmio_flip { - u32 seqno; - u32 ring_id; -}; - struct intel_crtc { struct drm_crtc base; enum pipe pipe; @@ -410,6 +422,7 @@ struct intel_crtc { uint32_t cursor_addr; int16_t cursor_width, cursor_height; uint32_t cursor_cntl; + uint32_t cursor_size; uint32_t cursor_base; struct intel_plane_config plane_config; @@ -430,10 +443,8 @@ struct intel_crtc { struct intel_pipe_wm active; } wm; - wait_queue_head_t vbl_wait; - int scanline_offset; - struct intel_mmio_flip mmio_flip; + struct i915_gem_request *mmio_flip; }; struct intel_plane_wm_parameters { @@ -455,6 +466,7 @@ struct intel_plane { unsigned int crtc_w, crtc_h; uint32_t src_x, src_y; uint32_t src_w, src_h; + unsigned int rotation; /* Since we need to change the watermarks before/after * enabling/disabling the planes, we need to store the parameters here @@ -565,6 +577,12 @@ struct intel_dp { struct notifier_block edp_notifier; + /* + * Pipe whose power sequencer is currently locked into + * this port. Only relevant on VLV/CHV. + */ + enum pipe pps_pipe; + bool use_tps3; bool can_mst; /* this port supports mst */ bool is_mst; @@ -663,6 +681,9 @@ struct intel_unpin_work { #define INTEL_FLIP_COMPLETE 2 u32 flip_count; u32 gtt_offset; + struct i915_gem_request *flip_queued_request; + int flip_queued_vblank; + int flip_ready_vblank; bool enable_stall_check; }; @@ -780,7 +801,7 @@ bool intel_has_pending_fb_unpin(struct drm_device *dev); int intel_pch_rawclk(struct drm_device *dev); void intel_mark_busy(struct drm_device *dev); void intel_fb_obj_invalidate(struct drm_i915_gem_object *obj, - struct intel_engine_cs *ring); + struct i915_gem_request *rq); void intel_frontbuffer_flip_prepare(struct drm_device *dev, unsigned frontbuffer_bits); void intel_frontbuffer_flip_complete(struct drm_device *dev, @@ -827,7 +848,6 @@ int intel_get_pipe_from_crtc_id(struct drm_device *dev, void *data, enum transcoder intel_pipe_to_cpu_transcoder(struct drm_i915_private *dev_priv, enum pipe pipe); void intel_wait_for_vblank(struct drm_device *dev, int pipe); -void intel_wait_for_pipe_off(struct drm_device *dev, int pipe); int ironlake_get_lanes_required(int target_clock, int link_bw, int bpp); void vlv_wait_port_ready(struct drm_i915_private *dev_priv, struct intel_digital_port *dport); @@ -839,7 +859,7 @@ void intel_release_load_detect_pipe(struct drm_connector *connector, struct intel_load_detect_pipe *old); int intel_pin_and_fence_fb_obj(struct drm_device *dev, struct drm_i915_gem_object *obj, - struct intel_engine_cs *pipelined); + struct i915_gem_request *pipelined); void intel_unpin_fb_obj(struct drm_i915_gem_object *obj); struct drm_framebuffer * __intel_framebuffer_create(struct drm_device *dev, @@ -848,6 +868,7 @@ __intel_framebuffer_create(struct drm_device *dev, void intel_prepare_page_flip(struct drm_device *dev, int plane); void intel_finish_page_flip(struct drm_device *dev, int pipe); void intel_finish_page_flip_plane(struct drm_device *dev, int plane); +void intel_check_page_flip(struct drm_device *dev, int pipe); /* shared dpll functions */ struct intel_shared_dpll *intel_crtc_to_shared_dpll(struct intel_crtc *crtc); @@ -882,6 +903,7 @@ void hsw_enable_pc8(struct drm_i915_private *dev_priv); void hsw_disable_pc8(struct drm_i915_private *dev_priv); void intel_dp_get_m_n(struct intel_crtc *crtc, struct intel_crtc_config *pipe_config); +void intel_dp_set_m_n(struct intel_crtc *crtc); int intel_dotclock_calculate(int link_freq, const struct intel_link_m_n *m_n); void ironlake_check_encoder_dotclock(const struct intel_crtc_config *pipe_config, @@ -896,7 +918,7 @@ void intel_mode_from_pipe_config(struct drm_display_mode *mode, struct intel_crtc_config *pipe_config); int intel_format_to_fourcc(int format); void intel_crtc_wait_for_pending_flips(struct drm_crtc *crtc); - +void intel_modeset_preclose(struct drm_device *dev, struct drm_file *file); /* intel_dp.c */ void intel_dp_init(struct drm_device *dev, int output_reg, enum port port); @@ -935,6 +957,7 @@ void intel_dp_mst_suspend(struct drm_device *dev); void intel_dp_mst_resume(struct drm_device *dev); int intel_dp_max_link_bw(struct intel_dp *intel_dp); void intel_dp_hot_plug(struct intel_encoder *intel_encoder); +void vlv_power_sequencer_reset(struct drm_i915_private *dev_priv); /* intel_dp_mst.c */ int intel_dp_mst_encoder_init(struct intel_digital_port *intel_dig_port, int conn_id); void intel_dp_mst_encoder_cleanup(struct intel_digital_port *intel_dig_port); @@ -949,9 +972,9 @@ void intel_dvo_init(struct drm_device *dev); /* legacy fbdev emulation in intel_fbdev.c */ #ifdef CONFIG_DRM_I915_FBDEV extern int intel_fbdev_init(struct drm_device *dev); -extern void intel_fbdev_initial_config(struct drm_device *dev); +extern void intel_fbdev_initial_config(void *data, async_cookie_t cookie); extern void intel_fbdev_fini(struct drm_device *dev); -extern void intel_fbdev_set_suspend(struct drm_device *dev, int state); +extern void intel_fbdev_set_suspend(struct drm_device *dev, int state, bool synchronous); extern void intel_fbdev_output_poll_changed(struct drm_device *dev); extern void intel_fbdev_restore_mode(struct drm_device *dev); #else @@ -960,7 +983,7 @@ static inline int intel_fbdev_init(struct drm_device *dev) return 0; } -static inline void intel_fbdev_initial_config(struct drm_device *dev) +static inline void intel_fbdev_initial_config(void *data, async_cookie_t cookie) { } @@ -968,7 +991,7 @@ static inline void intel_fbdev_fini(struct drm_device *dev) { } -static inline void intel_fbdev_set_suspend(struct drm_device *dev, int state) +static inline void intel_fbdev_set_suspend(struct drm_device *dev, int state, bool synchronous) { } @@ -1081,6 +1104,7 @@ void intel_runtime_pm_put(struct drm_i915_private *dev_priv); void intel_init_runtime_pm(struct drm_i915_private *dev_priv); void intel_fini_runtime_pm(struct drm_i915_private *dev_priv); void ilk_wm_get_hw_state(struct drm_device *dev); +uint32_t ilk_pipe_pixel_rate(const struct intel_crtc_config *pipe_config); /* intel_sdvo.c */ @@ -1091,7 +1115,10 @@ bool intel_sdvo_init(struct drm_device *dev, uint32_t sdvo_reg, bool is_sdvob); int intel_plane_init(struct drm_device *dev, enum pipe pipe, int plane); void intel_flush_primary_plane(struct drm_i915_private *dev_priv, enum plane plane); -void intel_plane_restore(struct drm_plane *plane); +int intel_plane_set_property(struct drm_plane *plane, + struct drm_property *prop, + uint64_t val); +int intel_plane_restore(struct drm_plane *plane); void intel_plane_disable(struct drm_plane *plane); int intel_sprite_set_colorkey(struct drm_device *dev, void *data, struct drm_file *file_priv); diff --git a/drivers/gpu/drm/i915/intel_dsi.c b/drivers/gpu/drm/i915/intel_dsi.c index 670c29a..5bd9e09 100644 --- a/drivers/gpu/drm/i915/intel_dsi.c +++ b/drivers/gpu/drm/i915/intel_dsi.c @@ -184,7 +184,7 @@ static void intel_dsi_pre_enable(struct intel_encoder *encoder) /* update the hw state for DPLL */ intel_crtc->config.dpll_hw_state.dpll = DPLL_INTEGRATED_CLOCK_VLV | - DPLL_REFA_CLK_ENABLE_VLV; + DPLL_REFA_CLK_ENABLE_VLV; tmp = I915_READ(DSPCLK_GATE_D); tmp |= DPOUNIT_CLOCK_GATE_DISABLE; @@ -259,8 +259,8 @@ static void intel_dsi_disable(struct intel_encoder *encoder) temp = I915_READ(MIPI_CTRL(pipe)); temp &= ~ESCAPE_CLOCK_DIVIDER_MASK; I915_WRITE(MIPI_CTRL(pipe), temp | - intel_dsi->escape_clk_div << - ESCAPE_CLOCK_DIVIDER_SHIFT); + intel_dsi->escape_clk_div << + ESCAPE_CLOCK_DIVIDER_SHIFT); I915_WRITE(MIPI_EOT_DISABLE(pipe), CLOCKSTOP); @@ -297,7 +297,7 @@ static void intel_dsi_clear_device_ready(struct intel_encoder *encoder) usleep_range(2000, 2500); if (wait_for(((I915_READ(MIPI_PORT_CTRL(pipe)) & AFE_LATCHOUT) - == 0x00000), 30)) + == 0x00000), 30)) DRM_ERROR("DSI LP not going Low\n"); val = I915_READ(MIPI_PORT_CTRL(pipe)); @@ -423,9 +423,11 @@ static u16 txclkesc(u32 divider, unsigned int us) } /* return pixels in terms of txbyteclkhs */ -static u16 txbyteclkhs(u16 pixels, int bpp, int lane_count) +static u16 txbyteclkhs(u16 pixels, int bpp, int lane_count, + u16 burst_mode_ratio) { - return DIV_ROUND_UP(DIV_ROUND_UP(pixels * bpp, 8), lane_count); + return DIV_ROUND_UP(DIV_ROUND_UP(pixels * bpp * burst_mode_ratio, + 8 * 100), lane_count); } static void set_dsi_timings(struct drm_encoder *encoder, @@ -451,10 +453,12 @@ static void set_dsi_timings(struct drm_encoder *encoder, vbp = mode->vtotal - mode->vsync_end; /* horizontal values are in terms of high speed byte clock */ - hactive = txbyteclkhs(hactive, bpp, lane_count); - hfp = txbyteclkhs(hfp, bpp, lane_count); - hsync = txbyteclkhs(hsync, bpp, lane_count); - hbp = txbyteclkhs(hbp, bpp, lane_count); + hactive = txbyteclkhs(hactive, bpp, lane_count, + intel_dsi->burst_mode_ratio); + hfp = txbyteclkhs(hfp, bpp, lane_count, intel_dsi->burst_mode_ratio); + hsync = txbyteclkhs(hsync, bpp, lane_count, + intel_dsi->burst_mode_ratio); + hbp = txbyteclkhs(hbp, bpp, lane_count, intel_dsi->burst_mode_ratio); I915_WRITE(MIPI_HACTIVE_AREA_COUNT(pipe), hactive); I915_WRITE(MIPI_HFP_COUNT(pipe), hfp); @@ -541,12 +545,14 @@ static void intel_dsi_prepare(struct intel_encoder *intel_encoder) intel_dsi->video_mode_format == VIDEO_MODE_BURST) { I915_WRITE(MIPI_HS_TX_TIMEOUT(pipe), txbyteclkhs(adjusted_mode->htotal, bpp, - intel_dsi->lane_count) + 1); + intel_dsi->lane_count, + intel_dsi->burst_mode_ratio) + 1); } else { I915_WRITE(MIPI_HS_TX_TIMEOUT(pipe), txbyteclkhs(adjusted_mode->vtotal * adjusted_mode->htotal, - bpp, intel_dsi->lane_count) + 1); + bpp, intel_dsi->lane_count, + intel_dsi->burst_mode_ratio) + 1); } I915_WRITE(MIPI_LP_RX_TIMEOUT(pipe), intel_dsi->lp_rx_timeout); I915_WRITE(MIPI_TURN_AROUND_TIMEOUT(pipe), intel_dsi->turn_arnd_val); @@ -576,7 +582,7 @@ static void intel_dsi_prepare(struct intel_encoder *intel_encoder) * XXX: write MIPI_STOP_STATE_STALL? */ I915_WRITE(MIPI_HIGH_LOW_SWITCH_COUNT(pipe), - intel_dsi->hs_to_lp_count); + intel_dsi->hs_to_lp_count); /* XXX: low power clock equivalence in terms of byte clock. the number * of byte clocks occupied in one low power clock. based on txbyteclkhs @@ -601,10 +607,10 @@ static void intel_dsi_prepare(struct intel_encoder *intel_encoder) * 64 like 1366 x 768. Enable RANDOM resolution support for such * panels by default */ I915_WRITE(MIPI_VIDEO_MODE_FORMAT(pipe), - intel_dsi->video_frmt_cfg_bits | - intel_dsi->video_mode_format | - IP_TG_CONFIG | - RANDOM_DPI_DISPLAY_RESOLUTION); + intel_dsi->video_frmt_cfg_bits | + intel_dsi->video_mode_format | + IP_TG_CONFIG | + RANDOM_DPI_DISPLAY_RESOLUTION); } static void intel_dsi_pre_pll_enable(struct intel_encoder *encoder) diff --git a/drivers/gpu/drm/i915/intel_dsi.h b/drivers/gpu/drm/i915/intel_dsi.h index fd51867..657eb5c 100644 --- a/drivers/gpu/drm/i915/intel_dsi.h +++ b/drivers/gpu/drm/i915/intel_dsi.h @@ -116,6 +116,8 @@ struct intel_dsi { u16 clk_hs_to_lp_count; u16 init_count; + u32 pclk; + u16 burst_mode_ratio; /* all delays in ms */ u16 backlight_off_delay; diff --git a/drivers/gpu/drm/i915/intel_dsi_cmd.c b/drivers/gpu/drm/i915/intel_dsi_cmd.c index 7f1430a..f4767fd 100644 --- a/drivers/gpu/drm/i915/intel_dsi_cmd.c +++ b/drivers/gpu/drm/i915/intel_dsi_cmd.c @@ -430,7 +430,7 @@ void wait_for_dsi_fifo_empty(struct intel_dsi *intel_dsi) u32 mask; mask = LP_CTRL_FIFO_EMPTY | HS_CTRL_FIFO_EMPTY | - LP_DATA_FIFO_EMPTY | HS_DATA_FIFO_EMPTY; + LP_DATA_FIFO_EMPTY | HS_DATA_FIFO_EMPTY; if (wait_for((I915_READ(MIPI_GEN_FIFO_STAT(pipe)) & mask) == mask, 100)) DRM_ERROR("DPI FIFOs are not empty\n"); diff --git a/drivers/gpu/drm/i915/intel_dsi_panel_vbt.c b/drivers/gpu/drm/i915/intel_dsi_panel_vbt.c index 47c7584..f6bdd44 100644 --- a/drivers/gpu/drm/i915/intel_dsi_panel_vbt.c +++ b/drivers/gpu/drm/i915/intel_dsi_panel_vbt.c @@ -271,6 +271,8 @@ static bool generic_init(struct intel_dsi_device *dsi) u32 ths_prepare_ns, tclk_trail_ns; u32 tclk_prepare_clkzero, ths_prepare_hszero; u32 lp_to_hs_switch, hs_to_lp_switch; + u32 pclk, computed_ddr; + u16 burst_mode_ratio; DRM_DEBUG_KMS("\n"); @@ -284,8 +286,6 @@ static bool generic_init(struct intel_dsi_device *dsi) else if (intel_dsi->pixel_format == VID_MODE_FORMAT_RGB565) bits_per_pixel = 16; - bitrate = (mode->clock * bits_per_pixel) / intel_dsi->lane_count; - intel_dsi->operation_mode = mipi_config->is_cmd_mode; intel_dsi->video_mode_format = mipi_config->video_transfer_mode; intel_dsi->escape_clk_div = mipi_config->byte_clk_sel; @@ -297,6 +297,40 @@ static bool generic_init(struct intel_dsi_device *dsi) intel_dsi->video_frmt_cfg_bits = mipi_config->bta_enabled ? DISABLE_VIDEO_BTA : 0; + pclk = mode->clock; + + /* Burst Mode Ratio + * Target ddr frequency from VBT / non burst ddr freq + * multiply by 100 to preserve remainder + */ + if (intel_dsi->video_mode_format == VIDEO_MODE_BURST) { + if (mipi_config->target_burst_mode_freq) { + computed_ddr = + (pclk * bits_per_pixel) / intel_dsi->lane_count; + + if (mipi_config->target_burst_mode_freq < + computed_ddr) { + DRM_ERROR("Burst mode freq is less than computed\n"); + return false; + } + + burst_mode_ratio = DIV_ROUND_UP( + mipi_config->target_burst_mode_freq * 100, + computed_ddr); + + pclk = DIV_ROUND_UP(pclk * burst_mode_ratio, 100); + } else { + DRM_ERROR("Burst mode target is not set\n"); + return false; + } + } else + burst_mode_ratio = 100; + + intel_dsi->burst_mode_ratio = burst_mode_ratio; + intel_dsi->pclk = pclk; + + bitrate = (pclk * bits_per_pixel) / intel_dsi->lane_count; + switch (intel_dsi->escape_clk_div) { case 0: tlpx_ns = 50; diff --git a/drivers/gpu/drm/i915/intel_dsi_pll.c b/drivers/gpu/drm/i915/intel_dsi_pll.c index d8bb1ea..fa7a6ca 100644 --- a/drivers/gpu/drm/i915/intel_dsi_pll.c +++ b/drivers/gpu/drm/i915/intel_dsi_pll.c @@ -134,8 +134,7 @@ static u32 dsi_rr_formula(const struct drm_display_mode *mode, #else /* Get DSI clock from pixel clock */ -static u32 dsi_clk_from_pclk(const struct drm_display_mode *mode, - int pixel_format, int lane_count) +static u32 dsi_clk_from_pclk(u32 pclk, int pixel_format, int lane_count) { u32 dsi_clk_khz; u32 bpp; @@ -156,7 +155,7 @@ static u32 dsi_clk_from_pclk(const struct drm_display_mode *mode, /* DSI data rate = pixel clock * bits per pixel / lane count pixel clock is converted from KHz to Hz */ - dsi_clk_khz = DIV_ROUND_CLOSEST(mode->clock * bpp, lane_count); + dsi_clk_khz = DIV_ROUND_CLOSEST(pclk * bpp, lane_count); return dsi_clk_khz; } @@ -191,7 +190,7 @@ static int dsi_calc_mnp(u32 dsi_clk, struct dsi_mnp *dsi_mnp) for (m = 62; m <= 92; m++) { for (p = 2; p <= 6; p++) { /* Find the optimal m and p divisors - with minimal error +/- the required clock */ + with minimal error +/- the required clock */ calc_dsi_clk = (m * ref_clk) / p; if (calc_dsi_clk == target_dsi_clk) { calc_m = m; @@ -228,15 +227,13 @@ static int dsi_calc_mnp(u32 dsi_clk, struct dsi_mnp *dsi_mnp) static void vlv_configure_dsi_pll(struct intel_encoder *encoder) { struct drm_i915_private *dev_priv = encoder->base.dev->dev_private; - struct intel_crtc *intel_crtc = to_intel_crtc(encoder->base.crtc); - const struct drm_display_mode *mode = &intel_crtc->config.adjusted_mode; struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base); int ret; struct dsi_mnp dsi_mnp; u32 dsi_clk; - dsi_clk = dsi_clk_from_pclk(mode, intel_dsi->pixel_format, - intel_dsi->lane_count); + dsi_clk = dsi_clk_from_pclk(intel_dsi->pclk, intel_dsi->pixel_format, + intel_dsi->lane_count); ret = dsi_calc_mnp(dsi_clk, &dsi_mnp); if (ret) { @@ -318,8 +315,8 @@ static void assert_bpp_mismatch(int pixel_format, int pipe_bpp) } WARN(bpp != pipe_bpp, - "bpp match assertion failure (expected %d, current %d)\n", - bpp, pipe_bpp); + "bpp match assertion failure (expected %d, current %d)\n", + bpp, pipe_bpp); } u32 vlv_get_dsi_pclk(struct intel_encoder *encoder, int pipe_bpp) diff --git a/drivers/gpu/drm/i915/intel_dvo.c b/drivers/gpu/drm/i915/intel_dvo.c index 56b47d2..e40e3df 100644 --- a/drivers/gpu/drm/i915/intel_dvo.c +++ b/drivers/gpu/drm/i915/intel_dvo.c @@ -85,7 +85,7 @@ static const struct intel_dvo_device intel_dvo_devices[] = { { .type = INTEL_DVO_CHIP_TMDS, .name = "ns2501", - .dvo_reg = DVOC, + .dvo_reg = DVOB, .slave_addr = NS2501_ADDR, .dev_ops = &ns2501_ops, } @@ -185,12 +185,13 @@ static void intel_enable_dvo(struct intel_encoder *encoder) u32 dvo_reg = intel_dvo->dev.dvo_reg; u32 temp = I915_READ(dvo_reg); - I915_WRITE(dvo_reg, temp | DVO_ENABLE); - I915_READ(dvo_reg); intel_dvo->dev.dev_ops->mode_set(&intel_dvo->dev, &crtc->config.requested_mode, &crtc->config.adjusted_mode); + I915_WRITE(dvo_reg, temp | DVO_ENABLE); + I915_READ(dvo_reg); + intel_dvo->dev.dev_ops->dpms(&intel_dvo->dev, true); } @@ -226,10 +227,6 @@ static void intel_dvo_dpms(struct drm_connector *connector, int mode) intel_crtc_update_dpms(crtc); - intel_dvo->dev.dev_ops->mode_set(&intel_dvo->dev, - &config->requested_mode, - &config->adjusted_mode); - intel_dvo->dev.dev_ops->dpms(&intel_dvo->dev, true); } else { intel_dvo->dev.dev_ops->dpms(&intel_dvo->dev, false); diff --git a/drivers/gpu/drm/i915/intel_fbdev.c b/drivers/gpu/drm/i915/intel_fbdev.c index f475414..9b584f3 100644 --- a/drivers/gpu/drm/i915/intel_fbdev.c +++ b/drivers/gpu/drm/i915/intel_fbdev.c @@ -24,8 +24,10 @@ * David Airlie */ +#include #include #include +#include #include #include #include @@ -331,24 +333,6 @@ static bool intel_fb_initial_config(struct drm_fb_helper *fb_helper, int num_connectors_enabled = 0; int num_connectors_detected = 0; - /* - * If the user specified any force options, just bail here - * and use that config. - */ - for (i = 0; i < fb_helper->connector_count; i++) { - struct drm_fb_helper_connector *fb_conn; - struct drm_connector *connector; - - fb_conn = fb_helper->connector_info[i]; - connector = fb_conn->connector; - - if (!enabled[i]) - continue; - - if (connector->force != DRM_FORCE_UNSPECIFIED) - return false; - } - save_enabled = kcalloc(dev->mode_config.num_connector, sizeof(bool), GFP_KERNEL); if (!save_enabled) @@ -374,8 +358,18 @@ static bool intel_fb_initial_config(struct drm_fb_helper *fb_helper, continue; } + if (connector->force == DRM_FORCE_OFF) { + DRM_DEBUG_KMS("connector %s is disabled by user, skipping\n", + connector->name); + enabled[i] = false; + continue; + } + encoder = connector->encoder; if (!encoder || WARN_ON(!encoder->crtc)) { + if (connector->force > DRM_FORCE_OFF) + goto bail; + DRM_DEBUG_KMS("connector %s has no encoder or crtc, skipping\n", connector->name); enabled[i] = false; @@ -394,8 +388,7 @@ static bool intel_fb_initial_config(struct drm_fb_helper *fb_helper, for (j = 0; j < fb_helper->connector_count; j++) { if (crtcs[j] == new_crtc) { DRM_DEBUG_KMS("fallback: cloned configuration\n"); - fallback = true; - goto out; + goto bail; } } @@ -466,8 +459,8 @@ static bool intel_fb_initial_config(struct drm_fb_helper *fb_helper, fallback = true; } -out: if (fallback) { +bail: DRM_DEBUG_KMS("Not using firmware configuration\n"); memcpy(enabled, save_enabled, dev->mode_config.num_connector); kfree(save_enabled); @@ -636,6 +629,15 @@ out: return false; } +static void intel_fbdev_suspend_worker(struct work_struct *work) +{ + intel_fbdev_set_suspend(container_of(work, + struct drm_i915_private, + fbdev_suspend_work)->dev, + FBINFO_STATE_RUNNING, + true); +} + int intel_fbdev_init(struct drm_device *dev) { struct intel_fbdev *ifbdev; @@ -662,14 +664,16 @@ int intel_fbdev_init(struct drm_device *dev) } dev_priv->fbdev = ifbdev; + INIT_WORK(&dev_priv->fbdev_suspend_work, intel_fbdev_suspend_worker); + drm_fb_helper_single_add_all_connectors(&ifbdev->helper); return 0; } -void intel_fbdev_initial_config(struct drm_device *dev) +void intel_fbdev_initial_config(void *data, async_cookie_t cookie) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = data; struct intel_fbdev *ifbdev = dev_priv->fbdev; /* Due to peculiar init order wrt to hpd handling this is separate. */ @@ -682,12 +686,15 @@ void intel_fbdev_fini(struct drm_device *dev) if (!dev_priv->fbdev) return; + flush_work(&dev_priv->fbdev_suspend_work); + + async_synchronize_full(); intel_fbdev_destroy(dev, dev_priv->fbdev); kfree(dev_priv->fbdev); dev_priv->fbdev = NULL; } -void intel_fbdev_set_suspend(struct drm_device *dev, int state) +void intel_fbdev_set_suspend(struct drm_device *dev, int state, bool synchronous) { struct drm_i915_private *dev_priv = dev->dev_private; struct intel_fbdev *ifbdev = dev_priv->fbdev; @@ -698,6 +705,33 @@ void intel_fbdev_set_suspend(struct drm_device *dev, int state) info = ifbdev->helper.fbdev; + if (synchronous) { + /* Flush any pending work to turn the console on, and then + * wait to turn it off. It must be synchronous as we are + * about to suspend or unload the driver. + * + * Note that from within the work-handler, we cannot flush + * ourselves, so only flush outstanding work upon suspend! + */ + if (state != FBINFO_STATE_RUNNING) + flush_work(&dev_priv->fbdev_suspend_work); + console_lock(); + } else { + /* + * The console lock can be pretty contented on resume due + * to all the printk activity. Try to keep it out of the hot + * path of resume if possible. + */ + WARN_ON(state != FBINFO_STATE_RUNNING); + if (!console_trylock()) { + /* Don't block our own workqueue as this can + * be run in parallel with other i915.ko tasks. + */ + schedule_work(&dev_priv->fbdev_suspend_work); + return; + } + } + /* On resume from hibernation: If the object is shmemfs backed, it has * been restored from swap. If the object is stolen however, it will be * full of whatever garbage was left in there. @@ -706,6 +740,7 @@ void intel_fbdev_set_suspend(struct drm_device *dev, int state) memset_io(info->screen_base, 0, info->screen_size); fb_set_suspend(info, state); + console_unlock(); } void intel_fbdev_output_poll_changed(struct drm_device *dev) diff --git a/drivers/gpu/drm/i915/intel_hdmi.c b/drivers/gpu/drm/i915/intel_hdmi.c index 5a9de21..1f61303 100644 --- a/drivers/gpu/drm/i915/intel_hdmi.c +++ b/drivers/gpu/drm/i915/intel_hdmi.c @@ -732,7 +732,7 @@ static void intel_hdmi_get_config(struct intel_encoder *encoder, if (tmp & HDMI_MODE_SELECT_HDMI) pipe_config->has_hdmi_sink = true; - if (tmp & SDVO_AUDIO_ENABLE) + if (tmp & HDMI_MODE_SELECT_HDMI) pipe_config->has_audio = true; if (!HAS_PCH_SPLIT(dev) && @@ -869,10 +869,15 @@ static enum drm_mode_status intel_hdmi_mode_valid(struct drm_connector *connector, struct drm_display_mode *mode) { - if (mode->clock > hdmi_portclock_limit(intel_attached_hdmi(connector), - true)) + int clock = mode->clock; + + if (mode->flags & DRM_MODE_FLAG_DBLCLK) + clock *= 2; + + if (clock > hdmi_portclock_limit(intel_attached_hdmi(connector), + true)) return MODE_CLOCK_HIGH; - if (mode->clock < 20000) + if (clock < 20000) return MODE_CLOCK_LOW; if (mode->flags & DRM_MODE_FLAG_DBLSCAN) @@ -890,7 +895,7 @@ static bool hdmi_12bpc_possible(struct intel_crtc *crtc) if (HAS_GMCH_DISPLAY(dev)) return false; - list_for_each_entry(encoder, &dev->mode_config.encoder_list, base.head) { + for_each_intel_encoder(dev, encoder) { if (encoder->new_crtc != crtc) continue; @@ -926,6 +931,10 @@ bool intel_hdmi_compute_config(struct intel_encoder *encoder, intel_hdmi->color_range = 0; } + if (adjusted_mode->flags & DRM_MODE_FLAG_DBLCLK) { + pipe_config->pixel_multiplier = 2; + } + if (intel_hdmi->color_range) pipe_config->limited_color_range = true; @@ -967,104 +976,117 @@ bool intel_hdmi_compute_config(struct intel_encoder *encoder, return true; } -static enum drm_connector_status -intel_hdmi_detect(struct drm_connector *connector, bool force) +static void +intel_hdmi_unset_edid(struct drm_connector *connector) { - struct drm_device *dev = connector->dev; struct intel_hdmi *intel_hdmi = intel_attached_hdmi(connector); - struct intel_digital_port *intel_dig_port = - hdmi_to_dig_port(intel_hdmi); - struct intel_encoder *intel_encoder = &intel_dig_port->base; - struct drm_i915_private *dev_priv = dev->dev_private; - struct edid *edid; - enum intel_display_power_domain power_domain; - enum drm_connector_status status = connector_status_disconnected; - DRM_DEBUG_KMS("[CONNECTOR:%d:%s]\n", - connector->base.id, connector->name); + intel_hdmi->has_hdmi_sink = false; + intel_hdmi->has_audio = false; + intel_hdmi->rgb_quant_range_selectable = false; + + kfree(to_intel_connector(connector)->detect_edid); + to_intel_connector(connector)->detect_edid = NULL; +} + +static bool +intel_hdmi_set_edid(struct drm_connector *connector) +{ + struct drm_i915_private *dev_priv = to_i915(connector->dev); + struct intel_hdmi *intel_hdmi = intel_attached_hdmi(connector); + struct intel_encoder *intel_encoder = + &hdmi_to_dig_port(intel_hdmi)->base; + enum intel_display_power_domain power_domain; + struct edid *edid; + bool connected = false; power_domain = intel_display_port_power_domain(intel_encoder); intel_display_power_get(dev_priv, power_domain); - intel_hdmi->has_hdmi_sink = false; - intel_hdmi->has_audio = false; - intel_hdmi->rgb_quant_range_selectable = false; edid = drm_get_edid(connector, intel_gmbus_get_adapter(dev_priv, intel_hdmi->ddc_bus)); - if (edid) { - if (edid->input & DRM_EDID_INPUT_DIGITAL) { - status = connector_status_connected; - if (intel_hdmi->force_audio != HDMI_AUDIO_OFF_DVI) - intel_hdmi->has_hdmi_sink = - drm_detect_hdmi_monitor(edid); - intel_hdmi->has_audio = drm_detect_monitor_audio(edid); - intel_hdmi->rgb_quant_range_selectable = - drm_rgb_quant_range_selectable(edid); - } - kfree(edid); - } + intel_display_power_put(dev_priv, power_domain); + + to_intel_connector(connector)->detect_edid = edid; + if (edid && edid->input & DRM_EDID_INPUT_DIGITAL) { + intel_hdmi->rgb_quant_range_selectable = + drm_rgb_quant_range_selectable(edid); - if (status == connector_status_connected) { + intel_hdmi->has_audio = drm_detect_monitor_audio(edid); if (intel_hdmi->force_audio != HDMI_AUDIO_AUTO) intel_hdmi->has_audio = - (intel_hdmi->force_audio == HDMI_AUDIO_ON); - intel_encoder->type = INTEL_OUTPUT_HDMI; + intel_hdmi->force_audio == HDMI_AUDIO_ON; + + if (intel_hdmi->force_audio != HDMI_AUDIO_OFF_DVI) + intel_hdmi->has_hdmi_sink = + drm_detect_hdmi_monitor(edid); + + connected = true; } - intel_display_power_put(dev_priv, power_domain); + return connected; +} + +static enum drm_connector_status +intel_hdmi_detect(struct drm_connector *connector, bool force) +{ + enum drm_connector_status status; + + DRM_DEBUG_KMS("[CONNECTOR:%d:%s]\n", + connector->base.id, connector->name); + + intel_hdmi_unset_edid(connector); + + if (intel_hdmi_set_edid(connector)) { + struct intel_hdmi *intel_hdmi = intel_attached_hdmi(connector); + + hdmi_to_dig_port(intel_hdmi)->base.type = INTEL_OUTPUT_HDMI; + status = connector_status_connected; + } else + status = connector_status_disconnected; return status; } -static int intel_hdmi_get_modes(struct drm_connector *connector) +static void +intel_hdmi_force(struct drm_connector *connector) { - struct intel_encoder *intel_encoder = intel_attached_encoder(connector); - struct intel_hdmi *intel_hdmi = enc_to_intel_hdmi(&intel_encoder->base); - struct drm_i915_private *dev_priv = connector->dev->dev_private; - enum intel_display_power_domain power_domain; - int ret; + struct intel_hdmi *intel_hdmi = intel_attached_hdmi(connector); - /* We should parse the EDID data and find out if it's an HDMI sink so - * we can send audio to it. - */ + DRM_DEBUG_KMS("[CONNECTOR:%d:%s]\n", + connector->base.id, connector->name); - power_domain = intel_display_port_power_domain(intel_encoder); - intel_display_power_get(dev_priv, power_domain); + intel_hdmi_unset_edid(connector); - ret = intel_ddc_get_modes(connector, - intel_gmbus_get_adapter(dev_priv, - intel_hdmi->ddc_bus)); + if (connector->status != connector_status_connected) + return; - intel_display_power_put(dev_priv, power_domain); + intel_hdmi_set_edid(connector); + hdmi_to_dig_port(intel_hdmi)->base.type = INTEL_OUTPUT_HDMI; +} - return ret; +static int intel_hdmi_get_modes(struct drm_connector *connector) +{ + struct edid *edid; + + edid = to_intel_connector(connector)->detect_edid; + if (edid == NULL) + return 0; + + return intel_connector_update_modes(connector, edid); } static bool intel_hdmi_detect_audio(struct drm_connector *connector) { - struct intel_encoder *intel_encoder = intel_attached_encoder(connector); - struct intel_hdmi *intel_hdmi = enc_to_intel_hdmi(&intel_encoder->base); - struct drm_i915_private *dev_priv = connector->dev->dev_private; - enum intel_display_power_domain power_domain; - struct edid *edid; bool has_audio = false; + struct edid *edid; - power_domain = intel_display_port_power_domain(intel_encoder); - intel_display_power_get(dev_priv, power_domain); - - edid = drm_get_edid(connector, - intel_gmbus_get_adapter(dev_priv, - intel_hdmi->ddc_bus)); - if (edid) { - if (edid->input & DRM_EDID_INPUT_DIGITAL) - has_audio = drm_detect_monitor_audio(edid); - kfree(edid); - } - - intel_display_power_put(dev_priv, power_domain); + edid = to_intel_connector(connector)->detect_edid; + if (edid && edid->input & DRM_EDID_INPUT_DIGITAL) + has_audio = drm_detect_monitor_audio(edid); return has_audio; } @@ -1265,6 +1287,8 @@ static void chv_hdmi_pre_pll_enable(struct intel_encoder *encoder) enum pipe pipe = intel_crtc->pipe; u32 val; + intel_hdmi_prepare(encoder); + mutex_lock(&dev_priv->dpio_lock); /* program left/right clock distribution */ @@ -1434,8 +1458,8 @@ static void chv_hdmi_pre_enable(struct intel_encoder *encoder) for (i = 0; i < 4; i++) { val = vlv_dpio_read(dev_priv, pipe, CHV_TX_DW2(ch, i)); - val &= ~DPIO_SWING_MARGIN_MASK; - val |= 102 << DPIO_SWING_MARGIN_SHIFT; + val &= ~DPIO_SWING_MARGIN000_MASK; + val |= 102 << DPIO_SWING_MARGIN000_SHIFT; vlv_dpio_write(dev_priv, pipe, CHV_TX_DW2(ch, i), val); } @@ -1482,6 +1506,7 @@ static void chv_hdmi_pre_enable(struct intel_encoder *encoder) static void intel_hdmi_destroy(struct drm_connector *connector) { + kfree(to_intel_connector(connector)->detect_edid); drm_connector_cleanup(connector); kfree(connector); } @@ -1489,6 +1514,7 @@ static void intel_hdmi_destroy(struct drm_connector *connector) static const struct drm_connector_funcs intel_hdmi_connector_funcs = { .dpms = intel_connector_dpms, .detect = intel_hdmi_detect, + .force = intel_hdmi_force, .fill_modes = drm_helper_probe_single_connector_modes, .set_property = intel_hdmi_set_property, .destroy = intel_hdmi_destroy, diff --git a/drivers/gpu/drm/i915/intel_lvds.c b/drivers/gpu/drm/i915/intel_lvds.c index fdf4026..a6bd142 100644 --- a/drivers/gpu/drm/i915/intel_lvds.c +++ b/drivers/gpu/drm/i915/intel_lvds.c @@ -823,8 +823,7 @@ bool intel_is_dual_link_lvds(struct drm_device *dev) struct intel_encoder *encoder; struct intel_lvds_encoder *lvds_encoder; - list_for_each_entry(encoder, &dev->mode_config.encoder_list, - base.head) { + for_each_intel_encoder(dev, encoder) { if (encoder->type == INTEL_OUTPUT_LVDS) { lvds_encoder = to_lvds_encoder(&encoder->base); diff --git a/drivers/gpu/drm/i915/intel_opregion.c b/drivers/gpu/drm/i915/intel_opregion.c index d8de1d5..ca52ad2 100644 --- a/drivers/gpu/drm/i915/intel_opregion.c +++ b/drivers/gpu/drm/i915/intel_opregion.c @@ -396,16 +396,6 @@ int intel_opregion_notify_adapter(struct drm_device *dev, pci_power_t state) return -EINVAL; } -/* - * If the vendor backlight interface is not in use and ACPI backlight interface - * is broken, do not bother processing backlight change requests from firmware. - */ -static bool should_ignore_backlight_request(void) -{ - return acpi_video_backlight_support() && - !acpi_video_verify_backlight_support(); -} - static u32 asle_set_backlight(struct drm_device *dev, u32 bclp) { struct drm_i915_private *dev_priv = dev->dev_private; @@ -414,7 +404,11 @@ static u32 asle_set_backlight(struct drm_device *dev, u32 bclp) DRM_DEBUG_DRIVER("bclp = 0x%08x\n", bclp); - if (should_ignore_backlight_request()) { + /* + * If the acpi_video interface is not supposed to be used, don't + * bother processing backlight level change requests from firmware. + */ + if (!acpi_video_verify_backlight_support()) { DRM_DEBUG_KMS("opregion backlight request ignored\n"); return 0; } diff --git a/drivers/gpu/drm/i915/intel_overlay.c b/drivers/gpu/drm/i915/intel_overlay.c index dc2f4f2..ae0e577 100644 --- a/drivers/gpu/drm/i915/intel_overlay.c +++ b/drivers/gpu/drm/i915/intel_overlay.c @@ -182,7 +182,7 @@ struct intel_overlay { u32 flip_addr; struct drm_i915_gem_object *reg_bo; /* flip handling */ - uint32_t last_flip_req; + struct i915_gem_request *flip_request; void (*flip_tail)(struct intel_overlay *); }; @@ -208,53 +208,86 @@ static void intel_overlay_unmap_regs(struct intel_overlay *overlay, io_mapping_unmap(regs); } -static int intel_overlay_do_wait_request(struct intel_overlay *overlay, - void (*tail)(struct intel_overlay *)) +/* recover from an interruption due to a signal + * We have to be careful not to repeat work forever an make forward progess. */ +static int intel_overlay_recover_from_interrupt(struct intel_overlay *overlay) { - struct drm_device *dev = overlay->dev; - struct drm_i915_private *dev_priv = dev->dev_private; - struct intel_engine_cs *ring = &dev_priv->ring[RCS]; int ret; - BUG_ON(overlay->last_flip_req); - ret = i915_add_request(ring, &overlay->last_flip_req); - if (ret) - return ret; + if (overlay->flip_request == NULL) + return 0; - overlay->flip_tail = tail; - ret = i915_wait_seqno(ring, overlay->last_flip_req); + ret = i915_request_wait(overlay->flip_request); if (ret) return ret; - i915_gem_retire_requests(dev); - overlay->last_flip_req = 0; + i915_request_put(overlay->flip_request); + overlay->flip_request = NULL; + + i915_gem_retire_requests(overlay->dev); + + if (overlay->flip_tail) + overlay->flip_tail(overlay); + return 0; } +static int intel_overlay_add_request(struct intel_overlay *overlay, + struct i915_gem_request *rq, + void (*tail)(struct intel_overlay *)) +{ + BUG_ON(overlay->flip_request); + overlay->flip_request = rq; + overlay->flip_tail = tail; + + return i915_request_commit(rq); +} + +static int intel_overlay_do_wait_request(struct intel_overlay *overlay, + struct i915_gem_request *rq, + void (*tail)(struct intel_overlay *)) +{ + intel_overlay_add_request(overlay, rq, tail); + return intel_overlay_recover_from_interrupt(overlay); +} + +static struct i915_gem_request * +intel_overlay_alloc_request(struct intel_overlay *overlay) +{ + struct drm_i915_private *i915 = to_i915(overlay->dev); + return intel_engine_alloc_request(RCS_ENGINE(i915), + RCS_ENGINE(i915)->default_context); +} + /* overlay needs to be disable in OCMD reg */ static int intel_overlay_on(struct intel_overlay *overlay) { struct drm_device *dev = overlay->dev; struct drm_i915_private *dev_priv = dev->dev_private; - struct intel_engine_cs *ring = &dev_priv->ring[RCS]; - int ret; + struct i915_gem_request *rq; + struct intel_ringbuffer *ring; BUG_ON(overlay->active); overlay->active = 1; WARN_ON(IS_I830(dev) && !(dev_priv->quirks & QUIRK_PIPEA_FORCE)); - ret = intel_ring_begin(ring, 4); - if (ret) - return ret; + rq = intel_overlay_alloc_request(overlay); + if (IS_ERR(rq)) + return PTR_ERR(rq); + + ring = intel_ring_begin(rq, 3); + if (IS_ERR(ring)) { + i915_request_put(rq); + return PTR_ERR(ring); + } intel_ring_emit(ring, MI_OVERLAY_FLIP | MI_OVERLAY_ON); intel_ring_emit(ring, overlay->flip_addr | OFC_UPDATE); intel_ring_emit(ring, MI_WAIT_FOR_EVENT | MI_WAIT_FOR_OVERLAY_FLIP); - intel_ring_emit(ring, MI_NOOP); intel_ring_advance(ring); - return intel_overlay_do_wait_request(overlay, NULL); + return intel_overlay_do_wait_request(overlay, rq, NULL); } /* overlay needs to be enabled in OCMD reg */ @@ -263,10 +296,10 @@ static int intel_overlay_continue(struct intel_overlay *overlay, { struct drm_device *dev = overlay->dev; struct drm_i915_private *dev_priv = dev->dev_private; - struct intel_engine_cs *ring = &dev_priv->ring[RCS]; u32 flip_addr = overlay->flip_addr; + struct i915_gem_request *rq; + struct intel_ringbuffer *ring; u32 tmp; - int ret; BUG_ON(!overlay->active); @@ -278,21 +311,30 @@ static int intel_overlay_continue(struct intel_overlay *overlay, if (tmp & (1 << 17)) DRM_DEBUG("overlay underrun, DOVSTA: %x\n", tmp); - ret = intel_ring_begin(ring, 2); - if (ret) - return ret; + rq = intel_overlay_alloc_request(overlay); + if (IS_ERR(rq)) + return PTR_ERR(rq); + + ring = intel_ring_begin(rq, 2); + if (IS_ERR(ring)) { + i915_request_put(rq); + return PTR_ERR(ring); + } intel_ring_emit(ring, MI_OVERLAY_FLIP | MI_OVERLAY_CONTINUE); intel_ring_emit(ring, flip_addr); intel_ring_advance(ring); - return i915_add_request(ring, &overlay->last_flip_req); + return intel_overlay_add_request(overlay, rq, NULL); } static void intel_overlay_release_old_vid_tail(struct intel_overlay *overlay) { struct drm_i915_gem_object *obj = overlay->old_vid_bo; + i915_gem_track_fb(obj, NULL, + INTEL_FRONTBUFFER_OVERLAY(overlay->crtc->pipe)); + i915_gem_object_ggtt_unpin(obj); drm_gem_object_unreference(&obj->base); @@ -319,10 +361,10 @@ static void intel_overlay_off_tail(struct intel_overlay *overlay) static int intel_overlay_off(struct intel_overlay *overlay) { struct drm_device *dev = overlay->dev; - struct drm_i915_private *dev_priv = dev->dev_private; - struct intel_engine_cs *ring = &dev_priv->ring[RCS]; u32 flip_addr = overlay->flip_addr; - int ret; + struct i915_gem_request *rq; + struct intel_ringbuffer *ring; + int len; BUG_ON(!overlay->active); @@ -332,53 +374,36 @@ static int intel_overlay_off(struct intel_overlay *overlay) * of the hw. Do it in both cases */ flip_addr |= OFC_UPDATE; - ret = intel_ring_begin(ring, 6); - if (ret) - return ret; + rq = intel_overlay_alloc_request(overlay); + if (IS_ERR(rq)) + return PTR_ERR(rq); + + len = 3; + if (!IS_I830(dev)) + len += 3; + + ring = intel_ring_begin(rq, len); + if (IS_ERR(ring)) { + i915_request_put(rq); + return PTR_ERR(ring); + } /* wait for overlay to go idle */ intel_ring_emit(ring, MI_OVERLAY_FLIP | MI_OVERLAY_CONTINUE); intel_ring_emit(ring, flip_addr); intel_ring_emit(ring, MI_WAIT_FOR_EVENT | MI_WAIT_FOR_OVERLAY_FLIP); - /* turn overlay off */ - if (IS_I830(dev)) { - /* Workaround: Don't disable the overlay fully, since otherwise - * it dies on the next OVERLAY_ON cmd. */ - intel_ring_emit(ring, MI_NOOP); - intel_ring_emit(ring, MI_NOOP); - intel_ring_emit(ring, MI_NOOP); - } else { + /* turn overlay off + * Workaround for i830: Don't disable the overlay fully, since + * otherwise it dies on the next OVERLAY_ON cmd. + */ + if (!IS_I830(dev)) { intel_ring_emit(ring, MI_OVERLAY_FLIP | MI_OVERLAY_OFF); intel_ring_emit(ring, flip_addr); intel_ring_emit(ring, MI_WAIT_FOR_EVENT | MI_WAIT_FOR_OVERLAY_FLIP); } intel_ring_advance(ring); - return intel_overlay_do_wait_request(overlay, intel_overlay_off_tail); -} - -/* recover from an interruption due to a signal - * We have to be careful not to repeat work forever an make forward progess. */ -static int intel_overlay_recover_from_interrupt(struct intel_overlay *overlay) -{ - struct drm_device *dev = overlay->dev; - struct drm_i915_private *dev_priv = dev->dev_private; - struct intel_engine_cs *ring = &dev_priv->ring[RCS]; - int ret; - - if (overlay->last_flip_req == 0) - return 0; - - ret = i915_wait_seqno(ring, overlay->last_flip_req); - if (ret) - return ret; - i915_gem_retire_requests(dev); - - if (overlay->flip_tail) - overlay->flip_tail(overlay); - - overlay->last_flip_req = 0; - return 0; + return intel_overlay_do_wait_request(overlay, rq, intel_overlay_off_tail); } /* Wait for pending overlay flip and release old frame. @@ -387,10 +412,8 @@ static int intel_overlay_recover_from_interrupt(struct intel_overlay *overlay) */ static int intel_overlay_release_old_vid(struct intel_overlay *overlay) { - struct drm_device *dev = overlay->dev; - struct drm_i915_private *dev_priv = dev->dev_private; - struct intel_engine_cs *ring = &dev_priv->ring[RCS]; - int ret; + struct drm_i915_private *dev_priv = to_i915(overlay->dev); + int ret = 0; /* Only wait if there is actually an old frame to release to * guarantee forward progress. @@ -399,27 +422,29 @@ static int intel_overlay_release_old_vid(struct intel_overlay *overlay) return 0; if (I915_READ(ISR) & I915_OVERLAY_PLANE_FLIP_PENDING_INTERRUPT) { + struct i915_gem_request *rq; + struct intel_ringbuffer *ring; + + rq = intel_overlay_alloc_request(overlay); + if (IS_ERR(rq)) + return PTR_ERR(rq); + /* synchronous slowpath */ - ret = intel_ring_begin(ring, 2); - if (ret) - return ret; + ring = intel_ring_begin(rq, 1); + if (IS_ERR(ring)) { + i915_request_put(rq); + return PTR_ERR(ring); + } intel_ring_emit(ring, MI_WAIT_FOR_EVENT | MI_WAIT_FOR_OVERLAY_FLIP); - intel_ring_emit(ring, MI_NOOP); intel_ring_advance(ring); - ret = intel_overlay_do_wait_request(overlay, + ret = intel_overlay_do_wait_request(overlay, rq, intel_overlay_release_old_vid_tail); - if (ret) - return ret; - } - - intel_overlay_release_old_vid_tail(overlay); - + } else + intel_overlay_release_old_vid_tail(overlay); - i915_gem_track_fb(overlay->old_vid_bo, NULL, - INTEL_FRONTBUFFER_OVERLAY(overlay->crtc->pipe)); - return 0; + return ret; } struct put_image_params { @@ -821,12 +846,7 @@ int intel_overlay_switch_off(struct intel_overlay *overlay) iowrite32(0, ®s->OCMD); intel_overlay_unmap_regs(overlay, regs); - ret = intel_overlay_off(overlay); - if (ret != 0) - return ret; - - intel_overlay_off_tail(overlay); - return 0; + return intel_overlay_off(overlay); } static int check_overlay_possible_on_crtc(struct intel_overlay *overlay, diff --git a/drivers/gpu/drm/i915/intel_panel.c b/drivers/gpu/drm/i915/intel_panel.c index 8e37444..f17ada3 100644 --- a/drivers/gpu/drm/i915/intel_panel.c +++ b/drivers/gpu/drm/i915/intel_panel.c @@ -538,14 +538,13 @@ static u32 intel_panel_get_backlight(struct intel_connector *connector) struct drm_device *dev = connector->base.dev; struct drm_i915_private *dev_priv = dev->dev_private; u32 val; - unsigned long flags; - spin_lock_irqsave(&dev_priv->backlight_lock, flags); + mutex_lock(&dev_priv->backlight_lock); val = dev_priv->display.get_backlight(connector); val = intel_panel_compute_brightness(connector, val); - spin_unlock_irqrestore(&dev_priv->backlight_lock, flags); + mutex_unlock(&dev_priv->backlight_lock); DRM_DEBUG_DRIVER("get backlight PWM = %d\n", val); return val; @@ -629,12 +628,11 @@ static void intel_panel_set_backlight(struct intel_connector *connector, struct intel_panel *panel = &connector->panel; enum pipe pipe = intel_get_pipe_from_connector(connector); u32 hw_level; - unsigned long flags; if (!panel->backlight.present || pipe == INVALID_PIPE) return; - spin_lock_irqsave(&dev_priv->backlight_lock, flags); + mutex_lock(&dev_priv->backlight_lock); WARN_ON(panel->backlight.max == 0); @@ -644,7 +642,7 @@ static void intel_panel_set_backlight(struct intel_connector *connector, if (panel->backlight.enabled) intel_panel_actually_set_backlight(connector, hw_level); - spin_unlock_irqrestore(&dev_priv->backlight_lock, flags); + mutex_unlock(&dev_priv->backlight_lock); } /* set backlight brightness to level in range [0..max], assuming hw min is @@ -658,12 +656,11 @@ void intel_panel_set_backlight_acpi(struct intel_connector *connector, struct intel_panel *panel = &connector->panel; enum pipe pipe = intel_get_pipe_from_connector(connector); u32 hw_level; - unsigned long flags; if (!panel->backlight.present || pipe == INVALID_PIPE) return; - spin_lock_irqsave(&dev_priv->backlight_lock, flags); + mutex_lock(&dev_priv->backlight_lock); WARN_ON(panel->backlight.max == 0); @@ -679,7 +676,7 @@ void intel_panel_set_backlight_acpi(struct intel_connector *connector, if (panel->backlight.enabled) intel_panel_actually_set_backlight(connector, hw_level); - spin_unlock_irqrestore(&dev_priv->backlight_lock, flags); + mutex_unlock(&dev_priv->backlight_lock); } static void pch_disable_backlight(struct intel_connector *connector) @@ -733,7 +730,6 @@ void intel_panel_disable_backlight(struct intel_connector *connector) struct drm_i915_private *dev_priv = dev->dev_private; struct intel_panel *panel = &connector->panel; enum pipe pipe = intel_get_pipe_from_connector(connector); - unsigned long flags; if (!panel->backlight.present || pipe == INVALID_PIPE) return; @@ -749,12 +745,14 @@ void intel_panel_disable_backlight(struct intel_connector *connector) return; } - spin_lock_irqsave(&dev_priv->backlight_lock, flags); + mutex_lock(&dev_priv->backlight_lock); + if (panel->backlight.device) + panel->backlight.device->props.power = FB_BLANK_POWERDOWN; panel->backlight.enabled = false; dev_priv->display.disable_backlight(connector); - spin_unlock_irqrestore(&dev_priv->backlight_lock, flags); + mutex_unlock(&dev_priv->backlight_lock); } static void bdw_enable_backlight(struct intel_connector *connector) @@ -935,14 +933,13 @@ void intel_panel_enable_backlight(struct intel_connector *connector) struct drm_i915_private *dev_priv = dev->dev_private; struct intel_panel *panel = &connector->panel; enum pipe pipe = intel_get_pipe_from_connector(connector); - unsigned long flags; if (!panel->backlight.present || pipe == INVALID_PIPE) return; DRM_DEBUG_KMS("pipe %c\n", pipe_name(pipe)); - spin_lock_irqsave(&dev_priv->backlight_lock, flags); + mutex_lock(&dev_priv->backlight_lock); WARN_ON(panel->backlight.max == 0); @@ -957,14 +954,17 @@ void intel_panel_enable_backlight(struct intel_connector *connector) dev_priv->display.enable_backlight(connector); panel->backlight.enabled = true; + if (panel->backlight.device) + panel->backlight.device->props.power = FB_BLANK_UNBLANK; - spin_unlock_irqrestore(&dev_priv->backlight_lock, flags); + mutex_unlock(&dev_priv->backlight_lock); } #if IS_ENABLED(CONFIG_BACKLIGHT_CLASS_DEVICE) static int intel_backlight_device_update_status(struct backlight_device *bd) { struct intel_connector *connector = bl_get_data(bd); + struct intel_panel *panel = &connector->panel; struct drm_device *dev = connector->base.dev; drm_modeset_lock(&dev->mode_config.connection_mutex, NULL); @@ -972,6 +972,23 @@ static int intel_backlight_device_update_status(struct backlight_device *bd) bd->props.brightness, bd->props.max_brightness); intel_panel_set_backlight(connector, bd->props.brightness, bd->props.max_brightness); + + /* + * Allow flipping bl_power as a sub-state of enabled. Sadly the + * backlight class device does not make it easy to to differentiate + * between callbacks for brightness and bl_power, so our backlight_power + * callback needs to take this into account. + */ + if (panel->backlight.enabled) { + if (panel->backlight_power) { + bool enable = bd->props.power == FB_BLANK_UNBLANK && + bd->props.brightness != 0; + panel->backlight_power(connector, enable); + } + } else { + bd->props.power = FB_BLANK_POWERDOWN; + } + drm_modeset_unlock(&dev->mode_config.connection_mutex); return 0; } @@ -1023,6 +1040,11 @@ static int intel_backlight_device_register(struct intel_connector *connector) panel->backlight.level, props.max_brightness); + if (panel->backlight.enabled) + props.power = FB_BLANK_UNBLANK; + else + props.power = FB_BLANK_POWERDOWN; + /* * Note: using the same name independent of the connector prevents * registration of multiple backlight devices in the driver. @@ -1203,7 +1225,7 @@ static int vlv_setup_backlight(struct intel_connector *connector) enum pipe pipe; u32 ctl, ctl2, val; - for_each_pipe(pipe) { + for_each_pipe(dev_priv, pipe) { u32 cur_val = I915_READ(VLV_BLC_PWM_CTL(pipe)); /* Skip if the modulation freq is already set */ @@ -1240,7 +1262,6 @@ int intel_panel_setup_backlight(struct drm_connector *connector) struct drm_i915_private *dev_priv = dev->dev_private; struct intel_connector *intel_connector = to_intel_connector(connector); struct intel_panel *panel = &intel_connector->panel; - unsigned long flags; int ret; if (!dev_priv->vbt.backlight.present) { @@ -1253,9 +1274,9 @@ int intel_panel_setup_backlight(struct drm_connector *connector) } /* set level and max in panel struct */ - spin_lock_irqsave(&dev_priv->backlight_lock, flags); + mutex_lock(&dev_priv->backlight_lock); ret = dev_priv->display.setup_backlight(intel_connector); - spin_unlock_irqrestore(&dev_priv->backlight_lock, flags); + mutex_unlock(&dev_priv->backlight_lock); if (ret) { DRM_DEBUG_KMS("failed to setup backlight for connector %s\n", diff --git a/drivers/gpu/drm/i915/intel_pm.c b/drivers/gpu/drm/i915/intel_pm.c index 40c1229..490bd69 100644 --- a/drivers/gpu/drm/i915/intel_pm.c +++ b/drivers/gpu/drm/i915/intel_pm.c @@ -309,6 +309,9 @@ static void gen7_enable_fbc(struct drm_crtc *crtc) dpfc_ctl |= IVB_DPFC_CTL_FENCE_EN; + if (dev_priv->fbc.false_color) + dpfc_ctl |= FBC_CTL_FALSE_COLOR; + I915_WRITE(ILK_DPFC_CONTROL, dpfc_ctl | DPFC_CTL_EN); if (IS_IVYBRIDGE(dev)) { @@ -336,12 +339,31 @@ bool intel_fbc_enabled(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; + /* If it wasn't never enabled by kernel parameter or platform default + * we can avoid reading registers so many times in vain + */ + if (!i915.enable_fbc) + return false; + if (!dev_priv->display.fbc_enabled) return false; return dev_priv->display.fbc_enabled(dev); } +void gen8_fbc_sw_flush(struct drm_device *dev, u32 value) +{ + struct drm_i915_private *dev_priv = dev->dev_private; + + if (!IS_GEN8(dev)) + return; + + if (!intel_fbc_enabled(dev)) + return; + + I915_WRITE(MSG_FBC_REND_STATE, value); +} + static void intel_fbc_work_fn(struct work_struct *__work) { struct intel_fbc_work *work = @@ -578,6 +600,12 @@ void intel_update_fbc(struct drm_device *dev) DRM_DEBUG_KMS("framebuffer not tiled or fenced, disabling compression\n"); goto out_disable; } + if (INTEL_INFO(dev)->gen <= 4 && !IS_G4X(dev) && + to_intel_plane(crtc->primary)->rotation != BIT(DRM_ROTATE_0)) { + if (set_no_fbc_reason(dev_priv, FBC_UNSUPPORTED_MODE)) + DRM_DEBUG_KMS("Rotation unsupported, disabling\n"); + goto out_disable; + } /* If the kernel debugger is active, always disable compression */ if (in_dbg_master()) @@ -853,7 +881,7 @@ void intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable) * A value of 5us seems to be a good balance; safe for very low end * platforms but not overly aggressive on lower latency configs. */ -static const int latency_ns = 5000; +static const int pessimal_latency_ns = 5000; static int i9xx_get_fifo_size(struct drm_device *dev, int plane) { @@ -982,13 +1010,20 @@ static const struct intel_watermark_params i915_wm_info = { .guard_size = 2, .cacheline_size = I915_FIFO_LINE_SIZE, }; -static const struct intel_watermark_params i830_wm_info = { +static const struct intel_watermark_params i830_a_wm_info = { .fifo_size = I855GM_FIFO_SIZE, .max_wm = I915_MAX_WM, .default_wm = 1, .guard_size = 2, .cacheline_size = I830_FIFO_LINE_SIZE, }; +static const struct intel_watermark_params i830_bc_wm_info = { + .fifo_size = I855GM_FIFO_SIZE, + .max_wm = I915_MAX_WM/2, + .default_wm = 1, + .guard_size = 2, + .cacheline_size = I830_FIFO_LINE_SIZE, +}; static const struct intel_watermark_params i845_wm_info = { .fifo_size = I830_FIFO_SIZE, .max_wm = I915_MAX_WM, @@ -1044,6 +1079,17 @@ static unsigned long intel_calculate_wm(unsigned long clock_in_khz, wm_size = wm->max_wm; if (wm_size <= 0) wm_size = wm->default_wm; + + /* + * Bspec seems to indicate that the value shouldn't be lower than + * 'burst size + 1'. Certainly 830 is quite unhappy with low values. + * Lets go for 8 which is the burst size since certain platforms + * already use a hardcoded 8 (which is what the spec says should be + * done). + */ + if (wm_size <= 8) + wm_size = 8; + return wm_size; } @@ -1268,33 +1314,27 @@ static bool g4x_compute_srwm(struct drm_device *dev, display, cursor); } -static bool vlv_compute_drain_latency(struct drm_device *dev, - int plane, - int *plane_prec_mult, - int *plane_dl, - int *cursor_prec_mult, - int *cursor_dl) +static bool vlv_compute_drain_latency(struct drm_crtc *crtc, + int pixel_size, + int *prec_mult, + int *drain_latency) { - struct drm_crtc *crtc; - int clock, pixel_size; int entries; + int clock = to_intel_crtc(crtc)->config.adjusted_mode.crtc_clock; - crtc = intel_get_crtc_for_plane(dev, plane); - if (!intel_crtc_active(crtc)) + if (WARN(clock == 0, "Pixel clock is zero!\n")) return false; - clock = to_intel_crtc(crtc)->config.adjusted_mode.crtc_clock; - pixel_size = crtc->primary->fb->bits_per_pixel / 8; /* BPP */ + if (WARN(pixel_size == 0, "Pixel size is zero!\n")) + return false; - entries = (clock / 1000) * pixel_size; - *plane_prec_mult = (entries > 128) ? - DRAIN_LATENCY_PRECISION_64 : DRAIN_LATENCY_PRECISION_32; - *plane_dl = (64 * (*plane_prec_mult) * 4) / entries; + entries = DIV_ROUND_UP(clock, 1000) * pixel_size; + *prec_mult = (entries > 128) ? DRAIN_LATENCY_PRECISION_64 : + DRAIN_LATENCY_PRECISION_32; + *drain_latency = (64 * (*prec_mult) * 4) / entries; - entries = (clock / 1000) * 4; /* BPP is always 4 for cursor */ - *cursor_prec_mult = (entries > 128) ? - DRAIN_LATENCY_PRECISION_64 : DRAIN_LATENCY_PRECISION_32; - *cursor_dl = (64 * (*cursor_prec_mult) * 4) / entries; + if (*drain_latency > DRAIN_LATENCY_MASK) + *drain_latency = DRAIN_LATENCY_MASK; return true; } @@ -1307,39 +1347,48 @@ static bool vlv_compute_drain_latency(struct drm_device *dev, * latency value. */ -static void vlv_update_drain_latency(struct drm_device *dev) +static void vlv_update_drain_latency(struct drm_crtc *crtc) { - struct drm_i915_private *dev_priv = dev->dev_private; - int planea_prec, planea_dl, planeb_prec, planeb_dl; - int cursora_prec, cursora_dl, cursorb_prec, cursorb_dl; - int plane_prec_mult, cursor_prec_mult; /* Precision multiplier is - either 16 or 32 */ + struct drm_i915_private *dev_priv = crtc->dev->dev_private; + struct intel_crtc *intel_crtc = to_intel_crtc(crtc); + int pixel_size; + int drain_latency; + enum pipe pipe = intel_crtc->pipe; + int plane_prec, prec_mult, plane_dl; + + plane_dl = I915_READ(VLV_DDL(pipe)) & ~(DDL_PLANE_PRECISION_64 | + DRAIN_LATENCY_MASK | DDL_CURSOR_PRECISION_64 | + (DRAIN_LATENCY_MASK << DDL_CURSOR_SHIFT)); - /* For plane A, Cursor A */ - if (vlv_compute_drain_latency(dev, 0, &plane_prec_mult, &planea_dl, - &cursor_prec_mult, &cursora_dl)) { - cursora_prec = (cursor_prec_mult == DRAIN_LATENCY_PRECISION_32) ? - DDL_CURSORA_PRECISION_32 : DDL_CURSORA_PRECISION_64; - planea_prec = (plane_prec_mult == DRAIN_LATENCY_PRECISION_32) ? - DDL_PLANEA_PRECISION_32 : DDL_PLANEA_PRECISION_64; + if (!intel_crtc_active(crtc)) { + I915_WRITE(VLV_DDL(pipe), plane_dl); + return; + } - I915_WRITE(VLV_DDL1, cursora_prec | - (cursora_dl << DDL_CURSORA_SHIFT) | - planea_prec | planea_dl); + /* Primary plane Drain Latency */ + pixel_size = crtc->primary->fb->bits_per_pixel / 8; /* BPP */ + if (vlv_compute_drain_latency(crtc, pixel_size, &prec_mult, &drain_latency)) { + plane_prec = (prec_mult == DRAIN_LATENCY_PRECISION_64) ? + DDL_PLANE_PRECISION_64 : + DDL_PLANE_PRECISION_32; + plane_dl |= plane_prec | drain_latency; } - /* For plane B, Cursor B */ - if (vlv_compute_drain_latency(dev, 1, &plane_prec_mult, &planeb_dl, - &cursor_prec_mult, &cursorb_dl)) { - cursorb_prec = (cursor_prec_mult == DRAIN_LATENCY_PRECISION_32) ? - DDL_CURSORB_PRECISION_32 : DDL_CURSORB_PRECISION_64; - planeb_prec = (plane_prec_mult == DRAIN_LATENCY_PRECISION_32) ? - DDL_PLANEB_PRECISION_32 : DDL_PLANEB_PRECISION_64; + /* Cursor Drain Latency + * BPP is always 4 for cursor + */ + pixel_size = 4; - I915_WRITE(VLV_DDL2, cursorb_prec | - (cursorb_dl << DDL_CURSORB_SHIFT) | - planeb_prec | planeb_dl); + /* Program cursor DL only if it is enabled */ + if (intel_crtc->cursor_base && + vlv_compute_drain_latency(crtc, pixel_size, &prec_mult, &drain_latency)) { + plane_prec = (prec_mult == DRAIN_LATENCY_PRECISION_64) ? + DDL_CURSOR_PRECISION_64 : + DDL_CURSOR_PRECISION_32; + plane_dl |= plane_prec | (drain_latency << DDL_CURSOR_SHIFT); } + + I915_WRITE(VLV_DDL(pipe), plane_dl); } #define single_plane_enabled(mask) is_power_of_2(mask) @@ -1355,17 +1404,17 @@ static void valleyview_update_wm(struct drm_crtc *crtc) unsigned int enabled = 0; bool cxsr_enabled; - vlv_update_drain_latency(dev); + vlv_update_drain_latency(crtc); if (g4x_compute_wm0(dev, PIPE_A, - &valleyview_wm_info, latency_ns, - &valleyview_cursor_wm_info, latency_ns, + &valleyview_wm_info, pessimal_latency_ns, + &valleyview_cursor_wm_info, pessimal_latency_ns, &planea_wm, &cursora_wm)) enabled |= 1 << PIPE_A; if (g4x_compute_wm0(dev, PIPE_B, - &valleyview_wm_info, latency_ns, - &valleyview_cursor_wm_info, latency_ns, + &valleyview_wm_info, pessimal_latency_ns, + &valleyview_cursor_wm_info, pessimal_latency_ns, &planeb_wm, &cursorb_wm)) enabled |= 1 << PIPE_B; @@ -1387,7 +1436,8 @@ static void valleyview_update_wm(struct drm_crtc *crtc) plane_sr = cursor_sr = 0; } - DRM_DEBUG_KMS("Setting FIFO watermarks - A: plane=%d, cursor=%d, B: plane=%d, cursor=%d, SR: plane=%d, cursor=%d\n", + DRM_DEBUG_KMS("Setting FIFO watermarks - A: plane=%d, cursor=%d, " + "B: plane=%d, cursor=%d, SR: plane=%d, cursor=%d\n", planea_wm, cursora_wm, planeb_wm, cursorb_wm, plane_sr, cursor_sr); @@ -1396,7 +1446,7 @@ static void valleyview_update_wm(struct drm_crtc *crtc) (plane_sr << DSPFW_SR_SHIFT) | (cursorb_wm << DSPFW_CURSORB_SHIFT) | (planeb_wm << DSPFW_PLANEB_SHIFT) | - planea_wm); + (planea_wm << DSPFW_PLANEA_SHIFT)); I915_WRITE(DSPFW2, (I915_READ(DSPFW2) & ~DSPFW_CURSORA_MASK) | (cursora_wm << DSPFW_CURSORA_SHIFT)); @@ -1408,6 +1458,116 @@ static void valleyview_update_wm(struct drm_crtc *crtc) intel_set_memory_cxsr(dev_priv, true); } +static void cherryview_update_wm(struct drm_crtc *crtc) +{ + struct drm_device *dev = crtc->dev; + static const int sr_latency_ns = 12000; + struct drm_i915_private *dev_priv = dev->dev_private; + int planea_wm, planeb_wm, planec_wm; + int cursora_wm, cursorb_wm, cursorc_wm; + int plane_sr, cursor_sr; + int ignore_plane_sr, ignore_cursor_sr; + unsigned int enabled = 0; + bool cxsr_enabled; + + vlv_update_drain_latency(crtc); + + if (g4x_compute_wm0(dev, PIPE_A, + &valleyview_wm_info, pessimal_latency_ns, + &valleyview_cursor_wm_info, pessimal_latency_ns, + &planea_wm, &cursora_wm)) + enabled |= 1 << PIPE_A; + + if (g4x_compute_wm0(dev, PIPE_B, + &valleyview_wm_info, pessimal_latency_ns, + &valleyview_cursor_wm_info, pessimal_latency_ns, + &planeb_wm, &cursorb_wm)) + enabled |= 1 << PIPE_B; + + if (g4x_compute_wm0(dev, PIPE_C, + &valleyview_wm_info, pessimal_latency_ns, + &valleyview_cursor_wm_info, pessimal_latency_ns, + &planec_wm, &cursorc_wm)) + enabled |= 1 << PIPE_C; + + if (single_plane_enabled(enabled) && + g4x_compute_srwm(dev, ffs(enabled) - 1, + sr_latency_ns, + &valleyview_wm_info, + &valleyview_cursor_wm_info, + &plane_sr, &ignore_cursor_sr) && + g4x_compute_srwm(dev, ffs(enabled) - 1, + 2*sr_latency_ns, + &valleyview_wm_info, + &valleyview_cursor_wm_info, + &ignore_plane_sr, &cursor_sr)) { + cxsr_enabled = true; + } else { + cxsr_enabled = false; + intel_set_memory_cxsr(dev_priv, false); + plane_sr = cursor_sr = 0; + } + + DRM_DEBUG_KMS("Setting FIFO watermarks - A: plane=%d, cursor=%d, " + "B: plane=%d, cursor=%d, C: plane=%d, cursor=%d, " + "SR: plane=%d, cursor=%d\n", + planea_wm, cursora_wm, + planeb_wm, cursorb_wm, + planec_wm, cursorc_wm, + plane_sr, cursor_sr); + + I915_WRITE(DSPFW1, + (plane_sr << DSPFW_SR_SHIFT) | + (cursorb_wm << DSPFW_CURSORB_SHIFT) | + (planeb_wm << DSPFW_PLANEB_SHIFT) | + (planea_wm << DSPFW_PLANEA_SHIFT)); + I915_WRITE(DSPFW2, + (I915_READ(DSPFW2) & ~DSPFW_CURSORA_MASK) | + (cursora_wm << DSPFW_CURSORA_SHIFT)); + I915_WRITE(DSPFW3, + (I915_READ(DSPFW3) & ~DSPFW_CURSOR_SR_MASK) | + (cursor_sr << DSPFW_CURSOR_SR_SHIFT)); + I915_WRITE(DSPFW9_CHV, + (I915_READ(DSPFW9_CHV) & ~(DSPFW_PLANEC_MASK | + DSPFW_CURSORC_MASK)) | + (planec_wm << DSPFW_PLANEC_SHIFT) | + (cursorc_wm << DSPFW_CURSORC_SHIFT)); + + if (cxsr_enabled) + intel_set_memory_cxsr(dev_priv, true); +} + +static void valleyview_update_sprite_wm(struct drm_plane *plane, + struct drm_crtc *crtc, + uint32_t sprite_width, + uint32_t sprite_height, + int pixel_size, + bool enabled, bool scaled) +{ + struct drm_device *dev = crtc->dev; + struct drm_i915_private *dev_priv = dev->dev_private; + int pipe = to_intel_plane(plane)->pipe; + int sprite = to_intel_plane(plane)->plane; + int drain_latency; + int plane_prec; + int sprite_dl; + int prec_mult; + + sprite_dl = I915_READ(VLV_DDL(pipe)) & ~(DDL_SPRITE_PRECISION_64(sprite) | + (DRAIN_LATENCY_MASK << DDL_SPRITE_SHIFT(sprite))); + + if (enabled && vlv_compute_drain_latency(crtc, pixel_size, &prec_mult, + &drain_latency)) { + plane_prec = (prec_mult == DRAIN_LATENCY_PRECISION_64) ? + DDL_SPRITE_PRECISION_64(sprite) : + DDL_SPRITE_PRECISION_32(sprite); + sprite_dl |= plane_prec | + (drain_latency << DDL_SPRITE_SHIFT(sprite)); + } + + I915_WRITE(VLV_DDL(pipe), sprite_dl); +} + static void g4x_update_wm(struct drm_crtc *crtc) { struct drm_device *dev = crtc->dev; @@ -1419,14 +1579,14 @@ static void g4x_update_wm(struct drm_crtc *crtc) bool cxsr_enabled; if (g4x_compute_wm0(dev, PIPE_A, - &g4x_wm_info, latency_ns, - &g4x_cursor_wm_info, latency_ns, + &g4x_wm_info, pessimal_latency_ns, + &g4x_cursor_wm_info, pessimal_latency_ns, &planea_wm, &cursora_wm)) enabled |= 1 << PIPE_A; if (g4x_compute_wm0(dev, PIPE_B, - &g4x_wm_info, latency_ns, - &g4x_cursor_wm_info, latency_ns, + &g4x_wm_info, pessimal_latency_ns, + &g4x_cursor_wm_info, pessimal_latency_ns, &planeb_wm, &cursorb_wm)) enabled |= 1 << PIPE_B; @@ -1443,7 +1603,8 @@ static void g4x_update_wm(struct drm_crtc *crtc) plane_sr = cursor_sr = 0; } - DRM_DEBUG_KMS("Setting FIFO watermarks - A: plane=%d, cursor=%d, B: plane=%d, cursor=%d, SR: plane=%d, cursor=%d\n", + DRM_DEBUG_KMS("Setting FIFO watermarks - A: plane=%d, cursor=%d, " + "B: plane=%d, cursor=%d, SR: plane=%d, cursor=%d\n", planea_wm, cursora_wm, planeb_wm, cursorb_wm, plane_sr, cursor_sr); @@ -1452,7 +1613,7 @@ static void g4x_update_wm(struct drm_crtc *crtc) (plane_sr << DSPFW_SR_SHIFT) | (cursorb_wm << DSPFW_CURSORB_SHIFT) | (planeb_wm << DSPFW_PLANEB_SHIFT) | - planea_wm); + (planea_wm << DSPFW_PLANEA_SHIFT)); I915_WRITE(DSPFW2, (I915_READ(DSPFW2) & ~DSPFW_CURSORA_MASK) | (cursora_wm << DSPFW_CURSORA_SHIFT)); @@ -1526,8 +1687,11 @@ static void i965_update_wm(struct drm_crtc *unused_crtc) /* 965 has limitations... */ I915_WRITE(DSPFW1, (srwm << DSPFW_SR_SHIFT) | - (8 << 16) | (8 << 8) | (8 << 0)); - I915_WRITE(DSPFW2, (8 << 8) | (8 << 0)); + (8 << DSPFW_CURSORB_SHIFT) | + (8 << DSPFW_PLANEB_SHIFT) | + (8 << DSPFW_PLANEA_SHIFT)); + I915_WRITE(DSPFW2, (8 << DSPFW_CURSORA_SHIFT) | + (8 << DSPFW_PLANEC_SHIFT_OLD)); /* update cursor SR watermark */ I915_WRITE(DSPFW3, (cursor_sr << DSPFW_CURSOR_SR_SHIFT)); @@ -1552,7 +1716,7 @@ static void i9xx_update_wm(struct drm_crtc *unused_crtc) else if (!IS_GEN2(dev)) wm_info = &i915_wm_info; else - wm_info = &i830_wm_info; + wm_info = &i830_a_wm_info; fifo_size = dev_priv->display.get_fifo_size(dev, 0); crtc = intel_get_crtc_for_plane(dev, 0); @@ -1565,10 +1729,16 @@ static void i9xx_update_wm(struct drm_crtc *unused_crtc) adjusted_mode = &to_intel_crtc(crtc)->config.adjusted_mode; planea_wm = intel_calculate_wm(adjusted_mode->crtc_clock, wm_info, fifo_size, cpp, - latency_ns); + pessimal_latency_ns); enabled = crtc; - } else + } else { planea_wm = fifo_size - wm_info->guard_size; + if (planea_wm > (long)wm_info->max_wm) + planea_wm = wm_info->max_wm; + } + + if (IS_GEN2(dev)) + wm_info = &i830_bc_wm_info; fifo_size = dev_priv->display.get_fifo_size(dev, 1); crtc = intel_get_crtc_for_plane(dev, 1); @@ -1581,13 +1751,16 @@ static void i9xx_update_wm(struct drm_crtc *unused_crtc) adjusted_mode = &to_intel_crtc(crtc)->config.adjusted_mode; planeb_wm = intel_calculate_wm(adjusted_mode->crtc_clock, wm_info, fifo_size, cpp, - latency_ns); + pessimal_latency_ns); if (enabled == NULL) enabled = crtc; else enabled = NULL; - } else + } else { planeb_wm = fifo_size - wm_info->guard_size; + if (planeb_wm > (long)wm_info->max_wm) + planeb_wm = wm_info->max_wm; + } DRM_DEBUG_KMS("FIFO watermarks - A: %d, B: %d\n", planea_wm, planeb_wm); @@ -1674,7 +1847,7 @@ static void i845_update_wm(struct drm_crtc *unused_crtc) planea_wm = intel_calculate_wm(adjusted_mode->crtc_clock, &i845_wm_info, dev_priv->display.get_fifo_size(dev, 0), - 4, latency_ns); + 4, pessimal_latency_ns); fwater_lo = I915_READ(FW_BLC) & ~0xfff; fwater_lo |= (3<<8) | planea_wm; @@ -1683,23 +1856,21 @@ static void i845_update_wm(struct drm_crtc *unused_crtc) I915_WRITE(FW_BLC, fwater_lo); } -static uint32_t ilk_pipe_pixel_rate(struct drm_device *dev, - struct drm_crtc *crtc) +uint32_t ilk_pipe_pixel_rate(const struct intel_crtc_config *pipe_config) { - struct intel_crtc *intel_crtc = to_intel_crtc(crtc); uint32_t pixel_rate; - pixel_rate = intel_crtc->config.adjusted_mode.crtc_clock; + pixel_rate = pipe_config->adjusted_mode.crtc_clock; /* We only use IF-ID interlacing. If we ever use PF-ID we'll need to * adjust the pixel_rate here. */ - if (intel_crtc->config.pch_pfit.enabled) { + if (pipe_config->pch_pfit.enabled) { uint64_t pipe_w, pipe_h, pfit_w, pfit_h; - uint32_t pfit_size = intel_crtc->config.pch_pfit.size; + uint32_t pfit_size = pipe_config->pch_pfit.size; - pipe_w = intel_crtc->config.pipe_src_w; - pipe_h = intel_crtc->config.pipe_src_h; + pipe_w = pipe_config->pipe_src_w; + pipe_h = pipe_config->pipe_src_h; pfit_w = (pfit_size >> 16) & 0xFFFF; pfit_h = pfit_size & 0xFFFF; if (pipe_w < pfit_w) @@ -2121,7 +2292,6 @@ int ilk_wm_max_level(const struct drm_device *dev) else return 2; } - static void intel_print_wm_latency(struct drm_device *dev, const char *name, const uint16_t wm[5]) @@ -2219,7 +2389,7 @@ static void ilk_compute_wm_parameters(struct drm_crtc *crtc, p->active = true; p->pipe_htotal = intel_crtc->config.adjusted_mode.crtc_htotal; - p->pixel_rate = ilk_pipe_pixel_rate(dev, crtc); + p->pixel_rate = ilk_pipe_pixel_rate(&intel_crtc->config); p->pri.bytes_per_pixel = crtc->primary->fb->bits_per_pixel / 8; p->cur.bytes_per_pixel = 4; p->pri.horiz_pixels = intel_crtc->config.pipe_src_w; @@ -2527,7 +2697,7 @@ static struct intel_pipe_wm *ilk_find_best_result(struct drm_device *dev, #define WM_DIRTY_FBC (1 << 24) #define WM_DIRTY_DDB (1 << 25) -static unsigned int ilk_compute_wm_dirty(struct drm_device *dev, +static unsigned int ilk_compute_wm_dirty(struct drm_i915_private *dev_priv, const struct ilk_wm_values *old, const struct ilk_wm_values *new) { @@ -2535,7 +2705,7 @@ static unsigned int ilk_compute_wm_dirty(struct drm_device *dev, enum pipe pipe; int wm_lp; - for_each_pipe(pipe) { + for_each_pipe(dev_priv, pipe) { if (old->wm_linetime[pipe] != new->wm_linetime[pipe]) { dirty |= WM_DIRTY_LINETIME(pipe); /* Must disable LP1+ watermarks too */ @@ -2621,7 +2791,7 @@ static void ilk_write_wm_values(struct drm_i915_private *dev_priv, unsigned int dirty; uint32_t val; - dirty = ilk_compute_wm_dirty(dev, previous, results); + dirty = ilk_compute_wm_dirty(dev_priv, previous, results); if (!dirty) return; @@ -3090,6 +3260,9 @@ static void gen6_set_rps_thresholds(struct drm_i915_private *dev_priv, u8 val) { int new_power; + if (dev_priv->rps.is_bdw_sw_turbo) + return; + new_power = dev_priv->rps.power; switch (dev_priv->rps.power) { case LOW_POWER: @@ -3297,8 +3470,11 @@ void gen6_rps_idle(struct drm_i915_private *dev_priv) valleyview_set_rps(dev_priv->dev, dev_priv->rps.min_freq_softlimit); else if (IS_VALLEYVIEW(dev)) vlv_set_rps_idle(dev_priv); - else + else if (!dev_priv->rps.is_bdw_sw_turbo + || atomic_read(&dev_priv->rps.sw_turbo.flip_received)){ gen6_set_rps(dev_priv->dev, dev_priv->rps.min_freq_softlimit); + } + dev_priv->rps.last_adj = 0; } mutex_unlock(&dev_priv->rps.hw_lock); @@ -3312,8 +3488,11 @@ void gen6_rps_boost(struct drm_i915_private *dev_priv) if (dev_priv->rps.enabled) { if (IS_VALLEYVIEW(dev)) valleyview_set_rps(dev_priv->dev, dev_priv->rps.max_freq_softlimit); - else + else if (!dev_priv->rps.is_bdw_sw_turbo + || atomic_read(&dev_priv->rps.sw_turbo.flip_received)){ gen6_set_rps(dev_priv->dev, dev_priv->rps.max_freq_softlimit); + } + dev_priv->rps.last_adj = 0; } mutex_unlock(&dev_priv->rps.hw_lock); @@ -3332,6 +3511,10 @@ void valleyview_set_rps(struct drm_device *dev, u8 val) dev_priv->rps.cur_freq, vlv_gpu_freq(dev_priv, val), val); + if (WARN_ONCE(IS_CHERRYVIEW(dev) && (val & 1), + "Odd GPU freq value\n")) + val &= ~1; + if (val != dev_priv->rps.cur_freq) vlv_punit_write(dev_priv, PUNIT_REG_GPU_FREQ_REQ, val); @@ -3344,21 +3527,26 @@ void valleyview_set_rps(struct drm_device *dev, u8 val) static void gen8_disable_rps_interrupts(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; + if (IS_BROADWELL(dev) && dev_priv->rps.is_bdw_sw_turbo){ + if (atomic_read(&dev_priv->rps.sw_turbo.flip_received)) + del_timer(&dev_priv->rps.sw_turbo.flip_timer); + dev_priv-> rps.is_bdw_sw_turbo = false; + } else { + I915_WRITE(GEN6_PMINTRMSK, ~GEN8_PMINTR_REDIRECT_TO_NON_DISP); + I915_WRITE(GEN8_GT_IER(2), I915_READ(GEN8_GT_IER(2)) & + ~dev_priv->pm_rps_events); + /* Complete PM interrupt masking here doesn't race with the rps work + * item again unmasking PM interrupts because that is using a different + * register (GEN8_GT_IMR(2)) to mask PM interrupts. The only risk is in + * leaving stale bits in GEN8_GT_IIR(2) and GEN8_GT_IMR(2) which + * gen8_enable_rps will clean up. */ - I915_WRITE(GEN6_PMINTRMSK, ~GEN8_PMINTR_REDIRECT_TO_NON_DISP); - I915_WRITE(GEN8_GT_IER(2), I915_READ(GEN8_GT_IER(2)) & - ~dev_priv->pm_rps_events); - /* Complete PM interrupt masking here doesn't race with the rps work - * item again unmasking PM interrupts because that is using a different - * register (GEN8_GT_IMR(2)) to mask PM interrupts. The only risk is in - * leaving stale bits in GEN8_GT_IIR(2) and GEN8_GT_IMR(2) which - * gen8_enable_rps will clean up. */ - - spin_lock_irq(&dev_priv->irq_lock); - dev_priv->rps.pm_iir = 0; - spin_unlock_irq(&dev_priv->irq_lock); + spin_lock_irq(&dev_priv->irq_lock); + dev_priv->rps.pm_iir = 0; + spin_unlock_irq(&dev_priv->irq_lock); - I915_WRITE(GEN8_GT_IIR(2), dev_priv->pm_rps_events); + I915_WRITE(GEN8_GT_IIR(2), dev_priv->pm_rps_events); + } } static void gen6_disable_rps_interrupts(struct drm_device *dev) @@ -3406,8 +3594,14 @@ static void valleyview_disable_rps(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; + /* we're doing forcewake before Disabling RC6, + * This what the BIOS expects when going into suspend */ + gen6_gt_force_wake_get(dev_priv, FORCEWAKE_ALL); + I915_WRITE(GEN6_RC_CONTROL, 0); + gen6_gt_force_wake_put(dev_priv, FORCEWAKE_ALL); + gen6_disable_rps_interrupts(dev); } @@ -3452,9 +3646,11 @@ static int sanitize_rc6_option(const struct drm_device *dev, int enable_rc6) return enable_rc6 & mask; } - /* Disable RC6 on Ironlake */ - if (INTEL_INFO(dev)->gen == 5) +#ifdef CONFIG_INTEL_IOMMU + /* Ironlake + RC6 + VT-d empirically blows up */ + if (IS_GEN5(dev) && intel_iommu_gfx_mapped) return 0; +#endif if (IS_IVYBRIDGE(dev)) return (INTEL_RC6_ENABLE | INTEL_RC6p_ENABLE); @@ -3510,13 +3706,111 @@ static void parse_rp_state_cap(struct drm_i915_private *dev_priv, u32 rp_state_c dev_priv->rps.min_freq_softlimit = dev_priv->rps.min_freq; } +static void bdw_sw_calculate_freq(struct drm_device *dev, + struct intel_rps_bdw_cal *c, u32 *cur_time, u32 *c0) +{ + struct drm_i915_private *dev_priv = dev->dev_private; + u64 busy = 0; + u32 busyness_pct = 0; + u32 elapsed_time = 0; + u16 new_freq = 0; + + if (!c || !cur_time || !c0) + return; + + if (0 == c->last_c0) + goto out; + + /* Check Evaluation interval */ + elapsed_time = *cur_time - c->last_ts; + if (elapsed_time < c->eval_interval) + return; + + mutex_lock(&dev_priv->rps.hw_lock); + + /* + * c0 unit in 32*1.28 usec, elapsed_time unit in 1 usec. + * Whole busyness_pct calculation should be + * busy = ((u64)(*c0 - c->last_c0) << 5 << 7) / 100; + * busyness_pct = (u32)(busy * 100 / elapsed_time); + * The final formula is to simplify CPU calculation + */ + busy = (u64)(*c0 - c->last_c0) << 12; + do_div(busy, elapsed_time); + busyness_pct = (u32)busy; + + if (c->is_up && busyness_pct >= c->it_threshold_pct) + new_freq = (u16)dev_priv->rps.cur_freq + 3; + if (!c->is_up && busyness_pct <= c->it_threshold_pct) + new_freq = (u16)dev_priv->rps.cur_freq - 1; + + /* Adjust to new frequency busyness and compare with threshold */ + if (0 != new_freq) { + if (new_freq > dev_priv->rps.max_freq_softlimit) + new_freq = dev_priv->rps.max_freq_softlimit; + else if (new_freq < dev_priv->rps.min_freq_softlimit) + new_freq = dev_priv->rps.min_freq_softlimit; + + gen6_set_rps(dev, new_freq); + } + + mutex_unlock(&dev_priv->rps.hw_lock); + +out: + c->last_c0 = *c0; + c->last_ts = *cur_time; +} + +static void gen8_set_frequency_RP0(struct work_struct *work) +{ + struct intel_rps_bdw_turbo *p_bdw_turbo = + container_of(work, struct intel_rps_bdw_turbo, work_max_freq); + struct intel_gen6_power_mgmt *p_power_mgmt = + container_of(p_bdw_turbo, struct intel_gen6_power_mgmt, sw_turbo); + struct drm_i915_private *dev_priv = + container_of(p_power_mgmt, struct drm_i915_private, rps); + + mutex_lock(&dev_priv->rps.hw_lock); + gen6_set_rps(dev_priv->dev, dev_priv->rps.rp0_freq); + mutex_unlock(&dev_priv->rps.hw_lock); +} + +static void flip_active_timeout_handler(unsigned long var) +{ + struct drm_i915_private *dev_priv = (struct drm_i915_private *) var; + + del_timer(&dev_priv->rps.sw_turbo.flip_timer); + atomic_set(&dev_priv->rps.sw_turbo.flip_received, false); + + queue_work(dev_priv->wq, &dev_priv->rps.sw_turbo.work_max_freq); +} + +void bdw_software_turbo(struct drm_device *dev) +{ + struct drm_i915_private *dev_priv = dev->dev_private; + + u32 current_time = I915_READ(TIMESTAMP_CTR); /* unit in usec */ + u32 current_c0 = I915_READ(MCHBAR_PCU_C0); /* unit in 32*1.28 usec */ + + bdw_sw_calculate_freq(dev, &dev_priv->rps.sw_turbo.up, + ¤t_time, ¤t_c0); + bdw_sw_calculate_freq(dev, &dev_priv->rps.sw_turbo.down, + ¤t_time, ¤t_c0); +} + static void gen8_enable_rps(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; - struct intel_engine_cs *ring; + struct intel_engine_cs *engine; uint32_t rc6_mask = 0, rp_state_cap; + uint32_t threshold_up_pct, threshold_down_pct; + uint32_t ei_up, ei_down; /* up and down evaluation interval */ + u32 rp_ctl_flag; int unused; + /* Use software Turbo for BDW */ + dev_priv->rps.is_bdw_sw_turbo = IS_BROADWELL(dev); + /* 1a: Software RC state - RC0 */ I915_WRITE(GEN6_RC_STATE, 0); @@ -3534,8 +3828,8 @@ static void gen8_enable_rps(struct drm_device *dev) I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 40 << 16); I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000); /* 12500 * 1280ns */ I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25); /* 25 * 1280ns */ - for_each_ring(ring, dev_priv, unused) - I915_WRITE(RING_MAX_IDLE(ring->mmio_base), 10); + for_each_engine(engine, dev_priv, unused) + I915_WRITE(RING_MAX_IDLE(engine->mmio_base), 10); I915_WRITE(GEN6_RC_SLEEP, 0); if (IS_BROADWELL(dev)) I915_WRITE(GEN6_RC6_THRESHOLD, 625); /* 800us/1.28 for TO */ @@ -3560,35 +3854,74 @@ static void gen8_enable_rps(struct drm_device *dev) HSW_FREQUENCY(dev_priv->rps.rp1_freq)); I915_WRITE(GEN6_RC_VIDEO_FREQ, HSW_FREQUENCY(dev_priv->rps.rp1_freq)); - /* NB: Docs say 1s, and 1000000 - which aren't equivalent */ - I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 100000000 / 128); /* 1 second timeout */ + ei_up = 84480; /* 84.48ms */ + ei_down = 448000; + threshold_up_pct = 90; /* x percent busy */ + threshold_down_pct = 70; + + if (dev_priv->rps.is_bdw_sw_turbo) { + dev_priv->rps.sw_turbo.up.it_threshold_pct = threshold_up_pct; + dev_priv->rps.sw_turbo.up.eval_interval = ei_up; + dev_priv->rps.sw_turbo.up.is_up = true; + dev_priv->rps.sw_turbo.up.last_ts = 0; + dev_priv->rps.sw_turbo.up.last_c0 = 0; + + dev_priv->rps.sw_turbo.down.it_threshold_pct = threshold_down_pct; + dev_priv->rps.sw_turbo.down.eval_interval = ei_down; + dev_priv->rps.sw_turbo.down.is_up = false; + dev_priv->rps.sw_turbo.down.last_ts = 0; + dev_priv->rps.sw_turbo.down.last_c0 = 0; + + /* Start the timer to track if flip comes*/ + dev_priv->rps.sw_turbo.timeout = 200*1000; /* in us */ + + init_timer(&dev_priv->rps.sw_turbo.flip_timer); + dev_priv->rps.sw_turbo.flip_timer.function = flip_active_timeout_handler; + dev_priv->rps.sw_turbo.flip_timer.data = (unsigned long) dev_priv; + dev_priv->rps.sw_turbo.flip_timer.expires = + usecs_to_jiffies(dev_priv->rps.sw_turbo.timeout) + jiffies; + add_timer(&dev_priv->rps.sw_turbo.flip_timer); + INIT_WORK(&dev_priv->rps.sw_turbo.work_max_freq, gen8_set_frequency_RP0); + + atomic_set(&dev_priv->rps.sw_turbo.flip_received, true); + } else { + /* NB: Docs say 1s, and 1000000 - which aren't equivalent + * 1 second timeout*/ + I915_WRITE(GEN6_RP_DOWN_TIMEOUT, FREQ_1_28_US(1000000)); - /* Docs recommend 900MHz, and 300 MHz respectively */ - I915_WRITE(GEN6_RP_INTERRUPT_LIMITS, - dev_priv->rps.max_freq_softlimit << 24 | - dev_priv->rps.min_freq_softlimit << 16); + /* Docs recommend 900MHz, and 300 MHz respectively */ + I915_WRITE(GEN6_RP_INTERRUPT_LIMITS, + dev_priv->rps.max_freq_softlimit << 24 | + dev_priv->rps.min_freq_softlimit << 16); - I915_WRITE(GEN6_RP_UP_THRESHOLD, 7600000 / 128); /* 76ms busyness per EI, 90% */ - I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 31300000 / 128); /* 313ms busyness per EI, 70%*/ - I915_WRITE(GEN6_RP_UP_EI, 66000); /* 84.48ms, XXX: random? */ - I915_WRITE(GEN6_RP_DOWN_EI, 350000); /* 448ms, XXX: random? */ + I915_WRITE(GEN6_RP_UP_THRESHOLD, + FREQ_1_28_US(ei_up * threshold_up_pct / 100)); + I915_WRITE(GEN6_RP_DOWN_THRESHOLD, + FREQ_1_28_US(ei_down * threshold_down_pct / 100)); + I915_WRITE(GEN6_RP_UP_EI, + FREQ_1_28_US(ei_up)); + I915_WRITE(GEN6_RP_DOWN_EI, + FREQ_1_28_US(ei_down)); - I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10); + I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10); + } /* 5: Enable RPS */ - I915_WRITE(GEN6_RP_CONTROL, - GEN6_RP_MEDIA_TURBO | - GEN6_RP_MEDIA_HW_NORMAL_MODE | - GEN6_RP_MEDIA_IS_GFX | - GEN6_RP_ENABLE | - GEN6_RP_UP_BUSY_AVG | - GEN6_RP_DOWN_IDLE_AVG); - - /* 6: Ring frequency + overclocking (our driver does this later */ - + rp_ctl_flag = GEN6_RP_MEDIA_TURBO | + GEN6_RP_MEDIA_HW_NORMAL_MODE | + GEN6_RP_MEDIA_IS_GFX | + GEN6_RP_UP_BUSY_AVG | + GEN6_RP_DOWN_IDLE_AVG; + if (!dev_priv->rps.is_bdw_sw_turbo) + rp_ctl_flag |= GEN6_RP_ENABLE; + + I915_WRITE(GEN6_RP_CONTROL, rp_ctl_flag); + + /* 6: Ring frequency + overclocking + * (our driver does this later */ gen6_set_rps(dev, (I915_READ(GEN6_GT_PERF_STATUS) & 0xff00) >> 8); - - gen8_enable_rps_interrupts(dev); + if (!dev_priv->rps.is_bdw_sw_turbo) + gen8_enable_rps_interrupts(dev); gen6_gt_force_wake_put(dev_priv, FORCEWAKE_ALL); } @@ -3596,9 +3929,8 @@ static void gen8_enable_rps(struct drm_device *dev) static void gen6_enable_rps(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; - struct intel_engine_cs *ring; + struct intel_engine_cs *engine; u32 rp_state_cap; - u32 gt_perf_status; u32 rc6vids, pcu_mbox = 0, rc6_mask = 0; u32 gtfifodbg; int rc6_mode; @@ -3623,7 +3955,6 @@ static void gen6_enable_rps(struct drm_device *dev) gen6_gt_force_wake_get(dev_priv, FORCEWAKE_ALL); rp_state_cap = I915_READ(GEN6_RP_STATE_CAP); - gt_perf_status = I915_READ(GEN6_GT_PERF_STATUS); parse_rp_state_cap(dev_priv, rp_state_cap); @@ -3636,8 +3967,8 @@ static void gen6_enable_rps(struct drm_device *dev) I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000); I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25); - for_each_ring(ring, dev_priv, i) - I915_WRITE(RING_MAX_IDLE(ring->mmio_base), 10); + for_each_engine(engine, dev_priv, i) + I915_WRITE(RING_MAX_IDLE(engine->mmio_base), 10); I915_WRITE(GEN6_RC_SLEEP, 0); I915_WRITE(GEN6_RC1e_THRESHOLD, 1000); @@ -3965,11 +4296,27 @@ static void valleyview_cleanup_pctx(struct drm_device *dev) static void valleyview_init_gt_powersave(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; + u32 val; valleyview_setup_pctx(dev); mutex_lock(&dev_priv->rps.hw_lock); + val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS); + switch ((val >> 6) & 3) { + case 0: + case 1: + dev_priv->mem_freq = 800; + break; + case 2: + dev_priv->mem_freq = 1066; + break; + case 3: + dev_priv->mem_freq = 1333; + break; + } + DRM_DEBUG_DRIVER("DDR speed: %d MHz", dev_priv->mem_freq); + dev_priv->rps.max_freq = valleyview_rps_max_freq(dev_priv); dev_priv->rps.rp0_freq = dev_priv->rps.max_freq; DRM_DEBUG_DRIVER("max GPU freq: %d MHz (%u)\n", @@ -4004,11 +4351,38 @@ static void valleyview_init_gt_powersave(struct drm_device *dev) static void cherryview_init_gt_powersave(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; + u32 val; cherryview_setup_pctx(dev); mutex_lock(&dev_priv->rps.hw_lock); + val = vlv_punit_read(dev_priv, CCK_FUSE_REG); + switch ((val >> 2) & 0x7) { + case 0: + case 1: + dev_priv->rps.cz_freq = 200; + dev_priv->mem_freq = 1600; + break; + case 2: + dev_priv->rps.cz_freq = 267; + dev_priv->mem_freq = 1600; + break; + case 3: + dev_priv->rps.cz_freq = 333; + dev_priv->mem_freq = 2000; + break; + case 4: + dev_priv->rps.cz_freq = 320; + dev_priv->mem_freq = 1600; + break; + case 5: + dev_priv->rps.cz_freq = 400; + dev_priv->mem_freq = 1600; + break; + } + DRM_DEBUG_DRIVER("DDR speed: %d MHz", dev_priv->mem_freq); + dev_priv->rps.max_freq = cherryview_rps_max_freq(dev_priv); dev_priv->rps.rp0_freq = dev_priv->rps.max_freq; DRM_DEBUG_DRIVER("max GPU freq: %d MHz (%u)\n", @@ -4030,6 +4404,12 @@ static void cherryview_init_gt_powersave(struct drm_device *dev) vlv_gpu_freq(dev_priv, dev_priv->rps.min_freq), dev_priv->rps.min_freq); + WARN_ONCE((dev_priv->rps.max_freq | + dev_priv->rps.efficient_freq | + dev_priv->rps.rp1_freq | + dev_priv->rps.min_freq) & 1, + "Odd GPU freq values\n"); + /* Preserve min/max settings in case of re-init */ if (dev_priv->rps.max_freq_softlimit == 0) dev_priv->rps.max_freq_softlimit = dev_priv->rps.max_freq; @@ -4048,7 +4428,7 @@ static void valleyview_cleanup_gt_powersave(struct drm_device *dev) static void cherryview_enable_rps(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; - struct intel_engine_cs *ring; + struct intel_engine_cs *engine; u32 gtfifodbg, val, rc6_mode = 0, pcbr; int i; @@ -4072,8 +4452,8 @@ static void cherryview_enable_rps(struct drm_device *dev) I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000); /* 12500 * 1280ns */ I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25); /* 25 * 1280ns */ - for_each_ring(ring, dev_priv, i) - I915_WRITE(RING_MAX_IDLE(ring->mmio_base), 10); + for_each_engine(engine, dev_priv, i) + I915_WRITE(RING_MAX_IDLE(engine->mmio_base), 10); I915_WRITE(GEN6_RC_SLEEP, 0); I915_WRITE(GEN6_RC6_THRESHOLD, 50000); /* 50/125ms per EI */ @@ -4140,7 +4520,7 @@ static void cherryview_enable_rps(struct drm_device *dev) static void valleyview_enable_rps(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; - struct intel_engine_cs *ring; + struct intel_engine_cs *engine; u32 gtfifodbg, val, rc6_mode = 0; int i; @@ -4177,8 +4557,8 @@ static void valleyview_enable_rps(struct drm_device *dev) I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000); I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25); - for_each_ring(ring, dev_priv, i) - I915_WRITE(RING_MAX_IDLE(ring->mmio_base), 10); + for_each_engine(engine, dev_priv, i) + I915_WRITE(RING_MAX_IDLE(engine->mmio_base), 10); I915_WRITE(GEN6_RC6_THRESHOLD, 0x557); @@ -4221,12 +4601,6 @@ void ironlake_teardown_rc6(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; - if (dev_priv->ips.renderctx) { - i915_gem_object_ggtt_unpin(dev_priv->ips.renderctx); - drm_gem_object_unreference(&dev_priv->ips.renderctx->base); - dev_priv->ips.renderctx = NULL; - } - if (dev_priv->ips.pwrctx) { i915_gem_object_ggtt_unpin(dev_priv->ips.pwrctx); drm_gem_object_unreference(&dev_priv->ips.pwrctx->base); @@ -4256,11 +4630,6 @@ static int ironlake_setup_rc6(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; - if (dev_priv->ips.renderctx == NULL) - dev_priv->ips.renderctx = intel_alloc_context_page(dev); - if (!dev_priv->ips.renderctx) - return -ENOMEM; - if (dev_priv->ips.pwrctx == NULL) dev_priv->ips.pwrctx = intel_alloc_context_page(dev); if (!dev_priv->ips.pwrctx) { @@ -4274,9 +4643,6 @@ static int ironlake_setup_rc6(struct drm_device *dev) static void ironlake_enable_rc6(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; - struct intel_engine_cs *ring = &dev_priv->ring[RCS]; - bool was_interruptible; - int ret; /* rc6 disabled by default due to repeated reports of hanging during * boot and resume. @@ -4286,46 +4652,8 @@ static void ironlake_enable_rc6(struct drm_device *dev) WARN_ON(!mutex_is_locked(&dev->struct_mutex)); - ret = ironlake_setup_rc6(dev); - if (ret) - return; - - was_interruptible = dev_priv->mm.interruptible; - dev_priv->mm.interruptible = false; - - /* - * GPU can automatically power down the render unit if given a page - * to save state. - */ - ret = intel_ring_begin(ring, 6); - if (ret) { - ironlake_teardown_rc6(dev); - dev_priv->mm.interruptible = was_interruptible; - return; - } - - intel_ring_emit(ring, MI_SUSPEND_FLUSH | MI_SUSPEND_FLUSH_EN); - intel_ring_emit(ring, MI_SET_CONTEXT); - intel_ring_emit(ring, i915_gem_obj_ggtt_offset(dev_priv->ips.renderctx) | - MI_MM_SPACE_GTT | - MI_SAVE_EXT_STATE_EN | - MI_RESTORE_EXT_STATE_EN | - MI_RESTORE_INHIBIT); - intel_ring_emit(ring, MI_SUSPEND_FLUSH); - intel_ring_emit(ring, MI_NOOP); - intel_ring_emit(ring, MI_FLUSH); - intel_ring_advance(ring); - - /* - * Wait for the command parser to advance past MI_SET_CONTEXT. The HW - * does an implicit flush, combined with MI_FLUSH above, it should be - * safe to assume that renderctx is valid - */ - ret = intel_ring_idle(ring); - dev_priv->mm.interruptible = was_interruptible; - if (ret) { + if (ironlake_setup_rc6(dev)) { DRM_ERROR("failed to enable ironlake power savings\n"); - ironlake_teardown_rc6(dev); return; } @@ -4784,7 +5112,7 @@ EXPORT_SYMBOL_GPL(i915_gpu_lower); bool i915_gpu_busy(void) { struct drm_i915_private *dev_priv; - struct intel_engine_cs *ring; + struct intel_engine_cs *engine; bool ret = false; int i; @@ -4793,8 +5121,8 @@ bool i915_gpu_busy(void) goto out_unlock; dev_priv = i915_mch_dev; - for_each_ring(ring, dev_priv, i) - ret |= !list_empty(&ring->request_list); + for_each_engine(engine, dev_priv, i) + ret |= engine->last_request != NULL; out_unlock: spin_unlock_irq(&mchdev_lock); @@ -5015,6 +5343,8 @@ static void intel_gen6_powersave_work(struct work_struct *work) rps.delayed_resume_work.work); struct drm_device *dev = dev_priv->dev; + dev_priv->rps.is_bdw_sw_turbo = false; + mutex_lock(&dev_priv->rps.hw_lock); if (IS_CHERRYVIEW(dev)) { @@ -5088,7 +5418,7 @@ static void g4x_disable_trickle_feed(struct drm_device *dev) struct drm_i915_private *dev_priv = dev->dev_private; int pipe; - for_each_pipe(pipe) { + for_each_pipe(dev_priv, pipe) { I915_WRITE(DSPCNTR(pipe), I915_READ(DSPCNTR(pipe)) | DISPPLANE_TRICKLE_FEED_DISABLE); @@ -5203,7 +5533,7 @@ static void cpt_init_clock_gating(struct drm_device *dev) /* The below fixes the weird display corruption, a few pixels shifted * downward, on (only) LVDS of some HP laptops with IVY. */ - for_each_pipe(pipe) { + for_each_pipe(dev_priv, pipe) { val = I915_READ(TRANS_CHICKEN2(pipe)); val |= TRANS_CHICKEN2_TIMING_OVERRIDE; val &= ~TRANS_CHICKEN2_FDI_POLARITY_REVERSED; @@ -5215,7 +5545,7 @@ static void cpt_init_clock_gating(struct drm_device *dev) I915_WRITE(TRANS_CHICKEN2(pipe), val); } /* WADP0ClockGatingDisable */ - for_each_pipe(pipe) { + for_each_pipe(dev_priv, pipe) { I915_WRITE(TRANS_CHICKEN1(pipe), TRANS_CHICKEN1_DP0UNIT_GC_DISABLE); } @@ -5383,7 +5713,7 @@ static void lpt_suspend_hw(struct drm_device *dev) } } -static void gen8_init_clock_gating(struct drm_device *dev) +static void broadwell_init_clock_gating(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; enum pipe pipe; @@ -5395,37 +5725,12 @@ static void gen8_init_clock_gating(struct drm_device *dev) /* FIXME(BDW): Check all the w/a, some might only apply to * pre-production hw. */ - /* WaDisablePartialInstShootdown:bdw */ - I915_WRITE(GEN8_ROW_CHICKEN, - _MASKED_BIT_ENABLE(PARTIAL_INSTRUCTION_SHOOTDOWN_DISABLE)); - /* WaDisableThreadStallDopClockGating:bdw */ - /* FIXME: Unclear whether we really need this on production bdw. */ - I915_WRITE(GEN8_ROW_CHICKEN, - _MASKED_BIT_ENABLE(STALL_DOP_GATING_DISABLE)); - - /* - * This GEN8_CENTROID_PIXEL_OPT_DIS W/A is only needed for - * pre-production hardware - */ - I915_WRITE(HALF_SLICE_CHICKEN3, - _MASKED_BIT_ENABLE(GEN8_CENTROID_PIXEL_OPT_DIS)); - I915_WRITE(HALF_SLICE_CHICKEN3, - _MASKED_BIT_ENABLE(GEN8_SAMPLER_POWER_BYPASS_DIS)); I915_WRITE(GAMTARBMODE, _MASKED_BIT_ENABLE(ARB_MODE_BWGTLB_DISABLE)); I915_WRITE(_3D_CHICKEN3, _MASKED_BIT_ENABLE(_3D_CHICKEN_SDE_LIMIT_FIFO_POLY_DEPTH(2))); - I915_WRITE(COMMON_SLICE_CHICKEN2, - _MASKED_BIT_ENABLE(GEN8_CSC2_SBE_VUE_CACHE_CONSERVATIVE)); - - I915_WRITE(GEN7_HALF_SLICE_CHICKEN1, - _MASKED_BIT_ENABLE(GEN7_SINGLE_SUBSCAN_DISPATCH_ENABLE)); - - /* WaDisableDopClockGating:bdw May not be needed for production */ - I915_WRITE(GEN7_ROW_CHICKEN2, - _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE)); /* WaSwitchSolVfFArbitrationPriority:bdw */ I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) | HSW_ECOCHK_ARB_PRIO_SOL); @@ -5435,37 +5740,18 @@ static void gen8_init_clock_gating(struct drm_device *dev) I915_READ(CHICKEN_PAR1_1) | DPA_MASK_VBLANK_SRD); /* WaPsrDPRSUnmaskVBlankInSRD:bdw */ - for_each_pipe(pipe) { + for_each_pipe(dev_priv, pipe) { I915_WRITE(CHICKEN_PIPESL_1(pipe), I915_READ(CHICKEN_PIPESL_1(pipe)) | BDW_DPRS_MASK_VBLANK_SRD); } - /* Use Force Non-Coherent whenever executing a 3D context. This is a - * workaround for for a possible hang in the unlikely event a TLB - * invalidation occurs during a PSD flush. - */ - I915_WRITE(HDC_CHICKEN0, - I915_READ(HDC_CHICKEN0) | - _MASKED_BIT_ENABLE(HDC_FORCE_NON_COHERENT)); - /* WaVSRefCountFullforceMissDisable:bdw */ /* WaDSRefCountFullforceMissDisable:bdw */ I915_WRITE(GEN7_FF_THREAD_MODE, I915_READ(GEN7_FF_THREAD_MODE) & ~(GEN8_FF_DS_REF_CNT_FFME | GEN7_FF_VS_REF_CNT_FFME)); - /* - * BSpec recommends 8x4 when MSAA is used, - * however in practice 16x4 seems fastest. - * - * Note that PS/WM thread counts depend on the WIZ hashing - * disable bit, which we don't touch here, but it's good - * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM). - */ - I915_WRITE(GEN7_GT_MODE, - GEN6_WIZ_HASHING_MASK | GEN6_WIZ_HASHING_16x4); - I915_WRITE(GEN6_RC_SLEEP_PSMI_CONTROL, _MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE)); @@ -5473,9 +5759,7 @@ static void gen8_init_clock_gating(struct drm_device *dev) I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) | GEN8_SDEUNIT_CLOCK_GATE_DISABLE); - /* Wa4x4STCOptimizationDisable:bdw */ - I915_WRITE(CACHE_MODE_1, - _MASKED_BIT_ENABLE(GEN8_4x4_STC_OPTIMIZATION_DISABLE)); + lpt_init_clock_gating(dev); } static void haswell_init_clock_gating(struct drm_device *dev) @@ -5631,24 +5915,6 @@ static void ivybridge_init_clock_gating(struct drm_device *dev) static void valleyview_init_clock_gating(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; - u32 val; - - mutex_lock(&dev_priv->rps.hw_lock); - val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS); - mutex_unlock(&dev_priv->rps.hw_lock); - switch ((val >> 6) & 3) { - case 0: - case 1: - dev_priv->mem_freq = 800; - break; - case 2: - dev_priv->mem_freq = 1066; - break; - case 3: - dev_priv->mem_freq = 1333; - break; - } - DRM_DEBUG_DRIVER("DDR speed: %d MHz", dev_priv->mem_freq); I915_WRITE(DSPCLK_GATE_D, VRHUNIT_CLOCK_GATE_DISABLE); @@ -5724,48 +5990,11 @@ static void valleyview_init_clock_gating(struct drm_device *dev) static void cherryview_init_clock_gating(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; - u32 val; - - mutex_lock(&dev_priv->rps.hw_lock); - val = vlv_punit_read(dev_priv, CCK_FUSE_REG); - mutex_unlock(&dev_priv->rps.hw_lock); - switch ((val >> 2) & 0x7) { - case 0: - case 1: - dev_priv->rps.cz_freq = CHV_CZ_CLOCK_FREQ_MODE_200; - dev_priv->mem_freq = 1600; - break; - case 2: - dev_priv->rps.cz_freq = CHV_CZ_CLOCK_FREQ_MODE_267; - dev_priv->mem_freq = 1600; - break; - case 3: - dev_priv->rps.cz_freq = CHV_CZ_CLOCK_FREQ_MODE_333; - dev_priv->mem_freq = 2000; - break; - case 4: - dev_priv->rps.cz_freq = CHV_CZ_CLOCK_FREQ_MODE_320; - dev_priv->mem_freq = 1600; - break; - case 5: - dev_priv->rps.cz_freq = CHV_CZ_CLOCK_FREQ_MODE_400; - dev_priv->mem_freq = 1600; - break; - } - DRM_DEBUG_DRIVER("DDR speed: %d MHz", dev_priv->mem_freq); I915_WRITE(DSPCLK_GATE_D, VRHUNIT_CLOCK_GATE_DISABLE); I915_WRITE(MI_ARB_VLV, MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE); - /* WaDisablePartialInstShootdown:chv */ - I915_WRITE(GEN8_ROW_CHICKEN, - _MASKED_BIT_ENABLE(PARTIAL_INSTRUCTION_SHOOTDOWN_DISABLE)); - - /* WaDisableThreadStallDopClockGating:chv */ - I915_WRITE(GEN8_ROW_CHICKEN, - _MASKED_BIT_ENABLE(STALL_DOP_GATING_DISABLE)); - /* WaVSRefCountFullforceMissDisable:chv */ /* WaDSRefCountFullforceMissDisable:chv */ I915_WRITE(GEN7_FF_THREAD_MODE, @@ -5784,10 +6013,6 @@ static void cherryview_init_clock_gating(struct drm_device *dev) I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) | GEN8_SDEUNIT_CLOCK_GATE_DISABLE); - /* WaDisableSamplerPowerBypass:chv (pre-production hw) */ - I915_WRITE(HALF_SLICE_CHICKEN3, - _MASKED_BIT_ENABLE(GEN8_SAMPLER_POWER_BYPASS_DIS)); - /* WaDisableGunitClockGating:chv (pre-production hw) */ I915_WRITE(VLV_GUNIT_CLOCK_GATE, I915_READ(VLV_GUNIT_CLOCK_GATE) | GINT_DIS); @@ -5797,8 +6022,6 @@ static void cherryview_init_clock_gating(struct drm_device *dev) _MASKED_BIT_ENABLE(GEN8_FF_DOP_CLOCK_GATE_DISABLE)); /* WaDisableDopClockGating:chv (pre-production hw) */ - I915_WRITE(GEN7_ROW_CHICKEN2, - _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE)); I915_WRITE(GEN6_UCGCTL1, I915_READ(GEN6_UCGCTL1) | GEN6_EU_TCUNIT_CLOCK_GATE_DISABLE); } @@ -5883,6 +6106,9 @@ static void gen3_init_clock_gating(struct drm_device *dev) /* On GEN3 we really need to make sure the ARB C3 LP bit is set */ I915_WRITE(MI_ARB_STATE, _MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE)); + + I915_WRITE(MI_ARB_STATE, + _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE)); } static void i85x_init_clock_gating(struct drm_device *dev) @@ -5894,6 +6120,9 @@ static void i85x_init_clock_gating(struct drm_device *dev) /* interrupts should cause a wake up from C3 */ I915_WRITE(MI_STATE, _MASKED_BIT_ENABLE(MI_AGPBUSY_INT_EN) | _MASKED_BIT_DISABLE(MI_AGPBUSY_830_MODE)); + + I915_WRITE(MEM_MODE, + _MASKED_BIT_ENABLE(MEM_DISPLAY_TRICKLE_FEED_DISABLE)); } static void i830_init_clock_gating(struct drm_device *dev) @@ -5901,6 +6130,10 @@ static void i830_init_clock_gating(struct drm_device *dev) struct drm_i915_private *dev_priv = dev->dev_private; I915_WRITE(DSPCLK_GATE_D, OVRUNIT_CLOCK_GATE_DISABLE); + + I915_WRITE(MEM_MODE, + _MASKED_BIT_ENABLE(MEM_DISPLAY_A_TRICKLE_FEED_DISABLE) | + _MASKED_BIT_ENABLE(MEM_DISPLAY_B_TRICKLE_FEED_DISABLE)); } void intel_init_clock_gating(struct drm_device *dev) @@ -6203,6 +6436,8 @@ static void vlv_display_power_well_disable(struct drm_i915_private *dev_priv, spin_unlock_irq(&dev_priv->irq_lock); vlv_set_power_well(dev_priv, power_well, false); + + vlv_power_sequencer_reset(dev_priv); } static void vlv_dpio_cmn_power_well_enable(struct drm_i915_private *dev_priv, @@ -6238,12 +6473,11 @@ static void vlv_dpio_cmn_power_well_enable(struct drm_i915_private *dev_priv, static void vlv_dpio_cmn_power_well_disable(struct drm_i915_private *dev_priv, struct i915_power_well *power_well) { - struct drm_device *dev = dev_priv->dev; enum pipe pipe; WARN_ON_ONCE(power_well->data != PUNIT_POWER_WELL_DPIO_CMN_BC); - for_each_pipe(pipe) + for_each_pipe(dev_priv, pipe) assert_pll_disabled(dev_priv, pipe); /* Assert common reset */ @@ -6252,6 +6486,153 @@ static void vlv_dpio_cmn_power_well_disable(struct drm_i915_private *dev_priv, vlv_set_power_well(dev_priv, power_well, false); } +static void chv_dpio_cmn_power_well_enable(struct drm_i915_private *dev_priv, + struct i915_power_well *power_well) +{ + enum dpio_phy phy; + + WARN_ON_ONCE(power_well->data != PUNIT_POWER_WELL_DPIO_CMN_BC && + power_well->data != PUNIT_POWER_WELL_DPIO_CMN_D); + + /* + * Enable the CRI clock source so we can get at the + * display and the reference clock for VGA + * hotplug / manual detection. + */ + if (power_well->data == PUNIT_POWER_WELL_DPIO_CMN_BC) { + phy = DPIO_PHY0; + I915_WRITE(DPLL(PIPE_B), I915_READ(DPLL(PIPE_B)) | + DPLL_REFA_CLK_ENABLE_VLV); + I915_WRITE(DPLL(PIPE_B), I915_READ(DPLL(PIPE_B)) | + DPLL_REFA_CLK_ENABLE_VLV | DPLL_INTEGRATED_CRI_CLK_VLV); + } else { + phy = DPIO_PHY1; + I915_WRITE(DPLL(PIPE_C), I915_READ(DPLL(PIPE_C)) | + DPLL_REFA_CLK_ENABLE_VLV | DPLL_INTEGRATED_CRI_CLK_VLV); + } + udelay(1); /* >10ns for cmnreset, >0ns for sidereset */ + vlv_set_power_well(dev_priv, power_well, true); + + /* Poll for phypwrgood signal */ + if (wait_for(I915_READ(DISPLAY_PHY_STATUS) & PHY_POWERGOOD(phy), 1)) + DRM_ERROR("Display PHY %d is not power up\n", phy); + + I915_WRITE(DISPLAY_PHY_CONTROL, I915_READ(DISPLAY_PHY_CONTROL) | + PHY_COM_LANE_RESET_DEASSERT(phy)); +} + +static void chv_dpio_cmn_power_well_disable(struct drm_i915_private *dev_priv, + struct i915_power_well *power_well) +{ + enum dpio_phy phy; + + WARN_ON_ONCE(power_well->data != PUNIT_POWER_WELL_DPIO_CMN_BC && + power_well->data != PUNIT_POWER_WELL_DPIO_CMN_D); + + if (power_well->data == PUNIT_POWER_WELL_DPIO_CMN_BC) { + phy = DPIO_PHY0; + assert_pll_disabled(dev_priv, PIPE_A); + assert_pll_disabled(dev_priv, PIPE_B); + } else { + phy = DPIO_PHY1; + assert_pll_disabled(dev_priv, PIPE_C); + } + + I915_WRITE(DISPLAY_PHY_CONTROL, I915_READ(DISPLAY_PHY_CONTROL) & + ~PHY_COM_LANE_RESET_DEASSERT(phy)); + + vlv_set_power_well(dev_priv, power_well, false); +} + +static bool chv_pipe_power_well_enabled(struct drm_i915_private *dev_priv, + struct i915_power_well *power_well) +{ + enum pipe pipe = power_well->data; + bool enabled; + u32 state, ctrl; + + mutex_lock(&dev_priv->rps.hw_lock); + + state = vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ) & DP_SSS_MASK(pipe); + /* + * We only ever set the power-on and power-gate states, anything + * else is unexpected. + */ + WARN_ON(state != DP_SSS_PWR_ON(pipe) && state != DP_SSS_PWR_GATE(pipe)); + enabled = state == DP_SSS_PWR_ON(pipe); + + /* + * A transient state at this point would mean some unexpected party + * is poking at the power controls too. + */ + ctrl = vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ) & DP_SSC_MASK(pipe); + WARN_ON(ctrl << 16 != state); + + mutex_unlock(&dev_priv->rps.hw_lock); + + return enabled; +} + +static void chv_set_pipe_power_well(struct drm_i915_private *dev_priv, + struct i915_power_well *power_well, + bool enable) +{ + enum pipe pipe = power_well->data; + u32 state; + u32 ctrl; + + state = enable ? DP_SSS_PWR_ON(pipe) : DP_SSS_PWR_GATE(pipe); + + mutex_lock(&dev_priv->rps.hw_lock); + +#define COND \ + ((vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ) & DP_SSS_MASK(pipe)) == state) + + if (COND) + goto out; + + ctrl = vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ); + ctrl &= ~DP_SSC_MASK(pipe); + ctrl |= enable ? DP_SSC_PWR_ON(pipe) : DP_SSC_PWR_GATE(pipe); + vlv_punit_write(dev_priv, PUNIT_REG_DSPFREQ, ctrl); + + if (wait_for(COND, 100)) + DRM_ERROR("timout setting power well state %08x (%08x)\n", + state, + vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ)); + +#undef COND + +out: + mutex_unlock(&dev_priv->rps.hw_lock); +} + +static void chv_pipe_power_well_sync_hw(struct drm_i915_private *dev_priv, + struct i915_power_well *power_well) +{ + chv_set_pipe_power_well(dev_priv, power_well, power_well->count > 0); +} + +static void chv_pipe_power_well_enable(struct drm_i915_private *dev_priv, + struct i915_power_well *power_well) +{ + WARN_ON_ONCE(power_well->data != PIPE_A && + power_well->data != PIPE_B && + power_well->data != PIPE_C); + + chv_set_pipe_power_well(dev_priv, power_well, true); +} + +static void chv_pipe_power_well_disable(struct drm_i915_private *dev_priv, + struct i915_power_well *power_well) +{ + WARN_ON_ONCE(power_well->data != PIPE_A && + power_well->data != PIPE_B && + power_well->data != PIPE_C); + + chv_set_pipe_power_well(dev_priv, power_well, false); +} + static void check_power_well_state(struct drm_i915_private *dev_priv, struct i915_power_well *power_well) { @@ -6443,6 +6824,39 @@ EXPORT_SYMBOL_GPL(i915_get_cdclk_freq); BIT(POWER_DOMAIN_PORT_DDI_C_4_LANES) | \ BIT(POWER_DOMAIN_INIT)) +#define CHV_PIPE_A_POWER_DOMAINS ( \ + BIT(POWER_DOMAIN_PIPE_A) | \ + BIT(POWER_DOMAIN_INIT)) + +#define CHV_PIPE_B_POWER_DOMAINS ( \ + BIT(POWER_DOMAIN_PIPE_B) | \ + BIT(POWER_DOMAIN_INIT)) + +#define CHV_PIPE_C_POWER_DOMAINS ( \ + BIT(POWER_DOMAIN_PIPE_C) | \ + BIT(POWER_DOMAIN_INIT)) + +#define CHV_DPIO_CMN_BC_POWER_DOMAINS ( \ + BIT(POWER_DOMAIN_PORT_DDI_B_2_LANES) | \ + BIT(POWER_DOMAIN_PORT_DDI_B_4_LANES) | \ + BIT(POWER_DOMAIN_PORT_DDI_C_2_LANES) | \ + BIT(POWER_DOMAIN_PORT_DDI_C_4_LANES) | \ + BIT(POWER_DOMAIN_INIT)) + +#define CHV_DPIO_CMN_D_POWER_DOMAINS ( \ + BIT(POWER_DOMAIN_PORT_DDI_D_2_LANES) | \ + BIT(POWER_DOMAIN_PORT_DDI_D_4_LANES) | \ + BIT(POWER_DOMAIN_INIT)) + +#define CHV_DPIO_TX_D_LANES_01_POWER_DOMAINS ( \ + BIT(POWER_DOMAIN_PORT_DDI_D_2_LANES) | \ + BIT(POWER_DOMAIN_PORT_DDI_D_4_LANES) | \ + BIT(POWER_DOMAIN_INIT)) + +#define CHV_DPIO_TX_D_LANES_23_POWER_DOMAINS ( \ + BIT(POWER_DOMAIN_PORT_DDI_D_4_LANES) | \ + BIT(POWER_DOMAIN_INIT)) + static const struct i915_power_well_ops i9xx_always_on_power_well_ops = { .sync_hw = i9xx_always_on_power_well_noop, .enable = i9xx_always_on_power_well_noop, @@ -6450,6 +6864,20 @@ static const struct i915_power_well_ops i9xx_always_on_power_well_ops = { .is_enabled = i9xx_always_on_power_well_enabled, }; +static const struct i915_power_well_ops chv_pipe_power_well_ops = { + .sync_hw = chv_pipe_power_well_sync_hw, + .enable = chv_pipe_power_well_enable, + .disable = chv_pipe_power_well_disable, + .is_enabled = chv_pipe_power_well_enabled, +}; + +static const struct i915_power_well_ops chv_dpio_cmn_power_well_ops = { + .sync_hw = vlv_power_well_sync_hw, + .enable = chv_dpio_cmn_power_well_enable, + .disable = chv_dpio_cmn_power_well_disable, + .is_enabled = vlv_power_well_enabled, +}; + static struct i915_power_well i9xx_always_on_power_well[] = { { .name = "always-on", @@ -6572,6 +7000,107 @@ static struct i915_power_well vlv_power_wells[] = { }, }; +static struct i915_power_well chv_power_wells[] = { + { + .name = "always-on", + .always_on = 1, + .domains = VLV_ALWAYS_ON_POWER_DOMAINS, + .ops = &i9xx_always_on_power_well_ops, + }, +#if 0 + { + .name = "display", + .domains = VLV_DISPLAY_POWER_DOMAINS, + .data = PUNIT_POWER_WELL_DISP2D, + .ops = &vlv_display_power_well_ops, + }, + { + .name = "pipe-a", + .domains = CHV_PIPE_A_POWER_DOMAINS, + .data = PIPE_A, + .ops = &chv_pipe_power_well_ops, + }, + { + .name = "pipe-b", + .domains = CHV_PIPE_B_POWER_DOMAINS, + .data = PIPE_B, + .ops = &chv_pipe_power_well_ops, + }, + { + .name = "pipe-c", + .domains = CHV_PIPE_C_POWER_DOMAINS, + .data = PIPE_C, + .ops = &chv_pipe_power_well_ops, + }, +#endif + { + .name = "dpio-common-bc", + /* + * XXX: cmnreset for one PHY seems to disturb the other. + * As a workaround keep both powered on at the same + * time for now. + */ + .domains = CHV_DPIO_CMN_BC_POWER_DOMAINS | CHV_DPIO_CMN_D_POWER_DOMAINS, + .data = PUNIT_POWER_WELL_DPIO_CMN_BC, + .ops = &chv_dpio_cmn_power_well_ops, + }, + { + .name = "dpio-common-d", + /* + * XXX: cmnreset for one PHY seems to disturb the other. + * As a workaround keep both powered on at the same + * time for now. + */ + .domains = CHV_DPIO_CMN_BC_POWER_DOMAINS | CHV_DPIO_CMN_D_POWER_DOMAINS, + .data = PUNIT_POWER_WELL_DPIO_CMN_D, + .ops = &chv_dpio_cmn_power_well_ops, + }, +#if 0 + { + .name = "dpio-tx-b-01", + .domains = VLV_DPIO_TX_B_LANES_01_POWER_DOMAINS | + VLV_DPIO_TX_B_LANES_23_POWER_DOMAINS, + .ops = &vlv_dpio_power_well_ops, + .data = PUNIT_POWER_WELL_DPIO_TX_B_LANES_01, + }, + { + .name = "dpio-tx-b-23", + .domains = VLV_DPIO_TX_B_LANES_01_POWER_DOMAINS | + VLV_DPIO_TX_B_LANES_23_POWER_DOMAINS, + .ops = &vlv_dpio_power_well_ops, + .data = PUNIT_POWER_WELL_DPIO_TX_B_LANES_23, + }, + { + .name = "dpio-tx-c-01", + .domains = VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS | + VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS, + .ops = &vlv_dpio_power_well_ops, + .data = PUNIT_POWER_WELL_DPIO_TX_C_LANES_01, + }, + { + .name = "dpio-tx-c-23", + .domains = VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS | + VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS, + .ops = &vlv_dpio_power_well_ops, + .data = PUNIT_POWER_WELL_DPIO_TX_C_LANES_23, + }, + { + .name = "dpio-tx-d-01", + .domains = CHV_DPIO_TX_D_LANES_01_POWER_DOMAINS | + CHV_DPIO_TX_D_LANES_23_POWER_DOMAINS, + .ops = &vlv_dpio_power_well_ops, + .data = PUNIT_POWER_WELL_DPIO_TX_D_LANES_01, + }, + { + .name = "dpio-tx-d-23", + .domains = CHV_DPIO_TX_D_LANES_01_POWER_DOMAINS | + CHV_DPIO_TX_D_LANES_23_POWER_DOMAINS, + .ops = &vlv_dpio_power_well_ops, + .data = PUNIT_POWER_WELL_DPIO_TX_D_LANES_23, + }, +#endif +}; + static struct i915_power_well *lookup_power_well(struct drm_i915_private *dev_priv, enum punit_power_well power_well_id) { @@ -6608,6 +7137,8 @@ int intel_power_domains_init(struct drm_i915_private *dev_priv) } else if (IS_BROADWELL(dev_priv->dev)) { set_power_wells(power_domains, bdw_power_wells); hsw_pwr = power_domains; + } else if (IS_CHERRYVIEW(dev_priv->dev)) { + set_power_wells(power_domains, chv_power_wells); } else if (IS_VALLEYVIEW(dev_priv->dev)) { set_power_wells(power_domains, vlv_power_wells); } else { @@ -6833,13 +7364,15 @@ void intel_init_pm(struct drm_device *dev) else if (IS_HASWELL(dev)) dev_priv->display.init_clock_gating = haswell_init_clock_gating; else if (INTEL_INFO(dev)->gen == 8) - dev_priv->display.init_clock_gating = gen8_init_clock_gating; + dev_priv->display.init_clock_gating = broadwell_init_clock_gating; } else if (IS_CHERRYVIEW(dev)) { - dev_priv->display.update_wm = valleyview_update_wm; + dev_priv->display.update_wm = cherryview_update_wm; + dev_priv->display.update_sprite_wm = valleyview_update_sprite_wm; dev_priv->display.init_clock_gating = cherryview_init_clock_gating; } else if (IS_VALLEYVIEW(dev)) { dev_priv->display.update_wm = valleyview_update_wm; + dev_priv->display.update_sprite_wm = valleyview_update_sprite_wm; dev_priv->display.init_clock_gating = valleyview_init_clock_gating; } else if (IS_PINEVIEW(dev)) { @@ -7025,6 +7558,7 @@ static int chv_freq_opcode(struct drm_i915_private *dev_priv, int val) return -1; } + /* CHV needs even values */ opcode = (DIV_ROUND_CLOSEST((val * 2 * mul), dev_priv->rps.cz_freq) * 2); return opcode; diff --git a/drivers/gpu/drm/i915/intel_ringbuffer.c b/drivers/gpu/drm/i915/intel_ringbuffer.c index 47a126a..24e7ec7 100644 --- a/drivers/gpu/drm/i915/intel_ringbuffer.c +++ b/drivers/gpu/drm/i915/intel_ringbuffer.c @@ -33,75 +33,39 @@ #include "i915_trace.h" #include "intel_drv.h" -/* Early gen2 devices have a cacheline of just 32 bytes, using 64 is overkill, - * but keeps the logic simple. Indeed, the whole purpose of this macro is just - * to give some inclination as to some of the magic values used in the various - * workarounds! - */ -#define CACHELINE_BYTES 64 - -static inline int __ring_space(int head, int tail, int size) -{ - int space = head - (tail + I915_RING_FREE_SPACE); - if (space < 0) - space += size; - return space; -} - -static inline int ring_space(struct intel_ringbuffer *ringbuf) -{ - return __ring_space(ringbuf->head & HEAD_ADDR, ringbuf->tail, ringbuf->size); -} - -static bool intel_ring_stopped(struct intel_engine_cs *ring) -{ - struct drm_i915_private *dev_priv = ring->dev->dev_private; - return dev_priv->gpu_error.stop_rings & intel_ring_flag(ring); -} - -void __intel_ring_advance(struct intel_engine_cs *ring) -{ - struct intel_ringbuffer *ringbuf = ring->buffer; - ringbuf->tail &= ringbuf->size - 1; - if (intel_ring_stopped(ring)) - return; - ring->write_tail(ring, ringbuf->tail); -} +/* Just userspace ABI convention to limit the wa batch bo to a resonable size */ +#define I830_BATCH_LIMIT (256*1024) +#define I830_TLB_ENTRIES (2) +#define I830_WA_SIZE max(I830_TLB_ENTRIES*4096, I830_BATCH_LIMIT) static int -gen2_render_ring_flush(struct intel_engine_cs *ring, - u32 invalidate_domains, - u32 flush_domains) +gen2_emit_flush(struct i915_gem_request *rq, u32 flags) { + struct intel_ringbuffer *ring; u32 cmd; - int ret; cmd = MI_FLUSH; - if (((invalidate_domains|flush_domains) & I915_GEM_DOMAIN_RENDER) == 0) + if ((flags & (I915_FLUSH_CACHES | I915_INVALIDATE_CACHES)) == 0) cmd |= MI_NO_WRITE_FLUSH; - if (invalidate_domains & I915_GEM_DOMAIN_SAMPLER) + if (flags & I915_INVALIDATE_CACHES) cmd |= MI_READ_FLUSH; - ret = intel_ring_begin(ring, 2); - if (ret) - return ret; + ring = intel_ring_begin(rq, 1); + if (IS_ERR(ring)) + return PTR_ERR(ring); intel_ring_emit(ring, cmd); - intel_ring_emit(ring, MI_NOOP); intel_ring_advance(ring); return 0; } static int -gen4_render_ring_flush(struct intel_engine_cs *ring, - u32 invalidate_domains, - u32 flush_domains) +gen4_emit_flush(struct i915_gem_request *rq, u32 flags) { - struct drm_device *dev = ring->dev; + struct intel_ringbuffer *ring; u32 cmd; - int ret; /* * read/write caches: @@ -131,22 +95,20 @@ gen4_render_ring_flush(struct intel_engine_cs *ring, * are flushed at any MI_FLUSH. */ - cmd = MI_FLUSH | MI_NO_WRITE_FLUSH; - if ((invalidate_domains|flush_domains) & I915_GEM_DOMAIN_RENDER) - cmd &= ~MI_NO_WRITE_FLUSH; - if (invalidate_domains & I915_GEM_DOMAIN_INSTRUCTION) + cmd = MI_FLUSH; + if ((flags & (I915_FLUSH_CACHES | I915_INVALIDATE_CACHES)) == 0) + cmd |= MI_NO_WRITE_FLUSH; + if (flags & I915_INVALIDATE_CACHES) { cmd |= MI_EXE_FLUSH; + if (IS_G4X(rq->i915) || IS_GEN5(rq->i915)) + cmd |= MI_INVALIDATE_ISP; + } - if (invalidate_domains & I915_GEM_DOMAIN_COMMAND && - (IS_G4X(dev) || IS_GEN5(dev))) - cmd |= MI_INVALIDATE_ISP; - - ret = intel_ring_begin(ring, 2); - if (ret) - return ret; + ring = intel_ring_begin(rq, 1); + if (IS_ERR(ring)) + return PTR_ERR(ring); intel_ring_emit(ring, cmd); - intel_ring_emit(ring, MI_NOOP); intel_ring_advance(ring); return 0; @@ -190,100 +152,89 @@ gen4_render_ring_flush(struct intel_engine_cs *ring, * really our business. That leaves only stall at scoreboard. */ static int -intel_emit_post_sync_nonzero_flush(struct intel_engine_cs *ring) +gen6_emit_post_sync_nonzero_flush(struct i915_gem_request *rq) { - u32 scratch_addr = ring->scratch.gtt_offset + 2 * CACHELINE_BYTES; - int ret; - + const u32 scratch = rq->engine->scratch.gtt_offset + 2*CACHELINE_BYTES; + struct intel_ringbuffer *ring; - ret = intel_ring_begin(ring, 6); - if (ret) - return ret; + ring = intel_ring_begin(rq, 8); + if (IS_ERR(ring)) + return PTR_ERR(ring); - intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(5)); + intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4)); intel_ring_emit(ring, PIPE_CONTROL_CS_STALL | PIPE_CONTROL_STALL_AT_SCOREBOARD); - intel_ring_emit(ring, scratch_addr | PIPE_CONTROL_GLOBAL_GTT); /* address */ - intel_ring_emit(ring, 0); /* low dword */ - intel_ring_emit(ring, 0); /* high dword */ - intel_ring_emit(ring, MI_NOOP); - intel_ring_advance(ring); - - ret = intel_ring_begin(ring, 6); - if (ret) - return ret; + intel_ring_emit(ring, scratch | PIPE_CONTROL_GLOBAL_GTT); + intel_ring_emit(ring, 0); - intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(5)); + intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4)); intel_ring_emit(ring, PIPE_CONTROL_QW_WRITE); - intel_ring_emit(ring, scratch_addr | PIPE_CONTROL_GLOBAL_GTT); /* address */ - intel_ring_emit(ring, 0); + intel_ring_emit(ring, scratch | PIPE_CONTROL_GLOBAL_GTT); intel_ring_emit(ring, 0); - intel_ring_emit(ring, MI_NOOP); - intel_ring_advance(ring); + intel_ring_advance(ring); return 0; } static int -gen6_render_ring_flush(struct intel_engine_cs *ring, - u32 invalidate_domains, u32 flush_domains) +gen6_render_emit_flush(struct i915_gem_request *rq, u32 flags) { - u32 flags = 0; - u32 scratch_addr = ring->scratch.gtt_offset + 2 * CACHELINE_BYTES; + const u32 scratch = rq->engine->scratch.gtt_offset + 2*CACHELINE_BYTES; + struct intel_ringbuffer *ring; + u32 cmd = 0; int ret; - /* Force SNB workarounds for PIPE_CONTROL flushes */ - ret = intel_emit_post_sync_nonzero_flush(ring); - if (ret) - return ret; + if (flags & I915_FLUSH_CACHES) { + /* Force SNB workarounds for PIPE_CONTROL flushes */ + ret = gen6_emit_post_sync_nonzero_flush(rq); + if (ret) + return ret; - /* Just flush everything. Experiments have shown that reducing the - * number of bits based on the write domains has little performance - * impact. - */ - if (flush_domains) { - flags |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH; - flags |= PIPE_CONTROL_DEPTH_CACHE_FLUSH; - /* - * Ensure that any following seqno writes only happen - * when the render cache is indeed flushed. - */ - flags |= PIPE_CONTROL_CS_STALL; - } - if (invalidate_domains) { - flags |= PIPE_CONTROL_TLB_INVALIDATE; - flags |= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE; - flags |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE; - flags |= PIPE_CONTROL_VF_CACHE_INVALIDATE; - flags |= PIPE_CONTROL_CONST_CACHE_INVALIDATE; - flags |= PIPE_CONTROL_STATE_CACHE_INVALIDATE; + cmd |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH; + cmd |= PIPE_CONTROL_DEPTH_CACHE_FLUSH; + } + if (flags & I915_INVALIDATE_CACHES) { + cmd |= PIPE_CONTROL_TLB_INVALIDATE; + cmd |= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE; + cmd |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE; + cmd |= PIPE_CONTROL_VF_CACHE_INVALIDATE; + cmd |= PIPE_CONTROL_CONST_CACHE_INVALIDATE; + cmd |= PIPE_CONTROL_STATE_CACHE_INVALIDATE; /* * TLB invalidate requires a post-sync write. */ - flags |= PIPE_CONTROL_QW_WRITE | PIPE_CONTROL_CS_STALL; + cmd |= PIPE_CONTROL_QW_WRITE | PIPE_CONTROL_CS_STALL; } + if (flags & I915_COMMAND_BARRIER) + /* + * Ensure that any following seqno writes only happen + * when the render cache is indeed flushed. + */ + cmd |= PIPE_CONTROL_CS_STALL; - ret = intel_ring_begin(ring, 4); - if (ret) - return ret; + if (cmd) { + ring = intel_ring_begin(rq, 4); + if (IS_ERR(ring)) + return PTR_ERR(ring); - intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4)); - intel_ring_emit(ring, flags); - intel_ring_emit(ring, scratch_addr | PIPE_CONTROL_GLOBAL_GTT); - intel_ring_emit(ring, 0); - intel_ring_advance(ring); + intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4)); + intel_ring_emit(ring, cmd); + intel_ring_emit(ring, scratch | PIPE_CONTROL_GLOBAL_GTT); + intel_ring_emit(ring, 0); + intel_ring_advance(ring); + } return 0; } static int -gen7_render_ring_cs_stall_wa(struct intel_engine_cs *ring) +gen7_render_ring_cs_stall_wa(struct i915_gem_request *rq) { - int ret; + struct intel_ringbuffer *ring; - ret = intel_ring_begin(ring, 4); - if (ret) - return ret; + ring = intel_ring_begin(rq, 4); + if (IS_ERR(ring)) + return PTR_ERR(ring); intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4)); intel_ring_emit(ring, PIPE_CONTROL_CS_STALL | @@ -295,35 +246,32 @@ gen7_render_ring_cs_stall_wa(struct intel_engine_cs *ring) return 0; } -static int gen7_ring_fbc_flush(struct intel_engine_cs *ring, u32 value) +static int gen7_ring_fbc_flush(struct i915_gem_request *rq, u32 value) { - int ret; + struct intel_ringbuffer *ring; - if (!ring->fbc_dirty) - return 0; + ring = intel_ring_begin(rq, 6); + if (IS_ERR(ring)) + return PTR_ERR(rq); - ret = intel_ring_begin(ring, 6); - if (ret) - return ret; /* WaFbcNukeOn3DBlt:ivb/hsw */ intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1)); intel_ring_emit(ring, MSG_FBC_REND_STATE); intel_ring_emit(ring, value); intel_ring_emit(ring, MI_STORE_REGISTER_MEM(1) | MI_SRM_LRM_GLOBAL_GTT); intel_ring_emit(ring, MSG_FBC_REND_STATE); - intel_ring_emit(ring, ring->scratch.gtt_offset + 256); + intel_ring_emit(ring, rq->engine->scratch.gtt_offset + 256); intel_ring_advance(ring); - ring->fbc_dirty = false; return 0; } static int -gen7_render_ring_flush(struct intel_engine_cs *ring, - u32 invalidate_domains, u32 flush_domains) +gen7_render_emit_flush(struct i915_gem_request *rq, u32 flags) { - u32 flags = 0; - u32 scratch_addr = ring->scratch.gtt_offset + 2 * CACHELINE_BYTES; + const u32 scratch_addr = rq->engine->scratch.gtt_offset + 2 * CACHELINE_BYTES; + struct intel_ringbuffer *ring; + u32 cmd = 0; int ret; /* @@ -334,63 +282,71 @@ gen7_render_ring_flush(struct intel_engine_cs *ring, * read-cache invalidate bits set) must have the CS_STALL bit set. We * don't try to be clever and just set it unconditionally. */ - flags |= PIPE_CONTROL_CS_STALL; + cmd |= PIPE_CONTROL_CS_STALL; /* Just flush everything. Experiments have shown that reducing the * number of bits based on the write domains has little performance * impact. */ - if (flush_domains) { - flags |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH; - flags |= PIPE_CONTROL_DEPTH_CACHE_FLUSH; - } - if (invalidate_domains) { - flags |= PIPE_CONTROL_TLB_INVALIDATE; - flags |= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE; - flags |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE; - flags |= PIPE_CONTROL_VF_CACHE_INVALIDATE; - flags |= PIPE_CONTROL_CONST_CACHE_INVALIDATE; - flags |= PIPE_CONTROL_STATE_CACHE_INVALIDATE; + if (flags & I915_FLUSH_CACHES) { + cmd |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH; + cmd |= PIPE_CONTROL_DEPTH_CACHE_FLUSH; + } + if (flags & I915_INVALIDATE_CACHES) { + cmd |= PIPE_CONTROL_TLB_INVALIDATE; + cmd |= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE; + cmd |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE; + cmd |= PIPE_CONTROL_VF_CACHE_INVALIDATE; + cmd |= PIPE_CONTROL_CONST_CACHE_INVALIDATE; + cmd |= PIPE_CONTROL_STATE_CACHE_INVALIDATE; /* * TLB invalidate requires a post-sync write. */ - flags |= PIPE_CONTROL_QW_WRITE; - flags |= PIPE_CONTROL_GLOBAL_GTT_IVB; + cmd |= PIPE_CONTROL_QW_WRITE; + cmd |= PIPE_CONTROL_GLOBAL_GTT_IVB; /* Workaround: we must issue a pipe_control with CS-stall bit * set before a pipe_control command that has the state cache * invalidate bit set. */ - gen7_render_ring_cs_stall_wa(ring); + ret = gen7_render_ring_cs_stall_wa(rq); + if (ret) + return ret; } - ret = intel_ring_begin(ring, 4); - if (ret) - return ret; + ring = intel_ring_begin(rq, 4); + if (IS_ERR(ring)) + return PTR_ERR(ring); intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4)); - intel_ring_emit(ring, flags); + intel_ring_emit(ring, cmd); intel_ring_emit(ring, scratch_addr); intel_ring_emit(ring, 0); intel_ring_advance(ring); - if (!invalidate_domains && flush_domains) - return gen7_ring_fbc_flush(ring, FBC_REND_NUKE); + if (flags & I915_KICK_FBC) { + ret = gen7_ring_fbc_flush(rq, FBC_REND_NUKE); + if (ret) + return ret; + } return 0; } static int -gen8_emit_pipe_control(struct intel_engine_cs *ring, - u32 flags, u32 scratch_addr) +gen8_emit_pipe_control(struct i915_gem_request *rq, + u32 cmd, u32 scratch_addr) { - int ret; + struct intel_ringbuffer *ring; - ret = intel_ring_begin(ring, 6); - if (ret) - return ret; + if (cmd == 0) + return 0; + + ring = intel_ring_begin(rq, 6); + if (IS_ERR(rq)) + return PTR_ERR(rq); intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(6)); - intel_ring_emit(ring, flags); + intel_ring_emit(ring, cmd); intel_ring_emit(ring, scratch_addr); intel_ring_emit(ring, 0); intel_ring_emit(ring, 0); @@ -401,31 +357,29 @@ gen8_emit_pipe_control(struct intel_engine_cs *ring, } static int -gen8_render_ring_flush(struct intel_engine_cs *ring, - u32 invalidate_domains, u32 flush_domains) +gen8_render_emit_flush(struct i915_gem_request *rq, + u32 flags) { - u32 flags = 0; - u32 scratch_addr = ring->scratch.gtt_offset + 2 * CACHELINE_BYTES; + const u32 scratch_addr = rq->engine->scratch.gtt_offset + 2 * CACHELINE_BYTES; + u32 cmd = 0; int ret; - flags |= PIPE_CONTROL_CS_STALL; - - if (flush_domains) { - flags |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH; - flags |= PIPE_CONTROL_DEPTH_CACHE_FLUSH; + if (flags & I915_FLUSH_CACHES) { + cmd |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH; + cmd |= PIPE_CONTROL_DEPTH_CACHE_FLUSH; } - if (invalidate_domains) { - flags |= PIPE_CONTROL_TLB_INVALIDATE; - flags |= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE; - flags |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE; - flags |= PIPE_CONTROL_VF_CACHE_INVALIDATE; - flags |= PIPE_CONTROL_CONST_CACHE_INVALIDATE; - flags |= PIPE_CONTROL_STATE_CACHE_INVALIDATE; - flags |= PIPE_CONTROL_QW_WRITE; - flags |= PIPE_CONTROL_GLOBAL_GTT_IVB; + if (flags & I915_INVALIDATE_CACHES) { + cmd |= PIPE_CONTROL_TLB_INVALIDATE; + cmd |= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE; + cmd |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE; + cmd |= PIPE_CONTROL_VF_CACHE_INVALIDATE; + cmd |= PIPE_CONTROL_CONST_CACHE_INVALIDATE; + cmd |= PIPE_CONTROL_STATE_CACHE_INVALIDATE; + cmd |= PIPE_CONTROL_QW_WRITE; + cmd |= PIPE_CONTROL_GLOBAL_GTT_IVB; /* WaCsStallBeforeStateCacheInvalidate:bdw,chv */ - ret = gen8_emit_pipe_control(ring, + ret = gen8_emit_pipe_control(rq, PIPE_CONTROL_CS_STALL | PIPE_CONTROL_STALL_AT_SCOREBOARD, 0); @@ -433,200 +387,481 @@ gen8_render_ring_flush(struct intel_engine_cs *ring, return ret; } - return gen8_emit_pipe_control(ring, flags, scratch_addr); + if (flags & I915_COMMAND_BARRIER) + cmd |= PIPE_CONTROL_CS_STALL; + + + ret = gen8_emit_pipe_control(rq, cmd, scratch_addr); + if (ret) + return ret; + + if (flags & I915_KICK_FBC) { + ret = gen7_ring_fbc_flush(rq, FBC_REND_NUKE); + if (ret) + return ret; + } + + return 0; } -static void ring_write_tail(struct intel_engine_cs *ring, +static void ring_write_tail(struct intel_engine_cs *engine, u32 value) { - struct drm_i915_private *dev_priv = ring->dev->dev_private; - I915_WRITE_TAIL(ring, value); + struct drm_i915_private *dev_priv = engine->i915; + I915_WRITE_TAIL(engine, value); } -u64 intel_ring_get_active_head(struct intel_engine_cs *ring) +u64 intel_engine_get_active_head(struct intel_engine_cs *engine) { - struct drm_i915_private *dev_priv = ring->dev->dev_private; + struct drm_i915_private *dev_priv = engine->i915; u64 acthd; - if (INTEL_INFO(ring->dev)->gen >= 8) - acthd = I915_READ64_2x32(RING_ACTHD(ring->mmio_base), - RING_ACTHD_UDW(ring->mmio_base)); - else if (INTEL_INFO(ring->dev)->gen >= 4) - acthd = I915_READ(RING_ACTHD(ring->mmio_base)); + if (INTEL_INFO(dev_priv)->gen >= 8) + acthd = I915_READ64_2x32(RING_ACTHD(engine->mmio_base), + RING_ACTHD_UDW(engine->mmio_base)); + else if (INTEL_INFO(dev_priv)->gen >= 4) + acthd = I915_READ(RING_ACTHD(engine->mmio_base)); else acthd = I915_READ(ACTHD); return acthd; } -static void ring_setup_phys_status_page(struct intel_engine_cs *ring) +static bool engine_stop(struct intel_engine_cs *engine) { - struct drm_i915_private *dev_priv = ring->dev->dev_private; - u32 addr; + struct drm_i915_private *dev_priv = engine->i915; + + if (!IS_GEN2(dev_priv)) { + I915_WRITE_MODE(engine, _MASKED_BIT_ENABLE(STOP_RING)); + if (wait_for((I915_READ_MODE(engine) & MODE_IDLE) != 0, 1000)) { + DRM_ERROR("%s : timed out trying to stop ring\n", engine->name); + /* Sometimes we observe that the idle flag is not + * set even though the ring is empty. So double + * check before giving up. + */ + if (I915_READ_HEAD(engine) != I915_READ_TAIL(engine)) + return false; + } + } - addr = dev_priv->status_page_dmah->busaddr; - if (INTEL_INFO(ring->dev)->gen >= 4) - addr |= (dev_priv->status_page_dmah->busaddr >> 28) & 0xf0; - I915_WRITE(HWS_PGA, addr); + I915_WRITE_CTL(engine, 0); + I915_WRITE_HEAD(engine, 0); + engine->write_tail(engine, 0); + + if (!IS_GEN2(dev_priv)) { + (void)I915_READ_CTL(engine); + I915_WRITE_MODE(engine, _MASKED_BIT_DISABLE(STOP_RING)); + } + + return (I915_READ_HEAD(engine) & HEAD_ADDR) == 0; +} + +static int engine_suspend(struct intel_engine_cs *engine) +{ + return engine_stop(engine) ? 0 : -EIO; } -static bool stop_ring(struct intel_engine_cs *ring) +static int enable_status_page(struct intel_engine_cs *engine) { - struct drm_i915_private *dev_priv = to_i915(ring->dev); + struct drm_i915_private *dev_priv = engine->i915; + u32 mmio, addr; + int ret = 0; - if (!IS_GEN2(ring->dev)) { - I915_WRITE_MODE(ring, _MASKED_BIT_ENABLE(STOP_RING)); - if (wait_for_atomic((I915_READ_MODE(ring) & MODE_IDLE) != 0, 1000)) { - DRM_ERROR("%s :timed out trying to stop ring\n", ring->name); - return false; + if (!I915_NEED_GFX_HWS(dev_priv)) { + addr = dev_priv->status_page_dmah->busaddr; + if (INTEL_INFO(dev_priv)->gen >= 4) + addr |= (dev_priv->status_page_dmah->busaddr >> 28) & 0xf0; + mmio = HWS_PGA; + } else { + addr = engine->status_page.gfx_addr; + /* The ring status page addresses are no longer next to the rest of + * the ring registers as of gen7. + */ + if (IS_GEN7(dev_priv)) { + switch (engine->id) { + default: + case RCS: + mmio = RENDER_HWS_PGA_GEN7; + break; + case BCS: + mmio = BLT_HWS_PGA_GEN7; + break; + /* + * VCS2 actually doesn't exist on Gen7. Only shut up + * gcc switch check warning + */ + case VCS2: + case VCS: + mmio = BSD_HWS_PGA_GEN7; + break; + case VECS: + mmio = VEBOX_HWS_PGA_GEN7; + break; + } + } else if (IS_GEN6(dev_priv)) { + mmio = RING_HWS_PGA_GEN6(engine->mmio_base); + } else { + /* XXX: gen8 returns to sanity */ + mmio = RING_HWS_PGA(engine->mmio_base); } } - I915_WRITE_CTL(ring, 0); - I915_WRITE_HEAD(ring, 0); - ring->write_tail(ring, 0); + I915_WRITE(mmio, addr); + POSTING_READ(mmio); + + /* + * Flush the TLB for this page + * + * FIXME: These two bits have disappeared on gen8, so a question + * arises: do we still need this and if so how should we go about + * invalidating the TLB? + */ + if (INTEL_INFO(dev_priv)->gen >= 6 && INTEL_INFO(dev_priv)->gen < 8) { + u32 reg = RING_INSTPM(engine->mmio_base); + + /* ring should be idle before issuing a sync flush*/ + WARN_ON((I915_READ_MODE(engine) & MODE_IDLE) == 0); - if (!IS_GEN2(ring->dev)) { - (void)I915_READ_CTL(ring); - I915_WRITE_MODE(ring, _MASKED_BIT_DISABLE(STOP_RING)); + I915_WRITE(reg, + _MASKED_BIT_ENABLE(INSTPM_TLB_INVALIDATE | + INSTPM_SYNC_FLUSH)); + if (wait_for((I915_READ(reg) & INSTPM_SYNC_FLUSH) == 0, + 1000)) { + DRM_ERROR("%s: wait for SyncFlush to complete for TLB invalidation timed out\n", + engine->name); + ret = -EIO; + } } - return (I915_READ_HEAD(ring) & HEAD_ADDR) == 0; + return ret; } -static int init_ring_common(struct intel_engine_cs *ring) +static struct intel_ringbuffer * +engine_get_ring(struct intel_engine_cs *engine, + struct intel_context *ctx) { - struct drm_device *dev = ring->dev; - struct drm_i915_private *dev_priv = dev->dev_private; - struct intel_ringbuffer *ringbuf = ring->buffer; - struct drm_i915_gem_object *obj = ringbuf->obj; + struct drm_i915_private *dev_priv = engine->i915; + struct intel_ringbuffer *ring; int ret = 0; - gen6_gt_force_wake_get(dev_priv, FORCEWAKE_ALL); + ring = engine->legacy_ring; + if (ring) + return ring; + + ring = intel_engine_alloc_ring(engine, ctx, 32 * PAGE_SIZE); + if (IS_ERR(ring)) { + DRM_ERROR("Failed to allocate ringbuffer for %s: %ld\n", engine->name, PTR_ERR(ring)); + return ERR_CAST(ring); + } - if (!stop_ring(ring)) { + gen6_gt_force_wake_get(dev_priv, FORCEWAKE_ALL); + if (!engine_stop(engine)) { /* G45 ring initialization often fails to reset head to zero */ DRM_DEBUG_KMS("%s head not reset to zero " "ctl %08x head %08x tail %08x start %08x\n", - ring->name, - I915_READ_CTL(ring), - I915_READ_HEAD(ring), - I915_READ_TAIL(ring), - I915_READ_START(ring)); - - if (!stop_ring(ring)) { + engine->name, + I915_READ_CTL(engine), + I915_READ_HEAD(engine), + I915_READ_TAIL(engine), + I915_READ_START(engine)); + if (!engine_stop(engine)) { DRM_ERROR("failed to set %s head to zero " "ctl %08x head %08x tail %08x start %08x\n", - ring->name, - I915_READ_CTL(ring), - I915_READ_HEAD(ring), - I915_READ_TAIL(ring), - I915_READ_START(ring)); + engine->name, + I915_READ_CTL(engine), + I915_READ_HEAD(engine), + I915_READ_TAIL(engine), + I915_READ_START(engine)); ret = -EIO; - goto out; } } + gen6_gt_force_wake_put(dev_priv, FORCEWAKE_ALL); + + if (ret == 0) { + engine->legacy_ring = ring; + } else { + intel_ring_free(ring); + ring = ERR_PTR(ret); + } - if (I915_NEED_GFX_HWS(dev)) - intel_ring_setup_status_page(ring); - else - ring_setup_phys_status_page(ring); + return ring; +} + +static int engine_resume(struct intel_engine_cs *engine) +{ + struct drm_i915_private *dev_priv = engine->i915; + struct intel_ringbuffer *ring = engine->legacy_ring; + int retry = 3, ret; + + if (WARN_ON(ring == NULL)) + return -ENODEV; + gen6_gt_force_wake_get(dev_priv, FORCEWAKE_ALL); + + ret = enable_status_page(engine); + +reset: /* Enforce ordering by reading HEAD register back */ - I915_READ_HEAD(ring); + engine->write_tail(engine, ring->tail); + I915_WRITE_HEAD(engine, ring->head); + (void)I915_READ_HEAD(engine); /* Initialize the ring. This must happen _after_ we've cleared the ring * registers with the above sequence (the readback of the HEAD registers * also enforces ordering), otherwise the hw might lose the new ring * register values. */ - I915_WRITE_START(ring, i915_gem_obj_ggtt_offset(obj)); - I915_WRITE_CTL(ring, - ((ringbuf->size - PAGE_SIZE) & RING_NR_PAGES) - | RING_VALID); - - /* If the head is still not zero, the ring is dead */ - if (wait_for((I915_READ_CTL(ring) & RING_VALID) != 0 && - I915_READ_START(ring) == i915_gem_obj_ggtt_offset(obj) && - (I915_READ_HEAD(ring) & HEAD_ADDR) == 0, 50)) { + I915_WRITE_START(engine, i915_gem_obj_ggtt_offset(ring->obj)); + + /* WaClearRingBufHeadRegAtInit:ctg,elk */ + if (I915_READ_HEAD(engine) != ring->head) + DRM_DEBUG("%s initialization failed [head=%08x], fudging\n", + engine->name, I915_READ_HEAD(engine)); + I915_WRITE_HEAD(engine, ring->head); + (void)I915_READ_HEAD(engine); + + I915_WRITE_CTL(engine, + ((ring->size - PAGE_SIZE) & RING_NR_PAGES) + | RING_VALID); + + if (wait_for((I915_READ_CTL(engine) & RING_VALID) != 0, 50)) { + if (retry-- && engine_stop(engine)) + goto reset; + } + + if ((I915_READ_CTL(engine) & RING_VALID) == 0 || + I915_READ_START(engine) != i915_gem_obj_ggtt_offset(ring->obj)) { DRM_ERROR("%s initialization failed " - "ctl %08x (valid? %d) head %08x tail %08x start %08x [expected %08lx]\n", - ring->name, - I915_READ_CTL(ring), I915_READ_CTL(ring) & RING_VALID, - I915_READ_HEAD(ring), I915_READ_TAIL(ring), - I915_READ_START(ring), (unsigned long)i915_gem_obj_ggtt_offset(obj)); + "ctl %08x (valid? %d) head %08x [expected %08x], tail %08x [expected %08x], start %08x [expected %08lx]\n", + engine->name, + I915_READ_CTL(engine), I915_READ_CTL(engine) & RING_VALID, + I915_READ_HEAD(engine), ring->head, + I915_READ_TAIL(engine), ring->tail, + I915_READ_START(engine), (unsigned long)i915_gem_obj_ggtt_offset(ring->obj)); ret = -EIO; - goto out; } - if (!drm_core_check_feature(ring->dev, DRIVER_MODESET)) - i915_kernel_lost_context(ring->dev); - else { - ringbuf->head = I915_READ_HEAD(ring); - ringbuf->tail = I915_READ_TAIL(ring) & TAIL_ADDR; - ringbuf->space = ring_space(ringbuf); - ringbuf->last_retired_head = -1; - } + gen6_gt_force_wake_put(dev_priv, FORCEWAKE_ALL); + return ret; +} - memset(&ring->hangcheck, 0, sizeof(ring->hangcheck)); +static void engine_put_ring(struct intel_ringbuffer *ring, + struct intel_context *ctx) +{ + if (ring->last_context == ctx) { + struct i915_gem_request *rq; + int ret = -EINVAL; -out: - gen6_gt_force_wake_put(dev_priv, FORCEWAKE_ALL); + rq = intel_engine_alloc_request(ring->engine, + ring->engine->default_context); + if (!IS_ERR(rq)) { + ret = i915_request_commit(rq); + i915_request_put(rq); + } + if (WARN_ON(ret)) + ring->last_context = ring->engine->default_context; + } +} - return ret; +static int engine_add_request(struct i915_gem_request *rq) +{ + rq->engine->write_tail(rq->engine, rq->tail); + list_add_tail(&rq->engine_list, &rq->engine->requests); + return 0; +} + +static bool engine_rq_is_complete(struct i915_gem_request *rq) +{ + return __i915_seqno_passed(rq->engine->get_seqno(rq->engine), + rq->seqno); } static int -init_pipe_control(struct intel_engine_cs *ring) +init_broken_cs_tlb_wa(struct intel_engine_cs *engine) { + struct drm_i915_gem_object *obj; int ret; - if (ring->scratch.obj) - return 0; + obj = i915_gem_object_create_stolen(engine->i915->dev, I830_WA_SIZE); + if (obj == NULL) + obj = i915_gem_alloc_object(engine->i915->dev, I830_WA_SIZE); + if (obj == NULL) { + DRM_ERROR("Failed to allocate batch bo\n"); + return -ENOMEM; + } + + ret = i915_gem_obj_ggtt_pin(obj, 0, 0); + if (ret != 0) { + drm_gem_object_unreference(&obj->base); + DRM_ERROR("Failed to ping batch bo\n"); + return ret; + } - ring->scratch.obj = i915_gem_alloc_object(ring->dev, 4096); - if (ring->scratch.obj == NULL) { + engine->scratch.obj = obj; + engine->scratch.gtt_offset = i915_gem_obj_ggtt_offset(obj); + return 0; +} + +static int +init_pipe_control(struct intel_engine_cs *engine) +{ + int ret; + + engine->scratch.obj = i915_gem_alloc_object(engine->i915->dev, 4096); + if (engine->scratch.obj == NULL) { DRM_ERROR("Failed to allocate seqno page\n"); ret = -ENOMEM; goto err; } - ret = i915_gem_object_set_cache_level(ring->scratch.obj, I915_CACHE_LLC); + ret = i915_gem_object_set_cache_level(engine->scratch.obj, I915_CACHE_LLC); if (ret) goto err_unref; - ret = i915_gem_obj_ggtt_pin(ring->scratch.obj, 4096, 0); + ret = i915_gem_obj_ggtt_pin(engine->scratch.obj, 4096, 0); if (ret) goto err_unref; - ring->scratch.gtt_offset = i915_gem_obj_ggtt_offset(ring->scratch.obj); - ring->scratch.cpu_page = kmap(sg_page(ring->scratch.obj->pages->sgl)); - if (ring->scratch.cpu_page == NULL) { + engine->scratch.gtt_offset = i915_gem_obj_ggtt_offset(engine->scratch.obj); + engine->scratch.cpu_page = kmap(sg_page(engine->scratch.obj->pages->sgl)); + if (engine->scratch.cpu_page == NULL) { ret = -ENOMEM; goto err_unpin; } DRM_DEBUG_DRIVER("%s pipe control offset: 0x%08x\n", - ring->name, ring->scratch.gtt_offset); + engine->name, engine->scratch.gtt_offset); return 0; err_unpin: - i915_gem_object_ggtt_unpin(ring->scratch.obj); + i915_gem_object_ggtt_unpin(engine->scratch.obj); err_unref: - drm_gem_object_unreference(&ring->scratch.obj->base); + drm_gem_object_unreference(&engine->scratch.obj->base); err: + engine->scratch.obj = NULL; return ret; } -static int init_render_ring(struct intel_engine_cs *ring) +static int +emit_lri(struct i915_gem_request *rq, + int num_registers, + ...) +{ + struct intel_ringbuffer *ring; + va_list ap; + + BUG_ON(num_registers > 60); + + ring = intel_ring_begin(rq, 2*num_registers + 1); + if (IS_ERR(ring)) + return PTR_ERR(ring); + + intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(num_registers)); + va_start(ap, num_registers); + while (num_registers--) { + intel_ring_emit(ring, va_arg(ap, u32)); + intel_ring_emit(ring, va_arg(ap, u32)); + } + va_end(ap); + intel_ring_advance(ring); + + return 0; +} + +static int bdw_render_init_context(struct i915_gem_request *rq) +{ + int ret; + + ret = emit_lri(rq, 8, + + /* FIXME: Unclear whether we really need this on production bdw. */ + GEN8_ROW_CHICKEN, + /* WaDisablePartialInstShootdown:bdw */ + _MASKED_BIT_ENABLE(PARTIAL_INSTRUCTION_SHOOTDOWN_DISABLE) | + /* WaDisableThreadStallDopClockGating:bdw */ + _MASKED_BIT_ENABLE(STALL_DOP_GATING_DISABLE), + + GEN7_ROW_CHICKEN2, + /* WaDisableDopClockGating:bdw May not be needed for production */ + _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE), + + /* + * This GEN8_CENTROID_PIXEL_OPT_DIS W/A is only needed for + * pre-production hardware + */ + HALF_SLICE_CHICKEN3, + _MASKED_BIT_ENABLE(GEN8_CENTROID_PIXEL_OPT_DIS) | + _MASKED_BIT_ENABLE(GEN8_SAMPLER_POWER_BYPASS_DIS), + + GEN7_HALF_SLICE_CHICKEN1, + _MASKED_BIT_ENABLE(GEN7_SINGLE_SUBSCAN_DISPATCH_ENABLE), + + COMMON_SLICE_CHICKEN2, + _MASKED_BIT_ENABLE(GEN8_CSC2_SBE_VUE_CACHE_CONSERVATIVE), + + /* Use Force Non-Coherent whenever executing a 3D context. This is a + * workaround for for a possible hang in the unlikely event a TLB + * invalidation occurs during a PSD flush. + */ + HDC_CHICKEN0, + _MASKED_BIT_ENABLE(HDC_FORCE_NON_COHERENT), + + CACHE_MODE_1, + /* Wa4x4STCOptimizationDisable:bdw */ + _MASKED_BIT_ENABLE(GEN8_4x4_STC_OPTIMIZATION_DISABLE), + + /* + * BSpec recommends 8x4 when MSAA is used, + * however in practice 16x4 seems fastest. + * + * Note that PS/WM thread counts depend on the WIZ hashing + * disable bit, which we don't touch here, but it's good + * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM). + */ + GEN7_GT_MODE, + GEN6_WIZ_HASHING_MASK | GEN6_WIZ_HASHING_16x4); + if (ret) + return ret; + + return i915_gem_render_state_init(rq); +} + +static int chv_render_init_context(struct i915_gem_request *rq) +{ + int ret; + + ret = emit_lri(rq, 3, + + GEN8_ROW_CHICKEN, + /* WaDisablePartialInstShootdown:chv */ + _MASKED_BIT_ENABLE(PARTIAL_INSTRUCTION_SHOOTDOWN_DISABLE) | + /* WaDisableThreadStallDopClockGating:chv */ + _MASKED_BIT_ENABLE(STALL_DOP_GATING_DISABLE), + + /* WaDisableDopClockGating:chv (pre-production hw) */ + GEN7_ROW_CHICKEN2, + _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE), + + /* WaDisableSamplerPowerBypass:chv (pre-production hw) */ + HALF_SLICE_CHICKEN3, + _MASKED_BIT_ENABLE(GEN8_SAMPLER_POWER_BYPASS_DIS)); + + if (ret) + return ret; + + return i915_gem_render_state_init(rq); +} + +static int render_resume(struct intel_engine_cs *engine) { - struct drm_device *dev = ring->dev; - struct drm_i915_private *dev_priv = dev->dev_private; - int ret = init_ring_common(ring); + struct drm_i915_private *dev_priv = engine->i915; + int ret; + + ret = engine_resume(engine); if (ret) return ret; /* WaTimedSingleVertexDispatch:cl,bw,ctg,elk,ilk,snb */ - if (INTEL_INFO(dev)->gen >= 4 && INTEL_INFO(dev)->gen < 7) + if (INTEL_INFO(dev_priv)->gen >= 4 && INTEL_INFO(dev_priv)->gen < 7) I915_WRITE(MI_MODE, _MASKED_BIT_ENABLE(VS_TIMER_DISPATCH)); /* We need to disable the AsyncFlip performance optimisations in order @@ -635,28 +870,22 @@ static int init_render_ring(struct intel_engine_cs *ring) * * WaDisableAsyncFlipPerfMode:snb,ivb,hsw,vlv,bdw,chv */ - if (INTEL_INFO(dev)->gen >= 6) + if (INTEL_INFO(dev_priv)->gen >= 6) I915_WRITE(MI_MODE, _MASKED_BIT_ENABLE(ASYNC_FLIP_PERF_DISABLE)); /* Required for the hardware to program scanline values for waiting */ /* WaEnableFlushTlbInvalidationMode:snb */ - if (INTEL_INFO(dev)->gen == 6) + if (INTEL_INFO(dev_priv)->gen == 6) I915_WRITE(GFX_MODE, _MASKED_BIT_ENABLE(GFX_TLB_INVALIDATE_EXPLICIT)); /* WaBCSVCSTlbInvalidationMode:ivb,vlv,hsw */ - if (IS_GEN7(dev)) + if (IS_GEN7(dev_priv)) I915_WRITE(GFX_MODE_GEN7, _MASKED_BIT_ENABLE(GFX_TLB_INVALIDATE_EXPLICIT) | _MASKED_BIT_ENABLE(GFX_REPLAY_MODE)); - if (INTEL_INFO(dev)->gen >= 5) { - ret = init_pipe_control(ring); - if (ret) - return ret; - } - - if (IS_GEN6(dev)) { + if (IS_GEN6(dev_priv)) { /* From the Sandybridge PRM, volume 1 part 3, page 24: * "If this bit is set, STCunit will have LRA as replacement * policy. [...] This bit must be reset. LRA replacement @@ -666,19 +895,40 @@ static int init_render_ring(struct intel_engine_cs *ring) _MASKED_BIT_DISABLE(CM0_STC_EVICT_DISABLE_LRA_SNB)); } - if (INTEL_INFO(dev)->gen >= 6) + if (INTEL_INFO(dev_priv)->gen >= 6) I915_WRITE(INSTPM, _MASKED_BIT_ENABLE(INSTPM_FORCE_ORDERING)); - if (HAS_L3_DPF(dev)) - I915_WRITE_IMR(ring, ~GT_PARITY_ERROR(dev)); + return 0; +} + +static void cleanup_status_page(struct intel_engine_cs *engine) +{ + struct drm_i915_gem_object *obj; - return ret; + obj = engine->status_page.obj; + if (obj == NULL) + return; + + kunmap(sg_page(obj->pages->sgl)); + i915_gem_object_ggtt_unpin(obj); + drm_gem_object_unreference(&obj->base); + engine->status_page.obj = NULL; +} + +static void engine_cleanup(struct intel_engine_cs *engine) +{ + if (engine->legacy_ring) + intel_ring_free(engine->legacy_ring); + + cleanup_status_page(engine); + i915_cmd_parser_fini_engine(engine); } -static void render_ring_cleanup(struct intel_engine_cs *ring) +static void render_cleanup(struct intel_engine_cs *engine) { - struct drm_device *dev = ring->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = engine->i915; + + engine_cleanup(engine); if (dev_priv->semaphore_obj) { i915_gem_object_ggtt_unpin(dev_priv->semaphore_obj); @@ -686,163 +936,86 @@ static void render_ring_cleanup(struct intel_engine_cs *ring) dev_priv->semaphore_obj = NULL; } - if (ring->scratch.obj == NULL) - return; - - if (INTEL_INFO(dev)->gen >= 5) { - kunmap(sg_page(ring->scratch.obj->pages->sgl)); - i915_gem_object_ggtt_unpin(ring->scratch.obj); + if (engine->scratch.obj) { + i915_gem_object_ggtt_unpin(engine->scratch.obj); + drm_gem_object_unreference(&engine->scratch.obj->base); + engine->scratch.obj = NULL; } - - drm_gem_object_unreference(&ring->scratch.obj->base); - ring->scratch.obj = NULL; } -static int gen8_rcs_signal(struct intel_engine_cs *signaller, - unsigned int num_dwords) +static int +gen8_rcs_emit_signal(struct i915_gem_request *rq, int id) { -#define MBOX_UPDATE_DWORDS 8 - struct drm_device *dev = signaller->dev; - struct drm_i915_private *dev_priv = dev->dev_private; - struct intel_engine_cs *waiter; - int i, ret, num_rings; + u64 offset = GEN8_SEMAPHORE_OFFSET(rq->i915, rq->engine->id, id); + struct intel_ringbuffer *ring; - num_rings = hweight32(INTEL_INFO(dev)->ring_mask); - num_dwords += (num_rings-1) * MBOX_UPDATE_DWORDS; -#undef MBOX_UPDATE_DWORDS + ring = intel_ring_begin(rq, 8); + if (IS_ERR(ring)) + return PTR_ERR(ring); - ret = intel_ring_begin(signaller, num_dwords); - if (ret) - return ret; - - for_each_ring(waiter, dev_priv, i) { - u64 gtt_offset = signaller->semaphore.signal_ggtt[i]; - if (gtt_offset == MI_SEMAPHORE_SYNC_INVALID) - continue; - - intel_ring_emit(signaller, GFX_OP_PIPE_CONTROL(6)); - intel_ring_emit(signaller, PIPE_CONTROL_GLOBAL_GTT_IVB | - PIPE_CONTROL_QW_WRITE | - PIPE_CONTROL_FLUSH_ENABLE); - intel_ring_emit(signaller, lower_32_bits(gtt_offset)); - intel_ring_emit(signaller, upper_32_bits(gtt_offset)); - intel_ring_emit(signaller, signaller->outstanding_lazy_seqno); - intel_ring_emit(signaller, 0); - intel_ring_emit(signaller, MI_SEMAPHORE_SIGNAL | - MI_SEMAPHORE_TARGET(waiter->id)); - intel_ring_emit(signaller, 0); - } - - return 0; -} - -static int gen8_xcs_signal(struct intel_engine_cs *signaller, - unsigned int num_dwords) -{ -#define MBOX_UPDATE_DWORDS 6 - struct drm_device *dev = signaller->dev; - struct drm_i915_private *dev_priv = dev->dev_private; - struct intel_engine_cs *waiter; - int i, ret, num_rings; - - num_rings = hweight32(INTEL_INFO(dev)->ring_mask); - num_dwords += (num_rings-1) * MBOX_UPDATE_DWORDS; -#undef MBOX_UPDATE_DWORDS - - ret = intel_ring_begin(signaller, num_dwords); - if (ret) - return ret; - - for_each_ring(waiter, dev_priv, i) { - u64 gtt_offset = signaller->semaphore.signal_ggtt[i]; - if (gtt_offset == MI_SEMAPHORE_SYNC_INVALID) - continue; - - intel_ring_emit(signaller, (MI_FLUSH_DW + 1) | - MI_FLUSH_DW_OP_STOREDW); - intel_ring_emit(signaller, lower_32_bits(gtt_offset) | - MI_FLUSH_DW_USE_GTT); - intel_ring_emit(signaller, upper_32_bits(gtt_offset)); - intel_ring_emit(signaller, signaller->outstanding_lazy_seqno); - intel_ring_emit(signaller, MI_SEMAPHORE_SIGNAL | - MI_SEMAPHORE_TARGET(waiter->id)); - intel_ring_emit(signaller, 0); - } + intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(6)); + intel_ring_emit(ring, + PIPE_CONTROL_GLOBAL_GTT_IVB | + PIPE_CONTROL_QW_WRITE | + PIPE_CONTROL_FLUSH_ENABLE); + intel_ring_emit(ring, lower_32_bits(offset)); + intel_ring_emit(ring, upper_32_bits(offset)); + intel_ring_emit(ring, rq->seqno); + intel_ring_emit(ring, 0); + intel_ring_emit(ring, + MI_SEMAPHORE_SIGNAL | + MI_SEMAPHORE_TARGET(id)); + intel_ring_emit(ring, 0); + intel_ring_advance(ring); return 0; } -static int gen6_signal(struct intel_engine_cs *signaller, - unsigned int num_dwords) +static int +gen8_xcs_emit_signal(struct i915_gem_request *rq, int id) { - struct drm_device *dev = signaller->dev; - struct drm_i915_private *dev_priv = dev->dev_private; - struct intel_engine_cs *useless; - int i, ret, num_rings; - -#define MBOX_UPDATE_DWORDS 3 - num_rings = hweight32(INTEL_INFO(dev)->ring_mask); - num_dwords += round_up((num_rings-1) * MBOX_UPDATE_DWORDS, 2); -#undef MBOX_UPDATE_DWORDS + u64 offset = GEN8_SEMAPHORE_OFFSET(rq->i915, rq->engine->id, id); + struct intel_ringbuffer *ring; - ret = intel_ring_begin(signaller, num_dwords); - if (ret) - return ret; - - for_each_ring(useless, dev_priv, i) { - u32 mbox_reg = signaller->semaphore.mbox.signal[i]; - if (mbox_reg != GEN6_NOSYNC) { - intel_ring_emit(signaller, MI_LOAD_REGISTER_IMM(1)); - intel_ring_emit(signaller, mbox_reg); - intel_ring_emit(signaller, signaller->outstanding_lazy_seqno); - } - } + ring = intel_ring_begin(rq, 6); + if (IS_ERR(ring)) + return PTR_ERR(ring); - /* If num_dwords was rounded, make sure the tail pointer is correct */ - if (num_rings % 2 == 0) - intel_ring_emit(signaller, MI_NOOP); + intel_ring_emit(ring, + MI_FLUSH_DW | + MI_FLUSH_DW_OP_STOREDW | + (4 - 2)); + intel_ring_emit(ring, + lower_32_bits(offset) | + MI_FLUSH_DW_USE_GTT); + intel_ring_emit(ring, upper_32_bits(offset)); + intel_ring_emit(ring, rq->seqno); + intel_ring_emit(ring, + MI_SEMAPHORE_SIGNAL | + MI_SEMAPHORE_TARGET(id)); + intel_ring_emit(ring, 0); + intel_ring_advance(ring); return 0; } -/** - * gen6_add_request - Update the semaphore mailbox registers - * - * @ring - ring that is adding a request - * @seqno - return seqno stuck into the ring - * - * Update the mailbox registers in the *other* rings with the current seqno. - * This acts like a signal in the canonical semaphore. - */ static int -gen6_add_request(struct intel_engine_cs *ring) +gen6_emit_signal(struct i915_gem_request *rq, int id) { - int ret; - - if (ring->semaphore.signal) - ret = ring->semaphore.signal(ring, 4); - else - ret = intel_ring_begin(ring, 4); + struct intel_ringbuffer *ring; - if (ret) - return ret; + ring = intel_ring_begin(rq, 3); + if (IS_ERR(ring)) + return PTR_ERR(ring); - intel_ring_emit(ring, MI_STORE_DWORD_INDEX); - intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT); - intel_ring_emit(ring, ring->outstanding_lazy_seqno); - intel_ring_emit(ring, MI_USER_INTERRUPT); - __intel_ring_advance(ring); + intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1)); + intel_ring_emit(ring, rq->engine->semaphore.mbox.signal[id]); + intel_ring_emit(ring, rq->seqno); + intel_ring_advance(ring); return 0; } -static inline bool i915_gem_has_seqno_wrapped(struct drm_device *dev, - u32 seqno) -{ - struct drm_i915_private *dev_priv = dev->dev_private; - return dev_priv->last_seqno < seqno; -} - /** * intel_ring_sync - sync the waiter to the signaller on seqno * @@ -852,66 +1025,52 @@ static inline bool i915_gem_has_seqno_wrapped(struct drm_device *dev, */ static int -gen8_ring_sync(struct intel_engine_cs *waiter, - struct intel_engine_cs *signaller, - u32 seqno) +gen8_emit_wait(struct i915_gem_request *waiter, + struct i915_gem_request *signaller) { - struct drm_i915_private *dev_priv = waiter->dev->dev_private; - int ret; + u64 offset = GEN8_SEMAPHORE_OFFSET(waiter->i915, signaller->engine->id, waiter->engine->id); + struct intel_ringbuffer *ring; - ret = intel_ring_begin(waiter, 4); - if (ret) - return ret; + ring = intel_ring_begin(waiter, 4); + if (IS_ERR(ring)) + return PTR_ERR(ring); - intel_ring_emit(waiter, MI_SEMAPHORE_WAIT | - MI_SEMAPHORE_GLOBAL_GTT | - MI_SEMAPHORE_POLL | - MI_SEMAPHORE_SAD_GTE_SDD); - intel_ring_emit(waiter, seqno); - intel_ring_emit(waiter, - lower_32_bits(GEN8_WAIT_OFFSET(waiter, signaller->id))); - intel_ring_emit(waiter, - upper_32_bits(GEN8_WAIT_OFFSET(waiter, signaller->id))); - intel_ring_advance(waiter); + intel_ring_emit(ring, + MI_SEMAPHORE_WAIT | + MI_SEMAPHORE_GLOBAL_GTT | + MI_SEMAPHORE_POLL | + MI_SEMAPHORE_SAD_GTE_SDD); + intel_ring_emit(ring, signaller->breadcrumb[waiter->engine->id]); + intel_ring_emit(ring, lower_32_bits(offset)); + intel_ring_emit(ring, upper_32_bits(offset)); + intel_ring_advance(ring); return 0; } static int -gen6_ring_sync(struct intel_engine_cs *waiter, - struct intel_engine_cs *signaller, - u32 seqno) +gen6_emit_wait(struct i915_gem_request *waiter, + struct i915_gem_request *signaller) { u32 dw1 = MI_SEMAPHORE_MBOX | MI_SEMAPHORE_COMPARE | MI_SEMAPHORE_REGISTER; - u32 wait_mbox = signaller->semaphore.mbox.wait[waiter->id]; - int ret; + u32 wait_mbox = signaller->engine->semaphore.mbox.wait[waiter->engine->id]; + struct intel_ringbuffer *ring; + + WARN_ON(wait_mbox == MI_SEMAPHORE_SYNC_INVALID); + + ring = intel_ring_begin(waiter, 3); + if (IS_ERR(ring)) + return PTR_ERR(ring); + intel_ring_emit(ring, dw1 | wait_mbox); /* Throughout all of the GEM code, seqno passed implies our current * seqno is >= the last seqno executed. However for hardware the * comparison is strictly greater than. */ - seqno -= 1; - - WARN_ON(wait_mbox == MI_SEMAPHORE_SYNC_INVALID); - - ret = intel_ring_begin(waiter, 4); - if (ret) - return ret; - - /* If seqno wrap happened, omit the wait with no-ops */ - if (likely(!i915_gem_has_seqno_wrapped(waiter->dev, seqno))) { - intel_ring_emit(waiter, dw1 | wait_mbox); - intel_ring_emit(waiter, seqno); - intel_ring_emit(waiter, 0); - intel_ring_emit(waiter, MI_NOOP); - } else { - intel_ring_emit(waiter, MI_NOOP); - intel_ring_emit(waiter, MI_NOOP); - intel_ring_emit(waiter, MI_NOOP); - intel_ring_emit(waiter, MI_NOOP); - } - intel_ring_advance(waiter); + intel_ring_emit(ring, signaller->breadcrumb[waiter->engine->id] - 1); + intel_ring_emit(ring, 0); + intel_ring_advance(ring); return 0; } @@ -926,10 +1085,10 @@ do { \ } while (0) static int -pc_render_add_request(struct intel_engine_cs *ring) +gen5_emit_breadcrumb(struct i915_gem_request *rq) { - u32 scratch_addr = ring->scratch.gtt_offset + 2 * CACHELINE_BYTES; - int ret; + u32 scratch_addr = rq->engine->scratch.gtt_offset + 2 * CACHELINE_BYTES; + struct intel_ringbuffer *ring; /* For Ironlake, MI_USER_INTERRUPT was deprecated and apparently * incoherent with writes to memory, i.e. completely fubar, @@ -939,16 +1098,17 @@ pc_render_add_request(struct intel_engine_cs *ring) * incoherence by flushing the 6 PIPE_NOTIFY buffers out to * memory before requesting an interrupt. */ - ret = intel_ring_begin(ring, 32); - if (ret) - return ret; + ring = intel_ring_begin(rq, 32); + if (IS_ERR(ring)) + return PTR_ERR(ring); intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4) | PIPE_CONTROL_QW_WRITE | PIPE_CONTROL_WRITE_FLUSH | PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE); - intel_ring_emit(ring, ring->scratch.gtt_offset | PIPE_CONTROL_GLOBAL_GTT); - intel_ring_emit(ring, ring->outstanding_lazy_seqno); + intel_ring_emit(ring, rq->engine->scratch.gtt_offset | PIPE_CONTROL_GLOBAL_GTT); + intel_ring_emit(ring, rq->seqno); intel_ring_emit(ring, 0); + PIPE_CONTROL_FLUSH(ring, scratch_addr); scratch_addr += 2 * CACHELINE_BYTES; /* write to separate cachelines */ PIPE_CONTROL_FLUSH(ring, scratch_addr); @@ -965,391 +1125,275 @@ pc_render_add_request(struct intel_engine_cs *ring) PIPE_CONTROL_WRITE_FLUSH | PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE | PIPE_CONTROL_NOTIFY); - intel_ring_emit(ring, ring->scratch.gtt_offset | PIPE_CONTROL_GLOBAL_GTT); - intel_ring_emit(ring, ring->outstanding_lazy_seqno); + intel_ring_emit(ring, rq->engine->scratch.gtt_offset | PIPE_CONTROL_GLOBAL_GTT); + intel_ring_emit(ring, rq->seqno); intel_ring_emit(ring, 0); - __intel_ring_advance(ring); - return 0; -} - -static u32 -gen6_ring_get_seqno(struct intel_engine_cs *ring, bool lazy_coherency) -{ - /* Workaround to force correct ordering between irq and seqno writes on - * ivb (and maybe also on snb) by reading from a CS register (like - * ACTHD) before reading the status page. */ - if (!lazy_coherency) { - struct drm_i915_private *dev_priv = ring->dev->dev_private; - POSTING_READ(RING_ACTHD(ring->mmio_base)); - } + intel_ring_advance(ring); - return intel_read_status_page(ring, I915_GEM_HWS_INDEX); + return 0; } static u32 -ring_get_seqno(struct intel_engine_cs *ring, bool lazy_coherency) +ring_get_seqno(struct intel_engine_cs *engine) { - return intel_read_status_page(ring, I915_GEM_HWS_INDEX); + return intel_read_status_page(engine, I915_GEM_HWS_INDEX); } static void -ring_set_seqno(struct intel_engine_cs *ring, u32 seqno) +ring_set_seqno(struct intel_engine_cs *engine, u32 seqno) { - intel_write_status_page(ring, I915_GEM_HWS_INDEX, seqno); + intel_write_status_page(engine, I915_GEM_HWS_INDEX, seqno); } static u32 -pc_render_get_seqno(struct intel_engine_cs *ring, bool lazy_coherency) +gen5_render_get_seqno(struct intel_engine_cs *engine) { - return ring->scratch.cpu_page[0]; + return engine->scratch.cpu_page[0]; } static void -pc_render_set_seqno(struct intel_engine_cs *ring, u32 seqno) +gen5_render_set_seqno(struct intel_engine_cs *engine, u32 seqno) { - ring->scratch.cpu_page[0] = seqno; + engine->scratch.cpu_page[0] = seqno; } -static bool -gen5_ring_get_irq(struct intel_engine_cs *ring) +static void +gen5_irq_get(struct intel_engine_cs *engine) { - struct drm_device *dev = ring->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *i915 = engine->i915; unsigned long flags; - if (!dev->irq_enabled) - return false; - - spin_lock_irqsave(&dev_priv->irq_lock, flags); - if (ring->irq_refcount++ == 0) - gen5_enable_gt_irq(dev_priv, ring->irq_enable_mask); - spin_unlock_irqrestore(&dev_priv->irq_lock, flags); - - return true; + spin_lock_irqsave(&i915->irq_lock, flags); + if (engine->irq_refcount++ == 0) + gen5_enable_gt_irq(i915, engine->irq_enable_mask); + spin_unlock_irqrestore(&i915->irq_lock, flags); } static void -gen5_ring_put_irq(struct intel_engine_cs *ring) +gen5_irq_put(struct intel_engine_cs *engine) { - struct drm_device *dev = ring->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *i915 = engine->i915; unsigned long flags; - spin_lock_irqsave(&dev_priv->irq_lock, flags); - if (--ring->irq_refcount == 0) - gen5_disable_gt_irq(dev_priv, ring->irq_enable_mask); - spin_unlock_irqrestore(&dev_priv->irq_lock, flags); + spin_lock_irqsave(&i915->irq_lock, flags); + if (--engine->irq_refcount == 0) + gen5_disable_gt_irq(i915, engine->irq_enable_mask); + spin_unlock_irqrestore(&i915->irq_lock, flags); } -static bool -i9xx_ring_get_irq(struct intel_engine_cs *ring) +static void +i9xx_irq_get(struct intel_engine_cs *engine) { - struct drm_device *dev = ring->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = engine->i915; unsigned long flags; - if (!dev->irq_enabled) - return false; - spin_lock_irqsave(&dev_priv->irq_lock, flags); - if (ring->irq_refcount++ == 0) { - dev_priv->irq_mask &= ~ring->irq_enable_mask; + if (engine->irq_refcount++ == 0) { + dev_priv->irq_mask &= ~engine->irq_enable_mask; I915_WRITE(IMR, dev_priv->irq_mask); POSTING_READ(IMR); } spin_unlock_irqrestore(&dev_priv->irq_lock, flags); - - return true; } static void -i9xx_ring_put_irq(struct intel_engine_cs *ring) +i9xx_irq_put(struct intel_engine_cs *engine) { - struct drm_device *dev = ring->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = engine->i915; unsigned long flags; spin_lock_irqsave(&dev_priv->irq_lock, flags); - if (--ring->irq_refcount == 0) { - dev_priv->irq_mask |= ring->irq_enable_mask; + if (--engine->irq_refcount == 0) { + dev_priv->irq_mask |= engine->irq_enable_mask; I915_WRITE(IMR, dev_priv->irq_mask); POSTING_READ(IMR); } spin_unlock_irqrestore(&dev_priv->irq_lock, flags); } -static bool -i8xx_ring_get_irq(struct intel_engine_cs *ring) +static void +i8xx_irq_get(struct intel_engine_cs *engine) { - struct drm_device *dev = ring->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = engine->i915; unsigned long flags; - if (!dev->irq_enabled) - return false; - spin_lock_irqsave(&dev_priv->irq_lock, flags); - if (ring->irq_refcount++ == 0) { - dev_priv->irq_mask &= ~ring->irq_enable_mask; + if (engine->irq_refcount++ == 0) { + dev_priv->irq_mask &= ~engine->irq_enable_mask; I915_WRITE16(IMR, dev_priv->irq_mask); POSTING_READ16(IMR); } spin_unlock_irqrestore(&dev_priv->irq_lock, flags); - - return true; } static void -i8xx_ring_put_irq(struct intel_engine_cs *ring) +i8xx_irq_put(struct intel_engine_cs *engine) { - struct drm_device *dev = ring->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = engine->i915; unsigned long flags; spin_lock_irqsave(&dev_priv->irq_lock, flags); - if (--ring->irq_refcount == 0) { - dev_priv->irq_mask |= ring->irq_enable_mask; + if (--engine->irq_refcount == 0) { + dev_priv->irq_mask |= engine->irq_enable_mask; I915_WRITE16(IMR, dev_priv->irq_mask); POSTING_READ16(IMR); } spin_unlock_irqrestore(&dev_priv->irq_lock, flags); } -void intel_ring_setup_status_page(struct intel_engine_cs *ring) -{ - struct drm_device *dev = ring->dev; - struct drm_i915_private *dev_priv = ring->dev->dev_private; - u32 mmio = 0; - - /* The ring status page addresses are no longer next to the rest of - * the ring registers as of gen7. - */ - if (IS_GEN7(dev)) { - switch (ring->id) { - case RCS: - mmio = RENDER_HWS_PGA_GEN7; - break; - case BCS: - mmio = BLT_HWS_PGA_GEN7; - break; - /* - * VCS2 actually doesn't exist on Gen7. Only shut up - * gcc switch check warning - */ - case VCS2: - case VCS: - mmio = BSD_HWS_PGA_GEN7; - break; - case VECS: - mmio = VEBOX_HWS_PGA_GEN7; - break; - } - } else if (IS_GEN6(ring->dev)) { - mmio = RING_HWS_PGA_GEN6(ring->mmio_base); - } else { - /* XXX: gen8 returns to sanity */ - mmio = RING_HWS_PGA(ring->mmio_base); - } - - I915_WRITE(mmio, (u32)ring->status_page.gfx_addr); - POSTING_READ(mmio); - - /* - * Flush the TLB for this page - * - * FIXME: These two bits have disappeared on gen8, so a question - * arises: do we still need this and if so how should we go about - * invalidating the TLB? - */ - if (INTEL_INFO(dev)->gen >= 6 && INTEL_INFO(dev)->gen < 8) { - u32 reg = RING_INSTPM(ring->mmio_base); - - /* ring should be idle before issuing a sync flush*/ - WARN_ON((I915_READ_MODE(ring) & MODE_IDLE) == 0); - - I915_WRITE(reg, - _MASKED_BIT_ENABLE(INSTPM_TLB_INVALIDATE | - INSTPM_SYNC_FLUSH)); - if (wait_for((I915_READ(reg) & INSTPM_SYNC_FLUSH) == 0, - 1000)) - DRM_ERROR("%s: wait for SyncFlush to complete for TLB invalidation timed out\n", - ring->name); - } -} - static int -bsd_ring_flush(struct intel_engine_cs *ring, - u32 invalidate_domains, - u32 flush_domains) +bsd_emit_flush(struct i915_gem_request *rq, + u32 flags) { - int ret; + struct intel_ringbuffer *ring; - ret = intel_ring_begin(ring, 2); - if (ret) - return ret; + ring = intel_ring_begin(rq, 1); + if (IS_ERR(ring)) + return PTR_ERR(ring); intel_ring_emit(ring, MI_FLUSH); - intel_ring_emit(ring, MI_NOOP); intel_ring_advance(ring); return 0; } static int -i9xx_add_request(struct intel_engine_cs *ring) +i9xx_emit_breadcrumb(struct i915_gem_request *rq) { - int ret; + struct intel_ringbuffer *ring; - ret = intel_ring_begin(ring, 4); - if (ret) - return ret; + ring = intel_ring_begin(rq, 4); + if (IS_ERR(ring)) + return PTR_ERR(ring); intel_ring_emit(ring, MI_STORE_DWORD_INDEX); intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT); - intel_ring_emit(ring, ring->outstanding_lazy_seqno); + intel_ring_emit(ring, rq->seqno); intel_ring_emit(ring, MI_USER_INTERRUPT); - __intel_ring_advance(ring); + intel_ring_advance(ring); return 0; } -static bool -gen6_ring_get_irq(struct intel_engine_cs *ring) +static void +gen6_irq_get(struct intel_engine_cs *engine) { - struct drm_device *dev = ring->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = engine->i915; unsigned long flags; - if (!dev->irq_enabled) - return false; - spin_lock_irqsave(&dev_priv->irq_lock, flags); - if (ring->irq_refcount++ == 0) { - if (HAS_L3_DPF(dev) && ring->id == RCS) - I915_WRITE_IMR(ring, - ~(ring->irq_enable_mask | - GT_PARITY_ERROR(dev))); - else - I915_WRITE_IMR(ring, ~ring->irq_enable_mask); - gen5_enable_gt_irq(dev_priv, ring->irq_enable_mask); + if (engine->irq_refcount++ == 0) { + I915_WRITE_IMR(engine, + ~(engine->irq_enable_mask | + engine->irq_keep_mask)); + gen5_enable_gt_irq(dev_priv, engine->irq_enable_mask); } spin_unlock_irqrestore(&dev_priv->irq_lock, flags); - return true; + /* Keep the device awake to save expensive CPU cycles when + * reading the registers. + */ + gen6_gt_force_wake_get(dev_priv, engine->power_domains); } static void -gen6_ring_put_irq(struct intel_engine_cs *ring) +gen6_irq_barrier(struct intel_engine_cs *engine) { - struct drm_device *dev = ring->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + /* w/a for lax serialisation of GPU writes with IRQs */ + struct drm_i915_private *dev_priv = engine->i915; + (void)I915_READ(RING_ACTHD(engine->mmio_base)); +} + +static void +gen6_irq_put(struct intel_engine_cs *engine) +{ + struct drm_i915_private *dev_priv = engine->i915; unsigned long flags; + gen6_gt_force_wake_put(dev_priv, engine->power_domains); + spin_lock_irqsave(&dev_priv->irq_lock, flags); - if (--ring->irq_refcount == 0) { - if (HAS_L3_DPF(dev) && ring->id == RCS) - I915_WRITE_IMR(ring, ~GT_PARITY_ERROR(dev)); - else - I915_WRITE_IMR(ring, ~0); - gen5_disable_gt_irq(dev_priv, ring->irq_enable_mask); + if (--engine->irq_refcount == 0) { + I915_WRITE_IMR(engine, ~engine->irq_keep_mask); + gen5_disable_gt_irq(dev_priv, engine->irq_enable_mask); } spin_unlock_irqrestore(&dev_priv->irq_lock, flags); } -static bool -hsw_vebox_get_irq(struct intel_engine_cs *ring) +static void +hsw_vebox_irq_get(struct intel_engine_cs *engine) { - struct drm_device *dev = ring->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = engine->i915; unsigned long flags; - if (!dev->irq_enabled) - return false; - spin_lock_irqsave(&dev_priv->irq_lock, flags); - if (ring->irq_refcount++ == 0) { - I915_WRITE_IMR(ring, ~ring->irq_enable_mask); - gen6_enable_pm_irq(dev_priv, ring->irq_enable_mask); + if (engine->irq_refcount++ == 0) { + I915_WRITE_IMR(engine, + ~(engine->irq_enable_mask | + engine->irq_keep_mask)); + gen6_enable_pm_irq(dev_priv, engine->irq_enable_mask); } spin_unlock_irqrestore(&dev_priv->irq_lock, flags); - return true; + gen6_gt_force_wake_get(dev_priv, engine->power_domains); } static void -hsw_vebox_put_irq(struct intel_engine_cs *ring) +hsw_vebox_irq_put(struct intel_engine_cs *engine) { - struct drm_device *dev = ring->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = engine->i915; unsigned long flags; - if (!dev->irq_enabled) - return; + gen6_gt_force_wake_put(dev_priv, engine->power_domains); spin_lock_irqsave(&dev_priv->irq_lock, flags); - if (--ring->irq_refcount == 0) { - I915_WRITE_IMR(ring, ~0); - gen6_disable_pm_irq(dev_priv, ring->irq_enable_mask); + if (--engine->irq_refcount == 0) { + I915_WRITE_IMR(engine, ~engine->irq_keep_mask); + gen6_disable_pm_irq(dev_priv, engine->irq_enable_mask); } spin_unlock_irqrestore(&dev_priv->irq_lock, flags); } -static bool -gen8_ring_get_irq(struct intel_engine_cs *ring) +static void +gen8_irq_get(struct intel_engine_cs *engine) { - struct drm_device *dev = ring->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = engine->i915; unsigned long flags; - if (!dev->irq_enabled) - return false; - spin_lock_irqsave(&dev_priv->irq_lock, flags); - if (ring->irq_refcount++ == 0) { - if (HAS_L3_DPF(dev) && ring->id == RCS) { - I915_WRITE_IMR(ring, - ~(ring->irq_enable_mask | - GT_RENDER_L3_PARITY_ERROR_INTERRUPT)); - } else { - I915_WRITE_IMR(ring, ~ring->irq_enable_mask); - } - POSTING_READ(RING_IMR(ring->mmio_base)); + if (engine->irq_refcount++ == 0) { + I915_WRITE_IMR(engine, + ~(engine->irq_enable_mask | + engine->irq_keep_mask)); + POSTING_READ(RING_IMR(engine->mmio_base)); } spin_unlock_irqrestore(&dev_priv->irq_lock, flags); - - return true; } static void -gen8_ring_put_irq(struct intel_engine_cs *ring) +gen8_irq_put(struct intel_engine_cs *engine) { - struct drm_device *dev = ring->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = engine->i915; unsigned long flags; spin_lock_irqsave(&dev_priv->irq_lock, flags); - if (--ring->irq_refcount == 0) { - if (HAS_L3_DPF(dev) && ring->id == RCS) { - I915_WRITE_IMR(ring, - ~GT_RENDER_L3_PARITY_ERROR_INTERRUPT); - } else { - I915_WRITE_IMR(ring, ~0); - } - POSTING_READ(RING_IMR(ring->mmio_base)); + if (--engine->irq_refcount == 0) { + I915_WRITE_IMR(engine, ~engine->irq_keep_mask); + POSTING_READ(RING_IMR(engine->mmio_base)); } spin_unlock_irqrestore(&dev_priv->irq_lock, flags); } static int -i965_dispatch_execbuffer(struct intel_engine_cs *ring, - u64 offset, u32 length, - unsigned flags) +i965_emit_batchbuffer(struct i915_gem_request *rq, + u64 offset, u32 length, + unsigned flags) { - int ret; + struct intel_ringbuffer *ring; - ret = intel_ring_begin(ring, 2); - if (ret) - return ret; + ring = intel_ring_begin(rq, 2); + if (IS_ERR(ring)) + return PTR_ERR(ring); intel_ring_emit(ring, MI_BATCH_BUFFER_START | @@ -1361,38 +1405,35 @@ i965_dispatch_execbuffer(struct intel_engine_cs *ring, return 0; } -/* Just userspace ABI convention to limit the wa batch bo to a resonable size */ -#define I830_BATCH_LIMIT (256*1024) -#define I830_TLB_ENTRIES (2) -#define I830_WA_SIZE max(I830_TLB_ENTRIES*4096, I830_BATCH_LIMIT) static int -i830_dispatch_execbuffer(struct intel_engine_cs *ring, - u64 offset, u32 len, - unsigned flags) -{ - u32 cs_offset = ring->scratch.gtt_offset; - int ret; - - ret = intel_ring_begin(ring, 6); - if (ret) - return ret; - - /* Evict the invalid PTE TLBs */ - intel_ring_emit(ring, COLOR_BLT_CMD | BLT_WRITE_RGBA); - intel_ring_emit(ring, BLT_DEPTH_32 | BLT_ROP_COLOR_COPY | 4096); - intel_ring_emit(ring, I830_TLB_ENTRIES << 16 | 4); /* load each page */ - intel_ring_emit(ring, cs_offset); - intel_ring_emit(ring, 0xdeadbeef); - intel_ring_emit(ring, MI_NOOP); - intel_ring_advance(ring); +i830_emit_batchbuffer(struct i915_gem_request *rq, + u64 offset, u32 len, + unsigned flags) +{ + u32 cs_offset = rq->engine->scratch.gtt_offset; + struct intel_ringbuffer *ring; + + if (rq->batch->vm->dirty) { + ring = intel_ring_begin(rq, 5); + if (IS_ERR(ring)) + return PTR_ERR(ring); + + /* Evict the invalid PTE TLBs */ + intel_ring_emit(ring, COLOR_BLT_CMD | BLT_WRITE_RGBA); + intel_ring_emit(ring, BLT_DEPTH_32 | BLT_ROP_COLOR_COPY | 4096); + intel_ring_emit(ring, I830_TLB_ENTRIES << 16 | 4); + intel_ring_emit(ring, cs_offset); + intel_ring_emit(ring, 0xdeadbeef); + intel_ring_advance(ring); + } if ((flags & I915_DISPATCH_PINNED) == 0) { if (len > I830_BATCH_LIMIT) return -ENOSPC; - ret = intel_ring_begin(ring, 6 + 2); - if (ret) - return ret; + ring = intel_ring_begin(rq, 6 + 1); + if (IS_ERR(ring)) + return PTR_ERR(ring); /* Blit the batch (which has now all relocs applied) to the * stable batch scratch bo area (so that the CS never @@ -1406,36 +1447,34 @@ i830_dispatch_execbuffer(struct intel_engine_cs *ring, intel_ring_emit(ring, offset); intel_ring_emit(ring, MI_FLUSH); - intel_ring_emit(ring, MI_NOOP); intel_ring_advance(ring); /* ... and execute it. */ offset = cs_offset; } - ret = intel_ring_begin(ring, 4); - if (ret) - return ret; + ring = intel_ring_begin(rq, 3); + if (IS_ERR(ring)) + return PTR_ERR(ring); intel_ring_emit(ring, MI_BATCH_BUFFER); intel_ring_emit(ring, offset | (flags & I915_DISPATCH_SECURE ? 0 : MI_BATCH_NON_SECURE)); intel_ring_emit(ring, offset + len - 8); - intel_ring_emit(ring, MI_NOOP); intel_ring_advance(ring); return 0; } static int -i915_dispatch_execbuffer(struct intel_engine_cs *ring, - u64 offset, u32 len, - unsigned flags) +i915_emit_batchbuffer(struct i915_gem_request *rq, + u64 offset, u32 len, + unsigned flags) { - int ret; + struct intel_ringbuffer *ring; - ret = intel_ring_begin(ring, 2); - if (ret) - return ret; + ring = intel_ring_begin(rq, 2); + if (IS_ERR(ring)) + return PTR_ERR(ring); intel_ring_emit(ring, MI_BATCH_BUFFER_START | MI_BATCH_GTT); intel_ring_emit(ring, offset | (flags & I915_DISPATCH_SECURE ? 0 : MI_BATCH_NON_SECURE)); @@ -1444,485 +1483,221 @@ i915_dispatch_execbuffer(struct intel_engine_cs *ring, return 0; } -static void cleanup_status_page(struct intel_engine_cs *ring) -{ - struct drm_i915_gem_object *obj; - - obj = ring->status_page.obj; - if (obj == NULL) - return; - - kunmap(sg_page(obj->pages->sgl)); - i915_gem_object_ggtt_unpin(obj); - drm_gem_object_unreference(&obj->base); - ring->status_page.obj = NULL; -} - -static int init_status_page(struct intel_engine_cs *ring) +static int setup_status_page(struct intel_engine_cs *engine) { struct drm_i915_gem_object *obj; + unsigned flags; + int ret; - if ((obj = ring->status_page.obj) == NULL) { - unsigned flags; - int ret; + obj = i915_gem_alloc_object(engine->i915->dev, 4096); + if (obj == NULL) { + DRM_ERROR("Failed to allocate status page\n"); + return -ENOMEM; + } - obj = i915_gem_alloc_object(ring->dev, 4096); - if (obj == NULL) { - DRM_ERROR("Failed to allocate status page\n"); - return -ENOMEM; - } + ret = i915_gem_object_set_cache_level(obj, I915_CACHE_LLC); + if (ret) + goto err_unref; - ret = i915_gem_object_set_cache_level(obj, I915_CACHE_LLC); - if (ret) - goto err_unref; - - flags = 0; - if (!HAS_LLC(ring->dev)) - /* On g33, we cannot place HWS above 256MiB, so - * restrict its pinning to the low mappable arena. - * Though this restriction is not documented for - * gen4, gen5, or byt, they also behave similarly - * and hang if the HWS is placed at the top of the - * GTT. To generalise, it appears that all !llc - * platforms have issues with us placing the HWS - * above the mappable region (even though we never - * actualy map it). - */ - flags |= PIN_MAPPABLE; - ret = i915_gem_obj_ggtt_pin(obj, 4096, flags); - if (ret) { + flags = 0; + if (!HAS_LLC(engine->i915)) + /* On g33, we cannot place HWS above 256MiB, so + * restrict its pinning to the low mappable arena. + * Though this restriction is not documented for + * gen4, gen5, or byt, they also behave similarly + * and hang if the HWS is placed at the top of the + * GTT. To generalise, it appears that all !llc + * platforms have issues with us placing the HWS + * above the mappable region (even though we never + * actualy map it). + */ + flags |= PIN_MAPPABLE; + ret = i915_gem_obj_ggtt_pin(obj, 4096, flags); + if (ret) { err_unref: - drm_gem_object_unreference(&obj->base); - return ret; - } - - ring->status_page.obj = obj; + drm_gem_object_unreference(&obj->base); + return ret; } - ring->status_page.gfx_addr = i915_gem_obj_ggtt_offset(obj); - ring->status_page.page_addr = kmap(sg_page(obj->pages->sgl)); - memset(ring->status_page.page_addr, 0, PAGE_SIZE); + engine->status_page.obj = obj; - DRM_DEBUG_DRIVER("%s hws offset: 0x%08x\n", - ring->name, ring->status_page.gfx_addr); + engine->status_page.gfx_addr = i915_gem_obj_ggtt_offset(obj); + engine->status_page.page_addr = kmap(sg_page(obj->pages->sgl)); + memset(engine->status_page.page_addr, 0, PAGE_SIZE); + DRM_DEBUG_DRIVER("%s hws offset: 0x%08x\n", + engine->name, engine->status_page.gfx_addr); return 0; } -static int init_phys_status_page(struct intel_engine_cs *ring) +static int setup_phys_status_page(struct intel_engine_cs *engine) { - struct drm_i915_private *dev_priv = ring->dev->dev_private; + struct drm_i915_private *i915 = engine->i915; - if (!dev_priv->status_page_dmah) { - dev_priv->status_page_dmah = - drm_pci_alloc(ring->dev, PAGE_SIZE, PAGE_SIZE); - if (!dev_priv->status_page_dmah) - return -ENOMEM; - } + i915->status_page_dmah = + drm_pci_alloc(i915->dev, PAGE_SIZE, PAGE_SIZE); + if (!i915->status_page_dmah) + return -ENOMEM; - ring->status_page.page_addr = dev_priv->status_page_dmah->vaddr; - memset(ring->status_page.page_addr, 0, PAGE_SIZE); + engine->status_page.page_addr = i915->status_page_dmah->vaddr; + memset(engine->status_page.page_addr, 0, PAGE_SIZE); return 0; } -static void intel_destroy_ringbuffer_obj(struct intel_ringbuffer *ringbuf) +void intel_ring_free(struct intel_ringbuffer *ring) { - if (!ringbuf->obj) - return; + if (ring->obj) { + iounmap(ring->virtual_start); + i915_gem_object_ggtt_unpin(ring->obj); + drm_gem_object_unreference(&ring->obj->base); + } - iounmap(ringbuf->virtual_start); - i915_gem_object_ggtt_unpin(ringbuf->obj); - drm_gem_object_unreference(&ringbuf->obj->base); - ringbuf->obj = NULL; + list_del(&ring->engine_list); + kfree(ring); } -static int intel_alloc_ringbuffer_obj(struct drm_device *dev, - struct intel_ringbuffer *ringbuf) +struct intel_ringbuffer * +intel_engine_alloc_ring(struct intel_engine_cs *engine, + struct intel_context *ctx, + int size) { - struct drm_i915_private *dev_priv = to_i915(dev); + struct drm_i915_private *i915 = engine->i915; + struct intel_ringbuffer *ring; struct drm_i915_gem_object *obj; int ret; - if (ringbuf->obj) - return 0; + DRM_DEBUG("creating ringbuffer for %s, size %d\n", engine->name, size); + + if (WARN_ON(!is_power_of_2(size))) + return ERR_PTR(-EINVAL); + + ring = kzalloc(sizeof(*ring), GFP_KERNEL); + if (ring == NULL) + return ERR_PTR(-ENOMEM); + + ring->engine = engine; + ring->ctx = ctx; - obj = NULL; - if (!HAS_LLC(dev)) - obj = i915_gem_object_create_stolen(dev, ringbuf->size); + obj = i915_gem_object_create_stolen(i915->dev, size); if (obj == NULL) - obj = i915_gem_alloc_object(dev, ringbuf->size); + obj = i915_gem_alloc_object(i915->dev, size); if (obj == NULL) - return -ENOMEM; + return ERR_PTR(-ENOMEM); /* mark ring buffers as read-only from GPU side by default */ obj->gt_ro = 1; ret = i915_gem_obj_ggtt_pin(obj, PAGE_SIZE, PIN_MAPPABLE); - if (ret) + if (ret) { + DRM_ERROR("failed pin ringbuffer into GGTT\n"); goto err_unref; + } ret = i915_gem_object_set_to_gtt_domain(obj, true); - if (ret) + if (ret) { + DRM_ERROR("failed mark ringbuffer for GTT writes\n"); goto err_unpin; + } - ringbuf->virtual_start = - ioremap_wc(dev_priv->gtt.mappable_base + i915_gem_obj_ggtt_offset(obj), - ringbuf->size); - if (ringbuf->virtual_start == NULL) { + ring->virtual_start = + ioremap_wc(i915->gtt.mappable_base + i915_gem_obj_ggtt_offset(obj), + size); + if (ring->virtual_start == NULL) { + DRM_ERROR("failed to map ringbuffer through GTT\n"); ret = -EINVAL; goto err_unpin; } - ringbuf->obj = obj; - return 0; + ring->obj = obj; + ring->size = size; + + /* Workaround an erratum on the i830 which causes a hang if + * the TAIL pointer points to within the last 2 cachelines + * of the buffer. + */ + ring->effective_size = size; + if (IS_I830(i915) || IS_845G(i915)) + ring->effective_size -= 2 * CACHELINE_BYTES; + + ring->space = intel_ring_space(ring); + ring->retired_head = -1; + + INIT_LIST_HEAD(&ring->requests); + INIT_LIST_HEAD(&ring->breadcrumbs); + list_add_tail(&ring->engine_list, &engine->rings); + + return ring; err_unpin: i915_gem_object_ggtt_unpin(obj); err_unref: drm_gem_object_unreference(&obj->base); - return ret; + return ERR_PTR(ret); +} + +static void +nop_irq_barrier(struct intel_engine_cs *engine) +{ } -static int intel_init_ring_buffer(struct drm_device *dev, - struct intel_engine_cs *ring) +static int intel_engine_init(struct intel_engine_cs *engine, + struct drm_i915_private *i915) { - struct intel_ringbuffer *ringbuf = ring->buffer; int ret; - if (ringbuf == NULL) { - ringbuf = kzalloc(sizeof(*ringbuf), GFP_KERNEL); - if (!ringbuf) - return -ENOMEM; - ring->buffer = ringbuf; - } + engine->i915 = i915; - ring->dev = dev; - INIT_LIST_HEAD(&ring->active_list); - INIT_LIST_HEAD(&ring->request_list); - ringbuf->size = 32 * PAGE_SIZE; - memset(ring->semaphore.sync_seqno, 0, sizeof(ring->semaphore.sync_seqno)); + INIT_LIST_HEAD(&engine->rings); + INIT_LIST_HEAD(&engine->read_list); + INIT_LIST_HEAD(&engine->write_list); + INIT_LIST_HEAD(&engine->requests); + INIT_LIST_HEAD(&engine->pending); + INIT_LIST_HEAD(&engine->submitted); - init_waitqueue_head(&ring->irq_queue); + spin_lock_init(&engine->lock); + spin_lock_init(&engine->irqlock); - if (I915_NEED_GFX_HWS(dev)) { - ret = init_status_page(ring); - if (ret) - goto error; - } else { - BUG_ON(ring->id != RCS); - ret = init_phys_status_page(ring); - if (ret) - goto error; - } - - ret = intel_alloc_ringbuffer_obj(dev, ringbuf); - if (ret) { - DRM_ERROR("Failed to allocate ringbuffer %s: %d\n", ring->name, ret); - goto error; - } - - /* Workaround an erratum on the i830 which causes a hang if - * the TAIL pointer points to within the last 2 cachelines - * of the buffer. - */ - ringbuf->effective_size = ringbuf->size; - if (IS_I830(dev) || IS_845G(dev)) - ringbuf->effective_size -= 2 * CACHELINE_BYTES; - - ret = i915_cmd_parser_init_ring(ring); - if (ret) - goto error; - - ret = ring->init(ring); - if (ret) - goto error; - - return 0; - -error: - kfree(ringbuf); - ring->buffer = NULL; - return ret; -} - -void intel_cleanup_ring_buffer(struct intel_engine_cs *ring) -{ - struct drm_i915_private *dev_priv = to_i915(ring->dev); - struct intel_ringbuffer *ringbuf = ring->buffer; - - if (!intel_ring_initialized(ring)) - return; - - intel_stop_ring_buffer(ring); - WARN_ON(!IS_GEN2(ring->dev) && (I915_READ_MODE(ring) & MODE_IDLE) == 0); - - intel_destroy_ringbuffer_obj(ringbuf); - ring->preallocated_lazy_request = NULL; - ring->outstanding_lazy_seqno = 0; - - if (ring->cleanup) - ring->cleanup(ring); - - cleanup_status_page(ring); - - i915_cmd_parser_fini_ring(ring); - - kfree(ringbuf); - ring->buffer = NULL; -} - -static int intel_ring_wait_request(struct intel_engine_cs *ring, int n) -{ - struct intel_ringbuffer *ringbuf = ring->buffer; - struct drm_i915_gem_request *request; - u32 seqno = 0; - int ret; - - if (ringbuf->last_retired_head != -1) { - ringbuf->head = ringbuf->last_retired_head; - ringbuf->last_retired_head = -1; - - ringbuf->space = ring_space(ringbuf); - if (ringbuf->space >= n) - return 0; - } - - list_for_each_entry(request, &ring->request_list, list) { - if (__ring_space(request->tail, ringbuf->tail, ringbuf->size) >= n) { - seqno = request->seqno; - break; - } - } - - if (seqno == 0) - return -ENOSPC; - - ret = i915_wait_seqno(ring, seqno); - if (ret) - return ret; - - i915_gem_retire_requests_ring(ring); - ringbuf->head = ringbuf->last_retired_head; - ringbuf->last_retired_head = -1; - - ringbuf->space = ring_space(ringbuf); - return 0; -} - -static int ring_wait_for_space(struct intel_engine_cs *ring, int n) -{ - struct drm_device *dev = ring->dev; - struct drm_i915_private *dev_priv = dev->dev_private; - struct intel_ringbuffer *ringbuf = ring->buffer; - unsigned long end; - int ret; - - ret = intel_ring_wait_request(ring, n); - if (ret != -ENOSPC) - return ret; - - /* force the tail write in case we have been skipping them */ - __intel_ring_advance(ring); + engine->suspend = engine_suspend; + engine->resume = engine_resume; + engine->cleanup = engine_cleanup; - /* With GEM the hangcheck timer should kick us out of the loop, - * leaving it early runs the risk of corrupting GEM state (due - * to running on almost untested codepaths). But on resume - * timers don't work yet, so prevent a complete hang in that - * case by choosing an insanely large timeout. */ - end = jiffies + 60 * HZ; + engine->get_seqno = ring_get_seqno; + engine->set_seqno = ring_set_seqno; - trace_i915_ring_wait_begin(ring); - do { - ringbuf->head = I915_READ_HEAD(ring); - ringbuf->space = ring_space(ringbuf); - if (ringbuf->space >= n) { - ret = 0; - break; - } - - if (!drm_core_check_feature(dev, DRIVER_MODESET) && - dev->primary->master) { - struct drm_i915_master_private *master_priv = dev->primary->master->driver_priv; - if (master_priv->sarea_priv) - master_priv->sarea_priv->perf_boxes |= I915_BOX_WAIT; - } - - msleep(1); - - if (dev_priv->mm.interruptible && signal_pending(current)) { - ret = -ERESTARTSYS; - break; - } - - ret = i915_gem_check_wedge(&dev_priv->gpu_error, - dev_priv->mm.interruptible); - if (ret) - break; - - if (time_after(jiffies, end)) { - ret = -EBUSY; - break; - } - } while (1); - trace_i915_ring_wait_end(ring); - return ret; -} - -static int intel_wrap_ring_buffer(struct intel_engine_cs *ring) -{ - uint32_t __iomem *virt; - struct intel_ringbuffer *ringbuf = ring->buffer; - int rem = ringbuf->size - ringbuf->tail; - - if (ringbuf->space < rem) { - int ret = ring_wait_for_space(ring, rem); - if (ret) - return ret; - } + engine->irq_barrier = nop_irq_barrier; - virt = ringbuf->virtual_start + ringbuf->tail; - rem /= 4; - while (rem--) - iowrite32(MI_NOOP, virt++); + engine->get_ring = engine_get_ring; + engine->put_ring = engine_put_ring; - ringbuf->tail = 0; - ringbuf->space = ring_space(ringbuf); + engine->semaphore.wait = NULL; - return 0; -} - -int intel_ring_idle(struct intel_engine_cs *ring) -{ - u32 seqno; - int ret; - - /* We need to add any requests required to flush the objects and ring */ - if (ring->outstanding_lazy_seqno) { - ret = i915_add_request(ring, NULL); - if (ret) - return ret; - } - - /* Wait upon the last request to be completed */ - if (list_empty(&ring->request_list)) - return 0; - - seqno = list_entry(ring->request_list.prev, - struct drm_i915_gem_request, - list)->seqno; - - return i915_wait_seqno(ring, seqno); -} - -static int -intel_ring_alloc_seqno(struct intel_engine_cs *ring) -{ - if (ring->outstanding_lazy_seqno) - return 0; - - if (ring->preallocated_lazy_request == NULL) { - struct drm_i915_gem_request *request; - - request = kmalloc(sizeof(*request), GFP_KERNEL); - if (request == NULL) - return -ENOMEM; - - ring->preallocated_lazy_request = request; - } - - return i915_gem_get_seqno(ring->dev, &ring->outstanding_lazy_seqno); -} - -static int __intel_ring_prepare(struct intel_engine_cs *ring, - int bytes) -{ - struct intel_ringbuffer *ringbuf = ring->buffer; - int ret; + engine->add_request = engine_add_request; + engine->write_tail = ring_write_tail; + engine->is_complete = engine_rq_is_complete; - if (unlikely(ringbuf->tail + bytes > ringbuf->effective_size)) { - ret = intel_wrap_ring_buffer(ring); - if (unlikely(ret)) - return ret; - } + init_waitqueue_head(&engine->irq_queue); - if (unlikely(ringbuf->space < bytes)) { - ret = ring_wait_for_space(ring, bytes); - if (unlikely(ret)) - return ret; + if (I915_NEED_GFX_HWS(i915)) { + ret = setup_status_page(engine); + } else { + BUG_ON(engine->id != RCS); + ret = setup_phys_status_page(engine); } - - return 0; -} - -int intel_ring_begin(struct intel_engine_cs *ring, - int num_dwords) -{ - struct drm_i915_private *dev_priv = ring->dev->dev_private; - int ret; - - ret = i915_gem_check_wedge(&dev_priv->gpu_error, - dev_priv->mm.interruptible); if (ret) return ret; - ret = __intel_ring_prepare(ring, num_dwords * sizeof(uint32_t)); + ret = i915_cmd_parser_init_engine(engine); if (ret) return ret; - /* Preallocate the olr before touching the ring */ - ret = intel_ring_alloc_seqno(ring); - if (ret) - return ret; - - ring->buffer->space -= num_dwords * sizeof(uint32_t); return 0; } -/* Align the ring tail to a cacheline boundary */ -int intel_ring_cacheline_align(struct intel_engine_cs *ring) -{ - int num_dwords = (ring->buffer->tail & (CACHELINE_BYTES - 1)) / sizeof(uint32_t); - int ret; - - if (num_dwords == 0) - return 0; - - num_dwords = CACHELINE_BYTES / sizeof(uint32_t) - num_dwords; - ret = intel_ring_begin(ring, num_dwords); - if (ret) - return ret; - - while (num_dwords--) - intel_ring_emit(ring, MI_NOOP); - - intel_ring_advance(ring); - - return 0; -} - -void intel_ring_init_seqno(struct intel_engine_cs *ring, u32 seqno) -{ - struct drm_device *dev = ring->dev; - struct drm_i915_private *dev_priv = dev->dev_private; - - BUG_ON(ring->outstanding_lazy_seqno); - - if (INTEL_INFO(dev)->gen == 6 || INTEL_INFO(dev)->gen == 7) { - I915_WRITE(RING_SYNC_0(ring->mmio_base), 0); - I915_WRITE(RING_SYNC_1(ring->mmio_base), 0); - if (HAS_VEBOX(dev)) - I915_WRITE(RING_SYNC_2(ring->mmio_base), 0); - } - - ring->set_seqno(ring, seqno); - ring->hangcheck.seqno = seqno; -} - -static void gen6_bsd_ring_write_tail(struct intel_engine_cs *ring, +static void gen6_bsd_ring_write_tail(struct intel_engine_cs *engine, u32 value) { - struct drm_i915_private *dev_priv = ring->dev->dev_private; + struct drm_i915_private *dev_priv = engine->i915; /* Every tail move must follow the sequence below */ @@ -1942,8 +1717,8 @@ static void gen6_bsd_ring_write_tail(struct intel_engine_cs *ring, DRM_ERROR("timed out waiting for the BSD ring to wake up\n"); /* Now that the ring is fully powered up, update the tail */ - I915_WRITE_TAIL(ring, value); - POSTING_READ(RING_TAIL(ring->mmio_base)); + I915_WRITE_TAIL(engine, value); + POSTING_READ(RING_TAIL(engine->mmio_base)); /* Let the ring send IDLE messages to the GT again, * and so let it sleep to conserve power when idle. @@ -1952,79 +1727,77 @@ static void gen6_bsd_ring_write_tail(struct intel_engine_cs *ring, _MASKED_BIT_DISABLE(GEN6_BSD_SLEEP_MSG_DISABLE)); } -static int gen6_bsd_ring_flush(struct intel_engine_cs *ring, - u32 invalidate, u32 flush) +static int gen6_bsd_emit_flush(struct i915_gem_request *rq, + u32 flags) { + struct intel_ringbuffer *ring; uint32_t cmd; - int ret; - ret = intel_ring_begin(ring, 4); - if (ret) - return ret; - - cmd = MI_FLUSH_DW; - if (INTEL_INFO(ring->dev)->gen >= 8) + cmd = 3; + if (INTEL_INFO(rq->i915)->gen >= 8) cmd += 1; + + ring = intel_ring_begin(rq, cmd); + if (IS_ERR(ring)) + return PTR_ERR(ring); + /* * Bspec vol 1c.5 - video engine command streamer: * "If ENABLED, all TLBs will be invalidated once the flush * operation is complete. This bit is only valid when the * Post-Sync Operation field is a value of 1h or 3h." */ - if (invalidate & I915_GEM_GPU_DOMAINS) - cmd |= MI_INVALIDATE_TLB | MI_INVALIDATE_BSD | - MI_FLUSH_DW_STORE_INDEX | MI_FLUSH_DW_OP_STOREDW; + cmd = MI_FLUSH_DW | (cmd - 2); + if (flags & I915_INVALIDATE_CACHES) + cmd |= (MI_INVALIDATE_TLB | + MI_INVALIDATE_BSD | + MI_FLUSH_DW_STORE_INDEX | + MI_FLUSH_DW_OP_STOREDW); intel_ring_emit(ring, cmd); intel_ring_emit(ring, I915_GEM_HWS_SCRATCH_ADDR | MI_FLUSH_DW_USE_GTT); - if (INTEL_INFO(ring->dev)->gen >= 8) { + if (INTEL_INFO(rq->i915)->gen >= 8) intel_ring_emit(ring, 0); /* upper addr */ - intel_ring_emit(ring, 0); /* value */ - } else { - intel_ring_emit(ring, 0); - intel_ring_emit(ring, MI_NOOP); - } + intel_ring_emit(ring, 0); /* value */ intel_ring_advance(ring); return 0; } static int -gen8_ring_dispatch_execbuffer(struct intel_engine_cs *ring, - u64 offset, u32 len, - unsigned flags) +gen8_emit_batchbuffer(struct i915_gem_request *rq, + u64 offset, u32 len, + unsigned flags) { - struct drm_i915_private *dev_priv = ring->dev->dev_private; - bool ppgtt = dev_priv->mm.aliasing_ppgtt != NULL && - !(flags & I915_DISPATCH_SECURE); - int ret; + bool ppgtt = USES_PPGTT(ring->dev) && !(flags & I915_DISPATCH_SECURE); + struct intel_ringbuffer *ring; - ret = intel_ring_begin(ring, 4); - if (ret) - return ret; + ring = intel_ring_begin(rq, 3); + if (IS_ERR(ring)) + return PTR_ERR(ring); /* FIXME(BDW): Address space and security selectors. */ intel_ring_emit(ring, MI_BATCH_BUFFER_START_GEN8 | (ppgtt<<8)); intel_ring_emit(ring, lower_32_bits(offset)); intel_ring_emit(ring, upper_32_bits(offset)); - intel_ring_emit(ring, MI_NOOP); intel_ring_advance(ring); return 0; } static int -hsw_ring_dispatch_execbuffer(struct intel_engine_cs *ring, - u64 offset, u32 len, - unsigned flags) +hsw_emit_batchbuffer(struct i915_gem_request *rq, + u64 offset, u32 len, + unsigned flags) { - int ret; + struct intel_ringbuffer *ring; - ret = intel_ring_begin(ring, 2); - if (ret) - return ret; + ring = intel_ring_begin(rq, 2); + if (IS_ERR(ring)) + return PTR_ERR(ring); intel_ring_emit(ring, - MI_BATCH_BUFFER_START | MI_BATCH_PPGTT_HSW | - (flags & I915_DISPATCH_SECURE ? 0 : MI_BATCH_NON_SECURE_HSW)); + MI_BATCH_BUFFER_START | + (flags & I915_DISPATCH_SECURE ? + 0 : MI_BATCH_PPGTT_HSW | MI_BATCH_NON_SECURE_HSW)); /* bit0-7 is the length on GEN6+ */ intel_ring_emit(ring, offset); intel_ring_advance(ring); @@ -2033,15 +1806,15 @@ hsw_ring_dispatch_execbuffer(struct intel_engine_cs *ring, } static int -gen6_ring_dispatch_execbuffer(struct intel_engine_cs *ring, - u64 offset, u32 len, - unsigned flags) +gen6_emit_batchbuffer(struct i915_gem_request *rq, + u64 offset, u32 len, + unsigned flags) { - int ret; + struct intel_ringbuffer *ring; - ret = intel_ring_begin(ring, 2); - if (ret) - return ret; + ring = intel_ring_begin(rq, 2); + if (IS_ERR(ring)) + return PTR_ERR(ring); intel_ring_emit(ring, MI_BATCH_BUFFER_START | @@ -2055,60 +1828,102 @@ gen6_ring_dispatch_execbuffer(struct intel_engine_cs *ring, /* Blitter support (SandyBridge+) */ -static int gen6_ring_flush(struct intel_engine_cs *ring, - u32 invalidate, u32 flush) +static int gen6_blt_emit_flush(struct i915_gem_request *rq, + u32 flags) { - struct drm_device *dev = ring->dev; + struct intel_ringbuffer *ring; uint32_t cmd; - int ret; - - ret = intel_ring_begin(ring, 4); - if (ret) - return ret; - cmd = MI_FLUSH_DW; - if (INTEL_INFO(ring->dev)->gen >= 8) + cmd = 3; + if (INTEL_INFO(rq->i915)->gen >= 8) cmd += 1; + + ring = intel_ring_begin(rq, cmd); + if (IS_ERR(ring)) + return PTR_ERR(ring); + /* * Bspec vol 1c.3 - blitter engine command streamer: * "If ENABLED, all TLBs will be invalidated once the flush * operation is complete. This bit is only valid when the * Post-Sync Operation field is a value of 1h or 3h." */ - if (invalidate & I915_GEM_DOMAIN_RENDER) - cmd |= MI_INVALIDATE_TLB | MI_FLUSH_DW_STORE_INDEX | - MI_FLUSH_DW_OP_STOREDW; + cmd = MI_FLUSH_DW | (cmd - 2); + if (flags & I915_INVALIDATE_CACHES) + cmd |= (MI_INVALIDATE_TLB | + MI_FLUSH_DW_STORE_INDEX | + MI_FLUSH_DW_OP_STOREDW); intel_ring_emit(ring, cmd); intel_ring_emit(ring, I915_GEM_HWS_SCRATCH_ADDR | MI_FLUSH_DW_USE_GTT); - if (INTEL_INFO(ring->dev)->gen >= 8) { + if (INTEL_INFO(rq->i915)->gen >= 8) intel_ring_emit(ring, 0); /* upper addr */ - intel_ring_emit(ring, 0); /* value */ - } else { - intel_ring_emit(ring, 0); - intel_ring_emit(ring, MI_NOOP); - } + intel_ring_emit(ring, 0); /* value */ intel_ring_advance(ring); - if (IS_GEN7(dev) && !invalidate && flush) - return gen7_ring_fbc_flush(ring, FBC_REND_CACHE_CLEAN); + if (IS_GEN7(rq->i915) && flags & I915_KICK_FBC) + return gen7_ring_fbc_flush(rq, FBC_REND_CACHE_CLEAN); return 0; } -int intel_init_render_ring_buffer(struct drm_device *dev) +static void gen8_engine_init_semaphore(struct intel_engine_cs *engine) { - struct drm_i915_private *dev_priv = dev->dev_private; - struct intel_engine_cs *ring = &dev_priv->ring[RCS]; + if (engine->i915->semaphore_obj == NULL) + return; + + engine->semaphore.wait = gen8_emit_wait; + engine->semaphore.signal = + engine->id == RCS ? gen8_rcs_emit_signal : gen8_xcs_emit_signal; +} + +static bool semaphores_enabled(struct drm_i915_private *dev_priv) +{ + if (INTEL_INFO(dev_priv)->gen < 6) + return false; + + if (i915.semaphores >= 0) + return i915.semaphores; + + /* Until we get further testing... */ + if (IS_GEN8(dev_priv)) + return false; + +#ifdef CONFIG_INTEL_IOMMU + /* Enable semaphores on SNB when IO remapping is off */ + if (INTEL_INFO(dev_priv)->gen == 6 && intel_iommu_gfx_mapped) + return false; +#endif + + return true; +} + +int intel_init_render_engine(struct drm_i915_private *dev_priv) +{ + struct intel_engine_cs *engine = &dev_priv->engine[RCS]; struct drm_i915_gem_object *obj; int ret; - ring->name = "render ring"; - ring->id = RCS; - ring->mmio_base = RENDER_RING_BASE; + ret = intel_engine_init(engine, dev_priv); + if (ret) + return ret; + + engine->name = "render ring"; + engine->id = RCS; + engine->power_domains = FORCEWAKE_RENDER; + engine->mmio_base = RENDER_RING_BASE; + + engine->init_context = i915_gem_render_state_init; + + if (HAS_L3_DPF(dev_priv)) { + if (INTEL_INFO(dev_priv)->gen >= 8) + engine->irq_keep_mask |= GT_RENDER_L3_PARITY_ERROR_INTERRUPT; + else + engine->irq_keep_mask |= GT_PARITY_ERROR(dev_priv); + } - if (INTEL_INFO(dev)->gen >= 8) { - if (i915_semaphore_is_enabled(dev)) { - obj = i915_gem_alloc_object(dev, 4096); + if (INTEL_INFO(dev_priv)->gen >= 8) { + if (semaphores_enabled(dev_priv)) { + obj = i915_gem_alloc_object(dev_priv->dev, 4096); if (obj == NULL) { DRM_ERROR("Failed to allocate semaphore bo. Disabling semaphores\n"); i915.semaphores = 0; @@ -2123,32 +1938,28 @@ int intel_init_render_ring_buffer(struct drm_device *dev) dev_priv->semaphore_obj = obj; } } - ring->add_request = gen6_add_request; - ring->flush = gen8_render_ring_flush; - ring->irq_get = gen8_ring_get_irq; - ring->irq_put = gen8_ring_put_irq; - ring->irq_enable_mask = GT_RENDER_USER_INTERRUPT; - ring->get_seqno = gen6_ring_get_seqno; - ring->set_seqno = ring_set_seqno; - if (i915_semaphore_is_enabled(dev)) { - WARN_ON(!dev_priv->semaphore_obj); - ring->semaphore.sync_to = gen8_ring_sync; - ring->semaphore.signal = gen8_rcs_signal; - GEN8_RING_SEMAPHORE_INIT; - } - } else if (INTEL_INFO(dev)->gen >= 6) { - ring->add_request = gen6_add_request; - ring->flush = gen7_render_ring_flush; - if (INTEL_INFO(dev)->gen == 6) - ring->flush = gen6_render_ring_flush; - ring->irq_get = gen6_ring_get_irq; - ring->irq_put = gen6_ring_put_irq; - ring->irq_enable_mask = GT_RENDER_USER_INTERRUPT; - ring->get_seqno = gen6_ring_get_seqno; - ring->set_seqno = ring_set_seqno; - if (i915_semaphore_is_enabled(dev)) { - ring->semaphore.sync_to = gen6_ring_sync; - ring->semaphore.signal = gen6_signal; + if (IS_CHERRYVIEW(dev_priv)) + engine->init_context = chv_render_init_context; + else + engine->init_context = bdw_render_init_context; + engine->emit_breadcrumb = i9xx_emit_breadcrumb; + engine->emit_flush = gen8_render_emit_flush; + engine->irq_get = gen8_irq_get; + engine->irq_put = gen8_irq_put; + engine->irq_enable_mask = GT_RENDER_USER_INTERRUPT; + gen8_engine_init_semaphore(engine); + } else if (INTEL_INFO(dev_priv)->gen >= 6) { + engine->emit_breadcrumb = i9xx_emit_breadcrumb; + engine->emit_flush = gen7_render_emit_flush; + if (INTEL_INFO(dev_priv)->gen == 6) + engine->emit_flush = gen6_render_emit_flush; + engine->irq_get = gen6_irq_get; + engine->irq_barrier = gen6_irq_barrier; + engine->irq_put = gen6_irq_put; + engine->irq_enable_mask = GT_RENDER_USER_INTERRUPT; + if (semaphores_enabled(dev_priv)) { + engine->semaphore.wait = gen6_emit_wait; + engine->semaphore.signal = gen6_emit_signal; /* * The current semaphore is only applied on pre-gen8 * platform. And there is no VCS2 ring on the pre-gen8 @@ -2156,312 +1967,206 @@ int intel_init_render_ring_buffer(struct drm_device *dev) * initialized as INVALID. Gen8 will initialize the * sema between VCS2 and RCS later. */ - ring->semaphore.mbox.wait[RCS] = MI_SEMAPHORE_SYNC_INVALID; - ring->semaphore.mbox.wait[VCS] = MI_SEMAPHORE_SYNC_RV; - ring->semaphore.mbox.wait[BCS] = MI_SEMAPHORE_SYNC_RB; - ring->semaphore.mbox.wait[VECS] = MI_SEMAPHORE_SYNC_RVE; - ring->semaphore.mbox.wait[VCS2] = MI_SEMAPHORE_SYNC_INVALID; - ring->semaphore.mbox.signal[RCS] = GEN6_NOSYNC; - ring->semaphore.mbox.signal[VCS] = GEN6_VRSYNC; - ring->semaphore.mbox.signal[BCS] = GEN6_BRSYNC; - ring->semaphore.mbox.signal[VECS] = GEN6_VERSYNC; - ring->semaphore.mbox.signal[VCS2] = GEN6_NOSYNC; + engine->semaphore.mbox.wait[RCS] = MI_SEMAPHORE_SYNC_INVALID; + engine->semaphore.mbox.wait[VCS] = MI_SEMAPHORE_SYNC_RV; + engine->semaphore.mbox.wait[BCS] = MI_SEMAPHORE_SYNC_RB; + engine->semaphore.mbox.wait[VECS] = MI_SEMAPHORE_SYNC_RVE; + engine->semaphore.mbox.wait[VCS2] = MI_SEMAPHORE_SYNC_INVALID; + engine->semaphore.mbox.signal[RCS] = GEN6_NOSYNC; + engine->semaphore.mbox.signal[VCS] = GEN6_VRSYNC; + engine->semaphore.mbox.signal[BCS] = GEN6_BRSYNC; + engine->semaphore.mbox.signal[VECS] = GEN6_VERSYNC; + engine->semaphore.mbox.signal[VCS2] = GEN6_NOSYNC; } - } else if (IS_GEN5(dev)) { - ring->add_request = pc_render_add_request; - ring->flush = gen4_render_ring_flush; - ring->get_seqno = pc_render_get_seqno; - ring->set_seqno = pc_render_set_seqno; - ring->irq_get = gen5_ring_get_irq; - ring->irq_put = gen5_ring_put_irq; - ring->irq_enable_mask = GT_RENDER_USER_INTERRUPT | - GT_RENDER_PIPECTL_NOTIFY_INTERRUPT; + } else if (IS_GEN5(dev_priv)) { + engine->emit_breadcrumb = gen5_emit_breadcrumb; + engine->emit_flush = gen4_emit_flush; + engine->get_seqno = gen5_render_get_seqno; + engine->set_seqno = gen5_render_set_seqno; + engine->irq_get = gen5_irq_get; + engine->irq_put = gen5_irq_put; + engine->irq_enable_mask = + GT_RENDER_USER_INTERRUPT | + GT_RENDER_PIPECTL_NOTIFY_INTERRUPT; } else { - ring->add_request = i9xx_add_request; - if (INTEL_INFO(dev)->gen < 4) - ring->flush = gen2_render_ring_flush; + engine->emit_breadcrumb = i9xx_emit_breadcrumb; + if (INTEL_INFO(dev_priv)->gen < 4) + engine->emit_flush = gen2_emit_flush; else - ring->flush = gen4_render_ring_flush; - ring->get_seqno = ring_get_seqno; - ring->set_seqno = ring_set_seqno; - if (IS_GEN2(dev)) { - ring->irq_get = i8xx_ring_get_irq; - ring->irq_put = i8xx_ring_put_irq; + engine->emit_flush = gen4_emit_flush; + if (IS_GEN2(dev_priv)) { + engine->irq_get = i8xx_irq_get; + engine->irq_put = i8xx_irq_put; } else { - ring->irq_get = i9xx_ring_get_irq; - ring->irq_put = i9xx_ring_put_irq; + engine->irq_get = i9xx_irq_get; + engine->irq_put = i9xx_irq_put; } - ring->irq_enable_mask = I915_USER_INTERRUPT; - } - ring->write_tail = ring_write_tail; - - if (IS_HASWELL(dev)) - ring->dispatch_execbuffer = hsw_ring_dispatch_execbuffer; - else if (IS_GEN8(dev)) - ring->dispatch_execbuffer = gen8_ring_dispatch_execbuffer; - else if (INTEL_INFO(dev)->gen >= 6) - ring->dispatch_execbuffer = gen6_ring_dispatch_execbuffer; - else if (INTEL_INFO(dev)->gen >= 4) - ring->dispatch_execbuffer = i965_dispatch_execbuffer; - else if (IS_I830(dev) || IS_845G(dev)) - ring->dispatch_execbuffer = i830_dispatch_execbuffer; + engine->irq_enable_mask = I915_USER_INTERRUPT; + } + + if (IS_GEN8(dev_priv)) + engine->emit_batchbuffer = gen8_emit_batchbuffer; + else if (IS_HASWELL(dev_priv)) + engine->emit_batchbuffer = hsw_emit_batchbuffer; + else if (INTEL_INFO(dev_priv)->gen >= 6) + engine->emit_batchbuffer = gen6_emit_batchbuffer; + else if (INTEL_INFO(dev_priv)->gen >= 4) + engine->emit_batchbuffer = i965_emit_batchbuffer; + else if (IS_I830(dev_priv) || IS_845G(dev_priv)) + engine->emit_batchbuffer = i830_emit_batchbuffer; else - ring->dispatch_execbuffer = i915_dispatch_execbuffer; - ring->init = init_render_ring; - ring->cleanup = render_ring_cleanup; - - /* Workaround batchbuffer to combat CS tlb bug. */ - if (HAS_BROKEN_CS_TLB(dev)) { - obj = i915_gem_alloc_object(dev, I830_WA_SIZE); - if (obj == NULL) { - DRM_ERROR("Failed to allocate batch bo\n"); - return -ENOMEM; - } + engine->emit_batchbuffer = i915_emit_batchbuffer; - ret = i915_gem_obj_ggtt_pin(obj, 0, 0); - if (ret != 0) { - drm_gem_object_unreference(&obj->base); - DRM_ERROR("Failed to ping batch bo\n"); - return ret; - } + engine->resume = render_resume; + engine->cleanup = render_cleanup; - ring->scratch.obj = obj; - ring->scratch.gtt_offset = i915_gem_obj_ggtt_offset(obj); - } + if (HAS_BROKEN_CS_TLB(dev_priv)) + /* Workaround batchbuffer to combat CS tlb bug. */ + ret = init_broken_cs_tlb_wa(engine); + else if (INTEL_INFO(dev_priv)->gen >= 5) + ret = init_pipe_control(engine); + if (ret) + return ret; - return intel_init_ring_buffer(dev, ring); + return 0; } -int intel_render_ring_init_dri(struct drm_device *dev, u64 start, u32 size) +int intel_init_bsd_engine(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = dev->dev_private; - struct intel_engine_cs *ring = &dev_priv->ring[RCS]; - struct intel_ringbuffer *ringbuf = ring->buffer; + struct intel_engine_cs *engine = &dev_priv->engine[VCS]; int ret; - if (ringbuf == NULL) { - ringbuf = kzalloc(sizeof(*ringbuf), GFP_KERNEL); - if (!ringbuf) - return -ENOMEM; - ring->buffer = ringbuf; - } - - ring->name = "render ring"; - ring->id = RCS; - ring->mmio_base = RENDER_RING_BASE; - - if (INTEL_INFO(dev)->gen >= 6) { - /* non-kms not supported on gen6+ */ - ret = -ENODEV; - goto err_ringbuf; - } - - /* Note: gem is not supported on gen5/ilk without kms (the corresponding - * gem_init ioctl returns with -ENODEV). Hence we do not need to set up - * the special gen5 functions. */ - ring->add_request = i9xx_add_request; - if (INTEL_INFO(dev)->gen < 4) - ring->flush = gen2_render_ring_flush; - else - ring->flush = gen4_render_ring_flush; - ring->get_seqno = ring_get_seqno; - ring->set_seqno = ring_set_seqno; - if (IS_GEN2(dev)) { - ring->irq_get = i8xx_ring_get_irq; - ring->irq_put = i8xx_ring_put_irq; - } else { - ring->irq_get = i9xx_ring_get_irq; - ring->irq_put = i9xx_ring_put_irq; - } - ring->irq_enable_mask = I915_USER_INTERRUPT; - ring->write_tail = ring_write_tail; - if (INTEL_INFO(dev)->gen >= 4) - ring->dispatch_execbuffer = i965_dispatch_execbuffer; - else if (IS_I830(dev) || IS_845G(dev)) - ring->dispatch_execbuffer = i830_dispatch_execbuffer; - else - ring->dispatch_execbuffer = i915_dispatch_execbuffer; - ring->init = init_render_ring; - ring->cleanup = render_ring_cleanup; - - ring->dev = dev; - INIT_LIST_HEAD(&ring->active_list); - INIT_LIST_HEAD(&ring->request_list); - - ringbuf->size = size; - ringbuf->effective_size = ringbuf->size; - if (IS_I830(ring->dev) || IS_845G(ring->dev)) - ringbuf->effective_size -= 2 * CACHELINE_BYTES; - - ringbuf->virtual_start = ioremap_wc(start, size); - if (ringbuf->virtual_start == NULL) { - DRM_ERROR("can not ioremap virtual address for" - " ring buffer\n"); - ret = -ENOMEM; - goto err_ringbuf; - } - - if (!I915_NEED_GFX_HWS(dev)) { - ret = init_phys_status_page(ring); - if (ret) - goto err_vstart; - } - - return 0; - -err_vstart: - iounmap(ringbuf->virtual_start); -err_ringbuf: - kfree(ringbuf); - ring->buffer = NULL; - return ret; -} + ret = intel_engine_init(engine, dev_priv); + if (ret) + return ret; -int intel_init_bsd_ring_buffer(struct drm_device *dev) -{ - struct drm_i915_private *dev_priv = dev->dev_private; - struct intel_engine_cs *ring = &dev_priv->ring[VCS]; + engine->name = "bsd ring"; + engine->id = VCS; + engine->power_domains = FORCEWAKE_MEDIA; - ring->name = "bsd ring"; - ring->id = VCS; + if (INTEL_INFO(dev_priv)->gen >= 6) { + engine->mmio_base = GEN6_BSD_RING_BASE; - ring->write_tail = ring_write_tail; - if (INTEL_INFO(dev)->gen >= 6) { - ring->mmio_base = GEN6_BSD_RING_BASE; /* gen6 bsd needs a special wa for tail updates */ - if (IS_GEN6(dev)) - ring->write_tail = gen6_bsd_ring_write_tail; - ring->flush = gen6_bsd_ring_flush; - ring->add_request = gen6_add_request; - ring->get_seqno = gen6_ring_get_seqno; - ring->set_seqno = ring_set_seqno; - if (INTEL_INFO(dev)->gen >= 8) { - ring->irq_enable_mask = + if (IS_GEN6(dev_priv)) + engine->write_tail = gen6_bsd_ring_write_tail; + engine->emit_flush = gen6_bsd_emit_flush; + engine->emit_breadcrumb = i9xx_emit_breadcrumb; + if (INTEL_INFO(dev_priv)->gen >= 8) { + engine->irq_enable_mask = GT_RENDER_USER_INTERRUPT << GEN8_VCS1_IRQ_SHIFT; - ring->irq_get = gen8_ring_get_irq; - ring->irq_put = gen8_ring_put_irq; - ring->dispatch_execbuffer = - gen8_ring_dispatch_execbuffer; - if (i915_semaphore_is_enabled(dev)) { - ring->semaphore.sync_to = gen8_ring_sync; - ring->semaphore.signal = gen8_xcs_signal; - GEN8_RING_SEMAPHORE_INIT; - } + engine->irq_get = gen8_irq_get; + engine->irq_put = gen8_irq_put; + engine->emit_batchbuffer = gen8_emit_batchbuffer; + gen8_engine_init_semaphore(engine); } else { - ring->irq_enable_mask = GT_BSD_USER_INTERRUPT; - ring->irq_get = gen6_ring_get_irq; - ring->irq_put = gen6_ring_put_irq; - ring->dispatch_execbuffer = - gen6_ring_dispatch_execbuffer; - if (i915_semaphore_is_enabled(dev)) { - ring->semaphore.sync_to = gen6_ring_sync; - ring->semaphore.signal = gen6_signal; - ring->semaphore.mbox.wait[RCS] = MI_SEMAPHORE_SYNC_VR; - ring->semaphore.mbox.wait[VCS] = MI_SEMAPHORE_SYNC_INVALID; - ring->semaphore.mbox.wait[BCS] = MI_SEMAPHORE_SYNC_VB; - ring->semaphore.mbox.wait[VECS] = MI_SEMAPHORE_SYNC_VVE; - ring->semaphore.mbox.wait[VCS2] = MI_SEMAPHORE_SYNC_INVALID; - ring->semaphore.mbox.signal[RCS] = GEN6_RVSYNC; - ring->semaphore.mbox.signal[VCS] = GEN6_NOSYNC; - ring->semaphore.mbox.signal[BCS] = GEN6_BVSYNC; - ring->semaphore.mbox.signal[VECS] = GEN6_VEVSYNC; - ring->semaphore.mbox.signal[VCS2] = GEN6_NOSYNC; + engine->irq_enable_mask = GT_BSD_USER_INTERRUPT; + engine->irq_get = gen6_irq_get; + engine->irq_barrier = gen6_irq_barrier; + engine->irq_put = gen6_irq_put; + engine->emit_batchbuffer = gen6_emit_batchbuffer; + if (semaphores_enabled(dev_priv)) { + engine->semaphore.wait = gen6_emit_wait; + engine->semaphore.signal = gen6_emit_signal; + engine->semaphore.mbox.wait[RCS] = MI_SEMAPHORE_SYNC_VR; + engine->semaphore.mbox.wait[VCS] = MI_SEMAPHORE_SYNC_INVALID; + engine->semaphore.mbox.wait[BCS] = MI_SEMAPHORE_SYNC_VB; + engine->semaphore.mbox.wait[VECS] = MI_SEMAPHORE_SYNC_VVE; + engine->semaphore.mbox.wait[VCS2] = MI_SEMAPHORE_SYNC_INVALID; + engine->semaphore.mbox.signal[RCS] = GEN6_RVSYNC; + engine->semaphore.mbox.signal[VCS] = GEN6_NOSYNC; + engine->semaphore.mbox.signal[BCS] = GEN6_BVSYNC; + engine->semaphore.mbox.signal[VECS] = GEN6_VEVSYNC; + engine->semaphore.mbox.signal[VCS2] = GEN6_NOSYNC; } } } else { - ring->mmio_base = BSD_RING_BASE; - ring->flush = bsd_ring_flush; - ring->add_request = i9xx_add_request; - ring->get_seqno = ring_get_seqno; - ring->set_seqno = ring_set_seqno; - if (IS_GEN5(dev)) { - ring->irq_enable_mask = ILK_BSD_USER_INTERRUPT; - ring->irq_get = gen5_ring_get_irq; - ring->irq_put = gen5_ring_put_irq; + engine->mmio_base = BSD_RING_BASE; + + engine->emit_flush = bsd_emit_flush; + engine->emit_breadcrumb = i9xx_emit_breadcrumb; + if (IS_GEN5(dev_priv)) { + engine->irq_enable_mask = ILK_BSD_USER_INTERRUPT; + engine->irq_get = gen5_irq_get; + engine->irq_put = gen5_irq_put; } else { - ring->irq_enable_mask = I915_BSD_USER_INTERRUPT; - ring->irq_get = i9xx_ring_get_irq; - ring->irq_put = i9xx_ring_put_irq; + engine->irq_enable_mask = I915_BSD_USER_INTERRUPT; + engine->irq_get = i9xx_irq_get; + engine->irq_put = i9xx_irq_put; } - ring->dispatch_execbuffer = i965_dispatch_execbuffer; + engine->emit_batchbuffer = i965_emit_batchbuffer; } - ring->init = init_ring_common; - return intel_init_ring_buffer(dev, ring); + return 0; } /** * Initialize the second BSD ring for Broadwell GT3. * It is noted that this only exists on Broadwell GT3. */ -int intel_init_bsd2_ring_buffer(struct drm_device *dev) +int intel_init_bsd2_engine(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = dev->dev_private; - struct intel_engine_cs *ring = &dev_priv->ring[VCS2]; + struct intel_engine_cs *engine = &dev_priv->engine[VCS2]; + int ret; - if ((INTEL_INFO(dev)->gen != 8)) { + if ((INTEL_INFO(dev_priv)->gen != 8)) { DRM_ERROR("No dual-BSD ring on non-BDW machine\n"); return -EINVAL; } - ring->name = "bsd2 ring"; - ring->id = VCS2; + ret = intel_engine_init(engine, dev_priv); + if (ret) + return ret; + + engine->name = "bsd2 ring"; + engine->id = VCS2; + engine->power_domains = FORCEWAKE_MEDIA; + engine->mmio_base = GEN8_BSD2_RING_BASE; - ring->write_tail = ring_write_tail; - ring->mmio_base = GEN8_BSD2_RING_BASE; - ring->flush = gen6_bsd_ring_flush; - ring->add_request = gen6_add_request; - ring->get_seqno = gen6_ring_get_seqno; - ring->set_seqno = ring_set_seqno; - ring->irq_enable_mask = + engine->emit_flush = gen6_bsd_emit_flush; + engine->emit_breadcrumb = i9xx_emit_breadcrumb; + engine->emit_batchbuffer = gen8_emit_batchbuffer; + engine->irq_enable_mask = GT_RENDER_USER_INTERRUPT << GEN8_VCS2_IRQ_SHIFT; - ring->irq_get = gen8_ring_get_irq; - ring->irq_put = gen8_ring_put_irq; - ring->dispatch_execbuffer = - gen8_ring_dispatch_execbuffer; - if (i915_semaphore_is_enabled(dev)) { - ring->semaphore.sync_to = gen8_ring_sync; - ring->semaphore.signal = gen8_xcs_signal; - GEN8_RING_SEMAPHORE_INIT; - } - ring->init = init_ring_common; + engine->irq_get = gen8_irq_get; + engine->irq_put = gen8_irq_put; + gen8_engine_init_semaphore(engine); - return intel_init_ring_buffer(dev, ring); + return 0; } -int intel_init_blt_ring_buffer(struct drm_device *dev) +int intel_init_blt_engine(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = dev->dev_private; - struct intel_engine_cs *ring = &dev_priv->ring[BCS]; + struct intel_engine_cs *engine = &dev_priv->engine[BCS]; + int ret; - ring->name = "blitter ring"; - ring->id = BCS; + ret = intel_engine_init(engine, dev_priv); + if (ret) + return ret; - ring->mmio_base = BLT_RING_BASE; - ring->write_tail = ring_write_tail; - ring->flush = gen6_ring_flush; - ring->add_request = gen6_add_request; - ring->get_seqno = gen6_ring_get_seqno; - ring->set_seqno = ring_set_seqno; - if (INTEL_INFO(dev)->gen >= 8) { - ring->irq_enable_mask = + engine->name = "blitter ring"; + engine->id = BCS; + engine->power_domains = FORCEWAKE_MEDIA; + engine->mmio_base = BLT_RING_BASE; + + engine->emit_flush = gen6_blt_emit_flush; + engine->emit_breadcrumb = i9xx_emit_breadcrumb; + if (INTEL_INFO(dev_priv)->gen >= 8) { + engine->irq_enable_mask = GT_RENDER_USER_INTERRUPT << GEN8_BCS_IRQ_SHIFT; - ring->irq_get = gen8_ring_get_irq; - ring->irq_put = gen8_ring_put_irq; - ring->dispatch_execbuffer = gen8_ring_dispatch_execbuffer; - if (i915_semaphore_is_enabled(dev)) { - ring->semaphore.sync_to = gen8_ring_sync; - ring->semaphore.signal = gen8_xcs_signal; - GEN8_RING_SEMAPHORE_INIT; - } + engine->irq_get = gen8_irq_get; + engine->irq_put = gen8_irq_put; + engine->emit_batchbuffer = gen8_emit_batchbuffer; + gen8_engine_init_semaphore(engine); } else { - ring->irq_enable_mask = GT_BLT_USER_INTERRUPT; - ring->irq_get = gen6_ring_get_irq; - ring->irq_put = gen6_ring_put_irq; - ring->dispatch_execbuffer = gen6_ring_dispatch_execbuffer; - if (i915_semaphore_is_enabled(dev)) { - ring->semaphore.signal = gen6_signal; - ring->semaphore.sync_to = gen6_ring_sync; + engine->irq_enable_mask = GT_BLT_USER_INTERRUPT; + engine->irq_get = gen6_irq_get; + engine->irq_barrier = gen6_irq_barrier; + engine->irq_put = gen6_irq_put; + engine->emit_batchbuffer = gen6_emit_batchbuffer; + if (semaphores_enabled(dev_priv)) { + engine->semaphore.signal = gen6_emit_signal; + engine->semaphore.wait = gen6_emit_wait; /* * The current semaphore is only applied on pre-gen8 * platform. And there is no VCS2 ring on the pre-gen8 @@ -2469,124 +2174,510 @@ int intel_init_blt_ring_buffer(struct drm_device *dev) * initialized as INVALID. Gen8 will initialize the * sema between BCS and VCS2 later. */ - ring->semaphore.mbox.wait[RCS] = MI_SEMAPHORE_SYNC_BR; - ring->semaphore.mbox.wait[VCS] = MI_SEMAPHORE_SYNC_BV; - ring->semaphore.mbox.wait[BCS] = MI_SEMAPHORE_SYNC_INVALID; - ring->semaphore.mbox.wait[VECS] = MI_SEMAPHORE_SYNC_BVE; - ring->semaphore.mbox.wait[VCS2] = MI_SEMAPHORE_SYNC_INVALID; - ring->semaphore.mbox.signal[RCS] = GEN6_RBSYNC; - ring->semaphore.mbox.signal[VCS] = GEN6_VBSYNC; - ring->semaphore.mbox.signal[BCS] = GEN6_NOSYNC; - ring->semaphore.mbox.signal[VECS] = GEN6_VEBSYNC; - ring->semaphore.mbox.signal[VCS2] = GEN6_NOSYNC; + engine->semaphore.mbox.wait[RCS] = MI_SEMAPHORE_SYNC_BR; + engine->semaphore.mbox.wait[VCS] = MI_SEMAPHORE_SYNC_BV; + engine->semaphore.mbox.wait[BCS] = MI_SEMAPHORE_SYNC_INVALID; + engine->semaphore.mbox.wait[VECS] = MI_SEMAPHORE_SYNC_BVE; + engine->semaphore.mbox.wait[VCS2] = MI_SEMAPHORE_SYNC_INVALID; + engine->semaphore.mbox.signal[RCS] = GEN6_RBSYNC; + engine->semaphore.mbox.signal[VCS] = GEN6_VBSYNC; + engine->semaphore.mbox.signal[BCS] = GEN6_NOSYNC; + engine->semaphore.mbox.signal[VECS] = GEN6_VEBSYNC; + engine->semaphore.mbox.signal[VCS2] = GEN6_NOSYNC; } } - ring->init = init_ring_common; - return intel_init_ring_buffer(dev, ring); + return 0; } -int intel_init_vebox_ring_buffer(struct drm_device *dev) +int intel_init_vebox_engine(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = dev->dev_private; - struct intel_engine_cs *ring = &dev_priv->ring[VECS]; + struct intel_engine_cs *engine = &dev_priv->engine[VECS]; + int ret; + + ret = intel_engine_init(engine, dev_priv); + if (ret) + return ret; - ring->name = "video enhancement ring"; - ring->id = VECS; + engine->name = "video enhancement ring"; + engine->id = VECS; + engine->power_domains = FORCEWAKE_MEDIA; + engine->mmio_base = VEBOX_RING_BASE; - ring->mmio_base = VEBOX_RING_BASE; - ring->write_tail = ring_write_tail; - ring->flush = gen6_ring_flush; - ring->add_request = gen6_add_request; - ring->get_seqno = gen6_ring_get_seqno; - ring->set_seqno = ring_set_seqno; + engine->emit_flush = gen6_blt_emit_flush; + engine->emit_breadcrumb = i9xx_emit_breadcrumb; - if (INTEL_INFO(dev)->gen >= 8) { - ring->irq_enable_mask = + if (INTEL_INFO(dev_priv)->gen >= 8) { + engine->irq_enable_mask = GT_RENDER_USER_INTERRUPT << GEN8_VECS_IRQ_SHIFT; - ring->irq_get = gen8_ring_get_irq; - ring->irq_put = gen8_ring_put_irq; - ring->dispatch_execbuffer = gen8_ring_dispatch_execbuffer; - if (i915_semaphore_is_enabled(dev)) { - ring->semaphore.sync_to = gen8_ring_sync; - ring->semaphore.signal = gen8_xcs_signal; - GEN8_RING_SEMAPHORE_INIT; - } + engine->irq_get = gen8_irq_get; + engine->irq_put = gen8_irq_put; + engine->emit_batchbuffer = gen8_emit_batchbuffer; + gen8_engine_init_semaphore(engine); } else { - ring->irq_enable_mask = PM_VEBOX_USER_INTERRUPT; - ring->irq_get = hsw_vebox_get_irq; - ring->irq_put = hsw_vebox_put_irq; - ring->dispatch_execbuffer = gen6_ring_dispatch_execbuffer; - if (i915_semaphore_is_enabled(dev)) { - ring->semaphore.sync_to = gen6_ring_sync; - ring->semaphore.signal = gen6_signal; - ring->semaphore.mbox.wait[RCS] = MI_SEMAPHORE_SYNC_VER; - ring->semaphore.mbox.wait[VCS] = MI_SEMAPHORE_SYNC_VEV; - ring->semaphore.mbox.wait[BCS] = MI_SEMAPHORE_SYNC_VEB; - ring->semaphore.mbox.wait[VECS] = MI_SEMAPHORE_SYNC_INVALID; - ring->semaphore.mbox.wait[VCS2] = MI_SEMAPHORE_SYNC_INVALID; - ring->semaphore.mbox.signal[RCS] = GEN6_RVESYNC; - ring->semaphore.mbox.signal[VCS] = GEN6_VVESYNC; - ring->semaphore.mbox.signal[BCS] = GEN6_BVESYNC; - ring->semaphore.mbox.signal[VECS] = GEN6_NOSYNC; - ring->semaphore.mbox.signal[VCS2] = GEN6_NOSYNC; + engine->irq_enable_mask = PM_VEBOX_USER_INTERRUPT; + engine->irq_get = hsw_vebox_irq_get; + engine->irq_barrier = gen6_irq_barrier; + engine->irq_put = hsw_vebox_irq_put; + engine->emit_batchbuffer = gen6_emit_batchbuffer; + if (semaphores_enabled(dev_priv)) { + engine->semaphore.wait = gen6_emit_wait; + engine->semaphore.signal = gen6_emit_signal; + engine->semaphore.mbox.wait[RCS] = MI_SEMAPHORE_SYNC_VER; + engine->semaphore.mbox.wait[VCS] = MI_SEMAPHORE_SYNC_VEV; + engine->semaphore.mbox.wait[BCS] = MI_SEMAPHORE_SYNC_VEB; + engine->semaphore.mbox.wait[VECS] = MI_SEMAPHORE_SYNC_INVALID; + engine->semaphore.mbox.wait[VCS2] = MI_SEMAPHORE_SYNC_INVALID; + engine->semaphore.mbox.signal[RCS] = GEN6_RVESYNC; + engine->semaphore.mbox.signal[VCS] = GEN6_VVESYNC; + engine->semaphore.mbox.signal[BCS] = GEN6_BVESYNC; + engine->semaphore.mbox.signal[VECS] = GEN6_NOSYNC; + engine->semaphore.mbox.signal[VCS2] = GEN6_NOSYNC; } } - ring->init = init_ring_common; - return intel_init_ring_buffer(dev, ring); + return 0; } int -intel_ring_flush_all_caches(struct intel_engine_cs *ring) +intel_engine_flush(struct intel_engine_cs *engine, + struct intel_context *ctx) { + struct i915_gem_request *rq; int ret; - if (!ring->gpu_caches_dirty) + rq = intel_engine_alloc_request(engine, ctx); + if (IS_ERR(rq)) + return PTR_ERR(rq); + + ret = i915_request_emit_breadcrumb(rq); + if (ret == 0) + ret = i915_request_commit(rq); + i915_request_put(rq); + + return ret; +} + +int intel_engine_sync(struct intel_engine_cs *engine) +{ + /* Wait upon the last request to be completed */ + if (engine->last_request == NULL) return 0; - ret = ring->flush(ring, 0, I915_GEM_GPU_DOMAINS); + return i915_request_wait(engine->last_request); +} + +static u32 +next_seqno(struct drm_i915_private *i915) +{ + /* reserve 0 for non-seqno */ + if (++i915->next_seqno == 0) + ++i915->next_seqno; + return i915->next_seqno; +} + +struct i915_gem_request * +intel_engine_alloc_request(struct intel_engine_cs *engine, + struct intel_context *ctx) +{ + struct intel_ringbuffer *ring; + struct i915_gem_request *rq; + int ret, n; + + ring = ctx->ring[engine->id].ring; + if (ring == NULL) { + ring = engine->get_ring(engine, ctx); + if (IS_ERR(ring)) + return ERR_CAST(ring); + + ctx->ring[engine->id].ring = ring; + } + + rq = kzalloc(sizeof(*rq), GFP_KERNEL); + if (rq == NULL) + return ERR_PTR(-ENOMEM); + + kref_init(&rq->kref); + INIT_LIST_HEAD(&rq->vmas); + INIT_LIST_HEAD(&rq->breadcrumb_link); + + rq->i915 = engine->i915; + rq->ring = ring; + rq->engine = engine; + + rq->reset_counter = atomic_read(&rq->i915->gpu_error.reset_counter); + if (rq->reset_counter & (I915_RESET_IN_PROGRESS_FLAG | I915_WEDGED)) { + ret = rq->reset_counter & I915_WEDGED ? -EIO : -EAGAIN; + goto err; + } + + rq->seqno = next_seqno(rq->i915); + memcpy(rq->semaphore, engine->semaphore.sync, sizeof(rq->semaphore)); + for (n = 0; n < ARRAY_SIZE(rq->semaphore); n++) + if (__i915_seqno_passed(rq->semaphore[n], rq->seqno)) + rq->semaphore[n] = 0; + rq->head = ring->tail; + rq->outstanding = true; + rq->pending_flush = ring->pending_flush; + + rq->ctx = ctx; + i915_gem_context_reference(rq->ctx); + + ret = i915_request_switch_context(rq); if (ret) - return ret; + goto err_ctx; + + return rq; + +err_ctx: + i915_gem_context_unreference(ctx); +err: + kfree(rq); + return ERR_PTR(ret); +} + +struct i915_gem_request * +intel_engine_seqno_to_request(struct intel_engine_cs *engine, + u32 seqno) +{ + struct i915_gem_request *rq; + + list_for_each_entry(rq, &engine->requests, engine_list) { + if (rq->seqno == seqno) + return rq; + + if (__i915_seqno_passed(rq->seqno, seqno)) + break; + } + + return NULL; +} + +void intel_engine_cleanup(struct intel_engine_cs *engine) +{ + WARN_ON(engine->last_request); + + if (engine->cleanup) + engine->cleanup(engine); +} + +static void intel_engine_clear_rings(struct intel_engine_cs *engine) +{ + struct intel_ringbuffer *ring; + + list_for_each_entry(ring, &engine->rings, engine_list) { + if (ring->retired_head != -1) { + ring->head = ring->retired_head; + ring->retired_head = -1; + + ring->space = intel_ring_space(ring); + } + + if (ring->last_context != NULL) { + struct drm_i915_gem_object *obj; + + obj = ring->last_context->ring[engine->id].state; + if (obj) + i915_gem_object_ggtt_unpin(obj); + + ring->last_context = NULL; + } + } +} + +int intel_engine_suspend(struct intel_engine_cs *engine) +{ + struct drm_i915_private *dev_priv = engine->i915; + int ret = 0; + + if (WARN_ON(!intel_engine_initialized(engine))) + return 0; + + I915_WRITE_IMR(engine, ~0); + + if (engine->suspend) + ret = engine->suspend(engine); + + intel_engine_clear_rings(engine); + + return ret; +} + +int intel_engine_resume(struct intel_engine_cs *engine) +{ + struct drm_i915_private *dev_priv = engine->i915; + int ret = 0; + + if (WARN_ON(!intel_engine_initialized(engine))) + return 0; + + if (engine->resume) + ret = engine->resume(engine); + + I915_WRITE_IMR(engine, ~engine->irq_keep_mask); + return ret; +} + +int intel_engine_retire(struct intel_engine_cs *engine, + u32 seqno) +{ + int count; - trace_i915_gem_ring_flush(ring, 0, I915_GEM_GPU_DOMAINS); + if (engine->retire) + engine->retire(engine, seqno); + + count = 0; + while (!list_empty(&engine->requests)) { + struct i915_gem_request *rq; + + rq = list_first_entry(&engine->requests, + struct i915_gem_request, + engine_list); + + if (!__i915_seqno_passed(seqno, rq->seqno)) + break; + + i915_request_retire(rq); + count++; + } + + if (unlikely(engine->trace_irq_seqno && + __i915_seqno_passed(seqno, engine->trace_irq_seqno))) { + engine->irq_put(engine); + engine->trace_irq_seqno = 0; + } + + return count; +} + +static struct i915_gem_request * +find_active_batch(struct list_head *list) +{ + struct i915_gem_request *rq, *last = NULL; + + list_for_each_entry(rq, list, engine_list) { + if (rq->batch == NULL) + continue; + + if (!__i915_request_complete__wa(rq)) + return rq; + + last = rq; + } + + return last; +} + +static bool context_is_banned(const struct intel_context *ctx, + unsigned long now) +{ + const struct i915_ctx_hang_stats *hs = &ctx->hang_stats; + + if (hs->banned) + return true; + + if (hs->ban_period_seconds == 0) + return false; + + if (now - hs->guilty_ts <= hs->ban_period_seconds) { + if (!i915_gem_context_is_default(ctx)) { + DRM_DEBUG("context hanging too fast, banning!\n"); + return true; + } else if (i915_stop_ring_allow_ban(ctx->i915)) { + if (i915_stop_ring_allow_warn(ctx->i915)) + DRM_ERROR("gpu hanging too fast, banning!\n"); + return true; + } + } + + return false; +} + +static void +intel_engine_hangstats(struct intel_engine_cs *engine) +{ + struct i915_ctx_hang_stats *hs; + struct i915_gem_request *rq; + + rq = find_active_batch(&engine->requests); + if (rq == NULL) + return; + + hs = &rq->ctx->hang_stats; + if (engine->hangcheck.score >= HANGCHECK_SCORE_RING_HUNG) { + unsigned long now = get_seconds(); + hs->banned = context_is_banned(rq->ctx, now); + hs->guilty_ts = now; + hs->batch_active++; + } else + hs->batch_pending++; + + list_for_each_entry_continue(rq, &engine->requests, engine_list) { + if (rq->batch == NULL) + continue; + + if (__i915_request_complete__wa(rq)) + continue; + + rq->ctx->hang_stats.batch_pending++; + } +} + +void intel_engine_reset(struct intel_engine_cs *engine) +{ + if (WARN_ON(!intel_engine_initialized(engine))) + return; + + if (engine->reset) + engine->reset(engine); + + memset(&engine->hangcheck, 0, sizeof(engine->hangcheck)); + intel_engine_hangstats(engine); + + intel_engine_retire(engine, engine->i915->next_seqno); + intel_engine_clear_rings(engine); +} + +static int ring_wait(struct intel_ringbuffer *ring, int n) +{ + int ret; + + trace_intel_ringbuffer_wait(ring, n); + + do { + struct i915_gem_request *rq; + + i915_gem_retire_requests__engine(ring->engine); + if (ring->retired_head != -1) { + ring->head = ring->retired_head; + ring->retired_head = -1; + + ring->space = intel_ring_space(ring); + if (ring->space >= n) + return 0; + } + + list_for_each_entry(rq, &ring->breadcrumbs, breadcrumb_link) + if (__intel_ring_space(rq->tail, ring->tail, + ring->size, I915_RING_RSVD) >= n) + break; + + if (WARN_ON(&rq->breadcrumb_link == &ring->breadcrumbs)) + return -EDEADLK; + + ret = i915_request_wait(rq); + } while (ret == 0); + + return ret; +} + +static int ring_wrap(struct intel_ringbuffer *ring, int bytes) +{ + uint32_t __iomem *virt; + int rem; + + rem = ring->size - ring->tail; + if (unlikely(ring->space < rem)) { + rem = ring_wait(ring, rem); + if (rem) + return rem; + } + + trace_intel_ringbuffer_wrap(ring, rem); + + virt = ring->virtual_start + ring->tail; + rem = ring->size - ring->tail; + + ring->space -= rem; + ring->tail = 0; + + rem /= 4; + while (rem--) + iowrite32(MI_NOOP, virt++); - ring->gpu_caches_dirty = false; return 0; } -int -intel_ring_invalidate_all_caches(struct intel_engine_cs *ring) +static int __intel_ring_prepare(struct intel_ringbuffer *ring, + int bytes) { - uint32_t flush_domains; int ret; - flush_domains = 0; - if (ring->gpu_caches_dirty) - flush_domains = I915_GEM_GPU_DOMAINS; + trace_intel_ringbuffer_begin(ring, bytes); - ret = ring->flush(ring, I915_GEM_GPU_DOMAINS, flush_domains); - if (ret) - return ret; + if (unlikely(ring->tail + bytes > ring->effective_size)) { + ret = ring_wrap(ring, bytes); + if (unlikely(ret)) + return ret; + } - trace_i915_gem_ring_flush(ring, I915_GEM_GPU_DOMAINS, flush_domains); + if (unlikely(ring->space < bytes)) { + ret = ring_wait(ring, bytes); + if (unlikely(ret)) + return ret; + } - ring->gpu_caches_dirty = false; return 0; } -void -intel_stop_ring_buffer(struct intel_engine_cs *ring) +struct intel_ringbuffer * +intel_ring_begin(struct i915_gem_request *rq, + int num_dwords) { + struct intel_ringbuffer *ring = rq->ring; int ret; - if (!intel_ring_initialized(ring)) - return; + /* TAIL updates must be aligned to a qword, so make sure we + * reserve space for any implicit padding required for this + * command. + */ + ret = __intel_ring_prepare(ring, + ALIGN(num_dwords, 2) * sizeof(uint32_t)); + if (ret) + return ERR_PTR(ret); + + ring->space -= num_dwords * sizeof(uint32_t); + + return ring; +} + +/* Align the ring tail to a cacheline boundary */ +int intel_ring_cacheline_align(struct i915_gem_request *rq) +{ + struct intel_ringbuffer *ring; + int tail, num_dwords; + + do { + tail = rq->ring->tail; + num_dwords = (tail & (CACHELINE_BYTES - 1)) / sizeof(uint32_t); + if (num_dwords == 0) + return 0; + + num_dwords = CACHELINE_BYTES / sizeof(uint32_t) - num_dwords; + ring = intel_ring_begin(rq, num_dwords); + if (IS_ERR(ring)) + return PTR_ERR(ring); + } while (tail != rq->ring->tail); + + while (num_dwords--) + intel_ring_emit(ring, MI_NOOP); + + intel_ring_advance(ring); + + return 0; +} + +struct i915_gem_request * +intel_engine_find_active_batch(struct intel_engine_cs *engine) +{ + struct i915_gem_request *rq; + unsigned long flags; - ret = intel_ring_idle(ring); - if (ret && !i915_reset_in_progress(&to_i915(ring->dev)->gpu_error)) - DRM_ERROR("failed to quiesce %s whilst cleaning up: %d\n", - ring->name, ret); + spin_lock_irqsave(&engine->irqlock, flags); + rq = find_active_batch(&engine->submitted); + spin_unlock_irqrestore(&engine->irqlock, flags); + if (rq) + return rq; - stop_ring(ring); + return find_active_batch(&engine->requests); } diff --git a/drivers/gpu/drm/i915/intel_ringbuffer.h b/drivers/gpu/drm/i915/intel_ringbuffer.h index 70525d0..b61e08c 100644 --- a/drivers/gpu/drm/i915/intel_ringbuffer.h +++ b/drivers/gpu/drm/i915/intel_ringbuffer.h @@ -5,6 +5,13 @@ #define I915_CMD_HASH_ORDER 9 +/* Early gen2 devices have a cacheline of just 32 bytes, using 64 is overkill, + * but keeps the logic simple. Indeed, the whole purpose of this macro is just + * to give some inclination as to some of the magic values used in the various + * workarounds! + */ +#define CACHELINE_BYTES 64 + /* * Gen2 BSpec "1. Programming Environment" / 1.4.4.6 "Ring Buffer Use" * Gen3 BSpec "vol1c Memory Interface Functions" / 2.3.4.5 "Ring Buffer Use" @@ -13,61 +20,48 @@ * "If the Ring Buffer Head Pointer and the Tail Pointer are on the same * cacheline, the Head Pointer must not be greater than the Tail * Pointer." + * + * To also accommodate errata on 830/845 which makes the last pair of + * cachelines in the ringbuffer unavailable, reduce the available space + * further. */ -#define I915_RING_FREE_SPACE 64 +#define I915_RING_RSVD (2*CACHELINE_BYTES) -struct intel_hw_status_page { +struct intel_hw_status_page { u32 *page_addr; unsigned int gfx_addr; struct drm_i915_gem_object *obj; }; -#define I915_READ_TAIL(ring) I915_READ(RING_TAIL((ring)->mmio_base)) -#define I915_WRITE_TAIL(ring, val) I915_WRITE(RING_TAIL((ring)->mmio_base), val) +#define I915_READ_TAIL(engine) I915_READ(RING_TAIL((engine)->mmio_base)) +#define I915_WRITE_TAIL(engine, val) I915_WRITE(RING_TAIL((engine)->mmio_base), val) -#define I915_READ_START(ring) I915_READ(RING_START((ring)->mmio_base)) -#define I915_WRITE_START(ring, val) I915_WRITE(RING_START((ring)->mmio_base), val) +#define I915_READ_START(engine) I915_READ(RING_START((engine)->mmio_base)) +#define I915_WRITE_START(engine, val) I915_WRITE(RING_START((engine)->mmio_base), val) -#define I915_READ_HEAD(ring) I915_READ(RING_HEAD((ring)->mmio_base)) -#define I915_WRITE_HEAD(ring, val) I915_WRITE(RING_HEAD((ring)->mmio_base), val) +#define I915_READ_HEAD(engine) I915_READ(RING_HEAD((engine)->mmio_base)) +#define I915_WRITE_HEAD(engine, val) I915_WRITE(RING_HEAD((engine)->mmio_base), val) -#define I915_READ_CTL(ring) I915_READ(RING_CTL((ring)->mmio_base)) -#define I915_WRITE_CTL(ring, val) I915_WRITE(RING_CTL((ring)->mmio_base), val) +#define I915_READ_CTL(engine) I915_READ(RING_CTL((engine)->mmio_base)) +#define I915_WRITE_CTL(engine, val) I915_WRITE(RING_CTL((engine)->mmio_base), val) -#define I915_READ_IMR(ring) I915_READ(RING_IMR((ring)->mmio_base)) -#define I915_WRITE_IMR(ring, val) I915_WRITE(RING_IMR((ring)->mmio_base), val) +#define I915_READ_IMR(engine) I915_READ(RING_IMR((engine)->mmio_base)) +#define I915_WRITE_IMR(engine, val) I915_WRITE(RING_IMR((engine)->mmio_base), val) -#define I915_READ_MODE(ring) I915_READ(RING_MI_MODE((ring)->mmio_base)) -#define I915_WRITE_MODE(ring, val) I915_WRITE(RING_MI_MODE((ring)->mmio_base), val) +#define I915_READ_MODE(engine) I915_READ(RING_MI_MODE((engine)->mmio_base)) +#define I915_WRITE_MODE(engine, val) I915_WRITE(RING_MI_MODE((engine)->mmio_base), val) /* seqno size is actually only a uint32, but since we plan to use MI_FLUSH_DW to * do the writes, and that must have qw aligned offsets, simply pretend it's 8b. */ #define i915_semaphore_seqno_size sizeof(uint64_t) -#define GEN8_SIGNAL_OFFSET(__ring, to) \ - (i915_gem_obj_ggtt_offset(dev_priv->semaphore_obj) + \ - ((__ring)->id * I915_NUM_RINGS * i915_semaphore_seqno_size) + \ - (i915_semaphore_seqno_size * (to))) - -#define GEN8_WAIT_OFFSET(__ring, from) \ - (i915_gem_obj_ggtt_offset(dev_priv->semaphore_obj) + \ - ((from) * I915_NUM_RINGS * i915_semaphore_seqno_size) + \ - (i915_semaphore_seqno_size * (__ring)->id)) - -#define GEN8_RING_SEMAPHORE_INIT do { \ - if (!dev_priv->semaphore_obj) { \ - break; \ - } \ - ring->semaphore.signal_ggtt[RCS] = GEN8_SIGNAL_OFFSET(ring, RCS); \ - ring->semaphore.signal_ggtt[VCS] = GEN8_SIGNAL_OFFSET(ring, VCS); \ - ring->semaphore.signal_ggtt[BCS] = GEN8_SIGNAL_OFFSET(ring, BCS); \ - ring->semaphore.signal_ggtt[VECS] = GEN8_SIGNAL_OFFSET(ring, VECS); \ - ring->semaphore.signal_ggtt[VCS2] = GEN8_SIGNAL_OFFSET(ring, VCS2); \ - ring->semaphore.signal_ggtt[ring->id] = MI_SEMAPHORE_SYNC_INVALID; \ - } while(0) - -enum intel_ring_hangcheck_action { +#define GEN8_SEMAPHORE_OFFSET(__dp, __from, __to) \ + (i915_gem_obj_ggtt_offset((__dp)->semaphore_obj) + \ + ((__from) * I915_NUM_ENGINES + (__to)) * i915_semaphore_seqno_size) + +enum intel_engine_hangcheck_action { HANGCHECK_IDLE = 0, + HANGCHECK_IDLE_WAITERS, HANGCHECK_WAIT, HANGCHECK_ACTIVE, HANGCHECK_ACTIVE_LOOP, @@ -77,38 +71,61 @@ enum intel_ring_hangcheck_action { #define HANGCHECK_SCORE_RING_HUNG 31 -struct intel_ring_hangcheck { +struct intel_engine_hangcheck { u64 acthd; u64 max_acthd; u32 seqno; + u32 interrupts; int score; - enum intel_ring_hangcheck_action action; + enum intel_engine_hangcheck_action action; int deadlock; }; +struct i915_gem_request; +struct intel_context; +struct intel_engine_cs; + struct intel_ringbuffer { + struct intel_context *last_context; + + struct intel_engine_cs *engine; + struct intel_context *ctx; + struct list_head engine_list; + struct drm_i915_gem_object *obj; void __iomem *virtual_start; - u32 head; - u32 tail; + /** + * List of breadcrumbs associated with GPU requests currently + * outstanding. + */ + struct list_head requests; + struct list_head breadcrumbs; + + int head; + int tail; int space; + int size; int effective_size; /** We track the position of the requests in the ring buffer, and - * when each is retired we increment last_retired_head as the GPU + * when each is retired we increment retired_head as the GPU * must have finished processing the request and so we know we * can advance the ringbuffer up to that position. * - * last_retired_head is set to -1 after the value is consumed so + * retired_head is set to -1 after the value is consumed so * we can detect new retirements. */ - u32 last_retired_head; + int retired_head; + int breadcrumb_tail; + + unsigned pending_flush:4; }; -struct intel_engine_cs { - const char *name; +struct intel_engine_cs { + struct drm_i915_private *i915; + const char *name; enum intel_ring_id { RCS = 0x0, VCS, @@ -116,44 +133,82 @@ struct intel_engine_cs { VECS, VCS2 } id; -#define I915_NUM_RINGS 5 +#define I915_NUM_ENGINES 5 +#define I915_NUM_ENGINE_BITS 4 #define LAST_USER_RING (VECS + 1) - u32 mmio_base; - struct drm_device *dev; - struct intel_ringbuffer *buffer; + u32 mmio_base; + u32 power_domains; + + /* protects requests against hangcheck */ + spinlock_t lock; + /* protects exlists: pending + submitted */ + spinlock_t irqlock; + + atomic_t interrupts; + u32 breadcrumb[I915_NUM_ENGINES]; + u16 tag, next_tag; + + struct list_head rings; + struct list_head requests; + struct list_head pending, submitted; + struct i915_gem_request *last_request; struct intel_hw_status_page status_page; - unsigned irq_refcount; /* protected by dev_priv->irq_lock */ - u32 irq_enable_mask; /* bitmask to enable ring interrupt */ - u32 trace_irq_seqno; - bool __must_check (*irq_get)(struct intel_engine_cs *ring); - void (*irq_put)(struct intel_engine_cs *ring); + struct intel_ringbuffer *legacy_ring; - int (*init)(struct intel_engine_cs *ring); + unsigned irq_refcount; /* protected by i915->irq_lock */ + u32 irq_enable_mask; /* bitmask to enable ring interrupt */ + u32 irq_keep_mask; /* never mask these interrupts */ + u32 trace_irq_seqno; + void (*irq_get)(struct intel_engine_cs *engine); + void (*irq_barrier)(struct intel_engine_cs *engine); + void (*irq_put)(struct intel_engine_cs *engine); + + struct intel_ringbuffer * + (*get_ring)(struct intel_engine_cs *engine, + struct intel_context *ctx); + void (*put_ring)(struct intel_ringbuffer *ring, + struct intel_context *ctx); + + void (*retire)(struct intel_engine_cs *engine, + u32 seqno); + void (*reset)(struct intel_engine_cs *engine); + int (*suspend)(struct intel_engine_cs *engine); + int (*resume)(struct intel_engine_cs *engine); + void (*cleanup)(struct intel_engine_cs *engine); - void (*write_tail)(struct intel_engine_cs *ring, - u32 value); - int __must_check (*flush)(struct intel_engine_cs *ring, - u32 invalidate_domains, - u32 flush_domains); - int (*add_request)(struct intel_engine_cs *ring); /* Some chipsets are not quite as coherent as advertised and need * an expensive kick to force a true read of the up-to-date seqno. * However, the up-to-date seqno is not always required and the last * seen value is good enough. Note that the seqno will always be * monotonic, even if not coherent. */ - u32 (*get_seqno)(struct intel_engine_cs *ring, - bool lazy_coherency); - void (*set_seqno)(struct intel_engine_cs *ring, + u32 (*get_seqno)(struct intel_engine_cs *engine); + void (*set_seqno)(struct intel_engine_cs *engine, u32 seqno); - int (*dispatch_execbuffer)(struct intel_engine_cs *ring, - u64 offset, u32 length, - unsigned flags); + + int (*init_context)(struct i915_gem_request *rq); + + int __must_check (*emit_flush)(struct i915_gem_request *rq, + u32 domains); +#define I915_COMMAND_BARRIER 0x1 +#define I915_FLUSH_CACHES 0x2 +#define I915_INVALIDATE_CACHES 0x4 +#define I915_KICK_FBC 0x8 + int __must_check (*emit_batchbuffer)(struct i915_gem_request *rq, + u64 offset, u32 length, + unsigned flags); #define I915_DISPATCH_SECURE 0x1 #define I915_DISPATCH_PINNED 0x2 - void (*cleanup)(struct intel_engine_cs *ring); + int __must_check (*emit_breadcrumb)(struct i915_gem_request *rq); + + int __must_check (*add_request)(struct i915_gem_request *rq); + void (*write_tail)(struct intel_engine_cs *engine, + u32 value); + + bool (*is_complete)(struct i915_gem_request *rq); + /* GEN8 signal/wait table - never trust comments! * signal to signal to signal to signal to signal to @@ -193,27 +248,25 @@ struct intel_engine_cs { * ie. transpose of f(x, y) */ struct { - u32 sync_seqno[I915_NUM_RINGS-1]; - - union { - struct { - /* our mbox written by others */ - u32 wait[I915_NUM_RINGS]; - /* mboxes this ring signals to */ - u32 signal[I915_NUM_RINGS]; - } mbox; - u64 signal_ggtt[I915_NUM_RINGS]; - }; - - /* AKA wait() */ - int (*sync_to)(struct intel_engine_cs *ring, - struct intel_engine_cs *to, - u32 seqno); - int (*signal)(struct intel_engine_cs *signaller, - /* num_dwords needed by caller */ - unsigned int num_dwords); + struct { + /* our mbox written by others */ + u32 wait[I915_NUM_ENGINES]; + /* mboxes this ring signals to */ + u32 signal[I915_NUM_ENGINES]; + } mbox; + + int (*wait)(struct i915_gem_request *waiter, + struct i915_gem_request *signaller); + int (*signal)(struct i915_gem_request *rq, int id); + + u32 sync[I915_NUM_ENGINES]; } semaphore; + /* Execlists */ + bool execlists_enabled; + u32 execlists_submitted; + u8 next_context_status_buffer; + /** * List of objects currently involved in rendering from the * ringbuffer. @@ -224,28 +277,13 @@ struct intel_engine_cs { * * A reference is held on the buffer while on this list. */ - struct list_head active_list; - - /** - * List of breadcrumbs associated with GPU requests currently - * outstanding. - */ - struct list_head request_list; - - /** - * Do we have some not yet emitted requests outstanding? - */ - struct drm_i915_gem_request *preallocated_lazy_request; - u32 outstanding_lazy_seqno; - bool gpu_caches_dirty; - bool fbc_dirty; + struct list_head read_list, write_list, fence_list; wait_queue_head_t irq_queue; struct intel_context *default_context; - struct intel_context *last_context; - struct intel_ring_hangcheck hangcheck; + struct intel_engine_hangcheck hangcheck; struct { struct drm_i915_gem_object *obj; @@ -288,52 +326,31 @@ struct intel_engine_cs { }; static inline bool -intel_ring_initialized(struct intel_engine_cs *ring) +intel_engine_initialized(struct intel_engine_cs *engine) { - return ring->buffer && ring->buffer->obj; + return engine->default_context; } static inline unsigned -intel_ring_flag(struct intel_engine_cs *ring) +intel_engine_flag(struct intel_engine_cs *engine) { - return 1 << ring->id; -} - -static inline u32 -intel_ring_sync_index(struct intel_engine_cs *ring, - struct intel_engine_cs *other) -{ - int idx; - - /* - * rcs -> 0 = vcs, 1 = bcs, 2 = vecs, 3 = vcs2; - * vcs -> 0 = bcs, 1 = vecs, 2 = vcs2, 3 = rcs; - * bcs -> 0 = vecs, 1 = vcs2. 2 = rcs, 3 = vcs; - * vecs -> 0 = vcs2, 1 = rcs, 2 = vcs, 3 = bcs; - * vcs2 -> 0 = rcs, 1 = vcs, 2 = bcs, 3 = vecs; - */ - - idx = (other - ring) - 1; - if (idx < 0) - idx += I915_NUM_RINGS; - - return idx; + return 1 << engine->id; } static inline u32 -intel_read_status_page(struct intel_engine_cs *ring, +intel_read_status_page(struct intel_engine_cs *engine, int reg) { /* Ensure that the compiler doesn't optimize away the load. */ barrier(); - return ring->status_page.page_addr[reg]; + return engine->status_page.page_addr[reg]; } static inline void -intel_write_status_page(struct intel_engine_cs *ring, +intel_write_status_page(struct intel_engine_cs *engine, int reg, u32 value) { - ring->status_page.page_addr[reg] = value; + engine->status_page.page_addr[reg] = value; } /** @@ -355,57 +372,77 @@ intel_write_status_page(struct intel_engine_cs *ring, #define I915_GEM_HWS_SCRATCH_INDEX 0x30 #define I915_GEM_HWS_SCRATCH_ADDR (I915_GEM_HWS_SCRATCH_INDEX << MI_STORE_DWORD_INDEX_SHIFT) -void intel_stop_ring_buffer(struct intel_engine_cs *ring); -void intel_cleanup_ring_buffer(struct intel_engine_cs *ring); +struct intel_ringbuffer * +intel_engine_alloc_ring(struct intel_engine_cs *engine, + struct intel_context *ctx, + int size); +void intel_ring_free(struct intel_ringbuffer *ring); -int __must_check intel_ring_begin(struct intel_engine_cs *ring, int n); -int __must_check intel_ring_cacheline_align(struct intel_engine_cs *ring); -static inline void intel_ring_emit(struct intel_engine_cs *ring, +struct intel_ringbuffer *__must_check +intel_ring_begin(struct i915_gem_request *rq, int n); +int __must_check intel_ring_cacheline_align(struct i915_gem_request *rq); +static inline void intel_ring_emit(struct intel_ringbuffer *ring, u32 data) { - struct intel_ringbuffer *ringbuf = ring->buffer; - iowrite32(data, ringbuf->virtual_start + ringbuf->tail); - ringbuf->tail += 4; + iowrite32(data, ring->virtual_start + ring->tail); + ring->tail += 4; } -static inline void intel_ring_advance(struct intel_engine_cs *ring) +static inline void intel_ring_advance(struct intel_ringbuffer *ring) { - struct intel_ringbuffer *ringbuf = ring->buffer; - ringbuf->tail &= ringbuf->size - 1; + ring->tail &= ring->size - 1; } -void __intel_ring_advance(struct intel_engine_cs *ring); -int __must_check intel_ring_idle(struct intel_engine_cs *ring); -void intel_ring_init_seqno(struct intel_engine_cs *ring, u32 seqno); -int intel_ring_flush_all_caches(struct intel_engine_cs *ring); -int intel_ring_invalidate_all_caches(struct intel_engine_cs *ring); - -int intel_init_render_ring_buffer(struct drm_device *dev); -int intel_init_bsd_ring_buffer(struct drm_device *dev); -int intel_init_bsd2_ring_buffer(struct drm_device *dev); -int intel_init_blt_ring_buffer(struct drm_device *dev); -int intel_init_vebox_ring_buffer(struct drm_device *dev); - -u64 intel_ring_get_active_head(struct intel_engine_cs *ring); -void intel_ring_setup_status_page(struct intel_engine_cs *ring); - -static inline u32 intel_ring_get_tail(struct intel_ringbuffer *ringbuf) +static inline int __intel_ring_space(int head, int tail, int size, int rsvd) { - return ringbuf->tail; + int space = head - (tail + 8); + if (space < 0) + space += size; + return space - rsvd; } -static inline u32 intel_ring_get_seqno(struct intel_engine_cs *ring) +static inline int intel_ring_space(struct intel_ringbuffer *ring) { - BUG_ON(ring->outstanding_lazy_seqno == 0); - return ring->outstanding_lazy_seqno; + return __intel_ring_space(ring->head, ring->tail, + ring->size, I915_RING_RSVD); } -static inline void i915_trace_irq_get(struct intel_engine_cs *ring, u32 seqno) + +struct i915_gem_request * __must_check __attribute__((nonnull)) +intel_engine_alloc_request(struct intel_engine_cs *engine, + struct intel_context *ctx); + +struct i915_gem_request * +intel_engine_find_active_batch(struct intel_engine_cs *engine); + +struct i915_gem_request * +intel_engine_seqno_to_request(struct intel_engine_cs *engine, + u32 seqno); + +int intel_init_render_engine(struct drm_i915_private *i915); +int intel_init_bsd_engine(struct drm_i915_private *i915); +int intel_init_bsd2_engine(struct drm_i915_private *i915); +int intel_init_blt_engine(struct drm_i915_private *i915); +int intel_init_vebox_engine(struct drm_i915_private *i915); + +int __must_check intel_engine_sync(struct intel_engine_cs *engine); +int __must_check intel_engine_flush(struct intel_engine_cs *engine, + struct intel_context *ctx); + +int intel_engine_retire(struct intel_engine_cs *engine, u32 seqno); +void intel_engine_reset(struct intel_engine_cs *engine); +int intel_engine_suspend(struct intel_engine_cs *engine); +int intel_engine_resume(struct intel_engine_cs *engine); +void intel_engine_cleanup(struct intel_engine_cs *engine); + + +u64 intel_engine_get_active_head(struct intel_engine_cs *engine); + +static inline void i915_trace_irq_get(struct intel_engine_cs *engine, u32 seqno) { - if (ring->trace_irq_seqno == 0 && ring->irq_get(ring)) - ring->trace_irq_seqno = seqno; -} + if (engine->trace_irq_seqno == 0) + engine->irq_get(engine); -/* DRI warts */ -int intel_render_ring_init_dri(struct drm_device *dev, u64 start, u32 size); + engine->trace_irq_seqno = seqno; +} #endif /* _INTEL_RINGBUFFER_H_ */ diff --git a/drivers/gpu/drm/i915/intel_sprite.c b/drivers/gpu/drm/i915/intel_sprite.c index 168c665..07059b4 100644 --- a/drivers/gpu/drm/i915/intel_sprite.c +++ b/drivers/gpu/drm/i915/intel_sprite.c @@ -37,6 +37,12 @@ #include #include "i915_drv.h" +/* Backport! From include/drm/drmP.h in later kernels */ +static inline wait_queue_head_t *drm_crtc_vblank_waitqueue(struct drm_crtc *crtc) +{ + return &crtc->dev->vblank[drm_crtc_index(crtc)].queue; +} + static int usecs_to_scanlines(const struct drm_display_mode *mode, int usecs) { /* paranoia */ @@ -53,6 +59,7 @@ static bool intel_pipe_update_start(struct intel_crtc *crtc, uint32_t *start_vbl enum pipe pipe = crtc->pipe; long timeout = msecs_to_jiffies_timeout(1); int scanline, min, max, vblank_start; + wait_queue_head_t *wq = drm_crtc_vblank_waitqueue(&crtc->base); DEFINE_WAIT(wait); WARN_ON(!drm_modeset_is_locked(&crtc->base.mutex)); @@ -81,7 +88,7 @@ static bool intel_pipe_update_start(struct intel_crtc *crtc, uint32_t *start_vbl * other CPUs can see the task state update by the time we * read the scanline. */ - prepare_to_wait(&crtc->vbl_wait, &wait, TASK_UNINTERRUPTIBLE); + prepare_to_wait(wq, &wait, TASK_UNINTERRUPTIBLE); scanline = intel_get_crtc_scanline(crtc); if (scanline < min || scanline > max) @@ -100,7 +107,7 @@ static bool intel_pipe_update_start(struct intel_crtc *crtc, uint32_t *start_vbl local_irq_disable(); } - finish_wait(&crtc->vbl_wait, &wait); + finish_wait(wq, &wait); drm_vblank_put(dev, pipe); @@ -163,6 +170,7 @@ vlv_update_plane(struct drm_plane *dplane, struct drm_crtc *crtc, sprctl &= ~SP_PIXFORMAT_MASK; sprctl &= ~SP_YUV_BYTE_ORDER_MASK; sprctl &= ~SP_TILED; + sprctl &= ~SP_ROTATE_180; switch (fb->pixel_format) { case DRM_FORMAT_YUYV: @@ -235,6 +243,14 @@ vlv_update_plane(struct drm_plane *dplane, struct drm_crtc *crtc, fb->pitches[0]); linear_offset -= sprsurf_offset; + if (intel_plane->rotation == BIT(DRM_ROTATE_180)) { + sprctl |= SP_ROTATE_180; + + x += src_w; + y += src_h; + linear_offset += src_h * fb->pitches[0] + src_w * pixel_size; + } + atomic_update = intel_pipe_update_start(intel_crtc, &start_vbl_count); intel_update_primary_plane(intel_crtc); @@ -364,6 +380,7 @@ ivb_update_plane(struct drm_plane *plane, struct drm_crtc *crtc, sprctl &= ~SPRITE_RGB_ORDER_RGBX; sprctl &= ~SPRITE_YUV_BYTE_ORDER_MASK; sprctl &= ~SPRITE_TILED; + sprctl &= ~SPRITE_ROTATE_180; switch (fb->pixel_format) { case DRM_FORMAT_XBGR8888: @@ -426,6 +443,18 @@ ivb_update_plane(struct drm_plane *plane, struct drm_crtc *crtc, pixel_size, fb->pitches[0]); linear_offset -= sprsurf_offset; + if (intel_plane->rotation == BIT(DRM_ROTATE_180)) { + sprctl |= SPRITE_ROTATE_180; + + /* HSW and BDW does this automagically in hardware */ + if (!IS_HASWELL(dev) && !IS_BROADWELL(dev)) { + x += src_w; + y += src_h; + linear_offset += src_h * fb->pitches[0] + + src_w * pixel_size; + } + } + atomic_update = intel_pipe_update_start(intel_crtc, &start_vbl_count); intel_update_primary_plane(intel_crtc); @@ -571,6 +600,7 @@ ilk_update_plane(struct drm_plane *plane, struct drm_crtc *crtc, dvscntr &= ~DVS_RGB_ORDER_XBGR; dvscntr &= ~DVS_YUV_BYTE_ORDER_MASK; dvscntr &= ~DVS_TILED; + dvscntr &= ~DVS_ROTATE_180; switch (fb->pixel_format) { case DRM_FORMAT_XBGR8888: @@ -628,6 +658,14 @@ ilk_update_plane(struct drm_plane *plane, struct drm_crtc *crtc, pixel_size, fb->pitches[0]); linear_offset -= dvssurf_offset; + if (intel_plane->rotation == BIT(DRM_ROTATE_180)) { + dvscntr |= DVS_ROTATE_180; + + x += src_w; + y += src_h; + linear_offset += src_h * fb->pitches[0] + src_w * pixel_size; + } + atomic_update = intel_pipe_update_start(intel_crtc, &start_vbl_count); intel_update_primary_plane(intel_crtc); @@ -813,57 +851,24 @@ static bool colorkey_enabled(struct intel_plane *intel_plane) } static int -intel_update_plane(struct drm_plane *plane, struct drm_crtc *crtc, - struct drm_framebuffer *fb, int crtc_x, int crtc_y, - unsigned int crtc_w, unsigned int crtc_h, - uint32_t src_x, uint32_t src_y, - uint32_t src_w, uint32_t src_h) +intel_check_sprite_plane(struct drm_plane *plane, + struct intel_plane_state *state) { - struct drm_device *dev = plane->dev; - struct intel_crtc *intel_crtc = to_intel_crtc(crtc); + struct intel_crtc *intel_crtc = to_intel_crtc(state->crtc); struct intel_plane *intel_plane = to_intel_plane(plane); - enum pipe pipe = intel_crtc->pipe; + struct drm_framebuffer *fb = state->fb; struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb); struct drm_i915_gem_object *obj = intel_fb->obj; - struct drm_i915_gem_object *old_obj = intel_plane->obj; - int ret; - bool primary_enabled; - bool visible; + int crtc_x, crtc_y; + unsigned int crtc_w, crtc_h; + uint32_t src_x, src_y, src_w, src_h; + struct drm_rect *src = &state->src; + struct drm_rect *dst = &state->dst; + struct drm_rect *orig_src = &state->orig_src; + const struct drm_rect *clip = &state->clip; int hscale, vscale; int max_scale, min_scale; int pixel_size = drm_format_plane_cpp(fb->pixel_format, 0); - struct drm_rect src = { - /* sample coordinates in 16.16 fixed point */ - .x1 = src_x, - .x2 = src_x + src_w, - .y1 = src_y, - .y2 = src_y + src_h, - }; - struct drm_rect dst = { - /* integer pixels */ - .x1 = crtc_x, - .x2 = crtc_x + crtc_w, - .y1 = crtc_y, - .y2 = crtc_y + crtc_h, - }; - const struct drm_rect clip = { - .x2 = intel_crtc->active ? intel_crtc->config.pipe_src_w : 0, - .y2 = intel_crtc->active ? intel_crtc->config.pipe_src_h : 0, - }; - const struct { - int crtc_x, crtc_y; - unsigned int crtc_w, crtc_h; - uint32_t src_x, src_y, src_w, src_h; - } orig = { - .crtc_x = crtc_x, - .crtc_y = crtc_y, - .crtc_w = crtc_w, - .crtc_h = crtc_h, - .src_x = src_x, - .src_y = src_y, - .src_w = src_w, - .src_h = src_h, - }; /* Don't modify another pipe's plane */ if (intel_plane->pipe != intel_crtc->pipe) { @@ -895,49 +900,55 @@ intel_update_plane(struct drm_plane *plane, struct drm_crtc *crtc, max_scale = intel_plane->max_downscale << 16; min_scale = intel_plane->can_scale ? 1 : (1 << 16); - hscale = drm_rect_calc_hscale_relaxed(&src, &dst, min_scale, max_scale); + drm_rect_rotate(src, fb->width << 16, fb->height << 16, + intel_plane->rotation); + + hscale = drm_rect_calc_hscale_relaxed(src, dst, min_scale, max_scale); BUG_ON(hscale < 0); - vscale = drm_rect_calc_vscale_relaxed(&src, &dst, min_scale, max_scale); + vscale = drm_rect_calc_vscale_relaxed(src, dst, min_scale, max_scale); BUG_ON(vscale < 0); - visible = drm_rect_clip_scaled(&src, &dst, &clip, hscale, vscale); + state->visible = drm_rect_clip_scaled(src, dst, clip, hscale, vscale); - crtc_x = dst.x1; - crtc_y = dst.y1; - crtc_w = drm_rect_width(&dst); - crtc_h = drm_rect_height(&dst); + crtc_x = dst->x1; + crtc_y = dst->y1; + crtc_w = drm_rect_width(dst); + crtc_h = drm_rect_height(dst); - if (visible) { + if (state->visible) { /* check again in case clipping clamped the results */ - hscale = drm_rect_calc_hscale(&src, &dst, min_scale, max_scale); + hscale = drm_rect_calc_hscale(src, dst, min_scale, max_scale); if (hscale < 0) { DRM_DEBUG_KMS("Horizontal scaling factor out of limits\n"); - drm_rect_debug_print(&src, true); - drm_rect_debug_print(&dst, false); + drm_rect_debug_print(src, true); + drm_rect_debug_print(dst, false); return hscale; } - vscale = drm_rect_calc_vscale(&src, &dst, min_scale, max_scale); + vscale = drm_rect_calc_vscale(src, dst, min_scale, max_scale); if (vscale < 0) { DRM_DEBUG_KMS("Vertical scaling factor out of limits\n"); - drm_rect_debug_print(&src, true); - drm_rect_debug_print(&dst, false); + drm_rect_debug_print(src, true); + drm_rect_debug_print(dst, false); return vscale; } /* Make the source viewport size an exact multiple of the scaling factors. */ - drm_rect_adjust_size(&src, - drm_rect_width(&dst) * hscale - drm_rect_width(&src), - drm_rect_height(&dst) * vscale - drm_rect_height(&src)); + drm_rect_adjust_size(src, + drm_rect_width(dst) * hscale - drm_rect_width(src), + drm_rect_height(dst) * vscale - drm_rect_height(src)); + + drm_rect_rotate_inv(src, fb->width << 16, fb->height << 16, + intel_plane->rotation); /* sanity check to make sure the src viewport wasn't enlarged */ - WARN_ON(src.x1 < (int) src_x || - src.y1 < (int) src_y || - src.x2 > (int) (src_x + src_w) || - src.y2 > (int) (src_y + src_h)); + WARN_ON(src->x1 < (int) orig_src->x1 || + src->y1 < (int) orig_src->y1 || + src->x2 > (int) orig_src->x2 || + src->y2 > (int) orig_src->y2); /* * Hardware doesn't handle subpixel coordinates. @@ -945,10 +956,10 @@ intel_update_plane(struct drm_plane *plane, struct drm_crtc *crtc, * increase the source viewport size, because that could * push the downscaling factor out of bounds. */ - src_x = src.x1 >> 16; - src_w = drm_rect_width(&src) >> 16; - src_y = src.y1 >> 16; - src_h = drm_rect_height(&src) >> 16; + src_x = src->x1 >> 16; + src_w = drm_rect_width(src) >> 16; + src_y = src->y1 >> 16; + src_h = drm_rect_height(src) >> 16; if (format_is_yuv(fb->pixel_format)) { src_x &= ~1; @@ -962,12 +973,12 @@ intel_update_plane(struct drm_plane *plane, struct drm_crtc *crtc, crtc_w &= ~1; if (crtc_w == 0) - visible = false; + state->visible = false; } } /* Check size restrictions when scaling */ - if (visible && (src_w != crtc_w || src_h != crtc_h)) { + if (state->visible && (src_w != crtc_w || src_h != crtc_h)) { unsigned int width_bytes; WARN_ON(!intel_plane->can_scale); @@ -975,12 +986,13 @@ intel_update_plane(struct drm_plane *plane, struct drm_crtc *crtc, /* FIXME interlacing min height is 6 */ if (crtc_w < 3 || crtc_h < 3) - visible = false; + state->visible = false; if (src_w < 3 || src_h < 3) - visible = false; + state->visible = false; - width_bytes = ((src_x * pixel_size) & 63) + src_w * pixel_size; + width_bytes = ((src_x * pixel_size) & 63) + + src_w * pixel_size; if (src_w > 2048 || src_h > 2048 || width_bytes > 4096 || fb->pitches[0] > 4096) { @@ -989,42 +1001,76 @@ intel_update_plane(struct drm_plane *plane, struct drm_crtc *crtc, } } - dst.x1 = crtc_x; - dst.x2 = crtc_x + crtc_w; - dst.y1 = crtc_y; - dst.y2 = crtc_y + crtc_h; + if (state->visible) { + src->x1 = src_x; + src->x2 = src_x + src_w; + src->y1 = src_y; + src->y2 = src_y + src_h; + } + + dst->x1 = crtc_x; + dst->x2 = crtc_x + crtc_w; + dst->y1 = crtc_y; + dst->y2 = crtc_y + crtc_h; + + return 0; +} + +static int +intel_commit_sprite_plane(struct drm_plane *plane, + struct intel_plane_state *state) +{ + struct drm_device *dev = plane->dev; + struct drm_crtc *crtc = state->crtc; + struct intel_crtc *intel_crtc = to_intel_crtc(crtc); + struct intel_plane *intel_plane = to_intel_plane(plane); + enum pipe pipe = intel_crtc->pipe; + struct drm_framebuffer *fb = state->fb; + struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb); + struct drm_i915_gem_object *obj = intel_fb->obj; + struct drm_i915_gem_object *old_obj = intel_plane->obj; + int crtc_x, crtc_y; + unsigned int crtc_w, crtc_h; + uint32_t src_x, src_y, src_w, src_h; + struct drm_rect *dst = &state->dst; + const struct drm_rect *clip = &state->clip; + bool primary_enabled; + int ret; /* * If the sprite is completely covering the primary plane, * we can disable the primary and save power. */ - primary_enabled = !drm_rect_equals(&dst, &clip) || colorkey_enabled(intel_plane); - WARN_ON(!primary_enabled && !visible && intel_crtc->active); - - mutex_lock(&dev->struct_mutex); + primary_enabled = !drm_rect_equals(dst, clip) || colorkey_enabled(intel_plane); + WARN_ON(!primary_enabled && !state->visible && intel_crtc->active); - /* Note that this will apply the VT-d workaround for scanouts, - * which is more restrictive than required for sprites. (The - * primary plane requires 256KiB alignment with 64 PTE padding, - * the sprite planes only require 128KiB alignment and 32 PTE padding. - */ - ret = intel_pin_and_fence_fb_obj(dev, obj, NULL); - i915_gem_track_fb(old_obj, obj, - INTEL_FRONTBUFFER_SPRITE(pipe)); - mutex_unlock(&dev->struct_mutex); + if (old_obj != obj) { + mutex_lock(&dev->struct_mutex); - if (ret) - return ret; + /* Note that this will apply the VT-d workaround for scanouts, + * which is more restrictive than required for sprites. (The + * primary plane requires 256KiB alignment with 64 PTE padding, + * the sprite planes only require 128KiB alignment and 32 PTE + * padding. + */ + ret = intel_pin_and_fence_fb_obj(dev, obj, NULL); + if (ret == 0) + i915_gem_track_fb(old_obj, obj, + INTEL_FRONTBUFFER_SPRITE(pipe)); + mutex_unlock(&dev->struct_mutex); + if (ret) + return ret; + } - intel_plane->crtc_x = orig.crtc_x; - intel_plane->crtc_y = orig.crtc_y; - intel_plane->crtc_w = orig.crtc_w; - intel_plane->crtc_h = orig.crtc_h; - intel_plane->src_x = orig.src_x; - intel_plane->src_y = orig.src_y; - intel_plane->src_w = orig.src_w; - intel_plane->src_h = orig.src_h; + intel_plane->crtc_x = state->orig_dst.x1; + intel_plane->crtc_y = state->orig_dst.y1; + intel_plane->crtc_w = drm_rect_width(&state->orig_dst); + intel_plane->crtc_h = drm_rect_height(&state->orig_dst); + intel_plane->src_x = state->orig_src.x1; + intel_plane->src_y = state->orig_src.y1; + intel_plane->src_w = drm_rect_width(&state->orig_src); + intel_plane->src_h = drm_rect_height(&state->orig_src); intel_plane->obj = obj; if (intel_crtc->active) { @@ -1038,12 +1084,22 @@ intel_update_plane(struct drm_plane *plane, struct drm_crtc *crtc, if (primary_was_enabled && !primary_enabled) intel_pre_disable_primary(crtc); - if (visible) + if (state->visible) { + crtc_x = state->dst.x1; + crtc_y = state->dst.y1; + crtc_w = drm_rect_width(&state->dst); + crtc_h = drm_rect_height(&state->dst); + src_x = state->src.x1; + src_y = state->src.y1; + src_w = drm_rect_width(&state->src); + src_h = drm_rect_height(&state->src); intel_plane->update_plane(plane, crtc, fb, obj, crtc_x, crtc_y, crtc_w, crtc_h, src_x, src_y, src_w, src_h); - else + } else { intel_plane->disable_plane(plane, crtc); + } + intel_frontbuffer_flip(dev, INTEL_FRONTBUFFER_SPRITE(pipe)); @@ -1052,14 +1108,15 @@ intel_update_plane(struct drm_plane *plane, struct drm_crtc *crtc, } /* Unpin old obj after new one is active to avoid ugliness */ - if (old_obj) { + if (old_obj && old_obj != obj) { + /* * It's fairly common to simply update the position of * an existing object. In that case, we don't need to * wait for vblank to avoid ugliness, we only need to * do the pin & ref bookkeeping. */ - if (old_obj != obj && intel_crtc->active) + if (intel_crtc->active) intel_wait_for_vblank(dev, intel_crtc->pipe); mutex_lock(&dev->struct_mutex); @@ -1071,6 +1128,46 @@ intel_update_plane(struct drm_plane *plane, struct drm_crtc *crtc, } static int +intel_update_plane(struct drm_plane *plane, struct drm_crtc *crtc, + struct drm_framebuffer *fb, int crtc_x, int crtc_y, + unsigned int crtc_w, unsigned int crtc_h, + uint32_t src_x, uint32_t src_y, + uint32_t src_w, uint32_t src_h) +{ + struct intel_plane_state state; + struct intel_crtc *intel_crtc = to_intel_crtc(crtc); + int ret; + + state.crtc = crtc; + state.fb = fb; + + /* sample coordinates in 16.16 fixed point */ + state.src.x1 = src_x; + state.src.x2 = src_x + src_w; + state.src.y1 = src_y; + state.src.y2 = src_y + src_h; + + /* integer pixels */ + state.dst.x1 = crtc_x; + state.dst.x2 = crtc_x + crtc_w; + state.dst.y1 = crtc_y; + state.dst.y2 = crtc_y + crtc_h; + + state.clip.x1 = 0; + state.clip.y1 = 0; + state.clip.x2 = intel_crtc->active ? intel_crtc->config.pipe_src_w : 0; + state.clip.y2 = intel_crtc->active ? intel_crtc->config.pipe_src_h : 0; + state.orig_src = state.src; + state.orig_dst = state.dst; + + ret = intel_check_sprite_plane(plane, &state); + if (ret) + return ret; + + return intel_commit_sprite_plane(plane, &state); +} + +static int intel_disable_plane(struct drm_plane *plane) { struct drm_device *dev = plane->dev; @@ -1180,18 +1277,27 @@ out_unlock: return ret; } -void intel_plane_restore(struct drm_plane *plane) +int intel_plane_set_property(struct drm_plane *plane, + struct drm_property *prop, + uint64_t val) +{ + int ret = -ENOENT; + + return ret; +} + +int intel_plane_restore(struct drm_plane *plane) { struct intel_plane *intel_plane = to_intel_plane(plane); if (!plane->crtc || !plane->fb) - return; + return 0; - intel_update_plane(plane, plane->crtc, plane->fb, - intel_plane->crtc_x, intel_plane->crtc_y, - intel_plane->crtc_w, intel_plane->crtc_h, - intel_plane->src_x, intel_plane->src_y, - intel_plane->src_w, intel_plane->src_h); + return plane->funcs->update_plane(plane, plane->crtc, plane->fb, + intel_plane->crtc_x, intel_plane->crtc_y, + intel_plane->crtc_w, intel_plane->crtc_h, + intel_plane->src_x, intel_plane->src_y, + intel_plane->src_w, intel_plane->src_h); } void intel_plane_disable(struct drm_plane *plane) @@ -1206,6 +1312,7 @@ static const struct drm_plane_funcs intel_plane_funcs = { .update_plane = intel_update_plane, .disable_plane = intel_disable_plane, .destroy = intel_destroy_plane, + .set_property = intel_plane_set_property, }; static uint32_t ilk_plane_formats[] = { @@ -1310,13 +1417,17 @@ intel_plane_init(struct drm_device *dev, enum pipe pipe, int plane) intel_plane->pipe = pipe; intel_plane->plane = plane; + intel_plane->rotation = BIT(DRM_ROTATE_0); possible_crtcs = (1 << pipe); - ret = drm_plane_init(dev, &intel_plane->base, possible_crtcs, - &intel_plane_funcs, - plane_formats, num_plane_formats, - false); - if (ret) + ret = drm_universal_plane_init(dev, &intel_plane->base, possible_crtcs, + &intel_plane_funcs, + plane_formats, num_plane_formats, + DRM_PLANE_TYPE_OVERLAY); + if (ret) { kfree(intel_plane); + goto out; + } + out: return ret; } diff --git a/drivers/gpu/drm/i915/intel_tv.c b/drivers/gpu/drm/i915/intel_tv.c index c14341c..6f5f59b 100644 --- a/drivers/gpu/drm/i915/intel_tv.c +++ b/drivers/gpu/drm/i915/intel_tv.c @@ -1182,18 +1182,17 @@ intel_tv_detect_type(struct intel_tv *intel_tv, struct intel_crtc *intel_crtc = to_intel_crtc(crtc); struct drm_device *dev = encoder->dev; struct drm_i915_private *dev_priv = dev->dev_private; - unsigned long irqflags; u32 tv_ctl, save_tv_ctl; u32 tv_dac, save_tv_dac; int type; /* Disable TV interrupts around load detect or we'll recurse */ if (connector->polled & DRM_CONNECTOR_POLL_HPD) { - spin_lock_irqsave(&dev_priv->irq_lock, irqflags); + spin_lock_irq(&dev_priv->irq_lock); i915_disable_pipestat(dev_priv, 0, PIPE_HOTPLUG_INTERRUPT_STATUS | PIPE_HOTPLUG_TV_INTERRUPT_STATUS); - spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags); + spin_unlock_irq(&dev_priv->irq_lock); } save_tv_dac = tv_dac = I915_READ(TV_DAC); @@ -1266,11 +1265,11 @@ intel_tv_detect_type(struct intel_tv *intel_tv, /* Restore interrupt config */ if (connector->polled & DRM_CONNECTOR_POLL_HPD) { - spin_lock_irqsave(&dev_priv->irq_lock, irqflags); + spin_lock_irq(&dev_priv->irq_lock); i915_enable_pipestat(dev_priv, 0, PIPE_HOTPLUG_INTERRUPT_STATUS | PIPE_HOTPLUG_TV_INTERRUPT_STATUS); - spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags); + spin_unlock_irq(&dev_priv->irq_lock); } return type; diff --git a/drivers/gpu/drm/i915/intel_uncore.c b/drivers/gpu/drm/i915/intel_uncore.c index e81bc3b..0e99852 100644 --- a/drivers/gpu/drm/i915/intel_uncore.c +++ b/drivers/gpu/drm/i915/intel_uncore.c @@ -101,7 +101,7 @@ static void __gen7_gt_force_wake_mt_get(struct drm_i915_private *dev_priv, { u32 forcewake_ack; - if (IS_HASWELL(dev_priv->dev) || IS_GEN8(dev_priv->dev)) + if (IS_HASWELL(dev_priv->dev) || IS_BROADWELL(dev_priv->dev)) forcewake_ack = FORCEWAKE_ACK_HSW; else forcewake_ack = FORCEWAKE_MT_ACK; @@ -194,13 +194,15 @@ static void vlv_force_wake_reset(struct drm_i915_private *dev_priv) static void __vlv_force_wake_get(struct drm_i915_private *dev_priv, int fw_engine) { + /* + * WaRsDontPollForAckOnClearingFWBits:vlv + * Hardware clears ack bits lazily (only when all ack + * bits become 0) so don't poll for individiual ack + * bits to be clear here like on other platforms. + */ + /* Check for Render Engine */ if (FORCEWAKE_RENDER & fw_engine) { - if (wait_for_atomic((__raw_i915_read32(dev_priv, - FORCEWAKE_ACK_VLV) & - FORCEWAKE_KERNEL) == 0, - FORCEWAKE_ACK_TIMEOUT_MS)) - DRM_ERROR("Timed out: Render forcewake old ack to clear.\n"); __raw_i915_write32(dev_priv, FORCEWAKE_VLV, _MASKED_BIT_ENABLE(FORCEWAKE_KERNEL)); @@ -214,11 +216,6 @@ static void __vlv_force_wake_get(struct drm_i915_private *dev_priv, /* Check for Media Engine */ if (FORCEWAKE_MEDIA & fw_engine) { - if (wait_for_atomic((__raw_i915_read32(dev_priv, - FORCEWAKE_ACK_MEDIA_VLV) & - FORCEWAKE_KERNEL) == 0, - FORCEWAKE_ACK_TIMEOUT_MS)) - DRM_ERROR("Timed out: Media forcewake old ack to clear.\n"); __raw_i915_write32(dev_priv, FORCEWAKE_MEDIA_VLV, _MASKED_BIT_ENABLE(FORCEWAKE_KERNEL)); @@ -334,7 +331,7 @@ void intel_uncore_forcewake_reset(struct drm_device *dev, bool restore) else if (IS_GEN6(dev) || IS_GEN7(dev)) __gen6_gt_force_wake_reset(dev_priv); - if (IS_IVYBRIDGE(dev) || IS_HASWELL(dev) || IS_GEN8(dev)) + if (IS_IVYBRIDGE(dev) || IS_HASWELL(dev) || IS_BROADWELL(dev)) __gen7_gt_force_wake_mt_reset(dev_priv); if (restore) { /* If reset with a user forcewake, try to restore */ @@ -838,7 +835,7 @@ void intel_uncore_init(struct drm_device *dev) if (IS_VALLEYVIEW(dev)) { dev_priv->uncore.funcs.force_wake_get = __vlv_force_wake_get; dev_priv->uncore.funcs.force_wake_put = __vlv_force_wake_put; - } else if (IS_HASWELL(dev) || IS_GEN8(dev)) { + } else if (IS_HASWELL(dev) || IS_BROADWELL(dev)) { dev_priv->uncore.funcs.force_wake_get = __gen7_gt_force_wake_mt_get; dev_priv->uncore.funcs.force_wake_put = __gen7_gt_force_wake_mt_put; } else if (IS_IVYBRIDGE(dev)) { -- 1.9.3