diff options
Diffstat (limited to 'drivers/gpu/drm/i915')
264 files changed, 17579 insertions, 10465 deletions
diff --git a/drivers/gpu/drm/i915/Kconfig b/drivers/gpu/drm/i915/Kconfig index 107762427648..84b6fc70cbf5 100644 --- a/drivers/gpu/drm/i915/Kconfig +++ b/drivers/gpu/drm/i915/Kconfig @@ -132,6 +132,17 @@ config DRM_I915_GVT_KVMGT Choose this option if you want to enable KVMGT support for Intel GVT-g. +config DRM_I915_PXP + bool "Enable Intel PXP support" + depends on DRM_I915 + depends on INTEL_MEI && INTEL_MEI_PXP + default n + help + PXP (Protected Xe Path) is an i915 component, available on graphics + version 12 and newer GPUs, that helps to establish the hardware + protected session and manage the status of the alive software session, + as well as its life cycle. + menu "drm/i915 Debugging" depends on DRM_I915 depends on EXPERT diff --git a/drivers/gpu/drm/i915/Makefile b/drivers/gpu/drm/i915/Makefile index 335ba9f43d8f..660bb03de6fc 100644 --- a/drivers/gpu/drm/i915/Makefile +++ b/drivers/gpu/drm/i915/Makefile @@ -13,13 +13,11 @@ # will most likely get a sudden build breakage... Hopefully we will fix # new warnings before CI updates! subdir-ccflags-y := -Wall -Wextra -subdir-ccflags-y += $(call cc-disable-warning, unused-parameter) -subdir-ccflags-y += $(call cc-disable-warning, type-limits) -subdir-ccflags-y += $(call cc-disable-warning, missing-field-initializers) +subdir-ccflags-y += -Wno-unused-parameter +subdir-ccflags-y += -Wno-type-limits +subdir-ccflags-y += -Wno-missing-field-initializers +subdir-ccflags-y += -Wno-sign-compare subdir-ccflags-y += $(call cc-disable-warning, unused-but-set-variable) -# clang warnings -subdir-ccflags-y += $(call cc-disable-warning, sign-compare) -subdir-ccflags-y += $(call cc-disable-warning, initializer-overrides) subdir-ccflags-y += $(call cc-disable-warning, frame-address) subdir-ccflags-$(CONFIG_DRM_I915_WERROR) += -Werror @@ -49,13 +47,15 @@ i915-y += i915_drv.o \ intel_dram.o \ intel_memory_region.o \ intel_pch.o \ + intel_pcode.o \ intel_pm.o \ intel_region_ttm.o \ intel_runtime_pm.o \ - intel_sideband.o \ + intel_sbi.o \ intel_step.o \ intel_uncore.o \ intel_wakeref.o \ + vlv_sideband.o \ vlv_suspend.o # core library code @@ -78,9 +78,6 @@ i915-$(CONFIG_PERF_EVENTS) += i915_pmu.o # "Graphics Technology" (aka we talk to the gpu) gt-y += \ - gt/debugfs_engines.o \ - gt/debugfs_gt.o \ - gt/debugfs_gt_pm.o \ gt/gen2_engine_cs.o \ gt/gen6_engine_cs.o \ gt/gen6_ppgtt.o \ @@ -100,8 +97,11 @@ gt-y += \ gt/intel_gt.o \ gt/intel_gt_buffer_pool.o \ gt/intel_gt_clock_utils.o \ + gt/intel_gt_debugfs.o \ + gt/intel_gt_engines_debugfs.o \ gt/intel_gt_irq.o \ gt/intel_gt_pm.o \ + gt/intel_gt_pm_debugfs.o \ gt/intel_gt_pm_irq.o \ gt/intel_gt_requests.o \ gt/intel_gtt.o \ @@ -154,6 +154,7 @@ gem-y += \ gem/i915_gem_throttle.o \ gem/i915_gem_tiling.o \ gem/i915_gem_ttm.o \ + gem/i915_gem_ttm_pm.o \ gem/i915_gem_userptr.o \ gem/i915_gem_wait.o \ gem/i915_gemfs.o @@ -211,8 +212,11 @@ i915-y += \ display/intel_dpio_phy.o \ display/intel_dpll.o \ display/intel_dpll_mgr.o \ + display/intel_dpt.o \ + display/intel_drrs.o \ display/intel_dsb.o \ display/intel_fb.o \ + display/intel_fb_pin.o \ display/intel_fbc.o \ display/intel_fdi.o \ display/intel_fifo_underrun.o \ @@ -222,6 +226,7 @@ i915-y += \ display/intel_hotplug.o \ display/intel_lpe_audio.o \ display/intel_overlay.o \ + display/intel_plane_initial.o \ display/intel_psr.o \ display/intel_quirks.o \ display/intel_sprite.o \ @@ -247,6 +252,7 @@ i915-y += \ display/g4x_dp.o \ display/g4x_hdmi.o \ display/icl_dsi.o \ + display/intel_backlight.o \ display/intel_crt.o \ display/intel_ddi.o \ display/intel_ddi_buf_trans.o \ @@ -277,6 +283,16 @@ i915-y += \ i915-y += i915_perf.o +# Protected execution platform (PXP) support +i915-$(CONFIG_DRM_I915_PXP) += \ + pxp/intel_pxp.o \ + pxp/intel_pxp_cmd.o \ + pxp/intel_pxp_debugfs.o \ + pxp/intel_pxp_irq.o \ + pxp/intel_pxp_pm.o \ + pxp/intel_pxp_session.o \ + pxp/intel_pxp_tee.o + # Post-mortem debug and GPU hang state capture i915-$(CONFIG_DRM_I915_CAPTURE_ERROR) += i915_gpu_error.o i915-$(CONFIG_DRM_I915_SELFTEST) += \ diff --git a/drivers/gpu/drm/i915/display/g4x_dp.c b/drivers/gpu/drm/i915/display/g4x_dp.c index de0f358184aa..dc41868d01ef 100644 --- a/drivers/gpu/drm/i915/display/g4x_dp.c +++ b/drivers/gpu/drm/i915/display/g4x_dp.c @@ -7,6 +7,7 @@ #include "g4x_dp.h" #include "intel_audio.h" +#include "intel_backlight.h" #include "intel_connector.h" #include "intel_de.h" #include "intel_display_types.h" @@ -16,9 +17,8 @@ #include "intel_fifo_underrun.h" #include "intel_hdmi.h" #include "intel_hotplug.h" -#include "intel_panel.h" #include "intel_pps.h" -#include "intel_sideband.h" +#include "vlv_sideband.h" struct dp_link_dpll { int clock; @@ -211,7 +211,7 @@ static void ilk_edp_pll_on(struct intel_dp *intel_dp, struct intel_crtc *crtc = to_intel_crtc(pipe_config->uapi.crtc); struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); - assert_pipe_disabled(dev_priv, pipe_config->cpu_transcoder); + assert_transcoder_disabled(dev_priv, pipe_config->cpu_transcoder); assert_dp_port_disabled(intel_dp); assert_edp_pll_disabled(dev_priv); @@ -251,7 +251,7 @@ static void ilk_edp_pll_off(struct intel_dp *intel_dp, struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->uapi.crtc); struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); - assert_pipe_disabled(dev_priv, old_crtc_state->cpu_transcoder); + assert_transcoder_disabled(dev_priv, old_crtc_state->cpu_transcoder); assert_dp_port_disabled(intel_dp); assert_edp_pll_enabled(dev_priv); @@ -426,7 +426,6 @@ intel_dp_link_down(struct intel_encoder *encoder, struct intel_dp *intel_dp = enc_to_intel_dp(encoder); struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->uapi.crtc); enum port port = encoder->port; - u32 DP = intel_dp->DP; if (drm_WARN_ON(&dev_priv->drm, (intel_de_read(dev_priv, intel_dp->output_reg) & @@ -437,17 +436,17 @@ intel_dp_link_down(struct intel_encoder *encoder, if ((IS_IVYBRIDGE(dev_priv) && port == PORT_A) || (HAS_PCH_CPT(dev_priv) && port != PORT_A)) { - DP &= ~DP_LINK_TRAIN_MASK_CPT; - DP |= DP_LINK_TRAIN_PAT_IDLE_CPT; + intel_dp->DP &= ~DP_LINK_TRAIN_MASK_CPT; + intel_dp->DP |= DP_LINK_TRAIN_PAT_IDLE_CPT; } else { - DP &= ~DP_LINK_TRAIN_MASK; - DP |= DP_LINK_TRAIN_PAT_IDLE; + intel_dp->DP &= ~DP_LINK_TRAIN_MASK; + intel_dp->DP |= DP_LINK_TRAIN_PAT_IDLE; } - intel_de_write(dev_priv, intel_dp->output_reg, DP); + intel_de_write(dev_priv, intel_dp->output_reg, intel_dp->DP); intel_de_posting_read(dev_priv, intel_dp->output_reg); - DP &= ~(DP_PORT_EN | DP_AUDIO_OUTPUT_ENABLE); - intel_de_write(dev_priv, intel_dp->output_reg, DP); + intel_dp->DP &= ~(DP_PORT_EN | DP_AUDIO_OUTPUT_ENABLE); + intel_de_write(dev_priv, intel_dp->output_reg, intel_dp->DP); intel_de_posting_read(dev_priv, intel_dp->output_reg); /* @@ -464,14 +463,14 @@ intel_dp_link_down(struct intel_encoder *encoder, intel_set_pch_fifo_underrun_reporting(dev_priv, PIPE_A, false); /* always enable with pattern 1 (as per spec) */ - DP &= ~(DP_PIPE_SEL_MASK | DP_LINK_TRAIN_MASK); - DP |= DP_PORT_EN | DP_PIPE_SEL(PIPE_A) | + intel_dp->DP &= ~(DP_PIPE_SEL_MASK | DP_LINK_TRAIN_MASK); + intel_dp->DP |= DP_PORT_EN | DP_PIPE_SEL(PIPE_A) | DP_LINK_TRAIN_PAT_1; - intel_de_write(dev_priv, intel_dp->output_reg, DP); + intel_de_write(dev_priv, intel_dp->output_reg, intel_dp->DP); intel_de_posting_read(dev_priv, intel_dp->output_reg); - DP &= ~DP_PORT_EN; - intel_de_write(dev_priv, intel_dp->output_reg, DP); + intel_dp->DP &= ~DP_PORT_EN; + intel_de_write(dev_priv, intel_dp->output_reg, intel_dp->DP); intel_de_posting_read(dev_priv, intel_dp->output_reg); intel_wait_for_vblank_if_active(dev_priv, PIPE_A); @@ -481,8 +480,6 @@ intel_dp_link_down(struct intel_encoder *encoder, msleep(intel_dp->pps.panel_power_down_delay); - intel_dp->DP = DP; - if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) { intel_wakeref_t wakeref; @@ -582,19 +579,18 @@ cpt_set_link_train(struct intel_dp *intel_dp, u8 dp_train_pat) { struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); - u32 *DP = &intel_dp->DP; - *DP &= ~DP_LINK_TRAIN_MASK_CPT; + intel_dp->DP &= ~DP_LINK_TRAIN_MASK_CPT; switch (intel_dp_training_pattern_symbol(dp_train_pat)) { case DP_TRAINING_PATTERN_DISABLE: - *DP |= DP_LINK_TRAIN_OFF_CPT; + intel_dp->DP |= DP_LINK_TRAIN_OFF_CPT; break; case DP_TRAINING_PATTERN_1: - *DP |= DP_LINK_TRAIN_PAT_1_CPT; + intel_dp->DP |= DP_LINK_TRAIN_PAT_1_CPT; break; case DP_TRAINING_PATTERN_2: - *DP |= DP_LINK_TRAIN_PAT_2_CPT; + intel_dp->DP |= DP_LINK_TRAIN_PAT_2_CPT; break; default: MISSING_CASE(intel_dp_training_pattern_symbol(dp_train_pat)); @@ -611,19 +607,18 @@ g4x_set_link_train(struct intel_dp *intel_dp, u8 dp_train_pat) { struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); - u32 *DP = &intel_dp->DP; - *DP &= ~DP_LINK_TRAIN_MASK; + intel_dp->DP &= ~DP_LINK_TRAIN_MASK; switch (intel_dp_training_pattern_symbol(dp_train_pat)) { case DP_TRAINING_PATTERN_DISABLE: - *DP |= DP_LINK_TRAIN_OFF; + intel_dp->DP |= DP_LINK_TRAIN_OFF; break; case DP_TRAINING_PATTERN_1: - *DP |= DP_LINK_TRAIN_PAT_1; + intel_dp->DP |= DP_LINK_TRAIN_PAT_1; break; case DP_TRAINING_PATTERN_2: - *DP |= DP_LINK_TRAIN_PAT_2; + intel_dp->DP |= DP_LINK_TRAIN_PAT_2; break; default: MISSING_CASE(intel_dp_training_pattern_symbol(dp_train_pat)); @@ -642,7 +637,7 @@ static void intel_dp_enable_port(struct intel_dp *intel_dp, /* enable with pattern 1 (as per spec) */ intel_dp_program_link_training_pattern(intel_dp, crtc_state, - DP_TRAINING_PATTERN_1); + DP_PHY_DPRX, DP_TRAINING_PATTERN_1); /* * Magic for VLV/CHV. We _must_ first set up the register @@ -813,10 +808,10 @@ static u8 intel_dp_preemph_max_3(struct intel_dp *intel_dp) return DP_TRAIN_PRE_EMPH_LEVEL_3; } -static void vlv_set_signal_levels(struct intel_dp *intel_dp, +static void vlv_set_signal_levels(struct intel_encoder *encoder, const struct intel_crtc_state *crtc_state) { - struct intel_encoder *encoder = &dp_to_dig_port(intel_dp)->base; + struct intel_dp *intel_dp = enc_to_intel_dp(encoder); unsigned long demph_reg_value, preemph_reg_value, uniqtranscale_reg_value; u8 train_set = intel_dp->train_set[0]; @@ -899,10 +894,10 @@ static void vlv_set_signal_levels(struct intel_dp *intel_dp, uniqtranscale_reg_value, 0); } -static void chv_set_signal_levels(struct intel_dp *intel_dp, +static void chv_set_signal_levels(struct intel_encoder *encoder, const struct intel_crtc_state *crtc_state) { - struct intel_encoder *encoder = &dp_to_dig_port(intel_dp)->base; + struct intel_dp *intel_dp = enc_to_intel_dp(encoder); u32 deemph_reg_value, margin_reg_value; bool uniq_trans_scale = false; u8 train_set = intel_dp->train_set[0]; @@ -1020,10 +1015,11 @@ static u32 g4x_signal_levels(u8 train_set) } static void -g4x_set_signal_levels(struct intel_dp *intel_dp, +g4x_set_signal_levels(struct intel_encoder *encoder, const struct intel_crtc_state *crtc_state) { - struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); + struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); + struct intel_dp *intel_dp = enc_to_intel_dp(encoder); u8 train_set = intel_dp->train_set[0]; u32 signal_levels; @@ -1067,10 +1063,11 @@ static u32 snb_cpu_edp_signal_levels(u8 train_set) } static void -snb_cpu_edp_set_signal_levels(struct intel_dp *intel_dp, +snb_cpu_edp_set_signal_levels(struct intel_encoder *encoder, const struct intel_crtc_state *crtc_state) { - struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); + struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); + struct intel_dp *intel_dp = enc_to_intel_dp(encoder); u8 train_set = intel_dp->train_set[0]; u32 signal_levels; @@ -1118,10 +1115,11 @@ static u32 ivb_cpu_edp_signal_levels(u8 train_set) } static void -ivb_cpu_edp_set_signal_levels(struct intel_dp *intel_dp, +ivb_cpu_edp_set_signal_levels(struct intel_encoder *encoder, const struct intel_crtc_state *crtc_state) { - struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); + struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); + struct intel_dp *intel_dp = enc_to_intel_dp(encoder); u8 train_set = intel_dp->train_set[0]; u32 signal_levels; @@ -1334,7 +1332,7 @@ bool g4x_dp_init(struct drm_i915_private *dev_priv, intel_encoder->get_config = intel_dp_get_config; intel_encoder->sync_state = intel_dp_sync_state; intel_encoder->initial_fastset_check = intel_dp_initial_fastset_check; - intel_encoder->update_pipe = intel_panel_update_backlight; + intel_encoder->update_pipe = intel_backlight_update; intel_encoder->suspend = intel_dp_encoder_suspend; intel_encoder->shutdown = intel_dp_encoder_shutdown; if (IS_CHERRYVIEW(dev_priv)) { @@ -1364,15 +1362,15 @@ bool g4x_dp_init(struct drm_i915_private *dev_priv, dig_port->dp.set_link_train = g4x_set_link_train; if (IS_CHERRYVIEW(dev_priv)) - dig_port->dp.set_signal_levels = chv_set_signal_levels; + intel_encoder->set_signal_levels = chv_set_signal_levels; else if (IS_VALLEYVIEW(dev_priv)) - dig_port->dp.set_signal_levels = vlv_set_signal_levels; + intel_encoder->set_signal_levels = vlv_set_signal_levels; else if (IS_IVYBRIDGE(dev_priv) && port == PORT_A) - dig_port->dp.set_signal_levels = ivb_cpu_edp_set_signal_levels; + intel_encoder->set_signal_levels = ivb_cpu_edp_set_signal_levels; else if (IS_SANDYBRIDGE(dev_priv) && port == PORT_A) - dig_port->dp.set_signal_levels = snb_cpu_edp_set_signal_levels; + intel_encoder->set_signal_levels = snb_cpu_edp_set_signal_levels; else - dig_port->dp.set_signal_levels = g4x_set_signal_levels; + intel_encoder->set_signal_levels = g4x_set_signal_levels; if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv) || (HAS_PCH_SPLIT(dev_priv) && port != PORT_A)) { diff --git a/drivers/gpu/drm/i915/display/g4x_hdmi.c b/drivers/gpu/drm/i915/display/g4x_hdmi.c index be352e9f0afc..88c427f3c346 100644 --- a/drivers/gpu/drm/i915/display/g4x_hdmi.c +++ b/drivers/gpu/drm/i915/display/g4x_hdmi.c @@ -14,8 +14,8 @@ #include "intel_fifo_underrun.h" #include "intel_hdmi.h" #include "intel_hotplug.h" -#include "intel_sideband.h" #include "intel_sdvo.h" +#include "vlv_sideband.h" static void intel_hdmi_prepare(struct intel_encoder *encoder, const struct intel_crtc_state *crtc_state) diff --git a/drivers/gpu/drm/i915/display/icl_dsi.c b/drivers/gpu/drm/i915/display/icl_dsi.c index a3eae3f3eadc..168c84a74d30 100644 --- a/drivers/gpu/drm/i915/display/icl_dsi.c +++ b/drivers/gpu/drm/i915/display/icl_dsi.c @@ -29,6 +29,7 @@ #include <drm/drm_mipi_dsi.h> #include "intel_atomic.h" +#include "intel_backlight.h" #include "intel_combo_phy.h" #include "intel_connector.h" #include "intel_crtc.h" @@ -54,20 +55,28 @@ static int payload_credits_available(struct drm_i915_private *dev_priv, >> FREE_PLOAD_CREDIT_SHIFT; } -static void wait_for_header_credits(struct drm_i915_private *dev_priv, - enum transcoder dsi_trans) +static bool wait_for_header_credits(struct drm_i915_private *dev_priv, + enum transcoder dsi_trans, int hdr_credit) { if (wait_for_us(header_credits_available(dev_priv, dsi_trans) >= - MAX_HEADER_CREDIT, 100)) + hdr_credit, 100)) { drm_err(&dev_priv->drm, "DSI header credits not released\n"); + return false; + } + + return true; } -static void wait_for_payload_credits(struct drm_i915_private *dev_priv, - enum transcoder dsi_trans) +static bool wait_for_payload_credits(struct drm_i915_private *dev_priv, + enum transcoder dsi_trans, int payld_credit) { if (wait_for_us(payload_credits_available(dev_priv, dsi_trans) >= - MAX_PLOAD_CREDIT, 100)) + payld_credit, 100)) { drm_err(&dev_priv->drm, "DSI payload credits not released\n"); + return false; + } + + return true; } static enum transcoder dsi_port_to_transcoder(enum port port) @@ -90,8 +99,8 @@ static void wait_for_cmds_dispatched_to_panel(struct intel_encoder *encoder) /* wait for header/payload credits to be released */ for_each_dsi_port(port, intel_dsi->ports) { dsi_trans = dsi_port_to_transcoder(port); - wait_for_header_credits(dev_priv, dsi_trans); - wait_for_payload_credits(dev_priv, dsi_trans); + wait_for_header_credits(dev_priv, dsi_trans, MAX_HEADER_CREDIT); + wait_for_payload_credits(dev_priv, dsi_trans, MAX_PLOAD_CREDIT); } /* send nop DCS command */ @@ -108,7 +117,7 @@ static void wait_for_cmds_dispatched_to_panel(struct intel_encoder *encoder) /* wait for header credits to be released */ for_each_dsi_port(port, intel_dsi->ports) { dsi_trans = dsi_port_to_transcoder(port); - wait_for_header_credits(dev_priv, dsi_trans); + wait_for_header_credits(dev_priv, dsi_trans, MAX_HEADER_CREDIT); } /* wait for LP TX in progress bit to be cleared */ @@ -120,54 +129,52 @@ static void wait_for_cmds_dispatched_to_panel(struct intel_encoder *encoder) } } -static bool add_payld_to_queue(struct intel_dsi_host *host, const u8 *data, - u32 len) +static int dsi_send_pkt_payld(struct intel_dsi_host *host, + const struct mipi_dsi_packet *packet) { struct intel_dsi *intel_dsi = host->intel_dsi; - struct drm_i915_private *dev_priv = to_i915(intel_dsi->base.base.dev); + struct drm_i915_private *i915 = to_i915(intel_dsi->base.base.dev); enum transcoder dsi_trans = dsi_port_to_transcoder(host->port); - int free_credits; + const u8 *data = packet->payload; + u32 len = packet->payload_length; int i, j; + /* payload queue can accept *256 bytes*, check limit */ + if (len > MAX_PLOAD_CREDIT * 4) { + drm_err(&i915->drm, "payload size exceeds max queue limit\n"); + return -EINVAL; + } + for (i = 0; i < len; i += 4) { u32 tmp = 0; - free_credits = payload_credits_available(dev_priv, dsi_trans); - if (free_credits < 1) { - drm_err(&dev_priv->drm, - "Payload credit not available\n"); - return false; - } + if (!wait_for_payload_credits(i915, dsi_trans, 1)) + return -EBUSY; for (j = 0; j < min_t(u32, len - i, 4); j++) tmp |= *data++ << 8 * j; - intel_de_write(dev_priv, DSI_CMD_TXPYLD(dsi_trans), tmp); + intel_de_write(i915, DSI_CMD_TXPYLD(dsi_trans), tmp); } - return true; + return 0; } static int dsi_send_pkt_hdr(struct intel_dsi_host *host, - struct mipi_dsi_packet pkt, bool enable_lpdt) + const struct mipi_dsi_packet *packet, + bool enable_lpdt) { struct intel_dsi *intel_dsi = host->intel_dsi; struct drm_i915_private *dev_priv = to_i915(intel_dsi->base.base.dev); enum transcoder dsi_trans = dsi_port_to_transcoder(host->port); u32 tmp; - int free_credits; - /* check if header credit available */ - free_credits = header_credits_available(dev_priv, dsi_trans); - if (free_credits < 1) { - drm_err(&dev_priv->drm, - "send pkt header failed, not enough hdr credits\n"); - return -1; - } + if (!wait_for_header_credits(dev_priv, dsi_trans, 1)) + return -EBUSY; tmp = intel_de_read(dev_priv, DSI_CMD_TXHDR(dsi_trans)); - if (pkt.payload) + if (packet->payload) tmp |= PAYLOAD_PRESENT; else tmp &= ~PAYLOAD_PRESENT; @@ -178,37 +185,15 @@ static int dsi_send_pkt_hdr(struct intel_dsi_host *host, tmp |= LP_DATA_TRANSFER; tmp &= ~(PARAM_WC_MASK | VC_MASK | DT_MASK); - tmp |= ((pkt.header[0] & VC_MASK) << VC_SHIFT); - tmp |= ((pkt.header[0] & DT_MASK) << DT_SHIFT); - tmp |= (pkt.header[1] << PARAM_WC_LOWER_SHIFT); - tmp |= (pkt.header[2] << PARAM_WC_UPPER_SHIFT); + tmp |= ((packet->header[0] & VC_MASK) << VC_SHIFT); + tmp |= ((packet->header[0] & DT_MASK) << DT_SHIFT); + tmp |= (packet->header[1] << PARAM_WC_LOWER_SHIFT); + tmp |= (packet->header[2] << PARAM_WC_UPPER_SHIFT); intel_de_write(dev_priv, DSI_CMD_TXHDR(dsi_trans), tmp); return 0; } -static int dsi_send_pkt_payld(struct intel_dsi_host *host, - struct mipi_dsi_packet pkt) -{ - struct intel_dsi *intel_dsi = host->intel_dsi; - struct drm_i915_private *i915 = to_i915(intel_dsi->base.base.dev); - - /* payload queue can accept *256 bytes*, check limit */ - if (pkt.payload_length > MAX_PLOAD_CREDIT * 4) { - drm_err(&i915->drm, "payload size exceeds max queue limit\n"); - return -1; - } - - /* load data into command payload queue */ - if (!add_payld_to_queue(host, pkt.payload, - pkt.payload_length)) { - drm_err(&i915->drm, "adding payload to queue failed\n"); - return -1; - } - - return 0; -} - void icl_dsi_frame_update(struct intel_crtc_state *crtc_state) { struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); @@ -248,7 +233,7 @@ static void dsi_program_swing_and_deemphasis(struct intel_encoder *encoder) * Program voltage swing and pre-emphasis level values as per * table in BSPEC under DDI buffer programing */ - tmp = intel_de_read(dev_priv, ICL_PORT_TX_DW5_LN0(phy)); + tmp = intel_de_read(dev_priv, ICL_PORT_TX_DW5_LN(0, phy)); tmp &= ~(SCALING_MODE_SEL_MASK | RTERM_SELECT_MASK); tmp |= SCALING_MODE_SEL(0x2); tmp |= TAP2_DISABLE | TAP3_DISABLE; @@ -262,7 +247,7 @@ static void dsi_program_swing_and_deemphasis(struct intel_encoder *encoder) tmp |= RTERM_SELECT(0x6); intel_de_write(dev_priv, ICL_PORT_TX_DW5_AUX(phy), tmp); - tmp = intel_de_read(dev_priv, ICL_PORT_TX_DW2_LN0(phy)); + tmp = intel_de_read(dev_priv, ICL_PORT_TX_DW2_LN(0, phy)); tmp &= ~(SWING_SEL_LOWER_MASK | SWING_SEL_UPPER_MASK | RCOMP_SCALAR_MASK); tmp |= SWING_SEL_UPPER(0x2); @@ -470,7 +455,7 @@ static void gen11_dsi_config_phy_lanes_sequence(struct intel_encoder *encoder) tmp &= ~FRC_LATENCY_OPTIM_MASK; tmp |= FRC_LATENCY_OPTIM_VAL(0x5); intel_de_write(dev_priv, ICL_PORT_TX_DW2_AUX(phy), tmp); - tmp = intel_de_read(dev_priv, ICL_PORT_TX_DW2_LN0(phy)); + tmp = intel_de_read(dev_priv, ICL_PORT_TX_DW2_LN(0, phy)); tmp &= ~FRC_LATENCY_OPTIM_MASK; tmp |= FRC_LATENCY_OPTIM_VAL(0x5); intel_de_write(dev_priv, ICL_PORT_TX_DW2_GRP(phy), tmp); @@ -485,7 +470,7 @@ static void gen11_dsi_config_phy_lanes_sequence(struct intel_encoder *encoder) tmp); tmp = intel_de_read(dev_priv, - ICL_PORT_PCS_DW1_LN0(phy)); + ICL_PORT_PCS_DW1_LN(0, phy)); tmp &= ~LATENCY_OPTIM_MASK; tmp |= LATENCY_OPTIM_VAL(0x1); intel_de_write(dev_priv, ICL_PORT_PCS_DW1_GRP(phy), @@ -504,7 +489,7 @@ static void gen11_dsi_voltage_swing_program_seq(struct intel_encoder *encoder) /* clear common keeper enable bit */ for_each_dsi_phy(phy, intel_dsi->phys) { - tmp = intel_de_read(dev_priv, ICL_PORT_PCS_DW1_LN0(phy)); + tmp = intel_de_read(dev_priv, ICL_PORT_PCS_DW1_LN(0, phy)); tmp &= ~COMMON_KEEPER_EN; intel_de_write(dev_priv, ICL_PORT_PCS_DW1_GRP(phy), tmp); tmp = intel_de_read(dev_priv, ICL_PORT_PCS_DW1_AUX(phy)); @@ -525,7 +510,7 @@ static void gen11_dsi_voltage_swing_program_seq(struct intel_encoder *encoder) /* Clear training enable to change swing values */ for_each_dsi_phy(phy, intel_dsi->phys) { - tmp = intel_de_read(dev_priv, ICL_PORT_TX_DW5_LN0(phy)); + tmp = intel_de_read(dev_priv, ICL_PORT_TX_DW5_LN(0, phy)); tmp &= ~TX_TRAINING_EN; intel_de_write(dev_priv, ICL_PORT_TX_DW5_GRP(phy), tmp); tmp = intel_de_read(dev_priv, ICL_PORT_TX_DW5_AUX(phy)); @@ -538,7 +523,7 @@ static void gen11_dsi_voltage_swing_program_seq(struct intel_encoder *encoder) /* Set training enable to trigger update */ for_each_dsi_phy(phy, intel_dsi->phys) { - tmp = intel_de_read(dev_priv, ICL_PORT_TX_DW5_LN0(phy)); + tmp = intel_de_read(dev_priv, ICL_PORT_TX_DW5_LN(0, phy)); tmp |= TX_TRAINING_EN; intel_de_write(dev_priv, ICL_PORT_TX_DW5_GRP(phy), tmp); tmp = intel_de_read(dev_priv, ICL_PORT_TX_DW5_AUX(phy)); @@ -1270,6 +1255,26 @@ static void icl_apply_kvmr_pipe_a_wa(struct intel_encoder *encoder, IGNORE_KVMR_PIPE_A, enable ? IGNORE_KVMR_PIPE_A : 0); } + +/* + * Wa_16012360555:adl-p + * SW will have to program the "LP to HS Wakeup Guardband" + * to account for the repeaters on the HS Request/Ready + * PPI signaling between the Display engine and the DPHY. + */ +static void adlp_set_lp_hs_wakeup_gb(struct intel_encoder *encoder) +{ + struct drm_i915_private *i915 = to_i915(encoder->base.dev); + struct intel_dsi *intel_dsi = enc_to_intel_dsi(encoder); + enum port port; + + if (DISPLAY_VER(i915) == 13) { + for_each_dsi_port(port, intel_dsi->ports) + intel_de_rmw(i915, TGL_DSI_CHKN_REG(port), + TGL_DSI_CHKN_LSHS_GB, 0x4); + } +} + static void gen11_dsi_enable(struct intel_atomic_state *state, struct intel_encoder *encoder, const struct intel_crtc_state *crtc_state, @@ -1283,11 +1288,14 @@ static void gen11_dsi_enable(struct intel_atomic_state *state, /* Wa_1409054076:icl,jsl,ehl */ icl_apply_kvmr_pipe_a_wa(encoder, crtc->pipe, true); + /* Wa_16012360555:adl-p */ + adlp_set_lp_hs_wakeup_gb(encoder); + /* step6d: enable dsi transcoder */ gen11_dsi_enable_transcoder(encoder); /* step7: enable backlight */ - intel_panel_enable_backlight(crtc_state, conn_state); + intel_backlight_enable(crtc_state, conn_state); intel_dsi_vbt_exec_sequence(intel_dsi, MIPI_SEQ_BACKLIGHT_ON); intel_crtc_vblank_on(crtc_state); @@ -1440,7 +1448,7 @@ static void gen11_dsi_disable(struct intel_atomic_state *state, /* step1: turn off backlight */ intel_dsi_vbt_exec_sequence(intel_dsi, MIPI_SEQ_BACKLIGHT_OFF); - intel_panel_disable_backlight(old_conn_state); + intel_backlight_disable(old_conn_state); /* step2d,e: disable transcoder and wait */ gen11_dsi_disable_transcoder(encoder); @@ -1650,16 +1658,17 @@ static int gen11_dsi_compute_config(struct intel_encoder *encoder, struct intel_dsi *intel_dsi = container_of(encoder, struct intel_dsi, base); struct intel_connector *intel_connector = intel_dsi->attached_connector; - const struct drm_display_mode *fixed_mode = - intel_connector->panel.fixed_mode; struct drm_display_mode *adjusted_mode = &pipe_config->hw.adjusted_mode; int ret; pipe_config->output_format = INTEL_OUTPUT_FORMAT_RGB; - intel_fixed_panel_mode(fixed_mode, adjusted_mode); - ret = intel_pch_panel_fitting(pipe_config, conn_state); + ret = intel_panel_compute_config(intel_connector, adjusted_mode); + if (ret) + return ret; + + ret = intel_panel_fitting(pipe_config, conn_state); if (ret) return ret; @@ -1815,18 +1824,18 @@ static ssize_t gen11_dsi_host_transfer(struct mipi_dsi_host *host, if (msg->flags & MIPI_DSI_MSG_USE_LPM) enable_lpdt = true; - /* send packet header */ - ret = dsi_send_pkt_hdr(intel_dsi_host, dsi_pkt, enable_lpdt); - if (ret < 0) - return ret; - /* only long packet contains payload */ if (mipi_dsi_packet_format_is_long(msg->type)) { - ret = dsi_send_pkt_payld(intel_dsi_host, dsi_pkt); + ret = dsi_send_pkt_payld(intel_dsi_host, &dsi_pkt); if (ret < 0) return ret; } + /* send packet header */ + ret = dsi_send_pkt_hdr(intel_dsi_host, &dsi_pkt, enable_lpdt); + if (ret < 0) + return ret; + //TODO: add payload receive code if needed ret = sizeof(dsi_pkt.header) + dsi_pkt.payload_length; @@ -2014,7 +2023,7 @@ void icl_dsi_init(struct drm_i915_private *dev_priv) encoder->port = port; encoder->get_config = gen11_dsi_get_config; encoder->sync_state = gen11_dsi_sync_state; - encoder->update_pipe = intel_panel_update_backlight; + encoder->update_pipe = intel_backlight_update; encoder->compute_config = gen11_dsi_compute_config; encoder->get_hw_state = gen11_dsi_get_hw_state; encoder->initial_fastset_check = gen11_dsi_initial_fastset_check; @@ -2048,7 +2057,7 @@ void icl_dsi_init(struct drm_i915_private *dev_priv) } intel_panel_init(&intel_connector->panel, fixed_mode, NULL); - intel_panel_setup_backlight(connector, INVALID_PIPE); + intel_backlight_setup(intel_connector, INVALID_PIPE); if (dev_priv->vbt.dsi.config->dual_link) intel_dsi->ports = BIT(PORT_A) | BIT(PORT_B); diff --git a/drivers/gpu/drm/i915/display/intel_acpi.c b/drivers/gpu/drm/i915/display/intel_acpi.c index 68abeaf2d7d4..e78430001f07 100644 --- a/drivers/gpu/drm/i915/display/intel_acpi.c +++ b/drivers/gpu/drm/i915/display/intel_acpi.c @@ -285,3 +285,49 @@ void intel_acpi_device_id_update(struct drm_i915_private *dev_priv) } drm_connector_list_iter_end(&conn_iter); } + +/* NOTE: The connector order must be final before this is called. */ +void intel_acpi_assign_connector_fwnodes(struct drm_i915_private *i915) +{ + struct drm_connector_list_iter conn_iter; + struct drm_device *drm_dev = &i915->drm; + struct fwnode_handle *fwnode = NULL; + struct drm_connector *connector; + struct acpi_device *adev; + + drm_connector_list_iter_begin(drm_dev, &conn_iter); + drm_for_each_connector_iter(connector, &conn_iter) { + /* Always getting the next, even when the last was not used. */ + fwnode = device_get_next_child_node(drm_dev->dev, fwnode); + if (!fwnode) + break; + + switch (connector->connector_type) { + case DRM_MODE_CONNECTOR_LVDS: + case DRM_MODE_CONNECTOR_eDP: + case DRM_MODE_CONNECTOR_DSI: + /* + * Integrated displays have a specific address 0x1f on + * most Intel platforms, but not on all of them. + */ + adev = acpi_find_child_device(ACPI_COMPANION(drm_dev->dev), + 0x1f, 0); + if (adev) { + connector->fwnode = + fwnode_handle_get(acpi_fwnode_handle(adev)); + break; + } + fallthrough; + default: + connector->fwnode = fwnode_handle_get(fwnode); + break; + } + } + drm_connector_list_iter_end(&conn_iter); + /* + * device_get_next_child_node() takes a reference on the fwnode, if + * we stopped iterating because we are out of connectors we need to + * put this, otherwise fwnode is NULL and the put is a no-op. + */ + fwnode_handle_put(fwnode); +} diff --git a/drivers/gpu/drm/i915/display/intel_acpi.h b/drivers/gpu/drm/i915/display/intel_acpi.h index 9f197401c313..4a760a2baed9 100644 --- a/drivers/gpu/drm/i915/display/intel_acpi.h +++ b/drivers/gpu/drm/i915/display/intel_acpi.h @@ -13,6 +13,7 @@ void intel_register_dsm_handler(void); void intel_unregister_dsm_handler(void); void intel_dsm_get_bios_data_funcs_supported(struct drm_i915_private *i915); void intel_acpi_device_id_update(struct drm_i915_private *i915); +void intel_acpi_assign_connector_fwnodes(struct drm_i915_private *i915); #else static inline void intel_register_dsm_handler(void) { return; } static inline void intel_unregister_dsm_handler(void) { return; } @@ -20,6 +21,8 @@ static inline void intel_dsm_get_bios_data_funcs_supported(struct drm_i915_private *i915) { return; } static inline void intel_acpi_device_id_update(struct drm_i915_private *i915) { return; } +static inline +void intel_acpi_assign_connector_fwnodes(struct drm_i915_private *i915) { return; } #endif /* CONFIG_ACPI */ #endif /* __INTEL_ACPI_H__ */ diff --git a/drivers/gpu/drm/i915/display/intel_atomic_plane.c b/drivers/gpu/drm/i915/display/intel_atomic_plane.c index 47234d898549..0be8c00e3db9 100644 --- a/drivers/gpu/drm/i915/display/intel_atomic_plane.c +++ b/drivers/gpu/drm/i915/display/intel_atomic_plane.c @@ -39,8 +39,10 @@ #include "intel_atomic_plane.h" #include "intel_cdclk.h" #include "intel_display_types.h" +#include "intel_fb_pin.h" #include "intel_pm.h" #include "intel_sprite.h" +#include "gt/intel_rps.h" static void intel_plane_state_reset(struct intel_plane_state *plane_state, struct intel_plane *plane) @@ -601,6 +603,213 @@ int intel_atomic_plane_check_clipping(struct intel_plane_state *plane_state, return 0; } +struct wait_rps_boost { + struct wait_queue_entry wait; + + struct drm_crtc *crtc; + struct i915_request *request; +}; + +static int do_rps_boost(struct wait_queue_entry *_wait, + unsigned mode, int sync, void *key) +{ + struct wait_rps_boost *wait = container_of(_wait, typeof(*wait), wait); + struct i915_request *rq = wait->request; + + /* + * If we missed the vblank, but the request is already running it + * is reasonable to assume that it will complete before the next + * vblank without our intervention, so leave RPS alone. + */ + if (!i915_request_started(rq)) + intel_rps_boost(rq); + i915_request_put(rq); + + drm_crtc_vblank_put(wait->crtc); + + list_del(&wait->wait.entry); + kfree(wait); + return 1; +} + +static void add_rps_boost_after_vblank(struct drm_crtc *crtc, + struct dma_fence *fence) +{ + struct wait_rps_boost *wait; + + if (!dma_fence_is_i915(fence)) + return; + + if (DISPLAY_VER(to_i915(crtc->dev)) < 6) + return; + + if (drm_crtc_vblank_get(crtc)) + return; + + wait = kmalloc(sizeof(*wait), GFP_KERNEL); + if (!wait) { + drm_crtc_vblank_put(crtc); + return; + } + + wait->request = to_request(dma_fence_get(fence)); + wait->crtc = crtc; + + wait->wait.func = do_rps_boost; + wait->wait.flags = 0; + + add_wait_queue(drm_crtc_vblank_waitqueue(crtc), &wait->wait); +} + +/** + * intel_prepare_plane_fb - Prepare fb for usage on plane + * @_plane: drm plane to prepare for + * @_new_plane_state: the plane state being prepared + * + * Prepares a framebuffer for usage on a display plane. Generally this + * involves pinning the underlying object and updating the frontbuffer tracking + * bits. Some older platforms need special physical address handling for + * cursor planes. + * + * Returns 0 on success, negative error code on failure. + */ +static int +intel_prepare_plane_fb(struct drm_plane *_plane, + struct drm_plane_state *_new_plane_state) +{ + struct i915_sched_attr attr = { .priority = I915_PRIORITY_DISPLAY }; + struct intel_plane *plane = to_intel_plane(_plane); + struct intel_plane_state *new_plane_state = + to_intel_plane_state(_new_plane_state); + struct intel_atomic_state *state = + to_intel_atomic_state(new_plane_state->uapi.state); + struct drm_i915_private *dev_priv = to_i915(plane->base.dev); + const struct intel_plane_state *old_plane_state = + intel_atomic_get_old_plane_state(state, plane); + struct drm_i915_gem_object *obj = intel_fb_obj(new_plane_state->hw.fb); + struct drm_i915_gem_object *old_obj = intel_fb_obj(old_plane_state->hw.fb); + int ret; + + if (old_obj) { + const struct intel_crtc_state *crtc_state = + intel_atomic_get_new_crtc_state(state, + to_intel_crtc(old_plane_state->hw.crtc)); + + /* Big Hammer, we also need to ensure that any pending + * MI_WAIT_FOR_EVENT inside a user batch buffer on the + * current scanout is retired before unpinning the old + * framebuffer. Note that we rely on userspace rendering + * into the buffer attached to the pipe they are waiting + * on. If not, userspace generates a GPU hang with IPEHR + * point to the MI_WAIT_FOR_EVENT. + * + * This should only fail upon a hung GPU, in which case we + * can safely continue. + */ + if (intel_crtc_needs_modeset(crtc_state)) { + ret = i915_sw_fence_await_reservation(&state->commit_ready, + old_obj->base.resv, NULL, + false, 0, + GFP_KERNEL); + if (ret < 0) + return ret; + } + } + + if (new_plane_state->uapi.fence) { /* explicit fencing */ + i915_gem_fence_wait_priority(new_plane_state->uapi.fence, + &attr); + ret = i915_sw_fence_await_dma_fence(&state->commit_ready, + new_plane_state->uapi.fence, + i915_fence_timeout(dev_priv), + GFP_KERNEL); + if (ret < 0) + return ret; + } + + if (!obj) + return 0; + + + ret = intel_plane_pin_fb(new_plane_state); + if (ret) + return ret; + + i915_gem_object_wait_priority(obj, 0, &attr); + + if (!new_plane_state->uapi.fence) { /* implicit fencing */ + struct dma_fence *fence; + + ret = i915_sw_fence_await_reservation(&state->commit_ready, + obj->base.resv, NULL, + false, + i915_fence_timeout(dev_priv), + GFP_KERNEL); + if (ret < 0) + goto unpin_fb; + + fence = dma_resv_get_excl_unlocked(obj->base.resv); + if (fence) { + add_rps_boost_after_vblank(new_plane_state->hw.crtc, + fence); + dma_fence_put(fence); + } + } else { + add_rps_boost_after_vblank(new_plane_state->hw.crtc, + new_plane_state->uapi.fence); + } + + /* + * We declare pageflips to be interactive and so merit a small bias + * towards upclocking to deliver the frame on time. By only changing + * the RPS thresholds to sample more regularly and aim for higher + * clocks we can hopefully deliver low power workloads (like kodi) + * that are not quite steady state without resorting to forcing + * maximum clocks following a vblank miss (see do_rps_boost()). + */ + if (!state->rps_interactive) { + intel_rps_mark_interactive(&dev_priv->gt.rps, true); + state->rps_interactive = true; + } + + return 0; + +unpin_fb: + intel_plane_unpin_fb(new_plane_state); + + return ret; +} + +/** + * intel_cleanup_plane_fb - Cleans up an fb after plane use + * @plane: drm plane to clean up for + * @_old_plane_state: the state from the previous modeset + * + * Cleans up a framebuffer that has just been removed from a plane. + */ +static void +intel_cleanup_plane_fb(struct drm_plane *plane, + struct drm_plane_state *_old_plane_state) +{ + struct intel_plane_state *old_plane_state = + to_intel_plane_state(_old_plane_state); + struct intel_atomic_state *state = + to_intel_atomic_state(old_plane_state->uapi.state); + struct drm_i915_private *dev_priv = to_i915(plane->dev); + struct drm_i915_gem_object *obj = intel_fb_obj(old_plane_state->hw.fb); + + if (!obj) + return; + + if (state->rps_interactive) { + intel_rps_mark_interactive(&dev_priv->gt.rps, false); + state->rps_interactive = false; + } + + /* Should only be called after a successful intel_prepare_plane_fb()! */ + intel_plane_unpin_fb(old_plane_state); +} + static const struct drm_plane_helper_funcs intel_plane_helper_funcs = { .prepare_fb = intel_prepare_plane_fb, .cleanup_fb = intel_cleanup_plane_fb, diff --git a/drivers/gpu/drm/i915/display/intel_audio.c b/drivers/gpu/drm/i915/display/intel_audio.c index 4e0f96bf6158..03e8c05a74f6 100644 --- a/drivers/gpu/drm/i915/display/intel_audio.c +++ b/drivers/gpu/drm/i915/display/intel_audio.c @@ -848,10 +848,10 @@ void intel_audio_codec_enable(struct intel_encoder *encoder, connector->eld[6] = drm_av_sync_delay(connector, adjusted_mode) / 2; - if (dev_priv->display.audio_codec_enable) - dev_priv->display.audio_codec_enable(encoder, - crtc_state, - conn_state); + if (dev_priv->audio_funcs) + dev_priv->audio_funcs->audio_codec_enable(encoder, + crtc_state, + conn_state); mutex_lock(&dev_priv->av_mutex); encoder->audio_connector = connector; @@ -893,10 +893,10 @@ void intel_audio_codec_disable(struct intel_encoder *encoder, enum port port = encoder->port; enum pipe pipe = crtc->pipe; - if (dev_priv->display.audio_codec_disable) - dev_priv->display.audio_codec_disable(encoder, - old_crtc_state, - old_conn_state); + if (dev_priv->audio_funcs) + dev_priv->audio_funcs->audio_codec_disable(encoder, + old_crtc_state, + old_conn_state); mutex_lock(&dev_priv->av_mutex); encoder->audio_connector = NULL; @@ -915,6 +915,21 @@ void intel_audio_codec_disable(struct intel_encoder *encoder, intel_lpe_audio_notify(dev_priv, pipe, port, NULL, 0, false); } +static const struct intel_audio_funcs g4x_audio_funcs = { + .audio_codec_enable = g4x_audio_codec_enable, + .audio_codec_disable = g4x_audio_codec_disable, +}; + +static const struct intel_audio_funcs ilk_audio_funcs = { + .audio_codec_enable = ilk_audio_codec_enable, + .audio_codec_disable = ilk_audio_codec_disable, +}; + +static const struct intel_audio_funcs hsw_audio_funcs = { + .audio_codec_enable = hsw_audio_codec_enable, + .audio_codec_disable = hsw_audio_codec_disable, +}; + /** * intel_init_audio_hooks - Set up chip specific audio hooks * @dev_priv: device private @@ -922,17 +937,13 @@ void intel_audio_codec_disable(struct intel_encoder *encoder, void intel_init_audio_hooks(struct drm_i915_private *dev_priv) { if (IS_G4X(dev_priv)) { - dev_priv->display.audio_codec_enable = g4x_audio_codec_enable; - dev_priv->display.audio_codec_disable = g4x_audio_codec_disable; + dev_priv->audio_funcs = &g4x_audio_funcs; } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) { - dev_priv->display.audio_codec_enable = ilk_audio_codec_enable; - dev_priv->display.audio_codec_disable = ilk_audio_codec_disable; + dev_priv->audio_funcs = &ilk_audio_funcs; } else if (IS_HASWELL(dev_priv) || DISPLAY_VER(dev_priv) >= 8) { - dev_priv->display.audio_codec_enable = hsw_audio_codec_enable; - dev_priv->display.audio_codec_disable = hsw_audio_codec_disable; + dev_priv->audio_funcs = &hsw_audio_funcs; } else if (HAS_PCH_SPLIT(dev_priv)) { - dev_priv->display.audio_codec_enable = ilk_audio_codec_enable; - dev_priv->display.audio_codec_disable = ilk_audio_codec_disable; + dev_priv->audio_funcs = &ilk_audio_funcs; } } diff --git a/drivers/gpu/drm/i915/display/intel_backlight.c b/drivers/gpu/drm/i915/display/intel_backlight.c new file mode 100644 index 000000000000..9523411cddd8 --- /dev/null +++ b/drivers/gpu/drm/i915/display/intel_backlight.c @@ -0,0 +1,1776 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2021 Intel Corporation + */ + +#include <linux/kernel.h> +#include <linux/pwm.h> + +#include "intel_backlight.h" +#include "intel_connector.h" +#include "intel_de.h" +#include "intel_display_types.h" +#include "intel_dp_aux_backlight.h" +#include "intel_dsi_dcs_backlight.h" +#include "intel_panel.h" + +/** + * scale - scale values from one range to another + * @source_val: value in range [@source_min..@source_max] + * @source_min: minimum legal value for @source_val + * @source_max: maximum legal value for @source_val + * @target_min: corresponding target value for @source_min + * @target_max: corresponding target value for @source_max + * + * Return @source_val in range [@source_min..@source_max] scaled to range + * [@target_min..@target_max]. + */ +static u32 scale(u32 source_val, + u32 source_min, u32 source_max, + u32 target_min, u32 target_max) +{ + u64 target_val; + + WARN_ON(source_min > source_max); + WARN_ON(target_min > target_max); + + /* defensive */ + source_val = clamp(source_val, source_min, source_max); + + /* avoid overflows */ + target_val = mul_u32_u32(source_val - source_min, + target_max - target_min); + target_val = DIV_ROUND_CLOSEST_ULL(target_val, source_max - source_min); + target_val += target_min; + + return target_val; +} + +/* + * Scale user_level in range [0..user_max] to [0..hw_max], clamping the result + * to [hw_min..hw_max]. + */ +static u32 clamp_user_to_hw(struct intel_connector *connector, + u32 user_level, u32 user_max) +{ + struct intel_panel *panel = &connector->panel; + u32 hw_level; + + hw_level = scale(user_level, 0, user_max, 0, panel->backlight.max); + hw_level = clamp(hw_level, panel->backlight.min, panel->backlight.max); + + return hw_level; +} + +/* Scale hw_level in range [hw_min..hw_max] to [0..user_max]. */ +static u32 scale_hw_to_user(struct intel_connector *connector, + u32 hw_level, u32 user_max) +{ + struct intel_panel *panel = &connector->panel; + + return scale(hw_level, panel->backlight.min, panel->backlight.max, + 0, user_max); +} + +u32 intel_backlight_invert_pwm_level(struct intel_connector *connector, u32 val) +{ + struct drm_i915_private *dev_priv = to_i915(connector->base.dev); + struct intel_panel *panel = &connector->panel; + + drm_WARN_ON(&dev_priv->drm, panel->backlight.pwm_level_max == 0); + + if (dev_priv->params.invert_brightness < 0) + return val; + + if (dev_priv->params.invert_brightness > 0 || + dev_priv->quirks & QUIRK_INVERT_BRIGHTNESS) { + return panel->backlight.pwm_level_max - val + panel->backlight.pwm_level_min; + } + + return val; +} + +void intel_backlight_set_pwm_level(const struct drm_connector_state *conn_state, u32 val) +{ + struct intel_connector *connector = to_intel_connector(conn_state->connector); + struct drm_i915_private *i915 = to_i915(connector->base.dev); + struct intel_panel *panel = &connector->panel; + + drm_dbg_kms(&i915->drm, "set backlight PWM = %d\n", val); + panel->backlight.pwm_funcs->set(conn_state, val); +} + +u32 intel_backlight_level_to_pwm(struct intel_connector *connector, u32 val) +{ + struct drm_i915_private *dev_priv = to_i915(connector->base.dev); + struct intel_panel *panel = &connector->panel; + + drm_WARN_ON_ONCE(&dev_priv->drm, + panel->backlight.max == 0 || panel->backlight.pwm_level_max == 0); + + val = scale(val, panel->backlight.min, panel->backlight.max, + panel->backlight.pwm_level_min, panel->backlight.pwm_level_max); + + return intel_backlight_invert_pwm_level(connector, val); +} + +u32 intel_backlight_level_from_pwm(struct intel_connector *connector, u32 val) +{ + struct drm_i915_private *dev_priv = to_i915(connector->base.dev); + struct intel_panel *panel = &connector->panel; + + drm_WARN_ON_ONCE(&dev_priv->drm, + panel->backlight.max == 0 || panel->backlight.pwm_level_max == 0); + + if (dev_priv->params.invert_brightness > 0 || + (dev_priv->params.invert_brightness == 0 && dev_priv->quirks & QUIRK_INVERT_BRIGHTNESS)) + val = panel->backlight.pwm_level_max - (val - panel->backlight.pwm_level_min); + + return scale(val, panel->backlight.pwm_level_min, panel->backlight.pwm_level_max, + panel->backlight.min, panel->backlight.max); +} + +static u32 lpt_get_backlight(struct intel_connector *connector, enum pipe unused) +{ + struct drm_i915_private *dev_priv = to_i915(connector->base.dev); + + return intel_de_read(dev_priv, BLC_PWM_PCH_CTL2) & BACKLIGHT_DUTY_CYCLE_MASK; +} + +static u32 pch_get_backlight(struct intel_connector *connector, enum pipe unused) +{ + struct drm_i915_private *dev_priv = to_i915(connector->base.dev); + + return intel_de_read(dev_priv, BLC_PWM_CPU_CTL) & BACKLIGHT_DUTY_CYCLE_MASK; +} + +static u32 i9xx_get_backlight(struct intel_connector *connector, enum pipe unused) +{ + struct drm_i915_private *dev_priv = to_i915(connector->base.dev); + struct intel_panel *panel = &connector->panel; + u32 val; + + val = intel_de_read(dev_priv, BLC_PWM_CTL) & BACKLIGHT_DUTY_CYCLE_MASK; + if (DISPLAY_VER(dev_priv) < 4) + val >>= 1; + + if (panel->backlight.combination_mode) { + u8 lbpc; + + pci_read_config_byte(to_pci_dev(dev_priv->drm.dev), LBPC, &lbpc); + val *= lbpc; + } + + return val; +} + +static u32 vlv_get_backlight(struct intel_connector *connector, enum pipe pipe) +{ + struct drm_i915_private *dev_priv = to_i915(connector->base.dev); + + if (drm_WARN_ON(&dev_priv->drm, pipe != PIPE_A && pipe != PIPE_B)) + return 0; + + return intel_de_read(dev_priv, VLV_BLC_PWM_CTL(pipe)) & BACKLIGHT_DUTY_CYCLE_MASK; +} + +static u32 bxt_get_backlight(struct intel_connector *connector, enum pipe unused) +{ + struct drm_i915_private *dev_priv = to_i915(connector->base.dev); + struct intel_panel *panel = &connector->panel; + + return intel_de_read(dev_priv, + BXT_BLC_PWM_DUTY(panel->backlight.controller)); +} + +static u32 ext_pwm_get_backlight(struct intel_connector *connector, enum pipe unused) +{ + struct intel_panel *panel = &connector->panel; + struct pwm_state state; + + pwm_get_state(panel->backlight.pwm, &state); + return pwm_get_relative_duty_cycle(&state, 100); +} + +static void lpt_set_backlight(const struct drm_connector_state *conn_state, u32 level) +{ + struct intel_connector *connector = to_intel_connector(conn_state->connector); + struct drm_i915_private *dev_priv = to_i915(connector->base.dev); + + u32 val = intel_de_read(dev_priv, BLC_PWM_PCH_CTL2) & ~BACKLIGHT_DUTY_CYCLE_MASK; + intel_de_write(dev_priv, BLC_PWM_PCH_CTL2, val | level); +} + +static void pch_set_backlight(const struct drm_connector_state *conn_state, u32 level) +{ + struct intel_connector *connector = to_intel_connector(conn_state->connector); + struct drm_i915_private *dev_priv = to_i915(connector->base.dev); + u32 tmp; + + tmp = intel_de_read(dev_priv, BLC_PWM_CPU_CTL) & ~BACKLIGHT_DUTY_CYCLE_MASK; + intel_de_write(dev_priv, BLC_PWM_CPU_CTL, tmp | level); +} + +static void i9xx_set_backlight(const struct drm_connector_state *conn_state, u32 level) +{ + struct intel_connector *connector = to_intel_connector(conn_state->connector); + struct drm_i915_private *dev_priv = to_i915(connector->base.dev); + struct intel_panel *panel = &connector->panel; + u32 tmp, mask; + + drm_WARN_ON(&dev_priv->drm, panel->backlight.pwm_level_max == 0); + + if (panel->backlight.combination_mode) { + u8 lbpc; + + lbpc = level * 0xfe / panel->backlight.pwm_level_max + 1; + level /= lbpc; + pci_write_config_byte(to_pci_dev(dev_priv->drm.dev), LBPC, lbpc); + } + + if (DISPLAY_VER(dev_priv) == 4) { + mask = BACKLIGHT_DUTY_CYCLE_MASK; + } else { + level <<= 1; + mask = BACKLIGHT_DUTY_CYCLE_MASK_PNV; + } + + tmp = intel_de_read(dev_priv, BLC_PWM_CTL) & ~mask; + intel_de_write(dev_priv, BLC_PWM_CTL, tmp | level); +} + +static void vlv_set_backlight(const struct drm_connector_state *conn_state, u32 level) +{ + struct intel_connector *connector = to_intel_connector(conn_state->connector); + struct drm_i915_private *dev_priv = to_i915(connector->base.dev); + enum pipe pipe = to_intel_crtc(conn_state->crtc)->pipe; + u32 tmp; + + tmp = intel_de_read(dev_priv, VLV_BLC_PWM_CTL(pipe)) & ~BACKLIGHT_DUTY_CYCLE_MASK; + intel_de_write(dev_priv, VLV_BLC_PWM_CTL(pipe), tmp | level); +} + +static void bxt_set_backlight(const struct drm_connector_state *conn_state, u32 level) +{ + struct intel_connector *connector = to_intel_connector(conn_state->connector); + struct drm_i915_private *dev_priv = to_i915(connector->base.dev); + struct intel_panel *panel = &connector->panel; + + intel_de_write(dev_priv, + BXT_BLC_PWM_DUTY(panel->backlight.controller), level); +} + +static void ext_pwm_set_backlight(const struct drm_connector_state *conn_state, u32 level) +{ + struct intel_panel *panel = &to_intel_connector(conn_state->connector)->panel; + + pwm_set_relative_duty_cycle(&panel->backlight.pwm_state, level, 100); + pwm_apply_state(panel->backlight.pwm, &panel->backlight.pwm_state); +} + +static void +intel_panel_actually_set_backlight(const struct drm_connector_state *conn_state, u32 level) +{ + struct intel_connector *connector = to_intel_connector(conn_state->connector); + struct drm_i915_private *i915 = to_i915(connector->base.dev); + struct intel_panel *panel = &connector->panel; + + drm_dbg_kms(&i915->drm, "set backlight level = %d\n", level); + + panel->backlight.funcs->set(conn_state, level); +} + +/* set backlight brightness to level in range [0..max], assuming hw min is + * respected. + */ +void intel_backlight_set_acpi(const struct drm_connector_state *conn_state, + u32 user_level, u32 user_max) +{ + struct intel_connector *connector = to_intel_connector(conn_state->connector); + struct drm_i915_private *dev_priv = to_i915(connector->base.dev); + struct intel_panel *panel = &connector->panel; + u32 hw_level; + + /* + * Lack of crtc may occur during driver init because + * connection_mutex isn't held across the entire backlight + * setup + modeset readout, and the BIOS can issue the + * requests at any time. + */ + if (!panel->backlight.present || !conn_state->crtc) + return; + + mutex_lock(&dev_priv->backlight_lock); + + drm_WARN_ON(&dev_priv->drm, panel->backlight.max == 0); + + hw_level = clamp_user_to_hw(connector, user_level, user_max); + panel->backlight.level = hw_level; + + if (panel->backlight.device) + panel->backlight.device->props.brightness = + scale_hw_to_user(connector, + panel->backlight.level, + panel->backlight.device->props.max_brightness); + + if (panel->backlight.enabled) + intel_panel_actually_set_backlight(conn_state, hw_level); + + mutex_unlock(&dev_priv->backlight_lock); +} + +static void lpt_disable_backlight(const struct drm_connector_state *old_conn_state, u32 level) +{ + struct intel_connector *connector = to_intel_connector(old_conn_state->connector); + struct drm_i915_private *dev_priv = to_i915(connector->base.dev); + u32 tmp; + + intel_backlight_set_pwm_level(old_conn_state, level); + + /* + * Although we don't support or enable CPU PWM with LPT/SPT based + * systems, it may have been enabled prior to loading the + * driver. Disable to avoid warnings on LCPLL disable. + * + * This needs rework if we need to add support for CPU PWM on PCH split + * platforms. + */ + tmp = intel_de_read(dev_priv, BLC_PWM_CPU_CTL2); + if (tmp & BLM_PWM_ENABLE) { + drm_dbg_kms(&dev_priv->drm, + "cpu backlight was enabled, disabling\n"); + intel_de_write(dev_priv, BLC_PWM_CPU_CTL2, + tmp & ~BLM_PWM_ENABLE); + } + + tmp = intel_de_read(dev_priv, BLC_PWM_PCH_CTL1); + intel_de_write(dev_priv, BLC_PWM_PCH_CTL1, tmp & ~BLM_PCH_PWM_ENABLE); +} + +static void pch_disable_backlight(const struct drm_connector_state *old_conn_state, u32 val) +{ + struct intel_connector *connector = to_intel_connector(old_conn_state->connector); + struct drm_i915_private *dev_priv = to_i915(connector->base.dev); + u32 tmp; + + intel_backlight_set_pwm_level(old_conn_state, val); + + tmp = intel_de_read(dev_priv, BLC_PWM_CPU_CTL2); + intel_de_write(dev_priv, BLC_PWM_CPU_CTL2, tmp & ~BLM_PWM_ENABLE); + + tmp = intel_de_read(dev_priv, BLC_PWM_PCH_CTL1); + intel_de_write(dev_priv, BLC_PWM_PCH_CTL1, tmp & ~BLM_PCH_PWM_ENABLE); +} + +static void i9xx_disable_backlight(const struct drm_connector_state *old_conn_state, u32 val) +{ + intel_backlight_set_pwm_level(old_conn_state, val); +} + +static void i965_disable_backlight(const struct drm_connector_state *old_conn_state, u32 val) +{ + struct drm_i915_private *dev_priv = to_i915(old_conn_state->connector->dev); + u32 tmp; + + intel_backlight_set_pwm_level(old_conn_state, val); + + tmp = intel_de_read(dev_priv, BLC_PWM_CTL2); + intel_de_write(dev_priv, BLC_PWM_CTL2, tmp & ~BLM_PWM_ENABLE); +} + +static void vlv_disable_backlight(const struct drm_connector_state *old_conn_state, u32 val) +{ + struct intel_connector *connector = to_intel_connector(old_conn_state->connector); + struct drm_i915_private *dev_priv = to_i915(connector->base.dev); + enum pipe pipe = to_intel_crtc(old_conn_state->crtc)->pipe; + u32 tmp; + + intel_backlight_set_pwm_level(old_conn_state, val); + + tmp = intel_de_read(dev_priv, VLV_BLC_PWM_CTL2(pipe)); + intel_de_write(dev_priv, VLV_BLC_PWM_CTL2(pipe), + tmp & ~BLM_PWM_ENABLE); +} + +static void bxt_disable_backlight(const struct drm_connector_state *old_conn_state, u32 val) +{ + struct intel_connector *connector = to_intel_connector(old_conn_state->connector); + struct drm_i915_private *dev_priv = to_i915(connector->base.dev); + struct intel_panel *panel = &connector->panel; + u32 tmp; + + intel_backlight_set_pwm_level(old_conn_state, val); + + tmp = intel_de_read(dev_priv, + BXT_BLC_PWM_CTL(panel->backlight.controller)); + intel_de_write(dev_priv, BXT_BLC_PWM_CTL(panel->backlight.controller), + tmp & ~BXT_BLC_PWM_ENABLE); + + if (panel->backlight.controller == 1) { + val = intel_de_read(dev_priv, UTIL_PIN_CTL); + val &= ~UTIL_PIN_ENABLE; + intel_de_write(dev_priv, UTIL_PIN_CTL, val); + } +} + +static void cnp_disable_backlight(const struct drm_connector_state *old_conn_state, u32 val) +{ + struct intel_connector *connector = to_intel_connector(old_conn_state->connector); + struct drm_i915_private *dev_priv = to_i915(connector->base.dev); + struct intel_panel *panel = &connector->panel; + u32 tmp; + + intel_backlight_set_pwm_level(old_conn_state, val); + + tmp = intel_de_read(dev_priv, + BXT_BLC_PWM_CTL(panel->backlight.controller)); + intel_de_write(dev_priv, BXT_BLC_PWM_CTL(panel->backlight.controller), + tmp & ~BXT_BLC_PWM_ENABLE); +} + +static void ext_pwm_disable_backlight(const struct drm_connector_state *old_conn_state, u32 level) +{ + struct intel_connector *connector = to_intel_connector(old_conn_state->connector); + struct intel_panel *panel = &connector->panel; + + panel->backlight.pwm_state.enabled = false; + pwm_apply_state(panel->backlight.pwm, &panel->backlight.pwm_state); +} + +void intel_backlight_disable(const struct drm_connector_state *old_conn_state) +{ + struct intel_connector *connector = to_intel_connector(old_conn_state->connector); + struct drm_i915_private *dev_priv = to_i915(connector->base.dev); + struct intel_panel *panel = &connector->panel; + + if (!panel->backlight.present) + return; + + /* + * Do not disable backlight on the vga_switcheroo path. When switching + * away from i915, the other client may depend on i915 to handle the + * backlight. This will leave the backlight on unnecessarily when + * another client is not activated. + */ + if (dev_priv->drm.switch_power_state == DRM_SWITCH_POWER_CHANGING) { + drm_dbg_kms(&dev_priv->drm, + "Skipping backlight disable on vga switch\n"); + return; + } + + mutex_lock(&dev_priv->backlight_lock); + + if (panel->backlight.device) + panel->backlight.device->props.power = FB_BLANK_POWERDOWN; + panel->backlight.enabled = false; + panel->backlight.funcs->disable(old_conn_state, 0); + + mutex_unlock(&dev_priv->backlight_lock); +} + +static void lpt_enable_backlight(const struct intel_crtc_state *crtc_state, + const struct drm_connector_state *conn_state, u32 level) +{ + struct intel_connector *connector = to_intel_connector(conn_state->connector); + struct drm_i915_private *dev_priv = to_i915(connector->base.dev); + struct intel_panel *panel = &connector->panel; + u32 pch_ctl1, pch_ctl2, schicken; + + pch_ctl1 = intel_de_read(dev_priv, BLC_PWM_PCH_CTL1); + if (pch_ctl1 & BLM_PCH_PWM_ENABLE) { + drm_dbg_kms(&dev_priv->drm, "pch backlight already enabled\n"); + pch_ctl1 &= ~BLM_PCH_PWM_ENABLE; + intel_de_write(dev_priv, BLC_PWM_PCH_CTL1, pch_ctl1); + } + + if (HAS_PCH_LPT(dev_priv)) { + schicken = intel_de_read(dev_priv, SOUTH_CHICKEN2); + if (panel->backlight.alternate_pwm_increment) + schicken |= LPT_PWM_GRANULARITY; + else + schicken &= ~LPT_PWM_GRANULARITY; + intel_de_write(dev_priv, SOUTH_CHICKEN2, schicken); + } else { + schicken = intel_de_read(dev_priv, SOUTH_CHICKEN1); + if (panel->backlight.alternate_pwm_increment) + schicken |= SPT_PWM_GRANULARITY; + else + schicken &= ~SPT_PWM_GRANULARITY; + intel_de_write(dev_priv, SOUTH_CHICKEN1, schicken); + } + + pch_ctl2 = panel->backlight.pwm_level_max << 16; + intel_de_write(dev_priv, BLC_PWM_PCH_CTL2, pch_ctl2); + + pch_ctl1 = 0; + if (panel->backlight.active_low_pwm) + pch_ctl1 |= BLM_PCH_POLARITY; + + /* After LPT, override is the default. */ + if (HAS_PCH_LPT(dev_priv)) + pch_ctl1 |= BLM_PCH_OVERRIDE_ENABLE; + + intel_de_write(dev_priv, BLC_PWM_PCH_CTL1, pch_ctl1); + intel_de_posting_read(dev_priv, BLC_PWM_PCH_CTL1); + intel_de_write(dev_priv, BLC_PWM_PCH_CTL1, + pch_ctl1 | BLM_PCH_PWM_ENABLE); + + /* This won't stick until the above enable. */ + intel_backlight_set_pwm_level(conn_state, level); +} + +static void pch_enable_backlight(const struct intel_crtc_state *crtc_state, + const struct drm_connector_state *conn_state, u32 level) +{ + struct intel_connector *connector = to_intel_connector(conn_state->connector); + struct drm_i915_private *dev_priv = to_i915(connector->base.dev); + struct intel_panel *panel = &connector->panel; + enum transcoder cpu_transcoder = crtc_state->cpu_transcoder; + u32 cpu_ctl2, pch_ctl1, pch_ctl2; + + cpu_ctl2 = intel_de_read(dev_priv, BLC_PWM_CPU_CTL2); + if (cpu_ctl2 & BLM_PWM_ENABLE) { + drm_dbg_kms(&dev_priv->drm, "cpu backlight already enabled\n"); + cpu_ctl2 &= ~BLM_PWM_ENABLE; + intel_de_write(dev_priv, BLC_PWM_CPU_CTL2, cpu_ctl2); + } + + pch_ctl1 = intel_de_read(dev_priv, BLC_PWM_PCH_CTL1); + if (pch_ctl1 & BLM_PCH_PWM_ENABLE) { + drm_dbg_kms(&dev_priv->drm, "pch backlight already enabled\n"); + pch_ctl1 &= ~BLM_PCH_PWM_ENABLE; + intel_de_write(dev_priv, BLC_PWM_PCH_CTL1, pch_ctl1); + } + + if (cpu_transcoder == TRANSCODER_EDP) + cpu_ctl2 = BLM_TRANSCODER_EDP; + else + cpu_ctl2 = BLM_PIPE(cpu_transcoder); + intel_de_write(dev_priv, BLC_PWM_CPU_CTL2, cpu_ctl2); + intel_de_posting_read(dev_priv, BLC_PWM_CPU_CTL2); + intel_de_write(dev_priv, BLC_PWM_CPU_CTL2, cpu_ctl2 | BLM_PWM_ENABLE); + + /* This won't stick until the above enable. */ + intel_backlight_set_pwm_level(conn_state, level); + + pch_ctl2 = panel->backlight.pwm_level_max << 16; + intel_de_write(dev_priv, BLC_PWM_PCH_CTL2, pch_ctl2); + + pch_ctl1 = 0; + if (panel->backlight.active_low_pwm) + pch_ctl1 |= BLM_PCH_POLARITY; + + intel_de_write(dev_priv, BLC_PWM_PCH_CTL1, pch_ctl1); + intel_de_posting_read(dev_priv, BLC_PWM_PCH_CTL1); + intel_de_write(dev_priv, BLC_PWM_PCH_CTL1, + pch_ctl1 | BLM_PCH_PWM_ENABLE); +} + +static void i9xx_enable_backlight(const struct intel_crtc_state *crtc_state, + const struct drm_connector_state *conn_state, u32 level) +{ + struct intel_connector *connector = to_intel_connector(conn_state->connector); + struct drm_i915_private *dev_priv = to_i915(connector->base.dev); + struct intel_panel *panel = &connector->panel; + u32 ctl, freq; + + ctl = intel_de_read(dev_priv, BLC_PWM_CTL); + if (ctl & BACKLIGHT_DUTY_CYCLE_MASK_PNV) { + drm_dbg_kms(&dev_priv->drm, "backlight already enabled\n"); + intel_de_write(dev_priv, BLC_PWM_CTL, 0); + } + + freq = panel->backlight.pwm_level_max; + if (panel->backlight.combination_mode) + freq /= 0xff; + + ctl = freq << 17; + if (panel->backlight.combination_mode) + ctl |= BLM_LEGACY_MODE; + if (IS_PINEVIEW(dev_priv) && panel->backlight.active_low_pwm) + ctl |= BLM_POLARITY_PNV; + + intel_de_write(dev_priv, BLC_PWM_CTL, ctl); + intel_de_posting_read(dev_priv, BLC_PWM_CTL); + + /* XXX: combine this into above write? */ + intel_backlight_set_pwm_level(conn_state, level); + + /* + * Needed to enable backlight on some 855gm models. BLC_HIST_CTL is + * 855gm only, but checking for gen2 is safe, as 855gm is the only gen2 + * that has backlight. + */ + if (DISPLAY_VER(dev_priv) == 2) + intel_de_write(dev_priv, BLC_HIST_CTL, BLM_HISTOGRAM_ENABLE); +} + +static void i965_enable_backlight(const struct intel_crtc_state *crtc_state, + const struct drm_connector_state *conn_state, u32 level) +{ + struct intel_connector *connector = to_intel_connector(conn_state->connector); + struct drm_i915_private *dev_priv = to_i915(connector->base.dev); + struct intel_panel *panel = &connector->panel; + enum pipe pipe = to_intel_crtc(conn_state->crtc)->pipe; + u32 ctl, ctl2, freq; + + ctl2 = intel_de_read(dev_priv, BLC_PWM_CTL2); + if (ctl2 & BLM_PWM_ENABLE) { + drm_dbg_kms(&dev_priv->drm, "backlight already enabled\n"); + ctl2 &= ~BLM_PWM_ENABLE; + intel_de_write(dev_priv, BLC_PWM_CTL2, ctl2); + } + + freq = panel->backlight.pwm_level_max; + if (panel->backlight.combination_mode) + freq /= 0xff; + + ctl = freq << 16; + intel_de_write(dev_priv, BLC_PWM_CTL, ctl); + + ctl2 = BLM_PIPE(pipe); + if (panel->backlight.combination_mode) + ctl2 |= BLM_COMBINATION_MODE; + if (panel->backlight.active_low_pwm) + ctl2 |= BLM_POLARITY_I965; + intel_de_write(dev_priv, BLC_PWM_CTL2, ctl2); + intel_de_posting_read(dev_priv, BLC_PWM_CTL2); + intel_de_write(dev_priv, BLC_PWM_CTL2, ctl2 | BLM_PWM_ENABLE); + + intel_backlight_set_pwm_level(conn_state, level); +} + +static void vlv_enable_backlight(const struct intel_crtc_state *crtc_state, + const struct drm_connector_state *conn_state, u32 level) +{ + struct intel_connector *connector = to_intel_connector(conn_state->connector); + struct drm_i915_private *dev_priv = to_i915(connector->base.dev); + struct intel_panel *panel = &connector->panel; + enum pipe pipe = to_intel_crtc(crtc_state->uapi.crtc)->pipe; + u32 ctl, ctl2; + + ctl2 = intel_de_read(dev_priv, VLV_BLC_PWM_CTL2(pipe)); + if (ctl2 & BLM_PWM_ENABLE) { + drm_dbg_kms(&dev_priv->drm, "backlight already enabled\n"); + ctl2 &= ~BLM_PWM_ENABLE; + intel_de_write(dev_priv, VLV_BLC_PWM_CTL2(pipe), ctl2); + } + + ctl = panel->backlight.pwm_level_max << 16; + intel_de_write(dev_priv, VLV_BLC_PWM_CTL(pipe), ctl); + + /* XXX: combine this into above write? */ + intel_backlight_set_pwm_level(conn_state, level); + + ctl2 = 0; + if (panel->backlight.active_low_pwm) + ctl2 |= BLM_POLARITY_I965; + intel_de_write(dev_priv, VLV_BLC_PWM_CTL2(pipe), ctl2); + intel_de_posting_read(dev_priv, VLV_BLC_PWM_CTL2(pipe)); + intel_de_write(dev_priv, VLV_BLC_PWM_CTL2(pipe), + ctl2 | BLM_PWM_ENABLE); +} + +static void bxt_enable_backlight(const struct intel_crtc_state *crtc_state, + const struct drm_connector_state *conn_state, u32 level) +{ + struct intel_connector *connector = to_intel_connector(conn_state->connector); + struct drm_i915_private *dev_priv = to_i915(connector->base.dev); + struct intel_panel *panel = &connector->panel; + enum pipe pipe = to_intel_crtc(crtc_state->uapi.crtc)->pipe; + u32 pwm_ctl, val; + + /* Controller 1 uses the utility pin. */ + if (panel->backlight.controller == 1) { + val = intel_de_read(dev_priv, UTIL_PIN_CTL); + if (val & UTIL_PIN_ENABLE) { + drm_dbg_kms(&dev_priv->drm, + "util pin already enabled\n"); + val &= ~UTIL_PIN_ENABLE; + intel_de_write(dev_priv, UTIL_PIN_CTL, val); + } + + val = 0; + if (panel->backlight.util_pin_active_low) + val |= UTIL_PIN_POLARITY; + intel_de_write(dev_priv, UTIL_PIN_CTL, + val | UTIL_PIN_PIPE(pipe) | UTIL_PIN_MODE_PWM | UTIL_PIN_ENABLE); + } + + pwm_ctl = intel_de_read(dev_priv, + BXT_BLC_PWM_CTL(panel->backlight.controller)); + if (pwm_ctl & BXT_BLC_PWM_ENABLE) { + drm_dbg_kms(&dev_priv->drm, "backlight already enabled\n"); + pwm_ctl &= ~BXT_BLC_PWM_ENABLE; + intel_de_write(dev_priv, + BXT_BLC_PWM_CTL(panel->backlight.controller), + pwm_ctl); + } + + intel_de_write(dev_priv, + BXT_BLC_PWM_FREQ(panel->backlight.controller), + panel->backlight.pwm_level_max); + + intel_backlight_set_pwm_level(conn_state, level); + + pwm_ctl = 0; + if (panel->backlight.active_low_pwm) + pwm_ctl |= BXT_BLC_PWM_POLARITY; + + intel_de_write(dev_priv, BXT_BLC_PWM_CTL(panel->backlight.controller), + pwm_ctl); + intel_de_posting_read(dev_priv, + BXT_BLC_PWM_CTL(panel->backlight.controller)); + intel_de_write(dev_priv, BXT_BLC_PWM_CTL(panel->backlight.controller), + pwm_ctl | BXT_BLC_PWM_ENABLE); +} + +static void cnp_enable_backlight(const struct intel_crtc_state *crtc_state, + const struct drm_connector_state *conn_state, u32 level) +{ + struct intel_connector *connector = to_intel_connector(conn_state->connector); + struct drm_i915_private *dev_priv = to_i915(connector->base.dev); + struct intel_panel *panel = &connector->panel; + u32 pwm_ctl; + + pwm_ctl = intel_de_read(dev_priv, + BXT_BLC_PWM_CTL(panel->backlight.controller)); + if (pwm_ctl & BXT_BLC_PWM_ENABLE) { + drm_dbg_kms(&dev_priv->drm, "backlight already enabled\n"); + pwm_ctl &= ~BXT_BLC_PWM_ENABLE; + intel_de_write(dev_priv, + BXT_BLC_PWM_CTL(panel->backlight.controller), + pwm_ctl); + } + + intel_de_write(dev_priv, + BXT_BLC_PWM_FREQ(panel->backlight.controller), + panel->backlight.pwm_level_max); + + intel_backlight_set_pwm_level(conn_state, level); + + pwm_ctl = 0; + if (panel->backlight.active_low_pwm) + pwm_ctl |= BXT_BLC_PWM_POLARITY; + + intel_de_write(dev_priv, BXT_BLC_PWM_CTL(panel->backlight.controller), + pwm_ctl); + intel_de_posting_read(dev_priv, + BXT_BLC_PWM_CTL(panel->backlight.controller)); + intel_de_write(dev_priv, BXT_BLC_PWM_CTL(panel->backlight.controller), + pwm_ctl | BXT_BLC_PWM_ENABLE); +} + +static void ext_pwm_enable_backlight(const struct intel_crtc_state *crtc_state, + const struct drm_connector_state *conn_state, u32 level) +{ + struct intel_connector *connector = to_intel_connector(conn_state->connector); + struct intel_panel *panel = &connector->panel; + + pwm_set_relative_duty_cycle(&panel->backlight.pwm_state, level, 100); + panel->backlight.pwm_state.enabled = true; + pwm_apply_state(panel->backlight.pwm, &panel->backlight.pwm_state); +} + +static void __intel_backlight_enable(const struct intel_crtc_state *crtc_state, + const struct drm_connector_state *conn_state) +{ + struct intel_connector *connector = to_intel_connector(conn_state->connector); + struct intel_panel *panel = &connector->panel; + + WARN_ON(panel->backlight.max == 0); + + if (panel->backlight.level <= panel->backlight.min) { + panel->backlight.level = panel->backlight.max; + if (panel->backlight.device) + panel->backlight.device->props.brightness = + scale_hw_to_user(connector, + panel->backlight.level, + panel->backlight.device->props.max_brightness); + } + + panel->backlight.funcs->enable(crtc_state, conn_state, panel->backlight.level); + panel->backlight.enabled = true; + if (panel->backlight.device) + panel->backlight.device->props.power = FB_BLANK_UNBLANK; +} + +void intel_backlight_enable(const struct intel_crtc_state *crtc_state, + const struct drm_connector_state *conn_state) +{ + struct intel_connector *connector = to_intel_connector(conn_state->connector); + struct drm_i915_private *dev_priv = to_i915(connector->base.dev); + struct intel_panel *panel = &connector->panel; + enum pipe pipe = to_intel_crtc(crtc_state->uapi.crtc)->pipe; + + if (!panel->backlight.present) + return; + + drm_dbg_kms(&dev_priv->drm, "pipe %c\n", pipe_name(pipe)); + + mutex_lock(&dev_priv->backlight_lock); + + __intel_backlight_enable(crtc_state, conn_state); + + mutex_unlock(&dev_priv->backlight_lock); +} + +#if IS_ENABLED(CONFIG_BACKLIGHT_CLASS_DEVICE) +static u32 intel_panel_get_backlight(struct intel_connector *connector) +{ + struct drm_i915_private *dev_priv = to_i915(connector->base.dev); + struct intel_panel *panel = &connector->panel; + u32 val = 0; + + mutex_lock(&dev_priv->backlight_lock); + + if (panel->backlight.enabled) + val = panel->backlight.funcs->get(connector, intel_connector_get_pipe(connector)); + + mutex_unlock(&dev_priv->backlight_lock); + + drm_dbg_kms(&dev_priv->drm, "get backlight PWM = %d\n", val); + return val; +} + +/* Scale user_level in range [0..user_max] to [hw_min..hw_max]. */ +static u32 scale_user_to_hw(struct intel_connector *connector, + u32 user_level, u32 user_max) +{ + struct intel_panel *panel = &connector->panel; + + return scale(user_level, 0, user_max, + panel->backlight.min, panel->backlight.max); +} + +/* set backlight brightness to level in range [0..max], scaling wrt hw min */ +static void intel_panel_set_backlight(const struct drm_connector_state *conn_state, + u32 user_level, u32 user_max) +{ + struct intel_connector *connector = to_intel_connector(conn_state->connector); + struct drm_i915_private *dev_priv = to_i915(connector->base.dev); + struct intel_panel *panel = &connector->panel; + u32 hw_level; + + if (!panel->backlight.present) + return; + + mutex_lock(&dev_priv->backlight_lock); + + drm_WARN_ON(&dev_priv->drm, panel->backlight.max == 0); + + hw_level = scale_user_to_hw(connector, user_level, user_max); + panel->backlight.level = hw_level; + + if (panel->backlight.enabled) + intel_panel_actually_set_backlight(conn_state, hw_level); + + mutex_unlock(&dev_priv->backlight_lock); +} + +static int intel_backlight_device_update_status(struct backlight_device *bd) +{ + struct intel_connector *connector = bl_get_data(bd); + struct intel_panel *panel = &connector->panel; + struct drm_device *dev = connector->base.dev; + + drm_modeset_lock(&dev->mode_config.connection_mutex, NULL); + DRM_DEBUG_KMS("updating intel_backlight, brightness=%d/%d\n", + bd->props.brightness, bd->props.max_brightness); + intel_panel_set_backlight(connector->base.state, bd->props.brightness, + bd->props.max_brightness); + + /* + * Allow flipping bl_power as a sub-state of enabled. Sadly the + * backlight class device does not make it easy to differentiate + * between callbacks for brightness and bl_power, so our backlight_power + * callback needs to take this into account. + */ + if (panel->backlight.enabled) { + if (panel->backlight.power) { + bool enable = bd->props.power == FB_BLANK_UNBLANK && + bd->props.brightness != 0; + panel->backlight.power(connector, enable); + } + } else { + bd->props.power = FB_BLANK_POWERDOWN; + } + + drm_modeset_unlock(&dev->mode_config.connection_mutex); + return 0; +} + +static int intel_backlight_device_get_brightness(struct backlight_device *bd) +{ + struct intel_connector *connector = bl_get_data(bd); + struct drm_device *dev = connector->base.dev; + struct drm_i915_private *dev_priv = to_i915(dev); + intel_wakeref_t wakeref; + int ret = 0; + + with_intel_runtime_pm(&dev_priv->runtime_pm, wakeref) { + u32 hw_level; + + drm_modeset_lock(&dev->mode_config.connection_mutex, NULL); + + hw_level = intel_panel_get_backlight(connector); + ret = scale_hw_to_user(connector, + hw_level, bd->props.max_brightness); + + drm_modeset_unlock(&dev->mode_config.connection_mutex); + } + + return ret; +} + +static const struct backlight_ops intel_backlight_device_ops = { + .update_status = intel_backlight_device_update_status, + .get_brightness = intel_backlight_device_get_brightness, +}; + +int intel_backlight_device_register(struct intel_connector *connector) +{ + struct drm_i915_private *i915 = to_i915(connector->base.dev); + struct intel_panel *panel = &connector->panel; + struct backlight_properties props; + struct backlight_device *bd; + const char *name; + int ret = 0; + + if (WARN_ON(panel->backlight.device)) + return -ENODEV; + + if (!panel->backlight.present) + return 0; + + WARN_ON(panel->backlight.max == 0); + + memset(&props, 0, sizeof(props)); + props.type = BACKLIGHT_RAW; + + /* + * Note: Everything should work even if the backlight device max + * presented to the userspace is arbitrarily chosen. + */ + props.max_brightness = panel->backlight.max; + props.brightness = scale_hw_to_user(connector, + panel->backlight.level, + props.max_brightness); + + if (panel->backlight.enabled) + props.power = FB_BLANK_UNBLANK; + else + props.power = FB_BLANK_POWERDOWN; + + name = kstrdup("intel_backlight", GFP_KERNEL); + if (!name) + return -ENOMEM; + + bd = backlight_device_register(name, connector->base.kdev, connector, + &intel_backlight_device_ops, &props); + + /* + * Using the same name independent of the drm device or connector + * prevents registration of multiple backlight devices in the + * driver. However, we need to use the default name for backward + * compatibility. Use unique names for subsequent backlight devices as a + * fallback when the default name already exists. + */ + if (IS_ERR(bd) && PTR_ERR(bd) == -EEXIST) { + kfree(name); + name = kasprintf(GFP_KERNEL, "card%d-%s-backlight", + i915->drm.primary->index, connector->base.name); + if (!name) + return -ENOMEM; + + bd = backlight_device_register(name, connector->base.kdev, connector, + &intel_backlight_device_ops, &props); + } + + if (IS_ERR(bd)) { + drm_err(&i915->drm, + "[CONNECTOR:%d:%s] backlight device %s register failed: %ld\n", + connector->base.base.id, connector->base.name, name, PTR_ERR(bd)); + ret = PTR_ERR(bd); + goto out; + } + + panel->backlight.device = bd; + + drm_dbg_kms(&i915->drm, + "[CONNECTOR:%d:%s] backlight device %s registered\n", + connector->base.base.id, connector->base.name, name); + +out: + kfree(name); + + return ret; +} + +void intel_backlight_device_unregister(struct intel_connector *connector) +{ + struct intel_panel *panel = &connector->panel; + + if (panel->backlight.device) { + backlight_device_unregister(panel->backlight.device); + panel->backlight.device = NULL; + } +} +#endif /* CONFIG_BACKLIGHT_CLASS_DEVICE */ + +/* + * CNP: PWM clock frequency is 19.2 MHz or 24 MHz. + * PWM increment = 1 + */ +static u32 cnp_hz_to_pwm(struct intel_connector *connector, u32 pwm_freq_hz) +{ + struct drm_i915_private *dev_priv = to_i915(connector->base.dev); + + return DIV_ROUND_CLOSEST(KHz(RUNTIME_INFO(dev_priv)->rawclk_freq), + pwm_freq_hz); +} + +/* + * BXT: PWM clock frequency = 19.2 MHz. + */ +static u32 bxt_hz_to_pwm(struct intel_connector *connector, u32 pwm_freq_hz) +{ + return DIV_ROUND_CLOSEST(KHz(19200), pwm_freq_hz); +} + +/* + * SPT: This value represents the period of the PWM stream in clock periods + * multiplied by 16 (default increment) or 128 (alternate increment selected in + * SCHICKEN_1 bit 0). PWM clock is 24 MHz. + */ +static u32 spt_hz_to_pwm(struct intel_connector *connector, u32 pwm_freq_hz) +{ + struct intel_panel *panel = &connector->panel; + u32 mul; + + if (panel->backlight.alternate_pwm_increment) + mul = 128; + else + mul = 16; + + return DIV_ROUND_CLOSEST(MHz(24), pwm_freq_hz * mul); +} + +/* + * LPT: This value represents the period of the PWM stream in clock periods + * multiplied by 128 (default increment) or 16 (alternate increment, selected in + * LPT SOUTH_CHICKEN2 register bit 5). + */ +static u32 lpt_hz_to_pwm(struct intel_connector *connector, u32 pwm_freq_hz) +{ + struct drm_i915_private *dev_priv = to_i915(connector->base.dev); + struct intel_panel *panel = &connector->panel; + u32 mul, clock; + + if (panel->backlight.alternate_pwm_increment) + mul = 16; + else + mul = 128; + + if (HAS_PCH_LPT_H(dev_priv)) + clock = MHz(135); /* LPT:H */ + else + clock = MHz(24); /* LPT:LP */ + + return DIV_ROUND_CLOSEST(clock, pwm_freq_hz * mul); +} + +/* + * ILK/SNB/IVB: This value represents the period of the PWM stream in PCH + * display raw clocks multiplied by 128. + */ +static u32 pch_hz_to_pwm(struct intel_connector *connector, u32 pwm_freq_hz) +{ + struct drm_i915_private *dev_priv = to_i915(connector->base.dev); + + return DIV_ROUND_CLOSEST(KHz(RUNTIME_INFO(dev_priv)->rawclk_freq), + pwm_freq_hz * 128); +} + +/* + * Gen2: This field determines the number of time base events (display core + * clock frequency/32) in total for a complete cycle of modulated backlight + * control. + * + * Gen3: A time base event equals the display core clock ([DevPNV] HRAW clock) + * divided by 32. + */ +static u32 i9xx_hz_to_pwm(struct intel_connector *connector, u32 pwm_freq_hz) +{ + struct drm_i915_private *dev_priv = to_i915(connector->base.dev); + int clock; + + if (IS_PINEVIEW(dev_priv)) + clock = KHz(RUNTIME_INFO(dev_priv)->rawclk_freq); + else + clock = KHz(dev_priv->cdclk.hw.cdclk); + + return DIV_ROUND_CLOSEST(clock, pwm_freq_hz * 32); +} + +/* + * Gen4: This value represents the period of the PWM stream in display core + * clocks ([DevCTG] HRAW clocks) multiplied by 128. + * + */ +static u32 i965_hz_to_pwm(struct intel_connector *connector, u32 pwm_freq_hz) +{ + struct drm_i915_private *dev_priv = to_i915(connector->base.dev); + int clock; + + if (IS_G4X(dev_priv)) + clock = KHz(RUNTIME_INFO(dev_priv)->rawclk_freq); + else + clock = KHz(dev_priv->cdclk.hw.cdclk); + + return DIV_ROUND_CLOSEST(clock, pwm_freq_hz * 128); +} + +/* + * VLV: This value represents the period of the PWM stream in display core + * clocks ([DevCTG] 200MHz HRAW clocks) multiplied by 128 or 25MHz S0IX clocks + * multiplied by 16. CHV uses a 19.2MHz S0IX clock. + */ +static u32 vlv_hz_to_pwm(struct intel_connector *connector, u32 pwm_freq_hz) +{ + struct drm_i915_private *dev_priv = to_i915(connector->base.dev); + int mul, clock; + + if ((intel_de_read(dev_priv, CBR1_VLV) & CBR_PWM_CLOCK_MUX_SELECT) == 0) { + if (IS_CHERRYVIEW(dev_priv)) + clock = KHz(19200); + else + clock = MHz(25); + mul = 16; + } else { + clock = KHz(RUNTIME_INFO(dev_priv)->rawclk_freq); + mul = 128; + } + + return DIV_ROUND_CLOSEST(clock, pwm_freq_hz * mul); +} + +static u16 get_vbt_pwm_freq(struct drm_i915_private *dev_priv) +{ + u16 pwm_freq_hz = dev_priv->vbt.backlight.pwm_freq_hz; + + if (pwm_freq_hz) { + drm_dbg_kms(&dev_priv->drm, + "VBT defined backlight frequency %u Hz\n", + pwm_freq_hz); + } else { + pwm_freq_hz = 200; + drm_dbg_kms(&dev_priv->drm, + "default backlight frequency %u Hz\n", + pwm_freq_hz); + } + + return pwm_freq_hz; +} + +static u32 get_backlight_max_vbt(struct intel_connector *connector) +{ + struct drm_i915_private *dev_priv = to_i915(connector->base.dev); + struct intel_panel *panel = &connector->panel; + u16 pwm_freq_hz = get_vbt_pwm_freq(dev_priv); + u32 pwm; + + if (!panel->backlight.pwm_funcs->hz_to_pwm) { + drm_dbg_kms(&dev_priv->drm, + "backlight frequency conversion not supported\n"); + return 0; + } + + pwm = panel->backlight.pwm_funcs->hz_to_pwm(connector, pwm_freq_hz); + if (!pwm) { + drm_dbg_kms(&dev_priv->drm, + "backlight frequency conversion failed\n"); + return 0; + } + + return pwm; +} + +/* + * Note: The setup hooks can't assume pipe is set! + */ +static u32 get_backlight_min_vbt(struct intel_connector *connector) +{ + struct drm_i915_private *dev_priv = to_i915(connector->base.dev); + struct intel_panel *panel = &connector->panel; + int min; + + drm_WARN_ON(&dev_priv->drm, panel->backlight.pwm_level_max == 0); + + /* + * XXX: If the vbt value is 255, it makes min equal to max, which leads + * to problems. There are such machines out there. Either our + * interpretation is wrong or the vbt has bogus data. Or both. Safeguard + * against this by letting the minimum be at most (arbitrarily chosen) + * 25% of the max. + */ + min = clamp_t(int, dev_priv->vbt.backlight.min_brightness, 0, 64); + if (min != dev_priv->vbt.backlight.min_brightness) { + drm_dbg_kms(&dev_priv->drm, + "clamping VBT min backlight %d/255 to %d/255\n", + dev_priv->vbt.backlight.min_brightness, min); + } + + /* vbt value is a coefficient in range [0..255] */ + return scale(min, 0, 255, 0, panel->backlight.pwm_level_max); +} + +static int lpt_setup_backlight(struct intel_connector *connector, enum pipe unused) +{ + struct drm_i915_private *dev_priv = to_i915(connector->base.dev); + struct intel_panel *panel = &connector->panel; + u32 cpu_ctl2, pch_ctl1, pch_ctl2, val; + bool alt, cpu_mode; + + if (HAS_PCH_LPT(dev_priv)) + alt = intel_de_read(dev_priv, SOUTH_CHICKEN2) & LPT_PWM_GRANULARITY; + else + alt = intel_de_read(dev_priv, SOUTH_CHICKEN1) & SPT_PWM_GRANULARITY; + panel->backlight.alternate_pwm_increment = alt; + + pch_ctl1 = intel_de_read(dev_priv, BLC_PWM_PCH_CTL1); + panel->backlight.active_low_pwm = pch_ctl1 & BLM_PCH_POLARITY; + + pch_ctl2 = intel_de_read(dev_priv, BLC_PWM_PCH_CTL2); + panel->backlight.pwm_level_max = pch_ctl2 >> 16; + + cpu_ctl2 = intel_de_read(dev_priv, BLC_PWM_CPU_CTL2); + + if (!panel->backlight.pwm_level_max) + panel->backlight.pwm_level_max = get_backlight_max_vbt(connector); + + if (!panel->backlight.pwm_level_max) + return -ENODEV; + + panel->backlight.pwm_level_min = get_backlight_min_vbt(connector); + + panel->backlight.pwm_enabled = pch_ctl1 & BLM_PCH_PWM_ENABLE; + + cpu_mode = panel->backlight.pwm_enabled && HAS_PCH_LPT(dev_priv) && + !(pch_ctl1 & BLM_PCH_OVERRIDE_ENABLE) && + (cpu_ctl2 & BLM_PWM_ENABLE); + + if (cpu_mode) { + val = pch_get_backlight(connector, unused); + + drm_dbg_kms(&dev_priv->drm, + "CPU backlight register was enabled, switching to PCH override\n"); + + /* Write converted CPU PWM value to PCH override register */ + lpt_set_backlight(connector->base.state, val); + intel_de_write(dev_priv, BLC_PWM_PCH_CTL1, + pch_ctl1 | BLM_PCH_OVERRIDE_ENABLE); + + intel_de_write(dev_priv, BLC_PWM_CPU_CTL2, + cpu_ctl2 & ~BLM_PWM_ENABLE); + } + + return 0; +} + +static int pch_setup_backlight(struct intel_connector *connector, enum pipe unused) +{ + struct drm_i915_private *dev_priv = to_i915(connector->base.dev); + struct intel_panel *panel = &connector->panel; + u32 cpu_ctl2, pch_ctl1, pch_ctl2; + + pch_ctl1 = intel_de_read(dev_priv, BLC_PWM_PCH_CTL1); + panel->backlight.active_low_pwm = pch_ctl1 & BLM_PCH_POLARITY; + + pch_ctl2 = intel_de_read(dev_priv, BLC_PWM_PCH_CTL2); + panel->backlight.pwm_level_max = pch_ctl2 >> 16; + + if (!panel->backlight.pwm_level_max) + panel->backlight.pwm_level_max = get_backlight_max_vbt(connector); + + if (!panel->backlight.pwm_level_max) + return -ENODEV; + + panel->backlight.pwm_level_min = get_backlight_min_vbt(connector); + + cpu_ctl2 = intel_de_read(dev_priv, BLC_PWM_CPU_CTL2); + panel->backlight.pwm_enabled = (cpu_ctl2 & BLM_PWM_ENABLE) && + (pch_ctl1 & BLM_PCH_PWM_ENABLE); + + return 0; +} + +static int i9xx_setup_backlight(struct intel_connector *connector, enum pipe unused) +{ + struct drm_i915_private *dev_priv = to_i915(connector->base.dev); + struct intel_panel *panel = &connector->panel; + u32 ctl, val; + + ctl = intel_de_read(dev_priv, BLC_PWM_CTL); + + if (DISPLAY_VER(dev_priv) == 2 || IS_I915GM(dev_priv) || IS_I945GM(dev_priv)) + panel->backlight.combination_mode = ctl & BLM_LEGACY_MODE; + + if (IS_PINEVIEW(dev_priv)) + panel->backlight.active_low_pwm = ctl & BLM_POLARITY_PNV; + + panel->backlight.pwm_level_max = ctl >> 17; + + if (!panel->backlight.pwm_level_max) { + panel->backlight.pwm_level_max = get_backlight_max_vbt(connector); + panel->backlight.pwm_level_max >>= 1; + } + + if (!panel->backlight.pwm_level_max) + return -ENODEV; + + if (panel->backlight.combination_mode) + panel->backlight.pwm_level_max *= 0xff; + + panel->backlight.pwm_level_min = get_backlight_min_vbt(connector); + + val = i9xx_get_backlight(connector, unused); + val = intel_backlight_invert_pwm_level(connector, val); + val = clamp(val, panel->backlight.pwm_level_min, panel->backlight.pwm_level_max); + + panel->backlight.pwm_enabled = val != 0; + + return 0; +} + +static int i965_setup_backlight(struct intel_connector *connector, enum pipe unused) +{ + struct drm_i915_private *dev_priv = to_i915(connector->base.dev); + struct intel_panel *panel = &connector->panel; + u32 ctl, ctl2; + + ctl2 = intel_de_read(dev_priv, BLC_PWM_CTL2); + panel->backlight.combination_mode = ctl2 & BLM_COMBINATION_MODE; + panel->backlight.active_low_pwm = ctl2 & BLM_POLARITY_I965; + + ctl = intel_de_read(dev_priv, BLC_PWM_CTL); + panel->backlight.pwm_level_max = ctl >> 16; + + if (!panel->backlight.pwm_level_max) + panel->backlight.pwm_level_max = get_backlight_max_vbt(connector); + + if (!panel->backlight.pwm_level_max) + return -ENODEV; + + if (panel->backlight.combination_mode) + panel->backlight.pwm_level_max *= 0xff; + + panel->backlight.pwm_level_min = get_backlight_min_vbt(connector); + + panel->backlight.pwm_enabled = ctl2 & BLM_PWM_ENABLE; + + return 0; +} + +static int vlv_setup_backlight(struct intel_connector *connector, enum pipe pipe) +{ + struct drm_i915_private *dev_priv = to_i915(connector->base.dev); + struct intel_panel *panel = &connector->panel; + u32 ctl, ctl2; + + if (drm_WARN_ON(&dev_priv->drm, pipe != PIPE_A && pipe != PIPE_B)) + return -ENODEV; + + ctl2 = intel_de_read(dev_priv, VLV_BLC_PWM_CTL2(pipe)); + panel->backlight.active_low_pwm = ctl2 & BLM_POLARITY_I965; + + ctl = intel_de_read(dev_priv, VLV_BLC_PWM_CTL(pipe)); + panel->backlight.pwm_level_max = ctl >> 16; + + if (!panel->backlight.pwm_level_max) + panel->backlight.pwm_level_max = get_backlight_max_vbt(connector); + + if (!panel->backlight.pwm_level_max) + return -ENODEV; + + panel->backlight.pwm_level_min = get_backlight_min_vbt(connector); + + panel->backlight.pwm_enabled = ctl2 & BLM_PWM_ENABLE; + + return 0; +} + +static int +bxt_setup_backlight(struct intel_connector *connector, enum pipe unused) +{ + struct drm_i915_private *dev_priv = to_i915(connector->base.dev); + struct intel_panel *panel = &connector->panel; + u32 pwm_ctl, val; + + panel->backlight.controller = dev_priv->vbt.backlight.controller; + + pwm_ctl = intel_de_read(dev_priv, + BXT_BLC_PWM_CTL(panel->backlight.controller)); + + /* Controller 1 uses the utility pin. */ + if (panel->backlight.controller == 1) { + val = intel_de_read(dev_priv, UTIL_PIN_CTL); + panel->backlight.util_pin_active_low = + val & UTIL_PIN_POLARITY; + } + + panel->backlight.active_low_pwm = pwm_ctl & BXT_BLC_PWM_POLARITY; + panel->backlight.pwm_level_max = + intel_de_read(dev_priv, BXT_BLC_PWM_FREQ(panel->backlight.controller)); + + if (!panel->backlight.pwm_level_max) + panel->backlight.pwm_level_max = get_backlight_max_vbt(connector); + + if (!panel->backlight.pwm_level_max) + return -ENODEV; + + panel->backlight.pwm_level_min = get_backlight_min_vbt(connector); + + panel->backlight.pwm_enabled = pwm_ctl & BXT_BLC_PWM_ENABLE; + + return 0; +} + +static int +cnp_setup_backlight(struct intel_connector *connector, enum pipe unused) +{ + struct drm_i915_private *dev_priv = to_i915(connector->base.dev); + struct intel_panel *panel = &connector->panel; + u32 pwm_ctl; + + /* + * CNP has the BXT implementation of backlight, but with only one + * controller. TODO: ICP has multiple controllers but we only use + * controller 0 for now. + */ + panel->backlight.controller = 0; + + pwm_ctl = intel_de_read(dev_priv, + BXT_BLC_PWM_CTL(panel->backlight.controller)); + + panel->backlight.active_low_pwm = pwm_ctl & BXT_BLC_PWM_POLARITY; + panel->backlight.pwm_level_max = + intel_de_read(dev_priv, BXT_BLC_PWM_FREQ(panel->backlight.controller)); + + if (!panel->backlight.pwm_level_max) + panel->backlight.pwm_level_max = get_backlight_max_vbt(connector); + + if (!panel->backlight.pwm_level_max) + return -ENODEV; + + panel->backlight.pwm_level_min = get_backlight_min_vbt(connector); + + panel->backlight.pwm_enabled = pwm_ctl & BXT_BLC_PWM_ENABLE; + + return 0; +} + +static int ext_pwm_setup_backlight(struct intel_connector *connector, + enum pipe pipe) +{ + struct drm_device *dev = connector->base.dev; + struct drm_i915_private *dev_priv = to_i915(dev); + struct intel_panel *panel = &connector->panel; + const char *desc; + u32 level; + + /* Get the right PWM chip for DSI backlight according to VBT */ + if (dev_priv->vbt.dsi.config->pwm_blc == PPS_BLC_PMIC) { + panel->backlight.pwm = pwm_get(dev->dev, "pwm_pmic_backlight"); + desc = "PMIC"; + } else { + panel->backlight.pwm = pwm_get(dev->dev, "pwm_soc_backlight"); + desc = "SoC"; + } + + if (IS_ERR(panel->backlight.pwm)) { + drm_err(&dev_priv->drm, "Failed to get the %s PWM chip\n", + desc); + panel->backlight.pwm = NULL; + return -ENODEV; + } + + panel->backlight.pwm_level_max = 100; /* 100% */ + panel->backlight.pwm_level_min = get_backlight_min_vbt(connector); + + if (pwm_is_enabled(panel->backlight.pwm)) { + /* PWM is already enabled, use existing settings */ + pwm_get_state(panel->backlight.pwm, &panel->backlight.pwm_state); + + level = pwm_get_relative_duty_cycle(&panel->backlight.pwm_state, + 100); + level = intel_backlight_invert_pwm_level(connector, level); + panel->backlight.pwm_enabled = true; + + drm_dbg_kms(&dev_priv->drm, "PWM already enabled at freq %ld, VBT freq %d, level %d\n", + NSEC_PER_SEC / (unsigned long)panel->backlight.pwm_state.period, + get_vbt_pwm_freq(dev_priv), level); + } else { + /* Set period from VBT frequency, leave other settings at 0. */ + panel->backlight.pwm_state.period = + NSEC_PER_SEC / get_vbt_pwm_freq(dev_priv); + } + + drm_info(&dev_priv->drm, "Using %s PWM for LCD backlight control\n", + desc); + return 0; +} + +static void intel_pwm_set_backlight(const struct drm_connector_state *conn_state, u32 level) +{ + struct intel_connector *connector = to_intel_connector(conn_state->connector); + struct intel_panel *panel = &connector->panel; + + panel->backlight.pwm_funcs->set(conn_state, + intel_backlight_invert_pwm_level(connector, level)); +} + +static u32 intel_pwm_get_backlight(struct intel_connector *connector, enum pipe pipe) +{ + struct intel_panel *panel = &connector->panel; + + return intel_backlight_invert_pwm_level(connector, + panel->backlight.pwm_funcs->get(connector, pipe)); +} + +static void intel_pwm_enable_backlight(const struct intel_crtc_state *crtc_state, + const struct drm_connector_state *conn_state, u32 level) +{ + struct intel_connector *connector = to_intel_connector(conn_state->connector); + struct intel_panel *panel = &connector->panel; + + panel->backlight.pwm_funcs->enable(crtc_state, conn_state, + intel_backlight_invert_pwm_level(connector, level)); +} + +static void intel_pwm_disable_backlight(const struct drm_connector_state *conn_state, u32 level) +{ + struct intel_connector *connector = to_intel_connector(conn_state->connector); + struct intel_panel *panel = &connector->panel; + + panel->backlight.pwm_funcs->disable(conn_state, + intel_backlight_invert_pwm_level(connector, level)); +} + +static int intel_pwm_setup_backlight(struct intel_connector *connector, enum pipe pipe) +{ + struct intel_panel *panel = &connector->panel; + int ret = panel->backlight.pwm_funcs->setup(connector, pipe); + + if (ret < 0) + return ret; + + panel->backlight.min = panel->backlight.pwm_level_min; + panel->backlight.max = panel->backlight.pwm_level_max; + panel->backlight.level = intel_pwm_get_backlight(connector, pipe); + panel->backlight.enabled = panel->backlight.pwm_enabled; + + return 0; +} + +void intel_backlight_update(struct intel_atomic_state *state, + struct intel_encoder *encoder, + const struct intel_crtc_state *crtc_state, + const struct drm_connector_state *conn_state) +{ + struct intel_connector *connector = to_intel_connector(conn_state->connector); + struct drm_i915_private *dev_priv = to_i915(connector->base.dev); + struct intel_panel *panel = &connector->panel; + + if (!panel->backlight.present) + return; + + mutex_lock(&dev_priv->backlight_lock); + if (!panel->backlight.enabled) + __intel_backlight_enable(crtc_state, conn_state); + + mutex_unlock(&dev_priv->backlight_lock); +} + +int intel_backlight_setup(struct intel_connector *connector, enum pipe pipe) +{ + struct drm_i915_private *dev_priv = to_i915(connector->base.dev); + struct intel_panel *panel = &connector->panel; + int ret; + + if (!dev_priv->vbt.backlight.present) { + if (dev_priv->quirks & QUIRK_BACKLIGHT_PRESENT) { + drm_dbg_kms(&dev_priv->drm, + "no backlight present per VBT, but present per quirk\n"); + } else { + drm_dbg_kms(&dev_priv->drm, + "no backlight present per VBT\n"); + return 0; + } + } + + /* ensure intel_panel has been initialized first */ + if (drm_WARN_ON(&dev_priv->drm, !panel->backlight.funcs)) + return -ENODEV; + + /* set level and max in panel struct */ + mutex_lock(&dev_priv->backlight_lock); + ret = panel->backlight.funcs->setup(connector, pipe); + mutex_unlock(&dev_priv->backlight_lock); + + if (ret) { + drm_dbg_kms(&dev_priv->drm, + "failed to setup backlight for connector %s\n", + connector->base.name); + return ret; + } + + panel->backlight.present = true; + + drm_dbg_kms(&dev_priv->drm, + "Connector %s backlight initialized, %s, brightness %u/%u\n", + connector->base.name, + enableddisabled(panel->backlight.enabled), + panel->backlight.level, panel->backlight.max); + + return 0; +} + +void intel_backlight_destroy(struct intel_panel *panel) +{ + /* dispose of the pwm */ + if (panel->backlight.pwm) + pwm_put(panel->backlight.pwm); + + panel->backlight.present = false; +} + +static const struct intel_panel_bl_funcs bxt_pwm_funcs = { + .setup = bxt_setup_backlight, + .enable = bxt_enable_backlight, + .disable = bxt_disable_backlight, + .set = bxt_set_backlight, + .get = bxt_get_backlight, + .hz_to_pwm = bxt_hz_to_pwm, +}; + +static const struct intel_panel_bl_funcs cnp_pwm_funcs = { + .setup = cnp_setup_backlight, + .enable = cnp_enable_backlight, + .disable = cnp_disable_backlight, + .set = bxt_set_backlight, + .get = bxt_get_backlight, + .hz_to_pwm = cnp_hz_to_pwm, +}; + +static const struct intel_panel_bl_funcs lpt_pwm_funcs = { + .setup = lpt_setup_backlight, + .enable = lpt_enable_backlight, + .disable = lpt_disable_backlight, + .set = lpt_set_backlight, + .get = lpt_get_backlight, + .hz_to_pwm = lpt_hz_to_pwm, +}; + +static const struct intel_panel_bl_funcs spt_pwm_funcs = { + .setup = lpt_setup_backlight, + .enable = lpt_enable_backlight, + .disable = lpt_disable_backlight, + .set = lpt_set_backlight, + .get = lpt_get_backlight, + .hz_to_pwm = spt_hz_to_pwm, +}; + +static const struct intel_panel_bl_funcs pch_pwm_funcs = { + .setup = pch_setup_backlight, + .enable = pch_enable_backlight, + .disable = pch_disable_backlight, + .set = pch_set_backlight, + .get = pch_get_backlight, + .hz_to_pwm = pch_hz_to_pwm, +}; + +static const struct intel_panel_bl_funcs ext_pwm_funcs = { + .setup = ext_pwm_setup_backlight, + .enable = ext_pwm_enable_backlight, + .disable = ext_pwm_disable_backlight, + .set = ext_pwm_set_backlight, + .get = ext_pwm_get_backlight, +}; + +static const struct intel_panel_bl_funcs vlv_pwm_funcs = { + .setup = vlv_setup_backlight, + .enable = vlv_enable_backlight, + .disable = vlv_disable_backlight, + .set = vlv_set_backlight, + .get = vlv_get_backlight, + .hz_to_pwm = vlv_hz_to_pwm, +}; + +static const struct intel_panel_bl_funcs i965_pwm_funcs = { + .setup = i965_setup_backlight, + .enable = i965_enable_backlight, + .disable = i965_disable_backlight, + .set = i9xx_set_backlight, + .get = i9xx_get_backlight, + .hz_to_pwm = i965_hz_to_pwm, +}; + +static const struct intel_panel_bl_funcs i9xx_pwm_funcs = { + .setup = i9xx_setup_backlight, + .enable = i9xx_enable_backlight, + .disable = i9xx_disable_backlight, + .set = i9xx_set_backlight, + .get = i9xx_get_backlight, + .hz_to_pwm = i9xx_hz_to_pwm, +}; + +static const struct intel_panel_bl_funcs pwm_bl_funcs = { + .setup = intel_pwm_setup_backlight, + .enable = intel_pwm_enable_backlight, + .disable = intel_pwm_disable_backlight, + .set = intel_pwm_set_backlight, + .get = intel_pwm_get_backlight, +}; + +/* Set up chip specific backlight functions */ +void intel_backlight_init_funcs(struct intel_panel *panel) +{ + struct intel_connector *connector = + container_of(panel, struct intel_connector, panel); + struct drm_i915_private *dev_priv = to_i915(connector->base.dev); + + if (connector->base.connector_type == DRM_MODE_CONNECTOR_DSI && + intel_dsi_dcs_init_backlight_funcs(connector) == 0) + return; + + if (IS_GEMINILAKE(dev_priv) || IS_BROXTON(dev_priv)) { + panel->backlight.pwm_funcs = &bxt_pwm_funcs; + } else if (INTEL_PCH_TYPE(dev_priv) >= PCH_CNP) { + panel->backlight.pwm_funcs = &cnp_pwm_funcs; + } else if (INTEL_PCH_TYPE(dev_priv) >= PCH_LPT) { + if (HAS_PCH_LPT(dev_priv)) + panel->backlight.pwm_funcs = &lpt_pwm_funcs; + else + panel->backlight.pwm_funcs = &spt_pwm_funcs; + } else if (HAS_PCH_SPLIT(dev_priv)) { + panel->backlight.pwm_funcs = &pch_pwm_funcs; + } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) { + if (connector->base.connector_type == DRM_MODE_CONNECTOR_DSI) { + panel->backlight.pwm_funcs = &ext_pwm_funcs; + } else { + panel->backlight.pwm_funcs = &vlv_pwm_funcs; + } + } else if (DISPLAY_VER(dev_priv) == 4) { + panel->backlight.pwm_funcs = &i965_pwm_funcs; + } else { + panel->backlight.pwm_funcs = &i9xx_pwm_funcs; + } + + if (connector->base.connector_type == DRM_MODE_CONNECTOR_eDP && + intel_dp_aux_init_backlight_funcs(connector) == 0) + return; + + /* We're using a standard PWM backlight interface */ + panel->backlight.funcs = &pwm_bl_funcs; +} diff --git a/drivers/gpu/drm/i915/display/intel_backlight.h b/drivers/gpu/drm/i915/display/intel_backlight.h new file mode 100644 index 000000000000..339643f63897 --- /dev/null +++ b/drivers/gpu/drm/i915/display/intel_backlight.h @@ -0,0 +1,52 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2021 Intel Corporation + */ + +#ifndef __INTEL_BACKLIGHT_H__ +#define __INTEL_BACKLIGHT_H__ + +#include <linux/types.h> + +struct drm_connector_state; +struct intel_atomic_state; +struct intel_connector; +struct intel_crtc_state; +struct intel_encoder; +struct intel_panel; +enum pipe; + +void intel_backlight_init_funcs(struct intel_panel *panel); +int intel_backlight_setup(struct intel_connector *connector, enum pipe pipe); +void intel_backlight_destroy(struct intel_panel *panel); + +void intel_backlight_enable(const struct intel_crtc_state *crtc_state, + const struct drm_connector_state *conn_state); +void intel_backlight_update(struct intel_atomic_state *state, + struct intel_encoder *encoder, + const struct intel_crtc_state *crtc_state, + const struct drm_connector_state *conn_state); +void intel_backlight_disable(const struct drm_connector_state *old_conn_state); + +void intel_backlight_set_acpi(const struct drm_connector_state *conn_state, + u32 level, u32 max); +void intel_backlight_set_pwm_level(const struct drm_connector_state *conn_state, + u32 level); +u32 intel_backlight_invert_pwm_level(struct intel_connector *connector, u32 level); +u32 intel_backlight_level_to_pwm(struct intel_connector *connector, u32 level); +u32 intel_backlight_level_from_pwm(struct intel_connector *connector, u32 val); + +#if IS_ENABLED(CONFIG_BACKLIGHT_CLASS_DEVICE) +int intel_backlight_device_register(struct intel_connector *connector); +void intel_backlight_device_unregister(struct intel_connector *connector); +#else /* CONFIG_BACKLIGHT_CLASS_DEVICE */ +static inline int intel_backlight_device_register(struct intel_connector *connector) +{ + return 0; +} +static inline void intel_backlight_device_unregister(struct intel_connector *connector) +{ +} +#endif /* CONFIG_BACKLIGHT_CLASS_DEVICE */ + +#endif /* __INTEL_BACKLIGHT_H__ */ diff --git a/drivers/gpu/drm/i915/display/intel_bios.c b/drivers/gpu/drm/i915/display/intel_bios.c index fd71346aac7b..b99907c656bb 100644 --- a/drivers/gpu/drm/i915/display/intel_bios.c +++ b/drivers/gpu/drm/i915/display/intel_bios.c @@ -493,6 +493,9 @@ parse_lfp_backlight(struct drm_i915_private *i915, level = 255; } i915->vbt.backlight.min_brightness = min_level; + + i915->vbt.backlight.brightness_precision_bits = + backlight_data->brightness_precision_bits[panel_type]; } else { level = backlight_data->level[panel_type]; i915->vbt.backlight.min_brightness = entry->min_brightness; @@ -1511,39 +1514,130 @@ static u8 translate_iboost(u8 val) return mapping[val]; } +static const u8 cnp_ddc_pin_map[] = { + [0] = 0, /* N/A */ + [DDC_BUS_DDI_B] = GMBUS_PIN_1_BXT, + [DDC_BUS_DDI_C] = GMBUS_PIN_2_BXT, + [DDC_BUS_DDI_D] = GMBUS_PIN_4_CNP, /* sic */ + [DDC_BUS_DDI_F] = GMBUS_PIN_3_BXT, /* sic */ +}; + +static const u8 icp_ddc_pin_map[] = { + [ICL_DDC_BUS_DDI_A] = GMBUS_PIN_1_BXT, + [ICL_DDC_BUS_DDI_B] = GMBUS_PIN_2_BXT, + [TGL_DDC_BUS_DDI_C] = GMBUS_PIN_3_BXT, + [ICL_DDC_BUS_PORT_1] = GMBUS_PIN_9_TC1_ICP, + [ICL_DDC_BUS_PORT_2] = GMBUS_PIN_10_TC2_ICP, + [ICL_DDC_BUS_PORT_3] = GMBUS_PIN_11_TC3_ICP, + [ICL_DDC_BUS_PORT_4] = GMBUS_PIN_12_TC4_ICP, + [TGL_DDC_BUS_PORT_5] = GMBUS_PIN_13_TC5_TGP, + [TGL_DDC_BUS_PORT_6] = GMBUS_PIN_14_TC6_TGP, +}; + +static const u8 rkl_pch_tgp_ddc_pin_map[] = { + [ICL_DDC_BUS_DDI_A] = GMBUS_PIN_1_BXT, + [ICL_DDC_BUS_DDI_B] = GMBUS_PIN_2_BXT, + [RKL_DDC_BUS_DDI_D] = GMBUS_PIN_9_TC1_ICP, + [RKL_DDC_BUS_DDI_E] = GMBUS_PIN_10_TC2_ICP, +}; + +static const u8 adls_ddc_pin_map[] = { + [ICL_DDC_BUS_DDI_A] = GMBUS_PIN_1_BXT, + [ADLS_DDC_BUS_PORT_TC1] = GMBUS_PIN_9_TC1_ICP, + [ADLS_DDC_BUS_PORT_TC2] = GMBUS_PIN_10_TC2_ICP, + [ADLS_DDC_BUS_PORT_TC3] = GMBUS_PIN_11_TC3_ICP, + [ADLS_DDC_BUS_PORT_TC4] = GMBUS_PIN_12_TC4_ICP, +}; + +static const u8 gen9bc_tgp_ddc_pin_map[] = { + [DDC_BUS_DDI_B] = GMBUS_PIN_2_BXT, + [DDC_BUS_DDI_C] = GMBUS_PIN_9_TC1_ICP, + [DDC_BUS_DDI_D] = GMBUS_PIN_10_TC2_ICP, +}; + +static u8 map_ddc_pin(struct drm_i915_private *i915, u8 vbt_pin) +{ + const u8 *ddc_pin_map; + int n_entries; + + if (IS_ALDERLAKE_S(i915)) { + ddc_pin_map = adls_ddc_pin_map; + n_entries = ARRAY_SIZE(adls_ddc_pin_map); + } else if (INTEL_PCH_TYPE(i915) >= PCH_DG1) { + return vbt_pin; + } else if (IS_ROCKETLAKE(i915) && INTEL_PCH_TYPE(i915) == PCH_TGP) { + ddc_pin_map = rkl_pch_tgp_ddc_pin_map; + n_entries = ARRAY_SIZE(rkl_pch_tgp_ddc_pin_map); + } else if (HAS_PCH_TGP(i915) && DISPLAY_VER(i915) == 9) { + ddc_pin_map = gen9bc_tgp_ddc_pin_map; + n_entries = ARRAY_SIZE(gen9bc_tgp_ddc_pin_map); + } else if (INTEL_PCH_TYPE(i915) >= PCH_ICP) { + ddc_pin_map = icp_ddc_pin_map; + n_entries = ARRAY_SIZE(icp_ddc_pin_map); + } else if (HAS_PCH_CNP(i915)) { + ddc_pin_map = cnp_ddc_pin_map; + n_entries = ARRAY_SIZE(cnp_ddc_pin_map); + } else { + /* Assuming direct map */ + return vbt_pin; + } + + if (vbt_pin < n_entries && ddc_pin_map[vbt_pin] != 0) + return ddc_pin_map[vbt_pin]; + + drm_dbg_kms(&i915->drm, + "Ignoring alternate pin: VBT claims DDC pin %d, which is not valid for this platform\n", + vbt_pin); + return 0; +} + static enum port get_port_by_ddc_pin(struct drm_i915_private *i915, u8 ddc_pin) { - const struct ddi_vbt_port_info *info; + const struct intel_bios_encoder_data *devdata; enum port port; if (!ddc_pin) return PORT_NONE; for_each_port(port) { - info = &i915->vbt.ddi_port_info[port]; + devdata = i915->vbt.ports[port]; - if (info->devdata && ddc_pin == info->alternate_ddc_pin) + if (devdata && ddc_pin == devdata->child.ddc_pin) return port; } return PORT_NONE; } -static void sanitize_ddc_pin(struct drm_i915_private *i915, +static void sanitize_ddc_pin(struct intel_bios_encoder_data *devdata, enum port port) { - struct ddi_vbt_port_info *info = &i915->vbt.ddi_port_info[port]; + struct drm_i915_private *i915 = devdata->i915; struct child_device_config *child; + u8 mapped_ddc_pin; enum port p; - p = get_port_by_ddc_pin(i915, info->alternate_ddc_pin); + if (!devdata->child.ddc_pin) + return; + + mapped_ddc_pin = map_ddc_pin(i915, devdata->child.ddc_pin); + if (!intel_gmbus_is_valid_pin(i915, mapped_ddc_pin)) { + drm_dbg_kms(&i915->drm, + "Port %c has invalid DDC pin %d, " + "sticking to defaults\n", + port_name(port), mapped_ddc_pin); + devdata->child.ddc_pin = 0; + return; + } + + p = get_port_by_ddc_pin(i915, devdata->child.ddc_pin); if (p == PORT_NONE) return; drm_dbg_kms(&i915->drm, "port %c trying to use the same DDC pin (0x%x) as port %c, " "disabling port %c DVI/HDMI support\n", - port_name(port), info->alternate_ddc_pin, + port_name(port), mapped_ddc_pin, port_name(p), port_name(p)); /* @@ -1555,48 +1649,47 @@ static void sanitize_ddc_pin(struct drm_i915_private *i915, * there are real machines (eg. Asrock B250M-HDV) where VBT has both * port A and port E with the same AUX ch and we must pick port E :( */ - info = &i915->vbt.ddi_port_info[p]; - child = &info->devdata->child; + child = &i915->vbt.ports[p]->child; child->device_type &= ~DEVICE_TYPE_TMDS_DVI_SIGNALING; child->device_type |= DEVICE_TYPE_NOT_HDMI_OUTPUT; - info->alternate_ddc_pin = 0; + child->ddc_pin = 0; } static enum port get_port_by_aux_ch(struct drm_i915_private *i915, u8 aux_ch) { - const struct ddi_vbt_port_info *info; + const struct intel_bios_encoder_data *devdata; enum port port; if (!aux_ch) return PORT_NONE; for_each_port(port) { - info = &i915->vbt.ddi_port_info[port]; + devdata = i915->vbt.ports[port]; - if (info->devdata && aux_ch == info->alternate_aux_channel) + if (devdata && aux_ch == devdata->child.aux_channel) return port; } return PORT_NONE; } -static void sanitize_aux_ch(struct drm_i915_private *i915, +static void sanitize_aux_ch(struct intel_bios_encoder_data *devdata, enum port port) { - struct ddi_vbt_port_info *info = &i915->vbt.ddi_port_info[port]; + struct drm_i915_private *i915 = devdata->i915; struct child_device_config *child; enum port p; - p = get_port_by_aux_ch(i915, info->alternate_aux_channel); + p = get_port_by_aux_ch(i915, devdata->child.aux_channel); if (p == PORT_NONE) return; drm_dbg_kms(&i915->drm, "port %c trying to use the same AUX CH (0x%x) as port %c, " "disabling port %c DP support\n", - port_name(port), info->alternate_aux_channel, + port_name(port), devdata->child.aux_channel, port_name(p), port_name(p)); /* @@ -1608,88 +1701,10 @@ static void sanitize_aux_ch(struct drm_i915_private *i915, * there are real machines (eg. Asrock B250M-HDV) where VBT has both * port A and port E with the same AUX ch and we must pick port E :( */ - info = &i915->vbt.ddi_port_info[p]; - child = &info->devdata->child; + child = &i915->vbt.ports[p]->child; child->device_type &= ~DEVICE_TYPE_DISPLAYPORT_OUTPUT; - info->alternate_aux_channel = 0; -} - -static const u8 cnp_ddc_pin_map[] = { - [0] = 0, /* N/A */ - [DDC_BUS_DDI_B] = GMBUS_PIN_1_BXT, - [DDC_BUS_DDI_C] = GMBUS_PIN_2_BXT, - [DDC_BUS_DDI_D] = GMBUS_PIN_4_CNP, /* sic */ - [DDC_BUS_DDI_F] = GMBUS_PIN_3_BXT, /* sic */ -}; - -static const u8 icp_ddc_pin_map[] = { - [ICL_DDC_BUS_DDI_A] = GMBUS_PIN_1_BXT, - [ICL_DDC_BUS_DDI_B] = GMBUS_PIN_2_BXT, - [TGL_DDC_BUS_DDI_C] = GMBUS_PIN_3_BXT, - [ICL_DDC_BUS_PORT_1] = GMBUS_PIN_9_TC1_ICP, - [ICL_DDC_BUS_PORT_2] = GMBUS_PIN_10_TC2_ICP, - [ICL_DDC_BUS_PORT_3] = GMBUS_PIN_11_TC3_ICP, - [ICL_DDC_BUS_PORT_4] = GMBUS_PIN_12_TC4_ICP, - [TGL_DDC_BUS_PORT_5] = GMBUS_PIN_13_TC5_TGP, - [TGL_DDC_BUS_PORT_6] = GMBUS_PIN_14_TC6_TGP, -}; - -static const u8 rkl_pch_tgp_ddc_pin_map[] = { - [ICL_DDC_BUS_DDI_A] = GMBUS_PIN_1_BXT, - [ICL_DDC_BUS_DDI_B] = GMBUS_PIN_2_BXT, - [RKL_DDC_BUS_DDI_D] = GMBUS_PIN_9_TC1_ICP, - [RKL_DDC_BUS_DDI_E] = GMBUS_PIN_10_TC2_ICP, -}; - -static const u8 adls_ddc_pin_map[] = { - [ICL_DDC_BUS_DDI_A] = GMBUS_PIN_1_BXT, - [ADLS_DDC_BUS_PORT_TC1] = GMBUS_PIN_9_TC1_ICP, - [ADLS_DDC_BUS_PORT_TC2] = GMBUS_PIN_10_TC2_ICP, - [ADLS_DDC_BUS_PORT_TC3] = GMBUS_PIN_11_TC3_ICP, - [ADLS_DDC_BUS_PORT_TC4] = GMBUS_PIN_12_TC4_ICP, -}; - -static const u8 gen9bc_tgp_ddc_pin_map[] = { - [DDC_BUS_DDI_B] = GMBUS_PIN_2_BXT, - [DDC_BUS_DDI_C] = GMBUS_PIN_9_TC1_ICP, - [DDC_BUS_DDI_D] = GMBUS_PIN_10_TC2_ICP, -}; - -static u8 map_ddc_pin(struct drm_i915_private *i915, u8 vbt_pin) -{ - const u8 *ddc_pin_map; - int n_entries; - - if (IS_ALDERLAKE_S(i915)) { - ddc_pin_map = adls_ddc_pin_map; - n_entries = ARRAY_SIZE(adls_ddc_pin_map); - } else if (INTEL_PCH_TYPE(i915) >= PCH_DG1) { - return vbt_pin; - } else if (IS_ROCKETLAKE(i915) && INTEL_PCH_TYPE(i915) == PCH_TGP) { - ddc_pin_map = rkl_pch_tgp_ddc_pin_map; - n_entries = ARRAY_SIZE(rkl_pch_tgp_ddc_pin_map); - } else if (HAS_PCH_TGP(i915) && DISPLAY_VER(i915) == 9) { - ddc_pin_map = gen9bc_tgp_ddc_pin_map; - n_entries = ARRAY_SIZE(gen9bc_tgp_ddc_pin_map); - } else if (INTEL_PCH_TYPE(i915) >= PCH_ICP) { - ddc_pin_map = icp_ddc_pin_map; - n_entries = ARRAY_SIZE(icp_ddc_pin_map); - } else if (HAS_PCH_CNP(i915)) { - ddc_pin_map = cnp_ddc_pin_map; - n_entries = ARRAY_SIZE(cnp_ddc_pin_map); - } else { - /* Assuming direct map */ - return vbt_pin; - } - - if (vbt_pin < n_entries && ddc_pin_map[vbt_pin] != 0) - return ddc_pin_map[vbt_pin]; - - drm_dbg_kms(&i915->drm, - "Ignoring alternate pin: VBT claims DDC pin %d, which is not valid for this platform\n", - vbt_pin); - return 0; + child->aux_channel = 0; } static enum port __dvo_port_to_port(int n_ports, int n_dvo, @@ -1825,6 +1840,17 @@ static int parse_bdb_216_dp_max_link_rate(const int vbt_max_link_rate) } } +static int _intel_bios_dp_max_link_rate(const struct intel_bios_encoder_data *devdata) +{ + if (!devdata || devdata->i915->vbt.version < 216) + return 0; + + if (devdata->i915->vbt.version >= 230) + return parse_bdb_230_dp_max_link_rate(devdata->child.dp_max_link_rate); + else + return parse_bdb_216_dp_max_link_rate(devdata->child.dp_max_link_rate); +} + static void sanitize_device_type(struct intel_bios_encoder_data *devdata, enum port port) { @@ -1878,6 +1904,76 @@ intel_bios_encoder_supports_edp(const struct intel_bios_encoder_data *devdata) devdata->child.device_type & DEVICE_TYPE_INTERNAL_CONNECTOR; } +static int _intel_bios_hdmi_level_shift(const struct intel_bios_encoder_data *devdata) +{ + if (!devdata || devdata->i915->vbt.version < 158) + return -1; + + return devdata->child.hdmi_level_shifter_value; +} + +static int _intel_bios_max_tmds_clock(const struct intel_bios_encoder_data *devdata) +{ + if (!devdata || devdata->i915->vbt.version < 204) + return 0; + + switch (devdata->child.hdmi_max_data_rate) { + default: + MISSING_CASE(devdata->child.hdmi_max_data_rate); + fallthrough; + case HDMI_MAX_DATA_RATE_PLATFORM: + return 0; + case HDMI_MAX_DATA_RATE_297: + return 297000; + case HDMI_MAX_DATA_RATE_165: + return 165000; + } +} + +static enum port get_edp_port(struct drm_i915_private *i915) +{ + const struct intel_bios_encoder_data *devdata; + enum port port; + + for_each_port(port) { + devdata = i915->vbt.ports[port]; + + if (devdata && intel_bios_encoder_supports_edp(devdata)) + return port; + } + + return PORT_NONE; +} + +/* + * FIXME: The power sequencer and backlight code currently do not support more + * than one set registers, at least not on anything other than VLV/CHV. It will + * clobber the registers. As a temporary workaround, gracefully prevent more + * than one eDP from being registered. + */ +static void sanitize_dual_edp(struct intel_bios_encoder_data *devdata, + enum port port) +{ + struct drm_i915_private *i915 = devdata->i915; + struct child_device_config *child = &devdata->child; + enum port p; + + /* CHV might not clobber PPS registers. */ + if (IS_CHERRYVIEW(i915)) + return; + + p = get_edp_port(i915); + if (p == PORT_NONE) + return; + + drm_dbg_kms(&i915->drm, "both ports %c and %c configured as eDP, " + "disabling port %c eDP\n", port_name(p), port_name(port), + port_name(port)); + + child->device_type &= ~DEVICE_TYPE_DISPLAYPORT_OUTPUT; + child->device_type &= ~DEVICE_TYPE_INTERNAL_CONNECTOR; +} + static bool is_port_valid(struct drm_i915_private *i915, enum port port) { /* @@ -1895,9 +1991,8 @@ static void parse_ddi_port(struct drm_i915_private *i915, struct intel_bios_encoder_data *devdata) { const struct child_device_config *child = &devdata->child; - struct ddi_vbt_port_info *info; bool is_dvi, is_hdmi, is_dp, is_edp, is_crt, supports_typec_usb, supports_tbt; - int dp_boost_level, hdmi_boost_level; + int dp_boost_level, dp_max_link_rate, hdmi_boost_level, hdmi_level_shift, max_tmds_clock; enum port port; port = dvo_port_to_port(i915, child->dvo_port); @@ -1911,9 +2006,7 @@ static void parse_ddi_port(struct drm_i915_private *i915, return; } - info = &i915->vbt.ddi_port_info[port]; - - if (info->devdata) { + if (i915->vbt.ports[port]) { drm_dbg_kms(&i915->drm, "More than one child device for port %c in VBT, using the first.\n", port_name(port)); @@ -1938,62 +2031,27 @@ static void parse_ddi_port(struct drm_i915_private *i915, supports_typec_usb, supports_tbt, devdata->dsc != NULL); - if (is_dvi) { - u8 ddc_pin; + if (is_edp) + sanitize_dual_edp(devdata, port); - ddc_pin = map_ddc_pin(i915, child->ddc_pin); - if (intel_gmbus_is_valid_pin(i915, ddc_pin)) { - info->alternate_ddc_pin = ddc_pin; - sanitize_ddc_pin(i915, port); - } else { - drm_dbg_kms(&i915->drm, - "Port %c has invalid DDC pin %d, " - "sticking to defaults\n", - port_name(port), ddc_pin); - } - } + if (is_dvi) + sanitize_ddc_pin(devdata, port); - if (is_dp) { - info->alternate_aux_channel = child->aux_channel; + if (is_dp) + sanitize_aux_ch(devdata, port); - sanitize_aux_ch(i915, port); - } - - if (i915->vbt.version >= 158) { - /* The VBT HDMI level shift values match the table we have. */ - u8 hdmi_level_shift = child->hdmi_level_shifter_value; + hdmi_level_shift = _intel_bios_hdmi_level_shift(devdata); + if (hdmi_level_shift >= 0) { drm_dbg_kms(&i915->drm, "Port %c VBT HDMI level shift: %d\n", - port_name(port), - hdmi_level_shift); - info->hdmi_level_shift = hdmi_level_shift; - info->hdmi_level_shift_set = true; + port_name(port), hdmi_level_shift); } - if (i915->vbt.version >= 204) { - int max_tmds_clock; - - switch (child->hdmi_max_data_rate) { - default: - MISSING_CASE(child->hdmi_max_data_rate); - fallthrough; - case HDMI_MAX_DATA_RATE_PLATFORM: - max_tmds_clock = 0; - break; - case HDMI_MAX_DATA_RATE_297: - max_tmds_clock = 297000; - break; - case HDMI_MAX_DATA_RATE_165: - max_tmds_clock = 165000; - break; - } - - if (max_tmds_clock) - drm_dbg_kms(&i915->drm, - "Port %c VBT HDMI max TMDS clock: %d kHz\n", - port_name(port), max_tmds_clock); - info->max_tmds_clock = max_tmds_clock; - } + max_tmds_clock = _intel_bios_max_tmds_clock(devdata); + if (max_tmds_clock) + drm_dbg_kms(&i915->drm, + "Port %c VBT HDMI max TMDS clock: %d kHz\n", + port_name(port), max_tmds_clock); /* I_boost config for SKL and above */ dp_boost_level = intel_bios_encoder_dp_boost_level(devdata); @@ -2008,19 +2066,13 @@ static void parse_ddi_port(struct drm_i915_private *i915, "Port %c VBT HDMI boost level: %d\n", port_name(port), hdmi_boost_level); - /* DP max link rate for GLK+ */ - if (i915->vbt.version >= 216) { - if (i915->vbt.version >= 230) - info->dp_max_link_rate = parse_bdb_230_dp_max_link_rate(child->dp_max_link_rate); - else - info->dp_max_link_rate = parse_bdb_216_dp_max_link_rate(child->dp_max_link_rate); - + dp_max_link_rate = _intel_bios_dp_max_link_rate(devdata); + if (dp_max_link_rate) drm_dbg_kms(&i915->drm, "Port %c VBT DP max link rate: %d\n", - port_name(port), info->dp_max_link_rate); - } + port_name(port), dp_max_link_rate); - info->devdata = devdata; + i915->vbt.ports[port] = devdata; } static void parse_ddi_ports(struct drm_i915_private *i915) @@ -2558,12 +2610,8 @@ bool intel_bios_is_port_present(struct drm_i915_private *i915, enum port port) [PORT_F] = { DVO_PORT_DPF, DVO_PORT_HDMIF, }, }; - if (HAS_DDI(i915)) { - const struct ddi_vbt_port_info *port_info = - &i915->vbt.ddi_port_info[port]; - - return port_info->devdata; - } + if (HAS_DDI(i915)) + return i915->vbt.ports[port]; /* FIXME maybe deal with port A as well? */ if (drm_WARN_ON(&i915->drm, @@ -2814,8 +2862,7 @@ bool intel_bios_is_port_hpd_inverted(const struct drm_i915_private *i915, enum port port) { - const struct intel_bios_encoder_data *devdata = - i915->vbt.ddi_port_info[port].devdata; + const struct intel_bios_encoder_data *devdata = i915->vbt.ports[port]; if (drm_WARN_ON_ONCE(&i915->drm, !IS_GEMINILAKE(i915) && !IS_BROXTON(i915))) @@ -2835,8 +2882,7 @@ bool intel_bios_is_lspcon_present(const struct drm_i915_private *i915, enum port port) { - const struct intel_bios_encoder_data *devdata = - i915->vbt.ddi_port_info[port].devdata; + const struct intel_bios_encoder_data *devdata = i915->vbt.ports[port]; return HAS_LSPCON(i915) && devdata && devdata->child.lspcon; } @@ -2852,8 +2898,7 @@ bool intel_bios_is_lane_reversal_needed(const struct drm_i915_private *i915, enum port port) { - const struct intel_bios_encoder_data *devdata = - i915->vbt.ddi_port_info[port].devdata; + const struct intel_bios_encoder_data *devdata = i915->vbt.ports[port]; return devdata && devdata->child.lane_reversal; } @@ -2861,11 +2906,10 @@ intel_bios_is_lane_reversal_needed(const struct drm_i915_private *i915, enum aux_ch intel_bios_port_aux_ch(struct drm_i915_private *i915, enum port port) { - const struct ddi_vbt_port_info *info = - &i915->vbt.ddi_port_info[port]; + const struct intel_bios_encoder_data *devdata = i915->vbt.ports[port]; enum aux_ch aux_ch; - if (!info->alternate_aux_channel) { + if (!devdata || !devdata->child.aux_channel) { aux_ch = (enum aux_ch)port; drm_dbg_kms(&i915->drm, @@ -2881,7 +2925,7 @@ enum aux_ch intel_bios_port_aux_ch(struct drm_i915_private *i915, * ADL-S VBT uses PHY based mapping. Combo PHYs A,B,C,D,E * map to DDI A,TC1,TC2,TC3,TC4 respectively. */ - switch (info->alternate_aux_channel) { + switch (devdata->child.aux_channel) { case DP_AUX_A: aux_ch = AUX_CH_A; break; @@ -2942,7 +2986,7 @@ enum aux_ch intel_bios_port_aux_ch(struct drm_i915_private *i915, aux_ch = AUX_CH_I; break; default: - MISSING_CASE(info->alternate_aux_channel); + MISSING_CASE(devdata->child.aux_channel); aux_ch = AUX_CH_A; break; } @@ -2956,17 +3000,18 @@ enum aux_ch intel_bios_port_aux_ch(struct drm_i915_private *i915, int intel_bios_max_tmds_clock(struct intel_encoder *encoder) { struct drm_i915_private *i915 = to_i915(encoder->base.dev); + const struct intel_bios_encoder_data *devdata = i915->vbt.ports[encoder->port]; - return i915->vbt.ddi_port_info[encoder->port].max_tmds_clock; + return _intel_bios_max_tmds_clock(devdata); } +/* This is an index in the HDMI/DVI DDI buffer translation table, or -1 */ int intel_bios_hdmi_level_shift(struct intel_encoder *encoder) { struct drm_i915_private *i915 = to_i915(encoder->base.dev); - const struct ddi_vbt_port_info *info = - &i915->vbt.ddi_port_info[encoder->port]; + const struct intel_bios_encoder_data *devdata = i915->vbt.ports[encoder->port]; - return info->hdmi_level_shift_set ? info->hdmi_level_shift : -1; + return _intel_bios_hdmi_level_shift(devdata); } int intel_bios_encoder_dp_boost_level(const struct intel_bios_encoder_data *devdata) @@ -2988,15 +3033,20 @@ int intel_bios_encoder_hdmi_boost_level(const struct intel_bios_encoder_data *de int intel_bios_dp_max_link_rate(struct intel_encoder *encoder) { struct drm_i915_private *i915 = to_i915(encoder->base.dev); + const struct intel_bios_encoder_data *devdata = i915->vbt.ports[encoder->port]; - return i915->vbt.ddi_port_info[encoder->port].dp_max_link_rate; + return _intel_bios_dp_max_link_rate(devdata); } int intel_bios_alternate_ddc_pin(struct intel_encoder *encoder) { struct drm_i915_private *i915 = to_i915(encoder->base.dev); + const struct intel_bios_encoder_data *devdata = i915->vbt.ports[encoder->port]; + + if (!devdata || !devdata->child.ddc_pin) + return 0; - return i915->vbt.ddi_port_info[encoder->port].alternate_ddc_pin; + return map_ddc_pin(i915, devdata->child.ddc_pin); } bool intel_bios_encoder_supports_typec_usb(const struct intel_bios_encoder_data *devdata) @@ -3012,5 +3062,5 @@ bool intel_bios_encoder_supports_tbt(const struct intel_bios_encoder_data *devda const struct intel_bios_encoder_data * intel_bios_encoder_data_lookup(struct drm_i915_private *i915, enum port port) { - return i915->vbt.ddi_port_info[port].devdata; + return i915->vbt.ports[port]; } diff --git a/drivers/gpu/drm/i915/display/intel_bw.c b/drivers/gpu/drm/i915/display/intel_bw.c index 4b94256d7319..8d9d888e9316 100644 --- a/drivers/gpu/drm/i915/display/intel_bw.c +++ b/drivers/gpu/drm/i915/display/intel_bw.c @@ -9,8 +9,8 @@ #include "intel_bw.h" #include "intel_cdclk.h" #include "intel_display_types.h" +#include "intel_pcode.h" #include "intel_pm.h" -#include "intel_sideband.h" /* Parameters for Qclk Geyserville (QGV) */ struct intel_qgv_point { diff --git a/drivers/gpu/drm/i915/display/intel_cdclk.c b/drivers/gpu/drm/i915/display/intel_cdclk.c index 34fa4130d5c4..9e466d829019 100644 --- a/drivers/gpu/drm/i915/display/intel_cdclk.c +++ b/drivers/gpu/drm/i915/display/intel_cdclk.c @@ -28,8 +28,9 @@ #include "intel_cdclk.h" #include "intel_de.h" #include "intel_display_types.h" +#include "intel_pcode.h" #include "intel_psr.h" -#include "intel_sideband.h" +#include "vlv_sideband.h" /** * DOC: CDCLK / RAWCLK @@ -59,6 +60,37 @@ * dividers can be programmed correctly. */ +void intel_cdclk_get_cdclk(struct drm_i915_private *dev_priv, + struct intel_cdclk_config *cdclk_config) +{ + dev_priv->cdclk_funcs->get_cdclk(dev_priv, cdclk_config); +} + +int intel_cdclk_bw_calc_min_cdclk(struct intel_atomic_state *state) +{ + struct drm_i915_private *dev_priv = to_i915(state->base.dev); + return dev_priv->cdclk_funcs->bw_calc_min_cdclk(state); +} + +static void intel_cdclk_set_cdclk(struct drm_i915_private *dev_priv, + const struct intel_cdclk_config *cdclk_config, + enum pipe pipe) +{ + dev_priv->cdclk_funcs->set_cdclk(dev_priv, cdclk_config, pipe); +} + +static int intel_cdclk_modeset_calc_cdclk(struct drm_i915_private *dev_priv, + struct intel_cdclk_state *cdclk_config) +{ + return dev_priv->cdclk_funcs->modeset_calc_cdclk(cdclk_config); +} + +static u8 intel_cdclk_calc_voltage_level(struct drm_i915_private *dev_priv, + int cdclk) +{ + return dev_priv->cdclk_funcs->calc_voltage_level(cdclk); +} + static void fixed_133mhz_get_cdclk(struct drm_i915_private *dev_priv, struct intel_cdclk_config *cdclk_config) { @@ -1466,7 +1498,7 @@ static void bxt_get_cdclk(struct drm_i915_private *dev_priv, * at least what the CDCLK frequency requires. */ cdclk_config->voltage_level = - dev_priv->display.calc_voltage_level(cdclk_config->cdclk); + intel_cdclk_calc_voltage_level(dev_priv, cdclk_config->cdclk); } static void bxt_de_pll_disable(struct drm_i915_private *dev_priv) @@ -1777,7 +1809,7 @@ static void bxt_cdclk_init_hw(struct drm_i915_private *dev_priv) cdclk_config.cdclk = bxt_calc_cdclk(dev_priv, 0); cdclk_config.vco = bxt_calc_cdclk_pll_vco(dev_priv, cdclk_config.cdclk); cdclk_config.voltage_level = - dev_priv->display.calc_voltage_level(cdclk_config.cdclk); + intel_cdclk_calc_voltage_level(dev_priv, cdclk_config.cdclk); bxt_set_cdclk(dev_priv, &cdclk_config, INVALID_PIPE); } @@ -1789,7 +1821,7 @@ static void bxt_cdclk_uninit_hw(struct drm_i915_private *dev_priv) cdclk_config.cdclk = cdclk_config.bypass; cdclk_config.vco = 0; cdclk_config.voltage_level = - dev_priv->display.calc_voltage_level(cdclk_config.cdclk); + intel_cdclk_calc_voltage_level(dev_priv, cdclk_config.cdclk); bxt_set_cdclk(dev_priv, &cdclk_config, INVALID_PIPE); } @@ -1932,7 +1964,7 @@ static void intel_set_cdclk(struct drm_i915_private *dev_priv, if (!intel_cdclk_changed(&dev_priv->cdclk.hw, cdclk_config)) return; - if (drm_WARN_ON_ONCE(&dev_priv->drm, !dev_priv->display.set_cdclk)) + if (drm_WARN_ON_ONCE(&dev_priv->drm, !dev_priv->cdclk_funcs->set_cdclk)) return; intel_dump_cdclk_config(cdclk_config, "Changing CDCLK to"); @@ -1956,7 +1988,7 @@ static void intel_set_cdclk(struct drm_i915_private *dev_priv, &dev_priv->gmbus_mutex); } - dev_priv->display.set_cdclk(dev_priv, cdclk_config, pipe); + intel_cdclk_set_cdclk(dev_priv, cdclk_config, pipe); for_each_intel_dp(&dev_priv->drm, encoder) { struct intel_dp *intel_dp = enc_to_intel_dp(encoder); @@ -2140,6 +2172,14 @@ int intel_crtc_compute_min_cdclk(const struct intel_crtc_state *crtc_state) min_cdclk = max(intel_planes_min_cdclk(crtc_state), min_cdclk); /* + * When we decide to use only one VDSC engine, since + * each VDSC operates with 1 ppc throughput, pixel clock + * cannot be higher than the VDSC clock (cdclk) + */ + if (crtc_state->dsc.compression_enable && !crtc_state->dsc.dsc_split) + min_cdclk = max(min_cdclk, (int)crtc_state->pixel_rate); + + /* * HACK. Currently for TGL platforms we calculate * min_cdclk initially based on pixel_rate divided * by 2, accounting for also plane requirements, @@ -2414,7 +2454,7 @@ static int bxt_modeset_calc_cdclk(struct intel_cdclk_state *cdclk_state) cdclk_state->logical.cdclk = cdclk; cdclk_state->logical.voltage_level = max_t(int, min_voltage_level, - dev_priv->display.calc_voltage_level(cdclk)); + intel_cdclk_calc_voltage_level(dev_priv, cdclk)); if (!cdclk_state->active_pipes) { cdclk = bxt_calc_cdclk(dev_priv, cdclk_state->force_min_cdclk); @@ -2423,7 +2463,7 @@ static int bxt_modeset_calc_cdclk(struct intel_cdclk_state *cdclk_state) cdclk_state->actual.vco = vco; cdclk_state->actual.cdclk = cdclk; cdclk_state->actual.voltage_level = - dev_priv->display.calc_voltage_level(cdclk); + intel_cdclk_calc_voltage_level(dev_priv, cdclk); } else { cdclk_state->actual = cdclk_state->logical; } @@ -2515,7 +2555,7 @@ int intel_modeset_calc_cdclk(struct intel_atomic_state *state) new_cdclk_state->active_pipes = intel_calc_active_pipes(state, old_cdclk_state->active_pipes); - ret = dev_priv->display.modeset_calc_cdclk(new_cdclk_state); + ret = intel_cdclk_modeset_calc_cdclk(dev_priv, new_cdclk_state); if (ret) return ret; @@ -2695,7 +2735,7 @@ void intel_update_max_cdclk(struct drm_i915_private *dev_priv) */ void intel_update_cdclk(struct drm_i915_private *dev_priv) { - dev_priv->display.get_cdclk(dev_priv, &dev_priv->cdclk.hw); + intel_cdclk_get_cdclk(dev_priv, &dev_priv->cdclk.hw); /* * 9:0 CMBUS [sic] CDCLK frequency (cdfreq): @@ -2845,6 +2885,157 @@ u32 intel_read_rawclk(struct drm_i915_private *dev_priv) return freq; } +static struct intel_cdclk_funcs tgl_cdclk_funcs = { + .get_cdclk = bxt_get_cdclk, + .set_cdclk = bxt_set_cdclk, + .bw_calc_min_cdclk = skl_bw_calc_min_cdclk, + .modeset_calc_cdclk = bxt_modeset_calc_cdclk, + .calc_voltage_level = tgl_calc_voltage_level, +}; + +static struct intel_cdclk_funcs ehl_cdclk_funcs = { + .get_cdclk = bxt_get_cdclk, + .set_cdclk = bxt_set_cdclk, + .bw_calc_min_cdclk = skl_bw_calc_min_cdclk, + .modeset_calc_cdclk = bxt_modeset_calc_cdclk, + .calc_voltage_level = ehl_calc_voltage_level, +}; + +static struct intel_cdclk_funcs icl_cdclk_funcs = { + .get_cdclk = bxt_get_cdclk, + .set_cdclk = bxt_set_cdclk, + .bw_calc_min_cdclk = skl_bw_calc_min_cdclk, + .modeset_calc_cdclk = bxt_modeset_calc_cdclk, + .calc_voltage_level = icl_calc_voltage_level, +}; + +static struct intel_cdclk_funcs bxt_cdclk_funcs = { + .get_cdclk = bxt_get_cdclk, + .set_cdclk = bxt_set_cdclk, + .bw_calc_min_cdclk = skl_bw_calc_min_cdclk, + .modeset_calc_cdclk = bxt_modeset_calc_cdclk, + .calc_voltage_level = bxt_calc_voltage_level, +}; + +static struct intel_cdclk_funcs skl_cdclk_funcs = { + .get_cdclk = skl_get_cdclk, + .set_cdclk = skl_set_cdclk, + .bw_calc_min_cdclk = skl_bw_calc_min_cdclk, + .modeset_calc_cdclk = skl_modeset_calc_cdclk, +}; + +static struct intel_cdclk_funcs bdw_cdclk_funcs = { + .get_cdclk = bdw_get_cdclk, + .set_cdclk = bdw_set_cdclk, + .bw_calc_min_cdclk = intel_bw_calc_min_cdclk, + .modeset_calc_cdclk = bdw_modeset_calc_cdclk, +}; + +static struct intel_cdclk_funcs chv_cdclk_funcs = { + .get_cdclk = vlv_get_cdclk, + .set_cdclk = chv_set_cdclk, + .bw_calc_min_cdclk = intel_bw_calc_min_cdclk, + .modeset_calc_cdclk = vlv_modeset_calc_cdclk, +}; + +static struct intel_cdclk_funcs vlv_cdclk_funcs = { + .get_cdclk = vlv_get_cdclk, + .set_cdclk = vlv_set_cdclk, + .bw_calc_min_cdclk = intel_bw_calc_min_cdclk, + .modeset_calc_cdclk = vlv_modeset_calc_cdclk, +}; + +static struct intel_cdclk_funcs hsw_cdclk_funcs = { + .get_cdclk = hsw_get_cdclk, + .bw_calc_min_cdclk = intel_bw_calc_min_cdclk, + .modeset_calc_cdclk = fixed_modeset_calc_cdclk, +}; + +/* SNB, IVB, 965G, 945G */ +static struct intel_cdclk_funcs fixed_400mhz_cdclk_funcs = { + .get_cdclk = fixed_400mhz_get_cdclk, + .bw_calc_min_cdclk = intel_bw_calc_min_cdclk, + .modeset_calc_cdclk = fixed_modeset_calc_cdclk, +}; + +static struct intel_cdclk_funcs ilk_cdclk_funcs = { + .get_cdclk = fixed_450mhz_get_cdclk, + .bw_calc_min_cdclk = intel_bw_calc_min_cdclk, + .modeset_calc_cdclk = fixed_modeset_calc_cdclk, +}; + +static struct intel_cdclk_funcs gm45_cdclk_funcs = { + .get_cdclk = gm45_get_cdclk, + .bw_calc_min_cdclk = intel_bw_calc_min_cdclk, + .modeset_calc_cdclk = fixed_modeset_calc_cdclk, +}; + +/* G45 uses G33 */ + +static struct intel_cdclk_funcs i965gm_cdclk_funcs = { + .get_cdclk = i965gm_get_cdclk, + .bw_calc_min_cdclk = intel_bw_calc_min_cdclk, + .modeset_calc_cdclk = fixed_modeset_calc_cdclk, +}; + +/* i965G uses fixed 400 */ + +static struct intel_cdclk_funcs pnv_cdclk_funcs = { + .get_cdclk = pnv_get_cdclk, + .bw_calc_min_cdclk = intel_bw_calc_min_cdclk, + .modeset_calc_cdclk = fixed_modeset_calc_cdclk, +}; + +static struct intel_cdclk_funcs g33_cdclk_funcs = { + .get_cdclk = g33_get_cdclk, + .bw_calc_min_cdclk = intel_bw_calc_min_cdclk, + .modeset_calc_cdclk = fixed_modeset_calc_cdclk, +}; + +static struct intel_cdclk_funcs i945gm_cdclk_funcs = { + .get_cdclk = i945gm_get_cdclk, + .bw_calc_min_cdclk = intel_bw_calc_min_cdclk, + .modeset_calc_cdclk = fixed_modeset_calc_cdclk, +}; + +/* i945G uses fixed 400 */ + +static struct intel_cdclk_funcs i915gm_cdclk_funcs = { + .get_cdclk = i915gm_get_cdclk, + .bw_calc_min_cdclk = intel_bw_calc_min_cdclk, + .modeset_calc_cdclk = fixed_modeset_calc_cdclk, +}; + +static struct intel_cdclk_funcs i915g_cdclk_funcs = { + .get_cdclk = fixed_333mhz_get_cdclk, + .bw_calc_min_cdclk = intel_bw_calc_min_cdclk, + .modeset_calc_cdclk = fixed_modeset_calc_cdclk, +}; + +static struct intel_cdclk_funcs i865g_cdclk_funcs = { + .get_cdclk = fixed_266mhz_get_cdclk, + .bw_calc_min_cdclk = intel_bw_calc_min_cdclk, + .modeset_calc_cdclk = fixed_modeset_calc_cdclk, +}; + +static struct intel_cdclk_funcs i85x_cdclk_funcs = { + .get_cdclk = i85x_get_cdclk, + .bw_calc_min_cdclk = intel_bw_calc_min_cdclk, + .modeset_calc_cdclk = fixed_modeset_calc_cdclk, +}; + +static struct intel_cdclk_funcs i845g_cdclk_funcs = { + .get_cdclk = fixed_200mhz_get_cdclk, + .bw_calc_min_cdclk = intel_bw_calc_min_cdclk, + .modeset_calc_cdclk = fixed_modeset_calc_cdclk, +}; + +static struct intel_cdclk_funcs i830_cdclk_funcs = { + .get_cdclk = fixed_133mhz_get_cdclk, + .bw_calc_min_cdclk = intel_bw_calc_min_cdclk, + .modeset_calc_cdclk = fixed_modeset_calc_cdclk, +}; + /** * intel_init_cdclk_hooks - Initialize CDCLK related modesetting hooks * @dev_priv: i915 device @@ -2852,119 +3043,78 @@ u32 intel_read_rawclk(struct drm_i915_private *dev_priv) void intel_init_cdclk_hooks(struct drm_i915_private *dev_priv) { if (IS_DG2(dev_priv)) { - dev_priv->display.set_cdclk = bxt_set_cdclk; - dev_priv->display.bw_calc_min_cdclk = skl_bw_calc_min_cdclk; - dev_priv->display.modeset_calc_cdclk = bxt_modeset_calc_cdclk; - dev_priv->display.calc_voltage_level = tgl_calc_voltage_level; + dev_priv->cdclk_funcs = &tgl_cdclk_funcs; dev_priv->cdclk.table = dg2_cdclk_table; } else if (IS_ALDERLAKE_P(dev_priv)) { - dev_priv->display.set_cdclk = bxt_set_cdclk; - dev_priv->display.bw_calc_min_cdclk = skl_bw_calc_min_cdclk; - dev_priv->display.modeset_calc_cdclk = bxt_modeset_calc_cdclk; - dev_priv->display.calc_voltage_level = tgl_calc_voltage_level; + dev_priv->cdclk_funcs = &tgl_cdclk_funcs; /* Wa_22011320316:adl-p[a0] */ if (IS_ADLP_DISPLAY_STEP(dev_priv, STEP_A0, STEP_B0)) dev_priv->cdclk.table = adlp_a_step_cdclk_table; else dev_priv->cdclk.table = adlp_cdclk_table; } else if (IS_ROCKETLAKE(dev_priv)) { - dev_priv->display.set_cdclk = bxt_set_cdclk; - dev_priv->display.bw_calc_min_cdclk = skl_bw_calc_min_cdclk; - dev_priv->display.modeset_calc_cdclk = bxt_modeset_calc_cdclk; - dev_priv->display.calc_voltage_level = tgl_calc_voltage_level; + dev_priv->cdclk_funcs = &tgl_cdclk_funcs; dev_priv->cdclk.table = rkl_cdclk_table; } else if (DISPLAY_VER(dev_priv) >= 12) { - dev_priv->display.set_cdclk = bxt_set_cdclk; - dev_priv->display.bw_calc_min_cdclk = skl_bw_calc_min_cdclk; - dev_priv->display.modeset_calc_cdclk = bxt_modeset_calc_cdclk; - dev_priv->display.calc_voltage_level = tgl_calc_voltage_level; + dev_priv->cdclk_funcs = &tgl_cdclk_funcs; dev_priv->cdclk.table = icl_cdclk_table; } else if (IS_JSL_EHL(dev_priv)) { - dev_priv->display.set_cdclk = bxt_set_cdclk; - dev_priv->display.bw_calc_min_cdclk = skl_bw_calc_min_cdclk; - dev_priv->display.modeset_calc_cdclk = bxt_modeset_calc_cdclk; - dev_priv->display.calc_voltage_level = ehl_calc_voltage_level; + dev_priv->cdclk_funcs = &ehl_cdclk_funcs; dev_priv->cdclk.table = icl_cdclk_table; } else if (DISPLAY_VER(dev_priv) >= 11) { - dev_priv->display.set_cdclk = bxt_set_cdclk; - dev_priv->display.bw_calc_min_cdclk = skl_bw_calc_min_cdclk; - dev_priv->display.modeset_calc_cdclk = bxt_modeset_calc_cdclk; - dev_priv->display.calc_voltage_level = icl_calc_voltage_level; + dev_priv->cdclk_funcs = &icl_cdclk_funcs; dev_priv->cdclk.table = icl_cdclk_table; } else if (IS_GEMINILAKE(dev_priv) || IS_BROXTON(dev_priv)) { - dev_priv->display.bw_calc_min_cdclk = skl_bw_calc_min_cdclk; - dev_priv->display.set_cdclk = bxt_set_cdclk; - dev_priv->display.modeset_calc_cdclk = bxt_modeset_calc_cdclk; - dev_priv->display.calc_voltage_level = bxt_calc_voltage_level; + dev_priv->cdclk_funcs = &bxt_cdclk_funcs; if (IS_GEMINILAKE(dev_priv)) dev_priv->cdclk.table = glk_cdclk_table; else dev_priv->cdclk.table = bxt_cdclk_table; } else if (DISPLAY_VER(dev_priv) == 9) { - dev_priv->display.bw_calc_min_cdclk = skl_bw_calc_min_cdclk; - dev_priv->display.set_cdclk = skl_set_cdclk; - dev_priv->display.modeset_calc_cdclk = skl_modeset_calc_cdclk; + dev_priv->cdclk_funcs = &skl_cdclk_funcs; } else if (IS_BROADWELL(dev_priv)) { - dev_priv->display.bw_calc_min_cdclk = intel_bw_calc_min_cdclk; - dev_priv->display.set_cdclk = bdw_set_cdclk; - dev_priv->display.modeset_calc_cdclk = bdw_modeset_calc_cdclk; + dev_priv->cdclk_funcs = &bdw_cdclk_funcs; + } else if (IS_HASWELL(dev_priv)) { + dev_priv->cdclk_funcs = &hsw_cdclk_funcs; } else if (IS_CHERRYVIEW(dev_priv)) { - dev_priv->display.bw_calc_min_cdclk = intel_bw_calc_min_cdclk; - dev_priv->display.set_cdclk = chv_set_cdclk; - dev_priv->display.modeset_calc_cdclk = vlv_modeset_calc_cdclk; + dev_priv->cdclk_funcs = &chv_cdclk_funcs; } else if (IS_VALLEYVIEW(dev_priv)) { - dev_priv->display.bw_calc_min_cdclk = intel_bw_calc_min_cdclk; - dev_priv->display.set_cdclk = vlv_set_cdclk; - dev_priv->display.modeset_calc_cdclk = vlv_modeset_calc_cdclk; - } else { - dev_priv->display.bw_calc_min_cdclk = intel_bw_calc_min_cdclk; - dev_priv->display.modeset_calc_cdclk = fixed_modeset_calc_cdclk; + dev_priv->cdclk_funcs = &vlv_cdclk_funcs; + } else if (IS_SANDYBRIDGE(dev_priv) || IS_IVYBRIDGE(dev_priv)) { + dev_priv->cdclk_funcs = &fixed_400mhz_cdclk_funcs; + } else if (IS_IRONLAKE(dev_priv)) { + dev_priv->cdclk_funcs = &ilk_cdclk_funcs; + } else if (IS_GM45(dev_priv)) { + dev_priv->cdclk_funcs = &gm45_cdclk_funcs; + } else if (IS_G45(dev_priv)) { + dev_priv->cdclk_funcs = &g33_cdclk_funcs; + } else if (IS_I965GM(dev_priv)) { + dev_priv->cdclk_funcs = &i965gm_cdclk_funcs; + } else if (IS_I965G(dev_priv)) { + dev_priv->cdclk_funcs = &fixed_400mhz_cdclk_funcs; + } else if (IS_PINEVIEW(dev_priv)) { + dev_priv->cdclk_funcs = &pnv_cdclk_funcs; + } else if (IS_G33(dev_priv)) { + dev_priv->cdclk_funcs = &g33_cdclk_funcs; + } else if (IS_I945GM(dev_priv)) { + dev_priv->cdclk_funcs = &i945gm_cdclk_funcs; + } else if (IS_I945G(dev_priv)) { + dev_priv->cdclk_funcs = &fixed_400mhz_cdclk_funcs; + } else if (IS_I915GM(dev_priv)) { + dev_priv->cdclk_funcs = &i915gm_cdclk_funcs; + } else if (IS_I915G(dev_priv)) { + dev_priv->cdclk_funcs = &i915g_cdclk_funcs; + } else if (IS_I865G(dev_priv)) { + dev_priv->cdclk_funcs = &i865g_cdclk_funcs; + } else if (IS_I85X(dev_priv)) { + dev_priv->cdclk_funcs = &i85x_cdclk_funcs; + } else if (IS_I845G(dev_priv)) { + dev_priv->cdclk_funcs = &i845g_cdclk_funcs; + } else if (IS_I830(dev_priv)) { + dev_priv->cdclk_funcs = &i830_cdclk_funcs; } - if (DISPLAY_VER(dev_priv) >= 10 || IS_BROXTON(dev_priv)) - dev_priv->display.get_cdclk = bxt_get_cdclk; - else if (DISPLAY_VER(dev_priv) == 9) - dev_priv->display.get_cdclk = skl_get_cdclk; - else if (IS_BROADWELL(dev_priv)) - dev_priv->display.get_cdclk = bdw_get_cdclk; - else if (IS_HASWELL(dev_priv)) - dev_priv->display.get_cdclk = hsw_get_cdclk; - else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) - dev_priv->display.get_cdclk = vlv_get_cdclk; - else if (IS_SANDYBRIDGE(dev_priv) || IS_IVYBRIDGE(dev_priv)) - dev_priv->display.get_cdclk = fixed_400mhz_get_cdclk; - else if (IS_IRONLAKE(dev_priv)) - dev_priv->display.get_cdclk = fixed_450mhz_get_cdclk; - else if (IS_GM45(dev_priv)) - dev_priv->display.get_cdclk = gm45_get_cdclk; - else if (IS_G45(dev_priv)) - dev_priv->display.get_cdclk = g33_get_cdclk; - else if (IS_I965GM(dev_priv)) - dev_priv->display.get_cdclk = i965gm_get_cdclk; - else if (IS_I965G(dev_priv)) - dev_priv->display.get_cdclk = fixed_400mhz_get_cdclk; - else if (IS_PINEVIEW(dev_priv)) - dev_priv->display.get_cdclk = pnv_get_cdclk; - else if (IS_G33(dev_priv)) - dev_priv->display.get_cdclk = g33_get_cdclk; - else if (IS_I945GM(dev_priv)) - dev_priv->display.get_cdclk = i945gm_get_cdclk; - else if (IS_I945G(dev_priv)) - dev_priv->display.get_cdclk = fixed_400mhz_get_cdclk; - else if (IS_I915GM(dev_priv)) - dev_priv->display.get_cdclk = i915gm_get_cdclk; - else if (IS_I915G(dev_priv)) - dev_priv->display.get_cdclk = fixed_333mhz_get_cdclk; - else if (IS_I865G(dev_priv)) - dev_priv->display.get_cdclk = fixed_266mhz_get_cdclk; - else if (IS_I85X(dev_priv)) - dev_priv->display.get_cdclk = i85x_get_cdclk; - else if (IS_I845G(dev_priv)) - dev_priv->display.get_cdclk = fixed_200mhz_get_cdclk; - else if (IS_I830(dev_priv)) - dev_priv->display.get_cdclk = fixed_133mhz_get_cdclk; - - if (drm_WARN(&dev_priv->drm, !dev_priv->display.get_cdclk, - "Unknown platform. Assuming 133 MHz CDCLK\n")) - dev_priv->display.get_cdclk = fixed_133mhz_get_cdclk; + if (drm_WARN(&dev_priv->drm, !dev_priv->cdclk_funcs, + "Unknown platform. Assuming i830\n")) + dev_priv->cdclk_funcs = &i830_cdclk_funcs; } diff --git a/drivers/gpu/drm/i915/display/intel_cdclk.h b/drivers/gpu/drm/i915/display/intel_cdclk.h index b34eb00fb327..309b3f394e24 100644 --- a/drivers/gpu/drm/i915/display/intel_cdclk.h +++ b/drivers/gpu/drm/i915/display/intel_cdclk.h @@ -68,7 +68,9 @@ void intel_set_cdclk_post_plane_update(struct intel_atomic_state *state); void intel_dump_cdclk_config(const struct intel_cdclk_config *cdclk_config, const char *context); int intel_modeset_calc_cdclk(struct intel_atomic_state *state); - +void intel_cdclk_get_cdclk(struct drm_i915_private *dev_priv, + struct intel_cdclk_config *cdclk_config); +int intel_cdclk_bw_calc_min_cdclk(struct intel_atomic_state *state); struct intel_cdclk_state * intel_atomic_get_cdclk_state(struct intel_atomic_state *state); diff --git a/drivers/gpu/drm/i915/display/intel_color.c b/drivers/gpu/drm/i915/display/intel_color.c index afcb4bf3826c..5359b7305a78 100644 --- a/drivers/gpu/drm/i915/display/intel_color.c +++ b/drivers/gpu/drm/i915/display/intel_color.c @@ -25,6 +25,8 @@ #include "intel_color.h" #include "intel_de.h" #include "intel_display_types.h" +#include "intel_dpll.h" +#include "intel_dsi.h" #define CTM_COEFF_SIGN (1ULL << 63) @@ -1137,14 +1139,14 @@ void intel_color_load_luts(const struct intel_crtc_state *crtc_state) { struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev); - dev_priv->display.load_luts(crtc_state); + dev_priv->color_funcs->load_luts(crtc_state); } void intel_color_commit(const struct intel_crtc_state *crtc_state) { struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev); - dev_priv->display.color_commit(crtc_state); + dev_priv->color_funcs->color_commit(crtc_state); } static bool intel_can_preload_luts(const struct intel_crtc_state *new_crtc_state) @@ -1200,15 +1202,15 @@ int intel_color_check(struct intel_crtc_state *crtc_state) { struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev); - return dev_priv->display.color_check(crtc_state); + return dev_priv->color_funcs->color_check(crtc_state); } void intel_color_get_config(struct intel_crtc_state *crtc_state) { struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev); - if (dev_priv->display.read_luts) - dev_priv->display.read_luts(crtc_state); + if (dev_priv->color_funcs->read_luts) + dev_priv->color_funcs->read_luts(crtc_state); } static bool need_plane_update(struct intel_plane *plane, @@ -2092,6 +2094,76 @@ static void icl_read_luts(struct intel_crtc_state *crtc_state) } } +static const struct intel_color_funcs chv_color_funcs = { + .color_check = chv_color_check, + .color_commit = i9xx_color_commit, + .load_luts = chv_load_luts, + .read_luts = chv_read_luts, +}; + +static const struct intel_color_funcs i965_color_funcs = { + .color_check = i9xx_color_check, + .color_commit = i9xx_color_commit, + .load_luts = i965_load_luts, + .read_luts = i965_read_luts, +}; + +static const struct intel_color_funcs i9xx_color_funcs = { + .color_check = i9xx_color_check, + .color_commit = i9xx_color_commit, + .load_luts = i9xx_load_luts, + .read_luts = i9xx_read_luts, +}; + +static const struct intel_color_funcs icl_color_funcs = { + .color_check = icl_color_check, + .color_commit = skl_color_commit, + .load_luts = icl_load_luts, + .read_luts = icl_read_luts, +}; + +static const struct intel_color_funcs glk_color_funcs = { + .color_check = glk_color_check, + .color_commit = skl_color_commit, + .load_luts = glk_load_luts, + .read_luts = glk_read_luts, +}; + +static const struct intel_color_funcs skl_color_funcs = { + .color_check = ivb_color_check, + .color_commit = skl_color_commit, + .load_luts = bdw_load_luts, + .read_luts = NULL, +}; + +static const struct intel_color_funcs bdw_color_funcs = { + .color_check = ivb_color_check, + .color_commit = hsw_color_commit, + .load_luts = bdw_load_luts, + .read_luts = NULL, +}; + +static const struct intel_color_funcs hsw_color_funcs = { + .color_check = ivb_color_check, + .color_commit = hsw_color_commit, + .load_luts = ivb_load_luts, + .read_luts = NULL, +}; + +static const struct intel_color_funcs ivb_color_funcs = { + .color_check = ivb_color_check, + .color_commit = ilk_color_commit, + .load_luts = ivb_load_luts, + .read_luts = NULL, +}; + +static const struct intel_color_funcs ilk_color_funcs = { + .color_check = ilk_color_check, + .color_commit = ilk_color_commit, + .load_luts = ilk_load_luts, + .read_luts = ilk_read_luts, +}; + void intel_color_init(struct intel_crtc *crtc) { struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); @@ -2101,52 +2173,28 @@ void intel_color_init(struct intel_crtc *crtc) if (HAS_GMCH(dev_priv)) { if (IS_CHERRYVIEW(dev_priv)) { - dev_priv->display.color_check = chv_color_check; - dev_priv->display.color_commit = i9xx_color_commit; - dev_priv->display.load_luts = chv_load_luts; - dev_priv->display.read_luts = chv_read_luts; + dev_priv->color_funcs = &chv_color_funcs; } else if (DISPLAY_VER(dev_priv) >= 4) { - dev_priv->display.color_check = i9xx_color_check; - dev_priv->display.color_commit = i9xx_color_commit; - dev_priv->display.load_luts = i965_load_luts; - dev_priv->display.read_luts = i965_read_luts; + dev_priv->color_funcs = &i965_color_funcs; } else { - dev_priv->display.color_check = i9xx_color_check; - dev_priv->display.color_commit = i9xx_color_commit; - dev_priv->display.load_luts = i9xx_load_luts; - dev_priv->display.read_luts = i9xx_read_luts; + dev_priv->color_funcs = &i9xx_color_funcs; } } else { if (DISPLAY_VER(dev_priv) >= 11) - dev_priv->display.color_check = icl_color_check; - else if (DISPLAY_VER(dev_priv) >= 10) - dev_priv->display.color_check = glk_color_check; - else if (DISPLAY_VER(dev_priv) >= 7) - dev_priv->display.color_check = ivb_color_check; - else - dev_priv->display.color_check = ilk_color_check; - - if (DISPLAY_VER(dev_priv) >= 9) - dev_priv->display.color_commit = skl_color_commit; - else if (IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv)) - dev_priv->display.color_commit = hsw_color_commit; - else - dev_priv->display.color_commit = ilk_color_commit; - - if (DISPLAY_VER(dev_priv) >= 11) { - dev_priv->display.load_luts = icl_load_luts; - dev_priv->display.read_luts = icl_read_luts; - } else if (DISPLAY_VER(dev_priv) == 10) { - dev_priv->display.load_luts = glk_load_luts; - dev_priv->display.read_luts = glk_read_luts; - } else if (DISPLAY_VER(dev_priv) >= 8) { - dev_priv->display.load_luts = bdw_load_luts; - } else if (DISPLAY_VER(dev_priv) >= 7) { - dev_priv->display.load_luts = ivb_load_luts; - } else { - dev_priv->display.load_luts = ilk_load_luts; - dev_priv->display.read_luts = ilk_read_luts; - } + dev_priv->color_funcs = &icl_color_funcs; + else if (DISPLAY_VER(dev_priv) == 10) + dev_priv->color_funcs = &glk_color_funcs; + else if (DISPLAY_VER(dev_priv) == 9) + dev_priv->color_funcs = &skl_color_funcs; + else if (DISPLAY_VER(dev_priv) == 8) + dev_priv->color_funcs = &bdw_color_funcs; + else if (DISPLAY_VER(dev_priv) == 7) { + if (IS_HASWELL(dev_priv)) + dev_priv->color_funcs = &hsw_color_funcs; + else + dev_priv->color_funcs = &ivb_color_funcs; + } else + dev_priv->color_funcs = &ilk_color_funcs; } drm_crtc_enable_color_mgmt(&crtc->base, diff --git a/drivers/gpu/drm/i915/display/intel_combo_phy.c b/drivers/gpu/drm/i915/display/intel_combo_phy.c index bacdf8a16bcb..634e8d449457 100644 --- a/drivers/gpu/drm/i915/display/intel_combo_phy.c +++ b/drivers/gpu/drm/i915/display/intel_combo_phy.c @@ -220,13 +220,13 @@ static bool icl_combo_phy_verify_state(struct drm_i915_private *dev_priv, return false; if (DISPLAY_VER(dev_priv) >= 12) { - ret &= check_phy_reg(dev_priv, phy, ICL_PORT_TX_DW8_LN0(phy), + ret &= check_phy_reg(dev_priv, phy, ICL_PORT_TX_DW8_LN(0, phy), ICL_PORT_TX_DW8_ODCC_CLK_SEL | ICL_PORT_TX_DW8_ODCC_CLK_DIV_SEL_MASK, ICL_PORT_TX_DW8_ODCC_CLK_SEL | ICL_PORT_TX_DW8_ODCC_CLK_DIV_SEL_DIV2); - ret &= check_phy_reg(dev_priv, phy, ICL_PORT_PCS_DW1_LN0(phy), + ret &= check_phy_reg(dev_priv, phy, ICL_PORT_PCS_DW1_LN(0, phy), DCC_MODE_SELECT_MASK, DCC_MODE_SELECT_CONTINUOSLY); } @@ -343,13 +343,13 @@ static void icl_combo_phys_init(struct drm_i915_private *dev_priv) skip_phy_misc: if (DISPLAY_VER(dev_priv) >= 12) { - val = intel_de_read(dev_priv, ICL_PORT_TX_DW8_LN0(phy)); + val = intel_de_read(dev_priv, ICL_PORT_TX_DW8_LN(0, phy)); val &= ~ICL_PORT_TX_DW8_ODCC_CLK_DIV_SEL_MASK; val |= ICL_PORT_TX_DW8_ODCC_CLK_SEL; val |= ICL_PORT_TX_DW8_ODCC_CLK_DIV_SEL_DIV2; intel_de_write(dev_priv, ICL_PORT_TX_DW8_GRP(phy), val); - val = intel_de_read(dev_priv, ICL_PORT_PCS_DW1_LN0(phy)); + val = intel_de_read(dev_priv, ICL_PORT_PCS_DW1_LN(0, phy)); val &= ~DCC_MODE_SELECT_MASK; val |= DCC_MODE_SELECT_CONTINUOSLY; intel_de_write(dev_priv, ICL_PORT_PCS_DW1_GRP(phy), val); diff --git a/drivers/gpu/drm/i915/display/intel_connector.c b/drivers/gpu/drm/i915/display/intel_connector.c index 9bed1ccecea0..c65f95a9a1ec 100644 --- a/drivers/gpu/drm/i915/display/intel_connector.c +++ b/drivers/gpu/drm/i915/display/intel_connector.c @@ -29,13 +29,13 @@ #include <drm/drm_atomic_helper.h> #include <drm/drm_edid.h> -#include "display/intel_panel.h" - #include "i915_drv.h" +#include "intel_backlight.h" #include "intel_connector.h" #include "intel_display_debugfs.h" #include "intel_display_types.h" #include "intel_hdcp.h" +#include "intel_panel.h" int intel_connector_init(struct intel_connector *connector) { @@ -124,7 +124,7 @@ int intel_connector_register(struct drm_connector *connector) goto err_backlight; } - intel_connector_debugfs_add(connector); + intel_connector_debugfs_add(intel_connector); return 0; diff --git a/drivers/gpu/drm/i915/display/intel_crt.c b/drivers/gpu/drm/i915/display/intel_crt.c index 408f82b0dc7d..1c161eeed82f 100644 --- a/drivers/gpu/drm/i915/display/intel_crt.c +++ b/drivers/gpu/drm/i915/display/intel_crt.c @@ -251,7 +251,7 @@ static void hsw_post_disable_crt(struct intel_atomic_state *state, intel_crtc_vblank_off(old_crtc_state); - intel_disable_pipe(old_crtc_state); + intel_disable_transcoder(old_crtc_state); intel_ddi_disable_transcoder_func(old_crtc_state); @@ -314,7 +314,7 @@ static void hsw_enable_crt(struct intel_atomic_state *state, intel_ddi_enable_transcoder_func(encoder, crtc_state); - intel_enable_pipe(crtc_state); + intel_enable_transcoder(crtc_state); lpt_pch_enable(crtc_state); diff --git a/drivers/gpu/drm/i915/display/intel_cursor.c b/drivers/gpu/drm/i915/display/intel_cursor.c index c7618fef0143..11842f212613 100644 --- a/drivers/gpu/drm/i915/display/intel_cursor.c +++ b/drivers/gpu/drm/i915/display/intel_cursor.c @@ -17,7 +17,7 @@ #include "intel_display_types.h" #include "intel_display.h" #include "intel_fb.h" - +#include "intel_fb_pin.h" #include "intel_frontbuffer.h" #include "intel_pm.h" #include "intel_psr.h" @@ -536,8 +536,10 @@ static void i9xx_update_cursor(struct intel_plane *plane, if (DISPLAY_VER(dev_priv) >= 9) skl_write_cursor_wm(plane, crtc_state); - if (!intel_crtc_needs_modeset(crtc_state)) + if (plane_state) intel_psr2_program_plane_sel_fetch(plane, crtc_state, plane_state, 0); + else + intel_psr2_disable_plane_sel_fetch(plane, crtc_state); if (plane->cursor.base != base || plane->cursor.size != fbc_ctl || @@ -637,8 +639,7 @@ intel_legacy_cursor_update(struct drm_plane *_plane, * FIXME bigjoiner fastpath would be good */ if (!crtc_state->hw.active || intel_crtc_needs_modeset(crtc_state) || - crtc_state->update_pipe || crtc_state->bigjoiner || - crtc_state->enable_psr2_sel_fetch) + crtc_state->update_pipe || crtc_state->bigjoiner) goto slow; /* @@ -696,7 +697,7 @@ intel_legacy_cursor_update(struct drm_plane *_plane, goto out_free; intel_frontbuffer_flush(to_intel_frontbuffer(new_plane_state->hw.fb), - ORIGIN_FLIP); + ORIGIN_CURSOR_UPDATE); intel_frontbuffer_track(to_intel_frontbuffer(old_plane_state->hw.fb), to_intel_frontbuffer(new_plane_state->hw.fb), plane->frontbuffer_bit); diff --git a/drivers/gpu/drm/i915/display/intel_ddi.c b/drivers/gpu/drm/i915/display/intel_ddi.c index bd184325d0c7..1dcfe31e6c6f 100644 --- a/drivers/gpu/drm/i915/display/intel_ddi.c +++ b/drivers/gpu/drm/i915/display/intel_ddi.c @@ -29,6 +29,7 @@ #include "i915_drv.h" #include "intel_audio.h" +#include "intel_backlight.h" #include "intel_combo_phy.h" #include "intel_connector.h" #include "intel_crtc.h" @@ -40,6 +41,7 @@ #include "intel_dp_link_training.h" #include "intel_dp_mst.h" #include "intel_dpio_phy.h" +#include "intel_drrs.h" #include "intel_dsi.h" #include "intel_fdi.h" #include "intel_fifo_underrun.h" @@ -48,7 +50,6 @@ #include "intel_hdmi.h" #include "intel_hotplug.h" #include "intel_lspcon.h" -#include "intel_panel.h" #include "intel_pps.h" #include "intel_psr.h" #include "intel_snps_phy.h" @@ -73,24 +74,27 @@ static const u8 index_to_dp_signal_levels[] = { }; static int intel_ddi_hdmi_level(struct intel_encoder *encoder, - const struct intel_crtc_state *crtc_state) + const struct intel_ddi_buf_trans *trans) { - struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); - int n_entries, level, default_entry; + int level; - n_entries = intel_ddi_hdmi_num_entries(encoder, crtc_state, &default_entry); - if (n_entries == 0) - return 0; level = intel_bios_hdmi_level_shift(encoder); if (level < 0) - level = default_entry; - - if (drm_WARN_ON_ONCE(&dev_priv->drm, level >= n_entries)) - level = n_entries - 1; + level = trans->hdmi_default_entry; return level; } +static bool has_buf_trans_select(struct drm_i915_private *i915) +{ + return DISPLAY_VER(i915) < 10 && !IS_BROXTON(i915); +} + +static bool has_iboost(struct drm_i915_private *i915) +{ + return DISPLAY_VER(i915) == 9 && !IS_BROXTON(i915); +} + /* * Starting with Haswell, DDI port buffers must be programmed with correct * values in advance. This function programs the correct values for @@ -103,22 +107,22 @@ void hsw_prepare_dp_ddi_buffers(struct intel_encoder *encoder, u32 iboost_bit = 0; int i, n_entries; enum port port = encoder->port; - const struct intel_ddi_buf_trans *ddi_translations; + const struct intel_ddi_buf_trans *trans; - ddi_translations = encoder->get_buf_trans(encoder, crtc_state, &n_entries); - if (drm_WARN_ON_ONCE(&dev_priv->drm, !ddi_translations)) + trans = encoder->get_buf_trans(encoder, crtc_state, &n_entries); + if (drm_WARN_ON_ONCE(&dev_priv->drm, !trans)) return; /* If we're boosting the current, set bit 31 of trans1 */ - if (DISPLAY_VER(dev_priv) == 9 && !IS_BROXTON(dev_priv) && + if (has_iboost(dev_priv) && intel_bios_encoder_dp_boost_level(encoder->devdata)) iboost_bit = DDI_BUF_BALANCE_LEG_ENABLE; for (i = 0; i < n_entries; i++) { intel_de_write(dev_priv, DDI_BUF_TRANS_LO(port, i), - ddi_translations->entries[i].hsw.trans1 | iboost_bit); + trans->entries[i].hsw.trans1 | iboost_bit); intel_de_write(dev_priv, DDI_BUF_TRANS_HI(port, i), - ddi_translations->entries[i].hsw.trans2); + trans->entries[i].hsw.trans2); } } @@ -128,31 +132,29 @@ void hsw_prepare_dp_ddi_buffers(struct intel_encoder *encoder, * HDMI/DVI use cases. */ static void hsw_prepare_hdmi_ddi_buffers(struct intel_encoder *encoder, - const struct intel_crtc_state *crtc_state, - int level) + const struct intel_crtc_state *crtc_state) { struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); + int level = intel_ddi_level(encoder, crtc_state, 0); u32 iboost_bit = 0; int n_entries; enum port port = encoder->port; - const struct intel_ddi_buf_trans *ddi_translations; + const struct intel_ddi_buf_trans *trans; - ddi_translations = encoder->get_buf_trans(encoder, crtc_state, &n_entries); - if (drm_WARN_ON_ONCE(&dev_priv->drm, !ddi_translations)) + trans = encoder->get_buf_trans(encoder, crtc_state, &n_entries); + if (drm_WARN_ON_ONCE(&dev_priv->drm, !trans)) return; - if (drm_WARN_ON_ONCE(&dev_priv->drm, level >= n_entries)) - level = n_entries - 1; /* If we're boosting the current, set bit 31 of trans1 */ - if (DISPLAY_VER(dev_priv) == 9 && !IS_BROXTON(dev_priv) && + if (has_iboost(dev_priv) && intel_bios_encoder_hdmi_boost_level(encoder->devdata)) iboost_bit = DDI_BUF_BALANCE_LEG_ENABLE; /* Entry 9 is for HDMI: */ intel_de_write(dev_priv, DDI_BUF_TRANS_LO(port, 9), - ddi_translations->entries[level].hsw.trans1 | iboost_bit); + trans->entries[level].hsw.trans1 | iboost_bit); intel_de_write(dev_priv, DDI_BUF_TRANS_HI(port, 9), - ddi_translations->entries[level].hsw.trans2); + trans->entries[level].hsw.trans2); } void intel_wait_ddi_buf_idle(struct drm_i915_private *dev_priv, @@ -281,13 +283,14 @@ static void intel_ddi_init_dp_buf_reg(struct intel_encoder *encoder, struct intel_digital_port *dig_port = enc_to_dig_port(encoder); enum phy phy = intel_port_to_phy(i915, encoder->port); + /* DDI_BUF_CTL_ENABLE will be set by intel_ddi_prepare_link_retrain() later */ intel_dp->DP = dig_port->saved_port_bits | - DDI_BUF_CTL_ENABLE | DDI_BUF_TRANS_SELECT(0); - intel_dp->DP |= DDI_PORT_WIDTH(crtc_state->lane_count); + DDI_PORT_WIDTH(crtc_state->lane_count) | + DDI_BUF_TRANS_SELECT(0); if (IS_ALDERLAKE_P(i915) && intel_phy_is_tc(i915, phy)) { intel_dp->DP |= ddi_buf_phy_link_rate(crtc_state->port_clock); - if (dig_port->tc_mode != TC_PORT_TBT_ALT) + if (!intel_tc_port_in_tbt_alt_mode(dig_port)) intel_dp->DP |= DDI_BUF_CTL_TC_PHY_OWNERSHIP; } } @@ -407,6 +410,20 @@ static u32 bdw_trans_port_sync_master_select(enum transcoder master_transcoder) return master_transcoder + 1; } +static void +intel_ddi_config_transcoder_dp2(struct intel_encoder *encoder, + const struct intel_crtc_state *crtc_state) +{ + struct drm_i915_private *i915 = to_i915(encoder->base.dev); + enum transcoder cpu_transcoder = crtc_state->cpu_transcoder; + u32 val = 0; + + if (intel_dp_is_uhbr(crtc_state)) + val = TRANS_DP2_128B132B_CHANNEL_CODING; + + intel_de_write(i915, TRANS_DP2_CTL(cpu_transcoder), val); +} + /* * Returns the TRANS_DDI_FUNC_CTL value based on CRTC state. * @@ -488,10 +505,13 @@ intel_ddi_transcoder_func_reg_val_get(struct intel_encoder *encoder, if (crtc_state->hdmi_high_tmds_clock_ratio) temp |= TRANS_DDI_HIGH_TMDS_CHAR_RATE; } else if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_ANALOG)) { - temp |= TRANS_DDI_MODE_SELECT_FDI; + temp |= TRANS_DDI_MODE_SELECT_FDI_OR_128B132B; temp |= (crtc_state->fdi_lanes - 1) << 1; } else if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DP_MST)) { - temp |= TRANS_DDI_MODE_SELECT_DP_MST; + if (intel_dp_is_uhbr(crtc_state)) + temp |= TRANS_DDI_MODE_SELECT_FDI_OR_128B132B; + else + temp |= TRANS_DDI_MODE_SELECT_DP_MST; temp |= DDI_PORT_WIDTH(crtc_state->lane_count); if (DISPLAY_VER(dev_priv) >= 12) { @@ -678,8 +698,13 @@ bool intel_ddi_connector_get_hw_state(struct intel_connector *intel_connector) ret = false; break; - case TRANS_DDI_MODE_SELECT_FDI: - ret = type == DRM_MODE_CONNECTOR_VGA; + case TRANS_DDI_MODE_SELECT_FDI_OR_128B132B: + if (HAS_DP20(dev_priv)) + /* 128b/132b */ + ret = false; + else + /* FDI */ + ret = type == DRM_MODE_CONNECTOR_VGA; break; default: @@ -766,8 +791,9 @@ static void intel_ddi_get_encoder_pipes(struct intel_encoder *encoder, if ((tmp & port_mask) != ddi_select) continue; - if ((tmp & TRANS_DDI_MODE_SELECT_MASK) == - TRANS_DDI_MODE_SELECT_DP_MST) + if ((tmp & TRANS_DDI_MODE_SELECT_MASK) == TRANS_DDI_MODE_SELECT_DP_MST || + (HAS_DP20(dev_priv) && + (tmp & TRANS_DDI_MODE_SELECT_MASK) == TRANS_DDI_MODE_SELECT_FDI_OR_128B132B)) mst_pipe_mask |= BIT(p); *pipe_mask |= BIT(p); @@ -861,8 +887,7 @@ static void intel_ddi_get_power_domains(struct intel_encoder *encoder, dig_port = enc_to_dig_port(encoder); - if (!intel_phy_is_tc(dev_priv, phy) || - dig_port->tc_mode != TC_PORT_TBT_ALT) { + if (!intel_tc_port_in_tbt_alt_mode(dig_port)) { drm_WARN_ON(&dev_priv->drm, dig_port->ddi_io_wakeref); dig_port->ddi_io_wakeref = intel_display_power_get(dev_priv, dig_port->ddi_io_power_domain); @@ -947,16 +972,14 @@ static void skl_ddi_set_iboost(struct intel_encoder *encoder, iboost = intel_bios_encoder_dp_boost_level(encoder->devdata); if (iboost == 0) { - const struct intel_ddi_buf_trans *ddi_translations; + const struct intel_ddi_buf_trans *trans; int n_entries; - ddi_translations = encoder->get_buf_trans(encoder, crtc_state, &n_entries); - if (drm_WARN_ON_ONCE(&dev_priv->drm, !ddi_translations)) + trans = encoder->get_buf_trans(encoder, crtc_state, &n_entries); + if (drm_WARN_ON_ONCE(&dev_priv->drm, !trans)) return; - if (drm_WARN_ON_ONCE(&dev_priv->drm, level >= n_entries)) - level = n_entries - 1; - iboost = ddi_translations->entries[level].hsw.i_boost; + iboost = trans->entries[level].hsw.i_boost; } /* Make sure that the requested I_boost is valid */ @@ -971,28 +994,6 @@ static void skl_ddi_set_iboost(struct intel_encoder *encoder, _skl_ddi_set_iboost(dev_priv, PORT_E, iboost); } -static void bxt_ddi_vswing_sequence(struct intel_encoder *encoder, - const struct intel_crtc_state *crtc_state, - int level) -{ - struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); - const struct intel_ddi_buf_trans *ddi_translations; - enum port port = encoder->port; - int n_entries; - - ddi_translations = encoder->get_buf_trans(encoder, crtc_state, &n_entries); - if (drm_WARN_ON_ONCE(&dev_priv->drm, !ddi_translations)) - return; - if (drm_WARN_ON_ONCE(&dev_priv->drm, level >= n_entries)) - level = n_entries - 1; - - bxt_ddi_phy_set_signal_level(dev_priv, port, - ddi_translations->entries[level].bxt.margin, - ddi_translations->entries[level].bxt.scale, - ddi_translations->entries[level].bxt.enable, - ddi_translations->entries[level].bxt.deemphasis); -} - static u8 intel_ddi_dp_voltage_max(struct intel_dp *intel_dp, const struct intel_crtc_state *crtc_state) { @@ -1022,33 +1023,43 @@ static u8 intel_ddi_dp_preemph_max(struct intel_dp *intel_dp) return DP_TRAIN_PRE_EMPH_LEVEL_3; } +static u32 icl_combo_phy_loadgen_select(const struct intel_crtc_state *crtc_state, + int lane) +{ + if (crtc_state->port_clock > 600000) + return 0; + + if (crtc_state->lane_count == 4) + return lane >= 1 ? LOADGEN_SELECT : 0; + else + return lane == 1 || lane == 2 ? LOADGEN_SELECT : 0; +} + static void icl_ddi_combo_vswing_program(struct intel_encoder *encoder, - const struct intel_crtc_state *crtc_state, - int level) + const struct intel_crtc_state *crtc_state) { struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); - const struct intel_ddi_buf_trans *ddi_translations; + int level = intel_ddi_level(encoder, crtc_state, 0); + const struct intel_ddi_buf_trans *trans; enum phy phy = intel_port_to_phy(dev_priv, encoder->port); int n_entries, ln; u32 val; - ddi_translations = encoder->get_buf_trans(encoder, crtc_state, &n_entries); - if (drm_WARN_ON_ONCE(&dev_priv->drm, !ddi_translations)) + trans = encoder->get_buf_trans(encoder, crtc_state, &n_entries); + if (drm_WARN_ON_ONCE(&dev_priv->drm, !trans)) return; - if (drm_WARN_ON_ONCE(&dev_priv->drm, level >= n_entries)) - level = n_entries - 1; if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_EDP)) { struct intel_dp *intel_dp = enc_to_intel_dp(encoder); val = EDP4K2K_MODE_OVRD_EN | EDP4K2K_MODE_OVRD_OPTIMIZED; - intel_dp->hobl_active = is_hobl_buf_trans(ddi_translations); + intel_dp->hobl_active = is_hobl_buf_trans(trans); intel_de_rmw(dev_priv, ICL_PORT_CL_DW10(phy), val, intel_dp->hobl_active ? val : 0); } /* Set PORT_TX_DW5 */ - val = intel_de_read(dev_priv, ICL_PORT_TX_DW5_LN0(phy)); + val = intel_de_read(dev_priv, ICL_PORT_TX_DW5_LN(0, phy)); val &= ~(SCALING_MODE_SEL_MASK | RTERM_SELECT_MASK | TAP2_DISABLE | TAP3_DISABLE); val |= SCALING_MODE_SEL(0x2); @@ -1057,52 +1068,48 @@ static void icl_ddi_combo_vswing_program(struct intel_encoder *encoder, intel_de_write(dev_priv, ICL_PORT_TX_DW5_GRP(phy), val); /* Program PORT_TX_DW2 */ - val = intel_de_read(dev_priv, ICL_PORT_TX_DW2_LN0(phy)); + val = intel_de_read(dev_priv, ICL_PORT_TX_DW2_LN(0, phy)); val &= ~(SWING_SEL_LOWER_MASK | SWING_SEL_UPPER_MASK | RCOMP_SCALAR_MASK); - val |= SWING_SEL_UPPER(ddi_translations->entries[level].icl.dw2_swing_sel); - val |= SWING_SEL_LOWER(ddi_translations->entries[level].icl.dw2_swing_sel); + val |= SWING_SEL_UPPER(trans->entries[level].icl.dw2_swing_sel); + val |= SWING_SEL_LOWER(trans->entries[level].icl.dw2_swing_sel); /* Program Rcomp scalar for every table entry */ val |= RCOMP_SCALAR(0x98); intel_de_write(dev_priv, ICL_PORT_TX_DW2_GRP(phy), val); /* Program PORT_TX_DW4 */ /* We cannot write to GRP. It would overwrite individual loadgen. */ - for (ln = 0; ln <= 3; ln++) { + for (ln = 0; ln < 4; ln++) { val = intel_de_read(dev_priv, ICL_PORT_TX_DW4_LN(ln, phy)); val &= ~(POST_CURSOR_1_MASK | POST_CURSOR_2_MASK | CURSOR_COEFF_MASK); - val |= POST_CURSOR_1(ddi_translations->entries[level].icl.dw4_post_cursor_1); - val |= POST_CURSOR_2(ddi_translations->entries[level].icl.dw4_post_cursor_2); - val |= CURSOR_COEFF(ddi_translations->entries[level].icl.dw4_cursor_coeff); + val |= POST_CURSOR_1(trans->entries[level].icl.dw4_post_cursor_1); + val |= POST_CURSOR_2(trans->entries[level].icl.dw4_post_cursor_2); + val |= CURSOR_COEFF(trans->entries[level].icl.dw4_cursor_coeff); intel_de_write(dev_priv, ICL_PORT_TX_DW4_LN(ln, phy), val); } /* Program PORT_TX_DW7 */ - val = intel_de_read(dev_priv, ICL_PORT_TX_DW7_LN0(phy)); + val = intel_de_read(dev_priv, ICL_PORT_TX_DW7_LN(0, phy)); val &= ~N_SCALAR_MASK; - val |= N_SCALAR(ddi_translations->entries[level].icl.dw7_n_scalar); + val |= N_SCALAR(trans->entries[level].icl.dw7_n_scalar); intel_de_write(dev_priv, ICL_PORT_TX_DW7_GRP(phy), val); } -static void icl_combo_phy_ddi_vswing_sequence(struct intel_encoder *encoder, - const struct intel_crtc_state *crtc_state, - int level) +static void icl_combo_phy_set_signal_levels(struct intel_encoder *encoder, + const struct intel_crtc_state *crtc_state) { struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); enum phy phy = intel_port_to_phy(dev_priv, encoder->port); - int width, rate, ln; u32 val; - - width = crtc_state->lane_count; - rate = crtc_state->port_clock; + int ln; /* * 1. If port type is eDP or DP, * set PORT_PCS_DW1 cmnkeeper_enable to 1b, * else clear to 0b. */ - val = intel_de_read(dev_priv, ICL_PORT_PCS_DW1_LN0(phy)); + val = intel_de_read(dev_priv, ICL_PORT_PCS_DW1_LN(0, phy)); if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI)) val &= ~COMMON_KEEPER_EN; else @@ -1111,19 +1118,15 @@ static void icl_combo_phy_ddi_vswing_sequence(struct intel_encoder *encoder, /* 2. Program loadgen select */ /* - * Program PORT_TX_DW4_LN depending on Bit rate and used lanes + * Program PORT_TX_DW4 depending on Bit rate and used lanes * <= 6 GHz and 4 lanes (LN0=0, LN1=1, LN2=1, LN3=1) * <= 6 GHz and 1,2 lanes (LN0=0, LN1=1, LN2=1, LN3=0) * > 6 GHz (LN0=0, LN1=0, LN2=0, LN3=0) */ - for (ln = 0; ln <= 3; ln++) { + for (ln = 0; ln < 4; ln++) { val = intel_de_read(dev_priv, ICL_PORT_TX_DW4_LN(ln, phy)); val &= ~LOADGEN_SELECT; - - if ((rate <= 600000 && width == 4 && ln >= 1) || - (rate <= 600000 && width < 4 && (ln == 1 || ln == 2))) { - val |= LOADGEN_SELECT; - } + val |= icl_combo_phy_loadgen_select(crtc_state, ln); intel_de_write(dev_priv, ICL_PORT_TX_DW4_LN(ln, phy), val); } @@ -1133,37 +1136,35 @@ static void icl_combo_phy_ddi_vswing_sequence(struct intel_encoder *encoder, intel_de_write(dev_priv, ICL_PORT_CL_DW5(phy), val); /* 4. Clear training enable to change swing values */ - val = intel_de_read(dev_priv, ICL_PORT_TX_DW5_LN0(phy)); + val = intel_de_read(dev_priv, ICL_PORT_TX_DW5_LN(0, phy)); val &= ~TX_TRAINING_EN; intel_de_write(dev_priv, ICL_PORT_TX_DW5_GRP(phy), val); /* 5. Program swing and de-emphasis */ - icl_ddi_combo_vswing_program(encoder, crtc_state, level); + icl_ddi_combo_vswing_program(encoder, crtc_state); /* 6. Set training enable to trigger update */ - val = intel_de_read(dev_priv, ICL_PORT_TX_DW5_LN0(phy)); + val = intel_de_read(dev_priv, ICL_PORT_TX_DW5_LN(0, phy)); val |= TX_TRAINING_EN; intel_de_write(dev_priv, ICL_PORT_TX_DW5_GRP(phy), val); } -static void icl_mg_phy_ddi_vswing_sequence(struct intel_encoder *encoder, - const struct intel_crtc_state *crtc_state, - int level) +static void icl_mg_phy_set_signal_levels(struct intel_encoder *encoder, + const struct intel_crtc_state *crtc_state) { struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); enum tc_port tc_port = intel_port_to_tc(dev_priv, encoder->port); - const struct intel_ddi_buf_trans *ddi_translations; + int level = intel_ddi_level(encoder, crtc_state, 0); + const struct intel_ddi_buf_trans *trans; int n_entries, ln; u32 val; - if (enc_to_dig_port(encoder)->tc_mode == TC_PORT_TBT_ALT) + if (intel_tc_port_in_tbt_alt_mode(enc_to_dig_port(encoder))) return; - ddi_translations = encoder->get_buf_trans(encoder, crtc_state, &n_entries); - if (drm_WARN_ON_ONCE(&dev_priv->drm, !ddi_translations)) + trans = encoder->get_buf_trans(encoder, crtc_state, &n_entries); + if (drm_WARN_ON_ONCE(&dev_priv->drm, !trans)) return; - if (drm_WARN_ON_ONCE(&dev_priv->drm, level >= n_entries)) - level = n_entries - 1; /* Set MG_TX_LINK_PARAMS cri_use_fs32 to 0. */ for (ln = 0; ln < 2; ln++) { @@ -1181,13 +1182,13 @@ static void icl_mg_phy_ddi_vswing_sequence(struct intel_encoder *encoder, val = intel_de_read(dev_priv, MG_TX1_SWINGCTRL(ln, tc_port)); val &= ~CRI_TXDEEMPH_OVERRIDE_17_12_MASK; val |= CRI_TXDEEMPH_OVERRIDE_17_12( - ddi_translations->entries[level].mg.cri_txdeemph_override_17_12); + trans->entries[level].mg.cri_txdeemph_override_17_12); intel_de_write(dev_priv, MG_TX1_SWINGCTRL(ln, tc_port), val); val = intel_de_read(dev_priv, MG_TX2_SWINGCTRL(ln, tc_port)); val &= ~CRI_TXDEEMPH_OVERRIDE_17_12_MASK; val |= CRI_TXDEEMPH_OVERRIDE_17_12( - ddi_translations->entries[level].mg.cri_txdeemph_override_17_12); + trans->entries[level].mg.cri_txdeemph_override_17_12); intel_de_write(dev_priv, MG_TX2_SWINGCTRL(ln, tc_port), val); } @@ -1197,9 +1198,9 @@ static void icl_mg_phy_ddi_vswing_sequence(struct intel_encoder *encoder, val &= ~(CRI_TXDEEMPH_OVERRIDE_11_6_MASK | CRI_TXDEEMPH_OVERRIDE_5_0_MASK); val |= CRI_TXDEEMPH_OVERRIDE_5_0( - ddi_translations->entries[level].mg.cri_txdeemph_override_5_0) | + trans->entries[level].mg.cri_txdeemph_override_5_0) | CRI_TXDEEMPH_OVERRIDE_11_6( - ddi_translations->entries[level].mg.cri_txdeemph_override_11_6) | + trans->entries[level].mg.cri_txdeemph_override_11_6) | CRI_TXDEEMPH_OVERRIDE_EN; intel_de_write(dev_priv, MG_TX1_DRVCTRL(ln, tc_port), val); @@ -1207,9 +1208,9 @@ static void icl_mg_phy_ddi_vswing_sequence(struct intel_encoder *encoder, val &= ~(CRI_TXDEEMPH_OVERRIDE_11_6_MASK | CRI_TXDEEMPH_OVERRIDE_5_0_MASK); val |= CRI_TXDEEMPH_OVERRIDE_5_0( - ddi_translations->entries[level].mg.cri_txdeemph_override_5_0) | + trans->entries[level].mg.cri_txdeemph_override_5_0) | CRI_TXDEEMPH_OVERRIDE_11_6( - ddi_translations->entries[level].mg.cri_txdeemph_override_11_6) | + trans->entries[level].mg.cri_txdeemph_override_11_6) | CRI_TXDEEMPH_OVERRIDE_EN; intel_de_write(dev_priv, MG_TX2_DRVCTRL(ln, tc_port), val); @@ -1269,45 +1270,29 @@ static void icl_mg_phy_ddi_vswing_sequence(struct intel_encoder *encoder, } } -static void icl_ddi_vswing_sequence(struct intel_encoder *encoder, - const struct intel_crtc_state *crtc_state, - int level) -{ - struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); - enum phy phy = intel_port_to_phy(dev_priv, encoder->port); - - if (intel_phy_is_combo(dev_priv, phy)) - icl_combo_phy_ddi_vswing_sequence(encoder, crtc_state, level); - else - icl_mg_phy_ddi_vswing_sequence(encoder, crtc_state, level); -} - -static void -tgl_dkl_phy_ddi_vswing_sequence(struct intel_encoder *encoder, - const struct intel_crtc_state *crtc_state, - int level) +static void tgl_dkl_phy_set_signal_levels(struct intel_encoder *encoder, + const struct intel_crtc_state *crtc_state) { struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); enum tc_port tc_port = intel_port_to_tc(dev_priv, encoder->port); - const struct intel_ddi_buf_trans *ddi_translations; + int level = intel_ddi_level(encoder, crtc_state, 0); + const struct intel_ddi_buf_trans *trans; u32 val, dpcnt_mask, dpcnt_val; int n_entries, ln; - if (enc_to_dig_port(encoder)->tc_mode == TC_PORT_TBT_ALT) + if (intel_tc_port_in_tbt_alt_mode(enc_to_dig_port(encoder))) return; - ddi_translations = encoder->get_buf_trans(encoder, crtc_state, &n_entries); - if (drm_WARN_ON_ONCE(&dev_priv->drm, !ddi_translations)) + trans = encoder->get_buf_trans(encoder, crtc_state, &n_entries); + if (drm_WARN_ON_ONCE(&dev_priv->drm, !trans)) return; - if (drm_WARN_ON_ONCE(&dev_priv->drm, level >= n_entries)) - level = n_entries - 1; dpcnt_mask = (DKL_TX_PRESHOOT_COEFF_MASK | DKL_TX_DE_EMPAHSIS_COEFF_MASK | DKL_TX_VSWING_CONTROL_MASK); - dpcnt_val = DKL_TX_VSWING_CONTROL(ddi_translations->entries[level].dkl.dkl_vswing_control); - dpcnt_val |= DKL_TX_DE_EMPHASIS_COEFF(ddi_translations->entries[level].dkl.dkl_de_emphasis_control); - dpcnt_val |= DKL_TX_PRESHOOT_COEFF(ddi_translations->entries[level].dkl.dkl_preshoot_control); + dpcnt_val = DKL_TX_VSWING_CONTROL(trans->entries[level].dkl.vswing); + dpcnt_val |= DKL_TX_DE_EMPHASIS_COEFF(trans->entries[level].dkl.de_emphasis); + dpcnt_val |= DKL_TX_PRESHOOT_COEFF(trans->entries[level].dkl.preshoot); for (ln = 0; ln < 2; ln++) { intel_de_write(dev_priv, HIP_INDEX_REG(tc_port), @@ -1329,30 +1314,9 @@ tgl_dkl_phy_ddi_vswing_sequence(struct intel_encoder *encoder, val = intel_de_read(dev_priv, DKL_TX_DPCNTL2(tc_port)); val &= ~DKL_TX_DP20BITMODE; intel_de_write(dev_priv, DKL_TX_DPCNTL2(tc_port), val); - - if ((intel_crtc_has_dp_encoder(crtc_state) && - crtc_state->port_clock == 162000) || - (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI) && - crtc_state->port_clock == 594000)) - val |= DKL_TX_LOADGEN_SHARING_PMD_DISABLE; - else - val &= ~DKL_TX_LOADGEN_SHARING_PMD_DISABLE; } } -static void tgl_ddi_vswing_sequence(struct intel_encoder *encoder, - const struct intel_crtc_state *crtc_state, - int level) -{ - struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); - enum phy phy = intel_port_to_phy(dev_priv, encoder->port); - - if (intel_phy_is_combo(dev_priv, phy)) - icl_combo_phy_ddi_vswing_sequence(encoder, crtc_state, level); - else - tgl_dkl_phy_ddi_vswing_sequence(encoder, crtc_state, level); -} - static int translate_signal_level(struct intel_dp *intel_dp, u8 signal_levels) { @@ -1371,65 +1335,63 @@ static int translate_signal_level(struct intel_dp *intel_dp, return 0; } -static int intel_ddi_dp_level(struct intel_dp *intel_dp) +static int intel_ddi_dp_level(struct intel_dp *intel_dp, + const struct intel_crtc_state *crtc_state, + int lane) { - u8 train_set = intel_dp->train_set[0]; - u8 signal_levels = train_set & (DP_TRAIN_VOLTAGE_SWING_MASK | - DP_TRAIN_PRE_EMPHASIS_MASK); - - return translate_signal_level(intel_dp, signal_levels); -} + u8 train_set = intel_dp->train_set[lane]; -static void -dg2_set_signal_levels(struct intel_dp *intel_dp, - const struct intel_crtc_state *crtc_state) -{ - struct intel_encoder *encoder = &dp_to_dig_port(intel_dp)->base; - int level = intel_ddi_dp_level(intel_dp); + if (intel_dp_is_uhbr(crtc_state)) { + return train_set & DP_TX_FFE_PRESET_VALUE_MASK; + } else { + u8 signal_levels = train_set & (DP_TRAIN_VOLTAGE_SWING_MASK | + DP_TRAIN_PRE_EMPHASIS_MASK); - intel_snps_phy_ddi_vswing_sequence(encoder, level); + return translate_signal_level(intel_dp, signal_levels); + } } -static void -tgl_set_signal_levels(struct intel_dp *intel_dp, - const struct intel_crtc_state *crtc_state) +int intel_ddi_level(struct intel_encoder *encoder, + const struct intel_crtc_state *crtc_state, + int lane) { - struct intel_encoder *encoder = &dp_to_dig_port(intel_dp)->base; - int level = intel_ddi_dp_level(intel_dp); - - tgl_ddi_vswing_sequence(encoder, crtc_state, level); -} + struct drm_i915_private *i915 = to_i915(encoder->base.dev); + const struct intel_ddi_buf_trans *trans; + int level, n_entries; -static void -icl_set_signal_levels(struct intel_dp *intel_dp, - const struct intel_crtc_state *crtc_state) -{ - struct intel_encoder *encoder = &dp_to_dig_port(intel_dp)->base; - int level = intel_ddi_dp_level(intel_dp); + trans = encoder->get_buf_trans(encoder, crtc_state, &n_entries); + if (drm_WARN_ON_ONCE(&i915->drm, !trans)) + return 0; - icl_ddi_vswing_sequence(encoder, crtc_state, level); -} + if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI)) + level = intel_ddi_hdmi_level(encoder, trans); + else + level = intel_ddi_dp_level(enc_to_intel_dp(encoder), crtc_state, + lane); -static void -bxt_set_signal_levels(struct intel_dp *intel_dp, - const struct intel_crtc_state *crtc_state) -{ - struct intel_encoder *encoder = &dp_to_dig_port(intel_dp)->base; - int level = intel_ddi_dp_level(intel_dp); + if (drm_WARN_ON_ONCE(&i915->drm, level >= n_entries)) + level = n_entries - 1; - bxt_ddi_vswing_sequence(encoder, crtc_state, level); + return level; } static void -hsw_set_signal_levels(struct intel_dp *intel_dp, +hsw_set_signal_levels(struct intel_encoder *encoder, const struct intel_crtc_state *crtc_state) { - struct intel_encoder *encoder = &dp_to_dig_port(intel_dp)->base; struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); - int level = intel_ddi_dp_level(intel_dp); + struct intel_dp *intel_dp = enc_to_intel_dp(encoder); + int level = intel_ddi_level(encoder, crtc_state, 0); enum port port = encoder->port; u32 signal_levels; + if (has_iboost(dev_priv)) + skl_ddi_set_iboost(encoder, crtc_state, level); + + /* HDMI ignores the rest */ + if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI)) + return; + signal_levels = DDI_BUF_TRANS_SELECT(level); drm_dbg_kms(&dev_priv->drm, "Using signal levels %08x\n", @@ -1438,9 +1400,6 @@ hsw_set_signal_levels(struct intel_dp *intel_dp, intel_dp->DP &= ~DDI_BUF_EMP_MASK; intel_dp->DP |= signal_levels; - if (DISPLAY_VER(dev_priv) == 9 && !IS_BROXTON(dev_priv)) - skl_ddi_set_iboost(encoder, crtc_state, level); - intel_de_write(dev_priv, DDI_BUF_CTL(port), intel_dp->DP); intel_de_posting_read(dev_priv, DDI_BUF_CTL(port)); } @@ -2059,7 +2018,7 @@ icl_program_mg_dp_mode(struct intel_digital_port *dig_port, u8 width; if (!intel_phy_is_tc(dev_priv, phy) || - dig_port->tc_mode == TC_PORT_TBT_ALT) + intel_tc_port_in_tbt_alt_mode(dig_port)) return; if (DISPLAY_VER(dev_priv) >= 12) { @@ -2084,7 +2043,7 @@ icl_program_mg_dp_mode(struct intel_digital_port *dig_port, switch (pin_assignment) { case 0x0: drm_WARN_ON(&dev_priv->drm, - dig_port->tc_mode != TC_PORT_LEGACY); + !intel_tc_port_in_legacy_mode(dig_port)); if (width == 1) { ln1 |= MG_DP_MODE_CFG_DP_X1_MODE; } else { @@ -2329,15 +2288,19 @@ static void dg2_ddi_pre_enable_dp(struct intel_atomic_state *state, { struct intel_dp *intel_dp = enc_to_intel_dp(encoder); struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); - enum phy phy = intel_port_to_phy(dev_priv, encoder->port); struct intel_digital_port *dig_port = enc_to_dig_port(encoder); bool is_mst = intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DP_MST); - int level = intel_ddi_dp_level(intel_dp); intel_dp_set_link_params(intel_dp, crtc_state->port_clock, crtc_state->lane_count); /* + * We only configure what the register value will be here. Actual + * enabling happens during link training farther down. + */ + intel_ddi_init_dp_buf_reg(encoder, crtc_state); + + /* * 1. Enable Power Wells * * This was handled at the beginning of intel_atomic_commit_tail(), @@ -2353,8 +2316,7 @@ static void dg2_ddi_pre_enable_dp(struct intel_atomic_state *state, intel_ddi_enable_clock(encoder, crtc_state); /* 4. Enable IO power */ - if (!intel_phy_is_tc(dev_priv, phy) || - dig_port->tc_mode != TC_PORT_TBT_ALT) + if (!intel_tc_port_in_tbt_alt_mode(dig_port)) dig_port->ddi_io_wakeref = intel_display_power_get(dev_priv, dig_port->ddi_io_power_domain); @@ -2374,7 +2336,8 @@ static void dg2_ddi_pre_enable_dp(struct intel_atomic_state *state, */ intel_ddi_enable_pipe_clock(encoder, crtc_state); - /* 5.b Not relevant to i915 for now */ + /* 5.b Configure transcoder for DP 2.0 128b/132b */ + intel_ddi_config_transcoder_dp2(encoder, crtc_state); /* * 5.c Configure TRANS_DDI_FUNC_CTL DDI Select, DDI Mode Select & MST @@ -2391,21 +2354,12 @@ static void dg2_ddi_pre_enable_dp(struct intel_atomic_state *state, */ /* 5.e Configure voltage swing and related IO settings */ - intel_snps_phy_ddi_vswing_sequence(encoder, level); - - /* - * 5.f Configure and enable DDI_BUF_CTL - * 5.g Wait for DDI_BUF_CTL DDI Idle Status = 0b (Not Idle), timeout - * after 1200 us. - * - * We only configure what the register value will be here. Actual - * enabling happens during link training farther down. - */ - intel_ddi_init_dp_buf_reg(encoder, crtc_state); + encoder->set_signal_levels(encoder, crtc_state); if (!is_mst) intel_dp_set_power(intel_dp, DP_SET_POWER_D0); + intel_dp_configure_protocol_converter(intel_dp, crtc_state); intel_dp_sink_set_decompression_state(intel_dp, crtc_state, true); /* * DDI FEC: "anticipates enabling FEC encoding sets the FEC_READY bit @@ -2413,6 +2367,8 @@ static void dg2_ddi_pre_enable_dp(struct intel_atomic_state *state, * training */ intel_dp_sink_set_fec_ready(intel_dp, crtc_state); + intel_dp_check_frl_training(intel_dp); + intel_dp_pcon_dsc_configure(intel_dp, crtc_state); /* * 5.h Follow DisplayPort specification training sequence (see notes for @@ -2439,16 +2395,20 @@ static void tgl_ddi_pre_enable_dp(struct intel_atomic_state *state, { struct intel_dp *intel_dp = enc_to_intel_dp(encoder); struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); - enum phy phy = intel_port_to_phy(dev_priv, encoder->port); struct intel_digital_port *dig_port = enc_to_dig_port(encoder); bool is_mst = intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DP_MST); - int level = intel_ddi_dp_level(intel_dp); intel_dp_set_link_params(intel_dp, crtc_state->port_clock, crtc_state->lane_count); /* + * We only configure what the register value will be here. Actual + * enabling happens during link training farther down. + */ + intel_ddi_init_dp_buf_reg(encoder, crtc_state); + + /* * 1. Enable Power Wells * * This was handled at the beginning of intel_atomic_commit_tail(), @@ -2476,8 +2436,7 @@ static void tgl_ddi_pre_enable_dp(struct intel_atomic_state *state, intel_ddi_enable_clock(encoder, crtc_state); /* 5. If IO power is controlled through PWR_WELL_CTL, Enable IO Power */ - if (!intel_phy_is_tc(dev_priv, phy) || - dig_port->tc_mode != TC_PORT_TBT_ALT) { + if (!intel_tc_port_in_tbt_alt_mode(dig_port)) { drm_WARN_ON(&dev_priv->drm, dig_port->ddi_io_wakeref); dig_port->ddi_io_wakeref = intel_display_power_get(dev_priv, dig_port->ddi_io_power_domain); @@ -2517,7 +2476,7 @@ static void tgl_ddi_pre_enable_dp(struct intel_atomic_state *state, */ /* 7.e Configure voltage swing and related IO settings */ - tgl_ddi_vswing_sequence(encoder, crtc_state, level); + encoder->set_signal_levels(encoder, crtc_state); /* * 7.f Combo PHY: Configure PORT_CL_DW10 Static Power Down to power up @@ -2530,16 +2489,6 @@ static void tgl_ddi_pre_enable_dp(struct intel_atomic_state *state, */ intel_ddi_mso_configure(crtc_state); - /* - * 7.g Configure and enable DDI_BUF_CTL - * 7.h Wait for DDI_BUF_CTL DDI Idle Status = 0b (Not Idle), timeout - * after 500 us. - * - * We only configure what the register value will be here. Actual - * enabling happens during link training farther down. - */ - intel_ddi_init_dp_buf_reg(encoder, crtc_state); - if (!is_mst) intel_dp_set_power(intel_dp, DP_SET_POWER_D0); @@ -2582,10 +2531,8 @@ static void hsw_ddi_pre_enable_dp(struct intel_atomic_state *state, struct intel_dp *intel_dp = enc_to_intel_dp(encoder); struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); enum port port = encoder->port; - enum phy phy = intel_port_to_phy(dev_priv, port); struct intel_digital_port *dig_port = enc_to_dig_port(encoder); bool is_mst = intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DP_MST); - int level = intel_ddi_dp_level(intel_dp); if (DISPLAY_VER(dev_priv) < 11) drm_WARN_ON(&dev_priv->drm, @@ -2597,12 +2544,17 @@ static void hsw_ddi_pre_enable_dp(struct intel_atomic_state *state, crtc_state->port_clock, crtc_state->lane_count); + /* + * We only configure what the register value will be here. Actual + * enabling happens during link training farther down. + */ + intel_ddi_init_dp_buf_reg(encoder, crtc_state); + intel_pps_on(intel_dp); intel_ddi_enable_clock(encoder, crtc_state); - if (!intel_phy_is_tc(dev_priv, phy) || - dig_port->tc_mode != TC_PORT_TBT_ALT) { + if (!intel_tc_port_in_tbt_alt_mode(dig_port)) { drm_WARN_ON(&dev_priv->drm, dig_port->ddi_io_wakeref); dig_port->ddi_io_wakeref = intel_display_power_get(dev_priv, dig_port->ddi_io_power_domain); @@ -2610,16 +2562,13 @@ static void hsw_ddi_pre_enable_dp(struct intel_atomic_state *state, icl_program_mg_dp_mode(dig_port, crtc_state); - if (DISPLAY_VER(dev_priv) >= 11) - icl_ddi_vswing_sequence(encoder, crtc_state, level); - else if (IS_GEMINILAKE(dev_priv) || IS_BROXTON(dev_priv)) - bxt_ddi_vswing_sequence(encoder, crtc_state, level); - else + if (has_buf_trans_select(dev_priv)) hsw_prepare_dp_ddi_buffers(encoder, crtc_state); + encoder->set_signal_levels(encoder, crtc_state); + intel_ddi_power_up_lanes(encoder, crtc_state); - intel_ddi_init_dp_buf_reg(encoder, crtc_state); if (!is_mst) intel_dp_set_power(intel_dp, DP_SET_POWER_D0); intel_dp_configure_protocol_converter(intel_dp, crtc_state); @@ -2772,7 +2721,6 @@ static void intel_ddi_post_disable_dp(struct intel_atomic_state *state, struct intel_dp *intel_dp = &dig_port->dp; bool is_mst = intel_crtc_has_type(old_crtc_state, INTEL_OUTPUT_DP_MST); - enum phy phy = intel_port_to_phy(dev_priv, encoder->port); if (!is_mst) intel_dp_set_infoframes(encoder, false, @@ -2815,8 +2763,7 @@ static void intel_ddi_post_disable_dp(struct intel_atomic_state *state, intel_pps_vdd_on(intel_dp); intel_pps_off(intel_dp); - if (!intel_phy_is_tc(dev_priv, phy) || - dig_port->tc_mode != TC_PORT_TBT_ALT) + if (!intel_tc_port_in_tbt_alt_mode(dig_port)) intel_display_power_put(dev_priv, dig_port->ddi_io_power_domain, fetch_and_zero(&dig_port->ddi_io_wakeref)); @@ -2862,7 +2809,7 @@ static void intel_ddi_post_disable(struct intel_atomic_state *state, if (!intel_crtc_has_type(old_crtc_state, INTEL_OUTPUT_DP_MST)) { intel_crtc_vblank_off(old_crtc_state); - intel_disable_pipe(old_crtc_state); + intel_disable_transcoder(old_crtc_state); intel_vrr_disable(old_crtc_state); @@ -3005,12 +2952,11 @@ static void intel_enable_ddi_dp(struct intel_atomic_state *state, intel_dp_stop_link_train(intel_dp, crtc_state); intel_edp_backlight_on(crtc_state, conn_state); - intel_psr_enable(intel_dp, crtc_state, conn_state); if (!dig_port->lspcon.active || dig_port->dp.has_hdmi_sink) intel_dp_set_infoframes(encoder, true, crtc_state, conn_state); - intel_edp_drrs_enable(intel_dp, crtc_state); + intel_drrs_enable(intel_dp, crtc_state); if (crtc_state->has_audio) intel_audio_codec_enable(encoder, crtc_state, conn_state); @@ -3046,7 +2992,6 @@ static void intel_enable_ddi_hdmi(struct intel_atomic_state *state, struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); struct intel_digital_port *dig_port = enc_to_dig_port(encoder); struct drm_connector *connector = conn_state->connector; - int level = intel_ddi_hdmi_level(encoder, crtc_state); enum port port = encoder->port; if (!intel_hdmi_handle_sink_scrambling(encoder, connector, @@ -3056,19 +3001,10 @@ static void intel_enable_ddi_hdmi(struct intel_atomic_state *state, "[CONNECTOR:%d:%s] Failed to configure sink scrambling/TMDS bit clock ratio\n", connector->base.id, connector->name); - if (IS_DG2(dev_priv)) - intel_snps_phy_ddi_vswing_sequence(encoder, U32_MAX); - else if (DISPLAY_VER(dev_priv) >= 12) - tgl_ddi_vswing_sequence(encoder, crtc_state, level); - else if (DISPLAY_VER(dev_priv) == 11) - icl_ddi_vswing_sequence(encoder, crtc_state, level); - else if (IS_GEMINILAKE(dev_priv) || IS_BROXTON(dev_priv)) - bxt_ddi_vswing_sequence(encoder, crtc_state, level); - else - hsw_prepare_hdmi_ddi_buffers(encoder, crtc_state, level); + if (has_buf_trans_select(dev_priv)) + hsw_prepare_hdmi_ddi_buffers(encoder, crtc_state); - if (DISPLAY_VER(dev_priv) == 9 && !IS_BROXTON(dev_priv)) - skl_ddi_set_iboost(encoder, crtc_state, level); + encoder->set_signal_levels(encoder, crtc_state); /* Display WA #1143: skl,kbl,cfl */ if (DISPLAY_VER(dev_priv) == 9 && !IS_BROXTON(dev_priv)) { @@ -3133,7 +3069,7 @@ static void intel_enable_ddi(struct intel_atomic_state *state, intel_vrr_enable(encoder, crtc_state); - intel_enable_pipe(crtc_state); + intel_enable_transcoder(crtc_state); intel_crtc_vblank_on(crtc_state); @@ -3198,7 +3134,7 @@ static void intel_pre_disable_ddi(struct intel_atomic_state *state, return; intel_dp = enc_to_intel_dp(encoder); - intel_edp_drrs_disable(intel_dp, old_crtc_state); + intel_drrs_disable(intel_dp, old_crtc_state); intel_psr_disable(intel_dp, old_crtc_state); } @@ -3226,11 +3162,10 @@ static void intel_ddi_update_pipe_dp(struct intel_atomic_state *state, intel_ddi_set_dp_msa(crtc_state, conn_state); - intel_psr_update(intel_dp, crtc_state, conn_state); intel_dp_set_infoframes(encoder, true, crtc_state, conn_state); - intel_edp_drrs_update(intel_dp, crtc_state); + intel_drrs_update(intel_dp, crtc_state); - intel_panel_update_backlight(state, encoder, crtc_state, conn_state); + intel_backlight_update(state, encoder, crtc_state, conn_state); } void intel_ddi_update_pipe(struct intel_atomic_state *state, @@ -3293,7 +3228,7 @@ intel_ddi_pre_pll_enable(struct intel_atomic_state *state, intel_ddi_main_link_aux_domain(dig_port)); } - if (is_tc_port && dig_port->tc_mode != TC_PORT_TBT_ALT) + if (is_tc_port && !intel_tc_port_in_tbt_alt_mode(dig_port)) /* * Program the lane count for static/dynamic connections on * Type-C ports. Skip this step for TBT. @@ -3553,9 +3488,6 @@ static void intel_ddi_read_func_ctl(struct intel_encoder *encoder, pipe_config->output_types |= BIT(INTEL_OUTPUT_HDMI); pipe_config->lane_count = 4; break; - case TRANS_DDI_MODE_SELECT_FDI: - pipe_config->output_types |= BIT(INTEL_OUTPUT_ANALOG); - break; case TRANS_DDI_MODE_SELECT_DP_SST: if (encoder->type == INTEL_OUTPUT_EDP) pipe_config->output_types |= BIT(INTEL_OUTPUT_EDP); @@ -3584,6 +3516,13 @@ static void intel_ddi_read_func_ctl(struct intel_encoder *encoder, pipe_config->infoframes.enable |= intel_hdmi_infoframes_enabled(encoder, pipe_config); break; + case TRANS_DDI_MODE_SELECT_FDI_OR_128B132B: + if (!HAS_DP20(dev_priv)) { + /* FDI */ + pipe_config->output_types |= BIT(INTEL_OUTPUT_ANALOG); + break; + } + fallthrough; /* 128b/132b */ case TRANS_DDI_MODE_SELECT_DP_MST: pipe_config->output_types |= BIT(INTEL_OUTPUT_DP_MST); pipe_config->lane_count = @@ -3995,13 +3934,15 @@ static void intel_ddi_encoder_destroy(struct drm_encoder *encoder) { struct drm_i915_private *i915 = to_i915(encoder->dev); struct intel_digital_port *dig_port = enc_to_dig_port(to_intel_encoder(encoder)); + enum phy phy = intel_port_to_phy(i915, dig_port->base.port); intel_dp_encoder_flush_work(encoder); + if (intel_phy_is_tc(i915, phy)) + intel_tc_port_flush_work(dig_port); intel_display_power_flush_work(i915); drm_encoder_cleanup(encoder); - if (dig_port) - kfree(dig_port->hdcp_port_data.streams); + kfree(dig_port->hdcp_port_data.streams); kfree(dig_port); } @@ -4022,7 +3963,6 @@ static const struct drm_encoder_funcs intel_ddi_funcs = { static struct intel_connector * intel_ddi_init_dp_connector(struct intel_digital_port *dig_port) { - struct drm_i915_private *dev_priv = to_i915(dig_port->base.base.dev); struct intel_connector *connector; enum port port = dig_port->base.port; @@ -4035,17 +3975,6 @@ intel_ddi_init_dp_connector(struct intel_digital_port *dig_port) dig_port->dp.set_link_train = intel_ddi_set_link_train; dig_port->dp.set_idle_link_train = intel_ddi_set_idle_link_train; - if (IS_DG2(dev_priv)) - dig_port->dp.set_signal_levels = dg2_set_signal_levels; - else if (DISPLAY_VER(dev_priv) >= 12) - dig_port->dp.set_signal_levels = tgl_set_signal_levels; - else if (DISPLAY_VER(dev_priv) >= 11) - dig_port->dp.set_signal_levels = icl_set_signal_levels; - else if (IS_GEMINILAKE(dev_priv) || IS_BROXTON(dev_priv)) - dig_port->dp.set_signal_levels = bxt_set_signal_levels; - else - dig_port->dp.set_signal_levels = hsw_set_signal_levels; - dig_port->dp.voltage_max = intel_ddi_dp_voltage_max; dig_port->dp.preemph_max = intel_ddi_dp_preemph_max; @@ -4421,7 +4350,7 @@ static void intel_ddi_encoder_suspend(struct intel_encoder *encoder) if (!intel_phy_is_tc(i915, phy)) return; - intel_tc_port_disconnect_phy(dig_port); + intel_tc_port_flush_work(dig_port); } static void intel_ddi_encoder_shutdown(struct intel_encoder *encoder) @@ -4436,7 +4365,7 @@ static void intel_ddi_encoder_shutdown(struct intel_encoder *encoder) if (!intel_phy_is_tc(i915, phy)) return; - intel_tc_port_disconnect_phy(dig_port); + intel_tc_port_flush_work(dig_port); } #define port_tc_name(port) ((port) - PORT_TC1 + '1') @@ -4617,6 +4546,24 @@ void intel_ddi_init(struct drm_i915_private *dev_priv, enum port port) encoder->get_config = hsw_ddi_get_config; } + if (IS_DG2(dev_priv)) { + encoder->set_signal_levels = intel_snps_phy_set_signal_levels; + } else if (DISPLAY_VER(dev_priv) >= 12) { + if (intel_phy_is_combo(dev_priv, phy)) + encoder->set_signal_levels = icl_combo_phy_set_signal_levels; + else + encoder->set_signal_levels = tgl_dkl_phy_set_signal_levels; + } else if (DISPLAY_VER(dev_priv) >= 11) { + if (intel_phy_is_combo(dev_priv, phy)) + encoder->set_signal_levels = icl_combo_phy_set_signal_levels; + else + encoder->set_signal_levels = icl_mg_phy_set_signal_levels; + } else if (IS_GEMINILAKE(dev_priv) || IS_BROXTON(dev_priv)) { + encoder->set_signal_levels = bxt_ddi_phy_set_signal_levels; + } else { + encoder->set_signal_levels = hsw_set_signal_levels; + } + intel_ddi_buf_trans_init(encoder); if (DISPLAY_VER(dev_priv) >= 13) diff --git a/drivers/gpu/drm/i915/display/intel_ddi.h b/drivers/gpu/drm/i915/display/intel_ddi.h index 7d448485d887..d6971717ef9c 100644 --- a/drivers/gpu/drm/i915/display/intel_ddi.h +++ b/drivers/gpu/drm/i915/display/intel_ddi.h @@ -59,13 +59,12 @@ void intel_ddi_set_vc_payload_alloc(const struct intel_crtc_state *crtc_state, bool state); void intel_ddi_compute_min_voltage_level(struct drm_i915_private *dev_priv, struct intel_crtc_state *crtc_state); -u32 bxt_signal_levels(struct intel_dp *intel_dp, - const struct intel_crtc_state *crtc_state); -u32 ddi_signal_levels(struct intel_dp *intel_dp, - const struct intel_crtc_state *crtc_state); int intel_ddi_toggle_hdcp_bits(struct intel_encoder *intel_encoder, enum transcoder cpu_transcoder, bool enable, u32 hdcp_mask); void intel_ddi_sanitize_encoder_pll_mapping(struct intel_encoder *encoder); +int intel_ddi_level(struct intel_encoder *encoder, + const struct intel_crtc_state *crtc_state, + int lane); #endif /* __INTEL_DDI_H__ */ diff --git a/drivers/gpu/drm/i915/display/intel_ddi_buf_trans.c b/drivers/gpu/drm/i915/display/intel_ddi_buf_trans.c index ba2c08f1a797..78cd8f77b49d 100644 --- a/drivers/gpu/drm/i915/display/intel_ddi_buf_trans.c +++ b/drivers/gpu/drm/i915/display/intel_ddi_buf_trans.c @@ -8,12 +8,13 @@ #include "intel_ddi_buf_trans.h" #include "intel_de.h" #include "intel_display_types.h" +#include "intel_dp.h" /* HDMI/DVI modes ignore everything but the last 2 items. So we share * them for both DP and FDI transports, allowing those ports to * automatically adapt to HDMI connections as well */ -static const union intel_ddi_buf_trans_entry _hsw_ddi_translations_dp[] = { +static const union intel_ddi_buf_trans_entry _hsw_trans_dp[] = { { .hsw = { 0x00FFFFFF, 0x0006000E, 0x0 } }, { .hsw = { 0x00D75FFF, 0x0005000A, 0x0 } }, { .hsw = { 0x00C30FFF, 0x00040006, 0x0 } }, @@ -25,12 +26,12 @@ static const union intel_ddi_buf_trans_entry _hsw_ddi_translations_dp[] = { { .hsw = { 0x80D75FFF, 0x000B0000, 0x0 } }, }; -static const struct intel_ddi_buf_trans hsw_ddi_translations_dp = { - .entries = _hsw_ddi_translations_dp, - .num_entries = ARRAY_SIZE(_hsw_ddi_translations_dp), +static const struct intel_ddi_buf_trans hsw_trans_dp = { + .entries = _hsw_trans_dp, + .num_entries = ARRAY_SIZE(_hsw_trans_dp), }; -static const union intel_ddi_buf_trans_entry _hsw_ddi_translations_fdi[] = { +static const union intel_ddi_buf_trans_entry _hsw_trans_fdi[] = { { .hsw = { 0x00FFFFFF, 0x0007000E, 0x0 } }, { .hsw = { 0x00D75FFF, 0x000F000A, 0x0 } }, { .hsw = { 0x00C30FFF, 0x00060006, 0x0 } }, @@ -42,12 +43,12 @@ static const union intel_ddi_buf_trans_entry _hsw_ddi_translations_fdi[] = { { .hsw = { 0x00D75FFF, 0x001E0000, 0x0 } }, }; -static const struct intel_ddi_buf_trans hsw_ddi_translations_fdi = { - .entries = _hsw_ddi_translations_fdi, - .num_entries = ARRAY_SIZE(_hsw_ddi_translations_fdi), +static const struct intel_ddi_buf_trans hsw_trans_fdi = { + .entries = _hsw_trans_fdi, + .num_entries = ARRAY_SIZE(_hsw_trans_fdi), }; -static const union intel_ddi_buf_trans_entry _hsw_ddi_translations_hdmi[] = { +static const union intel_ddi_buf_trans_entry _hsw_trans_hdmi[] = { /* Idx NT mV d T mV d db */ { .hsw = { 0x00FFFFFF, 0x0006000E, 0x0 } }, /* 0: 400 400 0 */ { .hsw = { 0x00E79FFF, 0x000E000C, 0x0 } }, /* 1: 400 500 2 */ @@ -63,13 +64,13 @@ static const union intel_ddi_buf_trans_entry _hsw_ddi_translations_hdmi[] = { { .hsw = { 0x80FFFFFF, 0x00030002, 0x0 } }, /* 11: 1000 1000 0 */ }; -static const struct intel_ddi_buf_trans hsw_ddi_translations_hdmi = { - .entries = _hsw_ddi_translations_hdmi, - .num_entries = ARRAY_SIZE(_hsw_ddi_translations_hdmi), +static const struct intel_ddi_buf_trans hsw_trans_hdmi = { + .entries = _hsw_trans_hdmi, + .num_entries = ARRAY_SIZE(_hsw_trans_hdmi), .hdmi_default_entry = 6, }; -static const union intel_ddi_buf_trans_entry _bdw_ddi_translations_edp[] = { +static const union intel_ddi_buf_trans_entry _bdw_trans_edp[] = { { .hsw = { 0x00FFFFFF, 0x00000012, 0x0 } }, { .hsw = { 0x00EBAFFF, 0x00020011, 0x0 } }, { .hsw = { 0x00C71FFF, 0x0006000F, 0x0 } }, @@ -81,12 +82,12 @@ static const union intel_ddi_buf_trans_entry _bdw_ddi_translations_edp[] = { { .hsw = { 0x00DB6FFF, 0x000A000C, 0x0 } }, }; -static const struct intel_ddi_buf_trans bdw_ddi_translations_edp = { - .entries = _bdw_ddi_translations_edp, - .num_entries = ARRAY_SIZE(_bdw_ddi_translations_edp), +static const struct intel_ddi_buf_trans bdw_trans_edp = { + .entries = _bdw_trans_edp, + .num_entries = ARRAY_SIZE(_bdw_trans_edp), }; -static const union intel_ddi_buf_trans_entry _bdw_ddi_translations_dp[] = { +static const union intel_ddi_buf_trans_entry _bdw_trans_dp[] = { { .hsw = { 0x00FFFFFF, 0x0007000E, 0x0 } }, { .hsw = { 0x00D75FFF, 0x000E000A, 0x0 } }, { .hsw = { 0x00BEFFFF, 0x00140006, 0x0 } }, @@ -98,12 +99,12 @@ static const union intel_ddi_buf_trans_entry _bdw_ddi_translations_dp[] = { { .hsw = { 0x80D75FFF, 0x001B0002, 0x0 } }, }; -static const struct intel_ddi_buf_trans bdw_ddi_translations_dp = { - .entries = _bdw_ddi_translations_dp, - .num_entries = ARRAY_SIZE(_bdw_ddi_translations_dp), +static const struct intel_ddi_buf_trans bdw_trans_dp = { + .entries = _bdw_trans_dp, + .num_entries = ARRAY_SIZE(_bdw_trans_dp), }; -static const union intel_ddi_buf_trans_entry _bdw_ddi_translations_fdi[] = { +static const union intel_ddi_buf_trans_entry _bdw_trans_fdi[] = { { .hsw = { 0x00FFFFFF, 0x0001000E, 0x0 } }, { .hsw = { 0x00D75FFF, 0x0004000A, 0x0 } }, { .hsw = { 0x00C30FFF, 0x00070006, 0x0 } }, @@ -115,12 +116,12 @@ static const union intel_ddi_buf_trans_entry _bdw_ddi_translations_fdi[] = { { .hsw = { 0x00D75FFF, 0x000C0000, 0x0 } }, }; -static const struct intel_ddi_buf_trans bdw_ddi_translations_fdi = { - .entries = _bdw_ddi_translations_fdi, - .num_entries = ARRAY_SIZE(_bdw_ddi_translations_fdi), +static const struct intel_ddi_buf_trans bdw_trans_fdi = { + .entries = _bdw_trans_fdi, + .num_entries = ARRAY_SIZE(_bdw_trans_fdi), }; -static const union intel_ddi_buf_trans_entry _bdw_ddi_translations_hdmi[] = { +static const union intel_ddi_buf_trans_entry _bdw_trans_hdmi[] = { /* Idx NT mV d T mV df db */ { .hsw = { 0x00FFFFFF, 0x0007000E, 0x0 } }, /* 0: 400 400 0 */ { .hsw = { 0x00D75FFF, 0x000E000A, 0x0 } }, /* 1: 400 600 3.5 */ @@ -134,14 +135,14 @@ static const union intel_ddi_buf_trans_entry _bdw_ddi_translations_hdmi[] = { { .hsw = { 0x80FFFFFF, 0x001B0002, 0x0 } }, /* 9: 1000 1000 0 */ }; -static const struct intel_ddi_buf_trans bdw_ddi_translations_hdmi = { - .entries = _bdw_ddi_translations_hdmi, - .num_entries = ARRAY_SIZE(_bdw_ddi_translations_hdmi), +static const struct intel_ddi_buf_trans bdw_trans_hdmi = { + .entries = _bdw_trans_hdmi, + .num_entries = ARRAY_SIZE(_bdw_trans_hdmi), .hdmi_default_entry = 7, }; /* Skylake H and S */ -static const union intel_ddi_buf_trans_entry _skl_ddi_translations_dp[] = { +static const union intel_ddi_buf_trans_entry _skl_trans_dp[] = { { .hsw = { 0x00002016, 0x000000A0, 0x0 } }, { .hsw = { 0x00005012, 0x0000009B, 0x0 } }, { .hsw = { 0x00007011, 0x00000088, 0x0 } }, @@ -153,13 +154,13 @@ static const union intel_ddi_buf_trans_entry _skl_ddi_translations_dp[] = { { .hsw = { 0x80005012, 0x000000C0, 0x1 } }, }; -static const struct intel_ddi_buf_trans skl_ddi_translations_dp = { - .entries = _skl_ddi_translations_dp, - .num_entries = ARRAY_SIZE(_skl_ddi_translations_dp), +static const struct intel_ddi_buf_trans skl_trans_dp = { + .entries = _skl_trans_dp, + .num_entries = ARRAY_SIZE(_skl_trans_dp), }; /* Skylake U */ -static const union intel_ddi_buf_trans_entry _skl_u_ddi_translations_dp[] = { +static const union intel_ddi_buf_trans_entry _skl_u_trans_dp[] = { { .hsw = { 0x0000201B, 0x000000A2, 0x0 } }, { .hsw = { 0x00005012, 0x00000088, 0x0 } }, { .hsw = { 0x80007011, 0x000000CD, 0x1 } }, @@ -171,13 +172,13 @@ static const union intel_ddi_buf_trans_entry _skl_u_ddi_translations_dp[] = { { .hsw = { 0x80005012, 0x000000C0, 0x1 } }, }; -static const struct intel_ddi_buf_trans skl_u_ddi_translations_dp = { - .entries = _skl_u_ddi_translations_dp, - .num_entries = ARRAY_SIZE(_skl_u_ddi_translations_dp), +static const struct intel_ddi_buf_trans skl_u_trans_dp = { + .entries = _skl_u_trans_dp, + .num_entries = ARRAY_SIZE(_skl_u_trans_dp), }; /* Skylake Y */ -static const union intel_ddi_buf_trans_entry _skl_y_ddi_translations_dp[] = { +static const union intel_ddi_buf_trans_entry _skl_y_trans_dp[] = { { .hsw = { 0x00000018, 0x000000A2, 0x0 } }, { .hsw = { 0x00005012, 0x00000088, 0x0 } }, { .hsw = { 0x80007011, 0x000000CD, 0x3 } }, @@ -189,13 +190,13 @@ static const union intel_ddi_buf_trans_entry _skl_y_ddi_translations_dp[] = { { .hsw = { 0x80005012, 0x000000C0, 0x3 } }, }; -static const struct intel_ddi_buf_trans skl_y_ddi_translations_dp = { - .entries = _skl_y_ddi_translations_dp, - .num_entries = ARRAY_SIZE(_skl_y_ddi_translations_dp), +static const struct intel_ddi_buf_trans skl_y_trans_dp = { + .entries = _skl_y_trans_dp, + .num_entries = ARRAY_SIZE(_skl_y_trans_dp), }; /* Kabylake H and S */ -static const union intel_ddi_buf_trans_entry _kbl_ddi_translations_dp[] = { +static const union intel_ddi_buf_trans_entry _kbl_trans_dp[] = { { .hsw = { 0x00002016, 0x000000A0, 0x0 } }, { .hsw = { 0x00005012, 0x0000009B, 0x0 } }, { .hsw = { 0x00007011, 0x00000088, 0x0 } }, @@ -207,13 +208,13 @@ static const union intel_ddi_buf_trans_entry _kbl_ddi_translations_dp[] = { { .hsw = { 0x80005012, 0x000000C0, 0x1 } }, }; -static const struct intel_ddi_buf_trans kbl_ddi_translations_dp = { - .entries = _kbl_ddi_translations_dp, - .num_entries = ARRAY_SIZE(_kbl_ddi_translations_dp), +static const struct intel_ddi_buf_trans kbl_trans_dp = { + .entries = _kbl_trans_dp, + .num_entries = ARRAY_SIZE(_kbl_trans_dp), }; /* Kabylake U */ -static const union intel_ddi_buf_trans_entry _kbl_u_ddi_translations_dp[] = { +static const union intel_ddi_buf_trans_entry _kbl_u_trans_dp[] = { { .hsw = { 0x0000201B, 0x000000A1, 0x0 } }, { .hsw = { 0x00005012, 0x00000088, 0x0 } }, { .hsw = { 0x80007011, 0x000000CD, 0x3 } }, @@ -225,13 +226,13 @@ static const union intel_ddi_buf_trans_entry _kbl_u_ddi_translations_dp[] = { { .hsw = { 0x80005012, 0x000000C0, 0x3 } }, }; -static const struct intel_ddi_buf_trans kbl_u_ddi_translations_dp = { - .entries = _kbl_u_ddi_translations_dp, - .num_entries = ARRAY_SIZE(_kbl_u_ddi_translations_dp), +static const struct intel_ddi_buf_trans kbl_u_trans_dp = { + .entries = _kbl_u_trans_dp, + .num_entries = ARRAY_SIZE(_kbl_u_trans_dp), }; /* Kabylake Y */ -static const union intel_ddi_buf_trans_entry _kbl_y_ddi_translations_dp[] = { +static const union intel_ddi_buf_trans_entry _kbl_y_trans_dp[] = { { .hsw = { 0x00001017, 0x000000A1, 0x0 } }, { .hsw = { 0x00005012, 0x00000088, 0x0 } }, { .hsw = { 0x80007011, 0x000000CD, 0x3 } }, @@ -243,16 +244,16 @@ static const union intel_ddi_buf_trans_entry _kbl_y_ddi_translations_dp[] = { { .hsw = { 0x80005012, 0x000000C0, 0x3 } }, }; -static const struct intel_ddi_buf_trans kbl_y_ddi_translations_dp = { - .entries = _kbl_y_ddi_translations_dp, - .num_entries = ARRAY_SIZE(_kbl_y_ddi_translations_dp), +static const struct intel_ddi_buf_trans kbl_y_trans_dp = { + .entries = _kbl_y_trans_dp, + .num_entries = ARRAY_SIZE(_kbl_y_trans_dp), }; /* * Skylake/Kabylake H and S * eDP 1.4 low vswing translation parameters */ -static const union intel_ddi_buf_trans_entry _skl_ddi_translations_edp[] = { +static const union intel_ddi_buf_trans_entry _skl_trans_edp[] = { { .hsw = { 0x00000018, 0x000000A8, 0x0 } }, { .hsw = { 0x00004013, 0x000000A9, 0x0 } }, { .hsw = { 0x00007011, 0x000000A2, 0x0 } }, @@ -265,16 +266,16 @@ static const union intel_ddi_buf_trans_entry _skl_ddi_translations_edp[] = { { .hsw = { 0x00000018, 0x000000DF, 0x0 } }, }; -static const struct intel_ddi_buf_trans skl_ddi_translations_edp = { - .entries = _skl_ddi_translations_edp, - .num_entries = ARRAY_SIZE(_skl_ddi_translations_edp), +static const struct intel_ddi_buf_trans skl_trans_edp = { + .entries = _skl_trans_edp, + .num_entries = ARRAY_SIZE(_skl_trans_edp), }; /* * Skylake/Kabylake U * eDP 1.4 low vswing translation parameters */ -static const union intel_ddi_buf_trans_entry _skl_u_ddi_translations_edp[] = { +static const union intel_ddi_buf_trans_entry _skl_u_trans_edp[] = { { .hsw = { 0x00000018, 0x000000A8, 0x0 } }, { .hsw = { 0x00004013, 0x000000A9, 0x0 } }, { .hsw = { 0x00007011, 0x000000A2, 0x0 } }, @@ -287,16 +288,16 @@ static const union intel_ddi_buf_trans_entry _skl_u_ddi_translations_edp[] = { { .hsw = { 0x00000018, 0x000000DF, 0x0 } }, }; -static const struct intel_ddi_buf_trans skl_u_ddi_translations_edp = { - .entries = _skl_u_ddi_translations_edp, - .num_entries = ARRAY_SIZE(_skl_u_ddi_translations_edp), +static const struct intel_ddi_buf_trans skl_u_trans_edp = { + .entries = _skl_u_trans_edp, + .num_entries = ARRAY_SIZE(_skl_u_trans_edp), }; /* * Skylake/Kabylake Y * eDP 1.4 low vswing translation parameters */ -static const union intel_ddi_buf_trans_entry _skl_y_ddi_translations_edp[] = { +static const union intel_ddi_buf_trans_entry _skl_y_trans_edp[] = { { .hsw = { 0x00000018, 0x000000A8, 0x0 } }, { .hsw = { 0x00004013, 0x000000AB, 0x0 } }, { .hsw = { 0x00007011, 0x000000A4, 0x0 } }, @@ -309,13 +310,13 @@ static const union intel_ddi_buf_trans_entry _skl_y_ddi_translations_edp[] = { { .hsw = { 0x00000018, 0x0000008A, 0x0 } }, }; -static const struct intel_ddi_buf_trans skl_y_ddi_translations_edp = { - .entries = _skl_y_ddi_translations_edp, - .num_entries = ARRAY_SIZE(_skl_y_ddi_translations_edp), +static const struct intel_ddi_buf_trans skl_y_trans_edp = { + .entries = _skl_y_trans_edp, + .num_entries = ARRAY_SIZE(_skl_y_trans_edp), }; /* Skylake/Kabylake U, H and S */ -static const union intel_ddi_buf_trans_entry _skl_ddi_translations_hdmi[] = { +static const union intel_ddi_buf_trans_entry _skl_trans_hdmi[] = { { .hsw = { 0x00000018, 0x000000AC, 0x0 } }, { .hsw = { 0x00005012, 0x0000009D, 0x0 } }, { .hsw = { 0x00007011, 0x00000088, 0x0 } }, @@ -329,14 +330,14 @@ static const union intel_ddi_buf_trans_entry _skl_ddi_translations_hdmi[] = { { .hsw = { 0x80000018, 0x000000C0, 0x1 } }, }; -static const struct intel_ddi_buf_trans skl_ddi_translations_hdmi = { - .entries = _skl_ddi_translations_hdmi, - .num_entries = ARRAY_SIZE(_skl_ddi_translations_hdmi), +static const struct intel_ddi_buf_trans skl_trans_hdmi = { + .entries = _skl_trans_hdmi, + .num_entries = ARRAY_SIZE(_skl_trans_hdmi), .hdmi_default_entry = 8, }; /* Skylake/Kabylake Y */ -static const union intel_ddi_buf_trans_entry _skl_y_ddi_translations_hdmi[] = { +static const union intel_ddi_buf_trans_entry _skl_y_trans_hdmi[] = { { .hsw = { 0x00000018, 0x000000A1, 0x0 } }, { .hsw = { 0x00005012, 0x000000DF, 0x0 } }, { .hsw = { 0x80007011, 0x000000CB, 0x3 } }, @@ -350,13 +351,13 @@ static const union intel_ddi_buf_trans_entry _skl_y_ddi_translations_hdmi[] = { { .hsw = { 0x80000018, 0x000000C0, 0x3 } }, }; -static const struct intel_ddi_buf_trans skl_y_ddi_translations_hdmi = { - .entries = _skl_y_ddi_translations_hdmi, - .num_entries = ARRAY_SIZE(_skl_y_ddi_translations_hdmi), +static const struct intel_ddi_buf_trans skl_y_trans_hdmi = { + .entries = _skl_y_trans_hdmi, + .num_entries = ARRAY_SIZE(_skl_y_trans_hdmi), .hdmi_default_entry = 8, }; -static const union intel_ddi_buf_trans_entry _bxt_ddi_translations_dp[] = { +static const union intel_ddi_buf_trans_entry _bxt_trans_dp[] = { /* Idx NT mV diff db */ { .bxt = { 52, 0x9A, 0, 128, } }, /* 0: 400 0 */ { .bxt = { 78, 0x9A, 0, 85, } }, /* 1: 400 3.5 */ @@ -370,12 +371,12 @@ static const union intel_ddi_buf_trans_entry _bxt_ddi_translations_dp[] = { { .bxt = { 154, 0x9A, 1, 128, } }, /* 9: 1200 0 */ }; -static const struct intel_ddi_buf_trans bxt_ddi_translations_dp = { - .entries = _bxt_ddi_translations_dp, - .num_entries = ARRAY_SIZE(_bxt_ddi_translations_dp), +static const struct intel_ddi_buf_trans bxt_trans_dp = { + .entries = _bxt_trans_dp, + .num_entries = ARRAY_SIZE(_bxt_trans_dp), }; -static const union intel_ddi_buf_trans_entry _bxt_ddi_translations_edp[] = { +static const union intel_ddi_buf_trans_entry _bxt_trans_edp[] = { /* Idx NT mV diff db */ { .bxt = { 26, 0, 0, 128, } }, /* 0: 200 0 */ { .bxt = { 38, 0, 0, 112, } }, /* 1: 200 1.5 */ @@ -389,15 +390,15 @@ static const union intel_ddi_buf_trans_entry _bxt_ddi_translations_edp[] = { { .bxt = { 48, 0, 0, 128, } }, /* 9: 300 0 */ }; -static const struct intel_ddi_buf_trans bxt_ddi_translations_edp = { - .entries = _bxt_ddi_translations_edp, - .num_entries = ARRAY_SIZE(_bxt_ddi_translations_edp), +static const struct intel_ddi_buf_trans bxt_trans_edp = { + .entries = _bxt_trans_edp, + .num_entries = ARRAY_SIZE(_bxt_trans_edp), }; /* BSpec has 2 recommended values - entries 0 and 8. * Using the entry with higher vswing. */ -static const union intel_ddi_buf_trans_entry _bxt_ddi_translations_hdmi[] = { +static const union intel_ddi_buf_trans_entry _bxt_trans_hdmi[] = { /* Idx NT mV diff db */ { .bxt = { 52, 0x9A, 0, 128, } }, /* 0: 400 0 */ { .bxt = { 52, 0x9A, 0, 85, } }, /* 1: 400 3.5 */ @@ -411,14 +412,14 @@ static const union intel_ddi_buf_trans_entry _bxt_ddi_translations_hdmi[] = { { .bxt = { 154, 0x9A, 1, 128, } }, /* 9: 1200 0 */ }; -static const struct intel_ddi_buf_trans bxt_ddi_translations_hdmi = { - .entries = _bxt_ddi_translations_hdmi, - .num_entries = ARRAY_SIZE(_bxt_ddi_translations_hdmi), - .hdmi_default_entry = ARRAY_SIZE(_bxt_ddi_translations_hdmi) - 1, +static const struct intel_ddi_buf_trans bxt_trans_hdmi = { + .entries = _bxt_trans_hdmi, + .num_entries = ARRAY_SIZE(_bxt_trans_hdmi), + .hdmi_default_entry = ARRAY_SIZE(_bxt_trans_hdmi) - 1, }; -/* icl_combo_phy_ddi_translations */ -static const union intel_ddi_buf_trans_entry _icl_combo_phy_ddi_translations_dp_hbr2_edp_hbr3[] = { +/* icl_combo_phy_trans */ +static const union intel_ddi_buf_trans_entry _icl_combo_phy_trans_dp_hbr2_edp_hbr3[] = { /* NT mV Trans mV db */ { .icl = { 0xA, 0x35, 0x3F, 0x00, 0x00 } }, /* 350 350 0.0 */ { .icl = { 0xA, 0x4F, 0x37, 0x00, 0x08 } }, /* 350 500 3.1 */ @@ -432,12 +433,12 @@ static const union intel_ddi_buf_trans_entry _icl_combo_phy_ddi_translations_dp_ { .icl = { 0x6, 0x7F, 0x3F, 0x00, 0x00 } }, /* 900 900 0.0 */ }; -static const struct intel_ddi_buf_trans icl_combo_phy_ddi_translations_dp_hbr2_edp_hbr3 = { - .entries = _icl_combo_phy_ddi_translations_dp_hbr2_edp_hbr3, - .num_entries = ARRAY_SIZE(_icl_combo_phy_ddi_translations_dp_hbr2_edp_hbr3), +static const struct intel_ddi_buf_trans icl_combo_phy_trans_dp_hbr2_edp_hbr3 = { + .entries = _icl_combo_phy_trans_dp_hbr2_edp_hbr3, + .num_entries = ARRAY_SIZE(_icl_combo_phy_trans_dp_hbr2_edp_hbr3), }; -static const union intel_ddi_buf_trans_entry _icl_combo_phy_ddi_translations_edp_hbr2[] = { +static const union intel_ddi_buf_trans_entry _icl_combo_phy_trans_edp_hbr2[] = { /* NT mV Trans mV db */ { .icl = { 0x0, 0x7F, 0x3F, 0x00, 0x00 } }, /* 200 200 0.0 */ { .icl = { 0x8, 0x7F, 0x38, 0x00, 0x07 } }, /* 200 250 1.9 */ @@ -451,12 +452,12 @@ static const union intel_ddi_buf_trans_entry _icl_combo_phy_ddi_translations_edp { .icl = { 0x9, 0x7F, 0x3F, 0x00, 0x00 } }, /* 350 350 0.0 */ }; -static const struct intel_ddi_buf_trans icl_combo_phy_ddi_translations_edp_hbr2 = { - .entries = _icl_combo_phy_ddi_translations_edp_hbr2, - .num_entries = ARRAY_SIZE(_icl_combo_phy_ddi_translations_edp_hbr2), +static const struct intel_ddi_buf_trans icl_combo_phy_trans_edp_hbr2 = { + .entries = _icl_combo_phy_trans_edp_hbr2, + .num_entries = ARRAY_SIZE(_icl_combo_phy_trans_edp_hbr2), }; -static const union intel_ddi_buf_trans_entry _icl_combo_phy_ddi_translations_hdmi[] = { +static const union intel_ddi_buf_trans_entry _icl_combo_phy_trans_hdmi[] = { /* NT mV Trans mV db */ { .icl = { 0xA, 0x60, 0x3F, 0x00, 0x00 } }, /* 450 450 0.0 */ { .icl = { 0xB, 0x73, 0x36, 0x00, 0x09 } }, /* 450 650 3.2 */ @@ -467,13 +468,13 @@ static const union intel_ddi_buf_trans_entry _icl_combo_phy_ddi_translations_hdm { .icl = { 0x6, 0x7F, 0x35, 0x00, 0x0A } }, /* 600 850 3.0 */ }; -static const struct intel_ddi_buf_trans icl_combo_phy_ddi_translations_hdmi = { - .entries = _icl_combo_phy_ddi_translations_hdmi, - .num_entries = ARRAY_SIZE(_icl_combo_phy_ddi_translations_hdmi), - .hdmi_default_entry = ARRAY_SIZE(_icl_combo_phy_ddi_translations_hdmi) - 1, +static const struct intel_ddi_buf_trans icl_combo_phy_trans_hdmi = { + .entries = _icl_combo_phy_trans_hdmi, + .num_entries = ARRAY_SIZE(_icl_combo_phy_trans_hdmi), + .hdmi_default_entry = ARRAY_SIZE(_icl_combo_phy_trans_hdmi) - 1, }; -static const union intel_ddi_buf_trans_entry _ehl_combo_phy_ddi_translations_dp[] = { +static const union intel_ddi_buf_trans_entry _ehl_combo_phy_trans_dp[] = { /* NT mV Trans mV db */ { .icl = { 0xA, 0x33, 0x3F, 0x00, 0x00 } }, /* 350 350 0.0 */ { .icl = { 0xA, 0x47, 0x36, 0x00, 0x09 } }, /* 350 500 3.1 */ @@ -487,12 +488,12 @@ static const union intel_ddi_buf_trans_entry _ehl_combo_phy_ddi_translations_dp[ { .icl = { 0x6, 0x7F, 0x3F, 0x00, 0x00 } }, /* 900 900 0.0 */ }; -static const struct intel_ddi_buf_trans ehl_combo_phy_ddi_translations_dp = { - .entries = _ehl_combo_phy_ddi_translations_dp, - .num_entries = ARRAY_SIZE(_ehl_combo_phy_ddi_translations_dp), +static const struct intel_ddi_buf_trans ehl_combo_phy_trans_dp = { + .entries = _ehl_combo_phy_trans_dp, + .num_entries = ARRAY_SIZE(_ehl_combo_phy_trans_dp), }; -static const union intel_ddi_buf_trans_entry _ehl_combo_phy_ddi_translations_edp_hbr2[] = { +static const union intel_ddi_buf_trans_entry _ehl_combo_phy_trans_edp_hbr2[] = { /* NT mV Trans mV db */ { .icl = { 0x8, 0x7F, 0x3F, 0x00, 0x00 } }, /* 200 200 0.0 */ { .icl = { 0x8, 0x7F, 0x3F, 0x00, 0x00 } }, /* 200 250 1.9 */ @@ -506,12 +507,12 @@ static const union intel_ddi_buf_trans_entry _ehl_combo_phy_ddi_translations_edp { .icl = { 0xA, 0x35, 0x3F, 0x00, 0x00 } }, /* 350 350 0.0 */ }; -static const struct intel_ddi_buf_trans ehl_combo_phy_ddi_translations_edp_hbr2 = { - .entries = _ehl_combo_phy_ddi_translations_edp_hbr2, - .num_entries = ARRAY_SIZE(_ehl_combo_phy_ddi_translations_edp_hbr2), +static const struct intel_ddi_buf_trans ehl_combo_phy_trans_edp_hbr2 = { + .entries = _ehl_combo_phy_trans_edp_hbr2, + .num_entries = ARRAY_SIZE(_ehl_combo_phy_trans_edp_hbr2), }; -static const union intel_ddi_buf_trans_entry _jsl_combo_phy_ddi_translations_edp_hbr[] = { +static const union intel_ddi_buf_trans_entry _jsl_combo_phy_trans_edp_hbr[] = { /* NT mV Trans mV db */ { .icl = { 0x8, 0x7F, 0x3F, 0x00, 0x00 } }, /* 200 200 0.0 */ { .icl = { 0x8, 0x7F, 0x38, 0x00, 0x07 } }, /* 200 250 1.9 */ @@ -525,12 +526,12 @@ static const union intel_ddi_buf_trans_entry _jsl_combo_phy_ddi_translations_edp { .icl = { 0xA, 0x35, 0x3F, 0x00, 0x00 } }, /* 350 350 0.0 */ }; -static const struct intel_ddi_buf_trans jsl_combo_phy_ddi_translations_edp_hbr = { - .entries = _jsl_combo_phy_ddi_translations_edp_hbr, - .num_entries = ARRAY_SIZE(_jsl_combo_phy_ddi_translations_edp_hbr), +static const struct intel_ddi_buf_trans jsl_combo_phy_trans_edp_hbr = { + .entries = _jsl_combo_phy_trans_edp_hbr, + .num_entries = ARRAY_SIZE(_jsl_combo_phy_trans_edp_hbr), }; -static const union intel_ddi_buf_trans_entry _jsl_combo_phy_ddi_translations_edp_hbr2[] = { +static const union intel_ddi_buf_trans_entry _jsl_combo_phy_trans_edp_hbr2[] = { /* NT mV Trans mV db */ { .icl = { 0x8, 0x7F, 0x3F, 0x00, 0x00 } }, /* 200 200 0.0 */ { .icl = { 0x8, 0x7F, 0x3F, 0x00, 0x00 } }, /* 200 250 1.9 */ @@ -544,12 +545,12 @@ static const union intel_ddi_buf_trans_entry _jsl_combo_phy_ddi_translations_edp { .icl = { 0xA, 0x35, 0x3F, 0x00, 0x00 } }, /* 350 350 0.0 */ }; -static const struct intel_ddi_buf_trans jsl_combo_phy_ddi_translations_edp_hbr2 = { - .entries = _jsl_combo_phy_ddi_translations_edp_hbr2, - .num_entries = ARRAY_SIZE(_jsl_combo_phy_ddi_translations_edp_hbr2), +static const struct intel_ddi_buf_trans jsl_combo_phy_trans_edp_hbr2 = { + .entries = _jsl_combo_phy_trans_edp_hbr2, + .num_entries = ARRAY_SIZE(_jsl_combo_phy_trans_edp_hbr2), }; -static const union intel_ddi_buf_trans_entry _dg1_combo_phy_ddi_translations_dp_rbr_hbr[] = { +static const union intel_ddi_buf_trans_entry _dg1_combo_phy_trans_dp_rbr_hbr[] = { /* NT mV Trans mV db */ { .icl = { 0xA, 0x32, 0x3F, 0x00, 0x00 } }, /* 350 350 0.0 */ { .icl = { 0xA, 0x48, 0x35, 0x00, 0x0A } }, /* 350 500 3.1 */ @@ -563,12 +564,12 @@ static const union intel_ddi_buf_trans_entry _dg1_combo_phy_ddi_translations_dp_ { .icl = { 0x6, 0x7F, 0x3F, 0x00, 0x00 } }, /* 900 900 0.0 */ }; -static const struct intel_ddi_buf_trans dg1_combo_phy_ddi_translations_dp_rbr_hbr = { - .entries = _dg1_combo_phy_ddi_translations_dp_rbr_hbr, - .num_entries = ARRAY_SIZE(_dg1_combo_phy_ddi_translations_dp_rbr_hbr), +static const struct intel_ddi_buf_trans dg1_combo_phy_trans_dp_rbr_hbr = { + .entries = _dg1_combo_phy_trans_dp_rbr_hbr, + .num_entries = ARRAY_SIZE(_dg1_combo_phy_trans_dp_rbr_hbr), }; -static const union intel_ddi_buf_trans_entry _dg1_combo_phy_ddi_translations_dp_hbr2_hbr3[] = { +static const union intel_ddi_buf_trans_entry _dg1_combo_phy_trans_dp_hbr2_hbr3[] = { /* NT mV Trans mV db */ { .icl = { 0xA, 0x32, 0x3F, 0x00, 0x00 } }, /* 350 350 0.0 */ { .icl = { 0xA, 0x48, 0x35, 0x00, 0x0A } }, /* 350 500 3.1 */ @@ -582,12 +583,12 @@ static const union intel_ddi_buf_trans_entry _dg1_combo_phy_ddi_translations_dp_ { .icl = { 0x6, 0x7F, 0x3F, 0x00, 0x00 } }, /* 900 900 0.0 */ }; -static const struct intel_ddi_buf_trans dg1_combo_phy_ddi_translations_dp_hbr2_hbr3 = { - .entries = _dg1_combo_phy_ddi_translations_dp_hbr2_hbr3, - .num_entries = ARRAY_SIZE(_dg1_combo_phy_ddi_translations_dp_hbr2_hbr3), +static const struct intel_ddi_buf_trans dg1_combo_phy_trans_dp_hbr2_hbr3 = { + .entries = _dg1_combo_phy_trans_dp_hbr2_hbr3, + .num_entries = ARRAY_SIZE(_dg1_combo_phy_trans_dp_hbr2_hbr3), }; -static const union intel_ddi_buf_trans_entry _icl_mg_phy_ddi_translations_rbr_hbr[] = { +static const union intel_ddi_buf_trans_entry _icl_mg_phy_trans_rbr_hbr[] = { /* Voltage swing pre-emphasis */ { .mg = { 0x18, 0x00, 0x00 } }, /* 0 0 */ { .mg = { 0x1D, 0x00, 0x05 } }, /* 0 1 */ @@ -601,12 +602,12 @@ static const union intel_ddi_buf_trans_entry _icl_mg_phy_ddi_translations_rbr_hb { .mg = { 0x3F, 0x00, 0x00 } }, /* 3 0 */ }; -static const struct intel_ddi_buf_trans icl_mg_phy_ddi_translations_rbr_hbr = { - .entries = _icl_mg_phy_ddi_translations_rbr_hbr, - .num_entries = ARRAY_SIZE(_icl_mg_phy_ddi_translations_rbr_hbr), +static const struct intel_ddi_buf_trans icl_mg_phy_trans_rbr_hbr = { + .entries = _icl_mg_phy_trans_rbr_hbr, + .num_entries = ARRAY_SIZE(_icl_mg_phy_trans_rbr_hbr), }; -static const union intel_ddi_buf_trans_entry _icl_mg_phy_ddi_translations_hbr2_hbr3[] = { +static const union intel_ddi_buf_trans_entry _icl_mg_phy_trans_hbr2_hbr3[] = { /* Voltage swing pre-emphasis */ { .mg = { 0x18, 0x00, 0x00 } }, /* 0 0 */ { .mg = { 0x1D, 0x00, 0x05 } }, /* 0 1 */ @@ -620,12 +621,12 @@ static const union intel_ddi_buf_trans_entry _icl_mg_phy_ddi_translations_hbr2_h { .mg = { 0x3F, 0x00, 0x00 } }, /* 3 0 */ }; -static const struct intel_ddi_buf_trans icl_mg_phy_ddi_translations_hbr2_hbr3 = { - .entries = _icl_mg_phy_ddi_translations_hbr2_hbr3, - .num_entries = ARRAY_SIZE(_icl_mg_phy_ddi_translations_hbr2_hbr3), +static const struct intel_ddi_buf_trans icl_mg_phy_trans_hbr2_hbr3 = { + .entries = _icl_mg_phy_trans_hbr2_hbr3, + .num_entries = ARRAY_SIZE(_icl_mg_phy_trans_hbr2_hbr3), }; -static const union intel_ddi_buf_trans_entry _icl_mg_phy_ddi_translations_hdmi[] = { +static const union intel_ddi_buf_trans_entry _icl_mg_phy_trans_hdmi[] = { /* HDMI Preset VS Pre-emph */ { .mg = { 0x1A, 0x0, 0x0 } }, /* 1 400mV 0dB */ { .mg = { 0x20, 0x0, 0x0 } }, /* 2 500mV 0dB */ @@ -639,13 +640,13 @@ static const union intel_ddi_buf_trans_entry _icl_mg_phy_ddi_translations_hdmi[] { .mg = { 0x36, 0x0, 0x9 } }, /* 10 Full -3 dB */ }; -static const struct intel_ddi_buf_trans icl_mg_phy_ddi_translations_hdmi = { - .entries = _icl_mg_phy_ddi_translations_hdmi, - .num_entries = ARRAY_SIZE(_icl_mg_phy_ddi_translations_hdmi), - .hdmi_default_entry = ARRAY_SIZE(_icl_mg_phy_ddi_translations_hdmi) - 1, +static const struct intel_ddi_buf_trans icl_mg_phy_trans_hdmi = { + .entries = _icl_mg_phy_trans_hdmi, + .num_entries = ARRAY_SIZE(_icl_mg_phy_trans_hdmi), + .hdmi_default_entry = ARRAY_SIZE(_icl_mg_phy_trans_hdmi) - 1, }; -static const union intel_ddi_buf_trans_entry _tgl_dkl_phy_ddi_translations_dp_hbr[] = { +static const union intel_ddi_buf_trans_entry _tgl_dkl_phy_trans_dp_hbr[] = { /* VS pre-emp Non-trans mV Pre-emph dB */ { .dkl = { 0x7, 0x0, 0x00 } }, /* 0 0 400mV 0 dB */ { .dkl = { 0x5, 0x0, 0x05 } }, /* 0 1 400mV 3.5 dB */ @@ -659,12 +660,12 @@ static const union intel_ddi_buf_trans_entry _tgl_dkl_phy_ddi_translations_dp_hb { .dkl = { 0x0, 0x0, 0x00 } }, /* 3 0 1200mV 0 dB HDMI default */ }; -static const struct intel_ddi_buf_trans tgl_dkl_phy_ddi_translations_dp_hbr = { - .entries = _tgl_dkl_phy_ddi_translations_dp_hbr, - .num_entries = ARRAY_SIZE(_tgl_dkl_phy_ddi_translations_dp_hbr), +static const struct intel_ddi_buf_trans tgl_dkl_phy_trans_dp_hbr = { + .entries = _tgl_dkl_phy_trans_dp_hbr, + .num_entries = ARRAY_SIZE(_tgl_dkl_phy_trans_dp_hbr), }; -static const union intel_ddi_buf_trans_entry _tgl_dkl_phy_ddi_translations_dp_hbr2[] = { +static const union intel_ddi_buf_trans_entry _tgl_dkl_phy_trans_dp_hbr2[] = { /* VS pre-emp Non-trans mV Pre-emph dB */ { .dkl = { 0x7, 0x0, 0x00 } }, /* 0 0 400mV 0 dB */ { .dkl = { 0x5, 0x0, 0x05 } }, /* 0 1 400mV 3.5 dB */ @@ -678,12 +679,12 @@ static const union intel_ddi_buf_trans_entry _tgl_dkl_phy_ddi_translations_dp_hb { .dkl = { 0x0, 0x0, 0x00 } }, /* 3 0 1200mV 0 dB HDMI default */ }; -static const struct intel_ddi_buf_trans tgl_dkl_phy_ddi_translations_dp_hbr2 = { - .entries = _tgl_dkl_phy_ddi_translations_dp_hbr2, - .num_entries = ARRAY_SIZE(_tgl_dkl_phy_ddi_translations_dp_hbr2), +static const struct intel_ddi_buf_trans tgl_dkl_phy_trans_dp_hbr2 = { + .entries = _tgl_dkl_phy_trans_dp_hbr2, + .num_entries = ARRAY_SIZE(_tgl_dkl_phy_trans_dp_hbr2), }; -static const union intel_ddi_buf_trans_entry _tgl_dkl_phy_ddi_translations_hdmi[] = { +static const union intel_ddi_buf_trans_entry _tgl_dkl_phy_trans_hdmi[] = { /* HDMI Preset VS Pre-emph */ { .dkl = { 0x7, 0x0, 0x0 } }, /* 1 400mV 0dB */ { .dkl = { 0x6, 0x0, 0x0 } }, /* 2 500mV 0dB */ @@ -697,13 +698,13 @@ static const union intel_ddi_buf_trans_entry _tgl_dkl_phy_ddi_translations_hdmi[ { .dkl = { 0x0, 0x0, 0xA } }, /* 10 Full -3 dB */ }; -static const struct intel_ddi_buf_trans tgl_dkl_phy_ddi_translations_hdmi = { - .entries = _tgl_dkl_phy_ddi_translations_hdmi, - .num_entries = ARRAY_SIZE(_tgl_dkl_phy_ddi_translations_hdmi), - .hdmi_default_entry = ARRAY_SIZE(_tgl_dkl_phy_ddi_translations_hdmi) - 1, +static const struct intel_ddi_buf_trans tgl_dkl_phy_trans_hdmi = { + .entries = _tgl_dkl_phy_trans_hdmi, + .num_entries = ARRAY_SIZE(_tgl_dkl_phy_trans_hdmi), + .hdmi_default_entry = ARRAY_SIZE(_tgl_dkl_phy_trans_hdmi) - 1, }; -static const union intel_ddi_buf_trans_entry _tgl_combo_phy_ddi_translations_dp_hbr[] = { +static const union intel_ddi_buf_trans_entry _tgl_combo_phy_trans_dp_hbr[] = { /* NT mV Trans mV db */ { .icl = { 0xA, 0x32, 0x3F, 0x00, 0x00 } }, /* 350 350 0.0 */ { .icl = { 0xA, 0x4F, 0x37, 0x00, 0x08 } }, /* 350 500 3.1 */ @@ -717,12 +718,12 @@ static const union intel_ddi_buf_trans_entry _tgl_combo_phy_ddi_translations_dp_ { .icl = { 0x6, 0x7F, 0x3F, 0x00, 0x00 } }, /* 900 900 0.0 */ }; -static const struct intel_ddi_buf_trans tgl_combo_phy_ddi_translations_dp_hbr = { - .entries = _tgl_combo_phy_ddi_translations_dp_hbr, - .num_entries = ARRAY_SIZE(_tgl_combo_phy_ddi_translations_dp_hbr), +static const struct intel_ddi_buf_trans tgl_combo_phy_trans_dp_hbr = { + .entries = _tgl_combo_phy_trans_dp_hbr, + .num_entries = ARRAY_SIZE(_tgl_combo_phy_trans_dp_hbr), }; -static const union intel_ddi_buf_trans_entry _tgl_combo_phy_ddi_translations_dp_hbr2[] = { +static const union intel_ddi_buf_trans_entry _tgl_combo_phy_trans_dp_hbr2[] = { /* NT mV Trans mV db */ { .icl = { 0xA, 0x35, 0x3F, 0x00, 0x00 } }, /* 350 350 0.0 */ { .icl = { 0xA, 0x4F, 0x37, 0x00, 0x08 } }, /* 350 500 3.1 */ @@ -736,12 +737,12 @@ static const union intel_ddi_buf_trans_entry _tgl_combo_phy_ddi_translations_dp_ { .icl = { 0x6, 0x7F, 0x3F, 0x00, 0x00 } }, /* 900 900 0.0 */ }; -static const struct intel_ddi_buf_trans tgl_combo_phy_ddi_translations_dp_hbr2 = { - .entries = _tgl_combo_phy_ddi_translations_dp_hbr2, - .num_entries = ARRAY_SIZE(_tgl_combo_phy_ddi_translations_dp_hbr2), +static const struct intel_ddi_buf_trans tgl_combo_phy_trans_dp_hbr2 = { + .entries = _tgl_combo_phy_trans_dp_hbr2, + .num_entries = ARRAY_SIZE(_tgl_combo_phy_trans_dp_hbr2), }; -static const union intel_ddi_buf_trans_entry _tgl_uy_combo_phy_ddi_translations_dp_hbr2[] = { +static const union intel_ddi_buf_trans_entry _tgl_uy_combo_phy_trans_dp_hbr2[] = { /* NT mV Trans mV db */ { .icl = { 0xA, 0x35, 0x3F, 0x00, 0x00 } }, /* 350 350 0.0 */ { .icl = { 0xA, 0x4F, 0x36, 0x00, 0x09 } }, /* 350 500 3.1 */ @@ -755,16 +756,16 @@ static const union intel_ddi_buf_trans_entry _tgl_uy_combo_phy_ddi_translations_ { .icl = { 0x6, 0x7F, 0x3F, 0x00, 0x00 } }, /* 900 900 0.0 */ }; -static const struct intel_ddi_buf_trans tgl_uy_combo_phy_ddi_translations_dp_hbr2 = { - .entries = _tgl_uy_combo_phy_ddi_translations_dp_hbr2, - .num_entries = ARRAY_SIZE(_tgl_uy_combo_phy_ddi_translations_dp_hbr2), +static const struct intel_ddi_buf_trans tgl_uy_combo_phy_trans_dp_hbr2 = { + .entries = _tgl_uy_combo_phy_trans_dp_hbr2, + .num_entries = ARRAY_SIZE(_tgl_uy_combo_phy_trans_dp_hbr2), }; /* * Cloned the HOBL entry to comply with the voltage and pre-emphasis entries * that DisplayPort specification requires */ -static const union intel_ddi_buf_trans_entry _tgl_combo_phy_ddi_translations_edp_hbr2_hobl[] = { +static const union intel_ddi_buf_trans_entry _tgl_combo_phy_trans_edp_hbr2_hobl[] = { /* VS pre-emp */ { .icl = { 0x6, 0x7F, 0x3F, 0x00, 0x00 } }, /* 0 0 */ { .icl = { 0x6, 0x7F, 0x3F, 0x00, 0x00 } }, /* 0 1 */ @@ -777,12 +778,12 @@ static const union intel_ddi_buf_trans_entry _tgl_combo_phy_ddi_translations_edp { .icl = { 0x6, 0x7F, 0x3F, 0x00, 0x00 } }, /* 2 1 */ }; -static const struct intel_ddi_buf_trans tgl_combo_phy_ddi_translations_edp_hbr2_hobl = { - .entries = _tgl_combo_phy_ddi_translations_edp_hbr2_hobl, - .num_entries = ARRAY_SIZE(_tgl_combo_phy_ddi_translations_edp_hbr2_hobl), +static const struct intel_ddi_buf_trans tgl_combo_phy_trans_edp_hbr2_hobl = { + .entries = _tgl_combo_phy_trans_edp_hbr2_hobl, + .num_entries = ARRAY_SIZE(_tgl_combo_phy_trans_edp_hbr2_hobl), }; -static const union intel_ddi_buf_trans_entry _rkl_combo_phy_ddi_translations_dp_hbr[] = { +static const union intel_ddi_buf_trans_entry _rkl_combo_phy_trans_dp_hbr[] = { /* NT mV Trans mV db */ { .icl = { 0xA, 0x2F, 0x3F, 0x00, 0x00 } }, /* 350 350 0.0 */ { .icl = { 0xA, 0x4F, 0x37, 0x00, 0x08 } }, /* 350 500 3.1 */ @@ -796,12 +797,12 @@ static const union intel_ddi_buf_trans_entry _rkl_combo_phy_ddi_translations_dp_ { .icl = { 0x6, 0x7F, 0x3F, 0x00, 0x00 } }, /* 900 900 0.0 */ }; -static const struct intel_ddi_buf_trans rkl_combo_phy_ddi_translations_dp_hbr = { - .entries = _rkl_combo_phy_ddi_translations_dp_hbr, - .num_entries = ARRAY_SIZE(_rkl_combo_phy_ddi_translations_dp_hbr), +static const struct intel_ddi_buf_trans rkl_combo_phy_trans_dp_hbr = { + .entries = _rkl_combo_phy_trans_dp_hbr, + .num_entries = ARRAY_SIZE(_rkl_combo_phy_trans_dp_hbr), }; -static const union intel_ddi_buf_trans_entry _rkl_combo_phy_ddi_translations_dp_hbr2_hbr3[] = { +static const union intel_ddi_buf_trans_entry _rkl_combo_phy_trans_dp_hbr2_hbr3[] = { /* NT mV Trans mV db */ { .icl = { 0xA, 0x35, 0x3F, 0x00, 0x00 } }, /* 350 350 0.0 */ { .icl = { 0xA, 0x50, 0x38, 0x00, 0x07 } }, /* 350 500 3.1 */ @@ -815,12 +816,12 @@ static const union intel_ddi_buf_trans_entry _rkl_combo_phy_ddi_translations_dp_ { .icl = { 0x6, 0x7F, 0x3F, 0x00, 0x00 } }, /* 900 900 0.0 */ }; -static const struct intel_ddi_buf_trans rkl_combo_phy_ddi_translations_dp_hbr2_hbr3 = { - .entries = _rkl_combo_phy_ddi_translations_dp_hbr2_hbr3, - .num_entries = ARRAY_SIZE(_rkl_combo_phy_ddi_translations_dp_hbr2_hbr3), +static const struct intel_ddi_buf_trans rkl_combo_phy_trans_dp_hbr2_hbr3 = { + .entries = _rkl_combo_phy_trans_dp_hbr2_hbr3, + .num_entries = ARRAY_SIZE(_rkl_combo_phy_trans_dp_hbr2_hbr3), }; -static const union intel_ddi_buf_trans_entry _adls_combo_phy_ddi_translations_dp_hbr2_hbr3[] = { +static const union intel_ddi_buf_trans_entry _adls_combo_phy_trans_dp_hbr2_hbr3[] = { /* NT mV Trans mV db */ { .icl = { 0xA, 0x35, 0x3F, 0x00, 0x00 } }, /* 350 350 0.0 */ { .icl = { 0xA, 0x4F, 0x37, 0x00, 0x08 } }, /* 350 500 3.1 */ @@ -834,12 +835,12 @@ static const union intel_ddi_buf_trans_entry _adls_combo_phy_ddi_translations_dp { .icl = { 0x6, 0x7F, 0x3F, 0x00, 0x00 } }, /* 900 900 0.0 */ }; -static const struct intel_ddi_buf_trans adls_combo_phy_ddi_translations_dp_hbr2_hbr3 = { - .entries = _adls_combo_phy_ddi_translations_dp_hbr2_hbr3, - .num_entries = ARRAY_SIZE(_adls_combo_phy_ddi_translations_dp_hbr2_hbr3), +static const struct intel_ddi_buf_trans adls_combo_phy_trans_dp_hbr2_hbr3 = { + .entries = _adls_combo_phy_trans_dp_hbr2_hbr3, + .num_entries = ARRAY_SIZE(_adls_combo_phy_trans_dp_hbr2_hbr3), }; -static const union intel_ddi_buf_trans_entry _adls_combo_phy_ddi_translations_edp_hbr2[] = { +static const union intel_ddi_buf_trans_entry _adls_combo_phy_trans_edp_hbr2[] = { /* NT mV Trans mV db */ { .icl = { 0x9, 0x73, 0x3D, 0x00, 0x02 } }, /* 200 200 0.0 */ { .icl = { 0x9, 0x7A, 0x3C, 0x00, 0x03 } }, /* 200 250 1.9 */ @@ -853,12 +854,12 @@ static const union intel_ddi_buf_trans_entry _adls_combo_phy_ddi_translations_ed { .icl = { 0x4, 0x6C, 0x3A, 0x00, 0x05 } }, /* 350 350 0.0 */ }; -static const struct intel_ddi_buf_trans adls_combo_phy_ddi_translations_edp_hbr2 = { - .entries = _adls_combo_phy_ddi_translations_edp_hbr2, - .num_entries = ARRAY_SIZE(_adls_combo_phy_ddi_translations_edp_hbr2), +static const struct intel_ddi_buf_trans adls_combo_phy_trans_edp_hbr2 = { + .entries = _adls_combo_phy_trans_edp_hbr2, + .num_entries = ARRAY_SIZE(_adls_combo_phy_trans_edp_hbr2), }; -static const union intel_ddi_buf_trans_entry _adls_combo_phy_ddi_translations_edp_hbr3[] = { +static const union intel_ddi_buf_trans_entry _adls_combo_phy_trans_edp_hbr3[] = { /* NT mV Trans mV db */ { .icl = { 0xA, 0x35, 0x3F, 0x00, 0x00 } }, /* 350 350 0.0 */ { .icl = { 0xA, 0x4F, 0x37, 0x00, 0x08 } }, /* 350 500 3.1 */ @@ -872,12 +873,12 @@ static const union intel_ddi_buf_trans_entry _adls_combo_phy_ddi_translations_ed { .icl = { 0x6, 0x7F, 0x3F, 0x00, 0x00 } }, /* 900 900 0.0 */ }; -static const struct intel_ddi_buf_trans adls_combo_phy_ddi_translations_edp_hbr3 = { - .entries = _adls_combo_phy_ddi_translations_edp_hbr3, - .num_entries = ARRAY_SIZE(_adls_combo_phy_ddi_translations_edp_hbr3), +static const struct intel_ddi_buf_trans adls_combo_phy_trans_edp_hbr3 = { + .entries = _adls_combo_phy_trans_edp_hbr3, + .num_entries = ARRAY_SIZE(_adls_combo_phy_trans_edp_hbr3), }; -static const union intel_ddi_buf_trans_entry _adlp_combo_phy_ddi_translations_hdmi[] = { +static const union intel_ddi_buf_trans_entry _adlp_combo_phy_trans_hdmi[] = { /* NT mV Trans mV db */ { .icl = { 0x6, 0x60, 0x3F, 0x00, 0x00 } }, /* 400 400 0.0 */ { .icl = { 0x6, 0x68, 0x3F, 0x00, 0x00 } }, /* 500 500 0.0 */ @@ -891,13 +892,13 @@ static const union intel_ddi_buf_trans_entry _adlp_combo_phy_ddi_translations_hd { .icl = { 0xB, 0x7F, 0x33, 0x00, 0x0C } }, /* Full Red -3.0 */ }; -static const struct intel_ddi_buf_trans adlp_combo_phy_ddi_translations_hdmi = { - .entries = _adlp_combo_phy_ddi_translations_hdmi, - .num_entries = ARRAY_SIZE(_adlp_combo_phy_ddi_translations_hdmi), - .hdmi_default_entry = ARRAY_SIZE(_adlp_combo_phy_ddi_translations_hdmi) - 1, +static const struct intel_ddi_buf_trans adlp_combo_phy_trans_hdmi = { + .entries = _adlp_combo_phy_trans_hdmi, + .num_entries = ARRAY_SIZE(_adlp_combo_phy_trans_hdmi), + .hdmi_default_entry = ARRAY_SIZE(_adlp_combo_phy_trans_hdmi) - 1, }; -static const union intel_ddi_buf_trans_entry _adlp_combo_phy_ddi_translations_dp_hbr[] = { +static const union intel_ddi_buf_trans_entry _adlp_combo_phy_trans_dp_hbr[] = { /* NT mV Trans mV db */ { .icl = { 0xA, 0x35, 0x3F, 0x00, 0x00 } }, /* 350 350 0.0 */ { .icl = { 0xA, 0x4F, 0x37, 0x00, 0x08 } }, /* 350 500 3.1 */ @@ -911,12 +912,12 @@ static const union intel_ddi_buf_trans_entry _adlp_combo_phy_ddi_translations_dp { .icl = { 0x6, 0x7F, 0x3F, 0x00, 0x00 } }, /* 900 900 0.0 */ }; -static const struct intel_ddi_buf_trans adlp_combo_phy_ddi_translations_dp_hbr = { - .entries = _adlp_combo_phy_ddi_translations_dp_hbr, - .num_entries = ARRAY_SIZE(_adlp_combo_phy_ddi_translations_dp_hbr), +static const struct intel_ddi_buf_trans adlp_combo_phy_trans_dp_hbr = { + .entries = _adlp_combo_phy_trans_dp_hbr, + .num_entries = ARRAY_SIZE(_adlp_combo_phy_trans_dp_hbr), }; -static const union intel_ddi_buf_trans_entry _adlp_combo_phy_ddi_translations_dp_hbr2_hbr3[] = { +static const union intel_ddi_buf_trans_entry _adlp_combo_phy_trans_dp_hbr2_hbr3[] = { /* NT mV Trans mV db */ { .icl = { 0xA, 0x35, 0x3F, 0x00, 0x00 } }, /* 350 350 0.0 */ { .icl = { 0xA, 0x4F, 0x37, 0x00, 0x08 } }, /* 350 500 3.1 */ @@ -930,22 +931,22 @@ static const union intel_ddi_buf_trans_entry _adlp_combo_phy_ddi_translations_dp { .icl = { 0x6, 0x7F, 0x3F, 0x00, 0x00 } }, /* 900 900 0.0 */ }; -static const struct intel_ddi_buf_trans adlp_combo_phy_ddi_translations_dp_hbr2_hbr3 = { - .entries = _adlp_combo_phy_ddi_translations_dp_hbr2_hbr3, - .num_entries = ARRAY_SIZE(_adlp_combo_phy_ddi_translations_dp_hbr2_hbr3), +static const struct intel_ddi_buf_trans adlp_combo_phy_trans_dp_hbr2_hbr3 = { + .entries = _adlp_combo_phy_trans_dp_hbr2_hbr3, + .num_entries = ARRAY_SIZE(_adlp_combo_phy_trans_dp_hbr2_hbr3), }; -static const struct intel_ddi_buf_trans adlp_combo_phy_ddi_translations_edp_hbr3 = { - .entries = _icl_combo_phy_ddi_translations_dp_hbr2_edp_hbr3, - .num_entries = ARRAY_SIZE(_icl_combo_phy_ddi_translations_dp_hbr2_edp_hbr3), +static const struct intel_ddi_buf_trans adlp_combo_phy_trans_edp_hbr3 = { + .entries = _icl_combo_phy_trans_dp_hbr2_edp_hbr3, + .num_entries = ARRAY_SIZE(_icl_combo_phy_trans_dp_hbr2_edp_hbr3), }; -static const struct intel_ddi_buf_trans adlp_combo_phy_ddi_translations_edp_up_to_hbr2 = { - .entries = _icl_combo_phy_ddi_translations_edp_hbr2, - .num_entries = ARRAY_SIZE(_icl_combo_phy_ddi_translations_edp_hbr2), +static const struct intel_ddi_buf_trans adlp_combo_phy_trans_edp_up_to_hbr2 = { + .entries = _icl_combo_phy_trans_edp_hbr2, + .num_entries = ARRAY_SIZE(_icl_combo_phy_trans_edp_hbr2), }; -static const union intel_ddi_buf_trans_entry _adlp_dkl_phy_ddi_translations_dp_hbr[] = { +static const union intel_ddi_buf_trans_entry _adlp_dkl_phy_trans_dp_hbr[] = { /* VS pre-emp Non-trans mV Pre-emph dB */ { .dkl = { 0x7, 0x0, 0x01 } }, /* 0 0 400mV 0 dB */ { .dkl = { 0x5, 0x0, 0x06 } }, /* 0 1 400mV 3.5 dB */ @@ -959,12 +960,12 @@ static const union intel_ddi_buf_trans_entry _adlp_dkl_phy_ddi_translations_dp_h { .dkl = { 0x0, 0x0, 0x00 } }, /* 3 0 1200mV 0 dB */ }; -static const struct intel_ddi_buf_trans adlp_dkl_phy_ddi_translations_dp_hbr = { - .entries = _adlp_dkl_phy_ddi_translations_dp_hbr, - .num_entries = ARRAY_SIZE(_adlp_dkl_phy_ddi_translations_dp_hbr), +static const struct intel_ddi_buf_trans adlp_dkl_phy_trans_dp_hbr = { + .entries = _adlp_dkl_phy_trans_dp_hbr, + .num_entries = ARRAY_SIZE(_adlp_dkl_phy_trans_dp_hbr), }; -static const union intel_ddi_buf_trans_entry _adlp_dkl_phy_ddi_translations_dp_hbr2_hbr3[] = { +static const union intel_ddi_buf_trans_entry _adlp_dkl_phy_trans_dp_hbr2_hbr3[] = { /* VS pre-emp Non-trans mV Pre-emph dB */ { .dkl = { 0x7, 0x0, 0x00 } }, /* 0 0 400mV 0 dB */ { .dkl = { 0x5, 0x0, 0x04 } }, /* 0 1 400mV 3.5 dB */ @@ -978,21 +979,64 @@ static const union intel_ddi_buf_trans_entry _adlp_dkl_phy_ddi_translations_dp_h { .dkl = { 0x0, 0x0, 0x00 } }, /* 3 0 1200mV 0 dB */ }; -static const struct intel_ddi_buf_trans adlp_dkl_phy_ddi_translations_dp_hbr2_hbr3 = { - .entries = _adlp_dkl_phy_ddi_translations_dp_hbr2_hbr3, - .num_entries = ARRAY_SIZE(_adlp_dkl_phy_ddi_translations_dp_hbr2_hbr3), +static const struct intel_ddi_buf_trans adlp_dkl_phy_trans_dp_hbr2_hbr3 = { + .entries = _adlp_dkl_phy_trans_dp_hbr2_hbr3, + .num_entries = ARRAY_SIZE(_adlp_dkl_phy_trans_dp_hbr2_hbr3), +}; + +static const union intel_ddi_buf_trans_entry _dg2_snps_trans[] = { + { .snps = { 26, 0, 0 } }, /* VS 0, pre-emph 0 */ + { .snps = { 33, 0, 6 } }, /* VS 0, pre-emph 1 */ + { .snps = { 38, 0, 12 } }, /* VS 0, pre-emph 2 */ + { .snps = { 43, 0, 19 } }, /* VS 0, pre-emph 3 */ + { .snps = { 39, 0, 0 } }, /* VS 1, pre-emph 0 */ + { .snps = { 44, 0, 8 } }, /* VS 1, pre-emph 1 */ + { .snps = { 47, 0, 15 } }, /* VS 1, pre-emph 2 */ + { .snps = { 52, 0, 0 } }, /* VS 2, pre-emph 0 */ + { .snps = { 51, 0, 10 } }, /* VS 2, pre-emph 1 */ + { .snps = { 62, 0, 0 } }, /* VS 3, pre-emph 0 */ +}; + +static const struct intel_ddi_buf_trans dg2_snps_trans = { + .entries = _dg2_snps_trans, + .num_entries = ARRAY_SIZE(_dg2_snps_trans), + .hdmi_default_entry = ARRAY_SIZE(_dg2_snps_trans) - 1, +}; + +static const union intel_ddi_buf_trans_entry _dg2_snps_trans_uhbr[] = { + { .snps = { 62, 0, 0 } }, /* preset 0 */ + { .snps = { 56, 0, 6 } }, /* preset 1 */ + { .snps = { 51, 0, 11 } }, /* preset 2 */ + { .snps = { 48, 0, 14 } }, /* preset 3 */ + { .snps = { 43, 0, 19 } }, /* preset 4 */ + { .snps = { 59, 3, 0 } }, /* preset 5 */ + { .snps = { 53, 3, 6 } }, /* preset 6 */ + { .snps = { 49, 3, 10 } }, /* preset 7 */ + { .snps = { 45, 3, 14 } }, /* preset 8 */ + { .snps = { 42, 3, 17 } }, /* preset 9 */ + { .snps = { 56, 6, 0 } }, /* preset 10 */ + { .snps = { 50, 6, 6 } }, /* preset 11 */ + { .snps = { 47, 6, 9 } }, /* preset 12 */ + { .snps = { 42, 6, 14 } }, /* preset 13 */ + { .snps = { 46, 8, 8 } }, /* preset 14 */ + { .snps = { 56, 3, 3 } }, /* preset 15 */ +}; + +static const struct intel_ddi_buf_trans dg2_snps_trans_uhbr = { + .entries = _dg2_snps_trans_uhbr, + .num_entries = ARRAY_SIZE(_dg2_snps_trans_uhbr), }; bool is_hobl_buf_trans(const struct intel_ddi_buf_trans *table) { - return table == &tgl_combo_phy_ddi_translations_edp_hbr2_hobl; + return table == &tgl_combo_phy_trans_edp_hbr2_hobl; } static const struct intel_ddi_buf_trans * -intel_get_buf_trans(const struct intel_ddi_buf_trans *ddi_translations, int *num_entries) +intel_get_buf_trans(const struct intel_ddi_buf_trans *trans, int *num_entries) { - *num_entries = ddi_translations->num_entries; - return ddi_translations; + *num_entries = trans->num_entries; + return trans; } static const struct intel_ddi_buf_trans * @@ -1001,11 +1045,11 @@ hsw_get_buf_trans(struct intel_encoder *encoder, int *n_entries) { if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_ANALOG)) - return intel_get_buf_trans(&hsw_ddi_translations_fdi, n_entries); + return intel_get_buf_trans(&hsw_trans_fdi, n_entries); else if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI)) - return intel_get_buf_trans(&hsw_ddi_translations_hdmi, n_entries); + return intel_get_buf_trans(&hsw_trans_hdmi, n_entries); else - return intel_get_buf_trans(&hsw_ddi_translations_dp, n_entries); + return intel_get_buf_trans(&hsw_trans_dp, n_entries); } static const struct intel_ddi_buf_trans * @@ -1016,14 +1060,14 @@ bdw_get_buf_trans(struct intel_encoder *encoder, struct drm_i915_private *i915 = to_i915(encoder->base.dev); if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_ANALOG)) - return intel_get_buf_trans(&bdw_ddi_translations_fdi, n_entries); + return intel_get_buf_trans(&bdw_trans_fdi, n_entries); else if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI)) - return intel_get_buf_trans(&bdw_ddi_translations_hdmi, n_entries); + return intel_get_buf_trans(&bdw_trans_hdmi, n_entries); else if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_EDP) && i915->vbt.edp.low_vswing) - return intel_get_buf_trans(&bdw_ddi_translations_edp, n_entries); + return intel_get_buf_trans(&bdw_trans_edp, n_entries); else - return intel_get_buf_trans(&bdw_ddi_translations_dp, n_entries); + return intel_get_buf_trans(&bdw_trans_dp, n_entries); } static int skl_buf_trans_num_entries(enum port port, int n_entries) @@ -1037,12 +1081,12 @@ static int skl_buf_trans_num_entries(enum port port, int n_entries) static const struct intel_ddi_buf_trans * _skl_get_buf_trans_dp(struct intel_encoder *encoder, - const struct intel_ddi_buf_trans *ddi_translations, + const struct intel_ddi_buf_trans *trans, int *n_entries) { - ddi_translations = intel_get_buf_trans(ddi_translations, n_entries); + trans = intel_get_buf_trans(trans, n_entries); *n_entries = skl_buf_trans_num_entries(encoder->port, *n_entries); - return ddi_translations; + return trans; } static const struct intel_ddi_buf_trans * @@ -1053,12 +1097,12 @@ skl_y_get_buf_trans(struct intel_encoder *encoder, struct drm_i915_private *i915 = to_i915(encoder->base.dev); if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI)) - return intel_get_buf_trans(&skl_y_ddi_translations_hdmi, n_entries); + return intel_get_buf_trans(&skl_y_trans_hdmi, n_entries); else if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_EDP) && i915->vbt.edp.low_vswing) - return _skl_get_buf_trans_dp(encoder, &skl_y_ddi_translations_edp, n_entries); + return _skl_get_buf_trans_dp(encoder, &skl_y_trans_edp, n_entries); else - return _skl_get_buf_trans_dp(encoder, &skl_y_ddi_translations_dp, n_entries); + return _skl_get_buf_trans_dp(encoder, &skl_y_trans_dp, n_entries); } static const struct intel_ddi_buf_trans * @@ -1069,12 +1113,12 @@ skl_u_get_buf_trans(struct intel_encoder *encoder, struct drm_i915_private *i915 = to_i915(encoder->base.dev); if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI)) - return intel_get_buf_trans(&skl_ddi_translations_hdmi, n_entries); + return intel_get_buf_trans(&skl_trans_hdmi, n_entries); else if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_EDP) && i915->vbt.edp.low_vswing) - return _skl_get_buf_trans_dp(encoder, &skl_u_ddi_translations_edp, n_entries); + return _skl_get_buf_trans_dp(encoder, &skl_u_trans_edp, n_entries); else - return _skl_get_buf_trans_dp(encoder, &skl_u_ddi_translations_dp, n_entries); + return _skl_get_buf_trans_dp(encoder, &skl_u_trans_dp, n_entries); } static const struct intel_ddi_buf_trans * @@ -1085,12 +1129,12 @@ skl_get_buf_trans(struct intel_encoder *encoder, struct drm_i915_private *i915 = to_i915(encoder->base.dev); if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI)) - return intel_get_buf_trans(&skl_ddi_translations_hdmi, n_entries); + return intel_get_buf_trans(&skl_trans_hdmi, n_entries); else if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_EDP) && i915->vbt.edp.low_vswing) - return _skl_get_buf_trans_dp(encoder, &skl_ddi_translations_edp, n_entries); + return _skl_get_buf_trans_dp(encoder, &skl_trans_edp, n_entries); else - return _skl_get_buf_trans_dp(encoder, &skl_ddi_translations_dp, n_entries); + return _skl_get_buf_trans_dp(encoder, &skl_trans_dp, n_entries); } static const struct intel_ddi_buf_trans * @@ -1101,12 +1145,12 @@ kbl_y_get_buf_trans(struct intel_encoder *encoder, struct drm_i915_private *i915 = to_i915(encoder->base.dev); if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI)) - return intel_get_buf_trans(&skl_y_ddi_translations_hdmi, n_entries); + return intel_get_buf_trans(&skl_y_trans_hdmi, n_entries); else if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_EDP) && i915->vbt.edp.low_vswing) - return _skl_get_buf_trans_dp(encoder, &skl_y_ddi_translations_edp, n_entries); + return _skl_get_buf_trans_dp(encoder, &skl_y_trans_edp, n_entries); else - return _skl_get_buf_trans_dp(encoder, &kbl_y_ddi_translations_dp, n_entries); + return _skl_get_buf_trans_dp(encoder, &kbl_y_trans_dp, n_entries); } static const struct intel_ddi_buf_trans * @@ -1117,12 +1161,12 @@ kbl_u_get_buf_trans(struct intel_encoder *encoder, struct drm_i915_private *i915 = to_i915(encoder->base.dev); if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI)) - return intel_get_buf_trans(&skl_ddi_translations_hdmi, n_entries); + return intel_get_buf_trans(&skl_trans_hdmi, n_entries); else if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_EDP) && i915->vbt.edp.low_vswing) - return _skl_get_buf_trans_dp(encoder, &skl_u_ddi_translations_edp, n_entries); + return _skl_get_buf_trans_dp(encoder, &skl_u_trans_edp, n_entries); else - return _skl_get_buf_trans_dp(encoder, &kbl_u_ddi_translations_dp, n_entries); + return _skl_get_buf_trans_dp(encoder, &kbl_u_trans_dp, n_entries); } static const struct intel_ddi_buf_trans * @@ -1133,12 +1177,12 @@ kbl_get_buf_trans(struct intel_encoder *encoder, struct drm_i915_private *i915 = to_i915(encoder->base.dev); if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI)) - return intel_get_buf_trans(&skl_ddi_translations_hdmi, n_entries); + return intel_get_buf_trans(&skl_trans_hdmi, n_entries); else if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_EDP) && i915->vbt.edp.low_vswing) - return _skl_get_buf_trans_dp(encoder, &skl_ddi_translations_edp, n_entries); + return _skl_get_buf_trans_dp(encoder, &skl_trans_edp, n_entries); else - return _skl_get_buf_trans_dp(encoder, &kbl_ddi_translations_dp, n_entries); + return _skl_get_buf_trans_dp(encoder, &kbl_trans_dp, n_entries); } static const struct intel_ddi_buf_trans * @@ -1149,12 +1193,12 @@ bxt_get_buf_trans(struct intel_encoder *encoder, struct drm_i915_private *i915 = to_i915(encoder->base.dev); if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI)) - return intel_get_buf_trans(&bxt_ddi_translations_hdmi, n_entries); + return intel_get_buf_trans(&bxt_trans_hdmi, n_entries); else if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_EDP) && i915->vbt.edp.low_vswing) - return intel_get_buf_trans(&bxt_ddi_translations_edp, n_entries); + return intel_get_buf_trans(&bxt_trans_edp, n_entries); else - return intel_get_buf_trans(&bxt_ddi_translations_dp, n_entries); + return intel_get_buf_trans(&bxt_trans_dp, n_entries); } static const struct intel_ddi_buf_trans * @@ -1162,7 +1206,7 @@ icl_get_combo_buf_trans_dp(struct intel_encoder *encoder, const struct intel_crtc_state *crtc_state, int *n_entries) { - return intel_get_buf_trans(&icl_combo_phy_ddi_translations_dp_hbr2_edp_hbr3, + return intel_get_buf_trans(&icl_combo_phy_trans_dp_hbr2_edp_hbr3, n_entries); } @@ -1174,10 +1218,10 @@ icl_get_combo_buf_trans_edp(struct intel_encoder *encoder, struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); if (crtc_state->port_clock > 540000) { - return intel_get_buf_trans(&icl_combo_phy_ddi_translations_dp_hbr2_edp_hbr3, + return intel_get_buf_trans(&icl_combo_phy_trans_dp_hbr2_edp_hbr3, n_entries); } else if (dev_priv->vbt.edp.low_vswing) { - return intel_get_buf_trans(&icl_combo_phy_ddi_translations_edp_hbr2, + return intel_get_buf_trans(&icl_combo_phy_trans_edp_hbr2, n_entries); } @@ -1190,7 +1234,7 @@ icl_get_combo_buf_trans(struct intel_encoder *encoder, int *n_entries) { if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI)) - return intel_get_buf_trans(&icl_combo_phy_ddi_translations_hdmi, n_entries); + return intel_get_buf_trans(&icl_combo_phy_trans_hdmi, n_entries); else if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_EDP)) return icl_get_combo_buf_trans_edp(encoder, crtc_state, n_entries); else @@ -1203,10 +1247,10 @@ icl_get_mg_buf_trans_dp(struct intel_encoder *encoder, int *n_entries) { if (crtc_state->port_clock > 270000) { - return intel_get_buf_trans(&icl_mg_phy_ddi_translations_hbr2_hbr3, + return intel_get_buf_trans(&icl_mg_phy_trans_hbr2_hbr3, n_entries); } else { - return intel_get_buf_trans(&icl_mg_phy_ddi_translations_rbr_hbr, + return intel_get_buf_trans(&icl_mg_phy_trans_rbr_hbr, n_entries); } } @@ -1217,7 +1261,7 @@ icl_get_mg_buf_trans(struct intel_encoder *encoder, int *n_entries) { if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI)) - return intel_get_buf_trans(&icl_mg_phy_ddi_translations_hdmi, n_entries); + return intel_get_buf_trans(&icl_mg_phy_trans_hdmi, n_entries); else return icl_get_mg_buf_trans_dp(encoder, crtc_state, n_entries); } @@ -1228,9 +1272,9 @@ ehl_get_combo_buf_trans_edp(struct intel_encoder *encoder, int *n_entries) { if (crtc_state->port_clock > 270000) - return intel_get_buf_trans(&ehl_combo_phy_ddi_translations_edp_hbr2, n_entries); + return intel_get_buf_trans(&ehl_combo_phy_trans_edp_hbr2, n_entries); else - return intel_get_buf_trans(&icl_combo_phy_ddi_translations_edp_hbr2, n_entries); + return intel_get_buf_trans(&icl_combo_phy_trans_edp_hbr2, n_entries); } static const struct intel_ddi_buf_trans * @@ -1241,12 +1285,12 @@ ehl_get_combo_buf_trans(struct intel_encoder *encoder, struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI)) - return intel_get_buf_trans(&icl_combo_phy_ddi_translations_hdmi, n_entries); + return intel_get_buf_trans(&icl_combo_phy_trans_hdmi, n_entries); else if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_EDP) && dev_priv->vbt.edp.low_vswing) return ehl_get_combo_buf_trans_edp(encoder, crtc_state, n_entries); else - return intel_get_buf_trans(&ehl_combo_phy_ddi_translations_dp, n_entries); + return intel_get_buf_trans(&ehl_combo_phy_trans_dp, n_entries); } static const struct intel_ddi_buf_trans * @@ -1255,9 +1299,9 @@ jsl_get_combo_buf_trans_edp(struct intel_encoder *encoder, int *n_entries) { if (crtc_state->port_clock > 270000) - return intel_get_buf_trans(&jsl_combo_phy_ddi_translations_edp_hbr2, n_entries); + return intel_get_buf_trans(&jsl_combo_phy_trans_edp_hbr2, n_entries); else - return intel_get_buf_trans(&jsl_combo_phy_ddi_translations_edp_hbr, n_entries); + return intel_get_buf_trans(&jsl_combo_phy_trans_edp_hbr, n_entries); } static const struct intel_ddi_buf_trans * @@ -1268,12 +1312,12 @@ jsl_get_combo_buf_trans(struct intel_encoder *encoder, struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI)) - return intel_get_buf_trans(&icl_combo_phy_ddi_translations_hdmi, n_entries); + return intel_get_buf_trans(&icl_combo_phy_trans_hdmi, n_entries); else if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_EDP) && dev_priv->vbt.edp.low_vswing) return jsl_get_combo_buf_trans_edp(encoder, crtc_state, n_entries); else - return intel_get_buf_trans(&icl_combo_phy_ddi_translations_dp_hbr2_edp_hbr3, n_entries); + return intel_get_buf_trans(&icl_combo_phy_trans_dp_hbr2_edp_hbr3, n_entries); } static const struct intel_ddi_buf_trans * @@ -1285,14 +1329,14 @@ tgl_get_combo_buf_trans_dp(struct intel_encoder *encoder, if (crtc_state->port_clock > 270000) { if (IS_TGL_U(dev_priv) || IS_TGL_Y(dev_priv)) { - return intel_get_buf_trans(&tgl_uy_combo_phy_ddi_translations_dp_hbr2, + return intel_get_buf_trans(&tgl_uy_combo_phy_trans_dp_hbr2, n_entries); } else { - return intel_get_buf_trans(&tgl_combo_phy_ddi_translations_dp_hbr2, + return intel_get_buf_trans(&tgl_combo_phy_trans_dp_hbr2, n_entries); } } else { - return intel_get_buf_trans(&tgl_combo_phy_ddi_translations_dp_hbr, + return intel_get_buf_trans(&tgl_combo_phy_trans_dp_hbr, n_entries); } } @@ -1306,13 +1350,13 @@ tgl_get_combo_buf_trans_edp(struct intel_encoder *encoder, struct intel_dp *intel_dp = enc_to_intel_dp(encoder); if (crtc_state->port_clock > 540000) { - return intel_get_buf_trans(&icl_combo_phy_ddi_translations_dp_hbr2_edp_hbr3, + return intel_get_buf_trans(&icl_combo_phy_trans_dp_hbr2_edp_hbr3, n_entries); } else if (dev_priv->vbt.edp.hobl && !intel_dp->hobl_failed) { - return intel_get_buf_trans(&tgl_combo_phy_ddi_translations_edp_hbr2_hobl, + return intel_get_buf_trans(&tgl_combo_phy_trans_edp_hbr2_hobl, n_entries); } else if (dev_priv->vbt.edp.low_vswing) { - return intel_get_buf_trans(&icl_combo_phy_ddi_translations_edp_hbr2, + return intel_get_buf_trans(&icl_combo_phy_trans_edp_hbr2, n_entries); } @@ -1325,7 +1369,7 @@ tgl_get_combo_buf_trans(struct intel_encoder *encoder, int *n_entries) { if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI)) - return intel_get_buf_trans(&icl_combo_phy_ddi_translations_hdmi, n_entries); + return intel_get_buf_trans(&icl_combo_phy_trans_hdmi, n_entries); else if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_EDP)) return tgl_get_combo_buf_trans_edp(encoder, crtc_state, n_entries); else @@ -1338,10 +1382,10 @@ dg1_get_combo_buf_trans_dp(struct intel_encoder *encoder, int *n_entries) { if (crtc_state->port_clock > 270000) - return intel_get_buf_trans(&dg1_combo_phy_ddi_translations_dp_hbr2_hbr3, + return intel_get_buf_trans(&dg1_combo_phy_trans_dp_hbr2_hbr3, n_entries); else - return intel_get_buf_trans(&dg1_combo_phy_ddi_translations_dp_rbr_hbr, + return intel_get_buf_trans(&dg1_combo_phy_trans_dp_rbr_hbr, n_entries); } @@ -1354,13 +1398,13 @@ dg1_get_combo_buf_trans_edp(struct intel_encoder *encoder, struct intel_dp *intel_dp = enc_to_intel_dp(encoder); if (crtc_state->port_clock > 540000) - return intel_get_buf_trans(&icl_combo_phy_ddi_translations_dp_hbr2_edp_hbr3, + return intel_get_buf_trans(&icl_combo_phy_trans_dp_hbr2_edp_hbr3, n_entries); else if (dev_priv->vbt.edp.hobl && !intel_dp->hobl_failed) - return intel_get_buf_trans(&tgl_combo_phy_ddi_translations_edp_hbr2_hobl, + return intel_get_buf_trans(&tgl_combo_phy_trans_edp_hbr2_hobl, n_entries); else if (dev_priv->vbt.edp.low_vswing) - return intel_get_buf_trans(&icl_combo_phy_ddi_translations_edp_hbr2, + return intel_get_buf_trans(&icl_combo_phy_trans_edp_hbr2, n_entries); else return dg1_get_combo_buf_trans_dp(encoder, crtc_state, n_entries); @@ -1372,7 +1416,7 @@ dg1_get_combo_buf_trans(struct intel_encoder *encoder, int *n_entries) { if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI)) - return intel_get_buf_trans(&icl_combo_phy_ddi_translations_hdmi, n_entries); + return intel_get_buf_trans(&icl_combo_phy_trans_hdmi, n_entries); else if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_EDP)) return dg1_get_combo_buf_trans_edp(encoder, crtc_state, n_entries); else @@ -1385,9 +1429,9 @@ rkl_get_combo_buf_trans_dp(struct intel_encoder *encoder, int *n_entries) { if (crtc_state->port_clock > 270000) - return intel_get_buf_trans(&rkl_combo_phy_ddi_translations_dp_hbr2_hbr3, n_entries); + return intel_get_buf_trans(&rkl_combo_phy_trans_dp_hbr2_hbr3, n_entries); else - return intel_get_buf_trans(&rkl_combo_phy_ddi_translations_dp_hbr, n_entries); + return intel_get_buf_trans(&rkl_combo_phy_trans_dp_hbr, n_entries); } static const struct intel_ddi_buf_trans * @@ -1399,13 +1443,13 @@ rkl_get_combo_buf_trans_edp(struct intel_encoder *encoder, struct intel_dp *intel_dp = enc_to_intel_dp(encoder); if (crtc_state->port_clock > 540000) { - return intel_get_buf_trans(&icl_combo_phy_ddi_translations_dp_hbr2_edp_hbr3, + return intel_get_buf_trans(&icl_combo_phy_trans_dp_hbr2_edp_hbr3, n_entries); } else if (dev_priv->vbt.edp.hobl && !intel_dp->hobl_failed) { - return intel_get_buf_trans(&tgl_combo_phy_ddi_translations_edp_hbr2_hobl, + return intel_get_buf_trans(&tgl_combo_phy_trans_edp_hbr2_hobl, n_entries); } else if (dev_priv->vbt.edp.low_vswing) { - return intel_get_buf_trans(&icl_combo_phy_ddi_translations_edp_hbr2, + return intel_get_buf_trans(&icl_combo_phy_trans_edp_hbr2, n_entries); } @@ -1418,7 +1462,7 @@ rkl_get_combo_buf_trans(struct intel_encoder *encoder, int *n_entries) { if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI)) - return intel_get_buf_trans(&icl_combo_phy_ddi_translations_hdmi, n_entries); + return intel_get_buf_trans(&icl_combo_phy_trans_hdmi, n_entries); else if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_EDP)) return rkl_get_combo_buf_trans_edp(encoder, crtc_state, n_entries); else @@ -1431,9 +1475,9 @@ adls_get_combo_buf_trans_dp(struct intel_encoder *encoder, int *n_entries) { if (crtc_state->port_clock > 270000) - return intel_get_buf_trans(&adls_combo_phy_ddi_translations_dp_hbr2_hbr3, n_entries); + return intel_get_buf_trans(&adls_combo_phy_trans_dp_hbr2_hbr3, n_entries); else - return intel_get_buf_trans(&tgl_combo_phy_ddi_translations_dp_hbr, n_entries); + return intel_get_buf_trans(&tgl_combo_phy_trans_dp_hbr, n_entries); } static const struct intel_ddi_buf_trans * @@ -1445,11 +1489,11 @@ adls_get_combo_buf_trans_edp(struct intel_encoder *encoder, struct intel_dp *intel_dp = enc_to_intel_dp(encoder); if (crtc_state->port_clock > 540000) - return intel_get_buf_trans(&adls_combo_phy_ddi_translations_edp_hbr3, n_entries); + return intel_get_buf_trans(&adls_combo_phy_trans_edp_hbr3, n_entries); else if (i915->vbt.edp.hobl && !intel_dp->hobl_failed) - return intel_get_buf_trans(&tgl_combo_phy_ddi_translations_edp_hbr2_hobl, n_entries); + return intel_get_buf_trans(&tgl_combo_phy_trans_edp_hbr2_hobl, n_entries); else if (i915->vbt.edp.low_vswing) - return intel_get_buf_trans(&adls_combo_phy_ddi_translations_edp_hbr2, n_entries); + return intel_get_buf_trans(&adls_combo_phy_trans_edp_hbr2, n_entries); else return adls_get_combo_buf_trans_dp(encoder, crtc_state, n_entries); } @@ -1460,7 +1504,7 @@ adls_get_combo_buf_trans(struct intel_encoder *encoder, int *n_entries) { if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI)) - return intel_get_buf_trans(&icl_combo_phy_ddi_translations_hdmi, n_entries); + return intel_get_buf_trans(&icl_combo_phy_trans_hdmi, n_entries); else if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_EDP)) return adls_get_combo_buf_trans_edp(encoder, crtc_state, n_entries); else @@ -1473,9 +1517,9 @@ adlp_get_combo_buf_trans_dp(struct intel_encoder *encoder, int *n_entries) { if (crtc_state->port_clock > 270000) - return intel_get_buf_trans(&adlp_combo_phy_ddi_translations_dp_hbr2_hbr3, n_entries); + return intel_get_buf_trans(&adlp_combo_phy_trans_dp_hbr2_hbr3, n_entries); else - return intel_get_buf_trans(&adlp_combo_phy_ddi_translations_dp_hbr, n_entries); + return intel_get_buf_trans(&adlp_combo_phy_trans_dp_hbr, n_entries); } static const struct intel_ddi_buf_trans * @@ -1487,13 +1531,13 @@ adlp_get_combo_buf_trans_edp(struct intel_encoder *encoder, struct intel_dp *intel_dp = enc_to_intel_dp(encoder); if (crtc_state->port_clock > 540000) { - return intel_get_buf_trans(&adlp_combo_phy_ddi_translations_edp_hbr3, + return intel_get_buf_trans(&adlp_combo_phy_trans_edp_hbr3, n_entries); } else if (dev_priv->vbt.edp.hobl && !intel_dp->hobl_failed) { - return intel_get_buf_trans(&tgl_combo_phy_ddi_translations_edp_hbr2_hobl, + return intel_get_buf_trans(&tgl_combo_phy_trans_edp_hbr2_hobl, n_entries); } else if (dev_priv->vbt.edp.low_vswing) { - return intel_get_buf_trans(&adlp_combo_phy_ddi_translations_edp_up_to_hbr2, + return intel_get_buf_trans(&adlp_combo_phy_trans_edp_up_to_hbr2, n_entries); } @@ -1506,7 +1550,7 @@ adlp_get_combo_buf_trans(struct intel_encoder *encoder, int *n_entries) { if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI)) - return intel_get_buf_trans(&adlp_combo_phy_ddi_translations_hdmi, n_entries); + return intel_get_buf_trans(&adlp_combo_phy_trans_hdmi, n_entries); else if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_EDP)) return adlp_get_combo_buf_trans_edp(encoder, crtc_state, n_entries); else @@ -1519,10 +1563,10 @@ tgl_get_dkl_buf_trans_dp(struct intel_encoder *encoder, int *n_entries) { if (crtc_state->port_clock > 270000) { - return intel_get_buf_trans(&tgl_dkl_phy_ddi_translations_dp_hbr2, + return intel_get_buf_trans(&tgl_dkl_phy_trans_dp_hbr2, n_entries); } else { - return intel_get_buf_trans(&tgl_dkl_phy_ddi_translations_dp_hbr, + return intel_get_buf_trans(&tgl_dkl_phy_trans_dp_hbr, n_entries); } } @@ -1533,7 +1577,7 @@ tgl_get_dkl_buf_trans(struct intel_encoder *encoder, int *n_entries) { if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI)) - return intel_get_buf_trans(&tgl_dkl_phy_ddi_translations_hdmi, n_entries); + return intel_get_buf_trans(&tgl_dkl_phy_trans_hdmi, n_entries); else return tgl_get_dkl_buf_trans_dp(encoder, crtc_state, n_entries); } @@ -1544,10 +1588,10 @@ adlp_get_dkl_buf_trans_dp(struct intel_encoder *encoder, int *n_entries) { if (crtc_state->port_clock > 270000) { - return intel_get_buf_trans(&adlp_dkl_phy_ddi_translations_dp_hbr2_hbr3, + return intel_get_buf_trans(&adlp_dkl_phy_trans_dp_hbr2_hbr3, n_entries); } else { - return intel_get_buf_trans(&adlp_dkl_phy_ddi_translations_dp_hbr, + return intel_get_buf_trans(&adlp_dkl_phy_trans_dp_hbr, n_entries); } } @@ -1558,29 +1602,21 @@ adlp_get_dkl_buf_trans(struct intel_encoder *encoder, int *n_entries) { if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI)) - return intel_get_buf_trans(&tgl_dkl_phy_ddi_translations_hdmi, n_entries); + return intel_get_buf_trans(&tgl_dkl_phy_trans_hdmi, n_entries); else return adlp_get_dkl_buf_trans_dp(encoder, crtc_state, n_entries); } -int intel_ddi_hdmi_num_entries(struct intel_encoder *encoder, - const struct intel_crtc_state *crtc_state, - int *default_entry) +static const struct intel_ddi_buf_trans * +dg2_get_snps_buf_trans(struct intel_encoder *encoder, + const struct intel_crtc_state *crtc_state, + int *n_entries) { - struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); - const struct intel_ddi_buf_trans *ddi_translations; - int n_entries; - - ddi_translations = encoder->get_buf_trans(encoder, crtc_state, &n_entries); - - if (drm_WARN_ON(&dev_priv->drm, !ddi_translations)) { - *default_entry = 0; - return 0; - } - - *default_entry = ddi_translations->hdmi_default_entry; - - return n_entries; + if (intel_crtc_has_dp_encoder(crtc_state) && + intel_dp_is_uhbr(crtc_state)) + return intel_get_buf_trans(&dg2_snps_trans_uhbr, n_entries); + else + return intel_get_buf_trans(&dg2_snps_trans, n_entries); } void intel_ddi_buf_trans_init(struct intel_encoder *encoder) @@ -1588,7 +1624,9 @@ void intel_ddi_buf_trans_init(struct intel_encoder *encoder) struct drm_i915_private *i915 = to_i915(encoder->base.dev); enum phy phy = intel_port_to_phy(i915, encoder->port); - if (IS_ALDERLAKE_P(i915)) { + if (IS_DG2(i915)) { + encoder->get_buf_trans = dg2_get_snps_buf_trans; + } else if (IS_ALDERLAKE_P(i915)) { if (intel_phy_is_combo(i915, phy)) encoder->get_buf_trans = adlp_get_combo_buf_trans; else diff --git a/drivers/gpu/drm/i915/display/intel_ddi_buf_trans.h b/drivers/gpu/drm/i915/display/intel_ddi_buf_trans.h index 2acd720f9d4f..2133984a572b 100644 --- a/drivers/gpu/drm/i915/display/intel_ddi_buf_trans.h +++ b/drivers/gpu/drm/i915/display/intel_ddi_buf_trans.h @@ -34,15 +34,21 @@ struct icl_ddi_buf_trans { }; struct icl_mg_phy_ddi_buf_trans { - u32 cri_txdeemph_override_11_6; - u32 cri_txdeemph_override_5_0; - u32 cri_txdeemph_override_17_12; + u8 cri_txdeemph_override_11_6; + u8 cri_txdeemph_override_5_0; + u8 cri_txdeemph_override_17_12; }; struct tgl_dkl_phy_ddi_buf_trans { - u32 dkl_vswing_control; - u32 dkl_preshoot_control; - u32 dkl_de_emphasis_control; + u8 vswing; + u8 preshoot; + u8 de_emphasis; +}; + +struct dg2_snps_phy_buf_trans { + u8 vswing; + u8 pre_cursor; + u8 post_cursor; }; union intel_ddi_buf_trans_entry { @@ -51,6 +57,7 @@ union intel_ddi_buf_trans_entry { struct icl_ddi_buf_trans icl; struct icl_mg_phy_ddi_buf_trans mg; struct tgl_dkl_phy_ddi_buf_trans dkl; + struct dg2_snps_phy_buf_trans snps; }; struct intel_ddi_buf_trans { @@ -61,10 +68,6 @@ struct intel_ddi_buf_trans { bool is_hobl_buf_trans(const struct intel_ddi_buf_trans *table); -int intel_ddi_hdmi_num_entries(struct intel_encoder *encoder, - const struct intel_crtc_state *crtc_state, - int *default_entry); - void intel_ddi_buf_trans_init(struct intel_encoder *encoder); #endif diff --git a/drivers/gpu/drm/i915/display/intel_display.c b/drivers/gpu/drm/i915/display/intel_display.c index 17f44ffea586..ff598b6cd953 100644 --- a/drivers/gpu/drm/i915/display/intel_display.c +++ b/drivers/gpu/drm/i915/display/intel_display.c @@ -52,6 +52,7 @@ #include "display/intel_dp_mst.h" #include "display/intel_dpll.h" #include "display/intel_dpll_mgr.h" +#include "display/intel_drrs.h" #include "display/intel_dsi.h" #include "display/intel_dvo.h" #include "display/intel_fb.h" @@ -67,9 +68,10 @@ #include "gem/i915_gem_lmem.h" #include "gem/i915_gem_object.h" -#include "gt/intel_rps.h" #include "gt/gen8_ppgtt.h" +#include "pxp/intel_pxp.h" + #include "g4x_dp.h" #include "g4x_hdmi.h" #include "i915_drv.h" @@ -84,35 +86,37 @@ #include "intel_display_types.h" #include "intel_dmc.h" #include "intel_dp_link_training.h" +#include "intel_dpt.h" #include "intel_fbc.h" -#include "intel_fdi.h" #include "intel_fbdev.h" +#include "intel_fdi.h" #include "intel_fifo_underrun.h" #include "intel_frontbuffer.h" #include "intel_hdcp.h" #include "intel_hotplug.h" #include "intel_overlay.h" +#include "intel_panel.h" +#include "intel_pcode.h" #include "intel_pipe_crc.h" +#include "intel_plane_initial.h" #include "intel_pm.h" #include "intel_pps.h" #include "intel_psr.h" #include "intel_quirks.h" -#include "intel_sideband.h" +#include "intel_sbi.h" #include "intel_sprite.h" #include "intel_tc.h" #include "intel_vga.h" #include "i9xx_plane.h" #include "skl_scaler.h" #include "skl_universal_plane.h" +#include "vlv_sideband.h" static void i9xx_crtc_clock_get(struct intel_crtc *crtc, struct intel_crtc_state *pipe_config); static void ilk_pch_clock_get(struct intel_crtc *crtc, struct intel_crtc_state *pipe_config); -static int intel_framebuffer_init(struct intel_framebuffer *ifb, - struct drm_i915_gem_object *obj, - struct drm_mode_fb_cmd2 *mode_cmd); static void intel_set_transcoder_timings(const struct intel_crtc_state *crtc_state); static void intel_set_pipe_src_size(const struct intel_crtc_state *crtc_state); static void intel_cpu_transcoder_set_m_n(const struct intel_crtc_state *crtc_state, @@ -120,186 +124,105 @@ static void intel_cpu_transcoder_set_m_n(const struct intel_crtc_state *crtc_sta const struct intel_link_m_n *m2_n2); static void i9xx_set_pipeconf(const struct intel_crtc_state *crtc_state); static void ilk_set_pipeconf(const struct intel_crtc_state *crtc_state); -static void hsw_set_pipeconf(const struct intel_crtc_state *crtc_state); +static void hsw_set_transconf(const struct intel_crtc_state *crtc_state); static void bdw_set_pipemisc(const struct intel_crtc_state *crtc_state); static void ilk_pfit_enable(const struct intel_crtc_state *crtc_state); static void intel_modeset_setup_hw_state(struct drm_device *dev, struct drm_modeset_acquire_ctx *ctx); -struct i915_dpt { - struct i915_address_space vm; - - struct drm_i915_gem_object *obj; - struct i915_vma *vma; - void __iomem *iomem; -}; - -#define i915_is_dpt(vm) ((vm)->is_dpt) - -static inline struct i915_dpt * -i915_vm_to_dpt(struct i915_address_space *vm) -{ - BUILD_BUG_ON(offsetof(struct i915_dpt, vm)); - GEM_BUG_ON(!i915_is_dpt(vm)); - return container_of(vm, struct i915_dpt, vm); -} - -#define dpt_total_entries(dpt) ((dpt)->vm.total >> PAGE_SHIFT) - -static void gen8_set_pte(void __iomem *addr, gen8_pte_t pte) -{ - writeq(pte, addr); -} - -static void dpt_insert_page(struct i915_address_space *vm, - dma_addr_t addr, - u64 offset, - enum i915_cache_level level, - u32 flags) +/** + * intel_update_watermarks - update FIFO watermark values based on current modes + * @dev_priv: i915 device + * + * Calculate watermark values for the various WM regs based on current mode + * and plane configuration. + * + * There are several cases to deal with here: + * - normal (i.e. non-self-refresh) + * - self-refresh (SR) mode + * - lines are large relative to FIFO size (buffer can hold up to 2) + * - lines are small relative to FIFO size (buffer can hold more than 2 + * lines), so need to account for TLB latency + * + * The normal calculation is: + * watermark = dotclock * bytes per pixel * latency + * where latency is platform & configuration dependent (we assume pessimal + * values here). + * + * The SR calculation is: + * watermark = (trunc(latency/line time)+1) * surface width * + * bytes per pixel + * where + * line time = htotal / dotclock + * surface width = hdisplay for normal plane and 64 for cursor + * and latency is assumed to be high, as above. + * + * The final value programmed to the register should always be rounded up, + * and include an extra 2 entries to account for clock crossings. + * + * We don't use the sprite, so we can ignore that. And on Crestline we have + * to set the non-SR watermarks to 8. + */ +static void intel_update_watermarks(struct drm_i915_private *dev_priv) { - struct i915_dpt *dpt = i915_vm_to_dpt(vm); - gen8_pte_t __iomem *base = dpt->iomem; - - gen8_set_pte(base + offset / I915_GTT_PAGE_SIZE, - vm->pte_encode(addr, level, flags)); + if (dev_priv->wm_disp->update_wm) + dev_priv->wm_disp->update_wm(dev_priv); } -static void dpt_insert_entries(struct i915_address_space *vm, - struct i915_vma *vma, - enum i915_cache_level level, - u32 flags) -{ - struct i915_dpt *dpt = i915_vm_to_dpt(vm); - gen8_pte_t __iomem *base = dpt->iomem; - const gen8_pte_t pte_encode = vm->pte_encode(0, level, flags); - struct sgt_iter sgt_iter; - dma_addr_t addr; - int i; - - /* - * Note that we ignore PTE_READ_ONLY here. The caller must be careful - * not to allow the user to override access to a read only page. - */ - - i = vma->node.start / I915_GTT_PAGE_SIZE; - for_each_sgt_daddr(addr, sgt_iter, vma->pages) - gen8_set_pte(&base[i++], pte_encode | addr); -} - -static void dpt_clear_range(struct i915_address_space *vm, - u64 start, u64 length) +static int intel_compute_pipe_wm(struct intel_atomic_state *state, + struct intel_crtc *crtc) { + struct drm_i915_private *dev_priv = to_i915(state->base.dev); + if (dev_priv->wm_disp->compute_pipe_wm) + return dev_priv->wm_disp->compute_pipe_wm(state, crtc); + return 0; } -static void dpt_bind_vma(struct i915_address_space *vm, - struct i915_vm_pt_stash *stash, - struct i915_vma *vma, - enum i915_cache_level cache_level, - u32 flags) +static int intel_compute_intermediate_wm(struct intel_atomic_state *state, + struct intel_crtc *crtc) { - struct drm_i915_gem_object *obj = vma->obj; - u32 pte_flags; - - /* Applicable to VLV (gen8+ do not support RO in the GGTT) */ - pte_flags = 0; - if (vma->vm->has_read_only && i915_gem_object_is_readonly(obj)) - pte_flags |= PTE_READ_ONLY; - if (i915_gem_object_is_lmem(obj)) - pte_flags |= PTE_LM; - - vma->vm->insert_entries(vma->vm, vma, cache_level, pte_flags); - - vma->page_sizes.gtt = I915_GTT_PAGE_SIZE; - - /* - * Without aliasing PPGTT there's no difference between - * GLOBAL/LOCAL_BIND, it's all the same ptes. Hence unconditionally - * upgrade to both bound if we bind either to avoid double-binding. - */ - atomic_or(I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND, &vma->flags); + struct drm_i915_private *dev_priv = to_i915(state->base.dev); + if (!dev_priv->wm_disp->compute_intermediate_wm) + return 0; + if (drm_WARN_ON(&dev_priv->drm, + !dev_priv->wm_disp->compute_pipe_wm)) + return 0; + return dev_priv->wm_disp->compute_intermediate_wm(state, crtc); } -static void dpt_unbind_vma(struct i915_address_space *vm, struct i915_vma *vma) +static bool intel_initial_watermarks(struct intel_atomic_state *state, + struct intel_crtc *crtc) { - vm->clear_range(vm, vma->node.start, vma->size); + struct drm_i915_private *dev_priv = to_i915(state->base.dev); + if (dev_priv->wm_disp->initial_watermarks) { + dev_priv->wm_disp->initial_watermarks(state, crtc); + return true; + } + return false; } -static void dpt_cleanup(struct i915_address_space *vm) +static void intel_atomic_update_watermarks(struct intel_atomic_state *state, + struct intel_crtc *crtc) { - struct i915_dpt *dpt = i915_vm_to_dpt(vm); - - i915_gem_object_put(dpt->obj); + struct drm_i915_private *dev_priv = to_i915(state->base.dev); + if (dev_priv->wm_disp->atomic_update_watermarks) + dev_priv->wm_disp->atomic_update_watermarks(state, crtc); } -static struct i915_address_space * -intel_dpt_create(struct intel_framebuffer *fb) +static void intel_optimize_watermarks(struct intel_atomic_state *state, + struct intel_crtc *crtc) { - struct drm_gem_object *obj = &intel_fb_obj(&fb->base)->base; - struct drm_i915_private *i915 = to_i915(obj->dev); - struct drm_i915_gem_object *dpt_obj; - struct i915_address_space *vm; - struct i915_dpt *dpt; - size_t size; - int ret; - - if (intel_fb_needs_pot_stride_remap(fb)) - size = intel_remapped_info_size(&fb->remapped_view.gtt.remapped); - else - size = DIV_ROUND_UP_ULL(obj->size, I915_GTT_PAGE_SIZE); - - size = round_up(size * sizeof(gen8_pte_t), I915_GTT_PAGE_SIZE); - - if (HAS_LMEM(i915)) - dpt_obj = i915_gem_object_create_lmem(i915, size, 0); - else - dpt_obj = i915_gem_object_create_stolen(i915, size); - if (IS_ERR(dpt_obj)) - return ERR_CAST(dpt_obj); - - ret = i915_gem_object_set_cache_level(dpt_obj, I915_CACHE_NONE); - if (ret) { - i915_gem_object_put(dpt_obj); - return ERR_PTR(ret); - } - - dpt = kzalloc(sizeof(*dpt), GFP_KERNEL); - if (!dpt) { - i915_gem_object_put(dpt_obj); - return ERR_PTR(-ENOMEM); - } - - vm = &dpt->vm; - - vm->gt = &i915->gt; - vm->i915 = i915; - vm->dma = i915->drm.dev; - vm->total = (size / sizeof(gen8_pte_t)) * I915_GTT_PAGE_SIZE; - vm->is_dpt = true; - - i915_address_space_init(vm, VM_CLASS_DPT); - - vm->insert_page = dpt_insert_page; - vm->clear_range = dpt_clear_range; - vm->insert_entries = dpt_insert_entries; - vm->cleanup = dpt_cleanup; - - vm->vma_ops.bind_vma = dpt_bind_vma; - vm->vma_ops.unbind_vma = dpt_unbind_vma; - vm->vma_ops.set_pages = ggtt_set_pages; - vm->vma_ops.clear_pages = clear_pages; - - vm->pte_encode = gen8_ggtt_pte_encode; - - dpt->obj = dpt_obj; - - return &dpt->vm; + struct drm_i915_private *dev_priv = to_i915(state->base.dev); + if (dev_priv->wm_disp->optimize_watermarks) + dev_priv->wm_disp->optimize_watermarks(state, crtc); } -static void intel_dpt_destroy(struct i915_address_space *vm) +static int intel_compute_global_watermarks(struct intel_atomic_state *state) { - struct i915_dpt *dpt = i915_vm_to_dpt(vm); - - i915_vm_close(&dpt->vm); + struct drm_i915_private *dev_priv = to_i915(state->base.dev); + if (dev_priv->wm_disp->compute_global_watermarks) + return dev_priv->wm_disp->compute_global_watermarks(state); + return 0; } /* returns HPLL frequency in kHz */ @@ -359,6 +282,12 @@ static void intel_update_czclk(struct drm_i915_private *dev_priv) dev_priv->czclk_freq); } +static bool is_hdr_mode(const struct intel_crtc_state *crtc_state) +{ + return (crtc_state->active_planes & + ~(icl_hdr_plane_mask() | BIT(PLANE_CURSOR))) == 0; +} + /* WA Display #0827: Gen9:all */ static void skl_wa_827(struct drm_i915_private *dev_priv, enum pipe pipe, bool enable) @@ -384,6 +313,15 @@ icl_wa_scalerclkgating(struct drm_i915_private *dev_priv, enum pipe pipe, intel_de_read(dev_priv, CLKGATE_DIS_PSL(pipe)) & ~DPFR_GATING_DIS); } +/* Wa_1604331009:icl,jsl,ehl */ +static void +icl_wa_cursorclkgating(struct drm_i915_private *dev_priv, enum pipe pipe, + bool enable) +{ + intel_de_rmw(dev_priv, CLKGATE_DIS_PSL(pipe), CURSOR_GATING_DIS, + enable ? CURSOR_GATING_DIS : 0); +} + static bool is_trans_port_sync_slave(const struct intel_crtc_state *crtc_state) { @@ -464,168 +402,8 @@ intel_wait_for_pipe_off(const struct intel_crtc_state *old_crtc_state) } } -/* Only for pre-ILK configs */ -void assert_pll(struct drm_i915_private *dev_priv, - enum pipe pipe, bool state) -{ - u32 val; - bool cur_state; - - val = intel_de_read(dev_priv, DPLL(pipe)); - cur_state = !!(val & DPLL_VCO_ENABLE); - I915_STATE_WARN(cur_state != state, - "PLL state assertion failure (expected %s, current %s)\n", - onoff(state), onoff(cur_state)); -} - -/* XXX: the dsi pll is shared between MIPI DSI ports */ -void assert_dsi_pll(struct drm_i915_private *dev_priv, bool state) -{ - u32 val; - bool cur_state; - - vlv_cck_get(dev_priv); - val = vlv_cck_read(dev_priv, CCK_REG_DSI_PLL_CONTROL); - vlv_cck_put(dev_priv); - - cur_state = val & DSI_PLL_VCO_EN; - I915_STATE_WARN(cur_state != state, - "DSI PLL state assertion failure (expected %s, current %s)\n", - onoff(state), onoff(cur_state)); -} - -static void assert_fdi_tx(struct drm_i915_private *dev_priv, - enum pipe pipe, bool state) -{ - bool cur_state; - - if (HAS_DDI(dev_priv)) { - /* - * DDI does not have a specific FDI_TX register. - * - * FDI is never fed from EDP transcoder - * so pipe->transcoder cast is fine here. - */ - enum transcoder cpu_transcoder = (enum transcoder)pipe; - u32 val = intel_de_read(dev_priv, - TRANS_DDI_FUNC_CTL(cpu_transcoder)); - cur_state = !!(val & TRANS_DDI_FUNC_ENABLE); - } else { - u32 val = intel_de_read(dev_priv, FDI_TX_CTL(pipe)); - cur_state = !!(val & FDI_TX_ENABLE); - } - I915_STATE_WARN(cur_state != state, - "FDI TX state assertion failure (expected %s, current %s)\n", - onoff(state), onoff(cur_state)); -} -#define assert_fdi_tx_enabled(d, p) assert_fdi_tx(d, p, true) -#define assert_fdi_tx_disabled(d, p) assert_fdi_tx(d, p, false) - -static void assert_fdi_rx(struct drm_i915_private *dev_priv, - enum pipe pipe, bool state) -{ - u32 val; - bool cur_state; - - val = intel_de_read(dev_priv, FDI_RX_CTL(pipe)); - cur_state = !!(val & FDI_RX_ENABLE); - I915_STATE_WARN(cur_state != state, - "FDI RX state assertion failure (expected %s, current %s)\n", - onoff(state), onoff(cur_state)); -} -#define assert_fdi_rx_enabled(d, p) assert_fdi_rx(d, p, true) -#define assert_fdi_rx_disabled(d, p) assert_fdi_rx(d, p, false) - -static void assert_fdi_tx_pll_enabled(struct drm_i915_private *dev_priv, - enum pipe pipe) -{ - u32 val; - - /* ILK FDI PLL is always enabled */ - if (IS_IRONLAKE(dev_priv)) - return; - - /* On Haswell, DDI ports are responsible for the FDI PLL setup */ - if (HAS_DDI(dev_priv)) - return; - - val = intel_de_read(dev_priv, FDI_TX_CTL(pipe)); - I915_STATE_WARN(!(val & FDI_TX_PLL_ENABLE), "FDI TX PLL assertion failure, should be active but is disabled\n"); -} - -void assert_fdi_rx_pll(struct drm_i915_private *dev_priv, - enum pipe pipe, bool state) -{ - u32 val; - bool cur_state; - - val = intel_de_read(dev_priv, FDI_RX_CTL(pipe)); - cur_state = !!(val & FDI_RX_PLL_ENABLE); - I915_STATE_WARN(cur_state != state, - "FDI RX PLL assertion failure (expected %s, current %s)\n", - onoff(state), onoff(cur_state)); -} - -void assert_panel_unlocked(struct drm_i915_private *dev_priv, enum pipe pipe) -{ - i915_reg_t pp_reg; - u32 val; - enum pipe panel_pipe = INVALID_PIPE; - bool locked = true; - - if (drm_WARN_ON(&dev_priv->drm, HAS_DDI(dev_priv))) - return; - - if (HAS_PCH_SPLIT(dev_priv)) { - u32 port_sel; - - pp_reg = PP_CONTROL(0); - port_sel = intel_de_read(dev_priv, PP_ON_DELAYS(0)) & PANEL_PORT_SELECT_MASK; - - switch (port_sel) { - case PANEL_PORT_SELECT_LVDS: - intel_lvds_port_enabled(dev_priv, PCH_LVDS, &panel_pipe); - break; - case PANEL_PORT_SELECT_DPA: - g4x_dp_port_enabled(dev_priv, DP_A, PORT_A, &panel_pipe); - break; - case PANEL_PORT_SELECT_DPC: - g4x_dp_port_enabled(dev_priv, PCH_DP_C, PORT_C, &panel_pipe); - break; - case PANEL_PORT_SELECT_DPD: - g4x_dp_port_enabled(dev_priv, PCH_DP_D, PORT_D, &panel_pipe); - break; - default: - MISSING_CASE(port_sel); - break; - } - } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) { - /* presumably write lock depends on pipe, not port select */ - pp_reg = PP_CONTROL(pipe); - panel_pipe = pipe; - } else { - u32 port_sel; - - pp_reg = PP_CONTROL(0); - port_sel = intel_de_read(dev_priv, PP_ON_DELAYS(0)) & PANEL_PORT_SELECT_MASK; - - drm_WARN_ON(&dev_priv->drm, - port_sel != PANEL_PORT_SELECT_LVDS); - intel_lvds_port_enabled(dev_priv, LVDS, &panel_pipe); - } - - val = intel_de_read(dev_priv, pp_reg); - if (!(val & PANEL_POWER_ON) || - ((val & PANEL_UNLOCK_MASK) == PANEL_UNLOCK_REGS)) - locked = false; - - I915_STATE_WARN(panel_pipe == pipe && locked, - "panel assertion failure, pipe %c regs locked\n", - pipe_name(pipe)); -} - -void assert_pipe(struct drm_i915_private *dev_priv, - enum transcoder cpu_transcoder, bool state) +void assert_transcoder(struct drm_i915_private *dev_priv, + enum transcoder cpu_transcoder, bool state) { bool cur_state; enum intel_display_power_domain power_domain; @@ -942,7 +720,7 @@ enum pipe intel_crtc_pch_transcoder(struct intel_crtc *crtc) return crtc->pipe; } -void intel_enable_pipe(const struct intel_crtc_state *new_crtc_state) +void intel_enable_transcoder(const struct intel_crtc_state *new_crtc_state) { struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc); struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); @@ -1003,7 +781,7 @@ void intel_enable_pipe(const struct intel_crtc_state *new_crtc_state) intel_wait_for_pipe_scanline_moving(crtc); } -void intel_disable_pipe(const struct intel_crtc_state *old_crtc_state) +void intel_disable_transcoder(const struct intel_crtc_state *old_crtc_state) { struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->uapi.crtc); struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); @@ -1053,69 +831,6 @@ intel_format_info_is_yuv_semiplanar(const struct drm_format_info *info, info->num_planes == (is_ccs_modifier(modifier) ? 4 : 2); } -unsigned int -intel_tile_width_bytes(const struct drm_framebuffer *fb, int color_plane) -{ - struct drm_i915_private *dev_priv = to_i915(fb->dev); - unsigned int cpp = fb->format->cpp[color_plane]; - - switch (fb->modifier) { - case DRM_FORMAT_MOD_LINEAR: - return intel_tile_size(dev_priv); - case I915_FORMAT_MOD_X_TILED: - if (DISPLAY_VER(dev_priv) == 2) - return 128; - else - return 512; - case I915_FORMAT_MOD_Y_TILED_CCS: - if (is_ccs_plane(fb, color_plane)) - return 128; - fallthrough; - case I915_FORMAT_MOD_Y_TILED_GEN12_RC_CCS: - case I915_FORMAT_MOD_Y_TILED_GEN12_RC_CCS_CC: - case I915_FORMAT_MOD_Y_TILED_GEN12_MC_CCS: - if (is_ccs_plane(fb, color_plane)) - return 64; - fallthrough; - case I915_FORMAT_MOD_Y_TILED: - if (DISPLAY_VER(dev_priv) == 2 || HAS_128_BYTE_Y_TILING(dev_priv)) - return 128; - else - return 512; - case I915_FORMAT_MOD_Yf_TILED_CCS: - if (is_ccs_plane(fb, color_plane)) - return 128; - fallthrough; - case I915_FORMAT_MOD_Yf_TILED: - switch (cpp) { - case 1: - return 64; - case 2: - case 4: - return 128; - case 8: - case 16: - return 256; - default: - MISSING_CASE(cpp); - return cpp; - } - break; - default: - MISSING_CASE(fb->modifier); - return cpp; - } -} - -unsigned int -intel_fb_align_height(const struct drm_framebuffer *fb, - int color_plane, unsigned int height) -{ - unsigned int tile_height = intel_tile_height(fb, color_plane); - - return ALIGN(height, tile_height); -} - unsigned int intel_rotation_info_size(const struct intel_rotation_info *rot_info) { unsigned int size = 0; @@ -1132,82 +847,16 @@ unsigned int intel_remapped_info_size(const struct intel_remapped_info *rem_info unsigned int size = 0; int i; - for (i = 0 ; i < ARRAY_SIZE(rem_info->plane); i++) + for (i = 0 ; i < ARRAY_SIZE(rem_info->plane); i++) { + if (rem_info->plane_alignment) + size = ALIGN(size, rem_info->plane_alignment); size += rem_info->plane[i].dst_stride * rem_info->plane[i].height; - - return size; -} - -static unsigned int intel_linear_alignment(const struct drm_i915_private *dev_priv) -{ - if (DISPLAY_VER(dev_priv) >= 9) - return 256 * 1024; - else if (IS_I965G(dev_priv) || IS_I965GM(dev_priv) || - IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) - return 128 * 1024; - else if (DISPLAY_VER(dev_priv) >= 4) - return 4 * 1024; - else - return 0; -} - -static bool has_async_flips(struct drm_i915_private *i915) -{ - return DISPLAY_VER(i915) >= 5; -} - -unsigned int intel_surf_alignment(const struct drm_framebuffer *fb, - int color_plane) -{ - struct drm_i915_private *dev_priv = to_i915(fb->dev); - - if (intel_fb_uses_dpt(fb)) - return 512 * 4096; - - /* AUX_DIST needs only 4K alignment */ - if (is_ccs_plane(fb, color_plane)) - return 4096; - - if (is_semiplanar_uv_plane(fb, color_plane)) { - /* - * TODO: cross-check wrt. the bspec stride in bytes * 64 bytes - * alignment for linear UV planes on all platforms. - */ - if (DISPLAY_VER(dev_priv) >= 12) { - if (fb->modifier == DRM_FORMAT_MOD_LINEAR) - return intel_linear_alignment(dev_priv); - - return intel_tile_row_size(fb, color_plane); - } - - return 4096; } - drm_WARN_ON(&dev_priv->drm, color_plane != 0); - - switch (fb->modifier) { - case DRM_FORMAT_MOD_LINEAR: - return intel_linear_alignment(dev_priv); - case I915_FORMAT_MOD_X_TILED: - if (has_async_flips(dev_priv)) - return 256 * 1024; - return 0; - case I915_FORMAT_MOD_Y_TILED_GEN12_MC_CCS: - case I915_FORMAT_MOD_Y_TILED_GEN12_RC_CCS: - case I915_FORMAT_MOD_Y_TILED_GEN12_RC_CCS_CC: - return 16 * 1024; - case I915_FORMAT_MOD_Y_TILED_CCS: - case I915_FORMAT_MOD_Yf_TILED_CCS: - case I915_FORMAT_MOD_Y_TILED: - case I915_FORMAT_MOD_Yf_TILED: - return 1 * 1024 * 1024; - default: - MISSING_CASE(fb->modifier); - return 0; - } + return size; } -static bool intel_plane_uses_fence(const struct intel_plane_state *plane_state) +bool intel_plane_uses_fence(const struct intel_plane_state *plane_state) { struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane); struct drm_i915_private *dev_priv = to_i915(plane->base.dev); @@ -1217,198 +866,6 @@ static bool intel_plane_uses_fence(const struct intel_plane_state *plane_state) plane_state->view.gtt.type == I915_GGTT_VIEW_NORMAL); } -static struct i915_vma * -intel_pin_fb_obj_dpt(struct drm_framebuffer *fb, - const struct i915_ggtt_view *view, - bool uses_fence, - unsigned long *out_flags, - struct i915_address_space *vm) -{ - struct drm_device *dev = fb->dev; - struct drm_i915_private *dev_priv = to_i915(dev); - struct drm_i915_gem_object *obj = intel_fb_obj(fb); - struct i915_vma *vma; - u32 alignment; - int ret; - - if (WARN_ON(!i915_gem_object_is_framebuffer(obj))) - return ERR_PTR(-EINVAL); - - alignment = 4096 * 512; - - atomic_inc(&dev_priv->gpu_error.pending_fb_pin); - - ret = i915_gem_object_set_cache_level(obj, I915_CACHE_NONE); - if (ret) { - vma = ERR_PTR(ret); - goto err; - } - - vma = i915_vma_instance(obj, vm, view); - if (IS_ERR(vma)) - goto err; - - if (i915_vma_misplaced(vma, 0, alignment, 0)) { - ret = i915_vma_unbind(vma); - if (ret) { - vma = ERR_PTR(ret); - goto err; - } - } - - ret = i915_vma_pin(vma, 0, alignment, PIN_GLOBAL); - if (ret) { - vma = ERR_PTR(ret); - goto err; - } - - vma->display_alignment = max_t(u64, vma->display_alignment, alignment); - - i915_gem_object_flush_if_display(obj); - - i915_vma_get(vma); -err: - atomic_dec(&dev_priv->gpu_error.pending_fb_pin); - - return vma; -} - -struct i915_vma * -intel_pin_and_fence_fb_obj(struct drm_framebuffer *fb, - bool phys_cursor, - const struct i915_ggtt_view *view, - bool uses_fence, - unsigned long *out_flags) -{ - struct drm_device *dev = fb->dev; - struct drm_i915_private *dev_priv = to_i915(dev); - struct drm_i915_gem_object *obj = intel_fb_obj(fb); - intel_wakeref_t wakeref; - struct i915_gem_ww_ctx ww; - struct i915_vma *vma; - unsigned int pinctl; - u32 alignment; - int ret; - - if (drm_WARN_ON(dev, !i915_gem_object_is_framebuffer(obj))) - return ERR_PTR(-EINVAL); - - if (phys_cursor) - alignment = intel_cursor_alignment(dev_priv); - else - alignment = intel_surf_alignment(fb, 0); - if (drm_WARN_ON(dev, alignment && !is_power_of_2(alignment))) - return ERR_PTR(-EINVAL); - - /* Note that the w/a also requires 64 PTE of padding following the - * bo. We currently fill all unused PTE with the shadow page and so - * we should always have valid PTE following the scanout preventing - * the VT-d warning. - */ - if (intel_scanout_needs_vtd_wa(dev_priv) && alignment < 256 * 1024) - alignment = 256 * 1024; - - /* - * Global gtt pte registers are special registers which actually forward - * writes to a chunk of system memory. Which means that there is no risk - * that the register values disappear as soon as we call - * intel_runtime_pm_put(), so it is correct to wrap only the - * pin/unpin/fence and not more. - */ - wakeref = intel_runtime_pm_get(&dev_priv->runtime_pm); - - atomic_inc(&dev_priv->gpu_error.pending_fb_pin); - - /* - * Valleyview is definitely limited to scanning out the first - * 512MiB. Lets presume this behaviour was inherited from the - * g4x display engine and that all earlier gen are similarly - * limited. Testing suggests that it is a little more - * complicated than this. For example, Cherryview appears quite - * happy to scanout from anywhere within its global aperture. - */ - pinctl = 0; - if (HAS_GMCH(dev_priv)) - pinctl |= PIN_MAPPABLE; - - i915_gem_ww_ctx_init(&ww, true); -retry: - ret = i915_gem_object_lock(obj, &ww); - if (!ret && phys_cursor) - ret = i915_gem_object_attach_phys(obj, alignment); - else if (!ret && HAS_LMEM(dev_priv)) - ret = i915_gem_object_migrate(obj, &ww, INTEL_REGION_LMEM); - /* TODO: Do we need to sync when migration becomes async? */ - if (!ret) - ret = i915_gem_object_pin_pages(obj); - if (ret) - goto err; - - if (!ret) { - vma = i915_gem_object_pin_to_display_plane(obj, &ww, alignment, - view, pinctl); - if (IS_ERR(vma)) { - ret = PTR_ERR(vma); - goto err_unpin; - } - } - - if (uses_fence && i915_vma_is_map_and_fenceable(vma)) { - /* - * Install a fence for tiled scan-out. Pre-i965 always needs a - * fence, whereas 965+ only requires a fence if using - * framebuffer compression. For simplicity, we always, when - * possible, install a fence as the cost is not that onerous. - * - * If we fail to fence the tiled scanout, then either the - * modeset will reject the change (which is highly unlikely as - * the affected systems, all but one, do not have unmappable - * space) or we will not be able to enable full powersaving - * techniques (also likely not to apply due to various limits - * FBC and the like impose on the size of the buffer, which - * presumably we violated anyway with this unmappable buffer). - * Anyway, it is presumably better to stumble onwards with - * something and try to run the system in a "less than optimal" - * mode that matches the user configuration. - */ - ret = i915_vma_pin_fence(vma); - if (ret != 0 && DISPLAY_VER(dev_priv) < 4) { - i915_vma_unpin(vma); - goto err_unpin; - } - ret = 0; - - if (vma->fence) - *out_flags |= PLANE_HAS_FENCE; - } - - i915_vma_get(vma); - -err_unpin: - i915_gem_object_unpin_pages(obj); -err: - if (ret == -EDEADLK) { - ret = i915_gem_ww_ctx_backoff(&ww); - if (!ret) - goto retry; - } - i915_gem_ww_ctx_fini(&ww); - if (ret) - vma = ERR_PTR(ret); - - atomic_dec(&dev_priv->gpu_error.pending_fb_pin); - intel_runtime_pm_put(&dev_priv->runtime_pm, wakeref); - return vma; -} - -void intel_unpin_fb_vma(struct i915_vma *vma, unsigned long flags) -{ - if (flags & PLANE_HAS_FENCE) - i915_vma_unpin_fence(vma); - i915_vma_unpin(vma); - i915_vma_put(vma); -} - /* * Convert the x/y offsets into a linear offset. * Only valid with 0/180 degree rotation, which is fine since linear @@ -1440,22 +897,6 @@ void intel_add_fb_offsets(int *x, int *y, *y += state->view.color_plane[color_plane].y; } -static unsigned int intel_fb_modifier_to_tiling(u64 fb_modifier) -{ - switch (fb_modifier) { - case I915_FORMAT_MOD_X_TILED: - return I915_TILING_X; - case I915_FORMAT_MOD_Y_TILED: - case I915_FORMAT_MOD_Y_TILED_CCS: - case I915_FORMAT_MOD_Y_TILED_GEN12_RC_CCS: - case I915_FORMAT_MOD_Y_TILED_GEN12_RC_CCS_CC: - case I915_FORMAT_MOD_Y_TILED_GEN12_MC_CCS: - return I915_TILING_Y; - default: - return I915_TILING_NONE; - } -} - /* * From the Sky Lake PRM: * "The Color Control Surface (CCS) contains the compression status of @@ -1586,12 +1027,6 @@ intel_get_format_info(const struct drm_mode_fb_cmd2 *cmd) } } -static int gen12_ccs_aux_stride(struct drm_framebuffer *fb, int ccs_plane) -{ - return DIV_ROUND_UP(fb->pitches[skl_ccs_to_main_plane(fb, ccs_plane)], - 512) * 64; -} - u32 intel_plane_fb_max_stride(struct drm_i915_private *dev_priv, u32 pixel_format, u64 modifier) { @@ -1616,188 +1051,6 @@ u32 intel_plane_fb_max_stride(struct drm_i915_private *dev_priv, DRM_MODE_ROTATE_0); } -static -u32 intel_fb_max_stride(struct drm_i915_private *dev_priv, - u32 pixel_format, u64 modifier) -{ - /* - * Arbitrary limit for gen4+ chosen to match the - * render engine max stride. - * - * The new CCS hash mode makes remapping impossible - */ - if (DISPLAY_VER(dev_priv) < 4 || is_ccs_modifier(modifier) || - intel_modifier_uses_dpt(dev_priv, modifier)) - return intel_plane_fb_max_stride(dev_priv, pixel_format, modifier); - else if (DISPLAY_VER(dev_priv) >= 7) - return 256 * 1024; - else - return 128 * 1024; -} - -static u32 -intel_fb_stride_alignment(const struct drm_framebuffer *fb, int color_plane) -{ - struct drm_i915_private *dev_priv = to_i915(fb->dev); - u32 tile_width; - - if (is_surface_linear(fb, color_plane)) { - u32 max_stride = intel_plane_fb_max_stride(dev_priv, - fb->format->format, - fb->modifier); - - /* - * To make remapping with linear generally feasible - * we need the stride to be page aligned. - */ - if (fb->pitches[color_plane] > max_stride && - !is_ccs_modifier(fb->modifier)) - return intel_tile_size(dev_priv); - else - return 64; - } - - tile_width = intel_tile_width_bytes(fb, color_plane); - if (is_ccs_modifier(fb->modifier)) { - /* - * Display WA #0531: skl,bxt,kbl,glk - * - * Render decompression and plane width > 3840 - * combined with horizontal panning requires the - * plane stride to be a multiple of 4. We'll just - * require the entire fb to accommodate that to avoid - * potential runtime errors at plane configuration time. - */ - if ((DISPLAY_VER(dev_priv) == 9 || IS_GEMINILAKE(dev_priv)) && - color_plane == 0 && fb->width > 3840) - tile_width *= 4; - /* - * The main surface pitch must be padded to a multiple of four - * tile widths. - */ - else if (DISPLAY_VER(dev_priv) >= 12) - tile_width *= 4; - } - return tile_width; -} - -static struct i915_vma * -initial_plane_vma(struct drm_i915_private *i915, - struct intel_initial_plane_config *plane_config) -{ - struct drm_i915_gem_object *obj; - struct i915_vma *vma; - u32 base, size; - - if (plane_config->size == 0) - return NULL; - - base = round_down(plane_config->base, - I915_GTT_MIN_ALIGNMENT); - size = round_up(plane_config->base + plane_config->size, - I915_GTT_MIN_ALIGNMENT); - size -= base; - - /* - * If the FB is too big, just don't use it since fbdev is not very - * important and we should probably use that space with FBC or other - * features. - */ - if (IS_ENABLED(CONFIG_FRAMEBUFFER_CONSOLE) && - size * 2 > i915->stolen_usable_size) - return NULL; - - obj = i915_gem_object_create_stolen_for_preallocated(i915, base, size); - if (IS_ERR(obj)) - return NULL; - - /* - * Mark it WT ahead of time to avoid changing the - * cache_level during fbdev initialization. The - * unbind there would get stuck waiting for rcu. - */ - i915_gem_object_set_cache_coherency(obj, HAS_WT(i915) ? - I915_CACHE_WT : I915_CACHE_NONE); - - switch (plane_config->tiling) { - case I915_TILING_NONE: - break; - case I915_TILING_X: - case I915_TILING_Y: - obj->tiling_and_stride = - plane_config->fb->base.pitches[0] | - plane_config->tiling; - break; - default: - MISSING_CASE(plane_config->tiling); - goto err_obj; - } - - vma = i915_vma_instance(obj, &i915->ggtt.vm, NULL); - if (IS_ERR(vma)) - goto err_obj; - - if (i915_ggtt_pin(vma, NULL, 0, PIN_MAPPABLE | PIN_OFFSET_FIXED | base)) - goto err_obj; - - if (i915_gem_object_is_tiled(obj) && - !i915_vma_is_map_and_fenceable(vma)) - goto err_obj; - - return vma; - -err_obj: - i915_gem_object_put(obj); - return NULL; -} - -static bool -intel_alloc_initial_plane_obj(struct intel_crtc *crtc, - struct intel_initial_plane_config *plane_config) -{ - struct drm_device *dev = crtc->base.dev; - struct drm_i915_private *dev_priv = to_i915(dev); - struct drm_mode_fb_cmd2 mode_cmd = { 0 }; - struct drm_framebuffer *fb = &plane_config->fb->base; - struct i915_vma *vma; - - switch (fb->modifier) { - case DRM_FORMAT_MOD_LINEAR: - case I915_FORMAT_MOD_X_TILED: - case I915_FORMAT_MOD_Y_TILED: - break; - default: - drm_dbg(&dev_priv->drm, - "Unsupported modifier for initial FB: 0x%llx\n", - fb->modifier); - return false; - } - - vma = initial_plane_vma(dev_priv, plane_config); - if (!vma) - return false; - - mode_cmd.pixel_format = fb->format->format; - mode_cmd.width = fb->width; - mode_cmd.height = fb->height; - mode_cmd.pitches[0] = fb->pitches[0]; - mode_cmd.modifier[0] = fb->modifier; - mode_cmd.flags = DRM_MODE_FB_MODIFIERS; - - if (intel_framebuffer_init(to_intel_framebuffer(fb), - vma->obj, &mode_cmd)) { - drm_dbg_kms(&dev_priv->drm, "intel fb init failed\n"); - goto err_vma; - } - - plane_config->vma = vma; - return true; - -err_vma: - i915_vma_put(vma); - return false; -} - static void intel_set_plane_visible(struct intel_crtc_state *crtc_state, struct intel_plane_state *plane_state, @@ -1833,8 +1086,8 @@ static void fixup_plane_bitmasks(struct intel_crtc_state *crtc_state) } } -static void intel_plane_disable_noatomic(struct intel_crtc *crtc, - struct intel_plane *plane) +void intel_plane_disable_noatomic(struct intel_crtc *crtc, + struct intel_plane *plane) { struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); struct intel_crtc_state *crtc_state = @@ -1879,168 +1132,6 @@ static void intel_plane_disable_noatomic(struct intel_crtc *crtc, intel_wait_for_vblank(dev_priv, crtc->pipe); } -static struct i915_vma *intel_dpt_pin(struct i915_address_space *vm) -{ - struct drm_i915_private *i915 = vm->i915; - struct i915_dpt *dpt = i915_vm_to_dpt(vm); - intel_wakeref_t wakeref; - struct i915_vma *vma; - void __iomem *iomem; - - wakeref = intel_runtime_pm_get(&i915->runtime_pm); - atomic_inc(&i915->gpu_error.pending_fb_pin); - - vma = i915_gem_object_ggtt_pin(dpt->obj, NULL, 0, 4096, - HAS_LMEM(i915) ? 0 : PIN_MAPPABLE); - if (IS_ERR(vma)) - goto err; - - iomem = i915_vma_pin_iomap(vma); - i915_vma_unpin(vma); - if (IS_ERR(iomem)) { - vma = iomem; - goto err; - } - - dpt->vma = vma; - dpt->iomem = iomem; - - i915_vma_get(vma); - -err: - atomic_dec(&i915->gpu_error.pending_fb_pin); - intel_runtime_pm_put(&i915->runtime_pm, wakeref); - - return vma; -} - -static void intel_dpt_unpin(struct i915_address_space *vm) -{ - struct i915_dpt *dpt = i915_vm_to_dpt(vm); - - i915_vma_unpin_iomap(dpt->vma); - i915_vma_put(dpt->vma); -} - -static bool -intel_reuse_initial_plane_obj(struct drm_i915_private *i915, - const struct intel_initial_plane_config *plane_config, - struct drm_framebuffer **fb, - struct i915_vma **vma) -{ - struct intel_crtc *crtc; - - for_each_intel_crtc(&i915->drm, crtc) { - struct intel_crtc_state *crtc_state = - to_intel_crtc_state(crtc->base.state); - struct intel_plane *plane = - to_intel_plane(crtc->base.primary); - struct intel_plane_state *plane_state = - to_intel_plane_state(plane->base.state); - - if (!crtc_state->uapi.active) - continue; - - if (!plane_state->ggtt_vma) - continue; - - if (intel_plane_ggtt_offset(plane_state) == plane_config->base) { - *fb = plane_state->hw.fb; - *vma = plane_state->ggtt_vma; - return true; - } - } - - return false; -} - -static void -intel_find_initial_plane_obj(struct intel_crtc *crtc, - struct intel_initial_plane_config *plane_config) -{ - struct drm_device *dev = crtc->base.dev; - struct drm_i915_private *dev_priv = to_i915(dev); - struct intel_crtc_state *crtc_state = - to_intel_crtc_state(crtc->base.state); - struct intel_plane *plane = - to_intel_plane(crtc->base.primary); - struct intel_plane_state *plane_state = - to_intel_plane_state(plane->base.state); - struct drm_framebuffer *fb; - struct i915_vma *vma; - - /* - * TODO: - * Disable planes if get_initial_plane_config() failed. - * Make sure things work if the surface base is not page aligned. - */ - if (!plane_config->fb) - return; - - if (intel_alloc_initial_plane_obj(crtc, plane_config)) { - fb = &plane_config->fb->base; - vma = plane_config->vma; - goto valid_fb; - } - - /* - * Failed to alloc the obj, check to see if we should share - * an fb with another CRTC instead - */ - if (intel_reuse_initial_plane_obj(dev_priv, plane_config, &fb, &vma)) - goto valid_fb; - - /* - * We've failed to reconstruct the BIOS FB. Current display state - * indicates that the primary plane is visible, but has a NULL FB, - * which will lead to problems later if we don't fix it up. The - * simplest solution is to just disable the primary plane now and - * pretend the BIOS never had it enabled. - */ - intel_plane_disable_noatomic(crtc, plane); - if (crtc_state->bigjoiner) { - struct intel_crtc *slave = - crtc_state->bigjoiner_linked_crtc; - intel_plane_disable_noatomic(slave, to_intel_plane(slave->base.primary)); - } - - return; - -valid_fb: - plane_state->uapi.rotation = plane_config->rotation; - intel_fb_fill_view(to_intel_framebuffer(fb), - plane_state->uapi.rotation, &plane_state->view); - - __i915_vma_pin(vma); - plane_state->ggtt_vma = i915_vma_get(vma); - if (intel_plane_uses_fence(plane_state) && - i915_vma_pin_fence(vma) == 0 && vma->fence) - plane_state->flags |= PLANE_HAS_FENCE; - - plane_state->uapi.src_x = 0; - plane_state->uapi.src_y = 0; - plane_state->uapi.src_w = fb->width << 16; - plane_state->uapi.src_h = fb->height << 16; - - plane_state->uapi.crtc_x = 0; - plane_state->uapi.crtc_y = 0; - plane_state->uapi.crtc_w = fb->width; - plane_state->uapi.crtc_h = fb->height; - - if (plane_config->tiling) - dev_priv->preserve_bios_swizzle = true; - - plane_state->uapi.fb = fb; - drm_framebuffer_get(fb); - - plane_state->uapi.crtc = &crtc->base; - intel_plane_copy_uapi_to_hw_state(plane_state, plane_state, crtc); - - intel_frontbuffer_flush(to_intel_frontbuffer(fb), ORIGIN_DIRTYFB); - - atomic_or(plane->frontbuffer_bit, &to_intel_frontbuffer(fb)->bits); -} - unsigned int intel_plane_fence_y_offset(const struct intel_plane_state *plane_state) { @@ -2449,55 +1540,6 @@ static void ilk_pch_transcoder_set_timings(const struct intel_crtc_state *crtc_s intel_de_read(dev_priv, VSYNCSHIFT(cpu_transcoder))); } -static void cpt_set_fdi_bc_bifurcation(struct drm_i915_private *dev_priv, bool enable) -{ - u32 temp; - - temp = intel_de_read(dev_priv, SOUTH_CHICKEN1); - if (!!(temp & FDI_BC_BIFURCATION_SELECT) == enable) - return; - - drm_WARN_ON(&dev_priv->drm, - intel_de_read(dev_priv, FDI_RX_CTL(PIPE_B)) & - FDI_RX_ENABLE); - drm_WARN_ON(&dev_priv->drm, - intel_de_read(dev_priv, FDI_RX_CTL(PIPE_C)) & - FDI_RX_ENABLE); - - temp &= ~FDI_BC_BIFURCATION_SELECT; - if (enable) - temp |= FDI_BC_BIFURCATION_SELECT; - - drm_dbg_kms(&dev_priv->drm, "%sabling fdi C rx\n", - enable ? "en" : "dis"); - intel_de_write(dev_priv, SOUTH_CHICKEN1, temp); - intel_de_posting_read(dev_priv, SOUTH_CHICKEN1); -} - -static void ivb_update_fdi_bc_bifurcation(const struct intel_crtc_state *crtc_state) -{ - struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); - struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); - - switch (crtc->pipe) { - case PIPE_A: - break; - case PIPE_B: - if (crtc_state->fdi_lanes > 2) - cpt_set_fdi_bc_bifurcation(dev_priv, false); - else - cpt_set_fdi_bc_bifurcation(dev_priv, true); - - break; - case PIPE_C: - cpt_set_fdi_bc_bifurcation(dev_priv, true); - - break; - default: - BUG(); - } -} - /* * Finds the encoder associated with the given CRTC. This can only be * used when we know that the CRTC isn't feeding multiple encoders! @@ -2547,16 +1589,8 @@ static void ilk_pch_enable(const struct intel_atomic_state *state, assert_pch_transcoder_disabled(dev_priv, pipe); - if (IS_IVYBRIDGE(dev_priv)) - ivb_update_fdi_bc_bifurcation(crtc_state); - - /* Write the TU size bits before fdi link training, so that error - * detection works. */ - intel_de_write(dev_priv, FDI_RX_TUSIZE1(pipe), - intel_de_read(dev_priv, PIPE_DATA_M1(pipe)) & TU_SIZE_MASK); - /* For PCH output, training FDI link */ - dev_priv->display.fdi_link_train(crtc, crtc_state); + intel_fdi_link_train(crtc, crtc_state); /* We need to program the right clock selection before writing the pixel * mutliplier into the DPLL. */ @@ -2584,7 +1618,7 @@ static void ilk_pch_enable(const struct intel_atomic_state *state, intel_enable_shared_dpll(crtc_state); /* set transcoder timing, panel must allow it */ - assert_panel_unlocked(dev_priv, pipe); + assert_pps_unlocked(dev_priv, pipe); ilk_pch_transcoder_set_timings(crtc_state, pipe); intel_fdi_normal_train(crtc); @@ -2842,6 +1876,46 @@ static bool needs_scalerclk_wa(const struct intel_crtc_state *crtc_state) return false; } +static bool needs_cursorclk_wa(const struct intel_crtc_state *crtc_state) +{ + struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev); + + /* Wa_1604331009:icl,jsl,ehl */ + if (is_hdr_mode(crtc_state) && + crtc_state->active_planes & BIT(PLANE_CURSOR) && + DISPLAY_VER(dev_priv) == 11) + return true; + + return false; +} + +static void intel_async_flip_vtd_wa(struct drm_i915_private *i915, + enum pipe pipe, bool enable) +{ + if (DISPLAY_VER(i915) == 9) { + /* + * "Plane N strech max must be programmed to 11b (x1) + * when Async flips are enabled on that plane." + */ + intel_de_rmw(i915, CHICKEN_PIPESL_1(pipe), + SKL_PLANE1_STRETCH_MAX_MASK, + enable ? SKL_PLANE1_STRETCH_MAX_X1 : SKL_PLANE1_STRETCH_MAX_X8); + } else { + /* Also needed on HSW/BDW albeit undocumented */ + intel_de_rmw(i915, CHICKEN_PIPESL_1(pipe), + HSW_PRI_STRETCH_MAX_MASK, + enable ? HSW_PRI_STRETCH_MAX_X1 : HSW_PRI_STRETCH_MAX_X8); + } +} + +static bool needs_async_flip_vtd_wa(const struct intel_crtc_state *crtc_state) +{ + struct drm_i915_private *i915 = to_i915(crtc_state->uapi.crtc->dev); + + return crtc_state->uapi.async_flip && intel_vtd_active() && + (DISPLAY_VER(i915) == 9 || IS_BROADWELL(i915) || IS_HASWELL(i915)); +} + static bool planes_enabling(const struct intel_crtc_state *old_crtc_state, const struct intel_crtc_state *new_crtc_state) { @@ -2869,12 +1943,17 @@ static void intel_post_plane_update(struct intel_atomic_state *state, intel_frontbuffer_flip(dev_priv, new_crtc_state->fb_bits); if (new_crtc_state->update_wm_post && new_crtc_state->hw.active) - intel_update_watermarks(crtc); + intel_update_watermarks(dev_priv); if (hsw_post_update_enable_ips(old_crtc_state, new_crtc_state)) hsw_enable_ips(new_crtc_state); intel_fbc_post_update(state, crtc); + intel_drrs_page_flip(state, crtc); + + if (needs_async_flip_vtd_wa(old_crtc_state) && + !needs_async_flip_vtd_wa(new_crtc_state)) + intel_async_flip_vtd_wa(dev_priv, pipe, false); if (needs_nv12_wa(old_crtc_state) && !needs_nv12_wa(new_crtc_state)) @@ -2883,6 +1962,11 @@ static void intel_post_plane_update(struct intel_atomic_state *state, if (needs_scalerclk_wa(old_crtc_state) && !needs_scalerclk_wa(new_crtc_state)) icl_wa_scalerclkgating(dev_priv, pipe, false); + + if (needs_cursorclk_wa(old_crtc_state) && + !needs_cursorclk_wa(new_crtc_state)) + icl_wa_cursorclkgating(dev_priv, pipe, false); + } static void intel_crtc_enable_flip_done(struct intel_atomic_state *state, @@ -2969,6 +2053,10 @@ static void intel_pre_plane_update(struct intel_atomic_state *state, if (intel_fbc_pre_update(state, crtc)) intel_wait_for_vblank(dev_priv, pipe); + if (!needs_async_flip_vtd_wa(old_crtc_state) && + needs_async_flip_vtd_wa(new_crtc_state)) + intel_async_flip_vtd_wa(dev_priv, pipe, true); + /* Display WA 827 */ if (!needs_nv12_wa(old_crtc_state) && needs_nv12_wa(new_crtc_state)) @@ -2979,6 +2067,11 @@ static void intel_pre_plane_update(struct intel_atomic_state *state, needs_scalerclk_wa(new_crtc_state)) icl_wa_scalerclkgating(dev_priv, pipe, true); + /* Wa_1604331009:icl,jsl,ehl */ + if (!needs_cursorclk_wa(old_crtc_state) && + needs_cursorclk_wa(new_crtc_state)) + icl_wa_cursorclkgating(dev_priv, pipe, true); + /* * Vblank time updates from the shadow to live plane control register * are blocked if the memory self-refresh mode is active at that @@ -3022,10 +2115,9 @@ static void intel_pre_plane_update(struct intel_atomic_state *state, * we'll continue to update watermarks the old way, if flags tell * us to. */ - if (dev_priv->display.initial_watermarks) - dev_priv->display.initial_watermarks(state, crtc); - else if (new_crtc_state->update_wm_pre) - intel_update_watermarks(crtc); + if (!intel_initial_watermarks(state, crtc)) + if (new_crtc_state->update_wm_pre) + intel_update_watermarks(dev_priv); } /* @@ -3360,9 +2452,6 @@ static void ilk_crtc_enable(struct intel_atomic_state *state, intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, false); intel_set_pch_fifo_underrun_reporting(dev_priv, pipe, false); - if (new_crtc_state->has_pch_encoder) - intel_prepare_shared_dpll(new_crtc_state); - if (intel_crtc_has_dp_encoder(new_crtc_state)) intel_dp_set_m_n(new_crtc_state, M1_N1); @@ -3400,9 +2489,8 @@ static void ilk_crtc_enable(struct intel_atomic_state *state, /* update DSPCNTR to configure gamma for pipe bottom color */ intel_disable_primary_plane(new_crtc_state); - if (dev_priv->display.initial_watermarks) - dev_priv->display.initial_watermarks(state, crtc); - intel_enable_pipe(new_crtc_state); + intel_initial_watermarks(state, crtc); + intel_enable_transcoder(new_crtc_state); if (new_crtc_state->has_pch_encoder) ilk_pch_enable(state, new_crtc_state); @@ -3578,10 +2666,9 @@ static void hsw_crtc_enable(struct intel_atomic_state *state, &new_crtc_state->fdi_m_n, NULL); hsw_set_frame_start_delay(new_crtc_state); - } - if (!transcoder_is_dsi(cpu_transcoder)) - hsw_set_pipeconf(new_crtc_state); + hsw_set_transconf(new_crtc_state); + } crtc->active = true; @@ -3611,8 +2698,7 @@ static void hsw_crtc_enable(struct intel_atomic_state *state, if (DISPLAY_VER(dev_priv) >= 11) icl_set_pipe_chicken(new_crtc_state); - if (dev_priv->display.initial_watermarks) - dev_priv->display.initial_watermarks(state, crtc); + intel_initial_watermarks(state, crtc); if (DISPLAY_VER(dev_priv) >= 11) { const struct intel_dbuf_state *dbuf_state = @@ -3676,7 +2762,7 @@ static void ilk_crtc_disable(struct intel_atomic_state *state, intel_crtc_vblank_off(old_crtc_state); - intel_disable_pipe(old_crtc_state); + intel_disable_transcoder(old_crtc_state); ilk_pfit_disable(old_crtc_state); @@ -3738,7 +2824,7 @@ static void i9xx_pfit_enable(const struct intel_crtc_state *crtc_state) */ drm_WARN_ON(&dev_priv->drm, intel_de_read(dev_priv, PFIT_CONTROL) & PFIT_ENABLE); - assert_pipe_disabled(dev_priv, crtc_state->cpu_transcoder); + assert_transcoder_disabled(dev_priv, crtc_state->cpu_transcoder); intel_de_write(dev_priv, PFIT_PGM_RATIOS, crtc_state->gmch_pfit.pgm_ratios); @@ -3858,11 +2944,7 @@ enum intel_display_power_domain intel_port_to_power_domain(enum port port) enum intel_display_power_domain intel_aux_power_domain(struct intel_digital_port *dig_port) { - struct drm_i915_private *dev_priv = to_i915(dig_port->base.base.dev); - enum phy phy = intel_port_to_phy(dev_priv, dig_port->base.port); - - if (intel_phy_is_tc(dev_priv, phy) && - dig_port->tc_mode == TC_PORT_TBT_ALT) { + if (intel_tc_port_in_tbt_alt_mode(dig_port)) { switch (dig_port->aux_ch) { case AUX_CH_C: return POWER_DOMAIN_AUX_C_TBT; @@ -3923,16 +3005,16 @@ static u64 get_crtc_power_domains(struct intel_crtc_state *crtc_state) { struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); + enum transcoder cpu_transcoder = crtc_state->cpu_transcoder; struct drm_encoder *encoder; enum pipe pipe = crtc->pipe; u64 mask; - enum transcoder transcoder = crtc_state->cpu_transcoder; if (!crtc_state->hw.active) return 0; mask = BIT_ULL(POWER_DOMAIN_PIPE(pipe)); - mask |= BIT_ULL(POWER_DOMAIN_TRANSCODER(transcoder)); + mask |= BIT_ULL(POWER_DOMAIN_TRANSCODER(cpu_transcoder)); if (crtc_state->pch_pfit.enabled || crtc_state->pch_pfit.force_thru) mask |= BIT_ULL(POWER_DOMAIN_PIPE_PANEL_FITTER(pipe)); @@ -3951,7 +3033,7 @@ static u64 get_crtc_power_domains(struct intel_crtc_state *crtc_state) mask |= BIT_ULL(POWER_DOMAIN_DISPLAY_CORE); if (crtc_state->dsc.compression_enable) - mask |= BIT_ULL(intel_dsc_power_domain(crtc_state)); + mask |= BIT_ULL(intel_dsc_power_domain(crtc, cpu_transcoder)); return mask; } @@ -4015,13 +3097,10 @@ static void valleyview_crtc_enable(struct intel_atomic_state *state, intel_encoders_pre_pll_enable(state, crtc); - if (IS_CHERRYVIEW(dev_priv)) { - chv_prepare_pll(crtc, new_crtc_state); - chv_enable_pll(crtc, new_crtc_state); - } else { - vlv_prepare_pll(crtc, new_crtc_state); - vlv_enable_pll(crtc, new_crtc_state); - } + if (IS_CHERRYVIEW(dev_priv)) + chv_enable_pll(new_crtc_state); + else + vlv_enable_pll(new_crtc_state); intel_encoders_pre_enable(state, crtc); @@ -4032,25 +3111,14 @@ static void valleyview_crtc_enable(struct intel_atomic_state *state, /* update DSPCNTR to configure gamma for pipe bottom color */ intel_disable_primary_plane(new_crtc_state); - dev_priv->display.initial_watermarks(state, crtc); - intel_enable_pipe(new_crtc_state); + intel_initial_watermarks(state, crtc); + intel_enable_transcoder(new_crtc_state); intel_crtc_vblank_on(new_crtc_state); intel_encoders_enable(state, crtc); } -static void i9xx_set_pll_dividers(const struct intel_crtc_state *crtc_state) -{ - struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); - struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); - - intel_de_write(dev_priv, FP0(crtc->pipe), - crtc_state->dpll_hw_state.fp0); - intel_de_write(dev_priv, FP1(crtc->pipe), - crtc_state->dpll_hw_state.fp1); -} - static void i9xx_crtc_enable(struct intel_atomic_state *state, struct intel_crtc *crtc) { @@ -4062,8 +3130,6 @@ static void i9xx_crtc_enable(struct intel_atomic_state *state, if (drm_WARN_ON(&dev_priv->drm, crtc->active)) return; - i9xx_set_pll_dividers(new_crtc_state); - if (intel_crtc_has_dp_encoder(new_crtc_state)) intel_dp_set_m_n(new_crtc_state, M1_N1); @@ -4079,7 +3145,7 @@ static void i9xx_crtc_enable(struct intel_atomic_state *state, intel_encoders_pre_enable(state, crtc); - i9xx_enable_pll(crtc, new_crtc_state); + i9xx_enable_pll(new_crtc_state); i9xx_pfit_enable(new_crtc_state); @@ -4088,11 +3154,9 @@ static void i9xx_crtc_enable(struct intel_atomic_state *state, /* update DSPCNTR to configure gamma for pipe bottom color */ intel_disable_primary_plane(new_crtc_state); - if (dev_priv->display.initial_watermarks) - dev_priv->display.initial_watermarks(state, crtc); - else - intel_update_watermarks(crtc); - intel_enable_pipe(new_crtc_state); + if (!intel_initial_watermarks(state, crtc)) + intel_update_watermarks(dev_priv); + intel_enable_transcoder(new_crtc_state); intel_crtc_vblank_on(new_crtc_state); @@ -4111,7 +3175,7 @@ static void i9xx_pfit_disable(const struct intel_crtc_state *old_crtc_state) if (!old_crtc_state->gmch_pfit.control) return; - assert_pipe_disabled(dev_priv, old_crtc_state->cpu_transcoder); + assert_transcoder_disabled(dev_priv, old_crtc_state->cpu_transcoder); drm_dbg_kms(&dev_priv->drm, "disabling pfit, current: 0x%08x\n", intel_de_read(dev_priv, PFIT_CONTROL)); @@ -4137,7 +3201,7 @@ static void i9xx_crtc_disable(struct intel_atomic_state *state, intel_crtc_vblank_off(old_crtc_state); - intel_disable_pipe(old_crtc_state); + intel_disable_transcoder(old_crtc_state); i9xx_pfit_disable(old_crtc_state); @@ -4157,8 +3221,8 @@ static void i9xx_crtc_disable(struct intel_atomic_state *state, if (DISPLAY_VER(dev_priv) != 2) intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, false); - if (!dev_priv->display.initial_watermarks) - intel_update_watermarks(crtc); + if (!dev_priv->wm_disp->initial_watermarks) + intel_update_watermarks(dev_priv); /* clock the pipe down to 640x480@60 to potentially save power */ if (IS_I830(dev_priv)) @@ -4211,7 +3275,7 @@ static void intel_crtc_disable_noatomic(struct intel_crtc *crtc, drm_WARN_ON(&dev_priv->drm, IS_ERR(temp_crtc_state) || ret); - dev_priv->display.crtc_disable(to_intel_atomic_state(state), crtc); + dev_priv->display->crtc_disable(to_intel_atomic_state(state), crtc); drm_atomic_state_put(state); @@ -4234,12 +3298,11 @@ static void intel_crtc_disable_noatomic(struct intel_crtc *crtc, encoder->base.crtc = NULL; intel_fbc_disable(crtc); - intel_update_watermarks(crtc); + intel_update_watermarks(dev_priv); intel_disable_shared_dpll(crtc_state); intel_display_power_put_all_in_set(dev_priv, &crtc->enabled_power_domains); - dev_priv->active_pipes &= ~BIT(pipe); cdclk_state->min_cdclk[pipe] = 0; cdclk_state->min_voltage_level[pipe] = 0; cdclk_state->active_pipes &= ~BIT(pipe); @@ -4596,13 +3659,6 @@ static int intel_crtc_compute_config(struct intel_crtc *crtc, } } - /* Cantiga+ cannot handle modes with a hsync front porch of 0. - * WaPruneModeWithIncorrectHsyncOffset:ctg,elk,ilk,snb,ivb,vlv,hsw. - */ - if ((DISPLAY_VER(dev_priv) > 4 || IS_G4X(dev_priv)) && - pipe_mode->crtc_hsync_start == pipe_mode->crtc_hdisplay) - return -EINVAL; - intel_crtc_compute_pixel_rate(pipe_config); if (pipe_config->has_pch_encoder) @@ -5412,102 +4468,6 @@ static void ilk_init_pch_refclk(struct drm_i915_private *dev_priv) BUG_ON(val != final); } -static void lpt_reset_fdi_mphy(struct drm_i915_private *dev_priv) -{ - u32 tmp; - - tmp = intel_de_read(dev_priv, SOUTH_CHICKEN2); - tmp |= FDI_MPHY_IOSFSB_RESET_CTL; - intel_de_write(dev_priv, SOUTH_CHICKEN2, tmp); - - if (wait_for_us(intel_de_read(dev_priv, SOUTH_CHICKEN2) & - FDI_MPHY_IOSFSB_RESET_STATUS, 100)) - drm_err(&dev_priv->drm, "FDI mPHY reset assert timeout\n"); - - tmp = intel_de_read(dev_priv, SOUTH_CHICKEN2); - tmp &= ~FDI_MPHY_IOSFSB_RESET_CTL; - intel_de_write(dev_priv, SOUTH_CHICKEN2, tmp); - - if (wait_for_us((intel_de_read(dev_priv, SOUTH_CHICKEN2) & - FDI_MPHY_IOSFSB_RESET_STATUS) == 0, 100)) - drm_err(&dev_priv->drm, "FDI mPHY reset de-assert timeout\n"); -} - -/* WaMPhyProgramming:hsw */ -static void lpt_program_fdi_mphy(struct drm_i915_private *dev_priv) -{ - u32 tmp; - - tmp = intel_sbi_read(dev_priv, 0x8008, SBI_MPHY); - tmp &= ~(0xFF << 24); - tmp |= (0x12 << 24); - intel_sbi_write(dev_priv, 0x8008, tmp, SBI_MPHY); - - tmp = intel_sbi_read(dev_priv, 0x2008, SBI_MPHY); - tmp |= (1 << 11); - intel_sbi_write(dev_priv, 0x2008, tmp, SBI_MPHY); - - tmp = intel_sbi_read(dev_priv, 0x2108, SBI_MPHY); - tmp |= (1 << 11); - intel_sbi_write(dev_priv, 0x2108, tmp, SBI_MPHY); - - tmp = intel_sbi_read(dev_priv, 0x206C, SBI_MPHY); - tmp |= (1 << 24) | (1 << 21) | (1 << 18); - intel_sbi_write(dev_priv, 0x206C, tmp, SBI_MPHY); - - tmp = intel_sbi_read(dev_priv, 0x216C, SBI_MPHY); - tmp |= (1 << 24) | (1 << 21) | (1 << 18); - intel_sbi_write(dev_priv, 0x216C, tmp, SBI_MPHY); - - tmp = intel_sbi_read(dev_priv, 0x2080, SBI_MPHY); - tmp &= ~(7 << 13); - tmp |= (5 << 13); - intel_sbi_write(dev_priv, 0x2080, tmp, SBI_MPHY); - - tmp = intel_sbi_read(dev_priv, 0x2180, SBI_MPHY); - tmp &= ~(7 << 13); - tmp |= (5 << 13); - intel_sbi_write(dev_priv, 0x2180, tmp, SBI_MPHY); - - tmp = intel_sbi_read(dev_priv, 0x208C, SBI_MPHY); - tmp &= ~0xFF; - tmp |= 0x1C; - intel_sbi_write(dev_priv, 0x208C, tmp, SBI_MPHY); - - tmp = intel_sbi_read(dev_priv, 0x218C, SBI_MPHY); - tmp &= ~0xFF; - tmp |= 0x1C; - intel_sbi_write(dev_priv, 0x218C, tmp, SBI_MPHY); - - tmp = intel_sbi_read(dev_priv, 0x2098, SBI_MPHY); - tmp &= ~(0xFF << 16); - tmp |= (0x1C << 16); - intel_sbi_write(dev_priv, 0x2098, tmp, SBI_MPHY); - - tmp = intel_sbi_read(dev_priv, 0x2198, SBI_MPHY); - tmp &= ~(0xFF << 16); - tmp |= (0x1C << 16); - intel_sbi_write(dev_priv, 0x2198, tmp, SBI_MPHY); - - tmp = intel_sbi_read(dev_priv, 0x20C4, SBI_MPHY); - tmp |= (1 << 27); - intel_sbi_write(dev_priv, 0x20C4, tmp, SBI_MPHY); - - tmp = intel_sbi_read(dev_priv, 0x21C4, SBI_MPHY); - tmp |= (1 << 27); - intel_sbi_write(dev_priv, 0x21C4, tmp, SBI_MPHY); - - tmp = intel_sbi_read(dev_priv, 0x20EC, SBI_MPHY); - tmp &= ~(0xF << 28); - tmp |= (4 << 28); - intel_sbi_write(dev_priv, 0x20EC, tmp, SBI_MPHY); - - tmp = intel_sbi_read(dev_priv, 0x21EC, SBI_MPHY); - tmp &= ~(0xF << 28); - tmp |= (4 << 28); - intel_sbi_write(dev_priv, 0x21EC, tmp, SBI_MPHY); -} - /* Implements 3 different sequences from BSpec chapter "Display iCLK * Programming" based on the parameters passed: * - Sequence to enable CLKOUT_DP @@ -5540,10 +4500,8 @@ static void lpt_enable_clkout_dp(struct drm_i915_private *dev_priv, tmp &= ~SBI_SSCCTL_PATHALT; intel_sbi_write(dev_priv, SBI_SSCCTL, tmp, SBI_ICLK); - if (with_fdi) { - lpt_reset_fdi_mphy(dev_priv); - lpt_program_fdi_mphy(dev_priv); - } + if (with_fdi) + lpt_fdi_program_mphy(dev_priv); } reg = HAS_PCH_LPT_LP(dev_priv) ? SBI_GEN0 : SBI_DBUFF0; @@ -5806,7 +4764,7 @@ static void ilk_set_pipeconf(const struct intel_crtc_state *crtc_state) intel_de_posting_read(dev_priv, PIPECONF(pipe)); } -static void hsw_set_pipeconf(const struct intel_crtc_state *crtc_state) +static void hsw_set_transconf(const struct intel_crtc_state *crtc_state) { struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); @@ -5870,9 +4828,7 @@ static void bdw_set_pipemisc(const struct intel_crtc_state *crtc_state) val |= PIPEMISC_YUV420_ENABLE | PIPEMISC_YUV420_MODE_FULL_BLEND; - if (DISPLAY_VER(dev_priv) >= 11 && - (crtc_state->active_planes & ~(icl_hdr_plane_mask() | - BIT(PLANE_CURSOR))) == 0) + if (DISPLAY_VER(dev_priv) >= 11 && is_hdr_mode(crtc_state)) val |= PIPEMISC_HDR_MODE_PRECISION; if (DISPLAY_VER(dev_priv) >= 12) @@ -6211,59 +5167,64 @@ out: return ret; } -static bool hsw_get_transcoder_state(struct intel_crtc *crtc, - struct intel_crtc_state *pipe_config, - struct intel_display_power_domain_set *power_domain_set) +static bool transcoder_ddi_func_is_enabled(struct drm_i915_private *dev_priv, + enum transcoder cpu_transcoder) { - struct drm_device *dev = crtc->base.dev; - struct drm_i915_private *dev_priv = to_i915(dev); - unsigned long panel_transcoder_mask = BIT(TRANSCODER_EDP); - unsigned long enabled_panel_transcoders = 0; - enum transcoder panel_transcoder; - u32 tmp; + enum intel_display_power_domain power_domain; + intel_wakeref_t wakeref; + u32 tmp = 0; - if (DISPLAY_VER(dev_priv) >= 11) - panel_transcoder_mask |= - BIT(TRANSCODER_DSI_0) | BIT(TRANSCODER_DSI_1); + power_domain = POWER_DOMAIN_TRANSCODER(cpu_transcoder); - /* - * The pipe->transcoder mapping is fixed with the exception of the eDP - * and DSI transcoders handled below. - */ - pipe_config->cpu_transcoder = (enum transcoder) crtc->pipe; + with_intel_display_power_if_enabled(dev_priv, power_domain, wakeref) + tmp = intel_de_read(dev_priv, TRANS_DDI_FUNC_CTL(cpu_transcoder)); + + return tmp & TRANS_DDI_FUNC_ENABLE; +} + +static u8 hsw_panel_transcoders(struct drm_i915_private *i915) +{ + u8 panel_transcoder_mask = BIT(TRANSCODER_EDP); + + if (DISPLAY_VER(i915) >= 11) + panel_transcoder_mask |= BIT(TRANSCODER_DSI_0) | BIT(TRANSCODER_DSI_1); + + return panel_transcoder_mask; +} + +static u8 hsw_enabled_transcoders(struct intel_crtc *crtc) +{ + struct drm_device *dev = crtc->base.dev; + struct drm_i915_private *dev_priv = to_i915(dev); + u8 panel_transcoder_mask = hsw_panel_transcoders(dev_priv); + enum transcoder cpu_transcoder; + u8 enabled_transcoders = 0; /* * XXX: Do intel_display_power_get_if_enabled before reading this (for * consistency and less surprising code; it's in always on power). */ - for_each_cpu_transcoder_masked(dev_priv, panel_transcoder, + for_each_cpu_transcoder_masked(dev_priv, cpu_transcoder, panel_transcoder_mask) { - bool force_thru = false; + enum intel_display_power_domain power_domain; + intel_wakeref_t wakeref; enum pipe trans_pipe; + u32 tmp = 0; - tmp = intel_de_read(dev_priv, - TRANS_DDI_FUNC_CTL(panel_transcoder)); - if (!(tmp & TRANS_DDI_FUNC_ENABLE)) - continue; + power_domain = POWER_DOMAIN_TRANSCODER(cpu_transcoder); + with_intel_display_power_if_enabled(dev_priv, power_domain, wakeref) + tmp = intel_de_read(dev_priv, TRANS_DDI_FUNC_CTL(cpu_transcoder)); - /* - * Log all enabled ones, only use the first one. - * - * FIXME: This won't work for two separate DSI displays. - */ - enabled_panel_transcoders |= BIT(panel_transcoder); - if (enabled_panel_transcoders != BIT(panel_transcoder)) + if (!(tmp & TRANS_DDI_FUNC_ENABLE)) continue; switch (tmp & TRANS_DDI_EDP_INPUT_MASK) { default: drm_WARN(dev, 1, "unknown pipe linked to transcoder %s\n", - transcoder_name(panel_transcoder)); + transcoder_name(cpu_transcoder)); fallthrough; case TRANS_DDI_EDP_INPUT_A_ONOFF: - force_thru = true; - fallthrough; case TRANS_DDI_EDP_INPUT_A_ON: trans_pipe = PIPE_A; break; @@ -6278,22 +5239,83 @@ static bool hsw_get_transcoder_state(struct intel_crtc *crtc, break; } - if (trans_pipe == crtc->pipe) { - pipe_config->cpu_transcoder = panel_transcoder; - pipe_config->pch_pfit.force_thru = force_thru; - } + if (trans_pipe == crtc->pipe) + enabled_transcoders |= BIT(cpu_transcoder); } + cpu_transcoder = (enum transcoder) crtc->pipe; + if (transcoder_ddi_func_is_enabled(dev_priv, cpu_transcoder)) + enabled_transcoders |= BIT(cpu_transcoder); + + return enabled_transcoders; +} + +static bool has_edp_transcoders(u8 enabled_transcoders) +{ + return enabled_transcoders & BIT(TRANSCODER_EDP); +} + +static bool has_dsi_transcoders(u8 enabled_transcoders) +{ + return enabled_transcoders & (BIT(TRANSCODER_DSI_0) | + BIT(TRANSCODER_DSI_1)); +} + +static bool has_pipe_transcoders(u8 enabled_transcoders) +{ + return enabled_transcoders & ~(BIT(TRANSCODER_EDP) | + BIT(TRANSCODER_DSI_0) | + BIT(TRANSCODER_DSI_1)); +} + +static void assert_enabled_transcoders(struct drm_i915_private *i915, + u8 enabled_transcoders) +{ + /* Only one type of transcoder please */ + drm_WARN_ON(&i915->drm, + has_edp_transcoders(enabled_transcoders) + + has_dsi_transcoders(enabled_transcoders) + + has_pipe_transcoders(enabled_transcoders) > 1); + + /* Only DSI transcoders can be ganged */ + drm_WARN_ON(&i915->drm, + !has_dsi_transcoders(enabled_transcoders) && + !is_power_of_2(enabled_transcoders)); +} + +static bool hsw_get_transcoder_state(struct intel_crtc *crtc, + struct intel_crtc_state *pipe_config, + struct intel_display_power_domain_set *power_domain_set) +{ + struct drm_device *dev = crtc->base.dev; + struct drm_i915_private *dev_priv = to_i915(dev); + unsigned long enabled_transcoders; + u32 tmp; + + enabled_transcoders = hsw_enabled_transcoders(crtc); + if (!enabled_transcoders) + return false; + + assert_enabled_transcoders(dev_priv, enabled_transcoders); + /* - * Valid combos: none, eDP, DSI0, DSI1, DSI0+DSI1 + * With the exception of DSI we should only ever have + * a single enabled transcoder. With DSI let's just + * pick the first one. */ - drm_WARN_ON(dev, (enabled_panel_transcoders & BIT(TRANSCODER_EDP)) && - enabled_panel_transcoders != BIT(TRANSCODER_EDP)); + pipe_config->cpu_transcoder = ffs(enabled_transcoders) - 1; if (!intel_display_power_get_in_set_if_enabled(dev_priv, power_domain_set, POWER_DOMAIN_TRANSCODER(pipe_config->cpu_transcoder))) return false; + if (hsw_panel_transcoders(dev_priv) & BIT(pipe_config->cpu_transcoder)) { + tmp = intel_de_read(dev_priv, TRANS_DDI_FUNC_CTL(pipe_config->cpu_transcoder)); + + if ((tmp & TRANS_DDI_EDP_INPUT_MASK) == TRANS_DDI_EDP_INPUT_A_ONOFF) + pipe_config->pch_pfit.force_thru = true; + } + tmp = intel_de_read(dev_priv, PIPECONF(pipe_config->cpu_transcoder)); return tmp & PIPECONF_ENABLE; @@ -6515,7 +5537,7 @@ static bool intel_crtc_get_pipe_config(struct intel_crtc_state *crtc_state) struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); struct drm_i915_private *i915 = to_i915(crtc->base.dev); - if (!i915->display.get_pipe_config(crtc, crtc_state)) + if (!i915->display->get_pipe_config(crtc, crtc_state)) return false; crtc_state->hw.active = true; @@ -6531,28 +5553,6 @@ static const struct drm_display_mode load_detect_mode = { 704, 832, 0, 480, 489, 491, 520, 0, DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC), }; -struct drm_framebuffer * -intel_framebuffer_create(struct drm_i915_gem_object *obj, - struct drm_mode_fb_cmd2 *mode_cmd) -{ - struct intel_framebuffer *intel_fb; - int ret; - - intel_fb = kzalloc(sizeof(*intel_fb), GFP_KERNEL); - if (!intel_fb) - return ERR_PTR(-ENOMEM); - - ret = intel_framebuffer_init(intel_fb, obj, mode_cmd); - if (ret) - goto err; - - return &intel_fb->base; - -err: - kfree(intel_fb); - return ERR_PTR(ret); -} - static int intel_modeset_disable_planes(struct drm_atomic_state *state, struct drm_crtc *crtc) { @@ -6780,7 +5780,6 @@ static void i9xx_crtc_clock_get(struct intel_crtc *crtc, { struct drm_device *dev = crtc->base.dev; struct drm_i915_private *dev_priv = to_i915(dev); - enum pipe pipe = crtc->pipe; u32 dpll = pipe_config->dpll_hw_state.dpll; u32 fp; struct dpll clock; @@ -6830,11 +5829,13 @@ static void i9xx_crtc_clock_get(struct intel_crtc *crtc, else port_clock = i9xx_calc_dpll_params(refclk, &clock); } else { - u32 lvds = IS_I830(dev_priv) ? 0 : intel_de_read(dev_priv, - LVDS); - bool is_lvds = (pipe == 1) && (lvds & LVDS_PORT_EN); + enum pipe lvds_pipe; + + if (IS_I85X(dev_priv) && + intel_lvds_port_enabled(dev_priv, LVDS, &lvds_pipe) && + lvds_pipe == crtc->pipe) { + u32 lvds = intel_de_read(dev_priv, LVDS); - if (is_lvds) { clock.p1 = ffs((dpll & DPLL_FPA01_P1_POST_DIV_MASK_I830_LVDS) >> DPLL_FPA01_P1_POST_DIV_SHIFT); @@ -6985,27 +5986,27 @@ static bool needs_scaling(const struct intel_plane_state *state) } int intel_plane_atomic_calc_changes(const struct intel_crtc_state *old_crtc_state, - struct intel_crtc_state *crtc_state, + struct intel_crtc_state *new_crtc_state, const struct intel_plane_state *old_plane_state, - struct intel_plane_state *plane_state) + struct intel_plane_state *new_plane_state) { - struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); - struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane); + struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc); + struct intel_plane *plane = to_intel_plane(new_plane_state->uapi.plane); struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); - bool mode_changed = intel_crtc_needs_modeset(crtc_state); + bool mode_changed = intel_crtc_needs_modeset(new_crtc_state); bool was_crtc_enabled = old_crtc_state->hw.active; - bool is_crtc_enabled = crtc_state->hw.active; + bool is_crtc_enabled = new_crtc_state->hw.active; bool turn_off, turn_on, visible, was_visible; int ret; if (DISPLAY_VER(dev_priv) >= 9 && plane->id != PLANE_CURSOR) { - ret = skl_update_scaler_plane(crtc_state, plane_state); + ret = skl_update_scaler_plane(new_crtc_state, new_plane_state); if (ret) return ret; } was_visible = old_plane_state->uapi.visible; - visible = plane_state->uapi.visible; + visible = new_plane_state->uapi.visible; if (!was_crtc_enabled && drm_WARN_ON(&dev_priv->drm, was_visible)) was_visible = false; @@ -7021,7 +6022,7 @@ int intel_plane_atomic_calc_changes(const struct intel_crtc_state *old_crtc_stat * only combine the results from all planes in the current place? */ if (!is_crtc_enabled) { - intel_plane_set_invisible(crtc_state, plane_state); + intel_plane_set_invisible(new_crtc_state, new_plane_state); visible = false; } @@ -7040,28 +6041,28 @@ int intel_plane_atomic_calc_changes(const struct intel_crtc_state *old_crtc_stat if (turn_on) { if (DISPLAY_VER(dev_priv) < 5 && !IS_G4X(dev_priv)) - crtc_state->update_wm_pre = true; + new_crtc_state->update_wm_pre = true; /* must disable cxsr around plane enable/disable */ if (plane->id != PLANE_CURSOR) - crtc_state->disable_cxsr = true; + new_crtc_state->disable_cxsr = true; } else if (turn_off) { if (DISPLAY_VER(dev_priv) < 5 && !IS_G4X(dev_priv)) - crtc_state->update_wm_post = true; + new_crtc_state->update_wm_post = true; /* must disable cxsr around plane enable/disable */ if (plane->id != PLANE_CURSOR) - crtc_state->disable_cxsr = true; - } else if (intel_wm_need_update(old_plane_state, plane_state)) { + new_crtc_state->disable_cxsr = true; + } else if (intel_wm_need_update(old_plane_state, new_plane_state)) { if (DISPLAY_VER(dev_priv) < 5 && !IS_G4X(dev_priv)) { /* FIXME bollocks */ - crtc_state->update_wm_pre = true; - crtc_state->update_wm_post = true; + new_crtc_state->update_wm_pre = true; + new_crtc_state->update_wm_post = true; } } if (visible || was_visible) - crtc_state->fb_bits |= plane->frontbuffer_bit; + new_crtc_state->fb_bits |= plane->frontbuffer_bit; /* * ILK/SNB DVSACNTR/Sprite Enable @@ -7100,8 +6101,8 @@ int intel_plane_atomic_calc_changes(const struct intel_crtc_state *old_crtc_stat (IS_IRONLAKE(dev_priv) || IS_SANDYBRIDGE(dev_priv) || IS_IVYBRIDGE(dev_priv)) && (turn_on || (!needs_scaling(old_plane_state) && - needs_scaling(plane_state)))) - crtc_state->disable_lp_wm = true; + needs_scaling(new_plane_state)))) + new_crtc_state->disable_lp_wm = true; return 0; } @@ -7363,10 +6364,10 @@ static int intel_crtc_atomic_check(struct intel_atomic_state *state, crtc_state->update_wm_post = true; if (mode_changed && crtc_state->hw.enable && - dev_priv->display.crtc_compute_clock && + dev_priv->dpll_funcs && !crtc_state->bigjoiner_slave && !drm_WARN_ON(&dev_priv->drm, crtc_state->shared_dpll)) { - ret = dev_priv->display.crtc_compute_clock(crtc, crtc_state); + ret = dev_priv->dpll_funcs->crtc_compute_clock(crtc_state); if (ret) return ret; } @@ -7385,32 +6386,23 @@ static int intel_crtc_atomic_check(struct intel_atomic_state *state, return ret; } - if (dev_priv->display.compute_pipe_wm) { - ret = dev_priv->display.compute_pipe_wm(state, crtc); - if (ret) { - drm_dbg_kms(&dev_priv->drm, - "Target pipe watermarks are invalid\n"); - return ret; - } - + ret = intel_compute_pipe_wm(state, crtc); + if (ret) { + drm_dbg_kms(&dev_priv->drm, + "Target pipe watermarks are invalid\n"); + return ret; } - if (dev_priv->display.compute_intermediate_wm) { - if (drm_WARN_ON(&dev_priv->drm, - !dev_priv->display.compute_pipe_wm)) - return 0; - - /* - * Calculate 'intermediate' watermarks that satisfy both the - * old state and the new state. We can program these - * immediately. - */ - ret = dev_priv->display.compute_intermediate_wm(state, crtc); - if (ret) { - drm_dbg_kms(&dev_priv->drm, - "No valid intermediate pipe watermarks are possible\n"); - return ret; - } + /* + * Calculate 'intermediate' watermarks that satisfy both the + * old state and the new state. We can program these + * immediately. + */ + ret = intel_compute_intermediate_wm(state, crtc); + if (ret) { + drm_dbg_kms(&dev_priv->drm, + "No valid intermediate pipe watermarks are possible\n"); + return ret; } if (DISPLAY_VER(dev_priv) >= 9) { @@ -7439,11 +6431,9 @@ static int intel_crtc_atomic_check(struct intel_atomic_state *state, } - if (!mode_changed) { - ret = intel_psr2_sel_fetch_update(state, crtc); - if (ret) - return ret; - } + ret = intel_psr2_sel_fetch_update(state, crtc); + if (ret) + return ret; return 0; } @@ -8153,11 +7143,10 @@ encoder_retry: ret = encoder->compute_config(encoder, pipe_config, connector_state); + if (ret == -EDEADLK) + return ret; if (ret < 0) { - if (ret != -EDEADLK) - drm_dbg_kms(&i915->drm, - "Encoder config failure: %d\n", - ret); + drm_dbg_kms(&i915->drm, "Encoder config failure: %d\n", ret); return ret; } } @@ -8171,12 +7160,7 @@ encoder_retry: ret = intel_crtc_compute_config(to_intel_crtc(crtc), pipe_config); if (ret == -EDEADLK) return ret; - if (ret < 0) { - drm_dbg_kms(&i915->drm, "CRTC fixup failed\n"); - return ret; - } - - if (ret == I915_DISPLAY_CONFIG_RETRY) { + if (ret == -EAGAIN) { if (drm_WARN(&i915->drm, !retry, "loop in pipe configuration computation\n")) return -EINVAL; @@ -8185,6 +7169,10 @@ encoder_retry: retry = false; goto encoder_retry; } + if (ret < 0) { + drm_dbg_kms(&i915->drm, "CRTC config failure: %d\n", ret); + return ret; + } /* Dithering seems to not pass-through bits correctly when it should, so * only enable it on 6bpc panels and when its not a compliance @@ -8720,10 +7708,12 @@ intel_pipe_config_compare(const struct intel_crtc_state *current_config, if (bp_gamma) PIPE_CONF_CHECK_COLOR_LUT(gamma_mode, hw.gamma_lut, bp_gamma); - PIPE_CONF_CHECK_BOOL(has_psr); - PIPE_CONF_CHECK_BOOL(has_psr2); - PIPE_CONF_CHECK_BOOL(enable_psr2_sel_fetch); - PIPE_CONF_CHECK_I(dc3co_exitline); + if (current_config->active_planes) { + PIPE_CONF_CHECK_BOOL(has_psr); + PIPE_CONF_CHECK_BOOL(has_psr2); + PIPE_CONF_CHECK_BOOL(enable_psr2_sel_fetch); + PIPE_CONF_CHECK_I(dc3co_exitline); + } } PIPE_CONF_CHECK_BOOL(double_wide); @@ -8780,7 +7770,7 @@ intel_pipe_config_compare(const struct intel_crtc_state *current_config, PIPE_CONF_CHECK_I(min_voltage_level); } - if (fastset && (current_config->has_psr || pipe_config->has_psr)) + if (current_config->has_psr || pipe_config->has_psr) PIPE_CONF_CHECK_X_WITH_MASK(infoframes.enable, ~intel_hdmi_infoframe_enable(DP_SDP_VSC)); else @@ -9402,7 +8392,7 @@ static void intel_modeset_clear_plls(struct intel_atomic_state *state) struct intel_crtc *crtc; int i; - if (!dev_priv->display.crtc_compute_clock) + if (!dev_priv->dpll_funcs) return; for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) { @@ -9503,23 +8493,6 @@ static int intel_modeset_checks(struct intel_atomic_state *state) return 0; } -/* - * Handle calculation of various watermark data at the end of the atomic check - * phase. The code here should be run after the per-crtc and per-plane 'check' - * handlers to ensure that all derived state has been updated. - */ -static int calc_watermark_data(struct intel_atomic_state *state) -{ - struct drm_device *dev = state->base.dev; - struct drm_i915_private *dev_priv = to_i915(dev); - - /* Is there platform-specific watermark information to calculate? */ - if (dev_priv->display.compute_global_watermarks) - return dev_priv->display.compute_global_watermarks(state); - - return 0; -} - static void intel_crtc_check_fastset(const struct intel_crtc_state *old_crtc_state, struct intel_crtc_state *new_crtc_state) { @@ -9627,13 +8600,28 @@ static int intel_bigjoiner_add_affected_planes(struct intel_atomic_state *state) return 0; } +static bool bo_has_valid_encryption(struct drm_i915_gem_object *obj) +{ + struct drm_i915_private *i915 = to_i915(obj->base.dev); + + return intel_pxp_key_check(&i915->gt.pxp, obj, false) == 0; +} + +static bool pxp_is_borked(struct drm_i915_gem_object *obj) +{ + return i915_gem_object_is_protected(obj) && !bo_has_valid_encryption(obj); +} + static int intel_atomic_check_planes(struct intel_atomic_state *state) { struct drm_i915_private *dev_priv = to_i915(state->base.dev); struct intel_crtc_state *old_crtc_state, *new_crtc_state; struct intel_plane_state *plane_state; struct intel_plane *plane; + struct intel_plane_state *new_plane_state; + struct intel_plane_state *old_plane_state; struct intel_crtc *crtc; + const struct drm_framebuffer *fb; int i, ret; ret = icl_add_linked_planes(state); @@ -9681,6 +8669,19 @@ static int intel_atomic_check_planes(struct intel_atomic_state *state) return ret; } + for_each_new_intel_plane_in_state(state, plane, plane_state, i) { + new_plane_state = intel_atomic_get_new_plane_state(state, plane); + old_plane_state = intel_atomic_get_old_plane_state(state, plane); + fb = new_plane_state->hw.fb; + if (fb) { + new_plane_state->decrypt = bo_has_valid_encryption(intel_fb_obj(fb)); + new_plane_state->force_black = pxp_is_borked(intel_fb_obj(fb)); + } else { + new_plane_state->decrypt = old_plane_state->decrypt; + new_plane_state->force_black = old_plane_state->force_black; + } + } + return 0; } @@ -9715,7 +8716,7 @@ static int intel_atomic_check_cdclk(struct intel_atomic_state *state, old_cdclk_state->force_min_cdclk != new_cdclk_state->force_min_cdclk) *need_cdclk_calc = true; - ret = dev_priv->display.bw_calc_min_cdclk(state); + ret = intel_cdclk_bw_calc_min_cdclk(state); if (ret) return ret; @@ -9967,6 +8968,10 @@ static int intel_atomic_check_async(struct intel_atomic_state *state) drm_dbg_kms(&i915->drm, "Color range cannot be changed in async flip\n"); return -EINVAL; } + + /* plane decryption is allow to change only in synchronous flips */ + if (old_plane_state->decrypt != new_plane_state->decrypt) + return -EINVAL; } return 0; @@ -10166,7 +9171,7 @@ static int intel_atomic_check(struct drm_device *dev, goto fail; intel_fbc_choose_crtc(dev_priv, state); - ret = calc_watermark_data(state); + ret = intel_compute_global_watermarks(state); if (ret) goto fail; @@ -10336,12 +9341,11 @@ static void commit_pipe_pre_planes(struct intel_atomic_state *state, if (new_crtc_state->update_pipe) intel_pipe_fastset(old_crtc_state, new_crtc_state); - - intel_psr2_program_trans_man_trk_ctl(new_crtc_state); } - if (dev_priv->display.atomic_update_watermarks) - dev_priv->display.atomic_update_watermarks(state, crtc); + intel_psr2_program_trans_man_trk_ctl(new_crtc_state); + + intel_atomic_update_watermarks(state, crtc); } static void commit_pipe_post_planes(struct intel_atomic_state *state, @@ -10373,7 +9377,7 @@ static void intel_enable_crtc(struct intel_atomic_state *state, intel_crtc_update_active_timings(new_crtc_state); - dev_priv->display.crtc_enable(state, crtc); + dev_priv->display->crtc_enable(state, crtc); if (new_crtc_state->bigjoiner_slave) return; @@ -10404,10 +9408,7 @@ static void intel_update_crtc(struct intel_atomic_state *state, intel_encoders_update_pipe(state, crtc); } - if (new_crtc_state->update_pipe && !new_crtc_state->enable_fbc) - intel_fbc_disable(crtc); - else - intel_fbc_enable(state, crtc); + intel_fbc_update(state, crtc); /* Perform vblank evasion around commit operation */ intel_pipe_update_start(new_crtc_state); @@ -10464,16 +9465,15 @@ static void intel_old_crtc_state_disables(struct intel_atomic_state *state, */ intel_crtc_disable_pipe_crc(crtc); - dev_priv->display.crtc_disable(state, crtc); + dev_priv->display->crtc_disable(state, crtc); crtc->active = false; intel_fbc_disable(crtc); intel_disable_shared_dpll(old_crtc_state); /* FIXME unify this for all platforms */ if (!new_crtc_state->hw.active && - !HAS_GMCH(dev_priv) && - dev_priv->display.initial_watermarks) - dev_priv->display.initial_watermarks(state, crtc); + !HAS_GMCH(dev_priv)) + intel_initial_watermarks(state, crtc); } static void intel_commit_modeset_disables(struct intel_atomic_state *state) @@ -10837,6 +9837,7 @@ static void intel_atomic_commit_tail(struct intel_atomic_state *state) intel_encoders_update_prepare(state); intel_dbuf_pre_plane_update(state); + intel_psr_pre_plane_update(state); for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) { if (new_crtc_state->uapi.async_flip) @@ -10844,7 +9845,7 @@ static void intel_atomic_commit_tail(struct intel_atomic_state *state) } /* Now enable the clocks, plane, pipe, and connectors that we set up. */ - dev_priv->display.commit_modeset_enables(state); + dev_priv->display->commit_modeset_enables(state); if (state->modeset) { intel_encoders_update_complete(state); @@ -10895,11 +9896,11 @@ static void intel_atomic_commit_tail(struct intel_atomic_state *state) if (DISPLAY_VER(dev_priv) == 2 && planes_enabling(old_crtc_state, new_crtc_state)) intel_set_cpu_fifo_underrun_reporting(dev_priv, crtc->pipe, true); - if (dev_priv->display.optimize_watermarks) - dev_priv->display.optimize_watermarks(state, crtc); + intel_optimize_watermarks(state, crtc); } intel_dbuf_post_plane_update(state); + intel_psr_post_plane_update(state); for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) { intel_post_plane_update(state, crtc); @@ -11088,280 +10089,6 @@ static int intel_atomic_commit(struct drm_device *dev, return 0; } -struct wait_rps_boost { - struct wait_queue_entry wait; - - struct drm_crtc *crtc; - struct i915_request *request; -}; - -static int do_rps_boost(struct wait_queue_entry *_wait, - unsigned mode, int sync, void *key) -{ - struct wait_rps_boost *wait = container_of(_wait, typeof(*wait), wait); - struct i915_request *rq = wait->request; - - /* - * If we missed the vblank, but the request is already running it - * is reasonable to assume that it will complete before the next - * vblank without our intervention, so leave RPS alone. - */ - if (!i915_request_started(rq)) - intel_rps_boost(rq); - i915_request_put(rq); - - drm_crtc_vblank_put(wait->crtc); - - list_del(&wait->wait.entry); - kfree(wait); - return 1; -} - -static void add_rps_boost_after_vblank(struct drm_crtc *crtc, - struct dma_fence *fence) -{ - struct wait_rps_boost *wait; - - if (!dma_fence_is_i915(fence)) - return; - - if (DISPLAY_VER(to_i915(crtc->dev)) < 6) - return; - - if (drm_crtc_vblank_get(crtc)) - return; - - wait = kmalloc(sizeof(*wait), GFP_KERNEL); - if (!wait) { - drm_crtc_vblank_put(crtc); - return; - } - - wait->request = to_request(dma_fence_get(fence)); - wait->crtc = crtc; - - wait->wait.func = do_rps_boost; - wait->wait.flags = 0; - - add_wait_queue(drm_crtc_vblank_waitqueue(crtc), &wait->wait); -} - -int intel_plane_pin_fb(struct intel_plane_state *plane_state) -{ - struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane); - struct drm_i915_private *dev_priv = to_i915(plane->base.dev); - struct drm_framebuffer *fb = plane_state->hw.fb; - struct i915_vma *vma; - bool phys_cursor = - plane->id == PLANE_CURSOR && - INTEL_INFO(dev_priv)->display.cursor_needs_physical; - - if (!intel_fb_uses_dpt(fb)) { - vma = intel_pin_and_fence_fb_obj(fb, phys_cursor, - &plane_state->view.gtt, - intel_plane_uses_fence(plane_state), - &plane_state->flags); - if (IS_ERR(vma)) - return PTR_ERR(vma); - - plane_state->ggtt_vma = vma; - } else { - struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb); - - vma = intel_dpt_pin(intel_fb->dpt_vm); - if (IS_ERR(vma)) - return PTR_ERR(vma); - - plane_state->ggtt_vma = vma; - - vma = intel_pin_fb_obj_dpt(fb, &plane_state->view.gtt, false, - &plane_state->flags, intel_fb->dpt_vm); - if (IS_ERR(vma)) { - intel_dpt_unpin(intel_fb->dpt_vm); - plane_state->ggtt_vma = NULL; - return PTR_ERR(vma); - } - - plane_state->dpt_vma = vma; - - WARN_ON(plane_state->ggtt_vma == plane_state->dpt_vma); - } - - return 0; -} - -void intel_plane_unpin_fb(struct intel_plane_state *old_plane_state) -{ - struct drm_framebuffer *fb = old_plane_state->hw.fb; - struct i915_vma *vma; - - if (!intel_fb_uses_dpt(fb)) { - vma = fetch_and_zero(&old_plane_state->ggtt_vma); - if (vma) - intel_unpin_fb_vma(vma, old_plane_state->flags); - } else { - struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb); - - vma = fetch_and_zero(&old_plane_state->dpt_vma); - if (vma) - intel_unpin_fb_vma(vma, old_plane_state->flags); - - vma = fetch_and_zero(&old_plane_state->ggtt_vma); - if (vma) - intel_dpt_unpin(intel_fb->dpt_vm); - } -} - -/** - * intel_prepare_plane_fb - Prepare fb for usage on plane - * @_plane: drm plane to prepare for - * @_new_plane_state: the plane state being prepared - * - * Prepares a framebuffer for usage on a display plane. Generally this - * involves pinning the underlying object and updating the frontbuffer tracking - * bits. Some older platforms need special physical address handling for - * cursor planes. - * - * Returns 0 on success, negative error code on failure. - */ -int -intel_prepare_plane_fb(struct drm_plane *_plane, - struct drm_plane_state *_new_plane_state) -{ - struct i915_sched_attr attr = { .priority = I915_PRIORITY_DISPLAY }; - struct intel_plane *plane = to_intel_plane(_plane); - struct intel_plane_state *new_plane_state = - to_intel_plane_state(_new_plane_state); - struct intel_atomic_state *state = - to_intel_atomic_state(new_plane_state->uapi.state); - struct drm_i915_private *dev_priv = to_i915(plane->base.dev); - const struct intel_plane_state *old_plane_state = - intel_atomic_get_old_plane_state(state, plane); - struct drm_i915_gem_object *obj = intel_fb_obj(new_plane_state->hw.fb); - struct drm_i915_gem_object *old_obj = intel_fb_obj(old_plane_state->hw.fb); - int ret; - - if (old_obj) { - const struct intel_crtc_state *crtc_state = - intel_atomic_get_new_crtc_state(state, - to_intel_crtc(old_plane_state->hw.crtc)); - - /* Big Hammer, we also need to ensure that any pending - * MI_WAIT_FOR_EVENT inside a user batch buffer on the - * current scanout is retired before unpinning the old - * framebuffer. Note that we rely on userspace rendering - * into the buffer attached to the pipe they are waiting - * on. If not, userspace generates a GPU hang with IPEHR - * point to the MI_WAIT_FOR_EVENT. - * - * This should only fail upon a hung GPU, in which case we - * can safely continue. - */ - if (intel_crtc_needs_modeset(crtc_state)) { - ret = i915_sw_fence_await_reservation(&state->commit_ready, - old_obj->base.resv, NULL, - false, 0, - GFP_KERNEL); - if (ret < 0) - return ret; - } - } - - if (new_plane_state->uapi.fence) { /* explicit fencing */ - i915_gem_fence_wait_priority(new_plane_state->uapi.fence, - &attr); - ret = i915_sw_fence_await_dma_fence(&state->commit_ready, - new_plane_state->uapi.fence, - i915_fence_timeout(dev_priv), - GFP_KERNEL); - if (ret < 0) - return ret; - } - - if (!obj) - return 0; - - - ret = intel_plane_pin_fb(new_plane_state); - if (ret) - return ret; - - i915_gem_object_wait_priority(obj, 0, &attr); - i915_gem_object_flush_frontbuffer(obj, ORIGIN_DIRTYFB); - - if (!new_plane_state->uapi.fence) { /* implicit fencing */ - struct dma_fence *fence; - - ret = i915_sw_fence_await_reservation(&state->commit_ready, - obj->base.resv, NULL, - false, - i915_fence_timeout(dev_priv), - GFP_KERNEL); - if (ret < 0) - goto unpin_fb; - - fence = dma_resv_get_excl_unlocked(obj->base.resv); - if (fence) { - add_rps_boost_after_vblank(new_plane_state->hw.crtc, - fence); - dma_fence_put(fence); - } - } else { - add_rps_boost_after_vblank(new_plane_state->hw.crtc, - new_plane_state->uapi.fence); - } - - /* - * We declare pageflips to be interactive and so merit a small bias - * towards upclocking to deliver the frame on time. By only changing - * the RPS thresholds to sample more regularly and aim for higher - * clocks we can hopefully deliver low power workloads (like kodi) - * that are not quite steady state without resorting to forcing - * maximum clocks following a vblank miss (see do_rps_boost()). - */ - if (!state->rps_interactive) { - intel_rps_mark_interactive(&dev_priv->gt.rps, true); - state->rps_interactive = true; - } - - return 0; - -unpin_fb: - intel_plane_unpin_fb(new_plane_state); - - return ret; -} - -/** - * intel_cleanup_plane_fb - Cleans up an fb after plane use - * @plane: drm plane to clean up for - * @_old_plane_state: the state from the previous modeset - * - * Cleans up a framebuffer that has just been removed from a plane. - */ -void -intel_cleanup_plane_fb(struct drm_plane *plane, - struct drm_plane_state *_old_plane_state) -{ - struct intel_plane_state *old_plane_state = - to_intel_plane_state(_old_plane_state); - struct intel_atomic_state *state = - to_intel_atomic_state(old_plane_state->uapi.state); - struct drm_i915_private *dev_priv = to_i915(plane->dev); - struct drm_i915_gem_object *obj = intel_fb_obj(old_plane_state->hw.fb); - - if (!obj) - return; - - if (state->rps_interactive) { - intel_rps_mark_interactive(&dev_priv->gt.rps, false); - state->rps_interactive = false; - } - - /* Should only be called after a successful intel_prepare_plane_fb()! */ - intel_plane_unpin_fb(old_plane_state); -} - /** * intel_plane_destroy - destroy a plane * @plane: plane to destroy @@ -11491,6 +10218,7 @@ static void intel_setup_outputs(struct drm_i915_private *dev_priv) intel_ddi_init(dev_priv, PORT_TC2); intel_ddi_init(dev_priv, PORT_TC3); intel_ddi_init(dev_priv, PORT_TC4); + icl_dsi_init(dev_priv); } else if (IS_ALDERLAKE_S(dev_priv)) { intel_ddi_init(dev_priv, PORT_A); intel_ddi_init(dev_priv, PORT_TC1); @@ -11708,249 +10436,6 @@ static void intel_setup_outputs(struct drm_i915_private *dev_priv) drm_helper_move_panel_connectors_to_head(&dev_priv->drm); } -static void intel_user_framebuffer_destroy(struct drm_framebuffer *fb) -{ - struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb); - - drm_framebuffer_cleanup(fb); - - if (intel_fb_uses_dpt(fb)) - intel_dpt_destroy(intel_fb->dpt_vm); - - intel_frontbuffer_put(intel_fb->frontbuffer); - - kfree(intel_fb); -} - -static int intel_user_framebuffer_create_handle(struct drm_framebuffer *fb, - struct drm_file *file, - unsigned int *handle) -{ - struct drm_i915_gem_object *obj = intel_fb_obj(fb); - struct drm_i915_private *i915 = to_i915(obj->base.dev); - - if (i915_gem_object_is_userptr(obj)) { - drm_dbg(&i915->drm, - "attempting to use a userptr for a framebuffer, denied\n"); - return -EINVAL; - } - - return drm_gem_handle_create(file, &obj->base, handle); -} - -static int intel_user_framebuffer_dirty(struct drm_framebuffer *fb, - struct drm_file *file, - unsigned flags, unsigned color, - struct drm_clip_rect *clips, - unsigned num_clips) -{ - struct drm_i915_gem_object *obj = intel_fb_obj(fb); - - i915_gem_object_flush_if_display(obj); - intel_frontbuffer_flush(to_intel_frontbuffer(fb), ORIGIN_DIRTYFB); - - return 0; -} - -static const struct drm_framebuffer_funcs intel_fb_funcs = { - .destroy = intel_user_framebuffer_destroy, - .create_handle = intel_user_framebuffer_create_handle, - .dirty = intel_user_framebuffer_dirty, -}; - -static int intel_framebuffer_init(struct intel_framebuffer *intel_fb, - struct drm_i915_gem_object *obj, - struct drm_mode_fb_cmd2 *mode_cmd) -{ - struct drm_i915_private *dev_priv = to_i915(obj->base.dev); - struct drm_framebuffer *fb = &intel_fb->base; - u32 max_stride; - unsigned int tiling, stride; - int ret = -EINVAL; - int i; - - intel_fb->frontbuffer = intel_frontbuffer_get(obj); - if (!intel_fb->frontbuffer) - return -ENOMEM; - - i915_gem_object_lock(obj, NULL); - tiling = i915_gem_object_get_tiling(obj); - stride = i915_gem_object_get_stride(obj); - i915_gem_object_unlock(obj); - - if (mode_cmd->flags & DRM_MODE_FB_MODIFIERS) { - /* - * If there's a fence, enforce that - * the fb modifier and tiling mode match. - */ - if (tiling != I915_TILING_NONE && - tiling != intel_fb_modifier_to_tiling(mode_cmd->modifier[0])) { - drm_dbg_kms(&dev_priv->drm, - "tiling_mode doesn't match fb modifier\n"); - goto err; - } - } else { - if (tiling == I915_TILING_X) { - mode_cmd->modifier[0] = I915_FORMAT_MOD_X_TILED; - } else if (tiling == I915_TILING_Y) { - drm_dbg_kms(&dev_priv->drm, - "No Y tiling for legacy addfb\n"); - goto err; - } - } - - if (!drm_any_plane_has_format(&dev_priv->drm, - mode_cmd->pixel_format, - mode_cmd->modifier[0])) { - drm_dbg_kms(&dev_priv->drm, - "unsupported pixel format %p4cc / modifier 0x%llx\n", - &mode_cmd->pixel_format, mode_cmd->modifier[0]); - goto err; - } - - /* - * gen2/3 display engine uses the fence if present, - * so the tiling mode must match the fb modifier exactly. - */ - if (DISPLAY_VER(dev_priv) < 4 && - tiling != intel_fb_modifier_to_tiling(mode_cmd->modifier[0])) { - drm_dbg_kms(&dev_priv->drm, - "tiling_mode must match fb modifier exactly on gen2/3\n"); - goto err; - } - - max_stride = intel_fb_max_stride(dev_priv, mode_cmd->pixel_format, - mode_cmd->modifier[0]); - if (mode_cmd->pitches[0] > max_stride) { - drm_dbg_kms(&dev_priv->drm, - "%s pitch (%u) must be at most %d\n", - mode_cmd->modifier[0] != DRM_FORMAT_MOD_LINEAR ? - "tiled" : "linear", - mode_cmd->pitches[0], max_stride); - goto err; - } - - /* - * If there's a fence, enforce that - * the fb pitch and fence stride match. - */ - if (tiling != I915_TILING_NONE && mode_cmd->pitches[0] != stride) { - drm_dbg_kms(&dev_priv->drm, - "pitch (%d) must match tiling stride (%d)\n", - mode_cmd->pitches[0], stride); - goto err; - } - - /* FIXME need to adjust LINOFF/TILEOFF accordingly. */ - if (mode_cmd->offsets[0] != 0) { - drm_dbg_kms(&dev_priv->drm, - "plane 0 offset (0x%08x) must be 0\n", - mode_cmd->offsets[0]); - goto err; - } - - drm_helper_mode_fill_fb_struct(&dev_priv->drm, fb, mode_cmd); - - for (i = 0; i < fb->format->num_planes; i++) { - u32 stride_alignment; - - if (mode_cmd->handles[i] != mode_cmd->handles[0]) { - drm_dbg_kms(&dev_priv->drm, "bad plane %d handle\n", - i); - goto err; - } - - stride_alignment = intel_fb_stride_alignment(fb, i); - if (fb->pitches[i] & (stride_alignment - 1)) { - drm_dbg_kms(&dev_priv->drm, - "plane %d pitch (%d) must be at least %u byte aligned\n", - i, fb->pitches[i], stride_alignment); - goto err; - } - - if (is_gen12_ccs_plane(fb, i) && !is_gen12_ccs_cc_plane(fb, i)) { - int ccs_aux_stride = gen12_ccs_aux_stride(fb, i); - - if (fb->pitches[i] != ccs_aux_stride) { - drm_dbg_kms(&dev_priv->drm, - "ccs aux plane %d pitch (%d) must be %d\n", - i, - fb->pitches[i], ccs_aux_stride); - goto err; - } - } - - /* TODO: Add POT stride remapping support for CCS formats as well. */ - if (IS_ALDERLAKE_P(dev_priv) && - mode_cmd->modifier[i] != DRM_FORMAT_MOD_LINEAR && - !intel_fb_needs_pot_stride_remap(intel_fb) && - !is_power_of_2(mode_cmd->pitches[i])) { - drm_dbg_kms(&dev_priv->drm, - "plane %d pitch (%d) must be power of two for tiled buffers\n", - i, mode_cmd->pitches[i]); - goto err; - } - - fb->obj[i] = &obj->base; - } - - ret = intel_fill_fb_info(dev_priv, intel_fb); - if (ret) - goto err; - - if (intel_fb_uses_dpt(fb)) { - struct i915_address_space *vm; - - vm = intel_dpt_create(intel_fb); - if (IS_ERR(vm)) { - ret = PTR_ERR(vm); - goto err; - } - - intel_fb->dpt_vm = vm; - } - - ret = drm_framebuffer_init(&dev_priv->drm, fb, &intel_fb_funcs); - if (ret) { - drm_err(&dev_priv->drm, "framebuffer init failed %d\n", ret); - goto err; - } - - return 0; - -err: - intel_frontbuffer_put(intel_fb->frontbuffer); - return ret; -} - -static struct drm_framebuffer * -intel_user_framebuffer_create(struct drm_device *dev, - struct drm_file *filp, - const struct drm_mode_fb_cmd2 *user_mode_cmd) -{ - struct drm_framebuffer *fb; - struct drm_i915_gem_object *obj; - struct drm_mode_fb_cmd2 mode_cmd = *user_mode_cmd; - struct drm_i915_private *i915; - - obj = i915_gem_object_lookup(filp, mode_cmd.handles[0]); - if (!obj) - return ERR_PTR(-ENOENT); - - /* object is backed with LMEM for discrete */ - i915 = to_i915(obj->base.dev); - if (HAS_LMEM(i915) && !i915_gem_object_can_migrate(obj, INTEL_REGION_LMEM)) { - /* object is "remote", not in local memory */ - i915_gem_object_put(obj); - return ERR_PTR(-EREMOTE); - } - - fb = intel_framebuffer_create(obj, &mode_cmd); - i915_gem_object_put(obj); - - return fb; -} - static enum drm_mode_status intel_mode_valid(struct drm_device *dev, const struct drm_display_mode *mode) @@ -12039,6 +10524,14 @@ intel_mode_valid(struct drm_device *dev, return MODE_V_ILLEGAL; } + /* + * Cantiga+ cannot handle modes with a hsync front porch of 0. + * WaPruneModeWithIncorrectHsyncOffset:ctg,elk,ilk,snb,ivb,vlv,hsw. + */ + if ((DISPLAY_VER(dev_priv) > 4 || IS_G4X(dev_priv)) && + mode->hsync_start == mode->hdisplay) + return MODE_H_ILLEGAL; + return MODE_OK; } @@ -12090,6 +10583,46 @@ static const struct drm_mode_config_funcs intel_mode_funcs = { .atomic_state_free = intel_atomic_state_free, }; +static const struct drm_i915_display_funcs skl_display_funcs = { + .get_pipe_config = hsw_get_pipe_config, + .crtc_enable = hsw_crtc_enable, + .crtc_disable = hsw_crtc_disable, + .commit_modeset_enables = skl_commit_modeset_enables, + .get_initial_plane_config = skl_get_initial_plane_config, +}; + +static const struct drm_i915_display_funcs ddi_display_funcs = { + .get_pipe_config = hsw_get_pipe_config, + .crtc_enable = hsw_crtc_enable, + .crtc_disable = hsw_crtc_disable, + .commit_modeset_enables = intel_commit_modeset_enables, + .get_initial_plane_config = i9xx_get_initial_plane_config, +}; + +static const struct drm_i915_display_funcs pch_split_display_funcs = { + .get_pipe_config = ilk_get_pipe_config, + .crtc_enable = ilk_crtc_enable, + .crtc_disable = ilk_crtc_disable, + .commit_modeset_enables = intel_commit_modeset_enables, + .get_initial_plane_config = i9xx_get_initial_plane_config, +}; + +static const struct drm_i915_display_funcs vlv_display_funcs = { + .get_pipe_config = i9xx_get_pipe_config, + .crtc_enable = valleyview_crtc_enable, + .crtc_disable = i9xx_crtc_disable, + .commit_modeset_enables = intel_commit_modeset_enables, + .get_initial_plane_config = i9xx_get_initial_plane_config, +}; + +static const struct drm_i915_display_funcs i9xx_display_funcs = { + .get_pipe_config = i9xx_get_pipe_config, + .crtc_enable = i9xx_crtc_enable, + .crtc_disable = i9xx_crtc_disable, + .commit_modeset_enables = intel_commit_modeset_enables, + .get_initial_plane_config = i9xx_get_initial_plane_config, +}; + /** * intel_init_display_hooks - initialize the display modesetting hooks * @dev_priv: device private @@ -12105,38 +10638,19 @@ void intel_init_display_hooks(struct drm_i915_private *dev_priv) intel_dpll_init_clock_hook(dev_priv); if (DISPLAY_VER(dev_priv) >= 9) { - dev_priv->display.get_pipe_config = hsw_get_pipe_config; - dev_priv->display.crtc_enable = hsw_crtc_enable; - dev_priv->display.crtc_disable = hsw_crtc_disable; + dev_priv->display = &skl_display_funcs; } else if (HAS_DDI(dev_priv)) { - dev_priv->display.get_pipe_config = hsw_get_pipe_config; - dev_priv->display.crtc_enable = hsw_crtc_enable; - dev_priv->display.crtc_disable = hsw_crtc_disable; + dev_priv->display = &ddi_display_funcs; } else if (HAS_PCH_SPLIT(dev_priv)) { - dev_priv->display.get_pipe_config = ilk_get_pipe_config; - dev_priv->display.crtc_enable = ilk_crtc_enable; - dev_priv->display.crtc_disable = ilk_crtc_disable; + dev_priv->display = &pch_split_display_funcs; } else if (IS_CHERRYVIEW(dev_priv) || IS_VALLEYVIEW(dev_priv)) { - dev_priv->display.get_pipe_config = i9xx_get_pipe_config; - dev_priv->display.crtc_enable = valleyview_crtc_enable; - dev_priv->display.crtc_disable = i9xx_crtc_disable; + dev_priv->display = &vlv_display_funcs; } else { - dev_priv->display.get_pipe_config = i9xx_get_pipe_config; - dev_priv->display.crtc_enable = i9xx_crtc_enable; - dev_priv->display.crtc_disable = i9xx_crtc_disable; + dev_priv->display = &i9xx_display_funcs; } intel_fdi_init_hook(dev_priv); - - if (DISPLAY_VER(dev_priv) >= 9) { - dev_priv->display.commit_modeset_enables = skl_commit_modeset_enables; - dev_priv->display.get_initial_plane_config = skl_get_initial_plane_config; - } else { - dev_priv->display.commit_modeset_enables = intel_commit_modeset_enables; - dev_priv->display.get_initial_plane_config = i9xx_get_initial_plane_config; - } - } void intel_modeset_init_hw(struct drm_i915_private *i915) @@ -12206,7 +10720,7 @@ static void sanitize_watermarks(struct drm_i915_private *dev_priv) int i; /* Only supported on platforms that use atomic watermark design */ - if (!dev_priv->display.optimize_watermarks) + if (!dev_priv->wm_disp->optimize_watermarks) return; state = drm_atomic_state_alloc(&dev_priv->drm); @@ -12239,7 +10753,7 @@ retry: /* Write calculated watermark values back */ for_each_new_intel_crtc_in_state(intel_state, crtc, crtc_state, i) { crtc_state->wm.need_postvbl_update = true; - dev_priv->display.optimize_watermarks(intel_state, crtc); + intel_optimize_watermarks(intel_state, crtc); to_intel_crtc_state(crtc->base.state)->wm = crtc_state->wm; } @@ -12271,22 +10785,6 @@ fail: drm_modeset_acquire_fini(&ctx); } -static void intel_update_fdi_pll_freq(struct drm_i915_private *dev_priv) -{ - if (IS_IRONLAKE(dev_priv)) { - u32 fdi_pll_clk = - intel_de_read(dev_priv, FDI_PLL_BIOS_0) & FDI_PLL_FB_CLOCK_MASK; - - dev_priv->fdi_pll_freq = (fdi_pll_clk + 2) * 10000; - } else if (IS_SANDYBRIDGE(dev_priv) || IS_IVYBRIDGE(dev_priv)) { - dev_priv->fdi_pll_freq = 270000; - } else { - return; - } - - drm_dbg(&dev_priv->drm, "FDI PLL freq=%d\n", dev_priv->fdi_pll_freq); -} - static int intel_initial_commit(struct drm_device *dev) { struct drm_atomic_state *state = NULL; @@ -12381,7 +10879,7 @@ static void intel_mode_config_init(struct drm_i915_private *i915) mode_config->funcs = &intel_mode_funcs; - mode_config->async_page_flip = has_async_flips(i915); + mode_config->async_page_flip = HAS_ASYNC_FLIPS(i915); /* * Maximum framebuffer dimensions, chosen to match @@ -12420,22 +10918,6 @@ static void intel_mode_config_cleanup(struct drm_i915_private *i915) drm_mode_config_cleanup(&i915->drm); } -static void plane_config_fini(struct intel_initial_plane_config *plane_config) -{ - if (plane_config->fb) { - struct drm_framebuffer *fb = &plane_config->fb->base; - - /* We may only have the stub and not a full framebuffer */ - if (drm_framebuffer_read_refcount(fb)) - drm_framebuffer_put(fb); - else - kfree(fb); - } - - if (plane_config->vma) - i915_vma_put(plane_config->vma); -} - /* part #1: call before irq install */ int intel_modeset_init_noirq(struct drm_i915_private *i915) { @@ -12540,7 +11022,7 @@ int intel_modeset_init_nogem(struct drm_i915_private *i915) intel_plane_possible_crtcs_init(i915); intel_shared_dpll_init(dev); - intel_update_fdi_pll_freq(i915); + intel_fdi_pll_freq_update(i915); intel_update_czclk(i915); intel_modeset_init_hw(i915); @@ -12564,30 +11046,13 @@ int intel_modeset_init_nogem(struct drm_i915_private *i915) drm_modeset_lock_all(dev); intel_modeset_setup_hw_state(dev, dev->mode_config.acquire_ctx); + intel_acpi_assign_connector_fwnodes(i915); drm_modeset_unlock_all(dev); for_each_intel_crtc(dev, crtc) { - struct intel_initial_plane_config plane_config = {}; - if (!to_intel_crtc_state(crtc->base.state)->uapi.active) continue; - - /* - * Note that reserving the BIOS fb up front prevents us - * from stuffing other stolen allocations like the ring - * on top. This prevents some ugliness at boot time, and - * can even allow for smooth boot transitions if the BIOS - * fb is large enough for the active pipe configuration. - */ - i915->display.get_initial_plane_config(crtc, &plane_config); - - /* - * If the fb is shared between multiple heads, we'll - * just get the first one. - */ - intel_find_initial_plane_obj(crtc, &plane_config); - - plane_config_fini(&plane_config); + intel_crtc_initial_plane_config(crtc); } /* @@ -12869,13 +11334,8 @@ static void intel_sanitize_crtc(struct intel_crtc *crtc, intel_plane_disable_noatomic(crtc, plane); } - /* - * Disable any background color set by the BIOS, but enable the - * gamma and CSC to match how we program our planes. - */ - if (DISPLAY_VER(dev_priv) >= 9) - intel_de_write(dev_priv, SKL_BOTTOM_COLOR(crtc->pipe), - SKL_BOTTOM_COLOR_GAMMA_ENABLE | SKL_BOTTOM_COLOR_CSC_ENABLE); + /* Disable any background color/etc. set by the BIOS */ + intel_color_commit(crtc_state); } /* Adjust the state of the output pipe according to whether we @@ -13076,8 +11536,7 @@ static void intel_modeset_readout_hw_state(struct drm_device *dev) enableddisabled(crtc_state->hw.active)); } - dev_priv->active_pipes = cdclk_state->active_pipes = - dbuf_state->active_pipes = active_pipes; + cdclk_state->active_pipes = dbuf_state->active_pipes = active_pipes; readout_plane_state(dev_priv); diff --git a/drivers/gpu/drm/i915/display/intel_display.h b/drivers/gpu/drm/i915/display/intel_display.h index 284936f0ddab..0c76bf57f86b 100644 --- a/drivers/gpu/drm/i915/display/intel_display.h +++ b/drivers/gpu/drm/i915/display/intel_display.h @@ -270,6 +270,7 @@ enum tc_port { }; enum tc_port_mode { + TC_PORT_DISCONNECTED, TC_PORT_TBT_ALT, TC_PORT_DP_ALT, TC_PORT_LEGACY, @@ -531,8 +532,8 @@ enum phy intel_port_to_phy(struct drm_i915_private *i915, enum port port); bool is_trans_port_sync_mode(const struct intel_crtc_state *state); void intel_plane_destroy(struct drm_plane *plane); -void intel_enable_pipe(const struct intel_crtc_state *new_crtc_state); -void intel_disable_pipe(const struct intel_crtc_state *old_crtc_state); +void intel_enable_transcoder(const struct intel_crtc_state *new_crtc_state); +void intel_disable_transcoder(const struct intel_crtc_state *old_crtc_state); void i830_enable_pipe(struct drm_i915_private *dev_priv, enum pipe pipe); void i830_disable_pipe(struct drm_i915_private *dev_priv, enum pipe pipe); enum pipe intel_crtc_pch_transcoder(struct intel_crtc *crtc); @@ -548,8 +549,6 @@ void intel_init_display_hooks(struct drm_i915_private *dev_priv); unsigned int intel_fb_xy_to_linear(int x, int y, const struct intel_plane_state *state, int plane); -unsigned int intel_fb_align_height(const struct drm_framebuffer *fb, - int color_plane, unsigned int height); void intel_add_fb_offsets(int *x, int *y, const struct intel_plane_state *state, int plane); unsigned int intel_rotation_info_size(const struct intel_rotation_info *rot_info); @@ -577,19 +576,9 @@ int intel_get_load_detect_pipe(struct drm_connector *connector, void intel_release_load_detect_pipe(struct drm_connector *connector, struct intel_load_detect_pipe *old, struct drm_modeset_acquire_ctx *ctx); -struct i915_vma * -intel_pin_and_fence_fb_obj(struct drm_framebuffer *fb, bool phys_cursor, - const struct i915_ggtt_view *view, - bool uses_fence, - unsigned long *out_flags); -void intel_unpin_fb_vma(struct i915_vma *vma, unsigned long flags); struct drm_framebuffer * intel_framebuffer_create(struct drm_i915_gem_object *obj, struct drm_mode_fb_cmd2 *mode_cmd); -int intel_prepare_plane_fb(struct drm_plane *plane, - struct drm_plane_state *new_state); -void intel_cleanup_plane_fb(struct drm_plane *plane, - struct drm_plane_state *old_state); void assert_pch_transcoder_disabled(struct drm_i915_private *dev_priv, enum pipe pipe); @@ -620,19 +609,16 @@ void ilk_pfit_disable(const struct intel_crtc_state *old_crtc_state); int bdw_get_pipemisc_bpp(struct intel_crtc *crtc); unsigned int intel_plane_fence_y_offset(const struct intel_plane_state *plane_state); +bool intel_plane_uses_fence(const struct intel_plane_state *plane_state); bool intel_format_info_is_yuv_semiplanar(const struct drm_format_info *info, u64 modifier); -int intel_plane_pin_fb(struct intel_plane_state *plane_state); -void intel_plane_unpin_fb(struct intel_plane_state *old_plane_state); struct intel_encoder * intel_get_crtc_new_encoder(const struct intel_atomic_state *state, const struct intel_crtc_state *crtc_state); - -unsigned int intel_surf_alignment(const struct drm_framebuffer *fb, - int color_plane); -unsigned int intel_tile_width_bytes(const struct drm_framebuffer *fb, int color_plane); +void intel_plane_disable_noatomic(struct intel_crtc *crtc, + struct intel_plane *plane); void intel_display_driver_register(struct drm_i915_private *i915); void intel_display_driver_unregister(struct drm_i915_private *i915); @@ -650,23 +636,10 @@ void intel_init_pch_refclk(struct drm_i915_private *dev_priv); int intel_modeset_all_pipes(struct intel_atomic_state *state); /* modesetting asserts */ -void assert_panel_unlocked(struct drm_i915_private *dev_priv, - enum pipe pipe); -void assert_pll(struct drm_i915_private *dev_priv, - enum pipe pipe, bool state); -#define assert_pll_enabled(d, p) assert_pll(d, p, true) -#define assert_pll_disabled(d, p) assert_pll(d, p, false) -void assert_dsi_pll(struct drm_i915_private *dev_priv, bool state); -#define assert_dsi_pll_enabled(d) assert_dsi_pll(d, true) -#define assert_dsi_pll_disabled(d) assert_dsi_pll(d, false) -void assert_fdi_rx_pll(struct drm_i915_private *dev_priv, - enum pipe pipe, bool state); -#define assert_fdi_rx_pll_enabled(d, p) assert_fdi_rx_pll(d, p, true) -#define assert_fdi_rx_pll_disabled(d, p) assert_fdi_rx_pll(d, p, false) -void assert_pipe(struct drm_i915_private *dev_priv, - enum transcoder cpu_transcoder, bool state); -#define assert_pipe_enabled(d, t) assert_pipe(d, t, true) -#define assert_pipe_disabled(d, t) assert_pipe(d, t, false) +void assert_transcoder(struct drm_i915_private *dev_priv, + enum transcoder cpu_transcoder, bool state); +#define assert_transcoder_enabled(d, t) assert_transcoder(d, t, true) +#define assert_transcoder_disabled(d, t) assert_transcoder(d, t, false) /* Use I915_STATE_WARN(x) and I915_STATE_WARN_ON() (rather than WARN() and * WARN_ON()) for hw state sanity checks to check for unexpected conditions diff --git a/drivers/gpu/drm/i915/display/intel_display_debugfs.c b/drivers/gpu/drm/i915/display/intel_display_debugfs.c index 8fdacb252bb1..e04767695530 100644 --- a/drivers/gpu/drm/i915/display/intel_display_debugfs.c +++ b/drivers/gpu/drm/i915/display/intel_display_debugfs.c @@ -7,18 +7,19 @@ #include <drm/drm_fourcc.h> #include "i915_debugfs.h" +#include "intel_de.h" #include "intel_display_debugfs.h" #include "intel_display_power.h" -#include "intel_de.h" #include "intel_display_types.h" #include "intel_dmc.h" #include "intel_dp.h" +#include "intel_dp_mst.h" +#include "intel_drrs.h" #include "intel_fbc.h" #include "intel_hdcp.h" #include "intel_hdmi.h" #include "intel_pm.h" #include "intel_psr.h" -#include "intel_sideband.h" #include "intel_sprite.h" static inline struct drm_i915_private *node_to_i915(struct drm_info_node *node) @@ -1323,9 +1324,6 @@ static int i915_drrs_status(struct seq_file *m, void *unused) return 0; } -#define LPSP_STATUS(COND) (COND ? seq_puts(m, "LPSP: enabled\n") : \ - seq_puts(m, "LPSP: disabled\n")) - static bool intel_lpsp_power_well_enabled(struct drm_i915_private *i915, enum i915_power_well_id power_well_id) @@ -1344,32 +1342,20 @@ intel_lpsp_power_well_enabled(struct drm_i915_private *i915, static int i915_lpsp_status(struct seq_file *m, void *unused) { struct drm_i915_private *i915 = node_to_i915(m->private); - - if (DISPLAY_VER(i915) >= 13) { - LPSP_STATUS(!intel_lpsp_power_well_enabled(i915, - SKL_DISP_PW_2)); + bool lpsp_enabled = false; + + if (DISPLAY_VER(i915) >= 13 || IS_DISPLAY_VER(i915, 9, 10)) { + lpsp_enabled = !intel_lpsp_power_well_enabled(i915, SKL_DISP_PW_2); + } else if (IS_DISPLAY_VER(i915, 11, 12)) { + lpsp_enabled = !intel_lpsp_power_well_enabled(i915, ICL_DISP_PW_3); + } else if (IS_HASWELL(i915) || IS_BROADWELL(i915)) { + lpsp_enabled = !intel_lpsp_power_well_enabled(i915, HSW_DISP_PW_GLOBAL); + } else { + seq_puts(m, "LPSP: not supported\n"); return 0; } - switch (DISPLAY_VER(i915)) { - case 12: - case 11: - LPSP_STATUS(!intel_lpsp_power_well_enabled(i915, ICL_DISP_PW_3)); - break; - case 10: - case 9: - LPSP_STATUS(!intel_lpsp_power_well_enabled(i915, SKL_DISP_PW_2)); - break; - default: - /* - * Apart from HASWELL/BROADWELL other legacy platform doesn't - * support lpsp. - */ - if (IS_HASWELL(i915) || IS_BROADWELL(i915)) - LPSP_STATUS(!intel_lpsp_power_well_enabled(i915, HSW_DISP_PW_GLOBAL)); - else - seq_puts(m, "LPSP: not supported\n"); - } + seq_printf(m, "LPSP: %s\n", enableddisabled(lpsp_enabled)); return 0; } @@ -1393,7 +1379,7 @@ static int i915_dp_mst_info(struct seq_file *m, void *unused) continue; dig_port = enc_to_dig_port(intel_encoder); - if (!dig_port->dp.can_mst) + if (!intel_dp_mst_source_support(&dig_port->dp)) continue; seq_printf(m, "MST Source Port [ENCODER:%d:%s]\n", @@ -2044,11 +2030,9 @@ static int i915_drrs_ctl_set(void *data, u64 val) intel_dp = enc_to_intel_dp(encoder); if (val) - intel_edp_drrs_enable(intel_dp, - crtc_state); + intel_drrs_enable(intel_dp, crtc_state); else - intel_edp_drrs_disable(intel_dp, - crtc_state); + intel_drrs_disable(intel_dp, crtc_state); } drm_connector_list_iter_end(&conn_iter); @@ -2240,14 +2224,12 @@ static int i915_psr_status_show(struct seq_file *m, void *data) } DEFINE_SHOW_ATTRIBUTE(i915_psr_status); -#define LPSP_CAPABLE(COND) (COND ? seq_puts(m, "LPSP: capable\n") : \ - seq_puts(m, "LPSP: incapable\n")) - static int i915_lpsp_capability_show(struct seq_file *m, void *data) { struct drm_connector *connector = m->private; struct drm_i915_private *i915 = to_i915(connector->dev); struct intel_encoder *encoder; + bool lpsp_capable = false; encoder = intel_attached_encoder(to_intel_connector(connector)); if (!encoder) @@ -2256,35 +2238,27 @@ static int i915_lpsp_capability_show(struct seq_file *m, void *data) if (connector->status != connector_status_connected) return -ENODEV; - if (DISPLAY_VER(i915) >= 13) { - LPSP_CAPABLE(encoder->port <= PORT_B); - return 0; - } - - switch (DISPLAY_VER(i915)) { - case 12: + if (DISPLAY_VER(i915) >= 13) + lpsp_capable = encoder->port <= PORT_B; + else if (DISPLAY_VER(i915) >= 12) /* * Actually TGL can drive LPSP on port till DDI_C * but there is no physical connected DDI_C on TGL sku's, * even driver is not initilizing DDI_C port for gen12. */ - LPSP_CAPABLE(encoder->port <= PORT_B); - break; - case 11: - LPSP_CAPABLE(connector->connector_type == DRM_MODE_CONNECTOR_DSI || - connector->connector_type == DRM_MODE_CONNECTOR_eDP); - break; - case 10: - case 9: - LPSP_CAPABLE(encoder->port == PORT_A && - (connector->connector_type == DRM_MODE_CONNECTOR_DSI || - connector->connector_type == DRM_MODE_CONNECTOR_eDP || - connector->connector_type == DRM_MODE_CONNECTOR_DisplayPort)); - break; - default: - if (IS_HASWELL(i915) || IS_BROADWELL(i915)) - LPSP_CAPABLE(connector->connector_type == DRM_MODE_CONNECTOR_eDP); - } + lpsp_capable = encoder->port <= PORT_B; + else if (DISPLAY_VER(i915) == 11) + lpsp_capable = (connector->connector_type == DRM_MODE_CONNECTOR_DSI || + connector->connector_type == DRM_MODE_CONNECTOR_eDP); + else if (IS_DISPLAY_VER(i915, 9, 10)) + lpsp_capable = (encoder->port == PORT_A && + (connector->connector_type == DRM_MODE_CONNECTOR_DSI || + connector->connector_type == DRM_MODE_CONNECTOR_eDP || + connector->connector_type == DRM_MODE_CONNECTOR_DisplayPort)); + else if (IS_HASWELL(i915) || IS_BROADWELL(i915)) + lpsp_capable = connector->connector_type == DRM_MODE_CONNECTOR_eDP; + + seq_printf(m, "LPSP: %s\n", lpsp_capable ? "capable" : "incapable"); return 0; } @@ -2468,17 +2442,16 @@ static const struct file_operations i915_dsc_bpp_fops = { * * Cleanup will be done by drm_connector_unregister() through a call to * drm_debugfs_connector_remove(). - * - * Returns 0 on success, negative error codes on error. */ -int intel_connector_debugfs_add(struct drm_connector *connector) +void intel_connector_debugfs_add(struct intel_connector *intel_connector) { + struct drm_connector *connector = &intel_connector->base; struct dentry *root = connector->debugfs_entry; struct drm_i915_private *dev_priv = to_i915(connector->dev); /* The connector must have been registered beforehands. */ if (!root) - return -ENODEV; + return; if (connector->connector_type == DRM_MODE_CONNECTOR_eDP) { debugfs_create_file("i915_panel_timings", S_IRUGO, root, @@ -2511,33 +2484,23 @@ int intel_connector_debugfs_add(struct drm_connector *connector) connector, &i915_dsc_bpp_fops); } - /* Legacy panels doesn't lpsp on any platform */ - if ((DISPLAY_VER(dev_priv) >= 9 || IS_HASWELL(dev_priv) || - IS_BROADWELL(dev_priv)) && - (connector->connector_type == DRM_MODE_CONNECTOR_DSI || - connector->connector_type == DRM_MODE_CONNECTOR_eDP || - connector->connector_type == DRM_MODE_CONNECTOR_DisplayPort || - connector->connector_type == DRM_MODE_CONNECTOR_HDMIA || - connector->connector_type == DRM_MODE_CONNECTOR_HDMIB)) + if (connector->connector_type == DRM_MODE_CONNECTOR_DSI || + connector->connector_type == DRM_MODE_CONNECTOR_eDP || + connector->connector_type == DRM_MODE_CONNECTOR_DisplayPort || + connector->connector_type == DRM_MODE_CONNECTOR_HDMIA || + connector->connector_type == DRM_MODE_CONNECTOR_HDMIB) debugfs_create_file("i915_lpsp_capability", 0444, root, connector, &i915_lpsp_capability_fops); - - return 0; } /** * intel_crtc_debugfs_add - add i915 specific crtc debugfs files * @crtc: pointer to a drm_crtc * - * Returns 0 on success, negative error codes on error. - * * Failure to add debugfs entries should generally be ignored. */ -int intel_crtc_debugfs_add(struct drm_crtc *crtc) +void intel_crtc_debugfs_add(struct drm_crtc *crtc) { - if (!crtc->debugfs_entry) - return -ENODEV; - - crtc_updates_add(crtc); - return 0; + if (crtc->debugfs_entry) + crtc_updates_add(crtc); } diff --git a/drivers/gpu/drm/i915/display/intel_display_debugfs.h b/drivers/gpu/drm/i915/display/intel_display_debugfs.h index 557901f3eb90..d3a79c07c384 100644 --- a/drivers/gpu/drm/i915/display/intel_display_debugfs.h +++ b/drivers/gpu/drm/i915/display/intel_display_debugfs.h @@ -6,18 +6,18 @@ #ifndef __INTEL_DISPLAY_DEBUGFS_H__ #define __INTEL_DISPLAY_DEBUGFS_H__ -struct drm_connector; struct drm_crtc; struct drm_i915_private; +struct intel_connector; #ifdef CONFIG_DEBUG_FS void intel_display_debugfs_register(struct drm_i915_private *i915); -int intel_connector_debugfs_add(struct drm_connector *connector); -int intel_crtc_debugfs_add(struct drm_crtc *crtc); +void intel_connector_debugfs_add(struct intel_connector *connector); +void intel_crtc_debugfs_add(struct drm_crtc *crtc); #else static inline void intel_display_debugfs_register(struct drm_i915_private *i915) {} -static inline int intel_connector_debugfs_add(struct drm_connector *connector) { return 0; } -static inline int intel_crtc_debugfs_add(struct drm_crtc *crtc) { return 0; } +static inline void intel_connector_debugfs_add(struct intel_connector *connector) {} +static inline void intel_crtc_debugfs_add(struct drm_crtc *crtc) {} #endif #endif /* __INTEL_DISPLAY_DEBUGFS_H__ */ diff --git a/drivers/gpu/drm/i915/display/intel_display_power.c b/drivers/gpu/drm/i915/display/intel_display_power.c index cce1a926fcc1..1672604f9ef7 100644 --- a/drivers/gpu/drm/i915/display/intel_display_power.c +++ b/drivers/gpu/drm/i915/display/intel_display_power.c @@ -3,24 +3,25 @@ * Copyright © 2019 Intel Corporation */ -#include "display/intel_crt.h" - #include "i915_drv.h" #include "i915_irq.h" #include "intel_cdclk.h" #include "intel_combo_phy.h" -#include "intel_display_power.h" +#include "intel_crt.h" #include "intel_de.h" +#include "intel_display_power.h" #include "intel_display_types.h" #include "intel_dmc.h" #include "intel_dpio_phy.h" +#include "intel_dpll.h" #include "intel_hotplug.h" +#include "intel_pcode.h" #include "intel_pm.h" #include "intel_pps.h" -#include "intel_sideband.h" #include "intel_snps_phy.h" #include "intel_tc.h" #include "intel_vga.h" +#include "vlv_sideband.h" bool intel_display_power_well_is_enabled(struct drm_i915_private *dev_priv, enum i915_power_well_id power_well_id); @@ -560,7 +561,7 @@ static void icl_tc_port_assert_ref_held(struct drm_i915_private *dev_priv, if (drm_WARN_ON(&dev_priv->drm, !dig_port)) return; - if (DISPLAY_VER(dev_priv) == 11 && dig_port->tc_legacy_port) + if (DISPLAY_VER(dev_priv) == 11 && intel_tc_cold_requires_aux_pw(dig_port)) return; drm_WARN_ON(&dev_priv->drm, !intel_tc_port_ref_held(dig_port)); @@ -629,7 +630,7 @@ icl_tc_phy_aux_power_well_enable(struct drm_i915_private *dev_priv, * exit sequence. */ timeout_expected = is_tbt || intel_tc_cold_requires_aux_pw(dig_port); - if (DISPLAY_VER(dev_priv) == 11 && dig_port->tc_legacy_port) + if (DISPLAY_VER(dev_priv) == 11 && intel_tc_cold_requires_aux_pw(dig_port)) icl_tc_cold_exit(dev_priv); hsw_wait_for_power_well_enable(dev_priv, power_well, timeout_expected); @@ -1195,7 +1196,7 @@ static void gen9_disable_dc_states(struct drm_i915_private *dev_priv) if (!HAS_DISPLAY(dev_priv)) return; - dev_priv->display.get_cdclk(dev_priv, &cdclk_config); + intel_cdclk_get_cdclk(dev_priv, &cdclk_config); /* Can't read out voltage_level so can't use intel_cdclk_changed() */ drm_WARN_ON(&dev_priv->drm, intel_cdclk_needs_modeset(&dev_priv->cdclk.hw, diff --git a/drivers/gpu/drm/i915/display/intel_display_power.h b/drivers/gpu/drm/i915/display/intel_display_power.h index 978531841fa3..0612e4b6e3c8 100644 --- a/drivers/gpu/drm/i915/display/intel_display_power.h +++ b/drivers/gpu/drm/i915/display/intel_display_power.h @@ -410,6 +410,10 @@ void gen9_dbuf_slices_update(struct drm_i915_private *dev_priv, for ((wf) = intel_display_power_get((i915), (domain)); (wf); \ intel_display_power_put_async((i915), (domain), (wf)), (wf) = 0) +#define with_intel_display_power_if_enabled(i915, domain, wf) \ + for ((wf) = intel_display_power_get_if_enabled((i915), (domain)); (wf); \ + intel_display_power_put_async((i915), (domain), (wf)), (wf) = 0) + void chv_phy_powergate_lanes(struct intel_encoder *encoder, bool override, unsigned int mask); bool chv_phy_powergate_ch(struct drm_i915_private *dev_priv, enum dpio_phy phy, diff --git a/drivers/gpu/drm/i915/display/intel_display_types.h b/drivers/gpu/drm/i915/display/intel_display_types.h index 6beeeeba1bed..39e11eaec1a3 100644 --- a/drivers/gpu/drm/i915/display/intel_display_types.h +++ b/drivers/gpu/drm/i915/display/intel_display_types.h @@ -103,8 +103,6 @@ struct intel_fb_view { * in the rotated and remapped GTT view all no-CCS formats (up to 2 * color planes) are supported. * - * TODO: add support for CCS formats in the remapped GTT view. - * * The view information shared by all FB color planes in the FB, * like dst x/y and src/dst width, is stored separately in * intel_plane_state. @@ -271,6 +269,9 @@ struct intel_encoder { const struct intel_ddi_buf_trans *(*get_buf_trans)(struct intel_encoder *encoder, const struct intel_crtc_state *crtc_state, int *n_entries); + void (*set_signal_levels)(struct intel_encoder *encoder, + const struct intel_crtc_state *crtc_state); + enum hpd_pin hpd_pin; enum intel_display_power_domain power_domain; /* for communication with audio component; protected by av_mutex */ @@ -428,10 +429,6 @@ struct intel_hdcp_shim { int (*hdcp_2_2_capable)(struct intel_digital_port *dig_port, bool *capable); - /* Detects whether a HDCP 1.4 sink connected in MST topology */ - int (*streams_type1_capable)(struct intel_connector *connector, - bool *capable); - /* Write HDCP2.2 messages */ int (*write_2_2_msg)(struct intel_digital_port *dig_port, void *buf, size_t size); @@ -629,6 +626,12 @@ struct intel_plane_state { struct intel_fb_view view; + /* Plane pxp decryption state */ + bool decrypt; + + /* Plane state to display black pixels when pxp is borked */ + bool force_black; + /* plane control register */ u32 ctl; @@ -1060,12 +1063,14 @@ struct intel_crtc_state { struct intel_link_m_n dp_m2_n2; bool has_drrs; + /* PSR is supported but might not be enabled due the lack of enabled planes */ bool has_psr; bool has_psr2; bool enable_psr2_sel_fetch; bool req_psr2_sdp_prior_scanline; u32 dc3co_exitline; u16 su_y_granularity; + struct drm_dp_vsc_sdp psr_vsc; /* * Frequence the dpll for the port should run at. Differs from the @@ -1529,7 +1534,6 @@ struct intel_psr { u32 dc3co_exitline; u32 dc3co_exit_delay; struct delayed_work dc3co_work; - struct drm_dp_vsc_sdp vsc; }; struct intel_dp { @@ -1576,7 +1580,6 @@ struct intel_dp { struct intel_pps pps; - bool can_mst; /* this port supports mst */ bool is_mst; int active_mst_links; @@ -1606,8 +1609,6 @@ struct intel_dp { u8 dp_train_pat); void (*set_idle_link_train)(struct intel_dp *intel_dp, const struct intel_crtc_state *crtc_state); - void (*set_signal_levels)(struct intel_dp *intel_dp, - const struct intel_crtc_state *crtc_state); u8 (*preemph_max)(struct intel_dp *intel_dp); u8 (*voltage_max)(struct intel_dp *intel_dp, @@ -1667,8 +1668,11 @@ struct intel_digital_port { enum intel_display_power_domain ddi_io_power_domain; intel_wakeref_t ddi_io_wakeref; intel_wakeref_t aux_wakeref; + struct mutex tc_lock; /* protects the TypeC port mode */ intel_wakeref_t tc_lock_wakeref; + enum intel_display_power_domain tc_lock_power_domain; + struct delayed_work tc_disconnect_phy_work; int tc_link_refcount; bool tc_legacy_port:1; char tc_port_name[8]; @@ -1684,6 +1688,8 @@ struct intel_digital_port { bool hdcp_auth_status; /* HDCP port data need to pass to security f/w */ struct hdcp_port_data hdcp_port_data; + /* Whether the MST topology supports HDCP Type 1 Content */ + bool hdcp_mst_type1_capable; void (*write_infoframe)(struct intel_encoder *encoder, const struct intel_crtc_state *crtc_state, @@ -2035,28 +2041,6 @@ to_intel_frontbuffer(struct drm_framebuffer *fb) return fb ? to_intel_framebuffer(fb)->frontbuffer : NULL; } -static inline bool intel_panel_use_ssc(struct drm_i915_private *dev_priv) -{ - if (dev_priv->params.panel_use_ssc >= 0) - return dev_priv->params.panel_use_ssc != 0; - return dev_priv->vbt.lvds_use_ssc - && !(dev_priv->quirks & QUIRK_LVDS_SSC_DISABLE); -} - -static inline u32 i9xx_dpll_compute_fp(struct dpll *dpll) -{ - return dpll->n << 16 | dpll->m1 << 8 | dpll->m2; -} - -static inline u32 intel_fdi_link_freq(struct drm_i915_private *dev_priv, - const struct intel_crtc_state *pipe_config) -{ - if (HAS_DDI(dev_priv)) - return pipe_config->port_clock; /* SPLL */ - else - return dev_priv->fdi_pll_freq; -} - static inline bool is_ccs_modifier(u64 modifier) { return modifier == I915_FORMAT_MOD_Y_TILED_GEN12_RC_CCS || diff --git a/drivers/gpu/drm/i915/display/intel_dmc.c b/drivers/gpu/drm/i915/display/intel_dmc.c index b3c8e1c450ef..2dc9d632969d 100644 --- a/drivers/gpu/drm/i915/display/intel_dmc.c +++ b/drivers/gpu/drm/i915/display/intel_dmc.c @@ -45,8 +45,8 @@ #define GEN12_DMC_MAX_FW_SIZE ICL_DMC_MAX_FW_SIZE -#define ADLP_DMC_PATH DMC_PATH(adlp, 2, 10) -#define ADLP_DMC_VERSION_REQUIRED DMC_VERSION(2, 10) +#define ADLP_DMC_PATH DMC_PATH(adlp, 2, 12) +#define ADLP_DMC_VERSION_REQUIRED DMC_VERSION(2, 12) MODULE_FIRMWARE(ADLP_DMC_PATH); #define ADLS_DMC_PATH DMC_PATH(adls, 2, 01) @@ -255,20 +255,10 @@ intel_get_stepping_info(struct drm_i915_private *i915, static void gen9_set_dc_state_debugmask(struct drm_i915_private *dev_priv) { - u32 val, mask; - - mask = DC_STATE_DEBUG_MASK_MEMORY_UP; - - if (IS_GEMINILAKE(dev_priv) || IS_BROXTON(dev_priv)) - mask |= DC_STATE_DEBUG_MASK_CORES; - /* The below bit doesn't need to be cleared ever afterwards */ - val = intel_de_read(dev_priv, DC_STATE_DEBUG); - if ((val & mask) != mask) { - val |= mask; - intel_de_write(dev_priv, DC_STATE_DEBUG, val); - intel_de_posting_read(dev_priv, DC_STATE_DEBUG); - } + intel_de_rmw(dev_priv, DC_STATE_DEBUG, 0, + DC_STATE_DEBUG_MASK_CORES | DC_STATE_DEBUG_MASK_MEMORY_UP); + intel_de_posting_read(dev_priv, DC_STATE_DEBUG); } /** diff --git a/drivers/gpu/drm/i915/display/intel_dp.c b/drivers/gpu/drm/i915/display/intel_dp.c index 5cf152be4487..23de500d56b5 100644 --- a/drivers/gpu/drm/i915/display/intel_dp.c +++ b/drivers/gpu/drm/i915/display/intel_dp.c @@ -44,6 +44,7 @@ #include "i915_drv.h" #include "intel_atomic.h" #include "intel_audio.h" +#include "intel_backlight.h" #include "intel_connector.h" #include "intel_ddi.h" #include "intel_de.h" @@ -55,6 +56,7 @@ #include "intel_dp_mst.h" #include "intel_dpio_phy.h" #include "intel_dpll.h" +#include "intel_drrs.h" #include "intel_fifo_underrun.h" #include "intel_hdcp.h" #include "intel_hdmi.h" @@ -64,7 +66,6 @@ #include "intel_panel.h" #include "intel_pps.h" #include "intel_psr.h" -#include "intel_sideband.h" #include "intel_tc.h" #include "intel_vdsc.h" #include "intel_vrr.h" @@ -100,6 +101,8 @@ static const u8 valid_dsc_slicecount[] = {1, 2, 4}; * * If a CPU or PCH DP output is attached to an eDP panel, this function * will return true, and false otherwise. + * + * This function is not safe to use prior to encoder type being set. */ bool intel_dp_is_edp(struct intel_dp *intel_dp) { @@ -111,6 +114,12 @@ bool intel_dp_is_edp(struct intel_dp *intel_dp) static void intel_dp_unset_edid(struct intel_dp *intel_dp); static int intel_dp_dsc_compute_bpp(struct intel_dp *intel_dp, u8 dsc_max_bpc); +/* Is link rate UHBR and thus 128b/132b? */ +bool intel_dp_is_uhbr(const struct intel_crtc_state *crtc_state) +{ + return crtc_state->port_clock >= 1000000; +} + /* update sink rates from dpcd */ static void intel_dp_set_sink_rates(struct intel_dp *intel_dp) { @@ -130,6 +139,9 @@ static void intel_dp_set_sink_rates(struct intel_dp *intel_dp) return; } + /* + * Sink rates for 8b/10b. + */ max_rate = drm_dp_bw_code_to_link_rate(intel_dp->dpcd[DP_MAX_LINK_RATE]); max_lttpr_rate = drm_dp_lttpr_max_link_rate(intel_dp->lttpr_common_caps); if (max_lttpr_rate) @@ -141,6 +153,41 @@ static void intel_dp_set_sink_rates(struct intel_dp *intel_dp) intel_dp->sink_rates[i] = dp_rates[i]; } + /* + * Sink rates for 128b/132b. If set, sink should support all 8b/10b + * rates and 10 Gbps. + */ + if (intel_dp->dpcd[DP_MAIN_LINK_CHANNEL_CODING] & DP_CAP_ANSI_128B132B) { + u8 uhbr_rates = 0; + + BUILD_BUG_ON(ARRAY_SIZE(intel_dp->sink_rates) < ARRAY_SIZE(dp_rates) + 3); + + drm_dp_dpcd_readb(&intel_dp->aux, + DP_128B132B_SUPPORTED_LINK_RATES, &uhbr_rates); + + if (drm_dp_lttpr_count(intel_dp->lttpr_common_caps)) { + /* We have a repeater */ + if (intel_dp->lttpr_common_caps[0] >= 0x20 && + intel_dp->lttpr_common_caps[DP_MAIN_LINK_CHANNEL_CODING_PHY_REPEATER - + DP_LT_TUNABLE_PHY_REPEATER_FIELD_DATA_STRUCTURE_REV] & + DP_PHY_REPEATER_128B132B_SUPPORTED) { + /* Repeater supports 128b/132b, valid UHBR rates */ + uhbr_rates &= intel_dp->lttpr_common_caps[DP_PHY_REPEATER_128B132B_RATES - + DP_LT_TUNABLE_PHY_REPEATER_FIELD_DATA_STRUCTURE_REV]; + } else { + /* Does not support 128b/132b */ + uhbr_rates = 0; + } + } + + if (uhbr_rates & DP_UHBR10) + intel_dp->sink_rates[i++] = 1000000; + if (uhbr_rates & DP_UHBR13_5) + intel_dp->sink_rates[i++] = 1350000; + if (uhbr_rates & DP_UHBR20) + intel_dp->sink_rates[i++] = 2000000; + } + intel_dp->num_sink_rates = i; } @@ -192,6 +239,10 @@ int intel_dp_max_lane_count(struct intel_dp *intel_dp) return intel_dp->max_link_lane_count; } +/* + * The required data bandwidth for a mode with given pixel clock and bpp. This + * is the required net bandwidth independent of the data bandwidth efficiency. + */ int intel_dp_link_required(int pixel_clock, int bpp) { @@ -199,16 +250,52 @@ intel_dp_link_required(int pixel_clock, int bpp) return DIV_ROUND_UP(pixel_clock * bpp, 8); } +/* + * Given a link rate and lanes, get the data bandwidth. + * + * Data bandwidth is the actual payload rate, which depends on the data + * bandwidth efficiency and the link rate. + * + * For 8b/10b channel encoding, SST and non-FEC, the data bandwidth efficiency + * is 80%. For example, for a 1.62 Gbps link, 1.62*10^9 bps * 0.80 * (1/8) = + * 162000 kBps. With 8-bit symbols, we have 162000 kHz symbol clock. Just by + * coincidence, the port clock in kHz matches the data bandwidth in kBps, and + * they equal the link bit rate in Gbps multiplied by 100000. (Note that this no + * longer holds for data bandwidth as soon as FEC or MST is taken into account!) + * + * For 128b/132b channel encoding, the data bandwidth efficiency is 96.71%. For + * example, for a 10 Gbps link, 10*10^9 bps * 0.9671 * (1/8) = 1208875 + * kBps. With 32-bit symbols, we have 312500 kHz symbol clock. The value 1000000 + * does not match the symbol clock, the port clock (not even if you think in + * terms of a byte clock), nor the data bandwidth. It only matches the link bit + * rate in units of 10000 bps. + */ int -intel_dp_max_data_rate(int max_link_clock, int max_lanes) +intel_dp_max_data_rate(int max_link_rate, int max_lanes) { - /* max_link_clock is the link symbol clock (LS_Clk) in kHz and not the - * link rate that is generally expressed in Gbps. Since, 8 bits of data - * is transmitted every LS_Clk per lane, there is no need to account for - * the channel encoding that is done in the PHY layer here. + if (max_link_rate >= 1000000) { + /* + * UHBR rates always use 128b/132b channel encoding, and have + * 97.71% data bandwidth efficiency. Consider max_link_rate the + * link bit rate in units of 10000 bps. + */ + int max_link_rate_kbps = max_link_rate * 10; + + max_link_rate_kbps = DIV_ROUND_CLOSEST_ULL(max_link_rate_kbps * 9671, 10000); + max_link_rate = max_link_rate_kbps / 8; + } + + /* + * Lower than UHBR rates always use 8b/10b channel encoding, and have + * 80% data bandwidth efficiency for SST non-FEC. However, this turns + * out to be a nop by coincidence, and can be skipped: + * + * int max_link_rate_kbps = max_link_rate * 10; + * max_link_rate_kbps = DIV_ROUND_CLOSEST_ULL(max_link_rate_kbps * 8, 10); + * max_link_rate = max_link_rate_kbps / 8; */ - return max_link_clock * max_lanes; + return max_link_rate * max_lanes; } bool intel_dp_can_bigjoiner(struct intel_dp *intel_dp) @@ -222,6 +309,20 @@ bool intel_dp_can_bigjoiner(struct intel_dp *intel_dp) encoder->port != PORT_A); } +static int dg2_max_source_rate(struct intel_dp *intel_dp) +{ + return intel_dp_is_edp(intel_dp) ? 810000 : 1350000; +} + +static bool is_low_voltage_sku(struct drm_i915_private *i915, enum phy phy) +{ + u32 voltage; + + voltage = intel_de_read(i915, ICL_PORT_COMP_DW3(phy)) & VOLTAGE_INFO_MASK; + + return voltage == VOLTAGE_INFO_0_85V; +} + static int icl_max_source_rate(struct intel_dp *intel_dp) { struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp); @@ -229,7 +330,7 @@ static int icl_max_source_rate(struct intel_dp *intel_dp) enum phy phy = intel_port_to_phy(dev_priv, dig_port->base.port); if (intel_phy_is_combo(dev_priv, phy) && - !intel_dp_is_edp(intel_dp)) + (is_low_voltage_sku(dev_priv, phy) || !intel_dp_is_edp(intel_dp))) return 540000; return 810000; @@ -237,7 +338,23 @@ static int icl_max_source_rate(struct intel_dp *intel_dp) static int ehl_max_source_rate(struct intel_dp *intel_dp) { - if (intel_dp_is_edp(intel_dp)) + struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp); + struct drm_i915_private *dev_priv = to_i915(dig_port->base.base.dev); + enum phy phy = intel_port_to_phy(dev_priv, dig_port->base.port); + + if (intel_dp_is_edp(intel_dp) || is_low_voltage_sku(dev_priv, phy)) + return 540000; + + return 810000; +} + +static int dg1_max_source_rate(struct intel_dp *intel_dp) +{ + struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp); + struct drm_i915_private *i915 = to_i915(dig_port->base.base.dev); + enum phy phy = intel_port_to_phy(i915, dig_port->base.port); + + if (intel_phy_is_combo(i915, phy) && is_low_voltage_sku(i915, phy)) return 540000; return 810000; @@ -248,7 +365,8 @@ intel_dp_set_source_rates(struct intel_dp *intel_dp) { /* The values must be in increasing order */ static const int icl_rates[] = { - 162000, 216000, 270000, 324000, 432000, 540000, 648000, 810000 + 162000, 216000, 270000, 324000, 432000, 540000, 648000, 810000, + 1000000, 1350000, }; static const int bxt_rates[] = { 162000, 216000, 243000, 270000, 324000, 432000, 540000 @@ -275,7 +393,12 @@ intel_dp_set_source_rates(struct intel_dp *intel_dp) if (DISPLAY_VER(dev_priv) >= 11) { source_rates = icl_rates; size = ARRAY_SIZE(icl_rates); - if (IS_JSL_EHL(dev_priv)) + if (IS_DG2(dev_priv)) + max_rate = dg2_max_source_rate(intel_dp); + else if (IS_ALDERLAKE_P(dev_priv) || IS_ALDERLAKE_S(dev_priv) || + IS_DG1(dev_priv) || IS_ROCKETLAKE(dev_priv)) + max_rate = dg1_max_source_rate(intel_dp); + else if (IS_JSL_EHL(dev_priv)) max_rate = ehl_max_source_rate(intel_dp); else max_rate = icl_max_source_rate(intel_dp); @@ -461,7 +584,9 @@ u32 intel_dp_mode_to_fec_clock(u32 mode_clock) static int small_joiner_ram_size_bits(struct drm_i915_private *i915) { - if (DISPLAY_VER(i915) >= 11) + if (DISPLAY_VER(i915) >= 13) + return 17280 * 8; + else if (DISPLAY_VER(i915) >= 11) return 7680 * 8; else return 6144 * 8; @@ -703,6 +828,17 @@ intel_dp_mode_valid_downstream(struct intel_connector *connector, return MODE_OK; } +static bool intel_dp_need_bigjoiner(struct intel_dp *intel_dp, + int hdisplay, int clock) +{ + struct drm_i915_private *i915 = dp_to_i915(intel_dp); + + if (!intel_dp_can_bigjoiner(intel_dp)) + return false; + + return clock > i915->max_dotclk_freq || hdisplay > 5120; +} + static enum drm_mode_status intel_dp_mode_valid(struct drm_connector *connector, struct drm_display_mode *mode) @@ -726,11 +862,9 @@ intel_dp_mode_valid(struct drm_connector *connector, return MODE_H_ILLEGAL; if (intel_dp_is_edp(intel_dp) && fixed_mode) { - if (mode->hdisplay != fixed_mode->hdisplay) - return MODE_PANEL; - - if (mode->vdisplay != fixed_mode->vdisplay) - return MODE_PANEL; + status = intel_panel_mode_valid(intel_connector, mode); + if (status != MODE_OK) + return status; target_clock = fixed_mode->clock; } @@ -738,8 +872,7 @@ intel_dp_mode_valid(struct drm_connector *connector, if (mode->clock < 10000) return MODE_CLOCK_LOW; - if ((target_clock > max_dotclk || mode->hdisplay > 5120) && - intel_dp_can_bigjoiner(intel_dp)) { + if (intel_dp_need_bigjoiner(intel_dp, mode->hdisplay, target_clock)) { bigjoiner = true; max_dotclk *= 2; } @@ -811,18 +944,14 @@ intel_dp_mode_valid(struct drm_connector *connector, return intel_mode_valid_max_plane_size(dev_priv, mode, bigjoiner); } -bool intel_dp_source_supports_hbr2(struct intel_dp *intel_dp) +bool intel_dp_source_supports_tps3(struct drm_i915_private *i915) { - int max_rate = intel_dp->source_rates[intel_dp->num_source_rates - 1]; - - return max_rate >= 540000; + return DISPLAY_VER(i915) >= 9 || IS_BROADWELL(i915) || IS_HASWELL(i915); } -bool intel_dp_source_supports_hbr3(struct intel_dp *intel_dp) +bool intel_dp_source_supports_tps4(struct drm_i915_private *i915) { - int max_rate = intel_dp->source_rates[intel_dp->num_source_rates - 1]; - - return max_rate >= 810000; + return DISPLAY_VER(i915) >= 10; } static void snprintf_int_array(char *str, size_t len, @@ -1044,7 +1173,8 @@ intel_dp_adjust_compliance_config(struct intel_dp *intel_dp, intel_dp->num_common_rates, intel_dp->compliance.test_link_rate); if (index >= 0) - limits->min_clock = limits->max_clock = index; + limits->min_rate = limits->max_rate = + intel_dp->compliance.test_link_rate; limits->min_lane_count = limits->max_lane_count = intel_dp->compliance.test_lane_count; } @@ -1058,8 +1188,8 @@ intel_dp_compute_link_config_wide(struct intel_dp *intel_dp, const struct link_config_limits *limits) { struct drm_display_mode *adjusted_mode = &pipe_config->hw.adjusted_mode; - int bpp, clock, lane_count; - int mode_rate, link_clock, link_avail; + int bpp, i, lane_count; + int mode_rate, link_rate, link_avail; for (bpp = limits->max_bpp; bpp >= limits->min_bpp; bpp -= 2 * 3) { int output_bpp = intel_dp_output_bpp(pipe_config->output_format, bpp); @@ -1067,18 +1197,22 @@ intel_dp_compute_link_config_wide(struct intel_dp *intel_dp, mode_rate = intel_dp_link_required(adjusted_mode->crtc_clock, output_bpp); - for (clock = limits->min_clock; clock <= limits->max_clock; clock++) { + for (i = 0; i < intel_dp->num_common_rates; i++) { + link_rate = intel_dp->common_rates[i]; + if (link_rate < limits->min_rate || + link_rate > limits->max_rate) + continue; + for (lane_count = limits->min_lane_count; lane_count <= limits->max_lane_count; lane_count <<= 1) { - link_clock = intel_dp->common_rates[clock]; - link_avail = intel_dp_max_data_rate(link_clock, + link_avail = intel_dp_max_data_rate(link_rate, lane_count); if (mode_rate <= link_avail) { pipe_config->lane_count = lane_count; pipe_config->pipe_bpp = bpp; - pipe_config->port_clock = link_clock; + pipe_config->port_clock = link_rate; return 0; } @@ -1212,7 +1346,7 @@ static int intel_dp_dsc_compute_config(struct intel_dp *intel_dp, * with DSC enabled for the requested mode. */ pipe_config->pipe_bpp = pipe_bpp; - pipe_config->port_clock = intel_dp->common_rates[limits->max_clock]; + pipe_config->port_clock = limits->max_rate; pipe_config->lane_count = limits->max_lane_count; if (intel_dp_is_edp(intel_dp)) { @@ -1321,8 +1455,8 @@ intel_dp_compute_link_config(struct intel_encoder *encoder, /* No common link rates between source and sink */ drm_WARN_ON(encoder->base.dev, common_len <= 0); - limits.min_clock = 0; - limits.max_clock = common_len - 1; + limits.min_rate = intel_dp->common_rates[0]; + limits.max_rate = intel_dp->common_rates[common_len - 1]; limits.min_lane_count = 1; limits.max_lane_count = intel_dp_max_lane_count(intel_dp); @@ -1340,20 +1474,18 @@ intel_dp_compute_link_config(struct intel_encoder *encoder, * values correspond to the native resolution of the panel. */ limits.min_lane_count = limits.max_lane_count; - limits.min_clock = limits.max_clock; + limits.min_rate = limits.max_rate; } intel_dp_adjust_compliance_config(intel_dp, pipe_config, &limits); drm_dbg_kms(&i915->drm, "DP link computation with max lane count %i " "max rate %d max bpp %d pixel clock %iKHz\n", - limits.max_lane_count, - intel_dp->common_rates[limits.max_clock], + limits.max_lane_count, limits.max_rate, limits.max_bpp, adjusted_mode->crtc_clock); - if ((adjusted_mode->crtc_clock > i915->max_dotclk_freq || - adjusted_mode->crtc_hdisplay > 5120) && - intel_dp_can_bigjoiner(intel_dp)) + if (intel_dp_need_bigjoiner(intel_dp, adjusted_mode->crtc_hdisplay, + adjusted_mode->crtc_clock)) pipe_config->bigjoiner = true; /* @@ -1553,7 +1685,7 @@ void intel_dp_compute_psr_vsc_sdp(struct intel_dp *intel_dp, { vsc->sdp_type = DP_SDP_VSC; - if (intel_dp->psr.psr2_enabled) { + if (crtc_state->has_psr2) { if (intel_dp->psr.colorimetry_support && intel_dp_needs_vsc_sdp(crtc_state, conn_state)) { /* [PSR2, +Colorimetry] */ @@ -1603,46 +1735,6 @@ intel_dp_compute_hdr_metadata_infoframe_sdp(struct intel_dp *intel_dp, intel_hdmi_infoframe_enable(HDMI_PACKET_TYPE_GAMUT_METADATA); } -static void -intel_dp_drrs_compute_config(struct intel_dp *intel_dp, - struct intel_crtc_state *pipe_config, - int output_bpp, bool constant_n) -{ - struct intel_connector *intel_connector = intel_dp->attached_connector; - struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); - int pixel_clock; - - if (pipe_config->vrr.enable) - return; - - /* - * DRRS and PSR can't be enable together, so giving preference to PSR - * as it allows more power-savings by complete shutting down display, - * so to guarantee this, intel_dp_drrs_compute_config() must be called - * after intel_psr_compute_config(). - */ - if (pipe_config->has_psr) - return; - - if (!intel_connector->panel.downclock_mode || - dev_priv->drrs.type != SEAMLESS_DRRS_SUPPORT) - return; - - pipe_config->has_drrs = true; - - pixel_clock = intel_connector->panel.downclock_mode->clock; - if (pipe_config->splitter.enable) - pixel_clock /= pipe_config->splitter.link_count; - - intel_link_compute_m_n(output_bpp, pipe_config->lane_count, pixel_clock, - pipe_config->port_clock, &pipe_config->dp_m2_n2, - constant_n, pipe_config->fec_enable); - - /* FIXME: abstract this better */ - if (pipe_config->splitter.enable) - pipe_config->dp_m2_n2.gmch_m *= pipe_config->splitter.link_count; -} - int intel_dp_compute_config(struct intel_encoder *encoder, struct intel_crtc_state *pipe_config, @@ -1665,7 +1757,7 @@ intel_dp_compute_config(struct intel_encoder *encoder, adjusted_mode); if (pipe_config->output_format == INTEL_OUTPUT_FORMAT_YCBCR420) { - ret = intel_pch_panel_fitting(pipe_config, conn_state); + ret = intel_panel_fitting(pipe_config, conn_state); if (ret) return ret; } @@ -1678,13 +1770,11 @@ intel_dp_compute_config(struct intel_encoder *encoder, pipe_config->has_audio = intel_conn_state->force_audio == HDMI_AUDIO_ON; if (intel_dp_is_edp(intel_dp) && intel_connector->panel.fixed_mode) { - intel_fixed_panel_mode(intel_connector->panel.fixed_mode, - adjusted_mode); + ret = intel_panel_compute_config(intel_connector, adjusted_mode); + if (ret) + return ret; - if (HAS_GMCH(dev_priv)) - ret = intel_gmch_panel_fitting(pipe_config, conn_state); - else - ret = intel_pch_panel_fitting(pipe_config, conn_state); + ret = intel_panel_fitting(pipe_config, conn_state); if (ret) return ret; } @@ -1750,9 +1840,9 @@ intel_dp_compute_config(struct intel_encoder *encoder, g4x_dp_set_clock(encoder, pipe_config); intel_vrr_compute_config(pipe_config, conn_state); - intel_psr_compute_config(intel_dp, pipe_config); - intel_dp_drrs_compute_config(intel_dp, pipe_config, output_bpp, - constant_n); + intel_psr_compute_config(intel_dp, pipe_config, conn_state); + intel_drrs_compute_config(intel_dp, pipe_config, output_bpp, + constant_n); intel_dp_compute_vsc_sdp(intel_dp, pipe_config, conn_state); intel_dp_compute_hdr_metadata_infoframe_sdp(intel_dp, pipe_config, conn_state); @@ -1762,6 +1852,7 @@ intel_dp_compute_config(struct intel_encoder *encoder, void intel_dp_set_link_params(struct intel_dp *intel_dp, int link_rate, int lane_count) { + memset(intel_dp->train_set, 0, sizeof(intel_dp->train_set)); intel_dp->link_trained = false; intel_dp->link_rate = link_rate; intel_dp->lane_count = lane_count; @@ -1779,7 +1870,7 @@ void intel_edp_backlight_on(const struct intel_crtc_state *crtc_state, drm_dbg_kms(&i915->drm, "\n"); - intel_panel_enable_backlight(crtc_state, conn_state); + intel_backlight_enable(crtc_state, conn_state); intel_pps_backlight_on(intel_dp); } @@ -1795,7 +1886,7 @@ void intel_edp_backlight_off(const struct drm_connector_state *old_conn_state) drm_dbg_kms(&i915->drm, "\n"); intel_pps_backlight_off(intel_dp); - intel_panel_disable_backlight(old_conn_state); + intel_backlight_disable(old_conn_state); } static bool downstream_hpd_needs_d0(struct intel_dp *intel_dp) @@ -2392,6 +2483,8 @@ static void intel_edp_mso_mode_fixup(struct intel_connector *connector, static void intel_edp_mso_init(struct intel_dp *intel_dp) { struct drm_i915_private *i915 = dp_to_i915(intel_dp); + struct intel_connector *connector = intel_dp->attached_connector; + struct drm_display_info *info = &connector->base.display_info; u8 mso; if (intel_dp->edp_dpcd[0] < DP_EDP_14) @@ -2410,8 +2503,9 @@ static void intel_edp_mso_init(struct intel_dp *intel_dp) } if (mso) { - drm_dbg_kms(&i915->drm, "Sink MSO %ux%u configuration\n", - mso, drm_dp_max_lane_count(intel_dp->dpcd) / mso); + drm_dbg_kms(&i915->drm, "Sink MSO %ux%u configuration, pixel overlap %u\n", + mso, drm_dp_max_lane_count(intel_dp->dpcd) / mso, + info->mso_pixel_overlap); if (!HAS_MSO(i915)) { drm_err(&i915->drm, "No source MSO support, disabling\n"); mso = 0; @@ -2419,7 +2513,7 @@ static void intel_edp_mso_init(struct intel_dp *intel_dp) } intel_dp->mso_link_count = mso; - intel_dp->mso_pixel_overlap = 0; /* FIXME: read from DisplayID v2.0 */ + intel_dp->mso_pixel_overlap = mso ? info->mso_pixel_overlap : 0; } static bool @@ -2508,8 +2602,6 @@ intel_edp_init_dpcd(struct intel_dp *intel_dp) */ intel_edp_init_source_oui(intel_dp, true); - intel_edp_mso_init(intel_dp); - return true; } @@ -2577,7 +2669,7 @@ intel_dp_can_mst(struct intel_dp *intel_dp) struct drm_i915_private *i915 = dp_to_i915(intel_dp); return i915->params.enable_dp_mst && - intel_dp->can_mst && + intel_dp_mst_source_support(intel_dp) && drm_dp_read_mst_cap(&intel_dp->aux, intel_dp->dpcd); } @@ -2592,10 +2684,10 @@ intel_dp_configure_mst(struct intel_dp *intel_dp) drm_dbg_kms(&i915->drm, "[ENCODER:%d:%s] MST support: port: %s, sink: %s, modparam: %s\n", encoder->base.base.id, encoder->base.name, - yesno(intel_dp->can_mst), yesno(sink_can_mst), + yesno(intel_dp_mst_source_support(intel_dp)), yesno(sink_can_mst), yesno(i915->params.enable_dp_mst)); - if (!intel_dp->can_mst) + if (!intel_dp_mst_source_support(intel_dp)) return; intel_dp->is_mst = sink_can_mst && @@ -2812,7 +2904,7 @@ static void intel_write_dp_sdp(struct intel_encoder *encoder, void intel_write_dp_vsc_sdp(struct intel_encoder *encoder, const struct intel_crtc_state *crtc_state, - struct drm_dp_vsc_sdp *vsc) + const struct drm_dp_vsc_sdp *vsc) { struct intel_digital_port *dig_port = enc_to_dig_port(encoder); struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); @@ -4591,6 +4683,17 @@ static int intel_dp_connector_atomic_check(struct drm_connector *conn, return intel_modeset_synced_crtcs(state, conn); } +static void intel_dp_oob_hotplug_event(struct drm_connector *connector) +{ + struct intel_encoder *encoder = intel_attached_encoder(to_intel_connector(connector)); + struct drm_i915_private *i915 = to_i915(connector->dev); + + spin_lock_irq(&i915->irq_lock); + i915->hotplug.event_bits |= BIT(encoder->hpd_pin); + spin_unlock_irq(&i915->irq_lock); + queue_delayed_work(system_wq, &i915->hotplug.hotplug_work, 0); +} + static const struct drm_connector_funcs intel_dp_connector_funcs = { .force = intel_dp_force, .fill_modes = drm_helper_probe_single_connector_modes, @@ -4601,6 +4704,7 @@ static const struct drm_connector_funcs intel_dp_connector_funcs = { .destroy = intel_connector_destroy, .atomic_destroy_state = drm_atomic_helper_connector_destroy_state, .atomic_duplicate_state = intel_digital_connector_duplicate_state, + .oob_hotplug_event = intel_dp_oob_hotplug_event, }; static const struct drm_connector_helper_funcs intel_dp_connector_helper_funcs = { @@ -4716,432 +4820,6 @@ intel_dp_add_properties(struct intel_dp *intel_dp, struct drm_connector *connect drm_connector_attach_vrr_capable_property(connector); } -/** - * intel_dp_set_drrs_state - program registers for RR switch to take effect - * @dev_priv: i915 device - * @crtc_state: a pointer to the active intel_crtc_state - * @refresh_rate: RR to be programmed - * - * This function gets called when refresh rate (RR) has to be changed from - * one frequency to another. Switches can be between high and low RR - * supported by the panel or to any other RR based on media playback (in - * this case, RR value needs to be passed from user space). - * - * The caller of this function needs to take a lock on dev_priv->drrs. - */ -static void intel_dp_set_drrs_state(struct drm_i915_private *dev_priv, - const struct intel_crtc_state *crtc_state, - int refresh_rate) -{ - struct intel_dp *intel_dp = dev_priv->drrs.dp; - struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); - enum drrs_refresh_rate_type index = DRRS_HIGH_RR; - - if (refresh_rate <= 0) { - drm_dbg_kms(&dev_priv->drm, - "Refresh rate should be positive non-zero.\n"); - return; - } - - if (intel_dp == NULL) { - drm_dbg_kms(&dev_priv->drm, "DRRS not supported.\n"); - return; - } - - if (!crtc) { - drm_dbg_kms(&dev_priv->drm, - "DRRS: intel_crtc not initialized\n"); - return; - } - - if (dev_priv->drrs.type < SEAMLESS_DRRS_SUPPORT) { - drm_dbg_kms(&dev_priv->drm, "Only Seamless DRRS supported.\n"); - return; - } - - if (drm_mode_vrefresh(intel_dp->attached_connector->panel.downclock_mode) == - refresh_rate) - index = DRRS_LOW_RR; - - if (index == dev_priv->drrs.refresh_rate_type) { - drm_dbg_kms(&dev_priv->drm, - "DRRS requested for previously set RR...ignoring\n"); - return; - } - - if (!crtc_state->hw.active) { - drm_dbg_kms(&dev_priv->drm, - "eDP encoder disabled. CRTC not Active\n"); - return; - } - - if (DISPLAY_VER(dev_priv) >= 8 && !IS_CHERRYVIEW(dev_priv)) { - switch (index) { - case DRRS_HIGH_RR: - intel_dp_set_m_n(crtc_state, M1_N1); - break; - case DRRS_LOW_RR: - intel_dp_set_m_n(crtc_state, M2_N2); - break; - case DRRS_MAX_RR: - default: - drm_err(&dev_priv->drm, - "Unsupported refreshrate type\n"); - } - } else if (DISPLAY_VER(dev_priv) > 6) { - i915_reg_t reg = PIPECONF(crtc_state->cpu_transcoder); - u32 val; - - val = intel_de_read(dev_priv, reg); - if (index > DRRS_HIGH_RR) { - if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) - val |= PIPECONF_EDP_RR_MODE_SWITCH_VLV; - else - val |= PIPECONF_EDP_RR_MODE_SWITCH; - } else { - if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) - val &= ~PIPECONF_EDP_RR_MODE_SWITCH_VLV; - else - val &= ~PIPECONF_EDP_RR_MODE_SWITCH; - } - intel_de_write(dev_priv, reg, val); - } - - dev_priv->drrs.refresh_rate_type = index; - - drm_dbg_kms(&dev_priv->drm, "eDP Refresh Rate set to : %dHz\n", - refresh_rate); -} - -static void -intel_edp_drrs_enable_locked(struct intel_dp *intel_dp) -{ - struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); - - dev_priv->drrs.busy_frontbuffer_bits = 0; - dev_priv->drrs.dp = intel_dp; -} - -/** - * intel_edp_drrs_enable - init drrs struct if supported - * @intel_dp: DP struct - * @crtc_state: A pointer to the active crtc state. - * - * Initializes frontbuffer_bits and drrs.dp - */ -void intel_edp_drrs_enable(struct intel_dp *intel_dp, - const struct intel_crtc_state *crtc_state) -{ - struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); - - if (!crtc_state->has_drrs) - return; - - drm_dbg_kms(&dev_priv->drm, "Enabling DRRS\n"); - - mutex_lock(&dev_priv->drrs.mutex); - - if (dev_priv->drrs.dp) { - drm_warn(&dev_priv->drm, "DRRS already enabled\n"); - goto unlock; - } - - intel_edp_drrs_enable_locked(intel_dp); - -unlock: - mutex_unlock(&dev_priv->drrs.mutex); -} - -static void -intel_edp_drrs_disable_locked(struct intel_dp *intel_dp, - const struct intel_crtc_state *crtc_state) -{ - struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); - - if (dev_priv->drrs.refresh_rate_type == DRRS_LOW_RR) { - int refresh; - - refresh = drm_mode_vrefresh(intel_dp->attached_connector->panel.fixed_mode); - intel_dp_set_drrs_state(dev_priv, crtc_state, refresh); - } - - dev_priv->drrs.dp = NULL; -} - -/** - * intel_edp_drrs_disable - Disable DRRS - * @intel_dp: DP struct - * @old_crtc_state: Pointer to old crtc_state. - * - */ -void intel_edp_drrs_disable(struct intel_dp *intel_dp, - const struct intel_crtc_state *old_crtc_state) -{ - struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); - - if (!old_crtc_state->has_drrs) - return; - - mutex_lock(&dev_priv->drrs.mutex); - if (!dev_priv->drrs.dp) { - mutex_unlock(&dev_priv->drrs.mutex); - return; - } - - intel_edp_drrs_disable_locked(intel_dp, old_crtc_state); - mutex_unlock(&dev_priv->drrs.mutex); - - cancel_delayed_work_sync(&dev_priv->drrs.work); -} - -/** - * intel_edp_drrs_update - Update DRRS state - * @intel_dp: Intel DP - * @crtc_state: new CRTC state - * - * This function will update DRRS states, disabling or enabling DRRS when - * executing fastsets. For full modeset, intel_edp_drrs_disable() and - * intel_edp_drrs_enable() should be called instead. - */ -void -intel_edp_drrs_update(struct intel_dp *intel_dp, - const struct intel_crtc_state *crtc_state) -{ - struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); - - if (dev_priv->drrs.type != SEAMLESS_DRRS_SUPPORT) - return; - - mutex_lock(&dev_priv->drrs.mutex); - - /* New state matches current one? */ - if (crtc_state->has_drrs == !!dev_priv->drrs.dp) - goto unlock; - - if (crtc_state->has_drrs) - intel_edp_drrs_enable_locked(intel_dp); - else - intel_edp_drrs_disable_locked(intel_dp, crtc_state); - -unlock: - mutex_unlock(&dev_priv->drrs.mutex); -} - -static void intel_edp_drrs_downclock_work(struct work_struct *work) -{ - struct drm_i915_private *dev_priv = - container_of(work, typeof(*dev_priv), drrs.work.work); - struct intel_dp *intel_dp; - - mutex_lock(&dev_priv->drrs.mutex); - - intel_dp = dev_priv->drrs.dp; - - if (!intel_dp) - goto unlock; - - /* - * The delayed work can race with an invalidate hence we need to - * recheck. - */ - - if (dev_priv->drrs.busy_frontbuffer_bits) - goto unlock; - - if (dev_priv->drrs.refresh_rate_type != DRRS_LOW_RR) { - struct drm_crtc *crtc = dp_to_dig_port(intel_dp)->base.base.crtc; - - intel_dp_set_drrs_state(dev_priv, to_intel_crtc(crtc)->config, - drm_mode_vrefresh(intel_dp->attached_connector->panel.downclock_mode)); - } - -unlock: - mutex_unlock(&dev_priv->drrs.mutex); -} - -/** - * intel_edp_drrs_invalidate - Disable Idleness DRRS - * @dev_priv: i915 device - * @frontbuffer_bits: frontbuffer plane tracking bits - * - * This function gets called everytime rendering on the given planes start. - * Hence DRRS needs to be Upclocked, i.e. (LOW_RR -> HIGH_RR). - * - * Dirty frontbuffers relevant to DRRS are tracked in busy_frontbuffer_bits. - */ -void intel_edp_drrs_invalidate(struct drm_i915_private *dev_priv, - unsigned int frontbuffer_bits) -{ - struct intel_dp *intel_dp; - struct drm_crtc *crtc; - enum pipe pipe; - - if (dev_priv->drrs.type == DRRS_NOT_SUPPORTED) - return; - - cancel_delayed_work(&dev_priv->drrs.work); - - mutex_lock(&dev_priv->drrs.mutex); - - intel_dp = dev_priv->drrs.dp; - if (!intel_dp) { - mutex_unlock(&dev_priv->drrs.mutex); - return; - } - - crtc = dp_to_dig_port(intel_dp)->base.base.crtc; - pipe = to_intel_crtc(crtc)->pipe; - - frontbuffer_bits &= INTEL_FRONTBUFFER_ALL_MASK(pipe); - dev_priv->drrs.busy_frontbuffer_bits |= frontbuffer_bits; - - /* invalidate means busy screen hence upclock */ - if (frontbuffer_bits && dev_priv->drrs.refresh_rate_type == DRRS_LOW_RR) - intel_dp_set_drrs_state(dev_priv, to_intel_crtc(crtc)->config, - drm_mode_vrefresh(intel_dp->attached_connector->panel.fixed_mode)); - - mutex_unlock(&dev_priv->drrs.mutex); -} - -/** - * intel_edp_drrs_flush - Restart Idleness DRRS - * @dev_priv: i915 device - * @frontbuffer_bits: frontbuffer plane tracking bits - * - * This function gets called every time rendering on the given planes has - * completed or flip on a crtc is completed. So DRRS should be upclocked - * (LOW_RR -> HIGH_RR). And also Idleness detection should be started again, - * if no other planes are dirty. - * - * Dirty frontbuffers relevant to DRRS are tracked in busy_frontbuffer_bits. - */ -void intel_edp_drrs_flush(struct drm_i915_private *dev_priv, - unsigned int frontbuffer_bits) -{ - struct intel_dp *intel_dp; - struct drm_crtc *crtc; - enum pipe pipe; - - if (dev_priv->drrs.type == DRRS_NOT_SUPPORTED) - return; - - cancel_delayed_work(&dev_priv->drrs.work); - - mutex_lock(&dev_priv->drrs.mutex); - - intel_dp = dev_priv->drrs.dp; - if (!intel_dp) { - mutex_unlock(&dev_priv->drrs.mutex); - return; - } - - crtc = dp_to_dig_port(intel_dp)->base.base.crtc; - pipe = to_intel_crtc(crtc)->pipe; - - frontbuffer_bits &= INTEL_FRONTBUFFER_ALL_MASK(pipe); - dev_priv->drrs.busy_frontbuffer_bits &= ~frontbuffer_bits; - - /* flush means busy screen hence upclock */ - if (frontbuffer_bits && dev_priv->drrs.refresh_rate_type == DRRS_LOW_RR) - intel_dp_set_drrs_state(dev_priv, to_intel_crtc(crtc)->config, - drm_mode_vrefresh(intel_dp->attached_connector->panel.fixed_mode)); - - /* - * flush also means no more activity hence schedule downclock, if all - * other fbs are quiescent too - */ - if (!dev_priv->drrs.busy_frontbuffer_bits) - schedule_delayed_work(&dev_priv->drrs.work, - msecs_to_jiffies(1000)); - mutex_unlock(&dev_priv->drrs.mutex); -} - -/** - * DOC: Display Refresh Rate Switching (DRRS) - * - * Display Refresh Rate Switching (DRRS) is a power conservation feature - * which enables swtching between low and high refresh rates, - * dynamically, based on the usage scenario. This feature is applicable - * for internal panels. - * - * Indication that the panel supports DRRS is given by the panel EDID, which - * would list multiple refresh rates for one resolution. - * - * DRRS is of 2 types - static and seamless. - * Static DRRS involves changing refresh rate (RR) by doing a full modeset - * (may appear as a blink on screen) and is used in dock-undock scenario. - * Seamless DRRS involves changing RR without any visual effect to the user - * and can be used during normal system usage. This is done by programming - * certain registers. - * - * Support for static/seamless DRRS may be indicated in the VBT based on - * inputs from the panel spec. - * - * DRRS saves power by switching to low RR based on usage scenarios. - * - * The implementation is based on frontbuffer tracking implementation. When - * there is a disturbance on the screen triggered by user activity or a periodic - * system activity, DRRS is disabled (RR is changed to high RR). When there is - * no movement on screen, after a timeout of 1 second, a switch to low RR is - * made. - * - * For integration with frontbuffer tracking code, intel_edp_drrs_invalidate() - * and intel_edp_drrs_flush() are called. - * - * DRRS can be further extended to support other internal panels and also - * the scenario of video playback wherein RR is set based on the rate - * requested by userspace. - */ - -/** - * intel_dp_drrs_init - Init basic DRRS work and mutex. - * @connector: eDP connector - * @fixed_mode: preferred mode of panel - * - * This function is called only once at driver load to initialize basic - * DRRS stuff. - * - * Returns: - * Downclock mode if panel supports it, else return NULL. - * DRRS support is determined by the presence of downclock mode (apart - * from VBT setting). - */ -static struct drm_display_mode * -intel_dp_drrs_init(struct intel_connector *connector, - struct drm_display_mode *fixed_mode) -{ - struct drm_i915_private *dev_priv = to_i915(connector->base.dev); - struct drm_display_mode *downclock_mode = NULL; - - INIT_DELAYED_WORK(&dev_priv->drrs.work, intel_edp_drrs_downclock_work); - mutex_init(&dev_priv->drrs.mutex); - - if (DISPLAY_VER(dev_priv) <= 6) { - drm_dbg_kms(&dev_priv->drm, - "DRRS supported for Gen7 and above\n"); - return NULL; - } - - if (dev_priv->vbt.drrs_type != SEAMLESS_DRRS_SUPPORT) { - drm_dbg_kms(&dev_priv->drm, "VBT doesn't support DRRS\n"); - return NULL; - } - - downclock_mode = intel_panel_edid_downclock_mode(connector, fixed_mode); - if (!downclock_mode) { - drm_dbg_kms(&dev_priv->drm, - "Downclock mode is not found. DRRS not supported\n"); - return NULL; - } - - dev_priv->drrs.type = dev_priv->vbt.drrs_type; - - dev_priv->drrs.refresh_rate_type = DRRS_HIGH_RR; - drm_dbg_kms(&dev_priv->drm, - "seamless DRRS supported for eDP panel.\n"); - return downclock_mode; -} - static bool intel_edp_init_connector(struct intel_dp *intel_dp, struct intel_connector *intel_connector) { @@ -5200,7 +4878,10 @@ static bool intel_edp_init_connector(struct intel_dp *intel_dp, fixed_mode = intel_panel_edid_fixed_mode(intel_connector); if (fixed_mode) - downclock_mode = intel_dp_drrs_init(intel_connector, fixed_mode); + downclock_mode = intel_drrs_init(intel_connector, fixed_mode); + + /* MSO requires information from the EDID */ + intel_edp_mso_init(intel_dp); /* multiply the mode clock and horizontal timings for MSO */ intel_edp_mso_mode_fixup(intel_connector, fixed_mode); @@ -5233,7 +4914,7 @@ static bool intel_edp_init_connector(struct intel_dp *intel_dp, intel_panel_init(&intel_connector->panel, fixed_mode, downclock_mode); if (!(dev_priv->quirks & QUIRK_NO_PPS_BACKLIGHT_POWER_HOOK)) intel_connector->panel.backlight.power = intel_pps_backlight_power; - intel_panel_setup_backlight(connector, pipe); + intel_backlight_setup(intel_connector, pipe); if (fixed_mode) { drm_connector_set_panel_orientation_with_quirk(connector, @@ -5295,8 +4976,6 @@ intel_dp_init_connector(struct intel_digital_port *dig_port, intel_encoder->base.name)) return false; - intel_dp_set_source_rates(intel_dp); - intel_dp->reset_link_params = true; intel_dp->pps.pps_pipe = INVALID_PIPE; intel_dp->pps.active_pipe = INVALID_PIPE; @@ -5312,28 +4991,22 @@ intel_dp_init_connector(struct intel_digital_port *dig_port, */ drm_WARN_ON(dev, intel_phy_is_tc(dev_priv, phy)); type = DRM_MODE_CONNECTOR_eDP; + intel_encoder->type = INTEL_OUTPUT_EDP; + + /* eDP only on port B and/or C on vlv/chv */ + if (drm_WARN_ON(dev, (IS_VALLEYVIEW(dev_priv) || + IS_CHERRYVIEW(dev_priv)) && + port != PORT_B && port != PORT_C)) + return false; } else { type = DRM_MODE_CONNECTOR_DisplayPort; } + intel_dp_set_source_rates(intel_dp); + if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) intel_dp->pps.active_pipe = vlv_active_pipe(intel_dp); - /* - * For eDP we always set the encoder type to INTEL_OUTPUT_EDP, but - * for DP the encoder type can be set by the caller to - * INTEL_OUTPUT_UNKNOWN for DDI, so don't rewrite it. - */ - if (type == DRM_MODE_CONNECTOR_eDP) - intel_encoder->type = INTEL_OUTPUT_EDP; - - /* eDP only on port B and/or C on vlv/chv */ - if (drm_WARN_ON(dev, (IS_VALLEYVIEW(dev_priv) || - IS_CHERRYVIEW(dev_priv)) && - intel_dp_is_edp(intel_dp) && - port != PORT_B && port != PORT_C)) - return false; - drm_dbg_kms(&dev_priv->drm, "Adding %s connector on [ENCODER:%d:%s]\n", type == DRM_MODE_CONNECTOR_eDP ? "eDP" : "DP", @@ -5414,7 +5087,7 @@ void intel_dp_mst_suspend(struct drm_i915_private *dev_priv) intel_dp = enc_to_intel_dp(encoder); - if (!intel_dp->can_mst) + if (!intel_dp_mst_source_support(intel_dp)) continue; if (intel_dp->is_mst) @@ -5438,7 +5111,7 @@ void intel_dp_mst_resume(struct drm_i915_private *dev_priv) intel_dp = enc_to_intel_dp(encoder); - if (!intel_dp->can_mst) + if (!intel_dp_mst_source_support(intel_dp)) continue; ret = drm_dp_mst_topology_mgr_resume(&intel_dp->mst_mgr, diff --git a/drivers/gpu/drm/i915/display/intel_dp.h b/drivers/gpu/drm/i915/display/intel_dp.h index 680631b5b437..ce229026dc91 100644 --- a/drivers/gpu/drm/i915/display/intel_dp.h +++ b/drivers/gpu/drm/i915/display/intel_dp.h @@ -26,7 +26,7 @@ struct intel_dp; struct intel_encoder; struct link_config_limits { - int min_clock, max_clock; + int min_rate, max_rate; int min_lane_count, max_lane_count; int min_bpp, max_bpp; }; @@ -58,6 +58,7 @@ int intel_dp_compute_config(struct intel_encoder *encoder, struct intel_crtc_state *pipe_config, struct drm_connector_state *conn_state); bool intel_dp_is_edp(struct intel_dp *intel_dp); +bool intel_dp_is_uhbr(const struct intel_crtc_state *crtc_state); bool intel_dp_is_port_edp(struct drm_i915_private *dev_priv, enum port port); enum irqreturn intel_dp_hpd_pulse(struct intel_digital_port *dig_port, bool long_hpd); @@ -70,25 +71,14 @@ int intel_dp_max_link_rate(struct intel_dp *intel_dp); int intel_dp_max_lane_count(struct intel_dp *intel_dp); int intel_dp_rate_select(struct intel_dp *intel_dp, int rate); -void intel_edp_drrs_enable(struct intel_dp *intel_dp, - const struct intel_crtc_state *crtc_state); -void intel_edp_drrs_disable(struct intel_dp *intel_dp, - const struct intel_crtc_state *crtc_state); -void intel_edp_drrs_update(struct intel_dp *intel_dp, - const struct intel_crtc_state *crtc_state); -void intel_edp_drrs_invalidate(struct drm_i915_private *dev_priv, - unsigned int frontbuffer_bits); -void intel_edp_drrs_flush(struct drm_i915_private *dev_priv, - unsigned int frontbuffer_bits); - void intel_dp_compute_rate(struct intel_dp *intel_dp, int port_clock, u8 *link_bw, u8 *rate_select); -bool intel_dp_source_supports_hbr2(struct intel_dp *intel_dp); -bool intel_dp_source_supports_hbr3(struct intel_dp *intel_dp); +bool intel_dp_source_supports_tps3(struct drm_i915_private *i915); +bool intel_dp_source_supports_tps4(struct drm_i915_private *i915); bool intel_dp_get_colorimetry_status(struct intel_dp *intel_dp); int intel_dp_link_required(int pixel_clock, int bpp); -int intel_dp_max_data_rate(int max_link_clock, int max_lanes); +int intel_dp_max_data_rate(int max_link_rate, int max_lanes); bool intel_dp_can_bigjoiner(struct intel_dp *intel_dp); bool intel_dp_needs_vsc_sdp(const struct intel_crtc_state *crtc_state, const struct drm_connector_state *conn_state); @@ -98,7 +88,7 @@ void intel_dp_compute_psr_vsc_sdp(struct intel_dp *intel_dp, struct drm_dp_vsc_sdp *vsc); void intel_write_dp_vsc_sdp(struct intel_encoder *encoder, const struct intel_crtc_state *crtc_state, - struct drm_dp_vsc_sdp *vsc); + const struct drm_dp_vsc_sdp *vsc); void intel_dp_set_infoframes(struct intel_encoder *encoder, bool enable, const struct intel_crtc_state *crtc_state, const struct drm_connector_state *conn_state); diff --git a/drivers/gpu/drm/i915/display/intel_dp_aux.c b/drivers/gpu/drm/i915/display/intel_dp_aux.c index f483f479dd0b..5fbb767fcd63 100644 --- a/drivers/gpu/drm/i915/display/intel_dp_aux.c +++ b/drivers/gpu/drm/i915/display/intel_dp_aux.c @@ -150,9 +150,6 @@ static u32 skl_get_aux_send_ctl(struct intel_dp *intel_dp, u32 unused) { struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp); - struct drm_i915_private *i915 = - to_i915(dig_port->base.base.dev); - enum phy phy = intel_port_to_phy(i915, dig_port->base.port); u32 ret; /* @@ -170,8 +167,7 @@ static u32 skl_get_aux_send_ctl(struct intel_dp *intel_dp, DP_AUX_CH_CTL_FW_SYNC_PULSE_SKL(32) | DP_AUX_CH_CTL_SYNC_PULSE_SKL(32); - if (intel_phy_is_tc(i915, phy) && - dig_port->tc_mode == TC_PORT_TBT_ALT) + if (intel_tc_port_in_tbt_alt_mode(dig_port)) ret |= DP_AUX_CH_CTL_TBT_IO; return ret; diff --git a/drivers/gpu/drm/i915/display/intel_dp_aux_backlight.c b/drivers/gpu/drm/i915/display/intel_dp_aux_backlight.c index 6ac568617ef3..569d17b4d00f 100644 --- a/drivers/gpu/drm/i915/display/intel_dp_aux_backlight.c +++ b/drivers/gpu/drm/i915/display/intel_dp_aux_backlight.c @@ -34,9 +34,9 @@ * for some reason. */ +#include "intel_backlight.h" #include "intel_display_types.h" #include "intel_dp_aux_backlight.h" -#include "intel_panel.h" /* TODO: * Implement HDR, right now we just implement the bare minimum to bring us back into SDR mode so we @@ -146,7 +146,7 @@ intel_dp_aux_hdr_get_backlight(struct intel_connector *connector, enum pipe pipe if (!panel->backlight.edp.intel.sdr_uses_aux) { u32 pwm_level = panel->backlight.pwm_funcs->get(connector, pipe); - return intel_panel_backlight_level_from_pwm(connector, pwm_level); + return intel_backlight_level_from_pwm(connector, pwm_level); } /* Assume 100% brightness if backlight controls aren't enabled yet */ @@ -187,9 +187,9 @@ intel_dp_aux_hdr_set_backlight(const struct drm_connector_state *conn_state, u32 if (panel->backlight.edp.intel.sdr_uses_aux) { intel_dp_aux_hdr_set_aux_backlight(conn_state, level); } else { - const u32 pwm_level = intel_panel_backlight_level_to_pwm(connector, level); + const u32 pwm_level = intel_backlight_level_to_pwm(connector, level); - intel_panel_set_pwm_level(conn_state, pwm_level); + intel_backlight_set_pwm_level(conn_state, pwm_level); } } @@ -215,7 +215,7 @@ intel_dp_aux_hdr_enable_backlight(const struct intel_crtc_state *crtc_state, ctrl |= INTEL_EDP_HDR_TCON_BRIGHTNESS_AUX_ENABLE; intel_dp_aux_hdr_set_aux_backlight(conn_state, level); } else { - u32 pwm_level = intel_panel_backlight_level_to_pwm(connector, level); + u32 pwm_level = intel_backlight_level_to_pwm(connector, level); panel->backlight.pwm_funcs->enable(crtc_state, conn_state, pwm_level); @@ -238,7 +238,7 @@ intel_dp_aux_hdr_disable_backlight(const struct drm_connector_state *conn_state, return; /* Note we want the actual pwm_level to be 0, regardless of pwm_min */ - panel->backlight.pwm_funcs->disable(conn_state, intel_panel_invert_pwm_level(connector, 0)); + panel->backlight.pwm_funcs->disable(conn_state, intel_backlight_invert_pwm_level(connector, 0)); } static int diff --git a/drivers/gpu/drm/i915/display/intel_dp_hdcp.c b/drivers/gpu/drm/i915/display/intel_dp_hdcp.c index d697d169e8c1..540a669e01dd 100644 --- a/drivers/gpu/drm/i915/display/intel_dp_hdcp.c +++ b/drivers/gpu/drm/i915/display/intel_dp_hdcp.c @@ -446,8 +446,6 @@ static int intel_dp_hdcp2_write_msg(struct intel_digital_port *dig_port, void *buf, size_t size) { - struct intel_dp *dp = &dig_port->dp; - struct intel_hdcp *hdcp = &dp->attached_connector->hdcp; unsigned int offset; u8 *byte = buf; ssize_t ret, bytes_to_write, len; @@ -463,8 +461,6 @@ int intel_dp_hdcp2_write_msg(struct intel_digital_port *dig_port, bytes_to_write = size - 1; byte++; - hdcp->cp_irq_count_cached = atomic_read(&hdcp->cp_irq_count); - while (bytes_to_write) { len = bytes_to_write > DP_AUX_MAX_PAYLOAD_BYTES ? DP_AUX_MAX_PAYLOAD_BYTES : bytes_to_write; @@ -482,29 +478,11 @@ int intel_dp_hdcp2_write_msg(struct intel_digital_port *dig_port, return size; } -static int -get_rxinfo_hdcp_1_dev_downstream(struct intel_digital_port *dig_port, bool *hdcp_1_x) -{ - u8 rx_info[HDCP_2_2_RXINFO_LEN]; - int ret; - - ret = drm_dp_dpcd_read(&dig_port->dp.aux, - DP_HDCP_2_2_REG_RXINFO_OFFSET, - (void *)rx_info, HDCP_2_2_RXINFO_LEN); - - if (ret != HDCP_2_2_RXINFO_LEN) - return ret >= 0 ? -EIO : ret; - - *hdcp_1_x = HDCP_2_2_HDCP1_DEVICE_CONNECTED(rx_info[1]) ? true : false; - return 0; -} - static -ssize_t get_receiver_id_list_size(struct intel_digital_port *dig_port) +ssize_t get_receiver_id_list_rx_info(struct intel_digital_port *dig_port, u32 *dev_cnt, u8 *byte) { - u8 rx_info[HDCP_2_2_RXINFO_LEN]; - u32 dev_cnt; ssize_t ret; + u8 *rx_info = byte; ret = drm_dp_dpcd_read(&dig_port->dp.aux, DP_HDCP_2_2_REG_RXINFO_OFFSET, @@ -512,15 +490,11 @@ ssize_t get_receiver_id_list_size(struct intel_digital_port *dig_port) if (ret != HDCP_2_2_RXINFO_LEN) return ret >= 0 ? -EIO : ret; - dev_cnt = (HDCP_2_2_DEV_COUNT_HI(rx_info[0]) << 4 | + *dev_cnt = (HDCP_2_2_DEV_COUNT_HI(rx_info[0]) << 4 | HDCP_2_2_DEV_COUNT_LO(rx_info[1])); - if (dev_cnt > HDCP_2_2_MAX_DEVICE_COUNT) - dev_cnt = HDCP_2_2_MAX_DEVICE_COUNT; - - ret = sizeof(struct hdcp2_rep_send_receiverid_list) - - HDCP_2_2_RECEIVER_IDS_MAX_LEN + - (dev_cnt * HDCP_2_2_RECEIVER_ID_LEN); + if (*dev_cnt > HDCP_2_2_MAX_DEVICE_COUNT) + *dev_cnt = HDCP_2_2_MAX_DEVICE_COUNT; return ret; } @@ -530,12 +504,15 @@ int intel_dp_hdcp2_read_msg(struct intel_digital_port *dig_port, u8 msg_id, void *buf, size_t size) { struct drm_i915_private *i915 = to_i915(dig_port->base.base.dev); + struct intel_dp *dp = &dig_port->dp; + struct intel_hdcp *hdcp = &dp->attached_connector->hdcp; unsigned int offset; u8 *byte = buf; ssize_t ret, bytes_to_recv, len; const struct hdcp2_dp_msg_data *hdcp2_msg_data; ktime_t msg_end = ktime_set(0, 0); bool msg_expired; + u32 dev_cnt; hdcp2_msg_data = get_hdcp2_dp_msg_data(msg_id); if (!hdcp2_msg_data) @@ -546,17 +523,24 @@ int intel_dp_hdcp2_read_msg(struct intel_digital_port *dig_port, if (ret < 0) return ret; + hdcp->cp_irq_count_cached = atomic_read(&hdcp->cp_irq_count); + + /* DP adaptation msgs has no msg_id */ + byte++; + if (msg_id == HDCP_2_2_REP_SEND_RECVID_LIST) { - ret = get_receiver_id_list_size(dig_port); + ret = get_receiver_id_list_rx_info(dig_port, &dev_cnt, byte); if (ret < 0) return ret; - size = ret; + byte += ret; + size = sizeof(struct hdcp2_rep_send_receiverid_list) - + HDCP_2_2_RXINFO_LEN - HDCP_2_2_RECEIVER_IDS_MAX_LEN + + (dev_cnt * HDCP_2_2_RECEIVER_ID_LEN); + offset += HDCP_2_2_RXINFO_LEN; } - bytes_to_recv = size - 1; - /* DP adaptation msgs has no msg_id */ - byte++; + bytes_to_recv = size - 1; while (bytes_to_recv) { len = bytes_to_recv > DP_AUX_MAX_PAYLOAD_BYTES ? @@ -664,27 +648,6 @@ int intel_dp_hdcp2_capable(struct intel_digital_port *dig_port, return 0; } -static -int intel_dp_mst_streams_type1_capable(struct intel_connector *connector, - bool *capable) -{ - struct intel_digital_port *dig_port = intel_attached_dig_port(connector); - struct drm_i915_private *i915 = to_i915(dig_port->base.base.dev); - int ret; - bool hdcp_1_x; - - ret = get_rxinfo_hdcp_1_dev_downstream(dig_port, &hdcp_1_x); - if (ret) { - drm_dbg_kms(&i915->drm, - "[%s:%d] failed to read RxInfo ret=%d\n", - connector->base.name, connector->base.base.id, ret); - return ret; - } - - *capable = !hdcp_1_x; - return 0; -} - static const struct intel_hdcp_shim intel_dp_hdcp_shim = { .write_an_aksv = intel_dp_hdcp_write_an_aksv, .read_bksv = intel_dp_hdcp_read_bksv, @@ -833,7 +796,6 @@ static const struct intel_hdcp_shim intel_dp_mst_hdcp_shim = { .stream_2_2_encryption = intel_dp_mst_hdcp2_stream_encryption, .check_2_2_link = intel_dp_mst_hdcp2_check_link, .hdcp_2_2_capable = intel_dp_hdcp2_capable, - .streams_type1_capable = intel_dp_mst_streams_type1_capable, .protocol = HDCP_PROTOCOL_DP, }; diff --git a/drivers/gpu/drm/i915/display/intel_dp_link_training.c b/drivers/gpu/drm/i915/display/intel_dp_link_training.c index 508a514c5e37..85676c953e0a 100644 --- a/drivers/gpu/drm/i915/display/intel_dp_link_training.c +++ b/drivers/gpu/drm/i915/display/intel_dp_link_training.c @@ -25,15 +25,6 @@ #include "intel_dp.h" #include "intel_dp_link_training.h" -static void -intel_dp_dump_link_status(struct drm_device *drm, - const u8 link_status[DP_LINK_STATUS_SIZE]) -{ - drm_dbg_kms(drm, - "ln0_1:0x%x ln2_3:0x%x align:0x%x sink:0x%x adj_req0_1:0x%x adj_req2_3:0x%x\n", - link_status[0], link_status[1], link_status[2], - link_status[3], link_status[4], link_status[5]); -} static void intel_dp_reset_lttpr_common_caps(struct intel_dp *intel_dp) { @@ -66,6 +57,7 @@ static u8 *intel_dp_lttpr_phy_caps(struct intel_dp *intel_dp, static void intel_dp_read_lttpr_phy_caps(struct intel_dp *intel_dp, enum drm_dp_phy dp_phy) { + struct intel_encoder *encoder = &dp_to_dig_port(intel_dp)->base; u8 *phy_caps = intel_dp_lttpr_phy_caps(intel_dp, dp_phy); char phy_name[10]; @@ -73,21 +65,22 @@ static void intel_dp_read_lttpr_phy_caps(struct intel_dp *intel_dp, if (drm_dp_read_lttpr_phy_caps(&intel_dp->aux, dp_phy, phy_caps) < 0) { drm_dbg_kms(&dp_to_i915(intel_dp)->drm, - "failed to read the PHY caps for %s\n", - phy_name); + "[ENCODER:%d:%s][%s] failed to read the PHY caps\n", + encoder->base.base.id, encoder->base.name, phy_name); return; } drm_dbg_kms(&dp_to_i915(intel_dp)->drm, - "%s PHY capabilities: %*ph\n", - phy_name, + "[ENCODER:%d:%s][%s] PHY capabilities: %*ph\n", + encoder->base.base.id, encoder->base.name, phy_name, (int)sizeof(intel_dp->lttpr_phy_caps[0]), phy_caps); } static bool intel_dp_read_lttpr_common_caps(struct intel_dp *intel_dp) { - struct drm_i915_private *i915 = dp_to_i915(intel_dp); + struct intel_encoder *encoder = &dp_to_dig_port(intel_dp)->base; + struct drm_i915_private *i915 = to_i915(encoder->base.dev); if (intel_dp_is_edp(intel_dp)) return false; @@ -104,7 +97,8 @@ static bool intel_dp_read_lttpr_common_caps(struct intel_dp *intel_dp) goto reset_caps; drm_dbg_kms(&dp_to_i915(intel_dp)->drm, - "LTTPR common capabilities: %*ph\n", + "[ENCODER:%d:%s] LTTPR common capabilities: %*ph\n", + encoder->base.base.id, encoder->base.name, (int)sizeof(intel_dp->lttpr_common_caps), intel_dp->lttpr_common_caps); @@ -130,6 +124,8 @@ intel_dp_set_lttpr_transparent_mode(struct intel_dp *intel_dp, bool enable) static int intel_dp_init_lttpr(struct intel_dp *intel_dp) { + struct intel_encoder *encoder = &dp_to_dig_port(intel_dp)->base; + struct drm_i915_private *i915 = to_i915(encoder->base.dev); int lttpr_count; int i; @@ -161,8 +157,9 @@ static int intel_dp_init_lttpr(struct intel_dp *intel_dp) return 0; if (!intel_dp_set_lttpr_transparent_mode(intel_dp, false)) { - drm_dbg_kms(&dp_to_i915(intel_dp)->drm, - "Switching to LTTPR non-transparent LT mode failed, fall-back to transparent mode\n"); + drm_dbg_kms(&i915->drm, + "[ENCODER:%d:%s] Switching to LTTPR non-transparent LT mode failed, fall-back to transparent mode\n", + encoder->base.base.id, encoder->base.name); intel_dp_set_lttpr_transparent_mode(intel_dp, true); intel_dp_reset_lttpr_count(intel_dp); @@ -301,21 +298,54 @@ static u8 intel_dp_phy_preemph_max(struct intel_dp *intel_dp, return preemph_max; } -void -intel_dp_get_adjust_train(struct intel_dp *intel_dp, - const struct intel_crtc_state *crtc_state, - enum drm_dp_phy dp_phy, - const u8 link_status[DP_LINK_STATUS_SIZE]) +static bool has_per_lane_signal_levels(struct intel_dp *intel_dp, + enum drm_dp_phy dp_phy) +{ + return !intel_dp_phy_is_downstream_of_source(intel_dp, dp_phy); +} + +/* 128b/132b */ +static u8 intel_dp_get_lane_adjust_tx_ffe_preset(struct intel_dp *intel_dp, + const struct intel_crtc_state *crtc_state, + enum drm_dp_phy dp_phy, + const u8 link_status[DP_LINK_STATUS_SIZE], + int lane) +{ + u8 tx_ffe = 0; + + if (has_per_lane_signal_levels(intel_dp, dp_phy)) { + lane = min(lane, crtc_state->lane_count - 1); + tx_ffe = drm_dp_get_adjust_tx_ffe_preset(link_status, lane); + } else { + for (lane = 0; lane < crtc_state->lane_count; lane++) + tx_ffe = max(tx_ffe, drm_dp_get_adjust_tx_ffe_preset(link_status, lane)); + } + + return tx_ffe; +} + +/* 8b/10b */ +static u8 intel_dp_get_lane_adjust_vswing_preemph(struct intel_dp *intel_dp, + const struct intel_crtc_state *crtc_state, + enum drm_dp_phy dp_phy, + const u8 link_status[DP_LINK_STATUS_SIZE], + int lane) { u8 v = 0; u8 p = 0; - int lane; u8 voltage_max; u8 preemph_max; - for (lane = 0; lane < crtc_state->lane_count; lane++) { - v = max(v, drm_dp_get_adjust_request_voltage(link_status, lane)); - p = max(p, drm_dp_get_adjust_request_pre_emphasis(link_status, lane)); + if (has_per_lane_signal_levels(intel_dp, dp_phy)) { + lane = min(lane, crtc_state->lane_count - 1); + + v = drm_dp_get_adjust_request_voltage(link_status, lane); + p = drm_dp_get_adjust_request_pre_emphasis(link_status, lane); + } else { + for (lane = 0; lane < crtc_state->lane_count; lane++) { + v = max(v, drm_dp_get_adjust_request_voltage(link_status, lane)); + p = max(p, drm_dp_get_adjust_request_pre_emphasis(link_status, lane)); + } } preemph_max = intel_dp_phy_preemph_max(intel_dp, dp_phy); @@ -328,8 +358,79 @@ intel_dp_get_adjust_train(struct intel_dp *intel_dp, if (v >= voltage_max) v = voltage_max | DP_TRAIN_MAX_SWING_REACHED; + return v | p; +} + +static u8 intel_dp_get_lane_adjust_train(struct intel_dp *intel_dp, + const struct intel_crtc_state *crtc_state, + enum drm_dp_phy dp_phy, + const u8 link_status[DP_LINK_STATUS_SIZE], + int lane) +{ + if (intel_dp_is_uhbr(crtc_state)) + return intel_dp_get_lane_adjust_tx_ffe_preset(intel_dp, crtc_state, + dp_phy, link_status, lane); + else + return intel_dp_get_lane_adjust_vswing_preemph(intel_dp, crtc_state, + dp_phy, link_status, lane); +} + +#define TRAIN_REQ_FMT "%d/%d/%d/%d" +#define _TRAIN_REQ_VSWING_ARGS(link_status, lane) \ + (drm_dp_get_adjust_request_voltage((link_status), (lane)) >> DP_TRAIN_VOLTAGE_SWING_SHIFT) +#define TRAIN_REQ_VSWING_ARGS(link_status) \ + _TRAIN_REQ_VSWING_ARGS(link_status, 0), \ + _TRAIN_REQ_VSWING_ARGS(link_status, 1), \ + _TRAIN_REQ_VSWING_ARGS(link_status, 2), \ + _TRAIN_REQ_VSWING_ARGS(link_status, 3) +#define _TRAIN_REQ_PREEMPH_ARGS(link_status, lane) \ + (drm_dp_get_adjust_request_pre_emphasis((link_status), (lane)) >> DP_TRAIN_PRE_EMPHASIS_SHIFT) +#define TRAIN_REQ_PREEMPH_ARGS(link_status) \ + _TRAIN_REQ_PREEMPH_ARGS(link_status, 0), \ + _TRAIN_REQ_PREEMPH_ARGS(link_status, 1), \ + _TRAIN_REQ_PREEMPH_ARGS(link_status, 2), \ + _TRAIN_REQ_PREEMPH_ARGS(link_status, 3) +#define _TRAIN_REQ_TX_FFE_ARGS(link_status, lane) \ + drm_dp_get_adjust_tx_ffe_preset((link_status), (lane)) +#define TRAIN_REQ_TX_FFE_ARGS(link_status) \ + _TRAIN_REQ_TX_FFE_ARGS(link_status, 0), \ + _TRAIN_REQ_TX_FFE_ARGS(link_status, 1), \ + _TRAIN_REQ_TX_FFE_ARGS(link_status, 2), \ + _TRAIN_REQ_TX_FFE_ARGS(link_status, 3) + +void +intel_dp_get_adjust_train(struct intel_dp *intel_dp, + const struct intel_crtc_state *crtc_state, + enum drm_dp_phy dp_phy, + const u8 link_status[DP_LINK_STATUS_SIZE]) +{ + struct intel_encoder *encoder = &dp_to_dig_port(intel_dp)->base; + struct drm_i915_private *i915 = to_i915(encoder->base.dev); + char phy_name[10]; + int lane; + + if (intel_dp_is_uhbr(crtc_state)) { + drm_dbg_kms(&i915->drm, "[ENCODER:%d:%s][%s] 128b/132b, lanes: %d, " + "TX FFE request: " TRAIN_REQ_FMT "\n", + encoder->base.base.id, encoder->base.name, + intel_dp_phy_name(dp_phy, phy_name, sizeof(phy_name)), + crtc_state->lane_count, + TRAIN_REQ_TX_FFE_ARGS(link_status)); + } else { + drm_dbg_kms(&i915->drm, "[ENCODER:%d:%s][%s] 8b/10b, lanes: %d, " + "vswing request: " TRAIN_REQ_FMT ", " + "pre-emphasis request: " TRAIN_REQ_FMT "\n", + encoder->base.base.id, encoder->base.name, + intel_dp_phy_name(dp_phy, phy_name, sizeof(phy_name)), + crtc_state->lane_count, + TRAIN_REQ_VSWING_ARGS(link_status), + TRAIN_REQ_PREEMPH_ARGS(link_status)); + } + for (lane = 0; lane < 4; lane++) - intel_dp->train_set[lane] = v | p; + intel_dp->train_set[lane] = + intel_dp_get_lane_adjust_train(intel_dp, crtc_state, + dp_phy, link_status, lane); } static int intel_dp_training_pattern_set_reg(struct intel_dp *intel_dp, @@ -351,7 +452,7 @@ intel_dp_set_link_train(struct intel_dp *intel_dp, int len; intel_dp_program_link_training_pattern(intel_dp, crtc_state, - dp_train_pat); + dp_phy, dp_train_pat); buf[0] = dp_train_pat; /* DP_TRAINING_LANEx_SET follow DP_TRAINING_PATTERN_SET */ @@ -379,40 +480,77 @@ static char dp_training_pattern_name(u8 train_pat) void intel_dp_program_link_training_pattern(struct intel_dp *intel_dp, const struct intel_crtc_state *crtc_state, + enum drm_dp_phy dp_phy, u8 dp_train_pat) { struct intel_encoder *encoder = &dp_to_dig_port(intel_dp)->base; - struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); + struct drm_i915_private *i915 = to_i915(encoder->base.dev); u8 train_pat = intel_dp_training_pattern_symbol(dp_train_pat); + char phy_name[10]; if (train_pat != DP_TRAINING_PATTERN_DISABLE) - drm_dbg_kms(&dev_priv->drm, - "[ENCODER:%d:%s] Using DP training pattern TPS%c\n", + drm_dbg_kms(&i915->drm, + "[ENCODER:%d:%s][%s] Using DP training pattern TPS%c\n", encoder->base.base.id, encoder->base.name, + intel_dp_phy_name(dp_phy, phy_name, sizeof(phy_name)), dp_training_pattern_name(train_pat)); intel_dp->set_link_train(intel_dp, crtc_state, dp_train_pat); } +#define TRAIN_SET_FMT "%d%s/%d%s/%d%s/%d%s" +#define _TRAIN_SET_VSWING_ARGS(train_set) \ + ((train_set) & DP_TRAIN_VOLTAGE_SWING_MASK) >> DP_TRAIN_VOLTAGE_SWING_SHIFT, \ + (train_set) & DP_TRAIN_MAX_SWING_REACHED ? "(max)" : "" +#define TRAIN_SET_VSWING_ARGS(train_set) \ + _TRAIN_SET_VSWING_ARGS((train_set)[0]), \ + _TRAIN_SET_VSWING_ARGS((train_set)[1]), \ + _TRAIN_SET_VSWING_ARGS((train_set)[2]), \ + _TRAIN_SET_VSWING_ARGS((train_set)[3]) +#define _TRAIN_SET_PREEMPH_ARGS(train_set) \ + ((train_set) & DP_TRAIN_PRE_EMPHASIS_MASK) >> DP_TRAIN_PRE_EMPHASIS_SHIFT, \ + (train_set) & DP_TRAIN_MAX_PRE_EMPHASIS_REACHED ? "(max)" : "" +#define TRAIN_SET_PREEMPH_ARGS(train_set) \ + _TRAIN_SET_PREEMPH_ARGS((train_set)[0]), \ + _TRAIN_SET_PREEMPH_ARGS((train_set)[1]), \ + _TRAIN_SET_PREEMPH_ARGS((train_set)[2]), \ + _TRAIN_SET_PREEMPH_ARGS((train_set)[3]) +#define _TRAIN_SET_TX_FFE_ARGS(train_set) \ + ((train_set) & DP_TX_FFE_PRESET_VALUE_MASK), "" +#define TRAIN_SET_TX_FFE_ARGS(train_set) \ + _TRAIN_SET_TX_FFE_ARGS((train_set)[0]), \ + _TRAIN_SET_TX_FFE_ARGS((train_set)[1]), \ + _TRAIN_SET_TX_FFE_ARGS((train_set)[2]), \ + _TRAIN_SET_TX_FFE_ARGS((train_set)[3]) + void intel_dp_set_signal_levels(struct intel_dp *intel_dp, const struct intel_crtc_state *crtc_state, enum drm_dp_phy dp_phy) { - struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); - u8 train_set = intel_dp->train_set[0]; + struct intel_encoder *encoder = &dp_to_dig_port(intel_dp)->base; + struct drm_i915_private *i915 = to_i915(encoder->base.dev); char phy_name[10]; - drm_dbg_kms(&dev_priv->drm, "Using vswing level %d%s, pre-emphasis level %d%s, at %s\n", - train_set & DP_TRAIN_VOLTAGE_SWING_MASK, - train_set & DP_TRAIN_MAX_SWING_REACHED ? " (max)" : "", - (train_set & DP_TRAIN_PRE_EMPHASIS_MASK) >> - DP_TRAIN_PRE_EMPHASIS_SHIFT, - train_set & DP_TRAIN_MAX_PRE_EMPHASIS_REACHED ? - " (max)" : "", - intel_dp_phy_name(dp_phy, phy_name, sizeof(phy_name))); + if (intel_dp_is_uhbr(crtc_state)) { + drm_dbg_kms(&i915->drm, "[ENCODER:%d:%s][%s] 128b/132b, lanes: %d, " + "TX FFE presets: " TRAIN_SET_FMT "\n", + encoder->base.base.id, encoder->base.name, + intel_dp_phy_name(dp_phy, phy_name, sizeof(phy_name)), + crtc_state->lane_count, + TRAIN_SET_TX_FFE_ARGS(intel_dp->train_set)); + } else { + drm_dbg_kms(&i915->drm, "[ENCODER:%d:%s][%s] 8b/10b, lanes: %d, " + "vswing levels: " TRAIN_SET_FMT ", " + "pre-emphasis levels: " TRAIN_SET_FMT "\n", + encoder->base.base.id, encoder->base.name, + intel_dp_phy_name(dp_phy, phy_name, sizeof(phy_name)), + crtc_state->lane_count, + TRAIN_SET_VSWING_ARGS(intel_dp->train_set), + TRAIN_SET_PREEMPH_ARGS(intel_dp->train_set)); + } if (intel_dp_phy_is_downstream_of_source(intel_dp, dp_phy)) - intel_dp->set_signal_levels(intel_dp, crtc_state); + encoder->set_signal_levels(encoder, crtc_state); } static bool @@ -444,15 +582,55 @@ intel_dp_update_link_train(struct intel_dp *intel_dp, return ret == crtc_state->lane_count; } +/* 128b/132b */ +static bool intel_dp_lane_max_tx_ffe_reached(u8 train_set_lane) +{ + return (train_set_lane & DP_TX_FFE_PRESET_VALUE_MASK) == + DP_TX_FFE_PRESET_VALUE_MASK; +} + +/* + * 8b/10b + * + * FIXME: The DP spec is very confusing here, also the Link CTS spec seems to + * have self contradicting tests around this area. + * + * In lieu of better ideas let's just stop when we've reached the max supported + * vswing with its max pre-emphasis, which is either 2+1 or 3+0 depending on + * whether vswing level 3 is supported or not. + */ +static bool intel_dp_lane_max_vswing_reached(u8 train_set_lane) +{ + u8 v = (train_set_lane & DP_TRAIN_VOLTAGE_SWING_MASK) >> + DP_TRAIN_VOLTAGE_SWING_SHIFT; + u8 p = (train_set_lane & DP_TRAIN_PRE_EMPHASIS_MASK) >> + DP_TRAIN_PRE_EMPHASIS_SHIFT; + + if ((train_set_lane & DP_TRAIN_MAX_SWING_REACHED) == 0) + return false; + + if (v + p != 3) + return false; + + return true; +} + static bool intel_dp_link_max_vswing_reached(struct intel_dp *intel_dp, const struct intel_crtc_state *crtc_state) { int lane; - for (lane = 0; lane < crtc_state->lane_count; lane++) - if ((intel_dp->train_set[lane] & - DP_TRAIN_MAX_SWING_REACHED) == 0) - return false; + for (lane = 0; lane < crtc_state->lane_count; lane++) { + u8 train_set_lane = intel_dp->train_set[lane]; + + if (intel_dp_is_uhbr(crtc_state)) { + if (!intel_dp_lane_max_tx_ffe_reached(train_set_lane)) + return false; + } else { + if (!intel_dp_lane_max_vswing_reached(train_set_lane)) + return false; + } + } return true; } @@ -465,7 +643,8 @@ static bool intel_dp_prepare_link_train(struct intel_dp *intel_dp, const struct intel_crtc_state *crtc_state) { - struct drm_i915_private *i915 = dp_to_i915(intel_dp); + struct intel_encoder *encoder = &dp_to_dig_port(intel_dp)->base; + struct drm_i915_private *i915 = to_i915(encoder->base.dev); u8 link_config[2]; u8 link_bw, rate_select; @@ -477,10 +656,12 @@ intel_dp_prepare_link_train(struct intel_dp *intel_dp, if (link_bw) drm_dbg_kms(&i915->drm, - "Using LINK_BW_SET value %02x\n", link_bw); + "[ENCODER:%d:%s] Using LINK_BW_SET value %02x\n", + encoder->base.base.id, encoder->base.name, link_bw); else drm_dbg_kms(&i915->drm, - "Using LINK_RATE_SET value %02x\n", rate_select); + "[ENCODER:%d:%s] Using LINK_RATE_SET value %02x\n", + encoder->base.base.id, encoder->base.name, rate_select); /* Write the link configuration data */ link_config[0] = link_bw; @@ -495,11 +676,10 @@ intel_dp_prepare_link_train(struct intel_dp *intel_dp, &rate_select, 1); link_config[0] = crtc_state->vrr.enable ? DP_MSA_TIMING_PAR_IGNORE_EN : 0; - link_config[1] = DP_SET_ANSI_8B10B; + link_config[1] = intel_dp_is_uhbr(crtc_state) ? + DP_SET_ANSI_128B132B : DP_SET_ANSI_8B10B; drm_dp_dpcd_write(&intel_dp->aux, DP_DOWNSPREAD_CTRL, link_config, 2); - intel_dp->DP |= DP_PORT_EN; - return true; } @@ -512,6 +692,48 @@ static void intel_dp_link_training_clock_recovery_delay(struct intel_dp *intel_d drm_dp_lttpr_link_train_clock_recovery_delay(); } +static bool intel_dp_adjust_request_changed(const struct intel_crtc_state *crtc_state, + const u8 old_link_status[DP_LINK_STATUS_SIZE], + const u8 new_link_status[DP_LINK_STATUS_SIZE]) +{ + int lane; + + for (lane = 0; lane < crtc_state->lane_count; lane++) { + u8 old, new; + + if (intel_dp_is_uhbr(crtc_state)) { + old = drm_dp_get_adjust_tx_ffe_preset(old_link_status, lane); + new = drm_dp_get_adjust_tx_ffe_preset(new_link_status, lane); + } else { + old = drm_dp_get_adjust_request_voltage(old_link_status, lane) | + drm_dp_get_adjust_request_pre_emphasis(old_link_status, lane); + new = drm_dp_get_adjust_request_voltage(new_link_status, lane) | + drm_dp_get_adjust_request_pre_emphasis(new_link_status, lane); + } + + if (old != new) + return true; + } + + return false; +} + +static void +intel_dp_dump_link_status(struct intel_dp *intel_dp, enum drm_dp_phy dp_phy, + const u8 link_status[DP_LINK_STATUS_SIZE]) +{ + struct intel_encoder *encoder = &dp_to_dig_port(intel_dp)->base; + struct drm_i915_private *i915 = to_i915(encoder->base.dev); + char phy_name[10]; + + drm_dbg_kms(&i915->drm, + "[ENCODER:%d:%s][%s] ln0_1:0x%x ln2_3:0x%x align:0x%x sink:0x%x adj_req0_1:0x%x adj_req2_3:0x%x\n", + encoder->base.base.id, encoder->base.name, + intel_dp_phy_name(dp_phy, phy_name, sizeof(phy_name)), + link_status[0], link_status[1], link_status[2], + link_status[3], link_status[4], link_status[5]); +} + /* * Perform the link training clock recovery phase on the given DP PHY using * training pattern 1. @@ -521,16 +743,22 @@ intel_dp_link_training_clock_recovery(struct intel_dp *intel_dp, const struct intel_crtc_state *crtc_state, enum drm_dp_phy dp_phy) { - struct drm_i915_private *i915 = dp_to_i915(intel_dp); - u8 voltage; + struct intel_encoder *encoder = &dp_to_dig_port(intel_dp)->base; + struct drm_i915_private *i915 = to_i915(encoder->base.dev); + u8 old_link_status[DP_LINK_STATUS_SIZE] = {}; int voltage_tries, cr_tries, max_cr_tries; + u8 link_status[DP_LINK_STATUS_SIZE]; bool max_vswing_reached = false; + char phy_name[10]; + + intel_dp_phy_name(dp_phy, phy_name, sizeof(phy_name)); /* clock recovery */ if (!intel_dp_reset_link_train(intel_dp, crtc_state, dp_phy, DP_TRAINING_PATTERN_1 | DP_LINK_SCRAMBLING_DISABLE)) { - drm_err(&i915->drm, "failed to enable link training\n"); + drm_err(&i915->drm, "[ENCODER:%d:%s][%s] Failed to enable link training\n", + encoder->base.base.id, encoder->base.name, phy_name); return false; } @@ -549,105 +777,118 @@ intel_dp_link_training_clock_recovery(struct intel_dp *intel_dp, voltage_tries = 1; for (cr_tries = 0; cr_tries < max_cr_tries; ++cr_tries) { - u8 link_status[DP_LINK_STATUS_SIZE]; - intel_dp_link_training_clock_recovery_delay(intel_dp, dp_phy); if (drm_dp_dpcd_read_phy_link_status(&intel_dp->aux, dp_phy, link_status) < 0) { - drm_err(&i915->drm, "failed to get link status\n"); + drm_err(&i915->drm, "[ENCODER:%d:%s][%s] Failed to get link status\n", + encoder->base.base.id, encoder->base.name, phy_name); return false; } if (drm_dp_clock_recovery_ok(link_status, crtc_state->lane_count)) { - drm_dbg_kms(&i915->drm, "clock recovery OK\n"); + drm_dbg_kms(&i915->drm, + "[ENCODER:%d:%s][%s] Clock recovery OK\n", + encoder->base.base.id, encoder->base.name, phy_name); return true; } if (voltage_tries == 5) { + intel_dp_dump_link_status(intel_dp, dp_phy, link_status); drm_dbg_kms(&i915->drm, - "Same voltage tried 5 times\n"); + "[ENCODER:%d:%s][%s] Same voltage tried 5 times\n", + encoder->base.base.id, encoder->base.name, phy_name); return false; } if (max_vswing_reached) { - drm_dbg_kms(&i915->drm, "Max Voltage Swing reached\n"); + intel_dp_dump_link_status(intel_dp, dp_phy, link_status); + drm_dbg_kms(&i915->drm, + "[ENCODER:%d:%s][%s] Max Voltage Swing reached\n", + encoder->base.base.id, encoder->base.name, phy_name); return false; } - voltage = intel_dp->train_set[0] & DP_TRAIN_VOLTAGE_SWING_MASK; - /* Update training set as requested by target */ intel_dp_get_adjust_train(intel_dp, crtc_state, dp_phy, link_status); if (!intel_dp_update_link_train(intel_dp, crtc_state, dp_phy)) { drm_err(&i915->drm, - "failed to update link training\n"); + "[ENCODER:%d:%s][%s] Failed to update link training\n", + encoder->base.base.id, encoder->base.name, phy_name); return false; } - if ((intel_dp->train_set[0] & DP_TRAIN_VOLTAGE_SWING_MASK) == - voltage) + if (!intel_dp_adjust_request_changed(crtc_state, old_link_status, link_status)) ++voltage_tries; else voltage_tries = 1; + memcpy(old_link_status, link_status, sizeof(link_status)); + if (intel_dp_link_max_vswing_reached(intel_dp, crtc_state)) max_vswing_reached = true; - } + + intel_dp_dump_link_status(intel_dp, dp_phy, link_status); drm_err(&i915->drm, - "Failed clock recovery %d times, giving up!\n", max_cr_tries); + "[ENCODER:%d:%s][%s] Failed clock recovery %d times, giving up!\n", + encoder->base.base.id, encoder->base.name, phy_name, max_cr_tries); + return false; } /* - * Pick training pattern for channel equalization. Training pattern 4 for HBR3 - * or for 1.4 devices that support it, training Pattern 3 for HBR2 - * or 1.2 devices that support it, Training Pattern 2 otherwise. + * Pick Training Pattern Sequence (TPS) for channel equalization. 128b/132b TPS2 + * for UHBR+, TPS4 for HBR3 or for 1.4 devices that support it, TPS3 for HBR2 or + * 1.2 devices that support it, TPS2 otherwise. */ static u32 intel_dp_training_pattern(struct intel_dp *intel_dp, const struct intel_crtc_state *crtc_state, enum drm_dp_phy dp_phy) { + struct drm_i915_private *i915 = dp_to_i915(intel_dp); bool source_tps3, sink_tps3, source_tps4, sink_tps4; + /* UHBR+ use separate 128b/132b TPS2 */ + if (intel_dp_is_uhbr(crtc_state)) + return DP_TRAINING_PATTERN_2; + /* - * Intel platforms that support HBR3 also support TPS4. It is mandatory - * for all downstream devices that support HBR3. There are no known eDP - * panels that support TPS4 as of Feb 2018 as per VESA eDP_v1.4b_E1 - * specification. + * TPS4 support is mandatory for all downstream devices that + * support HBR3. There are no known eDP panels that support + * TPS4 as of Feb 2018 as per VESA eDP_v1.4b_E1 specification. * LTTPRs must support TPS4. */ - source_tps4 = intel_dp_source_supports_hbr3(intel_dp); + source_tps4 = intel_dp_source_supports_tps4(i915); sink_tps4 = dp_phy != DP_PHY_DPRX || drm_dp_tps4_supported(intel_dp->dpcd); if (source_tps4 && sink_tps4) { return DP_TRAINING_PATTERN_4; } else if (crtc_state->port_clock == 810000) { if (!source_tps4) - drm_dbg_kms(&dp_to_i915(intel_dp)->drm, - "8.1 Gbps link rate without source HBR3/TPS4 support\n"); + drm_dbg_kms(&i915->drm, + "8.1 Gbps link rate without source TPS4 support\n"); if (!sink_tps4) - drm_dbg_kms(&dp_to_i915(intel_dp)->drm, + drm_dbg_kms(&i915->drm, "8.1 Gbps link rate without sink TPS4 support\n"); } + /* - * Intel platforms that support HBR2 also support TPS3. TPS3 support is - * also mandatory for downstream devices that support HBR2. However, not - * all sinks follow the spec. + * TPS3 support is mandatory for downstream devices that + * support HBR2. However, not all sinks follow the spec. */ - source_tps3 = intel_dp_source_supports_hbr2(intel_dp); + source_tps3 = intel_dp_source_supports_tps3(i915); sink_tps3 = dp_phy != DP_PHY_DPRX || drm_dp_tps3_supported(intel_dp->dpcd); if (source_tps3 && sink_tps3) { return DP_TRAINING_PATTERN_3; } else if (crtc_state->port_clock >= 540000) { if (!source_tps3) - drm_dbg_kms(&dp_to_i915(intel_dp)->drm, - ">=5.4/6.48 Gbps link rate without source HBR2/TPS3 support\n"); + drm_dbg_kms(&i915->drm, + ">=5.4/6.48 Gbps link rate without source TPS3 support\n"); if (!sink_tps3) - drm_dbg_kms(&dp_to_i915(intel_dp)->drm, + drm_dbg_kms(&i915->drm, ">=5.4/6.48 Gbps link rate without sink TPS3 support\n"); } @@ -677,11 +918,15 @@ intel_dp_link_training_channel_equalization(struct intel_dp *intel_dp, const struct intel_crtc_state *crtc_state, enum drm_dp_phy dp_phy) { - struct drm_i915_private *i915 = dp_to_i915(intel_dp); + struct intel_encoder *encoder = &dp_to_dig_port(intel_dp)->base; + struct drm_i915_private *i915 = to_i915(encoder->base.dev); int tries; u32 training_pattern; u8 link_status[DP_LINK_STATUS_SIZE]; bool channel_eq = false; + char phy_name[10]; + + intel_dp_phy_name(dp_phy, phy_name, sizeof(phy_name)); training_pattern = intel_dp_training_pattern(intel_dp, crtc_state, dp_phy); /* Scrambling is disabled for TPS2/3 and enabled for TPS4 */ @@ -691,7 +936,10 @@ intel_dp_link_training_channel_equalization(struct intel_dp *intel_dp, /* channel equalization */ if (!intel_dp_set_link_train(intel_dp, crtc_state, dp_phy, training_pattern)) { - drm_err(&i915->drm, "failed to start channel equalization\n"); + drm_err(&i915->drm, + "[ENCODER:%d:%s][%s] Failed to start channel equalization\n", + encoder->base.base.id, encoder->base.name, + phy_name); return false; } @@ -701,25 +949,28 @@ intel_dp_link_training_channel_equalization(struct intel_dp *intel_dp, if (drm_dp_dpcd_read_phy_link_status(&intel_dp->aux, dp_phy, link_status) < 0) { drm_err(&i915->drm, - "failed to get link status\n"); + "[ENCODER:%d:%s][%s] Failed to get link status\n", + encoder->base.base.id, encoder->base.name, phy_name); break; } /* Make sure clock is still ok */ if (!drm_dp_clock_recovery_ok(link_status, crtc_state->lane_count)) { - intel_dp_dump_link_status(&i915->drm, link_status); + intel_dp_dump_link_status(intel_dp, dp_phy, link_status); drm_dbg_kms(&i915->drm, - "Clock recovery check failed, cannot " - "continue channel equalization\n"); + "[ENCODER:%d:%s][%s] Clock recovery check failed, cannot " + "continue channel equalization\n", + encoder->base.base.id, encoder->base.name, phy_name); break; } if (drm_dp_channel_eq_ok(link_status, crtc_state->lane_count)) { channel_eq = true; - drm_dbg_kms(&i915->drm, "Channel EQ done. DP Training " - "successful\n"); + drm_dbg_kms(&i915->drm, + "[ENCODER:%d:%s][%s] Channel EQ done. DP Training successful\n", + encoder->base.base.id, encoder->base.name, phy_name); break; } @@ -728,16 +979,18 @@ intel_dp_link_training_channel_equalization(struct intel_dp *intel_dp, link_status); if (!intel_dp_update_link_train(intel_dp, crtc_state, dp_phy)) { drm_err(&i915->drm, - "failed to update link training\n"); + "[ENCODER:%d:%s][%s] Failed to update link training\n", + encoder->base.base.id, encoder->base.name, phy_name); break; } } /* Try 5 times, else fail and try at lower BW */ if (tries == 5) { - intel_dp_dump_link_status(&i915->drm, link_status); + intel_dp_dump_link_status(intel_dp, dp_phy, link_status); drm_dbg_kms(&i915->drm, - "Channel equalization failed 5 times\n"); + "[ENCODER:%d:%s][%s] Channel equalization failed 5 times\n", + encoder->base.base.id, encoder->base.name, phy_name); } return channel_eq; @@ -774,7 +1027,7 @@ void intel_dp_stop_link_train(struct intel_dp *intel_dp, intel_dp->link_trained = true; intel_dp_disable_dpcd_training_pattern(intel_dp, DP_PHY_DPRX); - intel_dp_program_link_training_pattern(intel_dp, crtc_state, + intel_dp_program_link_training_pattern(intel_dp, crtc_state, DP_PHY_DPRX, DP_TRAINING_PATTERN_DISABLE); } @@ -783,7 +1036,8 @@ intel_dp_link_train_phy(struct intel_dp *intel_dp, const struct intel_crtc_state *crtc_state, enum drm_dp_phy dp_phy) { - struct intel_connector *intel_connector = intel_dp->attached_connector; + struct intel_connector *connector = intel_dp->attached_connector; + struct intel_encoder *encoder = &dp_to_dig_port(intel_dp)->base; char phy_name[10]; bool ret = false; @@ -797,12 +1051,12 @@ intel_dp_link_train_phy(struct intel_dp *intel_dp, out: drm_dbg_kms(&dp_to_i915(intel_dp)->drm, - "[CONNECTOR:%d:%s] Link Training %s at link rate = %d, lane count = %d, at %s\n", - intel_connector->base.base.id, - intel_connector->base.name, + "[CONNECTOR:%d:%s][ENCODER:%d:%s][%s] Link Training %s at link rate = %d, lane count = %d\n", + connector->base.base.id, connector->base.name, + encoder->base.base.id, encoder->base.name, + intel_dp_phy_name(dp_phy, phy_name, sizeof(phy_name)), ret ? "passed" : "failed", - crtc_state->port_clock, crtc_state->lane_count, - intel_dp_phy_name(dp_phy, phy_name, sizeof(phy_name))); + crtc_state->port_clock, crtc_state->lane_count); return ret; } @@ -811,10 +1065,13 @@ static void intel_dp_schedule_fallback_link_training(struct intel_dp *intel_dp, const struct intel_crtc_state *crtc_state) { struct intel_connector *intel_connector = intel_dp->attached_connector; + struct intel_encoder *encoder = &dp_to_dig_port(intel_dp)->base; if (intel_dp->hobl_active) { drm_dbg_kms(&dp_to_i915(intel_dp)->drm, - "Link Training failed with HOBL active, not enabling it from now on"); + "[ENCODER:%d:%s] Link Training failed with HOBL active, " + "not enabling it from now on", + encoder->base.base.id, encoder->base.name); intel_dp->hobl_failed = true; } else if (intel_dp_get_link_train_fallback_values(intel_dp, crtc_state->port_clock, diff --git a/drivers/gpu/drm/i915/display/intel_dp_link_training.h b/drivers/gpu/drm/i915/display/intel_dp_link_training.h index 9d24d594368c..6a3a7b37349a 100644 --- a/drivers/gpu/drm/i915/display/intel_dp_link_training.h +++ b/drivers/gpu/drm/i915/display/intel_dp_link_training.h @@ -19,6 +19,7 @@ void intel_dp_get_adjust_train(struct intel_dp *intel_dp, const u8 link_status[DP_LINK_STATUS_SIZE]); void intel_dp_program_link_training_pattern(struct intel_dp *intel_dp, const struct intel_crtc_state *crtc_state, + enum drm_dp_phy dp_phy, u8 dp_train_pat); void intel_dp_set_signal_levels(struct intel_dp *intel_dp, const struct intel_crtc_state *crtc_state, diff --git a/drivers/gpu/drm/i915/display/intel_dp_mst.c b/drivers/gpu/drm/i915/display/intel_dp_mst.c index 8d13d7b26a25..89d701e8ae9d 100644 --- a/drivers/gpu/drm/i915/display/intel_dp_mst.c +++ b/drivers/gpu/drm/i915/display/intel_dp_mst.c @@ -61,7 +61,7 @@ static int intel_dp_mst_compute_link_config(struct intel_encoder *encoder, int bpp, slots = -EINVAL; crtc_state->lane_count = limits->max_lane_count; - crtc_state->port_clock = limits->max_clock; + crtc_state->port_clock = limits->max_rate; for (bpp = limits->max_bpp; bpp >= limits->min_bpp; bpp -= 2 * 3) { crtc_state->pipe_bpp = bpp; @@ -131,8 +131,8 @@ static int intel_dp_mst_compute_config(struct intel_encoder *encoder, * for MST we always configure max link bw - the spec doesn't * seem to suggest we should do otherwise. */ - limits.min_clock = - limits.max_clock = intel_dp_max_link_rate(intel_dp); + limits.min_rate = + limits.max_rate = intel_dp_max_link_rate(intel_dp); limits.min_lane_count = limits.max_lane_count = intel_dp_max_lane_count(intel_dp); @@ -378,7 +378,7 @@ static void intel_mst_disable_dp(struct intel_atomic_state *state, drm_dp_mst_reset_vcpi_slots(&intel_dp->mst_mgr, connector->port); - ret = drm_dp_update_payload_part1(&intel_dp->mst_mgr); + ret = drm_dp_update_payload_part1(&intel_dp->mst_mgr, 1); if (ret) { drm_dbg_kms(&i915->drm, "failed to update payload %d\n", ret); } @@ -396,7 +396,6 @@ static void intel_mst_post_disable_dp(struct intel_atomic_state *state, to_intel_connector(old_conn_state->connector); struct drm_i915_private *dev_priv = to_i915(connector->base.dev); bool last_mst_stream; - u32 val; intel_dp->active_mst_links--; last_mst_stream = intel_dp->active_mst_links == 0; @@ -406,18 +405,14 @@ static void intel_mst_post_disable_dp(struct intel_atomic_state *state, intel_crtc_vblank_off(old_crtc_state); - intel_disable_pipe(old_crtc_state); + intel_disable_transcoder(old_crtc_state); drm_dp_update_payload_part2(&intel_dp->mst_mgr); clear_act_sent(encoder, old_crtc_state); - val = intel_de_read(dev_priv, - TRANS_DDI_FUNC_CTL(old_crtc_state->cpu_transcoder)); - val &= ~TRANS_DDI_DP_VC_PAYLOAD_ALLOC; - intel_de_write(dev_priv, - TRANS_DDI_FUNC_CTL(old_crtc_state->cpu_transcoder), - val); + intel_de_rmw(dev_priv, TRANS_DDI_FUNC_CTL(old_crtc_state->cpu_transcoder), + TRANS_DDI_DP_VC_PAYLOAD_ALLOC, 0); wait_for_act_sent(encoder, old_crtc_state); @@ -523,7 +518,7 @@ static void intel_mst_pre_enable_dp(struct intel_atomic_state *state, intel_dp->active_mst_links++; - ret = drm_dp_update_payload_part1(&intel_dp->mst_mgr); + ret = drm_dp_update_payload_part1(&intel_dp->mst_mgr, 1); /* * Before Gen 12 this is not done as part of @@ -555,6 +550,17 @@ static void intel_mst_enable_dp(struct intel_atomic_state *state, clear_act_sent(encoder, pipe_config); + if (intel_dp_is_uhbr(pipe_config)) { + const struct drm_display_mode *adjusted_mode = + &pipe_config->hw.adjusted_mode; + u64 crtc_clock_hz = KHz(adjusted_mode->crtc_clock); + + intel_de_write(dev_priv, TRANS_DP2_VFREQHIGH(pipe_config->cpu_transcoder), + TRANS_DP2_VFREQ_PIXEL_CLOCK(crtc_clock_hz >> 24)); + intel_de_write(dev_priv, TRANS_DP2_VFREQLOW(pipe_config->cpu_transcoder), + TRANS_DP2_VFREQ_PIXEL_CLOCK(crtc_clock_hz & 0xffffff)); + } + intel_ddi_enable_transcoder_func(encoder, pipe_config); intel_de_rmw(dev_priv, TRANS_DDI_FUNC_CTL(trans), 0, @@ -571,7 +577,7 @@ static void intel_mst_enable_dp(struct intel_atomic_state *state, intel_de_rmw(dev_priv, CHICKEN_TRANS(trans), 0, FECSTALL_DIS_DPTSTREAM_DPTTG); - intel_enable_pipe(pipe_config); + intel_enable_transcoder(pipe_config); intel_crtc_vblank_on(pipe_config); @@ -971,24 +977,31 @@ intel_dp_mst_encoder_init(struct intel_digital_port *dig_port, int conn_base_id) dig_port->max_lanes, max_source_rate, conn_base_id); - if (ret) + if (ret) { + intel_dp->mst_mgr.cbs = NULL; return ret; - - intel_dp->can_mst = true; + } return 0; } +bool intel_dp_mst_source_support(struct intel_dp *intel_dp) +{ + return intel_dp->mst_mgr.cbs; +} + void intel_dp_mst_encoder_cleanup(struct intel_digital_port *dig_port) { struct intel_dp *intel_dp = &dig_port->dp; - if (!intel_dp->can_mst) + if (!intel_dp_mst_source_support(intel_dp)) return; drm_dp_mst_topology_mgr_destroy(&intel_dp->mst_mgr); /* encoders will get killed by normal cleanup */ + + intel_dp->mst_mgr.cbs = NULL; } bool intel_dp_mst_is_master_trans(const struct intel_crtc_state *crtc_state) diff --git a/drivers/gpu/drm/i915/display/intel_dp_mst.h b/drivers/gpu/drm/i915/display/intel_dp_mst.h index 6afda4e86b3c..f7301de6cdfb 100644 --- a/drivers/gpu/drm/i915/display/intel_dp_mst.h +++ b/drivers/gpu/drm/i915/display/intel_dp_mst.h @@ -8,13 +8,15 @@ #include <linux/types.h> -struct intel_digital_port; struct intel_crtc_state; +struct intel_digital_port; +struct intel_dp; int intel_dp_mst_encoder_init(struct intel_digital_port *dig_port, int conn_id); void intel_dp_mst_encoder_cleanup(struct intel_digital_port *dig_port); int intel_dp_mst_encoder_active_links(struct intel_digital_port *dig_port); bool intel_dp_mst_is_master_trans(const struct intel_crtc_state *crtc_state); bool intel_dp_mst_is_slave_trans(const struct intel_crtc_state *crtc_state); +bool intel_dp_mst_source_support(struct intel_dp *intel_dp); #endif /* __INTEL_DP_MST_H__ */ diff --git a/drivers/gpu/drm/i915/display/intel_dpio_phy.c b/drivers/gpu/drm/i915/display/intel_dpio_phy.c index 48507ed79950..44edeb2e55c0 100644 --- a/drivers/gpu/drm/i915/display/intel_dpio_phy.c +++ b/drivers/gpu/drm/i915/display/intel_dpio_phy.c @@ -21,12 +21,13 @@ * DEALINGS IN THE SOFTWARE. */ -#include "display/intel_dp.h" - +#include "intel_ddi.h" +#include "intel_ddi_buf_trans.h" #include "intel_de.h" #include "intel_display_types.h" +#include "intel_dp.h" #include "intel_dpio_phy.h" -#include "intel_sideband.h" +#include "vlv_sideband.h" /** * DOC: DPIO @@ -266,15 +267,22 @@ void bxt_port_to_phy_channel(struct drm_i915_private *dev_priv, enum port port, *ch = DPIO_CH0; } -void bxt_ddi_phy_set_signal_level(struct drm_i915_private *dev_priv, - enum port port, u32 margin, u32 scale, - u32 enable, u32 deemphasis) +void bxt_ddi_phy_set_signal_levels(struct intel_encoder *encoder, + const struct intel_crtc_state *crtc_state) { - u32 val; - enum dpio_phy phy; + struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); + int level = intel_ddi_level(encoder, crtc_state, 0); + const struct intel_ddi_buf_trans *trans; enum dpio_channel ch; + enum dpio_phy phy; + int n_entries; + u32 val; - bxt_port_to_phy_channel(dev_priv, port, &phy, &ch); + trans = encoder->get_buf_trans(encoder, crtc_state, &n_entries); + if (drm_WARN_ON_ONCE(&dev_priv->drm, !trans)) + return; + + bxt_port_to_phy_channel(dev_priv, encoder->port, &phy, &ch); /* * While we write to the group register to program all lanes at once we @@ -286,12 +294,13 @@ void bxt_ddi_phy_set_signal_level(struct drm_i915_private *dev_priv, val = intel_de_read(dev_priv, BXT_PORT_TX_DW2_LN0(phy, ch)); val &= ~(MARGIN_000 | UNIQ_TRANS_SCALE); - val |= margin << MARGIN_000_SHIFT | scale << UNIQ_TRANS_SCALE_SHIFT; + val |= trans->entries[level].bxt.margin << MARGIN_000_SHIFT | + trans->entries[level].bxt.scale << UNIQ_TRANS_SCALE_SHIFT; intel_de_write(dev_priv, BXT_PORT_TX_DW2_GRP(phy, ch), val); val = intel_de_read(dev_priv, BXT_PORT_TX_DW3_LN0(phy, ch)); val &= ~SCALE_DCOMP_METHOD; - if (enable) + if (trans->entries[level].bxt.enable) val |= SCALE_DCOMP_METHOD; if ((val & UNIQUE_TRANGE_EN_METHOD) && !(val & SCALE_DCOMP_METHOD)) @@ -302,7 +311,7 @@ void bxt_ddi_phy_set_signal_level(struct drm_i915_private *dev_priv, val = intel_de_read(dev_priv, BXT_PORT_TX_DW4_LN0(phy, ch)); val &= ~DE_EMPHASIS; - val |= deemphasis << DEEMPH_SHIFT; + val |= trans->entries[level].bxt.deemphasis << DEEMPH_SHIFT; intel_de_write(dev_priv, BXT_PORT_TX_DW4_GRP(phy, ch), val); val = intel_de_read(dev_priv, BXT_PORT_PCS_DW10_LN01(phy, ch)); diff --git a/drivers/gpu/drm/i915/display/intel_dpio_phy.h b/drivers/gpu/drm/i915/display/intel_dpio_phy.h index 6473440e7457..9c3d008e8e1a 100644 --- a/drivers/gpu/drm/i915/display/intel_dpio_phy.h +++ b/drivers/gpu/drm/i915/display/intel_dpio_phy.h @@ -17,9 +17,8 @@ struct intel_encoder; void bxt_port_to_phy_channel(struct drm_i915_private *dev_priv, enum port port, enum dpio_phy *phy, enum dpio_channel *ch); -void bxt_ddi_phy_set_signal_level(struct drm_i915_private *dev_priv, - enum port port, u32 margin, u32 scale, - u32 enable, u32 deemphasis); +void bxt_ddi_phy_set_signal_levels(struct intel_encoder *encoder, + const struct intel_crtc_state *crtc_state); void bxt_ddi_phy_init(struct drm_i915_private *dev_priv, enum dpio_phy phy); void bxt_ddi_phy_uninit(struct drm_i915_private *dev_priv, enum dpio_phy phy); bool bxt_ddi_phy_is_enabled(struct drm_i915_private *dev_priv, diff --git a/drivers/gpu/drm/i915/display/intel_dpll.c b/drivers/gpu/drm/i915/display/intel_dpll.c index 14515e62c05e..04a7af8340ca 100644 --- a/drivers/gpu/drm/i915/display/intel_dpll.c +++ b/drivers/gpu/drm/i915/display/intel_dpll.c @@ -2,16 +2,19 @@ /* * Copyright © 2020 Intel Corporation */ + #include <linux/kernel.h> + #include "intel_crtc.h" #include "intel_de.h" -#include "intel_display_types.h" #include "intel_display.h" +#include "intel_display_types.h" #include "intel_dpll.h" #include "intel_lvds.h" #include "intel_panel.h" -#include "intel_sideband.h" -#include "display/intel_snps_phy.h" +#include "intel_pps.h" +#include "intel_snps_phy.h" +#include "vlv_sideband.h" struct intel_limit { struct { @@ -309,7 +312,7 @@ int pnv_calc_dpll_params(int refclk, struct dpll *clock) return clock->dot; } -static u32 i9xx_dpll_compute_m(struct dpll *dpll) +static u32 i9xx_dpll_compute_m(const struct dpll *dpll) { return 5 * (dpll->m1 + 2) + (dpll->m2 + 2); } @@ -428,7 +431,8 @@ i9xx_select_p2_div(const struct intel_limit *limit, static bool i9xx_find_best_dpll(const struct intel_limit *limit, struct intel_crtc_state *crtc_state, - int target, int refclk, struct dpll *match_clock, + int target, int refclk, + const struct dpll *match_clock, struct dpll *best_clock) { struct drm_device *dev = crtc_state->uapi.crtc->dev; @@ -486,7 +490,8 @@ i9xx_find_best_dpll(const struct intel_limit *limit, static bool pnv_find_best_dpll(const struct intel_limit *limit, struct intel_crtc_state *crtc_state, - int target, int refclk, struct dpll *match_clock, + int target, int refclk, + const struct dpll *match_clock, struct dpll *best_clock) { struct drm_device *dev = crtc_state->uapi.crtc->dev; @@ -542,7 +547,8 @@ pnv_find_best_dpll(const struct intel_limit *limit, static bool g4x_find_best_dpll(const struct intel_limit *limit, struct intel_crtc_state *crtc_state, - int target, int refclk, struct dpll *match_clock, + int target, int refclk, + const struct dpll *match_clock, struct dpll *best_clock) { struct drm_device *dev = crtc_state->uapi.crtc->dev; @@ -636,7 +642,8 @@ static bool vlv_PLL_is_optimal(struct drm_device *dev, int target_freq, static bool vlv_find_best_dpll(const struct intel_limit *limit, struct intel_crtc_state *crtc_state, - int target, int refclk, struct dpll *match_clock, + int target, int refclk, + const struct dpll *match_clock, struct dpll *best_clock) { struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); @@ -696,7 +703,8 @@ vlv_find_best_dpll(const struct intel_limit *limit, static bool chv_find_best_dpll(const struct intel_limit *limit, struct intel_crtc_state *crtc_state, - int target, int refclk, struct dpll *match_clock, + int target, int refclk, + const struct dpll *match_clock, struct dpll *best_clock) { struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); @@ -763,47 +771,45 @@ bool bxt_find_best_dpll(struct intel_crtc_state *crtc_state, NULL, best_clock); } -static u32 pnv_dpll_compute_fp(struct dpll *dpll) +u32 i9xx_dpll_compute_fp(const struct dpll *dpll) +{ + return dpll->n << 16 | dpll->m1 << 8 | dpll->m2; +} + +static u32 pnv_dpll_compute_fp(const struct dpll *dpll) { return (1 << dpll->n) << 16 | dpll->m2; } -static void i9xx_update_pll_dividers(struct intel_crtc *crtc, - struct intel_crtc_state *crtc_state, - struct dpll *reduced_clock) +static void i9xx_update_pll_dividers(struct intel_crtc_state *crtc_state, + const struct dpll *clock, + const struct dpll *reduced_clock) { + struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); - u32 fp, fp2 = 0; + u32 fp, fp2; if (IS_PINEVIEW(dev_priv)) { - fp = pnv_dpll_compute_fp(&crtc_state->dpll); - if (reduced_clock) - fp2 = pnv_dpll_compute_fp(reduced_clock); + fp = pnv_dpll_compute_fp(clock); + fp2 = pnv_dpll_compute_fp(reduced_clock); } else { - fp = i9xx_dpll_compute_fp(&crtc_state->dpll); - if (reduced_clock) - fp2 = i9xx_dpll_compute_fp(reduced_clock); + fp = i9xx_dpll_compute_fp(clock); + fp2 = i9xx_dpll_compute_fp(reduced_clock); } crtc_state->dpll_hw_state.fp0 = fp; - - if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS) && - reduced_clock) { - crtc_state->dpll_hw_state.fp1 = fp2; - } else { - crtc_state->dpll_hw_state.fp1 = fp; - } + crtc_state->dpll_hw_state.fp1 = fp2; } -static void i9xx_compute_dpll(struct intel_crtc *crtc, - struct intel_crtc_state *crtc_state, - struct dpll *reduced_clock) +static void i9xx_compute_dpll(struct intel_crtc_state *crtc_state, + const struct dpll *clock, + const struct dpll *reduced_clock) { + struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); u32 dpll; - struct dpll *clock = &crtc_state->dpll; - i9xx_update_pll_dividers(crtc, crtc_state, reduced_clock); + i9xx_update_pll_dividers(crtc_state, clock, reduced_clock); dpll = DPLL_VGA_MODE_DIS; @@ -826,13 +832,17 @@ static void i9xx_compute_dpll(struct intel_crtc *crtc, dpll |= DPLL_SDVO_HIGH_SPEED; /* compute bitmask from p1 value */ - if (IS_PINEVIEW(dev_priv)) + if (IS_G4X(dev_priv)) { + dpll |= (1 << (clock->p1 - 1)) << DPLL_FPA01_P1_POST_DIV_SHIFT; + dpll |= (1 << (reduced_clock->p1 - 1)) << DPLL_FPA1_P1_POST_DIV_SHIFT; + } else if (IS_PINEVIEW(dev_priv)) { dpll |= (1 << (clock->p1 - 1)) << DPLL_FPA01_P1_POST_DIV_SHIFT_PINEVIEW; - else { + WARN_ON(reduced_clock->p1 != clock->p1); + } else { dpll |= (1 << (clock->p1 - 1)) << DPLL_FPA01_P1_POST_DIV_SHIFT; - if (IS_G4X(dev_priv) && reduced_clock) - dpll |= (1 << (reduced_clock->p1 - 1)) << DPLL_FPA1_P1_POST_DIV_SHIFT; + WARN_ON(reduced_clock->p1 != clock->p1); } + switch (clock->p2) { case 5: dpll |= DPLL_DAC_SERIAL_P2_CLOCK_DIV_5; @@ -847,6 +857,8 @@ static void i9xx_compute_dpll(struct intel_crtc *crtc, dpll |= DPLLB_LVDS_P2_CLOCK_DIV_14; break; } + WARN_ON(reduced_clock->p2 != clock->p2); + if (DISPLAY_VER(dev_priv) >= 4) dpll |= (6 << PLL_LOAD_PULSE_PHASE_SHIFT); @@ -868,16 +880,15 @@ static void i9xx_compute_dpll(struct intel_crtc *crtc, } } -static void i8xx_compute_dpll(struct intel_crtc *crtc, - struct intel_crtc_state *crtc_state, - struct dpll *reduced_clock) +static void i8xx_compute_dpll(struct intel_crtc_state *crtc_state, + const struct dpll *clock, + const struct dpll *reduced_clock) { - struct drm_device *dev = crtc->base.dev; - struct drm_i915_private *dev_priv = to_i915(dev); + struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); u32 dpll; - struct dpll *clock = &crtc_state->dpll; - i9xx_update_pll_dividers(crtc, crtc_state, reduced_clock); + i9xx_update_pll_dividers(crtc_state, clock, reduced_clock); dpll = DPLL_VGA_MODE_DIS; @@ -891,6 +902,8 @@ static void i8xx_compute_dpll(struct intel_crtc *crtc, if (clock->p2 == 4) dpll |= PLL_P2_DIVIDE_BY_4; } + WARN_ON(reduced_clock->p1 != clock->p1); + WARN_ON(reduced_clock->p2 != clock->p2); /* * Bspec: @@ -918,42 +931,44 @@ static void i8xx_compute_dpll(struct intel_crtc *crtc, crtc_state->dpll_hw_state.dpll = dpll; } -static int hsw_crtc_compute_clock(struct intel_crtc *crtc, - struct intel_crtc_state *crtc_state) +static int hsw_crtc_compute_clock(struct intel_crtc_state *crtc_state) { + struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); struct intel_atomic_state *state = to_intel_atomic_state(crtc_state->uapi.state); struct intel_encoder *encoder = intel_get_crtc_new_encoder(state, crtc_state); - if (IS_DG2(dev_priv)) { + if (IS_DG2(dev_priv)) return intel_mpllb_calc_state(crtc_state, encoder); - } else if (!intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DSI) || - DISPLAY_VER(dev_priv) >= 11) { - if (!intel_reserve_shared_dplls(state, crtc, encoder)) { - drm_dbg_kms(&dev_priv->drm, - "failed to find PLL for pipe %c\n", - pipe_name(crtc->pipe)); - return -EINVAL; - } + + if (DISPLAY_VER(dev_priv) < 11 && + intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DSI)) + return 0; + + if (!intel_reserve_shared_dplls(state, crtc, encoder)) { + drm_dbg_kms(&dev_priv->drm, + "failed to find PLL for pipe %c\n", + pipe_name(crtc->pipe)); + return -EINVAL; } return 0; } -static bool ilk_needs_fb_cb_tune(struct dpll *dpll, int factor) +static bool ilk_needs_fb_cb_tune(const struct dpll *dpll, int factor) { - return i9xx_dpll_compute_m(dpll) < factor * dpll->n; + return dpll->m < factor * dpll->n; } - -static void ilk_compute_dpll(struct intel_crtc *crtc, - struct intel_crtc_state *crtc_state, - struct dpll *reduced_clock) +static void ilk_update_pll_dividers(struct intel_crtc_state *crtc_state, + const struct dpll *clock, + const struct dpll *reduced_clock) { + struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); - u32 dpll, fp, fp2; + u32 fp, fp2; int factor; /* Enable autotuning of the PLL clock (if permissible) */ @@ -968,19 +983,27 @@ static void ilk_compute_dpll(struct intel_crtc *crtc, factor = 20; } - fp = i9xx_dpll_compute_fp(&crtc_state->dpll); - - if (ilk_needs_fb_cb_tune(&crtc_state->dpll, factor)) + fp = i9xx_dpll_compute_fp(clock); + if (ilk_needs_fb_cb_tune(clock, factor)) fp |= FP_CB_TUNE; - if (reduced_clock) { - fp2 = i9xx_dpll_compute_fp(reduced_clock); + fp2 = i9xx_dpll_compute_fp(reduced_clock); + if (ilk_needs_fb_cb_tune(reduced_clock, factor)) + fp2 |= FP_CB_TUNE; - if (reduced_clock->m < factor * reduced_clock->n) - fp2 |= FP_CB_TUNE; - } else { - fp2 = fp; - } + crtc_state->dpll_hw_state.fp0 = fp; + crtc_state->dpll_hw_state.fp1 = fp2; +} + +static void ilk_compute_dpll(struct intel_crtc_state *crtc_state, + const struct dpll *clock, + const struct dpll *reduced_clock) +{ + struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); + u32 dpll; + + ilk_update_pll_dividers(crtc_state, clock, reduced_clock); dpll = 0; @@ -1018,11 +1041,11 @@ static void ilk_compute_dpll(struct intel_crtc *crtc, dpll |= DPLL_SDVO_HIGH_SPEED; /* compute bitmask from p1 value */ - dpll |= (1 << (crtc_state->dpll.p1 - 1)) << DPLL_FPA01_P1_POST_DIV_SHIFT; + dpll |= (1 << (clock->p1 - 1)) << DPLL_FPA01_P1_POST_DIV_SHIFT; /* also FPA1 */ - dpll |= (1 << (crtc_state->dpll.p1 - 1)) << DPLL_FPA1_P1_POST_DIV_SHIFT; + dpll |= (1 << (reduced_clock->p1 - 1)) << DPLL_FPA1_P1_POST_DIV_SHIFT; - switch (crtc_state->dpll.p2) { + switch (clock->p2) { case 5: dpll |= DPLL_DAC_SERIAL_P2_CLOCK_DIV_5; break; @@ -1036,6 +1059,7 @@ static void ilk_compute_dpll(struct intel_crtc *crtc, dpll |= DPLLB_LVDS_P2_CLOCK_DIV_14; break; } + WARN_ON(reduced_clock->p2 != clock->p2); if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS) && intel_panel_use_ssc(dev_priv)) @@ -1046,13 +1070,11 @@ static void ilk_compute_dpll(struct intel_crtc *crtc, dpll |= DPLL_VCO_ENABLE; crtc_state->dpll_hw_state.dpll = dpll; - crtc_state->dpll_hw_state.fp0 = fp; - crtc_state->dpll_hw_state.fp1 = fp2; } -static int ilk_crtc_compute_clock(struct intel_crtc *crtc, - struct intel_crtc_state *crtc_state) +static int ilk_crtc_compute_clock(struct intel_crtc_state *crtc_state) { + struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); struct intel_atomic_state *state = to_intel_atomic_state(crtc_state->uapi.state); @@ -1097,7 +1119,8 @@ static int ilk_crtc_compute_clock(struct intel_crtc *crtc, return -EINVAL; } - ilk_compute_dpll(crtc, crtc_state, NULL); + ilk_compute_dpll(crtc_state, &crtc_state->dpll, + &crtc_state->dpll); if (!intel_reserve_shared_dplls(state, crtc, NULL)) { drm_dbg_kms(&dev_priv->drm, @@ -1109,41 +1132,42 @@ static int ilk_crtc_compute_clock(struct intel_crtc *crtc, return 0; } -void vlv_compute_dpll(struct intel_crtc *crtc, - struct intel_crtc_state *pipe_config) +void vlv_compute_dpll(struct intel_crtc_state *crtc_state) { - pipe_config->dpll_hw_state.dpll = DPLL_INTEGRATED_REF_CLK_VLV | + struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); + + crtc_state->dpll_hw_state.dpll = DPLL_INTEGRATED_REF_CLK_VLV | DPLL_REF_CLK_ENABLE_VLV | DPLL_VGA_MODE_DIS; if (crtc->pipe != PIPE_A) - pipe_config->dpll_hw_state.dpll |= DPLL_INTEGRATED_CRI_CLK_VLV; + crtc_state->dpll_hw_state.dpll |= DPLL_INTEGRATED_CRI_CLK_VLV; /* DPLL not used with DSI, but still need the rest set up */ - if (!intel_crtc_has_type(pipe_config, INTEL_OUTPUT_DSI)) - pipe_config->dpll_hw_state.dpll |= DPLL_VCO_ENABLE | + if (!intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DSI)) + crtc_state->dpll_hw_state.dpll |= DPLL_VCO_ENABLE | DPLL_EXT_BUFFER_ENABLE_VLV; - pipe_config->dpll_hw_state.dpll_md = - (pipe_config->pixel_multiplier - 1) << DPLL_MD_UDI_MULTIPLIER_SHIFT; + crtc_state->dpll_hw_state.dpll_md = + (crtc_state->pixel_multiplier - 1) << DPLL_MD_UDI_MULTIPLIER_SHIFT; } -void chv_compute_dpll(struct intel_crtc *crtc, - struct intel_crtc_state *pipe_config) +void chv_compute_dpll(struct intel_crtc_state *crtc_state) { - pipe_config->dpll_hw_state.dpll = DPLL_SSC_REF_CLK_CHV | + struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); + + crtc_state->dpll_hw_state.dpll = DPLL_SSC_REF_CLK_CHV | DPLL_REF_CLK_ENABLE_VLV | DPLL_VGA_MODE_DIS; if (crtc->pipe != PIPE_A) - pipe_config->dpll_hw_state.dpll |= DPLL_INTEGRATED_CRI_CLK_VLV; + crtc_state->dpll_hw_state.dpll |= DPLL_INTEGRATED_CRI_CLK_VLV; /* DPLL not used with DSI, but still need the rest set up */ - if (!intel_crtc_has_type(pipe_config, INTEL_OUTPUT_DSI)) - pipe_config->dpll_hw_state.dpll |= DPLL_VCO_ENABLE; + if (!intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DSI)) + crtc_state->dpll_hw_state.dpll |= DPLL_VCO_ENABLE; - pipe_config->dpll_hw_state.dpll_md = - (pipe_config->pixel_multiplier - 1) << DPLL_MD_UDI_MULTIPLIER_SHIFT; + crtc_state->dpll_hw_state.dpll_md = + (crtc_state->pixel_multiplier - 1) << DPLL_MD_UDI_MULTIPLIER_SHIFT; } -static int chv_crtc_compute_clock(struct intel_crtc *crtc, - struct intel_crtc_state *crtc_state) +static int chv_crtc_compute_clock(struct intel_crtc_state *crtc_state) { int refclk = 100000; const struct intel_limit *limit = &intel_limits_chv; @@ -1159,13 +1183,12 @@ static int chv_crtc_compute_clock(struct intel_crtc *crtc, return -EINVAL; } - chv_compute_dpll(crtc, crtc_state); + chv_compute_dpll(crtc_state); return 0; } -static int vlv_crtc_compute_clock(struct intel_crtc *crtc, - struct intel_crtc_state *crtc_state) +static int vlv_crtc_compute_clock(struct intel_crtc_state *crtc_state) { int refclk = 100000; const struct intel_limit *limit = &intel_limits_vlv; @@ -1181,14 +1204,14 @@ static int vlv_crtc_compute_clock(struct intel_crtc *crtc, return -EINVAL; } - vlv_compute_dpll(crtc, crtc_state); + vlv_compute_dpll(crtc_state); return 0; } -static int g4x_crtc_compute_clock(struct intel_crtc *crtc, - struct intel_crtc_state *crtc_state) +static int g4x_crtc_compute_clock(struct intel_crtc_state *crtc_state) { + struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); const struct intel_limit *limit; int refclk = 96000; @@ -1226,16 +1249,16 @@ static int g4x_crtc_compute_clock(struct intel_crtc *crtc, return -EINVAL; } - i9xx_compute_dpll(crtc, crtc_state, NULL); + i9xx_compute_dpll(crtc_state, &crtc_state->dpll, + &crtc_state->dpll); return 0; } -static int pnv_crtc_compute_clock(struct intel_crtc *crtc, - struct intel_crtc_state *crtc_state) +static int pnv_crtc_compute_clock(struct intel_crtc_state *crtc_state) { - struct drm_device *dev = crtc->base.dev; - struct drm_i915_private *dev_priv = to_i915(dev); + struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); const struct intel_limit *limit; int refclk = 96000; @@ -1263,16 +1286,16 @@ static int pnv_crtc_compute_clock(struct intel_crtc *crtc, return -EINVAL; } - i9xx_compute_dpll(crtc, crtc_state, NULL); + i9xx_compute_dpll(crtc_state, &crtc_state->dpll, + &crtc_state->dpll); return 0; } -static int i9xx_crtc_compute_clock(struct intel_crtc *crtc, - struct intel_crtc_state *crtc_state) +static int i9xx_crtc_compute_clock(struct intel_crtc_state *crtc_state) { - struct drm_device *dev = crtc->base.dev; - struct drm_i915_private *dev_priv = to_i915(dev); + struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); const struct intel_limit *limit; int refclk = 96000; @@ -1300,16 +1323,16 @@ static int i9xx_crtc_compute_clock(struct intel_crtc *crtc, return -EINVAL; } - i9xx_compute_dpll(crtc, crtc_state, NULL); + i9xx_compute_dpll(crtc_state, &crtc_state->dpll, + &crtc_state->dpll); return 0; } -static int i8xx_crtc_compute_clock(struct intel_crtc *crtc, - struct intel_crtc_state *crtc_state) +static int i8xx_crtc_compute_clock(struct intel_crtc_state *crtc_state) { - struct drm_device *dev = crtc->base.dev; - struct drm_i915_private *dev_priv = to_i915(dev); + struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); const struct intel_limit *limit; int refclk = 48000; @@ -1339,30 +1362,63 @@ static int i8xx_crtc_compute_clock(struct intel_crtc *crtc, return -EINVAL; } - i8xx_compute_dpll(crtc, crtc_state, NULL); + i8xx_compute_dpll(crtc_state, &crtc_state->dpll, + &crtc_state->dpll); return 0; } +static const struct intel_dpll_funcs hsw_dpll_funcs = { + .crtc_compute_clock = hsw_crtc_compute_clock, +}; + +static const struct intel_dpll_funcs ilk_dpll_funcs = { + .crtc_compute_clock = ilk_crtc_compute_clock, +}; + +static const struct intel_dpll_funcs chv_dpll_funcs = { + .crtc_compute_clock = chv_crtc_compute_clock, +}; + +static const struct intel_dpll_funcs vlv_dpll_funcs = { + .crtc_compute_clock = vlv_crtc_compute_clock, +}; + +static const struct intel_dpll_funcs g4x_dpll_funcs = { + .crtc_compute_clock = g4x_crtc_compute_clock, +}; + +static const struct intel_dpll_funcs pnv_dpll_funcs = { + .crtc_compute_clock = pnv_crtc_compute_clock, +}; + +static const struct intel_dpll_funcs i9xx_dpll_funcs = { + .crtc_compute_clock = i9xx_crtc_compute_clock, +}; + +static const struct intel_dpll_funcs i8xx_dpll_funcs = { + .crtc_compute_clock = i8xx_crtc_compute_clock, +}; + void intel_dpll_init_clock_hook(struct drm_i915_private *dev_priv) { if (DISPLAY_VER(dev_priv) >= 9 || HAS_DDI(dev_priv)) - dev_priv->display.crtc_compute_clock = hsw_crtc_compute_clock; + dev_priv->dpll_funcs = &hsw_dpll_funcs; else if (HAS_PCH_SPLIT(dev_priv)) - dev_priv->display.crtc_compute_clock = ilk_crtc_compute_clock; + dev_priv->dpll_funcs = &ilk_dpll_funcs; else if (IS_CHERRYVIEW(dev_priv)) - dev_priv->display.crtc_compute_clock = chv_crtc_compute_clock; + dev_priv->dpll_funcs = &chv_dpll_funcs; else if (IS_VALLEYVIEW(dev_priv)) - dev_priv->display.crtc_compute_clock = vlv_crtc_compute_clock; + dev_priv->dpll_funcs = &vlv_dpll_funcs; else if (IS_G4X(dev_priv)) - dev_priv->display.crtc_compute_clock = g4x_crtc_compute_clock; + dev_priv->dpll_funcs = &g4x_dpll_funcs; else if (IS_PINEVIEW(dev_priv)) - dev_priv->display.crtc_compute_clock = pnv_crtc_compute_clock; + dev_priv->dpll_funcs = &pnv_dpll_funcs; else if (DISPLAY_VER(dev_priv) != 2) - dev_priv->display.crtc_compute_clock = i9xx_crtc_compute_clock; + dev_priv->dpll_funcs = &i9xx_dpll_funcs; else - dev_priv->display.crtc_compute_clock = i8xx_crtc_compute_clock; + dev_priv->dpll_funcs = &i8xx_dpll_funcs; } static bool i9xx_has_pps(struct drm_i915_private *dev_priv) @@ -1373,34 +1429,37 @@ static bool i9xx_has_pps(struct drm_i915_private *dev_priv) return IS_PINEVIEW(dev_priv) || IS_MOBILE(dev_priv); } -void i9xx_enable_pll(struct intel_crtc *crtc, - const struct intel_crtc_state *crtc_state) +void i9xx_enable_pll(const struct intel_crtc_state *crtc_state) { + struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); - i915_reg_t reg = DPLL(crtc->pipe); u32 dpll = crtc_state->dpll_hw_state.dpll; + enum pipe pipe = crtc->pipe; int i; - assert_pipe_disabled(dev_priv, crtc_state->cpu_transcoder); + assert_transcoder_disabled(dev_priv, crtc_state->cpu_transcoder); /* PLL is protected by panel, make sure we can write it */ if (i9xx_has_pps(dev_priv)) - assert_panel_unlocked(dev_priv, crtc->pipe); + assert_pps_unlocked(dev_priv, pipe); + + intel_de_write(dev_priv, FP0(pipe), crtc_state->dpll_hw_state.fp0); + intel_de_write(dev_priv, FP1(pipe), crtc_state->dpll_hw_state.fp1); /* * Apparently we need to have VGA mode enabled prior to changing * the P1/P2 dividers. Otherwise the DPLL will keep using the old * dividers, even though the register value does change. */ - intel_de_write(dev_priv, reg, dpll & ~DPLL_VGA_MODE_DIS); - intel_de_write(dev_priv, reg, dpll); + intel_de_write(dev_priv, DPLL(pipe), dpll & ~DPLL_VGA_MODE_DIS); + intel_de_write(dev_priv, DPLL(pipe), dpll); /* Wait for the clocks to stabilize. */ - intel_de_posting_read(dev_priv, reg); + intel_de_posting_read(dev_priv, DPLL(pipe)); udelay(150); if (DISPLAY_VER(dev_priv) >= 4) { - intel_de_write(dev_priv, DPLL_MD(crtc->pipe), + intel_de_write(dev_priv, DPLL_MD(pipe), crtc_state->dpll_hw_state.dpll_md); } else { /* The pixel multiplier can only be updated once the @@ -1408,13 +1467,13 @@ void i9xx_enable_pll(struct intel_crtc *crtc, * * So write it again. */ - intel_de_write(dev_priv, reg, dpll); + intel_de_write(dev_priv, DPLL(pipe), dpll); } /* We do this three times for luck */ for (i = 0; i < 3; i++) { - intel_de_write(dev_priv, reg, dpll); - intel_de_posting_read(dev_priv, reg); + intel_de_write(dev_priv, DPLL(pipe), dpll); + intel_de_posting_read(dev_priv, DPLL(pipe)); udelay(150); /* wait for warmup */ } } @@ -1448,136 +1507,22 @@ static void vlv_pllb_recal_opamp(struct drm_i915_private *dev_priv, vlv_dpio_write(dev_priv, pipe, VLV_REF_DW13, reg_val); } -static void _vlv_enable_pll(struct intel_crtc *crtc, - const struct intel_crtc_state *pipe_config) -{ - struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); - enum pipe pipe = crtc->pipe; - - intel_de_write(dev_priv, DPLL(pipe), pipe_config->dpll_hw_state.dpll); - intel_de_posting_read(dev_priv, DPLL(pipe)); - udelay(150); - - if (intel_de_wait_for_set(dev_priv, DPLL(pipe), DPLL_LOCK_VLV, 1)) - drm_err(&dev_priv->drm, "DPLL %d failed to lock\n", pipe); -} - -void vlv_enable_pll(struct intel_crtc *crtc, - const struct intel_crtc_state *pipe_config) -{ - struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); - enum pipe pipe = crtc->pipe; - - assert_pipe_disabled(dev_priv, pipe_config->cpu_transcoder); - - /* PLL is protected by panel, make sure we can write it */ - assert_panel_unlocked(dev_priv, pipe); - - if (pipe_config->dpll_hw_state.dpll & DPLL_VCO_ENABLE) - _vlv_enable_pll(crtc, pipe_config); - - intel_de_write(dev_priv, DPLL_MD(pipe), - pipe_config->dpll_hw_state.dpll_md); - intel_de_posting_read(dev_priv, DPLL_MD(pipe)); -} - - -static void _chv_enable_pll(struct intel_crtc *crtc, - const struct intel_crtc_state *pipe_config) -{ - struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); - enum pipe pipe = crtc->pipe; - enum dpio_channel port = vlv_pipe_to_channel(pipe); - u32 tmp; - - vlv_dpio_get(dev_priv); - - /* Enable back the 10bit clock to display controller */ - tmp = vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW14(port)); - tmp |= DPIO_DCLKP_EN; - vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW14(port), tmp); - - vlv_dpio_put(dev_priv); - - /* - * Need to wait > 100ns between dclkp clock enable bit and PLL enable. - */ - udelay(1); - - /* Enable PLL */ - intel_de_write(dev_priv, DPLL(pipe), pipe_config->dpll_hw_state.dpll); - - /* Check PLL is locked */ - if (intel_de_wait_for_set(dev_priv, DPLL(pipe), DPLL_LOCK_VLV, 1)) - drm_err(&dev_priv->drm, "PLL %d failed to lock\n", pipe); -} - -void chv_enable_pll(struct intel_crtc *crtc, - const struct intel_crtc_state *pipe_config) +static void vlv_prepare_pll(const struct intel_crtc_state *crtc_state) { + struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); enum pipe pipe = crtc->pipe; - - assert_pipe_disabled(dev_priv, pipe_config->cpu_transcoder); - - /* PLL is protected by panel, make sure we can write it */ - assert_panel_unlocked(dev_priv, pipe); - - if (pipe_config->dpll_hw_state.dpll & DPLL_VCO_ENABLE) - _chv_enable_pll(crtc, pipe_config); - - if (pipe != PIPE_A) { - /* - * WaPixelRepeatModeFixForC0:chv - * - * DPLLCMD is AWOL. Use chicken bits to propagate - * the value from DPLLBMD to either pipe B or C. - */ - intel_de_write(dev_priv, CBR4_VLV, CBR_DPLLBMD_PIPE(pipe)); - intel_de_write(dev_priv, DPLL_MD(PIPE_B), - pipe_config->dpll_hw_state.dpll_md); - intel_de_write(dev_priv, CBR4_VLV, 0); - dev_priv->chv_dpll_md[pipe] = pipe_config->dpll_hw_state.dpll_md; - - /* - * DPLLB VGA mode also seems to cause problems. - * We should always have it disabled. - */ - drm_WARN_ON(&dev_priv->drm, - (intel_de_read(dev_priv, DPLL(PIPE_B)) & - DPLL_VGA_MODE_DIS) == 0); - } else { - intel_de_write(dev_priv, DPLL_MD(pipe), - pipe_config->dpll_hw_state.dpll_md); - intel_de_posting_read(dev_priv, DPLL_MD(pipe)); - } -} - -void vlv_prepare_pll(struct intel_crtc *crtc, - const struct intel_crtc_state *pipe_config) -{ - struct drm_device *dev = crtc->base.dev; - struct drm_i915_private *dev_priv = to_i915(dev); - enum pipe pipe = crtc->pipe; u32 mdiv; u32 bestn, bestm1, bestm2, bestp1, bestp2; u32 coreclk, reg_val; - /* Enable Refclk */ - intel_de_write(dev_priv, DPLL(pipe), - pipe_config->dpll_hw_state.dpll & ~(DPLL_VCO_ENABLE | DPLL_EXT_BUFFER_ENABLE_VLV)); - - /* No need to actually set up the DPLL with DSI */ - if ((pipe_config->dpll_hw_state.dpll & DPLL_VCO_ENABLE) == 0) - return; - vlv_dpio_get(dev_priv); - bestn = pipe_config->dpll.n; - bestm1 = pipe_config->dpll.m1; - bestm2 = pipe_config->dpll.m2; - bestp1 = pipe_config->dpll.p1; - bestp2 = pipe_config->dpll.p2; + bestn = crtc_state->dpll.n; + bestm1 = crtc_state->dpll.m1; + bestm2 = crtc_state->dpll.m2; + bestp1 = crtc_state->dpll.p1; + bestp2 = crtc_state->dpll.p2; /* See eDP HDMI DPIO driver vbios notes doc */ @@ -1614,16 +1559,16 @@ void vlv_prepare_pll(struct intel_crtc *crtc, vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW3(pipe), mdiv); /* Set HBR and RBR LPF coefficients */ - if (pipe_config->port_clock == 162000 || - intel_crtc_has_type(pipe_config, INTEL_OUTPUT_ANALOG) || - intel_crtc_has_type(pipe_config, INTEL_OUTPUT_HDMI)) + if (crtc_state->port_clock == 162000 || + intel_crtc_has_type(crtc_state, INTEL_OUTPUT_ANALOG) || + intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI)) vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW10(pipe), 0x009f0003); else vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW10(pipe), 0x00d0000f); - if (intel_crtc_has_dp_encoder(pipe_config)) { + if (intel_crtc_has_dp_encoder(crtc_state)) { /* Use SSC source */ if (pipe == PIPE_A) vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW5(pipe), @@ -1643,7 +1588,7 @@ void vlv_prepare_pll(struct intel_crtc *crtc, coreclk = vlv_dpio_read(dev_priv, pipe, VLV_PLL_DW7(pipe)); coreclk = (coreclk & 0x0000ff00) | 0x01c00000; - if (intel_crtc_has_dp_encoder(pipe_config)) + if (intel_crtc_has_dp_encoder(crtc_state)) coreclk |= 0x01000000; vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW7(pipe), coreclk); @@ -1652,11 +1597,50 @@ void vlv_prepare_pll(struct intel_crtc *crtc, vlv_dpio_put(dev_priv); } -void chv_prepare_pll(struct intel_crtc *crtc, - const struct intel_crtc_state *pipe_config) +static void _vlv_enable_pll(const struct intel_crtc_state *crtc_state) { - struct drm_device *dev = crtc->base.dev; - struct drm_i915_private *dev_priv = to_i915(dev); + struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); + enum pipe pipe = crtc->pipe; + + intel_de_write(dev_priv, DPLL(pipe), crtc_state->dpll_hw_state.dpll); + intel_de_posting_read(dev_priv, DPLL(pipe)); + udelay(150); + + if (intel_de_wait_for_set(dev_priv, DPLL(pipe), DPLL_LOCK_VLV, 1)) + drm_err(&dev_priv->drm, "DPLL %d failed to lock\n", pipe); +} + +void vlv_enable_pll(const struct intel_crtc_state *crtc_state) +{ + struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); + enum pipe pipe = crtc->pipe; + + assert_transcoder_disabled(dev_priv, crtc_state->cpu_transcoder); + + /* PLL is protected by panel, make sure we can write it */ + assert_pps_unlocked(dev_priv, pipe); + + /* Enable Refclk */ + intel_de_write(dev_priv, DPLL(pipe), + crtc_state->dpll_hw_state.dpll & + ~(DPLL_VCO_ENABLE | DPLL_EXT_BUFFER_ENABLE_VLV)); + + if (crtc_state->dpll_hw_state.dpll & DPLL_VCO_ENABLE) { + vlv_prepare_pll(crtc_state); + _vlv_enable_pll(crtc_state); + } + + intel_de_write(dev_priv, DPLL_MD(pipe), + crtc_state->dpll_hw_state.dpll_md); + intel_de_posting_read(dev_priv, DPLL_MD(pipe)); +} + +static void chv_prepare_pll(const struct intel_crtc_state *crtc_state) +{ + struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); enum pipe pipe = crtc->pipe; enum dpio_channel port = vlv_pipe_to_channel(pipe); u32 loopfilter, tribuf_calcntr; @@ -1664,21 +1648,13 @@ void chv_prepare_pll(struct intel_crtc *crtc, u32 dpio_val; int vco; - /* Enable Refclk and SSC */ - intel_de_write(dev_priv, DPLL(pipe), - pipe_config->dpll_hw_state.dpll & ~DPLL_VCO_ENABLE); - - /* No need to actually set up the DPLL with DSI */ - if ((pipe_config->dpll_hw_state.dpll & DPLL_VCO_ENABLE) == 0) - return; - - bestn = pipe_config->dpll.n; - bestm2_frac = pipe_config->dpll.m2 & 0x3fffff; - bestm1 = pipe_config->dpll.m1; - bestm2 = pipe_config->dpll.m2 >> 22; - bestp1 = pipe_config->dpll.p1; - bestp2 = pipe_config->dpll.p2; - vco = pipe_config->dpll.vco; + bestn = crtc_state->dpll.n; + bestm2_frac = crtc_state->dpll.m2 & 0x3fffff; + bestm1 = crtc_state->dpll.m1; + bestm2 = crtc_state->dpll.m2 >> 22; + bestp1 = crtc_state->dpll.p1; + bestp2 = crtc_state->dpll.p2; + vco = crtc_state->dpll.vco; dpio_val = 0; loopfilter = 0; @@ -1757,6 +1733,83 @@ void chv_prepare_pll(struct intel_crtc *crtc, vlv_dpio_put(dev_priv); } +static void _chv_enable_pll(const struct intel_crtc_state *crtc_state) +{ + struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); + enum pipe pipe = crtc->pipe; + enum dpio_channel port = vlv_pipe_to_channel(pipe); + u32 tmp; + + vlv_dpio_get(dev_priv); + + /* Enable back the 10bit clock to display controller */ + tmp = vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW14(port)); + tmp |= DPIO_DCLKP_EN; + vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW14(port), tmp); + + vlv_dpio_put(dev_priv); + + /* + * Need to wait > 100ns between dclkp clock enable bit and PLL enable. + */ + udelay(1); + + /* Enable PLL */ + intel_de_write(dev_priv, DPLL(pipe), crtc_state->dpll_hw_state.dpll); + + /* Check PLL is locked */ + if (intel_de_wait_for_set(dev_priv, DPLL(pipe), DPLL_LOCK_VLV, 1)) + drm_err(&dev_priv->drm, "PLL %d failed to lock\n", pipe); +} + +void chv_enable_pll(const struct intel_crtc_state *crtc_state) +{ + struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); + enum pipe pipe = crtc->pipe; + + assert_transcoder_disabled(dev_priv, crtc_state->cpu_transcoder); + + /* PLL is protected by panel, make sure we can write it */ + assert_pps_unlocked(dev_priv, pipe); + + /* Enable Refclk and SSC */ + intel_de_write(dev_priv, DPLL(pipe), + crtc_state->dpll_hw_state.dpll & ~DPLL_VCO_ENABLE); + + if (crtc_state->dpll_hw_state.dpll & DPLL_VCO_ENABLE) { + chv_prepare_pll(crtc_state); + _chv_enable_pll(crtc_state); + } + + if (pipe != PIPE_A) { + /* + * WaPixelRepeatModeFixForC0:chv + * + * DPLLCMD is AWOL. Use chicken bits to propagate + * the value from DPLLBMD to either pipe B or C. + */ + intel_de_write(dev_priv, CBR4_VLV, CBR_DPLLBMD_PIPE(pipe)); + intel_de_write(dev_priv, DPLL_MD(PIPE_B), + crtc_state->dpll_hw_state.dpll_md); + intel_de_write(dev_priv, CBR4_VLV, 0); + dev_priv->chv_dpll_md[pipe] = crtc_state->dpll_hw_state.dpll_md; + + /* + * DPLLB VGA mode also seems to cause problems. + * We should always have it disabled. + */ + drm_WARN_ON(&dev_priv->drm, + (intel_de_read(dev_priv, DPLL(PIPE_B)) & + DPLL_VGA_MODE_DIS) == 0); + } else { + intel_de_write(dev_priv, DPLL_MD(pipe), + crtc_state->dpll_hw_state.dpll_md); + intel_de_posting_read(dev_priv, DPLL_MD(pipe)); + } +} + /** * vlv_force_pll_on - forcibly enable just the PLL * @dev_priv: i915 private structure @@ -1771,27 +1824,26 @@ int vlv_force_pll_on(struct drm_i915_private *dev_priv, enum pipe pipe, const struct dpll *dpll) { struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, pipe); - struct intel_crtc_state *pipe_config; + struct intel_crtc_state *crtc_state; - pipe_config = intel_crtc_state_alloc(crtc); - if (!pipe_config) + crtc_state = intel_crtc_state_alloc(crtc); + if (!crtc_state) return -ENOMEM; - pipe_config->cpu_transcoder = (enum transcoder)pipe; - pipe_config->pixel_multiplier = 1; - pipe_config->dpll = *dpll; + crtc_state->cpu_transcoder = (enum transcoder)pipe; + crtc_state->pixel_multiplier = 1; + crtc_state->dpll = *dpll; + crtc_state->output_types = BIT(INTEL_OUTPUT_EDP); if (IS_CHERRYVIEW(dev_priv)) { - chv_compute_dpll(crtc, pipe_config); - chv_prepare_pll(crtc, pipe_config); - chv_enable_pll(crtc, pipe_config); + chv_compute_dpll(crtc_state); + chv_enable_pll(crtc_state); } else { - vlv_compute_dpll(crtc, pipe_config); - vlv_prepare_pll(crtc, pipe_config); - vlv_enable_pll(crtc, pipe_config); + vlv_compute_dpll(crtc_state); + vlv_enable_pll(crtc_state); } - kfree(pipe_config); + kfree(crtc_state); return 0; } @@ -1801,7 +1853,7 @@ void vlv_disable_pll(struct drm_i915_private *dev_priv, enum pipe pipe) u32 val; /* Make sure the pipe isn't still relying on us */ - assert_pipe_disabled(dev_priv, (enum transcoder)pipe); + assert_transcoder_disabled(dev_priv, (enum transcoder)pipe); val = DPLL_INTEGRATED_REF_CLK_VLV | DPLL_REF_CLK_ENABLE_VLV | DPLL_VGA_MODE_DIS; @@ -1818,7 +1870,7 @@ void chv_disable_pll(struct drm_i915_private *dev_priv, enum pipe pipe) u32 val; /* Make sure the pipe isn't still relying on us */ - assert_pipe_disabled(dev_priv, (enum transcoder)pipe); + assert_transcoder_disabled(dev_priv, (enum transcoder)pipe); val = DPLL_SSC_REF_CLK_CHV | DPLL_REF_CLK_ENABLE_VLV | DPLL_VGA_MODE_DIS; @@ -1849,7 +1901,7 @@ void i9xx_disable_pll(const struct intel_crtc_state *crtc_state) return; /* Make sure the pipe isn't still relying on us */ - assert_pipe_disabled(dev_priv, crtc_state->cpu_transcoder); + assert_transcoder_disabled(dev_priv, crtc_state->cpu_transcoder); intel_de_write(dev_priv, DPLL(pipe), DPLL_VGA_MODE_DIS); intel_de_posting_read(dev_priv, DPLL(pipe)); @@ -1871,3 +1923,25 @@ void vlv_force_pll_off(struct drm_i915_private *dev_priv, enum pipe pipe) else vlv_disable_pll(dev_priv, pipe); } + +/* Only for pre-ILK configs */ +static void assert_pll(struct drm_i915_private *dev_priv, + enum pipe pipe, bool state) +{ + bool cur_state; + + cur_state = intel_de_read(dev_priv, DPLL(pipe)) & DPLL_VCO_ENABLE; + I915_STATE_WARN(cur_state != state, + "PLL state assertion failure (expected %s, current %s)\n", + onoff(state), onoff(cur_state)); +} + +void assert_pll_enabled(struct drm_i915_private *i915, enum pipe pipe) +{ + assert_pll(i915, pipe, true); +} + +void assert_pll_disabled(struct drm_i915_private *i915, enum pipe pipe) +{ + assert_pll(i915, pipe, false); +} diff --git a/drivers/gpu/drm/i915/display/intel_dpll.h b/drivers/gpu/drm/i915/display/intel_dpll.h index 88247027fd5a..1af0ac43cca4 100644 --- a/drivers/gpu/drm/i915/display/intel_dpll.h +++ b/drivers/gpu/drm/i915/display/intel_dpll.h @@ -18,29 +18,25 @@ void intel_dpll_init_clock_hook(struct drm_i915_private *dev_priv); int vlv_calc_dpll_params(int refclk, struct dpll *clock); int pnv_calc_dpll_params(int refclk, struct dpll *clock); int i9xx_calc_dpll_params(int refclk, struct dpll *clock); -void vlv_compute_dpll(struct intel_crtc *crtc, - struct intel_crtc_state *pipe_config); -void chv_compute_dpll(struct intel_crtc *crtc, - struct intel_crtc_state *pipe_config); +u32 i9xx_dpll_compute_fp(const struct dpll *dpll); +void vlv_compute_dpll(struct intel_crtc_state *crtc_state); +void chv_compute_dpll(struct intel_crtc_state *crtc_state); int vlv_force_pll_on(struct drm_i915_private *dev_priv, enum pipe pipe, const struct dpll *dpll); void vlv_force_pll_off(struct drm_i915_private *dev_priv, enum pipe pipe); -void i9xx_enable_pll(struct intel_crtc *crtc, - const struct intel_crtc_state *crtc_state); -void vlv_enable_pll(struct intel_crtc *crtc, - const struct intel_crtc_state *pipe_config); -void chv_enable_pll(struct intel_crtc *crtc, - const struct intel_crtc_state *pipe_config); -void vlv_disable_pll(struct drm_i915_private *dev_priv, enum pipe pipe); + +void chv_enable_pll(const struct intel_crtc_state *crtc_state); void chv_disable_pll(struct drm_i915_private *dev_priv, enum pipe pipe); +void vlv_enable_pll(const struct intel_crtc_state *crtc_state); +void vlv_disable_pll(struct drm_i915_private *dev_priv, enum pipe pipe); +void i9xx_enable_pll(const struct intel_crtc_state *crtc_state); void i9xx_disable_pll(const struct intel_crtc_state *crtc_state); -void vlv_prepare_pll(struct intel_crtc *crtc, - const struct intel_crtc_state *pipe_config); -void chv_prepare_pll(struct intel_crtc *crtc, - const struct intel_crtc_state *pipe_config); bool bxt_find_best_dpll(struct intel_crtc_state *crtc_state, struct dpll *best_clock); int chv_calc_dpll_params(int refclk, struct dpll *pll_clock); +void assert_pll_enabled(struct drm_i915_private *i915, enum pipe pipe); +void assert_pll_disabled(struct drm_i915_private *i915, enum pipe pipe); + #endif diff --git a/drivers/gpu/drm/i915/display/intel_dpll_mgr.c b/drivers/gpu/drm/i915/display/intel_dpll_mgr.c index 5c91d125a337..0a7e04db04be 100644 --- a/drivers/gpu/drm/i915/display/intel_dpll_mgr.c +++ b/drivers/gpu/drm/i915/display/intel_dpll_mgr.c @@ -26,6 +26,7 @@ #include "intel_dpio_phy.h" #include "intel_dpll.h" #include "intel_dpll_mgr.h" +#include "intel_tc.h" /** * DOC: Display PLLs @@ -185,34 +186,6 @@ intel_tc_pll_enable_reg(struct drm_i915_private *i915, } /** - * intel_prepare_shared_dpll - call a dpll's prepare hook - * @crtc_state: CRTC, and its state, which has a shared dpll - * - * This calls the PLL's prepare hook if it has one and if the PLL is not - * already enabled. The prepare hook is platform specific. - */ -void intel_prepare_shared_dpll(const struct intel_crtc_state *crtc_state) -{ - struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); - struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); - struct intel_shared_dpll *pll = crtc_state->shared_dpll; - - if (drm_WARN_ON(&dev_priv->drm, pll == NULL)) - return; - - mutex_lock(&dev_priv->dpll.lock); - drm_WARN_ON(&dev_priv->drm, !pll->state.pipe_mask); - if (!pll->active_mask) { - drm_dbg(&dev_priv->drm, "setting up %s\n", pll->info->name); - drm_WARN_ON(&dev_priv->drm, pll->on); - assert_shared_dpll_disabled(dev_priv, pll); - - pll->info->funcs->prepare(dev_priv, pll); - } - mutex_unlock(&dev_priv->dpll.lock); -} - -/** * intel_enable_shared_dpll - enable a CRTC's shared DPLL * @crtc_state: CRTC, and its state, which has a shared DPLL * @@ -451,15 +424,6 @@ static bool ibx_pch_dpll_get_hw_state(struct drm_i915_private *dev_priv, return val & DPLL_VCO_ENABLE; } -static void ibx_pch_dpll_prepare(struct drm_i915_private *dev_priv, - struct intel_shared_dpll *pll) -{ - const enum intel_dpll_id id = pll->info->id; - - intel_de_write(dev_priv, PCH_FP0(id), pll->state.hw_state.fp0); - intel_de_write(dev_priv, PCH_FP1(id), pll->state.hw_state.fp1); -} - static void ibx_assert_pch_refclk_enabled(struct drm_i915_private *dev_priv) { u32 val; @@ -481,6 +445,9 @@ static void ibx_pch_dpll_enable(struct drm_i915_private *dev_priv, /* PCH refclock must be enabled first */ ibx_assert_pch_refclk_enabled(dev_priv); + intel_de_write(dev_priv, PCH_FP0(id), pll->state.hw_state.fp0); + intel_de_write(dev_priv, PCH_FP1(id), pll->state.hw_state.fp1); + intel_de_write(dev_priv, PCH_DPLL(id), pll->state.hw_state.dpll); /* Wait for the clocks to stabilize. */ @@ -558,7 +525,6 @@ static void ibx_dump_hw_state(struct drm_i915_private *dev_priv, } static const struct intel_shared_dpll_funcs ibx_pch_dpll_funcs = { - .prepare = ibx_pch_dpll_prepare, .enable = ibx_pch_dpll_enable, .disable = ibx_pch_dpll_disable, .get_hw_state = ibx_pch_dpll_get_hw_state, @@ -3136,8 +3102,8 @@ static void icl_update_active_dpll(struct intel_atomic_state *state, enc_to_dig_port(encoder); if (primary_port && - (primary_port->tc_mode == TC_PORT_DP_ALT || - primary_port->tc_mode == TC_PORT_LEGACY)) + (intel_tc_port_in_dp_alt_mode(primary_port) || + intel_tc_port_in_legacy_mode(primary_port))) port_dpll_id = ICL_PORT_DPLL_MG_PHY; icl_set_active_port_dpll(crtc_state, port_dpll_id); diff --git a/drivers/gpu/drm/i915/display/intel_dpll_mgr.h b/drivers/gpu/drm/i915/display/intel_dpll_mgr.h index 30e0aa5ca109..2f59d863be4c 100644 --- a/drivers/gpu/drm/i915/display/intel_dpll_mgr.h +++ b/drivers/gpu/drm/i915/display/intel_dpll_mgr.h @@ -256,16 +256,6 @@ struct intel_shared_dpll_state { */ struct intel_shared_dpll_funcs { /** - * @prepare: - * - * Optional hook to perform operations prior to enabling the PLL. - * Called from intel_prepare_shared_dpll() function unless the PLL - * is already enabled. - */ - void (*prepare)(struct drm_i915_private *dev_priv, - struct intel_shared_dpll *pll); - - /** * @enable: * * Hook for enabling the pll, called from intel_enable_shared_dpll() @@ -404,7 +394,6 @@ int intel_dpll_get_freq(struct drm_i915_private *i915, bool intel_dpll_get_hw_state(struct drm_i915_private *i915, struct intel_shared_dpll *pll, struct intel_dpll_hw_state *hw_state); -void intel_prepare_shared_dpll(const struct intel_crtc_state *crtc_state); void intel_enable_shared_dpll(const struct intel_crtc_state *crtc_state); void intel_disable_shared_dpll(const struct intel_crtc_state *crtc_state); void intel_shared_dpll_swap_state(struct intel_atomic_state *state); diff --git a/drivers/gpu/drm/i915/display/intel_dpt.c b/drivers/gpu/drm/i915/display/intel_dpt.c new file mode 100644 index 000000000000..8f7b1f7534a4 --- /dev/null +++ b/drivers/gpu/drm/i915/display/intel_dpt.c @@ -0,0 +1,239 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2021 Intel Corporation + */ + +#include "i915_drv.h" +#include "intel_display_types.h" +#include "intel_dpt.h" +#include "intel_fb.h" +#include "gt/gen8_ppgtt.h" + +struct i915_dpt { + struct i915_address_space vm; + + struct drm_i915_gem_object *obj; + struct i915_vma *vma; + void __iomem *iomem; +}; + +#define i915_is_dpt(vm) ((vm)->is_dpt) + +static inline struct i915_dpt * +i915_vm_to_dpt(struct i915_address_space *vm) +{ + BUILD_BUG_ON(offsetof(struct i915_dpt, vm)); + GEM_BUG_ON(!i915_is_dpt(vm)); + return container_of(vm, struct i915_dpt, vm); +} + +#define dpt_total_entries(dpt) ((dpt)->vm.total >> PAGE_SHIFT) + +static void gen8_set_pte(void __iomem *addr, gen8_pte_t pte) +{ + writeq(pte, addr); +} + +static void dpt_insert_page(struct i915_address_space *vm, + dma_addr_t addr, + u64 offset, + enum i915_cache_level level, + u32 flags) +{ + struct i915_dpt *dpt = i915_vm_to_dpt(vm); + gen8_pte_t __iomem *base = dpt->iomem; + + gen8_set_pte(base + offset / I915_GTT_PAGE_SIZE, + vm->pte_encode(addr, level, flags)); +} + +static void dpt_insert_entries(struct i915_address_space *vm, + struct i915_vma *vma, + enum i915_cache_level level, + u32 flags) +{ + struct i915_dpt *dpt = i915_vm_to_dpt(vm); + gen8_pte_t __iomem *base = dpt->iomem; + const gen8_pte_t pte_encode = vm->pte_encode(0, level, flags); + struct sgt_iter sgt_iter; + dma_addr_t addr; + int i; + + /* + * Note that we ignore PTE_READ_ONLY here. The caller must be careful + * not to allow the user to override access to a read only page. + */ + + i = vma->node.start / I915_GTT_PAGE_SIZE; + for_each_sgt_daddr(addr, sgt_iter, vma->pages) + gen8_set_pte(&base[i++], pte_encode | addr); +} + +static void dpt_clear_range(struct i915_address_space *vm, + u64 start, u64 length) +{ +} + +static void dpt_bind_vma(struct i915_address_space *vm, + struct i915_vm_pt_stash *stash, + struct i915_vma *vma, + enum i915_cache_level cache_level, + u32 flags) +{ + struct drm_i915_gem_object *obj = vma->obj; + u32 pte_flags; + + /* Applicable to VLV (gen8+ do not support RO in the GGTT) */ + pte_flags = 0; + if (vma->vm->has_read_only && i915_gem_object_is_readonly(obj)) + pte_flags |= PTE_READ_ONLY; + if (i915_gem_object_is_lmem(obj)) + pte_flags |= PTE_LM; + + vma->vm->insert_entries(vma->vm, vma, cache_level, pte_flags); + + vma->page_sizes.gtt = I915_GTT_PAGE_SIZE; + + /* + * Without aliasing PPGTT there's no difference between + * GLOBAL/LOCAL_BIND, it's all the same ptes. Hence unconditionally + * upgrade to both bound if we bind either to avoid double-binding. + */ + atomic_or(I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND, &vma->flags); +} + +static void dpt_unbind_vma(struct i915_address_space *vm, struct i915_vma *vma) +{ + vm->clear_range(vm, vma->node.start, vma->size); +} + +static void dpt_cleanup(struct i915_address_space *vm) +{ + struct i915_dpt *dpt = i915_vm_to_dpt(vm); + + i915_gem_object_put(dpt->obj); +} + +struct i915_vma *intel_dpt_pin(struct i915_address_space *vm) +{ + struct drm_i915_private *i915 = vm->i915; + struct i915_dpt *dpt = i915_vm_to_dpt(vm); + intel_wakeref_t wakeref; + struct i915_vma *vma; + void __iomem *iomem; + struct i915_gem_ww_ctx ww; + int err; + + wakeref = intel_runtime_pm_get(&i915->runtime_pm); + atomic_inc(&i915->gpu_error.pending_fb_pin); + + for_i915_gem_ww(&ww, err, true) { + err = i915_gem_object_lock(dpt->obj, &ww); + if (err) + continue; + + vma = i915_gem_object_ggtt_pin_ww(dpt->obj, &ww, NULL, 0, 4096, + HAS_LMEM(i915) ? 0 : PIN_MAPPABLE); + if (IS_ERR(vma)) { + err = PTR_ERR(vma); + continue; + } + + iomem = i915_vma_pin_iomap(vma); + i915_vma_unpin(vma); + + if (IS_ERR(iomem)) { + err = PTR_ERR(iomem); + continue; + } + + dpt->vma = vma; + dpt->iomem = iomem; + + i915_vma_get(vma); + } + + atomic_dec(&i915->gpu_error.pending_fb_pin); + intel_runtime_pm_put(&i915->runtime_pm, wakeref); + + return err ? ERR_PTR(err) : vma; +} + +void intel_dpt_unpin(struct i915_address_space *vm) +{ + struct i915_dpt *dpt = i915_vm_to_dpt(vm); + + i915_vma_unpin_iomap(dpt->vma); + i915_vma_put(dpt->vma); +} + +struct i915_address_space * +intel_dpt_create(struct intel_framebuffer *fb) +{ + struct drm_gem_object *obj = &intel_fb_obj(&fb->base)->base; + struct drm_i915_private *i915 = to_i915(obj->dev); + struct drm_i915_gem_object *dpt_obj; + struct i915_address_space *vm; + struct i915_dpt *dpt; + size_t size; + int ret; + + if (intel_fb_needs_pot_stride_remap(fb)) + size = intel_remapped_info_size(&fb->remapped_view.gtt.remapped); + else + size = DIV_ROUND_UP_ULL(obj->size, I915_GTT_PAGE_SIZE); + + size = round_up(size * sizeof(gen8_pte_t), I915_GTT_PAGE_SIZE); + + if (HAS_LMEM(i915)) + dpt_obj = i915_gem_object_create_lmem(i915, size, 0); + else + dpt_obj = i915_gem_object_create_stolen(i915, size); + if (IS_ERR(dpt_obj)) + return ERR_CAST(dpt_obj); + + ret = i915_gem_object_set_cache_level(dpt_obj, I915_CACHE_NONE); + if (ret) { + i915_gem_object_put(dpt_obj); + return ERR_PTR(ret); + } + + dpt = kzalloc(sizeof(*dpt), GFP_KERNEL); + if (!dpt) { + i915_gem_object_put(dpt_obj); + return ERR_PTR(-ENOMEM); + } + + vm = &dpt->vm; + + vm->gt = &i915->gt; + vm->i915 = i915; + vm->dma = i915->drm.dev; + vm->total = (size / sizeof(gen8_pte_t)) * I915_GTT_PAGE_SIZE; + vm->is_dpt = true; + + i915_address_space_init(vm, VM_CLASS_DPT); + + vm->insert_page = dpt_insert_page; + vm->clear_range = dpt_clear_range; + vm->insert_entries = dpt_insert_entries; + vm->cleanup = dpt_cleanup; + + vm->vma_ops.bind_vma = dpt_bind_vma; + vm->vma_ops.unbind_vma = dpt_unbind_vma; + vm->vma_ops.set_pages = ggtt_set_pages; + vm->vma_ops.clear_pages = clear_pages; + + vm->pte_encode = gen8_ggtt_pte_encode; + + dpt->obj = dpt_obj; + + return &dpt->vm; +} + +void intel_dpt_destroy(struct i915_address_space *vm) +{ + struct i915_dpt *dpt = i915_vm_to_dpt(vm); + + i915_vm_close(&dpt->vm); +} diff --git a/drivers/gpu/drm/i915/display/intel_dpt.h b/drivers/gpu/drm/i915/display/intel_dpt.h new file mode 100644 index 000000000000..45142b8f849f --- /dev/null +++ b/drivers/gpu/drm/i915/display/intel_dpt.h @@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2021 Intel Corporation + */ + +#ifndef __INTEL_DPT_H__ +#define __INTEL_DPT_H__ + +struct i915_address_space; +struct i915_vma; +struct intel_framebuffer; + +void intel_dpt_destroy(struct i915_address_space *vm); +struct i915_vma *intel_dpt_pin(struct i915_address_space *vm); +void intel_dpt_unpin(struct i915_address_space *vm); +struct i915_address_space * +intel_dpt_create(struct intel_framebuffer *fb); + +#endif /* __INTEL_DPT_H__ */ diff --git a/drivers/gpu/drm/i915/display/intel_drrs.c b/drivers/gpu/drm/i915/display/intel_drrs.c new file mode 100644 index 000000000000..c1439fcb5a95 --- /dev/null +++ b/drivers/gpu/drm/i915/display/intel_drrs.c @@ -0,0 +1,437 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2021 Intel Corporation + */ + +#include "i915_drv.h" +#include "intel_atomic.h" +#include "intel_de.h" +#include "intel_display_types.h" +#include "intel_drrs.h" +#include "intel_panel.h" + +/** + * DOC: Display Refresh Rate Switching (DRRS) + * + * Display Refresh Rate Switching (DRRS) is a power conservation feature + * which enables swtching between low and high refresh rates, + * dynamically, based on the usage scenario. This feature is applicable + * for internal panels. + * + * Indication that the panel supports DRRS is given by the panel EDID, which + * would list multiple refresh rates for one resolution. + * + * DRRS is of 2 types - static and seamless. + * Static DRRS involves changing refresh rate (RR) by doing a full modeset + * (may appear as a blink on screen) and is used in dock-undock scenario. + * Seamless DRRS involves changing RR without any visual effect to the user + * and can be used during normal system usage. This is done by programming + * certain registers. + * + * Support for static/seamless DRRS may be indicated in the VBT based on + * inputs from the panel spec. + * + * DRRS saves power by switching to low RR based on usage scenarios. + * + * The implementation is based on frontbuffer tracking implementation. When + * there is a disturbance on the screen triggered by user activity or a periodic + * system activity, DRRS is disabled (RR is changed to high RR). When there is + * no movement on screen, after a timeout of 1 second, a switch to low RR is + * made. + * + * For integration with frontbuffer tracking code, intel_drrs_invalidate() + * and intel_drrs_flush() are called. + * + * DRRS can be further extended to support other internal panels and also + * the scenario of video playback wherein RR is set based on the rate + * requested by userspace. + */ + +void +intel_drrs_compute_config(struct intel_dp *intel_dp, + struct intel_crtc_state *pipe_config, + int output_bpp, bool constant_n) +{ + struct intel_connector *intel_connector = intel_dp->attached_connector; + struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); + int pixel_clock; + + if (pipe_config->vrr.enable) + return; + + /* + * DRRS and PSR can't be enable together, so giving preference to PSR + * as it allows more power-savings by complete shutting down display, + * so to guarantee this, intel_drrs_compute_config() must be called + * after intel_psr_compute_config(). + */ + if (pipe_config->has_psr) + return; + + if (!intel_connector->panel.downclock_mode || + dev_priv->drrs.type != SEAMLESS_DRRS_SUPPORT) + return; + + pipe_config->has_drrs = true; + + pixel_clock = intel_connector->panel.downclock_mode->clock; + if (pipe_config->splitter.enable) + pixel_clock /= pipe_config->splitter.link_count; + + intel_link_compute_m_n(output_bpp, pipe_config->lane_count, pixel_clock, + pipe_config->port_clock, &pipe_config->dp_m2_n2, + constant_n, pipe_config->fec_enable); + + /* FIXME: abstract this better */ + if (pipe_config->splitter.enable) + pipe_config->dp_m2_n2.gmch_m *= pipe_config->splitter.link_count; +} + +static void intel_drrs_set_state(struct drm_i915_private *dev_priv, + const struct intel_crtc_state *crtc_state, + enum drrs_refresh_rate_type refresh_type) +{ + struct intel_dp *intel_dp = dev_priv->drrs.dp; + struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); + struct drm_display_mode *mode; + + if (!intel_dp) { + drm_dbg_kms(&dev_priv->drm, "DRRS not supported.\n"); + return; + } + + if (!crtc) { + drm_dbg_kms(&dev_priv->drm, + "DRRS: intel_crtc not initialized\n"); + return; + } + + if (dev_priv->drrs.type < SEAMLESS_DRRS_SUPPORT) { + drm_dbg_kms(&dev_priv->drm, "Only Seamless DRRS supported.\n"); + return; + } + + if (refresh_type == dev_priv->drrs.refresh_rate_type) + return; + + if (!crtc_state->hw.active) { + drm_dbg_kms(&dev_priv->drm, + "eDP encoder disabled. CRTC not Active\n"); + return; + } + + if (DISPLAY_VER(dev_priv) >= 8 && !IS_CHERRYVIEW(dev_priv)) { + switch (refresh_type) { + case DRRS_HIGH_RR: + intel_dp_set_m_n(crtc_state, M1_N1); + break; + case DRRS_LOW_RR: + intel_dp_set_m_n(crtc_state, M2_N2); + break; + case DRRS_MAX_RR: + default: + drm_err(&dev_priv->drm, + "Unsupported refreshrate type\n"); + } + } else if (DISPLAY_VER(dev_priv) > 6) { + i915_reg_t reg = PIPECONF(crtc_state->cpu_transcoder); + u32 val; + + val = intel_de_read(dev_priv, reg); + if (refresh_type == DRRS_LOW_RR) { + if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) + val |= PIPECONF_EDP_RR_MODE_SWITCH_VLV; + else + val |= PIPECONF_EDP_RR_MODE_SWITCH; + } else { + if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) + val &= ~PIPECONF_EDP_RR_MODE_SWITCH_VLV; + else + val &= ~PIPECONF_EDP_RR_MODE_SWITCH; + } + intel_de_write(dev_priv, reg, val); + } + + dev_priv->drrs.refresh_rate_type = refresh_type; + + if (refresh_type == DRRS_LOW_RR) + mode = intel_dp->attached_connector->panel.downclock_mode; + else + mode = intel_dp->attached_connector->panel.fixed_mode; + drm_dbg_kms(&dev_priv->drm, "eDP Refresh Rate set to : %dHz\n", + drm_mode_vrefresh(mode)); +} + +static void +intel_drrs_enable_locked(struct intel_dp *intel_dp) +{ + struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); + + dev_priv->drrs.busy_frontbuffer_bits = 0; + dev_priv->drrs.dp = intel_dp; +} + +/** + * intel_drrs_enable - init drrs struct if supported + * @intel_dp: DP struct + * @crtc_state: A pointer to the active crtc state. + * + * Initializes frontbuffer_bits and drrs.dp + */ +void intel_drrs_enable(struct intel_dp *intel_dp, + const struct intel_crtc_state *crtc_state) +{ + struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); + + if (!crtc_state->has_drrs) + return; + + drm_dbg_kms(&dev_priv->drm, "Enabling DRRS\n"); + + mutex_lock(&dev_priv->drrs.mutex); + + if (dev_priv->drrs.dp) { + drm_warn(&dev_priv->drm, "DRRS already enabled\n"); + goto unlock; + } + + intel_drrs_enable_locked(intel_dp); + +unlock: + mutex_unlock(&dev_priv->drrs.mutex); +} + +static void +intel_drrs_disable_locked(struct intel_dp *intel_dp, + const struct intel_crtc_state *crtc_state) +{ + struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); + + intel_drrs_set_state(dev_priv, crtc_state, DRRS_HIGH_RR); + dev_priv->drrs.dp = NULL; +} + +/** + * intel_drrs_disable - Disable DRRS + * @intel_dp: DP struct + * @old_crtc_state: Pointer to old crtc_state. + * + */ +void intel_drrs_disable(struct intel_dp *intel_dp, + const struct intel_crtc_state *old_crtc_state) +{ + struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); + + if (!old_crtc_state->has_drrs) + return; + + mutex_lock(&dev_priv->drrs.mutex); + if (!dev_priv->drrs.dp) { + mutex_unlock(&dev_priv->drrs.mutex); + return; + } + + intel_drrs_disable_locked(intel_dp, old_crtc_state); + mutex_unlock(&dev_priv->drrs.mutex); + + cancel_delayed_work_sync(&dev_priv->drrs.work); +} + +/** + * intel_drrs_update - Update DRRS state + * @intel_dp: Intel DP + * @crtc_state: new CRTC state + * + * This function will update DRRS states, disabling or enabling DRRS when + * executing fastsets. For full modeset, intel_drrs_disable() and + * intel_drrs_enable() should be called instead. + */ +void +intel_drrs_update(struct intel_dp *intel_dp, + const struct intel_crtc_state *crtc_state) +{ + struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); + + if (dev_priv->drrs.type != SEAMLESS_DRRS_SUPPORT) + return; + + mutex_lock(&dev_priv->drrs.mutex); + + /* New state matches current one? */ + if (crtc_state->has_drrs == !!dev_priv->drrs.dp) + goto unlock; + + if (crtc_state->has_drrs) + intel_drrs_enable_locked(intel_dp); + else + intel_drrs_disable_locked(intel_dp, crtc_state); + +unlock: + mutex_unlock(&dev_priv->drrs.mutex); +} + +static void intel_drrs_downclock_work(struct work_struct *work) +{ + struct drm_i915_private *dev_priv = + container_of(work, typeof(*dev_priv), drrs.work.work); + struct intel_dp *intel_dp; + struct drm_crtc *crtc; + + mutex_lock(&dev_priv->drrs.mutex); + + intel_dp = dev_priv->drrs.dp; + + if (!intel_dp) + goto unlock; + + /* + * The delayed work can race with an invalidate hence we need to + * recheck. + */ + + if (dev_priv->drrs.busy_frontbuffer_bits) + goto unlock; + + crtc = dp_to_dig_port(intel_dp)->base.base.crtc; + intel_drrs_set_state(dev_priv, to_intel_crtc(crtc)->config, DRRS_LOW_RR); + +unlock: + mutex_unlock(&dev_priv->drrs.mutex); +} + +static void intel_drrs_frontbuffer_update(struct drm_i915_private *dev_priv, + unsigned int frontbuffer_bits, + bool invalidate) +{ + struct intel_dp *intel_dp; + struct drm_crtc *crtc; + enum pipe pipe; + + if (dev_priv->drrs.type == DRRS_NOT_SUPPORTED) + return; + + cancel_delayed_work(&dev_priv->drrs.work); + + mutex_lock(&dev_priv->drrs.mutex); + + intel_dp = dev_priv->drrs.dp; + if (!intel_dp) { + mutex_unlock(&dev_priv->drrs.mutex); + return; + } + + crtc = dp_to_dig_port(intel_dp)->base.base.crtc; + pipe = to_intel_crtc(crtc)->pipe; + + frontbuffer_bits &= INTEL_FRONTBUFFER_ALL_MASK(pipe); + if (invalidate) + dev_priv->drrs.busy_frontbuffer_bits |= frontbuffer_bits; + else + dev_priv->drrs.busy_frontbuffer_bits &= ~frontbuffer_bits; + + /* flush/invalidate means busy screen hence upclock */ + if (frontbuffer_bits) + intel_drrs_set_state(dev_priv, to_intel_crtc(crtc)->config, + DRRS_HIGH_RR); + + /* + * flush also means no more activity hence schedule downclock, if all + * other fbs are quiescent too + */ + if (!invalidate && !dev_priv->drrs.busy_frontbuffer_bits) + schedule_delayed_work(&dev_priv->drrs.work, + msecs_to_jiffies(1000)); + mutex_unlock(&dev_priv->drrs.mutex); +} + +/** + * intel_drrs_invalidate - Disable Idleness DRRS + * @dev_priv: i915 device + * @frontbuffer_bits: frontbuffer plane tracking bits + * + * This function gets called everytime rendering on the given planes start. + * Hence DRRS needs to be Upclocked, i.e. (LOW_RR -> HIGH_RR). + * + * Dirty frontbuffers relevant to DRRS are tracked in busy_frontbuffer_bits. + */ +void intel_drrs_invalidate(struct drm_i915_private *dev_priv, + unsigned int frontbuffer_bits) +{ + intel_drrs_frontbuffer_update(dev_priv, frontbuffer_bits, true); +} + +/** + * intel_drrs_flush - Restart Idleness DRRS + * @dev_priv: i915 device + * @frontbuffer_bits: frontbuffer plane tracking bits + * + * This function gets called every time rendering on the given planes has + * completed or flip on a crtc is completed. So DRRS should be upclocked + * (LOW_RR -> HIGH_RR). And also Idleness detection should be started again, + * if no other planes are dirty. + * + * Dirty frontbuffers relevant to DRRS are tracked in busy_frontbuffer_bits. + */ +void intel_drrs_flush(struct drm_i915_private *dev_priv, + unsigned int frontbuffer_bits) +{ + intel_drrs_frontbuffer_update(dev_priv, frontbuffer_bits, false); +} + +void intel_drrs_page_flip(struct intel_atomic_state *state, + struct intel_crtc *crtc) +{ + struct drm_i915_private *dev_priv = to_i915(state->base.dev); + unsigned int frontbuffer_bits = INTEL_FRONTBUFFER_ALL_MASK(crtc->pipe); + + intel_drrs_frontbuffer_update(dev_priv, frontbuffer_bits, false); +} + +/** + * intel_drrs_init - Init basic DRRS work and mutex. + * @connector: eDP connector + * @fixed_mode: preferred mode of panel + * + * This function is called only once at driver load to initialize basic + * DRRS stuff. + * + * Returns: + * Downclock mode if panel supports it, else return NULL. + * DRRS support is determined by the presence of downclock mode (apart + * from VBT setting). + */ +struct drm_display_mode * +intel_drrs_init(struct intel_connector *connector, + struct drm_display_mode *fixed_mode) +{ + struct drm_i915_private *dev_priv = to_i915(connector->base.dev); + struct drm_display_mode *downclock_mode = NULL; + + INIT_DELAYED_WORK(&dev_priv->drrs.work, intel_drrs_downclock_work); + mutex_init(&dev_priv->drrs.mutex); + + if (DISPLAY_VER(dev_priv) <= 6) { + drm_dbg_kms(&dev_priv->drm, + "DRRS supported for Gen7 and above\n"); + return NULL; + } + + if (dev_priv->vbt.drrs_type != SEAMLESS_DRRS_SUPPORT) { + drm_dbg_kms(&dev_priv->drm, "VBT doesn't support DRRS\n"); + return NULL; + } + + downclock_mode = intel_panel_edid_downclock_mode(connector, fixed_mode); + if (!downclock_mode) { + drm_dbg_kms(&dev_priv->drm, + "Downclock mode is not found. DRRS not supported\n"); + return NULL; + } + + dev_priv->drrs.type = dev_priv->vbt.drrs_type; + + dev_priv->drrs.refresh_rate_type = DRRS_HIGH_RR; + drm_dbg_kms(&dev_priv->drm, + "seamless DRRS supported for eDP panel.\n"); + return downclock_mode; +} diff --git a/drivers/gpu/drm/i915/display/intel_drrs.h b/drivers/gpu/drm/i915/display/intel_drrs.h new file mode 100644 index 000000000000..9ec9c447211a --- /dev/null +++ b/drivers/gpu/drm/i915/display/intel_drrs.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2021 Intel Corporation + */ + +#ifndef __INTEL_DRRS_H__ +#define __INTEL_DRRS_H__ + +#include <linux/types.h> + +struct drm_i915_private; +struct intel_atomic_state; +struct intel_crtc; +struct intel_crtc_state; +struct intel_connector; +struct intel_dp; + +void intel_drrs_enable(struct intel_dp *intel_dp, + const struct intel_crtc_state *crtc_state); +void intel_drrs_disable(struct intel_dp *intel_dp, + const struct intel_crtc_state *crtc_state); +void intel_drrs_update(struct intel_dp *intel_dp, + const struct intel_crtc_state *crtc_state); +void intel_drrs_invalidate(struct drm_i915_private *dev_priv, + unsigned int frontbuffer_bits); +void intel_drrs_flush(struct drm_i915_private *dev_priv, + unsigned int frontbuffer_bits); +void intel_drrs_page_flip(struct intel_atomic_state *state, + struct intel_crtc *crtc); +void intel_drrs_compute_config(struct intel_dp *intel_dp, + struct intel_crtc_state *pipe_config, + int output_bpp, bool constant_n); +struct drm_display_mode *intel_drrs_init(struct intel_connector *connector, + struct drm_display_mode *fixed_mode); + +#endif /* __INTEL_DRRS_H__ */ diff --git a/drivers/gpu/drm/i915/display/intel_dsi.c b/drivers/gpu/drm/i915/display/intel_dsi.c index f453ceb8d149..6b0301ba046e 100644 --- a/drivers/gpu/drm/i915/display/intel_dsi.c +++ b/drivers/gpu/drm/i915/display/intel_dsi.c @@ -5,6 +5,7 @@ #include <drm/drm_mipi_dsi.h> #include "intel_dsi.h" +#include "intel_panel.h" int intel_dsi_bitrate(const struct intel_dsi *intel_dsi) { @@ -60,20 +61,19 @@ enum drm_mode_status intel_dsi_mode_valid(struct drm_connector *connector, struct intel_connector *intel_connector = to_intel_connector(connector); const struct drm_display_mode *fixed_mode = intel_connector->panel.fixed_mode; int max_dotclk = to_i915(connector->dev)->max_dotclk_freq; + enum drm_mode_status status; drm_dbg_kms(&dev_priv->drm, "\n"); if (mode->flags & DRM_MODE_FLAG_DBLSCAN) return MODE_NO_DBLESCAN; - if (fixed_mode) { - if (mode->hdisplay > fixed_mode->hdisplay) - return MODE_PANEL; - if (mode->vdisplay > fixed_mode->vdisplay) - return MODE_PANEL; - if (fixed_mode->clock > max_dotclk) - return MODE_CLOCK_HIGH; - } + status = intel_panel_mode_valid(intel_connector, mode); + if (status != MODE_OK) + return status; + + if (fixed_mode->clock > max_dotclk) + return MODE_CLOCK_HIGH; return intel_mode_valid_max_plane_size(dev_priv, mode, false); } diff --git a/drivers/gpu/drm/i915/display/intel_dsi.h b/drivers/gpu/drm/i915/display/intel_dsi.h index 50d6da0b2419..fbc40ffdc02e 100644 --- a/drivers/gpu/drm/i915/display/intel_dsi.h +++ b/drivers/gpu/drm/i915/display/intel_dsi.h @@ -207,6 +207,9 @@ u32 bxt_dsi_get_pclk(struct intel_encoder *encoder, struct intel_crtc_state *config); void bxt_dsi_reset_clocks(struct intel_encoder *encoder, enum port port); +void assert_dsi_pll_enabled(struct drm_i915_private *i915); +void assert_dsi_pll_disabled(struct drm_i915_private *i915); + /* intel_dsi_vbt.c */ bool intel_dsi_vbt_init(struct intel_dsi *intel_dsi, u16 panel_id); void intel_dsi_vbt_gpio_init(struct intel_dsi *intel_dsi, bool panel_is_on); diff --git a/drivers/gpu/drm/i915/display/intel_dsi_dcs_backlight.c b/drivers/gpu/drm/i915/display/intel_dsi_dcs_backlight.c index 584c14c4cbd0..f61ed82e8867 100644 --- a/drivers/gpu/drm/i915/display/intel_dsi_dcs_backlight.c +++ b/drivers/gpu/drm/i915/display/intel_dsi_dcs_backlight.c @@ -47,33 +47,42 @@ static u32 dcs_get_backlight(struct intel_connector *connector, enum pipe unused { struct intel_encoder *encoder = intel_attached_encoder(connector); struct intel_dsi *intel_dsi = enc_to_intel_dsi(encoder); + struct intel_panel *panel = &connector->panel; struct mipi_dsi_device *dsi_device; - u8 data = 0; + u8 data[2] = {}; enum port port; + size_t len = panel->backlight.max > U8_MAX ? 2 : 1; - /* FIXME: Need to take care of 16 bit brightness level */ for_each_dsi_port(port, intel_dsi->dcs_backlight_ports) { dsi_device = intel_dsi->dsi_hosts[port]->device; mipi_dsi_dcs_read(dsi_device, MIPI_DCS_GET_DISPLAY_BRIGHTNESS, - &data, sizeof(data)); + &data, len); break; } - return data; + return (data[1] << 8) | data[0]; } static void dcs_set_backlight(const struct drm_connector_state *conn_state, u32 level) { struct intel_dsi *intel_dsi = enc_to_intel_dsi(to_intel_encoder(conn_state->best_encoder)); + struct intel_panel *panel = &to_intel_connector(conn_state->connector)->panel; struct mipi_dsi_device *dsi_device; - u8 data = level; + u8 data[2] = {}; enum port port; + size_t len = panel->backlight.max > U8_MAX ? 2 : 1; + + if (len == 1) { + data[0] = level; + } else { + data[0] = level >> 8; + data[1] = level; + } - /* FIXME: Need to take care of 16 bit brightness level */ for_each_dsi_port(port, intel_dsi->dcs_backlight_ports) { dsi_device = intel_dsi->dsi_hosts[port]->device; mipi_dsi_dcs_write(dsi_device, MIPI_DCS_SET_DISPLAY_BRIGHTNESS, - &data, sizeof(data)); + &data, len); } } @@ -147,10 +156,16 @@ static void dcs_enable_backlight(const struct intel_crtc_state *crtc_state, static int dcs_setup_backlight(struct intel_connector *connector, enum pipe unused) { + struct drm_device *dev = connector->base.dev; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_panel *panel = &connector->panel; - panel->backlight.max = PANEL_PWM_MAX_VALUE; - panel->backlight.level = PANEL_PWM_MAX_VALUE; + if (dev_priv->vbt.backlight.brightness_precision_bits > 8) + panel->backlight.max = (1 << dev_priv->vbt.backlight.brightness_precision_bits) - 1; + else + panel->backlight.max = PANEL_PWM_MAX_VALUE; + + panel->backlight.level = panel->backlight.max; return 0; } diff --git a/drivers/gpu/drm/i915/display/intel_dsi_vbt.c b/drivers/gpu/drm/i915/display/intel_dsi_vbt.c index c2a2cd1f84dc..f241bedb8597 100644 --- a/drivers/gpu/drm/i915/display/intel_dsi_vbt.c +++ b/drivers/gpu/drm/i915/display/intel_dsi_vbt.c @@ -31,7 +31,6 @@ #include <linux/pinctrl/machine.h> #include <linux/slab.h> -#include <asm/intel-mid.h> #include <asm/unaligned.h> #include <drm/drm_crtc.h> @@ -42,7 +41,7 @@ #include "i915_drv.h" #include "intel_display_types.h" #include "intel_dsi.h" -#include "intel_sideband.h" +#include "vlv_sideband.h" #define MIPI_TRANSFER_MODE_SHIFT 0 #define MIPI_VIRTUAL_CHANNEL_SHIFT 1 diff --git a/drivers/gpu/drm/i915/display/intel_dvo.c b/drivers/gpu/drm/i915/display/intel_dvo.c index 77419f8c05e9..2eeb209afc64 100644 --- a/drivers/gpu/drm/i915/display/intel_dvo.c +++ b/drivers/gpu/drm/i915/display/intel_dvo.c @@ -223,9 +223,10 @@ static enum drm_mode_status intel_dvo_mode_valid(struct drm_connector *connector, struct drm_display_mode *mode) { - struct intel_dvo *intel_dvo = intel_attached_dvo(to_intel_connector(connector)); + struct intel_connector *intel_connector = to_intel_connector(connector); + struct intel_dvo *intel_dvo = intel_attached_dvo(intel_connector); const struct drm_display_mode *fixed_mode = - to_intel_connector(connector)->panel.fixed_mode; + intel_connector->panel.fixed_mode; int max_dotclk = to_i915(connector->dev)->max_dotclk_freq; int target_clock = mode->clock; @@ -235,10 +236,11 @@ intel_dvo_mode_valid(struct drm_connector *connector, /* XXX: Validate clock range */ if (fixed_mode) { - if (mode->hdisplay > fixed_mode->hdisplay) - return MODE_PANEL; - if (mode->vdisplay > fixed_mode->vdisplay) - return MODE_PANEL; + enum drm_mode_status status; + + status = intel_panel_mode_valid(intel_connector, mode); + if (status != MODE_OK) + return status; target_clock = fixed_mode->clock; } @@ -254,6 +256,7 @@ static int intel_dvo_compute_config(struct intel_encoder *encoder, struct drm_connector_state *conn_state) { struct intel_dvo *intel_dvo = enc_to_dvo(encoder); + struct intel_connector *connector = to_intel_connector(conn_state->connector); const struct drm_display_mode *fixed_mode = intel_dvo->attached_connector->panel.fixed_mode; struct drm_display_mode *adjusted_mode = &pipe_config->hw.adjusted_mode; @@ -264,8 +267,13 @@ static int intel_dvo_compute_config(struct intel_encoder *encoder, * with the panel scaling set up to source from the H/VDisplay * of the original mode. */ - if (fixed_mode) - intel_fixed_panel_mode(fixed_mode, adjusted_mode); + if (fixed_mode) { + int ret; + + ret = intel_panel_compute_config(connector, adjusted_mode); + if (ret) + return ret; + } if (adjusted_mode->flags & DRM_MODE_FLAG_DBLSCAN) return -EINVAL; diff --git a/drivers/gpu/drm/i915/display/intel_fb.c b/drivers/gpu/drm/i915/display/intel_fb.c index c60a81a81c09..fa1f375e696b 100644 --- a/drivers/gpu/drm/i915/display/intel_fb.c +++ b/drivers/gpu/drm/i915/display/intel_fb.c @@ -4,9 +4,11 @@ */ #include <drm/drm_framebuffer.h> +#include <drm/drm_modeset_helper.h> #include "intel_display.h" #include "intel_display_types.h" +#include "intel_dpt.h" #include "intel_fb.h" #define check_array_bounds(i915, a, i) drm_WARN_ON(&(i915)->drm, (i) >= ARRAY_SIZE(a)) @@ -61,6 +63,38 @@ int skl_ccs_to_main_plane(const struct drm_framebuffer *fb, int ccs_plane) return ccs_plane - fb->format->num_planes / 2; } +static unsigned int gen12_aligned_scanout_stride(const struct intel_framebuffer *fb, + int color_plane) +{ + struct drm_i915_private *i915 = to_i915(fb->base.dev); + unsigned int stride = fb->base.pitches[color_plane]; + + if (IS_ALDERLAKE_P(i915)) + return roundup_pow_of_two(max(stride, + 8u * intel_tile_width_bytes(&fb->base, color_plane))); + + return stride; +} + +static unsigned int gen12_ccs_aux_stride(struct intel_framebuffer *fb, int ccs_plane) +{ + struct drm_i915_private *i915 = to_i915(fb->base.dev); + int main_plane = skl_ccs_to_main_plane(&fb->base, ccs_plane); + unsigned int main_stride = fb->base.pitches[main_plane]; + unsigned int main_tile_width = intel_tile_width_bytes(&fb->base, main_plane); + + /* + * On ADL-P the AUX stride must align with a power-of-two aligned main + * surface stride. The stride of the allocated main surface object can + * be less than this POT stride, which is then autopadded to the POT + * size. + */ + if (IS_ALDERLAKE_P(i915)) + main_stride = gen12_aligned_scanout_stride(fb, main_plane); + + return DIV_ROUND_UP(main_stride, 4 * main_tile_width) * 64; +} + int skl_main_to_aux_plane(const struct drm_framebuffer *fb, int main_plane) { struct drm_i915_private *i915 = to_i915(fb->dev); @@ -79,16 +113,70 @@ unsigned int intel_tile_size(const struct drm_i915_private *i915) return DISPLAY_VER(i915) == 2 ? 2048 : 4096; } -unsigned int intel_tile_height(const struct drm_framebuffer *fb, int color_plane) +unsigned int +intel_tile_width_bytes(const struct drm_framebuffer *fb, int color_plane) { - if (is_gen12_ccs_plane(fb, color_plane)) - return 1; + struct drm_i915_private *dev_priv = to_i915(fb->dev); + unsigned int cpp = fb->format->cpp[color_plane]; + + switch (fb->modifier) { + case DRM_FORMAT_MOD_LINEAR: + return intel_tile_size(dev_priv); + case I915_FORMAT_MOD_X_TILED: + if (DISPLAY_VER(dev_priv) == 2) + return 128; + else + return 512; + case I915_FORMAT_MOD_Y_TILED_CCS: + if (is_ccs_plane(fb, color_plane)) + return 128; + fallthrough; + case I915_FORMAT_MOD_Y_TILED_GEN12_RC_CCS: + case I915_FORMAT_MOD_Y_TILED_GEN12_RC_CCS_CC: + case I915_FORMAT_MOD_Y_TILED_GEN12_MC_CCS: + if (is_ccs_plane(fb, color_plane)) + return 64; + fallthrough; + case I915_FORMAT_MOD_Y_TILED: + if (DISPLAY_VER(dev_priv) == 2 || HAS_128_BYTE_Y_TILING(dev_priv)) + return 128; + else + return 512; + case I915_FORMAT_MOD_Yf_TILED_CCS: + if (is_ccs_plane(fb, color_plane)) + return 128; + fallthrough; + case I915_FORMAT_MOD_Yf_TILED: + switch (cpp) { + case 1: + return 64; + case 2: + case 4: + return 128; + case 8: + case 16: + return 256; + default: + MISSING_CASE(cpp); + return cpp; + } + break; + default: + MISSING_CASE(fb->modifier); + return cpp; + } +} +unsigned int intel_tile_height(const struct drm_framebuffer *fb, int color_plane) +{ return intel_tile_size(to_i915(fb->dev)) / intel_tile_width_bytes(fb, color_plane); } -/* Return the tile dimensions in pixel units */ +/* + * Return the tile dimensions in pixel units, based on the (2 or 4 kbyte) GTT + * page tile size. + */ static void intel_tile_dims(const struct drm_framebuffer *fb, int color_plane, unsigned int *tile_width, unsigned int *tile_height) @@ -100,6 +188,21 @@ static void intel_tile_dims(const struct drm_framebuffer *fb, int color_plane, *tile_height = intel_tile_height(fb, color_plane); } +/* + * Return the tile dimensions in pixel units, based on the tile block size. + * The block covers the full GTT page sized tile on all tiled surfaces and + * it's a 64 byte portion of the tile on TGL+ CCS surfaces. + */ +static void intel_tile_block_dims(const struct drm_framebuffer *fb, int color_plane, + unsigned int *tile_width, + unsigned int *tile_height) +{ + intel_tile_dims(fb, color_plane, tile_width, tile_height); + + if (is_gen12_ccs_plane(fb, color_plane)) + *tile_height = 1; +} + unsigned int intel_tile_row_size(const struct drm_framebuffer *fb, int color_plane) { unsigned int tile_width, tile_height; @@ -109,6 +212,31 @@ unsigned int intel_tile_row_size(const struct drm_framebuffer *fb, int color_pla return fb->pitches[color_plane] * tile_height; } +unsigned int +intel_fb_align_height(const struct drm_framebuffer *fb, + int color_plane, unsigned int height) +{ + unsigned int tile_height = intel_tile_height(fb, color_plane); + + return ALIGN(height, tile_height); +} + +static unsigned int intel_fb_modifier_to_tiling(u64 fb_modifier) +{ + switch (fb_modifier) { + case I915_FORMAT_MOD_X_TILED: + return I915_TILING_X; + case I915_FORMAT_MOD_Y_TILED: + case I915_FORMAT_MOD_Y_TILED_CCS: + case I915_FORMAT_MOD_Y_TILED_GEN12_RC_CCS: + case I915_FORMAT_MOD_Y_TILED_GEN12_RC_CCS_CC: + case I915_FORMAT_MOD_Y_TILED_GEN12_MC_CCS: + return I915_TILING_Y; + default: + return I915_TILING_NONE; + } +} + unsigned int intel_cursor_alignment(const struct drm_i915_private *i915) { if (IS_I830(i915)) @@ -121,6 +249,70 @@ unsigned int intel_cursor_alignment(const struct drm_i915_private *i915) return 4 * 1024; } +static unsigned int intel_linear_alignment(const struct drm_i915_private *dev_priv) +{ + if (DISPLAY_VER(dev_priv) >= 9) + return 256 * 1024; + else if (IS_I965G(dev_priv) || IS_I965GM(dev_priv) || + IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) + return 128 * 1024; + else if (DISPLAY_VER(dev_priv) >= 4) + return 4 * 1024; + else + return 0; +} + +unsigned int intel_surf_alignment(const struct drm_framebuffer *fb, + int color_plane) +{ + struct drm_i915_private *dev_priv = to_i915(fb->dev); + + if (intel_fb_uses_dpt(fb)) + return 512 * 4096; + + /* AUX_DIST needs only 4K alignment */ + if (is_ccs_plane(fb, color_plane)) + return 4096; + + if (is_semiplanar_uv_plane(fb, color_plane)) { + /* + * TODO: cross-check wrt. the bspec stride in bytes * 64 bytes + * alignment for linear UV planes on all platforms. + */ + if (DISPLAY_VER(dev_priv) >= 12) { + if (fb->modifier == DRM_FORMAT_MOD_LINEAR) + return intel_linear_alignment(dev_priv); + + return intel_tile_row_size(fb, color_plane); + } + + return 4096; + } + + drm_WARN_ON(&dev_priv->drm, color_plane != 0); + + switch (fb->modifier) { + case DRM_FORMAT_MOD_LINEAR: + return intel_linear_alignment(dev_priv); + case I915_FORMAT_MOD_X_TILED: + if (HAS_ASYNC_FLIPS(dev_priv)) + return 256 * 1024; + return 0; + case I915_FORMAT_MOD_Y_TILED_GEN12_MC_CCS: + case I915_FORMAT_MOD_Y_TILED_GEN12_RC_CCS: + case I915_FORMAT_MOD_Y_TILED_GEN12_RC_CCS_CC: + return 16 * 1024; + case I915_FORMAT_MOD_Y_TILED_CCS: + case I915_FORMAT_MOD_Yf_TILED_CCS: + case I915_FORMAT_MOD_Y_TILED: + case I915_FORMAT_MOD_Yf_TILED: + return 1 * 1024 * 1024; + default: + MISSING_CASE(fb->modifier); + return 0; + } +} + void intel_fb_plane_get_subsampling(int *hsub, int *vsub, const struct drm_framebuffer *fb, int color_plane) @@ -165,15 +357,29 @@ void intel_fb_plane_get_subsampling(int *hsub, int *vsub, static void intel_fb_plane_dims(const struct intel_framebuffer *fb, int color_plane, int *w, int *h) { + struct drm_i915_private *i915 = to_i915(fb->base.dev); int main_plane = is_ccs_plane(&fb->base, color_plane) ? skl_ccs_to_main_plane(&fb->base, color_plane) : 0; + unsigned int main_width = fb->base.width; + unsigned int main_height = fb->base.height; int main_hsub, main_vsub; int hsub, vsub; + /* + * On ADL-P the CCS AUX surface layout always aligns with the + * power-of-two aligned main surface stride. The main surface + * stride in the allocated FB object may not be power-of-two + * sized, in which case it is auto-padded to the POT size. + */ + if (IS_ALDERLAKE_P(i915) && is_ccs_plane(&fb->base, color_plane)) + main_width = gen12_aligned_scanout_stride(fb, 0) / + fb->base.format->cpp[0]; + intel_fb_plane_get_subsampling(&main_hsub, &main_vsub, &fb->base, main_plane); intel_fb_plane_get_subsampling(&hsub, &vsub, &fb->base, color_plane); - *w = fb->base.width / main_hsub / hsub; - *h = fb->base.height / main_vsub / vsub; + + *w = main_width / main_hsub / hsub; + *h = main_height / main_vsub / vsub; } static u32 intel_adjust_tile_offset(int *x, int *y, @@ -355,17 +561,8 @@ static int intel_fb_offset_to_xy(int *x, int *y, unsigned int height; u32 alignment; - /* - * All DPT color planes must be 512*4k aligned (the amount mapped by a - * single DPT page). For ADL_P CCS FBs this only works by requiring - * the allocated offsets to be 2MB aligned. Once supoort to remap - * such FBs is added we can remove this requirement, as then all the - * planes can be remapped to an aligned offset. - */ - if (IS_ALDERLAKE_P(i915) && is_ccs_modifier(fb->modifier)) - alignment = 512 * 4096; - else if (DISPLAY_VER(i915) >= 12 && - is_semiplanar_uv_plane(fb, color_plane)) + if (DISPLAY_VER(i915) >= 12 && + is_semiplanar_uv_plane(fb, color_plane)) alignment = intel_tile_row_size(fb, color_plane); else if (fb->modifier != DRM_FORMAT_MOD_LINEAR) alignment = intel_tile_size(i915); @@ -416,7 +613,12 @@ static int intel_fb_check_ccs_xy(const struct drm_framebuffer *fb, int ccs_plane if (!is_ccs_plane(fb, ccs_plane) || is_gen12_ccs_cc_plane(fb, ccs_plane)) return 0; - intel_tile_dims(fb, ccs_plane, &tile_width, &tile_height); + /* + * While all the tile dimensions are based on a 2k or 4k GTT page size + * here the main and CCS coordinates must match only within a (64 byte + * on TGL+) block inside the tile. + */ + intel_tile_block_dims(fb, ccs_plane, &tile_width, &tile_height); intel_fb_plane_get_subsampling(&hsub, &vsub, fb, ccs_plane); tile_width *= hsub; @@ -491,8 +693,7 @@ bool intel_fb_needs_pot_stride_remap(const struct intel_framebuffer *fb) { struct drm_i915_private *i915 = to_i915(fb->base.dev); - return IS_ALDERLAKE_P(i915) && fb->base.modifier != DRM_FORMAT_MOD_LINEAR && - !is_ccs_modifier(fb->base.modifier); + return IS_ALDERLAKE_P(i915) && fb->base.modifier != DRM_FORMAT_MOD_LINEAR; } static int intel_fb_pitch(const struct intel_framebuffer *fb, int color_plane, unsigned int rotation) @@ -612,14 +813,16 @@ static unsigned int plane_view_dst_stride_tiles(const struct intel_framebuffer *fb, int color_plane, unsigned int pitch_tiles) { - if (intel_fb_needs_pot_stride_remap(fb)) + if (intel_fb_needs_pot_stride_remap(fb)) { + unsigned int min_stride = is_ccs_plane(&fb->base, color_plane) ? 2 : 8; /* * ADL_P, the only platform needing a POT stride has a minimum - * of 8 stride tiles. + * of 8 main surface and 2 CCS AUX stride tiles. */ - return roundup_pow_of_two(max(pitch_tiles, 8u)); - else + return roundup_pow_of_two(max(pitch_tiles, min_stride)); + } else { return pitch_tiles; + } } static unsigned int @@ -655,7 +858,7 @@ static u32 calc_plane_remap_info(const struct intel_framebuffer *fb, int color_p unsigned int tile_height = dims->tile_height; unsigned int tile_size = intel_tile_size(i915); struct drm_rect r; - u32 size; + u32 size = 0; assign_chk_ovf(i915, remap_info->offset, obj_offset); assign_chk_ovf(i915, remap_info->src_stride, plane_view_src_stride_tiles(fb, color_plane, dims)); @@ -680,7 +883,7 @@ static u32 calc_plane_remap_info(const struct intel_framebuffer *fb, int color_p color_plane_info->stride = remap_info->dst_stride * tile_height; - size = remap_info->dst_stride * remap_info->width; + size += remap_info->dst_stride * remap_info->width; /* rotate the tile dimensions to match the GTT view */ swap(tile_width, tile_height); @@ -689,6 +892,14 @@ static u32 calc_plane_remap_info(const struct intel_framebuffer *fb, int color_p check_array_bounds(i915, view->gtt.remapped.plane, color_plane); + if (view->gtt.remapped.plane_alignment) { + unsigned int aligned_offset = ALIGN(gtt_offset, + view->gtt.remapped.plane_alignment); + + size += aligned_offset - gtt_offset; + gtt_offset = aligned_offset; + } + assign_chk_ovf(i915, remap_info->dst_stride, plane_view_dst_stride_tiles(fb, color_plane, remap_info->width)); @@ -698,7 +909,7 @@ static u32 calc_plane_remap_info(const struct intel_framebuffer *fb, int color_p color_plane_info->stride = remap_info->dst_stride * tile_width * fb->base.format->cpp[color_plane]; - size = remap_info->dst_stride * remap_info->height; + size += remap_info->dst_stride * remap_info->height; } /* @@ -745,10 +956,14 @@ calc_plane_normal_size(const struct intel_framebuffer *fb, int color_plane, return tiles; } -static void intel_fb_view_init(struct intel_fb_view *view, enum i915_ggtt_view_type view_type) +static void intel_fb_view_init(struct drm_i915_private *i915, struct intel_fb_view *view, + enum i915_ggtt_view_type view_type) { memset(view, 0, sizeof(*view)); view->gtt.type = view_type; + + if (view_type == I915_GGTT_VIEW_REMAPPED && IS_ALDERLAKE_P(i915)) + view->gtt.remapped.plane_alignment = SZ_2M / PAGE_SIZE; } bool intel_fb_supports_90_270_rotation(const struct intel_framebuffer *fb) @@ -769,16 +984,16 @@ int intel_fill_fb_info(struct drm_i915_private *i915, struct intel_framebuffer * int i, num_planes = fb->base.format->num_planes; unsigned int tile_size = intel_tile_size(i915); - intel_fb_view_init(&fb->normal_view, I915_GGTT_VIEW_NORMAL); + intel_fb_view_init(i915, &fb->normal_view, I915_GGTT_VIEW_NORMAL); drm_WARN_ON(&i915->drm, intel_fb_supports_90_270_rotation(fb) && intel_fb_needs_pot_stride_remap(fb)); if (intel_fb_supports_90_270_rotation(fb)) - intel_fb_view_init(&fb->rotated_view, I915_GGTT_VIEW_ROTATED); + intel_fb_view_init(i915, &fb->rotated_view, I915_GGTT_VIEW_ROTATED); if (intel_fb_needs_pot_stride_remap(fb)) - intel_fb_view_init(&fb->remapped_view, I915_GGTT_VIEW_REMAPPED); + intel_fb_view_init(i915, &fb->remapped_view, I915_GGTT_VIEW_REMAPPED); for (i = 0; i < num_planes; i++) { struct fb_plane_view_dims view_dims; @@ -856,7 +1071,7 @@ static void intel_plane_remap_gtt(struct intel_plane_state *plane_state) unsigned int src_w, src_h; u32 gtt_offset = 0; - intel_fb_view_init(&plane_state->view, + intel_fb_view_init(i915, &plane_state->view, drm_rotation_90_or_270(rotation) ? I915_GGTT_VIEW_ROTATED : I915_GGTT_VIEW_REMAPPED); @@ -918,6 +1133,79 @@ void intel_fb_fill_view(const struct intel_framebuffer *fb, unsigned int rotatio *view = fb->normal_view; } +static +u32 intel_fb_max_stride(struct drm_i915_private *dev_priv, + u32 pixel_format, u64 modifier) +{ + /* + * Arbitrary limit for gen4+ chosen to match the + * render engine max stride. + * + * The new CCS hash mode makes remapping impossible + */ + if (DISPLAY_VER(dev_priv) < 4 || is_ccs_modifier(modifier) || + intel_modifier_uses_dpt(dev_priv, modifier)) + return intel_plane_fb_max_stride(dev_priv, pixel_format, modifier); + else if (DISPLAY_VER(dev_priv) >= 7) + return 256 * 1024; + else + return 128 * 1024; +} + +static u32 +intel_fb_stride_alignment(const struct drm_framebuffer *fb, int color_plane) +{ + struct drm_i915_private *dev_priv = to_i915(fb->dev); + u32 tile_width; + + if (is_surface_linear(fb, color_plane)) { + u32 max_stride = intel_plane_fb_max_stride(dev_priv, + fb->format->format, + fb->modifier); + + /* + * To make remapping with linear generally feasible + * we need the stride to be page aligned. + */ + if (fb->pitches[color_plane] > max_stride && + !is_ccs_modifier(fb->modifier)) + return intel_tile_size(dev_priv); + else + return 64; + } + + tile_width = intel_tile_width_bytes(fb, color_plane); + if (is_ccs_modifier(fb->modifier)) { + /* + * On ADL-P the stride must be either 8 tiles or a stride + * that is aligned to 16 tiles, required by the 16 tiles = + * 64 kbyte CCS AUX PTE granularity, allowing CCS FBs to be + * remapped. + */ + if (IS_ALDERLAKE_P(dev_priv)) + tile_width *= fb->pitches[0] <= tile_width * 8 ? 8 : 16; + /* + * On TGL the surface stride must be 4 tile aligned, mapped by + * one 64 byte cacheline on the CCS AUX surface. + */ + else if (DISPLAY_VER(dev_priv) >= 12) + tile_width *= 4; + /* + * Display WA #0531: skl,bxt,kbl,glk + * + * Render decompression and plane width > 3840 + * combined with horizontal panning requires the + * plane stride to be a multiple of 4. We'll just + * require the entire fb to accommodate that to avoid + * potential runtime errors at plane configuration time. + */ + else if ((DISPLAY_VER(dev_priv) == 9 || IS_GEMINILAKE(dev_priv)) && + color_plane == 0 && fb->width > 3840) + tile_width *= 4; + } + return tile_width; +} + static int intel_plane_check_stride(const struct intel_plane_state *plane_state) { struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane); @@ -981,3 +1269,257 @@ int intel_plane_compute_gtt(struct intel_plane_state *plane_state) return intel_plane_check_stride(plane_state); } + +static void intel_user_framebuffer_destroy(struct drm_framebuffer *fb) +{ + struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb); + + drm_framebuffer_cleanup(fb); + + if (intel_fb_uses_dpt(fb)) + intel_dpt_destroy(intel_fb->dpt_vm); + + intel_frontbuffer_put(intel_fb->frontbuffer); + + kfree(intel_fb); +} + +static int intel_user_framebuffer_create_handle(struct drm_framebuffer *fb, + struct drm_file *file, + unsigned int *handle) +{ + struct drm_i915_gem_object *obj = intel_fb_obj(fb); + struct drm_i915_private *i915 = to_i915(obj->base.dev); + + if (i915_gem_object_is_userptr(obj)) { + drm_dbg(&i915->drm, + "attempting to use a userptr for a framebuffer, denied\n"); + return -EINVAL; + } + + return drm_gem_handle_create(file, &obj->base, handle); +} + +static int intel_user_framebuffer_dirty(struct drm_framebuffer *fb, + struct drm_file *file, + unsigned int flags, unsigned int color, + struct drm_clip_rect *clips, + unsigned int num_clips) +{ + struct drm_i915_gem_object *obj = intel_fb_obj(fb); + + i915_gem_object_flush_if_display(obj); + intel_frontbuffer_flush(to_intel_frontbuffer(fb), ORIGIN_DIRTYFB); + + return 0; +} + +static const struct drm_framebuffer_funcs intel_fb_funcs = { + .destroy = intel_user_framebuffer_destroy, + .create_handle = intel_user_framebuffer_create_handle, + .dirty = intel_user_framebuffer_dirty, +}; + +int intel_framebuffer_init(struct intel_framebuffer *intel_fb, + struct drm_i915_gem_object *obj, + struct drm_mode_fb_cmd2 *mode_cmd) +{ + struct drm_i915_private *dev_priv = to_i915(obj->base.dev); + struct drm_framebuffer *fb = &intel_fb->base; + u32 max_stride; + unsigned int tiling, stride; + int ret = -EINVAL; + int i; + + intel_fb->frontbuffer = intel_frontbuffer_get(obj); + if (!intel_fb->frontbuffer) + return -ENOMEM; + + i915_gem_object_lock(obj, NULL); + tiling = i915_gem_object_get_tiling(obj); + stride = i915_gem_object_get_stride(obj); + i915_gem_object_unlock(obj); + + if (mode_cmd->flags & DRM_MODE_FB_MODIFIERS) { + /* + * If there's a fence, enforce that + * the fb modifier and tiling mode match. + */ + if (tiling != I915_TILING_NONE && + tiling != intel_fb_modifier_to_tiling(mode_cmd->modifier[0])) { + drm_dbg_kms(&dev_priv->drm, + "tiling_mode doesn't match fb modifier\n"); + goto err; + } + } else { + if (tiling == I915_TILING_X) { + mode_cmd->modifier[0] = I915_FORMAT_MOD_X_TILED; + } else if (tiling == I915_TILING_Y) { + drm_dbg_kms(&dev_priv->drm, + "No Y tiling for legacy addfb\n"); + goto err; + } + } + + if (!drm_any_plane_has_format(&dev_priv->drm, + mode_cmd->pixel_format, + mode_cmd->modifier[0])) { + drm_dbg_kms(&dev_priv->drm, + "unsupported pixel format %p4cc / modifier 0x%llx\n", + &mode_cmd->pixel_format, mode_cmd->modifier[0]); + goto err; + } + + /* + * gen2/3 display engine uses the fence if present, + * so the tiling mode must match the fb modifier exactly. + */ + if (DISPLAY_VER(dev_priv) < 4 && + tiling != intel_fb_modifier_to_tiling(mode_cmd->modifier[0])) { + drm_dbg_kms(&dev_priv->drm, + "tiling_mode must match fb modifier exactly on gen2/3\n"); + goto err; + } + + max_stride = intel_fb_max_stride(dev_priv, mode_cmd->pixel_format, + mode_cmd->modifier[0]); + if (mode_cmd->pitches[0] > max_stride) { + drm_dbg_kms(&dev_priv->drm, + "%s pitch (%u) must be at most %d\n", + mode_cmd->modifier[0] != DRM_FORMAT_MOD_LINEAR ? + "tiled" : "linear", + mode_cmd->pitches[0], max_stride); + goto err; + } + + /* + * If there's a fence, enforce that + * the fb pitch and fence stride match. + */ + if (tiling != I915_TILING_NONE && mode_cmd->pitches[0] != stride) { + drm_dbg_kms(&dev_priv->drm, + "pitch (%d) must match tiling stride (%d)\n", + mode_cmd->pitches[0], stride); + goto err; + } + + /* FIXME need to adjust LINOFF/TILEOFF accordingly. */ + if (mode_cmd->offsets[0] != 0) { + drm_dbg_kms(&dev_priv->drm, + "plane 0 offset (0x%08x) must be 0\n", + mode_cmd->offsets[0]); + goto err; + } + + drm_helper_mode_fill_fb_struct(&dev_priv->drm, fb, mode_cmd); + + for (i = 0; i < fb->format->num_planes; i++) { + u32 stride_alignment; + + if (mode_cmd->handles[i] != mode_cmd->handles[0]) { + drm_dbg_kms(&dev_priv->drm, "bad plane %d handle\n", + i); + goto err; + } + + stride_alignment = intel_fb_stride_alignment(fb, i); + if (fb->pitches[i] & (stride_alignment - 1)) { + drm_dbg_kms(&dev_priv->drm, + "plane %d pitch (%d) must be at least %u byte aligned\n", + i, fb->pitches[i], stride_alignment); + goto err; + } + + if (is_gen12_ccs_plane(fb, i) && !is_gen12_ccs_cc_plane(fb, i)) { + int ccs_aux_stride = gen12_ccs_aux_stride(intel_fb, i); + + if (fb->pitches[i] != ccs_aux_stride) { + drm_dbg_kms(&dev_priv->drm, + "ccs aux plane %d pitch (%d) must be %d\n", + i, + fb->pitches[i], ccs_aux_stride); + goto err; + } + } + + fb->obj[i] = &obj->base; + } + + ret = intel_fill_fb_info(dev_priv, intel_fb); + if (ret) + goto err; + + if (intel_fb_uses_dpt(fb)) { + struct i915_address_space *vm; + + vm = intel_dpt_create(intel_fb); + if (IS_ERR(vm)) { + ret = PTR_ERR(vm); + goto err; + } + + intel_fb->dpt_vm = vm; + } + + ret = drm_framebuffer_init(&dev_priv->drm, fb, &intel_fb_funcs); + if (ret) { + drm_err(&dev_priv->drm, "framebuffer init failed %d\n", ret); + goto err; + } + + return 0; + +err: + intel_frontbuffer_put(intel_fb->frontbuffer); + return ret; +} + +struct drm_framebuffer * +intel_user_framebuffer_create(struct drm_device *dev, + struct drm_file *filp, + const struct drm_mode_fb_cmd2 *user_mode_cmd) +{ + struct drm_framebuffer *fb; + struct drm_i915_gem_object *obj; + struct drm_mode_fb_cmd2 mode_cmd = *user_mode_cmd; + struct drm_i915_private *i915; + + obj = i915_gem_object_lookup(filp, mode_cmd.handles[0]); + if (!obj) + return ERR_PTR(-ENOENT); + + /* object is backed with LMEM for discrete */ + i915 = to_i915(obj->base.dev); + if (HAS_LMEM(i915) && !i915_gem_object_can_migrate(obj, INTEL_REGION_LMEM)) { + /* object is "remote", not in local memory */ + i915_gem_object_put(obj); + return ERR_PTR(-EREMOTE); + } + + fb = intel_framebuffer_create(obj, &mode_cmd); + i915_gem_object_put(obj); + + return fb; +} + +struct drm_framebuffer * +intel_framebuffer_create(struct drm_i915_gem_object *obj, + struct drm_mode_fb_cmd2 *mode_cmd) +{ + struct intel_framebuffer *intel_fb; + int ret; + + intel_fb = kzalloc(sizeof(*intel_fb), GFP_KERNEL); + if (!intel_fb) + return ERR_PTR(-ENOMEM); + + ret = intel_framebuffer_init(intel_fb, obj, mode_cmd); + if (ret) + goto err; + + return &intel_fb->base; + +err: + kfree(intel_fb); + return ERR_PTR(ret); +} diff --git a/drivers/gpu/drm/i915/display/intel_fb.h b/drivers/gpu/drm/i915/display/intel_fb.h index 739d1b91754b..1cbdd84502bd 100644 --- a/drivers/gpu/drm/i915/display/intel_fb.h +++ b/drivers/gpu/drm/i915/display/intel_fb.h @@ -8,10 +8,12 @@ #include <linux/types.h> +struct drm_device; +struct drm_file; struct drm_framebuffer; - +struct drm_i915_gem_object; struct drm_i915_private; - +struct drm_mode_fb_cmd2; struct intel_fb_view; struct intel_framebuffer; struct intel_plane_state; @@ -28,10 +30,14 @@ int skl_ccs_to_main_plane(const struct drm_framebuffer *fb, int ccs_plane); int skl_main_to_aux_plane(const struct drm_framebuffer *fb, int main_plane); unsigned int intel_tile_size(const struct drm_i915_private *i915); +unsigned int intel_tile_width_bytes(const struct drm_framebuffer *fb, int color_plane); unsigned int intel_tile_height(const struct drm_framebuffer *fb, int color_plane); unsigned int intel_tile_row_size(const struct drm_framebuffer *fb, int color_plane); - +unsigned int intel_fb_align_height(const struct drm_framebuffer *fb, + int color_plane, unsigned int height); unsigned int intel_cursor_alignment(const struct drm_i915_private *i915); +unsigned int intel_surf_alignment(const struct drm_framebuffer *fb, + int color_plane); void intel_fb_plane_get_subsampling(int *hsub, int *vsub, const struct drm_framebuffer *fb, @@ -53,4 +59,12 @@ void intel_fb_fill_view(const struct intel_framebuffer *fb, unsigned int rotatio struct intel_fb_view *view); int intel_plane_compute_gtt(struct intel_plane_state *plane_state); +int intel_framebuffer_init(struct intel_framebuffer *ifb, + struct drm_i915_gem_object *obj, + struct drm_mode_fb_cmd2 *mode_cmd); +struct drm_framebuffer * +intel_user_framebuffer_create(struct drm_device *dev, + struct drm_file *filp, + const struct drm_mode_fb_cmd2 *user_mode_cmd); + #endif /* __INTEL_FB_H__ */ diff --git a/drivers/gpu/drm/i915/display/intel_fb_pin.c b/drivers/gpu/drm/i915/display/intel_fb_pin.c new file mode 100644 index 000000000000..3f77f3013584 --- /dev/null +++ b/drivers/gpu/drm/i915/display/intel_fb_pin.c @@ -0,0 +1,274 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2021 Intel Corporation + */ + +/** + * DOC: display pinning helpers + */ + +#include "intel_display_types.h" +#include "intel_fb_pin.h" +#include "intel_fb.h" + +#include "intel_dpt.h" + +#include "gem/i915_gem_object.h" + +static struct i915_vma * +intel_pin_fb_obj_dpt(struct drm_framebuffer *fb, + const struct i915_ggtt_view *view, + bool uses_fence, + unsigned long *out_flags, + struct i915_address_space *vm) +{ + struct drm_device *dev = fb->dev; + struct drm_i915_private *dev_priv = to_i915(dev); + struct drm_i915_gem_object *obj = intel_fb_obj(fb); + struct i915_vma *vma; + u32 alignment; + int ret; + + if (WARN_ON(!i915_gem_object_is_framebuffer(obj))) + return ERR_PTR(-EINVAL); + + alignment = 4096 * 512; + + atomic_inc(&dev_priv->gpu_error.pending_fb_pin); + + ret = i915_gem_object_set_cache_level(obj, I915_CACHE_NONE); + if (ret) { + vma = ERR_PTR(ret); + goto err; + } + + vma = i915_vma_instance(obj, vm, view); + if (IS_ERR(vma)) + goto err; + + if (i915_vma_misplaced(vma, 0, alignment, 0)) { + ret = i915_vma_unbind(vma); + if (ret) { + vma = ERR_PTR(ret); + goto err; + } + } + + ret = i915_vma_pin(vma, 0, alignment, PIN_GLOBAL); + if (ret) { + vma = ERR_PTR(ret); + goto err; + } + + vma->display_alignment = max_t(u64, vma->display_alignment, alignment); + + i915_gem_object_flush_if_display(obj); + + i915_vma_get(vma); +err: + atomic_dec(&dev_priv->gpu_error.pending_fb_pin); + + return vma; +} + +struct i915_vma * +intel_pin_and_fence_fb_obj(struct drm_framebuffer *fb, + bool phys_cursor, + const struct i915_ggtt_view *view, + bool uses_fence, + unsigned long *out_flags) +{ + struct drm_device *dev = fb->dev; + struct drm_i915_private *dev_priv = to_i915(dev); + struct drm_i915_gem_object *obj = intel_fb_obj(fb); + intel_wakeref_t wakeref; + struct i915_gem_ww_ctx ww; + struct i915_vma *vma; + unsigned int pinctl; + u32 alignment; + int ret; + + if (drm_WARN_ON(dev, !i915_gem_object_is_framebuffer(obj))) + return ERR_PTR(-EINVAL); + + if (phys_cursor) + alignment = intel_cursor_alignment(dev_priv); + else + alignment = intel_surf_alignment(fb, 0); + if (drm_WARN_ON(dev, alignment && !is_power_of_2(alignment))) + return ERR_PTR(-EINVAL); + + /* Note that the w/a also requires 64 PTE of padding following the + * bo. We currently fill all unused PTE with the shadow page and so + * we should always have valid PTE following the scanout preventing + * the VT-d warning. + */ + if (intel_scanout_needs_vtd_wa(dev_priv) && alignment < 256 * 1024) + alignment = 256 * 1024; + + /* + * Global gtt pte registers are special registers which actually forward + * writes to a chunk of system memory. Which means that there is no risk + * that the register values disappear as soon as we call + * intel_runtime_pm_put(), so it is correct to wrap only the + * pin/unpin/fence and not more. + */ + wakeref = intel_runtime_pm_get(&dev_priv->runtime_pm); + + atomic_inc(&dev_priv->gpu_error.pending_fb_pin); + + /* + * Valleyview is definitely limited to scanning out the first + * 512MiB. Lets presume this behaviour was inherited from the + * g4x display engine and that all earlier gen are similarly + * limited. Testing suggests that it is a little more + * complicated than this. For example, Cherryview appears quite + * happy to scanout from anywhere within its global aperture. + */ + pinctl = 0; + if (HAS_GMCH(dev_priv)) + pinctl |= PIN_MAPPABLE; + + i915_gem_ww_ctx_init(&ww, true); +retry: + ret = i915_gem_object_lock(obj, &ww); + if (!ret && phys_cursor) + ret = i915_gem_object_attach_phys(obj, alignment); + else if (!ret && HAS_LMEM(dev_priv)) + ret = i915_gem_object_migrate(obj, &ww, INTEL_REGION_LMEM); + /* TODO: Do we need to sync when migration becomes async? */ + if (!ret) + ret = i915_gem_object_pin_pages(obj); + if (ret) + goto err; + + if (!ret) { + vma = i915_gem_object_pin_to_display_plane(obj, &ww, alignment, + view, pinctl); + if (IS_ERR(vma)) { + ret = PTR_ERR(vma); + goto err_unpin; + } + } + + if (uses_fence && i915_vma_is_map_and_fenceable(vma)) { + /* + * Install a fence for tiled scan-out. Pre-i965 always needs a + * fence, whereas 965+ only requires a fence if using + * framebuffer compression. For simplicity, we always, when + * possible, install a fence as the cost is not that onerous. + * + * If we fail to fence the tiled scanout, then either the + * modeset will reject the change (which is highly unlikely as + * the affected systems, all but one, do not have unmappable + * space) or we will not be able to enable full powersaving + * techniques (also likely not to apply due to various limits + * FBC and the like impose on the size of the buffer, which + * presumably we violated anyway with this unmappable buffer). + * Anyway, it is presumably better to stumble onwards with + * something and try to run the system in a "less than optimal" + * mode that matches the user configuration. + */ + ret = i915_vma_pin_fence(vma); + if (ret != 0 && DISPLAY_VER(dev_priv) < 4) { + i915_vma_unpin(vma); + goto err_unpin; + } + ret = 0; + + if (vma->fence) + *out_flags |= PLANE_HAS_FENCE; + } + + i915_vma_get(vma); + +err_unpin: + i915_gem_object_unpin_pages(obj); +err: + if (ret == -EDEADLK) { + ret = i915_gem_ww_ctx_backoff(&ww); + if (!ret) + goto retry; + } + i915_gem_ww_ctx_fini(&ww); + if (ret) + vma = ERR_PTR(ret); + + atomic_dec(&dev_priv->gpu_error.pending_fb_pin); + intel_runtime_pm_put(&dev_priv->runtime_pm, wakeref); + return vma; +} + +void intel_unpin_fb_vma(struct i915_vma *vma, unsigned long flags) +{ + if (flags & PLANE_HAS_FENCE) + i915_vma_unpin_fence(vma); + i915_vma_unpin(vma); + i915_vma_put(vma); +} + +int intel_plane_pin_fb(struct intel_plane_state *plane_state) +{ + struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane); + struct drm_i915_private *dev_priv = to_i915(plane->base.dev); + struct drm_framebuffer *fb = plane_state->hw.fb; + struct i915_vma *vma; + bool phys_cursor = + plane->id == PLANE_CURSOR && + INTEL_INFO(dev_priv)->display.cursor_needs_physical; + + if (!intel_fb_uses_dpt(fb)) { + vma = intel_pin_and_fence_fb_obj(fb, phys_cursor, + &plane_state->view.gtt, + intel_plane_uses_fence(plane_state), + &plane_state->flags); + if (IS_ERR(vma)) + return PTR_ERR(vma); + + plane_state->ggtt_vma = vma; + } else { + struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb); + + vma = intel_dpt_pin(intel_fb->dpt_vm); + if (IS_ERR(vma)) + return PTR_ERR(vma); + + plane_state->ggtt_vma = vma; + + vma = intel_pin_fb_obj_dpt(fb, &plane_state->view.gtt, false, + &plane_state->flags, intel_fb->dpt_vm); + if (IS_ERR(vma)) { + intel_dpt_unpin(intel_fb->dpt_vm); + plane_state->ggtt_vma = NULL; + return PTR_ERR(vma); + } + + plane_state->dpt_vma = vma; + + WARN_ON(plane_state->ggtt_vma == plane_state->dpt_vma); + } + + return 0; +} + +void intel_plane_unpin_fb(struct intel_plane_state *old_plane_state) +{ + struct drm_framebuffer *fb = old_plane_state->hw.fb; + struct i915_vma *vma; + + if (!intel_fb_uses_dpt(fb)) { + vma = fetch_and_zero(&old_plane_state->ggtt_vma); + if (vma) + intel_unpin_fb_vma(vma, old_plane_state->flags); + } else { + struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb); + + vma = fetch_and_zero(&old_plane_state->dpt_vma); + if (vma) + intel_unpin_fb_vma(vma, old_plane_state->flags); + + vma = fetch_and_zero(&old_plane_state->ggtt_vma); + if (vma) + intel_dpt_unpin(intel_fb->dpt_vm); + } +} diff --git a/drivers/gpu/drm/i915/display/intel_fb_pin.h b/drivers/gpu/drm/i915/display/intel_fb_pin.h new file mode 100644 index 000000000000..e4fcd0218d9d --- /dev/null +++ b/drivers/gpu/drm/i915/display/intel_fb_pin.h @@ -0,0 +1,28 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2021 Intel Corporation + */ + +#ifndef __INTEL_FB_PIN_H__ +#define __INTEL_FB_PIN_H__ + +#include <linux/types.h> + +struct drm_framebuffer; +struct i915_vma; +struct intel_plane_state; +struct i915_ggtt_view; + +struct i915_vma * +intel_pin_and_fence_fb_obj(struct drm_framebuffer *fb, + bool phys_cursor, + const struct i915_ggtt_view *view, + bool uses_fence, + unsigned long *out_flags); + +void intel_unpin_fb_vma(struct i915_vma *vma, unsigned long flags); + +int intel_plane_pin_fb(struct intel_plane_state *plane_state); +void intel_plane_unpin_fb(struct intel_plane_state *old_plane_state); + +#endif diff --git a/drivers/gpu/drm/i915/display/intel_fbc.c b/drivers/gpu/drm/i915/display/intel_fbc.c index ddfc17e21668..1f66de77a6b1 100644 --- a/drivers/gpu/drm/i915/display/intel_fbc.c +++ b/drivers/gpu/drm/i915/display/intel_fbc.c @@ -62,19 +62,84 @@ static void intel_fbc_get_plane_source_size(const struct intel_fbc_state_cache * *height = cache->plane.src_h; } -static int intel_fbc_calculate_cfb_size(struct drm_i915_private *dev_priv, - const struct intel_fbc_state_cache *cache) +/* plane stride in pixels */ +static unsigned int intel_fbc_plane_stride(const struct intel_plane_state *plane_state) { - int lines; + const struct drm_framebuffer *fb = plane_state->hw.fb; + unsigned int stride; + + stride = plane_state->view.color_plane[0].stride; + if (!drm_rotation_90_or_270(plane_state->hw.rotation)) + stride /= fb->format->cpp[0]; + + return stride; +} + +/* plane stride based cfb stride in bytes, assuming 1:1 compression limit */ +static unsigned int _intel_fbc_cfb_stride(const struct intel_fbc_state_cache *cache) +{ + unsigned int cpp = 4; /* FBC always 4 bytes per pixel */ + + return cache->fb.stride * cpp; +} + +/* minimum acceptable cfb stride in bytes, assuming 1:1 compression limit */ +static unsigned int skl_fbc_min_cfb_stride(struct drm_i915_private *i915, + const struct intel_fbc_state_cache *cache) +{ + unsigned int limit = 4; /* 1:4 compression limit is the worst case */ + unsigned int cpp = 4; /* FBC always 4 bytes per pixel */ + unsigned int height = 4; /* FBC segment is 4 lines */ + unsigned int stride; + + /* minimum segment stride we can use */ + stride = cache->plane.src_w * cpp * height / limit; + + /* + * Wa_16011863758: icl+ + * Avoid some hardware segment address miscalculation. + */ + if (DISPLAY_VER(i915) >= 11) + stride += 64; + + /* + * At least some of the platforms require each 4 line segment to + * be 512 byte aligned. Just do it always for simplicity. + */ + stride = ALIGN(stride, 512); + + /* convert back to single line equivalent with 1:1 compression limit */ + return stride * limit / height; +} + +/* properly aligned cfb stride in bytes, assuming 1:1 compression limit */ +static unsigned int intel_fbc_cfb_stride(struct drm_i915_private *i915, + const struct intel_fbc_state_cache *cache) +{ + unsigned int stride = _intel_fbc_cfb_stride(cache); + + /* + * At least some of the platforms require each 4 line segment to + * be 512 byte aligned. Aligning each line to 512 bytes guarantees + * that regardless of the compression limit we choose later. + */ + if (DISPLAY_VER(i915) >= 9) + return max(ALIGN(stride, 512), skl_fbc_min_cfb_stride(i915, cache)); + else + return stride; +} + +static unsigned int intel_fbc_cfb_size(struct drm_i915_private *dev_priv, + const struct intel_fbc_state_cache *cache) +{ + int lines = cache->plane.src_h; - intel_fbc_get_plane_source_size(cache, NULL, &lines); if (DISPLAY_VER(dev_priv) == 7) lines = min(lines, 2048); else if (DISPLAY_VER(dev_priv) >= 8) lines = min(lines, 2560); - /* Hardware needs the full buffer stride, not just the active area. */ - return lines * cache->fb.stride; + return lines * intel_fbc_cfb_stride(dev_priv, cache); } static void i8xx_fbc_deactivate(struct drm_i915_private *dev_priv) @@ -99,15 +164,13 @@ static void i8xx_fbc_deactivate(struct drm_i915_private *dev_priv) static void i8xx_fbc_activate(struct drm_i915_private *dev_priv) { - struct intel_fbc_reg_params *params = &dev_priv->fbc.params; + struct intel_fbc *fbc = &dev_priv->fbc; + const struct intel_fbc_reg_params *params = &fbc->params; int cfb_pitch; int i; u32 fbc_ctl; - /* Note: fbc.limit == 1 for i8xx */ - cfb_pitch = params->cfb_size / FBC_LL_SIZE; - if (params->fb.stride < cfb_pitch) - cfb_pitch = params->fb.stride; + cfb_pitch = params->cfb_stride / fbc->limit; /* FBC_CTL wants 32B or 64B units */ if (DISPLAY_VER(dev_priv) == 2) @@ -150,15 +213,9 @@ static bool i8xx_fbc_is_active(struct drm_i915_private *dev_priv) static u32 g4x_dpfc_ctl_limit(struct drm_i915_private *i915) { - const struct intel_fbc_reg_params *params = &i915->fbc.params; - int limit = i915->fbc.limit; - - if (params->fb.format->cpp[0] == 2) - limit <<= 1; - - switch (limit) { + switch (i915->fbc.limit) { default: - MISSING_CASE(limit); + MISSING_CASE(i915->fbc.limit); fallthrough; case 1: return DPFC_CTL_LIMIT_1X; @@ -232,16 +289,16 @@ static void i965_fbc_recompress(struct drm_i915_private *dev_priv) /* This function forces a CFB recompression through the nuke operation. */ static void snb_fbc_recompress(struct drm_i915_private *dev_priv) { - struct intel_fbc *fbc = &dev_priv->fbc; - - trace_intel_fbc_nuke(fbc->crtc); - intel_de_write(dev_priv, MSG_FBC_REND_STATE, FBC_REND_NUKE); intel_de_posting_read(dev_priv, MSG_FBC_REND_STATE); } static void intel_fbc_recompress(struct drm_i915_private *dev_priv) { + struct intel_fbc *fbc = &dev_priv->fbc; + + trace_intel_fbc_nuke(fbc->crtc); + if (DISPLAY_VER(dev_priv) >= 6) snb_fbc_recompress(dev_priv); else if (DISPLAY_VER(dev_priv) >= 4) @@ -280,8 +337,6 @@ static void ilk_fbc_activate(struct drm_i915_private *dev_priv) params->fence_y_offset); /* enable it... */ intel_de_write(dev_priv, ILK_DPFC_CONTROL, dpfc_ctl | DPFC_CTL_EN); - - intel_fbc_recompress(dev_priv); } static void ilk_fbc_deactivate(struct drm_i915_private *dev_priv) @@ -303,19 +358,29 @@ static bool ilk_fbc_is_active(struct drm_i915_private *dev_priv) static void gen7_fbc_activate(struct drm_i915_private *dev_priv) { - struct intel_fbc_reg_params *params = &dev_priv->fbc.params; + struct intel_fbc *fbc = &dev_priv->fbc; + const struct intel_fbc_reg_params *params = &fbc->params; u32 dpfc_ctl; - /* Display WA #0529: skl, kbl, bxt. */ - if (DISPLAY_VER(dev_priv) == 9) { - u32 val = intel_de_read(dev_priv, CHICKEN_MISC_4); + if (DISPLAY_VER(dev_priv) >= 10) { + u32 val = 0; - val &= ~(FBC_STRIDE_OVERRIDE | FBC_STRIDE_MASK); + if (params->override_cfb_stride) + val |= FBC_STRIDE_OVERRIDE | + FBC_STRIDE(params->override_cfb_stride / fbc->limit); - if (params->gen9_wa_cfb_stride) - val |= FBC_STRIDE_OVERRIDE | params->gen9_wa_cfb_stride; + intel_de_write(dev_priv, GLK_FBC_STRIDE, val); + } else if (DISPLAY_VER(dev_priv) == 9) { + u32 val = 0; - intel_de_write(dev_priv, CHICKEN_MISC_4, val); + /* Display WA #0529: skl, kbl, bxt. */ + if (params->override_cfb_stride) + val |= CHICKEN_FBC_STRIDE_OVERRIDE | + CHICKEN_FBC_STRIDE(params->override_cfb_stride / fbc->limit); + + intel_de_rmw(dev_priv, CHICKEN_MISC_4, + CHICKEN_FBC_STRIDE_OVERRIDE | + CHICKEN_FBC_STRIDE_MASK, val); } dpfc_ctl = 0; @@ -339,8 +404,6 @@ static void gen7_fbc_activate(struct drm_i915_private *dev_priv) dpfc_ctl |= FBC_CTL_FALSE_COLOR; intel_de_write(dev_priv, ILK_DPFC_CONTROL, dpfc_ctl | DPFC_CTL_EN); - - intel_fbc_recompress(dev_priv); } static bool intel_fbc_hw_is_active(struct drm_i915_private *dev_priv) @@ -402,6 +465,12 @@ bool intel_fbc_is_active(struct drm_i915_private *dev_priv) return dev_priv->fbc.active; } +static void intel_fbc_activate(struct drm_i915_private *dev_priv) +{ + intel_fbc_hw_activate(dev_priv); + intel_fbc_recompress(dev_priv); +} + static void intel_fbc_deactivate(struct drm_i915_private *dev_priv, const char *reason) { @@ -440,30 +509,32 @@ static u64 intel_fbc_stolen_end(struct drm_i915_private *dev_priv) return min(end, intel_fbc_cfb_base_max(dev_priv)); } -static int intel_fbc_max_limit(struct drm_i915_private *dev_priv, int fb_cpp) +static int intel_fbc_min_limit(int fb_cpp) { - /* - * FIXME: FBC1 can have arbitrary cfb stride, - * so we could support different compression ratios. - */ - if (DISPLAY_VER(dev_priv) < 5 && !IS_G4X(dev_priv)) - return 1; + return fb_cpp == 2 ? 2 : 1; +} +static int intel_fbc_max_limit(struct drm_i915_private *dev_priv) +{ /* WaFbcOnly1to1Ratio:ctg */ if (IS_G4X(dev_priv)) return 1; - /* FBC2 can only do 1:1, 1:2, 1:4 */ - return fb_cpp == 2 ? 2 : 4; + /* + * FBC2 can only do 1:1, 1:2, 1:4, we limit + * FBC1 to the same out of convenience. + */ + return 4; } static int find_compression_limit(struct drm_i915_private *dev_priv, - unsigned int size, - unsigned int fb_cpp) + unsigned int size, int min_limit) { struct intel_fbc *fbc = &dev_priv->fbc; u64 end = intel_fbc_stolen_end(dev_priv); - int ret, limit = 1; + int ret, limit = min_limit; + + size /= limit; /* Try to over-allocate to reduce reallocations and fragmentation. */ ret = i915_gem_stolen_insert_node_in_range(dev_priv, &fbc->compressed_fb, @@ -471,7 +542,7 @@ static int find_compression_limit(struct drm_i915_private *dev_priv, if (ret == 0) return limit; - for (; limit <= intel_fbc_max_limit(dev_priv, fb_cpp); limit <<= 1) { + for (; limit <= intel_fbc_max_limit(dev_priv); limit <<= 1) { ret = i915_gem_stolen_insert_node_in_range(dev_priv, &fbc->compressed_fb, size >>= 1, 4096, 0, end); if (ret == 0) @@ -482,7 +553,7 @@ static int find_compression_limit(struct drm_i915_private *dev_priv, } static int intel_fbc_alloc_cfb(struct drm_i915_private *dev_priv, - unsigned int size, unsigned int fb_cpp) + unsigned int size, int min_limit) { struct intel_fbc *fbc = &dev_priv->fbc; int ret; @@ -499,13 +570,12 @@ static int intel_fbc_alloc_cfb(struct drm_i915_private *dev_priv, goto err; } - ret = find_compression_limit(dev_priv, size, fb_cpp); + ret = find_compression_limit(dev_priv, size, min_limit); if (!ret) goto err_llb; - else if (ret > 1) { + else if (ret > min_limit) drm_info_once(&dev_priv->drm, "Reducing the compressed framebuffer size. This may lead to less power savings than a non-reduced-size. Try to increase stolen memory size if available in BIOS.\n"); - } fbc->limit = ret; @@ -675,11 +745,10 @@ static bool tiling_is_valid(struct drm_i915_private *dev_priv, { switch (modifier) { case DRM_FORMAT_MOD_LINEAR: - if (DISPLAY_VER(dev_priv) >= 9) - return true; - return false; - case I915_FORMAT_MOD_X_TILED: case I915_FORMAT_MOD_Y_TILED: + case I915_FORMAT_MOD_Yf_TILED: + return DISPLAY_VER(dev_priv) >= 9; + case I915_FORMAT_MOD_X_TILED: return true; default: return false; @@ -718,11 +787,7 @@ static void intel_fbc_update_state_cache(struct intel_crtc *crtc, cache->fb.format = fb->format; cache->fb.modifier = fb->modifier; - - /* FIXME is this correct? */ - cache->fb.stride = plane_state->view.color_plane[0].stride; - if (drm_rotation_90_or_270(plane_state->hw.rotation)) - cache->fb.stride *= fb->format->cpp[0]; + cache->fb.stride = intel_fbc_plane_stride(plane_state); /* FBC1 compression interval: arbitrary choice of 1 second */ cache->interval = drm_mode_vrefresh(&crtc_state->hw.adjusted_mode); @@ -745,27 +810,29 @@ static bool intel_fbc_cfb_size_changed(struct drm_i915_private *dev_priv) { struct intel_fbc *fbc = &dev_priv->fbc; - return intel_fbc_calculate_cfb_size(dev_priv, &fbc->state_cache) > + return intel_fbc_cfb_size(dev_priv, &fbc->state_cache) > fbc->compressed_fb.size * fbc->limit; } -static u16 intel_fbc_gen9_wa_cfb_stride(struct drm_i915_private *dev_priv) +static u16 intel_fbc_override_cfb_stride(struct drm_i915_private *dev_priv, + const struct intel_fbc_state_cache *cache) { - struct intel_fbc *fbc = &dev_priv->fbc; - struct intel_fbc_state_cache *cache = &fbc->state_cache; - - if ((DISPLAY_VER(dev_priv) == 9) && - cache->fb.modifier != I915_FORMAT_MOD_X_TILED) - return DIV_ROUND_UP(cache->plane.src_w, 32 * fbc->limit) * 8; - else - return 0; -} + unsigned int stride = _intel_fbc_cfb_stride(cache); + unsigned int stride_aligned = intel_fbc_cfb_stride(dev_priv, cache); -static bool intel_fbc_gen9_wa_cfb_stride_changed(struct drm_i915_private *dev_priv) -{ - struct intel_fbc *fbc = &dev_priv->fbc; + /* + * Override stride in 64 byte units per 4 line segment. + * + * Gen9 hw miscalculates cfb stride for linear as + * PLANE_STRIDE*512 instead of PLANE_STRIDE*64, so + * we always need to use the override there. + */ + if (stride != stride_aligned || + (DISPLAY_VER(dev_priv) == 9 && + cache->fb.modifier == DRM_FORMAT_MOD_LINEAR)) + return stride_aligned * 4 / 64; - return fbc->params.gen9_wa_cfb_stride != intel_fbc_gen9_wa_cfb_stride(dev_priv); + return 0; } static bool intel_fbc_can_enable(struct drm_i915_private *dev_priv) @@ -860,7 +927,8 @@ static bool intel_fbc_can_activate(struct intel_crtc *crtc) return false; } - if (!stride_is_valid(dev_priv, cache->fb.modifier, cache->fb.stride)) { + if (!stride_is_valid(dev_priv, cache->fb.modifier, + cache->fb.stride * cache->fb.format->cpp[0])) { fbc->no_fbc_reason = "framebuffer stride not supported"; return false; } @@ -948,9 +1016,9 @@ static void intel_fbc_get_reg_params(struct intel_crtc *crtc, params->fb.modifier = cache->fb.modifier; params->fb.stride = cache->fb.stride; - params->cfb_size = intel_fbc_calculate_cfb_size(dev_priv, cache); - - params->gen9_wa_cfb_stride = cache->gen9_wa_cfb_stride; + params->cfb_stride = intel_fbc_cfb_stride(dev_priv, cache); + params->cfb_size = intel_fbc_cfb_size(dev_priv, cache); + params->override_cfb_stride = intel_fbc_override_cfb_stride(dev_priv, cache); params->plane_visible = cache->plane.visible; } @@ -981,10 +1049,13 @@ static bool intel_fbc_can_flip_nuke(const struct intel_crtc_state *crtc_state) if (params->fb.stride != cache->fb.stride) return false; - if (params->cfb_size != intel_fbc_calculate_cfb_size(dev_priv, cache)) + if (params->cfb_stride != intel_fbc_cfb_stride(dev_priv, cache)) + return false; + + if (params->cfb_size != intel_fbc_cfb_size(dev_priv, cache)) return false; - if (params->gen9_wa_cfb_stride != cache->gen9_wa_cfb_stride) + if (params->override_cfb_stride != intel_fbc_override_cfb_stride(dev_priv, cache)) return false; return true; @@ -1090,7 +1161,7 @@ static void __intel_fbc_post_update(struct intel_crtc *crtc) return; if (!fbc->busy_bits) - intel_fbc_hw_activate(dev_priv); + intel_fbc_activate(dev_priv); else intel_fbc_deactivate(dev_priv, "frontbuffer write"); } @@ -1129,7 +1200,7 @@ void intel_fbc_invalidate(struct drm_i915_private *dev_priv, if (!HAS_FBC(dev_priv)) return; - if (origin == ORIGIN_GTT || origin == ORIGIN_FLIP) + if (origin == ORIGIN_FLIP || origin == ORIGIN_CURSOR_UPDATE) return; mutex_lock(&fbc->lock); @@ -1150,19 +1221,11 @@ void intel_fbc_flush(struct drm_i915_private *dev_priv, if (!HAS_FBC(dev_priv)) return; - /* - * GTT tracking does not nuke the entire cfb - * so don't clear busy_bits set for some other - * reason. - */ - if (origin == ORIGIN_GTT) - return; - mutex_lock(&fbc->lock); fbc->busy_bits &= ~frontbuffer_bits; - if (origin == ORIGIN_FLIP) + if (origin == ORIGIN_FLIP || origin == ORIGIN_CURSOR_UPDATE) goto out; if (!fbc->busy_bits && fbc->crtc && @@ -1246,8 +1309,8 @@ out: * intel_fbc_enable multiple times for the same pipe without an * intel_fbc_disable in the middle, as long as it is deactivated. */ -void intel_fbc_enable(struct intel_atomic_state *state, - struct intel_crtc *crtc) +static void intel_fbc_enable(struct intel_atomic_state *state, + struct intel_crtc *crtc) { struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); struct intel_plane *plane = to_intel_plane(crtc->base.primary); @@ -1257,16 +1320,22 @@ void intel_fbc_enable(struct intel_atomic_state *state, intel_atomic_get_new_plane_state(state, plane); struct intel_fbc *fbc = &dev_priv->fbc; struct intel_fbc_state_cache *cache = &fbc->state_cache; + int min_limit; if (!plane->has_fbc || !plane_state) return; + min_limit = intel_fbc_min_limit(plane_state->hw.fb ? + plane_state->hw.fb->format->cpp[0] : 0); + mutex_lock(&fbc->lock); if (fbc->crtc) { - if (fbc->crtc != crtc || - (!intel_fbc_cfb_size_changed(dev_priv) && - !intel_fbc_gen9_wa_cfb_stride_changed(dev_priv))) + if (fbc->crtc != crtc) + goto out; + + if (fbc->limit >= min_limit && + !intel_fbc_cfb_size_changed(dev_priv)) goto out; __intel_fbc_disable(dev_priv); @@ -1281,15 +1350,12 @@ void intel_fbc_enable(struct intel_atomic_state *state, goto out; if (intel_fbc_alloc_cfb(dev_priv, - intel_fbc_calculate_cfb_size(dev_priv, cache), - plane_state->hw.fb->format->cpp[0])) { + intel_fbc_cfb_size(dev_priv, cache), min_limit)) { cache->plane.visible = false; fbc->no_fbc_reason = "not enough stolen memory"; goto out; } - cache->gen9_wa_cfb_stride = intel_fbc_gen9_wa_cfb_stride(dev_priv); - drm_dbg_kms(&dev_priv->drm, "Enabling FBC on pipe %c\n", pipe_name(crtc->pipe)); fbc->no_fbc_reason = "FBC enabled but not active yet\n"; @@ -1323,6 +1389,28 @@ void intel_fbc_disable(struct intel_crtc *crtc) } /** + * intel_fbc_update: enable/disable FBC on the CRTC + * @state: atomic state + * @crtc: the CRTC + * + * This function checks if the given CRTC was chosen for FBC, then enables it if + * possible. Notice that it doesn't activate FBC. It is valid to call + * intel_fbc_update multiple times for the same pipe without an + * intel_fbc_disable in the middle. + */ +void intel_fbc_update(struct intel_atomic_state *state, + struct intel_crtc *crtc) +{ + const struct intel_crtc_state *crtc_state = + intel_atomic_get_new_crtc_state(state, crtc); + + if (crtc_state->update_pipe && !crtc_state->enable_fbc) + intel_fbc_disable(crtc); + else + intel_fbc_enable(state, crtc); +} + +/** * intel_fbc_global_disable - globally disable FBC * @dev_priv: i915 device instance * diff --git a/drivers/gpu/drm/i915/display/intel_fbc.h b/drivers/gpu/drm/i915/display/intel_fbc.h index 6dc1edefe81b..b97d908738e6 100644 --- a/drivers/gpu/drm/i915/display/intel_fbc.h +++ b/drivers/gpu/drm/i915/display/intel_fbc.h @@ -24,7 +24,7 @@ bool intel_fbc_pre_update(struct intel_atomic_state *state, void intel_fbc_post_update(struct intel_atomic_state *state, struct intel_crtc *crtc); void intel_fbc_init(struct drm_i915_private *dev_priv); -void intel_fbc_enable(struct intel_atomic_state *state, +void intel_fbc_update(struct intel_atomic_state *state, struct intel_crtc *crtc); void intel_fbc_disable(struct intel_crtc *crtc); void intel_fbc_global_disable(struct drm_i915_private *dev_priv); diff --git a/drivers/gpu/drm/i915/display/intel_fbdev.c b/drivers/gpu/drm/i915/display/intel_fbdev.c index df05d285f0bd..adc3a81be9f7 100644 --- a/drivers/gpu/drm/i915/display/intel_fbdev.c +++ b/drivers/gpu/drm/i915/display/intel_fbdev.c @@ -45,6 +45,8 @@ #include "i915_drv.h" #include "intel_display_types.h" +#include "intel_fb.h" +#include "intel_fb_pin.h" #include "intel_fbdev.h" #include "intel_frontbuffer.h" @@ -229,8 +231,6 @@ static int intelfb_create(struct drm_fb_helper *helper, goto out_unlock; } - intel_frontbuffer_flush(to_frontbuffer(ifbdev), ORIGIN_DIRTYFB); - info = drm_fb_helper_alloc_fbi(helper); if (IS_ERR(info)) { drm_err(&dev_priv->drm, "Failed to allocate fb_info\n"); diff --git a/drivers/gpu/drm/i915/display/intel_fdi.c b/drivers/gpu/drm/i915/display/intel_fdi.c index e10b9cd8e86e..dd2cf0c59921 100644 --- a/drivers/gpu/drm/i915/display/intel_fdi.c +++ b/drivers/gpu/drm/i915/display/intel_fdi.c @@ -2,11 +2,112 @@ /* * Copyright © 2020 Intel Corporation */ + #include "intel_atomic.h" #include "intel_ddi.h" #include "intel_de.h" #include "intel_display_types.h" #include "intel_fdi.h" +#include "intel_sbi.h" + +static void assert_fdi_tx(struct drm_i915_private *dev_priv, + enum pipe pipe, bool state) +{ + bool cur_state; + + if (HAS_DDI(dev_priv)) { + /* + * DDI does not have a specific FDI_TX register. + * + * FDI is never fed from EDP transcoder + * so pipe->transcoder cast is fine here. + */ + enum transcoder cpu_transcoder = (enum transcoder)pipe; + cur_state = intel_de_read(dev_priv, TRANS_DDI_FUNC_CTL(cpu_transcoder)) & TRANS_DDI_FUNC_ENABLE; + } else { + cur_state = intel_de_read(dev_priv, FDI_TX_CTL(pipe)) & FDI_TX_ENABLE; + } + I915_STATE_WARN(cur_state != state, + "FDI TX state assertion failure (expected %s, current %s)\n", + onoff(state), onoff(cur_state)); +} + +void assert_fdi_tx_enabled(struct drm_i915_private *i915, enum pipe pipe) +{ + assert_fdi_tx(i915, pipe, true); +} + +void assert_fdi_tx_disabled(struct drm_i915_private *i915, enum pipe pipe) +{ + assert_fdi_tx(i915, pipe, false); +} + +static void assert_fdi_rx(struct drm_i915_private *dev_priv, + enum pipe pipe, bool state) +{ + bool cur_state; + + cur_state = intel_de_read(dev_priv, FDI_RX_CTL(pipe)) & FDI_RX_ENABLE; + I915_STATE_WARN(cur_state != state, + "FDI RX state assertion failure (expected %s, current %s)\n", + onoff(state), onoff(cur_state)); +} + +void assert_fdi_rx_enabled(struct drm_i915_private *i915, enum pipe pipe) +{ + assert_fdi_rx(i915, pipe, true); +} + +void assert_fdi_rx_disabled(struct drm_i915_private *i915, enum pipe pipe) +{ + assert_fdi_rx(i915, pipe, false); +} + +void assert_fdi_tx_pll_enabled(struct drm_i915_private *i915, + enum pipe pipe) +{ + bool cur_state; + + /* ILK FDI PLL is always enabled */ + if (IS_IRONLAKE(i915)) + return; + + /* On Haswell, DDI ports are responsible for the FDI PLL setup */ + if (HAS_DDI(i915)) + return; + + cur_state = intel_de_read(i915, FDI_TX_CTL(pipe)) & FDI_TX_PLL_ENABLE; + I915_STATE_WARN(!cur_state, "FDI TX PLL assertion failure, should be active but is disabled\n"); +} + +static void assert_fdi_rx_pll(struct drm_i915_private *i915, + enum pipe pipe, bool state) +{ + bool cur_state; + + cur_state = intel_de_read(i915, FDI_RX_CTL(pipe)) & FDI_RX_PLL_ENABLE; + I915_STATE_WARN(cur_state != state, + "FDI RX PLL assertion failure (expected %s, current %s)\n", + onoff(state), onoff(cur_state)); +} + +void assert_fdi_rx_pll_enabled(struct drm_i915_private *i915, enum pipe pipe) +{ + assert_fdi_rx_pll(i915, pipe, true); +} + +void assert_fdi_rx_pll_disabled(struct drm_i915_private *i915, enum pipe pipe) +{ + assert_fdi_rx_pll(i915, pipe, false); +} + +void intel_fdi_link_train(struct intel_crtc *crtc, + const struct intel_crtc_state *crtc_state) +{ + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); + + dev_priv->fdi_funcs->fdi_link_train(crtc, crtc_state); +} /* units of 100MHz */ static int pipe_required_fdi_lanes(struct intel_crtc_state *crtc_state) @@ -91,8 +192,34 @@ static int ilk_check_fdi_lanes(struct drm_device *dev, enum pipe pipe, } return 0; default: - BUG(); + MISSING_CASE(pipe); + return 0; + } +} + +void intel_fdi_pll_freq_update(struct drm_i915_private *i915) +{ + if (IS_IRONLAKE(i915)) { + u32 fdi_pll_clk = + intel_de_read(i915, FDI_PLL_BIOS_0) & FDI_PLL_FB_CLOCK_MASK; + + i915->fdi_pll_freq = (fdi_pll_clk + 2) * 10000; + } else if (IS_SANDYBRIDGE(i915) || IS_IVYBRIDGE(i915)) { + i915->fdi_pll_freq = 270000; + } else { + return; } + + drm_dbg(&i915->drm, "FDI PLL freq=%d\n", i915->fdi_pll_freq); +} + +int intel_fdi_link_freq(struct drm_i915_private *i915, + const struct intel_crtc_state *pipe_config) +{ + if (HAS_DDI(i915)) + return pipe_config->port_clock; /* SPLL */ + else + return i915->fdi_pll_freq; } int ilk_fdi_compute_config(struct intel_crtc *crtc, @@ -140,11 +267,60 @@ retry: } if (needs_recompute) - return I915_DISPLAY_CONFIG_RETRY; + return -EAGAIN; return ret; } +static void cpt_set_fdi_bc_bifurcation(struct drm_i915_private *dev_priv, bool enable) +{ + u32 temp; + + temp = intel_de_read(dev_priv, SOUTH_CHICKEN1); + if (!!(temp & FDI_BC_BIFURCATION_SELECT) == enable) + return; + + drm_WARN_ON(&dev_priv->drm, + intel_de_read(dev_priv, FDI_RX_CTL(PIPE_B)) & + FDI_RX_ENABLE); + drm_WARN_ON(&dev_priv->drm, + intel_de_read(dev_priv, FDI_RX_CTL(PIPE_C)) & + FDI_RX_ENABLE); + + temp &= ~FDI_BC_BIFURCATION_SELECT; + if (enable) + temp |= FDI_BC_BIFURCATION_SELECT; + + drm_dbg_kms(&dev_priv->drm, "%sabling fdi C rx\n", + enable ? "en" : "dis"); + intel_de_write(dev_priv, SOUTH_CHICKEN1, temp); + intel_de_posting_read(dev_priv, SOUTH_CHICKEN1); +} + +static void ivb_update_fdi_bc_bifurcation(const struct intel_crtc_state *crtc_state) +{ + struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); + + switch (crtc->pipe) { + case PIPE_A: + break; + case PIPE_B: + if (crtc_state->fdi_lanes > 2) + cpt_set_fdi_bc_bifurcation(dev_priv, false); + else + cpt_set_fdi_bc_bifurcation(dev_priv, true); + + break; + case PIPE_C: + cpt_set_fdi_bc_bifurcation(dev_priv, true); + + break; + default: + MISSING_CASE(crtc->pipe); + } +} + void intel_fdi_normal_train(struct intel_crtc *crtc) { struct drm_device *dev = crtc->base.dev; @@ -196,8 +372,15 @@ static void ilk_fdi_link_train(struct intel_crtc *crtc, i915_reg_t reg; u32 temp, tries; + /* + * Write the TU size bits before fdi link training, so that error + * detection works. + */ + intel_de_write(dev_priv, FDI_RX_TUSIZE1(pipe), + intel_de_read(dev_priv, PIPE_DATA_M1(pipe)) & TU_SIZE_MASK); + /* FDI needs bits from pipe first */ - assert_pipe_enabled(dev_priv, crtc_state->cpu_transcoder); + assert_transcoder_enabled(dev_priv, crtc_state->cpu_transcoder); /* Train 1: umask FDI RX Interrupt symbol_lock and bit_lock bit for train result */ @@ -299,6 +482,13 @@ static void gen6_fdi_link_train(struct intel_crtc *crtc, i915_reg_t reg; u32 temp, i, retry; + /* + * Write the TU size bits before fdi link training, so that error + * detection works. + */ + intel_de_write(dev_priv, FDI_RX_TUSIZE1(pipe), + intel_de_read(dev_priv, PIPE_DATA_M1(pipe)) & TU_SIZE_MASK); + /* Train 1: umask FDI RX Interrupt symbol_lock and bit_lock bit for train result */ reg = FDI_RX_IMR(pipe); @@ -436,6 +626,15 @@ static void ivb_manual_fdi_link_train(struct intel_crtc *crtc, i915_reg_t reg; u32 temp, i, j; + ivb_update_fdi_bc_bifurcation(crtc_state); + + /* + * Write the TU size bits before fdi link training, so that error + * detection works. + */ + intel_de_write(dev_priv, FDI_RX_TUSIZE1(pipe), + intel_de_read(dev_priv, PIPE_DATA_M1(pipe)) & TU_SIZE_MASK); + /* Train 1: umask FDI RX Interrupt symbol_lock and bit_lock bit for train result */ reg = FDI_RX_IMR(pipe); @@ -807,15 +1006,125 @@ void ilk_fdi_disable(struct intel_crtc *crtc) udelay(100); } +static void lpt_fdi_reset_mphy(struct drm_i915_private *dev_priv) +{ + u32 tmp; + + tmp = intel_de_read(dev_priv, SOUTH_CHICKEN2); + tmp |= FDI_MPHY_IOSFSB_RESET_CTL; + intel_de_write(dev_priv, SOUTH_CHICKEN2, tmp); + + if (wait_for_us(intel_de_read(dev_priv, SOUTH_CHICKEN2) & + FDI_MPHY_IOSFSB_RESET_STATUS, 100)) + drm_err(&dev_priv->drm, "FDI mPHY reset assert timeout\n"); + + tmp = intel_de_read(dev_priv, SOUTH_CHICKEN2); + tmp &= ~FDI_MPHY_IOSFSB_RESET_CTL; + intel_de_write(dev_priv, SOUTH_CHICKEN2, tmp); + + if (wait_for_us((intel_de_read(dev_priv, SOUTH_CHICKEN2) & + FDI_MPHY_IOSFSB_RESET_STATUS) == 0, 100)) + drm_err(&dev_priv->drm, "FDI mPHY reset de-assert timeout\n"); +} + +/* WaMPhyProgramming:hsw */ +void lpt_fdi_program_mphy(struct drm_i915_private *dev_priv) +{ + u32 tmp; + + lpt_fdi_reset_mphy(dev_priv); + + tmp = intel_sbi_read(dev_priv, 0x8008, SBI_MPHY); + tmp &= ~(0xFF << 24); + tmp |= (0x12 << 24); + intel_sbi_write(dev_priv, 0x8008, tmp, SBI_MPHY); + + tmp = intel_sbi_read(dev_priv, 0x2008, SBI_MPHY); + tmp |= (1 << 11); + intel_sbi_write(dev_priv, 0x2008, tmp, SBI_MPHY); + + tmp = intel_sbi_read(dev_priv, 0x2108, SBI_MPHY); + tmp |= (1 << 11); + intel_sbi_write(dev_priv, 0x2108, tmp, SBI_MPHY); + + tmp = intel_sbi_read(dev_priv, 0x206C, SBI_MPHY); + tmp |= (1 << 24) | (1 << 21) | (1 << 18); + intel_sbi_write(dev_priv, 0x206C, tmp, SBI_MPHY); + + tmp = intel_sbi_read(dev_priv, 0x216C, SBI_MPHY); + tmp |= (1 << 24) | (1 << 21) | (1 << 18); + intel_sbi_write(dev_priv, 0x216C, tmp, SBI_MPHY); + + tmp = intel_sbi_read(dev_priv, 0x2080, SBI_MPHY); + tmp &= ~(7 << 13); + tmp |= (5 << 13); + intel_sbi_write(dev_priv, 0x2080, tmp, SBI_MPHY); + + tmp = intel_sbi_read(dev_priv, 0x2180, SBI_MPHY); + tmp &= ~(7 << 13); + tmp |= (5 << 13); + intel_sbi_write(dev_priv, 0x2180, tmp, SBI_MPHY); + + tmp = intel_sbi_read(dev_priv, 0x208C, SBI_MPHY); + tmp &= ~0xFF; + tmp |= 0x1C; + intel_sbi_write(dev_priv, 0x208C, tmp, SBI_MPHY); + + tmp = intel_sbi_read(dev_priv, 0x218C, SBI_MPHY); + tmp &= ~0xFF; + tmp |= 0x1C; + intel_sbi_write(dev_priv, 0x218C, tmp, SBI_MPHY); + + tmp = intel_sbi_read(dev_priv, 0x2098, SBI_MPHY); + tmp &= ~(0xFF << 16); + tmp |= (0x1C << 16); + intel_sbi_write(dev_priv, 0x2098, tmp, SBI_MPHY); + + tmp = intel_sbi_read(dev_priv, 0x2198, SBI_MPHY); + tmp &= ~(0xFF << 16); + tmp |= (0x1C << 16); + intel_sbi_write(dev_priv, 0x2198, tmp, SBI_MPHY); + + tmp = intel_sbi_read(dev_priv, 0x20C4, SBI_MPHY); + tmp |= (1 << 27); + intel_sbi_write(dev_priv, 0x20C4, tmp, SBI_MPHY); + + tmp = intel_sbi_read(dev_priv, 0x21C4, SBI_MPHY); + tmp |= (1 << 27); + intel_sbi_write(dev_priv, 0x21C4, tmp, SBI_MPHY); + + tmp = intel_sbi_read(dev_priv, 0x20EC, SBI_MPHY); + tmp &= ~(0xF << 28); + tmp |= (4 << 28); + intel_sbi_write(dev_priv, 0x20EC, tmp, SBI_MPHY); + + tmp = intel_sbi_read(dev_priv, 0x21EC, SBI_MPHY); + tmp &= ~(0xF << 28); + tmp |= (4 << 28); + intel_sbi_write(dev_priv, 0x21EC, tmp, SBI_MPHY); +} + +static const struct intel_fdi_funcs ilk_funcs = { + .fdi_link_train = ilk_fdi_link_train, +}; + +static const struct intel_fdi_funcs gen6_funcs = { + .fdi_link_train = gen6_fdi_link_train, +}; + +static const struct intel_fdi_funcs ivb_funcs = { + .fdi_link_train = ivb_manual_fdi_link_train, +}; + void intel_fdi_init_hook(struct drm_i915_private *dev_priv) { if (IS_IRONLAKE(dev_priv)) { - dev_priv->display.fdi_link_train = ilk_fdi_link_train; + dev_priv->fdi_funcs = &ilk_funcs; } else if (IS_SANDYBRIDGE(dev_priv)) { - dev_priv->display.fdi_link_train = gen6_fdi_link_train; + dev_priv->fdi_funcs = &gen6_funcs; } else if (IS_IVYBRIDGE(dev_priv)) { /* FIXME: detect B0+ stepping and use auto training */ - dev_priv->display.fdi_link_train = ivb_manual_fdi_link_train; + dev_priv->fdi_funcs = &ivb_funcs; } } diff --git a/drivers/gpu/drm/i915/display/intel_fdi.h b/drivers/gpu/drm/i915/display/intel_fdi.h index af01d2c173a8..640d6585c137 100644 --- a/drivers/gpu/drm/i915/display/intel_fdi.h +++ b/drivers/gpu/drm/i915/display/intel_fdi.h @@ -6,12 +6,14 @@ #ifndef _INTEL_FDI_H_ #define _INTEL_FDI_H_ +enum pipe; struct drm_i915_private; struct intel_crtc; struct intel_crtc_state; struct intel_encoder; -#define I915_DISPLAY_CONFIG_RETRY 1 +int intel_fdi_link_freq(struct drm_i915_private *i915, + const struct intel_crtc_state *pipe_config); int ilk_fdi_compute_config(struct intel_crtc *intel_crtc, struct intel_crtc_state *pipe_config); void intel_fdi_normal_train(struct intel_crtc *crtc); @@ -21,5 +23,18 @@ void ilk_fdi_pll_enable(const struct intel_crtc_state *crtc_state); void intel_fdi_init_hook(struct drm_i915_private *dev_priv); void hsw_fdi_link_train(struct intel_encoder *encoder, const struct intel_crtc_state *crtc_state); +void intel_fdi_pll_freq_update(struct drm_i915_private *i915); +void lpt_fdi_program_mphy(struct drm_i915_private *i915); + +void intel_fdi_link_train(struct intel_crtc *crtc, + const struct intel_crtc_state *crtc_state); + +void assert_fdi_tx_enabled(struct drm_i915_private *i915, enum pipe pipe); +void assert_fdi_tx_disabled(struct drm_i915_private *i915, enum pipe pipe); +void assert_fdi_rx_enabled(struct drm_i915_private *i915, enum pipe pipe); +void assert_fdi_rx_disabled(struct drm_i915_private *i915, enum pipe pipe); +void assert_fdi_tx_pll_enabled(struct drm_i915_private *i915, enum pipe pipe); +void assert_fdi_rx_pll_enabled(struct drm_i915_private *i915, enum pipe pipe); +void assert_fdi_rx_pll_disabled(struct drm_i915_private *i915, enum pipe pipe); #endif diff --git a/drivers/gpu/drm/i915/display/intel_frontbuffer.c b/drivers/gpu/drm/i915/display/intel_frontbuffer.c index 8e75debcce1a..0492446cd04a 100644 --- a/drivers/gpu/drm/i915/display/intel_frontbuffer.c +++ b/drivers/gpu/drm/i915/display/intel_frontbuffer.c @@ -62,6 +62,7 @@ #include "intel_display_types.h" #include "intel_fbc.h" #include "intel_frontbuffer.h" +#include "intel_drrs.h" #include "intel_psr.h" /** @@ -91,7 +92,7 @@ static void frontbuffer_flush(struct drm_i915_private *i915, trace_intel_frontbuffer_flush(frontbuffer_bits, origin); might_sleep(); - intel_edp_drrs_flush(i915, frontbuffer_bits); + intel_drrs_flush(i915, frontbuffer_bits); intel_psr_flush(i915, frontbuffer_bits, origin); intel_fbc_flush(i915, frontbuffer_bits, origin); } @@ -180,7 +181,7 @@ void __intel_fb_invalidate(struct intel_frontbuffer *front, might_sleep(); intel_psr_invalidate(i915, frontbuffer_bits, origin); - intel_edp_drrs_invalidate(i915, frontbuffer_bits); + intel_drrs_invalidate(i915, frontbuffer_bits); intel_fbc_invalidate(i915, frontbuffer_bits, origin); } diff --git a/drivers/gpu/drm/i915/display/intel_frontbuffer.h b/drivers/gpu/drm/i915/display/intel_frontbuffer.h index 6d41f5394425..a88441edc8f9 100644 --- a/drivers/gpu/drm/i915/display/intel_frontbuffer.h +++ b/drivers/gpu/drm/i915/display/intel_frontbuffer.h @@ -33,11 +33,11 @@ struct drm_i915_private; enum fb_op_origin { - ORIGIN_GTT, - ORIGIN_CPU, + ORIGIN_CPU = 0, ORIGIN_CS, ORIGIN_FLIP, ORIGIN_DIRTYFB, + ORIGIN_CURSOR_UPDATE, }; struct intel_frontbuffer { diff --git a/drivers/gpu/drm/i915/display/intel_hdcp.c b/drivers/gpu/drm/i915/display/intel_hdcp.c index ebc2e32aec0b..4509fe7438e8 100644 --- a/drivers/gpu/drm/i915/display/intel_hdcp.c +++ b/drivers/gpu/drm/i915/display/intel_hdcp.c @@ -17,12 +17,12 @@ #include "i915_drv.h" #include "i915_reg.h" -#include "intel_display_power.h" +#include "intel_connector.h" #include "intel_de.h" +#include "intel_display_power.h" #include "intel_display_types.h" #include "intel_hdcp.h" -#include "intel_sideband.h" -#include "intel_connector.h" +#include "intel_pcode.h" #define KEY_LOAD_TRIES 5 #define HDCP2_LC_RETRY_CNT 3 @@ -33,21 +33,6 @@ static int intel_conn_to_vcpi(struct intel_connector *connector) return connector->port ? connector->port->vcpi.vcpi : 0; } -static bool -intel_streams_type1_capable(struct intel_connector *connector) -{ - const struct intel_hdcp_shim *shim = connector->hdcp.shim; - bool capable = false; - - if (!shim) - return capable; - - if (shim->streams_type1_capable) - shim->streams_type1_capable(connector, &capable); - - return capable; -} - /* * intel_hdcp_required_content_stream selects the most highest common possible HDCP * content_type for all streams in DP MST topology because security f/w doesn't @@ -86,7 +71,7 @@ intel_hdcp_required_content_stream(struct intel_digital_port *dig_port) if (conn_dig_port != dig_port) continue; - if (!enforce_type0 && !intel_streams_type1_capable(connector)) + if (!enforce_type0 && !dig_port->hdcp_mst_type1_capable) enforce_type0 = true; data->streams[data->k].stream_id = intel_conn_to_vcpi(connector); @@ -112,6 +97,25 @@ intel_hdcp_required_content_stream(struct intel_digital_port *dig_port) return 0; } +static int intel_hdcp_prepare_streams(struct intel_connector *connector) +{ + struct intel_digital_port *dig_port = intel_attached_dig_port(connector); + struct hdcp_port_data *data = &dig_port->hdcp_port_data; + struct intel_hdcp *hdcp = &connector->hdcp; + int ret; + + if (!intel_encoder_is_mst(intel_attached_encoder(connector))) { + data->k = 1; + data->streams[0].stream_type = hdcp->content_type; + } else { + ret = intel_hdcp_required_content_stream(dig_port); + if (ret) + return ret; + } + + return 0; +} + static bool intel_hdcp_is_ksv_valid(u8 *ksv) { @@ -1632,6 +1636,14 @@ int hdcp2_authenticate_repeater_topology(struct intel_connector *connector) return -EINVAL; } + /* + * MST topology is not Type 1 capable if it contains a downstream + * device that is only HDCP 1.x or Legacy HDCP 2.0/2.1 compliant. + */ + dig_port->hdcp_mst_type1_capable = + !HDCP_2_2_HDCP1_DEVICE_CONNECTED(rx_info[1]) && + !HDCP_2_2_HDCP_2_0_REP_CONNECTED(rx_info[1]); + /* Converting and Storing the seq_num_v to local variable as DWORD */ seq_num_v = drm_hdcp_be24_to_cpu((const u8 *)msgs.recvid_list.seq_num_v); @@ -1876,6 +1888,14 @@ static int hdcp2_authenticate_and_encrypt(struct intel_connector *connector) for (i = 0; i < tries && !dig_port->hdcp_auth_status; i++) { ret = hdcp2_authenticate_sink(connector); if (!ret) { + ret = intel_hdcp_prepare_streams(connector); + if (ret) { + drm_dbg_kms(&i915->drm, + "Prepare streams failed.(%d)\n", + ret); + break; + } + ret = hdcp2_propagate_stream_management_info(connector); if (ret) { drm_dbg_kms(&i915->drm, @@ -1921,9 +1941,7 @@ static int hdcp2_authenticate_and_encrypt(struct intel_connector *connector) static int _intel_hdcp2_enable(struct intel_connector *connector) { - struct intel_digital_port *dig_port = intel_attached_dig_port(connector); struct drm_i915_private *i915 = to_i915(connector->base.dev); - struct hdcp_port_data *data = &dig_port->hdcp_port_data; struct intel_hdcp *hdcp = &connector->hdcp; int ret; @@ -1931,16 +1949,6 @@ static int _intel_hdcp2_enable(struct intel_connector *connector) connector->base.name, connector->base.base.id, hdcp->content_type); - /* Stream which requires encryption */ - if (!intel_encoder_is_mst(intel_attached_encoder(connector))) { - data->k = 1; - data->streams[0].stream_type = hdcp->content_type; - } else { - ret = intel_hdcp_required_content_stream(dig_port); - if (ret) - return ret; - } - ret = hdcp2_authenticate_and_encrypt(connector); if (ret) { drm_dbg_kms(&i915->drm, "HDCP2 Type%d Enabling Failed. (%d)\n", diff --git a/drivers/gpu/drm/i915/display/intel_hdmi.c b/drivers/gpu/drm/i915/display/intel_hdmi.c index b04685bb6439..d2e61f6c6e08 100644 --- a/drivers/gpu/drm/i915/display/intel_hdmi.c +++ b/drivers/gpu/drm/i915/display/intel_hdmi.c @@ -53,21 +53,20 @@ #include "intel_panel.h" #include "intel_snps_phy.h" -static struct drm_device *intel_hdmi_to_dev(struct intel_hdmi *intel_hdmi) +static struct drm_i915_private *intel_hdmi_to_i915(struct intel_hdmi *intel_hdmi) { - return hdmi_to_dig_port(intel_hdmi)->base.base.dev; + return to_i915(hdmi_to_dig_port(intel_hdmi)->base.base.dev); } static void assert_hdmi_port_disabled(struct intel_hdmi *intel_hdmi) { - struct drm_device *dev = intel_hdmi_to_dev(intel_hdmi); - struct drm_i915_private *dev_priv = to_i915(dev); + struct drm_i915_private *dev_priv = intel_hdmi_to_i915(intel_hdmi); u32 enabled_bits; enabled_bits = HAS_DDI(dev_priv) ? DDI_BUF_CTL_ENABLE : SDVO_ENABLE; - drm_WARN(dev, + drm_WARN(&dev_priv->drm, intel_de_read(dev_priv, intel_hdmi->hdmi_reg) & enabled_bits, "HDMI port enabled, expecting disabled\n"); } @@ -1246,7 +1245,7 @@ static void hsw_set_infoframes(struct intel_encoder *encoder, void intel_dp_dual_mode_set_tmds_output(struct intel_hdmi *hdmi, bool enable) { - struct drm_i915_private *dev_priv = to_i915(intel_hdmi_to_dev(hdmi)); + struct drm_i915_private *dev_priv = intel_hdmi_to_i915(hdmi); struct i2c_adapter *adapter = intel_gmbus_get_adapter(dev_priv, hdmi->ddc_bus); @@ -1703,7 +1702,7 @@ int intel_hdmi_hdcp2_read_msg(struct intel_digital_port *dig_port, drm_dbg_kms(&i915->drm, "msg_sz(%zd) is more than exp size(%zu)\n", ret, size); - return -1; + return -EINVAL; } offset = HDCP_2_2_HDMI_REG_RD_MSG_OFFSET; @@ -1830,7 +1829,7 @@ hdmi_port_clock_valid(struct intel_hdmi *hdmi, int clock, bool respect_downstream_limits, bool has_hdmi_sink) { - struct drm_i915_private *dev_priv = to_i915(intel_hdmi_to_dev(hdmi)); + struct drm_i915_private *dev_priv = intel_hdmi_to_i915(hdmi); if (clock < 25000) return MODE_CLOCK_LOW; @@ -1946,8 +1945,7 @@ intel_hdmi_mode_valid(struct drm_connector *connector, struct drm_display_mode *mode) { struct intel_hdmi *hdmi = intel_attached_hdmi(to_intel_connector(connector)); - struct drm_device *dev = intel_hdmi_to_dev(hdmi); - struct drm_i915_private *dev_priv = to_i915(dev); + struct drm_i915_private *dev_priv = intel_hdmi_to_i915(hdmi); enum drm_mode_status status; int clock = mode->clock; int max_dotclk = to_i915(connector->dev)->max_dotclk_freq; @@ -2210,7 +2208,7 @@ int intel_hdmi_compute_config(struct intel_encoder *encoder, return ret; if (pipe_config->output_format == INTEL_OUTPUT_FORMAT_YCBCR420) { - ret = intel_pch_panel_fitting(pipe_config, conn_state); + ret = intel_panel_fitting(pipe_config, conn_state); if (ret) return ret; } diff --git a/drivers/gpu/drm/i915/display/intel_hotplug.c b/drivers/gpu/drm/i915/display/intel_hotplug.c index 47c85ac97c87..955f6d07b0e1 100644 --- a/drivers/gpu/drm/i915/display/intel_hotplug.c +++ b/drivers/gpu/drm/i915/display/intel_hotplug.c @@ -215,8 +215,8 @@ intel_hpd_irq_storm_switch_to_polling(struct drm_i915_private *dev_priv) static void intel_hpd_irq_setup(struct drm_i915_private *i915) { - if (i915->display_irqs_enabled && i915->display.hpd_irq_setup) - i915->display.hpd_irq_setup(i915); + if (i915->display_irqs_enabled && i915->hotplug_funcs) + i915->hotplug_funcs->hpd_irq_setup(i915); } static void intel_hpd_irq_storm_reenable_work(struct work_struct *work) diff --git a/drivers/gpu/drm/i915/display/intel_lvds.c b/drivers/gpu/drm/i915/display/intel_lvds.c index e0381b0fce91..9fced37bed70 100644 --- a/drivers/gpu/drm/i915/display/intel_lvds.c +++ b/drivers/gpu/drm/i915/display/intel_lvds.c @@ -40,9 +40,12 @@ #include "i915_drv.h" #include "intel_atomic.h" +#include "intel_backlight.h" #include "intel_connector.h" #include "intel_de.h" #include "intel_display_types.h" +#include "intel_dpll.h" +#include "intel_fdi.h" #include "intel_gmbus.h" #include "intel_lvds.h" #include "intel_panel.h" @@ -323,7 +326,7 @@ static void intel_enable_lvds(struct intel_atomic_state *state, drm_err(&dev_priv->drm, "timed out waiting for panel to power on\n"); - intel_panel_enable_backlight(pipe_config, conn_state); + intel_backlight_enable(pipe_config, conn_state); } static void intel_disable_lvds(struct intel_atomic_state *state, @@ -351,7 +354,7 @@ static void gmch_disable_lvds(struct intel_atomic_state *state, const struct drm_connector_state *old_conn_state) { - intel_panel_disable_backlight(old_conn_state); + intel_backlight_disable(old_conn_state); intel_disable_lvds(state, encoder, old_crtc_state, old_conn_state); } @@ -361,7 +364,7 @@ static void pch_disable_lvds(struct intel_atomic_state *state, const struct intel_crtc_state *old_crtc_state, const struct drm_connector_state *old_conn_state) { - intel_panel_disable_backlight(old_conn_state); + intel_backlight_disable(old_conn_state); } static void pch_post_disable_lvds(struct intel_atomic_state *state, @@ -388,13 +391,15 @@ intel_lvds_mode_valid(struct drm_connector *connector, struct intel_connector *intel_connector = to_intel_connector(connector); struct drm_display_mode *fixed_mode = intel_connector->panel.fixed_mode; int max_pixclk = to_i915(connector->dev)->max_dotclk_freq; + enum drm_mode_status status; if (mode->flags & DRM_MODE_FLAG_DBLSCAN) return MODE_NO_DBLESCAN; - if (mode->hdisplay > fixed_mode->hdisplay) - return MODE_PANEL; - if (mode->vdisplay > fixed_mode->vdisplay) - return MODE_PANEL; + + status = intel_panel_mode_valid(intel_connector, mode); + if (status != MODE_OK) + return status; + if (fixed_mode->clock > max_pixclk) return MODE_CLOCK_HIGH; @@ -441,8 +446,9 @@ static int intel_lvds_compute_config(struct intel_encoder *intel_encoder, * with the panel scaling set up to source from the H/VDisplay * of the original mode. */ - intel_fixed_panel_mode(intel_connector->panel.fixed_mode, - adjusted_mode); + ret = intel_panel_compute_config(intel_connector, adjusted_mode); + if (ret) + return ret; if (adjusted_mode->flags & DRM_MODE_FLAG_DBLSCAN) return -EINVAL; @@ -450,10 +456,7 @@ static int intel_lvds_compute_config(struct intel_encoder *intel_encoder, if (HAS_PCH_SPLIT(dev_priv)) pipe_config->has_pch_encoder = true; - if (HAS_GMCH(dev_priv)) - ret = intel_gmch_panel_fitting(pipe_config, conn_state); - else - ret = intel_pch_panel_fitting(pipe_config, conn_state); + ret = intel_panel_fitting(pipe_config, conn_state); if (ret) return ret; @@ -906,7 +909,7 @@ void intel_lvds_init(struct drm_i915_private *dev_priv) } intel_encoder->get_hw_state = intel_lvds_get_hw_state; intel_encoder->get_config = intel_lvds_get_config; - intel_encoder->update_pipe = intel_panel_update_backlight; + intel_encoder->update_pipe = intel_backlight_update; intel_encoder->shutdown = intel_lvds_shutdown; intel_connector->get_hw_state = intel_connector_get_hw_state; @@ -999,7 +1002,7 @@ out: mutex_unlock(&dev->mode_config.mutex); intel_panel_init(&intel_connector->panel, fixed_mode, downclock_mode); - intel_panel_setup_backlight(connector, INVALID_PIPE); + intel_backlight_setup(intel_connector, INVALID_PIPE); lvds_encoder->is_dual_link = compute_is_dual_link_lvds(lvds_encoder); drm_dbg_kms(&dev_priv->drm, "detected %s-link lvds configuration\n", diff --git a/drivers/gpu/drm/i915/display/intel_opregion.c b/drivers/gpu/drm/i915/display/intel_opregion.c index 3855fba70980..0065111593a6 100644 --- a/drivers/gpu/drm/i915/display/intel_opregion.c +++ b/drivers/gpu/drm/i915/display/intel_opregion.c @@ -30,10 +30,9 @@ #include <linux/firmware.h> #include <acpi/video.h> -#include "display/intel_panel.h" - #include "i915_drv.h" #include "intel_acpi.h" +#include "intel_backlight.h" #include "intel_display_types.h" #include "intel_opregion.h" @@ -450,7 +449,7 @@ static u32 asle_set_backlight(struct drm_i915_private *dev_priv, u32 bclp) bclp); drm_connector_list_iter_begin(dev, &conn_iter); for_each_intel_connector_iter(connector, &conn_iter) - intel_panel_set_backlight_acpi(connector->base.state, bclp, 255); + intel_backlight_set_acpi(connector->base.state, bclp, 255); drm_connector_list_iter_end(&conn_iter); asle->cblv = DIV_ROUND_UP(bclp * 100, 255) | ASLE_CBLV_VALID; diff --git a/drivers/gpu/drm/i915/display/intel_panel.c b/drivers/gpu/drm/i915/display/intel_panel.c index 7d7a60b4d2de..a0c8e43db5eb 100644 --- a/drivers/gpu/drm/i915/display/intel_panel.c +++ b/drivers/gpu/drm/i915/display/intel_panel.c @@ -28,26 +28,51 @@ * Chris Wilson <chris@chris-wilson.co.uk> */ -#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt - #include <linux/kernel.h> -#include <linux/moduleparam.h> #include <linux/pwm.h> +#include "intel_backlight.h" #include "intel_connector.h" #include "intel_de.h" #include "intel_display_types.h" -#include "intel_dp_aux_backlight.h" -#include "intel_dsi_dcs_backlight.h" #include "intel_panel.h" -void -intel_fixed_panel_mode(const struct drm_display_mode *fixed_mode, - struct drm_display_mode *adjusted_mode) +bool intel_panel_use_ssc(struct drm_i915_private *i915) +{ + if (i915->params.panel_use_ssc >= 0) + return i915->params.panel_use_ssc != 0; + return i915->vbt.lvds_use_ssc + && !(i915->quirks & QUIRK_LVDS_SSC_DISABLE); +} + +int intel_panel_compute_config(struct intel_connector *connector, + struct drm_display_mode *adjusted_mode) { + const struct drm_display_mode *fixed_mode = connector->panel.fixed_mode; + + if (!fixed_mode) + return 0; + + /* + * We don't want to lie too much to the user about the refresh + * rate they're going to get. But we have to allow a bit of latitude + * for Xorg since it likes to automagically cook up modes with slightly + * off refresh rates. + */ + if (abs(drm_mode_vrefresh(adjusted_mode) - drm_mode_vrefresh(fixed_mode)) > 1) { + drm_dbg_kms(connector->base.dev, + "[CONNECTOR:%d:%s] Requested mode vrefresh (%d Hz) does not match fixed mode vrefresh (%d Hz)\n", + connector->base.base.id, connector->base.name, + drm_mode_vrefresh(adjusted_mode), drm_mode_vrefresh(fixed_mode)); + + return -EINVAL; + } + drm_mode_copy(adjusted_mode, fixed_mode); drm_mode_set_crtcinfo(adjusted_mode, 0); + + return 0; } static bool is_downclock_mode(const struct drm_display_mode *downclock_mode, @@ -175,8 +200,8 @@ intel_panel_vbt_fixed_mode(struct intel_connector *connector) } /* adjusted_mode has been preset to be the panel's fixed mode */ -int intel_pch_panel_fitting(struct intel_crtc_state *crtc_state, - const struct drm_connector_state *conn_state) +static int pch_panel_fitting(struct intel_crtc_state *crtc_state, + const struct drm_connector_state *conn_state) { const struct drm_display_mode *adjusted_mode = &crtc_state->hw.adjusted_mode; @@ -381,8 +406,8 @@ static void i9xx_scale_aspect(struct intel_crtc_state *crtc_state, } } -int intel_gmch_panel_fitting(struct intel_crtc_state *crtc_state, - const struct drm_connector_state *conn_state) +static int gmch_panel_fitting(struct intel_crtc_state *crtc_state, + const struct drm_connector_state *conn_state) { struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); @@ -456,1783 +481,55 @@ out: return 0; } -/** - * scale - scale values from one range to another - * @source_val: value in range [@source_min..@source_max] - * @source_min: minimum legal value for @source_val - * @source_max: maximum legal value for @source_val - * @target_min: corresponding target value for @source_min - * @target_max: corresponding target value for @source_max - * - * Return @source_val in range [@source_min..@source_max] scaled to range - * [@target_min..@target_max]. - */ -static u32 scale(u32 source_val, - u32 source_min, u32 source_max, - u32 target_min, u32 target_max) -{ - u64 target_val; - - WARN_ON(source_min > source_max); - WARN_ON(target_min > target_max); - - /* defensive */ - source_val = clamp(source_val, source_min, source_max); - - /* avoid overflows */ - target_val = mul_u32_u32(source_val - source_min, - target_max - target_min); - target_val = DIV_ROUND_CLOSEST_ULL(target_val, source_max - source_min); - target_val += target_min; - - return target_val; -} - -/* Scale user_level in range [0..user_max] to [0..hw_max], clamping the result - * to [hw_min..hw_max]. */ -static u32 clamp_user_to_hw(struct intel_connector *connector, - u32 user_level, u32 user_max) -{ - struct intel_panel *panel = &connector->panel; - u32 hw_level; - - hw_level = scale(user_level, 0, user_max, 0, panel->backlight.max); - hw_level = clamp(hw_level, panel->backlight.min, panel->backlight.max); - - return hw_level; -} - -/* Scale hw_level in range [hw_min..hw_max] to [0..user_max]. */ -static u32 scale_hw_to_user(struct intel_connector *connector, - u32 hw_level, u32 user_max) -{ - struct intel_panel *panel = &connector->panel; - - return scale(hw_level, panel->backlight.min, panel->backlight.max, - 0, user_max); -} - -u32 intel_panel_invert_pwm_level(struct intel_connector *connector, u32 val) -{ - struct drm_i915_private *dev_priv = to_i915(connector->base.dev); - struct intel_panel *panel = &connector->panel; - - drm_WARN_ON(&dev_priv->drm, panel->backlight.pwm_level_max == 0); - - if (dev_priv->params.invert_brightness < 0) - return val; - - if (dev_priv->params.invert_brightness > 0 || - dev_priv->quirks & QUIRK_INVERT_BRIGHTNESS) { - return panel->backlight.pwm_level_max - val + panel->backlight.pwm_level_min; - } - - return val; -} - -void intel_panel_set_pwm_level(const struct drm_connector_state *conn_state, u32 val) -{ - struct intel_connector *connector = to_intel_connector(conn_state->connector); - struct drm_i915_private *i915 = to_i915(connector->base.dev); - struct intel_panel *panel = &connector->panel; - - drm_dbg_kms(&i915->drm, "set backlight PWM = %d\n", val); - panel->backlight.pwm_funcs->set(conn_state, val); -} - -u32 intel_panel_backlight_level_to_pwm(struct intel_connector *connector, u32 val) -{ - struct drm_i915_private *dev_priv = to_i915(connector->base.dev); - struct intel_panel *panel = &connector->panel; - - drm_WARN_ON_ONCE(&dev_priv->drm, - panel->backlight.max == 0 || panel->backlight.pwm_level_max == 0); - - val = scale(val, panel->backlight.min, panel->backlight.max, - panel->backlight.pwm_level_min, panel->backlight.pwm_level_max); - - return intel_panel_invert_pwm_level(connector, val); -} - -u32 intel_panel_backlight_level_from_pwm(struct intel_connector *connector, u32 val) -{ - struct drm_i915_private *dev_priv = to_i915(connector->base.dev); - struct intel_panel *panel = &connector->panel; - - drm_WARN_ON_ONCE(&dev_priv->drm, - panel->backlight.max == 0 || panel->backlight.pwm_level_max == 0); - - if (dev_priv->params.invert_brightness > 0 || - (dev_priv->params.invert_brightness == 0 && dev_priv->quirks & QUIRK_INVERT_BRIGHTNESS)) - val = panel->backlight.pwm_level_max - (val - panel->backlight.pwm_level_min); - - return scale(val, panel->backlight.pwm_level_min, panel->backlight.pwm_level_max, - panel->backlight.min, panel->backlight.max); -} - -static u32 lpt_get_backlight(struct intel_connector *connector, enum pipe unused) -{ - struct drm_i915_private *dev_priv = to_i915(connector->base.dev); - - return intel_de_read(dev_priv, BLC_PWM_PCH_CTL2) & BACKLIGHT_DUTY_CYCLE_MASK; -} - -static u32 pch_get_backlight(struct intel_connector *connector, enum pipe unused) -{ - struct drm_i915_private *dev_priv = to_i915(connector->base.dev); - - return intel_de_read(dev_priv, BLC_PWM_CPU_CTL) & BACKLIGHT_DUTY_CYCLE_MASK; -} - -static u32 i9xx_get_backlight(struct intel_connector *connector, enum pipe unused) -{ - struct drm_i915_private *dev_priv = to_i915(connector->base.dev); - struct intel_panel *panel = &connector->panel; - u32 val; - - val = intel_de_read(dev_priv, BLC_PWM_CTL) & BACKLIGHT_DUTY_CYCLE_MASK; - if (DISPLAY_VER(dev_priv) < 4) - val >>= 1; - - if (panel->backlight.combination_mode) { - u8 lbpc; - - pci_read_config_byte(to_pci_dev(dev_priv->drm.dev), LBPC, &lbpc); - val *= lbpc; - } - - return val; -} - -static u32 vlv_get_backlight(struct intel_connector *connector, enum pipe pipe) -{ - struct drm_i915_private *dev_priv = to_i915(connector->base.dev); - - if (drm_WARN_ON(&dev_priv->drm, pipe != PIPE_A && pipe != PIPE_B)) - return 0; - - return intel_de_read(dev_priv, VLV_BLC_PWM_CTL(pipe)) & BACKLIGHT_DUTY_CYCLE_MASK; -} - -static u32 bxt_get_backlight(struct intel_connector *connector, enum pipe unused) -{ - struct drm_i915_private *dev_priv = to_i915(connector->base.dev); - struct intel_panel *panel = &connector->panel; - - return intel_de_read(dev_priv, - BXT_BLC_PWM_DUTY(panel->backlight.controller)); -} - -static u32 ext_pwm_get_backlight(struct intel_connector *connector, enum pipe unused) -{ - struct intel_panel *panel = &connector->panel; - struct pwm_state state; - - pwm_get_state(panel->backlight.pwm, &state); - return pwm_get_relative_duty_cycle(&state, 100); -} - -static void lpt_set_backlight(const struct drm_connector_state *conn_state, u32 level) -{ - struct intel_connector *connector = to_intel_connector(conn_state->connector); - struct drm_i915_private *dev_priv = to_i915(connector->base.dev); - - u32 val = intel_de_read(dev_priv, BLC_PWM_PCH_CTL2) & ~BACKLIGHT_DUTY_CYCLE_MASK; - intel_de_write(dev_priv, BLC_PWM_PCH_CTL2, val | level); -} - -static void pch_set_backlight(const struct drm_connector_state *conn_state, u32 level) -{ - struct intel_connector *connector = to_intel_connector(conn_state->connector); - struct drm_i915_private *dev_priv = to_i915(connector->base.dev); - u32 tmp; - - tmp = intel_de_read(dev_priv, BLC_PWM_CPU_CTL) & ~BACKLIGHT_DUTY_CYCLE_MASK; - intel_de_write(dev_priv, BLC_PWM_CPU_CTL, tmp | level); -} - -static void i9xx_set_backlight(const struct drm_connector_state *conn_state, u32 level) -{ - struct intel_connector *connector = to_intel_connector(conn_state->connector); - struct drm_i915_private *dev_priv = to_i915(connector->base.dev); - struct intel_panel *panel = &connector->panel; - u32 tmp, mask; - - drm_WARN_ON(&dev_priv->drm, panel->backlight.pwm_level_max == 0); - - if (panel->backlight.combination_mode) { - u8 lbpc; - - lbpc = level * 0xfe / panel->backlight.pwm_level_max + 1; - level /= lbpc; - pci_write_config_byte(to_pci_dev(dev_priv->drm.dev), LBPC, lbpc); - } - - if (DISPLAY_VER(dev_priv) == 4) { - mask = BACKLIGHT_DUTY_CYCLE_MASK; - } else { - level <<= 1; - mask = BACKLIGHT_DUTY_CYCLE_MASK_PNV; - } - - tmp = intel_de_read(dev_priv, BLC_PWM_CTL) & ~mask; - intel_de_write(dev_priv, BLC_PWM_CTL, tmp | level); -} - -static void vlv_set_backlight(const struct drm_connector_state *conn_state, u32 level) -{ - struct intel_connector *connector = to_intel_connector(conn_state->connector); - struct drm_i915_private *dev_priv = to_i915(connector->base.dev); - enum pipe pipe = to_intel_crtc(conn_state->crtc)->pipe; - u32 tmp; - - tmp = intel_de_read(dev_priv, VLV_BLC_PWM_CTL(pipe)) & ~BACKLIGHT_DUTY_CYCLE_MASK; - intel_de_write(dev_priv, VLV_BLC_PWM_CTL(pipe), tmp | level); -} - -static void bxt_set_backlight(const struct drm_connector_state *conn_state, u32 level) -{ - struct intel_connector *connector = to_intel_connector(conn_state->connector); - struct drm_i915_private *dev_priv = to_i915(connector->base.dev); - struct intel_panel *panel = &connector->panel; - - intel_de_write(dev_priv, - BXT_BLC_PWM_DUTY(panel->backlight.controller), level); -} - -static void ext_pwm_set_backlight(const struct drm_connector_state *conn_state, u32 level) -{ - struct intel_panel *panel = &to_intel_connector(conn_state->connector)->panel; - - pwm_set_relative_duty_cycle(&panel->backlight.pwm_state, level, 100); - pwm_apply_state(panel->backlight.pwm, &panel->backlight.pwm_state); -} - -static void -intel_panel_actually_set_backlight(const struct drm_connector_state *conn_state, u32 level) -{ - struct intel_connector *connector = to_intel_connector(conn_state->connector); - struct drm_i915_private *i915 = to_i915(connector->base.dev); - struct intel_panel *panel = &connector->panel; - - drm_dbg_kms(&i915->drm, "set backlight level = %d\n", level); - - panel->backlight.funcs->set(conn_state, level); -} - -/* set backlight brightness to level in range [0..max], assuming hw min is - * respected. - */ -void intel_panel_set_backlight_acpi(const struct drm_connector_state *conn_state, - u32 user_level, u32 user_max) -{ - struct intel_connector *connector = to_intel_connector(conn_state->connector); - struct drm_i915_private *dev_priv = to_i915(connector->base.dev); - struct intel_panel *panel = &connector->panel; - u32 hw_level; - - /* - * Lack of crtc may occur during driver init because - * connection_mutex isn't held across the entire backlight - * setup + modeset readout, and the BIOS can issue the - * requests at any time. - */ - if (!panel->backlight.present || !conn_state->crtc) - return; - - mutex_lock(&dev_priv->backlight_lock); - - drm_WARN_ON(&dev_priv->drm, panel->backlight.max == 0); - - hw_level = clamp_user_to_hw(connector, user_level, user_max); - panel->backlight.level = hw_level; - - if (panel->backlight.device) - panel->backlight.device->props.brightness = - scale_hw_to_user(connector, - panel->backlight.level, - panel->backlight.device->props.max_brightness); - - if (panel->backlight.enabled) - intel_panel_actually_set_backlight(conn_state, hw_level); - - mutex_unlock(&dev_priv->backlight_lock); -} - -static void lpt_disable_backlight(const struct drm_connector_state *old_conn_state, u32 level) -{ - struct intel_connector *connector = to_intel_connector(old_conn_state->connector); - struct drm_i915_private *dev_priv = to_i915(connector->base.dev); - u32 tmp; - - intel_panel_set_pwm_level(old_conn_state, level); - - /* - * Although we don't support or enable CPU PWM with LPT/SPT based - * systems, it may have been enabled prior to loading the - * driver. Disable to avoid warnings on LCPLL disable. - * - * This needs rework if we need to add support for CPU PWM on PCH split - * platforms. - */ - tmp = intel_de_read(dev_priv, BLC_PWM_CPU_CTL2); - if (tmp & BLM_PWM_ENABLE) { - drm_dbg_kms(&dev_priv->drm, - "cpu backlight was enabled, disabling\n"); - intel_de_write(dev_priv, BLC_PWM_CPU_CTL2, - tmp & ~BLM_PWM_ENABLE); - } - - tmp = intel_de_read(dev_priv, BLC_PWM_PCH_CTL1); - intel_de_write(dev_priv, BLC_PWM_PCH_CTL1, tmp & ~BLM_PCH_PWM_ENABLE); -} - -static void pch_disable_backlight(const struct drm_connector_state *old_conn_state, u32 val) -{ - struct intel_connector *connector = to_intel_connector(old_conn_state->connector); - struct drm_i915_private *dev_priv = to_i915(connector->base.dev); - u32 tmp; - - intel_panel_set_pwm_level(old_conn_state, val); - - tmp = intel_de_read(dev_priv, BLC_PWM_CPU_CTL2); - intel_de_write(dev_priv, BLC_PWM_CPU_CTL2, tmp & ~BLM_PWM_ENABLE); - - tmp = intel_de_read(dev_priv, BLC_PWM_PCH_CTL1); - intel_de_write(dev_priv, BLC_PWM_PCH_CTL1, tmp & ~BLM_PCH_PWM_ENABLE); -} - -static void i9xx_disable_backlight(const struct drm_connector_state *old_conn_state, u32 val) -{ - intel_panel_set_pwm_level(old_conn_state, val); -} - -static void i965_disable_backlight(const struct drm_connector_state *old_conn_state, u32 val) -{ - struct drm_i915_private *dev_priv = to_i915(old_conn_state->connector->dev); - u32 tmp; - - intel_panel_set_pwm_level(old_conn_state, val); - - tmp = intel_de_read(dev_priv, BLC_PWM_CTL2); - intel_de_write(dev_priv, BLC_PWM_CTL2, tmp & ~BLM_PWM_ENABLE); -} - -static void vlv_disable_backlight(const struct drm_connector_state *old_conn_state, u32 val) -{ - struct intel_connector *connector = to_intel_connector(old_conn_state->connector); - struct drm_i915_private *dev_priv = to_i915(connector->base.dev); - enum pipe pipe = to_intel_crtc(old_conn_state->crtc)->pipe; - u32 tmp; - - intel_panel_set_pwm_level(old_conn_state, val); - - tmp = intel_de_read(dev_priv, VLV_BLC_PWM_CTL2(pipe)); - intel_de_write(dev_priv, VLV_BLC_PWM_CTL2(pipe), - tmp & ~BLM_PWM_ENABLE); -} - -static void bxt_disable_backlight(const struct drm_connector_state *old_conn_state, u32 val) -{ - struct intel_connector *connector = to_intel_connector(old_conn_state->connector); - struct drm_i915_private *dev_priv = to_i915(connector->base.dev); - struct intel_panel *panel = &connector->panel; - u32 tmp; - - intel_panel_set_pwm_level(old_conn_state, val); - - tmp = intel_de_read(dev_priv, - BXT_BLC_PWM_CTL(panel->backlight.controller)); - intel_de_write(dev_priv, BXT_BLC_PWM_CTL(panel->backlight.controller), - tmp & ~BXT_BLC_PWM_ENABLE); - - if (panel->backlight.controller == 1) { - val = intel_de_read(dev_priv, UTIL_PIN_CTL); - val &= ~UTIL_PIN_ENABLE; - intel_de_write(dev_priv, UTIL_PIN_CTL, val); - } -} - -static void cnp_disable_backlight(const struct drm_connector_state *old_conn_state, u32 val) -{ - struct intel_connector *connector = to_intel_connector(old_conn_state->connector); - struct drm_i915_private *dev_priv = to_i915(connector->base.dev); - struct intel_panel *panel = &connector->panel; - u32 tmp; - - intel_panel_set_pwm_level(old_conn_state, val); - - tmp = intel_de_read(dev_priv, - BXT_BLC_PWM_CTL(panel->backlight.controller)); - intel_de_write(dev_priv, BXT_BLC_PWM_CTL(panel->backlight.controller), - tmp & ~BXT_BLC_PWM_ENABLE); -} - -static void ext_pwm_disable_backlight(const struct drm_connector_state *old_conn_state, u32 level) -{ - struct intel_connector *connector = to_intel_connector(old_conn_state->connector); - struct intel_panel *panel = &connector->panel; - - panel->backlight.pwm_state.enabled = false; - pwm_apply_state(panel->backlight.pwm, &panel->backlight.pwm_state); -} - -void intel_panel_disable_backlight(const struct drm_connector_state *old_conn_state) -{ - struct intel_connector *connector = to_intel_connector(old_conn_state->connector); - struct drm_i915_private *dev_priv = to_i915(connector->base.dev); - struct intel_panel *panel = &connector->panel; - - if (!panel->backlight.present) - return; - - /* - * Do not disable backlight on the vga_switcheroo path. When switching - * away from i915, the other client may depend on i915 to handle the - * backlight. This will leave the backlight on unnecessarily when - * another client is not activated. - */ - if (dev_priv->drm.switch_power_state == DRM_SWITCH_POWER_CHANGING) { - drm_dbg_kms(&dev_priv->drm, - "Skipping backlight disable on vga switch\n"); - return; - } - - mutex_lock(&dev_priv->backlight_lock); - - if (panel->backlight.device) - panel->backlight.device->props.power = FB_BLANK_POWERDOWN; - panel->backlight.enabled = false; - panel->backlight.funcs->disable(old_conn_state, 0); - - mutex_unlock(&dev_priv->backlight_lock); -} - -static void lpt_enable_backlight(const struct intel_crtc_state *crtc_state, - const struct drm_connector_state *conn_state, u32 level) -{ - struct intel_connector *connector = to_intel_connector(conn_state->connector); - struct drm_i915_private *dev_priv = to_i915(connector->base.dev); - struct intel_panel *panel = &connector->panel; - u32 pch_ctl1, pch_ctl2, schicken; - - pch_ctl1 = intel_de_read(dev_priv, BLC_PWM_PCH_CTL1); - if (pch_ctl1 & BLM_PCH_PWM_ENABLE) { - drm_dbg_kms(&dev_priv->drm, "pch backlight already enabled\n"); - pch_ctl1 &= ~BLM_PCH_PWM_ENABLE; - intel_de_write(dev_priv, BLC_PWM_PCH_CTL1, pch_ctl1); - } - - if (HAS_PCH_LPT(dev_priv)) { - schicken = intel_de_read(dev_priv, SOUTH_CHICKEN2); - if (panel->backlight.alternate_pwm_increment) - schicken |= LPT_PWM_GRANULARITY; - else - schicken &= ~LPT_PWM_GRANULARITY; - intel_de_write(dev_priv, SOUTH_CHICKEN2, schicken); - } else { - schicken = intel_de_read(dev_priv, SOUTH_CHICKEN1); - if (panel->backlight.alternate_pwm_increment) - schicken |= SPT_PWM_GRANULARITY; - else - schicken &= ~SPT_PWM_GRANULARITY; - intel_de_write(dev_priv, SOUTH_CHICKEN1, schicken); - } - - pch_ctl2 = panel->backlight.pwm_level_max << 16; - intel_de_write(dev_priv, BLC_PWM_PCH_CTL2, pch_ctl2); - - pch_ctl1 = 0; - if (panel->backlight.active_low_pwm) - pch_ctl1 |= BLM_PCH_POLARITY; - - /* After LPT, override is the default. */ - if (HAS_PCH_LPT(dev_priv)) - pch_ctl1 |= BLM_PCH_OVERRIDE_ENABLE; - - intel_de_write(dev_priv, BLC_PWM_PCH_CTL1, pch_ctl1); - intel_de_posting_read(dev_priv, BLC_PWM_PCH_CTL1); - intel_de_write(dev_priv, BLC_PWM_PCH_CTL1, - pch_ctl1 | BLM_PCH_PWM_ENABLE); - - /* This won't stick until the above enable. */ - intel_panel_set_pwm_level(conn_state, level); -} - -static void pch_enable_backlight(const struct intel_crtc_state *crtc_state, - const struct drm_connector_state *conn_state, u32 level) -{ - struct intel_connector *connector = to_intel_connector(conn_state->connector); - struct drm_i915_private *dev_priv = to_i915(connector->base.dev); - struct intel_panel *panel = &connector->panel; - enum transcoder cpu_transcoder = crtc_state->cpu_transcoder; - u32 cpu_ctl2, pch_ctl1, pch_ctl2; - - cpu_ctl2 = intel_de_read(dev_priv, BLC_PWM_CPU_CTL2); - if (cpu_ctl2 & BLM_PWM_ENABLE) { - drm_dbg_kms(&dev_priv->drm, "cpu backlight already enabled\n"); - cpu_ctl2 &= ~BLM_PWM_ENABLE; - intel_de_write(dev_priv, BLC_PWM_CPU_CTL2, cpu_ctl2); - } - - pch_ctl1 = intel_de_read(dev_priv, BLC_PWM_PCH_CTL1); - if (pch_ctl1 & BLM_PCH_PWM_ENABLE) { - drm_dbg_kms(&dev_priv->drm, "pch backlight already enabled\n"); - pch_ctl1 &= ~BLM_PCH_PWM_ENABLE; - intel_de_write(dev_priv, BLC_PWM_PCH_CTL1, pch_ctl1); - } - - if (cpu_transcoder == TRANSCODER_EDP) - cpu_ctl2 = BLM_TRANSCODER_EDP; - else - cpu_ctl2 = BLM_PIPE(cpu_transcoder); - intel_de_write(dev_priv, BLC_PWM_CPU_CTL2, cpu_ctl2); - intel_de_posting_read(dev_priv, BLC_PWM_CPU_CTL2); - intel_de_write(dev_priv, BLC_PWM_CPU_CTL2, cpu_ctl2 | BLM_PWM_ENABLE); - - /* This won't stick until the above enable. */ - intel_panel_set_pwm_level(conn_state, level); - - pch_ctl2 = panel->backlight.pwm_level_max << 16; - intel_de_write(dev_priv, BLC_PWM_PCH_CTL2, pch_ctl2); - - pch_ctl1 = 0; - if (panel->backlight.active_low_pwm) - pch_ctl1 |= BLM_PCH_POLARITY; - - intel_de_write(dev_priv, BLC_PWM_PCH_CTL1, pch_ctl1); - intel_de_posting_read(dev_priv, BLC_PWM_PCH_CTL1); - intel_de_write(dev_priv, BLC_PWM_PCH_CTL1, - pch_ctl1 | BLM_PCH_PWM_ENABLE); -} - -static void i9xx_enable_backlight(const struct intel_crtc_state *crtc_state, - const struct drm_connector_state *conn_state, u32 level) -{ - struct intel_connector *connector = to_intel_connector(conn_state->connector); - struct drm_i915_private *dev_priv = to_i915(connector->base.dev); - struct intel_panel *panel = &connector->panel; - u32 ctl, freq; - - ctl = intel_de_read(dev_priv, BLC_PWM_CTL); - if (ctl & BACKLIGHT_DUTY_CYCLE_MASK_PNV) { - drm_dbg_kms(&dev_priv->drm, "backlight already enabled\n"); - intel_de_write(dev_priv, BLC_PWM_CTL, 0); - } - - freq = panel->backlight.pwm_level_max; - if (panel->backlight.combination_mode) - freq /= 0xff; - - ctl = freq << 17; - if (panel->backlight.combination_mode) - ctl |= BLM_LEGACY_MODE; - if (IS_PINEVIEW(dev_priv) && panel->backlight.active_low_pwm) - ctl |= BLM_POLARITY_PNV; - - intel_de_write(dev_priv, BLC_PWM_CTL, ctl); - intel_de_posting_read(dev_priv, BLC_PWM_CTL); - - /* XXX: combine this into above write? */ - intel_panel_set_pwm_level(conn_state, level); - - /* - * Needed to enable backlight on some 855gm models. BLC_HIST_CTL is - * 855gm only, but checking for gen2 is safe, as 855gm is the only gen2 - * that has backlight. - */ - if (DISPLAY_VER(dev_priv) == 2) - intel_de_write(dev_priv, BLC_HIST_CTL, BLM_HISTOGRAM_ENABLE); -} - -static void i965_enable_backlight(const struct intel_crtc_state *crtc_state, - const struct drm_connector_state *conn_state, u32 level) -{ - struct intel_connector *connector = to_intel_connector(conn_state->connector); - struct drm_i915_private *dev_priv = to_i915(connector->base.dev); - struct intel_panel *panel = &connector->panel; - enum pipe pipe = to_intel_crtc(conn_state->crtc)->pipe; - u32 ctl, ctl2, freq; - - ctl2 = intel_de_read(dev_priv, BLC_PWM_CTL2); - if (ctl2 & BLM_PWM_ENABLE) { - drm_dbg_kms(&dev_priv->drm, "backlight already enabled\n"); - ctl2 &= ~BLM_PWM_ENABLE; - intel_de_write(dev_priv, BLC_PWM_CTL2, ctl2); - } - - freq = panel->backlight.pwm_level_max; - if (panel->backlight.combination_mode) - freq /= 0xff; - - ctl = freq << 16; - intel_de_write(dev_priv, BLC_PWM_CTL, ctl); - - ctl2 = BLM_PIPE(pipe); - if (panel->backlight.combination_mode) - ctl2 |= BLM_COMBINATION_MODE; - if (panel->backlight.active_low_pwm) - ctl2 |= BLM_POLARITY_I965; - intel_de_write(dev_priv, BLC_PWM_CTL2, ctl2); - intel_de_posting_read(dev_priv, BLC_PWM_CTL2); - intel_de_write(dev_priv, BLC_PWM_CTL2, ctl2 | BLM_PWM_ENABLE); - - intel_panel_set_pwm_level(conn_state, level); -} - -static void vlv_enable_backlight(const struct intel_crtc_state *crtc_state, - const struct drm_connector_state *conn_state, u32 level) -{ - struct intel_connector *connector = to_intel_connector(conn_state->connector); - struct drm_i915_private *dev_priv = to_i915(connector->base.dev); - struct intel_panel *panel = &connector->panel; - enum pipe pipe = to_intel_crtc(crtc_state->uapi.crtc)->pipe; - u32 ctl, ctl2; - - ctl2 = intel_de_read(dev_priv, VLV_BLC_PWM_CTL2(pipe)); - if (ctl2 & BLM_PWM_ENABLE) { - drm_dbg_kms(&dev_priv->drm, "backlight already enabled\n"); - ctl2 &= ~BLM_PWM_ENABLE; - intel_de_write(dev_priv, VLV_BLC_PWM_CTL2(pipe), ctl2); - } - - ctl = panel->backlight.pwm_level_max << 16; - intel_de_write(dev_priv, VLV_BLC_PWM_CTL(pipe), ctl); - - /* XXX: combine this into above write? */ - intel_panel_set_pwm_level(conn_state, level); - - ctl2 = 0; - if (panel->backlight.active_low_pwm) - ctl2 |= BLM_POLARITY_I965; - intel_de_write(dev_priv, VLV_BLC_PWM_CTL2(pipe), ctl2); - intel_de_posting_read(dev_priv, VLV_BLC_PWM_CTL2(pipe)); - intel_de_write(dev_priv, VLV_BLC_PWM_CTL2(pipe), - ctl2 | BLM_PWM_ENABLE); -} - -static void bxt_enable_backlight(const struct intel_crtc_state *crtc_state, - const struct drm_connector_state *conn_state, u32 level) -{ - struct intel_connector *connector = to_intel_connector(conn_state->connector); - struct drm_i915_private *dev_priv = to_i915(connector->base.dev); - struct intel_panel *panel = &connector->panel; - enum pipe pipe = to_intel_crtc(crtc_state->uapi.crtc)->pipe; - u32 pwm_ctl, val; - - /* Controller 1 uses the utility pin. */ - if (panel->backlight.controller == 1) { - val = intel_de_read(dev_priv, UTIL_PIN_CTL); - if (val & UTIL_PIN_ENABLE) { - drm_dbg_kms(&dev_priv->drm, - "util pin already enabled\n"); - val &= ~UTIL_PIN_ENABLE; - intel_de_write(dev_priv, UTIL_PIN_CTL, val); - } - - val = 0; - if (panel->backlight.util_pin_active_low) - val |= UTIL_PIN_POLARITY; - intel_de_write(dev_priv, UTIL_PIN_CTL, - val | UTIL_PIN_PIPE(pipe) | UTIL_PIN_MODE_PWM | UTIL_PIN_ENABLE); - } - - pwm_ctl = intel_de_read(dev_priv, - BXT_BLC_PWM_CTL(panel->backlight.controller)); - if (pwm_ctl & BXT_BLC_PWM_ENABLE) { - drm_dbg_kms(&dev_priv->drm, "backlight already enabled\n"); - pwm_ctl &= ~BXT_BLC_PWM_ENABLE; - intel_de_write(dev_priv, - BXT_BLC_PWM_CTL(panel->backlight.controller), - pwm_ctl); - } - - intel_de_write(dev_priv, - BXT_BLC_PWM_FREQ(panel->backlight.controller), - panel->backlight.pwm_level_max); - - intel_panel_set_pwm_level(conn_state, level); - - pwm_ctl = 0; - if (panel->backlight.active_low_pwm) - pwm_ctl |= BXT_BLC_PWM_POLARITY; - - intel_de_write(dev_priv, BXT_BLC_PWM_CTL(panel->backlight.controller), - pwm_ctl); - intel_de_posting_read(dev_priv, - BXT_BLC_PWM_CTL(panel->backlight.controller)); - intel_de_write(dev_priv, BXT_BLC_PWM_CTL(panel->backlight.controller), - pwm_ctl | BXT_BLC_PWM_ENABLE); -} - -static void cnp_enable_backlight(const struct intel_crtc_state *crtc_state, - const struct drm_connector_state *conn_state, u32 level) -{ - struct intel_connector *connector = to_intel_connector(conn_state->connector); - struct drm_i915_private *dev_priv = to_i915(connector->base.dev); - struct intel_panel *panel = &connector->panel; - u32 pwm_ctl; - - pwm_ctl = intel_de_read(dev_priv, - BXT_BLC_PWM_CTL(panel->backlight.controller)); - if (pwm_ctl & BXT_BLC_PWM_ENABLE) { - drm_dbg_kms(&dev_priv->drm, "backlight already enabled\n"); - pwm_ctl &= ~BXT_BLC_PWM_ENABLE; - intel_de_write(dev_priv, - BXT_BLC_PWM_CTL(panel->backlight.controller), - pwm_ctl); - } - - intel_de_write(dev_priv, - BXT_BLC_PWM_FREQ(panel->backlight.controller), - panel->backlight.pwm_level_max); - - intel_panel_set_pwm_level(conn_state, level); - - pwm_ctl = 0; - if (panel->backlight.active_low_pwm) - pwm_ctl |= BXT_BLC_PWM_POLARITY; - - intel_de_write(dev_priv, BXT_BLC_PWM_CTL(panel->backlight.controller), - pwm_ctl); - intel_de_posting_read(dev_priv, - BXT_BLC_PWM_CTL(panel->backlight.controller)); - intel_de_write(dev_priv, BXT_BLC_PWM_CTL(panel->backlight.controller), - pwm_ctl | BXT_BLC_PWM_ENABLE); -} - -static void ext_pwm_enable_backlight(const struct intel_crtc_state *crtc_state, - const struct drm_connector_state *conn_state, u32 level) -{ - struct intel_connector *connector = to_intel_connector(conn_state->connector); - struct intel_panel *panel = &connector->panel; - - pwm_set_relative_duty_cycle(&panel->backlight.pwm_state, level, 100); - panel->backlight.pwm_state.enabled = true; - pwm_apply_state(panel->backlight.pwm, &panel->backlight.pwm_state); -} - -static void __intel_panel_enable_backlight(const struct intel_crtc_state *crtc_state, - const struct drm_connector_state *conn_state) -{ - struct intel_connector *connector = to_intel_connector(conn_state->connector); - struct intel_panel *panel = &connector->panel; - - WARN_ON(panel->backlight.max == 0); - - if (panel->backlight.level <= panel->backlight.min) { - panel->backlight.level = panel->backlight.max; - if (panel->backlight.device) - panel->backlight.device->props.brightness = - scale_hw_to_user(connector, - panel->backlight.level, - panel->backlight.device->props.max_brightness); - } - - panel->backlight.funcs->enable(crtc_state, conn_state, panel->backlight.level); - panel->backlight.enabled = true; - if (panel->backlight.device) - panel->backlight.device->props.power = FB_BLANK_UNBLANK; -} - -void intel_panel_enable_backlight(const struct intel_crtc_state *crtc_state, - const struct drm_connector_state *conn_state) -{ - struct intel_connector *connector = to_intel_connector(conn_state->connector); - struct drm_i915_private *dev_priv = to_i915(connector->base.dev); - struct intel_panel *panel = &connector->panel; - enum pipe pipe = to_intel_crtc(crtc_state->uapi.crtc)->pipe; - - if (!panel->backlight.present) - return; - - drm_dbg_kms(&dev_priv->drm, "pipe %c\n", pipe_name(pipe)); - - mutex_lock(&dev_priv->backlight_lock); - - __intel_panel_enable_backlight(crtc_state, conn_state); - - mutex_unlock(&dev_priv->backlight_lock); -} - -#if IS_ENABLED(CONFIG_BACKLIGHT_CLASS_DEVICE) -static u32 intel_panel_get_backlight(struct intel_connector *connector) -{ - struct drm_i915_private *dev_priv = to_i915(connector->base.dev); - struct intel_panel *panel = &connector->panel; - u32 val = 0; - - mutex_lock(&dev_priv->backlight_lock); - - if (panel->backlight.enabled) - val = panel->backlight.funcs->get(connector, intel_connector_get_pipe(connector)); - - mutex_unlock(&dev_priv->backlight_lock); - - drm_dbg_kms(&dev_priv->drm, "get backlight PWM = %d\n", val); - return val; -} - -/* Scale user_level in range [0..user_max] to [hw_min..hw_max]. */ -static u32 scale_user_to_hw(struct intel_connector *connector, - u32 user_level, u32 user_max) -{ - struct intel_panel *panel = &connector->panel; - - return scale(user_level, 0, user_max, - panel->backlight.min, panel->backlight.max); -} - -/* set backlight brightness to level in range [0..max], scaling wrt hw min */ -static void intel_panel_set_backlight(const struct drm_connector_state *conn_state, - u32 user_level, u32 user_max) -{ - struct intel_connector *connector = to_intel_connector(conn_state->connector); - struct drm_i915_private *dev_priv = to_i915(connector->base.dev); - struct intel_panel *panel = &connector->panel; - u32 hw_level; - - if (!panel->backlight.present) - return; - - mutex_lock(&dev_priv->backlight_lock); - - drm_WARN_ON(&dev_priv->drm, panel->backlight.max == 0); - - hw_level = scale_user_to_hw(connector, user_level, user_max); - panel->backlight.level = hw_level; - - if (panel->backlight.enabled) - intel_panel_actually_set_backlight(conn_state, hw_level); - - mutex_unlock(&dev_priv->backlight_lock); -} - -static int intel_backlight_device_update_status(struct backlight_device *bd) -{ - struct intel_connector *connector = bl_get_data(bd); - struct intel_panel *panel = &connector->panel; - struct drm_device *dev = connector->base.dev; - - drm_modeset_lock(&dev->mode_config.connection_mutex, NULL); - DRM_DEBUG_KMS("updating intel_backlight, brightness=%d/%d\n", - bd->props.brightness, bd->props.max_brightness); - intel_panel_set_backlight(connector->base.state, bd->props.brightness, - bd->props.max_brightness); - - /* - * Allow flipping bl_power as a sub-state of enabled. Sadly the - * backlight class device does not make it easy to to differentiate - * between callbacks for brightness and bl_power, so our backlight_power - * callback needs to take this into account. - */ - if (panel->backlight.enabled) { - if (panel->backlight.power) { - bool enable = bd->props.power == FB_BLANK_UNBLANK && - bd->props.brightness != 0; - panel->backlight.power(connector, enable); - } - } else { - bd->props.power = FB_BLANK_POWERDOWN; - } - - drm_modeset_unlock(&dev->mode_config.connection_mutex); - return 0; -} - -static int intel_backlight_device_get_brightness(struct backlight_device *bd) -{ - struct intel_connector *connector = bl_get_data(bd); - struct drm_device *dev = connector->base.dev; - struct drm_i915_private *dev_priv = to_i915(dev); - intel_wakeref_t wakeref; - int ret = 0; - - with_intel_runtime_pm(&dev_priv->runtime_pm, wakeref) { - u32 hw_level; - - drm_modeset_lock(&dev->mode_config.connection_mutex, NULL); - - hw_level = intel_panel_get_backlight(connector); - ret = scale_hw_to_user(connector, - hw_level, bd->props.max_brightness); - - drm_modeset_unlock(&dev->mode_config.connection_mutex); - } - - return ret; -} - -static const struct backlight_ops intel_backlight_device_ops = { - .update_status = intel_backlight_device_update_status, - .get_brightness = intel_backlight_device_get_brightness, -}; - -int intel_backlight_device_register(struct intel_connector *connector) -{ - struct drm_i915_private *i915 = to_i915(connector->base.dev); - struct intel_panel *panel = &connector->panel; - struct backlight_properties props; - struct backlight_device *bd; - const char *name; - int ret = 0; - - if (WARN_ON(panel->backlight.device)) - return -ENODEV; - - if (!panel->backlight.present) - return 0; - - WARN_ON(panel->backlight.max == 0); - - memset(&props, 0, sizeof(props)); - props.type = BACKLIGHT_RAW; - - /* - * Note: Everything should work even if the backlight device max - * presented to the userspace is arbitrarily chosen. - */ - props.max_brightness = panel->backlight.max; - props.brightness = scale_hw_to_user(connector, - panel->backlight.level, - props.max_brightness); - - if (panel->backlight.enabled) - props.power = FB_BLANK_UNBLANK; - else - props.power = FB_BLANK_POWERDOWN; - - name = kstrdup("intel_backlight", GFP_KERNEL); - if (!name) - return -ENOMEM; - - bd = backlight_device_register(name, connector->base.kdev, connector, - &intel_backlight_device_ops, &props); - - /* - * Using the same name independent of the drm device or connector - * prevents registration of multiple backlight devices in the - * driver. However, we need to use the default name for backward - * compatibility. Use unique names for subsequent backlight devices as a - * fallback when the default name already exists. - */ - if (IS_ERR(bd) && PTR_ERR(bd) == -EEXIST) { - kfree(name); - name = kasprintf(GFP_KERNEL, "card%d-%s-backlight", - i915->drm.primary->index, connector->base.name); - if (!name) - return -ENOMEM; - - bd = backlight_device_register(name, connector->base.kdev, connector, - &intel_backlight_device_ops, &props); - } - - if (IS_ERR(bd)) { - drm_err(&i915->drm, - "[CONNECTOR:%d:%s] backlight device %s register failed: %ld\n", - connector->base.base.id, connector->base.name, name, PTR_ERR(bd)); - ret = PTR_ERR(bd); - goto out; - } - - panel->backlight.device = bd; - - drm_dbg_kms(&i915->drm, - "[CONNECTOR:%d:%s] backlight device %s registered\n", - connector->base.base.id, connector->base.name, name); - -out: - kfree(name); - - return ret; -} - -void intel_backlight_device_unregister(struct intel_connector *connector) -{ - struct intel_panel *panel = &connector->panel; - - if (panel->backlight.device) { - backlight_device_unregister(panel->backlight.device); - panel->backlight.device = NULL; - } -} -#endif /* CONFIG_BACKLIGHT_CLASS_DEVICE */ - -/* - * CNP: PWM clock frequency is 19.2 MHz or 24 MHz. - * PWM increment = 1 - */ -static u32 cnp_hz_to_pwm(struct intel_connector *connector, u32 pwm_freq_hz) -{ - struct drm_i915_private *dev_priv = to_i915(connector->base.dev); - - return DIV_ROUND_CLOSEST(KHz(RUNTIME_INFO(dev_priv)->rawclk_freq), - pwm_freq_hz); -} - -/* - * BXT: PWM clock frequency = 19.2 MHz. - */ -static u32 bxt_hz_to_pwm(struct intel_connector *connector, u32 pwm_freq_hz) -{ - return DIV_ROUND_CLOSEST(KHz(19200), pwm_freq_hz); -} - -/* - * SPT: This value represents the period of the PWM stream in clock periods - * multiplied by 16 (default increment) or 128 (alternate increment selected in - * SCHICKEN_1 bit 0). PWM clock is 24 MHz. - */ -static u32 spt_hz_to_pwm(struct intel_connector *connector, u32 pwm_freq_hz) -{ - struct intel_panel *panel = &connector->panel; - u32 mul; - - if (panel->backlight.alternate_pwm_increment) - mul = 128; - else - mul = 16; - - return DIV_ROUND_CLOSEST(MHz(24), pwm_freq_hz * mul); -} - -/* - * LPT: This value represents the period of the PWM stream in clock periods - * multiplied by 128 (default increment) or 16 (alternate increment, selected in - * LPT SOUTH_CHICKEN2 register bit 5). - */ -static u32 lpt_hz_to_pwm(struct intel_connector *connector, u32 pwm_freq_hz) -{ - struct drm_i915_private *dev_priv = to_i915(connector->base.dev); - struct intel_panel *panel = &connector->panel; - u32 mul, clock; - - if (panel->backlight.alternate_pwm_increment) - mul = 16; - else - mul = 128; - - if (HAS_PCH_LPT_H(dev_priv)) - clock = MHz(135); /* LPT:H */ - else - clock = MHz(24); /* LPT:LP */ - - return DIV_ROUND_CLOSEST(clock, pwm_freq_hz * mul); -} - -/* - * ILK/SNB/IVB: This value represents the period of the PWM stream in PCH - * display raw clocks multiplied by 128. - */ -static u32 pch_hz_to_pwm(struct intel_connector *connector, u32 pwm_freq_hz) -{ - struct drm_i915_private *dev_priv = to_i915(connector->base.dev); - - return DIV_ROUND_CLOSEST(KHz(RUNTIME_INFO(dev_priv)->rawclk_freq), - pwm_freq_hz * 128); -} - -/* - * Gen2: This field determines the number of time base events (display core - * clock frequency/32) in total for a complete cycle of modulated backlight - * control. - * - * Gen3: A time base event equals the display core clock ([DevPNV] HRAW clock) - * divided by 32. - */ -static u32 i9xx_hz_to_pwm(struct intel_connector *connector, u32 pwm_freq_hz) -{ - struct drm_i915_private *dev_priv = to_i915(connector->base.dev); - int clock; - - if (IS_PINEVIEW(dev_priv)) - clock = KHz(RUNTIME_INFO(dev_priv)->rawclk_freq); - else - clock = KHz(dev_priv->cdclk.hw.cdclk); - - return DIV_ROUND_CLOSEST(clock, pwm_freq_hz * 32); -} - -/* - * Gen4: This value represents the period of the PWM stream in display core - * clocks ([DevCTG] HRAW clocks) multiplied by 128. - * - */ -static u32 i965_hz_to_pwm(struct intel_connector *connector, u32 pwm_freq_hz) +int intel_panel_fitting(struct intel_crtc_state *crtc_state, + const struct drm_connector_state *conn_state) { - struct drm_i915_private *dev_priv = to_i915(connector->base.dev); - int clock; - - if (IS_G4X(dev_priv)) - clock = KHz(RUNTIME_INFO(dev_priv)->rawclk_freq); - else - clock = KHz(dev_priv->cdclk.hw.cdclk); - - return DIV_ROUND_CLOSEST(clock, pwm_freq_hz * 128); -} - -/* - * VLV: This value represents the period of the PWM stream in display core - * clocks ([DevCTG] 200MHz HRAW clocks) multiplied by 128 or 25MHz S0IX clocks - * multiplied by 16. CHV uses a 19.2MHz S0IX clock. - */ -static u32 vlv_hz_to_pwm(struct intel_connector *connector, u32 pwm_freq_hz) -{ - struct drm_i915_private *dev_priv = to_i915(connector->base.dev); - int mul, clock; - - if ((intel_de_read(dev_priv, CBR1_VLV) & CBR_PWM_CLOCK_MUX_SELECT) == 0) { - if (IS_CHERRYVIEW(dev_priv)) - clock = KHz(19200); - else - clock = MHz(25); - mul = 16; - } else { - clock = KHz(RUNTIME_INFO(dev_priv)->rawclk_freq); - mul = 128; - } - - return DIV_ROUND_CLOSEST(clock, pwm_freq_hz * mul); -} - -static u16 get_vbt_pwm_freq(struct drm_i915_private *dev_priv) -{ - u16 pwm_freq_hz = dev_priv->vbt.backlight.pwm_freq_hz; - - if (pwm_freq_hz) { - drm_dbg_kms(&dev_priv->drm, - "VBT defined backlight frequency %u Hz\n", - pwm_freq_hz); - } else { - pwm_freq_hz = 200; - drm_dbg_kms(&dev_priv->drm, - "default backlight frequency %u Hz\n", - pwm_freq_hz); - } - - return pwm_freq_hz; -} - -static u32 get_backlight_max_vbt(struct intel_connector *connector) -{ - struct drm_i915_private *dev_priv = to_i915(connector->base.dev); - struct intel_panel *panel = &connector->panel; - u16 pwm_freq_hz = get_vbt_pwm_freq(dev_priv); - u32 pwm; - - if (!panel->backlight.pwm_funcs->hz_to_pwm) { - drm_dbg_kms(&dev_priv->drm, - "backlight frequency conversion not supported\n"); - return 0; - } - - pwm = panel->backlight.pwm_funcs->hz_to_pwm(connector, pwm_freq_hz); - if (!pwm) { - drm_dbg_kms(&dev_priv->drm, - "backlight frequency conversion failed\n"); - return 0; - } - - return pwm; -} - -/* - * Note: The setup hooks can't assume pipe is set! - */ -static u32 get_backlight_min_vbt(struct intel_connector *connector) -{ - struct drm_i915_private *dev_priv = to_i915(connector->base.dev); - struct intel_panel *panel = &connector->panel; - int min; - - drm_WARN_ON(&dev_priv->drm, panel->backlight.pwm_level_max == 0); - - /* - * XXX: If the vbt value is 255, it makes min equal to max, which leads - * to problems. There are such machines out there. Either our - * interpretation is wrong or the vbt has bogus data. Or both. Safeguard - * against this by letting the minimum be at most (arbitrarily chosen) - * 25% of the max. - */ - min = clamp_t(int, dev_priv->vbt.backlight.min_brightness, 0, 64); - if (min != dev_priv->vbt.backlight.min_brightness) { - drm_dbg_kms(&dev_priv->drm, - "clamping VBT min backlight %d/255 to %d/255\n", - dev_priv->vbt.backlight.min_brightness, min); - } - - /* vbt value is a coefficient in range [0..255] */ - return scale(min, 0, 255, 0, panel->backlight.pwm_level_max); -} - -static int lpt_setup_backlight(struct intel_connector *connector, enum pipe unused) -{ - struct drm_i915_private *dev_priv = to_i915(connector->base.dev); - struct intel_panel *panel = &connector->panel; - u32 cpu_ctl2, pch_ctl1, pch_ctl2, val; - bool alt, cpu_mode; + struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); + struct drm_i915_private *i915 = to_i915(crtc->base.dev); - if (HAS_PCH_LPT(dev_priv)) - alt = intel_de_read(dev_priv, SOUTH_CHICKEN2) & LPT_PWM_GRANULARITY; + if (HAS_GMCH(i915)) + return gmch_panel_fitting(crtc_state, conn_state); else - alt = intel_de_read(dev_priv, SOUTH_CHICKEN1) & SPT_PWM_GRANULARITY; - panel->backlight.alternate_pwm_increment = alt; - - pch_ctl1 = intel_de_read(dev_priv, BLC_PWM_PCH_CTL1); - panel->backlight.active_low_pwm = pch_ctl1 & BLM_PCH_POLARITY; - - pch_ctl2 = intel_de_read(dev_priv, BLC_PWM_PCH_CTL2); - panel->backlight.pwm_level_max = pch_ctl2 >> 16; - - cpu_ctl2 = intel_de_read(dev_priv, BLC_PWM_CPU_CTL2); - - if (!panel->backlight.pwm_level_max) - panel->backlight.pwm_level_max = get_backlight_max_vbt(connector); - - if (!panel->backlight.pwm_level_max) - return -ENODEV; - - panel->backlight.pwm_level_min = get_backlight_min_vbt(connector); - - panel->backlight.pwm_enabled = pch_ctl1 & BLM_PCH_PWM_ENABLE; - - cpu_mode = panel->backlight.pwm_enabled && HAS_PCH_LPT(dev_priv) && - !(pch_ctl1 & BLM_PCH_OVERRIDE_ENABLE) && - (cpu_ctl2 & BLM_PWM_ENABLE); - - if (cpu_mode) { - val = pch_get_backlight(connector, unused); - - drm_dbg_kms(&dev_priv->drm, - "CPU backlight register was enabled, switching to PCH override\n"); - - /* Write converted CPU PWM value to PCH override register */ - lpt_set_backlight(connector->base.state, val); - intel_de_write(dev_priv, BLC_PWM_PCH_CTL1, - pch_ctl1 | BLM_PCH_OVERRIDE_ENABLE); - - intel_de_write(dev_priv, BLC_PWM_CPU_CTL2, - cpu_ctl2 & ~BLM_PWM_ENABLE); - } - - return 0; -} - -static int pch_setup_backlight(struct intel_connector *connector, enum pipe unused) -{ - struct drm_i915_private *dev_priv = to_i915(connector->base.dev); - struct intel_panel *panel = &connector->panel; - u32 cpu_ctl2, pch_ctl1, pch_ctl2; - - pch_ctl1 = intel_de_read(dev_priv, BLC_PWM_PCH_CTL1); - panel->backlight.active_low_pwm = pch_ctl1 & BLM_PCH_POLARITY; - - pch_ctl2 = intel_de_read(dev_priv, BLC_PWM_PCH_CTL2); - panel->backlight.pwm_level_max = pch_ctl2 >> 16; - - if (!panel->backlight.pwm_level_max) - panel->backlight.pwm_level_max = get_backlight_max_vbt(connector); - - if (!panel->backlight.pwm_level_max) - return -ENODEV; - - panel->backlight.pwm_level_min = get_backlight_min_vbt(connector); - - cpu_ctl2 = intel_de_read(dev_priv, BLC_PWM_CPU_CTL2); - panel->backlight.pwm_enabled = (cpu_ctl2 & BLM_PWM_ENABLE) && - (pch_ctl1 & BLM_PCH_PWM_ENABLE); - - return 0; -} - -static int i9xx_setup_backlight(struct intel_connector *connector, enum pipe unused) -{ - struct drm_i915_private *dev_priv = to_i915(connector->base.dev); - struct intel_panel *panel = &connector->panel; - u32 ctl, val; - - ctl = intel_de_read(dev_priv, BLC_PWM_CTL); - - if (DISPLAY_VER(dev_priv) == 2 || IS_I915GM(dev_priv) || IS_I945GM(dev_priv)) - panel->backlight.combination_mode = ctl & BLM_LEGACY_MODE; - - if (IS_PINEVIEW(dev_priv)) - panel->backlight.active_low_pwm = ctl & BLM_POLARITY_PNV; - - panel->backlight.pwm_level_max = ctl >> 17; - - if (!panel->backlight.pwm_level_max) { - panel->backlight.pwm_level_max = get_backlight_max_vbt(connector); - panel->backlight.pwm_level_max >>= 1; - } - - if (!panel->backlight.pwm_level_max) - return -ENODEV; - - if (panel->backlight.combination_mode) - panel->backlight.pwm_level_max *= 0xff; - - panel->backlight.pwm_level_min = get_backlight_min_vbt(connector); - - val = i9xx_get_backlight(connector, unused); - val = intel_panel_invert_pwm_level(connector, val); - val = clamp(val, panel->backlight.pwm_level_min, panel->backlight.pwm_level_max); - - panel->backlight.pwm_enabled = val != 0; - - return 0; -} - -static int i965_setup_backlight(struct intel_connector *connector, enum pipe unused) -{ - struct drm_i915_private *dev_priv = to_i915(connector->base.dev); - struct intel_panel *panel = &connector->panel; - u32 ctl, ctl2; - - ctl2 = intel_de_read(dev_priv, BLC_PWM_CTL2); - panel->backlight.combination_mode = ctl2 & BLM_COMBINATION_MODE; - panel->backlight.active_low_pwm = ctl2 & BLM_POLARITY_I965; - - ctl = intel_de_read(dev_priv, BLC_PWM_CTL); - panel->backlight.pwm_level_max = ctl >> 16; - - if (!panel->backlight.pwm_level_max) - panel->backlight.pwm_level_max = get_backlight_max_vbt(connector); - - if (!panel->backlight.pwm_level_max) - return -ENODEV; - - if (panel->backlight.combination_mode) - panel->backlight.pwm_level_max *= 0xff; - - panel->backlight.pwm_level_min = get_backlight_min_vbt(connector); - - panel->backlight.pwm_enabled = ctl2 & BLM_PWM_ENABLE; - - return 0; -} - -static int vlv_setup_backlight(struct intel_connector *connector, enum pipe pipe) -{ - struct drm_i915_private *dev_priv = to_i915(connector->base.dev); - struct intel_panel *panel = &connector->panel; - u32 ctl, ctl2; - - if (drm_WARN_ON(&dev_priv->drm, pipe != PIPE_A && pipe != PIPE_B)) - return -ENODEV; - - ctl2 = intel_de_read(dev_priv, VLV_BLC_PWM_CTL2(pipe)); - panel->backlight.active_low_pwm = ctl2 & BLM_POLARITY_I965; - - ctl = intel_de_read(dev_priv, VLV_BLC_PWM_CTL(pipe)); - panel->backlight.pwm_level_max = ctl >> 16; - - if (!panel->backlight.pwm_level_max) - panel->backlight.pwm_level_max = get_backlight_max_vbt(connector); - - if (!panel->backlight.pwm_level_max) - return -ENODEV; - - panel->backlight.pwm_level_min = get_backlight_min_vbt(connector); - - panel->backlight.pwm_enabled = ctl2 & BLM_PWM_ENABLE; - - return 0; -} - -static int -bxt_setup_backlight(struct intel_connector *connector, enum pipe unused) -{ - struct drm_i915_private *dev_priv = to_i915(connector->base.dev); - struct intel_panel *panel = &connector->panel; - u32 pwm_ctl, val; - - panel->backlight.controller = dev_priv->vbt.backlight.controller; - - pwm_ctl = intel_de_read(dev_priv, - BXT_BLC_PWM_CTL(panel->backlight.controller)); - - /* Controller 1 uses the utility pin. */ - if (panel->backlight.controller == 1) { - val = intel_de_read(dev_priv, UTIL_PIN_CTL); - panel->backlight.util_pin_active_low = - val & UTIL_PIN_POLARITY; - } - - panel->backlight.active_low_pwm = pwm_ctl & BXT_BLC_PWM_POLARITY; - panel->backlight.pwm_level_max = - intel_de_read(dev_priv, BXT_BLC_PWM_FREQ(panel->backlight.controller)); - - if (!panel->backlight.pwm_level_max) - panel->backlight.pwm_level_max = get_backlight_max_vbt(connector); - - if (!panel->backlight.pwm_level_max) - return -ENODEV; - - panel->backlight.pwm_level_min = get_backlight_min_vbt(connector); - - panel->backlight.pwm_enabled = pwm_ctl & BXT_BLC_PWM_ENABLE; - - return 0; -} - -static int -cnp_setup_backlight(struct intel_connector *connector, enum pipe unused) -{ - struct drm_i915_private *dev_priv = to_i915(connector->base.dev); - struct intel_panel *panel = &connector->panel; - u32 pwm_ctl; - - /* - * CNP has the BXT implementation of backlight, but with only one - * controller. TODO: ICP has multiple controllers but we only use - * controller 0 for now. - */ - panel->backlight.controller = 0; - - pwm_ctl = intel_de_read(dev_priv, - BXT_BLC_PWM_CTL(panel->backlight.controller)); - - panel->backlight.active_low_pwm = pwm_ctl & BXT_BLC_PWM_POLARITY; - panel->backlight.pwm_level_max = - intel_de_read(dev_priv, BXT_BLC_PWM_FREQ(panel->backlight.controller)); - - if (!panel->backlight.pwm_level_max) - panel->backlight.pwm_level_max = get_backlight_max_vbt(connector); - - if (!panel->backlight.pwm_level_max) - return -ENODEV; - - panel->backlight.pwm_level_min = get_backlight_min_vbt(connector); - - panel->backlight.pwm_enabled = pwm_ctl & BXT_BLC_PWM_ENABLE; - - return 0; -} - -static int ext_pwm_setup_backlight(struct intel_connector *connector, - enum pipe pipe) -{ - struct drm_device *dev = connector->base.dev; - struct drm_i915_private *dev_priv = to_i915(dev); - struct intel_panel *panel = &connector->panel; - const char *desc; - u32 level; - - /* Get the right PWM chip for DSI backlight according to VBT */ - if (dev_priv->vbt.dsi.config->pwm_blc == PPS_BLC_PMIC) { - panel->backlight.pwm = pwm_get(dev->dev, "pwm_pmic_backlight"); - desc = "PMIC"; - } else { - panel->backlight.pwm = pwm_get(dev->dev, "pwm_soc_backlight"); - desc = "SoC"; - } - - if (IS_ERR(panel->backlight.pwm)) { - drm_err(&dev_priv->drm, "Failed to get the %s PWM chip\n", - desc); - panel->backlight.pwm = NULL; - return -ENODEV; - } - - panel->backlight.pwm_level_max = 100; /* 100% */ - panel->backlight.pwm_level_min = get_backlight_min_vbt(connector); - - if (pwm_is_enabled(panel->backlight.pwm)) { - /* PWM is already enabled, use existing settings */ - pwm_get_state(panel->backlight.pwm, &panel->backlight.pwm_state); - - level = pwm_get_relative_duty_cycle(&panel->backlight.pwm_state, - 100); - level = intel_panel_invert_pwm_level(connector, level); - panel->backlight.pwm_enabled = true; - - drm_dbg_kms(&dev_priv->drm, "PWM already enabled at freq %ld, VBT freq %d, level %d\n", - NSEC_PER_SEC / (unsigned long)panel->backlight.pwm_state.period, - get_vbt_pwm_freq(dev_priv), level); - } else { - /* Set period from VBT frequency, leave other settings at 0. */ - panel->backlight.pwm_state.period = - NSEC_PER_SEC / get_vbt_pwm_freq(dev_priv); - } - - drm_info(&dev_priv->drm, "Using %s PWM for LCD backlight control\n", - desc); - return 0; -} - -static void intel_pwm_set_backlight(const struct drm_connector_state *conn_state, u32 level) -{ - struct intel_connector *connector = to_intel_connector(conn_state->connector); - struct intel_panel *panel = &connector->panel; - - panel->backlight.pwm_funcs->set(conn_state, - intel_panel_invert_pwm_level(connector, level)); + return pch_panel_fitting(crtc_state, conn_state); } -static u32 intel_pwm_get_backlight(struct intel_connector *connector, enum pipe pipe) -{ - struct intel_panel *panel = &connector->panel; - - return intel_panel_invert_pwm_level(connector, - panel->backlight.pwm_funcs->get(connector, pipe)); -} - -static void intel_pwm_enable_backlight(const struct intel_crtc_state *crtc_state, - const struct drm_connector_state *conn_state, u32 level) -{ - struct intel_connector *connector = to_intel_connector(conn_state->connector); - struct intel_panel *panel = &connector->panel; - - panel->backlight.pwm_funcs->enable(crtc_state, conn_state, - intel_panel_invert_pwm_level(connector, level)); -} - -static void intel_pwm_disable_backlight(const struct drm_connector_state *conn_state, u32 level) -{ - struct intel_connector *connector = to_intel_connector(conn_state->connector); - struct intel_panel *panel = &connector->panel; - - panel->backlight.pwm_funcs->disable(conn_state, - intel_panel_invert_pwm_level(connector, level)); -} - -static int intel_pwm_setup_backlight(struct intel_connector *connector, enum pipe pipe) -{ - struct intel_panel *panel = &connector->panel; - int ret = panel->backlight.pwm_funcs->setup(connector, pipe); - - if (ret < 0) - return ret; - - panel->backlight.min = panel->backlight.pwm_level_min; - panel->backlight.max = panel->backlight.pwm_level_max; - panel->backlight.level = intel_pwm_get_backlight(connector, pipe); - panel->backlight.enabled = panel->backlight.pwm_enabled; - - return 0; -} - -void intel_panel_update_backlight(struct intel_atomic_state *state, - struct intel_encoder *encoder, - const struct intel_crtc_state *crtc_state, - const struct drm_connector_state *conn_state) -{ - struct intel_connector *connector = to_intel_connector(conn_state->connector); - struct drm_i915_private *dev_priv = to_i915(connector->base.dev); - struct intel_panel *panel = &connector->panel; - - if (!panel->backlight.present) - return; - - mutex_lock(&dev_priv->backlight_lock); - if (!panel->backlight.enabled) - __intel_panel_enable_backlight(crtc_state, conn_state); - - mutex_unlock(&dev_priv->backlight_lock); -} - -int intel_panel_setup_backlight(struct drm_connector *connector, enum pipe pipe) +enum drm_connector_status +intel_panel_detect(struct drm_connector *connector, bool force) { - struct drm_i915_private *dev_priv = to_i915(connector->dev); - struct intel_connector *intel_connector = to_intel_connector(connector); - struct intel_panel *panel = &intel_connector->panel; - int ret; - - if (!dev_priv->vbt.backlight.present) { - if (dev_priv->quirks & QUIRK_BACKLIGHT_PRESENT) { - drm_dbg_kms(&dev_priv->drm, - "no backlight present per VBT, but present per quirk\n"); - } else { - drm_dbg_kms(&dev_priv->drm, - "no backlight present per VBT\n"); - return 0; - } - } - - /* ensure intel_panel has been initialized first */ - if (drm_WARN_ON(&dev_priv->drm, !panel->backlight.funcs)) - return -ENODEV; - - /* set level and max in panel struct */ - mutex_lock(&dev_priv->backlight_lock); - ret = panel->backlight.funcs->setup(intel_connector, pipe); - mutex_unlock(&dev_priv->backlight_lock); - - if (ret) { - drm_dbg_kms(&dev_priv->drm, - "failed to setup backlight for connector %s\n", - connector->name); - return ret; - } - - panel->backlight.present = true; - - drm_dbg_kms(&dev_priv->drm, - "Connector %s backlight initialized, %s, brightness %u/%u\n", - connector->name, - enableddisabled(panel->backlight.enabled), - panel->backlight.level, panel->backlight.max); - - return 0; -} + struct drm_i915_private *i915 = to_i915(connector->dev); -static void intel_panel_destroy_backlight(struct intel_panel *panel) -{ - /* dispose of the pwm */ - if (panel->backlight.pwm) - pwm_put(panel->backlight.pwm); + if (!INTEL_DISPLAY_ENABLED(i915)) + return connector_status_disconnected; - panel->backlight.present = false; + return connector_status_connected; } -static const struct intel_panel_bl_funcs bxt_pwm_funcs = { - .setup = bxt_setup_backlight, - .enable = bxt_enable_backlight, - .disable = bxt_disable_backlight, - .set = bxt_set_backlight, - .get = bxt_get_backlight, - .hz_to_pwm = bxt_hz_to_pwm, -}; - -static const struct intel_panel_bl_funcs cnp_pwm_funcs = { - .setup = cnp_setup_backlight, - .enable = cnp_enable_backlight, - .disable = cnp_disable_backlight, - .set = bxt_set_backlight, - .get = bxt_get_backlight, - .hz_to_pwm = cnp_hz_to_pwm, -}; - -static const struct intel_panel_bl_funcs lpt_pwm_funcs = { - .setup = lpt_setup_backlight, - .enable = lpt_enable_backlight, - .disable = lpt_disable_backlight, - .set = lpt_set_backlight, - .get = lpt_get_backlight, - .hz_to_pwm = lpt_hz_to_pwm, -}; - -static const struct intel_panel_bl_funcs spt_pwm_funcs = { - .setup = lpt_setup_backlight, - .enable = lpt_enable_backlight, - .disable = lpt_disable_backlight, - .set = lpt_set_backlight, - .get = lpt_get_backlight, - .hz_to_pwm = spt_hz_to_pwm, -}; - -static const struct intel_panel_bl_funcs pch_pwm_funcs = { - .setup = pch_setup_backlight, - .enable = pch_enable_backlight, - .disable = pch_disable_backlight, - .set = pch_set_backlight, - .get = pch_get_backlight, - .hz_to_pwm = pch_hz_to_pwm, -}; - -static const struct intel_panel_bl_funcs ext_pwm_funcs = { - .setup = ext_pwm_setup_backlight, - .enable = ext_pwm_enable_backlight, - .disable = ext_pwm_disable_backlight, - .set = ext_pwm_set_backlight, - .get = ext_pwm_get_backlight, -}; - -static const struct intel_panel_bl_funcs vlv_pwm_funcs = { - .setup = vlv_setup_backlight, - .enable = vlv_enable_backlight, - .disable = vlv_disable_backlight, - .set = vlv_set_backlight, - .get = vlv_get_backlight, - .hz_to_pwm = vlv_hz_to_pwm, -}; - -static const struct intel_panel_bl_funcs i965_pwm_funcs = { - .setup = i965_setup_backlight, - .enable = i965_enable_backlight, - .disable = i965_disable_backlight, - .set = i9xx_set_backlight, - .get = i9xx_get_backlight, - .hz_to_pwm = i965_hz_to_pwm, -}; - -static const struct intel_panel_bl_funcs i9xx_pwm_funcs = { - .setup = i9xx_setup_backlight, - .enable = i9xx_enable_backlight, - .disable = i9xx_disable_backlight, - .set = i9xx_set_backlight, - .get = i9xx_get_backlight, - .hz_to_pwm = i9xx_hz_to_pwm, -}; - -static const struct intel_panel_bl_funcs pwm_bl_funcs = { - .setup = intel_pwm_setup_backlight, - .enable = intel_pwm_enable_backlight, - .disable = intel_pwm_disable_backlight, - .set = intel_pwm_set_backlight, - .get = intel_pwm_get_backlight, -}; - -/* Set up chip specific backlight functions */ -static void -intel_panel_init_backlight_funcs(struct intel_panel *panel) +enum drm_mode_status +intel_panel_mode_valid(struct intel_connector *connector, + const struct drm_display_mode *mode) { - struct intel_connector *connector = - container_of(panel, struct intel_connector, panel); - struct drm_i915_private *dev_priv = to_i915(connector->base.dev); + const struct drm_display_mode *fixed_mode = connector->panel.fixed_mode; - if (connector->base.connector_type == DRM_MODE_CONNECTOR_DSI && - intel_dsi_dcs_init_backlight_funcs(connector) == 0) - return; - - if (IS_GEMINILAKE(dev_priv) || IS_BROXTON(dev_priv)) { - panel->backlight.pwm_funcs = &bxt_pwm_funcs; - } else if (INTEL_PCH_TYPE(dev_priv) >= PCH_CNP) { - panel->backlight.pwm_funcs = &cnp_pwm_funcs; - } else if (INTEL_PCH_TYPE(dev_priv) >= PCH_LPT) { - if (HAS_PCH_LPT(dev_priv)) - panel->backlight.pwm_funcs = &lpt_pwm_funcs; - else - panel->backlight.pwm_funcs = &spt_pwm_funcs; - } else if (HAS_PCH_SPLIT(dev_priv)) { - panel->backlight.pwm_funcs = &pch_pwm_funcs; - } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) { - if (connector->base.connector_type == DRM_MODE_CONNECTOR_DSI) { - panel->backlight.pwm_funcs = &ext_pwm_funcs; - } else { - panel->backlight.pwm_funcs = &vlv_pwm_funcs; - } - } else if (DISPLAY_VER(dev_priv) == 4) { - panel->backlight.pwm_funcs = &i965_pwm_funcs; - } else { - panel->backlight.pwm_funcs = &i9xx_pwm_funcs; - } - - if (connector->base.connector_type == DRM_MODE_CONNECTOR_eDP && - intel_dp_aux_init_backlight_funcs(connector) == 0) - return; + if (!fixed_mode) + return MODE_OK; - /* We're using a standard PWM backlight interface */ - panel->backlight.funcs = &pwm_bl_funcs; -} + if (mode->hdisplay != fixed_mode->hdisplay) + return MODE_PANEL; -enum drm_connector_status -intel_panel_detect(struct drm_connector *connector, bool force) -{ - struct drm_i915_private *i915 = to_i915(connector->dev); + if (mode->vdisplay != fixed_mode->vdisplay) + return MODE_PANEL; - if (!INTEL_DISPLAY_ENABLED(i915)) - return connector_status_disconnected; + if (drm_mode_vrefresh(mode) != drm_mode_vrefresh(fixed_mode)) + return MODE_PANEL; - return connector_status_connected; + return MODE_OK; } int intel_panel_init(struct intel_panel *panel, struct drm_display_mode *fixed_mode, struct drm_display_mode *downclock_mode) { - intel_panel_init_backlight_funcs(panel); + intel_backlight_init_funcs(panel); panel->fixed_mode = fixed_mode; panel->downclock_mode = downclock_mode; @@ -2245,7 +542,7 @@ void intel_panel_fini(struct intel_panel *panel) struct intel_connector *intel_connector = container_of(panel, struct intel_connector, panel); - intel_panel_destroy_backlight(panel); + intel_backlight_destroy(panel); if (panel->fixed_mode) drm_mode_destroy(intel_connector->base.dev, panel->fixed_mode); diff --git a/drivers/gpu/drm/i915/display/intel_panel.h b/drivers/gpu/drm/i915/display/intel_panel.h index 1d340f77bffc..d50b3f7e9e58 100644 --- a/drivers/gpu/drm/i915/display/intel_panel.h +++ b/drivers/gpu/drm/i915/display/intel_panel.h @@ -8,15 +8,13 @@ #include <linux/types.h> -#include "intel_display.h" - +enum drm_connector_status; struct drm_connector; struct drm_connector_state; struct drm_display_mode; +struct drm_i915_private; struct intel_connector; -struct intel_crtc; struct intel_crtc_state; -struct intel_encoder; struct intel_panel; int intel_panel_init(struct intel_panel *panel, @@ -25,23 +23,16 @@ int intel_panel_init(struct intel_panel *panel, void intel_panel_fini(struct intel_panel *panel); enum drm_connector_status intel_panel_detect(struct drm_connector *connector, bool force); -void intel_fixed_panel_mode(const struct drm_display_mode *fixed_mode, +bool intel_panel_use_ssc(struct drm_i915_private *i915); +void intel_panel_fixed_mode(const struct drm_display_mode *fixed_mode, struct drm_display_mode *adjusted_mode); -int intel_pch_panel_fitting(struct intel_crtc_state *crtc_state, - const struct drm_connector_state *conn_state); -int intel_gmch_panel_fitting(struct intel_crtc_state *crtc_state, - const struct drm_connector_state *conn_state); -void intel_panel_set_backlight_acpi(const struct drm_connector_state *conn_state, - u32 level, u32 max); -int intel_panel_setup_backlight(struct drm_connector *connector, - enum pipe pipe); -void intel_panel_enable_backlight(const struct intel_crtc_state *crtc_state, - const struct drm_connector_state *conn_state); -void intel_panel_update_backlight(struct intel_atomic_state *state, - struct intel_encoder *encoder, - const struct intel_crtc_state *crtc_state, - const struct drm_connector_state *conn_state); -void intel_panel_disable_backlight(const struct drm_connector_state *old_conn_state); +enum drm_mode_status +intel_panel_mode_valid(struct intel_connector *connector, + const struct drm_display_mode *mode); +int intel_panel_fitting(struct intel_crtc_state *crtc_state, + const struct drm_connector_state *conn_state); +int intel_panel_compute_config(struct intel_connector *connector, + struct drm_display_mode *adjusted_mode); struct drm_display_mode * intel_panel_edid_downclock_mode(struct intel_connector *connector, const struct drm_display_mode *fixed_mode); @@ -49,22 +40,5 @@ struct drm_display_mode * intel_panel_edid_fixed_mode(struct intel_connector *connector); struct drm_display_mode * intel_panel_vbt_fixed_mode(struct intel_connector *connector); -void intel_panel_set_pwm_level(const struct drm_connector_state *conn_state, u32 level); -u32 intel_panel_invert_pwm_level(struct intel_connector *connector, u32 level); -u32 intel_panel_backlight_level_to_pwm(struct intel_connector *connector, u32 level); -u32 intel_panel_backlight_level_from_pwm(struct intel_connector *connector, u32 val); - -#if IS_ENABLED(CONFIG_BACKLIGHT_CLASS_DEVICE) -int intel_backlight_device_register(struct intel_connector *connector); -void intel_backlight_device_unregister(struct intel_connector *connector); -#else /* CONFIG_BACKLIGHT_CLASS_DEVICE */ -static inline int intel_backlight_device_register(struct intel_connector *connector) -{ - return 0; -} -static inline void intel_backlight_device_unregister(struct intel_connector *connector) -{ -} -#endif /* CONFIG_BACKLIGHT_CLASS_DEVICE */ #endif /* __INTEL_PANEL_H__ */ diff --git a/drivers/gpu/drm/i915/display/intel_plane_initial.c b/drivers/gpu/drm/i915/display/intel_plane_initial.c new file mode 100644 index 000000000000..dcd698a02da2 --- /dev/null +++ b/drivers/gpu/drm/i915/display/intel_plane_initial.c @@ -0,0 +1,283 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2021 Intel Corporation + */ + +#include "intel_display_types.h" +#include "intel_plane_initial.h" +#include "intel_atomic_plane.h" +#include "intel_display.h" +#include "intel_fb.h" + +static bool +intel_reuse_initial_plane_obj(struct drm_i915_private *i915, + const struct intel_initial_plane_config *plane_config, + struct drm_framebuffer **fb, + struct i915_vma **vma) +{ + struct intel_crtc *crtc; + + for_each_intel_crtc(&i915->drm, crtc) { + struct intel_crtc_state *crtc_state = + to_intel_crtc_state(crtc->base.state); + struct intel_plane *plane = + to_intel_plane(crtc->base.primary); + struct intel_plane_state *plane_state = + to_intel_plane_state(plane->base.state); + + if (!crtc_state->uapi.active) + continue; + + if (!plane_state->ggtt_vma) + continue; + + if (intel_plane_ggtt_offset(plane_state) == plane_config->base) { + *fb = plane_state->hw.fb; + *vma = plane_state->ggtt_vma; + return true; + } + } + + return false; +} + +static struct i915_vma * +initial_plane_vma(struct drm_i915_private *i915, + struct intel_initial_plane_config *plane_config) +{ + struct drm_i915_gem_object *obj; + struct i915_vma *vma; + u32 base, size; + + if (plane_config->size == 0) + return NULL; + + base = round_down(plane_config->base, + I915_GTT_MIN_ALIGNMENT); + size = round_up(plane_config->base + plane_config->size, + I915_GTT_MIN_ALIGNMENT); + size -= base; + + /* + * If the FB is too big, just don't use it since fbdev is not very + * important and we should probably use that space with FBC or other + * features. + */ + if (IS_ENABLED(CONFIG_FRAMEBUFFER_CONSOLE) && + size * 2 > i915->stolen_usable_size) + return NULL; + + obj = i915_gem_object_create_stolen_for_preallocated(i915, base, size); + if (IS_ERR(obj)) + return NULL; + + /* + * Mark it WT ahead of time to avoid changing the + * cache_level during fbdev initialization. The + * unbind there would get stuck waiting for rcu. + */ + i915_gem_object_set_cache_coherency(obj, HAS_WT(i915) ? + I915_CACHE_WT : I915_CACHE_NONE); + + switch (plane_config->tiling) { + case I915_TILING_NONE: + break; + case I915_TILING_X: + case I915_TILING_Y: + obj->tiling_and_stride = + plane_config->fb->base.pitches[0] | + plane_config->tiling; + break; + default: + MISSING_CASE(plane_config->tiling); + goto err_obj; + } + + vma = i915_vma_instance(obj, &i915->ggtt.vm, NULL); + if (IS_ERR(vma)) + goto err_obj; + + if (i915_ggtt_pin(vma, NULL, 0, PIN_MAPPABLE | PIN_OFFSET_FIXED | base)) + goto err_obj; + + if (i915_gem_object_is_tiled(obj) && + !i915_vma_is_map_and_fenceable(vma)) + goto err_obj; + + return vma; + +err_obj: + i915_gem_object_put(obj); + return NULL; +} + +static bool +intel_alloc_initial_plane_obj(struct intel_crtc *crtc, + struct intel_initial_plane_config *plane_config) +{ + struct drm_device *dev = crtc->base.dev; + struct drm_i915_private *dev_priv = to_i915(dev); + struct drm_mode_fb_cmd2 mode_cmd = { 0 }; + struct drm_framebuffer *fb = &plane_config->fb->base; + struct i915_vma *vma; + + switch (fb->modifier) { + case DRM_FORMAT_MOD_LINEAR: + case I915_FORMAT_MOD_X_TILED: + case I915_FORMAT_MOD_Y_TILED: + break; + default: + drm_dbg(&dev_priv->drm, + "Unsupported modifier for initial FB: 0x%llx\n", + fb->modifier); + return false; + } + + vma = initial_plane_vma(dev_priv, plane_config); + if (!vma) + return false; + + mode_cmd.pixel_format = fb->format->format; + mode_cmd.width = fb->width; + mode_cmd.height = fb->height; + mode_cmd.pitches[0] = fb->pitches[0]; + mode_cmd.modifier[0] = fb->modifier; + mode_cmd.flags = DRM_MODE_FB_MODIFIERS; + + if (intel_framebuffer_init(to_intel_framebuffer(fb), + vma->obj, &mode_cmd)) { + drm_dbg_kms(&dev_priv->drm, "intel fb init failed\n"); + goto err_vma; + } + + plane_config->vma = vma; + return true; + +err_vma: + i915_vma_put(vma); + return false; +} + +static void +intel_find_initial_plane_obj(struct intel_crtc *crtc, + struct intel_initial_plane_config *plane_config) +{ + struct drm_device *dev = crtc->base.dev; + struct drm_i915_private *dev_priv = to_i915(dev); + struct intel_crtc_state *crtc_state = + to_intel_crtc_state(crtc->base.state); + struct intel_plane *plane = + to_intel_plane(crtc->base.primary); + struct intel_plane_state *plane_state = + to_intel_plane_state(plane->base.state); + struct drm_framebuffer *fb; + struct i915_vma *vma; + + /* + * TODO: + * Disable planes if get_initial_plane_config() failed. + * Make sure things work if the surface base is not page aligned. + */ + if (!plane_config->fb) + return; + + if (intel_alloc_initial_plane_obj(crtc, plane_config)) { + fb = &plane_config->fb->base; + vma = plane_config->vma; + goto valid_fb; + } + + /* + * Failed to alloc the obj, check to see if we should share + * an fb with another CRTC instead + */ + if (intel_reuse_initial_plane_obj(dev_priv, plane_config, &fb, &vma)) + goto valid_fb; + + /* + * We've failed to reconstruct the BIOS FB. Current display state + * indicates that the primary plane is visible, but has a NULL FB, + * which will lead to problems later if we don't fix it up. The + * simplest solution is to just disable the primary plane now and + * pretend the BIOS never had it enabled. + */ + intel_plane_disable_noatomic(crtc, plane); + if (crtc_state->bigjoiner) { + struct intel_crtc *slave = + crtc_state->bigjoiner_linked_crtc; + intel_plane_disable_noatomic(slave, to_intel_plane(slave->base.primary)); + } + + return; + +valid_fb: + plane_state->uapi.rotation = plane_config->rotation; + intel_fb_fill_view(to_intel_framebuffer(fb), + plane_state->uapi.rotation, &plane_state->view); + + __i915_vma_pin(vma); + plane_state->ggtt_vma = i915_vma_get(vma); + if (intel_plane_uses_fence(plane_state) && + i915_vma_pin_fence(vma) == 0 && vma->fence) + plane_state->flags |= PLANE_HAS_FENCE; + + plane_state->uapi.src_x = 0; + plane_state->uapi.src_y = 0; + plane_state->uapi.src_w = fb->width << 16; + plane_state->uapi.src_h = fb->height << 16; + + plane_state->uapi.crtc_x = 0; + plane_state->uapi.crtc_y = 0; + plane_state->uapi.crtc_w = fb->width; + plane_state->uapi.crtc_h = fb->height; + + if (plane_config->tiling) + dev_priv->preserve_bios_swizzle = true; + + plane_state->uapi.fb = fb; + drm_framebuffer_get(fb); + + plane_state->uapi.crtc = &crtc->base; + intel_plane_copy_uapi_to_hw_state(plane_state, plane_state, crtc); + + atomic_or(plane->frontbuffer_bit, &to_intel_frontbuffer(fb)->bits); +} + +static void plane_config_fini(struct intel_initial_plane_config *plane_config) +{ + if (plane_config->fb) { + struct drm_framebuffer *fb = &plane_config->fb->base; + + /* We may only have the stub and not a full framebuffer */ + if (drm_framebuffer_read_refcount(fb)) + drm_framebuffer_put(fb); + else + kfree(fb); + } + + if (plane_config->vma) + i915_vma_put(plane_config->vma); +} + +void intel_crtc_initial_plane_config(struct intel_crtc *crtc) +{ + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); + struct intel_initial_plane_config plane_config = {}; + + /* + * Note that reserving the BIOS fb up front prevents us + * from stuffing other stolen allocations like the ring + * on top. This prevents some ugliness at boot time, and + * can even allow for smooth boot transitions if the BIOS + * fb is large enough for the active pipe configuration. + */ + dev_priv->display->get_initial_plane_config(crtc, &plane_config); + + /* + * If the fb is shared between multiple heads, we'll + * just get the first one. + */ + intel_find_initial_plane_obj(crtc, &plane_config); + + plane_config_fini(&plane_config); +} diff --git a/drivers/gpu/drm/i915/display/intel_plane_initial.h b/drivers/gpu/drm/i915/display/intel_plane_initial.h new file mode 100644 index 000000000000..c7e35ab3182b --- /dev/null +++ b/drivers/gpu/drm/i915/display/intel_plane_initial.h @@ -0,0 +1,13 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2021 Intel Corporation + */ + +#ifndef __INTEL_PLANE_INITIAL_H__ +#define __INTEL_PLANE_INITIAL_H__ + +struct intel_crtc; + +void intel_crtc_initial_plane_config(struct intel_crtc *crtc); + +#endif diff --git a/drivers/gpu/drm/i915/display/intel_pps.c b/drivers/gpu/drm/i915/display/intel_pps.c index a36ec4a818ff..e9c679bb1b2e 100644 --- a/drivers/gpu/drm/i915/display/intel_pps.c +++ b/drivers/gpu/drm/i915/display/intel_pps.c @@ -9,6 +9,7 @@ #include "intel_display_types.h" #include "intel_dp.h" #include "intel_dpll.h" +#include "intel_lvds.h" #include "intel_pps.h" static void vlv_steal_power_sequencer(struct drm_i915_private *dev_priv, @@ -1408,3 +1409,61 @@ void intel_pps_setup(struct drm_i915_private *i915) else i915->pps_mmio_base = PPS_BASE; } + +void assert_pps_unlocked(struct drm_i915_private *dev_priv, enum pipe pipe) +{ + i915_reg_t pp_reg; + u32 val; + enum pipe panel_pipe = INVALID_PIPE; + bool locked = true; + + if (drm_WARN_ON(&dev_priv->drm, HAS_DDI(dev_priv))) + return; + + if (HAS_PCH_SPLIT(dev_priv)) { + u32 port_sel; + + pp_reg = PP_CONTROL(0); + port_sel = intel_de_read(dev_priv, PP_ON_DELAYS(0)) & PANEL_PORT_SELECT_MASK; + + switch (port_sel) { + case PANEL_PORT_SELECT_LVDS: + intel_lvds_port_enabled(dev_priv, PCH_LVDS, &panel_pipe); + break; + case PANEL_PORT_SELECT_DPA: + g4x_dp_port_enabled(dev_priv, DP_A, PORT_A, &panel_pipe); + break; + case PANEL_PORT_SELECT_DPC: + g4x_dp_port_enabled(dev_priv, PCH_DP_C, PORT_C, &panel_pipe); + break; + case PANEL_PORT_SELECT_DPD: + g4x_dp_port_enabled(dev_priv, PCH_DP_D, PORT_D, &panel_pipe); + break; + default: + MISSING_CASE(port_sel); + break; + } + } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) { + /* presumably write lock depends on pipe, not port select */ + pp_reg = PP_CONTROL(pipe); + panel_pipe = pipe; + } else { + u32 port_sel; + + pp_reg = PP_CONTROL(0); + port_sel = intel_de_read(dev_priv, PP_ON_DELAYS(0)) & PANEL_PORT_SELECT_MASK; + + drm_WARN_ON(&dev_priv->drm, + port_sel != PANEL_PORT_SELECT_LVDS); + intel_lvds_port_enabled(dev_priv, LVDS, &panel_pipe); + } + + val = intel_de_read(dev_priv, pp_reg); + if (!(val & PANEL_POWER_ON) || + ((val & PANEL_UNLOCK_MASK) == PANEL_UNLOCK_REGS)) + locked = false; + + I915_STATE_WARN(panel_pipe == pipe && locked, + "panel assertion failure, pipe %c regs locked\n", + pipe_name(pipe)); +} diff --git a/drivers/gpu/drm/i915/display/intel_pps.h b/drivers/gpu/drm/i915/display/intel_pps.h index fbbcca782e7b..fbb47f6f453e 100644 --- a/drivers/gpu/drm/i915/display/intel_pps.h +++ b/drivers/gpu/drm/i915/display/intel_pps.h @@ -10,6 +10,7 @@ #include "intel_wakeref.h" +enum pipe; struct drm_i915_private; struct intel_connector; struct intel_crtc_state; @@ -49,4 +50,6 @@ void vlv_pps_init(struct intel_encoder *encoder, void intel_pps_unlock_regs_wa(struct drm_i915_private *i915); void intel_pps_setup(struct drm_i915_private *i915); +void assert_pps_unlocked(struct drm_i915_private *i915, enum pipe pipe); + #endif /* __INTEL_PPS_H__ */ diff --git a/drivers/gpu/drm/i915/display/intel_psr.c b/drivers/gpu/drm/i915/display/intel_psr.c index 1b0daf649e82..7a205fd5023b 100644 --- a/drivers/gpu/drm/i915/display/intel_psr.c +++ b/drivers/gpu/drm/i915/display/intel_psr.c @@ -22,6 +22,7 @@ */ #include <drm/drm_atomic_helper.h> +#include <drm/drm_damage_helper.h> #include "display/intel_dp.h" @@ -364,41 +365,6 @@ void intel_psr_init_dpcd(struct intel_dp *intel_dp) } } -static void hsw_psr_setup_aux(struct intel_dp *intel_dp) -{ - struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); - u32 aux_clock_divider, aux_ctl; - int i; - static const u8 aux_msg[] = { - [0] = DP_AUX_NATIVE_WRITE << 4, - [1] = DP_SET_POWER >> 8, - [2] = DP_SET_POWER & 0xff, - [3] = 1 - 1, - [4] = DP_SET_POWER_D0, - }; - u32 psr_aux_mask = EDP_PSR_AUX_CTL_TIME_OUT_MASK | - EDP_PSR_AUX_CTL_MESSAGE_SIZE_MASK | - EDP_PSR_AUX_CTL_PRECHARGE_2US_MASK | - EDP_PSR_AUX_CTL_BIT_CLOCK_2X_MASK; - - BUILD_BUG_ON(sizeof(aux_msg) > 20); - for (i = 0; i < sizeof(aux_msg); i += 4) - intel_de_write(dev_priv, - EDP_PSR_AUX_DATA(intel_dp->psr.transcoder, i >> 2), - intel_dp_pack_aux(&aux_msg[i], sizeof(aux_msg) - i)); - - aux_clock_divider = intel_dp->get_aux_clock_divider(intel_dp, 0); - - /* Start with bits set for DDI_AUX_CTL register */ - aux_ctl = intel_dp->get_aux_send_ctl(intel_dp, sizeof(aux_msg), - aux_clock_divider); - - /* Select only valid bits for SRD_AUX_CTL */ - aux_ctl &= psr_aux_mask; - intel_de_write(dev_priv, EDP_PSR_AUX_CTL(intel_dp->psr.transcoder), - aux_ctl); -} - static void intel_psr_enable_sink(struct intel_dp *intel_dp) { struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); @@ -460,7 +426,7 @@ static u32 intel_psr1_get_tp_time(struct intel_dp *intel_dp) val |= EDP_PSR_TP2_TP3_TIME_2500us; check_tp3_sel: - if (intel_dp_source_supports_hbr2(intel_dp) && + if (intel_dp_source_supports_tps3(dev_priv) && drm_dp_tps3_supported(intel_dp->dpcd)) val |= EDP_PSR_TP1_TP3_SEL; else @@ -545,7 +511,7 @@ static void hsw_activate_psr2(struct intel_dp *intel_dp) if (DISPLAY_VER(dev_priv) >= 10 && DISPLAY_VER(dev_priv) <= 12) val |= EDP_Y_COORDINATE_ENABLE; - val |= EDP_PSR2_FRAME_BEFORE_SU(intel_dp->psr.sink_sync_latency + 1); + val |= EDP_PSR2_FRAME_BEFORE_SU(max_t(u8, intel_dp->psr.sink_sync_latency + 1, 2)); val |= intel_psr2_get_tp_time(intel_dp); /* Wa_22012278275:adl-p */ @@ -595,15 +561,16 @@ static void hsw_activate_psr2(struct intel_dp *intel_dp) val |= EDP_PSR2_SU_SDP_SCANLINE; if (intel_dp->psr.psr2_sel_fetch_enabled) { + u32 tmp; + /* Wa_1408330847 */ if (IS_TGL_DISPLAY_STEP(dev_priv, STEP_A0, STEP_B0)) intel_de_rmw(dev_priv, CHICKEN_PAR1_1, DIS_RAM_BYPASS_PSR2_MAN_TRACK, DIS_RAM_BYPASS_PSR2_MAN_TRACK); - intel_de_write(dev_priv, - PSR2_MAN_TRK_CTL(intel_dp->psr.transcoder), - PSR2_MAN_TRK_CTL_ENABLE); + tmp = intel_de_read(dev_priv, PSR2_MAN_TRK_CTL(intel_dp->psr.transcoder)); + drm_WARN_ON(&dev_priv->drm, !(tmp & PSR2_MAN_TRK_CTL_ENABLE)); } else if (HAS_PSR2_SEL_FETCH(dev_priv)) { intel_de_write(dev_priv, PSR2_MAN_TRK_CTL(intel_dp->psr.transcoder), 0); @@ -621,9 +588,7 @@ static void hsw_activate_psr2(struct intel_dp *intel_dp) static bool transcoder_has_psr2(struct drm_i915_private *dev_priv, enum transcoder trans) { - if (DISPLAY_VER(dev_priv) < 9) - return false; - else if (DISPLAY_VER(dev_priv) >= 12) + if (DISPLAY_VER(dev_priv) >= 12) return trans == TRANSCODER_A; else return trans == TRANSCODER_EDP; @@ -755,11 +720,7 @@ tgl_dc3co_exitline_compute_config(struct intel_dp *intel_dp, static bool intel_psr2_sel_fetch_config_valid(struct intel_dp *intel_dp, struct intel_crtc_state *crtc_state) { - struct intel_atomic_state *state = to_intel_atomic_state(crtc_state->uapi.state); struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); - struct intel_plane_state *plane_state; - struct intel_plane *plane; - int i; if (!dev_priv->params.enable_psr2_sel_fetch && intel_dp->psr.debug != I915_PSR_DEBUG_ENABLE_SEL_FETCH) { @@ -774,14 +735,6 @@ static bool intel_psr2_sel_fetch_config_valid(struct intel_dp *intel_dp, return false; } - for_each_new_intel_plane_in_state(state, plane, plane_state, i) { - if (plane_state->uapi.rotation != DRM_MODE_ROTATE_0) { - drm_dbg_kms(&dev_priv->drm, - "PSR2 sel fetch not enabled, plane rotated\n"); - return false; - } - } - /* Wa_14010254185 Wa_14010103792 */ if (IS_TGL_DISPLAY_STEP(dev_priv, STEP_A0, STEP_C0)) { drm_dbg_kms(&dev_priv->drm, @@ -877,12 +830,8 @@ static bool intel_psr2_config_valid(struct intel_dp *intel_dp, return false; } - /* - * We are missing the implementation of some workarounds to enabled PSR2 - * in Alderlake_P, until ready PSR2 should be kept disabled. - */ - if (IS_ALDERLAKE_P(dev_priv)) { - drm_dbg_kms(&dev_priv->drm, "PSR2 is missing the implementation of workarounds\n"); + if (IS_ADLP_DISPLAY_STEP(dev_priv, STEP_A0, STEP_B0)) { + drm_dbg_kms(&dev_priv->drm, "PSR2 not completely functional in this stepping\n"); return false; } @@ -985,7 +934,8 @@ static bool intel_psr2_config_valid(struct intel_dp *intel_dp, } void intel_psr_compute_config(struct intel_dp *intel_dp, - struct intel_crtc_state *crtc_state) + struct intel_crtc_state *crtc_state, + struct drm_connector_state *conn_state) { struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); const struct drm_display_mode *adjusted_mode = @@ -1037,7 +987,10 @@ void intel_psr_compute_config(struct intel_dp *intel_dp, crtc_state->has_psr = true; crtc_state->has_psr2 = intel_psr2_config_valid(intel_dp, crtc_state); + crtc_state->infoframes.enable |= intel_hdmi_infoframe_enable(DP_SDP_VSC); + intel_dp_compute_psr_vsc_sdp(intel_dp, crtc_state, conn_state, + &crtc_state->psr_vsc); } void intel_psr_get_config(struct intel_encoder *encoder, @@ -1114,12 +1067,6 @@ static void intel_psr_enable_source(struct intel_dp *intel_dp) enum transcoder cpu_transcoder = intel_dp->psr.transcoder; u32 mask; - /* Only HSW and BDW have PSR AUX registers that need to be setup. SKL+ - * use hardcoded values PSR AUX transactions - */ - if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) - hsw_psr_setup_aux(intel_dp); - if (intel_dp->psr.psr2_enabled && DISPLAY_VER(dev_priv) == 9) { i915_reg_t reg = CHICKEN_TRANS(cpu_transcoder); u32 chicken = intel_de_read(dev_priv, reg); @@ -1130,6 +1077,16 @@ static void intel_psr_enable_source(struct intel_dp *intel_dp) } /* + * Wa_16014451276:adlp + * All supported adlp panels have 1-based X granularity, this may + * cause issues if non-supported panels are used. + */ + if (IS_ALDERLAKE_P(dev_priv) && + intel_dp->psr.psr2_enabled) + intel_de_rmw(dev_priv, CHICKEN_TRANS(cpu_transcoder), 0, + ADLP_1_BASED_X_GRANULARITY); + + /* * Per Spec: Avoid continuous PSR exit by masking MEMUP and HPD also * mask LPSP to avoid dependency on other drivers that might block * runtime_pm besides preventing other hw tracking issues now we @@ -1174,6 +1131,11 @@ static void intel_psr_enable_source(struct intel_dp *intel_dp) TRANS_SET_CONTEXT_LATENCY(intel_dp->psr.transcoder), TRANS_SET_CONTEXT_LATENCY_MASK, TRANS_SET_CONTEXT_LATENCY_VALUE(1)); + + /* Wa_16012604467:adlp */ + if (IS_ALDERLAKE_P(dev_priv) && intel_dp->psr.psr2_enabled) + intel_de_rmw(dev_priv, CLKGATE_DIS_MISC, 0, + CLKGATE_DIS_MISC_DMASC_GATING_DIS); } static bool psr_interrupt_error_check(struct intel_dp *intel_dp) @@ -1208,8 +1170,7 @@ static bool psr_interrupt_error_check(struct intel_dp *intel_dp) } static void intel_psr_enable_locked(struct intel_dp *intel_dp, - const struct intel_crtc_state *crtc_state, - const struct drm_connector_state *conn_state) + const struct intel_crtc_state *crtc_state) { struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp); struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); @@ -1236,9 +1197,7 @@ static void intel_psr_enable_locked(struct intel_dp *intel_dp, drm_dbg_kms(&dev_priv->drm, "Enabling PSR%s\n", intel_dp->psr.psr2_enabled ? "2" : "1"); - intel_dp_compute_psr_vsc_sdp(intel_dp, crtc_state, conn_state, - &intel_dp->psr.vsc); - intel_write_dp_vsc_sdp(encoder, crtc_state, &intel_dp->psr.vsc); + intel_write_dp_vsc_sdp(encoder, crtc_state, &crtc_state->psr_vsc); intel_snps_phy_update_psr_power_state(dev_priv, phy, true); intel_psr_enable_sink(intel_dp); intel_psr_enable_source(intel_dp); @@ -1248,33 +1207,6 @@ static void intel_psr_enable_locked(struct intel_dp *intel_dp, intel_psr_activate(intel_dp); } -/** - * intel_psr_enable - Enable PSR - * @intel_dp: Intel DP - * @crtc_state: new CRTC state - * @conn_state: new CONNECTOR state - * - * This function can only be called after the pipe is fully trained and enabled. - */ -void intel_psr_enable(struct intel_dp *intel_dp, - const struct intel_crtc_state *crtc_state, - const struct drm_connector_state *conn_state) -{ - struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); - - if (!CAN_PSR(intel_dp)) - return; - - if (!crtc_state->has_psr) - return; - - drm_WARN_ON(&dev_priv->drm, dev_priv->drrs.dp); - - mutex_lock(&intel_dp->psr.lock); - intel_psr_enable_locked(intel_dp, crtc_state, conn_state); - mutex_unlock(&intel_dp->psr.lock); -} - static void intel_psr_exit(struct intel_dp *intel_dp) { struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); @@ -1363,6 +1295,11 @@ static void intel_psr_disable_locked(struct intel_dp *intel_dp) TRANS_SET_CONTEXT_LATENCY(intel_dp->psr.transcoder), TRANS_SET_CONTEXT_LATENCY_MASK, 0); + /* Wa_16012604467:adlp */ + if (IS_ALDERLAKE_P(dev_priv) && intel_dp->psr.psr2_enabled) + intel_de_rmw(dev_priv, CLKGATE_DIS_MISC, + CLKGATE_DIS_MISC_DMASC_GATING_DIS, 0); + intel_snps_phy_update_psr_power_state(dev_priv, phy, false); /* Disable PSR on Sink */ @@ -1456,27 +1393,48 @@ unlock: mutex_unlock(&psr->lock); } +static inline u32 man_trk_ctl_single_full_frame_bit_get(struct drm_i915_private *dev_priv) +{ + return IS_ALDERLAKE_P(dev_priv) ? + ADLP_PSR2_MAN_TRK_CTL_SF_SINGLE_FULL_FRAME : + PSR2_MAN_TRK_CTL_SF_SINGLE_FULL_FRAME; +} + static void psr_force_hw_tracking_exit(struct intel_dp *intel_dp) { struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); - if (DISPLAY_VER(dev_priv) >= 9) - /* - * Display WA #0884: skl+ - * This documented WA for bxt can be safely applied - * broadly so we can force HW tracking to exit PSR - * instead of disabling and re-enabling. - * Workaround tells us to write 0 to CUR_SURFLIVE_A, - * but it makes more sense write to the current active - * pipe. - */ - intel_de_write(dev_priv, CURSURFLIVE(intel_dp->psr.pipe), 0); - else - /* - * A write to CURSURFLIVE do not cause HW tracking to exit PSR - * on older gens so doing the manual exit instead. - */ - intel_psr_exit(intel_dp); + if (intel_dp->psr.psr2_sel_fetch_enabled) + intel_de_rmw(dev_priv, + PSR2_MAN_TRK_CTL(intel_dp->psr.transcoder), 0, + man_trk_ctl_single_full_frame_bit_get(dev_priv)); + + /* + * Display WA #0884: skl+ + * This documented WA for bxt can be safely applied + * broadly so we can force HW tracking to exit PSR + * instead of disabling and re-enabling. + * Workaround tells us to write 0 to CUR_SURFLIVE_A, + * but it makes more sense write to the current active + * pipe. + * + * This workaround do not exist for platforms with display 10 or newer + * but testing proved that it works for up display 13, for newer + * than that testing will be needed. + */ + intel_de_write(dev_priv, CURSURFLIVE(intel_dp->psr.pipe), 0); +} + +void intel_psr2_disable_plane_sel_fetch(struct intel_plane *plane, + const struct intel_crtc_state *crtc_state) +{ + struct drm_i915_private *dev_priv = to_i915(plane->base.dev); + enum pipe pipe = plane->pipe; + + if (!crtc_state->enable_psr2_sel_fetch) + return; + + intel_de_write_fw(dev_priv, PLANE_SEL_FETCH_CTL(pipe, plane->id), 0); } void intel_psr2_program_plane_sel_fetch(struct intel_plane *plane, @@ -1487,17 +1445,17 @@ void intel_psr2_program_plane_sel_fetch(struct intel_plane *plane, struct drm_i915_private *dev_priv = to_i915(plane->base.dev); enum pipe pipe = plane->pipe; const struct drm_rect *clip; - u32 val, offset; - int ret, x, y; + u32 val; + int x, y; if (!crtc_state->enable_psr2_sel_fetch) return; - val = plane_state ? plane_state->ctl : 0; - val &= plane->id == PLANE_CURSOR ? val : PLANE_SEL_FETCH_CTL_ENABLE; - intel_de_write_fw(dev_priv, PLANE_SEL_FETCH_CTL(pipe, plane->id), val); - if (!val || plane->id == PLANE_CURSOR) + if (plane->id == PLANE_CURSOR) { + intel_de_write_fw(dev_priv, PLANE_SEL_FETCH_CTL(pipe, plane->id), + plane_state->ctl); return; + } clip = &plane_state->psr2_sel_fetch_area; @@ -1508,10 +1466,6 @@ void intel_psr2_program_plane_sel_fetch(struct intel_plane *plane, /* TODO: consider auxiliary surfaces */ x = plane_state->uapi.src.x1 >> 16; y = (plane_state->uapi.src.y1 >> 16) + clip->y1; - ret = skl_calc_main_surface_offset(plane_state, &x, &y, &offset); - if (ret) - drm_warn_once(&dev_priv->drm, "skl_calc_main_surface_offset() returned %i\n", - ret); val = y << 16 | x; intel_de_write_fw(dev_priv, PLANE_SEL_FETCH_OFFSET(pipe, plane->id), val); @@ -1520,14 +1474,16 @@ void intel_psr2_program_plane_sel_fetch(struct intel_plane *plane, val = (drm_rect_height(clip) - 1) << 16; val |= (drm_rect_width(&plane_state->uapi.src) >> 16) - 1; intel_de_write_fw(dev_priv, PLANE_SEL_FETCH_SIZE(pipe, plane->id), val); + + intel_de_write_fw(dev_priv, PLANE_SEL_FETCH_CTL(pipe, plane->id), + PLANE_SEL_FETCH_CTL_ENABLE); } void intel_psr2_program_trans_man_trk_ctl(const struct intel_crtc_state *crtc_state) { struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev); - if (!HAS_PSR2_SEL_FETCH(dev_priv) || - !crtc_state->enable_psr2_sel_fetch) + if (!crtc_state->enable_psr2_sel_fetch) return; intel_de_write(dev_priv, PSR2_MAN_TRK_CTL(crtc_state->cpu_transcoder), @@ -1542,11 +1498,11 @@ static void psr2_man_trk_ctl_calc(struct intel_crtc_state *crtc_state, u32 val = PSR2_MAN_TRK_CTL_ENABLE; if (full_update) { - if (IS_ALDERLAKE_P(dev_priv)) - val |= ADLP_PSR2_MAN_TRK_CTL_SF_SINGLE_FULL_FRAME; - else - val |= PSR2_MAN_TRK_CTL_SF_SINGLE_FULL_FRAME; - + /* + * Not applying Wa_14014971508:adlp as we do not support the + * feature that requires this workaround. + */ + val |= man_trk_ctl_single_full_frame_bit_get(dev_priv); goto exit; } @@ -1555,7 +1511,7 @@ static void psr2_man_trk_ctl_calc(struct intel_crtc_state *crtc_state, if (IS_ALDERLAKE_P(dev_priv)) { val |= ADLP_PSR2_MAN_TRK_CTL_SU_REGION_START_ADDR(clip->y1); - val |= ADLP_PSR2_MAN_TRK_CTL_SU_REGION_END_ADDR(clip->y2); + val |= ADLP_PSR2_MAN_TRK_CTL_SU_REGION_END_ADDR(clip->y2 - 1); } else { drm_WARN_ON(crtc_state->uapi.crtc->dev, clip->y1 % 4 || clip->y2 % 4); @@ -1597,6 +1553,45 @@ static void intel_psr2_sel_fetch_pipe_alignment(const struct intel_crtc_state *c drm_warn(&dev_priv->drm, "Missing PSR2 sel fetch alignment with DSC\n"); } +/* + * TODO: Not clear how to handle planes with negative position, + * also planes are not updated if they have a negative X + * position so for now doing a full update in this cases + * + * TODO: We are missing multi-planar formats handling, until it is + * implemented it will send full frame updates. + * + * Plane scaling and rotation is not supported by selective fetch and both + * properties can change without a modeset, so need to be check at every + * atomic commmit. + */ +static bool psr2_sel_fetch_plane_state_supported(const struct intel_plane_state *plane_state) +{ + if (plane_state->uapi.dst.y1 < 0 || + plane_state->uapi.dst.x1 < 0 || + plane_state->scaler_id >= 0 || + plane_state->hw.fb->format->num_planes > 1 || + plane_state->uapi.rotation != DRM_MODE_ROTATE_0) + return false; + + return true; +} + +/* + * Check for pipe properties that is not supported by selective fetch. + * + * TODO: pipe scaling causes a modeset but skl_update_scaler_crtc() is executed + * after intel_psr_compute_config(), so for now keeping PSR2 selective fetch + * enabled and going to the full update path. + */ +static bool psr2_sel_fetch_pipe_state_supported(const struct intel_crtc_state *crtc_state) +{ + if (crtc_state->scaler_state.scaler_id >= 0) + return false; + + return true; +} + int intel_psr2_sel_fetch_update(struct intel_atomic_state *state, struct intel_crtc *crtc) { @@ -1610,9 +1605,10 @@ int intel_psr2_sel_fetch_update(struct intel_atomic_state *state, if (!crtc_state->enable_psr2_sel_fetch) return 0; - ret = drm_atomic_add_affected_planes(&state->base, &crtc->base); - if (ret) - return ret; + if (!psr2_sel_fetch_pipe_state_supported(crtc_state)) { + full_update = true; + goto skip_sel_fetch_set_loop; + } /* * Calculate minimal selective fetch area of each plane and calculate @@ -1623,8 +1619,8 @@ int intel_psr2_sel_fetch_update(struct intel_atomic_state *state, for_each_oldnew_intel_plane_in_state(state, plane, old_plane_state, new_plane_state, i) { struct drm_rect src, damaged_area = { .y1 = -1 }; - struct drm_mode_rect *damaged_clips; - u32 num_clips, j; + struct drm_atomic_helper_damage_iter iter; + struct drm_rect clip; if (new_plane_state->uapi.crtc != crtc_state->uapi.crtc) continue; @@ -1633,19 +1629,11 @@ int intel_psr2_sel_fetch_update(struct intel_atomic_state *state, !old_plane_state->uapi.visible) continue; - /* - * TODO: Not clear how to handle planes with negative position, - * also planes are not updated if they have a negative X - * position so for now doing a full update in this cases - */ - if (new_plane_state->uapi.dst.y1 < 0 || - new_plane_state->uapi.dst.x1 < 0) { + if (!psr2_sel_fetch_plane_state_supported(new_plane_state)) { full_update = true; break; } - num_clips = drm_plane_get_damage_clips_count(&new_plane_state->uapi); - /* * If visibility or plane moved, mark the whole plane area as * damaged as it needs to be complete redraw in the new and old @@ -1666,14 +1654,8 @@ int intel_psr2_sel_fetch_update(struct intel_atomic_state *state, clip_area_update(&pipe_clip, &damaged_area); } continue; - } else if (new_plane_state->uapi.alpha != old_plane_state->uapi.alpha || - (!num_clips && - new_plane_state->uapi.fb != old_plane_state->uapi.fb)) { - /* - * If the plane don't have damaged areas but the - * framebuffer changed or alpha changed, mark the whole - * plane area as damaged. - */ + } else if (new_plane_state->uapi.alpha != old_plane_state->uapi.alpha) { + /* If alpha changed mark the whole plane area as damaged */ damaged_area.y1 = new_plane_state->uapi.dst.y1; damaged_area.y2 = new_plane_state->uapi.dst.y2; clip_area_update(&pipe_clip, &damaged_area); @@ -1681,15 +1663,11 @@ int intel_psr2_sel_fetch_update(struct intel_atomic_state *state, } drm_rect_fp_to_int(&src, &new_plane_state->uapi.src); - damaged_clips = drm_plane_get_damage_clips(&new_plane_state->uapi); - for (j = 0; j < num_clips; j++) { - struct drm_rect clip; - - clip.x1 = damaged_clips[j].x1; - clip.y1 = damaged_clips[j].y1; - clip.x2 = damaged_clips[j].x2; - clip.y2 = damaged_clips[j].y2; + drm_atomic_helper_damage_iter_init(&iter, + &old_plane_state->uapi, + &new_plane_state->uapi); + drm_atomic_for_each_plane_damage(&iter, &clip) { if (drm_rect_intersect(&clip, &src)) clip_area_update(&damaged_area, &clip); } @@ -1705,6 +1683,10 @@ int intel_psr2_sel_fetch_update(struct intel_atomic_state *state, if (full_update) goto skip_sel_fetch_set_loop; + ret = drm_atomic_add_affected_planes(&state->base, &crtc->base); + if (ret) + return ret; + intel_psr2_sel_fetch_pipe_alignment(crtc_state, &pipe_clip); /* @@ -1723,6 +1705,11 @@ int intel_psr2_sel_fetch_update(struct intel_atomic_state *state, if (!drm_rect_intersect(&inter, &new_plane_state->uapi.dst)) continue; + if (!psr2_sel_fetch_plane_state_supported(new_plane_state)) { + full_update = true; + break; + } + sel_fetch_area = &new_plane_state->psr2_sel_fetch_area; sel_fetch_area->y1 = inter.y1 - new_plane_state->uapi.dst.y1; sel_fetch_area->y2 = inter.y2 - new_plane_state->uapi.dst.y1; @@ -1734,58 +1721,92 @@ skip_sel_fetch_set_loop: return 0; } -/** - * intel_psr_update - Update PSR state - * @intel_dp: Intel DP - * @crtc_state: new CRTC state - * @conn_state: new CONNECTOR state - * - * This functions will update PSR states, disabling, enabling or switching PSR - * version when executing fastsets. For full modeset, intel_psr_disable() and - * intel_psr_enable() should be called instead. - */ -void intel_psr_update(struct intel_dp *intel_dp, - const struct intel_crtc_state *crtc_state, - const struct drm_connector_state *conn_state) +static void _intel_psr_pre_plane_update(const struct intel_atomic_state *state, + const struct intel_crtc_state *crtc_state) { - struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); - struct intel_psr *psr = &intel_dp->psr; - bool enable, psr2_enable; + struct intel_encoder *encoder; - if (!CAN_PSR(intel_dp)) + for_each_intel_encoder_mask_with_psr(state->base.dev, encoder, + crtc_state->uapi.encoder_mask) { + struct intel_dp *intel_dp = enc_to_intel_dp(encoder); + struct intel_psr *psr = &intel_dp->psr; + bool needs_to_disable = false; + + mutex_lock(&psr->lock); + + /* + * Reasons to disable: + * - PSR disabled in new state + * - All planes will go inactive + * - Changing between PSR versions + */ + needs_to_disable |= !crtc_state->has_psr; + needs_to_disable |= !crtc_state->active_planes; + needs_to_disable |= crtc_state->has_psr2 != psr->psr2_enabled; + + if (psr->enabled && needs_to_disable) + intel_psr_disable_locked(intel_dp); + + mutex_unlock(&psr->lock); + } +} + +void intel_psr_pre_plane_update(const struct intel_atomic_state *state) +{ + struct drm_i915_private *dev_priv = to_i915(state->base.dev); + struct intel_crtc_state *crtc_state; + struct intel_crtc *crtc; + int i; + + if (!HAS_PSR(dev_priv)) return; - mutex_lock(&intel_dp->psr.lock); + for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) + _intel_psr_pre_plane_update(state, crtc_state); +} + +static void _intel_psr_post_plane_update(const struct intel_atomic_state *state, + const struct intel_crtc_state *crtc_state) +{ + struct drm_i915_private *dev_priv = to_i915(state->base.dev); + struct intel_encoder *encoder; + + if (!crtc_state->has_psr) + return; + + for_each_intel_encoder_mask_with_psr(state->base.dev, encoder, + crtc_state->uapi.encoder_mask) { + struct intel_dp *intel_dp = enc_to_intel_dp(encoder); + struct intel_psr *psr = &intel_dp->psr; + + mutex_lock(&psr->lock); + + drm_WARN_ON(&dev_priv->drm, psr->enabled && !crtc_state->active_planes); - enable = crtc_state->has_psr; - psr2_enable = crtc_state->has_psr2; + /* Only enable if there is active planes */ + if (!psr->enabled && crtc_state->active_planes) + intel_psr_enable_locked(intel_dp, crtc_state); - if (enable == psr->enabled && psr2_enable == psr->psr2_enabled && - crtc_state->enable_psr2_sel_fetch == psr->psr2_sel_fetch_enabled) { /* Force a PSR exit when enabling CRC to avoid CRC timeouts */ if (crtc_state->crc_enabled && psr->enabled) psr_force_hw_tracking_exit(intel_dp); - else if (DISPLAY_VER(dev_priv) < 9 && psr->enabled) { - /* - * Activate PSR again after a force exit when enabling - * CRC in older gens - */ - if (!intel_dp->psr.active && - !intel_dp->psr.busy_frontbuffer_bits) - schedule_work(&intel_dp->psr.work); - } - goto unlock; + mutex_unlock(&psr->lock); } +} - if (psr->enabled) - intel_psr_disable_locked(intel_dp); +void intel_psr_post_plane_update(const struct intel_atomic_state *state) +{ + struct drm_i915_private *dev_priv = to_i915(state->base.dev); + struct intel_crtc_state *crtc_state; + struct intel_crtc *crtc; + int i; - if (enable) - intel_psr_enable_locked(intel_dp, crtc_state, conn_state); + if (!HAS_PSR(dev_priv)) + return; -unlock: - mutex_unlock(&intel_dp->psr.lock); + for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) + _intel_psr_post_plane_update(state, crtc_state); } /** @@ -2065,20 +2086,16 @@ void intel_psr_invalidate(struct drm_i915_private *dev_priv, /* * When we will be completely rely on PSR2 S/W tracking in future, * intel_psr_flush() will invalidate and flush the PSR for ORIGIN_FLIP - * event also therefore tgl_dc3co_flush() require to be changed + * event also therefore tgl_dc3co_flush_locked() require to be changed * accordingly in future. */ static void -tgl_dc3co_flush(struct intel_dp *intel_dp, unsigned int frontbuffer_bits, - enum fb_op_origin origin) +tgl_dc3co_flush_locked(struct intel_dp *intel_dp, unsigned int frontbuffer_bits, + enum fb_op_origin origin) { - mutex_lock(&intel_dp->psr.lock); - - if (!intel_dp->psr.dc3co_exitline) - goto unlock; - - if (!intel_dp->psr.psr2_enabled || !intel_dp->psr.active) - goto unlock; + if (!intel_dp->psr.dc3co_exitline || !intel_dp->psr.psr2_enabled || + !intel_dp->psr.active) + return; /* * At every frontbuffer flush flip event modified delay of delayed work, @@ -2086,14 +2103,11 @@ tgl_dc3co_flush(struct intel_dp *intel_dp, unsigned int frontbuffer_bits, */ if (!(frontbuffer_bits & INTEL_FRONTBUFFER_ALL_MASK(intel_dp->psr.pipe))) - goto unlock; + return; tgl_psr2_enable_dc3co(intel_dp); mod_delayed_work(system_wq, &intel_dp->psr.dc3co_work, intel_dp->psr.dc3co_exit_delay); - -unlock: - mutex_unlock(&intel_dp->psr.lock); } /** @@ -2118,11 +2132,6 @@ void intel_psr_flush(struct drm_i915_private *dev_priv, unsigned int pipe_frontbuffer_bits = frontbuffer_bits; struct intel_dp *intel_dp = enc_to_intel_dp(encoder); - if (origin == ORIGIN_FLIP) { - tgl_dc3co_flush(intel_dp, frontbuffer_bits, origin); - continue; - } - mutex_lock(&intel_dp->psr.lock); if (!intel_dp->psr.enabled) { mutex_unlock(&intel_dp->psr.lock); @@ -2143,6 +2152,14 @@ void intel_psr_flush(struct drm_i915_private *dev_priv, continue; } + if (origin == ORIGIN_FLIP || + (origin == ORIGIN_CURSOR_UPDATE && + !intel_dp->psr.psr2_sel_fetch_enabled)) { + tgl_dc3co_flush_locked(intel_dp, frontbuffer_bits, origin); + mutex_unlock(&intel_dp->psr.lock); + continue; + } + /* By definition flush = invalidate + flush */ if (pipe_frontbuffer_bits) psr_force_hw_tracking_exit(intel_dp); @@ -2186,23 +2203,12 @@ void intel_psr_init(struct intel_dp *intel_dp) intel_dp->psr.source_support = true; - if (IS_HASWELL(dev_priv)) - /* - * HSW don't have PSR registers on the same space as transcoder - * so set this to a value that when subtract to the register - * in transcoder space results in the right offset for HSW - */ - dev_priv->hsw_psr_mmio_adjust = _SRD_CTL_EDP - _HSW_EDP_PSR_BASE; - if (dev_priv->params.enable_psr == -1) - if (DISPLAY_VER(dev_priv) < 9 || !dev_priv->vbt.psr.enable) + if (!dev_priv->vbt.psr.enable) dev_priv->params.enable_psr = 0; /* Set link_standby x link_off defaults */ - if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) - /* HSW and BDW require workarounds that we don't implement. */ - intel_dp->psr.link_standby = false; - else if (DISPLAY_VER(dev_priv) < 12) + if (DISPLAY_VER(dev_priv) < 12) /* For new platforms up to TGL let's respect VBT back again */ intel_dp->psr.link_standby = dev_priv->vbt.psr.full_link; diff --git a/drivers/gpu/drm/i915/display/intel_psr.h b/drivers/gpu/drm/i915/display/intel_psr.h index 641521b101c8..facffbacd357 100644 --- a/drivers/gpu/drm/i915/display/intel_psr.h +++ b/drivers/gpu/drm/i915/display/intel_psr.h @@ -20,14 +20,10 @@ struct intel_plane; struct intel_encoder; void intel_psr_init_dpcd(struct intel_dp *intel_dp); -void intel_psr_enable(struct intel_dp *intel_dp, - const struct intel_crtc_state *crtc_state, - const struct drm_connector_state *conn_state); +void intel_psr_pre_plane_update(const struct intel_atomic_state *state); +void intel_psr_post_plane_update(const struct intel_atomic_state *state); void intel_psr_disable(struct intel_dp *intel_dp, const struct intel_crtc_state *old_crtc_state); -void intel_psr_update(struct intel_dp *intel_dp, - const struct intel_crtc_state *crtc_state, - const struct drm_connector_state *conn_state); int intel_psr_debug_set(struct intel_dp *intel_dp, u64 value); void intel_psr_invalidate(struct drm_i915_private *dev_priv, unsigned frontbuffer_bits, @@ -37,7 +33,8 @@ void intel_psr_flush(struct drm_i915_private *dev_priv, enum fb_op_origin origin); void intel_psr_init(struct intel_dp *intel_dp); void intel_psr_compute_config(struct intel_dp *intel_dp, - struct intel_crtc_state *crtc_state); + struct intel_crtc_state *crtc_state, + struct drm_connector_state *conn_state); void intel_psr_get_config(struct intel_encoder *encoder, struct intel_crtc_state *pipe_config); void intel_psr_irq_handler(struct intel_dp *intel_dp, u32 psr_iir); @@ -51,6 +48,8 @@ void intel_psr2_program_plane_sel_fetch(struct intel_plane *plane, const struct intel_crtc_state *crtc_state, const struct intel_plane_state *plane_state, int color_plane); +void intel_psr2_disable_plane_sel_fetch(struct intel_plane *plane, + const struct intel_crtc_state *crtc_state); void intel_psr_pause(struct intel_dp *intel_dp); void intel_psr_resume(struct intel_dp *intel_dp); diff --git a/drivers/gpu/drm/i915/display/intel_sdvo.c b/drivers/gpu/drm/i915/display/intel_sdvo.c index 6cb27599ea03..2dc6c3742ba2 100644 --- a/drivers/gpu/drm/i915/display/intel_sdvo.c +++ b/drivers/gpu/drm/i915/display/intel_sdvo.c @@ -1335,6 +1335,13 @@ static int intel_sdvo_compute_config(struct intel_encoder *encoder, adjusted_mode); pipe_config->sdvo_tv_clock = true; } else if (IS_LVDS(intel_sdvo_connector)) { + int ret; + + ret = intel_panel_compute_config(&intel_sdvo_connector->base, + adjusted_mode); + if (ret) + return ret; + if (!intel_sdvo_set_output_timings_from_mode(intel_sdvo, intel_sdvo_connector->base.panel.fixed_mode)) return -EINVAL; @@ -1873,7 +1880,6 @@ intel_sdvo_mode_valid(struct drm_connector *connector, if (mode->flags & DRM_MODE_FLAG_DBLSCAN) return MODE_NO_DBLESCAN; - if (clock > max_dotclk) return MODE_CLOCK_HIGH; @@ -1890,14 +1896,11 @@ intel_sdvo_mode_valid(struct drm_connector *connector, return MODE_CLOCK_HIGH; if (IS_LVDS(intel_sdvo_connector)) { - const struct drm_display_mode *fixed_mode = - intel_sdvo_connector->base.panel.fixed_mode; - - if (mode->hdisplay > fixed_mode->hdisplay) - return MODE_PANEL; + enum drm_mode_status status; - if (mode->vdisplay > fixed_mode->vdisplay) - return MODE_PANEL; + status = intel_panel_mode_valid(&intel_sdvo_connector->base, mode); + if (status != MODE_OK) + return status; } return MODE_OK; diff --git a/drivers/gpu/drm/i915/display/intel_snps_phy.c b/drivers/gpu/drm/i915/display/intel_snps_phy.c index 18b52b64af95..5e20f340730f 100644 --- a/drivers/gpu/drm/i915/display/intel_snps_phy.c +++ b/drivers/gpu/drm/i915/display/intel_snps_phy.c @@ -5,6 +5,8 @@ #include <linux/util_macros.h> +#include "intel_ddi.h" +#include "intel_ddi_buf_trans.h" #include "intel_de.h" #include "intel_display_types.h" #include "intel_snps_phy.h" @@ -50,58 +52,28 @@ void intel_snps_phy_update_psr_power_state(struct drm_i915_private *dev_priv, SNPS_PHY_TX_REQ_LN_DIS_PWR_STATE_PSR, val); } -static const u32 dg2_ddi_translations[] = { - /* VS 0, pre-emph 0 */ - REG_FIELD_PREP(SNPS_PHY_TX_EQ_MAIN, 26), - - /* VS 0, pre-emph 1 */ - REG_FIELD_PREP(SNPS_PHY_TX_EQ_MAIN, 33) | - REG_FIELD_PREP(SNPS_PHY_TX_EQ_POST, 6), - - /* VS 0, pre-emph 2 */ - REG_FIELD_PREP(SNPS_PHY_TX_EQ_MAIN, 38) | - REG_FIELD_PREP(SNPS_PHY_TX_EQ_POST, 12), - - /* VS 0, pre-emph 3 */ - REG_FIELD_PREP(SNPS_PHY_TX_EQ_MAIN, 43) | - REG_FIELD_PREP(SNPS_PHY_TX_EQ_POST, 19), - - /* VS 1, pre-emph 0 */ - REG_FIELD_PREP(SNPS_PHY_TX_EQ_MAIN, 39), - - /* VS 1, pre-emph 1 */ - REG_FIELD_PREP(SNPS_PHY_TX_EQ_MAIN, 44) | - REG_FIELD_PREP(SNPS_PHY_TX_EQ_POST, 8), - - /* VS 1, pre-emph 2 */ - REG_FIELD_PREP(SNPS_PHY_TX_EQ_MAIN, 47) | - REG_FIELD_PREP(SNPS_PHY_TX_EQ_POST, 15), - - /* VS 2, pre-emph 0 */ - REG_FIELD_PREP(SNPS_PHY_TX_EQ_MAIN, 52), - - /* VS 2, pre-emph 1 */ - REG_FIELD_PREP(SNPS_PHY_TX_EQ_MAIN, 51) | - REG_FIELD_PREP(SNPS_PHY_TX_EQ_POST, 10), - - /* VS 3, pre-emph 0 */ - REG_FIELD_PREP(SNPS_PHY_TX_EQ_MAIN, 62), -}; - -void intel_snps_phy_ddi_vswing_sequence(struct intel_encoder *encoder, - u32 level) +void intel_snps_phy_set_signal_levels(struct intel_encoder *encoder, + const struct intel_crtc_state *crtc_state) { struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); + const struct intel_ddi_buf_trans *trans; enum phy phy = intel_port_to_phy(dev_priv, encoder->port); + int level = intel_ddi_level(encoder, crtc_state, 0); int n_entries, ln; - n_entries = ARRAY_SIZE(dg2_ddi_translations); - if (level >= n_entries) - level = n_entries - 1; + trans = encoder->get_buf_trans(encoder, crtc_state, &n_entries); + if (drm_WARN_ON_ONCE(&dev_priv->drm, !trans)) + return; + + for (ln = 0; ln < 4; ln++) { + u32 val = 0; + + val |= REG_FIELD_PREP(SNPS_PHY_TX_EQ_MAIN, trans->entries[level].snps.vswing); + val |= REG_FIELD_PREP(SNPS_PHY_TX_EQ_PRE, trans->entries[level].snps.pre_cursor); + val |= REG_FIELD_PREP(SNPS_PHY_TX_EQ_POST, trans->entries[level].snps.post_cursor); - for (ln = 0; ln < 4; ln++) - intel_de_write(dev_priv, SNPS_PHY_TX_EQ(ln, phy), - dg2_ddi_translations[level]); + intel_de_write(dev_priv, SNPS_PHY_TX_EQ(ln, phy), val); + } } /* @@ -198,11 +170,81 @@ static const struct intel_mpllb_state dg2_dp_hbr3_100 = { REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_DEN, 1), }; -static const struct intel_mpllb_state *dg2_dp_100_tables[] = { +static const struct intel_mpllb_state dg2_dp_uhbr10_100 = { + .clock = 1000000, + .ref_control = + REG_FIELD_PREP(SNPS_PHY_REF_CONTROL_REF_RANGE, 3), + .mpllb_cp = + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT, 4) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP, 21) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT_GS, 65) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP_GS, 127), + .mpllb_div = + REG_FIELD_PREP(SNPS_PHY_MPLLB_DIV5_CLK_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_DIV_CLK_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_DIV_MULTIPLIER, 8) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_PMIX_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_WORD_DIV2_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_DP2_MODE, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_V2I, 2), + .mpllb_div2 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_REF_CLK_DIV, 2) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_MULTIPLIER, 368), + .mpllb_fracn1 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_CGG_UPDATE_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_DEN, 1), + + /* + * SSC will be enabled, DP UHBR has a minimum SSC requirement. + */ + .mpllb_sscen = + REG_FIELD_PREP(SNPS_PHY_MPLLB_SSC_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_SSC_PEAK, 58982), + .mpllb_sscstep = + REG_FIELD_PREP(SNPS_PHY_MPLLB_SSC_STEPSIZE, 76101), +}; + +static const struct intel_mpllb_state dg2_dp_uhbr13_100 = { + .clock = 1350000, + .ref_control = + REG_FIELD_PREP(SNPS_PHY_REF_CONTROL_REF_RANGE, 3), + .mpllb_cp = + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT, 5) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP, 45) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT_GS, 65) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP_GS, 127), + .mpllb_div = + REG_FIELD_PREP(SNPS_PHY_MPLLB_DIV5_CLK_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_DIV_CLK_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_DIV_MULTIPLIER, 8) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_PMIX_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_WORD_DIV2_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_DP2_MODE, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_V2I, 3), + .mpllb_div2 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_REF_CLK_DIV, 2) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_MULTIPLIER, 508), + .mpllb_fracn1 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_CGG_UPDATE_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_DEN, 1), + + /* + * SSC will be enabled, DP UHBR has a minimum SSC requirement. + */ + .mpllb_sscen = + REG_FIELD_PREP(SNPS_PHY_MPLLB_SSC_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_SSC_PEAK, 79626), + .mpllb_sscstep = + REG_FIELD_PREP(SNPS_PHY_MPLLB_SSC_STEPSIZE, 102737), +}; + +static const struct intel_mpllb_state * const dg2_dp_100_tables[] = { &dg2_dp_rbr_100, &dg2_dp_hbr1_100, &dg2_dp_hbr2_100, &dg2_dp_hbr3_100, + &dg2_dp_uhbr10_100, + &dg2_dp_uhbr13_100, NULL, }; @@ -311,11 +353,88 @@ static const struct intel_mpllb_state dg2_dp_hbr3_38_4 = { REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_QUOT, 61440), }; -static const struct intel_mpllb_state *dg2_dp_38_4_tables[] = { +static const struct intel_mpllb_state dg2_dp_uhbr10_38_4 = { + .clock = 1000000, + .ref_control = + REG_FIELD_PREP(SNPS_PHY_REF_CONTROL_REF_RANGE, 1), + .mpllb_cp = + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT, 5) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP, 26) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT_GS, 65) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP_GS, 127), + .mpllb_div = + REG_FIELD_PREP(SNPS_PHY_MPLLB_DIV5_CLK_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_DIV_CLK_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_DIV_MULTIPLIER, 8) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_PMIX_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_WORD_DIV2_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_DP2_MODE, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_V2I, 2), + .mpllb_div2 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_REF_CLK_DIV, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_MULTIPLIER, 488), + .mpllb_fracn1 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_CGG_UPDATE_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_DEN, 3), + .mpllb_fracn2 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_REM, 2) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_QUOT, 27306), + + /* + * SSC will be enabled, DP UHBR has a minimum SSC requirement. + */ + .mpllb_sscen = + REG_FIELD_PREP(SNPS_PHY_MPLLB_SSC_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_SSC_PEAK, 76800), + .mpllb_sscstep = + REG_FIELD_PREP(SNPS_PHY_MPLLB_SSC_STEPSIZE, 129024), +}; + +static const struct intel_mpllb_state dg2_dp_uhbr13_38_4 = { + .clock = 1350000, + .ref_control = + REG_FIELD_PREP(SNPS_PHY_REF_CONTROL_REF_RANGE, 1), + .mpllb_cp = + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT, 6) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP, 56) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT_GS, 65) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP_GS, 127), + .mpllb_div = + REG_FIELD_PREP(SNPS_PHY_MPLLB_DIV5_CLK_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_DIV_CLK_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_DIV_MULTIPLIER, 8) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_PMIX_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_WORD_DIV2_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_DP2_MODE, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_V2I, 3), + .mpllb_div2 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_REF_CLK_DIV, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_MULTIPLIER, 670), + .mpllb_fracn1 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_CGG_UPDATE_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_DEN, 1), + .mpllb_fracn2 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_QUOT, 36864), + + /* + * SSC will be enabled, DP UHBR has a minimum SSC requirement. + */ + .mpllb_sscen = + REG_FIELD_PREP(SNPS_PHY_MPLLB_SSC_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_SSC_PEAK, 103680), + .mpllb_sscstep = + REG_FIELD_PREP(SNPS_PHY_MPLLB_SSC_STEPSIZE, 174182), +}; + +static const struct intel_mpllb_state * const dg2_dp_38_4_tables[] = { &dg2_dp_rbr_38_4, &dg2_dp_hbr1_38_4, &dg2_dp_hbr2_38_4, &dg2_dp_hbr3_38_4, + &dg2_dp_uhbr10_38_4, + &dg2_dp_uhbr13_38_4, NULL, }; @@ -448,7 +567,7 @@ static const struct intel_mpllb_state dg2_edp_r432 = { REG_FIELD_PREP(SNPS_PHY_MPLLB_SSC_STEPSIZE, 65752), }; -static const struct intel_mpllb_state *dg2_edp_tables[] = { +static const struct intel_mpllb_state * const dg2_edp_tables[] = { &dg2_dp_rbr_100, &dg2_edp_r216, &dg2_edp_r243, @@ -611,7 +730,7 @@ static const struct intel_mpllb_state dg2_hdmi_594 = { REG_FIELD_PREP(SNPS_PHY_MPLLB_SSC_UP_SPREAD, 1), }; -static const struct intel_mpllb_state *dg2_hdmi_tables[] = { +static const struct intel_mpllb_state * const dg2_hdmi_tables[] = { &dg2_hdmi_25_175, &dg2_hdmi_27_0, &dg2_hdmi_74_25, @@ -620,7 +739,7 @@ static const struct intel_mpllb_state *dg2_hdmi_tables[] = { NULL, }; -static const struct intel_mpllb_state ** +static const struct intel_mpllb_state * const * intel_mpllb_tables_get(struct intel_crtc_state *crtc_state, struct intel_encoder *encoder) { @@ -654,7 +773,7 @@ intel_mpllb_tables_get(struct intel_crtc_state *crtc_state, int intel_mpllb_calc_state(struct intel_crtc_state *crtc_state, struct intel_encoder *encoder) { - const struct intel_mpllb_state **tables; + const struct intel_mpllb_state * const *tables; int i; if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI)) { @@ -850,7 +969,7 @@ void intel_mpllb_readout_hw_state(struct intel_encoder *encoder, int intel_snps_phy_check_hdmi_link_rate(int clock) { - const struct intel_mpllb_state **tables = dg2_hdmi_tables; + const struct intel_mpllb_state * const *tables = dg2_hdmi_tables; int i; for (i = 0; tables[i]; i++) { diff --git a/drivers/gpu/drm/i915/display/intel_snps_phy.h b/drivers/gpu/drm/i915/display/intel_snps_phy.h index 6261ff88ef5c..11dcd6deb070 100644 --- a/drivers/gpu/drm/i915/display/intel_snps_phy.h +++ b/drivers/gpu/drm/i915/display/intel_snps_phy.h @@ -29,7 +29,7 @@ int intel_mpllb_calc_port_clock(struct intel_encoder *encoder, const struct intel_mpllb_state *pll_state); int intel_snps_phy_check_hdmi_link_rate(int clock); -void intel_snps_phy_ddi_vswing_sequence(struct intel_encoder *encoder, - u32 level); +void intel_snps_phy_set_signal_levels(struct intel_encoder *encoder, + const struct intel_crtc_state *crtc_state); #endif /* __INTEL_SNPS_PHY_H__ */ diff --git a/drivers/gpu/drm/i915/display/intel_tc.c b/drivers/gpu/drm/i915/display/intel_tc.c index 3ffece568ed9..40faa18947c9 100644 --- a/drivers/gpu/drm/i915/display/intel_tc.c +++ b/drivers/gpu/drm/i915/display/intel_tc.c @@ -12,44 +12,81 @@ static const char *tc_port_mode_name(enum tc_port_mode mode) { static const char * const names[] = { + [TC_PORT_DISCONNECTED] = "disconnected", [TC_PORT_TBT_ALT] = "tbt-alt", [TC_PORT_DP_ALT] = "dp-alt", [TC_PORT_LEGACY] = "legacy", }; if (WARN_ON(mode >= ARRAY_SIZE(names))) - mode = TC_PORT_TBT_ALT; + mode = TC_PORT_DISCONNECTED; return names[mode]; } +static bool intel_tc_port_in_mode(struct intel_digital_port *dig_port, + enum tc_port_mode mode) +{ + struct drm_i915_private *i915 = to_i915(dig_port->base.base.dev); + enum phy phy = intel_port_to_phy(i915, dig_port->base.port); + + return intel_phy_is_tc(i915, phy) && dig_port->tc_mode == mode; +} + +bool intel_tc_port_in_tbt_alt_mode(struct intel_digital_port *dig_port) +{ + return intel_tc_port_in_mode(dig_port, TC_PORT_TBT_ALT); +} + +bool intel_tc_port_in_dp_alt_mode(struct intel_digital_port *dig_port) +{ + return intel_tc_port_in_mode(dig_port, TC_PORT_DP_ALT); +} + +bool intel_tc_port_in_legacy_mode(struct intel_digital_port *dig_port) +{ + return intel_tc_port_in_mode(dig_port, TC_PORT_LEGACY); +} + +bool intel_tc_cold_requires_aux_pw(struct intel_digital_port *dig_port) +{ + struct drm_i915_private *i915 = to_i915(dig_port->base.base.dev); + + return (DISPLAY_VER(i915) == 11 && dig_port->tc_legacy_port) || + IS_ALDERLAKE_P(i915); +} + static enum intel_display_power_domain -tc_cold_get_power_domain(struct intel_digital_port *dig_port) +tc_cold_get_power_domain(struct intel_digital_port *dig_port, enum tc_port_mode mode) { - if (intel_tc_cold_requires_aux_pw(dig_port)) - return intel_legacy_aux_to_power_domain(dig_port->aux_ch); - else + if (mode == TC_PORT_TBT_ALT || !intel_tc_cold_requires_aux_pw(dig_port)) return POWER_DOMAIN_TC_COLD_OFF; + + return intel_legacy_aux_to_power_domain(dig_port->aux_ch); } static intel_wakeref_t -tc_cold_block(struct intel_digital_port *dig_port) +tc_cold_block_in_mode(struct intel_digital_port *dig_port, enum tc_port_mode mode, + enum intel_display_power_domain *domain) { struct drm_i915_private *i915 = to_i915(dig_port->base.base.dev); - enum intel_display_power_domain domain; - if (DISPLAY_VER(i915) == 11 && !dig_port->tc_legacy_port) - return 0; + *domain = tc_cold_get_power_domain(dig_port, mode); - domain = tc_cold_get_power_domain(dig_port); - return intel_display_power_get(i915, domain); + return intel_display_power_get(i915, *domain); +} + +static intel_wakeref_t +tc_cold_block(struct intel_digital_port *dig_port, enum intel_display_power_domain *domain) +{ + return tc_cold_block_in_mode(dig_port, dig_port->tc_mode, domain); } static void -tc_cold_unblock(struct intel_digital_port *dig_port, intel_wakeref_t wakeref) +tc_cold_unblock(struct intel_digital_port *dig_port, enum intel_display_power_domain domain, + intel_wakeref_t wakeref) { struct drm_i915_private *i915 = to_i915(dig_port->base.base.dev); - enum intel_display_power_domain domain; /* * wakeref == -1, means some error happened saving save_depot_stack but @@ -59,8 +96,7 @@ tc_cold_unblock(struct intel_digital_port *dig_port, intel_wakeref_t wakeref) if (wakeref == 0) return; - domain = tc_cold_get_power_domain(dig_port); - intel_display_power_put_async(i915, domain, wakeref); + intel_display_power_put(i915, domain, wakeref); } static void @@ -69,11 +105,9 @@ assert_tc_cold_blocked(struct intel_digital_port *dig_port) struct drm_i915_private *i915 = to_i915(dig_port->base.base.dev); bool enabled; - if (DISPLAY_VER(i915) == 11 && !dig_port->tc_legacy_port) - return; - enabled = intel_display_power_is_enabled(i915, - tc_cold_get_power_domain(dig_port)); + tc_cold_get_power_domain(dig_port, + dig_port->tc_mode)); drm_WARN_ON(&i915->drm, !enabled); } @@ -244,6 +278,11 @@ static u32 adl_tc_port_live_status_mask(struct intel_digital_port *dig_port) struct intel_uncore *uncore = &i915->uncore; u32 val, mask = 0; + /* + * On ADL-P HW/FW will wake from TCCOLD to complete the read access of + * registers in IOM. Note that this doesn't apply to PHY and FIA + * registers. + */ val = intel_uncore_read(uncore, TCSS_DDI_STATUS(tc_port)); if (val & TCSS_DDI_STATUS_HPD_LIVE_STATUS_ALT) mask |= BIT(TC_PORT_DP_ALT); @@ -270,6 +309,14 @@ static u32 tc_port_live_status_mask(struct intel_digital_port *dig_port) return icl_tc_port_live_status_mask(dig_port); } +/* + * Return the PHY status complete flag indicating that display can acquire the + * PHY ownership. The IOM firmware sets this flag when a DP-alt or legacy sink + * is connected and it's ready to switch the ownership to display. The flag + * will be left cleared when a TBT-alt sink is connected, where the PHY is + * owned by the TBT subsystem and so switching the ownership to display is not + * required. + */ static bool icl_tc_phy_status_complete(struct intel_digital_port *dig_port) { struct drm_i915_private *i915 = to_i915(dig_port->base.base.dev); @@ -288,6 +335,13 @@ static bool icl_tc_phy_status_complete(struct intel_digital_port *dig_port) return val & DP_PHY_MODE_STATUS_COMPLETED(dig_port->tc_phy_fia_idx); } +/* + * Return the PHY status complete flag indicating that display can acquire the + * PHY ownership. The IOM firmware sets this flag when it's ready to switch + * the ownership to display, regardless of what sink is connected (TBT-alt, + * DP-alt, legacy or nothing). For TBT-alt sinks the PHY is owned by the TBT + * subsystem and so switching the ownership to display is not required. + */ static bool adl_tc_phy_status_complete(struct intel_digital_port *dig_port) { struct drm_i915_private *i915 = to_i915(dig_port->base.base.dev); @@ -339,11 +393,6 @@ static bool icl_tc_phy_take_ownership(struct intel_digital_port *dig_port, intel_uncore_write(uncore, PORT_TX_DFLEXDPCSSS(dig_port->tc_phy_fia), val); - if (!take && wait_for(!tc_phy_status_complete(dig_port), 10)) - drm_dbg_kms(&i915->drm, - "Port %s: PHY complete clear timed out\n", - dig_port->tc_port_name); - return true; } @@ -429,6 +478,7 @@ static void icl_tc_phy_connect(struct intel_digital_port *dig_port, int required_lanes) { struct drm_i915_private *i915 = to_i915(dig_port->base.base.dev); + u32 live_status_mask; int max_lanes; if (!tc_phy_status_complete(dig_port)) { @@ -437,6 +487,13 @@ static void icl_tc_phy_connect(struct intel_digital_port *dig_port, goto out_set_tbt_alt_mode; } + live_status_mask = tc_port_live_status_mask(dig_port); + if (!(live_status_mask & (BIT(TC_PORT_DP_ALT) | BIT(TC_PORT_LEGACY)))) { + drm_dbg_kms(&i915->drm, "Port %s: PHY ownership not required (live status %02x)\n", + dig_port->tc_port_name, live_status_mask); + goto out_set_tbt_alt_mode; + } + if (!tc_phy_take_ownership(dig_port, true) && !drm_WARN_ON(&i915->drm, dig_port->tc_legacy_port)) goto out_set_tbt_alt_mode; @@ -485,14 +542,13 @@ static void icl_tc_phy_disconnect(struct intel_digital_port *dig_port) { switch (dig_port->tc_mode) { case TC_PORT_LEGACY: - /* Nothing to do, we never disconnect from legacy mode */ - break; case TC_PORT_DP_ALT: tc_phy_take_ownership(dig_port, false); - dig_port->tc_mode = TC_PORT_TBT_ALT; - break; + fallthrough; case TC_PORT_TBT_ALT: - /* Nothing to do, we stay in TBT-alt mode */ + dig_port->tc_mode = TC_PORT_DISCONNECTED; + fallthrough; + case TC_PORT_DISCONNECTED: break; default: MISSING_CASE(dig_port->tc_mode); @@ -509,6 +565,10 @@ static bool icl_tc_phy_is_connected(struct intel_digital_port *dig_port) return dig_port->tc_mode == TC_PORT_TBT_ALT; } + /* On ADL-P the PHY complete flag is set in TBT mode as well. */ + if (IS_ALDERLAKE_P(i915) && dig_port->tc_mode == TC_PORT_TBT_ALT) + return true; + if (!tc_phy_is_owned(dig_port)) { drm_dbg_kms(&i915->drm, "Port %s: PHY not owned\n", dig_port->tc_port_name); @@ -550,9 +610,7 @@ intel_tc_port_get_target_mode(struct intel_digital_port *dig_port) if (live_status_mask) return fls(live_status_mask) - 1; - return tc_phy_status_complete(dig_port) && - dig_port->tc_legacy_port ? TC_PORT_LEGACY : - TC_PORT_TBT_ALT; + return TC_PORT_TBT_ALT; } static void intel_tc_port_reset_mode(struct intel_digital_port *dig_port, @@ -581,6 +639,43 @@ static void intel_tc_port_reset_mode(struct intel_digital_port *dig_port, tc_port_mode_name(dig_port->tc_mode)); } +static bool intel_tc_port_needs_reset(struct intel_digital_port *dig_port) +{ + return intel_tc_port_get_target_mode(dig_port) != dig_port->tc_mode; +} + +static void intel_tc_port_update_mode(struct intel_digital_port *dig_port, + int required_lanes, bool force_disconnect) +{ + enum intel_display_power_domain domain; + intel_wakeref_t wref; + bool needs_reset = force_disconnect; + + if (!needs_reset) { + /* Get power domain required to check the hotplug live status. */ + wref = tc_cold_block(dig_port, &domain); + needs_reset = intel_tc_port_needs_reset(dig_port); + tc_cold_unblock(dig_port, domain, wref); + } + + if (!needs_reset) + return; + + /* Get power domain required for resetting the mode. */ + wref = tc_cold_block_in_mode(dig_port, TC_PORT_DISCONNECTED, &domain); + + intel_tc_port_reset_mode(dig_port, required_lanes, force_disconnect); + + /* Get power domain matching the new mode after reset. */ + tc_cold_unblock(dig_port, dig_port->tc_lock_power_domain, + fetch_and_zero(&dig_port->tc_lock_wakeref)); + if (dig_port->tc_mode != TC_PORT_DISCONNECTED) + dig_port->tc_lock_wakeref = tc_cold_block(dig_port, + &dig_port->tc_lock_power_domain); + + tc_cold_unblock(dig_port, domain, wref); +} + static void intel_tc_port_link_init_refcount(struct intel_digital_port *dig_port, int refcount) @@ -595,45 +690,42 @@ void intel_tc_port_sanitize(struct intel_digital_port *dig_port) { struct drm_i915_private *i915 = to_i915(dig_port->base.base.dev); struct intel_encoder *encoder = &dig_port->base; - intel_wakeref_t tc_cold_wref; int active_links = 0; mutex_lock(&dig_port->tc_lock); - tc_cold_wref = tc_cold_block(dig_port); - dig_port->tc_mode = intel_tc_port_get_current_mode(dig_port); if (dig_port->dp.is_mst) active_links = intel_dp_mst_encoder_active_links(dig_port); else if (encoder->base.crtc) active_links = to_intel_crtc(encoder->base.crtc)->active; + drm_WARN_ON(&i915->drm, dig_port->tc_mode != TC_PORT_DISCONNECTED); + drm_WARN_ON(&i915->drm, dig_port->tc_lock_wakeref); if (active_links) { + enum intel_display_power_domain domain; + intel_wakeref_t tc_cold_wref = tc_cold_block(dig_port, &domain); + + dig_port->tc_mode = intel_tc_port_get_current_mode(dig_port); + if (!icl_tc_phy_is_connected(dig_port)) drm_dbg_kms(&i915->drm, "Port %s: PHY disconnected with %d active link(s)\n", dig_port->tc_port_name, active_links); intel_tc_port_link_init_refcount(dig_port, active_links); - goto out; - } + dig_port->tc_lock_wakeref = tc_cold_block(dig_port, + &dig_port->tc_lock_power_domain); - if (dig_port->tc_legacy_port) - icl_tc_phy_connect(dig_port, 1); + tc_cold_unblock(dig_port, domain, tc_cold_wref); + } -out: drm_dbg_kms(&i915->drm, "Port %s: sanitize mode (%s)\n", dig_port->tc_port_name, tc_port_mode_name(dig_port->tc_mode)); - tc_cold_unblock(dig_port, tc_cold_wref); mutex_unlock(&dig_port->tc_lock); } -static bool intel_tc_port_needs_reset(struct intel_digital_port *dig_port) -{ - return intel_tc_port_get_target_mode(dig_port) != dig_port->tc_mode; -} - /* * The type-C ports are different because even when they are connected, they may * not be available/usable by the graphics driver: see the comment on @@ -648,78 +740,79 @@ bool intel_tc_port_connected(struct intel_encoder *encoder) { struct intel_digital_port *dig_port = enc_to_dig_port(encoder); bool is_connected; - intel_wakeref_t tc_cold_wref; intel_tc_port_lock(dig_port); - tc_cold_wref = tc_cold_block(dig_port); is_connected = tc_port_live_status_mask(dig_port) & BIT(dig_port->tc_mode); - tc_cold_unblock(dig_port, tc_cold_wref); intel_tc_port_unlock(dig_port); return is_connected; } static void __intel_tc_port_lock(struct intel_digital_port *dig_port, - int required_lanes, bool force_disconnect) + int required_lanes) { struct drm_i915_private *i915 = to_i915(dig_port->base.base.dev); - intel_wakeref_t wakeref; - - wakeref = intel_display_power_get(i915, POWER_DOMAIN_DISPLAY_CORE); mutex_lock(&dig_port->tc_lock); - if (!dig_port->tc_link_refcount) { - intel_wakeref_t tc_cold_wref; - - tc_cold_wref = tc_cold_block(dig_port); + cancel_delayed_work(&dig_port->tc_disconnect_phy_work); - if (force_disconnect || intel_tc_port_needs_reset(dig_port)) - intel_tc_port_reset_mode(dig_port, required_lanes, - force_disconnect); - - tc_cold_unblock(dig_port, tc_cold_wref); - } + if (!dig_port->tc_link_refcount) + intel_tc_port_update_mode(dig_port, required_lanes, + false); - drm_WARN_ON(&i915->drm, dig_port->tc_lock_wakeref); - dig_port->tc_lock_wakeref = wakeref; + drm_WARN_ON(&i915->drm, dig_port->tc_mode == TC_PORT_DISCONNECTED); + drm_WARN_ON(&i915->drm, dig_port->tc_mode != TC_PORT_TBT_ALT && + !tc_phy_is_owned(dig_port)); } void intel_tc_port_lock(struct intel_digital_port *dig_port) { - __intel_tc_port_lock(dig_port, 1, false); + __intel_tc_port_lock(dig_port, 1); } -void intel_tc_port_unlock(struct intel_digital_port *dig_port) +/** + * intel_tc_port_disconnect_phy_work: disconnect TypeC PHY from display port + * @dig_port: digital port + * + * Disconnect the given digital port from its TypeC PHY (handing back the + * control of the PHY to the TypeC subsystem). This will happen in a delayed + * manner after each aux transactions and modeset disables. + */ +static void intel_tc_port_disconnect_phy_work(struct work_struct *work) { - struct drm_i915_private *i915 = to_i915(dig_port->base.base.dev); - intel_wakeref_t wakeref = fetch_and_zero(&dig_port->tc_lock_wakeref); + struct intel_digital_port *dig_port = + container_of(work, struct intel_digital_port, tc_disconnect_phy_work.work); - mutex_unlock(&dig_port->tc_lock); + mutex_lock(&dig_port->tc_lock); + + if (!dig_port->tc_link_refcount) + intel_tc_port_update_mode(dig_port, 1, true); - intel_display_power_put_async(i915, POWER_DOMAIN_DISPLAY_CORE, - wakeref); + mutex_unlock(&dig_port->tc_lock); } /** - * intel_tc_port_disconnect_phy: disconnect TypeC PHY from display port + * intel_tc_port_flush_work: flush the work disconnecting the PHY * @dig_port: digital port * - * Disconnect the given digital port from its TypeC PHY (handing back the - * control of the PHY to the TypeC subsystem). The only purpose of this - * function is to force the disconnect even with a TypeC display output still - * plugged to the TypeC connector, which is required by the TypeC firmwares - * during system suspend and shutdown. Otherwise - during the unplug event - * handling - the PHY ownership is released automatically by - * intel_tc_port_reset_mode(), when calling this function is not required. + * Flush the delayed work disconnecting an idle PHY. */ -void intel_tc_port_disconnect_phy(struct intel_digital_port *dig_port) +void intel_tc_port_flush_work(struct intel_digital_port *dig_port) { - __intel_tc_port_lock(dig_port, 1, true); - intel_tc_port_unlock(dig_port); + flush_delayed_work(&dig_port->tc_disconnect_phy_work); +} + +void intel_tc_port_unlock(struct intel_digital_port *dig_port) +{ + if (!dig_port->tc_link_refcount && dig_port->tc_mode != TC_PORT_DISCONNECTED) + queue_delayed_work(system_unbound_wq, &dig_port->tc_disconnect_phy_work, + msecs_to_jiffies(1000)); + + mutex_unlock(&dig_port->tc_lock); } bool intel_tc_port_ref_held(struct intel_digital_port *dig_port) @@ -731,21 +824,30 @@ bool intel_tc_port_ref_held(struct intel_digital_port *dig_port) void intel_tc_port_get_link(struct intel_digital_port *dig_port, int required_lanes) { - __intel_tc_port_lock(dig_port, required_lanes, false); + __intel_tc_port_lock(dig_port, required_lanes); dig_port->tc_link_refcount++; intel_tc_port_unlock(dig_port); } void intel_tc_port_put_link(struct intel_digital_port *dig_port) { - mutex_lock(&dig_port->tc_lock); - dig_port->tc_link_refcount--; - mutex_unlock(&dig_port->tc_lock); + intel_tc_port_lock(dig_port); + --dig_port->tc_link_refcount; + intel_tc_port_unlock(dig_port); + + /* + * Disconnecting the PHY after the PHY's PLL gets disabled may + * hang the system on ADL-P, so disconnect the PHY here synchronously. + * TODO: remove this once the root cause of the ordering requirement + * is found/fixed. + */ + intel_tc_port_flush_work(dig_port); } static bool tc_has_modular_fia(struct drm_i915_private *i915, struct intel_digital_port *dig_port) { + enum intel_display_power_domain domain; intel_wakeref_t wakeref; u32 val; @@ -753,9 +855,9 @@ tc_has_modular_fia(struct drm_i915_private *i915, struct intel_digital_port *dig return false; mutex_lock(&dig_port->tc_lock); - wakeref = tc_cold_block(dig_port); + wakeref = tc_cold_block(dig_port, &domain); val = intel_uncore_read(&i915->uncore, PORT_TX_DFLEXDPSP(FIA1)); - tc_cold_unblock(dig_port, wakeref); + tc_cold_unblock(dig_port, domain, wakeref); mutex_unlock(&dig_port->tc_lock); drm_WARN_ON(&i915->drm, val == 0xffffffff); @@ -795,15 +897,9 @@ void intel_tc_port_init(struct intel_digital_port *dig_port, bool is_legacy) "%c/TC#%d", port_name(port), tc_port + 1); mutex_init(&dig_port->tc_lock); + INIT_DELAYED_WORK(&dig_port->tc_disconnect_phy_work, intel_tc_port_disconnect_phy_work); dig_port->tc_legacy_port = is_legacy; + dig_port->tc_mode = TC_PORT_DISCONNECTED; dig_port->tc_link_refcount = 0; tc_port_load_fia_params(i915, dig_port); } - -bool intel_tc_cold_requires_aux_pw(struct intel_digital_port *dig_port) -{ - struct drm_i915_private *i915 = to_i915(dig_port->base.base.dev); - - return (DISPLAY_VER(i915) == 11 && dig_port->tc_legacy_port) || - IS_ALDERLAKE_P(i915); -} diff --git a/drivers/gpu/drm/i915/display/intel_tc.h b/drivers/gpu/drm/i915/display/intel_tc.h index 0c881f645e27..6b47b29f551c 100644 --- a/drivers/gpu/drm/i915/display/intel_tc.h +++ b/drivers/gpu/drm/i915/display/intel_tc.h @@ -12,8 +12,11 @@ struct intel_digital_port; struct intel_encoder; +bool intel_tc_port_in_tbt_alt_mode(struct intel_digital_port *dig_port); +bool intel_tc_port_in_dp_alt_mode(struct intel_digital_port *dig_port); +bool intel_tc_port_in_legacy_mode(struct intel_digital_port *dig_port); + bool intel_tc_port_connected(struct intel_encoder *encoder); -void intel_tc_port_disconnect_phy(struct intel_digital_port *dig_port); u32 intel_tc_port_get_lane_mask(struct intel_digital_port *dig_port); u32 intel_tc_port_get_pin_assignment_mask(struct intel_digital_port *dig_port); @@ -24,6 +27,7 @@ void intel_tc_port_set_fia_lane_count(struct intel_digital_port *dig_port, void intel_tc_port_sanitize(struct intel_digital_port *dig_port); void intel_tc_port_lock(struct intel_digital_port *dig_port); void intel_tc_port_unlock(struct intel_digital_port *dig_port); +void intel_tc_port_flush_work(struct intel_digital_port *dig_port); void intel_tc_port_get_link(struct intel_digital_port *dig_port, int required_lanes); void intel_tc_port_put_link(struct intel_digital_port *dig_port); diff --git a/drivers/gpu/drm/i915/display/intel_tv.c b/drivers/gpu/drm/i915/display/intel_tv.c index d02f09f7e750..88a398df9621 100644 --- a/drivers/gpu/drm/i915/display/intel_tv.c +++ b/drivers/gpu/drm/i915/display/intel_tv.c @@ -1529,7 +1529,7 @@ static void intel_tv_pre_enable(struct intel_atomic_state *state, intel_de_write(dev_priv, TV_CLR_LEVEL, ((video_levels->black << TV_BLACK_LEVEL_SHIFT) | (video_levels->blank << TV_BLANK_LEVEL_SHIFT))); - assert_pipe_disabled(dev_priv, pipe_config->cpu_transcoder); + assert_transcoder_disabled(dev_priv, pipe_config->cpu_transcoder); /* Filter ctl must be set before TV_WIN_SIZE */ tv_filter_ctl = TV_AUTO_SCALE; diff --git a/drivers/gpu/drm/i915/display/intel_vdsc.c b/drivers/gpu/drm/i915/display/intel_vdsc.c index df3286aa6999..2275f99ce9d7 100644 --- a/drivers/gpu/drm/i915/display/intel_vdsc.c +++ b/drivers/gpu/drm/i915/display/intel_vdsc.c @@ -357,11 +357,9 @@ bool intel_dsc_source_support(const struct intel_crtc_state *crtc_state) return false; } -static bool is_pipe_dsc(const struct intel_crtc_state *crtc_state) +static bool is_pipe_dsc(struct intel_crtc *crtc, enum transcoder cpu_transcoder) { - const struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); - const struct drm_i915_private *i915 = to_i915(crtc->base.dev); - enum transcoder cpu_transcoder = crtc_state->cpu_transcoder; + struct drm_i915_private *i915 = to_i915(crtc->base.dev); if (DISPLAY_VER(i915) >= 12) return true; @@ -547,9 +545,8 @@ int intel_dsc_compute_params(struct intel_encoder *encoder, } enum intel_display_power_domain -intel_dsc_power_domain(const struct intel_crtc_state *crtc_state) +intel_dsc_power_domain(struct intel_crtc *crtc, enum transcoder cpu_transcoder) { - struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); struct drm_i915_private *i915 = to_i915(crtc->base.dev); enum pipe pipe = crtc->pipe; @@ -566,7 +563,7 @@ intel_dsc_power_domain(const struct intel_crtc_state *crtc_state) */ if (DISPLAY_VER(i915) == 12 && !IS_ROCKETLAKE(i915) && pipe == PIPE_A) return POWER_DOMAIN_TRANSCODER_VDSC_PW2; - else if (is_pipe_dsc(crtc_state)) + else if (is_pipe_dsc(crtc, cpu_transcoder)) return POWER_DOMAIN_PIPE(pipe); else return POWER_DOMAIN_TRANSCODER_VDSC_PW2; @@ -577,6 +574,7 @@ static void intel_dsc_pps_configure(const struct intel_crtc_state *crtc_state) struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); const struct drm_dsc_config *vdsc_cfg = &crtc_state->dsc.config; + enum transcoder cpu_transcoder = crtc_state->cpu_transcoder; enum pipe pipe = crtc->pipe; u32 pps_val = 0; u32 rc_buf_thresh_dword[4]; @@ -601,7 +599,7 @@ static void intel_dsc_pps_configure(const struct intel_crtc_state *crtc_state) if (vdsc_cfg->vbr_enable) pps_val |= DSC_VBR_ENABLE; drm_info(&dev_priv->drm, "PPS0 = 0x%08x\n", pps_val); - if (!is_pipe_dsc(crtc_state)) { + if (!is_pipe_dsc(crtc, cpu_transcoder)) { intel_de_write(dev_priv, DSCA_PICTURE_PARAMETER_SET_0, pps_val); /* @@ -625,7 +623,7 @@ static void intel_dsc_pps_configure(const struct intel_crtc_state *crtc_state) pps_val = 0; pps_val |= DSC_BPP(vdsc_cfg->bits_per_pixel); drm_info(&dev_priv->drm, "PPS1 = 0x%08x\n", pps_val); - if (!is_pipe_dsc(crtc_state)) { + if (!is_pipe_dsc(crtc, cpu_transcoder)) { intel_de_write(dev_priv, DSCA_PICTURE_PARAMETER_SET_1, pps_val); /* @@ -650,7 +648,7 @@ static void intel_dsc_pps_configure(const struct intel_crtc_state *crtc_state) pps_val |= DSC_PIC_HEIGHT(vdsc_cfg->pic_height) | DSC_PIC_WIDTH(vdsc_cfg->pic_width / num_vdsc_instances); drm_info(&dev_priv->drm, "PPS2 = 0x%08x\n", pps_val); - if (!is_pipe_dsc(crtc_state)) { + if (!is_pipe_dsc(crtc, cpu_transcoder)) { intel_de_write(dev_priv, DSCA_PICTURE_PARAMETER_SET_2, pps_val); /* @@ -675,7 +673,7 @@ static void intel_dsc_pps_configure(const struct intel_crtc_state *crtc_state) pps_val |= DSC_SLICE_HEIGHT(vdsc_cfg->slice_height) | DSC_SLICE_WIDTH(vdsc_cfg->slice_width); drm_info(&dev_priv->drm, "PPS3 = 0x%08x\n", pps_val); - if (!is_pipe_dsc(crtc_state)) { + if (!is_pipe_dsc(crtc, cpu_transcoder)) { intel_de_write(dev_priv, DSCA_PICTURE_PARAMETER_SET_3, pps_val); /* @@ -700,7 +698,7 @@ static void intel_dsc_pps_configure(const struct intel_crtc_state *crtc_state) pps_val |= DSC_INITIAL_XMIT_DELAY(vdsc_cfg->initial_xmit_delay) | DSC_INITIAL_DEC_DELAY(vdsc_cfg->initial_dec_delay); drm_info(&dev_priv->drm, "PPS4 = 0x%08x\n", pps_val); - if (!is_pipe_dsc(crtc_state)) { + if (!is_pipe_dsc(crtc, cpu_transcoder)) { intel_de_write(dev_priv, DSCA_PICTURE_PARAMETER_SET_4, pps_val); /* @@ -725,7 +723,7 @@ static void intel_dsc_pps_configure(const struct intel_crtc_state *crtc_state) pps_val |= DSC_SCALE_INC_INT(vdsc_cfg->scale_increment_interval) | DSC_SCALE_DEC_INT(vdsc_cfg->scale_decrement_interval); drm_info(&dev_priv->drm, "PPS5 = 0x%08x\n", pps_val); - if (!is_pipe_dsc(crtc_state)) { + if (!is_pipe_dsc(crtc, cpu_transcoder)) { intel_de_write(dev_priv, DSCA_PICTURE_PARAMETER_SET_5, pps_val); /* @@ -752,7 +750,7 @@ static void intel_dsc_pps_configure(const struct intel_crtc_state *crtc_state) DSC_FLATNESS_MIN_QP(vdsc_cfg->flatness_min_qp) | DSC_FLATNESS_MAX_QP(vdsc_cfg->flatness_max_qp); drm_info(&dev_priv->drm, "PPS6 = 0x%08x\n", pps_val); - if (!is_pipe_dsc(crtc_state)) { + if (!is_pipe_dsc(crtc, cpu_transcoder)) { intel_de_write(dev_priv, DSCA_PICTURE_PARAMETER_SET_6, pps_val); /* @@ -777,7 +775,7 @@ static void intel_dsc_pps_configure(const struct intel_crtc_state *crtc_state) pps_val |= DSC_SLICE_BPG_OFFSET(vdsc_cfg->slice_bpg_offset) | DSC_NFL_BPG_OFFSET(vdsc_cfg->nfl_bpg_offset); drm_info(&dev_priv->drm, "PPS7 = 0x%08x\n", pps_val); - if (!is_pipe_dsc(crtc_state)) { + if (!is_pipe_dsc(crtc, cpu_transcoder)) { intel_de_write(dev_priv, DSCA_PICTURE_PARAMETER_SET_7, pps_val); /* @@ -802,7 +800,7 @@ static void intel_dsc_pps_configure(const struct intel_crtc_state *crtc_state) pps_val |= DSC_FINAL_OFFSET(vdsc_cfg->final_offset) | DSC_INITIAL_OFFSET(vdsc_cfg->initial_offset); drm_info(&dev_priv->drm, "PPS8 = 0x%08x\n", pps_val); - if (!is_pipe_dsc(crtc_state)) { + if (!is_pipe_dsc(crtc, cpu_transcoder)) { intel_de_write(dev_priv, DSCA_PICTURE_PARAMETER_SET_8, pps_val); /* @@ -827,7 +825,7 @@ static void intel_dsc_pps_configure(const struct intel_crtc_state *crtc_state) pps_val |= DSC_RC_MODEL_SIZE(vdsc_cfg->rc_model_size) | DSC_RC_EDGE_FACTOR(DSC_RC_EDGE_FACTOR_CONST); drm_info(&dev_priv->drm, "PPS9 = 0x%08x\n", pps_val); - if (!is_pipe_dsc(crtc_state)) { + if (!is_pipe_dsc(crtc, cpu_transcoder)) { intel_de_write(dev_priv, DSCA_PICTURE_PARAMETER_SET_9, pps_val); /* @@ -854,7 +852,7 @@ static void intel_dsc_pps_configure(const struct intel_crtc_state *crtc_state) DSC_RC_TARGET_OFF_HIGH(DSC_RC_TGT_OFFSET_HI_CONST) | DSC_RC_TARGET_OFF_LOW(DSC_RC_TGT_OFFSET_LO_CONST); drm_info(&dev_priv->drm, "PPS10 = 0x%08x\n", pps_val); - if (!is_pipe_dsc(crtc_state)) { + if (!is_pipe_dsc(crtc, cpu_transcoder)) { intel_de_write(dev_priv, DSCA_PICTURE_PARAMETER_SET_10, pps_val); /* @@ -882,7 +880,7 @@ static void intel_dsc_pps_configure(const struct intel_crtc_state *crtc_state) DSC_SLICE_ROW_PER_FRAME(vdsc_cfg->pic_height / vdsc_cfg->slice_height); drm_info(&dev_priv->drm, "PPS16 = 0x%08x\n", pps_val); - if (!is_pipe_dsc(crtc_state)) { + if (!is_pipe_dsc(crtc, cpu_transcoder)) { intel_de_write(dev_priv, DSCA_PICTURE_PARAMETER_SET_16, pps_val); /* @@ -911,7 +909,7 @@ static void intel_dsc_pps_configure(const struct intel_crtc_state *crtc_state) drm_info(&dev_priv->drm, " RC_BUF_THRESH%d = 0x%08x\n", i, rc_buf_thresh_dword[i / 4]); } - if (!is_pipe_dsc(crtc_state)) { + if (!is_pipe_dsc(crtc, cpu_transcoder)) { intel_de_write(dev_priv, DSCA_RC_BUF_THRESH_0, rc_buf_thresh_dword[0]); intel_de_write(dev_priv, DSCA_RC_BUF_THRESH_0_UDW, @@ -968,7 +966,7 @@ static void intel_dsc_pps_configure(const struct intel_crtc_state *crtc_state) drm_info(&dev_priv->drm, " RC_RANGE_PARAM_%d = 0x%08x\n", i, rc_range_params_dword[i / 2]); } - if (!is_pipe_dsc(crtc_state)) { + if (!is_pipe_dsc(crtc, cpu_transcoder)) { intel_de_write(dev_priv, DSCA_RC_RANGE_PARAMETERS_0, rc_range_params_dword[0]); intel_de_write(dev_priv, DSCA_RC_RANGE_PARAMETERS_0_UDW, @@ -1095,18 +1093,16 @@ static void intel_dsc_dp_pps_write(struct intel_encoder *encoder, sizeof(dp_dsc_pps_sdp)); } -static i915_reg_t dss_ctl1_reg(const struct intel_crtc_state *crtc_state) +static i915_reg_t dss_ctl1_reg(struct intel_crtc *crtc, enum transcoder cpu_transcoder) { - enum pipe pipe = to_intel_crtc(crtc_state->uapi.crtc)->pipe; - - return is_pipe_dsc(crtc_state) ? ICL_PIPE_DSS_CTL1(pipe) : DSS_CTL1; + return is_pipe_dsc(crtc, cpu_transcoder) ? + ICL_PIPE_DSS_CTL1(crtc->pipe) : DSS_CTL1; } -static i915_reg_t dss_ctl2_reg(const struct intel_crtc_state *crtc_state) +static i915_reg_t dss_ctl2_reg(struct intel_crtc *crtc, enum transcoder cpu_transcoder) { - enum pipe pipe = to_intel_crtc(crtc_state->uapi.crtc)->pipe; - - return is_pipe_dsc(crtc_state) ? ICL_PIPE_DSS_CTL2(pipe) : DSS_CTL2; + return is_pipe_dsc(crtc, cpu_transcoder) ? + ICL_PIPE_DSS_CTL2(crtc->pipe) : DSS_CTL2; } static struct intel_crtc * @@ -1142,7 +1138,7 @@ void intel_uncompressed_joiner_enable(const struct intel_crtc_state *crtc_state) else dss_ctl1_val |= UNCOMPRESSED_JOINER_MASTER; - intel_de_write(dev_priv, dss_ctl1_reg(crtc_state), dss_ctl1_val); + intel_de_write(dev_priv, dss_ctl1_reg(crtc, crtc_state->cpu_transcoder), dss_ctl1_val); } } @@ -1176,8 +1172,8 @@ void intel_dsc_enable(struct intel_encoder *encoder, if (!crtc_state->bigjoiner_slave) dss_ctl1_val |= MASTER_BIG_JOINER_ENABLE; } - intel_de_write(dev_priv, dss_ctl1_reg(crtc_state), dss_ctl1_val); - intel_de_write(dev_priv, dss_ctl2_reg(crtc_state), dss_ctl2_val); + intel_de_write(dev_priv, dss_ctl1_reg(crtc, crtc_state->cpu_transcoder), dss_ctl1_val); + intel_de_write(dev_priv, dss_ctl2_reg(crtc, crtc_state->cpu_transcoder), dss_ctl2_val); } void intel_dsc_disable(const struct intel_crtc_state *old_crtc_state) @@ -1188,8 +1184,8 @@ void intel_dsc_disable(const struct intel_crtc_state *old_crtc_state) /* Disable only if either of them is enabled */ if (old_crtc_state->dsc.compression_enable || old_crtc_state->bigjoiner) { - intel_de_write(dev_priv, dss_ctl1_reg(old_crtc_state), 0); - intel_de_write(dev_priv, dss_ctl2_reg(old_crtc_state), 0); + intel_de_write(dev_priv, dss_ctl1_reg(crtc, old_crtc_state->cpu_transcoder), 0); + intel_de_write(dev_priv, dss_ctl2_reg(crtc, old_crtc_state->cpu_transcoder), 0); } } @@ -1199,7 +1195,7 @@ void intel_uncompressed_joiner_get_config(struct intel_crtc_state *crtc_state) struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); u32 dss_ctl1; - dss_ctl1 = intel_de_read(dev_priv, dss_ctl1_reg(crtc_state)); + dss_ctl1 = intel_de_read(dev_priv, dss_ctl1_reg(crtc, crtc_state->cpu_transcoder)); if (dss_ctl1 & UNCOMPRESSED_JOINER_MASTER) { crtc_state->bigjoiner = true; crtc_state->bigjoiner_linked_crtc = intel_dsc_get_bigjoiner_secondary(crtc); @@ -1214,9 +1210,10 @@ void intel_uncompressed_joiner_get_config(struct intel_crtc_state *crtc_state) void intel_dsc_get_config(struct intel_crtc_state *crtc_state) { - struct drm_dsc_config *vdsc_cfg = &crtc_state->dsc.config; struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); + struct drm_dsc_config *vdsc_cfg = &crtc_state->dsc.config; + enum transcoder cpu_transcoder = crtc_state->cpu_transcoder; enum pipe pipe = crtc->pipe; enum intel_display_power_domain power_domain; intel_wakeref_t wakeref; @@ -1225,14 +1222,14 @@ void intel_dsc_get_config(struct intel_crtc_state *crtc_state) if (!intel_dsc_source_support(crtc_state)) return; - power_domain = intel_dsc_power_domain(crtc_state); + power_domain = intel_dsc_power_domain(crtc, cpu_transcoder); wakeref = intel_display_power_get_if_enabled(dev_priv, power_domain); if (!wakeref) return; - dss_ctl1 = intel_de_read(dev_priv, dss_ctl1_reg(crtc_state)); - dss_ctl2 = intel_de_read(dev_priv, dss_ctl2_reg(crtc_state)); + dss_ctl1 = intel_de_read(dev_priv, dss_ctl1_reg(crtc, cpu_transcoder)); + dss_ctl2 = intel_de_read(dev_priv, dss_ctl2_reg(crtc, cpu_transcoder)); crtc_state->dsc.compression_enable = dss_ctl2 & LEFT_BRANCH_VDSC_ENABLE; if (!crtc_state->dsc.compression_enable) @@ -1256,7 +1253,7 @@ void intel_dsc_get_config(struct intel_crtc_state *crtc_state) /* FIXME: add more state readout as needed */ /* PPS1 */ - if (!is_pipe_dsc(crtc_state)) + if (!is_pipe_dsc(crtc, cpu_transcoder)) val = intel_de_read(dev_priv, DSCA_PICTURE_PARAMETER_SET_1); else val = intel_de_read(dev_priv, diff --git a/drivers/gpu/drm/i915/display/intel_vdsc.h b/drivers/gpu/drm/i915/display/intel_vdsc.h index dfb1fd38deb4..0c5d80a572da 100644 --- a/drivers/gpu/drm/i915/display/intel_vdsc.h +++ b/drivers/gpu/drm/i915/display/intel_vdsc.h @@ -8,8 +8,10 @@ #include <linux/types.h> -struct intel_encoder; +enum transcoder; +struct intel_crtc; struct intel_crtc_state; +struct intel_encoder; bool intel_dsc_source_support(const struct intel_crtc_state *crtc_state); void intel_uncompressed_joiner_enable(const struct intel_crtc_state *crtc_state); @@ -21,7 +23,7 @@ int intel_dsc_compute_params(struct intel_encoder *encoder, void intel_uncompressed_joiner_get_config(struct intel_crtc_state *crtc_state); void intel_dsc_get_config(struct intel_crtc_state *crtc_state); enum intel_display_power_domain -intel_dsc_power_domain(const struct intel_crtc_state *crtc_state); +intel_dsc_power_domain(struct intel_crtc *crtc, enum transcoder cpu_transcoder); struct intel_crtc *intel_dsc_get_bigjoiner_secondary(const struct intel_crtc *primary_crtc); #endif /* __INTEL_VDSC_H__ */ diff --git a/drivers/gpu/drm/i915/display/skl_universal_plane.c b/drivers/gpu/drm/i915/display/skl_universal_plane.c index 724e7b04f3b6..a0e53a3b267a 100644 --- a/drivers/gpu/drm/i915/display/skl_universal_plane.c +++ b/drivers/gpu/drm/i915/display/skl_universal_plane.c @@ -18,6 +18,7 @@ #include "intel_sprite.h" #include "skl_scaler.h" #include "skl_universal_plane.h" +#include "pxp/intel_pxp.h" static const u32 skl_plane_formats[] = { DRM_FORMAT_C8, @@ -656,6 +657,7 @@ skl_disable_plane(struct intel_plane *plane, skl_write_plane_wm(plane, crtc_state); + intel_psr2_disable_plane_sel_fetch(plane, crtc_state); intel_de_write_fw(dev_priv, PLANE_CTL(pipe, plane_id), 0); intel_de_write_fw(dev_priv, PLANE_SURF(pipe, plane_id), 0); @@ -993,6 +995,11 @@ static u32 skl_surf_address(const struct intel_plane_state *plane_state, u32 offset = plane_state->view.color_plane[color_plane].offset; if (intel_fb_uses_dpt(fb)) { + /* + * The DPT object contains only one vma, so the VMA's offset + * within the DPT is always 0. + */ + WARN_ON(plane_state->dpt_vma->node.start); WARN_ON(offset & 0x1fffff); return offset >> 9; } else { @@ -1001,6 +1008,33 @@ static u32 skl_surf_address(const struct intel_plane_state *plane_state, } } +static void intel_load_plane_csc_black(struct intel_plane *intel_plane) +{ + struct drm_i915_private *dev_priv = to_i915(intel_plane->base.dev); + enum pipe pipe = intel_plane->pipe; + enum plane_id plane = intel_plane->id; + u16 postoff = 0; + + drm_dbg_kms(&dev_priv->drm, "plane color CTM to black %s:%d\n", + intel_plane->base.name, plane); + intel_de_write_fw(dev_priv, PLANE_CSC_COEFF(pipe, plane, 0), 0); + intel_de_write_fw(dev_priv, PLANE_CSC_COEFF(pipe, plane, 1), 0); + + intel_de_write_fw(dev_priv, PLANE_CSC_COEFF(pipe, plane, 2), 0); + intel_de_write_fw(dev_priv, PLANE_CSC_COEFF(pipe, plane, 3), 0); + + intel_de_write_fw(dev_priv, PLANE_CSC_COEFF(pipe, plane, 4), 0); + intel_de_write_fw(dev_priv, PLANE_CSC_COEFF(pipe, plane, 5), 0); + + intel_de_write_fw(dev_priv, PLANE_CSC_PREOFF(pipe, plane, 0), 0); + intel_de_write_fw(dev_priv, PLANE_CSC_PREOFF(pipe, plane, 1), 0); + intel_de_write_fw(dev_priv, PLANE_CSC_PREOFF(pipe, plane, 2), 0); + + intel_de_write_fw(dev_priv, PLANE_CSC_POSTOFF(pipe, plane, 0), postoff); + intel_de_write_fw(dev_priv, PLANE_CSC_POSTOFF(pipe, plane, 1), postoff); + intel_de_write_fw(dev_priv, PLANE_CSC_POSTOFF(pipe, plane, 2), postoff); +} + static void skl_program_plane(struct intel_plane *plane, const struct intel_crtc_state *crtc_state, @@ -1024,7 +1058,7 @@ skl_program_plane(struct intel_plane *plane, u8 alpha = plane_state->hw.alpha >> 8; u32 plane_color_ctl = 0, aux_dist = 0; unsigned long irqflags; - u32 keymsk, keymax; + u32 keymsk, keymax, plane_surf; u32 plane_ctl = plane_state->ctl; plane_ctl |= skl_plane_ctl_crtc(crtc_state); @@ -1096,8 +1130,7 @@ skl_program_plane(struct intel_plane *plane, (plane_state->view.color_plane[1].y << 16) | plane_state->view.color_plane[1].x); - if (!drm_atomic_crtc_needs_modeset(&crtc_state->uapi)) - intel_psr2_program_plane_sel_fetch(plane, crtc_state, plane_state, color_plane); + intel_psr2_program_plane_sel_fetch(plane, crtc_state, plane_state, color_plane); /* * Enable the scaler before the plane so that we don't @@ -1113,8 +1146,23 @@ skl_program_plane(struct intel_plane *plane, * the control register just before the surface register. */ intel_de_write_fw(dev_priv, PLANE_CTL(pipe, plane_id), plane_ctl); - intel_de_write_fw(dev_priv, PLANE_SURF(pipe, plane_id), - intel_plane_ggtt_offset(plane_state) + surf_addr); + plane_surf = intel_plane_ggtt_offset(plane_state) + surf_addr; + plane_color_ctl = intel_de_read_fw(dev_priv, PLANE_COLOR_CTL(pipe, plane_id)); + + /* + * FIXME: pxp session invalidation can hit any time even at time of commit + * or after the commit, display content will be garbage. + */ + if (plane_state->decrypt) { + plane_surf |= PLANE_SURF_DECRYPT; + } else if (plane_state->force_black) { + intel_load_plane_csc_black(plane); + plane_color_ctl |= PLANE_COLOR_PLANE_CSC_ENABLE; + } + + intel_de_write_fw(dev_priv, PLANE_COLOR_CTL(pipe, plane_id), + plane_color_ctl); + intel_de_write_fw(dev_priv, PLANE_SURF(pipe, plane_id), plane_surf); spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); } diff --git a/drivers/gpu/drm/i915/display/vlv_dsi.c b/drivers/gpu/drm/i915/display/vlv_dsi.c index 0ee4ff341e25..07584695514b 100644 --- a/drivers/gpu/drm/i915/display/vlv_dsi.c +++ b/drivers/gpu/drm/i915/display/vlv_dsi.c @@ -32,6 +32,7 @@ #include "i915_drv.h" #include "intel_atomic.h" +#include "intel_backlight.h" #include "intel_connector.h" #include "intel_crtc.h" #include "intel_de.h" @@ -39,8 +40,8 @@ #include "intel_dsi.h" #include "intel_fifo_underrun.h" #include "intel_panel.h" -#include "intel_sideband.h" #include "skl_scaler.h" +#include "vlv_sideband.h" /* return pixels in terms of txbyteclkhs */ static u16 txbyteclkhs(u16 pixels, int bpp, int lane_count, @@ -270,23 +271,19 @@ static int intel_dsi_compute_config(struct intel_encoder *encoder, struct intel_dsi *intel_dsi = container_of(encoder, struct intel_dsi, base); struct intel_connector *intel_connector = intel_dsi->attached_connector; - const struct drm_display_mode *fixed_mode = intel_connector->panel.fixed_mode; struct drm_display_mode *adjusted_mode = &pipe_config->hw.adjusted_mode; int ret; drm_dbg_kms(&dev_priv->drm, "\n"); pipe_config->output_format = INTEL_OUTPUT_FORMAT_RGB; - if (fixed_mode) { - intel_fixed_panel_mode(fixed_mode, adjusted_mode); + ret = intel_panel_compute_config(intel_connector, adjusted_mode); + if (ret) + return ret; - if (HAS_GMCH(dev_priv)) - ret = intel_gmch_panel_fitting(pipe_config, conn_state); - else - ret = intel_pch_panel_fitting(pipe_config, conn_state); - if (ret) - return ret; - } + ret = intel_panel_fitting(pipe_config, conn_state); + if (ret) + return ret; if (adjusted_mode->flags & DRM_MODE_FLAG_DBLSCAN) return -EINVAL; @@ -883,7 +880,7 @@ static void intel_dsi_pre_enable(struct intel_atomic_state *state, intel_dsi_port_enable(encoder, pipe_config); } - intel_panel_enable_backlight(pipe_config, conn_state); + intel_backlight_enable(pipe_config, conn_state); intel_dsi_vbt_exec_sequence(intel_dsi, MIPI_SEQ_BACKLIGHT_ON); } @@ -913,7 +910,7 @@ static void intel_dsi_disable(struct intel_atomic_state *state, drm_dbg_kms(&i915->drm, "\n"); intel_dsi_vbt_exec_sequence(intel_dsi, MIPI_SEQ_BACKLIGHT_OFF); - intel_panel_disable_backlight(old_conn_state); + intel_backlight_disable(old_conn_state); /* * According to the spec we should send SHUTDOWN before @@ -1633,25 +1630,21 @@ static const struct drm_connector_funcs intel_dsi_connector_funcs = { static void vlv_dsi_add_properties(struct intel_connector *connector) { struct drm_i915_private *dev_priv = to_i915(connector->base.dev); + u32 allowed_scalers; - if (connector->panel.fixed_mode) { - u32 allowed_scalers; - - allowed_scalers = BIT(DRM_MODE_SCALE_ASPECT) | BIT(DRM_MODE_SCALE_FULLSCREEN); - if (!HAS_GMCH(dev_priv)) - allowed_scalers |= BIT(DRM_MODE_SCALE_CENTER); + allowed_scalers = BIT(DRM_MODE_SCALE_ASPECT) | BIT(DRM_MODE_SCALE_FULLSCREEN); + if (!HAS_GMCH(dev_priv)) + allowed_scalers |= BIT(DRM_MODE_SCALE_CENTER); - drm_connector_attach_scaling_mode_property(&connector->base, - allowed_scalers); + drm_connector_attach_scaling_mode_property(&connector->base, + allowed_scalers); - connector->base.state->scaling_mode = DRM_MODE_SCALE_ASPECT; + connector->base.state->scaling_mode = DRM_MODE_SCALE_ASPECT; - drm_connector_set_panel_orientation_with_quirk( - &connector->base, - intel_dsi_get_panel_orientation(connector), - connector->panel.fixed_mode->hdisplay, - connector->panel.fixed_mode->vdisplay); - } + drm_connector_set_panel_orientation_with_quirk(&connector->base, + intel_dsi_get_panel_orientation(connector), + connector->panel.fixed_mode->hdisplay, + connector->panel.fixed_mode->vdisplay); } #define NS_KHZ_RATIO 1000000 @@ -1876,7 +1869,7 @@ void vlv_dsi_init(struct drm_i915_private *dev_priv) intel_encoder->post_disable = intel_dsi_post_disable; intel_encoder->get_hw_state = intel_dsi_get_hw_state; intel_encoder->get_config = intel_dsi_get_config; - intel_encoder->update_pipe = intel_panel_update_backlight; + intel_encoder->update_pipe = intel_backlight_update; intel_encoder->shutdown = intel_dsi_shutdown; intel_connector->get_hw_state = intel_connector_get_hw_state; @@ -1964,7 +1957,7 @@ void vlv_dsi_init(struct drm_i915_private *dev_priv) } intel_panel_init(&intel_connector->panel, fixed_mode, NULL); - intel_panel_setup_backlight(connector, INVALID_PIPE); + intel_backlight_setup(intel_connector, INVALID_PIPE); vlv_dsi_add_properties(intel_connector); diff --git a/drivers/gpu/drm/i915/display/vlv_dsi_pll.c b/drivers/gpu/drm/i915/display/vlv_dsi_pll.c index 90185b219447..5413b52ab6ba 100644 --- a/drivers/gpu/drm/i915/display/vlv_dsi_pll.c +++ b/drivers/gpu/drm/i915/display/vlv_dsi_pll.c @@ -31,7 +31,7 @@ #include "intel_de.h" #include "intel_display_types.h" #include "intel_dsi.h" -#include "intel_sideband.h" +#include "vlv_sideband.h" static const u16 lfsr_converts[] = { 426, 469, 234, 373, 442, 221, 110, 311, 411, /* 62 - 70 */ @@ -568,3 +568,26 @@ void bxt_dsi_reset_clocks(struct intel_encoder *encoder, enum port port) } intel_de_write(dev_priv, MIPI_EOT_DISABLE(port), CLOCKSTOP); } + +static void assert_dsi_pll(struct drm_i915_private *i915, bool state) +{ + bool cur_state; + + vlv_cck_get(i915); + cur_state = vlv_cck_read(i915, CCK_REG_DSI_PLL_CONTROL) & DSI_PLL_VCO_EN; + vlv_cck_put(i915); + + I915_STATE_WARN(cur_state != state, + "DSI PLL state assertion failure (expected %s, current %s)\n", + onoff(state), onoff(cur_state)); +} + +void assert_dsi_pll_enabled(struct drm_i915_private *i915) +{ + assert_dsi_pll(i915, true); +} + +void assert_dsi_pll_disabled(struct drm_i915_private *i915) +{ + assert_dsi_pll(i915, false); +} diff --git a/drivers/gpu/drm/i915/gem/i915_gem_busy.c b/drivers/gpu/drm/i915/gem/i915_gem_busy.c index 6234e17259c1..7358bebef15c 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_busy.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_busy.c @@ -4,6 +4,8 @@ * Copyright © 2014-2016 Intel Corporation */ +#include <linux/dma-fence-array.h> + #include "gt/intel_engine.h" #include "i915_gem_ioctls.h" @@ -36,7 +38,7 @@ static __always_inline u32 __busy_write_id(u16 id) } static __always_inline unsigned int -__busy_set_if_active(const struct dma_fence *fence, u32 (*flag)(u16 id)) +__busy_set_if_active(struct dma_fence *fence, u32 (*flag)(u16 id)) { const struct i915_request *rq; @@ -46,29 +48,60 @@ __busy_set_if_active(const struct dma_fence *fence, u32 (*flag)(u16 id)) * to eventually flush us, but to minimise latency just ask the * hardware. * - * Note we only report on the status of native fences. + * Note we only report on the status of native fences and we currently + * have two native fences: + * + * 1. A composite fence (dma_fence_array) constructed of i915 requests + * created during a parallel submission. In this case we deconstruct the + * composite fence into individual i915 requests and check the status of + * each request. + * + * 2. A single i915 request. */ - if (!dma_fence_is_i915(fence)) + if (dma_fence_is_array(fence)) { + struct dma_fence_array *array = to_dma_fence_array(fence); + struct dma_fence **child = array->fences; + unsigned int nchild = array->num_fences; + + do { + struct dma_fence *current_fence = *child++; + + /* Not an i915 fence, can't be busy per above */ + if (!dma_fence_is_i915(current_fence) || + !test_bit(I915_FENCE_FLAG_COMPOSITE, + ¤t_fence->flags)) { + return 0; + } + + rq = to_request(current_fence); + if (!i915_request_completed(rq)) + return flag(rq->engine->uabi_class); + } while (--nchild); + + /* All requests in array complete, not busy */ return 0; + } else { + if (!dma_fence_is_i915(fence)) + return 0; - /* opencode to_request() in order to avoid const warnings */ - rq = container_of(fence, const struct i915_request, fence); - if (i915_request_completed(rq)) - return 0; + rq = to_request(fence); + if (i915_request_completed(rq)) + return 0; - /* Beware type-expansion follies! */ - BUILD_BUG_ON(!typecheck(u16, rq->engine->uabi_class)); - return flag(rq->engine->uabi_class); + /* Beware type-expansion follies! */ + BUILD_BUG_ON(!typecheck(u16, rq->engine->uabi_class)); + return flag(rq->engine->uabi_class); + } } static __always_inline unsigned int -busy_check_reader(const struct dma_fence *fence) +busy_check_reader(struct dma_fence *fence) { return __busy_set_if_active(fence, __busy_read_flag); } static __always_inline unsigned int -busy_check_writer(const struct dma_fence *fence) +busy_check_writer(struct dma_fence *fence) { if (!fence) return 0; diff --git a/drivers/gpu/drm/i915/gem/i915_gem_context.c b/drivers/gpu/drm/i915/gem/i915_gem_context.c index 166bb46408a9..fb33d0322960 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_context.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_context.c @@ -77,6 +77,8 @@ #include "gt/intel_gpu_commands.h" #include "gt/intel_ring.h" +#include "pxp/intel_pxp.h" + #include "i915_gem_context.h" #include "i915_trace.h" #include "i915_user_extensions.h" @@ -186,10 +188,13 @@ static int validate_priority(struct drm_i915_private *i915, return 0; } -static void proto_context_close(struct i915_gem_proto_context *pc) +static void proto_context_close(struct drm_i915_private *i915, + struct i915_gem_proto_context *pc) { int i; + if (pc->pxp_wakeref) + intel_runtime_pm_put(&i915->runtime_pm, pc->pxp_wakeref); if (pc->vm) i915_vm_put(pc->vm); if (pc->user_engines) { @@ -241,6 +246,35 @@ static int proto_context_set_persistence(struct drm_i915_private *i915, return 0; } +static int proto_context_set_protected(struct drm_i915_private *i915, + struct i915_gem_proto_context *pc, + bool protected) +{ + int ret = 0; + + if (!protected) { + pc->uses_protected_content = false; + } else if (!intel_pxp_is_enabled(&i915->gt.pxp)) { + ret = -ENODEV; + } else if ((pc->user_flags & BIT(UCONTEXT_RECOVERABLE)) || + !(pc->user_flags & BIT(UCONTEXT_BANNABLE))) { + ret = -EPERM; + } else { + pc->uses_protected_content = true; + + /* + * protected context usage requires the PXP session to be up, + * which in turn requires the device to be active. + */ + pc->pxp_wakeref = intel_runtime_pm_get(&i915->runtime_pm); + + if (!intel_pxp_is_active(&i915->gt.pxp)) + ret = intel_pxp_start(&i915->gt.pxp); + } + + return ret; +} + static struct i915_gem_proto_context * proto_context_create(struct drm_i915_private *i915, unsigned int flags) { @@ -269,7 +303,7 @@ proto_context_create(struct drm_i915_private *i915, unsigned int flags) return pc; proto_close: - proto_context_close(pc); + proto_context_close(i915, pc); return err; } @@ -442,6 +476,13 @@ set_proto_ctx_engines_bond(struct i915_user_extension __user *base, void *data) u16 idx, num_bonds; int err, n; + if (GRAPHICS_VER(i915) >= 12 && !IS_TIGERLAKE(i915) && + !IS_ROCKETLAKE(i915) && !IS_ALDERLAKE_S(i915)) { + drm_dbg(&i915->drm, + "Bonding on gen12+ aside from TGL, RKL, and ADL_S not supported\n"); + return -ENODEV; + } + if (get_user(idx, &ext->virtual_index)) return -EFAULT; @@ -515,9 +556,147 @@ set_proto_ctx_engines_bond(struct i915_user_extension __user *base, void *data) return 0; } +static int +set_proto_ctx_engines_parallel_submit(struct i915_user_extension __user *base, + void *data) +{ + struct i915_context_engines_parallel_submit __user *ext = + container_of_user(base, typeof(*ext), base); + const struct set_proto_ctx_engines *set = data; + struct drm_i915_private *i915 = set->i915; + u64 flags; + int err = 0, n, i, j; + u16 slot, width, num_siblings; + struct intel_engine_cs **siblings = NULL; + intel_engine_mask_t prev_mask; + + /* FIXME: This is NIY for execlists */ + if (!(intel_uc_uses_guc_submission(&i915->gt.uc))) + return -ENODEV; + + if (get_user(slot, &ext->engine_index)) + return -EFAULT; + + if (get_user(width, &ext->width)) + return -EFAULT; + + if (get_user(num_siblings, &ext->num_siblings)) + return -EFAULT; + + if (slot >= set->num_engines) { + drm_dbg(&i915->drm, "Invalid placement value, %d >= %d\n", + slot, set->num_engines); + return -EINVAL; + } + + if (set->engines[slot].type != I915_GEM_ENGINE_TYPE_INVALID) { + drm_dbg(&i915->drm, + "Invalid placement[%d], already occupied\n", slot); + return -EINVAL; + } + + if (get_user(flags, &ext->flags)) + return -EFAULT; + + if (flags) { + drm_dbg(&i915->drm, "Unknown flags 0x%02llx", flags); + return -EINVAL; + } + + for (n = 0; n < ARRAY_SIZE(ext->mbz64); n++) { + err = check_user_mbz(&ext->mbz64[n]); + if (err) + return err; + } + + if (width < 2) { + drm_dbg(&i915->drm, "Width (%d) < 2\n", width); + return -EINVAL; + } + + if (num_siblings < 1) { + drm_dbg(&i915->drm, "Number siblings (%d) < 1\n", + num_siblings); + return -EINVAL; + } + + siblings = kmalloc_array(num_siblings * width, + sizeof(*siblings), + GFP_KERNEL); + if (!siblings) + return -ENOMEM; + + /* Create contexts / engines */ + for (i = 0; i < width; ++i) { + intel_engine_mask_t current_mask = 0; + struct i915_engine_class_instance prev_engine; + + for (j = 0; j < num_siblings; ++j) { + struct i915_engine_class_instance ci; + + n = i * num_siblings + j; + if (copy_from_user(&ci, &ext->engines[n], sizeof(ci))) { + err = -EFAULT; + goto out_err; + } + + siblings[n] = + intel_engine_lookup_user(i915, ci.engine_class, + ci.engine_instance); + if (!siblings[n]) { + drm_dbg(&i915->drm, + "Invalid sibling[%d]: { class:%d, inst:%d }\n", + n, ci.engine_class, ci.engine_instance); + err = -EINVAL; + goto out_err; + } + + if (n) { + if (prev_engine.engine_class != + ci.engine_class) { + drm_dbg(&i915->drm, + "Mismatched class %d, %d\n", + prev_engine.engine_class, + ci.engine_class); + err = -EINVAL; + goto out_err; + } + } + + prev_engine = ci; + current_mask |= siblings[n]->logical_mask; + } + + if (i > 0) { + if (current_mask != prev_mask << 1) { + drm_dbg(&i915->drm, + "Non contiguous logical mask 0x%x, 0x%x\n", + prev_mask, current_mask); + err = -EINVAL; + goto out_err; + } + } + prev_mask = current_mask; + } + + set->engines[slot].type = I915_GEM_ENGINE_TYPE_PARALLEL; + set->engines[slot].num_siblings = num_siblings; + set->engines[slot].width = width; + set->engines[slot].siblings = siblings; + + return 0; + +out_err: + kfree(siblings); + + return err; +} + static const i915_user_extension_fn set_proto_ctx_engines_extensions[] = { [I915_CONTEXT_ENGINES_EXT_LOAD_BALANCE] = set_proto_ctx_engines_balance, [I915_CONTEXT_ENGINES_EXT_BOND] = set_proto_ctx_engines_bond, + [I915_CONTEXT_ENGINES_EXT_PARALLEL_SUBMIT] = + set_proto_ctx_engines_parallel_submit, }; static int set_proto_ctx_engines(struct drm_i915_file_private *fpriv, @@ -686,6 +865,8 @@ static int set_proto_ctx_param(struct drm_i915_file_private *fpriv, ret = -EPERM; else if (args->value) pc->user_flags |= BIT(UCONTEXT_BANNABLE); + else if (pc->uses_protected_content) + ret = -EPERM; else pc->user_flags &= ~BIT(UCONTEXT_BANNABLE); break; @@ -693,10 +874,12 @@ static int set_proto_ctx_param(struct drm_i915_file_private *fpriv, case I915_CONTEXT_PARAM_RECOVERABLE: if (args->size) ret = -EINVAL; - else if (args->value) - pc->user_flags |= BIT(UCONTEXT_RECOVERABLE); - else + else if (!args->value) pc->user_flags &= ~BIT(UCONTEXT_RECOVERABLE); + else if (pc->uses_protected_content) + ret = -EPERM; + else + pc->user_flags |= BIT(UCONTEXT_RECOVERABLE); break; case I915_CONTEXT_PARAM_PRIORITY: @@ -724,6 +907,11 @@ static int set_proto_ctx_param(struct drm_i915_file_private *fpriv, args->value); break; + case I915_CONTEXT_PARAM_PROTECTED_CONTENT: + ret = proto_context_set_protected(fpriv->dev_priv, pc, + args->value); + break; + case I915_CONTEXT_PARAM_NO_ZEROMAP: case I915_CONTEXT_PARAM_BAN_PERIOD: case I915_CONTEXT_PARAM_RINGSIZE: @@ -735,44 +923,6 @@ static int set_proto_ctx_param(struct drm_i915_file_private *fpriv, return ret; } -static struct i915_address_space * -context_get_vm_rcu(struct i915_gem_context *ctx) -{ - GEM_BUG_ON(!rcu_access_pointer(ctx->vm)); - - do { - struct i915_address_space *vm; - - /* - * We do not allow downgrading from full-ppgtt [to a shared - * global gtt], so ctx->vm cannot become NULL. - */ - vm = rcu_dereference(ctx->vm); - if (!kref_get_unless_zero(&vm->ref)) - continue; - - /* - * This ppgtt may have be reallocated between - * the read and the kref, and reassigned to a third - * context. In order to avoid inadvertent sharing - * of this ppgtt with that third context (and not - * src), we have to confirm that we have the same - * ppgtt after passing through the strong memory - * barrier implied by a successful - * kref_get_unless_zero(). - * - * Once we have acquired the current ppgtt of ctx, - * we no longer care if it is released from ctx, as - * it cannot be reallocated elsewhere. - */ - - if (vm == rcu_access_pointer(ctx->vm)) - return rcu_pointer_handoff(vm); - - i915_vm_put(vm); - } while (1); -} - static int intel_context_set_gem(struct intel_context *ce, struct i915_gem_context *ctx, struct intel_sseu sseu) @@ -782,25 +932,18 @@ static int intel_context_set_gem(struct intel_context *ce, GEM_BUG_ON(rcu_access_pointer(ce->gem_context)); RCU_INIT_POINTER(ce->gem_context, ctx); + GEM_BUG_ON(intel_context_is_pinned(ce)); ce->ring_size = SZ_16K; - if (rcu_access_pointer(ctx->vm)) { - struct i915_address_space *vm; - - rcu_read_lock(); - vm = context_get_vm_rcu(ctx); /* hmm */ - rcu_read_unlock(); - - i915_vm_put(ce->vm); - ce->vm = vm; - } + i915_vm_put(ce->vm); + ce->vm = i915_gem_context_get_eb_vm(ctx); if (ctx->sched.priority >= I915_PRIORITY_NORMAL && intel_engine_has_timeslices(ce->engine) && intel_engine_has_semaphores(ce->engine)) __set_bit(CONTEXT_USE_SEMAPHORES, &ce->flags); - if (IS_ACTIVE(CONFIG_DRM_I915_REQUEST_TIMEOUT) && + if (CONFIG_DRM_I915_REQUEST_TIMEOUT && ctx->i915->params.request_timeout_ms) { unsigned int timeout_ms = ctx->i915->params.request_timeout_ms; @@ -814,6 +957,25 @@ static int intel_context_set_gem(struct intel_context *ce, return ret; } +static void __unpin_engines(struct i915_gem_engines *e, unsigned int count) +{ + while (count--) { + struct intel_context *ce = e->engines[count], *child; + + if (!ce || !test_bit(CONTEXT_PERMA_PIN, &ce->flags)) + continue; + + for_each_child(ce, child) + intel_context_unpin(child); + intel_context_unpin(ce); + } +} + +static void unpin_engines(struct i915_gem_engines *e) +{ + __unpin_engines(e, e->num_engines); +} + static void __free_engines(struct i915_gem_engines *e, unsigned int count) { while (count--) { @@ -929,6 +1091,40 @@ free_engines: return err; } +static int perma_pin_contexts(struct intel_context *ce) +{ + struct intel_context *child; + int i = 0, j = 0, ret; + + GEM_BUG_ON(!intel_context_is_parent(ce)); + + ret = intel_context_pin(ce); + if (unlikely(ret)) + return ret; + + for_each_child(ce, child) { + ret = intel_context_pin(child); + if (unlikely(ret)) + goto unwind; + ++i; + } + + set_bit(CONTEXT_PERMA_PIN, &ce->flags); + + return 0; + +unwind: + intel_context_unpin(ce); + for_each_child(ce, child) { + if (j++ < i) + intel_context_unpin(child); + else + break; + } + + return ret; +} + static struct i915_gem_engines *user_engines(struct i915_gem_context *ctx, unsigned int num_engines, struct i915_gem_proto_engine *pe) @@ -942,7 +1138,7 @@ static struct i915_gem_engines *user_engines(struct i915_gem_context *ctx, e->num_engines = num_engines; for (n = 0; n < num_engines; n++) { - struct intel_context *ce; + struct intel_context *ce, *child; int ret; switch (pe[n].type) { @@ -952,7 +1148,13 @@ static struct i915_gem_engines *user_engines(struct i915_gem_context *ctx, case I915_GEM_ENGINE_TYPE_BALANCED: ce = intel_engine_create_virtual(pe[n].siblings, - pe[n].num_siblings); + pe[n].num_siblings, 0); + break; + + case I915_GEM_ENGINE_TYPE_PARALLEL: + ce = intel_engine_create_parallel(pe[n].siblings, + pe[n].num_siblings, + pe[n].width); break; case I915_GEM_ENGINE_TYPE_INVALID: @@ -973,6 +1175,30 @@ static struct i915_gem_engines *user_engines(struct i915_gem_context *ctx, err = ERR_PTR(ret); goto free_engines; } + for_each_child(ce, child) { + ret = intel_context_set_gem(child, ctx, pe->sseu); + if (ret) { + err = ERR_PTR(ret); + goto free_engines; + } + } + + /* + * XXX: Must be done after calling intel_context_set_gem as that + * function changes the ring size. The ring is allocated when + * the context is pinned. If the ring size is changed after + * allocation we have a mismatch of the ring size and will cause + * the context to hang. Presumably with a bit of reordering we + * could move the perma-pin step to the backend function + * intel_engine_create_parallel. + */ + if (pe[n].type == I915_GEM_ENGINE_TYPE_PARALLEL) { + ret = perma_pin_contexts(ce); + if (ret) { + err = ERR_PTR(ret); + goto free_engines; + } + } } return e; @@ -982,9 +1208,11 @@ free_engines: return err; } -void i915_gem_context_release(struct kref *ref) +static void i915_gem_context_release_work(struct work_struct *work) { - struct i915_gem_context *ctx = container_of(ref, typeof(*ctx), ref); + struct i915_gem_context *ctx = container_of(work, typeof(*ctx), + release_work); + struct i915_address_space *vm; trace_i915_context_free(ctx); GEM_BUG_ON(!i915_gem_context_is_closed(ctx)); @@ -992,6 +1220,13 @@ void i915_gem_context_release(struct kref *ref) if (ctx->syncobj) drm_syncobj_put(ctx->syncobj); + vm = ctx->vm; + if (vm) + i915_vm_put(vm); + + if (ctx->pxp_wakeref) + intel_runtime_pm_put(&ctx->i915->runtime_pm, ctx->pxp_wakeref); + mutex_destroy(&ctx->engines_mutex); mutex_destroy(&ctx->lut_mutex); @@ -1001,6 +1236,13 @@ void i915_gem_context_release(struct kref *ref) kfree_rcu(ctx, rcu); } +void i915_gem_context_release(struct kref *ref) +{ + struct i915_gem_context *ctx = container_of(ref, typeof(*ctx), ref); + + queue_work(ctx->i915->wq, &ctx->release_work); +} + static inline struct i915_gem_engines * __context_engines_static(const struct i915_gem_context *ctx) { @@ -1199,6 +1441,7 @@ static void context_close(struct i915_gem_context *ctx) /* Flush any concurrent set_engines() */ mutex_lock(&ctx->engines_mutex); + unpin_engines(__context_engines_static(ctx)); engines_idle_release(ctx, rcu_replace_pointer(ctx->engines, NULL, 1)); i915_gem_context_set_closed(ctx); mutex_unlock(&ctx->engines_mutex); @@ -1207,9 +1450,16 @@ static void context_close(struct i915_gem_context *ctx) set_closed_name(ctx); - vm = i915_gem_context_vm(ctx); - if (vm) + vm = ctx->vm; + if (vm) { + /* i915_vm_close drops the final reference, which is a bit too + * early and could result in surprises with concurrent + * operations racing with thist ctx close. Keep a full reference + * until the end. + */ + i915_vm_get(vm); i915_vm_close(vm); + } ctx->file_priv = ERR_PTR(-EBADF); @@ -1280,49 +1530,6 @@ static int __context_set_persistence(struct i915_gem_context *ctx, bool state) return 0; } -static inline struct i915_gem_engines * -__context_engines_await(const struct i915_gem_context *ctx, - bool *user_engines) -{ - struct i915_gem_engines *engines; - - rcu_read_lock(); - do { - engines = rcu_dereference(ctx->engines); - GEM_BUG_ON(!engines); - - if (user_engines) - *user_engines = i915_gem_context_user_engines(ctx); - - /* successful await => strong mb */ - if (unlikely(!i915_sw_fence_await(&engines->fence))) - continue; - - if (likely(engines == rcu_access_pointer(ctx->engines))) - break; - - i915_sw_fence_complete(&engines->fence); - } while (1); - rcu_read_unlock(); - - return engines; -} - -static void -context_apply_all(struct i915_gem_context *ctx, - void (*fn)(struct intel_context *ce, void *data), - void *data) -{ - struct i915_gem_engines_iter it; - struct i915_gem_engines *e; - struct intel_context *ce; - - e = __context_engines_await(ctx, NULL); - for_each_gem_engine(ce, e, it) - fn(ce, data); - i915_sw_fence_complete(&e->fence); -} - static struct i915_gem_context * i915_gem_create_context(struct drm_i915_private *i915, const struct i915_gem_proto_context *pc) @@ -1342,6 +1549,7 @@ i915_gem_create_context(struct drm_i915_private *i915, ctx->sched = pc->sched; mutex_init(&ctx->mutex); INIT_LIST_HEAD(&ctx->link); + INIT_WORK(&ctx->release_work, i915_gem_context_release_work); spin_lock_init(&ctx->stale.lock); INIT_LIST_HEAD(&ctx->stale.engines); @@ -1351,7 +1559,7 @@ i915_gem_create_context(struct drm_i915_private *i915, } else if (HAS_FULL_PPGTT(i915)) { struct i915_ppgtt *ppgtt; - ppgtt = i915_ppgtt_create(&i915->gt); + ppgtt = i915_ppgtt_create(&i915->gt, 0); if (IS_ERR(ppgtt)) { drm_dbg(&i915->drm, "PPGTT setup failed (%ld)\n", PTR_ERR(ppgtt)); @@ -1361,7 +1569,7 @@ i915_gem_create_context(struct drm_i915_private *i915, vm = &ppgtt->vm; } if (vm) { - RCU_INIT_POINTER(ctx->vm, i915_vm_open(vm)); + ctx->vm = i915_vm_open(vm); /* i915_vm_open() takes a reference */ i915_vm_put(vm); @@ -1402,6 +1610,11 @@ i915_gem_create_context(struct drm_i915_private *i915, goto err_engines; } + if (pc->uses_protected_content) { + ctx->pxp_wakeref = intel_runtime_pm_get(&i915->runtime_pm); + ctx->uses_protected_content = true; + } + trace_i915_context_create(ctx); return ctx; @@ -1473,7 +1686,7 @@ int i915_gem_context_open(struct drm_i915_private *i915, } ctx = i915_gem_create_context(i915, pc); - proto_context_close(pc); + proto_context_close(i915, pc); if (IS_ERR(ctx)) { err = PTR_ERR(ctx); goto err; @@ -1500,7 +1713,7 @@ void i915_gem_context_close(struct drm_file *file) unsigned long idx; xa_for_each(&file_priv->proto_context_xa, idx, pc) - proto_context_close(pc); + proto_context_close(file_priv->dev_priv, pc); xa_destroy(&file_priv->proto_context_xa); mutex_destroy(&file_priv->proto_context_lock); @@ -1529,7 +1742,7 @@ int i915_gem_vm_create_ioctl(struct drm_device *dev, void *data, if (args->flags) return -EINVAL; - ppgtt = i915_ppgtt_create(&i915->gt); + ppgtt = i915_ppgtt_create(&i915->gt, 0); if (IS_ERR(ppgtt)) return PTR_ERR(ppgtt); @@ -1584,18 +1797,15 @@ static int get_ppgtt(struct drm_i915_file_private *file_priv, int err; u32 id; - if (!rcu_access_pointer(ctx->vm)) + if (!i915_gem_context_has_full_ppgtt(ctx)) return -ENODEV; - rcu_read_lock(); - vm = context_get_vm_rcu(ctx); - rcu_read_unlock(); - if (!vm) - return -ENODEV; + vm = ctx->vm; + GEM_BUG_ON(!vm); err = xa_alloc(&file_priv->vm_xa, &id, vm, xa_limit_32b, GFP_KERNEL); if (err) - goto err_put; + return err; i915_vm_open(vm); @@ -1603,8 +1813,6 @@ static int get_ppgtt(struct drm_i915_file_private *file_priv, args->value = id; args->size = 0; -err_put: - i915_vm_put(vm); return err; } @@ -1772,23 +1980,11 @@ set_persistence(struct i915_gem_context *ctx, return __context_set_persistence(ctx, args->value); } -static void __apply_priority(struct intel_context *ce, void *arg) -{ - struct i915_gem_context *ctx = arg; - - if (!intel_engine_has_timeslices(ce->engine)) - return; - - if (ctx->sched.priority >= I915_PRIORITY_NORMAL && - intel_engine_has_semaphores(ce->engine)) - intel_context_set_use_semaphores(ce); - else - intel_context_clear_use_semaphores(ce); -} - static int set_priority(struct i915_gem_context *ctx, const struct drm_i915_gem_context_param *args) { + struct i915_gem_engines_iter it; + struct intel_context *ce; int err; err = validate_priority(ctx->i915, args); @@ -1796,7 +1992,27 @@ static int set_priority(struct i915_gem_context *ctx, return err; ctx->sched.priority = args->value; - context_apply_all(ctx, __apply_priority, ctx); + + for_each_gem_engine(ce, i915_gem_context_lock_engines(ctx), it) { + if (!intel_engine_has_timeslices(ce->engine)) + continue; + + if (ctx->sched.priority >= I915_PRIORITY_NORMAL && + intel_engine_has_semaphores(ce->engine)) + intel_context_set_use_semaphores(ce); + else + intel_context_clear_use_semaphores(ce); + } + i915_gem_context_unlock_engines(ctx); + + return 0; +} + +static int get_protected(struct i915_gem_context *ctx, + struct drm_i915_gem_context_param *args) +{ + args->size = 0; + args->value = i915_gem_context_uses_protected_content(ctx); return 0; } @@ -1824,6 +2040,8 @@ static int ctx_setparam(struct drm_i915_file_private *fpriv, ret = -EPERM; else if (args->value) i915_gem_context_set_bannable(ctx); + else if (i915_gem_context_uses_protected_content(ctx)) + ret = -EPERM; /* can't clear this for protected contexts */ else i915_gem_context_clear_bannable(ctx); break; @@ -1831,10 +2049,12 @@ static int ctx_setparam(struct drm_i915_file_private *fpriv, case I915_CONTEXT_PARAM_RECOVERABLE: if (args->size) ret = -EINVAL; - else if (args->value) - i915_gem_context_set_recoverable(ctx); - else + else if (!args->value) i915_gem_context_clear_recoverable(ctx); + else if (i915_gem_context_uses_protected_content(ctx)) + ret = -EPERM; /* can't set this for protected contexts */ + else + i915_gem_context_set_recoverable(ctx); break; case I915_CONTEXT_PARAM_PRIORITY: @@ -1849,6 +2069,7 @@ static int ctx_setparam(struct drm_i915_file_private *fpriv, ret = set_persistence(ctx, args); break; + case I915_CONTEXT_PARAM_PROTECTED_CONTENT: case I915_CONTEXT_PARAM_NO_ZEROMAP: case I915_CONTEXT_PARAM_BAN_PERIOD: case I915_CONTEXT_PARAM_RINGSIZE: @@ -1927,7 +2148,7 @@ finalize_create_context_locked(struct drm_i915_file_private *file_priv, old = xa_erase(&file_priv->proto_context_xa, id); GEM_BUG_ON(old != pc); - proto_context_close(pc); + proto_context_close(file_priv->dev_priv, pc); /* One for the xarray and one for the caller */ return i915_gem_context_get(ctx); @@ -2013,7 +2234,7 @@ int i915_gem_context_create_ioctl(struct drm_device *dev, void *data, goto err_pc; } - proto_context_close(ext_data.pc); + proto_context_close(i915, ext_data.pc); gem_context_register(ctx, ext_data.fpriv, id); } else { ret = proto_context_register(ext_data.fpriv, ext_data.pc, &id); @@ -2027,7 +2248,7 @@ int i915_gem_context_create_ioctl(struct drm_device *dev, void *data, return 0; err_pc: - proto_context_close(ext_data.pc); + proto_context_close(i915, ext_data.pc); return ret; } @@ -2058,7 +2279,7 @@ int i915_gem_context_destroy_ioctl(struct drm_device *dev, void *data, GEM_WARN_ON(ctx && pc); if (pc) - proto_context_close(pc); + proto_context_close(file_priv->dev_priv, pc); if (ctx) context_close(ctx); @@ -2127,6 +2348,7 @@ int i915_gem_context_getparam_ioctl(struct drm_device *dev, void *data, struct drm_i915_file_private *file_priv = file->driver_priv; struct drm_i915_gem_context_param *args = data; struct i915_gem_context *ctx; + struct i915_address_space *vm; int ret = 0; ctx = i915_gem_context_lookup(file_priv, args->ctx_id); @@ -2136,12 +2358,10 @@ int i915_gem_context_getparam_ioctl(struct drm_device *dev, void *data, switch (args->param) { case I915_CONTEXT_PARAM_GTT_SIZE: args->size = 0; - rcu_read_lock(); - if (rcu_access_pointer(ctx->vm)) - args->value = rcu_dereference(ctx->vm)->total; - else - args->value = to_i915(dev)->ggtt.vm.total; - rcu_read_unlock(); + vm = i915_gem_context_get_eb_vm(ctx); + args->value = vm->total; + i915_vm_put(vm); + break; case I915_CONTEXT_PARAM_NO_ERROR_CAPTURE: @@ -2177,6 +2397,10 @@ int i915_gem_context_getparam_ioctl(struct drm_device *dev, void *data, args->value = i915_gem_context_is_persistent(ctx); break; + case I915_CONTEXT_PARAM_PROTECTED_CONTENT: + ret = get_protected(ctx, args); + break; + case I915_CONTEXT_PARAM_NO_ZEROMAP: case I915_CONTEXT_PARAM_BAN_PERIOD: case I915_CONTEXT_PARAM_ENGINES: diff --git a/drivers/gpu/drm/i915/gem/i915_gem_context.h b/drivers/gpu/drm/i915/gem/i915_gem_context.h index 18060536b0c2..babfecb17ad1 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_context.h +++ b/drivers/gpu/drm/i915/gem/i915_gem_context.h @@ -108,6 +108,12 @@ i915_gem_context_clear_user_engines(struct i915_gem_context *ctx) clear_bit(CONTEXT_USER_ENGINES, &ctx->flags); } +static inline bool +i915_gem_context_uses_protected_content(const struct i915_gem_context *ctx) +{ + return ctx->uses_protected_content; +} + /* i915_gem_context.c */ void i915_gem_init__contexts(struct drm_i915_private *i915); @@ -154,17 +160,22 @@ i915_gem_context_vm(struct i915_gem_context *ctx) return rcu_dereference_protected(ctx->vm, lockdep_is_held(&ctx->mutex)); } +static inline bool i915_gem_context_has_full_ppgtt(struct i915_gem_context *ctx) +{ + GEM_BUG_ON(!!ctx->vm != HAS_FULL_PPGTT(ctx->i915)); + + return !!ctx->vm; +} + static inline struct i915_address_space * -i915_gem_context_get_vm_rcu(struct i915_gem_context *ctx) +i915_gem_context_get_eb_vm(struct i915_gem_context *ctx) { struct i915_address_space *vm; - rcu_read_lock(); - vm = rcu_dereference(ctx->vm); + vm = ctx->vm; if (!vm) vm = &ctx->i915->ggtt.vm; vm = i915_vm_get(vm); - rcu_read_unlock(); return vm; } diff --git a/drivers/gpu/drm/i915/gem/i915_gem_context_types.h b/drivers/gpu/drm/i915/gem/i915_gem_context_types.h index 94c03a97cb77..282cdb8a5c5a 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_context_types.h +++ b/drivers/gpu/drm/i915/gem/i915_gem_context_types.h @@ -78,13 +78,16 @@ enum i915_gem_engine_type { /** @I915_GEM_ENGINE_TYPE_BALANCED: A load-balanced engine set */ I915_GEM_ENGINE_TYPE_BALANCED, + + /** @I915_GEM_ENGINE_TYPE_PARALLEL: A parallel engine set */ + I915_GEM_ENGINE_TYPE_PARALLEL, }; /** * struct i915_gem_proto_engine - prototype engine * * This struct describes an engine that a context may contain. Engines - * have three types: + * have four types: * * - I915_GEM_ENGINE_TYPE_INVALID: Invalid engines can be created but they * show up as a NULL in i915_gem_engines::engines[i] and any attempt to @@ -97,6 +100,10 @@ enum i915_gem_engine_type { * * - I915_GEM_ENGINE_TYPE_BALANCED: A load-balanced engine set, described * i915_gem_proto_engine::num_siblings and i915_gem_proto_engine::siblings. + * + * - I915_GEM_ENGINE_TYPE_PARALLEL: A parallel submission engine set, described + * i915_gem_proto_engine::width, i915_gem_proto_engine::num_siblings, and + * i915_gem_proto_engine::siblings. */ struct i915_gem_proto_engine { /** @type: Type of this engine */ @@ -105,10 +112,13 @@ struct i915_gem_proto_engine { /** @engine: Engine, for physical */ struct intel_engine_cs *engine; - /** @num_siblings: Number of balanced siblings */ + /** @num_siblings: Number of balanced or parallel siblings */ unsigned int num_siblings; - /** @siblings: Balanced siblings */ + /** @width: Width of each sibling */ + unsigned int width; + + /** @siblings: Balanced siblings or num_siblings * width for parallel */ struct intel_engine_cs **siblings; /** @sseu: Client-set SSEU parameters */ @@ -198,6 +208,12 @@ struct i915_gem_proto_context { /** @single_timeline: See See &i915_gem_context.syncobj */ bool single_timeline; + + /** @uses_protected_content: See &i915_gem_context.uses_protected_content */ + bool uses_protected_content; + + /** @pxp_wakeref: See &i915_gem_context.pxp_wakeref */ + intel_wakeref_t pxp_wakeref; }; /** @@ -262,7 +278,7 @@ struct i915_gem_context { * In other modes, this is a NULL pointer with the expectation that * the caller uses the shared global GTT. */ - struct i915_address_space __rcu *vm; + struct i915_address_space *vm; /** * @pid: process id of creator @@ -289,6 +305,18 @@ struct i915_gem_context { struct kref ref; /** + * @release_work: + * + * Work item for deferred cleanup, since i915_gem_context_put() tends to + * be called from hardirq context. + * + * FIXME: The only real reason for this is &i915_gem_engines.fence, all + * other callers are from process context and need at most some mild + * shuffling to pull the i915_gem_context_put() call out of a spinlock. + */ + struct work_struct release_work; + + /** * @rcu: rcu_head for deferred freeing. */ struct rcu_head rcu; @@ -309,6 +337,28 @@ struct i915_gem_context { #define CONTEXT_CLOSED 0 #define CONTEXT_USER_ENGINES 1 + /** + * @uses_protected_content: context uses PXP-encrypted objects. + * + * This flag can only be set at ctx creation time and it's immutable for + * the lifetime of the context. See I915_CONTEXT_PARAM_PROTECTED_CONTENT + * in uapi/drm/i915_drm.h for more info on setting restrictions and + * expected behaviour of marked contexts. + */ + bool uses_protected_content; + + /** + * @pxp_wakeref: wakeref to keep the device awake when PXP is in use + * + * PXP sessions are invalidated when the device is suspended, which in + * turns invalidates all contexts and objects using it. To keep the + * flow simple, we keep the device awake when contexts using PXP objects + * are in use. It is expected that the userspace application only uses + * PXP when the display is on, so taking a wakeref here shouldn't worsen + * our power metrics. + */ + intel_wakeref_t pxp_wakeref; + /** @mutex: guards everything that isn't engines or handles_vma */ struct mutex mutex; diff --git a/drivers/gpu/drm/i915/gem/i915_gem_create.c b/drivers/gpu/drm/i915/gem/i915_gem_create.c index 23fee13a3384..8955d6abcef1 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_create.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_create.c @@ -6,6 +6,7 @@ #include "gem/i915_gem_ioctls.h" #include "gem/i915_gem_lmem.h" #include "gem/i915_gem_region.h" +#include "pxp/intel_pxp.h" #include "i915_drv.h" #include "i915_trace.h" @@ -82,21 +83,11 @@ static int i915_gem_publish(struct drm_i915_gem_object *obj, return 0; } -/** - * Creates a new object using the same path as DRM_I915_GEM_CREATE_EXT - * @i915: i915 private - * @size: size of the buffer, in bytes - * @placements: possible placement regions, in priority order - * @n_placements: number of possible placement regions - * - * This function is exposed primarily for selftests and does very little - * error checking. It is assumed that the set of placement regions has - * already been verified to be valid. - */ -struct drm_i915_gem_object * -__i915_gem_object_create_user(struct drm_i915_private *i915, u64 size, - struct intel_memory_region **placements, - unsigned int n_placements) +static struct drm_i915_gem_object * +__i915_gem_object_create_user_ext(struct drm_i915_private *i915, u64 size, + struct intel_memory_region **placements, + unsigned int n_placements, + unsigned int ext_flags) { struct intel_memory_region *mr = placements[0]; struct drm_i915_gem_object *obj; @@ -135,6 +126,9 @@ __i915_gem_object_create_user(struct drm_i915_private *i915, u64 size, GEM_BUG_ON(size != obj->base.size); + /* Add any flag set by create_ext options */ + obj->flags |= ext_flags; + trace_i915_gem_object_create(obj); return obj; @@ -145,6 +139,26 @@ object_free: return ERR_PTR(ret); } +/** + * Creates a new object using the same path as DRM_I915_GEM_CREATE_EXT + * @i915: i915 private + * @size: size of the buffer, in bytes + * @placements: possible placement regions, in priority order + * @n_placements: number of possible placement regions + * + * This function is exposed primarily for selftests and does very little + * error checking. It is assumed that the set of placement regions has + * already been verified to be valid. + */ +struct drm_i915_gem_object * +__i915_gem_object_create_user(struct drm_i915_private *i915, u64 size, + struct intel_memory_region **placements, + unsigned int n_placements) +{ + return __i915_gem_object_create_user_ext(i915, size, placements, + n_placements, 0); +} + int i915_gem_dumb_create(struct drm_file *file, struct drm_device *dev, @@ -224,6 +238,7 @@ struct create_ext { struct drm_i915_private *i915; struct intel_memory_region *placements[INTEL_REGION_UNKNOWN]; unsigned int n_placements; + unsigned long flags; }; static void repr_placements(char *buf, size_t size, @@ -347,17 +362,34 @@ static int ext_set_placements(struct i915_user_extension __user *base, { struct drm_i915_gem_create_ext_memory_regions ext; - if (!IS_ENABLED(CONFIG_DRM_I915_UNSTABLE_FAKE_LMEM)) - return -ENODEV; - if (copy_from_user(&ext, base, sizeof(ext))) return -EFAULT; return set_placements(&ext, data); } +static int ext_set_protected(struct i915_user_extension __user *base, void *data) +{ + struct drm_i915_gem_create_ext_protected_content ext; + struct create_ext *ext_data = data; + + if (copy_from_user(&ext, base, sizeof(ext))) + return -EFAULT; + + if (ext.flags) + return -EINVAL; + + if (!intel_pxp_is_enabled(&ext_data->i915->gt.pxp)) + return -ENODEV; + + ext_data->flags |= I915_BO_PROTECTED; + + return 0; +} + static const i915_user_extension_fn create_extensions[] = { [I915_GEM_CREATE_EXT_MEMORY_REGIONS] = ext_set_placements, + [I915_GEM_CREATE_EXT_PROTECTED_CONTENT] = ext_set_protected, }; /** @@ -392,9 +424,10 @@ i915_gem_create_ext_ioctl(struct drm_device *dev, void *data, ext_data.n_placements = 1; } - obj = __i915_gem_object_create_user(i915, args->size, - ext_data.placements, - ext_data.n_placements); + obj = __i915_gem_object_create_user_ext(i915, args->size, + ext_data.placements, + ext_data.n_placements, + ext_data.flags); if (IS_ERR(obj)) return PTR_ERR(obj); diff --git a/drivers/gpu/drm/i915/gem/i915_gem_dmabuf.c b/drivers/gpu/drm/i915/gem/i915_gem_dmabuf.c index afa34111de02..1adcd8e02d29 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_dmabuf.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_dmabuf.c @@ -232,6 +232,7 @@ struct dma_buf *i915_gem_prime_export(struct drm_gem_object *gem_obj, int flags) static int i915_gem_object_get_pages_dmabuf(struct drm_i915_gem_object *obj) { + struct drm_i915_private *i915 = to_i915(obj->base.dev); struct sg_table *pages; unsigned int sg_page_sizes; @@ -242,8 +243,11 @@ static int i915_gem_object_get_pages_dmabuf(struct drm_i915_gem_object *obj) if (IS_ERR(pages)) return PTR_ERR(pages); - sg_page_sizes = i915_sg_dma_sizes(pages->sgl); + /* XXX: consider doing a vmap flush or something */ + if (!HAS_LLC(i915) || i915_gem_object_can_bypass_llc(obj)) + wbinvd_on_all_cpus(); + sg_page_sizes = i915_sg_dma_sizes(pages->sgl); __i915_gem_object_set_pages(obj, pages, sg_page_sizes); return 0; @@ -301,7 +305,8 @@ struct drm_gem_object *i915_gem_prime_import(struct drm_device *dev, } drm_gem_private_object_init(dev, &obj->base, dma_buf->size); - i915_gem_object_init(obj, &i915_gem_object_dmabuf_ops, &lock_class, 0); + i915_gem_object_init(obj, &i915_gem_object_dmabuf_ops, &lock_class, + I915_BO_ALLOC_USER); obj->base.import_attach = attach; obj->base.resv = dma_buf->resv; diff --git a/drivers/gpu/drm/i915/gem/i915_gem_execbuffer.c b/drivers/gpu/drm/i915/gem/i915_gem_execbuffer.c index 1aa249908b64..4d7da07442f2 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_execbuffer.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_execbuffer.c @@ -21,6 +21,8 @@ #include "gt/intel_gt_pm.h" #include "gt/intel_ring.h" +#include "pxp/intel_pxp.h" + #include "i915_drv.h" #include "i915_gem_clflush.h" #include "i915_gem_context.h" @@ -244,17 +246,25 @@ struct i915_execbuffer { struct drm_i915_gem_exec_object2 *exec; /** ioctl execobj[] */ struct eb_vma *vma; - struct intel_engine_cs *engine; /** engine to queue the request to */ + struct intel_gt *gt; /* gt for the execbuf */ struct intel_context *context; /* logical state for the request */ struct i915_gem_context *gem_context; /** caller's context */ - struct i915_request *request; /** our request to build */ - struct eb_vma *batch; /** identity of the batch obj/vma */ + /** our requests to build */ + struct i915_request *requests[MAX_ENGINE_INSTANCE + 1]; + /** identity of the batch obj/vma */ + struct eb_vma *batches[MAX_ENGINE_INSTANCE + 1]; struct i915_vma *trampoline; /** trampoline used for chaining */ + /** used for excl fence in dma_resv objects when > 1 BB submitted */ + struct dma_fence *composite_fence; + /** actual size of execobj[] as we may extend it for the cmdparser */ unsigned int buffer_count; + /* number of batches in execbuf IOCTL */ + unsigned int num_batches; + /** list of vma not yet bound during reservation phase */ struct list_head unbound; @@ -281,7 +291,8 @@ struct i915_execbuffer { u64 invalid_flags; /** Set of execobj.flags that are invalid */ - u64 batch_len; /** Length of batch within object */ + /** Length of batch within object */ + u64 batch_len[MAX_ENGINE_INSTANCE + 1]; u32 batch_start_offset; /** Location within object of batch */ u32 batch_flags; /** Flags composed for emit_bb_start() */ struct intel_gt_buffer_pool_node *batch_pool; /** pool node for batch buffer */ @@ -299,14 +310,13 @@ struct i915_execbuffer { }; static int eb_parse(struct i915_execbuffer *eb); -static struct i915_request *eb_pin_engine(struct i915_execbuffer *eb, - bool throttle); +static int eb_pin_engine(struct i915_execbuffer *eb, bool throttle); static void eb_unpin_engine(struct i915_execbuffer *eb); static inline bool eb_use_cmdparser(const struct i915_execbuffer *eb) { - return intel_engine_requires_cmd_parser(eb->engine) || - (intel_engine_using_cmd_parser(eb->engine) && + return intel_engine_requires_cmd_parser(eb->context->engine) || + (intel_engine_using_cmd_parser(eb->context->engine) && eb->args->batch_len); } @@ -533,11 +543,21 @@ eb_validate_vma(struct i915_execbuffer *eb, return 0; } -static void +static inline bool +is_batch_buffer(struct i915_execbuffer *eb, unsigned int buffer_idx) +{ + return eb->args->flags & I915_EXEC_BATCH_FIRST ? + buffer_idx < eb->num_batches : + buffer_idx >= eb->args->buffer_count - eb->num_batches; +} + +static int eb_add_vma(struct i915_execbuffer *eb, - unsigned int i, unsigned batch_idx, + unsigned int *current_batch, + unsigned int i, struct i915_vma *vma) { + struct drm_i915_private *i915 = eb->i915; struct drm_i915_gem_exec_object2 *entry = &eb->exec[i]; struct eb_vma *ev = &eb->vma[i]; @@ -564,15 +584,43 @@ eb_add_vma(struct i915_execbuffer *eb, * Note that actual hangs have only been observed on gen7, but for * paranoia do it everywhere. */ - if (i == batch_idx) { + if (is_batch_buffer(eb, i)) { if (entry->relocation_count && !(ev->flags & EXEC_OBJECT_PINNED)) ev->flags |= __EXEC_OBJECT_NEEDS_BIAS; if (eb->reloc_cache.has_fence) ev->flags |= EXEC_OBJECT_NEEDS_FENCE; - eb->batch = ev; + eb->batches[*current_batch] = ev; + + if (unlikely(ev->flags & EXEC_OBJECT_WRITE)) { + drm_dbg(&i915->drm, + "Attempting to use self-modifying batch buffer\n"); + return -EINVAL; + } + + if (range_overflows_t(u64, + eb->batch_start_offset, + eb->args->batch_len, + ev->vma->size)) { + drm_dbg(&i915->drm, "Attempting to use out-of-bounds batch\n"); + return -EINVAL; + } + + if (eb->args->batch_len == 0) + eb->batch_len[*current_batch] = ev->vma->size - + eb->batch_start_offset; + else + eb->batch_len[*current_batch] = eb->args->batch_len; + if (unlikely(eb->batch_len[*current_batch] == 0)) { /* impossible! */ + drm_dbg(&i915->drm, "Invalid batch length\n"); + return -EINVAL; + } + + ++*current_batch; } + + return 0; } static inline int use_cpu_reloc(const struct reloc_cache *cache, @@ -716,14 +764,6 @@ static int eb_reserve(struct i915_execbuffer *eb) } while (1); } -static unsigned int eb_batch_index(const struct i915_execbuffer *eb) -{ - if (eb->args->flags & I915_EXEC_BATCH_FIRST) - return 0; - else - return eb->buffer_count - 1; -} - static int eb_select_context(struct i915_execbuffer *eb) { struct i915_gem_context *ctx; @@ -733,7 +773,7 @@ static int eb_select_context(struct i915_execbuffer *eb) return PTR_ERR(ctx); eb->gem_context = ctx; - if (rcu_access_pointer(ctx->vm)) + if (i915_gem_context_has_full_ppgtt(ctx)) eb->invalid_flags |= EXEC_OBJECT_NEEDS_GTT; return 0; @@ -759,11 +799,7 @@ static int __eb_add_lut(struct i915_execbuffer *eb, /* Check that the context hasn't been closed in the meantime */ err = -EINTR; if (!mutex_lock_interruptible(&ctx->lut_mutex)) { - struct i915_address_space *vm = rcu_access_pointer(ctx->vm); - - if (unlikely(vm && vma->vm != vm)) - err = -EAGAIN; /* user racing with ctx set-vm */ - else if (likely(!i915_gem_context_is_closed(ctx))) + if (likely(!i915_gem_context_is_closed(ctx))) err = radix_tree_insert(&ctx->handles_vma, handle, vma); else err = -ENOENT; @@ -814,6 +850,22 @@ static struct i915_vma *eb_lookup_vma(struct i915_execbuffer *eb, u32 handle) if (unlikely(!obj)) return ERR_PTR(-ENOENT); + /* + * If the user has opted-in for protected-object tracking, make + * sure the object encryption can be used. + * We only need to do this when the object is first used with + * this context, because the context itself will be banned when + * the protected objects become invalid. + */ + if (i915_gem_context_uses_protected_content(eb->gem_context) && + i915_gem_object_is_protected(obj)) { + err = intel_pxp_key_check(&vm->gt->pxp, obj, true); + if (err) { + i915_gem_object_put(obj); + return ERR_PTR(err); + } + } + vma = i915_vma_instance(obj, vm, NULL); if (IS_ERR(vma)) { i915_gem_object_put(obj); @@ -832,9 +884,7 @@ static struct i915_vma *eb_lookup_vma(struct i915_execbuffer *eb, u32 handle) static int eb_lookup_vmas(struct i915_execbuffer *eb) { - struct drm_i915_private *i915 = eb->i915; - unsigned int batch = eb_batch_index(eb); - unsigned int i; + unsigned int i, current_batch = 0; int err = 0; INIT_LIST_HEAD(&eb->relocs); @@ -854,7 +904,9 @@ static int eb_lookup_vmas(struct i915_execbuffer *eb) goto err; } - eb_add_vma(eb, i, batch, vma); + err = eb_add_vma(eb, ¤t_batch, i, vma); + if (err) + return err; if (i915_gem_object_is_userptr(vma->obj)) { err = i915_gem_object_userptr_submit_init(vma->obj); @@ -877,26 +929,6 @@ static int eb_lookup_vmas(struct i915_execbuffer *eb) } } - if (unlikely(eb->batch->flags & EXEC_OBJECT_WRITE)) { - drm_dbg(&i915->drm, - "Attempting to use self-modifying batch buffer\n"); - return -EINVAL; - } - - if (range_overflows_t(u64, - eb->batch_start_offset, eb->batch_len, - eb->batch->vma->size)) { - drm_dbg(&i915->drm, "Attempting to use out-of-bounds batch\n"); - return -EINVAL; - } - - if (eb->batch_len == 0) - eb->batch_len = eb->batch->vma->size - eb->batch_start_offset; - if (unlikely(eb->batch_len == 0)) { /* impossible! */ - drm_dbg(&i915->drm, "Invalid batch length\n"); - return -EINVAL; - } - return 0; err: @@ -1629,8 +1661,7 @@ static int eb_reinit_userptr(struct i915_execbuffer *eb) return 0; } -static noinline int eb_relocate_parse_slow(struct i915_execbuffer *eb, - struct i915_request *rq) +static noinline int eb_relocate_parse_slow(struct i915_execbuffer *eb) { bool have_copy = false; struct eb_vma *ev; @@ -1646,21 +1677,6 @@ repeat: eb_release_vmas(eb, false); i915_gem_ww_ctx_fini(&eb->ww); - if (rq) { - /* nonblocking is always false */ - if (i915_request_wait(rq, I915_WAIT_INTERRUPTIBLE, - MAX_SCHEDULE_TIMEOUT) < 0) { - i915_request_put(rq); - rq = NULL; - - err = -EINTR; - goto err_relock; - } - - i915_request_put(rq); - rq = NULL; - } - /* * We take 3 passes through the slowpatch. * @@ -1687,28 +1703,21 @@ repeat: if (!err) err = eb_reinit_userptr(eb); -err_relock: i915_gem_ww_ctx_init(&eb->ww, true); if (err) goto out; /* reacquire the objects */ repeat_validate: - rq = eb_pin_engine(eb, false); - if (IS_ERR(rq)) { - err = PTR_ERR(rq); - rq = NULL; + err = eb_pin_engine(eb, false); + if (err) goto err; - } - - /* We didn't throttle, should be NULL */ - GEM_WARN_ON(rq); err = eb_validate_vmas(eb); if (err) goto err; - GEM_BUG_ON(!eb->batch); + GEM_BUG_ON(!eb->batches[0]); list_for_each_entry(ev, &eb->relocs, reloc_link) { if (!have_copy) { @@ -1772,46 +1781,23 @@ out: } } - if (rq) - i915_request_put(rq); - return err; } static int eb_relocate_parse(struct i915_execbuffer *eb) { int err; - struct i915_request *rq = NULL; bool throttle = true; retry: - rq = eb_pin_engine(eb, throttle); - if (IS_ERR(rq)) { - err = PTR_ERR(rq); - rq = NULL; + err = eb_pin_engine(eb, throttle); + if (err) { if (err != -EDEADLK) return err; goto err; } - if (rq) { - bool nonblock = eb->file->filp->f_flags & O_NONBLOCK; - - /* Need to drop all locks now for throttling, take slowpath */ - err = i915_request_wait(rq, I915_WAIT_INTERRUPTIBLE, 0); - if (err == -ETIME) { - if (nonblock) { - err = -EWOULDBLOCK; - i915_request_put(rq); - goto err; - } - goto slow; - } - i915_request_put(rq); - rq = NULL; - } - /* only throttle once, even if we didn't need to throttle */ throttle = false; @@ -1851,7 +1837,7 @@ err: return err; slow: - err = eb_relocate_parse_slow(eb, rq); + err = eb_relocate_parse_slow(eb); if (err) /* * If the user expects the execobject.offset and @@ -1865,11 +1851,40 @@ slow: return err; } +/* + * Using two helper loops for the order of which requests / batches are created + * and added the to backend. Requests are created in order from the parent to + * the last child. Requests are added in the reverse order, from the last child + * to parent. This is done for locking reasons as the timeline lock is acquired + * during request creation and released when the request is added to the + * backend. To make lockdep happy (see intel_context_timeline_lock) this must be + * the ordering. + */ +#define for_each_batch_create_order(_eb, _i) \ + for ((_i) = 0; (_i) < (_eb)->num_batches; ++(_i)) +#define for_each_batch_add_order(_eb, _i) \ + BUILD_BUG_ON(!typecheck(int, _i)); \ + for ((_i) = (_eb)->num_batches - 1; (_i) >= 0; --(_i)) + +static struct i915_request * +eb_find_first_request_added(struct i915_execbuffer *eb) +{ + int i; + + for_each_batch_add_order(eb, i) + if (eb->requests[i]) + return eb->requests[i]; + + GEM_BUG_ON("Request not found"); + + return NULL; +} + static int eb_move_to_gpu(struct i915_execbuffer *eb) { const unsigned int count = eb->buffer_count; unsigned int i = count; - int err = 0; + int err = 0, j; while (i--) { struct eb_vma *ev = &eb->vma[i]; @@ -1882,11 +1897,17 @@ static int eb_move_to_gpu(struct i915_execbuffer *eb) if (flags & EXEC_OBJECT_CAPTURE) { struct i915_capture_list *capture; - capture = kmalloc(sizeof(*capture), GFP_KERNEL); - if (capture) { - capture->next = eb->request->capture_list; - capture->vma = vma; - eb->request->capture_list = capture; + for_each_batch_create_order(eb, j) { + if (!eb->requests[j]) + break; + + capture = kmalloc(sizeof(*capture), GFP_KERNEL); + if (capture) { + capture->next = + eb->requests[j]->capture_list; + capture->vma = vma; + eb->requests[j]->capture_list = capture; + } } } @@ -1901,20 +1922,43 @@ static int eb_move_to_gpu(struct i915_execbuffer *eb) * !(obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_READ) * but gcc's optimiser doesn't handle that as well and emits * two jumps instead of one. Maybe one day... + * + * FIXME: There is also sync flushing in set_pages(), which + * serves a different purpose(some of the time at least). + * + * We should consider: + * + * 1. Rip out the async flush code. + * + * 2. Or make the sync flushing use the async clflush path + * using mandatory fences underneath. Currently the below + * async flush happens after we bind the object. */ if (unlikely(obj->cache_dirty & ~obj->cache_coherent)) { if (i915_gem_clflush_object(obj, 0)) flags &= ~EXEC_OBJECT_ASYNC; } + /* We only need to await on the first request */ if (err == 0 && !(flags & EXEC_OBJECT_ASYNC)) { err = i915_request_await_object - (eb->request, obj, flags & EXEC_OBJECT_WRITE); + (eb_find_first_request_added(eb), obj, + flags & EXEC_OBJECT_WRITE); } - if (err == 0) - err = i915_vma_move_to_active(vma, eb->request, - flags | __EXEC_OBJECT_NO_RESERVE); + for_each_batch_add_order(eb, j) { + if (err) + break; + if (!eb->requests[j]) + continue; + + err = _i915_vma_move_to_active(vma, eb->requests[j], + j ? NULL : + eb->composite_fence ? + eb->composite_fence : + &eb->requests[j]->fence, + flags | __EXEC_OBJECT_NO_RESERVE); + } } #ifdef CONFIG_MMU_NOTIFIER @@ -1945,11 +1989,16 @@ static int eb_move_to_gpu(struct i915_execbuffer *eb) goto err_skip; /* Unconditionally flush any chipset caches (for streaming writes). */ - intel_gt_chipset_flush(eb->engine->gt); + intel_gt_chipset_flush(eb->gt); return 0; err_skip: - i915_request_set_error_once(eb->request, err); + for_each_batch_create_order(eb, j) { + if (!eb->requests[j]) + break; + + i915_request_set_error_once(eb->requests[j], err); + } return err; } @@ -2044,14 +2093,17 @@ static int eb_parse(struct i915_execbuffer *eb) int err; if (!eb_use_cmdparser(eb)) { - batch = eb_dispatch_secure(eb, eb->batch->vma); + batch = eb_dispatch_secure(eb, eb->batches[0]->vma); if (IS_ERR(batch)) return PTR_ERR(batch); goto secure_batch; } - len = eb->batch_len; + if (intel_context_is_parallel(eb->context)) + return -EINVAL; + + len = eb->batch_len[0]; if (!CMDPARSER_USES_GGTT(eb->i915)) { /* * ppGTT backed shadow buffers must be mapped RO, to prevent @@ -2065,11 +2117,11 @@ static int eb_parse(struct i915_execbuffer *eb) } else { len += I915_CMD_PARSER_TRAMPOLINE_SIZE; } - if (unlikely(len < eb->batch_len)) /* last paranoid check of overflow */ + if (unlikely(len < eb->batch_len[0])) /* last paranoid check of overflow */ return -EINVAL; if (!pool) { - pool = intel_gt_get_buffer_pool(eb->engine->gt, len, + pool = intel_gt_get_buffer_pool(eb->gt, len, I915_MAP_WB); if (IS_ERR(pool)) return PTR_ERR(pool); @@ -2094,7 +2146,7 @@ static int eb_parse(struct i915_execbuffer *eb) trampoline = shadow; shadow = shadow_batch_pin(eb, pool->obj, - &eb->engine->gt->ggtt->vm, + &eb->gt->ggtt->vm, PIN_GLOBAL); if (IS_ERR(shadow)) { err = PTR_ERR(shadow); @@ -2116,26 +2168,29 @@ static int eb_parse(struct i915_execbuffer *eb) if (err) goto err_trampoline; - err = intel_engine_cmd_parser(eb->engine, - eb->batch->vma, + err = intel_engine_cmd_parser(eb->context->engine, + eb->batches[0]->vma, eb->batch_start_offset, - eb->batch_len, + eb->batch_len[0], shadow, trampoline); if (err) goto err_unpin_batch; - eb->batch = &eb->vma[eb->buffer_count++]; - eb->batch->vma = i915_vma_get(shadow); - eb->batch->flags = __EXEC_OBJECT_HAS_PIN; + eb->batches[0] = &eb->vma[eb->buffer_count++]; + eb->batches[0]->vma = i915_vma_get(shadow); + eb->batches[0]->flags = __EXEC_OBJECT_HAS_PIN; eb->trampoline = trampoline; eb->batch_start_offset = 0; secure_batch: if (batch) { - eb->batch = &eb->vma[eb->buffer_count++]; - eb->batch->flags = __EXEC_OBJECT_HAS_PIN; - eb->batch->vma = i915_vma_get(batch); + if (intel_context_is_parallel(eb->context)) + return -EINVAL; + + eb->batches[0] = &eb->vma[eb->buffer_count++]; + eb->batches[0]->flags = __EXEC_OBJECT_HAS_PIN; + eb->batches[0]->vma = i915_vma_get(batch); } return 0; @@ -2151,19 +2206,18 @@ err: return err; } -static int eb_submit(struct i915_execbuffer *eb, struct i915_vma *batch) +static int eb_request_submit(struct i915_execbuffer *eb, + struct i915_request *rq, + struct i915_vma *batch, + u64 batch_len) { int err; - if (intel_context_nopreempt(eb->context)) - __set_bit(I915_FENCE_FLAG_NOPREEMPT, &eb->request->fence.flags); - - err = eb_move_to_gpu(eb); - if (err) - return err; + if (intel_context_nopreempt(rq->context)) + __set_bit(I915_FENCE_FLAG_NOPREEMPT, &rq->fence.flags); if (eb->args->flags & I915_EXEC_GEN7_SOL_RESET) { - err = i915_reset_gen7_sol_offsets(eb->request); + err = i915_reset_gen7_sol_offsets(rq); if (err) return err; } @@ -2174,26 +2228,26 @@ static int eb_submit(struct i915_execbuffer *eb, struct i915_vma *batch) * allows us to determine if the batch is still waiting on the GPU * or actually running by checking the breadcrumb. */ - if (eb->engine->emit_init_breadcrumb) { - err = eb->engine->emit_init_breadcrumb(eb->request); + if (rq->context->engine->emit_init_breadcrumb) { + err = rq->context->engine->emit_init_breadcrumb(rq); if (err) return err; } - err = eb->engine->emit_bb_start(eb->request, - batch->node.start + - eb->batch_start_offset, - eb->batch_len, - eb->batch_flags); + err = rq->context->engine->emit_bb_start(rq, + batch->node.start + + eb->batch_start_offset, + batch_len, + eb->batch_flags); if (err) return err; if (eb->trampoline) { + GEM_BUG_ON(intel_context_is_parallel(rq->context)); GEM_BUG_ON(eb->batch_start_offset); - err = eb->engine->emit_bb_start(eb->request, - eb->trampoline->node.start + - eb->batch_len, - 0, 0); + err = rq->context->engine->emit_bb_start(rq, + eb->trampoline->node.start + + batch_len, 0, 0); if (err) return err; } @@ -2201,6 +2255,27 @@ static int eb_submit(struct i915_execbuffer *eb, struct i915_vma *batch) return 0; } +static int eb_submit(struct i915_execbuffer *eb) +{ + unsigned int i; + int err; + + err = eb_move_to_gpu(eb); + + for_each_batch_create_order(eb, i) { + if (!eb->requests[i]) + break; + + trace_i915_request_queue(eb->requests[i], eb->batch_flags); + if (!err) + err = eb_request_submit(eb, eb->requests[i], + eb->batches[i]->vma, + eb->batch_len[i]); + } + + return err; +} + static int num_vcs_engines(const struct drm_i915_private *i915) { return hweight_long(VDBOX_MASK(&i915->gt)); @@ -2266,26 +2341,11 @@ static struct i915_request *eb_throttle(struct i915_execbuffer *eb, struct intel return i915_request_get(rq); } -static struct i915_request *eb_pin_engine(struct i915_execbuffer *eb, bool throttle) +static int eb_pin_timeline(struct i915_execbuffer *eb, struct intel_context *ce, + bool throttle) { - struct intel_context *ce = eb->context; struct intel_timeline *tl; struct i915_request *rq = NULL; - int err; - - GEM_BUG_ON(eb->args->flags & __EXEC_ENGINE_PINNED); - - if (unlikely(intel_context_is_banned(ce))) - return ERR_PTR(-EIO); - - /* - * Pinning the contexts may generate requests in order to acquire - * GGTT space, so do this first before we reserve a seqno for - * ourselves. - */ - err = intel_context_pin_ww(ce, &eb->ww); - if (err) - return ERR_PTR(err); /* * Take a local wakeref for preparing to dispatch the execbuf as @@ -2296,33 +2356,108 @@ static struct i915_request *eb_pin_engine(struct i915_execbuffer *eb, bool throt * taken on the engine, and the parent device. */ tl = intel_context_timeline_lock(ce); - if (IS_ERR(tl)) { - intel_context_unpin(ce); - return ERR_CAST(tl); - } + if (IS_ERR(tl)) + return PTR_ERR(tl); intel_context_enter(ce); if (throttle) rq = eb_throttle(eb, ce); intel_context_timeline_unlock(tl); + if (rq) { + bool nonblock = eb->file->filp->f_flags & O_NONBLOCK; + long timeout = nonblock ? 0 : MAX_SCHEDULE_TIMEOUT; + + if (i915_request_wait(rq, I915_WAIT_INTERRUPTIBLE, + timeout) < 0) { + i915_request_put(rq); + + tl = intel_context_timeline_lock(ce); + intel_context_exit(ce); + intel_context_timeline_unlock(tl); + + if (nonblock) + return -EWOULDBLOCK; + else + return -EINTR; + } + i915_request_put(rq); + } + + return 0; +} + +static int eb_pin_engine(struct i915_execbuffer *eb, bool throttle) +{ + struct intel_context *ce = eb->context, *child; + int err; + int i = 0, j = 0; + + GEM_BUG_ON(eb->args->flags & __EXEC_ENGINE_PINNED); + + if (unlikely(intel_context_is_banned(ce))) + return -EIO; + + /* + * Pinning the contexts may generate requests in order to acquire + * GGTT space, so do this first before we reserve a seqno for + * ourselves. + */ + err = intel_context_pin_ww(ce, &eb->ww); + if (err) + return err; + for_each_child(ce, child) { + err = intel_context_pin_ww(child, &eb->ww); + GEM_BUG_ON(err); /* perma-pinned should incr a counter */ + } + + for_each_child(ce, child) { + err = eb_pin_timeline(eb, child, throttle); + if (err) + goto unwind; + ++i; + } + err = eb_pin_timeline(eb, ce, throttle); + if (err) + goto unwind; + eb->args->flags |= __EXEC_ENGINE_PINNED; - return rq; + return 0; + +unwind: + for_each_child(ce, child) { + if (j++ < i) { + mutex_lock(&child->timeline->mutex); + intel_context_exit(child); + mutex_unlock(&child->timeline->mutex); + } + } + for_each_child(ce, child) + intel_context_unpin(child); + intel_context_unpin(ce); + return err; } static void eb_unpin_engine(struct i915_execbuffer *eb) { - struct intel_context *ce = eb->context; - struct intel_timeline *tl = ce->timeline; + struct intel_context *ce = eb->context, *child; if (!(eb->args->flags & __EXEC_ENGINE_PINNED)) return; eb->args->flags &= ~__EXEC_ENGINE_PINNED; - mutex_lock(&tl->mutex); + for_each_child(ce, child) { + mutex_lock(&child->timeline->mutex); + intel_context_exit(child); + mutex_unlock(&child->timeline->mutex); + + intel_context_unpin(child); + } + + mutex_lock(&ce->timeline->mutex); intel_context_exit(ce); - mutex_unlock(&tl->mutex); + mutex_unlock(&ce->timeline->mutex); intel_context_unpin(ce); } @@ -2373,7 +2508,7 @@ eb_select_legacy_ring(struct i915_execbuffer *eb) static int eb_select_engine(struct i915_execbuffer *eb) { - struct intel_context *ce; + struct intel_context *ce, *child; unsigned int idx; int err; @@ -2386,6 +2521,20 @@ eb_select_engine(struct i915_execbuffer *eb) if (IS_ERR(ce)) return PTR_ERR(ce); + if (intel_context_is_parallel(ce)) { + if (eb->buffer_count < ce->parallel.number_children + 1) { + intel_context_put(ce); + return -EINVAL; + } + if (eb->batch_start_offset || eb->args->batch_len) { + intel_context_put(ce); + return -EINVAL; + } + } + eb->num_batches = ce->parallel.number_children + 1; + + for_each_child(ce, child) + intel_context_get(child); intel_gt_pm_get(ce->engine->gt); if (!test_bit(CONTEXT_ALLOC_BIT, &ce->flags)) { @@ -2393,6 +2542,13 @@ eb_select_engine(struct i915_execbuffer *eb) if (err) goto err; } + for_each_child(ce, child) { + if (!test_bit(CONTEXT_ALLOC_BIT, &child->flags)) { + err = intel_context_alloc_state(child); + if (err) + goto err; + } + } /* * ABI: Before userspace accesses the GPU (e.g. execbuffer), report @@ -2403,7 +2559,7 @@ eb_select_engine(struct i915_execbuffer *eb) goto err; eb->context = ce; - eb->engine = ce->engine; + eb->gt = ce->engine->gt; /* * Make sure engine pool stays alive even if we call intel_context_put @@ -2414,6 +2570,8 @@ eb_select_engine(struct i915_execbuffer *eb) err: intel_gt_pm_put(ce->engine->gt); + for_each_child(ce, child) + intel_context_put(child); intel_context_put(ce); return err; } @@ -2421,7 +2579,11 @@ err: static void eb_put_engine(struct i915_execbuffer *eb) { - intel_gt_pm_put(eb->engine->gt); + struct intel_context *child; + + intel_gt_pm_put(eb->gt); + for_each_child(eb->context, child) + intel_context_put(child); intel_context_put(eb->context); } @@ -2644,7 +2806,8 @@ static void put_fence_array(struct eb_fence *fences, int num_fences) } static int -await_fence_array(struct i915_execbuffer *eb) +await_fence_array(struct i915_execbuffer *eb, + struct i915_request *rq) { unsigned int n; int err; @@ -2658,8 +2821,7 @@ await_fence_array(struct i915_execbuffer *eb) if (!eb->fences[n].dma_fence) continue; - err = i915_request_await_dma_fence(eb->request, - eb->fences[n].dma_fence); + err = i915_request_await_dma_fence(rq, eb->fences[n].dma_fence); if (err < 0) return err; } @@ -2667,9 +2829,9 @@ await_fence_array(struct i915_execbuffer *eb) return 0; } -static void signal_fence_array(const struct i915_execbuffer *eb) +static void signal_fence_array(const struct i915_execbuffer *eb, + struct dma_fence * const fence) { - struct dma_fence * const fence = &eb->request->fence; unsigned int n; for (n = 0; n < eb->num_fences; n++) { @@ -2717,9 +2879,9 @@ static void retire_requests(struct intel_timeline *tl, struct i915_request *end) break; } -static int eb_request_add(struct i915_execbuffer *eb, int err) +static int eb_request_add(struct i915_execbuffer *eb, struct i915_request *rq, + int err, bool last_parallel) { - struct i915_request *rq = eb->request; struct intel_timeline * const tl = i915_request_timeline(rq); struct i915_sched_attr attr = {}; struct i915_request *prev; @@ -2741,6 +2903,17 @@ static int eb_request_add(struct i915_execbuffer *eb, int err) err = -ENOENT; /* override any transient errors */ } + if (intel_context_is_parallel(eb->context)) { + if (err) { + __i915_request_skip(rq); + set_bit(I915_FENCE_FLAG_SKIP_PARALLEL, + &rq->fence.flags); + } + if (last_parallel) + set_bit(I915_FENCE_FLAG_SUBMIT_PARALLEL, + &rq->fence.flags); + } + __i915_request_queue(rq, &attr); /* Try to clean up the client's timeline after submitting the request */ @@ -2752,6 +2925,25 @@ static int eb_request_add(struct i915_execbuffer *eb, int err) return err; } +static int eb_requests_add(struct i915_execbuffer *eb, int err) +{ + int i; + + /* + * We iterate in reverse order of creation to release timeline mutexes in + * same order. + */ + for_each_batch_add_order(eb, i) { + struct i915_request *rq = eb->requests[i]; + + if (!rq) + continue; + err |= eb_request_add(eb, rq, err, i == 0); + } + + return err; +} + static const i915_user_extension_fn execbuf_extensions[] = { [DRM_I915_GEM_EXECBUFFER_EXT_TIMELINE_FENCES] = parse_timeline_fences, }; @@ -2778,6 +2970,185 @@ parse_execbuf2_extensions(struct drm_i915_gem_execbuffer2 *args, eb); } +static void eb_requests_get(struct i915_execbuffer *eb) +{ + unsigned int i; + + for_each_batch_create_order(eb, i) { + if (!eb->requests[i]) + break; + + i915_request_get(eb->requests[i]); + } +} + +static void eb_requests_put(struct i915_execbuffer *eb) +{ + unsigned int i; + + for_each_batch_create_order(eb, i) { + if (!eb->requests[i]) + break; + + i915_request_put(eb->requests[i]); + } +} + +static struct sync_file * +eb_composite_fence_create(struct i915_execbuffer *eb, int out_fence_fd) +{ + struct sync_file *out_fence = NULL; + struct dma_fence_array *fence_array; + struct dma_fence **fences; + unsigned int i; + + GEM_BUG_ON(!intel_context_is_parent(eb->context)); + + fences = kmalloc_array(eb->num_batches, sizeof(*fences), GFP_KERNEL); + if (!fences) + return ERR_PTR(-ENOMEM); + + for_each_batch_create_order(eb, i) { + fences[i] = &eb->requests[i]->fence; + __set_bit(I915_FENCE_FLAG_COMPOSITE, + &eb->requests[i]->fence.flags); + } + + fence_array = dma_fence_array_create(eb->num_batches, + fences, + eb->context->parallel.fence_context, + eb->context->parallel.seqno, + false); + if (!fence_array) { + kfree(fences); + return ERR_PTR(-ENOMEM); + } + + /* Move ownership to the dma_fence_array created above */ + for_each_batch_create_order(eb, i) + dma_fence_get(fences[i]); + + if (out_fence_fd != -1) { + out_fence = sync_file_create(&fence_array->base); + /* sync_file now owns fence_arry, drop creation ref */ + dma_fence_put(&fence_array->base); + if (!out_fence) + return ERR_PTR(-ENOMEM); + } + + eb->composite_fence = &fence_array->base; + + return out_fence; +} + +static struct sync_file * +eb_fences_add(struct i915_execbuffer *eb, struct i915_request *rq, + struct dma_fence *in_fence, int out_fence_fd) +{ + struct sync_file *out_fence = NULL; + int err; + + if (unlikely(eb->gem_context->syncobj)) { + struct dma_fence *fence; + + fence = drm_syncobj_fence_get(eb->gem_context->syncobj); + err = i915_request_await_dma_fence(rq, fence); + dma_fence_put(fence); + if (err) + return ERR_PTR(err); + } + + if (in_fence) { + if (eb->args->flags & I915_EXEC_FENCE_SUBMIT) + err = i915_request_await_execution(rq, in_fence); + else + err = i915_request_await_dma_fence(rq, in_fence); + if (err < 0) + return ERR_PTR(err); + } + + if (eb->fences) { + err = await_fence_array(eb, rq); + if (err) + return ERR_PTR(err); + } + + if (intel_context_is_parallel(eb->context)) { + out_fence = eb_composite_fence_create(eb, out_fence_fd); + if (IS_ERR(out_fence)) + return ERR_PTR(-ENOMEM); + } else if (out_fence_fd != -1) { + out_fence = sync_file_create(&rq->fence); + if (!out_fence) + return ERR_PTR(-ENOMEM); + } + + return out_fence; +} + +static struct intel_context * +eb_find_context(struct i915_execbuffer *eb, unsigned int context_number) +{ + struct intel_context *child; + + if (likely(context_number == 0)) + return eb->context; + + for_each_child(eb->context, child) + if (!--context_number) + return child; + + GEM_BUG_ON("Context not found"); + + return NULL; +} + +static struct sync_file * +eb_requests_create(struct i915_execbuffer *eb, struct dma_fence *in_fence, + int out_fence_fd) +{ + struct sync_file *out_fence = NULL; + unsigned int i; + + for_each_batch_create_order(eb, i) { + /* Allocate a request for this batch buffer nice and early. */ + eb->requests[i] = i915_request_create(eb_find_context(eb, i)); + if (IS_ERR(eb->requests[i])) { + out_fence = ERR_PTR(PTR_ERR(eb->requests[i])); + eb->requests[i] = NULL; + return out_fence; + } + + /* + * Only the first request added (committed to backend) has to + * take the in fences into account as all subsequent requests + * will have fences inserted inbetween them. + */ + if (i + 1 == eb->num_batches) { + out_fence = eb_fences_add(eb, eb->requests[i], + in_fence, out_fence_fd); + if (IS_ERR(out_fence)) + return out_fence; + } + + /* + * Whilst this request exists, batch_obj will be on the + * active_list, and so will hold the active reference. Only when + * this request is retired will the batch_obj be moved onto + * the inactive_list and lose its active reference. Hence we do + * not need to explicitly hold another reference here. + */ + eb->requests[i]->batch = eb->batches[i]->vma; + if (eb->batch_pool) { + GEM_BUG_ON(intel_context_is_parallel(eb->context)); + intel_gt_buffer_pool_mark_active(eb->batch_pool, + eb->requests[i]); + } + } + + return out_fence; +} + static int i915_gem_do_execbuffer(struct drm_device *dev, struct drm_file *file, @@ -2788,7 +3159,6 @@ i915_gem_do_execbuffer(struct drm_device *dev, struct i915_execbuffer eb; struct dma_fence *in_fence = NULL; struct sync_file *out_fence = NULL; - struct i915_vma *batch; int out_fence_fd = -1; int err; @@ -2812,12 +3182,15 @@ i915_gem_do_execbuffer(struct drm_device *dev, eb.buffer_count = args->buffer_count; eb.batch_start_offset = args->batch_start_offset; - eb.batch_len = args->batch_len; eb.trampoline = NULL; eb.fences = NULL; eb.num_fences = 0; + memset(eb.requests, 0, sizeof(struct i915_request *) * + ARRAY_SIZE(eb.requests)); + eb.composite_fence = NULL; + eb.batch_flags = 0; if (args->flags & I915_EXEC_SECURE) { if (GRAPHICS_VER(i915) >= 11) @@ -2901,70 +3274,25 @@ i915_gem_do_execbuffer(struct drm_device *dev, ww_acquire_done(&eb.ww.ctx); - batch = eb.batch->vma; - - /* Allocate a request for this batch buffer nice and early. */ - eb.request = i915_request_create(eb.context); - if (IS_ERR(eb.request)) { - err = PTR_ERR(eb.request); - goto err_vma; - } - - if (unlikely(eb.gem_context->syncobj)) { - struct dma_fence *fence; - - fence = drm_syncobj_fence_get(eb.gem_context->syncobj); - err = i915_request_await_dma_fence(eb.request, fence); - dma_fence_put(fence); - if (err) - goto err_ext; - } - - if (in_fence) { - if (args->flags & I915_EXEC_FENCE_SUBMIT) - err = i915_request_await_execution(eb.request, - in_fence); - else - err = i915_request_await_dma_fence(eb.request, - in_fence); - if (err < 0) - goto err_request; - } - - if (eb.fences) { - err = await_fence_array(&eb); - if (err) - goto err_request; - } - - if (out_fence_fd != -1) { - out_fence = sync_file_create(&eb.request->fence); - if (!out_fence) { - err = -ENOMEM; + out_fence = eb_requests_create(&eb, in_fence, out_fence_fd); + if (IS_ERR(out_fence)) { + err = PTR_ERR(out_fence); + if (eb.requests[0]) goto err_request; - } + else + goto err_vma; } - /* - * Whilst this request exists, batch_obj will be on the - * active_list, and so will hold the active reference. Only when this - * request is retired will the the batch_obj be moved onto the - * inactive_list and lose its active reference. Hence we do not need - * to explicitly hold another reference here. - */ - eb.request->batch = batch; - if (eb.batch_pool) - intel_gt_buffer_pool_mark_active(eb.batch_pool, eb.request); - - trace_i915_request_queue(eb.request, eb.batch_flags); - err = eb_submit(&eb, batch); + err = eb_submit(&eb); err_request: - i915_request_get(eb.request); - err = eb_request_add(&eb, err); + eb_requests_get(&eb); + err = eb_requests_add(&eb, err); if (eb.fences) - signal_fence_array(&eb); + signal_fence_array(&eb, eb.composite_fence ? + eb.composite_fence : + &eb.requests[0]->fence); if (out_fence) { if (err == 0) { @@ -2979,10 +3307,15 @@ err_request: if (unlikely(eb.gem_context->syncobj)) { drm_syncobj_replace_fence(eb.gem_context->syncobj, - &eb.request->fence); + eb.composite_fence ? + eb.composite_fence : + &eb.requests[0]->fence); } - i915_request_put(eb.request); + if (!out_fence && eb.composite_fence) + dma_fence_put(eb.composite_fence); + + eb_requests_put(&eb); err_vma: eb_release_vmas(&eb, true); diff --git a/drivers/gpu/drm/i915/gem/i915_gem_internal.c b/drivers/gpu/drm/i915/gem/i915_gem_internal.c index e5ae9c06510c..a57a6b7013c2 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_internal.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_internal.c @@ -134,6 +134,8 @@ static void i915_gem_object_put_pages_internal(struct drm_i915_gem_object *obj, internal_free_pages(pages); obj->mm.dirty = false; + + __start_cpu_write(obj); } static const struct drm_i915_gem_object_ops i915_gem_object_internal_ops = { diff --git a/drivers/gpu/drm/i915/gem/i915_gem_lmem.c b/drivers/gpu/drm/i915/gem/i915_gem_lmem.c index eb345305dc52..444f8268b9c5 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_lmem.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_lmem.c @@ -56,8 +56,8 @@ bool i915_gem_object_is_lmem(struct drm_i915_gem_object *obj) * @obj: The object to check. * * This function is intended to be called from within the fence signaling - * path where the fence keeps the object from being migrated. For example - * during gpu reset or similar. + * path where the fence, or a pin, keeps the object from being migrated. For + * example during gpu reset or similar. * * Return: Whether the object is resident in lmem. */ @@ -66,7 +66,8 @@ bool __i915_gem_object_is_lmem(struct drm_i915_gem_object *obj) struct intel_memory_region *mr = READ_ONCE(obj->mm.region); #ifdef CONFIG_LOCKDEP - GEM_WARN_ON(dma_resv_test_signaled(obj->base.resv, true)); + GEM_WARN_ON(dma_resv_test_signaled(obj->base.resv, true) && + i915_gem_object_evictable(obj)); #endif return mr && (mr->type == INTEL_MEMORY_LOCAL || mr->type == INTEL_MEMORY_STOLEN_LOCAL); @@ -104,6 +105,32 @@ __i915_gem_object_create_lmem_with_ps(struct drm_i915_private *i915, } struct drm_i915_gem_object * +i915_gem_object_create_lmem_from_data(struct drm_i915_private *i915, + const void *data, size_t size) +{ + struct drm_i915_gem_object *obj; + void *map; + + obj = i915_gem_object_create_lmem(i915, + round_up(size, PAGE_SIZE), + I915_BO_ALLOC_CONTIGUOUS); + if (IS_ERR(obj)) + return obj; + + map = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WC); + if (IS_ERR(map)) { + i915_gem_object_put(obj); + return map; + } + + memcpy(map, data, size); + + i915_gem_object_unpin_map(obj); + + return obj; +} + +struct drm_i915_gem_object * i915_gem_object_create_lmem(struct drm_i915_private *i915, resource_size_t size, unsigned int flags) diff --git a/drivers/gpu/drm/i915/gem/i915_gem_lmem.h b/drivers/gpu/drm/i915/gem/i915_gem_lmem.h index 4ee81fc66302..1b88ea13435c 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_lmem.h +++ b/drivers/gpu/drm/i915/gem/i915_gem_lmem.h @@ -24,6 +24,10 @@ bool i915_gem_object_is_lmem(struct drm_i915_gem_object *obj); bool __i915_gem_object_is_lmem(struct drm_i915_gem_object *obj); struct drm_i915_gem_object * +i915_gem_object_create_lmem_from_data(struct drm_i915_private *i915, + const void *data, size_t size); + +struct drm_i915_gem_object * __i915_gem_object_create_lmem_with_ps(struct drm_i915_private *i915, resource_size_t size, resource_size_t page_size, diff --git a/drivers/gpu/drm/i915/gem/i915_gem_mman.c b/drivers/gpu/drm/i915/gem/i915_gem_mman.c index 5130e8ed9564..65fc6ff5f59d 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_mman.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_mman.c @@ -395,7 +395,7 @@ retry: /* Track the mmo associated with the fenced vma */ vma->mmo = mmo; - if (IS_ACTIVE(CONFIG_DRM_I915_USERFAULT_AUTOSUSPEND)) + if (CONFIG_DRM_I915_USERFAULT_AUTOSUSPEND) intel_wakeref_auto(&i915->ggtt.userfault_wakeref, msecs_to_jiffies_timeout(CONFIG_DRM_I915_USERFAULT_AUTOSUSPEND)); diff --git a/drivers/gpu/drm/i915/gem/i915_gem_object.c b/drivers/gpu/drm/i915/gem/i915_gem_object.c index 6fb9afb65034..1e426a42a36c 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_object.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_object.c @@ -25,6 +25,7 @@ #include <linux/sched/mm.h> #include "display/intel_frontbuffer.h" +#include "pxp/intel_pxp.h" #include "i915_drv.h" #include "i915_gem_clflush.h" #include "i915_gem_context.h" @@ -90,6 +91,22 @@ void i915_gem_object_init(struct drm_i915_gem_object *obj, } /** + * i915_gem_object_fini - Clean up a GEM object initialization + * @obj: The gem object to cleanup + * + * This function cleans up gem object fields that are set up by + * drm_gem_private_object_init() and i915_gem_object_init(). + * It's primarily intended as a helper for backends that need to + * clean up the gem object in separate steps. + */ +void __i915_gem_object_fini(struct drm_i915_gem_object *obj) +{ + mutex_destroy(&obj->mm.get_page.lock); + mutex_destroy(&obj->mm.get_dma_page.lock); + dma_resv_fini(&obj->base._resv); +} + +/** * Mark up the object's coherency levels for a given cache_level * @obj: #drm_i915_gem_object * @cache_level: cache level @@ -111,6 +128,32 @@ void i915_gem_object_set_cache_coherency(struct drm_i915_gem_object *obj, !(obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_WRITE); } +bool i915_gem_object_can_bypass_llc(struct drm_i915_gem_object *obj) +{ + struct drm_i915_private *i915 = to_i915(obj->base.dev); + + /* + * This is purely from a security perspective, so we simply don't care + * about non-userspace objects being able to bypass the LLC. + */ + if (!(obj->flags & I915_BO_ALLOC_USER)) + return false; + + /* + * EHL and JSL add the 'Bypass LLC' MOCS entry, which should make it + * possible for userspace to bypass the GTT caching bits set by the + * kernel, as per the given object cache_level. This is troublesome + * since the heavy flush we apply when first gathering the pages is + * skipped if the kernel thinks the object is coherent with the GPU. As + * a result it might be possible to bypass the cache and read the + * contents of the page directly, which could be stale data. If it's + * just a case of userspace shooting themselves in the foot then so be + * it, but since i915 takes the stance of always zeroing memory before + * handing it to userspace, we need to prevent this. + */ + return IS_JSL_EHL(i915); +} + static void i915_gem_close_object(struct drm_gem_object *gem, struct drm_file *file) { struct drm_i915_gem_object *obj = to_intel_bo(gem); @@ -174,7 +217,6 @@ void __i915_gem_free_object_rcu(struct rcu_head *head) container_of(head, typeof(*obj), rcu); struct drm_i915_private *i915 = to_i915(obj->base.dev); - dma_resv_fini(&obj->base._resv); i915_gem_object_free(obj); GEM_BUG_ON(!atomic_read(&i915->mm.free_count)); @@ -204,10 +246,17 @@ static void __i915_gem_object_free_mmaps(struct drm_i915_gem_object *obj) } } -void __i915_gem_free_object(struct drm_i915_gem_object *obj) +/** + * __i915_gem_object_pages_fini - Clean up pages use of a gem object + * @obj: The gem object to clean up + * + * This function cleans up usage of the object mm.pages member. It + * is intended for backends that need to clean up a gem object in + * separate steps and needs to be called when the object is idle before + * the object's backing memory is freed. + */ +void __i915_gem_object_pages_fini(struct drm_i915_gem_object *obj) { - trace_i915_gem_object_destroy(obj); - if (!list_empty(&obj->vma.list)) { struct i915_vma *vma; @@ -233,11 +282,17 @@ void __i915_gem_free_object(struct drm_i915_gem_object *obj) __i915_gem_object_free_mmaps(obj); - GEM_BUG_ON(!list_empty(&obj->lut_list)); - atomic_set(&obj->mm.pages_pin_count, 0); __i915_gem_object_put_pages(obj); GEM_BUG_ON(i915_gem_object_has_pages(obj)); +} + +void __i915_gem_free_object(struct drm_i915_gem_object *obj) +{ + trace_i915_gem_object_destroy(obj); + + GEM_BUG_ON(!list_empty(&obj->lut_list)); + bitmap_free(obj->bit_17); if (obj->base.import_attach) @@ -253,6 +308,8 @@ void __i915_gem_free_object(struct drm_i915_gem_object *obj) if (obj->shares_resv_from) i915_vm_resv_put(obj->shares_resv_from); + + __i915_gem_object_fini(obj); } static void __i915_gem_free_objects(struct drm_i915_private *i915, @@ -266,6 +323,7 @@ static void __i915_gem_free_objects(struct drm_i915_private *i915, obj->ops->delayed_free(obj); continue; } + __i915_gem_object_pages_fini(obj); __i915_gem_free_object(obj); /* But keep the pointer alive for RCU-protected lookups */ diff --git a/drivers/gpu/drm/i915/gem/i915_gem_object.h b/drivers/gpu/drm/i915/gem/i915_gem_object.h index 48112b9d76df..59201801cec5 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_object.h +++ b/drivers/gpu/drm/i915/gem/i915_gem_object.h @@ -58,6 +58,9 @@ void i915_gem_object_init(struct drm_i915_gem_object *obj, const struct drm_i915_gem_object_ops *ops, struct lock_class_key *key, unsigned alloc_flags); + +void __i915_gem_object_fini(struct drm_i915_gem_object *obj); + struct drm_i915_gem_object * i915_gem_object_create_shmem(struct drm_i915_private *i915, resource_size_t size); @@ -270,6 +273,12 @@ i915_gem_object_clear_tiling_quirk(struct drm_i915_gem_object *obj) } static inline bool +i915_gem_object_is_protected(const struct drm_i915_gem_object *obj) +{ + return obj->flags & I915_BO_PROTECTED; +} + +static inline bool i915_gem_object_type_has(const struct drm_i915_gem_object *obj, unsigned long flags) { @@ -503,25 +512,9 @@ i915_gem_object_finish_access(struct drm_i915_gem_object *obj) i915_gem_object_unpin_pages(obj); } -static inline struct intel_engine_cs * -i915_gem_object_last_write_engine(struct drm_i915_gem_object *obj) -{ - struct intel_engine_cs *engine = NULL; - struct dma_fence *fence; - - rcu_read_lock(); - fence = dma_resv_get_excl_unlocked(obj->base.resv); - rcu_read_unlock(); - - if (fence && dma_fence_is_i915(fence) && !dma_fence_is_signaled(fence)) - engine = to_request(fence)->engine; - dma_fence_put(fence); - - return engine; -} - void i915_gem_object_set_cache_coherency(struct drm_i915_gem_object *obj, unsigned int cache_level); +bool i915_gem_object_can_bypass_llc(struct drm_i915_gem_object *obj); void i915_gem_object_flush_if_display(struct drm_i915_gem_object *obj); void i915_gem_object_flush_if_display_locked(struct drm_i915_gem_object *obj); @@ -599,6 +592,8 @@ bool i915_gem_object_is_shmem(const struct drm_i915_gem_object *obj); void __i915_gem_free_object_rcu(struct rcu_head *head); +void __i915_gem_object_pages_fini(struct drm_i915_gem_object *obj); + void __i915_gem_free_object(struct drm_i915_gem_object *obj); bool i915_gem_object_evictable(struct drm_i915_gem_object *obj); diff --git a/drivers/gpu/drm/i915/gem/i915_gem_object_types.h b/drivers/gpu/drm/i915/gem/i915_gem_object_types.h index 2471f36aaff3..da85169006d4 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_object_types.h +++ b/drivers/gpu/drm/i915/gem/i915_gem_object_types.h @@ -288,17 +288,23 @@ struct drm_i915_gem_object { I915_SELFTEST_DECLARE(struct list_head st_link); unsigned long flags; -#define I915_BO_ALLOC_CONTIGUOUS BIT(0) -#define I915_BO_ALLOC_VOLATILE BIT(1) -#define I915_BO_ALLOC_CPU_CLEAR BIT(2) -#define I915_BO_ALLOC_USER BIT(3) +#define I915_BO_ALLOC_CONTIGUOUS BIT(0) +#define I915_BO_ALLOC_VOLATILE BIT(1) +#define I915_BO_ALLOC_CPU_CLEAR BIT(2) +#define I915_BO_ALLOC_USER BIT(3) +/* Object is allowed to lose its contents on suspend / resume, even if pinned */ +#define I915_BO_ALLOC_PM_VOLATILE BIT(4) +/* Object needs to be restored early using memcpy during resume */ +#define I915_BO_ALLOC_PM_EARLY BIT(5) #define I915_BO_ALLOC_FLAGS (I915_BO_ALLOC_CONTIGUOUS | \ I915_BO_ALLOC_VOLATILE | \ I915_BO_ALLOC_CPU_CLEAR | \ - I915_BO_ALLOC_USER) -#define I915_BO_READONLY BIT(4) -#define I915_TILING_QUIRK_BIT 5 /* unknown swizzling; do not release! */ - + I915_BO_ALLOC_USER | \ + I915_BO_ALLOC_PM_VOLATILE | \ + I915_BO_ALLOC_PM_EARLY) +#define I915_BO_READONLY BIT(6) +#define I915_TILING_QUIRK_BIT 7 /* unknown swizzling; do not release! */ +#define I915_BO_PROTECTED BIT(8) /** * @mem_flags - Mutable placement-related flags * @@ -421,6 +427,33 @@ struct drm_i915_gem_object { * can freely bypass the CPU cache when touching the pages with the GPU, * where the kernel is completely unaware. On such platform we need * apply the sledgehammer-on-acquire regardless of the @cache_coherent. + * + * Special care is taken on non-LLC platforms, to prevent potential + * information leak. The driver currently ensures: + * + * 1. All userspace objects, by default, have @cache_level set as + * I915_CACHE_NONE. The only exception is userptr objects, where we + * instead force I915_CACHE_LLC, but we also don't allow userspace to + * ever change the @cache_level for such objects. Another special case + * is dma-buf, which doesn't rely on @cache_dirty, but there we + * always do a forced flush when acquiring the pages, if there is a + * chance that the pages can be read directly from main memory with + * the GPU. + * + * 2. All I915_CACHE_NONE objects have @cache_dirty initially true. + * + * 3. All swapped-out objects(i.e shmem) have @cache_dirty set to + * true. + * + * 4. The @cache_dirty is never freely reset before the initial + * flush, even if userspace adjusts the @cache_level through the + * i915_gem_set_caching_ioctl. + * + * 5. All @cache_dirty objects(including swapped-in) are initially + * flushed with a synchronous call to drm_clflush_sg in + * __i915_gem_object_set_pages. The @cache_dirty can be freely reset + * at this point. All further asynchronous clfushes are never security + * critical, i.e userspace is free to race against itself. */ unsigned int cache_dirty:1; @@ -534,9 +567,17 @@ struct drm_i915_gem_object { struct { struct sg_table *cached_io_st; struct i915_gem_object_page_iter get_io_page; + struct drm_i915_gem_object *backup; bool created:1; } ttm; + /* + * Record which PXP key instance this object was created against (if + * any), so we can use it to determine if the encryption is valid by + * comparing against the current key instance. + */ + u32 pxp_key_instance; + /** Record of address bit 17 of each page at last unbind. */ unsigned long *bit_17; diff --git a/drivers/gpu/drm/i915/gem/i915_gem_pm.c b/drivers/gpu/drm/i915/gem/i915_gem_pm.c index 8b9d7d14c4bd..726b40e1fbb0 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_pm.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_pm.c @@ -5,6 +5,7 @@ */ #include "gem/i915_gem_pm.h" +#include "gem/i915_gem_ttm_pm.h" #include "gt/intel_gt.h" #include "gt/intel_gt_pm.h" #include "gt/intel_gt_requests.h" @@ -39,6 +40,88 @@ void i915_gem_suspend(struct drm_i915_private *i915) i915_gem_drain_freed_objects(i915); } +static int lmem_restore(struct drm_i915_private *i915, u32 flags) +{ + struct intel_memory_region *mr; + int ret = 0, id; + + for_each_memory_region(mr, i915, id) { + if (mr->type == INTEL_MEMORY_LOCAL) { + ret = i915_ttm_restore_region(mr, flags); + if (ret) + break; + } + } + + return ret; +} + +static int lmem_suspend(struct drm_i915_private *i915, u32 flags) +{ + struct intel_memory_region *mr; + int ret = 0, id; + + for_each_memory_region(mr, i915, id) { + if (mr->type == INTEL_MEMORY_LOCAL) { + ret = i915_ttm_backup_region(mr, flags); + if (ret) + break; + } + } + + return ret; +} + +static void lmem_recover(struct drm_i915_private *i915) +{ + struct intel_memory_region *mr; + int id; + + for_each_memory_region(mr, i915, id) + if (mr->type == INTEL_MEMORY_LOCAL) + i915_ttm_recover_region(mr); +} + +int i915_gem_backup_suspend(struct drm_i915_private *i915) +{ + int ret; + + /* Opportunistically try to evict unpinned objects */ + ret = lmem_suspend(i915, I915_TTM_BACKUP_ALLOW_GPU); + if (ret) + goto out_recover; + + i915_gem_suspend(i915); + + /* + * More objects may have become unpinned as requests were + * retired. Now try to evict again. The gt may be wedged here + * in which case we automatically fall back to memcpy. + * We allow also backing up pinned objects that have not been + * marked for early recover, and that may contain, for example, + * page-tables for the migrate context. + */ + ret = lmem_suspend(i915, I915_TTM_BACKUP_ALLOW_GPU | + I915_TTM_BACKUP_PINNED); + if (ret) + goto out_recover; + + /* + * Remaining objects are backed up using memcpy once we've stopped + * using the migrate context. + */ + ret = lmem_suspend(i915, I915_TTM_BACKUP_PINNED); + if (ret) + goto out_recover; + + return 0; + +out_recover: + lmem_recover(i915); + + return ret; +} + void i915_gem_suspend_late(struct drm_i915_private *i915) { struct drm_i915_gem_object *obj; @@ -128,12 +211,20 @@ int i915_gem_freeze_late(struct drm_i915_private *i915) void i915_gem_resume(struct drm_i915_private *i915) { + int ret; + GEM_TRACE("%s\n", dev_name(i915->drm.dev)); + ret = lmem_restore(i915, 0); + GEM_WARN_ON(ret); + /* * As we didn't flush the kernel context before suspend, we cannot * guarantee that the context image is complete. So let's just reset * it and start again. */ intel_gt_resume(&i915->gt); + + ret = lmem_restore(i915, I915_TTM_BACKUP_ALLOW_GPU); + GEM_WARN_ON(ret); } diff --git a/drivers/gpu/drm/i915/gem/i915_gem_pm.h b/drivers/gpu/drm/i915/gem/i915_gem_pm.h index c9a66630e92e..bedf1e95941a 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_pm.h +++ b/drivers/gpu/drm/i915/gem/i915_gem_pm.h @@ -18,6 +18,7 @@ void i915_gem_idle_work_handler(struct work_struct *work); void i915_gem_suspend(struct drm_i915_private *i915); void i915_gem_suspend_late(struct drm_i915_private *i915); +int i915_gem_backup_suspend(struct drm_i915_private *i915); int i915_gem_freeze(struct drm_i915_private *i915); int i915_gem_freeze_late(struct drm_i915_private *i915); diff --git a/drivers/gpu/drm/i915/gem/i915_gem_region.c b/drivers/gpu/drm/i915/gem/i915_gem_region.c index 1f557b2178ed..a016ccec36f3 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_region.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_region.c @@ -80,3 +80,73 @@ err_object_free: i915_gem_object_free(obj); return ERR_PTR(err); } + +/** + * i915_gem_process_region - Iterate over all objects of a region using ops + * to process and optionally skip objects + * @mr: The memory region + * @apply: ops and private data + * + * This function can be used to iterate over the regions object list, + * checking whether to skip objects, and, if not, lock the objects and + * process them using the supplied ops. Note that this function temporarily + * removes objects from the region list while iterating, so that if run + * concurrently with itself may not iterate over all objects. + * + * Return: 0 if successful, negative error code on failure. + */ +int i915_gem_process_region(struct intel_memory_region *mr, + struct i915_gem_apply_to_region *apply) +{ + const struct i915_gem_apply_to_region_ops *ops = apply->ops; + struct drm_i915_gem_object *obj; + struct list_head still_in_list; + int ret = 0; + + /* + * In the future, a non-NULL apply->ww could mean the caller is + * already in a locking transaction and provides its own context. + */ + GEM_WARN_ON(apply->ww); + + INIT_LIST_HEAD(&still_in_list); + mutex_lock(&mr->objects.lock); + for (;;) { + struct i915_gem_ww_ctx ww; + + obj = list_first_entry_or_null(&mr->objects.list, typeof(*obj), + mm.region_link); + if (!obj) + break; + + list_move_tail(&obj->mm.region_link, &still_in_list); + if (!kref_get_unless_zero(&obj->base.refcount)) + continue; + + /* + * Note: Someone else might be migrating the object at this + * point. The object's region is not stable until we lock + * the object. + */ + mutex_unlock(&mr->objects.lock); + apply->ww = &ww; + for_i915_gem_ww(&ww, ret, apply->interruptible) { + ret = i915_gem_object_lock(obj, apply->ww); + if (ret) + continue; + + if (obj->mm.region == mr) + ret = ops->process_obj(apply, obj); + /* Implicit object unlock */ + } + + i915_gem_object_put(obj); + mutex_lock(&mr->objects.lock); + if (ret) + break; + } + list_splice_tail(&still_in_list, &mr->objects.list); + mutex_unlock(&mr->objects.lock); + + return ret; +} diff --git a/drivers/gpu/drm/i915/gem/i915_gem_region.h b/drivers/gpu/drm/i915/gem/i915_gem_region.h index 1008e580a89a..fcaa12d657d4 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_region.h +++ b/drivers/gpu/drm/i915/gem/i915_gem_region.h @@ -12,6 +12,41 @@ struct intel_memory_region; struct drm_i915_gem_object; struct sg_table; +struct i915_gem_apply_to_region; + +/** + * struct i915_gem_apply_to_region_ops - ops to use when iterating over all + * region objects. + */ +struct i915_gem_apply_to_region_ops { + /** + * process_obj - Process the current object + * @apply: Embed this for private data. + * @obj: The current object. + * + * Note that if this function is part of a ww transaction, and + * if returns -EDEADLK for one of the objects, it may be + * rerun for that same object in the same pass. + */ + int (*process_obj)(struct i915_gem_apply_to_region *apply, + struct drm_i915_gem_object *obj); +}; + +/** + * struct i915_gem_apply_to_region - Argument to the struct + * i915_gem_apply_to_region_ops functions. + * @ops: The ops for the operation. + * @ww: Locking context used for the transaction. + * @interruptible: Whether to perform object locking interruptible. + * + * This structure is intended to be embedded in a private struct if needed + */ +struct i915_gem_apply_to_region { + const struct i915_gem_apply_to_region_ops *ops; + struct i915_gem_ww_ctx *ww; + u32 interruptible:1; +}; + void i915_gem_object_init_memory_region(struct drm_i915_gem_object *obj, struct intel_memory_region *mem); void i915_gem_object_release_memory_region(struct drm_i915_gem_object *obj); @@ -22,4 +57,6 @@ i915_gem_object_create_region(struct intel_memory_region *mem, resource_size_t page_size, unsigned int flags); +int i915_gem_process_region(struct intel_memory_region *mr, + struct i915_gem_apply_to_region *apply); #endif diff --git a/drivers/gpu/drm/i915/gem/i915_gem_shmem.c b/drivers/gpu/drm/i915/gem/i915_gem_shmem.c index 11f072193f3b..d77da59fae04 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_shmem.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_shmem.c @@ -182,22 +182,7 @@ rebuild_st: if (i915_gem_object_needs_bit17_swizzle(obj)) i915_gem_object_do_bit_17_swizzle(obj, st); - /* - * EHL and JSL add the 'Bypass LLC' MOCS entry, which should make it - * possible for userspace to bypass the GTT caching bits set by the - * kernel, as per the given object cache_level. This is troublesome - * since the heavy flush we apply when first gathering the pages is - * skipped if the kernel thinks the object is coherent with the GPU. As - * a result it might be possible to bypass the cache and read the - * contents of the page directly, which could be stale data. If it's - * just a case of userspace shooting themselves in the foot then so be - * it, but since i915 takes the stance of always zeroing memory before - * handing it to userspace, we need to prevent this. - * - * By setting cache_dirty here we make the clflush in set_pages - * unconditional on such platforms. - */ - if (IS_JSL_EHL(i915) && obj->flags & I915_BO_ALLOC_USER) + if (i915_gem_object_can_bypass_llc(obj)) obj->cache_dirty = true; __i915_gem_object_set_pages(obj, st, sg_page_sizes); @@ -301,6 +286,8 @@ __i915_gem_object_release_shmem(struct drm_i915_gem_object *obj, struct sg_table *pages, bool needs_clflush) { + struct drm_i915_private *i915 = to_i915(obj->base.dev); + GEM_BUG_ON(obj->mm.madv == __I915_MADV_PURGED); if (obj->mm.madv == I915_MADV_DONTNEED) @@ -312,6 +299,16 @@ __i915_gem_object_release_shmem(struct drm_i915_gem_object *obj, drm_clflush_sg(pages); __start_cpu_write(obj); + /* + * On non-LLC platforms, force the flush-on-acquire if this is ever + * swapped-in. Our async flush path is not trust worthy enough yet(and + * happens in the wrong order), and with some tricks it's conceivable + * for userspace to change the cache-level to I915_CACHE_NONE after the + * pages are swapped-in, and since execbuf binds the object before doing + * the async flush, we have a race window. + */ + if (!HAS_LLC(i915)) + obj->cache_dirty = true; } void i915_gem_object_put_pages_shmem(struct drm_i915_gem_object *obj, struct sg_table *pages) diff --git a/drivers/gpu/drm/i915/gem/i915_gem_ttm.c b/drivers/gpu/drm/i915/gem/i915_gem_ttm.c index 6ea13159bffc..74a1ffd0d7dd 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_ttm.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_ttm.c @@ -10,18 +10,16 @@ #include "intel_memory_region.h" #include "intel_region_ttm.h" +#include "gem/i915_gem_mman.h" #include "gem/i915_gem_object.h" #include "gem/i915_gem_region.h" #include "gem/i915_gem_ttm.h" -#include "gem/i915_gem_mman.h" +#include "gem/i915_gem_ttm_pm.h" -#include "gt/intel_migrate.h" -#include "gt/intel_engine_pm.h" -#define I915_PL_LMEM0 TTM_PL_PRIV -#define I915_PL_SYSTEM TTM_PL_SYSTEM -#define I915_PL_STOLEN TTM_PL_VRAM -#define I915_PL_GGTT TTM_PL_TT +#include "gt/intel_engine_pm.h" +#include "gt/intel_gt.h" +#include "gt/intel_migrate.h" #define I915_TTM_PRIO_PURGE 0 #define I915_TTM_PRIO_NO_PAGES 1 @@ -64,6 +62,20 @@ static struct ttm_placement i915_sys_placement = { .busy_placement = &sys_placement_flags, }; +/** + * i915_ttm_sys_placement - Return the struct ttm_placement to be + * used for an object in system memory. + * + * Rather than making the struct extern, use this + * function. + * + * Return: A pointer to a static variable for sys placement. + */ +struct ttm_placement *i915_ttm_sys_placement(void) +{ + return &i915_sys_placement; +} + static int i915_ttm_err_to_gem(int err) { /* Fastpath */ @@ -182,7 +194,7 @@ static struct ttm_tt *i915_ttm_tt_create(struct ttm_buffer_object *bo, if (obj->flags & I915_BO_ALLOC_CPU_CLEAR && man->use_tt) - page_flags |= TTM_PAGE_FLAG_ZERO_ALLOC; + page_flags |= TTM_TT_FLAG_ZERO_ALLOC; ret = ttm_tt_init(&i915_tt->ttm, bo, page_flags, i915_ttm_select_tt_caching(obj)); @@ -214,7 +226,6 @@ static void i915_ttm_tt_destroy(struct ttm_device *bdev, struct ttm_tt *ttm) { struct i915_ttm_tt *i915_tt = container_of(ttm, typeof(*i915_tt), ttm); - ttm_tt_destroy_common(bdev, ttm); ttm_tt_fini(ttm); kfree(i915_tt); } @@ -356,8 +367,10 @@ static void i915_ttm_delete_mem_notify(struct ttm_buffer_object *bo) { struct drm_i915_gem_object *obj = i915_ttm_to_gem(bo); - if (likely(obj)) + if (likely(obj)) { + __i915_gem_object_pages_fini(obj); i915_ttm_free_cached_io_st(obj); + } } static struct intel_memory_region * @@ -427,7 +440,9 @@ i915_ttm_resource_get_st(struct drm_i915_gem_object *obj, } static int i915_ttm_accel_move(struct ttm_buffer_object *bo, + bool clear, struct ttm_resource *dst_mem, + struct ttm_tt *dst_ttm, struct sg_table *dst_st) { struct drm_i915_private *i915 = container_of(bo->bdev, typeof(*i915), @@ -437,21 +452,18 @@ static int i915_ttm_accel_move(struct ttm_buffer_object *bo, struct drm_i915_gem_object *obj = i915_ttm_to_gem(bo); struct sg_table *src_st; struct i915_request *rq; - struct ttm_tt *ttm = bo->ttm; + struct ttm_tt *src_ttm = bo->ttm; enum i915_cache_level src_level, dst_level; int ret; - if (!i915->gt.migrate.context) + if (!i915->gt.migrate.context || intel_gt_is_wedged(&i915->gt)) return -EINVAL; - dst_level = i915_ttm_cache_level(i915, dst_mem, ttm); - if (!ttm || !ttm_tt_is_populated(ttm)) { + dst_level = i915_ttm_cache_level(i915, dst_mem, dst_ttm); + if (clear) { if (bo->type == ttm_bo_type_kernel) return -EINVAL; - if (ttm && !(ttm->page_flags & TTM_PAGE_FLAG_ZERO_ALLOC)) - return 0; - intel_engine_pm_get(i915->gt.migrate.context->engine); ret = intel_context_migrate_clear(i915->gt.migrate.context, NULL, dst_st->sgl, dst_level, @@ -464,10 +476,10 @@ static int i915_ttm_accel_move(struct ttm_buffer_object *bo, } intel_engine_pm_put(i915->gt.migrate.context->engine); } else { - src_st = src_man->use_tt ? i915_ttm_tt_get_st(ttm) : + src_st = src_man->use_tt ? i915_ttm_tt_get_st(src_ttm) : obj->ttm.cached_io_st; - src_level = i915_ttm_cache_level(i915, bo->resource, ttm); + src_level = i915_ttm_cache_level(i915, bo->resource, src_ttm); intel_engine_pm_get(i915->gt.migrate.context->engine); ret = intel_context_migrate_copy(i915->gt.migrate.context, NULL, src_st->sgl, src_level, @@ -485,6 +497,44 @@ static int i915_ttm_accel_move(struct ttm_buffer_object *bo, return ret; } +static void __i915_ttm_move(struct ttm_buffer_object *bo, bool clear, + struct ttm_resource *dst_mem, + struct ttm_tt *dst_ttm, + struct sg_table *dst_st, + bool allow_accel) +{ + int ret = -EINVAL; + + if (allow_accel) + ret = i915_ttm_accel_move(bo, clear, dst_mem, dst_ttm, dst_st); + if (ret) { + struct drm_i915_gem_object *obj = i915_ttm_to_gem(bo); + struct intel_memory_region *dst_reg, *src_reg; + union { + struct ttm_kmap_iter_tt tt; + struct ttm_kmap_iter_iomap io; + } _dst_iter, _src_iter; + struct ttm_kmap_iter *dst_iter, *src_iter; + + dst_reg = i915_ttm_region(bo->bdev, dst_mem->mem_type); + src_reg = i915_ttm_region(bo->bdev, bo->resource->mem_type); + GEM_BUG_ON(!dst_reg || !src_reg); + + dst_iter = !cpu_maps_iomem(dst_mem) ? + ttm_kmap_iter_tt_init(&_dst_iter.tt, dst_ttm) : + ttm_kmap_iter_iomap_init(&_dst_iter.io, &dst_reg->iomap, + dst_st, dst_reg->region.start); + + src_iter = !cpu_maps_iomem(bo->resource) ? + ttm_kmap_iter_tt_init(&_src_iter.tt, bo->ttm) : + ttm_kmap_iter_iomap_init(&_src_iter.io, &src_reg->iomap, + obj->ttm.cached_io_st, + src_reg->region.start); + + ttm_move_memcpy(clear, dst_mem->num_pages, dst_iter, src_iter); + } +} + static int i915_ttm_move(struct ttm_buffer_object *bo, bool evict, struct ttm_operation_ctx *ctx, struct ttm_resource *dst_mem, @@ -493,19 +543,11 @@ static int i915_ttm_move(struct ttm_buffer_object *bo, bool evict, struct drm_i915_gem_object *obj = i915_ttm_to_gem(bo); struct ttm_resource_manager *dst_man = ttm_manager_type(bo->bdev, dst_mem->mem_type); - struct intel_memory_region *dst_reg, *src_reg; - union { - struct ttm_kmap_iter_tt tt; - struct ttm_kmap_iter_iomap io; - } _dst_iter, _src_iter; - struct ttm_kmap_iter *dst_iter, *src_iter; + struct ttm_tt *ttm = bo->ttm; struct sg_table *dst_st; + bool clear; int ret; - dst_reg = i915_ttm_region(bo->bdev, dst_mem->mem_type); - src_reg = i915_ttm_region(bo->bdev, bo->resource->mem_type); - GEM_BUG_ON(!dst_reg || !src_reg); - /* Sync for now. We could do the actual copy async. */ ret = ttm_bo_wait_ctx(bo, ctx); if (ret) @@ -522,9 +564,8 @@ static int i915_ttm_move(struct ttm_buffer_object *bo, bool evict, } /* Populate ttm with pages if needed. Typically system memory. */ - if (bo->ttm && (dst_man->use_tt || - (bo->ttm->page_flags & TTM_PAGE_FLAG_SWAPPED))) { - ret = ttm_tt_populate(bo->bdev, bo->ttm, ctx); + if (ttm && (dst_man->use_tt || (ttm->page_flags & TTM_TT_FLAG_SWAPPED))) { + ret = ttm_tt_populate(bo->bdev, ttm, ctx); if (ret) return ret; } @@ -533,23 +574,10 @@ static int i915_ttm_move(struct ttm_buffer_object *bo, bool evict, if (IS_ERR(dst_st)) return PTR_ERR(dst_st); - ret = i915_ttm_accel_move(bo, dst_mem, dst_st); - if (ret) { - /* If we start mapping GGTT, we can no longer use man::use_tt here. */ - dst_iter = !cpu_maps_iomem(dst_mem) ? - ttm_kmap_iter_tt_init(&_dst_iter.tt, bo->ttm) : - ttm_kmap_iter_iomap_init(&_dst_iter.io, &dst_reg->iomap, - dst_st, dst_reg->region.start); + clear = !cpu_maps_iomem(bo->resource) && (!ttm || !ttm_tt_is_populated(ttm)); + if (!(clear && ttm && !(ttm->page_flags & TTM_TT_FLAG_ZERO_ALLOC))) + __i915_ttm_move(bo, clear, dst_mem, bo->ttm, dst_st, true); - src_iter = !cpu_maps_iomem(bo->resource) ? - ttm_kmap_iter_tt_init(&_src_iter.tt, bo->ttm) : - ttm_kmap_iter_iomap_init(&_src_iter.io, &src_reg->iomap, - obj->ttm.cached_io_st, - src_reg->region.start); - - ttm_move_memcpy(bo, dst_mem->num_pages, dst_iter, src_iter); - } - /* Below dst_mem becomes bo->resource. */ ttm_bo_move_sync_cleanup(bo, dst_mem); i915_ttm_adjust_domains_after_move(obj); i915_ttm_free_cached_io_st(obj); @@ -787,12 +815,9 @@ static void i915_ttm_adjust_lru(struct drm_i915_gem_object *obj) */ static void i915_ttm_delayed_free(struct drm_i915_gem_object *obj) { - if (obj->ttm.created) { - ttm_bo_put(i915_gem_to_ttm(obj)); - } else { - __i915_gem_free_object(obj); - call_rcu(&obj->rcu, __i915_gem_free_object_rcu); - } + GEM_BUG_ON(!obj->ttm.created); + + ttm_bo_put(i915_gem_to_ttm(obj)); } static vm_fault_t vm_fault_ttm(struct vm_fault *vmf) @@ -872,14 +897,19 @@ void i915_ttm_bo_destroy(struct ttm_buffer_object *bo) { struct drm_i915_gem_object *obj = i915_ttm_to_gem(bo); - /* This releases all gem object bindings to the backend. */ - __i915_gem_free_object(obj); - i915_gem_object_release_memory_region(obj); mutex_destroy(&obj->ttm.get_io_page.lock); - if (obj->ttm.created) + if (obj->ttm.created) { + i915_ttm_backup_free(obj); + + /* This releases all gem object bindings to the backend. */ + __i915_gem_free_object(obj); + call_rcu(&obj->rcu, __i915_gem_free_object_rcu); + } else { + __i915_gem_object_fini(obj); + } } /** @@ -908,7 +938,11 @@ int __i915_gem_ttm_object_init(struct intel_memory_region *mem, drm_gem_private_object_init(&i915->drm, &obj->base, size); i915_gem_object_init(obj, &i915_gem_ttm_obj_ops, &lock_class, flags); - i915_gem_object_init_memory_region(obj, mem); + + /* Don't put on a region list until we're either locked or fully initialized. */ + obj->mm.region = intel_memory_region_get(mem); + INIT_LIST_HEAD(&obj->mm.region_link); + i915_gem_object_make_unshrinkable(obj); INIT_RADIX_TREE(&obj->ttm.get_io_page.radix, GFP_KERNEL | __GFP_NOWARN); mutex_init(&obj->ttm.get_io_page.lock); @@ -935,6 +969,8 @@ int __i915_gem_ttm_object_init(struct intel_memory_region *mem, return i915_ttm_err_to_gem(ret); obj->ttm.created = true; + i915_gem_object_release_memory_region(obj); + i915_gem_object_init_memory_region(obj, mem); i915_ttm_adjust_domains_after_move(obj); i915_ttm_adjust_gem_after_move(obj); i915_gem_object_unlock(obj); @@ -963,3 +999,50 @@ i915_gem_ttm_system_setup(struct drm_i915_private *i915, intel_memory_region_set_name(mr, "system-ttm"); return mr; } + +/** + * i915_gem_obj_copy_ttm - Copy the contents of one ttm-based gem object to + * another + * @dst: The destination object + * @src: The source object + * @allow_accel: Allow using the blitter. Otherwise TTM memcpy is used. + * @intr: Whether to perform waits interruptible: + * + * Note: The caller is responsible for assuring that the underlying + * TTM objects are populated if needed and locked. + * + * Return: Zero on success. Negative error code on error. If @intr == true, + * then it may return -ERESTARTSYS or -EINTR. + */ +int i915_gem_obj_copy_ttm(struct drm_i915_gem_object *dst, + struct drm_i915_gem_object *src, + bool allow_accel, bool intr) +{ + struct ttm_buffer_object *dst_bo = i915_gem_to_ttm(dst); + struct ttm_buffer_object *src_bo = i915_gem_to_ttm(src); + struct ttm_operation_ctx ctx = { + .interruptible = intr, + }; + struct sg_table *dst_st; + int ret; + + assert_object_held(dst); + assert_object_held(src); + + /* + * Sync for now. This will change with async moves. + */ + ret = ttm_bo_wait_ctx(dst_bo, &ctx); + if (!ret) + ret = ttm_bo_wait_ctx(src_bo, &ctx); + if (ret) + return ret; + + dst_st = gpu_binds_iomem(dst_bo->resource) ? + dst->ttm.cached_io_st : i915_ttm_tt_get_st(dst_bo->ttm); + + __i915_ttm_move(src_bo, false, dst_bo->resource, dst_bo->ttm, + dst_st, allow_accel); + + return 0; +} diff --git a/drivers/gpu/drm/i915/gem/i915_gem_ttm.h b/drivers/gpu/drm/i915/gem/i915_gem_ttm.h index 40927f67b6d9..0b7291dd897c 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_ttm.h +++ b/drivers/gpu/drm/i915/gem/i915_gem_ttm.h @@ -46,4 +46,18 @@ int __i915_gem_ttm_object_init(struct intel_memory_region *mem, resource_size_t size, resource_size_t page_size, unsigned int flags); + +int i915_gem_obj_copy_ttm(struct drm_i915_gem_object *dst, + struct drm_i915_gem_object *src, + bool allow_accel, bool intr); + +/* Internal I915 TTM declarations and definitions below. */ + +#define I915_PL_LMEM0 TTM_PL_PRIV +#define I915_PL_SYSTEM TTM_PL_SYSTEM +#define I915_PL_STOLEN TTM_PL_VRAM +#define I915_PL_GGTT TTM_PL_TT + +struct ttm_placement *i915_ttm_sys_placement(void); + #endif diff --git a/drivers/gpu/drm/i915/gem/i915_gem_ttm_pm.c b/drivers/gpu/drm/i915/gem/i915_gem_ttm_pm.c new file mode 100644 index 000000000000..3b6d14b5c604 --- /dev/null +++ b/drivers/gpu/drm/i915/gem/i915_gem_ttm_pm.c @@ -0,0 +1,206 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2021 Intel Corporation + */ + +#include <drm/ttm/ttm_placement.h> +#include <drm/ttm/ttm_tt.h> + +#include "i915_drv.h" +#include "intel_memory_region.h" +#include "intel_region_ttm.h" + +#include "gem/i915_gem_region.h" +#include "gem/i915_gem_ttm.h" +#include "gem/i915_gem_ttm_pm.h" + +/** + * i915_ttm_backup_free - Free any backup attached to this object + * @obj: The object whose backup is to be freed. + */ +void i915_ttm_backup_free(struct drm_i915_gem_object *obj) +{ + if (obj->ttm.backup) { + i915_gem_object_put(obj->ttm.backup); + obj->ttm.backup = NULL; + } +} + +/** + * struct i915_gem_ttm_pm_apply - Apply-to-region subclass for restore + * @base: The i915_gem_apply_to_region we derive from. + * @allow_gpu: Whether using the gpu blitter is allowed. + * @backup_pinned: On backup, backup also pinned objects. + */ +struct i915_gem_ttm_pm_apply { + struct i915_gem_apply_to_region base; + bool allow_gpu : 1; + bool backup_pinned : 1; +}; + +static int i915_ttm_backup(struct i915_gem_apply_to_region *apply, + struct drm_i915_gem_object *obj) +{ + struct i915_gem_ttm_pm_apply *pm_apply = + container_of(apply, typeof(*pm_apply), base); + struct ttm_buffer_object *bo = i915_gem_to_ttm(obj); + struct ttm_buffer_object *backup_bo; + struct drm_i915_private *i915 = + container_of(bo->bdev, typeof(*i915), bdev); + struct drm_i915_gem_object *backup; + struct ttm_operation_ctx ctx = {}; + int err = 0; + + if (bo->resource->mem_type == I915_PL_SYSTEM || obj->ttm.backup) + return 0; + + if (pm_apply->allow_gpu && i915_gem_object_evictable(obj)) + return ttm_bo_validate(bo, i915_ttm_sys_placement(), &ctx); + + if (!pm_apply->backup_pinned || + (pm_apply->allow_gpu && (obj->flags & I915_BO_ALLOC_PM_EARLY))) + return 0; + + if (obj->flags & I915_BO_ALLOC_PM_VOLATILE) + return 0; + + backup = i915_gem_object_create_shmem(i915, obj->base.size); + if (IS_ERR(backup)) + return PTR_ERR(backup); + + err = i915_gem_object_lock(backup, apply->ww); + if (err) + goto out_no_lock; + + backup_bo = i915_gem_to_ttm(backup); + err = ttm_tt_populate(backup_bo->bdev, backup_bo->ttm, &ctx); + if (err) + goto out_no_populate; + + err = i915_gem_obj_copy_ttm(backup, obj, pm_apply->allow_gpu, false); + GEM_WARN_ON(err); + + obj->ttm.backup = backup; + return 0; + +out_no_populate: + i915_gem_ww_unlock_single(backup); +out_no_lock: + i915_gem_object_put(backup); + + return err; +} + +static int i915_ttm_recover(struct i915_gem_apply_to_region *apply, + struct drm_i915_gem_object *obj) +{ + i915_ttm_backup_free(obj); + return 0; +} + +/** + * i915_ttm_recover_region - Free the backup of all objects of a region + * @mr: The memory region + * + * Checks all objects of a region if there is backup attached and if so + * frees that backup. Typically this is called to recover after a partially + * performed backup. + */ +void i915_ttm_recover_region(struct intel_memory_region *mr) +{ + static const struct i915_gem_apply_to_region_ops recover_ops = { + .process_obj = i915_ttm_recover, + }; + struct i915_gem_apply_to_region apply = {.ops = &recover_ops}; + int ret; + + ret = i915_gem_process_region(mr, &apply); + GEM_WARN_ON(ret); +} + +/** + * i915_ttm_backup_region - Back up all objects of a region to smem. + * @mr: The memory region + * @allow_gpu: Whether to allow the gpu blitter for this backup. + * @backup_pinned: Backup also pinned objects. + * + * Loops over all objects of a region and either evicts them if they are + * evictable or backs them up using a backup object if they are pinned. + * + * Return: Zero on success. Negative error code on error. + */ +int i915_ttm_backup_region(struct intel_memory_region *mr, u32 flags) +{ + static const struct i915_gem_apply_to_region_ops backup_ops = { + .process_obj = i915_ttm_backup, + }; + struct i915_gem_ttm_pm_apply pm_apply = { + .base = {.ops = &backup_ops}, + .allow_gpu = flags & I915_TTM_BACKUP_ALLOW_GPU, + .backup_pinned = flags & I915_TTM_BACKUP_PINNED, + }; + + return i915_gem_process_region(mr, &pm_apply.base); +} + +static int i915_ttm_restore(struct i915_gem_apply_to_region *apply, + struct drm_i915_gem_object *obj) +{ + struct i915_gem_ttm_pm_apply *pm_apply = + container_of(apply, typeof(*pm_apply), base); + struct drm_i915_gem_object *backup = obj->ttm.backup; + struct ttm_buffer_object *backup_bo = i915_gem_to_ttm(backup); + struct ttm_operation_ctx ctx = {}; + int err; + + if (!backup) + return 0; + + if (!pm_apply->allow_gpu && !(obj->flags & I915_BO_ALLOC_PM_EARLY)) + return 0; + + err = i915_gem_object_lock(backup, apply->ww); + if (err) + return err; + + /* Content may have been swapped. */ + err = ttm_tt_populate(backup_bo->bdev, backup_bo->ttm, &ctx); + if (!err) { + err = i915_gem_obj_copy_ttm(obj, backup, pm_apply->allow_gpu, + false); + GEM_WARN_ON(err); + + obj->ttm.backup = NULL; + err = 0; + } + + i915_gem_ww_unlock_single(backup); + + if (!err) + i915_gem_object_put(backup); + + return err; +} + +/** + * i915_ttm_restore_region - Restore backed-up objects of a region from smem. + * @mr: The memory region + * @allow_gpu: Whether to allow the gpu blitter to recover. + * + * Loops over all objects of a region and if they are backed-up, restores + * them from smem. + * + * Return: Zero on success. Negative error code on error. + */ +int i915_ttm_restore_region(struct intel_memory_region *mr, u32 flags) +{ + static const struct i915_gem_apply_to_region_ops restore_ops = { + .process_obj = i915_ttm_restore, + }; + struct i915_gem_ttm_pm_apply pm_apply = { + .base = {.ops = &restore_ops}, + .allow_gpu = flags & I915_TTM_BACKUP_ALLOW_GPU, + }; + + return i915_gem_process_region(mr, &pm_apply.base); +} diff --git a/drivers/gpu/drm/i915/gem/i915_gem_ttm_pm.h b/drivers/gpu/drm/i915/gem/i915_gem_ttm_pm.h new file mode 100644 index 000000000000..25ed67a31571 --- /dev/null +++ b/drivers/gpu/drm/i915/gem/i915_gem_ttm_pm.h @@ -0,0 +1,26 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2021 Intel Corporation + */ + +#ifndef _I915_GEM_TTM_PM_H_ +#define _I915_GEM_TTM_PM_H_ + +#include <linux/types.h> + +struct intel_memory_region; +struct drm_i915_gem_object; + +#define I915_TTM_BACKUP_ALLOW_GPU BIT(0) +#define I915_TTM_BACKUP_PINNED BIT(1) + +int i915_ttm_backup_region(struct intel_memory_region *mr, u32 flags); + +void i915_ttm_recover_region(struct intel_memory_region *mr); + +int i915_ttm_restore_region(struct intel_memory_region *mr, u32 flags); + +/* Internal I915 TTM functions below. */ +void i915_ttm_backup_free(struct drm_i915_gem_object *obj); + +#endif diff --git a/drivers/gpu/drm/i915/gem/i915_gem_userptr.c b/drivers/gpu/drm/i915/gem/i915_gem_userptr.c index 8ea0fa665e53..3173c9f9a040 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_userptr.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_userptr.c @@ -165,8 +165,11 @@ alloc_table: goto err; } - sg_page_sizes = i915_sg_dma_sizes(st->sgl); + WARN_ON_ONCE(!(obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_WRITE)); + if (i915_gem_object_can_bypass_llc(obj)) + obj->cache_dirty = true; + sg_page_sizes = i915_sg_dma_sizes(st->sgl); __i915_gem_object_set_pages(obj, st, sg_page_sizes); return 0; @@ -546,7 +549,8 @@ i915_gem_userptr_ioctl(struct drm_device *dev, return -ENOMEM; drm_gem_private_object_init(dev, &obj->base, args->user_size); - i915_gem_object_init(obj, &i915_gem_userptr_ops, &lock_class, 0); + i915_gem_object_init(obj, &i915_gem_userptr_ops, &lock_class, + I915_BO_ALLOC_USER); obj->mem_flags = I915_BO_FLAG_STRUCT_PAGE; obj->read_domains = I915_GEM_DOMAIN_CPU; obj->write_domain = I915_GEM_DOMAIN_CPU; diff --git a/drivers/gpu/drm/i915/gem/i915_gemfs.c b/drivers/gpu/drm/i915/gem/i915_gemfs.c index 5e6e8c91ab38..dbdbdc344d87 100644 --- a/drivers/gpu/drm/i915/gem/i915_gemfs.c +++ b/drivers/gpu/drm/i915/gem/i915_gemfs.c @@ -6,7 +6,6 @@ #include <linux/fs.h> #include <linux/mount.h> -#include <linux/pagemap.h> #include "i915_drv.h" #include "i915_gemfs.h" @@ -15,6 +14,7 @@ int i915_gemfs_init(struct drm_i915_private *i915) { struct file_system_type *type; struct vfsmount *gemfs; + char *opts; type = get_fs_type("tmpfs"); if (!type) @@ -26,10 +26,26 @@ int i915_gemfs_init(struct drm_i915_private *i915) * * One example, although it is probably better with a per-file * control, is selecting huge page allocations ("huge=within_size"). - * Currently unused due to bandwidth issues (slow reads) on Broadwell+. + * However, we only do so to offset the overhead of iommu lookups + * due to bandwidth issues (slow reads) on Broadwell+. */ - gemfs = kern_mount(type); + opts = NULL; + if (intel_vtd_active()) { + if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) { + static char huge_opt[] = "huge=within_size"; /* r/w */ + + opts = huge_opt; + drm_info(&i915->drm, + "Transparent Hugepage mode '%s'\n", + opts); + } else { + drm_notice(&i915->drm, + "Transparent Hugepage support is recommended for optimal performance when IOMMU is enabled!\n"); + } + } + + gemfs = vfs_kern_mount(type, SB_KERNMOUNT, type->name, opts); if (IS_ERR(gemfs)) return PTR_ERR(gemfs); diff --git a/drivers/gpu/drm/i915/gem/selftests/huge_pages.c b/drivers/gpu/drm/i915/gem/selftests/huge_pages.c index a094f3ce1a90..b2003133deaf 100644 --- a/drivers/gpu/drm/i915/gem/selftests/huge_pages.c +++ b/drivers/gpu/drm/i915/gem/selftests/huge_pages.c @@ -136,6 +136,8 @@ static void put_huge_pages(struct drm_i915_gem_object *obj, huge_pages_free_pages(pages); obj->mm.dirty = false; + + __start_cpu_write(obj); } static const struct drm_i915_gem_object_ops huge_page_ops = { @@ -152,6 +154,7 @@ huge_pages_object(struct drm_i915_private *i915, { static struct lock_class_key lock_class; struct drm_i915_gem_object *obj; + unsigned int cache_level; GEM_BUG_ON(!size); GEM_BUG_ON(!IS_ALIGNED(size, BIT(__ffs(page_mask)))); @@ -173,7 +176,9 @@ huge_pages_object(struct drm_i915_private *i915, obj->write_domain = I915_GEM_DOMAIN_CPU; obj->read_domains = I915_GEM_DOMAIN_CPU; - obj->cache_level = I915_CACHE_NONE; + + cache_level = HAS_LLC(i915) ? I915_CACHE_LLC : I915_CACHE_NONE; + i915_gem_object_set_cache_coherency(obj, cache_level); obj->mm.page_mask = page_mask; @@ -1456,7 +1461,7 @@ static int igt_tmpfs_fallback(void *arg) struct i915_gem_context *ctx = arg; struct drm_i915_private *i915 = ctx->i915; struct vfsmount *gemfs = i915->mm.gemfs; - struct i915_address_space *vm = i915_gem_context_get_vm_rcu(ctx); + struct i915_address_space *vm = i915_gem_context_get_eb_vm(ctx); struct drm_i915_gem_object *obj; struct i915_vma *vma; u32 *vaddr; @@ -1512,13 +1517,14 @@ static int igt_shrink_thp(void *arg) { struct i915_gem_context *ctx = arg; struct drm_i915_private *i915 = ctx->i915; - struct i915_address_space *vm = i915_gem_context_get_vm_rcu(ctx); + struct i915_address_space *vm = i915_gem_context_get_eb_vm(ctx); struct drm_i915_gem_object *obj; struct i915_gem_engines_iter it; struct intel_context *ce; struct i915_vma *vma; unsigned int flags = PIN_USER; unsigned int n; + bool should_swap; int err = 0; /* @@ -1567,23 +1573,39 @@ static int igt_shrink_thp(void *arg) break; } i915_gem_context_unlock_engines(ctx); + /* + * Nuke everything *before* we unpin the pages so we can be reasonably + * sure that when later checking get_nr_swap_pages() that some random + * leftover object doesn't steal the remaining swap space. + */ + i915_gem_shrink(NULL, i915, -1UL, NULL, + I915_SHRINK_BOUND | + I915_SHRINK_UNBOUND | + I915_SHRINK_ACTIVE); i915_vma_unpin(vma); if (err) goto out_put; /* - * Now that the pages are *unpinned* shrink-all should invoke - * shmem to truncate our pages. + * Now that the pages are *unpinned* shrinking should invoke + * shmem to truncate our pages, if we have available swap. */ - i915_gem_shrink_all(i915); - if (i915_gem_object_has_pages(obj)) { - pr_err("shrink-all didn't truncate the pages\n"); + should_swap = get_nr_swap_pages() > 0; + i915_gem_shrink(NULL, i915, -1UL, NULL, + I915_SHRINK_BOUND | + I915_SHRINK_UNBOUND | + I915_SHRINK_ACTIVE | + I915_SHRINK_WRITEBACK); + if (should_swap == i915_gem_object_has_pages(obj)) { + pr_err("unexpected pages mismatch, should_swap=%s\n", + yesno(should_swap)); err = -EINVAL; goto out_put; } - if (obj->mm.page_sizes.sg || obj->mm.page_sizes.phys) { - pr_err("residual page-size bits left\n"); + if (should_swap == (obj->mm.page_sizes.sg || obj->mm.page_sizes.phys)) { + pr_err("unexpected residual page-size bits, should_swap=%s\n", + yesno(should_swap)); err = -EINVAL; goto out_put; } @@ -1629,7 +1651,7 @@ int i915_gem_huge_page_mock_selftests(void) mkwrite_device_info(dev_priv)->ppgtt_type = INTEL_PPGTT_FULL; mkwrite_device_info(dev_priv)->ppgtt_size = 48; - ppgtt = i915_ppgtt_create(&dev_priv->gt); + ppgtt = i915_ppgtt_create(&dev_priv->gt, 0); if (IS_ERR(ppgtt)) { err = PTR_ERR(ppgtt); goto out_unlock; @@ -1688,11 +1710,9 @@ int i915_gem_huge_page_live_selftests(struct drm_i915_private *i915) goto out_file; } - mutex_lock(&ctx->mutex); - vm = i915_gem_context_vm(ctx); + vm = ctx->vm; if (vm) WRITE_ONCE(vm->scrub_64K, true); - mutex_unlock(&ctx->mutex); err = i915_subtests(tests, ctx); diff --git a/drivers/gpu/drm/i915/gem/selftests/i915_gem_client_blt.c b/drivers/gpu/drm/i915/gem/selftests/i915_gem_client_blt.c index ecbcbb86ae1e..8402ed925a69 100644 --- a/drivers/gpu/drm/i915/gem/selftests/i915_gem_client_blt.c +++ b/drivers/gpu/drm/i915/gem/selftests/i915_gem_client_blt.c @@ -17,13 +17,20 @@ #include "huge_gem_object.h" #include "mock_context.h" +enum client_tiling { + CLIENT_TILING_LINEAR, + CLIENT_TILING_X, + CLIENT_TILING_Y, + CLIENT_NUM_TILING_TYPES +}; + #define WIDTH 512 #define HEIGHT 32 struct blit_buffer { struct i915_vma *vma; u32 start_val; - u32 tiling; + enum client_tiling tiling; }; struct tiled_blits { @@ -53,9 +60,9 @@ static int prepare_blit(const struct tiled_blits *t, *cs++ = MI_LOAD_REGISTER_IMM(1); *cs++ = i915_mmio_reg_offset(BCS_SWCTRL); cmd = (BCS_SRC_Y | BCS_DST_Y) << 16; - if (src->tiling == I915_TILING_Y) + if (src->tiling == CLIENT_TILING_Y) cmd |= BCS_SRC_Y; - if (dst->tiling == I915_TILING_Y) + if (dst->tiling == CLIENT_TILING_Y) cmd |= BCS_DST_Y; *cs++ = cmd; @@ -172,7 +179,7 @@ static int tiled_blits_create_buffers(struct tiled_blits *t, t->buffers[i].vma = vma; t->buffers[i].tiling = - i915_prandom_u32_max_state(I915_TILING_Y + 1, prng); + i915_prandom_u32_max_state(CLIENT_TILING_Y + 1, prng); } return 0; @@ -197,17 +204,17 @@ static u64 swizzle_bit(unsigned int bit, u64 offset) static u64 tiled_offset(const struct intel_gt *gt, u64 v, unsigned int stride, - unsigned int tiling) + enum client_tiling tiling) { unsigned int swizzle; u64 x, y; - if (tiling == I915_TILING_NONE) + if (tiling == CLIENT_TILING_LINEAR) return v; y = div64_u64_rem(v, stride, &x); - if (tiling == I915_TILING_X) { + if (tiling == CLIENT_TILING_X) { v = div64_u64_rem(y, 8, &y) * stride * 8; v += y * 512; v += div64_u64_rem(x, 512, &x) << 12; @@ -244,12 +251,12 @@ static u64 tiled_offset(const struct intel_gt *gt, return v; } -static const char *repr_tiling(int tiling) +static const char *repr_tiling(enum client_tiling tiling) { switch (tiling) { - case I915_TILING_NONE: return "linear"; - case I915_TILING_X: return "X"; - case I915_TILING_Y: return "Y"; + case CLIENT_TILING_LINEAR: return "linear"; + case CLIENT_TILING_X: return "X"; + case CLIENT_TILING_Y: return "Y"; default: return "unknown"; } } diff --git a/drivers/gpu/drm/i915/gem/selftests/i915_gem_context.c b/drivers/gpu/drm/i915/gem/selftests/i915_gem_context.c index 8eb5050f8cb3..b32f7fed2d9c 100644 --- a/drivers/gpu/drm/i915/gem/selftests/i915_gem_context.c +++ b/drivers/gpu/drm/i915/gem/selftests/i915_gem_context.c @@ -27,12 +27,6 @@ #define DW_PER_PAGE (PAGE_SIZE / sizeof(u32)) -static inline struct i915_address_space *ctx_vm(struct i915_gem_context *ctx) -{ - /* single threaded, private ctx */ - return rcu_dereference_protected(ctx->vm, true); -} - static int live_nop_switch(void *arg) { const unsigned int nctx = 1024; @@ -94,7 +88,7 @@ static int live_nop_switch(void *arg) rq = i915_request_get(this); i915_request_add(this); } - if (i915_request_wait(rq, 0, HZ / 5) < 0) { + if (i915_request_wait(rq, 0, HZ) < 0) { pr_err("Failed to populated %d contexts\n", nctx); intel_gt_set_wedged(&i915->gt); i915_request_put(rq); @@ -704,7 +698,7 @@ static int igt_ctx_exec(void *arg) pr_err("Failed to fill dword %lu [%lu/%lu] with gpu (%s) [full-ppgtt? %s], err=%d\n", ndwords, dw, max_dwords(obj), engine->name, - yesno(!!rcu_access_pointer(ctx->vm)), + yesno(i915_gem_context_has_full_ppgtt(ctx)), err); intel_context_put(ce); kernel_context_close(ctx); @@ -813,7 +807,7 @@ static int igt_shared_ctx_exec(void *arg) struct i915_gem_context *ctx; struct intel_context *ce; - ctx = kernel_context(i915, ctx_vm(parent)); + ctx = kernel_context(i915, parent->vm); if (IS_ERR(ctx)) { err = PTR_ERR(ctx); goto out_test; @@ -823,7 +817,7 @@ static int igt_shared_ctx_exec(void *arg) GEM_BUG_ON(IS_ERR(ce)); if (!obj) { - obj = create_test_object(ctx_vm(parent), + obj = create_test_object(parent->vm, file, &objects); if (IS_ERR(obj)) { err = PTR_ERR(obj); @@ -838,7 +832,7 @@ static int igt_shared_ctx_exec(void *arg) pr_err("Failed to fill dword %lu [%lu/%lu] with gpu (%s) [full-ppgtt? %s], err=%d\n", ndwords, dw, max_dwords(obj), engine->name, - yesno(!!rcu_access_pointer(ctx->vm)), + yesno(i915_gem_context_has_full_ppgtt(ctx)), err); intel_context_put(ce); kernel_context_close(ctx); @@ -1380,7 +1374,7 @@ static int igt_ctx_readonly(void *arg) goto out_file; } - vm = ctx_vm(ctx) ?: &i915->ggtt.alias->vm; + vm = ctx->vm ?: &i915->ggtt.alias->vm; if (!vm || !vm->has_read_only) { err = 0; goto out_file; @@ -1417,7 +1411,7 @@ static int igt_ctx_readonly(void *arg) pr_err("Failed to fill dword %lu [%lu/%lu] with gpu (%s) [full-ppgtt? %s], err=%d\n", ndwords, dw, max_dwords(obj), ce->engine->name, - yesno(!!ctx_vm(ctx)), + yesno(i915_gem_context_has_full_ppgtt(ctx)), err); i915_gem_context_unlock_engines(ctx); goto out_file; @@ -1499,7 +1493,7 @@ static int write_to_scratch(struct i915_gem_context *ctx, GEM_BUG_ON(offset < I915_GTT_PAGE_SIZE); - err = check_scratch(ctx_vm(ctx), offset); + err = check_scratch(ctx->vm, offset); if (err) return err; @@ -1528,7 +1522,7 @@ static int write_to_scratch(struct i915_gem_context *ctx, intel_gt_chipset_flush(engine->gt); - vm = i915_gem_context_get_vm_rcu(ctx); + vm = i915_gem_context_get_eb_vm(ctx); vma = i915_vma_instance(obj, vm, NULL); if (IS_ERR(vma)) { err = PTR_ERR(vma); @@ -1596,7 +1590,7 @@ static int read_from_scratch(struct i915_gem_context *ctx, GEM_BUG_ON(offset < I915_GTT_PAGE_SIZE); - err = check_scratch(ctx_vm(ctx), offset); + err = check_scratch(ctx->vm, offset); if (err) return err; @@ -1607,7 +1601,7 @@ static int read_from_scratch(struct i915_gem_context *ctx, if (GRAPHICS_VER(i915) >= 8) { const u32 GPR0 = engine->mmio_base + 0x600; - vm = i915_gem_context_get_vm_rcu(ctx); + vm = i915_gem_context_get_eb_vm(ctx); vma = i915_vma_instance(obj, vm, NULL); if (IS_ERR(vma)) { err = PTR_ERR(vma); @@ -1739,7 +1733,7 @@ static int check_scratch_page(struct i915_gem_context *ctx, u32 *out) u32 *vaddr; int err = 0; - vm = ctx_vm(ctx); + vm = ctx->vm; if (!vm) return -ENODEV; @@ -1801,7 +1795,7 @@ static int igt_vm_isolation(void *arg) } /* We can only test vm isolation, if the vm are distinct */ - if (ctx_vm(ctx_a) == ctx_vm(ctx_b)) + if (ctx_a->vm == ctx_b->vm) goto out_file; /* Read the initial state of the scratch page */ @@ -1813,8 +1807,8 @@ static int igt_vm_isolation(void *arg) if (err) goto out_file; - vm_total = ctx_vm(ctx_a)->total; - GEM_BUG_ON(ctx_vm(ctx_b)->total != vm_total); + vm_total = ctx_a->vm->total; + GEM_BUG_ON(ctx_b->vm->total != vm_total); count = 0; num_engines = 0; diff --git a/drivers/gpu/drm/i915/gem/selftests/i915_gem_execbuffer.c b/drivers/gpu/drm/i915/gem/selftests/i915_gem_execbuffer.c deleted file mode 100644 index 16162fc2782d..000000000000 --- a/drivers/gpu/drm/i915/gem/selftests/i915_gem_execbuffer.c +++ /dev/null @@ -1,190 +0,0 @@ -// SPDX-License-Identifier: MIT -/* - * Copyright © 2020 Intel Corporation - */ - -#include "i915_selftest.h" - -#include "gt/intel_engine_pm.h" -#include "selftests/igt_flush_test.h" - -static u64 read_reloc(const u32 *map, int x, const u64 mask) -{ - u64 reloc; - - memcpy(&reloc, &map[x], sizeof(reloc)); - return reloc & mask; -} - -static int __igt_gpu_reloc(struct i915_execbuffer *eb, - struct drm_i915_gem_object *obj) -{ - const unsigned int offsets[] = { 8, 3, 0 }; - const u64 mask = - GENMASK_ULL(eb->reloc_cache.use_64bit_reloc ? 63 : 31, 0); - const u32 *map = page_mask_bits(obj->mm.mapping); - struct i915_request *rq; - struct i915_vma *vma; - int err; - int i; - - vma = i915_vma_instance(obj, eb->context->vm, NULL); - if (IS_ERR(vma)) - return PTR_ERR(vma); - - err = i915_gem_object_lock(obj, &eb->ww); - if (err) - return err; - - err = i915_vma_pin_ww(vma, &eb->ww, 0, 0, PIN_USER | PIN_HIGH); - if (err) - return err; - - /* 8-Byte aligned */ - err = __reloc_entry_gpu(eb, vma, offsets[0] * sizeof(u32), 0); - if (err <= 0) - goto reloc_err; - - /* !8-Byte aligned */ - err = __reloc_entry_gpu(eb, vma, offsets[1] * sizeof(u32), 1); - if (err <= 0) - goto reloc_err; - - /* Skip to the end of the cmd page */ - i = PAGE_SIZE / sizeof(u32) - 1; - i -= eb->reloc_cache.rq_size; - memset32(eb->reloc_cache.rq_cmd + eb->reloc_cache.rq_size, - MI_NOOP, i); - eb->reloc_cache.rq_size += i; - - /* Force next batch */ - err = __reloc_entry_gpu(eb, vma, offsets[2] * sizeof(u32), 2); - if (err <= 0) - goto reloc_err; - - GEM_BUG_ON(!eb->reloc_cache.rq); - rq = i915_request_get(eb->reloc_cache.rq); - reloc_gpu_flush(eb, &eb->reloc_cache); - GEM_BUG_ON(eb->reloc_cache.rq); - - err = i915_gem_object_wait(obj, I915_WAIT_INTERRUPTIBLE, HZ / 2); - if (err) { - intel_gt_set_wedged(eb->engine->gt); - goto put_rq; - } - - if (!i915_request_completed(rq)) { - pr_err("%s: did not wait for relocations!\n", eb->engine->name); - err = -EINVAL; - goto put_rq; - } - - for (i = 0; i < ARRAY_SIZE(offsets); i++) { - u64 reloc = read_reloc(map, offsets[i], mask); - - if (reloc != i) { - pr_err("%s[%d]: map[%d] %llx != %x\n", - eb->engine->name, i, offsets[i], reloc, i); - err = -EINVAL; - } - } - if (err) - igt_hexdump(map, 4096); - -put_rq: - i915_request_put(rq); -unpin_vma: - i915_vma_unpin(vma); - return err; - -reloc_err: - if (!err) - err = -EIO; - goto unpin_vma; -} - -static int igt_gpu_reloc(void *arg) -{ - struct i915_execbuffer eb; - struct drm_i915_gem_object *scratch; - int err = 0; - u32 *map; - - eb.i915 = arg; - - scratch = i915_gem_object_create_internal(eb.i915, 4096); - if (IS_ERR(scratch)) - return PTR_ERR(scratch); - - map = i915_gem_object_pin_map_unlocked(scratch, I915_MAP_WC); - if (IS_ERR(map)) { - err = PTR_ERR(map); - goto err_scratch; - } - - intel_gt_pm_get(&eb.i915->gt); - - for_each_uabi_engine(eb.engine, eb.i915) { - if (intel_engine_requires_cmd_parser(eb.engine) || - intel_engine_using_cmd_parser(eb.engine)) - continue; - - reloc_cache_init(&eb.reloc_cache, eb.i915); - memset(map, POISON_INUSE, 4096); - - intel_engine_pm_get(eb.engine); - eb.context = intel_context_create(eb.engine); - if (IS_ERR(eb.context)) { - err = PTR_ERR(eb.context); - goto err_pm; - } - eb.reloc_pool = NULL; - eb.reloc_context = NULL; - - i915_gem_ww_ctx_init(&eb.ww, false); -retry: - err = intel_context_pin_ww(eb.context, &eb.ww); - if (!err) { - err = __igt_gpu_reloc(&eb, scratch); - - intel_context_unpin(eb.context); - } - if (err == -EDEADLK) { - err = i915_gem_ww_ctx_backoff(&eb.ww); - if (!err) - goto retry; - } - i915_gem_ww_ctx_fini(&eb.ww); - - if (eb.reloc_pool) - intel_gt_buffer_pool_put(eb.reloc_pool); - if (eb.reloc_context) - intel_context_put(eb.reloc_context); - - intel_context_put(eb.context); -err_pm: - intel_engine_pm_put(eb.engine); - if (err) - break; - } - - if (igt_flush_test(eb.i915)) - err = -EIO; - - intel_gt_pm_put(&eb.i915->gt); -err_scratch: - i915_gem_object_put(scratch); - return err; -} - -int i915_gem_execbuffer_live_selftests(struct drm_i915_private *i915) -{ - static const struct i915_subtest tests[] = { - SUBTEST(igt_gpu_reloc), - }; - - if (intel_gt_is_wedged(&i915->gt)) - return 0; - - return i915_live_subtests(tests, i915); -} diff --git a/drivers/gpu/drm/i915/gem/selftests/i915_gem_mman.c b/drivers/gpu/drm/i915/gem/selftests/i915_gem_mman.c index a2c34e5a1c54..6d30cdfa80f3 100644 --- a/drivers/gpu/drm/i915/gem/selftests/i915_gem_mman.c +++ b/drivers/gpu/drm/i915/gem/selftests/i915_gem_mman.c @@ -903,7 +903,9 @@ static int __igt_mmap(struct drm_i915_private *i915, pr_debug("igt_mmap(%s, %d) @ %lx\n", obj->mm.region->name, type, addr); + mmap_read_lock(current->mm); area = vma_lookup(current->mm, addr); + mmap_read_unlock(current->mm); if (!area) { pr_err("%s: Did not create a vm_area_struct for the mmap\n", obj->mm.region->name); diff --git a/drivers/gpu/drm/i915/gem/selftests/mock_context.c b/drivers/gpu/drm/i915/gem/selftests/mock_context.c index fee070df1c97..c0a8ef368044 100644 --- a/drivers/gpu/drm/i915/gem/selftests/mock_context.c +++ b/drivers/gpu/drm/i915/gem/selftests/mock_context.c @@ -23,6 +23,7 @@ mock_context(struct drm_i915_private *i915, kref_init(&ctx->ref); INIT_LIST_HEAD(&ctx->link); ctx->i915 = i915; + INIT_WORK(&ctx->release_work, i915_gem_context_release_work); mutex_init(&ctx->mutex); @@ -87,7 +88,7 @@ live_context(struct drm_i915_private *i915, struct file *file) return ERR_CAST(pc); ctx = i915_gem_create_context(i915, pc); - proto_context_close(pc); + proto_context_close(i915, pc); if (IS_ERR(ctx)) return ctx; @@ -162,7 +163,7 @@ kernel_context(struct drm_i915_private *i915, } ctx = i915_gem_create_context(i915, pc); - proto_context_close(pc); + proto_context_close(i915, pc); if (IS_ERR(ctx)) return ctx; diff --git a/drivers/gpu/drm/i915/gt/debugfs_engines.h b/drivers/gpu/drm/i915/gt/debugfs_engines.h deleted file mode 100644 index f69257eaa1cc..000000000000 --- a/drivers/gpu/drm/i915/gt/debugfs_engines.h +++ /dev/null @@ -1,14 +0,0 @@ -/* SPDX-License-Identifier: MIT */ -/* - * Copyright © 2019 Intel Corporation - */ - -#ifndef DEBUGFS_ENGINES_H -#define DEBUGFS_ENGINES_H - -struct intel_gt; -struct dentry; - -void debugfs_engines_register(struct intel_gt *gt, struct dentry *root); - -#endif /* DEBUGFS_ENGINES_H */ diff --git a/drivers/gpu/drm/i915/gt/debugfs_gt.c b/drivers/gpu/drm/i915/gt/debugfs_gt.c deleted file mode 100644 index 591eb60785db..000000000000 --- a/drivers/gpu/drm/i915/gt/debugfs_gt.c +++ /dev/null @@ -1,47 +0,0 @@ -// SPDX-License-Identifier: MIT -/* - * Copyright © 2019 Intel Corporation - */ - -#include <linux/debugfs.h> - -#include "debugfs_engines.h" -#include "debugfs_gt.h" -#include "debugfs_gt_pm.h" -#include "intel_sseu_debugfs.h" -#include "uc/intel_uc_debugfs.h" -#include "i915_drv.h" - -void debugfs_gt_register(struct intel_gt *gt) -{ - struct dentry *root; - - if (!gt->i915->drm.primary->debugfs_root) - return; - - root = debugfs_create_dir("gt", gt->i915->drm.primary->debugfs_root); - if (IS_ERR(root)) - return; - - debugfs_engines_register(gt, root); - debugfs_gt_pm_register(gt, root); - intel_sseu_debugfs_register(gt, root); - - intel_uc_debugfs_register(>->uc, root); -} - -void intel_gt_debugfs_register_files(struct dentry *root, - const struct debugfs_gt_file *files, - unsigned long count, void *data) -{ - while (count--) { - umode_t mode = files->fops->write ? 0644 : 0444; - - if (!files->eval || files->eval(data)) - debugfs_create_file(files->name, - mode, root, data, - files->fops); - - files++; - } -} diff --git a/drivers/gpu/drm/i915/gt/debugfs_gt_pm.h b/drivers/gpu/drm/i915/gt/debugfs_gt_pm.h deleted file mode 100644 index 4cf5f5c9da7d..000000000000 --- a/drivers/gpu/drm/i915/gt/debugfs_gt_pm.h +++ /dev/null @@ -1,14 +0,0 @@ -/* SPDX-License-Identifier: MIT */ -/* - * Copyright © 2019 Intel Corporation - */ - -#ifndef DEBUGFS_GT_PM_H -#define DEBUGFS_GT_PM_H - -struct intel_gt; -struct dentry; - -void debugfs_gt_pm_register(struct intel_gt *gt, struct dentry *root); - -#endif /* DEBUGFS_GT_PM_H */ diff --git a/drivers/gpu/drm/i915/gt/gen6_ppgtt.c b/drivers/gpu/drm/i915/gt/gen6_ppgtt.c index 1aee5e6b1b23..890191f286e3 100644 --- a/drivers/gpu/drm/i915/gt/gen6_ppgtt.c +++ b/drivers/gpu/drm/i915/gt/gen6_ppgtt.c @@ -429,7 +429,7 @@ struct i915_ppgtt *gen6_ppgtt_create(struct intel_gt *gt) mutex_init(&ppgtt->flush); mutex_init(&ppgtt->pin_mutex); - ppgtt_init(&ppgtt->base, gt); + ppgtt_init(&ppgtt->base, gt, 0); ppgtt->base.vm.pd_shift = ilog2(SZ_4K * SZ_4K / sizeof(gen6_pte_t)); ppgtt->base.vm.top = 1; diff --git a/drivers/gpu/drm/i915/gt/gen8_ppgtt.c b/drivers/gpu/drm/i915/gt/gen8_ppgtt.c index 6e0e52eeb87a..037a9a6e4889 100644 --- a/drivers/gpu/drm/i915/gt/gen8_ppgtt.c +++ b/drivers/gpu/drm/i915/gt/gen8_ppgtt.c @@ -548,6 +548,7 @@ static void gen8_ppgtt_insert_huge(struct i915_vma *vma, I915_GTT_PAGE_SIZE_2M)))) { vaddr = px_vaddr(pd); vaddr[maybe_64K] |= GEN8_PDE_IPS_64K; + clflush_cache_range(vaddr, PAGE_SIZE); page_size = I915_GTT_PAGE_SIZE_64K; /* @@ -568,6 +569,7 @@ static void gen8_ppgtt_insert_huge(struct i915_vma *vma, for (i = 1; i < index; i += 16) memset64(vaddr + i, encode, 15); + clflush_cache_range(vaddr, PAGE_SIZE); } } @@ -751,7 +753,8 @@ err_pd: * space. * */ -struct i915_ppgtt *gen8_ppgtt_create(struct intel_gt *gt) +struct i915_ppgtt *gen8_ppgtt_create(struct intel_gt *gt, + unsigned long lmem_pt_obj_flags) { struct i915_ppgtt *ppgtt; int err; @@ -760,7 +763,7 @@ struct i915_ppgtt *gen8_ppgtt_create(struct intel_gt *gt) if (!ppgtt) return ERR_PTR(-ENOMEM); - ppgtt_init(ppgtt, gt); + ppgtt_init(ppgtt, gt, lmem_pt_obj_flags); ppgtt->vm.top = i915_vm_is_4lvl(&ppgtt->vm) ? 3 : 2; ppgtt->vm.pd_shift = ilog2(SZ_4K * SZ_4K / sizeof(gen8_pte_t)); diff --git a/drivers/gpu/drm/i915/gt/gen8_ppgtt.h b/drivers/gpu/drm/i915/gt/gen8_ppgtt.h index b9028c2ad3c7..f541d19264b4 100644 --- a/drivers/gpu/drm/i915/gt/gen8_ppgtt.h +++ b/drivers/gpu/drm/i915/gt/gen8_ppgtt.h @@ -12,7 +12,9 @@ struct i915_address_space; struct intel_gt; enum i915_cache_level; -struct i915_ppgtt *gen8_ppgtt_create(struct intel_gt *gt); +struct i915_ppgtt *gen8_ppgtt_create(struct intel_gt *gt, + unsigned long lmem_pt_obj_flags); + u64 gen8_ggtt_pte_encode(dma_addr_t addr, enum i915_cache_level level, u32 flags); diff --git a/drivers/gpu/drm/i915/gt/intel_context.c b/drivers/gpu/drm/i915/gt/intel_context.c index 17ca4dc4d0cb..5634d14052bc 100644 --- a/drivers/gpu/drm/i915/gt/intel_context.c +++ b/drivers/gpu/drm/i915/gt/intel_context.c @@ -240,6 +240,8 @@ int __intel_context_do_pin_ww(struct intel_context *ce, if (err) goto err_post_unpin; + intel_engine_pm_might_get(ce->engine); + if (unlikely(intel_context_is_closed(ce))) { err = -ENOENT; goto err_unlock; @@ -395,19 +397,22 @@ intel_context_init(struct intel_context *ce, struct intel_engine_cs *engine) spin_lock_init(&ce->guc_state.lock); INIT_LIST_HEAD(&ce->guc_state.fences); + INIT_LIST_HEAD(&ce->guc_state.requests); + + ce->guc_id.id = GUC_INVALID_LRC_ID; + INIT_LIST_HEAD(&ce->guc_id.link); - spin_lock_init(&ce->guc_active.lock); - INIT_LIST_HEAD(&ce->guc_active.requests); + INIT_LIST_HEAD(&ce->destroyed_link); - ce->guc_id = GUC_INVALID_LRC_ID; - INIT_LIST_HEAD(&ce->guc_id_link); + INIT_LIST_HEAD(&ce->parallel.child_list); /* * Initialize fence to be complete as this is expected to be complete * unless there is a pending schedule disable outstanding. */ - i915_sw_fence_init(&ce->guc_blocked, sw_fence_dummy_notify); - i915_sw_fence_commit(&ce->guc_blocked); + i915_sw_fence_init(&ce->guc_state.blocked, + sw_fence_dummy_notify); + i915_sw_fence_commit(&ce->guc_state.blocked); i915_active_init(&ce->active, __intel_context_active, __intel_context_retire, 0); @@ -415,13 +420,20 @@ intel_context_init(struct intel_context *ce, struct intel_engine_cs *engine) void intel_context_fini(struct intel_context *ce) { + struct intel_context *child, *next; + if (ce->timeline) intel_timeline_put(ce->timeline); i915_vm_put(ce->vm); + /* Need to put the creation ref for the children */ + if (intel_context_is_parent(ce)) + for_each_child_safe(ce, child, next) + intel_context_put(child); + mutex_destroy(&ce->pin_mutex); i915_active_fini(&ce->active); - i915_sw_fence_fini(&ce->guc_blocked); + i915_sw_fence_fini(&ce->guc_state.blocked); } void i915_context_module_exit(void) @@ -517,24 +529,53 @@ retry: struct i915_request *intel_context_find_active_request(struct intel_context *ce) { + struct intel_context *parent = intel_context_to_parent(ce); struct i915_request *rq, *active = NULL; unsigned long flags; GEM_BUG_ON(!intel_engine_uses_guc(ce->engine)); - spin_lock_irqsave(&ce->guc_active.lock, flags); - list_for_each_entry_reverse(rq, &ce->guc_active.requests, + /* + * We search the parent list to find an active request on the submitted + * context. The parent list contains the requests for all the contexts + * in the relationship so we have to do a compare of each request's + * context. + */ + spin_lock_irqsave(&parent->guc_state.lock, flags); + list_for_each_entry_reverse(rq, &parent->guc_state.requests, sched.link) { + if (rq->context != ce) + continue; if (i915_request_completed(rq)) break; active = rq; } - spin_unlock_irqrestore(&ce->guc_active.lock, flags); + spin_unlock_irqrestore(&parent->guc_state.lock, flags); return active; } +void intel_context_bind_parent_child(struct intel_context *parent, + struct intel_context *child) +{ + /* + * Callers responsibility to validate that this function is used + * correctly but we use GEM_BUG_ON here ensure that they do. + */ + GEM_BUG_ON(!intel_engine_uses_guc(parent->engine)); + GEM_BUG_ON(intel_context_is_pinned(parent)); + GEM_BUG_ON(intel_context_is_child(parent)); + GEM_BUG_ON(intel_context_is_pinned(child)); + GEM_BUG_ON(intel_context_is_child(child)); + GEM_BUG_ON(intel_context_is_parent(child)); + + parent->parallel.child_index = parent->parallel.number_children++; + list_add_tail(&child->parallel.child_link, + &parent->parallel.child_list); + child->parallel.parent = parent; +} + #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) #include "selftest_context.c" #endif diff --git a/drivers/gpu/drm/i915/gt/intel_context.h b/drivers/gpu/drm/i915/gt/intel_context.h index c41098950746..246c37d72cd7 100644 --- a/drivers/gpu/drm/i915/gt/intel_context.h +++ b/drivers/gpu/drm/i915/gt/intel_context.h @@ -44,6 +44,54 @@ void intel_context_free(struct intel_context *ce); int intel_context_reconfigure_sseu(struct intel_context *ce, const struct intel_sseu sseu); +#define PARENT_SCRATCH_SIZE PAGE_SIZE + +static inline bool intel_context_is_child(struct intel_context *ce) +{ + return !!ce->parallel.parent; +} + +static inline bool intel_context_is_parent(struct intel_context *ce) +{ + return !!ce->parallel.number_children; +} + +static inline bool intel_context_is_pinned(struct intel_context *ce); + +static inline struct intel_context * +intel_context_to_parent(struct intel_context *ce) +{ + if (intel_context_is_child(ce)) { + /* + * The parent holds ref count to the child so it is always safe + * for the parent to access the child, but the child has a + * pointer to the parent without a ref. To ensure this is safe + * the child should only access the parent pointer while the + * parent is pinned. + */ + GEM_BUG_ON(!intel_context_is_pinned(ce->parallel.parent)); + + return ce->parallel.parent; + } else { + return ce; + } +} + +static inline bool intel_context_is_parallel(struct intel_context *ce) +{ + return intel_context_is_child(ce) || intel_context_is_parent(ce); +} + +void intel_context_bind_parent_child(struct intel_context *parent, + struct intel_context *child); + +#define for_each_child(parent, ce)\ + list_for_each_entry(ce, &(parent)->parallel.child_list,\ + parallel.child_link) +#define for_each_child_safe(parent, ce, cn)\ + list_for_each_entry_safe(ce, cn, &(parent)->parallel.child_list,\ + parallel.child_link) + /** * intel_context_lock_pinned - Stablises the 'pinned' status of the HW context * @ce - the context @@ -193,7 +241,13 @@ intel_context_timeline_lock(struct intel_context *ce) struct intel_timeline *tl = ce->timeline; int err; - err = mutex_lock_interruptible(&tl->mutex); + if (intel_context_is_parent(ce)) + err = mutex_lock_interruptible_nested(&tl->mutex, 0); + else if (intel_context_is_child(ce)) + err = mutex_lock_interruptible_nested(&tl->mutex, + ce->parallel.child_index + 1); + else + err = mutex_lock_interruptible(&tl->mutex); if (err) return ERR_PTR(err); diff --git a/drivers/gpu/drm/i915/gt/intel_context_types.h b/drivers/gpu/drm/i915/gt/intel_context_types.h index e54351a170e2..9e0177dc5484 100644 --- a/drivers/gpu/drm/i915/gt/intel_context_types.h +++ b/drivers/gpu/drm/i915/gt/intel_context_types.h @@ -55,9 +55,13 @@ struct intel_context_ops { void (*reset)(struct intel_context *ce); void (*destroy)(struct kref *kref); - /* virtual engine/context interface */ + /* virtual/parallel engine/context interface */ struct intel_context *(*create_virtual)(struct intel_engine_cs **engine, - unsigned int count); + unsigned int count, + unsigned long flags); + struct intel_context *(*create_parallel)(struct intel_engine_cs **engines, + unsigned int num_siblings, + unsigned int width); struct intel_engine_cs *(*get_sibling)(struct intel_engine_cs *engine, unsigned int sibling); }; @@ -112,6 +116,8 @@ struct intel_context { #define CONTEXT_FORCE_SINGLE_SUBMISSION 7 #define CONTEXT_NOPREEMPT 8 #define CONTEXT_LRCA_DIRTY 9 +#define CONTEXT_GUC_INIT 10 +#define CONTEXT_PERMA_PIN 11 struct { u64 timeout_us; @@ -152,52 +158,141 @@ struct intel_context { /** sseu: Control eu/slice partitioning */ struct intel_sseu sseu; + /** + * pinned_contexts_link: List link for the engine's pinned contexts. + * This is only used if this is a perma-pinned kernel context and + * the list is assumed to only be manipulated during driver load + * or unload time so no mutex protection currently. + */ + struct list_head pinned_contexts_link; + u8 wa_bb_page; /* if set, page num reserved for context workarounds */ struct { - /** lock: protects everything in guc_state */ + /** @lock: protects everything in guc_state */ spinlock_t lock; /** - * sched_state: scheduling state of this context using GuC + * @sched_state: scheduling state of this context using GuC * submission */ - u16 sched_state; + u32 sched_state; /* - * fences: maintains of list of requests that have a submit - * fence related to GuC submission + * @fences: maintains a list of requests that are currently + * being fenced until a GuC operation completes */ struct list_head fences; + /** + * @blocked: fence used to signal when the blocking of a + * context's submissions is complete. + */ + struct i915_sw_fence blocked; + /** @number_committed_requests: number of committed requests */ + int number_committed_requests; + /** @requests: list of active requests on this context */ + struct list_head requests; + /** @prio: the context's current guc priority */ + u8 prio; + /** + * @prio_count: a counter of the number requests in flight in + * each priority bucket + */ + u32 prio_count[GUC_CLIENT_PRIORITY_NUM]; } guc_state; struct { - /** lock: protects everything in guc_active */ - spinlock_t lock; - /** requests: active requests on this context */ - struct list_head requests; - } guc_active; - - /* GuC scheduling state flags that do not require a lock. */ - atomic_t guc_sched_state_no_lock; - - /* GuC LRC descriptor ID */ - u16 guc_id; + /** + * @id: handle which is used to uniquely identify this context + * with the GuC, protected by guc->submission_state.lock + */ + u16 id; + /** + * @ref: the number of references to the guc_id, when + * transitioning in and out of zero protected by + * guc->submission_state.lock + */ + atomic_t ref; + /** + * @link: in guc->guc_id_list when the guc_id has no refs but is + * still valid, protected by guc->submission_state.lock + */ + struct list_head link; + } guc_id; - /* GuC LRC descriptor reference count */ - atomic_t guc_id_ref; + /** + * @destroyed_link: link in guc->submission_state.destroyed_contexts, in + * list when context is pending to be destroyed (deregistered with the + * GuC), protected by guc->submission_state.lock + */ + struct list_head destroyed_link; - /* - * GuC ID link - in list when unpinned but guc_id still valid in GuC + /** @parallel: sub-structure for parallel submission members */ + struct { + union { + /** + * @child_list: parent's list of children + * contexts, no protection as immutable after context + * creation + */ + struct list_head child_list; + /** + * @child_link: child's link into parent's list of + * children + */ + struct list_head child_link; + }; + /** @parent: pointer to parent if child */ + struct intel_context *parent; + /** + * @last_rq: last request submitted on a parallel context, used + * to insert submit fences between requests in the parallel + * context + */ + struct i915_request *last_rq; + /** + * @fence_context: fence context composite fence when doing + * parallel submission + */ + u64 fence_context; + /** + * @seqno: seqno for composite fence when doing parallel + * submission + */ + u32 seqno; + /** @number_children: number of children if parent */ + u8 number_children; + /** @child_index: index into child_list if child */ + u8 child_index; + /** @guc: GuC specific members for parallel submission */ + struct { + /** @wqi_head: head pointer in work queue */ + u16 wqi_head; + /** @wqi_tail: tail pointer in work queue */ + u16 wqi_tail; + /** + * @parent_page: page in context state (ce->state) used + * by parent for work queue, process descriptor + */ + u8 parent_page; + } guc; + } parallel; + +#ifdef CONFIG_DRM_I915_SELFTEST + /** + * @drop_schedule_enable: Force drop of schedule enable G2H for selftest */ - struct list_head guc_id_link; + bool drop_schedule_enable; - /* GuC context blocked fence */ - struct i915_sw_fence guc_blocked; + /** + * @drop_schedule_disable: Force drop of schedule disable G2H for + * selftest + */ + bool drop_schedule_disable; - /* - * GuC priority management + /** + * @drop_deregister: Force drop of deregister G2H for selftest */ - u8 guc_prio; - u32 guc_prio_count[GUC_CLIENT_PRIORITY_NUM]; + bool drop_deregister; +#endif }; #endif /* __INTEL_CONTEXT_TYPES__ */ diff --git a/drivers/gpu/drm/i915/gt/intel_engine.h b/drivers/gpu/drm/i915/gt/intel_engine.h index 87579affb952..08559ace0ada 100644 --- a/drivers/gpu/drm/i915/gt/intel_engine.h +++ b/drivers/gpu/drm/i915/gt/intel_engine.h @@ -2,6 +2,7 @@ #ifndef _INTEL_RINGBUFFER_H_ #define _INTEL_RINGBUFFER_H_ +#include <asm/cacheflush.h> #include <drm/drm_util.h> #include <linux/hashtable.h> @@ -175,6 +176,8 @@ intel_write_status_page(struct intel_engine_cs *engine, int reg, u32 value) #define I915_GEM_HWS_SEQNO 0x40 #define I915_GEM_HWS_SEQNO_ADDR (I915_GEM_HWS_SEQNO * sizeof(u32)) #define I915_GEM_HWS_MIGRATE (0x42 * sizeof(u32)) +#define I915_GEM_HWS_PXP 0x60 +#define I915_GEM_HWS_PXP_ADDR (I915_GEM_HWS_PXP * sizeof(u32)) #define I915_GEM_HWS_SCRATCH 0x80 #define I915_HWS_CSB_BUF0_INDEX 0x10 @@ -273,15 +276,25 @@ static inline bool intel_engine_uses_guc(const struct intel_engine_cs *engine) static inline bool intel_engine_has_preempt_reset(const struct intel_engine_cs *engine) { - if (!IS_ACTIVE(CONFIG_DRM_I915_PREEMPT_TIMEOUT)) + if (!CONFIG_DRM_I915_PREEMPT_TIMEOUT) return false; return intel_engine_has_preemption(engine); } +#define FORCE_VIRTUAL BIT(0) struct intel_context * intel_engine_create_virtual(struct intel_engine_cs **siblings, - unsigned int count); + unsigned int count, unsigned long flags); + +static inline struct intel_context * +intel_engine_create_parallel(struct intel_engine_cs **engines, + unsigned int num_engines, + unsigned int width) +{ + GEM_BUG_ON(!engines[0]->cops->create_parallel); + return engines[0]->cops->create_parallel(engines, num_engines, width); +} static inline bool intel_virtual_engine_has_heartbeat(const struct intel_engine_cs *engine) @@ -300,7 +313,7 @@ intel_virtual_engine_has_heartbeat(const struct intel_engine_cs *engine) static inline bool intel_engine_has_heartbeat(const struct intel_engine_cs *engine) { - if (!IS_ACTIVE(CONFIG_DRM_I915_HEARTBEAT_INTERVAL)) + if (!CONFIG_DRM_I915_HEARTBEAT_INTERVAL) return false; if (intel_engine_is_virtual(engine)) diff --git a/drivers/gpu/drm/i915/gt/intel_engine_cs.c b/drivers/gpu/drm/i915/gt/intel_engine_cs.c index 0d9105a31d84..ff6753ccb129 100644 --- a/drivers/gpu/drm/i915/gt/intel_engine_cs.c +++ b/drivers/gpu/drm/i915/gt/intel_engine_cs.c @@ -290,7 +290,8 @@ static void nop_irq_handler(struct intel_engine_cs *engine, u16 iir) GEM_DEBUG_WARN_ON(iir); } -static int intel_engine_setup(struct intel_gt *gt, enum intel_engine_id id) +static int intel_engine_setup(struct intel_gt *gt, enum intel_engine_id id, + u8 logical_instance) { const struct engine_info *info = &intel_engines[id]; struct drm_i915_private *i915 = gt->i915; @@ -320,6 +321,7 @@ static int intel_engine_setup(struct intel_gt *gt, enum intel_engine_id id) BUILD_BUG_ON(BITS_PER_TYPE(engine->mask) < I915_NUM_ENGINES); + INIT_LIST_HEAD(&engine->pinned_contexts_list); engine->id = id; engine->legacy_idx = INVALID_ENGINE; engine->mask = BIT(id); @@ -334,6 +336,7 @@ static int intel_engine_setup(struct intel_gt *gt, enum intel_engine_id id) engine->class = info->class; engine->instance = info->instance; + engine->logical_mask = BIT(logical_instance); __sprint_engine_name(engine); engine->props.heartbeat_interval_ms = @@ -398,7 +401,8 @@ static void __setup_engine_capabilities(struct intel_engine_cs *engine) engine->uabi_capabilities |= I915_VIDEO_AND_ENHANCE_CLASS_CAPABILITY_SFC; } else if (engine->class == VIDEO_ENHANCEMENT_CLASS) { - if (GRAPHICS_VER(i915) >= 9) + if (GRAPHICS_VER(i915) >= 9 && + engine->gt->info.sfc_mask & BIT(engine->instance)) engine->uabi_capabilities |= I915_VIDEO_AND_ENHANCE_CLASS_CAPABILITY_SFC; } @@ -474,18 +478,25 @@ void intel_engines_free(struct intel_gt *gt) } static -bool gen11_vdbox_has_sfc(struct drm_i915_private *i915, +bool gen11_vdbox_has_sfc(struct intel_gt *gt, unsigned int physical_vdbox, unsigned int logical_vdbox, u16 vdbox_mask) { + struct drm_i915_private *i915 = gt->i915; + /* * In Gen11, only even numbered logical VDBOXes are hooked * up to an SFC (Scaler & Format Converter) unit. * In Gen12, Even numbered physical instance always are connected * to an SFC. Odd numbered physical instances have SFC only if * previous even instance is fused off. + * + * Starting with Xe_HP, there's also a dedicated SFC_ENABLE field + * in the fuse register that tells us whether a specific SFC is present. */ - if (GRAPHICS_VER(i915) == 12) + if ((gt->info.sfc_mask & BIT(physical_vdbox / 2)) == 0) + return false; + else if (GRAPHICS_VER(i915) == 12) return (physical_vdbox % 2 == 0) || !(BIT(physical_vdbox - 1) & vdbox_mask); else if (GRAPHICS_VER(i915) == 11) @@ -512,7 +523,7 @@ static intel_engine_mask_t init_engine_mask(struct intel_gt *gt) struct intel_uncore *uncore = gt->uncore; unsigned int logical_vdbox = 0; unsigned int i; - u32 media_fuse; + u32 media_fuse, fuse1; u16 vdbox_mask; u16 vebox_mask; @@ -534,6 +545,13 @@ static intel_engine_mask_t init_engine_mask(struct intel_gt *gt) vebox_mask = (media_fuse & GEN11_GT_VEBOX_DISABLE_MASK) >> GEN11_GT_VEBOX_DISABLE_SHIFT; + if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 50)) { + fuse1 = intel_uncore_read(uncore, HSW_PAVP_FUSE1); + gt->info.sfc_mask = REG_FIELD_GET(XEHP_SFC_ENABLE_MASK, fuse1); + } else { + gt->info.sfc_mask = ~0; + } + for (i = 0; i < I915_MAX_VCS; i++) { if (!HAS_ENGINE(gt, _VCS(i))) { vdbox_mask &= ~BIT(i); @@ -546,7 +564,7 @@ static intel_engine_mask_t init_engine_mask(struct intel_gt *gt) continue; } - if (gen11_vdbox_has_sfc(i915, i, logical_vdbox, vdbox_mask)) + if (gen11_vdbox_has_sfc(gt, i, logical_vdbox, vdbox_mask)) gt->info.vdbox_sfc_access |= BIT(i); logical_vdbox++; } @@ -572,6 +590,37 @@ static intel_engine_mask_t init_engine_mask(struct intel_gt *gt) return info->engine_mask; } +static void populate_logical_ids(struct intel_gt *gt, u8 *logical_ids, + u8 class, const u8 *map, u8 num_instances) +{ + int i, j; + u8 current_logical_id = 0; + + for (j = 0; j < num_instances; ++j) { + for (i = 0; i < ARRAY_SIZE(intel_engines); ++i) { + if (!HAS_ENGINE(gt, i) || + intel_engines[i].class != class) + continue; + + if (intel_engines[i].instance == map[j]) { + logical_ids[intel_engines[i].instance] = + current_logical_id++; + break; + } + } + } +} + +static void setup_logical_ids(struct intel_gt *gt, u8 *logical_ids, u8 class) +{ + int i; + u8 map[MAX_ENGINE_INSTANCE + 1]; + + for (i = 0; i < MAX_ENGINE_INSTANCE + 1; ++i) + map[i] = i; + populate_logical_ids(gt, logical_ids, class, map, ARRAY_SIZE(map)); +} + /** * intel_engines_init_mmio() - allocate and prepare the Engine Command Streamers * @gt: pointer to struct intel_gt @@ -583,7 +632,8 @@ int intel_engines_init_mmio(struct intel_gt *gt) struct drm_i915_private *i915 = gt->i915; const unsigned int engine_mask = init_engine_mask(gt); unsigned int mask = 0; - unsigned int i; + unsigned int i, class; + u8 logical_ids[MAX_ENGINE_INSTANCE + 1]; int err; drm_WARN_ON(&i915->drm, engine_mask == 0); @@ -593,15 +643,23 @@ int intel_engines_init_mmio(struct intel_gt *gt) if (i915_inject_probe_failure(i915)) return -ENODEV; - for (i = 0; i < ARRAY_SIZE(intel_engines); i++) { - if (!HAS_ENGINE(gt, i)) - continue; + for (class = 0; class < MAX_ENGINE_CLASS + 1; ++class) { + setup_logical_ids(gt, logical_ids, class); - err = intel_engine_setup(gt, i); - if (err) - goto cleanup; + for (i = 0; i < ARRAY_SIZE(intel_engines); ++i) { + u8 instance = intel_engines[i].instance; - mask |= BIT(i); + if (intel_engines[i].class != class || + !HAS_ENGINE(gt, i)) + continue; + + err = intel_engine_setup(gt, i, + logical_ids[instance]); + if (err) + goto cleanup; + + mask |= BIT(i); + } } /* @@ -875,6 +933,8 @@ intel_engine_create_pinned_context(struct intel_engine_cs *engine, return ERR_PTR(err); } + list_add_tail(&ce->pinned_contexts_link, &engine->pinned_contexts_list); + /* * Give our perma-pinned kernel timelines a separate lockdep class, * so that we can use them from within the normal user timelines @@ -897,6 +957,7 @@ void intel_engine_destroy_pinned_context(struct intel_context *ce) list_del(&ce->timeline->engine_link); mutex_unlock(&hwsp->vm->mutex); + list_del(&ce->pinned_contexts_link); intel_context_unpin(ce); intel_context_put(ce); } @@ -1163,16 +1224,16 @@ void intel_engine_get_instdone(const struct intel_engine_cs *engine, u32 mmio_base = engine->mmio_base; int slice; int subslice; + int iter; memset(instdone, 0, sizeof(*instdone)); - switch (GRAPHICS_VER(i915)) { - default: + if (GRAPHICS_VER(i915) >= 8) { instdone->instdone = intel_uncore_read(uncore, RING_INSTDONE(mmio_base)); if (engine->id != RCS0) - break; + return; instdone->slice_common = intel_uncore_read(uncore, GEN7_SC_INSTDONE); @@ -1182,21 +1243,39 @@ void intel_engine_get_instdone(const struct intel_engine_cs *engine, instdone->slice_common_extra[1] = intel_uncore_read(uncore, GEN12_SC_INSTDONE_EXTRA2); } - for_each_instdone_slice_subslice(i915, sseu, slice, subslice) { - instdone->sampler[slice][subslice] = - read_subslice_reg(engine, slice, subslice, - GEN7_SAMPLER_INSTDONE); - instdone->row[slice][subslice] = - read_subslice_reg(engine, slice, subslice, - GEN7_ROW_INSTDONE); + + if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 50)) { + for_each_instdone_gslice_dss_xehp(i915, sseu, iter, slice, subslice) { + instdone->sampler[slice][subslice] = + read_subslice_reg(engine, slice, subslice, + GEN7_SAMPLER_INSTDONE); + instdone->row[slice][subslice] = + read_subslice_reg(engine, slice, subslice, + GEN7_ROW_INSTDONE); + } + } else { + for_each_instdone_slice_subslice(i915, sseu, slice, subslice) { + instdone->sampler[slice][subslice] = + read_subslice_reg(engine, slice, subslice, + GEN7_SAMPLER_INSTDONE); + instdone->row[slice][subslice] = + read_subslice_reg(engine, slice, subslice, + GEN7_ROW_INSTDONE); + } } - break; - case 7: + + if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 55)) { + for_each_instdone_gslice_dss_xehp(i915, sseu, iter, slice, subslice) + instdone->geom_svg[slice][subslice] = + read_subslice_reg(engine, slice, subslice, + XEHPG_INSTDONE_GEOM_SVG); + } + } else if (GRAPHICS_VER(i915) >= 7) { instdone->instdone = intel_uncore_read(uncore, RING_INSTDONE(mmio_base)); if (engine->id != RCS0) - break; + return; instdone->slice_common = intel_uncore_read(uncore, GEN7_SC_INSTDONE); @@ -1204,22 +1283,15 @@ void intel_engine_get_instdone(const struct intel_engine_cs *engine, intel_uncore_read(uncore, GEN7_SAMPLER_INSTDONE); instdone->row[0][0] = intel_uncore_read(uncore, GEN7_ROW_INSTDONE); - - break; - case 6: - case 5: - case 4: + } else if (GRAPHICS_VER(i915) >= 4) { instdone->instdone = intel_uncore_read(uncore, RING_INSTDONE(mmio_base)); if (engine->id == RCS0) /* HACK: Using the wrong struct member */ instdone->slice_common = intel_uncore_read(uncore, GEN4_INSTDONE1); - break; - case 3: - case 2: + } else { instdone->instdone = intel_uncore_read(uncore, GEN2_INSTDONE); - break; } } @@ -1881,16 +1953,16 @@ ktime_t intel_engine_get_busy_time(struct intel_engine_cs *engine, ktime_t *now) struct intel_context * intel_engine_create_virtual(struct intel_engine_cs **siblings, - unsigned int count) + unsigned int count, unsigned long flags) { if (count == 0) return ERR_PTR(-EINVAL); - if (count == 1) + if (count == 1 && !(flags & FORCE_VIRTUAL)) return intel_context_create(siblings[0]); GEM_BUG_ON(!siblings[0]->cops->create_virtual); - return siblings[0]->cops->create_virtual(siblings, count); + return siblings[0]->cops->create_virtual(siblings, count, flags); } struct i915_request * diff --git a/drivers/gpu/drm/i915/gt/intel_engine_heartbeat.c b/drivers/gpu/drm/i915/gt/intel_engine_heartbeat.c index 74775ae961b2..a3698f611f45 100644 --- a/drivers/gpu/drm/i915/gt/intel_engine_heartbeat.c +++ b/drivers/gpu/drm/i915/gt/intel_engine_heartbeat.c @@ -207,7 +207,7 @@ out: void intel_engine_unpark_heartbeat(struct intel_engine_cs *engine) { - if (!IS_ACTIVE(CONFIG_DRM_I915_HEARTBEAT_INTERVAL)) + if (!CONFIG_DRM_I915_HEARTBEAT_INTERVAL) return; next_heartbeat(engine); diff --git a/drivers/gpu/drm/i915/gt/intel_engine_pm.c b/drivers/gpu/drm/i915/gt/intel_engine_pm.c index 1f07ac4e0672..a1334b48dde7 100644 --- a/drivers/gpu/drm/i915/gt/intel_engine_pm.c +++ b/drivers/gpu/drm/i915/gt/intel_engine_pm.c @@ -162,6 +162,19 @@ static bool switch_to_kernel_context(struct intel_engine_cs *engine) unsigned long flags; bool result = true; + /* + * This is execlist specific behaviour intended to ensure the GPU is + * idle by switching to a known 'safe' context. With GuC submission, the + * same idle guarantee is achieved by other means (disabling + * scheduling). Further, switching to a 'safe' context has no effect + * with GuC submission as the scheduler can just switch back again. + * + * FIXME: Move this backend scheduler specific behaviour into the + * scheduler backend. + */ + if (intel_engine_uses_guc(engine)) + return true; + /* GPU is pointing to the void, as good as in the kernel context. */ if (intel_gt_is_wedged(engine->gt)) return true; @@ -298,6 +311,29 @@ void intel_engine_init__pm(struct intel_engine_cs *engine) intel_engine_init_heartbeat(engine); } +/** + * intel_engine_reset_pinned_contexts - Reset the pinned contexts of + * an engine. + * @engine: The engine whose pinned contexts we want to reset. + * + * Typically the pinned context LMEM images lose or get their content + * corrupted on suspend. This function resets their images. + */ +void intel_engine_reset_pinned_contexts(struct intel_engine_cs *engine) +{ + struct intel_context *ce; + + list_for_each_entry(ce, &engine->pinned_contexts_list, + pinned_contexts_link) { + /* kernel context gets reset at __engine_unpark() */ + if (ce == engine->kernel_context) + continue; + + dbg_poison_ce(ce); + ce->ops->reset(ce); + } +} + #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) #include "selftest_engine_pm.c" #endif diff --git a/drivers/gpu/drm/i915/gt/intel_engine_pm.h b/drivers/gpu/drm/i915/gt/intel_engine_pm.h index 70ea46d6cfb0..d68675925b79 100644 --- a/drivers/gpu/drm/i915/gt/intel_engine_pm.h +++ b/drivers/gpu/drm/i915/gt/intel_engine_pm.h @@ -6,9 +6,11 @@ #ifndef INTEL_ENGINE_PM_H #define INTEL_ENGINE_PM_H +#include "i915_drv.h" #include "i915_request.h" #include "intel_engine_types.h" #include "intel_wakeref.h" +#include "intel_gt_pm.h" static inline bool intel_engine_pm_is_awake(const struct intel_engine_cs *engine) @@ -16,6 +18,11 @@ intel_engine_pm_is_awake(const struct intel_engine_cs *engine) return intel_wakeref_is_active(&engine->wakeref); } +static inline void __intel_engine_pm_get(struct intel_engine_cs *engine) +{ + __intel_wakeref_get(&engine->wakeref); +} + static inline void intel_engine_pm_get(struct intel_engine_cs *engine) { intel_wakeref_get(&engine->wakeref); @@ -26,6 +33,21 @@ static inline bool intel_engine_pm_get_if_awake(struct intel_engine_cs *engine) return intel_wakeref_get_if_active(&engine->wakeref); } +static inline void intel_engine_pm_might_get(struct intel_engine_cs *engine) +{ + if (!intel_engine_is_virtual(engine)) { + intel_wakeref_might_get(&engine->wakeref); + } else { + struct intel_gt *gt = engine->gt; + struct intel_engine_cs *tengine; + intel_engine_mask_t tmp, mask = engine->mask; + + for_each_engine_masked(tengine, gt, mask, tmp) + intel_wakeref_might_get(&tengine->wakeref); + } + intel_gt_pm_might_get(engine->gt); +} + static inline void intel_engine_pm_put(struct intel_engine_cs *engine) { intel_wakeref_put(&engine->wakeref); @@ -47,6 +69,21 @@ static inline void intel_engine_pm_flush(struct intel_engine_cs *engine) intel_wakeref_unlock_wait(&engine->wakeref); } +static inline void intel_engine_pm_might_put(struct intel_engine_cs *engine) +{ + if (!intel_engine_is_virtual(engine)) { + intel_wakeref_might_put(&engine->wakeref); + } else { + struct intel_gt *gt = engine->gt; + struct intel_engine_cs *tengine; + intel_engine_mask_t tmp, mask = engine->mask; + + for_each_engine_masked(tengine, gt, mask, tmp) + intel_wakeref_might_put(&tengine->wakeref); + } + intel_gt_pm_might_put(engine->gt); +} + static inline struct i915_request * intel_engine_create_kernel_request(struct intel_engine_cs *engine) { @@ -69,4 +106,6 @@ intel_engine_create_kernel_request(struct intel_engine_cs *engine) void intel_engine_init__pm(struct intel_engine_cs *engine); +void intel_engine_reset_pinned_contexts(struct intel_engine_cs *engine); + #endif /* INTEL_ENGINE_PM_H */ diff --git a/drivers/gpu/drm/i915/gt/intel_engine_types.h b/drivers/gpu/drm/i915/gt/intel_engine_types.h index ed91bcff20eb..e0f773585c29 100644 --- a/drivers/gpu/drm/i915/gt/intel_engine_types.h +++ b/drivers/gpu/drm/i915/gt/intel_engine_types.h @@ -67,8 +67,11 @@ struct intel_instdone { /* The following exist only in the RCS engine */ u32 slice_common; u32 slice_common_extra[2]; - u32 sampler[I915_MAX_SLICES][I915_MAX_SUBSLICES]; - u32 row[I915_MAX_SLICES][I915_MAX_SUBSLICES]; + u32 sampler[GEN_MAX_GSLICES][I915_MAX_SUBSLICES]; + u32 row[GEN_MAX_GSLICES][I915_MAX_SUBSLICES]; + + /* Added in XeHPG */ + u32 geom_svg[GEN_MAX_GSLICES][I915_MAX_SUBSLICES]; }; /* @@ -266,6 +269,13 @@ struct intel_engine_cs { unsigned int guc_id; intel_engine_mask_t mask; + /** + * @logical_mask: logical mask of engine, reported to user space via + * query IOCTL and used to communicate with the GuC in logical space. + * The logical instance of a physical engine can change based on product + * and fusing. + */ + intel_engine_mask_t logical_mask; u8 class; u8 instance; @@ -304,6 +314,13 @@ struct intel_engine_cs { struct intel_context *kernel_context; /* pinned */ + /** + * pinned_contexts_list: List of pinned contexts. This list is only + * assumed to be manipulated during driver load- or unload time and + * does therefore not have any additional protection. + */ + struct list_head pinned_contexts_list; + intel_engine_mask_t saturated; /* submitting semaphores too late? */ struct { @@ -546,7 +563,7 @@ intel_engine_has_semaphores(const struct intel_engine_cs *engine) static inline bool intel_engine_has_timeslices(const struct intel_engine_cs *engine) { - if (!IS_ACTIVE(CONFIG_DRM_I915_TIMESLICE_DURATION)) + if (!CONFIG_DRM_I915_TIMESLICE_DURATION) return false; return engine->flags & I915_ENGINE_HAS_TIMESLICES; @@ -578,4 +595,12 @@ intel_engine_has_relative_mmio(const struct intel_engine_cs * const engine) for_each_if((instdone_has_slice(dev_priv_, sseu_, slice_)) && \ (instdone_has_subslice(dev_priv_, sseu_, slice_, \ subslice_))) + +#define for_each_instdone_gslice_dss_xehp(dev_priv_, sseu_, iter_, gslice_, dss_) \ + for ((iter_) = 0, (gslice_) = 0, (dss_) = 0; \ + (iter_) < GEN_MAX_SUBSLICES; \ + (iter_)++, (gslice_) = (iter_) / GEN_DSS_PER_GSLICE, \ + (dss_) = (iter_) % GEN_DSS_PER_GSLICE) \ + for_each_if(intel_sseu_has_subslice((sseu_), 0, (iter_))) + #endif /* __INTEL_ENGINE_TYPES_H__ */ diff --git a/drivers/gpu/drm/i915/gt/intel_execlists_submission.c b/drivers/gpu/drm/i915/gt/intel_execlists_submission.c index de5f9c86b9a4..bedb80057046 100644 --- a/drivers/gpu/drm/i915/gt/intel_execlists_submission.c +++ b/drivers/gpu/drm/i915/gt/intel_execlists_submission.c @@ -201,7 +201,8 @@ static struct virtual_engine *to_virtual_engine(struct intel_engine_cs *engine) } static struct intel_context * -execlists_create_virtual(struct intel_engine_cs **siblings, unsigned int count); +execlists_create_virtual(struct intel_engine_cs **siblings, unsigned int count, + unsigned long flags); static struct i915_request * __active_request(const struct intel_timeline * const tl, @@ -2140,10 +2141,6 @@ static void __execlists_unhold(struct i915_request *rq) if (p->flags & I915_DEPENDENCY_WEAK) continue; - /* Propagate any change in error status */ - if (rq->fence.error) - i915_request_set_error_once(w, rq->fence.error); - if (w->engine != rq->engine) continue; @@ -2565,7 +2562,7 @@ __execlists_context_pre_pin(struct intel_context *ce, if (!__test_and_set_bit(CONTEXT_INIT_BIT, &ce->flags)) { lrc_init_state(ce, engine, *vaddr); - __i915_gem_object_flush_map(ce->state->obj, 0, engine->context_size); + __i915_gem_object_flush_map(ce->state->obj, 0, engine->context_size); } return 0; @@ -2791,6 +2788,8 @@ static void execlists_sanitize(struct intel_engine_cs *engine) /* And scrub the dirty cachelines for the HWSP */ clflush_cache_range(engine->status_page.addr, PAGE_SIZE); + + intel_engine_reset_pinned_contexts(engine); } static void enable_error_interrupt(struct intel_engine_cs *engine) @@ -3341,7 +3340,7 @@ logical_ring_default_vfuncs(struct intel_engine_cs *engine) engine->flags |= I915_ENGINE_HAS_SEMAPHORES; if (can_preempt(engine)) { engine->flags |= I915_ENGINE_HAS_PREEMPTION; - if (IS_ACTIVE(CONFIG_DRM_I915_TIMESLICE_DURATION)) + if (CONFIG_DRM_I915_TIMESLICE_DURATION) engine->flags |= I915_ENGINE_HAS_TIMESLICES; } } @@ -3786,7 +3785,8 @@ unlock: } static struct intel_context * -execlists_create_virtual(struct intel_engine_cs **siblings, unsigned int count) +execlists_create_virtual(struct intel_engine_cs **siblings, unsigned int count, + unsigned long flags) { struct virtual_engine *ve; unsigned int n; @@ -3879,6 +3879,7 @@ execlists_create_virtual(struct intel_engine_cs **siblings, unsigned int count) ve->siblings[ve->num_siblings++] = sibling; ve->base.mask |= sibling->mask; + ve->base.logical_mask |= sibling->logical_mask; /* * All physical engines must be compatible for their emission diff --git a/drivers/gpu/drm/i915/gt/intel_ggtt.c b/drivers/gpu/drm/i915/gt/intel_ggtt.c index de3ac58fceec..f17383e76eb7 100644 --- a/drivers/gpu/drm/i915/gt/intel_ggtt.c +++ b/drivers/gpu/drm/i915/gt/intel_ggtt.c @@ -644,7 +644,7 @@ static int init_aliasing_ppgtt(struct i915_ggtt *ggtt) struct i915_ppgtt *ppgtt; int err; - ppgtt = i915_ppgtt_create(ggtt->vm.gt); + ppgtt = i915_ppgtt_create(ggtt->vm.gt, 0); if (IS_ERR(ppgtt)) return PTR_ERR(ppgtt); @@ -727,7 +727,6 @@ static void ggtt_cleanup_hw(struct i915_ggtt *ggtt) atomic_set(&ggtt->vm.open, 0); - rcu_barrier(); /* flush the RCU'ed__i915_vm_release */ flush_workqueue(ggtt->vm.i915->wq); mutex_lock(&ggtt->vm.mutex); @@ -814,6 +813,21 @@ static unsigned int chv_get_total_gtt_size(u16 gmch_ctrl) return 0; } +static unsigned int gen6_gttmmadr_size(struct drm_i915_private *i915) +{ + /* + * GEN6: GTTMMADR size is 4MB and GTTADR starts at 2MB offset + * GEN8: GTTMMADR size is 16MB and GTTADR starts at 8MB offset + */ + GEM_BUG_ON(GRAPHICS_VER(i915) < 6); + return (GRAPHICS_VER(i915) < 8) ? SZ_4M : SZ_16M; +} + +static unsigned int gen6_gttadr_offset(struct drm_i915_private *i915) +{ + return gen6_gttmmadr_size(i915) / 2; +} + static int ggtt_probe_common(struct i915_ggtt *ggtt, u64 size) { struct drm_i915_private *i915 = ggtt->vm.i915; @@ -822,8 +836,8 @@ static int ggtt_probe_common(struct i915_ggtt *ggtt, u64 size) u32 pte_flags; int ret; - /* For Modern GENs the PTEs and register space are split in the BAR */ - phys_addr = pci_resource_start(pdev, 0) + pci_resource_len(pdev, 0) / 2; + GEM_WARN_ON(pci_resource_len(pdev, 0) != gen6_gttmmadr_size(i915)); + phys_addr = pci_resource_start(pdev, 0) + gen6_gttadr_offset(i915); /* * On BXT+/ICL+ writes larger than 64 bit to the GTT pagetable range @@ -910,6 +924,7 @@ static int gen8_gmch_probe(struct i915_ggtt *ggtt) size = gen8_get_total_gtt_size(snb_gmch_ctl); ggtt->vm.alloc_pt_dma = alloc_pt_dma; + ggtt->vm.lmem_pt_obj_flags = I915_BO_ALLOC_PM_EARLY; ggtt->vm.total = (size / sizeof(gen8_pte_t)) * I915_GTT_PAGE_SIZE; ggtt->vm.cleanup = gen6_gmch_remove; @@ -1373,13 +1388,28 @@ err_st_alloc: } static struct scatterlist * -remap_pages(struct drm_i915_gem_object *obj, unsigned int offset, +remap_pages(struct drm_i915_gem_object *obj, + unsigned int offset, unsigned int alignment_pad, unsigned int width, unsigned int height, unsigned int src_stride, unsigned int dst_stride, struct sg_table *st, struct scatterlist *sg) { unsigned int row; + if (alignment_pad) { + st->nents++; + + /* + * The DE ignores the PTEs for the padding tiles, the sg entry + * here is just a convenience to indicate how many padding PTEs + * to insert at this spot. + */ + sg_set_page(sg, NULL, alignment_pad * 4096, 0); + sg_dma_address(sg) = 0; + sg_dma_len(sg) = alignment_pad * 4096; + sg = sg_next(sg); + } + for (row = 0; row < height; row++) { unsigned int left = width * I915_GTT_PAGE_SIZE; @@ -1439,6 +1469,7 @@ intel_remap_pages(struct intel_remapped_info *rem_info, struct drm_i915_private *i915 = to_i915(obj->base.dev); struct sg_table *st; struct scatterlist *sg; + unsigned int gtt_offset = 0; int ret = -ENOMEM; int i; @@ -1455,10 +1486,19 @@ intel_remap_pages(struct intel_remapped_info *rem_info, sg = st->sgl; for (i = 0 ; i < ARRAY_SIZE(rem_info->plane); i++) { - sg = remap_pages(obj, rem_info->plane[i].offset, + unsigned int alignment_pad = 0; + + if (rem_info->plane_alignment) + alignment_pad = ALIGN(gtt_offset, rem_info->plane_alignment) - gtt_offset; + + sg = remap_pages(obj, + rem_info->plane[i].offset, alignment_pad, rem_info->plane[i].width, rem_info->plane[i].height, rem_info->plane[i].src_stride, rem_info->plane[i].dst_stride, st, sg); + + gtt_offset += alignment_pad + + rem_info->plane[i].dst_stride * rem_info->plane[i].height; } i915_sg_trim(st); diff --git a/drivers/gpu/drm/i915/gt/intel_gpu_commands.h b/drivers/gpu/drm/i915/gt/intel_gpu_commands.h index 1c3af0fc0456..f8253012d166 100644 --- a/drivers/gpu/drm/i915/gt/intel_gpu_commands.h +++ b/drivers/gpu/drm/i915/gt/intel_gpu_commands.h @@ -28,10 +28,13 @@ #define INSTR_26_TO_24_MASK 0x7000000 #define INSTR_26_TO_24_SHIFT 24 +#define __INSTR(client) ((client) << INSTR_CLIENT_SHIFT) + /* * Memory interface instructions used by the kernel */ -#define MI_INSTR(opcode, flags) (((opcode) << 23) | (flags)) +#define MI_INSTR(opcode, flags) \ + (__INSTR(INSTR_MI_CLIENT) | (opcode) << 23 | (flags)) /* Many MI commands use bit 22 of the header dword for GGTT vs PPGTT */ #define MI_GLOBAL_GTT (1<<22) @@ -57,6 +60,7 @@ #define MI_SUSPEND_FLUSH MI_INSTR(0x0b, 0) #define MI_SUSPEND_FLUSH_EN (1<<0) #define MI_SET_APPID MI_INSTR(0x0e, 0) +#define MI_SET_APPID_SESSION_ID(x) ((x) << 0) #define MI_OVERLAY_FLIP MI_INSTR(0x11, 0) #define MI_OVERLAY_CONTINUE (0x0<<21) #define MI_OVERLAY_ON (0x1<<21) @@ -146,6 +150,7 @@ #define MI_STORE_REGISTER_MEM_GEN8 MI_INSTR(0x24, 2) #define MI_SRM_LRM_GLOBAL_GTT (1<<22) #define MI_FLUSH_DW MI_INSTR(0x26, 1) /* for GEN6 */ +#define MI_FLUSH_DW_PROTECTED_MEM_EN (1 << 22) #define MI_FLUSH_DW_STORE_INDEX (1<<21) #define MI_INVALIDATE_TLB (1<<18) #define MI_FLUSH_DW_OP_STOREDW (1<<14) @@ -273,6 +278,19 @@ #define MI_MATH_REG_CF 0x33 /* + * Media instructions used by the kernel + */ +#define MEDIA_INSTR(pipe, op, sub_op, flags) \ + (__INSTR(INSTR_RC_CLIENT) | (pipe) << INSTR_SUBCLIENT_SHIFT | \ + (op) << INSTR_26_TO_24_SHIFT | (sub_op) << 16 | (flags)) + +#define MFX_WAIT MEDIA_INSTR(1, 0, 0, 0) +#define MFX_WAIT_DW0_MFX_SYNC_CONTROL_FLAG REG_BIT(8) +#define MFX_WAIT_DW0_PXP_SYNC_CONTROL_FLAG REG_BIT(9) + +#define CRYPTO_KEY_EXCHANGE MEDIA_INSTR(2, 6, 9, 0) + +/* * Commands used only by the command parser */ #define MI_SET_PREDICATE MI_INSTR(0x01, 0) @@ -328,8 +346,6 @@ #define GFX_OP_3DSTATE_BINDING_TABLE_EDIT_PS \ ((0x3<<29)|(0x3<<27)|(0x0<<24)|(0x47<<16)) -#define MFX_WAIT ((0x3<<29)|(0x1<<27)|(0x0<<16)) - #define COLOR_BLT ((0x2<<29)|(0x40<<22)) #define SRC_COPY_BLT ((0x2<<29)|(0x43<<22)) diff --git a/drivers/gpu/drm/i915/gt/intel_gt.c b/drivers/gpu/drm/i915/gt/intel_gt.c index 62d40c986642..1cb1948ac959 100644 --- a/drivers/gpu/drm/i915/gt/intel_gt.c +++ b/drivers/gpu/drm/i915/gt/intel_gt.c @@ -3,7 +3,7 @@ * Copyright © 2019 Intel Corporation */ -#include "debugfs_gt.h" +#include "intel_gt_debugfs.h" #include "gem/i915_gem_lmem.h" #include "i915_drv.h" @@ -15,12 +15,13 @@ #include "intel_gt_requests.h" #include "intel_migrate.h" #include "intel_mocs.h" +#include "intel_pm.h" #include "intel_rc6.h" #include "intel_renderstate.h" #include "intel_rps.h" #include "intel_uncore.h" -#include "intel_pm.h" #include "shmem_utils.h" +#include "pxp/intel_pxp.h" void intel_gt_init_early(struct intel_gt *gt, struct drm_i915_private *i915) { @@ -434,7 +435,7 @@ void intel_gt_driver_register(struct intel_gt *gt) { intel_rps_driver_register(>->rps); - debugfs_gt_register(gt); + intel_gt_debugfs_register(gt); } static int intel_gt_init_scratch(struct intel_gt *gt, unsigned int size) @@ -481,7 +482,7 @@ static void intel_gt_fini_scratch(struct intel_gt *gt) static struct i915_address_space *kernel_vm(struct intel_gt *gt) { if (INTEL_PPGTT(gt->i915) > INTEL_PPGTT_ALIASING) - return &i915_ppgtt_create(gt)->vm; + return &i915_ppgtt_create(gt, I915_BO_ALLOC_PM_EARLY)->vm; else return i915_vm_get(>->ggtt->vm); } @@ -660,6 +661,8 @@ int intel_gt_init(struct intel_gt *gt) if (err) return err; + intel_gt_init_workarounds(gt); + /* * This is just a security blanket to placate dragons. * On some systems, we very sporadically observe that the first TLBs @@ -682,6 +685,8 @@ int intel_gt_init(struct intel_gt *gt) goto err_pm; } + intel_set_mocs_index(gt); + err = intel_engines_init(gt); if (err) goto err_engines; @@ -710,6 +715,8 @@ int intel_gt_init(struct intel_gt *gt) intel_migrate_init(>->migrate, gt); + intel_pxp_init(>->pxp); + goto out_fw; err_gt: __intel_gt_disable(gt); @@ -737,6 +744,8 @@ void intel_gt_driver_remove(struct intel_gt *gt) intel_uc_driver_remove(>->uc); intel_engines_release(gt); + + intel_gt_flush_buffer_pool(gt); } void intel_gt_driver_unregister(struct intel_gt *gt) @@ -745,12 +754,14 @@ void intel_gt_driver_unregister(struct intel_gt *gt) intel_rps_driver_unregister(>->rps); + intel_pxp_fini(>->pxp); + /* * Upon unregistering the device to prevent any new users, cancel * all in-flight requests so that we can quickly unbind the active * resources. */ - intel_gt_set_wedged(gt); + intel_gt_set_wedged_on_fini(gt); /* Scrub all HW state upon release */ with_intel_runtime_pm(gt->uncore->rpm, wakeref) @@ -765,6 +776,7 @@ void intel_gt_driver_release(struct intel_gt *gt) if (vm) /* FIXME being called twice on error paths :( */ i915_vm_put(vm); + intel_wa_list_free(>->wa_list); intel_gt_pm_fini(gt); intel_gt_fini_scratch(gt); intel_gt_fini_buffer_pool(gt); diff --git a/drivers/gpu/drm/i915/gt/intel_gt_buffer_pool.c b/drivers/gpu/drm/i915/gt/intel_gt_buffer_pool.c index aa0a59c5b614..acc49c56a9f3 100644 --- a/drivers/gpu/drm/i915/gt/intel_gt_buffer_pool.c +++ b/drivers/gpu/drm/i915/gt/intel_gt_buffer_pool.c @@ -245,8 +245,6 @@ void intel_gt_fini_buffer_pool(struct intel_gt *gt) struct intel_gt_buffer_pool *pool = >->buffer_pool; int n; - intel_gt_flush_buffer_pool(gt); - for (n = 0; n < ARRAY_SIZE(pool->cache_list); n++) GEM_BUG_ON(!list_empty(&pool->cache_list[n])); } diff --git a/drivers/gpu/drm/i915/gt/intel_gt_debugfs.c b/drivers/gpu/drm/i915/gt/intel_gt_debugfs.c new file mode 100644 index 000000000000..f103664b71d4 --- /dev/null +++ b/drivers/gpu/drm/i915/gt/intel_gt_debugfs.c @@ -0,0 +1,104 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2019 Intel Corporation + */ + +#include <linux/debugfs.h> + +#include "i915_drv.h" +#include "intel_gt_debugfs.h" +#include "intel_gt_engines_debugfs.h" +#include "intel_gt_pm_debugfs.h" +#include "intel_sseu_debugfs.h" +#include "pxp/intel_pxp_debugfs.h" +#include "uc/intel_uc_debugfs.h" + +int intel_gt_debugfs_reset_show(struct intel_gt *gt, u64 *val) +{ + int ret = intel_gt_terminally_wedged(gt); + + switch (ret) { + case -EIO: + *val = 1; + return 0; + case 0: + *val = 0; + return 0; + default: + return ret; + } +} + +int intel_gt_debugfs_reset_store(struct intel_gt *gt, u64 val) +{ + /* Flush any previous reset before applying for a new one */ + wait_event(gt->reset.queue, + !test_bit(I915_RESET_BACKOFF, >->reset.flags)); + + intel_gt_handle_error(gt, val, I915_ERROR_CAPTURE, + "Manually reset engine mask to %llx", val); + return 0; +} + +/* + * keep the interface clean where the first parameter + * is a 'struct intel_gt *' instead of 'void *' + */ +static int __intel_gt_debugfs_reset_show(void *data, u64 *val) +{ + return intel_gt_debugfs_reset_show(data, val); +} + +static int __intel_gt_debugfs_reset_store(void *data, u64 val) +{ + return intel_gt_debugfs_reset_store(data, val); +} + +DEFINE_SIMPLE_ATTRIBUTE(reset_fops, __intel_gt_debugfs_reset_show, + __intel_gt_debugfs_reset_store, "%llu\n"); + +static void gt_debugfs_register(struct intel_gt *gt, struct dentry *root) +{ + static const struct intel_gt_debugfs_file files[] = { + { "reset", &reset_fops, NULL }, + }; + + intel_gt_debugfs_register_files(root, files, ARRAY_SIZE(files), gt); +} + +void intel_gt_debugfs_register(struct intel_gt *gt) +{ + struct dentry *root; + + if (!gt->i915->drm.primary->debugfs_root) + return; + + root = debugfs_create_dir("gt", gt->i915->drm.primary->debugfs_root); + if (IS_ERR(root)) + return; + + gt_debugfs_register(gt, root); + + intel_gt_engines_debugfs_register(gt, root); + intel_gt_pm_debugfs_register(gt, root); + intel_sseu_debugfs_register(gt, root); + + intel_uc_debugfs_register(>->uc, root); + intel_pxp_debugfs_register(>->pxp, root); +} + +void intel_gt_debugfs_register_files(struct dentry *root, + const struct intel_gt_debugfs_file *files, + unsigned long count, void *data) +{ + while (count--) { + umode_t mode = files->fops->write ? 0644 : 0444; + + if (!files->eval || files->eval(data)) + debugfs_create_file(files->name, + mode, root, data, + files->fops); + + files++; + } +} diff --git a/drivers/gpu/drm/i915/gt/debugfs_gt.h b/drivers/gpu/drm/i915/gt/intel_gt_debugfs.h index f77540f727e9..e307ceb99031 100644 --- a/drivers/gpu/drm/i915/gt/debugfs_gt.h +++ b/drivers/gpu/drm/i915/gt/intel_gt_debugfs.h @@ -3,14 +3,14 @@ * Copyright © 2019 Intel Corporation */ -#ifndef DEBUGFS_GT_H -#define DEBUGFS_GT_H +#ifndef INTEL_GT_DEBUGFS_H +#define INTEL_GT_DEBUGFS_H #include <linux/file.h> struct intel_gt; -#define DEFINE_GT_DEBUGFS_ATTRIBUTE(__name) \ +#define DEFINE_INTEL_GT_DEBUGFS_ATTRIBUTE(__name) \ static int __name ## _open(struct inode *inode, struct file *file) \ { \ return single_open(file, __name ## _show, inode->i_private); \ @@ -23,16 +23,20 @@ static const struct file_operations __name ## _fops = { \ .release = single_release, \ } -void debugfs_gt_register(struct intel_gt *gt); +void intel_gt_debugfs_register(struct intel_gt *gt); -struct debugfs_gt_file { +struct intel_gt_debugfs_file { const char *name; const struct file_operations *fops; bool (*eval)(void *data); }; void intel_gt_debugfs_register_files(struct dentry *root, - const struct debugfs_gt_file *files, + const struct intel_gt_debugfs_file *files, unsigned long count, void *data); -#endif /* DEBUGFS_GT_H */ +/* functions that need to be accessed by the upper level non-gt interfaces */ +int intel_gt_debugfs_reset_show(struct intel_gt *gt, u64 *val); +int intel_gt_debugfs_reset_store(struct intel_gt *gt, u64 val); + +#endif /* INTEL_GT_DEBUGFS_H */ diff --git a/drivers/gpu/drm/i915/gt/debugfs_engines.c b/drivers/gpu/drm/i915/gt/intel_gt_engines_debugfs.c index 5e3725e62241..8f9b874fdc9c 100644 --- a/drivers/gpu/drm/i915/gt/debugfs_engines.c +++ b/drivers/gpu/drm/i915/gt/intel_gt_engines_debugfs.c @@ -6,10 +6,10 @@ #include <drm/drm_print.h> -#include "debugfs_engines.h" -#include "debugfs_gt.h" #include "i915_drv.h" /* for_each_engine! */ #include "intel_engine.h" +#include "intel_gt_debugfs.h" +#include "intel_gt_engines_debugfs.h" static int engines_show(struct seq_file *m, void *data) { @@ -24,11 +24,11 @@ static int engines_show(struct seq_file *m, void *data) return 0; } -DEFINE_GT_DEBUGFS_ATTRIBUTE(engines); +DEFINE_INTEL_GT_DEBUGFS_ATTRIBUTE(engines); -void debugfs_engines_register(struct intel_gt *gt, struct dentry *root) +void intel_gt_engines_debugfs_register(struct intel_gt *gt, struct dentry *root) { - static const struct debugfs_gt_file files[] = { + static const struct intel_gt_debugfs_file files[] = { { "engines", &engines_fops }, }; diff --git a/drivers/gpu/drm/i915/gt/intel_gt_engines_debugfs.h b/drivers/gpu/drm/i915/gt/intel_gt_engines_debugfs.h new file mode 100644 index 000000000000..dda113452da9 --- /dev/null +++ b/drivers/gpu/drm/i915/gt/intel_gt_engines_debugfs.h @@ -0,0 +1,14 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2019 Intel Corporation + */ + +#ifndef INTEL_GT_ENGINES_DEBUGFS_H +#define INTEL_GT_ENGINES_DEBUGFS_H + +struct intel_gt; +struct dentry; + +void intel_gt_engines_debugfs_register(struct intel_gt *gt, struct dentry *root); + +#endif /* INTEL_GT_ENGINES_DEBUGFS_H */ diff --git a/drivers/gpu/drm/i915/gt/intel_gt_irq.c b/drivers/gpu/drm/i915/gt/intel_gt_irq.c index b2de83be4d97..699a74582d32 100644 --- a/drivers/gpu/drm/i915/gt/intel_gt_irq.c +++ b/drivers/gpu/drm/i915/gt/intel_gt_irq.c @@ -13,6 +13,7 @@ #include "intel_lrc_reg.h" #include "intel_uncore.h" #include "intel_rps.h" +#include "pxp/intel_pxp_irq.h" static void guc_irq_handler(struct intel_guc *guc, u16 iir) { @@ -64,6 +65,9 @@ gen11_other_irq_handler(struct intel_gt *gt, const u8 instance, if (instance == OTHER_GTPM_INSTANCE) return gen11_rps_irq_handler(>->rps, iir); + if (instance == OTHER_KCR_INSTANCE) + return intel_pxp_irq_handler(>->pxp, iir); + WARN_ONCE(1, "unhandled other interrupt instance=0x%x, iir=0x%x\n", instance, iir); } @@ -196,6 +200,9 @@ void gen11_gt_irq_reset(struct intel_gt *gt) intel_uncore_write(uncore, GEN11_GPM_WGBOXPERF_INTR_MASK, ~0); intel_uncore_write(uncore, GEN11_GUC_SG_INTR_ENABLE, 0); intel_uncore_write(uncore, GEN11_GUC_SG_INTR_MASK, ~0); + + intel_uncore_write(uncore, GEN11_CRYPTO_RSVD_INTR_ENABLE, 0); + intel_uncore_write(uncore, GEN11_CRYPTO_RSVD_INTR_MASK, ~0); } void gen11_gt_irq_postinstall(struct intel_gt *gt) diff --git a/drivers/gpu/drm/i915/gt/intel_gt_pm.c b/drivers/gpu/drm/i915/gt/intel_gt_pm.c index dea8e2479897..524eaf678790 100644 --- a/drivers/gpu/drm/i915/gt/intel_gt_pm.c +++ b/drivers/gpu/drm/i915/gt/intel_gt_pm.c @@ -18,6 +18,9 @@ #include "intel_rc6.h" #include "intel_rps.h" #include "intel_wakeref.h" +#include "pxp/intel_pxp_pm.h" + +#define I915_GT_SUSPEND_IDLE_TIMEOUT (HZ / 2) static void user_forcewake(struct intel_gt *gt, bool suspend) { @@ -262,6 +265,8 @@ int intel_gt_resume(struct intel_gt *gt) intel_uc_resume(>->uc); + intel_pxp_resume(>->pxp); + user_forcewake(gt, false); out_fw: @@ -279,7 +284,7 @@ static void wait_for_suspend(struct intel_gt *gt) if (!intel_gt_pm_is_awake(gt)) return; - if (intel_gt_wait_for_idle(gt, I915_GEM_IDLE_TIMEOUT) == -ETIME) { + if (intel_gt_wait_for_idle(gt, I915_GT_SUSPEND_IDLE_TIMEOUT) == -ETIME) { /* * Forcibly cancel outstanding work and leave * the gpu quiet. @@ -296,7 +301,7 @@ void intel_gt_suspend_prepare(struct intel_gt *gt) user_forcewake(gt, true); wait_for_suspend(gt); - intel_uc_suspend(>->uc); + intel_pxp_suspend(>->pxp, false); } static suspend_state_t pm_suspend_target(void) @@ -320,6 +325,8 @@ void intel_gt_suspend_late(struct intel_gt *gt) GEM_BUG_ON(gt->awake); + intel_uc_suspend(>->uc); + /* * On disabling the device, we want to turn off HW access to memory * that we no longer own. @@ -346,6 +353,7 @@ void intel_gt_suspend_late(struct intel_gt *gt) void intel_gt_runtime_suspend(struct intel_gt *gt) { + intel_pxp_suspend(>->pxp, true); intel_uc_runtime_suspend(>->uc); GT_TRACE(gt, "\n"); @@ -353,11 +361,19 @@ void intel_gt_runtime_suspend(struct intel_gt *gt) int intel_gt_runtime_resume(struct intel_gt *gt) { + int ret; + GT_TRACE(gt, "\n"); intel_gt_init_swizzling(gt); intel_ggtt_restore_fences(gt->ggtt); - return intel_uc_runtime_resume(>->uc); + ret = intel_uc_runtime_resume(>->uc); + if (ret) + return ret; + + intel_pxp_resume(>->pxp); + + return 0; } static ktime_t __intel_gt_get_awake_time(const struct intel_gt *gt) diff --git a/drivers/gpu/drm/i915/gt/intel_gt_pm.h b/drivers/gpu/drm/i915/gt/intel_gt_pm.h index d0588d8aaa44..bc898df7a48c 100644 --- a/drivers/gpu/drm/i915/gt/intel_gt_pm.h +++ b/drivers/gpu/drm/i915/gt/intel_gt_pm.h @@ -31,6 +31,11 @@ static inline bool intel_gt_pm_get_if_awake(struct intel_gt *gt) return intel_wakeref_get_if_active(>->wakeref); } +static inline void intel_gt_pm_might_get(struct intel_gt *gt) +{ + intel_wakeref_might_get(>->wakeref); +} + static inline void intel_gt_pm_put(struct intel_gt *gt) { intel_wakeref_put(>->wakeref); @@ -41,6 +46,15 @@ static inline void intel_gt_pm_put_async(struct intel_gt *gt) intel_wakeref_put_async(>->wakeref); } +static inline void intel_gt_pm_might_put(struct intel_gt *gt) +{ + intel_wakeref_might_put(>->wakeref); +} + +#define with_intel_gt_pm(gt, tmp) \ + for (tmp = 1, intel_gt_pm_get(gt); tmp; \ + intel_gt_pm_put(gt), tmp = 0) + static inline int intel_gt_pm_wait_for_idle(struct intel_gt *gt) { return intel_wakeref_wait_for_idle(>->wakeref); diff --git a/drivers/gpu/drm/i915/gt/debugfs_gt_pm.c b/drivers/gpu/drm/i915/gt/intel_gt_pm_debugfs.c index d6f5836396f8..404dfa7673c6 100644 --- a/drivers/gpu/drm/i915/gt/debugfs_gt_pm.c +++ b/drivers/gpu/drm/i915/gt/intel_gt_pm_debugfs.c @@ -6,18 +6,59 @@ #include <linux/seq_file.h> -#include "debugfs_gt.h" -#include "debugfs_gt_pm.h" #include "i915_drv.h" #include "intel_gt.h" #include "intel_gt_clock_utils.h" +#include "intel_gt_debugfs.h" #include "intel_gt_pm.h" +#include "intel_gt_pm_debugfs.h" #include "intel_llc.h" +#include "intel_pcode.h" #include "intel_rc6.h" #include "intel_rps.h" #include "intel_runtime_pm.h" -#include "intel_sideband.h" #include "intel_uncore.h" +#include "vlv_sideband.h" + +int intel_gt_pm_debugfs_forcewake_user_open(struct intel_gt *gt) +{ + atomic_inc(>->user_wakeref); + intel_gt_pm_get(gt); + if (GRAPHICS_VER(gt->i915) >= 6) + intel_uncore_forcewake_user_get(gt->uncore); + + return 0; +} + +int intel_gt_pm_debugfs_forcewake_user_release(struct intel_gt *gt) +{ + if (GRAPHICS_VER(gt->i915) >= 6) + intel_uncore_forcewake_user_put(gt->uncore); + intel_gt_pm_put(gt); + atomic_dec(>->user_wakeref); + + return 0; +} + +static int forcewake_user_open(struct inode *inode, struct file *file) +{ + struct intel_gt *gt = inode->i_private; + + return intel_gt_pm_debugfs_forcewake_user_open(gt); +} + +static int forcewake_user_release(struct inode *inode, struct file *file) +{ + struct intel_gt *gt = inode->i_private; + + return intel_gt_pm_debugfs_forcewake_user_release(gt); +} + +static const struct file_operations forcewake_user_fops = { + .owner = THIS_MODULE, + .open = forcewake_user_open, + .release = forcewake_user_release, +}; static int fw_domains_show(struct seq_file *m, void *data) { @@ -36,7 +77,7 @@ static int fw_domains_show(struct seq_file *m, void *data) return 0; } -DEFINE_GT_DEBUGFS_ATTRIBUTE(fw_domains); +DEFINE_INTEL_GT_DEBUGFS_ATTRIBUTE(fw_domains); static void print_rc6_res(struct seq_file *m, const char *title, @@ -238,11 +279,10 @@ static int drpc_show(struct seq_file *m, void *unused) return err; } -DEFINE_GT_DEBUGFS_ATTRIBUTE(drpc); +DEFINE_INTEL_GT_DEBUGFS_ATTRIBUTE(drpc); -static int frequency_show(struct seq_file *m, void *unused) +void intel_gt_pm_frequency_dump(struct intel_gt *gt, struct drm_printer *p) { - struct intel_gt *gt = m->private; struct drm_i915_private *i915 = gt->i915; struct intel_uncore *uncore = gt->uncore; struct intel_rps *rps = >->rps; @@ -254,21 +294,21 @@ static int frequency_show(struct seq_file *m, void *unused) u16 rgvswctl = intel_uncore_read16(uncore, MEMSWCTL); u16 rgvstat = intel_uncore_read16(uncore, MEMSTAT_ILK); - seq_printf(m, "Requested P-state: %d\n", (rgvswctl >> 8) & 0xf); - seq_printf(m, "Requested VID: %d\n", rgvswctl & 0x3f); - seq_printf(m, "Current VID: %d\n", (rgvstat & MEMSTAT_VID_MASK) >> + drm_printf(p, "Requested P-state: %d\n", (rgvswctl >> 8) & 0xf); + drm_printf(p, "Requested VID: %d\n", rgvswctl & 0x3f); + drm_printf(p, "Current VID: %d\n", (rgvstat & MEMSTAT_VID_MASK) >> MEMSTAT_VID_SHIFT); - seq_printf(m, "Current P-state: %d\n", + drm_printf(p, "Current P-state: %d\n", (rgvstat & MEMSTAT_PSTATE_MASK) >> MEMSTAT_PSTATE_SHIFT); } else if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915)) { u32 rpmodectl, freq_sts; rpmodectl = intel_uncore_read(uncore, GEN6_RP_CONTROL); - seq_printf(m, "Video Turbo Mode: %s\n", + drm_printf(p, "Video Turbo Mode: %s\n", yesno(rpmodectl & GEN6_RP_MEDIA_TURBO)); - seq_printf(m, "HW control enabled: %s\n", + drm_printf(p, "HW control enabled: %s\n", yesno(rpmodectl & GEN6_RP_ENABLE)); - seq_printf(m, "SW control enabled: %s\n", + drm_printf(p, "SW control enabled: %s\n", yesno((rpmodectl & GEN6_RP_MEDIA_MODE_MASK) == GEN6_RP_MEDIA_SW_MODE)); @@ -276,25 +316,25 @@ static int frequency_show(struct seq_file *m, void *unused) freq_sts = vlv_punit_read(i915, PUNIT_REG_GPU_FREQ_STS); vlv_punit_put(i915); - seq_printf(m, "PUNIT_REG_GPU_FREQ_STS: 0x%08x\n", freq_sts); - seq_printf(m, "DDR freq: %d MHz\n", i915->mem_freq); + drm_printf(p, "PUNIT_REG_GPU_FREQ_STS: 0x%08x\n", freq_sts); + drm_printf(p, "DDR freq: %d MHz\n", i915->mem_freq); - seq_printf(m, "actual GPU freq: %d MHz\n", + drm_printf(p, "actual GPU freq: %d MHz\n", intel_gpu_freq(rps, (freq_sts >> 8) & 0xff)); - seq_printf(m, "current GPU freq: %d MHz\n", + drm_printf(p, "current GPU freq: %d MHz\n", intel_gpu_freq(rps, rps->cur_freq)); - seq_printf(m, "max GPU freq: %d MHz\n", + drm_printf(p, "max GPU freq: %d MHz\n", intel_gpu_freq(rps, rps->max_freq)); - seq_printf(m, "min GPU freq: %d MHz\n", + drm_printf(p, "min GPU freq: %d MHz\n", intel_gpu_freq(rps, rps->min_freq)); - seq_printf(m, "idle GPU freq: %d MHz\n", + drm_printf(p, "idle GPU freq: %d MHz\n", intel_gpu_freq(rps, rps->idle_freq)); - seq_printf(m, "efficient (RPe) frequency: %d MHz\n", + drm_printf(p, "efficient (RPe) frequency: %d MHz\n", intel_gpu_freq(rps, rps->efficient_freq)); } else if (GRAPHICS_VER(i915) >= 6) { u32 rp_state_limits; @@ -309,13 +349,11 @@ static int frequency_show(struct seq_file *m, void *unused) int max_freq; rp_state_limits = intel_uncore_read(uncore, GEN6_RP_STATE_LIMITS); - if (IS_GEN9_LP(i915)) { - rp_state_cap = intel_uncore_read(uncore, BXT_RP_STATE_CAP); + rp_state_cap = intel_rps_read_state_cap(rps); + if (IS_GEN9_LP(i915)) gt_perf_status = intel_uncore_read(uncore, BXT_GT_PERF_STATUS); - } else { - rp_state_cap = intel_uncore_read(uncore, GEN6_RP_STATE_CAP); + else gt_perf_status = intel_uncore_read(uncore, GEN6_GT_PERF_STATUS); - } /* RPSTAT1 is in the GT power well */ intel_uncore_forcewake_get(uncore, FORCEWAKE_ALL); @@ -376,113 +414,121 @@ static int frequency_show(struct seq_file *m, void *unused) } pm_mask = intel_uncore_read(uncore, GEN6_PMINTRMSK); - seq_printf(m, "Video Turbo Mode: %s\n", + drm_printf(p, "Video Turbo Mode: %s\n", yesno(rpmodectl & GEN6_RP_MEDIA_TURBO)); - seq_printf(m, "HW control enabled: %s\n", + drm_printf(p, "HW control enabled: %s\n", yesno(rpmodectl & GEN6_RP_ENABLE)); - seq_printf(m, "SW control enabled: %s\n", + drm_printf(p, "SW control enabled: %s\n", yesno((rpmodectl & GEN6_RP_MEDIA_MODE_MASK) == GEN6_RP_MEDIA_SW_MODE)); - seq_printf(m, "PM IER=0x%08x IMR=0x%08x, MASK=0x%08x\n", + drm_printf(p, "PM IER=0x%08x IMR=0x%08x, MASK=0x%08x\n", pm_ier, pm_imr, pm_mask); if (GRAPHICS_VER(i915) <= 10) - seq_printf(m, "PM ISR=0x%08x IIR=0x%08x\n", + drm_printf(p, "PM ISR=0x%08x IIR=0x%08x\n", pm_isr, pm_iir); - seq_printf(m, "pm_intrmsk_mbz: 0x%08x\n", + drm_printf(p, "pm_intrmsk_mbz: 0x%08x\n", rps->pm_intrmsk_mbz); - seq_printf(m, "GT_PERF_STATUS: 0x%08x\n", gt_perf_status); - seq_printf(m, "Render p-state ratio: %d\n", + drm_printf(p, "GT_PERF_STATUS: 0x%08x\n", gt_perf_status); + drm_printf(p, "Render p-state ratio: %d\n", (gt_perf_status & (GRAPHICS_VER(i915) >= 9 ? 0x1ff00 : 0xff00)) >> 8); - seq_printf(m, "Render p-state VID: %d\n", + drm_printf(p, "Render p-state VID: %d\n", gt_perf_status & 0xff); - seq_printf(m, "Render p-state limit: %d\n", + drm_printf(p, "Render p-state limit: %d\n", rp_state_limits & 0xff); - seq_printf(m, "RPSTAT1: 0x%08x\n", rpstat); - seq_printf(m, "RPMODECTL: 0x%08x\n", rpmodectl); - seq_printf(m, "RPINCLIMIT: 0x%08x\n", rpinclimit); - seq_printf(m, "RPDECLIMIT: 0x%08x\n", rpdeclimit); - seq_printf(m, "RPNSWREQ: %dMHz\n", reqf); - seq_printf(m, "CAGF: %dMHz\n", cagf); - seq_printf(m, "RP CUR UP EI: %d (%lldns)\n", + drm_printf(p, "RPSTAT1: 0x%08x\n", rpstat); + drm_printf(p, "RPMODECTL: 0x%08x\n", rpmodectl); + drm_printf(p, "RPINCLIMIT: 0x%08x\n", rpinclimit); + drm_printf(p, "RPDECLIMIT: 0x%08x\n", rpdeclimit); + drm_printf(p, "RPNSWREQ: %dMHz\n", reqf); + drm_printf(p, "CAGF: %dMHz\n", cagf); + drm_printf(p, "RP CUR UP EI: %d (%lldns)\n", rpcurupei, intel_gt_pm_interval_to_ns(gt, rpcurupei)); - seq_printf(m, "RP CUR UP: %d (%lldns)\n", + drm_printf(p, "RP CUR UP: %d (%lldns)\n", rpcurup, intel_gt_pm_interval_to_ns(gt, rpcurup)); - seq_printf(m, "RP PREV UP: %d (%lldns)\n", + drm_printf(p, "RP PREV UP: %d (%lldns)\n", rpprevup, intel_gt_pm_interval_to_ns(gt, rpprevup)); - seq_printf(m, "Up threshold: %d%%\n", + drm_printf(p, "Up threshold: %d%%\n", rps->power.up_threshold); - seq_printf(m, "RP UP EI: %d (%lldns)\n", + drm_printf(p, "RP UP EI: %d (%lldns)\n", rpupei, intel_gt_pm_interval_to_ns(gt, rpupei)); - seq_printf(m, "RP UP THRESHOLD: %d (%lldns)\n", + drm_printf(p, "RP UP THRESHOLD: %d (%lldns)\n", rpupt, intel_gt_pm_interval_to_ns(gt, rpupt)); - seq_printf(m, "RP CUR DOWN EI: %d (%lldns)\n", + drm_printf(p, "RP CUR DOWN EI: %d (%lldns)\n", rpcurdownei, intel_gt_pm_interval_to_ns(gt, rpcurdownei)); - seq_printf(m, "RP CUR DOWN: %d (%lldns)\n", + drm_printf(p, "RP CUR DOWN: %d (%lldns)\n", rpcurdown, intel_gt_pm_interval_to_ns(gt, rpcurdown)); - seq_printf(m, "RP PREV DOWN: %d (%lldns)\n", + drm_printf(p, "RP PREV DOWN: %d (%lldns)\n", rpprevdown, intel_gt_pm_interval_to_ns(gt, rpprevdown)); - seq_printf(m, "Down threshold: %d%%\n", + drm_printf(p, "Down threshold: %d%%\n", rps->power.down_threshold); - seq_printf(m, "RP DOWN EI: %d (%lldns)\n", + drm_printf(p, "RP DOWN EI: %d (%lldns)\n", rpdownei, intel_gt_pm_interval_to_ns(gt, rpdownei)); - seq_printf(m, "RP DOWN THRESHOLD: %d (%lldns)\n", + drm_printf(p, "RP DOWN THRESHOLD: %d (%lldns)\n", rpdownt, intel_gt_pm_interval_to_ns(gt, rpdownt)); max_freq = (IS_GEN9_LP(i915) ? rp_state_cap >> 0 : rp_state_cap >> 16) & 0xff; max_freq *= (IS_GEN9_BC(i915) || GRAPHICS_VER(i915) >= 11 ? GEN9_FREQ_SCALER : 1); - seq_printf(m, "Lowest (RPN) frequency: %dMHz\n", + drm_printf(p, "Lowest (RPN) frequency: %dMHz\n", intel_gpu_freq(rps, max_freq)); max_freq = (rp_state_cap & 0xff00) >> 8; max_freq *= (IS_GEN9_BC(i915) || GRAPHICS_VER(i915) >= 11 ? GEN9_FREQ_SCALER : 1); - seq_printf(m, "Nominal (RP1) frequency: %dMHz\n", + drm_printf(p, "Nominal (RP1) frequency: %dMHz\n", intel_gpu_freq(rps, max_freq)); max_freq = (IS_GEN9_LP(i915) ? rp_state_cap >> 16 : rp_state_cap >> 0) & 0xff; max_freq *= (IS_GEN9_BC(i915) || GRAPHICS_VER(i915) >= 11 ? GEN9_FREQ_SCALER : 1); - seq_printf(m, "Max non-overclocked (RP0) frequency: %dMHz\n", + drm_printf(p, "Max non-overclocked (RP0) frequency: %dMHz\n", intel_gpu_freq(rps, max_freq)); - seq_printf(m, "Max overclocked frequency: %dMHz\n", + drm_printf(p, "Max overclocked frequency: %dMHz\n", intel_gpu_freq(rps, rps->max_freq)); - seq_printf(m, "Current freq: %d MHz\n", + drm_printf(p, "Current freq: %d MHz\n", intel_gpu_freq(rps, rps->cur_freq)); - seq_printf(m, "Actual freq: %d MHz\n", cagf); - seq_printf(m, "Idle freq: %d MHz\n", + drm_printf(p, "Actual freq: %d MHz\n", cagf); + drm_printf(p, "Idle freq: %d MHz\n", intel_gpu_freq(rps, rps->idle_freq)); - seq_printf(m, "Min freq: %d MHz\n", + drm_printf(p, "Min freq: %d MHz\n", intel_gpu_freq(rps, rps->min_freq)); - seq_printf(m, "Boost freq: %d MHz\n", + drm_printf(p, "Boost freq: %d MHz\n", intel_gpu_freq(rps, rps->boost_freq)); - seq_printf(m, "Max freq: %d MHz\n", + drm_printf(p, "Max freq: %d MHz\n", intel_gpu_freq(rps, rps->max_freq)); - seq_printf(m, + drm_printf(p, "efficient (RPe) frequency: %d MHz\n", intel_gpu_freq(rps, rps->efficient_freq)); } else { - seq_puts(m, "no P-state info available\n"); + drm_puts(p, "no P-state info available\n"); } - seq_printf(m, "Current CD clock frequency: %d kHz\n", i915->cdclk.hw.cdclk); - seq_printf(m, "Max CD clock frequency: %d kHz\n", i915->max_cdclk_freq); - seq_printf(m, "Max pixel clock frequency: %d kHz\n", i915->max_dotclk_freq); + drm_printf(p, "Current CD clock frequency: %d kHz\n", i915->cdclk.hw.cdclk); + drm_printf(p, "Max CD clock frequency: %d kHz\n", i915->max_cdclk_freq); + drm_printf(p, "Max pixel clock frequency: %d kHz\n", i915->max_dotclk_freq); intel_runtime_pm_put(uncore->rpm, wakeref); +} + +static int frequency_show(struct seq_file *m, void *unused) +{ + struct intel_gt *gt = m->private; + struct drm_printer p = drm_seq_file_printer(m); + + intel_gt_pm_frequency_dump(gt, &p); return 0; } -DEFINE_GT_DEBUGFS_ATTRIBUTE(frequency); +DEFINE_INTEL_GT_DEBUGFS_ATTRIBUTE(frequency); static int llc_show(struct seq_file *m, void *data) { @@ -535,7 +581,7 @@ static bool llc_eval(void *data) return HAS_LLC(gt->i915); } -DEFINE_GT_DEBUGFS_ATTRIBUTE(llc); +DEFINE_INTEL_GT_DEBUGFS_ATTRIBUTE(llc); static const char *rps_power_to_str(unsigned int power) { @@ -614,14 +660,15 @@ static bool rps_eval(void *data) return HAS_RPS(gt->i915); } -DEFINE_GT_DEBUGFS_ATTRIBUTE(rps_boost); +DEFINE_INTEL_GT_DEBUGFS_ATTRIBUTE(rps_boost); -void debugfs_gt_pm_register(struct intel_gt *gt, struct dentry *root) +void intel_gt_pm_debugfs_register(struct intel_gt *gt, struct dentry *root) { - static const struct debugfs_gt_file files[] = { + static const struct intel_gt_debugfs_file files[] = { { "drpc", &drpc_fops, NULL }, { "frequency", &frequency_fops, NULL }, { "forcewake", &fw_domains_fops, NULL }, + { "forcewake_user", &forcewake_user_fops, NULL}, { "llc", &llc_fops, llc_eval }, { "rps_boost", &rps_boost_fops, rps_eval }, }; diff --git a/drivers/gpu/drm/i915/gt/intel_gt_pm_debugfs.h b/drivers/gpu/drm/i915/gt/intel_gt_pm_debugfs.h new file mode 100644 index 000000000000..a8457887ec65 --- /dev/null +++ b/drivers/gpu/drm/i915/gt/intel_gt_pm_debugfs.h @@ -0,0 +1,20 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2019 Intel Corporation + */ + +#ifndef INTEL_GT_PM_DEBUGFS_H +#define INTEL_GT_PM_DEBUGFS_H + +struct intel_gt; +struct dentry; +struct drm_printer; + +void intel_gt_pm_debugfs_register(struct intel_gt *gt, struct dentry *root); +void intel_gt_pm_frequency_dump(struct intel_gt *gt, struct drm_printer *m); + +/* functions that need to be accessed by the upper level non-gt interfaces */ +int intel_gt_pm_debugfs_forcewake_user_open(struct intel_gt *gt); +int intel_gt_pm_debugfs_forcewake_user_release(struct intel_gt *gt); + +#endif /* INTEL_GT_PM_DEBUGFS_H */ diff --git a/drivers/gpu/drm/i915/gt/intel_gt_types.h b/drivers/gpu/drm/i915/gt/intel_gt_types.h index a81e21bf1bd1..14216cc471b1 100644 --- a/drivers/gpu/drm/i915/gt/intel_gt_types.h +++ b/drivers/gpu/drm/i915/gt/intel_gt_types.h @@ -26,6 +26,7 @@ #include "intel_rps_types.h" #include "intel_migrate_types.h" #include "intel_wakeref.h" +#include "pxp/intel_pxp_types.h" struct drm_i915_private; struct i915_ggtt; @@ -72,6 +73,8 @@ struct intel_gt { struct intel_uc uc; + struct i915_wa_list wa_list; + struct intel_gt_timelines { spinlock_t lock; /* protects active_list */ struct list_head active_list; @@ -184,6 +187,9 @@ struct intel_gt { u8 num_engines; + /* General presence of SFC units */ + u8 sfc_mask; + /* Media engine access to SFC per instance */ u8 vdbox_sfc_access; @@ -192,6 +198,12 @@ struct intel_gt { unsigned long mslice_mask; } info; + + struct { + u8 uc_index; + } mocs; + + struct intel_pxp pxp; }; enum intel_gt_scratch_field { diff --git a/drivers/gpu/drm/i915/gt/intel_gtt.c b/drivers/gpu/drm/i915/gt/intel_gtt.c index e137dd32b5b8..67d14afa6623 100644 --- a/drivers/gpu/drm/i915/gt/intel_gtt.c +++ b/drivers/gpu/drm/i915/gt/intel_gtt.c @@ -28,7 +28,8 @@ struct drm_i915_gem_object *alloc_pt_lmem(struct i915_address_space *vm, int sz) * used the passed in size for the page size, which should ensure it * also has the same alignment. */ - obj = __i915_gem_object_create_lmem_with_ps(vm->i915, sz, sz, 0); + obj = __i915_gem_object_create_lmem_with_ps(vm->i915, sz, sz, + vm->lmem_pt_obj_flags); /* * Ensure all paging structures for this vm share the same dma-resv * object underneath, with the idea that one object_lock() will lock @@ -155,7 +156,7 @@ void i915_vm_resv_release(struct kref *kref) static void __i915_vm_release(struct work_struct *work) { struct i915_address_space *vm = - container_of(work, struct i915_address_space, rcu.work); + container_of(work, struct i915_address_space, release_work); vm->cleanup(vm); i915_address_space_fini(vm); @@ -171,7 +172,7 @@ void i915_vm_release(struct kref *kref) GEM_BUG_ON(i915_is_ggtt(vm)); trace_i915_ppgtt_release(vm); - queue_rcu_work(vm->i915->wq, &vm->rcu); + queue_work(vm->i915->wq, &vm->release_work); } void i915_address_space_init(struct i915_address_space *vm, int subclass) @@ -185,7 +186,7 @@ void i915_address_space_init(struct i915_address_space *vm, int subclass) if (!kref_read(&vm->resv_ref)) kref_init(&vm->resv_ref); - INIT_RCU_WORK(&vm->rcu, __i915_vm_release); + INIT_WORK(&vm->release_work, __i915_vm_release); atomic_set(&vm->open, 1); /* diff --git a/drivers/gpu/drm/i915/gt/intel_gtt.h b/drivers/gpu/drm/i915/gt/intel_gtt.h index bc7153018ebd..bc6750263359 100644 --- a/drivers/gpu/drm/i915/gt/intel_gtt.h +++ b/drivers/gpu/drm/i915/gt/intel_gtt.h @@ -213,7 +213,7 @@ struct i915_vma_ops { struct i915_address_space { struct kref ref; - struct rcu_work rcu; + struct work_struct release_work; struct drm_mm mm; struct intel_gt *gt; @@ -260,6 +260,9 @@ struct i915_address_space { u8 pd_shift; u8 scratch_order; + /* Flags used when creating page-table objects for this vm */ + unsigned long lmem_pt_obj_flags; + struct drm_i915_gem_object * (*alloc_pt_dma)(struct i915_address_space *vm, int sz); @@ -519,7 +522,8 @@ i915_page_dir_dma_addr(const struct i915_ppgtt *ppgtt, const unsigned int n) return __px_dma(pt ? px_base(pt) : ppgtt->vm.scratch[ppgtt->vm.top]); } -void ppgtt_init(struct i915_ppgtt *ppgtt, struct intel_gt *gt); +void ppgtt_init(struct i915_ppgtt *ppgtt, struct intel_gt *gt, + unsigned long lmem_pt_obj_flags); int i915_ggtt_probe_hw(struct drm_i915_private *i915); int i915_ggtt_init_hw(struct drm_i915_private *i915); @@ -537,7 +541,8 @@ static inline bool i915_ggtt_has_aperture(const struct i915_ggtt *ggtt) int i915_ppgtt_init_hw(struct intel_gt *gt); -struct i915_ppgtt *i915_ppgtt_create(struct intel_gt *gt); +struct i915_ppgtt *i915_ppgtt_create(struct intel_gt *gt, + unsigned long lmem_pt_obj_flags); void i915_ggtt_suspend(struct i915_ggtt *gtt); void i915_ggtt_resume(struct i915_ggtt *ggtt); diff --git a/drivers/gpu/drm/i915/gt/intel_llc.c b/drivers/gpu/drm/i915/gt/intel_llc.c index eb1a15deed22..08d7d5ae263a 100644 --- a/drivers/gpu/drm/i915/gt/intel_llc.c +++ b/drivers/gpu/drm/i915/gt/intel_llc.c @@ -3,12 +3,13 @@ * Copyright © 2019 Intel Corporation */ +#include <asm/tsc.h> #include <linux/cpufreq.h> #include "i915_drv.h" #include "intel_gt.h" #include "intel_llc.h" -#include "intel_sideband.h" +#include "intel_pcode.h" struct ia_constants { unsigned int min_gpu_freq; diff --git a/drivers/gpu/drm/i915/gt/intel_lrc.c b/drivers/gpu/drm/i915/gt/intel_lrc.c index bb4af4977920..56156cf18c41 100644 --- a/drivers/gpu/drm/i915/gt/intel_lrc.c +++ b/drivers/gpu/drm/i915/gt/intel_lrc.c @@ -226,6 +226,40 @@ static const u8 gen12_xcs_offsets[] = { END }; +static const u8 dg2_xcs_offsets[] = { + NOP(1), + LRI(15, POSTED), + REG16(0x244), + REG(0x034), + REG(0x030), + REG(0x038), + REG(0x03c), + REG(0x168), + REG(0x140), + REG(0x110), + REG(0x1c0), + REG(0x1c4), + REG(0x1c8), + REG(0x180), + REG16(0x2b4), + REG(0x120), + REG(0x124), + + NOP(1), + LRI(9, POSTED), + REG16(0x3a8), + REG16(0x28c), + REG16(0x288), + REG16(0x284), + REG16(0x280), + REG16(0x27c), + REG16(0x278), + REG16(0x274), + REG16(0x270), + + END +}; + static const u8 gen8_rcs_offsets[] = { NOP(1), LRI(14, POSTED), @@ -525,6 +559,49 @@ static const u8 xehp_rcs_offsets[] = { END }; +static const u8 dg2_rcs_offsets[] = { + NOP(1), + LRI(15, POSTED), + REG16(0x244), + REG(0x034), + REG(0x030), + REG(0x038), + REG(0x03c), + REG(0x168), + REG(0x140), + REG(0x110), + REG(0x1c0), + REG(0x1c4), + REG(0x1c8), + REG(0x180), + REG16(0x2b4), + REG(0x120), + REG(0x124), + + NOP(1), + LRI(9, POSTED), + REG16(0x3a8), + REG16(0x28c), + REG16(0x288), + REG16(0x284), + REG16(0x280), + REG16(0x27c), + REG16(0x278), + REG16(0x274), + REG16(0x270), + + LRI(3, POSTED), + REG(0x1b0), + REG16(0x5a8), + REG16(0x5ac), + + NOP(6), + LRI(1, 0), + REG(0x0c8), + + END +}; + #undef END #undef REG16 #undef REG @@ -543,7 +620,9 @@ static const u8 *reg_offsets(const struct intel_engine_cs *engine) !intel_engine_has_relative_mmio(engine)); if (engine->class == RENDER_CLASS) { - if (GRAPHICS_VER_FULL(engine->i915) >= IP_VER(12, 50)) + if (GRAPHICS_VER_FULL(engine->i915) >= IP_VER(12, 55)) + return dg2_rcs_offsets; + else if (GRAPHICS_VER_FULL(engine->i915) >= IP_VER(12, 50)) return xehp_rcs_offsets; else if (GRAPHICS_VER(engine->i915) >= 12) return gen12_rcs_offsets; @@ -554,7 +633,9 @@ static const u8 *reg_offsets(const struct intel_engine_cs *engine) else return gen8_rcs_offsets; } else { - if (GRAPHICS_VER(engine->i915) >= 12) + if (GRAPHICS_VER_FULL(engine->i915) >= IP_VER(12, 55)) + return dg2_xcs_offsets; + else if (GRAPHICS_VER(engine->i915) >= 12) return gen12_xcs_offsets; else if (GRAPHICS_VER(engine->i915) >= 9) return gen9_xcs_offsets; @@ -861,7 +942,13 @@ __lrc_alloc_state(struct intel_context *ce, struct intel_engine_cs *engine) context_size += PAGE_SIZE; } - obj = i915_gem_object_create_lmem(engine->i915, context_size, 0); + if (intel_context_is_parent(ce) && intel_engine_uses_guc(engine)) { + ce->parallel.guc.parent_page = context_size / PAGE_SIZE; + context_size += PARENT_SCRATCH_SIZE; + } + + obj = i915_gem_object_create_lmem(engine->i915, context_size, + I915_BO_ALLOC_PM_VOLATILE); if (IS_ERR(obj)) obj = i915_gem_object_create_shmem(engine->i915, context_size); if (IS_ERR(obj)) diff --git a/drivers/gpu/drm/i915/gt/intel_migrate.c b/drivers/gpu/drm/i915/gt/intel_migrate.c index 1dac21aa7e5c..afb1cce9a352 100644 --- a/drivers/gpu/drm/i915/gt/intel_migrate.c +++ b/drivers/gpu/drm/i915/gt/intel_migrate.c @@ -78,7 +78,7 @@ static struct i915_address_space *migrate_vm(struct intel_gt *gt) * TODO: Add support for huge LMEM PTEs */ - vm = i915_ppgtt_create(gt); + vm = i915_ppgtt_create(gt, I915_BO_ALLOC_PM_EARLY); if (IS_ERR(vm)) return ERR_CAST(vm); diff --git a/drivers/gpu/drm/i915/gt/intel_mocs.c b/drivers/gpu/drm/i915/gt/intel_mocs.c index 582c4423b95d..15f9ada28a7a 100644 --- a/drivers/gpu/drm/i915/gt/intel_mocs.c +++ b/drivers/gpu/drm/i915/gt/intel_mocs.c @@ -22,6 +22,8 @@ struct drm_i915_mocs_table { unsigned int size; unsigned int n_entries; const struct drm_i915_mocs_entry *table; + u8 uc_index; + u8 unused_entries_index; }; /* Defines for the tables (XXX_MOCS_0 - XXX_MOCS_63) */ @@ -40,6 +42,8 @@ struct drm_i915_mocs_table { #define L3_ESC(value) ((value) << 0) #define L3_SCC(value) ((value) << 1) #define _L3_CACHEABILITY(value) ((value) << 4) +#define L3_GLBGO(value) ((value) << 6) +#define L3_LKUP(value) ((value) << 7) /* Helper defines */ #define GEN9_NUM_MOCS_ENTRIES 64 /* 63-64 are reserved, but configured. */ @@ -88,18 +92,25 @@ struct drm_i915_mocs_table { * * Entries not part of the following tables are undefined as far as * userspace is concerned and shouldn't be relied upon. For Gen < 12 - * they will be initialized to PTE. Gen >= 12 onwards don't have a setting for - * PTE and will be initialized to an invalid value. + * they will be initialized to PTE. Gen >= 12 don't have a setting for + * PTE and those platforms except TGL/RKL will be initialized L3 WB to + * catch accidental use of reserved and unused mocs indexes. * * The last few entries are reserved by the hardware. For ICL+ they * should be initialized according to bspec and never used, for older * platforms they should never be written to. * - * NOTE: These tables are part of bspec and defined as part of hardware + * NOTE1: These tables are part of bspec and defined as part of hardware * interface for ICL+. For older platforms, they are part of kernel * ABI. It is expected that, for specific hardware platform, existing * entries will remain constant and the table will only be updated by * adding new entries, filling unused positions. + * + * NOTE2: For GEN >= 12 except TGL and RKL, reserved and unspecified MOCS + * indices have been set to L3 WB. These reserved entries should never + * be used, they may be changed to low performant variants with better + * coherency in the future if more entries are needed. + * For TGL/RKL, all the unspecified MOCS indexes are mapped to L3 UC. */ #define GEN9_MOCS_ENTRIES \ MOCS_ENTRY(I915_MOCS_UNCACHED, \ @@ -282,17 +293,9 @@ static const struct drm_i915_mocs_entry icl_mocs_table[] = { }; static const struct drm_i915_mocs_entry dg1_mocs_table[] = { - /* Error */ - MOCS_ENTRY(0, 0, L3_0_DIRECT), /* UC */ MOCS_ENTRY(1, 0, L3_1_UC), - - /* Reserved */ - MOCS_ENTRY(2, 0, L3_0_DIRECT), - MOCS_ENTRY(3, 0, L3_0_DIRECT), - MOCS_ENTRY(4, 0, L3_0_DIRECT), - /* WB - L3 */ MOCS_ENTRY(5, 0, L3_3_WB), /* WB - L3 50% */ @@ -314,6 +317,83 @@ static const struct drm_i915_mocs_entry dg1_mocs_table[] = { MOCS_ENTRY(63, 0, L3_1_UC), }; +static const struct drm_i915_mocs_entry gen12_mocs_table[] = { + GEN11_MOCS_ENTRIES, + /* Implicitly enable L1 - HDC:L1 + L3 + LLC */ + MOCS_ENTRY(48, + LE_3_WB | LE_TC_1_LLC | LE_LRUM(3), + L3_3_WB), + /* Implicitly enable L1 - HDC:L1 + L3 */ + MOCS_ENTRY(49, + LE_1_UC | LE_TC_1_LLC, + L3_3_WB), + /* Implicitly enable L1 - HDC:L1 + LLC */ + MOCS_ENTRY(50, + LE_3_WB | LE_TC_1_LLC | LE_LRUM(3), + L3_1_UC), + /* Implicitly enable L1 - HDC:L1 */ + MOCS_ENTRY(51, + LE_1_UC | LE_TC_1_LLC, + L3_1_UC), + /* HW Special Case (CCS) */ + MOCS_ENTRY(60, + LE_3_WB | LE_TC_1_LLC | LE_LRUM(3), + L3_1_UC), + /* HW Special Case (Displayable) */ + MOCS_ENTRY(61, + LE_1_UC | LE_TC_1_LLC, + L3_3_WB), +}; + +static const struct drm_i915_mocs_entry xehpsdv_mocs_table[] = { + /* wa_1608975824 */ + MOCS_ENTRY(0, 0, L3_3_WB | L3_LKUP(1)), + + /* UC - Coherent; GO:L3 */ + MOCS_ENTRY(1, 0, L3_1_UC | L3_LKUP(1)), + /* UC - Coherent; GO:Memory */ + MOCS_ENTRY(2, 0, L3_1_UC | L3_GLBGO(1) | L3_LKUP(1)), + /* UC - Non-Coherent; GO:Memory */ + MOCS_ENTRY(3, 0, L3_1_UC | L3_GLBGO(1)), + /* UC - Non-Coherent; GO:L3 */ + MOCS_ENTRY(4, 0, L3_1_UC), + + /* WB */ + MOCS_ENTRY(5, 0, L3_3_WB | L3_LKUP(1)), + + /* HW Reserved - SW program but never use. */ + MOCS_ENTRY(48, 0, L3_3_WB | L3_LKUP(1)), + MOCS_ENTRY(49, 0, L3_1_UC | L3_LKUP(1)), + MOCS_ENTRY(60, 0, L3_1_UC), + MOCS_ENTRY(61, 0, L3_1_UC), + MOCS_ENTRY(62, 0, L3_1_UC), + MOCS_ENTRY(63, 0, L3_1_UC), +}; + +static const struct drm_i915_mocs_entry dg2_mocs_table[] = { + /* UC - Coherent; GO:L3 */ + MOCS_ENTRY(0, 0, L3_1_UC | L3_LKUP(1)), + /* UC - Coherent; GO:Memory */ + MOCS_ENTRY(1, 0, L3_1_UC | L3_GLBGO(1) | L3_LKUP(1)), + /* UC - Non-Coherent; GO:Memory */ + MOCS_ENTRY(2, 0, L3_1_UC | L3_GLBGO(1)), + + /* WB - LC */ + MOCS_ENTRY(3, 0, L3_3_WB | L3_LKUP(1)), +}; + +static const struct drm_i915_mocs_entry dg2_mocs_table_g10_ax[] = { + /* Wa_14011441408: Set Go to Memory for MOCS#0 */ + MOCS_ENTRY(0, 0, L3_1_UC | L3_GLBGO(1) | L3_LKUP(1)), + /* UC - Coherent; GO:Memory */ + MOCS_ENTRY(1, 0, L3_1_UC | L3_GLBGO(1) | L3_LKUP(1)), + /* UC - Non-Coherent; GO:Memory */ + MOCS_ENTRY(2, 0, L3_1_UC | L3_GLBGO(1)), + + /* WB - LC */ + MOCS_ENTRY(3, 0, L3_3_WB | L3_LKUP(1)), +}; + enum { HAS_GLOBAL_MOCS = BIT(0), HAS_ENGINE_MOCS = BIT(1), @@ -340,14 +420,45 @@ static unsigned int get_mocs_settings(const struct drm_i915_private *i915, { unsigned int flags; - if (IS_DG1(i915)) { + memset(table, 0, sizeof(struct drm_i915_mocs_table)); + + table->unused_entries_index = I915_MOCS_PTE; + if (IS_DG2(i915)) { + if (IS_DG2_GT_STEP(i915, G10, STEP_A0, STEP_B0)) { + table->size = ARRAY_SIZE(dg2_mocs_table_g10_ax); + table->table = dg2_mocs_table_g10_ax; + } else { + table->size = ARRAY_SIZE(dg2_mocs_table); + table->table = dg2_mocs_table; + } + table->uc_index = 1; + table->n_entries = GEN9_NUM_MOCS_ENTRIES; + table->unused_entries_index = 3; + } else if (IS_XEHPSDV(i915)) { + table->size = ARRAY_SIZE(xehpsdv_mocs_table); + table->table = xehpsdv_mocs_table; + table->uc_index = 2; + table->n_entries = GEN9_NUM_MOCS_ENTRIES; + table->unused_entries_index = 5; + } else if (IS_DG1(i915)) { table->size = ARRAY_SIZE(dg1_mocs_table); table->table = dg1_mocs_table; + table->uc_index = 1; table->n_entries = GEN9_NUM_MOCS_ENTRIES; - } else if (GRAPHICS_VER(i915) >= 12) { + table->uc_index = 1; + table->unused_entries_index = 5; + } else if (IS_TIGERLAKE(i915) || IS_ROCKETLAKE(i915)) { + /* For TGL/RKL, Can't be changed now for ABI reasons */ table->size = ARRAY_SIZE(tgl_mocs_table); table->table = tgl_mocs_table; table->n_entries = GEN9_NUM_MOCS_ENTRIES; + table->uc_index = 3; + } else if (GRAPHICS_VER(i915) >= 12) { + table->size = ARRAY_SIZE(gen12_mocs_table); + table->table = gen12_mocs_table; + table->n_entries = GEN9_NUM_MOCS_ENTRIES; + table->uc_index = 3; + table->unused_entries_index = 2; } else if (GRAPHICS_VER(i915) == 11) { table->size = ARRAY_SIZE(icl_mocs_table); table->table = icl_mocs_table; @@ -393,16 +504,16 @@ static unsigned int get_mocs_settings(const struct drm_i915_private *i915, } /* - * Get control_value from MOCS entry taking into account when it's not used: - * I915_MOCS_PTE's value is returned in this case. + * Get control_value from MOCS entry taking into account when it's not used + * then if unused_entries_index is non-zero then its value will be returned + * otherwise I915_MOCS_PTE's value is returned in this case. */ static u32 get_entry_control(const struct drm_i915_mocs_table *table, unsigned int index) { if (index < table->size && table->table[index].used) return table->table[index].control_value; - - return table->table[I915_MOCS_PTE].control_value; + return table->table[table->unused_entries_index].control_value; } #define for_each_mocs(mocs, t, i) \ @@ -417,6 +528,8 @@ static void __init_mocs_table(struct intel_uncore *uncore, unsigned int i; u32 mocs; + drm_WARN_ONCE(&uncore->i915->drm, !table->unused_entries_index, + "Unused entries index should have been defined\n"); for_each_mocs(mocs, table, i) intel_uncore_write_fw(uncore, _MMIO(addr + i * 4), mocs); } @@ -443,16 +556,16 @@ static void init_mocs_table(struct intel_engine_cs *engine, } /* - * Get l3cc_value from MOCS entry taking into account when it's not used: - * I915_MOCS_PTE's value is returned in this case. + * Get l3cc_value from MOCS entry taking into account when it's not used + * then if unused_entries_index is not zero then its value will be returned + * otherwise I915_MOCS_PTE's value is returned in this case. */ static u16 get_entry_l3cc(const struct drm_i915_mocs_table *table, unsigned int index) { if (index < table->size && table->table[index].used) return table->table[index].l3cc_value; - - return table->table[I915_MOCS_PTE].l3cc_value; + return table->table[table->unused_entries_index].l3cc_value; } static u32 l3cc_combine(u16 low, u16 high) @@ -468,10 +581,9 @@ static u32 l3cc_combine(u16 low, u16 high) 0; \ i++) -static void init_l3cc_table(struct intel_engine_cs *engine, +static void init_l3cc_table(struct intel_uncore *uncore, const struct drm_i915_mocs_table *table) { - struct intel_uncore *uncore = engine->uncore; unsigned int i; u32 l3cc; @@ -496,7 +608,7 @@ void intel_mocs_init_engine(struct intel_engine_cs *engine) init_mocs_table(engine, &table); if (flags & HAS_RENDER_L3CC && engine->class == RENDER_CLASS) - init_l3cc_table(engine, &table); + init_l3cc_table(engine->uncore, &table); } static u32 global_mocs_offset(void) @@ -504,6 +616,14 @@ static u32 global_mocs_offset(void) return i915_mmio_reg_offset(GEN12_GLOBAL_MOCS(0)); } +void intel_set_mocs_index(struct intel_gt *gt) +{ + struct drm_i915_mocs_table table; + + get_mocs_settings(gt->i915, &table); + gt->mocs.uc_index = table.uc_index; +} + void intel_mocs_init(struct intel_gt *gt) { struct drm_i915_mocs_table table; @@ -515,6 +635,14 @@ void intel_mocs_init(struct intel_gt *gt) flags = get_mocs_settings(gt->i915, &table); if (flags & HAS_GLOBAL_MOCS) __init_mocs_table(gt->uncore, &table, global_mocs_offset()); + + /* + * Initialize the L3CC table as part of mocs initalization to make + * sure the LNCFCMOCSx registers are programmed for the subsequent + * memory transactions including guc transactions + */ + if (flags & HAS_RENDER_L3CC) + init_l3cc_table(gt->uncore, &table); } #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) diff --git a/drivers/gpu/drm/i915/gt/intel_mocs.h b/drivers/gpu/drm/i915/gt/intel_mocs.h index d83274f5163b..76db827210c0 100644 --- a/drivers/gpu/drm/i915/gt/intel_mocs.h +++ b/drivers/gpu/drm/i915/gt/intel_mocs.h @@ -36,5 +36,6 @@ struct intel_gt; void intel_mocs_init(struct intel_gt *gt); void intel_mocs_init_engine(struct intel_engine_cs *engine); +void intel_set_mocs_index(struct intel_gt *gt); #endif diff --git a/drivers/gpu/drm/i915/gt/intel_ppgtt.c b/drivers/gpu/drm/i915/gt/intel_ppgtt.c index 886060f7e6fc..4396bfd630d8 100644 --- a/drivers/gpu/drm/i915/gt/intel_ppgtt.c +++ b/drivers/gpu/drm/i915/gt/intel_ppgtt.c @@ -155,19 +155,20 @@ int i915_ppgtt_init_hw(struct intel_gt *gt) } static struct i915_ppgtt * -__ppgtt_create(struct intel_gt *gt) +__ppgtt_create(struct intel_gt *gt, unsigned long lmem_pt_obj_flags) { if (GRAPHICS_VER(gt->i915) < 8) return gen6_ppgtt_create(gt); else - return gen8_ppgtt_create(gt); + return gen8_ppgtt_create(gt, lmem_pt_obj_flags); } -struct i915_ppgtt *i915_ppgtt_create(struct intel_gt *gt) +struct i915_ppgtt *i915_ppgtt_create(struct intel_gt *gt, + unsigned long lmem_pt_obj_flags) { struct i915_ppgtt *ppgtt; - ppgtt = __ppgtt_create(gt); + ppgtt = __ppgtt_create(gt, lmem_pt_obj_flags); if (IS_ERR(ppgtt)) return ppgtt; @@ -298,7 +299,8 @@ int ppgtt_set_pages(struct i915_vma *vma) return 0; } -void ppgtt_init(struct i915_ppgtt *ppgtt, struct intel_gt *gt) +void ppgtt_init(struct i915_ppgtt *ppgtt, struct intel_gt *gt, + unsigned long lmem_pt_obj_flags) { struct drm_i915_private *i915 = gt->i915; @@ -306,6 +308,7 @@ void ppgtt_init(struct i915_ppgtt *ppgtt, struct intel_gt *gt) ppgtt->vm.i915 = i915; ppgtt->vm.dma = i915->drm.dev; ppgtt->vm.total = BIT_ULL(INTEL_INFO(i915)->ppgtt_size); + ppgtt->vm.lmem_pt_obj_flags = lmem_pt_obj_flags; dma_resv_init(&ppgtt->vm._resv); i915_address_space_init(&ppgtt->vm, VM_CLASS_PPGTT); diff --git a/drivers/gpu/drm/i915/gt/intel_rc6.c b/drivers/gpu/drm/i915/gt/intel_rc6.c index 799d382eea79..43093dd2d0c9 100644 --- a/drivers/gpu/drm/i915/gt/intel_rc6.c +++ b/drivers/gpu/drm/i915/gt/intel_rc6.c @@ -9,8 +9,8 @@ #include "i915_vgpu.h" #include "intel_gt.h" #include "intel_gt_pm.h" +#include "intel_pcode.h" #include "intel_rc6.h" -#include "intel_sideband.h" /** * DOC: RC6 diff --git a/drivers/gpu/drm/i915/gt/intel_region_lmem.c b/drivers/gpu/drm/i915/gt/intel_region_lmem.c index a74b72f50cc9..afb35d2e5c73 100644 --- a/drivers/gpu/drm/i915/gt/intel_region_lmem.c +++ b/drivers/gpu/drm/i915/gt/intel_region_lmem.c @@ -32,7 +32,7 @@ static int init_fake_lmem_bar(struct intel_memory_region *mem) mem->remap_addr = dma_map_resource(i915->drm.dev, mem->region.start, mem->fake_mappable.size, - PCI_DMA_BIDIRECTIONAL, + DMA_BIDIRECTIONAL, DMA_ATTR_FORCE_CONTIGUOUS); if (dma_mapping_error(i915->drm.dev, mem->remap_addr)) { drm_mm_remove_node(&mem->fake_mappable); @@ -62,7 +62,7 @@ static void release_fake_lmem_bar(struct intel_memory_region *mem) dma_unmap_resource(mem->i915->drm.dev, mem->remap_addr, mem->fake_mappable.size, - PCI_DMA_BIDIRECTIONAL, + DMA_BIDIRECTIONAL, DMA_ATTR_FORCE_CONTIGUOUS); } diff --git a/drivers/gpu/drm/i915/gt/intel_ring.c b/drivers/gpu/drm/i915/gt/intel_ring.c index 7c4d5158e03b..2fdd52b62092 100644 --- a/drivers/gpu/drm/i915/gt/intel_ring.c +++ b/drivers/gpu/drm/i915/gt/intel_ring.c @@ -112,7 +112,8 @@ static struct i915_vma *create_ring_vma(struct i915_ggtt *ggtt, int size) struct drm_i915_gem_object *obj; struct i915_vma *vma; - obj = i915_gem_object_create_lmem(i915, size, I915_BO_ALLOC_VOLATILE); + obj = i915_gem_object_create_lmem(i915, size, I915_BO_ALLOC_VOLATILE | + I915_BO_ALLOC_PM_VOLATILE); if (IS_ERR(obj) && i915_ggtt_has_aperture(ggtt)) obj = i915_gem_object_create_stolen(i915, size); if (IS_ERR(obj)) diff --git a/drivers/gpu/drm/i915/gt/intel_ring_submission.c b/drivers/gpu/drm/i915/gt/intel_ring_submission.c index 2958e2fae380..586dca1731ce 100644 --- a/drivers/gpu/drm/i915/gt/intel_ring_submission.c +++ b/drivers/gpu/drm/i915/gt/intel_ring_submission.c @@ -17,6 +17,7 @@ #include "intel_ring.h" #include "shmem_utils.h" #include "intel_engine_heartbeat.h" +#include "intel_engine_pm.h" /* Rough estimate of the typical request size, performing a flush, * set-context and then emitting the batch. @@ -291,7 +292,9 @@ static void xcs_sanitize(struct intel_engine_cs *engine) sanitize_hwsp(engine); /* And scrub the dirty cachelines for the HWSP */ - clflush_cache_range(engine->status_page.addr, PAGE_SIZE); + drm_clflush_virt_range(engine->status_page.addr, PAGE_SIZE); + + intel_engine_reset_pinned_contexts(engine); } static void reset_prepare(struct intel_engine_cs *engine) @@ -1265,7 +1268,7 @@ static struct i915_vma *gen7_ctx_vma(struct intel_engine_cs *engine) int size, err; if (GRAPHICS_VER(engine->i915) != 7 || engine->class != RENDER_CLASS) - return 0; + return NULL; err = gen7_ctx_switch_bb_setup(engine, NULL /* probe size */); if (err < 0) diff --git a/drivers/gpu/drm/i915/gt/intel_rps.c b/drivers/gpu/drm/i915/gt/intel_rps.c index 0a03fbed9f9b..5e275f8dda8c 100644 --- a/drivers/gpu/drm/i915/gt/intel_rps.c +++ b/drivers/gpu/drm/i915/gt/intel_rps.c @@ -11,8 +11,9 @@ #include "intel_gt_clock_utils.h" #include "intel_gt_irq.h" #include "intel_gt_pm_irq.h" +#include "intel_pcode.h" #include "intel_rps.h" -#include "intel_sideband.h" +#include "vlv_sideband.h" #include "../../../platform/x86/intel_ips.h" #define BUSY_MAX_EI 20u /* ms */ @@ -994,20 +995,16 @@ int intel_rps_set(struct intel_rps *rps, u8 val) static void gen6_rps_init(struct intel_rps *rps) { struct drm_i915_private *i915 = rps_to_i915(rps); - struct intel_uncore *uncore = rps_to_uncore(rps); + u32 rp_state_cap = intel_rps_read_state_cap(rps); /* All of these values are in units of 50MHz */ /* static values from HW: RP0 > RP1 > RPn (min_freq) */ if (IS_GEN9_LP(i915)) { - u32 rp_state_cap = intel_uncore_read(uncore, BXT_RP_STATE_CAP); - rps->rp0_freq = (rp_state_cap >> 16) & 0xff; rps->rp1_freq = (rp_state_cap >> 8) & 0xff; rps->min_freq = (rp_state_cap >> 0) & 0xff; } else { - u32 rp_state_cap = intel_uncore_read(uncore, GEN6_RP_STATE_CAP); - rps->rp0_freq = (rp_state_cap >> 0) & 0xff; rps->rp1_freq = (rp_state_cap >> 8) & 0xff; rps->min_freq = (rp_state_cap >> 16) & 0xff; @@ -2144,6 +2141,19 @@ int intel_rps_set_min_frequency(struct intel_rps *rps, u32 val) return set_min_freq(rps, val); } +u32 intel_rps_read_state_cap(struct intel_rps *rps) +{ + struct drm_i915_private *i915 = rps_to_i915(rps); + struct intel_uncore *uncore = rps_to_uncore(rps); + + if (IS_XEHPSDV(i915)) + return intel_uncore_read(uncore, XEHPSDV_RP_STATE_CAP); + else if (IS_GEN9_LP(i915)) + return intel_uncore_read(uncore, BXT_RP_STATE_CAP); + else + return intel_uncore_read(uncore, GEN6_RP_STATE_CAP); +} + /* External interface for intel_ips.ko */ static struct drm_i915_private __rcu *ips_mchdev; diff --git a/drivers/gpu/drm/i915/gt/intel_rps.h b/drivers/gpu/drm/i915/gt/intel_rps.h index 4213bcce1667..11960d64ca82 100644 --- a/drivers/gpu/drm/i915/gt/intel_rps.h +++ b/drivers/gpu/drm/i915/gt/intel_rps.h @@ -41,6 +41,7 @@ u32 intel_rps_get_rp1_frequency(struct intel_rps *rps); u32 intel_rps_get_rpn_frequency(struct intel_rps *rps); u32 intel_rps_read_punit_req(struct intel_rps *rps); u32 intel_rps_read_punit_req_frequency(struct intel_rps *rps); +u32 intel_rps_read_state_cap(struct intel_rps *rps); void gen5_rps_irq_handler(struct intel_rps *rps); void gen6_rps_irq_handler(struct intel_rps *rps, u32 pm_iir); diff --git a/drivers/gpu/drm/i915/gt/intel_sseu.c b/drivers/gpu/drm/i915/gt/intel_sseu.c index bbd272943c3f..bdf09051b8a0 100644 --- a/drivers/gpu/drm/i915/gt/intel_sseu.c +++ b/drivers/gpu/drm/i915/gt/intel_sseu.c @@ -46,11 +46,11 @@ u32 intel_sseu_get_subslices(const struct sseu_dev_info *sseu, u8 slice) } void intel_sseu_set_subslices(struct sseu_dev_info *sseu, int slice, - u32 ss_mask) + u8 *subslice_mask, u32 ss_mask) { int offset = slice * sseu->ss_stride; - memcpy(&sseu->subslice_mask[offset], &ss_mask, sseu->ss_stride); + memcpy(&subslice_mask[offset], &ss_mask, sseu->ss_stride); } unsigned int @@ -100,14 +100,24 @@ static u16 compute_eu_total(const struct sseu_dev_info *sseu) return total; } -static void gen11_compute_sseu_info(struct sseu_dev_info *sseu, - u8 s_en, u32 ss_en, u16 eu_en) +static u32 get_ss_stride_mask(struct sseu_dev_info *sseu, u8 s, u32 ss_en) +{ + u32 ss_mask; + + ss_mask = ss_en >> (s * sseu->max_subslices); + ss_mask &= GENMASK(sseu->max_subslices - 1, 0); + + return ss_mask; +} + +static void gen11_compute_sseu_info(struct sseu_dev_info *sseu, u8 s_en, + u32 g_ss_en, u32 c_ss_en, u16 eu_en) { int s, ss; - /* ss_en represents entire subslice mask across all slices */ + /* g_ss_en/c_ss_en represent entire subslice mask across all slices */ GEM_BUG_ON(sseu->max_slices * sseu->max_subslices > - sizeof(ss_en) * BITS_PER_BYTE); + sizeof(g_ss_en) * BITS_PER_BYTE); for (s = 0; s < sseu->max_slices; s++) { if ((s_en & BIT(s)) == 0) @@ -115,7 +125,22 @@ static void gen11_compute_sseu_info(struct sseu_dev_info *sseu, sseu->slice_mask |= BIT(s); - intel_sseu_set_subslices(sseu, s, ss_en); + /* + * XeHP introduces the concept of compute vs geometry DSS. To + * reduce variation between GENs around subslice usage, store a + * mask for both the geometry and compute enabled masks since + * userspace will need to be able to query these masks + * independently. Also compute a total enabled subslice count + * for the purposes of selecting subslices to use in a + * particular GEM context. + */ + intel_sseu_set_subslices(sseu, s, sseu->compute_subslice_mask, + get_ss_stride_mask(sseu, s, c_ss_en)); + intel_sseu_set_subslices(sseu, s, sseu->geometry_subslice_mask, + get_ss_stride_mask(sseu, s, g_ss_en)); + intel_sseu_set_subslices(sseu, s, sseu->subslice_mask, + get_ss_stride_mask(sseu, s, + g_ss_en | c_ss_en)); for (ss = 0; ss < sseu->max_subslices; ss++) if (intel_sseu_has_subslice(sseu, s, ss)) @@ -129,7 +154,7 @@ static void gen12_sseu_info_init(struct intel_gt *gt) { struct sseu_dev_info *sseu = >->info.sseu; struct intel_uncore *uncore = gt->uncore; - u32 dss_en; + u32 g_dss_en, c_dss_en = 0; u16 eu_en = 0; u8 eu_en_fuse; u8 s_en; @@ -160,7 +185,9 @@ static void gen12_sseu_info_init(struct intel_gt *gt) s_en = intel_uncore_read(uncore, GEN11_GT_SLICE_ENABLE) & GEN11_GT_S_ENA_MASK; - dss_en = intel_uncore_read(uncore, GEN12_GT_DSS_ENABLE); + g_dss_en = intel_uncore_read(uncore, GEN12_GT_GEOMETRY_DSS_ENABLE); + if (GRAPHICS_VER_FULL(gt->i915) >= IP_VER(12, 50)) + c_dss_en = intel_uncore_read(uncore, GEN12_GT_COMPUTE_DSS_ENABLE); /* one bit per pair of EUs */ if (GRAPHICS_VER_FULL(gt->i915) >= IP_VER(12, 50)) @@ -173,7 +200,7 @@ static void gen12_sseu_info_init(struct intel_gt *gt) if (eu_en_fuse & BIT(eu)) eu_en |= BIT(eu * 2) | BIT(eu * 2 + 1); - gen11_compute_sseu_info(sseu, s_en, dss_en, eu_en); + gen11_compute_sseu_info(sseu, s_en, g_dss_en, c_dss_en, eu_en); /* TGL only supports slice-level power gating */ sseu->has_slice_pg = 1; @@ -199,7 +226,7 @@ static void gen11_sseu_info_init(struct intel_gt *gt) eu_en = ~(intel_uncore_read(uncore, GEN11_EU_DISABLE) & GEN11_EU_DIS_MASK); - gen11_compute_sseu_info(sseu, s_en, ss_en, eu_en); + gen11_compute_sseu_info(sseu, s_en, ss_en, 0, eu_en); /* ICL has no power gating restrictions. */ sseu->has_slice_pg = 1; @@ -240,7 +267,7 @@ static void cherryview_sseu_info_init(struct intel_gt *gt) sseu_set_eus(sseu, 0, 1, ~disabled_mask); } - intel_sseu_set_subslices(sseu, 0, subslice_mask); + intel_sseu_set_subslices(sseu, 0, sseu->subslice_mask, subslice_mask); sseu->eu_total = compute_eu_total(sseu); @@ -296,7 +323,8 @@ static void gen9_sseu_info_init(struct intel_gt *gt) /* skip disabled slice */ continue; - intel_sseu_set_subslices(sseu, s, subslice_mask); + intel_sseu_set_subslices(sseu, s, sseu->subslice_mask, + subslice_mask); eu_disable = intel_uncore_read(uncore, GEN9_EU_DISABLE(s)); for (ss = 0; ss < sseu->max_subslices; ss++) { @@ -408,7 +436,8 @@ static void bdw_sseu_info_init(struct intel_gt *gt) /* skip disabled slice */ continue; - intel_sseu_set_subslices(sseu, s, subslice_mask); + intel_sseu_set_subslices(sseu, s, sseu->subslice_mask, + subslice_mask); for (ss = 0; ss < sseu->max_subslices; ss++) { u8 eu_disabled_mask; @@ -485,10 +514,9 @@ static void hsw_sseu_info_init(struct intel_gt *gt) } fuse1 = intel_uncore_read(gt->uncore, HSW_PAVP_FUSE1); - switch ((fuse1 & HSW_F1_EU_DIS_MASK) >> HSW_F1_EU_DIS_SHIFT) { + switch (REG_FIELD_GET(HSW_F1_EU_DIS_MASK, fuse1)) { default: - MISSING_CASE((fuse1 & HSW_F1_EU_DIS_MASK) >> - HSW_F1_EU_DIS_SHIFT); + MISSING_CASE(REG_FIELD_GET(HSW_F1_EU_DIS_MASK, fuse1)); fallthrough; case HSW_F1_EU_DIS_10EUS: sseu->eu_per_subslice = 10; @@ -506,7 +534,8 @@ static void hsw_sseu_info_init(struct intel_gt *gt) sseu->eu_per_subslice); for (s = 0; s < sseu->max_slices; s++) { - intel_sseu_set_subslices(sseu, s, subslice_mask); + intel_sseu_set_subslices(sseu, s, sseu->subslice_mask, + subslice_mask); for (ss = 0; ss < sseu->max_subslices; ss++) { sseu_set_eus(sseu, s, ss, diff --git a/drivers/gpu/drm/i915/gt/intel_sseu.h b/drivers/gpu/drm/i915/gt/intel_sseu.h index 22fef98887c0..60882a74741e 100644 --- a/drivers/gpu/drm/i915/gt/intel_sseu.h +++ b/drivers/gpu/drm/i915/gt/intel_sseu.h @@ -26,9 +26,14 @@ struct drm_printer; #define GEN_DSS_PER_CSLICE 8 #define GEN_DSS_PER_MSLICE 8 +#define GEN_MAX_GSLICES (GEN_MAX_SUBSLICES / GEN_DSS_PER_GSLICE) +#define GEN_MAX_CSLICES (GEN_MAX_SUBSLICES / GEN_DSS_PER_CSLICE) + struct sseu_dev_info { u8 slice_mask; u8 subslice_mask[GEN_MAX_SLICES * GEN_MAX_SUBSLICE_STRIDE]; + u8 geometry_subslice_mask[GEN_MAX_SLICES * GEN_MAX_SUBSLICE_STRIDE]; + u8 compute_subslice_mask[GEN_MAX_SLICES * GEN_MAX_SUBSLICE_STRIDE]; u8 eu_mask[GEN_MAX_SLICES * GEN_MAX_SUBSLICES * GEN_MAX_EU_STRIDE]; u16 eu_total; u8 eu_per_subslice; @@ -78,6 +83,10 @@ intel_sseu_has_subslice(const struct sseu_dev_info *sseu, int slice, u8 mask; int ss_idx = subslice / BITS_PER_BYTE; + if (slice >= sseu->max_slices || + subslice >= sseu->max_subslices) + return false; + GEM_BUG_ON(ss_idx >= sseu->ss_stride); mask = sseu->subslice_mask[slice * sseu->ss_stride + ss_idx]; @@ -97,7 +106,7 @@ intel_sseu_subslices_per_slice(const struct sseu_dev_info *sseu, u8 slice); u32 intel_sseu_get_subslices(const struct sseu_dev_info *sseu, u8 slice); void intel_sseu_set_subslices(struct sseu_dev_info *sseu, int slice, - u32 ss_mask); + u8 *subslice_mask, u32 ss_mask); void intel_sseu_info_init(struct intel_gt *gt); diff --git a/drivers/gpu/drm/i915/gt/intel_sseu_debugfs.c b/drivers/gpu/drm/i915/gt/intel_sseu_debugfs.c index 1ba8b7da9d37..8bb3a91dad82 100644 --- a/drivers/gpu/drm/i915/gt/intel_sseu_debugfs.c +++ b/drivers/gpu/drm/i915/gt/intel_sseu_debugfs.c @@ -4,9 +4,9 @@ * Copyright © 2020 Intel Corporation */ -#include "debugfs_gt.h" -#include "intel_sseu_debugfs.h" #include "i915_drv.h" +#include "intel_gt_debugfs.h" +#include "intel_sseu_debugfs.h" static void sseu_copy_subslices(const struct sseu_dev_info *sseu, int slice, u8 *to_mask) @@ -282,7 +282,7 @@ static int sseu_status_show(struct seq_file *m, void *unused) return intel_sseu_status(m, gt); } -DEFINE_GT_DEBUGFS_ATTRIBUTE(sseu_status); +DEFINE_INTEL_GT_DEBUGFS_ATTRIBUTE(sseu_status); static int rcs_topology_show(struct seq_file *m, void *unused) { @@ -293,11 +293,11 @@ static int rcs_topology_show(struct seq_file *m, void *unused) return 0; } -DEFINE_GT_DEBUGFS_ATTRIBUTE(rcs_topology); +DEFINE_INTEL_GT_DEBUGFS_ATTRIBUTE(rcs_topology); void intel_sseu_debugfs_register(struct intel_gt *gt, struct dentry *root) { - static const struct debugfs_gt_file files[] = { + static const struct intel_gt_debugfs_file files[] = { { "sseu_status", &sseu_status_fops, NULL }, { "rcs_topology", &rcs_topology_fops, NULL }, }; diff --git a/drivers/gpu/drm/i915/gt/intel_workarounds.c b/drivers/gpu/drm/i915/gt/intel_workarounds.c index aae609d7d85d..e1f362530889 100644 --- a/drivers/gpu/drm/i915/gt/intel_workarounds.c +++ b/drivers/gpu/drm/i915/gt/intel_workarounds.c @@ -644,6 +644,72 @@ static void dg1_ctx_workarounds_init(struct intel_engine_cs *engine, DG1_HZ_READ_SUPPRESSION_OPTIMIZATION_DISABLE); } +static void fakewa_disable_nestedbb_mode(struct intel_engine_cs *engine, + struct i915_wa_list *wal) +{ + /* + * This is a "fake" workaround defined by software to ensure we + * maintain reliable, backward-compatible behavior for userspace with + * regards to how nested MI_BATCH_BUFFER_START commands are handled. + * + * The per-context setting of MI_MODE[12] determines whether the bits + * of a nested MI_BATCH_BUFFER_START instruction should be interpreted + * in the traditional manner or whether they should instead use a new + * tgl+ meaning that breaks backward compatibility, but allows nesting + * into 3rd-level batchbuffers. When this new capability was first + * added in TGL, it remained off by default unless a context + * intentionally opted in to the new behavior. However Xe_HPG now + * flips this on by default and requires that we explicitly opt out if + * we don't want the new behavior. + * + * From a SW perspective, we want to maintain the backward-compatible + * behavior for userspace, so we'll apply a fake workaround to set it + * back to the legacy behavior on platforms where the hardware default + * is to break compatibility. At the moment there is no Linux + * userspace that utilizes third-level batchbuffers, so this will avoid + * userspace from needing to make any changes. using the legacy + * meaning is the correct thing to do. If/when we have userspace + * consumers that want to utilize third-level batch nesting, we can + * provide a context parameter to allow them to opt-in. + */ + wa_masked_dis(wal, RING_MI_MODE(engine->mmio_base), TGL_NESTED_BB_EN); +} + +static void gen12_ctx_gt_mocs_init(struct intel_engine_cs *engine, + struct i915_wa_list *wal) +{ + u8 mocs; + + /* + * Some blitter commands do not have a field for MOCS, those + * commands will use MOCS index pointed by BLIT_CCTL. + * BLIT_CCTL registers are needed to be programmed to un-cached. + */ + if (engine->class == COPY_ENGINE_CLASS) { + mocs = engine->gt->mocs.uc_index; + wa_write_clr_set(wal, + BLIT_CCTL(engine->mmio_base), + BLIT_CCTL_MASK, + BLIT_CCTL_MOCS(mocs, mocs)); + } +} + +/* + * gen12_ctx_gt_fake_wa_init() aren't programmingan official workaround + * defined by the hardware team, but it programming general context registers. + * Adding those context register programming in context workaround + * allow us to use the wa framework for proper application and validation. + */ +static void +gen12_ctx_gt_fake_wa_init(struct intel_engine_cs *engine, + struct i915_wa_list *wal) +{ + if (GRAPHICS_VER_FULL(engine->i915) >= IP_VER(12, 55)) + fakewa_disable_nestedbb_mode(engine, wal); + + gen12_ctx_gt_mocs_init(engine, wal); +} + static void __intel_engine_init_ctx_wa(struct intel_engine_cs *engine, struct i915_wa_list *wal, @@ -651,11 +717,19 @@ __intel_engine_init_ctx_wa(struct intel_engine_cs *engine, { struct drm_i915_private *i915 = engine->i915; - if (engine->class != RENDER_CLASS) - return; - wa_init_start(wal, name, engine->name); + /* Applies to all engines */ + /* + * Fake workarounds are not the actual workaround but + * programming of context registers using workaround framework. + */ + if (GRAPHICS_VER(i915) >= 12) + gen12_ctx_gt_fake_wa_init(engine, wal); + + if (engine->class != RENDER_CLASS) + goto done; + if (IS_DG1(i915)) dg1_ctx_workarounds_init(engine, wal); else if (GRAPHICS_VER(i915) == 12) @@ -685,6 +759,7 @@ __intel_engine_init_ctx_wa(struct intel_engine_cs *engine, else MISSING_CASE(GRAPHICS_VER(i915)); +done: wa_init_finish(wal); } @@ -729,7 +804,7 @@ int intel_engine_emit_ctx_wa(struct i915_request *rq) } static void -gen4_gt_workarounds_init(struct drm_i915_private *i915, +gen4_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal) { /* WaDisable_RenderCache_OperationalFlush:gen4,ilk */ @@ -737,29 +812,29 @@ gen4_gt_workarounds_init(struct drm_i915_private *i915, } static void -g4x_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal) +g4x_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal) { - gen4_gt_workarounds_init(i915, wal); + gen4_gt_workarounds_init(gt, wal); /* WaDisableRenderCachePipelinedFlush:g4x,ilk */ wa_masked_en(wal, CACHE_MODE_0, CM0_PIPELINED_RENDER_FLUSH_DISABLE); } static void -ilk_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal) +ilk_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal) { - g4x_gt_workarounds_init(i915, wal); + g4x_gt_workarounds_init(gt, wal); wa_masked_en(wal, _3D_CHICKEN2, _3D_CHICKEN2_WM_READ_PIPELINED); } static void -snb_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal) +snb_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal) { } static void -ivb_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal) +ivb_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal) { /* Apply the WaDisableRHWOOptimizationForRenderHang:ivb workaround. */ wa_masked_dis(wal, @@ -775,7 +850,7 @@ ivb_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal) } static void -vlv_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal) +vlv_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal) { /* WaForceL3Serialization:vlv */ wa_write_clr(wal, GEN7_L3SQCREG4, L3SQ_URB_READ_CAM_MATCH_DISABLE); @@ -788,7 +863,7 @@ vlv_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal) } static void -hsw_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal) +hsw_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal) { /* L3 caching of data atomics doesn't work -- disable it. */ wa_write(wal, HSW_SCRATCH1, HSW_SCRATCH1_L3_DATA_ATOMICS_DISABLE); @@ -803,8 +878,10 @@ hsw_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal) } static void -gen9_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal) +gen9_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal) { + struct drm_i915_private *i915 = gt->i915; + /* WaDisableKillLogic:bxt,skl,kbl */ if (!IS_COFFEELAKE(i915) && !IS_COMETLAKE(i915)) wa_write_or(wal, @@ -829,9 +906,9 @@ gen9_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal } static void -skl_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal) +skl_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal) { - gen9_gt_workarounds_init(i915, wal); + gen9_gt_workarounds_init(gt, wal); /* WaDisableGafsUnitClkGating:skl */ wa_write_or(wal, @@ -839,19 +916,19 @@ skl_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal) GEN8_EU_GAUNIT_CLOCK_GATE_DISABLE); /* WaInPlaceDecompressionHang:skl */ - if (IS_SKL_GT_STEP(i915, STEP_A0, STEP_H0)) + if (IS_SKL_GT_STEP(gt->i915, STEP_A0, STEP_H0)) wa_write_or(wal, GEN9_GAMT_ECO_REG_RW_IA, GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS); } static void -kbl_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal) +kbl_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal) { - gen9_gt_workarounds_init(i915, wal); + gen9_gt_workarounds_init(gt, wal); /* WaDisableDynamicCreditSharing:kbl */ - if (IS_KBL_GT_STEP(i915, 0, STEP_C0)) + if (IS_KBL_GT_STEP(gt->i915, 0, STEP_C0)) wa_write_or(wal, GAMT_CHKN_BIT_REG, GAMT_CHKN_DISABLE_DYNAMIC_CREDIT_SHARING); @@ -868,15 +945,15 @@ kbl_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal) } static void -glk_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal) +glk_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal) { - gen9_gt_workarounds_init(i915, wal); + gen9_gt_workarounds_init(gt, wal); } static void -cfl_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal) +cfl_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal) { - gen9_gt_workarounds_init(i915, wal); + gen9_gt_workarounds_init(gt, wal); /* WaDisableGafsUnitClkGating:cfl */ wa_write_or(wal, @@ -901,21 +978,21 @@ static void __set_mcr_steering(struct i915_wa_list *wal, wa_write_clr_set(wal, steering_reg, mcr_mask, mcr); } -static void __add_mcr_wa(struct drm_i915_private *i915, struct i915_wa_list *wal, +static void __add_mcr_wa(struct intel_gt *gt, struct i915_wa_list *wal, unsigned int slice, unsigned int subslice) { - drm_dbg(&i915->drm, "MCR slice=0x%x, subslice=0x%x\n", slice, subslice); + drm_dbg(>->i915->drm, "MCR slice=0x%x, subslice=0x%x\n", slice, subslice); __set_mcr_steering(wal, GEN8_MCR_SELECTOR, slice, subslice); } static void -icl_wa_init_mcr(struct drm_i915_private *i915, struct i915_wa_list *wal) +icl_wa_init_mcr(struct intel_gt *gt, struct i915_wa_list *wal) { - const struct sseu_dev_info *sseu = &i915->gt.info.sseu; + const struct sseu_dev_info *sseu = >->info.sseu; unsigned int slice, subslice; - GEM_BUG_ON(GRAPHICS_VER(i915) < 11); + GEM_BUG_ON(GRAPHICS_VER(gt->i915) < 11); GEM_BUG_ON(hweight8(sseu->slice_mask) > 1); slice = 0; @@ -935,16 +1012,15 @@ icl_wa_init_mcr(struct drm_i915_private *i915, struct i915_wa_list *wal) * then we can just rely on the default steering and won't need to * worry about explicitly re-steering L3BANK reads later. */ - if (i915->gt.info.l3bank_mask & BIT(subslice)) - i915->gt.steering_table[L3BANK] = NULL; + if (gt->info.l3bank_mask & BIT(subslice)) + gt->steering_table[L3BANK] = NULL; - __add_mcr_wa(i915, wal, slice, subslice); + __add_mcr_wa(gt, wal, slice, subslice); } static void xehp_init_mcr(struct intel_gt *gt, struct i915_wa_list *wal) { - struct drm_i915_private *i915 = gt->i915; const struct sseu_dev_info *sseu = >->info.sseu; unsigned long slice, subslice = 0, slice_mask = 0; u64 dss_mask = 0; @@ -1008,7 +1084,7 @@ xehp_init_mcr(struct intel_gt *gt, struct i915_wa_list *wal) WARN_ON(subslice > GEN_DSS_PER_GSLICE); WARN_ON(dss_mask >> (slice * GEN_DSS_PER_GSLICE) == 0); - __add_mcr_wa(i915, wal, slice, subslice); + __add_mcr_wa(gt, wal, slice, subslice); /* * SQIDI ranges are special because they use different steering @@ -1024,9 +1100,11 @@ xehp_init_mcr(struct intel_gt *gt, struct i915_wa_list *wal) } static void -icl_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal) +icl_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal) { - icl_wa_init_mcr(i915, wal); + struct drm_i915_private *i915 = gt->i915; + + icl_wa_init_mcr(gt, wal); /* WaModifyGamTlbPartitioning:icl */ wa_write_clr_set(wal, @@ -1077,10 +1155,9 @@ icl_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal) * the engine-specific workaround list. */ static void -wa_14011060649(struct drm_i915_private *i915, struct i915_wa_list *wal) +wa_14011060649(struct intel_gt *gt, struct i915_wa_list *wal) { struct intel_engine_cs *engine; - struct intel_gt *gt = &i915->gt; int id; for_each_engine(engine, gt, id) { @@ -1094,22 +1171,23 @@ wa_14011060649(struct drm_i915_private *i915, struct i915_wa_list *wal) } static void -gen12_gt_workarounds_init(struct drm_i915_private *i915, - struct i915_wa_list *wal) +gen12_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal) { - icl_wa_init_mcr(i915, wal); + icl_wa_init_mcr(gt, wal); /* Wa_14011060649:tgl,rkl,dg1,adl-s,adl-p */ - wa_14011060649(i915, wal); + wa_14011060649(gt, wal); /* Wa_14011059788:tgl,rkl,adl-s,dg1,adl-p */ wa_write_or(wal, GEN10_DFR_RATIO_EN_AND_CHICKEN, DFR_DISABLE); } static void -tgl_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal) +tgl_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal) { - gen12_gt_workarounds_init(i915, wal); + struct drm_i915_private *i915 = gt->i915; + + gen12_gt_workarounds_init(gt, wal); /* Wa_1409420604:tgl */ if (IS_TGL_UY_GT_STEP(i915, STEP_A0, STEP_B0)) @@ -1130,9 +1208,11 @@ tgl_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal) } static void -dg1_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal) +dg1_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal) { - gen12_gt_workarounds_init(i915, wal); + struct drm_i915_private *i915 = gt->i915; + + gen12_gt_workarounds_init(gt, wal); /* Wa_1607087056:dg1 */ if (IS_DG1_GT_STEP(i915, STEP_A0, STEP_B0)) @@ -1154,60 +1234,62 @@ dg1_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal) } static void -xehpsdv_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal) +xehpsdv_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal) { - xehp_init_mcr(&i915->gt, wal); + xehp_init_mcr(gt, wal); } static void -gt_init_workarounds(struct drm_i915_private *i915, struct i915_wa_list *wal) +gt_init_workarounds(struct intel_gt *gt, struct i915_wa_list *wal) { + struct drm_i915_private *i915 = gt->i915; + if (IS_XEHPSDV(i915)) - xehpsdv_gt_workarounds_init(i915, wal); + xehpsdv_gt_workarounds_init(gt, wal); else if (IS_DG1(i915)) - dg1_gt_workarounds_init(i915, wal); + dg1_gt_workarounds_init(gt, wal); else if (IS_TIGERLAKE(i915)) - tgl_gt_workarounds_init(i915, wal); + tgl_gt_workarounds_init(gt, wal); else if (GRAPHICS_VER(i915) == 12) - gen12_gt_workarounds_init(i915, wal); + gen12_gt_workarounds_init(gt, wal); else if (GRAPHICS_VER(i915) == 11) - icl_gt_workarounds_init(i915, wal); + icl_gt_workarounds_init(gt, wal); else if (IS_COFFEELAKE(i915) || IS_COMETLAKE(i915)) - cfl_gt_workarounds_init(i915, wal); + cfl_gt_workarounds_init(gt, wal); else if (IS_GEMINILAKE(i915)) - glk_gt_workarounds_init(i915, wal); + glk_gt_workarounds_init(gt, wal); else if (IS_KABYLAKE(i915)) - kbl_gt_workarounds_init(i915, wal); + kbl_gt_workarounds_init(gt, wal); else if (IS_BROXTON(i915)) - gen9_gt_workarounds_init(i915, wal); + gen9_gt_workarounds_init(gt, wal); else if (IS_SKYLAKE(i915)) - skl_gt_workarounds_init(i915, wal); + skl_gt_workarounds_init(gt, wal); else if (IS_HASWELL(i915)) - hsw_gt_workarounds_init(i915, wal); + hsw_gt_workarounds_init(gt, wal); else if (IS_VALLEYVIEW(i915)) - vlv_gt_workarounds_init(i915, wal); + vlv_gt_workarounds_init(gt, wal); else if (IS_IVYBRIDGE(i915)) - ivb_gt_workarounds_init(i915, wal); + ivb_gt_workarounds_init(gt, wal); else if (GRAPHICS_VER(i915) == 6) - snb_gt_workarounds_init(i915, wal); + snb_gt_workarounds_init(gt, wal); else if (GRAPHICS_VER(i915) == 5) - ilk_gt_workarounds_init(i915, wal); + ilk_gt_workarounds_init(gt, wal); else if (IS_G4X(i915)) - g4x_gt_workarounds_init(i915, wal); + g4x_gt_workarounds_init(gt, wal); else if (GRAPHICS_VER(i915) == 4) - gen4_gt_workarounds_init(i915, wal); + gen4_gt_workarounds_init(gt, wal); else if (GRAPHICS_VER(i915) <= 8) ; else MISSING_CASE(GRAPHICS_VER(i915)); } -void intel_gt_init_workarounds(struct drm_i915_private *i915) +void intel_gt_init_workarounds(struct intel_gt *gt) { - struct i915_wa_list *wal = &i915->gt_wa_list; + struct i915_wa_list *wal = >->wa_list; wa_init_start(wal, "GT", "global"); - gt_init_workarounds(i915, wal); + gt_init_workarounds(gt, wal); wa_init_finish(wal); } @@ -1278,7 +1360,7 @@ wa_list_apply(struct intel_gt *gt, const struct i915_wa_list *wal) void intel_gt_apply_workarounds(struct intel_gt *gt) { - wa_list_apply(gt, >->i915->gt_wa_list); + wa_list_apply(gt, >->wa_list); } static bool wa_list_verify(struct intel_gt *gt, @@ -1310,7 +1392,7 @@ static bool wa_list_verify(struct intel_gt *gt, bool intel_gt_verify_workarounds(struct intel_gt *gt, const char *from) { - return wa_list_verify(gt, >->i915->gt_wa_list, from); + return wa_list_verify(gt, >->wa_list, from); } __maybe_unused @@ -1604,6 +1686,31 @@ void intel_engine_apply_whitelist(struct intel_engine_cs *engine) i915_mmio_reg_offset(RING_NOPID(base))); } +/* + * engine_fake_wa_init(), a place holder to program the registers + * which are not part of an official workaround defined by the + * hardware team. + * Adding programming of those register inside workaround will + * allow utilizing wa framework to proper application and verification. + */ +static void +engine_fake_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal) +{ + u8 mocs; + + /* + * RING_CMD_CCTL are need to be programed to un-cached + * for memory writes and reads outputted by Command + * Streamers on Gen12 onward platforms. + */ + if (GRAPHICS_VER(engine->i915) >= 12) { + mocs = engine->gt->mocs.uc_index; + wa_masked_field_set(wal, + RING_CMD_CCTL(engine->mmio_base), + CMD_CCTL_MOCS_MASK, + CMD_CCTL_MOCS_OVERRIDE(mocs, mocs)); + } +} static void rcs_engine_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal) { @@ -2044,6 +2151,8 @@ engine_init_workarounds(struct intel_engine_cs *engine, struct i915_wa_list *wal if (I915_SELFTEST_ONLY(GRAPHICS_VER(engine->i915) < 4)) return; + engine_fake_wa_init(engine, wal); + if (engine->class == RENDER_CLASS) rcs_engine_wa_init(engine, wal); else @@ -2067,12 +2176,7 @@ void intel_engine_apply_workarounds(struct intel_engine_cs *engine) wa_list_apply(engine->gt, &engine->wa_list); } -struct mcr_range { - u32 start; - u32 end; -}; - -static const struct mcr_range mcr_ranges_gen8[] = { +static const struct i915_range mcr_ranges_gen8[] = { { .start = 0x5500, .end = 0x55ff }, { .start = 0x7000, .end = 0x7fff }, { .start = 0x9400, .end = 0x97ff }, @@ -2081,7 +2185,7 @@ static const struct mcr_range mcr_ranges_gen8[] = { {}, }; -static const struct mcr_range mcr_ranges_gen12[] = { +static const struct i915_range mcr_ranges_gen12[] = { { .start = 0x8150, .end = 0x815f }, { .start = 0x9520, .end = 0x955f }, { .start = 0xb100, .end = 0xb3ff }, @@ -2090,7 +2194,7 @@ static const struct mcr_range mcr_ranges_gen12[] = { {}, }; -static const struct mcr_range mcr_ranges_xehp[] = { +static const struct i915_range mcr_ranges_xehp[] = { { .start = 0x4000, .end = 0x4aff }, { .start = 0x5200, .end = 0x52ff }, { .start = 0x5400, .end = 0x7fff }, @@ -2109,7 +2213,7 @@ static const struct mcr_range mcr_ranges_xehp[] = { static bool mcr_range(struct drm_i915_private *i915, u32 offset) { - const struct mcr_range *mcr_ranges; + const struct i915_range *mcr_ranges; int i; if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 50)) diff --git a/drivers/gpu/drm/i915/gt/intel_workarounds.h b/drivers/gpu/drm/i915/gt/intel_workarounds.h index 15abb68b6c00..9beaab77c7f0 100644 --- a/drivers/gpu/drm/i915/gt/intel_workarounds.h +++ b/drivers/gpu/drm/i915/gt/intel_workarounds.h @@ -24,7 +24,7 @@ static inline void intel_wa_list_free(struct i915_wa_list *wal) void intel_engine_init_ctx_wa(struct intel_engine_cs *engine); int intel_engine_emit_ctx_wa(struct i915_request *rq); -void intel_gt_init_workarounds(struct drm_i915_private *i915); +void intel_gt_init_workarounds(struct intel_gt *gt); void intel_gt_apply_workarounds(struct intel_gt *gt); bool intel_gt_verify_workarounds(struct intel_gt *gt, const char *from); diff --git a/drivers/gpu/drm/i915/gt/mock_engine.c b/drivers/gpu/drm/i915/gt/mock_engine.c index 2c1af030310c..8b89215afe46 100644 --- a/drivers/gpu/drm/i915/gt/mock_engine.c +++ b/drivers/gpu/drm/i915/gt/mock_engine.c @@ -376,6 +376,8 @@ int mock_engine_init(struct intel_engine_cs *engine) { struct intel_context *ce; + INIT_LIST_HEAD(&engine->pinned_contexts_list); + engine->sched_engine = i915_sched_engine_create(ENGINE_MOCK); if (!engine->sched_engine) return -ENOMEM; diff --git a/drivers/gpu/drm/i915/gt/selftest_engine_heartbeat.c b/drivers/gpu/drm/i915/gt/selftest_engine_heartbeat.c index 317eebf086c3..6e6e4d747cca 100644 --- a/drivers/gpu/drm/i915/gt/selftest_engine_heartbeat.c +++ b/drivers/gpu/drm/i915/gt/selftest_engine_heartbeat.c @@ -290,7 +290,7 @@ static int live_heartbeat_fast(void *arg) int err = 0; /* Check that the heartbeat ticks at the desired rate. */ - if (!IS_ACTIVE(CONFIG_DRM_I915_HEARTBEAT_INTERVAL)) + if (!CONFIG_DRM_I915_HEARTBEAT_INTERVAL) return 0; for_each_engine(engine, gt, id) { @@ -352,7 +352,7 @@ static int live_heartbeat_off(void *arg) int err = 0; /* Check that we can turn off heartbeat and not interrupt VIP */ - if (!IS_ACTIVE(CONFIG_DRM_I915_HEARTBEAT_INTERVAL)) + if (!CONFIG_DRM_I915_HEARTBEAT_INTERVAL) return 0; for_each_engine(engine, gt, id) { diff --git a/drivers/gpu/drm/i915/gt/selftest_execlists.c b/drivers/gpu/drm/i915/gt/selftest_execlists.c index f12ffe797639..b367ecfa42de 100644 --- a/drivers/gpu/drm/i915/gt/selftest_execlists.c +++ b/drivers/gpu/drm/i915/gt/selftest_execlists.c @@ -992,7 +992,7 @@ static int live_timeslice_preempt(void *arg) * need to preempt the current task and replace it with another * ready task. */ - if (!IS_ACTIVE(CONFIG_DRM_I915_TIMESLICE_DURATION)) + if (!CONFIG_DRM_I915_TIMESLICE_DURATION) return 0; obj = i915_gem_object_create_internal(gt->i915, PAGE_SIZE); @@ -1122,7 +1122,7 @@ static int live_timeslice_rewind(void *arg) * but only a few of those requests, forcing us to rewind the * RING_TAIL of the original request. */ - if (!IS_ACTIVE(CONFIG_DRM_I915_TIMESLICE_DURATION)) + if (!CONFIG_DRM_I915_TIMESLICE_DURATION) return 0; for_each_engine(engine, gt, id) { @@ -1299,7 +1299,7 @@ static int live_timeslice_queue(void *arg) * ELSP[1] is already occupied, so must rely on timeslicing to * eject ELSP[0] in favour of the queue.) */ - if (!IS_ACTIVE(CONFIG_DRM_I915_TIMESLICE_DURATION)) + if (!CONFIG_DRM_I915_TIMESLICE_DURATION) return 0; obj = i915_gem_object_create_internal(gt->i915, PAGE_SIZE); @@ -1420,7 +1420,7 @@ static int live_timeslice_nopreempt(void *arg) * We should not timeslice into a request that is marked with * I915_REQUEST_NOPREEMPT. */ - if (!IS_ACTIVE(CONFIG_DRM_I915_TIMESLICE_DURATION)) + if (!CONFIG_DRM_I915_TIMESLICE_DURATION) return 0; if (igt_spinner_init(&spin, gt)) @@ -2260,7 +2260,7 @@ static int __cancel_hostile(struct live_preempt_cancel *arg) int err; /* Preempt cancel non-preemptible spinner in ELSP0 */ - if (!IS_ACTIVE(CONFIG_DRM_I915_PREEMPT_TIMEOUT)) + if (!CONFIG_DRM_I915_PREEMPT_TIMEOUT) return 0; if (!intel_has_reset_engine(arg->engine->gt)) @@ -2316,7 +2316,7 @@ static int __cancel_fail(struct live_preempt_cancel *arg) struct i915_request *rq; int err; - if (!IS_ACTIVE(CONFIG_DRM_I915_PREEMPT_TIMEOUT)) + if (!CONFIG_DRM_I915_PREEMPT_TIMEOUT) return 0; if (!intel_has_reset_engine(engine->gt)) @@ -3375,7 +3375,7 @@ static int live_preempt_timeout(void *arg) * Check that we force preemption to occur by cancelling the previous * context if it refuses to yield the GPU. */ - if (!IS_ACTIVE(CONFIG_DRM_I915_PREEMPT_TIMEOUT)) + if (!CONFIG_DRM_I915_PREEMPT_TIMEOUT) return 0; if (!intel_has_reset_engine(gt)) @@ -3493,7 +3493,7 @@ static int smoke_submit(struct preempt_smoke *smoke, if (batch) { struct i915_address_space *vm; - vm = i915_gem_context_get_vm_rcu(ctx); + vm = i915_gem_context_get_eb_vm(ctx); vma = i915_vma_instance(batch, vm, NULL); i915_vm_put(vm); if (IS_ERR(vma)) @@ -3733,7 +3733,7 @@ static int nop_virtual_engine(struct intel_gt *gt, GEM_BUG_ON(!nctx || nctx > ARRAY_SIZE(ve)); for (n = 0; n < nctx; n++) { - ve[n] = intel_engine_create_virtual(siblings, nsibling); + ve[n] = intel_engine_create_virtual(siblings, nsibling, 0); if (IS_ERR(ve[n])) { err = PTR_ERR(ve[n]); nctx = n; @@ -3929,7 +3929,7 @@ static int mask_virtual_engine(struct intel_gt *gt, * restrict it to our desired engine within the virtual engine. */ - ve = intel_engine_create_virtual(siblings, nsibling); + ve = intel_engine_create_virtual(siblings, nsibling, 0); if (IS_ERR(ve)) { err = PTR_ERR(ve); goto out_close; @@ -4060,7 +4060,7 @@ static int slicein_virtual_engine(struct intel_gt *gt, i915_request_add(rq); } - ce = intel_engine_create_virtual(siblings, nsibling); + ce = intel_engine_create_virtual(siblings, nsibling, 0); if (IS_ERR(ce)) { err = PTR_ERR(ce); goto out; @@ -4112,7 +4112,7 @@ static int sliceout_virtual_engine(struct intel_gt *gt, /* XXX We do not handle oversubscription and fairness with normal rq */ for (n = 0; n < nsibling; n++) { - ce = intel_engine_create_virtual(siblings, nsibling); + ce = intel_engine_create_virtual(siblings, nsibling, 0); if (IS_ERR(ce)) { err = PTR_ERR(ce); goto out; @@ -4214,7 +4214,7 @@ static int preserved_virtual_engine(struct intel_gt *gt, if (err) goto out_scratch; - ve = intel_engine_create_virtual(siblings, nsibling); + ve = intel_engine_create_virtual(siblings, nsibling, 0); if (IS_ERR(ve)) { err = PTR_ERR(ve); goto out_scratch; @@ -4354,7 +4354,7 @@ static int reset_virtual_engine(struct intel_gt *gt, if (igt_spinner_init(&spin, gt)) return -ENOMEM; - ve = intel_engine_create_virtual(siblings, nsibling); + ve = intel_engine_create_virtual(siblings, nsibling, 0); if (IS_ERR(ve)) { err = PTR_ERR(ve); goto out_spin; diff --git a/drivers/gpu/drm/i915/gt/selftest_hangcheck.c b/drivers/gpu/drm/i915/gt/selftest_hangcheck.c index 2c1ed32ca5ac..7e2d99dd012d 100644 --- a/drivers/gpu/drm/i915/gt/selftest_hangcheck.c +++ b/drivers/gpu/drm/i915/gt/selftest_hangcheck.c @@ -117,7 +117,7 @@ static struct i915_request * hang_create_request(struct hang *h, struct intel_engine_cs *engine) { struct intel_gt *gt = h->gt; - struct i915_address_space *vm = i915_gem_context_get_vm_rcu(h->ctx); + struct i915_address_space *vm = i915_gem_context_get_eb_vm(h->ctx); struct drm_i915_gem_object *obj; struct i915_request *rq = NULL; struct i915_vma *hws, *vma; @@ -789,7 +789,7 @@ static int __igt_reset_engine(struct intel_gt *gt, bool active) if (err) pr_err("[%s] Wait for request %lld:%lld [0x%04X] failed: %d!\n", engine->name, rq->fence.context, - rq->fence.seqno, rq->context->guc_id, err); + rq->fence.seqno, rq->context->guc_id.id, err); } skip: @@ -1098,7 +1098,7 @@ static int __igt_reset_engines(struct intel_gt *gt, if (err) pr_err("[%s] Wait for request %lld:%lld [0x%04X] failed: %d!\n", engine->name, rq->fence.context, - rq->fence.seqno, rq->context->guc_id, err); + rq->fence.seqno, rq->context->guc_id.id, err); } count++; @@ -1108,7 +1108,7 @@ static int __igt_reset_engines(struct intel_gt *gt, pr_err("i915_reset_engine(%s:%s): failed to reset request %lld:%lld [0x%04X]\n", engine->name, test_name, rq->fence.context, - rq->fence.seqno, rq->context->guc_id); + rq->fence.seqno, rq->context->guc_id.id); i915_request_put(rq); GEM_TRACE_DUMP(); @@ -1596,7 +1596,7 @@ static int igt_reset_evict_ppgtt(void *arg) if (INTEL_PPGTT(gt->i915) < INTEL_PPGTT_FULL) return 0; - ppgtt = i915_ppgtt_create(gt); + ppgtt = i915_ppgtt_create(gt, 0); if (IS_ERR(ppgtt)) return PTR_ERR(ppgtt); diff --git a/drivers/gpu/drm/i915/gt/selftest_workarounds.c b/drivers/gpu/drm/i915/gt/selftest_workarounds.c index e623ac45f4aa..962e91ba3be4 100644 --- a/drivers/gpu/drm/i915/gt/selftest_workarounds.c +++ b/drivers/gpu/drm/i915/gt/selftest_workarounds.c @@ -66,7 +66,7 @@ reference_lists_init(struct intel_gt *gt, struct wa_lists *lists) memset(lists, 0, sizeof(*lists)); wa_init_start(&lists->gt_wa_list, "GT_REF", "global"); - gt_init_workarounds(gt->i915, &lists->gt_wa_list); + gt_init_workarounds(gt, &lists->gt_wa_list); wa_init_finish(&lists->gt_wa_list); for_each_engine(engine, gt, id) { diff --git a/drivers/gpu/drm/i915/gt/uc/abi/guc_actions_abi.h b/drivers/gpu/drm/i915/gt/uc/abi/guc_actions_abi.h index 8ff582222aff..ba10bd374cee 100644 --- a/drivers/gpu/drm/i915/gt/uc/abi/guc_actions_abi.h +++ b/drivers/gpu/drm/i915/gt/uc/abi/guc_actions_abi.h @@ -142,6 +142,7 @@ enum intel_guc_action { INTEL_GUC_ACTION_REGISTER_COMMAND_TRANSPORT_BUFFER = 0x4505, INTEL_GUC_ACTION_DEREGISTER_COMMAND_TRANSPORT_BUFFER = 0x4506, INTEL_GUC_ACTION_DEREGISTER_CONTEXT_DONE = 0x4600, + INTEL_GUC_ACTION_REGISTER_CONTEXT_MULTI_LRC = 0x4601, INTEL_GUC_ACTION_RESET_CLIENT = 0x5507, INTEL_GUC_ACTION_LIMIT }; diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc.c b/drivers/gpu/drm/i915/gt/uc/intel_guc.c index fbfcae727d7f..6e228343e8cb 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_guc.c +++ b/drivers/gpu/drm/i915/gt/uc/intel_guc.c @@ -3,6 +3,7 @@ * Copyright © 2014-2019 Intel Corporation */ +#include "gem/i915_gem_lmem.h" #include "gt/intel_gt.h" #include "gt/intel_gt_irq.h" #include "gt/intel_gt_pm_irq.h" @@ -647,7 +648,14 @@ struct i915_vma *intel_guc_allocate_vma(struct intel_guc *guc, u32 size) u64 flags; int ret; - obj = i915_gem_object_create_shmem(gt->i915, size); + if (HAS_LMEM(gt->i915)) + obj = i915_gem_object_create_lmem(gt->i915, size, + I915_BO_ALLOC_CPU_CLEAR | + I915_BO_ALLOC_CONTIGUOUS | + I915_BO_ALLOC_PM_EARLY); + else + obj = i915_gem_object_create_shmem(gt->i915, size); + if (IS_ERR(obj)) return ERR_CAST(obj); @@ -748,3 +756,32 @@ void intel_guc_load_status(struct intel_guc *guc, struct drm_printer *p) } } } + +void intel_guc_write_barrier(struct intel_guc *guc) +{ + struct intel_gt *gt = guc_to_gt(guc); + + if (i915_gem_object_is_lmem(guc->ct.vma->obj)) { + /* + * Ensure intel_uncore_write_fw can be used rather than + * intel_uncore_write. + */ + GEM_BUG_ON(guc->send_regs.fw_domains); + + /* + * This register is used by the i915 and GuC for MMIO based + * communication. Once we are in this code CTBs are the only + * method the i915 uses to communicate with the GuC so it is + * safe to write to this register (a value of 0 is NOP for MMIO + * communication). If we ever start mixing CTBs and MMIOs a new + * register will have to be chosen. This function is also used + * to enforce ordering of a work queue item write and an update + * to the process descriptor. When a work queue is being used, + * CTBs are also the only mechanism of communication. + */ + intel_uncore_write_fw(gt->uncore, GEN11_SOFT_SCRATCH(0), 0); + } else { + /* wmb() sufficient for a barrier if in smem */ + wmb(); + } +} diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc.h b/drivers/gpu/drm/i915/gt/uc/intel_guc.h index 2e27fe59786b..31cf9fb48c7e 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_guc.h +++ b/drivers/gpu/drm/i915/gt/uc/intel_guc.h @@ -22,74 +22,155 @@ struct __guc_ads_blob; -/* - * Top level structure of GuC. It handles firmware loading and manages client - * pool. intel_guc owns a intel_guc_client to replace the legacy ExecList - * submission. +/** + * struct intel_guc - Top level structure of GuC. + * + * It handles firmware loading and manages client pool. intel_guc owns an + * i915_sched_engine for submission. */ struct intel_guc { + /** @fw: the GuC firmware */ struct intel_uc_fw fw; + /** @log: sub-structure containing GuC log related data and objects */ struct intel_guc_log log; + /** @ct: the command transport communication channel */ struct intel_guc_ct ct; + /** @slpc: sub-structure containing SLPC related data and objects */ struct intel_guc_slpc slpc; - /* Global engine used to submit requests to GuC */ + /** @sched_engine: Global engine used to submit requests to GuC */ struct i915_sched_engine *sched_engine; + /** + * @stalled_request: if GuC can't process a request for any reason, we + * save it until GuC restarts processing. No other request can be + * submitted until the stalled request is processed. + */ struct i915_request *stalled_request; + /** + * @submission_stall_reason: reason why submission is stalled + */ + enum { + STALL_NONE, + STALL_REGISTER_CONTEXT, + STALL_MOVE_LRC_TAIL, + STALL_ADD_REQUEST, + } submission_stall_reason; /* intel_guc_recv interrupt related state */ + /** @irq_lock: protects GuC irq state */ spinlock_t irq_lock; + /** + * @msg_enabled_mask: mask of events that are processed when receiving + * an INTEL_GUC_ACTION_DEFAULT G2H message. + */ unsigned int msg_enabled_mask; + /** + * @outstanding_submission_g2h: number of outstanding GuC to Host + * responses related to GuC submission, used to determine if the GT is + * idle + */ atomic_t outstanding_submission_g2h; + /** @interrupts: pointers to GuC interrupt-managing functions. */ struct { void (*reset)(struct intel_guc *guc); void (*enable)(struct intel_guc *guc); void (*disable)(struct intel_guc *guc); } interrupts; - /* - * contexts_lock protects the pool of free guc ids and a linked list of - * guc ids available to be stolen + /** + * @submission_state: sub-structure for submission state protected by + * single lock + */ + struct { + /** + * @lock: protects everything in submission_state, + * ce->guc_id.id, and ce->guc_id.ref when transitioning in and + * out of zero + */ + spinlock_t lock; + /** + * @guc_ids: used to allocate new guc_ids, single-lrc + */ + struct ida guc_ids; + /** + * @guc_ids_bitmap: used to allocate new guc_ids, multi-lrc + */ + unsigned long *guc_ids_bitmap; + /** + * @guc_id_list: list of intel_context with valid guc_ids but no + * refs + */ + struct list_head guc_id_list; + /** + * @destroyed_contexts: list of contexts waiting to be destroyed + * (deregistered with the GuC) + */ + struct list_head destroyed_contexts; + /** + * @destroyed_worker: worker to deregister contexts, need as we + * need to take a GT PM reference and can't from destroy + * function as it might be in an atomic context (no sleeping) + */ + struct work_struct destroyed_worker; + } submission_state; + + /** + * @submission_supported: tracks whether we support GuC submission on + * the current platform */ - spinlock_t contexts_lock; - struct ida guc_ids; - struct list_head guc_id_list; - bool submission_supported; + /** @submission_selected: tracks whether the user enabled GuC submission */ bool submission_selected; + /** + * @rc_supported: tracks whether we support GuC rc on the current platform + */ bool rc_supported; + /** @rc_selected: tracks whether the user enabled GuC rc */ bool rc_selected; + /** @ads_vma: object allocated to hold the GuC ADS */ struct i915_vma *ads_vma; + /** @ads_blob: contents of the GuC ADS */ struct __guc_ads_blob *ads_blob; + /** @ads_regset_size: size of the save/restore regsets in the ADS */ u32 ads_regset_size; + /** @ads_golden_ctxt_size: size of the golden contexts in the ADS */ u32 ads_golden_ctxt_size; + /** @lrc_desc_pool: object allocated to hold the GuC LRC descriptor pool */ struct i915_vma *lrc_desc_pool; + /** @lrc_desc_pool_vaddr: contents of the GuC LRC descriptor pool */ void *lrc_desc_pool_vaddr; - /* guc_id to intel_context lookup */ + /** + * @context_lookup: used to resolve intel_context from guc_id, if a + * context is present in this structure it is registered with the GuC + */ struct xarray context_lookup; - /* Control params for fw initialization */ + /** @params: Control params for fw initialization */ u32 params[GUC_CTL_MAX_DWORDS]; - /* GuC's FW specific registers used in MMIO send */ + /** @send_regs: GuC's FW specific registers used for sending MMIO H2G */ struct { u32 base; unsigned int count; enum forcewake_domains fw_domains; } send_regs; - /* register used to send interrupts to the GuC FW */ + /** @notify_reg: register used to send interrupts to the GuC FW */ i915_reg_t notify_reg; - /* Store msg (e.g. log flush) that we see while CTBs are disabled */ + /** + * @mmio_msg: notification bitmask that the GuC writes in one of its + * registers when the CT channel is disabled, to be processed when the + * channel is back up. + */ u32 mmio_msg; - /* To serialize the intel_guc_send actions */ + /** @send_mutex: used to serialize the intel_guc_send actions */ struct mutex send_mutex; }; @@ -295,4 +376,6 @@ void intel_guc_submission_cancel_requests(struct intel_guc *guc); void intel_guc_load_status(struct intel_guc *guc, struct drm_printer *p); +void intel_guc_write_barrier(struct intel_guc *guc); + #endif diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_ads.c b/drivers/gpu/drm/i915/gt/uc/intel_guc_ads.c index 6926919bcac6..621c893a009f 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_guc_ads.c +++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_ads.c @@ -176,7 +176,7 @@ static void guc_mapping_table_init(struct intel_gt *gt, for_each_engine(engine, gt, id) { u8 guc_class = engine_class_to_guc_class(engine->class); - system_info->mapping_table[guc_class][engine->instance] = + system_info->mapping_table[guc_class][ilog2(engine->logical_mask)] = engine->instance; } } @@ -349,6 +349,8 @@ static void fill_engine_enable_masks(struct intel_gt *gt, info->engine_enabled_masks[GUC_VIDEOENHANCE_CLASS] = VEBOX_MASK(gt); } +#define LR_HW_CONTEXT_SIZE (80 * sizeof(u32)) +#define LRC_SKIP_SIZE (LRC_PPHWSP_SZ * PAGE_SIZE + LR_HW_CONTEXT_SIZE) static int guc_prep_golden_context(struct intel_guc *guc, struct __guc_ads_blob *blob) { @@ -396,7 +398,18 @@ static int guc_prep_golden_context(struct intel_guc *guc, if (!blob) continue; - blob->ads.eng_state_size[guc_class] = real_size; + /* + * This interface is slightly confusing. We need to pass the + * base address of the full golden context and the size of just + * the engine state, which is the section of the context image + * that starts after the execlists context. This is required to + * allow the GuC to restore just the engine state when a + * watchdog reset occurs. + * We calculate the engine state size by removing the size of + * what comes before it in the context image (which is identical + * on all engines). + */ + blob->ads.eng_state_size[guc_class] = real_size - LRC_SKIP_SIZE; blob->ads.golden_context_lrca[guc_class] = addr_ggtt; addr_ggtt += alloc_size; } @@ -436,11 +449,6 @@ static void guc_init_golden_context(struct intel_guc *guc) u8 engine_class, guc_class; u8 *ptr; - /* Skip execlist and PPGTT registers + HWSP */ - const u32 lr_hw_context_size = 80 * sizeof(u32); - const u32 skip_size = LRC_PPHWSP_SZ * PAGE_SIZE + - lr_hw_context_size; - if (!intel_uc_uses_guc_submission(>->uc)) return; @@ -476,12 +484,12 @@ static void guc_init_golden_context(struct intel_guc *guc) continue; } - GEM_BUG_ON(blob->ads.eng_state_size[guc_class] != real_size); + GEM_BUG_ON(blob->ads.eng_state_size[guc_class] != + real_size - LRC_SKIP_SIZE); GEM_BUG_ON(blob->ads.golden_context_lrca[guc_class] != addr_ggtt); addr_ggtt += alloc_size; - shmem_read(engine->default_state, skip_size, ptr + skip_size, - real_size - skip_size); + shmem_read(engine->default_state, 0, ptr, real_size); ptr += alloc_size; } diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_ct.c b/drivers/gpu/drm/i915/gt/uc/intel_guc_ct.c index 22b4733b55e2..a0cc34be7b56 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_guc_ct.c +++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_ct.c @@ -168,12 +168,15 @@ static int guc_action_register_ct_buffer(struct intel_guc *guc, u32 type, FIELD_PREP(HOST2GUC_REGISTER_CTB_REQUEST_MSG_2_DESC_ADDR, desc_addr), FIELD_PREP(HOST2GUC_REGISTER_CTB_REQUEST_MSG_3_BUFF_ADDR, buff_addr), }; + int ret; GEM_BUG_ON(type != GUC_CTB_TYPE_HOST2GUC && type != GUC_CTB_TYPE_GUC2HOST); GEM_BUG_ON(size % SZ_4K); /* CT registration must go over MMIO */ - return intel_guc_send_mmio(guc, request, ARRAY_SIZE(request), NULL, 0); + ret = intel_guc_send_mmio(guc, request, ARRAY_SIZE(request), NULL, 0); + + return ret > 0 ? -EPROTO : ret; } static int ct_register_buffer(struct intel_guc_ct *ct, u32 type, @@ -188,8 +191,8 @@ static int ct_register_buffer(struct intel_guc_ct *ct, u32 type, err = guc_action_register_ct_buffer(ct_to_guc(ct), type, desc_addr, buff_addr, size); if (unlikely(err)) - CT_ERROR(ct, "Failed to register %s buffer (err=%d)\n", - guc_ct_buffer_type_to_str(type), err); + CT_ERROR(ct, "Failed to register %s buffer (%pe)\n", + guc_ct_buffer_type_to_str(type), ERR_PTR(err)); return err; } @@ -201,11 +204,14 @@ static int guc_action_deregister_ct_buffer(struct intel_guc *guc, u32 type) FIELD_PREP(GUC_HXG_REQUEST_MSG_0_ACTION, GUC_ACTION_HOST2GUC_DEREGISTER_CTB), FIELD_PREP(HOST2GUC_DEREGISTER_CTB_REQUEST_MSG_1_TYPE, type), }; + int ret; GEM_BUG_ON(type != GUC_CTB_TYPE_HOST2GUC && type != GUC_CTB_TYPE_GUC2HOST); /* CT deregistration must go over MMIO */ - return intel_guc_send_mmio(guc, request, ARRAY_SIZE(request), NULL, 0); + ret = intel_guc_send_mmio(guc, request, ARRAY_SIZE(request), NULL, 0); + + return ret > 0 ? -EPROTO : ret; } static int ct_deregister_buffer(struct intel_guc_ct *ct, u32 type) @@ -213,8 +219,8 @@ static int ct_deregister_buffer(struct intel_guc_ct *ct, u32 type) int err = guc_action_deregister_ct_buffer(ct_to_guc(ct), type); if (unlikely(err)) - CT_ERROR(ct, "Failed to deregister %s buffer (err=%d)\n", - guc_ct_buffer_type_to_str(type), err); + CT_ERROR(ct, "Failed to deregister %s buffer (%pe)\n", + guc_ct_buffer_type_to_str(type), ERR_PTR(err)); return err; } @@ -377,28 +383,6 @@ static u32 ct_get_next_fence(struct intel_guc_ct *ct) return ++ct->requests.last_fence; } -static void write_barrier(struct intel_guc_ct *ct) -{ - struct intel_guc *guc = ct_to_guc(ct); - struct intel_gt *gt = guc_to_gt(guc); - - if (i915_gem_object_is_lmem(guc->ct.vma->obj)) { - GEM_BUG_ON(guc->send_regs.fw_domains); - /* - * This register is used by the i915 and GuC for MMIO based - * communication. Once we are in this code CTBs are the only - * method the i915 uses to communicate with the GuC so it is - * safe to write to this register (a value of 0 is NOP for MMIO - * communication). If we ever start mixing CTBs and MMIOs a new - * register will have to be chosen. - */ - intel_uncore_write_fw(gt->uncore, GEN11_SOFT_SCRATCH(0), 0); - } else { - /* wmb() sufficient for a barrier if in smem */ - wmb(); - } -} - static int ct_write(struct intel_guc_ct *ct, const u32 *action, u32 len /* in dwords */, @@ -468,7 +452,7 @@ static int ct_write(struct intel_guc_ct *ct, * make sure H2G buffer update and LRC tail update (if this triggering a * submission) are visible before updating the descriptor tail */ - write_barrier(ct); + intel_guc_write_barrier(ct_to_guc(ct)); /* update local copies */ ctb->tail = tail; @@ -522,9 +506,6 @@ static int wait_for_ct_request_update(struct ct_request *req, u32 *status) err = wait_for(done, GUC_CTB_RESPONSE_TIMEOUT_LONG_MS); #undef done - if (unlikely(err)) - DRM_ERROR("CT: fence %u err %d\n", req->fence, err); - *status = req->status; return err; } @@ -722,8 +703,11 @@ retry: err = wait_for_ct_request_update(&request, status); g2h_release_space(ct, GUC_CTB_HXG_MSG_MAX_LEN); - if (unlikely(err)) + if (unlikely(err)) { + CT_ERROR(ct, "No response for request %#x (fence %u)\n", + action[0], request.fence); goto unlink; + } if (FIELD_GET(GUC_HXG_MSG_0_TYPE, *status) != GUC_HXG_TYPE_RESPONSE_SUCCESS) { err = -EIO; @@ -775,8 +759,8 @@ int intel_guc_ct_send(struct intel_guc_ct *ct, const u32 *action, u32 len, ret = ct_send(ct, action, len, response_buf, response_buf_size, &status); if (unlikely(ret < 0)) { - CT_ERROR(ct, "Sending action %#x failed (err=%d status=%#X)\n", - action[0], ret, status); + CT_ERROR(ct, "Sending action %#x failed (%pe) status=%#X\n", + action[0], ERR_PTR(ret), status); } else if (unlikely(ret)) { CT_DEBUG(ct, "send action %#x returned %d (%#x)\n", action[0], ret, ret); @@ -1042,9 +1026,9 @@ static void ct_incoming_request_worker_func(struct work_struct *w) container_of(w, struct intel_guc_ct, requests.worker); bool done; - done = ct_process_incoming_requests(ct); - if (!done) - queue_work(system_unbound_wq, &ct->requests.worker); + do { + done = ct_process_incoming_requests(ct); + } while (!done); } static int ct_handle_event(struct intel_guc_ct *ct, struct ct_incoming_msg *request) diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_debugfs.c b/drivers/gpu/drm/i915/gt/uc/intel_guc_debugfs.c index 887c8c8f35db..25f09a420561 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_guc_debugfs.c +++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_debugfs.c @@ -5,14 +5,14 @@ #include <drm/drm_print.h> -#include "gt/debugfs_gt.h" +#include "gt/intel_gt_debugfs.h" +#include "gt/uc/intel_guc_ads.h" +#include "gt/uc/intel_guc_ct.h" +#include "gt/uc/intel_guc_slpc.h" +#include "gt/uc/intel_guc_submission.h" #include "intel_guc.h" #include "intel_guc_debugfs.h" #include "intel_guc_log_debugfs.h" -#include "gt/uc/intel_guc_ct.h" -#include "gt/uc/intel_guc_ads.h" -#include "gt/uc/intel_guc_submission.h" -#include "gt/uc/intel_guc_slpc.h" static int guc_info_show(struct seq_file *m, void *data) { @@ -35,7 +35,7 @@ static int guc_info_show(struct seq_file *m, void *data) return 0; } -DEFINE_GT_DEBUGFS_ATTRIBUTE(guc_info); +DEFINE_INTEL_GT_DEBUGFS_ATTRIBUTE(guc_info); static int guc_registered_contexts_show(struct seq_file *m, void *data) { @@ -49,7 +49,7 @@ static int guc_registered_contexts_show(struct seq_file *m, void *data) return 0; } -DEFINE_GT_DEBUGFS_ATTRIBUTE(guc_registered_contexts); +DEFINE_INTEL_GT_DEBUGFS_ATTRIBUTE(guc_registered_contexts); static int guc_slpc_info_show(struct seq_file *m, void *unused) { @@ -62,7 +62,7 @@ static int guc_slpc_info_show(struct seq_file *m, void *unused) return intel_guc_slpc_print_info(slpc, &p); } -DEFINE_GT_DEBUGFS_ATTRIBUTE(guc_slpc_info); +DEFINE_INTEL_GT_DEBUGFS_ATTRIBUTE(guc_slpc_info); static bool intel_eval_slpc_support(void *data) { @@ -73,7 +73,7 @@ static bool intel_eval_slpc_support(void *data) void intel_guc_debugfs_register(struct intel_guc *guc, struct dentry *root) { - static const struct debugfs_gt_file files[] = { + static const struct intel_gt_debugfs_file files[] = { { "guc_info", &guc_info_fops, NULL }, { "guc_registered_contexts", &guc_registered_contexts_fops, NULL }, { "guc_slpc_info", &guc_slpc_info_fops, &intel_eval_slpc_support}, diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_fw.c b/drivers/gpu/drm/i915/gt/uc/intel_guc_fw.c index 76fe766ad1bc..196424be0998 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_guc_fw.c +++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_fw.c @@ -41,18 +41,21 @@ static void guc_prepare_xfer(struct intel_uncore *uncore) } /* Copy RSA signature from the fw image to HW for verification */ -static void guc_xfer_rsa(struct intel_uc_fw *guc_fw, - struct intel_uncore *uncore) +static int guc_xfer_rsa(struct intel_uc_fw *guc_fw, + struct intel_uncore *uncore) { u32 rsa[UOS_RSA_SCRATCH_COUNT]; size_t copied; int i; copied = intel_uc_fw_copy_rsa(guc_fw, rsa, sizeof(rsa)); - GEM_BUG_ON(copied < sizeof(rsa)); + if (copied < sizeof(rsa)) + return -ENOMEM; for (i = 0; i < UOS_RSA_SCRATCH_COUNT; i++) intel_uncore_write(uncore, UOS_RSA_SCRATCH(i), rsa[i]); + + return 0; } /* @@ -141,7 +144,9 @@ int intel_guc_fw_upload(struct intel_guc *guc) * by the DMA engine in one operation, whereas the RSA signature is * loaded via MMIO. */ - guc_xfer_rsa(&guc->fw, uncore); + ret = guc_xfer_rsa(&guc->fw, uncore); + if (ret) + goto out; /* * Current uCode expects the code to be loaded at 8k; locations below diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_fwif.h b/drivers/gpu/drm/i915/gt/uc/intel_guc_fwif.h index fa4be13c8854..722933e26347 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_guc_fwif.h +++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_fwif.h @@ -52,27 +52,27 @@ #define GUC_DOORBELL_INVALID 256 -#define GUC_WQ_SIZE (PAGE_SIZE * 2) - -/* Work queue item header definitions */ +/* + * Work queue item header definitions + * + * Work queue is circular buffer used to submit complex (multi-lrc) submissions + * to the GuC. A work queue item is an entry in the circular buffer. + */ #define WQ_STATUS_ACTIVE 1 #define WQ_STATUS_SUSPENDED 2 #define WQ_STATUS_CMD_ERROR 3 #define WQ_STATUS_ENGINE_ID_NOT_USED 4 #define WQ_STATUS_SUSPENDED_FROM_RESET 5 -#define WQ_TYPE_SHIFT 0 -#define WQ_TYPE_BATCH_BUF (0x1 << WQ_TYPE_SHIFT) -#define WQ_TYPE_PSEUDO (0x2 << WQ_TYPE_SHIFT) -#define WQ_TYPE_INORDER (0x3 << WQ_TYPE_SHIFT) -#define WQ_TYPE_NOOP (0x4 << WQ_TYPE_SHIFT) -#define WQ_TARGET_SHIFT 10 -#define WQ_LEN_SHIFT 16 -#define WQ_NO_WCFLUSH_WAIT (1 << 27) -#define WQ_PRESENT_WORKLOAD (1 << 28) - -#define WQ_RING_TAIL_SHIFT 20 -#define WQ_RING_TAIL_MAX 0x7FF /* 2^11 QWords */ -#define WQ_RING_TAIL_MASK (WQ_RING_TAIL_MAX << WQ_RING_TAIL_SHIFT) +#define WQ_TYPE_BATCH_BUF 0x1 +#define WQ_TYPE_PSEUDO 0x2 +#define WQ_TYPE_INORDER 0x3 +#define WQ_TYPE_NOOP 0x4 +#define WQ_TYPE_MULTI_LRC 0x5 +#define WQ_TYPE_MASK GENMASK(7, 0) +#define WQ_LEN_MASK GENMASK(26, 16) + +#define WQ_GUC_ID_MASK GENMASK(15, 0) +#define WQ_RING_TAIL_MASK GENMASK(28, 18) #define GUC_STAGE_DESC_ATTR_ACTIVE BIT(0) #define GUC_STAGE_DESC_ATTR_PENDING_DB BIT(1) @@ -186,7 +186,7 @@ struct guc_process_desc { u32 wq_status; u32 engine_presence; u32 priority; - u32 reserved[30]; + u32 reserved[36]; } __packed; #define CONTEXT_REGISTRATION_FLAG_KMD BIT(0) diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_log_debugfs.c b/drivers/gpu/drm/i915/gt/uc/intel_guc_log_debugfs.c index 64e0b86bf258..46026c2c1722 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_guc_log_debugfs.c +++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_log_debugfs.c @@ -6,7 +6,7 @@ #include <linux/fs.h> #include <drm/drm_print.h> -#include "gt/debugfs_gt.h" +#include "gt/intel_gt_debugfs.h" #include "intel_guc.h" #include "intel_guc_log.h" #include "intel_guc_log_debugfs.h" @@ -17,7 +17,7 @@ static int guc_log_dump_show(struct seq_file *m, void *data) return intel_guc_log_dump(m->private, &p, false); } -DEFINE_GT_DEBUGFS_ATTRIBUTE(guc_log_dump); +DEFINE_INTEL_GT_DEBUGFS_ATTRIBUTE(guc_log_dump); static int guc_load_err_log_dump_show(struct seq_file *m, void *data) { @@ -25,7 +25,7 @@ static int guc_load_err_log_dump_show(struct seq_file *m, void *data) return intel_guc_log_dump(m->private, &p, true); } -DEFINE_GT_DEBUGFS_ATTRIBUTE(guc_load_err_log_dump); +DEFINE_INTEL_GT_DEBUGFS_ATTRIBUTE(guc_load_err_log_dump); static int guc_log_level_get(void *data, u64 *val) { @@ -109,7 +109,7 @@ static const struct file_operations guc_log_relay_fops = { void intel_guc_log_debugfs_register(struct intel_guc_log *log, struct dentry *root) { - static const struct debugfs_gt_file files[] = { + static const struct intel_gt_debugfs_file files[] = { { "guc_log_dump", &guc_log_dump_fops, NULL }, { "guc_load_err_log_dump", &guc_load_err_log_dump_fops, NULL }, { "guc_log_level", &guc_log_level_fops, NULL }, diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c b/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c index 87d8dc8f51b9..d7710debcd47 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c +++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c @@ -11,6 +11,7 @@ #include "gt/intel_context.h" #include "gt/intel_engine_pm.h" #include "gt/intel_engine_heartbeat.h" +#include "gt/intel_gpu_commands.h" #include "gt/intel_gt.h" #include "gt/intel_gt_irq.h" #include "gt/intel_gt_pm.h" @@ -28,21 +29,6 @@ /** * DOC: GuC-based command submission * - * IMPORTANT NOTE: GuC submission is currently not supported in i915. The GuC - * firmware is moving to an updated submission interface and we plan to - * turn submission back on when that lands. The below documentation (and related - * code) matches the old submission model and will be updated as part of the - * upgrade to the new flow. - * - * GuC stage descriptor: - * During initialization, the driver allocates a static pool of 1024 such - * descriptors, and shares them with the GuC. Currently, we only use one - * descriptor. This stage descriptor lets the GuC know about the workqueue and - * process descriptor. Theoretically, it also lets the GuC know about our HW - * contexts (context ID, etc...), but we actually employ a kind of submission - * where the GuC uses the LRCA sent via the work item instead. This is called - * a "proxy" submission. - * * The Scratch registers: * There are 16 MMIO-based registers start from 0xC180. The kernel driver writes * a value to the action register (SOFT_SCRATCH_0) along with any data. It then @@ -51,14 +37,85 @@ * processes the request. The kernel driver polls waiting for this update and * then proceeds. * - * Work Items: - * There are several types of work items that the host may place into a - * workqueue, each with its own requirements and limitations. Currently only - * WQ_TYPE_INORDER is needed to support legacy submission via GuC, which - * represents in-order queue. The kernel driver packs ring tail pointer and an - * ELSP context descriptor dword into Work Item. - * See guc_add_request() + * Command Transport buffers (CTBs): + * Covered in detail in other sections but CTBs (Host to GuC - H2G, GuC to Host + * - G2H) are a message interface between the i915 and GuC. + * + * Context registration: + * Before a context can be submitted it must be registered with the GuC via a + * H2G. A unique guc_id is associated with each context. The context is either + * registered at request creation time (normal operation) or at submission time + * (abnormal operation, e.g. after a reset). + * + * Context submission: + * The i915 updates the LRC tail value in memory. The i915 must enable the + * scheduling of the context within the GuC for the GuC to actually consider it. + * Therefore, the first time a disabled context is submitted we use a schedule + * enable H2G, while follow up submissions are done via the context submit H2G, + * which informs the GuC that a previously enabled context has new work + * available. + * + * Context unpin: + * To unpin a context a H2G is used to disable scheduling. When the + * corresponding G2H returns indicating the scheduling disable operation has + * completed it is safe to unpin the context. While a disable is in flight it + * isn't safe to resubmit the context so a fence is used to stall all future + * requests of that context until the G2H is returned. + * + * Context deregistration: + * Before a context can be destroyed or if we steal its guc_id we must + * deregister the context with the GuC via H2G. If stealing the guc_id it isn't + * safe to submit anything to this guc_id until the deregister completes so a + * fence is used to stall all requests associated with this guc_id until the + * corresponding G2H returns indicating the guc_id has been deregistered. + * + * submission_state.guc_ids: + * Unique number associated with private GuC context data passed in during + * context registration / submission / deregistration. 64k available. Simple ida + * is used for allocation. + * + * Stealing guc_ids: + * If no guc_ids are available they can be stolen from another context at + * request creation time if that context is unpinned. If a guc_id can't be found + * we punt this problem to the user as we believe this is near impossible to hit + * during normal use cases. + * + * Locking: + * In the GuC submission code we have 3 basic spin locks which protect + * everything. Details about each below. + * + * sched_engine->lock + * This is the submission lock for all contexts that share an i915 schedule + * engine (sched_engine), thus only one of the contexts which share a + * sched_engine can be submitting at a time. Currently only one sched_engine is + * used for all of GuC submission but that could change in the future. + * + * guc->submission_state.lock + * Global lock for GuC submission state. Protects guc_ids and destroyed contexts + * list. + * + * ce->guc_state.lock + * Protects everything under ce->guc_state. Ensures that a context is in the + * correct state before issuing a H2G. e.g. We don't issue a schedule disable + * on a disabled context (bad idea), we don't issue a schedule enable when a + * schedule disable is in flight, etc... Also protects list of inflight requests + * on the context and the priority management state. Lock is individual to each + * context. + * + * Lock ordering rules: + * sched_engine->lock -> ce->guc_state.lock + * guc->submission_state.lock -> ce->guc_state.lock * + * Reset races: + * When a full GT reset is triggered it is assumed that some G2H responses to + * H2Gs can be lost as the GuC is also reset. Losing these G2H can prove to be + * fatal as we do certain operations upon receiving a G2H (e.g. destroy + * contexts, release guc_ids, etc...). When this occurs we can scrub the + * context state and cleanup appropriately, however this is quite racey. + * To avoid races, the reset code must disable submission before scrubbing for + * the missing G2H, while the submission code must check for submission being + * disabled and skip sending H2Gs and updating context states when it is. Both + * sides must also make sure to hold the relevant locks. */ /* GuC Virtual Engine */ @@ -68,91 +125,56 @@ struct guc_virtual_engine { }; static struct intel_context * -guc_create_virtual(struct intel_engine_cs **siblings, unsigned int count); +guc_create_virtual(struct intel_engine_cs **siblings, unsigned int count, + unsigned long flags); + +static struct intel_context * +guc_create_parallel(struct intel_engine_cs **engines, + unsigned int num_siblings, + unsigned int width); #define GUC_REQUEST_SIZE 64 /* bytes */ /* - * Below is a set of functions which control the GuC scheduling state which do - * not require a lock as all state transitions are mutually exclusive. i.e. It - * is not possible for the context pinning code and submission, for the same - * context, to be executing simultaneously. We still need an atomic as it is - * possible for some of the bits to changing at the same time though. + * We reserve 1/16 of the guc_ids for multi-lrc as these need to be contiguous + * per the GuC submission interface. A different allocation algorithm is used + * (bitmap vs. ida) between multi-lrc and single-lrc hence the reason to + * partition the guc_id space. We believe the number of multi-lrc contexts in + * use should be low and 1/16 should be sufficient. Minimum of 32 guc_ids for + * multi-lrc. */ -#define SCHED_STATE_NO_LOCK_ENABLED BIT(0) -#define SCHED_STATE_NO_LOCK_PENDING_ENABLE BIT(1) -#define SCHED_STATE_NO_LOCK_REGISTERED BIT(2) -static inline bool context_enabled(struct intel_context *ce) -{ - return (atomic_read(&ce->guc_sched_state_no_lock) & - SCHED_STATE_NO_LOCK_ENABLED); -} - -static inline void set_context_enabled(struct intel_context *ce) -{ - atomic_or(SCHED_STATE_NO_LOCK_ENABLED, &ce->guc_sched_state_no_lock); -} - -static inline void clr_context_enabled(struct intel_context *ce) -{ - atomic_and((u32)~SCHED_STATE_NO_LOCK_ENABLED, - &ce->guc_sched_state_no_lock); -} - -static inline bool context_pending_enable(struct intel_context *ce) -{ - return (atomic_read(&ce->guc_sched_state_no_lock) & - SCHED_STATE_NO_LOCK_PENDING_ENABLE); -} - -static inline void set_context_pending_enable(struct intel_context *ce) -{ - atomic_or(SCHED_STATE_NO_LOCK_PENDING_ENABLE, - &ce->guc_sched_state_no_lock); -} - -static inline void clr_context_pending_enable(struct intel_context *ce) -{ - atomic_and((u32)~SCHED_STATE_NO_LOCK_PENDING_ENABLE, - &ce->guc_sched_state_no_lock); -} - -static inline bool context_registered(struct intel_context *ce) -{ - return (atomic_read(&ce->guc_sched_state_no_lock) & - SCHED_STATE_NO_LOCK_REGISTERED); -} - -static inline void set_context_registered(struct intel_context *ce) -{ - atomic_or(SCHED_STATE_NO_LOCK_REGISTERED, - &ce->guc_sched_state_no_lock); -} - -static inline void clr_context_registered(struct intel_context *ce) -{ - atomic_and((u32)~SCHED_STATE_NO_LOCK_REGISTERED, - &ce->guc_sched_state_no_lock); -} +#define NUMBER_MULTI_LRC_GUC_ID (GUC_MAX_LRC_DESCRIPTORS / 16) /* * Below is a set of functions which control the GuC scheduling state which - * require a lock, aside from the special case where the functions are called - * from guc_lrc_desc_pin(). In that case it isn't possible for any other code - * path to be executing on the context. + * require a lock. */ #define SCHED_STATE_WAIT_FOR_DEREGISTER_TO_REGISTER BIT(0) #define SCHED_STATE_DESTROYED BIT(1) #define SCHED_STATE_PENDING_DISABLE BIT(2) #define SCHED_STATE_BANNED BIT(3) -#define SCHED_STATE_BLOCKED_SHIFT 4 +#define SCHED_STATE_ENABLED BIT(4) +#define SCHED_STATE_PENDING_ENABLE BIT(5) +#define SCHED_STATE_REGISTERED BIT(6) +#define SCHED_STATE_BLOCKED_SHIFT 7 #define SCHED_STATE_BLOCKED BIT(SCHED_STATE_BLOCKED_SHIFT) #define SCHED_STATE_BLOCKED_MASK (0xfff << SCHED_STATE_BLOCKED_SHIFT) + static inline void init_sched_state(struct intel_context *ce) { - /* Only should be called from guc_lrc_desc_pin() */ - atomic_set(&ce->guc_sched_state_no_lock, 0); - ce->guc_state.sched_state = 0; + lockdep_assert_held(&ce->guc_state.lock); + ce->guc_state.sched_state &= SCHED_STATE_BLOCKED_MASK; +} + +__maybe_unused +static bool sched_state_is_init(struct intel_context *ce) +{ + /* + * XXX: Kernel contexts can have SCHED_STATE_NO_LOCK_REGISTERED after + * suspend. + */ + return !(ce->guc_state.sched_state &= + ~(SCHED_STATE_BLOCKED_MASK | SCHED_STATE_REGISTERED)); } static inline bool @@ -165,7 +187,7 @@ context_wait_for_deregister_to_register(struct intel_context *ce) static inline void set_context_wait_for_deregister_to_register(struct intel_context *ce) { - /* Only should be called from guc_lrc_desc_pin() without lock */ + lockdep_assert_held(&ce->guc_state.lock); ce->guc_state.sched_state |= SCHED_STATE_WAIT_FOR_DEREGISTER_TO_REGISTER; } @@ -225,6 +247,57 @@ static inline void clr_context_banned(struct intel_context *ce) ce->guc_state.sched_state &= ~SCHED_STATE_BANNED; } +static inline bool context_enabled(struct intel_context *ce) +{ + return ce->guc_state.sched_state & SCHED_STATE_ENABLED; +} + +static inline void set_context_enabled(struct intel_context *ce) +{ + lockdep_assert_held(&ce->guc_state.lock); + ce->guc_state.sched_state |= SCHED_STATE_ENABLED; +} + +static inline void clr_context_enabled(struct intel_context *ce) +{ + lockdep_assert_held(&ce->guc_state.lock); + ce->guc_state.sched_state &= ~SCHED_STATE_ENABLED; +} + +static inline bool context_pending_enable(struct intel_context *ce) +{ + return ce->guc_state.sched_state & SCHED_STATE_PENDING_ENABLE; +} + +static inline void set_context_pending_enable(struct intel_context *ce) +{ + lockdep_assert_held(&ce->guc_state.lock); + ce->guc_state.sched_state |= SCHED_STATE_PENDING_ENABLE; +} + +static inline void clr_context_pending_enable(struct intel_context *ce) +{ + lockdep_assert_held(&ce->guc_state.lock); + ce->guc_state.sched_state &= ~SCHED_STATE_PENDING_ENABLE; +} + +static inline bool context_registered(struct intel_context *ce) +{ + return ce->guc_state.sched_state & SCHED_STATE_REGISTERED; +} + +static inline void set_context_registered(struct intel_context *ce) +{ + lockdep_assert_held(&ce->guc_state.lock); + ce->guc_state.sched_state |= SCHED_STATE_REGISTERED; +} + +static inline void clr_context_registered(struct intel_context *ce) +{ + lockdep_assert_held(&ce->guc_state.lock); + ce->guc_state.sched_state &= ~SCHED_STATE_REGISTERED; +} + static inline u32 context_blocked(struct intel_context *ce) { return (ce->guc_state.sched_state & SCHED_STATE_BLOCKED_MASK) >> @@ -233,7 +306,6 @@ static inline u32 context_blocked(struct intel_context *ce) static inline void incr_context_blocked(struct intel_context *ce) { - lockdep_assert_held(&ce->engine->sched_engine->lock); lockdep_assert_held(&ce->guc_state.lock); ce->guc_state.sched_state += SCHED_STATE_BLOCKED; @@ -243,7 +315,6 @@ static inline void incr_context_blocked(struct intel_context *ce) static inline void decr_context_blocked(struct intel_context *ce) { - lockdep_assert_held(&ce->engine->sched_engine->lock); lockdep_assert_held(&ce->guc_state.lock); GEM_BUG_ON(!context_blocked(ce)); /* Underflow check */ @@ -251,14 +322,39 @@ static inline void decr_context_blocked(struct intel_context *ce) ce->guc_state.sched_state -= SCHED_STATE_BLOCKED; } +static inline bool context_has_committed_requests(struct intel_context *ce) +{ + return !!ce->guc_state.number_committed_requests; +} + +static inline void incr_context_committed_requests(struct intel_context *ce) +{ + lockdep_assert_held(&ce->guc_state.lock); + ++ce->guc_state.number_committed_requests; + GEM_BUG_ON(ce->guc_state.number_committed_requests < 0); +} + +static inline void decr_context_committed_requests(struct intel_context *ce) +{ + lockdep_assert_held(&ce->guc_state.lock); + --ce->guc_state.number_committed_requests; + GEM_BUG_ON(ce->guc_state.number_committed_requests < 0); +} + +static struct intel_context * +request_to_scheduling_context(struct i915_request *rq) +{ + return intel_context_to_parent(rq->context); +} + static inline bool context_guc_id_invalid(struct intel_context *ce) { - return ce->guc_id == GUC_INVALID_LRC_ID; + return ce->guc_id.id == GUC_INVALID_LRC_ID; } static inline void set_context_guc_id_invalid(struct intel_context *ce) { - ce->guc_id = GUC_INVALID_LRC_ID; + ce->guc_id.id = GUC_INVALID_LRC_ID; } static inline struct intel_guc *ce_to_guc(struct intel_context *ce) @@ -271,6 +367,104 @@ static inline struct i915_priolist *to_priolist(struct rb_node *rb) return rb_entry(rb, struct i915_priolist, node); } +/* + * When using multi-lrc submission a scratch memory area is reserved in the + * parent's context state for the process descriptor, work queue, and handshake + * between the parent + children contexts to insert safe preemption points + * between each of the BBs. Currently the scratch area is sized to a page. + * + * The layout of this scratch area is below: + * 0 guc_process_desc + * + sizeof(struct guc_process_desc) child go + * + CACHELINE_BYTES child join[0] + * ... + * + CACHELINE_BYTES child join[n - 1] + * ... unused + * PARENT_SCRATCH_SIZE / 2 work queue start + * ... work queue + * PARENT_SCRATCH_SIZE - 1 work queue end + */ +#define WQ_SIZE (PARENT_SCRATCH_SIZE / 2) +#define WQ_OFFSET (PARENT_SCRATCH_SIZE - WQ_SIZE) + +struct sync_semaphore { + u32 semaphore; + u8 unused[CACHELINE_BYTES - sizeof(u32)]; +}; + +struct parent_scratch { + struct guc_process_desc pdesc; + + struct sync_semaphore go; + struct sync_semaphore join[MAX_ENGINE_INSTANCE + 1]; + + u8 unused[WQ_OFFSET - sizeof(struct guc_process_desc) - + sizeof(struct sync_semaphore) * (MAX_ENGINE_INSTANCE + 2)]; + + u32 wq[WQ_SIZE / sizeof(u32)]; +}; + +static u32 __get_parent_scratch_offset(struct intel_context *ce) +{ + GEM_BUG_ON(!ce->parallel.guc.parent_page); + + return ce->parallel.guc.parent_page * PAGE_SIZE; +} + +static u32 __get_wq_offset(struct intel_context *ce) +{ + BUILD_BUG_ON(offsetof(struct parent_scratch, wq) != WQ_OFFSET); + + return __get_parent_scratch_offset(ce) + WQ_OFFSET; +} + +static struct parent_scratch * +__get_parent_scratch(struct intel_context *ce) +{ + BUILD_BUG_ON(sizeof(struct parent_scratch) != PARENT_SCRATCH_SIZE); + BUILD_BUG_ON(sizeof(struct sync_semaphore) != CACHELINE_BYTES); + + /* + * Need to subtract LRC_STATE_OFFSET here as the + * parallel.guc.parent_page is the offset into ce->state while + * ce->lrc_reg_reg is ce->state + LRC_STATE_OFFSET. + */ + return (struct parent_scratch *) + (ce->lrc_reg_state + + ((__get_parent_scratch_offset(ce) - + LRC_STATE_OFFSET) / sizeof(u32))); +} + +static struct guc_process_desc * +__get_process_desc(struct intel_context *ce) +{ + struct parent_scratch *ps = __get_parent_scratch(ce); + + return &ps->pdesc; +} + +static u32 *get_wq_pointer(struct guc_process_desc *desc, + struct intel_context *ce, + u32 wqi_size) +{ + /* + * Check for space in work queue. Caching a value of head pointer in + * intel_context structure in order reduce the number accesses to shared + * GPU memory which may be across a PCIe bus. + */ +#define AVAILABLE_SPACE \ + CIRC_SPACE(ce->parallel.guc.wqi_tail, ce->parallel.guc.wqi_head, WQ_SIZE) + if (wqi_size > AVAILABLE_SPACE) { + ce->parallel.guc.wqi_head = READ_ONCE(desc->head); + + if (wqi_size > AVAILABLE_SPACE) + return NULL; + } +#undef AVAILABLE_SPACE + + return &__get_parent_scratch(ce)->wq[ce->parallel.guc.wqi_tail / sizeof(u32)]; +} + static struct guc_lrc_desc *__get_lrc_desc(struct intel_guc *guc, u32 index) { struct guc_lrc_desc *base = guc->lrc_desc_pool_vaddr; @@ -352,20 +546,29 @@ static inline void set_lrc_desc_registered(struct intel_guc *guc, u32 id, xa_unlock_irqrestore(&guc->context_lookup, flags); } +static void decr_outstanding_submission_g2h(struct intel_guc *guc) +{ + if (atomic_dec_and_test(&guc->outstanding_submission_g2h)) + wake_up_all(&guc->ct.wq); +} + static int guc_submission_send_busy_loop(struct intel_guc *guc, const u32 *action, u32 len, u32 g2h_len_dw, bool loop) { - int err; - - err = intel_guc_send_busy_loop(guc, action, len, g2h_len_dw, loop); + /* + * We always loop when a send requires a reply (i.e. g2h_len_dw > 0), + * so we don't handle the case where we don't get a reply because we + * aborted the send due to the channel being busy. + */ + GEM_BUG_ON(g2h_len_dw && !loop); - if (!err && g2h_len_dw) + if (g2h_len_dw) atomic_inc(&guc->outstanding_submission_g2h); - return err; + return intel_guc_send_busy_loop(guc, action, len, g2h_len_dw, loop); } int intel_guc_wait_for_pending_msg(struct intel_guc *guc, @@ -421,15 +624,17 @@ int intel_guc_wait_for_idle(struct intel_guc *guc, long timeout) static int guc_lrc_desc_pin(struct intel_context *ce, bool loop); -static int guc_add_request(struct intel_guc *guc, struct i915_request *rq) +static int __guc_add_request(struct intel_guc *guc, struct i915_request *rq) { int err = 0; - struct intel_context *ce = rq->context; + struct intel_context *ce = request_to_scheduling_context(rq); u32 action[3]; int len = 0; u32 g2h_len_dw = 0; bool enabled; + lockdep_assert_held(&rq->engine->sched_engine->lock); + /* * Corner case where requests were sitting in the priority list or a * request resubmitted after the context was banned. @@ -437,41 +642,34 @@ static int guc_add_request(struct intel_guc *guc, struct i915_request *rq) if (unlikely(intel_context_is_banned(ce))) { i915_request_put(i915_request_mark_eio(rq)); intel_engine_signal_breadcrumbs(ce->engine); - goto out; + return 0; } - GEM_BUG_ON(!atomic_read(&ce->guc_id_ref)); + GEM_BUG_ON(!atomic_read(&ce->guc_id.ref)); GEM_BUG_ON(context_guc_id_invalid(ce)); - /* - * Corner case where the GuC firmware was blown away and reloaded while - * this context was pinned. - */ - if (unlikely(!lrc_desc_registered(guc, ce->guc_id))) { - err = guc_lrc_desc_pin(ce, false); - if (unlikely(err)) - goto out; - } + spin_lock(&ce->guc_state.lock); /* * The request / context will be run on the hardware when scheduling - * gets enabled in the unblock. + * gets enabled in the unblock. For multi-lrc we still submit the + * context to move the LRC tails. */ - if (unlikely(context_blocked(ce))) + if (unlikely(context_blocked(ce) && !intel_context_is_parent(ce))) goto out; - enabled = context_enabled(ce); + enabled = context_enabled(ce) || context_blocked(ce); if (!enabled) { action[len++] = INTEL_GUC_ACTION_SCHED_CONTEXT_MODE_SET; - action[len++] = ce->guc_id; + action[len++] = ce->guc_id.id; action[len++] = GUC_CONTEXT_ENABLE; set_context_pending_enable(ce); intel_context_get(ce); g2h_len_dw = G2H_LEN_DW_SCHED_CONTEXT_MODE_SET; } else { action[len++] = INTEL_GUC_ACTION_SCHED_CONTEXT; - action[len++] = ce->guc_id; + action[len++] = ce->guc_id.id; } err = intel_guc_send_nb(guc, action, len, g2h_len_dw); @@ -479,6 +677,18 @@ static int guc_add_request(struct intel_guc *guc, struct i915_request *rq) trace_intel_context_sched_enable(ce); atomic_inc(&guc->outstanding_submission_g2h); set_context_enabled(ce); + + /* + * Without multi-lrc KMD does the submission step (moving the + * lrc tail) so enabling scheduling is sufficient to submit the + * context. This isn't the case in multi-lrc submission as the + * GuC needs to move the tails, hence the need for another H2G + * to submit a multi-lrc context after enabling scheduling. + */ + if (intel_context_is_parent(ce)) { + action[0] = INTEL_GUC_ACTION_SCHED_CONTEXT; + err = intel_guc_send_nb(guc, action, len - 1, 0); + } } else if (!enabled) { clr_context_pending_enable(ce); intel_context_put(ce); @@ -487,9 +697,22 @@ static int guc_add_request(struct intel_guc *guc, struct i915_request *rq) trace_i915_request_guc_submit(rq); out: + spin_unlock(&ce->guc_state.lock); return err; } +static int guc_add_request(struct intel_guc *guc, struct i915_request *rq) +{ + int ret = __guc_add_request(guc, rq); + + if (unlikely(ret == -EBUSY)) { + guc->stalled_request = rq; + guc->submission_stall_reason = STALL_ADD_REQUEST; + } + + return ret; +} + static inline void guc_set_lrc_tail(struct i915_request *rq) { rq->context->lrc_reg_state[CTX_RING_TAIL] = @@ -501,6 +724,135 @@ static inline int rq_prio(const struct i915_request *rq) return rq->sched.attr.priority; } +static bool is_multi_lrc_rq(struct i915_request *rq) +{ + return intel_context_is_parallel(rq->context); +} + +static bool can_merge_rq(struct i915_request *rq, + struct i915_request *last) +{ + return request_to_scheduling_context(rq) == + request_to_scheduling_context(last); +} + +static u32 wq_space_until_wrap(struct intel_context *ce) +{ + return (WQ_SIZE - ce->parallel.guc.wqi_tail); +} + +static void write_wqi(struct guc_process_desc *desc, + struct intel_context *ce, + u32 wqi_size) +{ + BUILD_BUG_ON(!is_power_of_2(WQ_SIZE)); + + /* + * Ensure WQI are visible before updating tail + */ + intel_guc_write_barrier(ce_to_guc(ce)); + + ce->parallel.guc.wqi_tail = (ce->parallel.guc.wqi_tail + wqi_size) & + (WQ_SIZE - 1); + WRITE_ONCE(desc->tail, ce->parallel.guc.wqi_tail); +} + +static int guc_wq_noop_append(struct intel_context *ce) +{ + struct guc_process_desc *desc = __get_process_desc(ce); + u32 *wqi = get_wq_pointer(desc, ce, wq_space_until_wrap(ce)); + u32 len_dw = wq_space_until_wrap(ce) / sizeof(u32) - 1; + + if (!wqi) + return -EBUSY; + + GEM_BUG_ON(!FIELD_FIT(WQ_LEN_MASK, len_dw)); + + *wqi = FIELD_PREP(WQ_TYPE_MASK, WQ_TYPE_NOOP) | + FIELD_PREP(WQ_LEN_MASK, len_dw); + ce->parallel.guc.wqi_tail = 0; + + return 0; +} + +static int __guc_wq_item_append(struct i915_request *rq) +{ + struct intel_context *ce = request_to_scheduling_context(rq); + struct intel_context *child; + struct guc_process_desc *desc = __get_process_desc(ce); + unsigned int wqi_size = (ce->parallel.number_children + 4) * + sizeof(u32); + u32 *wqi; + u32 len_dw = (wqi_size / sizeof(u32)) - 1; + int ret; + + /* Ensure context is in correct state updating work queue */ + GEM_BUG_ON(!atomic_read(&ce->guc_id.ref)); + GEM_BUG_ON(context_guc_id_invalid(ce)); + GEM_BUG_ON(context_wait_for_deregister_to_register(ce)); + GEM_BUG_ON(!lrc_desc_registered(ce_to_guc(ce), ce->guc_id.id)); + + /* Insert NOOP if this work queue item will wrap the tail pointer. */ + if (wqi_size > wq_space_until_wrap(ce)) { + ret = guc_wq_noop_append(ce); + if (ret) + return ret; + } + + wqi = get_wq_pointer(desc, ce, wqi_size); + if (!wqi) + return -EBUSY; + + GEM_BUG_ON(!FIELD_FIT(WQ_LEN_MASK, len_dw)); + + *wqi++ = FIELD_PREP(WQ_TYPE_MASK, WQ_TYPE_MULTI_LRC) | + FIELD_PREP(WQ_LEN_MASK, len_dw); + *wqi++ = ce->lrc.lrca; + *wqi++ = FIELD_PREP(WQ_GUC_ID_MASK, ce->guc_id.id) | + FIELD_PREP(WQ_RING_TAIL_MASK, ce->ring->tail / sizeof(u64)); + *wqi++ = 0; /* fence_id */ + for_each_child(ce, child) + *wqi++ = child->ring->tail / sizeof(u64); + + write_wqi(desc, ce, wqi_size); + + return 0; +} + +static int guc_wq_item_append(struct intel_guc *guc, + struct i915_request *rq) +{ + struct intel_context *ce = request_to_scheduling_context(rq); + int ret = 0; + + if (likely(!intel_context_is_banned(ce))) { + ret = __guc_wq_item_append(rq); + + if (unlikely(ret == -EBUSY)) { + guc->stalled_request = rq; + guc->submission_stall_reason = STALL_MOVE_LRC_TAIL; + } + } + + return ret; +} + +static bool multi_lrc_submit(struct i915_request *rq) +{ + struct intel_context *ce = request_to_scheduling_context(rq); + + intel_ring_set_tail(rq->ring, rq->tail); + + /* + * We expect the front end (execbuf IOCTL) to set this flag on the last + * request generated from a multi-BB submission. This indicates to the + * backend (GuC interface) that we should submit this context thus + * submitting all the requests generated in parallel. + */ + return test_bit(I915_FENCE_FLAG_SUBMIT_PARALLEL, &rq->fence.flags) || + intel_context_is_banned(ce); +} + static int guc_dequeue_one_context(struct intel_guc *guc) { struct i915_sched_engine * const sched_engine = guc->sched_engine; @@ -514,7 +866,17 @@ static int guc_dequeue_one_context(struct intel_guc *guc) if (guc->stalled_request) { submit = true; last = guc->stalled_request; - goto resubmit; + + switch (guc->submission_stall_reason) { + case STALL_REGISTER_CONTEXT: + goto register_context; + case STALL_MOVE_LRC_TAIL: + goto move_lrc_tail; + case STALL_ADD_REQUEST: + goto add_request; + default: + MISSING_CASE(guc->submission_stall_reason); + } } while ((rb = rb_first_cached(&sched_engine->queue))) { @@ -522,8 +884,8 @@ static int guc_dequeue_one_context(struct intel_guc *guc) struct i915_request *rq, *rn; priolist_for_each_request_consume(rq, rn, p) { - if (last && rq->context != last->context) - goto done; + if (last && !can_merge_rq(rq, last)) + goto register_context; list_del_init(&rq->sched.link); @@ -531,33 +893,84 @@ static int guc_dequeue_one_context(struct intel_guc *guc) trace_i915_request_in(rq, 0); last = rq; - submit = true; + + if (is_multi_lrc_rq(rq)) { + /* + * We need to coalesce all multi-lrc requests in + * a relationship into a single H2G. We are + * guaranteed that all of these requests will be + * submitted sequentially. + */ + if (multi_lrc_submit(rq)) { + submit = true; + goto register_context; + } + } else { + submit = true; + } } rb_erase_cached(&p->node, &sched_engine->queue); i915_priolist_free(p); } -done: + +register_context: if (submit) { - guc_set_lrc_tail(last); -resubmit: + struct intel_context *ce = request_to_scheduling_context(last); + + if (unlikely(!lrc_desc_registered(guc, ce->guc_id.id) && + !intel_context_is_banned(ce))) { + ret = guc_lrc_desc_pin(ce, false); + if (unlikely(ret == -EPIPE)) { + goto deadlk; + } else if (ret == -EBUSY) { + guc->stalled_request = last; + guc->submission_stall_reason = + STALL_REGISTER_CONTEXT; + goto schedule_tasklet; + } else if (ret != 0) { + GEM_WARN_ON(ret); /* Unexpected */ + goto deadlk; + } + } + +move_lrc_tail: + if (is_multi_lrc_rq(last)) { + ret = guc_wq_item_append(guc, last); + if (ret == -EBUSY) { + goto schedule_tasklet; + } else if (ret != 0) { + GEM_WARN_ON(ret); /* Unexpected */ + goto deadlk; + } + } else { + guc_set_lrc_tail(last); + } + +add_request: ret = guc_add_request(guc, last); - if (unlikely(ret == -EPIPE)) + if (unlikely(ret == -EPIPE)) { + goto deadlk; + } else if (ret == -EBUSY) { + goto schedule_tasklet; + } else if (ret != 0) { + GEM_WARN_ON(ret); /* Unexpected */ goto deadlk; - else if (ret == -EBUSY) { - tasklet_schedule(&sched_engine->tasklet); - guc->stalled_request = last; - return false; } } guc->stalled_request = NULL; + guc->submission_stall_reason = STALL_NONE; return submit; deadlk: sched_engine->tasklet.callback = NULL; tasklet_disable_nosync(&sched_engine->tasklet); return false; + +schedule_tasklet: + tasklet_schedule(&sched_engine->tasklet); + return false; } static void guc_submission_tasklet(struct tasklet_struct *t) @@ -596,10 +1009,18 @@ static void scrub_guc_desc_for_outstanding_g2h(struct intel_guc *guc) unsigned long index, flags; bool pending_disable, pending_enable, deregister, destroyed, banned; + xa_lock_irqsave(&guc->context_lookup, flags); xa_for_each(&guc->context_lookup, index, ce) { - /* Flush context */ - spin_lock_irqsave(&ce->guc_state.lock, flags); - spin_unlock_irqrestore(&ce->guc_state.lock, flags); + /* + * Corner case where the ref count on the object is zero but and + * deregister G2H was lost. In this case we don't touch the ref + * count and finish the destroy of the context. + */ + bool do_put = kref_get_unless_zero(&ce->ref); + + xa_unlock(&guc->context_lookup); + + spin_lock(&ce->guc_state.lock); /* * Once we are at this point submission_disabled() is guaranteed @@ -615,11 +1036,16 @@ static void scrub_guc_desc_for_outstanding_g2h(struct intel_guc *guc) banned = context_banned(ce); init_sched_state(ce); + spin_unlock(&ce->guc_state.lock); + + GEM_BUG_ON(!do_put && !destroyed); + if (pending_enable || destroyed || deregister) { - atomic_dec(&guc->outstanding_submission_g2h); + decr_outstanding_submission_g2h(guc); if (deregister) guc_signal_context_fence(ce); if (destroyed) { + intel_gt_pm_put_async(guc_to_gt(guc)); release_guc_id(guc, ce); __guc_context_destroy(ce); } @@ -635,14 +1061,20 @@ static void scrub_guc_desc_for_outstanding_g2h(struct intel_guc *guc) intel_engine_signal_breadcrumbs(ce->engine); } intel_context_sched_disable_unpin(ce); - atomic_dec(&guc->outstanding_submission_g2h); - spin_lock_irqsave(&ce->guc_state.lock, flags); + decr_outstanding_submission_g2h(guc); + + spin_lock(&ce->guc_state.lock); guc_blocked_fence_complete(ce); - spin_unlock_irqrestore(&ce->guc_state.lock, flags); + spin_unlock(&ce->guc_state.lock); intel_context_put(ce); } + + if (do_put) + intel_context_put(ce); + xa_lock(&guc->context_lookup); } + xa_unlock_irqrestore(&guc->context_lookup, flags); } static inline bool @@ -692,6 +1124,8 @@ static void guc_flush_submissions(struct intel_guc *guc) spin_unlock_irqrestore(&sched_engine->lock, flags); } +static void guc_flush_destroyed_contexts(struct intel_guc *guc); + void intel_guc_submission_reset_prepare(struct intel_guc *guc) { int i; @@ -710,6 +1144,7 @@ void intel_guc_submission_reset_prepare(struct intel_guc *guc) spin_unlock_irq(&guc_to_gt(guc)->irq_lock); guc_flush_submissions(guc); + guc_flush_destroyed_contexts(guc); /* * Handle any outstanding G2Hs before reset. Call IRQ handler directly @@ -725,6 +1160,7 @@ void intel_guc_submission_reset_prepare(struct intel_guc *guc) wait_for_reset(guc, &guc->outstanding_submission_g2h); } while (!list_empty(&guc->ct.requests.incoming)); } + scrub_guc_desc_for_outstanding_g2h(guc); } @@ -796,16 +1232,14 @@ __unwind_incomplete_requests(struct intel_context *ce) unsigned long flags; spin_lock_irqsave(&sched_engine->lock, flags); - spin_lock(&ce->guc_active.lock); - list_for_each_entry_safe(rq, rn, - &ce->guc_active.requests, - sched.link) { + spin_lock(&ce->guc_state.lock); + list_for_each_entry_safe_reverse(rq, rn, + &ce->guc_state.requests, + sched.link) { if (i915_request_completed(rq)) continue; list_del_init(&rq->sched.link); - spin_unlock(&ce->guc_active.lock); - __i915_request_unsubmit(rq); /* Push the request back into the queue for later resubmission. */ @@ -816,64 +1250,111 @@ __unwind_incomplete_requests(struct intel_context *ce) } GEM_BUG_ON(i915_sched_engine_is_empty(sched_engine)); - list_add_tail(&rq->sched.link, pl); + list_add(&rq->sched.link, pl); set_bit(I915_FENCE_FLAG_PQUEUE, &rq->fence.flags); - - spin_lock(&ce->guc_active.lock); } - spin_unlock(&ce->guc_active.lock); + spin_unlock(&ce->guc_state.lock); spin_unlock_irqrestore(&sched_engine->lock, flags); } static void __guc_reset_context(struct intel_context *ce, bool stalled) { + bool local_stalled; struct i915_request *rq; + unsigned long flags; u32 head; + int i, number_children = ce->parallel.number_children; + bool skip = false; + struct intel_context *parent = ce; + + GEM_BUG_ON(intel_context_is_child(ce)); intel_context_get(ce); /* - * GuC will implicitly mark the context as non-schedulable - * when it sends the reset notification. Make sure our state - * reflects this change. The context will be marked enabled - * on resubmission. + * GuC will implicitly mark the context as non-schedulable when it sends + * the reset notification. Make sure our state reflects this change. The + * context will be marked enabled on resubmission. + * + * XXX: If the context is reset as a result of the request cancellation + * this G2H is received after the schedule disable complete G2H which is + * wrong as this creates a race between the request cancellation code + * re-submitting the context and this G2H handler. This is a bug in the + * GuC but can be worked around in the meantime but converting this to a + * NOP if a pending enable is in flight as this indicates that a request + * cancellation has occurred. */ - clr_context_enabled(ce); + spin_lock_irqsave(&ce->guc_state.lock, flags); + if (likely(!context_pending_enable(ce))) + clr_context_enabled(ce); + else + skip = true; + spin_unlock_irqrestore(&ce->guc_state.lock, flags); + if (unlikely(skip)) + goto out_put; - rq = intel_context_find_active_request(ce); - if (!rq) { - head = ce->ring->tail; - stalled = false; - goto out_replay; - } + /* + * For each context in the relationship find the hanging request + * resetting each context / request as needed + */ + for (i = 0; i < number_children + 1; ++i) { + if (!intel_context_is_pinned(ce)) + goto next_context; + + local_stalled = false; + rq = intel_context_find_active_request(ce); + if (!rq) { + head = ce->ring->tail; + goto out_replay; + } - if (!i915_request_started(rq)) - stalled = false; + if (i915_request_started(rq)) + local_stalled = true; - GEM_BUG_ON(i915_active_is_idle(&ce->active)); - head = intel_ring_wrap(ce->ring, rq->head); - __i915_request_reset(rq, stalled); + GEM_BUG_ON(i915_active_is_idle(&ce->active)); + head = intel_ring_wrap(ce->ring, rq->head); + __i915_request_reset(rq, local_stalled && stalled); out_replay: - guc_reset_state(ce, head, stalled); - __unwind_incomplete_requests(ce); - intel_context_put(ce); + guc_reset_state(ce, head, local_stalled && stalled); +next_context: + if (i != number_children) + ce = list_next_entry(ce, parallel.child_link); + } + + __unwind_incomplete_requests(parent); +out_put: + intel_context_put(parent); } void intel_guc_submission_reset(struct intel_guc *guc, bool stalled) { struct intel_context *ce; unsigned long index; + unsigned long flags; if (unlikely(!guc_submission_initialized(guc))) { /* Reset called during driver load? GuC not yet initialised! */ return; } - xa_for_each(&guc->context_lookup, index, ce) - if (intel_context_is_pinned(ce)) + xa_lock_irqsave(&guc->context_lookup, flags); + xa_for_each(&guc->context_lookup, index, ce) { + if (!kref_get_unless_zero(&ce->ref)) + continue; + + xa_unlock(&guc->context_lookup); + + if (intel_context_is_pinned(ce) && + !intel_context_is_child(ce)) __guc_reset_context(ce, stalled); + intel_context_put(ce); + + xa_lock(&guc->context_lookup); + } + xa_unlock_irqrestore(&guc->context_lookup, flags); + /* GuC is blown away, drop all references to contexts */ xa_destroy(&guc->context_lookup); } @@ -886,10 +1367,10 @@ static void guc_cancel_context_requests(struct intel_context *ce) /* Mark all executing requests as skipped. */ spin_lock_irqsave(&sched_engine->lock, flags); - spin_lock(&ce->guc_active.lock); - list_for_each_entry(rq, &ce->guc_active.requests, sched.link) + spin_lock(&ce->guc_state.lock); + list_for_each_entry(rq, &ce->guc_state.requests, sched.link) i915_request_put(i915_request_mark_eio(rq)); - spin_unlock(&ce->guc_active.lock); + spin_unlock(&ce->guc_state.lock); spin_unlock_irqrestore(&sched_engine->lock, flags); } @@ -948,11 +1429,25 @@ void intel_guc_submission_cancel_requests(struct intel_guc *guc) { struct intel_context *ce; unsigned long index; + unsigned long flags; + + xa_lock_irqsave(&guc->context_lookup, flags); + xa_for_each(&guc->context_lookup, index, ce) { + if (!kref_get_unless_zero(&ce->ref)) + continue; - xa_for_each(&guc->context_lookup, index, ce) - if (intel_context_is_pinned(ce)) + xa_unlock(&guc->context_lookup); + + if (intel_context_is_pinned(ce) && + !intel_context_is_child(ce)) guc_cancel_context_requests(ce); + intel_context_put(ce); + + xa_lock(&guc->context_lookup); + } + xa_unlock_irqrestore(&guc->context_lookup, flags); + guc_cancel_sched_engine_requests(guc->sched_engine); /* GuC is blown away, drop all references to contexts */ @@ -981,6 +1476,8 @@ void intel_guc_submission_reset_finish(struct intel_guc *guc) intel_gt_unpark_heartbeats(guc_to_gt(guc)); } +static void destroyed_worker_func(struct work_struct *w); + /* * Set up the memory resources to be shared with the GuC (via the GGTT) * at firmware loading time. @@ -1003,9 +1500,17 @@ int intel_guc_submission_init(struct intel_guc *guc) xa_init_flags(&guc->context_lookup, XA_FLAGS_LOCK_IRQ); - spin_lock_init(&guc->contexts_lock); - INIT_LIST_HEAD(&guc->guc_id_list); - ida_init(&guc->guc_ids); + spin_lock_init(&guc->submission_state.lock); + INIT_LIST_HEAD(&guc->submission_state.guc_id_list); + ida_init(&guc->submission_state.guc_ids); + INIT_LIST_HEAD(&guc->submission_state.destroyed_contexts); + INIT_WORK(&guc->submission_state.destroyed_worker, + destroyed_worker_func); + + guc->submission_state.guc_ids_bitmap = + bitmap_zalloc(NUMBER_MULTI_LRC_GUC_ID, GFP_KERNEL); + if (!guc->submission_state.guc_ids_bitmap) + return -ENOMEM; return 0; } @@ -1015,8 +1520,10 @@ void intel_guc_submission_fini(struct intel_guc *guc) if (!guc->lrc_desc_pool) return; + guc_flush_destroyed_contexts(guc); guc_lrc_desc_pool_destroy(guc); i915_sched_engine_put(guc->sched_engine); + bitmap_free(guc->submission_state.guc_ids_bitmap); } static inline void queue_request(struct i915_sched_engine *sched_engine, @@ -1027,21 +1534,28 @@ static inline void queue_request(struct i915_sched_engine *sched_engine, list_add_tail(&rq->sched.link, i915_sched_lookup_priolist(sched_engine, prio)); set_bit(I915_FENCE_FLAG_PQUEUE, &rq->fence.flags); + tasklet_hi_schedule(&sched_engine->tasklet); } static int guc_bypass_tasklet_submit(struct intel_guc *guc, struct i915_request *rq) { - int ret; + int ret = 0; __i915_request_submit(rq); trace_i915_request_in(rq, 0); - guc_set_lrc_tail(rq); - ret = guc_add_request(guc, rq); - if (ret == -EBUSY) - guc->stalled_request = rq; + if (is_multi_lrc_rq(rq)) { + if (multi_lrc_submit(rq)) { + ret = guc_wq_item_append(guc, rq); + if (!ret) + ret = guc_add_request(guc, rq); + } + } else { + guc_set_lrc_tail(rq); + ret = guc_add_request(guc, rq); + } if (unlikely(ret == -EPIPE)) disable_submission(guc); @@ -1049,6 +1563,16 @@ static int guc_bypass_tasklet_submit(struct intel_guc *guc, return ret; } +static bool need_tasklet(struct intel_guc *guc, struct i915_request *rq) +{ + struct i915_sched_engine *sched_engine = rq->engine->sched_engine; + struct intel_context *ce = request_to_scheduling_context(rq); + + return submission_disabled(guc) || guc->stalled_request || + !i915_sched_engine_is_empty(sched_engine) || + !lrc_desc_registered(guc, ce->guc_id.id); +} + static void guc_submit_request(struct i915_request *rq) { struct i915_sched_engine *sched_engine = rq->engine->sched_engine; @@ -1058,8 +1582,7 @@ static void guc_submit_request(struct i915_request *rq) /* Will be called from irq-context when using foreign fences. */ spin_lock_irqsave(&sched_engine->lock, flags); - if (submission_disabled(guc) || guc->stalled_request || - !i915_sched_engine_is_empty(sched_engine)) + if (need_tasklet(guc, rq)) queue_request(sched_engine, rq, rq_prio(rq)); else if (guc_bypass_tasklet_submit(guc, rq) == -EBUSY) tasklet_hi_schedule(&sched_engine->tasklet); @@ -1067,72 +1590,117 @@ static void guc_submit_request(struct i915_request *rq) spin_unlock_irqrestore(&sched_engine->lock, flags); } -static int new_guc_id(struct intel_guc *guc) +static int new_guc_id(struct intel_guc *guc, struct intel_context *ce) { - return ida_simple_get(&guc->guc_ids, 0, - GUC_MAX_LRC_DESCRIPTORS, GFP_KERNEL | - __GFP_RETRY_MAYFAIL | __GFP_NOWARN); + int ret; + + GEM_BUG_ON(intel_context_is_child(ce)); + + if (intel_context_is_parent(ce)) + ret = bitmap_find_free_region(guc->submission_state.guc_ids_bitmap, + NUMBER_MULTI_LRC_GUC_ID, + order_base_2(ce->parallel.number_children + + 1)); + else + ret = ida_simple_get(&guc->submission_state.guc_ids, + NUMBER_MULTI_LRC_GUC_ID, + GUC_MAX_LRC_DESCRIPTORS, + GFP_KERNEL | __GFP_RETRY_MAYFAIL | + __GFP_NOWARN); + if (unlikely(ret < 0)) + return ret; + + ce->guc_id.id = ret; + return 0; } static void __release_guc_id(struct intel_guc *guc, struct intel_context *ce) { + GEM_BUG_ON(intel_context_is_child(ce)); + if (!context_guc_id_invalid(ce)) { - ida_simple_remove(&guc->guc_ids, ce->guc_id); - reset_lrc_desc(guc, ce->guc_id); + if (intel_context_is_parent(ce)) + bitmap_release_region(guc->submission_state.guc_ids_bitmap, + ce->guc_id.id, + order_base_2(ce->parallel.number_children + + 1)); + else + ida_simple_remove(&guc->submission_state.guc_ids, + ce->guc_id.id); + reset_lrc_desc(guc, ce->guc_id.id); set_context_guc_id_invalid(ce); } - if (!list_empty(&ce->guc_id_link)) - list_del_init(&ce->guc_id_link); + if (!list_empty(&ce->guc_id.link)) + list_del_init(&ce->guc_id.link); } static void release_guc_id(struct intel_guc *guc, struct intel_context *ce) { unsigned long flags; - spin_lock_irqsave(&guc->contexts_lock, flags); + spin_lock_irqsave(&guc->submission_state.lock, flags); __release_guc_id(guc, ce); - spin_unlock_irqrestore(&guc->contexts_lock, flags); + spin_unlock_irqrestore(&guc->submission_state.lock, flags); } -static int steal_guc_id(struct intel_guc *guc) +static int steal_guc_id(struct intel_guc *guc, struct intel_context *ce) { - struct intel_context *ce; - int guc_id; + struct intel_context *cn; - lockdep_assert_held(&guc->contexts_lock); + lockdep_assert_held(&guc->submission_state.lock); + GEM_BUG_ON(intel_context_is_child(ce)); + GEM_BUG_ON(intel_context_is_parent(ce)); - if (!list_empty(&guc->guc_id_list)) { - ce = list_first_entry(&guc->guc_id_list, + if (!list_empty(&guc->submission_state.guc_id_list)) { + cn = list_first_entry(&guc->submission_state.guc_id_list, struct intel_context, - guc_id_link); + guc_id.link); - GEM_BUG_ON(atomic_read(&ce->guc_id_ref)); - GEM_BUG_ON(context_guc_id_invalid(ce)); + GEM_BUG_ON(atomic_read(&cn->guc_id.ref)); + GEM_BUG_ON(context_guc_id_invalid(cn)); + GEM_BUG_ON(intel_context_is_child(cn)); + GEM_BUG_ON(intel_context_is_parent(cn)); - list_del_init(&ce->guc_id_link); - guc_id = ce->guc_id; - clr_context_registered(ce); - set_context_guc_id_invalid(ce); - return guc_id; + list_del_init(&cn->guc_id.link); + ce->guc_id = cn->guc_id; + + spin_lock(&ce->guc_state.lock); + clr_context_registered(cn); + spin_unlock(&ce->guc_state.lock); + + set_context_guc_id_invalid(cn); + + return 0; } else { return -EAGAIN; } } -static int assign_guc_id(struct intel_guc *guc, u16 *out) +static int assign_guc_id(struct intel_guc *guc, struct intel_context *ce) { int ret; - lockdep_assert_held(&guc->contexts_lock); + lockdep_assert_held(&guc->submission_state.lock); + GEM_BUG_ON(intel_context_is_child(ce)); - ret = new_guc_id(guc); + ret = new_guc_id(guc, ce); if (unlikely(ret < 0)) { - ret = steal_guc_id(guc); + if (intel_context_is_parent(ce)) + return -ENOSPC; + + ret = steal_guc_id(guc, ce); if (ret < 0) return ret; } - *out = ret; + if (intel_context_is_parent(ce)) { + struct intel_context *child; + int i = 1; + + for_each_child(ce, child) + child->guc_id.id = ce->guc_id.id + i++; + } + return 0; } @@ -1142,26 +1710,28 @@ static int pin_guc_id(struct intel_guc *guc, struct intel_context *ce) int ret = 0; unsigned long flags, tries = PIN_GUC_ID_TRIES; - GEM_BUG_ON(atomic_read(&ce->guc_id_ref)); + GEM_BUG_ON(atomic_read(&ce->guc_id.ref)); try_again: - spin_lock_irqsave(&guc->contexts_lock, flags); + spin_lock_irqsave(&guc->submission_state.lock, flags); + + might_lock(&ce->guc_state.lock); if (context_guc_id_invalid(ce)) { - ret = assign_guc_id(guc, &ce->guc_id); + ret = assign_guc_id(guc, ce); if (ret) goto out_unlock; ret = 1; /* Indidcates newly assigned guc_id */ } - if (!list_empty(&ce->guc_id_link)) - list_del_init(&ce->guc_id_link); - atomic_inc(&ce->guc_id_ref); + if (!list_empty(&ce->guc_id.link)) + list_del_init(&ce->guc_id.link); + atomic_inc(&ce->guc_id.ref); out_unlock: - spin_unlock_irqrestore(&guc->contexts_lock, flags); + spin_unlock_irqrestore(&guc->submission_state.lock, flags); /* - * -EAGAIN indicates no guc_ids are available, let's retire any + * -EAGAIN indicates no guc_id are available, let's retire any * outstanding requests to see if that frees up a guc_id. If the first * retire didn't help, insert a sleep with the timeslice duration before * attempting to retire more requests. Double the sleep period each @@ -1189,16 +1759,43 @@ static void unpin_guc_id(struct intel_guc *guc, struct intel_context *ce) { unsigned long flags; - GEM_BUG_ON(atomic_read(&ce->guc_id_ref) < 0); + GEM_BUG_ON(atomic_read(&ce->guc_id.ref) < 0); + GEM_BUG_ON(intel_context_is_child(ce)); - if (unlikely(context_guc_id_invalid(ce))) + if (unlikely(context_guc_id_invalid(ce) || + intel_context_is_parent(ce))) return; - spin_lock_irqsave(&guc->contexts_lock, flags); - if (!context_guc_id_invalid(ce) && list_empty(&ce->guc_id_link) && - !atomic_read(&ce->guc_id_ref)) - list_add_tail(&ce->guc_id_link, &guc->guc_id_list); - spin_unlock_irqrestore(&guc->contexts_lock, flags); + spin_lock_irqsave(&guc->submission_state.lock, flags); + if (!context_guc_id_invalid(ce) && list_empty(&ce->guc_id.link) && + !atomic_read(&ce->guc_id.ref)) + list_add_tail(&ce->guc_id.link, + &guc->submission_state.guc_id_list); + spin_unlock_irqrestore(&guc->submission_state.lock, flags); +} + +static int __guc_action_register_multi_lrc(struct intel_guc *guc, + struct intel_context *ce, + u32 guc_id, + u32 offset, + bool loop) +{ + struct intel_context *child; + u32 action[4 + MAX_ENGINE_INSTANCE]; + int len = 0; + + GEM_BUG_ON(ce->parallel.number_children > MAX_ENGINE_INSTANCE); + + action[len++] = INTEL_GUC_ACTION_REGISTER_CONTEXT_MULTI_LRC; + action[len++] = guc_id; + action[len++] = ce->parallel.number_children + 1; + action[len++] = offset; + for_each_child(ce, child) { + offset += sizeof(struct guc_lrc_desc); + action[len++] = offset; + } + + return guc_submission_send_busy_loop(guc, action, len, 0, loop); } static int __guc_action_register_context(struct intel_guc *guc, @@ -1220,21 +1817,31 @@ static int register_context(struct intel_context *ce, bool loop) { struct intel_guc *guc = ce_to_guc(ce); u32 offset = intel_guc_ggtt_offset(guc, guc->lrc_desc_pool) + - ce->guc_id * sizeof(struct guc_lrc_desc); + ce->guc_id.id * sizeof(struct guc_lrc_desc); int ret; + GEM_BUG_ON(intel_context_is_child(ce)); trace_intel_context_register(ce); - ret = __guc_action_register_context(guc, ce->guc_id, offset, loop); - if (likely(!ret)) + if (intel_context_is_parent(ce)) + ret = __guc_action_register_multi_lrc(guc, ce, ce->guc_id.id, + offset, loop); + else + ret = __guc_action_register_context(guc, ce->guc_id.id, offset, + loop); + if (likely(!ret)) { + unsigned long flags; + + spin_lock_irqsave(&ce->guc_state.lock, flags); set_context_registered(ce); + spin_unlock_irqrestore(&ce->guc_state.lock, flags); + } return ret; } static int __guc_action_deregister_context(struct intel_guc *guc, - u32 guc_id, - bool loop) + u32 guc_id) { u32 action[] = { INTEL_GUC_ACTION_DEREGISTER_CONTEXT, @@ -1243,33 +1850,38 @@ static int __guc_action_deregister_context(struct intel_guc *guc, return guc_submission_send_busy_loop(guc, action, ARRAY_SIZE(action), G2H_LEN_DW_DEREGISTER_CONTEXT, - loop); + true); } -static int deregister_context(struct intel_context *ce, u32 guc_id, bool loop) +static int deregister_context(struct intel_context *ce, u32 guc_id) { struct intel_guc *guc = ce_to_guc(ce); + GEM_BUG_ON(intel_context_is_child(ce)); trace_intel_context_deregister(ce); - return __guc_action_deregister_context(guc, guc_id, loop); + return __guc_action_deregister_context(guc, guc_id); } -static intel_engine_mask_t adjust_engine_mask(u8 class, intel_engine_mask_t mask) +static inline void clear_children_join_go_memory(struct intel_context *ce) { - switch (class) { - case RENDER_CLASS: - return mask >> RCS0; - case VIDEO_ENHANCEMENT_CLASS: - return mask >> VECS0; - case VIDEO_DECODE_CLASS: - return mask >> VCS0; - case COPY_ENGINE_CLASS: - return mask >> BCS0; - default: - MISSING_CASE(class); - return 0; - } + struct parent_scratch *ps = __get_parent_scratch(ce); + int i; + + ps->go.semaphore = 0; + for (i = 0; i < ce->parallel.number_children + 1; ++i) + ps->join[i].semaphore = 0; +} + +static inline u32 get_children_go_value(struct intel_context *ce) +{ + return __get_parent_scratch(ce)->go.semaphore; +} + +static inline u32 get_children_join_value(struct intel_context *ce, + u8 child_index) +{ + return __get_parent_scratch(ce)->join[child_index].semaphore; } static void guc_context_policy_init(struct intel_engine_cs *engine, @@ -1285,22 +1897,20 @@ static void guc_context_policy_init(struct intel_engine_cs *engine, desc->preemption_timeout = engine->props.preempt_timeout_ms * 1000; } -static inline u8 map_i915_prio_to_guc_prio(int prio); - static int guc_lrc_desc_pin(struct intel_context *ce, bool loop) { struct intel_engine_cs *engine = ce->engine; struct intel_runtime_pm *runtime_pm = engine->uncore->rpm; struct intel_guc *guc = &engine->gt->uc.guc; - u32 desc_idx = ce->guc_id; + u32 desc_idx = ce->guc_id.id; struct guc_lrc_desc *desc; - const struct i915_gem_context *ctx; - int prio = I915_CONTEXT_DEFAULT_PRIORITY; bool context_registered; intel_wakeref_t wakeref; + struct intel_context *child; int ret = 0; GEM_BUG_ON(!engine->mask); + GEM_BUG_ON(!sched_state_is_init(ce)); /* * Ensure LRC + CT vmas are is same region as write barrier is done @@ -1311,25 +1921,53 @@ static int guc_lrc_desc_pin(struct intel_context *ce, bool loop) context_registered = lrc_desc_registered(guc, desc_idx); - rcu_read_lock(); - ctx = rcu_dereference(ce->gem_context); - if (ctx) - prio = ctx->sched.priority; - rcu_read_unlock(); - reset_lrc_desc(guc, desc_idx); set_lrc_desc_registered(guc, desc_idx, ce); desc = __get_lrc_desc(guc, desc_idx); desc->engine_class = engine_class_to_guc_class(engine->class); - desc->engine_submit_mask = adjust_engine_mask(engine->class, - engine->mask); + desc->engine_submit_mask = engine->logical_mask; desc->hw_context_desc = ce->lrc.lrca; - ce->guc_prio = map_i915_prio_to_guc_prio(prio); - desc->priority = ce->guc_prio; + desc->priority = ce->guc_state.prio; desc->context_flags = CONTEXT_REGISTRATION_FLAG_KMD; guc_context_policy_init(engine, desc); - init_sched_state(ce); + + /* + * If context is a parent, we need to register a process descriptor + * describing a work queue and register all child contexts. + */ + if (intel_context_is_parent(ce)) { + struct guc_process_desc *pdesc; + + ce->parallel.guc.wqi_tail = 0; + ce->parallel.guc.wqi_head = 0; + + desc->process_desc = i915_ggtt_offset(ce->state) + + __get_parent_scratch_offset(ce); + desc->wq_addr = i915_ggtt_offset(ce->state) + + __get_wq_offset(ce); + desc->wq_size = WQ_SIZE; + + pdesc = __get_process_desc(ce); + memset(pdesc, 0, sizeof(*(pdesc))); + pdesc->stage_id = ce->guc_id.id; + pdesc->wq_base_addr = desc->wq_addr; + pdesc->wq_size_bytes = desc->wq_size; + pdesc->wq_status = WQ_STATUS_ACTIVE; + + for_each_child(ce, child) { + desc = __get_lrc_desc(guc, child->guc_id.id); + + desc->engine_class = + engine_class_to_guc_class(engine->class); + desc->hw_context_desc = child->lrc.lrca; + desc->priority = ce->guc_state.prio; + desc->context_flags = CONTEXT_REGISTRATION_FLAG_KMD; + guc_context_policy_init(engine, desc); + } + + clear_children_join_go_memory(ce); + } /* * The context_lookup xarray is used to determine if the hardware @@ -1340,26 +1978,23 @@ static int guc_lrc_desc_pin(struct intel_context *ce, bool loop) * registering this context. */ if (context_registered) { + bool disabled; + unsigned long flags; + trace_intel_context_steal_guc_id(ce); - if (!loop) { + GEM_BUG_ON(!loop); + + /* Seal race with Reset */ + spin_lock_irqsave(&ce->guc_state.lock, flags); + disabled = submission_disabled(guc); + if (likely(!disabled)) { set_context_wait_for_deregister_to_register(ce); intel_context_get(ce); - } else { - bool disabled; - unsigned long flags; - - /* Seal race with Reset */ - spin_lock_irqsave(&ce->guc_state.lock, flags); - disabled = submission_disabled(guc); - if (likely(!disabled)) { - set_context_wait_for_deregister_to_register(ce); - intel_context_get(ce); - } - spin_unlock_irqrestore(&ce->guc_state.lock, flags); - if (unlikely(disabled)) { - reset_lrc_desc(guc, desc_idx); - return 0; /* Will get registered later */ - } + } + spin_unlock_irqrestore(&ce->guc_state.lock, flags); + if (unlikely(disabled)) { + reset_lrc_desc(guc, desc_idx); + return 0; /* Will get registered later */ } /* @@ -1367,20 +2002,18 @@ static int guc_lrc_desc_pin(struct intel_context *ce, bool loop) * context whose guc_id was stolen. */ with_intel_runtime_pm(runtime_pm, wakeref) - ret = deregister_context(ce, ce->guc_id, loop); - if (unlikely(ret == -EBUSY)) { - clr_context_wait_for_deregister_to_register(ce); - intel_context_put(ce); - } else if (unlikely(ret == -ENODEV)) { + ret = deregister_context(ce, ce->guc_id.id); + if (unlikely(ret == -ENODEV)) ret = 0; /* Will get registered later */ - } } else { with_intel_runtime_pm(runtime_pm, wakeref) ret = register_context(ce, loop); - if (unlikely(ret == -EBUSY)) + if (unlikely(ret == -EBUSY)) { + reset_lrc_desc(guc, desc_idx); + } else if (unlikely(ret == -ENODEV)) { reset_lrc_desc(guc, desc_idx); - else if (unlikely(ret == -ENODEV)) ret = 0; /* Will get registered later */ + } } return ret; @@ -1419,7 +2052,12 @@ static int guc_context_pre_pin(struct intel_context *ce, static int guc_context_pin(struct intel_context *ce, void *vaddr) { - return __guc_context_pin(ce, ce->engine, vaddr); + int ret = __guc_context_pin(ce, ce->engine, vaddr); + + if (likely(!ret && !intel_context_is_barrier(ce))) + intel_engine_pm_get(ce->engine); + + return ret; } static void guc_context_unpin(struct intel_context *ce) @@ -1428,6 +2066,9 @@ static void guc_context_unpin(struct intel_context *ce) unpin_guc_id(guc, ce); lrc_unpin(ce); + + if (likely(!intel_context_is_barrier(ce))) + intel_engine_pm_put_async(ce->engine); } static void guc_context_post_unpin(struct intel_context *ce) @@ -1440,7 +2081,7 @@ static void __guc_context_sched_enable(struct intel_guc *guc, { u32 action[] = { INTEL_GUC_ACTION_SCHED_CONTEXT_MODE_SET, - ce->guc_id, + ce->guc_id.id, GUC_CONTEXT_ENABLE }; @@ -1456,12 +2097,13 @@ static void __guc_context_sched_disable(struct intel_guc *guc, { u32 action[] = { INTEL_GUC_ACTION_SCHED_CONTEXT_MODE_SET, - guc_id, /* ce->guc_id not stable */ + guc_id, /* ce->guc_id.id not stable */ GUC_CONTEXT_DISABLE }; GEM_BUG_ON(guc_id == GUC_INVALID_LRC_ID); + GEM_BUG_ON(intel_context_is_child(ce)); trace_intel_context_sched_disable(ce); guc_submission_send_busy_loop(guc, action, ARRAY_SIZE(action), @@ -1472,24 +2114,24 @@ static void guc_blocked_fence_complete(struct intel_context *ce) { lockdep_assert_held(&ce->guc_state.lock); - if (!i915_sw_fence_done(&ce->guc_blocked)) - i915_sw_fence_complete(&ce->guc_blocked); + if (!i915_sw_fence_done(&ce->guc_state.blocked)) + i915_sw_fence_complete(&ce->guc_state.blocked); } static void guc_blocked_fence_reinit(struct intel_context *ce) { lockdep_assert_held(&ce->guc_state.lock); - GEM_BUG_ON(!i915_sw_fence_done(&ce->guc_blocked)); + GEM_BUG_ON(!i915_sw_fence_done(&ce->guc_state.blocked)); /* * This fence is always complete unless a pending schedule disable is * outstanding. We arm the fence here and complete it when we receive * the pending schedule disable complete message. */ - i915_sw_fence_fini(&ce->guc_blocked); - i915_sw_fence_reinit(&ce->guc_blocked); - i915_sw_fence_await(&ce->guc_blocked); - i915_sw_fence_commit(&ce->guc_blocked); + i915_sw_fence_fini(&ce->guc_state.blocked); + i915_sw_fence_reinit(&ce->guc_state.blocked); + i915_sw_fence_await(&ce->guc_state.blocked); + i915_sw_fence_commit(&ce->guc_state.blocked); } static u16 prep_context_pending_disable(struct intel_context *ce) @@ -1501,35 +2143,30 @@ static u16 prep_context_pending_disable(struct intel_context *ce) guc_blocked_fence_reinit(ce); intel_context_get(ce); - return ce->guc_id; + return ce->guc_id.id; } static struct i915_sw_fence *guc_context_block(struct intel_context *ce) { struct intel_guc *guc = ce_to_guc(ce); - struct i915_sched_engine *sched_engine = ce->engine->sched_engine; unsigned long flags; struct intel_runtime_pm *runtime_pm = ce->engine->uncore->rpm; intel_wakeref_t wakeref; u16 guc_id; bool enabled; + GEM_BUG_ON(intel_context_is_child(ce)); + spin_lock_irqsave(&ce->guc_state.lock, flags); - /* - * Sync with submission path, increment before below changes to context - * state. - */ - spin_lock(&sched_engine->lock); incr_context_blocked(ce); - spin_unlock(&sched_engine->lock); enabled = context_enabled(ce); if (unlikely(!enabled || submission_disabled(guc))) { if (enabled) clr_context_enabled(ce); spin_unlock_irqrestore(&ce->guc_state.lock, flags); - return &ce->guc_blocked; + return &ce->guc_state.blocked; } /* @@ -1545,26 +2182,41 @@ static struct i915_sw_fence *guc_context_block(struct intel_context *ce) with_intel_runtime_pm(runtime_pm, wakeref) __guc_context_sched_disable(guc, ce, guc_id); - return &ce->guc_blocked; + return &ce->guc_state.blocked; +} + +#define SCHED_STATE_MULTI_BLOCKED_MASK \ + (SCHED_STATE_BLOCKED_MASK & ~SCHED_STATE_BLOCKED) +#define SCHED_STATE_NO_UNBLOCK \ + (SCHED_STATE_MULTI_BLOCKED_MASK | \ + SCHED_STATE_PENDING_DISABLE | \ + SCHED_STATE_BANNED) + +static bool context_cant_unblock(struct intel_context *ce) +{ + lockdep_assert_held(&ce->guc_state.lock); + + return (ce->guc_state.sched_state & SCHED_STATE_NO_UNBLOCK) || + context_guc_id_invalid(ce) || + !lrc_desc_registered(ce_to_guc(ce), ce->guc_id.id) || + !intel_context_is_pinned(ce); } static void guc_context_unblock(struct intel_context *ce) { struct intel_guc *guc = ce_to_guc(ce); - struct i915_sched_engine *sched_engine = ce->engine->sched_engine; unsigned long flags; struct intel_runtime_pm *runtime_pm = ce->engine->uncore->rpm; intel_wakeref_t wakeref; bool enable; GEM_BUG_ON(context_enabled(ce)); + GEM_BUG_ON(intel_context_is_child(ce)); spin_lock_irqsave(&ce->guc_state.lock, flags); if (unlikely(submission_disabled(guc) || - !intel_context_is_pinned(ce) || - context_pending_disable(ce) || - context_blocked(ce) > 1)) { + context_cant_unblock(ce))) { enable = false; } else { enable = true; @@ -1573,13 +2225,7 @@ static void guc_context_unblock(struct intel_context *ce) intel_context_get(ce); } - /* - * Sync with submission path, decrement after above changes to context - * state. - */ - spin_lock(&sched_engine->lock); decr_context_blocked(ce); - spin_unlock(&sched_engine->lock); spin_unlock_irqrestore(&ce->guc_state.lock, flags); @@ -1592,16 +2238,29 @@ static void guc_context_unblock(struct intel_context *ce) static void guc_context_cancel_request(struct intel_context *ce, struct i915_request *rq) { + struct intel_context *block_context = + request_to_scheduling_context(rq); + if (i915_sw_fence_signaled(&rq->submit)) { - struct i915_sw_fence *fence = guc_context_block(ce); + struct i915_sw_fence *fence; + intel_context_get(ce); + fence = guc_context_block(block_context); i915_sw_fence_wait(fence); if (!i915_request_completed(rq)) { __i915_request_skip(rq); guc_reset_state(ce, intel_ring_wrap(ce->ring, rq->head), true); } - guc_context_unblock(ce); + + /* + * XXX: Racey if context is reset, see comment in + * __guc_reset_context(). + */ + flush_work(&ce_to_guc(ce)->ct.requests.worker); + + guc_context_unblock(block_context); + intel_context_put(ce); } } @@ -1626,6 +2285,8 @@ static void guc_context_ban(struct intel_context *ce, struct i915_request *rq) intel_wakeref_t wakeref; unsigned long flags; + GEM_BUG_ON(intel_context_is_child(ce)); + guc_flush_submissions(guc); spin_lock_irqsave(&ce->guc_state.lock, flags); @@ -1662,7 +2323,7 @@ static void guc_context_ban(struct intel_context *ce, struct i915_request *rq) if (!context_guc_id_invalid(ce)) with_intel_runtime_pm(runtime_pm, wakeref) __guc_context_set_preemption_timeout(guc, - ce->guc_id, + ce->guc_id.id, 1); spin_unlock_irqrestore(&ce->guc_state.lock, flags); } @@ -1675,40 +2336,24 @@ static void guc_context_sched_disable(struct intel_context *ce) struct intel_runtime_pm *runtime_pm = &ce->engine->gt->i915->runtime_pm; intel_wakeref_t wakeref; u16 guc_id; - bool enabled; - if (submission_disabled(guc) || context_guc_id_invalid(ce) || - !lrc_desc_registered(guc, ce->guc_id)) { - clr_context_enabled(ce); - goto unpin; - } - - if (!context_enabled(ce)) - goto unpin; + GEM_BUG_ON(intel_context_is_child(ce)); spin_lock_irqsave(&ce->guc_state.lock, flags); /* - * We have to check if the context has been disabled by another thread. - * We also have to check if the context has been pinned again as another - * pin operation is allowed to pass this function. Checking the pin - * count, within ce->guc_state.lock, synchronizes this function with - * guc_request_alloc ensuring a request doesn't slip through the - * 'context_pending_disable' fence. Checking within the spin lock (can't - * sleep) ensures another process doesn't pin this context and generate - * a request before we set the 'context_pending_disable' flag here. + * We have to check if the context has been disabled by another thread, + * check if submssion has been disabled to seal a race with reset and + * finally check if any more requests have been committed to the + * context ensursing that a request doesn't slip through the + * 'context_pending_disable' fence. */ - enabled = context_enabled(ce); - if (unlikely(!enabled || submission_disabled(guc))) { - if (enabled) - clr_context_enabled(ce); + if (unlikely(!context_enabled(ce) || submission_disabled(guc) || + context_has_committed_requests(ce))) { + clr_context_enabled(ce); spin_unlock_irqrestore(&ce->guc_state.lock, flags); goto unpin; } - if (unlikely(atomic_add_unless(&ce->pin_count, -2, 2))) { - spin_unlock_irqrestore(&ce->guc_state.lock, flags); - return; - } guc_id = prep_context_pending_disable(ce); spin_unlock_irqrestore(&ce->guc_state.lock, flags); @@ -1724,21 +2369,40 @@ unpin: static inline void guc_lrc_desc_unpin(struct intel_context *ce) { struct intel_guc *guc = ce_to_guc(ce); + struct intel_gt *gt = guc_to_gt(guc); + unsigned long flags; + bool disabled; - GEM_BUG_ON(!lrc_desc_registered(guc, ce->guc_id)); - GEM_BUG_ON(ce != __get_context(guc, ce->guc_id)); + GEM_BUG_ON(!intel_gt_pm_is_awake(gt)); + GEM_BUG_ON(!lrc_desc_registered(guc, ce->guc_id.id)); + GEM_BUG_ON(ce != __get_context(guc, ce->guc_id.id)); GEM_BUG_ON(context_enabled(ce)); - clr_context_registered(ce); - deregister_context(ce, ce->guc_id, true); + /* Seal race with Reset */ + spin_lock_irqsave(&ce->guc_state.lock, flags); + disabled = submission_disabled(guc); + if (likely(!disabled)) { + __intel_gt_pm_get(gt); + set_context_destroyed(ce); + clr_context_registered(ce); + } + spin_unlock_irqrestore(&ce->guc_state.lock, flags); + if (unlikely(disabled)) { + release_guc_id(guc, ce); + __guc_context_destroy(ce); + return; + } + + deregister_context(ce, ce->guc_id.id); } static void __guc_context_destroy(struct intel_context *ce) { - GEM_BUG_ON(ce->guc_prio_count[GUC_CLIENT_PRIORITY_KMD_HIGH] || - ce->guc_prio_count[GUC_CLIENT_PRIORITY_HIGH] || - ce->guc_prio_count[GUC_CLIENT_PRIORITY_KMD_NORMAL] || - ce->guc_prio_count[GUC_CLIENT_PRIORITY_NORMAL]); + GEM_BUG_ON(ce->guc_state.prio_count[GUC_CLIENT_PRIORITY_KMD_HIGH] || + ce->guc_state.prio_count[GUC_CLIENT_PRIORITY_HIGH] || + ce->guc_state.prio_count[GUC_CLIENT_PRIORITY_KMD_NORMAL] || + ce->guc_state.prio_count[GUC_CLIENT_PRIORITY_NORMAL]); + GEM_BUG_ON(ce->guc_state.number_committed_requests); lrc_fini(ce); intel_context_fini(ce); @@ -1756,76 +2420,86 @@ static void __guc_context_destroy(struct intel_context *ce) } } +static void guc_flush_destroyed_contexts(struct intel_guc *guc) +{ + struct intel_context *ce, *cn; + unsigned long flags; + + GEM_BUG_ON(!submission_disabled(guc) && + guc_submission_initialized(guc)); + + spin_lock_irqsave(&guc->submission_state.lock, flags); + list_for_each_entry_safe(ce, cn, + &guc->submission_state.destroyed_contexts, + destroyed_link) { + list_del_init(&ce->destroyed_link); + __release_guc_id(guc, ce); + __guc_context_destroy(ce); + } + spin_unlock_irqrestore(&guc->submission_state.lock, flags); +} + +static void deregister_destroyed_contexts(struct intel_guc *guc) +{ + struct intel_context *ce, *cn; + unsigned long flags; + + spin_lock_irqsave(&guc->submission_state.lock, flags); + list_for_each_entry_safe(ce, cn, + &guc->submission_state.destroyed_contexts, + destroyed_link) { + list_del_init(&ce->destroyed_link); + guc_lrc_desc_unpin(ce); + } + spin_unlock_irqrestore(&guc->submission_state.lock, flags); +} + +static void destroyed_worker_func(struct work_struct *w) +{ + struct intel_guc *guc = container_of(w, struct intel_guc, + submission_state.destroyed_worker); + struct intel_gt *gt = guc_to_gt(guc); + int tmp; + + with_intel_gt_pm(gt, tmp) + deregister_destroyed_contexts(guc); +} + static void guc_context_destroy(struct kref *kref) { struct intel_context *ce = container_of(kref, typeof(*ce), ref); - struct intel_runtime_pm *runtime_pm = ce->engine->uncore->rpm; struct intel_guc *guc = ce_to_guc(ce); - intel_wakeref_t wakeref; unsigned long flags; - bool disabled; + bool destroy; /* * If the guc_id is invalid this context has been stolen and we can free * it immediately. Also can be freed immediately if the context is not * registered with the GuC or the GuC is in the middle of a reset. */ - if (context_guc_id_invalid(ce)) { - __guc_context_destroy(ce); - return; - } else if (submission_disabled(guc) || - !lrc_desc_registered(guc, ce->guc_id)) { - release_guc_id(guc, ce); - __guc_context_destroy(ce); - return; - } - - /* - * We have to acquire the context spinlock and check guc_id again, if it - * is valid it hasn't been stolen and needs to be deregistered. We - * delete this context from the list of unpinned guc_ids available to - * steal to seal a race with guc_lrc_desc_pin(). When the G2H CTB - * returns indicating this context has been deregistered the guc_id is - * returned to the pool of available guc_ids. - */ - spin_lock_irqsave(&guc->contexts_lock, flags); - if (context_guc_id_invalid(ce)) { - spin_unlock_irqrestore(&guc->contexts_lock, flags); - __guc_context_destroy(ce); - return; + spin_lock_irqsave(&guc->submission_state.lock, flags); + destroy = submission_disabled(guc) || context_guc_id_invalid(ce) || + !lrc_desc_registered(guc, ce->guc_id.id); + if (likely(!destroy)) { + if (!list_empty(&ce->guc_id.link)) + list_del_init(&ce->guc_id.link); + list_add_tail(&ce->destroyed_link, + &guc->submission_state.destroyed_contexts); + } else { + __release_guc_id(guc, ce); } - - if (!list_empty(&ce->guc_id_link)) - list_del_init(&ce->guc_id_link); - spin_unlock_irqrestore(&guc->contexts_lock, flags); - - /* Seal race with Reset */ - spin_lock_irqsave(&ce->guc_state.lock, flags); - disabled = submission_disabled(guc); - if (likely(!disabled)) - set_context_destroyed(ce); - spin_unlock_irqrestore(&ce->guc_state.lock, flags); - if (unlikely(disabled)) { - release_guc_id(guc, ce); + spin_unlock_irqrestore(&guc->submission_state.lock, flags); + if (unlikely(destroy)) { __guc_context_destroy(ce); return; } /* - * We defer GuC context deregistration until the context is destroyed - * in order to save on CTBs. With this optimization ideally we only need - * 1 CTB to register the context during the first pin and 1 CTB to - * deregister the context when the context is destroyed. Without this - * optimization, a CTB would be needed every pin & unpin. - * - * XXX: Need to acqiure the runtime wakeref as this can be triggered - * from context_free_worker when runtime wakeref is not held. - * guc_lrc_desc_unpin requires the runtime as a GuC register is written - * in H2G CTB to deregister the context. A future patch may defer this - * H2G CTB if the runtime wakeref is zero. + * We use a worker to issue the H2G to deregister the context as we can + * take the GT PM for the first time which isn't allowed from an atomic + * context. */ - with_intel_runtime_pm(runtime_pm, wakeref) - guc_lrc_desc_unpin(ce); + queue_work(system_unbound_wq, &guc->submission_state.destroyed_worker); } static int guc_context_alloc(struct intel_context *ce) @@ -1839,20 +2513,23 @@ static void guc_context_set_prio(struct intel_guc *guc, { u32 action[] = { INTEL_GUC_ACTION_SET_CONTEXT_PRIORITY, - ce->guc_id, + ce->guc_id.id, prio, }; GEM_BUG_ON(prio < GUC_CLIENT_PRIORITY_KMD_HIGH || prio > GUC_CLIENT_PRIORITY_NORMAL); + lockdep_assert_held(&ce->guc_state.lock); - if (ce->guc_prio == prio || submission_disabled(guc) || - !context_registered(ce)) + if (ce->guc_state.prio == prio || submission_disabled(guc) || + !context_registered(ce)) { + ce->guc_state.prio = prio; return; + } guc_submission_send_busy_loop(guc, action, ARRAY_SIZE(action), 0, true); - ce->guc_prio = prio; + ce->guc_state.prio = prio; trace_intel_context_set_prio(ce); } @@ -1871,25 +2548,25 @@ static inline u8 map_i915_prio_to_guc_prio(int prio) static inline void add_context_inflight_prio(struct intel_context *ce, u8 guc_prio) { - lockdep_assert_held(&ce->guc_active.lock); - GEM_BUG_ON(guc_prio >= ARRAY_SIZE(ce->guc_prio_count)); + lockdep_assert_held(&ce->guc_state.lock); + GEM_BUG_ON(guc_prio >= ARRAY_SIZE(ce->guc_state.prio_count)); - ++ce->guc_prio_count[guc_prio]; + ++ce->guc_state.prio_count[guc_prio]; /* Overflow protection */ - GEM_WARN_ON(!ce->guc_prio_count[guc_prio]); + GEM_WARN_ON(!ce->guc_state.prio_count[guc_prio]); } static inline void sub_context_inflight_prio(struct intel_context *ce, u8 guc_prio) { - lockdep_assert_held(&ce->guc_active.lock); - GEM_BUG_ON(guc_prio >= ARRAY_SIZE(ce->guc_prio_count)); + lockdep_assert_held(&ce->guc_state.lock); + GEM_BUG_ON(guc_prio >= ARRAY_SIZE(ce->guc_state.prio_count)); /* Underflow protection */ - GEM_WARN_ON(!ce->guc_prio_count[guc_prio]); + GEM_WARN_ON(!ce->guc_state.prio_count[guc_prio]); - --ce->guc_prio_count[guc_prio]; + --ce->guc_state.prio_count[guc_prio]; } static inline void update_context_prio(struct intel_context *ce) @@ -1900,10 +2577,10 @@ static inline void update_context_prio(struct intel_context *ce) BUILD_BUG_ON(GUC_CLIENT_PRIORITY_KMD_HIGH != 0); BUILD_BUG_ON(GUC_CLIENT_PRIORITY_KMD_HIGH > GUC_CLIENT_PRIORITY_NORMAL); - lockdep_assert_held(&ce->guc_active.lock); + lockdep_assert_held(&ce->guc_state.lock); - for (i = 0; i < ARRAY_SIZE(ce->guc_prio_count); ++i) { - if (ce->guc_prio_count[i]) { + for (i = 0; i < ARRAY_SIZE(ce->guc_state.prio_count); ++i) { + if (ce->guc_state.prio_count[i]) { guc_context_set_prio(guc, ce, i); break; } @@ -1918,13 +2595,14 @@ static inline bool new_guc_prio_higher(u8 old_guc_prio, u8 new_guc_prio) static void add_to_context(struct i915_request *rq) { - struct intel_context *ce = rq->context; + struct intel_context *ce = request_to_scheduling_context(rq); u8 new_guc_prio = map_i915_prio_to_guc_prio(rq_prio(rq)); + GEM_BUG_ON(intel_context_is_child(ce)); GEM_BUG_ON(rq->guc_prio == GUC_PRIO_FINI); - spin_lock(&ce->guc_active.lock); - list_move_tail(&rq->sched.link, &ce->guc_active.requests); + spin_lock(&ce->guc_state.lock); + list_move_tail(&rq->sched.link, &ce->guc_state.requests); if (rq->guc_prio == GUC_PRIO_INIT) { rq->guc_prio = new_guc_prio; @@ -1936,12 +2614,12 @@ static void add_to_context(struct i915_request *rq) } update_context_prio(ce); - spin_unlock(&ce->guc_active.lock); + spin_unlock(&ce->guc_state.lock); } static void guc_prio_fini(struct i915_request *rq, struct intel_context *ce) { - lockdep_assert_held(&ce->guc_active.lock); + lockdep_assert_held(&ce->guc_state.lock); if (rq->guc_prio != GUC_PRIO_INIT && rq->guc_prio != GUC_PRIO_FINI) { @@ -1953,9 +2631,11 @@ static void guc_prio_fini(struct i915_request *rq, struct intel_context *ce) static void remove_from_context(struct i915_request *rq) { - struct intel_context *ce = rq->context; + struct intel_context *ce = request_to_scheduling_context(rq); - spin_lock_irq(&ce->guc_active.lock); + GEM_BUG_ON(intel_context_is_child(ce)); + + spin_lock_irq(&ce->guc_state.lock); list_del_init(&rq->sched.link); clear_bit(I915_FENCE_FLAG_PQUEUE, &rq->fence.flags); @@ -1965,9 +2645,11 @@ static void remove_from_context(struct i915_request *rq) guc_prio_fini(rq, ce); - spin_unlock_irq(&ce->guc_active.lock); + decr_context_committed_requests(ce); + + spin_unlock_irq(&ce->guc_state.lock); - atomic_dec(&ce->guc_id_ref); + atomic_dec(&ce->guc_id.ref); i915_request_notify_execute_cb_imm(rq); } @@ -1992,19 +2674,35 @@ static const struct intel_context_ops guc_context_ops = { .destroy = guc_context_destroy, .create_virtual = guc_create_virtual, + .create_parallel = guc_create_parallel, }; +static void submit_work_cb(struct irq_work *wrk) +{ + struct i915_request *rq = container_of(wrk, typeof(*rq), submit_work); + + might_lock(&rq->engine->sched_engine->lock); + i915_sw_fence_complete(&rq->submit); +} + static void __guc_signal_context_fence(struct intel_context *ce) { - struct i915_request *rq; + struct i915_request *rq, *rn; lockdep_assert_held(&ce->guc_state.lock); if (!list_empty(&ce->guc_state.fences)) trace_intel_context_fence_release(ce); - list_for_each_entry(rq, &ce->guc_state.fences, guc_fence_link) - i915_sw_fence_complete(&rq->submit); + /* + * Use an IRQ to ensure locking order of sched_engine->lock -> + * ce->guc_state.lock is preserved. + */ + list_for_each_entry_safe(rq, rn, &ce->guc_state.fences, + guc_fence_link) { + list_del(&rq->guc_fence_link); + irq_work_queue(&rq->submit_work); + } INIT_LIST_HEAD(&ce->guc_state.fences); } @@ -2013,6 +2711,8 @@ static void guc_signal_context_fence(struct intel_context *ce) { unsigned long flags; + GEM_BUG_ON(intel_context_is_child(ce)); + spin_lock_irqsave(&ce->guc_state.lock, flags); clr_context_wait_for_deregister_to_register(ce); __guc_signal_context_fence(ce); @@ -2022,13 +2722,28 @@ static void guc_signal_context_fence(struct intel_context *ce) static bool context_needs_register(struct intel_context *ce, bool new_guc_id) { return (new_guc_id || test_bit(CONTEXT_LRCA_DIRTY, &ce->flags) || - !lrc_desc_registered(ce_to_guc(ce), ce->guc_id)) && + !lrc_desc_registered(ce_to_guc(ce), ce->guc_id.id)) && !submission_disabled(ce_to_guc(ce)); } +static void guc_context_init(struct intel_context *ce) +{ + const struct i915_gem_context *ctx; + int prio = I915_CONTEXT_DEFAULT_PRIORITY; + + rcu_read_lock(); + ctx = rcu_dereference(ce->gem_context); + if (ctx) + prio = ctx->sched.priority; + rcu_read_unlock(); + + ce->guc_state.prio = map_i915_prio_to_guc_prio(prio); + set_bit(CONTEXT_GUC_INIT, &ce->flags); +} + static int guc_request_alloc(struct i915_request *rq) { - struct intel_context *ce = rq->context; + struct intel_context *ce = request_to_scheduling_context(rq); struct intel_guc *guc = ce_to_guc(ce); unsigned long flags; int ret; @@ -2057,14 +2772,17 @@ static int guc_request_alloc(struct i915_request *rq) rq->reserved_space -= GUC_REQUEST_SIZE; + if (unlikely(!test_bit(CONTEXT_GUC_INIT, &ce->flags))) + guc_context_init(ce); + /* * Call pin_guc_id here rather than in the pinning step as with * dma_resv, contexts can be repeatedly pinned / unpinned trashing the - * guc_ids and creating horrible race conditions. This is especially bad - * when guc_ids are being stolen due to over subscription. By the time + * guc_id and creating horrible race conditions. This is especially bad + * when guc_id are being stolen due to over subscription. By the time * this function is reached, it is guaranteed that the guc_id will be * persistent until the generated request is retired. Thus, sealing these - * race conditions. It is still safe to fail here if guc_ids are + * race conditions. It is still safe to fail here if guc_id are * exhausted and return -EAGAIN to the user indicating that they can try * again in the future. * @@ -2074,7 +2792,7 @@ static int guc_request_alloc(struct i915_request *rq) * decremented on each retire. When it is zero, a lock around the * increment (in pin_guc_id) is needed to seal a race with unpin_guc_id. */ - if (atomic_add_unless(&ce->guc_id_ref, 1, 0)) + if (atomic_add_unless(&ce->guc_id.ref, 1, 0)) goto out; ret = pin_guc_id(guc, ce); /* returns 1 if new guc_id assigned */ @@ -2087,7 +2805,7 @@ static int guc_request_alloc(struct i915_request *rq) disable_submission(guc); goto out; /* GPU will be reset */ } - atomic_dec(&ce->guc_id_ref); + atomic_dec(&ce->guc_id.ref); unpin_guc_id(guc, ce); return ret; } @@ -2102,22 +2820,16 @@ out: * schedule enable or context registration if either G2H is pending * respectfully. Once a G2H returns, the fence is released that is * blocking these requests (see guc_signal_context_fence). - * - * We can safely check the below fields outside of the lock as it isn't - * possible for these fields to transition from being clear to set but - * converse is possible, hence the need for the check within the lock. */ - if (likely(!context_wait_for_deregister_to_register(ce) && - !context_pending_disable(ce))) - return 0; - spin_lock_irqsave(&ce->guc_state.lock, flags); if (context_wait_for_deregister_to_register(ce) || context_pending_disable(ce)) { + init_irq_work(&rq->submit_work, submit_work_cb); i915_sw_fence_await(&rq->submit); list_add_tail(&rq->guc_fence_link, &ce->guc_state.fences); } + incr_context_committed_requests(ce); spin_unlock_irqrestore(&ce->guc_state.lock, flags); return 0; @@ -2135,8 +2847,30 @@ static int guc_virtual_context_pre_pin(struct intel_context *ce, static int guc_virtual_context_pin(struct intel_context *ce, void *vaddr) { struct intel_engine_cs *engine = guc_virtual_get_sibling(ce->engine, 0); + int ret = __guc_context_pin(ce, engine, vaddr); + intel_engine_mask_t tmp, mask = ce->engine->mask; - return __guc_context_pin(ce, engine, vaddr); + if (likely(!ret)) + for_each_engine_masked(engine, ce->engine->gt, mask, tmp) + intel_engine_pm_get(engine); + + return ret; +} + +static void guc_virtual_context_unpin(struct intel_context *ce) +{ + intel_engine_mask_t tmp, mask = ce->engine->mask; + struct intel_engine_cs *engine; + struct intel_guc *guc = ce_to_guc(ce); + + GEM_BUG_ON(context_enabled(ce)); + GEM_BUG_ON(intel_context_is_barrier(ce)); + + unpin_guc_id(guc, ce); + lrc_unpin(ce); + + for_each_engine_masked(engine, ce->engine->gt, mask, tmp) + intel_engine_pm_put_async(engine); } static void guc_virtual_context_enter(struct intel_context *ce) @@ -2173,7 +2907,98 @@ static const struct intel_context_ops virtual_guc_context_ops = { .pre_pin = guc_virtual_context_pre_pin, .pin = guc_virtual_context_pin, - .unpin = guc_context_unpin, + .unpin = guc_virtual_context_unpin, + .post_unpin = guc_context_post_unpin, + + .ban = guc_context_ban, + + .cancel_request = guc_context_cancel_request, + + .enter = guc_virtual_context_enter, + .exit = guc_virtual_context_exit, + + .sched_disable = guc_context_sched_disable, + + .destroy = guc_context_destroy, + + .get_sibling = guc_virtual_get_sibling, +}; + +static int guc_parent_context_pin(struct intel_context *ce, void *vaddr) +{ + struct intel_engine_cs *engine = guc_virtual_get_sibling(ce->engine, 0); + struct intel_guc *guc = ce_to_guc(ce); + int ret; + + GEM_BUG_ON(!intel_context_is_parent(ce)); + GEM_BUG_ON(!intel_engine_is_virtual(ce->engine)); + + ret = pin_guc_id(guc, ce); + if (unlikely(ret < 0)) + return ret; + + return __guc_context_pin(ce, engine, vaddr); +} + +static int guc_child_context_pin(struct intel_context *ce, void *vaddr) +{ + struct intel_engine_cs *engine = guc_virtual_get_sibling(ce->engine, 0); + + GEM_BUG_ON(!intel_context_is_child(ce)); + GEM_BUG_ON(!intel_engine_is_virtual(ce->engine)); + + __intel_context_pin(ce->parallel.parent); + return __guc_context_pin(ce, engine, vaddr); +} + +static void guc_parent_context_unpin(struct intel_context *ce) +{ + struct intel_guc *guc = ce_to_guc(ce); + + GEM_BUG_ON(context_enabled(ce)); + GEM_BUG_ON(intel_context_is_barrier(ce)); + GEM_BUG_ON(!intel_context_is_parent(ce)); + GEM_BUG_ON(!intel_engine_is_virtual(ce->engine)); + + if (ce->parallel.last_rq) + i915_request_put(ce->parallel.last_rq); + unpin_guc_id(guc, ce); + lrc_unpin(ce); +} + +static void guc_child_context_unpin(struct intel_context *ce) +{ + GEM_BUG_ON(context_enabled(ce)); + GEM_BUG_ON(intel_context_is_barrier(ce)); + GEM_BUG_ON(!intel_context_is_child(ce)); + GEM_BUG_ON(!intel_engine_is_virtual(ce->engine)); + + lrc_unpin(ce); +} + +static void guc_child_context_post_unpin(struct intel_context *ce) +{ + GEM_BUG_ON(!intel_context_is_child(ce)); + GEM_BUG_ON(!intel_context_is_pinned(ce->parallel.parent)); + GEM_BUG_ON(!intel_engine_is_virtual(ce->engine)); + + lrc_post_unpin(ce); + intel_context_unpin(ce->parallel.parent); +} + +static void guc_child_context_destroy(struct kref *kref) +{ + struct intel_context *ce = container_of(kref, typeof(*ce), ref); + + __guc_context_destroy(ce); +} + +static const struct intel_context_ops virtual_parent_context_ops = { + .alloc = guc_virtual_context_alloc, + + .pre_pin = guc_context_pre_pin, + .pin = guc_parent_context_pin, + .unpin = guc_parent_context_unpin, .post_unpin = guc_context_post_unpin, .ban = guc_context_ban, @@ -2190,6 +3015,110 @@ static const struct intel_context_ops virtual_guc_context_ops = { .get_sibling = guc_virtual_get_sibling, }; +static const struct intel_context_ops virtual_child_context_ops = { + .alloc = guc_virtual_context_alloc, + + .pre_pin = guc_context_pre_pin, + .pin = guc_child_context_pin, + .unpin = guc_child_context_unpin, + .post_unpin = guc_child_context_post_unpin, + + .cancel_request = guc_context_cancel_request, + + .enter = guc_virtual_context_enter, + .exit = guc_virtual_context_exit, + + .destroy = guc_child_context_destroy, + + .get_sibling = guc_virtual_get_sibling, +}; + +/* + * The below override of the breadcrumbs is enabled when the user configures a + * context for parallel submission (multi-lrc, parent-child). + * + * The overridden breadcrumbs implements an algorithm which allows the GuC to + * safely preempt all the hw contexts configured for parallel submission + * between each BB. The contract between the i915 and GuC is if the parent + * context can be preempted, all the children can be preempted, and the GuC will + * always try to preempt the parent before the children. A handshake between the + * parent / children breadcrumbs ensures the i915 holds up its end of the deal + * creating a window to preempt between each set of BBs. + */ +static int emit_bb_start_parent_no_preempt_mid_batch(struct i915_request *rq, + u64 offset, u32 len, + const unsigned int flags); +static int emit_bb_start_child_no_preempt_mid_batch(struct i915_request *rq, + u64 offset, u32 len, + const unsigned int flags); +static u32 * +emit_fini_breadcrumb_parent_no_preempt_mid_batch(struct i915_request *rq, + u32 *cs); +static u32 * +emit_fini_breadcrumb_child_no_preempt_mid_batch(struct i915_request *rq, + u32 *cs); + +static struct intel_context * +guc_create_parallel(struct intel_engine_cs **engines, + unsigned int num_siblings, + unsigned int width) +{ + struct intel_engine_cs **siblings = NULL; + struct intel_context *parent = NULL, *ce, *err; + int i, j; + + siblings = kmalloc_array(num_siblings, + sizeof(*siblings), + GFP_KERNEL); + if (!siblings) + return ERR_PTR(-ENOMEM); + + for (i = 0; i < width; ++i) { + for (j = 0; j < num_siblings; ++j) + siblings[j] = engines[i * num_siblings + j]; + + ce = intel_engine_create_virtual(siblings, num_siblings, + FORCE_VIRTUAL); + if (!ce) { + err = ERR_PTR(-ENOMEM); + goto unwind; + } + + if (i == 0) { + parent = ce; + parent->ops = &virtual_parent_context_ops; + } else { + ce->ops = &virtual_child_context_ops; + intel_context_bind_parent_child(parent, ce); + } + } + + parent->parallel.fence_context = dma_fence_context_alloc(1); + + parent->engine->emit_bb_start = + emit_bb_start_parent_no_preempt_mid_batch; + parent->engine->emit_fini_breadcrumb = + emit_fini_breadcrumb_parent_no_preempt_mid_batch; + parent->engine->emit_fini_breadcrumb_dw = + 12 + 4 * parent->parallel.number_children; + for_each_child(parent, ce) { + ce->engine->emit_bb_start = + emit_bb_start_child_no_preempt_mid_batch; + ce->engine->emit_fini_breadcrumb = + emit_fini_breadcrumb_child_no_preempt_mid_batch; + ce->engine->emit_fini_breadcrumb_dw = 16; + } + + kfree(siblings); + return parent; + +unwind: + if (parent) + intel_context_put(parent); + kfree(siblings); + return err; +} + static bool guc_irq_enable_breadcrumbs(struct intel_breadcrumbs *b) { @@ -2249,7 +3178,7 @@ static void guc_init_breadcrumbs(struct intel_engine_cs *engine) static void guc_bump_inflight_request_prio(struct i915_request *rq, int prio) { - struct intel_context *ce = rq->context; + struct intel_context *ce = request_to_scheduling_context(rq); u8 new_guc_prio = map_i915_prio_to_guc_prio(prio); /* Short circuit function */ @@ -2259,7 +3188,7 @@ static void guc_bump_inflight_request_prio(struct i915_request *rq, !new_guc_prio_higher(rq->guc_prio, new_guc_prio))) return; - spin_lock(&ce->guc_active.lock); + spin_lock(&ce->guc_state.lock); if (rq->guc_prio != GUC_PRIO_FINI) { if (rq->guc_prio != GUC_PRIO_INIT) sub_context_inflight_prio(ce, rq->guc_prio); @@ -2267,16 +3196,16 @@ static void guc_bump_inflight_request_prio(struct i915_request *rq, add_context_inflight_prio(ce, rq->guc_prio); update_context_prio(ce); } - spin_unlock(&ce->guc_active.lock); + spin_unlock(&ce->guc_state.lock); } static void guc_retire_inflight_request_prio(struct i915_request *rq) { - struct intel_context *ce = rq->context; + struct intel_context *ce = request_to_scheduling_context(rq); - spin_lock(&ce->guc_active.lock); + spin_lock(&ce->guc_state.lock); guc_prio_fini(rq, ce); - spin_unlock(&ce->guc_active.lock); + spin_unlock(&ce->guc_state.lock); } static void sanitize_hwsp(struct intel_engine_cs *engine) @@ -2310,6 +3239,8 @@ static void guc_sanitize(struct intel_engine_cs *engine) /* And scrub the dirty cachelines for the HWSP */ clflush_cache_range(engine->status_page.addr, PAGE_SIZE); + + intel_engine_reset_pinned_contexts(engine); } static void setup_hwsp(struct intel_engine_cs *engine) @@ -2385,9 +3316,13 @@ static inline void guc_init_lrc_mapping(struct intel_guc *guc) * and even it did this code would be run again. */ - for_each_engine(engine, gt, id) - if (engine->kernel_context) - guc_kernel_context_pin(guc, engine->kernel_context); + for_each_engine(engine, gt, id) { + struct intel_context *ce; + + list_for_each_entry(ce, &engine->pinned_contexts_list, + pinned_contexts_link) + guc_kernel_context_pin(guc, ce); + } } static void guc_release(struct intel_engine_cs *engine) @@ -2580,13 +3515,13 @@ g2h_context_lookup(struct intel_guc *guc, u32 desc_idx) return NULL; } - return ce; -} + if (unlikely(intel_context_is_child(ce))) { + drm_err(&guc_to_gt(guc)->i915->drm, + "Context is child, desc_idx %u", desc_idx); + return NULL; + } -static void decr_outstanding_submission_g2h(struct intel_guc *guc) -{ - if (atomic_dec_and_test(&guc->outstanding_submission_g2h)) - wake_up_all(&guc->ct.wq); + return ce; } int intel_guc_deregister_done_process_msg(struct intel_guc *guc, @@ -2607,6 +3542,13 @@ int intel_guc_deregister_done_process_msg(struct intel_guc *guc, trace_intel_context_deregister_done(ce); +#ifdef CONFIG_DRM_I915_SELFTEST + if (unlikely(ce->drop_deregister)) { + ce->drop_deregister = false; + return 0; + } +#endif + if (context_wait_for_deregister_to_register(ce)) { struct intel_runtime_pm *runtime_pm = &ce->engine->gt->i915->runtime_pm; @@ -2622,6 +3564,7 @@ int intel_guc_deregister_done_process_msg(struct intel_guc *guc, intel_context_put(ce); } else if (context_destroyed(ce)) { /* Context has been destroyed */ + intel_gt_pm_put_async(guc_to_gt(guc)); release_guc_id(guc, ce); __guc_context_destroy(ce); } @@ -2652,8 +3595,7 @@ int intel_guc_sched_done_process_msg(struct intel_guc *guc, (!context_pending_enable(ce) && !context_pending_disable(ce)))) { drm_err(&guc_to_gt(guc)->i915->drm, - "Bad context sched_state 0x%x, 0x%x, desc_idx %u", - atomic_read(&ce->guc_sched_state_no_lock), + "Bad context sched_state 0x%x, desc_idx %u", ce->guc_state.sched_state, desc_idx); return -EPROTO; } @@ -2661,10 +3603,26 @@ int intel_guc_sched_done_process_msg(struct intel_guc *guc, trace_intel_context_sched_done(ce); if (context_pending_enable(ce)) { +#ifdef CONFIG_DRM_I915_SELFTEST + if (unlikely(ce->drop_schedule_enable)) { + ce->drop_schedule_enable = false; + return 0; + } +#endif + + spin_lock_irqsave(&ce->guc_state.lock, flags); clr_context_pending_enable(ce); + spin_unlock_irqrestore(&ce->guc_state.lock, flags); } else if (context_pending_disable(ce)) { bool banned; +#ifdef CONFIG_DRM_I915_SELFTEST + if (unlikely(ce->drop_schedule_disable)) { + ce->drop_schedule_disable = false; + return 0; + } +#endif + /* * Unpin must be done before __guc_signal_context_fence, * otherwise a race exists between the requests getting @@ -2721,7 +3679,12 @@ static void guc_handle_context_reset(struct intel_guc *guc, { trace_intel_context_reset(ce); - if (likely(!intel_context_is_banned(ce))) { + /* + * XXX: Racey if request cancellation has occurred, see comment in + * __guc_reset_context(). + */ + if (likely(!intel_context_is_banned(ce) && + !context_blocked(ce))) { capture_error_state(guc, ce); guc_context_replay(ce); } @@ -2797,33 +3760,47 @@ void intel_guc_find_hung_context(struct intel_engine_cs *engine) struct intel_context *ce; struct i915_request *rq; unsigned long index; + unsigned long flags; /* Reset called during driver load? GuC not yet initialised! */ if (unlikely(!guc_submission_initialized(guc))) return; + xa_lock_irqsave(&guc->context_lookup, flags); xa_for_each(&guc->context_lookup, index, ce) { - if (!intel_context_is_pinned(ce)) + if (!kref_get_unless_zero(&ce->ref)) continue; + xa_unlock(&guc->context_lookup); + + if (!intel_context_is_pinned(ce)) + goto next; + if (intel_engine_is_virtual(ce->engine)) { if (!(ce->engine->mask & engine->mask)) - continue; + goto next; } else { if (ce->engine != engine) - continue; + goto next; } - list_for_each_entry(rq, &ce->guc_active.requests, sched.link) { + list_for_each_entry(rq, &ce->guc_state.requests, sched.link) { if (i915_test_request_state(rq) != I915_REQUEST_ACTIVE) continue; intel_engine_set_hung_context(engine, ce); /* Can only cope with one hang at a time... */ - return; + intel_context_put(ce); + xa_lock(&guc->context_lookup); + goto done; } +next: + intel_context_put(ce); + xa_lock(&guc->context_lookup); } +done: + xa_unlock_irqrestore(&guc->context_lookup, flags); } void intel_guc_dump_active_requests(struct intel_engine_cs *engine, @@ -2839,23 +3816,34 @@ void intel_guc_dump_active_requests(struct intel_engine_cs *engine, if (unlikely(!guc_submission_initialized(guc))) return; + xa_lock_irqsave(&guc->context_lookup, flags); xa_for_each(&guc->context_lookup, index, ce) { - if (!intel_context_is_pinned(ce)) + if (!kref_get_unless_zero(&ce->ref)) continue; + xa_unlock(&guc->context_lookup); + + if (!intel_context_is_pinned(ce)) + goto next; + if (intel_engine_is_virtual(ce->engine)) { if (!(ce->engine->mask & engine->mask)) - continue; + goto next; } else { if (ce->engine != engine) - continue; + goto next; } - spin_lock_irqsave(&ce->guc_active.lock, flags); - intel_engine_dump_active_requests(&ce->guc_active.requests, + spin_lock(&ce->guc_state.lock); + intel_engine_dump_active_requests(&ce->guc_state.requests, hung_rq, m); - spin_unlock_irqrestore(&ce->guc_active.lock, flags); + spin_unlock(&ce->guc_state.lock); + +next: + intel_context_put(ce); + xa_lock(&guc->context_lookup); } + xa_unlock_irqrestore(&guc->context_lookup, flags); } void intel_guc_submission_print_info(struct intel_guc *guc, @@ -2881,7 +3869,7 @@ void intel_guc_submission_print_info(struct intel_guc *guc, priolist_for_each_request(rq, pl) drm_printf(p, "guc_id=%u, seqno=%llu\n", - rq->context->guc_id, + rq->context->guc_id.id, rq->fence.seqno); } spin_unlock_irqrestore(&sched_engine->lock, flags); @@ -2893,46 +3881,348 @@ static inline void guc_log_context_priority(struct drm_printer *p, { int i; - drm_printf(p, "\t\tPriority: %d\n", - ce->guc_prio); + drm_printf(p, "\t\tPriority: %d\n", ce->guc_state.prio); drm_printf(p, "\t\tNumber Requests (lower index == higher priority)\n"); for (i = GUC_CLIENT_PRIORITY_KMD_HIGH; i < GUC_CLIENT_PRIORITY_NUM; ++i) { drm_printf(p, "\t\tNumber requests in priority band[%d]: %d\n", - i, ce->guc_prio_count[i]); + i, ce->guc_state.prio_count[i]); } drm_printf(p, "\n"); } +static inline void guc_log_context(struct drm_printer *p, + struct intel_context *ce) +{ + drm_printf(p, "GuC lrc descriptor %u:\n", ce->guc_id.id); + drm_printf(p, "\tHW Context Desc: 0x%08x\n", ce->lrc.lrca); + drm_printf(p, "\t\tLRC Head: Internal %u, Memory %u\n", + ce->ring->head, + ce->lrc_reg_state[CTX_RING_HEAD]); + drm_printf(p, "\t\tLRC Tail: Internal %u, Memory %u\n", + ce->ring->tail, + ce->lrc_reg_state[CTX_RING_TAIL]); + drm_printf(p, "\t\tContext Pin Count: %u\n", + atomic_read(&ce->pin_count)); + drm_printf(p, "\t\tGuC ID Ref Count: %u\n", + atomic_read(&ce->guc_id.ref)); + drm_printf(p, "\t\tSchedule State: 0x%x\n\n", + ce->guc_state.sched_state); +} + void intel_guc_submission_print_context_info(struct intel_guc *guc, struct drm_printer *p) { struct intel_context *ce; unsigned long index; + unsigned long flags; + xa_lock_irqsave(&guc->context_lookup, flags); xa_for_each(&guc->context_lookup, index, ce) { - drm_printf(p, "GuC lrc descriptor %u:\n", ce->guc_id); - drm_printf(p, "\tHW Context Desc: 0x%08x\n", ce->lrc.lrca); - drm_printf(p, "\t\tLRC Head: Internal %u, Memory %u\n", - ce->ring->head, - ce->lrc_reg_state[CTX_RING_HEAD]); - drm_printf(p, "\t\tLRC Tail: Internal %u, Memory %u\n", - ce->ring->tail, - ce->lrc_reg_state[CTX_RING_TAIL]); - drm_printf(p, "\t\tContext Pin Count: %u\n", - atomic_read(&ce->pin_count)); - drm_printf(p, "\t\tGuC ID Ref Count: %u\n", - atomic_read(&ce->guc_id_ref)); - drm_printf(p, "\t\tSchedule State: 0x%x, 0x%x\n\n", - ce->guc_state.sched_state, - atomic_read(&ce->guc_sched_state_no_lock)); + GEM_BUG_ON(intel_context_is_child(ce)); + guc_log_context(p, ce); guc_log_context_priority(p, ce); + + if (intel_context_is_parent(ce)) { + struct guc_process_desc *desc = __get_process_desc(ce); + struct intel_context *child; + + drm_printf(p, "\t\tNumber children: %u\n", + ce->parallel.number_children); + drm_printf(p, "\t\tWQI Head: %u\n", + READ_ONCE(desc->head)); + drm_printf(p, "\t\tWQI Tail: %u\n", + READ_ONCE(desc->tail)); + drm_printf(p, "\t\tWQI Status: %u\n\n", + READ_ONCE(desc->wq_status)); + + if (ce->engine->emit_bb_start == + emit_bb_start_parent_no_preempt_mid_batch) { + u8 i; + + drm_printf(p, "\t\tChildren Go: %u\n\n", + get_children_go_value(ce)); + for (i = 0; i < ce->parallel.number_children; ++i) + drm_printf(p, "\t\tChildren Join: %u\n", + get_children_join_value(ce, i)); + } + + for_each_child(ce, child) + guc_log_context(p, child); + } + } + xa_unlock_irqrestore(&guc->context_lookup, flags); +} + +static inline u32 get_children_go_addr(struct intel_context *ce) +{ + GEM_BUG_ON(!intel_context_is_parent(ce)); + + return i915_ggtt_offset(ce->state) + + __get_parent_scratch_offset(ce) + + offsetof(struct parent_scratch, go.semaphore); +} + +static inline u32 get_children_join_addr(struct intel_context *ce, + u8 child_index) +{ + GEM_BUG_ON(!intel_context_is_parent(ce)); + + return i915_ggtt_offset(ce->state) + + __get_parent_scratch_offset(ce) + + offsetof(struct parent_scratch, join[child_index].semaphore); +} + +#define PARENT_GO_BB 1 +#define PARENT_GO_FINI_BREADCRUMB 0 +#define CHILD_GO_BB 1 +#define CHILD_GO_FINI_BREADCRUMB 0 +static int emit_bb_start_parent_no_preempt_mid_batch(struct i915_request *rq, + u64 offset, u32 len, + const unsigned int flags) +{ + struct intel_context *ce = rq->context; + u32 *cs; + u8 i; + + GEM_BUG_ON(!intel_context_is_parent(ce)); + + cs = intel_ring_begin(rq, 10 + 4 * ce->parallel.number_children); + if (IS_ERR(cs)) + return PTR_ERR(cs); + + /* Wait on children */ + for (i = 0; i < ce->parallel.number_children; ++i) { + *cs++ = (MI_SEMAPHORE_WAIT | + MI_SEMAPHORE_GLOBAL_GTT | + MI_SEMAPHORE_POLL | + MI_SEMAPHORE_SAD_EQ_SDD); + *cs++ = PARENT_GO_BB; + *cs++ = get_children_join_addr(ce, i); + *cs++ = 0; + } + + /* Turn off preemption */ + *cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE; + *cs++ = MI_NOOP; + + /* Tell children go */ + cs = gen8_emit_ggtt_write(cs, + CHILD_GO_BB, + get_children_go_addr(ce), + 0); + + /* Jump to batch */ + *cs++ = MI_BATCH_BUFFER_START_GEN8 | + (flags & I915_DISPATCH_SECURE ? 0 : BIT(8)); + *cs++ = lower_32_bits(offset); + *cs++ = upper_32_bits(offset); + *cs++ = MI_NOOP; + + intel_ring_advance(rq, cs); + + return 0; +} + +static int emit_bb_start_child_no_preempt_mid_batch(struct i915_request *rq, + u64 offset, u32 len, + const unsigned int flags) +{ + struct intel_context *ce = rq->context; + struct intel_context *parent = intel_context_to_parent(ce); + u32 *cs; + + GEM_BUG_ON(!intel_context_is_child(ce)); + + cs = intel_ring_begin(rq, 12); + if (IS_ERR(cs)) + return PTR_ERR(cs); + + /* Signal parent */ + cs = gen8_emit_ggtt_write(cs, + PARENT_GO_BB, + get_children_join_addr(parent, + ce->parallel.child_index), + 0); + + /* Wait on parent for go */ + *cs++ = (MI_SEMAPHORE_WAIT | + MI_SEMAPHORE_GLOBAL_GTT | + MI_SEMAPHORE_POLL | + MI_SEMAPHORE_SAD_EQ_SDD); + *cs++ = CHILD_GO_BB; + *cs++ = get_children_go_addr(parent); + *cs++ = 0; + + /* Turn off preemption */ + *cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE; + + /* Jump to batch */ + *cs++ = MI_BATCH_BUFFER_START_GEN8 | + (flags & I915_DISPATCH_SECURE ? 0 : BIT(8)); + *cs++ = lower_32_bits(offset); + *cs++ = upper_32_bits(offset); + + intel_ring_advance(rq, cs); + + return 0; +} + +static u32 * +__emit_fini_breadcrumb_parent_no_preempt_mid_batch(struct i915_request *rq, + u32 *cs) +{ + struct intel_context *ce = rq->context; + u8 i; + + GEM_BUG_ON(!intel_context_is_parent(ce)); + + /* Wait on children */ + for (i = 0; i < ce->parallel.number_children; ++i) { + *cs++ = (MI_SEMAPHORE_WAIT | + MI_SEMAPHORE_GLOBAL_GTT | + MI_SEMAPHORE_POLL | + MI_SEMAPHORE_SAD_EQ_SDD); + *cs++ = PARENT_GO_FINI_BREADCRUMB; + *cs++ = get_children_join_addr(ce, i); + *cs++ = 0; + } + + /* Turn on preemption */ + *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE; + *cs++ = MI_NOOP; + + /* Tell children go */ + cs = gen8_emit_ggtt_write(cs, + CHILD_GO_FINI_BREADCRUMB, + get_children_go_addr(ce), + 0); + + return cs; +} + +/* + * If this true, a submission of multi-lrc requests had an error and the + * requests need to be skipped. The front end (execuf IOCTL) should've called + * i915_request_skip which squashes the BB but we still need to emit the fini + * breadrcrumbs seqno write. At this point we don't know how many of the + * requests in the multi-lrc submission were generated so we can't do the + * handshake between the parent and children (e.g. if 4 requests should be + * generated but 2nd hit an error only 1 would be seen by the GuC backend). + * Simply skip the handshake, but still emit the breadcrumbd seqno, if an error + * has occurred on any of the requests in submission / relationship. + */ +static inline bool skip_handshake(struct i915_request *rq) +{ + return test_bit(I915_FENCE_FLAG_SKIP_PARALLEL, &rq->fence.flags); +} + +static u32 * +emit_fini_breadcrumb_parent_no_preempt_mid_batch(struct i915_request *rq, + u32 *cs) +{ + struct intel_context *ce = rq->context; + + GEM_BUG_ON(!intel_context_is_parent(ce)); + + if (unlikely(skip_handshake(rq))) { + /* + * NOP everything in __emit_fini_breadcrumb_parent_no_preempt_mid_batch, + * the -6 comes from the length of the emits below. + */ + memset(cs, 0, sizeof(u32) * + (ce->engine->emit_fini_breadcrumb_dw - 6)); + cs += ce->engine->emit_fini_breadcrumb_dw - 6; + } else { + cs = __emit_fini_breadcrumb_parent_no_preempt_mid_batch(rq, cs); + } + + /* Emit fini breadcrumb */ + cs = gen8_emit_ggtt_write(cs, + rq->fence.seqno, + i915_request_active_timeline(rq)->hwsp_offset, + 0); + + /* User interrupt */ + *cs++ = MI_USER_INTERRUPT; + *cs++ = MI_NOOP; + + rq->tail = intel_ring_offset(rq, cs); + + return cs; +} + +static u32 * +__emit_fini_breadcrumb_child_no_preempt_mid_batch(struct i915_request *rq, + u32 *cs) +{ + struct intel_context *ce = rq->context; + struct intel_context *parent = intel_context_to_parent(ce); + + GEM_BUG_ON(!intel_context_is_child(ce)); + + /* Turn on preemption */ + *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE; + *cs++ = MI_NOOP; + + /* Signal parent */ + cs = gen8_emit_ggtt_write(cs, + PARENT_GO_FINI_BREADCRUMB, + get_children_join_addr(parent, + ce->parallel.child_index), + 0); + + /* Wait parent on for go */ + *cs++ = (MI_SEMAPHORE_WAIT | + MI_SEMAPHORE_GLOBAL_GTT | + MI_SEMAPHORE_POLL | + MI_SEMAPHORE_SAD_EQ_SDD); + *cs++ = CHILD_GO_FINI_BREADCRUMB; + *cs++ = get_children_go_addr(parent); + *cs++ = 0; + + return cs; +} + +static u32 * +emit_fini_breadcrumb_child_no_preempt_mid_batch(struct i915_request *rq, + u32 *cs) +{ + struct intel_context *ce = rq->context; + + GEM_BUG_ON(!intel_context_is_child(ce)); + + if (unlikely(skip_handshake(rq))) { + /* + * NOP everything in __emit_fini_breadcrumb_child_no_preempt_mid_batch, + * the -6 comes from the length of the emits below. + */ + memset(cs, 0, sizeof(u32) * + (ce->engine->emit_fini_breadcrumb_dw - 6)); + cs += ce->engine->emit_fini_breadcrumb_dw - 6; + } else { + cs = __emit_fini_breadcrumb_child_no_preempt_mid_batch(rq, cs); } + + /* Emit fini breadcrumb */ + cs = gen8_emit_ggtt_write(cs, + rq->fence.seqno, + i915_request_active_timeline(rq)->hwsp_offset, + 0); + + /* User interrupt */ + *cs++ = MI_USER_INTERRUPT; + *cs++ = MI_NOOP; + + rq->tail = intel_ring_offset(rq, cs); + + return cs; } static struct intel_context * -guc_create_virtual(struct intel_engine_cs **siblings, unsigned int count) +guc_create_virtual(struct intel_engine_cs **siblings, unsigned int count, + unsigned long flags) { struct guc_virtual_engine *ve; struct intel_guc *guc; @@ -2981,6 +4271,7 @@ guc_create_virtual(struct intel_engine_cs **siblings, unsigned int count) } ve->base.mask |= sibling->mask; + ve->base.logical_mask |= sibling->logical_mask; if (n != 0 && ve->base.class != sibling->class) { DRM_DEBUG("invalid mixing of engine class, sibling %d, already %d\n", @@ -3036,3 +4327,8 @@ bool intel_guc_virtual_engine_has_heartbeat(const struct intel_engine_cs *ve) return false; } + +#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) +#include "selftest_guc.c" +#include "selftest_guc_multi_lrc.c" +#endif diff --git a/drivers/gpu/drm/i915/gt/uc/intel_huc.c b/drivers/gpu/drm/i915/gt/uc/intel_huc.c index fc5387b410a2..ff4b6869b80b 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_huc.c +++ b/drivers/gpu/drm/i915/gt/uc/intel_huc.c @@ -87,17 +87,25 @@ static int intel_huc_rsa_data_create(struct intel_huc *huc) vma->obj, true)); if (IS_ERR(vaddr)) { i915_vma_unpin_and_release(&vma, 0); - return PTR_ERR(vaddr); + err = PTR_ERR(vaddr); + goto unpin_out; } copied = intel_uc_fw_copy_rsa(&huc->fw, vaddr, vma->size); - GEM_BUG_ON(copied < huc->fw.rsa_size); - i915_gem_object_unpin_map(vma->obj); + if (copied < huc->fw.rsa_size) { + err = -ENOMEM; + goto unpin_out; + } + huc->rsa_data = vma; return 0; + +unpin_out: + i915_vma_unpin_and_release(&vma, 0); + return err; } static void intel_huc_rsa_data_destroy(struct intel_huc *huc) diff --git a/drivers/gpu/drm/i915/gt/uc/intel_huc_debugfs.c b/drivers/gpu/drm/i915/gt/uc/intel_huc_debugfs.c index 5733c15fd123..15998963b863 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_huc_debugfs.c +++ b/drivers/gpu/drm/i915/gt/uc/intel_huc_debugfs.c @@ -5,7 +5,7 @@ #include <drm/drm_print.h> -#include "gt/debugfs_gt.h" +#include "gt/intel_gt_debugfs.h" #include "intel_huc.h" #include "intel_huc_debugfs.h" @@ -21,11 +21,11 @@ static int huc_info_show(struct seq_file *m, void *data) return 0; } -DEFINE_GT_DEBUGFS_ATTRIBUTE(huc_info); +DEFINE_INTEL_GT_DEBUGFS_ATTRIBUTE(huc_info); void intel_huc_debugfs_register(struct intel_huc *huc, struct dentry *root) { - static const struct debugfs_gt_file files[] = { + static const struct intel_gt_debugfs_file files[] = { { "huc_info", &huc_info_fops, NULL }, }; diff --git a/drivers/gpu/drm/i915/gt/uc/intel_uc.c b/drivers/gpu/drm/i915/gt/uc/intel_uc.c index 86c318516e14..2fef3b0bbe95 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_uc.c +++ b/drivers/gpu/drm/i915/gt/uc/intel_uc.c @@ -35,7 +35,7 @@ static void uc_expand_default_options(struct intel_uc *uc) } /* Intermediate platforms are HuC authentication only */ - if (IS_DG1(i915) || IS_ALDERLAKE_S(i915)) { + if (IS_ALDERLAKE_S(i915)) { i915->params.enable_guc = ENABLE_GUC_LOAD_HUC; return; } diff --git a/drivers/gpu/drm/i915/gt/uc/intel_uc_debugfs.c b/drivers/gpu/drm/i915/gt/uc/intel_uc_debugfs.c index 089d98662f46..c2f7924295e7 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_uc_debugfs.c +++ b/drivers/gpu/drm/i915/gt/uc/intel_uc_debugfs.c @@ -6,7 +6,7 @@ #include <linux/debugfs.h> #include <drm/drm_print.h> -#include "gt/debugfs_gt.h" +#include "gt/intel_gt_debugfs.h" #include "intel_guc_debugfs.h" #include "intel_huc_debugfs.h" #include "intel_uc.h" @@ -32,11 +32,11 @@ static int uc_usage_show(struct seq_file *m, void *data) return 0; } -DEFINE_GT_DEBUGFS_ATTRIBUTE(uc_usage); +DEFINE_INTEL_GT_DEBUGFS_ATTRIBUTE(uc_usage); void intel_uc_debugfs_register(struct intel_uc *uc, struct dentry *gt_root) { - static const struct debugfs_gt_file files[] = { + static const struct intel_gt_debugfs_file files[] = { { "usage", &uc_usage_fops, NULL }, }; struct dentry *root; diff --git a/drivers/gpu/drm/i915/gt/uc/intel_uc_fw.c b/drivers/gpu/drm/i915/gt/uc/intel_uc_fw.c index 3a16d08608a5..3aa87be4f2e4 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_uc_fw.c +++ b/drivers/gpu/drm/i915/gt/uc/intel_uc_fw.c @@ -7,6 +7,7 @@ #include <linux/firmware.h> #include <drm/drm_print.h> +#include "gem/i915_gem_lmem.h" #include "intel_uc_fw.h" #include "intel_uc_fw_abi.h" #include "i915_drv.h" @@ -50,6 +51,7 @@ void intel_uc_fw_change_status(struct intel_uc_fw *uc_fw, #define INTEL_UC_FIRMWARE_DEFS(fw_def, guc_def, huc_def) \ fw_def(ALDERLAKE_P, 0, guc_def(adlp, 62, 0, 3), huc_def(tgl, 7, 9, 3)) \ fw_def(ALDERLAKE_S, 0, guc_def(tgl, 62, 0, 0), huc_def(tgl, 7, 9, 3)) \ + fw_def(DG1, 0, guc_def(dg1, 62, 0, 0), huc_def(dg1, 7, 9, 3)) \ fw_def(ROCKETLAKE, 0, guc_def(tgl, 62, 0, 0), huc_def(tgl, 7, 9, 3)) \ fw_def(TIGERLAKE, 0, guc_def(tgl, 62, 0, 0), huc_def(tgl, 7, 9, 3)) \ fw_def(JASPERLAKE, 0, guc_def(ehl, 62, 0, 0), huc_def(ehl, 9, 0, 0)) \ @@ -370,7 +372,14 @@ int intel_uc_fw_fetch(struct intel_uc_fw *uc_fw) if (uc_fw->type == INTEL_UC_FW_TYPE_GUC) uc_fw->private_data_size = css->private_data_size; - obj = i915_gem_object_create_shmem_from_data(i915, fw->data, fw->size); + if (HAS_LMEM(i915)) { + obj = i915_gem_object_create_lmem_from_data(i915, fw->data, fw->size); + if (!IS_ERR(obj)) + obj->flags |= I915_BO_ALLOC_PM_EARLY; + } else { + obj = i915_gem_object_create_shmem_from_data(i915, fw->data, fw->size); + } + if (IS_ERR(obj)) { err = PTR_ERR(obj); goto fail; @@ -413,20 +422,25 @@ static void uc_fw_bind_ggtt(struct intel_uc_fw *uc_fw) { struct drm_i915_gem_object *obj = uc_fw->obj; struct i915_ggtt *ggtt = __uc_fw_to_gt(uc_fw)->ggtt; - struct i915_vma dummy = { - .node.start = uc_fw_ggtt_offset(uc_fw), - .node.size = obj->base.size, - .pages = obj->mm.pages, - .vm = &ggtt->vm, - }; + struct i915_vma *dummy = &uc_fw->dummy; + u32 pte_flags = 0; + + dummy->node.start = uc_fw_ggtt_offset(uc_fw); + dummy->node.size = obj->base.size; + dummy->pages = obj->mm.pages; + dummy->vm = &ggtt->vm; GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj)); - GEM_BUG_ON(dummy.node.size > ggtt->uc_fw.size); + GEM_BUG_ON(dummy->node.size > ggtt->uc_fw.size); /* uc_fw->obj cache domains were not controlled across suspend */ - drm_clflush_sg(dummy.pages); + if (i915_gem_object_has_struct_page(obj)) + drm_clflush_sg(dummy->pages); - ggtt->vm.insert_entries(&ggtt->vm, &dummy, I915_CACHE_NONE, 0); + if (i915_gem_object_is_lmem(obj)) + pte_flags |= PTE_LM; + + ggtt->vm.insert_entries(&ggtt->vm, dummy, I915_CACHE_NONE, pte_flags); } static void uc_fw_unbind_ggtt(struct intel_uc_fw *uc_fw) @@ -585,13 +599,68 @@ void intel_uc_fw_cleanup_fetch(struct intel_uc_fw *uc_fw) */ size_t intel_uc_fw_copy_rsa(struct intel_uc_fw *uc_fw, void *dst, u32 max_len) { - struct sg_table *pages = uc_fw->obj->mm.pages; + struct intel_memory_region *mr = uc_fw->obj->mm.region; u32 size = min_t(u32, uc_fw->rsa_size, max_len); u32 offset = sizeof(struct uc_css_header) + uc_fw->ucode_size; + struct sgt_iter iter; + size_t count = 0; + int idx; + /* Called during reset handling, must be atomic [no fs_reclaim] */ GEM_BUG_ON(!intel_uc_fw_is_available(uc_fw)); - return sg_pcopy_to_buffer(pages->sgl, pages->nents, dst, size, offset); + idx = offset >> PAGE_SHIFT; + offset = offset_in_page(offset); + if (i915_gem_object_has_struct_page(uc_fw->obj)) { + struct page *page; + + for_each_sgt_page(page, iter, uc_fw->obj->mm.pages) { + u32 len = min_t(u32, size, PAGE_SIZE - offset); + void *vaddr; + + if (idx > 0) { + idx--; + continue; + } + + vaddr = kmap_atomic(page); + memcpy(dst, vaddr + offset, len); + kunmap_atomic(vaddr); + + offset = 0; + dst += len; + size -= len; + count += len; + if (!size) + break; + } + } else { + dma_addr_t addr; + + for_each_sgt_daddr(addr, iter, uc_fw->obj->mm.pages) { + u32 len = min_t(u32, size, PAGE_SIZE - offset); + void __iomem *vaddr; + + if (idx > 0) { + idx--; + continue; + } + + vaddr = io_mapping_map_atomic_wc(&mr->iomap, + addr - mr->region.start); + memcpy_fromio(dst, vaddr + offset, len); + io_mapping_unmap_atomic(vaddr); + + offset = 0; + dst += len; + size -= len; + count += len; + if (!size) + break; + } + } + + return count; } /** diff --git a/drivers/gpu/drm/i915/gt/uc/intel_uc_fw.h b/drivers/gpu/drm/i915/gt/uc/intel_uc_fw.h index 99bb1fe1af66..1e00bf65639e 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_uc_fw.h +++ b/drivers/gpu/drm/i915/gt/uc/intel_uc_fw.h @@ -10,6 +10,7 @@ #include "intel_uc_fw_abi.h" #include "intel_device_info.h" #include "i915_gem.h" +#include "i915_vma.h" struct drm_printer; struct drm_i915_private; @@ -75,6 +76,14 @@ struct intel_uc_fw { bool user_overridden; size_t size; struct drm_i915_gem_object *obj; + /** + * @dummy: A vma used in binding the uc fw to ggtt. We can't define this + * vma on the stack as it can lead to a stack overflow, so we define it + * here. Safe to have 1 copy per uc fw because the binding is single + * threaded as it done during driver load (inherently single threaded) + * or during a GT reset (mutex guarantees single threaded). + */ + struct i915_vma dummy; /* * The firmware build process will generate a version header file with major and diff --git a/drivers/gpu/drm/i915/gt/uc/selftest_guc.c b/drivers/gpu/drm/i915/gt/uc/selftest_guc.c new file mode 100644 index 000000000000..fb0e4a7bd8ca --- /dev/null +++ b/drivers/gpu/drm/i915/gt/uc/selftest_guc.c @@ -0,0 +1,127 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright �� 2021 Intel Corporation + */ + +#include "selftests/intel_scheduler_helpers.h" + +static struct i915_request *nop_user_request(struct intel_context *ce, + struct i915_request *from) +{ + struct i915_request *rq; + int ret; + + rq = intel_context_create_request(ce); + if (IS_ERR(rq)) + return rq; + + if (from) { + ret = i915_sw_fence_await_dma_fence(&rq->submit, + &from->fence, 0, + I915_FENCE_GFP); + if (ret < 0) { + i915_request_put(rq); + return ERR_PTR(ret); + } + } + + i915_request_get(rq); + i915_request_add(rq); + + return rq; +} + +static int intel_guc_scrub_ctbs(void *arg) +{ + struct intel_gt *gt = arg; + int ret = 0; + int i; + struct i915_request *last[3] = {NULL, NULL, NULL}, *rq; + intel_wakeref_t wakeref; + struct intel_engine_cs *engine; + struct intel_context *ce; + + wakeref = intel_runtime_pm_get(gt->uncore->rpm); + engine = intel_selftest_find_any_engine(gt); + + /* Submit requests and inject errors forcing G2H to be dropped */ + for (i = 0; i < 3; ++i) { + ce = intel_context_create(engine); + if (IS_ERR(ce)) { + ret = PTR_ERR(ce); + pr_err("Failed to create context, %d: %d\n", i, ret); + goto err; + } + + switch (i) { + case 0: + ce->drop_schedule_enable = true; + break; + case 1: + ce->drop_schedule_disable = true; + break; + case 2: + ce->drop_deregister = true; + break; + } + + rq = nop_user_request(ce, NULL); + intel_context_put(ce); + + if (IS_ERR(rq)) { + ret = PTR_ERR(rq); + pr_err("Failed to create request, %d: %d\n", i, ret); + goto err; + } + + last[i] = rq; + } + + for (i = 0; i < 3; ++i) { + ret = i915_request_wait(last[i], 0, HZ); + if (ret < 0) { + pr_err("Last request failed to complete: %d\n", ret); + goto err; + } + i915_request_put(last[i]); + last[i] = NULL; + } + + /* Force all H2G / G2H to be submitted / processed */ + intel_gt_retire_requests(gt); + msleep(500); + + /* Scrub missing G2H */ + intel_gt_handle_error(engine->gt, -1, 0, "selftest reset"); + + /* GT will not idle if G2H are lost */ + ret = intel_gt_wait_for_idle(gt, HZ); + if (ret < 0) { + pr_err("GT failed to idle: %d\n", ret); + goto err; + } + +err: + for (i = 0; i < 3; ++i) + if (last[i]) + i915_request_put(last[i]); + intel_runtime_pm_put(gt->uncore->rpm, wakeref); + + return ret; +} + +int intel_guc_live_selftests(struct drm_i915_private *i915) +{ + static const struct i915_subtest tests[] = { + SUBTEST(intel_guc_scrub_ctbs), + }; + struct intel_gt *gt = &i915->gt; + + if (intel_gt_is_wedged(gt)) + return 0; + + if (!intel_uc_uses_guc_submission(>->uc)) + return 0; + + return intel_gt_live_subtests(tests, gt); +} diff --git a/drivers/gpu/drm/i915/gt/uc/selftest_guc_multi_lrc.c b/drivers/gpu/drm/i915/gt/uc/selftest_guc_multi_lrc.c new file mode 100644 index 000000000000..50953c8e8b53 --- /dev/null +++ b/drivers/gpu/drm/i915/gt/uc/selftest_guc_multi_lrc.c @@ -0,0 +1,179 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright �� 2019 Intel Corporation + */ + +#include "selftests/igt_spinner.h" +#include "selftests/igt_reset.h" +#include "selftests/intel_scheduler_helpers.h" +#include "gt/intel_engine_heartbeat.h" +#include "gem/selftests/mock_context.h" + +static void logical_sort(struct intel_engine_cs **engines, int num_engines) +{ + struct intel_engine_cs *sorted[MAX_ENGINE_INSTANCE + 1]; + int i, j; + + for (i = 0; i < num_engines; ++i) + for (j = 0; j < MAX_ENGINE_INSTANCE + 1; ++j) { + if (engines[j]->logical_mask & BIT(i)) { + sorted[i] = engines[j]; + break; + } + } + + memcpy(*engines, *sorted, + sizeof(struct intel_engine_cs *) * num_engines); +} + +static struct intel_context * +multi_lrc_create_parent(struct intel_gt *gt, u8 class, + unsigned long flags) +{ + struct intel_engine_cs *siblings[MAX_ENGINE_INSTANCE + 1]; + struct intel_engine_cs *engine; + enum intel_engine_id id; + int i = 0; + + for_each_engine(engine, gt, id) { + if (engine->class != class) + continue; + + siblings[i++] = engine; + } + + if (i <= 1) + return ERR_PTR(0); + + logical_sort(siblings, i); + + return intel_engine_create_parallel(siblings, 1, i); +} + +static void multi_lrc_context_unpin(struct intel_context *ce) +{ + struct intel_context *child; + + GEM_BUG_ON(!intel_context_is_parent(ce)); + + for_each_child(ce, child) + intel_context_unpin(child); + intel_context_unpin(ce); +} + +static void multi_lrc_context_put(struct intel_context *ce) +{ + GEM_BUG_ON(!intel_context_is_parent(ce)); + + /* + * Only the parent gets the creation ref put in the uAPI, the parent + * itself is responsible for creation ref put on the children. + */ + intel_context_put(ce); +} + +static struct i915_request * +multi_lrc_nop_request(struct intel_context *ce) +{ + struct intel_context *child; + struct i915_request *rq, *child_rq; + int i = 0; + + GEM_BUG_ON(!intel_context_is_parent(ce)); + + rq = intel_context_create_request(ce); + if (IS_ERR(rq)) + return rq; + + i915_request_get(rq); + i915_request_add(rq); + + for_each_child(ce, child) { + child_rq = intel_context_create_request(child); + if (IS_ERR(child_rq)) + goto child_error; + + if (++i == ce->parallel.number_children) + set_bit(I915_FENCE_FLAG_SUBMIT_PARALLEL, + &child_rq->fence.flags); + i915_request_add(child_rq); + } + + return rq; + +child_error: + i915_request_put(rq); + + return ERR_PTR(-ENOMEM); +} + +static int __intel_guc_multi_lrc_basic(struct intel_gt *gt, unsigned int class) +{ + struct intel_context *parent; + struct i915_request *rq; + int ret; + + parent = multi_lrc_create_parent(gt, class, 0); + if (IS_ERR(parent)) { + pr_err("Failed creating contexts: %ld", PTR_ERR(parent)); + return PTR_ERR(parent); + } else if (!parent) { + pr_debug("Not enough engines in class: %d", class); + return 0; + } + + rq = multi_lrc_nop_request(parent); + if (IS_ERR(rq)) { + ret = PTR_ERR(rq); + pr_err("Failed creating requests: %d", ret); + goto out; + } + + ret = intel_selftest_wait_for_rq(rq); + if (ret) + pr_err("Failed waiting on request: %d", ret); + + i915_request_put(rq); + + if (ret >= 0) { + ret = intel_gt_wait_for_idle(gt, HZ * 5); + if (ret < 0) + pr_err("GT failed to idle: %d\n", ret); + } + +out: + multi_lrc_context_unpin(parent); + multi_lrc_context_put(parent); + return ret; +} + +static int intel_guc_multi_lrc_basic(void *arg) +{ + struct intel_gt *gt = arg; + unsigned int class; + int ret; + + for (class = 0; class < MAX_ENGINE_CLASS + 1; ++class) { + ret = __intel_guc_multi_lrc_basic(gt, class); + if (ret) + return ret; + } + + return 0; +} + +int intel_guc_multi_lrc_live_selftests(struct drm_i915_private *i915) +{ + static const struct i915_subtest tests[] = { + SUBTEST(intel_guc_multi_lrc_basic), + }; + struct intel_gt *gt = &i915->gt; + + if (intel_gt_is_wedged(gt)) + return 0; + + if (!intel_uc_uses_guc_submission(>->uc)) + return 0; + + return intel_gt_live_subtests(tests, gt); +} diff --git a/drivers/gpu/drm/i915/gvt/gtt.c b/drivers/gpu/drm/i915/gvt/gtt.c index e5c2fdfc20e3..53d0cb327539 100644 --- a/drivers/gpu/drm/i915/gvt/gtt.c +++ b/drivers/gpu/drm/i915/gvt/gtt.c @@ -745,7 +745,7 @@ static void ppgtt_free_spt(struct intel_vgpu_ppgtt_spt *spt) trace_spt_free(spt->vgpu->id, spt, spt->guest_page.type); dma_unmap_page(kdev, spt->shadow_page.mfn << I915_GTT_PAGE_SHIFT, 4096, - PCI_DMA_BIDIRECTIONAL); + DMA_BIDIRECTIONAL); radix_tree_delete(&spt->vgpu->gtt.spt_tree, spt->shadow_page.mfn); @@ -849,7 +849,7 @@ retry: */ spt->shadow_page.type = type; daddr = dma_map_page(kdev, spt->shadow_page.page, - 0, 4096, PCI_DMA_BIDIRECTIONAL); + 0, 4096, DMA_BIDIRECTIONAL); if (dma_mapping_error(kdev, daddr)) { gvt_vgpu_err("fail to map dma addr\n"); ret = -EINVAL; @@ -865,7 +865,7 @@ retry: return spt; err_unmap_dma: - dma_unmap_page(kdev, daddr, PAGE_SIZE, PCI_DMA_BIDIRECTIONAL); + dma_unmap_page(kdev, daddr, PAGE_SIZE, DMA_BIDIRECTIONAL); err_free_spt: free_spt(spt); return ERR_PTR(ret); @@ -2409,8 +2409,7 @@ static int alloc_scratch_pages(struct intel_vgpu *vgpu, return -ENOMEM; } - daddr = dma_map_page(dev, virt_to_page(scratch_pt), 0, - 4096, PCI_DMA_BIDIRECTIONAL); + daddr = dma_map_page(dev, virt_to_page(scratch_pt), 0, 4096, DMA_BIDIRECTIONAL); if (dma_mapping_error(dev, daddr)) { gvt_vgpu_err("fail to dmamap scratch_pt\n"); __free_page(virt_to_page(scratch_pt)); @@ -2461,7 +2460,7 @@ static int release_scratch_page_tree(struct intel_vgpu *vgpu) if (vgpu->gtt.scratch_pt[i].page != NULL) { daddr = (dma_addr_t)(vgpu->gtt.scratch_pt[i].page_mfn << I915_GTT_PAGE_SHIFT); - dma_unmap_page(dev, daddr, 4096, PCI_DMA_BIDIRECTIONAL); + dma_unmap_page(dev, daddr, 4096, DMA_BIDIRECTIONAL); __free_page(vgpu->gtt.scratch_pt[i].page); vgpu->gtt.scratch_pt[i].page = NULL; vgpu->gtt.scratch_pt[i].page_mfn = 0; @@ -2741,7 +2740,7 @@ int intel_gvt_init_gtt(struct intel_gvt *gvt) } daddr = dma_map_page(dev, virt_to_page(page), 0, - 4096, PCI_DMA_BIDIRECTIONAL); + 4096, DMA_BIDIRECTIONAL); if (dma_mapping_error(dev, daddr)) { gvt_err("fail to dmamap scratch ggtt page\n"); __free_page(virt_to_page(page)); @@ -2755,7 +2754,7 @@ int intel_gvt_init_gtt(struct intel_gvt *gvt) ret = setup_spt_oos(gvt); if (ret) { gvt_err("fail to initialize SPT oos\n"); - dma_unmap_page(dev, daddr, 4096, PCI_DMA_BIDIRECTIONAL); + dma_unmap_page(dev, daddr, 4096, DMA_BIDIRECTIONAL); __free_page(gvt->gtt.scratch_page); return ret; } @@ -2779,7 +2778,7 @@ void intel_gvt_clean_gtt(struct intel_gvt *gvt) dma_addr_t daddr = (dma_addr_t)(gvt->gtt.scratch_mfn << I915_GTT_PAGE_SHIFT); - dma_unmap_page(dev, daddr, 4096, PCI_DMA_BIDIRECTIONAL); + dma_unmap_page(dev, daddr, 4096, DMA_BIDIRECTIONAL); __free_page(gvt->gtt.scratch_page); diff --git a/drivers/gpu/drm/i915/gvt/kvmgt.c b/drivers/gpu/drm/i915/gvt/kvmgt.c index 7efa386449d1..20b82fb036f8 100644 --- a/drivers/gpu/drm/i915/gvt/kvmgt.c +++ b/drivers/gpu/drm/i915/gvt/kvmgt.c @@ -328,7 +328,7 @@ static int gvt_dma_map_page(struct intel_vgpu *vgpu, unsigned long gfn, return ret; /* Setup DMA mapping. */ - *dma_addr = dma_map_page(dev, page, 0, size, PCI_DMA_BIDIRECTIONAL); + *dma_addr = dma_map_page(dev, page, 0, size, DMA_BIDIRECTIONAL); if (dma_mapping_error(dev, *dma_addr)) { gvt_vgpu_err("DMA mapping failed for pfn 0x%lx, ret %d\n", page_to_pfn(page), ret); @@ -344,7 +344,7 @@ static void gvt_dma_unmap_page(struct intel_vgpu *vgpu, unsigned long gfn, { struct device *dev = vgpu->gvt->gt->i915->drm.dev; - dma_unmap_page(dev, dma_addr, size, PCI_DMA_BIDIRECTIONAL); + dma_unmap_page(dev, dma_addr, size, DMA_BIDIRECTIONAL); gvt_unpin_guest_page(vgpu, gfn, size); } diff --git a/drivers/gpu/drm/i915/gvt/scheduler.c b/drivers/gpu/drm/i915/gvt/scheduler.c index 1bb1be5c48c8..6c804102528b 100644 --- a/drivers/gpu/drm/i915/gvt/scheduler.c +++ b/drivers/gpu/drm/i915/gvt/scheduler.c @@ -1386,7 +1386,7 @@ int intel_vgpu_setup_submission(struct intel_vgpu *vgpu) enum intel_engine_id i; int ret; - ppgtt = i915_ppgtt_create(&i915->gt); + ppgtt = i915_ppgtt_create(&i915->gt, I915_BO_ALLOC_PM_EARLY); if (IS_ERR(ppgtt)) return PTR_ERR(ppgtt); diff --git a/drivers/gpu/drm/i915/i915_buddy.c b/drivers/gpu/drm/i915/i915_buddy.c index 7b274c51cac0..6e2ad68f8f3f 100644 --- a/drivers/gpu/drm/i915/i915_buddy.c +++ b/drivers/gpu/drm/i915/i915_buddy.c @@ -4,6 +4,7 @@ */ #include <linux/kmemleak.h> +#include <linux/sizes.h> #include "i915_buddy.h" @@ -82,6 +83,7 @@ int i915_buddy_init(struct i915_buddy_mm *mm, u64 size, u64 chunk_size) size = round_down(size, chunk_size); mm->size = size; + mm->avail = size; mm->chunk_size = chunk_size; mm->max_order = ilog2(size) - ilog2(chunk_size); @@ -155,6 +157,8 @@ void i915_buddy_fini(struct i915_buddy_mm *mm) i915_block_free(mm, mm->roots[i]); } + GEM_WARN_ON(mm->avail != mm->size); + kfree(mm->roots); kfree(mm->free_list); } @@ -230,6 +234,7 @@ void i915_buddy_free(struct i915_buddy_mm *mm, struct i915_buddy_block *block) { GEM_BUG_ON(!i915_buddy_block_is_allocated(block)); + mm->avail += i915_buddy_block_size(mm, block); __i915_buddy_free(mm, block); } @@ -283,6 +288,7 @@ i915_buddy_alloc(struct i915_buddy_mm *mm, unsigned int order) } mark_allocated(block); + mm->avail -= i915_buddy_block_size(mm, block); kmemleak_update_trace(block); return block; @@ -368,6 +374,7 @@ int i915_buddy_alloc_range(struct i915_buddy_mm *mm, } mark_allocated(block); + mm->avail -= i915_buddy_block_size(mm, block); list_add_tail(&block->link, &allocated); continue; } @@ -402,6 +409,44 @@ err_free: return err; } +void i915_buddy_block_print(struct i915_buddy_mm *mm, + struct i915_buddy_block *block, + struct drm_printer *p) +{ + u64 start = i915_buddy_block_offset(block); + u64 size = i915_buddy_block_size(mm, block); + + drm_printf(p, "%#018llx-%#018llx: %llu\n", start, start + size, size); +} + +void i915_buddy_print(struct i915_buddy_mm *mm, struct drm_printer *p) +{ + int order; + + drm_printf(p, "chunk_size: %lluKiB, total: %lluMiB, free: %lluMiB\n", + mm->chunk_size >> 10, mm->size >> 20, mm->avail >> 20); + + for (order = mm->max_order; order >= 0; order--) { + struct i915_buddy_block *block; + u64 count = 0, free; + + list_for_each_entry(block, &mm->free_list[order], link) { + GEM_BUG_ON(!i915_buddy_block_is_free(block)); + count++; + } + + drm_printf(p, "order-%d ", order); + + free = count * (mm->chunk_size << order); + if (free < SZ_1M) + drm_printf(p, "free: %lluKiB", free >> 10); + else + drm_printf(p, "free: %lluMiB", free >> 20); + + drm_printf(p, ", pages: %llu\n", count); + } +} + #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) #include "selftests/i915_buddy.c" #endif diff --git a/drivers/gpu/drm/i915/i915_buddy.h b/drivers/gpu/drm/i915/i915_buddy.h index 3940d632f208..7077742112ac 100644 --- a/drivers/gpu/drm/i915/i915_buddy.h +++ b/drivers/gpu/drm/i915/i915_buddy.h @@ -10,6 +10,8 @@ #include <linux/list.h> #include <linux/slab.h> +#include <drm/drm_print.h> + struct i915_buddy_block { #define I915_BUDDY_HEADER_OFFSET GENMASK_ULL(63, 12) #define I915_BUDDY_HEADER_STATE GENMASK_ULL(11, 10) @@ -69,6 +71,7 @@ struct i915_buddy_mm { /* Must be at least PAGE_SIZE */ u64 chunk_size; u64 size; + u64 avail; }; static inline u64 @@ -129,6 +132,11 @@ void i915_buddy_free(struct i915_buddy_mm *mm, struct i915_buddy_block *block); void i915_buddy_free_list(struct i915_buddy_mm *mm, struct list_head *objects); +void i915_buddy_print(struct i915_buddy_mm *mm, struct drm_printer *p); +void i915_buddy_block_print(struct i915_buddy_mm *mm, + struct i915_buddy_block *block, + struct drm_printer *p); + void i915_buddy_module_exit(void); int i915_buddy_module_init(void); diff --git a/drivers/gpu/drm/i915/i915_config.c b/drivers/gpu/drm/i915/i915_config.c index b79b5f6d2cfa..afb828dab53b 100644 --- a/drivers/gpu/drm/i915/i915_config.c +++ b/drivers/gpu/drm/i915/i915_config.c @@ -8,7 +8,7 @@ unsigned long i915_fence_context_timeout(const struct drm_i915_private *i915, u64 context) { - if (context && IS_ACTIVE(CONFIG_DRM_I915_FENCE_TIMEOUT)) + if (CONFIG_DRM_I915_FENCE_TIMEOUT && context) return msecs_to_jiffies_timeout(CONFIG_DRM_I915_FENCE_TIMEOUT); return 0; diff --git a/drivers/gpu/drm/i915/i915_debugfs.c b/drivers/gpu/drm/i915/i915_debugfs.c index 44969f5dde50..fe638b5da7c0 100644 --- a/drivers/gpu/drm/i915/i915_debugfs.c +++ b/drivers/gpu/drm/i915/i915_debugfs.c @@ -32,13 +32,15 @@ #include <drm/drm_debugfs.h> #include "gem/i915_gem_context.h" +#include "gt/intel_gt.h" #include "gt/intel_gt_buffer_pool.h" #include "gt/intel_gt_clock_utils.h" -#include "gt/intel_gt.h" +#include "gt/intel_gt_debugfs.h" #include "gt/intel_gt_pm.h" +#include "gt/intel_gt_pm_debugfs.h" #include "gt/intel_gt_requests.h" -#include "gt/intel_reset.h" #include "gt/intel_rc6.h" +#include "gt/intel_reset.h" #include "gt/intel_rps.h" #include "gt/intel_sseu_debugfs.h" @@ -48,7 +50,6 @@ #include "i915_scheduler.h" #include "i915_trace.h" #include "intel_pm.h" -#include "intel_sideband.h" static inline struct drm_i915_private *node_to_i915(struct drm_info_node *node) { @@ -139,7 +140,6 @@ void i915_debugfs_describe_obj(struct seq_file *m, struct drm_i915_gem_object *obj) { struct drm_i915_private *dev_priv = to_i915(obj->base.dev); - struct intel_engine_cs *engine; struct i915_vma *vma; int pin_count = 0; @@ -229,15 +229,12 @@ i915_debugfs_describe_obj(struct seq_file *m, struct drm_i915_gem_object *obj) seq_printf(m, " (stolen: %08llx)", obj->stolen->start); if (i915_gem_object_is_framebuffer(obj)) seq_printf(m, " (fb)"); - - engine = i915_gem_object_last_write_engine(obj); - if (engine) - seq_printf(m, " (%s)", engine->name); } static int i915_gem_object_info(struct seq_file *m, void *data) { struct drm_i915_private *i915 = node_to_i915(m->private); + struct drm_printer p = drm_seq_file_printer(m); struct intel_memory_region *mr; enum intel_region_id id; @@ -246,8 +243,7 @@ static int i915_gem_object_info(struct seq_file *m, void *data) atomic_read(&i915->mm.free_count), i915->mm.shrink_memory); for_each_memory_region(mr, i915, id) - seq_printf(m, "%s: total:%pa, available:%pa bytes\n", - mr->name, &mr->total, &mr->avail); + intel_memory_region_debug(mr, &p); return 0; } @@ -354,232 +350,12 @@ static const struct file_operations i915_error_state_fops = { static int i915_frequency_info(struct seq_file *m, void *unused) { - struct drm_i915_private *dev_priv = node_to_i915(m->private); - struct intel_uncore *uncore = &dev_priv->uncore; - struct intel_rps *rps = &dev_priv->gt.rps; - intel_wakeref_t wakeref; - - wakeref = intel_runtime_pm_get(&dev_priv->runtime_pm); - - if (GRAPHICS_VER(dev_priv) == 5) { - u16 rgvswctl = intel_uncore_read16(uncore, MEMSWCTL); - u16 rgvstat = intel_uncore_read16(uncore, MEMSTAT_ILK); - - seq_printf(m, "Requested P-state: %d\n", (rgvswctl >> 8) & 0xf); - seq_printf(m, "Requested VID: %d\n", rgvswctl & 0x3f); - seq_printf(m, "Current VID: %d\n", (rgvstat & MEMSTAT_VID_MASK) >> - MEMSTAT_VID_SHIFT); - seq_printf(m, "Current P-state: %d\n", - (rgvstat & MEMSTAT_PSTATE_MASK) >> MEMSTAT_PSTATE_SHIFT); - } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) { - u32 rpmodectl, freq_sts; - - rpmodectl = intel_uncore_read(&dev_priv->uncore, GEN6_RP_CONTROL); - seq_printf(m, "Video Turbo Mode: %s\n", - yesno(rpmodectl & GEN6_RP_MEDIA_TURBO)); - seq_printf(m, "HW control enabled: %s\n", - yesno(rpmodectl & GEN6_RP_ENABLE)); - seq_printf(m, "SW control enabled: %s\n", - yesno((rpmodectl & GEN6_RP_MEDIA_MODE_MASK) == - GEN6_RP_MEDIA_SW_MODE)); - - vlv_punit_get(dev_priv); - freq_sts = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS); - vlv_punit_put(dev_priv); - - seq_printf(m, "PUNIT_REG_GPU_FREQ_STS: 0x%08x\n", freq_sts); - seq_printf(m, "DDR freq: %d MHz\n", dev_priv->mem_freq); - - seq_printf(m, "actual GPU freq: %d MHz\n", - intel_gpu_freq(rps, (freq_sts >> 8) & 0xff)); - - seq_printf(m, "current GPU freq: %d MHz\n", - intel_gpu_freq(rps, rps->cur_freq)); - - seq_printf(m, "max GPU freq: %d MHz\n", - intel_gpu_freq(rps, rps->max_freq)); - - seq_printf(m, "min GPU freq: %d MHz\n", - intel_gpu_freq(rps, rps->min_freq)); - - seq_printf(m, "idle GPU freq: %d MHz\n", - intel_gpu_freq(rps, rps->idle_freq)); - - seq_printf(m, - "efficient (RPe) frequency: %d MHz\n", - intel_gpu_freq(rps, rps->efficient_freq)); - } else if (GRAPHICS_VER(dev_priv) >= 6) { - u32 rp_state_limits; - u32 gt_perf_status; - u32 rp_state_cap; - u32 rpmodectl, rpinclimit, rpdeclimit; - u32 rpstat, cagf, reqf; - u32 rpupei, rpcurup, rpprevup; - u32 rpdownei, rpcurdown, rpprevdown; - u32 pm_ier, pm_imr, pm_isr, pm_iir, pm_mask; - int max_freq; - - rp_state_limits = intel_uncore_read(&dev_priv->uncore, GEN6_RP_STATE_LIMITS); - if (IS_GEN9_LP(dev_priv)) { - rp_state_cap = intel_uncore_read(&dev_priv->uncore, BXT_RP_STATE_CAP); - gt_perf_status = intel_uncore_read(&dev_priv->uncore, BXT_GT_PERF_STATUS); - } else { - rp_state_cap = intel_uncore_read(&dev_priv->uncore, GEN6_RP_STATE_CAP); - gt_perf_status = intel_uncore_read(&dev_priv->uncore, GEN6_GT_PERF_STATUS); - } - - /* RPSTAT1 is in the GT power well */ - intel_uncore_forcewake_get(&dev_priv->uncore, FORCEWAKE_ALL); - - reqf = intel_uncore_read(&dev_priv->uncore, GEN6_RPNSWREQ); - if (GRAPHICS_VER(dev_priv) >= 9) - reqf >>= 23; - else { - reqf &= ~GEN6_TURBO_DISABLE; - if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) - reqf >>= 24; - else - reqf >>= 25; - } - reqf = intel_gpu_freq(rps, reqf); - - rpmodectl = intel_uncore_read(&dev_priv->uncore, GEN6_RP_CONTROL); - rpinclimit = intel_uncore_read(&dev_priv->uncore, GEN6_RP_UP_THRESHOLD); - rpdeclimit = intel_uncore_read(&dev_priv->uncore, GEN6_RP_DOWN_THRESHOLD); - - rpstat = intel_uncore_read(&dev_priv->uncore, GEN6_RPSTAT1); - rpupei = intel_uncore_read(&dev_priv->uncore, GEN6_RP_CUR_UP_EI) & GEN6_CURICONT_MASK; - rpcurup = intel_uncore_read(&dev_priv->uncore, GEN6_RP_CUR_UP) & GEN6_CURBSYTAVG_MASK; - rpprevup = intel_uncore_read(&dev_priv->uncore, GEN6_RP_PREV_UP) & GEN6_CURBSYTAVG_MASK; - rpdownei = intel_uncore_read(&dev_priv->uncore, GEN6_RP_CUR_DOWN_EI) & GEN6_CURIAVG_MASK; - rpcurdown = intel_uncore_read(&dev_priv->uncore, GEN6_RP_CUR_DOWN) & GEN6_CURBSYTAVG_MASK; - rpprevdown = intel_uncore_read(&dev_priv->uncore, GEN6_RP_PREV_DOWN) & GEN6_CURBSYTAVG_MASK; - cagf = intel_rps_read_actual_frequency(rps); - - intel_uncore_forcewake_put(&dev_priv->uncore, FORCEWAKE_ALL); - - if (GRAPHICS_VER(dev_priv) >= 11) { - pm_ier = intel_uncore_read(&dev_priv->uncore, GEN11_GPM_WGBOXPERF_INTR_ENABLE); - pm_imr = intel_uncore_read(&dev_priv->uncore, GEN11_GPM_WGBOXPERF_INTR_MASK); - /* - * The equivalent to the PM ISR & IIR cannot be read - * without affecting the current state of the system - */ - pm_isr = 0; - pm_iir = 0; - } else if (GRAPHICS_VER(dev_priv) >= 8) { - pm_ier = intel_uncore_read(&dev_priv->uncore, GEN8_GT_IER(2)); - pm_imr = intel_uncore_read(&dev_priv->uncore, GEN8_GT_IMR(2)); - pm_isr = intel_uncore_read(&dev_priv->uncore, GEN8_GT_ISR(2)); - pm_iir = intel_uncore_read(&dev_priv->uncore, GEN8_GT_IIR(2)); - } else { - pm_ier = intel_uncore_read(&dev_priv->uncore, GEN6_PMIER); - pm_imr = intel_uncore_read(&dev_priv->uncore, GEN6_PMIMR); - pm_isr = intel_uncore_read(&dev_priv->uncore, GEN6_PMISR); - pm_iir = intel_uncore_read(&dev_priv->uncore, GEN6_PMIIR); - } - pm_mask = intel_uncore_read(&dev_priv->uncore, GEN6_PMINTRMSK); - - seq_printf(m, "Video Turbo Mode: %s\n", - yesno(rpmodectl & GEN6_RP_MEDIA_TURBO)); - seq_printf(m, "HW control enabled: %s\n", - yesno(rpmodectl & GEN6_RP_ENABLE)); - seq_printf(m, "SW control enabled: %s\n", - yesno((rpmodectl & GEN6_RP_MEDIA_MODE_MASK) == - GEN6_RP_MEDIA_SW_MODE)); - - seq_printf(m, "PM IER=0x%08x IMR=0x%08x, MASK=0x%08x\n", - pm_ier, pm_imr, pm_mask); - if (GRAPHICS_VER(dev_priv) <= 10) - seq_printf(m, "PM ISR=0x%08x IIR=0x%08x\n", - pm_isr, pm_iir); - seq_printf(m, "pm_intrmsk_mbz: 0x%08x\n", - rps->pm_intrmsk_mbz); - seq_printf(m, "GT_PERF_STATUS: 0x%08x\n", gt_perf_status); - seq_printf(m, "Render p-state ratio: %d\n", - (gt_perf_status & (GRAPHICS_VER(dev_priv) >= 9 ? 0x1ff00 : 0xff00)) >> 8); - seq_printf(m, "Render p-state VID: %d\n", - gt_perf_status & 0xff); - seq_printf(m, "Render p-state limit: %d\n", - rp_state_limits & 0xff); - seq_printf(m, "RPSTAT1: 0x%08x\n", rpstat); - seq_printf(m, "RPMODECTL: 0x%08x\n", rpmodectl); - seq_printf(m, "RPINCLIMIT: 0x%08x\n", rpinclimit); - seq_printf(m, "RPDECLIMIT: 0x%08x\n", rpdeclimit); - seq_printf(m, "RPNSWREQ: %dMHz\n", reqf); - seq_printf(m, "CAGF: %dMHz\n", cagf); - seq_printf(m, "RP CUR UP EI: %d (%lldns)\n", - rpupei, - intel_gt_pm_interval_to_ns(&dev_priv->gt, rpupei)); - seq_printf(m, "RP CUR UP: %d (%lldun)\n", - rpcurup, - intel_gt_pm_interval_to_ns(&dev_priv->gt, rpcurup)); - seq_printf(m, "RP PREV UP: %d (%lldns)\n", - rpprevup, - intel_gt_pm_interval_to_ns(&dev_priv->gt, rpprevup)); - seq_printf(m, "Up threshold: %d%%\n", - rps->power.up_threshold); - - seq_printf(m, "RP CUR DOWN EI: %d (%lldns)\n", - rpdownei, - intel_gt_pm_interval_to_ns(&dev_priv->gt, - rpdownei)); - seq_printf(m, "RP CUR DOWN: %d (%lldns)\n", - rpcurdown, - intel_gt_pm_interval_to_ns(&dev_priv->gt, - rpcurdown)); - seq_printf(m, "RP PREV DOWN: %d (%lldns)\n", - rpprevdown, - intel_gt_pm_interval_to_ns(&dev_priv->gt, - rpprevdown)); - seq_printf(m, "Down threshold: %d%%\n", - rps->power.down_threshold); - - max_freq = (IS_GEN9_LP(dev_priv) ? rp_state_cap >> 0 : - rp_state_cap >> 16) & 0xff; - max_freq *= (IS_GEN9_BC(dev_priv) || - GRAPHICS_VER(dev_priv) >= 11 ? GEN9_FREQ_SCALER : 1); - seq_printf(m, "Lowest (RPN) frequency: %dMHz\n", - intel_gpu_freq(rps, max_freq)); - - max_freq = (rp_state_cap & 0xff00) >> 8; - max_freq *= (IS_GEN9_BC(dev_priv) || - GRAPHICS_VER(dev_priv) >= 11 ? GEN9_FREQ_SCALER : 1); - seq_printf(m, "Nominal (RP1) frequency: %dMHz\n", - intel_gpu_freq(rps, max_freq)); - - max_freq = (IS_GEN9_LP(dev_priv) ? rp_state_cap >> 16 : - rp_state_cap >> 0) & 0xff; - max_freq *= (IS_GEN9_BC(dev_priv) || - GRAPHICS_VER(dev_priv) >= 11 ? GEN9_FREQ_SCALER : 1); - seq_printf(m, "Max non-overclocked (RP0) frequency: %dMHz\n", - intel_gpu_freq(rps, max_freq)); - seq_printf(m, "Max overclocked frequency: %dMHz\n", - intel_gpu_freq(rps, rps->max_freq)); - - seq_printf(m, "Current freq: %d MHz\n", - intel_gpu_freq(rps, rps->cur_freq)); - seq_printf(m, "Actual freq: %d MHz\n", cagf); - seq_printf(m, "Idle freq: %d MHz\n", - intel_gpu_freq(rps, rps->idle_freq)); - seq_printf(m, "Min freq: %d MHz\n", - intel_gpu_freq(rps, rps->min_freq)); - seq_printf(m, "Boost freq: %d MHz\n", - intel_gpu_freq(rps, rps->boost_freq)); - seq_printf(m, "Max freq: %d MHz\n", - intel_gpu_freq(rps, rps->max_freq)); - seq_printf(m, - "efficient (RPe) frequency: %d MHz\n", - intel_gpu_freq(rps, rps->efficient_freq)); - } else { - seq_puts(m, "no P-state info available\n"); - } + struct drm_i915_private *i915 = node_to_i915(m->private); + struct intel_gt *gt = &i915->gt; + struct drm_printer p = drm_seq_file_printer(m); - seq_printf(m, "Current CD clock frequency: %d kHz\n", dev_priv->cdclk.hw.cdclk); - seq_printf(m, "Max CD clock frequency: %d kHz\n", dev_priv->max_cdclk_freq); - seq_printf(m, "Max pixel clock frequency: %d kHz\n", dev_priv->max_dotclk_freq); + intel_gt_pm_frequency_dump(gt, &p); - intel_runtime_pm_put(&dev_priv->runtime_pm, wakeref); return 0; } @@ -778,36 +554,18 @@ static int i915_wa_registers(struct seq_file *m, void *unused) return 0; } -static int -i915_wedged_get(void *data, u64 *val) +static int i915_wedged_get(void *data, u64 *val) { struct drm_i915_private *i915 = data; - int ret = intel_gt_terminally_wedged(&i915->gt); - switch (ret) { - case -EIO: - *val = 1; - return 0; - case 0: - *val = 0; - return 0; - default: - return ret; - } + return intel_gt_debugfs_reset_show(&i915->gt, val); } -static int -i915_wedged_set(void *data, u64 val) +static int i915_wedged_set(void *data, u64 val) { struct drm_i915_private *i915 = data; - /* Flush any previous reset before applying for a new one */ - wait_event(i915->gt.reset.queue, - !test_bit(I915_RESET_BACKOFF, &i915->gt.reset.flags)); - - intel_gt_handle_error(&i915->gt, val, I915_ERROR_CAPTURE, - "Manually set wedged engine mask = %llx", val); - return 0; + return intel_gt_debugfs_reset_store(&i915->gt, val); } DEFINE_SIMPLE_ATTRIBUTE(i915_wedged_fops, @@ -952,27 +710,15 @@ static int i915_sseu_status(struct seq_file *m, void *unused) static int i915_forcewake_open(struct inode *inode, struct file *file) { struct drm_i915_private *i915 = inode->i_private; - struct intel_gt *gt = &i915->gt; - atomic_inc(>->user_wakeref); - intel_gt_pm_get(gt); - if (GRAPHICS_VER(i915) >= 6) - intel_uncore_forcewake_user_get(gt->uncore); - - return 0; + return intel_gt_pm_debugfs_forcewake_user_open(&i915->gt); } static int i915_forcewake_release(struct inode *inode, struct file *file) { struct drm_i915_private *i915 = inode->i_private; - struct intel_gt *gt = &i915->gt; - if (GRAPHICS_VER(i915) >= 6) - intel_uncore_forcewake_user_put(&i915->uncore); - intel_gt_pm_put(gt); - atomic_dec(>->user_wakeref); - - return 0; + return intel_gt_pm_debugfs_forcewake_user_release(&i915->gt); } static const struct file_operations i915_forcewake_fops = { diff --git a/drivers/gpu/drm/i915/i915_drv.c b/drivers/gpu/drm/i915/i915_drv.c index 59fb4c710c8c..b18a250e5d2e 100644 --- a/drivers/gpu/drm/i915/i915_drv.c +++ b/drivers/gpu/drm/i915/i915_drv.c @@ -67,6 +67,8 @@ #include "gt/intel_gt_pm.h" #include "gt/intel_rc6.h" +#include "pxp/intel_pxp_pm.h" + #include "i915_debugfs.h" #include "i915_drv.h" #include "i915_ioc32.h" @@ -82,9 +84,9 @@ #include "intel_dram.h" #include "intel_gvt.h" #include "intel_memory_region.h" +#include "intel_pcode.h" #include "intel_pm.h" #include "intel_region_ttm.h" -#include "intel_sideband.h" #include "vlv_suspend.h" static const struct drm_driver driver; @@ -97,7 +99,7 @@ static int i915_get_bridge_dev(struct drm_i915_private *dev_priv) pci_get_domain_bus_and_slot(domain, 0, PCI_DEVFN(0, 0)); if (!dev_priv->bridge_dev) { drm_err(&dev_priv->drm, "bridge device not found\n"); - return -1; + return -EIO; } return 0; } @@ -409,8 +411,9 @@ static int i915_driver_mmio_probe(struct drm_i915_private *dev_priv) if (i915_inject_probe_failure(dev_priv)) return -ENODEV; - if (i915_get_bridge_dev(dev_priv)) - return -EIO; + ret = i915_get_bridge_dev(dev_priv); + if (ret < 0) + return ret; ret = intel_uncore_init_mmio(&dev_priv->uncore); if (ret < 0) @@ -588,8 +591,6 @@ static int i915_driver_hw_probe(struct drm_i915_private *dev_priv) pci_set_master(pdev); - intel_gt_init_workarounds(dev_priv); - /* On the 945G/GM, the chipset reports the MSI capability on the * integrated graphics even though the support isn't actually there * according to the published specs. It doesn't appear to function @@ -1096,9 +1097,7 @@ static int i915_drm_prepare(struct drm_device *dev) * split out that work and pull it forward so that after point, * the GPU is not woken again. */ - i915_gem_suspend(i915); - - return 0; + return i915_gem_backup_suspend(i915); } static int i915_drm_suspend(struct drm_device *dev) diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h index 005b1cec7007..12256218634f 100644 --- a/drivers/gpu/drm/i915/i915_drv.h +++ b/drivers/gpu/drm/i915/i915_drv.h @@ -323,15 +323,15 @@ struct intel_crtc; struct intel_limit; struct dpll; -struct drm_i915_display_funcs { - void (*get_cdclk)(struct drm_i915_private *dev_priv, - struct intel_cdclk_config *cdclk_config); - void (*set_cdclk)(struct drm_i915_private *dev_priv, - const struct intel_cdclk_config *cdclk_config, - enum pipe pipe); - int (*bw_calc_min_cdclk)(struct intel_atomic_state *state); - int (*get_fifo_size)(struct drm_i915_private *dev_priv, - enum i9xx_plane_id i9xx_plane); +/* functions used internal in intel_pm.c */ +struct drm_i915_clock_gating_funcs { + void (*init_clock_gating)(struct drm_i915_private *dev_priv); +}; + +/* functions used for watermark calcs for display. */ +struct drm_i915_wm_disp_funcs { + /* update_wm is for legacy wm management */ + void (*update_wm)(struct drm_i915_private *dev_priv); int (*compute_pipe_wm)(struct intel_atomic_state *state, struct intel_crtc *crtc); int (*compute_intermediate_wm)(struct intel_atomic_state *state, @@ -343,39 +343,9 @@ struct drm_i915_display_funcs { void (*optimize_watermarks)(struct intel_atomic_state *state, struct intel_crtc *crtc); int (*compute_global_watermarks)(struct intel_atomic_state *state); - void (*update_wm)(struct intel_crtc *crtc); - int (*modeset_calc_cdclk)(struct intel_cdclk_state *state); - u8 (*calc_voltage_level)(int cdclk); - /* Returns the active state of the crtc, and if the crtc is active, - * fills out the pipe-config with the hw state. */ - bool (*get_pipe_config)(struct intel_crtc *, - struct intel_crtc_state *); - void (*get_initial_plane_config)(struct intel_crtc *, - struct intel_initial_plane_config *); - int (*crtc_compute_clock)(struct intel_crtc *crtc, - struct intel_crtc_state *crtc_state); - void (*crtc_enable)(struct intel_atomic_state *state, - struct intel_crtc *crtc); - void (*crtc_disable)(struct intel_atomic_state *state, - struct intel_crtc *crtc); - void (*commit_modeset_enables)(struct intel_atomic_state *state); - void (*commit_modeset_disables)(struct intel_atomic_state *state); - void (*audio_codec_enable)(struct intel_encoder *encoder, - const struct intel_crtc_state *crtc_state, - const struct drm_connector_state *conn_state); - void (*audio_codec_disable)(struct intel_encoder *encoder, - const struct intel_crtc_state *old_crtc_state, - const struct drm_connector_state *old_conn_state); - void (*fdi_link_train)(struct intel_crtc *crtc, - const struct intel_crtc_state *crtc_state); - void (*init_clock_gating)(struct drm_i915_private *dev_priv); - void (*hpd_irq_setup)(struct drm_i915_private *dev_priv); - /* clock updates for mode set */ - /* cursor updates */ - /* render clock increase/decrease */ - /* display clock increase/decrease */ - /* pll clock increase/decrease */ +}; +struct intel_color_funcs { int (*color_check)(struct intel_crtc_state *crtc_state); /* * Program double buffered color management registers during @@ -394,6 +364,53 @@ struct drm_i915_display_funcs { void (*read_luts)(struct intel_crtc_state *crtc_state); }; +struct intel_audio_funcs { + void (*audio_codec_enable)(struct intel_encoder *encoder, + const struct intel_crtc_state *crtc_state, + const struct drm_connector_state *conn_state); + void (*audio_codec_disable)(struct intel_encoder *encoder, + const struct intel_crtc_state *old_crtc_state, + const struct drm_connector_state *old_conn_state); +}; + +struct intel_cdclk_funcs { + void (*get_cdclk)(struct drm_i915_private *dev_priv, + struct intel_cdclk_config *cdclk_config); + void (*set_cdclk)(struct drm_i915_private *dev_priv, + const struct intel_cdclk_config *cdclk_config, + enum pipe pipe); + int (*bw_calc_min_cdclk)(struct intel_atomic_state *state); + int (*modeset_calc_cdclk)(struct intel_cdclk_state *state); + u8 (*calc_voltage_level)(int cdclk); +}; + +struct intel_hotplug_funcs { + void (*hpd_irq_setup)(struct drm_i915_private *dev_priv); +}; + +struct intel_fdi_funcs { + void (*fdi_link_train)(struct intel_crtc *crtc, + const struct intel_crtc_state *crtc_state); +}; + +struct intel_dpll_funcs { + int (*crtc_compute_clock)(struct intel_crtc_state *crtc_state); +}; + +struct drm_i915_display_funcs { + /* Returns the active state of the crtc, and if the crtc is active, + * fills out the pipe-config with the hw state. */ + bool (*get_pipe_config)(struct intel_crtc *, + struct intel_crtc_state *); + void (*get_initial_plane_config)(struct intel_crtc *, + struct intel_initial_plane_config *); + void (*crtc_enable)(struct intel_atomic_state *state, + struct intel_crtc *crtc); + void (*crtc_disable)(struct intel_atomic_state *state, + struct intel_crtc *crtc); + void (*commit_modeset_enables)(struct intel_atomic_state *state); +}; + #define I915_COLOR_UNEVICTABLE (-1) /* a non-vma sharing the address space */ @@ -454,7 +471,6 @@ struct intel_fbc { } fb; unsigned int fence_y_offset; - u16 gen9_wa_cfb_stride; u16 interval; s8 fence_id; bool psr2_active; @@ -479,9 +495,10 @@ struct intel_fbc { u64 modifier; } fb; - int cfb_size; + unsigned int cfb_stride; + unsigned int cfb_size; unsigned int fence_y_offset; - u16 gen9_wa_cfb_stride; + u16 override_cfb_stride; u16 interval; s8 fence_id; bool plane_visible; @@ -636,22 +653,6 @@ i915_fence_timeout(const struct drm_i915_private *i915) /* Amount of PSF GV points, BSpec precisely defines this */ #define I915_NUM_PSF_GV_POINTS 3 -struct ddi_vbt_port_info { - /* Non-NULL if port present. */ - struct intel_bios_encoder_data *devdata; - - int max_tmds_clock; - - /* This is an index in the HDMI/DVI DDI buffer translation table. */ - u8 hdmi_level_shift; - u8 hdmi_level_shift_set:1; - - u8 alternate_aux_channel; - u8 alternate_ddc_pin; - - int dp_max_link_rate; /* 0 for not limited by VBT */ -}; - enum psr_lines_to_wait { PSR_0_LINES_TO_WAIT = 0, PSR_1_LINE_TO_WAIT, @@ -706,6 +707,7 @@ struct intel_vbt_data { struct { u16 pwm_freq_hz; + u16 brightness_precision_bits; bool present; bool active_low_pwm; u8 min_brightness; /* min_brightness/255 of max */ @@ -732,7 +734,7 @@ struct intel_vbt_data { struct list_head display_devices; - struct ddi_vbt_port_info ddi_port_info[I915_MAX_PORTS]; + struct intel_bios_encoder_data *ports[I915_MAX_PORTS]; /* Non-NULL if port present. */ struct sdvo_device_mapping sdvo_mappings[2]; }; @@ -886,8 +888,6 @@ struct drm_i915_private { */ u32 gpio_mmio_base; - u32 hsw_psr_mmio_adjust; - /* MMIO base address for MIPI regs */ u32 mipi_mmio_base; @@ -974,8 +974,32 @@ struct drm_i915_private { /* unbound hipri wq for page flips/plane updates */ struct workqueue_struct *flip_wq; + /* pm private clock gating functions */ + const struct drm_i915_clock_gating_funcs *clock_gating_funcs; + + /* pm display functions */ + const struct drm_i915_wm_disp_funcs *wm_disp; + + /* irq display functions */ + const struct intel_hotplug_funcs *hotplug_funcs; + + /* fdi display functions */ + const struct intel_fdi_funcs *fdi_funcs; + + /* display pll funcs */ + const struct intel_dpll_funcs *dpll_funcs; + /* Display functions */ - struct drm_i915_display_funcs display; + const struct drm_i915_display_funcs *display; + + /* Display internal color functions */ + const struct intel_color_funcs *color_funcs; + + /* Display internal audio functions */ + const struct intel_audio_funcs *audio_funcs; + + /* Display CDCLK functions */ + const struct intel_cdclk_funcs *cdclk_funcs; /* PCH chipset type */ enum intel_pch pch_type; @@ -1022,8 +1046,6 @@ struct drm_i915_private { */ u8 active_pipes; - struct i915_wa_list gt_wa_list; - struct i915_frontbuffer_tracking fb_tracking; struct intel_atomic_helper { @@ -1665,6 +1687,7 @@ IS_SUBPLATFORM(const struct drm_i915_private *i915, #define HAS_IPS(dev_priv) (IS_HSW_ULT(dev_priv) || IS_BROADWELL(dev_priv)) #define HAS_DP_MST(dev_priv) (INTEL_INFO(dev_priv)->display.has_dp_mst) +#define HAS_DP20(dev_priv) (IS_DG2(dev_priv)) #define HAS_CDCLK_CRAWL(dev_priv) (INTEL_INFO(dev_priv)->display.has_cdclk_crawl) #define HAS_DDI(dev_priv) (INTEL_INFO(dev_priv)->display.has_ddi) @@ -1702,6 +1725,9 @@ IS_SUBPLATFORM(const struct drm_i915_private *i915, #define HAS_GLOBAL_MOCS_REGISTERS(dev_priv) (INTEL_INFO(dev_priv)->has_global_mocs) +#define HAS_PXP(dev_priv) ((IS_ENABLED(CONFIG_DRM_I915_PXP) && \ + INTEL_INFO(dev_priv)->has_pxp) && \ + VDBOX_MASK(&dev_priv->gt)) #define HAS_GMCH(dev_priv) (INTEL_INFO(dev_priv)->display.has_gmch) @@ -1721,6 +1747,8 @@ IS_SUBPLATFORM(const struct drm_i915_private *i915, #define HAS_VRR(i915) (GRAPHICS_VER(i915) >= 12) +#define HAS_ASYNC_FLIPS(i915) (DISPLAY_VER(i915) >= 5) + /* Only valid when HAS_DISPLAY() is true */ #define INTEL_DISPLAY_ENABLED(dev_priv) \ (drm_WARN_ON(&(dev_priv)->drm, !HAS_DISPLAY(dev_priv)), !(dev_priv)->params.disable_display) @@ -1881,11 +1909,11 @@ i915_gem_vm_lookup(struct drm_i915_file_private *file_priv, u32 id) { struct i915_address_space *vm; - rcu_read_lock(); + xa_lock(&file_priv->vm_xa); vm = xa_load(&file_priv->vm_xa, id); - if (vm && !kref_get_unless_zero(&vm->ref)) - vm = NULL; - rcu_read_unlock(); + if (vm) + kref_get(&vm->ref); + xa_unlock(&file_priv->vm_xa); return vm; } diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c index 590efc8b0265..981e383d1a5d 100644 --- a/drivers/gpu/drm/i915/i915_gem.c +++ b/drivers/gpu/drm/i915/i915_gem.c @@ -1139,8 +1139,6 @@ void i915_gem_driver_release(struct drm_i915_private *dev_priv) { intel_gt_driver_release(&dev_priv->gt); - intel_wa_list_free(&dev_priv->gt_wa_list); - intel_uc_cleanup_firmwares(&dev_priv->gt.uc); i915_gem_drain_freed_objects(dev_priv); diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.c b/drivers/gpu/drm/i915/i915_gem_gtt.c index 36489be4896b..cd5f2348a187 100644 --- a/drivers/gpu/drm/i915/i915_gem_gtt.c +++ b/drivers/gpu/drm/i915/i915_gem_gtt.c @@ -30,7 +30,7 @@ int i915_gem_gtt_prepare_pages(struct drm_i915_gem_object *obj, do { if (dma_map_sg_attrs(obj->base.dev->dev, pages->sgl, pages->nents, - PCI_DMA_BIDIRECTIONAL, + DMA_BIDIRECTIONAL, DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_NO_KERNEL_MAPPING | DMA_ATTR_NO_WARN)) @@ -64,7 +64,7 @@ void i915_gem_gtt_finish_pages(struct drm_i915_gem_object *obj, usleep_range(100, 250); dma_unmap_sg(i915->drm.dev, pages->sgl, pages->nents, - PCI_DMA_BIDIRECTIONAL); + DMA_BIDIRECTIONAL); } /** diff --git a/drivers/gpu/drm/i915/i915_gem_ww.h b/drivers/gpu/drm/i915/i915_gem_ww.h index f6b1a796667b..86f0fe343de6 100644 --- a/drivers/gpu/drm/i915/i915_gem_ww.h +++ b/drivers/gpu/drm/i915/i915_gem_ww.h @@ -11,8 +11,7 @@ struct i915_gem_ww_ctx { struct ww_acquire_ctx ctx; struct list_head obj_list; struct drm_i915_gem_object *contended; - unsigned short intr; - unsigned short loop; + bool intr; }; void i915_gem_ww_ctx_init(struct i915_gem_ww_ctx *ctx, bool intr); @@ -20,31 +19,23 @@ void i915_gem_ww_ctx_fini(struct i915_gem_ww_ctx *ctx); int __must_check i915_gem_ww_ctx_backoff(struct i915_gem_ww_ctx *ctx); void i915_gem_ww_unlock_single(struct drm_i915_gem_object *obj); -/* Internal functions used by the inlines! Don't use. */ +/* Internal function used by the inlines! Don't use. */ static inline int __i915_gem_ww_fini(struct i915_gem_ww_ctx *ww, int err) { - ww->loop = 0; if (err == -EDEADLK) { err = i915_gem_ww_ctx_backoff(ww); if (!err) - ww->loop = 1; + err = -EDEADLK; } - if (!ww->loop) + if (err != -EDEADLK) i915_gem_ww_ctx_fini(ww); return err; } -static inline void -__i915_gem_ww_init(struct i915_gem_ww_ctx *ww, bool intr) -{ - i915_gem_ww_ctx_init(ww, intr); - ww->loop = 1; -} - -#define for_i915_gem_ww(_ww, _err, _intr) \ - for (__i915_gem_ww_init(_ww, _intr); (_ww)->loop; \ - _err = __i915_gem_ww_fini(_ww, _err)) - +#define for_i915_gem_ww(_ww, _err, _intr) \ + for (i915_gem_ww_ctx_init(_ww, _intr), (_err) = -EDEADLK; \ + (_err) == -EDEADLK; \ + (_err) = __i915_gem_ww_fini(_ww, _err)) #endif diff --git a/drivers/gpu/drm/i915/i915_gpu_error.c b/drivers/gpu/drm/i915/i915_gpu_error.c index 9cf6ac575de1..2a2d7643b551 100644 --- a/drivers/gpu/drm/i915/i915_gpu_error.c +++ b/drivers/gpu/drm/i915/i915_gpu_error.c @@ -431,6 +431,7 @@ static void error_print_instdone(struct drm_i915_error_state_buf *m, const struct sseu_dev_info *sseu = &ee->engine->gt->info.sseu; int slice; int subslice; + int iter; err_printf(m, " INSTDONE: 0x%08x\n", ee->instdone.instdone); @@ -444,19 +445,38 @@ static void error_print_instdone(struct drm_i915_error_state_buf *m, if (GRAPHICS_VER(m->i915) <= 6) return; - for_each_instdone_slice_subslice(m->i915, sseu, slice, subslice) - err_printf(m, " SAMPLER_INSTDONE[%d][%d]: 0x%08x\n", - slice, subslice, - ee->instdone.sampler[slice][subslice]); + if (GRAPHICS_VER_FULL(m->i915) >= IP_VER(12, 50)) { + for_each_instdone_gslice_dss_xehp(m->i915, sseu, iter, slice, subslice) + err_printf(m, " SAMPLER_INSTDONE[%d][%d]: 0x%08x\n", + slice, subslice, + ee->instdone.sampler[slice][subslice]); - for_each_instdone_slice_subslice(m->i915, sseu, slice, subslice) - err_printf(m, " ROW_INSTDONE[%d][%d]: 0x%08x\n", - slice, subslice, - ee->instdone.row[slice][subslice]); + for_each_instdone_gslice_dss_xehp(m->i915, sseu, iter, slice, subslice) + err_printf(m, " ROW_INSTDONE[%d][%d]: 0x%08x\n", + slice, subslice, + ee->instdone.row[slice][subslice]); + } else { + for_each_instdone_slice_subslice(m->i915, sseu, slice, subslice) + err_printf(m, " SAMPLER_INSTDONE[%d][%d]: 0x%08x\n", + slice, subslice, + ee->instdone.sampler[slice][subslice]); + + for_each_instdone_slice_subslice(m->i915, sseu, slice, subslice) + err_printf(m, " ROW_INSTDONE[%d][%d]: 0x%08x\n", + slice, subslice, + ee->instdone.row[slice][subslice]); + } if (GRAPHICS_VER(m->i915) < 12) return; + if (GRAPHICS_VER_FULL(m->i915) >= IP_VER(12, 55)) { + for_each_instdone_gslice_dss_xehp(m->i915, sseu, iter, slice, subslice) + err_printf(m, " GEOM_SVGUNIT_INSTDONE[%d][%d]: 0x%08x\n", + slice, subslice, + ee->instdone.geom_svg[slice][subslice]); + } + err_printf(m, " SC_INSTDONE_EXTRA: 0x%08x\n", ee->instdone.slice_common_extra[0]); err_printf(m, " SC_INSTDONE_EXTRA2: 0x%08x\n", @@ -733,7 +753,8 @@ static void err_print_gt(struct drm_i915_error_state_buf *m, * only exists if the corresponding VCS engine is * present. */ - if (!HAS_ENGINE(gt->_gt, _VCS(i * 2))) + if ((gt->_gt->info.sfc_mask & BIT(i)) == 0 || + !HAS_ENGINE(gt->_gt, _VCS(i * 2))) continue; err_printf(m, " SFC_DONE[%d]: 0x%08x\n", i, @@ -1612,7 +1633,8 @@ static void gt_record_regs(struct intel_gt_coredump *gt) * only exists if the corresponding VCS engine is * present. */ - if (!HAS_ENGINE(gt->_gt, _VCS(i * 2))) + if ((gt->_gt->info.sfc_mask & BIT(i)) == 0 || + !HAS_ENGINE(gt->_gt, _VCS(i * 2))) continue; gt->sfc_done[i] = diff --git a/drivers/gpu/drm/i915/i915_irq.c b/drivers/gpu/drm/i915/i915_irq.c index 9bc4f4a8e12e..77680bca46ee 100644 --- a/drivers/gpu/drm/i915/i915_irq.c +++ b/drivers/gpu/drm/i915/i915_irq.c @@ -359,9 +359,8 @@ void i915_hotplug_interrupt_update(struct drm_i915_private *dev_priv, * @interrupt_mask: mask of interrupt bits to update * @enabled_irq_mask: mask of interrupt bits to enable */ -void ilk_update_display_irq(struct drm_i915_private *dev_priv, - u32 interrupt_mask, - u32 enabled_irq_mask) +static void ilk_update_display_irq(struct drm_i915_private *dev_priv, + u32 interrupt_mask, u32 enabled_irq_mask) { u32 new_val; @@ -380,6 +379,16 @@ void ilk_update_display_irq(struct drm_i915_private *dev_priv, } } +void ilk_enable_display_irq(struct drm_i915_private *i915, u32 bits) +{ + ilk_update_display_irq(i915, bits, bits); +} + +void ilk_disable_display_irq(struct drm_i915_private *i915, u32 bits) +{ + ilk_update_display_irq(i915, bits, 0); +} + /** * bdw_update_port_irq - update DE port interrupt * @dev_priv: driver private @@ -419,10 +428,9 @@ static void bdw_update_port_irq(struct drm_i915_private *dev_priv, * @interrupt_mask: mask of interrupt bits to update * @enabled_irq_mask: mask of interrupt bits to enable */ -void bdw_update_pipe_irq(struct drm_i915_private *dev_priv, - enum pipe pipe, - u32 interrupt_mask, - u32 enabled_irq_mask) +static void bdw_update_pipe_irq(struct drm_i915_private *dev_priv, + enum pipe pipe, u32 interrupt_mask, + u32 enabled_irq_mask) { u32 new_val; @@ -444,15 +452,27 @@ void bdw_update_pipe_irq(struct drm_i915_private *dev_priv, } } +void bdw_enable_pipe_irq(struct drm_i915_private *i915, + enum pipe pipe, u32 bits) +{ + bdw_update_pipe_irq(i915, pipe, bits, bits); +} + +void bdw_disable_pipe_irq(struct drm_i915_private *i915, + enum pipe pipe, u32 bits) +{ + bdw_update_pipe_irq(i915, pipe, bits, 0); +} + /** * ibx_display_interrupt_update - update SDEIMR * @dev_priv: driver private * @interrupt_mask: mask of interrupt bits to update * @enabled_irq_mask: mask of interrupt bits to enable */ -void ibx_display_interrupt_update(struct drm_i915_private *dev_priv, - u32 interrupt_mask, - u32 enabled_irq_mask) +static void ibx_display_interrupt_update(struct drm_i915_private *dev_priv, + u32 interrupt_mask, + u32 enabled_irq_mask) { u32 sdeimr = intel_uncore_read(&dev_priv->uncore, SDEIMR); sdeimr &= ~interrupt_mask; @@ -469,6 +489,16 @@ void ibx_display_interrupt_update(struct drm_i915_private *dev_priv, intel_uncore_posting_read(&dev_priv->uncore, SDEIMR); } +void ibx_enable_display_interrupt(struct drm_i915_private *i915, u32 bits) +{ + ibx_display_interrupt_update(i915, bits, bits); +} + +void ibx_disable_display_interrupt(struct drm_i915_private *i915, u32 bits) +{ + ibx_display_interrupt_update(i915, bits, 0); +} + u32 i915_pipestat_enable_mask(struct drm_i915_private *dev_priv, enum pipe pipe) { @@ -2093,22 +2123,6 @@ static void ivb_display_irq_handler(struct drm_i915_private *dev_priv, if (de_iir & DE_ERR_INT_IVB) ivb_err_int_handler(dev_priv); - if (de_iir & DE_EDP_PSR_INT_HSW) { - struct intel_encoder *encoder; - - for_each_intel_encoder_with_psr(&dev_priv->drm, encoder) { - struct intel_dp *intel_dp = enc_to_intel_dp(encoder); - - u32 psr_iir = intel_uncore_read(&dev_priv->uncore, - EDP_PSR_IIR); - - intel_psr_irq_handler(intel_dp, psr_iir); - intel_uncore_write(&dev_priv->uncore, - EDP_PSR_IIR, psr_iir); - break; - } - } - if (de_iir & DE_AUX_CHANNEL_A_IVB) dp_aux_irq_handler(dev_priv); @@ -4331,6 +4345,20 @@ static irqreturn_t i965_irq_handler(int irq, void *arg) return ret; } +#define HPD_FUNCS(platform) \ +static const struct intel_hotplug_funcs platform##_hpd_funcs = { \ + .hpd_irq_setup = platform##_hpd_irq_setup, \ +} + +HPD_FUNCS(i915); +HPD_FUNCS(dg1); +HPD_FUNCS(gen11); +HPD_FUNCS(bxt); +HPD_FUNCS(icp); +HPD_FUNCS(spt); +HPD_FUNCS(ilk); +#undef HPD_FUNCS + /** * intel_irq_init - initializes irq support * @dev_priv: i915 device instance @@ -4381,20 +4409,20 @@ void intel_irq_init(struct drm_i915_private *dev_priv) if (HAS_GMCH(dev_priv)) { if (I915_HAS_HOTPLUG(dev_priv)) - dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup; + dev_priv->hotplug_funcs = &i915_hpd_funcs; } else { if (HAS_PCH_DG1(dev_priv)) - dev_priv->display.hpd_irq_setup = dg1_hpd_irq_setup; + dev_priv->hotplug_funcs = &dg1_hpd_funcs; else if (DISPLAY_VER(dev_priv) >= 11) - dev_priv->display.hpd_irq_setup = gen11_hpd_irq_setup; + dev_priv->hotplug_funcs = &gen11_hpd_funcs; else if (IS_GEMINILAKE(dev_priv) || IS_BROXTON(dev_priv)) - dev_priv->display.hpd_irq_setup = bxt_hpd_irq_setup; + dev_priv->hotplug_funcs = &bxt_hpd_funcs; else if (INTEL_PCH_TYPE(dev_priv) >= PCH_ICP) - dev_priv->display.hpd_irq_setup = icp_hpd_irq_setup; + dev_priv->hotplug_funcs = &icp_hpd_funcs; else if (INTEL_PCH_TYPE(dev_priv) >= PCH_SPT) - dev_priv->display.hpd_irq_setup = spt_hpd_irq_setup; + dev_priv->hotplug_funcs = &spt_hpd_funcs; else - dev_priv->display.hpd_irq_setup = ilk_hpd_irq_setup; + dev_priv->hotplug_funcs = &ilk_hpd_funcs; } } diff --git a/drivers/gpu/drm/i915/i915_irq.h b/drivers/gpu/drm/i915/i915_irq.h index e43b6734f21b..0eb90d271fa7 100644 --- a/drivers/gpu/drm/i915/i915_irq.h +++ b/drivers/gpu/drm/i915/i915_irq.h @@ -9,9 +9,9 @@ #include <linux/ktime.h> #include <linux/types.h> -#include "display/intel_display.h" #include "i915_reg.h" +enum pipe; struct drm_crtc; struct drm_device; struct drm_display_mode; @@ -40,46 +40,15 @@ void valleyview_disable_display_irqs(struct drm_i915_private *dev_priv); void i915_hotplug_interrupt_update(struct drm_i915_private *dev_priv, u32 mask, u32 bits); -void ilk_update_display_irq(struct drm_i915_private *dev_priv, - u32 interrupt_mask, - u32 enabled_irq_mask); -static inline void -ilk_enable_display_irq(struct drm_i915_private *dev_priv, u32 bits) -{ - ilk_update_display_irq(dev_priv, bits, bits); -} -static inline void -ilk_disable_display_irq(struct drm_i915_private *dev_priv, u32 bits) -{ - ilk_update_display_irq(dev_priv, bits, 0); -} -void bdw_update_pipe_irq(struct drm_i915_private *dev_priv, - enum pipe pipe, - u32 interrupt_mask, - u32 enabled_irq_mask); -static inline void bdw_enable_pipe_irq(struct drm_i915_private *dev_priv, - enum pipe pipe, u32 bits) -{ - bdw_update_pipe_irq(dev_priv, pipe, bits, bits); -} -static inline void bdw_disable_pipe_irq(struct drm_i915_private *dev_priv, - enum pipe pipe, u32 bits) -{ - bdw_update_pipe_irq(dev_priv, pipe, bits, 0); -} -void ibx_display_interrupt_update(struct drm_i915_private *dev_priv, - u32 interrupt_mask, - u32 enabled_irq_mask); -static inline void -ibx_enable_display_interrupt(struct drm_i915_private *dev_priv, u32 bits) -{ - ibx_display_interrupt_update(dev_priv, bits, bits); -} -static inline void -ibx_disable_display_interrupt(struct drm_i915_private *dev_priv, u32 bits) -{ - ibx_display_interrupt_update(dev_priv, bits, 0); -} + +void ilk_enable_display_irq(struct drm_i915_private *i915, u32 bits); +void ilk_disable_display_irq(struct drm_i915_private *i915, u32 bits); + +void bdw_enable_pipe_irq(struct drm_i915_private *i915, enum pipe pipe, u32 bits); +void bdw_disable_pipe_irq(struct drm_i915_private *i915, enum pipe pipe, u32 bits); + +void ibx_enable_display_interrupt(struct drm_i915_private *i915, u32 bits); +void ibx_disable_display_interrupt(struct drm_i915_private *i915, u32 bits); void gen5_enable_gt_irq(struct drm_i915_private *dev_priv, u32 mask); void gen5_disable_gt_irq(struct drm_i915_private *dev_priv, u32 mask); diff --git a/drivers/gpu/drm/i915/i915_module.c b/drivers/gpu/drm/i915/i915_module.c index d8b4482c69d0..ab2295dd4500 100644 --- a/drivers/gpu/drm/i915/i915_module.c +++ b/drivers/gpu/drm/i915/i915_module.c @@ -67,8 +67,8 @@ static const struct { { .init = i915_mock_selftests }, { .init = i915_pmu_init, .exit = i915_pmu_exit }, - { .init = i915_register_pci_driver, - .exit = i915_unregister_pci_driver }, + { .init = i915_pci_register_driver, + .exit = i915_pci_unregister_driver }, { .init = i915_perf_sysctl_register, .exit = i915_perf_sysctl_unregister }, }; diff --git a/drivers/gpu/drm/i915/i915_params.h b/drivers/gpu/drm/i915/i915_params.h index f27eceb82c0f..8d725b64592d 100644 --- a/drivers/gpu/drm/i915/i915_params.h +++ b/drivers/gpu/drm/i915/i915_params.h @@ -55,7 +55,7 @@ struct drm_printer; param(int, enable_fbc, -1, 0600) \ param(int, enable_psr, -1, 0600) \ param(bool, psr_safest_params, false, 0400) \ - param(bool, enable_psr2_sel_fetch, false, 0400) \ + param(bool, enable_psr2_sel_fetch, true, 0400) \ param(int, disable_power_well, -1, 0400) \ param(int, enable_ips, 1, 0600) \ param(int, invert_brightness, 0, 0600) \ diff --git a/drivers/gpu/drm/i915/i915_pci.c b/drivers/gpu/drm/i915/i915_pci.c index 1bbd09ad5287..169837de395d 100644 --- a/drivers/gpu/drm/i915/i915_pci.c +++ b/drivers/gpu/drm/i915/i915_pci.c @@ -537,8 +537,6 @@ static const struct intel_device_info vlv_info = { BIT(TRANSCODER_C) | BIT(TRANSCODER_EDP), \ .display.has_ddi = 1, \ .display.has_fpga_dbg = 1, \ - .display.has_psr = 1, \ - .display.has_psr_hw_tracking = 1, \ .display.has_dp_mst = 1, \ .has_rc6p = 0 /* RC6p removed-by HSW */, \ HSW_PIPE_OFFSETS, \ @@ -642,6 +640,8 @@ static const struct intel_device_info chv_info = { .has_gt_uc = 1, \ .display.has_hdcp = 1, \ .display.has_ipc = 1, \ + .display.has_psr = 1, \ + .display.has_psr_hw_tracking = 1, \ .dbuf.size = 896 - 4, /* 4 blocks for bypass path allocation */ \ .dbuf.slice_mask = BIT(DBUF_S1) @@ -865,6 +865,7 @@ static const struct intel_device_info jsl_info = { }, \ TGL_CURSOR_OFFSETS, \ .has_global_mocs = 1, \ + .has_pxp = 1, \ .display.has_dsb = 1 static const struct intel_device_info tgl_info = { @@ -891,10 +892,11 @@ static const struct intel_device_info rkl_info = { #define DGFX_FEATURES \ .memory_regions = REGION_SMEM | REGION_LMEM | REGION_STOLEN_LMEM, \ .has_llc = 0, \ + .has_pxp = 0, \ .has_snoop = 1, \ .is_dgfx = 1 -static const struct intel_device_info dg1_info __maybe_unused = { +static const struct intel_device_info dg1_info = { GEN12_FEATURES, DGFX_FEATURES, .graphics_rel = 10, @@ -912,7 +914,6 @@ static const struct intel_device_info adl_s_info = { GEN12_FEATURES, PLATFORM(INTEL_ALDERLAKE_S), .pipe_mask = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D), - .require_force_probe = 1, .display.has_hti = 1, .display.has_psr_hw_tracking = 0, .platform_engine_mask = @@ -1115,6 +1116,7 @@ static const struct pci_device_id pciidlist[] = { INTEL_RKL_IDS(&rkl_info), INTEL_ADLS_IDS(&adl_s_info), INTEL_ADLP_IDS(&adl_p_info), + INTEL_DG1_IDS(&dg1_info), {0, 0, 0} }; MODULE_DEVICE_TABLE(pci, pciidlist); @@ -1234,12 +1236,12 @@ static struct pci_driver i915_pci_driver = { .driver.pm = &i915_pm_ops, }; -int i915_register_pci_driver(void) +int i915_pci_register_driver(void) { return pci_register_driver(&i915_pci_driver); } -void i915_unregister_pci_driver(void) +void i915_pci_unregister_driver(void) { pci_unregister_driver(&i915_pci_driver); } diff --git a/drivers/gpu/drm/i915/i915_pci.h b/drivers/gpu/drm/i915/i915_pci.h index b386f319f52e..ee048c238174 100644 --- a/drivers/gpu/drm/i915/i915_pci.h +++ b/drivers/gpu/drm/i915/i915_pci.h @@ -1,8 +1,12 @@ +/* SPDX-License-Identifier: MIT */ /* - * SPDX-License-Identifier: MIT - * * Copyright © 2021 Intel Corporation */ -int i915_register_pci_driver(void); -void i915_unregister_pci_driver(void); +#ifndef __I915_PCI_H__ +#define __I915_PCI_H__ + +int i915_pci_register_driver(void); +void i915_pci_unregister_driver(void); + +#endif /* __I915_PCI_H__ */ diff --git a/drivers/gpu/drm/i915/i915_query.c b/drivers/gpu/drm/i915/i915_query.c index e49da36c62fb..51b368be0fc4 100644 --- a/drivers/gpu/drm/i915/i915_query.c +++ b/drivers/gpu/drm/i915/i915_query.c @@ -124,7 +124,9 @@ query_engine_info(struct drm_i915_private *i915, for_each_uabi_engine(engine, i915) { info.engine.engine_class = engine->uabi_class; info.engine.engine_instance = engine->uabi_instance; + info.flags = I915_ENGINE_INFO_HAS_LOGICAL_INSTANCE; info.capabilities = engine->uabi_capabilities; + info.logical_instance = ilog2(engine->logical_mask); if (copy_to_user(info_ptr, &info, sizeof(info))) return -EFAULT; @@ -432,9 +434,6 @@ static int query_memregion_info(struct drm_i915_private *i915, u32 total_length; int ret, id, i; - if (!IS_ENABLED(CONFIG_DRM_I915_UNSTABLE_FAKE_LMEM)) - return -ENODEV; - if (query_item->flags != 0) return -EINVAL; diff --git a/drivers/gpu/drm/i915/i915_reg.h b/drivers/gpu/drm/i915/i915_reg.h index 9023d4ecf3b3..da9055c3ebf0 100644 --- a/drivers/gpu/drm/i915/i915_reg.h +++ b/drivers/gpu/drm/i915/i915_reg.h @@ -1968,7 +1968,7 @@ static inline bool i915_mmio_reg_valid(i915_reg_t reg) _ICL_PORT_PCS_LN(ln) + 4 * (dw)) #define ICL_PORT_PCS_DW1_AUX(phy) _MMIO(_ICL_PORT_PCS_DW_AUX(1, phy)) #define ICL_PORT_PCS_DW1_GRP(phy) _MMIO(_ICL_PORT_PCS_DW_GRP(1, phy)) -#define ICL_PORT_PCS_DW1_LN0(phy) _MMIO(_ICL_PORT_PCS_DW_LN(1, 0, phy)) +#define ICL_PORT_PCS_DW1_LN(ln, phy) _MMIO(_ICL_PORT_PCS_DW_LN(1, ln, phy)) #define DCC_MODE_SELECT_MASK (0x3 << 20) #define DCC_MODE_SELECT_CONTINUOSLY (0x3 << 20) #define COMMON_KEEPER_EN (1 << 26) @@ -1989,7 +1989,7 @@ static inline bool i915_mmio_reg_valid(i915_reg_t reg) #define ICL_PORT_TX_DW2_AUX(phy) _MMIO(_ICL_PORT_TX_DW_AUX(2, phy)) #define ICL_PORT_TX_DW2_GRP(phy) _MMIO(_ICL_PORT_TX_DW_GRP(2, phy)) -#define ICL_PORT_TX_DW2_LN0(phy) _MMIO(_ICL_PORT_TX_DW_LN(2, 0, phy)) +#define ICL_PORT_TX_DW2_LN(ln, phy) _MMIO(_ICL_PORT_TX_DW_LN(2, ln, phy)) #define SWING_SEL_UPPER(x) (((x) >> 3) << 15) #define SWING_SEL_UPPER_MASK (1 << 15) #define SWING_SEL_LOWER(x) (((x) & 0x7) << 11) @@ -2001,7 +2001,6 @@ static inline bool i915_mmio_reg_valid(i915_reg_t reg) #define ICL_PORT_TX_DW4_AUX(phy) _MMIO(_ICL_PORT_TX_DW_AUX(4, phy)) #define ICL_PORT_TX_DW4_GRP(phy) _MMIO(_ICL_PORT_TX_DW_GRP(4, phy)) -#define ICL_PORT_TX_DW4_LN0(phy) _MMIO(_ICL_PORT_TX_DW_LN(4, 0, phy)) #define ICL_PORT_TX_DW4_LN(ln, phy) _MMIO(_ICL_PORT_TX_DW_LN(4, ln, phy)) #define LOADGEN_SELECT (1 << 31) #define POST_CURSOR_1(x) ((x) << 12) @@ -2013,7 +2012,7 @@ static inline bool i915_mmio_reg_valid(i915_reg_t reg) #define ICL_PORT_TX_DW5_AUX(phy) _MMIO(_ICL_PORT_TX_DW_AUX(5, phy)) #define ICL_PORT_TX_DW5_GRP(phy) _MMIO(_ICL_PORT_TX_DW_GRP(5, phy)) -#define ICL_PORT_TX_DW5_LN0(phy) _MMIO(_ICL_PORT_TX_DW_LN(5, 0, phy)) +#define ICL_PORT_TX_DW5_LN(ln, phy) _MMIO(_ICL_PORT_TX_DW_LN(5, ln, phy)) #define TX_TRAINING_EN (1 << 31) #define TAP2_DISABLE (1 << 30) #define TAP3_DISABLE (1 << 29) @@ -2024,14 +2023,13 @@ static inline bool i915_mmio_reg_valid(i915_reg_t reg) #define ICL_PORT_TX_DW7_AUX(phy) _MMIO(_ICL_PORT_TX_DW_AUX(7, phy)) #define ICL_PORT_TX_DW7_GRP(phy) _MMIO(_ICL_PORT_TX_DW_GRP(7, phy)) -#define ICL_PORT_TX_DW7_LN0(phy) _MMIO(_ICL_PORT_TX_DW_LN(7, 0, phy)) #define ICL_PORT_TX_DW7_LN(ln, phy) _MMIO(_ICL_PORT_TX_DW_LN(7, ln, phy)) #define N_SCALAR(x) ((x) << 24) #define N_SCALAR_MASK (0x7F << 24) #define ICL_PORT_TX_DW8_AUX(phy) _MMIO(_ICL_PORT_TX_DW_AUX(8, phy)) #define ICL_PORT_TX_DW8_GRP(phy) _MMIO(_ICL_PORT_TX_DW_GRP(8, phy)) -#define ICL_PORT_TX_DW8_LN0(phy) _MMIO(_ICL_PORT_TX_DW_LN(8, 0, phy)) +#define ICL_PORT_TX_DW8_LN(ln, phy) _MMIO(_ICL_PORT_TX_DW_LN(8, ln, phy)) #define ICL_PORT_TX_DW8_ODCC_CLK_SEL REG_BIT(31) #define ICL_PORT_TX_DW8_ODCC_CLK_DIV_SEL_MASK REG_GENMASK(30, 29) #define ICL_PORT_TX_DW8_ODCC_CLK_DIV_SEL_DIV2 REG_FIELD_PREP(ICL_PORT_TX_DW8_ODCC_CLK_DIV_SEL_MASK, 0x1) @@ -2237,10 +2235,14 @@ static inline bool i915_mmio_reg_valid(i915_reg_t reg) #define SNPS_PHY_MPLLB_DIV(phy) _MMIO_SNPS(phy, 0x168004) #define SNPS_PHY_MPLLB_FORCE_EN REG_BIT(31) +#define SNPS_PHY_MPLLB_DIV_CLK_EN REG_BIT(30) #define SNPS_PHY_MPLLB_DIV5_CLK_EN REG_BIT(29) #define SNPS_PHY_MPLLB_V2I REG_GENMASK(27, 26) #define SNPS_PHY_MPLLB_FREQ_VCO REG_GENMASK(25, 24) +#define SNPS_PHY_MPLLB_DIV_MULTIPLIER REG_GENMASK(23, 16) #define SNPS_PHY_MPLLB_PMIX_EN REG_BIT(10) +#define SNPS_PHY_MPLLB_DP2_MODE REG_BIT(9) +#define SNPS_PHY_MPLLB_WORD_DIV2_EN REG_BIT(8) #define SNPS_PHY_MPLLB_TX_CLK_DIV REG_GENMASK(7, 5) #define SNPS_PHY_MPLLB_FRACN1(phy) _MMIO_SNPS(phy, 0x168008) @@ -2551,6 +2553,32 @@ static inline bool i915_mmio_reg_valid(i915_reg_t reg) #define RING_HWS_PGA(base) _MMIO((base) + 0x80) #define RING_ID(base) _MMIO((base) + 0x8c) #define RING_HWS_PGA_GEN6(base) _MMIO((base) + 0x2080) + +#define RING_CMD_CCTL(base) _MMIO((base) + 0xc4) +/* + * CMD_CCTL read/write fields take a MOCS value and _not_ a table index. + * The lsb of each can be considered a separate enabling bit for encryption. + * 6:0 == default MOCS value for reads => 6:1 == table index for reads. + * 13:7 == default MOCS value for writes => 13:8 == table index for writes. + * 15:14 == Reserved => 31:30 are set to 0. + */ +#define CMD_CCTL_WRITE_OVERRIDE_MASK REG_GENMASK(13, 7) +#define CMD_CCTL_READ_OVERRIDE_MASK REG_GENMASK(6, 0) +#define CMD_CCTL_MOCS_MASK (CMD_CCTL_WRITE_OVERRIDE_MASK | \ + CMD_CCTL_READ_OVERRIDE_MASK) +#define CMD_CCTL_MOCS_OVERRIDE(write, read) \ + (REG_FIELD_PREP(CMD_CCTL_WRITE_OVERRIDE_MASK, (write) << 1) | \ + REG_FIELD_PREP(CMD_CCTL_READ_OVERRIDE_MASK, (read) << 1)) + +#define BLIT_CCTL(base) _MMIO((base) + 0x204) +#define BLIT_CCTL_DST_MOCS_MASK REG_GENMASK(14, 8) +#define BLIT_CCTL_SRC_MOCS_MASK REG_GENMASK(6, 0) +#define BLIT_CCTL_MASK (BLIT_CCTL_DST_MOCS_MASK | \ + BLIT_CCTL_SRC_MOCS_MASK) +#define BLIT_CCTL_MOCS(dst, src) \ + (REG_FIELD_PREP(BLIT_CCTL_DST_MOCS_MASK, (dst) << 1) | \ + REG_FIELD_PREP(BLIT_CCTL_SRC_MOCS_MASK, (src) << 1)) + #define RING_RESET_CTL(base) _MMIO((base) + 0xd0) #define RESET_CTL_CAT_ERROR REG_BIT(2) #define RESET_CTL_READY_TO_RESET REG_BIT(1) @@ -2686,6 +2714,7 @@ static inline bool i915_mmio_reg_valid(i915_reg_t reg) #define GEN12_SC_INSTDONE_EXTRA2 _MMIO(0x7108) #define GEN7_SAMPLER_INSTDONE _MMIO(0xe160) #define GEN7_ROW_INSTDONE _MMIO(0xe164) +#define XEHPG_INSTDONE_GEOM_SVG _MMIO(0x666c) #define MCFG_MCR_SELECTOR _MMIO(0xfd0) #define SF_MCR_SELECTOR _MMIO(0xfd8) #define GEN8_MCR_SELECTOR _MMIO(0xfdc) @@ -2820,6 +2849,7 @@ static inline bool i915_mmio_reg_valid(i915_reg_t reg) #define MI_MODE _MMIO(0x209c) # define VS_TIMER_DISPATCH (1 << 6) # define MI_FLUSH_ENABLE (1 << 12) +# define TGL_NESTED_BB_EN (1 << 12) # define ASYNC_FLIP_PERF_DISABLE (1 << 14) # define MODE_IDLE (1 << 9) # define STOP_RING (1 << 8) @@ -3080,8 +3110,8 @@ static inline bool i915_mmio_reg_valid(i915_reg_t reg) /* Fuse readout registers for GT */ #define HSW_PAVP_FUSE1 _MMIO(0x911C) -#define HSW_F1_EU_DIS_SHIFT 16 -#define HSW_F1_EU_DIS_MASK (0x3 << HSW_F1_EU_DIS_SHIFT) +#define XEHP_SFC_ENABLE_MASK REG_GENMASK(27, 24) +#define HSW_F1_EU_DIS_MASK REG_GENMASK(17, 16) #define HSW_F1_EU_DIS_10EUS 0 #define HSW_F1_EU_DIS_8EUS 1 #define HSW_F1_EU_DIS_6EUS 2 @@ -3150,7 +3180,8 @@ static inline bool i915_mmio_reg_valid(i915_reg_t reg) #define GEN11_GT_SUBSLICE_DISABLE _MMIO(0x913C) -#define GEN12_GT_DSS_ENABLE _MMIO(0x913C) +#define GEN12_GT_GEOMETRY_DSS_ENABLE _MMIO(0x913C) +#define GEN12_GT_COMPUTE_DSS_ENABLE _MMIO(0x9144) #define XEHP_EU_ENABLE _MMIO(0x9134) #define XEHP_EU_ENA_MASK 0xFF @@ -3356,6 +3387,10 @@ static inline bool i915_mmio_reg_valid(i915_reg_t reg) #define ILK_DPFC_DISABLE_DUMMY0 (1 << 8) #define ILK_DPFC_CHICKEN_COMP_DUMMY_PIXEL (1 << 14) #define ILK_DPFC_NUKE_ON_ANY_MODIFICATION (1 << 23) +#define GLK_FBC_STRIDE _MMIO(0x43228) +#define FBC_STRIDE_OVERRIDE REG_BIT(15) +#define FBC_STRIDE_MASK REG_GENMASK(14, 0) +#define FBC_STRIDE(x) REG_FIELD_PREP(FBC_STRIDE_MASK, (x)) #define ILK_FBC_RT_BASE _MMIO(0x2128) #define ILK_FBC_RT_VALID (1 << 0) #define SNB_FBC_FRONT_BUFFER (1 << 1) @@ -4113,6 +4148,7 @@ static inline bool i915_mmio_reg_valid(i915_reg_t reg) #define RPN_CAP_MASK REG_GENMASK(23, 16) #define BXT_RP_STATE_CAP _MMIO(0x138170) #define GEN9_RP_STATE_LIMITS _MMIO(0x138148) +#define XEHPSDV_RP_STATE_CAP _MMIO(0x250014) /* * Logical Context regs @@ -4231,6 +4267,7 @@ enum { #define DUPS1_GATING_DIS (1 << 15) #define DUPS2_GATING_DIS (1 << 19) #define DUPS3_GATING_DIS (1 << 23) +#define CURSOR_GATING_DIS REG_BIT(28) #define DPF_GATING_DIS (1 << 10) #define DPF_RAM_GATING_DIS (1 << 9) #define DPFR_GATING_DIS (1 << 8) @@ -4509,11 +4546,9 @@ enum { * HSW PSR registers are relative to DDIA(_DDI_BUF_CTL_A + 0x800) with just one * instance of it */ -#define _HSW_EDP_PSR_BASE 0x64800 #define _SRD_CTL_A 0x60800 #define _SRD_CTL_EDP 0x6f800 -#define _PSR_ADJ(tran, reg) (_TRANS2(tran, reg) - dev_priv->hsw_psr_mmio_adjust) -#define EDP_PSR_CTL(tran) _MMIO(_PSR_ADJ(tran, _SRD_CTL_A)) +#define EDP_PSR_CTL(tran) _MMIO(_TRANS2(tran, _SRD_CTL_A)) #define EDP_PSR_ENABLE (1 << 31) #define BDW_PSR_SINGLE_FRAME (1 << 30) #define EDP_PSR_RESTORE_PSR_ACTIVE_CTX_MASK (1 << 29) /* SW can't modify */ @@ -4557,22 +4592,13 @@ enum { #define EDP_PSR_POST_EXIT(trans) (0x2 << _EDP_PSR_TRANS_SHIFT(trans)) #define EDP_PSR_PRE_ENTRY(trans) (0x1 << _EDP_PSR_TRANS_SHIFT(trans)) -#define _SRD_AUX_CTL_A 0x60810 -#define _SRD_AUX_CTL_EDP 0x6f810 -#define EDP_PSR_AUX_CTL(tran) _MMIO(_PSR_ADJ(tran, _SRD_AUX_CTL_A)) -#define EDP_PSR_AUX_CTL_TIME_OUT_MASK (3 << 26) -#define EDP_PSR_AUX_CTL_MESSAGE_SIZE_MASK (0x1f << 20) -#define EDP_PSR_AUX_CTL_PRECHARGE_2US_MASK (0xf << 16) -#define EDP_PSR_AUX_CTL_ERROR_INTERRUPT (1 << 11) -#define EDP_PSR_AUX_CTL_BIT_CLOCK_2X_MASK (0x7ff) - #define _SRD_AUX_DATA_A 0x60814 #define _SRD_AUX_DATA_EDP 0x6f814 -#define EDP_PSR_AUX_DATA(tran, i) _MMIO(_PSR_ADJ(tran, _SRD_AUX_DATA_A) + (i) + 4) /* 5 registers */ +#define EDP_PSR_AUX_DATA(tran, i) _MMIO(_TRANS2(tran, _SRD_AUX_DATA_A) + (i) + 4) /* 5 registers */ #define _SRD_STATUS_A 0x60840 #define _SRD_STATUS_EDP 0x6f840 -#define EDP_PSR_STATUS(tran) _MMIO(_PSR_ADJ(tran, _SRD_STATUS_A)) +#define EDP_PSR_STATUS(tran) _MMIO(_TRANS2(tran, _SRD_STATUS_A)) #define EDP_PSR_STATUS_STATE_MASK (7 << 29) #define EDP_PSR_STATUS_STATE_SHIFT 29 #define EDP_PSR_STATUS_STATE_IDLE (0 << 29) @@ -4599,13 +4625,13 @@ enum { #define _SRD_PERF_CNT_A 0x60844 #define _SRD_PERF_CNT_EDP 0x6f844 -#define EDP_PSR_PERF_CNT(tran) _MMIO(_PSR_ADJ(tran, _SRD_PERF_CNT_A)) +#define EDP_PSR_PERF_CNT(tran) _MMIO(_TRANS2(tran, _SRD_PERF_CNT_A)) #define EDP_PSR_PERF_CNT_MASK 0xffffff /* PSR_MASK on SKL+ */ #define _SRD_DEBUG_A 0x60860 #define _SRD_DEBUG_EDP 0x6f860 -#define EDP_PSR_DEBUG(tran) _MMIO(_PSR_ADJ(tran, _SRD_DEBUG_A)) +#define EDP_PSR_DEBUG(tran) _MMIO(_TRANS2(tran, _SRD_DEBUG_A)) #define EDP_PSR_DEBUG_MASK_MAX_SLEEP (1 << 28) #define EDP_PSR_DEBUG_MASK_LPSP (1 << 27) #define EDP_PSR_DEBUG_MASK_MEMUP (1 << 26) @@ -7230,6 +7256,7 @@ enum { #define _PLANE_COLOR_CTL_3_A 0x703CC /* GLK+ */ #define PLANE_COLOR_PIPE_GAMMA_ENABLE (1 << 30) /* Pre-ICL */ #define PLANE_COLOR_YUV_RANGE_CORRECTION_DISABLE (1 << 28) +#define PLANE_COLOR_PLANE_CSC_ENABLE REG_BIT(21) /* ICL+ */ #define PLANE_COLOR_INPUT_CSC_ENABLE (1 << 20) /* ICL+ */ #define PLANE_COLOR_PIPE_CSC_ENABLE (1 << 23) /* Pre-ICL */ #define PLANE_COLOR_CSC_MODE_BYPASS (0 << 17) @@ -7353,6 +7380,7 @@ enum { #define _PLANE_SURF_3(pipe) _PIPE(pipe, _PLANE_SURF_3_A, _PLANE_SURF_3_B) #define PLANE_SURF(pipe, plane) \ _MMIO_PLANE(plane, _PLANE_SURF_1(pipe), _PLANE_SURF_2(pipe)) +#define PLANE_SURF_DECRYPT REG_BIT(2) #define _PLANE_OFFSET_1_B 0x711a4 #define _PLANE_OFFSET_2_B 0x712a4 @@ -8094,6 +8122,7 @@ enum { /* irq instances for OTHER_CLASS */ #define OTHER_GUC_INSTANCE 0 #define OTHER_GTPM_INSTANCE 1 +#define OTHER_KCR_INSTANCE 4 #define GEN11_INTR_IDENTITY_REG(x) _MMIO(0x190060 + ((x) * 4)) @@ -8176,8 +8205,9 @@ enum { #define GLK_CL0_PWR_DOWN (1 << 10) #define CHICKEN_MISC_4 _MMIO(0x4208c) -#define FBC_STRIDE_OVERRIDE (1 << 13) -#define FBC_STRIDE_MASK 0x1FFF +#define CHICKEN_FBC_STRIDE_OVERRIDE REG_BIT(13) +#define CHICKEN_FBC_STRIDE_MASK REG_GENMASK(12, 0) +#define CHICKEN_FBC_STRIDE(x) REG_FIELD_PREP(CHICKEN_FBC_STRIDE_MASK, (x)) #define _CHICKEN_PIPESL_1_A 0x420b0 #define _CHICKEN_PIPESL_1_B 0x420b4 @@ -8216,6 +8246,7 @@ enum { #define VSC_DATA_SEL_SOFTWARE_CONTROL REG_BIT(25) /* GLK */ #define FECSTALL_DIS_DPTSTREAM_DPTTG REG_BIT(23) #define DDI_TRAINING_OVERRIDE_ENABLE REG_BIT(19) +#define ADLP_1_BASED_X_GRANULARITY REG_BIT(18) #define DDI_TRAINING_OVERRIDE_VALUE REG_BIT(18) #define DDIE_TRAINING_OVERRIDE_ENABLE REG_BIT(17) /* CHICKEN_TRANS_A only */ #define DDIE_TRAINING_OVERRIDE_VALUE REG_BIT(16) /* CHICKEN_TRANS_A only */ @@ -9101,6 +9132,29 @@ enum { #define TRANS_DP_HSYNC_ACTIVE_LOW 0 #define TRANS_DP_SYNC_MASK (3 << 3) +#define _TRANS_DP2_CTL_A 0x600a0 +#define _TRANS_DP2_CTL_B 0x610a0 +#define _TRANS_DP2_CTL_C 0x620a0 +#define _TRANS_DP2_CTL_D 0x630a0 +#define TRANS_DP2_CTL(trans) _MMIO_TRANS(trans, _TRANS_DP2_CTL_A, _TRANS_DP2_CTL_B) +#define TRANS_DP2_128B132B_CHANNEL_CODING REG_BIT(31) +#define TRANS_DP2_PANEL_REPLAY_ENABLE REG_BIT(30) +#define TRANS_DP2_DEBUG_ENABLE REG_BIT(23) + +#define _TRANS_DP2_VFREQHIGH_A 0x600a4 +#define _TRANS_DP2_VFREQHIGH_B 0x610a4 +#define _TRANS_DP2_VFREQHIGH_C 0x620a4 +#define _TRANS_DP2_VFREQHIGH_D 0x630a4 +#define TRANS_DP2_VFREQHIGH(trans) _MMIO_TRANS(trans, _TRANS_DP2_VFREQHIGH_A, _TRANS_DP2_VFREQHIGH_B) +#define TRANS_DP2_VFREQ_PIXEL_CLOCK_MASK REG_GENMASK(31, 8) +#define TRANS_DP2_VFREQ_PIXEL_CLOCK(clk_hz) REG_FIELD_PREP(TRANS_DP2_VFREQ_PIXEL_CLOCK_MASK, (clk_hz)) + +#define _TRANS_DP2_VFREQLOW_A 0x600a8 +#define _TRANS_DP2_VFREQLOW_B 0x610a8 +#define _TRANS_DP2_VFREQLOW_C 0x620a8 +#define _TRANS_DP2_VFREQLOW_D 0x630a8 +#define TRANS_DP2_VFREQLOW(trans) _MMIO_TRANS(trans, _TRANS_DP2_VFREQLOW_A, _TRANS_DP2_VFREQLOW_B) + /* SNB eDP training params */ /* SNB A-stepping */ #define EDP_LINK_TRAIN_400MV_0DB_SNB_A (0x38 << 22) @@ -9715,6 +9769,11 @@ enum { #define AUDIO_CP_READY(trans) ((1 << 1) << ((trans) * 4)) #define AUDIO_ELD_VALID(trans) ((1 << 0) << ((trans) * 4)) +#define _AUD_TCA_DP_2DOT0_CTRL 0x650bc +#define _AUD_TCB_DP_2DOT0_CTRL 0x651bc +#define AUD_DP_2DOT0_CTRL(trans) _MMIO_TRANS(trans, _AUD_TCA_DP_2DOT0_CTRL, _AUD_TCB_DP_2DOT0_CTRL) +#define AUD_ENABLE_SDP_SPLIT REG_BIT(31) + #define HSW_AUD_CHICKENBIT _MMIO(0x65f10) #define SKL_AUD_CODEC_WAKE_SIGNAL (1 << 15) @@ -10160,7 +10219,7 @@ enum skl_power_gate { #define TRANS_DDI_MODE_SELECT_DVI (1 << 24) #define TRANS_DDI_MODE_SELECT_DP_SST (2 << 24) #define TRANS_DDI_MODE_SELECT_DP_MST (3 << 24) -#define TRANS_DDI_MODE_SELECT_FDI (4 << 24) +#define TRANS_DDI_MODE_SELECT_FDI_OR_128B132B (4 << 24) #define TRANS_DDI_BPC_MASK (7 << 20) #define TRANS_DDI_BPC_8 (0 << 20) #define TRANS_DDI_BPC_10 (1 << 20) @@ -10963,7 +11022,6 @@ enum skl_power_gate { _DKL_TX_DPCNTL1) #define _DKL_TX_DPCNTL2 0x2C8 -#define DKL_TX_LOADGEN_SHARING_PMD_DISABLE REG_BIT(12) #define DKL_TX_DP20BITMODE (1 << 2) #define DKL_TX_DPCNTL2(tc_port) _MMIO(_PORT(tc_port, \ _DKL_PHY1_BASE, \ @@ -11342,6 +11400,51 @@ enum skl_power_gate { _PAL_PREC_MULTI_SEG_DATA_A, \ _PAL_PREC_MULTI_SEG_DATA_B) +#define _MMIO_PLANE_GAMC(plane, i, a, b) _MMIO(_PIPE(plane, a, b) + (i) * 4) + +/* Plane CSC Registers */ +#define _PLANE_CSC_RY_GY_1_A 0x70210 +#define _PLANE_CSC_RY_GY_2_A 0x70310 + +#define _PLANE_CSC_RY_GY_1_B 0x71210 +#define _PLANE_CSC_RY_GY_2_B 0x71310 + +#define _PLANE_CSC_RY_GY_1(pipe) _PIPE(pipe, _PLANE_CSC_RY_GY_1_A, \ + _PLANE_CSC_RY_GY_1_B) +#define _PLANE_CSC_RY_GY_2(pipe) _PIPE(pipe, _PLANE_INPUT_CSC_RY_GY_2_A, \ + _PLANE_INPUT_CSC_RY_GY_2_B) +#define PLANE_CSC_COEFF(pipe, plane, index) _MMIO_PLANE(plane, \ + _PLANE_CSC_RY_GY_1(pipe) + (index) * 4, \ + _PLANE_CSC_RY_GY_2(pipe) + (index) * 4) + +#define _PLANE_CSC_PREOFF_HI_1_A 0x70228 +#define _PLANE_CSC_PREOFF_HI_2_A 0x70328 + +#define _PLANE_CSC_PREOFF_HI_1_B 0x71228 +#define _PLANE_CSC_PREOFF_HI_2_B 0x71328 + +#define _PLANE_CSC_PREOFF_HI_1(pipe) _PIPE(pipe, _PLANE_CSC_PREOFF_HI_1_A, \ + _PLANE_CSC_PREOFF_HI_1_B) +#define _PLANE_CSC_PREOFF_HI_2(pipe) _PIPE(pipe, _PLANE_CSC_PREOFF_HI_2_A, \ + _PLANE_CSC_PREOFF_HI_2_B) +#define PLANE_CSC_PREOFF(pipe, plane, index) _MMIO_PLANE(plane, _PLANE_CSC_PREOFF_HI_1(pipe) + \ + (index) * 4, _PLANE_CSC_PREOFF_HI_2(pipe) + \ + (index) * 4) + +#define _PLANE_CSC_POSTOFF_HI_1_A 0x70234 +#define _PLANE_CSC_POSTOFF_HI_2_A 0x70334 + +#define _PLANE_CSC_POSTOFF_HI_1_B 0x71234 +#define _PLANE_CSC_POSTOFF_HI_2_B 0x71334 + +#define _PLANE_CSC_POSTOFF_HI_1(pipe) _PIPE(pipe, _PLANE_CSC_POSTOFF_HI_1_A, \ + _PLANE_CSC_POSTOFF_HI_1_B) +#define _PLANE_CSC_POSTOFF_HI_2(pipe) _PIPE(pipe, _PLANE_CSC_POSTOFF_HI_2_A, \ + _PLANE_CSC_POSTOFF_HI_2_B) +#define PLANE_CSC_POSTOFF(pipe, plane, index) _MMIO_PLANE(plane, _PLANE_CSC_POSTOFF_HI_1(pipe) + \ + (index) * 4, _PLANE_CSC_POSTOFF_HI_2(pipe) + \ + (index) * 4) + /* pipe CSC & degamma/gamma LUTs on CHV */ #define _CGM_PIPE_A_CSC_COEFF01 (VLV_DISPLAY_BASE + 0x67900) #define _CGM_PIPE_A_CSC_COEFF23 (VLV_DISPLAY_BASE + 0x67904) @@ -11608,6 +11711,14 @@ enum skl_power_gate { _ICL_DSI_IO_MODECTL_1) #define COMBO_PHY_MODE_DSI (1 << 0) +/* TGL DSI Chicken register */ +#define _TGL_DSI_CHKN_REG_0 0x6B0C0 +#define _TGL_DSI_CHKN_REG_1 0x6B8C0 +#define TGL_DSI_CHKN_REG(port) _MMIO_PORT(port, \ + _TGL_DSI_CHKN_REG_0, \ + _TGL_DSI_CHKN_REG_1) +#define TGL_DSI_CHKN_LSHS_GB REG_GENMASK(15, 12) + /* Display Stream Splitter Control */ #define DSS_CTL1 _MMIO(0x67400) #define SPLITTER_ENABLE (1 << 31) @@ -12731,4 +12842,7 @@ enum skl_power_gate { #define CLKREQ_POLICY _MMIO(0x101038) #define CLKREQ_POLICY_MEM_UP_OVRD REG_BIT(1) +#define CLKGATE_DIS_MISC _MMIO(0x46534) +#define CLKGATE_DIS_MISC_DMASC_GATING_DIS REG_BIT(21) + #endif /* _I915_REG_H_ */ diff --git a/drivers/gpu/drm/i915/i915_request.c b/drivers/gpu/drm/i915/i915_request.c index 79da5eca60af..2c3cd6e635b5 100644 --- a/drivers/gpu/drm/i915/i915_request.c +++ b/drivers/gpu/drm/i915/i915_request.c @@ -1145,6 +1145,12 @@ __emit_semaphore_wait(struct i915_request *to, return 0; } +static bool +can_use_semaphore_wait(struct i915_request *to, struct i915_request *from) +{ + return to->engine->gt->ggtt == from->engine->gt->ggtt; +} + static int emit_semaphore_wait(struct i915_request *to, struct i915_request *from, @@ -1153,6 +1159,9 @@ emit_semaphore_wait(struct i915_request *to, const intel_engine_mask_t mask = READ_ONCE(from->engine)->mask; struct i915_sw_fence *wait = &to->submit; + if (!can_use_semaphore_wait(to, from)) + goto await_fence; + if (!intel_context_use_semaphores(to->context)) goto await_fence; @@ -1256,7 +1265,8 @@ __i915_request_await_execution(struct i915_request *to, * immediate execution, and so we must wait until it reaches the * active slot. */ - if (intel_engine_has_semaphores(to->engine) && + if (can_use_semaphore_wait(to, from) && + intel_engine_has_semaphores(to->engine) && !i915_request_has_initial_breadcrumb(to)) { err = __emit_semaphore_wait(to, from, from->fence.seqno - 1); if (err < 0) @@ -1325,6 +1335,25 @@ i915_request_await_external(struct i915_request *rq, struct dma_fence *fence) return err; } +static inline bool is_parallel_rq(struct i915_request *rq) +{ + return intel_context_is_parallel(rq->context); +} + +static inline struct intel_context *request_to_parent(struct i915_request *rq) +{ + return intel_context_to_parent(rq->context); +} + +static bool is_same_parallel_context(struct i915_request *to, + struct i915_request *from) +{ + if (is_parallel_rq(to)) + return request_to_parent(to) == request_to_parent(from); + + return false; +} + int i915_request_await_execution(struct i915_request *rq, struct dma_fence *fence) @@ -1356,11 +1385,14 @@ i915_request_await_execution(struct i915_request *rq, * want to run our callback in all cases. */ - if (dma_fence_is_i915(fence)) + if (dma_fence_is_i915(fence)) { + if (is_same_parallel_context(rq, to_request(fence))) + continue; ret = __i915_request_await_execution(rq, to_request(fence)); - else + } else { ret = i915_request_await_external(rq, fence); + } if (ret < 0) return ret; } while (--nchild); @@ -1461,10 +1493,13 @@ i915_request_await_dma_fence(struct i915_request *rq, struct dma_fence *fence) fence)) continue; - if (dma_fence_is_i915(fence)) + if (dma_fence_is_i915(fence)) { + if (is_same_parallel_context(rq, to_request(fence))) + continue; ret = i915_request_await_request(rq, to_request(fence)); - else + } else { ret = i915_request_await_external(rq, fence); + } if (ret < 0) return ret; @@ -1540,35 +1575,51 @@ i915_request_await_object(struct i915_request *to, } static struct i915_request * -__i915_request_add_to_timeline(struct i915_request *rq) +__i915_request_ensure_parallel_ordering(struct i915_request *rq, + struct intel_timeline *timeline) { - struct intel_timeline *timeline = i915_request_timeline(rq); struct i915_request *prev; - /* - * Dependency tracking and request ordering along the timeline - * is special cased so that we can eliminate redundant ordering - * operations while building the request (we know that the timeline - * itself is ordered, and here we guarantee it). - * - * As we know we will need to emit tracking along the timeline, - * we embed the hooks into our request struct -- at the cost of - * having to have specialised no-allocation interfaces (which will - * be beneficial elsewhere). - * - * A second benefit to open-coding i915_request_await_request is - * that we can apply a slight variant of the rules specialised - * for timelines that jump between engines (such as virtual engines). - * If we consider the case of virtual engine, we must emit a dma-fence - * to prevent scheduling of the second request until the first is - * complete (to maximise our greedy late load balancing) and this - * precludes optimising to use semaphores serialisation of a single - * timeline across engines. - */ + GEM_BUG_ON(!is_parallel_rq(rq)); + + prev = request_to_parent(rq)->parallel.last_rq; + if (prev) { + if (!__i915_request_is_complete(prev)) { + i915_sw_fence_await_sw_fence(&rq->submit, + &prev->submit, + &rq->submitq); + + if (rq->engine->sched_engine->schedule) + __i915_sched_node_add_dependency(&rq->sched, + &prev->sched, + &rq->dep, + 0); + } + i915_request_put(prev); + } + + request_to_parent(rq)->parallel.last_rq = i915_request_get(rq); + + return to_request(__i915_active_fence_set(&timeline->last_request, + &rq->fence)); +} + +static struct i915_request * +__i915_request_ensure_ordering(struct i915_request *rq, + struct intel_timeline *timeline) +{ + struct i915_request *prev; + + GEM_BUG_ON(is_parallel_rq(rq)); + prev = to_request(__i915_active_fence_set(&timeline->last_request, &rq->fence)); + if (prev && !__i915_request_is_complete(prev)) { bool uses_guc = intel_engine_uses_guc(rq->engine); + bool pow2 = is_power_of_2(READ_ONCE(prev->engine)->mask | + rq->engine->mask); + bool same_context = prev->context == rq->context; /* * The requests are supposed to be kept in order. However, @@ -1576,13 +1627,11 @@ __i915_request_add_to_timeline(struct i915_request *rq) * is used as a barrier for external modification to this * context. */ - GEM_BUG_ON(prev->context == rq->context && + GEM_BUG_ON(same_context && i915_seqno_passed(prev->fence.seqno, rq->fence.seqno)); - if ((!uses_guc && - is_power_of_2(READ_ONCE(prev->engine)->mask | rq->engine->mask)) || - (uses_guc && prev->context == rq->context)) + if ((same_context && uses_guc) || (!uses_guc && pow2)) i915_sw_fence_await_sw_fence(&rq->submit, &prev->submit, &rq->submitq); @@ -1597,6 +1646,50 @@ __i915_request_add_to_timeline(struct i915_request *rq) 0); } + return prev; +} + +static struct i915_request * +__i915_request_add_to_timeline(struct i915_request *rq) +{ + struct intel_timeline *timeline = i915_request_timeline(rq); + struct i915_request *prev; + + /* + * Dependency tracking and request ordering along the timeline + * is special cased so that we can eliminate redundant ordering + * operations while building the request (we know that the timeline + * itself is ordered, and here we guarantee it). + * + * As we know we will need to emit tracking along the timeline, + * we embed the hooks into our request struct -- at the cost of + * having to have specialised no-allocation interfaces (which will + * be beneficial elsewhere). + * + * A second benefit to open-coding i915_request_await_request is + * that we can apply a slight variant of the rules specialised + * for timelines that jump between engines (such as virtual engines). + * If we consider the case of virtual engine, we must emit a dma-fence + * to prevent scheduling of the second request until the first is + * complete (to maximise our greedy late load balancing) and this + * precludes optimising to use semaphores serialisation of a single + * timeline across engines. + * + * We do not order parallel submission requests on the timeline as each + * parallel submission context has its own timeline and the ordering + * rules for parallel requests are that they must be submitted in the + * order received from the execbuf IOCTL. So rather than using the + * timeline we store a pointer to last request submitted in the + * relationship in the gem context and insert a submission fence + * between that request and request passed into this function or + * alternatively we use completion fence if gem context has a single + * timeline and this is the first submission of an execbuf IOCTL. + */ + if (likely(!is_parallel_rq(rq))) + prev = __i915_request_ensure_ordering(rq, timeline); + else + prev = __i915_request_ensure_parallel_ordering(rq, timeline); + /* * Make sure that no request gazumped us - if it was allocated after * our i915_request_alloc() and called __i915_request_add() before @@ -1852,7 +1945,7 @@ long i915_request_wait(struct i915_request *rq, * completion. That requires having a good predictor for the request * duration, which we currently lack. */ - if (IS_ACTIVE(CONFIG_DRM_I915_MAX_REQUEST_BUSYWAIT) && + if (CONFIG_DRM_I915_MAX_REQUEST_BUSYWAIT && __i915_spin_request(rq, state)) goto out; diff --git a/drivers/gpu/drm/i915/i915_request.h b/drivers/gpu/drm/i915/i915_request.h index 1bc1349ba3c2..dc359242d1ae 100644 --- a/drivers/gpu/drm/i915/i915_request.h +++ b/drivers/gpu/drm/i915/i915_request.h @@ -139,6 +139,29 @@ enum { * the GPU. Here we track such boost requests on a per-request basis. */ I915_FENCE_FLAG_BOOST, + + /* + * I915_FENCE_FLAG_SUBMIT_PARALLEL - request with a context in a + * parent-child relationship (parallel submission, multi-lrc) should + * trigger a submission to the GuC rather than just moving the context + * tail. + */ + I915_FENCE_FLAG_SUBMIT_PARALLEL, + + /* + * I915_FENCE_FLAG_SKIP_PARALLEL - request with a context in a + * parent-child relationship (parallel submission, multi-lrc) that + * hit an error while generating requests in the execbuf IOCTL. + * Indicates this request should be skipped as another request in + * submission / relationship encoutered an error. + */ + I915_FENCE_FLAG_SKIP_PARALLEL, + + /* + * I915_FENCE_FLAG_COMPOSITE - Indicates fence is part of a composite + * fence (dma_fence_array) and i915 generated for parallel submission. + */ + I915_FENCE_FLAG_COMPOSITE, }; /** @@ -218,6 +241,11 @@ struct i915_request { }; struct llist_head execute_cb; struct i915_sw_fence semaphore; + /** + * @submit_work: complete submit fence from an IRQ if needed for + * locking hierarchy reasons. + */ + struct irq_work submit_work; /* * A list of everyone we wait upon, and everyone who waits upon us. @@ -285,18 +313,23 @@ struct i915_request { struct hrtimer timer; } watchdog; - /* - * Requests may need to be stalled when using GuC submission waiting for - * certain GuC operations to complete. If that is the case, stalled - * requests are added to a per context list of stalled requests. The - * below list_head is the link in that list. + /** + * @guc_fence_link: Requests may need to be stalled when using GuC + * submission waiting for certain GuC operations to complete. If that is + * the case, stalled requests are added to a per context list of stalled + * requests. The below list_head is the link in that list. Protected by + * ce->guc_state.lock. */ struct list_head guc_fence_link; /** - * Priority level while the request is inflight. Differs from i915 - * scheduler priority. See comment above - * I915_SCHEDULER_CAP_STATIC_PRIORITY_MAP for details. + * @guc_prio: Priority level while the request is in flight. Differs + * from i915 scheduler priority. See comment above + * I915_SCHEDULER_CAP_STATIC_PRIORITY_MAP for details. Protected by + * ce->guc_active.lock. Two special values (GUC_PRIO_INIT and + * GUC_PRIO_FINI) outside the GuC priority range are used to indicate + * if the priority has not been initialized yet or if no more updates + * are possible because the request has completed. */ #define GUC_PRIO_INIT 0xff #define GUC_PRIO_FINI 0xfe diff --git a/drivers/gpu/drm/i915/i915_sysfs.c b/drivers/gpu/drm/i915/i915_sysfs.c index cdf0e9c6fd73..1804f4142740 100644 --- a/drivers/gpu/drm/i915/i915_sysfs.c +++ b/drivers/gpu/drm/i915/i915_sysfs.c @@ -37,7 +37,6 @@ #include "i915_drv.h" #include "i915_sysfs.h" #include "intel_pm.h" -#include "intel_sideband.h" static inline struct drm_i915_private *kdev_minor_to_i915(struct device *kdev) { diff --git a/drivers/gpu/drm/i915/i915_trace.h b/drivers/gpu/drm/i915/i915_trace.h index 63fec1c3c132..8104981a6604 100644 --- a/drivers/gpu/drm/i915/i915_trace.h +++ b/drivers/gpu/drm/i915/i915_trace.h @@ -900,23 +900,19 @@ DECLARE_EVENT_CLASS(intel_context, __field(u32, guc_id) __field(int, pin_count) __field(u32, sched_state) - __field(u32, guc_sched_state_no_lock) __field(u8, guc_prio) ), TP_fast_assign( - __entry->guc_id = ce->guc_id; + __entry->guc_id = ce->guc_id.id; __entry->pin_count = atomic_read(&ce->pin_count); __entry->sched_state = ce->guc_state.sched_state; - __entry->guc_sched_state_no_lock = - atomic_read(&ce->guc_sched_state_no_lock); - __entry->guc_prio = ce->guc_prio; + __entry->guc_prio = ce->guc_state.prio; ), - TP_printk("guc_id=%d, pin_count=%d sched_state=0x%x,0x%x, guc_prio=%u", + TP_printk("guc_id=%d, pin_count=%d sched_state=0x%x, guc_prio=%u", __entry->guc_id, __entry->pin_count, __entry->sched_state, - __entry->guc_sched_state_no_lock, __entry->guc_prio) ); @@ -1243,7 +1239,7 @@ DECLARE_EVENT_CLASS(i915_context, TP_fast_assign( __entry->dev = ctx->i915->drm.primary->index; __entry->ctx = ctx; - __entry->vm = rcu_access_pointer(ctx->vm); + __entry->vm = ctx->vm; ), TP_printk("dev=%u, ctx=%p, ctx_vm=%p", diff --git a/drivers/gpu/drm/i915/i915_ttm_buddy_manager.c b/drivers/gpu/drm/i915/i915_ttm_buddy_manager.c index 6877362f6b85..d59fbb019032 100644 --- a/drivers/gpu/drm/i915/i915_ttm_buddy_manager.c +++ b/drivers/gpu/drm/i915/i915_ttm_buddy_manager.c @@ -126,12 +126,30 @@ static void i915_ttm_buddy_man_free(struct ttm_resource_manager *man, kfree(bman_res); } +static void i915_ttm_buddy_man_debug(struct ttm_resource_manager *man, + struct drm_printer *printer) +{ + struct i915_ttm_buddy_manager *bman = to_buddy_manager(man); + struct i915_buddy_block *block; + + mutex_lock(&bman->lock); + drm_printf(printer, "default_page_size: %lluKiB\n", + bman->default_page_size >> 10); + + i915_buddy_print(&bman->mm, printer); + + drm_printf(printer, "reserved:\n"); + list_for_each_entry(block, &bman->reserved, link) + i915_buddy_block_print(&bman->mm, block, printer); + mutex_unlock(&bman->lock); +} + static const struct ttm_resource_manager_func i915_ttm_buddy_manager_func = { .alloc = i915_ttm_buddy_man_alloc, .free = i915_ttm_buddy_man_free, + .debug = i915_ttm_buddy_man_debug, }; - /** * i915_ttm_buddy_man_init - Setup buddy allocator based ttm manager * @bdev: The ttm device diff --git a/drivers/gpu/drm/i915/i915_utils.h b/drivers/gpu/drm/i915/i915_utils.h index 066a9118c374..7a5925072466 100644 --- a/drivers/gpu/drm/i915/i915_utils.h +++ b/drivers/gpu/drm/i915/i915_utils.h @@ -459,17 +459,4 @@ static inline bool timer_expired(const struct timer_list *t) return timer_active(t) && !timer_pending(t); } -/* - * This is a lookalike for IS_ENABLED() that takes a kconfig value, - * e.g. CONFIG_DRM_I915_SPIN_REQUEST, and evaluates whether it is non-zero - * i.e. whether the configuration is active. Wrapping up the config inside - * a boolean context prevents clang and smatch from complaining about potential - * issues in confusing logical-&& with bitwise-& for constants. - * - * Sadly IS_ENABLED() itself does not work with kconfig values. - * - * Returns 0 if @config is 0, 1 if set to any value. - */ -#define IS_ACTIVE(config) ((config) != 0) - #endif /* !__I915_UTILS_H */ diff --git a/drivers/gpu/drm/i915/i915_vma.c b/drivers/gpu/drm/i915/i915_vma.c index 4b7fc4647e46..90546fa58fc1 100644 --- a/drivers/gpu/drm/i915/i915_vma.c +++ b/drivers/gpu/drm/i915/i915_vma.c @@ -1234,9 +1234,10 @@ int __i915_vma_move_to_active(struct i915_vma *vma, struct i915_request *rq) return i915_active_add_request(&vma->active, rq); } -int i915_vma_move_to_active(struct i915_vma *vma, - struct i915_request *rq, - unsigned int flags) +int _i915_vma_move_to_active(struct i915_vma *vma, + struct i915_request *rq, + struct dma_fence *fence, + unsigned int flags) { struct drm_i915_gem_object *obj = vma->obj; int err; @@ -1257,9 +1258,11 @@ int i915_vma_move_to_active(struct i915_vma *vma, intel_frontbuffer_put(front); } - dma_resv_add_excl_fence(vma->resv, &rq->fence); - obj->write_domain = I915_GEM_DOMAIN_RENDER; - obj->read_domains = 0; + if (fence) { + dma_resv_add_excl_fence(vma->resv, fence); + obj->write_domain = I915_GEM_DOMAIN_RENDER; + obj->read_domains = 0; + } } else { if (!(flags & __EXEC_OBJECT_NO_RESERVE)) { err = dma_resv_reserve_shared(vma->resv, 1); @@ -1267,8 +1270,10 @@ int i915_vma_move_to_active(struct i915_vma *vma, return err; } - dma_resv_add_shared_fence(vma->resv, &rq->fence); - obj->write_domain = 0; + if (fence) { + dma_resv_add_shared_fence(vma->resv, fence); + obj->write_domain = 0; + } } if (flags & EXEC_OBJECT_NEEDS_FENCE && vma->fence) diff --git a/drivers/gpu/drm/i915/i915_vma.h b/drivers/gpu/drm/i915/i915_vma.h index ed69f66c7ab0..648dbe744c96 100644 --- a/drivers/gpu/drm/i915/i915_vma.h +++ b/drivers/gpu/drm/i915/i915_vma.h @@ -57,9 +57,16 @@ static inline bool i915_vma_is_active(const struct i915_vma *vma) int __must_check __i915_vma_move_to_active(struct i915_vma *vma, struct i915_request *rq); -int __must_check i915_vma_move_to_active(struct i915_vma *vma, - struct i915_request *rq, - unsigned int flags); +int __must_check _i915_vma_move_to_active(struct i915_vma *vma, + struct i915_request *rq, + struct dma_fence *fence, + unsigned int flags); +static inline int __must_check +i915_vma_move_to_active(struct i915_vma *vma, struct i915_request *rq, + unsigned int flags) +{ + return _i915_vma_move_to_active(vma, rq, &rq->fence, flags); +} #define __i915_vma_flags(v) ((unsigned long *)&(v)->flags.counter) diff --git a/drivers/gpu/drm/i915/i915_vma_types.h b/drivers/gpu/drm/i915/i915_vma_types.h index 995b502d7e5d..80e93bf00f2e 100644 --- a/drivers/gpu/drm/i915/i915_vma_types.h +++ b/drivers/gpu/drm/i915/i915_vma_types.h @@ -105,8 +105,9 @@ struct intel_remapped_plane_info { } __packed; struct intel_remapped_info { - struct intel_remapped_plane_info plane[2]; - u32 unused_mbz; + struct intel_remapped_plane_info plane[4]; + /* in gtt pages */ + u32 plane_alignment; } __packed; struct intel_rotation_info { @@ -129,7 +130,7 @@ static inline void assert_i915_gem_gtt_types(void) { BUILD_BUG_ON(sizeof(struct intel_rotation_info) != 2 * sizeof(u32) + 8 * sizeof(u16)); BUILD_BUG_ON(sizeof(struct intel_partial_info) != sizeof(u64) + sizeof(unsigned int)); - BUILD_BUG_ON(sizeof(struct intel_remapped_info) != 3 * sizeof(u32) + 8 * sizeof(u16)); + BUILD_BUG_ON(sizeof(struct intel_remapped_info) != 5 * sizeof(u32) + 16 * sizeof(u16)); /* Check that rotation/remapped shares offsets for simplicity */ BUILD_BUG_ON(offsetof(struct intel_remapped_info, plane[0]) != diff --git a/drivers/gpu/drm/i915/intel_device_info.h b/drivers/gpu/drm/i915/intel_device_info.h index d328bb95c49b..8e6f48d1eb7b 100644 --- a/drivers/gpu/drm/i915/intel_device_info.h +++ b/drivers/gpu/drm/i915/intel_device_info.h @@ -133,6 +133,7 @@ enum intel_ppgtt_type { func(has_logical_ring_elsq); \ func(has_mslices); \ func(has_pooled_eu); \ + func(has_pxp); \ func(has_rc6); \ func(has_rc6p); \ func(has_rps); \ diff --git a/drivers/gpu/drm/i915/intel_dram.c b/drivers/gpu/drm/i915/intel_dram.c index 7acce64b0941..84bb212bae4b 100644 --- a/drivers/gpu/drm/i915/intel_dram.c +++ b/drivers/gpu/drm/i915/intel_dram.c @@ -5,7 +5,7 @@ #include "i915_drv.h" #include "intel_dram.h" -#include "intel_sideband.h" +#include "intel_pcode.h" struct dram_dimm_info { u16 size; @@ -418,7 +418,7 @@ static int icl_pcode_read_mem_global_info(struct drm_i915_private *dev_priv) break; default: MISSING_CASE(val & 0xf); - return -1; + return -EINVAL; } } else { switch (val & 0xf) { @@ -436,7 +436,7 @@ static int icl_pcode_read_mem_global_info(struct drm_i915_private *dev_priv) break; default: MISSING_CASE(val & 0xf); - return -1; + return -EINVAL; } } diff --git a/drivers/gpu/drm/i915/intel_memory_region.c b/drivers/gpu/drm/i915/intel_memory_region.c index 779eb2fa90b6..e7f7e6627750 100644 --- a/drivers/gpu/drm/i915/intel_memory_region.c +++ b/drivers/gpu/drm/i915/intel_memory_region.c @@ -78,6 +78,18 @@ int intel_memory_region_reserve(struct intel_memory_region *mem, return i915_ttm_buddy_man_reserve(man, offset, size); } +void intel_memory_region_debug(struct intel_memory_region *mr, + struct drm_printer *printer) +{ + drm_printf(printer, "%s: ", mr->name); + + if (mr->region_private) + ttm_resource_manager_debug(mr->region_private, printer); + else + drm_printf(printer, "total:%pa, available:%pa bytes\n", + &mr->total, &mr->avail); +} + struct intel_memory_region * intel_memory_region_create(struct drm_i915_private *i915, resource_size_t start, diff --git a/drivers/gpu/drm/i915/intel_memory_region.h b/drivers/gpu/drm/i915/intel_memory_region.h index 1f2b96efa69d..3feae3353d33 100644 --- a/drivers/gpu/drm/i915/intel_memory_region.h +++ b/drivers/gpu/drm/i915/intel_memory_region.h @@ -15,6 +15,7 @@ struct drm_i915_private; struct drm_i915_gem_object; +struct drm_printer; struct intel_memory_region; struct sg_table; struct ttm_resource; @@ -127,6 +128,9 @@ int intel_memory_region_reserve(struct intel_memory_region *mem, resource_size_t offset, resource_size_t size); +void intel_memory_region_debug(struct intel_memory_region *mr, + struct drm_printer *printer); + struct intel_memory_region * i915_gem_ttm_system_setup(struct drm_i915_private *i915, u16 type, u16 instance); diff --git a/drivers/gpu/drm/i915/intel_pcode.c b/drivers/gpu/drm/i915/intel_pcode.c new file mode 100644 index 000000000000..e8c886e4e78d --- /dev/null +++ b/drivers/gpu/drm/i915/intel_pcode.c @@ -0,0 +1,235 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2013-2021 Intel Corporation + */ + +#include "i915_drv.h" +#include "intel_pcode.h" + +static int gen6_check_mailbox_status(u32 mbox) +{ + switch (mbox & GEN6_PCODE_ERROR_MASK) { + case GEN6_PCODE_SUCCESS: + return 0; + case GEN6_PCODE_UNIMPLEMENTED_CMD: + return -ENODEV; + case GEN6_PCODE_ILLEGAL_CMD: + return -ENXIO; + case GEN6_PCODE_MIN_FREQ_TABLE_GT_RATIO_OUT_OF_RANGE: + case GEN7_PCODE_MIN_FREQ_TABLE_GT_RATIO_OUT_OF_RANGE: + return -EOVERFLOW; + case GEN6_PCODE_TIMEOUT: + return -ETIMEDOUT; + default: + MISSING_CASE(mbox & GEN6_PCODE_ERROR_MASK); + return 0; + } +} + +static int gen7_check_mailbox_status(u32 mbox) +{ + switch (mbox & GEN6_PCODE_ERROR_MASK) { + case GEN6_PCODE_SUCCESS: + return 0; + case GEN6_PCODE_ILLEGAL_CMD: + return -ENXIO; + case GEN7_PCODE_TIMEOUT: + return -ETIMEDOUT; + case GEN7_PCODE_ILLEGAL_DATA: + return -EINVAL; + case GEN11_PCODE_ILLEGAL_SUBCOMMAND: + return -ENXIO; + case GEN11_PCODE_LOCKED: + return -EBUSY; + case GEN11_PCODE_REJECTED: + return -EACCES; + case GEN7_PCODE_MIN_FREQ_TABLE_GT_RATIO_OUT_OF_RANGE: + return -EOVERFLOW; + default: + MISSING_CASE(mbox & GEN6_PCODE_ERROR_MASK); + return 0; + } +} + +static int __sandybridge_pcode_rw(struct drm_i915_private *i915, + u32 mbox, u32 *val, u32 *val1, + int fast_timeout_us, + int slow_timeout_ms, + bool is_read) +{ + struct intel_uncore *uncore = &i915->uncore; + + lockdep_assert_held(&i915->sb_lock); + + /* + * GEN6_PCODE_* are outside of the forcewake domain, we can use + * intel_uncore_read/write_fw variants to reduce the amount of work + * required when reading/writing. + */ + + if (intel_uncore_read_fw(uncore, GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) + return -EAGAIN; + + intel_uncore_write_fw(uncore, GEN6_PCODE_DATA, *val); + intel_uncore_write_fw(uncore, GEN6_PCODE_DATA1, val1 ? *val1 : 0); + intel_uncore_write_fw(uncore, + GEN6_PCODE_MAILBOX, GEN6_PCODE_READY | mbox); + + if (__intel_wait_for_register_fw(uncore, + GEN6_PCODE_MAILBOX, + GEN6_PCODE_READY, 0, + fast_timeout_us, + slow_timeout_ms, + &mbox)) + return -ETIMEDOUT; + + if (is_read) + *val = intel_uncore_read_fw(uncore, GEN6_PCODE_DATA); + if (is_read && val1) + *val1 = intel_uncore_read_fw(uncore, GEN6_PCODE_DATA1); + + if (GRAPHICS_VER(i915) > 6) + return gen7_check_mailbox_status(mbox); + else + return gen6_check_mailbox_status(mbox); +} + +int sandybridge_pcode_read(struct drm_i915_private *i915, u32 mbox, + u32 *val, u32 *val1) +{ + int err; + + mutex_lock(&i915->sb_lock); + err = __sandybridge_pcode_rw(i915, mbox, val, val1, + 500, 20, + true); + mutex_unlock(&i915->sb_lock); + + if (err) { + drm_dbg(&i915->drm, + "warning: pcode (read from mbox %x) mailbox access failed for %ps: %d\n", + mbox, __builtin_return_address(0), err); + } + + return err; +} + +int sandybridge_pcode_write_timeout(struct drm_i915_private *i915, + u32 mbox, u32 val, + int fast_timeout_us, + int slow_timeout_ms) +{ + int err; + + mutex_lock(&i915->sb_lock); + err = __sandybridge_pcode_rw(i915, mbox, &val, NULL, + fast_timeout_us, slow_timeout_ms, + false); + mutex_unlock(&i915->sb_lock); + + if (err) { + drm_dbg(&i915->drm, + "warning: pcode (write of 0x%08x to mbox %x) mailbox access failed for %ps: %d\n", + val, mbox, __builtin_return_address(0), err); + } + + return err; +} + +static bool skl_pcode_try_request(struct drm_i915_private *i915, u32 mbox, + u32 request, u32 reply_mask, u32 reply, + u32 *status) +{ + *status = __sandybridge_pcode_rw(i915, mbox, &request, NULL, + 500, 0, + true); + + return *status || ((request & reply_mask) == reply); +} + +/** + * skl_pcode_request - send PCODE request until acknowledgment + * @i915: device private + * @mbox: PCODE mailbox ID the request is targeted for + * @request: request ID + * @reply_mask: mask used to check for request acknowledgment + * @reply: value used to check for request acknowledgment + * @timeout_base_ms: timeout for polling with preemption enabled + * + * Keep resending the @request to @mbox until PCODE acknowledges it, PCODE + * reports an error or an overall timeout of @timeout_base_ms+50 ms expires. + * The request is acknowledged once the PCODE reply dword equals @reply after + * applying @reply_mask. Polling is first attempted with preemption enabled + * for @timeout_base_ms and if this times out for another 50 ms with + * preemption disabled. + * + * Returns 0 on success, %-ETIMEDOUT in case of a timeout, <0 in case of some + * other error as reported by PCODE. + */ +int skl_pcode_request(struct drm_i915_private *i915, u32 mbox, u32 request, + u32 reply_mask, u32 reply, int timeout_base_ms) +{ + u32 status; + int ret; + + mutex_lock(&i915->sb_lock); + +#define COND \ + skl_pcode_try_request(i915, mbox, request, reply_mask, reply, &status) + + /* + * Prime the PCODE by doing a request first. Normally it guarantees + * that a subsequent request, at most @timeout_base_ms later, succeeds. + * _wait_for() doesn't guarantee when its passed condition is evaluated + * first, so send the first request explicitly. + */ + if (COND) { + ret = 0; + goto out; + } + ret = _wait_for(COND, timeout_base_ms * 1000, 10, 10); + if (!ret) + goto out; + + /* + * The above can time out if the number of requests was low (2 in the + * worst case) _and_ PCODE was busy for some reason even after a + * (queued) request and @timeout_base_ms delay. As a workaround retry + * the poll with preemption disabled to maximize the number of + * requests. Increase the timeout from @timeout_base_ms to 50ms to + * account for interrupts that could reduce the number of these + * requests, and for any quirks of the PCODE firmware that delays + * the request completion. + */ + drm_dbg_kms(&i915->drm, + "PCODE timeout, retrying with preemption disabled\n"); + drm_WARN_ON_ONCE(&i915->drm, timeout_base_ms > 3); + preempt_disable(); + ret = wait_for_atomic(COND, 50); + preempt_enable(); + +out: + mutex_unlock(&i915->sb_lock); + return ret ? ret : status; +#undef COND +} + +int intel_pcode_init(struct drm_i915_private *i915) +{ + int ret = 0; + + if (!IS_DGFX(i915)) + return ret; + + ret = skl_pcode_request(i915, DG1_PCODE_STATUS, + DG1_UNCORE_GET_INIT_STATUS, + DG1_UNCORE_INIT_STATUS_COMPLETE, + DG1_UNCORE_INIT_STATUS_COMPLETE, 180000); + + drm_dbg(&i915->drm, "PCODE init status %d\n", ret); + + if (ret) + drm_err(&i915->drm, "Pcode did not report uncore initialization completion!\n"); + + return ret; +} diff --git a/drivers/gpu/drm/i915/intel_pcode.h b/drivers/gpu/drm/i915/intel_pcode.h new file mode 100644 index 000000000000..50806649d4b6 --- /dev/null +++ b/drivers/gpu/drm/i915/intel_pcode.h @@ -0,0 +1,26 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2013-2021 Intel Corporation + */ + +#ifndef _INTEL_PCODE_H_ +#define _INTEL_PCODE_H_ + +#include <linux/types.h> + +struct drm_i915_private; + +int sandybridge_pcode_read(struct drm_i915_private *i915, u32 mbox, + u32 *val, u32 *val1); +int sandybridge_pcode_write_timeout(struct drm_i915_private *i915, u32 mbox, + u32 val, int fast_timeout_us, + int slow_timeout_ms); +#define sandybridge_pcode_write(i915, mbox, val) \ + sandybridge_pcode_write_timeout(i915, mbox, val, 500, 0) + +int skl_pcode_request(struct drm_i915_private *i915, u32 mbox, u32 request, + u32 reply_mask, u32 reply, int timeout_base_ms); + +int intel_pcode_init(struct drm_i915_private *i915); + +#endif /* _INTEL_PCODE_H */ diff --git a/drivers/gpu/drm/i915/intel_pm.c b/drivers/gpu/drm/i915/intel_pm.c index a725792d5248..ecbb3d141632 100644 --- a/drivers/gpu/drm/i915/intel_pm.c +++ b/drivers/gpu/drm/i915/intel_pm.c @@ -47,8 +47,9 @@ #include "i915_fixed.h" #include "i915_irq.h" #include "i915_trace.h" +#include "intel_pcode.h" #include "intel_pm.h" -#include "intel_sideband.h" +#include "vlv_sideband.h" #include "../../../platform/x86/intel_ips.h" /* Stores plane specific WM parameters */ @@ -893,9 +894,8 @@ static struct intel_crtc *single_enabled_crtc(struct drm_i915_private *dev_priv) return enabled; } -static void pnv_update_wm(struct intel_crtc *unused_crtc) +static void pnv_update_wm(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = to_i915(unused_crtc->base.dev); struct intel_crtc *crtc; const struct cxsr_latency *latency; u32 reg; @@ -1164,17 +1164,13 @@ static u16 g4x_compute_wm(const struct intel_crtc_state *crtc_state, cpp = plane_state->hw.fb->format->cpp[0]; /* - * Not 100% sure which way ELK should go here as the - * spec only says CL/CTG should assume 32bpp and BW - * doesn't need to. But as these things followed the - * mobile vs. desktop lines on gen3 as well, let's - * assume ELK doesn't need this. + * WaUse32BppForSRWM:ctg,elk * - * The spec also fails to list such a restriction for - * the HPLL watermark, which seems a little strange. + * The spec fails to list this restriction for the + * HPLL watermark, which seems a little strange. * Let's use 32bpp for the HPLL watermark as well. */ - if (IS_GM45(dev_priv) && plane->id == PLANE_PRIMARY && + if (plane->id == PLANE_PRIMARY && level != G4X_WM_LEVEL_NORMAL) cpp = max(cpp, 4u); @@ -1388,8 +1384,7 @@ static int g4x_compute_pipe_wm(struct intel_atomic_state *state, struct intel_crtc_state *crtc_state = intel_atomic_get_new_crtc_state(state, crtc); struct g4x_wm_state *wm_state = &crtc_state->wm.g4x.optimal; - int num_active_planes = hweight8(crtc_state->active_planes & - ~BIT(PLANE_CURSOR)); + u8 active_planes = crtc_state->active_planes & ~BIT(PLANE_CURSOR); const struct g4x_pipe_wm *raw; const struct intel_plane_state *old_plane_state; const struct intel_plane_state *new_plane_state; @@ -1429,7 +1424,7 @@ static int g4x_compute_pipe_wm(struct intel_atomic_state *state, wm_state->sr.cursor = raw->plane[PLANE_CURSOR]; wm_state->sr.fbc = raw->fbc; - wm_state->cxsr = num_active_planes == BIT(PLANE_PRIMARY); + wm_state->cxsr = active_planes == BIT(PLANE_PRIMARY); level = G4X_WM_LEVEL_HPLL; if (!g4x_raw_crtc_wm_is_valid(crtc_state, level)) @@ -1720,7 +1715,7 @@ static int vlv_compute_fifo(struct intel_crtc_state *crtc_state) const struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM2]; struct vlv_fifo_state *fifo_state = &crtc_state->wm.vlv.fifo_state; - unsigned int active_planes = crtc_state->active_planes & ~BIT(PLANE_CURSOR); + u8 active_planes = crtc_state->active_planes & ~BIT(PLANE_CURSOR); int num_active_planes = hweight8(active_planes); const int fifo_size = 511; int fifo_extra, fifo_left = fifo_size; @@ -1912,8 +1907,8 @@ static int vlv_compute_pipe_wm(struct intel_atomic_state *state, struct vlv_wm_state *wm_state = &crtc_state->wm.vlv.optimal; const struct vlv_fifo_state *fifo_state = &crtc_state->wm.vlv.fifo_state; - int num_active_planes = hweight8(crtc_state->active_planes & - ~BIT(PLANE_CURSOR)); + u8 active_planes = crtc_state->active_planes & ~BIT(PLANE_CURSOR); + int num_active_planes = hweight8(active_planes); bool needs_modeset = drm_atomic_crtc_needs_modeset(&crtc_state->uapi); const struct intel_plane_state *old_plane_state; const struct intel_plane_state *new_plane_state; @@ -2265,9 +2260,8 @@ static void vlv_optimize_watermarks(struct intel_atomic_state *state, mutex_unlock(&dev_priv->wm.wm_mutex); } -static void i965_update_wm(struct intel_crtc *unused_crtc) +static void i965_update_wm(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = to_i915(unused_crtc->base.dev); struct intel_crtc *crtc; int srwm = 1; int cursor_sr = 16; @@ -2341,9 +2335,8 @@ static void i965_update_wm(struct intel_crtc *unused_crtc) #undef FW_WM -static void i9xx_update_wm(struct intel_crtc *unused_crtc) +static void i9xx_update_wm(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = to_i915(unused_crtc->base.dev); const struct intel_watermark_params *wm_info; u32 fwater_lo; u32 fwater_hi; @@ -2359,7 +2352,10 @@ static void i9xx_update_wm(struct intel_crtc *unused_crtc) else wm_info = &i830_a_wm_info; - fifo_size = dev_priv->display.get_fifo_size(dev_priv, PLANE_A); + if (DISPLAY_VER(dev_priv) == 2) + fifo_size = i830_get_fifo_size(dev_priv, PLANE_A); + else + fifo_size = i9xx_get_fifo_size(dev_priv, PLANE_A); crtc = intel_get_crtc_for_plane(dev_priv, PLANE_A); if (intel_crtc_active(crtc)) { const struct drm_display_mode *pipe_mode = @@ -2386,7 +2382,10 @@ static void i9xx_update_wm(struct intel_crtc *unused_crtc) if (DISPLAY_VER(dev_priv) == 2) wm_info = &i830_bc_wm_info; - fifo_size = dev_priv->display.get_fifo_size(dev_priv, PLANE_B); + if (DISPLAY_VER(dev_priv) == 2) + fifo_size = i830_get_fifo_size(dev_priv, PLANE_B); + else + fifo_size = i9xx_get_fifo_size(dev_priv, PLANE_B); crtc = intel_get_crtc_for_plane(dev_priv, PLANE_B); if (intel_crtc_active(crtc)) { const struct drm_display_mode *pipe_mode = @@ -2487,9 +2486,8 @@ static void i9xx_update_wm(struct intel_crtc *unused_crtc) intel_set_memory_cxsr(dev_priv, true); } -static void i845_update_wm(struct intel_crtc *unused_crtc) +static void i845_update_wm(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = to_i915(unused_crtc->base.dev); struct intel_crtc *crtc; const struct drm_display_mode *pipe_mode; u32 fwater_lo; @@ -2502,7 +2500,7 @@ static void i845_update_wm(struct intel_crtc *unused_crtc) pipe_mode = &crtc->config->hw.pipe_mode; planea_wm = intel_calculate_wm(pipe_mode->crtc_clock, &i845_wm_info, - dev_priv->display.get_fifo_size(dev_priv, PLANE_A), + i845_get_fifo_size(dev_priv, PLANE_A), 4, pessimal_latency_ns); fwater_lo = intel_uncore_read(&dev_priv->uncore, FW_BLC) & ~0xfff; fwater_lo |= (3<<8) | planea_wm; @@ -2871,6 +2869,7 @@ static void intel_read_wm_latency(struct drm_i915_private *dev_priv, u32 val; int ret, i; int level, max_level = ilk_wm_max_level(dev_priv); + int mult = IS_DG2(dev_priv) ? 2 : 1; /* read the first set of memory latencies[0:3] */ val = 0; /* data0 to be programmed to 0 for first set */ @@ -2884,13 +2883,13 @@ static void intel_read_wm_latency(struct drm_i915_private *dev_priv, return; } - wm[0] = val & GEN9_MEM_LATENCY_LEVEL_MASK; - wm[1] = (val >> GEN9_MEM_LATENCY_LEVEL_1_5_SHIFT) & - GEN9_MEM_LATENCY_LEVEL_MASK; - wm[2] = (val >> GEN9_MEM_LATENCY_LEVEL_2_6_SHIFT) & - GEN9_MEM_LATENCY_LEVEL_MASK; - wm[3] = (val >> GEN9_MEM_LATENCY_LEVEL_3_7_SHIFT) & - GEN9_MEM_LATENCY_LEVEL_MASK; + wm[0] = (val & GEN9_MEM_LATENCY_LEVEL_MASK) * mult; + wm[1] = ((val >> GEN9_MEM_LATENCY_LEVEL_1_5_SHIFT) & + GEN9_MEM_LATENCY_LEVEL_MASK) * mult; + wm[2] = ((val >> GEN9_MEM_LATENCY_LEVEL_2_6_SHIFT) & + GEN9_MEM_LATENCY_LEVEL_MASK) * mult; + wm[3] = ((val >> GEN9_MEM_LATENCY_LEVEL_3_7_SHIFT) & + GEN9_MEM_LATENCY_LEVEL_MASK) * mult; /* read the second set of memory latencies[4:7] */ val = 1; /* data0 to be programmed to 1 for second set */ @@ -2903,13 +2902,13 @@ static void intel_read_wm_latency(struct drm_i915_private *dev_priv, return; } - wm[4] = val & GEN9_MEM_LATENCY_LEVEL_MASK; - wm[5] = (val >> GEN9_MEM_LATENCY_LEVEL_1_5_SHIFT) & - GEN9_MEM_LATENCY_LEVEL_MASK; - wm[6] = (val >> GEN9_MEM_LATENCY_LEVEL_2_6_SHIFT) & - GEN9_MEM_LATENCY_LEVEL_MASK; - wm[7] = (val >> GEN9_MEM_LATENCY_LEVEL_3_7_SHIFT) & - GEN9_MEM_LATENCY_LEVEL_MASK; + wm[4] = (val & GEN9_MEM_LATENCY_LEVEL_MASK) * mult; + wm[5] = ((val >> GEN9_MEM_LATENCY_LEVEL_1_5_SHIFT) & + GEN9_MEM_LATENCY_LEVEL_MASK) * mult; + wm[6] = ((val >> GEN9_MEM_LATENCY_LEVEL_2_6_SHIFT) & + GEN9_MEM_LATENCY_LEVEL_MASK) * mult; + wm[7] = ((val >> GEN9_MEM_LATENCY_LEVEL_3_7_SHIFT) & + GEN9_MEM_LATENCY_LEVEL_MASK) * mult; /* * If a level n (n > 1) has a 0us latency, all levels m (m >= n) @@ -6844,7 +6843,8 @@ void g4x_wm_get_hw_state(struct drm_i915_private *dev_priv) for_each_plane_id_on_crtc(crtc, plane_id) raw->plane[plane_id] = active->wm.plane[plane_id]; - if (++level > max_level) + level = G4X_WM_LEVEL_SR; + if (level > max_level) goto out; raw = &crtc_state->wm.g4x.raw[level]; @@ -6853,7 +6853,8 @@ void g4x_wm_get_hw_state(struct drm_i915_private *dev_priv) raw->plane[PLANE_SPRITE0] = 0; raw->fbc = active->sr.fbc; - if (++level > max_level) + level = G4X_WM_LEVEL_HPLL; + if (level > max_level) goto out; raw = &crtc_state->wm.g4x.raw[level]; @@ -6862,6 +6863,7 @@ void g4x_wm_get_hw_state(struct drm_i915_private *dev_priv) raw->plane[PLANE_SPRITE0] = 0; raw->fbc = active->hpll.fbc; + level++; out: for_each_plane_id_on_crtc(crtc, plane_id) g4x_raw_plane_wm_set(crtc_state, level, @@ -7141,47 +7143,6 @@ void ilk_wm_get_hw_state(struct drm_i915_private *dev_priv) !(intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) & DISP_FBC_WM_DIS); } -/** - * intel_update_watermarks - update FIFO watermark values based on current modes - * @crtc: the #intel_crtc on which to compute the WM - * - * Calculate watermark values for the various WM regs based on current mode - * and plane configuration. - * - * There are several cases to deal with here: - * - normal (i.e. non-self-refresh) - * - self-refresh (SR) mode - * - lines are large relative to FIFO size (buffer can hold up to 2) - * - lines are small relative to FIFO size (buffer can hold more than 2 - * lines), so need to account for TLB latency - * - * The normal calculation is: - * watermark = dotclock * bytes per pixel * latency - * where latency is platform & configuration dependent (we assume pessimal - * values here). - * - * The SR calculation is: - * watermark = (trunc(latency/line time)+1) * surface width * - * bytes per pixel - * where - * line time = htotal / dotclock - * surface width = hdisplay for normal plane and 64 for cursor - * and latency is assumed to be high, as above. - * - * The final value programmed to the register should always be rounded up, - * and include an extra 2 entries to account for clock crossings. - * - * We don't use the sprite, so we can ignore that. And on Crestline we have - * to set the non-SR watermarks to 8. - */ -void intel_update_watermarks(struct intel_crtc *crtc) -{ - struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); - - if (dev_priv->display.update_wm) - dev_priv->display.update_wm(crtc); -} - void intel_enable_ipc(struct drm_i915_private *dev_priv) { u32 val; @@ -7639,11 +7600,6 @@ static void bdw_init_clock_gating(struct drm_i915_private *dev_priv) intel_uncore_write(&dev_priv->uncore, CHICKEN_PIPESL_1(pipe), intel_uncore_read(&dev_priv->uncore, CHICKEN_PIPESL_1(pipe)) | BDW_DPRS_MASK_VBLANK_SRD); - - /* Undocumented but fixes async flip + VT-d corruption */ - if (intel_vtd_active()) - intel_uncore_rmw(&dev_priv->uncore, CHICKEN_PIPESL_1(pipe), - HSW_PRI_STRETCH_MAX_MASK, HSW_PRI_STRETCH_MAX_X1); } /* WaVSRefCountFullforceMissDisable:bdw */ @@ -7679,20 +7635,11 @@ static void bdw_init_clock_gating(struct drm_i915_private *dev_priv) static void hsw_init_clock_gating(struct drm_i915_private *dev_priv) { - enum pipe pipe; - /* WaFbcAsynchFlipDisableFbcQueue:hsw,bdw */ intel_uncore_write(&dev_priv->uncore, CHICKEN_PIPESL_1(PIPE_A), intel_uncore_read(&dev_priv->uncore, CHICKEN_PIPESL_1(PIPE_A)) | HSW_FBCQ_DIS); - for_each_pipe(dev_priv, pipe) { - /* Undocumented but fixes async flip + VT-d corruption */ - if (intel_vtd_active()) - intel_uncore_rmw(&dev_priv->uncore, CHICKEN_PIPESL_1(pipe), - HSW_PRI_STRETCH_MAX_MASK, HSW_PRI_STRETCH_MAX_X1); - } - /* This is required by WaCatErrorRejectionIssue:hsw */ intel_uncore_write(&dev_priv->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG, intel_uncore_read(&dev_priv->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) | @@ -7921,7 +7868,7 @@ static void i830_init_clock_gating(struct drm_i915_private *dev_priv) void intel_init_clock_gating(struct drm_i915_private *dev_priv) { - dev_priv->display.init_clock_gating(dev_priv); + dev_priv->clock_gating_funcs->init_clock_gating(dev_priv); } void intel_suspend_hw(struct drm_i915_private *dev_priv) @@ -7936,6 +7883,36 @@ static void nop_init_clock_gating(struct drm_i915_private *dev_priv) "No clock gating settings or workarounds applied.\n"); } +#define CG_FUNCS(platform) \ +static const struct drm_i915_clock_gating_funcs platform##_clock_gating_funcs = { \ + .init_clock_gating = platform##_init_clock_gating, \ +} + +CG_FUNCS(adlp); +CG_FUNCS(dg1); +CG_FUNCS(gen12lp); +CG_FUNCS(icl); +CG_FUNCS(cfl); +CG_FUNCS(skl); +CG_FUNCS(kbl); +CG_FUNCS(bxt); +CG_FUNCS(glk); +CG_FUNCS(bdw); +CG_FUNCS(chv); +CG_FUNCS(hsw); +CG_FUNCS(ivb); +CG_FUNCS(vlv); +CG_FUNCS(gen6); +CG_FUNCS(ilk); +CG_FUNCS(g4x); +CG_FUNCS(i965gm); +CG_FUNCS(i965g); +CG_FUNCS(gen3); +CG_FUNCS(i85x); +CG_FUNCS(i830); +CG_FUNCS(nop); +#undef CG_FUNCS + /** * intel_init_clock_gating_hooks - setup the clock gating hooks * @dev_priv: device private @@ -7948,55 +7925,100 @@ static void nop_init_clock_gating(struct drm_i915_private *dev_priv) void intel_init_clock_gating_hooks(struct drm_i915_private *dev_priv) { if (IS_ALDERLAKE_P(dev_priv)) - dev_priv->display.init_clock_gating = adlp_init_clock_gating; + dev_priv->clock_gating_funcs = &adlp_clock_gating_funcs; else if (IS_DG1(dev_priv)) - dev_priv->display.init_clock_gating = dg1_init_clock_gating; + dev_priv->clock_gating_funcs = &dg1_clock_gating_funcs; else if (GRAPHICS_VER(dev_priv) == 12) - dev_priv->display.init_clock_gating = gen12lp_init_clock_gating; + dev_priv->clock_gating_funcs = &gen12lp_clock_gating_funcs; else if (GRAPHICS_VER(dev_priv) == 11) - dev_priv->display.init_clock_gating = icl_init_clock_gating; + dev_priv->clock_gating_funcs = &icl_clock_gating_funcs; else if (IS_COFFEELAKE(dev_priv) || IS_COMETLAKE(dev_priv)) - dev_priv->display.init_clock_gating = cfl_init_clock_gating; + dev_priv->clock_gating_funcs = &cfl_clock_gating_funcs; else if (IS_SKYLAKE(dev_priv)) - dev_priv->display.init_clock_gating = skl_init_clock_gating; + dev_priv->clock_gating_funcs = &skl_clock_gating_funcs; else if (IS_KABYLAKE(dev_priv)) - dev_priv->display.init_clock_gating = kbl_init_clock_gating; + dev_priv->clock_gating_funcs = &kbl_clock_gating_funcs; else if (IS_BROXTON(dev_priv)) - dev_priv->display.init_clock_gating = bxt_init_clock_gating; + dev_priv->clock_gating_funcs = &bxt_clock_gating_funcs; else if (IS_GEMINILAKE(dev_priv)) - dev_priv->display.init_clock_gating = glk_init_clock_gating; + dev_priv->clock_gating_funcs = &glk_clock_gating_funcs; else if (IS_BROADWELL(dev_priv)) - dev_priv->display.init_clock_gating = bdw_init_clock_gating; + dev_priv->clock_gating_funcs = &bdw_clock_gating_funcs; else if (IS_CHERRYVIEW(dev_priv)) - dev_priv->display.init_clock_gating = chv_init_clock_gating; + dev_priv->clock_gating_funcs = &chv_clock_gating_funcs; else if (IS_HASWELL(dev_priv)) - dev_priv->display.init_clock_gating = hsw_init_clock_gating; + dev_priv->clock_gating_funcs = &hsw_clock_gating_funcs; else if (IS_IVYBRIDGE(dev_priv)) - dev_priv->display.init_clock_gating = ivb_init_clock_gating; + dev_priv->clock_gating_funcs = &ivb_clock_gating_funcs; else if (IS_VALLEYVIEW(dev_priv)) - dev_priv->display.init_clock_gating = vlv_init_clock_gating; + dev_priv->clock_gating_funcs = &vlv_clock_gating_funcs; else if (GRAPHICS_VER(dev_priv) == 6) - dev_priv->display.init_clock_gating = gen6_init_clock_gating; + dev_priv->clock_gating_funcs = &gen6_clock_gating_funcs; else if (GRAPHICS_VER(dev_priv) == 5) - dev_priv->display.init_clock_gating = ilk_init_clock_gating; + dev_priv->clock_gating_funcs = &ilk_clock_gating_funcs; else if (IS_G4X(dev_priv)) - dev_priv->display.init_clock_gating = g4x_init_clock_gating; + dev_priv->clock_gating_funcs = &g4x_clock_gating_funcs; else if (IS_I965GM(dev_priv)) - dev_priv->display.init_clock_gating = i965gm_init_clock_gating; + dev_priv->clock_gating_funcs = &i965gm_clock_gating_funcs; else if (IS_I965G(dev_priv)) - dev_priv->display.init_clock_gating = i965g_init_clock_gating; + dev_priv->clock_gating_funcs = &i965g_clock_gating_funcs; else if (GRAPHICS_VER(dev_priv) == 3) - dev_priv->display.init_clock_gating = gen3_init_clock_gating; + dev_priv->clock_gating_funcs = &gen3_clock_gating_funcs; else if (IS_I85X(dev_priv) || IS_I865G(dev_priv)) - dev_priv->display.init_clock_gating = i85x_init_clock_gating; + dev_priv->clock_gating_funcs = &i85x_clock_gating_funcs; else if (GRAPHICS_VER(dev_priv) == 2) - dev_priv->display.init_clock_gating = i830_init_clock_gating; + dev_priv->clock_gating_funcs = &i830_clock_gating_funcs; else { MISSING_CASE(INTEL_DEVID(dev_priv)); - dev_priv->display.init_clock_gating = nop_init_clock_gating; + dev_priv->clock_gating_funcs = &nop_clock_gating_funcs; } } +static const struct drm_i915_wm_disp_funcs skl_wm_funcs = { + .compute_global_watermarks = skl_compute_wm, +}; + +static const struct drm_i915_wm_disp_funcs ilk_wm_funcs = { + .compute_pipe_wm = ilk_compute_pipe_wm, + .compute_intermediate_wm = ilk_compute_intermediate_wm, + .initial_watermarks = ilk_initial_watermarks, + .optimize_watermarks = ilk_optimize_watermarks, +}; + +static const struct drm_i915_wm_disp_funcs vlv_wm_funcs = { + .compute_pipe_wm = vlv_compute_pipe_wm, + .compute_intermediate_wm = vlv_compute_intermediate_wm, + .initial_watermarks = vlv_initial_watermarks, + .optimize_watermarks = vlv_optimize_watermarks, + .atomic_update_watermarks = vlv_atomic_update_fifo, +}; + +static const struct drm_i915_wm_disp_funcs g4x_wm_funcs = { + .compute_pipe_wm = g4x_compute_pipe_wm, + .compute_intermediate_wm = g4x_compute_intermediate_wm, + .initial_watermarks = g4x_initial_watermarks, + .optimize_watermarks = g4x_optimize_watermarks, +}; + +static const struct drm_i915_wm_disp_funcs pnv_wm_funcs = { + .update_wm = pnv_update_wm, +}; + +static const struct drm_i915_wm_disp_funcs i965_wm_funcs = { + .update_wm = i965_update_wm, +}; + +static const struct drm_i915_wm_disp_funcs i9xx_wm_funcs = { + .update_wm = i9xx_update_wm, +}; + +static const struct drm_i915_wm_disp_funcs i845_wm_funcs = { + .update_wm = i845_update_wm, +}; + +static const struct drm_i915_wm_disp_funcs nop_funcs = { +}; + /* Set up chip specific power management-related functions */ void intel_init_pm(struct drm_i915_private *dev_priv) { @@ -8012,7 +8034,7 @@ void intel_init_pm(struct drm_i915_private *dev_priv) /* For FIFO watermark updates */ if (DISPLAY_VER(dev_priv) >= 9) { skl_setup_wm_latency(dev_priv); - dev_priv->display.compute_global_watermarks = skl_compute_wm; + dev_priv->wm_disp = &skl_wm_funcs; } else if (HAS_PCH_SPLIT(dev_priv)) { ilk_setup_wm_latency(dev_priv); @@ -8020,31 +8042,19 @@ void intel_init_pm(struct drm_i915_private *dev_priv) dev_priv->wm.spr_latency[1] && dev_priv->wm.cur_latency[1]) || (DISPLAY_VER(dev_priv) != 5 && dev_priv->wm.pri_latency[0] && dev_priv->wm.spr_latency[0] && dev_priv->wm.cur_latency[0])) { - dev_priv->display.compute_pipe_wm = ilk_compute_pipe_wm; - dev_priv->display.compute_intermediate_wm = - ilk_compute_intermediate_wm; - dev_priv->display.initial_watermarks = - ilk_initial_watermarks; - dev_priv->display.optimize_watermarks = - ilk_optimize_watermarks; + dev_priv->wm_disp = &ilk_wm_funcs; } else { drm_dbg_kms(&dev_priv->drm, "Failed to read display plane latency. " "Disable CxSR\n"); + dev_priv->wm_disp = &nop_funcs; } } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) { vlv_setup_wm_latency(dev_priv); - dev_priv->display.compute_pipe_wm = vlv_compute_pipe_wm; - dev_priv->display.compute_intermediate_wm = vlv_compute_intermediate_wm; - dev_priv->display.initial_watermarks = vlv_initial_watermarks; - dev_priv->display.optimize_watermarks = vlv_optimize_watermarks; - dev_priv->display.atomic_update_watermarks = vlv_atomic_update_fifo; + dev_priv->wm_disp = &vlv_wm_funcs; } else if (IS_G4X(dev_priv)) { g4x_setup_wm_latency(dev_priv); - dev_priv->display.compute_pipe_wm = g4x_compute_pipe_wm; - dev_priv->display.compute_intermediate_wm = g4x_compute_intermediate_wm; - dev_priv->display.initial_watermarks = g4x_initial_watermarks; - dev_priv->display.optimize_watermarks = g4x_optimize_watermarks; + dev_priv->wm_disp = &g4x_wm_funcs; } else if (IS_PINEVIEW(dev_priv)) { if (!intel_get_cxsr_latency(!IS_MOBILE(dev_priv), dev_priv->is_ddr3, @@ -8058,25 +8068,22 @@ void intel_init_pm(struct drm_i915_private *dev_priv) dev_priv->fsb_freq, dev_priv->mem_freq); /* Disable CxSR and never update its watermark again */ intel_set_memory_cxsr(dev_priv, false); - dev_priv->display.update_wm = NULL; + dev_priv->wm_disp = &nop_funcs; } else - dev_priv->display.update_wm = pnv_update_wm; + dev_priv->wm_disp = &pnv_wm_funcs; } else if (DISPLAY_VER(dev_priv) == 4) { - dev_priv->display.update_wm = i965_update_wm; + dev_priv->wm_disp = &i965_wm_funcs; } else if (DISPLAY_VER(dev_priv) == 3) { - dev_priv->display.update_wm = i9xx_update_wm; - dev_priv->display.get_fifo_size = i9xx_get_fifo_size; + dev_priv->wm_disp = &i9xx_wm_funcs; } else if (DISPLAY_VER(dev_priv) == 2) { - if (INTEL_NUM_PIPES(dev_priv) == 1) { - dev_priv->display.update_wm = i845_update_wm; - dev_priv->display.get_fifo_size = i845_get_fifo_size; - } else { - dev_priv->display.update_wm = i9xx_update_wm; - dev_priv->display.get_fifo_size = i830_get_fifo_size; - } + if (INTEL_NUM_PIPES(dev_priv) == 1) + dev_priv->wm_disp = &i845_wm_funcs; + else + dev_priv->wm_disp = &i9xx_wm_funcs; } else { drm_err(&dev_priv->drm, "unexpected fall-through in %s\n", __func__); + dev_priv->wm_disp = &nop_funcs; } } diff --git a/drivers/gpu/drm/i915/intel_pm.h b/drivers/gpu/drm/i915/intel_pm.h index 91f23b7f0af2..990cdcaf85ce 100644 --- a/drivers/gpu/drm/i915/intel_pm.h +++ b/drivers/gpu/drm/i915/intel_pm.h @@ -8,7 +8,6 @@ #include <linux/types.h> -#include "display/intel_bw.h" #include "display/intel_display.h" #include "display/intel_global_state.h" @@ -19,6 +18,7 @@ struct drm_device; struct drm_i915_private; struct i915_request; struct intel_atomic_state; +struct intel_bw_state; struct intel_crtc; struct intel_crtc_state; struct intel_plane; @@ -29,7 +29,6 @@ struct skl_wm_level; void intel_init_clock_gating(struct drm_i915_private *dev_priv); void intel_suspend_hw(struct drm_i915_private *dev_priv); int ilk_wm_max_level(const struct drm_i915_private *dev_priv); -void intel_update_watermarks(struct intel_crtc *crtc); void intel_init_pm(struct drm_i915_private *dev_priv); void intel_init_clock_gating_hooks(struct drm_i915_private *dev_priv); void intel_pm_setup(struct drm_i915_private *dev_priv); diff --git a/drivers/gpu/drm/i915/intel_runtime_pm.h b/drivers/gpu/drm/i915/intel_runtime_pm.h index 183ea2b187fe..47a85fab4130 100644 --- a/drivers/gpu/drm/i915/intel_runtime_pm.h +++ b/drivers/gpu/drm/i915/intel_runtime_pm.h @@ -8,8 +8,6 @@ #include <linux/types.h> -#include "display/intel_display.h" - #include "intel_wakeref.h" #include "i915_utils.h" diff --git a/drivers/gpu/drm/i915/intel_sbi.c b/drivers/gpu/drm/i915/intel_sbi.c new file mode 100644 index 000000000000..5ba8490a31e6 --- /dev/null +++ b/drivers/gpu/drm/i915/intel_sbi.c @@ -0,0 +1,73 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2013-2021 Intel Corporation + * + * LPT/WPT IOSF sideband. + */ + +#include "i915_drv.h" +#include "intel_sbi.h" + +/* SBI access */ +static int intel_sbi_rw(struct drm_i915_private *i915, u16 reg, + enum intel_sbi_destination destination, + u32 *val, bool is_read) +{ + struct intel_uncore *uncore = &i915->uncore; + u32 cmd; + + lockdep_assert_held(&i915->sb_lock); + + if (intel_wait_for_register_fw(uncore, + SBI_CTL_STAT, SBI_BUSY, 0, + 100)) { + drm_err(&i915->drm, + "timeout waiting for SBI to become ready\n"); + return -EBUSY; + } + + intel_uncore_write_fw(uncore, SBI_ADDR, (u32)reg << 16); + intel_uncore_write_fw(uncore, SBI_DATA, is_read ? 0 : *val); + + if (destination == SBI_ICLK) + cmd = SBI_CTL_DEST_ICLK | SBI_CTL_OP_CRRD; + else + cmd = SBI_CTL_DEST_MPHY | SBI_CTL_OP_IORD; + if (!is_read) + cmd |= BIT(8); + intel_uncore_write_fw(uncore, SBI_CTL_STAT, cmd | SBI_BUSY); + + if (__intel_wait_for_register_fw(uncore, + SBI_CTL_STAT, SBI_BUSY, 0, + 100, 100, &cmd)) { + drm_err(&i915->drm, + "timeout waiting for SBI to complete read\n"); + return -ETIMEDOUT; + } + + if (cmd & SBI_RESPONSE_FAIL) { + drm_err(&i915->drm, "error during SBI read of reg %x\n", reg); + return -ENXIO; + } + + if (is_read) + *val = intel_uncore_read_fw(uncore, SBI_DATA); + + return 0; +} + +u32 intel_sbi_read(struct drm_i915_private *i915, u16 reg, + enum intel_sbi_destination destination) +{ + u32 result = 0; + + intel_sbi_rw(i915, reg, destination, &result, true); + + return result; +} + +void intel_sbi_write(struct drm_i915_private *i915, u16 reg, u32 value, + enum intel_sbi_destination destination) +{ + intel_sbi_rw(i915, reg, destination, &value, false); +} diff --git a/drivers/gpu/drm/i915/intel_sbi.h b/drivers/gpu/drm/i915/intel_sbi.h new file mode 100644 index 000000000000..f5a862210454 --- /dev/null +++ b/drivers/gpu/drm/i915/intel_sbi.h @@ -0,0 +1,23 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2013-2021 Intel Corporation + */ + +#ifndef _INTEL_SBI_H_ +#define _INTEL_SBI_H_ + +#include <linux/types.h> + +struct drm_i915_private; + +enum intel_sbi_destination { + SBI_ICLK, + SBI_MPHY, +}; + +u32 intel_sbi_read(struct drm_i915_private *i915, u16 reg, + enum intel_sbi_destination destination); +void intel_sbi_write(struct drm_i915_private *i915, u16 reg, u32 value, + enum intel_sbi_destination destination); + +#endif /* _INTEL_SBI_H_ */ diff --git a/drivers/gpu/drm/i915/intel_sideband.c b/drivers/gpu/drm/i915/intel_sideband.c deleted file mode 100644 index e304bf44e1ff..000000000000 --- a/drivers/gpu/drm/i915/intel_sideband.c +++ /dev/null @@ -1,577 +0,0 @@ -/* - * Copyright © 2013 Intel Corporation - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice (including the next - * paragraph) shall be included in all copies or substantial portions of the - * Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - * - */ - -#include <asm/iosf_mbi.h> - -#include "i915_drv.h" -#include "intel_sideband.h" - -/* - * IOSF sideband, see VLV2_SidebandMsg_HAS.docx and - * VLV_VLV2_PUNIT_HAS_0.8.docx - */ - -/* Standard MMIO read, non-posted */ -#define SB_MRD_NP 0x00 -/* Standard MMIO write, non-posted */ -#define SB_MWR_NP 0x01 -/* Private register read, double-word addressing, non-posted */ -#define SB_CRRDDA_NP 0x06 -/* Private register write, double-word addressing, non-posted */ -#define SB_CRWRDA_NP 0x07 - -static void ping(void *info) -{ -} - -static void __vlv_punit_get(struct drm_i915_private *i915) -{ - iosf_mbi_punit_acquire(); - - /* - * Prevent the cpu from sleeping while we use this sideband, otherwise - * the punit may cause a machine hang. The issue appears to be isolated - * with changing the power state of the CPU package while changing - * the power state via the punit, and we have only observed it - * reliably on 4-core Baytail systems suggesting the issue is in the - * power delivery mechanism and likely to be be board/function - * specific. Hence we presume the workaround needs only be applied - * to the Valleyview P-unit and not all sideband communications. - */ - if (IS_VALLEYVIEW(i915)) { - cpu_latency_qos_update_request(&i915->sb_qos, 0); - on_each_cpu(ping, NULL, 1); - } -} - -static void __vlv_punit_put(struct drm_i915_private *i915) -{ - if (IS_VALLEYVIEW(i915)) - cpu_latency_qos_update_request(&i915->sb_qos, - PM_QOS_DEFAULT_VALUE); - - iosf_mbi_punit_release(); -} - -void vlv_iosf_sb_get(struct drm_i915_private *i915, unsigned long ports) -{ - if (ports & BIT(VLV_IOSF_SB_PUNIT)) - __vlv_punit_get(i915); - - mutex_lock(&i915->sb_lock); -} - -void vlv_iosf_sb_put(struct drm_i915_private *i915, unsigned long ports) -{ - mutex_unlock(&i915->sb_lock); - - if (ports & BIT(VLV_IOSF_SB_PUNIT)) - __vlv_punit_put(i915); -} - -static int vlv_sideband_rw(struct drm_i915_private *i915, - u32 devfn, u32 port, u32 opcode, - u32 addr, u32 *val) -{ - struct intel_uncore *uncore = &i915->uncore; - const bool is_read = (opcode == SB_MRD_NP || opcode == SB_CRRDDA_NP); - int err; - - lockdep_assert_held(&i915->sb_lock); - if (port == IOSF_PORT_PUNIT) - iosf_mbi_assert_punit_acquired(); - - /* Flush the previous comms, just in case it failed last time. */ - if (intel_wait_for_register(uncore, - VLV_IOSF_DOORBELL_REQ, IOSF_SB_BUSY, 0, - 5)) { - drm_dbg(&i915->drm, "IOSF sideband idle wait (%s) timed out\n", - is_read ? "read" : "write"); - return -EAGAIN; - } - - preempt_disable(); - - intel_uncore_write_fw(uncore, VLV_IOSF_ADDR, addr); - intel_uncore_write_fw(uncore, VLV_IOSF_DATA, is_read ? 0 : *val); - intel_uncore_write_fw(uncore, VLV_IOSF_DOORBELL_REQ, - (devfn << IOSF_DEVFN_SHIFT) | - (opcode << IOSF_OPCODE_SHIFT) | - (port << IOSF_PORT_SHIFT) | - (0xf << IOSF_BYTE_ENABLES_SHIFT) | - (0 << IOSF_BAR_SHIFT) | - IOSF_SB_BUSY); - - if (__intel_wait_for_register_fw(uncore, - VLV_IOSF_DOORBELL_REQ, IOSF_SB_BUSY, 0, - 10000, 0, NULL) == 0) { - if (is_read) - *val = intel_uncore_read_fw(uncore, VLV_IOSF_DATA); - err = 0; - } else { - drm_dbg(&i915->drm, "IOSF sideband finish wait (%s) timed out\n", - is_read ? "read" : "write"); - err = -ETIMEDOUT; - } - - preempt_enable(); - - return err; -} - -u32 vlv_punit_read(struct drm_i915_private *i915, u32 addr) -{ - u32 val = 0; - - vlv_sideband_rw(i915, PCI_DEVFN(0, 0), IOSF_PORT_PUNIT, - SB_CRRDDA_NP, addr, &val); - - return val; -} - -int vlv_punit_write(struct drm_i915_private *i915, u32 addr, u32 val) -{ - return vlv_sideband_rw(i915, PCI_DEVFN(0, 0), IOSF_PORT_PUNIT, - SB_CRWRDA_NP, addr, &val); -} - -u32 vlv_bunit_read(struct drm_i915_private *i915, u32 reg) -{ - u32 val = 0; - - vlv_sideband_rw(i915, PCI_DEVFN(0, 0), IOSF_PORT_BUNIT, - SB_CRRDDA_NP, reg, &val); - - return val; -} - -void vlv_bunit_write(struct drm_i915_private *i915, u32 reg, u32 val) -{ - vlv_sideband_rw(i915, PCI_DEVFN(0, 0), IOSF_PORT_BUNIT, - SB_CRWRDA_NP, reg, &val); -} - -u32 vlv_nc_read(struct drm_i915_private *i915, u8 addr) -{ - u32 val = 0; - - vlv_sideband_rw(i915, PCI_DEVFN(0, 0), IOSF_PORT_NC, - SB_CRRDDA_NP, addr, &val); - - return val; -} - -u32 vlv_iosf_sb_read(struct drm_i915_private *i915, u8 port, u32 reg) -{ - u32 val = 0; - - vlv_sideband_rw(i915, PCI_DEVFN(0, 0), port, - SB_CRRDDA_NP, reg, &val); - - return val; -} - -void vlv_iosf_sb_write(struct drm_i915_private *i915, - u8 port, u32 reg, u32 val) -{ - vlv_sideband_rw(i915, PCI_DEVFN(0, 0), port, - SB_CRWRDA_NP, reg, &val); -} - -u32 vlv_cck_read(struct drm_i915_private *i915, u32 reg) -{ - u32 val = 0; - - vlv_sideband_rw(i915, PCI_DEVFN(0, 0), IOSF_PORT_CCK, - SB_CRRDDA_NP, reg, &val); - - return val; -} - -void vlv_cck_write(struct drm_i915_private *i915, u32 reg, u32 val) -{ - vlv_sideband_rw(i915, PCI_DEVFN(0, 0), IOSF_PORT_CCK, - SB_CRWRDA_NP, reg, &val); -} - -u32 vlv_ccu_read(struct drm_i915_private *i915, u32 reg) -{ - u32 val = 0; - - vlv_sideband_rw(i915, PCI_DEVFN(0, 0), IOSF_PORT_CCU, - SB_CRRDDA_NP, reg, &val); - - return val; -} - -void vlv_ccu_write(struct drm_i915_private *i915, u32 reg, u32 val) -{ - vlv_sideband_rw(i915, PCI_DEVFN(0, 0), IOSF_PORT_CCU, - SB_CRWRDA_NP, reg, &val); -} - -static u32 vlv_dpio_phy_iosf_port(struct drm_i915_private *i915, enum dpio_phy phy) -{ - /* - * IOSF_PORT_DPIO: VLV x2 PHY (DP/HDMI B and C), CHV x1 PHY (DP/HDMI D) - * IOSF_PORT_DPIO_2: CHV x2 PHY (DP/HDMI B and C) - */ - if (IS_CHERRYVIEW(i915)) - return phy == DPIO_PHY0 ? IOSF_PORT_DPIO_2 : IOSF_PORT_DPIO; - else - return IOSF_PORT_DPIO; -} - -u32 vlv_dpio_read(struct drm_i915_private *i915, enum pipe pipe, int reg) -{ - u32 port = vlv_dpio_phy_iosf_port(i915, DPIO_PHY(pipe)); - u32 val = 0; - - vlv_sideband_rw(i915, DPIO_DEVFN, port, SB_MRD_NP, reg, &val); - - /* - * FIXME: There might be some registers where all 1's is a valid value, - * so ideally we should check the register offset instead... - */ - drm_WARN(&i915->drm, val == 0xffffffff, - "DPIO read pipe %c reg 0x%x == 0x%x\n", - pipe_name(pipe), reg, val); - - return val; -} - -void vlv_dpio_write(struct drm_i915_private *i915, - enum pipe pipe, int reg, u32 val) -{ - u32 port = vlv_dpio_phy_iosf_port(i915, DPIO_PHY(pipe)); - - vlv_sideband_rw(i915, DPIO_DEVFN, port, SB_MWR_NP, reg, &val); -} - -u32 vlv_flisdsi_read(struct drm_i915_private *i915, u32 reg) -{ - u32 val = 0; - - vlv_sideband_rw(i915, DPIO_DEVFN, IOSF_PORT_FLISDSI, SB_CRRDDA_NP, - reg, &val); - return val; -} - -void vlv_flisdsi_write(struct drm_i915_private *i915, u32 reg, u32 val) -{ - vlv_sideband_rw(i915, DPIO_DEVFN, IOSF_PORT_FLISDSI, SB_CRWRDA_NP, - reg, &val); -} - -/* SBI access */ -static int intel_sbi_rw(struct drm_i915_private *i915, u16 reg, - enum intel_sbi_destination destination, - u32 *val, bool is_read) -{ - struct intel_uncore *uncore = &i915->uncore; - u32 cmd; - - lockdep_assert_held(&i915->sb_lock); - - if (intel_wait_for_register_fw(uncore, - SBI_CTL_STAT, SBI_BUSY, 0, - 100)) { - drm_err(&i915->drm, - "timeout waiting for SBI to become ready\n"); - return -EBUSY; - } - - intel_uncore_write_fw(uncore, SBI_ADDR, (u32)reg << 16); - intel_uncore_write_fw(uncore, SBI_DATA, is_read ? 0 : *val); - - if (destination == SBI_ICLK) - cmd = SBI_CTL_DEST_ICLK | SBI_CTL_OP_CRRD; - else - cmd = SBI_CTL_DEST_MPHY | SBI_CTL_OP_IORD; - if (!is_read) - cmd |= BIT(8); - intel_uncore_write_fw(uncore, SBI_CTL_STAT, cmd | SBI_BUSY); - - if (__intel_wait_for_register_fw(uncore, - SBI_CTL_STAT, SBI_BUSY, 0, - 100, 100, &cmd)) { - drm_err(&i915->drm, - "timeout waiting for SBI to complete read\n"); - return -ETIMEDOUT; - } - - if (cmd & SBI_RESPONSE_FAIL) { - drm_err(&i915->drm, "error during SBI read of reg %x\n", reg); - return -ENXIO; - } - - if (is_read) - *val = intel_uncore_read_fw(uncore, SBI_DATA); - - return 0; -} - -u32 intel_sbi_read(struct drm_i915_private *i915, u16 reg, - enum intel_sbi_destination destination) -{ - u32 result = 0; - - intel_sbi_rw(i915, reg, destination, &result, true); - - return result; -} - -void intel_sbi_write(struct drm_i915_private *i915, u16 reg, u32 value, - enum intel_sbi_destination destination) -{ - intel_sbi_rw(i915, reg, destination, &value, false); -} - -static int gen6_check_mailbox_status(u32 mbox) -{ - switch (mbox & GEN6_PCODE_ERROR_MASK) { - case GEN6_PCODE_SUCCESS: - return 0; - case GEN6_PCODE_UNIMPLEMENTED_CMD: - return -ENODEV; - case GEN6_PCODE_ILLEGAL_CMD: - return -ENXIO; - case GEN6_PCODE_MIN_FREQ_TABLE_GT_RATIO_OUT_OF_RANGE: - case GEN7_PCODE_MIN_FREQ_TABLE_GT_RATIO_OUT_OF_RANGE: - return -EOVERFLOW; - case GEN6_PCODE_TIMEOUT: - return -ETIMEDOUT; - default: - MISSING_CASE(mbox & GEN6_PCODE_ERROR_MASK); - return 0; - } -} - -static int gen7_check_mailbox_status(u32 mbox) -{ - switch (mbox & GEN6_PCODE_ERROR_MASK) { - case GEN6_PCODE_SUCCESS: - return 0; - case GEN6_PCODE_ILLEGAL_CMD: - return -ENXIO; - case GEN7_PCODE_TIMEOUT: - return -ETIMEDOUT; - case GEN7_PCODE_ILLEGAL_DATA: - return -EINVAL; - case GEN11_PCODE_ILLEGAL_SUBCOMMAND: - return -ENXIO; - case GEN11_PCODE_LOCKED: - return -EBUSY; - case GEN11_PCODE_REJECTED: - return -EACCES; - case GEN7_PCODE_MIN_FREQ_TABLE_GT_RATIO_OUT_OF_RANGE: - return -EOVERFLOW; - default: - MISSING_CASE(mbox & GEN6_PCODE_ERROR_MASK); - return 0; - } -} - -static int __sandybridge_pcode_rw(struct drm_i915_private *i915, - u32 mbox, u32 *val, u32 *val1, - int fast_timeout_us, - int slow_timeout_ms, - bool is_read) -{ - struct intel_uncore *uncore = &i915->uncore; - - lockdep_assert_held(&i915->sb_lock); - - /* - * GEN6_PCODE_* are outside of the forcewake domain, we can use - * intel_uncore_read/write_fw variants to reduce the amount of work - * required when reading/writing. - */ - - if (intel_uncore_read_fw(uncore, GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) - return -EAGAIN; - - intel_uncore_write_fw(uncore, GEN6_PCODE_DATA, *val); - intel_uncore_write_fw(uncore, GEN6_PCODE_DATA1, val1 ? *val1 : 0); - intel_uncore_write_fw(uncore, - GEN6_PCODE_MAILBOX, GEN6_PCODE_READY | mbox); - - if (__intel_wait_for_register_fw(uncore, - GEN6_PCODE_MAILBOX, - GEN6_PCODE_READY, 0, - fast_timeout_us, - slow_timeout_ms, - &mbox)) - return -ETIMEDOUT; - - if (is_read) - *val = intel_uncore_read_fw(uncore, GEN6_PCODE_DATA); - if (is_read && val1) - *val1 = intel_uncore_read_fw(uncore, GEN6_PCODE_DATA1); - - if (GRAPHICS_VER(i915) > 6) - return gen7_check_mailbox_status(mbox); - else - return gen6_check_mailbox_status(mbox); -} - -int sandybridge_pcode_read(struct drm_i915_private *i915, u32 mbox, - u32 *val, u32 *val1) -{ - int err; - - mutex_lock(&i915->sb_lock); - err = __sandybridge_pcode_rw(i915, mbox, val, val1, - 500, 20, - true); - mutex_unlock(&i915->sb_lock); - - if (err) { - drm_dbg(&i915->drm, - "warning: pcode (read from mbox %x) mailbox access failed for %ps: %d\n", - mbox, __builtin_return_address(0), err); - } - - return err; -} - -int sandybridge_pcode_write_timeout(struct drm_i915_private *i915, - u32 mbox, u32 val, - int fast_timeout_us, - int slow_timeout_ms) -{ - int err; - - mutex_lock(&i915->sb_lock); - err = __sandybridge_pcode_rw(i915, mbox, &val, NULL, - fast_timeout_us, slow_timeout_ms, - false); - mutex_unlock(&i915->sb_lock); - - if (err) { - drm_dbg(&i915->drm, - "warning: pcode (write of 0x%08x to mbox %x) mailbox access failed for %ps: %d\n", - val, mbox, __builtin_return_address(0), err); - } - - return err; -} - -static bool skl_pcode_try_request(struct drm_i915_private *i915, u32 mbox, - u32 request, u32 reply_mask, u32 reply, - u32 *status) -{ - *status = __sandybridge_pcode_rw(i915, mbox, &request, NULL, - 500, 0, - true); - - return *status || ((request & reply_mask) == reply); -} - -/** - * skl_pcode_request - send PCODE request until acknowledgment - * @i915: device private - * @mbox: PCODE mailbox ID the request is targeted for - * @request: request ID - * @reply_mask: mask used to check for request acknowledgment - * @reply: value used to check for request acknowledgment - * @timeout_base_ms: timeout for polling with preemption enabled - * - * Keep resending the @request to @mbox until PCODE acknowledges it, PCODE - * reports an error or an overall timeout of @timeout_base_ms+50 ms expires. - * The request is acknowledged once the PCODE reply dword equals @reply after - * applying @reply_mask. Polling is first attempted with preemption enabled - * for @timeout_base_ms and if this times out for another 50 ms with - * preemption disabled. - * - * Returns 0 on success, %-ETIMEDOUT in case of a timeout, <0 in case of some - * other error as reported by PCODE. - */ -int skl_pcode_request(struct drm_i915_private *i915, u32 mbox, u32 request, - u32 reply_mask, u32 reply, int timeout_base_ms) -{ - u32 status; - int ret; - - mutex_lock(&i915->sb_lock); - -#define COND \ - skl_pcode_try_request(i915, mbox, request, reply_mask, reply, &status) - - /* - * Prime the PCODE by doing a request first. Normally it guarantees - * that a subsequent request, at most @timeout_base_ms later, succeeds. - * _wait_for() doesn't guarantee when its passed condition is evaluated - * first, so send the first request explicitly. - */ - if (COND) { - ret = 0; - goto out; - } - ret = _wait_for(COND, timeout_base_ms * 1000, 10, 10); - if (!ret) - goto out; - - /* - * The above can time out if the number of requests was low (2 in the - * worst case) _and_ PCODE was busy for some reason even after a - * (queued) request and @timeout_base_ms delay. As a workaround retry - * the poll with preemption disabled to maximize the number of - * requests. Increase the timeout from @timeout_base_ms to 50ms to - * account for interrupts that could reduce the number of these - * requests, and for any quirks of the PCODE firmware that delays - * the request completion. - */ - drm_dbg_kms(&i915->drm, - "PCODE timeout, retrying with preemption disabled\n"); - drm_WARN_ON_ONCE(&i915->drm, timeout_base_ms > 3); - preempt_disable(); - ret = wait_for_atomic(COND, 50); - preempt_enable(); - -out: - mutex_unlock(&i915->sb_lock); - return ret ? ret : status; -#undef COND -} - -int intel_pcode_init(struct drm_i915_private *i915) -{ - int ret = 0; - - if (!IS_DGFX(i915)) - return ret; - - ret = skl_pcode_request(i915, DG1_PCODE_STATUS, - DG1_UNCORE_GET_INIT_STATUS, - DG1_UNCORE_INIT_STATUS_COMPLETE, - DG1_UNCORE_INIT_STATUS_COMPLETE, 180000); - - drm_dbg(&i915->drm, "PCODE init status %d\n", ret); - - if (ret) - drm_err(&i915->drm, "Pcode did not report uncore initialization completion!\n"); - - return ret; -} diff --git a/drivers/gpu/drm/i915/intel_uncore.c b/drivers/gpu/drm/i915/intel_uncore.c index 6b38bc2811c1..e072054adac5 100644 --- a/drivers/gpu/drm/i915/intel_uncore.c +++ b/drivers/gpu/drm/i915/intel_uncore.c @@ -36,6 +36,12 @@ #define __raw_posting_read(...) ((void)__raw_uncore_read32(__VA_ARGS__)) +static void +fw_domains_get(struct intel_uncore *uncore, enum forcewake_domains fw_domains) +{ + uncore->fw_get_funcs->force_wake_get(uncore, fw_domains); +} + void intel_uncore_mmio_debug_init_early(struct intel_uncore_mmio_debug *mmio_debug) { @@ -64,7 +70,7 @@ static void mmio_debug_resume(struct intel_uncore_mmio_debug *mmio_debug) static const char * const forcewake_domain_names[] = { "render", - "blitter", + "gt", "media", "vdbox0", "vdbox1", @@ -248,7 +254,7 @@ fw_domain_put(const struct intel_uncore_forcewake_domain *d) } static void -fw_domains_get(struct intel_uncore *uncore, enum forcewake_domains fw_domains) +fw_domains_get_normal(struct intel_uncore *uncore, enum forcewake_domains fw_domains) { struct intel_uncore_forcewake_domain *d; unsigned int tmp; @@ -340,7 +346,7 @@ static void __gen6_gt_wait_for_thread_c0(struct intel_uncore *uncore) static void fw_domains_get_with_thread_status(struct intel_uncore *uncore, enum forcewake_domains fw_domains) { - fw_domains_get(uncore, fw_domains); + fw_domains_get_normal(uncore, fw_domains); /* WaRsForcewakeWaitTC0:snb,ivb,hsw,bdw,vlv */ __gen6_gt_wait_for_thread_c0(uncore); @@ -396,7 +402,7 @@ intel_uncore_fw_release_timer(struct hrtimer *timer) GEM_BUG_ON(!domain->wake_count); if (--domain->wake_count == 0) - uncore->funcs.force_wake_put(uncore, domain->mask); + fw_domains_put(uncore, domain->mask); spin_unlock_irqrestore(&uncore->lock, irqflags); @@ -454,7 +460,7 @@ intel_uncore_forcewake_reset(struct intel_uncore *uncore) fw = uncore->fw_domains_active; if (fw) - uncore->funcs.force_wake_put(uncore, fw); + fw_domains_put(uncore, fw); fw_domains_reset(uncore, uncore->fw_domains); assert_forcewakes_inactive(uncore); @@ -562,7 +568,7 @@ static void forcewake_early_sanitize(struct intel_uncore *uncore, intel_uncore_forcewake_reset(uncore); if (restore_forcewake) { spin_lock_irq(&uncore->lock); - uncore->funcs.force_wake_get(uncore, restore_forcewake); + fw_domains_get(uncore, restore_forcewake); if (intel_uncore_has_fifo(uncore)) uncore->fifo_count = fifo_free_entries(uncore); @@ -623,7 +629,7 @@ static void __intel_uncore_forcewake_get(struct intel_uncore *uncore, } if (fw_domains) - uncore->funcs.force_wake_get(uncore, fw_domains); + fw_domains_get(uncore, fw_domains); } /** @@ -644,7 +650,7 @@ void intel_uncore_forcewake_get(struct intel_uncore *uncore, { unsigned long irqflags; - if (!uncore->funcs.force_wake_get) + if (!uncore->fw_get_funcs) return; assert_rpm_wakelock_held(uncore->rpm); @@ -711,7 +717,7 @@ void intel_uncore_forcewake_get__locked(struct intel_uncore *uncore, { lockdep_assert_held(&uncore->lock); - if (!uncore->funcs.force_wake_get) + if (!uncore->fw_get_funcs) return; __intel_uncore_forcewake_get(uncore, fw_domains); @@ -733,7 +739,7 @@ static void __intel_uncore_forcewake_put(struct intel_uncore *uncore, continue; } - uncore->funcs.force_wake_put(uncore, domain->mask); + fw_domains_put(uncore, domain->mask); } } @@ -750,7 +756,7 @@ void intel_uncore_forcewake_put(struct intel_uncore *uncore, { unsigned long irqflags; - if (!uncore->funcs.force_wake_put) + if (!uncore->fw_get_funcs) return; spin_lock_irqsave(&uncore->lock, irqflags); @@ -769,7 +775,7 @@ void intel_uncore_forcewake_flush(struct intel_uncore *uncore, struct intel_uncore_forcewake_domain *domain; unsigned int tmp; - if (!uncore->funcs.force_wake_put) + if (!uncore->fw_get_funcs) return; fw_domains &= uncore->fw_domains; @@ -793,7 +799,7 @@ void intel_uncore_forcewake_put__locked(struct intel_uncore *uncore, { lockdep_assert_held(&uncore->lock); - if (!uncore->funcs.force_wake_put) + if (!uncore->fw_get_funcs) return; __intel_uncore_forcewake_put(uncore, fw_domains); @@ -801,7 +807,7 @@ void intel_uncore_forcewake_put__locked(struct intel_uncore *uncore, void assert_forcewakes_inactive(struct intel_uncore *uncore) { - if (!uncore->funcs.force_wake_get) + if (!uncore->fw_get_funcs) return; drm_WARN(&uncore->i915->drm, uncore->fw_domains_active, @@ -818,7 +824,7 @@ void assert_forcewakes_active(struct intel_uncore *uncore, if (!IS_ENABLED(CONFIG_DRM_I915_DEBUG_RUNTIME_PM)) return; - if (!uncore->funcs.force_wake_get) + if (!uncore->fw_get_funcs) return; spin_lock_irq(&uncore->lock); @@ -851,16 +857,9 @@ void assert_forcewakes_active(struct intel_uncore *uncore, } /* We give fast paths for the really cool registers */ -#define NEEDS_FORCE_WAKE(reg) ((reg) < 0x40000) - -#define __gen6_reg_read_fw_domains(uncore, offset) \ -({ \ - enum forcewake_domains __fwd; \ - if (NEEDS_FORCE_WAKE(offset)) \ - __fwd = FORCEWAKE_RENDER; \ - else \ - __fwd = 0; \ - __fwd; \ +#define NEEDS_FORCE_WAKE(reg) ({ \ + u32 __reg = (reg); \ + __reg < 0x40000 || __reg >= GEN11_BSD_RING_BASE; \ }) static int fw_range_cmp(u32 offset, const struct intel_forcewake_range *entry) @@ -942,125 +941,146 @@ static const struct intel_forcewake_range __vlv_fw_ranges[] = { __fwd; \ }) -#define __gen11_fwtable_reg_read_fw_domains(uncore, offset) \ - find_fw_domain(uncore, offset) - -#define __gen12_fwtable_reg_read_fw_domains(uncore, offset) \ - find_fw_domain(uncore, offset) - /* *Must* be sorted by offset! See intel_shadow_table_check(). */ -static const i915_reg_t gen8_shadowed_regs[] = { - RING_TAIL(RENDER_RING_BASE), /* 0x2000 (base) */ - GEN6_RPNSWREQ, /* 0xA008 */ - GEN6_RC_VIDEO_FREQ, /* 0xA00C */ - RING_TAIL(GEN6_BSD_RING_BASE), /* 0x12000 (base) */ - RING_TAIL(VEBOX_RING_BASE), /* 0x1a000 (base) */ - RING_TAIL(BLT_RING_BASE), /* 0x22000 (base) */ +static const struct i915_range gen8_shadowed_regs[] = { + { .start = 0x2030, .end = 0x2030 }, + { .start = 0xA008, .end = 0xA00C }, + { .start = 0x12030, .end = 0x12030 }, + { .start = 0x1a030, .end = 0x1a030 }, + { .start = 0x22030, .end = 0x22030 }, /* TODO: Other registers are not yet used */ }; -static const i915_reg_t gen11_shadowed_regs[] = { - RING_TAIL(RENDER_RING_BASE), /* 0x2000 (base) */ - RING_EXECLIST_CONTROL(RENDER_RING_BASE), /* 0x2550 */ - GEN6_RPNSWREQ, /* 0xA008 */ - GEN6_RC_VIDEO_FREQ, /* 0xA00C */ - RING_TAIL(BLT_RING_BASE), /* 0x22000 (base) */ - RING_EXECLIST_CONTROL(BLT_RING_BASE), /* 0x22550 */ - RING_TAIL(GEN11_BSD_RING_BASE), /* 0x1C0000 (base) */ - RING_EXECLIST_CONTROL(GEN11_BSD_RING_BASE), /* 0x1C0550 */ - RING_TAIL(GEN11_BSD2_RING_BASE), /* 0x1C4000 (base) */ - RING_EXECLIST_CONTROL(GEN11_BSD2_RING_BASE), /* 0x1C4550 */ - RING_TAIL(GEN11_VEBOX_RING_BASE), /* 0x1C8000 (base) */ - RING_EXECLIST_CONTROL(GEN11_VEBOX_RING_BASE), /* 0x1C8550 */ - RING_TAIL(GEN11_BSD3_RING_BASE), /* 0x1D0000 (base) */ - RING_EXECLIST_CONTROL(GEN11_BSD3_RING_BASE), /* 0x1D0550 */ - RING_TAIL(GEN11_BSD4_RING_BASE), /* 0x1D4000 (base) */ - RING_EXECLIST_CONTROL(GEN11_BSD4_RING_BASE), /* 0x1D4550 */ - RING_TAIL(GEN11_VEBOX2_RING_BASE), /* 0x1D8000 (base) */ - RING_EXECLIST_CONTROL(GEN11_VEBOX2_RING_BASE), /* 0x1D8550 */ - /* TODO: Other registers are not yet used */ +static const struct i915_range gen11_shadowed_regs[] = { + { .start = 0x2030, .end = 0x2030 }, + { .start = 0x2550, .end = 0x2550 }, + { .start = 0xA008, .end = 0xA00C }, + { .start = 0x22030, .end = 0x22030 }, + { .start = 0x22230, .end = 0x22230 }, + { .start = 0x22510, .end = 0x22550 }, + { .start = 0x1C0030, .end = 0x1C0030 }, + { .start = 0x1C0230, .end = 0x1C0230 }, + { .start = 0x1C0510, .end = 0x1C0550 }, + { .start = 0x1C4030, .end = 0x1C4030 }, + { .start = 0x1C4230, .end = 0x1C4230 }, + { .start = 0x1C4510, .end = 0x1C4550 }, + { .start = 0x1C8030, .end = 0x1C8030 }, + { .start = 0x1C8230, .end = 0x1C8230 }, + { .start = 0x1C8510, .end = 0x1C8550 }, + { .start = 0x1D0030, .end = 0x1D0030 }, + { .start = 0x1D0230, .end = 0x1D0230 }, + { .start = 0x1D0510, .end = 0x1D0550 }, + { .start = 0x1D4030, .end = 0x1D4030 }, + { .start = 0x1D4230, .end = 0x1D4230 }, + { .start = 0x1D4510, .end = 0x1D4550 }, + { .start = 0x1D8030, .end = 0x1D8030 }, + { .start = 0x1D8230, .end = 0x1D8230 }, + { .start = 0x1D8510, .end = 0x1D8550 }, }; -static const i915_reg_t gen12_shadowed_regs[] = { - RING_TAIL(RENDER_RING_BASE), /* 0x2000 (base) */ - RING_EXECLIST_CONTROL(RENDER_RING_BASE), /* 0x2550 */ - GEN6_RPNSWREQ, /* 0xA008 */ - GEN6_RC_VIDEO_FREQ, /* 0xA00C */ - RING_TAIL(BLT_RING_BASE), /* 0x22000 (base) */ - RING_EXECLIST_CONTROL(BLT_RING_BASE), /* 0x22550 */ - RING_TAIL(GEN11_BSD_RING_BASE), /* 0x1C0000 (base) */ - RING_EXECLIST_CONTROL(GEN11_BSD_RING_BASE), /* 0x1C0550 */ - RING_TAIL(GEN11_BSD2_RING_BASE), /* 0x1C4000 (base) */ - RING_EXECLIST_CONTROL(GEN11_BSD2_RING_BASE), /* 0x1C4550 */ - RING_TAIL(GEN11_VEBOX_RING_BASE), /* 0x1C8000 (base) */ - RING_EXECLIST_CONTROL(GEN11_VEBOX_RING_BASE), /* 0x1C8550 */ - RING_TAIL(GEN11_BSD3_RING_BASE), /* 0x1D0000 (base) */ - RING_EXECLIST_CONTROL(GEN11_BSD3_RING_BASE), /* 0x1D0550 */ - RING_TAIL(GEN11_BSD4_RING_BASE), /* 0x1D4000 (base) */ - RING_EXECLIST_CONTROL(GEN11_BSD4_RING_BASE), /* 0x1D4550 */ - RING_TAIL(GEN11_VEBOX2_RING_BASE), /* 0x1D8000 (base) */ - RING_EXECLIST_CONTROL(GEN11_VEBOX2_RING_BASE), /* 0x1D8550 */ - /* TODO: Other registers are not yet used */ +static const struct i915_range gen12_shadowed_regs[] = { + { .start = 0x2030, .end = 0x2030 }, + { .start = 0x2510, .end = 0x2550 }, + { .start = 0xA008, .end = 0xA00C }, + { .start = 0xA188, .end = 0xA188 }, + { .start = 0xA278, .end = 0xA278 }, + { .start = 0xA540, .end = 0xA56C }, + { .start = 0xC4C8, .end = 0xC4C8 }, + { .start = 0xC4D4, .end = 0xC4D4 }, + { .start = 0xC600, .end = 0xC600 }, + { .start = 0x22030, .end = 0x22030 }, + { .start = 0x22510, .end = 0x22550 }, + { .start = 0x1C0030, .end = 0x1C0030 }, + { .start = 0x1C0510, .end = 0x1C0550 }, + { .start = 0x1C4030, .end = 0x1C4030 }, + { .start = 0x1C4510, .end = 0x1C4550 }, + { .start = 0x1C8030, .end = 0x1C8030 }, + { .start = 0x1C8510, .end = 0x1C8550 }, + { .start = 0x1D0030, .end = 0x1D0030 }, + { .start = 0x1D0510, .end = 0x1D0550 }, + { .start = 0x1D4030, .end = 0x1D4030 }, + { .start = 0x1D4510, .end = 0x1D4550 }, + { .start = 0x1D8030, .end = 0x1D8030 }, + { .start = 0x1D8510, .end = 0x1D8550 }, + + /* + * The rest of these ranges are specific to Xe_HP and beyond, but + * are reserved/unused ranges on earlier gen12 platforms, so they can + * be safely added to the gen12 table. + */ + { .start = 0x1E0030, .end = 0x1E0030 }, + { .start = 0x1E0510, .end = 0x1E0550 }, + { .start = 0x1E4030, .end = 0x1E4030 }, + { .start = 0x1E4510, .end = 0x1E4550 }, + { .start = 0x1E8030, .end = 0x1E8030 }, + { .start = 0x1E8510, .end = 0x1E8550 }, + { .start = 0x1F0030, .end = 0x1F0030 }, + { .start = 0x1F0510, .end = 0x1F0550 }, + { .start = 0x1F4030, .end = 0x1F4030 }, + { .start = 0x1F4510, .end = 0x1F4550 }, + { .start = 0x1F8030, .end = 0x1F8030 }, + { .start = 0x1F8510, .end = 0x1F8550 }, }; -static const i915_reg_t xehp_shadowed_regs[] = { - RING_TAIL(RENDER_RING_BASE), /* 0x2000 (base) */ - RING_EXECLIST_CONTROL(RENDER_RING_BASE), /* 0x2550 */ - GEN6_RPNSWREQ, /* 0xA008 */ - GEN6_RC_VIDEO_FREQ, /* 0xA00C */ - RING_TAIL(BLT_RING_BASE), /* 0x22000 (base) */ - RING_EXECLIST_CONTROL(BLT_RING_BASE), /* 0x22550 */ - RING_TAIL(GEN11_BSD_RING_BASE), /* 0x1C0000 (base) */ - RING_EXECLIST_CONTROL(GEN11_BSD_RING_BASE), /* 0x1C0550 */ - RING_TAIL(GEN11_BSD2_RING_BASE), /* 0x1C4000 (base) */ - RING_EXECLIST_CONTROL(GEN11_BSD2_RING_BASE), /* 0x1C4550 */ - RING_TAIL(GEN11_VEBOX_RING_BASE), /* 0x1C8000 (base) */ - RING_EXECLIST_CONTROL(GEN11_VEBOX_RING_BASE), /* 0x1C8550 */ - RING_TAIL(GEN11_BSD3_RING_BASE), /* 0x1D0000 (base) */ - RING_EXECLIST_CONTROL(GEN11_BSD3_RING_BASE), /* 0x1D0550 */ - RING_TAIL(GEN11_BSD4_RING_BASE), /* 0x1D4000 (base) */ - RING_EXECLIST_CONTROL(GEN11_BSD4_RING_BASE), /* 0x1D4550 */ - RING_TAIL(GEN11_VEBOX2_RING_BASE), /* 0x1D8000 (base) */ - RING_EXECLIST_CONTROL(GEN11_VEBOX2_RING_BASE), /* 0x1D8550 */ - RING_TAIL(XEHP_BSD5_RING_BASE), /* 0x1E0000 (base) */ - RING_EXECLIST_CONTROL(XEHP_BSD5_RING_BASE), /* 0x1E0550 */ - RING_TAIL(XEHP_BSD6_RING_BASE), /* 0x1E4000 (base) */ - RING_EXECLIST_CONTROL(XEHP_BSD6_RING_BASE), /* 0x1E4550 */ - RING_TAIL(XEHP_VEBOX3_RING_BASE), /* 0x1E8000 (base) */ - RING_EXECLIST_CONTROL(XEHP_VEBOX3_RING_BASE), /* 0x1E8550 */ - RING_TAIL(XEHP_BSD7_RING_BASE), /* 0x1F0000 (base) */ - RING_EXECLIST_CONTROL(XEHP_BSD7_RING_BASE), /* 0x1F0550 */ - RING_TAIL(XEHP_BSD8_RING_BASE), /* 0x1F4000 (base) */ - RING_EXECLIST_CONTROL(XEHP_BSD8_RING_BASE), /* 0x1F4550 */ - RING_TAIL(XEHP_VEBOX4_RING_BASE), /* 0x1F8000 (base) */ - RING_EXECLIST_CONTROL(XEHP_VEBOX4_RING_BASE), /* 0x1F8550 */ - /* TODO: Other registers are not yet used */ +static const struct i915_range dg2_shadowed_regs[] = { + { .start = 0x2030, .end = 0x2030 }, + { .start = 0x2510, .end = 0x2550 }, + { .start = 0xA008, .end = 0xA00C }, + { .start = 0xA188, .end = 0xA188 }, + { .start = 0xA278, .end = 0xA278 }, + { .start = 0xA540, .end = 0xA56C }, + { .start = 0xC4C8, .end = 0xC4C8 }, + { .start = 0xC4E0, .end = 0xC4E0 }, + { .start = 0xC600, .end = 0xC600 }, + { .start = 0xC658, .end = 0xC658 }, + { .start = 0x22030, .end = 0x22030 }, + { .start = 0x22510, .end = 0x22550 }, + { .start = 0x1C0030, .end = 0x1C0030 }, + { .start = 0x1C0510, .end = 0x1C0550 }, + { .start = 0x1C4030, .end = 0x1C4030 }, + { .start = 0x1C4510, .end = 0x1C4550 }, + { .start = 0x1C8030, .end = 0x1C8030 }, + { .start = 0x1C8510, .end = 0x1C8550 }, + { .start = 0x1D0030, .end = 0x1D0030 }, + { .start = 0x1D0510, .end = 0x1D0550 }, + { .start = 0x1D4030, .end = 0x1D4030 }, + { .start = 0x1D4510, .end = 0x1D4550 }, + { .start = 0x1D8030, .end = 0x1D8030 }, + { .start = 0x1D8510, .end = 0x1D8550 }, + { .start = 0x1E0030, .end = 0x1E0030 }, + { .start = 0x1E0510, .end = 0x1E0550 }, + { .start = 0x1E4030, .end = 0x1E4030 }, + { .start = 0x1E4510, .end = 0x1E4550 }, + { .start = 0x1E8030, .end = 0x1E8030 }, + { .start = 0x1E8510, .end = 0x1E8550 }, + { .start = 0x1F0030, .end = 0x1F0030 }, + { .start = 0x1F0510, .end = 0x1F0550 }, + { .start = 0x1F4030, .end = 0x1F4030 }, + { .start = 0x1F4510, .end = 0x1F4550 }, + { .start = 0x1F8030, .end = 0x1F8030 }, + { .start = 0x1F8510, .end = 0x1F8550 }, }; -static int mmio_reg_cmp(u32 key, const i915_reg_t *reg) +static int mmio_range_cmp(u32 key, const struct i915_range *range) { - u32 offset = i915_mmio_reg_offset(*reg); - - if (key < offset) + if (key < range->start) return -1; - else if (key > offset) + else if (key > range->end) return 1; else return 0; } -#define __is_X_shadowed(x) \ -static bool is_##x##_shadowed(u32 offset) \ -{ \ - const i915_reg_t *regs = x##_shadowed_regs; \ - return BSEARCH(offset, regs, ARRAY_SIZE(x##_shadowed_regs), \ - mmio_reg_cmp); \ -} +static bool is_shadowed(struct intel_uncore *uncore, u32 offset) +{ + if (drm_WARN_ON(&uncore->i915->drm, !uncore->shadowed_reg_table)) + return false; -__is_X_shadowed(gen8) -__is_X_shadowed(gen11) -__is_X_shadowed(gen12) -__is_X_shadowed(xehp) + return BSEARCH(offset, + uncore->shadowed_reg_table, + uncore->shadowed_reg_table_entries, + mmio_range_cmp); +} static enum forcewake_domains gen6_reg_write_fw_domains(struct intel_uncore *uncore, i915_reg_t reg) @@ -1068,15 +1088,9 @@ gen6_reg_write_fw_domains(struct intel_uncore *uncore, i915_reg_t reg) return FORCEWAKE_RENDER; } -#define __gen8_reg_write_fw_domains(uncore, offset) \ -({ \ - enum forcewake_domains __fwd; \ - if (NEEDS_FORCE_WAKE(offset) && !is_gen8_shadowed(offset)) \ - __fwd = FORCEWAKE_RENDER; \ - else \ - __fwd = 0; \ - __fwd; \ -}) +static const struct intel_forcewake_range __gen6_fw_ranges[] = { + GEN_FW_RANGE(0x0, 0x3ffff, FORCEWAKE_RENDER), +}; /* *Must* be sorted by offset ranges! See intel_fw_table_check(). */ static const struct intel_forcewake_range __chv_fw_ranges[] = { @@ -1101,34 +1115,8 @@ static const struct intel_forcewake_range __chv_fw_ranges[] = { #define __fwtable_reg_write_fw_domains(uncore, offset) \ ({ \ enum forcewake_domains __fwd = 0; \ - if (NEEDS_FORCE_WAKE((offset)) && !is_gen8_shadowed(offset)) \ - __fwd = find_fw_domain(uncore, offset); \ - __fwd; \ -}) - -#define __gen11_fwtable_reg_write_fw_domains(uncore, offset) \ -({ \ - enum forcewake_domains __fwd = 0; \ const u32 __offset = (offset); \ - if (!is_gen11_shadowed(__offset)) \ - __fwd = find_fw_domain(uncore, __offset); \ - __fwd; \ -}) - -#define __gen12_fwtable_reg_write_fw_domains(uncore, offset) \ -({ \ - enum forcewake_domains __fwd = 0; \ - const u32 __offset = (offset); \ - if (!is_gen12_shadowed(__offset)) \ - __fwd = find_fw_domain(uncore, __offset); \ - __fwd; \ -}) - -#define __xehp_fwtable_reg_write_fw_domains(uncore, offset) \ -({ \ - enum forcewake_domains __fwd = 0; \ - const u32 __offset = (offset); \ - if (!is_xehp_shadowed(__offset)) \ + if (NEEDS_FORCE_WAKE((__offset)) && !is_shadowed(uncore, __offset)) \ __fwd = find_fw_domain(uncore, __offset); \ __fwd; \ }) @@ -1605,7 +1593,7 @@ static noinline void ___force_wake_auto(struct intel_uncore *uncore, for_each_fw_domain_masked(domain, fw_domains, uncore, tmp) fw_domain_arm_timer(domain); - uncore->funcs.force_wake_get(uncore, fw_domains); + fw_domains_get(uncore, fw_domains); } static inline void __force_wake_auto(struct intel_uncore *uncore, @@ -1621,35 +1609,30 @@ static inline void __force_wake_auto(struct intel_uncore *uncore, ___force_wake_auto(uncore, fw_domains); } -#define __gen_read(func, x) \ +#define __gen_fwtable_read(x) \ static u##x \ -func##_read##x(struct intel_uncore *uncore, i915_reg_t reg, bool trace) { \ +fwtable_read##x(struct intel_uncore *uncore, i915_reg_t reg, bool trace) \ +{ \ enum forcewake_domains fw_engine; \ GEN6_READ_HEADER(x); \ - fw_engine = __##func##_reg_read_fw_domains(uncore, offset); \ + fw_engine = __fwtable_reg_read_fw_domains(uncore, offset); \ if (fw_engine) \ __force_wake_auto(uncore, fw_engine); \ val = __raw_uncore_read##x(uncore, reg); \ GEN6_READ_FOOTER; \ } -#define __gen_reg_read_funcs(func) \ -static enum forcewake_domains \ -func##_reg_read_fw_domains(struct intel_uncore *uncore, i915_reg_t reg) { \ - return __##func##_reg_read_fw_domains(uncore, i915_mmio_reg_offset(reg)); \ -} \ -\ -__gen_read(func, 8) \ -__gen_read(func, 16) \ -__gen_read(func, 32) \ -__gen_read(func, 64) +static enum forcewake_domains +fwtable_reg_read_fw_domains(struct intel_uncore *uncore, i915_reg_t reg) { + return __fwtable_reg_read_fw_domains(uncore, i915_mmio_reg_offset(reg)); +} -__gen_reg_read_funcs(gen12_fwtable); -__gen_reg_read_funcs(gen11_fwtable); -__gen_reg_read_funcs(fwtable); -__gen_reg_read_funcs(gen6); +__gen_fwtable_read(8) +__gen_fwtable_read(16) +__gen_fwtable_read(32) +__gen_fwtable_read(64) -#undef __gen_reg_read_funcs +#undef __gen_fwtable_read #undef GEN6_READ_FOOTER #undef GEN6_READ_HEADER @@ -1714,35 +1697,29 @@ __gen6_write(8) __gen6_write(16) __gen6_write(32) -#define __gen_write(func, x) \ +#define __gen_fwtable_write(x) \ static void \ -func##_write##x(struct intel_uncore *uncore, i915_reg_t reg, u##x val, bool trace) { \ +fwtable_write##x(struct intel_uncore *uncore, i915_reg_t reg, u##x val, bool trace) { \ enum forcewake_domains fw_engine; \ GEN6_WRITE_HEADER; \ - fw_engine = __##func##_reg_write_fw_domains(uncore, offset); \ + fw_engine = __fwtable_reg_write_fw_domains(uncore, offset); \ if (fw_engine) \ __force_wake_auto(uncore, fw_engine); \ __raw_uncore_write##x(uncore, reg, val); \ GEN6_WRITE_FOOTER; \ } -#define __gen_reg_write_funcs(func) \ -static enum forcewake_domains \ -func##_reg_write_fw_domains(struct intel_uncore *uncore, i915_reg_t reg) { \ - return __##func##_reg_write_fw_domains(uncore, i915_mmio_reg_offset(reg)); \ -} \ -\ -__gen_write(func, 8) \ -__gen_write(func, 16) \ -__gen_write(func, 32) +static enum forcewake_domains +fwtable_reg_write_fw_domains(struct intel_uncore *uncore, i915_reg_t reg) +{ + return __fwtable_reg_write_fw_domains(uncore, i915_mmio_reg_offset(reg)); +} -__gen_reg_write_funcs(xehp_fwtable); -__gen_reg_write_funcs(gen12_fwtable); -__gen_reg_write_funcs(gen11_fwtable); -__gen_reg_write_funcs(fwtable); -__gen_reg_write_funcs(gen8); +__gen_fwtable_write(8) +__gen_fwtable_write(16) +__gen_fwtable_write(32) -#undef __gen_reg_write_funcs +#undef __gen_fwtable_write #undef GEN6_WRITE_FOOTER #undef GEN6_WRITE_HEADER @@ -1866,6 +1843,18 @@ static void intel_uncore_fw_domains_fini(struct intel_uncore *uncore) fw_domain_fini(uncore, d->id); } +static const struct intel_uncore_fw_get uncore_get_fallback = { + .force_wake_get = fw_domains_get_with_fallback +}; + +static const struct intel_uncore_fw_get uncore_get_normal = { + .force_wake_get = fw_domains_get_normal, +}; + +static const struct intel_uncore_fw_get uncore_get_thread_status = { + .force_wake_get = fw_domains_get_with_thread_status +}; + static int intel_uncore_fw_domains_init(struct intel_uncore *uncore) { struct drm_i915_private *i915 = uncore->i915; @@ -1881,8 +1870,7 @@ static int intel_uncore_fw_domains_init(struct intel_uncore *uncore) intel_engine_mask_t emask = INTEL_INFO(i915)->platform_engine_mask; int i; - uncore->funcs.force_wake_get = fw_domains_get_with_fallback; - uncore->funcs.force_wake_put = fw_domains_put; + uncore->fw_get_funcs = &uncore_get_fallback; fw_domain_init(uncore, FW_DOMAIN_ID_RENDER, FORCEWAKE_RENDER_GEN9, FORCEWAKE_ACK_RENDER_GEN9); @@ -1907,8 +1895,7 @@ static int intel_uncore_fw_domains_init(struct intel_uncore *uncore) FORCEWAKE_ACK_MEDIA_VEBOX_GEN11(i)); } } else if (IS_GRAPHICS_VER(i915, 9, 10)) { - uncore->funcs.force_wake_get = fw_domains_get_with_fallback; - uncore->funcs.force_wake_put = fw_domains_put; + uncore->fw_get_funcs = &uncore_get_fallback; fw_domain_init(uncore, FW_DOMAIN_ID_RENDER, FORCEWAKE_RENDER_GEN9, FORCEWAKE_ACK_RENDER_GEN9); @@ -1918,16 +1905,13 @@ static int intel_uncore_fw_domains_init(struct intel_uncore *uncore) fw_domain_init(uncore, FW_DOMAIN_ID_MEDIA, FORCEWAKE_MEDIA_GEN9, FORCEWAKE_ACK_MEDIA_GEN9); } else if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915)) { - uncore->funcs.force_wake_get = fw_domains_get; - uncore->funcs.force_wake_put = fw_domains_put; + uncore->fw_get_funcs = &uncore_get_normal; fw_domain_init(uncore, FW_DOMAIN_ID_RENDER, FORCEWAKE_VLV, FORCEWAKE_ACK_VLV); fw_domain_init(uncore, FW_DOMAIN_ID_MEDIA, FORCEWAKE_MEDIA_VLV, FORCEWAKE_ACK_MEDIA_VLV); } else if (IS_HASWELL(i915) || IS_BROADWELL(i915)) { - uncore->funcs.force_wake_get = - fw_domains_get_with_thread_status; - uncore->funcs.force_wake_put = fw_domains_put; + uncore->fw_get_funcs = &uncore_get_thread_status; fw_domain_init(uncore, FW_DOMAIN_ID_RENDER, FORCEWAKE_MT, FORCEWAKE_ACK_HSW); } else if (IS_IVYBRIDGE(i915)) { @@ -1942,9 +1926,7 @@ static int intel_uncore_fw_domains_init(struct intel_uncore *uncore) * (correctly) interpreted by the test below as MT * forcewake being disabled. */ - uncore->funcs.force_wake_get = - fw_domains_get_with_thread_status; - uncore->funcs.force_wake_put = fw_domains_put; + uncore->fw_get_funcs = &uncore_get_thread_status; /* We need to init first for ECOBUS access and then * determine later if we want to reinit, in case of MT access is @@ -1975,9 +1957,7 @@ static int intel_uncore_fw_domains_init(struct intel_uncore *uncore) FORCEWAKE, FORCEWAKE_ACK); } } else if (GRAPHICS_VER(i915) == 6) { - uncore->funcs.force_wake_get = - fw_domains_get_with_thread_status; - uncore->funcs.force_wake_put = fw_domains_put; + uncore->fw_get_funcs = &uncore_get_thread_status; fw_domain_init(uncore, FW_DOMAIN_ID_RENDER, FORCEWAKE, FORCEWAKE_ACK); } @@ -2001,6 +1981,12 @@ out: (uncore)->fw_domains_table_entries = ARRAY_SIZE((d)); \ } +#define ASSIGN_SHADOW_TABLE(uncore, d) \ +{ \ + (uncore)->shadowed_reg_table = d; \ + (uncore)->shadowed_reg_table_entries = ARRAY_SIZE((d)); \ +} + static int i915_pmic_bus_access_notifier(struct notifier_block *nb, unsigned long action, void *data) { @@ -2111,40 +2097,42 @@ static int uncore_forcewake_init(struct intel_uncore *uncore) return ret; forcewake_early_sanitize(uncore, 0); + ASSIGN_READ_MMIO_VFUNCS(uncore, fwtable); + if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 55)) { ASSIGN_FW_DOMAINS_TABLE(uncore, __dg2_fw_ranges); - ASSIGN_WRITE_MMIO_VFUNCS(uncore, xehp_fwtable); - ASSIGN_READ_MMIO_VFUNCS(uncore, gen11_fwtable); + ASSIGN_SHADOW_TABLE(uncore, dg2_shadowed_regs); + ASSIGN_WRITE_MMIO_VFUNCS(uncore, fwtable); } else if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 50)) { ASSIGN_FW_DOMAINS_TABLE(uncore, __xehp_fw_ranges); - ASSIGN_WRITE_MMIO_VFUNCS(uncore, xehp_fwtable); - ASSIGN_READ_MMIO_VFUNCS(uncore, gen11_fwtable); + ASSIGN_SHADOW_TABLE(uncore, gen12_shadowed_regs); + ASSIGN_WRITE_MMIO_VFUNCS(uncore, fwtable); } else if (GRAPHICS_VER(i915) >= 12) { ASSIGN_FW_DOMAINS_TABLE(uncore, __gen12_fw_ranges); - ASSIGN_WRITE_MMIO_VFUNCS(uncore, gen12_fwtable); - ASSIGN_READ_MMIO_VFUNCS(uncore, gen12_fwtable); + ASSIGN_SHADOW_TABLE(uncore, gen12_shadowed_regs); + ASSIGN_WRITE_MMIO_VFUNCS(uncore, fwtable); } else if (GRAPHICS_VER(i915) == 11) { ASSIGN_FW_DOMAINS_TABLE(uncore, __gen11_fw_ranges); - ASSIGN_WRITE_MMIO_VFUNCS(uncore, gen11_fwtable); - ASSIGN_READ_MMIO_VFUNCS(uncore, gen11_fwtable); + ASSIGN_SHADOW_TABLE(uncore, gen11_shadowed_regs); + ASSIGN_WRITE_MMIO_VFUNCS(uncore, fwtable); } else if (IS_GRAPHICS_VER(i915, 9, 10)) { ASSIGN_FW_DOMAINS_TABLE(uncore, __gen9_fw_ranges); + ASSIGN_SHADOW_TABLE(uncore, gen8_shadowed_regs); ASSIGN_WRITE_MMIO_VFUNCS(uncore, fwtable); - ASSIGN_READ_MMIO_VFUNCS(uncore, fwtable); } else if (IS_CHERRYVIEW(i915)) { ASSIGN_FW_DOMAINS_TABLE(uncore, __chv_fw_ranges); + ASSIGN_SHADOW_TABLE(uncore, gen8_shadowed_regs); ASSIGN_WRITE_MMIO_VFUNCS(uncore, fwtable); - ASSIGN_READ_MMIO_VFUNCS(uncore, fwtable); } else if (GRAPHICS_VER(i915) == 8) { - ASSIGN_WRITE_MMIO_VFUNCS(uncore, gen8); - ASSIGN_READ_MMIO_VFUNCS(uncore, gen6); + ASSIGN_FW_DOMAINS_TABLE(uncore, __gen6_fw_ranges); + ASSIGN_SHADOW_TABLE(uncore, gen8_shadowed_regs); + ASSIGN_WRITE_MMIO_VFUNCS(uncore, fwtable); } else if (IS_VALLEYVIEW(i915)) { ASSIGN_FW_DOMAINS_TABLE(uncore, __vlv_fw_ranges); ASSIGN_WRITE_MMIO_VFUNCS(uncore, gen6); - ASSIGN_READ_MMIO_VFUNCS(uncore, fwtable); } else if (IS_GRAPHICS_VER(i915, 6, 7)) { + ASSIGN_FW_DOMAINS_TABLE(uncore, __gen6_fw_ranges); ASSIGN_WRITE_MMIO_VFUNCS(uncore, gen6); - ASSIGN_READ_MMIO_VFUNCS(uncore, gen6); } uncore->pmic_bus_access_nb.notifier_call = i915_pmic_bus_access_notifier; @@ -2186,8 +2174,7 @@ int intel_uncore_init_mmio(struct intel_uncore *uncore) } /* make sure fw funcs are set if and only if we have fw*/ - GEM_BUG_ON(intel_uncore_has_forcewake(uncore) != !!uncore->funcs.force_wake_get); - GEM_BUG_ON(intel_uncore_has_forcewake(uncore) != !!uncore->funcs.force_wake_put); + GEM_BUG_ON(intel_uncore_has_forcewake(uncore) != !!uncore->fw_get_funcs); GEM_BUG_ON(intel_uncore_has_forcewake(uncore) != !!uncore->funcs.read_fw_domains); GEM_BUG_ON(intel_uncore_has_forcewake(uncore) != !!uncore->funcs.write_fw_domains); diff --git a/drivers/gpu/drm/i915/intel_uncore.h b/drivers/gpu/drm/i915/intel_uncore.h index 3c0b0a8b5250..3248e4e2c540 100644 --- a/drivers/gpu/drm/i915/intel_uncore.h +++ b/drivers/gpu/drm/i915/intel_uncore.h @@ -84,12 +84,12 @@ enum forcewake_domains { FORCEWAKE_ALL = BIT(FW_DOMAIN_ID_COUNT) - 1, }; -struct intel_uncore_funcs { +struct intel_uncore_fw_get { void (*force_wake_get)(struct intel_uncore *uncore, enum forcewake_domains domains); - void (*force_wake_put)(struct intel_uncore *uncore, - enum forcewake_domains domains); +}; +struct intel_uncore_funcs { enum forcewake_domains (*read_fw_domains)(struct intel_uncore *uncore, i915_reg_t r); enum forcewake_domains (*write_fw_domains)(struct intel_uncore *uncore, @@ -119,6 +119,12 @@ struct intel_forcewake_range { enum forcewake_domains domains; }; +/* Other register ranges (e.g., shadow tables, MCR tables, etc.) */ +struct i915_range { + u32 start; + u32 end; +}; + struct intel_uncore { void __iomem *regs; @@ -136,7 +142,15 @@ struct intel_uncore { const struct intel_forcewake_range *fw_domains_table; unsigned int fw_domains_table_entries; + /* + * Shadowed registers are special cases where we can safely write + * to the register *without* grabbing forcewake. + */ + const struct i915_range *shadowed_reg_table; + unsigned int shadowed_reg_table_entries; + struct notifier_block pmic_bus_access_nb; + const struct intel_uncore_fw_get *fw_get_funcs; struct intel_uncore_funcs funcs; unsigned int fifo_count; diff --git a/drivers/gpu/drm/i915/intel_wakeref.h b/drivers/gpu/drm/i915/intel_wakeref.h index 545c8f277c46..4f4c2e15e736 100644 --- a/drivers/gpu/drm/i915/intel_wakeref.h +++ b/drivers/gpu/drm/i915/intel_wakeref.h @@ -123,6 +123,12 @@ enum { __INTEL_WAKEREF_PUT_LAST_BIT__ }; +static inline void +intel_wakeref_might_get(struct intel_wakeref *wf) +{ + might_lock(&wf->mutex); +} + /** * intel_wakeref_put_flags: Release the wakeref * @wf: the wakeref @@ -170,6 +176,12 @@ intel_wakeref_put_delay(struct intel_wakeref *wf, unsigned long delay) FIELD_PREP(INTEL_WAKEREF_PUT_DELAY, delay)); } +static inline void +intel_wakeref_might_put(struct intel_wakeref *wf) +{ + might_lock(&wf->mutex); +} + /** * intel_wakeref_lock: Lock the wakeref (mutex) * @wf: the wakeref diff --git a/drivers/gpu/drm/i915/pxp/intel_pxp.c b/drivers/gpu/drm/i915/pxp/intel_pxp.c new file mode 100644 index 000000000000..e2314ad9546d --- /dev/null +++ b/drivers/gpu/drm/i915/pxp/intel_pxp.c @@ -0,0 +1,299 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright(c) 2020 Intel Corporation. + */ +#include <linux/workqueue.h> +#include "intel_pxp.h" +#include "intel_pxp_irq.h" +#include "intel_pxp_session.h" +#include "intel_pxp_tee.h" +#include "gem/i915_gem_context.h" +#include "gt/intel_context.h" +#include "i915_drv.h" + +/** + * DOC: PXP + * + * PXP (Protected Xe Path) is a feature available in Gen12 and newer platforms. + * It allows execution and flip to display of protected (i.e. encrypted) + * objects. The SW support is enabled via the CONFIG_DRM_I915_PXP kconfig. + * + * Objects can opt-in to PXP encryption at creation time via the + * I915_GEM_CREATE_EXT_PROTECTED_CONTENT create_ext flag. For objects to be + * correctly protected they must be used in conjunction with a context created + * with the I915_CONTEXT_PARAM_PROTECTED_CONTENT flag. See the documentation + * of those two uapi flags for details and restrictions. + * + * Protected objects are tied to a pxp session; currently we only support one + * session, which i915 manages and whose index is available in the uapi + * (I915_PROTECTED_CONTENT_DEFAULT_SESSION) for use in instructions targeting + * protected objects. + * The session is invalidated by the HW when certain events occur (e.g. + * suspend/resume). When this happens, all the objects that were used with the + * session are marked as invalid and all contexts marked as using protected + * content are banned. Any further attempt at using them in an execbuf call is + * rejected, while flips are converted to black frames. + * + * Some of the PXP setup operations are performed by the Management Engine, + * which is handled by the mei driver; communication between i915 and mei is + * performed via the mei_pxp component module. + */ + +struct intel_gt *pxp_to_gt(const struct intel_pxp *pxp) +{ + return container_of(pxp, struct intel_gt, pxp); +} + +bool intel_pxp_is_active(const struct intel_pxp *pxp) +{ + return pxp->arb_is_valid; +} + +/* KCR register definitions */ +#define KCR_INIT _MMIO(0x320f0) +/* Setting KCR Init bit is required after system boot */ +#define KCR_INIT_ALLOW_DISPLAY_ME_WRITES REG_BIT(14) + +static void kcr_pxp_enable(struct intel_gt *gt) +{ + intel_uncore_write(gt->uncore, KCR_INIT, + _MASKED_BIT_ENABLE(KCR_INIT_ALLOW_DISPLAY_ME_WRITES)); +} + +static void kcr_pxp_disable(struct intel_gt *gt) +{ + intel_uncore_write(gt->uncore, KCR_INIT, + _MASKED_BIT_DISABLE(KCR_INIT_ALLOW_DISPLAY_ME_WRITES)); +} + +static int create_vcs_context(struct intel_pxp *pxp) +{ + static struct lock_class_key pxp_lock; + struct intel_gt *gt = pxp_to_gt(pxp); + struct intel_engine_cs *engine; + struct intel_context *ce; + int i; + + /* + * Find the first VCS engine present. We're guaranteed there is one + * if we're in this function due to the check in has_pxp + */ + for (i = 0, engine = NULL; !engine; i++) + engine = gt->engine_class[VIDEO_DECODE_CLASS][i]; + + GEM_BUG_ON(!engine || engine->class != VIDEO_DECODE_CLASS); + + ce = intel_engine_create_pinned_context(engine, engine->gt->vm, SZ_4K, + I915_GEM_HWS_PXP_ADDR, + &pxp_lock, "pxp_context"); + if (IS_ERR(ce)) { + drm_err(>->i915->drm, "failed to create VCS ctx for PXP\n"); + return PTR_ERR(ce); + } + + pxp->ce = ce; + + return 0; +} + +static void destroy_vcs_context(struct intel_pxp *pxp) +{ + intel_engine_destroy_pinned_context(fetch_and_zero(&pxp->ce)); +} + +void intel_pxp_init(struct intel_pxp *pxp) +{ + struct intel_gt *gt = pxp_to_gt(pxp); + int ret; + + if (!HAS_PXP(gt->i915)) + return; + + mutex_init(&pxp->tee_mutex); + + /* + * we'll use the completion to check if there is a termination pending, + * so we start it as completed and we reinit it when a termination + * is triggered. + */ + init_completion(&pxp->termination); + complete_all(&pxp->termination); + + mutex_init(&pxp->arb_mutex); + INIT_WORK(&pxp->session_work, intel_pxp_session_work); + + ret = create_vcs_context(pxp); + if (ret) + return; + + ret = intel_pxp_tee_component_init(pxp); + if (ret) + goto out_context; + + drm_info(>->i915->drm, "Protected Xe Path (PXP) protected content support initialized\n"); + + return; + +out_context: + destroy_vcs_context(pxp); +} + +void intel_pxp_fini(struct intel_pxp *pxp) +{ + if (!intel_pxp_is_enabled(pxp)) + return; + + pxp->arb_is_valid = false; + + intel_pxp_tee_component_fini(pxp); + + destroy_vcs_context(pxp); +} + +void intel_pxp_mark_termination_in_progress(struct intel_pxp *pxp) +{ + pxp->arb_is_valid = false; + reinit_completion(&pxp->termination); +} + +static void pxp_queue_termination(struct intel_pxp *pxp) +{ + struct intel_gt *gt = pxp_to_gt(pxp); + + /* + * We want to get the same effect as if we received a termination + * interrupt, so just pretend that we did. + */ + spin_lock_irq(>->irq_lock); + intel_pxp_mark_termination_in_progress(pxp); + pxp->session_events |= PXP_TERMINATION_REQUEST; + queue_work(system_unbound_wq, &pxp->session_work); + spin_unlock_irq(>->irq_lock); +} + +/* + * the arb session is restarted from the irq work when we receive the + * termination completion interrupt + */ +int intel_pxp_start(struct intel_pxp *pxp) +{ + int ret = 0; + + if (!intel_pxp_is_enabled(pxp)) + return -ENODEV; + + mutex_lock(&pxp->arb_mutex); + + if (pxp->arb_is_valid) + goto unlock; + + pxp_queue_termination(pxp); + + if (!wait_for_completion_timeout(&pxp->termination, + msecs_to_jiffies(250))) { + ret = -ETIMEDOUT; + goto unlock; + } + + /* make sure the compiler doesn't optimize the double access */ + barrier(); + + if (!pxp->arb_is_valid) + ret = -EIO; + +unlock: + mutex_unlock(&pxp->arb_mutex); + return ret; +} + +void intel_pxp_init_hw(struct intel_pxp *pxp) +{ + kcr_pxp_enable(pxp_to_gt(pxp)); + intel_pxp_irq_enable(pxp); +} + +void intel_pxp_fini_hw(struct intel_pxp *pxp) +{ + kcr_pxp_disable(pxp_to_gt(pxp)); + + intel_pxp_irq_disable(pxp); +} + +int intel_pxp_key_check(struct intel_pxp *pxp, + struct drm_i915_gem_object *obj, + bool assign) +{ + if (!intel_pxp_is_active(pxp)) + return -ENODEV; + + if (!i915_gem_object_is_protected(obj)) + return -EINVAL; + + GEM_BUG_ON(!pxp->key_instance); + + /* + * If this is the first time we're using this object, it's not + * encrypted yet; it will be encrypted with the current key, so mark it + * as such. If the object is already encrypted, check instead if the + * used key is still valid. + */ + if (!obj->pxp_key_instance && assign) + obj->pxp_key_instance = pxp->key_instance; + + if (obj->pxp_key_instance != pxp->key_instance) + return -ENOEXEC; + + return 0; +} + +void intel_pxp_invalidate(struct intel_pxp *pxp) +{ + struct drm_i915_private *i915 = pxp_to_gt(pxp)->i915; + struct i915_gem_context *ctx, *cn; + + /* ban all contexts marked as protected */ + spin_lock_irq(&i915->gem.contexts.lock); + list_for_each_entry_safe(ctx, cn, &i915->gem.contexts.list, link) { + struct i915_gem_engines_iter it; + struct intel_context *ce; + + if (!kref_get_unless_zero(&ctx->ref)) + continue; + + if (likely(!i915_gem_context_uses_protected_content(ctx))) { + i915_gem_context_put(ctx); + continue; + } + + spin_unlock_irq(&i915->gem.contexts.lock); + + /* + * By the time we get here we are either going to suspend with + * quiesced execution or the HW keys are already long gone and + * in this case it is worthless to attempt to close the context + * and wait for its execution. It will hang the GPU if it has + * not already. So, as a fast mitigation, we can ban the + * context as quick as we can. That might race with the + * execbuffer, but currently this is the best that can be done. + */ + for_each_gem_engine(ce, i915_gem_context_lock_engines(ctx), it) + intel_context_ban(ce, NULL); + i915_gem_context_unlock_engines(ctx); + + /* + * The context has been banned, no need to keep the wakeref. + * This is safe from races because the only other place this + * is touched is context_release and we're holding a ctx ref + */ + if (ctx->pxp_wakeref) { + intel_runtime_pm_put(&i915->runtime_pm, + ctx->pxp_wakeref); + ctx->pxp_wakeref = 0; + } + + spin_lock_irq(&i915->gem.contexts.lock); + list_safe_reset_next(ctx, cn, link); + i915_gem_context_put(ctx); + } + spin_unlock_irq(&i915->gem.contexts.lock); +} diff --git a/drivers/gpu/drm/i915/pxp/intel_pxp.h b/drivers/gpu/drm/i915/pxp/intel_pxp.h new file mode 100644 index 000000000000..aa262258d4d4 --- /dev/null +++ b/drivers/gpu/drm/i915/pxp/intel_pxp.h @@ -0,0 +1,64 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright(c) 2020, Intel Corporation. All rights reserved. + */ + +#ifndef __INTEL_PXP_H__ +#define __INTEL_PXP_H__ + +#include "intel_pxp_types.h" + +struct drm_i915_gem_object; + +static inline bool intel_pxp_is_enabled(const struct intel_pxp *pxp) +{ + return pxp->ce; +} + +#ifdef CONFIG_DRM_I915_PXP +struct intel_gt *pxp_to_gt(const struct intel_pxp *pxp); +bool intel_pxp_is_active(const struct intel_pxp *pxp); + +void intel_pxp_init(struct intel_pxp *pxp); +void intel_pxp_fini(struct intel_pxp *pxp); + +void intel_pxp_init_hw(struct intel_pxp *pxp); +void intel_pxp_fini_hw(struct intel_pxp *pxp); + +void intel_pxp_mark_termination_in_progress(struct intel_pxp *pxp); + +int intel_pxp_start(struct intel_pxp *pxp); + +int intel_pxp_key_check(struct intel_pxp *pxp, + struct drm_i915_gem_object *obj, + bool assign); + +void intel_pxp_invalidate(struct intel_pxp *pxp); +#else +static inline void intel_pxp_init(struct intel_pxp *pxp) +{ +} + +static inline void intel_pxp_fini(struct intel_pxp *pxp) +{ +} + +static inline int intel_pxp_start(struct intel_pxp *pxp) +{ + return -ENODEV; +} + +static inline bool intel_pxp_is_active(const struct intel_pxp *pxp) +{ + return false; +} + +static inline int intel_pxp_key_check(struct intel_pxp *pxp, + struct drm_i915_gem_object *obj, + bool assign) +{ + return -ENODEV; +} +#endif + +#endif /* __INTEL_PXP_H__ */ diff --git a/drivers/gpu/drm/i915/pxp/intel_pxp_cmd.c b/drivers/gpu/drm/i915/pxp/intel_pxp_cmd.c new file mode 100644 index 000000000000..f41e45763d0d --- /dev/null +++ b/drivers/gpu/drm/i915/pxp/intel_pxp_cmd.c @@ -0,0 +1,141 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright(c) 2020, Intel Corporation. All rights reserved. + */ + +#include "intel_pxp.h" +#include "intel_pxp_cmd.h" +#include "intel_pxp_session.h" +#include "gt/intel_context.h" +#include "gt/intel_engine_pm.h" +#include "gt/intel_gpu_commands.h" +#include "gt/intel_ring.h" + +#include "i915_trace.h" + +/* stall until prior PXP and MFX/HCP/HUC objects are cmopleted */ +#define MFX_WAIT_PXP (MFX_WAIT | \ + MFX_WAIT_DW0_PXP_SYNC_CONTROL_FLAG | \ + MFX_WAIT_DW0_MFX_SYNC_CONTROL_FLAG) + +static u32 *pxp_emit_session_selection(u32 *cs, u32 idx) +{ + *cs++ = MFX_WAIT_PXP; + + /* pxp off */ + *cs++ = MI_FLUSH_DW; + *cs++ = 0; + *cs++ = 0; + + /* select session */ + *cs++ = MI_SET_APPID | MI_SET_APPID_SESSION_ID(idx); + + *cs++ = MFX_WAIT_PXP; + + /* pxp on */ + *cs++ = MI_FLUSH_DW | MI_FLUSH_DW_PROTECTED_MEM_EN | + MI_FLUSH_DW_OP_STOREDW | MI_FLUSH_DW_STORE_INDEX; + *cs++ = I915_GEM_HWS_PXP_ADDR | MI_FLUSH_DW_USE_GTT; + *cs++ = 0; + + *cs++ = MFX_WAIT_PXP; + + return cs; +} + +static u32 *pxp_emit_inline_termination(u32 *cs) +{ + /* session inline termination */ + *cs++ = CRYPTO_KEY_EXCHANGE; + *cs++ = 0; + + return cs; +} + +static u32 *pxp_emit_session_termination(u32 *cs, u32 idx) +{ + cs = pxp_emit_session_selection(cs, idx); + cs = pxp_emit_inline_termination(cs); + + return cs; +} + +static u32 *pxp_emit_wait(u32 *cs) +{ + /* wait for cmds to go through */ + *cs++ = MFX_WAIT_PXP; + *cs++ = 0; + + return cs; +} + +/* + * if we ever need to terminate more than one session, we can submit multiple + * selections and terminations back-to-back with a single wait at the end + */ +#define SELECTION_LEN 10 +#define TERMINATION_LEN 2 +#define SESSION_TERMINATION_LEN(x) ((SELECTION_LEN + TERMINATION_LEN) * (x)) +#define WAIT_LEN 2 + +static void pxp_request_commit(struct i915_request *rq) +{ + struct i915_sched_attr attr = { .priority = I915_PRIORITY_MAX }; + struct intel_timeline * const tl = i915_request_timeline(rq); + + lockdep_unpin_lock(&tl->mutex, rq->cookie); + + trace_i915_request_add(rq); + __i915_request_commit(rq); + __i915_request_queue(rq, &attr); + + mutex_unlock(&tl->mutex); +} + +int intel_pxp_terminate_session(struct intel_pxp *pxp, u32 id) +{ + struct i915_request *rq; + struct intel_context *ce = pxp->ce; + u32 *cs; + int err = 0; + + if (!intel_pxp_is_enabled(pxp)) + return 0; + + rq = i915_request_create(ce); + if (IS_ERR(rq)) + return PTR_ERR(rq); + + if (ce->engine->emit_init_breadcrumb) { + err = ce->engine->emit_init_breadcrumb(rq); + if (err) + goto out_rq; + } + + cs = intel_ring_begin(rq, SESSION_TERMINATION_LEN(1) + WAIT_LEN); + if (IS_ERR(cs)) { + err = PTR_ERR(cs); + goto out_rq; + } + + cs = pxp_emit_session_termination(cs, id); + cs = pxp_emit_wait(cs); + + intel_ring_advance(rq, cs); + +out_rq: + i915_request_get(rq); + + if (unlikely(err)) + i915_request_set_error_once(rq, err); + + pxp_request_commit(rq); + + if (!err && i915_request_wait(rq, 0, HZ / 5) < 0) + err = -ETIME; + + i915_request_put(rq); + + return err; +} + diff --git a/drivers/gpu/drm/i915/pxp/intel_pxp_cmd.h b/drivers/gpu/drm/i915/pxp/intel_pxp_cmd.h new file mode 100644 index 000000000000..6d6299543578 --- /dev/null +++ b/drivers/gpu/drm/i915/pxp/intel_pxp_cmd.h @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright(c) 2020, Intel Corporation. All rights reserved. + */ + +#ifndef __INTEL_PXP_CMD_H__ +#define __INTEL_PXP_CMD_H__ + +#include <linux/types.h> + +struct intel_pxp; + +int intel_pxp_terminate_session(struct intel_pxp *pxp, u32 idx); + +#endif /* __INTEL_PXP_CMD_H__ */ diff --git a/drivers/gpu/drm/i915/pxp/intel_pxp_debugfs.c b/drivers/gpu/drm/i915/pxp/intel_pxp_debugfs.c new file mode 100644 index 000000000000..10e1e45471f1 --- /dev/null +++ b/drivers/gpu/drm/i915/pxp/intel_pxp_debugfs.c @@ -0,0 +1,78 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2021 Intel Corporation + */ + +#include <linux/debugfs.h> +#include <drm/drm_print.h> + +#include "gt/intel_gt_debugfs.h" +#include "pxp/intel_pxp.h" +#include "pxp/intel_pxp_irq.h" +#include "i915_drv.h" + +static int pxp_info_show(struct seq_file *m, void *data) +{ + struct intel_pxp *pxp = m->private; + struct drm_printer p = drm_seq_file_printer(m); + bool enabled = intel_pxp_is_enabled(pxp); + + if (!enabled) { + drm_printf(&p, "pxp disabled\n"); + return 0; + } + + drm_printf(&p, "active: %s\n", yesno(intel_pxp_is_active(pxp))); + drm_printf(&p, "instance counter: %u\n", pxp->key_instance); + + return 0; +} +DEFINE_INTEL_GT_DEBUGFS_ATTRIBUTE(pxp_info); + +static int pxp_terminate_get(void *data, u64 *val) +{ + /* nothing to read */ + return -EPERM; +} + +static int pxp_terminate_set(void *data, u64 val) +{ + struct intel_pxp *pxp = data; + struct intel_gt *gt = pxp_to_gt(pxp); + + if (!intel_pxp_is_active(pxp)) + return -ENODEV; + + /* simulate a termination interrupt */ + spin_lock_irq(>->irq_lock); + intel_pxp_irq_handler(pxp, GEN12_DISPLAY_PXP_STATE_TERMINATED_INTERRUPT); + spin_unlock_irq(>->irq_lock); + + if (!wait_for_completion_timeout(&pxp->termination, + msecs_to_jiffies(100))) + return -ETIMEDOUT; + + return 0; +} + +DEFINE_SIMPLE_ATTRIBUTE(pxp_terminate_fops, pxp_terminate_get, pxp_terminate_set, "%llx\n"); +void intel_pxp_debugfs_register(struct intel_pxp *pxp, struct dentry *gt_root) +{ + static const struct intel_gt_debugfs_file files[] = { + { "info", &pxp_info_fops, NULL }, + { "terminate_state", &pxp_terminate_fops, NULL }, + }; + struct dentry *root; + + if (!gt_root) + return; + + if (!HAS_PXP((pxp_to_gt(pxp)->i915))) + return; + + root = debugfs_create_dir("pxp", gt_root); + if (IS_ERR(root)) + return; + + intel_gt_debugfs_register_files(root, files, ARRAY_SIZE(files), pxp); +} diff --git a/drivers/gpu/drm/i915/pxp/intel_pxp_debugfs.h b/drivers/gpu/drm/i915/pxp/intel_pxp_debugfs.h new file mode 100644 index 000000000000..7e0c3d2f5d7e --- /dev/null +++ b/drivers/gpu/drm/i915/pxp/intel_pxp_debugfs.h @@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2021 Intel Corporation + */ + +#ifndef __INTEL_PXP_DEBUGFS_H__ +#define __INTEL_PXP_DEBUGFS_H__ + +struct intel_pxp; +struct dentry; + +#ifdef CONFIG_DRM_I915_PXP +void intel_pxp_debugfs_register(struct intel_pxp *pxp, struct dentry *root); +#else +static inline void +intel_pxp_debugfs_register(struct intel_pxp *pxp, struct dentry *root) +{ +} +#endif + +#endif /* __INTEL_PXP_DEBUGFS_H__ */ diff --git a/drivers/gpu/drm/i915/pxp/intel_pxp_irq.c b/drivers/gpu/drm/i915/pxp/intel_pxp_irq.c new file mode 100644 index 000000000000..8d5553772ded --- /dev/null +++ b/drivers/gpu/drm/i915/pxp/intel_pxp_irq.c @@ -0,0 +1,101 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright(c) 2020 Intel Corporation. + */ +#include <linux/workqueue.h> +#include "intel_pxp.h" +#include "intel_pxp_irq.h" +#include "intel_pxp_session.h" +#include "gt/intel_gt_irq.h" +#include "gt/intel_gt_types.h" +#include "i915_irq.h" +#include "i915_reg.h" +#include "intel_runtime_pm.h" + +/** + * intel_pxp_irq_handler - Handles PXP interrupts. + * @pxp: pointer to pxp struct + * @iir: interrupt vector + */ +void intel_pxp_irq_handler(struct intel_pxp *pxp, u16 iir) +{ + struct intel_gt *gt = pxp_to_gt(pxp); + + if (GEM_WARN_ON(!intel_pxp_is_enabled(pxp))) + return; + + lockdep_assert_held(>->irq_lock); + + if (unlikely(!iir)) + return; + + if (iir & (GEN12_DISPLAY_PXP_STATE_TERMINATED_INTERRUPT | + GEN12_DISPLAY_APP_TERMINATED_PER_FW_REQ_INTERRUPT)) { + /* immediately mark PXP as inactive on termination */ + intel_pxp_mark_termination_in_progress(pxp); + pxp->session_events |= PXP_TERMINATION_REQUEST | PXP_INVAL_REQUIRED; + } + + if (iir & GEN12_DISPLAY_STATE_RESET_COMPLETE_INTERRUPT) + pxp->session_events |= PXP_TERMINATION_COMPLETE; + + if (pxp->session_events) + queue_work(system_unbound_wq, &pxp->session_work); +} + +static inline void __pxp_set_interrupts(struct intel_gt *gt, u32 interrupts) +{ + struct intel_uncore *uncore = gt->uncore; + const u32 mask = interrupts << 16; + + intel_uncore_write(uncore, GEN11_CRYPTO_RSVD_INTR_ENABLE, mask); + intel_uncore_write(uncore, GEN11_CRYPTO_RSVD_INTR_MASK, ~mask); +} + +static inline void pxp_irq_reset(struct intel_gt *gt) +{ + spin_lock_irq(>->irq_lock); + gen11_gt_reset_one_iir(gt, 0, GEN11_KCR); + spin_unlock_irq(>->irq_lock); +} + +void intel_pxp_irq_enable(struct intel_pxp *pxp) +{ + struct intel_gt *gt = pxp_to_gt(pxp); + + spin_lock_irq(>->irq_lock); + + if (!pxp->irq_enabled) + WARN_ON_ONCE(gen11_gt_reset_one_iir(gt, 0, GEN11_KCR)); + + __pxp_set_interrupts(gt, GEN12_PXP_INTERRUPTS); + pxp->irq_enabled = true; + + spin_unlock_irq(>->irq_lock); +} + +void intel_pxp_irq_disable(struct intel_pxp *pxp) +{ + struct intel_gt *gt = pxp_to_gt(pxp); + + /* + * We always need to submit a global termination when we re-enable the + * interrupts, so there is no need to make sure that the session state + * makes sense at the end of this function. Just make sure this is not + * called in a path were the driver consider the session as valid and + * doesn't call a termination on restart. + */ + GEM_WARN_ON(intel_pxp_is_active(pxp)); + + spin_lock_irq(>->irq_lock); + + pxp->irq_enabled = false; + __pxp_set_interrupts(gt, 0); + + spin_unlock_irq(>->irq_lock); + intel_synchronize_irq(gt->i915); + + pxp_irq_reset(gt); + + flush_work(&pxp->session_work); +} diff --git a/drivers/gpu/drm/i915/pxp/intel_pxp_irq.h b/drivers/gpu/drm/i915/pxp/intel_pxp_irq.h new file mode 100644 index 000000000000..8b5793654844 --- /dev/null +++ b/drivers/gpu/drm/i915/pxp/intel_pxp_irq.h @@ -0,0 +1,32 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright(c) 2020, Intel Corporation. All rights reserved. + */ + +#ifndef __INTEL_PXP_IRQ_H__ +#define __INTEL_PXP_IRQ_H__ + +#include <linux/types.h> + +struct intel_pxp; + +#define GEN12_DISPLAY_PXP_STATE_TERMINATED_INTERRUPT BIT(1) +#define GEN12_DISPLAY_APP_TERMINATED_PER_FW_REQ_INTERRUPT BIT(2) +#define GEN12_DISPLAY_STATE_RESET_COMPLETE_INTERRUPT BIT(3) + +#define GEN12_PXP_INTERRUPTS \ + (GEN12_DISPLAY_PXP_STATE_TERMINATED_INTERRUPT | \ + GEN12_DISPLAY_APP_TERMINATED_PER_FW_REQ_INTERRUPT | \ + GEN12_DISPLAY_STATE_RESET_COMPLETE_INTERRUPT) + +#ifdef CONFIG_DRM_I915_PXP +void intel_pxp_irq_enable(struct intel_pxp *pxp); +void intel_pxp_irq_disable(struct intel_pxp *pxp); +void intel_pxp_irq_handler(struct intel_pxp *pxp, u16 iir); +#else +static inline void intel_pxp_irq_handler(struct intel_pxp *pxp, u16 iir) +{ +} +#endif + +#endif /* __INTEL_PXP_IRQ_H__ */ diff --git a/drivers/gpu/drm/i915/pxp/intel_pxp_pm.c b/drivers/gpu/drm/i915/pxp/intel_pxp_pm.c new file mode 100644 index 000000000000..23fd86de5a24 --- /dev/null +++ b/drivers/gpu/drm/i915/pxp/intel_pxp_pm.c @@ -0,0 +1,46 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright(c) 2020 Intel Corporation. + */ + +#include "intel_pxp.h" +#include "intel_pxp_irq.h" +#include "intel_pxp_pm.h" +#include "intel_pxp_session.h" + +void intel_pxp_suspend(struct intel_pxp *pxp, bool runtime) +{ + if (!intel_pxp_is_enabled(pxp)) + return; + + pxp->arb_is_valid = false; + + /* + * Contexts using protected objects keep a runtime PM reference, so we + * can only runtime suspend when all of them have been either closed + * or banned. Therefore, there is no need to invalidate in that + * scenario. + */ + if (!runtime) + intel_pxp_invalidate(pxp); + + intel_pxp_fini_hw(pxp); + + pxp->hw_state_invalidated = false; +} + +void intel_pxp_resume(struct intel_pxp *pxp) +{ + if (!intel_pxp_is_enabled(pxp)) + return; + + /* + * The PXP component gets automatically unbound when we go into S3 and + * re-bound after we come out, so in that scenario we can defer the + * hw init to the bind call. + */ + if (!pxp->pxp_component) + return; + + intel_pxp_init_hw(pxp); +} diff --git a/drivers/gpu/drm/i915/pxp/intel_pxp_pm.h b/drivers/gpu/drm/i915/pxp/intel_pxp_pm.h new file mode 100644 index 000000000000..c89e97a0c3d0 --- /dev/null +++ b/drivers/gpu/drm/i915/pxp/intel_pxp_pm.h @@ -0,0 +1,24 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright(c) 2020, Intel Corporation. All rights reserved. + */ + +#ifndef __INTEL_PXP_PM_H__ +#define __INTEL_PXP_PM_H__ + +#include "intel_pxp_types.h" + +#ifdef CONFIG_DRM_I915_PXP +void intel_pxp_suspend(struct intel_pxp *pxp, bool runtime); +void intel_pxp_resume(struct intel_pxp *pxp); +#else +static inline void intel_pxp_suspend(struct intel_pxp *pxp, bool runtime) +{ +} + +static inline void intel_pxp_resume(struct intel_pxp *pxp) +{ +} +#endif + +#endif /* __INTEL_PXP_PM_H__ */ diff --git a/drivers/gpu/drm/i915/pxp/intel_pxp_session.c b/drivers/gpu/drm/i915/pxp/intel_pxp_session.c new file mode 100644 index 000000000000..d02732f04757 --- /dev/null +++ b/drivers/gpu/drm/i915/pxp/intel_pxp_session.c @@ -0,0 +1,175 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright(c) 2020, Intel Corporation. All rights reserved. + */ + +#include "drm/i915_drm.h" +#include "i915_drv.h" + +#include "intel_pxp.h" +#include "intel_pxp_cmd.h" +#include "intel_pxp_session.h" +#include "intel_pxp_tee.h" +#include "intel_pxp_types.h" + +#define ARB_SESSION I915_PROTECTED_CONTENT_DEFAULT_SESSION /* shorter define */ + +#define GEN12_KCR_SIP _MMIO(0x32260) /* KCR hwdrm session in play 0-31 */ + +/* PXP global terminate register for session termination */ +#define PXP_GLOBAL_TERMINATE _MMIO(0x320f8) + +static bool intel_pxp_session_is_in_play(struct intel_pxp *pxp, u32 id) +{ + struct intel_uncore *uncore = pxp_to_gt(pxp)->uncore; + intel_wakeref_t wakeref; + u32 sip = 0; + + /* if we're suspended the session is considered off */ + with_intel_runtime_pm_if_in_use(uncore->rpm, wakeref) + sip = intel_uncore_read(uncore, GEN12_KCR_SIP); + + return sip & BIT(id); +} + +static int pxp_wait_for_session_state(struct intel_pxp *pxp, u32 id, bool in_play) +{ + struct intel_uncore *uncore = pxp_to_gt(pxp)->uncore; + intel_wakeref_t wakeref; + u32 mask = BIT(id); + int ret; + + /* if we're suspended the session is considered off */ + wakeref = intel_runtime_pm_get_if_in_use(uncore->rpm); + if (!wakeref) + return in_play ? -ENODEV : 0; + + ret = intel_wait_for_register(uncore, + GEN12_KCR_SIP, + mask, + in_play ? mask : 0, + 100); + + intel_runtime_pm_put(uncore->rpm, wakeref); + + return ret; +} + +static int pxp_create_arb_session(struct intel_pxp *pxp) +{ + struct intel_gt *gt = pxp_to_gt(pxp); + int ret; + + pxp->arb_is_valid = false; + + if (intel_pxp_session_is_in_play(pxp, ARB_SESSION)) { + drm_err(>->i915->drm, "arb session already in play at creation time\n"); + return -EEXIST; + } + + ret = intel_pxp_tee_cmd_create_arb_session(pxp, ARB_SESSION); + if (ret) { + drm_err(>->i915->drm, "tee cmd for arb session creation failed\n"); + return ret; + } + + ret = pxp_wait_for_session_state(pxp, ARB_SESSION, true); + if (ret) { + drm_err(>->i915->drm, "arb session failed to go in play\n"); + return ret; + } + + if (!++pxp->key_instance) + ++pxp->key_instance; + + pxp->arb_is_valid = true; + + return 0; +} + +static int pxp_terminate_arb_session_and_global(struct intel_pxp *pxp) +{ + int ret; + struct intel_gt *gt = pxp_to_gt(pxp); + + /* must mark termination in progress calling this function */ + GEM_WARN_ON(pxp->arb_is_valid); + + /* terminate the hw sessions */ + ret = intel_pxp_terminate_session(pxp, ARB_SESSION); + if (ret) { + drm_err(>->i915->drm, "Failed to submit session termination\n"); + return ret; + } + + ret = pxp_wait_for_session_state(pxp, ARB_SESSION, false); + if (ret) { + drm_err(>->i915->drm, "Session state did not clear\n"); + return ret; + } + + intel_uncore_write(gt->uncore, PXP_GLOBAL_TERMINATE, 1); + + return ret; +} + +static void pxp_terminate(struct intel_pxp *pxp) +{ + int ret; + + pxp->hw_state_invalidated = true; + + /* + * if we fail to submit the termination there is no point in waiting for + * it to complete. PXP will be marked as non-active until the next + * termination is issued. + */ + ret = pxp_terminate_arb_session_and_global(pxp); + if (ret) + complete_all(&pxp->termination); +} + +static void pxp_terminate_complete(struct intel_pxp *pxp) +{ + /* Re-create the arb session after teardown handle complete */ + if (fetch_and_zero(&pxp->hw_state_invalidated)) + pxp_create_arb_session(pxp); + + complete_all(&pxp->termination); +} + +void intel_pxp_session_work(struct work_struct *work) +{ + struct intel_pxp *pxp = container_of(work, typeof(*pxp), session_work); + struct intel_gt *gt = pxp_to_gt(pxp); + intel_wakeref_t wakeref; + u32 events = 0; + + spin_lock_irq(>->irq_lock); + events = fetch_and_zero(&pxp->session_events); + spin_unlock_irq(>->irq_lock); + + if (!events) + return; + + if (events & PXP_INVAL_REQUIRED) + intel_pxp_invalidate(pxp); + + /* + * If we're processing an event while suspending then don't bother, + * we're going to re-init everything on resume anyway. + */ + wakeref = intel_runtime_pm_get_if_in_use(gt->uncore->rpm); + if (!wakeref) + return; + + if (events & PXP_TERMINATION_REQUEST) { + events &= ~PXP_TERMINATION_COMPLETE; + pxp_terminate(pxp); + } + + if (events & PXP_TERMINATION_COMPLETE) + pxp_terminate_complete(pxp); + + intel_runtime_pm_put(gt->uncore->rpm, wakeref); +} diff --git a/drivers/gpu/drm/i915/pxp/intel_pxp_session.h b/drivers/gpu/drm/i915/pxp/intel_pxp_session.h new file mode 100644 index 000000000000..ba4c9d2b94b7 --- /dev/null +++ b/drivers/gpu/drm/i915/pxp/intel_pxp_session.h @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright(c) 2020, Intel Corporation. All rights reserved. + */ + +#ifndef __INTEL_PXP_SESSION_H__ +#define __INTEL_PXP_SESSION_H__ + +#include <linux/types.h> + +struct work_struct; + +void intel_pxp_session_work(struct work_struct *work); + +#endif /* __INTEL_PXP_SESSION_H__ */ diff --git a/drivers/gpu/drm/i915/pxp/intel_pxp_tee.c b/drivers/gpu/drm/i915/pxp/intel_pxp_tee.c new file mode 100644 index 000000000000..49508f31dcb7 --- /dev/null +++ b/drivers/gpu/drm/i915/pxp/intel_pxp_tee.c @@ -0,0 +1,172 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright(c) 2020 Intel Corporation. + */ + +#include <linux/component.h> +#include "drm/i915_pxp_tee_interface.h" +#include "drm/i915_component.h" +#include "i915_drv.h" +#include "intel_pxp.h" +#include "intel_pxp_session.h" +#include "intel_pxp_tee.h" +#include "intel_pxp_tee_interface.h" + +static inline struct intel_pxp *i915_dev_to_pxp(struct device *i915_kdev) +{ + return &kdev_to_i915(i915_kdev)->gt.pxp; +} + +static int intel_pxp_tee_io_message(struct intel_pxp *pxp, + void *msg_in, u32 msg_in_size, + void *msg_out, u32 msg_out_max_size, + u32 *msg_out_rcv_size) +{ + struct drm_i915_private *i915 = pxp_to_gt(pxp)->i915; + struct i915_pxp_component *pxp_component = pxp->pxp_component; + int ret = 0; + + mutex_lock(&pxp->tee_mutex); + + /* + * The binding of the component is asynchronous from i915 probe, so we + * can't be sure it has happened. + */ + if (!pxp_component) { + ret = -ENODEV; + goto unlock; + } + + ret = pxp_component->ops->send(pxp_component->tee_dev, msg_in, msg_in_size); + if (ret) { + drm_err(&i915->drm, "Failed to send PXP TEE message\n"); + goto unlock; + } + + ret = pxp_component->ops->recv(pxp_component->tee_dev, msg_out, msg_out_max_size); + if (ret < 0) { + drm_err(&i915->drm, "Failed to receive PXP TEE message\n"); + goto unlock; + } + + if (ret > msg_out_max_size) { + drm_err(&i915->drm, + "Failed to receive PXP TEE message due to unexpected output size\n"); + ret = -ENOSPC; + goto unlock; + } + + if (msg_out_rcv_size) + *msg_out_rcv_size = ret; + + ret = 0; +unlock: + mutex_unlock(&pxp->tee_mutex); + return ret; +} + +/** + * i915_pxp_tee_component_bind - bind function to pass the function pointers to pxp_tee + * @i915_kdev: pointer to i915 kernel device + * @tee_kdev: pointer to tee kernel device + * @data: pointer to pxp_tee_master containing the function pointers + * + * This bind function is called during the system boot or resume from system sleep. + * + * Return: return 0 if successful. + */ +static int i915_pxp_tee_component_bind(struct device *i915_kdev, + struct device *tee_kdev, void *data) +{ + struct drm_i915_private *i915 = kdev_to_i915(i915_kdev); + struct intel_pxp *pxp = i915_dev_to_pxp(i915_kdev); + intel_wakeref_t wakeref; + + mutex_lock(&pxp->tee_mutex); + pxp->pxp_component = data; + pxp->pxp_component->tee_dev = tee_kdev; + mutex_unlock(&pxp->tee_mutex); + + /* if we are suspended, the HW will be re-initialized on resume */ + wakeref = intel_runtime_pm_get_if_in_use(&i915->runtime_pm); + if (!wakeref) + return 0; + + /* the component is required to fully start the PXP HW */ + intel_pxp_init_hw(pxp); + + intel_runtime_pm_put(&i915->runtime_pm, wakeref); + + return 0; +} + +static void i915_pxp_tee_component_unbind(struct device *i915_kdev, + struct device *tee_kdev, void *data) +{ + struct intel_pxp *pxp = i915_dev_to_pxp(i915_kdev); + + intel_pxp_fini_hw(pxp); + + mutex_lock(&pxp->tee_mutex); + pxp->pxp_component = NULL; + mutex_unlock(&pxp->tee_mutex); +} + +static const struct component_ops i915_pxp_tee_component_ops = { + .bind = i915_pxp_tee_component_bind, + .unbind = i915_pxp_tee_component_unbind, +}; + +int intel_pxp_tee_component_init(struct intel_pxp *pxp) +{ + int ret; + struct intel_gt *gt = pxp_to_gt(pxp); + struct drm_i915_private *i915 = gt->i915; + + ret = component_add_typed(i915->drm.dev, &i915_pxp_tee_component_ops, + I915_COMPONENT_PXP); + if (ret < 0) { + drm_err(&i915->drm, "Failed to add PXP component (%d)\n", ret); + return ret; + } + + pxp->pxp_component_added = true; + + return 0; +} + +void intel_pxp_tee_component_fini(struct intel_pxp *pxp) +{ + struct drm_i915_private *i915 = pxp_to_gt(pxp)->i915; + + if (!pxp->pxp_component_added) + return; + + component_del(i915->drm.dev, &i915_pxp_tee_component_ops); + pxp->pxp_component_added = false; +} + +int intel_pxp_tee_cmd_create_arb_session(struct intel_pxp *pxp, + int arb_session_id) +{ + struct drm_i915_private *i915 = pxp_to_gt(pxp)->i915; + struct pxp_tee_create_arb_in msg_in = {0}; + struct pxp_tee_create_arb_out msg_out = {0}; + int ret; + + msg_in.header.api_version = PXP_TEE_APIVER; + msg_in.header.command_id = PXP_TEE_ARB_CMDID; + msg_in.header.buffer_len = sizeof(msg_in) - sizeof(msg_in.header); + msg_in.protection_mode = PXP_TEE_ARB_PROTECTION_MODE; + msg_in.session_id = arb_session_id; + + ret = intel_pxp_tee_io_message(pxp, + &msg_in, sizeof(msg_in), + &msg_out, sizeof(msg_out), + NULL); + + if (ret) + drm_err(&i915->drm, "Failed to send tee msg ret=[%d]\n", ret); + + return ret; +} diff --git a/drivers/gpu/drm/i915/pxp/intel_pxp_tee.h b/drivers/gpu/drm/i915/pxp/intel_pxp_tee.h new file mode 100644 index 000000000000..c136053ce340 --- /dev/null +++ b/drivers/gpu/drm/i915/pxp/intel_pxp_tee.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright(c) 2020, Intel Corporation. All rights reserved. + */ + +#ifndef __INTEL_PXP_TEE_H__ +#define __INTEL_PXP_TEE_H__ + +#include "intel_pxp.h" + +int intel_pxp_tee_component_init(struct intel_pxp *pxp); +void intel_pxp_tee_component_fini(struct intel_pxp *pxp); + +int intel_pxp_tee_cmd_create_arb_session(struct intel_pxp *pxp, + int arb_session_id); + +#endif /* __INTEL_PXP_TEE_H__ */ diff --git a/drivers/gpu/drm/i915/pxp/intel_pxp_tee_interface.h b/drivers/gpu/drm/i915/pxp/intel_pxp_tee_interface.h new file mode 100644 index 000000000000..36e9b0868f5c --- /dev/null +++ b/drivers/gpu/drm/i915/pxp/intel_pxp_tee_interface.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright(c) 2020, Intel Corporation. All rights reserved. + */ + +#ifndef __INTEL_PXP_TEE_INTERFACE_H__ +#define __INTEL_PXP_TEE_INTERFACE_H__ + +#include <linux/types.h> + +#define PXP_TEE_APIVER 0x40002 +#define PXP_TEE_ARB_CMDID 0x1e +#define PXP_TEE_ARB_PROTECTION_MODE 0x2 + +/* PXP TEE message header */ +struct pxp_tee_cmd_header { + u32 api_version; + u32 command_id; + u32 status; + /* Length of the message (excluding the header) */ + u32 buffer_len; +} __packed; + +/* PXP TEE message input to create a arbitrary session */ +struct pxp_tee_create_arb_in { + struct pxp_tee_cmd_header header; + u32 protection_mode; + u32 session_id; +} __packed; + +/* PXP TEE message output to create a arbitrary session */ +struct pxp_tee_create_arb_out { + struct pxp_tee_cmd_header header; +} __packed; + +#endif /* __INTEL_PXP_TEE_INTERFACE_H__ */ diff --git a/drivers/gpu/drm/i915/pxp/intel_pxp_types.h b/drivers/gpu/drm/i915/pxp/intel_pxp_types.h new file mode 100644 index 000000000000..73ef7d1754e1 --- /dev/null +++ b/drivers/gpu/drm/i915/pxp/intel_pxp_types.h @@ -0,0 +1,83 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright(c) 2020, Intel Corporation. All rights reserved. + */ + +#ifndef __INTEL_PXP_TYPES_H__ +#define __INTEL_PXP_TYPES_H__ + +#include <linux/completion.h> +#include <linux/list.h> +#include <linux/mutex.h> +#include <linux/spinlock.h> +#include <linux/types.h> +#include <linux/workqueue.h> + +struct intel_context; +struct i915_pxp_component; + +/** + * struct intel_pxp - pxp state + */ +struct intel_pxp { + /** + * @pxp_component: i915_pxp_component struct of the bound mei_pxp + * module. Only set and cleared inside component bind/unbind functions, + * which are protected by &tee_mutex. + */ + struct i915_pxp_component *pxp_component; + /** + * @pxp_component_added: track if the pxp component has been added. + * Set and cleared in tee init and fini functions respectively. + */ + bool pxp_component_added; + + /** @ce: kernel-owned context used for PXP operations */ + struct intel_context *ce; + + /** @arb_mutex: protects arb session start */ + struct mutex arb_mutex; + /** + * @arb_is_valid: tracks arb session status. + * After a teardown, the arb session can still be in play on the HW + * even if the keys are gone, so we can't rely on the HW state of the + * session to know if it's valid and need to track the status in SW. + */ + bool arb_is_valid; + + /** + * @key_instance: tracks which key instance we're on, so we can use it + * to determine if an object was created using the current key or a + * previous one. + */ + u32 key_instance; + + /** @tee_mutex: protects the tee channel binding and messaging. */ + struct mutex tee_mutex; + + /** + * @hw_state_invalidated: if the HW perceives an attack on the integrity + * of the encryption it will invalidate the keys and expect SW to + * re-initialize the session. We keep track of this state to make sure + * we only re-start the arb session when required. + */ + bool hw_state_invalidated; + + /** @irq_enabled: tracks the status of the kcr irqs */ + bool irq_enabled; + /** + * @termination: tracks the status of a pending termination. Only + * re-initialized under gt->irq_lock and completed in &session_work. + */ + struct completion termination; + + /** @session_work: worker that manages session events. */ + struct work_struct session_work; + /** @session_events: pending session events, protected with gt->irq_lock. */ + u32 session_events; +#define PXP_TERMINATION_REQUEST BIT(0) +#define PXP_TERMINATION_COMPLETE BIT(1) +#define PXP_INVAL_REQUIRED BIT(2) +}; + +#endif /* __INTEL_PXP_TYPES_H__ */ diff --git a/drivers/gpu/drm/i915/selftests/i915_gem_gtt.c b/drivers/gpu/drm/i915/selftests/i915_gem_gtt.c index f843a5040706..46f4236039a9 100644 --- a/drivers/gpu/drm/i915/selftests/i915_gem_gtt.c +++ b/drivers/gpu/drm/i915/selftests/i915_gem_gtt.c @@ -155,7 +155,7 @@ static int igt_ppgtt_alloc(void *arg) if (!HAS_PPGTT(dev_priv)) return 0; - ppgtt = i915_ppgtt_create(&dev_priv->gt); + ppgtt = i915_ppgtt_create(&dev_priv->gt, 0); if (IS_ERR(ppgtt)) return PTR_ERR(ppgtt); @@ -1053,7 +1053,7 @@ static int exercise_ppgtt(struct drm_i915_private *dev_priv, if (IS_ERR(file)) return PTR_ERR(file); - ppgtt = i915_ppgtt_create(&dev_priv->gt); + ppgtt = i915_ppgtt_create(&dev_priv->gt, 0); if (IS_ERR(ppgtt)) { err = PTR_ERR(ppgtt); goto out_free; @@ -1300,7 +1300,7 @@ static int exercise_mock(struct drm_i915_private *i915, if (!ctx) return -ENOMEM; - vm = i915_gem_context_get_vm_rcu(ctx); + vm = i915_gem_context_get_eb_vm(ctx); err = func(vm, 0, min(vm->total, limit), end_time); i915_vm_put(vm); @@ -1848,7 +1848,7 @@ static int igt_cs_tlb(void *arg) goto out_unlock; } - vm = i915_gem_context_get_vm_rcu(ctx); + vm = i915_gem_context_get_eb_vm(ctx); if (i915_is_ggtt(vm)) goto out_vm; diff --git a/drivers/gpu/drm/i915/selftests/i915_live_selftests.h b/drivers/gpu/drm/i915/selftests/i915_live_selftests.h index cfa5c4165a4f..bdd290f2bf3c 100644 --- a/drivers/gpu/drm/i915/selftests/i915_live_selftests.h +++ b/drivers/gpu/drm/i915/selftests/i915_live_selftests.h @@ -47,5 +47,7 @@ selftest(execlists, intel_execlists_live_selftests) selftest(ring_submission, intel_ring_submission_live_selftests) selftest(perf, i915_perf_live_selftests) selftest(slpc, intel_slpc_live_selftests) +selftest(guc, intel_guc_live_selftests) +selftest(guc_multi_lrc, intel_guc_multi_lrc_live_selftests) /* Here be dragons: keep last to run last! */ selftest(late_gt_pm, intel_gt_pm_late_selftests) diff --git a/drivers/gpu/drm/i915/selftests/i915_vma.c b/drivers/gpu/drm/i915/selftests/i915_vma.c index dd0607254a95..1f10fe36619b 100644 --- a/drivers/gpu/drm/i915/selftests/i915_vma.c +++ b/drivers/gpu/drm/i915/selftests/i915_vma.c @@ -39,7 +39,7 @@ static bool assert_vma(struct i915_vma *vma, { bool ok = true; - if (vma->vm != rcu_access_pointer(ctx->vm)) { + if (vma->vm != ctx->vm) { pr_err("VMA created with wrong VM\n"); ok = false; } @@ -118,7 +118,7 @@ static int create_vmas(struct drm_i915_private *i915, struct i915_vma *vma; int err; - vm = i915_gem_context_get_vm_rcu(ctx); + vm = i915_gem_context_get_eb_vm(ctx); vma = checked_vma_instance(obj, vm, NULL); i915_vm_put(vm); if (IS_ERR(vma)) diff --git a/drivers/gpu/drm/i915/selftests/intel_scheduler_helpers.c b/drivers/gpu/drm/i915/selftests/intel_scheduler_helpers.c index 4b328346b48a..310fb83c527e 100644 --- a/drivers/gpu/drm/i915/selftests/intel_scheduler_helpers.c +++ b/drivers/gpu/drm/i915/selftests/intel_scheduler_helpers.c @@ -14,6 +14,18 @@ #define REDUCED_PREEMPT 10 #define WAIT_FOR_RESET_TIME 10000 +struct intel_engine_cs *intel_selftest_find_any_engine(struct intel_gt *gt) +{ + struct intel_engine_cs *engine; + enum intel_engine_id id; + + for_each_engine(engine, gt, id) + return engine; + + pr_err("No valid engine found!\n"); + return NULL; +} + int intel_selftest_modify_policy(struct intel_engine_cs *engine, struct intel_selftest_saved_policy *saved, u32 modify_type) diff --git a/drivers/gpu/drm/i915/selftests/intel_scheduler_helpers.h b/drivers/gpu/drm/i915/selftests/intel_scheduler_helpers.h index 35c098601ac0..ae60bb507f45 100644 --- a/drivers/gpu/drm/i915/selftests/intel_scheduler_helpers.h +++ b/drivers/gpu/drm/i915/selftests/intel_scheduler_helpers.h @@ -10,6 +10,7 @@ struct i915_request; struct intel_engine_cs; +struct intel_gt; struct intel_selftest_saved_policy { u32 flags; @@ -23,6 +24,7 @@ enum selftest_scheduler_modify { SELFTEST_SCHEDULER_MODIFY_FAST_RESET, }; +struct intel_engine_cs *intel_selftest_find_any_engine(struct intel_gt *gt); int intel_selftest_modify_policy(struct intel_engine_cs *engine, struct intel_selftest_saved_policy *saved, enum selftest_scheduler_modify modify_type); diff --git a/drivers/gpu/drm/i915/selftests/intel_uncore.c b/drivers/gpu/drm/i915/selftests/intel_uncore.c index 720b60853f8b..bc8128170a99 100644 --- a/drivers/gpu/drm/i915/selftests/intel_uncore.c +++ b/drivers/gpu/drm/i915/selftests/intel_uncore.c @@ -62,30 +62,40 @@ static int intel_fw_table_check(const struct intel_forcewake_range *ranges, static int intel_shadow_table_check(void) { struct { - const i915_reg_t *regs; + const struct i915_range *regs; unsigned int size; - } reg_lists[] = { + } range_lists[] = { { gen8_shadowed_regs, ARRAY_SIZE(gen8_shadowed_regs) }, { gen11_shadowed_regs, ARRAY_SIZE(gen11_shadowed_regs) }, { gen12_shadowed_regs, ARRAY_SIZE(gen12_shadowed_regs) }, - { xehp_shadowed_regs, ARRAY_SIZE(xehp_shadowed_regs) }, + { dg2_shadowed_regs, ARRAY_SIZE(dg2_shadowed_regs) }, }; - const i915_reg_t *reg; + const struct i915_range *range; unsigned int i, j; s32 prev; - for (j = 0; j < ARRAY_SIZE(reg_lists); ++j) { - reg = reg_lists[j].regs; - for (i = 0, prev = -1; i < reg_lists[j].size; i++, reg++) { - u32 offset = i915_mmio_reg_offset(*reg); + for (j = 0; j < ARRAY_SIZE(range_lists); ++j) { + range = range_lists[j].regs; + for (i = 0, prev = -1; i < range_lists[j].size; i++, range++) { + if (range->end < range->start) { + pr_err("%s: range[%d]:(%06x-%06x) has end before start\n", + __func__, i, range->start, range->end); + return -EINVAL; + } + + if (prev >= (s32)range->start) { + pr_err("%s: range[%d]:(%06x-%06x) is before end of previous (%06x)\n", + __func__, i, range->start, range->end, prev); + return -EINVAL; + } - if (prev >= (s32)offset) { - pr_err("%s: entry[%d]:(%x) is before previous (%x)\n", - __func__, i, offset, prev); + if (range->start % 4) { + pr_err("%s: range[%d]:(%06x-%06x) has non-dword-aligned start\n", + __func__, i, range->start, range->end); return -EINVAL; } - prev = offset; + prev = range->end; } } diff --git a/drivers/gpu/drm/i915/selftests/mock_region.c b/drivers/gpu/drm/i915/selftests/mock_region.c index efa86dffe3c6..75793008c4ef 100644 --- a/drivers/gpu/drm/i915/selftests/mock_region.c +++ b/drivers/gpu/drm/i915/selftests/mock_region.c @@ -6,8 +6,6 @@ #include <drm/ttm/ttm_placement.h> #include <linux/scatterlist.h> -#include <drm/ttm/ttm_placement.h> - #include "gem/i915_gem_region.h" #include "intel_memory_region.h" #include "intel_region_ttm.h" diff --git a/drivers/gpu/drm/i915/vlv_sideband.c b/drivers/gpu/drm/i915/vlv_sideband.c new file mode 100644 index 000000000000..35380738a951 --- /dev/null +++ b/drivers/gpu/drm/i915/vlv_sideband.c @@ -0,0 +1,266 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2013-2021 Intel Corporation + */ + +#include <asm/iosf_mbi.h> + +#include "i915_drv.h" +#include "vlv_sideband.h" + +/* + * IOSF sideband, see VLV2_SidebandMsg_HAS.docx and + * VLV_VLV2_PUNIT_HAS_0.8.docx + */ + +/* Standard MMIO read, non-posted */ +#define SB_MRD_NP 0x00 +/* Standard MMIO write, non-posted */ +#define SB_MWR_NP 0x01 +/* Private register read, double-word addressing, non-posted */ +#define SB_CRRDDA_NP 0x06 +/* Private register write, double-word addressing, non-posted */ +#define SB_CRWRDA_NP 0x07 + +static void ping(void *info) +{ +} + +static void __vlv_punit_get(struct drm_i915_private *i915) +{ + iosf_mbi_punit_acquire(); + + /* + * Prevent the cpu from sleeping while we use this sideband, otherwise + * the punit may cause a machine hang. The issue appears to be isolated + * with changing the power state of the CPU package while changing + * the power state via the punit, and we have only observed it + * reliably on 4-core Baytail systems suggesting the issue is in the + * power delivery mechanism and likely to be board/function + * specific. Hence we presume the workaround needs only be applied + * to the Valleyview P-unit and not all sideband communications. + */ + if (IS_VALLEYVIEW(i915)) { + cpu_latency_qos_update_request(&i915->sb_qos, 0); + on_each_cpu(ping, NULL, 1); + } +} + +static void __vlv_punit_put(struct drm_i915_private *i915) +{ + if (IS_VALLEYVIEW(i915)) + cpu_latency_qos_update_request(&i915->sb_qos, + PM_QOS_DEFAULT_VALUE); + + iosf_mbi_punit_release(); +} + +void vlv_iosf_sb_get(struct drm_i915_private *i915, unsigned long ports) +{ + if (ports & BIT(VLV_IOSF_SB_PUNIT)) + __vlv_punit_get(i915); + + mutex_lock(&i915->sb_lock); +} + +void vlv_iosf_sb_put(struct drm_i915_private *i915, unsigned long ports) +{ + mutex_unlock(&i915->sb_lock); + + if (ports & BIT(VLV_IOSF_SB_PUNIT)) + __vlv_punit_put(i915); +} + +static int vlv_sideband_rw(struct drm_i915_private *i915, + u32 devfn, u32 port, u32 opcode, + u32 addr, u32 *val) +{ + struct intel_uncore *uncore = &i915->uncore; + const bool is_read = (opcode == SB_MRD_NP || opcode == SB_CRRDDA_NP); + int err; + + lockdep_assert_held(&i915->sb_lock); + if (port == IOSF_PORT_PUNIT) + iosf_mbi_assert_punit_acquired(); + + /* Flush the previous comms, just in case it failed last time. */ + if (intel_wait_for_register(uncore, + VLV_IOSF_DOORBELL_REQ, IOSF_SB_BUSY, 0, + 5)) { + drm_dbg(&i915->drm, "IOSF sideband idle wait (%s) timed out\n", + is_read ? "read" : "write"); + return -EAGAIN; + } + + preempt_disable(); + + intel_uncore_write_fw(uncore, VLV_IOSF_ADDR, addr); + intel_uncore_write_fw(uncore, VLV_IOSF_DATA, is_read ? 0 : *val); + intel_uncore_write_fw(uncore, VLV_IOSF_DOORBELL_REQ, + (devfn << IOSF_DEVFN_SHIFT) | + (opcode << IOSF_OPCODE_SHIFT) | + (port << IOSF_PORT_SHIFT) | + (0xf << IOSF_BYTE_ENABLES_SHIFT) | + (0 << IOSF_BAR_SHIFT) | + IOSF_SB_BUSY); + + if (__intel_wait_for_register_fw(uncore, + VLV_IOSF_DOORBELL_REQ, IOSF_SB_BUSY, 0, + 10000, 0, NULL) == 0) { + if (is_read) + *val = intel_uncore_read_fw(uncore, VLV_IOSF_DATA); + err = 0; + } else { + drm_dbg(&i915->drm, "IOSF sideband finish wait (%s) timed out\n", + is_read ? "read" : "write"); + err = -ETIMEDOUT; + } + + preempt_enable(); + + return err; +} + +u32 vlv_punit_read(struct drm_i915_private *i915, u32 addr) +{ + u32 val = 0; + + vlv_sideband_rw(i915, PCI_DEVFN(0, 0), IOSF_PORT_PUNIT, + SB_CRRDDA_NP, addr, &val); + + return val; +} + +int vlv_punit_write(struct drm_i915_private *i915, u32 addr, u32 val) +{ + return vlv_sideband_rw(i915, PCI_DEVFN(0, 0), IOSF_PORT_PUNIT, + SB_CRWRDA_NP, addr, &val); +} + +u32 vlv_bunit_read(struct drm_i915_private *i915, u32 reg) +{ + u32 val = 0; + + vlv_sideband_rw(i915, PCI_DEVFN(0, 0), IOSF_PORT_BUNIT, + SB_CRRDDA_NP, reg, &val); + + return val; +} + +void vlv_bunit_write(struct drm_i915_private *i915, u32 reg, u32 val) +{ + vlv_sideband_rw(i915, PCI_DEVFN(0, 0), IOSF_PORT_BUNIT, + SB_CRWRDA_NP, reg, &val); +} + +u32 vlv_nc_read(struct drm_i915_private *i915, u8 addr) +{ + u32 val = 0; + + vlv_sideband_rw(i915, PCI_DEVFN(0, 0), IOSF_PORT_NC, + SB_CRRDDA_NP, addr, &val); + + return val; +} + +u32 vlv_iosf_sb_read(struct drm_i915_private *i915, u8 port, u32 reg) +{ + u32 val = 0; + + vlv_sideband_rw(i915, PCI_DEVFN(0, 0), port, + SB_CRRDDA_NP, reg, &val); + + return val; +} + +void vlv_iosf_sb_write(struct drm_i915_private *i915, + u8 port, u32 reg, u32 val) +{ + vlv_sideband_rw(i915, PCI_DEVFN(0, 0), port, + SB_CRWRDA_NP, reg, &val); +} + +u32 vlv_cck_read(struct drm_i915_private *i915, u32 reg) +{ + u32 val = 0; + + vlv_sideband_rw(i915, PCI_DEVFN(0, 0), IOSF_PORT_CCK, + SB_CRRDDA_NP, reg, &val); + + return val; +} + +void vlv_cck_write(struct drm_i915_private *i915, u32 reg, u32 val) +{ + vlv_sideband_rw(i915, PCI_DEVFN(0, 0), IOSF_PORT_CCK, + SB_CRWRDA_NP, reg, &val); +} + +u32 vlv_ccu_read(struct drm_i915_private *i915, u32 reg) +{ + u32 val = 0; + + vlv_sideband_rw(i915, PCI_DEVFN(0, 0), IOSF_PORT_CCU, + SB_CRRDDA_NP, reg, &val); + + return val; +} + +void vlv_ccu_write(struct drm_i915_private *i915, u32 reg, u32 val) +{ + vlv_sideband_rw(i915, PCI_DEVFN(0, 0), IOSF_PORT_CCU, + SB_CRWRDA_NP, reg, &val); +} + +static u32 vlv_dpio_phy_iosf_port(struct drm_i915_private *i915, enum dpio_phy phy) +{ + /* + * IOSF_PORT_DPIO: VLV x2 PHY (DP/HDMI B and C), CHV x1 PHY (DP/HDMI D) + * IOSF_PORT_DPIO_2: CHV x2 PHY (DP/HDMI B and C) + */ + if (IS_CHERRYVIEW(i915)) + return phy == DPIO_PHY0 ? IOSF_PORT_DPIO_2 : IOSF_PORT_DPIO; + else + return IOSF_PORT_DPIO; +} + +u32 vlv_dpio_read(struct drm_i915_private *i915, enum pipe pipe, int reg) +{ + u32 port = vlv_dpio_phy_iosf_port(i915, DPIO_PHY(pipe)); + u32 val = 0; + + vlv_sideband_rw(i915, DPIO_DEVFN, port, SB_MRD_NP, reg, &val); + + /* + * FIXME: There might be some registers where all 1's is a valid value, + * so ideally we should check the register offset instead... + */ + drm_WARN(&i915->drm, val == 0xffffffff, + "DPIO read pipe %c reg 0x%x == 0x%x\n", + pipe_name(pipe), reg, val); + + return val; +} + +void vlv_dpio_write(struct drm_i915_private *i915, + enum pipe pipe, int reg, u32 val) +{ + u32 port = vlv_dpio_phy_iosf_port(i915, DPIO_PHY(pipe)); + + vlv_sideband_rw(i915, DPIO_DEVFN, port, SB_MWR_NP, reg, &val); +} + +u32 vlv_flisdsi_read(struct drm_i915_private *i915, u32 reg) +{ + u32 val = 0; + + vlv_sideband_rw(i915, DPIO_DEVFN, IOSF_PORT_FLISDSI, SB_CRRDDA_NP, + reg, &val); + return val; +} + +void vlv_flisdsi_write(struct drm_i915_private *i915, u32 reg, u32 val) +{ + vlv_sideband_rw(i915, DPIO_DEVFN, IOSF_PORT_FLISDSI, SB_CRWRDA_NP, + reg, &val); +} diff --git a/drivers/gpu/drm/i915/intel_sideband.h b/drivers/gpu/drm/i915/vlv_sideband.h index d1d14bcb8f56..d7732f612e7f 100644 --- a/drivers/gpu/drm/i915/intel_sideband.h +++ b/drivers/gpu/drm/i915/vlv_sideband.h @@ -1,18 +1,16 @@ /* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2013-2021 Intel Corporation + */ -#ifndef _INTEL_SIDEBAND_H_ -#define _INTEL_SIDEBAND_H_ +#ifndef _VLV_SIDEBAND_H_ +#define _VLV_SIDEBAND_H_ #include <linux/bitops.h> #include <linux/types.h> -struct drm_i915_private; enum pipe; - -enum intel_sbi_destination { - SBI_ICLK, - SBI_MPHY, -}; +struct drm_i915_private; enum { VLV_IOSF_SB_BUNIT, @@ -122,22 +120,4 @@ static inline void vlv_punit_put(struct drm_i915_private *i915) vlv_iosf_sb_put(i915, BIT(VLV_IOSF_SB_PUNIT)); } -u32 intel_sbi_read(struct drm_i915_private *i915, u16 reg, - enum intel_sbi_destination destination); -void intel_sbi_write(struct drm_i915_private *i915, u16 reg, u32 value, - enum intel_sbi_destination destination); - -int sandybridge_pcode_read(struct drm_i915_private *i915, u32 mbox, - u32 *val, u32 *val1); -int sandybridge_pcode_write_timeout(struct drm_i915_private *i915, u32 mbox, - u32 val, int fast_timeout_us, - int slow_timeout_ms); -#define sandybridge_pcode_write(i915, mbox, val) \ - sandybridge_pcode_write_timeout(i915, mbox, val, 500, 0) - -int skl_pcode_request(struct drm_i915_private *i915, u32 mbox, u32 request, - u32 reply_mask, u32 reply, int timeout_base_ms); - -int intel_pcode_init(struct drm_i915_private *i915); - -#endif /* _INTEL_SIDEBAND_H */ +#endif /* _VLV_SIDEBAND_H_ */ |